Release of version 2.2.2

In case a node can not be downloaded or its metadata is needed and can
not be retrieved, a NodeUnavailableException is raised and the according
node will be skipped.
Concerns #26

.gitignore

@@ -1,4 +1 @@
 __pycache__/
-build/
-dist/
-instaloader.egg-info/

.travis.yml

@@ -9,4 +9,12 @@ python:
 install:
  - pip install pylint requests
 script:
- - python3 -m pylint -r n -d bad-whitespace,missing-docstring,too-many-arguments,locally-disabled,line-too-long,too-many-public-methods,too-many-lines,too-many-instance-attributes instaloader
+ - python3 -m pylint -r n -d bad-whitespace,missing-docstring,too-many-arguments,locally-disabled,line-too-long,too-many-public-methods,too-many-lines,too-many-instance-attributes,too-many-locals,too-many-branches,too-many-statements instaloader
+deploy:
+  provider: pypi
+  user: aandergr
+  on:
+    tags: true
+    python: 3.6
+  password:
+    secure: KBsATtXNnEzqr0X2ALdyx2zaYj2EUfK8YjP4jEOilBELpJ4evHYXVgRkn9CKXGsD7LKbWmR3gz2rr8NZ6U42pHw0fpQHizz91MpJSKbNY7RZZh4UVWMDh81izeglchWAF+m3Wdpxpo/GBDbrZIbRQFhPgB+kjSzJo3uiCS+yVXtGG+6csJHzd0WkJjsVHz+XSdxyOiLy/qFF3nmY6ZrVrgw6WbX2YLbx9je6DxK3YRrnBqgMgH7Ui2xWWbgJCk3KKpQTMiMKnTa9mdeMJAd26YMCz1gG/X7ck6VNly/6bs5+Zq7/bU0dSolhPBhNthyccYdS0IeGPdPF3YqatblJ4fLGTfHn8oJs3/T11yIW9pvOT3UftEynPOUX+en2Qnb/3WfgwUfVts8i8DAYgEFLGEzbF5XMGy4gwAD/bZ0JW0pmgAG2PZsdOokAt+ydlNj/uxq1GFW2doZbul6k4t03p1mik6itYscvuhKoEm8Yw6wr1P9aDvMR6FFpSu4gDt2XrKmRmWkSXxpNeCfZujTjyAnISxLVQGSY7BG5K0Z2HbDtqqNz9nj4WgNGzZmngiRofCevWR+vPIDUDW7NCX03csfcFS9RRllBO/BrL7NVUV+Qp9dfTQYp+8SqYLWgHtDoXdgyMREjJaAmbx2u/TYTY8bADZ5F3vGUpv2j5hP3NTU=

README.rst

@@ -68,6 +68,13 @@ in your temporary directory, which will be reused later when ``--login`` is give
 you can download private profiles **non-interactively** when you already
 have a valid session cookie file.
 
+Besides downloading private profiles, being logged in allows to
+**download stories**:
+
+::
+
+    instaloader --login=your_username --stories profile [profile ...]
+
 You may also download
 **the most recent pictures by hashtag**:
 
@@ -102,6 +109,12 @@ or to **download all pictures from your feed**:
 
     instaloader --login=your_username :feed-all
 
+**Download all stories** from the profiles you follow:
+
+::
+
+    instaloader --login=your_username --filename-pattern={date}_{profile} :stories
+
 Advanced Options
 ----------------
 
@@ -120,6 +133,11 @@ captions and the current **profile picture**. If an already-downloaded profile
 has been renamed, Instaloader automatically **finds it by its unique ID** and
 renames the folder likewise.
 
+Instead of a *profile* or a *#hashtag*, the special targets
+``:feed-all`` (pictures from your feed),
+``:feed-liked`` (pictures from your feed which you liked), and
+``:stories`` (stories of your followees) can be specified.
+
 --profile-pic-only         Only download profile picture.
 --skip-videos              Do not download videos.
 --geotags                  **Download geotags** when available. Geotags are stored as
@@ -130,6 +148,11 @@ renames the folder likewise.
 --comments                 Download and update comments for each post. This
                            requires an additional request to the Instagram server
                            for each post, which is why it is disabled by default.
+--stories                  Also **download stories** of each profile that is
+                           downloaded. Requires ``--login``.
+--stories-only             Rather than downloading regular posts of each
+                           specified profile, only download stories.
+                           Requires ``--login``.
 
 When to Stop Downloading
 ^^^^^^^^^^^^^^^^^^^^^^^^
@@ -237,7 +260,7 @@ You could also download your last 20 liked pics with
 
 .. code:: python
 
-    oader.download_feed_pics(max_count=20, fast_update=True,
+    loader.download_feed_pics(max_count=20, fast_update=True,
                              filter_func=lambda node:
                                    not node["likes"]["viewer_has_liked"] if "likes" in node else not node["viewer_has_liked"])
 
@@ -247,6 +270,12 @@ To download the last 20 pictures with hashtag #cat, do
 
     loader.download_hashtag('cat', max_count=20)
 
+If logged in, Instaloader is also able to download user stories:
+
+.. code:: python
+
+    loader.download_stories()
+
 Each Instagram profile has its own unique ID which stays unmodified even
 if a user changes his/her username. To get said ID, given the profile's
 name, you may call
@@ -262,3 +291,9 @@ get the current username of a profile, given this unique ID
 .. code:: python
 
     loader.get_username_by_id(followees[0]['id'])
+
+Disclaimer
+----------
+
+This code is in no way affiliated with, authorized, maintained or endorsed by Instagram or any of its affiliates or
+subsidiaries. This is an independent and unofficial project. Use at your own risk.

instaloader.py

@@ -2,7 +2,6 @@
 
 """Download pictures (or videos) along with their captions and other metadata from Instagram."""
 
-import datetime
 import getpass
 import json
 import os
@@ -16,11 +15,13 @@ import tempfile
 import time
 from argparse import ArgumentParser
 from base64 import b64decode, b64encode
+from datetime import datetime
 from io import BytesIO
-from typing import Any, Callable, Dict, List, Optional
+from typing import Any, Callable, Dict, List, Optional, Tuple
 
 import requests
 import requests.utils
+import urllib3
 
 
 # To get version from setup.py for instaloader --version
@@ -75,6 +76,14 @@ class InvalidArgumentException(NonfatalException):
     pass
 
 
+class BadResponseException(NonfatalException):
+    pass
+
+
+class NodeUnavailableException(NonfatalException):
+    pass
+
+
 class BadCredentialsException(InstaloaderException):
     pass
 
@@ -83,10 +92,6 @@ class ConnectionException(InstaloaderException):
     pass
 
 
-def _epoch_to_string(epoch: float) -> str:
-    return datetime.datetime.fromtimestamp(epoch).strftime('%Y-%m-%d_%H-%M-%S')
-
-
 def get_default_session_filename(username: str) -> str:
     """Returns default session filename for given username."""
     dirname = tempfile.gettempdir() + "/" + ".instaloader-" + getpass.getuser()
@@ -99,7 +104,7 @@ def copy_session(session: requests.Session) -> requests.Session:
     new = requests.Session()
     new.cookies = \
         requests.utils.cookiejar_from_dict(requests.utils.dict_from_cookiejar(session.cookies))
-    new.headers = session.headers
+    new.headers = session.headers.copy()
     return new
 
 
@@ -154,6 +159,23 @@ class Instaloader:
         if self.sleep:
             time.sleep(random.uniform(0.25, 2.0))
 
+    def _get_and_write_raw(self, url: str, filename: str, tries: int = 3) -> None:
+        try:
+            resp = self.get_anonymous_session().get(url, stream=True)
+            if resp.status_code == 200:
+                self._log(filename, end=' ', flush=True)
+                with open(filename, 'wb') as file:
+                    resp.raw.decode_content = True
+                    shutil.copyfileobj(resp.raw, file)
+            else:
+                raise ConnectionException("Request returned HTTP error code {}.".format(resp.status_code))
+        except (urllib3.exceptions.HTTPError, requests.exceptions.RequestException, ConnectionException) as err:
+            print("URL: {}\n{}".format(url, err), file=sys.stderr)
+            if tries <= 1:
+                raise NodeUnavailableException
+            self._sleep()
+            self._get_and_write_raw(url, filename, tries - 1)
+
     def get_json(self, name: str, session: requests.Session = None,
                  max_id: Optional[str] = None) -> Optional[Dict[str, Any]]:
         """Return JSON of a profile"""
@@ -323,7 +345,7 @@ class Instaloader:
                 break
         return comments
 
-    def download_pic(self, filename: str, url: str, date_epoch: float,
+    def download_pic(self, filename: str, url: str, mtime: datetime,
                      filename_suffix: Optional[str] = None) -> bool:
         """Downloads and saves picture with given url under given directory with given timestamp.
         Returns true, if file was actually downloaded, i.e. updated."""
@@ -335,16 +357,9 @@ class Instaloader:
         if os.path.isfile(filename):
             self._log(filename + ' exists', end=' ', flush=True)
             return False
-        resp = self.get_anonymous_session().get(url, stream=True)
-        if resp.status_code == 200:
-            self._log(filename, end=' ', flush=True)
-            with open(filename, 'wb') as file:
-                resp.raw.decode_content = True
-                shutil.copyfileobj(resp.raw, file)
-            os.utime(filename, (datetime.datetime.now().timestamp(), date_epoch))
-            return True
-        else:
-            raise ConnectionException("File \'" + url + "\' could not be downloaded.")
+        self._get_and_write_raw(url, filename)
+        os.utime(filename, (datetime.now().timestamp(), mtime.timestamp()))
+        return True
 
     def update_comments(self, filename: str, shortcode: str) -> None:
         filename += '_comments.json'
@@ -369,9 +384,8 @@ class Instaloader:
                 file.write(json.dumps(unique_comments_list, indent=4))
             self._log('comments', end=' ', flush=True)
 
-    def save_caption(self, filename: str, date_epoch: float, caption: str) -> None:
+    def save_caption(self, filename: str, mtime: datetime, caption: str) -> None:
         """Updates picture caption"""
-        # pylint:disable=too-many-branches
         filename += '.txt'
         pcaption = caption.replace('\n', ' ').strip()
         caption = caption.encode("UTF-8")
@@ -410,30 +424,34 @@ class Instaloader:
             self._log('txt', end=' ', flush=True)
         with open(filename, 'wb') as text_file:
             shutil.copyfileobj(BytesIO(caption), text_file)
-        os.utime(filename, (datetime.datetime.now().timestamp(), date_epoch))
+        os.utime(filename, (datetime.now().timestamp(), mtime.timestamp()))
 
-    def save_location(self, filename: str, location_json: Dict[str, str], date_epoch: float) -> None:
+    def save_location(self, filename: str, location_json: Dict[str, str], mtime: datetime) -> None:
         filename += '_location.txt'
         location_string = (location_json["name"] + "\n" +
                            "https://maps.google.com/maps?q={0},{1}&ll={0},{1}\n".format(location_json["lat"],
                                                                                         location_json["lng"]))
         with open(filename, 'wb') as text_file:
             shutil.copyfileobj(BytesIO(location_string.encode()), text_file)
-        os.utime(filename, (datetime.datetime.now().timestamp(), date_epoch))
+        os.utime(filename, (datetime.now().timestamp(), mtime.timestamp()))
         self._log('geo', end=' ', flush=True)
 
     def download_profilepic(self, name: str, url: str) -> None:
         """Downloads and saves profile pic with given url."""
-        date_object = datetime.datetime.strptime(requests.head(url).headers["Last-Modified"],
-                                                 '%a, %d %b %Y %H:%M:%S GMT')
+
+        def _epoch_to_string(epoch: datetime) -> str:
+            return epoch.strftime('%Y-%m-%d_%H-%M-%S')
+
+        date_object = datetime.strptime(requests.head(url).headers["Last-Modified"],
+                                        '%a, %d %b %Y %H:%M:%S GMT')
         if ((format_string_contains_key(self.dirname_pattern, 'profile') or
              format_string_contains_key(self.dirname_pattern, 'target'))):
             filename = '{0}/{1}_UTC_profile_pic.{2}'.format(self.dirname_pattern.format(profile=name.lower(),
                                                                                         target=name.lower()),
-                                                            _epoch_to_string(date_object.timestamp()), url[-3:])
+                                                            _epoch_to_string(date_object), url[-3:])
         else:
             filename = '{0}/{1}_{2}_UTC_profile_pic.{3}'.format(self.dirname_pattern.format(), name.lower(),
-                                                                _epoch_to_string(date_object.timestamp()), url[-3:])
+                                                                _epoch_to_string(date_object), url[-3:])
         if os.path.isfile(filename):
             self._log(filename + ' already exists')
             return None
@@ -443,15 +461,8 @@ class Instaloader:
         index = len(match.group(0)) - 1
         offset = 8 if match.group(0)[-1:] == 's' else 0
         url = url[:index] + 's2048x2048' + ('/' if offset == 0 else str()) + url[index + offset:]
-        resp = self.get_anonymous_session().get(url, stream=True)
-        if resp.status_code == 200:
-            self._log(filename)
-            with open(filename, 'wb') as file:
-                resp.raw.decode_content = True
-                shutil.copyfileobj(resp.raw, file)
-            os.utime(filename, (datetime.datetime.now().timestamp(), date_object.timestamp()))
-        else:
-            raise ConnectionException("File \'" + url + "\' could not be downloaded.")
+        self._get_and_write_raw(url, filename)
+        os.utime(filename, (datetime.now().timestamp(), date_object.timestamp()))
 
     def save_session_to_file(self, filename: Optional[str] = None) -> None:
         """Saves requests.Session object."""
@@ -530,15 +541,28 @@ class Instaloader:
                                                       'fetch_comment_count': 4,
                                                       'fetch_like': 10})
 
-    def get_node_metadata(self, node_code: str) -> Dict[str, Any]:
+    def get_node_metadata(self, node_code: str, tries: int = 3) -> Dict[str, Any]:
         pic_json = self.get_json("p/" + node_code)
-        media = pic_json["entry_data"]["PostPage"][0]["graphql"]["shortcode_media"] \
-            if "graphql" in pic_json["entry_data"]["PostPage"][0] \
-            else pic_json["entry_data"]["PostPage"][0]["media"]
+        try:
+            media = pic_json["entry_data"]["PostPage"][0]["graphql"]["shortcode_media"] \
+                if "graphql" in pic_json["entry_data"]["PostPage"][0] \
+                else pic_json["entry_data"]["PostPage"][0]["media"]
+        except KeyError as err:
+            print(err, file=sys.stderr)
+            print(json.dumps(pic_json, indent=4), file=sys.stderr)
+            if tries <= 1:
+                raise NodeUnavailableException
+            self._sleep()
+            media = self.get_node_metadata(node_code, tries - 1)
         return media
 
     def get_location(self, node_code: str) -> Dict[str, str]:
-        media = self.get_node_metadata(node_code)
+        try:
+            media = self.get_node_metadata(node_code)
+        except NodeUnavailableException:
+            print("Unable to lookup location for node \"https://www.instagram.com/p/{}/\".".format(node_code),
+                  file=sys.stderr)
+            return dict()
         if media["location"] is not None:
             location_json = self.get_json("explore/locations/" +
                                           media["location"]["id"])
@@ -557,7 +581,6 @@ class Instaloader:
         :param download_comments: Update comments
         :return: True if something was downloaded, False otherwise, i.e. file was already there
         """
-        # pylint:disable=too-many-branches,too-many-locals,too-many-statements
         already_has_profilename = profile is not None or ('owner' in node and 'username' in node['owner'])
         needs_profilename = (format_string_contains_key(self.dirname_pattern, 'profile') or
                              format_string_contains_key(self.filename_pattern, 'profile'))
@@ -566,17 +589,22 @@ class Instaloader:
             if already_has_profilename:
                 profilename = profile if profile is not None else node['owner']['username']
             else:
-                metadata = self.get_node_metadata(shortcode)
-                profilename = metadata['owner']['username']
+                try:
+                    metadata = self.get_node_metadata(shortcode)
+                    profilename = metadata['owner']['username']
+                except NodeUnavailableException:
+                    print("Unable to gather profilename for node "
+                          "\"https://www.instagram.com/p/{}/\".".format(shortcode), file=sys.stderr)
+                    profilename = 'UNKNOWN'
         else:
             profilename = None
         profilename = profilename.lower() if profilename else None
-        date = node["date"] if "date" in node else node["taken_at_timestamp"]
+        date = datetime.fromtimestamp(node["date"] if "date" in node else node["taken_at_timestamp"])
         dirname = self.dirname_pattern.format(profile=profilename, target=target.lower())
         filename = dirname + '/' + self.filename_pattern.format(profile=profilename, target=target.lower(),
-                                                                date=datetime.datetime.fromtimestamp(date),
+                                                                date=date,
                                                                 shortcode=shortcode)
-        os.makedirs(dirname, exist_ok=True)
+        os.makedirs(os.path.dirname(filename), exist_ok=True)
         if '__typename' in node:
             if node['__typename'] == 'GraphSidecar':
                 self._sleep()
@@ -589,7 +617,7 @@ class Instaloader:
                 for edge in media['edge_sidecar_to_children']['edges']:
                     edge_downloaded = self.download_pic(filename=filename,
                                                         url=edge['node']['display_url'],
-                                                        date_epoch=date,
+                                                        mtime=date,
                                                         filename_suffix=str(edge_number))
                     downloaded = downloaded and edge_downloaded
                     edge_number += 1
@@ -597,13 +625,13 @@ class Instaloader:
                 url = node["display_url"] if "display_url" in node else node["display_src"]
                 downloaded = self.download_pic(filename=filename,
                                                url=url,
-                                               date_epoch=date)
+                                               mtime=date)
             else:
                 self._log("Warning: Unknown typename discovered:" + node['__typename'])
                 downloaded = False
         else:
             # Node is an old image or video.
-            downloaded = self.download_pic(filename=filename, url=node["display_src"], date_epoch=date)
+            downloaded = self.download_pic(filename=filename, url=node["display_src"], mtime=date)
         if "edge_media_to_caption" in node and node["edge_media_to_caption"]["edges"]:
             self.save_caption(filename, date, node["edge_media_to_caption"]["edges"][0]["node"]["text"])
         elif "caption" in node:
@@ -614,7 +642,7 @@ class Instaloader:
             video_data = self.get_json('p/' + shortcode)
             self.download_pic(filename=filename,
                               url=video_data['entry_data']['PostPage'][0]['graphql']['shortcode_media']['video_url'],
-                              date_epoch=date)
+                              mtime=date)
         if geotags:
             location = self.get_location(shortcode)
             if location:
@@ -624,6 +652,114 @@ class Instaloader:
         self._log()
         return downloaded
 
+    def download_stories(self,
+                         userids: Optional[List[int]] = None,
+                         download_videos: bool = True,
+                         fast_update: bool = False,
+                         filename_target: str = ':stories') -> None:
+        """
+        Download available stories from user followees or all stories of users whose ID are given.
+        Does not mark stories as seen.
+        To use this, one needs to be logged in
+
+        :param userids: List of user IDs to be processed in terms of downloading their stories
+        :param download_videos: True, if videos should be downloaded
+        :param fast_update: If true, abort when first already-downloaded picture is encountered
+        :param filename_target: Replacement for {target} in dirname_pattern and filename_pattern
+        """
+
+        if self.username is None:
+            raise LoginRequiredException('Login required to download stories')
+
+        tempsession = copy_session(self.session)
+        header = tempsession.headers
+        header['User-Agent'] = 'Instagram 10.3.2 (iPhone7,2; iPhone OS 9_3_3; en_US; en-US; scale=2.00; 750x1334) ' \
+                               'AppleWebKit/420+'
+        del header['Host']
+        del header['Origin']
+        del header['X-Instagram-AJAX']
+        del header['X-Requested-With']
+
+        def _user_stories():
+            def _get(url):
+                self._sleep()
+                resp = tempsession.get(url)
+                if resp.status_code != 200:
+                    raise ConnectionException('Failed to fetch stories.')
+                return json.loads(resp.text)
+            url_reel_media = 'https://i.instagram.com/api/v1/feed/user/{0}/reel_media/'
+            url_reels_tray = 'https://i.instagram.com/api/v1/feed/reels_tray/'
+            if userids is not None:
+                for userid in userids:
+                    yield _get(url_reel_media.format(userid))
+            else:
+                data = _get(url_reels_tray)
+                if not 'tray' in data:
+                    raise BadResponseException('Bad story reel JSON.')
+                for user in data["tray"]:
+                    yield user if "items" in user else _get(url_reel_media.format(user['user']['pk']))
+
+        for user_stories in _user_stories():
+            if "items" not in user_stories:
+                continue
+            name = user_stories["user"]["username"].lower()
+            self._log("Retrieving stories from profile {}.".format(name))
+            totalcount = len(user_stories["items"]) if "items" in user_stories else 0
+            count = 1
+            for item in user_stories["items"]:
+                self._log("[%3i/%3i] " % (count, totalcount), end="", flush=True)
+                count += 1
+
+                self._sleep()
+                shortcode = item["code"] if "code" in item else "no_code"
+
+                date_float = item["device_timestamp"] if "device_timestamp" in item else item["taken_at"]
+                try:
+                    date = datetime.fromtimestamp(date_float)
+                except ValueError:
+                    # device_timestamp seems to sometime be in milliseconds
+                    date_float /= 1000
+                    date = datetime.fromtimestamp(date_float)
+
+                dirname = self.dirname_pattern.format(profile=name, target=filename_target)
+                filename = dirname + '/' + self.filename_pattern.format(profile=name, target=filename_target,
+                                                                        date=date,
+                                                                        shortcode=shortcode)
+                os.makedirs(os.path.dirname(filename), exist_ok=True)
+                try:
+                    if "image_versions2" in item:
+                        url = item["image_versions2"]["candidates"][0]["url"]
+                        downloaded = self.download_pic(filename=filename,
+                                                       url=url,
+                                                       mtime=date)
+                    else:
+                        self._log("Warning: Unable to find story image.")
+                        downloaded = False
+                    if "caption" in item and item["caption"] is not None:
+                        caption = item["caption"]
+                        if isinstance(caption, dict) and "text" in caption:
+                            caption = caption["text"]
+                        self.save_caption(filename, date, caption)
+                    else:
+                        self._log("<no caption>", end=' ', flush=True)
+                    if "video_versions" in item and download_videos:
+                        downloaded = self.download_pic(filename=filename,
+                                                       url=item["video_versions"][0]["url"],
+                                                       mtime=date)
+                        if "video_duration" in item and self.sleep and downloaded:
+                            time.sleep(item["video_duration"])
+                except NodeUnavailableException:
+                    print("Unable to download node \"https://www.instagram.com/p/{}/\" of user {} from stories."
+                          .format(shortcode, name), file=sys.stderr)
+                    continue
+                if item["story_locations"]:
+                    location = item["story_locations"][0]["location"]
+                    if location:
+                        self.save_location(filename, location, date)
+                self._log()
+                if fast_update and not downloaded:
+                    break
+
     def download_feed_pics(self, max_count: int = None, fast_update: bool = False,
                            filter_func: Optional[Callable[[Dict[str, Dict[str, Any]]], bool]] = None,
                            download_videos: bool = True, geotags: bool = False,
@@ -647,7 +783,6 @@ class Instaloader:
         :param geotags: Download geotags
         :param download_comments: Update comments
         """
-        # pylint:disable=too-many-locals
         data = self.get_feed_json()
         count = 1
         while True:
@@ -670,9 +805,14 @@ class Instaloader:
                     continue
                 self._log("[%3i] %s " % (count, name), end="", flush=True)
                 count += 1
-                downloaded = self.download_node(node, profile=name, target=':feed',
-                                                download_videos=download_videos, geotags=geotags,
-                                                download_comments=download_comments)
+                try:
+                    downloaded = self.download_node(node, profile=name, target=':feed',
+                                                    download_videos=download_videos, geotags=geotags,
+                                                    download_comments=download_comments)
+                except NodeUnavailableException:
+                    print("Unable to download node \"https://www.instagram.com/p/{}/\" of user {} from feed."
+                          .format(node['shortcode'], name), file=sys.stderr)
+                    continue
                 if fast_update and not downloaded:
                     return
             if not feed["page_info"]["has_next_page"]:
@@ -714,9 +854,14 @@ class Instaloader:
                     self._log('<skipped>')
                     continue
                 count += 1
-                downloaded = self.download_node(node=node, profile=None, target='#'+hashtag,
-                                                download_videos=download_videos, geotags=geotags,
-                                                download_comments=download_comments)
+                try:
+                    downloaded = self.download_node(node=node, profile=None, target='#'+hashtag,
+                                                    download_videos=download_videos, geotags=geotags,
+                                                    download_comments=download_comments)
+                except NodeUnavailableException:
+                    print("Unable to download node \"https://www.instagram.com/p/{}/\" "
+                          "while downloading hashtag \"{}\".".format(node['shortcode'], hashtag), file=sys.stderr)
+                    continue
                 if fast_update and not downloaded:
                     return
             if data['entry_data']['TagPage'][0]['tag']['media']['page_info']['has_next_page']:
@@ -726,10 +871,12 @@ class Instaloader:
             else:
                 break
 
-    def check_id(self, profile: str, json_data: Dict[str, Any]) -> str:
+    def check_id(self, profile: str, json_data: Dict[str, Any]) -> Tuple[str, int]:
         """
         Consult locally stored ID of profile with given name, check whether ID matches and whether name
         has changed and return current name of the profile, and store ID of profile.
+
+        :return: current profile name, profile id
         """
         profile_exists = "ProfilePage" in json_data["entry_data"]
         if ((format_string_contains_key(self.dirname_pattern, 'profile') or
@@ -758,8 +905,8 @@ class Instaloader:
                 else:
                     os.rename('{0}/{1}_id'.format(self.dirname_pattern.format(), profile.lower()),
                               '{0}/{1}_id'.format(self.dirname_pattern.format(), newname.lower()))
-                return newname
-            return profile
+                return newname, profile_id
+            return profile, profile_id
         except FileNotFoundError:
             pass
         if profile_exists:
@@ -769,41 +916,51 @@ class Instaloader:
                 profile_id = json_data['entry_data']['ProfilePage'][0]['user']['id']
                 text_file.write(profile_id + "\n")
                 self._log("Stored ID {0} for profile {1}.".format(profile_id, profile))
-            return profile
+            return profile, profile_id
         raise ProfileNotExistsException("Profile {0} does not exist.".format(profile))
 
     def download(self, name: str,
                  profile_pic_only: bool = False, download_videos: bool = True, geotags: bool = False,
-                 download_comments: bool = False, fast_update: bool = False) -> None:
+                 download_comments: bool = False, fast_update: bool = False,
+                 download_stories: bool = False, download_stories_only: bool = False) -> None:
         """Download one profile"""
-        # pylint:disable=too-many-branches,too-many-locals
         # Get profile main page json
         data = self.get_json(name)
         # check if profile does exist or name has changed since last download
         # and update name and json data if necessary
-        name_updated = self.check_id(name, data)
+        name_updated, profile_id = self.check_id(name, data)
         if name_updated != name:
             name = name_updated
             data = self.get_json(name)
         # Download profile picture
-        self.download_profilepic(name, data["entry_data"]["ProfilePage"][0]["user"]["profile_pic_url"])
+        try:
+            self.download_profilepic(name, data["entry_data"]["ProfilePage"][0]["user"]["profile_pic_url"])
+        except NodeUnavailableException:
+            print("Unable to download profilepic of user {}.".format(name), file=sys.stderr)
         if profile_pic_only:
             return
         # Catch some errors
         if data["entry_data"]["ProfilePage"][0]["user"]["is_private"]:
             if data["config"]["viewer"] is None:
                 raise LoginRequiredException("profile %s requires login" % name)
-            if not data["entry_data"]["ProfilePage"][0]["user"]["followed_by_viewer"]:
+            if not data["entry_data"]["ProfilePage"][0]["user"]["followed_by_viewer"] and \
+                    self.username != data["entry_data"]["ProfilePage"][0]["user"]["username"]:
                 raise PrivateProfileNotFollowedException("Profile %s: private but not followed." % name)
         else:
-            if data["config"]["viewer"] is not None:
+            if data["config"]["viewer"] is not None and not (download_stories or download_stories_only):
                 self._log("profile %s could also be downloaded anonymously." % name)
+        if download_stories or download_stories_only:
+            self.download_stories(userids=[profile_id], filename_target=name,
+                                  download_videos=download_videos, fast_update=fast_update)
+        if download_stories_only:
+            return
         if ("nodes" not in data["entry_data"]["ProfilePage"][0]["user"]["media"] or
                 not data["entry_data"]["ProfilePage"][0]["user"]["media"]["nodes"]) \
                 and not profile_pic_only:
             raise ProfileHasNoPicsException("Profile %s: no pics found." % name)
 
         # Iterate over pictures and download them
+        self._log("Retrieving posts from profile {}.".format(name))
         def get_last_id(data):
             if data["entry_data"] and data["entry_data"]["ProfilePage"][0]["user"]["media"]["nodes"]:
                 return data["entry_data"]["ProfilePage"][0]["user"]["media"]["nodes"][-1]["id"]
@@ -814,9 +971,14 @@ class Instaloader:
             for node in data["entry_data"]["ProfilePage"][0]["user"]["media"]["nodes"]:
                 self._log("[%3i/%3i] " % (count, totalcount), end="", flush=True)
                 count += 1
-                downloaded = self.download_node(node=node, profile=name, target=name,
-                                                download_videos=download_videos, geotags=geotags,
-                                                download_comments=download_comments)
+                try:
+                    downloaded = self.download_node(node=node, profile=name, target=name,
+                                                    download_videos=download_videos, geotags=geotags,
+                                                    download_comments=download_comments)
+                except NodeUnavailableException:
+                    print("Unable to download node \"https://www.instagram.com/p/{}/\" of user {}."
+                          .format(node['shortcode'], name), file=sys.stderr)
+                    continue
                 if fast_update and not downloaded:
                     return
             data = self.get_json(name, max_id=get_last_id(data))
@@ -840,9 +1002,9 @@ class Instaloader:
                           sessionfile: Optional[str] = None, max_count: Optional[int] = None,
                           profile_pic_only: bool = False, download_videos: bool = True, geotags: bool = False,
                           download_comments: bool = False,
-                          fast_update: bool = False) -> None:
+                          fast_update: bool = False,
+                          stories: bool = False, stories_only: bool = False) -> None:
         """Download set of profiles and handle sessions"""
-        # pylint:disable=too-many-branches,too-many-locals,too-many-statements
         # Login, if desired
         if username is not None:
             try:
@@ -895,6 +1057,11 @@ class Instaloader:
                                                 download_comments=download_comments)
                     else:
                         print("--login=USERNAME required to download {}.".format(pentry), file=sys.stderr)
+                elif pentry == ":stories":
+                    if username is not None:
+                        self.download_stories(download_videos=download_videos, fast_update=fast_update)
+                    else:
+                        print("--login=USERNAME required to download {}.".format(pentry), file=sys.stderr)
                 else:
                     targets.add(pentry)
             if len(targets) > 1:
@@ -904,7 +1071,7 @@ class Instaloader:
                 try:
                     try:
                         self.download(target, profile_pic_only, download_videos,
-                                      geotags, download_comments, fast_update)
+                                      geotags, download_comments, fast_update, stories, stories_only)
                     except ProfileNotExistsException as err:
                         if username is not None:
                             self._log(err)
@@ -942,9 +1109,9 @@ def main():
     g_what.add_argument('profile', nargs='*', metavar='profile|#hashtag',
                         help='Name of profile or #hashtag to download. '
                              'Alternatively, if --login is given: @<profile> to download all followees of '
-                             '<profile>; or the special targets :feed-all or :feed-liked to '
-                             'download pictures from your feed (using '
-                             '--fast-update is recommended).')
+                             '<profile>; the special targets :feed-all or :feed-liked to '
+                             'download pictures from your feed; or :stories to download the stories of your '
+                             'followees.')
     g_what.add_argument('-P', '--profile-pic-only', action='store_true',
                         help='Only download profile picture.')
     g_what.add_argument('-V', '--skip-videos', action='store_true',
@@ -958,6 +1125,11 @@ def main():
                         help='Download and update comments for each post. '
                              'This requires an additional request to the Instagram '
                              'server for each post, which is why it is disabled by default.')
+    g_what.add_argument('-s', '--stories', action='store_true',
+                        help='Also download stories of each profile that is downloaded. Requires --login.')
+    g_what.add_argument('--stories-only', action='store_true',
+                        help='Rather than downloading regular posts of each specified profile, only download '
+                             'stories. Requires --login.')
 
     g_stop = parser.add_argument_group('When to Stop Downloading',
                                        'If none of these options are given, Instaloader goes through all pictures '
@@ -1014,13 +1186,19 @@ def main():
 
     args = parser.parse_args()
     try:
+        if args.login is None and (args.stories or args.stories_only):
+            print("--login=USERNAME required to download stories.", file=sys.stderr)
+            args.stories = False
+            if args.stories_only:
+                raise SystemExit(1)
         loader = Instaloader(sleep=not args.no_sleep, quiet=args.quiet, shorter_output=args.shorter_output,
                              user_agent=args.user_agent,
                              dirname_pattern=args.dirname_pattern, filename_pattern=args.filename_pattern)
-        loader.download_profiles(args.profile, args.login, args.password, args.sessionfile,
+        loader.download_profiles(args.profile, args.login.lower() if args.login is not None else None, args.password,
+                                 args.sessionfile,
                                  int(args.count) if args.count is not None else None,
                                  args.profile_pic_only, not args.skip_videos, args.geotags, args.comments,
-                                 args.fast_update)
+                                 args.fast_update, args.stories, args.stories_only)
     except InstaloaderException as err:
         raise SystemExit("Fatal error: %s" % err)
 

setup.py

@@ -8,7 +8,7 @@ if sys.version_info < (3, 5):
 
 setup(
     name='instaloader',
-    version='2.1',
+    version='2.2.2',
     py_modules=['instaloader'],
     url='https://github.com/Thammus/instaloader',
     license='MIT',