Release of version 2.2.3

Fixes #32.

.gitignore

@@ -1,4 +1 @@
 __pycache__/
-build/
-dist/
-instaloader.egg-info/

.travis.yml

@@ -10,3 +10,11 @@ install:
  - pip install pylint requests
 script:
  - python3 -m pylint -r n -d bad-whitespace,missing-docstring,too-many-arguments,locally-disabled,line-too-long,too-many-public-methods,too-many-lines,too-many-instance-attributes,too-many-locals,too-many-branches,too-many-statements instaloader
+deploy:
+  provider: pypi
+  user: aandergr
+  on:
+    tags: true
+    python: 3.6
+  password:
+    secure: KBsATtXNnEzqr0X2ALdyx2zaYj2EUfK8YjP4jEOilBELpJ4evHYXVgRkn9CKXGsD7LKbWmR3gz2rr8NZ6U42pHw0fpQHizz91MpJSKbNY7RZZh4UVWMDh81izeglchWAF+m3Wdpxpo/GBDbrZIbRQFhPgB+kjSzJo3uiCS+yVXtGG+6csJHzd0WkJjsVHz+XSdxyOiLy/qFF3nmY6ZrVrgw6WbX2YLbx9je6DxK3YRrnBqgMgH7Ui2xWWbgJCk3KKpQTMiMKnTa9mdeMJAd26YMCz1gG/X7ck6VNly/6bs5+Zq7/bU0dSolhPBhNthyccYdS0IeGPdPF3YqatblJ4fLGTfHn8oJs3/T11yIW9pvOT3UftEynPOUX+en2Qnb/3WfgwUfVts8i8DAYgEFLGEzbF5XMGy4gwAD/bZ0JW0pmgAG2PZsdOokAt+ydlNj/uxq1GFW2doZbul6k4t03p1mik6itYscvuhKoEm8Yw6wr1P9aDvMR6FFpSu4gDt2XrKmRmWkSXxpNeCfZujTjyAnISxLVQGSY7BG5K0Z2HbDtqqNz9nj4WgNGzZmngiRofCevWR+vPIDUDW7NCX03csfcFS9RRllBO/BrL7NVUV+Qp9dfTQYp+8SqYLWgHtDoXdgyMREjJaAmbx2u/TYTY8bADZ5F3vGUpv2j5hP3NTU=

instaloader.py

@@ -80,6 +80,10 @@ class BadResponseException(NonfatalException):
     pass
 
 
+class NodeUnavailableException(NonfatalException):
+    pass
+
+
 class BadCredentialsException(InstaloaderException):
     pass
 
@@ -145,6 +149,7 @@ class Instaloader:
         self.dirname_pattern = dirname_pattern if dirname_pattern is not None else '{target}'
         self.filename_pattern = filename_pattern.replace('{date}', '{date:%Y-%m-%d_%H-%M-%S}') \
             if filename_pattern is not None else '{date:%Y-%m-%d_%H-%M-%S}'
+        self.previous_queries = dict()
 
     def _log(self, *msg, sep='', end='\n', flush=False):
         if not self.quiet:
@@ -166,9 +171,9 @@ class Instaloader:
             else:
                 raise ConnectionException("Request returned HTTP error code {}.".format(resp.status_code))
         except (urllib3.exceptions.HTTPError, requests.exceptions.RequestException, ConnectionException) as err:
-            print("URL: " + url + "\n" + err, file=sys.stderr)
+            print("URL: {}\n{}".format(url, err), file=sys.stderr)
             if tries <= 1:
-                raise err
+                raise NodeUnavailableException
             self._sleep()
             self._get_and_write_raw(url, filename, tries - 1)
 
@@ -216,13 +221,14 @@ class Instaloader:
         return session
 
     def graphql_query(self, query_id: int, variables: Dict[str, Any],
-                      referer: Optional[str] = None) -> Dict[str, Any]:
+                      referer: Optional[str] = None, tries: int = 3) -> Dict[str, Any]:
         """
         Do a GraphQL Query.
 
         :param query_id: Query ID.
         :param variables: Variables for the Query.
         :param referer: HTTP Referer, or None.
+        :param tries: Number of total tries. Retries if a RequestException occurs.
         :return: The server's response dictionary.
         """
         tmpsession = copy_session(self.session)
@@ -234,13 +240,57 @@ class Instaloader:
         tmpsession.headers['accept'] = '*/*'
         if referer is not None:
             tmpsession.headers['referer'] = referer
+        sliding_window = 660
+        def graphql_query_waittime(query_id: int) -> int:
+            timestamp_list = self.previous_queries.get(query_id)
+            if not timestamp_list:
+                return 0
+            current_time = datetime.now().timestamp()
+            timestamp_list = list(filter(lambda t: t > current_time - sliding_window, timestamp_list))
+            self.previous_queries[query_id] = timestamp_list
+            if len(timestamp_list) < 100:
+                return 0
+            return round(min(timestamp_list) + sliding_window - current_time) + 6
+        waittime = graphql_query_waittime(query_id)
+        if waittime > 0:
+            self._log('\nToo many queries in the last time. Need to wait {} seconds.'.format(waittime), flush=True)
+            time.sleep(waittime)
         self._sleep()
-        response = tmpsession.get('https://www.instagram.com/graphql/query',
-                                  params={'query_id': query_id,
-                                          'variables': json.dumps(variables, separators=(',', ':'))})
-        if response.status_code != 200:
-            raise ConnectionException("GraphQL query returned HTTP error code {}.".format(response.status_code))
-        return response.json()
+        try:
+            response = tmpsession.get('https://www.instagram.com/graphql/query',
+                                      params={'query_id': query_id,
+                                              'variables': json.dumps(variables, separators=(',', ':'))})
+        except requests.exceptions.RequestException as err:
+            print(err, file=sys.stderr)
+            if tries <= 1:
+                raise ConnectionException('GraphQL query {} failed multiple times with following variables:\n{}'
+                                          .format(query_id, json.dumps(variables, indent=4)))
+            self._sleep()
+            return self.graphql_query(query_id, variables, referer, tries - 1)
+        else:
+            if response.status_code != 200:
+                if response.status_code == 429:
+                    print('GraphQL query returned HTTP error code 429 because too many queries occured '
+                          'in the last time.', file=sys.stderr)
+                    waittime = round(min(self.previous_queries.get(query_id))
+                                     + sliding_window - datetime.now().timestamp()) + 6 \
+                        if self.previous_queries.get(query_id) else sliding_window
+                    if waittime > 0:
+                        sys.stderr.flush()
+                        self._log('Please do not use Instagram in your browser or run multiple instances '
+                                  'of Instaloader in parallel while using options that require GraphQL '
+                                  'queries.\nNeed to wait {} seconds.'.format(waittime), flush=True)
+                        time.sleep(waittime)
+                    return self.graphql_query(query_id, variables, referer, tries)
+                else:
+                    raise ConnectionException("GraphQL query returned HTTP error code {}.".format(response.status_code))
+            return response.json()
+        finally:
+            timestamp_list = self.previous_queries.get(query_id)
+            if timestamp_list:
+                self.previous_queries.get(query_id).append(datetime.now().timestamp())
+            else:
+                self.previous_queries[query_id] = [datetime.now().timestamp()]
 
     def get_username_by_id(self, profile_id: int) -> str:
         """To get the current username of a profile, given its unique ID, this function can be used."""
@@ -322,14 +372,25 @@ class Instaloader:
                 break
         return followees
 
-    def get_comments(self, shortcode: str) -> List[Dict[str, Any]]:
+    def get_comments(self, shortcode: str, tries: int = 3) -> List[Dict[str, Any]]:
         """Retrieve comments of node with given shortcode."""
+        data = self.get_node_metadata(shortcode)
+        if data['edge_media_to_comment']['count'] == len(data['edge_media_to_comment']['edges']):
+            return [comment['node'] for comment in data['edge_media_to_comment']['edges']]
         data = self.graphql_query(17852405266163336, {'shortcode': shortcode,
                                                       'first': 500},
                                   referer='https://www.instagram.com/p/' + shortcode + '/')
         comments = []
         while True:
-            edge_media_to_comment = data['data']['shortcode_media']['edge_media_to_comment']
+            try:
+                edge_media_to_comment = data['data']['shortcode_media']['edge_media_to_comment']
+            except KeyError as err:
+                print('Missing key {} in response of GraphQL query for retrieving comments for node \'{}\'.'
+                      .format(err, shortcode), file=sys.stderr)
+                if tries <= 1:
+                    raise BadResponseException('GraphQL query returned strange JSON for shortcode {}:\n{}'.format(shortcode, data))
+                self._sleep()
+                return self.get_comments(shortcode, tries - 1)
             comments.extend([comment['node'] for comment in edge_media_to_comment['edges']])
             page_info = edge_media_to_comment['page_info']
             if page_info['has_next_page']:
@@ -543,17 +604,22 @@ class Instaloader:
             media = pic_json["entry_data"]["PostPage"][0]["graphql"]["shortcode_media"] \
                 if "graphql" in pic_json["entry_data"]["PostPage"][0] \
                 else pic_json["entry_data"]["PostPage"][0]["media"]
-        except KeyError as err:
+        except (KeyError, TypeError) as err:
             print(err, file=sys.stderr)
             print(json.dumps(pic_json, indent=4), file=sys.stderr)
             if tries <= 1:
-                raise err
+                raise NodeUnavailableException
             self._sleep()
             media = self.get_node_metadata(node_code, tries - 1)
         return media
 
     def get_location(self, node_code: str) -> Dict[str, str]:
-        media = self.get_node_metadata(node_code)
+        try:
+            media = self.get_node_metadata(node_code)
+        except NodeUnavailableException:
+            print("Unable to lookup location for node \"https://www.instagram.com/p/{}/\".".format(node_code),
+                  file=sys.stderr)
+            return dict()
         if media["location"] is not None:
             location_json = self.get_json("explore/locations/" +
                                           media["location"]["id"])
@@ -580,8 +646,13 @@ class Instaloader:
             if already_has_profilename:
                 profilename = profile if profile is not None else node['owner']['username']
             else:
-                metadata = self.get_node_metadata(shortcode)
-                profilename = metadata['owner']['username']
+                try:
+                    metadata = self.get_node_metadata(shortcode)
+                    profilename = metadata['owner']['username']
+                except NodeUnavailableException:
+                    print("Unable to gather profilename for node "
+                          "\"https://www.instagram.com/p/{}/\".".format(shortcode), file=sys.stderr)
+                    profilename = 'UNKNOWN'
         else:
             profilename = None
         profilename = profilename.lower() if profilename else None
@@ -700,9 +771,9 @@ class Instaloader:
                 shortcode = item["code"] if "code" in item else "no_code"
 
                 date_float = item["device_timestamp"] if "device_timestamp" in item else item["taken_at"]
-                try:
+                if date_float < 10000000000:
                     date = datetime.fromtimestamp(date_float)
-                except ValueError:
+                else:
                     # device_timestamp seems to sometime be in milliseconds
                     date_float /= 1000
                     date = datetime.fromtimestamp(date_float)
@@ -712,27 +783,32 @@ class Instaloader:
                                                                         date=date,
                                                                         shortcode=shortcode)
                 os.makedirs(os.path.dirname(filename), exist_ok=True)
-                if "image_versions2" in item:
-                    url = item["image_versions2"]["candidates"][0]["url"]
-                    downloaded = self.download_pic(filename=filename,
-                                                   url=url,
-                                                   mtime=date)
-                else:
-                    self._log("Warning: Unable to find story image.")
-                    downloaded = False
-                if "caption" in item and item["caption"] is not None:
-                    caption = item["caption"]
-                    if isinstance(caption, dict) and "text" in caption:
-                        caption = caption["text"]
-                    self.save_caption(filename, date, caption)
-                else:
-                    self._log("<no caption>", end=' ', flush=True)
-                if "video_versions" in item and download_videos:
-                    downloaded = self.download_pic(filename=filename,
-                                                   url=item["video_versions"][0]["url"],
-                                                   mtime=date)
-                    if "video_duration" in item and self.sleep and downloaded:
-                        time.sleep(item["video_duration"])
+                try:
+                    if "image_versions2" in item:
+                        url = item["image_versions2"]["candidates"][0]["url"]
+                        downloaded = self.download_pic(filename=filename,
+                                                       url=url,
+                                                       mtime=date)
+                    else:
+                        self._log("Warning: Unable to find story image.")
+                        downloaded = False
+                    if "caption" in item and item["caption"] is not None:
+                        caption = item["caption"]
+                        if isinstance(caption, dict) and "text" in caption:
+                            caption = caption["text"]
+                        self.save_caption(filename, date, caption)
+                    else:
+                        self._log("<no caption>", end=' ', flush=True)
+                    if "video_versions" in item and download_videos:
+                        downloaded = self.download_pic(filename=filename,
+                                                       url=item["video_versions"][0]["url"],
+                                                       mtime=date)
+                        if "video_duration" in item and self.sleep and downloaded:
+                            time.sleep(item["video_duration"])
+                except NodeUnavailableException:
+                    print("Unable to download node \"https://www.instagram.com/p/{}/\" of user {} from stories."
+                          .format(shortcode, name), file=sys.stderr)
+                    continue
                 if item["story_locations"]:
                     location = item["story_locations"][0]["location"]
                     if location:
@@ -786,9 +862,14 @@ class Instaloader:
                     continue
                 self._log("[%3i] %s " % (count, name), end="", flush=True)
                 count += 1
-                downloaded = self.download_node(node, profile=name, target=':feed',
-                                                download_videos=download_videos, geotags=geotags,
-                                                download_comments=download_comments)
+                try:
+                    downloaded = self.download_node(node, profile=name, target=':feed',
+                                                    download_videos=download_videos, geotags=geotags,
+                                                    download_comments=download_comments)
+                except NodeUnavailableException:
+                    print("Unable to download node \"https://www.instagram.com/p/{}/\" of user {} from feed."
+                          .format(node['shortcode'] if 'shortcode' in node else node['code'], name), file=sys.stderr)
+                    continue
                 if fast_update and not downloaded:
                     return
             if not feed["page_info"]["has_next_page"]:
@@ -830,9 +911,15 @@ class Instaloader:
                     self._log('<skipped>')
                     continue
                 count += 1
-                downloaded = self.download_node(node=node, profile=None, target='#'+hashtag,
-                                                download_videos=download_videos, geotags=geotags,
-                                                download_comments=download_comments)
+                try:
+                    downloaded = self.download_node(node=node, profile=None, target='#'+hashtag,
+                                                    download_videos=download_videos, geotags=geotags,
+                                                    download_comments=download_comments)
+                except NodeUnavailableException:
+                    print("Unable to download node \"https://www.instagram.com/p/{}/\" "
+                          "while downloading hashtag \"{}\"."
+                          .format(node['shortcode'] if 'shortcode' in node else node['code'], hashtag), file=sys.stderr)
+                    continue
                 if fast_update and not downloaded:
                     return
             if data['entry_data']['TagPage'][0]['tag']['media']['page_info']['has_next_page']:
@@ -904,7 +991,10 @@ class Instaloader:
             name = name_updated
             data = self.get_json(name)
         # Download profile picture
-        self.download_profilepic(name, data["entry_data"]["ProfilePage"][0]["user"]["profile_pic_url"])
+        try:
+            self.download_profilepic(name, data["entry_data"]["ProfilePage"][0]["user"]["profile_pic_url"])
+        except NodeUnavailableException:
+            print("Unable to download profilepic of user {}.".format(name), file=sys.stderr)
         if profile_pic_only:
             return
         # Catch some errors
@@ -939,9 +1029,14 @@ class Instaloader:
             for node in data["entry_data"]["ProfilePage"][0]["user"]["media"]["nodes"]:
                 self._log("[%3i/%3i] " % (count, totalcount), end="", flush=True)
                 count += 1
-                downloaded = self.download_node(node=node, profile=name, target=name,
-                                                download_videos=download_videos, geotags=geotags,
-                                                download_comments=download_comments)
+                try:
+                    downloaded = self.download_node(node=node, profile=name, target=name,
+                                                    download_videos=download_videos, geotags=geotags,
+                                                    download_comments=download_comments)
+                except NodeUnavailableException:
+                    print("Unable to download node \"https://www.instagram.com/p/{}/\" of user {}."
+                          .format(node['shortcode'] if 'shortcode' in node else node['code'], name), file=sys.stderr)
+                    continue
                 if fast_update and not downloaded:
                     return
             data = self.get_json(name, max_id=get_last_id(data))
@@ -1157,7 +1252,8 @@ def main():
         loader = Instaloader(sleep=not args.no_sleep, quiet=args.quiet, shorter_output=args.shorter_output,
                              user_agent=args.user_agent,
                              dirname_pattern=args.dirname_pattern, filename_pattern=args.filename_pattern)
-        loader.download_profiles(args.profile, args.login.lower(), args.password, args.sessionfile,
+        loader.download_profiles(args.profile, args.login.lower() if args.login is not None else None, args.password,
+                                 args.sessionfile,
                                  int(args.count) if args.count is not None else None,
                                  args.profile_pic_only, not args.skip_videos, args.geotags, args.comments,
                                  args.fast_update, args.stories, args.stories_only)

setup.py

@@ -8,7 +8,7 @@ if sys.version_info < (3, 5):
 
 setup(
     name='instaloader',
-    version='2.2',
+    version='2.2.3',
     py_modules=['instaloader'],
     url='https://github.com/Thammus/instaloader',
     license='MIT',