Release of version 2.2.3

Fixes #32.

instaloader.py

@@ -149,6 +149,7 @@ class Instaloader:
         self.dirname_pattern = dirname_pattern if dirname_pattern is not None else '{target}'
         self.filename_pattern = filename_pattern.replace('{date}', '{date:%Y-%m-%d_%H-%M-%S}') \
             if filename_pattern is not None else '{date:%Y-%m-%d_%H-%M-%S}'
+        self.previous_queries = dict()
 
     def _log(self, *msg, sep='', end='\n', flush=False):
         if not self.quiet:
@@ -220,13 +221,14 @@ class Instaloader:
         return session
 
     def graphql_query(self, query_id: int, variables: Dict[str, Any],
-                      referer: Optional[str] = None) -> Dict[str, Any]:
+                      referer: Optional[str] = None, tries: int = 3) -> Dict[str, Any]:
         """
         Do a GraphQL Query.
 
         :param query_id: Query ID.
         :param variables: Variables for the Query.
         :param referer: HTTP Referer, or None.
+        :param tries: Number of total tries. Retries if a RequestException occurs.
         :return: The server's response dictionary.
         """
         tmpsession = copy_session(self.session)
@@ -238,13 +240,57 @@ class Instaloader:
         tmpsession.headers['accept'] = '*/*'
         if referer is not None:
             tmpsession.headers['referer'] = referer
+        sliding_window = 660
+        def graphql_query_waittime(query_id: int) -> int:
+            timestamp_list = self.previous_queries.get(query_id)
+            if not timestamp_list:
+                return 0
+            current_time = datetime.now().timestamp()
+            timestamp_list = list(filter(lambda t: t > current_time - sliding_window, timestamp_list))
+            self.previous_queries[query_id] = timestamp_list
+            if len(timestamp_list) < 100:
+                return 0
+            return round(min(timestamp_list) + sliding_window - current_time) + 6
+        waittime = graphql_query_waittime(query_id)
+        if waittime > 0:
+            self._log('\nToo many queries in the last time. Need to wait {} seconds.'.format(waittime), flush=True)
+            time.sleep(waittime)
         self._sleep()
-        response = tmpsession.get('https://www.instagram.com/graphql/query',
-                                  params={'query_id': query_id,
-                                          'variables': json.dumps(variables, separators=(',', ':'))})
-        if response.status_code != 200:
-            raise ConnectionException("GraphQL query returned HTTP error code {}.".format(response.status_code))
-        return response.json()
+        try:
+            response = tmpsession.get('https://www.instagram.com/graphql/query',
+                                      params={'query_id': query_id,
+                                              'variables': json.dumps(variables, separators=(',', ':'))})
+        except requests.exceptions.RequestException as err:
+            print(err, file=sys.stderr)
+            if tries <= 1:
+                raise ConnectionException('GraphQL query {} failed multiple times with following variables:\n{}'
+                                          .format(query_id, json.dumps(variables, indent=4)))
+            self._sleep()
+            return self.graphql_query(query_id, variables, referer, tries - 1)
+        else:
+            if response.status_code != 200:
+                if response.status_code == 429:
+                    print('GraphQL query returned HTTP error code 429 because too many queries occured '
+                          'in the last time.', file=sys.stderr)
+                    waittime = round(min(self.previous_queries.get(query_id))
+                                     + sliding_window - datetime.now().timestamp()) + 6 \
+                        if self.previous_queries.get(query_id) else sliding_window
+                    if waittime > 0:
+                        sys.stderr.flush()
+                        self._log('Please do not use Instagram in your browser or run multiple instances '
+                                  'of Instaloader in parallel while using options that require GraphQL '
+                                  'queries.\nNeed to wait {} seconds.'.format(waittime), flush=True)
+                        time.sleep(waittime)
+                    return self.graphql_query(query_id, variables, referer, tries)
+                else:
+                    raise ConnectionException("GraphQL query returned HTTP error code {}.".format(response.status_code))
+            return response.json()
+        finally:
+            timestamp_list = self.previous_queries.get(query_id)
+            if timestamp_list:
+                self.previous_queries.get(query_id).append(datetime.now().timestamp())
+            else:
+                self.previous_queries[query_id] = [datetime.now().timestamp()]
 
     def get_username_by_id(self, profile_id: int) -> str:
         """To get the current username of a profile, given its unique ID, this function can be used."""
@@ -326,14 +372,25 @@ class Instaloader:
                 break
         return followees
 
-    def get_comments(self, shortcode: str) -> List[Dict[str, Any]]:
+    def get_comments(self, shortcode: str, tries: int = 3) -> List[Dict[str, Any]]:
         """Retrieve comments of node with given shortcode."""
+        data = self.get_node_metadata(shortcode)
+        if data['edge_media_to_comment']['count'] == len(data['edge_media_to_comment']['edges']):
+            return [comment['node'] for comment in data['edge_media_to_comment']['edges']]
         data = self.graphql_query(17852405266163336, {'shortcode': shortcode,
                                                       'first': 500},
                                   referer='https://www.instagram.com/p/' + shortcode + '/')
         comments = []
         while True:
-            edge_media_to_comment = data['data']['shortcode_media']['edge_media_to_comment']
+            try:
+                edge_media_to_comment = data['data']['shortcode_media']['edge_media_to_comment']
+            except KeyError as err:
+                print('Missing key {} in response of GraphQL query for retrieving comments for node \'{}\'.'
+                      .format(err, shortcode), file=sys.stderr)
+                if tries <= 1:
+                    raise BadResponseException('GraphQL query returned strange JSON for shortcode {}:\n{}'.format(shortcode, data))
+                self._sleep()
+                return self.get_comments(shortcode, tries - 1)
             comments.extend([comment['node'] for comment in edge_media_to_comment['edges']])
             page_info = edge_media_to_comment['page_info']
             if page_info['has_next_page']:
@@ -547,7 +604,7 @@ class Instaloader:
             media = pic_json["entry_data"]["PostPage"][0]["graphql"]["shortcode_media"] \
                 if "graphql" in pic_json["entry_data"]["PostPage"][0] \
                 else pic_json["entry_data"]["PostPage"][0]["media"]
-        except KeyError as err:
+        except (KeyError, TypeError) as err:
             print(err, file=sys.stderr)
             print(json.dumps(pic_json, indent=4), file=sys.stderr)
             if tries <= 1:
@@ -714,9 +771,9 @@ class Instaloader:
                 shortcode = item["code"] if "code" in item else "no_code"
 
                 date_float = item["device_timestamp"] if "device_timestamp" in item else item["taken_at"]
-                try:
+                if date_float < 10000000000:
                     date = datetime.fromtimestamp(date_float)
-                except ValueError:
+                else:
                     # device_timestamp seems to sometime be in milliseconds
                     date_float /= 1000
                     date = datetime.fromtimestamp(date_float)
@@ -811,7 +868,7 @@ class Instaloader:
                                                     download_comments=download_comments)
                 except NodeUnavailableException:
                     print("Unable to download node \"https://www.instagram.com/p/{}/\" of user {} from feed."
-                          .format(node['shortcode'], name), file=sys.stderr)
+                          .format(node['shortcode'] if 'shortcode' in node else node['code'], name), file=sys.stderr)
                     continue
                 if fast_update and not downloaded:
                     return
@@ -860,7 +917,8 @@ class Instaloader:
                                                     download_comments=download_comments)
                 except NodeUnavailableException:
                     print("Unable to download node \"https://www.instagram.com/p/{}/\" "
-                          "while downloading hashtag \"{}\".".format(node['shortcode'], hashtag), file=sys.stderr)
+                          "while downloading hashtag \"{}\"."
+                          .format(node['shortcode'] if 'shortcode' in node else node['code'], hashtag), file=sys.stderr)
                     continue
                 if fast_update and not downloaded:
                     return
@@ -977,7 +1035,7 @@ class Instaloader:
                                                     download_comments=download_comments)
                 except NodeUnavailableException:
                     print("Unable to download node \"https://www.instagram.com/p/{}/\" of user {}."
-                          .format(node['shortcode'], name), file=sys.stderr)
+                          .format(node['shortcode'] if 'shortcode' in node else node['code'], name), file=sys.stderr)
                     continue
                 if fast_update and not downloaded:
                     return

setup.py

@@ -8,7 +8,7 @@ if sys.version_info < (3, 5):
 
 setup(
     name='instaloader',
-    version='2.2.2',
+    version='2.2.3',
     py_modules=['instaloader'],
     url='https://github.com/Thammus/instaloader',
     license='MIT',