Release of version 3.3.10

When fetching comments, likes, followees or followers, output a warning
that the used queries are rate limited.

instaloader.py

@@ -31,7 +31,7 @@ import requests.utils
 import urllib3
 
 
-__version__ = '3.3.8'
+__version__ = '3.3.10'
 
 # NOTE: duplicated in README.rst and docs/index.rst
 USAGE_STRING = """
@@ -377,6 +377,7 @@ class Post:
         Each comment is represented by a dictionary having the keys text, created_at, id and owner, which is a
         dictionary with keys username, profile_pic_url and id.
         """
+        # pylint:disable=protected-access
         if self.comments == 0:
             # Avoid doing additional requests if there are no comments
             return
@@ -385,6 +386,7 @@ class Post:
             # If the Post's metadata already contains all comments, don't do GraphQL requests to obtain them
             yield from (comment['node'] for comment in comment_edges)
             return
+        self._instaloader._log('(Warning: Fetching comments uses many queries, which are rate limited.)')
         yield from self._instaloader.graphql_node_list(17852405266163336, {'shortcode': self.shortcode},
                                                        'https://www.instagram.com/p/' + self.shortcode + '/',
                                                        lambda d: d['data']['shortcode_media']['edge_media_to_comment'],
@@ -396,6 +398,7 @@ class Post:
         Each like is represented by a dictionary having the keys username, followed_by_viewer, id, is_verified,
         requested_by_viewer, followed_by_viewer, profile_pic_url.
         """
+        # pylint:disable=protected-access
         if self.likes == 0:
             # Avoid doing additional requests if there are no comments
             return
@@ -404,6 +407,7 @@ class Post:
             # If the Post's metadata already contains all likes, don't do GraphQL requests to obtain them
             yield from (like['node'] for like in likes_edges)
             return
+        self._instaloader._log('(Warning: Fetching likes uses many queries, which are rate limited.)')
         yield from self._instaloader.graphql_node_list("1cb6ec562846122743b61e492c85999f", {'shortcode': self.shortcode},
                                                        'https://www.instagram.com/p/' + self.shortcode + '/',
                                                        lambda d: d['data']['shortcode_media']['edge_liked_by'],
@@ -460,7 +464,7 @@ class _PathPattern(str):
 
 
 class Instaloader:
-    GRAPHQL_PAGE_LENGTH = 200
+    GRAPHQL_PAGE_LENGTH = 50
 
     def __init__(self,
                  sleep: bool = True, quiet: bool = False,
@@ -506,7 +510,7 @@ class Instaloader:
         self.error_log = []
 
         # For the adaption of sleep intervals (rate control)
-        self.previous_queries = dict()
+        self.query_timestamps = list()
 
     @property
     def is_logged_in(self) -> bool:
@@ -523,10 +527,10 @@ class Instaloader:
                                  self.download_geotags,
                                  self.save_captions, self.download_comments,
                                  self.save_metadata, self.max_connection_attempts)
-        new_loader.previous_queries = self.previous_queries
+        new_loader.query_timestamps = self.query_timestamps
         yield new_loader
         self.error_log.extend(new_loader.error_log)
-        self.previous_queries = new_loader.previous_queries
+        self.query_timestamps = new_loader.query_timestamps
 
     def _log(self, *msg, sep='', end='\n', flush=False):
         """Log a message to stdout that can be suppressed with --quiet."""
@@ -604,29 +608,25 @@ class Instaloader:
         :raises QueryReturnedNotFoundException: When the server responds with a 404.
         :raises ConnectionException: When query repeatedly failed.
         """
-        def graphql_query_waittime(query_id: Union[int, str], untracked_queries: bool = False) -> int:
+        def graphql_query_waittime(untracked_queries: bool = False) -> int:
             sliding_window = 660
-            timestamps = self.previous_queries.get(query_id)
-            if not timestamps:
+            if not self.query_timestamps:
                 return sliding_window if untracked_queries else 0
             current_time = time.monotonic()
-            timestamps = list(filter(lambda t: t > current_time - sliding_window, timestamps))
-            self.previous_queries[query_id] = timestamps
-            if len(timestamps) < 100 and not untracked_queries:
+            self.query_timestamps = list(filter(lambda t: t > current_time - sliding_window, self.query_timestamps))
+            if len(self.query_timestamps) < 20 and not untracked_queries:
                 return 0
-            return round(min(timestamps) + sliding_window - current_time) + 6
+            return round(min(self.query_timestamps) + sliding_window - current_time) + 6
         is_graphql_query = 'graphql/query' in path
         if is_graphql_query:
-            query_id = params['query_id'] if 'query_id' in params else params['query_hash']
-            waittime = graphql_query_waittime(query_id)
+            waittime = graphql_query_waittime()
             if waittime > 0:
                 self._log('\nToo many queries in the last time. Need to wait {} seconds.'.format(waittime))
                 time.sleep(waittime)
-            timestamp_list = self.previous_queries.get(query_id)
-            if timestamp_list is not None:
-                timestamp_list.append(time.monotonic())
+            if self.query_timestamps is not None:
+                self.query_timestamps.append(time.monotonic())
             else:
-                self.previous_queries[query_id] = [time.monotonic()]
+                self.query_timestamps = [time.monotonic()]
         sess = session if session else self.session
         try:
             self._sleep()
@@ -663,7 +663,7 @@ class Instaloader:
                 if isinstance(err, TooManyRequests):
                     print(textwrap.fill(text_for_429), file=sys.stderr)
                     if is_graphql_query:
-                        waittime = graphql_query_waittime(query_id=params['query_id'], untracked_queries=True)
+                        waittime = graphql_query_waittime(untracked_queries=True)
                         if waittime > 0:
                             self._log('The request will be retried in {} seconds.'.format(waittime))
                             time.sleep(waittime)
@@ -783,6 +783,7 @@ class Instaloader:
 
         :param profile: Name of profile to lookup followers.
         """
+        self._log('(Warning: Fetching followers uses many queries, which are rate limited.)')
         yield from self.graphql_node_list(17851374694183129, {'id': str(self.get_id_by_username(profile))},
                                           'https://www.instagram.com/' + profile + '/',
                                           lambda d: d['data']['user']['edge_followed_by'])
@@ -795,6 +796,7 @@ class Instaloader:
 
         :param profile: Name of profile to lookup followers.
         """
+        self._log('(Warning: Fetching followees uses many queries, which are rate limited.)')
         yield from self.graphql_node_list(17874545323001329, {'id': str(self.get_id_by_username(profile))},
                                           'https://www.instagram.com/' + profile + '/',
                                           lambda d: d['data']['user']['edge_follow'])
@@ -1161,7 +1163,7 @@ class Instaloader:
             self._log("Retrieving stories from profile {}.".format(name))
             totalcount = len(user_stories["items"])
             count = 1
-            for item in user_stories["items"]:
+            for item in reversed(user_stories["items"]):
                 self._log("[%3i/%3i] " % (count, totalcount), end="", flush=True)
                 count += 1
                 with self._error_catcher('Download story from user {}'.format(name)):