Release of version 4.2.4

Freeze pylint version to 2.3.1.

.travis.yml

@@ -5,10 +5,10 @@ python:
  - "3.6"
  - "3.6-dev"
 install:
- - pip install pylint requests
+ - pip install pylint~=2.3.1 requests
  - pip install -r docs/requirements.txt
 script:
- - python3 -m pylint -r n -d bad-whitespace,missing-docstring,too-many-arguments,locally-disabled,line-too-long,too-many-public-methods,too-many-lines,too-many-instance-attributes,too-many-locals,too-many-branches,too-many-statements,inconsistent-return-statements,invalid-name,wildcard-import,unused-wildcard-import,no-else-return,cyclic-import,unnecessary-pass instaloader
+ - python3 -m pylint -r n -d bad-whitespace,missing-docstring,too-many-arguments,locally-disabled,line-too-long,no-else-raise,too-many-public-methods,too-many-lines,too-many-instance-attributes,too-many-locals,too-many-branches,too-many-statements,inconsistent-return-statements,invalid-name,wildcard-import,unused-wildcard-import,no-else-return,cyclic-import,unnecessary-pass instaloader
  - make -C docs html SPHINXOPTS="-W -n"
 deploy:
   - provider: pypi

docs/as-module.rst

@@ -123,6 +123,9 @@ Additionally, the following trivial structures are defined:
 .. autoclass:: PostComment
    :no-show-inheritance:
 
+.. autoclass:: PostCommentAnswer
+   :no-show-inheritance:
+
 .. autoclass:: PostLocation
    :no-show-inheritance:
 

instaloader/__init__.py

@@ -1,7 +1,7 @@
 """Download pictures (or videos) along with their captions and other metadata from Instagram."""
 
 
-__version__ = '4.2.2'
+__version__ = '4.2.4'
 
 
 try:
@@ -15,5 +15,5 @@ else:
 from .exceptions import *
 from .instaloader import Instaloader
 from .instaloadercontext import InstaloaderContext
-from .structures import (Highlight, Post, PostSidecarNode, PostComment, PostLocation, Profile, Story, StoryItem,
+from .structures import (Highlight, Post, PostSidecarNode, PostComment, PostCommentAnswer, PostLocation, Profile, Story, StoryItem,
                          load_structure_from_file, save_structure_to_file)

instaloader/__main__.py

@@ -331,7 +331,6 @@ def main():
     g_how.add_argument('--user-agent',
                        help='User Agent to use for HTTP requests. Defaults to \'{}\'.'.format(default_user_agent()))
     g_how.add_argument('-S', '--no-sleep', action='store_true', help=SUPPRESS)
-    g_how.add_argument('--graphql-rate-limit', type=int, help=SUPPRESS)
     g_how.add_argument('--max-connection-attempts', metavar='N', type=int, default=3,
                        help='Maximum number of connection attempts until a request is aborted. Defaults to 3. If a '
                             'connection fails, it can be manually skipped by hitting CTRL+C. Set this to 0 to retry '
@@ -393,7 +392,6 @@ def main():
                              compress_json=not args.no_compress_json,
                              post_metadata_txt_pattern=post_metadata_txt_pattern,
                              storyitem_metadata_txt_pattern=storyitem_metadata_txt_pattern,
-                             graphql_rate_limit=args.graphql_rate_limit,
                              max_connection_attempts=args.max_connection_attempts,
                              commit_mode=args.commit_mode)
         _main(loader,

instaloader/instaloader.py

@@ -152,11 +152,10 @@ class Instaloader:
                  compress_json: bool = True,
                  post_metadata_txt_pattern: str = None,
                  storyitem_metadata_txt_pattern: str = None,
-                 graphql_rate_limit: Optional[int] = None,
                  max_connection_attempts: int = 3,
                  commit_mode: bool = False):
 
-        self.context = InstaloaderContext(sleep, quiet, user_agent, graphql_rate_limit, max_connection_attempts)
+        self.context = InstaloaderContext(sleep, quiet, user_agent, max_connection_attempts)
 
         # configuration parameters
         self.dirname_pattern = dirname_pattern or "{target}"
@@ -190,13 +189,10 @@ class Instaloader:
                                  save_metadata=self.save_metadata, compress_json=self.compress_json,
                                  post_metadata_txt_pattern=self.post_metadata_txt_pattern,
                                  storyitem_metadata_txt_pattern=self.storyitem_metadata_txt_pattern,
-                                 graphql_rate_limit=self.context.graphql_count_per_slidingwindow,
                                  max_connection_attempts=self.context.max_connection_attempts)
-        new_loader.context.query_timestamps = self.context.query_timestamps
         yield new_loader
         self.context.error_log.extend(new_loader.context.error_log)
         new_loader.context.error_log = []  # avoid double-printing of errors
-        self.context.query_timestamps = new_loader.context.query_timestamps
         new_loader.close()
 
     def close(self):
@@ -244,11 +240,33 @@ class Instaloader:
             self.context.log('json', end=' ', flush=True)
 
     def update_comments(self, filename: str, post: Post) -> None:
-        def _postcomment_asdict(comment):
+        def _postcommentanswer_asdict(comment):
             return {'id': comment.id,
                     'created_at': int(comment.created_at_utc.replace(tzinfo=timezone.utc).timestamp()),
                     'text': comment.text,
                     'owner': comment.owner._asdict()}
+
+        def _postcomment_asdict(comment):
+            return {**_postcommentanswer_asdict(comment),
+                    'answers': sorted([_postcommentanswer_asdict(answer) for answer in comment.answers],
+                                      key=lambda t: int(t['id']),
+                                      reverse=True)}
+
+        def get_unique_comments(comments, combine_answers=False):
+            if not comments:
+                return list()
+            comments_list = sorted(sorted(list(comments), key=lambda t: int(t['id'])),
+                                   key=lambda t: int(t['created_at']), reverse=True)
+            unique_comments_list = [comments_list[0]]
+            for x, y in zip(comments_list[:-1], comments_list[1:]):
+                if x['id'] != y['id']:
+                    unique_comments_list.append(y)
+                elif combine_answers:
+                    combined_answers = unique_comments_list[-1].get('answers') or list()
+                    if 'answers' in y:
+                        combined_answers.extend(y['answers'])
+                    unique_comments_list[-1]['answers'] = get_unique_comments(combined_answers)
+            return unique_comments_list
         filename += '_comments.json'
         try:
             with open(filename) as fp:
@@ -257,18 +275,10 @@ class Instaloader:
             comments = list()
         comments.extend(_postcomment_asdict(comment) for comment in post.get_comments())
         if comments:
-            comments_list = sorted(sorted(list(comments), key=lambda t: int(t['id'])),
-                                   key=lambda t: int(t['created_at']), reverse=True)
-            unique_comments_list = [comments_list[0]]
-            #for comment in comments_list:
-            #    if unique_comments_list[-1]['id'] != comment['id']:
-            #        unique_comments_list.append(comment)
-            #file.write(json.dumps(unique_comments_list, indent=4))
-            for x, y in zip(comments_list[:-1], comments_list[1:]):
-                if x['id'] != y['id']:
-                    unique_comments_list.append(y)
+            comments = get_unique_comments(comments, combine_answers=True)
+            answer_ids = set(int(answer['id']) for comment in comments for answer in comment.get('answers'))
             with open(filename, 'w') as file:
-                file.write(json.dumps(unique_comments_list, indent=4))
+                file.write(json.dumps(list(filter(lambda t: int(t['id']) not in answer_ids, comments)), indent=4))
             self.context.log('comments', end=' ', flush=True)
 
     def save_caption(self, filename: str, mtime: datetime, caption: str) -> None:

instaloader/instaloadercontext.py

@@ -47,7 +47,7 @@ class InstaloaderContext:
     """
 
     def __init__(self, sleep: bool = True, quiet: bool = False, user_agent: Optional[str] = None,
-                 graphql_count_per_slidingwindow: Optional[int] = None, max_connection_attempts: int = 3):
+                 max_connection_attempts: int = 3):
 
         self.user_agent = user_agent if user_agent is not None else default_user_agent()
         self._session = self.get_anonymous_session()
@@ -56,7 +56,6 @@ class InstaloaderContext:
         self.quiet = quiet
         self.max_connection_attempts = max_connection_attempts
         self._graphql_page_length = 50
-        self.graphql_count_per_slidingwindow = graphql_count_per_slidingwindow or 200
         self._root_rhx_gis = None
         self.two_factor_auth_pending = None
 
@@ -64,7 +63,8 @@ class InstaloaderContext:
         self.error_log = []
 
         # For the adaption of sleep intervals (rate control)
-        self.query_timestamps = list()
+        self._graphql_query_timestamps = dict()
+        self._graphql_earliest_next_request_time = 0
 
         # Can be set to True for testing, disables supression of InstaloaderContext._error_catcher
         self.raise_all_errors = False
@@ -265,6 +265,72 @@ class InstaloaderContext:
         if self.sleep:
             time.sleep(min(random.expovariate(0.7), 5.0))
 
+    def _dump_query_timestamps(self, current_time: float):
+        """Output the number of GraphQL queries grouped by their query_hash within the last time."""
+        windows = [10, 11, 15, 20, 30, 60]
+        print("GraphQL requests:", file=sys.stderr)
+        for query_hash, times in self._graphql_query_timestamps.items():
+            print("  {}".format(query_hash), file=sys.stderr)
+            for window in windows:
+                reqs_in_sliding_window = sum(t > current_time - window * 60 for t in times)
+                print("    last {} minutes: {} requests".format(window, reqs_in_sliding_window), file=sys.stderr)
+
+    def _graphql_request_count_per_sliding_window(self, query_hash: str) -> int:
+        """Return how many GraphQL requests can be done within the sliding window."""
+        if self.is_logged_in:
+            max_reqs = {'1cb6ec562846122743b61e492c85999f': 20, '33ba35852cb50da46f5b5e889df7d159': 20}
+        else:
+            max_reqs = {'1cb6ec562846122743b61e492c85999f': 200, '33ba35852cb50da46f5b5e889df7d159': 200}
+        return max_reqs.get(query_hash) or min(max_reqs.values())
+
+    def _graphql_query_waittime(self, query_hash: str, current_time: float, untracked_queries: bool = False) -> int:
+        """Calculate time needed to wait before GraphQL query can be executed."""
+        sliding_window = 660
+        if query_hash not in self._graphql_query_timestamps:
+            self._graphql_query_timestamps[query_hash] = []
+        self._graphql_query_timestamps[query_hash] = list(filter(lambda t: t > current_time - 60 * 60,
+                                                                 self._graphql_query_timestamps[query_hash]))
+        reqs_in_sliding_window = list(filter(lambda t: t > current_time - sliding_window,
+                                             self._graphql_query_timestamps[query_hash]))
+        count_per_sliding_window = self._graphql_request_count_per_sliding_window(query_hash)
+        if len(reqs_in_sliding_window) < count_per_sliding_window and not untracked_queries:
+            return max(0, self._graphql_earliest_next_request_time - current_time)
+        next_request_time = min(reqs_in_sliding_window) + sliding_window + 6
+        if untracked_queries:
+            self._graphql_earliest_next_request_time = next_request_time
+        return round(max(next_request_time, self._graphql_earliest_next_request_time) - current_time)
+
+    def _ratecontrol_graphql_query(self, query_hash: str, untracked_queries: bool = False):
+        """Called before a GraphQL query is made in order to stay within Instagram's rate limits.
+
+        :param query_hash: The query_hash parameter of the query.
+        :param untracked_queries: True, if 429 has been returned to apply 429 logic.
+        """
+        if not untracked_queries:
+            waittime = self._graphql_query_waittime(query_hash, time.monotonic(), untracked_queries)
+            assert waittime >= 0
+            if waittime > 10:
+                self.log('\nToo many queries in the last time. Need to wait {} seconds, until {:%H:%M}.'
+                         .format(waittime, datetime.now() + timedelta(seconds=waittime)))
+            time.sleep(waittime)
+            if query_hash not in self._graphql_query_timestamps:
+                self._graphql_query_timestamps[query_hash] = [time.monotonic()]
+            else:
+                self._graphql_query_timestamps[query_hash].append(time.monotonic())
+        else:
+            text_for_429 = ("HTTP error code 429 was returned because too many queries occured in the last time. "
+                            "Please do not use Instagram in your browser or run multiple instances of Instaloader "
+                            "in parallel.")
+            print(textwrap.fill(text_for_429), file=sys.stderr)
+            current_time = time.monotonic()
+            waittime = self._graphql_query_waittime(query_hash, current_time, untracked_queries)
+            assert waittime >= 0
+            if waittime > 10:
+                self.log('The request will be retried in {} seconds, at {:%H:%M}.'
+                         .format(waittime, datetime.now() + timedelta(seconds=waittime)))
+            self._dump_query_timestamps(current_time)
+            time.sleep(waittime)
+
     def get_json(self, path: str, params: Dict[str, Any], host: str = 'www.instagram.com',
                  session: Optional[requests.Session] = None, _attempt=1) -> Dict[str, Any]:
         """JSON request to Instagram.
@@ -278,32 +344,12 @@ class InstaloaderContext:
         :raises QueryReturnedNotFoundException: When the server responds with a 404.
         :raises ConnectionException: When query repeatedly failed.
         """
-        def graphql_query_waittime(untracked_queries: bool = False) -> int:
-            sliding_window = 660
-            if not self.query_timestamps:
-                return sliding_window if untracked_queries else 0
-            current_time = time.monotonic()
-            self.query_timestamps = list(filter(lambda t: t > current_time - sliding_window, self.query_timestamps))
-            if len(self.query_timestamps) < self.graphql_count_per_slidingwindow and not untracked_queries:
-                return 0
-            return round(min(self.query_timestamps) + sliding_window - current_time) + 6
         is_graphql_query = 'query_hash' in params and 'graphql/query' in path
-        # some queries are not rate limited if invoked anonymously:
-        query_not_limited = is_graphql_query and not self.is_logged_in \
-                            and params['query_hash'] in ['9ca88e465c3f866a76f7adee3871bdd8']
-        if is_graphql_query and not query_not_limited:
-            waittime = graphql_query_waittime()
-            if waittime > 0:
-                self.log('\nToo many queries in the last time. Need to wait {} seconds, until {:%H:%M}.'
-                         .format(waittime, datetime.now() + timedelta(seconds=waittime)))
-                time.sleep(waittime)
-            if self.query_timestamps is not None:
-                self.query_timestamps.append(time.monotonic())
-            else:
-                self.query_timestamps = [time.monotonic()]
         sess = session if session else self._session
         try:
             self.do_sleep()
+            if is_graphql_query:
+                self._ratecontrol_graphql_query(params['query_hash'])
             resp = sess.get('https://{0}/{1}'.format(host, path), params=params, allow_redirects=False)
             while resp.is_redirect:
                 redirect_url = resp.headers['location']
@@ -341,20 +387,9 @@ class InstaloaderContext:
             if _attempt == self.max_connection_attempts:
                 raise ConnectionException(error_string) from err
             self.error(error_string + " [retrying; skip with ^C]", repeat_at_end=False)
-            text_for_429 = ("HTTP error code 429 was returned because too many queries occured in the last time. "
-                            "Please do not use Instagram in your browser or run multiple instances of Instaloader "
-                            "in parallel.")
             try:
                 if isinstance(err, TooManyRequestsException):
-                    print(textwrap.fill(text_for_429), file=sys.stderr)
-                    if is_graphql_query:
-                        print("Made {} GraphQL requests.".format(len(self.query_timestamps)), file=sys.stderr)
-                        waittime = graphql_query_waittime(untracked_queries=True)
-                        if waittime > 0:
-                            self.log('The request will be retried in {} seconds, at {:%H:%M}.'
-                                     .format(waittime, datetime.now() + timedelta(seconds=waittime)))
-                            time.sleep(waittime)
-                self.do_sleep()
+                    self._ratecontrol_graphql_query(params['query_hash'], untracked_queries=True)
                 return self.get_json(path=path, params=params, host=host, session=sess, _attempt=_attempt + 1)
             except KeyboardInterrupt:
                 self.error("[skipped by user]", repeat_at_end=False)

instaloader/structures.py

@@ -17,11 +17,16 @@ PostSidecarNode.is_video.__doc__ = "Whether this node is a video."
 PostSidecarNode.display_url.__doc__ = "URL of image or video thumbnail."
 PostSidecarNode.video_url.__doc__ = "URL of video or None."
 
-PostComment = namedtuple('PostComment', ['id', 'created_at_utc', 'text', 'owner'])
-PostComment.id.__doc__ = "ID number of comment."
-PostComment.created_at_utc.__doc__ = ":class:`~datetime.datetime` when comment was created (UTC)."
-PostComment.text.__doc__ = "Comment text."
-PostComment.owner.__doc__ = "Owner :class:`Profile` of the comment."
+PostCommentAnswer = namedtuple('PostCommentAnswer', ['id', 'created_at_utc', 'text', 'owner'])
+PostCommentAnswer.id.__doc__ = "ID number of comment."
+PostCommentAnswer.created_at_utc.__doc__ = ":class:`~datetime.datetime` when comment was created (UTC)."
+PostCommentAnswer.text.__doc__ = "Comment text."
+PostCommentAnswer.owner.__doc__ = "Owner :class:`Profile` of the comment."
+
+PostComment = namedtuple('PostComment', (*PostCommentAnswer._fields, 'answers'))
+for field in PostCommentAnswer._fields:
+    getattr(PostComment, field).__doc__ = getattr(PostCommentAnswer, field).__doc__
+PostComment.answers.__doc__ = r"Iterator which yields all :class:`PostCommentAnswer`\ s for the comment."
 
 PostLocation = namedtuple('PostLocation', ['id', 'name', 'slug', 'has_public_page', 'lat', 'lng'])
 PostLocation.id.__doc__ = "ID number of location."
@@ -283,34 +288,72 @@ class Post:
 
     @property
     def comments(self) -> int:
-        """Comment count"""
-        return self._field('edge_media_to_comment', 'count')
+        """Comment count including answers"""
+        try:
+            return self._field('edge_media_to_parent_comment', 'count')
+        except KeyError:
+            return self._field('edge_media_to_comment', 'count')
 
     def get_comments(self) -> Iterator[PostComment]:
-        """Iterate over all comments of the post.
+        r"""Iterate over all comments of the post.
 
         Each comment is represented by a PostComment namedtuple with fields text (string), created_at (datetime),
-        id (int) and owner (:class:`Profile`).
+        id (int), owner (:class:`Profile`) and answers (:class:`~typing.Iterator`\ [:class:`PostCommentAnswer`])
+        if available.
         """
+        def _postcommentanswer(node):
+            return PostCommentAnswer(id=int(node['id']),
+                                     created_at_utc=datetime.utcfromtimestamp(node['created_at']),
+                                     text=node['text'],
+                                     owner=Profile(self._context, node['owner']))
+
+        def _postcommentanswers(node):
+            if 'edge_threaded_comments' not in node:
+                return
+            answer_count = node['edge_threaded_comments']['count']
+            if answer_count == 0:
+                # Avoid doing additional requests if there are no comment answers
+                return
+            answer_edges = node['edge_threaded_comments']['edges']
+            if answer_count == len(answer_edges):
+                # If the answer's metadata already contains all comments, don't do GraphQL requests to obtain them
+                yield from (_postcommentanswer(comment['node']) for comment in answer_edges)
+                return
+            yield from (_postcommentanswer(answer_node) for answer_node in
+                        self._context.graphql_node_list("51fdd02b67508306ad4484ff574a0b62",
+                                                        {'comment_id': node['id']},
+                                                        'https://www.instagram.com/p/' + self.shortcode + '/',
+                                                        lambda d: d['data']['comment']['edge_threaded_comments']))
+
         def _postcomment(node):
-            return PostComment(id=int(node['id']),
-                               created_at_utc=datetime.utcfromtimestamp(node['created_at']),
-                               text=node['text'],
-                               owner=Profile(self._context, node['owner']))
+            return PostComment(*_postcommentanswer(node),
+                               answers=_postcommentanswers(node))
         if self.comments == 0:
             # Avoid doing additional requests if there are no comments
             return
-        comment_edges = self._field('edge_media_to_comment', 'edges')
-        if self.comments == len(comment_edges):
-            # If the Post's metadata already contains all comments, don't do GraphQL requests to obtain them
+        try:
+            comment_edges = self._field('edge_media_to_parent_comment', 'edges')
+            answers_count = sum([edge['node']['edge_threaded_comments']['count'] for edge in comment_edges])
+            threaded_comments_available = True
+        except KeyError:
+            comment_edges = self._field('edge_media_to_comment', 'edges')
+            answers_count = 0
+            threaded_comments_available = False
+
+        if self.comments == len(comment_edges) + answers_count:
+            # If the Post's metadata already contains all parent comments, don't do GraphQL requests to obtain them
             yield from (_postcomment(comment['node']) for comment in comment_edges)
             return
         yield from (_postcomment(node) for node in
-                    self._context.graphql_node_list("33ba35852cb50da46f5b5e889df7d159",
-                                                    {'shortcode': self.shortcode},
-                                                    'https://www.instagram.com/p/' + self.shortcode + '/',
-                                                    lambda d: d['data']['shortcode_media']['edge_media_to_comment'],
-                                                    self._rhx_gis))
+                    self._context.graphql_node_list(
+                        "97b41c52301f77ce508f55e66d17620e" if threaded_comments_available
+                        else "f0986789a5c5d17c2400faebf16efd0d",
+                        {'shortcode': self.shortcode},
+                        'https://www.instagram.com/p/' + self.shortcode + '/',
+                        lambda d:
+                        d['data']['shortcode_media'][
+                            'edge_media_to_parent_comment' if threaded_comments_available else 'edge_media_to_comment'],
+                        self._rhx_gis))
 
     def get_likes(self) -> Iterator['Profile']:
         """Iterate over all likes of the post. A :class:`Profile` instance of each likee is yielded."""

test/instaloader_unittests.py

@@ -22,7 +22,7 @@ EMPTY_PROFILE = "not_public"
 EMPTY_PROFILE_ID = 1928659031
 
 # Preserve query timestamps (rate control) between tests to not get rate limited
-instaloadercontext_query_timestamps = list()
+instaloadercontext_query_timestamps = dict()
 
 
 class TestInstaloaderAnonymously(unittest.TestCase):
@@ -35,11 +35,11 @@ class TestInstaloaderAnonymously(unittest.TestCase):
                                          download_comments=True,
                                          save_metadata=True)
         self.L.context.raise_all_errors = True
-        self.L.context.query_timestamps = instaloadercontext_query_timestamps
+        self.L.context.query_timestamps = instaloadercontext_query_timestamps.copy()
 
     def tearDown(self):
         global instaloadercontext_query_timestamps
-        instaloadercontext_query_timestamps = self.L.context.query_timestamps
+        instaloadercontext_query_timestamps = self.L.context.query_timestamps.copy()
         self.L.close()
         os.chdir('/')
         print("Removing {}".format(self.dir))