Release of version 4.10.3

Closes issue #2125 by using an iphone endpoint for posts that have more
than 50 comments.

AUTHORS.md

@@ -6,4 +6,5 @@ Instaloader is written by
 - Alexander Graf (@aandergr)
 - André Koch-Kramer (@Thammus)
 - Lars Lindqvist (@e5150)
+- Eduardo M Kalinowksi (@ekalin)
 - ... and many more, see https://github.com/instaloader/instaloader/graphs/contributors

instaloader/__init__.py

@@ -1,7 +1,7 @@
 """Download pictures (or videos) along with their captions and other metadata from Instagram."""
 
 
-__version__ = '4.10.2'
+__version__ = '4.10.3'
 
 
 try:

instaloader/instaloader.py

@@ -991,7 +991,8 @@ class Instaloader:
                             max_count: Optional[int] = None,
                             total_count: Optional[int] = None,
                             owner_profile: Optional[Profile] = None,
-                            takewhile: Optional[Callable[[Post], bool]] = None) -> None:
+                            takewhile: Optional[Callable[[Post], bool]] = None,
+                            possibly_pinned: int = 0) -> None:
         """
         Download the Posts returned by given Post Iterator.
 
@@ -1003,6 +1004,9 @@ class Instaloader:
         .. versionchanged:: 4.8
            Add `takewhile` parameter.
 
+        .. versionchanged:: 4.10.3
+           Add `possibly_pinned` parameter.
+
         :param posts: Post Iterator to loop through.
         :param target: Target name.
         :param fast_update: :option:`--fast-update`.
@@ -1011,6 +1015,8 @@ class Instaloader:
         :param total_count: Total number of posts returned by given iterator.
         :param owner_profile: Associated profile, if any.
         :param takewhile: Expression evaluated for each post. Once it returns false, downloading stops.
+        :param possibly_pinned: Number of posts that might be pinned. These posts do not cause download
+               to stop even if they've already been downloaded.
         """
         displayed_count = (max_count if total_count is None or max_count is not None and max_count < total_count
                            else total_count)
@@ -1032,7 +1038,7 @@ class Instaloader:
         ) as (is_resuming, start_index):
             for number, post in enumerate(posts, start=start_index + 1):
                 should_stop = not takewhile(post)
-                if should_stop and post.is_pinned:
+                if should_stop and number <= possibly_pinned:
                     continue
                 if (max_count is not None and number > max_count) or should_stop:
                     break
@@ -1066,7 +1072,7 @@ class Instaloader:
                         except PostChangedException:
                             post_changed = True
                             continue
-                    if fast_update and not downloaded and not post_changed and not post.is_pinned:
+                    if fast_update and not downloaded and not post_changed and number > possibly_pinned:
                         # disengage fast_update for first post when resuming
                         if not is_resuming or number > 0:
                             break
@@ -1488,7 +1494,7 @@ class Instaloader:
                     posts_to_download = profile.get_posts()
                     self.posts_download_loop(posts_to_download, profile_name, fast_update, post_filter,
                                              total_count=profile.mediacount, owner_profile=profile,
-                                             takewhile=posts_takewhile)
+                                             takewhile=posts_takewhile, possibly_pinned=3)
                     if latest_stamps is not None and posts_to_download.first_item is not None:
                         latest_stamps.set_last_post_timestamp(profile_name,
                                                               posts_to_download.first_item.date_local)

instaloader/nodeiterator.py

@@ -184,6 +184,10 @@ class NodeIterator(Iterator[T]):
         """
         return self._node_wrapper(self._first_node) if self._first_node is not None else None
 
+    @staticmethod
+    def page_length() -> int:
+        return NodeIterator._graphql_page_length
+
     def freeze(self) -> FrozenNodeIterator:
         """Freeze the iterator for later resuming."""
         remaining_data = None

instaloader/structures.py

@@ -545,11 +545,74 @@ class Post:
         except KeyError:
             return self._field('edge_media_to_comment', 'count')
 
+    def _get_comments_via_iphone_endpoint(self) -> Iterable[PostComment]:
+        """
+        Iterate over all comments of the post via an iPhone endpoint.
+
+        .. versionadded:: 4.10.3
+           fallback for :issue:`2125`.
+        """
+        def _query(min_id=None):
+            pagination_params = {"min_id": min_id} if min_id is not None else {}
+            return self._context.get_iphone_json(
+                f"api/v1/media/{self.mediaid}/comments/",
+                {
+                    "can_support_threading": "true",
+                    "permalink_enabled": "false",
+                    **pagination_params,
+                },
+            )
+
+        def _answers(comment_node):
+            def _answer(child_comment):
+                return PostCommentAnswer(
+                    id=int(child_comment["pk"]),
+                    created_at_utc=datetime.utcfromtimestamp(child_comment["created_at"]),
+                    text=child_comment["text"],
+                    owner=Profile.from_iphone_struct(self._context, child_comment["user"]),
+                    likes_count=child_comment["comment_like_count"],
+                )
+
+            child_comment_count = comment_node["child_comment_count"]
+            if child_comment_count == 0:
+                return
+            preview_child_comments = comment_node["preview_child_comments"]
+            if child_comment_count == len(preview_child_comments):
+                yield from (
+                    _answer(child_comment) for child_comment in preview_child_comments
+                )
+                return
+            pk = comment_node["pk"]
+            answers_json = self._context.get_iphone_json(
+                f"api/v1/media/{self.mediaid}/comments/{pk}/child_comments/",
+                {"max_id": ""},
+            )
+            yield from (
+                _answer(child_comment) for child_comment in answers_json["child_comments"]
+            )
+
+        def _paginated_comments(comments_json):
+            for comment_node in comments_json.get("comments", []):
+                yield PostComment(
+                    id=int(comment_node["pk"]),
+                    created_at_utc=datetime.utcfromtimestamp(comment_node["created_at"]),
+                    text=comment_node["text"],
+                    owner=Profile.from_iphone_struct(self._context, comment_node["user"]),
+                    likes_count=comment_node["comment_like_count"],
+                    answers=_answers(comment_node),
+                )
+
+            next_min_id = comments_json.get("next_min_id")
+            if next_min_id:
+                yield from _paginated_comments(_query(next_min_id))
+
+        return _paginated_comments(_query())
+
     def get_comments(self) -> Iterable[PostComment]:
-        r"""Iterate over all comments of the post.
+        """Iterate over all comments of the post.
 
         Each comment is represented by a PostComment NamedTuple with fields text (string), created_at (datetime),
-        id (int), owner (:class:`Profile`) and answers (:class:`~typing.Iterator`\ [:class:`PostCommentAnswer`])
+        id (int), owner (:class:`Profile`) and answers (:class:`~typing.Iterator` [:class:`PostCommentAnswer`])
         if available.
 
         .. versionchanged:: 4.7
@@ -596,6 +659,12 @@ class Post:
         if self.comments == len(comment_edges) + answers_count:
             # If the Post's metadata already contains all parent comments, don't do GraphQL requests to obtain them
             return [_postcomment(comment['node']) for comment in comment_edges]
+
+        if self.comments > NodeIterator.page_length():
+            # comments pagination via our graphql query does not work reliably anymore (issue #2125), fallback to an
+            # iphone endpoint if needed.
+            return self._get_comments_via_iphone_endpoint()
+
         return NodeIterator(
             self._context,
             '97b41c52301f77ce508f55e66d17620e',
@@ -678,7 +747,11 @@ class Post:
 
     @property
     def is_pinned(self) -> bool:
-        """True if this Post has been pinned by at least one user.
+        """
+        .. deprecated: 4.10.3
+           This information is not returned by IG anymore
+
+        Used to return True if this Post has been pinned by at least one user, now likely returns always false.
 
         .. versionadded: 4.9.2"""
         return 'pinned_for_users' in self._node and bool(self._node['pinned_for_users'])