From 47235bf2b87e7d1a1eb0cea2f199e49683032e67 Mon Sep 17 00:00:00 2001 From: Eduardo Kalinowski Date: Mon, 8 Jan 2024 03:46:27 -0300 Subject: [PATCH 1/5] Change pinned post detection, fixes #2142 (#2147) The property that indicated whether a post is pinned is not available anymore. As a workaround, the first five posts are treated differently (as pinned posts were): if they were already downloaded, they are skipped, but they do not cause the iteration to stop. --- instaloader/instaloader.py | 14 ++++++++++---- instaloader/structures.py | 6 +++++- 2 files changed, 15 insertions(+), 5 deletions(-) diff --git a/instaloader/instaloader.py b/instaloader/instaloader.py index c5def2f..e854360 100644 --- a/instaloader/instaloader.py +++ b/instaloader/instaloader.py @@ -991,7 +991,8 @@ class Instaloader: max_count: Optional[int] = None, total_count: Optional[int] = None, owner_profile: Optional[Profile] = None, - takewhile: Optional[Callable[[Post], bool]] = None) -> None: + takewhile: Optional[Callable[[Post], bool]] = None, + possibly_pinned: int = 0) -> None: """ Download the Posts returned by given Post Iterator. @@ -1003,6 +1004,9 @@ class Instaloader: .. versionchanged:: 4.8 Add `takewhile` parameter. + .. versionchanged:: 4.10.3 + Add `possibly_pinned` parameter. + :param posts: Post Iterator to loop through. :param target: Target name. :param fast_update: :option:`--fast-update`. @@ -1011,6 +1015,8 @@ class Instaloader: :param total_count: Total number of posts returned by given iterator. :param owner_profile: Associated profile, if any. :param takewhile: Expression evaluated for each post. Once it returns false, downloading stops. + :param possibly_pinned: Number of posts that might be pinned. These posts do not cause download + to stop even if they've already been downloaded. """ displayed_count = (max_count if total_count is None or max_count is not None and max_count < total_count else total_count) @@ -1032,7 +1038,7 @@ class Instaloader: ) as (is_resuming, start_index): for number, post in enumerate(posts, start=start_index + 1): should_stop = not takewhile(post) - if should_stop and post.is_pinned: + if should_stop and number <= possibly_pinned: continue if (max_count is not None and number > max_count) or should_stop: break @@ -1066,7 +1072,7 @@ class Instaloader: except PostChangedException: post_changed = True continue - if fast_update and not downloaded and not post_changed and not post.is_pinned: + if fast_update and not downloaded and not post_changed and number > possibly_pinned: # disengage fast_update for first post when resuming if not is_resuming or number > 0: break @@ -1488,7 +1494,7 @@ class Instaloader: posts_to_download = profile.get_posts() self.posts_download_loop(posts_to_download, profile_name, fast_update, post_filter, total_count=profile.mediacount, owner_profile=profile, - takewhile=posts_takewhile) + takewhile=posts_takewhile, possibly_pinned=3) if latest_stamps is not None and posts_to_download.first_item is not None: latest_stamps.set_last_post_timestamp(profile_name, posts_to_download.first_item.date_local) diff --git a/instaloader/structures.py b/instaloader/structures.py index ef74d0b..86d5840 100644 --- a/instaloader/structures.py +++ b/instaloader/structures.py @@ -678,7 +678,11 @@ class Post: @property def is_pinned(self) -> bool: - """True if this Post has been pinned by at least one user. + """ + .. deprecated: 4.10.3 + This information is not returned by IG anymore + + Used to return True if this Post has been pinned by at least one user, now likely returns always false. .. versionadded: 4.9.2""" return 'pinned_for_users' in self._node and bool(self._node['pinned_for_users']) From baaa2c2998f019333d079572c97a5d32e0a0827d Mon Sep 17 00:00:00 2001 From: Eduardo Kalinowski Date: Mon, 8 Jan 2024 08:15:13 -0300 Subject: [PATCH 2/5] Add myself to AUTHORS.md (#2160) --- AUTHORS.md | 1 + 1 file changed, 1 insertion(+) diff --git a/AUTHORS.md b/AUTHORS.md index 8dd917f..01119cf 100644 --- a/AUTHORS.md +++ b/AUTHORS.md @@ -6,4 +6,5 @@ Instaloader is written by - Alexander Graf (@aandergr) - André Koch-Kramer (@Thammus) - Lars Lindqvist (@e5150) +- Eduardo M Kalinowksi (@ekalin) - ... and many more, see https://github.com/instaloader/instaloader/graphs/contributors From 185244782519ed8ba98e204f22c5039c4a775dd3 Mon Sep 17 00:00:00 2001 From: Alexander Graf <17130992+aandergr@users.noreply.github.com> Date: Fri, 12 Jan 2024 08:07:54 +0100 Subject: [PATCH 3/5] Fix downloading comments Closes issue #2125 by using an iphone endpoint for posts that have more than 50 comments. --- instaloader/nodeiterator.py | 4 ++ instaloader/structures.py | 73 ++++++++++++++++++++++++++++++++++++- 2 files changed, 75 insertions(+), 2 deletions(-) diff --git a/instaloader/nodeiterator.py b/instaloader/nodeiterator.py index a5c907c..37544d7 100644 --- a/instaloader/nodeiterator.py +++ b/instaloader/nodeiterator.py @@ -184,6 +184,10 @@ class NodeIterator(Iterator[T]): """ return self._node_wrapper(self._first_node) if self._first_node is not None else None + @staticmethod + def page_length() -> int: + return NodeIterator._graphql_page_length + def freeze(self) -> FrozenNodeIterator: """Freeze the iterator for later resuming.""" remaining_data = None diff --git a/instaloader/structures.py b/instaloader/structures.py index 86d5840..998adf2 100644 --- a/instaloader/structures.py +++ b/instaloader/structures.py @@ -545,11 +545,74 @@ class Post: except KeyError: return self._field('edge_media_to_comment', 'count') + def _get_comments_via_iphone_endpoint(self) -> Iterable[PostComment]: + """ + Iterate over all comments of the post via an iPhone endpoint. + + .. versionadded:: 4.10.3 + fallback for :issue:`2125`. + """ + def _query(min_id=None): + pagination_params = {"min_id": min_id} if min_id is not None else {} + return self._context.get_iphone_json( + f"api/v1/media/{self.mediaid}/comments/", + { + "can_support_threading": "true", + "permalink_enabled": "false", + **pagination_params, + }, + ) + + def _answers(comment_node): + def _answer(child_comment): + return PostCommentAnswer( + id=int(child_comment["pk"]), + created_at_utc=datetime.utcfromtimestamp(child_comment["created_at"]), + text=child_comment["text"], + owner=Profile.from_iphone_struct(self._context, child_comment["user"]), + likes_count=child_comment["comment_like_count"], + ) + + child_comment_count = comment_node["child_comment_count"] + if child_comment_count == 0: + return + preview_child_comments = comment_node["preview_child_comments"] + if child_comment_count == len(preview_child_comments): + yield from ( + _answer(child_comment) for child_comment in preview_child_comments + ) + return + pk = comment_node["pk"] + answers_json = self._context.get_iphone_json( + f"api/v1/media/{self.mediaid}/comments/{pk}/child_comments/", + {"max_id": ""}, + ) + yield from ( + _answer(child_comment) for child_comment in answers_json["child_comments"] + ) + + def _paginated_comments(comments_json): + for comment_node in comments_json.get("comments", []): + yield PostComment( + id=int(comment_node["pk"]), + created_at_utc=datetime.utcfromtimestamp(comment_node["created_at"]), + text=comment_node["text"], + owner=Profile.from_iphone_struct(self._context, comment_node["user"]), + likes_count=comment_node["comment_like_count"], + answers=_answers(comment_node), + ) + + next_min_id = comments_json.get("next_min_id") + if next_min_id: + yield from _paginated_comments(_query(next_min_id)) + + return _paginated_comments(_query()) + def get_comments(self) -> Iterable[PostComment]: - r"""Iterate over all comments of the post. + """Iterate over all comments of the post. Each comment is represented by a PostComment NamedTuple with fields text (string), created_at (datetime), - id (int), owner (:class:`Profile`) and answers (:class:`~typing.Iterator`\ [:class:`PostCommentAnswer`]) + id (int), owner (:class:`Profile`) and answers (:class:`~typing.Iterator` [:class:`PostCommentAnswer`]) if available. .. versionchanged:: 4.7 @@ -596,6 +659,12 @@ class Post: if self.comments == len(comment_edges) + answers_count: # If the Post's metadata already contains all parent comments, don't do GraphQL requests to obtain them return [_postcomment(comment['node']) for comment in comment_edges] + + if self.comments > NodeIterator.page_length(): + # comments pagination via our graphql query does not work reliably anymore (issue #2125), fallback to an + # iphone endpoint if needed. + return self._get_comments_via_iphone_endpoint() + return NodeIterator( self._context, '97b41c52301f77ce508f55e66d17620e', From 2b594909bb097919d954b562a37cce57a394970a Mon Sep 17 00:00:00 2001 From: Alexander Graf <17130992+aandergr@users.noreply.github.com> Date: Fri, 12 Jan 2024 08:15:56 +0100 Subject: [PATCH 4/5] Release of version 4.10.3 --- instaloader/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/instaloader/__init__.py b/instaloader/__init__.py index 53ef0ca..e08f73e 100644 --- a/instaloader/__init__.py +++ b/instaloader/__init__.py @@ -1,7 +1,7 @@ """Download pictures (or videos) along with their captions and other metadata from Instagram.""" -__version__ = '4.10.2' +__version__ = '4.10.3' try: From 5509ea7daf7f5e81534d4eb996c8afd478704426 Mon Sep 17 00:00:00 2001 From: Alexander Graf <17130992+aandergr@users.noreply.github.com> Date: Fri, 12 Jan 2024 08:29:09 +0100 Subject: [PATCH 5/5] Update sponsors in README --- README.rst | 1 - 1 file changed, 1 deletion(-) diff --git a/README.rst b/README.rst index 1a10421..2e28575 100644 --- a/README.rst +++ b/README.rst @@ -125,7 +125,6 @@ Supporters | Instaloader is proudly sponsored by | `@rocketapi-io `__ -| `@socialmethod `__ See `Alex' GitHub Sponsors `__ page for how you can sponsor the development of Instaloader!