From fbd7df2a90c459807cd8af3e8bf881529082c8f0 Mon Sep 17 00:00:00 2001 From: Alexander Graf <17130992+aandergr@users.noreply.github.com> Date: Sat, 26 Mar 2022 16:59:42 +0100 Subject: [PATCH] Use resumable iteration to download hashtags Closes #1457. --- docs/cli-options.rst | 3 ++- instaloader/instaloader.py | 6 +++--- instaloader/structures.py | 7 +++++-- 3 files changed, 10 insertions(+), 6 deletions(-) diff --git a/docs/cli-options.rst b/docs/cli-options.rst index 88b4408..f8a187f 100644 --- a/docs/cli-options.rst +++ b/docs/cli-options.rst @@ -258,7 +258,8 @@ How to Download - Profile posts, - Profile IGTV posts (:option:`--igtv`), - Profile tagged posts (:option:`--tagged`), - - Saved posts (``:saved``). + - Saved posts (``:saved``), + - Hashtags. This feature is enabled by default for targets where it is supported; :option:`--resume-prefix` only changes the name of the iterator files. diff --git a/instaloader/instaloader.py b/instaloader/instaloader.py index 9371c3f..31f8c90 100644 --- a/instaloader/instaloader.py +++ b/instaloader/instaloader.py @@ -1176,8 +1176,8 @@ class Instaloader: """Get Posts associated with a #hashtag. .. deprecated:: 4.4 - Use :meth:`Hashtag.get_posts`.""" - return Hashtag.from_name(self.context, hashtag).get_posts() + Use :meth:`Hashtag.get_posts_resumable`.""" + return Hashtag.from_name(self.context, hashtag).get_posts_resumable() def download_hashtag(self, hashtag: Union[Hashtag, str], max_count: Optional[int] = None, @@ -1213,7 +1213,7 @@ class Instaloader: self.download_hashtag_profilepic(hashtag) if posts: self.context.log("Retrieving pictures with hashtag #{}...".format(hashtag.name)) - self.posts_download_loop(hashtag.get_all_posts(), target, fast_update, post_filter, + self.posts_download_loop(hashtag.get_posts_resumable(), target, fast_update, post_filter, max_count=max_count) if self.save_metadata: json_filename = '{0}/{1}'.format(self.dirname_pattern.format(profile=target, diff --git a/instaloader/structures.py b/instaloader/structures.py index 11f1da7..eb638d0 100644 --- a/instaloader/structures.py +++ b/instaloader/structures.py @@ -1553,7 +1553,10 @@ class Hashtag: return self._metadata("media_count") def get_posts(self) -> Iterator[Post]: - """Yields the recent posts associated with this hashtag.""" + """Yields the recent posts associated with this hashtag. + + .. deprecated:: 4.9 + Use :meth:`Hashtag.get_posts_resumable` as this method may return incorrect results (:issue:`1457`)""" try: self._metadata("edge_hashtag_to_media", "edges") self._metadata("edge_hashtag_to_media", "page_info") @@ -1575,7 +1578,7 @@ class Hashtag: def get_all_posts(self) -> Iterator[Post]: """Yields all posts, i.e. all most recent posts and the top posts, in almost-chronological order.""" sorted_top_posts = iter(sorted(islice(self.get_top_posts(), 9), key=lambda p: p.date_utc, reverse=True)) - other_posts = self.get_posts() + other_posts = self.get_posts_resumable() next_top = next(sorted_top_posts, None) next_other = next(other_posts, None) while next_top is not None or next_other is not None: