From b5aa268344a52c86e7aa15df594ac1b4d1a2e4c4 Mon Sep 17 00:00:00 2001 From: Lars Lindqvist Date: Fri, 3 Aug 2018 19:25:57 +0200 Subject: [PATCH 1/8] Fix owner_profile for Profile.get_tagged_posts() --- instaloader/structures.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/instaloader/structures.py b/instaloader/structures.py index 8d9312e..68db660 100644 --- a/instaloader/structures.py +++ b/instaloader/structures.py @@ -613,7 +613,7 @@ class Profile: def get_tagged_posts(self) -> Iterator[Post]: """Retrieve all posts where a profile is tagged.""" self._obtain_metadata() - yield from (Post(self._context, node, self) for node in + yield from (Post(self._context, node, self if int(node['owner']['id']) == self.userid else None) for node in self._context.graphql_node_list("e31a871f7301132ceaab56507a66bbb7", {'id': self.userid}, 'https://www.instagram.com/{0}/'.format(self.username), From 7f81985911721115566e5516bb34e7b2c80c2165 Mon Sep 17 00:00:00 2001 From: Alexander Graf Date: Sat, 4 Aug 2018 11:39:43 +0200 Subject: [PATCH 2/8] _ArbitraryItemFormatter: Allow chaining attributes Override get_value() rather than get_field(), to allow chaining of item attributes, such as --post-metadata-txt={owner_profile.profile_pic_url}. Motivated by #151. --- instaloader/instaloader.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/instaloader/instaloader.py b/instaloader/instaloader.py index 41d819f..36a2367 100644 --- a/instaloader/instaloader.py +++ b/instaloader/instaloader.py @@ -49,11 +49,11 @@ class _ArbitraryItemFormatter(string.Formatter): def __init__(self, item: Any): self._item = item - def get_field(self, field_name, args, kwargs): + def get_value(self, key, args, kwargs): """Override to substitute {ATTRIBUTE} by attributes of our _item.""" - if hasattr(self._item, field_name): - return self._item.__getattribute__(field_name), None - return super().get_field(field_name, args, kwargs) + if hasattr(self._item, key): + return getattr(self._item, key) + return super().get_value(key, args, kwargs) def format_field(self, value, format_spec): """Override :meth:`string.Formatter.format_field` to have our From bfa5d8ed41d2cd8e82c21a481a38db7f5d7564b3 Mon Sep 17 00:00:00 2001 From: Lars Lindqvist Date: Sun, 5 Aug 2018 03:29:24 +0200 Subject: [PATCH 3/8] Fix :stories 50a5330fecb1272487fdd600416354c83ef8922f breaks `:stories` if userids is None (non-iterable). --- instaloader/instaloader.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/instaloader/instaloader.py b/instaloader/instaloader.py index 36a2367..5c0d1ac 100644 --- a/instaloader/instaloader.py +++ b/instaloader/instaloader.py @@ -428,8 +428,10 @@ class Instaloader: if not userids: self.context.log("Retrieving all visible stories...") + else: + userids = [p if isinstance(p, int) else p.userid for p in userids] - for user_story in self.get_stories([p if isinstance(p, int) else p.userid for p in userids]): + for user_story in self.get_stories(userids): name = user_story.owner_username self.context.log("Retrieving stories from profile {}.".format(name)) totalcount = user_story.itemcount From a34d750152c5afc35c82737f2578de1979910dc9 Mon Sep 17 00:00:00 2001 From: Alexander Graf Date: Sun, 5 Aug 2018 07:58:25 +0200 Subject: [PATCH 4/8] rate handling: Show %H:%M when request is retried --- instaloader/instaloadercontext.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/instaloader/instaloadercontext.py b/instaloader/instaloadercontext.py index d7b65a3..7cf4700 100644 --- a/instaloader/instaloadercontext.py +++ b/instaloader/instaloadercontext.py @@ -9,6 +9,7 @@ import textwrap import time import urllib.parse from contextlib import contextmanager +from datetime import datetime, timedelta from typing import Any, Callable, Dict, Iterator, Optional import requests @@ -246,7 +247,8 @@ class InstaloaderContext: if is_graphql_query and not query_not_limited: waittime = graphql_query_waittime() if waittime > 0: - self.log('\nToo many queries in the last time. Need to wait {} seconds.'.format(waittime)) + self.log('\nToo many queries in the last time. Need to wait {} seconds, until {:%H:%M}.' + .format(waittime, datetime.now() + timedelta(seconds=waittime))) time.sleep(waittime) if self.query_timestamps is not None: self.query_timestamps.append(time.monotonic()) @@ -301,7 +303,8 @@ class InstaloaderContext: if is_graphql_query: waittime = graphql_query_waittime(untracked_queries=True) if waittime > 0: - self.log('The request will be retried in {} seconds.'.format(waittime)) + self.log('The request will be retried in {} seconds, at {:%H:%M}.' + .format(waittime, datetime.now() + timedelta(seconds=waittime))) time.sleep(waittime) self._sleep() return self.get_json(path=path, params=params, host=host, session=sess, _attempt=_attempt + 1) From a070ed42aebe0d2157058fdc4c632fe7e2d49028 Mon Sep 17 00:00:00 2001 From: Alexander Graf Date: Sun, 5 Aug 2018 08:06:53 +0200 Subject: [PATCH 5/8] get_stories(): Fetch in chunks of 100 users It seems there is a limit of how many user stories can be requested at once. Now, Instaloader.get_stories() chunks the stories request. Fixes #157 and fixes #134. --- instaloader/instaloader.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/instaloader/instaloader.py b/instaloader/instaloader.py index 5c0d1ac..d8e5ec6 100644 --- a/instaloader/instaloader.py +++ b/instaloader/instaloader.py @@ -396,17 +396,22 @@ class Instaloader: :param userids: List of user IDs to be processed in terms of downloading their stories, or None. """ - if userids is None: + if not userids: data = self.context.graphql_query("d15efd8c0c5b23f0ef71f18bf363c704", {"only_stories": True})["data"]["user"] if data is None: raise BadResponseException('Bad stories reel JSON.') userids = list(edge["node"]["id"] for edge in data["feed_reels_tray"]["edge_reels_tray_to_reel"]["edges"]) - stories = self.context.graphql_query("bf41e22b1c4ba4c9f31b844ebb7d9056", - {"reel_ids": userids, "precomposed_overlay": False})["data"] + def _userid_chunks(): + userids_per_query = 100 + for i in range(0, len(userids), userids_per_query): + yield userids[i:i + userids_per_query] - yield from (Story(self.context, media) for media in stories['reels_media']) + for userid_chunk in _userid_chunks(): + stories = self.context.graphql_query("bf41e22b1c4ba4c9f31b844ebb7d9056", + {"reel_ids": userid_chunk, "precomposed_overlay": False})["data"] + yield from (Story(self.context, media) for media in stories['reels_media']) @_requires_login def download_stories(self, From c634f21dccffae0c051f96b8efdfd3038558b5b1 Mon Sep 17 00:00:00 2001 From: Alexander Graf Date: Sun, 5 Aug 2018 09:01:22 +0200 Subject: [PATCH 6/8] Fix anon-retry for profiles that blocked viewer This was broken with 50a5330fecb1272487fdd600416354c83ef8922f. --- instaloader/__main__.py | 47 +++++++++++++++++++++++++++++------------ 1 file changed, 33 insertions(+), 14 deletions(-) diff --git a/instaloader/__main__.py b/instaloader/__main__.py index 8de51a5..1eaaf81 100644 --- a/instaloader/__main__.py +++ b/instaloader/__main__.py @@ -90,6 +90,7 @@ def _main(instaloader: Instaloader, targetlist: List[str], instaloader.context.log("Logged in as %s." % username) # Try block for KeyboardInterrupt (save session on ^C) profiles = set() + anonymous_retry_profiles = set() try: # Generate set of profiles, already downloading non-profile targets for target in targetlist: @@ -136,7 +137,28 @@ def _main(instaloader: Instaloader, targetlist: List[str], instaloader.download_saved_posts(fast_update=fast_update, max_count=max_count, post_filter=post_filter) else: - profiles.add(instaloader.check_profile_id(target)) + try: + profile = instaloader.check_profile_id(target) + if instaloader.context.is_logged_in and profile.has_blocked_viewer: + if not stories_only and not profile.is_private: + raise ProfileNotExistsException("{} blocked you; But she's public, " + "so we just download her anonymously.".format(target)) + else: + instaloader.context.error("{} blocked you.".format(target)) + else: + profiles.add(profile) + except ProfileNotExistsException as err: + # Not only our profile.has_blocked_viewer condition raises ProfileNotExistsException, + # check_profile_id() also does, since access to blocked profile may be responded with 404. + if instaloader.context.is_logged_in and not stories_only: + instaloader.context.log(err) + instaloader.context.log("Trying again anonymously, helps in case you are just blocked.") + with instaloader.anonymous_copy() as anonymous_loader: + with instaloader.context.error_catcher(): + anonymous_retry_profiles.add(anonymous_loader.check_profile_id(target)) + instaloader.context.log("Looks good.") + else: + raise if len(profiles) > 1: instaloader.context.log("Downloading {} profiles: {}".format(len(profiles), ' '.join([p.username for p in profiles]))) @@ -144,19 +166,16 @@ def _main(instaloader: Instaloader, targetlist: List[str], # Iterate through profiles list and download them for target in profiles: with instaloader.context.error_catcher(target): - try: - instaloader.download_profile(target, profile_pic, profile_pic_only, - fast_update, post_filter=post_filter) - except ProfileNotExistsException as err: - if instaloader.context.is_logged_in and not stories_only: - instaloader.context.log(err) - instaloader.context.log("Trying again anonymously, helps in case you are just blocked.") - with instaloader.anonymous_copy() as anonymous_loader: - with instaloader.context.error_catcher(): - anonymous_loader.download_profile(target, profile_pic, profile_pic_only, - fast_update, post_filter=post_filter) - else: - raise + instaloader.download_profile(target, profile_pic, profile_pic_only, + fast_update, post_filter=post_filter) + if anonymous_retry_profiles: + instaloader.context.log("Downloading anonymously: {}" + .format(' '.join([p.username for p in anonymous_retry_profiles]))) + with instaloader.anonymous_copy() as anonymous_loader: + for target in anonymous_retry_profiles: + with instaloader.context.error_catcher(target): + anonymous_loader.download_profile(target, profile_pic, profile_pic_only, + fast_update, post_filter=post_filter) if stories or stories_only: with instaloader.context.error_catcher("Download stories"): instaloader.context.log("Downloading stories") From 809b23a99665b13d9986abe5938acbd79cddf726 Mon Sep 17 00:00:00 2001 From: Alexander Graf Date: Sun, 5 Aug 2018 09:13:35 +0200 Subject: [PATCH 7/8] doc: Mark Profile.has_higlight_reels as deprecated --- instaloader/structures.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/instaloader/structures.py b/instaloader/structures.py index 68db660..da85d35 100644 --- a/instaloader/structures.py +++ b/instaloader/structures.py @@ -526,7 +526,8 @@ class Profile: @property def has_highlight_reels(self) -> bool: """ - Always returns `True` since :issue:`153`. + .. deprecated:: 4.0.6 + Always returns `True` since :issue:`153`. Before broken, this indicated whether the :class:`Profile` had available stories. """ From dcea0e9672df6aae096f9692c12d40e4200bace0 Mon Sep 17 00:00:00 2001 From: Alexander Graf Date: Sun, 5 Aug 2018 09:37:13 +0200 Subject: [PATCH 8/8] Release of version 4.0.7 --- instaloader/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/instaloader/__init__.py b/instaloader/__init__.py index 96f1b19..13ac807 100644 --- a/instaloader/__init__.py +++ b/instaloader/__init__.py @@ -1,7 +1,7 @@ """Download pictures (or videos) along with their captions and other metadata from Instagram.""" -__version__ = '4.0.6' +__version__ = '4.0.7' try: