Release of Version 4.5.2

Fixes #774.

instaloader/__init__.py

@@ -1,7 +1,7 @@
 """Download pictures (or videos) along with their captions and other metadata from Instagram."""
 
 
-__version__ = '4.5'
+__version__ = '4.5.2'
 
 
 try:

instaloader/instaloader.py

@@ -13,7 +13,7 @@ from functools import wraps
 from hashlib import md5
 from io import BytesIO
 from pathlib import Path
-from typing import Any, Callable, ContextManager, IO, Iterator, List, Optional, Set, Union, cast
+from typing import Any, Callable, IO, Iterator, List, Optional, Set, Union, cast
 
 import requests
 import urllib3  # type: ignore
@@ -470,13 +470,7 @@ class Instaloader:
 
     def test_login(self) -> Optional[str]:
         """Returns the Instagram username to which given :class:`requests.Session` object belongs, or None."""
-        data = self.context.graphql_query("d6f4427fbe92d846298cf93df0b937d3", {}).get('data', {}).get('user')
-        if data is not None:
-            profile = Profile.from_node(self.context, {'id': data.get('id'),
-                                                       'username': data.get('username'),
-                                                       'profile_pic_url_hd': data.get('profile_pic_url')})
-            return profile.username
-        return None
+        return self.context.test_login()
 
     def login(self, user: str, passwd: str) -> None:
         """Log in to instagram with given username and password and internally store session object.
@@ -519,26 +513,28 @@ class Instaloader:
 
         # Download the image(s) / video thumbnail and videos within sidecars if desired
         downloaded = True
-        if self.download_pictures:
-            if post.typename == 'GraphSidecar':
-                edge_number = 1
-                for sidecar_node in post.get_sidecar_nodes():
-                    # Download picture or video thumbnail
-                    if not sidecar_node.is_video or self.download_video_thumbnails is True:
+        if post.typename == 'GraphSidecar':
+            if self.download_pictures or self.download_videos:
+                for edge_number, sidecar_node in enumerate(post.get_sidecar_nodes(), start=1):
+                    if self.download_pictures and (not sidecar_node.is_video or self.download_video_thumbnails):
+                        # Download sidecar picture or video thumbnail (--no-pictures implies --no-video-thumbnails)
                         downloaded &= self.download_pic(filename=filename, url=sidecar_node.display_url,
                                                         mtime=post.date_local, filename_suffix=str(edge_number))
-                    # Additionally download video if available and desired
-                    if sidecar_node.is_video and self.download_videos is True:
+                    if sidecar_node.is_video and self.download_videos:
+                        # Download sidecar video if desired
                         downloaded &= self.download_pic(filename=filename, url=sidecar_node.video_url,
                                                         mtime=post.date_local, filename_suffix=str(edge_number))
-                    edge_number += 1
-            elif post.typename == 'GraphImage':
+        elif post.typename == 'GraphImage':
+            # Download picture
+            if self.download_pictures:
                 downloaded = self.download_pic(filename=filename, url=post.url, mtime=post.date_local)
-            elif post.typename == 'GraphVideo':
-                if self.download_video_thumbnails is True:
+        elif post.typename == 'GraphVideo':
+            # Download video thumbnail (--no-pictures implies --no-video-thumbnails)
+            if self.download_pictures and self.download_video_thumbnails:
+                with self.context.error_catcher("Video thumbnail of {}".format(post)):
                     downloaded = self.download_pic(filename=filename, url=post.url, mtime=post.date_local)
-            else:
-                self.context.error("Warning: {0} has unknown typename: {1}".format(post, post.typename))
+        else:
+            self.context.error("Warning: {0} has unknown typename: {1}".format(post, post.typename))
 
         # Save caption if desired
         metadata_string = _ArbitraryItemFormatter(post).format(self.post_metadata_txt_pattern).strip()
@@ -546,7 +542,7 @@ class Instaloader:
             self.save_caption(filename=filename, mtime=post.date_local, caption=metadata_string)
 
         # Download video if desired
-        if post.is_video and self.download_videos is True:
+        if post.is_video and self.download_videos:
             downloaded &= self.download_pic(filename=filename, url=post.video_url, mtime=post.date_local)
 
         # Download geotags if desired
@@ -554,11 +550,11 @@ class Instaloader:
             self.save_location(filename, post.location, post.date_local)
 
         # Update comments if desired
-        if self.download_comments is True:
+        if self.download_comments:
             self.update_comments(filename=filename, post=post)
 
         # Save metadata as JSON if desired.
-        if self.save_metadata is not False:
+        if self.save_metadata:
             self.save_metadata_json(filename, post)
 
         self.context.log()
@@ -861,7 +857,7 @@ class Instaloader:
         """
         self.context.log("Retrieving saved posts...")
         assert self.context.username is not None  # safe due to @_requires_login; required by typechecker
-        node_iterator = Profile.from_username(self.context, self.context.username).get_saved_posts()
+        node_iterator = Profile.own_profile(self.context).get_saved_posts()
         self.posts_download_loop(node_iterator, ":saved",
                                  fast_update, post_filter,
                                  max_count=max_count, total_count=node_iterator.count)
@@ -1113,11 +1109,11 @@ class Instaloader:
         def _error_raiser(_str):
             yield
 
-        error_handler = cast(Callable[[Optional[str]], ContextManager[None]],
-                             _error_raiser if raise_errors else self.context.error_catcher)
+        # error_handler type is Callable[[Optional[str]], ContextManager[None]] (not supported with Python 3.5.0..3.5.3)
+        error_handler = _error_raiser if raise_errors else self.context.error_catcher
 
         for profile in profiles:
-            with error_handler(profile.username):
+            with error_handler(profile.username):  # type: ignore # (ignore type for Python 3.5 support)
                 profile_name = profile.username
 
                 # Download profile picture

instaloader/instaloadercontext.py

@@ -78,9 +78,6 @@ class InstaloaderContext:
         # Cache profile from id (mapping from id to Profile)
         self.profile_id_cache = dict()           # type: Dict[int, Any]
 
-        # Cache profile from name (mapping from username to Profile)
-        self.profile_name_cache = dict()         # type: Dict[str, Any]
-
     @contextmanager
     def anonymous_copy(self):
         session = self._session
@@ -188,12 +185,7 @@ class InstaloaderContext:
         self.username = username
 
     def test_login(self) -> Optional[str]:
-        """Not meant to be used directly, use :meth:`Instaloader.test_login`.
-
-           .. deprecated:: 4.5
-              :meth:`Instaloader.test_login` now calls :meth:`InstaloaderContext.graphql_query` directly
-              without using this function.
-        """
+        """Not meant to be used directly, use :meth:`Instaloader.test_login`."""
         data = self.graphql_query("d6f4427fbe92d846298cf93df0b937d3", {})
         return data["data"]["user"]["username"] if data["data"]["user"] is not None else None
 
@@ -495,7 +487,7 @@ class InstaloaderContext:
                 shutil.copyfileobj(resp.raw, file)
             else:
                 file.write(resp)
-        os.rename(filename + '.temp', filename)
+        os.replace(filename + '.temp', filename)
 
     def get_raw(self, url: str, _attempt=1) -> requests.Response:
         """Downloads a file anonymously.
@@ -557,8 +549,8 @@ class RateController:
 
     def __init__(self, context: InstaloaderContext):
         self._context = context
-        self._graphql_query_timestamps = dict()  # type: Dict[str, List[float]]
-        self._graphql_earliest_next_request_time = 0.0
+        self._query_timestamps = dict()  # type: Dict[str, List[float]]
+        self._earliest_next_request_time = 0.0
 
     def sleep(self, secs: float):
         """Wait given number of seconds."""
@@ -567,38 +559,75 @@ class RateController:
         # pylint:disable=no-self-use
         time.sleep(secs)
 
-    def _dump_query_timestamps(self, current_time: float):
-        windows = [10, 11, 15, 20, 30, 60]
-        print("GraphQL requests:", file=sys.stderr)
-        for query_hash, times in self._graphql_query_timestamps.items():
-            print("  {}".format(query_hash), file=sys.stderr)
-            for window in windows:
-                reqs_in_sliding_window = sum(t > current_time - window * 60 for t in times)
-                print("    last {} minutes: {} requests".format(window, reqs_in_sliding_window), file=sys.stderr)
+    def _dump_query_timestamps(self, current_time: float, failed_query_type: str):
+        windows = [10, 11, 20, 22, 30, 60]
+        self._context.error("Requests within last {} minutes grouped by type:"
+                            .format('/'.join(str(w) for w in windows)),
+                            repeat_at_end=False)
+        for query_type, times in self._query_timestamps.items():
+            reqs_in_sliding_window = [sum(t > current_time - w * 60 for t in times) for w in windows]
+            self._context.error(" {} {:>32}: {}".format(
+                "*" if query_type == failed_query_type else " ",
+                query_type,
+                " ".join("{:4}".format(reqs) for reqs in reqs_in_sliding_window)
+            ), repeat_at_end=False)
 
     def count_per_sliding_window(self, query_type: str) -> int:
-        """Return how many GraphQL requests can be done within the sliding window."""
+        """Return how many requests can be done within the sliding window."""
         # Not static, to allow for the count_per_sliding_window to depend on context-inherent properties, such as
         # whether we are logged in.
-        # pylint:disable=no-self-use,unused-argument
-        return 200
+        # pylint:disable=no-self-use
+        return 100 if query_type in ['iphone', 'other'] else 200
+
+    def _reqs_in_sliding_window(self, query_type: Optional[str], current_time: float, window: float) -> List[float]:
+        if query_type is not None:
+            # timestamps of type query_type
+            relevant_timestamps = self._query_timestamps[query_type]
+        else:
+            # all GraphQL queries, i.e. not 'iphone' or 'other'
+            graphql_query_timestamps = filter(lambda tp: tp[0] not in ['iphone', 'other'],
+                                              self._query_timestamps.items())
+            relevant_timestamps = [t for times in (tp[1] for tp in graphql_query_timestamps) for t in times]
+        return list(filter(lambda t: t > current_time - window, relevant_timestamps))
 
     def query_waittime(self, query_type: str, current_time: float, untracked_queries: bool = False) -> float:
-        """Calculate time needed to wait before GraphQL query can be executed."""
-        sliding_window = 660
-        if query_type not in self._graphql_query_timestamps:
-            self._graphql_query_timestamps[query_type] = []
-        self._graphql_query_timestamps[query_type] = list(filter(lambda t: t > current_time - 60 * 60,
-                                                                 self._graphql_query_timestamps[query_type]))
-        reqs_in_sliding_window = list(filter(lambda t: t > current_time - sliding_window,
-                                             self._graphql_query_timestamps[query_type]))
-        count_per_sliding_window = self.count_per_sliding_window(query_type)
-        if len(reqs_in_sliding_window) < count_per_sliding_window and not untracked_queries:
-            return max(0.0, self._graphql_earliest_next_request_time - current_time)
-        next_request_time = min(reqs_in_sliding_window) + sliding_window + 6
-        if untracked_queries:
-            self._graphql_earliest_next_request_time = next_request_time
-        return max(next_request_time, self._graphql_earliest_next_request_time) - current_time
+        """Calculate time needed to wait before query can be executed."""
+        per_type_sliding_window = 660
+        if query_type not in self._query_timestamps:
+            self._query_timestamps[query_type] = []
+        self._query_timestamps[query_type] = list(filter(lambda t: t > current_time - 60 * 60,
+                                                         self._query_timestamps[query_type]))
+
+        def per_type_next_request_time():
+            reqs_in_sliding_window = self._reqs_in_sliding_window(query_type, current_time, per_type_sliding_window)
+            if len(reqs_in_sliding_window) < self.count_per_sliding_window(query_type):
+                return 0.0
+            else:
+                return min(reqs_in_sliding_window) + per_type_sliding_window + 6
+
+        def gql_accumulated_next_request_time():
+            if query_type in ['iphone', 'other']:
+                return 0.0
+            gql_accumulated_sliding_window = 600
+            gql_accumulated_max_count = 275
+            reqs_in_sliding_window = self._reqs_in_sliding_window(None, current_time, gql_accumulated_sliding_window)
+            if len(reqs_in_sliding_window) < gql_accumulated_max_count:
+                return 0.0
+            else:
+                return min(reqs_in_sliding_window) + gql_accumulated_sliding_window
+
+        def untracked_next_request_time():
+            if untracked_queries:
+                reqs_in_sliding_window = self._reqs_in_sliding_window(query_type, current_time, per_type_sliding_window)
+                self._earliest_next_request_time = min(reqs_in_sliding_window) + per_type_sliding_window + 6
+            return self._earliest_next_request_time
+
+        return max(0.0,
+                   max(
+                       per_type_next_request_time(),
+                       gql_accumulated_next_request_time(),
+                       untracked_next_request_time(),
+                   ) - current_time)
 
     def wait_before_query(self, query_type: str) -> None:
         """This method is called before a query to Instagram. It calls :meth:`RateController.sleep` to wait
@@ -610,24 +639,25 @@ class RateController:
                               .format(round(waittime), datetime.now() + timedelta(seconds=waittime)))
         if waittime > 0:
             self.sleep(waittime)
-        if query_type not in self._graphql_query_timestamps:
-            self._graphql_query_timestamps[query_type] = [time.monotonic()]
+        if query_type not in self._query_timestamps:
+            self._query_timestamps[query_type] = [time.monotonic()]
         else:
-            self._graphql_query_timestamps[query_type].append(time.monotonic())
+            self._query_timestamps[query_type].append(time.monotonic())
 
     def handle_429(self, query_type: str) -> None:
         """This method is called to handle a 429 Too Many Requests response. It calls :meth:`RateController.sleep` to
          wait until we can repeat the same request."""
-        text_for_429 = ("HTTP error code 429 was returned because too many queries occurred in the last time. "
-                        "Please do not use Instagram in your browser or run multiple instances of Instaloader "
-                        "in parallel.")
-        print(textwrap.fill(text_for_429), file=sys.stderr)
         current_time = time.monotonic()
         waittime = self.query_waittime(query_type, current_time, True)
         assert waittime >= 0
-        if waittime > 15:
-            self._context.log("The request will be retried in {} seconds, at {:%H:%M}."
-                              .format(round(waittime), datetime.now() + timedelta(seconds=waittime)))
-        self._dump_query_timestamps(current_time)
+        self._dump_query_timestamps(current_time, query_type)
+        text_for_429 = ("Instagram responded with HTTP error \"429 - Too Many Requests\". Please do not run multiple "
+                        "instances of Instaloader in parallel or within short sequence. Also, do not use any Instagram "
+                        "App while Instaloader is running.")
+        self._context.error(textwrap.fill(text_for_429), repeat_at_end=False)
+        if waittime > 1.5:
+            self._context.error("The request will be retried in {} seconds, at {:%H:%M}."
+                                .format(round(waittime), datetime.now() + timedelta(seconds=waittime)),
+                                repeat_at_end=False)
         if waittime > 0:
             self.sleep(waittime)

instaloader/nodeiterator.py

@@ -81,11 +81,13 @@ class NodeIterator(Iterator[T]):
         self._node_wrapper = node_wrapper
         self._query_variables = query_variables if query_variables is not None else {}
         self._query_referer = query_referer
-        self._data = first_data
         self._page_index = 0
         self._total_index = 0
-        self._best_before = (None if first_data is None else
-                             datetime.now() + NodeIterator._shelf_life)
+        if first_data is not None:
+            self._data = first_data
+            self._best_before = datetime.now() + NodeIterator._shelf_life
+        else:
+            self._data = self._query()
 
     def _query(self, after: Optional[str] = None) -> Dict:
         pagination_variables = {'first': NodeIterator._graphql_page_length}  # type: Dict[str, Any]
@@ -113,8 +115,6 @@ class NodeIterator(Iterator[T]):
         return self
 
     def __next__(self) -> T:
-        if self._data is None:
-            self._data = self._query()
         if self._page_index < len(self._data['edges']):
             node = self._data['edges'][self._page_index]['node']
             page_index, total_index = self._page_index, self._total_index
@@ -193,8 +193,12 @@ class NodeIterator(Iterator[T]):
                 self._query_referer != frozen.query_referer or
                 self._context.username != frozen.context_username):
             raise InvalidArgumentException("Mismatching resume information.")
+        if not frozen.best_before:
+            raise InvalidArgumentException("\"best before\" date missing.")
+        if frozen.remaining_data is None:
+            raise InvalidArgumentException("\"remaining_data\" missing.")
         self._total_index = frozen.total_index
-        self._best_before = datetime.fromtimestamp(frozen.best_before) if frozen.best_before else None
+        self._best_before = datetime.fromtimestamp(frozen.best_before)
         self._data = frozen.remaining_data
 
 

instaloader/structures.py

@@ -196,7 +196,7 @@ class Post:
                 # is contained.
                 # Note that we cannot use Profile.from_id() here since that would lead us into a recursion.
                 owner_struct = self._full_metadata['owner']
-            self._owner_profile = Profile.from_node(self._context, owner_struct)
+            self._owner_profile = Profile(self._context, owner_struct)
         return self._owner_profile
 
     @property
@@ -387,7 +387,7 @@ class Post:
             return PostCommentAnswer(id=int(node['id']),
                                      created_at_utc=datetime.utcfromtimestamp(node['created_at']),
                                      text=node['text'],
-                                     owner=Profile.from_node(self._context, node['owner']),
+                                     owner=Profile(self._context, node['owner']),
                                      likes_count=node.get('edge_liked_by', {}).get('count', 0))
 
         def _postcommentanswers(node):
@@ -442,13 +442,13 @@ class Post:
         likes_edges = self._field('edge_media_preview_like', 'edges')
         if self.likes == len(likes_edges):
             # If the Post's metadata already contains all likes, don't do GraphQL requests to obtain them
-            yield from (Profile.from_node(self._context, like['node']) for like in likes_edges)
+            yield from (Profile(self._context, like['node']) for like in likes_edges)
             return
         yield from NodeIterator(
             self._context,
             '1cb6ec562846122743b61e492c85999f',
             lambda d: d['data']['shortcode_media']['edge_liked_by'],
-            lambda n: Profile.from_node(self._context, n),
+            lambda n: Profile(self._context, n),
             {'shortcode': self.shortcode},
             'https://www.instagram.com/p/{0}/'.format(self.shortcode),
         )
@@ -474,7 +474,7 @@ class Post:
         .. versionadded:: 4.4
         """
         return ([] if not self.is_sponsored else
-                [Profile.from_node(self._context, edge['node']['sponsor']) for edge in
+                [Profile(self._context, edge['node']['sponsor']) for edge in
                  self._field('edge_media_to_sponsor_user', 'edges')])
 
     @property
@@ -537,17 +537,6 @@ class Profile:
             # if loaded from JSON with load_structure_from_file()
             self._iphone_struct_ = node['iphone_struct']
 
-    @classmethod
-    def from_node(cls, context: InstaloaderContext, node: Dict[str, Any]) -> 'Profile':
-        assert 'username' in node
-        if node['username'] in context.profile_name_cache:
-            return context.profile_name_cache[node['username']]
-        profile = cls(context, node)
-        if 'id' in node:
-            context.profile_id_cache[node['id']] = profile
-        context.profile_name_cache[profile.username] = profile
-        return profile
-
     @classmethod
     def from_username(cls, context: InstaloaderContext, username: str):
         """Create a Profile instance from a given username, raise exception if it does not exist.
@@ -559,11 +548,8 @@ class Profile:
         :raises: :class:`ProfileNotExistsException`
         """
         # pylint:disable=protected-access
-        if username.lower() in context.profile_name_cache:
-            return context.profile_name_cache[username.lower()]
-        profile = cls.from_node(context, {'username': username.lower()})
+        profile = cls(context, {'username': username.lower()})
         profile._obtain_metadata()  # to raise ProfileNotExistException now in case username is invalid
-        context.profile_id_cache[profile.userid] = profile
         return profile
 
     @classmethod
@@ -586,10 +572,23 @@ class Profile:
                                       'include_highlight_reels': False},
                                      rhx_gis=context.root_rhx_gis)['data']['user']
         if data:
-            return cls.from_node(context, data['reel']['owner'])
+            profile = cls(context, data['reel']['owner'])
         else:
             raise ProfileNotExistsException("No profile found, the user may have blocked you (ID: " +
                                             str(profile_id) + ").")
+        context.profile_id_cache[profile_id] = profile
+        return profile
+
+    @classmethod
+    def own_profile(cls, context: InstaloaderContext):
+        """Return own profile if logged-in.
+
+        :param context: :attr:`Instaloader.context`
+
+        .. versionadded:: 4.5.2"""
+        if not context.is_logged_in:
+            raise LoginRequiredException("--login required to access own profile.")
+        return cls(context, context.graphql_query("d6f4427fbe92d846298cf93df0b937d3", {})["data"]["user"])
 
     def _asdict(self):
         json_node = self._node.copy()
@@ -814,7 +813,6 @@ class Profile:
         if self.username != self._context.username:
             raise LoginRequiredException("--login={} required to get that profile's saved posts.".format(self.username))
 
-        self._obtain_metadata()
         return NodeIterator(
             self._context,
             'f883d95537fbcd400f466f63d42bd8a1',
@@ -822,7 +820,6 @@ class Profile:
             lambda n: Post(self._context, n),
             {'id': self.userid},
             'https://www.instagram.com/{0}/'.format(self.username),
-            self._metadata('edge_saved_media'),
         )
 
     def get_tagged_posts(self) -> NodeIterator[Post]:
@@ -872,7 +869,7 @@ class Profile:
             self._context,
             '37479f2b8209594dde7facb0d904896a',
             lambda d: d['data']['user']['edge_followed_by'],
-            lambda n: Profile.from_node(self._context, n),
+            lambda n: Profile(self._context, n),
             {'id': str(self.userid)},
             'https://www.instagram.com/{0}/'.format(self.username),
         )
@@ -891,7 +888,7 @@ class Profile:
             self._context,
             '58712303d941c6855d4e888c5f0cd22f',
             lambda d: d['data']['user']['edge_follow'],
-            lambda n: Profile.from_node(self._context, n),
+            lambda n: Profile(self._context, n),
             {'id': str(self.userid)},
             'https://www.instagram.com/{0}/'.format(self.username),
         )
@@ -906,7 +903,7 @@ class Profile:
         if not self._context.is_logged_in:
             raise LoginRequiredException("--login required to get a profile's similar accounts.")
         self._obtain_metadata()
-        yield from (Profile.from_node(self._context, edge["node"]) for edge in
+        yield from (Profile(self._context, edge["node"]) for edge in
                     self._context.graphql_query("ad99dd9d3646cc3c0dda65debcd266a7",
                                                 {"user_id": str(self.userid), "include_chaining": True},
                                                 "https://www.instagram.com/{0}/"
@@ -1111,7 +1108,7 @@ class Story:
     def owner_profile(self) -> Profile:
         """:class:`Profile` instance of the story owner."""
         if not self._owner_profile:
-            self._owner_profile = Profile.from_node(self._context, self._node['user'])
+            self._owner_profile = Profile(self._context, self._node['user'])
         return self._owner_profile
 
     @property
@@ -1169,7 +1166,7 @@ class Highlight(Story):
     def owner_profile(self) -> Profile:
         """:class:`Profile` instance of the highlights' owner."""
         if not self._owner_profile:
-            self._owner_profile = Profile.from_node(self._context, self._node['owner'])
+            self._owner_profile = Profile(self._context, self._node['owner'])
         return self._owner_profile
 
     @property
@@ -1400,7 +1397,7 @@ class TopSearchResults:
             user_node = user['user']
             if 'pk' in user_node:
                 user_node['id'] = user_node['pk']
-            yield Profile.from_node(self._context, user_node)
+            yield Profile(self._context, user_node)
 
     def get_prefixed_usernames(self) -> Iterator[str]:
         """
@@ -1493,7 +1490,7 @@ def load_structure_from_file(context: InstaloaderContext, filename: str) -> Json
         if node_type == "Post":
             return Post(context, json_structure['node'])
         elif node_type == "Profile":
-            return Profile.from_node(context, json_structure['node'])
+            return Profile(context, json_structure['node'])
         elif node_type == "StoryItem":
             return StoryItem(context, json_structure['node'])
         elif node_type == "Hashtag":