Cache Profiles on creation and create one at test_login()

instaloader/__init__.py

@@ -1,7 +1,7 @@
 """Download pictures (or videos) along with their captions and other metadata from Instagram."""
 
 
-__version__ = '4.5.1'
+__version__ = '4.5'
 
 
 try:

instaloader/instaloader.py

@@ -13,7 +13,7 @@ from functools import wraps
 from hashlib import md5
 from io import BytesIO
 from pathlib import Path
-from typing import Any, Callable, IO, Iterator, List, Optional, Set, Union, cast
+from typing import Any, Callable, ContextManager, IO, Iterator, List, Optional, Set, Union, cast
 
 import requests
 import urllib3  # type: ignore
@@ -470,7 +470,13 @@ class Instaloader:
 
     def test_login(self) -> Optional[str]:
         """Returns the Instagram username to which given :class:`requests.Session` object belongs, or None."""
-        return self.context.test_login()
+        data = self.context.graphql_query("d6f4427fbe92d846298cf93df0b937d3", {}).get('data', {}).get('user')
+        if data is not None:
+            profile = Profile.from_node(self.context, {'id': data.get('id'),
+                                                       'username': data.get('username'),
+                                                       'profile_pic_url_hd': data.get('profile_pic_url')})
+            return profile.username
+        return None
 
     def login(self, user: str, passwd: str) -> None:
         """Log in to instagram with given username and password and internally store session object.
@@ -513,27 +519,26 @@ class Instaloader:
 
         # Download the image(s) / video thumbnail and videos within sidecars if desired
         downloaded = True
-        if post.typename == 'GraphSidecar':
-            for edge_number, sidecar_node in enumerate(post.get_sidecar_nodes(), start=1):
-                if self.download_pictures and (not sidecar_node.is_video or self.download_video_thumbnails):
-                    # Download sidecar picture or video thumbnail (--no-pictures implies --no-video-thumbnails)
-                    downloaded &= self.download_pic(filename=filename, url=sidecar_node.display_url,
-                                                    mtime=post.date_local, filename_suffix=str(edge_number))
-                if sidecar_node.is_video and self.download_videos:
-                    # Download sidecar video if desired
-                    downloaded &= self.download_pic(filename=filename, url=sidecar_node.video_url,
-                                                    mtime=post.date_local, filename_suffix=str(edge_number))
-        elif post.typename == 'GraphImage':
-            # Download picture
-            if self.download_pictures:
+        if self.download_pictures:
+            if post.typename == 'GraphSidecar':
+                edge_number = 1
+                for sidecar_node in post.get_sidecar_nodes():
+                    # Download picture or video thumbnail
+                    if not sidecar_node.is_video or self.download_video_thumbnails is True:
+                        downloaded &= self.download_pic(filename=filename, url=sidecar_node.display_url,
+                                                        mtime=post.date_local, filename_suffix=str(edge_number))
+                    # Additionally download video if available and desired
+                    if sidecar_node.is_video and self.download_videos is True:
+                        downloaded &= self.download_pic(filename=filename, url=sidecar_node.video_url,
+                                                        mtime=post.date_local, filename_suffix=str(edge_number))
+                    edge_number += 1
+            elif post.typename == 'GraphImage':
                 downloaded = self.download_pic(filename=filename, url=post.url, mtime=post.date_local)
-        elif post.typename == 'GraphVideo':
-            # Download video thumbnail (--no-pictures implies --no-video-thumbnails)
-            if self.download_pictures and self.download_video_thumbnails:
-                with self.context.error_catcher("Video thumbnail of {}".format(post)):
+            elif post.typename == 'GraphVideo':
+                if self.download_video_thumbnails is True:
                     downloaded = self.download_pic(filename=filename, url=post.url, mtime=post.date_local)
-        else:
-            self.context.error("Warning: {0} has unknown typename: {1}".format(post, post.typename))
+            else:
+                self.context.error("Warning: {0} has unknown typename: {1}".format(post, post.typename))
 
         # Save caption if desired
         metadata_string = _ArbitraryItemFormatter(post).format(self.post_metadata_txt_pattern).strip()
@@ -541,7 +546,7 @@ class Instaloader:
             self.save_caption(filename=filename, mtime=post.date_local, caption=metadata_string)
 
         # Download video if desired
-        if post.is_video and self.download_videos:
+        if post.is_video and self.download_videos is True:
             downloaded &= self.download_pic(filename=filename, url=post.video_url, mtime=post.date_local)
 
         # Download geotags if desired
@@ -549,11 +554,11 @@ class Instaloader:
             self.save_location(filename, post.location, post.date_local)
 
         # Update comments if desired
-        if self.download_comments:
+        if self.download_comments is True:
             self.update_comments(filename=filename, post=post)
 
         # Save metadata as JSON if desired.
-        if self.save_metadata:
+        if self.save_metadata is not False:
             self.save_metadata_json(filename, post)
 
         self.context.log()
@@ -1108,11 +1113,11 @@ class Instaloader:
         def _error_raiser(_str):
             yield
 
-        # error_handler type is Callable[[Optional[str]], ContextManager[None]] (not supported with Python 3.5.0..3.5.3)
-        error_handler = _error_raiser if raise_errors else self.context.error_catcher
+        error_handler = cast(Callable[[Optional[str]], ContextManager[None]],
+                             _error_raiser if raise_errors else self.context.error_catcher)
 
         for profile in profiles:
-            with error_handler(profile.username):  # type: ignore # (ignore type for Python 3.5 support)
+            with error_handler(profile.username):
                 profile_name = profile.username
 
                 # Download profile picture

instaloader/instaloadercontext.py

@@ -78,6 +78,9 @@ class InstaloaderContext:
         # Cache profile from id (mapping from id to Profile)
         self.profile_id_cache = dict()           # type: Dict[int, Any]
 
+        # Cache profile from name (mapping from username to Profile)
+        self.profile_name_cache = dict()         # type: Dict[str, Any]
+
     @contextmanager
     def anonymous_copy(self):
         session = self._session
@@ -185,7 +188,12 @@ class InstaloaderContext:
         self.username = username
 
     def test_login(self) -> Optional[str]:
-        """Not meant to be used directly, use :meth:`Instaloader.test_login`."""
+        """Not meant to be used directly, use :meth:`Instaloader.test_login`.
+
+           .. deprecated:: 4.5
+              :meth:`Instaloader.test_login` now calls :meth:`InstaloaderContext.graphql_query` directly
+              without using this function.
+        """
         data = self.graphql_query("d6f4427fbe92d846298cf93df0b937d3", {})
         return data["data"]["user"]["username"] if data["data"]["user"] is not None else None
 
@@ -487,7 +495,7 @@ class InstaloaderContext:
                 shutil.copyfileobj(resp.raw, file)
             else:
                 file.write(resp)
-        os.replace(filename + '.temp', filename)
+        os.rename(filename + '.temp', filename)
 
     def get_raw(self, url: str, _attempt=1) -> requests.Response:
         """Downloads a file anonymously.
@@ -559,18 +567,14 @@ class RateController:
         # pylint:disable=no-self-use
         time.sleep(secs)
 
-    def _dump_query_timestamps(self, current_time: float, failed_query_type: str):
+    def _dump_query_timestamps(self, current_time: float):
         windows = [10, 11, 15, 20, 30, 60]
-        self._context.error("Requests within last {} minutes grouped by type:"
-                            .format('/'.join(str(w) for w in windows)),
-                            repeat_at_end=False)
-        for query_type, times in self._graphql_query_timestamps.items():
-            reqs_in_sliding_window = [sum(t > current_time - w * 60 for t in times) for w in windows]
-            self._context.error(" {} {:>32}: {}".format(
-                "*" if query_type == failed_query_type else " ",
-                query_type,
-                " ".join("{:4}".format(reqs) for reqs in reqs_in_sliding_window)
-            ), repeat_at_end=False)
+        print("GraphQL requests:", file=sys.stderr)
+        for query_hash, times in self._graphql_query_timestamps.items():
+            print("  {}".format(query_hash), file=sys.stderr)
+            for window in windows:
+                reqs_in_sliding_window = sum(t > current_time - window * 60 for t in times)
+                print("    last {} minutes: {} requests".format(window, reqs_in_sliding_window), file=sys.stderr)
 
     def count_per_sliding_window(self, query_type: str) -> int:
         """Return how many GraphQL requests can be done within the sliding window."""
@@ -614,17 +618,16 @@ class RateController:
     def handle_429(self, query_type: str) -> None:
         """This method is called to handle a 429 Too Many Requests response. It calls :meth:`RateController.sleep` to
          wait until we can repeat the same request."""
+        text_for_429 = ("HTTP error code 429 was returned because too many queries occurred in the last time. "
+                        "Please do not use Instagram in your browser or run multiple instances of Instaloader "
+                        "in parallel.")
+        print(textwrap.fill(text_for_429), file=sys.stderr)
         current_time = time.monotonic()
         waittime = self.query_waittime(query_type, current_time, True)
         assert waittime >= 0
-        self._dump_query_timestamps(current_time, query_type)
-        text_for_429 = ("Instagram responded with HTTP error \"429 - Too Many Requests\". Please do not run multiple "
-                        "instances of Instaloader in parallel or within short sequence. Also, do not use any Instagram "
-                        "App while Instaloader is running.")
-        self._context.error(textwrap.fill(text_for_429), repeat_at_end=False)
-        if waittime > 1.5:
-            self._context.error("The request will be retried in {} seconds, at {:%H:%M}."
-                                .format(round(waittime), datetime.now() + timedelta(seconds=waittime)),
-                                repeat_at_end=False)
+        if waittime > 15:
+            self._context.log("The request will be retried in {} seconds, at {:%H:%M}."
+                              .format(round(waittime), datetime.now() + timedelta(seconds=waittime)))
+        self._dump_query_timestamps(current_time)
         if waittime > 0:
             self.sleep(waittime)

instaloader/structures.py

@@ -196,7 +196,7 @@ class Post:
                 # is contained.
                 # Note that we cannot use Profile.from_id() here since that would lead us into a recursion.
                 owner_struct = self._full_metadata['owner']
-            self._owner_profile = Profile(self._context, owner_struct)
+            self._owner_profile = Profile.from_node(self._context, owner_struct)
         return self._owner_profile
 
     @property
@@ -387,7 +387,7 @@ class Post:
             return PostCommentAnswer(id=int(node['id']),
                                      created_at_utc=datetime.utcfromtimestamp(node['created_at']),
                                      text=node['text'],
-                                     owner=Profile(self._context, node['owner']),
+                                     owner=Profile.from_node(self._context, node['owner']),
                                      likes_count=node.get('edge_liked_by', {}).get('count', 0))
 
         def _postcommentanswers(node):
@@ -442,13 +442,13 @@ class Post:
         likes_edges = self._field('edge_media_preview_like', 'edges')
         if self.likes == len(likes_edges):
             # If the Post's metadata already contains all likes, don't do GraphQL requests to obtain them
-            yield from (Profile(self._context, like['node']) for like in likes_edges)
+            yield from (Profile.from_node(self._context, like['node']) for like in likes_edges)
             return
         yield from NodeIterator(
             self._context,
             '1cb6ec562846122743b61e492c85999f',
             lambda d: d['data']['shortcode_media']['edge_liked_by'],
-            lambda n: Profile(self._context, n),
+            lambda n: Profile.from_node(self._context, n),
             {'shortcode': self.shortcode},
             'https://www.instagram.com/p/{0}/'.format(self.shortcode),
         )
@@ -474,7 +474,7 @@ class Post:
         .. versionadded:: 4.4
         """
         return ([] if not self.is_sponsored else
-                [Profile(self._context, edge['node']['sponsor']) for edge in
+                [Profile.from_node(self._context, edge['node']['sponsor']) for edge in
                  self._field('edge_media_to_sponsor_user', 'edges')])
 
     @property
@@ -537,6 +537,17 @@ class Profile:
             # if loaded from JSON with load_structure_from_file()
             self._iphone_struct_ = node['iphone_struct']
 
+    @classmethod
+    def from_node(cls, context: InstaloaderContext, node: Dict[str, Any]) -> 'Profile':
+        assert 'username' in node
+        if node['username'] in context.profile_name_cache:
+            return context.profile_name_cache[node['username']]
+        profile = cls(context, node)
+        if 'id' in node:
+            context.profile_id_cache[node['id']] = profile
+        context.profile_name_cache[profile.username] = profile
+        return profile
+
     @classmethod
     def from_username(cls, context: InstaloaderContext, username: str):
         """Create a Profile instance from a given username, raise exception if it does not exist.
@@ -548,8 +559,11 @@ class Profile:
         :raises: :class:`ProfileNotExistsException`
         """
         # pylint:disable=protected-access
-        profile = cls(context, {'username': username.lower()})
+        if username.lower() in context.profile_name_cache:
+            return context.profile_name_cache[username.lower()]
+        profile = cls.from_node(context, {'username': username.lower()})
         profile._obtain_metadata()  # to raise ProfileNotExistException now in case username is invalid
+        context.profile_id_cache[profile.userid] = profile
         return profile
 
     @classmethod
@@ -572,12 +586,10 @@ class Profile:
                                       'include_highlight_reels': False},
                                      rhx_gis=context.root_rhx_gis)['data']['user']
         if data:
-            profile = cls(context, data['reel']['owner'])
+            return cls.from_node(context, data['reel']['owner'])
         else:
             raise ProfileNotExistsException("No profile found, the user may have blocked you (ID: " +
                                             str(profile_id) + ").")
-        context.profile_id_cache[profile_id] = profile
-        return profile
 
     def _asdict(self):
         json_node = self._node.copy()
@@ -860,7 +872,7 @@ class Profile:
             self._context,
             '37479f2b8209594dde7facb0d904896a',
             lambda d: d['data']['user']['edge_followed_by'],
-            lambda n: Profile(self._context, n),
+            lambda n: Profile.from_node(self._context, n),
             {'id': str(self.userid)},
             'https://www.instagram.com/{0}/'.format(self.username),
         )
@@ -879,7 +891,7 @@ class Profile:
             self._context,
             '58712303d941c6855d4e888c5f0cd22f',
             lambda d: d['data']['user']['edge_follow'],
-            lambda n: Profile(self._context, n),
+            lambda n: Profile.from_node(self._context, n),
             {'id': str(self.userid)},
             'https://www.instagram.com/{0}/'.format(self.username),
         )
@@ -894,7 +906,7 @@ class Profile:
         if not self._context.is_logged_in:
             raise LoginRequiredException("--login required to get a profile's similar accounts.")
         self._obtain_metadata()
-        yield from (Profile(self._context, edge["node"]) for edge in
+        yield from (Profile.from_node(self._context, edge["node"]) for edge in
                     self._context.graphql_query("ad99dd9d3646cc3c0dda65debcd266a7",
                                                 {"user_id": str(self.userid), "include_chaining": True},
                                                 "https://www.instagram.com/{0}/"
@@ -1099,7 +1111,7 @@ class Story:
     def owner_profile(self) -> Profile:
         """:class:`Profile` instance of the story owner."""
         if not self._owner_profile:
-            self._owner_profile = Profile(self._context, self._node['user'])
+            self._owner_profile = Profile.from_node(self._context, self._node['user'])
         return self._owner_profile
 
     @property
@@ -1157,7 +1169,7 @@ class Highlight(Story):
     def owner_profile(self) -> Profile:
         """:class:`Profile` instance of the highlights' owner."""
         if not self._owner_profile:
-            self._owner_profile = Profile(self._context, self._node['owner'])
+            self._owner_profile = Profile.from_node(self._context, self._node['owner'])
         return self._owner_profile
 
     @property
@@ -1388,7 +1400,7 @@ class TopSearchResults:
             user_node = user['user']
             if 'pk' in user_node:
                 user_node['id'] = user_node['pk']
-            yield Profile(self._context, user_node)
+            yield Profile.from_node(self._context, user_node)
 
     def get_prefixed_usernames(self) -> Iterator[str]:
         """
@@ -1481,7 +1493,7 @@ def load_structure_from_file(context: InstaloaderContext, filename: str) -> Json
         if node_type == "Post":
             return Post(context, json_structure['node'])
         elif node_type == "Profile":
-            return Profile(context, json_structure['node'])
+            return Profile.from_node(context, json_structure['node'])
         elif node_type == "StoryItem":
             return StoryItem(context, json_structure['node'])
         elif node_type == "Hashtag":