Release of version 4.1.1

If Instaloader was called with --fast-update and encountered a post where
nothing has been downloaded because of flags that disable download of certain
types of media, the download was aborted. This behavior is now fixed.

Closes #183.

docs/basic-usage.rst

@@ -192,11 +192,6 @@ where the attributes from :class:`Post` or
 Id est, the following attributes can be used with both
 :option:`--post-filter` and :option:`--storyitem-filter`:
 
-- :attr:`~Post.is_video` (bool)
-   Post/StoryItem is a video. For example, you may skip videos::
-
-      instaloader --post-filter="not is_video" target
-
 - :attr:`~Post.owner_username` (str), :attr:`~Post.owner_id` (int)
    Owner profile username / userid.
 
@@ -206,6 +201,15 @@ Id est, the following attributes can be used with both
 
       instaloader --post-filter="date_utc <= datetime(2018, 5, 31)" target
 
+- :attr:`~Post.is_video` (bool)
+   Post/StoryItem is a video. For example, you may skip videos::
+
+      instaloader --post-filter="not is_video" target
+
+   This is not the same as :option:`--no-videos` and
+   :option:`--no-video-thumbnails`, since sidecar posts (posts that contain
+   multiple pictures/videos in one post) have this attribute set to False.
+
 As :option:`--post-filter`, the following attributes can be used additionally:
 
 - :attr:`~Post.viewer_has_liked` (bool)

docs/codesnippets.rst

@@ -14,6 +14,9 @@ Here we present code examples that use the :ref:`python-module-instaloader` for
 more advanced Instagram downloading or metadata mining than what is possible
 with the Instaloader command line interface.
 
+The scripts presented here can be downloaded from our source tree:
+`instaloader/docs/codesnippets/ <https://github.com/instaloader/instaloader/tree/master/docs/codesnippets>`__
+
 .. For each code snippet:
    - title
    - brief description of what it does / motivation / how it works

docs/codesnippets/README.md

@@ -0,0 +1,4 @@
+# Advanced Instaloader Examples
+
+This directory contains the scripts that are presented at:
+https://instaloader.github.io/codesnippets.html

instaloader/__init__.py

@@ -1,7 +1,7 @@
 """Download pictures (or videos) along with their captions and other metadata from Instagram."""
 
 
-__version__ = '4.1a1'
+__version__ = '4.1.1'
 
 
 try:

instaloader/instaloader.py

@@ -338,6 +338,12 @@ class Instaloader:
         :raises ConnectionException: If connection to Instagram failed."""
         self.context.login(user, passwd)
 
+    def format_filename(self, item: Union[Post, StoryItem], target: Optional[str] = None):
+        """Format filename of a :class:`Post` or :class:`StoryItem` according to ``filename-pattern`` parameter.
+
+        .. versionadded:: 4.1"""
+        return _PostPathFormatter(item).format(self.filename_pattern, target=target)
+
     def download_post(self, post: Post, target: str) -> bool:
         """
         Download everything associated with one instagram post node, i.e. picture, caption and video.
@@ -348,22 +354,22 @@ class Instaloader:
         """
 
         dirname = _PostPathFormatter(post).format(self.dirname_pattern, target=target)
-        filename = dirname + '/' + _PostPathFormatter(post).format(self.filename_pattern, target=target)
+        filename = dirname + '/' + self.format_filename(post, target=target)
         os.makedirs(os.path.dirname(filename), exist_ok=True)
 
         # Download the image(s) / video thumbnail and videos within sidecars if desired
-        downloaded = False
+        downloaded = True
         if self.download_pictures:
             if post.typename == 'GraphSidecar':
                 edge_number = 1
                 for sidecar_node in post.get_sidecar_nodes():
                     # Download picture or video thumbnail
                     if not sidecar_node.is_video or self.download_video_thumbnails is True:
-                        downloaded |= self.download_pic(filename=filename, url=sidecar_node.display_url,
+                        downloaded &= self.download_pic(filename=filename, url=sidecar_node.display_url,
                                                         mtime=post.date_local, filename_suffix=str(edge_number))
                     # Additionally download video if available and desired
                     if sidecar_node.is_video and self.download_videos is True:
-                        downloaded |= self.download_pic(filename=filename, url=sidecar_node.video_url,
+                        downloaded &= self.download_pic(filename=filename, url=sidecar_node.video_url,
                                                         mtime=post.date_local, filename_suffix=str(edge_number))
                     edge_number += 1
             elif post.typename == 'GraphImage':
@@ -381,7 +387,7 @@ class Instaloader:
 
         # Download video if desired
         if post.is_video and self.download_videos is True:
-            downloaded |= self.download_pic(filename=filename, url=post.video_url, mtime=post.date_local)
+            downloaded &= self.download_pic(filename=filename, url=post.video_url, mtime=post.date_local)
 
         # Download geotags if desired
         if self.download_geotags and post.location:
@@ -473,7 +479,7 @@ class Instaloader:
 
         date_local = item.date_local
         dirname = _PostPathFormatter(item).format(self.dirname_pattern, target=target)
-        filename = dirname + '/' + _PostPathFormatter(item).format(self.filename_pattern, target=target)
+        filename = dirname + '/' + self.format_filename(item, target=target)
         os.makedirs(os.path.dirname(filename), exist_ok=True)
         downloaded = False
         if not item.is_video or self.download_video_thumbnails is True:
@@ -693,7 +699,7 @@ class Instaloader:
 
         .. versionadded:: 4.1"""
         if target is None:
-            target = profile.username + ':tagged'
+            target = profile.username + '/:tagged'
         self.context.log("Retrieving tagged posts for profile {}.".format(profile.username))
         count = 1
         for post in profile.get_tagged_posts():
@@ -817,7 +823,7 @@ class Instaloader:
                         self.download_profilepic(profile)
 
                 # Catch some errors
-                if profile.is_private:
+                if profile.is_private and (tagged or highlights or posts):
                     if not self.context.is_logged_in:
                         raise LoginRequiredException("--login=USERNAME required.")
                     if not profile.followed_by_viewer and self.context.username != profile.username:

instaloader/instaloadercontext.py

@@ -57,7 +57,7 @@ class InstaloaderContext:
         self.quiet = quiet
         self.max_connection_attempts = max_connection_attempts
         self._graphql_page_length = 50
-        self.graphql_count_per_slidingwindow = graphql_count_per_slidingwindow or 20
+        self.graphql_count_per_slidingwindow = graphql_count_per_slidingwindow or 200
         self._root_rhx_gis = None
 
         # error log, filled with error() and printed at the end of Instaloader.main()
@@ -69,6 +69,9 @@ class InstaloaderContext:
         # Can be set to True for testing, disables supression of InstaloaderContext._error_catcher
         self.raise_all_errors = False
 
+        # Cache profile from id (mapping from id to Profile)
+        self.profile_id_cache = dict()
+
     @contextmanager
     def anonymous_copy(self):
         session = self._session
@@ -218,7 +221,7 @@ class InstaloaderContext:
     def _sleep(self):
         """Sleep a short time if self.sleep is set. Called before each request to instagram.com."""
         if self.sleep:
-            time.sleep(min(random.expovariate(0.6), 5.0))
+            time.sleep(min(random.expovariate(0.7), 5.0))
 
     def get_json(self, path: str, params: Dict[str, Any], host: str = 'www.instagram.com',
                  session: Optional[requests.Session] = None, _attempt=1) -> Dict[str, Any]:
@@ -303,6 +306,7 @@ class InstaloaderContext:
                 if isinstance(err, TooManyRequestsException):
                     print(textwrap.fill(text_for_429), file=sys.stderr)
                     if is_graphql_query:
+                        print("Made {} GraphQL requests.".format(len(self.query_timestamps)), file=sys.stderr)
                         waittime = graphql_query_waittime(untracked_queries=True)
                         if waittime > 0:
                             self.log('The request will be retried in {} seconds, at {:%H:%M}.'

instaloader/structures.py

@@ -402,6 +402,8 @@ class Profile:
         :param profile_id: userid
         :raises: :class:`ProfileNotExistsException`, :class:`ProfileHasNoPicsException`
         """
+        if profile_id in context.profile_id_cache:
+            return context.profile_id_cache[profile_id]
         data = context.graphql_query("472f257a40c653c64c666ce877d59d2b",
                                      {'id': str(profile_id), 'first': 1},
                                      rhx_gis=context.root_rhx_gis)['data']['user']
@@ -415,7 +417,9 @@ class Profile:
                 raise ProfileHasNoPicsException("Profile with ID {0}: no pics found.".format(str(profile_id)))
             else:
                 raise LoginRequiredException("Login required to determine username (ID: " + str(profile_id) + ").")
-        return Post(context, data['edges'][0]['node']).owner_profile
+        profile = Post(context, data['edges'][0]['node']).owner_profile
+        context.profile_id_cache[profile_id] = profile
+        return profile
 
     def _asdict(self):
         json_node = self._node.copy()

test/instaloader_unittests.py

@@ -63,6 +63,12 @@ class TestInstaloaderAnonymously(unittest.TestCase):
     def test_hashtag_download(self):
         self.L.download_hashtag(HASHTAG, NORMAL_MAX_COUNT)
 
+    def test_hashtag_paging(self):
+        for count, post in enumerate(self.L.get_hashtag_posts(HASHTAG)):
+            print(post)
+            if count == PAGING_MAX_COUNT:
+                break
+
     def test_get_id_by_username(self):
         self.assertEqual(PUBLIC_PROFILE_ID,
                          instaloader.Profile.from_username(self.L.context, PUBLIC_PROFILE).userid)