Release of version 4.0.1

Increment graphql_count_per_slidingwindow to 25; Let context._sleep() choose
sleep times from upper-bounded exponential distribution with similar expected
value as before, letting wait times between queries seem more natural.

instaloader/__init__.py

@@ -1,7 +1,7 @@
 """Download pictures (or videos) along with their captions and other metadata from Instagram."""
 
 
-__version__ = '4.0'
+__version__ = '4.0.1'
 
 
 try:

instaloader/__main__.py

@@ -146,7 +146,7 @@ def _main(instaloader: Instaloader, targetlist: List[str],
                                                  stories, stories_only, post_filter=post_filter,
                                                  storyitem_filter=storyitem_filter)
                 except ProfileNotExistsException as err:
-                    if not instaloader.context.is_logged_in:
+                    if instaloader.context.is_logged_in:
                         instaloader.context.log(err)
                         instaloader.context.log("Trying again anonymously, helps in case you are just blocked.")
                         with instaloader.anonymous_copy() as anonymous_loader:

instaloader/instaloader.py

@@ -178,27 +178,28 @@ class Instaloader:
     def update_comments(self, filename: str, post: Post) -> None:
         def _postcomment_asdict(comment):
             return {'id': comment.id,
-                    'created_at': comment.created_at_utc.replace(tzinfo=timezone.utc).timestamp(),
+                    'created_at': int(comment.created_at_utc.replace(tzinfo=timezone.utc).timestamp()),
                     'text': comment.text,
                     'owner': comment.owner._asdict()}
         filename += '_comments.json'
         try:
-            comments = json.load(open(filename))
-        except FileNotFoundError:
+            with open(filename) as fp:
+                comments = json.load(fp)
+        except (FileNotFoundError, json.decoder.JSONDecodeError):
             comments = list()
         comments.extend(_postcomment_asdict(comment) for comment in post.get_comments())
         if comments:
+            comments_list = sorted(sorted(list(comments), key=lambda t: int(t['id'])),
+                                   key=lambda t: int(t['created_at']), reverse=True)
+            unique_comments_list = [comments_list[0]]
+            #for comment in comments_list:
+            #    if unique_comments_list[-1]['id'] != comment['id']:
+            #        unique_comments_list.append(comment)
+            #file.write(json.dumps(unique_comments_list, indent=4))
+            for x, y in zip(comments_list[:-1], comments_list[1:]):
+                if x['id'] != y['id']:
+                    unique_comments_list.append(y)
             with open(filename, 'w') as file:
-                comments_list = sorted(sorted(list(comments), key=lambda t: t['id']),
-                                       key=lambda t: t['created_at'], reverse=True)
-                unique_comments_list = [comments_list[0]]
-                #for comment in comments_list:
-                #    if unique_comments_list[-1]['id'] != comment['id']:
-                #        unique_comments_list.append(comment)
-                #file.write(json.dumps(unique_comments_list, indent=4))
-                for x, y in zip(comments_list[:-1], comments_list[1:]):
-                    if x['id'] != y['id']:
-                        unique_comments_list.append(y)
                 file.write(json.dumps(unique_comments_list, indent=4))
             self.context.log('comments', end=' ', flush=True)
 
@@ -698,9 +699,12 @@ class Instaloader:
 
         # Download stories, if requested
         if download_stories or download_stories_only:
-            with self.context.error_catcher("Download stories of {}".format(profile_name)):
-                self.download_stories(userids=[profile.userid], filename_target=profile_name, fast_update=fast_update,
-                                      storyitem_filter=storyitem_filter)
+            if profile.has_highlight_reel:
+                with self.context.error_catcher("Download stories of {}".format(profile_name)):
+                    self.download_stories(userids=[profile.userid], filename_target=profile_name,
+                                          fast_update=fast_update, storyitem_filter=storyitem_filter)
+            else:
+                self.context.log("{} does not have any stories.".format(profile_name))
         if download_stories_only:
             return
 

instaloader/instaloadercontext.py

@@ -56,6 +56,7 @@ class InstaloaderContext:
         self.quiet = quiet
         self.max_connection_attempts = max_connection_attempts
         self._graphql_page_length = 50
+        self._graphql_count_per_slidingwindow = 25
         self._root_rhx_gis = None
 
         # error log, filled with error() and printed at the end of Instaloader.main()
@@ -185,7 +186,7 @@ class InstaloaderContext:
     def _sleep(self):
         """Sleep a short time if self.sleep is set. Called before each request to instagram.com."""
         if self.sleep:
-            time.sleep(random.uniform(0.5, 3))
+            time.sleep(min(random.expovariate(0.6), 5.0))
 
     def get_json(self, path: str, params: Dict[str, Any], host: str = 'www.instagram.com',
                  session: Optional[requests.Session] = None, _attempt=1) -> Dict[str, Any]:
@@ -206,7 +207,7 @@ class InstaloaderContext:
                 return sliding_window if untracked_queries else 0
             current_time = time.monotonic()
             self.query_timestamps = list(filter(lambda t: t > current_time - sliding_window, self.query_timestamps))
-            if len(self.query_timestamps) < 20 and not untracked_queries:
+            if len(self.query_timestamps) < self._graphql_count_per_slidingwindow and not untracked_queries:
                 return 0
             return round(min(self.query_timestamps) + sliding_window - current_time) + 6
         is_graphql_query = 'query_hash' in params and 'graphql/query' in path

instaloader/structures.py

@@ -365,6 +365,8 @@ class Profile:
     def from_username(cls, context: InstaloaderContext, username: str):
         """Create a Profile instance from a given username, raise exception if it does not exist.
 
+        See also :meth:`Instaloader.check_profile_id`.
+
         :param context: :attr:`Instaloader.context`
         :param username: Username
         :raises: :class:`ProfileNotExistsException`
@@ -412,8 +414,8 @@ class Profile:
                 metadata = self._context.get_json('{}/'.format(self.username), params={})
                 self._node = metadata['entry_data']['ProfilePage'][0]['graphql']['user']
                 self._rhx_gis = metadata['rhx_gis']
-        except QueryReturnedNotFoundException:
-            raise ProfileNotExistsException('Profile {} does not exist.'.format(self.username))
+        except (QueryReturnedNotFoundException, KeyError) as err:
+            raise ProfileNotExistsException('Profile {} does not exist.'.format(self.username)) from err
 
     def _metadata(self, *keys) -> Any:
         try:
@@ -481,6 +483,10 @@ class Profile:
     def has_blocked_viewer(self) -> bool:
         return self._metadata('has_blocked_viewer')
 
+    @property
+    def has_highlight_reel(self) -> bool:
+        return self._metadata('has_highlight_reel')
+
     @property
     def has_requested_viewer(self) -> bool:
         return self._metadata('has_requested_viewer')