Fix obtaining post metadata when not logged-in

Fixes #960.

README.rst

@@ -29,7 +29,7 @@
 
 .. |downloads| image:: https://pepy.tech/badge/instaloader/month
    :alt: PyPI Download Count
-   :target: https://pepy.tech/project/instaloader
+   :target: https://pepy.tech/project/instaloader/month
 
 .. badges-end
 
@@ -84,9 +84,7 @@ To later **update your local copy** of that profiles, you may run
     instaloader --fast-update profile [profile ...]
 
 If ``--fast-update`` is given, Instaloader stops when arriving at the
-first already-downloaded picture.
-
-When updating profiles, Instaloader
+first already-downloaded picture. When updating profiles, Instaloader
 automatically **detects profile name changes** and renames the target directory
 accordingly.
 

docs/cli-options.rst

@@ -220,20 +220,12 @@ How to Download
    used to save the information to resume an interrupted download.  The default
    prefix is ``iterator``.
 
-   Resuming an interrupted download is supported for the following targets:
-    - Profile posts,
-    - Profile IGTV posts (:option:`--igtv`),
-    - Profile tagged posts (:option:`--tagged`),
-    - Saved posts (``:saved``).
-
-   This feature is enabled by default for targets where it is supported;
-   :option:`--resume-prefix` only changes the name of the iterator files.
-
-   To turn this feature off, use :option:`--no-resume`.
-
+   Resuming an interrupted download is supported for most, but not all targets.
    JSON files with resume information are always compressed, regardless of
    :option:`--no-compress-json`.
 
+   This feature is turned off entirely with :option:`--no-resume`.
+
    .. versionadded:: 4.5
 
 .. option:: --no-resume
@@ -246,7 +238,7 @@ How to Download
 .. option:: --user-agent USER_AGENT
 
    User Agent to use for HTTP requests. Per default, Instaloader pretends being
-   Chrome/88 on Linux.
+   Chrome/51.
 
 .. option:: --max-connection-attempts N
 

instaloader/__init__.py

@@ -1,7 +1,7 @@
 """Download pictures (or videos) along with their captions and other metadata from Instagram."""
 
 
-__version__ = '4.6.2'
+__version__ = '4.6'
 
 
 try:

instaloader/instaloader.py

@@ -563,20 +563,23 @@ class Instaloader:
                         post.get_sidecar_nodes(self.slide_start, self.slide_end),
                         start=post.mediacount if self.slide_start < 0 else self.slide_start + 1
                 ):
-                    suffix = str(edge_number)  # type: Optional[str]
-                    if '{filename}' in self.filename_pattern:
-                        suffix = None
                     if self.download_pictures and (not sidecar_node.is_video or self.download_video_thumbnails):
+                        suffix = str(edge_number)
+                        if '{filename}' in self.filename_pattern:
+                            suffix = ''
                         # pylint:disable=cell-var-from-loop
-                        sidecar_filename = self.__prepare_filename(filename_template, lambda: sidecar_node.display_url)
+                        filename = self.__prepare_filename(filename_template, lambda: sidecar_node.display_url)
                         # Download sidecar picture or video thumbnail (--no-pictures implies --no-video-thumbnails)
-                        downloaded &= self.download_pic(filename=sidecar_filename, url=sidecar_node.display_url,
+                        downloaded &= self.download_pic(filename=filename, url=sidecar_node.display_url,
                                                         mtime=post.date_local, filename_suffix=suffix)
                     if sidecar_node.is_video and self.download_videos:
+                        suffix = str(edge_number)
+                        if '{filename}' in self.filename_pattern:
+                            suffix = ''
                         # pylint:disable=cell-var-from-loop
-                        sidecar_filename = self.__prepare_filename(filename_template, lambda: sidecar_node.video_url)
+                        filename = self.__prepare_filename(filename_template, lambda: sidecar_node.video_url)
                         # Download sidecar video if desired
-                        downloaded &= self.download_pic(filename=sidecar_filename, url=sidecar_node.video_url,
+                        downloaded &= self.download_pic(filename=filename, url=sidecar_node.video_url,
                                                         mtime=post.date_local, filename_suffix=suffix)
         elif post.typename == 'GraphImage':
             # Download picture

instaloader/instaloadercontext.py

@@ -33,7 +33,7 @@ def copy_session(session: requests.Session, request_timeout: Optional[float] = N
 
 def default_user_agent() -> str:
     return 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 ' \
-           '(KHTML, like Gecko) Chrome/88.0.4324.96 Safari/537.36'
+           '(KHTML, like Gecko) Chrome/51.0.2704.79 Safari/537.36'
 
 
 class InstaloaderContext:
@@ -288,7 +288,7 @@ class InstaloaderContext:
     def do_sleep(self):
         """Sleep a short time if self.sleep is set. Called before each request to instagram.com."""
         if self.sleep:
-            time.sleep(min(random.expovariate(0.6), 15.0))
+            time.sleep(min(random.expovariate(0.7), 5.0))
 
     def get_json(self, path: str, params: Dict[str, Any], host: str = 'www.instagram.com',
                  session: Optional[requests.Session] = None, _attempt=1) -> Dict[str, Any]:
@@ -549,7 +549,6 @@ class RateController:
         self._context = context
         self._query_timestamps = dict()  # type: Dict[str, List[float]]
         self._earliest_next_request_time = 0.0
-        self._iphone_earliest_next_request_time = 0.0
 
     def sleep(self, secs: float):
         """Wait given number of seconds."""
@@ -560,7 +559,7 @@ class RateController:
 
     def _dump_query_timestamps(self, current_time: float, failed_query_type: str):
         windows = [10, 11, 20, 22, 30, 60]
-        self._context.error("Number of requests within last {} minutes grouped by type:"
+        self._context.error("Requests within last {} minutes grouped by type:"
                             .format('/'.join(str(w) for w in windows)),
                             repeat_at_end=False)
         for query_type, times in self._query_timestamps.items():
@@ -572,15 +571,11 @@ class RateController:
             ), repeat_at_end=False)
 
     def count_per_sliding_window(self, query_type: str) -> int:
-        """Return how many requests of the given type can be done within a sliding window of 11 minutes.
-
-        This is called by :meth:`RateController.query_waittime` and allows to simply customize wait times before queries
-        at query_type granularity. Consider overriding :meth:`RateController.query_waittime` directly if you need more
-        control."""
+        """Return how many requests can be done within the sliding window."""
         # Not static, to allow for the count_per_sliding_window to depend on context-inherent properties, such as
         # whether we are logged in.
         # pylint:disable=no-self-use
-        return 75 if query_type == 'other' else 200
+        return 75 if query_type in ['iphone', 'other'] else 200
 
     def _reqs_in_sliding_window(self, query_type: Optional[str], current_time: float, window: float) -> List[float]:
         if query_type is not None:
@@ -596,7 +591,6 @@ class RateController:
     def query_waittime(self, query_type: str, current_time: float, untracked_queries: bool = False) -> float:
         """Calculate time needed to wait before query can be executed."""
         per_type_sliding_window = 660
-        iphone_sliding_window = 1800
         if query_type not in self._query_timestamps:
             self._query_timestamps[query_type] = []
         self._query_timestamps[query_type] = list(filter(lambda t: t > current_time - 60 * 60,
@@ -622,43 +616,25 @@ class RateController:
 
         def untracked_next_request_time():
             if untracked_queries:
-                if query_type == "iphone":
-                    reqs_in_sliding_window = self._reqs_in_sliding_window(query_type, current_time,
-                                                                          iphone_sliding_window)
-                    self._iphone_earliest_next_request_time = min(reqs_in_sliding_window) + iphone_sliding_window + 18
-                else:
-                    reqs_in_sliding_window = self._reqs_in_sliding_window(query_type, current_time,
-                                                                          per_type_sliding_window)
-                    self._earliest_next_request_time = min(reqs_in_sliding_window) + per_type_sliding_window + 6
-            return max(self._iphone_earliest_next_request_time, self._earliest_next_request_time)
-
-        def iphone_next_request():
-            if query_type == "iphone":
-                reqs_in_sliding_window = self._reqs_in_sliding_window(query_type, current_time, iphone_sliding_window)
-                if len(reqs_in_sliding_window) >= 199:
-                    return min(reqs_in_sliding_window) + iphone_sliding_window + 18
-            return 0.0
+                reqs_in_sliding_window = self._reqs_in_sliding_window(query_type, current_time, per_type_sliding_window)
+                self._earliest_next_request_time = min(reqs_in_sliding_window) + per_type_sliding_window + 6
+            return self._earliest_next_request_time
 
         return max(0.0,
                    max(
                        per_type_next_request_time(),
                        gql_accumulated_next_request_time(),
                        untracked_next_request_time(),
-                       iphone_next_request(),
                    ) - current_time)
 
     def wait_before_query(self, query_type: str) -> None:
-        """This method is called before a query to Instagram.
-
-        It calls :meth:`RateController.query_waittime` to determine the time needed to wait and then calls
-        :meth:`RateController.sleep` to wait until the request can be made."""
+        """This method is called before a query to Instagram. It calls :meth:`RateController.sleep` to wait
+        until the request can be made."""
         waittime = self.query_waittime(query_type, time.monotonic(), False)
         assert waittime >= 0
         if waittime > 15:
-            formatted_waittime = ("{} seconds".format(round(waittime)) if waittime <= 666 else
-                                  "{} minutes".format(round(waittime / 60)))
-            self._context.log("\nToo many queries in the last time. Need to wait {}, until {:%H:%M}."
-                              .format(formatted_waittime, datetime.now() + timedelta(seconds=waittime)))
+            self._context.log("\nToo many queries in the last time. Need to wait {} seconds, until {:%H:%M}."
+                              .format(round(waittime), datetime.now() + timedelta(seconds=waittime)))
         if waittime > 0:
             self.sleep(waittime)
         if query_type not in self._query_timestamps:
@@ -667,10 +643,8 @@ class RateController:
             self._query_timestamps[query_type].append(time.monotonic())
 
     def handle_429(self, query_type: str) -> None:
-        """This method is called to handle a 429 Too Many Requests response.
-
-        It calls :meth:`RateController.query_waittime` to determine the time needed to wait and then calls
-        :meth:`RateController.sleep` to wait until we can repeat the same request."""
+        """This method is called to handle a 429 Too Many Requests response. It calls :meth:`RateController.sleep` to
+         wait until we can repeat the same request."""
         current_time = time.monotonic()
         waittime = self.query_waittime(query_type, current_time, True)
         assert waittime >= 0
@@ -680,10 +654,8 @@ class RateController:
                         "App while Instaloader is running.")
         self._context.error(textwrap.fill(text_for_429), repeat_at_end=False)
         if waittime > 1.5:
-            formatted_waittime = ("{} seconds".format(round(waittime)) if waittime <= 666 else
-                                  "{} minutes".format(round(waittime / 60)))
-            self._context.error("The request will be retried in {}, at {:%H:%M}."
-                                .format(formatted_waittime, datetime.now() + timedelta(seconds=waittime)),
+            self._context.error("The request will be retried in {} seconds, at {:%H:%M}."
+                                .format(round(waittime), datetime.now() + timedelta(seconds=waittime)),
                                 repeat_at_end=False)
         if waittime > 0:
             self.sleep(waittime)

instaloader/structures.py

@@ -141,11 +141,16 @@ class Post:
 
     def _obtain_metadata(self):
         if not self._full_metadata_dict:
-            pic_json = self._context.graphql_query(
-                '2b0673e0dc4580674a88d426fe00ea90',
-                {'shortcode': self.shortcode}
-            )
-            self._full_metadata_dict = pic_json['data']['shortcode_media']
+            if self._context.is_logged_in:
+                pic_json = self._context.graphql_query(
+                    '2b0673e0dc4580674a88d426fe00ea90',
+                    {'shortcode': self.shortcode}
+                )
+                self._full_metadata_dict = pic_json['data']['shortcode_media']
+            else:
+                # if not logged in, above graphql query is responded with a redirect to login page
+                pic_json = self._context.get_json("p/{0}/".format(self.shortcode), params={})
+                self._full_metadata_dict = pic_json['entry_data']['PostPage'][0]['graphql']['shortcode_media']
             if self._full_metadata_dict is None:
                 raise BadResponseException("Fetching Post metadata failed.")
             if self.shortcode != self._full_metadata_dict['shortcode']:
@@ -273,7 +278,7 @@ class Post:
         """
         if self.typename == 'GraphSidecar':
             edges = self._field('edge_sidecar_to_children', 'edges')
-            if any(edge['node']['is_video'] and 'video_url' not in edge['node'] for edge in edges):
+            if any(edge['node']['is_video'] for edge in edges):
                 # video_url is only present in full metadata, issue #558.
                 edges = self._full_metadata['edge_sidecar_to_children']['edges']
             if end < 0:
@@ -449,8 +454,16 @@ class Post:
             # Avoid doing additional requests if there are no comments
             return
 
-        comment_edges = self._field('edge_media_to_comment', 'edges')
-        answers_count = sum([edge['node'].get('edge_threaded_comments', {}).get('count', 0) for edge in comment_edges])
+        try:
+            # this seems to be the structure for logged-in accesses
+            comment_edges = self._field('edge_media_to_comment', 'edges')
+            answers_count = sum([edge['node'].get('edge_threaded_comments', {}).get('count', 0)
+                                 for edge in comment_edges])
+        except KeyError:
+            # ... and this for anonymous accesses
+            comment_edges = self._field('edge_media_to_parent_comment', 'edges')
+            answers_count = sum([edge['node']['edge_threaded_comments']['count']
+                                 for edge in comment_edges])
 
         if self.comments == len(comment_edges) + answers_count:
             # If the Post's metadata already contains all parent comments, don't do GraphQL requests to obtain them
@@ -835,7 +848,7 @@ class Profile:
         self._obtain_metadata()
         return NodeIterator(
             self._context,
-            '003056d32c2554def87228bc3fd9668a',
+            '472f257a40c653c64c666ce877d59d2b',
             lambda d: d['data']['user']['edge_owner_to_timeline_media'],
             lambda n: Post(self._context, n, self),
             {'id': self.userid},