Release of version 4.6.1

Avoid evaluating 'url' when preparing the filename if the
filename-pattern does not contain the "{filename}" token. This avoids
the URL to be queried even though it is not needed (e.g. with
--no-pictures).

.github/funding.yml

@@ -0,0 +1 @@
+custom: "https://instaloader.github.io/contributing.html#donations"

.github/pull_request_template.md

@@ -1,7 +1,8 @@
 
 
 <!--
-Please describe:
+Thanks for your willingness to contribute to Instaloader! Please briefly
+describe
 
 - A motivation for this change, e.g.
   - Fixes # .

docs/cli-options.rst

@@ -238,7 +238,7 @@ How to Download
 .. option:: --user-agent USER_AGENT
 
    User Agent to use for HTTP requests. Per default, Instaloader pretends being
-   Chrome/51.
+   Chrome/88 on Linux.
 
 .. option:: --max-connection-attempts N
 

docs/conf.py

@@ -65,7 +65,7 @@ extlinks = {
         'Issue #'
     ),
     'example': (
-        'https://raw.githubusercontent.com/instaloader/instaloader/v' + current_release + '/docs/codesnippets/%s',
+        'https://raw.githubusercontent.com/instaloader/instaloader/master/docs/codesnippets/%s',
         'Example '
     ),
 }

docs/contributing.rst

@@ -11,11 +11,19 @@ Instaloader's development is organized on
 `GitHub <https://github.com/instaloader/instaloader>`__, where Issues and Pull
 Requests are discussed.
 
+Contributing to an open source project is an unmatchable opportunity to improve
+and manifest programming skills. For issues that are good starting points to get
+involved in Instaloader development, as they do not require deep insight into
+Instaloader's code base, have a look at our
+`Issues labelled 'good first issue' <https://github.com/instaloader/instaloader/contribute>`__.
+
+This document covers several ways of how you can contribute back to Instaloader.
+
 Answering Questions
 -------------------
 
 The easiest way to help out is to answer questions. If you are interested in
-answering questions regarding Instaloader, good places to start are
+answering questions regarding Instaloader, good places to begin are
 
 - `Questions tagged 'instaloader' on Stack Overflow <https://stackoverflow.com/questions/tagged/instaloader>`__,
 - `Instaloader Issues labeled 'question' <https://github.com/instaloader/instaloader/issues?q=is%3Aissue+is%3Aopen+label%3Aquestion>`__.
@@ -81,8 +89,8 @@ Improvements of the Instaloader source or its documentation can be proposed as a
      make -C docs html
 
 - Feel free to create an issue to make sure someone from the Instaloader team
-  agrees that the change might be an improvement, and is happy with your basic
-  proposal, before working on a pull request.
+  agrees that the change might be an improvement, or if you want to discuss
+  your basic proposal, before working on a pull request.
 
 Proposing Features
 ------------------

instaloader/__init__.py

@@ -1,7 +1,7 @@
 """Download pictures (or videos) along with their captions and other metadata from Instagram."""
 
 
-__version__ = '4.6rc1'
+__version__ = '4.6.1'
 
 
 try:

instaloader/instaloader.py

@@ -524,11 +524,15 @@ class Instaloader:
         self.context.two_factor_login(two_factor_code)
 
     @staticmethod
-    def __prepare_filename(filename_template: str, url: str) -> str:
+    def __prepare_filename(filename_template: str, url: Callable[[], str]) -> str:
         """Replace filename token inside filename_template with url's filename and assure the directories exist.
 
         .. versionadded:: 4.6"""
-        filename = filename_template.replace("{filename}", os.path.splitext(os.path.basename(urlparse(url).path))[0])
+        if "{filename}" in filename_template:
+            filename = filename_template.replace("{filename}",
+                                                 os.path.splitext(os.path.basename(urlparse(url()).path))[0])
+        else:
+            filename = filename_template
         os.makedirs(os.path.dirname(filename), exist_ok=True)
         return filename
 
@@ -549,7 +553,7 @@ class Instaloader:
 
         dirname = _PostPathFormatter(post).format(self.dirname_pattern, target=target)
         filename_template = os.path.join(dirname, self.format_filename(post, target=target))
-        filename = self.__prepare_filename(filename_template, post.url)
+        filename = self.__prepare_filename(filename_template, lambda: post.url)
 
         # Download the image(s) / video thumbnail and videos within sidecars if desired
         downloaded = True
@@ -563,7 +567,8 @@ class Instaloader:
                         suffix = str(edge_number)
                         if '{filename}' in self.filename_pattern:
                             suffix = ''
-                        filename = self.__prepare_filename(filename_template, sidecar_node.display_url)
+                        # pylint:disable=cell-var-from-loop
+                        filename = self.__prepare_filename(filename_template, lambda: sidecar_node.display_url)
                         # Download sidecar picture or video thumbnail (--no-pictures implies --no-video-thumbnails)
                         downloaded &= self.download_pic(filename=filename, url=sidecar_node.display_url,
                                                         mtime=post.date_local, filename_suffix=suffix)
@@ -571,7 +576,8 @@ class Instaloader:
                         suffix = str(edge_number)
                         if '{filename}' in self.filename_pattern:
                             suffix = ''
-                        filename = self.__prepare_filename(filename_template, sidecar_node.video_url)
+                        # pylint:disable=cell-var-from-loop
+                        filename = self.__prepare_filename(filename_template, lambda: sidecar_node.video_url)
                         # Download sidecar video if desired
                         downloaded &= self.download_pic(filename=filename, url=sidecar_node.video_url,
                                                         mtime=post.date_local, filename_suffix=suffix)
@@ -690,13 +696,13 @@ class Instaloader:
         date_local = item.date_local
         dirname = _PostPathFormatter(item).format(self.dirname_pattern, target=target)
         filename_template = os.path.join(dirname, self.format_filename(item, target=target))
-        filename = self.__prepare_filename(filename_template, item.url)
+        filename = self.__prepare_filename(filename_template, lambda: item.url)
         downloaded = False
         if not item.is_video or self.download_video_thumbnails is True:
             url = item.url
             downloaded = self.download_pic(filename=filename, url=url, mtime=date_local)
         if item.is_video and self.download_videos is True:
-            filename = self.__prepare_filename(filename_template, str(item.video_url))
+            filename = self.__prepare_filename(filename_template, lambda: str(item.video_url))
             downloaded |= self.download_pic(filename=filename, url=item.video_url, mtime=date_local)
         # Save caption if desired
         metadata_string = _ArbitraryItemFormatter(item).format(self.storyitem_metadata_txt_pattern).strip()

instaloader/instaloadercontext.py

@@ -33,7 +33,7 @@ def copy_session(session: requests.Session, request_timeout: Optional[float] = N
 
 def default_user_agent() -> str:
     return 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 ' \
-           '(KHTML, like Gecko) Chrome/51.0.2704.79 Safari/537.36'
+           '(KHTML, like Gecko) Chrome/88.0.4324.96 Safari/537.36'
 
 
 class InstaloaderContext:
@@ -288,7 +288,7 @@ class InstaloaderContext:
     def do_sleep(self):
         """Sleep a short time if self.sleep is set. Called before each request to instagram.com."""
         if self.sleep:
-            time.sleep(min(random.expovariate(0.7), 5.0))
+            time.sleep(min(random.expovariate(0.6), 15.0))
 
     def get_json(self, path: str, params: Dict[str, Any], host: str = 'www.instagram.com',
                  session: Optional[requests.Session] = None, _attempt=1) -> Dict[str, Any]:
@@ -549,6 +549,7 @@ class RateController:
         self._context = context
         self._query_timestamps = dict()  # type: Dict[str, List[float]]
         self._earliest_next_request_time = 0.0
+        self._iphone_earliest_next_request_time = 0.0
 
     def sleep(self, secs: float):
         """Wait given number of seconds."""
@@ -559,7 +560,7 @@ class RateController:
 
     def _dump_query_timestamps(self, current_time: float, failed_query_type: str):
         windows = [10, 11, 20, 22, 30, 60]
-        self._context.error("Requests within last {} minutes grouped by type:"
+        self._context.error("Number of requests within last {} minutes grouped by type:"
                             .format('/'.join(str(w) for w in windows)),
                             repeat_at_end=False)
         for query_type, times in self._query_timestamps.items():
@@ -571,11 +572,15 @@ class RateController:
             ), repeat_at_end=False)
 
     def count_per_sliding_window(self, query_type: str) -> int:
-        """Return how many requests can be done within the sliding window."""
+        """Return how many requests of the given type can be done within a sliding window of 11 minutes.
+
+        This is called by :meth:`RateController.query_waittime` and allows to simply customize wait times before queries
+        at query_type granularity. Consider overriding :meth:`RateController.query_waittime` directly if you need more
+        control."""
         # Not static, to allow for the count_per_sliding_window to depend on context-inherent properties, such as
         # whether we are logged in.
         # pylint:disable=no-self-use
-        return 75 if query_type in ['iphone', 'other'] else 200
+        return 75 if query_type == 'other' else 200
 
     def _reqs_in_sliding_window(self, query_type: Optional[str], current_time: float, window: float) -> List[float]:
         if query_type is not None:
@@ -591,6 +596,7 @@ class RateController:
     def query_waittime(self, query_type: str, current_time: float, untracked_queries: bool = False) -> float:
         """Calculate time needed to wait before query can be executed."""
         per_type_sliding_window = 660
+        iphone_sliding_window = 1800
         if query_type not in self._query_timestamps:
             self._query_timestamps[query_type] = []
         self._query_timestamps[query_type] = list(filter(lambda t: t > current_time - 60 * 60,
@@ -616,25 +622,43 @@ class RateController:
 
         def untracked_next_request_time():
             if untracked_queries:
-                reqs_in_sliding_window = self._reqs_in_sliding_window(query_type, current_time, per_type_sliding_window)
-                self._earliest_next_request_time = min(reqs_in_sliding_window) + per_type_sliding_window + 6
-            return self._earliest_next_request_time
+                if query_type == "iphone":
+                    reqs_in_sliding_window = self._reqs_in_sliding_window(query_type, current_time,
+                                                                          iphone_sliding_window)
+                    self._iphone_earliest_next_request_time = min(reqs_in_sliding_window) + iphone_sliding_window + 18
+                else:
+                    reqs_in_sliding_window = self._reqs_in_sliding_window(query_type, current_time,
+                                                                          per_type_sliding_window)
+                    self._earliest_next_request_time = min(reqs_in_sliding_window) + per_type_sliding_window + 6
+            return max(self._iphone_earliest_next_request_time, self._earliest_next_request_time)
+
+        def iphone_next_request():
+            if query_type == "iphone":
+                reqs_in_sliding_window = self._reqs_in_sliding_window(query_type, current_time, iphone_sliding_window)
+                if len(reqs_in_sliding_window) >= 199:
+                    return min(reqs_in_sliding_window) + iphone_sliding_window + 18
+            return 0.0
 
         return max(0.0,
                    max(
                        per_type_next_request_time(),
                        gql_accumulated_next_request_time(),
                        untracked_next_request_time(),
+                       iphone_next_request(),
                    ) - current_time)
 
     def wait_before_query(self, query_type: str) -> None:
-        """This method is called before a query to Instagram. It calls :meth:`RateController.sleep` to wait
-        until the request can be made."""
+        """This method is called before a query to Instagram.
+
+        It calls :meth:`RateController.query_waittime` to determine the time needed to wait and then calls
+        :meth:`RateController.sleep` to wait until the request can be made."""
         waittime = self.query_waittime(query_type, time.monotonic(), False)
         assert waittime >= 0
         if waittime > 15:
-            self._context.log("\nToo many queries in the last time. Need to wait {} seconds, until {:%H:%M}."
-                              .format(round(waittime), datetime.now() + timedelta(seconds=waittime)))
+            formatted_waittime = ("{} seconds".format(round(waittime)) if waittime <= 666 else
+                                  "{} minutes".format(round(waittime / 60)))
+            self._context.log("\nToo many queries in the last time. Need to wait {}, until {:%H:%M}."
+                              .format(formatted_waittime, datetime.now() + timedelta(seconds=waittime)))
         if waittime > 0:
             self.sleep(waittime)
         if query_type not in self._query_timestamps:
@@ -643,8 +667,10 @@ class RateController:
             self._query_timestamps[query_type].append(time.monotonic())
 
     def handle_429(self, query_type: str) -> None:
-        """This method is called to handle a 429 Too Many Requests response. It calls :meth:`RateController.sleep` to
-         wait until we can repeat the same request."""
+        """This method is called to handle a 429 Too Many Requests response.
+
+        It calls :meth:`RateController.query_waittime` to determine the time needed to wait and then calls
+        :meth:`RateController.sleep` to wait until we can repeat the same request."""
         current_time = time.monotonic()
         waittime = self.query_waittime(query_type, current_time, True)
         assert waittime >= 0
@@ -654,8 +680,10 @@ class RateController:
                         "App while Instaloader is running.")
         self._context.error(textwrap.fill(text_for_429), repeat_at_end=False)
         if waittime > 1.5:
-            self._context.error("The request will be retried in {} seconds, at {:%H:%M}."
-                                .format(round(waittime), datetime.now() + timedelta(seconds=waittime)),
+            formatted_waittime = ("{} seconds".format(round(waittime)) if waittime <= 666 else
+                                  "{} minutes".format(round(waittime / 60)))
+            self._context.error("The request will be retried in {}, at {:%H:%M}."
+                                .format(formatted_waittime, datetime.now() + timedelta(seconds=waittime)),
                                 repeat_at_end=False)
         if waittime > 0:
             self.sleep(waittime)