First Release Candidate for Version 4.7

Co-Authored-By: André Koch-Kramer <koch-kramer@web.de>

.github/funding.yml

@@ -0,0 +1 @@
+custom: "https://instaloader.github.io/contributing.html#donations"

.github/pull_request_template.md

@@ -1,7 +1,8 @@
 
 
 <!--
-Please describe:
+Thanks for your willingness to contribute to Instaloader! Please briefly
+describe
 
 - A motivation for this change, e.g.
   - Fixes # .

README.rst

@@ -29,7 +29,7 @@
 
 .. |downloads| image:: https://pepy.tech/badge/instaloader/month
    :alt: PyPI Download Count
-   :target: https://pepy.tech/project/instaloader/month
+   :target: https://pepy.tech/project/instaloader
 
 .. badges-end
 
@@ -84,7 +84,9 @@ To later **update your local copy** of that profiles, you may run
     instaloader --fast-update profile [profile ...]
 
 If ``--fast-update`` is given, Instaloader stops when arriving at the
-first already-downloaded picture. When updating profiles, Instaloader
+first already-downloaded picture.
+
+When updating profiles, Instaloader
 automatically **detects profile name changes** and renames the target directory
 accordingly.
 

docs/cli-options.rst

@@ -220,12 +220,20 @@ How to Download
    used to save the information to resume an interrupted download.  The default
    prefix is ``iterator``.
 
-   Resuming an interrupted download is supported for most, but not all targets.
+   Resuming an interrupted download is supported for the following targets:
+    - Profile posts,
+    - Profile IGTV posts (:option:`--igtv`),
+    - Profile tagged posts (:option:`--tagged`),
+    - Saved posts (``:saved``).
+
+   This feature is enabled by default for targets where it is supported;
+   :option:`--resume-prefix` only changes the name of the iterator files.
+
+   To turn this feature off, use :option:`--no-resume`.
+
    JSON files with resume information are always compressed, regardless of
    :option:`--no-compress-json`.
 
-   This feature is turned off entirely with :option:`--no-resume`.
-
    .. versionadded:: 4.5
 
 .. option:: --no-resume
@@ -238,7 +246,7 @@ How to Download
 .. option:: --user-agent USER_AGENT
 
    User Agent to use for HTTP requests. Per default, Instaloader pretends being
-   Chrome/51.
+   Chrome/88 on Linux.
 
 .. option:: --max-connection-attempts N
 
@@ -255,6 +263,17 @@ How to Download
    .. versionchanged:: 4.6
       Enabled this option by default with a timeout of 300 seconds.
 
+.. option:: --abort-on STATUS_CODE_LIST
+
+   Comma-separated list of HTTP status codes that cause Instaloader to abort,
+   bypassing all retry logic.
+
+   For example, with ``--abort-on=302,400,429``, Instaloader will stop if a
+   request is responded with a 302 redirect, a Bad Request error, or a Too Many
+   Requests error.
+
+   .. versionadded:: 4.7
+
 Miscellaneous Options
 ^^^^^^^^^^^^^^^^^^^^^
 

docs/conf.py

@@ -65,7 +65,7 @@ extlinks = {
         'Issue #'
     ),
     'example': (
-        'https://raw.githubusercontent.com/instaloader/instaloader/v' + current_release + '/docs/codesnippets/%s',
+        'https://raw.githubusercontent.com/instaloader/instaloader/master/docs/codesnippets/%s',
         'Example '
     ),
 }

docs/contributing.rst

@@ -11,11 +11,19 @@ Instaloader's development is organized on
 `GitHub <https://github.com/instaloader/instaloader>`__, where Issues and Pull
 Requests are discussed.
 
+Contributing to an open source project is an unmatchable opportunity to improve
+and manifest programming skills. For issues that are good starting points to get
+involved in Instaloader development, as they do not require deep insight into
+Instaloader's code base, have a look at our
+`Issues labelled 'good first issue' <https://github.com/instaloader/instaloader/contribute>`__.
+
+This document covers several ways of how you can contribute back to Instaloader.
+
 Answering Questions
 -------------------
 
 The easiest way to help out is to answer questions. If you are interested in
-answering questions regarding Instaloader, good places to start are
+answering questions regarding Instaloader, good places to begin are
 
 - `Questions tagged 'instaloader' on Stack Overflow <https://stackoverflow.com/questions/tagged/instaloader>`__,
 - `Instaloader Issues labeled 'question' <https://github.com/instaloader/instaloader/issues?q=is%3Aissue+is%3Aopen+label%3Aquestion>`__.
@@ -81,8 +89,8 @@ Improvements of the Instaloader source or its documentation can be proposed as a
      make -C docs html
 
 - Feel free to create an issue to make sure someone from the Instaloader team
-  agrees that the change might be an improvement, and is happy with your basic
-  proposal, before working on a pull request.
+  agrees that the change might be an improvement, or if you want to discuss
+  your basic proposal, before working on a pull request.
 
 Proposing Features
 ------------------

docs/module/exceptions.rst

@@ -47,3 +47,5 @@ Exceptions
 
 
 .. autoexception:: TooManyRequestsException
+
+.. autoexception:: AbortDownloadException

instaloader/__init__.py

@@ -1,7 +1,7 @@
 """Download pictures (or videos) along with their captions and other metadata from Instagram."""
 
 
-__version__ = '4.6a1'
+__version__ = '4.7rc1'
 
 
 try:

instaloader/__main__.py

@@ -5,12 +5,12 @@ import datetime
 import os
 import re
 import sys
-from argparse import ArgumentParser, SUPPRESS
+from argparse import ArgumentParser, ArgumentTypeError, SUPPRESS
 from typing import List, Optional
 
-from . import (Instaloader, InstaloaderException, InvalidArgumentException, Post, Profile, ProfileNotExistsException,
-               StoryItem, __version__, load_structure_from_file, TwoFactorAuthRequiredException,
-               BadCredentialsException)
+from . import (AbortDownloadException, BadCredentialsException, Instaloader, InstaloaderException,
+               InvalidArgumentException, Post, Profile, ProfileNotExistsException, StoryItem,
+               TwoFactorAuthRequiredException, __version__, load_structure_from_file)
 from .instaloader import get_default_session_filename
 from .instaloadercontext import default_user_agent
 
@@ -27,6 +27,14 @@ def usage_string():
 {0} --help""".format(argv0, len(argv0), '')
 
 
+def http_status_code_list(code_list_str: str) -> List[int]:
+    codes = [int(s) for s in code_list_str.split(',')]
+    for code in codes:
+        if not 100 <= code <= 599:
+            raise ArgumentTypeError("Invalid HTTP status code: {}".format(code))
+    return codes
+
+
 def filterstr_to_filterfunc(filter_str: str, item_type: type):
     """Takes an --post-filter=... or --storyitem-filter=... filter
      specification and makes a filter_func Callable out of it."""
@@ -199,7 +207,7 @@ def _main(instaloader: Instaloader, targetlist: List[str],
             instaloader.context.log("Downloading {} profiles: {}".format(len(profiles),
                                                                          ' '.join([p.username for p in profiles])))
         if profiles and (download_profile_pic or download_posts) and not instaloader.context.is_logged_in:
-            instaloader.context.error("Warning: Use --login to download higher-quality versions of pictures.")
+            instaloader.context.log("Hint: Use --login to download higher-quality versions of pictures.")
         instaloader.download_profiles(profiles,
                                       download_profile_pic, download_posts, download_tagged, download_igtv,
                                       download_highlights, download_stories,
@@ -213,6 +221,8 @@ def _main(instaloader: Instaloader, targetlist: List[str],
                                                    fast_update=fast_update, post_filter=post_filter)
     except KeyboardInterrupt:
         print("\nInterrupted by user.", file=sys.stderr)
+    except AbortDownloadException as exc:
+        print("\nDownload aborted: {}.".format(exc), file=sys.stderr)
     # Save session if it is useful
     if instaloader.context.is_logged_in:
         instaloader.save_session_to_file(sessionfile)
@@ -369,6 +379,9 @@ def main():
     g_how.add_argument('--commit-mode', action='store_true', help=SUPPRESS)
     g_how.add_argument('--request-timeout', metavar='N', type=float, default=300.0,
                        help='Seconds to wait before timing out a connection request. Defaults to 300.')
+    g_how.add_argument('--abort-on', type=http_status_code_list, metavar="STATUS_CODES",
+                       help='Comma-separated list of HTTP status codes that cause Instaloader to abort, bypassing all '
+                            'retry logic.')
 
     g_misc = parser.add_argument_group('Miscellaneous Options')
     g_misc.add_argument('-q', '--quiet', action='store_true',
@@ -427,7 +440,8 @@ def main():
                              request_timeout=args.request_timeout,
                              resume_prefix=resume_prefix,
                              check_resume_bbd=not args.use_aged_resume_files,
-                             slide=args.slide)
+                             slide=args.slide,
+                             fatal_status_codes=args.abort_on)
         _main(loader,
               args.profile,
               username=args.login.lower() if args.login is not None else None,

instaloader/exceptions.py

@@ -64,3 +64,15 @@ class QueryReturnedNotFoundException(ConnectionException):
 
 class TooManyRequestsException(ConnectionException):
     pass
+
+
+class AbortDownloadException(Exception):
+    """
+    Exception that is not catched in the error catchers inside the download loop and so aborts the
+    download loop.
+
+    This exception is not a subclass of ``InstaloaderException``.
+
+    .. versionadded:: 4.7
+    """
+    pass

instaloader/instaloader.py

@@ -161,6 +161,7 @@ class Instaloader:
     :param resume_prefix: :option:`--resume-prefix`, or None for :option:`--no-resume`.
     :param check_resume_bbd: Whether to check the date of expiry of resume files and reject them if expired.
     :param slide: :option:`--slide`
+    :param fatal_status_codes: :option:`--abort-on`
 
     .. attribute:: context
 
@@ -187,10 +188,11 @@ class Instaloader:
                  rate_controller: Optional[Callable[[InstaloaderContext], RateController]] = None,
                  resume_prefix: Optional[str] = "iterator",
                  check_resume_bbd: bool = True,
-                 slide: Optional[str] = None):
+                 slide: Optional[str] = None,
+                 fatal_status_codes: Optional[List[int]] = None):
 
         self.context = InstaloaderContext(sleep, quiet, user_agent, max_connection_attempts,
-                                          request_timeout, rate_controller)
+                                          request_timeout, rate_controller, fatal_status_codes)
 
         # configuration parameters
         self.dirname_pattern = dirname_pattern or "{target}"
@@ -256,7 +258,8 @@ class Instaloader:
             request_timeout=self.context.request_timeout,
             resume_prefix=self.resume_prefix,
             check_resume_bbd=self.check_resume_bbd,
-            slide=self.slide)
+            slide=self.slide,
+            fatal_status_codes=self.context.fatal_status_codes)
         yield new_loader
         self.context.error_log.extend(new_loader.context.error_log)
         new_loader.context.error_log = []  # avoid double-printing of errors
@@ -332,18 +335,47 @@ class Instaloader:
                             combined_answers.extend(y['answers'])
                         unique_comments_list[-1]['answers'] = get_unique_comments(combined_answers)
             return unique_comments_list
+
+        def get_new_comments(new_comments, start):
+            for idx, comment in enumerate(new_comments, start=start+1):
+                if idx % 250 == 0:
+                    self.context.log('{}'.format(idx), end='…', flush=True)
+                yield comment
+
+        def save_comments(extended_comments):
+            unique_comments = get_unique_comments(extended_comments, combine_answers=True)
+            answer_ids = set(int(answer['id']) for comment in unique_comments for answer in comment.get('answers', []))
+            with open(filename, 'w') as file:
+                file.write(json.dumps(list(filter(lambda t: int(t['id']) not in answer_ids, unique_comments)),
+                                      indent=4))
+
+        base_filename = filename
         filename += '_comments.json'
         try:
             with open(filename) as fp:
                 comments = json.load(fp)
         except (FileNotFoundError, json.decoder.JSONDecodeError):
             comments = list()
-        comments.extend(_postcomment_asdict(comment) for comment in post.get_comments())
+
+        comments_iterator = post.get_comments()
+        try:
+            with resumable_iteration(
+                    context=self.context,
+                    iterator=comments_iterator,
+                    load=load_structure_from_file,
+                    save=save_structure_to_file,
+                    format_path=lambda magic: "{}_{}_{}.json.xz".format(base_filename, self.resume_prefix, magic),
+                    check_bbd=self.check_resume_bbd,
+                    enabled=self.resume_prefix is not None
+            ) as (_is_resuming, start_index):
+                comments.extend(_postcomment_asdict(comment)
+                                for comment in get_new_comments(comments_iterator, start_index))
+        except (KeyboardInterrupt, AbortDownloadException):
+            if comments:
+                save_comments(comments)
+            raise
         if comments:
-            comments = get_unique_comments(comments, combine_answers=True)
-            answer_ids = set(int(answer['id']) for comment in comments for answer in comment.get('answers', []))
-            with open(filename, 'w') as file:
-                file.write(json.dumps(list(filter(lambda t: int(t['id']) not in answer_ids, comments)), indent=4))
+            save_comments(comments)
             self.context.log('comments', end=' ', flush=True)
 
     def save_caption(self, filename: str, mtime: datetime, caption: str) -> None:
@@ -524,11 +556,15 @@ class Instaloader:
         self.context.two_factor_login(two_factor_code)
 
     @staticmethod
-    def __prepare_filename(filename_template: str, url: str) -> str:
+    def __prepare_filename(filename_template: str, url: Callable[[], str]) -> str:
         """Replace filename token inside filename_template with url's filename and assure the directories exist.
 
         .. versionadded:: 4.6"""
-        filename = filename_template.replace("{filename}", os.path.splitext(os.path.basename(urlparse(url).path))[0])
+        if "{filename}" in filename_template:
+            filename = filename_template.replace("{filename}",
+                                                 os.path.splitext(os.path.basename(urlparse(url()).path))[0])
+        else:
+            filename = filename_template
         os.makedirs(os.path.dirname(filename), exist_ok=True)
         return filename
 
@@ -547,43 +583,76 @@ class Instaloader:
         :return: True if something was downloaded, False otherwise, i.e. file was already there
         """
 
+        def _already_downloaded(path: str) -> bool:
+            if not os.path.isfile(path):
+                return False
+            else:
+                self.context.log(path + ' exists', end=' ', flush=True)
+                return True
+
+        def _all_already_downloaded(path_base, is_videos_enumerated) -> bool:
+            if '{filename}' in self.filename_pattern:
+                # full URL needed to evaluate actual filename, cannot determine at
+                # this point if all sidecar nodes were already downloaded.
+                return False
+            for idx, is_video in is_videos_enumerated:
+                if self.download_pictures and (not is_video or self.download_video_thumbnails):
+                    if not _already_downloaded("{0}_{1}.jpg".format(path_base, idx)):
+                        return False
+                if is_video and self.download_videos:
+                    if not _already_downloaded("{0}_{1}.mp4".format(path_base, idx)):
+                        return False
+            return True
+
         dirname = _PostPathFormatter(post).format(self.dirname_pattern, target=target)
         filename_template = os.path.join(dirname, self.format_filename(post, target=target))
-        filename = self.__prepare_filename(filename_template, post.url)
+        filename = self.__prepare_filename(filename_template, lambda: post.url)
 
         # Download the image(s) / video thumbnail and videos within sidecars if desired
         downloaded = True
         if post.typename == 'GraphSidecar':
             if self.download_pictures or self.download_videos:
-                for edge_number, sidecar_node in enumerate(
-                        post.get_sidecar_nodes(self.slide_start, self.slide_end),
-                        start=post.mediacount if self.slide_start < 0 else self.slide_start + 1
+                if not _all_already_downloaded(
+                        filename_template, enumerate(
+                            (post.get_is_videos()[i]
+                             for i in range(self.slide_start % post.mediacount, self.slide_end % post.mediacount + 1)),
+                            start=self.slide_start % post.mediacount + 1
+                        )
                 ):
-                    if self.download_pictures and (not sidecar_node.is_video or self.download_video_thumbnails):
-                        suffix = str(edge_number)
+                    for edge_number, sidecar_node in enumerate(
+                            post.get_sidecar_nodes(self.slide_start, self.slide_end),
+                            start=self.slide_start % post.mediacount + 1
+                    ):
+                        suffix = str(edge_number)  # type: Optional[str]
                         if '{filename}' in self.filename_pattern:
-                            suffix = ''
-                        filename = self.__prepare_filename(filename_template, sidecar_node.display_url)
-                        # Download sidecar picture or video thumbnail (--no-pictures implies --no-video-thumbnails)
-                        downloaded &= self.download_pic(filename=filename, url=sidecar_node.display_url,
-                                                        mtime=post.date_local, filename_suffix=suffix)
-                    if sidecar_node.is_video and self.download_videos:
-                        suffix = str(edge_number)
-                        if '{filename}' in self.filename_pattern:
-                            suffix = ''
-                        filename = self.__prepare_filename(filename_template, sidecar_node.video_url)
-                        # Download sidecar video if desired
-                        downloaded &= self.download_pic(filename=filename, url=sidecar_node.video_url,
-                                                        mtime=post.date_local, filename_suffix=suffix)
+                            suffix = None
+                        if self.download_pictures and (not sidecar_node.is_video or self.download_video_thumbnails):
+                            # pylint:disable=cell-var-from-loop
+                            sidecar_filename = self.__prepare_filename(filename_template,
+                                                                       lambda: sidecar_node.display_url)
+                            # Download sidecar picture or video thumbnail (--no-pictures implies --no-video-thumbnails)
+                            downloaded &= self.download_pic(filename=sidecar_filename, url=sidecar_node.display_url,
+                                                            mtime=post.date_local, filename_suffix=suffix)
+                        if sidecar_node.is_video and self.download_videos:
+                            # pylint:disable=cell-var-from-loop
+                            sidecar_filename = self.__prepare_filename(filename_template,
+                                                                       lambda: sidecar_node.video_url)
+                            # Download sidecar video if desired
+                            downloaded &= self.download_pic(filename=sidecar_filename, url=sidecar_node.video_url,
+                                                            mtime=post.date_local, filename_suffix=suffix)
+                else:
+                    downloaded = False
         elif post.typename == 'GraphImage':
             # Download picture
             if self.download_pictures:
-                downloaded = self.download_pic(filename=filename, url=post.url, mtime=post.date_local)
+                downloaded = (not _already_downloaded(filename + ".jpg") and
+                              self.download_pic(filename=filename, url=post.url, mtime=post.date_local))
         elif post.typename == 'GraphVideo':
             # Download video thumbnail (--no-pictures implies --no-video-thumbnails)
             if self.download_pictures and self.download_video_thumbnails:
                 with self.context.error_catcher("Video thumbnail of {}".format(post)):
-                    downloaded = self.download_pic(filename=filename, url=post.url, mtime=post.date_local)
+                    downloaded = (not _already_downloaded(filename + ".jpg") and
+                                  self.download_pic(filename=filename, url=post.url, mtime=post.date_local))
         else:
             self.context.error("Warning: {0} has unknown typename: {1}".format(post, post.typename))
 
@@ -594,7 +663,8 @@ class Instaloader:
 
         # Download video if desired
         if post.is_video and self.download_videos:
-            downloaded &= self.download_pic(filename=filename, url=post.video_url, mtime=post.date_local)
+            downloaded &= (not _already_downloaded(filename + ".mp4") and
+                           self.download_pic(filename=filename, url=post.video_url, mtime=post.date_local))
 
         # Download geotags if desired
         if self.download_geotags and post.location:
@@ -687,17 +757,25 @@ class Instaloader:
         :return: True if something was downloaded, False otherwise, i.e. file was already there
         """
 
+        def _already_downloaded(path: str) -> bool:
+            if not os.path.isfile(path):
+                return False
+            else:
+                self.context.log(path + ' exists', end=' ', flush=True)
+                return True
+
         date_local = item.date_local
         dirname = _PostPathFormatter(item).format(self.dirname_pattern, target=target)
         filename_template = os.path.join(dirname, self.format_filename(item, target=target))
-        filename = self.__prepare_filename(filename_template, item.url)
+        filename = self.__prepare_filename(filename_template, lambda: item.url)
         downloaded = False
         if not item.is_video or self.download_video_thumbnails is True:
-            url = item.url
-            downloaded = self.download_pic(filename=filename, url=url, mtime=date_local)
+            downloaded = (not _already_downloaded(filename + ".jpg") and
+                          self.download_pic(filename=filename, url=item.url, mtime=date_local))
         if item.is_video and self.download_videos is True:
-            filename = self.__prepare_filename(filename_template, str(item.video_url))
-            downloaded |= self.download_pic(filename=filename, url=item.video_url, mtime=date_local)
+            filename = self.__prepare_filename(filename_template, lambda: str(item.video_url))
+            downloaded |= (not _already_downloaded(filename + ".mp4") and
+                           self.download_pic(filename=filename, url=item.video_url, mtime=date_local))
         # Save caption if desired
         metadata_string = _ArbitraryItemFormatter(item).format(self.storyitem_metadata_txt_pattern).strip()
         if metadata_string:

instaloader/instaloadercontext.py

@@ -33,7 +33,7 @@ def copy_session(session: requests.Session, request_timeout: Optional[float] = N
 
 def default_user_agent() -> str:
     return 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 ' \
-           '(KHTML, like Gecko) Chrome/51.0.2704.79 Safari/537.36'
+           '(KHTML, like Gecko) Chrome/88.0.4324.96 Safari/537.36'
 
 
 class InstaloaderContext:
@@ -53,7 +53,8 @@ class InstaloaderContext:
 
     def __init__(self, sleep: bool = True, quiet: bool = False, user_agent: Optional[str] = None,
                  max_connection_attempts: int = 3, request_timeout: float = 300.0,
-                 rate_controller: Optional[Callable[["InstaloaderContext"], "RateController"]] = None):
+                 rate_controller: Optional[Callable[["InstaloaderContext"], "RateController"]] = None,
+                 fatal_status_codes: Optional[List[int]] = None):
 
         self.user_agent = user_agent if user_agent is not None else default_user_agent()
         self.request_timeout = request_timeout
@@ -74,6 +75,9 @@ class InstaloaderContext:
         # Can be set to True for testing, disables supression of InstaloaderContext._error_catcher
         self.raise_all_errors = False
 
+        # HTTP status codes that should cause an AbortDownloadException
+        self.fatal_status_codes = fatal_status_codes or []
+
         # Cache profile from id (mapping from id to Profile)
         self.profile_id_cache = dict()           # type: Dict[int, Any]
 
@@ -288,7 +292,7 @@ class InstaloaderContext:
     def do_sleep(self):
         """Sleep a short time if self.sleep is set. Called before each request to instagram.com."""
         if self.sleep:
-            time.sleep(min(random.expovariate(0.7), 5.0))
+            time.sleep(min(random.expovariate(0.6), 15.0))
 
     def get_json(self, path: str, params: Dict[str, Any], host: str = 'www.instagram.com',
                  session: Optional[requests.Session] = None, _attempt=1) -> Dict[str, Any]:
@@ -316,6 +320,11 @@ class InstaloaderContext:
             if is_other_query:
                 self._rate_controller.wait_before_query('other')
             resp = sess.get('https://{0}/{1}'.format(host, path), params=params, allow_redirects=False)
+            if resp.status_code in self.fatal_status_codes:
+                redirect = " redirect to {}".format(resp.headers['location']) if 'location' in resp.headers else ""
+                raise AbortDownloadException("Query to https://{}/{} responded with \"{} {}\"{}".format(
+                    host, path, resp.status_code, resp.reason, redirect
+                ))
             while resp.is_redirect:
                 redirect_url = resp.headers['location']
                 self.log('\nHTTP redirect from https://{0}/{1} to {2}'.format(host, path, redirect_url))
@@ -549,6 +558,7 @@ class RateController:
         self._context = context
         self._query_timestamps = dict()  # type: Dict[str, List[float]]
         self._earliest_next_request_time = 0.0
+        self._iphone_earliest_next_request_time = 0.0
 
     def sleep(self, secs: float):
         """Wait given number of seconds."""
@@ -559,7 +569,7 @@ class RateController:
 
     def _dump_query_timestamps(self, current_time: float, failed_query_type: str):
         windows = [10, 11, 20, 22, 30, 60]
-        self._context.error("Requests within last {} minutes grouped by type:"
+        self._context.error("Number of requests within last {} minutes grouped by type:"
                             .format('/'.join(str(w) for w in windows)),
                             repeat_at_end=False)
         for query_type, times in self._query_timestamps.items():
@@ -571,11 +581,15 @@ class RateController:
             ), repeat_at_end=False)
 
     def count_per_sliding_window(self, query_type: str) -> int:
-        """Return how many requests can be done within the sliding window."""
+        """Return how many requests of the given type can be done within a sliding window of 11 minutes.
+
+        This is called by :meth:`RateController.query_waittime` and allows to simply customize wait times before queries
+        at query_type granularity. Consider overriding :meth:`RateController.query_waittime` directly if you need more
+        control."""
         # Not static, to allow for the count_per_sliding_window to depend on context-inherent properties, such as
         # whether we are logged in.
         # pylint:disable=no-self-use
-        return 75 if query_type in ['iphone', 'other'] else 200
+        return 75 if query_type == 'other' else 200
 
     def _reqs_in_sliding_window(self, query_type: Optional[str], current_time: float, window: float) -> List[float]:
         if query_type is not None:
@@ -591,6 +605,7 @@ class RateController:
     def query_waittime(self, query_type: str, current_time: float, untracked_queries: bool = False) -> float:
         """Calculate time needed to wait before query can be executed."""
         per_type_sliding_window = 660
+        iphone_sliding_window = 1800
         if query_type not in self._query_timestamps:
             self._query_timestamps[query_type] = []
         self._query_timestamps[query_type] = list(filter(lambda t: t > current_time - 60 * 60,
@@ -616,25 +631,43 @@ class RateController:
 
         def untracked_next_request_time():
             if untracked_queries:
-                reqs_in_sliding_window = self._reqs_in_sliding_window(query_type, current_time, per_type_sliding_window)
-                self._earliest_next_request_time = min(reqs_in_sliding_window) + per_type_sliding_window + 6
-            return self._earliest_next_request_time
+                if query_type == "iphone":
+                    reqs_in_sliding_window = self._reqs_in_sliding_window(query_type, current_time,
+                                                                          iphone_sliding_window)
+                    self._iphone_earliest_next_request_time = min(reqs_in_sliding_window) + iphone_sliding_window + 18
+                else:
+                    reqs_in_sliding_window = self._reqs_in_sliding_window(query_type, current_time,
+                                                                          per_type_sliding_window)
+                    self._earliest_next_request_time = min(reqs_in_sliding_window) + per_type_sliding_window + 6
+            return max(self._iphone_earliest_next_request_time, self._earliest_next_request_time)
+
+        def iphone_next_request():
+            if query_type == "iphone":
+                reqs_in_sliding_window = self._reqs_in_sliding_window(query_type, current_time, iphone_sliding_window)
+                if len(reqs_in_sliding_window) >= 199:
+                    return min(reqs_in_sliding_window) + iphone_sliding_window + 18
+            return 0.0
 
         return max(0.0,
                    max(
                        per_type_next_request_time(),
                        gql_accumulated_next_request_time(),
                        untracked_next_request_time(),
+                       iphone_next_request(),
                    ) - current_time)
 
     def wait_before_query(self, query_type: str) -> None:
-        """This method is called before a query to Instagram. It calls :meth:`RateController.sleep` to wait
-        until the request can be made."""
+        """This method is called before a query to Instagram.
+
+        It calls :meth:`RateController.query_waittime` to determine the time needed to wait and then calls
+        :meth:`RateController.sleep` to wait until the request can be made."""
         waittime = self.query_waittime(query_type, time.monotonic(), False)
         assert waittime >= 0
         if waittime > 15:
-            self._context.log("\nToo many queries in the last time. Need to wait {} seconds, until {:%H:%M}."
-                              .format(round(waittime), datetime.now() + timedelta(seconds=waittime)))
+            formatted_waittime = ("{} seconds".format(round(waittime)) if waittime <= 666 else
+                                  "{} minutes".format(round(waittime / 60)))
+            self._context.log("\nToo many queries in the last time. Need to wait {}, until {:%H:%M}."
+                              .format(formatted_waittime, datetime.now() + timedelta(seconds=waittime)))
         if waittime > 0:
             self.sleep(waittime)
         if query_type not in self._query_timestamps:
@@ -643,8 +676,10 @@ class RateController:
             self._query_timestamps[query_type].append(time.monotonic())
 
     def handle_429(self, query_type: str) -> None:
-        """This method is called to handle a 429 Too Many Requests response. It calls :meth:`RateController.sleep` to
-         wait until we can repeat the same request."""
+        """This method is called to handle a 429 Too Many Requests response.
+
+        It calls :meth:`RateController.query_waittime` to determine the time needed to wait and then calls
+        :meth:`RateController.sleep` to wait until we can repeat the same request."""
         current_time = time.monotonic()
         waittime = self.query_waittime(query_type, current_time, True)
         assert waittime >= 0
@@ -654,8 +689,10 @@ class RateController:
                         "App while Instaloader is running.")
         self._context.error(textwrap.fill(text_for_429), repeat_at_end=False)
         if waittime > 1.5:
-            self._context.error("The request will be retried in {} seconds, at {:%H:%M}."
-                                .format(round(waittime), datetime.now() + timedelta(seconds=waittime)),
+            formatted_waittime = ("{} seconds".format(round(waittime)) if waittime <= 666 else
+                                  "{} minutes".format(round(waittime / 60)))
+            self._context.error("The request will be retried in {}, at {:%H:%M}."
+                                .format(formatted_waittime, datetime.now() + timedelta(seconds=waittime)),
                                 repeat_at_end=False)
         if waittime > 0:
             self.sleep(waittime)

instaloader/nodeiterator.py

@@ -5,9 +5,9 @@ import os
 from contextlib import contextmanager
 from datetime import datetime, timedelta
 from lzma import LZMAError
-from typing import Any, Callable, Dict, Iterator, NamedTuple, Optional, Tuple, TypeVar
+from typing import Any, Callable, Dict, Iterable, Iterator, NamedTuple, Optional, Tuple, TypeVar
 
-from .exceptions import InvalidArgumentException, QueryReturnedBadRequestException
+from .exceptions import AbortDownloadException, InvalidArgumentException, QueryReturnedBadRequestException
 from .instaloadercontext import InstaloaderContext
 
 FrozenNodeIterator = NamedTuple('FrozenNodeIterator',
@@ -204,14 +204,15 @@ class NodeIterator(Iterator[T]):
 
 @contextmanager
 def resumable_iteration(context: InstaloaderContext,
-                        iterator: Iterator,
+                        iterator: Iterable,
                         load: Callable[[InstaloaderContext, str], Any],
                         save: Callable[[FrozenNodeIterator, str], None],
                         format_path: Callable[[str], str],
                         check_bbd: bool = True,
                         enabled: bool = True) -> Iterator[Tuple[bool, int]]:
     """
-    High-level context manager to handle a resumable iteration that can be interrupted with a KeyboardInterrupt.
+    High-level context manager to handle a resumable iteration that can be interrupted
+    with a :class:`KeyboardInterrupt` or an :class:`AbortDownloadException`.
 
     It can be used as follows to automatically load a previously-saved state into the iterator, save the iterator's
     state when interrupted, and delete the resume file upon completion::
@@ -239,6 +240,9 @@ def resumable_iteration(context: InstaloaderContext,
     :param format_path: Returns the path to the resume file for the given magic.
     :param check_bbd: Whether to check the best before date and reject an expired FrozenNodeIterator.
     :param enabled: Set to False to disable all functionality and simply execute the inner body.
+
+    .. versionchanged:: 4.7
+       Also interrupt on :class:`AbortDownloadException`.
     """
     if not enabled or not isinstance(iterator, NodeIterator):
         yield False, 0
@@ -262,7 +266,7 @@ def resumable_iteration(context: InstaloaderContext,
             context.error("Warning: Not resuming from {}: {}".format(resume_file_path, exc))
     try:
         yield is_resuming, start_index
-    except KeyboardInterrupt:
+    except (KeyboardInterrupt, AbortDownloadException):
         if os.path.dirname(resume_file_path):
             os.makedirs(os.path.dirname(resume_file_path), exist_ok=True)
         save(iterator.freeze(), resume_file_path)

instaloader/structures.py

@@ -4,7 +4,7 @@ import re
 from base64 import b64decode, b64encode
 from collections import namedtuple
 from datetime import datetime
-from typing import Any, Dict, Iterator, List, Optional, Union
+from typing import Any, Dict, Iterable, Iterator, List, Optional, Union
 
 from . import __version__
 from .exceptions import *
@@ -128,6 +128,14 @@ class Post:
         """The mediaid is a decimal representation of the media shortcode."""
         return int(self._node['id'])
 
+    @property
+    def title(self) -> Optional[str]:
+        """Title of post"""
+        try:
+            return self._field('title')
+        except KeyError:
+            return None
+
     def __repr__(self):
         return '<Post {}>'.format(self.shortcode)
 
@@ -264,6 +272,17 @@ class Post:
             return len(edges)
         return 1
 
+    def get_is_videos(self) -> List[bool]:
+        """
+        Return a list containing the ``is_video`` property for each media in the post.
+
+        .. versionadded:: 4.7
+        """
+        if self.typename == 'GraphSidecar':
+            edges = self._field('edge_sidecar_to_children', 'edges')
+            return [edge['node']['is_video'] for edge in edges]
+        return [self.is_video]
+
     def get_sidecar_nodes(self, start=0, end=-1) -> Iterator[PostSidecarNode]:
         """
         Sidecar nodes of a Post with typename==GraphSidecar.
@@ -273,13 +292,13 @@ class Post:
         """
         if self.typename == 'GraphSidecar':
             edges = self._field('edge_sidecar_to_children', 'edges')
-            if any(edge['node']['is_video'] for edge in edges):
-                # video_url is only present in full metadata, issue #558.
-                edges = self._full_metadata['edge_sidecar_to_children']['edges']
             if end < 0:
                 end = len(edges)-1
             if start < 0:
                 start = len(edges)-1
+            if any(edge['node']['is_video'] and 'video_url' not in edge['node'] for edge in edges[start:(end+1)]):
+                # video_url is only present in full metadata, issue #558.
+                edges = self._full_metadata['edge_sidecar_to_children']['edges']
             for idx, edge in enumerate(edges):
                 if start <= idx <= end:
                     node = edge['node']
@@ -407,12 +426,15 @@ class Post:
         except KeyError:
             return self._field('edge_media_to_comment', 'count')
 
-    def get_comments(self) -> Iterator[PostComment]:
+    def get_comments(self) -> Iterable[PostComment]:
         r"""Iterate over all comments of the post.
 
         Each comment is represented by a PostComment namedtuple with fields text (string), created_at (datetime),
         id (int), owner (:class:`Profile`) and answers (:class:`~typing.Iterator`\ [:class:`PostCommentAnswer`])
         if available.
+
+        .. versionchanged:: 4.7
+           Change return type to ``Iterable``.
         """
         def _postcommentanswer(node):
             return PostCommentAnswer(id=int(node['id']),
@@ -447,16 +469,15 @@ class Post:
                                answers=_postcommentanswers(node))
         if self.comments == 0:
             # Avoid doing additional requests if there are no comments
-            return
+            return []
 
         comment_edges = self._field('edge_media_to_comment', 'edges')
         answers_count = sum([edge['node'].get('edge_threaded_comments', {}).get('count', 0) for edge in comment_edges])
 
         if self.comments == len(comment_edges) + answers_count:
             # If the Post's metadata already contains all parent comments, don't do GraphQL requests to obtain them
-            yield from (_postcomment(comment['node']) for comment in comment_edges)
-            return
-        yield from NodeIterator(
+            return [_postcomment(comment['node']) for comment in comment_edges]
+        return NodeIterator(
             self._context,
             '97b41c52301f77ce508f55e66d17620e',
             lambda d: d['data']['shortcode_media']['edge_media_to_parent_comment'],
@@ -835,7 +856,7 @@ class Profile:
         self._obtain_metadata()
         return NodeIterator(
             self._context,
-            '472f257a40c653c64c666ce877d59d2b',
+            '003056d32c2554def87228bc3fd9668a',
             lambda d: d['data']['user']['edge_owner_to_timeline_media'],
             lambda n: Post(self._context, n, self),
             {'id': self.userid},