First Alpha Release for Version 4.5

With this change, Instaloader is capable of resuming a previously-aborted download loop. To do so, it creates a JSON file within the target directory when interrupted, that contains all the necessary information to later resume that operation.

Resuming an interrupted download is supported for most, but not all targets. It is supported for:

- Regular profile posts,
- IGTV posts
- Saved posts,
- Tagged posts,
- Explore posts.

README.rst

@@ -49,7 +49,9 @@
   directory accordingly,
 
 - allows **fine-grained customization** of filters and where to store
-  downloaded media.
+  downloaded media,
+
+- automatically **resumes previously-interrupted** download iterations.
 
 ::
 

docs/as-module.rst

@@ -226,8 +226,32 @@ Exceptions
 
 .. autoexception:: TooManyRequestsException
 
+Resumable Iterations
+^^^^^^^^^^^^^^^^^^^^
+
+.. versionadded:: 4.5
+
+.. autoclass:: NodeIterator
+   :no-show-inheritance:
+
+.. autoclass:: FrozenNodeIterator
+   :no-show-inheritance:
+
+.. autofunction:: resumable_iteration
+
 ``InstaloaderContext`` (Low-level functions)
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
+``InstaloaderContext``
+""""""""""""""""""""""
+
 .. autoclass:: InstaloaderContext
    :no-show-inheritance:
+
+``RateController``
+""""""""""""""""""
+
+.. autoclass:: RateController
+   :no-show-inheritance:
+
+   .. versionadded:: 4.5

docs/cli-options.rst

@@ -223,6 +223,29 @@ How to Download
    ``#hashtag`` or the profile name. Defaults to ``{date_utc}_UTC``.
    See :ref:`filename-specification` for a list of supported tokens.
 
+.. option:: --resume-prefix prefix
+
+   For many targets, Instaloader is capable of resuming a previously-aborted
+   download loop.  To do so, it creates a JSON file within the target directory
+   when interrupted.  This option controls the prefix for filenames that are
+   used to save the information to resume an interrupted download.  The default
+   prefix is ``iterator``.
+
+   Resuming an interrupted download is supported for most, but not all targets.
+   JSON files with resume information are always compressed, regardless of
+   :option:`--no-compress-json`.
+
+   This feature is turned off entirely with :option:`--no-resume`.
+
+   .. versionadded:: 4.5
+
+.. option:: --no-resume
+
+   Do not resume a previously-aborted download iteration, and do not save such
+   information when interrupted.
+
+   .. versionadded:: 4.5
+
 .. option:: --user-agent USER_AGENT
 
    User Agent to use for HTTP requests. Per default, Instaloader pretends being

docs/conf.py

@@ -50,6 +50,8 @@ autodoc_member_order = 'bysource'
 intersphinx_mapping = {'python': ('https://docs.python.org/3', None),
                        'requests': ('https://requests.kennethreitz.org/en/master/', None)}
 
+nitpick_ignore = [('py:class', 'typing.Tuple')]
+
 current_release = subprocess.check_output(["git", "describe", "--abbrev=0"]).decode("ascii")[1:-1]
 date_format = "%e %b %Y" if platform.system() != "Windows" else "%d %b %Y"
 current_release_date = subprocess.check_output(

docs/index.rst

@@ -40,6 +40,8 @@ See :ref:`install` for more options on how to install Instaloader.
 - allows **fine-grained customization** of filters and where to store
   downloaded media,
 
+- automatically **resumes previously-interrupted** download iterations,
+
 - is free `open source <https://github.com/instaloader/instaloader>`__
   software written in Python.
 

instaloader/__init__.py

@@ -1,7 +1,7 @@
 """Download pictures (or videos) along with their captions and other metadata from Instagram."""
 
 
-__version__ = '4.4.5'
+__version__ = '4.5a1'
 
 
 try:
@@ -14,6 +14,7 @@ else:
 
 from .exceptions import *
 from .instaloader import Instaloader
-from .instaloadercontext import InstaloaderContext
+from .instaloadercontext import InstaloaderContext, RateController
+from .nodeiterator import NodeIterator, FrozenNodeIterator, resumable_iteration
 from .structures import (Hashtag, Highlight, Post, PostSidecarNode, PostComment, PostCommentAnswer, PostLocation,
                          Profile, Story, StoryItem, TopSearchResults, load_structure_from_file, save_structure_to_file)

instaloader/__main__.py

@@ -351,6 +351,13 @@ def main():
                             '--dirname-pattern. {profile} is replaced by the profile name,'
                             '{target} is replaced by the target you specified, i.e. either :feed'
                             '#hashtag or the profile name. Defaults to \'{date_utc}_UTC\'')
+    g_how.add_argument('--resume-prefix', metavar='PREFIX',
+                       help='Prefix for filenames that are used to save the information to resume an interrupted '
+                            'download.')
+    g_how.add_argument('--no-resume', action='store_true',
+                       help='Do not resume a previously-aborted download iteration, and do not save such information '
+                            'when interrupted.')
+    g_how.add_argument('--use-aged-resume-files', action='store_true', help=SUPPRESS)
     g_how.add_argument('--user-agent',
                        help='User Agent to use for HTTP requests. Defaults to \'{}\'.'.format(default_user_agent()))
     g_how.add_argument('-S', '--no-sleep', action='store_true', help=SUPPRESS)
@@ -358,10 +365,7 @@ def main():
                        help='Maximum number of connection attempts until a request is aborted. Defaults to 3. If a '
                             'connection fails, it can be manually skipped by hitting CTRL+C. Set this to 0 to retry '
                             'infinitely.')
-    g_how.add_argument('--commit-mode', action='store_true',
+    g_how.add_argument('--commit-mode', action='store_true', help=SUPPRESS)
-                       help='Tries to ensure downloaded images avoid corruption in case of unexpected interruption. '
-                       'If the last picture is corrupted, Instaloader will fix the picture the next time it is run. '
-                       'Requires the JSON metadata to be saved.')
     g_how.add_argument('--request-timeout', metavar='N', type=float,
                        help='seconds to wait before timing out a connection request')
 
@@ -397,6 +401,10 @@ def main():
                 raise SystemExit("--no-captions and --post-metadata-txt or --storyitem-metadata-txt given; "
                                  "That contradicts.")
 
+        if args.no_resume and args.resume_prefix:
+            raise SystemExit("--no-resume and --resume-prefix given; That contradicts.")
+        resume_prefix = (args.resume_prefix if args.resume_prefix else 'iterator') if not args.no_resume else None
+
         if args.no_pictures and args.fast_update:
             raise SystemExit('--no-pictures and --fast-update cannot be used together.')
 
@@ -405,9 +413,6 @@ def main():
         download_posts = not (args.no_posts or args.stories_only or args.profile_pic_only)
         download_stories = args.stories or args.stories_only
 
-        if args.commit_mode and args.no_metadata_json:
-            raise SystemExit('--commit-mode requires JSON metadata to be saved.')
-
         loader = Instaloader(sleep=not args.no_sleep, quiet=args.quiet, user_agent=args.user_agent,
                              dirname_pattern=args.dirname_pattern, filename_pattern=args.filename_pattern,
                              download_pictures=not args.no_pictures,
@@ -419,7 +424,8 @@ def main():
                              storyitem_metadata_txt_pattern=storyitem_metadata_txt_pattern,
                              max_connection_attempts=args.max_connection_attempts,
                              request_timeout=args.request_timeout,
-                             commit_mode=args.commit_mode)
+                             resume_prefix=resume_prefix,
+                             check_resume_bbd=not args.use_aged_resume_files)
         _main(loader,
               args.profile,
               username=args.login.lower() if args.login is not None else None,

instaloader/instaloader.py

@@ -1,6 +1,5 @@
 import getpass
 import json
-import lzma
 import os
 import platform
 import re
@@ -20,9 +19,10 @@ import requests
 import urllib3  # type: ignore
 
 from .exceptions import *
-from .instaloadercontext import InstaloaderContext
+from .instaloadercontext import InstaloaderContext, RateController
+from .nodeiterator import NodeIterator, resumable_iteration
 from .structures import (Hashtag, Highlight, JsonExportable, Post, PostLocation, Profile, Story, StoryItem,
-                         save_structure_to_file, load_structure_from_file)
+                         load_structure_from_file, save_structure_to_file)
 
 
 def get_default_session_filename(username: str) -> str:
@@ -153,8 +153,10 @@ class Instaloader:
        txt file.
     :param storyitem_metadata_txt_pattern: :option:`--storyitem-metadata-txt`, default is empty (=none)
     :param max_connection_attempts: :option:`--max-connection-attempts`
-    :param commit_mode: :option:`--commit-mode`
     :param request_timeout: :option:`--request-timeout`, set per-request timeout (seconds)
+    :param rate_controller: Generator for a :class:`RateController` to override rate controlling behavior
+    :param resume_prefix: :option:`--resume-prefix`, or None for :option:`--no-resume`.
+    :param check_resume_bbd: Whether to check the date of expiry of resume files and reject them if expired.
 
     .. attribute:: context
 
@@ -178,9 +180,12 @@ class Instaloader:
                  storyitem_metadata_txt_pattern: str = None,
                  max_connection_attempts: int = 3,
                  request_timeout: Optional[float] = None,
-                 commit_mode: bool = False):
+                 rate_controller: Optional[Callable[[InstaloaderContext], RateController]] = None,
+                 resume_prefix: Optional[str] = "iterator",
+                 check_resume_bbd: bool = True):
 
-        self.context = InstaloaderContext(sleep, quiet, user_agent, max_connection_attempts, request_timeout)
+        self.context = InstaloaderContext(sleep, quiet, user_agent, max_connection_attempts,
+                                          request_timeout, rate_controller)
 
         # configuration parameters
         self.dirname_pattern = dirname_pattern or "{target}"
@@ -196,12 +201,8 @@ class Instaloader:
             else post_metadata_txt_pattern
         self.storyitem_metadata_txt_pattern = '' if storyitem_metadata_txt_pattern is None \
             else storyitem_metadata_txt_pattern
-        self.commit_mode = commit_mode
+        self.resume_prefix = resume_prefix
-        if self.commit_mode and not self.save_metadata:
+        self.check_resume_bbd = check_resume_bbd
-            raise InvalidArgumentException("Commit mode requires JSON metadata to be saved.")
-
-        # Used to keep state in commit mode
-        self._committed = None  # type: Optional[bool]
 
     @contextmanager
     def anonymous_copy(self):
@@ -223,7 +224,8 @@ class Instaloader:
             storyitem_metadata_txt_pattern=self.storyitem_metadata_txt_pattern,
             max_connection_attempts=self.context.max_connection_attempts,
             request_timeout=self.context.request_timeout,
-            commit_mode=self.commit_mode)
+            resume_prefix=self.resume_prefix,
+            check_resume_bbd=self.check_resume_bbd)
         yield new_loader
         self.context.error_log.extend(new_loader.context.error_log)
         new_loader.context.error_log = []  # avoid double-printing of errors
@@ -249,15 +251,9 @@ class Instaloader:
         if filename_suffix is not None:
             filename += '_' + filename_suffix
         filename += '.' + file_extension
-        # A post is considered "commited" if the json file exists and is not malformed.
+        if os.path.isfile(filename):
-        if self.commit_mode:
+            self.context.log(filename + ' exists', end=' ', flush=True)
-            if self._committed and os.path.isfile(filename):
+            return False
-                self.context.log(filename + ' exists', end=' ', flush=True)
-                return False
-        else:
-            if os.path.isfile(filename):
-                self.context.log(filename + ' exists', end=' ', flush=True)
-                return False
         self.context.get_and_write_raw(url, filename)
         os.utime(filename, (datetime.now().timestamp(), mtime.timestamp()))
         return True
@@ -369,6 +365,24 @@ class Instaloader:
         os.utime(filename, (datetime.now().timestamp(), mtime.timestamp()))
         self.context.log('geo', end=' ', flush=True)
 
+    def format_filename_within_target_path(self,
+                                           target: Union[str, Path],
+                                           owner_profile: Optional[Profile],
+                                           identifier: str,
+                                           name_suffix: str,
+                                           extension: str):
+        """Returns a filename within the target path.
+
+        .. versionadded:: 4.5"""
+        if ((format_string_contains_key(self.dirname_pattern, 'profile') or
+             format_string_contains_key(self.dirname_pattern, 'target'))):
+            profile_str = owner_profile.username.lower() if owner_profile is not None else target
+            return os.path.join(self.dirname_pattern.format(profile=profile_str, target=target),
+                                '{0}_{1}.{2}'.format(identifier, name_suffix, extension))
+        else:
+            return os.path.join(self.dirname_pattern.format(),
+                                '{0}_{1}_{2}.{3}'.format(target, identifier, name_suffix, extension))
+
     @_retry_on_connection_error
     def download_title_pic(self, url: str, target: Union[str, Path], name_suffix: str, owner_profile: Optional[Profile],
                            _attempt: int = 1) -> None:
@@ -389,16 +403,7 @@ class Instaloader:
         else:
             pic_bytes = http_response.content
             pic_identifier = md5(pic_bytes).hexdigest()[:16]
-        pic_extension = 'jpg'
+        filename = self.format_filename_within_target_path(target, owner_profile, pic_identifier, name_suffix, 'jpg')
-        if ((format_string_contains_key(self.dirname_pattern, 'profile') or
-             format_string_contains_key(self.dirname_pattern, 'target'))):
-            profile_str = owner_profile.username.lower() if owner_profile is not None else target
-            filename = os.path.join(self.dirname_pattern.format(profile=profile_str,
-                                                                target=target),
-                                    '{0}_{1}.{2}'.format(pic_identifier, name_suffix, pic_extension))
-        else:
-            filename = os.path.join(self.dirname_pattern.format(),
-                                    '{0}_{1}_{2}.{3}'.format(target, pic_identifier, name_suffix, pic_extension))
         content_length = http_response.headers.get('Content-Length', None)
         if os.path.isfile(filename) and (not self.context.is_logged_in or
                                          (content_length is not None and
@@ -506,7 +511,6 @@ class Instaloader:
 
         # Download the image(s) / video thumbnail and videos within sidecars if desired
         downloaded = True
-        self._committed = self.check_if_committed(filename)
         if self.download_pictures:
             if post.typename == 'GraphSidecar':
                 edge_number = 1
@@ -719,59 +723,75 @@ class Instaloader:
                             fast_update: bool = False,
                             post_filter: Optional[Callable[[Post], bool]] = None,
                             max_count: Optional[int] = None,
-                            total_count: Optional[int] = None) -> None:
+                            total_count: Optional[int] = None,
+                            owner_profile: Optional[Profile] = None) -> None:
         """
         Download the Posts returned by given Post Iterator.
 
         ..versionadded:: 4.4
 
+        ..versionchanged:: 4.5
+           Transparently resume an aborted operation if `posts` is a :class:`NodeIterator`.
+
         :param posts: Post Iterator to loop through.
-        :param target: Target name
+        :param target: Target name.
-        :param fast_update: :option:`--fast-update`
+        :param fast_update: :option:`--fast-update`.
-        :param post_filter: :option:`--post-filter`
+        :param post_filter: :option:`--post-filter`.
-        :param max_count: Maximum count of Posts to download (:option:`--count`)
+        :param max_count: Maximum count of Posts to download (:option:`--count`).
-        :param total_count: Total number of posts returned by given iterator
+        :param total_count: Total number of posts returned by given iterator.
+        :param owner_profile: Associated profile, if any.
         """
-        for number, post in enumerate(posts):
+        displayed_count = (max_count if total_count is None or max_count is not None and max_count < total_count
-            if max_count is not None and number >= max_count:
+                           else total_count)
-                break
+        with resumable_iteration(
-            if total_count is not None:
+                context=self.context,
-                self.context.log("[{0:{w}d}/{1:{w}d}] ".format(number + 1, total_count,
+                iterator=posts,
-                                                               w=len(str(total_count))),
+                load=load_structure_from_file,
-                                 end="", flush=True)
+                save=save_structure_to_file,
-            else:
+                format_path=lambda magic: self.format_filename_within_target_path(
-                if max_count is not None:
+                    target, owner_profile, self.resume_prefix or '', magic, 'json.xz'
-                    self.context.log("[{0:{w}d}/{1:{w}d}] ".format(number + 1, max_count,
+                ),
-                                                                   w=len(str(max_count))),
+                check_bbd=self.check_resume_bbd,
+                enabled=self.resume_prefix is not None
+        ) as resume_info:
+            is_resuming, start_index = resume_info
+            for number, post in enumerate(posts):
+                if max_count is not None and number + start_index >= max_count:
+                    break
+                if displayed_count is not None:
+                    self.context.log("[{0:{w}d}/{1:{w}d}] ".format(number + start_index + 1, displayed_count,
+                                                                   w=len(str(displayed_count))),
                                      end="", flush=True)
                 else:
-                    self.context.log("[{:3d}] ".format(number + 1), end="", flush=True)
+                    self.context.log("[{:3d}] ".format(number + start_index + 1), end="", flush=True)
-            if post_filter is not None:
+                if post_filter is not None:
-                try:
-                    if not post_filter(post):
-                        self.context.log("{} skipped".format(post))
-                        continue
-                except (InstaloaderException, KeyError, TypeError) as err:
-                    self.context.error("{} skipped. Filter evaluation failed: {}".format(post, err))
-                    continue
-            with self.context.error_catcher("Download {} of {}".format(post, target)):
-                # The PostChangedException gets raised if the Post's id/shortcode changed while obtaining
-                # additional metadata. This is most likely the case if a HTTP redirect takes place while
-                # resolving the shortcode URL.
-                # The `post_changed` variable keeps the fast-update functionality alive: A Post which is
-                # obained after a redirect has probably already been downloaded as a previous Post of the
-                # same Profile.
-                # Observed in issue #225: https://github.com/instaloader/instaloader/issues/225
-                post_changed = False
-                while True:
                     try:
-                        downloaded = self.download_post(post, target=target)
+                        if not post_filter(post):
-                        break
+                            self.context.log("{} skipped".format(post))
-                    except PostChangedException:
+                            continue
-                        post_changed = True
+                    except (InstaloaderException, KeyError, TypeError) as err:
+                        self.context.error("{} skipped. Filter evaluation failed: {}".format(post, err))
                         continue
-                if fast_update and not downloaded and not post_changed:
+                with self.context.error_catcher("Download {} of {}".format(post, target)):
-                    break
+                    # The PostChangedException gets raised if the Post's id/shortcode changed while obtaining
+                    # additional metadata. This is most likely the case if a HTTP redirect takes place while
+                    # resolving the shortcode URL.
+                    # The `post_changed` variable keeps the fast-update functionality alive: A Post which is
+                    # obained after a redirect has probably already been downloaded as a previous Post of the
+                    # same Profile.
+                    # Observed in issue #225: https://github.com/instaloader/instaloader/issues/225
+                    post_changed = False
+                    while True:
+                        try:
+                            downloaded = self.download_post(post, target=target)
+                            break
+                        except PostChangedException:
+                            post_changed = True
+                            continue
+                    if fast_update and not downloaded and not post_changed:
+                        # disengage fast_update for first post when resuming
+                        if not is_resuming or number > 0:
+                            break
 
     @_requires_login
     def get_feed_posts(self) -> Iterator[Post]:
@@ -831,8 +851,10 @@ class Instaloader:
         """
         self.context.log("Retrieving saved posts...")
         assert self.context.username is not None  # safe due to @_requires_login; required by typechecker
-        self.posts_download_loop(Profile.from_username(self.context, self.context.username).get_saved_posts(), ":saved",
+        node_iterator = Profile.from_username(self.context, self.context.username).get_saved_posts()
-                                 fast_update, post_filter, max_count=max_count)
+        self.posts_download_loop(node_iterator, ":saved",
+                                 fast_update, post_filter,
+                                 max_count=max_count, total_count=node_iterator.count)
 
     @_requires_login
     def get_location_posts(self, location: str) -> Iterator[Post]:
@@ -887,18 +909,20 @@ class Instaloader:
                                  max_count=max_count)
 
     @_requires_login
-    def get_explore_posts(self) -> Iterator[Post]:
+    def get_explore_posts(self) -> NodeIterator[Post]:
         """Get Posts which are worthy of exploring suggested by Instagram.
 
         :return: Iterator over Posts of the user's suggested posts.
+        :rtype: NodeIterator[Post]
         :raises LoginRequiredException: If called without being logged in.
         """
-        data = self.context.get_json('explore/', {})
+        return NodeIterator(
-        yield from (Post(self.context, node)
+            self.context,
-                    for node in self.context.graphql_node_list("df0dcc250c2b18d9fd27c5581ef33c7c",
+            'df0dcc250c2b18d9fd27c5581ef33c7c',
-                                                               {}, 'https://www.instagram.com/explore/',
+            lambda d: d['data']['user']['edge_web_discover_media'],
-                                                               lambda d: d['data']['user']['edge_web_discover_media'],
+            lambda n: Post(self.context, n),
-                                                               data.get('rhx_gis')))
+            query_referer='https://www.instagram.com/explore/',
+        )
 
     def get_hashtag_posts(self, hashtag: str) -> Iterator[Post]:
         """Get Posts associated with a #hashtag.
@@ -969,7 +993,7 @@ class Instaloader:
         .. versionadded:: 4.3"""
         self.context.log("Retrieving IGTV videos for profile {}.".format(profile.username))
         self.posts_download_loop(profile.get_igtv_posts(), profile.username, fast_update, post_filter,
-                                 total_count=profile.igtvcount)
+                                 total_count=profile.igtvcount, owner_profile=profile)
 
     def _get_id_filename(self, profile_name: str) -> str:
         if ((format_string_contains_key(self.dirname_pattern, 'profile') or
@@ -1124,7 +1148,7 @@ class Instaloader:
                 if posts:
                     self.context.log("Retrieving posts from profile {}.".format(profile_name))
                     self.posts_download_loop(profile.get_posts(), profile_name, fast_update, post_filter,
-                                             total_count=profile.mediacount)
+                                             total_count=profile.mediacount, owner_profile=profile)
 
         if stories and profiles:
             with self.context.error_catcher("Download stories"):
@@ -1204,26 +1228,7 @@ class Instaloader:
         # Iterate over pictures and download them
         self.context.log("Retrieving posts from profile {}.".format(profile_name))
         self.posts_download_loop(profile.get_posts(), profile_name, fast_update, post_filter,
-                                 total_count=profile.mediacount)
+                                 total_count=profile.mediacount, owner_profile=profile)
-
-    def check_if_committed(self, filename: str) -> bool:
-        """Checks to see if the current post has been committed.
-
-        A post is considered committed if its json metadata file exists and is not malformed.
-
-        .. versionadded:: 4.2
-        """
-        if os.path.isfile(filename + '.json.xz'):
-            filename += '.json.xz'
-        elif os.path.isfile(filename + '.json'):
-            filename += '.json'
-        else:
-            return False
-        try:
-            load_structure_from_file(self.context, filename)
-            return True
-        except (FileNotFoundError, lzma.LZMAError, json.decoder.JSONDecodeError):
-            return False
 
     def interactive_login(self, username: str) -> None:
         """Logs in and internally stores session, asking user for password interactively.

instaloader/instaloadercontext.py

@@ -1,5 +1,6 @@
 import hashlib
 import json
+import os
 import pickle
 import random
 import re
@@ -52,7 +53,8 @@ class InstaloaderContext:
     """
 
     def __init__(self, sleep: bool = True, quiet: bool = False, user_agent: Optional[str] = None,
-                 max_connection_attempts: int = 3, request_timeout: Optional[float] = None):
+                 max_connection_attempts: int = 3, request_timeout: Optional[float] = None,
+                 rate_controller: Optional[Callable[["InstaloaderContext"], "RateController"]] = None):
 
         self.user_agent = user_agent if user_agent is not None else default_user_agent()
         self.request_timeout = request_timeout
@@ -68,9 +70,7 @@ class InstaloaderContext:
         # error log, filled with error() and printed at the end of Instaloader.main()
         self.error_log = []                      # type: List[str]
 
-        # For the adaption of sleep intervals (rate control)
+        self._rate_controller = rate_controller(self) if rate_controller is not None else RateController(self)
-        self._graphql_query_timestamps = dict()  # type: Dict[str, List[float]]
-        self._graphql_earliest_next_request_time = 0.0
 
         # Can be set to True for testing, disables supression of InstaloaderContext._error_catcher
         self.raise_all_errors = False
@@ -294,77 +294,6 @@ class InstaloaderContext:
         if self.sleep:
             time.sleep(min(random.expovariate(0.7), 5.0))
 
-    def _dump_query_timestamps(self, current_time: float):
-        """Output the number of GraphQL queries grouped by their query_hash within the last time."""
-        windows = [10, 11, 15, 20, 30, 60]
-        print("GraphQL requests:", file=sys.stderr)
-        for query_hash, times in self._graphql_query_timestamps.items():
-            print("  {}".format(query_hash), file=sys.stderr)
-            for window in windows:
-                reqs_in_sliding_window = sum(t > current_time - window * 60 for t in times)
-                print("    last {} minutes: {} requests".format(window, reqs_in_sliding_window), file=sys.stderr)
-
-    def _graphql_request_count_per_sliding_window(self, query_hash: str) -> int:
-        """Return how many GraphQL requests can be done within the sliding window."""
-        if self.is_logged_in:
-            max_reqs = {'1cb6ec562846122743b61e492c85999f': 20,
-                        '33ba35852cb50da46f5b5e889df7d159': 20,
-                        'iphone': 100,
-                        'other': 100}
-        else:
-            max_reqs = {'1cb6ec562846122743b61e492c85999f': 200,
-                        '33ba35852cb50da46f5b5e889df7d159': 200,
-                        'other': 200}
-        return max_reqs.get(query_hash) or min(max_reqs.values())
-
-    def _graphql_query_waittime(self, query_hash: str, current_time: float, untracked_queries: bool = False) -> float:
-        """Calculate time needed to wait before GraphQL query can be executed."""
-        sliding_window = 660
-        if query_hash not in self._graphql_query_timestamps:
-            self._graphql_query_timestamps[query_hash] = []
-        self._graphql_query_timestamps[query_hash] = list(filter(lambda t: t > current_time - 60 * 60,
-                                                                 self._graphql_query_timestamps[query_hash]))
-        reqs_in_sliding_window = list(filter(lambda t: t > current_time - sliding_window,
-                                             self._graphql_query_timestamps[query_hash]))
-        count_per_sliding_window = self._graphql_request_count_per_sliding_window(query_hash)
-        if len(reqs_in_sliding_window) < count_per_sliding_window and not untracked_queries:
-            return max(0, self._graphql_earliest_next_request_time - current_time)
-        next_request_time = min(reqs_in_sliding_window) + sliding_window + 6
-        if untracked_queries:
-            self._graphql_earliest_next_request_time = next_request_time
-        return round(max(next_request_time, self._graphql_earliest_next_request_time) - current_time)
-
-    def _ratecontrol_graphql_query(self, query_hash: str, untracked_queries: bool = False):
-        """Called before a GraphQL query is made in order to stay within Instagram's rate limits.
-
-        :param query_hash: The query_hash parameter of the query.
-        :param untracked_queries: True, if 429 has been returned to apply 429 logic.
-        """
-        if not untracked_queries:
-            waittime = self._graphql_query_waittime(query_hash, time.monotonic(), untracked_queries)
-            assert waittime >= 0
-            if waittime > 10:
-                self.log('\nToo many queries in the last time. Need to wait {} seconds, until {:%H:%M}.'
-                         .format(waittime, datetime.now() + timedelta(seconds=waittime)))
-            time.sleep(waittime)
-            if query_hash not in self._graphql_query_timestamps:
-                self._graphql_query_timestamps[query_hash] = [time.monotonic()]
-            else:
-                self._graphql_query_timestamps[query_hash].append(time.monotonic())
-        else:
-            text_for_429 = ("HTTP error code 429 was returned because too many queries occurred in the last time. "
-                            "Please do not use Instagram in your browser or run multiple instances of Instaloader "
-                            "in parallel.")
-            print(textwrap.fill(text_for_429), file=sys.stderr)
-            current_time = time.monotonic()
-            waittime = self._graphql_query_waittime(query_hash, current_time, untracked_queries)
-            assert waittime >= 0
-            if waittime > 10:
-                self.log('The request will be retried in {} seconds, at {:%H:%M}.'
-                         .format(waittime, datetime.now() + timedelta(seconds=waittime)))
-            self._dump_query_timestamps(current_time)
-            time.sleep(waittime)
-
     def get_json(self, path: str, params: Dict[str, Any], host: str = 'www.instagram.com',
                  session: Optional[requests.Session] = None, _attempt=1) -> Dict[str, Any]:
         """JSON request to Instagram.
@@ -385,11 +314,11 @@ class InstaloaderContext:
         try:
             self.do_sleep()
             if is_graphql_query:
-                self._ratecontrol_graphql_query(params['query_hash'])
+                self._rate_controller.wait_before_query(params['query_hash'])
             if is_iphone_query:
-                self._ratecontrol_graphql_query('iphone')
+                self._rate_controller.wait_before_query('iphone')
             if is_other_query:
-                self._ratecontrol_graphql_query('other')
+                self._rate_controller.wait_before_query('other')
             resp = sess.get('https://{0}/{1}'.format(host, path), params=params, allow_redirects=False)
             while resp.is_redirect:
                 redirect_url = resp.headers['location']
@@ -445,12 +374,13 @@ class InstaloaderContext:
                     raise ConnectionException(error_string) from err
             self.error(error_string + " [retrying; skip with ^C]", repeat_at_end=False)
             try:
-                if is_graphql_query and isinstance(err, TooManyRequestsException):
+                if isinstance(err, TooManyRequestsException):
-                    self._ratecontrol_graphql_query(params['query_hash'], untracked_queries=True)
+                    if is_graphql_query:
-                if is_iphone_query and isinstance(err, TooManyRequestsException):
+                        self._rate_controller.handle_429(params['query_hash'])
-                    self._ratecontrol_graphql_query('iphone', untracked_queries=True)
+                    if is_iphone_query:
-                if is_other_query and isinstance(err, TooManyRequestsException):
+                        self._rate_controller.handle_429('iphone')
-                    self._ratecontrol_graphql_query('other', untracked_queries=True)
+                    if is_other_query:
+                        self._rate_controller.handle_429('other')
                 return self.get_json(path=path, params=params, host=host, session=sess, _attempt=_attempt + 1)
             except KeyboardInterrupt:
                 self.error("[skipped by user]", repeat_at_end=False)
@@ -498,7 +428,12 @@ class InstaloaderContext:
                           edge_extractor: Callable[[Dict[str, Any]], Dict[str, Any]],
                           rhx_gis: Optional[str] = None,
                           first_data: Optional[Dict[str, Any]] = None) -> Iterator[Dict[str, Any]]:
-        """Retrieve a list of GraphQL nodes."""
+        """
+        Retrieve a list of GraphQL nodes.
+
+        ..deprecated:: 4.5
+           Use :class:`NodeIterator` instead, which provides more functionality.
+        """
 
         def _query():
             query_variables['first'] = self._graphql_page_length
@@ -547,11 +482,12 @@ class InstaloaderContext:
 
         .. versionadded:: 4.2.1"""
         self.log(filename, end=' ', flush=True)
-        with open(filename, 'wb') as file:
+        with open(filename + '.temp', 'wb') as file:
             if isinstance(resp, requests.Response):
                 shutil.copyfileobj(resp.raw, file)
             else:
                 file.write(resp)
+        os.rename(filename + '.temp', filename)
 
     def get_raw(self, url: str, _attempt=1) -> requests.Response:
         """Downloads a file anonymously.
@@ -593,3 +529,97 @@ class InstaloaderContext:
         if self._root_rhx_gis is None:
             self._root_rhx_gis = self.get_json('', {}).get('rhx_gis', '')
         return self._root_rhx_gis or None
+
+
+class RateController:
+    """
+    Class providing request tracking and rate controlling to stay within rate limits.
+
+    It can be overridden to change Instaloader's behavior regarding rate limits, for example to raise a custom
+    exception when the rate limit is hit::
+
+       import instaloader
+
+       class MyRateController(instaloader.RateController):
+           def sleep(self, secs):
+               raise MyCustomException()
+
+       L = instaloader.Instaloader(rate_controller=lambda ctx: MyRateController(ctx))
+    """
+
+    def __init__(self, context: InstaloaderContext):
+        self._context = context
+        self._graphql_query_timestamps = dict()  # type: Dict[str, List[float]]
+        self._graphql_earliest_next_request_time = 0.0
+
+    def sleep(self, secs: float):
+        """Wait given number of seconds."""
+        # Not static, to allow for the behavior of this method to depend on context-inherent properties, such as
+        # whether we are logged in.
+        # pylint:disable=no-self-use
+        time.sleep(secs)
+
+    def _dump_query_timestamps(self, current_time: float):
+        windows = [10, 11, 15, 20, 30, 60]
+        print("GraphQL requests:", file=sys.stderr)
+        for query_hash, times in self._graphql_query_timestamps.items():
+            print("  {}".format(query_hash), file=sys.stderr)
+            for window in windows:
+                reqs_in_sliding_window = sum(t > current_time - window * 60 for t in times)
+                print("    last {} minutes: {} requests".format(window, reqs_in_sliding_window), file=sys.stderr)
+
+    def count_per_sliding_window(self, query_type: str) -> int:
+        """Return how many GraphQL requests can be done within the sliding window."""
+        # Not static, to allow for the count_per_sliding_window to depend on context-inherent properties, such as
+        # whether we are logged in.
+        # pylint:disable=no-self-use,unused-argument
+        return 200
+
+    def query_waittime(self, query_type: str, current_time: float, untracked_queries: bool = False) -> float:
+        """Calculate time needed to wait before GraphQL query can be executed."""
+        sliding_window = 660
+        if query_type not in self._graphql_query_timestamps:
+            self._graphql_query_timestamps[query_type] = []
+        self._graphql_query_timestamps[query_type] = list(filter(lambda t: t > current_time - 60 * 60,
+                                                                 self._graphql_query_timestamps[query_type]))
+        reqs_in_sliding_window = list(filter(lambda t: t > current_time - sliding_window,
+                                             self._graphql_query_timestamps[query_type]))
+        count_per_sliding_window = self.count_per_sliding_window(query_type)
+        if len(reqs_in_sliding_window) < count_per_sliding_window and not untracked_queries:
+            return max(0.0, self._graphql_earliest_next_request_time - current_time)
+        next_request_time = min(reqs_in_sliding_window) + sliding_window + 6
+        if untracked_queries:
+            self._graphql_earliest_next_request_time = next_request_time
+        return max(next_request_time, self._graphql_earliest_next_request_time) - current_time
+
+    def wait_before_query(self, query_type: str) -> None:
+        """This method is called before a query to Instagram. It calls :meth:`RateController.sleep` to wait
+        until the request can be made."""
+        waittime = self.query_waittime(query_type, time.monotonic(), False)
+        assert waittime >= 0
+        if waittime > 15:
+            self._context.log("\nToo many queries in the last time. Need to wait {} seconds, until {:%H:%M}."
+                              .format(round(waittime), datetime.now() + timedelta(seconds=waittime)))
+        if waittime > 0:
+            self.sleep(waittime)
+        if query_type not in self._graphql_query_timestamps:
+            self._graphql_query_timestamps[query_type] = [time.monotonic()]
+        else:
+            self._graphql_query_timestamps[query_type].append(time.monotonic())
+
+    def handle_429(self, query_type: str) -> None:
+        """This method is called to handle a 429 Too Many Requests response. It calls :meth:`RateController.sleep` to
+         wait until we can repeat the same request."""
+        text_for_429 = ("HTTP error code 429 was returned because too many queries occurred in the last time. "
+                        "Please do not use Instagram in your browser or run multiple instances of Instaloader "
+                        "in parallel.")
+        print(textwrap.fill(text_for_429), file=sys.stderr)
+        current_time = time.monotonic()
+        waittime = self.query_waittime(query_type, current_time, True)
+        assert waittime >= 0
+        if waittime > 15:
+            self._context.log("The request will be retried in {} seconds, at {:%H:%M}."
+                              .format(round(waittime), datetime.now() + timedelta(seconds=waittime)))
+        self._dump_query_timestamps(current_time)
+        if waittime > 0:
+            self.sleep(waittime)

instaloader/nodeiterator.py

@@ -0,0 +1,261 @@
+import base64
+import hashlib
+import json
+import os
+from contextlib import contextmanager
+from datetime import datetime, timedelta
+from lzma import LZMAError
+from typing import Any, Callable, Dict, Iterator, NamedTuple, Optional, Tuple, TypeVar
+
+from .exceptions import InvalidArgumentException, QueryReturnedBadRequestException
+from .instaloadercontext import InstaloaderContext
+
+FrozenNodeIterator = NamedTuple('FrozenNodeIterator',
+                                [('query_hash', str),
+                                 ('query_variables', Dict),
+                                 ('query_referer', Optional[str]),
+                                 ('context_username', Optional[str]),
+                                 ('total_index', int),
+                                 ('best_before', Optional[float]),
+                                 ('remaining_data', Optional[Dict])])
+FrozenNodeIterator.__doc__ = \
+    """A serializable representation of a :class:`NodeIterator` instance, saving its iteration state."""
+FrozenNodeIterator.query_hash.__doc__ = """The GraphQL ``query_hash`` parameter."""
+FrozenNodeIterator.query_variables.__doc__ = """The GraphQL ``query_variables`` parameter."""
+FrozenNodeIterator.query_referer.__doc__ = """The HTTP referer used for the GraphQL query."""
+FrozenNodeIterator.context_username.__doc__ = """The username who created the iterator, or ``None``."""
+FrozenNodeIterator.total_index.__doc__ = """Number of items that have already been returned."""
+FrozenNodeIterator.best_before.__doc__ = """Date when parts of the stored nodes might have expired."""
+FrozenNodeIterator.remaining_data.__doc__ = \
+    """The already-retrieved, yet-unprocessed ``edges`` and the ``page_info`` at time of freezing."""
+
+
+T = TypeVar('T')
+
+
+class NodeIterator(Iterator[T]):
+    """
+    Iterate the nodes within edges in a GraphQL pagination. Instances of this class are returned by many (but not all)
+    of Instaloader's :class:`Post`-returning functions (such as :meth:`Profile.get_posts` etc.).
+
+    What makes this iterator special is its ability to freeze/store its current state, e.g. to interrupt an iteration,
+    and later thaw/resume from where it left off.
+
+    You can freeze a NodeIterator with :meth:`NodeIterator.freeze`::
+
+       post_iterator = profile.get_posts()
+       try:
+           for post in post_iterator:
+               do_something_with(post)
+       except KeyboardInterrupt:
+           save("resume_information.json", post_iterator.freeze())
+
+    and later reuse it with :meth:`NodeIterator.thaw` on an equally-constructed NodeIterator::
+
+       post_iterator = profile.get_posts()
+       post_iterator.thaw(load("resume_information.json"))
+
+    A :class:`FrozenNodeIterator` can only be thawn with a matching NodeIterator, i.e. a NodeIterator instance that has
+    been constructed with the same parameters as the instance that is represented by the :class:`FrozenNodeIterator` in
+    question. This is to ensure that an iteration cannot be resumed in a wrong, unmatching loop. As a quick way to
+    distinguish iterators that are saved e.g. in files, there is the :attr:`NodeIterator.magic` string: Two
+    NodeIterators are matching if they have the same magic.
+
+    See also :func:`resumable_iteration` for a high-level context manager that handles a resumable iteration.
+    """
+
+    _graphql_page_length = 50
+    shelf_life = timedelta(days=29)
+
+    def __init__(self,
+                 context: InstaloaderContext,
+                 query_hash: str,
+                 edge_extractor: Callable[[Dict[str, Any]], Dict[str, Any]],
+                 node_wrapper: Callable[[Dict], T],
+                 query_variables: Optional[Dict[str, Any]] = None,
+                 query_referer: Optional[str] = None,
+                 first_data: Optional[Dict[str, Any]] = None):
+        self._context = context
+        self._query_hash = query_hash
+        self._edge_extractor = edge_extractor
+        self._node_wrapper = node_wrapper
+        self._query_variables = query_variables if query_variables is not None else {}
+        self._query_referer = query_referer
+        self._data = first_data
+        self._page_index = 0
+        self._total_index = 0
+        self._best_before = (None if first_data is None else
+                             datetime.now() + NodeIterator.shelf_life)
+
+    def _query(self, after: Optional[str] = None) -> Dict:
+        pagination_variables = {'first': NodeIterator._graphql_page_length}  # type: Dict[str, Any]
+        if after is not None:
+            pagination_variables['after'] = after
+        try:
+            data = self._edge_extractor(
+                self._context.graphql_query(
+                    self._query_hash, {**self._query_variables, **pagination_variables}, self._query_referer
+                )
+            )
+            self._best_before = datetime.now() + NodeIterator.shelf_life
+            return data
+        except QueryReturnedBadRequestException:
+            new_page_length = int(NodeIterator._graphql_page_length / 2)
+            if new_page_length >= 12:
+                NodeIterator._graphql_page_length = new_page_length
+                self._context.error("HTTP Error 400 (Bad Request) on GraphQL Query. Retrying with shorter page length.",
+                                    repeat_at_end=False)
+                return self._query(after)
+            else:
+                raise
+
+    def __iter__(self):
+        return self
+
+    def __next__(self):
+        if self._data is None:
+            self._data = self._query()
+        if self._page_index < len(self._data['edges']):
+            node = self._data['edges'][self._page_index]['node']
+            page_index, total_index = self._page_index, self._total_index
+            try:
+                self._page_index += 1
+                self._total_index += 1
+            except KeyboardInterrupt:
+                self._page_index, self._total_index = page_index, total_index
+                raise
+            return self._node_wrapper(node)
+        if self._data['page_info']['has_next_page']:
+            query_response = self._query(self._data['page_info']['end_cursor'])
+            page_index, data = self._page_index, self._data
+            try:
+                self._page_index = 0
+                self._data = query_response
+            except KeyboardInterrupt:
+                self._page_index, self._data = page_index, data
+                raise
+            return self.__next__()
+        raise StopIteration()
+
+    @property
+    def count(self) -> Optional[int]:
+        """The ``count`` as returned by Instagram. This is not always the total count this iterator will yield."""
+        return self._data.get('count') if self._data is not None else None
+
+    @property
+    def total_index(self) -> int:
+        """Number of items that have already been returned."""
+        return self._total_index
+
+    @property
+    def magic(self) -> str:
+        """Magic string for easily identifying a matching iterator file for resuming (hash of some parameters)."""
+        if 'blake2b' not in hashlib.algorithms_available:
+            magic_hash = hashlib.new('sha224')
+        else:
+            # Use blake2b when possible, i.e. on Python >= 3.6.
+            magic_hash = hashlib.blake2b(digest_size=6)  # type:ignore  # pylint: disable=no-member
+        magic_hash.update(json.dumps(
+            [self._query_hash, self._query_variables, self._query_referer, self._context.username]
+        ).encode())
+        return base64.urlsafe_b64encode(magic_hash.digest()).decode()
+
+    def freeze(self) -> FrozenNodeIterator:
+        """Freeze the iterator for later resuming."""
+        remaining_data = None
+        if self._data is not None:
+            remaining_data = {**self._data,
+                              'edges': (self._data['edges'][(max(self._page_index - 1, 0)):])}
+        return FrozenNodeIterator(
+            query_hash=self._query_hash,
+            query_variables=self._query_variables,
+            query_referer=self._query_referer,
+            context_username=self._context.username,
+            total_index=max(self.total_index - 1, 0),
+            best_before=self._best_before.timestamp() if self._best_before else None,
+            remaining_data=remaining_data,
+        )
+
+    def thaw(self, frozen: FrozenNodeIterator) -> None:
+        """Use this iterator for resuming from earlier iteration."""
+        if self._total_index or self._page_index:
+            raise InvalidArgumentException("thaw() called on already-used iterator.")
+        if (self._query_hash != frozen.query_hash or
+                self._query_variables != frozen.query_variables or
+                self._query_referer != frozen.query_referer or
+                self._context.username != frozen.context_username):
+            raise InvalidArgumentException("Mismatching resume information.")
+        self._total_index = frozen.total_index
+        self._best_before = datetime.fromtimestamp(frozen.best_before) if frozen.best_before else None
+        self._data = frozen.remaining_data
+
+
+@contextmanager
+def resumable_iteration(context: InstaloaderContext,
+                        iterator: Iterator,
+                        load: Callable[[InstaloaderContext, str], Any],
+                        save: Callable[[FrozenNodeIterator, str], None],
+                        format_path: Callable[[str], str],
+                        check_bbd: bool = True,
+                        enabled: bool = True) -> Iterator[Tuple[bool, int]]:
+    """
+    High-level context manager to handle a resumable iteration that can be interrupted with a KeyboardInterrupt.
+
+    It can be used as follows to automatically load a previously-saved state into the iterator, save the iterator's
+    state when interrupted, and delete the resume file upon completion::
+
+       post_iterator = profile.get_posts()
+       with resumable_iteration(
+               context=L.context,
+               iterator=post_iterator,
+               load=lambda _, path: FrozenNodeIterator(**json.load(open(path))),
+               save=lambda fni, path: json.dump(fni._asdict(), open(path, 'w')),
+               format_path=lambda magic: "resume_info_{}.json".format(magic)
+       ) as resume_info:
+           is_resuming, start_index = resume_info
+           for post in post_iterator:
+               do_something_with(post)
+
+    It yields a tuple (is_resuming, start_index).
+
+    When the passed iterator is not a :class:`NodeIterator`, it behaves as if ``resumable_iteration`` was not used,
+    just executing the inner body.
+
+    :param context: The :class:`InstaloaderContext`.
+    :param iterator: The fresh :class:`NodeIterator`.
+    :param load: Loads a FrozenNodeIterator from given path. The object is ignored if it has a different type.
+    :param save: Saves the given FrozenNodeIterator to the given path.
+    :param format_path: Returns the path to the resume file for the given magic.
+    :param check_bbd: Whether to check the best before date and reject an expired FrozenNodeIterator.
+    :param enabled: Set to False to disable all functionality and simply execute the inner body.
+    """
+    if not enabled or not isinstance(iterator, NodeIterator):
+        yield False, 0
+        return
+    is_resuming = False
+    start_index = 0
+    resume_file_path = format_path(iterator.magic)
+    resume_file_exists = os.path.isfile(resume_file_path)
+    if resume_file_exists:
+        try:
+            fni = load(context, resume_file_path)
+            if not isinstance(fni, FrozenNodeIterator):
+                raise InvalidArgumentException("Invalid type.")
+            if check_bbd and fni.best_before and datetime.fromtimestamp(fni.best_before) < datetime.now():
+                raise InvalidArgumentException("\"Best before\" date exceeded.")
+            iterator.thaw(fni)
+            is_resuming = True
+            start_index = iterator.total_index
+            context.log("Resuming from {}.".format(resume_file_path))
+        except (InvalidArgumentException, LZMAError, json.decoder.JSONDecodeError) as exc:
+            context.error("Warning: Not resuming from {}: {}".format(resume_file_path, exc))
+    try:
+        yield is_resuming, start_index
+    except KeyboardInterrupt:
+        os.makedirs(os.path.dirname(resume_file_path), exist_ok=True)
+        save(iterator.freeze(), resume_file_path)
+        context.log("\nSaved resume information to {}.".format(resume_file_path))
+        raise
+    if resume_file_exists:
+        os.unlink(resume_file_path)
+        context.log("Iteration complete, deleted resume information file {}.".format(resume_file_path))

instaloader/structures.py

@@ -9,7 +9,7 @@ from typing import Any, Dict, Iterator, List, Optional, Union
 from . import __version__
 from .exceptions import *
 from .instaloadercontext import InstaloaderContext
-
+from .nodeiterator import FrozenNodeIterator, NodeIterator
 
 PostSidecarNode = namedtuple('PostSidecarNode', ['is_video', 'display_url', 'video_url'])
 PostSidecarNode.__doc__ = "Item of a Sidecar Post."
@@ -402,11 +402,14 @@ class Post:
                 # If the answer's metadata already contains all comments, don't do GraphQL requests to obtain them
                 yield from (_postcommentanswer(comment['node']) for comment in answer_edges)
                 return
-            yield from (_postcommentanswer(answer_node) for answer_node in
+            yield from NodeIterator(
-                        self._context.graphql_node_list("51fdd02b67508306ad4484ff574a0b62",
+                self._context,
-                                                        {'comment_id': node['id']},
+                '51fdd02b67508306ad4484ff574a0b62',
-                                                        'https://www.instagram.com/p/' + self.shortcode + '/',
+                lambda d: d['data']['comment']['edge_threaded_comments'],
-                                                        lambda d: d['data']['comment']['edge_threaded_comments']))
+                _postcommentanswer,
+                {'comment_id': node['id']},
+                'https://www.instagram.com/p/{0}/'.format(self.shortcode),
+            )
 
         def _postcomment(node):
             return PostComment(*_postcommentanswer(node),
@@ -422,12 +425,14 @@ class Post:
             # If the Post's metadata already contains all parent comments, don't do GraphQL requests to obtain them
             yield from (_postcomment(comment['node']) for comment in comment_edges)
             return
-        yield from (_postcomment(node) for node in
+        yield from NodeIterator(
-                    self._context.graphql_node_list(
+            self._context,
-                        "97b41c52301f77ce508f55e66d17620e",
+            '97b41c52301f77ce508f55e66d17620e',
-                        {'shortcode': self.shortcode},
+            lambda d: d['data']['shortcode_media']['edge_media_to_parent_comment'],
-                        'https://www.instagram.com/p/' + self.shortcode + '/',
+            _postcomment,
-                        lambda d: d['data']['shortcode_media']['edge_media_to_parent_comment']))
+            {'shortcode': self.shortcode},
+            'https://www.instagram.com/p/{0}/'.format(self.shortcode),
+        )
 
     def get_likes(self) -> Iterator['Profile']:
         """Iterate over all likes of the post. A :class:`Profile` instance of each likee is yielded."""
@@ -439,10 +444,14 @@ class Post:
             # If the Post's metadata already contains all likes, don't do GraphQL requests to obtain them
             yield from (Profile(self._context, like['node']) for like in likes_edges)
             return
-        yield from (Profile(self._context, node) for node in
+        yield from NodeIterator(
-                    self._context.graphql_node_list("1cb6ec562846122743b61e492c85999f", {'shortcode': self.shortcode},
+            self._context,
-                                                    'https://www.instagram.com/p/' + self.shortcode + '/',
+            '1cb6ec562846122743b61e492c85999f',
-                                                    lambda d: d['data']['shortcode_media']['edge_liked_by']))
+            lambda d: d['data']['shortcode_media']['edge_liked_by'],
+            lambda n: Profile(self._context, n),
+            {'shortcode': self.shortcode},
+            'https://www.instagram.com/p/{0}/'.format(self.shortcode),
+        )
 
     @property
     def is_sponsored(self) -> bool:
@@ -770,80 +779,110 @@ class Profile:
 	   Use :attr:`profile_pic_url`."""
         return self.profile_pic_url
 
-    def get_posts(self) -> Iterator[Post]:
+    def get_posts(self) -> NodeIterator[Post]:
-        """Retrieve all posts from a profile."""
+        """Retrieve all posts from a profile.
-        self._obtain_metadata()
-        yield from (Post(self._context, node, self) for node in
-                    self._context.graphql_node_list("472f257a40c653c64c666ce877d59d2b",
-                                                    {'id': self.userid},
-                                                    'https://www.instagram.com/{0}/'.format(self.username),
-                                                    lambda d: d['data']['user']['edge_owner_to_timeline_media'],
-                                                    first_data=self._metadata('edge_owner_to_timeline_media')))
 
-    def get_saved_posts(self) -> Iterator[Post]:
+        :rtype:NodeIterator[Post]"""
-        """Get Posts that are marked as saved by the user."""
+        self._obtain_metadata()
+        return NodeIterator(
+            self._context,
+            '472f257a40c653c64c666ce877d59d2b',
+            lambda d: d['data']['user']['edge_owner_to_timeline_media'],
+            lambda n: Post(self._context, n, self),
+            {'id': self.userid},
+            'https://www.instagram.com/{0}/'.format(self.username),
+            self._metadata('edge_owner_to_timeline_media'),
+        )
+
+    def get_saved_posts(self) -> NodeIterator[Post]:
+        """Get Posts that are marked as saved by the user.
+
+        :rtype:NodeIterator[Post]"""
 
         if self.username != self._context.username:
             raise LoginRequiredException("--login={} required to get that profile's saved posts.".format(self.username))
 
         self._obtain_metadata()
-        yield from (Post(self._context, node) for node in
+        return NodeIterator(
-                    self._context.graphql_node_list("f883d95537fbcd400f466f63d42bd8a1",
+            self._context,
-                                                    {'id': self.userid},
+            'f883d95537fbcd400f466f63d42bd8a1',
-                                                    'https://www.instagram.com/{0}/'.format(self.username),
+            lambda d: d['data']['user']['edge_saved_media'],
-                                                    lambda d: d['data']['user']['edge_saved_media'],
+            lambda n: Post(self._context, n),
-                                                    first_data=self._metadata('edge_saved_media')))
+            {'id': self.userid},
+            'https://www.instagram.com/{0}/'.format(self.username),
+            self._metadata('edge_saved_media'),
+        )
 
-    def get_tagged_posts(self) -> Iterator[Post]:
+    def get_tagged_posts(self) -> NodeIterator[Post]:
         """Retrieve all posts where a profile is tagged.
 
+        :rtype: NodeIterator[Post]
+
         .. versionadded:: 4.0.7"""
         self._obtain_metadata()
-        yield from (Post(self._context, node, self if int(node['owner']['id']) == self.userid else None) for node in
+        return NodeIterator(
-                    self._context.graphql_node_list("e31a871f7301132ceaab56507a66bbb7",
+            self._context,
-                                                    {'id': self.userid},
+            'e31a871f7301132ceaab56507a66bbb7',
-                                                    'https://www.instagram.com/{0}/'.format(self.username),
+            lambda d: d['data']['user']['edge_user_to_photos_of_you'],
-                                                    lambda d: d['data']['user']['edge_user_to_photos_of_you']))
+            lambda n: Post(self._context, n, self if int(n['owner']['id']) == self.userid else None),
+            {'id': self.userid},
+            'https://www.instagram.com/{0}/'.format(self.username),
+        )
 
-    def get_igtv_posts(self) -> Iterator[Post]:
+    def get_igtv_posts(self) -> NodeIterator[Post]:
         """Retrieve all IGTV posts.
 
+        :rtype: NodeIterator[Post]
+
         .. versionadded:: 4.3"""
         self._obtain_metadata()
-        yield from (Post(self._context, node, self) for node in
+        return NodeIterator(
-                    self._context.graphql_node_list('bc78b344a68ed16dd5d7f264681c4c76',
+            self._context,
-                                                    {'id': self.userid},
+            'bc78b344a68ed16dd5d7f264681c4c76',
-                                                    'https://www.instagram.com/{0}/channel/'.format(self.username),
+            lambda d: d['data']['user']['edge_felix_video_timeline'],
-                                                    lambda d: d['data']['user']['edge_felix_video_timeline'],
+            lambda n: Post(self._context, n, self),
-                                                    first_data=self._metadata('edge_felix_video_timeline')))
+            {'id': self.userid},
+            'https://www.instagram.com/{0}/channel/'.format(self.username),
+            self._metadata('edge_felix_video_timeline'),
+        )
 
-    def get_followers(self) -> Iterator['Profile']:
+    def get_followers(self) -> NodeIterator['Profile']:
         """
         Retrieve list of followers of given profile.
         To use this, one needs to be logged in and private profiles has to be followed.
+
+        :rtype:NodeIterator[Profile]
         """
         if not self._context.is_logged_in:
             raise LoginRequiredException("--login required to get a profile's followers.")
         self._obtain_metadata()
-        yield from (Profile(self._context, node) for node in
+        return NodeIterator(
-                    self._context.graphql_node_list("37479f2b8209594dde7facb0d904896a",
+            self._context,
-                                                    {'id': str(self.userid)},
+            '37479f2b8209594dde7facb0d904896a',
-                                                    'https://www.instagram.com/' + self.username + '/',
+            lambda d: d['data']['user']['edge_followed_by'],
-                                                    lambda d: d['data']['user']['edge_followed_by']))
+            lambda n: Profile(self._context, n),
+            {'id': str(self.userid)},
+            'https://www.instagram.com/{0}/'.format(self.username),
+        )
 
-    def get_followees(self) -> Iterator['Profile']:
+    def get_followees(self) -> NodeIterator['Profile']:
         """
         Retrieve list of followees (followings) of given profile.
         To use this, one needs to be logged in and private profiles has to be followed.
+
+        :rtype:NodeIterator[Profile]
         """
         if not self._context.is_logged_in:
             raise LoginRequiredException("--login required to get a profile's followees.")
         self._obtain_metadata()
-        yield from (Profile(self._context, node) for node in
+        return NodeIterator(
-                    self._context.graphql_node_list("58712303d941c6855d4e888c5f0cd22f",
+            self._context,
-                                                    {'id': str(self.userid)},
+            '58712303d941c6855d4e888c5f0cd22f',
-                                                    'https://www.instagram.com/' + self.username + '/',
+            lambda d: d['data']['user']['edge_follow'],
-                                                    lambda d: d['data']['user']['edge_follow']))
+            lambda n: Profile(self._context, n),
+            {'id': str(self.userid)},
+            'https://www.instagram.com/{0}/'.format(self.username),
+        )
 
     def get_similar_accounts(self) -> Iterator['Profile']:
         """
@@ -1398,7 +1437,7 @@ class TopSearchResults:
         return self._searchstring
 
 
-JsonExportable = Union[Post, Profile, StoryItem, Hashtag]
+JsonExportable = Union[Post, Profile, StoryItem, Hashtag, FrozenNodeIterator]
 
 
 def save_structure_to_file(structure: JsonExportable, filename: str) -> None:
@@ -1447,6 +1486,8 @@ def load_structure_from_file(context: InstaloaderContext, filename: str) -> Json
             return StoryItem(context, json_structure['node'])
         elif node_type == "Hashtag":
             return Hashtag(context, json_structure['node'])
+        elif node_type == "FrozenNodeIterator":
+            return FrozenNodeIterator(**json_structure['node'])
         else:
             raise InvalidArgumentException("{}: Not an Instaloader JSON.".format(filename))
     elif 'shortcode' in json_structure:

test/instaloader_unittests.py

@@ -5,7 +5,7 @@ import shutil
 import tempfile
 import unittest
 from itertools import islice
-from typing import Dict, List
+from typing import Optional
 
 import instaloader
 
@@ -23,8 +23,7 @@ PRIVATE_PROFILE_ID = 1706625676
 EMPTY_PROFILE = "not_public"
 EMPTY_PROFILE_ID = 1928659031
 
-# Preserve query timestamps (rate control) between tests to not get rate limited
+ratecontroller = None  # type: Optional[instaloader.RateController]
-instaloadercontext_query_timestamps = dict()  # type: Dict[str, List[float]]
 
 
 class TestInstaloaderAnonymously(unittest.TestCase):
@@ -37,13 +36,15 @@ class TestInstaloaderAnonymously(unittest.TestCase):
                                          download_comments=True,
                                          save_metadata=True)
         self.L.context.raise_all_errors = True
-        # pylint:disable=protected-access
+        if ratecontroller is not None:
-        self.L.context._graphql_query_timestamps = instaloadercontext_query_timestamps.copy()
+            # pylint:disable=protected-access
+            ratecontroller._context = self.L.context
+            self.L.context._rate_controller = ratecontroller
 
     def tearDown(self):
         # pylint:disable=global-statement,protected-access
-        global instaloadercontext_query_timestamps
+        global ratecontroller
-        instaloadercontext_query_timestamps = self.L.context._graphql_query_timestamps.copy()
+        ratecontroller = self.L.context._rate_controller
         self.L.close()
         os.chdir('/')
         print("Removing {}".format(self.dir))