Version 4.2

Motivated by #92.

.travis.yml

@@ -8,7 +8,7 @@ install:
  - pip install pylint requests
  - pip install -r docs/requirements.txt
 script:
- - python3 -m pylint -r n -d bad-whitespace,missing-docstring,too-many-arguments,locally-disabled,line-too-long,too-many-public-methods,too-many-lines,too-many-instance-attributes,too-many-locals,too-many-branches,too-many-statements,inconsistent-return-statements,invalid-name,wildcard-import,unused-wildcard-import,no-else-return instaloader
+ - python3 -m pylint -r n -d bad-whitespace,missing-docstring,too-many-arguments,locally-disabled,line-too-long,too-many-public-methods,too-many-lines,too-many-instance-attributes,too-many-locals,too-many-branches,too-many-statements,inconsistent-return-statements,invalid-name,wildcard-import,unused-wildcard-import,no-else-return,cyclic-import,unnecessary-pass instaloader
  - make -C docs html SPHINXOPTS="-W -n"
 deploy:
   - provider: pypi

docs/as-module.rst

@@ -188,6 +188,10 @@ Exceptions
 
 .. autoexception:: LoginRequiredException
 
+.. autoexception:: TwoFactorAuthRequiredException
+
+   .. versionadded:: 4.2
+
 .. autoexception:: InvalidArgumentException
 
 .. autoexception:: BadResponseException

docs/basic-usage.rst

@@ -85,6 +85,13 @@ Instaloader supports the following targets:
 - ``"#hashtag"``
    Posts with a certain **hashtag** (the quotes are usually necessary),
 
+- ``%location id``
+   Posts tagged with a given location; the location ID is the numerical ID
+   Instagram labels a location with (e.g.
+   \https://www.instagram.com/explore/locations/**362629379**/plymouth-naval-memorial/).
+
+   .. versionadded:: 4.2
+
 - ``:stories``
    The currently-visible **stories** of your followees (requires
    :option:`--login`),
@@ -126,7 +133,7 @@ Filename Specification
 ^^^^^^^^^^^^^^^^^^^^^^
 
 For each target, Instaloader creates a directory named after the target,
-i.e. ``profile``, ``#hashtag``, ``:feed``, etc. and therein saves the
+i.e. ``profile``, ``#hashtag``, ``%location id``, ``:feed``, etc. and therein saves the
 posts in files named after the post's timestamp.
 
 :option:`--dirname-pattern` allows to configure the directory name of each
@@ -192,11 +199,6 @@ where the attributes from :class:`Post` or
 Id est, the following attributes can be used with both
 :option:`--post-filter` and :option:`--storyitem-filter`:
 
-- :attr:`~Post.is_video` (bool)
-   Post/StoryItem is a video. For example, you may skip videos::
-
-      instaloader --post-filter="not is_video" target
-
 - :attr:`~Post.owner_username` (str), :attr:`~Post.owner_id` (int)
    Owner profile username / userid.
 
@@ -206,6 +208,15 @@ Id est, the following attributes can be used with both
 
       instaloader --post-filter="date_utc <= datetime(2018, 5, 31)" target
 
+- :attr:`~Post.is_video` (bool)
+   Post/StoryItem is a video. For example, you may skip videos::
+
+      instaloader --post-filter="not is_video" target
+
+   This is not the same as :option:`--no-videos` and
+   :option:`--no-video-thumbnails`, since sidecar posts (posts that contain
+   multiple pictures/videos in one post) have this attribute set to False.
+
 As :option:`--post-filter`, the following attributes can be used additionally:
 
 - :attr:`~Post.viewer_has_liked` (bool)

docs/cli-options.rst

@@ -8,7 +8,7 @@ Instaloader is invoked with::
    $ instaloader [options] target [target ...]
 
 where ``target`` is a ``profile``, a ``"#hashtag"``, ``@profile`` (all profiles
-that *profile* is following), or if logged in ``:feed`` (pictures from your
+that *profile* is following), ``%location ID``, or if logged in ``:feed`` (pictures from your
 feed), ``:stories`` (stories of your followees) or ``:saved`` (collection of
 posts marked as saved).
 
@@ -171,7 +171,7 @@ Which Posts to Download
 .. option:: --count COUNT, -c
 
    Do not attempt to download more than COUNT posts.  Applies only to
-   ``#hashtag`` and ``:feed``.
+   ``#hashtag``, ``%location id``, and ``:feed``.
 
 
 Login (Download Private Profiles)
@@ -230,6 +230,14 @@ How to Download
    to ``3``. If a connection fails, it can be manually skipped by hitting
    :kbd:`Control-c`. Set this to ``0`` to retry infinitely.
 
+.. option:: --commit-mode
+
+   Tries to ensure downloaded images avoid corruption in case of unexpected
+   interruption. If the last picture is corrupted, Instaloader will fix the
+   picture the next time it is run.
+
+   .. versionadded:: 4.2
+
 Miscellaneous Options
 ^^^^^^^^^^^^^^^^^^^^^
 

docs/codesnippets.rst

@@ -14,6 +14,9 @@ Here we present code examples that use the :ref:`python-module-instaloader` for
 more advanced Instagram downloading or metadata mining than what is possible
 with the Instaloader command line interface.
 
+The scripts presented here can be downloaded from our source tree:
+`instaloader/docs/codesnippets/ <https://github.com/instaloader/instaloader/tree/master/docs/codesnippets>`__
+
 .. For each code snippet:
    - title
    - brief description of what it does / motivation / how it works

docs/codesnippets/README.md

@@ -0,0 +1,4 @@
+# Advanced Instaloader Examples
+
+This directory contains the scripts that are presented at:
+https://instaloader.github.io/codesnippets.html

docs/conf.py

@@ -20,6 +20,7 @@
 import os
 import subprocess
 import sys
+import typing
 
 import sphinx_bootstrap_theme
 
@@ -374,6 +375,7 @@ html_context = {'current_release': current_release, 'current_release_date': curr
 
 
 def setup(app):
+    typing.TYPE_CHECKING = True
     app.add_stylesheet("style.css")
     app.connect('autodoc-process-signature', sphinx_autodoc_typehints.process_signature)
     app.connect('autodoc-process-docstring', sphinx_autodoc_typehints.process_docstring)

docs/index.rst

@@ -43,7 +43,7 @@ See :ref:`install` for more options on how to install Instaloader.
 
     instaloader [--comments] [--geotags] [--stories] [--highlights] [--tagged]
                 [--login YOUR-USERNAME] [--fast-update]
-                profile | "#hashtag" |
+                profile | "#hashtag" | %location_id |
                 :stories | :feed | :saved
 
 See :ref:`download-pictures-from-instagram` for a detailed introduction on how

docs/sphinx_autodoc_typehints.py

@@ -1,4 +1,4 @@
-# sphinx_autodoc_typehints from Mar 18 2018.
+# sphinx_autodoc_typehints from Nov 20 2018 (version 1.5.2).
 #  https://github.com/agronholm/sphinx-autodoc-typehints
 # (slightly modified to fix class links, maybe related to agronholm/sphinx-autodoc-typehints#38)
 #
@@ -27,8 +27,8 @@
 import inspect
 from typing import get_type_hints, TypeVar, Any, AnyStr, Generic, Union
 
-from sphinx.util.inspect import getargspec
-from sphinx.ext.autodoc import formatargspec
+from sphinx.util import logging
+from sphinx.util.inspect import Signature
 
 try:
     from inspect import unwrap
@@ -51,6 +51,8 @@ except ImportError:
             memo.add(id_func)
         return func
 
+logger = logging.getLogger(__name__)
+
 
 def format_annotation(annotation):
     if inspect.isclass(annotation) and annotation.__module__ == 'builtins':
@@ -162,23 +164,37 @@ def process_signature(app, what: str, name: str, obj, options, signature, return
         return
 
     obj = unwrap(obj)
-    try:
-        argspec = getargspec(obj)
-    except (TypeError, ValueError):
-        return
+    signature = Signature(obj)
+    parameters = [
+        param.replace(annotation=inspect.Parameter.empty)
+        for param in signature.signature.parameters.values()
+    ]
 
-    if argspec.args:
+    if parameters:
         if what in ('class', 'exception'):
-            del argspec.args[0]
+            del parameters[0]
         elif what == 'method':
             outer = inspect.getmodule(obj)
             for clsname in obj.__qualname__.split('.')[:-1]:
                 outer = getattr(outer, clsname)
-            method_object = outer.__dict__[obj.__name__]
-            if not isinstance(method_object, (classmethod, staticmethod)):
-                del argspec.args[0]
 
-    return formatargspec(obj, *argspec[:-1]), None
+            method_name = obj.__name__
+            if method_name.startswith("__") and not method_name.endswith("__"):
+                # If the method starts with double underscore (dunder)
+                # Python applies mangling so we need to prepend the class name.
+                # This doesn't happen if it always ends with double underscore.
+                class_name = obj.__qualname__.split('.')[-2]
+                method_name = "_{c}{m}".format(c=class_name, m=method_name)
+
+            method_object = outer.__dict__[method_name]
+            if not isinstance(method_object, (classmethod, staticmethod)):
+                del parameters[0]
+
+    signature.signature = signature.signature.replace(
+        parameters=parameters,
+        return_annotation=inspect.Signature.empty)
+
+    return signature.format_args().replace('\\', '\\\\'), None
 
 
 def process_docstring(app, what, name, obj, options, lines):
@@ -195,8 +211,15 @@ def process_docstring(app, what, name, obj, options, lines):
         except (AttributeError, TypeError):
             # Introspecting a slot wrapper will raise TypeError
             return
+        except NameError as exc:
+            logger.warning('Cannot resolve forward reference in type annotations of "%s": %s',
+                           name, exc)
+            type_hints = obj.__annotations__
 
         for argname, annotation in type_hints.items():
+            if argname.endswith('_'):
+                argname = '{}\\_'.format(argname[:-1])
+
             formatted_annotation = format_annotation(annotation)
 
             if argname == 'return':

instaloader/__init__.py

@@ -1,7 +1,7 @@
 """Download pictures (or videos) along with their captions and other metadata from Instagram."""
 
 
-__version__ = '4.1a1'
+__version__ = '4.2'
 
 
 try:

instaloader/__main__.py

@@ -8,7 +8,8 @@ from argparse import ArgumentParser, SUPPRESS
 from typing import List, Optional
 
 from . import (Instaloader, InstaloaderException, InvalidArgumentException, Post, Profile, ProfileNotExistsException,
-               StoryItem, __version__, load_structure_from_file)
+               StoryItem, __version__, load_structure_from_file, TwoFactorAuthRequiredException,
+               BadCredentialsException)
 from .instaloader import get_default_session_filename
 from .instaloadercontext import default_user_agent
 
@@ -20,7 +21,7 @@ def usage_string():
     return """
 {0} [--comments] [--geotags] [--stories] [--highlights] [--tagged]
 {2:{1}} [--login YOUR-USERNAME] [--fast-update]
-{2:{1}} profile | "#hashtag" | :stories | :feed | :saved
+{2:{1}} profile | "#hashtag" | %%location_id | :stories | :feed | :saved
 {0} --help""".format(argv0, len(argv0), '')
 
 
@@ -84,7 +85,16 @@ def _main(instaloader: Instaloader, targetlist: List[str],
             instaloader.context.log("Session file does not exist yet - Logging in.")
         if not instaloader.context.is_logged_in or username != instaloader.test_login():
             if password is not None:
-                instaloader.login(username, password)
+                try:
+                    instaloader.login(username, password)
+                except TwoFactorAuthRequiredException:
+                    while True:
+                        try:
+                            code = input("Enter 2FA verification code: ")
+                            instaloader.two_factor_login(code)
+                            break
+                        except BadCredentialsException:
+                            pass
             else:
                 instaloader.interactive_login(username)
         instaloader.context.log("Logged in as %s." % username)
@@ -130,6 +140,9 @@ def _main(instaloader: Instaloader, targetlist: List[str],
                                                  post_filter=post_filter)
                 elif target[0] == '-':
                     instaloader.download_post(Post.from_shortcode(instaloader.context, target[1:]), target)
+                elif target[0] == "%":
+                    instaloader.download_location(location=target[1:], max_count=max_count, fast_update=fast_update,
+                                                  post_filter=post_filter)
                 elif target == ":feed":
                     instaloader.download_feed_posts(fast_update=fast_update, max_count=max_count,
                                                     post_filter=post_filter)
@@ -208,6 +221,7 @@ def main():
                            help="Download all followees of profile. Requires --login. "
                                 "Consider using :feed rather than @yourself.")
     g_targets.add_argument('_hashtag', nargs='*', metavar='"#hashtag"', help="Download #hashtag.")
+    g_targets.add_argument('_location', nargs='*', metavar='%location_id', help="Download %%location_id.")
     g_targets.add_argument('_feed', nargs='*', metavar=":feed",
                            help="Download pictures from your feed. Requires --login.")
     g_targets.add_argument('_stories', nargs='*', metavar=":stories",
@@ -320,6 +334,10 @@ def main():
                        help='Maximum number of connection attempts until a request is aborted. Defaults to 3. If a '
                             'connection fails, it can be manually skipped by hitting CTRL+C. Set this to 0 to retry '
                             'infinitely.')
+    g_how.add_argument('--commit-mode', action='store_true',
+                       help='Tries to ensure downloaded images avoid corruption in case of unexpected interruption. '
+                       'If the last picture is corrupted, Instaloader will fix the picture the next time it is run. '
+                       'Requires the JSON metadata to be saved.')
 
     g_misc = parser.add_argument_group('Miscellaneous Options')
     g_misc.add_argument('-q', '--quiet', action='store_true',
@@ -361,6 +379,9 @@ def main():
         download_posts = not (args.no_posts or args.stories_only or args.profile_pic_only)
         download_stories = args.stories or args.stories_only
 
+        if args.commit_mode and args.no_metadata_json:
+            raise SystemExit('--commit-mode requires JSON metadata to be saved.')
+
         loader = Instaloader(sleep=not args.no_sleep, quiet=args.quiet, user_agent=args.user_agent,
                              dirname_pattern=args.dirname_pattern, filename_pattern=args.filename_pattern,
                              download_pictures=not args.no_pictures,
@@ -371,7 +392,8 @@ def main():
                              post_metadata_txt_pattern=post_metadata_txt_pattern,
                              storyitem_metadata_txt_pattern=storyitem_metadata_txt_pattern,
                              graphql_rate_limit=args.graphql_rate_limit,
-                             max_connection_attempts=args.max_connection_attempts)
+                             max_connection_attempts=args.max_connection_attempts,
+                             commit_mode=args.commit_mode)
         _main(loader,
               args.profile,
               username=args.login.lower() if args.login is not None else None,

instaloader/exceptions.py

@@ -33,6 +33,10 @@ class LoginRequiredException(InstaloaderException):
     pass
 
 
+class TwoFactorAuthRequiredException(InstaloaderException):
+    pass
+
+
 class InvalidArgumentException(InstaloaderException):
     pass
 

instaloader/instaloader.py

@@ -1,5 +1,6 @@
 import getpass
 import json
+import lzma
 import os
 import platform
 import re
@@ -15,7 +16,7 @@ from typing import Any, Callable, Iterator, List, Optional, Set, Union
 
 from .exceptions import *
 from .instaloadercontext import InstaloaderContext
-from .structures import Highlight, JsonExportable, Post, PostLocation, Profile, Story, StoryItem, save_structure_to_file
+from .structures import Highlight, JsonExportable, Post, PostLocation, Profile, Story, StoryItem, save_structure_to_file, load_structure_from_file
 
 
 def get_default_session_filename(username: str) -> str:
@@ -95,6 +96,7 @@ class Instaloader:
        txt file.
     :param storyitem_metadata_txt_pattern: :option:`--storyitem-metadata-txt`, default is empty (=none)
     :param max_connection_attempts: :option:`--max-connection-attempts`
+    :param commit_mode: :option:`--commit-mode`
 
     .. attribute:: context
 
@@ -116,7 +118,8 @@ class Instaloader:
                  post_metadata_txt_pattern: str = None,
                  storyitem_metadata_txt_pattern: str = None,
                  graphql_rate_limit: Optional[int] = None,
-                 max_connection_attempts: int = 3):
+                 max_connection_attempts: int = 3,
+                 commit_mode: bool = False):
 
         self.context = InstaloaderContext(sleep, quiet, user_agent, graphql_rate_limit, max_connection_attempts)
 
@@ -134,6 +137,12 @@ class Instaloader:
             else post_metadata_txt_pattern
         self.storyitem_metadata_txt_pattern = '' if storyitem_metadata_txt_pattern is None \
             else storyitem_metadata_txt_pattern
+        self.commit_mode = commit_mode
+        if self.commit_mode and not self.save_metadata:
+            raise InvalidArgumentException("Commit mode requires JSON metadata to be saved.")
+
+        # Used to keep state in commit mode
+        self._committed = None
 
     @contextmanager
     def anonymous_copy(self):
@@ -173,9 +182,15 @@ class Instaloader:
         if filename_suffix is not None:
             filename += '_' + filename_suffix
         filename += '.' + file_extension
-        if os.path.isfile(filename):
-            self.context.log(filename + ' exists', end=' ', flush=True)
-            return False
+        # A post is considered "commited" if the json file exists and is not malformed.
+        if self.commit_mode:
+            if self._committed and os.path.isfile(filename):
+                self.context.log(filename + ' exists', end=' ', flush=True)
+                return False
+        else:
+            if os.path.isfile(filename):
+                self.context.log(filename + ' exists', end=' ', flush=True)
+                return False
         self.context.get_and_write_raw(url, filename)
         os.utime(filename, (datetime.now().timestamp(), mtime.timestamp()))
         return True
@@ -335,9 +350,26 @@ class Instaloader:
 
         :raises InvalidArgumentException: If the provided username does not exist.
         :raises BadCredentialsException: If the provided password is wrong.
-        :raises ConnectionException: If connection to Instagram failed."""
+        :raises ConnectionException: If connection to Instagram failed.
+        :raises TwoFactorAuthRequiredException: First step of 2FA login done, now call :meth:`Instaloader.two_factor_login`."""
         self.context.login(user, passwd)
 
+    def two_factor_login(self, two_factor_code) -> None:
+        """Second step of login if 2FA is enabled.
+        Not meant to be used directly, use :meth:`Instaloader.two_factor_login`.
+
+        :raises InvalidArgumentException: No two-factor authentication pending.
+        :raises BadCredentialsException: 2FA verification code invalid.
+
+        .. versionadded:: 4.2"""
+        self.context.two_factor_login(two_factor_code)
+
+    def format_filename(self, item: Union[Post, StoryItem], target: Optional[str] = None):
+        """Format filename of a :class:`Post` or :class:`StoryItem` according to ``filename-pattern`` parameter.
+
+        .. versionadded:: 4.1"""
+        return _PostPathFormatter(item).format(self.filename_pattern, target=target)
+
     def download_post(self, post: Post, target: str) -> bool:
         """
         Download everything associated with one instagram post node, i.e. picture, caption and video.
@@ -348,22 +380,23 @@ class Instaloader:
         """
 
         dirname = _PostPathFormatter(post).format(self.dirname_pattern, target=target)
-        filename = dirname + '/' + _PostPathFormatter(post).format(self.filename_pattern, target=target)
+        filename = dirname + '/' + self.format_filename(post, target=target)
         os.makedirs(os.path.dirname(filename), exist_ok=True)
 
         # Download the image(s) / video thumbnail and videos within sidecars if desired
-        downloaded = False
+        downloaded = True
+        self._committed = self.check_if_committed(filename)
         if self.download_pictures:
             if post.typename == 'GraphSidecar':
                 edge_number = 1
                 for sidecar_node in post.get_sidecar_nodes():
                     # Download picture or video thumbnail
                     if not sidecar_node.is_video or self.download_video_thumbnails is True:
-                        downloaded |= self.download_pic(filename=filename, url=sidecar_node.display_url,
+                        downloaded &= self.download_pic(filename=filename, url=sidecar_node.display_url,
                                                         mtime=post.date_local, filename_suffix=str(edge_number))
                     # Additionally download video if available and desired
                     if sidecar_node.is_video and self.download_videos is True:
-                        downloaded |= self.download_pic(filename=filename, url=sidecar_node.video_url,
+                        downloaded &= self.download_pic(filename=filename, url=sidecar_node.video_url,
                                                         mtime=post.date_local, filename_suffix=str(edge_number))
                     edge_number += 1
             elif post.typename == 'GraphImage':
@@ -381,7 +414,7 @@ class Instaloader:
 
         # Download video if desired
         if post.is_video and self.download_videos is True:
-            downloaded |= self.download_pic(filename=filename, url=post.video_url, mtime=post.date_local)
+            downloaded &= self.download_pic(filename=filename, url=post.video_url, mtime=post.date_local)
 
         # Download geotags if desired
         if self.download_geotags and post.location:
@@ -473,7 +506,7 @@ class Instaloader:
 
         date_local = item.date_local
         dirname = _PostPathFormatter(item).format(self.dirname_pattern, target=target)
-        filename = dirname + '/' + _PostPathFormatter(item).format(self.filename_pattern, target=target)
+        filename = dirname + '/' + self.format_filename(item, target=target)
         os.makedirs(os.path.dirname(filename), exist_ok=True)
         downloaded = False
         if not item.is_video or self.download_video_thumbnails is True:
@@ -626,6 +659,59 @@ class Instaloader:
                 if fast_update and not downloaded:
                     break
 
+    def get_location_posts(self, location: str) -> Iterator[Post]:
+        """Get Posts which are listed by Instagram for a given Location.
+
+        :return:  Iterator over Posts of a location's posts
+
+        .. versionadded:: 4.2
+        """
+        has_next_page = True
+        end_cursor = None
+        while has_next_page:
+            if end_cursor:
+                params = {'__a': 1, 'max_id': end_cursor}
+            else:
+                params = {'__a': 1}
+            location_data = self.context.get_json('explore/locations/{0}/'.format(location),
+                                                  params)['graphql']['location']['edge_location_to_media']
+            yield from (Post(self.context, edge['node']) for edge in location_data['edges'])
+            has_next_page = location_data['page_info']['has_next_page']
+            end_cursor = location_data['page_info']['end_cursor']
+
+    def download_location(self, location: str,
+                          max_count: Optional[int] = None,
+                          post_filter: Optional[Callable[[Post], bool]] = None,
+                          fast_update: bool = False) -> None:
+        """Download pictures of one location.
+
+        To download the last 30 pictures with location 362629379, do::
+
+            loader = Instaloader()
+            loader.download_location(362629379, max_count=30)
+
+        :param location: Location to download, as Instagram numerical ID
+        :param max_count: Maximum count of pictures to download
+        :param post_filter: function(post), which returns True if given picture should be downloaded
+        :param fast_update: If true, abort when first already-downloaded picture is encountered
+
+        .. versionadded:: 4.2
+        """
+        self.context.log("Retrieving pictures for location {}...".format(location))
+        count = 1
+        for post in self.get_location_posts(location):
+            if max_count is not None and count > max_count:
+                break
+            self.context.log('[{0:3d}] %{1} '.format(count, location), end='', flush=True)
+            if post_filter is not None and not post_filter(post):
+                self.context.log('<skipped>')
+                continue
+            count += 1
+            with self.context.error_catcher('Download location {}'.format(location)):
+                downloaded = self.download_post(post, target='%' + location)
+                if fast_update and not downloaded:
+                    break
+
     @_requires_login
     def get_explore_posts(self) -> Iterator[Post]:
         """Get Posts which are worthy of exploring suggested by Instagram.
@@ -693,7 +779,7 @@ class Instaloader:
 
         .. versionadded:: 4.1"""
         if target is None:
-            target = profile.username + ':tagged'
+            target = profile.username + '/:tagged'
         self.context.log("Retrieving tagged posts for profile {}.".format(profile.username))
         count = 1
         for post in profile.get_tagged_posts():
@@ -817,7 +903,7 @@ class Instaloader:
                         self.download_profilepic(profile)
 
                 # Catch some errors
-                if profile.is_private:
+                if profile.is_private and (tagged or highlights or posts):
                     if not self.context.is_logged_in:
                         raise LoginRequiredException("--login=USERNAME required.")
                     if not profile.followed_by_viewer and self.context.username != profile.username:
@@ -939,6 +1025,25 @@ class Instaloader:
                 if fast_update and not downloaded:
                     break
 
+    def check_if_committed(self, filename: str) -> bool:
+        """Checks to see if the current post has been committed.
+
+        A post is considered committed if its json metadata file exists and is not malformed.
+
+        .. versionadded:: 4.2
+        """
+        if os.path.isfile(filename + '.json.xz'):
+            filename += '.json.xz'
+        elif os.path.isfile(filename + '.json'):
+            filename += '.json'
+        else:
+            return False
+        try:
+            load_structure_from_file(self.context, filename)
+            return True
+        except (FileNotFoundError, lzma.LZMAError, json.decoder.JSONDecodeError):
+            return False
+
     def interactive_login(self, username: str) -> None:
         """Logs in and internally stores session, asking user for password interactively.
 
@@ -947,11 +1052,20 @@ class Instaloader:
         :raises ConnectionException: If connection to Instagram failed."""
         if self.context.quiet:
             raise LoginRequiredException("Quiet mode requires given password or valid session file.")
-        password = None
-        while password is None:
-            password = getpass.getpass(prompt="Enter Instagram password for %s: " % username)
-            try:
-                self.login(username, password)
-            except BadCredentialsException as err:
-                print(err, file=sys.stderr)
-                password = None
+        try:
+            password = None
+            while password is None:
+                password = getpass.getpass(prompt="Enter Instagram password for %s: " % username)
+                try:
+                    self.login(username, password)
+                except BadCredentialsException as err:
+                    print(err, file=sys.stderr)
+                    password = None
+        except TwoFactorAuthRequiredException:
+            while True:
+                try:
+                    code = input("Enter 2FA verification code: ")
+                    self.two_factor_login(code)
+                    break
+                except BadCredentialsException:
+                    pass

instaloader/instaloadercontext.py

@@ -57,8 +57,9 @@ class InstaloaderContext:
         self.quiet = quiet
         self.max_connection_attempts = max_connection_attempts
         self._graphql_page_length = 50
-        self.graphql_count_per_slidingwindow = graphql_count_per_slidingwindow or 20
+        self.graphql_count_per_slidingwindow = graphql_count_per_slidingwindow or 200
         self._root_rhx_gis = None
+        self.two_factor_auth_pending = None
 
         # error log, filled with error() and printed at the end of Instaloader.main()
         self.error_log = []
@@ -69,6 +70,9 @@ class InstaloaderContext:
         # Can be set to True for testing, disables supression of InstaloaderContext._error_catcher
         self.raise_all_errors = False
 
+        # Cache profile from id (mapping from id to Profile)
+        self.profile_id_cache = dict()
+
     @contextmanager
     def anonymous_copy(self):
         session = self._session
@@ -177,23 +181,39 @@ class InstaloaderContext:
 
         :raises InvalidArgumentException: If the provided username does not exist.
         :raises BadCredentialsException: If the provided password is wrong.
-        :raises ConnectionException: If connection to Instagram failed."""
+        :raises ConnectionException: If connection to Instagram failed.
+        :raises TwoFactorAuthRequiredException: First step of 2FA login done, now call :meth:`Instaloader.two_factor_login`."""
         import http.client
         # pylint:disable=protected-access
         http.client._MAXHEADERS = 200
         session = requests.Session()
         session.cookies.update({'sessionid': '', 'mid': '', 'ig_pr': '1',
-                                'ig_vw': '1920', 'csrftoken': '',
+                                'ig_vw': '1920', 'ig_cb': '1', 'csrftoken': '',
                                 's_network': '', 'ds_user_id': ''})
         session.headers.update(self._default_http_header())
-        session.headers.update({'X-CSRFToken': self.get_json('', {})['config']['csrf_token']})
+        session.get('https://www.instagram.com/web/__mid/')
+        csrf_token = session.cookies.get_dict()['csrftoken']
+        session.headers.update({'X-CSRFToken': csrf_token})
         # Not using self.get_json() here, because we need to access csrftoken cookie
         self._sleep()
         login = session.post('https://www.instagram.com/accounts/login/ajax/',
                              data={'password': passwd, 'username': user}, allow_redirects=True)
-        if login.status_code != 200:
-            raise ConnectionException("Login error: {} {}".format(login.status_code, login.reason))
-        resp_json = login.json()
+        try:
+            resp_json = login.json()
+        except json.decoder.JSONDecodeError:
+            raise ConnectionException("Login error: JSON decode fail, {} - {}.".format(login.status_code, login.reason))
+        if resp_json.get('two_factor_required'):
+            two_factor_session = copy_session(session)
+            two_factor_session.headers.update({'X-CSRFToken': csrf_token})
+            two_factor_session.cookies.update({'csrftoken': csrf_token})
+            self.two_factor_auth_pending = (two_factor_session,
+                                            user,
+                                            resp_json['two_factor_info']['two_factor_identifier'])
+            raise TwoFactorAuthRequiredException("Login error: two-factor authentication required.")
+        if resp_json.get('checkpoint_url'):
+            raise ConnectionException("Login: Checkpoint required. Point your browser to "
+                                      "https://www.instagram.com{}, "
+                                      "follow the instructions, then retry.".format(resp_json.get('checkpoint_url')))
         if resp_json['status'] != 'ok':
             if 'message' in resp_json:
                 raise ConnectionException("Login error: \"{}\" status, message \"{}\".".format(resp_json['status'],
@@ -215,10 +235,36 @@ class InstaloaderContext:
         self._session = session
         self.username = user
 
+    def two_factor_login(self, two_factor_code):
+        """Second step of login if 2FA is enabled.
+        Not meant to be used directly, use :meth:`Instaloader.two_factor_login`.
+
+        :raises InvalidArgumentException: No two-factor authentication pending.
+        :raises BadCredentialsException: 2FA verification code invalid.
+
+        .. versionadded:: 4.2"""
+        if not self.two_factor_auth_pending:
+            raise InvalidArgumentException("No two-factor authentication pending.")
+        (session, user, two_factor_id) = self.two_factor_auth_pending
+
+        login = session.post('https://www.instagram.com/accounts/login/ajax/two_factor/',
+                             data={'username': user, 'verificationCode': two_factor_code, 'identifier': two_factor_id},
+                             allow_redirects=True)
+        resp_json = login.json()
+        if resp_json['status'] != 'ok':
+            if 'message' in resp_json:
+                raise BadCredentialsException("Login error: {}".format(resp_json['message']))
+            else:
+                raise BadCredentialsException("Login error: \"{}\" status.".format(resp_json['status']))
+        session.headers.update({'X-CSRFToken': login.cookies['csrftoken']})
+        self._session = session
+        self.username = user
+        self.two_factor_auth_pending = None
+
     def _sleep(self):
         """Sleep a short time if self.sleep is set. Called before each request to instagram.com."""
         if self.sleep:
-            time.sleep(min(random.expovariate(0.6), 5.0))
+            time.sleep(min(random.expovariate(0.7), 5.0))
 
     def get_json(self, path: str, params: Dict[str, Any], host: str = 'www.instagram.com',
                  session: Optional[requests.Session] = None, _attempt=1) -> Dict[str, Any]:
@@ -303,6 +349,7 @@ class InstaloaderContext:
                 if isinstance(err, TooManyRequestsException):
                     print(textwrap.fill(text_for_429), file=sys.stderr)
                     if is_graphql_query:
+                        print("Made {} GraphQL requests.".format(len(self.query_timestamps)), file=sys.stderr)
                         waittime = graphql_query_waittime(untracked_queries=True)
                         if waittime > 0:
                             self.log('The request will be retried in {} seconds, at {:%H:%M}.'

instaloader/structures.py

@@ -373,10 +373,6 @@ class Profile:
         self._has_public_story = None
         self._node = node
         self._rhx_gis = None
-        self._iphone_struct_ = None
-        if 'iphone_struct' in node:
-            # if loaded from JSON with load_structure_from_file()
-            self._iphone_struct_ = node['iphone_struct']
 
     @classmethod
     def from_username(cls, context: InstaloaderContext, username: str):
@@ -402,6 +398,8 @@ class Profile:
         :param profile_id: userid
         :raises: :class:`ProfileNotExistsException`, :class:`ProfileHasNoPicsException`
         """
+        if profile_id in context.profile_id_cache:
+            return context.profile_id_cache[profile_id]
         data = context.graphql_query("472f257a40c653c64c666ce877d59d2b",
                                      {'id': str(profile_id), 'first': 1},
                                      rhx_gis=context.root_rhx_gis)['data']['user']
@@ -415,7 +413,9 @@ class Profile:
                 raise ProfileHasNoPicsException("Profile with ID {0}: no pics found.".format(str(profile_id)))
             else:
                 raise LoginRequiredException("Login required to determine username (ID: " + str(profile_id) + ").")
-        return Post(context, data['edges'][0]['node']).owner_profile
+        profile = Post(context, data['edges'][0]['node']).owner_profile
+        context.profile_id_cache[profile_id] = profile
+        return profile
 
     def _asdict(self):
         json_node = self._node.copy()
@@ -423,8 +423,6 @@ class Profile:
         json_node.pop('edge_media_collections', None)
         json_node.pop('edge_owner_to_timeline_media', None)
         json_node.pop('edge_saved_media', None)
-        if self._iphone_struct_:
-            json_node['iphone_struct'] = self._iphone_struct_
         return json_node
 
     def _obtain_metadata(self):
@@ -449,15 +447,6 @@ class Profile:
                 d = d[key]
             return d
 
-    @property
-    def _iphone_struct(self) -> Dict[str, Any]:
-        if not self._iphone_struct_:
-            with self._context.anonymous_copy() as anonymous_context:
-                data = anonymous_context.get_json(path='api/v1/users/{}/info/'.format(self.userid),
-                                                  params={}, host='i.instagram.com')
-            self._iphone_struct_ = data['user']
-        return self._iphone_struct_
-
     @property
     def userid(self) -> int:
         """User ID"""
@@ -577,11 +566,7 @@ class Profile:
         """Return URL of profile picture
 
         .. versionadded:: 4.0.3"""
-        try:
-            return self._iphone_struct['hd_profile_pic_url_info']['url']
-        except (InstaloaderException, KeyError) as err:
-            self._context.error('{} Unable to fetch high quality profile pic.'.format(err))
-            return self._metadata("profile_pic_url_hd")
+        return self._metadata("profile_pic_url_hd")
 
     def get_profile_pic_url(self) -> str:
         """.. deprecated:: 4.0.3

test/instaloader_unittests.py

@@ -12,6 +12,7 @@ PROFILE_WITH_HIGHLIGHTS = 325732271
 PUBLIC_PROFILE = "selenagomez"
 PUBLIC_PROFILE_ID = 460563723
 HASHTAG = "kitten"
+LOCATION = "362629379"
 OWN_USERNAME = "aandergr"
 NORMAL_MAX_COUNT = 2
 PAGING_MAX_COUNT = 15
@@ -63,6 +64,21 @@ class TestInstaloaderAnonymously(unittest.TestCase):
     def test_hashtag_download(self):
         self.L.download_hashtag(HASHTAG, NORMAL_MAX_COUNT)
 
+    def test_hashtag_paging(self):
+        for count, post in enumerate(self.L.get_hashtag_posts(HASHTAG)):
+            print(post)
+            if count == PAGING_MAX_COUNT:
+                break
+
+    def test_location_download(self):
+        self.L.download_location(LOCATION, NORMAL_MAX_COUNT)
+
+    def test_location_paging(self):
+        for count, post in enumerate(self.L.get_location_posts(LOCATION)):
+            print(post)
+            if count == PAGING_MAX_COUNT:
+                break
+
     def test_get_id_by_username(self):
         self.assertEqual(PUBLIC_PROFILE_ID,
                          instaloader.Profile.from_username(self.L.context, PUBLIC_PROFILE).userid)