Release of Version 4.3.2

Fixes #478.

.github/ISSUE_TEMPLATE/bug_report.md

@@ -1,6 +1,7 @@
 ---
-name: Bug report
-about: Create a report to help us improve
+name: 🐛 Bug report
+about: Create a report to help us improve 🤕
+labels: bug
 
 ---
 
@@ -22,3 +23,5 @@ If applicable, add error messages and tracebacks to help explain your problem.
 
 **Additional context**
 Add any other context about the problem here.
+
+(please also see https://instaloader.github.io/contributing.html for how to report a bug)

.github/ISSUE_TEMPLATE/feature_request.md

@@ -1,6 +1,7 @@
 ---
-name: Feature request
-about: Suggest an idea for this project
+name: 🚀 Feature Suggestion
+about: Suggest an idea for this project 💡
+labels: "feature suggestion"
 
 ---
 
@@ -13,5 +14,11 @@ A clear and concise description of what you want to happen.
 **Describe alternatives you've considered**
 A clear and concise description of any alternative solutions or features you've considered.
 
+**If the feature request is accepted, would you be willing to submit a PR?**
+
+Yes / No _(Help can be provided if you need assistance submitting a PR)_
+
 **Additional context**
 Add any other context about the feature request here.
+
+(please also see https://instaloader.github.io/contributing.html for how to suggest a feature)

.github/ISSUE_TEMPLATE/question.md

@@ -0,0 +1,8 @@
+---
+name: ❓ Question
+about: If you have a question about how to use Instaloader that's not well covered by our documentation
+labels: question
+
+---
+
+Your question here...

.github/workflows/stale.yml

@@ -0,0 +1,24 @@
+name: Mark stale issues and pull requests
+
+on:
+  schedule:
+  - cron: "15 16 * * *"
+
+jobs:
+  stale:
+
+    runs-on: ubuntu-latest
+    
+    steps:
+    - uses: actions/stale@v1
+      with:
+        repo-token: ${{ secrets.GITHUB_TOKEN }}
+        stale-issue-message: 'There has been no activity on this issue for an extended period of time. This issue will be closed after further 14 days of inactivity.'
+        stale-pr-message: 'There has been no activity on this Pull Request for an extended period of time. This Pull Request will be closed after further 14 days of inactivity.'
+        stale-issue-label: 'stale'
+        stale-pr-label: 'stale'
+        exempt-issue-label: 'leave open'
+        exempt-pr-label: 'leave open'
+        days-before-stale: 90
+        days-before-close: 14
+        operations-per-run: 7

.github/workflows/windows_exe.yml

@@ -0,0 +1,47 @@
+name: Windows EXE build
+
+on: [push, pull_request]
+
+jobs:
+  build:
+    runs-on: windows-latest
+    steps:
+      - name: Checkout Instaloader repository
+        uses: actions/checkout@v2
+      - name: Setup Python
+        uses: actions/setup-python@v1
+        with:
+          python-version: 3.8
+          architecture: x64
+      - name: Get the tagged version
+        id: get_version
+        env:
+          GITHUB_REF: ${{ github.ref }}
+        run: echo ::set-output name=VERSION::${GITHUB_REF/refs\/tags\/v/}
+        shell: bash
+      - name: Run custom python script for EXE creation
+        env:
+          VERSION_TAG: v${{ steps.get_version.outputs.VERSION }}
+        run: python deploy/windows/create_exe.py
+      - name: Create draft release
+        if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v')
+        id: create_release
+        uses: actions/create-release@v1
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        with:
+          tag_name: v${{ steps.get_version.outputs.VERSION }}
+          release_name: Version ${{ steps.get_version.outputs.VERSION }}
+          draft: true
+          prerelease: contains(steps.get_version.outputs.VERSION, 'a') || contains(steps.get_version.outputs.VERSION, 'b') || contains(steps.get_version.outputs.VERSION, 'c')
+      - name: Upload EXE as release asset
+        if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v')
+        id: upload-release-asset
+        uses: actions/upload-release-asset@v1
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        with:
+          upload_url: ${{ steps.create_release.outputs.upload_url }}
+          asset_path: ./instaloader-v${{ steps.get_version.outputs.VERSION }}-windows-standalone.zip
+          asset_name: instaloader-v${{ steps.get_version.outputs.VERSION }}-windows-standalone.zip
+          asset_content_type: application/zip

.gitignore

@@ -1 +1,2 @@
 __pycache__/
+.mypy_cache/

.pylintrc

@@ -0,0 +1,584 @@
+[MASTER]
+
+# A comma-separated list of package or module names from where C extensions may
+# be loaded. Extensions are loading into the active Python interpreter and may
+# run arbitrary code.
+extension-pkg-whitelist=
+
+# Add files or directories to the blacklist. They should be base names, not
+# paths.
+ignore=CVS
+
+# Add files or directories matching the regex patterns to the blacklist. The
+# regex matches against base names, not paths.
+ignore-patterns=
+
+# Python code to execute, usually for sys.path manipulation such as
+# pygtk.require().
+#init-hook=
+
+# Use multiple processes to speed up Pylint. Specifying 0 will auto-detect the
+# number of processors available to use.
+jobs=0
+
+# Control the amount of potential inferred values when inferring a single
+# object. This can help the performance when dealing with large functions or
+# complex, nested conditions.
+limit-inference-results=100
+
+# List of plugins (as comma separated values of python modules names) to load,
+# usually to register additional checkers.
+load-plugins=
+
+# Pickle collected data for later comparisons.
+persistent=yes
+
+# Specify a configuration file.
+#rcfile=
+
+# When enabled, pylint would attempt to guess common misconfiguration and emit
+# user-friendly hints instead of false-positive error messages.
+suggestion-mode=yes
+
+# Allow loading of arbitrary C extensions. Extensions are imported into the
+# active Python interpreter and may run arbitrary code.
+unsafe-load-any-extension=no
+
+
+[MESSAGES CONTROL]
+
+# Only show warnings with the listed confidence levels. Leave empty to show
+# all. Valid levels: HIGH, INFERENCE, INFERENCE_FAILURE, UNDEFINED.
+confidence=
+
+# Disable the message, report, category or checker with the given id(s). You
+# can either give multiple identifiers separated by comma (,) or put this
+# option multiple times (only on the command line, not in the configuration
+# file where it should appear only once). You can also use "--disable=all" to
+# disable everything first and then reenable specific checks. For example, if
+# you want to run only the similarities checker, you can use "--disable=all
+# --enable=similarities". If you want to run only the classes checker, but have
+# no Warning level messages displayed, use "--disable=all --enable=classes
+# --disable=W".
+disable=invalid-name,
+        missing-docstring,
+        too-many-lines,
+        print-statement,
+        parameter-unpacking,
+        unpacking-in-except,
+        old-raise-syntax,
+        backtick,
+        long-suffix,
+        old-ne-operator,
+        old-octal-literal,
+        import-star-module-level,
+        non-ascii-bytes-literal,
+        raw-checker-failed,
+        bad-inline-option,
+        locally-disabled,
+        file-ignored,
+        suppressed-message,
+        useless-suppression,
+        deprecated-pragma,
+        use-symbolic-message-instead,
+        too-many-instance-attributes,
+        too-many-public-methods,
+        too-many-branches,
+        too-many-arguments,
+        too-many-locals,
+        too-many-statements,
+        no-else-return,
+        no-else-raise,
+        unnecessary-pass,
+        wildcard-import,
+        unused-wildcard-import,
+        apply-builtin,
+        basestring-builtin,
+        buffer-builtin,
+        cmp-builtin,
+        coerce-builtin,
+        execfile-builtin,
+        file-builtin,
+        long-builtin,
+        raw_input-builtin,
+        reduce-builtin,
+        standarderror-builtin,
+        unicode-builtin,
+        xrange-builtin,
+        coerce-method,
+        delslice-method,
+        getslice-method,
+        setslice-method,
+        no-absolute-import,
+        old-division,
+        dict-iter-method,
+        dict-view-method,
+        next-method-called,
+        metaclass-assignment,
+        indexing-exception,
+        raising-string,
+        reload-builtin,
+        oct-method,
+        hex-method,
+        nonzero-method,
+        cmp-method,
+        input-builtin,
+        round-builtin,
+        intern-builtin,
+        unichr-builtin,
+        map-builtin-not-iterating,
+        zip-builtin-not-iterating,
+        range-builtin-not-iterating,
+        filter-builtin-not-iterating,
+        using-cmp-argument,
+        eq-without-hash,
+        div-method,
+        idiv-method,
+        rdiv-method,
+        exception-message-attribute,
+        invalid-str-codec,
+        sys-max-int,
+        bad-python3-import,
+        deprecated-string-function,
+        deprecated-str-translate-call,
+        deprecated-itertools-function,
+        deprecated-types-field,
+        next-method-defined,
+        dict-items-not-iterating,
+        dict-keys-not-iterating,
+        dict-values-not-iterating,
+        deprecated-operator-function,
+        deprecated-urllib-function,
+        xreadlines-attribute,
+        deprecated-sys-function,
+        exception-escape,
+        comprehension-escape
+
+# Enable the message, report, category or checker with the given id(s). You can
+# either give multiple identifier separated by comma (,) or put this option
+# multiple time (only on the command line, not in the configuration file where
+# it should appear only once). See also the "--disable" option for examples.
+enable=c-extension-no-member
+
+
+[REPORTS]
+
+# Python expression which should return a note less than 10 (10 is the highest
+# note). You have access to the variables errors warning, statement which
+# respectively contain the number of errors / warnings messages and the total
+# number of statements analyzed. This is used by the global evaluation report
+# (RP0004).
+evaluation=10.0 - ((float(5 * error + warning + refactor + convention) / statement) * 10)
+
+# Template used to display messages. This is a python new-style format string
+# used to format the message information. See doc for all details.
+#msg-template=
+
+# Set the output format. Available formats are text, parseable, colorized, json
+# and msvs (visual studio). You can also give a reporter class, e.g.
+# mypackage.mymodule.MyReporterClass.
+output-format=text
+
+# Tells whether to display a full report or only the messages.
+reports=no
+
+# Activate the evaluation score.
+score=yes
+
+
+[REFACTORING]
+
+# Maximum number of nested blocks for function / method body
+max-nested-blocks=5
+
+# Complete name of functions that never returns. When checking for
+# inconsistent-return-statements if a never returning function is called then
+# it will be considered as an explicit return statement and no message will be
+# printed.
+never-returning-functions=sys.exit
+
+
+[VARIABLES]
+
+# List of additional names supposed to be defined in builtins. Remember that
+# you should avoid defining new builtins when possible.
+additional-builtins=
+
+# Tells whether unused global variables should be treated as a violation.
+allow-global-unused-variables=yes
+
+# List of strings which can identify a callback function by name. A callback
+# name must start or end with one of those strings.
+callbacks=cb_,
+          _cb
+
+# A regular expression matching the name of dummy variables (i.e. expected to
+# not be used).
+dummy-variables-rgx=_+$|(_[a-zA-Z0-9_]*[a-zA-Z0-9]+?$)|dummy|^ignored_|^unused_
+
+# Argument names that match this expression will be ignored. Default to name
+# with leading underscore.
+ignored-argument-names=_.*|^ignored_|^unused_
+
+# Tells whether we should check for unused import in __init__ files.
+init-import=no
+
+# List of qualified module names which can have objects that can redefine
+# builtins.
+redefining-builtins-modules=six.moves,past.builtins,future.builtins,builtins,io
+
+
+[TYPECHECK]
+
+# List of decorators that produce context managers, such as
+# contextlib.contextmanager. Add to this list to register other decorators that
+# produce valid context managers.
+contextmanager-decorators=contextlib.contextmanager
+
+# List of members which are set dynamically and missed by pylint inference
+# system, and so shouldn't trigger E1101 when accessed. Python regular
+# expressions are accepted.
+generated-members=
+
+# Tells whether missing members accessed in mixin class should be ignored. A
+# mixin class is detected if its name ends with "mixin" (case insensitive).
+ignore-mixin-members=yes
+
+# Tells whether to warn about missing members when the owner of the attribute
+# is inferred to be None.
+ignore-none=yes
+
+# This flag controls whether pylint should warn about no-member and similar
+# checks whenever an opaque object is returned when inferring. The inference
+# can return multiple potential results while evaluating a Python object, but
+# some branches might not be evaluated, which results in partial inference. In
+# that case, it might be useful to still emit no-member and other checks for
+# the rest of the inferred objects.
+ignore-on-opaque-inference=yes
+
+# List of class names for which member attributes should not be checked (useful
+# for classes with dynamically set attributes). This supports the use of
+# qualified names.
+ignored-classes=optparse.Values,thread._local,_thread._local
+
+# List of module names for which member attributes should not be checked
+# (useful for modules/projects where namespaces are manipulated during runtime
+# and thus existing member attributes cannot be deduced by static analysis. It
+# supports qualified module names, as well as Unix pattern matching.
+ignored-modules=
+
+# Show a hint with possible names when a member name was not found. The aspect
+# of finding the hint is based on edit distance.
+missing-member-hint=yes
+
+# The minimum edit distance a name should have in order to be considered a
+# similar match for a missing member name.
+missing-member-hint-distance=1
+
+# The total number of similar names that should be taken in consideration when
+# showing a hint for a missing member.
+missing-member-max-choices=1
+
+
+[STRING]
+
+# This flag controls whether the implicit-str-concat-in-sequence should
+# generate a warning on implicit string concatenation in sequences defined over
+# several lines.
+check-str-concat-over-line-jumps=no
+
+
+[SPELLING]
+
+# Limits count of emitted suggestions for spelling mistakes.
+max-spelling-suggestions=4
+
+# Spelling dictionary name. Available dictionaries: none. To make it working
+# install python-enchant package..
+spelling-dict=
+
+# List of comma separated words that should not be checked.
+spelling-ignore-words=
+
+# A path to a file that contains private dictionary; one word per line.
+spelling-private-dict-file=
+
+# Tells whether to store unknown words to indicated private dictionary in
+# --spelling-private-dict-file option instead of raising a message.
+spelling-store-unknown-words=no
+
+
+[SIMILARITIES]
+
+# Ignore comments when computing similarities.
+ignore-comments=yes
+
+# Ignore docstrings when computing similarities.
+ignore-docstrings=yes
+
+# Ignore imports when computing similarities.
+ignore-imports=no
+
+# Minimum lines number of a similarity.
+min-similarity-lines=4
+
+
+[MISCELLANEOUS]
+
+# List of note tags to take in consideration, separated by a comma.
+notes=FIXME,
+      XXX,
+      TODO
+
+
+[LOGGING]
+
+# Format style used to check logging format string. `old` means using %
+# formatting, while `new` is for `{}` formatting.
+logging-format-style=old
+
+# Logging modules to check that the string format arguments are in logging
+# function parameter format.
+logging-modules=logging
+
+
+[FORMAT]
+
+# Expected format of line ending, e.g. empty (any line ending), LF or CRLF.
+expected-line-ending-format=
+
+# Regexp for a line that is allowed to be longer than the limit.
+ignore-long-lines=^\s*(# )?<?https?://\S+>?$
+
+# Number of spaces of indent required inside a hanging or continued line.
+indent-after-paren=4
+
+# String used as indentation unit. This is usually "    " (4 spaces) or "\t" (1
+# tab).
+indent-string='    '
+
+# Maximum number of characters on a single line.
+max-line-length=120
+
+# Maximum number of lines in a module.
+max-module-lines=1000
+
+# List of optional constructs for which whitespace checking is disabled. `dict-
+# separator` is used to allow tabulation in dicts, etc.: {1  : 1,\n222: 2}.
+# `trailing-comma` allows a space between comma and closing bracket: (a, ).
+# `empty-line` allows space-only lines.
+no-space-check=trailing-comma,
+               dict-separator
+
+# Allow the body of a class to be on the same line as the declaration if body
+# contains single statement.
+single-line-class-stmt=no
+
+# Allow the body of an if to be on the same line as the test if there is no
+# else.
+single-line-if-stmt=no
+
+
+[BASIC]
+
+# Naming style matching correct argument names.
+argument-naming-style=snake_case
+
+# Regular expression matching correct argument names. Overrides argument-
+# naming-style.
+#argument-rgx=
+
+# Naming style matching correct attribute names.
+attr-naming-style=snake_case
+
+# Regular expression matching correct attribute names. Overrides attr-naming-
+# style.
+#attr-rgx=
+
+# Bad variable names which should always be refused, separated by a comma.
+bad-names=foo,
+          bar,
+          baz,
+          toto,
+          tutu,
+          tata
+
+# Naming style matching correct class attribute names.
+class-attribute-naming-style=any
+
+# Regular expression matching correct class attribute names. Overrides class-
+# attribute-naming-style.
+#class-attribute-rgx=
+
+# Naming style matching correct class names.
+class-naming-style=PascalCase
+
+# Regular expression matching correct class names. Overrides class-naming-
+# style.
+#class-rgx=
+
+# Naming style matching correct constant names.
+const-naming-style=UPPER_CASE
+
+# Regular expression matching correct constant names. Overrides const-naming-
+# style.
+#const-rgx=
+
+# Minimum line length for functions/classes that require docstrings, shorter
+# ones are exempt.
+docstring-min-length=-1
+
+# Naming style matching correct function names.
+function-naming-style=snake_case
+
+# Regular expression matching correct function names. Overrides function-
+# naming-style.
+#function-rgx=
+
+# Good variable names which should always be accepted, separated by a comma.
+good-names=i,
+           j,
+           k,
+           ex,
+           Run,
+           _
+
+# Include a hint for the correct naming format with invalid-name.
+include-naming-hint=no
+
+# Naming style matching correct inline iteration names.
+inlinevar-naming-style=any
+
+# Regular expression matching correct inline iteration names. Overrides
+# inlinevar-naming-style.
+#inlinevar-rgx=
+
+# Naming style matching correct method names.
+method-naming-style=snake_case
+
+# Regular expression matching correct method names. Overrides method-naming-
+# style.
+#method-rgx=
+
+# Naming style matching correct module names.
+module-naming-style=snake_case
+
+# Regular expression matching correct module names. Overrides module-naming-
+# style.
+#module-rgx=
+
+# Colon-delimited sets of names that determine each other's naming style when
+# the name regexes allow several styles.
+name-group=
+
+# Regular expression which should only match function or class names that do
+# not require a docstring.
+no-docstring-rgx=^_
+
+# List of decorators that produce properties, such as abc.abstractproperty. Add
+# to this list to register other decorators that produce valid properties.
+# These decorators are taken in consideration only for invalid-name.
+property-classes=abc.abstractproperty
+
+# Naming style matching correct variable names.
+variable-naming-style=snake_case
+
+# Regular expression matching correct variable names. Overrides variable-
+# naming-style.
+#variable-rgx=
+
+
+[IMPORTS]
+
+# Allow wildcard imports from modules that define __all__.
+allow-wildcard-with-all=no
+
+# Analyse import fallback blocks. This can be used to support both Python 2 and
+# 3 compatible code, which means that the block might have code that exists
+# only in one or another interpreter, leading to false positives when analysed.
+analyse-fallback-blocks=no
+
+# Deprecated modules which should not be used, separated by a comma.
+deprecated-modules=optparse,tkinter.tix
+
+# Create a graph of external dependencies in the given file (report RP0402 must
+# not be disabled).
+ext-import-graph=
+
+# Create a graph of every (i.e. internal and external) dependencies in the
+# given file (report RP0402 must not be disabled).
+import-graph=
+
+# Create a graph of internal dependencies in the given file (report RP0402 must
+# not be disabled).
+int-import-graph=
+
+# Force import order to recognize a module as part of the standard
+# compatibility libraries.
+known-standard-library=
+
+# Force import order to recognize a module as part of a third party library.
+known-third-party=enchant
+
+
+[DESIGN]
+
+# Maximum number of arguments for function / method.
+max-args=5
+
+# Maximum number of attributes for a class (see R0902).
+max-attributes=7
+
+# Maximum number of boolean expressions in an if statement.
+max-bool-expr=5
+
+# Maximum number of branch for function / method body.
+max-branches=12
+
+# Maximum number of locals for function / method body.
+max-locals=15
+
+# Maximum number of parents for a class (see R0901).
+max-parents=7
+
+# Maximum number of public methods for a class (see R0904).
+max-public-methods=20
+
+# Maximum number of return / yield for function / method body.
+max-returns=6
+
+# Maximum number of statements in function / method body.
+max-statements=50
+
+# Minimum number of public methods for a class (see R0903).
+min-public-methods=2
+
+
+[CLASSES]
+
+# List of method names used to declare (i.e. assign) instance attributes.
+defining-attr-methods=__init__,
+                      __new__,
+                      setUp
+
+# List of member names, which should be excluded from the protected access
+# warning.
+exclude-protected=_asdict,
+                  _fields,
+                  _replace,
+                  _source,
+                  _make
+
+# List of valid names for the first argument in a class method.
+valid-classmethod-first-arg=cls
+
+# List of valid names for the first argument in a metaclass class method.
+valid-metaclass-classmethod-first-arg=cls
+
+
+[EXCEPTIONS]
+
+# Exceptions that will emit a warning when being caught. Defaults to
+# "BaseException, Exception".
+overgeneral-exceptions=BaseException,
+                       Exception

.travis.yml

@@ -1,15 +1,17 @@
+dist: xenial
 language: python
 python:
  - "3.5"
- - "3.5-dev"
  - "3.6"
- - "3.6-dev"
+ - "3.7"
 install:
- - pip install pylint requests
- - pip install -r docs/requirements.txt
+ - pipenv sync --dev
 script:
- - python3 -m pylint -r n -d bad-whitespace,missing-docstring,too-many-arguments,locally-disabled,line-too-long,too-many-public-methods,too-many-lines,too-many-instance-attributes,too-many-locals,too-many-branches,too-many-statements,inconsistent-return-statements,invalid-name,wildcard-import,unused-wildcard-import,no-else-return instaloader
+ - pylint instaloader
+ - mypy -m instaloader
  - make -C docs html SPHINXOPTS="-W -n"
+addons:
+  ssh_known_hosts: aur.archlinux.org
 deploy:
   - provider: pypi
     user: aandergr
@@ -25,5 +27,10 @@ deploy:
     repo: instaloader/instaloader.github.io
     target-branch: master
     on:
-      branch: master
+      tags: true
+      python: 3.6
+  - provider: script
+    script: deploy/arch/deploy.sh $TRAVIS_TAG
+    on:
+      tags: true
       python: 3.6

LICENSE

@@ -1,6 +1,6 @@
 The MIT License (MIT)
 
-Copyright (c) 2016-2018 Alexander Graf and André Koch-Kramer.
+Copyright (c) 2016-2019 Alexander Graf and André Koch-Kramer.
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

Pipfile

@@ -0,0 +1,16 @@
+[[source]]
+name = "pypi"
+url = "https://pypi.org/simple"
+verify_ssl = true
+
+[dev-packages]
+pylint = "*"
+mypy = "*"
+sphinx = "*"
+pyinstaller = "*"
+pefile = "*"
+pywin32-ctypes = "*"
+psutil = "*"
+
+[packages]
+requests = "*"

Pipfile.lock

@@ -0,0 +1,431 @@
+{
+    "_meta": {
+        "hash": {
+            "sha256": "33408a60fda1a56bf680462fad21643ae0159bc13f3b85d8f10ebf3eef7d8725"
+        },
+        "pipfile-spec": 6,
+        "requires": {},
+        "sources": [
+            {
+                "name": "pypi",
+                "url": "https://pypi.org/simple",
+                "verify_ssl": true
+            }
+        ]
+    },
+    "default": {
+        "certifi": {
+            "hashes": [
+                "sha256:017c25db2a153ce562900032d5bc68e9f191e44e9a0f762f373977de9df1fbb3",
+                "sha256:25b64c7da4cd7479594d035c08c2d809eb4aab3a26e5a990ea98cc450c320f1f"
+            ],
+            "version": "==2019.11.28"
+        },
+        "chardet": {
+            "hashes": [
+                "sha256:84ab92ed1c4d4f16916e05906b6b75a6c0fb5db821cc65e70cbd64a3e2a5eaae",
+                "sha256:fc323ffcaeaed0e0a02bf4d117757b98aed530d9ed4531e3e15460124c106691"
+            ],
+            "version": "==3.0.4"
+        },
+        "idna": {
+            "hashes": [
+                "sha256:c357b3f628cf53ae2c4c05627ecc484553142ca23264e593d327bcde5e9c3407",
+                "sha256:ea8b7f6188e6fa117537c3df7da9fc686d485087abf6ac197f9c46432f7e4a3c"
+            ],
+            "version": "==2.8"
+        },
+        "requests": {
+            "hashes": [
+                "sha256:11e007a8a2aa0323f5a921e9e6a2d7e4e67d9877e85773fba9ba6419025cbeb4",
+                "sha256:9cf5292fcd0f598c671cfc1e0d7d1a7f13bb8085e9a590f48c010551dc6c4b31"
+            ],
+            "index": "pypi",
+            "version": "==2.22.0"
+        },
+        "urllib3": {
+            "hashes": [
+                "sha256:2f3db8b19923a873b3e5256dc9c2dedfa883e33d87c690d9c7913e1f40673cdc",
+                "sha256:87716c2d2a7121198ebcb7ce7cccf6ce5e9ba539041cfbaeecfb641dc0bf6acc"
+            ],
+            "version": "==1.25.8"
+        }
+    },
+    "develop": {
+        "alabaster": {
+            "hashes": [
+                "sha256:446438bdcca0e05bd45ea2de1668c1d9b032e1a9154c2c259092d77031ddd359",
+                "sha256:a661d72d58e6ea8a57f7a86e37d86716863ee5e92788398526d58b26a4e4dc02"
+            ],
+            "version": "==0.7.12"
+        },
+        "altgraph": {
+            "hashes": [
+                "sha256:1f05a47122542f97028caf78775a095fbe6a2699b5089de8477eb583167d69aa",
+                "sha256:c623e5f3408ca61d4016f23a681b9adb100802ca3e3da5e718915a9e4052cebe"
+            ],
+            "version": "==0.17"
+        },
+        "astroid": {
+            "hashes": [
+                "sha256:71ea07f44df9568a75d0f354c49143a4575d90645e9fead6dfb52c26a85ed13a",
+                "sha256:840947ebfa8b58f318d42301cf8c0a20fd794a33b61cc4638e28e9e61ba32f42"
+            ],
+            "version": "==2.3.3"
+        },
+        "babel": {
+            "hashes": [
+                "sha256:1aac2ae2d0d8ea368fa90906567f5c08463d98ade155c0c4bfedd6a0f7160e38",
+                "sha256:d670ea0b10f8b723672d3a6abeb87b565b244da220d76b4dba1b66269ec152d4"
+            ],
+            "version": "==2.8.0"
+        },
+        "certifi": {
+            "hashes": [
+                "sha256:017c25db2a153ce562900032d5bc68e9f191e44e9a0f762f373977de9df1fbb3",
+                "sha256:25b64c7da4cd7479594d035c08c2d809eb4aab3a26e5a990ea98cc450c320f1f"
+            ],
+            "version": "==2019.11.28"
+        },
+        "chardet": {
+            "hashes": [
+                "sha256:84ab92ed1c4d4f16916e05906b6b75a6c0fb5db821cc65e70cbd64a3e2a5eaae",
+                "sha256:fc323ffcaeaed0e0a02bf4d117757b98aed530d9ed4531e3e15460124c106691"
+            ],
+            "version": "==3.0.4"
+        },
+        "docutils": {
+            "hashes": [
+                "sha256:0c5b78adfbf7762415433f5515cd5c9e762339e23369dbe8000d84a4bf4ab3af",
+                "sha256:c2de3a60e9e7d07be26b7f2b00ca0309c207e06c100f9cc2a94931fc75a478fc"
+            ],
+            "version": "==0.16"
+        },
+        "future": {
+            "hashes": [
+                "sha256:b1bead90b70cf6ec3f0710ae53a525360fa360d306a86583adc6bf83a4db537d"
+            ],
+            "version": "==0.18.2"
+        },
+        "idna": {
+            "hashes": [
+                "sha256:c357b3f628cf53ae2c4c05627ecc484553142ca23264e593d327bcde5e9c3407",
+                "sha256:ea8b7f6188e6fa117537c3df7da9fc686d485087abf6ac197f9c46432f7e4a3c"
+            ],
+            "version": "==2.8"
+        },
+        "imagesize": {
+            "hashes": [
+                "sha256:6965f19a6a2039c7d48bca7dba2473069ff854c36ae6f19d2cde309d998228a1",
+                "sha256:b1f6b5a4eab1f73479a50fb79fcf729514a900c341d8503d62a62dbc4127a2b1"
+            ],
+            "version": "==1.2.0"
+        },
+        "isort": {
+            "hashes": [
+                "sha256:54da7e92468955c4fceacd0c86bd0ec997b0e1ee80d97f67c35a78b719dccab1",
+                "sha256:6e811fcb295968434526407adb8796944f1988c5b65e8139058f2014cbe100fd"
+            ],
+            "version": "==4.3.21"
+        },
+        "jinja2": {
+            "hashes": [
+                "sha256:93187ffbc7808079673ef52771baa950426fd664d3aad1d0fa3e95644360e250",
+                "sha256:b0eaf100007721b5c16c1fc1eecb87409464edc10469ddc9a22a27a99123be49"
+            ],
+            "version": "==2.11.1"
+        },
+        "lazy-object-proxy": {
+            "hashes": [
+                "sha256:0c4b206227a8097f05c4dbdd323c50edf81f15db3b8dc064d08c62d37e1a504d",
+                "sha256:194d092e6f246b906e8f70884e620e459fc54db3259e60cf69a4d66c3fda3449",
+                "sha256:1be7e4c9f96948003609aa6c974ae59830a6baecc5376c25c92d7d697e684c08",
+                "sha256:4677f594e474c91da97f489fea5b7daa17b5517190899cf213697e48d3902f5a",
+                "sha256:48dab84ebd4831077b150572aec802f303117c8cc5c871e182447281ebf3ac50",
+                "sha256:5541cada25cd173702dbd99f8e22434105456314462326f06dba3e180f203dfd",
+                "sha256:59f79fef100b09564bc2df42ea2d8d21a64fdcda64979c0fa3db7bdaabaf6239",
+                "sha256:8d859b89baf8ef7f8bc6b00aa20316483d67f0b1cbf422f5b4dc56701c8f2ffb",
+                "sha256:9254f4358b9b541e3441b007a0ea0764b9d056afdeafc1a5569eee1cc6c1b9ea",
+                "sha256:9651375199045a358eb6741df3e02a651e0330be090b3bc79f6d0de31a80ec3e",
+                "sha256:97bb5884f6f1cdce0099f86b907aa41c970c3c672ac8b9c8352789e103cf3156",
+                "sha256:9b15f3f4c0f35727d3a0fba4b770b3c4ebbb1fa907dbcc046a1d2799f3edd142",
+                "sha256:a2238e9d1bb71a56cd710611a1614d1194dc10a175c1e08d75e1a7bcc250d442",
+                "sha256:a6ae12d08c0bf9909ce12385803a543bfe99b95fe01e752536a60af2b7797c62",
+                "sha256:ca0a928a3ddbc5725be2dd1cf895ec0a254798915fb3a36af0964a0a4149e3db",
+                "sha256:cb2c7c57005a6804ab66f106ceb8482da55f5314b7fcb06551db1edae4ad1531",
+                "sha256:d74bb8693bf9cf75ac3b47a54d716bbb1a92648d5f781fc799347cfc95952383",
+                "sha256:d945239a5639b3ff35b70a88c5f2f491913eb94871780ebfabb2568bd58afc5a",
+                "sha256:eba7011090323c1dadf18b3b689845fd96a61ba0a1dfbd7f24b921398affc357",
+                "sha256:efa1909120ce98bbb3777e8b6f92237f5d5c8ea6758efea36a473e1d38f7d3e4",
+                "sha256:f3900e8a5de27447acbf900b4750b0ddfd7ec1ea7fbaf11dfa911141bc522af0"
+            ],
+            "version": "==1.4.3"
+        },
+        "markupsafe": {
+            "hashes": [
+                "sha256:00bc623926325b26bb9605ae9eae8a215691f33cae5df11ca5424f06f2d1f473",
+                "sha256:09027a7803a62ca78792ad89403b1b7a73a01c8cb65909cd876f7fcebd79b161",
+                "sha256:09c4b7f37d6c648cb13f9230d847adf22f8171b1ccc4d5682398e77f40309235",
+                "sha256:1027c282dad077d0bae18be6794e6b6b8c91d58ed8a8d89a89d59693b9131db5",
+                "sha256:13d3144e1e340870b25e7b10b98d779608c02016d5184cfb9927a9f10c689f42",
+                "sha256:24982cc2533820871eba85ba648cd53d8623687ff11cbb805be4ff7b4c971aff",
+                "sha256:29872e92839765e546828bb7754a68c418d927cd064fd4708fab9fe9c8bb116b",
+                "sha256:43a55c2930bbc139570ac2452adf3d70cdbb3cfe5912c71cdce1c2c6bbd9c5d1",
+                "sha256:46c99d2de99945ec5cb54f23c8cd5689f6d7177305ebff350a58ce5f8de1669e",
+                "sha256:500d4957e52ddc3351cabf489e79c91c17f6e0899158447047588650b5e69183",
+                "sha256:535f6fc4d397c1563d08b88e485c3496cf5784e927af890fb3c3aac7f933ec66",
+                "sha256:596510de112c685489095da617b5bcbbac7dd6384aeebeda4df6025d0256a81b",
+                "sha256:62fe6c95e3ec8a7fad637b7f3d372c15ec1caa01ab47926cfdf7a75b40e0eac1",
+                "sha256:6788b695d50a51edb699cb55e35487e430fa21f1ed838122d722e0ff0ac5ba15",
+                "sha256:6dd73240d2af64df90aa7c4e7481e23825ea70af4b4922f8ede5b9e35f78a3b1",
+                "sha256:717ba8fe3ae9cc0006d7c451f0bb265ee07739daf76355d06366154ee68d221e",
+                "sha256:79855e1c5b8da654cf486b830bd42c06e8780cea587384cf6545b7d9ac013a0b",
+                "sha256:7c1699dfe0cf8ff607dbdcc1e9b9af1755371f92a68f706051cc8c37d447c905",
+                "sha256:88e5fcfb52ee7b911e8bb6d6aa2fd21fbecc674eadd44118a9cc3863f938e735",
+                "sha256:8defac2f2ccd6805ebf65f5eeb132adcf2ab57aa11fdf4c0dd5169a004710e7d",
+                "sha256:98c7086708b163d425c67c7a91bad6e466bb99d797aa64f965e9d25c12111a5e",
+                "sha256:9add70b36c5666a2ed02b43b335fe19002ee5235efd4b8a89bfcf9005bebac0d",
+                "sha256:9bf40443012702a1d2070043cb6291650a0841ece432556f784f004937f0f32c",
+                "sha256:ade5e387d2ad0d7ebf59146cc00c8044acbd863725f887353a10df825fc8ae21",
+                "sha256:b00c1de48212e4cc9603895652c5c410df699856a2853135b3967591e4beebc2",
+                "sha256:b1282f8c00509d99fef04d8ba936b156d419be841854fe901d8ae224c59f0be5",
+                "sha256:b2051432115498d3562c084a49bba65d97cf251f5a331c64a12ee7e04dacc51b",
+                "sha256:ba59edeaa2fc6114428f1637ffff42da1e311e29382d81b339c1817d37ec93c6",
+                "sha256:c8716a48d94b06bb3b2524c2b77e055fb313aeb4ea620c8dd03a105574ba704f",
+                "sha256:cd5df75523866410809ca100dc9681e301e3c27567cf498077e8551b6d20e42f",
+                "sha256:cdb132fc825c38e1aeec2c8aa9338310d29d337bebbd7baa06889d09a60a1fa2",
+                "sha256:e249096428b3ae81b08327a63a485ad0878de3fb939049038579ac0ef61e17e7",
+                "sha256:e8313f01ba26fbbe36c7be1966a7b7424942f670f38e666995b88d012765b9be"
+            ],
+            "version": "==1.1.1"
+        },
+        "mccabe": {
+            "hashes": [
+                "sha256:ab8a6258860da4b6677da4bd2fe5dc2c659cff31b3ee4f7f5d64e79735b80d42",
+                "sha256:dd8d182285a0fe56bace7f45b5e7d1a6ebcbf524e8f3bd87eb0f125271b8831f"
+            ],
+            "version": "==0.6.1"
+        },
+        "mypy": {
+            "hashes": [
+                "sha256:0a9a45157e532da06fe56adcfef8a74629566b607fa2c1ac0122d1ff995c748a",
+                "sha256:2c35cae79ceb20d47facfad51f952df16c2ae9f45db6cb38405a3da1cf8fc0a7",
+                "sha256:4b9365ade157794cef9685791032521233729cb00ce76b0ddc78749abea463d2",
+                "sha256:53ea810ae3f83f9c9b452582261ea859828a9ed666f2e1ca840300b69322c474",
+                "sha256:634aef60b4ff0f650d3e59d4374626ca6153fcaff96ec075b215b568e6ee3cb0",
+                "sha256:7e396ce53cacd5596ff6d191b47ab0ea18f8e0ec04e15d69728d530e86d4c217",
+                "sha256:7eadc91af8270455e0d73565b8964da1642fe226665dd5c9560067cd64d56749",
+                "sha256:7f672d02fffcbace4db2b05369142e0506cdcde20cea0e07c7c2171c4fd11dd6",
+                "sha256:85baab8d74ec601e86134afe2bcccd87820f79d2f8d5798c889507d1088287bf",
+                "sha256:87c556fb85d709dacd4b4cb6167eecc5bbb4f0a9864b69136a0d4640fdc76a36",
+                "sha256:a6bd44efee4dc8c3324c13785a9dc3519b3ee3a92cada42d2b57762b7053b49b",
+                "sha256:c6d27bd20c3ba60d5b02f20bd28e20091d6286a699174dfad515636cb09b5a72",
+                "sha256:e2bb577d10d09a2d8822a042a23b8d62bc3b269667c9eb8e60a6edfa000211b1",
+                "sha256:f97a605d7c8bc2c6d1172c2f0d5a65b24142e11a58de689046e62c2d632ca8c1"
+            ],
+            "index": "pypi",
+            "version": "==0.761"
+        },
+        "mypy-extensions": {
+            "hashes": [
+                "sha256:090fedd75945a69ae91ce1303b5824f428daf5a028d2f6ab8a299250a846f15d",
+                "sha256:2d82818f5bb3e369420cb3c4060a7970edba416647068eb4c5343488a6c604a8"
+            ],
+            "version": "==0.4.3"
+        },
+        "packaging": {
+            "hashes": [
+                "sha256:170748228214b70b672c581a3dd610ee51f733018650740e98c7df862a583f73",
+                "sha256:e665345f9eef0c621aa0bf2f8d78cf6d21904eef16a93f020240b704a57f1334"
+            ],
+            "version": "==20.1"
+        },
+        "pefile": {
+            "hashes": [
+                "sha256:a5d6e8305c6b210849b47a6174ddf9c452b2888340b8177874b862ba6c207645"
+            ],
+            "index": "pypi",
+            "version": "==2019.4.18"
+        },
+        "psutil": {
+            "hashes": [
+                "sha256:094f899ac3ef72422b7e00411b4ed174e3c5a2e04c267db6643937ddba67a05b",
+                "sha256:10b7f75cc8bd676cfc6fa40cd7d5c25b3f45a0e06d43becd7c2d2871cbb5e806",
+                "sha256:1b1575240ca9a90b437e5a40db662acd87bbf181f6aa02f0204978737b913c6b",
+                "sha256:21231ef1c1a89728e29b98a885b8e0a8e00d09018f6da5cdc1f43f988471a995",
+                "sha256:28f771129bfee9fc6b63d83a15d857663bbdcae3828e1cb926e91320a9b5b5cd",
+                "sha256:70387772f84fa5c3bb6a106915a2445e20ac8f9821c5914d7cbde148f4d7ff73",
+                "sha256:b560f5cd86cf8df7bcd258a851ca1ad98f0d5b8b98748e877a0aec4e9032b465",
+                "sha256:b74b43fecce384a57094a83d2778cdfc2e2d9a6afaadd1ebecb2e75e0d34e10d",
+                "sha256:e85f727ffb21539849e6012f47b12f6dd4c44965e56591d8dec6e8bc9ab96f4a",
+                "sha256:fd2e09bb593ad9bdd7429e779699d2d47c1268cbde4dda95fcd1bd17544a0217",
+                "sha256:ffad8eb2ac614518bbe3c0b8eb9dffdb3a8d2e3a7d5da51c5b974fb723a5c5aa"
+            ],
+            "index": "pypi",
+            "version": "==5.6.7"
+        },
+        "pygments": {
+            "hashes": [
+                "sha256:2a3fe295e54a20164a9df49c75fa58526d3be48e14aceba6d6b1e8ac0bfd6f1b",
+                "sha256:98c8aa5a9f778fcd1026a17361ddaf7330d1b7c62ae97c3bb0ae73e0b9b6b0fe"
+            ],
+            "version": "==2.5.2"
+        },
+        "pyinstaller": {
+            "hashes": [
+                "sha256:3730fa80d088f8bb7084d32480eb87cbb4ddb64123363763cf8f2a1378c1c4b7"
+            ],
+            "index": "pypi",
+            "version": "==3.6"
+        },
+        "pylint": {
+            "hashes": [
+                "sha256:3db5468ad013380e987410a8d6956226963aed94ecb5f9d3a28acca6d9ac36cd",
+                "sha256:886e6afc935ea2590b462664b161ca9a5e40168ea99e5300935f6591ad467df4"
+            ],
+            "index": "pypi",
+            "version": "==2.4.4"
+        },
+        "pyparsing": {
+            "hashes": [
+                "sha256:4c830582a84fb022400b85429791bc551f1f4871c33f23e44f353119e92f969f",
+                "sha256:c342dccb5250c08d45fd6f8b4a559613ca603b57498511740e65cd11a2e7dcec"
+            ],
+            "version": "==2.4.6"
+        },
+        "pytz": {
+            "hashes": [
+                "sha256:1c557d7d0e871de1f5ccd5833f60fb2550652da6be2693c1e02300743d21500d",
+                "sha256:b02c06db6cf09c12dd25137e563b31700d3b80fcc4ad23abb7a315f2789819be"
+            ],
+            "version": "==2019.3"
+        },
+        "pywin32-ctypes": {
+            "hashes": [
+                "sha256:24ffc3b341d457d48e8922352130cf2644024a4ff09762a2261fd34c36ee5942",
+                "sha256:9dc2d991b3479cc2df15930958b674a48a227d5361d413827a4cfd0b5876fc98"
+            ],
+            "index": "pypi",
+            "version": "==0.2.0"
+        },
+        "requests": {
+            "hashes": [
+                "sha256:11e007a8a2aa0323f5a921e9e6a2d7e4e67d9877e85773fba9ba6419025cbeb4",
+                "sha256:9cf5292fcd0f598c671cfc1e0d7d1a7f13bb8085e9a590f48c010551dc6c4b31"
+            ],
+            "index": "pypi",
+            "version": "==2.22.0"
+        },
+        "six": {
+            "hashes": [
+                "sha256:236bdbdce46e6e6a3d61a337c0f8b763ca1e8717c03b369e87a7ec7ce1319c0a",
+                "sha256:8f3cd2e254d8f793e7f3d6d9df77b92252b52637291d0f0da013c76ea2724b6c"
+            ],
+            "version": "==1.14.0"
+        },
+        "snowballstemmer": {
+            "hashes": [
+                "sha256:209f257d7533fdb3cb73bdbd24f436239ca3b2fa67d56f6ff88e86be08cc5ef0",
+                "sha256:df3bac3df4c2c01363f3dd2cfa78cce2840a79b9f1c2d2de9ce8d31683992f52"
+            ],
+            "version": "==2.0.0"
+        },
+        "sphinx": {
+            "hashes": [
+                "sha256:298537cb3234578b2d954ff18c5608468229e116a9757af3b831c2b2b4819159",
+                "sha256:e6e766b74f85f37a5f3e0773a1e1be8db3fcb799deb58ca6d18b70b0b44542a5"
+            ],
+            "index": "pypi",
+            "version": "==2.3.1"
+        },
+        "sphinxcontrib-applehelp": {
+            "hashes": [
+                "sha256:edaa0ab2b2bc74403149cb0209d6775c96de797dfd5b5e2a71981309efab3897",
+                "sha256:fb8dee85af95e5c30c91f10e7eb3c8967308518e0f7488a2828ef7bc191d0d5d"
+            ],
+            "version": "==1.0.1"
+        },
+        "sphinxcontrib-devhelp": {
+            "hashes": [
+                "sha256:6c64b077937330a9128a4da74586e8c2130262f014689b4b89e2d08ee7294a34",
+                "sha256:9512ecb00a2b0821a146736b39f7aeb90759834b07e81e8cc23a9c70bacb9981"
+            ],
+            "version": "==1.0.1"
+        },
+        "sphinxcontrib-htmlhelp": {
+            "hashes": [
+                "sha256:4670f99f8951bd78cd4ad2ab962f798f5618b17675c35c5ac3b2132a14ea8422",
+                "sha256:d4fd39a65a625c9df86d7fa8a2d9f3cd8299a3a4b15db63b50aac9e161d8eff7"
+            ],
+            "version": "==1.0.2"
+        },
+        "sphinxcontrib-jsmath": {
+            "hashes": [
+                "sha256:2ec2eaebfb78f3f2078e73666b1415417a116cc848b72e5172e596c871103178",
+                "sha256:a9925e4a4587247ed2191a22df5f6970656cb8ca2bd6284309578f2153e0c4b8"
+            ],
+            "version": "==1.0.1"
+        },
+        "sphinxcontrib-qthelp": {
+            "hashes": [
+                "sha256:513049b93031beb1f57d4daea74068a4feb77aa5630f856fcff2e50de14e9a20",
+                "sha256:79465ce11ae5694ff165becda529a600c754f4bc459778778c7017374d4d406f"
+            ],
+            "version": "==1.0.2"
+        },
+        "sphinxcontrib-serializinghtml": {
+            "hashes": [
+                "sha256:c0efb33f8052c04fd7a26c0a07f1678e8512e0faec19f4aa8f2473a8b81d5227",
+                "sha256:db6615af393650bf1151a6cd39120c29abaf93cc60db8c48eb2dddbfdc3a9768"
+            ],
+            "version": "==1.1.3"
+        },
+        "typed-ast": {
+            "hashes": [
+                "sha256:0666aa36131496aed8f7be0410ff974562ab7eeac11ef351def9ea6fa28f6355",
+                "sha256:0c2c07682d61a629b68433afb159376e24e5b2fd4641d35424e462169c0a7919",
+                "sha256:249862707802d40f7f29f6e1aad8d84b5aa9e44552d2cc17384b209f091276aa",
+                "sha256:24995c843eb0ad11a4527b026b4dde3da70e1f2d8806c99b7b4a7cf491612652",
+                "sha256:269151951236b0f9a6f04015a9004084a5ab0d5f19b57de779f908621e7d8b75",
+                "sha256:4083861b0aa07990b619bd7ddc365eb7fa4b817e99cf5f8d9cf21a42780f6e01",
+                "sha256:498b0f36cc7054c1fead3d7fc59d2150f4d5c6c56ba7fb150c013fbc683a8d2d",
+                "sha256:4e3e5da80ccbebfff202a67bf900d081906c358ccc3d5e3c8aea42fdfdfd51c1",
+                "sha256:6daac9731f172c2a22ade6ed0c00197ee7cc1221aa84cfdf9c31defeb059a907",
+                "sha256:715ff2f2df46121071622063fc7543d9b1fd19ebfc4f5c8895af64a77a8c852c",
+                "sha256:73d785a950fc82dd2a25897d525d003f6378d1cb23ab305578394694202a58c3",
+                "sha256:8c8aaad94455178e3187ab22c8b01a3837f8ee50e09cf31f1ba129eb293ec30b",
+                "sha256:8ce678dbaf790dbdb3eba24056d5364fb45944f33553dd5869b7580cdbb83614",
+                "sha256:aaee9905aee35ba5905cfb3c62f3e83b3bec7b39413f0a7f19be4e547ea01ebb",
+                "sha256:bcd3b13b56ea479b3650b82cabd6b5343a625b0ced5429e4ccad28a8973f301b",
+                "sha256:c9e348e02e4d2b4a8b2eedb48210430658df6951fa484e59de33ff773fbd4b41",
+                "sha256:d205b1b46085271b4e15f670058ce182bd1199e56b317bf2ec004b6a44f911f6",
+                "sha256:d43943ef777f9a1c42bf4e552ba23ac77a6351de620aa9acf64ad54933ad4d34",
+                "sha256:d5d33e9e7af3b34a40dc05f498939f0ebf187f07c385fd58d591c533ad8562fe",
+                "sha256:fc0fea399acb12edbf8a628ba8d2312f583bdbdb3335635db062fa98cf71fca4",
+                "sha256:fe460b922ec15dd205595c9b5b99e2f056fd98ae8f9f56b888e7a17dc2b757e7"
+            ],
+            "version": "==1.4.1"
+        },
+        "typing-extensions": {
+            "hashes": [
+                "sha256:091ecc894d5e908ac75209f10d5b4f118fbdb2eb1ede6a63544054bb1edb41f2",
+                "sha256:910f4656f54de5993ad9304959ce9bb903f90aadc7c67a0bef07e678014e892d",
+                "sha256:cf8b63fedea4d89bab840ecbb93e75578af28f76f66c35889bd7065f5af88575"
+            ],
+            "version": "==3.7.4.1"
+        },
+        "urllib3": {
+            "hashes": [
+                "sha256:2f3db8b19923a873b3e5256dc9c2dedfa883e33d87c690d9c7913e1f40673cdc",
+                "sha256:87716c2d2a7121198ebcb7ce7cccf6ce5e9ba539041cfbaeecfb641dc0bf6acc"
+            ],
+            "version": "==1.25.8"
+        },
+        "wrapt": {
+            "hashes": [
+                "sha256:565a021fd19419476b9362b05eeaa094178de64f8361e44468f9e9d7843901e1"
+            ],
+            "version": "==1.11.2"
+        }
+    }
+}

README.rst

@@ -1,5 +1,37 @@
 .. image:: https://raw.githubusercontent.com/instaloader/instaloader/master/docs/logo_heading.png
 
+.. badges-start
+
+|travis| |pypi| |pyversion| |license| |aur| |contributors| |downloads|
+
+.. |travis| image:: https://img.shields.io/travis/instaloader/instaloader/master.svg
+   :alt: Travis-CI Build Status
+   :target: https://travis-ci.org/instaloader/instaloader
+
+.. |pypi| image:: https://img.shields.io/pypi/v/instaloader.svg
+   :alt: Instaloader PyPI Project Page
+   :target: https://pypi.org/project/instaloader/
+
+.. |license| image:: https://img.shields.io/github/license/instaloader/instaloader.svg
+   :alt: MIT License
+   :target: https://github.com/instaloader/instaloader/blob/master/LICENSE
+
+.. |pyversion| image:: https://img.shields.io/pypi/pyversions/instaloader.svg
+   :alt: Supported Python Versions
+
+.. |contributors| image:: https://img.shields.io/github/contributors/instaloader/instaloader.svg
+   :alt: Contributor Count
+   :target: https://github.com/instaloader/instaloader/graphs/contributors
+
+.. |aur| image:: https://img.shields.io/aur/version/instaloader.svg
+   :alt: Arch User Repository Package
+   :target: https://aur.archlinux.org/packages/instaloader/
+
+.. |downloads| image:: https://img.shields.io/pypi/dm/instaloader.svg
+   :alt: PyPI Download Count
+
+.. badges-end
+
 ::
 
     $ pip3 install instaloader
@@ -21,7 +53,8 @@
 
 ::
 
-    instaloader [--comments] [--geotags] [--stories] [--highlights] [--tagged]
+    instaloader [--comments] [--geotags]
+                [--stories] [--highlights] [--tagged] [--igtv]
                 [--login YOUR-USERNAME] [--fast-update]
                 profile | "#hashtag" | :stories | :feed | :saved
 

deploy/arch/.SRCINFO.template

@@ -0,0 +1,15 @@
+pkgbase = instaloader
+	pkgdesc = Command line tool to download pictures, videos and metadata from Instagram
+	pkgver = {{version}}
+	pkgrel = 1
+	url = https://instaloader.github.io/
+	arch = any
+	license = MIT
+	makedepends = python-setuptools
+	depends = python>=3.5
+	depends = python-requests>=2.4
+	options = !emptydirs
+	source = instaloader-{{version}}.tar.gz::https://codeload.github.com/instaloader/instaloader/tar.gz/v{{version}}
+	sha512sums = {{hash}}
+
+pkgname = instaloader

deploy/arch/PKGBUILD.template

@@ -0,0 +1,27 @@
+# Maintainer: André Koch-Kramer <koch-kramer@web.de>
+
+pkgname=instaloader
+pkgver={{version}}
+pkgrel=1
+pkgdesc="Command line tool to download pictures, videos and metadata from Instagram"
+arch=('any')
+url="https://instaloader.github.io/"
+license=('MIT')
+groups=()
+depends=('python>=3.5' 'python-requests>=2.4')
+makedepends=('python-setuptools')
+options=('!emptydirs')
+source=($pkgname-$pkgver.tar.gz::https://codeload.github.com/instaloader/instaloader/tar.gz/v$pkgver)
+sha512sums=('{{hash}}')
+
+build() {
+	cd "$srcdir/$pkgname-$pkgver"
+	python setup.py clean --all
+	python setup.py build
+}
+
+package() {
+	cd "$srcdir/$pkgname-$pkgver"
+	install -Dm644 LICENSE "$pkgdir"/usr/share/licenses/$pkgname/LICENSE
+	python setup.py install --root "$pkgdir/" --skip-build
+}

deploy/arch/deploy.sh

@@ -0,0 +1,30 @@
+#!/usr/bin/env bash
+
+VERSION=${1:1}
+
+# do not deploy pre-releases
+echo $VERSION | grep -qe "[abc]" && exit 0
+
+cd $(dirname $0)
+
+# decrypt and add ssh key
+openssl aes-256-cbc -K $encrypted_354637631c28_key -iv $encrypted_354637631c28_iv -in id_rsa_AUR.enc -out /tmp/AUR_openssh -d
+eval "$(ssh-agent -s)"
+chmod 600 /tmp/AUR_openssh
+ssh-add /tmp/AUR_openssh
+
+# clone and modify AUR repo
+git clone --depth 1 ssh://aur@aur.archlinux.org/instaloader.git
+
+curl -sSfOJ https://codeload.github.com/instaloader/instaloader/tar.gz/$1
+HASH=$(sha512sum instaloader-$VERSION.tar.gz | cut -f1 -d " ")
+sed -e "s/{{version}}/$VERSION/g" -e "s/{{hash}}/$HASH/g" PKGBUILD.template > instaloader/PKGBUILD
+sed -e "s/{{version}}/$VERSION/g" -e "s/{{hash}}/$HASH/g" .SRCINFO.template > instaloader/.SRCINFO
+
+# commit and push changes
+cd instaloader
+git config user.email "koch-kramer@web.de"
+git config user.name "André Koch-Kramer"
+git add .
+git commit -m "Release of version $VERSION"
+git push

deploy/windows/create_exe.py

@@ -0,0 +1,86 @@
+import hashlib
+import os
+import re
+import shutil
+import subprocess
+import sys
+
+# copy the required files into repo root
+shutil.copy('docs/favicon.ico', '.')
+shutil.copy('deploy/windows/instaloader.spec', '.')
+shutil.unpack_archive('deploy/windows/ps', '.', 'xztar')
+
+code = """
+import psutil
+import subprocess
+
+def __main():
+    grandparentpid = psutil.Process(os.getppid()).ppid()
+    grandparentpidsearchstring = ' ' + str(grandparentpid) + ' '
+    if hasattr(sys, "_MEIPASS"):
+        ps = os.path.join(sys._MEIPASS, 'tasklist.exe')
+    else:
+        ps = 'tasklist'
+    popen = subprocess.Popen(ps, stdout=subprocess.PIPE, universal_newlines=True)
+    for examine in iter(popen.stdout.readline, ""):
+        if grandparentpidsearchstring in examine:
+            pname = examine
+            break
+    popen.stdout.close()
+    return_code = popen.wait()
+    if return_code:
+        raise subprocess.CalledProcessError(return_code, ps)
+    if pname[0:12] == 'explorer.exe':
+        subprocess.Popen("cmd /K \\\"{0}\\\"".format(os.path.splitext(os.path.basename(sys.argv[0]))[0]))
+    else:
+        main()
+
+
+if __name__ == "__main__":
+    __main()
+"""
+
+with open('instaloader/__main__.py', 'r') as f:
+    # adjust imports for changed file structure
+    regex = re.compile(r'from (?:(\.[^ ]+ )|\.( ))import')
+    lines = [regex.sub(r'from instaloader\1\2import', line) for line in f.readlines()]
+
+    # insert code for magic exe behavior
+    index = lines.index('if __name__ == "__main__":\n')
+    code_lines = [cl + '\n' for cl in code.splitlines()]
+    for i, code_line in enumerate(code_lines):
+        if i + index < len(lines):
+            lines[i + index] = code_line
+        else:
+            lines.extend(code_lines[i:])
+            break
+
+with open('__main__.py', 'w+') as f:
+    f.writelines(lines)
+
+# install dependencies and invoke PyInstaller
+commands = ["pip install pipenv==2018.11.26",
+            "pipenv sync --dev",
+            "pipenv run pyinstaller --log-level=DEBUG instaloader.spec"]
+
+for command in commands:
+    print()
+    print('#' * (len(command) + 6))
+    print('## {} ##'.format(command))
+    print('#' * (len(command) + 6))
+    print(flush=True)
+    err = subprocess.Popen(command).wait()
+    if err != 0:
+        sys.exit(err)
+
+# calculate and store MD5 hash for created executable
+hash_md5 = hashlib.md5()
+with open('dist/instaloader.exe', 'rb') as f:
+    for chunk in iter(lambda: f.read(4096), b''):
+        hash_md5.update(chunk)
+
+with open('dist/instaloader.exe.md5', 'w+') as f:
+    f.write('{} *instaloader.exe\n'.format(hash_md5.hexdigest()))
+
+# Create ZIP file
+shutil.make_archive('instaloader-{}-windows-standalone'.format(os.getenv('VERSION_TAG')), 'zip', 'dist')

deploy/windows/instaloader.spec

@@ -0,0 +1,28 @@
+# -*- mode: python -*-
+
+block_cipher = None
+
+
+a = Analysis(['__main__.py'],
+             binaries=[],
+             datas=[('tasklist.exe', '.'), ('framedyn.dll', '.')],
+             hiddenimports=[],
+             hookspath=[],
+             runtime_hooks=[],
+             excludes=[],
+             win_no_prefer_redirects=False,
+             win_private_assemblies=False,
+             cipher=block_cipher)
+pyz = PYZ(a.pure, a.zipped_data,
+             cipher=block_cipher)
+exe = EXE(pyz,
+          a.scripts,
+          a.binaries,
+          a.zipfiles,
+          a.datas,
+          name='instaloader',
+          debug=False,
+          strip=False,
+          upx=False,
+          runtime_tmpdir=None,
+          console=True , icon='favicon.ico')

docs/_static/instaloader.css

@@ -0,0 +1,75 @@
+@import url(bootstrap-4.1.3.min.css);
+a {
+  color: #008d06; }
+  a:hover, a.current, a.active {
+    color: #f48400; }
+  a.current, a.active {
+    font-weight: 500; }
+
+.doc-sidebar a:hover, .modal-body a:hover {
+  text-decoration: none; }
+
+code {
+  color: #212529; }
+
+.highlight {
+  border-radius: 0.3em; }
+  .highlight pre {
+    padding: 0.7em;
+    color: #fff; }
+  .highlight .c1 {
+    color: #008d06; }
+  .highlight .k, .highlight .kn, .highlight .ow {
+    color: #f48400;
+    font-weight: bold; }
+  .highlight .nb {
+    color: #f48400; }
+  .highlight .s1, .highlight .s2, .highlight .mi {
+    color: #ff7efd; }
+
+.doc-content {
+  padding-top: 0.5em;
+  top: 3.5625em;
+  position: fixed;
+  bottom: 0;
+  overflow-y: auto; }
+
+.doc-sidebar {
+  position: fixed;
+  padding-top: 0.5em;
+  top: 3.5625em;
+  bottom: 0;
+  right: 0;
+  overflow-y: auto; }
+  .doc-sidebar ul {
+    list-style-type: none;
+    padding-left: 1em; }
+    .doc-sidebar ul.current {
+      padding-left: 0; }
+  .doc-sidebar li {
+    padding-top: 0.2em;
+    padding-bottom: 0.2em; }
+
+.admonition {
+  border-radius: 0.3em;
+  border: 1px solid #ccc;
+  background-color: #f8f9fa; }
+  .admonition .admonition-title {
+    color: #ff7efd; }
+
+.topic {
+  border-radius: 0.3em;
+  background-color: #f8f9fa; }
+
+.nav-link {
+  padding-top: 0;
+  padding-bottom: 0; }
+
+.btn-success {
+  background-color: #008d06;
+  border-color: #008d06; }
+  .btn-success:hover {
+    background-color: #f48400;
+    border-color: #f48400; }
+
+/*# sourceMappingURL=instaloader.css.map */

docs/_static/instaloader.js

@@ -0,0 +1,22 @@
+$(function () {
+    $('[data-toggle="tooltip"]').tooltip();
+
+    $('.doc-sidebar > ul > li.current').attr("id", "localtoc");
+    $('#localtoc ul').addClass("nav flex-column");
+    $('#localtoc ul li').addClass("nav-item");
+    $('#localtoc ul li a').addClass("nav-link");
+    $('.doc-content').scrollspy({target: '#localtoc'});
+
+    const top_href = '#' + $('.section:first').attr("id");
+    $('#localtoc > a.current').attr("href", top_href);
+
+    $('#navbarToc a').on("click", function () {
+        const href = $(this).attr("href");
+        if (href === '#') {
+            window.location.href = top_href;
+        } else {
+            window.location.href = href;
+        }
+        $('#navbarToc').modal('hide');
+    });
+});

docs/_static/instaloader.scss

@@ -0,0 +1,117 @@
+@import 'bootstrap-4.1.3.min.css';
+
+$color_instaloader_main: #008d06;
+$color_instaloader_accent: #f48400;
+$color_instaloader_hyper: #ff7efd;
+$color_normal_text: #212529;
+$color_light_bg: #f8f9fa;
+
+a {
+  color: $color_instaloader_main;
+
+  &:hover, &.current, &.active {
+    color: $color_instaloader_accent;
+  }
+
+  &.current, &.active {
+    font-weight: 500;
+  }
+}
+
+.doc-sidebar, .modal-body {
+  a:hover {
+    text-decoration: none
+  }
+}
+
+code {
+  color: $color_normal_text;
+}
+
+.highlight {
+  border-radius: 0.3em;
+
+  pre {
+    padding: 0.7em;
+    color: #fff;
+  }
+
+  .c1 {
+    color: $color_instaloader_main
+  }
+
+  .k, .kn, .ow {
+    color: $color_instaloader_accent;
+    font-weight: bold
+  }
+
+  .nb {
+    color: $color_instaloader_accent
+  }
+
+  .s1, .s2, .mi {
+    color: $color_instaloader_hyper
+  }
+}
+
+.doc-content {
+  padding-top: 0.5em;
+  top: 3.5625em;
+  position: fixed;
+  bottom: 0;
+  overflow-y: auto;
+}
+
+.doc-sidebar {
+  position: fixed;
+  padding-top: 0.5em;
+  top: 3.5625em;
+  bottom: 0;
+  right: 0;
+  overflow-y: auto;
+
+  ul {
+    list-style-type: none;
+    padding-left: 1em;
+
+    &.current {
+      padding-left: 0;
+    }
+  }
+
+  li {
+    padding-top: 0.2em;
+    padding-bottom: 0.2em;
+
+  }
+}
+
+.admonition {
+  border-radius: 0.3em;
+  border: 1px solid #ccc;
+  background-color: $color_light_bg;
+
+  .admonition-title {
+    color: $color_instaloader_hyper;
+  }
+}
+
+.topic {
+  border-radius: 0.3em;
+  background-color: $color_light_bg;
+}
+
+.nav-link {
+  padding-top: 0;
+  padding-bottom: 0;
+}
+
+.btn-success {
+  background-color: $color_instaloader_main;
+  border-color: $color_instaloader_main;
+
+  &:hover {
+    background-color: $color_instaloader_accent;
+    border-color: $color_instaloader_accent;
+  }
+}

docs/_static/style.css

@@ -1,28 +0,0 @@
-code {
-    color: #222;
-    background-color: #f5f5f5;
-}
-
-dd {
-    padding-left: 8px;
-}
-
-dl.option dd {
-    padding-left: 12px;
-}
-
-a {
-    color: #008d06;
-}
-
-a:hover {
-    color: #f48400;
-}
-
-body {
-    font-size: 16px;
-}
-
-.versionmodified {
-    font-size: 14px;
-}

docs/_templates/globaltoc.html

@@ -1,14 +0,0 @@
-<li class="dropdown globaltoc-container">
-  <a role="button"
-     id="dLabelGlobalToc"
-     data-toggle="dropdown"
-     data-target="#"
-     href="{{ pathto(master_doc) }}">{{ theme_navbar_site_name }} <b class="caret"></b></a>
-  <ul class="dropdown-menu globaltoc"
-      role="menu"
-      aria-labelledby="dLabelGlobalToc">
-    {{ toctree(maxdepth=theme_globaltoc_depth|toint, collapse=False, includehidden=theme_globaltoc_includehidden|tobool) }}
-    <li role="separator" class="divider"></li>
-    <li class="toctree-l1"><a class="reference internal" href="genindex.html">Index</a></li>
-  </ul>
-</li>

docs/_templates/layout.html

@@ -1,18 +1,115 @@
 {% extends "!layout.html" %}
+
+{% set html5_doctype = True %}
+
+{% set script_files = script_files + ['_static/bootstrap-4.1.3.bundle.min.js', '_static/instaloader.js'] %}
+
 {% block htmltitle %}
   {% if pagename == "index" %}
-  <title>{{ title|striptags|e }}</title>
+    <title>{{ title|striptags|e }}</title>
   {% else %}
-  <title>{{ title|striptags|e }}{{ titlesuffix }}</title>
+    <title>{{ title|striptags|e }}{{ titlesuffix }}</title>
   {% endif %}
 {% endblock %}
+
 {% block extrahead %}
-{{ super() }}
-  <meta name="google-site-verification" content="FNUsDdkOJM9obm9QEdrNBVd9AtAzHzA-aaSMCUrS6C8" />
+  {{ super() }}
+  <meta name="viewport" content="width=device-width, initial-scale=1, shrink-to-fit=no">
+  <meta name="google-site-verification" content="FNUsDdkOJM9obm9QEdrNBVd9AtAzHzA-aaSMCUrS6C8"/>
   {% if pagename == "index" %}
-  <link rel="canonical" href="https://instaloader.github.io/" />
+    <link rel="canonical" href="https://instaloader.github.io/"/>
   {% else %}
-  <link rel="canonical" href="https://instaloader.github.io/{{ pagename }}.html" />
+    <link rel="canonical" href="https://instaloader.github.io/{{ pagename }}.html"/>
   {% endif %}
-  <script async defer src="https://buttons.github.io/buttons.js"></script>
 {% endblock %}
+
+{% block header %}
+  <nav class="navbar navbar-expand-sm navbar-light bg-light fixed-top border-bottom">
+    <a class="navbar-brand" href="{{ pathto(master_doc) }}">
+      <img src="{{ pathto('_static/' + logo, 1) }}" width="30" height="30" class="d-inline-block align-top" alt="logo">
+      Instaloader
+    </a>
+    <ul class="navbar-nav mr-auto">
+      <li class="nav-item ml-0 ml-sm-2 ml-md-0">
+        <a target="_blank" class="nav-link"
+           href="https://github.com/instaloader/instaloader/releases/tag/v{{ current_release }}"
+           title="Released on {{ current_release_date }}" data-toggle="tooltip">v<b>{{ current_release }}</b></a>
+      </li>
+    </ul>
+    <button class="navbar-toggler" type="button" data-toggle="collapse" data-target="#navbarLinks"
+            aria-controls="navbarLinks" aria-expanded="false" aria-label="Toggle navigation">
+      <span class="navbar-toggler-icon"></span>
+    </button>
+    <div class="collapse navbar-collapse" id="navbarLinks">
+      <ul class="navbar-nav ml-auto">
+        <li class="nav-item">
+          <a target="_blank" class="nav-link" href="https://github.com/instaloader/instaloader/issues">Issues</a>
+        </li>
+        <li class="nav-item">
+          <a target="_blank" class="nav-link" href="https://github.com/instaloader/instaloader/releases">Releases</a>
+        </li>
+        <li class="nav-item">
+          <a target="_blank" class="nav-link" href="https://github.com/instaloader/instaloader">GitHub</a>
+        </li>
+      </ul>
+      <form class="form-inline" action="search.html" method="get">
+        <div class="input-group my-2 my-sm-0 ml-2">
+          <input class="form-control" type="search" placeholder="Search" aria-label="Search" name="q">
+          <div class="input-group-append">
+            <button class="btn btn-success btn-small" type="submit">Go</button>
+          </div>
+        </div>
+      </form>
+    </div>
+    {% if pagename != "index" %}
+      <button type="button" class="btn btn-success d-md-none btn-small ml-2" data-toggle="modal"
+              data-target="#navbarToc">
+        TOC
+      </button>
+    {% endif %}
+  </nav>
+  {% if pagename != "index" %}
+    <div class="modal fade" id="navbarToc" tabindex="-1" role="dialog"
+         aria-hidden="true" aria-labelledby="navbarTocTitle">
+      <div class="modal-dialog modal-dialog-centered" role="document">
+        <div class="modal-content">
+          <div class="modal-header">
+            <h5 class="modal-title" id="navbarTocTitle">Contents</h5>
+            <button type="button" class="close" data-dismiss="modal" aria-label="Close">
+              <span aria-hidden="true">&times;</span>
+            </button>
+          </div>
+          <div class="modal-body">
+            {{ toctree(maxdepth=-1) }}
+          </div>
+        </div>
+      </div>
+    </div>
+  {% endif %}
+{% endblock %}
+
+{% block relbar1 %}{% endblock %}
+{% block relbar2 %}{% endblock %}
+{% block footer %}{% endblock %}
+{% block content %}
+  <div class="container-fluid row">
+    <div class="col-12 {% if pagename != "index" %}col-md-9 col-xl-10{% endif %} doc-content">
+      {% block body %}{% endblock %}
+    </div>
+    {% if pagename != "index" %}
+      <div class="d-none col-md-3 col-xl-2 bg-light doc-sidebar d-md-flex flex-column border-left">
+
+        <h3>{{ _('Table of Contents') }}</h3>
+        {{ toctree(maxdepth=-1) }}
+
+        <div class="d-flex justify-content-between small mt-auto mb-2">
+          <a class="reference internal{% if pagename == "genindex" %} current{% endif %}" href="genindex.html">Index</a>
+          {% if pagename != "genindex" and pagename != "search" %}
+            <a class="reference external"
+               href="https://github.com/instaloader/instaloader/edit/master/docs/{{ pagename }}.rst">Edit this page</a>
+          {% endif %}
+        </div>
+      </div>
+    {% endif %}
+  </div>
+{% endblock %}

docs/_templates/navbar.html

@@ -1,60 +0,0 @@
-  <div id="navbar" class="{{ theme_navbar_class }} navbar-default {% if theme_navbar_fixed_top|tobool -%} navbar-fixed-top{%- endif -%}">
-    <div class="container">
-      <div class="navbar-header">
-        <!-- .btn-navbar is used as the toggle for collapsed navbar content -->
-        <button type="button" class="navbar-toggle" data-toggle="collapse" data-target=".nav-collapse">
-          <span class="icon-bar"></span>
-          <span class="icon-bar"></span>
-          <span class="icon-bar"></span>
-        </button>
-        <a class="navbar-brand" href="{{ pathto(master_doc) }}">
-          {%- block sidebarlogo %}
-            {%- if logo %}<span><img src="{{ pathto('_static/' + logo, 1) }}"></span>{%- endif %}
-          {%- endblock %}
-          {% if theme_navbar_title -%}{{ theme_navbar_title|e }}{%- else -%}{{ project|e }}{%- endif -%}
-        </a>
-        <span class="navbar-text navbar-version pull-left">
-          <a href="https://github.com/instaloader/instaloader/releases/tag/v{{current_release}}" title="Released on {{current_release_date}}">v<b>{{current_release}}</b></a>
-        </span>
-      </div>
-
-        <div class="collapse navbar-collapse nav-collapse">
-          <ul class="nav navbar-nav">
-            {% if theme_navbar_links %}
-              {%- for link in theme_navbar_links %}
-                <li><a href="{{ pathto(*link[1:]) }}">{{ link[0] }}</a></li>
-              {%- endfor %}
-            {% endif %}
-            {% block navbartoc %}
-              {% include "globaltoc.html" %}
-              {% if theme_navbar_pagenav and pagename not in ("index", "genindex", "installation") %}
-                {% include "navbartoc.html" %}
-              {% endif %}
-            {% endblock %}
-            {% if theme_navbar_sidebarrel %}
-              {% block sidebarrel %}
-                {% include "relations.html" %}
-              {% endblock %}
-            {% endif %}
-            </ul>
-            <ul class="nav navbar-nav navbar-right">
-              <li class="dropdown">
-                <a role="button" id="dLabelUsefulLinks" data-toggle="dropdown" data-target="#" href="#">
-                  Useful Links
-                  <b class="caret"></b>
-                </a>
-                <ul class="dropdown-menu" role="menu" aria-labelledby="dLabelUsefulLinks">
-                  <li><a href="https://github.com/instaloader/instaloader/issues">Issue Tracker</a></li>
-                  <li><a href="https://github.com/instaloader/instaloader/releases">Version History</a></li>
-                  <li><a href="https://pypi.org/project/instaloader/">Instaloader on PyPI</a></li>
-                  <li><a href="https://github.com/instaloader/instaloader">Instaloader on GitHub</a></li>
-                  <li role="separator" class="divider"></li>
-                  <li style="padding-left: 20px;"><a class="github-button" href="https://github.com/instaloader/instaloader" data-icon="octicon-star" data-size="large" data-show-count="true" aria-label="Star Instaloader on GitHub">Star</a></li>
-                  <li role="separator" class="divider"></li>
-                  <li><a href="https://www.paypal.me/aandergr"><b>Donate (via Paypal)</b></a></li>
-                </ul>
-              </li>
-            </ul>
-        </div>
-    </div>
-  </div>

docs/_templates/search.html

@@ -0,0 +1,28 @@
+{% extends "!search.html" %}
+
+{% block body %}
+  <div id="fallback" class="admonition warning">
+    <script type="text/javascript">$('#fallback').hide();</script>
+    <p>
+      {% trans %}Please activate JavaScript to enable the search
+        functionality.{% endtrans %}
+    </p>
+  </div>
+  {% if search_performed %}
+    <h1>{{ _('Search Results') }}</h1>
+    {% if not search_results %}
+      <p>{{ _('Your search did not match any documents. Please make sure that all words are spelled correctly and that you\'ve selected enough categories.') }}</p>
+    {% endif %}
+  {% endif %}
+  <div id="search-results">
+    {% if search_results %}
+      <ul>
+        {% for href, caption, context in search_results %}
+          <li><a href="{{ pathto(item.href) }}">{{ caption }}</a>
+            <div class="context">{{ context|e }}</div>
+          </li>
+        {% endfor %}
+      </ul>
+    {% endif %}
+  </div>
+{% endblock %}

docs/as-module.rst

@@ -123,6 +123,9 @@ Additionally, the following trivial structures are defined:
 .. autoclass:: PostComment
    :no-show-inheritance:
 
+.. autoclass:: PostCommentAnswer
+   :no-show-inheritance:
+
 .. autoclass:: PostLocation
    :no-show-inheritance:
 
@@ -152,6 +155,14 @@ Profiles
 .. autoclass:: Profile
    :no-show-inheritance:
 
+TopSearchResults
+""""""""""""""""
+
+.. autoclass:: TopSearchResults
+   :no-show-inheritance:
+
+   .. versionadded:: 4.3
+
 Loading and Saving
 """"""""""""""""""
 
@@ -176,8 +187,6 @@ Exceptions
 
 .. autoexception:: QueryReturnedBadRequestException
 
-.. autoexception:: QueryReturnedNotFoundException
-
 .. autoexception:: QueryReturnedForbiddenException
 
 .. autoexception:: ProfileNotExistsException
@@ -188,12 +197,25 @@ Exceptions
 
 .. autoexception:: LoginRequiredException
 
+.. autoexception:: TwoFactorAuthRequiredException
+
+   .. versionadded:: 4.2
+
 .. autoexception:: InvalidArgumentException
 
 .. autoexception:: BadResponseException
 
 .. autoexception:: BadCredentialsException
 
+.. autoexception:: PostChangedException
+
+.. autoexception:: QueryReturnedNotFoundException
+
+   .. versionchanged:: 4.3
+      QueryReturnedNotFoundException now inherits ConnectionException
+      to retry on 404 errors.
+
+
 .. autoexception:: TooManyRequestsException
 
 ``InstaloaderContext`` (Low-level functions)

docs/basic-usage.rst

@@ -77,14 +77,25 @@ Instaloader supports the following targets:
       instructs Instaloader to also **download the user's stories**,
 
    - :option:`--highlights`
-      to **download highlights of each profile that is downloaded**, and
+      to **download highlights of each profile that is downloaded**,
 
    - :option:`--tagged`
-      to **download posts where the user is tagged**.
+      to **download posts where the user is tagged**, and
+
+   - :option:`--igtv`
+      to **download IGTV videos**.
 
 - ``"#hashtag"``
    Posts with a certain **hashtag** (the quotes are usually necessary),
 
+- ``%location id``
+   Posts tagged with a given location; the location ID is the numerical ID
+   Instagram labels a location with (e.g.
+   \https://www.instagram.com/explore/locations/**362629379**/plymouth-naval-memorial/).
+   Requires :option:`--login`.
+
+   .. versionadded:: 4.2
+
 - ``:stories``
    The currently-visible **stories** of your followees (requires
    :option:`--login`),
@@ -115,7 +126,7 @@ downloads the pictures and videos and their captions. You can specify
 
 - :option:`--geotags`
    **download geotags** of each post and save them as
-   Google Maps link,
+   Google Maps link (requires :option:`--login`),
 
 For a reference of all supported command line options, see
 :ref:`command-line-options`.
@@ -126,7 +137,7 @@ Filename Specification
 ^^^^^^^^^^^^^^^^^^^^^^
 
 For each target, Instaloader creates a directory named after the target,
-i.e. ``profile``, ``#hashtag``, ``:feed``, etc. and therein saves the
+i.e. ``profile``, ``#hashtag``, ``%location id``, ``:feed``, etc. and therein saves the
 posts in files named after the post's timestamp.
 
 :option:`--dirname-pattern` allows to configure the directory name of each

docs/cli-options.rst

@@ -8,7 +8,7 @@ Instaloader is invoked with::
    $ instaloader [options] target [target ...]
 
 where ``target`` is a ``profile``, a ``"#hashtag"``, ``@profile`` (all profiles
-that *profile* is following), or if logged in ``:feed`` (pictures from your
+that *profile* is following), ``%location ID``, or if logged in ``:feed`` (pictures from your
 feed), ``:stories`` (stories of your followees) or ``:saved`` (collection of
 posts marked as saved).
 
@@ -61,8 +61,7 @@ What to Download of each Post
 
    **Download geotags** when available. Geotags are stored as a text file with
    the location's name and a Google Maps link. This requires an additional
-   request to the Instagram server for each picture, which is why it is disabled
-   by default.
+   request to the Instagram server for each picture. Requires :option:`--login`.
 
 .. option:: --comments, -C
 
@@ -130,6 +129,12 @@ What to Download of each Profile
 
    .. versionadded:: 4.1
 
+.. option:: --igtv
+
+   Also download IGTV videos.
+
+   .. versionadded:: 4.3
+
 .. option:: --stories-only
 
    .. deprecated:: 4.1
@@ -171,7 +176,7 @@ Which Posts to Download
 .. option:: --count COUNT, -c
 
    Do not attempt to download more than COUNT posts.  Applies only to
-   ``#hashtag`` and ``:feed``.
+   ``#hashtag``, ``%location id``, and ``:feed``.
 
 
 Login (Download Private Profiles)
@@ -230,6 +235,20 @@ How to Download
    to ``3``. If a connection fails, it can be manually skipped by hitting
    :kbd:`Control-c`. Set this to ``0`` to retry infinitely.
 
+.. option:: --commit-mode
+
+   Tries to ensure downloaded images avoid corruption in case of unexpected
+   interruption. If the last picture is corrupted, Instaloader will fix the
+   picture the next time it is run.
+
+   .. versionadded:: 4.2
+
+.. option:: --request-timeout N
+
+   Seconds to wait before timing out a connection request.
+
+   .. versionadded:: 4.3
+
 Miscellaneous Options
 ^^^^^^^^^^^^^^^^^^^^^
 
@@ -241,18 +260,20 @@ Miscellaneous Options
 
 .. option:: +args.txt
 
-   Read arguments from file `args.txt`, a shortcut to provide argument from
-   file rather than command-line. This provide a convient way to hide login
-   info from CLI. and also can use for simplify managment of long arguments.
+   Read arguments from file `args.txt`, a shortcut to provide arguments from
+   file rather than command-line. This provides a convenient way to hide login
+   info from CLI, and can also be used to simplify managment of long arguments.
 
    .. note::
 
-      text file should separate arg with line break.
+      Text file should separate arguments with line breaks.
 
       args.txt example::
 
-         --login MYUSENAME
-         --password MYPASSWORD
+         --login=MYUSERNAME
+         --password=MYPASSWORD
          --fast-update
+         profile1
+         profile2
 
    .. versionadded:: 4.1

docs/codesnippets.rst

@@ -82,6 +82,15 @@ See also :class:`Post`, :meth:`Instaloader.download_post`,
 
 Discussed in :issue:`113`.
 
+Top X Posts of User
+-------------------
+
+With Instaloader, it is easy to download the few most-liked pictres of a user.
+
+.. literalinclude:: codesnippets/194_top_x_of_user.py
+
+Discussed in :issue:`194`.
+
 Upgrade Images by Local Copies
 ------------------------------
 

docs/codesnippets/113_only_one_per_user.py

@@ -2,13 +2,13 @@ import instaloader
 
 L = instaloader.Instaloader()
 
-posts = L.get_hashtag_posts('milfgarden')
+posts = L.get_hashtag_posts('urbanphotography')
 
 users = set()
 
 for post in posts:
     if not post.owner_profile in users:
-        L.download_post(post, '#milfgarden')
+        L.download_post(post, '#urbanphotography')
         users.add(post.owner_profile)
     else:
         print("{} from {} skipped.".format(post, post.owner_profile))

docs/codesnippets/121_since_until.py

@@ -5,7 +5,7 @@ import instaloader
 
 L = instaloader.Instaloader()
 
-posts = L.get_hashtag_posts('milfgarden')
+posts = L.get_hashtag_posts('urbanphotography')
 # or
 # posts = instaloader.Profile.from_username(L.context, PROFILE).get_posts()
 
@@ -14,4 +14,4 @@ UNTIL = datetime(2015, 3, 1)
 
 for post in takewhile(lambda p: p.date > UNTIL, dropwhile(lambda p: p.date > SINCE, posts)):
     print(post.date)
-    L.download_post(post, '#milfgarden')
+    L.download_post(post, '#urbanphotography')

docs/codesnippets/194_top_x_of_user.py

@@ -0,0 +1,17 @@
+from itertools import islice
+from math import ceil
+
+from instaloader import Instaloader, Profile
+
+PROFILE = ...        # profile to download from
+X_percentage = 10    # percentage of posts that should be downloaded
+
+L = Instaloader()
+
+profile = Profile.from_username(L.context, PROFILE)
+posts_sorted_by_likes = sorted(profile.get_posts(),
+                               key=lambda p: p.likes + p.comments,
+                               reverse=True)
+
+for post in islice(posts_sorted_by_likes, ceil(profile.mediacount * X_percentage / 100)):
+    L.download_post(post, PROFILE)

docs/codesnippets/92_import_firefox_session.py

@@ -0,0 +1,23 @@
+from glob import glob
+from os.path import expanduser
+from sqlite3 import connect
+
+from instaloader import ConnectionException, Instaloader
+
+# FIREFOXCOOKIEFILE = "/home/alex/.mozilla/firefox/l96w6b90.default/cookies.sqlite"
+FIREFOXCOOKIEFILE = glob(expanduser("~/.mozilla/firefox/*.default/cookies.sqlite"))[0]
+
+instaloader = Instaloader(max_connection_attempts=1)
+instaloader.context._session.cookies.update(connect(FIREFOXCOOKIEFILE)
+                                            .execute("SELECT name, value FROM moz_cookies "
+                                                     "WHERE baseDomain='instagram.com'"))
+
+try:
+    username = instaloader.test_login()
+    if not username:
+        raise ConnectionException()
+except ConnectionException:
+    raise SystemExit("Cookie import failed. Are you logged in successfully in Firefox?")
+
+instaloader.context.username = username
+instaloader.save_session_to_file()

docs/conf.py

@@ -20,8 +20,7 @@
 import os
 import subprocess
 import sys
-
-import sphinx_bootstrap_theme
+import typing
 
 sys.path.insert(0, os.path.abspath('..'))
 
@@ -48,7 +47,7 @@ autodoc_default_flags = ['show-inheritance', 'members', 'undoc-members']
 autodoc_member_order = 'bysource'
 
 intersphinx_mapping = {'python': ('https://docs.python.org/3', None),
-                       'requests': ('http://docs.python-requests.org/en/master', None)}
+                       'requests': ('https://requests.kennethreitz.org/en/master/', None)}
 
 extlinks = {'issue': ('https://github.com/instaloader/instaloader/issues/%s', 'issue #')}
 
@@ -87,7 +86,7 @@ author = 'Alexander Graf and Andre Koch-Kramer'
 #
 # This is also used if you do content translation via gettext catalogs.
 # Usually you set "language" from the command line for these cases.
-language = None
+language = "en"
 
 # There are two options for replacing |today|: either, you set today to some
 # non-false value, then it is used:
@@ -123,7 +122,7 @@ add_module_names = False
 # show_authors = False
 
 # The name of the Pygments (syntax highlighting) style to use.
-pygments_style = 'sphinx'
+pygments_style = 'fruity'
 
 # A list of ignored prefixes for module index sorting.
 # modindex_common_prefix = []
@@ -140,20 +139,7 @@ todo_include_todos = False
 # The theme to use for HTML and HTML Help pages.  See the documentation for
 # a list of builtin themes.
 #
-html_theme = 'bootstrap'
-html_theme_path = sphinx_bootstrap_theme.get_html_theme_path()
-
-# Theme options are theme-specific and customize the look and feel of a theme
-# further.  For a list of options available for each theme, see the
-# documentation.
-#
-html_theme_options = {
-    'navbar_site_name': 'Site Contents',
-    'navbar_pagenav_name': 'Page Contents',
-    'navbar_pagenav': True,
-    'navbar_sidebarrel': True,
-    'nosidebar': True,
-     }
+html_theme = 'basic'
 
 # Add any paths that contain custom themes here, relative to this directory.
 # html_theme_path = []
@@ -374,6 +360,7 @@ html_context = {'current_release': current_release, 'current_release_date': curr
 
 
 def setup(app):
-    app.add_stylesheet("style.css")
+    typing.TYPE_CHECKING = True
+    app.add_stylesheet("instaloader.css")
     app.connect('autodoc-process-signature', sphinx_autodoc_typehints.process_signature)
     app.connect('autodoc-process-docstring', sphinx_autodoc_typehints.process_docstring)

docs/contributing.rst

@@ -15,14 +15,14 @@ Reporting Bugs
 --------------
 
 If you encounter a bug, do not hesitate to report it in our
-`Issue Tracker <https://github.com/instaloader/instaloader/issues>`__. Since
-Instaloader is actively developed, the majority of bugs is fixed within only
-**3 days** after being reported. When reporting a problem, please keep the
-following in mind:
+`Issue Tracker <https://github.com/instaloader/instaloader/issues>`__. When
+reporting a problem, please keep the following in mind:
 
 - Ensure you **use the latest version** of Instaloader. The currently-installed
   version can be found out with ``instaloader --version``.
 
+- Check whether there is a valid solution in our :ref:`troubleshooting` section.
+
 - Briefly **check whether the bug has already been reported**. If you find an
   issue reporting the same bug you encountered, comment there rather than
   opening a new issue. However, if unsure, please create a new issue.
@@ -64,6 +64,16 @@ few things to consider:
 Suggesting Features
 -------------------
 
+.. goal-start
+
+Instaloader's goal is to mimick the browser's behavior to access the data that
+is available through the Instagram web interface, pack this data into complete
+and easily-(re)usable python objects, and provide a user interface for the most
+common downloading and metadata collection tasks, without changing any of the
+online data.
+
+.. goal-end
+
 If you have an idea of how Instaloader should be enhanced, but do not want to
 implement the feature yourself, feel free to open a ticket in our 
 `Issue Tracker <https://github.com/instaloader/instaloader/issues>`__.
@@ -80,6 +90,9 @@ Please consider the following:
   invoke the new function? Which problem would it solve? If new information is
   obtained, how would it be further processed?
 
+- If not obvious, briefly motivate how your suggested feature **conforms with
+  Instaloader's project goal**.
+
 - **Be patient**. Naturally, bugs and pull requests have a higher priority than
   feature suggestions. Keep in mind that this is a free software project, and
   unfortunately we only have limited time to work on it.

docs/index.rst

@@ -15,6 +15,10 @@ Instaloader
 **Instaloader** is a tool to download pictures (or videos) along with
 their captions and other metadata from Instagram.
 
+.. include:: ../README.rst
+   :start-after: badges-start
+   :end-before: badges-end
+
 With `Python <https://www.python.org/>`__ installed, do::
 
     $ pip3 install instaloader
@@ -41,9 +45,10 @@ See :ref:`install` for more options on how to install Instaloader.
 
 ::
 
-    instaloader [--comments] [--geotags] [--stories] [--highlights] [--tagged]
+    instaloader [--comments] [--geotags]
+                [--stories] [--highlights] [--tagged] [--igtv]
                 [--login YOUR-USERNAME] [--fast-update]
-                profile | "#hashtag" |
+                profile | "#hashtag" | %location_id |
                 :stories | :feed | :saved
 
 See :ref:`download-pictures-from-instagram` for a detailed introduction on how
@@ -60,6 +65,7 @@ Instaloader Documentation
    cli-options
    as-module
    codesnippets
+   troubleshooting
    contributing
 
 Useful Links

docs/requirements.txt

@@ -1,3 +1,2 @@
 requests
 sphinx!=1.7.7
-sphinx-bootstrap-theme

docs/sphinx_autodoc_typehints.py

@@ -1,4 +1,4 @@
-# sphinx_autodoc_typehints from Mar 18 2018.
+# sphinx_autodoc_typehints from Nov 20 2018 (version 1.5.2).
 #  https://github.com/agronholm/sphinx-autodoc-typehints
 # (slightly modified to fix class links, maybe related to agronholm/sphinx-autodoc-typehints#38)
 #
@@ -27,8 +27,8 @@
 import inspect
 from typing import get_type_hints, TypeVar, Any, AnyStr, Generic, Union
 
-from sphinx.util.inspect import getargspec
-from sphinx.ext.autodoc import formatargspec
+from sphinx.util import logging
+from sphinx.util.inspect import Signature
 
 try:
     from inspect import unwrap
@@ -51,6 +51,8 @@ except ImportError:
             memo.add(id_func)
         return func
 
+logger = logging.getLogger(__name__)
+
 
 def format_annotation(annotation):
     if inspect.isclass(annotation) and annotation.__module__ == 'builtins':
@@ -162,23 +164,37 @@ def process_signature(app, what: str, name: str, obj, options, signature, return
         return
 
     obj = unwrap(obj)
-    try:
-        argspec = getargspec(obj)
-    except (TypeError, ValueError):
-        return
+    signature = Signature(obj)
+    parameters = [
+        param.replace(annotation=inspect.Parameter.empty)
+        for param in signature.signature.parameters.values()
+    ]
 
-    if argspec.args:
+    if parameters:
         if what in ('class', 'exception'):
-            del argspec.args[0]
+            del parameters[0]
         elif what == 'method':
             outer = inspect.getmodule(obj)
             for clsname in obj.__qualname__.split('.')[:-1]:
                 outer = getattr(outer, clsname)
-            method_object = outer.__dict__[obj.__name__]
-            if not isinstance(method_object, (classmethod, staticmethod)):
-                del argspec.args[0]
 
-    return formatargspec(obj, *argspec[:-1]), None
+            method_name = obj.__name__
+            if method_name.startswith("__") and not method_name.endswith("__"):
+                # If the method starts with double underscore (dunder)
+                # Python applies mangling so we need to prepend the class name.
+                # This doesn't happen if it always ends with double underscore.
+                class_name = obj.__qualname__.split('.')[-2]
+                method_name = "_{c}{m}".format(c=class_name, m=method_name)
+
+            method_object = outer.__dict__[method_name]
+            if not isinstance(method_object, (classmethod, staticmethod)):
+                del parameters[0]
+
+    signature.signature = signature.signature.replace(
+        parameters=parameters,
+        return_annotation=inspect.Signature.empty)
+
+    return signature.format_args().replace('\\', '\\\\'), None
 
 
 def process_docstring(app, what, name, obj, options, lines):
@@ -195,8 +211,15 @@ def process_docstring(app, what, name, obj, options, lines):
         except (AttributeError, TypeError):
             # Introspecting a slot wrapper will raise TypeError
             return
+        except NameError as exc:
+            logger.warning('Cannot resolve forward reference in type annotations of "%s": %s',
+                           name, exc)
+            type_hints = obj.__annotations__
 
         for argname, annotation in type_hints.items():
+            if argname.endswith('_'):
+                argname = '{}\\_'.format(argname[:-1])
+
             formatted_annotation = format_annotation(annotation)
 
             if argname == 'return':

docs/troubleshooting.rst

@@ -0,0 +1,60 @@
+.. _troubleshooting:
+
+Troubleshooting
+===============
+
+.. highlight:: python
+
+429 - Too Many Requests
+-----------------------
+
+Instaloader has a logic to keep track of its requests to Instagram and to obey
+their rate limits. Since they are nowhere documented, we try them out
+experimentally. We have a daily cron job running to confirm that Instaloader
+still stays within the rate limits. Nevertheless, the rate control logic assumes
+that
+
+- at one time, Instaloader is the only application that consumes requests. I.e.
+  neither the Instagram browser interface, nor a mobile app, nor another
+  Instaloader instance is running in parallel,
+
+- no requests had been consumed when Instaloader starts.
+
+The latter one implies that restarting or reinstantiating Instaloader often
+within short time is prone to cause a 429. When a request is denied with a 429,
+Instaloader retries the request as soon as the temporary ban is assumed to be
+expired. In case the retry continuously fails for some reason, which should not
+happen in normal conditions, consider adjusting the
+:option:`--max-connection-attempts` option.
+
+**"Too many queries in the last time"** is not an error. It is a notice that the
+rate limit has almost been reached, according to Instaloader's own rate
+accounting mechanism. We regularly adjust this mechanism to match Instagram's
+current rate limiting.
+
+Login error
+-----------
+
+Instaloader's login *should* work fine, both with and without
+Two-Factor-Authentication. It also supports handling the *checkpoint challenge*,
+issued when Instagram suspects authentication activity on your account, by
+pointing the user to an URL to be opened in a browser.
+
+Nevertheless, in :issue:`92` users report problems with logging in. To still use
+Instaloader's logged-in functionality, you may use the following script to
+workaround login problems by importing the session cookies from Firefox and
+bypassing Instaloader's login.
+
+.. literalinclude:: codesnippets/92_import_firefox_session.py
+
+To use this,
+
+#. login to Instagram in Firefox, 
+
+#. execute the snippet,
+
+#. then, ``instaloader -l USERNAME`` should work fine.
+
+If you do not use your default firefox profile, or your operating system has the
+paths differently set up, you may have to alter the ``FIREFOXCOOKIEFILE``
+variable first.

instaloader/__init__.py

@@ -1,12 +1,12 @@
 """Download pictures (or videos) along with their captions and other metadata from Instagram."""
 
 
-__version__ = '4.1'
+__version__ = '4.3.2'
 
 
 try:
     # pylint:disable=wrong-import-position
-    import win_unicode_console
+    import win_unicode_console  # type: ignore
 except ImportError:
     pass
 else:
@@ -15,5 +15,5 @@ else:
 from .exceptions import *
 from .instaloader import Instaloader
 from .instaloadercontext import InstaloaderContext
-from .structures import (Highlight, Post, PostSidecarNode, PostComment, PostLocation, Profile, Story, StoryItem,
-                         load_structure_from_file, save_structure_to_file)
+from .structures import (Highlight, Post, PostSidecarNode, PostComment, PostCommentAnswer, PostLocation, Profile, Story,
+                         StoryItem, TopSearchResults, load_structure_from_file, save_structure_to_file)

instaloader/__main__.py

@@ -8,7 +8,8 @@ from argparse import ArgumentParser, SUPPRESS
 from typing import List, Optional
 
 from . import (Instaloader, InstaloaderException, InvalidArgumentException, Post, Profile, ProfileNotExistsException,
-               StoryItem, __version__, load_structure_from_file)
+               StoryItem, __version__, load_structure_from_file, TwoFactorAuthRequiredException,
+               BadCredentialsException)
 from .instaloader import get_default_session_filename
 from .instaloadercontext import default_user_agent
 
@@ -18,9 +19,10 @@ def usage_string():
     argv0 = os.path.basename(sys.argv[0])
     argv0 = "instaloader" if argv0 == "__main__.py" else argv0
     return """
-{0} [--comments] [--geotags] [--stories] [--highlights] [--tagged]
+{0} [--comments] [--geotags]
+{2:{1}} [--stories] [--highlights] [--tagged] [--igtv]
 {2:{1}} [--login YOUR-USERNAME] [--fast-update]
-{2:{1}} profile | "#hashtag" | :stories | :feed | :saved
+{2:{1}} profile | "#hashtag" | %%location_id | :stories | :feed | :saved
 {0} --help""".format(argv0, len(argv0), '')
 
 
@@ -60,7 +62,10 @@ def _main(instaloader: Instaloader, targetlist: List[str],
           username: Optional[str] = None, password: Optional[str] = None,
           sessionfile: Optional[str] = None,
           download_profile_pic: bool = True, download_posts=True,
-          download_stories: bool = False, download_highlights: bool = False, download_tagged: bool = False,
+          download_stories: bool = False,
+          download_highlights: bool = False,
+          download_tagged: bool = False,
+          download_igtv: bool = False,
           fast_update: bool = False,
           max_count: Optional[int] = None, post_filter_str: Optional[str] = None,
           storyitem_filter_str: Optional[str] = None) -> None:
@@ -84,10 +89,22 @@ def _main(instaloader: Instaloader, targetlist: List[str],
             instaloader.context.log("Session file does not exist yet - Logging in.")
         if not instaloader.context.is_logged_in or username != instaloader.test_login():
             if password is not None:
-                instaloader.login(username, password)
+                try:
+                    instaloader.login(username, password)
+                except TwoFactorAuthRequiredException:
+                    while True:
+                        try:
+                            code = input("Enter 2FA verification code: ")
+                            instaloader.two_factor_login(code)
+                            break
+                        except BadCredentialsException:
+                            pass
             else:
                 instaloader.interactive_login(username)
         instaloader.context.log("Logged in as %s." % username)
+    # since 4.2.9 login is required for geotags
+    if instaloader.download_geotags and not instaloader.context.is_logged_in:
+        instaloader.context.error("Warning: Use --login to download geotags of posts.")
     # Try block for KeyboardInterrupt (save session on ^C)
     profiles = set()
     anonymous_retry_profiles = set()
@@ -130,6 +147,9 @@ def _main(instaloader: Instaloader, targetlist: List[str],
                                                  post_filter=post_filter)
                 elif target[0] == '-':
                     instaloader.download_post(Post.from_shortcode(instaloader.context, target[1:]), target)
+                elif target[0] == "%":
+                    instaloader.download_location(location=target[1:], max_count=max_count, fast_update=fast_update,
+                                                  post_filter=post_filter)
                 elif target == ":feed":
                     instaloader.download_feed_posts(fast_update=fast_update, max_count=max_count,
                                                     post_filter=post_filter)
@@ -142,7 +162,8 @@ def _main(instaloader: Instaloader, targetlist: List[str],
                     try:
                         profile = instaloader.check_profile_id(target)
                         if instaloader.context.is_logged_in and profile.has_blocked_viewer:
-                            if download_profile_pic or ((download_posts or download_tagged) and not profile.is_private):
+                            if download_profile_pic or ((download_posts or download_tagged or download_igtv)
+                                                        and not profile.is_private):
                                 raise ProfileNotExistsException("{} blocked you; But we download her anonymously."
                                                                 .format(target))
                             else:
@@ -153,7 +174,7 @@ def _main(instaloader: Instaloader, targetlist: List[str],
                         # Not only our profile.has_blocked_viewer condition raises ProfileNotExistsException,
                         # check_profile_id() also does, since access to blocked profile may be responded with 404.
                         if instaloader.context.is_logged_in and (download_profile_pic or download_posts or
-                                                                 download_tagged):
+                                                                 download_tagged or download_igtv):
                             instaloader.context.log(err)
                             instaloader.context.log("Trying again anonymously, helps in case you are just blocked.")
                             with instaloader.anonymous_copy() as anonymous_loader:
@@ -166,15 +187,18 @@ def _main(instaloader: Instaloader, targetlist: List[str],
         if len(profiles) > 1:
             instaloader.context.log("Downloading {} profiles: {}".format(len(profiles),
                                                                          ' '.join([p.username for p in profiles])))
+        if profiles and download_profile_pic and not instaloader.context.is_logged_in:
+            instaloader.context.error("Warning: Use --login to download HD version of profile pictures.")
         instaloader.download_profiles(profiles,
-                                      download_profile_pic, download_posts, download_tagged, download_highlights,
-                                      download_stories, fast_update, post_filter, storyitem_filter)
+                                      download_profile_pic, download_posts, download_tagged, download_igtv,
+                                      download_highlights, download_stories,
+                                      fast_update, post_filter, storyitem_filter)
         if anonymous_retry_profiles:
             instaloader.context.log("Downloading anonymously: {}"
                                     .format(' '.join([p.username for p in anonymous_retry_profiles])))
             with instaloader.anonymous_copy() as anonymous_loader:
                 anonymous_loader.download_profiles(anonymous_retry_profiles,
-                                                   download_profile_pic, download_posts, download_tagged,
+                                                   download_profile_pic, download_posts, download_tagged, download_igtv,
                                                    fast_update=fast_update, post_filter=post_filter)
     except KeyboardInterrupt:
         print("\nInterrupted by user.", file=sys.stderr)
@@ -208,6 +232,8 @@ def main():
                            help="Download all followees of profile. Requires --login. "
                                 "Consider using :feed rather than @yourself.")
     g_targets.add_argument('_hashtag', nargs='*', metavar='"#hashtag"', help="Download #hashtag.")
+    g_targets.add_argument('_location', nargs='*', metavar='%location_id',
+                           help="Download %%location_id. Requires --login.")
     g_targets.add_argument('_feed', nargs='*', metavar=":feed",
                            help="Download pictures from your feed. Requires --login.")
     g_targets.add_argument('_stories', nargs='*', metavar=":stories",
@@ -242,7 +268,7 @@ def main():
                         help='Download geotags when available. Geotags are stored as a '
                              'text file with the location\'s name and a Google Maps link. '
                              'This requires an additional request to the Instagram '
-                             'server for each picture, which is why it is disabled by default.')
+                             'server for each picture. Requires --login.')
     g_post.add_argument('-C', '--comments', action='store_true',
                         help='Download and update comments for each post. '
                              'This requires an additional request to the Instagram '
@@ -267,6 +293,8 @@ def main():
                         help='Also download highlights of each profile that is downloaded. Requires --login.')
     g_prof.add_argument('--tagged', action='store_true',
                         help='Also download posts where each profile is tagged.')
+    g_prof.add_argument('--igtv', action='store_true',
+                        help='Also download IGTV videos.')
 
     g_cond = parser.add_argument_group("Which Posts to Download")
 
@@ -315,11 +343,16 @@ def main():
     g_how.add_argument('--user-agent',
                        help='User Agent to use for HTTP requests. Defaults to \'{}\'.'.format(default_user_agent()))
     g_how.add_argument('-S', '--no-sleep', action='store_true', help=SUPPRESS)
-    g_how.add_argument('--graphql-rate-limit', type=int, help=SUPPRESS)
     g_how.add_argument('--max-connection-attempts', metavar='N', type=int, default=3,
                        help='Maximum number of connection attempts until a request is aborted. Defaults to 3. If a '
                             'connection fails, it can be manually skipped by hitting CTRL+C. Set this to 0 to retry '
                             'infinitely.')
+    g_how.add_argument('--commit-mode', action='store_true',
+                       help='Tries to ensure downloaded images avoid corruption in case of unexpected interruption. '
+                       'If the last picture is corrupted, Instaloader will fix the picture the next time it is run. '
+                       'Requires the JSON metadata to be saved.')
+    g_how.add_argument('--request-timeout', metavar='N', type=float,
+                       help='seconds to wait before timing out a connection request')
 
     g_misc = parser.add_argument_group('Miscellaneous Options')
     g_misc.add_argument('-q', '--quiet', action='store_true',
@@ -361,6 +394,9 @@ def main():
         download_posts = not (args.no_posts or args.stories_only or args.profile_pic_only)
         download_stories = args.stories or args.stories_only
 
+        if args.commit_mode and args.no_metadata_json:
+            raise SystemExit('--commit-mode requires JSON metadata to be saved.')
+
         loader = Instaloader(sleep=not args.no_sleep, quiet=args.quiet, user_agent=args.user_agent,
                              dirname_pattern=args.dirname_pattern, filename_pattern=args.filename_pattern,
                              download_pictures=not args.no_pictures,
@@ -370,8 +406,9 @@ def main():
                              compress_json=not args.no_compress_json,
                              post_metadata_txt_pattern=post_metadata_txt_pattern,
                              storyitem_metadata_txt_pattern=storyitem_metadata_txt_pattern,
-                             graphql_rate_limit=args.graphql_rate_limit,
-                             max_connection_attempts=args.max_connection_attempts)
+                             max_connection_attempts=args.max_connection_attempts,
+                             request_timeout=args.request_timeout,
+                             commit_mode=args.commit_mode)
         _main(loader,
               args.profile,
               username=args.login.lower() if args.login is not None else None,
@@ -382,6 +419,7 @@ def main():
               download_stories=download_stories,
               download_highlights=args.highlights,
               download_tagged=args.tagged,
+              download_igtv=args.igtv,
               fast_update=args.fast_update,
               max_count=int(args.count) if args.count is not None else None,
               post_filter_str=args.post_filter,

instaloader/exceptions.py

@@ -9,10 +9,6 @@ class QueryReturnedBadRequestException(InstaloaderException):
     pass
 
 
-class QueryReturnedNotFoundException(InstaloaderException):
-    pass
-
-
 class QueryReturnedForbiddenException(InstaloaderException):
     pass
 
@@ -22,6 +18,10 @@ class ProfileNotExistsException(InstaloaderException):
 
 
 class ProfileHasNoPicsException(InstaloaderException):
+    """
+    .. deprecated:: 4.2.2
+       Not raised anymore.
+    """
     pass
 
 
@@ -33,6 +33,10 @@ class LoginRequiredException(InstaloaderException):
     pass
 
 
+class TwoFactorAuthRequiredException(InstaloaderException):
+    pass
+
+
 class InvalidArgumentException(InstaloaderException):
     pass
 
@@ -49,5 +53,14 @@ class ConnectionException(InstaloaderException):
     pass
 
 
+class PostChangedException(InstaloaderException):
+    """.. versionadded:: 4.2.2"""
+    pass
+
+
+class QueryReturnedNotFoundException(ConnectionException):
+    pass
+
+
 class TooManyRequestsException(ConnectionException):
     pass

instaloader/instaloader.py

@@ -1,5 +1,6 @@
 import getpass
 import json
+import lzma
 import os
 import platform
 import re
@@ -10,12 +11,18 @@ import tempfile
 from contextlib import contextmanager, suppress
 from datetime import datetime, timezone
 from functools import wraps
+from hashlib import md5
 from io import BytesIO
+from pathlib import Path
 from typing import Any, Callable, Iterator, List, Optional, Set, Union
 
+import requests
+import urllib3  # type: ignore
+
 from .exceptions import *
 from .instaloadercontext import InstaloaderContext
-from .structures import Highlight, JsonExportable, Post, PostLocation, Profile, Story, StoryItem, save_structure_to_file
+from .structures import (Highlight, JsonExportable, Post, PostLocation, Profile, Story, StoryItem,
+                         save_structure_to_file, load_structure_from_file)
 
 
 def get_default_session_filename(username: str) -> str:
@@ -40,8 +47,37 @@ def _requires_login(func: Callable) -> Callable:
         if not instaloader.context.is_logged_in:
             raise LoginRequiredException("--login=USERNAME required.")
         return func(instaloader, *args, **kwargs)
-    # pylint:disable=no-member
-    call.__doc__ += ":raises LoginRequiredException: If called without being logged in.\n"
+    return call
+
+
+def _retry_on_connection_error(func: Callable) -> Callable:
+    """Decorator to retry the function max_connection_attemps number of times.
+
+    Herewith-decorated functions need an ``_attempt`` keyword argument.
+
+    This is to decorate functions that do network requests that may fail. Note that
+    :meth:`.get_json`, :meth:`.get_iphone_json`, :meth:`.graphql_query` and :meth:`.graphql_node_list` already have
+    their own logic for retrying, hence functions that only use these for network access must not be decorated with this
+    decorator."""
+    @wraps(func)
+    def call(instaloader, *args, **kwargs):
+        try:
+            return func(instaloader, *args, **kwargs)
+        except (urllib3.exceptions.HTTPError, requests.exceptions.RequestException, ConnectionException) as err:
+            error_string = "{}({}): {}".format(func.__name__, ', '.join([repr(arg) for arg in args]), err)
+            if (kwargs.get('_attempt') or 1) == instaloader.context.max_connection_attempts:
+                raise ConnectionException(error_string) from None
+            instaloader.context.error(error_string + " [retrying; skip with ^C]", repeat_at_end=False)
+            try:
+                if kwargs.get('_attempt'):
+                    kwargs['_attempt'] += 1
+                else:
+                    kwargs['_attempt'] = 2
+                instaloader.context.do_sleep()
+                return call(instaloader, *args, **kwargs)
+            except KeyboardInterrupt:
+                instaloader.context.error("[skipped by user]", repeat_at_end=False)
+                raise ConnectionException(error_string) from None
     return call
 
 
@@ -72,7 +108,17 @@ class _PostPathFormatter(_ArbitraryItemFormatter):
         ret = super().vformat(format_string, args, kwargs)
         if platform.system() == 'Windows':
             ret = ret.replace(':', '\ua789').replace('<', '\ufe64').replace('>', '\ufe65').replace('\"', '\uff02')
-            ret = ret.replace('\\', '\uff3c').replace('|', '\uff5c').replace('?', '\ufe16').replace('*', '\uff0a')
+            ret = ret.replace('\\', '\ufe68').replace('|', '\uff5c').replace('?', '\ufe16').replace('*', '\uff0a')
+        return ret
+
+    def get_value(self, key, args, kwargs):
+        """Replaces '/' with similar looking Division Slash and on windows newline with space"""
+        ret = super().get_value(key, args, kwargs)
+        if not isinstance(ret, str):
+            return ret
+        ret = ret.replace('/', '\u2215')
+        if platform.system() == 'Windows':
+            ret = ret.replace('\n', ' ')
         return ret
 
 
@@ -95,6 +141,8 @@ class Instaloader:
        txt file.
     :param storyitem_metadata_txt_pattern: :option:`--storyitem-metadata-txt`, default is empty (=none)
     :param max_connection_attempts: :option:`--max-connection-attempts`
+    :param commit_mode: :option:`--commit-mode`
+    :param request_timeout: :option:`--request-timeout`, set per-request timeout (seconds)
 
     .. attribute:: context
 
@@ -102,7 +150,8 @@ class Instaloader:
     """
 
     def __init__(self,
-                 sleep: bool = True, quiet: bool = False,
+                 sleep: bool = True,
+                 quiet: bool = False,
                  user_agent: Optional[str] = None,
                  dirname_pattern: Optional[str] = None,
                  filename_pattern: Optional[str] = None,
@@ -115,10 +164,11 @@ class Instaloader:
                  compress_json: bool = True,
                  post_metadata_txt_pattern: str = None,
                  storyitem_metadata_txt_pattern: str = None,
-                 graphql_rate_limit: Optional[int] = None,
-                 max_connection_attempts: int = 3):
+                 max_connection_attempts: int = 3,
+                 request_timeout: Optional[float] = None,
+                 commit_mode: bool = False):
 
-        self.context = InstaloaderContext(sleep, quiet, user_agent, graphql_rate_limit, max_connection_attempts)
+        self.context = InstaloaderContext(sleep, quiet, user_agent, max_connection_attempts, request_timeout)
 
         # configuration parameters
         self.dirname_pattern = dirname_pattern or "{target}"
@@ -134,25 +184,37 @@ class Instaloader:
             else post_metadata_txt_pattern
         self.storyitem_metadata_txt_pattern = '' if storyitem_metadata_txt_pattern is None \
             else storyitem_metadata_txt_pattern
+        self.commit_mode = commit_mode
+        if self.commit_mode and not self.save_metadata:
+            raise InvalidArgumentException("Commit mode requires JSON metadata to be saved.")
+
+        # Used to keep state in commit mode
+        self._committed = None  # type: Optional[bool]
 
     @contextmanager
     def anonymous_copy(self):
         """Yield an anonymous, otherwise equally-configured copy of an Instaloader instance; Then copy its error log."""
-        new_loader = Instaloader(self.context.sleep, self.context.quiet, self.context.user_agent, self.dirname_pattern,
-                                 self.filename_pattern, download_pictures=self.download_pictures,
-                                 download_videos=self.download_videos,
-                                 download_video_thumbnails=self.download_video_thumbnails,
-                                 download_geotags=self.download_geotags, download_comments=self.download_comments,
-                                 save_metadata=self.save_metadata, compress_json=self.compress_json,
-                                 post_metadata_txt_pattern=self.post_metadata_txt_pattern,
-                                 storyitem_metadata_txt_pattern=self.storyitem_metadata_txt_pattern,
-                                 graphql_rate_limit=self.context.graphql_count_per_slidingwindow,
-                                 max_connection_attempts=self.context.max_connection_attempts)
-        new_loader.context.query_timestamps = self.context.query_timestamps
+        new_loader = Instaloader(
+            sleep=self.context.sleep,
+            quiet=self.context.quiet,
+            user_agent=self.context.user_agent,
+            dirname_pattern=self.dirname_pattern,
+            filename_pattern=self.filename_pattern,
+            download_pictures=self.download_pictures,
+            download_videos=self.download_videos,
+            download_video_thumbnails=self.download_video_thumbnails,
+            download_geotags=self.download_geotags,
+            download_comments=self.download_comments,
+            save_metadata=self.save_metadata,
+            compress_json=self.compress_json,
+            post_metadata_txt_pattern=self.post_metadata_txt_pattern,
+            storyitem_metadata_txt_pattern=self.storyitem_metadata_txt_pattern,
+            max_connection_attempts=self.context.max_connection_attempts,
+            request_timeout=self.context.request_timeout,
+            commit_mode=self.commit_mode)
         yield new_loader
         self.context.error_log.extend(new_loader.context.error_log)
         new_loader.context.error_log = []  # avoid double-printing of errors
-        self.context.query_timestamps = new_loader.context.query_timestamps
         new_loader.close()
 
     def close(self):
@@ -165,7 +227,9 @@ class Instaloader:
     def __exit__(self, *args):
         self.close()
 
-    def download_pic(self, filename: str, url: str, mtime: datetime, filename_suffix: Optional[str] = None) -> bool:
+    @_retry_on_connection_error
+    def download_pic(self, filename: str, url: str, mtime: datetime,
+                     filename_suffix: Optional[str] = None, _attempt: int = 1) -> bool:
         """Downloads and saves picture with given url under given directory with given timestamp.
         Returns true, if file was actually downloaded, i.e. updated."""
         urlmatch = re.search('\\.[a-z0-9]*\\?', url)
@@ -173,9 +237,15 @@ class Instaloader:
         if filename_suffix is not None:
             filename += '_' + filename_suffix
         filename += '.' + file_extension
-        if os.path.isfile(filename):
-            self.context.log(filename + ' exists', end=' ', flush=True)
-            return False
+        # A post is considered "commited" if the json file exists and is not malformed.
+        if self.commit_mode:
+            if self._committed and os.path.isfile(filename):
+                self.context.log(filename + ' exists', end=' ', flush=True)
+                return False
+        else:
+            if os.path.isfile(filename):
+                self.context.log(filename + ' exists', end=' ', flush=True)
+                return False
         self.context.get_and_write_raw(url, filename)
         os.utime(filename, (datetime.now().timestamp(), mtime.timestamp()))
         return True
@@ -186,17 +256,43 @@ class Instaloader:
             filename += '.json.xz'
         else:
             filename += '.json'
+        os.makedirs(os.path.dirname(filename), exist_ok=True)
         save_structure_to_file(structure, filename)
         if isinstance(structure, (Post, StoryItem)):
             # log 'json ' message when saving Post or StoryItem
             self.context.log('json', end=' ', flush=True)
 
     def update_comments(self, filename: str, post: Post) -> None:
-        def _postcomment_asdict(comment):
+        def _postcommentanswer_asdict(comment):
             return {'id': comment.id,
                     'created_at': int(comment.created_at_utc.replace(tzinfo=timezone.utc).timestamp()),
                     'text': comment.text,
-                    'owner': comment.owner._asdict()}
+                    'owner': comment.owner._asdict(),
+                    'likes_count': comment.likes_count}
+
+        def _postcomment_asdict(comment):
+            return {**_postcommentanswer_asdict(comment),
+                    'answers': sorted([_postcommentanswer_asdict(answer) for answer in comment.answers],
+                                      key=lambda t: int(t['id']),
+                                      reverse=True)}
+
+        def get_unique_comments(comments, combine_answers=False):
+            if not comments:
+                return list()
+            comments_list = sorted(sorted(list(comments), key=lambda t: int(t['id'])),
+                                   key=lambda t: int(t['created_at']), reverse=True)
+            unique_comments_list = [comments_list[0]]
+            for x, y in zip(comments_list[:-1], comments_list[1:]):
+                if x['id'] != y['id']:
+                    unique_comments_list.append(y)
+                else:
+                    unique_comments_list[-1]['likes_count'] = y.get('likes_count')
+                    if combine_answers:
+                        combined_answers = unique_comments_list[-1].get('answers') or list()
+                        if 'answers' in y:
+                            combined_answers.extend(y['answers'])
+                        unique_comments_list[-1]['answers'] = get_unique_comments(combined_answers)
+            return unique_comments_list
         filename += '_comments.json'
         try:
             with open(filename) as fp:
@@ -205,18 +301,10 @@ class Instaloader:
             comments = list()
         comments.extend(_postcomment_asdict(comment) for comment in post.get_comments())
         if comments:
-            comments_list = sorted(sorted(list(comments), key=lambda t: int(t['id'])),
-                                   key=lambda t: int(t['created_at']), reverse=True)
-            unique_comments_list = [comments_list[0]]
-            #for comment in comments_list:
-            #    if unique_comments_list[-1]['id'] != comment['id']:
-            #        unique_comments_list.append(comment)
-            #file.write(json.dumps(unique_comments_list, indent=4))
-            for x, y in zip(comments_list[:-1], comments_list[1:]):
-                if x['id'] != y['id']:
-                    unique_comments_list.append(y)
+            comments = get_unique_comments(comments, combine_answers=True)
+            answer_ids = set(int(answer['id']) for comment in comments for answer in comment.get('answers', []))
             with open(filename, 'w') as file:
-                file.write(json.dumps(unique_comments_list, indent=4))
+                file.write(json.dumps(list(filter(lambda t: int(t['id']) not in answer_ids, comments)), indent=4))
             self.context.log('comments', end=' ', flush=True)
 
     def save_caption(self, filename: str, mtime: datetime, caption: str) -> None:
@@ -227,11 +315,11 @@ class Instaloader:
         filename += '.txt'
         caption += '\n'
         pcaption = _elliptify(caption)
-        caption = caption.encode("UTF-8")
+        bcaption = caption.encode("UTF-8")
         with suppress(FileNotFoundError):
             with open(filename, 'rb') as file:
                 file_caption = file.read()
-            if file_caption.replace(b'\r\n', b'\n') == caption.replace(b'\r\n', b'\n'):
+            if file_caption.replace(b'\r\n', b'\n') == bcaption.replace(b'\r\n', b'\n'):
                 try:
                     self.context.log(pcaption + ' unchanged', end=' ', flush=True)
                 except UnicodeEncodeError:
@@ -239,8 +327,7 @@ class Instaloader:
                 return None
             else:
                 def get_filename(index):
-                    return filename if index == 0 else (filename[:-4] + '_old_' +
-                                                        (str(0) if index < 10 else str()) + str(index) + filename[-4:])
+                    return filename if index == 0 else '{0}_old_{2:02}{1}'.format(*os.path.splitext(filename), index)
 
                 i = 0
                 while os.path.isfile(get_filename(i)):
@@ -256,7 +343,7 @@ class Instaloader:
         except UnicodeEncodeError:
             self.context.log('txt', end=' ', flush=True)
         with open(filename, 'wb') as text_file:
-            shutil.copyfileobj(BytesIO(caption), text_file)
+            shutil.copyfileobj(BytesIO(bcaption), text_file)
         os.utime(filename, (datetime.now().timestamp(), mtime.timestamp()))
 
     def save_location(self, filename: str, location: PostLocation, mtime: datetime) -> None:
@@ -270,39 +357,66 @@ class Instaloader:
         os.utime(filename, (datetime.now().timestamp(), mtime.timestamp()))
         self.context.log('geo', end=' ', flush=True)
 
-    def download_profilepic(self, profile: Profile) -> None:
-        """Downloads and saves profile pic."""
+    @_retry_on_connection_error
+    def download_title_pic(self, url: str, target: Union[str, Path], name_suffix: str, owner_profile: Profile,
+                           _attempt: int = 1) -> None:
+        """Downloads and saves a picture that does not have an association with a Post or StoryItem, such as a
+        Profile picture or a Highlight cover picture. Modification time is taken from the HTTP response headers.
+
+        .. versionadded:: 4.3"""
 
         def _epoch_to_string(epoch: datetime) -> str:
-            return epoch.strftime('%Y-%m-%d_%H-%M-%S')
+            return epoch.strftime('%Y-%m-%d_%H-%M-%S_UTC')
 
-        profile_pic_url = profile.profile_pic_url
-        with self.context.get_anonymous_session() as anonymous_session:
-            date_object = datetime.strptime(anonymous_session.head(profile_pic_url).headers["Last-Modified"],
-                                            '%a, %d %b %Y %H:%M:%S GMT')
-        profile_pic_extension = 'jpg'
+        http_response = self.context.get_raw(url)
+        date_object = None  # type: Optional[datetime]
+        if 'Last-Modified' in http_response.headers:
+            date_object = datetime.strptime(http_response.headers["Last-Modified"], '%a, %d %b %Y %H:%M:%S GMT')
+            pic_bytes = None
+            pic_identifier = _epoch_to_string(date_object)
+        else:
+            pic_bytes = http_response.content
+            pic_identifier = md5(pic_bytes).hexdigest()[:16]
+        pic_extension = 'jpg'
         if ((format_string_contains_key(self.dirname_pattern, 'profile') or
              format_string_contains_key(self.dirname_pattern, 'target'))):
-            filename = '{0}/{1}_UTC_profile_pic.{2}'.format(self.dirname_pattern.format(profile=profile.username.lower(),
-                                                                                        target=profile.username.lower()),
-                                                            _epoch_to_string(date_object), profile_pic_extension)
+            filename = '{0}/{1}_{2}.{3}'.format(self.dirname_pattern.format(profile=owner_profile.username.lower(),
+                                                                            target=target),
+                                                pic_identifier, name_suffix, pic_extension)
         else:
-            filename = '{0}/{1}_{2}_UTC_profile_pic.{3}'.format(self.dirname_pattern.format(), profile.username.lower(),
-                                                                _epoch_to_string(date_object), profile_pic_extension)
-        if os.path.isfile(filename):
+            filename = '{0}/{1}_{2}_{3}.{4}'.format(self.dirname_pattern.format(), target,
+                                                    pic_identifier, name_suffix, pic_extension)
+        content_length = http_response.headers.get('Content-Length', None)
+        if os.path.isfile(filename) and (not self.context.is_logged_in or
+                                         (content_length is not None and
+                                          os.path.getsize(filename) >= int(content_length))):
             self.context.log(filename + ' already exists')
             return None
-        self.context.get_and_write_raw(profile_pic_url, filename)
-        os.utime(filename, (datetime.now().timestamp(), date_object.timestamp()))
+        os.makedirs(os.path.dirname(filename), exist_ok=True)
+        self.context.write_raw(pic_bytes if pic_bytes else http_response, filename)
+        if date_object:
+            os.utime(filename, (datetime.now().timestamp(), date_object.timestamp()))
         self.context.log('')  # log output of _get_and_write_raw() does not produce \n
 
+    def download_profilepic(self, profile: Profile) -> None:
+        """Downloads and saves profile pic."""
+        self.download_title_pic(profile.profile_pic_url, profile.username.lower(), 'profile_pic', profile)
+
+    def download_highlight_cover(self, highlight: Highlight, target: Union[str, Path]) -> None:
+        """Downloads and saves Highlight cover picture.
+
+        .. versionadded:: 4.3"""
+        self.download_title_pic(highlight.cover_url, target, 'cover', highlight.owner_profile)
+
     @_requires_login
     def save_session_to_file(self, filename: Optional[str] = None) -> None:
         """Saves internally stored :class:`requests.Session` object.
 
         :param filename: Filename, or None to use default filename.
+        :raises LoginRequiredException: If called without being logged in.
         """
         if filename is None:
+            assert self.context.username is not None
             filename = get_default_session_filename(self.context.username)
         dirname = os.path.dirname(filename)
         if dirname != '' and not os.path.exists(dirname):
@@ -335,16 +449,28 @@ class Instaloader:
 
         :raises InvalidArgumentException: If the provided username does not exist.
         :raises BadCredentialsException: If the provided password is wrong.
-        :raises ConnectionException: If connection to Instagram failed."""
+        :raises ConnectionException: If connection to Instagram failed.
+        :raises TwoFactorAuthRequiredException: First step of 2FA login done, now call
+           :meth:`Instaloader.two_factor_login`."""
         self.context.login(user, passwd)
 
-    def format_filename(self, item: Union[Post, StoryItem], target: Optional[str] = None):
+    def two_factor_login(self, two_factor_code) -> None:
+        """Second step of login if 2FA is enabled.
+        Not meant to be used directly, use :meth:`Instaloader.two_factor_login`.
+
+        :raises InvalidArgumentException: No two-factor authentication pending.
+        :raises BadCredentialsException: 2FA verification code invalid.
+
+        .. versionadded:: 4.2"""
+        self.context.two_factor_login(two_factor_code)
+
+    def format_filename(self, item: Union[Post, StoryItem], target: Optional[Union[str, Path]] = None):
         """Format filename of a :class:`Post` or :class:`StoryItem` according to ``filename-pattern`` parameter.
 
         .. versionadded:: 4.1"""
         return _PostPathFormatter(item).format(self.filename_pattern, target=target)
 
-    def download_post(self, post: Post, target: str) -> bool:
+    def download_post(self, post: Post, target: Union[str, Path]) -> bool:
         """
         Download everything associated with one instagram post node, i.e. picture, caption and video.
 
@@ -358,18 +484,19 @@ class Instaloader:
         os.makedirs(os.path.dirname(filename), exist_ok=True)
 
         # Download the image(s) / video thumbnail and videos within sidecars if desired
-        downloaded = False
+        downloaded = True
+        self._committed = self.check_if_committed(filename)
         if self.download_pictures:
             if post.typename == 'GraphSidecar':
                 edge_number = 1
                 for sidecar_node in post.get_sidecar_nodes():
                     # Download picture or video thumbnail
                     if not sidecar_node.is_video or self.download_video_thumbnails is True:
-                        downloaded |= self.download_pic(filename=filename, url=sidecar_node.display_url,
+                        downloaded &= self.download_pic(filename=filename, url=sidecar_node.display_url,
                                                         mtime=post.date_local, filename_suffix=str(edge_number))
                     # Additionally download video if available and desired
                     if sidecar_node.is_video and self.download_videos is True:
-                        downloaded |= self.download_pic(filename=filename, url=sidecar_node.video_url,
+                        downloaded &= self.download_pic(filename=filename, url=sidecar_node.video_url,
                                                         mtime=post.date_local, filename_suffix=str(edge_number))
                     edge_number += 1
             elif post.typename == 'GraphImage':
@@ -387,7 +514,7 @@ class Instaloader:
 
         # Download video if desired
         if post.is_video and self.download_videos is True:
-            downloaded |= self.download_pic(filename=filename, url=post.video_url, mtime=post.date_local)
+            downloaded &= self.download_pic(filename=filename, url=post.video_url, mtime=post.date_local)
 
         # Download geotags if desired
         if self.download_geotags and post.location:
@@ -411,6 +538,7 @@ class Instaloader:
         To use this, one needs to be logged in
 
         :param userids: List of user IDs to be processed in terms of downloading their stories, or None.
+        :raises LoginRequiredException: If called without being logged in.
         """
 
         if not userids:
@@ -421,6 +549,7 @@ class Instaloader:
             userids = list(edge["node"]["id"] for edge in data["feed_reels_tray"]["edge_reels_tray_to_reel"]["edges"])
 
         def _userid_chunks():
+            assert userids is not None
             userids_per_query = 100
             for i in range(0, len(userids), userids_per_query):
                 yield userids[i:i + userids_per_query]
@@ -446,6 +575,7 @@ class Instaloader:
         :param filename_target: Replacement for {target} in dirname_pattern and filename_pattern
                or None if profile name should be used instead
         :param storyitem_filter: function(storyitem), which returns True if given StoryItem should be downloaded
+        :raises LoginRequiredException: If called without being logged in.
         """
 
         if not userids:
@@ -469,7 +599,7 @@ class Instaloader:
                     if fast_update and not downloaded:
                         break
 
-    def download_storyitem(self, item: StoryItem, target: str) -> bool:
+    def download_storyitem(self, item: StoryItem, target: Union[str, Path]) -> bool:
         """Download one user story.
 
         :param item: Story item, as in story['items'] for story in :meth:`get_stories`
@@ -505,6 +635,7 @@ class Instaloader:
         .. versionadded:: 4.1
 
         :param user: ID or Profile of the user whose highlights should get fetched.
+        :raises LoginRequiredException: If called without being logged in.
         """
 
         userid = user if isinstance(user, int) else user.userid
@@ -529,15 +660,23 @@ class Instaloader:
 
         .. versionadded:: 4.1
 
+        .. versionchanged:: 4.3
+           Also downloads and saves the Highlight's cover pictures.
+
         :param user: ID or Profile of the user whose highlights should get downloaded.
         :param fast_update: If true, abort when first already-downloaded picture is encountered
         :param filename_target: Replacement for {target} in dirname_pattern and filename_pattern
                or None if profile name and the highlights' titles should be used instead
         :param storyitem_filter: function(storyitem), which returns True if given StoryItem should be downloaded
+        :raises LoginRequiredException: If called without being logged in.
         """
         for user_highlight in self.get_highlights(user):
             name = user_highlight.owner_username
+            highlight_target = (filename_target
+                                if filename_target
+                                else Path(name) / Path(user_highlight.title))  # type: Union[str, Path]
             self.context.log("Retrieving highlights \"{}\" from profile {}".format(user_highlight.title, name))
+            self.download_highlight_cover(user_highlight, highlight_target)
             totalcount = user_highlight.itemcount
             count = 1
             for item in user_highlight.get_items():
@@ -546,10 +685,9 @@ class Instaloader:
                     continue
                 self.context.log("[%3i/%3i] " % (count, totalcount), end="", flush=True)
                 count += 1
-                with self.context.error_catcher('Download highlights \"{}\" from user {}'.format(user_highlight.title, name)):
-                    downloaded = self.download_storyitem(item, filename_target
-                                                         if filename_target
-                                                         else '{}/{}'.format(name, user_highlight.title))
+                with self.context.error_catcher('Download highlights \"{}\" from user {}'.format(user_highlight.title,
+                                                                                                 name)):
+                    downloaded = self.download_storyitem(item, highlight_target)
                     if fast_update and not downloaded:
                         break
 
@@ -558,14 +696,17 @@ class Instaloader:
         """Get Posts of the user's feed.
 
         :return: Iterator over Posts of the user's feed.
+        :raises LoginRequiredException: If called without being logged in.
         """
 
         data = self.context.graphql_query("d6f4427fbe92d846298cf93df0b937d3", {})["data"]
 
         while True:
             feed = data["user"]["edge_web_feed_timeline"]
-            yield from (Post(self.context, edge["node"]) for edge in feed["edges"]
-                        if not edge["node"]["__typename"] == "GraphSuggestedUserFeedUnit")
+            for edge in feed["edges"]:
+                node = edge["node"]
+                if node.get("__typename") in Post.supported_graphql_types() and node.get("shortcode") is not None:
+                    yield Post(self.context, node)
             if not feed["page_info"]["has_next_page"]:
                 break
             data = self.context.graphql_query("d6f4427fbe92d846298cf93df0b937d3",
@@ -591,6 +732,7 @@ class Instaloader:
         :param max_count: Maximum count of pictures to download
         :param fast_update: If true, abort when first already-downloaded picture is encountered
         :param post_filter: function(post), which returns True if given picture should be downloaded
+        :raises LoginRequiredException: If called without being logged in.
         """
         self.context.log("Retrieving pictures from your feed...")
         count = 1
@@ -616,9 +758,11 @@ class Instaloader:
         :param max_count: Maximum count of pictures to download
         :param fast_update: If true, abort when first already-downloaded picture is encountered
         :param post_filter: function(post), which returns True if given picture should be downloaded
+        :raises LoginRequiredException: If called without being logged in.
         """
         self.context.log("Retrieving saved posts...")
         count = 1
+        assert self.context.username is not None
         for post in Profile.from_username(self.context, self.context.username).get_saved_posts():
             if max_count is not None and count > max_count:
                 break
@@ -632,18 +776,82 @@ class Instaloader:
                 if fast_update and not downloaded:
                     break
 
+    @_requires_login
+    def get_location_posts(self, location: str) -> Iterator[Post]:
+        """Get Posts which are listed by Instagram for a given Location.
+
+        :return:  Iterator over Posts of a location's posts
+        :raises LoginRequiredException: If called without being logged in.
+
+        .. versionadded:: 4.2
+
+        .. versionchanged:: 4.2.9
+           Require being logged in (as required by Instagram)
+        """
+        has_next_page = True
+        end_cursor = None
+        while has_next_page:
+            if end_cursor:
+                params = {'__a': 1, 'max_id': end_cursor}
+            else:
+                params = {'__a': 1}
+            location_data = self.context.get_json('explore/locations/{0}/'.format(location),
+                                                  params)['graphql']['location']['edge_location_to_media']
+            yield from (Post(self.context, edge['node']) for edge in location_data['edges'])
+            has_next_page = location_data['page_info']['has_next_page']
+            end_cursor = location_data['page_info']['end_cursor']
+
+    @_requires_login
+    def download_location(self, location: str,
+                          max_count: Optional[int] = None,
+                          post_filter: Optional[Callable[[Post], bool]] = None,
+                          fast_update: bool = False) -> None:
+        """Download pictures of one location.
+
+        To download the last 30 pictures with location 362629379, do::
+
+            loader = Instaloader()
+            loader.download_location(362629379, max_count=30)
+
+        :param location: Location to download, as Instagram numerical ID
+        :param max_count: Maximum count of pictures to download
+        :param post_filter: function(post), which returns True if given picture should be downloaded
+        :param fast_update: If true, abort when first already-downloaded picture is encountered
+        :raises LoginRequiredException: If called without being logged in.
+
+        .. versionadded:: 4.2
+
+        .. versionchanged:: 4.2.9
+           Require being logged in (as required by Instagram)
+        """
+        self.context.log("Retrieving pictures for location {}...".format(location))
+        count = 1
+        for post in self.get_location_posts(location):
+            if max_count is not None and count > max_count:
+                break
+            self.context.log('[{0:3d}] %{1} '.format(count, location), end='', flush=True)
+            if post_filter is not None and not post_filter(post):
+                self.context.log('<skipped>')
+                continue
+            count += 1
+            with self.context.error_catcher('Download location {}'.format(location)):
+                downloaded = self.download_post(post, target='%' + location)
+                if fast_update and not downloaded:
+                    break
+
     @_requires_login
     def get_explore_posts(self) -> Iterator[Post]:
         """Get Posts which are worthy of exploring suggested by Instagram.
 
         :return: Iterator over Posts of the user's suggested posts.
+        :raises LoginRequiredException: If called without being logged in.
         """
         data = self.context.get_json('explore/', {})
         yield from (Post(self.context, node)
                     for node in self.context.graphql_node_list("df0dcc250c2b18d9fd27c5581ef33c7c",
                                                                {}, 'https://www.instagram.com/explore/',
                                                                lambda d: d['data']['user']['edge_web_discover_media'],
-                                                               data['rhx_gis']))
+                                                               data.get('rhx_gis')))
 
     def get_hashtag_posts(self, hashtag: str) -> Iterator[Post]:
         """Get Posts associated with a #hashtag."""
@@ -698,8 +906,6 @@ class Instaloader:
         """Download all posts where a profile is tagged.
 
         .. versionadded:: 4.1"""
-        if target is None:
-            target = profile.username + '/:tagged'
         self.context.log("Retrieving tagged posts for profile {}.".format(profile.username))
         count = 1
         for post in profile.get_tagged_posts():
@@ -707,8 +913,26 @@ class Instaloader:
             count += 1
             if post_filter is not None and not post_filter(post):
                 self.context.log('<{} skipped>'.format(post))
+                continue
             with self.context.error_catcher('Download tagged {}'.format(profile.username)):
-                downloaded = self.download_post(post, target)
+                downloaded = self.download_post(post, target if target else Path(profile.username) / Path(':tagged'))
+                if fast_update and not downloaded:
+                    break
+
+    def download_igtv(self, profile: Profile, fast_update: bool = False,
+                      post_filter: Optional[Callable[[Post], bool]] = None) -> None:
+        """Download IGTV videos of a profile.
+
+        .. versionadded:: 4.3"""
+        self.context.log("Retrieving IGTV videos for profile {}.".format(profile.username))
+        for number, post in enumerate(profile.get_igtv_posts()):
+            self.context.log("[{0:{w}d}/{1:{w}d}] ".format(number+1, profile.igtvcount, w=len(str(profile.igtvcount))),
+                             end="", flush=True)
+            if post_filter is not None and not post_filter(post):
+                self.context.log('<{} skipped>'.format(post))
+                continue
+            with self.context.error_catcher('Download IGTV {}'.format(post.shortcode)):
+                downloaded = self.download_post(post, target=profile.username)
                 if fast_update and not downloaded:
                     break
 
@@ -741,16 +965,18 @@ class Instaloader:
         :return: Instance of current profile
         """
         profile = None
-        with suppress(ProfileNotExistsException):
+        profile_name_not_exists_err = None
+        try:
             profile = Profile.from_username(self.context, profile_name)
-        profile_exists = profile is not None
+        except ProfileNotExistsException as err:
+            profile_name_not_exists_err = err
         id_filename = self._get_id_filename(profile_name)
         try:
             with open(id_filename, 'rb') as id_file:
                 profile_id = int(id_file.read())
-            if (not profile_exists) or \
+            if (profile is None) or \
                     (profile_id != profile.userid):
-                if profile_exists:
+                if profile is not None:
                     self.context.log("Profile {0} does not match the stored unique ID {1}.".format(profile_name,
                                                                                                    profile_id))
                 else:
@@ -769,17 +995,23 @@ class Instaloader:
                     os.rename('{0}/{1}_id'.format(self.dirname_pattern.format(), profile_name.lower()),
                               '{0}/{1}_id'.format(self.dirname_pattern.format(), newname.lower()))
                 return profile_from_id
+            # profile exists and profile id matches saved id
             return profile
         except (FileNotFoundError, ValueError):
             pass
-        if profile_exists:
+        if profile is not None:
             self.save_profile_id(profile)
             return profile
+        if profile_name_not_exists_err:
+            raise profile_name_not_exists_err
         raise ProfileNotExistsException("Profile {0} does not exist.".format(profile_name))
 
     def download_profiles(self, profiles: Set[Profile],
                           profile_pic: bool = True, posts: bool = True,
-                          tagged: bool = False, highlights: bool = False, stories: bool = False,
+                          tagged: bool = False,
+                          igtv: bool = False,
+                          highlights: bool = False,
+                          stories: bool = False,
                           fast_update: bool = False,
                           post_filter: Optional[Callable[[Post], bool]] = None,
                           storyitem_filter: Optional[Callable[[Post], bool]] = None,
@@ -790,6 +1022,7 @@ class Instaloader:
         :param profile_pic: not :option:`--no-profile-pic`.
         :param posts: not :option:`--no-posts`.
         :param tagged: :option:`--tagged`.
+        :param igtv: :option:`--igtv`.
         :param highlights: :option:`--highlights`.
         :param stories: :option:`--stories`.
         :param fast_update: :option:`--fast-update`.
@@ -799,17 +1032,28 @@ class Instaloader:
            Whether :exc:`LoginRequiredException` and :exc:`PrivateProfileNotFollowedException` should be raised or
            catched and printed with :meth:`InstaloaderContext.error_catcher`.
 
-        .. versionadded:: 4.1"""
+        .. versionadded:: 4.1
 
+        .. versionchanged:: 4.3
+           Add `igtv` parameter.
+        """
+
+        @contextmanager
         def _error_raiser(_str):
             yield
 
+        # error_handler type is Callable[[Optional[str]], ContextManager[None]] (not supported with Python 3.5)
         error_handler = _error_raiser if raise_errors else self.context.error_catcher
 
         for profile in profiles:
-            with error_handler(profile.username):
+            with error_handler(profile.username):  # type: ignore
                 profile_name = profile.username
 
+                # Download profile picture
+                if profile_pic:
+                    with self.context.error_catcher('Download profile picture of {}'.format(profile_name)):
+                        self.download_profilepic(profile)
+
                 # Save metadata as JSON if desired.
                 if self.save_metadata:
                     json_filename = '{0}/{1}_{2}'.format(self.dirname_pattern.format(profile=profile_name,
@@ -817,13 +1061,8 @@ class Instaloader:
                                                          profile_name, profile.userid)
                     self.save_metadata_json(json_filename, profile)
 
-                # Download profile picture
-                if profile_pic:
-                    with self.context.error_catcher('Download profile picture of {}'.format(profile_name)):
-                        self.download_profilepic(profile)
-
                 # Catch some errors
-                if profile.is_private:
+                if profile.is_private and (tagged or igtv or highlights or posts):
                     if not self.context.is_logged_in:
                         raise LoginRequiredException("--login=USERNAME required.")
                     if not profile.followed_by_viewer and self.context.username != profile.username:
@@ -834,6 +1073,11 @@ class Instaloader:
                     with self.context.error_catcher('Download tagged of {}'.format(profile_name)):
                         self.download_tagged(profile, fast_update=fast_update, post_filter=post_filter)
 
+                # Download IGTV, if requested
+                if igtv:
+                    with self.context.error_catcher('Download IGTV of {}'.format(profile_name)):
+                        self.download_igtv(profile, fast_update=fast_update, post_filter=post_filter)
+
                 # Download highlights, if requested
                 if highlights:
                     with self.context.error_catcher('Download highlights of {}'.format(profile_name)):
@@ -851,8 +1095,22 @@ class Instaloader:
                             self.context.log('<skipped>')
                             continue
                         with self.context.error_catcher("Download {} of {}".format(post, profile_name)):
-                            downloaded = self.download_post(post, target=profile_name)
-                            if fast_update and not downloaded:
+                            # The PostChangedException gets raised if the Post's id/shortcode changed while obtaining
+                            # additional metadata. This is most likely the case if a HTTP redirect takes place while
+                            # resolving the shortcode URL.
+                            # The `post_changed` variable keeps the fast-update functionality alive: A Post which is
+                            # obained after a redirect has probably already been downloaded as a previous Post of the
+                            # same Profile.
+                            # Observed in issue #225: https://github.com/instaloader/instaloader/issues/225
+                            post_changed = False
+                            while True:
+                                try:
+                                    downloaded = self.download_post(post, target=profile_name)
+                                    break
+                                except PostChangedException:
+                                    post_changed = True
+                                    continue
+                            if fast_update and not downloaded and not post_changed:
                                 break
 
         if stories and profiles:
@@ -945,6 +1203,25 @@ class Instaloader:
                 if fast_update and not downloaded:
                     break
 
+    def check_if_committed(self, filename: str) -> bool:
+        """Checks to see if the current post has been committed.
+
+        A post is considered committed if its json metadata file exists and is not malformed.
+
+        .. versionadded:: 4.2
+        """
+        if os.path.isfile(filename + '.json.xz'):
+            filename += '.json.xz'
+        elif os.path.isfile(filename + '.json'):
+            filename += '.json'
+        else:
+            return False
+        try:
+            load_structure_from_file(self.context, filename)
+            return True
+        except (FileNotFoundError, lzma.LZMAError, json.decoder.JSONDecodeError):
+            return False
+
     def interactive_login(self, username: str) -> None:
         """Logs in and internally stores session, asking user for password interactively.
 
@@ -953,11 +1230,20 @@ class Instaloader:
         :raises ConnectionException: If connection to Instagram failed."""
         if self.context.quiet:
             raise LoginRequiredException("Quiet mode requires given password or valid session file.")
-        password = None
-        while password is None:
-            password = getpass.getpass(prompt="Enter Instagram password for %s: " % username)
-            try:
-                self.login(username, password)
-            except BadCredentialsException as err:
-                print(err, file=sys.stderr)
-                password = None
+        try:
+            password = None
+            while password is None:
+                password = getpass.getpass(prompt="Enter Instagram password for %s: " % username)
+                try:
+                    self.login(username, password)
+                except BadCredentialsException as err:
+                    print(err, file=sys.stderr)
+                    password = None
+        except TwoFactorAuthRequiredException:
+            while True:
+                try:
+                    code = input("Enter 2FA verification code: ")
+                    self.two_factor_login(code)
+                    break
+                except BadCredentialsException:
+                    pass

instaloader/instaloadercontext.py

@@ -10,20 +10,24 @@ import time
 import urllib.parse
 from contextlib import contextmanager
 from datetime import datetime, timedelta
-from typing import Any, Callable, Dict, Iterator, Optional
+from functools import partial
+from typing import Any, Callable, Dict, Iterator, List, Optional, Union
 
 import requests
 import requests.utils
-import urllib3
 
 from .exceptions import *
 
 
-def copy_session(session: requests.Session) -> requests.Session:
+def copy_session(session: requests.Session, request_timeout: Optional[float] = None) -> requests.Session:
     """Duplicates a requests.Session."""
     new = requests.Session()
     new.cookies = requests.utils.cookiejar_from_dict(requests.utils.dict_from_cookiejar(session.cookies))
-    new.headers = session.headers.copy()
+    new.headers = session.headers.copy() # type: ignore
+    if request_timeout is not None:
+        # Override default timeout behavior.
+        # Need to silence mypy bug for this. See: https://github.com/python/mypy/issues/2427
+        new.request = partial(new.request, timeout=request_timeout) # type: ignore
     return new
 
 
@@ -48,29 +52,31 @@ class InstaloaderContext:
     """
 
     def __init__(self, sleep: bool = True, quiet: bool = False, user_agent: Optional[str] = None,
-                 graphql_count_per_slidingwindow: Optional[int] = None, max_connection_attempts: int = 3):
+                 max_connection_attempts: int = 3, request_timeout: Optional[float] = None):
 
         self.user_agent = user_agent if user_agent is not None else default_user_agent()
+        self.request_timeout = request_timeout
         self._session = self.get_anonymous_session()
         self.username = None
         self.sleep = sleep
         self.quiet = quiet
         self.max_connection_attempts = max_connection_attempts
         self._graphql_page_length = 50
-        self.graphql_count_per_slidingwindow = graphql_count_per_slidingwindow or 200
         self._root_rhx_gis = None
+        self.two_factor_auth_pending = None
 
         # error log, filled with error() and printed at the end of Instaloader.main()
-        self.error_log = []
+        self.error_log = []                      # type: List[str]
 
         # For the adaption of sleep intervals (rate control)
-        self.query_timestamps = list()
+        self._graphql_query_timestamps = dict()  # type: Dict[str, List[float]]
+        self._graphql_earliest_next_request_time = 0.0
 
         # Can be set to True for testing, disables supression of InstaloaderContext._error_catcher
         self.raise_all_errors = False
 
         # Cache profile from id (mapping from id to Profile)
-        self.profile_id_cache = dict()
+        self.profile_id_cache = dict()           # type: Dict[int, Any]
 
     @contextmanager
     def anonymous_copy(self):
@@ -155,6 +161,10 @@ class InstaloaderContext:
                                 'ig_vw': '1920', 'csrftoken': '',
                                 's_network': '', 'ds_user_id': ''})
         session.headers.update(self._default_http_header(empty_session_only=True))
+        if self.request_timeout is not None:
+            # Override default timeout behavior.
+            # Need to silence mypy bug for this. See: https://github.com/python/mypy/issues/2427
+            session.request = partial(session.request, timeout=self.request_timeout) # type: ignore
         return session
 
     def save_session_to_file(self, sessionfile):
@@ -167,6 +177,10 @@ class InstaloaderContext:
         session.cookies = requests.utils.cookiejar_from_dict(pickle.load(sessionfile))
         session.headers.update(self._default_http_header())
         session.headers.update({'X-CSRFToken': session.cookies.get_dict()['csrftoken']})
+        if self.request_timeout is not None:
+            # Override default timeout behavior.
+            # Need to silence mypy bug for this. See: https://github.com/python/mypy/issues/2427
+            session.request = partial(session.request, timeout=self.request_timeout) # type: ignore
         self._session = session
         self.username = username
 
@@ -180,29 +194,57 @@ class InstaloaderContext:
 
         :raises InvalidArgumentException: If the provided username does not exist.
         :raises BadCredentialsException: If the provided password is wrong.
-        :raises ConnectionException: If connection to Instagram failed."""
+        :raises ConnectionException: If connection to Instagram failed.
+        :raises TwoFactorAuthRequiredException: First step of 2FA login done, now call
+           :meth:`Instaloader.two_factor_login`."""
+        # pylint:disable=import-outside-toplevel
         import http.client
         # pylint:disable=protected-access
         http.client._MAXHEADERS = 200
         session = requests.Session()
         session.cookies.update({'sessionid': '', 'mid': '', 'ig_pr': '1',
-                                'ig_vw': '1920', 'csrftoken': '',
+                                'ig_vw': '1920', 'ig_cb': '1', 'csrftoken': '',
                                 's_network': '', 'ds_user_id': ''})
         session.headers.update(self._default_http_header())
-        session.headers.update({'X-CSRFToken': self.get_json('', {})['config']['csrf_token']})
+        if self.request_timeout is not None:
+            # Override default timeout behavior.
+            # Need to silence mypy bug for this. See: https://github.com/python/mypy/issues/2427
+            session.request = partial(session.request, timeout=self.request_timeout) # type: ignore
+        session.get('https://www.instagram.com/web/__mid/')
+        csrf_token = session.cookies.get_dict()['csrftoken']
+        session.headers.update({'X-CSRFToken': csrf_token})
         # Not using self.get_json() here, because we need to access csrftoken cookie
-        self._sleep()
+        self.do_sleep()
         login = session.post('https://www.instagram.com/accounts/login/ajax/',
                              data={'password': passwd, 'username': user}, allow_redirects=True)
-        if login.status_code != 200:
-            raise ConnectionException("Login error: {} {}".format(login.status_code, login.reason))
-        resp_json = login.json()
+        try:
+            resp_json = login.json()
+        except json.decoder.JSONDecodeError:
+            raise ConnectionException("Login error: JSON decode fail, {} - {}.".format(login.status_code, login.reason))
+        if resp_json.get('two_factor_required'):
+            two_factor_session = copy_session(session, self.request_timeout)
+            two_factor_session.headers.update({'X-CSRFToken': csrf_token})
+            two_factor_session.cookies.update({'csrftoken': csrf_token})
+            self.two_factor_auth_pending = (two_factor_session,
+                                            user,
+                                            resp_json['two_factor_info']['two_factor_identifier'])
+            raise TwoFactorAuthRequiredException("Login error: two-factor authentication required.")
+        if resp_json.get('checkpoint_url'):
+            raise ConnectionException("Login: Checkpoint required. Point your browser to "
+                                      "https://www.instagram.com{} - "
+                                      "follow the instructions, then retry.".format(resp_json.get('checkpoint_url')))
         if resp_json['status'] != 'ok':
             if 'message' in resp_json:
                 raise ConnectionException("Login error: \"{}\" status, message \"{}\".".format(resp_json['status'],
                                                                                                resp_json['message']))
             else:
                 raise ConnectionException("Login error: \"{}\" status.".format(resp_json['status']))
+        if 'authenticated' not in resp_json:
+            # Issue #472
+            if 'message' in resp_json:
+                raise ConnectionException("Login error: Unexpected response, \"{}\".".format(resp_json['message']))
+            else:
+                raise ConnectionException("Login error: Unexpected response, this might indicate a blocked IP.")
         if not resp_json['authenticated']:
             if resp_json['user']:
                 # '{"authenticated": false, "user": true, "status": "ok"}'
@@ -218,11 +260,108 @@ class InstaloaderContext:
         self._session = session
         self.username = user
 
-    def _sleep(self):
+    def two_factor_login(self, two_factor_code):
+        """Second step of login if 2FA is enabled.
+        Not meant to be used directly, use :meth:`Instaloader.two_factor_login`.
+
+        :raises InvalidArgumentException: No two-factor authentication pending.
+        :raises BadCredentialsException: 2FA verification code invalid.
+
+        .. versionadded:: 4.2"""
+        if not self.two_factor_auth_pending:
+            raise InvalidArgumentException("No two-factor authentication pending.")
+        (session, user, two_factor_id) = self.two_factor_auth_pending
+
+        login = session.post('https://www.instagram.com/accounts/login/ajax/two_factor/',
+                             data={'username': user, 'verificationCode': two_factor_code, 'identifier': two_factor_id},
+                             allow_redirects=True)
+        resp_json = login.json()
+        if resp_json['status'] != 'ok':
+            if 'message' in resp_json:
+                raise BadCredentialsException("Login error: {}".format(resp_json['message']))
+            else:
+                raise BadCredentialsException("Login error: \"{}\" status.".format(resp_json['status']))
+        session.headers.update({'X-CSRFToken': login.cookies['csrftoken']})
+        self._session = session
+        self.username = user
+        self.two_factor_auth_pending = None
+
+    def do_sleep(self):
         """Sleep a short time if self.sleep is set. Called before each request to instagram.com."""
         if self.sleep:
             time.sleep(min(random.expovariate(0.7), 5.0))
 
+    def _dump_query_timestamps(self, current_time: float):
+        """Output the number of GraphQL queries grouped by their query_hash within the last time."""
+        windows = [10, 11, 15, 20, 30, 60]
+        print("GraphQL requests:", file=sys.stderr)
+        for query_hash, times in self._graphql_query_timestamps.items():
+            print("  {}".format(query_hash), file=sys.stderr)
+            for window in windows:
+                reqs_in_sliding_window = sum(t > current_time - window * 60 for t in times)
+                print("    last {} minutes: {} requests".format(window, reqs_in_sliding_window), file=sys.stderr)
+
+    def _graphql_request_count_per_sliding_window(self, query_hash: str) -> int:
+        """Return how many GraphQL requests can be done within the sliding window."""
+        if self.is_logged_in:
+            max_reqs = {'1cb6ec562846122743b61e492c85999f': 20,
+                        '33ba35852cb50da46f5b5e889df7d159': 20,
+                        'iphone': 100,
+                        'other': 100}
+        else:
+            max_reqs = {'1cb6ec562846122743b61e492c85999f': 200,
+                        '33ba35852cb50da46f5b5e889df7d159': 200,
+                        'other': 200}
+        return max_reqs.get(query_hash) or min(max_reqs.values())
+
+    def _graphql_query_waittime(self, query_hash: str, current_time: float, untracked_queries: bool = False) -> float:
+        """Calculate time needed to wait before GraphQL query can be executed."""
+        sliding_window = 660
+        if query_hash not in self._graphql_query_timestamps:
+            self._graphql_query_timestamps[query_hash] = []
+        self._graphql_query_timestamps[query_hash] = list(filter(lambda t: t > current_time - 60 * 60,
+                                                                 self._graphql_query_timestamps[query_hash]))
+        reqs_in_sliding_window = list(filter(lambda t: t > current_time - sliding_window,
+                                             self._graphql_query_timestamps[query_hash]))
+        count_per_sliding_window = self._graphql_request_count_per_sliding_window(query_hash)
+        if len(reqs_in_sliding_window) < count_per_sliding_window and not untracked_queries:
+            return max(0, self._graphql_earliest_next_request_time - current_time)
+        next_request_time = min(reqs_in_sliding_window) + sliding_window + 6
+        if untracked_queries:
+            self._graphql_earliest_next_request_time = next_request_time
+        return round(max(next_request_time, self._graphql_earliest_next_request_time) - current_time)
+
+    def _ratecontrol_graphql_query(self, query_hash: str, untracked_queries: bool = False):
+        """Called before a GraphQL query is made in order to stay within Instagram's rate limits.
+
+        :param query_hash: The query_hash parameter of the query.
+        :param untracked_queries: True, if 429 has been returned to apply 429 logic.
+        """
+        if not untracked_queries:
+            waittime = self._graphql_query_waittime(query_hash, time.monotonic(), untracked_queries)
+            assert waittime >= 0
+            if waittime > 10:
+                self.log('\nToo many queries in the last time. Need to wait {} seconds, until {:%H:%M}.'
+                         .format(waittime, datetime.now() + timedelta(seconds=waittime)))
+            time.sleep(waittime)
+            if query_hash not in self._graphql_query_timestamps:
+                self._graphql_query_timestamps[query_hash] = [time.monotonic()]
+            else:
+                self._graphql_query_timestamps[query_hash].append(time.monotonic())
+        else:
+            text_for_429 = ("HTTP error code 429 was returned because too many queries occured in the last time. "
+                            "Please do not use Instagram in your browser or run multiple instances of Instaloader "
+                            "in parallel.")
+            print(textwrap.fill(text_for_429), file=sys.stderr)
+            current_time = time.monotonic()
+            waittime = self._graphql_query_waittime(query_hash, current_time, untracked_queries)
+            assert waittime >= 0
+            if waittime > 10:
+                self.log('The request will be retried in {} seconds, at {:%H:%M}.'
+                         .format(waittime, datetime.now() + timedelta(seconds=waittime)))
+            self._dump_query_timestamps(current_time)
+            time.sleep(waittime)
+
     def get_json(self, path: str, params: Dict[str, Any], host: str = 'www.instagram.com',
                  session: Optional[requests.Session] = None, _attempt=1) -> Dict[str, Any]:
         """JSON request to Instagram.
@@ -236,36 +375,25 @@ class InstaloaderContext:
         :raises QueryReturnedNotFoundException: When the server responds with a 404.
         :raises ConnectionException: When query repeatedly failed.
         """
-        def graphql_query_waittime(untracked_queries: bool = False) -> int:
-            sliding_window = 660
-            if not self.query_timestamps:
-                return sliding_window if untracked_queries else 0
-            current_time = time.monotonic()
-            self.query_timestamps = list(filter(lambda t: t > current_time - sliding_window, self.query_timestamps))
-            if len(self.query_timestamps) < self.graphql_count_per_slidingwindow and not untracked_queries:
-                return 0
-            return round(min(self.query_timestamps) + sliding_window - current_time) + 6
         is_graphql_query = 'query_hash' in params and 'graphql/query' in path
-        # some queries are not rate limited if invoked anonymously:
-        query_not_limited = is_graphql_query and not self.is_logged_in \
-                            and params['query_hash'] in ['9ca88e465c3f866a76f7adee3871bdd8']
-        if is_graphql_query and not query_not_limited:
-            waittime = graphql_query_waittime()
-            if waittime > 0:
-                self.log('\nToo many queries in the last time. Need to wait {} seconds, until {:%H:%M}.'
-                         .format(waittime, datetime.now() + timedelta(seconds=waittime)))
-                time.sleep(waittime)
-            if self.query_timestamps is not None:
-                self.query_timestamps.append(time.monotonic())
-            else:
-                self.query_timestamps = [time.monotonic()]
+        is_iphone_query = host == 'i.instagram.com'
+        is_other_query = not is_graphql_query and host == "www.instagram.com"
         sess = session if session else self._session
         try:
-            self._sleep()
+            self.do_sleep()
+            if is_graphql_query:
+                self._ratecontrol_graphql_query(params['query_hash'])
+            if is_iphone_query:
+                self._ratecontrol_graphql_query('iphone')
+            if is_other_query:
+                self._ratecontrol_graphql_query('other')
             resp = sess.get('https://{0}/{1}'.format(host, path), params=params, allow_redirects=False)
             while resp.is_redirect:
                 redirect_url = resp.headers['location']
                 self.log('\nHTTP redirect from https://{0}/{1} to {2}'.format(host, path, redirect_url))
+                if redirect_url.startswith('https://www.instagram.com/accounts/login'):
+                    # alternate rate limit exceeded behavior
+                    raise TooManyRequestsException("429 Too Many Requests: redirected to login")
                 if redirect_url.startswith('https://{}/'.format(host)):
                     resp = sess.get(redirect_url if redirect_url.endswith('/') else redirect_url + '/',
                                     params=params, allow_redirects=False)
@@ -283,8 +411,19 @@ class InstaloaderContext:
             if is_html_query:
                 match = re.search(r'window\._sharedData = (.*);</script>', resp.text)
                 if match is None:
-                    raise ConnectionException("Could not find \"window._sharedData\" in html response.")
-                return json.loads(match.group(1))
+                    raise QueryReturnedNotFoundException("Could not find \"window._sharedData\" in html response.")
+                resp_json = json.loads(match.group(1))
+                entry_data = resp_json.get('entry_data')
+                post_or_profile_page = list(entry_data.values())[0] if entry_data is not None else None
+                if post_or_profile_page is None:
+                    raise ConnectionException("\"window._sharedData\" does not contain required keys.")
+                # If GraphQL data is missing in `window._sharedData`, search for it in `__additionalDataLoaded`.
+                if 'graphql' not in post_or_profile_page[0]:
+                    match = re.search(r'window\.__additionalDataLoaded\([^{]+{"graphql":({.*})}\);</script>',
+                                      resp.text)
+                    if match is not None:
+                        post_or_profile_page[0]['graphql'] = json.loads(match.group(1))
+                return resp_json
             else:
                 resp_json = resp.json()
             if 'status' in resp_json and resp_json['status'] != "ok":
@@ -297,21 +436,18 @@ class InstaloaderContext:
         except (ConnectionException, json.decoder.JSONDecodeError, requests.exceptions.RequestException) as err:
             error_string = "JSON Query to {}: {}".format(path, err)
             if _attempt == self.max_connection_attempts:
-                raise ConnectionException(error_string) from err
+                if isinstance(err, QueryReturnedNotFoundException):
+                    raise QueryReturnedNotFoundException(error_string) from err
+                else:
+                    raise ConnectionException(error_string) from err
             self.error(error_string + " [retrying; skip with ^C]", repeat_at_end=False)
-            text_for_429 = ("HTTP error code 429 was returned because too many queries occured in the last time. "
-                            "Please do not use Instagram in your browser or run multiple instances of Instaloader "
-                            "in parallel.")
             try:
-                if isinstance(err, TooManyRequestsException):
-                    print(textwrap.fill(text_for_429), file=sys.stderr)
-                    if is_graphql_query:
-                        waittime = graphql_query_waittime(untracked_queries=True)
-                        if waittime > 0:
-                            self.log('The request will be retried in {} seconds, at {:%H:%M}.'
-                                     .format(waittime, datetime.now() + timedelta(seconds=waittime)))
-                            time.sleep(waittime)
-                self._sleep()
+                if is_graphql_query and isinstance(err, TooManyRequestsException):
+                    self._ratecontrol_graphql_query(params['query_hash'], untracked_queries=True)
+                if is_iphone_query and isinstance(err, TooManyRequestsException):
+                    self._ratecontrol_graphql_query('iphone', untracked_queries=True)
+                if is_other_query and isinstance(err, TooManyRequestsException):
+                    self._ratecontrol_graphql_query('other', untracked_queries=True)
                 return self.get_json(path=path, params=params, host=host, session=sess, _attempt=_attempt + 1)
             except KeyboardInterrupt:
                 self.error("[skipped by user]", repeat_at_end=False)
@@ -328,7 +464,7 @@ class InstaloaderContext:
         :param rhx_gis: 'rhx_gis' variable as somewhere returned by Instagram, needed to 'sign' request
         :return: The server's response dictionary.
         """
-        with copy_session(self._session) as tmpsession:
+        with copy_session(self._session, self.request_timeout) as tmpsession:
             tmpsession.headers.update(self._default_http_header(empty_session_only=True))
             del tmpsession.headers['Connection']
             del tmpsession.headers['Content-Length']
@@ -385,39 +521,64 @@ class InstaloaderContext:
             data = _query()
             yield from (edge['node'] for edge in data['edges'])
 
-    def get_and_write_raw(self, url: str, filename: str, _attempt=1) -> None:
-        """Downloads raw data.
+    def get_iphone_json(self, path: str, params: Dict[str, Any]) -> Dict[str, Any]:
+        """JSON request to ``i.instagram.com``.
+
+        :param path: URL, relative to ``i.instagram.com/``
+        :param params: GET parameters
+        :return: Decoded response dictionary
+        :raises QueryReturnedBadRequestException: When the server responds with a 400.
+        :raises QueryReturnedNotFoundException: When the server responds with a 404.
+        :raises ConnectionException: When query repeatedly failed.
+
+        .. versionadded:: 4.2.1"""
+        with copy_session(self._session, self.request_timeout) as tempsession:
+            tempsession.headers['User-Agent'] = 'Instagram 10.3.2 (iPhone7,2; iPhone OS 9_3_3; en_US; en-US; ' \
+                                                'scale=2.00; 750x1334) AppleWebKit/420+'
+            for header in ['Host', 'Origin', 'X-Instagram-AJAX', 'X-Requested-With']:
+                tempsession.headers.pop(header, None)
+            return self.get_json(path, params, 'i.instagram.com', tempsession)
+
+    def write_raw(self, resp: Union[bytes, requests.Response], filename: str) -> None:
+        """Write raw response data into a file.
+
+        .. versionadded:: 4.2.1"""
+        self.log(filename, end=' ', flush=True)
+        with open(filename, 'wb') as file:
+            if isinstance(resp, requests.Response):
+                shutil.copyfileobj(resp.raw, file)
+            else:
+                file.write(resp)
+
+    def get_raw(self, url: str, _attempt=1) -> requests.Response:
+        """Downloads a file anonymously.
+
+        :raises QueryReturnedNotFoundException: When the server responds with a 404.
+        :raises QueryReturnedForbiddenException: When the server responds with a 403.
+        :raises ConnectionException: When download failed.
+
+        .. versionadded:: 4.2.1"""
+        with self.get_anonymous_session() as anonymous_session:
+            resp = anonymous_session.get(url, stream=True)
+        if resp.status_code == 200:
+            resp.raw.decode_content = True
+            return resp
+        else:
+            if resp.status_code == 403:
+                # suspected invalid URL signature
+                raise QueryReturnedForbiddenException("403 when accessing {}.".format(url))
+            if resp.status_code == 404:
+                # 404 not worth retrying.
+                raise QueryReturnedNotFoundException("404 when accessing {}.".format(url))
+            raise ConnectionException("HTTP error code {}.".format(resp.status_code))
+
+    def get_and_write_raw(self, url: str, filename: str) -> None:
+        """Downloads and writes anonymously-requested raw data into a file.
 
         :raises QueryReturnedNotFoundException: When the server responds with a 404.
         :raises QueryReturnedForbiddenException: When the server responds with a 403.
         :raises ConnectionException: When download repeatedly failed."""
-        try:
-            with self.get_anonymous_session() as anonymous_session:
-                resp = anonymous_session.get(url, stream=True)
-            if resp.status_code == 200:
-                self.log(filename, end=' ', flush=True)
-                with open(filename, 'wb') as file:
-                    resp.raw.decode_content = True
-                    shutil.copyfileobj(resp.raw, file)
-            else:
-                if resp.status_code == 403:
-                    # suspected invalid URL signature
-                    raise QueryReturnedForbiddenException("403 when accessing {}.".format(url))
-                if resp.status_code == 404:
-                    # 404 not worth retrying.
-                    raise QueryReturnedNotFoundException("404 when accessing {}.".format(url))
-                raise ConnectionException("HTTP error code {}.".format(resp.status_code))
-        except (urllib3.exceptions.HTTPError, requests.exceptions.RequestException, ConnectionException) as err:
-            error_string = "URL {}: {}".format(url, err)
-            if _attempt == self.max_connection_attempts:
-                raise ConnectionException(error_string) from err
-            self.error(error_string + " [retrying; skip with ^C]", repeat_at_end=False)
-            try:
-                self._sleep()
-                self.get_and_write_raw(url, filename, _attempt + 1)
-            except KeyboardInterrupt:
-                self.error("[skipped by user]", repeat_at_end=False)
-                raise ConnectionException(error_string) from err
+        self.write_raw(self.get_raw(url), filename)
 
     @property
     def root_rhx_gis(self) -> Optional[str]:
@@ -426,6 +587,6 @@ class InstaloaderContext:
             # At the moment, rhx_gis seems to be required for anonymous requests only. By returning None when logged
             # in, we can save the root_rhx_gis lookup query.
             return None
-        if not self._root_rhx_gis:
-            self._root_rhx_gis = self.get_json('', {})['rhx_gis']
-        return self._root_rhx_gis
+        if self._root_rhx_gis is None:
+            self._root_rhx_gis = self.get_json('', {}).get('rhx_gis', '')
+        return self._root_rhx_gis or None

instaloader/structures.py

@@ -17,11 +17,17 @@ PostSidecarNode.is_video.__doc__ = "Whether this node is a video."
 PostSidecarNode.display_url.__doc__ = "URL of image or video thumbnail."
 PostSidecarNode.video_url.__doc__ = "URL of video or None."
 
-PostComment = namedtuple('PostComment', ['id', 'created_at_utc', 'text', 'owner'])
-PostComment.id.__doc__ = "ID number of comment."
-PostComment.created_at_utc.__doc__ = ":class:`~datetime.datetime` when comment was created (UTC)."
-PostComment.text.__doc__ = "Comment text."
-PostComment.owner.__doc__ = "Owner :class:`Profile` of the comment."
+PostCommentAnswer = namedtuple('PostCommentAnswer', ['id', 'created_at_utc', 'text', 'owner', 'likes_count'])
+PostCommentAnswer.id.__doc__ = "ID number of comment."
+PostCommentAnswer.created_at_utc.__doc__ = ":class:`~datetime.datetime` when comment was created (UTC)."
+PostCommentAnswer.text.__doc__ = "Comment text."
+PostCommentAnswer.owner.__doc__ = "Owner :class:`Profile` of the comment."
+PostCommentAnswer.likes_count.__doc__ = "Number of likes on comment."
+
+PostComment = namedtuple('PostComment', (*PostCommentAnswer._fields, 'answers')) # type: ignore
+for field in PostCommentAnswer._fields:
+    getattr(PostComment, field).__doc__ = getattr(PostCommentAnswer, field).__doc__
+PostComment.answers.__doc__ = r"Iterator which yields all :class:`PostCommentAnswer`\ s for the comment." # type: ignore
 
 PostLocation = namedtuple('PostLocation', ['id', 'name', 'slug', 'has_public_page', 'lat', 'lng'])
 PostLocation.id.__doc__ = "ID number of location."
@@ -62,9 +68,9 @@ class Post:
         self._context = context
         self._node = node
         self._owner_profile = owner_profile
-        self._full_metadata_dict = None
-        self._rhx_gis_str = None
-        self._location = None
+        self._full_metadata_dict = None  # type: Optional[Dict[str, Any]]
+        self._rhx_gis_str = None         # type: Optional[str]
+        self._location = None            # type: Optional[PostLocation]
 
     @classmethod
     def from_shortcode(cls, context: InstaloaderContext, shortcode: str):
@@ -92,6 +98,11 @@ class Post:
             raise InvalidArgumentException("Wrong mediaid {0}, unable to convert to shortcode".format(str(mediaid)))
         return b64encode(mediaid.to_bytes(9, 'big'), b'-_').decode().replace('A', ' ').lstrip().replace(' ', 'A')
 
+    @staticmethod
+    def supported_graphql_types() -> List[str]:
+        """The values of __typename fields that the :class:`Post` class can handle."""
+        return ["GraphImage", "GraphVideo", "GraphSidecar"]
+
     def _asdict(self):
         node = self._node
         if self._full_metadata_dict:
@@ -127,15 +138,25 @@ class Post:
         if not self._full_metadata_dict:
             pic_json = self._context.get_json("p/{0}/".format(self.shortcode), params={})
             self._full_metadata_dict = pic_json['entry_data']['PostPage'][0]['graphql']['shortcode_media']
-            self._rhx_gis_str = pic_json['rhx_gis']
+            self._rhx_gis_str = pic_json.get('rhx_gis')
+            if self._full_metadata_dict is None:
+                # issue #449
+                self._context.error("Fetching Post metadata failed (issue #449). "
+                                    "The following data has been returned:\n"
+                                    + json.dumps(pic_json['entry_data'], indent=2))
+                raise BadResponseException("Fetching Post metadata failed.")
+            if self.shortcode != self._full_metadata_dict['shortcode']:
+                self._node.update(self._full_metadata_dict)
+                raise PostChangedException
 
     @property
     def _full_metadata(self) -> Dict[str, Any]:
         self._obtain_metadata()
+        assert self._full_metadata_dict is not None
         return self._full_metadata_dict
 
     @property
-    def _rhx_gis(self) -> str:
+    def _rhx_gis(self) -> Optional[str]:
         self._obtain_metadata()
         return self._rhx_gis_str
 
@@ -180,12 +201,16 @@ class Post:
     @property
     def date_local(self) -> datetime:
         """Timestamp when the post was created (local time zone)."""
-        return datetime.fromtimestamp(self._node["date"] if "date" in self._node else self._node["taken_at_timestamp"])
+        return datetime.fromtimestamp(self._node["date"]
+                                      if "date" in self._node
+                                      else self._node["taken_at_timestamp"])
 
     @property
     def date_utc(self) -> datetime:
         """Timestamp when the post was created (UTC)."""
-        return datetime.utcfromtimestamp(self._node["date"] if "date" in self._node else self._node["taken_at_timestamp"])
+        return datetime.utcfromtimestamp(self._node["date"]
+                                         if "date" in self._node
+                                         else self._node["taken_at_timestamp"])
 
     @property
     def date(self) -> datetime:
@@ -210,7 +235,11 @@ class Post:
     def get_sidecar_nodes(self) -> Iterator[PostSidecarNode]:
         """Sidecar nodes of a Post with typename==GraphSidecar."""
         if self.typename == 'GraphSidecar':
-            for edge in self._field('edge_sidecar_to_children', 'edges'):
+            edges = self._field('edge_sidecar_to_children', 'edges')
+            if any(edge['node']['is_video'] for edge in edges):
+                # video_url is only present in full metadata, issue #558.
+                edges = self._full_metadata['edge_sidecar_to_children']['edges']
+            for edge in edges:
                 node = edge['node']
                 is_video = node['is_video']
                 yield PostSidecarNode(is_video=is_video, display_url=node['display_url'],
@@ -223,6 +252,7 @@ class Post:
             return self._node["edge_media_to_caption"]["edges"][0]["node"]["text"]
         elif "caption" in self._node:
             return self._node["caption"]
+        return None
 
     @property
     def caption_hashtags(self) -> List[str]:
@@ -244,6 +274,16 @@ class Post:
         mention_regex = re.compile(r"(?:@)(\w(?:(?:\w|(?:\.(?!\.))){0,28}(?:\w))?)")
         return re.findall(mention_regex, self.caption.lower())
 
+    @property
+    def pcaption(self) -> str:
+        """Printable caption, useful as a format specifier for --filename-pattern.
+
+        .. versionadded:: 4.2.6"""
+        def _elliptify(caption):
+            pcaption = ' '.join([s.replace('/', '\u2215') for s in caption.splitlines() if s]).strip()
+            return (pcaption[:30] + u"\u2026") if len(pcaption) > 31 else pcaption
+        return _elliptify(self.caption) if self.caption else ''
+
     @property
     def tagged_users(self) -> List[str]:
         """List of all lowercased users that are tagged in the Post."""
@@ -263,6 +303,25 @@ class Post:
         """URL of the video, or None."""
         if self.is_video:
             return self._field('video_url')
+        return None
+
+    @property
+    def video_view_count(self) -> Optional[int]:
+        """View count of the video, or None.
+
+        .. versionadded:: 4.2.6"""
+        if self.is_video:
+            return self._field('video_view_count')
+        return None
+
+    @property
+    def video_duration(self) -> Optional[float]:
+        """Duration of the video in seconds, or None.
+
+        .. versionadded:: 4.2.6"""
+        if self.is_video:
+            return self._field('video_duration')
+        return None
 
     @property
     def viewer_has_liked(self) -> Optional[bool]:
@@ -280,34 +339,78 @@ class Post:
 
     @property
     def comments(self) -> int:
-        """Comment count"""
-        return self._field('edge_media_to_comment', 'count')
+        """Comment count including answers"""
+        # If the count is already present in `self._node`, do not use `self._field` which could trigger fetching the
+        # full metadata dict.
+        comments = self._node.get('edge_media_to_comment')
+        if comments and 'count' in comments:
+            return comments['count']
+        try:
+            return self._field('edge_media_to_parent_comment', 'count')
+        except KeyError:
+            return self._field('edge_media_to_comment', 'count')
 
     def get_comments(self) -> Iterator[PostComment]:
-        """Iterate over all comments of the post.
+        r"""Iterate over all comments of the post.
 
         Each comment is represented by a PostComment namedtuple with fields text (string), created_at (datetime),
-        id (int) and owner (:class:`Profile`).
+        id (int), owner (:class:`Profile`) and answers (:class:`~typing.Iterator`\ [:class:`PostCommentAnswer`])
+        if available.
         """
+        def _postcommentanswer(node):
+            return PostCommentAnswer(id=int(node['id']),
+                                     created_at_utc=datetime.utcfromtimestamp(node['created_at']),
+                                     text=node['text'],
+                                     owner=Profile(self._context, node['owner']),
+                                     likes_count=node['edge_liked_by']['count'])
+
+        def _postcommentanswers(node):
+            if 'edge_threaded_comments' not in node:
+                return
+            answer_count = node['edge_threaded_comments']['count']
+            if answer_count == 0:
+                # Avoid doing additional requests if there are no comment answers
+                return
+            answer_edges = node['edge_threaded_comments']['edges']
+            if answer_count == len(answer_edges):
+                # If the answer's metadata already contains all comments, don't do GraphQL requests to obtain them
+                yield from (_postcommentanswer(comment['node']) for comment in answer_edges)
+                return
+            yield from (_postcommentanswer(answer_node) for answer_node in
+                        self._context.graphql_node_list("51fdd02b67508306ad4484ff574a0b62",
+                                                        {'comment_id': node['id']},
+                                                        'https://www.instagram.com/p/' + self.shortcode + '/',
+                                                        lambda d: d['data']['comment']['edge_threaded_comments']))
+
         def _postcomment(node):
-            return PostComment(id=int(node['id']),
-                               created_at_utc=datetime.utcfromtimestamp(node['created_at']),
-                               text=node['text'],
-                               owner=Profile(self._context, node['owner']))
+            return PostComment(*_postcommentanswer(node),
+                               answers=_postcommentanswers(node))
         if self.comments == 0:
             # Avoid doing additional requests if there are no comments
             return
-        comment_edges = self._field('edge_media_to_comment', 'edges')
-        if self.comments == len(comment_edges):
-            # If the Post's metadata already contains all comments, don't do GraphQL requests to obtain them
+        try:
+            comment_edges = self._field('edge_media_to_parent_comment', 'edges')
+            answers_count = sum([edge['node']['edge_threaded_comments']['count'] for edge in comment_edges])
+            threaded_comments_available = True
+        except KeyError:
+            comment_edges = self._field('edge_media_to_comment', 'edges')
+            answers_count = 0
+            threaded_comments_available = False
+
+        if self.comments == len(comment_edges) + answers_count:
+            # If the Post's metadata already contains all parent comments, don't do GraphQL requests to obtain them
             yield from (_postcomment(comment['node']) for comment in comment_edges)
             return
         yield from (_postcomment(node) for node in
-                    self._context.graphql_node_list("33ba35852cb50da46f5b5e889df7d159",
-                                                    {'shortcode': self.shortcode},
-                                                    'https://www.instagram.com/p/' + self.shortcode + '/',
-                                                    lambda d: d['data']['shortcode_media']['edge_media_to_comment'],
-                                                    self._rhx_gis))
+                    self._context.graphql_node_list(
+                        "97b41c52301f77ce508f55e66d17620e" if threaded_comments_available
+                        else "f0986789a5c5d17c2400faebf16efd0d",
+                        {'shortcode': self.shortcode},
+                        'https://www.instagram.com/p/' + self.shortcode + '/',
+                        lambda d:
+                        d['data']['shortcode_media'][
+                            'edge_media_to_parent_comment' if threaded_comments_available else 'edge_media_to_comment'],
+                        self._rhx_gis))
 
     def get_likes(self) -> Iterator['Profile']:
         """Iterate over all likes of the post. A :class:`Profile` instance of each likee is yielded."""
@@ -327,10 +430,17 @@ class Post:
 
     @property
     def location(self) -> Optional[PostLocation]:
-        """If the Post has a location, returns PostLocation namedtuple with fields 'id', 'lat' and 'lng' and 'name'."""
+        """
+        If the Post has a location, returns PostLocation namedtuple with fields 'id', 'lat' and 'lng' and 'name'.
+
+        .. versionchanged:: 4.2.9
+           Require being logged in (as required by Instagram), return None if not logged-in.
+        """
         loc = self._field("location")
         if self._location or not loc:
             return self._location
+        if not self._context.is_logged_in:
+            return None
         location_id = int(loc['id'])
         if any(k not in loc for k in ('name', 'slug', 'has_public_page', 'lat', 'lng')):
             loc = self._context.get_json("explore/locations/{0}/".format(location_id),
@@ -370,8 +480,9 @@ class Profile:
     def __init__(self, context: InstaloaderContext, node: Dict[str, Any]):
         assert 'username' in node
         self._context = context
-        self._has_public_story = None
+        self._has_public_story = None  # type: Optional[bool]
         self._node = node
+        self._has_full_metadata = False
         self._rhx_gis = None
         self._iphone_struct_ = None
         if 'iphone_struct' in node:
@@ -396,48 +507,56 @@ class Profile:
     @classmethod
     def from_id(cls, context: InstaloaderContext, profile_id: int):
         """Create a Profile instance from a given userid. If possible, use :meth:`Profile.from_username`
-        or constructor directly rather than this method, since it does many requests.
+        or constructor directly rather than this method, since it requires more requests.
 
         :param context: :attr:`Instaloader.context`
         :param profile_id: userid
-        :raises: :class:`ProfileNotExistsException`, :class:`ProfileHasNoPicsException`
+        :raises: :class:`ProfileNotExistsException`
         """
         if profile_id in context.profile_id_cache:
             return context.profile_id_cache[profile_id]
-        data = context.graphql_query("472f257a40c653c64c666ce877d59d2b",
-                                     {'id': str(profile_id), 'first': 1},
+        data = context.graphql_query('7c16654f22c819fb63d1183034a5162f',
+                                     {'user_id': str(profile_id),
+                                      'include_chaining': False,
+                                      'include_reel': True,
+                                      'include_suggested_users': False,
+                                      'include_logged_out_extras': False,
+                                      'include_highlight_reels': False},
                                      rhx_gis=context.root_rhx_gis)['data']['user']
         if data:
-            data = data["edge_owner_to_timeline_media"]
+            profile = cls(context, data['reel']['owner'])
         else:
             raise ProfileNotExistsException("No profile found, the user may have blocked you (ID: " +
                                             str(profile_id) + ").")
-        if not data['edges']:
-            if data['count'] == 0:
-                raise ProfileHasNoPicsException("Profile with ID {0}: no pics found.".format(str(profile_id)))
-            else:
-                raise LoginRequiredException("Login required to determine username (ID: " + str(profile_id) + ").")
-        profile = Post(context, data['edges'][0]['node']).owner_profile
         context.profile_id_cache[profile_id] = profile
         return profile
 
     def _asdict(self):
         json_node = self._node.copy()
-        # remove posts
+        # remove posts to avoid "Circular reference detected" exception
         json_node.pop('edge_media_collections', None)
         json_node.pop('edge_owner_to_timeline_media', None)
         json_node.pop('edge_saved_media', None)
+        json_node.pop('edge_felix_video_timeline', None)
         if self._iphone_struct_:
             json_node['iphone_struct'] = self._iphone_struct_
         return json_node
 
     def _obtain_metadata(self):
         try:
-            if not self._rhx_gis:
+            if not self._has_full_metadata:
                 metadata = self._context.get_json('{}/'.format(self.username), params={})
                 self._node = metadata['entry_data']['ProfilePage'][0]['graphql']['user']
-                self._rhx_gis = metadata['rhx_gis']
+                self._has_full_metadata = True
+                self._rhx_gis = metadata.get('rhx_gis')
         except (QueryReturnedNotFoundException, KeyError) as err:
+            top_search_results = TopSearchResults(self._context, self.username)
+            similar_profiles = [profile.username for profile in top_search_results.get_profiles()]
+            if similar_profiles:
+                raise ProfileNotExistsException('Profile {} does not exist.\nThe most similar profile{}: {}.'
+                                                .format(self.username,
+                                                        's are' if len(similar_profiles) > 1 else ' is',
+                                                        ', '.join(similar_profiles[0:5]))) from err
             raise ProfileNotExistsException('Profile {} does not exist.'.format(self.username)) from err
 
     def _metadata(self, *keys) -> Any:
@@ -455,10 +574,10 @@ class Profile:
 
     @property
     def _iphone_struct(self) -> Dict[str, Any]:
+        if not self._context.is_logged_in:
+            raise LoginRequiredException("--login required to access iPhone profile info endpoint.")
         if not self._iphone_struct_:
-            with self._context.anonymous_copy() as anonymous_context:
-                data = anonymous_context.get_json(path='api/v1/users/{}/info/'.format(self.userid),
-                                                  params={}, host='i.instagram.com')
+            data = self._context.get_iphone_json(path='api/v1/users/{}/info/'.format(self.userid), params={})
             self._iphone_struct_ = data['user']
         return self._iphone_struct_
 
@@ -495,6 +614,10 @@ class Profile:
     def mediacount(self) -> int:
         return self._metadata('edge_owner_to_timeline_media', 'count')
 
+    @property
+    def igtvcount(self) -> int:
+        return self._metadata('edge_felix_video_timeline', 'count')
+
     @property
     def followers(self) -> int:
         return self._metadata('edge_followed_by', 'count')
@@ -551,6 +674,7 @@ class Profile:
                                                        'https://www.instagram.com/{}/'.format(self.username),
                                                        self._rhx_gis)
             self._has_public_story = data['data']['user']['has_public_story']
+        assert self._has_public_story is not None
         return self._has_public_story
 
     @property
@@ -578,13 +702,19 @@ class Profile:
 
     @property
     def profile_pic_url(self) -> str:
-        """Return URL of profile picture
+        """Return URL of profile picture. If logged in, the HD version is returned, otherwise a lower-quality version.
 
-        .. versionadded:: 4.0.3"""
-        try:
-            return self._iphone_struct['hd_profile_pic_url_info']['url']
-        except (InstaloaderException, KeyError) as err:
-            self._context.error('{} Unable to fetch high quality profile pic.'.format(err))
+        .. versionadded:: 4.0.3
+
+        .. versionchanged:: 4.2.1
+           Require being logged in for HD version (as required by Instagram)."""
+        if self._context.is_logged_in:
+            try:
+                return self._iphone_struct['hd_profile_pic_url_info']['url']
+            except (InstaloaderException, KeyError) as err:
+                self._context.error('{} Unable to fetch high quality profile pic.'.format(err))
+                return self._metadata("profile_pic_url_hd")
+        else:
             return self._metadata("profile_pic_url_hd")
 
     def get_profile_pic_url(self) -> str:
@@ -631,6 +761,19 @@ class Profile:
                                                     lambda d: d['data']['user']['edge_user_to_photos_of_you'],
                                                     self._rhx_gis))
 
+    def get_igtv_posts(self) -> Iterator[Post]:
+        """Retrieve all IGTV posts.
+
+        .. versionadded:: 4.3"""
+        self._obtain_metadata()
+        yield from (Post(self._context, node, self) for node in
+                    self._context.graphql_node_list('bc78b344a68ed16dd5d7f264681c4c76',
+                                                    {'id': self.userid},
+                                                    'https://www.instagram.com/{0}/channel/'.format(self.username),
+                                                    lambda d: d['data']['user']['edge_felix_video_timeline'],
+                                                    self._rhx_gis,
+                                                    self._metadata('edge_felix_video_timeline')))
+
     def get_followers(self) -> Iterator['Profile']:
         """
         Retrieve list of followers of given profile.
@@ -711,6 +854,7 @@ class StoryItem:
         """:class:`Profile` instance of the story item's owner."""
         if not self._owner_profile:
             self._owner_profile = Profile.from_id(self._context, self._node['owner']['id'])
+        assert self._owner_profile is not None
         return self._owner_profile
 
     @property
@@ -773,6 +917,7 @@ class StoryItem:
         """URL of the video, or None."""
         if self.is_video:
             return self._node['video_resources'][-1]['src']
+        return None
 
 
 class Story:
@@ -799,8 +944,8 @@ class Story:
     def __init__(self, context: InstaloaderContext, node: Dict[str, Any]):
         self._context = context
         self._node = node
-        self._unique_id = None
-        self._owner_profile = None
+        self._unique_id = None      # type: Optional[str]
+        self._owner_profile = None  # type: Optional[Profile]
 
     def __repr__(self):
         return '<Story by {} changed {:%Y-%m-%d_%H-%M-%S_UTC}>'.format(self.owner_username, self.latest_media_utc)
@@ -814,7 +959,7 @@ class Story:
         return hash(self.unique_id)
 
     @property
-    def unique_id(self) -> str:
+    def unique_id(self) -> Union[str, int]:
         """
         This ID only equals amongst :class:`Story` instances which have the same owner and the same set of
         :class:`StoryItem`. For all other :class:`Story` instances this ID is different.
@@ -830,12 +975,14 @@ class Story:
         """Timestamp when the story has last been watched or None (local time zone)."""
         if self._node['seen']:
             return datetime.fromtimestamp(self._node['seen'])
+        return None
 
     @property
     def last_seen_utc(self) -> Optional[datetime]:
         """Timestamp when the story has last been watched or None (UTC)."""
         if self._node['seen']:
             return datetime.utcfromtimestamp(self._node['seen'])
+        return None
 
     @property
     def latest_media_local(self) -> datetime:
@@ -900,7 +1047,7 @@ class Highlight(Story):
     def __init__(self, context: InstaloaderContext, node: Dict[str, Any], owner: Optional[Profile] = None):
         super().__init__(context, node)
         self._owner_profile = owner
-        self._items = None
+        self._items = None  # type: Optional[List[Dict[str, Any]]]
 
     def __repr__(self):
         return '<Highlight by {}: {}>'.format(self.owner_username, self.title)
@@ -943,14 +1090,83 @@ class Highlight(Story):
     def itemcount(self) -> int:
         """Count of items associated with the :class:`Highlight` instance."""
         self._fetch_items()
+        assert self._items is not None
         return len(self._items)
 
     def get_items(self) -> Iterator[StoryItem]:
         """Retrieve all associated highlight items."""
         self._fetch_items()
+        assert self._items is not None
         yield from (StoryItem(self._context, item, self.owner_profile) for item in self._items)
 
 
+class TopSearchResults:
+    """
+    An invocation of this class triggers a search on Instagram for the provided search string.
+
+    Provides methods to access the search results as profiles (:class:`Profile`), locations (:class:`PostLocation`) and
+    hashtags.
+
+    :param context: :attr:`Instaloader.context` used to send the query for the search.
+    :param searchstring: String to search for with Instagram's "top search".
+    """
+
+    def __init__(self, context: InstaloaderContext, searchstring: str):
+        self._context = context
+        self._searchstring = searchstring
+        # The `__a` param is only needed to prevent `get_json()` from searching for 'window._sharedData'.
+        self._node = context.get_json('web/search/topsearch/',
+                                      params={'context': 'blended',
+                                              'query': searchstring,
+                                              'include_reel': False,
+                                              '__a': 1})
+
+    def get_profiles(self) -> Iterator[Profile]:
+        """
+        Provides the :class:`Profile` instances from the search result.
+        """
+        for user in self._node.get('users', []):
+            user_node = user['user']
+            if 'pk' in user_node:
+                user_node['id'] = user_node['pk']
+            yield Profile(self._context, user_node)
+
+    def get_prefixed_usernames(self) -> Iterator[str]:
+        """
+        Provides all profile names from the search result that start with the search string.
+        """
+        for user in self._node.get('users', []):
+            username = user.get('user', {}).get('username', '')
+            if username.startswith(self._searchstring):
+                yield username
+
+    def get_locations(self) -> Iterator[PostLocation]:
+        """
+        Provides instances of :class:`PostLocation` from the search result.
+        """
+        for location in self._node.get('places', []):
+            place = location.get('place', {})
+            slug = place.get('slug')
+            loc = place.get('location', {})
+            yield PostLocation(int(loc['pk']), loc['name'], slug, None, loc['lat'], loc['lng'])
+
+    def get_hashtag_strings(self) -> Iterator[str]:
+        """
+        Provides the hashtags from the search result as strings.
+        """
+        for hashtag in self._node.get('hashtags', []):
+            name = hashtag.get('hashtag', {}).get('name')
+            if name:
+                yield name
+
+    @property
+    def searchstring(self) -> str:
+        """
+        The string that was searched for on Instagram to produce this :class:`TopSearchResults` instance.
+        """
+        return self._searchstring
+
+
 JsonExportable = Union[Post, Profile, StoryItem]
 
 

setup.py

@@ -60,6 +60,7 @@ setup(
         'Programming Language :: Python :: 3.5',
         'Programming Language :: Python :: 3.6',
         'Programming Language :: Python :: 3.7',
+        'Programming Language :: Python :: 3.8',
         'Programming Language :: Python :: 3 :: Only',
         'Topic :: Internet',
         'Topic :: Multimedia :: Graphics'

test/instaloader_unittests.py

@@ -5,20 +5,26 @@ import shutil
 import tempfile
 import unittest
 from itertools import islice
+from typing import Dict, List
 
 import instaloader
 
 PROFILE_WITH_HIGHLIGHTS = 325732271
 PUBLIC_PROFILE = "selenagomez"
 PUBLIC_PROFILE_ID = 460563723
+PUBLIC_PROFILE_WITH_IGTV = "natgeo"
 HASHTAG = "kitten"
+LOCATION = "362629379"
 OWN_USERNAME = "aandergr"
 NORMAL_MAX_COUNT = 2
 PAGING_MAX_COUNT = 15
 PRIVATE_PROFILE = "aandergr"
+PRIVATE_PROFILE_ID = 1706625676
+EMPTY_PROFILE = "not_public"
+EMPTY_PROFILE_ID = 1928659031
 
 # Preserve query timestamps (rate control) between tests to not get rate limited
-instaloadercontext_query_timestamps = list()
+instaloadercontext_query_timestamps = dict()  # type: Dict[str, List[float]]
 
 
 class TestInstaloaderAnonymously(unittest.TestCase):
@@ -31,11 +37,13 @@ class TestInstaloaderAnonymously(unittest.TestCase):
                                          download_comments=True,
                                          save_metadata=True)
         self.L.context.raise_all_errors = True
-        self.L.context.query_timestamps = instaloadercontext_query_timestamps
+        # pylint:disable=protected-access
+        self.L.context._graphql_query_timestamps = instaloadercontext_query_timestamps.copy()
 
     def tearDown(self):
+        # pylint:disable=global-statement,protected-access
         global instaloadercontext_query_timestamps
-        instaloadercontext_query_timestamps = self.L.context.query_timestamps
+        instaloadercontext_query_timestamps = self.L.context._graphql_query_timestamps.copy()
         self.L.close()
         os.chdir('/')
         print("Removing {}".format(self.dir))
@@ -49,28 +57,43 @@ class TestInstaloaderAnonymously(unittest.TestCase):
                 self.assertTrue(post.date_utc < previous_post.date_utc)
             previous_post = post
 
-    @unittest.SkipTest
+    @unittest.skip('')
     def test_public_profile_download(self):
-        self.L.download_profile(PUBLIC_PROFILE, profile_pic=False, fast_update=True)
-        self.L.download_profile(PUBLIC_PROFILE, profile_pic=False, fast_update=True)
+        profiles = {self.L.check_profile_id(PUBLIC_PROFILE)}
+        self.L.download_profiles(profiles, profile_pic=False, fast_update=True, raise_errors=True)
+        self.L.download_profiles(profiles, profile_pic=False, fast_update=True, raise_errors=True)
 
     def test_public_profile_paging(self):
         self.post_paging_test(instaloader.Profile.from_username(self.L.context, PUBLIC_PROFILE).get_posts())
 
     def test_profile_pic_download(self):
-        self.L.download_profile(PUBLIC_PROFILE, profile_pic_only=True)
+        self.L.download_profiles({self.L.check_profile_id(PUBLIC_PROFILE)}, posts=False, raise_errors=True)
 
     def test_hashtag_download(self):
         self.L.download_hashtag(HASHTAG, NORMAL_MAX_COUNT)
 
+    def test_hashtag_paging(self):
+        for count, post in enumerate(self.L.get_hashtag_posts(HASHTAG)):
+            print(post)
+            if count == PAGING_MAX_COUNT:
+                break
+
     def test_get_id_by_username(self):
         self.assertEqual(PUBLIC_PROFILE_ID,
                          instaloader.Profile.from_username(self.L.context, PUBLIC_PROFILE).userid)
 
-    def test_get_username_by_id(self):
+    def test_get_username_by_id_private(self):
+        self.assertEqual(PRIVATE_PROFILE.lower(),
+                         instaloader.Profile.from_id(self.L.context, PRIVATE_PROFILE_ID).username)
+
+    def test_get_username_by_id_public(self):
         self.assertEqual(PUBLIC_PROFILE.lower(),
                          instaloader.Profile.from_id(self.L.context, PUBLIC_PROFILE_ID).username)
 
+    def test_get_username_by_id_empty(self):
+        self.assertEqual(EMPTY_PROFILE.lower(),
+                         instaloader.Profile.from_id(self.L.context, EMPTY_PROFILE_ID).username)
+
     def test_post_from_mediaid(self):
         for post in instaloader.Profile.from_username(self.L.context, PUBLIC_PROFILE).get_posts():
             post2 = instaloader.Post.from_mediaid(self.L.context, post.mediaid)
@@ -82,6 +105,11 @@ class TestInstaloaderAnonymously(unittest.TestCase):
                            PAGING_MAX_COUNT):
             print(post)
 
+    def test_public_profile_igtv(self):
+        for post in islice(instaloader.Profile.from_username(self.L.context, PUBLIC_PROFILE_WITH_IGTV).get_igtv_posts(),
+                           PAGING_MAX_COUNT):
+            print(post)
+
 
 class TestInstaloaderLoggedIn(TestInstaloaderAnonymously):
 
@@ -89,13 +117,13 @@ class TestInstaloaderLoggedIn(TestInstaloaderAnonymously):
         super().setUp()
         self.L.load_session_from_file(OWN_USERNAME)
 
-    @unittest.SkipTest
+    @unittest.skip('')
     def test_stories_download(self):
         self.L.download_stories()
 
-    @unittest.SkipTest
+    @unittest.skip('')
     def test_private_profile_download(self):
-        self.L.download_profile(PRIVATE_PROFILE, download_stories=True)
+        self.L.download_profiles({self.L.check_profile_id(PRIVATE_PROFILE)}, stories=True, raise_errors=True)
 
     def test_stories_paging(self):
         for user_story in self.L.get_stories():
@@ -137,8 +165,8 @@ class TestInstaloaderLoggedIn(TestInstaloaderAnonymously):
 
     def test_followees_and_stories(self):
         profile = instaloader.Profile.from_username(self.L.context, OWN_USERNAME)
-        for f in islice(profile.get_followees(), PAGING_MAX_COUNT):
-            self.L.download_profile(f.username, profile_pic=False, download_stories_only=True)
+        followees = set(islice(profile.get_followees(), PAGING_MAX_COUNT))
+        self.L.download_profiles(followees, profile_pic=False, stories=True, posts=False, raise_errors=True)
 
     def test_get_followees(self):
         profile = instaloader.Profile.from_username(self.L.context, OWN_USERNAME)
@@ -150,10 +178,6 @@ class TestInstaloaderLoggedIn(TestInstaloaderAnonymously):
         for f in profile.get_followers():
             print(f.username)
 
-    def test_get_username_by_id(self):
-        self.assertEqual(PUBLIC_PROFILE.lower(),
-                         instaloader.Profile.from_id(self.L.context, PUBLIC_PROFILE_ID).username)
-
     def test_get_likes(self):
         for post in instaloader.Profile.from_username(self.L.context, OWN_USERNAME).get_posts():
             for like in post.get_likes():
@@ -166,6 +190,15 @@ class TestInstaloaderLoggedIn(TestInstaloaderAnonymously):
             if count == PAGING_MAX_COUNT:
                 break
 
+    def test_location_download(self):
+        self.L.download_location(LOCATION, NORMAL_MAX_COUNT)
+
+    def test_location_paging(self):
+        for count, post in enumerate(self.L.get_location_posts(LOCATION)):
+            print(post)
+            if count == PAGING_MAX_COUNT:
+                break
+
 
 if __name__ == '__main__':
     unittest.main()