Release of Version 4.4.2

* Workaround for Instagram's encrypted login.

Credit: https://github.com/pgrimaud/
See: https://github.com/pgrimaud/instagram-user-feed/commit/96ad4cf54d1ad331b337f325c73e664999a6d066

Closes #615.

Co-authored-by: Alexander Graf <17130992+aandergr@users.noreply.github.com>

.github/ISSUE_TEMPLATE/bug_report.md

@@ -0,0 +1,27 @@
+---
+name: 🐛 Bug report
+about: Create a report to help us improve 🤕
+labels: bug
+
+---
+
+**Describe the bug**
+A clear and concise description of what the bug is.
+
+**To Reproduce**
+Steps to reproduce the behavior:
+(e.g. Instaloader command line)
+
+**Expected behavior**
+A clear and concise description of what you expected to happen (if not obvious).
+
+**Error messages and tracebacks**
+If applicable, add error messages and tracebacks to help explain your problem.
+
+**Instaloader version**
+`instaloader --version` output.
+
+**Additional context**
+Add any other context about the problem here.
+
+(please also see https://instaloader.github.io/contributing.html for how to report a bug)

.github/ISSUE_TEMPLATE/feature_request.md

@@ -0,0 +1,24 @@
+---
+name: 🚀 Feature Suggestion
+about: Suggest an idea for this project 💡
+labels: "feature suggestion"
+
+---
+
+**Provide us a use case of the feature**
+How could the user invoke the new function? Which problem would it solve? If new information is obtained, how would it be further processed?
+
+**Describe the solution you'd like**
+A clear and concise description of what you want to happen.
+
+**Describe alternatives you've considered**
+A clear and concise description of any alternative solutions or features you've considered.
+
+**If the feature request is accepted, would you be willing to submit a PR?**
+
+Yes / No _(Help can be provided if you need assistance submitting a PR)_
+
+**Additional context**
+Add any other context about the feature request here.
+
+(please also see https://instaloader.github.io/contributing.html for how to suggest a feature)

.github/ISSUE_TEMPLATE/question.md

@@ -0,0 +1,8 @@
+---
+name: ❓ Question
+about: If you have a question about how to use Instaloader that's not well covered by our documentation
+labels: question
+
+---
+
+Your question here...

.github/workflows/stale.yml

@@ -0,0 +1,24 @@
+name: Mark stale issues and pull requests
+
+on:
+  schedule:
+  - cron: "15 16 * * *"
+
+jobs:
+  stale:
+
+    runs-on: ubuntu-latest
+    
+    steps:
+    - uses: actions/stale@v1
+      with:
+        repo-token: ${{ secrets.GITHUB_TOKEN }}
+        stale-issue-message: 'There has been no activity on this issue for an extended period of time. This issue will be closed after further 14 days of inactivity.'
+        stale-pr-message: 'There has been no activity on this Pull Request for an extended period of time. This Pull Request will be closed after further 14 days of inactivity.'
+        stale-issue-label: 'stale'
+        stale-pr-label: 'stale'
+        exempt-issue-label: 'leave open'
+        exempt-pr-label: 'leave open'
+        days-before-stale: 90
+        days-before-close: 14
+        operations-per-run: 7

.github/workflows/windows_exe.yml

@@ -0,0 +1,47 @@
+name: Windows EXE build
+
+on: [push, pull_request]
+
+jobs:
+  build:
+    runs-on: windows-latest
+    steps:
+      - name: Checkout Instaloader repository
+        uses: actions/checkout@v2
+      - name: Setup Python
+        uses: actions/setup-python@v1
+        with:
+          python-version: 3.8
+          architecture: x64
+      - name: Get the tagged version
+        id: get_version
+        env:
+          GITHUB_REF: ${{ github.ref }}
+        run: echo ::set-output name=VERSION::${GITHUB_REF/refs\/tags\/v/}
+        shell: bash
+      - name: Run custom python script for EXE creation
+        env:
+          VERSION_TAG: v${{ steps.get_version.outputs.VERSION }}
+        run: python deploy/windows/create_exe.py
+      - name: Create draft release
+        if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v')
+        id: create_release
+        uses: actions/create-release@v1
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        with:
+          tag_name: v${{ steps.get_version.outputs.VERSION }}
+          release_name: Version ${{ steps.get_version.outputs.VERSION }}
+          draft: true
+          prerelease: contains(steps.get_version.outputs.VERSION, 'a') || contains(steps.get_version.outputs.VERSION, 'b') || contains(steps.get_version.outputs.VERSION, 'c')
+      - name: Upload EXE as release asset
+        if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v')
+        id: upload-release-asset
+        uses: actions/upload-release-asset@v1
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        with:
+          upload_url: ${{ steps.create_release.outputs.upload_url }}
+          asset_path: ./instaloader-v${{ steps.get_version.outputs.VERSION }}-windows-standalone.zip
+          asset_name: instaloader-v${{ steps.get_version.outputs.VERSION }}-windows-standalone.zip
+          asset_content_type: application/zip

.gitignore

@@ -1 +1,2 @@
 __pycache__/
+.mypy_cache/

.pylintrc

@@ -0,0 +1,584 @@
+[MASTER]
+
+# A comma-separated list of package or module names from where C extensions may
+# be loaded. Extensions are loading into the active Python interpreter and may
+# run arbitrary code.
+extension-pkg-whitelist=
+
+# Add files or directories to the blacklist. They should be base names, not
+# paths.
+ignore=CVS
+
+# Add files or directories matching the regex patterns to the blacklist. The
+# regex matches against base names, not paths.
+ignore-patterns=
+
+# Python code to execute, usually for sys.path manipulation such as
+# pygtk.require().
+#init-hook=
+
+# Use multiple processes to speed up Pylint. Specifying 0 will auto-detect the
+# number of processors available to use.
+jobs=0
+
+# Control the amount of potential inferred values when inferring a single
+# object. This can help the performance when dealing with large functions or
+# complex, nested conditions.
+limit-inference-results=100
+
+# List of plugins (as comma separated values of python modules names) to load,
+# usually to register additional checkers.
+load-plugins=
+
+# Pickle collected data for later comparisons.
+persistent=yes
+
+# Specify a configuration file.
+#rcfile=
+
+# When enabled, pylint would attempt to guess common misconfiguration and emit
+# user-friendly hints instead of false-positive error messages.
+suggestion-mode=yes
+
+# Allow loading of arbitrary C extensions. Extensions are imported into the
+# active Python interpreter and may run arbitrary code.
+unsafe-load-any-extension=no
+
+
+[MESSAGES CONTROL]
+
+# Only show warnings with the listed confidence levels. Leave empty to show
+# all. Valid levels: HIGH, INFERENCE, INFERENCE_FAILURE, UNDEFINED.
+confidence=
+
+# Disable the message, report, category or checker with the given id(s). You
+# can either give multiple identifiers separated by comma (,) or put this
+# option multiple times (only on the command line, not in the configuration
+# file where it should appear only once). You can also use "--disable=all" to
+# disable everything first and then reenable specific checks. For example, if
+# you want to run only the similarities checker, you can use "--disable=all
+# --enable=similarities". If you want to run only the classes checker, but have
+# no Warning level messages displayed, use "--disable=all --enable=classes
+# --disable=W".
+disable=invalid-name,
+        missing-docstring,
+        too-many-lines,
+        print-statement,
+        parameter-unpacking,
+        unpacking-in-except,
+        old-raise-syntax,
+        backtick,
+        long-suffix,
+        old-ne-operator,
+        old-octal-literal,
+        import-star-module-level,
+        non-ascii-bytes-literal,
+        raw-checker-failed,
+        bad-inline-option,
+        locally-disabled,
+        file-ignored,
+        suppressed-message,
+        useless-suppression,
+        deprecated-pragma,
+        use-symbolic-message-instead,
+        too-many-instance-attributes,
+        too-many-public-methods,
+        too-many-branches,
+        too-many-arguments,
+        too-many-locals,
+        too-many-statements,
+        no-else-return,
+        no-else-raise,
+        unnecessary-pass,
+        wildcard-import,
+        unused-wildcard-import,
+        apply-builtin,
+        basestring-builtin,
+        buffer-builtin,
+        cmp-builtin,
+        coerce-builtin,
+        execfile-builtin,
+        file-builtin,
+        long-builtin,
+        raw_input-builtin,
+        reduce-builtin,
+        standarderror-builtin,
+        unicode-builtin,
+        xrange-builtin,
+        coerce-method,
+        delslice-method,
+        getslice-method,
+        setslice-method,
+        no-absolute-import,
+        old-division,
+        dict-iter-method,
+        dict-view-method,
+        next-method-called,
+        metaclass-assignment,
+        indexing-exception,
+        raising-string,
+        reload-builtin,
+        oct-method,
+        hex-method,
+        nonzero-method,
+        cmp-method,
+        input-builtin,
+        round-builtin,
+        intern-builtin,
+        unichr-builtin,
+        map-builtin-not-iterating,
+        zip-builtin-not-iterating,
+        range-builtin-not-iterating,
+        filter-builtin-not-iterating,
+        using-cmp-argument,
+        eq-without-hash,
+        div-method,
+        idiv-method,
+        rdiv-method,
+        exception-message-attribute,
+        invalid-str-codec,
+        sys-max-int,
+        bad-python3-import,
+        deprecated-string-function,
+        deprecated-str-translate-call,
+        deprecated-itertools-function,
+        deprecated-types-field,
+        next-method-defined,
+        dict-items-not-iterating,
+        dict-keys-not-iterating,
+        dict-values-not-iterating,
+        deprecated-operator-function,
+        deprecated-urllib-function,
+        xreadlines-attribute,
+        deprecated-sys-function,
+        exception-escape,
+        comprehension-escape
+
+# Enable the message, report, category or checker with the given id(s). You can
+# either give multiple identifier separated by comma (,) or put this option
+# multiple time (only on the command line, not in the configuration file where
+# it should appear only once). See also the "--disable" option for examples.
+enable=c-extension-no-member
+
+
+[REPORTS]
+
+# Python expression which should return a note less than 10 (10 is the highest
+# note). You have access to the variables errors warning, statement which
+# respectively contain the number of errors / warnings messages and the total
+# number of statements analyzed. This is used by the global evaluation report
+# (RP0004).
+evaluation=10.0 - ((float(5 * error + warning + refactor + convention) / statement) * 10)
+
+# Template used to display messages. This is a python new-style format string
+# used to format the message information. See doc for all details.
+#msg-template=
+
+# Set the output format. Available formats are text, parseable, colorized, json
+# and msvs (visual studio). You can also give a reporter class, e.g.
+# mypackage.mymodule.MyReporterClass.
+output-format=text
+
+# Tells whether to display a full report or only the messages.
+reports=no
+
+# Activate the evaluation score.
+score=yes
+
+
+[REFACTORING]
+
+# Maximum number of nested blocks for function / method body
+max-nested-blocks=5
+
+# Complete name of functions that never returns. When checking for
+# inconsistent-return-statements if a never returning function is called then
+# it will be considered as an explicit return statement and no message will be
+# printed.
+never-returning-functions=sys.exit
+
+
+[VARIABLES]
+
+# List of additional names supposed to be defined in builtins. Remember that
+# you should avoid defining new builtins when possible.
+additional-builtins=
+
+# Tells whether unused global variables should be treated as a violation.
+allow-global-unused-variables=yes
+
+# List of strings which can identify a callback function by name. A callback
+# name must start or end with one of those strings.
+callbacks=cb_,
+          _cb
+
+# A regular expression matching the name of dummy variables (i.e. expected to
+# not be used).
+dummy-variables-rgx=_+$|(_[a-zA-Z0-9_]*[a-zA-Z0-9]+?$)|dummy|^ignored_|^unused_
+
+# Argument names that match this expression will be ignored. Default to name
+# with leading underscore.
+ignored-argument-names=_.*|^ignored_|^unused_
+
+# Tells whether we should check for unused import in __init__ files.
+init-import=no
+
+# List of qualified module names which can have objects that can redefine
+# builtins.
+redefining-builtins-modules=six.moves,past.builtins,future.builtins,builtins,io
+
+
+[TYPECHECK]
+
+# List of decorators that produce context managers, such as
+# contextlib.contextmanager. Add to this list to register other decorators that
+# produce valid context managers.
+contextmanager-decorators=contextlib.contextmanager
+
+# List of members which are set dynamically and missed by pylint inference
+# system, and so shouldn't trigger E1101 when accessed. Python regular
+# expressions are accepted.
+generated-members=
+
+# Tells whether missing members accessed in mixin class should be ignored. A
+# mixin class is detected if its name ends with "mixin" (case insensitive).
+ignore-mixin-members=yes
+
+# Tells whether to warn about missing members when the owner of the attribute
+# is inferred to be None.
+ignore-none=yes
+
+# This flag controls whether pylint should warn about no-member and similar
+# checks whenever an opaque object is returned when inferring. The inference
+# can return multiple potential results while evaluating a Python object, but
+# some branches might not be evaluated, which results in partial inference. In
+# that case, it might be useful to still emit no-member and other checks for
+# the rest of the inferred objects.
+ignore-on-opaque-inference=yes
+
+# List of class names for which member attributes should not be checked (useful
+# for classes with dynamically set attributes). This supports the use of
+# qualified names.
+ignored-classes=optparse.Values,thread._local,_thread._local
+
+# List of module names for which member attributes should not be checked
+# (useful for modules/projects where namespaces are manipulated during runtime
+# and thus existing member attributes cannot be deduced by static analysis. It
+# supports qualified module names, as well as Unix pattern matching.
+ignored-modules=
+
+# Show a hint with possible names when a member name was not found. The aspect
+# of finding the hint is based on edit distance.
+missing-member-hint=yes
+
+# The minimum edit distance a name should have in order to be considered a
+# similar match for a missing member name.
+missing-member-hint-distance=1
+
+# The total number of similar names that should be taken in consideration when
+# showing a hint for a missing member.
+missing-member-max-choices=1
+
+
+[STRING]
+
+# This flag controls whether the implicit-str-concat-in-sequence should
+# generate a warning on implicit string concatenation in sequences defined over
+# several lines.
+check-str-concat-over-line-jumps=no
+
+
+[SPELLING]
+
+# Limits count of emitted suggestions for spelling mistakes.
+max-spelling-suggestions=4
+
+# Spelling dictionary name. Available dictionaries: none. To make it working
+# install python-enchant package..
+spelling-dict=
+
+# List of comma separated words that should not be checked.
+spelling-ignore-words=
+
+# A path to a file that contains private dictionary; one word per line.
+spelling-private-dict-file=
+
+# Tells whether to store unknown words to indicated private dictionary in
+# --spelling-private-dict-file option instead of raising a message.
+spelling-store-unknown-words=no
+
+
+[SIMILARITIES]
+
+# Ignore comments when computing similarities.
+ignore-comments=yes
+
+# Ignore docstrings when computing similarities.
+ignore-docstrings=yes
+
+# Ignore imports when computing similarities.
+ignore-imports=no
+
+# Minimum lines number of a similarity.
+min-similarity-lines=4
+
+
+[MISCELLANEOUS]
+
+# List of note tags to take in consideration, separated by a comma.
+notes=FIXME,
+      XXX,
+      TODO
+
+
+[LOGGING]
+
+# Format style used to check logging format string. `old` means using %
+# formatting, while `new` is for `{}` formatting.
+logging-format-style=old
+
+# Logging modules to check that the string format arguments are in logging
+# function parameter format.
+logging-modules=logging
+
+
+[FORMAT]
+
+# Expected format of line ending, e.g. empty (any line ending), LF or CRLF.
+expected-line-ending-format=
+
+# Regexp for a line that is allowed to be longer than the limit.
+ignore-long-lines=^\s*(# )?<?https?://\S+>?$
+
+# Number of spaces of indent required inside a hanging or continued line.
+indent-after-paren=4
+
+# String used as indentation unit. This is usually "    " (4 spaces) or "\t" (1
+# tab).
+indent-string='    '
+
+# Maximum number of characters on a single line.
+max-line-length=120
+
+# Maximum number of lines in a module.
+max-module-lines=1000
+
+# List of optional constructs for which whitespace checking is disabled. `dict-
+# separator` is used to allow tabulation in dicts, etc.: {1  : 1,\n222: 2}.
+# `trailing-comma` allows a space between comma and closing bracket: (a, ).
+# `empty-line` allows space-only lines.
+no-space-check=trailing-comma,
+               dict-separator
+
+# Allow the body of a class to be on the same line as the declaration if body
+# contains single statement.
+single-line-class-stmt=no
+
+# Allow the body of an if to be on the same line as the test if there is no
+# else.
+single-line-if-stmt=no
+
+
+[BASIC]
+
+# Naming style matching correct argument names.
+argument-naming-style=snake_case
+
+# Regular expression matching correct argument names. Overrides argument-
+# naming-style.
+#argument-rgx=
+
+# Naming style matching correct attribute names.
+attr-naming-style=snake_case
+
+# Regular expression matching correct attribute names. Overrides attr-naming-
+# style.
+#attr-rgx=
+
+# Bad variable names which should always be refused, separated by a comma.
+bad-names=foo,
+          bar,
+          baz,
+          toto,
+          tutu,
+          tata
+
+# Naming style matching correct class attribute names.
+class-attribute-naming-style=any
+
+# Regular expression matching correct class attribute names. Overrides class-
+# attribute-naming-style.
+#class-attribute-rgx=
+
+# Naming style matching correct class names.
+class-naming-style=PascalCase
+
+# Regular expression matching correct class names. Overrides class-naming-
+# style.
+#class-rgx=
+
+# Naming style matching correct constant names.
+const-naming-style=UPPER_CASE
+
+# Regular expression matching correct constant names. Overrides const-naming-
+# style.
+#const-rgx=
+
+# Minimum line length for functions/classes that require docstrings, shorter
+# ones are exempt.
+docstring-min-length=-1
+
+# Naming style matching correct function names.
+function-naming-style=snake_case
+
+# Regular expression matching correct function names. Overrides function-
+# naming-style.
+#function-rgx=
+
+# Good variable names which should always be accepted, separated by a comma.
+good-names=i,
+           j,
+           k,
+           ex,
+           Run,
+           _
+
+# Include a hint for the correct naming format with invalid-name.
+include-naming-hint=no
+
+# Naming style matching correct inline iteration names.
+inlinevar-naming-style=any
+
+# Regular expression matching correct inline iteration names. Overrides
+# inlinevar-naming-style.
+#inlinevar-rgx=
+
+# Naming style matching correct method names.
+method-naming-style=snake_case
+
+# Regular expression matching correct method names. Overrides method-naming-
+# style.
+#method-rgx=
+
+# Naming style matching correct module names.
+module-naming-style=snake_case
+
+# Regular expression matching correct module names. Overrides module-naming-
+# style.
+#module-rgx=
+
+# Colon-delimited sets of names that determine each other's naming style when
+# the name regexes allow several styles.
+name-group=
+
+# Regular expression which should only match function or class names that do
+# not require a docstring.
+no-docstring-rgx=^_
+
+# List of decorators that produce properties, such as abc.abstractproperty. Add
+# to this list to register other decorators that produce valid properties.
+# These decorators are taken in consideration only for invalid-name.
+property-classes=abc.abstractproperty
+
+# Naming style matching correct variable names.
+variable-naming-style=snake_case
+
+# Regular expression matching correct variable names. Overrides variable-
+# naming-style.
+#variable-rgx=
+
+
+[IMPORTS]
+
+# Allow wildcard imports from modules that define __all__.
+allow-wildcard-with-all=no
+
+# Analyse import fallback blocks. This can be used to support both Python 2 and
+# 3 compatible code, which means that the block might have code that exists
+# only in one or another interpreter, leading to false positives when analysed.
+analyse-fallback-blocks=no
+
+# Deprecated modules which should not be used, separated by a comma.
+deprecated-modules=optparse,tkinter.tix
+
+# Create a graph of external dependencies in the given file (report RP0402 must
+# not be disabled).
+ext-import-graph=
+
+# Create a graph of every (i.e. internal and external) dependencies in the
+# given file (report RP0402 must not be disabled).
+import-graph=
+
+# Create a graph of internal dependencies in the given file (report RP0402 must
+# not be disabled).
+int-import-graph=
+
+# Force import order to recognize a module as part of the standard
+# compatibility libraries.
+known-standard-library=
+
+# Force import order to recognize a module as part of a third party library.
+known-third-party=enchant
+
+
+[DESIGN]
+
+# Maximum number of arguments for function / method.
+max-args=5
+
+# Maximum number of attributes for a class (see R0902).
+max-attributes=7
+
+# Maximum number of boolean expressions in an if statement.
+max-bool-expr=5
+
+# Maximum number of branch for function / method body.
+max-branches=12
+
+# Maximum number of locals for function / method body.
+max-locals=15
+
+# Maximum number of parents for a class (see R0901).
+max-parents=7
+
+# Maximum number of public methods for a class (see R0904).
+max-public-methods=20
+
+# Maximum number of return / yield for function / method body.
+max-returns=6
+
+# Maximum number of statements in function / method body.
+max-statements=50
+
+# Minimum number of public methods for a class (see R0903).
+min-public-methods=2
+
+
+[CLASSES]
+
+# List of method names used to declare (i.e. assign) instance attributes.
+defining-attr-methods=__init__,
+                      __new__,
+                      setUp
+
+# List of member names, which should be excluded from the protected access
+# warning.
+exclude-protected=_asdict,
+                  _fields,
+                  _replace,
+                  _source,
+                  _make
+
+# List of valid names for the first argument in a class method.
+valid-classmethod-first-arg=cls
+
+# List of valid names for the first argument in a metaclass class method.
+valid-metaclass-classmethod-first-arg=cls
+
+
+[EXCEPTIONS]
+
+# Exceptions that will emit a warning when being caught. Defaults to
+# "BaseException, Exception".
+overgeneral-exceptions=BaseException,
+                       Exception

.travis.yml

@@ -1,15 +1,17 @@
+dist: xenial
 language: python
 python:
  - "3.5"
- - "3.5-dev"
  - "3.6"
- - "3.6-dev"
+ - "3.7"
 install:
- - pip install pylint requests
+ - pipenv sync --dev
- - pip install -r docs/requirements.txt
 script:
- - python3 -m pylint -r n -d bad-whitespace,missing-docstring,too-many-arguments,locally-disabled,line-too-long,too-many-public-methods,too-many-lines,too-many-instance-attributes,too-many-locals,too-many-branches,too-many-statements,inconsistent-return-statements instaloader
+ - pylint instaloader
- - make -C docs html
+ - mypy -m instaloader
+ - make -C docs html SPHINXOPTS="-W -n"
+addons:
+  ssh_known_hosts: aur.archlinux.org
 deploy:
   - provider: pypi
     user: aandergr
@@ -25,5 +27,10 @@ deploy:
     repo: instaloader/instaloader.github.io
     target-branch: master
     on:
-      branch: master
+      tags: true
+      python: 3.6
+  - provider: script
+    script: deploy/arch/deploy.sh $TRAVIS_TAG
+    on:
+      tags: true
       python: 3.6

AUTHORS.md

@@ -0,0 +1,8 @@
+Authors
+=======
+
+Instaloader is written by
+
+- Alexander Graf (@aandergr)
+- André Koch-Kramer (@Thammus)
+- Lars Lindqvist (@e5150)

LICENSE

@@ -1,6 +1,6 @@
 The MIT License (MIT)
 
-Copyright (c) 2016-2018 Alexander Graf and André Koch-Kramer.
+Copyright (c) 2016-2019 Alexander Graf and André Koch-Kramer.
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

Pipfile

@@ -0,0 +1,16 @@
+[[source]]
+name = "pypi"
+url = "https://pypi.org/simple"
+verify_ssl = true
+
+[dev-packages]
+pylint = "*"
+mypy = "*"
+sphinx = "*"
+pyinstaller = "*"
+pefile = "*"
+pywin32-ctypes = "*"
+psutil = "*"
+
+[packages]
+requests = "*"

Pipfile.lock

@@ -0,0 +1,431 @@
+{
+    "_meta": {
+        "hash": {
+            "sha256": "33408a60fda1a56bf680462fad21643ae0159bc13f3b85d8f10ebf3eef7d8725"
+        },
+        "pipfile-spec": 6,
+        "requires": {},
+        "sources": [
+            {
+                "name": "pypi",
+                "url": "https://pypi.org/simple",
+                "verify_ssl": true
+            }
+        ]
+    },
+    "default": {
+        "certifi": {
+            "hashes": [
+                "sha256:017c25db2a153ce562900032d5bc68e9f191e44e9a0f762f373977de9df1fbb3",
+                "sha256:25b64c7da4cd7479594d035c08c2d809eb4aab3a26e5a990ea98cc450c320f1f"
+            ],
+            "version": "==2019.11.28"
+        },
+        "chardet": {
+            "hashes": [
+                "sha256:84ab92ed1c4d4f16916e05906b6b75a6c0fb5db821cc65e70cbd64a3e2a5eaae",
+                "sha256:fc323ffcaeaed0e0a02bf4d117757b98aed530d9ed4531e3e15460124c106691"
+            ],
+            "version": "==3.0.4"
+        },
+        "idna": {
+            "hashes": [
+                "sha256:c357b3f628cf53ae2c4c05627ecc484553142ca23264e593d327bcde5e9c3407",
+                "sha256:ea8b7f6188e6fa117537c3df7da9fc686d485087abf6ac197f9c46432f7e4a3c"
+            ],
+            "version": "==2.8"
+        },
+        "requests": {
+            "hashes": [
+                "sha256:11e007a8a2aa0323f5a921e9e6a2d7e4e67d9877e85773fba9ba6419025cbeb4",
+                "sha256:9cf5292fcd0f598c671cfc1e0d7d1a7f13bb8085e9a590f48c010551dc6c4b31"
+            ],
+            "index": "pypi",
+            "version": "==2.22.0"
+        },
+        "urllib3": {
+            "hashes": [
+                "sha256:2f3db8b19923a873b3e5256dc9c2dedfa883e33d87c690d9c7913e1f40673cdc",
+                "sha256:87716c2d2a7121198ebcb7ce7cccf6ce5e9ba539041cfbaeecfb641dc0bf6acc"
+            ],
+            "version": "==1.25.8"
+        }
+    },
+    "develop": {
+        "alabaster": {
+            "hashes": [
+                "sha256:446438bdcca0e05bd45ea2de1668c1d9b032e1a9154c2c259092d77031ddd359",
+                "sha256:a661d72d58e6ea8a57f7a86e37d86716863ee5e92788398526d58b26a4e4dc02"
+            ],
+            "version": "==0.7.12"
+        },
+        "altgraph": {
+            "hashes": [
+                "sha256:1f05a47122542f97028caf78775a095fbe6a2699b5089de8477eb583167d69aa",
+                "sha256:c623e5f3408ca61d4016f23a681b9adb100802ca3e3da5e718915a9e4052cebe"
+            ],
+            "version": "==0.17"
+        },
+        "astroid": {
+            "hashes": [
+                "sha256:71ea07f44df9568a75d0f354c49143a4575d90645e9fead6dfb52c26a85ed13a",
+                "sha256:840947ebfa8b58f318d42301cf8c0a20fd794a33b61cc4638e28e9e61ba32f42"
+            ],
+            "version": "==2.3.3"
+        },
+        "babel": {
+            "hashes": [
+                "sha256:1aac2ae2d0d8ea368fa90906567f5c08463d98ade155c0c4bfedd6a0f7160e38",
+                "sha256:d670ea0b10f8b723672d3a6abeb87b565b244da220d76b4dba1b66269ec152d4"
+            ],
+            "version": "==2.8.0"
+        },
+        "certifi": {
+            "hashes": [
+                "sha256:017c25db2a153ce562900032d5bc68e9f191e44e9a0f762f373977de9df1fbb3",
+                "sha256:25b64c7da4cd7479594d035c08c2d809eb4aab3a26e5a990ea98cc450c320f1f"
+            ],
+            "version": "==2019.11.28"
+        },
+        "chardet": {
+            "hashes": [
+                "sha256:84ab92ed1c4d4f16916e05906b6b75a6c0fb5db821cc65e70cbd64a3e2a5eaae",
+                "sha256:fc323ffcaeaed0e0a02bf4d117757b98aed530d9ed4531e3e15460124c106691"
+            ],
+            "version": "==3.0.4"
+        },
+        "docutils": {
+            "hashes": [
+                "sha256:0c5b78adfbf7762415433f5515cd5c9e762339e23369dbe8000d84a4bf4ab3af",
+                "sha256:c2de3a60e9e7d07be26b7f2b00ca0309c207e06c100f9cc2a94931fc75a478fc"
+            ],
+            "version": "==0.16"
+        },
+        "future": {
+            "hashes": [
+                "sha256:b1bead90b70cf6ec3f0710ae53a525360fa360d306a86583adc6bf83a4db537d"
+            ],
+            "version": "==0.18.2"
+        },
+        "idna": {
+            "hashes": [
+                "sha256:c357b3f628cf53ae2c4c05627ecc484553142ca23264e593d327bcde5e9c3407",
+                "sha256:ea8b7f6188e6fa117537c3df7da9fc686d485087abf6ac197f9c46432f7e4a3c"
+            ],
+            "version": "==2.8"
+        },
+        "imagesize": {
+            "hashes": [
+                "sha256:6965f19a6a2039c7d48bca7dba2473069ff854c36ae6f19d2cde309d998228a1",
+                "sha256:b1f6b5a4eab1f73479a50fb79fcf729514a900c341d8503d62a62dbc4127a2b1"
+            ],
+            "version": "==1.2.0"
+        },
+        "isort": {
+            "hashes": [
+                "sha256:54da7e92468955c4fceacd0c86bd0ec997b0e1ee80d97f67c35a78b719dccab1",
+                "sha256:6e811fcb295968434526407adb8796944f1988c5b65e8139058f2014cbe100fd"
+            ],
+            "version": "==4.3.21"
+        },
+        "jinja2": {
+            "hashes": [
+                "sha256:93187ffbc7808079673ef52771baa950426fd664d3aad1d0fa3e95644360e250",
+                "sha256:b0eaf100007721b5c16c1fc1eecb87409464edc10469ddc9a22a27a99123be49"
+            ],
+            "version": "==2.11.1"
+        },
+        "lazy-object-proxy": {
+            "hashes": [
+                "sha256:0c4b206227a8097f05c4dbdd323c50edf81f15db3b8dc064d08c62d37e1a504d",
+                "sha256:194d092e6f246b906e8f70884e620e459fc54db3259e60cf69a4d66c3fda3449",
+                "sha256:1be7e4c9f96948003609aa6c974ae59830a6baecc5376c25c92d7d697e684c08",
+                "sha256:4677f594e474c91da97f489fea5b7daa17b5517190899cf213697e48d3902f5a",
+                "sha256:48dab84ebd4831077b150572aec802f303117c8cc5c871e182447281ebf3ac50",
+                "sha256:5541cada25cd173702dbd99f8e22434105456314462326f06dba3e180f203dfd",
+                "sha256:59f79fef100b09564bc2df42ea2d8d21a64fdcda64979c0fa3db7bdaabaf6239",
+                "sha256:8d859b89baf8ef7f8bc6b00aa20316483d67f0b1cbf422f5b4dc56701c8f2ffb",
+                "sha256:9254f4358b9b541e3441b007a0ea0764b9d056afdeafc1a5569eee1cc6c1b9ea",
+                "sha256:9651375199045a358eb6741df3e02a651e0330be090b3bc79f6d0de31a80ec3e",
+                "sha256:97bb5884f6f1cdce0099f86b907aa41c970c3c672ac8b9c8352789e103cf3156",
+                "sha256:9b15f3f4c0f35727d3a0fba4b770b3c4ebbb1fa907dbcc046a1d2799f3edd142",
+                "sha256:a2238e9d1bb71a56cd710611a1614d1194dc10a175c1e08d75e1a7bcc250d442",
+                "sha256:a6ae12d08c0bf9909ce12385803a543bfe99b95fe01e752536a60af2b7797c62",
+                "sha256:ca0a928a3ddbc5725be2dd1cf895ec0a254798915fb3a36af0964a0a4149e3db",
+                "sha256:cb2c7c57005a6804ab66f106ceb8482da55f5314b7fcb06551db1edae4ad1531",
+                "sha256:d74bb8693bf9cf75ac3b47a54d716bbb1a92648d5f781fc799347cfc95952383",
+                "sha256:d945239a5639b3ff35b70a88c5f2f491913eb94871780ebfabb2568bd58afc5a",
+                "sha256:eba7011090323c1dadf18b3b689845fd96a61ba0a1dfbd7f24b921398affc357",
+                "sha256:efa1909120ce98bbb3777e8b6f92237f5d5c8ea6758efea36a473e1d38f7d3e4",
+                "sha256:f3900e8a5de27447acbf900b4750b0ddfd7ec1ea7fbaf11dfa911141bc522af0"
+            ],
+            "version": "==1.4.3"
+        },
+        "markupsafe": {
+            "hashes": [
+                "sha256:00bc623926325b26bb9605ae9eae8a215691f33cae5df11ca5424f06f2d1f473",
+                "sha256:09027a7803a62ca78792ad89403b1b7a73a01c8cb65909cd876f7fcebd79b161",
+                "sha256:09c4b7f37d6c648cb13f9230d847adf22f8171b1ccc4d5682398e77f40309235",
+                "sha256:1027c282dad077d0bae18be6794e6b6b8c91d58ed8a8d89a89d59693b9131db5",
+                "sha256:13d3144e1e340870b25e7b10b98d779608c02016d5184cfb9927a9f10c689f42",
+                "sha256:24982cc2533820871eba85ba648cd53d8623687ff11cbb805be4ff7b4c971aff",
+                "sha256:29872e92839765e546828bb7754a68c418d927cd064fd4708fab9fe9c8bb116b",
+                "sha256:43a55c2930bbc139570ac2452adf3d70cdbb3cfe5912c71cdce1c2c6bbd9c5d1",
+                "sha256:46c99d2de99945ec5cb54f23c8cd5689f6d7177305ebff350a58ce5f8de1669e",
+                "sha256:500d4957e52ddc3351cabf489e79c91c17f6e0899158447047588650b5e69183",
+                "sha256:535f6fc4d397c1563d08b88e485c3496cf5784e927af890fb3c3aac7f933ec66",
+                "sha256:596510de112c685489095da617b5bcbbac7dd6384aeebeda4df6025d0256a81b",
+                "sha256:62fe6c95e3ec8a7fad637b7f3d372c15ec1caa01ab47926cfdf7a75b40e0eac1",
+                "sha256:6788b695d50a51edb699cb55e35487e430fa21f1ed838122d722e0ff0ac5ba15",
+                "sha256:6dd73240d2af64df90aa7c4e7481e23825ea70af4b4922f8ede5b9e35f78a3b1",
+                "sha256:717ba8fe3ae9cc0006d7c451f0bb265ee07739daf76355d06366154ee68d221e",
+                "sha256:79855e1c5b8da654cf486b830bd42c06e8780cea587384cf6545b7d9ac013a0b",
+                "sha256:7c1699dfe0cf8ff607dbdcc1e9b9af1755371f92a68f706051cc8c37d447c905",
+                "sha256:88e5fcfb52ee7b911e8bb6d6aa2fd21fbecc674eadd44118a9cc3863f938e735",
+                "sha256:8defac2f2ccd6805ebf65f5eeb132adcf2ab57aa11fdf4c0dd5169a004710e7d",
+                "sha256:98c7086708b163d425c67c7a91bad6e466bb99d797aa64f965e9d25c12111a5e",
+                "sha256:9add70b36c5666a2ed02b43b335fe19002ee5235efd4b8a89bfcf9005bebac0d",
+                "sha256:9bf40443012702a1d2070043cb6291650a0841ece432556f784f004937f0f32c",
+                "sha256:ade5e387d2ad0d7ebf59146cc00c8044acbd863725f887353a10df825fc8ae21",
+                "sha256:b00c1de48212e4cc9603895652c5c410df699856a2853135b3967591e4beebc2",
+                "sha256:b1282f8c00509d99fef04d8ba936b156d419be841854fe901d8ae224c59f0be5",
+                "sha256:b2051432115498d3562c084a49bba65d97cf251f5a331c64a12ee7e04dacc51b",
+                "sha256:ba59edeaa2fc6114428f1637ffff42da1e311e29382d81b339c1817d37ec93c6",
+                "sha256:c8716a48d94b06bb3b2524c2b77e055fb313aeb4ea620c8dd03a105574ba704f",
+                "sha256:cd5df75523866410809ca100dc9681e301e3c27567cf498077e8551b6d20e42f",
+                "sha256:cdb132fc825c38e1aeec2c8aa9338310d29d337bebbd7baa06889d09a60a1fa2",
+                "sha256:e249096428b3ae81b08327a63a485ad0878de3fb939049038579ac0ef61e17e7",
+                "sha256:e8313f01ba26fbbe36c7be1966a7b7424942f670f38e666995b88d012765b9be"
+            ],
+            "version": "==1.1.1"
+        },
+        "mccabe": {
+            "hashes": [
+                "sha256:ab8a6258860da4b6677da4bd2fe5dc2c659cff31b3ee4f7f5d64e79735b80d42",
+                "sha256:dd8d182285a0fe56bace7f45b5e7d1a6ebcbf524e8f3bd87eb0f125271b8831f"
+            ],
+            "version": "==0.6.1"
+        },
+        "mypy": {
+            "hashes": [
+                "sha256:0a9a45157e532da06fe56adcfef8a74629566b607fa2c1ac0122d1ff995c748a",
+                "sha256:2c35cae79ceb20d47facfad51f952df16c2ae9f45db6cb38405a3da1cf8fc0a7",
+                "sha256:4b9365ade157794cef9685791032521233729cb00ce76b0ddc78749abea463d2",
+                "sha256:53ea810ae3f83f9c9b452582261ea859828a9ed666f2e1ca840300b69322c474",
+                "sha256:634aef60b4ff0f650d3e59d4374626ca6153fcaff96ec075b215b568e6ee3cb0",
+                "sha256:7e396ce53cacd5596ff6d191b47ab0ea18f8e0ec04e15d69728d530e86d4c217",
+                "sha256:7eadc91af8270455e0d73565b8964da1642fe226665dd5c9560067cd64d56749",
+                "sha256:7f672d02fffcbace4db2b05369142e0506cdcde20cea0e07c7c2171c4fd11dd6",
+                "sha256:85baab8d74ec601e86134afe2bcccd87820f79d2f8d5798c889507d1088287bf",
+                "sha256:87c556fb85d709dacd4b4cb6167eecc5bbb4f0a9864b69136a0d4640fdc76a36",
+                "sha256:a6bd44efee4dc8c3324c13785a9dc3519b3ee3a92cada42d2b57762b7053b49b",
+                "sha256:c6d27bd20c3ba60d5b02f20bd28e20091d6286a699174dfad515636cb09b5a72",
+                "sha256:e2bb577d10d09a2d8822a042a23b8d62bc3b269667c9eb8e60a6edfa000211b1",
+                "sha256:f97a605d7c8bc2c6d1172c2f0d5a65b24142e11a58de689046e62c2d632ca8c1"
+            ],
+            "index": "pypi",
+            "version": "==0.761"
+        },
+        "mypy-extensions": {
+            "hashes": [
+                "sha256:090fedd75945a69ae91ce1303b5824f428daf5a028d2f6ab8a299250a846f15d",
+                "sha256:2d82818f5bb3e369420cb3c4060a7970edba416647068eb4c5343488a6c604a8"
+            ],
+            "version": "==0.4.3"
+        },
+        "packaging": {
+            "hashes": [
+                "sha256:170748228214b70b672c581a3dd610ee51f733018650740e98c7df862a583f73",
+                "sha256:e665345f9eef0c621aa0bf2f8d78cf6d21904eef16a93f020240b704a57f1334"
+            ],
+            "version": "==20.1"
+        },
+        "pefile": {
+            "hashes": [
+                "sha256:a5d6e8305c6b210849b47a6174ddf9c452b2888340b8177874b862ba6c207645"
+            ],
+            "index": "pypi",
+            "version": "==2019.4.18"
+        },
+        "psutil": {
+            "hashes": [
+                "sha256:094f899ac3ef72422b7e00411b4ed174e3c5a2e04c267db6643937ddba67a05b",
+                "sha256:10b7f75cc8bd676cfc6fa40cd7d5c25b3f45a0e06d43becd7c2d2871cbb5e806",
+                "sha256:1b1575240ca9a90b437e5a40db662acd87bbf181f6aa02f0204978737b913c6b",
+                "sha256:21231ef1c1a89728e29b98a885b8e0a8e00d09018f6da5cdc1f43f988471a995",
+                "sha256:28f771129bfee9fc6b63d83a15d857663bbdcae3828e1cb926e91320a9b5b5cd",
+                "sha256:70387772f84fa5c3bb6a106915a2445e20ac8f9821c5914d7cbde148f4d7ff73",
+                "sha256:b560f5cd86cf8df7bcd258a851ca1ad98f0d5b8b98748e877a0aec4e9032b465",
+                "sha256:b74b43fecce384a57094a83d2778cdfc2e2d9a6afaadd1ebecb2e75e0d34e10d",
+                "sha256:e85f727ffb21539849e6012f47b12f6dd4c44965e56591d8dec6e8bc9ab96f4a",
+                "sha256:fd2e09bb593ad9bdd7429e779699d2d47c1268cbde4dda95fcd1bd17544a0217",
+                "sha256:ffad8eb2ac614518bbe3c0b8eb9dffdb3a8d2e3a7d5da51c5b974fb723a5c5aa"
+            ],
+            "index": "pypi",
+            "version": "==5.6.7"
+        },
+        "pygments": {
+            "hashes": [
+                "sha256:2a3fe295e54a20164a9df49c75fa58526d3be48e14aceba6d6b1e8ac0bfd6f1b",
+                "sha256:98c8aa5a9f778fcd1026a17361ddaf7330d1b7c62ae97c3bb0ae73e0b9b6b0fe"
+            ],
+            "version": "==2.5.2"
+        },
+        "pyinstaller": {
+            "hashes": [
+                "sha256:3730fa80d088f8bb7084d32480eb87cbb4ddb64123363763cf8f2a1378c1c4b7"
+            ],
+            "index": "pypi",
+            "version": "==3.6"
+        },
+        "pylint": {
+            "hashes": [
+                "sha256:3db5468ad013380e987410a8d6956226963aed94ecb5f9d3a28acca6d9ac36cd",
+                "sha256:886e6afc935ea2590b462664b161ca9a5e40168ea99e5300935f6591ad467df4"
+            ],
+            "index": "pypi",
+            "version": "==2.4.4"
+        },
+        "pyparsing": {
+            "hashes": [
+                "sha256:4c830582a84fb022400b85429791bc551f1f4871c33f23e44f353119e92f969f",
+                "sha256:c342dccb5250c08d45fd6f8b4a559613ca603b57498511740e65cd11a2e7dcec"
+            ],
+            "version": "==2.4.6"
+        },
+        "pytz": {
+            "hashes": [
+                "sha256:1c557d7d0e871de1f5ccd5833f60fb2550652da6be2693c1e02300743d21500d",
+                "sha256:b02c06db6cf09c12dd25137e563b31700d3b80fcc4ad23abb7a315f2789819be"
+            ],
+            "version": "==2019.3"
+        },
+        "pywin32-ctypes": {
+            "hashes": [
+                "sha256:24ffc3b341d457d48e8922352130cf2644024a4ff09762a2261fd34c36ee5942",
+                "sha256:9dc2d991b3479cc2df15930958b674a48a227d5361d413827a4cfd0b5876fc98"
+            ],
+            "index": "pypi",
+            "version": "==0.2.0"
+        },
+        "requests": {
+            "hashes": [
+                "sha256:11e007a8a2aa0323f5a921e9e6a2d7e4e67d9877e85773fba9ba6419025cbeb4",
+                "sha256:9cf5292fcd0f598c671cfc1e0d7d1a7f13bb8085e9a590f48c010551dc6c4b31"
+            ],
+            "index": "pypi",
+            "version": "==2.22.0"
+        },
+        "six": {
+            "hashes": [
+                "sha256:236bdbdce46e6e6a3d61a337c0f8b763ca1e8717c03b369e87a7ec7ce1319c0a",
+                "sha256:8f3cd2e254d8f793e7f3d6d9df77b92252b52637291d0f0da013c76ea2724b6c"
+            ],
+            "version": "==1.14.0"
+        },
+        "snowballstemmer": {
+            "hashes": [
+                "sha256:209f257d7533fdb3cb73bdbd24f436239ca3b2fa67d56f6ff88e86be08cc5ef0",
+                "sha256:df3bac3df4c2c01363f3dd2cfa78cce2840a79b9f1c2d2de9ce8d31683992f52"
+            ],
+            "version": "==2.0.0"
+        },
+        "sphinx": {
+            "hashes": [
+                "sha256:298537cb3234578b2d954ff18c5608468229e116a9757af3b831c2b2b4819159",
+                "sha256:e6e766b74f85f37a5f3e0773a1e1be8db3fcb799deb58ca6d18b70b0b44542a5"
+            ],
+            "index": "pypi",
+            "version": "==2.3.1"
+        },
+        "sphinxcontrib-applehelp": {
+            "hashes": [
+                "sha256:edaa0ab2b2bc74403149cb0209d6775c96de797dfd5b5e2a71981309efab3897",
+                "sha256:fb8dee85af95e5c30c91f10e7eb3c8967308518e0f7488a2828ef7bc191d0d5d"
+            ],
+            "version": "==1.0.1"
+        },
+        "sphinxcontrib-devhelp": {
+            "hashes": [
+                "sha256:6c64b077937330a9128a4da74586e8c2130262f014689b4b89e2d08ee7294a34",
+                "sha256:9512ecb00a2b0821a146736b39f7aeb90759834b07e81e8cc23a9c70bacb9981"
+            ],
+            "version": "==1.0.1"
+        },
+        "sphinxcontrib-htmlhelp": {
+            "hashes": [
+                "sha256:4670f99f8951bd78cd4ad2ab962f798f5618b17675c35c5ac3b2132a14ea8422",
+                "sha256:d4fd39a65a625c9df86d7fa8a2d9f3cd8299a3a4b15db63b50aac9e161d8eff7"
+            ],
+            "version": "==1.0.2"
+        },
+        "sphinxcontrib-jsmath": {
+            "hashes": [
+                "sha256:2ec2eaebfb78f3f2078e73666b1415417a116cc848b72e5172e596c871103178",
+                "sha256:a9925e4a4587247ed2191a22df5f6970656cb8ca2bd6284309578f2153e0c4b8"
+            ],
+            "version": "==1.0.1"
+        },
+        "sphinxcontrib-qthelp": {
+            "hashes": [
+                "sha256:513049b93031beb1f57d4daea74068a4feb77aa5630f856fcff2e50de14e9a20",
+                "sha256:79465ce11ae5694ff165becda529a600c754f4bc459778778c7017374d4d406f"
+            ],
+            "version": "==1.0.2"
+        },
+        "sphinxcontrib-serializinghtml": {
+            "hashes": [
+                "sha256:c0efb33f8052c04fd7a26c0a07f1678e8512e0faec19f4aa8f2473a8b81d5227",
+                "sha256:db6615af393650bf1151a6cd39120c29abaf93cc60db8c48eb2dddbfdc3a9768"
+            ],
+            "version": "==1.1.3"
+        },
+        "typed-ast": {
+            "hashes": [
+                "sha256:0666aa36131496aed8f7be0410ff974562ab7eeac11ef351def9ea6fa28f6355",
+                "sha256:0c2c07682d61a629b68433afb159376e24e5b2fd4641d35424e462169c0a7919",
+                "sha256:249862707802d40f7f29f6e1aad8d84b5aa9e44552d2cc17384b209f091276aa",
+                "sha256:24995c843eb0ad11a4527b026b4dde3da70e1f2d8806c99b7b4a7cf491612652",
+                "sha256:269151951236b0f9a6f04015a9004084a5ab0d5f19b57de779f908621e7d8b75",
+                "sha256:4083861b0aa07990b619bd7ddc365eb7fa4b817e99cf5f8d9cf21a42780f6e01",
+                "sha256:498b0f36cc7054c1fead3d7fc59d2150f4d5c6c56ba7fb150c013fbc683a8d2d",
+                "sha256:4e3e5da80ccbebfff202a67bf900d081906c358ccc3d5e3c8aea42fdfdfd51c1",
+                "sha256:6daac9731f172c2a22ade6ed0c00197ee7cc1221aa84cfdf9c31defeb059a907",
+                "sha256:715ff2f2df46121071622063fc7543d9b1fd19ebfc4f5c8895af64a77a8c852c",
+                "sha256:73d785a950fc82dd2a25897d525d003f6378d1cb23ab305578394694202a58c3",
+                "sha256:8c8aaad94455178e3187ab22c8b01a3837f8ee50e09cf31f1ba129eb293ec30b",
+                "sha256:8ce678dbaf790dbdb3eba24056d5364fb45944f33553dd5869b7580cdbb83614",
+                "sha256:aaee9905aee35ba5905cfb3c62f3e83b3bec7b39413f0a7f19be4e547ea01ebb",
+                "sha256:bcd3b13b56ea479b3650b82cabd6b5343a625b0ced5429e4ccad28a8973f301b",
+                "sha256:c9e348e02e4d2b4a8b2eedb48210430658df6951fa484e59de33ff773fbd4b41",
+                "sha256:d205b1b46085271b4e15f670058ce182bd1199e56b317bf2ec004b6a44f911f6",
+                "sha256:d43943ef777f9a1c42bf4e552ba23ac77a6351de620aa9acf64ad54933ad4d34",
+                "sha256:d5d33e9e7af3b34a40dc05f498939f0ebf187f07c385fd58d591c533ad8562fe",
+                "sha256:fc0fea399acb12edbf8a628ba8d2312f583bdbdb3335635db062fa98cf71fca4",
+                "sha256:fe460b922ec15dd205595c9b5b99e2f056fd98ae8f9f56b888e7a17dc2b757e7"
+            ],
+            "version": "==1.4.1"
+        },
+        "typing-extensions": {
+            "hashes": [
+                "sha256:091ecc894d5e908ac75209f10d5b4f118fbdb2eb1ede6a63544054bb1edb41f2",
+                "sha256:910f4656f54de5993ad9304959ce9bb903f90aadc7c67a0bef07e678014e892d",
+                "sha256:cf8b63fedea4d89bab840ecbb93e75578af28f76f66c35889bd7065f5af88575"
+            ],
+            "version": "==3.7.4.1"
+        },
+        "urllib3": {
+            "hashes": [
+                "sha256:2f3db8b19923a873b3e5256dc9c2dedfa883e33d87c690d9c7913e1f40673cdc",
+                "sha256:87716c2d2a7121198ebcb7ce7cccf6ce5e9ba539041cfbaeecfb641dc0bf6acc"
+            ],
+            "version": "==1.25.8"
+        },
+        "wrapt": {
+            "hashes": [
+                "sha256:565a021fd19419476b9362b05eeaa094178de64f8361e44468f9e9d7843901e1"
+            ],
+            "version": "==1.11.2"
+        }
+    }
+}

README.rst

@@ -1,5 +1,37 @@
 .. image:: https://raw.githubusercontent.com/instaloader/instaloader/master/docs/logo_heading.png
 
+.. badges-start
+
+|travis| |pypi| |pyversion| |license| |aur| |contributors| |downloads|
+
+.. |travis| image:: https://img.shields.io/travis/instaloader/instaloader/master.svg
+   :alt: Travis-CI Build Status
+   :target: https://travis-ci.org/instaloader/instaloader
+
+.. |pypi| image:: https://img.shields.io/pypi/v/instaloader.svg
+   :alt: Instaloader PyPI Project Page
+   :target: https://pypi.org/project/instaloader/
+
+.. |license| image:: https://img.shields.io/github/license/instaloader/instaloader.svg
+   :alt: MIT License
+   :target: https://github.com/instaloader/instaloader/blob/master/LICENSE
+
+.. |pyversion| image:: https://img.shields.io/pypi/pyversions/instaloader.svg
+   :alt: Supported Python Versions
+
+.. |contributors| image:: https://img.shields.io/github/contributors/instaloader/instaloader.svg
+   :alt: Contributor Count
+   :target: https://github.com/instaloader/instaloader/graphs/contributors
+
+.. |aur| image:: https://img.shields.io/aur/version/instaloader.svg
+   :alt: Arch User Repository Package
+   :target: https://aur.archlinux.org/packages/instaloader/
+
+.. |downloads| image:: https://img.shields.io/pypi/dm/instaloader.svg
+   :alt: PyPI Download Count
+
+.. badges-end
+
 ::
 
     $ pip3 install instaloader
@@ -21,7 +53,8 @@
 
 ::
 
-    instaloader [--comments] [--geotags] [--stories]
+    instaloader [--comments] [--geotags]
+                [--stories] [--highlights] [--tagged] [--igtv]
                 [--login YOUR-USERNAME] [--fast-update]
                 profile | "#hashtag" | :stories | :feed | :saved
 
@@ -75,6 +108,24 @@ Disclaimer
 Instaloader is in no way affiliated with, authorized, maintained or endorsed by Instagram or any of its affiliates or
 subsidiaries. This is an independent and unofficial project. Use at your own risk.
 
+Instaloader is licensed under an MIT license. Refer to ``LICENSE`` file for more information.
+
 .. disclaimer-end
 
-Instaloader is licensed under an MIT license. Refer to ``LICENSE`` file for more information.
+Contributing
+------------
+
+As an open source project, Instaloader heavily depends on the contributions from
+its community. See
+`contributing <https://instaloader.github.io/contributing.html>`__
+for how you may help Instaloader to become an even greater tool.
+
+It is a pleasure for us to share our Instaloader to the world, and we are proud
+to have attracted such an active and motivating community, with so many users
+who share their suggestions and ideas with us. Buying a community-sponsored beer
+or coffee from time to time is very likely to further raise our passion for the
+development of Instaloader.
+
+| For Donations, we provide a PayPal.Me link and a Bitcoin address.
+|  PayPal: `PayPal.me/aandergr <https://www.paypal.me/aandergr>`__
+|  BTC: 1Nst4LoadeYzrKjJ1DX9CpbLXBYE9RKLwY

deploy/arch/.SRCINFO.template

@@ -0,0 +1,15 @@
+pkgbase = instaloader
+	pkgdesc = Command line tool to download pictures, videos and metadata from Instagram
+	pkgver = {{version}}
+	pkgrel = 1
+	url = https://instaloader.github.io/
+	arch = any
+	license = MIT
+	makedepends = python-setuptools
+	depends = python>=3.5
+	depends = python-requests>=2.4
+	options = !emptydirs
+	source = instaloader-{{version}}.tar.gz::https://codeload.github.com/instaloader/instaloader/tar.gz/v{{version}}
+	sha512sums = {{hash}}
+
+pkgname = instaloader

deploy/arch/PKGBUILD.template

@@ -0,0 +1,27 @@
+# Maintainer: André Koch-Kramer <koch-kramer@web.de>
+
+pkgname=instaloader
+pkgver={{version}}
+pkgrel=1
+pkgdesc="Command line tool to download pictures, videos and metadata from Instagram"
+arch=('any')
+url="https://instaloader.github.io/"
+license=('MIT')
+groups=()
+depends=('python>=3.5' 'python-requests>=2.4')
+makedepends=('python-setuptools')
+options=('!emptydirs')
+source=($pkgname-$pkgver.tar.gz::https://codeload.github.com/instaloader/instaloader/tar.gz/v$pkgver)
+sha512sums=('{{hash}}')
+
+build() {
+	cd "$srcdir/$pkgname-$pkgver"
+	python setup.py clean --all
+	python setup.py build
+}
+
+package() {
+	cd "$srcdir/$pkgname-$pkgver"
+	install -Dm644 LICENSE "$pkgdir"/usr/share/licenses/$pkgname/LICENSE
+	python setup.py install --root "$pkgdir/" --skip-build
+}

deploy/arch/deploy.sh

@@ -0,0 +1,30 @@
+#!/usr/bin/env bash
+
+VERSION=${1:1}
+
+# do not deploy pre-releases
+echo $VERSION | grep -qe "[abc]" && exit 0
+
+cd $(dirname $0)
+
+# decrypt and add ssh key
+openssl aes-256-cbc -K $encrypted_354637631c28_key -iv $encrypted_354637631c28_iv -in id_rsa_AUR.enc -out /tmp/AUR_openssh -d
+eval "$(ssh-agent -s)"
+chmod 600 /tmp/AUR_openssh
+ssh-add /tmp/AUR_openssh
+
+# clone and modify AUR repo
+git clone --depth 1 ssh://aur@aur.archlinux.org/instaloader.git
+
+curl -sSfOJ https://codeload.github.com/instaloader/instaloader/tar.gz/$1
+HASH=$(sha512sum instaloader-$VERSION.tar.gz | cut -f1 -d " ")
+sed -e "s/{{version}}/$VERSION/g" -e "s/{{hash}}/$HASH/g" PKGBUILD.template > instaloader/PKGBUILD
+sed -e "s/{{version}}/$VERSION/g" -e "s/{{hash}}/$HASH/g" .SRCINFO.template > instaloader/.SRCINFO
+
+# commit and push changes
+cd instaloader
+git config user.email "koch-kramer@web.de"
+git config user.name "André Koch-Kramer"
+git add .
+git commit -m "Release of version $VERSION"
+git push

deploy/windows/create_exe.py

@@ -0,0 +1,86 @@
+import hashlib
+import os
+import re
+import shutil
+import subprocess
+import sys
+
+# copy the required files into repo root
+shutil.copy('docs/favicon.ico', '.')
+shutil.copy('deploy/windows/instaloader.spec', '.')
+shutil.unpack_archive('deploy/windows/ps', '.', 'xztar')
+
+code = """
+import psutil
+import subprocess
+
+def __main():
+    grandparentpid = psutil.Process(os.getppid()).ppid()
+    grandparentpidsearchstring = ' ' + str(grandparentpid) + ' '
+    if hasattr(sys, "_MEIPASS"):
+        ps = os.path.join(sys._MEIPASS, 'tasklist.exe')
+    else:
+        ps = 'tasklist'
+    popen = subprocess.Popen(ps, stdout=subprocess.PIPE, universal_newlines=True)
+    for examine in iter(popen.stdout.readline, ""):
+        if grandparentpidsearchstring in examine:
+            pname = examine
+            break
+    popen.stdout.close()
+    return_code = popen.wait()
+    if return_code:
+        raise subprocess.CalledProcessError(return_code, ps)
+    if pname[0:12] == 'explorer.exe':
+        subprocess.Popen("cmd /K \\\"{0}\\\"".format(os.path.splitext(os.path.basename(sys.argv[0]))[0]))
+    else:
+        main()
+
+
+if __name__ == "__main__":
+    __main()
+"""
+
+with open('instaloader/__main__.py', 'r') as f:
+    # adjust imports for changed file structure
+    regex = re.compile(r'from (?:(\.[^ ]+ )|\.( ))import')
+    lines = [regex.sub(r'from instaloader\1\2import', line) for line in f.readlines()]
+
+    # insert code for magic exe behavior
+    index = lines.index('if __name__ == "__main__":\n')
+    code_lines = [cl + '\n' for cl in code.splitlines()]
+    for i, code_line in enumerate(code_lines):
+        if i + index < len(lines):
+            lines[i + index] = code_line
+        else:
+            lines.extend(code_lines[i:])
+            break
+
+with open('__main__.py', 'w+') as f:
+    f.writelines(lines)
+
+# install dependencies and invoke PyInstaller
+commands = ["pip install pipenv==2018.11.26",
+            "pipenv sync --dev",
+            "pipenv run pyinstaller --log-level=DEBUG instaloader.spec"]
+
+for command in commands:
+    print()
+    print('#' * (len(command) + 6))
+    print('## {} ##'.format(command))
+    print('#' * (len(command) + 6))
+    print(flush=True)
+    err = subprocess.Popen(command).wait()
+    if err != 0:
+        sys.exit(err)
+
+# calculate and store MD5 hash for created executable
+hash_md5 = hashlib.md5()
+with open('dist/instaloader.exe', 'rb') as f:
+    for chunk in iter(lambda: f.read(4096), b''):
+        hash_md5.update(chunk)
+
+with open('dist/instaloader.exe.md5', 'w+') as f:
+    f.write('{} *instaloader.exe\n'.format(hash_md5.hexdigest()))
+
+# Create ZIP file
+shutil.make_archive('instaloader-{}-windows-standalone'.format(os.getenv('VERSION_TAG')), 'zip', 'dist')

deploy/windows/instaloader.spec

@@ -0,0 +1,28 @@
+# -*- mode: python -*-
+
+block_cipher = None
+
+
+a = Analysis(['__main__.py'],
+             binaries=[],
+             datas=[('tasklist.exe', '.'), ('framedyn.dll', '.')],
+             hiddenimports=[],
+             hookspath=[],
+             runtime_hooks=[],
+             excludes=[],
+             win_no_prefer_redirects=False,
+             win_private_assemblies=False,
+             cipher=block_cipher)
+pyz = PYZ(a.pure, a.zipped_data,
+             cipher=block_cipher)
+exe = EXE(pyz,
+          a.scripts,
+          a.binaries,
+          a.zipfiles,
+          a.datas,
+          name='instaloader',
+          debug=False,
+          strip=False,
+          upx=False,
+          runtime_tmpdir=None,
+          console=True , icon='favicon.ico')

docs/_static/instaloader.css

@@ -0,0 +1,75 @@
+@import url(bootstrap-4.1.3.min.css);
+a {
+  color: #008d06; }
+  a:hover, a.current, a.active {
+    color: #f48400; }
+  a.current, a.active {
+    font-weight: 500; }
+
+.doc-sidebar a:hover, .modal-body a:hover {
+  text-decoration: none; }
+
+code {
+  color: #212529; }
+
+.highlight {
+  border-radius: 0.3em; }
+  .highlight pre {
+    padding: 0.7em;
+    color: #fff; }
+  .highlight .c1 {
+    color: #008d06; }
+  .highlight .k, .highlight .kn, .highlight .ow {
+    color: #f48400;
+    font-weight: bold; }
+  .highlight .nb {
+    color: #f48400; }
+  .highlight .s1, .highlight .s2, .highlight .mi {
+    color: #ff7efd; }
+
+.doc-content {
+  padding-top: 0.5em;
+  top: 3.5625em;
+  position: fixed;
+  bottom: 0;
+  overflow-y: auto; }
+
+.doc-sidebar {
+  position: fixed;
+  padding-top: 0.5em;
+  top: 3.5625em;
+  bottom: 0;
+  right: 0;
+  overflow-y: auto; }
+  .doc-sidebar ul {
+    list-style-type: none;
+    padding-left: 1em; }
+    .doc-sidebar ul.current {
+      padding-left: 0; }
+  .doc-sidebar li {
+    padding-top: 0.2em;
+    padding-bottom: 0.2em; }
+
+.admonition {
+  border-radius: 0.3em;
+  border: 1px solid #ccc;
+  background-color: #f8f9fa; }
+  .admonition .admonition-title {
+    color: #ff7efd; }
+
+.topic {
+  border-radius: 0.3em;
+  background-color: #f8f9fa; }
+
+.nav-link {
+  padding-top: 0;
+  padding-bottom: 0; }
+
+.btn-success {
+  background-color: #008d06;
+  border-color: #008d06; }
+  .btn-success:hover {
+    background-color: #f48400;
+    border-color: #f48400; }
+
+/*# sourceMappingURL=instaloader.css.map */

docs/_static/instaloader.js

@@ -0,0 +1,22 @@
+$(function () {
+    $('[data-toggle="tooltip"]').tooltip();
+
+    $('.doc-sidebar > ul > li.current').attr("id", "localtoc");
+    $('#localtoc ul').addClass("nav flex-column");
+    $('#localtoc ul li').addClass("nav-item");
+    $('#localtoc ul li a').addClass("nav-link");
+    $('.doc-content').scrollspy({target: '#localtoc'});
+
+    const top_href = '#' + $('.section:first').attr("id");
+    $('#localtoc > a.current').attr("href", top_href);
+
+    $('#navbarToc a').on("click", function () {
+        const href = $(this).attr("href");
+        if (href === '#') {
+            window.location.href = top_href;
+        } else {
+            window.location.href = href;
+        }
+        $('#navbarToc').modal('hide');
+    });
+});

docs/_static/instaloader.scss

@@ -0,0 +1,117 @@
+@import 'bootstrap-4.1.3.min.css';
+
+$color_instaloader_main: #008d06;
+$color_instaloader_accent: #f48400;
+$color_instaloader_hyper: #ff7efd;
+$color_normal_text: #212529;
+$color_light_bg: #f8f9fa;
+
+a {
+  color: $color_instaloader_main;
+
+  &:hover, &.current, &.active {
+    color: $color_instaloader_accent;
+  }
+
+  &.current, &.active {
+    font-weight: 500;
+  }
+}
+
+.doc-sidebar, .modal-body {
+  a:hover {
+    text-decoration: none
+  }
+}
+
+code {
+  color: $color_normal_text;
+}
+
+.highlight {
+  border-radius: 0.3em;
+
+  pre {
+    padding: 0.7em;
+    color: #fff;
+  }
+
+  .c1 {
+    color: $color_instaloader_main
+  }
+
+  .k, .kn, .ow {
+    color: $color_instaloader_accent;
+    font-weight: bold
+  }
+
+  .nb {
+    color: $color_instaloader_accent
+  }
+
+  .s1, .s2, .mi {
+    color: $color_instaloader_hyper
+  }
+}
+
+.doc-content {
+  padding-top: 0.5em;
+  top: 3.5625em;
+  position: fixed;
+  bottom: 0;
+  overflow-y: auto;
+}
+
+.doc-sidebar {
+  position: fixed;
+  padding-top: 0.5em;
+  top: 3.5625em;
+  bottom: 0;
+  right: 0;
+  overflow-y: auto;
+
+  ul {
+    list-style-type: none;
+    padding-left: 1em;
+
+    &.current {
+      padding-left: 0;
+    }
+  }
+
+  li {
+    padding-top: 0.2em;
+    padding-bottom: 0.2em;
+
+  }
+}
+
+.admonition {
+  border-radius: 0.3em;
+  border: 1px solid #ccc;
+  background-color: $color_light_bg;
+
+  .admonition-title {
+    color: $color_instaloader_hyper;
+  }
+}
+
+.topic {
+  border-radius: 0.3em;
+  background-color: $color_light_bg;
+}
+
+.nav-link {
+  padding-top: 0;
+  padding-bottom: 0;
+}
+
+.btn-success {
+  background-color: $color_instaloader_main;
+  border-color: $color_instaloader_main;
+
+  &:hover {
+    background-color: $color_instaloader_accent;
+    border-color: $color_instaloader_accent;
+  }
+}

docs/_static/style.css

@@ -1,3 +0,0 @@
-code {
-    color: #222;
-}

docs/_templates/layout.html

@@ -1,11 +1,115 @@
 {% extends "!layout.html" %}
-{% block extrahead %}
+
-{{ super() }}
+{% set html5_doctype = True %}
-  <meta name="google-site-verification" content="FNUsDdkOJM9obm9QEdrNBVd9AtAzHzA-aaSMCUrS6C8" />
+
+{% set script_files = script_files + ['_static/bootstrap-4.1.3.bundle.min.js', '_static/instaloader.js'] %}
+
+{% block htmltitle %}
   {% if pagename == "index" %}
-  <link rel="canonical" href="https://instaloader.github.io/" />
+    <title>{{ title|striptags|e }}</title>
   {% else %}
-  <link rel="canonical" href="https://instaloader.github.io/{{ pagename }}.html" />
+    <title>{{ title|striptags|e }}{{ titlesuffix }}</title>
   {% endif %}
-  <script async defer src="https://buttons.github.io/buttons.js"></script>
 {% endblock %}
+
+{% block extrahead %}
+  {{ super() }}
+  <meta name="viewport" content="width=device-width, initial-scale=1, shrink-to-fit=no">
+  <meta name="google-site-verification" content="FNUsDdkOJM9obm9QEdrNBVd9AtAzHzA-aaSMCUrS6C8"/>
+  {% if pagename == "index" %}
+    <link rel="canonical" href="https://instaloader.github.io/"/>
+  {% else %}
+    <link rel="canonical" href="https://instaloader.github.io/{{ pagename }}.html"/>
+  {% endif %}
+{% endblock %}
+
+{% block header %}
+  <nav class="navbar navbar-expand-sm navbar-light bg-light fixed-top border-bottom">
+    <a class="navbar-brand" href="{{ pathto(master_doc) }}">
+      <img src="{{ pathto('_static/' + logo, 1) }}" width="30" height="30" class="d-inline-block align-top" alt="logo">
+      Instaloader
+    </a>
+    <ul class="navbar-nav mr-auto">
+      <li class="nav-item ml-0 ml-sm-2 ml-md-0">
+        <a target="_blank" class="nav-link"
+           href="https://github.com/instaloader/instaloader/releases/tag/v{{ current_release }}"
+           title="Released on {{ current_release_date }}" data-toggle="tooltip">v<b>{{ current_release }}</b></a>
+      </li>
+    </ul>
+    <button class="navbar-toggler" type="button" data-toggle="collapse" data-target="#navbarLinks"
+            aria-controls="navbarLinks" aria-expanded="false" aria-label="Toggle navigation">
+      <span class="navbar-toggler-icon"></span>
+    </button>
+    <div class="collapse navbar-collapse" id="navbarLinks">
+      <ul class="navbar-nav ml-auto">
+        <li class="nav-item">
+          <a target="_blank" class="nav-link" href="https://github.com/instaloader/instaloader/issues">Issues</a>
+        </li>
+        <li class="nav-item">
+          <a target="_blank" class="nav-link" href="https://github.com/instaloader/instaloader/releases">Releases</a>
+        </li>
+        <li class="nav-item">
+          <a target="_blank" class="nav-link" href="https://github.com/instaloader/instaloader">GitHub</a>
+        </li>
+      </ul>
+      <form class="form-inline" action="search.html" method="get">
+        <div class="input-group my-2 my-sm-0 ml-2">
+          <input class="form-control" type="search" placeholder="Search" aria-label="Search" name="q">
+          <div class="input-group-append">
+            <button class="btn btn-success btn-small" type="submit">Go</button>
+          </div>
+        </div>
+      </form>
+    </div>
+    {% if pagename != "index" %}
+      <button type="button" class="btn btn-success d-md-none btn-small ml-2" data-toggle="modal"
+              data-target="#navbarToc">
+        TOC
+      </button>
+    {% endif %}
+  </nav>
+  {% if pagename != "index" %}
+    <div class="modal fade" id="navbarToc" tabindex="-1" role="dialog"
+         aria-hidden="true" aria-labelledby="navbarTocTitle">
+      <div class="modal-dialog modal-dialog-centered" role="document">
+        <div class="modal-content">
+          <div class="modal-header">
+            <h5 class="modal-title" id="navbarTocTitle">Contents</h5>
+            <button type="button" class="close" data-dismiss="modal" aria-label="Close">
+              <span aria-hidden="true">&times;</span>
+            </button>
+          </div>
+          <div class="modal-body">
+            {{ toctree(maxdepth=-1) }}
+          </div>
+        </div>
+      </div>
+    </div>
+  {% endif %}
+{% endblock %}
+
+{% block relbar1 %}{% endblock %}
+{% block relbar2 %}{% endblock %}
+{% block footer %}{% endblock %}
+{% block content %}
+  <div class="container-fluid row">
+    <div class="col-12 {% if pagename != "index" %}col-md-9 col-xl-10{% endif %} doc-content">
+      {% block body %}{% endblock %}
+    </div>
+    {% if pagename != "index" %}
+      <div class="d-none col-md-3 col-xl-2 bg-light doc-sidebar d-md-flex flex-column border-left">
+
+        <h3>{{ _('Table of Contents') }}</h3>
+        {{ toctree(maxdepth=-1) }}
+
+        <div class="d-flex justify-content-between small mt-auto mb-2">
+          <a class="reference internal{% if pagename == "genindex" %} current{% endif %}" href="genindex.html">Index</a>
+          {% if pagename != "genindex" and pagename != "search" %}
+            <a class="reference external"
+               href="https://github.com/instaloader/instaloader/edit/master/docs/{{ pagename }}.rst">Edit this page</a>
+          {% endif %}
+        </div>
+      </div>
+    {% endif %}
+  </div>
+{% endblock %}

docs/_templates/navbar.html

@@ -1,58 +0,0 @@
-  <div id="navbar" class="{{ theme_navbar_class }} navbar-default {% if theme_navbar_fixed_top|tobool -%} navbar-fixed-top{%- endif -%}">
-    <div class="container">
-      <div class="navbar-header">
-        <!-- .btn-navbar is used as the toggle for collapsed navbar content -->
-        <button type="button" class="navbar-toggle" data-toggle="collapse" data-target=".nav-collapse">
-          <span class="icon-bar"></span>
-          <span class="icon-bar"></span>
-          <span class="icon-bar"></span>
-        </button>
-        <a class="navbar-brand" href="{{ pathto(master_doc) }}">
-          {%- block sidebarlogo %}
-            {%- if logo %}<span><img src="{{ pathto('_static/' + logo, 1) }}"></span>{%- endif %}
-          {%- endblock %}
-          {% if theme_navbar_title -%}{{ theme_navbar_title|e }}{%- else -%}{{ project|e }}{%- endif -%}
-        </a>
-        <span class="navbar-text navbar-version pull-left">
-          <a href="https://github.com/instaloader/instaloader/releases/tag/v{{current_release}}" title="Released on {{current_release_date}}.">v<b>{{current_release}}</b></a>
-        </span>
-      </div>
-
-        <div class="collapse navbar-collapse nav-collapse">
-          <ul class="nav navbar-nav">
-            {% if theme_navbar_links %}
-              {%- for link in theme_navbar_links %}
-                <li><a href="{{ pathto(*link[1:]) }}">{{ link[0] }}</a></li>
-              {%- endfor %}
-            {% endif %}
-            {% block navbartoc %}
-              {% include "globaltoc.html" %}
-              {% if theme_navbar_pagenav %}
-                {% include "navbartoc.html" %}
-              {% endif %}
-            {% endblock %}
-            {% if theme_navbar_sidebarrel %}
-              {% block sidebarrel %}
-                {% include "relations.html" %}
-              {% endblock %}
-            {% endif %}
-            </ul>
-            <ul class="nav navbar-nav navbar-right">
-              <li class="dropdown">
-                <a role="button" id="dLabelUsefulLinks" data-toggle="dropdown" data-target="#" href="#">
-                  Useful Links
-                  <b class="caret"></b>
-                </a>
-                <ul class="dropdown-menu" role="menu" aria-labelledby="dLabelUsefulLinks">
-                  <li><a href="https://github.com/instaloader/instaloader/issues">Issue Tracker</a></li>
-                  <li><a href="https://github.com/instaloader/instaloader/releases">Version History</a></li>
-                  <li><a href="https://pypi.python.org/pypi/instaloader">Instaloader on PyPI</a></li>
-                  <li><a href="https://github.com/instaloader/instaloader">Instaloader on GitHub</a></li>
-                  <li role="separator" class="divider"></li>
-                  <li style="padding-left: 20px;"><a class="github-button" href="https://github.com/instaloader/instaloader" data-icon="octicon-star" data-size="large" data-show-count="true" aria-label="Star Instaloader on GitHub">Star</a></li>
-                </ul>
-              </li>
-            </ul>
-        </div>
-    </div>
-  </div>

docs/_templates/search.html

@@ -0,0 +1,28 @@
+{% extends "!search.html" %}
+
+{% block body %}
+  <div id="fallback" class="admonition warning">
+    <script type="text/javascript">$('#fallback').hide();</script>
+    <p>
+      {% trans %}Please activate JavaScript to enable the search
+        functionality.{% endtrans %}
+    </p>
+  </div>
+  {% if search_performed %}
+    <h1>{{ _('Search Results') }}</h1>
+    {% if not search_results %}
+      <p>{{ _('Your search did not match any documents. Please make sure that all words are spelled correctly and that you\'ve selected enough categories.') }}</p>
+    {% endif %}
+  {% endif %}
+  <div id="search-results">
+    {% if search_results %}
+      <ul>
+        {% for href, caption, context in search_results %}
+          <li><a href="{{ pathto(item.href) }}">{{ caption }}</a>
+            <div class="context">{{ context|e }}</div>
+          </li>
+        {% endfor %}
+      </ul>
+    {% endif %}
+  </div>
+{% endblock %}

docs/as-module.rst

@@ -1,8 +1,10 @@
 .. meta::
    :description:
-      Documentation of Instaloader module, a powerful and easy-to-use
+      Documentation of Instaloader module, a powerful and intuitive
       Python library to download Instagram media and metadata.
 
+.. _python-module-instaloader:
+
 Python Module :mod:`instaloader`
 --------------------------------
 
@@ -10,9 +12,12 @@ Python Module :mod:`instaloader`
 
 .. highlight:: python
 
-Instaloader exposes its internally used methods as a Python module, making it a
+.. contents::
-**powerful and easy-to-use Python API for Instagram**, allowing to further
+  :backlinks: none
-customize obtaining media and metadata.
+
+Instaloader exposes its internally used methods and structures as a Python
+module, making it a **powerful and intuitive Python API for Instagram**,
+allowing to further customize obtaining media and metadata.
 
 Start with getting an instance of :class:`Instaloader`::
 
@@ -32,44 +37,68 @@ picture, video or sidecar (set of multiple pictures/videos) posted in a user's
 profile. :class:`Instaloader` provides methods to iterate over Posts from a
 certain source::
 
-    for post in L.get_hashtag_posts('cat'):
+    for post in instaloader.Hashtag.from_name(L.context, 'cat').get_posts():
         # post is an instance of instaloader.Post
         L.download_post(post, target='#cat')
 
-Besides :func:`Instaloader.get_hashtag_posts`, there is
+:class:`Post` instances can be created with:
-:func:`Instaloader.get_feed_posts`, :func:`Instaloader.get_profile_posts` and
-:func:`Instaloader.get_saved_posts`.
-Also, :class:`Post` instances can be created with :func:`Post.from_shortcode`
-and :func:`Post.from_mediaid`.
 
-Further, information about profiles can be easily obtained. For example, you may
+- :func:`Post.from_shortcode`
-print a list of all followees and followers of a profile with::
+   Use a Post shortcode (part of the Post URL,
+   ``https://www.instagram.com/p/SHORTCODE/``) to create a Post
+   object::
 
-    # Print followees
+      post = Post.from_shortcode(L.context, SHORTCODE)
-    print(PROFILE + " follows these profiles:")
-    for f in L.get_followees(PROFILE):
-        print("\t%s\t%s" % (f['username'], f['full_name']))
 
-    # Print followers
+- :meth:`Profile.get_posts`
-    print("Followers of " + PROFILE + ":")
+   All media of a :class:`Profile`.
-    for f in L.get_followers(PROFILE):
-        print("\t%s\t%s" % (f['username'], f['full_name']))
 
-Then, you may download all pictures of all followees with::
+- :meth:`Profile.get_saved_posts`
+   Media that the user marked as saved (:class:`Profile` must be own profile
+   for this to work)
 
-    for f in L.get_followees(PROFILE):
+- :meth:`Instaloader.get_feed_posts`
-        L.download_profile(f['username'])
+   Media in the user's feed.
 
-Each Instagram profile has its own unique ID which stays unmodified even if a
+- :meth:`Instaloader.get_explore_posts`
-user changes his/her username. To get said ID, given the profile's name, you may
+   Media that is suggested by Instagram to explore.
-call::
 
-    L.get_id_by_username(PROFILE_NAME)
+- :meth:`Hashtag.get_posts`
+   Media associated with given hashtag.
+
+With the :class:`Profile` class, Instaloader also makes it easy to access
+metadata of a Profile. :class:`Profile` instances can be created with:
+
+- :meth:`Profile.from_username`::
+
+     profile = Profile.from_username(L.context, USERNAME)
+
+- :meth:`Profile.from_id`
+   given its User ID. This allows to easily lookup a Profile's username given
+   its ID::
+
+      Profile.from_id(L.context, USERID).username
+
+- :meth:`Profile.get_followees`
+   Profiles that are followed by given user.
+
+- :meth:`Profile.get_followers`
+   Profiles that follow given user.
+
+- :attr:`Post.owner_profile`, :attr:`Story.owner_profile` and :attr:`StoryItem.owner_profile`
+   Owner profile of particular object.
+
+- :meth:`Post.get_likes`
+   Profiles that liked a given :class:`Post`
+
+- :attr:`PostComment.owner` attribute for comment in :meth:`Post.get_comments`
+   Profile of a Post comment.
 
 A reference of the many methods provided by the :mod:`instaloader` module is
-provided in the remainder of this document. Feel free to direct any issue or
+provided in the remainder of this document.
-contribution to
+
-`Instaloader on Github <https://github.com/instaloader/instaloader>`__.
+For a list of real code examples using the Instaloader module for advanced
+tasks, see :ref:`codesnippets`.
 
 ``Instaloader`` (Main Class)
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^
@@ -77,28 +106,96 @@ contribution to
 .. autoclass:: Instaloader
    :no-show-inheritance:
 
-``Post`` Class
+Instagram Structures
-^^^^^^^^^^^^^^
+^^^^^^^^^^^^^^^^^^^^
+
+Posts
+"""""
 
 .. autoclass:: Post
    :no-show-inheritance:
 
-Miscellaneous Functions
+Additionally, the following trivial structures are defined:
-^^^^^^^^^^^^^^^^^^^^^^^
 
-.. autofunction:: shortcode_to_mediaid
+.. autoclass:: PostSidecarNode
+   :no-show-inheritance:
 
-.. autofunction:: mediaid_to_shortcode
+.. autoclass:: PostComment
+   :no-show-inheritance:
 
-.. autoclass:: Tristate
+.. autoclass:: PostCommentAnswer
+   :no-show-inheritance:
+
+.. autoclass:: PostLocation
+   :no-show-inheritance:
+
+User Stories
+""""""""""""
+
+.. autoclass:: Story
+   :no-show-inheritance:
+
+.. autoclass:: StoryItem
+   :no-show-inheritance:
+
+Highlights
+""""""""""
+
+.. autoclass:: Highlight
+   :no-show-inheritance:
+   :inherited-members:
+
+   Bases: :class:`Story`
+
+   .. versionadded:: 4.1
+
+Profiles
+""""""""
+
+.. autoclass:: Profile
+   :no-show-inheritance:
+
+Hashtags
+""""""""
+
+.. autoclass:: Hashtag
+   :no-show-inheritance:
+
+   .. versionadded:: 4.4
+
+TopSearchResults
+""""""""""""""""
+
+.. autoclass:: TopSearchResults
+   :no-show-inheritance:
+
+   .. versionadded:: 4.3
+
+Loading and Saving
+""""""""""""""""""
+
+:class:`Post`, :class:`StoryItem`, :class:`Profile` and :class:`Hashtag` can be saved and loaded
+to/from JSON files.
+
+.. autofunction:: load_structure_from_file
+
+.. autofunction:: save_structure_to_file
 
 Exceptions
 ^^^^^^^^^^
 
+.. currentmodule:: instaloader.exceptions
+
 .. autoexception:: InstaloaderException
    :no-show-inheritance:
 
-.. autoexception:: QueryReturnedNotFoundException
+.. autoexception:: ConnectionException
+
+.. currentmodule:: instaloader
+
+.. autoexception:: QueryReturnedBadRequestException
+
+.. autoexception:: QueryReturnedForbiddenException
 
 .. autoexception:: ProfileNotExistsException
 
@@ -108,10 +205,29 @@ Exceptions
 
 .. autoexception:: LoginRequiredException
 
+.. autoexception:: TwoFactorAuthRequiredException
+
+   .. versionadded:: 4.2
+
 .. autoexception:: InvalidArgumentException
 
 .. autoexception:: BadResponseException
 
 .. autoexception:: BadCredentialsException
 
-.. autoexception:: ConnectionException
+.. autoexception:: PostChangedException
+
+.. autoexception:: QueryReturnedNotFoundException
+
+   .. versionchanged:: 4.3
+      QueryReturnedNotFoundException now inherits ConnectionException
+      to retry on 404 errors.
+
+
+.. autoexception:: TooManyRequestsException
+
+``InstaloaderContext`` (Low-level functions)
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+.. autoclass:: InstaloaderContext
+   :no-show-inheritance:

docs/basic-usage.rst

@@ -5,11 +5,16 @@
       and private profiles, hashtags, stories, feeds, saved media, and
       their metadata, comments and captions.
 
+.. _download-pictures-from-instagram:
+
 Download Pictures from Instagram
 ---------------------------------
 
 .. highlight:: none
 
+Here we describe how to use Instaloader to download pictures from Instagram. If
+you do not have Instaloader installed yet, see :ref:`install`.
+
 .. NOTE that Section "Basic Usage" is duplicated in README.rst.
 
 Basic Usage
@@ -32,7 +37,9 @@ To later **update your local copy** of that profiles, you may run
     instaloader --fast-update profile [profile ...]
 
 If :option:`--fast-update` is given, Instaloader stops when arriving at the
-first already-downloaded picture. When updating profiles, Instaloader
+first already-downloaded picture.
+
+When updating profiles, Instaloader
 automatically **detects profile name changes** and renames the target directory
 accordingly.
 
@@ -53,40 +60,76 @@ already have a valid session cookie file.
 What to Download
 ^^^^^^^^^^^^^^^^
 
+.. targets-start
+
 Instaloader supports the following targets:
 
-``profile``
+- ``profile``
-   Public profile, or private profile with :option:`--login`. For each profile
+   Public profile, or private profile with :option:`--login`.
-   you download, :option:`--stories` instructs Instaloader to also
-   **download the user's stories**.
 
-``"#hashtag"``
+   If an already-downloaded profile has been renamed, Instaloader automatically
-   Posts with a certain **hashtag** (the quotes are usually neccessary),
+   finds it by its unique ID and renames the folder accordingly.
 
-``:stories``
+   Besides the profile's posts, its current profile picture is downloaded. For
+   each profile you download,
+
+   - :option:`--stories`
+      instructs Instaloader to also **download the user's stories**,
+
+   - :option:`--highlights`
+      to **download highlights of each profile that is downloaded**,
+
+   - :option:`--tagged`
+      to **download posts where the user is tagged**, and
+
+   - :option:`--igtv`
+      to **download IGTV videos**.
+
+- ``"#hashtag"``
+   Posts with a certain **hashtag** (the quotes are usually necessary),
+
+- ``%location id``
+   Posts tagged with a given location; the location ID is the numerical ID
+   Instagram labels a location with (e.g.
+   \https://www.instagram.com/explore/locations/**362629379**/plymouth-naval-memorial/).
+   Requires :option:`--login`.
+
+   .. versionadded:: 4.2
+
+- ``:stories``
    The currently-visible **stories** of your followees (requires
    :option:`--login`),
 
-``:feed``
+- ``:feed``
    Your **feed** (requires :option:`--login`),
 
-``:saved``
+- ``:saved``
    Posts which are marked as **saved** (requires :option:`--login`),
 
-``@profile``
+- ``@profile``
    All profiles that are followed by ``profile``, i.e. the *followees* of
    ``profile`` (requires :option:`--login`).
 
+- ``-post``
+   The single **post** with the given shortcode. Must be preceeded by ``--`` in
+   the argument list to not be mistaken as an option flag.
+
+   .. versionadded:: 4.1
+
+.. targets-end
+
 Instaloader goes through all media matching the specified targets and
 downloads the pictures and videos and their captions. You can specify
 
-- :option:`--comments`, to also **download comments** of each post,
+- :option:`--comments`
+   also **download comments** of each post,
 
-- :option:`--geotags`, to **download geotags** of each post and save them as
+- :option:`--geotags`
-  Google Maps link,
+   **download geotags** of each post and save them as
+   Google Maps link (requires :option:`--login`),
 
-- :option:`--metadata-json`, to store further post metadata in a separate JSON
+For a reference of all supported command line options, see
-  file.
+:ref:`command-line-options`.
 
 .. _filename-specification:
 
@@ -94,7 +137,7 @@ Filename Specification
 ^^^^^^^^^^^^^^^^^^^^^^
 
 For each target, Instaloader creates a directory named after the target,
-i.e. ``profile``, ``#hashtag``, ``:feed``, etc. and therein saves the
+i.e. ``profile``, ``#hashtag``, ``%location id``, ``:feed``, etc. and therein saves the
 posts in files named after the post's timestamp.
 
 :option:`--dirname-pattern` allows to configure the directory name of each
@@ -103,60 +146,171 @@ pattern, the token ``{target}`` is replaced by the target name, and
 ``{profile}`` is replaced by the owner of the post which is downloaded.
 
 :option:`--filename-pattern` configures the path of the post's files relative
-to the target directory. The default is ``--filename-pattern={date}``.
+to the target directory that is specified with :option:`--dirname-pattern`.
+The default is ``--filename-pattern={date_utc}_UTC``.
 The tokens ``{target}`` and ``{profile}`` are replaced like in the
-dirname pattern. Further, the tokens ``{date}``, ``{date_utc}`` and ``{shortcode}`` are
+dirname pattern. The following tokens are defined for usage with
-defined. Additionally, in case of not downloading stories, the attributes of
+:option:`--filename-pattern`:
-:class:`.Post` can be used, e.g. ``{post.owner_id}`` or ``{post.mediaid}``.
 
-For example, encode the poster's profile name in the filenames with:
+- ``{target}``
+   Target name (as given in Instaloader command line)
 
-::
+- ``{profile}`` (same as ``{owner_username}``)
+   Owner of the Post / StoryItem.
 
-    instaloader --filename-pattern={date}_{profile} "#hashtag"
+- ``{owner_id}``
+   Unique integer ID of owner profile.
 
-The pattern string is formatted with Python's string formatter. This
+- ``{shortcode}``
-gives additional flexibilty for pattern specification. For example,
+   Shortcode (identifier string).
-`strftime-style formatting options <https://docs.python.org/3/library/datetime.html#strftime-and-strptime-behavior>`__
+
-are supported for the post's
+- ``{mediaid}``
-timestamp. The default for ``{date}`` is ``{date:%Y-%m-%d_%H-%M-%S}``.
+   Integer representation of shortcode.
+
+- ``{date_utc}`` (same as ``{date}``)
+   Creation time in UTC timezone.
+   `strftime()-style formatting options <https://docs.python.org/3/library/datetime.html#strftime-and-strptime-behavior>`__
+   are supported as format specifier. The default date format specifier used by
+   Instaloader is::
+
+      {date_utc:%Y-%m-%d_%H-%M-%S}
+
+For example, encode the poster's profile name in the filenames with::
+
+    instaloader --filename-pattern={date_utc}_UTC_{profile} "#hashtag"
+
+As another example, you may instruct Instaloader to store posts in a
+``PROFILE/YEAR/SHORTCODE.jpg`` directory structure::
+
+    instaloader --dirname-pattern={profile} --filename-pattern={date_utc:%Y}/{shortcode} <target> ...
 
 .. _filter-posts:
 
 Filter Posts
 ^^^^^^^^^^^^
 
-The :option:`--only-if` option allows to specify criterias that posts have to
+.. py:currentmodule:: instaloader
-meet to be downloaded. If not given, all posts are downloaded. It must be a
-boolean Python expression where the variables :attr:`.likes`, :attr:`.comments`,
-:attr:`.viewer_has_liked`, :attr:`.is_video`, and many more are defined.
 
-A few examples:
+The options :option:`--post-filter` and :option:`--storyitem-filter`
+allows to specify criteria that posts or story items have to
+meet to be downloaded. If not given, all posts are downloaded.
 
-To **download the pictures from your feed that you have liked**:
+The filter string must be a
+`Python boolean expression <https://docs.python.org/3/reference/expressions.html#boolean-operations>`__
+where the attributes from :class:`Post` or
+:class:`StoryItem` respectively are defined.
 
-::
+Id est, the following attributes can be used with both
+:option:`--post-filter` and :option:`--storyitem-filter`:
 
-    instaloader --login=your_username --only-if=viewer_has_liked :feed
+- :attr:`~Post.owner_username` (str), :attr:`~Post.owner_id` (int)
+   Owner profile username / userid.
 
-Or you might only want to download **posts that either you liked or were
+- :attr:`~Post.date_utc` (datetime), :attr:`~Post.date_local` (datetime)
-liked by many others**:
+   Creation timestamp. Since :class:`~datetime.datetime` objects can be created
+   inside filter strings, this easily allows filtering by creation date. E.g.::
 
-::
+      instaloader --post-filter="date_utc <= datetime(2018, 5, 31)" target
 
-    instaloader --login=your_username --only-if="likes>100 or viewer_has_liked" profile
+- :attr:`~Post.is_video` (bool)
+   Post/StoryItem is a video. For example, you may skip videos::
 
-Or you may **skip videos**:
+      instaloader --post-filter="not is_video" target
 
-::
+   This is not the same as :option:`--no-videos` and
+   :option:`--no-video-thumbnails`, since sidecar posts (posts that contain
+   multiple pictures/videos in one post) have this attribute set to False.
 
-    instaloader --only-if="not is_video" target
+As :option:`--post-filter`, the following attributes can be used additionally:
 
-Or you may filter by hashtags that occur in the Post's caption. For
+- :attr:`~Post.viewer_has_liked` (bool)
-example, to download posts of kittens that are cute: ::
+   Whether user (with :option:`--login`) has liked given post. To download the
+   pictures from your feed that you have liked::
 
-    instaloader --only-if="'cute' in caption_hashtags" "#kitten"
+      instaloader --login=your_username --post-filter=viewer_has_liked :feed
 
-The given string is evaluated as a
+- :attr:`~Post.likes` (int), :attr:`~Post.comments` (int)
-`Python boolean expression <https://docs.python.org/3/reference/expressions.html#boolean-operations>`__,
+   Likes count / Comments count. You might only want to download posts that
-where all occuring variables are attributes of the :class:`.Post` class.
+   either you liked or were liked by many others**::
+
+      instaloader --login=your_username --post-filter="likes>100 or viewer_has_liked" profile
+
+- :attr:`~Post.caption_hashtags` (list of str) / :attr:`~Post.caption_mentions` (list of str)
+   ``#hashtags`` or ``@mentions`` (lowercased) in the Post's caption. For example, to
+   download posts of kittens that are cute::
+
+       instaloader --post-filter="'cute' in caption_hashtags" "#kitten"
+
+- :attr:`~Post.tagged_users` (list of str)
+   Lowercased usernames that are tagged in the Post.
+
+For :option:`--storyitem-filter`, the following additional attributes are
+defined:
+
+- :attr:`~StoryItem.expiring_utc` (datetime) / :attr:`~StoryItem.expiring_local` (datetime)
+   Timestamp when StoryItem will get unavailable.
+
+.. _metadata-text-files:
+
+Metadata Text Files
+^^^^^^^^^^^^^^^^^^^
+
+Unless :option:`--no-captions` is given, Instaloader creates a ``.txt`` file
+along with each post where the Post's caption is saved.
+
+You can customize what metadata to save for each Post or StoryItem with
+:option:`--post-metadata-txt` and :option:`--storyitem-metadata-txt`. The
+default is ``--post-metadata-txt={caption}`` and no storyitem metadata txt.
+These strings are formatted similar as in the :ref:`filename-specification` and
+the result is saved in text files, unless it is empty.
+
+Specifying these options multiple times results in output having multiple lines,
+in the order they were given to Instaloader.
+
+The field names are evaluated to :class:`Post` or :class:`StoryItem` attributes,
+and as such, the same fields are supported as in :ref:`filename-specification`
+and :ref:`filter-posts`.
+
+For example, to save the current number of likes for each post, rather than
+the post's caption::
+
+   instaloader --post-metadata-txt="{likes} likes." <target>
+
+Note that with this feature, it is possible to easily and fastly extract
+additional metadata of already-downloaded posts, by reimporting their JSON
+files. Say, you now also want to export the number of comments the Posts had
+when they were downloaded::
+
+   instaloader --post-metadata-txt="{likes} likes, {comments} comments." <target>/*.json.xz
+
+.. _instaloader-as-cronjob:
+
+Instaloader as Cronjob
+^^^^^^^^^^^^^^^^^^^^^^
+
+Instaloader is suitable for running as a cronjob to periodically update your
+personal Instagram archive. The :option:`--quiet` option disables user
+interactions and logging of non-error messages. To non-interactively use
+Instaloader logged-in, create a session file, e.g. in your home directory::
+
+   instaloader --login=your_username --sessionfile=~/.instaloader-session
+
+Then use the same parameters in your cronjob to load the session and download
+the given targets::
+
+   instaloader --login=your_username --sessionfile=~/.instaloader-session --quiet <target> [...]
+
+Without :option:`--sessionfile` option, Instaloader saves the session file in
+a path within your temporary directory.
+
+Programming Instaloader
+^^^^^^^^^^^^^^^^^^^^^^^
+
+If your task cannot be done with the command line interface of Instaloader,
+consider taking a look at the :ref:`python-module-instaloader`.
+Instaloader exposes its internally used methods and structures, making it a
+powerful and intuitive Python API for Instagram, allowing to further customize
+obtaining media and metadata.
+
+Also see :ref:`codesnippets`, where we collect example scripts that use
+Instaloader to achieve more complex tasks.

docs/cli-options.rst

@@ -1,3 +1,5 @@
+.. _command-line-options:
+
 Command Line Options
 ====================
 
@@ -6,32 +8,48 @@ Instaloader is invoked with::
    $ instaloader [options] target [target ...]
 
 where ``target`` is a ``profile``, a ``"#hashtag"``, ``@profile`` (all profiles
-that *profile* is following), or if logged in ``:feed`` (pictures from your
+that *profile* is following), ``%location ID``, or if logged in ``:feed`` (pictures from your
 feed), ``:stories`` (stories of your followees) or ``:saved`` (collection of
 posts marked as saved).
 
 Here we explain the additional options that can be given to Instaloader to
-customize its behavior.  To get a list of all flags, their abbreviations and
+customize its behavior.  For an
-their descriptions, you may also run ``instaloader --help``.
+introduction on how to use Instaloader, see
+:ref:`download-pictures-from-instagram`.
 
-What to Download
+To get a list of all flags, their abbreviations and
-^^^^^^^^^^^^^^^^
+their descriptions, you may also run::
 
-Specify a list of targets (profiles, #hashtags, ``:feed``, ``:stories`` or
+   instaloader --help
-``:saved``). For each of these, Instaloader creates a folder and stores all
-posts along with the pictures's captions and the current **profile picture**
-there. If an already-downloaded profile has been renamed, Instaloader
-automatically **finds it by its unique ID** and renames the folder likewise.
 
-.. option:: --profile-pic-only
+Targets
+^^^^^^^
 
-   Only download profile picture.
+Specify a list of targets. For each of these, Instaloader creates a folder and
+stores all posts along with the pictures's captions there.
 
-.. option:: --no-profile-pic
+.. include:: basic-usage.rst
+   :start-after: targets-start
+   :end-before: targets-end
 
-   Do not download profile picture.
+- ``filename.json[.xz]``
+   Re-Download the given object
 
-.. option:: --no-videos
+- ``+args.txt``
+   Read targets (and options) from given textfile. See :option:`+args.txt`.
+
+What to Download of each Post
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+.. option:: --no-pictures
+
+   Do not download post pictures. Cannot be used together with
+   :option:`--fast-update`. Implies :option:`--no-video-thumbnails`, does not
+   imply :option:`--no-videos`.
+
+   .. versionadded:: 4.1
+
+.. option:: --no-videos, -V
 
    Do not download videos.
 
@@ -39,19 +57,13 @@ automatically **finds it by its unique ID** and renames the folder likewise.
 
    Do not download thumbnails of videos.
 
-.. option:: --geotags
+.. option:: --geotags, -G
 
    **Download geotags** when available. Geotags are stored as a text file with
    the location's name and a Google Maps link. This requires an additional
-   request to the Instagram server for each picture, which is why it is disabled
+   request to the Instagram server for each picture. Requires :option:`--login`.
-   by default.
 
-.. option:: --no-geotags
+.. option:: --comments, -C
-
-   Do not store geotags, even if they can be obtained without any additional
-   request.
-
-.. option:: --comments
 
    Download and update comments for each post. This requires an additional
    request to the Instagram server for each post, which is why it is disabled by
@@ -59,52 +71,112 @@ automatically **finds it by its unique ID** and renames the folder likewise.
 
 .. option:: --no-captions
 
-   Do not store media captions, although no additional request is needed to
+   Do not create txt files.
-   obtain them.
 
-.. option:: --stories
+.. option:: --post-metadata-txt
+
+   Template to write in txt file for each Post. See :ref:`metadata-text-files`.
+
+.. option:: --storyitem-metadata-txt
+
+   Template to write in txt file for each StoryItem. See
+   :ref:`metadata-text-files`.
+
+.. option:: --no-metadata-json
+
+   Do not create a JSON file containing the metadata of each post.
+
+.. option:: --no-compress-json
+
+   Do not xz compress JSON files, rather create pretty formatted JSONs.
+
+
+What to Download of each Profile
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+.. option:: --profile-pic-only, -P
+
+   .. deprecated:: 4.1
+      Use :option:`--no-posts`.
+
+   Only download profile picture.
+
+.. option:: --no-posts
+
+   Do not download regular posts.
+
+   .. versionadded:: 4.1
+
+.. option:: --no-profile-pic
+
+   Do not download profile picture.
+
+.. option:: --stories, -s
 
    Also **download stories** of each profile that is downloaded. Requires
    :option:`--login`.
 
-.. option:: --metadata-json
+.. option:: --highlights
 
-   Create a JSON file containing the metadata of each post. This does not
+   Also **download highlights** of each profile that is downloaded. Requires
-   include comments (see :option:`--comments`) nor geotags (see
+   :option:`--login`.
-   :option:`--geotags`). The JSON files contain the properties of
+
-   :class:`instaloader.Post`.
+   .. versionadded:: 4.1
+
+.. option:: --tagged
+
+   Also download posts where each profile is tagged.
+
+   .. versionadded:: 4.1
+
+.. option:: --igtv
+
+   Also download IGTV videos.
+
+   .. versionadded:: 4.3
 
 .. option:: --stories-only
 
+   .. deprecated:: 4.1
+      Use :option:`--stories` :option:`--no-posts`.
+
    Rather than downloading regular posts of each specified profile, only
    download stories.  Requires :option:`--login`. Does not imply
    :option:`--no-profile-pic`.
 
-.. option:: --only-if filter
+   .. note::
 
-   Expression that, if given, must evaluate to True for each post to be
+      If possible, use ``:stories`` target rather than :option:`--stories-only`
-   downloaded.  Must be a syntactically valid Python expression. Variables are
+      with all your followees. ``:stories`` uses fewer API requests.
-   evaluated to :class:`instaloader.Post` attributes.  Example:
-   ``--only-if=viewer_has_liked``. See :ref:`filter-posts` for more
-   examples.
 
+Which Posts to Download
+^^^^^^^^^^^^^^^^^^^^^^^
 
-When to Stop Downloading
+.. option:: --fast-update, -F
-^^^^^^^^^^^^^^^^^^^^^^^^
-
-If none of these options are given, Instaloader goes through all pictures
-matching the specified targets.
-
-.. option:: --fast-update
 
    For each target, stop when encountering the first already-downloaded picture.
    This flag is recommended when you use Instaloader to update your personal
    Instagram archive.
 
-.. option:: --count COUNT
+.. option:: --post-filter filter, --only-if filter
+
+   Expression that, if given, must evaluate to True for each post to be
+   downloaded.  Must be a syntactically valid Python expression. Variables are
+   evaluated to :class:`instaloader.Post` attributes.  Example:
+   ``--post-filter=viewer_has_liked``. See :ref:`filter-posts` for more
+   examples.
+
+.. option:: --storyitem-filter filter
+
+   Expression that, if given, must evaluate to True for each storyitem to be
+   downloaded.  Must be a syntactically valid Python expression. Variables are
+   evaluated to :class:`instaloader.StoryItem` attributes.
+   See :ref:`filter-posts` for more examples.
+
+.. option:: --count COUNT, -c
 
    Do not attempt to download more than COUNT posts.  Applies only to
-   ``#hashtag`` and ``:feed``.
+   ``#hashtag``, ``%location id``, and ``:feed``.
 
 
 Login (Download Private Profiles)
@@ -115,21 +187,25 @@ profiles. To login, pass the :option:`--login` option. Your session cookie (not
 password!) will be saved to a local file to be reused next time you want
 Instaloader to login.
 
-.. option:: --login YOUR-USERNAME
+.. option:: --login YOUR-USERNAME, -l YOUR-USERNAME
 
    Login name (profile name) for your Instagram account.
 
-.. option:: --sessionfile SESSIONFILE
+.. option:: --sessionfile SESSIONFILE, -f SESSIONFILE
 
    Path for loading and storing session key file.  Defaults to a path within
    your temporary directory, encoding your local username and your Instagram
    profile name.
 
-.. option:: --password YOUR-PASSWORD
+.. option:: --password YOUR-PASSWORD, -p YOUR-PASSWORD
 
    Password for your Instagram account.  Without this option, you'll be prompted
    for your password interactively if there is not yet a valid session file.
 
+   .. warning:: Using :option:`--password` option is discouraged for security
+      reasons.  Enter your password interactively when asked, or use the
+      sessionfile feature (:option:`--sessionfile` to customize path).
+
 How to Download
 ^^^^^^^^^^^^^^^
 
@@ -142,15 +218,11 @@ How to Download
 
 .. option:: --filename-pattern FILENAME_PATTERN
 
-   Prefix of filenames. Posts are stored in the directory whose pattern is given
+   Prefix of filenames, relative to the directory given with
-   with ``--dirname-pattern``.  ``{profile}`` is replaced by the profile name,
+   :option:`--dirname-pattern`. ``{profile}`` is replaced by the profile name,
    ``{target}`` is replaced by the target you specified, i.e.  either ``:feed``,
-   ``#hashtag`` or the profile name.  Also, the fields ``{date}``, ``{date_utc}`` and
+   ``#hashtag`` or the profile name. Defaults to ``{date_utc}_UTC``.
-   ``{shortcode}`` can be specified.  In case of not downloading stories, the
+   See :ref:`filename-specification` for a list of supported tokens.
-   attributes of the :class:`.Post` class can be used in addition, e.g.
-   ``{post.owner_id}`` or ``{post.mediaid}``.
-   Defaults to
-   ``{date:%Y-%m-%d_%H-%M-%S}``.  See :ref:`filename-specification`.
 
 .. option:: --user-agent USER_AGENT
 
@@ -163,11 +235,45 @@ How to Download
    to ``3``. If a connection fails, it can be manually skipped by hitting
    :kbd:`Control-c`. Set this to ``0`` to retry infinitely.
 
+.. option:: --commit-mode
+
+   Tries to ensure downloaded images avoid corruption in case of unexpected
+   interruption. If the last picture is corrupted, Instaloader will fix the
+   picture the next time it is run.
+
+   .. versionadded:: 4.2
+
+.. option:: --request-timeout N
+
+   Seconds to wait before timing out a connection request.
+
+   .. versionadded:: 4.3
+
 Miscellaneous Options
 ^^^^^^^^^^^^^^^^^^^^^
 
-.. option:: --quiet
+.. option:: --quiet, -q
 
    Disable user interaction, i.e. do not print messages (except errors) and fail
-   if login credentials are needed but not given. This makes Instaloader
+   if login credentials are needed but not given.
-   **suitable as a cron job**.
+   This is handy for running :ref:`instaloader-as-cronjob`.
+
+.. option:: +args.txt
+
+   Read arguments from file `args.txt`, a shortcut to provide arguments from
+   file rather than command-line. This provides a convenient way to hide login
+   info from CLI, and can also be used to simplify managment of long arguments.
+
+   .. note::
+
+      Text file should separate arguments with line breaks.
+
+      args.txt example::
+
+         --login=MYUSERNAME
+         --password=MYPASSWORD
+         --fast-update
+         profile1
+         profile2
+
+   .. versionadded:: 4.1

docs/codesnippets.rst

@@ -0,0 +1,140 @@
+.. _codesnippets:
+
+Advanced Instaloader Examples
+=============================
+
+.. currentmodule:: instaloader
+
+.. highlight:: python
+
+.. contents::
+   :backlinks: none
+
+Here we present code examples that use the :ref:`python-module-instaloader` for
+more advanced Instagram downloading or metadata mining than what is possible
+with the Instaloader command line interface.
+
+The scripts presented here can be downloaded from our source tree:
+`instaloader/docs/codesnippets/ <https://github.com/instaloader/instaloader/tree/master/docs/codesnippets>`__
+
+.. For each code snippet:
+   - title
+   - brief description of what it does / motivation / how it works
+   - code, or link to code
+   - link to discussion issue
+   - link used methods
+
+Download Posts in a Specific Period
+-----------------------------------
+
+To only download Instagram pictures (and metadata) that are within a specific
+period, you can play around with :func:`~itertools.dropwhile` and
+:func:`~itertools.takewhile` from :mod:`itertools` like in this snippet.
+
+.. literalinclude:: codesnippets/121_since_until.py
+
+See also :class:`Post`, :meth:`Instaloader.download_post`.
+
+Discussed in :issue:`121`.
+
+Likes of a Profile / Ghost Followers
+------------------------------------
+
+To obtain a list of your inactive followers, i.e. followers that did not like
+any of your pictures, into a file you can use this approach.
+
+.. literalinclude:: codesnippets/120_ghost_followers.py
+
+See also :meth:`Profile.get_posts`, :meth:`Post.get_likes`,
+:meth:`Profile.get_followers`, :meth:`Instaloader.load_session_from_file`,
+:meth:`Profile.from_username`.
+
+Discussed in :issue:`120`.
+
+Track Deleted Posts
+-------------------
+
+This script uses Instaloader to obtain a list of currently-online Instagram and
+compares it with the set of posts that you already have downloaded.  It outputs
+a list of posts which are online but not offline (i.e. not yet downloaded) and a
+list of posts which are offline but not online (i.e. deleted in the profile).
+
+.. literalinclude:: codesnippets/56_track_deleted.py
+
+See also :func:`load_structure_from_file`, :meth:`Profile.from_username`,
+:meth:`Profile.get_posts`, :class:`Post`.
+
+Discussed in :issue:`56`.
+
+Only one Post per User
+----------------------
+
+To download only the one most recent post from each user, this snippet creates a
+:class:`set` that contains the users of which a post has already been
+downloaded. While iterating the posts, it checks whether the post's owner
+already is in the set. If not, the post is downloaded from Instagram and the
+user is added to that set.
+
+.. literalinclude:: codesnippets/113_only_one_per_user.py
+
+See also :class:`Post`, :meth:`Instaloader.download_post`,
+:attr:`Post.owner_profile`, :class:`Profile`.
+
+Discussed in :issue:`113`.
+
+Top X Posts of User
+-------------------
+
+With Instaloader, it is easy to download the few most-liked pictres of a user.
+
+.. literalinclude:: codesnippets/194_top_x_of_user.py
+
+Discussed in :issue:`194`.
+
+Upgrade Images by Local Copies
+------------------------------
+
+The following script finds local versions of images fetched by Instaloader, in
+order to upgrade the downloaded images by locally-found versions with better
+quality. It uses image hashing to identify similar images.
+
+`updgrade-instaloader-images.py <https://gist.github.com/pavelkryukov/15f93d19a99428a284a8bcec27e0187b>`__ (external link to GitHub Gist)
+
+Discussed in :issue:`46`.
+
+Add Captions to Images
+----------------------
+
+Instaloader does not modify the downloaded JPEG file. However, one could combine
+it with an imaging library such as Pillow or PIL to render the caption on
+Instagram pictures.  The following shows an approach.
+
+.. literalinclude:: codesnippets/110_pil_captions.py
+
+See also :attr:`Post.caption`, :attr:`Post.url`, :meth:`Post.from_shortcode`,
+:func:`load_structure_from_file`.
+
+Discussed in :issue:`110`.
+
+Metadata JSON Files
+-------------------
+
+The JSON files Instaloader saves along with each Post contain all the metadata
+that has been retrieved from Instagram while downloading the picture and
+associated required information.
+
+With `jq <https://stedolan.github.io/jq/>`__, a command-line JSON processor, the
+metadata can be easily post-processed. For example, Instaloader's JSON files can
+be pretty-formatted with:
+
+.. code-block:: none
+
+   xzcat 2018-05-13_11-18-45_UTC.json.xz | jq .node
+
+However, Instaloader tries to do as few metadata requests as possible, so,
+depending on how Instaloader has been invoked, it may occur that these files do
+not contain the complete available metadata structure. Nevertheless, the file
+can be loaded into Instaloader with :func:`load_structure_from_file` and the
+required metadata then be accessed via the :class:`Post` or :class:`Profile`
+attributes, which trigger an Instagram request if that particular information is
+not present in the JSON file.

docs/codesnippets/110_pil_captions.py

@@ -0,0 +1,20 @@
+from io import BytesIO
+
+from requests import get
+from PIL import Image, ImageDraw
+from instaloader import *
+
+L = Instaloader()
+
+# Load Post instance
+post = load_structure_from_file(L.context, '2017-10-01_18-53-03_UTC.json.xz')
+# or post = Post.from_shortcode(L.context, SHORTCODE)
+
+# Render caption
+image = Image.open(BytesIO(get(post.url).content))
+draw = ImageDraw.Draw(image)
+color = 'rgb(0, 0, 0)'  # black color
+draw.text((300,100), post.caption.encode('latin1', errors='ignore'), fill=color)
+
+# Save image
+image.save('test.jpg')

docs/codesnippets/113_only_one_per_user.py

@@ -0,0 +1,14 @@
+import instaloader
+
+L = instaloader.Instaloader()
+
+posts = instaloader.Hashtag.from_name(L.context, 'urbanphotography').get_posts()
+
+users = set()
+
+for post in posts:
+    if not post.owner_profile in users:
+        L.download_post(post, '#urbanphotography')
+        users.add(post.owner_profile)
+    else:
+        print("{} from {} skipped.".format(post, post.owner_profile))

docs/codesnippets/120_ghost_followers.py

@@ -0,0 +1,27 @@
+import instaloader
+
+L = instaloader.Instaloader()
+
+USER = 'your_account'
+PROFILE = USER
+
+# Your preferred way of logging in:
+L.load_session_from_file(USER)
+
+profile = instaloader.Profile.from_username(L.context, PROFILE)
+
+likes = set()
+print('Fetching likes of all posts of profile {}.'.format(profile.username))
+for post in profile.get_posts():
+    print(post)
+    likes = likes | set(post.get_likes())
+
+print('Fetching followers of profile {}.'.format(profile.username))
+followers = set(profile.get_followers())
+
+ghosts = followers - likes
+
+print('Storing ghosts into file.')
+with open('/YOUR PATH/inactive-users.txt', 'w') as f:
+    for ghost in ghosts:
+        print(ghost.username, file=f)

docs/codesnippets/121_since_until.py

@@ -0,0 +1,17 @@
+from datetime import datetime
+from itertools import dropwhile, takewhile
+
+import instaloader
+
+L = instaloader.Instaloader()
+
+posts = instaloader.Hashtag.from_name(L.context, 'urbanphotography').get_posts()
+# or
+# posts = instaloader.Profile.from_username(L.context, PROFILE).get_posts()
+
+SINCE = datetime(2015, 5, 1)
+UNTIL = datetime(2015, 3, 1)
+
+for post in takewhile(lambda p: p.date > UNTIL, dropwhile(lambda p: p.date > SINCE, posts)):
+    print(post.date)
+    L.download_post(post, '#urbanphotography')

docs/codesnippets/194_top_x_of_user.py

@@ -0,0 +1,17 @@
+from itertools import islice
+from math import ceil
+
+from instaloader import Instaloader, Profile
+
+PROFILE = ...        # profile to download from
+X_percentage = 10    # percentage of posts that should be downloaded
+
+L = Instaloader()
+
+profile = Profile.from_username(L.context, PROFILE)
+posts_sorted_by_likes = sorted(profile.get_posts(),
+                               key=lambda p: p.likes + p.comments,
+                               reverse=True)
+
+for post in islice(posts_sorted_by_likes, ceil(profile.mediacount * X_percentage / 100)):
+    L.download_post(post, PROFILE)

docs/codesnippets/56_track_deleted.py

@@ -0,0 +1,34 @@
+from glob import glob
+from sys import argv
+from os import chdir
+
+from instaloader import Instaloader, Post, Profile, load_structure_from_file
+
+# Instaloader instantiation - you may pass additional arguments to the constructor here
+L = Instaloader()
+
+# If desired, load session previously saved with `instaloader -l USERNAME`:
+#L.load_session_from_file(USERNAME)
+
+try:
+    TARGET = argv[1]
+except IndexError:
+    raise SystemExit("Pass profile name as argument!")
+
+# Obtain set of posts that are on hard disk
+chdir(TARGET)
+offline_posts = set(filter(lambda s: isinstance(s, Post),
+                           (load_structure_from_file(L.context, file)
+                            for file in (glob('*.json.xz') + glob('*.json')))))
+
+# Obtain set of posts that are currently online
+post_iterator = Profile.from_username(L.context, TARGET).get_posts()
+online_posts = set(post_iterator)
+
+if online_posts - offline_posts:
+    print("Not yet downloaded posts:")
+    print(" ".join(str(p) for p in (online_posts - offline_posts)))
+
+if offline_posts - online_posts:
+    print("Deleted posts:")
+    print(" ".join(str(p) for p in (offline_posts - online_posts)))

docs/codesnippets/92_import_firefox_session.py

@@ -0,0 +1,23 @@
+from glob import glob
+from os.path import expanduser
+from sqlite3 import connect
+
+from instaloader import ConnectionException, Instaloader
+
+# FIREFOXCOOKIEFILE = "/home/alex/.mozilla/firefox/l96w6b90.default/cookies.sqlite"
+FIREFOXCOOKIEFILE = glob(expanduser("~/.mozilla/firefox/*.default/cookies.sqlite"))[0]
+
+instaloader = Instaloader(max_connection_attempts=1)
+instaloader.context._session.cookies.update(connect(FIREFOXCOOKIEFILE)
+                                            .execute("SELECT name, value FROM moz_cookies "
+                                                     "WHERE baseDomain='instagram.com'"))
+
+try:
+    username = instaloader.test_login()
+    if not username:
+        raise ConnectionException()
+except ConnectionException:
+    raise SystemExit("Cookie import failed. Are you logged in successfully in Firefox?")
+
+instaloader.context.username = username
+instaloader.save_session_to_file()

docs/codesnippets/README.md

@@ -0,0 +1,4 @@
+# Advanced Instaloader Examples
+
+This directory contains the scripts that are presented at:
+https://instaloader.github.io/codesnippets.html

docs/conf.py

@@ -20,11 +20,12 @@
 import os
 import subprocess
 import sys
-
+import typing
-import sphinx_bootstrap_theme
 
 sys.path.insert(0, os.path.abspath('..'))
 
+import docs.sphinx_autodoc_typehints as sphinx_autodoc_typehints
+
 # -- General configuration ------------------------------------------------
 
 # If your documentation needs a minimal Sphinx version, state it here.
@@ -36,9 +37,9 @@ sys.path.insert(0, os.path.abspath('..'))
 # ones.
 extensions = [
     'sphinx.ext.autodoc',
-    'sphinx_autodoc_typehints',
     'sphinx.ext.githubpages',
-    'sphinx.ext.intersphinx'
+    'sphinx.ext.intersphinx',
+    'sphinx.ext.extlinks'
 ]
 
 autodoc_default_flags = ['show-inheritance', 'members', 'undoc-members']
@@ -46,7 +47,9 @@ autodoc_default_flags = ['show-inheritance', 'members', 'undoc-members']
 autodoc_member_order = 'bysource'
 
 intersphinx_mapping = {'python': ('https://docs.python.org/3', None),
-                       'requests': ('http://docs.python-requests.org/en/master', None)}
+                       'requests': ('https://requests.kennethreitz.org/en/master/', None)}
+
+extlinks = {'issue': ('https://github.com/instaloader/instaloader/issues/%s', 'issue #')}
 
 # Add any paths that contain templates here, relative to this directory.
 templates_path = ['_templates']
@@ -83,7 +86,7 @@ author = 'Alexander Graf and Andre Koch-Kramer'
 #
 # This is also used if you do content translation via gettext catalogs.
 # Usually you set "language" from the command line for these cases.
-language = None
+language = "en"
 
 # There are two options for replacing |today|: either, you set today to some
 # non-false value, then it is used:
@@ -119,7 +122,7 @@ add_module_names = False
 # show_authors = False
 
 # The name of the Pygments (syntax highlighting) style to use.
-pygments_style = 'sphinx'
+pygments_style = 'fruity'
 
 # A list of ignored prefixes for module index sorting.
 # modindex_common_prefix = []
@@ -136,20 +139,7 @@ todo_include_todos = False
 # The theme to use for HTML and HTML Help pages.  See the documentation for
 # a list of builtin themes.
 #
-html_theme = 'bootstrap'
+html_theme = 'basic'
-html_theme_path = sphinx_bootstrap_theme.get_html_theme_path()
-
-# Theme options are theme-specific and customize the look and feel of a theme
-# further.  For a list of options available for each theme, see the
-# documentation.
-#
-html_theme_options = {
-    'navbar_site_name': 'Site Contents',
-    'navbar_pagenav_name': 'Page Contents',
-    'navbar_pagenav': True,
-    'navbar_sidebarrel': True,
-    'nosidebar': True,
-     }
 
 # Add any paths that contain custom themes here, relative to this directory.
 # html_theme_path = []
@@ -212,7 +202,7 @@ html_domain_indices = False
 
 # If false, no index is generated.
 #
-html_use_index = False
+html_use_index = True
 
 # If true, the index is split into individual pages for each letter.
 #
@@ -368,5 +358,9 @@ current_release_date = subprocess.check_output(["git", "log", "-1", "--tags", "-
 
 html_context = {'current_release': current_release, 'current_release_date': current_release_date}
 
+
 def setup(app):
-    app.add_stylesheet("style.css")
+    typing.TYPE_CHECKING = True
+    app.add_stylesheet("instaloader.css")
+    app.connect('autodoc-process-signature', sphinx_autodoc_typehints.process_signature)
+    app.connect('autodoc-process-docstring', sphinx_autodoc_typehints.process_docstring)

docs/contributing.rst

@@ -1,3 +1,5 @@
+.. _contributing:
+
 Contributing to Instaloader
 ===========================
 
@@ -13,20 +15,20 @@ Reporting Bugs
 --------------
 
 If you encounter a bug, do not hesitate to report it in our
-`Issue Tracker <https://github.com/instaloader/instaloader/issues>`__. Since
+`Issue Tracker <https://github.com/instaloader/instaloader/issues>`__. When
-Instaloader is actively developed, the majority of bugs is fixed within only
+reporting a problem, please keep the following in mind:
-**4 days** after being reported. When reporting a problem, please keep the
-following in mind:
 
 - Ensure you **use the latest version** of Instaloader. The currently-installed
   version can be found out with ``instaloader --version``.
 
+- Check whether there is a valid solution in our :ref:`troubleshooting` section.
+
 - Briefly **check whether the bug has already been reported**. If you find an
   issue reporting the same bug you encountered, comment there rather than
-  opening a new issue. However, if unsure, it is advised to create a new issue.
+  opening a new issue. However, if unsure, please create a new issue.
 
 - State **how the bug can be reproduced**, i.e. how Instaloader was invoked
-  when the problem occured (of course, you may anonymize profile names etc.).
+  when the problem occurred (of course, you may anonymize profile names etc.).
 
 - Include all **error messages and tracebacks** in the report.
 
@@ -47,13 +49,13 @@ few things to consider:
   you forked from the most recent branch.
 
 - We use `Pylint <https://www.pylint.org/>`__ for error and syntax checking of
-  ``instaloader.py``. The file ``.travis.yml`` in the project's root directory
+  the source. The file ``.travis.yml`` in the project's root directory
   shows how it is invoked. Note that sometimes it might be better to disable a
   warning rather than adapting the code to Pylint's desires.
 
 - The documentation source is located in the ``docs`` folder. The file
-  ``cli-options.rst`` is merely an RST-formatted copy of ``ìnstaloader --help``
+  ``cli-options.rst`` is merely an RST-formatted copy of ``instaloader --help``
-  output, of which the source is near the bottom of ``instaloader.py``.
+  output, of which the source is in ``instaloader/__main__.py``.
 
 - Feel free to contact us, even if you "only" have Proof-of-Concepts or
   not-fully integrated changes. They already might be an advance for the
@@ -62,6 +64,16 @@ few things to consider:
 Suggesting Features
 -------------------
 
+.. goal-start
+
+Instaloader's goal is to mimick the browser's behavior to access the data that
+is available through the Instagram web interface, pack this data into complete
+and easily-(re)usable python objects, and provide a user interface for the most
+common downloading and metadata collection tasks, without changing any of the
+online data.
+
+.. goal-end
+
 If you have an idea of how Instaloader should be enhanced, but do not want to
 implement the feature yourself, feel free to open a ticket in our 
 `Issue Tracker <https://github.com/instaloader/instaloader/issues>`__.
@@ -78,6 +90,28 @@ Please consider the following:
   invoke the new function? Which problem would it solve? If new information is
   obtained, how would it be further processed?
 
+- If not obvious, briefly motivate how your suggested feature **conforms with
+  Instaloader's project goal**.
+
 - **Be patient**. Naturally, bugs and pull requests have a higher priority than
   feature suggestions. Keep in mind that this is a free software project, and
   unfortunately we only have limited time to work on it.
+
+Donations
+---------
+
+.. donations-start
+
+It is a pleasure for us to share our Instaloader to the world, and we are proud
+to have attracted such an active and motivating community, with so many users
+who share their suggestions and ideas with us. Buying a community-sponsored beer
+or coffee from time to time is very likely to further raise our passion for the
+development of Instaloader.
+
+| For donations, we provide a PayPal.Me link and a Bitcoin address.
+|  `PayPal.me/aandergr <https://www.paypal.me/aandergr>`__
+|  BTC: 1Nst4LoadeYzrKjJ1DX9CpbLXBYE9RKLwY
+
+.. donations-end
+
+.. (Discussion in :issue:`130`)

docs/index.rst

@@ -5,6 +5,8 @@
       saved media, and their metadata, comments and captions.
       Written in Python.
 
+.. title:: Instaloader — Download Instagram Photos and Metadata
+
 Instaloader
 ===========
 
@@ -13,12 +15,18 @@ Instaloader
 **Instaloader** is a tool to download pictures (or videos) along with
 their captions and other metadata from Instagram.
 
+.. include:: ../README.rst
+   :start-after: badges-start
+   :end-before: badges-end
+
 With `Python <https://www.python.org/>`__ installed, do::
 
     $ pip3 install instaloader
 
     $ instaloader profile [profile ...]
 
+See :ref:`install` for more options on how to install Instaloader.
+
 **Instaloader**
 
 - downloads **public and private profiles, hashtags, user stories,
@@ -37,14 +45,17 @@ With `Python <https://www.python.org/>`__ installed, do::
 
 ::
 
-    instaloader [--comments] [--geotags] [--stories]
+    instaloader [--comments] [--geotags]
+                [--stories] [--highlights] [--tagged] [--igtv]
                 [--login YOUR-USERNAME] [--fast-update]
-                profile | "#hashtag" |
+                profile | "#hashtag" | %location_id |
                 :stories | :feed | :saved
 
+See :ref:`download-pictures-from-instagram` for a detailed introduction on how
+to use Instaloader to download pictures from Instagram.
 
-Table of Contents
+Instaloader Documentation
------------------
+-------------------------
 
 .. toctree::
    :maxdepth: 2
@@ -53,20 +64,35 @@ Table of Contents
    basic-usage
    cli-options
    as-module
+   codesnippets
+   troubleshooting
    contributing
 
 Useful Links
 ------------
 
 - `Git Repository (on GitHub) <https://github.com/instaloader/instaloader>`__
-- `PyPI Project Page <https://pypi.python.org/pypi/instaloader>`__
+- `PyPI Project Page <https://pypi.org/project/instaloader/>`__
 - `Issue Tracker / Bug Tracker <https://github.com/instaloader/instaloader/issues>`__
 - `Version History <https://github.com/instaloader/instaloader/releases>`__
 
+Disclaimer
+----------
+
 .. include:: ../README.rst
    :start-after: disclaimer-start
    :end-before: disclaimer-end
 
+Contributing
+------------
+
+As an open source project, Instaloader heavily depends on the contributions from
+its community. See :ref:`contributing` for how you may help Instaloader to
+become an even greater tool.
+
+.. include:: contributing.rst
+   :start-after: donations-start
+   :end-before: donations-end
 
 ..
    * :ref:`genindex`

docs/installation.rst

@@ -1,28 +1,24 @@
-Installation
+.. _install:
-============
+
+Install Instaloader
+===================
 
 .. highlight:: none
 
-Instaloader requires `Python <https://www.python.org/>`__, at least
+**To install Instaloader**,
-version 3.5.  If you have `pip <https://pypi.python.org/pypi/pip>`__
-installed, you may install Instaloader using
 
-::
+#. Ensure that you have `Python <https://www.python.org/>`__, at least
+   version 3.5 and `pip <https://pypi.python.org/pypi/pip>`__
+   installed
 
-    pip3 install instaloader
+#. Then, install Instaloader using::
 
-Alternatively, to get the most current version of Instaloader from our
+      pip3 install instaloader
-`Git repository <https://github.com/instaloader/instaloader>`__:
 
-::
+**To upgrade Instaloader**, do::
 
-    pip3 install git+https://github.com/instaloader/instaloader
+   pip3 install --upgrade instaloader
 
-(pass ``--upgrade`` to upgrade if Instaloader is already installed)
-
-Instaloader requires
-`requests <http://python-requests.org/>`__, which
-will be installed automatically, if it is not already installed.
 
 **Alternative methods for installing Instaloader:**
 
@@ -34,8 +30,9 @@ will be installed automatically, if it is not already installed.
 - On **Arch Linux**, you may install Instaloader using the
   `Instaloader AUR package <https://aur.archlinux.org/packages/instaloader/>`__.
 
-- On **Gentoo Linux**, you may install Instaloader using the
-  `Instaloader Ebuild <https://bugs.gentoo.org/642100>`__.
-
 - On **Windows 10**, you may download the standalone executable from the
   `current release page <https://github.com/instaloader/instaloader/releases/latest>`__.
+
+- To test the most current pre-release or development version of Instaloader::
+
+     pip3 install --pre instaloader

docs/requirements.txt

@@ -1,4 +1,2 @@
 requests
-sphinx
+sphinx!=1.7.7
-sphinx-autodoc-typehints
-sphinx-bootstrap-theme

docs/sphinx_autodoc_typehints.py

@@ -0,0 +1,251 @@
+# sphinx_autodoc_typehints from Nov 20 2018 (version 1.5.2).
+#  https://github.com/agronholm/sphinx-autodoc-typehints
+# (slightly modified to fix class links, maybe related to agronholm/sphinx-autodoc-typehints#38)
+#
+# This is the MIT license: http://www.opensource.org/licenses/mit-license.php
+#
+# Copyright (c) Alex Grönholm
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy of this
+# software and associated documentation files (the "Software"), to deal in the Software
+# without restriction, including without limitation the rights to use, copy, modify, merge,
+# publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons
+# to whom the Software is furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in all copies or
+# substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
+# INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+# PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE
+# FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
+# OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+# DEALINGS IN THE SOFTWARE.
+#
+
+
+import inspect
+from typing import get_type_hints, TypeVar, Any, AnyStr, Generic, Union
+
+from sphinx.util import logging
+from sphinx.util.inspect import Signature
+
+try:
+    from inspect import unwrap
+except ImportError:
+    def unwrap(func, *, stop=None):
+        """This is the inspect.unwrap() method copied from Python 3.5's standard library."""
+        if stop is None:
+            def _is_wrapper(f):
+                return hasattr(f, '__wrapped__')
+        else:
+            def _is_wrapper(f):
+                return hasattr(f, '__wrapped__') and not stop(f)
+        f = func  # remember the original func for error reporting
+        memo = {id(f)}  # Memoise by id to tolerate non-hashable objects
+        while _is_wrapper(func):
+            func = func.__wrapped__
+            id_func = id(func)
+            if id_func in memo:
+                raise ValueError('wrapper loop when unwrapping {!r}'.format(f))
+            memo.add(id_func)
+        return func
+
+logger = logging.getLogger(__name__)
+
+
+def format_annotation(annotation):
+    if inspect.isclass(annotation) and annotation.__module__ == 'builtins':
+        if annotation.__qualname__ == 'NoneType':
+            return '``None``'
+        else:
+            return ':py:class:`{}`'.format(annotation.__qualname__)
+
+    annotation_cls = annotation if inspect.isclass(annotation) else type(annotation)
+    class_name = None
+    if annotation_cls.__module__ == 'typing':
+        params = None
+        prefix = ':py:class:'
+        module = 'typing'
+        extra = ''
+
+        if inspect.isclass(getattr(annotation, '__origin__', None)):
+            annotation_cls = annotation.__origin__
+            try:
+                if Generic in annotation_cls.mro():
+                    module = annotation_cls.__module__
+            except TypeError:
+                pass  # annotation_cls was either the "type" object or typing.Type
+
+        if annotation is Any:
+            return ':py:data:`~typing.Any`'
+        elif annotation is AnyStr:
+            return ':py:data:`~typing.AnyStr`'
+        elif isinstance(annotation, TypeVar):
+            return '\\%r' % annotation
+        elif (annotation is Union or getattr(annotation, '__origin__', None) is Union or
+              hasattr(annotation, '__union_params__')):
+            prefix = ':py:data:'
+            class_name = 'Union'
+            if hasattr(annotation, '__union_params__'):
+                params = annotation.__union_params__
+            elif hasattr(annotation, '__args__'):
+                params = annotation.__args__
+
+            if params and len(params) == 2 and (hasattr(params[1], '__qualname__') and
+                                                params[1].__qualname__ == 'NoneType'):
+                class_name = 'Optional'
+                params = (params[0],)
+        elif annotation_cls.__qualname__ == 'Tuple' and hasattr(annotation, '__tuple_params__'):
+            params = annotation.__tuple_params__
+            if annotation.__tuple_use_ellipsis__:
+                params += (Ellipsis,)
+        elif annotation_cls.__qualname__ == 'Callable':
+            prefix = ':py:data:'
+            arg_annotations = result_annotation = None
+            if hasattr(annotation, '__result__'):
+                arg_annotations = annotation.__args__
+                result_annotation = annotation.__result__
+            elif getattr(annotation, '__args__', None):
+                arg_annotations = annotation.__args__[:-1]
+                result_annotation = annotation.__args__[-1]
+
+            if arg_annotations in (Ellipsis, (Ellipsis,)):
+                params = [Ellipsis, result_annotation]
+            elif arg_annotations is not None:
+                params = [
+                    '\\[{}]'.format(
+                        ', '.join(format_annotation(param) for param in arg_annotations)),
+                    result_annotation
+                ]
+        elif hasattr(annotation, 'type_var'):
+            # Type alias
+            class_name = annotation.name
+            params = (annotation.type_var,)
+        elif getattr(annotation, '__args__', None) is not None:
+            params = annotation.__args__
+        elif hasattr(annotation, '__parameters__'):
+            params = annotation.__parameters__
+
+        if params:
+            extra = '\\[{}]'.format(', '.join(format_annotation(param) for param in params))
+
+        if not class_name:
+            class_name = annotation_cls.__qualname__.title()
+
+        return '{}`~{}.{}`{}'.format(prefix, module, class_name, extra)
+    elif annotation is Ellipsis:
+        return '...'
+    elif inspect.isclass(annotation) or inspect.isclass(getattr(annotation, '__origin__', None)):
+        if not inspect.isclass(annotation):
+            annotation_cls = annotation.__origin__
+
+        extra = ''
+        if Generic in annotation_cls.mro():
+            params = (getattr(annotation, '__parameters__', None) or
+                      getattr(annotation, '__args__', None))
+            extra = '\\[{}]'.format(', '.join(format_annotation(param) for param in params))
+
+        module = annotation.__module__.split('.')[0]    # hack to 'fix' class linking for Instaloader project
+        return ':py:class:`~{}.{}`{}'.format(module, annotation_cls.__qualname__,
+                                             extra)
+
+    return str(annotation)
+
+
+def process_signature(app, what: str, name: str, obj, options, signature, return_annotation):
+    if not callable(obj):
+        return
+
+    if what in ('class', 'exception'):
+        obj = getattr(obj, '__init__', getattr(obj, '__new__', None))
+
+    if not getattr(obj, '__annotations__', None):
+        return
+
+    obj = unwrap(obj)
+    signature = Signature(obj)
+    parameters = [
+        param.replace(annotation=inspect.Parameter.empty)
+        for param in signature.signature.parameters.values()
+    ]
+
+    if parameters:
+        if what in ('class', 'exception'):
+            del parameters[0]
+        elif what == 'method':
+            outer = inspect.getmodule(obj)
+            for clsname in obj.__qualname__.split('.')[:-1]:
+                outer = getattr(outer, clsname)
+
+            method_name = obj.__name__
+            if method_name.startswith("__") and not method_name.endswith("__"):
+                # If the method starts with double underscore (dunder)
+                # Python applies mangling so we need to prepend the class name.
+                # This doesn't happen if it always ends with double underscore.
+                class_name = obj.__qualname__.split('.')[-2]
+                method_name = "_{c}{m}".format(c=class_name, m=method_name)
+
+            method_object = outer.__dict__[method_name]
+            if not isinstance(method_object, (classmethod, staticmethod)):
+                del parameters[0]
+
+    signature.signature = signature.signature.replace(
+        parameters=parameters,
+        return_annotation=inspect.Signature.empty)
+
+    return signature.format_args().replace('\\', '\\\\'), None
+
+
+def process_docstring(app, what, name, obj, options, lines):
+    if isinstance(obj, property):
+        obj = obj.fget
+
+    if callable(obj):
+        if what in ('class', 'exception'):
+            obj = getattr(obj, '__init__')
+
+        obj = unwrap(obj)
+        try:
+            type_hints = get_type_hints(obj)
+        except (AttributeError, TypeError):
+            # Introspecting a slot wrapper will raise TypeError
+            return
+        except NameError as exc:
+            logger.warning('Cannot resolve forward reference in type annotations of "%s": %s',
+                           name, exc)
+            type_hints = obj.__annotations__
+
+        for argname, annotation in type_hints.items():
+            if argname.endswith('_'):
+                argname = '{}\\_'.format(argname[:-1])
+
+            formatted_annotation = format_annotation(annotation)
+
+            if argname == 'return':
+                if what in ('class', 'exception'):
+                    # Don't add return type None from __init__()
+                    continue
+
+                insert_index = len(lines)
+                for i, line in enumerate(lines):
+                    if line.startswith(':rtype:'):
+                        insert_index = None
+                        break
+                    elif line.startswith(':return:') or line.startswith(':returns:'):
+                        insert_index = i
+
+                if insert_index is not None:
+                    if insert_index == len(lines):
+                        # Ensure that :rtype: doesn't get joined with a paragraph of text, which
+                        # prevents it being interpreted.
+                        lines.append('')
+                        insert_index += 1
+
+                    lines.insert(insert_index, ':rtype: {}'.format(formatted_annotation))
+            else:
+                searchfor = ':param {}:'.format(argname)
+                for i, line in enumerate(lines):
+                    if line.startswith(searchfor):
+                        lines.insert(i, ':type {}: {}'.format(argname, formatted_annotation))
+                        break

docs/troubleshooting.rst

@@ -0,0 +1,60 @@
+.. _troubleshooting:
+
+Troubleshooting
+===============
+
+.. highlight:: python
+
+429 - Too Many Requests
+-----------------------
+
+Instaloader has a logic to keep track of its requests to Instagram and to obey
+their rate limits. Since they are nowhere documented, we try them out
+experimentally. We have a daily cron job running to confirm that Instaloader
+still stays within the rate limits. Nevertheless, the rate control logic assumes
+that
+
+- at one time, Instaloader is the only application that consumes requests. I.e.
+  neither the Instagram browser interface, nor a mobile app, nor another
+  Instaloader instance is running in parallel,
+
+- no requests had been consumed when Instaloader starts.
+
+The latter one implies that restarting or reinstantiating Instaloader often
+within short time is prone to cause a 429. When a request is denied with a 429,
+Instaloader retries the request as soon as the temporary ban is assumed to be
+expired. In case the retry continuously fails for some reason, which should not
+happen in normal conditions, consider adjusting the
+:option:`--max-connection-attempts` option.
+
+**"Too many queries in the last time"** is not an error. It is a notice that the
+rate limit has almost been reached, according to Instaloader's own rate
+accounting mechanism. We regularly adjust this mechanism to match Instagram's
+current rate limiting.
+
+Login error
+-----------
+
+Instaloader's login *should* work fine, both with and without
+Two-Factor-Authentication. It also supports handling the *checkpoint challenge*,
+issued when Instagram suspects authentication activity on your account, by
+pointing the user to an URL to be opened in a browser.
+
+Nevertheless, in :issue:`92` users report problems with logging in. To still use
+Instaloader's logged-in functionality, you may use the following script to
+workaround login problems by importing the session cookies from Firefox and
+bypassing Instaloader's login.
+
+.. literalinclude:: codesnippets/92_import_firefox_session.py
+
+To use this,
+
+#. login to Instagram in Firefox, 
+
+#. execute the snippet,
+
+#. then, ``instaloader -l USERNAME`` should work fine.
+
+If you do not use your default firefox profile, or your operating system has the
+paths differently set up, you may have to alter the ``FIREFOXCOOKIEFILE``
+variable first.

instaloader/__init__.py

@@ -0,0 +1,19 @@
+"""Download pictures (or videos) along with their captions and other metadata from Instagram."""
+
+
+__version__ = '4.4.2'
+
+
+try:
+    # pylint:disable=wrong-import-position
+    import win_unicode_console  # type: ignore
+except ImportError:
+    pass
+else:
+    win_unicode_console.enable()
+
+from .exceptions import *
+from .instaloader import Instaloader
+from .instaloadercontext import InstaloaderContext
+from .structures import (Hashtag, Highlight, Post, PostSidecarNode, PostComment, PostCommentAnswer, PostLocation,
+                         Profile, Story, StoryItem, TopSearchResults, load_structure_from_file, save_structure_to_file)

instaloader/__main__.py

@@ -0,0 +1,442 @@
+"""Download pictures (or videos) along with their captions and other metadata from Instagram."""
+
+import ast
+import datetime
+import os
+import re
+import sys
+from argparse import ArgumentParser, SUPPRESS
+from typing import List, Optional
+
+from . import (Instaloader, InstaloaderException, InvalidArgumentException, Post, Profile, ProfileNotExistsException,
+               StoryItem, __version__, load_structure_from_file, TwoFactorAuthRequiredException,
+               BadCredentialsException)
+from .instaloader import get_default_session_filename
+from .instaloadercontext import default_user_agent
+
+
+def usage_string():
+    # NOTE: duplicated in README.rst and docs/index.rst
+    argv0 = os.path.basename(sys.argv[0])
+    argv0 = "instaloader" if argv0 == "__main__.py" else argv0
+    return """
+{0} [--comments] [--geotags]
+{2:{1}} [--stories] [--highlights] [--tagged] [--igtv]
+{2:{1}} [--login YOUR-USERNAME] [--fast-update]
+{2:{1}} profile | "#hashtag" | %%location_id | :stories | :feed | :saved
+{0} --help""".format(argv0, len(argv0), '')
+
+
+def filterstr_to_filterfunc(filter_str: str, item_type: type):
+    """Takes an --post-filter=... or --storyitem-filter=... filter
+     specification and makes a filter_func Callable out of it."""
+
+    # The filter_str is parsed, then all names occurring in its AST are replaced by loads to post.<name>. A
+    # function Post->bool is returned which evaluates the filter with the post as 'post' in its namespace.
+
+    class TransformFilterAst(ast.NodeTransformer):
+        def visit_Name(self, node: ast.Name):
+            # pylint:disable=no-self-use
+            if not isinstance(node.ctx, ast.Load):
+                raise InvalidArgumentException("Invalid filter: Modifying variables ({}) not allowed.".format(node.id))
+            if node.id == "datetime":
+                return node
+            if not hasattr(item_type, node.id):
+                raise InvalidArgumentException("Invalid filter: {} not a {} attribute.".format(node.id,
+                                                                                               item_type.__name__))
+            new_node = ast.Attribute(ast.copy_location(ast.Name('item', ast.Load()), node), node.id,
+                                     ast.copy_location(ast.Load(), node))
+            return ast.copy_location(new_node, node)
+
+    input_filename = '<command line filter parameter>'
+    compiled_filter = compile(TransformFilterAst().visit(ast.parse(filter_str, filename=input_filename, mode='eval')),
+                              filename=input_filename, mode='eval')
+
+    def filterfunc(item) -> bool:
+        # pylint:disable=eval-used
+        return bool(eval(compiled_filter, {'item': item, 'datetime': datetime.datetime}))
+
+    return filterfunc
+
+
+def _main(instaloader: Instaloader, targetlist: List[str],
+          username: Optional[str] = None, password: Optional[str] = None,
+          sessionfile: Optional[str] = None,
+          download_profile_pic: bool = True, download_posts=True,
+          download_stories: bool = False,
+          download_highlights: bool = False,
+          download_tagged: bool = False,
+          download_igtv: bool = False,
+          fast_update: bool = False,
+          max_count: Optional[int] = None, post_filter_str: Optional[str] = None,
+          storyitem_filter_str: Optional[str] = None) -> None:
+    """Download set of profiles, hashtags etc. and handle logging in and session files if desired."""
+    # Parse and generate filter function
+    post_filter = None
+    if post_filter_str is not None:
+        post_filter = filterstr_to_filterfunc(post_filter_str, Post)
+        instaloader.context.log('Only download posts with property "{}".'.format(post_filter_str))
+    storyitem_filter = None
+    if storyitem_filter_str is not None:
+        storyitem_filter = filterstr_to_filterfunc(storyitem_filter_str, StoryItem)
+        instaloader.context.log('Only download storyitems with property "{}".'.format(storyitem_filter_str))
+    # Login, if desired
+    if username is not None:
+        try:
+            instaloader.load_session_from_file(username, sessionfile)
+        except FileNotFoundError as err:
+            if sessionfile is not None:
+                print(err, file=sys.stderr)
+            instaloader.context.log("Session file does not exist yet - Logging in.")
+        if not instaloader.context.is_logged_in or username != instaloader.test_login():
+            if password is not None:
+                try:
+                    instaloader.login(username, password)
+                except TwoFactorAuthRequiredException:
+                    while True:
+                        try:
+                            code = input("Enter 2FA verification code: ")
+                            instaloader.two_factor_login(code)
+                            break
+                        except BadCredentialsException:
+                            pass
+            else:
+                instaloader.interactive_login(username)
+        instaloader.context.log("Logged in as %s." % username)
+    # since 4.2.9 login is required for geotags
+    if instaloader.download_geotags and not instaloader.context.is_logged_in:
+        instaloader.context.error("Warning: Use --login to download geotags of posts.")
+    # Try block for KeyboardInterrupt (save session on ^C)
+    profiles = set()
+    anonymous_retry_profiles = set()
+    try:
+        # Generate set of profiles, already downloading non-profile targets
+        for target in targetlist:
+            if (target.endswith('.json') or target.endswith('.json.xz')) and os.path.isfile(target):
+                with instaloader.context.error_catcher(target):
+                    structure = load_structure_from_file(instaloader.context, target)
+                    if isinstance(structure, Post):
+                        if post_filter is not None and not post_filter(structure):
+                            instaloader.context.log("<{} ({}) skipped>".format(structure, target), flush=True)
+                            continue
+                        instaloader.context.log("Downloading {} ({})".format(structure, target))
+                        instaloader.download_post(structure, os.path.dirname(target))
+                    elif isinstance(structure, StoryItem):
+                        if storyitem_filter is not None and not storyitem_filter(structure):
+                            instaloader.context.log("<{} ({}) skipped>".format(structure, target), flush=True)
+                            continue
+                        instaloader.context.log("Attempting to download {} ({})".format(structure, target))
+                        instaloader.download_storyitem(structure, os.path.dirname(target))
+                    elif isinstance(structure, Profile):
+                        raise InvalidArgumentException("Profile JSON are ignored. Pass \"{}\" to download that profile"
+                                                       .format(structure.username))
+                    else:
+                        raise InvalidArgumentException("{} JSON file not supported as target"
+                                                       .format(structure.__class__.__name__))
+                continue
+            # strip '/' characters to be more shell-autocompletion-friendly
+            target = target.rstrip('/')
+            with instaloader.context.error_catcher(target):
+                if re.match(r"^@[A-Za-z0-9._]+$", target):
+                    instaloader.context.log("Retrieving followees of %s..." % target[1:])
+                    profile = Profile.from_username(instaloader.context, target[1:])
+                    for followee in profile.get_followees():
+                        instaloader.save_profile_id(followee)
+                        profiles.add(followee)
+                elif re.match(r"^#\w+$", target):
+                    instaloader.download_hashtag(hashtag=target[1:], max_count=max_count, fast_update=fast_update,
+                                                 post_filter=post_filter,
+                                                 profile_pic=download_profile_pic, posts=download_posts)
+                elif re.match(r"^-[A-Za-z0-9-_]+$", target):
+                    instaloader.download_post(Post.from_shortcode(instaloader.context, target[1:]), target)
+                elif re.match(r"^%[0-9]+$", target):
+                    instaloader.download_location(location=target[1:], max_count=max_count, fast_update=fast_update,
+                                                  post_filter=post_filter)
+                elif target == ":feed":
+                    instaloader.download_feed_posts(fast_update=fast_update, max_count=max_count,
+                                                    post_filter=post_filter)
+                elif target == ":stories":
+                    instaloader.download_stories(fast_update=fast_update, storyitem_filter=storyitem_filter)
+                elif target == ":saved":
+                    instaloader.download_saved_posts(fast_update=fast_update, max_count=max_count,
+                                                     post_filter=post_filter)
+                elif re.match(r"^[A-Za-z0-9._]+$", target):
+                    try:
+                        profile = instaloader.check_profile_id(target)
+                        if instaloader.context.is_logged_in and profile.has_blocked_viewer:
+                            if download_profile_pic or ((download_posts or download_tagged or download_igtv)
+                                                        and not profile.is_private):
+                                raise ProfileNotExistsException("{} blocked you; But we download her anonymously."
+                                                                .format(target))
+                            else:
+                                instaloader.context.error("{} blocked you.".format(target))
+                        else:
+                            profiles.add(profile)
+                    except ProfileNotExistsException as err:
+                        # Not only our profile.has_blocked_viewer condition raises ProfileNotExistsException,
+                        # check_profile_id() also does, since access to blocked profile may be responded with 404.
+                        if instaloader.context.is_logged_in and (download_profile_pic or download_posts or
+                                                                 download_tagged or download_igtv):
+                            instaloader.context.log(err)
+                            instaloader.context.log("Trying again anonymously, helps in case you are just blocked.")
+                            with instaloader.anonymous_copy() as anonymous_loader:
+                                with instaloader.context.error_catcher():
+                                    anonymous_retry_profiles.add(anonymous_loader.check_profile_id(target))
+                                    instaloader.context.error("Warning: {} will be downloaded anonymously (\"{}\")."
+                                                              .format(target, err))
+                        else:
+                            raise
+                else:
+                    target_type = {
+                        '#': 'hashtag',
+                        '%': 'location',
+                        '-': 'shortcode',
+                    }.get(target[0], 'username')
+                    raise ProfileNotExistsException('Invalid {} {}'.format(target_type, target))
+        if len(profiles) > 1:
+            instaloader.context.log("Downloading {} profiles: {}".format(len(profiles),
+                                                                         ' '.join([p.username for p in profiles])))
+        if profiles and download_profile_pic and not instaloader.context.is_logged_in:
+            instaloader.context.error("Warning: Use --login to download HD version of profile pictures.")
+        instaloader.download_profiles(profiles,
+                                      download_profile_pic, download_posts, download_tagged, download_igtv,
+                                      download_highlights, download_stories,
+                                      fast_update, post_filter, storyitem_filter)
+        if anonymous_retry_profiles:
+            instaloader.context.log("Downloading anonymously: {}"
+                                    .format(' '.join([p.username for p in anonymous_retry_profiles])))
+            with instaloader.anonymous_copy() as anonymous_loader:
+                anonymous_loader.download_profiles(anonymous_retry_profiles,
+                                                   download_profile_pic, download_posts, download_tagged, download_igtv,
+                                                   fast_update=fast_update, post_filter=post_filter)
+    except KeyboardInterrupt:
+        print("\nInterrupted by user.", file=sys.stderr)
+    # Save session if it is useful
+    if instaloader.context.is_logged_in:
+        instaloader.save_session_to_file(sessionfile)
+    # User might be confused if Instaloader does nothing
+    if not targetlist:
+        if instaloader.context.is_logged_in:
+            # Instaloader did at least save a session file
+            instaloader.context.log("No targets were specified, thus nothing has been downloaded.")
+        else:
+            # Instloader did not do anything
+            instaloader.context.log("usage:" + usage_string())
+
+
+def main():
+    parser = ArgumentParser(description=__doc__, add_help=False, usage=usage_string(),
+                            epilog="Report issues at https://github.com/instaloader/instaloader/issues. "
+                                   "The complete documentation can be found at "
+                                   "https://instaloader.github.io/.",
+                            fromfile_prefix_chars='+')
+
+    g_targets = parser.add_argument_group("What to Download",
+                                          "Specify a list of targets. For each of these, Instaloader creates a folder "
+                                          "and downloads all posts. The following targets are supported:")
+    g_targets.add_argument('profile', nargs='*',
+                           help="Download profile. If an already-downloaded profile has been renamed, Instaloader "
+                                "automatically finds it by its unique ID and renames the folder likewise.")
+    g_targets.add_argument('_at_profile', nargs='*', metavar="@profile",
+                           help="Download all followees of profile. Requires --login. "
+                                "Consider using :feed rather than @yourself.")
+    g_targets.add_argument('_hashtag', nargs='*', metavar='"#hashtag"', help="Download #hashtag.")
+    g_targets.add_argument('_location', nargs='*', metavar='%location_id',
+                           help="Download %%location_id. Requires --login.")
+    g_targets.add_argument('_feed', nargs='*', metavar=":feed",
+                           help="Download pictures from your feed. Requires --login.")
+    g_targets.add_argument('_stories', nargs='*', metavar=":stories",
+                           help="Download the stories of your followees. Requires --login.")
+    g_targets.add_argument('_saved', nargs='*', metavar=":saved",
+                           help="Download the posts that you marked as saved. Requires --login.")
+    g_targets.add_argument('_singlepost', nargs='*', metavar="-- -shortcode",
+                           help="Download the post with the given shortcode")
+    g_targets.add_argument('_json', nargs='*', metavar="filename.json[.xz]",
+                           help="Re-Download the given object.")
+    g_targets.add_argument('_fromfile', nargs='*', metavar="+args.txt",
+                           help="Read targets (and options) from given textfile.")
+
+    g_post = parser.add_argument_group("What to Download of each Post")
+
+    g_prof = parser.add_argument_group("What to Download of each Profile")
+
+    g_prof.add_argument('-P', '--profile-pic-only', action='store_true',
+                        help=SUPPRESS)
+    g_prof.add_argument('--no-posts', action='store_true',
+                        help="Do not download regular posts.")
+    g_prof.add_argument('--no-profile-pic', action='store_true',
+                        help='Do not download profile picture.')
+    g_post.add_argument('--no-pictures', action='store_true',
+                        help='Do not download post pictures. Cannot be used together with --fast-update. '
+                             'Implies --no-video-thumbnails, does not imply --no-videos.')
+    g_post.add_argument('-V', '--no-videos', action='store_true',
+                        help='Do not download videos.')
+    g_post.add_argument('--no-video-thumbnails', action='store_true',
+                        help='Do not download thumbnails of videos.')
+    g_post.add_argument('-G', '--geotags', action='store_true',
+                        help='Download geotags when available. Geotags are stored as a '
+                             'text file with the location\'s name and a Google Maps link. '
+                             'This requires an additional request to the Instagram '
+                             'server for each picture. Requires --login.')
+    g_post.add_argument('-C', '--comments', action='store_true',
+                        help='Download and update comments for each post. '
+                             'This requires an additional request to the Instagram '
+                             'server for each post, which is why it is disabled by default.')
+    g_post.add_argument('--no-captions', action='store_true',
+                        help='Do not create txt files.')
+    g_post.add_argument('--post-metadata-txt', action='append',
+                        help='Template to write in txt file for each Post.')
+    g_post.add_argument('--storyitem-metadata-txt', action='append',
+                        help='Template to write in txt file for each StoryItem.')
+    g_post.add_argument('--no-metadata-json', action='store_true',
+                        help='Do not create a JSON file containing the metadata of each post.')
+    g_post.add_argument('--metadata-json', action='store_true',
+                        help=SUPPRESS)
+    g_post.add_argument('--no-compress-json', action='store_true',
+                        help='Do not xz compress JSON files, rather create pretty formatted JSONs.')
+    g_prof.add_argument('-s', '--stories', action='store_true',
+                        help='Also download stories of each profile that is downloaded. Requires --login.')
+    g_prof.add_argument('--stories-only', action='store_true',
+                        help=SUPPRESS)
+    g_prof.add_argument('--highlights', action='store_true',
+                        help='Also download highlights of each profile that is downloaded. Requires --login.')
+    g_prof.add_argument('--tagged', action='store_true',
+                        help='Also download posts where each profile is tagged.')
+    g_prof.add_argument('--igtv', action='store_true',
+                        help='Also download IGTV videos.')
+
+    g_cond = parser.add_argument_group("Which Posts to Download")
+
+    g_cond.add_argument('-F', '--fast-update', action='store_true',
+                        help='For each target, stop when encountering the first already-downloaded picture. This '
+                             'flag is recommended when you use Instaloader to update your personal Instagram archive.')
+
+    g_cond.add_argument('--post-filter', '--only-if', metavar='filter',
+                        help='Expression that, if given, must evaluate to True for each post to be downloaded. Must be '
+                             'a syntactically valid python expression. Variables are evaluated to '
+                             'instaloader.Post attributes. Example: --post-filter=viewer_has_liked.')
+    g_cond.add_argument('--storyitem-filter', metavar='filter',
+                        help='Expression that, if given, must evaluate to True for each storyitem to be downloaded. '
+                             'Must be a syntactically valid python expression. Variables are evaluated to '
+                             'instaloader.StoryItem attributes.')
+
+    g_cond.add_argument('-c', '--count',
+                        help='Do not attempt to download more than COUNT posts. '
+                             'Applies only to #hashtag and :feed.')
+
+    g_login = parser.add_argument_group('Login (Download Private Profiles)',
+                                        'Instaloader can login to Instagram. This allows downloading private profiles. '
+                                        'To login, pass the --login option. Your session cookie (not your password!) '
+                                        'will be saved to a local file to be reused next time you want Instaloader '
+                                        'to login.')
+    g_login.add_argument('-l', '--login', metavar='YOUR-USERNAME',
+                         help='Login name (profile name) for your Instagram account.')
+    g_login.add_argument('-f', '--sessionfile',
+                         help='Path for loading and storing session key file. '
+                              'Defaults to ' + get_default_session_filename("<login_name>"))
+    g_login.add_argument('-p', '--password', metavar='YOUR-PASSWORD',
+                         help='Password for your Instagram account. Without this option, '
+                              'you\'ll be prompted for your password interactively if '
+                              'there is not yet a valid session file.')
+
+    g_how = parser.add_argument_group('How to Download')
+    g_how.add_argument('--dirname-pattern',
+                       help='Name of directory where to store posts. {profile} is replaced by the profile name, '
+                            '{target} is replaced by the target you specified, i.e. either :feed, #hashtag or the '
+                            'profile name. Defaults to \'{target}\'.')
+    g_how.add_argument('--filename-pattern',
+                       help='Prefix of filenames, relative to the directory given with '
+                            '--dirname-pattern. {profile} is replaced by the profile name,'
+                            '{target} is replaced by the target you specified, i.e. either :feed'
+                            '#hashtag or the profile name. Defaults to \'{date_utc}_UTC\'')
+    g_how.add_argument('--user-agent',
+                       help='User Agent to use for HTTP requests. Defaults to \'{}\'.'.format(default_user_agent()))
+    g_how.add_argument('-S', '--no-sleep', action='store_true', help=SUPPRESS)
+    g_how.add_argument('--max-connection-attempts', metavar='N', type=int, default=3,
+                       help='Maximum number of connection attempts until a request is aborted. Defaults to 3. If a '
+                            'connection fails, it can be manually skipped by hitting CTRL+C. Set this to 0 to retry '
+                            'infinitely.')
+    g_how.add_argument('--commit-mode', action='store_true',
+                       help='Tries to ensure downloaded images avoid corruption in case of unexpected interruption. '
+                       'If the last picture is corrupted, Instaloader will fix the picture the next time it is run. '
+                       'Requires the JSON metadata to be saved.')
+    g_how.add_argument('--request-timeout', metavar='N', type=float,
+                       help='seconds to wait before timing out a connection request')
+
+    g_misc = parser.add_argument_group('Miscellaneous Options')
+    g_misc.add_argument('-q', '--quiet', action='store_true',
+                        help='Disable user interaction, i.e. do not print messages (except errors) and fail '
+                             'if login credentials are needed but not given. This makes Instaloader suitable as a '
+                             'cron job.')
+    g_misc.add_argument('-h', '--help', action='help', help='Show this help message and exit.')
+    g_misc.add_argument('--version', action='version', help='Show version number and exit.',
+                        version=__version__)
+
+    args = parser.parse_args()
+    try:
+        if args.login is None and (args.stories or args.stories_only):
+            print("--login=USERNAME required to download stories.", file=sys.stderr)
+            args.stories = False
+            if args.stories_only:
+                raise SystemExit(1)
+
+        if ':feed-all' in args.profile or ':feed-liked' in args.profile:
+            raise SystemExit(":feed-all and :feed-liked were removed. Use :feed as target and "
+                             "eventually --post-filter=viewer_has_liked.")
+
+        post_metadata_txt_pattern = '\n'.join(args.post_metadata_txt) if args.post_metadata_txt else None
+        storyitem_metadata_txt_pattern = '\n'.join(args.storyitem_metadata_txt) if args.storyitem_metadata_txt else None
+
+        if args.no_captions:
+            if not (post_metadata_txt_pattern or storyitem_metadata_txt_pattern):
+                post_metadata_txt_pattern = ''
+                storyitem_metadata_txt_pattern = ''
+            else:
+                raise SystemExit("--no-captions and --post-metadata-txt or --storyitem-metadata-txt given; "
+                                 "That contradicts.")
+
+        if args.no_pictures and args.fast_update:
+            raise SystemExit('--no-pictures and --fast-update cannot be used together.')
+
+        # Determine what to download
+        download_profile_pic = not args.no_profile_pic or args.profile_pic_only
+        download_posts = not (args.no_posts or args.stories_only or args.profile_pic_only)
+        download_stories = args.stories or args.stories_only
+
+        if args.commit_mode and args.no_metadata_json:
+            raise SystemExit('--commit-mode requires JSON metadata to be saved.')
+
+        loader = Instaloader(sleep=not args.no_sleep, quiet=args.quiet, user_agent=args.user_agent,
+                             dirname_pattern=args.dirname_pattern, filename_pattern=args.filename_pattern,
+                             download_pictures=not args.no_pictures,
+                             download_videos=not args.no_videos, download_video_thumbnails=not args.no_video_thumbnails,
+                             download_geotags=args.geotags,
+                             download_comments=args.comments, save_metadata=not args.no_metadata_json,
+                             compress_json=not args.no_compress_json,
+                             post_metadata_txt_pattern=post_metadata_txt_pattern,
+                             storyitem_metadata_txt_pattern=storyitem_metadata_txt_pattern,
+                             max_connection_attempts=args.max_connection_attempts,
+                             request_timeout=args.request_timeout,
+                             commit_mode=args.commit_mode)
+        _main(loader,
+              args.profile,
+              username=args.login.lower() if args.login is not None else None,
+              password=args.password,
+              sessionfile=args.sessionfile,
+              download_profile_pic=download_profile_pic,
+              download_posts=download_posts,
+              download_stories=download_stories,
+              download_highlights=args.highlights,
+              download_tagged=args.tagged,
+              download_igtv=args.igtv,
+              fast_update=args.fast_update,
+              max_count=int(args.count) if args.count is not None else None,
+              post_filter_str=args.post_filter,
+              storyitem_filter_str=args.storyitem_filter)
+        loader.close()
+    except InstaloaderException as err:
+        raise SystemExit("Fatal error: %s" % err)
+
+
+if __name__ == "__main__":
+    main()

instaloader/exceptions.py

@@ -0,0 +1,66 @@
+class InstaloaderException(Exception):
+    """Base exception for this script.
+
+    :note: This exception should not be raised directly."""
+    pass
+
+
+class QueryReturnedBadRequestException(InstaloaderException):
+    pass
+
+
+class QueryReturnedForbiddenException(InstaloaderException):
+    pass
+
+
+class ProfileNotExistsException(InstaloaderException):
+    pass
+
+
+class ProfileHasNoPicsException(InstaloaderException):
+    """
+    .. deprecated:: 4.2.2
+       Not raised anymore.
+    """
+    pass
+
+
+class PrivateProfileNotFollowedException(InstaloaderException):
+    pass
+
+
+class LoginRequiredException(InstaloaderException):
+    pass
+
+
+class TwoFactorAuthRequiredException(InstaloaderException):
+    pass
+
+
+class InvalidArgumentException(InstaloaderException):
+    pass
+
+
+class BadResponseException(InstaloaderException):
+    pass
+
+
+class BadCredentialsException(InstaloaderException):
+    pass
+
+
+class ConnectionException(InstaloaderException):
+    pass
+
+
+class PostChangedException(InstaloaderException):
+    """.. versionadded:: 4.2.2"""
+    pass
+
+
+class QueryReturnedNotFoundException(ConnectionException):
+    pass
+
+
+class TooManyRequestsException(ConnectionException):
+    pass

instaloader/instaloadercontext.py

@@ -0,0 +1,595 @@
+import hashlib
+import json
+import pickle
+import random
+import re
+import shutil
+import sys
+import textwrap
+import time
+import urllib.parse
+from contextlib import contextmanager
+from datetime import datetime, timedelta
+from functools import partial
+from typing import Any, Callable, Dict, Iterator, List, Optional, Union
+
+import requests
+import requests.utils
+
+from .exceptions import *
+
+
+def copy_session(session: requests.Session, request_timeout: Optional[float] = None) -> requests.Session:
+    """Duplicates a requests.Session."""
+    new = requests.Session()
+    new.cookies = requests.utils.cookiejar_from_dict(requests.utils.dict_from_cookiejar(session.cookies))
+    new.headers = session.headers.copy() # type: ignore
+    if request_timeout is not None:
+        # Override default timeout behavior.
+        # Need to silence mypy bug for this. See: https://github.com/python/mypy/issues/2427
+        new.request = partial(new.request, timeout=request_timeout) # type: ignore
+    return new
+
+
+def default_user_agent() -> str:
+    return 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 ' \
+           '(KHTML, like Gecko) Chrome/51.0.2704.79 Safari/537.36'
+
+
+class InstaloaderContext:
+    """Class providing methods for (error) logging and low-level communication with Instagram.
+
+    It is not thought to be instantiated directly, rather :class:`Instaloader` instances maintain a context
+    object.
+
+    For logging, it provides :meth:`log`, :meth:`error`, :meth:`error_catcher`.
+
+    It provides low-level communication routines :meth:`get_json`, :meth:`graphql_query`, :meth:`graphql_node_list`,
+    :meth:`get_and_write_raw` and implements mechanisms for rate controlling and error handling.
+
+    Further, it provides methods for logging in and general session handles, which are used by that routines in
+    class :class:`Instaloader`.
+    """
+
+    def __init__(self, sleep: bool = True, quiet: bool = False, user_agent: Optional[str] = None,
+                 max_connection_attempts: int = 3, request_timeout: Optional[float] = None):
+
+        self.user_agent = user_agent if user_agent is not None else default_user_agent()
+        self.request_timeout = request_timeout
+        self._session = self.get_anonymous_session()
+        self.username = None
+        self.sleep = sleep
+        self.quiet = quiet
+        self.max_connection_attempts = max_connection_attempts
+        self._graphql_page_length = 50
+        self._root_rhx_gis = None
+        self.two_factor_auth_pending = None
+
+        # error log, filled with error() and printed at the end of Instaloader.main()
+        self.error_log = []                      # type: List[str]
+
+        # For the adaption of sleep intervals (rate control)
+        self._graphql_query_timestamps = dict()  # type: Dict[str, List[float]]
+        self._graphql_earliest_next_request_time = 0.0
+
+        # Can be set to True for testing, disables supression of InstaloaderContext._error_catcher
+        self.raise_all_errors = False
+
+        # Cache profile from id (mapping from id to Profile)
+        self.profile_id_cache = dict()           # type: Dict[int, Any]
+
+    @contextmanager
+    def anonymous_copy(self):
+        session = self._session
+        username = self.username
+        self._session = self.get_anonymous_session()
+        self.username = None
+        try:
+            yield self
+        finally:
+            self._session.close()
+            self.username = username
+            self._session = session
+
+    @property
+    def is_logged_in(self) -> bool:
+        """True, if this Instaloader instance is logged in."""
+        return bool(self.username)
+
+    def log(self, *msg, sep='', end='\n', flush=False):
+        """Log a message to stdout that can be suppressed with --quiet."""
+        if not self.quiet:
+            print(*msg, sep=sep, end=end, flush=flush)
+
+    def error(self, msg, repeat_at_end=True):
+        """Log a non-fatal error message to stderr, which is repeated at program termination.
+
+        :param msg: Message to be printed.
+        :param repeat_at_end: Set to false if the message should be printed, but not repeated at program termination."""
+        print(msg, file=sys.stderr)
+        if repeat_at_end:
+            self.error_log.append(msg)
+
+    def close(self):
+        """Print error log and close session"""
+        if self.error_log and not self.quiet:
+            print("\nErrors occurred:", file=sys.stderr)
+            for err in self.error_log:
+                print(err, file=sys.stderr)
+        self._session.close()
+
+    @contextmanager
+    def error_catcher(self, extra_info: Optional[str] = None):
+        """
+        Context manager to catch, print and record InstaloaderExceptions.
+
+        :param extra_info: String to prefix error message with."""
+        try:
+            yield
+        except InstaloaderException as err:
+            if extra_info:
+                self.error('{}: {}'.format(extra_info, err))
+            else:
+                self.error('{}'.format(err))
+            if self.raise_all_errors:
+                raise
+
+    def _default_http_header(self, empty_session_only: bool = False) -> Dict[str, str]:
+        """Returns default HTTP header we use for requests."""
+        header = {'Accept-Encoding': 'gzip, deflate',
+                  'Accept-Language': 'en-US,en;q=0.8',
+                  'Connection': 'keep-alive',
+                  'Content-Length': '0',
+                  'Host': 'www.instagram.com',
+                  'Origin': 'https://www.instagram.com',
+                  'Referer': 'https://www.instagram.com/',
+                  'User-Agent': self.user_agent,
+                  'X-Instagram-AJAX': '1',
+                  'X-Requested-With': 'XMLHttpRequest'}
+        if empty_session_only:
+            del header['Host']
+            del header['Origin']
+            del header['Referer']
+            del header['X-Instagram-AJAX']
+            del header['X-Requested-With']
+        return header
+
+    def get_anonymous_session(self) -> requests.Session:
+        """Returns our default anonymous requests.Session object."""
+        session = requests.Session()
+        session.cookies.update({'sessionid': '', 'mid': '', 'ig_pr': '1',
+                                'ig_vw': '1920', 'csrftoken': '',
+                                's_network': '', 'ds_user_id': ''})
+        session.headers.update(self._default_http_header(empty_session_only=True))
+        if self.request_timeout is not None:
+            # Override default timeout behavior.
+            # Need to silence mypy bug for this. See: https://github.com/python/mypy/issues/2427
+            session.request = partial(session.request, timeout=self.request_timeout) # type: ignore
+        return session
+
+    def save_session_to_file(self, sessionfile):
+        """Not meant to be used directly, use :meth:`Instaloader.save_session_to_file`."""
+        pickle.dump(requests.utils.dict_from_cookiejar(self._session.cookies), sessionfile)
+
+    def load_session_from_file(self, username, sessionfile):
+        """Not meant to be used directly, use :meth:`Instaloader.load_session_from_file`."""
+        session = requests.Session()
+        session.cookies = requests.utils.cookiejar_from_dict(pickle.load(sessionfile))
+        session.headers.update(self._default_http_header())
+        session.headers.update({'X-CSRFToken': session.cookies.get_dict()['csrftoken']})
+        if self.request_timeout is not None:
+            # Override default timeout behavior.
+            # Need to silence mypy bug for this. See: https://github.com/python/mypy/issues/2427
+            session.request = partial(session.request, timeout=self.request_timeout) # type: ignore
+        self._session = session
+        self.username = username
+
+    def test_login(self) -> Optional[str]:
+        """Not meant to be used directly, use :meth:`Instaloader.test_login`."""
+        data = self.graphql_query("d6f4427fbe92d846298cf93df0b937d3", {})
+        return data["data"]["user"]["username"] if data["data"]["user"] is not None else None
+
+    def login(self, user, passwd):
+        """Not meant to be used directly, use :meth:`Instaloader.login`.
+
+        :raises InvalidArgumentException: If the provided username does not exist.
+        :raises BadCredentialsException: If the provided password is wrong.
+        :raises ConnectionException: If connection to Instagram failed.
+        :raises TwoFactorAuthRequiredException: First step of 2FA login done, now call
+           :meth:`Instaloader.two_factor_login`."""
+        # pylint:disable=import-outside-toplevel
+        import http.client
+        # pylint:disable=protected-access
+        http.client._MAXHEADERS = 200
+        session = requests.Session()
+        session.cookies.update({'sessionid': '', 'mid': '', 'ig_pr': '1',
+                                'ig_vw': '1920', 'ig_cb': '1', 'csrftoken': '',
+                                's_network': '', 'ds_user_id': ''})
+        session.headers.update(self._default_http_header())
+        if self.request_timeout is not None:
+            # Override default timeout behavior.
+            # Need to silence mypy bug for this. See: https://github.com/python/mypy/issues/2427
+            session.request = partial(session.request, timeout=self.request_timeout) # type: ignore
+        session.get('https://www.instagram.com/web/__mid/')
+        csrf_token = session.cookies.get_dict()['csrftoken']
+        session.headers.update({'X-CSRFToken': csrf_token})
+        # Not using self.get_json() here, because we need to access csrftoken cookie
+        self.do_sleep()
+        # Workaround credits to pgrimaud.
+        # See: https://github.com/pgrimaud/instagram-user-feed/commit/96ad4cf54d1ad331b337f325c73e664999a6d066
+        enc_password = '#PWD_INSTAGRAM_BROWSER:0:{}:{}'.format(int(datetime.now().timestamp()), passwd)
+        login = session.post('https://www.instagram.com/accounts/login/ajax/',
+                             data={'enc_password': enc_password, 'username': user}, allow_redirects=True)
+        try:
+            resp_json = login.json()
+        except json.decoder.JSONDecodeError:
+            raise ConnectionException("Login error: JSON decode fail, {} - {}.".format(login.status_code, login.reason))
+        if resp_json.get('two_factor_required'):
+            two_factor_session = copy_session(session, self.request_timeout)
+            two_factor_session.headers.update({'X-CSRFToken': csrf_token})
+            two_factor_session.cookies.update({'csrftoken': csrf_token})
+            self.two_factor_auth_pending = (two_factor_session,
+                                            user,
+                                            resp_json['two_factor_info']['two_factor_identifier'])
+            raise TwoFactorAuthRequiredException("Login error: two-factor authentication required.")
+        if resp_json.get('checkpoint_url'):
+            raise ConnectionException("Login: Checkpoint required. Point your browser to "
+                                      "https://www.instagram.com{} - "
+                                      "follow the instructions, then retry.".format(resp_json.get('checkpoint_url')))
+        if resp_json['status'] != 'ok':
+            if 'message' in resp_json:
+                raise ConnectionException("Login error: \"{}\" status, message \"{}\".".format(resp_json['status'],
+                                                                                               resp_json['message']))
+            else:
+                raise ConnectionException("Login error: \"{}\" status.".format(resp_json['status']))
+        if 'authenticated' not in resp_json:
+            # Issue #472
+            if 'message' in resp_json:
+                raise ConnectionException("Login error: Unexpected response, \"{}\".".format(resp_json['message']))
+            else:
+                raise ConnectionException("Login error: Unexpected response, this might indicate a blocked IP.")
+        if not resp_json['authenticated']:
+            if resp_json['user']:
+                # '{"authenticated": false, "user": true, "status": "ok"}'
+                raise BadCredentialsException('Login error: Wrong password.')
+            else:
+                # '{"authenticated": false, "user": false, "status": "ok"}'
+                # Raise InvalidArgumentException rather than BadCredentialException, because BadCredentialException
+                # triggers re-asking of password in Instaloader.interactive_login(), which makes no sense if the
+                # username is invalid.
+                raise InvalidArgumentException('Login error: User {} does not exist.'.format(user))
+        # '{"authenticated": true, "user": true, "userId": ..., "oneTapPrompt": false, "status": "ok"}'
+        session.headers.update({'X-CSRFToken': login.cookies['csrftoken']})
+        self._session = session
+        self.username = user
+
+    def two_factor_login(self, two_factor_code):
+        """Second step of login if 2FA is enabled.
+        Not meant to be used directly, use :meth:`Instaloader.two_factor_login`.
+
+        :raises InvalidArgumentException: No two-factor authentication pending.
+        :raises BadCredentialsException: 2FA verification code invalid.
+
+        .. versionadded:: 4.2"""
+        if not self.two_factor_auth_pending:
+            raise InvalidArgumentException("No two-factor authentication pending.")
+        (session, user, two_factor_id) = self.two_factor_auth_pending
+
+        login = session.post('https://www.instagram.com/accounts/login/ajax/two_factor/',
+                             data={'username': user, 'verificationCode': two_factor_code, 'identifier': two_factor_id},
+                             allow_redirects=True)
+        resp_json = login.json()
+        if resp_json['status'] != 'ok':
+            if 'message' in resp_json:
+                raise BadCredentialsException("Login error: {}".format(resp_json['message']))
+            else:
+                raise BadCredentialsException("Login error: \"{}\" status.".format(resp_json['status']))
+        session.headers.update({'X-CSRFToken': login.cookies['csrftoken']})
+        self._session = session
+        self.username = user
+        self.two_factor_auth_pending = None
+
+    def do_sleep(self):
+        """Sleep a short time if self.sleep is set. Called before each request to instagram.com."""
+        if self.sleep:
+            time.sleep(min(random.expovariate(0.7), 5.0))
+
+    def _dump_query_timestamps(self, current_time: float):
+        """Output the number of GraphQL queries grouped by their query_hash within the last time."""
+        windows = [10, 11, 15, 20, 30, 60]
+        print("GraphQL requests:", file=sys.stderr)
+        for query_hash, times in self._graphql_query_timestamps.items():
+            print("  {}".format(query_hash), file=sys.stderr)
+            for window in windows:
+                reqs_in_sliding_window = sum(t > current_time - window * 60 for t in times)
+                print("    last {} minutes: {} requests".format(window, reqs_in_sliding_window), file=sys.stderr)
+
+    def _graphql_request_count_per_sliding_window(self, query_hash: str) -> int:
+        """Return how many GraphQL requests can be done within the sliding window."""
+        if self.is_logged_in:
+            max_reqs = {'1cb6ec562846122743b61e492c85999f': 20,
+                        '33ba35852cb50da46f5b5e889df7d159': 20,
+                        'iphone': 100,
+                        'other': 100}
+        else:
+            max_reqs = {'1cb6ec562846122743b61e492c85999f': 200,
+                        '33ba35852cb50da46f5b5e889df7d159': 200,
+                        'other': 200}
+        return max_reqs.get(query_hash) or min(max_reqs.values())
+
+    def _graphql_query_waittime(self, query_hash: str, current_time: float, untracked_queries: bool = False) -> float:
+        """Calculate time needed to wait before GraphQL query can be executed."""
+        sliding_window = 660
+        if query_hash not in self._graphql_query_timestamps:
+            self._graphql_query_timestamps[query_hash] = []
+        self._graphql_query_timestamps[query_hash] = list(filter(lambda t: t > current_time - 60 * 60,
+                                                                 self._graphql_query_timestamps[query_hash]))
+        reqs_in_sliding_window = list(filter(lambda t: t > current_time - sliding_window,
+                                             self._graphql_query_timestamps[query_hash]))
+        count_per_sliding_window = self._graphql_request_count_per_sliding_window(query_hash)
+        if len(reqs_in_sliding_window) < count_per_sliding_window and not untracked_queries:
+            return max(0, self._graphql_earliest_next_request_time - current_time)
+        next_request_time = min(reqs_in_sliding_window) + sliding_window + 6
+        if untracked_queries:
+            self._graphql_earliest_next_request_time = next_request_time
+        return round(max(next_request_time, self._graphql_earliest_next_request_time) - current_time)
+
+    def _ratecontrol_graphql_query(self, query_hash: str, untracked_queries: bool = False):
+        """Called before a GraphQL query is made in order to stay within Instagram's rate limits.
+
+        :param query_hash: The query_hash parameter of the query.
+        :param untracked_queries: True, if 429 has been returned to apply 429 logic.
+        """
+        if not untracked_queries:
+            waittime = self._graphql_query_waittime(query_hash, time.monotonic(), untracked_queries)
+            assert waittime >= 0
+            if waittime > 10:
+                self.log('\nToo many queries in the last time. Need to wait {} seconds, until {:%H:%M}.'
+                         .format(waittime, datetime.now() + timedelta(seconds=waittime)))
+            time.sleep(waittime)
+            if query_hash not in self._graphql_query_timestamps:
+                self._graphql_query_timestamps[query_hash] = [time.monotonic()]
+            else:
+                self._graphql_query_timestamps[query_hash].append(time.monotonic())
+        else:
+            text_for_429 = ("HTTP error code 429 was returned because too many queries occurred in the last time. "
+                            "Please do not use Instagram in your browser or run multiple instances of Instaloader "
+                            "in parallel.")
+            print(textwrap.fill(text_for_429), file=sys.stderr)
+            current_time = time.monotonic()
+            waittime = self._graphql_query_waittime(query_hash, current_time, untracked_queries)
+            assert waittime >= 0
+            if waittime > 10:
+                self.log('The request will be retried in {} seconds, at {:%H:%M}.'
+                         .format(waittime, datetime.now() + timedelta(seconds=waittime)))
+            self._dump_query_timestamps(current_time)
+            time.sleep(waittime)
+
+    def get_json(self, path: str, params: Dict[str, Any], host: str = 'www.instagram.com',
+                 session: Optional[requests.Session] = None, _attempt=1) -> Dict[str, Any]:
+        """JSON request to Instagram.
+
+        :param path: URL, relative to the given domain which defaults to www.instagram.com/
+        :param params: GET parameters
+        :param host: Domain part of the URL from where to download the requested JSON; defaults to www.instagram.com
+        :param session: Session to use, or None to use self.session
+        :return: Decoded response dictionary
+        :raises QueryReturnedBadRequestException: When the server responds with a 400.
+        :raises QueryReturnedNotFoundException: When the server responds with a 404.
+        :raises ConnectionException: When query repeatedly failed.
+        """
+        is_graphql_query = 'query_hash' in params and 'graphql/query' in path
+        is_iphone_query = host == 'i.instagram.com'
+        is_other_query = not is_graphql_query and host == "www.instagram.com"
+        sess = session if session else self._session
+        try:
+            self.do_sleep()
+            if is_graphql_query:
+                self._ratecontrol_graphql_query(params['query_hash'])
+            if is_iphone_query:
+                self._ratecontrol_graphql_query('iphone')
+            if is_other_query:
+                self._ratecontrol_graphql_query('other')
+            resp = sess.get('https://{0}/{1}'.format(host, path), params=params, allow_redirects=False)
+            while resp.is_redirect:
+                redirect_url = resp.headers['location']
+                self.log('\nHTTP redirect from https://{0}/{1} to {2}'.format(host, path, redirect_url))
+                if redirect_url.startswith('https://www.instagram.com/accounts/login'):
+                    # alternate rate limit exceeded behavior
+                    raise TooManyRequestsException("429 Too Many Requests: redirected to login")
+                if redirect_url.startswith('https://{}/'.format(host)):
+                    resp = sess.get(redirect_url if redirect_url.endswith('/') else redirect_url + '/',
+                                    params=params, allow_redirects=False)
+                else:
+                    break
+            if resp.status_code == 400:
+                raise QueryReturnedBadRequestException("400 Bad Request")
+            if resp.status_code == 404:
+                raise QueryReturnedNotFoundException("404 Not Found")
+            if resp.status_code == 429:
+                raise TooManyRequestsException("429 Too Many Requests")
+            if resp.status_code != 200:
+                raise ConnectionException("HTTP error code {}.".format(resp.status_code))
+            is_html_query = not is_graphql_query and not "__a" in params and host == "www.instagram.com"
+            if is_html_query:
+                match = re.search(r'window\._sharedData = (.*);</script>', resp.text)
+                if match is None:
+                    raise QueryReturnedNotFoundException("Could not find \"window._sharedData\" in html response.")
+                resp_json = json.loads(match.group(1))
+                entry_data = resp_json.get('entry_data')
+                post_or_profile_page = list(entry_data.values())[0] if entry_data is not None else None
+                if post_or_profile_page is None:
+                    raise ConnectionException("\"window._sharedData\" does not contain required keys.")
+                # If GraphQL data is missing in `window._sharedData`, search for it in `__additionalDataLoaded`.
+                if 'graphql' not in post_or_profile_page[0]:
+                    match = re.search(r'window\.__additionalDataLoaded\([^{]+{"graphql":({.*})}\);</script>',
+                                      resp.text)
+                    if match is not None:
+                        post_or_profile_page[0]['graphql'] = json.loads(match.group(1))
+                return resp_json
+            else:
+                resp_json = resp.json()
+            if 'status' in resp_json and resp_json['status'] != "ok":
+                if 'message' in resp_json:
+                    raise ConnectionException("Returned \"{}\" status, message \"{}\".".format(resp_json['status'],
+                                                                                               resp_json['message']))
+                else:
+                    raise ConnectionException("Returned \"{}\" status.".format(resp_json['status']))
+            return resp_json
+        except (ConnectionException, json.decoder.JSONDecodeError, requests.exceptions.RequestException) as err:
+            error_string = "JSON Query to {}: {}".format(path, err)
+            if _attempt == self.max_connection_attempts:
+                if isinstance(err, QueryReturnedNotFoundException):
+                    raise QueryReturnedNotFoundException(error_string) from err
+                else:
+                    raise ConnectionException(error_string) from err
+            self.error(error_string + " [retrying; skip with ^C]", repeat_at_end=False)
+            try:
+                if is_graphql_query and isinstance(err, TooManyRequestsException):
+                    self._ratecontrol_graphql_query(params['query_hash'], untracked_queries=True)
+                if is_iphone_query and isinstance(err, TooManyRequestsException):
+                    self._ratecontrol_graphql_query('iphone', untracked_queries=True)
+                if is_other_query and isinstance(err, TooManyRequestsException):
+                    self._ratecontrol_graphql_query('other', untracked_queries=True)
+                return self.get_json(path=path, params=params, host=host, session=sess, _attempt=_attempt + 1)
+            except KeyboardInterrupt:
+                self.error("[skipped by user]", repeat_at_end=False)
+                raise ConnectionException(error_string) from err
+
+    def graphql_query(self, query_hash: str, variables: Dict[str, Any],
+                      referer: Optional[str] = None, rhx_gis: Optional[str] = None) -> Dict[str, Any]:
+        """
+        Do a GraphQL Query.
+
+        :param query_hash: Query identifying hash.
+        :param variables: Variables for the Query.
+        :param referer: HTTP Referer, or None.
+        :param rhx_gis: 'rhx_gis' variable as somewhere returned by Instagram, needed to 'sign' request
+        :return: The server's response dictionary.
+        """
+        with copy_session(self._session, self.request_timeout) as tmpsession:
+            tmpsession.headers.update(self._default_http_header(empty_session_only=True))
+            del tmpsession.headers['Connection']
+            del tmpsession.headers['Content-Length']
+            tmpsession.headers['authority'] = 'www.instagram.com'
+            tmpsession.headers['scheme'] = 'https'
+            tmpsession.headers['accept'] = '*/*'
+            if referer is not None:
+                tmpsession.headers['referer'] = urllib.parse.quote(referer)
+
+            variables_json = json.dumps(variables, separators=(',', ':'))
+
+            if rhx_gis:
+                #self.log("rhx_gis {} query_hash {}".format(rhx_gis, query_hash))
+                values = "{}:{}".format(rhx_gis, variables_json)
+                x_instagram_gis = hashlib.md5(values.encode()).hexdigest()
+                tmpsession.headers['x-instagram-gis'] = x_instagram_gis
+
+            resp_json = self.get_json('graphql/query',
+                                      params={'query_hash': query_hash,
+                                              'variables': variables_json},
+                                      session=tmpsession)
+        if 'status' not in resp_json:
+            self.error("GraphQL response did not contain a \"status\" field.")
+        return resp_json
+
+    def graphql_node_list(self, query_hash: str, query_variables: Dict[str, Any],
+                          query_referer: Optional[str],
+                          edge_extractor: Callable[[Dict[str, Any]], Dict[str, Any]],
+                          rhx_gis: Optional[str] = None,
+                          first_data: Optional[Dict[str, Any]] = None) -> Iterator[Dict[str, Any]]:
+        """Retrieve a list of GraphQL nodes."""
+
+        def _query():
+            query_variables['first'] = self._graphql_page_length
+            try:
+                return edge_extractor(self.graphql_query(query_hash, query_variables, query_referer, rhx_gis))
+            except QueryReturnedBadRequestException:
+                new_page_length = int(self._graphql_page_length / 2)
+                if new_page_length >= 12:
+                    self._graphql_page_length = new_page_length
+                    self.error("HTTP Error 400 (Bad Request) on GraphQL Query. Retrying with shorter page length.",
+                               repeat_at_end=False)
+                    return _query()
+                else:
+                    raise
+
+        if first_data:
+            data = first_data
+        else:
+            data = _query()
+        yield from (edge['node'] for edge in data['edges'])
+        while data['page_info']['has_next_page']:
+            query_variables['after'] = data['page_info']['end_cursor']
+            data = _query()
+            yield from (edge['node'] for edge in data['edges'])
+
+    def get_iphone_json(self, path: str, params: Dict[str, Any]) -> Dict[str, Any]:
+        """JSON request to ``i.instagram.com``.
+
+        :param path: URL, relative to ``i.instagram.com/``
+        :param params: GET parameters
+        :return: Decoded response dictionary
+        :raises QueryReturnedBadRequestException: When the server responds with a 400.
+        :raises QueryReturnedNotFoundException: When the server responds with a 404.
+        :raises ConnectionException: When query repeatedly failed.
+
+        .. versionadded:: 4.2.1"""
+        with copy_session(self._session, self.request_timeout) as tempsession:
+            tempsession.headers['User-Agent'] = 'Instagram 10.3.2 (iPhone7,2; iPhone OS 9_3_3; en_US; en-US; ' \
+                                                'scale=2.00; 750x1334) AppleWebKit/420+'
+            for header in ['Host', 'Origin', 'X-Instagram-AJAX', 'X-Requested-With']:
+                tempsession.headers.pop(header, None)
+            return self.get_json(path, params, 'i.instagram.com', tempsession)
+
+    def write_raw(self, resp: Union[bytes, requests.Response], filename: str) -> None:
+        """Write raw response data into a file.
+
+        .. versionadded:: 4.2.1"""
+        self.log(filename, end=' ', flush=True)
+        with open(filename, 'wb') as file:
+            if isinstance(resp, requests.Response):
+                shutil.copyfileobj(resp.raw, file)
+            else:
+                file.write(resp)
+
+    def get_raw(self, url: str, _attempt=1) -> requests.Response:
+        """Downloads a file anonymously.
+
+        :raises QueryReturnedNotFoundException: When the server responds with a 404.
+        :raises QueryReturnedForbiddenException: When the server responds with a 403.
+        :raises ConnectionException: When download failed.
+
+        .. versionadded:: 4.2.1"""
+        with self.get_anonymous_session() as anonymous_session:
+            resp = anonymous_session.get(url, stream=True)
+        if resp.status_code == 200:
+            resp.raw.decode_content = True
+            return resp
+        else:
+            if resp.status_code == 403:
+                # suspected invalid URL signature
+                raise QueryReturnedForbiddenException("403 when accessing {}.".format(url))
+            if resp.status_code == 404:
+                # 404 not worth retrying.
+                raise QueryReturnedNotFoundException("404 when accessing {}.".format(url))
+            raise ConnectionException("HTTP error code {}.".format(resp.status_code))
+
+    def get_and_write_raw(self, url: str, filename: str) -> None:
+        """Downloads and writes anonymously-requested raw data into a file.
+
+        :raises QueryReturnedNotFoundException: When the server responds with a 404.
+        :raises QueryReturnedForbiddenException: When the server responds with a 403.
+        :raises ConnectionException: When download repeatedly failed."""
+        self.write_raw(self.get_raw(url), filename)
+
+    @property
+    def root_rhx_gis(self) -> Optional[str]:
+        """rhx_gis string returned in the / query."""
+        if self.is_logged_in:
+            # At the moment, rhx_gis seems to be required for anonymous requests only. By returning None when logged
+            # in, we can save the root_rhx_gis lookup query.
+            return None
+        if self._root_rhx_gis is None:
+            self._root_rhx_gis = self.get_json('', {}).get('rhx_gis', '')
+        return self._root_rhx_gis or None

setup.py

@@ -11,7 +11,7 @@ SRC = os.path.abspath(os.path.dirname(__file__))
 
 
 def get_version():
-    with open(os.path.join(SRC, 'instaloader.py')) as f:
+    with open(os.path.join(SRC, 'instaloader/__init__.py')) as f:
         for line in f:
             m = re.match("__version__ = '(.*)'", line)
             if m:
@@ -37,7 +37,7 @@ keywords = (['instagram', 'instagram-scraper', 'instagram-client', 'instagram-fe
 setup(
     name='instaloader',
     version=get_version(),
-    py_modules=['instaloader'],
+    packages=['instaloader'],
     url='https://instaloader.github.io/',
     license='MIT',
     author='Alexander Graf, André Koch-Kramer',
@@ -47,17 +47,20 @@ setup(
     long_description=open(os.path.join(SRC, 'README.rst')).read(),
     install_requires=requirements,
     python_requires='>=3.5',
-    entry_points={'console_scripts': ['instaloader=instaloader:main']},
+    entry_points={'console_scripts': ['instaloader=instaloader.__main__:main']},
     zip_safe=True,
     keywords=keywords,
     classifiers=[
         'Development Status :: 5 - Production/Stable',
         'Environment :: Console',
         'Intended Audience :: End Users/Desktop',
+        'Intended Audience :: Developers',
         'License :: OSI Approved :: MIT License',
         'Operating System :: OS Independent',
         'Programming Language :: Python :: 3.5',
         'Programming Language :: Python :: 3.6',
+        'Programming Language :: Python :: 3.7',
+        'Programming Language :: Python :: 3.8',
         'Programming Language :: Python :: 3 :: Only',
         'Topic :: Internet',
         'Topic :: Multimedia :: Graphics'

test/instaloader_unittests.py

@@ -0,0 +1,209 @@
+"""Unit Tests for Instaloader"""
+
+import os
+import shutil
+import tempfile
+import unittest
+from itertools import islice
+from typing import Dict, List
+
+import instaloader
+
+PROFILE_WITH_HIGHLIGHTS = 325732271
+PUBLIC_PROFILE = "selenagomez"
+PUBLIC_PROFILE_ID = 460563723
+PUBLIC_PROFILE_WITH_IGTV = "natgeo"
+HASHTAG = "kitten"
+LOCATION = "362629379"
+OWN_USERNAME = "aandergr"
+NORMAL_MAX_COUNT = 2
+PAGING_MAX_COUNT = 15
+PRIVATE_PROFILE = "aandergr"
+PRIVATE_PROFILE_ID = 1706625676
+EMPTY_PROFILE = "not_public"
+EMPTY_PROFILE_ID = 1928659031
+
+# Preserve query timestamps (rate control) between tests to not get rate limited
+instaloadercontext_query_timestamps = dict()  # type: Dict[str, List[float]]
+
+
+class TestInstaloaderAnonymously(unittest.TestCase):
+
+    def setUp(self):
+        self.dir = tempfile.mkdtemp()
+        print("Testing in {}".format(self.dir))
+        os.chdir(self.dir)
+        self.L = instaloader.Instaloader(download_geotags=True,
+                                         download_comments=True,
+                                         save_metadata=True)
+        self.L.context.raise_all_errors = True
+        # pylint:disable=protected-access
+        self.L.context._graphql_query_timestamps = instaloadercontext_query_timestamps.copy()
+
+    def tearDown(self):
+        # pylint:disable=global-statement,protected-access
+        global instaloadercontext_query_timestamps
+        instaloadercontext_query_timestamps = self.L.context._graphql_query_timestamps.copy()
+        self.L.close()
+        os.chdir('/')
+        print("Removing {}".format(self.dir))
+        shutil.rmtree(self.dir)
+
+    def post_paging_test(self, iterator):
+        previous_post = None
+        for post in islice(iterator, PAGING_MAX_COUNT):
+            print(post)
+            if previous_post:
+                self.assertTrue(post.date_utc < previous_post.date_utc)
+            previous_post = post
+
+    @unittest.skip('')
+    def test_public_profile_download(self):
+        profiles = {self.L.check_profile_id(PUBLIC_PROFILE)}
+        self.L.download_profiles(profiles, profile_pic=False, fast_update=True, raise_errors=True)
+        self.L.download_profiles(profiles, profile_pic=False, fast_update=True, raise_errors=True)
+
+    def test_public_profile_paging(self):
+        self.post_paging_test(instaloader.Profile.from_username(self.L.context, PUBLIC_PROFILE).get_posts())
+
+    def test_profile_pic_download(self):
+        self.L.download_profiles({self.L.check_profile_id(PUBLIC_PROFILE)}, posts=False, raise_errors=True)
+
+    def test_hashtag_download(self):
+        self.L.download_hashtag(HASHTAG, NORMAL_MAX_COUNT)
+
+    def test_hashtag_paging(self):
+        for count, post in enumerate(instaloader.Hashtag.from_name(self.L.context, HASHTAG).get_all_posts()):
+            print(post)
+            if count == PAGING_MAX_COUNT:
+                break
+
+    def test_get_id_by_username(self):
+        self.assertEqual(PUBLIC_PROFILE_ID,
+                         instaloader.Profile.from_username(self.L.context, PUBLIC_PROFILE).userid)
+
+    def test_get_username_by_id_private(self):
+        self.assertEqual(PRIVATE_PROFILE.lower(),
+                         instaloader.Profile.from_id(self.L.context, PRIVATE_PROFILE_ID).username)
+
+    def test_get_username_by_id_public(self):
+        self.assertEqual(PUBLIC_PROFILE.lower(),
+                         instaloader.Profile.from_id(self.L.context, PUBLIC_PROFILE_ID).username)
+
+    def test_get_username_by_id_empty(self):
+        self.assertEqual(EMPTY_PROFILE.lower(),
+                         instaloader.Profile.from_id(self.L.context, EMPTY_PROFILE_ID).username)
+
+    def test_post_from_mediaid(self):
+        for post in instaloader.Profile.from_username(self.L.context, PUBLIC_PROFILE).get_posts():
+            post2 = instaloader.Post.from_mediaid(self.L.context, post.mediaid)
+            self.assertEqual(post, post2)
+            break
+
+    def test_public_profile_tagged_paging(self):
+        for post in islice(instaloader.Profile.from_username(self.L.context, PUBLIC_PROFILE).get_tagged_posts(),
+                           PAGING_MAX_COUNT):
+            print(post)
+
+    def test_public_profile_igtv(self):
+        for post in islice(instaloader.Profile.from_username(self.L.context, PUBLIC_PROFILE_WITH_IGTV).get_igtv_posts(),
+                           PAGING_MAX_COUNT):
+            print(post)
+
+
+class TestInstaloaderLoggedIn(TestInstaloaderAnonymously):
+
+    def setUp(self):
+        super().setUp()
+        self.L.load_session_from_file(OWN_USERNAME)
+
+    @unittest.skip('')
+    def test_stories_download(self):
+        self.L.download_stories()
+
+    @unittest.skip('')
+    def test_private_profile_download(self):
+        self.L.download_profiles({self.L.check_profile_id(PRIVATE_PROFILE)}, stories=True, raise_errors=True)
+
+    def test_stories_paging(self):
+        for user_story in self.L.get_stories():
+            print("profile {}.".format(user_story.owner_username))
+            for item in user_story.get_items():
+                print(item)
+
+    def test_highlights_paging(self):
+        for user_highlight in self.L.get_highlights(PROFILE_WITH_HIGHLIGHTS):
+            print("Retrieving {} highlights \"{}\" from profile {}".format(user_highlight.itemcount,
+                                                                           user_highlight.title,
+                                                                           user_highlight.owner_username))
+            for item in user_highlight.get_items():
+                print(item)
+
+    def test_private_profile_paging(self):
+        self.post_paging_test(instaloader.Profile.from_username(self.L.context, PRIVATE_PROFILE).get_posts())
+
+    def test_feed_download(self):
+        self.L.download_feed_posts(NORMAL_MAX_COUNT)
+
+    def test_feed_paging(self):
+        for count, post in enumerate(self.L.get_feed_posts()):
+            print(post)
+            if count == PAGING_MAX_COUNT:
+                break
+
+    def test_saved_download(self):
+        self.L.download_saved_posts(NORMAL_MAX_COUNT)
+
+    def test_saved_paging(self):
+        for count, post in enumerate(instaloader.Profile.from_username(self.L.context, OWN_USERNAME).get_saved_posts()):
+            print(post)
+            if count == PAGING_MAX_COUNT:
+                break
+
+    def test_test_login(self):
+        self.assertEqual(OWN_USERNAME, self.L.test_login())
+
+    def test_followees_and_stories(self):
+        profile = instaloader.Profile.from_username(self.L.context, OWN_USERNAME)
+        followees = set(islice(profile.get_followees(), PAGING_MAX_COUNT))
+        self.L.download_profiles(followees, profile_pic=False, stories=True, posts=False, raise_errors=True)
+
+    def test_get_followees(self):
+        profile = instaloader.Profile.from_username(self.L.context, OWN_USERNAME)
+        for f in profile.get_followees():
+            print(f.username)
+
+    def test_get_followers(self):
+        profile = instaloader.Profile.from_username(self.L.context, OWN_USERNAME)
+        for f in profile.get_followers():
+            print(f.username)
+
+    def test_get_similar_accounts(self):
+        profile = instaloader.Profile.from_username(self.L.context, OWN_USERNAME)
+        for f in profile.get_similar_accounts():
+            print(f.username)
+
+    def test_get_likes(self):
+        for post in instaloader.Profile.from_username(self.L.context, OWN_USERNAME).get_posts():
+            for like in post.get_likes():
+                print(like.username)
+            break
+
+    def test_explore_paging(self):
+        for count, post in enumerate(self.L.get_explore_posts()):
+            print(post)
+            if count == PAGING_MAX_COUNT:
+                break
+
+    def test_location_download(self):
+        self.L.download_location(LOCATION, NORMAL_MAX_COUNT)
+
+    def test_location_paging(self):
+        for count, post in enumerate(self.L.get_location_posts(LOCATION)):
+            print(post)
+            if count == PAGING_MAX_COUNT:
+                break
+
+
+if __name__ == '__main__':
+    unittest.main()