Release of version 4.2.5

Closes #239.

.github/ISSUE_TEMPLATE/bug_report.md

@@ -0,0 +1,24 @@
+---
+name: Bug report
+about: Create a report to help us improve
+
+---
+
+**Describe the bug**
+A clear and concise description of what the bug is.
+
+**To Reproduce**
+Steps to reproduce the behavior:
+(e.g. Instaloader command line)
+
+**Expected behavior**
+A clear and concise description of what you expected to happen (if not obvious).
+
+**Error messages and tracebacks**
+If applicable, add error messages and tracebacks to help explain your problem.
+
+**Instaloader version**
+`instaloader --version` output.
+
+**Additional context**
+Add any other context about the problem here.

.github/ISSUE_TEMPLATE/feature_request.md

@@ -0,0 +1,17 @@
+---
+name: Feature request
+about: Suggest an idea for this project
+
+---
+
+**Provide us a use case of the feature**
+How could the user invoke the new function? Which problem would it solve? If new information is obtained, how would it be further processed?
+
+**Describe the solution you'd like**
+A clear and concise description of what you want to happen.
+
+**Describe alternatives you've considered**
+A clear and concise description of any alternative solutions or features you've considered.
+
+**Additional context**
+Add any other context about the feature request here.

.pylintrc

@@ -0,0 +1,588 @@
+[MASTER]
+
+# A comma-separated list of package or module names from where C extensions may
+# be loaded. Extensions are loading into the active Python interpreter and may
+# run arbitrary code.
+extension-pkg-whitelist=
+
+# Add files or directories to the blacklist. They should be base names, not
+# paths.
+ignore=CVS
+
+# Add files or directories matching the regex patterns to the blacklist. The
+# regex matches against base names, not paths.
+ignore-patterns=
+
+# Python code to execute, usually for sys.path manipulation such as
+# pygtk.require().
+#init-hook=
+
+# Use multiple processes to speed up Pylint. Specifying 0 will auto-detect the
+# number of processors available to use.
+jobs=0
+
+# Control the amount of potential inferred values when inferring a single
+# object. This can help the performance when dealing with large functions or
+# complex, nested conditions.
+limit-inference-results=100
+
+# List of plugins (as comma separated values of python modules names) to load,
+# usually to register additional checkers.
+load-plugins=
+
+# Pickle collected data for later comparisons.
+persistent=yes
+
+# Specify a configuration file.
+#rcfile=
+
+# When enabled, pylint would attempt to guess common misconfiguration and emit
+# user-friendly hints instead of false-positive error messages.
+suggestion-mode=yes
+
+# Allow loading of arbitrary C extensions. Extensions are imported into the
+# active Python interpreter and may run arbitrary code.
+unsafe-load-any-extension=no
+
+
+[MESSAGES CONTROL]
+
+# Only show warnings with the listed confidence levels. Leave empty to show
+# all. Valid levels: HIGH, INFERENCE, INFERENCE_FAILURE, UNDEFINED.
+confidence=
+
+# Disable the message, report, category or checker with the given id(s). You
+# can either give multiple identifiers separated by comma (,) or put this
+# option multiple times (only on the command line, not in the configuration
+# file where it should appear only once). You can also use "--disable=all" to
+# disable everything first and then reenable specific checks. For example, if
+# you want to run only the similarities checker, you can use "--disable=all
+# --enable=similarities". If you want to run only the classes checker, but have
+# no Warning level messages displayed, use "--disable=all --enable=classes
+# --disable=W".
+disable=invalid-name,
+        missing-docstring,
+        line-too-long,
+        too-many-lines,
+        bad-whitespace,
+        print-statement,
+        parameter-unpacking,
+        unpacking-in-except,
+        old-raise-syntax,
+        backtick,
+        long-suffix,
+        old-ne-operator,
+        old-octal-literal,
+        import-star-module-level,
+        non-ascii-bytes-literal,
+        raw-checker-failed,
+        bad-inline-option,
+        locally-disabled,
+        file-ignored,
+        suppressed-message,
+        useless-suppression,
+        deprecated-pragma,
+        use-symbolic-message-instead,
+        cyclic-import,
+        too-many-instance-attributes,
+        too-many-public-methods,
+        too-many-branches,
+        too-many-arguments,
+        too-many-locals,
+        too-many-statements,
+        no-else-return,
+        inconsistent-return-statements,
+        no-else-raise,
+        unnecessary-pass,
+        wildcard-import,
+        unused-wildcard-import,
+        apply-builtin,
+        basestring-builtin,
+        buffer-builtin,
+        cmp-builtin,
+        coerce-builtin,
+        execfile-builtin,
+        file-builtin,
+        long-builtin,
+        raw_input-builtin,
+        reduce-builtin,
+        standarderror-builtin,
+        unicode-builtin,
+        xrange-builtin,
+        coerce-method,
+        delslice-method,
+        getslice-method,
+        setslice-method,
+        no-absolute-import,
+        old-division,
+        dict-iter-method,
+        dict-view-method,
+        next-method-called,
+        metaclass-assignment,
+        indexing-exception,
+        raising-string,
+        reload-builtin,
+        oct-method,
+        hex-method,
+        nonzero-method,
+        cmp-method,
+        input-builtin,
+        round-builtin,
+        intern-builtin,
+        unichr-builtin,
+        map-builtin-not-iterating,
+        zip-builtin-not-iterating,
+        range-builtin-not-iterating,
+        filter-builtin-not-iterating,
+        using-cmp-argument,
+        eq-without-hash,
+        div-method,
+        idiv-method,
+        rdiv-method,
+        exception-message-attribute,
+        invalid-str-codec,
+        sys-max-int,
+        bad-python3-import,
+        deprecated-string-function,
+        deprecated-str-translate-call,
+        deprecated-itertools-function,
+        deprecated-types-field,
+        next-method-defined,
+        dict-items-not-iterating,
+        dict-keys-not-iterating,
+        dict-values-not-iterating,
+        deprecated-operator-function,
+        deprecated-urllib-function,
+        xreadlines-attribute,
+        deprecated-sys-function,
+        exception-escape,
+        comprehension-escape
+
+# Enable the message, report, category or checker with the given id(s). You can
+# either give multiple identifier separated by comma (,) or put this option
+# multiple time (only on the command line, not in the configuration file where
+# it should appear only once). See also the "--disable" option for examples.
+enable=c-extension-no-member
+
+
+[REPORTS]
+
+# Python expression which should return a note less than 10 (10 is the highest
+# note). You have access to the variables errors warning, statement which
+# respectively contain the number of errors / warnings messages and the total
+# number of statements analyzed. This is used by the global evaluation report
+# (RP0004).
+evaluation=10.0 - ((float(5 * error + warning + refactor + convention) / statement) * 10)
+
+# Template used to display messages. This is a python new-style format string
+# used to format the message information. See doc for all details.
+#msg-template=
+
+# Set the output format. Available formats are text, parseable, colorized, json
+# and msvs (visual studio). You can also give a reporter class, e.g.
+# mypackage.mymodule.MyReporterClass.
+output-format=text
+
+# Tells whether to display a full report or only the messages.
+reports=no
+
+# Activate the evaluation score.
+score=yes
+
+
+[REFACTORING]
+
+# Maximum number of nested blocks for function / method body
+max-nested-blocks=5
+
+# Complete name of functions that never returns. When checking for
+# inconsistent-return-statements if a never returning function is called then
+# it will be considered as an explicit return statement and no message will be
+# printed.
+never-returning-functions=sys.exit
+
+
+[VARIABLES]
+
+# List of additional names supposed to be defined in builtins. Remember that
+# you should avoid defining new builtins when possible.
+additional-builtins=
+
+# Tells whether unused global variables should be treated as a violation.
+allow-global-unused-variables=yes
+
+# List of strings which can identify a callback function by name. A callback
+# name must start or end with one of those strings.
+callbacks=cb_,
+          _cb
+
+# A regular expression matching the name of dummy variables (i.e. expected to
+# not be used).
+dummy-variables-rgx=_+$|(_[a-zA-Z0-9_]*[a-zA-Z0-9]+?$)|dummy|^ignored_|^unused_
+
+# Argument names that match this expression will be ignored. Default to name
+# with leading underscore.
+ignored-argument-names=_.*|^ignored_|^unused_
+
+# Tells whether we should check for unused import in __init__ files.
+init-import=no
+
+# List of qualified module names which can have objects that can redefine
+# builtins.
+redefining-builtins-modules=six.moves,past.builtins,future.builtins,builtins,io
+
+
+[TYPECHECK]
+
+# List of decorators that produce context managers, such as
+# contextlib.contextmanager. Add to this list to register other decorators that
+# produce valid context managers.
+contextmanager-decorators=contextlib.contextmanager
+
+# List of members which are set dynamically and missed by pylint inference
+# system, and so shouldn't trigger E1101 when accessed. Python regular
+# expressions are accepted.
+generated-members=
+
+# Tells whether missing members accessed in mixin class should be ignored. A
+# mixin class is detected if its name ends with "mixin" (case insensitive).
+ignore-mixin-members=yes
+
+# Tells whether to warn about missing members when the owner of the attribute
+# is inferred to be None.
+ignore-none=yes
+
+# This flag controls whether pylint should warn about no-member and similar
+# checks whenever an opaque object is returned when inferring. The inference
+# can return multiple potential results while evaluating a Python object, but
+# some branches might not be evaluated, which results in partial inference. In
+# that case, it might be useful to still emit no-member and other checks for
+# the rest of the inferred objects.
+ignore-on-opaque-inference=yes
+
+# List of class names for which member attributes should not be checked (useful
+# for classes with dynamically set attributes). This supports the use of
+# qualified names.
+ignored-classes=optparse.Values,thread._local,_thread._local
+
+# List of module names for which member attributes should not be checked
+# (useful for modules/projects where namespaces are manipulated during runtime
+# and thus existing member attributes cannot be deduced by static analysis. It
+# supports qualified module names, as well as Unix pattern matching.
+ignored-modules=
+
+# Show a hint with possible names when a member name was not found. The aspect
+# of finding the hint is based on edit distance.
+missing-member-hint=yes
+
+# The minimum edit distance a name should have in order to be considered a
+# similar match for a missing member name.
+missing-member-hint-distance=1
+
+# The total number of similar names that should be taken in consideration when
+# showing a hint for a missing member.
+missing-member-max-choices=1
+
+
+[STRING]
+
+# This flag controls whether the implicit-str-concat-in-sequence should
+# generate a warning on implicit string concatenation in sequences defined over
+# several lines.
+check-str-concat-over-line-jumps=no
+
+
+[SPELLING]
+
+# Limits count of emitted suggestions for spelling mistakes.
+max-spelling-suggestions=4
+
+# Spelling dictionary name. Available dictionaries: none. To make it working
+# install python-enchant package..
+spelling-dict=
+
+# List of comma separated words that should not be checked.
+spelling-ignore-words=
+
+# A path to a file that contains private dictionary; one word per line.
+spelling-private-dict-file=
+
+# Tells whether to store unknown words to indicated private dictionary in
+# --spelling-private-dict-file option instead of raising a message.
+spelling-store-unknown-words=no
+
+
+[SIMILARITIES]
+
+# Ignore comments when computing similarities.
+ignore-comments=yes
+
+# Ignore docstrings when computing similarities.
+ignore-docstrings=yes
+
+# Ignore imports when computing similarities.
+ignore-imports=no
+
+# Minimum lines number of a similarity.
+min-similarity-lines=4
+
+
+[MISCELLANEOUS]
+
+# List of note tags to take in consideration, separated by a comma.
+notes=FIXME,
+      XXX,
+      TODO
+
+
+[LOGGING]
+
+# Format style used to check logging format string. `old` means using %
+# formatting, while `new` is for `{}` formatting.
+logging-format-style=old
+
+# Logging modules to check that the string format arguments are in logging
+# function parameter format.
+logging-modules=logging
+
+
+[FORMAT]
+
+# Expected format of line ending, e.g. empty (any line ending), LF or CRLF.
+expected-line-ending-format=
+
+# Regexp for a line that is allowed to be longer than the limit.
+ignore-long-lines=^\s*(# )?<?https?://\S+>?$
+
+# Number of spaces of indent required inside a hanging or continued line.
+indent-after-paren=4
+
+# String used as indentation unit. This is usually "    " (4 spaces) or "\t" (1
+# tab).
+indent-string='    '
+
+# Maximum number of characters on a single line.
+max-line-length=100
+
+# Maximum number of lines in a module.
+max-module-lines=1000
+
+# List of optional constructs for which whitespace checking is disabled. `dict-
+# separator` is used to allow tabulation in dicts, etc.: {1  : 1,\n222: 2}.
+# `trailing-comma` allows a space between comma and closing bracket: (a, ).
+# `empty-line` allows space-only lines.
+no-space-check=trailing-comma,
+               dict-separator
+
+# Allow the body of a class to be on the same line as the declaration if body
+# contains single statement.
+single-line-class-stmt=no
+
+# Allow the body of an if to be on the same line as the test if there is no
+# else.
+single-line-if-stmt=no
+
+
+[BASIC]
+
+# Naming style matching correct argument names.
+argument-naming-style=snake_case
+
+# Regular expression matching correct argument names. Overrides argument-
+# naming-style.
+#argument-rgx=
+
+# Naming style matching correct attribute names.
+attr-naming-style=snake_case
+
+# Regular expression matching correct attribute names. Overrides attr-naming-
+# style.
+#attr-rgx=
+
+# Bad variable names which should always be refused, separated by a comma.
+bad-names=foo,
+          bar,
+          baz,
+          toto,
+          tutu,
+          tata
+
+# Naming style matching correct class attribute names.
+class-attribute-naming-style=any
+
+# Regular expression matching correct class attribute names. Overrides class-
+# attribute-naming-style.
+#class-attribute-rgx=
+
+# Naming style matching correct class names.
+class-naming-style=PascalCase
+
+# Regular expression matching correct class names. Overrides class-naming-
+# style.
+#class-rgx=
+
+# Naming style matching correct constant names.
+const-naming-style=UPPER_CASE
+
+# Regular expression matching correct constant names. Overrides const-naming-
+# style.
+#const-rgx=
+
+# Minimum line length for functions/classes that require docstrings, shorter
+# ones are exempt.
+docstring-min-length=-1
+
+# Naming style matching correct function names.
+function-naming-style=snake_case
+
+# Regular expression matching correct function names. Overrides function-
+# naming-style.
+#function-rgx=
+
+# Good variable names which should always be accepted, separated by a comma.
+good-names=i,
+           j,
+           k,
+           ex,
+           Run,
+           _
+
+# Include a hint for the correct naming format with invalid-name.
+include-naming-hint=no
+
+# Naming style matching correct inline iteration names.
+inlinevar-naming-style=any
+
+# Regular expression matching correct inline iteration names. Overrides
+# inlinevar-naming-style.
+#inlinevar-rgx=
+
+# Naming style matching correct method names.
+method-naming-style=snake_case
+
+# Regular expression matching correct method names. Overrides method-naming-
+# style.
+#method-rgx=
+
+# Naming style matching correct module names.
+module-naming-style=snake_case
+
+# Regular expression matching correct module names. Overrides module-naming-
+# style.
+#module-rgx=
+
+# Colon-delimited sets of names that determine each other's naming style when
+# the name regexes allow several styles.
+name-group=
+
+# Regular expression which should only match function or class names that do
+# not require a docstring.
+no-docstring-rgx=^_
+
+# List of decorators that produce properties, such as abc.abstractproperty. Add
+# to this list to register other decorators that produce valid properties.
+# These decorators are taken in consideration only for invalid-name.
+property-classes=abc.abstractproperty
+
+# Naming style matching correct variable names.
+variable-naming-style=snake_case
+
+# Regular expression matching correct variable names. Overrides variable-
+# naming-style.
+#variable-rgx=
+
+
+[IMPORTS]
+
+# Allow wildcard imports from modules that define __all__.
+allow-wildcard-with-all=no
+
+# Analyse import fallback blocks. This can be used to support both Python 2 and
+# 3 compatible code, which means that the block might have code that exists
+# only in one or another interpreter, leading to false positives when analysed.
+analyse-fallback-blocks=no
+
+# Deprecated modules which should not be used, separated by a comma.
+deprecated-modules=optparse,tkinter.tix
+
+# Create a graph of external dependencies in the given file (report RP0402 must
+# not be disabled).
+ext-import-graph=
+
+# Create a graph of every (i.e. internal and external) dependencies in the
+# given file (report RP0402 must not be disabled).
+import-graph=
+
+# Create a graph of internal dependencies in the given file (report RP0402 must
+# not be disabled).
+int-import-graph=
+
+# Force import order to recognize a module as part of the standard
+# compatibility libraries.
+known-standard-library=
+
+# Force import order to recognize a module as part of a third party library.
+known-third-party=enchant
+
+
+[DESIGN]
+
+# Maximum number of arguments for function / method.
+max-args=5
+
+# Maximum number of attributes for a class (see R0902).
+max-attributes=7
+
+# Maximum number of boolean expressions in an if statement.
+max-bool-expr=5
+
+# Maximum number of branch for function / method body.
+max-branches=12
+
+# Maximum number of locals for function / method body.
+max-locals=15
+
+# Maximum number of parents for a class (see R0901).
+max-parents=7
+
+# Maximum number of public methods for a class (see R0904).
+max-public-methods=20
+
+# Maximum number of return / yield for function / method body.
+max-returns=6
+
+# Maximum number of statements in function / method body.
+max-statements=50
+
+# Minimum number of public methods for a class (see R0903).
+min-public-methods=2
+
+
+[CLASSES]
+
+# List of method names used to declare (i.e. assign) instance attributes.
+defining-attr-methods=__init__,
+                      __new__,
+                      setUp
+
+# List of member names, which should be excluded from the protected access
+# warning.
+exclude-protected=_asdict,
+                  _fields,
+                  _replace,
+                  _source,
+                  _make
+
+# List of valid names for the first argument in a class method.
+valid-classmethod-first-arg=cls
+
+# List of valid names for the first argument in a metaclass class method.
+valid-metaclass-classmethod-first-arg=cls
+
+
+[EXCEPTIONS]
+
+# Exceptions that will emit a warning when being caught. Defaults to
+# "BaseException, Exception".
+overgeneral-exceptions=BaseException,
+                       Exception

.travis.yml

@@ -1,15 +1,18 @@
+dist: xenial
 language: python
 python:
  - "3.5"
- - "3.5-dev"
  - "3.6"
- - "3.6-dev"
+ - "3.7"
 install:
- - pip install pylint requests
+ - pip install pylint~=2.3.1 requests mypy
  - pip install -r docs/requirements.txt
 script:
- - python3 -m pylint -r n -d bad-whitespace,missing-docstring,too-many-arguments,locally-disabled,line-too-long,too-many-public-methods,too-many-lines,too-many-instance-attributes,too-many-locals,too-many-branches,too-many-statements,inconsistent-return-statements,invalid-name,wildcard-import,unused-wildcard-import instaloader
+ - python3 -m pylint instaloader
+ - python3 -m mypy -m instaloader
  - make -C docs html SPHINXOPTS="-W -n"
+addons:
+  ssh_known_hosts: aur.archlinux.org
 deploy:
   - provider: pypi
     user: aandergr
@@ -27,3 +30,8 @@ deploy:
     on:
       branch: master
       python: 3.6
+  - provider: script
+    script: deploy/arch/deploy.sh $TRAVIS_TAG
+    on:
+      tags: true
+      python: 3.6

AUTHORS.md

@@ -0,0 +1,8 @@
+Authors
+=======
+
+Instaloader is written by
+
+- Alexander Graf (@aandergr)
+- André Koch-Kramer (@Thammus)
+- Lars Lindqvist (@e5150)

LICENSE

@@ -1,6 +1,6 @@
 The MIT License (MIT)
 
-Copyright (c) 2016-2018 Alexander Graf and André Koch-Kramer.
+Copyright (c) 2016-2019 Alexander Graf and André Koch-Kramer.
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

README.rst

@@ -1,5 +1,41 @@
 .. image:: https://raw.githubusercontent.com/instaloader/instaloader/master/docs/logo_heading.png
 
+.. badges-start
+
+|travis| |pypi| |pyversion| |license| |aur| |contributors| |downloads| |saythanks|
+
+.. |travis| image:: https://img.shields.io/travis/instaloader/instaloader/master.svg
+   :alt: Travis-CI Build Status
+   :target: https://travis-ci.org/instaloader/instaloader
+
+.. |pypi| image:: https://img.shields.io/pypi/v/instaloader.svg
+   :alt: Instaloader PyPI Project Page
+   :target: https://pypi.org/project/instaloader/
+
+.. |license| image:: https://img.shields.io/github/license/instaloader/instaloader.svg
+   :alt: MIT License
+   :target: https://github.com/instaloader/instaloader/blob/master/LICENSE
+
+.. |pyversion| image:: https://img.shields.io/pypi/pyversions/instaloader.svg
+   :alt: Supported Python Versions
+
+.. |contributors| image:: https://img.shields.io/github/contributors/instaloader/instaloader.svg
+   :alt: Contributor Count
+   :target: https://github.com/instaloader/instaloader/graphs/contributors
+
+.. |saythanks| image:: https://img.shields.io/badge/Say%20Thanks-!-ff7efd.svg
+   :alt: Say Thanks!
+   :target: https://saythanks.io/to/aandergr
+
+.. |aur| image:: https://img.shields.io/aur/version/instaloader.svg
+   :alt: Arch User Repository Package
+   :target: https://aur.archlinux.org/packages/instaloader/
+
+.. |downloads| image:: https://img.shields.io/pypi/dm/instaloader.svg
+   :alt: PyPI Download Count
+
+.. badges-end
+
 ::
 
     $ pip3 install instaloader
@@ -21,7 +57,7 @@
 
 ::
 
-    instaloader [--comments] [--geotags] [--stories]
+    instaloader [--comments] [--geotags] [--stories] [--highlights] [--tagged]
                 [--login YOUR-USERNAME] [--fast-update]
                 profile | "#hashtag" | :stories | :feed | :saved
 
@@ -75,6 +111,24 @@ Disclaimer
 Instaloader is in no way affiliated with, authorized, maintained or endorsed by Instagram or any of its affiliates or
 subsidiaries. This is an independent and unofficial project. Use at your own risk.
 
+Instaloader is licensed under an MIT license. Refer to ``LICENSE`` file for more information.
+
 .. disclaimer-end
 
-Instaloader is licensed under an MIT license. Refer to ``LICENSE`` file for more information.
+Contributing
+------------
+
+As an open source project, Instaloader heavily depends on the contributions from
+its community. See
+`contributing <https://instaloader.github.io/contributing.html>`__
+for how you may help Instaloader to become an even greater tool.
+
+It is a pleasure for us to share our Instaloader to the world, and we are proud
+to have attracted such an active and motivating community, with so many users
+who share their suggestions and ideas with us. Buying a community-sponsored beer
+or coffee from time to time is very likely to further raise our passion for the
+development of Instaloader.
+
+| For Donations, we provide a PayPal.Me link and a Bitcoin address.
+|  PayPal: `PayPal.me/aandergr <https://www.paypal.me/aandergr>`__
+|  BTC: 1Nst4LoadeYzrKjJ1DX9CpbLXBYE9RKLwY

deploy/arch/.SRCINFO.template

@@ -0,0 +1,15 @@
+pkgbase = instaloader
+	pkgdesc = Command line tool to download pictures, videos and metadata from Instagram
+	pkgver = {{version}}
+	pkgrel = 1
+	url = https://instaloader.github.io/
+	arch = any
+	license = MIT
+	makedepends = python-setuptools
+	depends = python>=3.5
+	depends = python-requests>=2.4
+	options = !emptydirs
+	source = instaloader-{{version}}.tar.gz::https://codeload.github.com/instaloader/instaloader/tar.gz/v{{version}}
+	sha512sums = {{hash}}
+
+pkgname = instaloader

deploy/arch/PKGBUILD.template

@@ -0,0 +1,27 @@
+# Maintainer: André Koch-Kramer <koch-kramer@web.de>
+
+pkgname=instaloader
+pkgver={{version}}
+pkgrel=1
+pkgdesc="Command line tool to download pictures, videos and metadata from Instagram"
+arch=('any')
+url="https://instaloader.github.io/"
+license=('MIT')
+groups=()
+depends=('python>=3.5' 'python-requests>=2.4')
+makedepends=('python-setuptools')
+options=('!emptydirs')
+source=($pkgname-$pkgver.tar.gz::https://codeload.github.com/instaloader/instaloader/tar.gz/v$pkgver)
+sha512sums=('{{hash}}')
+
+build() {
+	cd "$srcdir/$pkgname-$pkgver"
+	python setup.py clean --all
+	python setup.py build
+}
+
+package() {
+	cd "$srcdir/$pkgname-$pkgver"
+	install -Dm644 LICENSE "$pkgdir"/usr/share/licenses/$pkgname/LICENSE
+	python setup.py install --root "$pkgdir/" --skip-build
+}

deploy/arch/deploy.sh

@@ -0,0 +1,26 @@
+#!/usr/bin/env bash
+
+cd $(dirname $0)
+
+# decrypt and add ssh key
+openssl aes-256-cbc -K $encrypted_354637631c28_key -iv $encrypted_354637631c28_iv -in id_rsa_AUR.enc -out /tmp/AUR_openssh -d
+eval "$(ssh-agent -s)"
+chmod 600 /tmp/AUR_openssh
+ssh-add /tmp/AUR_openssh
+
+# clone and modify AUR repo
+git clone --depth 1 ssh://aur@aur.archlinux.org/instaloader.git
+
+curl -sSfOJ https://codeload.github.com/instaloader/instaloader/tar.gz/$1
+VERSION=${1:1}
+HASH=$(sha512sum instaloader-$VERSION.tar.gz | cut -f1 -d " ")
+sed -e "s/{{version}}/$VERSION/g" -e "s/{{hash}}/$HASH/g" PKGBUILD.template > instaloader/PKGBUILD
+sed -e "s/{{version}}/$VERSION/g" -e "s/{{hash}}/$HASH/g" .SRCINFO.template > instaloader/.SRCINFO
+
+# commit and push changes
+cd instaloader
+git config user.email "koch-kramer@web.de"
+git config user.name "André Koch-Kramer"
+git add .
+git commit -m "Release of version $VERSION"
+git push

docs/_static/instaloader.css

@@ -0,0 +1,75 @@
+@import url(bootstrap-4.1.3.min.css);
+a {
+  color: #008d06; }
+  a:hover, a.current, a.active {
+    color: #f48400; }
+  a.current, a.active {
+    font-weight: 500; }
+
+.doc-sidebar a:hover, .modal-body a:hover {
+  text-decoration: none; }
+
+code {
+  color: #212529; }
+
+.highlight {
+  border-radius: 0.3em; }
+  .highlight pre {
+    padding: 0.7em;
+    color: #fff; }
+  .highlight .c1 {
+    color: #008d06; }
+  .highlight .k, .highlight .kn, .highlight .ow {
+    color: #f48400;
+    font-weight: bold; }
+  .highlight .nb {
+    color: #f48400; }
+  .highlight .s1, .highlight .s2, .highlight .mi {
+    color: #ff7efd; }
+
+.doc-content {
+  padding-top: 0.5em;
+  top: 3.5625em;
+  position: fixed;
+  bottom: 0;
+  overflow-y: auto; }
+
+.doc-sidebar {
+  position: fixed;
+  padding-top: 0.5em;
+  top: 3.5625em;
+  bottom: 0;
+  right: 0;
+  overflow-y: auto; }
+  .doc-sidebar ul {
+    list-style-type: none;
+    padding-left: 1em; }
+    .doc-sidebar ul.current {
+      padding-left: 0; }
+  .doc-sidebar li {
+    padding-top: 0.2em;
+    padding-bottom: 0.2em; }
+
+.admonition {
+  border-radius: 0.3em;
+  border: 1px solid #ccc;
+  background-color: #f8f9fa; }
+  .admonition .admonition-title {
+    color: #ff7efd; }
+
+.topic {
+  border-radius: 0.3em;
+  background-color: #f8f9fa; }
+
+.nav-link {
+  padding-top: 0;
+  padding-bottom: 0; }
+
+.btn-success {
+  background-color: #008d06;
+  border-color: #008d06; }
+  .btn-success:hover {
+    background-color: #f48400;
+    border-color: #f48400; }
+
+/*# sourceMappingURL=instaloader.css.map */

docs/_static/instaloader.js

@@ -0,0 +1,22 @@
+$(function () {
+    $('[data-toggle="tooltip"]').tooltip();
+
+    $('.doc-sidebar > ul > li.current').attr("id", "localtoc");
+    $('#localtoc ul').addClass("nav flex-column");
+    $('#localtoc ul li').addClass("nav-item");
+    $('#localtoc ul li a').addClass("nav-link");
+    $('.doc-content').scrollspy({target: '#localtoc'});
+
+    const top_href = '#' + $('.section:first').attr("id");
+    $('#localtoc > a.current').attr("href", top_href);
+
+    $('#navbarToc a').on("click", function () {
+        const href = $(this).attr("href");
+        if (href === '#') {
+            window.location.href = top_href;
+        } else {
+            window.location.href = href;
+        }
+        $('#navbarToc').modal('hide');
+    });
+});

docs/_static/instaloader.scss

@@ -0,0 +1,117 @@
+@import 'bootstrap-4.1.3.min.css';
+
+$color_instaloader_main: #008d06;
+$color_instaloader_accent: #f48400;
+$color_instaloader_hyper: #ff7efd;
+$color_normal_text: #212529;
+$color_light_bg: #f8f9fa;
+
+a {
+  color: $color_instaloader_main;
+
+  &:hover, &.current, &.active {
+    color: $color_instaloader_accent;
+  }
+
+  &.current, &.active {
+    font-weight: 500;
+  }
+}
+
+.doc-sidebar, .modal-body {
+  a:hover {
+    text-decoration: none
+  }
+}
+
+code {
+  color: $color_normal_text;
+}
+
+.highlight {
+  border-radius: 0.3em;
+
+  pre {
+    padding: 0.7em;
+    color: #fff;
+  }
+
+  .c1 {
+    color: $color_instaloader_main
+  }
+
+  .k, .kn, .ow {
+    color: $color_instaloader_accent;
+    font-weight: bold
+  }
+
+  .nb {
+    color: $color_instaloader_accent
+  }
+
+  .s1, .s2, .mi {
+    color: $color_instaloader_hyper
+  }
+}
+
+.doc-content {
+  padding-top: 0.5em;
+  top: 3.5625em;
+  position: fixed;
+  bottom: 0;
+  overflow-y: auto;
+}
+
+.doc-sidebar {
+  position: fixed;
+  padding-top: 0.5em;
+  top: 3.5625em;
+  bottom: 0;
+  right: 0;
+  overflow-y: auto;
+
+  ul {
+    list-style-type: none;
+    padding-left: 1em;
+
+    &.current {
+      padding-left: 0;
+    }
+  }
+
+  li {
+    padding-top: 0.2em;
+    padding-bottom: 0.2em;
+
+  }
+}
+
+.admonition {
+  border-radius: 0.3em;
+  border: 1px solid #ccc;
+  background-color: $color_light_bg;
+
+  .admonition-title {
+    color: $color_instaloader_hyper;
+  }
+}
+
+.topic {
+  border-radius: 0.3em;
+  background-color: $color_light_bg;
+}
+
+.nav-link {
+  padding-top: 0;
+  padding-bottom: 0;
+}
+
+.btn-success {
+  background-color: $color_instaloader_main;
+  border-color: $color_instaloader_main;
+
+  &:hover {
+    background-color: $color_instaloader_accent;
+    border-color: $color_instaloader_accent;
+  }
+}

docs/_static/style.css

@@ -1,12 +0,0 @@
-code {
-    color: #222;
-    background-color: #f5f5f5;
-}
-
-dd {
-    padding-left: 8px;
-}
-
-dl.option dd {
-    padding-left: 12px;
-}

docs/_templates/globaltoc.html

@@ -1,14 +0,0 @@
-<li class="dropdown globaltoc-container">
-  <a role="button"
-     id="dLabelGlobalToc"
-     data-toggle="dropdown"
-     data-target="#"
-     href="{{ pathto(master_doc) }}">{{ theme_navbar_site_name }} <b class="caret"></b></a>
-  <ul class="dropdown-menu globaltoc"
-      role="menu"
-      aria-labelledby="dLabelGlobalToc">
-    {{ toctree(maxdepth=theme_globaltoc_depth|toint, collapse=False, includehidden=theme_globaltoc_includehidden|tobool) }}
-    <li role="separator" class="divider"></li>
-    <li class="toctree-l1"><a class="reference internal" href="genindex.html">Index</a></li>
-  </ul>
-</li>

docs/_templates/layout.html

@@ -1,18 +1,118 @@
 {% extends "!layout.html" %}
+
+{% set html5_doctype = True %}
+
+{% set script_files = script_files + ['_static/bootstrap-4.1.3.bundle.min.js', '_static/instaloader.js'] %}
+
 {% block htmltitle %}
   {% if pagename == "index" %}
-  <title>{{ title|striptags|e }}</title>
+    <title>{{ title|striptags|e }}</title>
   {% else %}
-  <title>{{ title|striptags|e }}{{ titlesuffix }}</title>
+    <title>{{ title|striptags|e }}{{ titlesuffix }}</title>
   {% endif %}
 {% endblock %}
+
 {% block extrahead %}
-{{ super() }}
-  <meta name="google-site-verification" content="FNUsDdkOJM9obm9QEdrNBVd9AtAzHzA-aaSMCUrS6C8" />
+  {{ super() }}
+  <meta name="viewport" content="width=device-width, initial-scale=1, shrink-to-fit=no">
+  <meta name="google-site-verification" content="FNUsDdkOJM9obm9QEdrNBVd9AtAzHzA-aaSMCUrS6C8"/>
   {% if pagename == "index" %}
-  <link rel="canonical" href="https://instaloader.github.io/" />
+    <link rel="canonical" href="https://instaloader.github.io/"/>
   {% else %}
-  <link rel="canonical" href="https://instaloader.github.io/{{ pagename }}.html" />
+    <link rel="canonical" href="https://instaloader.github.io/{{ pagename }}.html"/>
   {% endif %}
-  <script async defer src="https://buttons.github.io/buttons.js"></script>
 {% endblock %}
+
+{% block header %}
+  <nav class="navbar navbar-expand-sm navbar-light bg-light fixed-top border-bottom">
+    <a class="navbar-brand" href="{{ pathto(master_doc) }}">
+      <img src="{{ pathto('_static/' + logo, 1) }}" width="30" height="30" class="d-inline-block align-top" alt="logo">
+      Instaloader
+    </a>
+    <ul class="navbar-nav mr-auto">
+      <li class="nav-item ml-0 ml-sm-2 ml-md-0">
+        <a target="_blank" class="nav-link"
+           href="https://github.com/instaloader/instaloader/releases/tag/v{{ current_release }}"
+           title="Released on {{ current_release_date }}" data-toggle="tooltip">v<b>{{ current_release }}</b></a>
+      </li>
+    </ul>
+    <button class="navbar-toggler" type="button" data-toggle="collapse" data-target="#navbarLinks"
+            aria-controls="navbarLinks" aria-expanded="false" aria-label="Toggle navigation">
+      <span class="navbar-toggler-icon"></span>
+    </button>
+    <div class="collapse navbar-collapse" id="navbarLinks">
+      <ul class="navbar-nav ml-auto">
+        <li class="nav-item">
+          <a target="_blank" class="nav-link" href="https://github.com/instaloader/instaloader/issues">Issues</a>
+        </li>
+        <li class="nav-item">
+          <a target="_blank" class="nav-link" href="https://github.com/instaloader/instaloader/releases">Releases</a>
+        </li>
+        <li class="nav-item">
+          <a target="_blank" class="nav-link" href="https://github.com/instaloader/instaloader">GitHub</a>
+        </li>
+        <li class="nav-item d-sm-none d-lg-block">
+          <a target="_blank" class="nav-link" href="https://saythanks.io/to/aandergr">Say Thanks!</a>
+        </li>
+      </ul>
+      <form class="form-inline" action="search.html" method="get">
+        <div class="input-group my-2 my-sm-0 ml-2">
+          <input class="form-control" type="search" placeholder="Search" aria-label="Search" name="q">
+          <div class="input-group-append">
+            <button class="btn btn-success btn-small" type="submit">Go</button>
+          </div>
+        </div>
+      </form>
+    </div>
+    {% if pagename != "index" %}
+      <button type="button" class="btn btn-success d-md-none btn-small ml-2" data-toggle="modal"
+              data-target="#navbarToc">
+        TOC
+      </button>
+    {% endif %}
+  </nav>
+  {% if pagename != "index" %}
+    <div class="modal fade" id="navbarToc" tabindex="-1" role="dialog"
+         aria-hidden="true" aria-labelledby="navbarTocTitle">
+      <div class="modal-dialog modal-dialog-centered" role="document">
+        <div class="modal-content">
+          <div class="modal-header">
+            <h5 class="modal-title" id="navbarTocTitle">Contents</h5>
+            <button type="button" class="close" data-dismiss="modal" aria-label="Close">
+              <span aria-hidden="true">&times;</span>
+            </button>
+          </div>
+          <div class="modal-body">
+            {{ toctree(maxdepth=-1) }}
+          </div>
+        </div>
+      </div>
+    </div>
+  {% endif %}
+{% endblock %}
+
+{% block relbar1 %}{% endblock %}
+{% block relbar2 %}{% endblock %}
+{% block footer %}{% endblock %}
+{% block content %}
+  <div class="container-fluid row">
+    <div class="col-12 {% if pagename != "index" %}col-md-9 col-xl-10{% endif %} doc-content">
+      {% block body %}{% endblock %}
+    </div>
+    {% if pagename != "index" %}
+      <div class="d-none col-md-3 col-xl-2 bg-light doc-sidebar d-md-flex flex-column border-left">
+
+        <h3>{{ _('Table of Contents') }}</h3>
+        {{ toctree(maxdepth=-1) }}
+
+        <div class="d-flex justify-content-between small mt-auto mb-2">
+          <a class="reference internal{% if pagename == "genindex" %} current{% endif %}" href="genindex.html">Index</a>
+          {% if pagename != "genindex" and pagename != "search" %}
+            <a class="reference external"
+               href="https://github.com/instaloader/instaloader/edit/master/docs/{{ pagename }}.rst">Edit this page</a>
+          {% endif %}
+        </div>
+      </div>
+    {% endif %}
+  </div>
+{% endblock %}

docs/_templates/navbar.html

@@ -1,58 +0,0 @@
-  <div id="navbar" class="{{ theme_navbar_class }} navbar-default {% if theme_navbar_fixed_top|tobool -%} navbar-fixed-top{%- endif -%}">
-    <div class="container">
-      <div class="navbar-header">
-        <!-- .btn-navbar is used as the toggle for collapsed navbar content -->
-        <button type="button" class="navbar-toggle" data-toggle="collapse" data-target=".nav-collapse">
-          <span class="icon-bar"></span>
-          <span class="icon-bar"></span>
-          <span class="icon-bar"></span>
-        </button>
-        <a class="navbar-brand" href="{{ pathto(master_doc) }}">
-          {%- block sidebarlogo %}
-            {%- if logo %}<span><img src="{{ pathto('_static/' + logo, 1) }}"></span>{%- endif %}
-          {%- endblock %}
-          {% if theme_navbar_title -%}{{ theme_navbar_title|e }}{%- else -%}{{ project|e }}{%- endif -%}
-        </a>
-        <span class="navbar-text navbar-version pull-left">
-          <a href="https://github.com/instaloader/instaloader/releases/tag/v{{current_release}}" title="Released on {{current_release_date}}">v<b>{{current_release}}</b></a>
-        </span>
-      </div>
-
-        <div class="collapse navbar-collapse nav-collapse">
-          <ul class="nav navbar-nav">
-            {% if theme_navbar_links %}
-              {%- for link in theme_navbar_links %}
-                <li><a href="{{ pathto(*link[1:]) }}">{{ link[0] }}</a></li>
-              {%- endfor %}
-            {% endif %}
-            {% block navbartoc %}
-              {% include "globaltoc.html" %}
-              {% if theme_navbar_pagenav and pagename not in ("index", "genindex", "installation") %}
-                {% include "navbartoc.html" %}
-              {% endif %}
-            {% endblock %}
-            {% if theme_navbar_sidebarrel %}
-              {% block sidebarrel %}
-                {% include "relations.html" %}
-              {% endblock %}
-            {% endif %}
-            </ul>
-            <ul class="nav navbar-nav navbar-right">
-              <li class="dropdown">
-                <a role="button" id="dLabelUsefulLinks" data-toggle="dropdown" data-target="#" href="#">
-                  Useful Links
-                  <b class="caret"></b>
-                </a>
-                <ul class="dropdown-menu" role="menu" aria-labelledby="dLabelUsefulLinks">
-                  <li><a href="https://github.com/instaloader/instaloader/issues">Issue Tracker</a></li>
-                  <li><a href="https://github.com/instaloader/instaloader/releases">Version History</a></li>
-                  <li><a href="https://pypi.org/project/instaloader/">Instaloader on PyPI</a></li>
-                  <li><a href="https://github.com/instaloader/instaloader">Instaloader on GitHub</a></li>
-                  <li role="separator" class="divider"></li>
-                  <li style="padding-left: 20px;"><a class="github-button" href="https://github.com/instaloader/instaloader" data-icon="octicon-star" data-size="large" data-show-count="true" aria-label="Star Instaloader on GitHub">Star</a></li>
-                </ul>
-              </li>
-            </ul>
-        </div>
-    </div>
-  </div>

docs/_templates/search.html

@@ -0,0 +1,28 @@
+{% extends "!search.html" %}
+
+{% block body %}
+  <div id="fallback" class="admonition warning">
+    <script type="text/javascript">$('#fallback').hide();</script>
+    <p>
+      {% trans %}Please activate JavaScript to enable the search
+        functionality.{% endtrans %}
+    </p>
+  </div>
+  {% if search_performed %}
+    <h1>{{ _('Search Results') }}</h1>
+    {% if not search_results %}
+      <p>{{ _('Your search did not match any documents. Please make sure that all words are spelled correctly and that you\'ve selected enough categories.') }}</p>
+    {% endif %}
+  {% endif %}
+  <div id="search-results">
+    {% if search_results %}
+      <ul>
+        {% for href, caption, context in search_results %}
+          <li><a href="{{ pathto(item.href) }}">{{ caption }}</a>
+            <div class="context">{{ context|e }}</div>
+          </li>
+        {% endfor %}
+      </ul>
+    {% endif %}
+  </div>
+{% endblock %}

docs/as-module.rst

@@ -123,6 +123,9 @@ Additionally, the following trivial structures are defined:
 .. autoclass:: PostComment
    :no-show-inheritance:
 
+.. autoclass:: PostCommentAnswer
+   :no-show-inheritance:
+
 .. autoclass:: PostLocation
    :no-show-inheritance:
 
@@ -135,6 +138,17 @@ User Stories
 .. autoclass:: StoryItem
    :no-show-inheritance:
 
+Highlights
+""""""""""
+
+.. autoclass:: Highlight
+   :no-show-inheritance:
+   :inherited-members:
+
+   Bases: :class:`Story`
+
+   .. versionadded:: 4.1
+
 Profiles
 """"""""
 
@@ -177,12 +191,18 @@ Exceptions
 
 .. autoexception:: LoginRequiredException
 
+.. autoexception:: TwoFactorAuthRequiredException
+
+   .. versionadded:: 4.2
+
 .. autoexception:: InvalidArgumentException
 
 .. autoexception:: BadResponseException
 
 .. autoexception:: BadCredentialsException
 
+.. autoexception:: PostChangedException
+
 .. autoexception:: TooManyRequestsException
 
 ``InstaloaderContext`` (Low-level functions)

docs/basic-usage.rst

@@ -60,15 +60,37 @@ already have a valid session cookie file.
 What to Download
 ^^^^^^^^^^^^^^^^
 
+.. targets-start
+
 Instaloader supports the following targets:
 
 - ``profile``
-   Public profile, or private profile with :option:`--login`. For each profile
-   you download, :option:`--stories` instructs Instaloader to also
-   **download the user's stories**.
+   Public profile, or private profile with :option:`--login`.
+
+   If an already-downloaded profile has been renamed, Instaloader automatically
+   finds it by its unique ID and renames the folder accordingly.
+
+   Besides the profile's posts, its current profile picture is downloaded. For
+   each profile you download,
+
+   - :option:`--stories`
+      instructs Instaloader to also **download the user's stories**,
+
+   - :option:`--highlights`
+      to **download highlights of each profile that is downloaded**, and
+
+   - :option:`--tagged`
+      to **download posts where the user is tagged**.
 
 - ``"#hashtag"``
-   Posts with a certain **hashtag** (the quotes are usually neccessary),
+   Posts with a certain **hashtag** (the quotes are usually necessary),
+
+- ``%location id``
+   Posts tagged with a given location; the location ID is the numerical ID
+   Instagram labels a location with (e.g.
+   \https://www.instagram.com/explore/locations/**362629379**/plymouth-naval-memorial/).
+
+   .. versionadded:: 4.2
 
 - ``:stories``
    The currently-visible **stories** of your followees (requires
@@ -84,6 +106,14 @@ Instaloader supports the following targets:
    All profiles that are followed by ``profile``, i.e. the *followees* of
    ``profile`` (requires :option:`--login`).
 
+- ``-post``
+   The single **post** with the given shortcode. Must be preceeded by ``--`` in
+   the argument list to not be mistaken as an option flag.
+
+   .. versionadded:: 4.1
+
+.. targets-end
+
 Instaloader goes through all media matching the specified targets and
 downloads the pictures and videos and their captions. You can specify
 
@@ -103,7 +133,7 @@ Filename Specification
 ^^^^^^^^^^^^^^^^^^^^^^
 
 For each target, Instaloader creates a directory named after the target,
-i.e. ``profile``, ``#hashtag``, ``:feed``, etc. and therein saves the
+i.e. ``profile``, ``#hashtag``, ``%location id``, ``:feed``, etc. and therein saves the
 posts in files named after the post's timestamp.
 
 :option:`--dirname-pattern` allows to configure the directory name of each
@@ -158,7 +188,7 @@ Filter Posts
 .. py:currentmodule:: instaloader
 
 The options :option:`--post-filter` and :option:`--storyitem-filter`
-allows to specify criterias that posts or story items have to
+allows to specify criteria that posts or story items have to
 meet to be downloaded. If not given, all posts are downloaded.
 
 The filter string must be a
@@ -169,11 +199,6 @@ where the attributes from :class:`Post` or
 Id est, the following attributes can be used with both
 :option:`--post-filter` and :option:`--storyitem-filter`:
 
-- :attr:`~Post.is_video` (bool)
-   Post/StoryItem is a video. For example, you may skip videos::
-
-      instaloader --post-filter="not is_video" target
-
 - :attr:`~Post.owner_username` (str), :attr:`~Post.owner_id` (int)
    Owner profile username / userid.
 
@@ -183,6 +208,15 @@ Id est, the following attributes can be used with both
 
       instaloader --post-filter="date_utc <= datetime(2018, 5, 31)" target
 
+- :attr:`~Post.is_video` (bool)
+   Post/StoryItem is a video. For example, you may skip videos::
+
+      instaloader --post-filter="not is_video" target
+
+   This is not the same as :option:`--no-videos` and
+   :option:`--no-video-thumbnails`, since sidecar posts (posts that contain
+   multiple pictures/videos in one post) have this attribute set to False.
+
 As :option:`--post-filter`, the following attributes can be used additionally:
 
 - :attr:`~Post.viewer_has_liked` (bool)

docs/cli-options.rst

@@ -8,32 +8,46 @@ Instaloader is invoked with::
    $ instaloader [options] target [target ...]
 
 where ``target`` is a ``profile``, a ``"#hashtag"``, ``@profile`` (all profiles
-that *profile* is following), or if logged in ``:feed`` (pictures from your
+that *profile* is following), ``%location ID``, or if logged in ``:feed`` (pictures from your
 feed), ``:stories`` (stories of your followees) or ``:saved`` (collection of
 posts marked as saved).
 
 Here we explain the additional options that can be given to Instaloader to
-customize its behavior.  To get a list of all flags, their abbreviations and
-their descriptions, you may also run ``instaloader --help``.  For an
+customize its behavior.  For an
 introduction on how to use Instaloader, see
 :ref:`download-pictures-from-instagram`.
 
-What to Download
-^^^^^^^^^^^^^^^^
+To get a list of all flags, their abbreviations and
+their descriptions, you may also run::
 
-Specify a list of targets (profiles, #hashtags, ``:feed``, ``:stories`` or
-``:saved``). For each of these, Instaloader creates a folder and stores all
-posts along with the pictures's captions and the current **profile picture**
-there. If an already-downloaded profile has been renamed, Instaloader
-automatically **finds it by its unique ID** and renames the folder likewise.
+   instaloader --help
 
-.. option:: --profile-pic-only, -P
+Targets
+^^^^^^^
 
-   Only download profile picture.
+Specify a list of targets. For each of these, Instaloader creates a folder and
+stores all posts along with the pictures's captions there.
 
-.. option:: --no-profile-pic
+.. include:: basic-usage.rst
+   :start-after: targets-start
+   :end-before: targets-end
 
-   Do not download profile picture.
+- ``filename.json[.xz]``
+   Re-Download the given object
+
+- ``+args.txt``
+   Read targets (and options) from given textfile. See :option:`+args.txt`.
+
+What to Download of each Post
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+.. option:: --no-pictures
+
+   Do not download post pictures. Cannot be used together with
+   :option:`--fast-update`. Implies :option:`--no-video-thumbnails`, does not
+   imply :option:`--no-videos`.
+
+   .. versionadded:: 4.1
 
 .. option:: --no-videos, -V
 
@@ -69,11 +83,6 @@ automatically **finds it by its unique ID** and renames the folder likewise.
    Template to write in txt file for each StoryItem. See
    :ref:`metadata-text-files`.
 
-.. option:: --stories, -s
-
-   Also **download stories** of each profile that is downloaded. Requires
-   :option:`--login`.
-
 .. option:: --no-metadata-json
 
    Do not create a JSON file containing the metadata of each post.
@@ -82,8 +91,50 @@ automatically **finds it by its unique ID** and renames the folder likewise.
 
    Do not xz compress JSON files, rather create pretty formatted JSONs.
 
+
+What to Download of each Profile
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+.. option:: --profile-pic-only, -P
+
+   .. deprecated:: 4.1
+      Use :option:`--no-posts`.
+
+   Only download profile picture.
+
+.. option:: --no-posts
+
+   Do not download regular posts.
+
+   .. versionadded:: 4.1
+
+.. option:: --no-profile-pic
+
+   Do not download profile picture.
+
+.. option:: --stories, -s
+
+   Also **download stories** of each profile that is downloaded. Requires
+   :option:`--login`.
+
+.. option:: --highlights
+
+   Also **download highlights** of each profile that is downloaded. Requires
+   :option:`--login`.
+
+   .. versionadded:: 4.1
+
+.. option:: --tagged
+
+   Also download posts where each profile is tagged.
+
+   .. versionadded:: 4.1
+
 .. option:: --stories-only
 
+   .. deprecated:: 4.1
+      Use :option:`--stories` :option:`--no-posts`.
+
    Rather than downloading regular posts of each specified profile, only
    download stories.  Requires :option:`--login`. Does not imply
    :option:`--no-profile-pic`.
@@ -93,6 +144,15 @@ automatically **finds it by its unique ID** and renames the folder likewise.
       If possible, use ``:stories`` target rather than :option:`--stories-only`
       with all your followees. ``:stories`` uses fewer API requests.
 
+Which Posts to Download
+^^^^^^^^^^^^^^^^^^^^^^^
+
+.. option:: --fast-update, -F
+
+   For each target, stop when encountering the first already-downloaded picture.
+   This flag is recommended when you use Instaloader to update your personal
+   Instagram archive.
+
 .. option:: --post-filter filter, --only-if filter
 
    Expression that, if given, must evaluate to True for each post to be
@@ -108,24 +168,10 @@ automatically **finds it by its unique ID** and renames the folder likewise.
    evaluated to :class:`instaloader.StoryItem` attributes.
    See :ref:`filter-posts` for more examples.
 
-
-
-When to Stop Downloading
-^^^^^^^^^^^^^^^^^^^^^^^^
-
-If none of these options are given, Instaloader goes through all pictures
-matching the specified targets.
-
-.. option:: --fast-update, -F
-
-   For each target, stop when encountering the first already-downloaded picture.
-   This flag is recommended when you use Instaloader to update your personal
-   Instagram archive.
-
 .. option:: --count COUNT, -c
 
    Do not attempt to download more than COUNT posts.  Applies only to
-   ``#hashtag`` and ``:feed``.
+   ``#hashtag``, ``%location id``, and ``:feed``.
 
 
 Login (Download Private Profiles)
@@ -184,6 +230,14 @@ How to Download
    to ``3``. If a connection fails, it can be manually skipped by hitting
    :kbd:`Control-c`. Set this to ``0`` to retry infinitely.
 
+.. option:: --commit-mode
+
+   Tries to ensure downloaded images avoid corruption in case of unexpected
+   interruption. If the last picture is corrupted, Instaloader will fix the
+   picture the next time it is run.
+
+   .. versionadded:: 4.2
+
 Miscellaneous Options
 ^^^^^^^^^^^^^^^^^^^^^
 
@@ -192,3 +246,23 @@ Miscellaneous Options
    Disable user interaction, i.e. do not print messages (except errors) and fail
    if login credentials are needed but not given.
    This is handy for running :ref:`instaloader-as-cronjob`.
+
+.. option:: +args.txt
+
+   Read arguments from file `args.txt`, a shortcut to provide arguments from
+   file rather than command-line. This provides a convenient way to hide login
+   info from CLI, and can also be used to simplify managment of long arguments.
+
+   .. note::
+
+      Text file should separate arguments with line breaks.
+
+      args.txt example::
+
+         --login=MYUSERNAME
+         --password=MYPASSWORD
+         --fast-update
+         profile1
+         profile2
+
+   .. versionadded:: 4.1

docs/codesnippets.rst

@@ -11,8 +11,11 @@ Advanced Instaloader Examples
    :backlinks: none
 
 Here we present code examples that use the :ref:`python-module-instaloader` for
-more advanced tasks than what is possible with the Instaloader command line
-interface.
+more advanced Instagram downloading or metadata mining than what is possible
+with the Instaloader command line interface.
+
+The scripts presented here can be downloaded from our source tree:
+`instaloader/docs/codesnippets/ <https://github.com/instaloader/instaloader/tree/master/docs/codesnippets>`__
 
 .. For each code snippet:
    - title
@@ -21,12 +24,12 @@ interface.
    - link to discussion issue
    - link used methods
 
-Download Posts in a specific period
+Download Posts in a Specific Period
 -----------------------------------
 
-To collect pictures (and metadata) only from a specific period, you can play
-around with :func:`~itertools.dropwhile` and :func:`~itertools.takewhile` from
-:mod:`itertools` like in this snippet.
+To only download Instagram pictures (and metadata) that are within a specific
+period, you can play around with :func:`~itertools.dropwhile` and
+:func:`~itertools.takewhile` from :mod:`itertools` like in this snippet.
 
 .. literalinclude:: codesnippets/121_since_until.py
 
@@ -37,8 +40,8 @@ Discussed in :issue:`121`.
 Likes of a Profile / Ghost Followers
 ------------------------------------
 
-To store inactive followers, i.e. followers that did not like any of your
-pictures, into a file you can use this approach.
+To obtain a list of your inactive followers, i.e. followers that did not like
+any of your pictures, into a file you can use this approach.
 
 .. literalinclude:: codesnippets/120_ghost_followers.py
 
@@ -51,10 +54,10 @@ Discussed in :issue:`120`.
 Track Deleted Posts
 -------------------
 
-This script uses Instaloader to obtain a list of currently-online posts, and
-generates the matching filename of each post. It outputs a list of posts which
-are online but not offline (i.e. not yet downloaded) and a list of posts which
-are offline but not online (i.e. deleted in the profile).
+This script uses Instaloader to obtain a list of currently-online Instagram and
+compares it with the set of posts that you already have downloaded.  It outputs
+a list of posts which are online but not offline (i.e. not yet downloaded) and a
+list of posts which are offline but not online (i.e. deleted in the profile).
 
 .. literalinclude:: codesnippets/56_track_deleted.py
 
@@ -68,9 +71,9 @@ Only one Post per User
 
 To download only the one most recent post from each user, this snippet creates a
 :class:`set` that contains the users of which a post has already been
-downloaded. When iterating the posts, check whether the post's owner already is
-in the set. If so, skip the post. Otherwise, download it and add the user to
-that set.
+downloaded. While iterating the posts, it checks whether the post's owner
+already is in the set. If not, the post is downloaded from Instagram and the
+user is added to that set.
 
 .. literalinclude:: codesnippets/113_only_one_per_user.py
 
@@ -79,23 +82,32 @@ See also :class:`Post`, :meth:`Instaloader.download_post`,
 
 Discussed in :issue:`113`.
 
-Upgrade Images by local Copies
+Top X Posts of User
+-------------------
+
+With Instaloader, it is easy to download the few most-liked pictres of a user.
+
+.. literalinclude:: codesnippets/194_top_x_of_user.py
+
+Discussed in :issue:`194`.
+
+Upgrade Images by Local Copies
 ------------------------------
 
 The following script finds local versions of images fetched by Instaloader, in
-order to ugprade the downloaded images by locally-found versions with better
+order to upgrade the downloaded images by locally-found versions with better
 quality. It uses image hashing to identify similar images.
 
 `updgrade-instaloader-images.py <https://gist.github.com/pavelkryukov/15f93d19a99428a284a8bcec27e0187b>`__ (external link to GitHub Gist)
 
 Discussed in :issue:`46`.
 
-Render Captions to downloaded Images
-------------------------------------
+Add Captions to Images
+----------------------
 
 Instaloader does not modify the downloaded JPEG file. However, one could combine
-it with an imaging library such as Pillow or PIL to render the
-:attr:`Post.caption` on pictures.  The following shows an approach.
+it with an imaging library such as Pillow or PIL to render the caption on
+Instagram pictures.  The following shows an approach.
 
 .. literalinclude:: codesnippets/110_pil_captions.py
 
@@ -103,3 +115,26 @@ See also :attr:`Post.caption`, :attr:`Post.url`, :meth:`Post.from_shortcode`,
 :func:`load_structure_from_file`.
 
 Discussed in :issue:`110`.
+
+Metadata JSON Files
+-------------------
+
+The JSON files Instaloader saves along with each Post contain all the metadata
+that has been retrieved from Instagram while downloading the picture and
+associated required information.
+
+With `jq <https://stedolan.github.io/jq/>`__, a command-line JSON processor, the
+metadata can be easily post-processed. For example, Instaloader's JSON files can
+be pretty-formatted with:
+
+.. code-block:: none
+
+   xzcat 2018-05-13_11-18-45_UTC.json.xz | jq .node
+
+However, Instaloader tries to do as few metadata requests as possible, so,
+depending on how Instaloader has been invoked, it may occur that these files do
+not contain the complete available metadata structure. Nevertheless, the file
+can be loaded into Instaloader with :func:`load_structure_from_file` and the
+required metadata then be accessed via the :class:`Post` or :class:`Profile`
+attributes, which trigger an Instagram request if that particular information is
+not present in the JSON file.

docs/codesnippets/194_top_x_of_user.py

@@ -0,0 +1,15 @@
+from itertools import islice
+from math import ceil
+
+from instaloader import Instaloader, Profile
+
+PROFILE = ...        # profile to download from
+X_percentage = 10    # percentage of posts that should be downloaded
+
+L = Instaloader()
+
+profile = Profile.from_username(L.context, PROFILE)
+posts_sorted_by_likes = sorted(profile.get_posts(), key = lambda p: p.likes + p.comments)
+
+for post in islice(posts_sorted_by_likes, ceil(profile.mediacount * X_percentage / 100)):
+    L.download_post(post, PROFILE)

docs/codesnippets/92_import_firefox_session.py

@@ -0,0 +1,23 @@
+from glob import glob
+from os.path import expanduser
+from sqlite3 import connect
+
+from instaloader import ConnectionException, Instaloader
+
+# FIREFOXCOOKIEFILE = "/home/alex/.mozilla/firefox/l96w6b90.default/cookies.sqlite"
+FIREFOXCOOKIEFILE = glob(expanduser("~/.mozilla/firefox/*.default/cookies.sqlite"))[0]
+
+instaloader = Instaloader(max_connection_attempts=1)
+instaloader.context._session.cookies.update(connect(FIREFOXCOOKIEFILE)
+                                            .execute("SELECT name, value FROM moz_cookies "
+                                                     "WHERE baseDomain='instagram.com'"))
+
+try:
+    username = instaloader.test_login()
+    if not username:
+        raise ConnectionException()
+except ConnectionException:
+    raise SystemExit("Cookie import failed. Are you logged in successfully in Firefox?")
+
+instaloader.context.username = username
+instaloader.save_session_to_file()

docs/codesnippets/README.md

@@ -0,0 +1,4 @@
+# Advanced Instaloader Examples
+
+This directory contains the scripts that are presented at:
+https://instaloader.github.io/codesnippets.html

docs/conf.py

@@ -20,8 +20,7 @@
 import os
 import subprocess
 import sys
-
-import sphinx_bootstrap_theme
+import typing
 
 sys.path.insert(0, os.path.abspath('..'))
 
@@ -87,7 +86,7 @@ author = 'Alexander Graf and Andre Koch-Kramer'
 #
 # This is also used if you do content translation via gettext catalogs.
 # Usually you set "language" from the command line for these cases.
-language = None
+language = "en"
 
 # There are two options for replacing |today|: either, you set today to some
 # non-false value, then it is used:
@@ -123,7 +122,7 @@ add_module_names = False
 # show_authors = False
 
 # The name of the Pygments (syntax highlighting) style to use.
-pygments_style = 'sphinx'
+pygments_style = 'fruity'
 
 # A list of ignored prefixes for module index sorting.
 # modindex_common_prefix = []
@@ -140,20 +139,7 @@ todo_include_todos = False
 # The theme to use for HTML and HTML Help pages.  See the documentation for
 # a list of builtin themes.
 #
-html_theme = 'bootstrap'
-html_theme_path = sphinx_bootstrap_theme.get_html_theme_path()
-
-# Theme options are theme-specific and customize the look and feel of a theme
-# further.  For a list of options available for each theme, see the
-# documentation.
-#
-html_theme_options = {
-    'navbar_site_name': 'Site Contents',
-    'navbar_pagenav_name': 'Page Contents',
-    'navbar_pagenav': True,
-    'navbar_sidebarrel': True,
-    'nosidebar': True,
-     }
+html_theme = 'basic'
 
 # Add any paths that contain custom themes here, relative to this directory.
 # html_theme_path = []
@@ -374,6 +360,7 @@ html_context = {'current_release': current_release, 'current_release_date': curr
 
 
 def setup(app):
-    app.add_stylesheet("style.css")
+    typing.TYPE_CHECKING = True
+    app.add_stylesheet("instaloader.css")
     app.connect('autodoc-process-signature', sphinx_autodoc_typehints.process_signature)
     app.connect('autodoc-process-docstring', sphinx_autodoc_typehints.process_docstring)

docs/contributing.rst

@@ -1,3 +1,5 @@
+.. _contributing:
+
 Contributing to Instaloader
 ===========================
 
@@ -21,12 +23,14 @@ following in mind:
 - Ensure you **use the latest version** of Instaloader. The currently-installed
   version can be found out with ``instaloader --version``.
 
+- Check whether there is a valid solution in our :ref:`troubleshooting` section.
+
 - Briefly **check whether the bug has already been reported**. If you find an
   issue reporting the same bug you encountered, comment there rather than
-  opening a new issue. However, if unsure, it is advised to create a new issue.
+  opening a new issue. However, if unsure, please create a new issue.
 
 - State **how the bug can be reproduced**, i.e. how Instaloader was invoked
-  when the problem occured (of course, you may anonymize profile names etc.).
+  when the problem occurred (of course, you may anonymize profile names etc.).
 
 - Include all **error messages and tracebacks** in the report.
 
@@ -47,13 +51,13 @@ few things to consider:
   you forked from the most recent branch.
 
 - We use `Pylint <https://www.pylint.org/>`__ for error and syntax checking of
-  ``instaloader.py``. The file ``.travis.yml`` in the project's root directory
+  the source. The file ``.travis.yml`` in the project's root directory
   shows how it is invoked. Note that sometimes it might be better to disable a
   warning rather than adapting the code to Pylint's desires.
 
 - The documentation source is located in the ``docs`` folder. The file
-  ``cli-options.rst`` is merely an RST-formatted copy of ``ìnstaloader --help``
-  output, of which the source is near the bottom of ``instaloader.py``.
+  ``cli-options.rst`` is merely an RST-formatted copy of ``instaloader --help``
+  output, of which the source is in ``instaloader/__main__.py``.
 
 - Feel free to contact us, even if you "only" have Proof-of-Concepts or
   not-fully integrated changes. They already might be an advance for the
@@ -81,3 +85,22 @@ Please consider the following:
 - **Be patient**. Naturally, bugs and pull requests have a higher priority than
   feature suggestions. Keep in mind that this is a free software project, and
   unfortunately we only have limited time to work on it.
+
+Donations
+---------
+
+.. donations-start
+
+It is a pleasure for us to share our Instaloader to the world, and we are proud
+to have attracted such an active and motivating community, with so many users
+who share their suggestions and ideas with us. Buying a community-sponsored beer
+or coffee from time to time is very likely to further raise our passion for the
+development of Instaloader.
+
+| For donations, we provide a PayPal.Me link and a Bitcoin address.
+|  `PayPal.me/aandergr <https://www.paypal.me/aandergr>`__
+|  BTC: 1Nst4LoadeYzrKjJ1DX9CpbLXBYE9RKLwY
+
+.. donations-end
+
+.. (Discussion in :issue:`130`)

docs/index.rst

@@ -15,6 +15,10 @@ Instaloader
 **Instaloader** is a tool to download pictures (or videos) along with
 their captions and other metadata from Instagram.
 
+.. include:: ../README.rst
+   :start-after: badges-start
+   :end-before: badges-end
+
 With `Python <https://www.python.org/>`__ installed, do::
 
     $ pip3 install instaloader
@@ -41,9 +45,9 @@ See :ref:`install` for more options on how to install Instaloader.
 
 ::
 
-    instaloader [--comments] [--geotags] [--stories]
+    instaloader [--comments] [--geotags] [--stories] [--highlights] [--tagged]
                 [--login YOUR-USERNAME] [--fast-update]
-                profile | "#hashtag" |
+                profile | "#hashtag" | %location_id |
                 :stories | :feed | :saved
 
 See :ref:`download-pictures-from-instagram` for a detailed introduction on how
@@ -60,6 +64,7 @@ Instaloader Documentation
    cli-options
    as-module
    codesnippets
+   troubleshooting
    contributing
 
 Useful Links
@@ -70,10 +75,23 @@ Useful Links
 - `Issue Tracker / Bug Tracker <https://github.com/instaloader/instaloader/issues>`__
 - `Version History <https://github.com/instaloader/instaloader/releases>`__
 
+Disclaimer
+----------
+
 .. include:: ../README.rst
    :start-after: disclaimer-start
    :end-before: disclaimer-end
 
+Contributing
+------------
+
+As an open source project, Instaloader heavily depends on the contributions from
+its community. See :ref:`contributing` for how you may help Instaloader to
+become an even greater tool.
+
+.. include:: contributing.rst
+   :start-after: donations-start
+   :end-before: donations-end
 
 ..
    * :ref:`genindex`

docs/installation.rst

@@ -33,7 +33,6 @@ Install Instaloader
 - On **Windows 10**, you may download the standalone executable from the
   `current release page <https://github.com/instaloader/instaloader/releases/latest>`__.
 
-- To get the most current version of Instaloader from our
-  `Git repository <https://github.com/instaloader/instaloader>`__::
+- To test the most current pre-release or development version of Instaloader::
 
-     pip3 install --upgrade git+https://github.com/instaloader/instaloader
+     pip3 install --pre instaloader

docs/requirements.txt

@@ -1,3 +1,2 @@
 requests
-sphinx
-sphinx-bootstrap-theme
+sphinx!=1.7.7

docs/sphinx_autodoc_typehints.py

@@ -1,4 +1,4 @@
-# sphinx_autodoc_typehints from Mar 18 2018.
+# sphinx_autodoc_typehints from Nov 20 2018 (version 1.5.2).
 #  https://github.com/agronholm/sphinx-autodoc-typehints
 # (slightly modified to fix class links, maybe related to agronholm/sphinx-autodoc-typehints#38)
 #
@@ -27,8 +27,8 @@
 import inspect
 from typing import get_type_hints, TypeVar, Any, AnyStr, Generic, Union
 
-from sphinx.util.inspect import getargspec
-from sphinx.ext.autodoc import formatargspec
+from sphinx.util import logging
+from sphinx.util.inspect import Signature
 
 try:
     from inspect import unwrap
@@ -51,6 +51,8 @@ except ImportError:
             memo.add(id_func)
         return func
 
+logger = logging.getLogger(__name__)
+
 
 def format_annotation(annotation):
     if inspect.isclass(annotation) and annotation.__module__ == 'builtins':
@@ -162,23 +164,37 @@ def process_signature(app, what: str, name: str, obj, options, signature, return
         return
 
     obj = unwrap(obj)
-    try:
-        argspec = getargspec(obj)
-    except (TypeError, ValueError):
-        return
+    signature = Signature(obj)
+    parameters = [
+        param.replace(annotation=inspect.Parameter.empty)
+        for param in signature.signature.parameters.values()
+    ]
 
-    if argspec.args:
+    if parameters:
         if what in ('class', 'exception'):
-            del argspec.args[0]
+            del parameters[0]
         elif what == 'method':
             outer = inspect.getmodule(obj)
             for clsname in obj.__qualname__.split('.')[:-1]:
                 outer = getattr(outer, clsname)
-            method_object = outer.__dict__[obj.__name__]
-            if not isinstance(method_object, (classmethod, staticmethod)):
-                del argspec.args[0]
 
-    return formatargspec(obj, *argspec[:-1]), None
+            method_name = obj.__name__
+            if method_name.startswith("__") and not method_name.endswith("__"):
+                # If the method starts with double underscore (dunder)
+                # Python applies mangling so we need to prepend the class name.
+                # This doesn't happen if it always ends with double underscore.
+                class_name = obj.__qualname__.split('.')[-2]
+                method_name = "_{c}{m}".format(c=class_name, m=method_name)
+
+            method_object = outer.__dict__[method_name]
+            if not isinstance(method_object, (classmethod, staticmethod)):
+                del parameters[0]
+
+    signature.signature = signature.signature.replace(
+        parameters=parameters,
+        return_annotation=inspect.Signature.empty)
+
+    return signature.format_args().replace('\\', '\\\\'), None
 
 
 def process_docstring(app, what, name, obj, options, lines):
@@ -195,8 +211,15 @@ def process_docstring(app, what, name, obj, options, lines):
         except (AttributeError, TypeError):
             # Introspecting a slot wrapper will raise TypeError
             return
+        except NameError as exc:
+            logger.warning('Cannot resolve forward reference in type annotations of "%s": %s',
+                           name, exc)
+            type_hints = obj.__annotations__
 
         for argname, annotation in type_hints.items():
+            if argname.endswith('_'):
+                argname = '{}\\_'.format(argname[:-1])
+
             formatted_annotation = format_annotation(annotation)
 
             if argname == 'return':

docs/troubleshooting.rst

@@ -0,0 +1,60 @@
+.. _troubleshooting:
+
+Troubleshooting
+===============
+
+.. highlight:: python
+
+429 - Too Many Requests
+-----------------------
+
+Instaloader has a logic to keep track of its requests to Instagram and to obey
+their rate limits. Since they are nowhere documented, we try them out
+experimentally. We have a daily cron job running to confirm that Instaloader
+still stays within the rate limits. Nevertheless, the rate control logic assumes
+that
+
+- at one time, Instaloader is the only application that consumes requests. I.e.
+  neither the Instagram browser interface, nor a mobile app, nor another
+  Instaloader instance is running in parallel,
+
+- no requests had been consumed when Instaloader starts.
+
+The latter one implies that restarting or reinstantiating Instaloader often
+within short time is prone to cause a 429. When a request is denied with a 429,
+Instaloader retries the request as soon as the temporary ban is assumed to be
+expired. In case the retry continuously fails for some reason, which should not
+happen in normal conditions, consider adjusting the
+:option:`--max-connection-attempts` option.
+
+**"Too many queries in the last time"** is not an error. It is a notice that the
+rate limit has almost been reached, according to Instaloader's own rate
+accounting mechanism. We regularly adjust this mechanism to match Instagram's
+current rate limiting.
+
+Login error
+-----------
+
+Instaloader's login *should* work fine, both with and without
+Two-Factor-Authentication. It also supports handling the *checkpoint challenge*,
+issued when Instagram suspects authentication activity on your account, by
+pointing the user to an URL to be opened in a browser.
+
+Nevertheless, in :issue:`92` users report problems with logging in. To still use
+Instaloader's logged-in functionality, you may use the following script to
+workaround login problems by importing the session cookies from Firefox and
+bypassing Instaloader's login.
+
+.. literalinclude:: codesnippets/92_import_firefox_session.py
+
+To use this,
+
+#. login to Instagram in Firefox, 
+
+#. execute the snippet,
+
+#. then, ``instaloader -l USERNAME`` should work fine.
+
+If you do not use your default firefox profile, or your operating system has the
+paths differently set up, you may have to alter the ``FIREFOXCOOKIEFILE``
+variable first.

instaloader/__init__.py

@@ -1,12 +1,12 @@
 """Download pictures (or videos) along with their captions and other metadata from Instagram."""
 
 
-__version__ = '4.0.4'
+__version__ = '4.2.5'
 
 
 try:
     # pylint:disable=wrong-import-position
-    import win_unicode_console
+    import win_unicode_console  # type: ignore
 except ImportError:
     pass
 else:
@@ -15,5 +15,5 @@ else:
 from .exceptions import *
 from .instaloader import Instaloader
 from .instaloadercontext import InstaloaderContext
-from .structures import (Post, PostSidecarNode, PostComment, PostLocation, Profile, Story, StoryItem,
+from .structures import (Highlight, Post, PostSidecarNode, PostComment, PostCommentAnswer, PostLocation, Profile, Story, StoryItem,
                          load_structure_from_file, save_structure_to_file)

instaloader/__main__.py

@@ -8,7 +8,8 @@ from argparse import ArgumentParser, SUPPRESS
 from typing import List, Optional
 
 from . import (Instaloader, InstaloaderException, InvalidArgumentException, Post, Profile, ProfileNotExistsException,
-               StoryItem, __version__, load_structure_from_file)
+               StoryItem, __version__, load_structure_from_file, TwoFactorAuthRequiredException,
+               BadCredentialsException)
 from .instaloader import get_default_session_filename
 from .instaloadercontext import default_user_agent
 
@@ -18,9 +19,9 @@ def usage_string():
     argv0 = os.path.basename(sys.argv[0])
     argv0 = "instaloader" if argv0 == "__main__.py" else argv0
     return """
-{0} [--comments] [--geotags] [--stories]
+{0} [--comments] [--geotags] [--stories] [--highlights] [--tagged]
 {2:{1}} [--login YOUR-USERNAME] [--fast-update]
-{2:{1}} profile | "#hashtag" | :stories | :feed | :saved
+{2:{1}} profile | "#hashtag" | %%location_id | :stories | :feed | :saved
 {0} --help""".format(argv0, len(argv0), '')
 
 
@@ -58,11 +59,11 @@ def filterstr_to_filterfunc(filter_str: str, item_type: type):
 
 def _main(instaloader: Instaloader, targetlist: List[str],
           username: Optional[str] = None, password: Optional[str] = None,
-          sessionfile: Optional[str] = None, max_count: Optional[int] = None,
-          profile_pic: bool = True, profile_pic_only: bool = False,
+          sessionfile: Optional[str] = None,
+          download_profile_pic: bool = True, download_posts=True,
+          download_stories: bool = False, download_highlights: bool = False, download_tagged: bool = False,
           fast_update: bool = False,
-          stories: bool = False, stories_only: bool = False,
-          post_filter_str: Optional[str] = None,
+          max_count: Optional[int] = None, post_filter_str: Optional[str] = None,
           storyitem_filter_str: Optional[str] = None) -> None:
     """Download set of profiles, hashtags etc. and handle logging in and session files if desired."""
     # Parse and generate filter function
@@ -84,12 +85,22 @@ def _main(instaloader: Instaloader, targetlist: List[str],
             instaloader.context.log("Session file does not exist yet - Logging in.")
         if not instaloader.context.is_logged_in or username != instaloader.test_login():
             if password is not None:
-                instaloader.login(username, password)
+                try:
+                    instaloader.login(username, password)
+                except TwoFactorAuthRequiredException:
+                    while True:
+                        try:
+                            code = input("Enter 2FA verification code: ")
+                            instaloader.two_factor_login(code)
+                            break
+                        except BadCredentialsException:
+                            pass
             else:
                 instaloader.interactive_login(username)
         instaloader.context.log("Logged in as %s." % username)
     # Try block for KeyboardInterrupt (save session on ^C)
     profiles = set()
+    anonymous_retry_profiles = set()
     try:
         # Generate set of profiles, already downloading non-profile targets
         for target in targetlist:
@@ -121,11 +132,17 @@ def _main(instaloader: Instaloader, targetlist: List[str],
                 if target[0] == '@':
                     instaloader.context.log("Retrieving followees of %s..." % target[1:])
                     profile = Profile.from_username(instaloader.context, target[1:])
-                    followees = profile.get_followees()
-                    profiles.update([followee.username for followee in followees])
+                    for followee in profile.get_followees():
+                        instaloader.save_profile_id(followee)
+                        profiles.add(followee)
                 elif target[0] == '#':
                     instaloader.download_hashtag(hashtag=target[1:], max_count=max_count, fast_update=fast_update,
                                                  post_filter=post_filter)
+                elif target[0] == '-':
+                    instaloader.download_post(Post.from_shortcode(instaloader.context, target[1:]), target)
+                elif target[0] == "%":
+                    instaloader.download_location(location=target[1:], max_count=max_count, fast_update=fast_update,
+                                                  post_filter=post_filter)
                 elif target == ":feed":
                     instaloader.download_feed_posts(fast_update=fast_update, max_count=max_count,
                                                     post_filter=post_filter)
@@ -135,26 +152,45 @@ def _main(instaloader: Instaloader, targetlist: List[str],
                     instaloader.download_saved_posts(fast_update=fast_update, max_count=max_count,
                                                      post_filter=post_filter)
                 else:
-                    profiles.add(target)
+                    try:
+                        profile = instaloader.check_profile_id(target)
+                        if instaloader.context.is_logged_in and profile.has_blocked_viewer:
+                            if download_profile_pic or ((download_posts or download_tagged) and not profile.is_private):
+                                raise ProfileNotExistsException("{} blocked you; But we download her anonymously."
+                                                                .format(target))
+                            else:
+                                instaloader.context.error("{} blocked you.".format(target))
+                        else:
+                            profiles.add(profile)
+                    except ProfileNotExistsException as err:
+                        # Not only our profile.has_blocked_viewer condition raises ProfileNotExistsException,
+                        # check_profile_id() also does, since access to blocked profile may be responded with 404.
+                        if instaloader.context.is_logged_in and (download_profile_pic or download_posts or
+                                                                 download_tagged):
+                            instaloader.context.log(err)
+                            instaloader.context.log("Trying again anonymously, helps in case you are just blocked.")
+                            with instaloader.anonymous_copy() as anonymous_loader:
+                                with instaloader.context.error_catcher():
+                                    anonymous_retry_profiles.add(anonymous_loader.check_profile_id(target))
+                                    instaloader.context.error("Warning: {} will be downloaded anonymously (\"{}\")."
+                                                              .format(target, err))
+                        else:
+                            raise
         if len(profiles) > 1:
-            instaloader.context.log("Downloading {} profiles: {}".format(len(profiles), ' '.join(profiles)))
-        # Iterate through profiles list and download them
-        for target in profiles:
-            with instaloader.context.error_catcher(target):
-                try:
-                    instaloader.download_profile(target, profile_pic, profile_pic_only, fast_update,
-                                                 stories, stories_only, post_filter=post_filter,
-                                                 storyitem_filter=storyitem_filter)
-                except ProfileNotExistsException as err:
-                    if instaloader.context.is_logged_in:
-                        instaloader.context.log(err)
-                        instaloader.context.log("Trying again anonymously, helps in case you are just blocked.")
-                        with instaloader.anonymous_copy() as anonymous_loader:
-                            with instaloader.context.error_catcher():
-                                anonymous_loader.download_profile(target, profile_pic, profile_pic_only,
-                                                                  fast_update, post_filter=post_filter)
-                    else:
-                        raise
+            instaloader.context.log("Downloading {} profiles: {}".format(len(profiles),
+                                                                         ' '.join([p.username for p in profiles])))
+        if profiles and download_profile_pic and not instaloader.context.is_logged_in:
+            instaloader.context.error("Warning: Use --login to download HD version of profile pictures.")
+        instaloader.download_profiles(profiles,
+                                      download_profile_pic, download_posts, download_tagged, download_highlights,
+                                      download_stories, fast_update, post_filter, storyitem_filter)
+        if anonymous_retry_profiles:
+            instaloader.context.log("Downloading anonymously: {}"
+                                    .format(' '.join([p.username for p in anonymous_retry_profiles])))
+            with instaloader.anonymous_copy() as anonymous_loader:
+                anonymous_loader.download_profiles(anonymous_retry_profiles,
+                                                   download_profile_pic, download_posts, download_tagged,
+                                                   fast_update=fast_update, post_filter=post_filter)
     except KeyboardInterrupt:
         print("\nInterrupted by user.", file=sys.stderr)
     # Save session if it is useful
@@ -174,72 +210,96 @@ def main():
     parser = ArgumentParser(description=__doc__, add_help=False, usage=usage_string(),
                             epilog="Report issues at https://github.com/instaloader/instaloader/issues. "
                                    "The complete documentation can be found at "
-                                   "https://instaloader.github.io/.")
+                                   "https://instaloader.github.io/.",
+                            fromfile_prefix_chars='+')
 
-    g_what = parser.add_argument_group('What to Download',
-                                       'Specify a list of profiles or #hashtags. For each of these, Instaloader '
-                                       'creates a folder and '
-                                       'downloads all posts along with the pictures\'s '
-                                       'captions and the current profile picture. '
-                                       'If an already-downloaded profile has been renamed, Instaloader automatically '
-                                       'finds it by its unique ID and renames the folder likewise.')
-    g_what.add_argument('profile', nargs='*', metavar='profile|#hashtag',
-                        help='Name of profile or #hashtag to download. '
-                             'Alternatively, if --login is given: @<profile> to download all followees of '
-                             '<profile>; the special targets '
-                             ':feed to download pictures from your feed; '
-                             ':stories to download the stories of your followees; or '
-                             ':saved to download the posts marked as saved.')
-    g_what.add_argument('-P', '--profile-pic-only', action='store_true',
-                        help='Only download profile picture.')
-    g_what.add_argument('--no-profile-pic', action='store_true',
+    g_targets = parser.add_argument_group("What to Download",
+                                          "Specify a list of targets. For each of these, Instaloader creates a folder "
+                                          "and downloads all posts. The following targets are supported:")
+    g_targets.add_argument('profile', nargs='*',
+                           help="Download profile. If an already-downloaded profile has been renamed, Instaloader "
+                                "automatically finds it by its unique ID and renames the folder likewise.")
+    g_targets.add_argument('_at_profile', nargs='*', metavar="@profile",
+                           help="Download all followees of profile. Requires --login. "
+                                "Consider using :feed rather than @yourself.")
+    g_targets.add_argument('_hashtag', nargs='*', metavar='"#hashtag"', help="Download #hashtag.")
+    g_targets.add_argument('_location', nargs='*', metavar='%location_id', help="Download %%location_id.")
+    g_targets.add_argument('_feed', nargs='*', metavar=":feed",
+                           help="Download pictures from your feed. Requires --login.")
+    g_targets.add_argument('_stories', nargs='*', metavar=":stories",
+                           help="Download the stories of your followees. Requires --login.")
+    g_targets.add_argument('_saved', nargs='*', metavar=":saved",
+                           help="Download the posts that you marked as saved. Requires --login.")
+    g_targets.add_argument('_singlepost', nargs='*', metavar="-- -shortcode",
+                           help="Download the post with the given shortcode")
+    g_targets.add_argument('_json', nargs='*', metavar="filename.json[.xz]",
+                           help="Re-Download the given object.")
+    g_targets.add_argument('_fromfile', nargs='*', metavar="+args.txt",
+                           help="Read targets (and options) from given textfile.")
+
+    g_post = parser.add_argument_group("What to Download of each Post")
+
+    g_prof = parser.add_argument_group("What to Download of each Profile")
+
+    g_prof.add_argument('-P', '--profile-pic-only', action='store_true',
+                        help=SUPPRESS)
+    g_prof.add_argument('--no-posts', action='store_true',
+                        help="Do not download regular posts.")
+    g_prof.add_argument('--no-profile-pic', action='store_true',
                         help='Do not download profile picture.')
-    g_what.add_argument('-V', '--no-videos', action='store_true',
+    g_post.add_argument('--no-pictures', action='store_true',
+                        help='Do not download post pictures. Cannot be used together with --fast-update. '
+                             'Implies --no-video-thumbnails, does not imply --no-videos.')
+    g_post.add_argument('-V', '--no-videos', action='store_true',
                         help='Do not download videos.')
-    g_what.add_argument('--no-video-thumbnails', action='store_true',
+    g_post.add_argument('--no-video-thumbnails', action='store_true',
                         help='Do not download thumbnails of videos.')
-    g_what.add_argument('-G', '--geotags', action='store_true',
+    g_post.add_argument('-G', '--geotags', action='store_true',
                         help='Download geotags when available. Geotags are stored as a '
                              'text file with the location\'s name and a Google Maps link. '
                              'This requires an additional request to the Instagram '
                              'server for each picture, which is why it is disabled by default.')
-    g_what.add_argument('-C', '--comments', action='store_true',
+    g_post.add_argument('-C', '--comments', action='store_true',
                         help='Download and update comments for each post. '
                              'This requires an additional request to the Instagram '
                              'server for each post, which is why it is disabled by default.')
-    g_what.add_argument('--no-captions', action='store_true',
+    g_post.add_argument('--no-captions', action='store_true',
                         help='Do not create txt files.')
-    g_what.add_argument('--post-metadata-txt', action='append',
+    g_post.add_argument('--post-metadata-txt', action='append',
                         help='Template to write in txt file for each Post.')
-    g_what.add_argument('--storyitem-metadata-txt', action='append',
+    g_post.add_argument('--storyitem-metadata-txt', action='append',
                         help='Template to write in txt file for each StoryItem.')
-    g_what.add_argument('--no-metadata-json', action='store_true',
+    g_post.add_argument('--no-metadata-json', action='store_true',
                         help='Do not create a JSON file containing the metadata of each post.')
-    g_what.add_argument('--metadata-json', action='store_true',
+    g_post.add_argument('--metadata-json', action='store_true',
                         help=SUPPRESS)
-    g_what.add_argument('--no-compress-json', action='store_true',
+    g_post.add_argument('--no-compress-json', action='store_true',
                         help='Do not xz compress JSON files, rather create pretty formatted JSONs.')
-    g_what.add_argument('-s', '--stories', action='store_true',
+    g_prof.add_argument('-s', '--stories', action='store_true',
                         help='Also download stories of each profile that is downloaded. Requires --login.')
-    g_what.add_argument('--stories-only', action='store_true',
-                        help='Rather than downloading regular posts of each specified profile, only download '
-                             'stories. Requires --login. Does not imply --no-profile-pic.')
-    g_what.add_argument('--post-filter', '--only-if', metavar='filter',
+    g_prof.add_argument('--stories-only', action='store_true',
+                        help=SUPPRESS)
+    g_prof.add_argument('--highlights', action='store_true',
+                        help='Also download highlights of each profile that is downloaded. Requires --login.')
+    g_prof.add_argument('--tagged', action='store_true',
+                        help='Also download posts where each profile is tagged.')
+
+    g_cond = parser.add_argument_group("Which Posts to Download")
+
+    g_cond.add_argument('-F', '--fast-update', action='store_true',
+                        help='For each target, stop when encountering the first already-downloaded picture. This '
+                             'flag is recommended when you use Instaloader to update your personal Instagram archive.')
+
+    g_cond.add_argument('--post-filter', '--only-if', metavar='filter',
                         help='Expression that, if given, must evaluate to True for each post to be downloaded. Must be '
                              'a syntactically valid python expression. Variables are evaluated to '
                              'instaloader.Post attributes. Example: --post-filter=viewer_has_liked.')
-    g_what.add_argument('--storyitem-filter', metavar='filter',
+    g_cond.add_argument('--storyitem-filter', metavar='filter',
                         help='Expression that, if given, must evaluate to True for each storyitem to be downloaded. '
                              'Must be a syntactically valid python expression. Variables are evaluated to '
                              'instaloader.StoryItem attributes.')
 
-    g_stop = parser.add_argument_group('When to Stop Downloading',
-                                       'If none of these options are given, Instaloader goes through all pictures '
-                                       'matching the specified targets.')
-    g_stop.add_argument('-F', '--fast-update', action='store_true',
-                        help='For each target, stop when encountering the first already-downloaded picture. This '
-                             'flag is recommended when you use Instaloader to update your personal Instagram archive.')
-    g_stop.add_argument('-c', '--count',
+    g_cond.add_argument('-c', '--count',
                         help='Do not attempt to download more than COUNT posts. '
                              'Applies only to #hashtag and :feed.')
 
@@ -271,11 +331,14 @@ def main():
     g_how.add_argument('--user-agent',
                        help='User Agent to use for HTTP requests. Defaults to \'{}\'.'.format(default_user_agent()))
     g_how.add_argument('-S', '--no-sleep', action='store_true', help=SUPPRESS)
-    g_how.add_argument('--graphql-rate-limit', type=int, help=SUPPRESS)
     g_how.add_argument('--max-connection-attempts', metavar='N', type=int, default=3,
                        help='Maximum number of connection attempts until a request is aborted. Defaults to 3. If a '
                             'connection fails, it can be manually skipped by hitting CTRL+C. Set this to 0 to retry '
                             'infinitely.')
+    g_how.add_argument('--commit-mode', action='store_true',
+                       help='Tries to ensure downloaded images avoid corruption in case of unexpected interruption. '
+                       'If the last picture is corrupted, Instaloader will fix the picture the next time it is run. '
+                       'Requires the JSON metadata to be saved.')
 
     g_misc = parser.add_argument_group('Miscellaneous Options')
     g_misc.add_argument('-q', '--quiet', action='store_true',
@@ -309,27 +372,40 @@ def main():
                 raise SystemExit("--no-captions and --post-metadata-txt or --storyitem-metadata-txt given; "
                                  "That contradicts.")
 
+        if args.no_pictures and args.fast_update:
+            raise SystemExit('--no-pictures and --fast-update cannot be used together.')
+
+        # Determine what to download
+        download_profile_pic = not args.no_profile_pic or args.profile_pic_only
+        download_posts = not (args.no_posts or args.stories_only or args.profile_pic_only)
+        download_stories = args.stories or args.stories_only
+
+        if args.commit_mode and args.no_metadata_json:
+            raise SystemExit('--commit-mode requires JSON metadata to be saved.')
+
         loader = Instaloader(sleep=not args.no_sleep, quiet=args.quiet, user_agent=args.user_agent,
                              dirname_pattern=args.dirname_pattern, filename_pattern=args.filename_pattern,
+                             download_pictures=not args.no_pictures,
                              download_videos=not args.no_videos, download_video_thumbnails=not args.no_video_thumbnails,
                              download_geotags=args.geotags,
                              download_comments=args.comments, save_metadata=not args.no_metadata_json,
                              compress_json=not args.no_compress_json,
                              post_metadata_txt_pattern=post_metadata_txt_pattern,
                              storyitem_metadata_txt_pattern=storyitem_metadata_txt_pattern,
-                             graphql_rate_limit=args.graphql_rate_limit,
-                             max_connection_attempts=args.max_connection_attempts)
+                             max_connection_attempts=args.max_connection_attempts,
+                             commit_mode=args.commit_mode)
         _main(loader,
               args.profile,
               username=args.login.lower() if args.login is not None else None,
               password=args.password,
               sessionfile=args.sessionfile,
-              max_count=int(args.count) if args.count is not None else None,
-              profile_pic=not args.no_profile_pic,
-              profile_pic_only=args.profile_pic_only,
+              download_profile_pic=download_profile_pic,
+              download_posts=download_posts,
+              download_stories=download_stories,
+              download_highlights=args.highlights,
+              download_tagged=args.tagged,
               fast_update=args.fast_update,
-              stories=args.stories,
-              stories_only=args.stories_only,
+              max_count=int(args.count) if args.count is not None else None,
               post_filter_str=args.post_filter,
               storyitem_filter_str=args.storyitem_filter)
         loader.close()

instaloader/exceptions.py

@@ -22,6 +22,10 @@ class ProfileNotExistsException(InstaloaderException):
 
 
 class ProfileHasNoPicsException(InstaloaderException):
+    """
+    .. deprecated:: 4.2.2
+       Not raised anymore.
+    """
     pass
 
 
@@ -33,6 +37,10 @@ class LoginRequiredException(InstaloaderException):
     pass
 
 
+class TwoFactorAuthRequiredException(InstaloaderException):
+    pass
+
+
 class InvalidArgumentException(InstaloaderException):
     pass
 
@@ -49,5 +57,10 @@ class ConnectionException(InstaloaderException):
     pass
 
 
+class PostChangedException(InstaloaderException):
+    """.. versionadded:: 4.2.2"""
+    pass
+
+
 class TooManyRequestsException(ConnectionException):
     pass

instaloader/instaloader.py

@@ -1,5 +1,6 @@
 import getpass
 import json
+import lzma
 import os
 import platform
 import re
@@ -10,12 +11,16 @@ import tempfile
 from contextlib import contextmanager, suppress
 from datetime import datetime, timezone
 from functools import wraps
+from hashlib import md5
 from io import BytesIO
-from typing import Callable, Iterator, List, Optional, Any
+from typing import Any, Callable, ContextManager, Iterator, List, Optional, Set, Union, cast
+
+import requests
+import urllib3  # type: ignore
 
 from .exceptions import *
 from .instaloadercontext import InstaloaderContext
-from .structures import JsonExportable, Post, PostLocation, Profile, Story, StoryItem, save_structure_to_file
+from .structures import Highlight, JsonExportable, Post, PostLocation, Profile, Story, StoryItem, save_structure_to_file, load_structure_from_file
 
 
 def get_default_session_filename(username: str) -> str:
@@ -40,8 +45,39 @@ def _requires_login(func: Callable) -> Callable:
         if not instaloader.context.is_logged_in:
             raise LoginRequiredException("--login=USERNAME required.")
         return func(instaloader, *args, **kwargs)
-    # pylint:disable=no-member
-    call.__doc__ += ":raises LoginRequiredException: If called without being logged in.\n"
+    docstring_text = ":raises LoginRequiredException: If called without being logged in.\n"
+    call.__doc__ = call.__doc__ + docstring_text if call.__doc__ is not None else docstring_text
+    return call
+
+
+def _retry_on_connection_error(func: Callable) -> Callable:
+    """Decorator to retry the function max_connection_attemps number of times.
+
+    Herewith-decorated functions need an ``_attempt`` keyword argument.
+
+    This is to decorate functions that do network requests that may fail. Note that
+    :meth:`.get_json`, :meth:`.get_iphone_json`, :meth:`.graphql_query` and :meth:`.graphql_node_list` already have
+    their own logic for retrying, hence functions that only use these for network access must not be decorated with this
+    decorator."""
+    @wraps(func)
+    def call(instaloader, *args, **kwargs):
+        try:
+            return func(instaloader, *args, **kwargs)
+        except (urllib3.exceptions.HTTPError, requests.exceptions.RequestException, ConnectionException) as err:
+            error_string = "{}({}): {}".format(func.__name__, ', '.join([repr(arg) for arg in args]), err)
+            if (kwargs.get('_attempt') or 1) == instaloader.context.max_connection_attempts:
+                raise ConnectionException(error_string) from None
+            instaloader.context.error(error_string + " [retrying; skip with ^C]", repeat_at_end=False)
+            try:
+                if kwargs.get('_attempt'):
+                    kwargs['_attempt'] += 1
+                else:
+                    kwargs['_attempt'] = 2
+                instaloader.context.do_sleep()
+                return call(instaloader, *args, **kwargs)
+            except KeyboardInterrupt:
+                instaloader.context.error("[skipped by user]", repeat_at_end=False)
+                raise ConnectionException(error_string) from None
     return call
 
 
@@ -49,11 +85,11 @@ class _ArbitraryItemFormatter(string.Formatter):
     def __init__(self, item: Any):
         self._item = item
 
-    def get_field(self, field_name, args, kwargs):
+    def get_value(self, key, args, kwargs):
         """Override to substitute {ATTRIBUTE} by attributes of our _item."""
-        if hasattr(self._item, field_name):
-            return self._item.__getattribute__(field_name), None
-        return super().get_field(field_name, args, kwargs)
+        if hasattr(self._item, key):
+            return getattr(self._item, key)
+        return super().get_value(key, args, kwargs)
 
     def format_field(self, value, format_spec):
         """Override :meth:`string.Formatter.format_field` to have our
@@ -70,7 +106,20 @@ class _PostPathFormatter(_ArbitraryItemFormatter):
     def vformat(self, format_string, args, kwargs):
         """Override :meth:`string.Formatter.vformat` for character substitution in paths for Windows, see issue #84."""
         ret = super().vformat(format_string, args, kwargs)
-        return ret.replace(':', '\ua789') if platform.system() == 'Windows' else ret
+        if platform.system() == 'Windows':
+            ret = ret.replace(':', '\ua789').replace('<', '\ufe64').replace('>', '\ufe65').replace('\"', '\uff02')
+            ret = ret.replace('\\', '\ufe68').replace('|', '\uff5c').replace('?', '\ufe16').replace('*', '\uff0a')
+        return ret
+
+    def get_value(self, key, args, kwargs):
+        """Replaces '/' with similar looking Division Slash and on windows newline with space"""
+        ret = super().get_value(key, args, kwargs)
+        if not isinstance(ret, str):
+            return ret
+        ret = ret.replace('/', '\u2215')
+        if platform.system() == 'Windows':
+            ret = ret.replace('\n', ' ')
+        return ret
 
 
 class Instaloader:
@@ -80,6 +129,7 @@ class Instaloader:
     :param user_agent: :option:`--user-agent`
     :param dirname_pattern: :option:`--dirname-pattern`, default is ``{target}``
     :param filename_pattern: :option:`--filename-pattern`, default is ``{date_utc}_UTC``
+    :param download_pictures: not :option:`--no-pictures`
     :param download_videos: not :option:`--no-videos`
     :param download_video_thumbnails: not :option:`--no-video-thumbnails`
     :param download_geotags: :option:`--geotags`
@@ -91,6 +141,7 @@ class Instaloader:
        txt file.
     :param storyitem_metadata_txt_pattern: :option:`--storyitem-metadata-txt`, default is empty (=none)
     :param max_connection_attempts: :option:`--max-connection-attempts`
+    :param commit_mode: :option:`--commit-mode`
 
     .. attribute:: context
 
@@ -102,6 +153,7 @@ class Instaloader:
                  user_agent: Optional[str] = None,
                  dirname_pattern: Optional[str] = None,
                  filename_pattern: Optional[str] = None,
+                 download_pictures=True,
                  download_videos: bool = True,
                  download_video_thumbnails: bool = True,
                  download_geotags: bool = True,
@@ -110,14 +162,15 @@ class Instaloader:
                  compress_json: bool = True,
                  post_metadata_txt_pattern: str = None,
                  storyitem_metadata_txt_pattern: str = None,
-                 graphql_rate_limit: Optional[int] = None,
-                 max_connection_attempts: int = 3):
+                 max_connection_attempts: int = 3,
+                 commit_mode: bool = False):
 
-        self.context = InstaloaderContext(sleep, quiet, user_agent, graphql_rate_limit, max_connection_attempts)
+        self.context = InstaloaderContext(sleep, quiet, user_agent, max_connection_attempts)
 
         # configuration parameters
         self.dirname_pattern = dirname_pattern or "{target}"
         self.filename_pattern = filename_pattern or "{date_utc}_UTC"
+        self.download_pictures = download_pictures
         self.download_videos = download_videos
         self.download_video_thumbnails = download_video_thumbnails
         self.download_geotags = download_geotags
@@ -128,21 +181,28 @@ class Instaloader:
             else post_metadata_txt_pattern
         self.storyitem_metadata_txt_pattern = '' if storyitem_metadata_txt_pattern is None \
             else storyitem_metadata_txt_pattern
+        self.commit_mode = commit_mode
+        if self.commit_mode and not self.save_metadata:
+            raise InvalidArgumentException("Commit mode requires JSON metadata to be saved.")
+
+        # Used to keep state in commit mode
+        self._committed = None  # type: Optional[bool]
 
     @contextmanager
     def anonymous_copy(self):
         """Yield an anonymous, otherwise equally-configured copy of an Instaloader instance; Then copy its error log."""
         new_loader = Instaloader(self.context.sleep, self.context.quiet, self.context.user_agent, self.dirname_pattern,
-                                 self.filename_pattern, self.download_videos, self.download_video_thumbnails,
-                                 self.download_geotags, self.download_comments, self.save_metadata,
-                                 self.compress_json, self.post_metadata_txt_pattern,
-                                 self.storyitem_metadata_txt_pattern, self.context.graphql_count_per_slidingwindow,
-                                 self.context.max_connection_attempts)
-        new_loader.context.query_timestamps = self.context.query_timestamps
+                                 self.filename_pattern, download_pictures=self.download_pictures,
+                                 download_videos=self.download_videos,
+                                 download_video_thumbnails=self.download_video_thumbnails,
+                                 download_geotags=self.download_geotags, download_comments=self.download_comments,
+                                 save_metadata=self.save_metadata, compress_json=self.compress_json,
+                                 post_metadata_txt_pattern=self.post_metadata_txt_pattern,
+                                 storyitem_metadata_txt_pattern=self.storyitem_metadata_txt_pattern,
+                                 max_connection_attempts=self.context.max_connection_attempts)
         yield new_loader
         self.context.error_log.extend(new_loader.context.error_log)
         new_loader.context.error_log = []  # avoid double-printing of errors
-        self.context.query_timestamps = new_loader.context.query_timestamps
         new_loader.close()
 
     def close(self):
@@ -155,7 +215,9 @@ class Instaloader:
     def __exit__(self, *args):
         self.close()
 
-    def download_pic(self, filename: str, url: str, mtime: datetime, filename_suffix: Optional[str] = None) -> bool:
+    @_retry_on_connection_error
+    def download_pic(self, filename: str, url: str, mtime: datetime,
+                     filename_suffix: Optional[str] = None, _attempt: int = 1) -> bool:
         """Downloads and saves picture with given url under given directory with given timestamp.
         Returns true, if file was actually downloaded, i.e. updated."""
         urlmatch = re.search('\\.[a-z0-9]*\\?', url)
@@ -163,9 +225,15 @@ class Instaloader:
         if filename_suffix is not None:
             filename += '_' + filename_suffix
         filename += '.' + file_extension
-        if os.path.isfile(filename):
-            self.context.log(filename + ' exists', end=' ', flush=True)
-            return False
+        # A post is considered "commited" if the json file exists and is not malformed.
+        if self.commit_mode:
+            if self._committed and os.path.isfile(filename):
+                self.context.log(filename + ' exists', end=' ', flush=True)
+                return False
+        else:
+            if os.path.isfile(filename):
+                self.context.log(filename + ' exists', end=' ', flush=True)
+                return False
         self.context.get_and_write_raw(url, filename)
         os.utime(filename, (datetime.now().timestamp(), mtime.timestamp()))
         return True
@@ -182,11 +250,33 @@ class Instaloader:
             self.context.log('json', end=' ', flush=True)
 
     def update_comments(self, filename: str, post: Post) -> None:
-        def _postcomment_asdict(comment):
+        def _postcommentanswer_asdict(comment):
             return {'id': comment.id,
                     'created_at': int(comment.created_at_utc.replace(tzinfo=timezone.utc).timestamp()),
                     'text': comment.text,
                     'owner': comment.owner._asdict()}
+
+        def _postcomment_asdict(comment):
+            return {**_postcommentanswer_asdict(comment),
+                    'answers': sorted([_postcommentanswer_asdict(answer) for answer in comment.answers],
+                                      key=lambda t: int(t['id']),
+                                      reverse=True)}
+
+        def get_unique_comments(comments, combine_answers=False):
+            if not comments:
+                return list()
+            comments_list = sorted(sorted(list(comments), key=lambda t: int(t['id'])),
+                                   key=lambda t: int(t['created_at']), reverse=True)
+            unique_comments_list = [comments_list[0]]
+            for x, y in zip(comments_list[:-1], comments_list[1:]):
+                if x['id'] != y['id']:
+                    unique_comments_list.append(y)
+                elif combine_answers:
+                    combined_answers = unique_comments_list[-1].get('answers') or list()
+                    if 'answers' in y:
+                        combined_answers.extend(y['answers'])
+                    unique_comments_list[-1]['answers'] = get_unique_comments(combined_answers)
+            return unique_comments_list
         filename += '_comments.json'
         try:
             with open(filename) as fp:
@@ -195,18 +285,10 @@ class Instaloader:
             comments = list()
         comments.extend(_postcomment_asdict(comment) for comment in post.get_comments())
         if comments:
-            comments_list = sorted(sorted(list(comments), key=lambda t: int(t['id'])),
-                                   key=lambda t: int(t['created_at']), reverse=True)
-            unique_comments_list = [comments_list[0]]
-            #for comment in comments_list:
-            #    if unique_comments_list[-1]['id'] != comment['id']:
-            #        unique_comments_list.append(comment)
-            #file.write(json.dumps(unique_comments_list, indent=4))
-            for x, y in zip(comments_list[:-1], comments_list[1:]):
-                if x['id'] != y['id']:
-                    unique_comments_list.append(y)
+            comments = get_unique_comments(comments, combine_answers=True)
+            answer_ids = set(int(answer['id']) for comment in comments for answer in comment.get('answers', []))
             with open(filename, 'w') as file:
-                file.write(json.dumps(unique_comments_list, indent=4))
+                file.write(json.dumps(list(filter(lambda t: int(t['id']) not in answer_ids, comments)), indent=4))
             self.context.log('comments', end=' ', flush=True)
 
     def save_caption(self, filename: str, mtime: datetime, caption: str) -> None:
@@ -217,11 +299,11 @@ class Instaloader:
         filename += '.txt'
         caption += '\n'
         pcaption = _elliptify(caption)
-        caption = caption.encode("UTF-8")
+        bcaption = caption.encode("UTF-8")
         with suppress(FileNotFoundError):
             with open(filename, 'rb') as file:
                 file_caption = file.read()
-            if file_caption.replace(b'\r\n', b'\n') == caption.replace(b'\r\n', b'\n'):
+            if file_caption.replace(b'\r\n', b'\n') == bcaption.replace(b'\r\n', b'\n'):
                 try:
                     self.context.log(pcaption + ' unchanged', end=' ', flush=True)
                 except UnicodeEncodeError:
@@ -229,8 +311,7 @@ class Instaloader:
                 return None
             else:
                 def get_filename(index):
-                    return filename if index == 0 else (filename[:-4] + '_old_' +
-                                                        (str(0) if index < 10 else str()) + str(index) + filename[-4:])
+                    return filename if index == 0 else '{0}_old_{2:02}{1}'.format(*os.path.splitext(filename), index)
 
                 i = 0
                 while os.path.isfile(get_filename(i)):
@@ -246,7 +327,7 @@ class Instaloader:
         except UnicodeEncodeError:
             self.context.log('txt', end=' ', flush=True)
         with open(filename, 'wb') as text_file:
-            shutil.copyfileobj(BytesIO(caption), text_file)
+            shutil.copyfileobj(BytesIO(bcaption), text_file)
         os.utime(filename, (datetime.now().timestamp(), mtime.timestamp()))
 
     def save_location(self, filename: str, location: PostLocation, mtime: datetime) -> None:
@@ -260,29 +341,39 @@ class Instaloader:
         os.utime(filename, (datetime.now().timestamp(), mtime.timestamp()))
         self.context.log('geo', end=' ', flush=True)
 
-    def download_profilepic(self, profile: Profile) -> None:
+    @_retry_on_connection_error
+    def download_profilepic(self, profile: Profile, _attempt: int = 1) -> None:
         """Downloads and saves profile pic."""
 
         def _epoch_to_string(epoch: datetime) -> str:
-            return epoch.strftime('%Y-%m-%d_%H-%M-%S')
+            return epoch.strftime('%Y-%m-%d_%H-%M-%S_UTC')
 
-        profile_pic_url = profile.profile_pic_url
-        with self.context.get_anonymous_session() as anonymous_session:
-            date_object = datetime.strptime(anonymous_session.head(profile_pic_url).headers["Last-Modified"],
-                                            '%a, %d %b %Y %H:%M:%S GMT')
+        profile_pic_response = self.context.get_raw(profile.profile_pic_url)
+        date_object = None  # type: Optional[datetime]
+        if 'Last-Modified' in profile_pic_response.headers:
+            date_object = datetime.strptime(profile_pic_response.headers["Last-Modified"], '%a, %d %b %Y %H:%M:%S GMT')
+            profile_pic_bytes = None
+            profile_pic_identifier = _epoch_to_string(date_object)
+        else:
+            profile_pic_bytes = profile_pic_response.content
+            profile_pic_identifier = md5(profile_pic_bytes).hexdigest()[:16]
+        profile_pic_extension = 'jpg'
         if ((format_string_contains_key(self.dirname_pattern, 'profile') or
              format_string_contains_key(self.dirname_pattern, 'target'))):
-            filename = '{0}/{1}_UTC_profile_pic.{2}'.format(self.dirname_pattern.format(profile=profile.username.lower(),
-                                                                                        target=profile.username.lower()),
-                                                            _epoch_to_string(date_object), profile_pic_url[-3:])
+            filename = '{0}/{1}_profile_pic.{2}'.format(self.dirname_pattern.format(profile=profile.username.lower(),
+                                                                                    target=profile.username.lower()),
+                                                        profile_pic_identifier, profile_pic_extension)
         else:
-            filename = '{0}/{1}_{2}_UTC_profile_pic.{3}'.format(self.dirname_pattern.format(), profile.username.lower(),
-                                                                _epoch_to_string(date_object), profile_pic_url[-3:])
-        if os.path.isfile(filename):
+            filename = '{0}/{1}_{2}_profile_pic.{3}'.format(self.dirname_pattern.format(), profile.username.lower(),
+                                                            profile_pic_identifier, profile_pic_extension)
+        content_length = profile_pic_response.headers.get('Content-Length', None)
+        if os.path.isfile(filename) and (not self.context.is_logged_in or
+                                         content_length is not None and os.path.getsize(filename) >= int(content_length)):
             self.context.log(filename + ' already exists')
             return None
-        self.context.get_and_write_raw(profile_pic_url, filename)
-        os.utime(filename, (datetime.now().timestamp(), date_object.timestamp()))
+        self.context.write_raw(profile_pic_bytes if profile_pic_bytes else profile_pic_response, filename)
+        if date_object:
+            os.utime(filename, (datetime.now().timestamp(), date_object.timestamp()))
         self.context.log('')  # log output of _get_and_write_raw() does not produce \n
 
     @_requires_login
@@ -292,6 +383,7 @@ class Instaloader:
         :param filename: Filename, or None to use default filename.
         """
         if filename is None:
+            assert self.context.username is not None
             filename = get_default_session_filename(self.context.username)
         dirname = os.path.dirname(filename)
         if dirname != '' and not os.path.exists(dirname):
@@ -324,9 +416,26 @@ class Instaloader:
 
         :raises InvalidArgumentException: If the provided username does not exist.
         :raises BadCredentialsException: If the provided password is wrong.
-        :raises ConnectionException: If connection to Instagram failed."""
+        :raises ConnectionException: If connection to Instagram failed.
+        :raises TwoFactorAuthRequiredException: First step of 2FA login done, now call :meth:`Instaloader.two_factor_login`."""
         self.context.login(user, passwd)
 
+    def two_factor_login(self, two_factor_code) -> None:
+        """Second step of login if 2FA is enabled.
+        Not meant to be used directly, use :meth:`Instaloader.two_factor_login`.
+
+        :raises InvalidArgumentException: No two-factor authentication pending.
+        :raises BadCredentialsException: 2FA verification code invalid.
+
+        .. versionadded:: 4.2"""
+        self.context.two_factor_login(two_factor_code)
+
+    def format_filename(self, item: Union[Post, StoryItem], target: Optional[str] = None):
+        """Format filename of a :class:`Post` or :class:`StoryItem` according to ``filename-pattern`` parameter.
+
+        .. versionadded:: 4.1"""
+        return _PostPathFormatter(item).format(self.filename_pattern, target=target)
+
     def download_post(self, post: Post, target: str) -> bool:
         """
         Download everything associated with one instagram post node, i.e. picture, caption and video.
@@ -337,30 +446,32 @@ class Instaloader:
         """
 
         dirname = _PostPathFormatter(post).format(self.dirname_pattern, target=target)
-        filename = dirname + '/' + _PostPathFormatter(post).format(self.filename_pattern, target=target)
+        filename = dirname + '/' + self.format_filename(post, target=target)
         os.makedirs(os.path.dirname(filename), exist_ok=True)
 
         # Download the image(s) / video thumbnail and videos within sidecars if desired
-        downloaded = False
-        if post.typename == 'GraphSidecar':
-            edge_number = 1
-            for sidecar_node in post.get_sidecar_nodes():
-                # Download picture or video thumbnail
-                if not sidecar_node.is_video or self.download_video_thumbnails is True:
-                    downloaded |= self.download_pic(filename=filename, url=sidecar_node.display_url,
-                                                    mtime=post.date_local, filename_suffix=str(edge_number))
-                # Additionally download video if available and desired
-                if sidecar_node.is_video and self.download_videos is True:
-                    downloaded |= self.download_pic(filename=filename, url=sidecar_node.video_url,
-                                                    mtime=post.date_local, filename_suffix=str(edge_number))
-                edge_number += 1
-        elif post.typename == 'GraphImage':
-            downloaded = self.download_pic(filename=filename, url=post.url, mtime=post.date_local)
-        elif post.typename == 'GraphVideo':
-            if self.download_video_thumbnails is True:
+        downloaded = True
+        self._committed = self.check_if_committed(filename)
+        if self.download_pictures:
+            if post.typename == 'GraphSidecar':
+                edge_number = 1
+                for sidecar_node in post.get_sidecar_nodes():
+                    # Download picture or video thumbnail
+                    if not sidecar_node.is_video or self.download_video_thumbnails is True:
+                        downloaded &= self.download_pic(filename=filename, url=sidecar_node.display_url,
+                                                        mtime=post.date_local, filename_suffix=str(edge_number))
+                    # Additionally download video if available and desired
+                    if sidecar_node.is_video and self.download_videos is True:
+                        downloaded &= self.download_pic(filename=filename, url=sidecar_node.video_url,
+                                                        mtime=post.date_local, filename_suffix=str(edge_number))
+                    edge_number += 1
+            elif post.typename == 'GraphImage':
                 downloaded = self.download_pic(filename=filename, url=post.url, mtime=post.date_local)
-        else:
-            self.context.error("Warning: {0} has unknown typename: {1}".format(post, post.typename))
+            elif post.typename == 'GraphVideo':
+                if self.download_video_thumbnails is True:
+                    downloaded = self.download_pic(filename=filename, url=post.url, mtime=post.date_local)
+            else:
+                self.context.error("Warning: {0} has unknown typename: {1}".format(post, post.typename))
 
         # Save caption if desired
         metadata_string = _ArbitraryItemFormatter(post).format(self.post_metadata_txt_pattern).strip()
@@ -369,7 +480,7 @@ class Instaloader:
 
         # Download video if desired
         if post.is_video and self.download_videos is True:
-            downloaded |= self.download_pic(filename=filename, url=post.video_url, mtime=post.date_local)
+            downloaded &= self.download_pic(filename=filename, url=post.video_url, mtime=post.date_local)
 
         # Download geotags if desired
         if self.download_geotags and post.location:
@@ -395,37 +506,46 @@ class Instaloader:
         :param userids: List of user IDs to be processed in terms of downloading their stories, or None.
         """
 
-        if userids is None:
+        if not userids:
             data = self.context.graphql_query("d15efd8c0c5b23f0ef71f18bf363c704",
                                               {"only_stories": True})["data"]["user"]
             if data is None:
                 raise BadResponseException('Bad stories reel JSON.')
             userids = list(edge["node"]["id"] for edge in data["feed_reels_tray"]["edge_reels_tray_to_reel"]["edges"])
 
-        stories = self.context.graphql_query("bf41e22b1c4ba4c9f31b844ebb7d9056",
-                                             {"reel_ids": userids, "precomposed_overlay": False})["data"]
+        def _userid_chunks():
+            assert userids is not None
+            userids_per_query = 100
+            for i in range(0, len(userids), userids_per_query):
+                yield userids[i:i + userids_per_query]
 
-        yield from (Story(self.context, media) for media in stories['reels_media'])
+        for userid_chunk in _userid_chunks():
+            stories = self.context.graphql_query("bf41e22b1c4ba4c9f31b844ebb7d9056",
+                                                 {"reel_ids": userid_chunk, "precomposed_overlay": False})["data"]
+            yield from (Story(self.context, media) for media in stories['reels_media'])
 
     @_requires_login
     def download_stories(self,
-                         userids: Optional[List[int]] = None,
+                         userids: Optional[List[Union[int, Profile]]] = None,
                          fast_update: bool = False,
-                         filename_target: str = ':stories',
+                         filename_target: Optional[str] = ':stories',
                          storyitem_filter: Optional[Callable[[StoryItem], bool]] = None) -> None:
         """
         Download available stories from user followees or all stories of users whose ID are given.
         Does not mark stories as seen.
         To use this, one needs to be logged in
 
-        :param userids: List of user IDs to be processed in terms of downloading their stories
+        :param userids: List of user IDs or Profiles to be processed in terms of downloading their stories
         :param fast_update: If true, abort when first already-downloaded picture is encountered
         :param filename_target: Replacement for {target} in dirname_pattern and filename_pattern
+               or None if profile name should be used instead
         :param storyitem_filter: function(storyitem), which returns True if given StoryItem should be downloaded
         """
 
         if not userids:
             self.context.log("Retrieving all visible stories...")
+        else:
+            userids = [p if isinstance(p, int) else p.userid for p in userids]
 
         for user_story in self.get_stories(userids):
             name = user_story.owner_username
@@ -439,7 +559,7 @@ class Instaloader:
                 self.context.log("[%3i/%3i] " % (count, totalcount), end="", flush=True)
                 count += 1
                 with self.context.error_catcher('Download story from user {}'.format(name)):
-                    downloaded = self.download_storyitem(item, filename_target)
+                    downloaded = self.download_storyitem(item, filename_target if filename_target else name)
                     if fast_update and not downloaded:
                         break
 
@@ -453,7 +573,7 @@ class Instaloader:
 
         date_local = item.date_local
         dirname = _PostPathFormatter(item).format(self.dirname_pattern, target=target)
-        filename = dirname + '/' + _PostPathFormatter(item).format(self.filename_pattern, target=target)
+        filename = dirname + '/' + self.format_filename(item, target=target)
         os.makedirs(os.path.dirname(filename), exist_ok=True)
         downloaded = False
         if not item.is_video or self.download_video_thumbnails is True:
@@ -471,6 +591,62 @@ class Instaloader:
         self.context.log()
         return downloaded
 
+    @_requires_login
+    def get_highlights(self, user: Union[int, Profile]) -> Iterator[Highlight]:
+        """Get all highlights from a user.
+        To use this, one needs to be logged in.
+
+        .. versionadded:: 4.1
+
+        :param user: ID or Profile of the user whose highlights should get fetched.
+        """
+
+        userid = user if isinstance(user, int) else user.userid
+        data = self.context.graphql_query("7c16654f22c819fb63d1183034a5162f",
+                                          {"user_id": userid, "include_chaining": False, "include_reel": False,
+                                           "include_suggested_users": False, "include_logged_out_extras": False,
+                                           "include_highlight_reels": True})["data"]["user"]['edge_highlight_reels']
+        if data is None:
+            raise BadResponseException('Bad highlights reel JSON.')
+        yield from (Highlight(self.context, edge['node'], user if isinstance(user, Profile) else None)
+                    for edge in data['edges'])
+
+    @_requires_login
+    def download_highlights(self,
+                            user: Union[int, Profile],
+                            fast_update: bool = False,
+                            filename_target: Optional[str] = None,
+                            storyitem_filter: Optional[Callable[[StoryItem], bool]] = None) -> None:
+        """
+        Download available highlights from a user whose ID is given.
+        To use this, one needs to be logged in.
+
+        .. versionadded:: 4.1
+
+        :param user: ID or Profile of the user whose highlights should get downloaded.
+        :param fast_update: If true, abort when first already-downloaded picture is encountered
+        :param filename_target: Replacement for {target} in dirname_pattern and filename_pattern
+               or None if profile name and the highlights' titles should be used instead
+        :param storyitem_filter: function(storyitem), which returns True if given StoryItem should be downloaded
+        """
+        for user_highlight in self.get_highlights(user):
+            name = user_highlight.owner_username
+            self.context.log("Retrieving highlights \"{}\" from profile {}".format(user_highlight.title, name))
+            totalcount = user_highlight.itemcount
+            count = 1
+            for item in user_highlight.get_items():
+                if storyitem_filter is not None and not storyitem_filter(item):
+                    self.context.log("<{} skipped>".format(item), flush=True)
+                    continue
+                self.context.log("[%3i/%3i] " % (count, totalcount), end="", flush=True)
+                count += 1
+                with self.context.error_catcher('Download highlights \"{}\" from user {}'.format(user_highlight.title, name)):
+                    downloaded = self.download_storyitem(item, filename_target
+                                                         if filename_target
+                                                         else '{}/{}'.format(name, user_highlight.title))
+                    if fast_update and not downloaded:
+                        break
+
     @_requires_login
     def get_feed_posts(self) -> Iterator[Post]:
         """Get Posts of the user's feed.
@@ -537,6 +713,7 @@ class Instaloader:
         """
         self.context.log("Retrieving saved posts...")
         count = 1
+        assert self.context.username is not None
         for post in Profile.from_username(self.context, self.context.username).get_saved_posts():
             if max_count is not None and count > max_count:
                 break
@@ -550,6 +727,59 @@ class Instaloader:
                 if fast_update and not downloaded:
                     break
 
+    def get_location_posts(self, location: str) -> Iterator[Post]:
+        """Get Posts which are listed by Instagram for a given Location.
+
+        :return:  Iterator over Posts of a location's posts
+
+        .. versionadded:: 4.2
+        """
+        has_next_page = True
+        end_cursor = None
+        while has_next_page:
+            if end_cursor:
+                params = {'__a': 1, 'max_id': end_cursor}
+            else:
+                params = {'__a': 1}
+            location_data = self.context.get_json('explore/locations/{0}/'.format(location),
+                                                  params)['graphql']['location']['edge_location_to_media']
+            yield from (Post(self.context, edge['node']) for edge in location_data['edges'])
+            has_next_page = location_data['page_info']['has_next_page']
+            end_cursor = location_data['page_info']['end_cursor']
+
+    def download_location(self, location: str,
+                          max_count: Optional[int] = None,
+                          post_filter: Optional[Callable[[Post], bool]] = None,
+                          fast_update: bool = False) -> None:
+        """Download pictures of one location.
+
+        To download the last 30 pictures with location 362629379, do::
+
+            loader = Instaloader()
+            loader.download_location(362629379, max_count=30)
+
+        :param location: Location to download, as Instagram numerical ID
+        :param max_count: Maximum count of pictures to download
+        :param post_filter: function(post), which returns True if given picture should be downloaded
+        :param fast_update: If true, abort when first already-downloaded picture is encountered
+
+        .. versionadded:: 4.2
+        """
+        self.context.log("Retrieving pictures for location {}...".format(location))
+        count = 1
+        for post in self.get_location_posts(location):
+            if max_count is not None and count > max_count:
+                break
+            self.context.log('[{0:3d}] %{1} '.format(count, location), end='', flush=True)
+            if post_filter is not None and not post_filter(post):
+                self.context.log('<skipped>')
+                continue
+            count += 1
+            with self.context.error_catcher('Download location {}'.format(location)):
+                downloaded = self.download_post(post, target='%' + location)
+                if fast_update and not downloaded:
+                    break
+
     @_requires_login
     def get_explore_posts(self) -> Iterator[Post]:
         """Get Posts which are worthy of exploring suggested by Instagram.
@@ -610,6 +840,46 @@ class Instaloader:
                 if fast_update and not downloaded:
                     break
 
+    def download_tagged(self, profile: Profile, fast_update: bool = False,
+                        target: Optional[str] = None,
+                        post_filter: Optional[Callable[[Post], bool]] = None) -> None:
+        """Download all posts where a profile is tagged.
+
+        .. versionadded:: 4.1"""
+        if target is None:
+            target = profile.username + '/:tagged'
+        self.context.log("Retrieving tagged posts for profile {}.".format(profile.username))
+        count = 1
+        for post in profile.get_tagged_posts():
+            self.context.log("[%3i/???] " % (count), end="", flush=True)
+            count += 1
+            if post_filter is not None and not post_filter(post):
+                self.context.log('<{} skipped>'.format(post))
+            with self.context.error_catcher('Download tagged {}'.format(profile.username)):
+                downloaded = self.download_post(post, target)
+                if fast_update and not downloaded:
+                    break
+
+    def _get_id_filename(self, profile_name: str) -> str:
+        if ((format_string_contains_key(self.dirname_pattern, 'profile') or
+             format_string_contains_key(self.dirname_pattern, 'target'))):
+            return '{0}/id'.format(self.dirname_pattern.format(profile=profile_name.lower(),
+                                                               target=profile_name.lower()))
+        else:
+            return '{0}/{1}_id'.format(self.dirname_pattern.format(), profile_name.lower())
+
+    def save_profile_id(self, profile: Profile):
+        """
+        Store ID of profile locally.
+
+        .. versionadded:: 4.0.6
+        """
+        os.makedirs(self.dirname_pattern.format(profile=profile.username,
+                                                target=profile.username), exist_ok=True)
+        with open(self._get_id_filename(profile.username), 'w') as text_file:
+            text_file.write(str(profile.userid) + "\n")
+            self.context.log("Stored ID {0} for profile {1}.".format(profile.userid, profile.username))
+
     def check_profile_id(self, profile_name: str) -> Profile:
         """
         Consult locally stored ID of profile with given name, check whether ID matches and whether name
@@ -622,12 +892,7 @@ class Instaloader:
         with suppress(ProfileNotExistsException):
             profile = Profile.from_username(self.context, profile_name)
         profile_exists = profile is not None
-        if ((format_string_contains_key(self.dirname_pattern, 'profile') or
-             format_string_contains_key(self.dirname_pattern, 'target'))):
-            id_filename = '{0}/id'.format(self.dirname_pattern.format(profile=profile_name.lower(),
-                                                                      target=profile_name.lower()))
-        else:
-            id_filename = '{0}/{1}_id'.format(self.dirname_pattern.format(), profile_name.lower())
+        id_filename = self._get_id_filename(profile_name)
         try:
             with open(id_filename, 'rb') as id_file:
                 profile_id = int(id_file.read())
@@ -656,26 +921,130 @@ class Instaloader:
         except (FileNotFoundError, ValueError):
             pass
         if profile_exists:
-            os.makedirs(self.dirname_pattern.format(profile=profile_name.lower(),
-                                                    target=profile_name.lower()), exist_ok=True)
-            with open(id_filename, 'w') as text_file:
-                text_file.write(str(profile.userid) + "\n")
-                self.context.log("Stored ID {0} for profile {1}.".format(profile.userid, profile_name))
+            self.save_profile_id(profile)
             return profile
         raise ProfileNotExistsException("Profile {0} does not exist.".format(profile_name))
 
-    def download_profile(self, profile_name: str,
+    def download_profiles(self, profiles: Set[Profile],
+                          profile_pic: bool = True, posts: bool = True,
+                          tagged: bool = False, highlights: bool = False, stories: bool = False,
+                          fast_update: bool = False,
+                          post_filter: Optional[Callable[[Post], bool]] = None,
+                          storyitem_filter: Optional[Callable[[Post], bool]] = None,
+                          raise_errors: bool = False):
+        """High-level method to download set of profiles.
+
+        :param profiles: Set of profiles to download.
+        :param profile_pic: not :option:`--no-profile-pic`.
+        :param posts: not :option:`--no-posts`.
+        :param tagged: :option:`--tagged`.
+        :param highlights: :option:`--highlights`.
+        :param stories: :option:`--stories`.
+        :param fast_update: :option:`--fast-update`.
+        :param post_filter: :option:`--post-filter`.
+        :param storyitem_filter: :option:`--post-filter`.
+        :param raise_errors:
+           Whether :exc:`LoginRequiredException` and :exc:`PrivateProfileNotFollowedException` should be raised or
+           catched and printed with :meth:`InstaloaderContext.error_catcher`.
+
+        .. versionadded:: 4.1"""
+
+        @contextmanager
+        def _error_raiser(_str):
+            yield
+
+        error_handler = cast(Callable[[Optional[str]], ContextManager[None]],
+                             _error_raiser if raise_errors else self.context.error_catcher)
+
+        for profile in profiles:
+            with error_handler(profile.username):
+                profile_name = profile.username
+
+                # Download profile picture
+                if profile_pic:
+                    with self.context.error_catcher('Download profile picture of {}'.format(profile_name)):
+                        self.download_profilepic(profile)
+
+                # Save metadata as JSON if desired.
+                if self.save_metadata:
+                    json_filename = '{0}/{1}_{2}'.format(self.dirname_pattern.format(profile=profile_name,
+                                                                                     target=profile_name),
+                                                         profile_name, profile.userid)
+                    self.save_metadata_json(json_filename, profile)
+
+                # Catch some errors
+                if profile.is_private and (tagged or highlights or posts):
+                    if not self.context.is_logged_in:
+                        raise LoginRequiredException("--login=USERNAME required.")
+                    if not profile.followed_by_viewer and self.context.username != profile.username:
+                        raise PrivateProfileNotFollowedException("Private but not followed.")
+
+                # Download tagged, if requested
+                if tagged:
+                    with self.context.error_catcher('Download tagged of {}'.format(profile_name)):
+                        self.download_tagged(profile, fast_update=fast_update, post_filter=post_filter)
+
+                # Download highlights, if requested
+                if highlights:
+                    with self.context.error_catcher('Download highlights of {}'.format(profile_name)):
+                        self.download_highlights(profile, fast_update=fast_update, storyitem_filter=storyitem_filter)
+
+                # Iterate over pictures and download them
+                if posts:
+                    self.context.log("Retrieving posts from profile {}.".format(profile_name))
+                    totalcount = profile.mediacount
+                    count = 1
+                    for post in profile.get_posts():
+                        self.context.log("[%3i/%3i] " % (count, totalcount), end="", flush=True)
+                        count += 1
+                        if post_filter is not None and not post_filter(post):
+                            self.context.log('<skipped>')
+                            continue
+                        with self.context.error_catcher("Download {} of {}".format(post, profile_name)):
+                            # The PostChangedException gets raised if the Post's id/shortcode changed while obtaining
+                            # additional metadata. This is most likely the case if a HTTP redirect takes place while
+                            # resolving the shortcode URL.
+                            # The `post_changed` variable keeps the fast-update functionality alive: A Post which is
+                            # obained after a redirect has probably already been downloaded as a previous Post of the
+                            # same Profile.
+                            # Observed in issue #225: https://github.com/instaloader/instaloader/issues/225
+                            post_changed = False
+                            while True:
+                                try:
+                                    downloaded = self.download_post(post, target=profile_name)
+                                    break
+                                except PostChangedException:
+                                    post_changed = True
+                                    continue
+                            if fast_update and not downloaded and not post_changed:
+                                break
+
+        if stories and profiles:
+            with self.context.error_catcher("Download stories"):
+                self.context.log("Downloading stories")
+                self.download_stories(userids=list(profiles), fast_update=fast_update, filename_target=None,
+                                      storyitem_filter=storyitem_filter)
+
+    def download_profile(self, profile_name: Union[str, Profile],
                          profile_pic: bool = True, profile_pic_only: bool = False,
                          fast_update: bool = False,
                          download_stories: bool = False, download_stories_only: bool = False,
+                         download_tagged: bool = False, download_tagged_only: bool = False,
                          post_filter: Optional[Callable[[Post], bool]] = None,
                          storyitem_filter: Optional[Callable[[StoryItem], bool]] = None) -> None:
-        """Download one profile"""
+        """Download one profile
+
+        .. deprecated:: 4.1
+           Use :meth:`Instaloader.download_profiles`.
+        """
 
         # Get profile main page json
         # check if profile does exist or name has changed since last download
         # and update name and json data if necessary
-        profile = self.check_profile_id(profile_name.lower())
+        if isinstance(profile_name, str):
+            profile = self.check_profile_id(profile_name.lower())
+        else:
+            profile = profile_name
 
         profile_name = profile.username
 
@@ -718,6 +1087,13 @@ class Instaloader:
         if download_stories_only:
             return
 
+        # Download tagged, if requested
+        if download_tagged or download_tagged_only:
+            with self.context.error_catcher('Download tagged of {}'.format(profile_name)):
+                self.download_tagged(profile, fast_update=fast_update, post_filter=post_filter)
+        if download_tagged_only:
+            return
+
         # Iterate over pictures and download them
         self.context.log("Retrieving posts from profile {}.".format(profile_name))
         totalcount = profile.mediacount
@@ -733,6 +1109,25 @@ class Instaloader:
                 if fast_update and not downloaded:
                     break
 
+    def check_if_committed(self, filename: str) -> bool:
+        """Checks to see if the current post has been committed.
+
+        A post is considered committed if its json metadata file exists and is not malformed.
+
+        .. versionadded:: 4.2
+        """
+        if os.path.isfile(filename + '.json.xz'):
+            filename += '.json.xz'
+        elif os.path.isfile(filename + '.json'):
+            filename += '.json'
+        else:
+            return False
+        try:
+            load_structure_from_file(self.context, filename)
+            return True
+        except (FileNotFoundError, lzma.LZMAError, json.decoder.JSONDecodeError):
+            return False
+
     def interactive_login(self, username: str) -> None:
         """Logs in and internally stores session, asking user for password interactively.
 
@@ -741,11 +1136,20 @@ class Instaloader:
         :raises ConnectionException: If connection to Instagram failed."""
         if self.context.quiet:
             raise LoginRequiredException("Quiet mode requires given password or valid session file.")
-        password = None
-        while password is None:
-            password = getpass.getpass(prompt="Enter Instagram password for %s: " % username)
-            try:
-                self.login(username, password)
-            except BadCredentialsException as err:
-                print(err, file=sys.stderr)
-                password = None
+        try:
+            password = None
+            while password is None:
+                password = getpass.getpass(prompt="Enter Instagram password for %s: " % username)
+                try:
+                    self.login(username, password)
+                except BadCredentialsException as err:
+                    print(err, file=sys.stderr)
+                    password = None
+        except TwoFactorAuthRequiredException:
+            while True:
+                try:
+                    code = input("Enter 2FA verification code: ")
+                    self.two_factor_login(code)
+                    break
+                except BadCredentialsException:
+                    pass

instaloader/instaloadercontext.py

@@ -9,11 +9,11 @@ import textwrap
 import time
 import urllib.parse
 from contextlib import contextmanager
-from typing import Any, Callable, Dict, Iterator, Optional
+from datetime import datetime, timedelta
+from typing import Any, Callable, Dict, Iterator, List, Optional, Union
 
 import requests
 import requests.utils
-import urllib3
 
 from .exceptions import *
 
@@ -22,7 +22,7 @@ def copy_session(session: requests.Session) -> requests.Session:
     """Duplicates a requests.Session."""
     new = requests.Session()
     new.cookies = requests.utils.cookiejar_from_dict(requests.utils.dict_from_cookiejar(session.cookies))
-    new.headers = session.headers.copy()
+    new.headers = session.headers.copy() # type: ignore
     return new
 
 
@@ -47,7 +47,7 @@ class InstaloaderContext:
     """
 
     def __init__(self, sleep: bool = True, quiet: bool = False, user_agent: Optional[str] = None,
-                 graphql_count_per_slidingwindow: Optional[int] = None, max_connection_attempts: int = 3):
+                 max_connection_attempts: int = 3):
 
         self.user_agent = user_agent if user_agent is not None else default_user_agent()
         self._session = self.get_anonymous_session()
@@ -56,28 +56,34 @@ class InstaloaderContext:
         self.quiet = quiet
         self.max_connection_attempts = max_connection_attempts
         self._graphql_page_length = 50
-        self.graphql_count_per_slidingwindow = graphql_count_per_slidingwindow or 20
         self._root_rhx_gis = None
+        self.two_factor_auth_pending = None
 
         # error log, filled with error() and printed at the end of Instaloader.main()
-        self.error_log = []
+        self.error_log = []                      # type: List[str]
 
         # For the adaption of sleep intervals (rate control)
-        self.query_timestamps = list()
+        self._graphql_query_timestamps = dict()  # type: Dict[str, List[float]]
+        self._graphql_earliest_next_request_time = 0.0
 
         # Can be set to True for testing, disables supression of InstaloaderContext._error_catcher
         self.raise_all_errors = False
 
+        # Cache profile from id (mapping from id to Profile)
+        self.profile_id_cache = dict()           # type: Dict[int, Any]
+
     @contextmanager
     def anonymous_copy(self):
         session = self._session
         username = self.username
         self._session = self.get_anonymous_session()
         self.username = None
-        yield self
-        self._session.close()
-        self.username = username
-        self._session = session
+        try:
+            yield self
+        finally:
+            self._session.close()
+            self.username = username
+            self._session = session
 
     @property
     def is_logged_in(self) -> bool:
@@ -174,20 +180,39 @@ class InstaloaderContext:
 
         :raises InvalidArgumentException: If the provided username does not exist.
         :raises BadCredentialsException: If the provided password is wrong.
-        :raises ConnectionException: If connection to Instagram failed."""
+        :raises ConnectionException: If connection to Instagram failed.
+        :raises TwoFactorAuthRequiredException: First step of 2FA login done, now call :meth:`Instaloader.two_factor_login`."""
+        import http.client
+        # pylint:disable=protected-access
+        http.client._MAXHEADERS = 200
         session = requests.Session()
         session.cookies.update({'sessionid': '', 'mid': '', 'ig_pr': '1',
-                                'ig_vw': '1920', 'csrftoken': '',
+                                'ig_vw': '1920', 'ig_cb': '1', 'csrftoken': '',
                                 's_network': '', 'ds_user_id': ''})
         session.headers.update(self._default_http_header())
-        session.headers.update({'X-CSRFToken': self.get_json('', {})['config']['csrf_token']})
+        session.get('https://www.instagram.com/web/__mid/')
+        csrf_token = session.cookies.get_dict()['csrftoken']
+        session.headers.update({'X-CSRFToken': csrf_token})
         # Not using self.get_json() here, because we need to access csrftoken cookie
-        self._sleep()
+        self.do_sleep()
         login = session.post('https://www.instagram.com/accounts/login/ajax/',
                              data={'password': passwd, 'username': user}, allow_redirects=True)
-        if login.status_code != 200:
-            raise ConnectionException("Login error: {} {}".format(login.status_code, login.reason))
-        resp_json = login.json()
+        try:
+            resp_json = login.json()
+        except json.decoder.JSONDecodeError:
+            raise ConnectionException("Login error: JSON decode fail, {} - {}.".format(login.status_code, login.reason))
+        if resp_json.get('two_factor_required'):
+            two_factor_session = copy_session(session)
+            two_factor_session.headers.update({'X-CSRFToken': csrf_token})
+            two_factor_session.cookies.update({'csrftoken': csrf_token})
+            self.two_factor_auth_pending = (two_factor_session,
+                                            user,
+                                            resp_json['two_factor_info']['two_factor_identifier'])
+            raise TwoFactorAuthRequiredException("Login error: two-factor authentication required.")
+        if resp_json.get('checkpoint_url'):
+            raise ConnectionException("Login: Checkpoint required. Point your browser to "
+                                      "https://www.instagram.com{}, "
+                                      "follow the instructions, then retry.".format(resp_json.get('checkpoint_url')))
         if resp_json['status'] != 'ok':
             if 'message' in resp_json:
                 raise ConnectionException("Login error: \"{}\" status, message \"{}\".".format(resp_json['status'],
@@ -209,10 +234,102 @@ class InstaloaderContext:
         self._session = session
         self.username = user
 
-    def _sleep(self):
+    def two_factor_login(self, two_factor_code):
+        """Second step of login if 2FA is enabled.
+        Not meant to be used directly, use :meth:`Instaloader.two_factor_login`.
+
+        :raises InvalidArgumentException: No two-factor authentication pending.
+        :raises BadCredentialsException: 2FA verification code invalid.
+
+        .. versionadded:: 4.2"""
+        if not self.two_factor_auth_pending:
+            raise InvalidArgumentException("No two-factor authentication pending.")
+        (session, user, two_factor_id) = self.two_factor_auth_pending
+
+        login = session.post('https://www.instagram.com/accounts/login/ajax/two_factor/',
+                             data={'username': user, 'verificationCode': two_factor_code, 'identifier': two_factor_id},
+                             allow_redirects=True)
+        resp_json = login.json()
+        if resp_json['status'] != 'ok':
+            if 'message' in resp_json:
+                raise BadCredentialsException("Login error: {}".format(resp_json['message']))
+            else:
+                raise BadCredentialsException("Login error: \"{}\" status.".format(resp_json['status']))
+        session.headers.update({'X-CSRFToken': login.cookies['csrftoken']})
+        self._session = session
+        self.username = user
+        self.two_factor_auth_pending = None
+
+    def do_sleep(self):
         """Sleep a short time if self.sleep is set. Called before each request to instagram.com."""
         if self.sleep:
-            time.sleep(min(random.expovariate(0.6), 5.0))
+            time.sleep(min(random.expovariate(0.7), 5.0))
+
+    def _dump_query_timestamps(self, current_time: float):
+        """Output the number of GraphQL queries grouped by their query_hash within the last time."""
+        windows = [10, 11, 15, 20, 30, 60]
+        print("GraphQL requests:", file=sys.stderr)
+        for query_hash, times in self._graphql_query_timestamps.items():
+            print("  {}".format(query_hash), file=sys.stderr)
+            for window in windows:
+                reqs_in_sliding_window = sum(t > current_time - window * 60 for t in times)
+                print("    last {} minutes: {} requests".format(window, reqs_in_sliding_window), file=sys.stderr)
+
+    def _graphql_request_count_per_sliding_window(self, query_hash: str) -> int:
+        """Return how many GraphQL requests can be done within the sliding window."""
+        if self.is_logged_in:
+            max_reqs = {'1cb6ec562846122743b61e492c85999f': 20, '33ba35852cb50da46f5b5e889df7d159': 20}
+        else:
+            max_reqs = {'1cb6ec562846122743b61e492c85999f': 200, '33ba35852cb50da46f5b5e889df7d159': 200}
+        return max_reqs.get(query_hash) or min(max_reqs.values())
+
+    def _graphql_query_waittime(self, query_hash: str, current_time: float, untracked_queries: bool = False) -> float:
+        """Calculate time needed to wait before GraphQL query can be executed."""
+        sliding_window = 660
+        if query_hash not in self._graphql_query_timestamps:
+            self._graphql_query_timestamps[query_hash] = []
+        self._graphql_query_timestamps[query_hash] = list(filter(lambda t: t > current_time - 60 * 60,
+                                                                 self._graphql_query_timestamps[query_hash]))
+        reqs_in_sliding_window = list(filter(lambda t: t > current_time - sliding_window,
+                                             self._graphql_query_timestamps[query_hash]))
+        count_per_sliding_window = self._graphql_request_count_per_sliding_window(query_hash)
+        if len(reqs_in_sliding_window) < count_per_sliding_window and not untracked_queries:
+            return max(0, self._graphql_earliest_next_request_time - current_time)
+        next_request_time = min(reqs_in_sliding_window) + sliding_window + 6
+        if untracked_queries:
+            self._graphql_earliest_next_request_time = next_request_time
+        return round(max(next_request_time, self._graphql_earliest_next_request_time) - current_time)
+
+    def _ratecontrol_graphql_query(self, query_hash: str, untracked_queries: bool = False):
+        """Called before a GraphQL query is made in order to stay within Instagram's rate limits.
+
+        :param query_hash: The query_hash parameter of the query.
+        :param untracked_queries: True, if 429 has been returned to apply 429 logic.
+        """
+        if not untracked_queries:
+            waittime = self._graphql_query_waittime(query_hash, time.monotonic(), untracked_queries)
+            assert waittime >= 0
+            if waittime > 10:
+                self.log('\nToo many queries in the last time. Need to wait {} seconds, until {:%H:%M}.'
+                         .format(waittime, datetime.now() + timedelta(seconds=waittime)))
+            time.sleep(waittime)
+            if query_hash not in self._graphql_query_timestamps:
+                self._graphql_query_timestamps[query_hash] = [time.monotonic()]
+            else:
+                self._graphql_query_timestamps[query_hash].append(time.monotonic())
+        else:
+            text_for_429 = ("HTTP error code 429 was returned because too many queries occured in the last time. "
+                            "Please do not use Instagram in your browser or run multiple instances of Instaloader "
+                            "in parallel.")
+            print(textwrap.fill(text_for_429), file=sys.stderr)
+            current_time = time.monotonic()
+            waittime = self._graphql_query_waittime(query_hash, current_time, untracked_queries)
+            assert waittime >= 0
+            if waittime > 10:
+                self.log('The request will be retried in {} seconds, at {:%H:%M}.'
+                         .format(waittime, datetime.now() + timedelta(seconds=waittime)))
+            self._dump_query_timestamps(current_time)
+            time.sleep(waittime)
 
     def get_json(self, path: str, params: Dict[str, Any], host: str = 'www.instagram.com',
                  session: Optional[requests.Session] = None, _attempt=1) -> Dict[str, Any]:
@@ -227,31 +344,12 @@ class InstaloaderContext:
         :raises QueryReturnedNotFoundException: When the server responds with a 404.
         :raises ConnectionException: When query repeatedly failed.
         """
-        def graphql_query_waittime(untracked_queries: bool = False) -> int:
-            sliding_window = 660
-            if not self.query_timestamps:
-                return sliding_window if untracked_queries else 0
-            current_time = time.monotonic()
-            self.query_timestamps = list(filter(lambda t: t > current_time - sliding_window, self.query_timestamps))
-            if len(self.query_timestamps) < self.graphql_count_per_slidingwindow and not untracked_queries:
-                return 0
-            return round(min(self.query_timestamps) + sliding_window - current_time) + 6
         is_graphql_query = 'query_hash' in params and 'graphql/query' in path
-        # some queries are not rate limited if invoked anonymously:
-        query_not_limited = is_graphql_query and not self.is_logged_in \
-                            and params['query_hash'] in ['9ca88e465c3f866a76f7adee3871bdd8']
-        if is_graphql_query and not query_not_limited:
-            waittime = graphql_query_waittime()
-            if waittime > 0:
-                self.log('\nToo many queries in the last time. Need to wait {} seconds.'.format(waittime))
-                time.sleep(waittime)
-            if self.query_timestamps is not None:
-                self.query_timestamps.append(time.monotonic())
-            else:
-                self.query_timestamps = [time.monotonic()]
         sess = session if session else self._session
         try:
-            self._sleep()
+            self.do_sleep()
+            if is_graphql_query:
+                self._ratecontrol_graphql_query(params['query_hash'])
             resp = sess.get('https://{0}/{1}'.format(host, path), params=params, allow_redirects=False)
             while resp.is_redirect:
                 redirect_url = resp.headers['location']
@@ -289,18 +387,9 @@ class InstaloaderContext:
             if _attempt == self.max_connection_attempts:
                 raise ConnectionException(error_string) from err
             self.error(error_string + " [retrying; skip with ^C]", repeat_at_end=False)
-            text_for_429 = ("HTTP error code 429 was returned because too many queries occured in the last time. "
-                            "Please do not use Instagram in your browser or run multiple instances of Instaloader "
-                            "in parallel.")
             try:
                 if isinstance(err, TooManyRequestsException):
-                    print(textwrap.fill(text_for_429), file=sys.stderr)
-                    if is_graphql_query:
-                        waittime = graphql_query_waittime(untracked_queries=True)
-                        if waittime > 0:
-                            self.log('The request will be retried in {} seconds.'.format(waittime))
-                            time.sleep(waittime)
-                self._sleep()
+                    self._ratecontrol_graphql_query(params['query_hash'], untracked_queries=True)
                 return self.get_json(path=path, params=params, host=host, session=sess, _attempt=_attempt + 1)
             except KeyboardInterrupt:
                 self.error("[skipped by user]", repeat_at_end=False)
@@ -374,39 +463,64 @@ class InstaloaderContext:
             data = _query()
             yield from (edge['node'] for edge in data['edges'])
 
-    def get_and_write_raw(self, url: str, filename: str, _attempt=1) -> None:
-        """Downloads raw data.
+    def get_iphone_json(self, path: str, params: Dict[str, Any]) -> Dict[str, Any]:
+        """JSON request to ``i.instagram.com``.
+
+        :param path: URL, relative to ``i.instagram.com/``
+        :param params: GET parameters
+        :return: Decoded response dictionary
+        :raises QueryReturnedBadRequestException: When the server responds with a 400.
+        :raises QueryReturnedNotFoundException: When the server responds with a 404.
+        :raises ConnectionException: When query repeatedly failed.
+
+        .. versionadded:: 4.2.1"""
+        with copy_session(self._session) as tempsession:
+            tempsession.headers['User-Agent'] = 'Instagram 10.3.2 (iPhone7,2; iPhone OS 9_3_3; en_US; en-US; ' \
+                                                'scale=2.00; 750x1334) AppleWebKit/420+'
+            for header in ['Host', 'Origin', 'X-Instagram-AJAX', 'X-Requested-With']:
+                tempsession.headers.pop(header, None)
+            return self.get_json(path, params, 'i.instagram.com', tempsession)
+
+    def write_raw(self, resp: Union[bytes, requests.Response], filename: str) -> None:
+        """Write raw response data into a file.
+
+        .. versionadded:: 4.2.1"""
+        self.log(filename, end=' ', flush=True)
+        with open(filename, 'wb') as file:
+            if isinstance(resp, requests.Response):
+                shutil.copyfileobj(resp.raw, file)
+            else:
+                file.write(resp)
+
+    def get_raw(self, url: str, _attempt=1) -> requests.Response:
+        """Downloads a file anonymously.
+
+        :raises QueryReturnedNotFoundException: When the server responds with a 404.
+        :raises QueryReturnedForbiddenException: When the server responds with a 403.
+        :raises ConnectionException: When download failed.
+
+        .. versionadded:: 4.2.1"""
+        with self.get_anonymous_session() as anonymous_session:
+            resp = anonymous_session.get(url, stream=True)
+        if resp.status_code == 200:
+            resp.raw.decode_content = True
+            return resp
+        else:
+            if resp.status_code == 403:
+                # suspected invalid URL signature
+                raise QueryReturnedForbiddenException("403 when accessing {}.".format(url))
+            if resp.status_code == 404:
+                # 404 not worth retrying.
+                raise QueryReturnedNotFoundException("404 when accessing {}.".format(url))
+            raise ConnectionException("HTTP error code {}.".format(resp.status_code))
+
+    def get_and_write_raw(self, url: str, filename: str) -> None:
+        """Downloads and writes anonymously-requested raw data into a file.
 
         :raises QueryReturnedNotFoundException: When the server responds with a 404.
         :raises QueryReturnedForbiddenException: When the server responds with a 403.
         :raises ConnectionException: When download repeatedly failed."""
-        try:
-            with self.get_anonymous_session() as anonymous_session:
-                resp = anonymous_session.get(url, stream=True)
-            if resp.status_code == 200:
-                self.log(filename, end=' ', flush=True)
-                with open(filename, 'wb') as file:
-                    resp.raw.decode_content = True
-                    shutil.copyfileobj(resp.raw, file)
-            else:
-                if resp.status_code == 403:
-                    # suspected invalid URL signature
-                    raise QueryReturnedForbiddenException("403 when accessing {}.".format(url))
-                if resp.status_code == 404:
-                    # 404 not worth retrying.
-                    raise QueryReturnedNotFoundException("404 when accessing {}.".format(url))
-                raise ConnectionException("HTTP error code {}.".format(resp.status_code))
-        except (urllib3.exceptions.HTTPError, requests.exceptions.RequestException, ConnectionException) as err:
-            error_string = "URL {}: {}".format(url, err)
-            if _attempt == self.max_connection_attempts:
-                raise ConnectionException(error_string) from err
-            self.error(error_string + " [retrying; skip with ^C]", repeat_at_end=False)
-            try:
-                self._sleep()
-                self.get_and_write_raw(url, filename, _attempt + 1)
-            except KeyboardInterrupt:
-                self.error("[skipped by user]", repeat_at_end=False)
-                raise ConnectionException(error_string) from err
+        self.write_raw(self.get_raw(url), filename)
 
     @property
     def root_rhx_gis(self) -> Optional[str]:

instaloader/structures.py

@@ -17,11 +17,16 @@ PostSidecarNode.is_video.__doc__ = "Whether this node is a video."
 PostSidecarNode.display_url.__doc__ = "URL of image or video thumbnail."
 PostSidecarNode.video_url.__doc__ = "URL of video or None."
 
-PostComment = namedtuple('PostComment', ['id', 'created_at_utc', 'text', 'owner'])
-PostComment.id.__doc__ = "ID number of comment."
-PostComment.created_at_utc.__doc__ = ":class:`~datetime.datetime` when comment was created (UTC)."
-PostComment.text.__doc__ = "Comment text."
-PostComment.owner.__doc__ = "Owner :class:`Profile` of the comment."
+PostCommentAnswer = namedtuple('PostCommentAnswer', ['id', 'created_at_utc', 'text', 'owner'])
+PostCommentAnswer.id.__doc__ = "ID number of comment."
+PostCommentAnswer.created_at_utc.__doc__ = ":class:`~datetime.datetime` when comment was created (UTC)."
+PostCommentAnswer.text.__doc__ = "Comment text."
+PostCommentAnswer.owner.__doc__ = "Owner :class:`Profile` of the comment."
+
+PostComment = namedtuple('PostComment', (*PostCommentAnswer._fields, 'answers')) # type: ignore
+for field in PostCommentAnswer._fields:
+    getattr(PostComment, field).__doc__ = getattr(PostCommentAnswer, field).__doc__
+PostComment.answers.__doc__ = r"Iterator which yields all :class:`PostCommentAnswer`\ s for the comment." # type: ignore
 
 PostLocation = namedtuple('PostLocation', ['id', 'name', 'slug', 'has_public_page', 'lat', 'lng'])
 PostLocation.id.__doc__ = "ID number of location."
@@ -62,9 +67,9 @@ class Post:
         self._context = context
         self._node = node
         self._owner_profile = owner_profile
-        self._full_metadata_dict = None
-        self._rhx_gis_str = None
-        self._location = None
+        self._full_metadata_dict = None  # type: Optional[Dict[str, Any]]
+        self._rhx_gis_str = None         # type: Optional[str]
+        self._location = None            # type: Optional[PostLocation]
 
     @classmethod
     def from_shortcode(cls, context: InstaloaderContext, shortcode: str):
@@ -93,10 +98,9 @@ class Post:
         return b64encode(mediaid.to_bytes(9, 'big'), b'-_').decode().replace('A', ' ').lstrip().replace(' ', 'A')
 
     def _asdict(self):
+        node = self._node
         if self._full_metadata_dict:
-            node = self._full_metadata_dict
-        else:
-            node = self._node
+            node.update(self._full_metadata_dict)
         if self._owner_profile:
             node['owner'] = self.owner_profile._asdict()
         if self._location:
@@ -129,15 +133,20 @@ class Post:
             pic_json = self._context.get_json("p/{0}/".format(self.shortcode), params={})
             self._full_metadata_dict = pic_json['entry_data']['PostPage'][0]['graphql']['shortcode_media']
             self._rhx_gis_str = pic_json['rhx_gis']
+            if self.shortcode != self._full_metadata_dict['shortcode']:
+                self._node.update(self._full_metadata_dict)
+                raise PostChangedException
 
     @property
     def _full_metadata(self) -> Dict[str, Any]:
         self._obtain_metadata()
+        assert self._full_metadata_dict is not None
         return self._full_metadata_dict
 
     @property
     def _rhx_gis(self) -> str:
         self._obtain_metadata()
+        assert self._rhx_gis_str is not None
         return self._rhx_gis_str
 
     def _field(self, *keys) -> Any:
@@ -224,6 +233,7 @@ class Post:
             return self._node["edge_media_to_caption"]["edges"][0]["node"]["text"]
         elif "caption" in self._node:
             return self._node["caption"]
+        return None
 
     @property
     def caption_hashtags(self) -> List[str]:
@@ -245,6 +255,14 @@ class Post:
         mention_regex = re.compile(r"(?:@)(\w(?:(?:\w|(?:\.(?!\.))){0,28}(?:\w))?)")
         return re.findall(mention_regex, self.caption.lower())
 
+    @property
+    def pcaption(self) -> str:
+        """Printable caption, useful as a format specifier for --filename-pattern."""
+        def _elliptify(caption):
+            pcaption = ' '.join([s.replace('/', '\u2215') for s in caption.splitlines() if s]).strip()
+            return (pcaption[:30] + u"\u2026") if len(pcaption) > 31 else pcaption
+        return _elliptify(self.caption) if self.caption else ''
+
     @property
     def tagged_users(self) -> List[str]:
         """List of all lowercased users that are tagged in the Post."""
@@ -264,6 +282,21 @@ class Post:
         """URL of the video, or None."""
         if self.is_video:
             return self._field('video_url')
+        return None
+
+    @property
+    def video_view_count(self) -> Optional[int]:
+        """View count of the video, or None."""
+        if self.is_video:
+            return self._field('video_view_count')
+        return None
+
+    @property
+    def video_duration(self) -> Optional[float]:
+        """Duration of the video in seconds, or None."""
+        if self.is_video:
+            return self._field('video_duration')
+        return None
 
     @property
     def viewer_has_liked(self) -> Optional[bool]:
@@ -281,34 +314,72 @@ class Post:
 
     @property
     def comments(self) -> int:
-        """Comment count"""
-        return self._field('edge_media_to_comment', 'count')
+        """Comment count including answers"""
+        try:
+            return self._field('edge_media_to_parent_comment', 'count')
+        except KeyError:
+            return self._field('edge_media_to_comment', 'count')
 
     def get_comments(self) -> Iterator[PostComment]:
-        """Iterate over all comments of the post.
+        r"""Iterate over all comments of the post.
 
         Each comment is represented by a PostComment namedtuple with fields text (string), created_at (datetime),
-        id (int) and owner (:class:`Profile`).
+        id (int), owner (:class:`Profile`) and answers (:class:`~typing.Iterator`\ [:class:`PostCommentAnswer`])
+        if available.
         """
+        def _postcommentanswer(node):
+            return PostCommentAnswer(id=int(node['id']),
+                                     created_at_utc=datetime.utcfromtimestamp(node['created_at']),
+                                     text=node['text'],
+                                     owner=Profile(self._context, node['owner']))
+
+        def _postcommentanswers(node):
+            if 'edge_threaded_comments' not in node:
+                return
+            answer_count = node['edge_threaded_comments']['count']
+            if answer_count == 0:
+                # Avoid doing additional requests if there are no comment answers
+                return
+            answer_edges = node['edge_threaded_comments']['edges']
+            if answer_count == len(answer_edges):
+                # If the answer's metadata already contains all comments, don't do GraphQL requests to obtain them
+                yield from (_postcommentanswer(comment['node']) for comment in answer_edges)
+                return
+            yield from (_postcommentanswer(answer_node) for answer_node in
+                        self._context.graphql_node_list("51fdd02b67508306ad4484ff574a0b62",
+                                                        {'comment_id': node['id']},
+                                                        'https://www.instagram.com/p/' + self.shortcode + '/',
+                                                        lambda d: d['data']['comment']['edge_threaded_comments']))
+
         def _postcomment(node):
-            return PostComment(id=int(node['id']),
-                               created_at_utc=datetime.utcfromtimestamp(node['created_at']),
-                               text=node['text'],
-                               owner=Profile(self._context, node['owner']))
+            return PostComment(*_postcommentanswer(node),
+                               answers=_postcommentanswers(node))
         if self.comments == 0:
             # Avoid doing additional requests if there are no comments
             return
-        comment_edges = self._field('edge_media_to_comment', 'edges')
-        if self.comments == len(comment_edges):
-            # If the Post's metadata already contains all comments, don't do GraphQL requests to obtain them
+        try:
+            comment_edges = self._field('edge_media_to_parent_comment', 'edges')
+            answers_count = sum([edge['node']['edge_threaded_comments']['count'] for edge in comment_edges])
+            threaded_comments_available = True
+        except KeyError:
+            comment_edges = self._field('edge_media_to_comment', 'edges')
+            answers_count = 0
+            threaded_comments_available = False
+
+        if self.comments == len(comment_edges) + answers_count:
+            # If the Post's metadata already contains all parent comments, don't do GraphQL requests to obtain them
             yield from (_postcomment(comment['node']) for comment in comment_edges)
             return
         yield from (_postcomment(node) for node in
-                    self._context.graphql_node_list("33ba35852cb50da46f5b5e889df7d159",
-                                                    {'shortcode': self.shortcode},
-                                                    'https://www.instagram.com/p/' + self.shortcode + '/',
-                                                    lambda d: d['data']['shortcode_media']['edge_media_to_comment'],
-                                                    self._rhx_gis))
+                    self._context.graphql_node_list(
+                        "97b41c52301f77ce508f55e66d17620e" if threaded_comments_available
+                        else "f0986789a5c5d17c2400faebf16efd0d",
+                        {'shortcode': self.shortcode},
+                        'https://www.instagram.com/p/' + self.shortcode + '/',
+                        lambda d:
+                        d['data']['shortcode_media'][
+                            'edge_media_to_parent_comment' if threaded_comments_available else 'edge_media_to_comment'],
+                        self._rhx_gis))
 
     def get_likes(self) -> Iterator['Profile']:
         """Iterate over all likes of the post. A :class:`Profile` instance of each likee is yielded."""
@@ -371,7 +442,7 @@ class Profile:
     def __init__(self, context: InstaloaderContext, node: Dict[str, Any]):
         assert 'username' in node
         self._context = context
-        self._has_public_story = None
+        self._has_public_story = None  # type: Optional[bool]
         self._node = node
         self._rhx_gis = None
         self._iphone_struct_ = None
@@ -397,26 +468,29 @@ class Profile:
     @classmethod
     def from_id(cls, context: InstaloaderContext, profile_id: int):
         """Create a Profile instance from a given userid. If possible, use :meth:`Profile.from_username`
-        or constructor directly rather than this method, since it does many requests.
+        or constructor directly rather than this method, since it requires more requests.
 
         :param context: :attr:`Instaloader.context`
         :param profile_id: userid
-        :raises: :class:`ProfileNotExistsException`, :class:`ProfileHasNoPicsException`
+        :raises: :class:`ProfileNotExistsException`
         """
-        data = context.graphql_query("472f257a40c653c64c666ce877d59d2b",
-                                     {'id': str(profile_id), 'first': 1},
+        if profile_id in context.profile_id_cache:
+            return context.profile_id_cache[profile_id]
+        data = context.graphql_query('7c16654f22c819fb63d1183034a5162f',
+                                     {'user_id': str(profile_id),
+                                      'include_chaining': False,
+                                      'include_reel': True,
+                                      'include_suggested_users': False,
+                                      'include_logged_out_extras': False,
+                                      'include_highlight_reels': False},
                                      rhx_gis=context.root_rhx_gis)['data']['user']
         if data:
-            data = data["edge_owner_to_timeline_media"]
+            profile = cls(context, data['reel']['owner'])
         else:
             raise ProfileNotExistsException("No profile found, the user may have blocked you (ID: " +
                                             str(profile_id) + ").")
-        if not data['edges']:
-            if data['count'] == 0:
-                raise ProfileHasNoPicsException("Profile with ID {0}: no pics found.".format(str(profile_id)))
-            else:
-                raise LoginRequiredException("Login required to determine username (ID: " + str(profile_id) + ").")
-        return Post(context, data['edges'][0]['node']).owner_profile
+        context.profile_id_cache[profile_id] = profile
+        return profile
 
     def _asdict(self):
         json_node = self._node.copy()
@@ -452,10 +526,10 @@ class Profile:
 
     @property
     def _iphone_struct(self) -> Dict[str, Any]:
+        if not self._context.is_logged_in:
+            raise LoginRequiredException("--login required to access iPhone profile info endpoint.")
         if not self._iphone_struct_:
-            with self._context.anonymous_copy() as anonymous_context:
-                data = anonymous_context.get_json(path='api/v1/users/{}/info/'.format(self.userid),
-                                                  params={}, host='i.instagram.com')
+            data = self._context.get_iphone_json(path='api/v1/users/{}/info/'.format(self.userid), params={})
             self._iphone_struct_ = data['user']
         return self._iphone_struct_
 
@@ -527,10 +601,12 @@ class Profile:
     @property
     def has_highlight_reels(self) -> bool:
         """
-        This becomes `True` if the :class:`Profile` has any stories currently available,
-        even if not viewable by the viewer.
+        .. deprecated:: 4.0.6
+           Always returns `True` since :issue:`153`.
+
+        Before broken, this indicated whether the :class:`Profile` had available stories.
         """
-        return self._iphone_struct['has_highlight_reels']
+        return True
 
     @property
     def has_public_story(self) -> bool:
@@ -546,11 +622,14 @@ class Profile:
                                                        'https://www.instagram.com/{}/'.format(self.username),
                                                        self._rhx_gis)
             self._has_public_story = data['data']['user']['has_public_story']
+        assert self._has_public_story is not None
         return self._has_public_story
 
     @property
     def has_viewable_story(self) -> bool:
         """
+        .. deprecated:: 4.0.6
+
         Some stories are private. This property determines if the :class:`Profile`
         has at least one story which can be viewed using the associated :class:`InstaloaderContext`,
         i.e. the viewer has privileges to view it.
@@ -571,11 +650,19 @@ class Profile:
 
     @property
     def profile_pic_url(self) -> str:
-        """Return URL of profile picture"""
-        try:
-            return self._iphone_struct['hd_profile_pic_url_info']['url']
-        except (InstaloaderException, KeyError) as err:
-            self._context.error('{} Unable to fetch high quality profile pic.'.format(err))
+        """Return URL of profile picture. If logged in, the HD version is returned, otherwise a lower-quality version.
+
+        .. versionadded:: 4.0.3
+
+        .. versionchanged:: 4.2.1
+           Require being logged in for HD version (as required by Instagram)."""
+        if self._context.is_logged_in:
+            try:
+                return self._iphone_struct['hd_profile_pic_url_info']['url']
+            except (InstaloaderException, KeyError) as err:
+                self._context.error('{} Unable to fetch high quality profile pic.'.format(err))
+                return self._metadata("profile_pic_url_hd")
+        else:
             return self._metadata("profile_pic_url_hd")
 
     def get_profile_pic_url(self) -> str:
@@ -610,6 +697,18 @@ class Profile:
                                                     self._rhx_gis,
                                                     self._metadata('edge_saved_media')))
 
+    def get_tagged_posts(self) -> Iterator[Post]:
+        """Retrieve all posts where a profile is tagged.
+
+        .. versionadded:: 4.0.7"""
+        self._obtain_metadata()
+        yield from (Post(self._context, node, self if int(node['owner']['id']) == self.userid else None) for node in
+                    self._context.graphql_node_list("e31a871f7301132ceaab56507a66bbb7",
+                                                    {'id': self.userid},
+                                                    'https://www.instagram.com/{0}/'.format(self.username),
+                                                    lambda d: d['data']['user']['edge_user_to_photos_of_you'],
+                                                    self._rhx_gis))
+
     def get_followers(self) -> Iterator['Profile']:
         """
         Retrieve list of followers of given profile.
@@ -690,6 +789,7 @@ class StoryItem:
         """:class:`Profile` instance of the story item's owner."""
         if not self._owner_profile:
             self._owner_profile = Profile.from_id(self._context, self._node['owner']['id'])
+        assert self._owner_profile is not None
         return self._owner_profile
 
     @property
@@ -752,6 +852,7 @@ class StoryItem:
         """URL of the video, or None."""
         if self.is_video:
             return self._node['video_resources'][-1]['src']
+        return None
 
 
 class Story:
@@ -767,7 +868,7 @@ class Story:
            # story is a Story object
            for item in story.get_items():
                # item is a StoryItem object
-               L.download_storyitem(item, ':stores')
+               L.download_storyitem(item, ':stories')
 
     This class implements == and is hashable.
 
@@ -778,8 +879,8 @@ class Story:
     def __init__(self, context: InstaloaderContext, node: Dict[str, Any]):
         self._context = context
         self._node = node
-        self._unique_id = None
-        self._owner_profile = None
+        self._unique_id = None      # type: Optional[str]
+        self._owner_profile = None  # type: Optional[Profile]
 
     def __repr__(self):
         return '<Story by {} changed {:%Y-%m-%d_%H-%M-%S_UTC}>'.format(self.owner_username, self.latest_media_utc)
@@ -790,10 +891,10 @@ class Story:
         return NotImplemented
 
     def __hash__(self) -> int:
-        return hash(self._unique_id)
+        return hash(self.unique_id)
 
     @property
-    def unique_id(self) -> str:
+    def unique_id(self) -> Union[str, int]:
         """
         This ID only equals amongst :class:`Story` instances which have the same owner and the same set of
         :class:`StoryItem`. For all other :class:`Story` instances this ID is different.
@@ -809,12 +910,14 @@ class Story:
         """Timestamp when the story has last been watched or None (local time zone)."""
         if self._node['seen']:
             return datetime.fromtimestamp(self._node['seen'])
+        return None
 
     @property
     def last_seen_utc(self) -> Optional[datetime]:
         """Timestamp when the story has last been watched or None (UTC)."""
         if self._node['seen']:
             return datetime.utcfromtimestamp(self._node['seen'])
+        return None
 
     @property
     def latest_media_local(self) -> datetime:
@@ -853,6 +956,85 @@ class Story:
         yield from (StoryItem(self._context, item, self.owner_profile) for item in reversed(self._node['items']))
 
 
+class Highlight(Story):
+    """
+    Structure representing a user's highlight with its associated story items.
+
+    Provides methods for accessing highlight properties, as well as :meth:`Highlight.get_items` to request associated
+    :class:`StoryItem` nodes. Highlights are returned by :meth:`Instaloader.get_highlights`.
+
+    With a logged-in :class:`Instaloader` instance `L`, you may download all highlights of a :class:`Profile` instance
+    USER with::
+
+       for highlight in L.get_highlights(USER):
+           # highlight is a Highlight object
+           for item in highlight.get_items():
+               # item is a StoryItem object
+               L.download_storyitem(item, '{}/{}'.format(highlight.owner_username, highlight.title))
+
+    This class implements == and is hashable.
+
+    :param context: :class:`InstaloaderContext` instance used for additional queries if necessary.
+    :param node: Dictionary containing the available information of the highlight as returned by Instagram.
+    :param owner: :class:`Profile` instance representing the owner profile of the highlight.
+    """
+
+    def __init__(self, context: InstaloaderContext, node: Dict[str, Any], owner: Optional[Profile] = None):
+        super().__init__(context, node)
+        self._owner_profile = owner
+        self._items = None  # type: Optional[List[Dict[str, Any]]]
+
+    def __repr__(self):
+        return '<Highlight by {}: {}>'.format(self.owner_username, self.title)
+
+    @property
+    def unique_id(self) -> int:
+        """A unique ID identifying this set of highlights."""
+        return int(self._node['id'])
+
+    @property
+    def owner_profile(self) -> Profile:
+        """:class:`Profile` instance of the highlights' owner."""
+        if not self._owner_profile:
+            self._owner_profile = Profile(self._context, self._node['owner'])
+        return self._owner_profile
+
+    @property
+    def title(self) -> str:
+        """The title of these highlights."""
+        return self._node['title']
+
+    @property
+    def cover_url(self) -> str:
+        """URL of the highlights' cover."""
+        return self._node['cover_media']['thumbnail_src']
+
+    @property
+    def cover_cropped_url(self) -> str:
+        """URL of the cropped version of the cover."""
+        return self._node['cover_media_cropped_thumbnail']['url']
+
+    def _fetch_items(self):
+        if not self._items:
+            self._items = self._context.graphql_query("45246d3fe16ccc6577e0bd297a5db1ab",
+                                                      {"reel_ids": [], "tag_names": [], "location_ids": [],
+                                                       "highlight_reel_ids": [str(self.unique_id)],
+                                                       "precomposed_overlay": False})['data']['reels_media'][0]['items']
+
+    @property
+    def itemcount(self) -> int:
+        """Count of items associated with the :class:`Highlight` instance."""
+        self._fetch_items()
+        assert self._items is not None
+        return len(self._items)
+
+    def get_items(self) -> Iterator[StoryItem]:
+        """Retrieve all associated highlight items."""
+        self._fetch_items()
+        assert self._items is not None
+        yield from (StoryItem(self._context, item, self.owner_profile) for item in self._items)
+
+
 JsonExportable = Union[Post, Profile, StoryItem]
 
 

test/instaloader_unittests.py

@@ -8,16 +8,21 @@ from itertools import islice
 
 import instaloader
 
+PROFILE_WITH_HIGHLIGHTS = 325732271
 PUBLIC_PROFILE = "selenagomez"
 PUBLIC_PROFILE_ID = 460563723
 HASHTAG = "kitten"
+LOCATION = "362629379"
 OWN_USERNAME = "aandergr"
 NORMAL_MAX_COUNT = 2
 PAGING_MAX_COUNT = 15
 PRIVATE_PROFILE = "aandergr"
+PRIVATE_PROFILE_ID = 1706625676
+EMPTY_PROFILE = "not_public"
+EMPTY_PROFILE_ID = 1928659031
 
 # Preserve query timestamps (rate control) between tests to not get rate limited
-instaloadercontext_query_timestamps = list()
+instaloadercontext_query_timestamps = dict()
 
 
 class TestInstaloaderAnonymously(unittest.TestCase):
@@ -30,26 +35,31 @@ class TestInstaloaderAnonymously(unittest.TestCase):
                                          download_comments=True,
                                          save_metadata=True)
         self.L.context.raise_all_errors = True
-        self.L.context.query_timestamps = instaloadercontext_query_timestamps
+        self.L.context.query_timestamps = instaloadercontext_query_timestamps.copy()
 
     def tearDown(self):
         global instaloadercontext_query_timestamps
-        instaloadercontext_query_timestamps = self.L.context.query_timestamps
+        instaloadercontext_query_timestamps = self.L.context.query_timestamps.copy()
         self.L.close()
         os.chdir('/')
         print("Removing {}".format(self.dir))
         shutil.rmtree(self.dir)
 
+    def post_paging_test(self, iterator):
+        previous_post = None
+        for post in islice(iterator, PAGING_MAX_COUNT):
+            print(post)
+            if previous_post:
+                self.assertTrue(post.date_utc < previous_post.date_utc)
+            previous_post = post
+
     @unittest.SkipTest
     def test_public_profile_download(self):
         self.L.download_profile(PUBLIC_PROFILE, profile_pic=False, fast_update=True)
         self.L.download_profile(PUBLIC_PROFILE, profile_pic=False, fast_update=True)
 
     def test_public_profile_paging(self):
-        for count, post in enumerate(instaloader.Profile.from_username(self.L.context, PUBLIC_PROFILE).get_posts()):
-            print(post)
-            if count == PAGING_MAX_COUNT:
-                break
+        self.post_paging_test(instaloader.Profile.from_username(self.L.context, PUBLIC_PROFILE).get_posts())
 
     def test_profile_pic_download(self):
         self.L.download_profile(PUBLIC_PROFILE, profile_pic_only=True)
@@ -57,20 +67,48 @@ class TestInstaloaderAnonymously(unittest.TestCase):
     def test_hashtag_download(self):
         self.L.download_hashtag(HASHTAG, NORMAL_MAX_COUNT)
 
+    def test_hashtag_paging(self):
+        for count, post in enumerate(self.L.get_hashtag_posts(HASHTAG)):
+            print(post)
+            if count == PAGING_MAX_COUNT:
+                break
+
+    def test_location_download(self):
+        self.L.download_location(LOCATION, NORMAL_MAX_COUNT)
+
+    def test_location_paging(self):
+        for count, post in enumerate(self.L.get_location_posts(LOCATION)):
+            print(post)
+            if count == PAGING_MAX_COUNT:
+                break
+
     def test_get_id_by_username(self):
         self.assertEqual(PUBLIC_PROFILE_ID,
                          instaloader.Profile.from_username(self.L.context, PUBLIC_PROFILE).userid)
 
-    def test_get_username_by_id(self):
+    def test_get_username_by_id_private(self):
+        self.assertEqual(PRIVATE_PROFILE.lower(),
+                         instaloader.Profile.from_id(self.L.context, PRIVATE_PROFILE_ID).username)
+
+    def test_get_username_by_id_public(self):
         self.assertEqual(PUBLIC_PROFILE.lower(),
                          instaloader.Profile.from_id(self.L.context, PUBLIC_PROFILE_ID).username)
 
+    def test_get_username_by_id_empty(self):
+        self.assertEqual(EMPTY_PROFILE.lower(),
+                         instaloader.Profile.from_id(self.L.context, EMPTY_PROFILE_ID).username)
+
     def test_post_from_mediaid(self):
         for post in instaloader.Profile.from_username(self.L.context, PUBLIC_PROFILE).get_posts():
             post2 = instaloader.Post.from_mediaid(self.L.context, post.mediaid)
             self.assertEqual(post, post2)
             break
 
+    def test_public_profile_tagged_paging(self):
+        for post in islice(instaloader.Profile.from_username(self.L.context, PUBLIC_PROFILE).get_tagged_posts(),
+                           PAGING_MAX_COUNT):
+            print(post)
+
 
 class TestInstaloaderLoggedIn(TestInstaloaderAnonymously):
 
@@ -92,11 +130,19 @@ class TestInstaloaderLoggedIn(TestInstaloaderAnonymously):
             for item in user_story.get_items():
                 print(item)
 
+    def test_highlights_paging(self):
+        for user_highlight in self.L.get_highlights(PROFILE_WITH_HIGHLIGHTS):
+            print("Retrieving {} highlights \"{}\" from profile {}".format(user_highlight.itemcount,
+                                                                           user_highlight.title,
+                                                                           user_highlight.owner_username))
+            for item in user_highlight.get_items():
+                print(item)
+
     def test_private_profile_paging(self):
-        for count, post in enumerate(instaloader.Profile.from_username(self.L.context, PRIVATE_PROFILE).get_posts()):
-            print(post)
-            if count == PAGING_MAX_COUNT:
-                break
+        self.post_paging_test(instaloader.Profile.from_username(self.L.context, PRIVATE_PROFILE).get_posts())
+
+    def test_profile_pic_download(self):
+        self.L.download_profile(PUBLIC_PROFILE, profile_pic_only=True)
 
     def test_feed_download(self):
         self.L.download_feed_posts(NORMAL_MAX_COUNT)
@@ -134,10 +180,6 @@ class TestInstaloaderLoggedIn(TestInstaloaderAnonymously):
         for f in profile.get_followers():
             print(f.username)
 
-    def test_get_username_by_id(self):
-        self.assertEqual(PUBLIC_PROFILE.lower(),
-                         instaloader.Profile.from_id(self.L.context, PUBLIC_PROFILE_ID).username)
-
     def test_get_likes(self):
         for post in instaloader.Profile.from_username(self.L.context, OWN_USERNAME).get_posts():
             for like in post.get_likes():