Fix obtaining post metadata when not logged-in

Fixes #960.

.github/ISSUE_TEMPLATE/bug_report.md

@@ -1,6 +1,7 @@
 ---
-name: Bug report
-about: Create a report to help us improve
+name: 🐛 Bug report
+about: Create a report to help us improve 🤕
+labels: bug
 
 ---
 
@@ -12,7 +13,7 @@ Steps to reproduce the behavior:
 (e.g. Instaloader command line)
 
 **Expected behavior**
-A clear and concise description of what you expected to happen (if not obvious).
+A clear and concise description of what you expected to happen (even if it seems obvious).
 
 **Error messages and tracebacks**
 If applicable, add error messages and tracebacks to help explain your problem.
@@ -22,3 +23,10 @@ If applicable, add error messages and tracebacks to help explain your problem.
 
 **Additional context**
 Add any other context about the problem here.
+
+<!--
+Also see https://instaloader.github.io/contributing.html for how to report a bug.
+
+Please understand that a bug issue that lacks the required information cannot be
+processed.
+-->

.github/ISSUE_TEMPLATE/config.yml

@@ -0,0 +1,11 @@
+blank_issues_enabled: false
+contact_links:
+  - name: Contributing to Instaloader
+    url: https://instaloader.github.io/contributing.html
+    about: Guidelines for how to report an issue.
+  - name: Instaloader Troubleshooting
+    url: https://instaloader.github.io/troubleshooting.html
+    about: Many common problems and their workarounds are described here.
+  - name: Instaloader Documentation
+    url: https://instaloader.github.io/
+    about: Many questions might be already answered here.

.github/ISSUE_TEMPLATE/feature_request.md

@@ -1,6 +1,7 @@
 ---
-name: Feature request
-about: Suggest an idea for this project
+name: 🚀 Feature Suggestion
+about: Suggest an idea for this project 💡
+labels: "feature suggestion"
 
 ---
 
@@ -13,5 +14,11 @@ A clear and concise description of what you want to happen.
 **Describe alternatives you've considered**
 A clear and concise description of any alternative solutions or features you've considered.
 
+**If the feature request is accepted, would you be willing to submit a pull request?**
+Yes / No
+<!-- Help can be provided if you need assistance submitting a PR -->
+
 **Additional context**
 Add any other context about the feature request here.
+
+<!-- please also see https://instaloader.github.io/contributing.html for how to suggest a feature -->

.github/ISSUE_TEMPLATE/question.md

@@ -0,0 +1,18 @@
+---
+name: ❓ Question
+about: If you have a question about how to use Instaloader that's not well covered by our documentation
+labels: question
+
+---
+
+Your question here...
+
+<!--
+People will be better able and more motivated to provide help if your question
+
+ - has a concise and meaningful title,
+ - is not covered by the documentation: https://instaloader.github.io/,
+ - is actually related to Instaloader (this is not a Python help forum),
+ - has not already been answered here or on other sites such as Stack Overflow: https://stackoverflow.com/questions/tagged/instaloader,
+ - is written in a polite and welcoming tone.
+-->

.github/funding.yml

@@ -0,0 +1 @@
+custom: "https://instaloader.github.io/contributing.html#donations"

.github/pull_request_template.md

@@ -0,0 +1,18 @@
+
+
+<!--
+Thanks for your willingness to contribute to Instaloader! Please briefly
+describe
+
+- A motivation for this change, e.g.
+  - Fixes # .
+  - More general: What problem does the pull request solve?
+
+- The changes proposed in this pull request
+
+- The completeness of this change
+  - Is it just a proof of concept?
+  - Is the documentation updated (if appropriate)?
+  - Do you consider it ready to be merged or is it a draft?
+  - Can we help you at some point?
+-->

.github/workflows/lint.yml

@@ -0,0 +1,30 @@
+name: PyLint and MyPy
+
+on: [push, pull_request]
+
+jobs:
+  lint:
+    name: PyLint and MyPy
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: [3.8]
+    steps:
+      - name: Checkout Instaloader Repository
+        uses: actions/checkout@v2
+        with:
+          fetch-depth: 0  # needed for building docs
+      - name: Setup Python
+        uses: actions/setup-python@v2
+        with:
+          python-version: ${{ matrix.python-version }}
+      - name: Install Dependencies
+        run: |
+          python -m pip install pipenv==2020.6.2
+          pipenv sync --dev
+      - name: PyLint
+        run: pipenv run pylint instaloader
+      - name: MyPy
+        run: pipenv run mypy -m instaloader
+      - name: Build Documentation
+        run: pipenv run make -C docs html SPHINXOPTS="-W -n"

.github/workflows/stale.yml

@@ -0,0 +1,34 @@
+name: Mark stale issues and pull requests
+
+on:
+  schedule:
+  - cron: "10 14 * * *"
+
+jobs:
+  stale:
+
+    name: Mark stale issues and pull requests
+    runs-on: ubuntu-latest
+    
+    steps:
+    - uses: actions/stale@v3
+      with:
+        repo-token: ${{ secrets.GITHUB_TOKEN }}
+        only-labels: 'question'
+        stale-issue-message: 'There has been no activity on this question for an extended period of time. This issue will be closed after further 14 days of inactivity.'
+        stale-issue-label: 'stale'
+        exempt-issue-labels: 'leave open'
+        days-before-stale: 15
+        days-before-close: -1
+        remove-stale-when-updated: false
+    - uses: actions/stale@v1
+      with:
+        repo-token: ${{ secrets.GITHUB_TOKEN }}
+        stale-issue-message: 'There has been no activity on this issue for an extended period of time. This issue will be closed after further 14 days of inactivity.'
+        stale-pr-message: 'There has been no activity on this Pull Request for an extended period of time. This Pull Request will be closed after further 14 days of inactivity.'
+        stale-issue-label: 'stale'
+        stale-pr-label: 'stale'
+        exempt-issue-label: 'leave open'
+        exempt-pr-label: 'leave open'
+        days-before-stale: 90
+        days-before-close: 14

.github/workflows/windows_exe.yml

@@ -0,0 +1,48 @@
+name: Windows EXE build
+
+on: [push, pull_request]
+
+jobs:
+  build:
+    name: Windows EXE build
+    runs-on: windows-latest
+    steps:
+      - name: Checkout Instaloader repository
+        uses: actions/checkout@v2
+      - name: Setup Python
+        uses: actions/setup-python@v1
+        with:
+          python-version: 3.8
+          architecture: x64
+      - name: Get the tagged version
+        id: get_version
+        env:
+          GITHUB_REF: ${{ github.ref }}
+        run: echo ::set-output name=VERSION::${GITHUB_REF/refs\/tags\/v/}
+        shell: bash
+      - name: Run custom python script for EXE creation
+        env:
+          VERSION_TAG: v${{ steps.get_version.outputs.VERSION }}
+        run: python deploy/windows/create_exe.py
+      - name: Create draft release
+        if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v')
+        id: create_release
+        uses: actions/create-release@v1
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        with:
+          tag_name: v${{ steps.get_version.outputs.VERSION }}
+          release_name: Version ${{ steps.get_version.outputs.VERSION }}
+          draft: true
+          prerelease: contains(steps.get_version.outputs.VERSION, 'a') || contains(steps.get_version.outputs.VERSION, 'b') || contains(steps.get_version.outputs.VERSION, 'c')
+      - name: Upload EXE as release asset
+        if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v')
+        id: upload-release-asset
+        uses: actions/upload-release-asset@v1
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        with:
+          upload_url: ${{ steps.create_release.outputs.upload_url }}
+          asset_path: ./instaloader-v${{ steps.get_version.outputs.VERSION }}-windows-standalone.zip
+          asset_name: instaloader-v${{ steps.get_version.outputs.VERSION }}-windows-standalone.zip
+          asset_content_type: application/zip

.gitignore

@@ -1 +1,2 @@
 __pycache__/
+.mypy_cache/

.pylintrc

@@ -62,9 +62,7 @@ confidence=
 # --disable=W".
 disable=invalid-name,
         missing-docstring,
-        line-too-long,
         too-many-lines,
-        bad-whitespace,
         print-statement,
         parameter-unpacking,
         unpacking-in-except,
@@ -83,7 +81,6 @@ disable=invalid-name,
         useless-suppression,
         deprecated-pragma,
         use-symbolic-message-instead,
-        cyclic-import,
         too-many-instance-attributes,
         too-many-public-methods,
         too-many-branches,
@@ -91,7 +88,6 @@ disable=invalid-name,
         too-many-locals,
         too-many-statements,
         no-else-return,
-        inconsistent-return-statements,
         no-else-raise,
         unnecessary-pass,
         wildcard-import,
@@ -362,7 +358,7 @@ indent-after-paren=4
 indent-string='    '
 
 # Maximum number of characters on a single line.
-max-line-length=100
+max-line-length=120
 
 # Maximum number of lines in a module.
 max-module-lines=1000

.travis.yml

@@ -5,11 +5,10 @@ python:
  - "3.6"
  - "3.7"
 install:
- - pip install pylint~=2.3.1 requests mypy
- - pip install -r docs/requirements.txt
+ - pipenv sync --dev
 script:
- - python3 -m pylint instaloader
- - python3 -m mypy -m instaloader
+ - pylint instaloader
+ - mypy -m instaloader
  - make -C docs html SPHINXOPTS="-W -n"
 addons:
   ssh_known_hosts: aur.archlinux.org

AUTHORS.md

@@ -6,3 +6,4 @@ Instaloader is written by
 - Alexander Graf (@aandergr)
 - André Koch-Kramer (@Thammus)
 - Lars Lindqvist (@e5150)
+- ... and many more, see https://github.com/instaloader/instaloader/graphs/contributors

CODE_OF_CONDUCT.md

@@ -0,0 +1,135 @@
+
+# Contributor Covenant Code of Conduct
+
+## Our Pledge
+
+We as members, contributors, and leaders pledge to make participation in our
+community a harassment-free experience for everyone, regardless of age, body
+size, visible or invisible disability, ethnicity, sex characteristics, gender
+identity and expression, level of experience, education, socio-economic status,
+nationality, personal appearance, race, religion, or sexual identity
+and orientation.
+
+We pledge to act and interact in ways that contribute to an open, welcoming,
+diverse, inclusive, and healthy community.
+
+## Our Standards
+
+Examples of behavior that contributes to a positive environment for our
+community include:
+
+* Demonstrating empathy and kindness toward other people
+* Being respectful of differing opinions, viewpoints, and experiences
+* Giving and gracefully accepting constructive feedback
+* Accepting responsibility and apologizing to those affected by our mistakes,
+  and learning from the experience
+* Focusing on what is best not just for us as individuals, but for the
+  overall community
+
+Examples of unacceptable behavior include:
+
+* The use of sexualized language or imagery, and sexual attention or
+  advances of any kind
+* Trolling, insulting or derogatory comments, and personal or political attacks
+* Public or private harassment
+* Publishing others' private information, such as a physical or email
+  address, without their explicit permission
+* Other conduct which could reasonably be considered inappropriate in a
+  professional setting
+
+## Enforcement Responsibilities
+
+Community leaders are responsible for clarifying and enforcing our standards of
+acceptable behavior and will take appropriate and fair corrective action in
+response to any behavior that they deem inappropriate, threatening, offensive,
+or harmful.
+
+Community leaders have the right and responsibility to remove, edit, or reject
+comments, commits, code, wiki edits, issues, and other contributions that are
+not aligned to this Code of Conduct, and will communicate reasons for moderation
+decisions when appropriate.
+
+## Scope
+
+This Code of Conduct applies within all community spaces, and also applies when
+an individual is officially representing the community in public spaces.
+Examples of representing our community include using an official e-mail address,
+posting via an official social media account, or acting as an appointed
+representative at an online or offline event.
+
+## Enforcement
+
+Instances of abusive, harassing, or otherwise unacceptable behavior may be
+reported to the community leaders responsible for enforcement by opening an
+issue or contacting one or more of the project maintainers.
+
+All complaints will be reviewed and investigated promptly and fairly.
+
+All community leaders are obligated to respect the privacy and security of the
+reporter of any incident.
+
+## Enforcement Guidelines
+
+Community leaders will follow these Community Impact Guidelines in determining
+the consequences for any action they deem in violation of this Code of Conduct:
+
+### 1. Correction
+
+**Community Impact**: Use of inappropriate language or other behavior deemed
+unprofessional or unwelcome in the community.
+
+**Consequence**: A private, written warning from community leaders, providing
+clarity around the nature of the violation and an explanation of why the
+behavior was inappropriate. A public apology may be requested.
+
+### 2. Warning
+
+**Community Impact**: A violation through a single incident or series
+of actions.
+
+**Consequence**: A warning with consequences for continued behavior. No
+interaction with the people involved, including unsolicited interaction with
+those enforcing the Code of Conduct, for a specified period of time. This
+includes avoiding interactions in community spaces as well as external channels
+like social media. Violating these terms may lead to a temporary or
+permanent ban.
+
+### 3. Temporary Ban
+
+**Community Impact**: A serious violation of community standards, including
+sustained inappropriate behavior.
+
+**Consequence**: A temporary ban from any sort of interaction or public
+communication with the community for a specified period of time. No public or
+private interaction with the people involved, including unsolicited interaction
+with those enforcing the Code of Conduct, is allowed during this period.
+Violating these terms may lead to a permanent ban.
+
+### 4. Permanent Ban
+
+**Community Impact**: Demonstrating a pattern of violation of community
+standards, including sustained inappropriate behavior,  harassment of an
+individual, or aggression toward or disparagement of classes of individuals.
+
+**Consequence**: A permanent ban from any sort of public interaction within
+the community.
+
+## Attribution
+
+This Code of Conduct is adapted from the [Contributor Covenant][homepage],
+version 2.0, available at
+[https://www.contributor-covenant.org/version/2/0/code_of_conduct.html][v2.0].
+
+Community Impact Guidelines were inspired by 
+[Mozilla's code of conduct enforcement ladder][Mozilla CoC].
+
+For answers to common questions about this code of conduct, see the FAQ at
+[https://www.contributor-covenant.org/faq][FAQ]. Translations are available 
+at [https://www.contributor-covenant.org/translations][translations].
+
+[homepage]: https://www.contributor-covenant.org
+[v2.0]: https://www.contributor-covenant.org/version/2/0/code_of_conduct.html
+[Mozilla CoC]: https://github.com/mozilla/diversity
+[FAQ]: https://www.contributor-covenant.org/faq
+[translations]: https://www.contributor-covenant.org/translations
+

LICENSE

@@ -1,6 +1,6 @@
 The MIT License (MIT)
 
-Copyright (c) 2016-2019 Alexander Graf and André Koch-Kramer.
+Copyright (c) 2016-2020 Alexander Graf and André Koch-Kramer.
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

Pipfile

@@ -0,0 +1,16 @@
+[[source]]
+name = "pypi"
+url = "https://pypi.org/simple"
+verify_ssl = true
+
+[dev-packages]
+pylint = "*"
+mypy = "*"
+sphinx = "~=2.3"
+pyinstaller = "*"
+pefile = "*"
+pywin32-ctypes = "*"
+psutil = "*"
+
+[packages]
+requests = "*"

Pipfile.lock

@@ -0,0 +1,439 @@
+{
+    "_meta": {
+        "hash": {
+            "sha256": "58caf1be034bed9aa7ebb8c78e75ffa7d7abe820d014b6398c444a5db566b306"
+        },
+        "pipfile-spec": 6,
+        "requires": {},
+        "sources": [
+            {
+                "name": "pypi",
+                "url": "https://pypi.org/simple",
+                "verify_ssl": true
+            }
+        ]
+    },
+    "default": {
+        "certifi": {
+            "hashes": [
+                "sha256:5930595817496dd21bb8dc35dad090f1c2cd0adfaf21204bf6732ca5d8ee34d3",
+                "sha256:8fc0819f1f30ba15bdb34cceffb9ef04d99f420f68eb75d901e9560b8749fc41"
+            ],
+            "version": "==2020.6.20"
+        },
+        "chardet": {
+            "hashes": [
+                "sha256:84ab92ed1c4d4f16916e05906b6b75a6c0fb5db821cc65e70cbd64a3e2a5eaae",
+                "sha256:fc323ffcaeaed0e0a02bf4d117757b98aed530d9ed4531e3e15460124c106691"
+            ],
+            "version": "==3.0.4"
+        },
+        "idna": {
+            "hashes": [
+                "sha256:b307872f855b18632ce0c21c5e45be78c0ea7ae4c15c828c20788b26921eb3f6",
+                "sha256:b97d804b1e9b523befed77c48dacec60e6dcb0b5391d57af6a65a312a90648c0"
+            ],
+            "version": "==2.10"
+        },
+        "requests": {
+            "hashes": [
+                "sha256:b3559a131db72c33ee969480840fff4bb6dd111de7dd27c8ee1f820f4f00231b",
+                "sha256:fe75cc94a9443b9246fc7049224f75604b113c36acb93f87b80ed42c44cbb898"
+            ],
+            "index": "pypi",
+            "version": "==2.24.0"
+        },
+        "urllib3": {
+            "hashes": [
+                "sha256:91056c15fa70756691db97756772bb1eb9678fa585d9184f24534b100dc60f4a",
+                "sha256:e7983572181f5e1522d9c98453462384ee92a0be7fac5f1413a1e35c56cc0461"
+            ],
+            "version": "==1.25.10"
+        }
+    },
+    "develop": {
+        "alabaster": {
+            "hashes": [
+                "sha256:446438bdcca0e05bd45ea2de1668c1d9b032e1a9154c2c259092d77031ddd359",
+                "sha256:a661d72d58e6ea8a57f7a86e37d86716863ee5e92788398526d58b26a4e4dc02"
+            ],
+            "version": "==0.7.12"
+        },
+        "altgraph": {
+            "hashes": [
+                "sha256:1f05a47122542f97028caf78775a095fbe6a2699b5089de8477eb583167d69aa",
+                "sha256:c623e5f3408ca61d4016f23a681b9adb100802ca3e3da5e718915a9e4052cebe"
+            ],
+            "version": "==0.17"
+        },
+        "astroid": {
+            "hashes": [
+                "sha256:2f4078c2a41bf377eea06d71c9d2ba4eb8f6b1af2135bec27bbbb7d8f12bb703",
+                "sha256:bc58d83eb610252fd8de6363e39d4f1d0619c894b0ed24603b881c02e64c7386"
+            ],
+            "version": "==2.4.2"
+        },
+        "babel": {
+            "hashes": [
+                "sha256:1aac2ae2d0d8ea368fa90906567f5c08463d98ade155c0c4bfedd6a0f7160e38",
+                "sha256:d670ea0b10f8b723672d3a6abeb87b565b244da220d76b4dba1b66269ec152d4"
+            ],
+            "version": "==2.8.0"
+        },
+        "certifi": {
+            "hashes": [
+                "sha256:5930595817496dd21bb8dc35dad090f1c2cd0adfaf21204bf6732ca5d8ee34d3",
+                "sha256:8fc0819f1f30ba15bdb34cceffb9ef04d99f420f68eb75d901e9560b8749fc41"
+            ],
+            "version": "==2020.6.20"
+        },
+        "chardet": {
+            "hashes": [
+                "sha256:84ab92ed1c4d4f16916e05906b6b75a6c0fb5db821cc65e70cbd64a3e2a5eaae",
+                "sha256:fc323ffcaeaed0e0a02bf4d117757b98aed530d9ed4531e3e15460124c106691"
+            ],
+            "version": "==3.0.4"
+        },
+        "docutils": {
+            "hashes": [
+                "sha256:0c5b78adfbf7762415433f5515cd5c9e762339e23369dbe8000d84a4bf4ab3af",
+                "sha256:c2de3a60e9e7d07be26b7f2b00ca0309c207e06c100f9cc2a94931fc75a478fc"
+            ],
+            "version": "==0.16"
+        },
+        "future": {
+            "hashes": [
+                "sha256:b1bead90b70cf6ec3f0710ae53a525360fa360d306a86583adc6bf83a4db537d"
+            ],
+            "version": "==0.18.2"
+        },
+        "idna": {
+            "hashes": [
+                "sha256:b307872f855b18632ce0c21c5e45be78c0ea7ae4c15c828c20788b26921eb3f6",
+                "sha256:b97d804b1e9b523befed77c48dacec60e6dcb0b5391d57af6a65a312a90648c0"
+            ],
+            "version": "==2.10"
+        },
+        "imagesize": {
+            "hashes": [
+                "sha256:6965f19a6a2039c7d48bca7dba2473069ff854c36ae6f19d2cde309d998228a1",
+                "sha256:b1f6b5a4eab1f73479a50fb79fcf729514a900c341d8503d62a62dbc4127a2b1"
+            ],
+            "version": "==1.2.0"
+        },
+        "isort": {
+            "hashes": [
+                "sha256:54da7e92468955c4fceacd0c86bd0ec997b0e1ee80d97f67c35a78b719dccab1",
+                "sha256:6e811fcb295968434526407adb8796944f1988c5b65e8139058f2014cbe100fd"
+            ],
+            "version": "==4.3.21"
+        },
+        "jinja2": {
+            "hashes": [
+                "sha256:89aab215427ef59c34ad58735269eb58b1a5808103067f7bb9d5836c651b3bb0",
+                "sha256:f0a4641d3cf955324a89c04f3d94663aa4d638abe8f733ecd3582848e1c37035"
+            ],
+            "version": "==2.11.2"
+        },
+        "lazy-object-proxy": {
+            "hashes": [
+                "sha256:0c4b206227a8097f05c4dbdd323c50edf81f15db3b8dc064d08c62d37e1a504d",
+                "sha256:194d092e6f246b906e8f70884e620e459fc54db3259e60cf69a4d66c3fda3449",
+                "sha256:1be7e4c9f96948003609aa6c974ae59830a6baecc5376c25c92d7d697e684c08",
+                "sha256:4677f594e474c91da97f489fea5b7daa17b5517190899cf213697e48d3902f5a",
+                "sha256:48dab84ebd4831077b150572aec802f303117c8cc5c871e182447281ebf3ac50",
+                "sha256:5541cada25cd173702dbd99f8e22434105456314462326f06dba3e180f203dfd",
+                "sha256:59f79fef100b09564bc2df42ea2d8d21a64fdcda64979c0fa3db7bdaabaf6239",
+                "sha256:8d859b89baf8ef7f8bc6b00aa20316483d67f0b1cbf422f5b4dc56701c8f2ffb",
+                "sha256:9254f4358b9b541e3441b007a0ea0764b9d056afdeafc1a5569eee1cc6c1b9ea",
+                "sha256:9651375199045a358eb6741df3e02a651e0330be090b3bc79f6d0de31a80ec3e",
+                "sha256:97bb5884f6f1cdce0099f86b907aa41c970c3c672ac8b9c8352789e103cf3156",
+                "sha256:9b15f3f4c0f35727d3a0fba4b770b3c4ebbb1fa907dbcc046a1d2799f3edd142",
+                "sha256:a2238e9d1bb71a56cd710611a1614d1194dc10a175c1e08d75e1a7bcc250d442",
+                "sha256:a6ae12d08c0bf9909ce12385803a543bfe99b95fe01e752536a60af2b7797c62",
+                "sha256:ca0a928a3ddbc5725be2dd1cf895ec0a254798915fb3a36af0964a0a4149e3db",
+                "sha256:cb2c7c57005a6804ab66f106ceb8482da55f5314b7fcb06551db1edae4ad1531",
+                "sha256:d74bb8693bf9cf75ac3b47a54d716bbb1a92648d5f781fc799347cfc95952383",
+                "sha256:d945239a5639b3ff35b70a88c5f2f491913eb94871780ebfabb2568bd58afc5a",
+                "sha256:eba7011090323c1dadf18b3b689845fd96a61ba0a1dfbd7f24b921398affc357",
+                "sha256:efa1909120ce98bbb3777e8b6f92237f5d5c8ea6758efea36a473e1d38f7d3e4",
+                "sha256:f3900e8a5de27447acbf900b4750b0ddfd7ec1ea7fbaf11dfa911141bc522af0"
+            ],
+            "version": "==1.4.3"
+        },
+        "markupsafe": {
+            "hashes": [
+                "sha256:00bc623926325b26bb9605ae9eae8a215691f33cae5df11ca5424f06f2d1f473",
+                "sha256:09027a7803a62ca78792ad89403b1b7a73a01c8cb65909cd876f7fcebd79b161",
+                "sha256:09c4b7f37d6c648cb13f9230d847adf22f8171b1ccc4d5682398e77f40309235",
+                "sha256:1027c282dad077d0bae18be6794e6b6b8c91d58ed8a8d89a89d59693b9131db5",
+                "sha256:13d3144e1e340870b25e7b10b98d779608c02016d5184cfb9927a9f10c689f42",
+                "sha256:24982cc2533820871eba85ba648cd53d8623687ff11cbb805be4ff7b4c971aff",
+                "sha256:29872e92839765e546828bb7754a68c418d927cd064fd4708fab9fe9c8bb116b",
+                "sha256:43a55c2930bbc139570ac2452adf3d70cdbb3cfe5912c71cdce1c2c6bbd9c5d1",
+                "sha256:46c99d2de99945ec5cb54f23c8cd5689f6d7177305ebff350a58ce5f8de1669e",
+                "sha256:500d4957e52ddc3351cabf489e79c91c17f6e0899158447047588650b5e69183",
+                "sha256:535f6fc4d397c1563d08b88e485c3496cf5784e927af890fb3c3aac7f933ec66",
+                "sha256:596510de112c685489095da617b5bcbbac7dd6384aeebeda4df6025d0256a81b",
+                "sha256:62fe6c95e3ec8a7fad637b7f3d372c15ec1caa01ab47926cfdf7a75b40e0eac1",
+                "sha256:6788b695d50a51edb699cb55e35487e430fa21f1ed838122d722e0ff0ac5ba15",
+                "sha256:6dd73240d2af64df90aa7c4e7481e23825ea70af4b4922f8ede5b9e35f78a3b1",
+                "sha256:717ba8fe3ae9cc0006d7c451f0bb265ee07739daf76355d06366154ee68d221e",
+                "sha256:79855e1c5b8da654cf486b830bd42c06e8780cea587384cf6545b7d9ac013a0b",
+                "sha256:7c1699dfe0cf8ff607dbdcc1e9b9af1755371f92a68f706051cc8c37d447c905",
+                "sha256:88e5fcfb52ee7b911e8bb6d6aa2fd21fbecc674eadd44118a9cc3863f938e735",
+                "sha256:8defac2f2ccd6805ebf65f5eeb132adcf2ab57aa11fdf4c0dd5169a004710e7d",
+                "sha256:98c7086708b163d425c67c7a91bad6e466bb99d797aa64f965e9d25c12111a5e",
+                "sha256:9add70b36c5666a2ed02b43b335fe19002ee5235efd4b8a89bfcf9005bebac0d",
+                "sha256:9bf40443012702a1d2070043cb6291650a0841ece432556f784f004937f0f32c",
+                "sha256:ade5e387d2ad0d7ebf59146cc00c8044acbd863725f887353a10df825fc8ae21",
+                "sha256:b00c1de48212e4cc9603895652c5c410df699856a2853135b3967591e4beebc2",
+                "sha256:b1282f8c00509d99fef04d8ba936b156d419be841854fe901d8ae224c59f0be5",
+                "sha256:b2051432115498d3562c084a49bba65d97cf251f5a331c64a12ee7e04dacc51b",
+                "sha256:ba59edeaa2fc6114428f1637ffff42da1e311e29382d81b339c1817d37ec93c6",
+                "sha256:c8716a48d94b06bb3b2524c2b77e055fb313aeb4ea620c8dd03a105574ba704f",
+                "sha256:cd5df75523866410809ca100dc9681e301e3c27567cf498077e8551b6d20e42f",
+                "sha256:cdb132fc825c38e1aeec2c8aa9338310d29d337bebbd7baa06889d09a60a1fa2",
+                "sha256:e249096428b3ae81b08327a63a485ad0878de3fb939049038579ac0ef61e17e7",
+                "sha256:e8313f01ba26fbbe36c7be1966a7b7424942f670f38e666995b88d012765b9be"
+            ],
+            "version": "==1.1.1"
+        },
+        "mccabe": {
+            "hashes": [
+                "sha256:ab8a6258860da4b6677da4bd2fe5dc2c659cff31b3ee4f7f5d64e79735b80d42",
+                "sha256:dd8d182285a0fe56bace7f45b5e7d1a6ebcbf524e8f3bd87eb0f125271b8831f"
+            ],
+            "version": "==0.6.1"
+        },
+        "mypy": {
+            "hashes": [
+                "sha256:2c6cde8aa3426c1682d35190b59b71f661237d74b053822ea3d748e2c9578a7c",
+                "sha256:3fdda71c067d3ddfb21da4b80e2686b71e9e5c72cca65fa216d207a358827f86",
+                "sha256:5dd13ff1f2a97f94540fd37a49e5d255950ebcdf446fb597463a40d0df3fac8b",
+                "sha256:6731603dfe0ce4352c555c6284c6db0dc935b685e9ce2e4cf220abe1e14386fd",
+                "sha256:6bb93479caa6619d21d6e7160c552c1193f6952f0668cdda2f851156e85186fc",
+                "sha256:81c7908b94239c4010e16642c9102bfc958ab14e36048fa77d0be3289dda76ea",
+                "sha256:9c7a9a7ceb2871ba4bac1cf7217a7dd9ccd44c27c2950edbc6dc08530f32ad4e",
+                "sha256:a4a2cbcfc4cbf45cd126f531dedda8485671545b43107ded25ce952aac6fb308",
+                "sha256:b7fbfabdbcc78c4f6fc4712544b9b0d6bf171069c6e0e3cb82440dd10ced3406",
+                "sha256:c05b9e4fb1d8a41d41dec8786c94f3b95d3c5f528298d769eb8e73d293abc48d",
+                "sha256:d7df6eddb6054d21ca4d3c6249cae5578cb4602951fd2b6ee2f5510ffb098707",
+                "sha256:e0b61738ab504e656d1fe4ff0c0601387a5489ca122d55390ade31f9ca0e252d",
+                "sha256:eff7d4a85e9eea55afa34888dfeaccde99e7520b51f867ac28a48492c0b1130c",
+                "sha256:f05644db6779387ccdb468cc47a44b4356fc2ffa9287135d05b70a98dc83b89a"
+            ],
+            "index": "pypi",
+            "version": "==0.782"
+        },
+        "mypy-extensions": {
+            "hashes": [
+                "sha256:090fedd75945a69ae91ce1303b5824f428daf5a028d2f6ab8a299250a846f15d",
+                "sha256:2d82818f5bb3e369420cb3c4060a7970edba416647068eb4c5343488a6c604a8"
+            ],
+            "version": "==0.4.3"
+        },
+        "packaging": {
+            "hashes": [
+                "sha256:4357f74f47b9c12db93624a82154e9b120fa8293699949152b22065d556079f8",
+                "sha256:998416ba6962ae7fbd6596850b80e17859a5753ba17c32284f67bfff33784181"
+            ],
+            "version": "==20.4"
+        },
+        "pefile": {
+            "hashes": [
+                "sha256:a5d6e8305c6b210849b47a6174ddf9c452b2888340b8177874b862ba6c207645"
+            ],
+            "index": "pypi",
+            "version": "==2019.4.18"
+        },
+        "psutil": {
+            "hashes": [
+                "sha256:0ee3c36428f160d2d8fce3c583a0353e848abb7de9732c50cf3356dd49ad63f8",
+                "sha256:10512b46c95b02842c225f58fa00385c08fa00c68bac7da2d9a58ebe2c517498",
+                "sha256:4080869ed93cce662905b029a1770fe89c98787e543fa7347f075ade761b19d6",
+                "sha256:5e9d0f26d4194479a13d5f4b3798260c20cecf9ac9a461e718eb59ea520a360c",
+                "sha256:66c18ca7680a31bf16ee22b1d21b6397869dda8059dbdb57d9f27efa6615f195",
+                "sha256:68d36986ded5dac7c2dcd42f2682af1db80d4bce3faa126a6145c1637e1b559f",
+                "sha256:90990af1c3c67195c44c9a889184f84f5b2320dce3ee3acbd054e3ba0b4a7beb",
+                "sha256:a5b120bb3c0c71dfe27551f9da2f3209a8257a178ed6c628a819037a8df487f1",
+                "sha256:d8a82162f23c53b8525cf5f14a355f5d1eea86fa8edde27287dd3a98399e4fdf",
+                "sha256:f2018461733b23f308c298653c8903d32aaad7873d25e1d228765e91ae42c3f2",
+                "sha256:ff1977ba1a5f71f89166d5145c3da1cea89a0fdb044075a12c720ee9123ec818"
+            ],
+            "index": "pypi",
+            "version": "==5.7.2"
+        },
+        "pygments": {
+            "hashes": [
+                "sha256:647344a061c249a3b74e230c739f434d7ea4d8b1d5f3721bc0f3558049b38f44",
+                "sha256:ff7a40b4860b727ab48fad6360eb351cc1b33cbf9b15a0f689ca5353e9463324"
+            ],
+            "version": "==2.6.1"
+        },
+        "pyinstaller": {
+            "hashes": [
+                "sha256:3730fa80d088f8bb7084d32480eb87cbb4ddb64123363763cf8f2a1378c1c4b7"
+            ],
+            "index": "pypi",
+            "version": "==3.6"
+        },
+        "pylint": {
+            "hashes": [
+                "sha256:7dd78437f2d8d019717dbf287772d0b2dbdfd13fc016aa7faa08d67bccc46adc",
+                "sha256:d0ece7d223fe422088b0e8f13fa0a1e8eb745ebffcb8ed53d3e95394b6101a1c"
+            ],
+            "index": "pypi",
+            "version": "==2.5.3"
+        },
+        "pyparsing": {
+            "hashes": [
+                "sha256:c203ec8783bf771a155b207279b9bccb8dea02d8f0c9e5f8ead507bc3246ecc1",
+                "sha256:ef9d7589ef3c200abe66653d3f1ab1033c3c419ae9b9bdb1240a85b024efc88b"
+            ],
+            "version": "==2.4.7"
+        },
+        "pytz": {
+            "hashes": [
+                "sha256:a494d53b6d39c3c6e44c3bec237336e14305e4f29bbf800b599253057fbb79ed",
+                "sha256:c35965d010ce31b23eeb663ed3cc8c906275d6be1a34393a1d73a41febf4a048"
+            ],
+            "version": "==2020.1"
+        },
+        "pywin32-ctypes": {
+            "hashes": [
+                "sha256:24ffc3b341d457d48e8922352130cf2644024a4ff09762a2261fd34c36ee5942",
+                "sha256:9dc2d991b3479cc2df15930958b674a48a227d5361d413827a4cfd0b5876fc98"
+            ],
+            "index": "pypi",
+            "version": "==0.2.0"
+        },
+        "requests": {
+            "hashes": [
+                "sha256:b3559a131db72c33ee969480840fff4bb6dd111de7dd27c8ee1f820f4f00231b",
+                "sha256:fe75cc94a9443b9246fc7049224f75604b113c36acb93f87b80ed42c44cbb898"
+            ],
+            "index": "pypi",
+            "version": "==2.24.0"
+        },
+        "six": {
+            "hashes": [
+                "sha256:30639c035cdb23534cd4aa2dd52c3bf48f06e5f4a941509c8bafd8ce11080259",
+                "sha256:8b74bedcbbbaca38ff6d7491d76f2b06b3592611af620f8426e82dddb04a5ced"
+            ],
+            "version": "==1.15.0"
+        },
+        "snowballstemmer": {
+            "hashes": [
+                "sha256:209f257d7533fdb3cb73bdbd24f436239ca3b2fa67d56f6ff88e86be08cc5ef0",
+                "sha256:df3bac3df4c2c01363f3dd2cfa78cce2840a79b9f1c2d2de9ce8d31683992f52"
+            ],
+            "version": "==2.0.0"
+        },
+        "sphinx": {
+            "hashes": [
+                "sha256:b4c750d546ab6d7e05bdff6ac24db8ae3e8b8253a3569b754e445110a0a12b66",
+                "sha256:fc312670b56cb54920d6cc2ced455a22a547910de10b3142276495ced49231cb"
+            ],
+            "index": "pypi",
+            "version": "==2.4.4"
+        },
+        "sphinxcontrib-applehelp": {
+            "hashes": [
+                "sha256:806111e5e962be97c29ec4c1e7fe277bfd19e9652fb1a4392105b43e01af885a",
+                "sha256:a072735ec80e7675e3f432fcae8610ecf509c5f1869d17e2eecff44389cdbc58"
+            ],
+            "version": "==1.0.2"
+        },
+        "sphinxcontrib-devhelp": {
+            "hashes": [
+                "sha256:8165223f9a335cc1af7ffe1ed31d2871f325254c0423bc0c4c7cd1c1e4734a2e",
+                "sha256:ff7f1afa7b9642e7060379360a67e9c41e8f3121f2ce9164266f61b9f4b338e4"
+            ],
+            "version": "==1.0.2"
+        },
+        "sphinxcontrib-htmlhelp": {
+            "hashes": [
+                "sha256:3c0bc24a2c41e340ac37c85ced6dafc879ab485c095b1d65d2461ac2f7cca86f",
+                "sha256:e8f5bb7e31b2dbb25b9cc435c8ab7a79787ebf7f906155729338f3156d93659b"
+            ],
+            "version": "==1.0.3"
+        },
+        "sphinxcontrib-jsmath": {
+            "hashes": [
+                "sha256:2ec2eaebfb78f3f2078e73666b1415417a116cc848b72e5172e596c871103178",
+                "sha256:a9925e4a4587247ed2191a22df5f6970656cb8ca2bd6284309578f2153e0c4b8"
+            ],
+            "version": "==1.0.1"
+        },
+        "sphinxcontrib-qthelp": {
+            "hashes": [
+                "sha256:4c33767ee058b70dba89a6fc5c1892c0d57a54be67ddd3e7875a18d14cba5a72",
+                "sha256:bd9fc24bcb748a8d51fd4ecaade681350aa63009a347a8c14e637895444dfab6"
+            ],
+            "version": "==1.0.3"
+        },
+        "sphinxcontrib-serializinghtml": {
+            "hashes": [
+                "sha256:eaa0eccc86e982a9b939b2b82d12cc5d013385ba5eadcc7e4fed23f4405f77bc",
+                "sha256:f242a81d423f59617a8e5cf16f5d4d74e28ee9a66f9e5b637a18082991db5a9a"
+            ],
+            "version": "==1.1.4"
+        },
+        "toml": {
+            "hashes": [
+                "sha256:926b612be1e5ce0634a2ca03470f95169cf16f939018233a670519cb4ac58b0f",
+                "sha256:bda89d5935c2eac546d648028b9901107a595863cb36bae0c73ac804a9b4ce88"
+            ],
+            "version": "==0.10.1"
+        },
+        "typed-ast": {
+            "hashes": [
+                "sha256:0666aa36131496aed8f7be0410ff974562ab7eeac11ef351def9ea6fa28f6355",
+                "sha256:0c2c07682d61a629b68433afb159376e24e5b2fd4641d35424e462169c0a7919",
+                "sha256:249862707802d40f7f29f6e1aad8d84b5aa9e44552d2cc17384b209f091276aa",
+                "sha256:24995c843eb0ad11a4527b026b4dde3da70e1f2d8806c99b7b4a7cf491612652",
+                "sha256:269151951236b0f9a6f04015a9004084a5ab0d5f19b57de779f908621e7d8b75",
+                "sha256:4083861b0aa07990b619bd7ddc365eb7fa4b817e99cf5f8d9cf21a42780f6e01",
+                "sha256:498b0f36cc7054c1fead3d7fc59d2150f4d5c6c56ba7fb150c013fbc683a8d2d",
+                "sha256:4e3e5da80ccbebfff202a67bf900d081906c358ccc3d5e3c8aea42fdfdfd51c1",
+                "sha256:6daac9731f172c2a22ade6ed0c00197ee7cc1221aa84cfdf9c31defeb059a907",
+                "sha256:715ff2f2df46121071622063fc7543d9b1fd19ebfc4f5c8895af64a77a8c852c",
+                "sha256:73d785a950fc82dd2a25897d525d003f6378d1cb23ab305578394694202a58c3",
+                "sha256:8c8aaad94455178e3187ab22c8b01a3837f8ee50e09cf31f1ba129eb293ec30b",
+                "sha256:8ce678dbaf790dbdb3eba24056d5364fb45944f33553dd5869b7580cdbb83614",
+                "sha256:aaee9905aee35ba5905cfb3c62f3e83b3bec7b39413f0a7f19be4e547ea01ebb",
+                "sha256:bcd3b13b56ea479b3650b82cabd6b5343a625b0ced5429e4ccad28a8973f301b",
+                "sha256:c9e348e02e4d2b4a8b2eedb48210430658df6951fa484e59de33ff773fbd4b41",
+                "sha256:d205b1b46085271b4e15f670058ce182bd1199e56b317bf2ec004b6a44f911f6",
+                "sha256:d43943ef777f9a1c42bf4e552ba23ac77a6351de620aa9acf64ad54933ad4d34",
+                "sha256:d5d33e9e7af3b34a40dc05f498939f0ebf187f07c385fd58d591c533ad8562fe",
+                "sha256:fc0fea399acb12edbf8a628ba8d2312f583bdbdb3335635db062fa98cf71fca4",
+                "sha256:fe460b922ec15dd205595c9b5b99e2f056fd98ae8f9f56b888e7a17dc2b757e7"
+            ],
+            "markers": "implementation_name == 'cpython' and python_version < '3.8'",
+            "version": "==1.4.1"
+        },
+        "typing-extensions": {
+            "hashes": [
+                "sha256:6e95524d8a547a91e08f404ae485bbb71962de46967e1b71a0cb89af24e761c5",
+                "sha256:79ee589a3caca649a9bfd2a8de4709837400dfa00b6cc81962a1e6a1815969ae",
+                "sha256:f8d2bd89d25bc39dabe7d23df520442fa1d8969b82544370e03d88b5a591c392"
+            ],
+            "version": "==3.7.4.2"
+        },
+        "urllib3": {
+            "hashes": [
+                "sha256:91056c15fa70756691db97756772bb1eb9678fa585d9184f24534b100dc60f4a",
+                "sha256:e7983572181f5e1522d9c98453462384ee92a0be7fac5f1413a1e35c56cc0461"
+            ],
+            "version": "==1.25.10"
+        },
+        "wrapt": {
+            "hashes": [
+                "sha256:b62ffa81fb85f4332a4f609cab4ac40709470da05643a082ec1eb88e6d9b97d7"
+            ],
+            "version": "==1.12.1"
+        }
+    }
+}

README.rst

@@ -2,7 +2,7 @@
 
 .. badges-start
 
-|travis| |pypi| |pyversion| |license| |aur| |contributors| |downloads| |saythanks|
+|travis| |pypi| |pyversion| |license| |aur| |contributors| |downloads|
 
 .. |travis| image:: https://img.shields.io/travis/instaloader/instaloader/master.svg
    :alt: Travis-CI Build Status
@@ -23,16 +23,13 @@
    :alt: Contributor Count
    :target: https://github.com/instaloader/instaloader/graphs/contributors
 
-.. |saythanks| image:: https://img.shields.io/badge/Say%20Thanks-!-ff7efd.svg
-   :alt: Say Thanks!
-   :target: https://saythanks.io/to/aandergr
-
 .. |aur| image:: https://img.shields.io/aur/version/instaloader.svg
    :alt: Arch User Repository Package
    :target: https://aur.archlinux.org/packages/instaloader/
 
-.. |downloads| image:: https://img.shields.io/pypi/dm/instaloader.svg
+.. |downloads| image:: https://pepy.tech/badge/instaloader/month
    :alt: PyPI Download Count
+   :target: https://pepy.tech/project/instaloader/month
 
 .. badges-end
 
@@ -53,11 +50,14 @@
   directory accordingly,
 
 - allows **fine-grained customization** of filters and where to store
-  downloaded media.
+  downloaded media,
+
+- automatically **resumes previously-interrupted** download iterations.
 
 ::
 
-    instaloader [--comments] [--geotags] [--stories] [--highlights] [--tagged]
+    instaloader [--comments] [--geotags]
+                [--stories] [--highlights] [--tagged] [--igtv]
                 [--login YOUR-USERNAME] [--fast-update]
                 profile | "#hashtag" | :stories | :feed | :saved
 

deploy/arch/deploy.sh

@@ -1,5 +1,10 @@
 #!/usr/bin/env bash
 
+VERSION=${1:1}
+
+# do not deploy pre-releases
+echo $VERSION | grep -qe "[abc]" && exit 0
+
 cd $(dirname $0)
 
 # decrypt and add ssh key
@@ -12,7 +17,6 @@ ssh-add /tmp/AUR_openssh
 git clone --depth 1 ssh://aur@aur.archlinux.org/instaloader.git
 
 curl -sSfOJ https://codeload.github.com/instaloader/instaloader/tar.gz/$1
-VERSION=${1:1}
 HASH=$(sha512sum instaloader-$VERSION.tar.gz | cut -f1 -d " ")
 sed -e "s/{{version}}/$VERSION/g" -e "s/{{hash}}/$HASH/g" PKGBUILD.template > instaloader/PKGBUILD
 sed -e "s/{{version}}/$VERSION/g" -e "s/{{hash}}/$HASH/g" .SRCINFO.template > instaloader/.SRCINFO

deploy/windows/create_exe.py

@@ -0,0 +1,71 @@
+import hashlib
+import os
+import re
+import shutil
+import subprocess
+import sys
+
+# copy the required files into repo root
+shutil.copy('docs/favicon.ico', '.')
+shutil.copy('deploy/windows/instaloader.spec', '.')
+
+code = """
+import contextlib
+import psutil
+import subprocess
+
+def __main():
+    with contextlib.suppress(AttributeError, psutil.Error):
+        if psutil.Process().parent().parent().name() == "explorer.exe":
+            subprocess.Popen("powershell -NoExit -Command \\\"& '{0}'\\\"".format(sys.argv[0]))
+            return
+    main()
+
+if __name__ == "__main__":
+    __main()
+"""
+
+with open('instaloader/__main__.py', 'r') as f:
+    # adjust imports for changed file structure
+    regex = re.compile(r'from (?:(\.[^ ]+ )|\.( ))import')
+    lines = [regex.sub(r'from instaloader\1\2import', line) for line in f.readlines()]
+
+    # insert code for magic exe behavior
+    index = lines.index('if __name__ == "__main__":\n')
+    code_lines = [cl + '\n' for cl in code.splitlines()]
+    for i, code_line in enumerate(code_lines):
+        if i + index < len(lines):
+            lines[i + index] = code_line
+        else:
+            lines.extend(code_lines[i:])
+            break
+
+with open('__main__.py', 'w+') as f:
+    f.writelines(lines)
+
+# install dependencies and invoke PyInstaller
+commands = ["pip install pipenv==2020.6.2",
+            "pipenv sync --dev",
+            "pipenv run pyinstaller --log-level=DEBUG instaloader.spec"]
+
+for command in commands:
+    print()
+    print('#' * (len(command) + 6))
+    print('## {} ##'.format(command))
+    print('#' * (len(command) + 6))
+    print(flush=True)
+    err = subprocess.Popen(command).wait()
+    if err != 0:
+        sys.exit(err)
+
+# calculate and store MD5 hash for created executable
+hash_md5 = hashlib.md5()
+with open('dist/instaloader.exe', 'rb') as f:
+    for chunk in iter(lambda: f.read(4096), b''):
+        hash_md5.update(chunk)
+
+with open('dist/instaloader.exe.md5', 'w+') as f:
+    f.write('{} *instaloader.exe\n'.format(hash_md5.hexdigest()))
+
+# Create ZIP file
+shutil.make_archive('instaloader-{}-windows-standalone'.format(os.getenv('VERSION_TAG')), 'zip', 'dist')

deploy/windows/instaloader.spec

@@ -0,0 +1,28 @@
+# -*- mode: python -*-
+
+block_cipher = None
+
+
+a = Analysis(['__main__.py'],
+             binaries=[],
+             datas=[],
+             hiddenimports=[],
+             hookspath=[],
+             runtime_hooks=[],
+             excludes=[],
+             win_no_prefer_redirects=False,
+             win_private_assemblies=False,
+             cipher=block_cipher)
+pyz = PYZ(a.pure, a.zipped_data,
+             cipher=block_cipher)
+exe = EXE(pyz,
+          a.scripts,
+          a.binaries,
+          a.zipfiles,
+          a.datas,
+          name='instaloader',
+          debug=False,
+          strip=False,
+          upx=False,
+          runtime_tmpdir=None,
+          console=True , icon='favicon.ico')

docs/_static/instaloader.css

@@ -1,75 +0,0 @@
-@import url(bootstrap-4.1.3.min.css);
-a {
-  color: #008d06; }
-  a:hover, a.current, a.active {
-    color: #f48400; }
-  a.current, a.active {
-    font-weight: 500; }
-
-.doc-sidebar a:hover, .modal-body a:hover {
-  text-decoration: none; }
-
-code {
-  color: #212529; }
-
-.highlight {
-  border-radius: 0.3em; }
-  .highlight pre {
-    padding: 0.7em;
-    color: #fff; }
-  .highlight .c1 {
-    color: #008d06; }
-  .highlight .k, .highlight .kn, .highlight .ow {
-    color: #f48400;
-    font-weight: bold; }
-  .highlight .nb {
-    color: #f48400; }
-  .highlight .s1, .highlight .s2, .highlight .mi {
-    color: #ff7efd; }
-
-.doc-content {
-  padding-top: 0.5em;
-  top: 3.5625em;
-  position: fixed;
-  bottom: 0;
-  overflow-y: auto; }
-
-.doc-sidebar {
-  position: fixed;
-  padding-top: 0.5em;
-  top: 3.5625em;
-  bottom: 0;
-  right: 0;
-  overflow-y: auto; }
-  .doc-sidebar ul {
-    list-style-type: none;
-    padding-left: 1em; }
-    .doc-sidebar ul.current {
-      padding-left: 0; }
-  .doc-sidebar li {
-    padding-top: 0.2em;
-    padding-bottom: 0.2em; }
-
-.admonition {
-  border-radius: 0.3em;
-  border: 1px solid #ccc;
-  background-color: #f8f9fa; }
-  .admonition .admonition-title {
-    color: #ff7efd; }
-
-.topic {
-  border-radius: 0.3em;
-  background-color: #f8f9fa; }
-
-.nav-link {
-  padding-top: 0;
-  padding-bottom: 0; }
-
-.btn-success {
-  background-color: #008d06;
-  border-color: #008d06; }
-  .btn-success:hover {
-    background-color: #f48400;
-    border-color: #f48400; }
-
-/*# sourceMappingURL=instaloader.css.map */

docs/_static/instaloader.js

@@ -1,22 +0,0 @@
-$(function () {
-    $('[data-toggle="tooltip"]').tooltip();
-
-    $('.doc-sidebar > ul > li.current').attr("id", "localtoc");
-    $('#localtoc ul').addClass("nav flex-column");
-    $('#localtoc ul li').addClass("nav-item");
-    $('#localtoc ul li a').addClass("nav-link");
-    $('.doc-content').scrollspy({target: '#localtoc'});
-
-    const top_href = '#' + $('.section:first').attr("id");
-    $('#localtoc > a.current').attr("href", top_href);
-
-    $('#navbarToc a').on("click", function () {
-        const href = $(this).attr("href");
-        if (href === '#') {
-            window.location.href = top_href;
-        } else {
-            window.location.href = href;
-        }
-        $('#navbarToc').modal('hide');
-    });
-});

docs/_static/instaloader.scss

@@ -1,117 +0,0 @@
-@import 'bootstrap-4.1.3.min.css';
-
-$color_instaloader_main: #008d06;
-$color_instaloader_accent: #f48400;
-$color_instaloader_hyper: #ff7efd;
-$color_normal_text: #212529;
-$color_light_bg: #f8f9fa;
-
-a {
-  color: $color_instaloader_main;
-
-  &:hover, &.current, &.active {
-    color: $color_instaloader_accent;
-  }
-
-  &.current, &.active {
-    font-weight: 500;
-  }
-}
-
-.doc-sidebar, .modal-body {
-  a:hover {
-    text-decoration: none
-  }
-}
-
-code {
-  color: $color_normal_text;
-}
-
-.highlight {
-  border-radius: 0.3em;
-
-  pre {
-    padding: 0.7em;
-    color: #fff;
-  }
-
-  .c1 {
-    color: $color_instaloader_main
-  }
-
-  .k, .kn, .ow {
-    color: $color_instaloader_accent;
-    font-weight: bold
-  }
-
-  .nb {
-    color: $color_instaloader_accent
-  }
-
-  .s1, .s2, .mi {
-    color: $color_instaloader_hyper
-  }
-}
-
-.doc-content {
-  padding-top: 0.5em;
-  top: 3.5625em;
-  position: fixed;
-  bottom: 0;
-  overflow-y: auto;
-}
-
-.doc-sidebar {
-  position: fixed;
-  padding-top: 0.5em;
-  top: 3.5625em;
-  bottom: 0;
-  right: 0;
-  overflow-y: auto;
-
-  ul {
-    list-style-type: none;
-    padding-left: 1em;
-
-    &.current {
-      padding-left: 0;
-    }
-  }
-
-  li {
-    padding-top: 0.2em;
-    padding-bottom: 0.2em;
-
-  }
-}
-
-.admonition {
-  border-radius: 0.3em;
-  border: 1px solid #ccc;
-  background-color: $color_light_bg;
-
-  .admonition-title {
-    color: $color_instaloader_hyper;
-  }
-}
-
-.topic {
-  border-radius: 0.3em;
-  background-color: $color_light_bg;
-}
-
-.nav-link {
-  padding-top: 0;
-  padding-bottom: 0;
-}
-
-.btn-success {
-  background-color: $color_instaloader_main;
-  border-color: $color_instaloader_main;
-
-  &:hover {
-    background-color: $color_instaloader_accent;
-    border-color: $color_instaloader_accent;
-  }
-}

docs/_templates/layout.html

@@ -1,9 +1,6 @@
 {% extends "!layout.html" %}
-
 {% set html5_doctype = True %}
-
-{% set script_files = script_files + ['_static/bootstrap-4.1.3.bundle.min.js', '_static/instaloader.js'] %}
-
+{% set script_files = script_files + ['_static/instaloaderdoc.js'] %}
 {% block htmltitle %}
   {% if pagename == "index" %}
     <title>{{ title|striptags|e }}</title>
@@ -11,7 +8,6 @@
     <title>{{ title|striptags|e }}{{ titlesuffix }}</title>
   {% endif %}
 {% endblock %}
-
 {% block extrahead %}
   {{ super() }}
   <meta name="viewport" content="width=device-width, initial-scale=1, shrink-to-fit=no">
@@ -22,97 +18,66 @@
     <link rel="canonical" href="https://instaloader.github.io/{{ pagename }}.html"/>
   {% endif %}
 {% endblock %}
-
 {% block header %}
-  <nav class="navbar navbar-expand-sm navbar-light bg-light fixed-top border-bottom">
+  <nav class="navbar navbar-light navbar-expand bg-light border-bottom">
     <a class="navbar-brand" href="{{ pathto(master_doc) }}">
       <img src="{{ pathto('_static/' + logo, 1) }}" width="30" height="30" class="d-inline-block align-top" alt="logo">
-      Instaloader
+      <span class="ml-3">Instaloader</span>
     </a>
     <ul class="navbar-nav mr-auto">
-      <li class="nav-item ml-0 ml-sm-2 ml-md-0">
-        <a target="_blank" class="nav-link"
+      <li class="nav-item ml-0 ml-sm-2">
+        <a class="nav-link"
            href="https://github.com/instaloader/instaloader/releases/tag/v{{ current_release }}"
-           title="Released on {{ current_release_date }}" data-toggle="tooltip">v<b>{{ current_release }}</b></a>
+           title="Released on {{ current_release_date }}" data-toggle="tooltip">
+          v<b>{{ current_release }}</b>
+        </a>
       </li>
     </ul>
-    <button class="navbar-toggler" type="button" data-toggle="collapse" data-target="#navbarLinks"
-            aria-controls="navbarLinks" aria-expanded="false" aria-label="Toggle navigation">
+    <ul class="navbar-nav ml-auto d-none d-sm-flex">
+      <li class="nav-item">
+        <a class="nav-link" href="https://github.com/instaloader/instaloader/issues">
+          Issues
+        </a>
+      </li>
+      <li class="nav-item">
+        <a class="nav-link" href="https://github.com/instaloader/instaloader/releases">
+          Releases
+        </a>
+      </li>
+      <li class="nav-item">
+        <a class="nav-link" href="https://github.com/instaloader/instaloader">
+          GitHub
+        </a>
+      </li>
+    </ul>
+    <button class="d-inline-block d-md-none btn btn-outline-primary btn-sm" type="button"
+            data-toggle="collapse" data-target="#instaloaderdocSidebar" aria-controls="instaloaderdocSidebar"
+            aria-expanded="false" aria-label="Toggle navigation">
       <span class="navbar-toggler-icon"></span>
     </button>
-    <div class="collapse navbar-collapse" id="navbarLinks">
-      <ul class="navbar-nav ml-auto">
-        <li class="nav-item">
-          <a target="_blank" class="nav-link" href="https://github.com/instaloader/instaloader/issues">Issues</a>
-        </li>
-        <li class="nav-item">
-          <a target="_blank" class="nav-link" href="https://github.com/instaloader/instaloader/releases">Releases</a>
-        </li>
-        <li class="nav-item">
-          <a target="_blank" class="nav-link" href="https://github.com/instaloader/instaloader">GitHub</a>
-        </li>
-        <li class="nav-item d-sm-none d-lg-block">
-          <a target="_blank" class="nav-link" href="https://saythanks.io/to/aandergr">Say Thanks!</a>
-        </li>
-      </ul>
-      <form class="form-inline" action="search.html" method="get">
-        <div class="input-group my-2 my-sm-0 ml-2">
-          <input class="form-control" type="search" placeholder="Search" aria-label="Search" name="q">
-          <div class="input-group-append">
-            <button class="btn btn-success btn-small" type="submit">Go</button>
-          </div>
-        </div>
-      </form>
-    </div>
-    {% if pagename != "index" %}
-      <button type="button" class="btn btn-success d-md-none btn-small ml-2" data-toggle="modal"
-              data-target="#navbarToc">
-        TOC
-      </button>
-    {% endif %}
   </nav>
-  {% if pagename != "index" %}
-    <div class="modal fade" id="navbarToc" tabindex="-1" role="dialog"
-         aria-hidden="true" aria-labelledby="navbarTocTitle">
-      <div class="modal-dialog modal-dialog-centered" role="document">
-        <div class="modal-content">
-          <div class="modal-header">
-            <h5 class="modal-title" id="navbarTocTitle">Contents</h5>
-            <button type="button" class="close" data-dismiss="modal" aria-label="Close">
-              <span aria-hidden="true">&times;</span>
-            </button>
-          </div>
-          <div class="modal-body">
-            {{ toctree(maxdepth=-1) }}
-          </div>
-        </div>
-      </div>
-    </div>
-  {% endif %}
 {% endblock %}
-
 {% block relbar1 %}{% endblock %}
 {% block relbar2 %}{% endblock %}
 {% block footer %}{% endblock %}
 {% block content %}
-  <div class="container-fluid row">
-    <div class="col-12 {% if pagename != "index" %}col-md-9 col-xl-10{% endif %} doc-content">
-      {% block body %}{% endblock %}
-    </div>
-    {% if pagename != "index" %}
-      <div class="d-none col-md-3 col-xl-2 bg-light doc-sidebar d-md-flex flex-column border-left">
-
-        <h3>{{ _('Table of Contents') }}</h3>
-        {{ toctree(maxdepth=-1) }}
-
-        <div class="d-flex justify-content-between small mt-auto mb-2">
-          <a class="reference internal{% if pagename == "genindex" %} current{% endif %}" href="genindex.html">Index</a>
-          {% if pagename != "genindex" and pagename != "search" %}
-            <a class="reference external"
-               href="https://github.com/instaloader/instaloader/edit/master/docs/{{ pagename }}.rst">Edit this page</a>
-          {% endif %}
-        </div>
+  <div class="container-fluid">
+    <div class="row">
+      <div class="instaloaderdoc-sidebar collapse d-md-block col-md-3 col-xl-2 bg-light"
+           id="instaloaderdocSidebar">
+        <nav role="navigation" aria-label="Table of Contents">
+          <form action="{{ pathto('search') }}" method="get" role="search" class="border-bottom mr-2 pb-2 mb-2 px-1">
+            <label class="sr-only" for="searchQueryInput">Search</label>
+            <input type="search" name="q" id="searchQueryInput" class="form-control my-2"
+                   placeholder="Search..." aria-label="Search">
+          </form>
+          <h4>Contents</h4>
+          {{ toctree(maxdepth=-1) }}
+        </nav>
       </div>
-    {% endif %}
+      <main class="col-12 col-md-9 col-xl-10 pt-2" role="main">
+        {% block body %}{% endblock %}
+      </main>
+    </div>
   </div>
-{% endblock %}
+{% endblock %}

docs/as-module.rst

@@ -12,11 +12,8 @@ Python Module :mod:`instaloader`
 
 .. highlight:: python
 
-.. contents::
-  :backlinks: none
-
 Instaloader exposes its internally used methods and structures as a Python
-module, making it a **powerful and intuitive Python API for Instagram**,
+module, making it a powerful and intuitive Python API for Instagram,
 allowing to further customize obtaining media and metadata.
 
 Start with getting an instance of :class:`Instaloader`::
@@ -37,7 +34,7 @@ picture, video or sidecar (set of multiple pictures/videos) posted in a user's
 profile. :class:`Instaloader` provides methods to iterate over Posts from a
 certain source::
 
-    for post in L.get_hashtag_posts('cat'):
+    for post in instaloader.Hashtag.from_name(L.context, 'cat').get_posts():
         # post is an instance of instaloader.Post
         L.download_post(post, target='#cat')
 
@@ -63,7 +60,7 @@ certain source::
 - :meth:`Instaloader.get_explore_posts`
    Media that is suggested by Instagram to explore.
 
-- :meth:`Instaloader.get_hashtag_posts`
+- :meth:`Hashtag.get_posts`
    Media associated with given hashtag.
 
 With the :class:`Profile` class, Instaloader also makes it easy to access
@@ -85,128 +82,26 @@ metadata of a Profile. :class:`Profile` instances can be created with:
 - :meth:`Profile.get_followers`
    Profiles that follow given user.
 
-- :attr:`Post.owner_profile`, :attr:`Story.owner_profile` and :attr:`StoryItem.owner_profile`
+- :attr:`Post.owner_profile`, :attr:`Story.owner_profile`, :attr:`StoryItem.owner_profile`
    Owner profile of particular object.
 
 - :meth:`Post.get_likes`
    Profiles that liked a given :class:`Post`
 
-- :attr:`PostComment.owner` attribute for comment in :meth:`Post.get_comments`
+- :attr:`PostComment.owner`
    Profile of a Post comment.
 
-A reference of the many methods provided by the :mod:`instaloader` module is
-provided in the remainder of this document.
+For a list of a few code examples that use the Instaloader module, see
+:ref:`codesnippets`.
 
-For a list of real code examples using the Instaloader module for advanced
-tasks, see :ref:`codesnippets`.
+The reference of the classes and functions provided by the :mod:`instaloader` module is
+divided into the following subsections:
 
-``Instaloader`` (Main Class)
-^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+.. toctree::
+   :maxdepth: 2
 
-.. autoclass:: Instaloader
-   :no-show-inheritance:
-
-Instagram Structures
-^^^^^^^^^^^^^^^^^^^^
-
-Posts
-"""""
-
-.. autoclass:: Post
-   :no-show-inheritance:
-
-Additionally, the following trivial structures are defined:
-
-.. autoclass:: PostSidecarNode
-   :no-show-inheritance:
-
-.. autoclass:: PostComment
-   :no-show-inheritance:
-
-.. autoclass:: PostCommentAnswer
-   :no-show-inheritance:
-
-.. autoclass:: PostLocation
-   :no-show-inheritance:
-
-User Stories
-""""""""""""
-
-.. autoclass:: Story
-   :no-show-inheritance:
-
-.. autoclass:: StoryItem
-   :no-show-inheritance:
-
-Highlights
-""""""""""
-
-.. autoclass:: Highlight
-   :no-show-inheritance:
-   :inherited-members:
-
-   Bases: :class:`Story`
-
-   .. versionadded:: 4.1
-
-Profiles
-""""""""
-
-.. autoclass:: Profile
-   :no-show-inheritance:
-
-Loading and Saving
-""""""""""""""""""
-
-:class:`Post`, :class:`StoryItem` and :class:`Profile` can be saved and loaded
-to/from JSON files.
-
-.. autofunction:: load_structure_from_file
-
-.. autofunction:: save_structure_to_file
-
-Exceptions
-^^^^^^^^^^
-
-.. currentmodule:: instaloader.exceptions
-
-.. autoexception:: InstaloaderException
-   :no-show-inheritance:
-
-.. autoexception:: ConnectionException
-
-.. currentmodule:: instaloader
-
-.. autoexception:: QueryReturnedBadRequestException
-
-.. autoexception:: QueryReturnedNotFoundException
-
-.. autoexception:: QueryReturnedForbiddenException
-
-.. autoexception:: ProfileNotExistsException
-
-.. autoexception:: ProfileHasNoPicsException
-
-.. autoexception:: PrivateProfileNotFollowedException
-
-.. autoexception:: LoginRequiredException
-
-.. autoexception:: TwoFactorAuthRequiredException
-
-   .. versionadded:: 4.2
-
-.. autoexception:: InvalidArgumentException
-
-.. autoexception:: BadResponseException
-
-.. autoexception:: BadCredentialsException
-
-.. autoexception:: PostChangedException
-
-.. autoexception:: TooManyRequestsException
-
-``InstaloaderContext`` (Low-level functions)
-^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-
-.. autoclass:: InstaloaderContext
-   :no-show-inheritance:
+   module/instaloader
+   module/structures
+   module/nodeiterator
+   module/instaloadercontext
+   module/exceptions

docs/basic-usage.rst

@@ -51,7 +51,7 @@ invoke it with
     instaloader --login=your_username profile [profile ...]
 
 When logging in, Instaloader **stores the session cookies** in a file in your
-temporary directory, which will be reused later the next time :option:`--login`
+home directory, which will be reused later the next time :option:`--login`
 is given.  So you can download private profiles **non-interactively** when you
 already have a valid session cookie file.
 
@@ -77,38 +77,46 @@ Instaloader supports the following targets:
       instructs Instaloader to also **download the user's stories**,
 
    - :option:`--highlights`
-      to **download highlights of each profile that is downloaded**, and
+      to **download the highlights of that profile**,
 
    - :option:`--tagged`
-      to **download posts where the user is tagged**.
+      to **download posts where the user is tagged**, and
+
+   - :option:`--igtv`
+      to **download IGTV videos**.
 
 - ``"#hashtag"``
-   Posts with a certain **hashtag** (the quotes are usually necessary),
+   Posts with a certain **hashtag** (the quotes are usually necessary).
 
 - ``%location id``
    Posts tagged with a given location; the location ID is the numerical ID
    Instagram labels a location with (e.g.
    \https://www.instagram.com/explore/locations/**362629379**/plymouth-naval-memorial/).
+   Requires :option:`--login`.
 
    .. versionadded:: 4.2
 
 - ``:stories``
    The currently-visible **stories** of your followees (requires
-   :option:`--login`),
+   :option:`--login`).
 
 - ``:feed``
-   Your **feed** (requires :option:`--login`),
+   Your **feed** (requires :option:`--login`).
 
 - ``:saved``
-   Posts which are marked as **saved** (requires :option:`--login`),
+   Posts which are marked as **saved** (requires :option:`--login`).
 
 - ``@profile``
    All profiles that are followed by ``profile``, i.e. the *followees* of
    ``profile`` (requires :option:`--login`).
 
 - ``-post``
-   The single **post** with the given shortcode. Must be preceeded by ``--`` in
-   the argument list to not be mistaken as an option flag.
+   Replace **post** with the post's shortcode to download single post. Must be preceded by ``--`` in
+   the argument list to not be mistaken as an option flag::
+
+    instaloader -- -B_K4CykAOtf
+
+
 
    .. versionadded:: 4.1
 
@@ -122,7 +130,7 @@ downloads the pictures and videos and their captions. You can specify
 
 - :option:`--geotags`
    **download geotags** of each post and save them as
-   Google Maps link,
+   Google Maps link (requires :option:`--login`),
 
 For a reference of all supported command line options, see
 :ref:`command-line-options`.
@@ -163,6 +171,9 @@ dirname pattern. The following tokens are defined for usage with
 - ``{mediaid}``
    Integer representation of shortcode.
 
+- ``{filename}``
+   Instagram's internal filename.
+
 - ``{date_utc}`` (same as ``{date}``)
    Creation time in UTC timezone.
    `strftime()-style formatting options <https://docs.python.org/3/library/datetime.html#strftime-and-strptime-behavior>`__
@@ -188,7 +199,7 @@ Filter Posts
 .. py:currentmodule:: instaloader
 
 The options :option:`--post-filter` and :option:`--storyitem-filter`
-allows to specify criteria that posts or story items have to
+allow to specify criteria that posts or story items have to
 meet to be downloaded. If not given, all posts are downloaded.
 
 The filter string must be a
@@ -200,7 +211,7 @@ Id est, the following attributes can be used with both
 :option:`--post-filter` and :option:`--storyitem-filter`:
 
 - :attr:`~Post.owner_username` (str), :attr:`~Post.owner_id` (int)
-   Owner profile username / userid.
+   Owner profile username / user ID.
 
 - :attr:`~Post.date_utc` (datetime), :attr:`~Post.date_local` (datetime)
    Creation timestamp. Since :class:`~datetime.datetime` objects can be created
@@ -209,7 +220,7 @@ Id est, the following attributes can be used with both
       instaloader --post-filter="date_utc <= datetime(2018, 5, 31)" target
 
 - :attr:`~Post.is_video` (bool)
-   Post/StoryItem is a video. For example, you may skip videos::
+   Whether Post/StoryItem is a video. For example, you may skip videos::
 
       instaloader --post-filter="not is_video" target
 
@@ -226,8 +237,8 @@ As :option:`--post-filter`, the following attributes can be used additionally:
       instaloader --login=your_username --post-filter=viewer_has_liked :feed
 
 - :attr:`~Post.likes` (int), :attr:`~Post.comments` (int)
-   Likes count / Comments count. You might only want to download posts that
-   either you liked or were liked by many others**::
+   Likes count / comments count. You might only want to download posts that
+   were either liked by yourself or by many others::
 
       instaloader --login=your_username --post-filter="likes>100 or viewer_has_liked" profile
 
@@ -257,7 +268,7 @@ along with each post where the Post's caption is saved.
 You can customize what metadata to save for each Post or StoryItem with
 :option:`--post-metadata-txt` and :option:`--storyitem-metadata-txt`. The
 default is ``--post-metadata-txt={caption}`` and no storyitem metadata txt.
-These strings are formatted similar as in the :ref:`filename-specification` and
+These strings are formatted similar as the path patterns described in :ref:`filename-specification` and
 the result is saved in text files, unless it is empty.
 
 Specifying these options multiple times results in output having multiple lines,
@@ -287,17 +298,18 @@ Instaloader as Cronjob
 Instaloader is suitable for running as a cronjob to periodically update your
 personal Instagram archive. The :option:`--quiet` option disables user
 interactions and logging of non-error messages. To non-interactively use
-Instaloader logged-in, create a session file, e.g. in your home directory::
+Instaloader logged-in, create a session file::
 
-   instaloader --login=your_username --sessionfile=~/.instaloader-session
+   instaloader --login=your_username
 
-Then use the same parameters in your cronjob to load the session and download
+Then use the same username in your cronjob to load the session and download
 the given targets::
 
-   instaloader --login=your_username --sessionfile=~/.instaloader-session --quiet <target> [...]
+   instaloader --login=your_username --quiet target [...]
 
-Without :option:`--sessionfile` option, Instaloader saves the session file in
-a path within your temporary directory.
+Instaloader saves the session file to
+``~/.config/instaloader/session-YOUR-USERNAME``. See
+:option:`--sessionfile` option for how to override this path.
 
 Programming Instaloader
 ^^^^^^^^^^^^^^^^^^^^^^^
@@ -308,5 +320,6 @@ Instaloader exposes its internally used methods and structures, making it a
 powerful and intuitive Python API for Instagram, allowing to further customize
 obtaining media and metadata.
 
-Also see :ref:`codesnippets`, where we collect example scripts that use
-Instaloader to achieve more complex tasks.
+Also see :ref:`codesnippets`, where we collect a few example scripts that use
+Instaloader for simple tasks that cannot be done with the command line
+interface.

docs/cli-options.rst

@@ -36,7 +36,7 @@ stores all posts along with the pictures's captions there.
    Re-Download the given object
 
 - ``+args.txt``
-   Read targets (and options) from given textfile. See :option:`+args.txt`.
+   Read targets (and options) from given text file. See :option:`+args.txt`.
 
 What to Download of each Post
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
@@ -59,10 +59,9 @@ What to Download of each Post
 
 .. option:: --geotags, -G
 
-   **Download geotags** when available. Geotags are stored as a text file with
+   Download geotags when available. Geotags are stored as a text file with
    the location's name and a Google Maps link. This requires an additional
-   request to the Instagram server for each picture, which is why it is disabled
-   by default.
+   request to the Instagram server for each picture. Requires :option:`--login`.
 
 .. option:: --comments, -C
 
@@ -83,6 +82,16 @@ What to Download of each Post
    Template to write in txt file for each StoryItem. See
    :ref:`metadata-text-files`.
 
+.. option:: --slide
+
+   Download only selected images of a sidecar. You can select single images using their
+   index in the sidecar starting with the leftmost or you can specify a range of images
+   with the following syntax: ``start_index-end_index``. Example:
+   ``--slide 1`` will select only the first image, ``--slide last`` only the last one and ``--slide 1-3`` will select only
+   the first three images.
+
+   .. versionadded:: 4.6
+
 .. option:: --no-metadata-json
 
    Do not create a JSON file containing the metadata of each post.
@@ -95,13 +104,6 @@ What to Download of each Post
 What to Download of each Profile
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
-.. option:: --profile-pic-only, -P
-
-   .. deprecated:: 4.1
-      Use :option:`--no-posts`.
-
-   Only download profile picture.
-
 .. option:: --no-posts
 
    Do not download regular posts.
@@ -114,12 +116,12 @@ What to Download of each Profile
 
 .. option:: --stories, -s
 
-   Also **download stories** of each profile that is downloaded. Requires
+   Also download stories of each profile that is downloaded. Requires
    :option:`--login`.
 
 .. option:: --highlights
 
-   Also **download highlights** of each profile that is downloaded. Requires
+   Also download highlights of each profile that is downloaded. Requires
    :option:`--login`.
 
    .. versionadded:: 4.1
@@ -130,19 +132,11 @@ What to Download of each Profile
 
    .. versionadded:: 4.1
 
-.. option:: --stories-only
+.. option:: --igtv
 
-   .. deprecated:: 4.1
-      Use :option:`--stories` :option:`--no-posts`.
+   Also download IGTV videos.
 
-   Rather than downloading regular posts of each specified profile, only
-   download stories.  Requires :option:`--login`. Does not imply
-   :option:`--no-profile-pic`.
-
-   .. note::
-
-      If possible, use ``:stories`` target rather than :option:`--stories-only`
-      with all your followees. ``:stories`` uses fewer API requests.
+   .. versionadded:: 4.3
 
 Which Posts to Download
 ^^^^^^^^^^^^^^^^^^^^^^^
@@ -177,7 +171,7 @@ Which Posts to Download
 Login (Download Private Profiles)
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
-Instaloader can **login to Instagram**. This allows downloading private
+Instaloader can login to Instagram. This allows downloading private
 profiles. To login, pass the :option:`--login` option. Your session cookie (not your
 password!) will be saved to a local file to be reused next time you want
 Instaloader to login.
@@ -188,9 +182,8 @@ Instaloader to login.
 
 .. option:: --sessionfile SESSIONFILE, -f SESSIONFILE
 
-   Path for loading and storing session key file.  Defaults to a path within
-   your temporary directory, encoding your local username and your Instagram
-   profile name.
+   Path for loading and storing session key file.  Defaults to
+   ``~/.config/instaloader/session-YOUR-USERNAME``.
 
 .. option:: --password YOUR-PASSWORD, -p YOUR-PASSWORD
 
@@ -219,6 +212,29 @@ How to Download
    ``#hashtag`` or the profile name. Defaults to ``{date_utc}_UTC``.
    See :ref:`filename-specification` for a list of supported tokens.
 
+.. option:: --resume-prefix prefix
+
+   For many targets, Instaloader is capable of resuming a previously-aborted
+   download loop.  To do so, it creates a JSON file within the target directory
+   when interrupted.  This option controls the prefix for filenames that are
+   used to save the information to resume an interrupted download.  The default
+   prefix is ``iterator``.
+
+   Resuming an interrupted download is supported for most, but not all targets.
+   JSON files with resume information are always compressed, regardless of
+   :option:`--no-compress-json`.
+
+   This feature is turned off entirely with :option:`--no-resume`.
+
+   .. versionadded:: 4.5
+
+.. option:: --no-resume
+
+   Do not resume a previously-aborted download iteration, and do not save such
+   information when interrupted.
+
+   .. versionadded:: 4.5
+
 .. option:: --user-agent USER_AGENT
 
    User Agent to use for HTTP requests. Per default, Instaloader pretends being
@@ -230,13 +246,14 @@ How to Download
    to ``3``. If a connection fails, it can be manually skipped by hitting
    :kbd:`Control-c`. Set this to ``0`` to retry infinitely.
 
-.. option:: --commit-mode
+.. option:: --request-timeout N
 
-   Tries to ensure downloaded images avoid corruption in case of unexpected
-   interruption. If the last picture is corrupted, Instaloader will fix the
-   picture the next time it is run.
+   Seconds to wait before timing out a connection request. Defaults to 300.
 
-   .. versionadded:: 4.2
+   .. versionadded:: 4.3
+
+   .. versionchanged:: 4.6
+      Enabled this option by default with a timeout of 300 seconds.
 
 Miscellaneous Options
 ^^^^^^^^^^^^^^^^^^^^^
@@ -251,18 +268,18 @@ Miscellaneous Options
 
    Read arguments from file `args.txt`, a shortcut to provide arguments from
    file rather than command-line. This provides a convenient way to hide login
-   info from CLI, and can also be used to simplify managment of long arguments.
+   info from CLI, and can also be used to simplify management of long arguments.
 
    .. note::
 
       Text file should separate arguments with line breaks.
 
-      args.txt example::
+   args.txt example::
 
-         --login=MYUSERNAME
-         --password=MYPASSWORD
-         --fast-update
-         profile1
-         profile2
+      --login=MYUSERNAME
+      --password=MYPASSWORD
+      --fast-update
+      profile1
+      profile2
 
    .. versionadded:: 4.1

docs/codesnippets.rst

@@ -28,8 +28,9 @@ Download Posts in a Specific Period
 -----------------------------------
 
 To only download Instagram pictures (and metadata) that are within a specific
-period, you can play around with :func:`~itertools.dropwhile` and
-:func:`~itertools.takewhile` from :mod:`itertools` like in this snippet.
+period, you can simply use :func:`~itertools.dropwhile` and
+:func:`~itertools.takewhile` from :mod:`itertools` on a generator that returns
+Posts in **exact chronological order**, such as :meth:`Profile.get_posts`.
 
 .. literalinclude:: codesnippets/121_since_until.py
 
@@ -37,11 +38,19 @@ See also :class:`Post`, :meth:`Instaloader.download_post`.
 
 Discussed in :issue:`121`.
 
+The code example with :func:`~itertools.dropwhile` and
+:func:`~itertools.takewhile` makes the assumption that the post iterator returns
+posts in exact chronological order.  As discussed in :issue:`666`, the following
+approach fits for an **almost chronological order**, where up to *k* older posts
+are inserted into an otherwise chronological order, such as a Hashtag feed.
+
+.. literalinclude:: codesnippets/666_historical_hashtag_data.py
+
 Likes of a Profile / Ghost Followers
 ------------------------------------
 
 To obtain a list of your inactive followers, i.e. followers that did not like
-any of your pictures, into a file you can use this approach.
+any of your pictures, you can use this approach.
 
 .. literalinclude:: codesnippets/120_ghost_followers.py
 
@@ -69,11 +78,11 @@ Discussed in :issue:`56`.
 Only one Post per User
 ----------------------
 
-To download only the one most recent post from each user, this snippet creates a
-:class:`set` that contains the users of which a post has already been
-downloaded. While iterating the posts, it checks whether the post's owner
-already is in the set. If not, the post is downloaded from Instagram and the
-user is added to that set.
+To download only the single most recent post per user within a hashtag feed,
+this snippet uses a :class:`set` that contains the users of whom a post has
+already been downloaded. For each post, it checks whether the post's creator is
+already contained in that set. If not, the post is downloaded from Instagram and
+the user is added to that set.
 
 .. literalinclude:: codesnippets/113_only_one_per_user.py
 
@@ -85,37 +94,12 @@ Discussed in :issue:`113`.
 Top X Posts of User
 -------------------
 
-With Instaloader, it is easy to download the few most-liked pictres of a user.
+With Instaloader, it is easy to download the few most-liked pictures of a user.
 
 .. literalinclude:: codesnippets/194_top_x_of_user.py
 
 Discussed in :issue:`194`.
 
-Upgrade Images by Local Copies
-------------------------------
-
-The following script finds local versions of images fetched by Instaloader, in
-order to upgrade the downloaded images by locally-found versions with better
-quality. It uses image hashing to identify similar images.
-
-`updgrade-instaloader-images.py <https://gist.github.com/pavelkryukov/15f93d19a99428a284a8bcec27e0187b>`__ (external link to GitHub Gist)
-
-Discussed in :issue:`46`.
-
-Add Captions to Images
-----------------------
-
-Instaloader does not modify the downloaded JPEG file. However, one could combine
-it with an imaging library such as Pillow or PIL to render the caption on
-Instagram pictures.  The following shows an approach.
-
-.. literalinclude:: codesnippets/110_pil_captions.py
-
-See also :attr:`Post.caption`, :attr:`Post.url`, :meth:`Post.from_shortcode`,
-:func:`load_structure_from_file`.
-
-Discussed in :issue:`110`.
-
 Metadata JSON Files
 -------------------
 

docs/codesnippets/110_pil_captions.py

@@ -1,20 +0,0 @@
-from io import BytesIO
-
-from requests import get
-from PIL import Image, ImageDraw
-from instaloader import *
-
-L = Instaloader()
-
-# Load Post instance
-post = load_structure_from_file(L.context, '2017-10-01_18-53-03_UTC.json.xz')
-# or post = Post.from_shortcode(L.context, SHORTCODE)
-
-# Render caption
-image = Image.open(BytesIO(get(post.url).content))
-draw = ImageDraw.Draw(image)
-color = 'rgb(0, 0, 0)'  # black color
-draw.text((300,100), post.caption.encode('latin1', errors='ignore'), fill=color)
-
-# Save image
-image.save('test.jpg')

docs/codesnippets/113_only_one_per_user.py

@@ -2,13 +2,13 @@ import instaloader
 
 L = instaloader.Instaloader()
 
-posts = L.get_hashtag_posts('milfgarden')
+posts = instaloader.Hashtag.from_name(L.context, 'urbanphotography').get_posts()
 
 users = set()
 
 for post in posts:
     if not post.owner_profile in users:
-        L.download_post(post, '#milfgarden')
+        L.download_post(post, '#urbanphotography')
         users.add(post.owner_profile)
     else:
         print("{} from {} skipped.".format(post, post.owner_profile))

docs/codesnippets/120_ghost_followers.py

@@ -2,26 +2,26 @@ import instaloader
 
 L = instaloader.Instaloader()
 
-USER = 'your_account'
+USER = "your_account"
 PROFILE = USER
 
-# Your preferred way of logging in:
+# Load session previously saved with `instaloader -l USERNAME`:
 L.load_session_from_file(USER)
 
 profile = instaloader.Profile.from_username(L.context, PROFILE)
 
 likes = set()
-print('Fetching likes of all posts of profile {}.'.format(profile.username))
+print("Fetching likes of all posts of profile {}.".format(profile.username))
 for post in profile.get_posts():
     print(post)
     likes = likes | set(post.get_likes())
 
-print('Fetching followers of profile {}.'.format(profile.username))
+print("Fetching followers of profile {}.".format(profile.username))
 followers = set(profile.get_followers())
 
 ghosts = followers - likes
 
-print('Storing ghosts into file.')
-with open('/YOUR PATH/inactive-users.txt', 'w') as f:
+print("Storing ghosts into file.")
+with open("inactive-users.txt", 'w') as f:
     for ghost in ghosts:
         print(ghost.username, file=f)

docs/codesnippets/121_since_until.py

@@ -5,13 +5,11 @@ import instaloader
 
 L = instaloader.Instaloader()
 
-posts = L.get_hashtag_posts('milfgarden')
-# or
-# posts = instaloader.Profile.from_username(L.context, PROFILE).get_posts()
+posts = instaloader.Profile.from_username(L.context, "instagram").get_posts()
 
 SINCE = datetime(2015, 5, 1)
 UNTIL = datetime(2015, 3, 1)
 
 for post in takewhile(lambda p: p.date > UNTIL, dropwhile(lambda p: p.date > SINCE, posts)):
     print(post.date)
-    L.download_post(post, '#milfgarden')
+    L.download_post(post, "instagram")

docs/codesnippets/194_top_x_of_user.py

@@ -9,7 +9,9 @@ X_percentage = 10    # percentage of posts that should be downloaded
 L = Instaloader()
 
 profile = Profile.from_username(L.context, PROFILE)
-posts_sorted_by_likes = sorted(profile.get_posts(), key = lambda p: p.likes + p.comments)
+posts_sorted_by_likes = sorted(profile.get_posts(),
+                               key=lambda p: p.likes + p.comments,
+                               reverse=True)
 
 for post in islice(posts_sorted_by_likes, ceil(profile.mediacount * X_percentage / 100)):
     L.download_post(post, PROFILE)

docs/codesnippets/615_import_firefox_session.py

@@ -0,0 +1,53 @@
+from argparse import ArgumentParser
+from glob import glob
+from os.path import expanduser
+from platform import system
+from sqlite3 import OperationalError, connect
+
+try:
+    from instaloader import ConnectionException, Instaloader
+except ModuleNotFoundError:
+    raise SystemExit("Instaloader not found.\n  pip install [--user] instaloader")
+
+
+def get_cookiefile():
+    default_cookiefile = {
+        "Windows": "~/AppData/Roaming/Mozilla/Firefox/Profiles/*/cookies.sqlite",
+        "Darwin": "~/Library/Application Support/Firefox/Profiles/*/cookies.sqlite",
+    }.get(system(), "~/.mozilla/firefox/*/cookies.sqlite")
+    cookiefiles = glob(expanduser(default_cookiefile))
+    if not cookiefiles:
+        raise SystemExit("No Firefox cookies.sqlite file found. Use -c COOKIEFILE.")
+    return cookiefiles[0]
+
+
+def import_session(cookiefile, sessionfile):
+    print("Using cookies from {}.".format(cookiefile))
+    conn = connect(f"file:{cookiefile}?immutable=1", uri=True)
+    try:
+        cookie_data = conn.execute(
+            "SELECT name, value FROM moz_cookies WHERE baseDomain='instagram.com'"
+        )
+    except OperationalError:
+        cookie_data = conn.execute(
+            "SELECT name, value FROM moz_cookies WHERE host LIKE '%instagram.com'"
+        )
+    instaloader = Instaloader(max_connection_attempts=1)
+    instaloader.context._session.cookies.update(cookie_data)
+    username = instaloader.test_login()
+    if not username:
+        raise SystemExit("Not logged in. Are you logged in successfully in Firefox?")
+    print("Imported session cookie for {}.".format(username))
+    instaloader.context.username = username
+    instaloader.save_session_to_file(sessionfile)
+
+
+if __name__ == "__main__":
+    p = ArgumentParser()
+    p.add_argument("-c", "--cookiefile")
+    p.add_argument("-f", "--sessionfile")
+    args = p.parse_args()
+    try:
+        import_session(args.cookiefile or get_cookiefile(), args.sessionfile)
+    except (ConnectionException, OperationalError) as e:
+        raise SystemExit("Cookie import failed: {}".format(e))

docs/codesnippets/666_historical_hashtag_data.py

@@ -0,0 +1,31 @@
+from datetime import datetime
+import instaloader
+
+L = instaloader.Instaloader()
+
+posts = instaloader.Hashtag.from_name(L.context, "urbanphotography").get_posts()
+
+SINCE = datetime(2020, 5, 10)  # further from today, inclusive
+UNTIL = datetime(2020, 5, 11)  # closer to today, not inclusive
+
+k = 0  # initiate k
+#k_list = []  # uncomment this to tune k
+
+for post in posts:
+    postdate = post.date
+
+    if postdate > UNTIL:
+        continue
+    elif postdate <= SINCE:
+        k += 1
+        if k == 50:
+            break
+        else:
+            continue
+    else:
+        L.download_post(post, "#urbanphotography")
+        # if you want to tune k, uncomment below to get your k max
+        #k_list.append(k)
+        k = 0  # set k to 0
+
+#max(k_list)

docs/codesnippets/92_import_firefox_session.py

@@ -1,23 +0,0 @@
-from glob import glob
-from os.path import expanduser
-from sqlite3 import connect
-
-from instaloader import ConnectionException, Instaloader
-
-# FIREFOXCOOKIEFILE = "/home/alex/.mozilla/firefox/l96w6b90.default/cookies.sqlite"
-FIREFOXCOOKIEFILE = glob(expanduser("~/.mozilla/firefox/*.default/cookies.sqlite"))[0]
-
-instaloader = Instaloader(max_connection_attempts=1)
-instaloader.context._session.cookies.update(connect(FIREFOXCOOKIEFILE)
-                                            .execute("SELECT name, value FROM moz_cookies "
-                                                     "WHERE baseDomain='instagram.com'"))
-
-try:
-    username = instaloader.test_login()
-    if not username:
-        raise ConnectionException()
-except ConnectionException:
-    raise SystemExit("Cookie import failed. Are you logged in successfully in Firefox?")
-
-instaloader.context.username = username
-instaloader.save_session_to_file()

docs/conf.py

@@ -18,6 +18,7 @@
 # documentation root, use os.path.abspath to make it absolute, like shown here.
 #
 import os
+import platform
 import subprocess
 import sys
 import typing
@@ -47,9 +48,27 @@ autodoc_default_flags = ['show-inheritance', 'members', 'undoc-members']
 autodoc_member_order = 'bysource'
 
 intersphinx_mapping = {'python': ('https://docs.python.org/3', None),
-                       'requests': ('http://docs.python-requests.org/en/master', None)}
+                       'requests': ('https://requests.kennethreitz.org/en/master/', None)}
 
-extlinks = {'issue': ('https://github.com/instaloader/instaloader/issues/%s', 'issue #')}
+nitpick_ignore = [('py:class', 'typing.Tuple')]
+
+current_release = subprocess.check_output(["git", "describe", "--abbrev=0"]).decode("ascii")[1:-1]
+date_format = "%e %b %Y" if platform.system() != "Windows" else "%d %b %Y"
+current_release_date = subprocess.check_output(
+    ["git", "log", "-1", "--tags", "--format=%ad", "--date=format:"+date_format]).decode("ascii")[:-1]
+
+html_context = {'current_release': current_release, 'current_release_date': current_release_date}
+
+extlinks = {
+    'issue': (
+        'https://github.com/instaloader/instaloader/issues/%s',
+        'Issue #'
+    ),
+    'example': (
+        'https://raw.githubusercontent.com/instaloader/instaloader/master/docs/codesnippets/%s',
+        'Example '
+    ),
+}
 
 # Add any paths that contain templates here, relative to this directory.
 templates_path = ['_templates']
@@ -69,7 +88,7 @@ master_doc = 'index'
 
 # General information about the project.
 project = 'Instaloader'
-copyright = '2017, Alexander Graf and Andre Koch-Kramer'
+copyright = '2017-2020, Alexander Graf and Andre Koch-Kramer'
 author = 'Alexander Graf and Andre Koch-Kramer'
 
 # The version info for the project you're documenting, acts as replacement for
@@ -353,14 +372,8 @@ texinfo_documents = [
 # texinfo_no_detailmenu = False
 
 
-current_release = subprocess.check_output(["git", "describe", "--abbrev=0"]).decode("ascii")[1:-1]
-current_release_date = subprocess.check_output(["git", "log", "-1", "--tags", "--format=%ad", "--date=format:%e %b %Y"]).decode("ascii")[:-1]
-
-html_context = {'current_release': current_release, 'current_release_date': current_release_date}
-
-
 def setup(app):
     typing.TYPE_CHECKING = True
-    app.add_stylesheet("instaloader.css")
+    app.add_stylesheet("instaloaderdoc.css")
     app.connect('autodoc-process-signature', sphinx_autodoc_typehints.process_signature)
     app.connect('autodoc-process-docstring', sphinx_autodoc_typehints.process_docstring)

docs/contributing.rst

@@ -11,14 +11,29 @@ Instaloader's development is organized on
 `GitHub <https://github.com/instaloader/instaloader>`__, where Issues and Pull
 Requests are discussed.
 
+Contributing to an open source project is an unmatchable opportunity to improve
+and manifest programming skills. For issues that are good starting points to get
+involved in Instaloader development, as they do not require deep insight into
+Instaloader's code base, have a look at our
+`Issues labelled 'good first issue' <https://github.com/instaloader/instaloader/contribute>`__.
+
+This document covers several ways of how you can contribute back to Instaloader.
+
+Answering Questions
+-------------------
+
+The easiest way to help out is to answer questions. If you are interested in
+answering questions regarding Instaloader, good places to begin are
+
+- `Questions tagged 'instaloader' on Stack Overflow <https://stackoverflow.com/questions/tagged/instaloader>`__,
+- `Instaloader Issues labeled 'question' <https://github.com/instaloader/instaloader/issues?q=is%3Aissue+is%3Aopen+label%3Aquestion>`__.
+
 Reporting Bugs
 --------------
 
 If you encounter a bug, do not hesitate to report it in our
-`Issue Tracker <https://github.com/instaloader/instaloader/issues>`__. Since
-Instaloader is actively developed, the majority of bugs is fixed within only
-**4 days** after being reported. When reporting a problem, please keep the
-following in mind:
+`Issue Tracker <https://github.com/instaloader/instaloader/issues>`__. When
+reporting a problem, please keep the following in mind:
 
 - Ensure you **use the latest version** of Instaloader. The currently-installed
   version can be found out with ``instaloader --version``.
@@ -34,42 +49,65 @@ following in mind:
 
 - Include all **error messages and tracebacks** in the report.
 
-- If not obvious, describe **which behavior you expected**
+- Even if it seems obvious, describe **which behavior you expected**
   instead of what actually happened.
 
-- If you think an issue has been closed accidentally or inappropriately, feel
-  free to reopen it.
+- If we have closed an issue apparently inadvertently or inappropriately, please
+  let us know.
 
 Writing Code or Improving the Documentation
 -------------------------------------------
 
-Changes of the Instaloader source can be proposed as a
-`Pull Request <https://github.com/instaloader/instaloader/pulls>`__. There are only
-few things to consider:
+Improvements of the Instaloader source or its documentation can be proposed as a
+`Pull Request <https://github.com/instaloader/instaloader/pulls>`__.
 
-- Sometimes, the most current code is not in the ``master`` branch. Check that
-  you forked from the most recent branch.
+- Please base your Pull Request on
 
-- We use `Pylint <https://www.pylint.org/>`__ for error and syntax checking of
-  the source. The file ``.travis.yml`` in the project's root directory
-  shows how it is invoked. Note that sometimes it might be better to disable a
-  warning rather than adapting the code to Pylint's desires.
+  - ``master``, which will be released with the next minor release, if it is
 
-- The documentation source is located in the ``docs`` folder. The file
-  ``cli-options.rst`` is merely an RST-formatted copy of ``instaloader --help``
-  output, of which the source is in ``instaloader/__main__.py``.
+    - a bug fix that does not require extensive testing,
+    - an improvement to the documentation,
 
-- Feel free to contact us, even if you "only" have Proof-of-Concepts or
-  not-fully integrated changes. They already might be an advance for the
-  project.
+  - ``upcoming/v4.X``, if it is
 
-Suggesting Features
--------------------
+    - a new feature,
+    - a bug fix that does require thorough testing before being released to a
+      final version.
 
-If you have an idea of how Instaloader should be enhanced, but do not want to
-implement the feature yourself, feel free to open a ticket in our 
-`Issue Tracker <https://github.com/instaloader/instaloader/issues>`__.
-Please consider the following:
+- All Pull Requests are analyzed by `Pylint <https://www.pylint.org/>`__ for
+  error and syntax checking of the source and
+  `Mypy <https://github.com/python/mypy>`__ for type checking. You can run them
+  locally with::
+
+     pylint instaloader
+     mypy -m instaloader
+
+- Improvements to the documentation are very welcome. The documentation is
+  created with `Sphinx <https://www.sphinx-doc.org/en/2.0/>`__, version 2,
+  and can be build locally using::
+
+     make -C docs html
+
+- Feel free to create an issue to make sure someone from the Instaloader team
+  agrees that the change might be an improvement, or if you want to discuss
+  your basic proposal, before working on a pull request.
+
+Proposing Features
+------------------
+
+.. goal-start
+
+Instaloader's goal is to mimic the browser's behavior to access the data that
+is available through the Instagram web interface, pack this data into complete
+and easily-(re)usable python objects, and provide a user interface for the most
+common downloading and metadata collection tasks, without changing any of the
+online data.
+
+.. goal-end
+
+Prior spending effort on implementing a new feature, it might be appropriate to
+clarify how it could fit into the project's scope or discuss implementation
+details. If you feel the need to do so, please create a "feature suggestion".
 
 - Instaloader already has plenty of features. **Check the documentation**
   beforehand to ensure your desired suggestion is not already implemented.
@@ -78,13 +116,21 @@ Please consider the following:
   an issue suggesting the same or a similar feature, share your thoughts in a
   comment there, instead of opening a new issue.
 
-- If possible, provide us a **use case of the feature**: How could the user
-  invoke the new function? Which problem would it solve? If new information is
-  obtained, how would it be further processed?
+- **Motivate the feature**, i.e.
 
-- **Be patient**. Naturally, bugs and pull requests have a higher priority than
-  feature suggestions. Keep in mind that this is a free software project, and
-  unfortunately we only have limited time to work on it.
+  - Provide us a **use case of the feature**: How could the user
+    invoke the new function? Which problem would it solve? If new information is
+    obtained, how would it be further processed?
+
+  - Describe already-working **alternatives of the feature** and how they
+    compare to your proposed feature.
+
+  - Briefly describe how your suggested feature **conforms with Instaloader's
+    project goal**.
+
+- Explain your **solution ideas**. Describe your ideas on how the feature could
+  be implemented and the underlying problem could be solved. Also **describe
+  alternatives** that you have considered.
 
 Donations
 ---------

docs/index.rst

@@ -40,12 +40,15 @@ See :ref:`install` for more options on how to install Instaloader.
 - allows **fine-grained customization** of filters and where to store
   downloaded media,
 
+- automatically **resumes previously-interrupted** download iterations,
+
 - is free `open source <https://github.com/instaloader/instaloader>`__
   software written in Python.
 
 ::
 
-    instaloader [--comments] [--geotags] [--stories] [--highlights] [--tagged]
+    instaloader [--comments] [--geotags]
+                [--stories] [--highlights] [--tagged] [--igtv]
                 [--login YOUR-USERNAME] [--fast-update]
                 profile | "#hashtag" | %location_id |
                 :stories | :feed | :saved

docs/installation.rst

@@ -5,27 +5,27 @@ Install Instaloader
 
 .. highlight:: none
 
-**To install Instaloader**,
+To **install Instaloader**,
 
 #. Ensure that you have `Python <https://www.python.org/>`__, at least
-   version 3.5 and `pip <https://pypi.python.org/pypi/pip>`__
-   installed
+   version 3.5, and `pip <https://pypi.python.org/pypi/pip>`__
+   installed.
 
 #. Then, install Instaloader using::
 
       pip3 install instaloader
 
-**To upgrade Instaloader**, do::
+To **upgrade Instaloader** to its current version, do::
 
    pip3 install --upgrade instaloader
 
 
-**Alternative methods for installing Instaloader:**
+**Alternative methods** for installing Instaloader:
 
 - If you do not want to use pip, even though it is highly recommended,
   and prefer to **install Instaloader manually**,
   `Download the Source <https://github.com/instaloader/instaloader/releases/latest>`__,
-  extract the Zip or Tarball and execute ``instaloader.py`` from there.
+  extract the Zip or Tarball and run ``setup.py`` from there.
 
 - On **Arch Linux**, you may install Instaloader using the
   `Instaloader AUR package <https://aur.archlinux.org/packages/instaloader/>`__.
@@ -33,6 +33,9 @@ Install Instaloader
 - On **Windows 10**, you may download the standalone executable from the
   `current release page <https://github.com/instaloader/instaloader/releases/latest>`__.
 
-- To test the most current pre-release or development version of Instaloader::
+- On **Android**, you can use Instaloader with `Termux <https://play.google.com/store/apps/details?id=com.termux>`__
+  after typing ``pkg install python`` and ``pip3 install instaloader``.
+
+- To test the most current **pre-release** version of Instaloader::
 
      pip3 install --pre instaloader

docs/module/exceptions.rst

@@ -0,0 +1,49 @@
+Exceptions
+^^^^^^^^^^
+
+.. module:: instaloader
+   :noindex:
+
+.. highlight:: python
+
+.. currentmodule:: instaloader.exceptions
+
+.. autoexception:: InstaloaderException
+   :no-show-inheritance:
+
+.. autoexception:: ConnectionException
+
+.. currentmodule:: instaloader
+
+.. autoexception:: QueryReturnedBadRequestException
+
+.. autoexception:: QueryReturnedForbiddenException
+
+.. autoexception:: ProfileNotExistsException
+
+.. autoexception:: ProfileHasNoPicsException
+
+.. autoexception:: PrivateProfileNotFollowedException
+
+.. autoexception:: LoginRequiredException
+
+.. autoexception:: TwoFactorAuthRequiredException
+
+   .. versionadded:: 4.2
+
+.. autoexception:: InvalidArgumentException
+
+.. autoexception:: BadResponseException
+
+.. autoexception:: BadCredentialsException
+
+.. autoexception:: PostChangedException
+
+.. autoexception:: QueryReturnedNotFoundException
+
+   .. versionchanged:: 4.3
+      QueryReturnedNotFoundException now inherits ConnectionException
+      to retry on 404 errors.
+
+
+.. autoexception:: TooManyRequestsException

docs/module/instaloader.rst

@@ -0,0 +1,10 @@
+``Instaloader`` (Main Class)
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+.. module:: instaloader
+   :noindex:
+
+.. highlight:: python
+
+.. autoclass:: Instaloader
+   :no-show-inheritance:

docs/module/instaloadercontext.rst

@@ -0,0 +1,24 @@
+``InstaloaderContext`` (Low-level functions)
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+.. module:: instaloader
+   :noindex:
+
+.. highlight:: python
+
+.. contents::
+   :backlinks: none
+
+``InstaloaderContext``
+""""""""""""""""""""""
+
+.. autoclass:: InstaloaderContext
+   :no-show-inheritance:
+
+``RateController``
+""""""""""""""""""
+
+.. autoclass:: RateController
+   :no-show-inheritance:
+
+   .. versionadded:: 4.5

docs/module/nodeiterator.rst

@@ -0,0 +1,31 @@
+Resumable Iterations
+^^^^^^^^^^^^^^^^^^^^
+
+.. module:: instaloader
+   :noindex:
+
+.. highlight:: python
+
+.. contents::
+   :backlinks: none
+
+For many download targets, Instaloader is able to resume a
+previously-interrupted iteration. It provides an interruptible
+Iterator :class:`NodeIterator` and a context manager
+:func:`resumable_iteration`, which we both present here.
+
+.. versionadded:: 4.5
+
+``NodeIterator``
+""""""""""""""""
+
+.. autoclass:: NodeIterator
+   :no-show-inheritance:
+
+.. autoclass:: FrozenNodeIterator
+   :no-show-inheritance:
+
+``resumable_iteration``
+"""""""""""""""""""""""
+
+.. autofunction:: resumable_iteration

docs/module/structures.rst

@@ -0,0 +1,82 @@
+Instagram Structures
+^^^^^^^^^^^^^^^^^^^^
+
+.. module:: instaloader
+   :noindex:
+
+.. highlight:: python
+
+.. contents::
+   :backlinks: none
+
+Posts
+"""""
+
+.. autoclass:: Post
+   :no-show-inheritance:
+
+Additionally, the following trivial structures are defined:
+
+.. autoclass:: PostSidecarNode
+   :no-show-inheritance:
+
+.. autoclass:: PostComment
+   :no-show-inheritance:
+
+.. autoclass:: PostCommentAnswer
+   :no-show-inheritance:
+
+.. autoclass:: PostLocation
+   :no-show-inheritance:
+
+User Stories
+""""""""""""
+
+.. autoclass:: Story
+   :no-show-inheritance:
+
+.. autoclass:: StoryItem
+   :no-show-inheritance:
+
+Highlights
+""""""""""
+
+.. autoclass:: Highlight
+   :no-show-inheritance:
+   :inherited-members:
+
+   Bases: :class:`Story`
+
+   .. versionadded:: 4.1
+
+Profiles
+""""""""
+
+.. autoclass:: Profile
+   :no-show-inheritance:
+
+Hashtags
+""""""""
+
+.. autoclass:: Hashtag
+   :no-show-inheritance:
+
+   .. versionadded:: 4.4
+
+TopSearchResults
+""""""""""""""""
+
+.. autoclass:: TopSearchResults
+   :no-show-inheritance:
+
+   .. versionadded:: 4.3
+
+Loading and Saving
+""""""""""""""""""
+
+:class:`Post`, :class:`StoryItem`, :class:`Profile`, :class:`Hashtag` and
+:class:`FrozenNodeIterator` can be saved and loaded to/from JSON files.
+
+.. autofunction:: load_structure_from_file
+
+.. autofunction:: save_structure_to_file

docs/requirements.txt

@@ -1,2 +1,2 @@
 requests
-sphinx!=1.7.7
+sphinx~=2.3

docs/troubleshooting.rst

@@ -5,32 +5,54 @@ Troubleshooting
 
 .. highlight:: python
 
-429 - Too Many Requests
------------------------
+429 Too Many Requests
+---------------------
 
 Instaloader has a logic to keep track of its requests to Instagram and to obey
-their rate limits. Since they are nowhere documented, we try them out
-experimentally. We have a daily cron job running to confirm that Instaloader
-still stays within the rate limits. Nevertheless, the rate control logic assumes
-that
+their rate limits. The rate controller assumes that
 
-- at one time, Instaloader is the only application that consumes requests. I.e.
+- at one time, Instaloader is the only application that consumes requests, i.e.
   neither the Instagram browser interface, nor a mobile app, nor another
-  Instaloader instance is running in parallel,
+  Instaloader instance is running in parallel, and
 
 - no requests had been consumed when Instaloader starts.
 
 The latter one implies that restarting or reinstantiating Instaloader often
-within short time is prone to cause a 429. When a request is denied with a 429,
+within short time is prone to cause a 429.
+
+Since the behavior of the rate controller might change between different
+versions of Instaloader, make sure to use the current version of Instaloader,
+especially when encountering many 429 errors.
+
+If a request is denied with a 429,
 Instaloader retries the request as soon as the temporary ban is assumed to be
 expired. In case the retry continuously fails for some reason, which should not
-happen in normal conditions, consider adjusting the
+happen under normal conditions, consider adjusting the
 :option:`--max-connection-attempts` option.
 
+There have been observations that services, that in their nature offer
+promiscuous IP addresses, such as cloud, VPN and public proxy services, might be
+subject to significantly stricter limits for anonymous access. However,
+logged-in accesses (see :option:`--login`) do not seem to be affected.
+
+Instaloader allows to adjust the rate controlling behavior by overriding
+:class:`instaloader.RateController`.
+
+Too many queries in the last time
+---------------------------------
+
 **"Too many queries in the last time"** is not an error. It is a notice that the
 rate limit has almost been reached, according to Instaloader's own rate
-accounting mechanism. We regularly adjust this mechanism to match Instagram's
-current rate limiting.
+accounting mechanism.
+
+Instaloader allows to adjust the rate controlling behavior by overriding
+:class:`instaloader.RateController`.
+
+Private but not followed
+------------------------
+
+You have to follow a private account to access most of its associated
+information.
 
 Login error
 -----------
@@ -40,21 +62,30 @@ Two-Factor-Authentication. It also supports handling the *checkpoint challenge*,
 issued when Instagram suspects authentication activity on your account, by
 pointing the user to an URL to be opened in a browser.
 
-Nevertheless, in :issue:`92` users report problems with logging in. To still use
-Instaloader's logged-in functionality, you may use the following script to
-workaround login problems by importing the session cookies from Firefox and
-bypassing Instaloader's login.
+Nevertheless, in :issue:`92` and :issue:`615` users reported problems with
+logging in. We recommend to always keep the session file which Instaloader
+creates when using :option:`--login`. If a session file is present,
+:option:`--login` does not make use of the failure-prone login procedure.
+Also, session files usually do not expire.
 
-.. literalinclude:: codesnippets/92_import_firefox_session.py
+If you do not have a session file present, you may use the following script
+(:example:`615_import_firefox_session.py`) to workaround login problems by
+importing the session cookies from Firefox and bypassing Instaloader's login and
+so still use Instaloader's logged-in functionality.
 
-To use this,
+.. literalinclude:: codesnippets/615_import_firefox_session.py
 
-#. login to Instagram in Firefox, 
+To use this script,
 
-#. execute the snippet,
+#. Download the script: :example:`615_import_firefox_session.py`,
 
-#. then, ``instaloader -l USERNAME`` should work fine.
+#. Login to Instagram in Firefox,
 
-If you do not use your default firefox profile, or your operating system has the
-paths differently set up, you may have to alter the ``FIREFOXCOOKIEFILE``
-variable first.
+#. Execute the snippet, e.g. with ``python 615_import_firefox_session.py``,
+
+#. Then, ``instaloader -l USERNAME`` should work fine.
+
+This script also supports specifying a cookie file path, which may be useful if
+you use multiple Firefox profiles or if your operating system has the directory
+structure differently set up. Also, you can specify an alternative session file
+path.

instaloader/__init__.py

@@ -1,7 +1,7 @@
 """Download pictures (or videos) along with their captions and other metadata from Instagram."""
 
 
-__version__ = '4.2.5'
+__version__ = '4.6'
 
 
 try:
@@ -14,6 +14,7 @@ else:
 
 from .exceptions import *
 from .instaloader import Instaloader
-from .instaloadercontext import InstaloaderContext
-from .structures import (Highlight, Post, PostSidecarNode, PostComment, PostCommentAnswer, PostLocation, Profile, Story, StoryItem,
-                         load_structure_from_file, save_structure_to_file)
+from .instaloadercontext import InstaloaderContext, RateController
+from .nodeiterator import NodeIterator, FrozenNodeIterator, resumable_iteration
+from .structures import (Hashtag, Highlight, Post, PostSidecarNode, PostComment, PostCommentAnswer, PostLocation,
+                         Profile, Story, StoryItem, TopSearchResults, load_structure_from_file, save_structure_to_file)

instaloader/__main__.py

@@ -3,6 +3,7 @@
 import ast
 import datetime
 import os
+import re
 import sys
 from argparse import ArgumentParser, SUPPRESS
 from typing import List, Optional
@@ -19,7 +20,8 @@ def usage_string():
     argv0 = os.path.basename(sys.argv[0])
     argv0 = "instaloader" if argv0 == "__main__.py" else argv0
     return """
-{0} [--comments] [--geotags] [--stories] [--highlights] [--tagged]
+{0} [--comments] [--geotags]
+{2:{1}} [--stories] [--highlights] [--tagged] [--igtv]
 {2:{1}} [--login YOUR-USERNAME] [--fast-update]
 {2:{1}} profile | "#hashtag" | %%location_id | :stories | :feed | :saved
 {0} --help""".format(argv0, len(argv0), '')
@@ -61,7 +63,10 @@ def _main(instaloader: Instaloader, targetlist: List[str],
           username: Optional[str] = None, password: Optional[str] = None,
           sessionfile: Optional[str] = None,
           download_profile_pic: bool = True, download_posts=True,
-          download_stories: bool = False, download_highlights: bool = False, download_tagged: bool = False,
+          download_stories: bool = False,
+          download_highlights: bool = False,
+          download_tagged: bool = False,
+          download_igtv: bool = False,
           fast_update: bool = False,
           max_count: Optional[int] = None, post_filter_str: Optional[str] = None,
           storyitem_filter_str: Optional[str] = None) -> None:
@@ -77,6 +82,8 @@ def _main(instaloader: Instaloader, targetlist: List[str],
         instaloader.context.log('Only download storyitems with property "{}".'.format(storyitem_filter_str))
     # Login, if desired
     if username is not None:
+        if not re.match(r"^[A-Za-z0-9._]+$", username):
+            instaloader.context.error("Warning: Parameter \"{}\" for --login is not a valid username.".format(username))
         try:
             instaloader.load_session_from_file(username, sessionfile)
         except FileNotFoundError as err:
@@ -98,6 +105,9 @@ def _main(instaloader: Instaloader, targetlist: List[str],
             else:
                 instaloader.interactive_login(username)
         instaloader.context.log("Logged in as %s." % username)
+    # since 4.2.9 login is required for geotags
+    if instaloader.download_geotags and not instaloader.context.is_logged_in:
+        instaloader.context.error("Warning: Use --login to download geotags of posts.")
     # Try block for KeyboardInterrupt (save session on ^C)
     profiles = set()
     anonymous_retry_profiles = set()
@@ -129,18 +139,19 @@ def _main(instaloader: Instaloader, targetlist: List[str],
             # strip '/' characters to be more shell-autocompletion-friendly
             target = target.rstrip('/')
             with instaloader.context.error_catcher(target):
-                if target[0] == '@':
+                if re.match(r"^@[A-Za-z0-9._]+$", target):
                     instaloader.context.log("Retrieving followees of %s..." % target[1:])
                     profile = Profile.from_username(instaloader.context, target[1:])
                     for followee in profile.get_followees():
                         instaloader.save_profile_id(followee)
                         profiles.add(followee)
-                elif target[0] == '#':
+                elif re.match(r"^#\w+$", target):
                     instaloader.download_hashtag(hashtag=target[1:], max_count=max_count, fast_update=fast_update,
-                                                 post_filter=post_filter)
-                elif target[0] == '-':
+                                                 post_filter=post_filter,
+                                                 profile_pic=download_profile_pic, posts=download_posts)
+                elif re.match(r"^-[A-Za-z0-9-_]+$", target):
                     instaloader.download_post(Post.from_shortcode(instaloader.context, target[1:]), target)
-                elif target[0] == "%":
+                elif re.match(r"^%[0-9]+$", target):
                     instaloader.download_location(location=target[1:], max_count=max_count, fast_update=fast_update,
                                                   post_filter=post_filter)
                 elif target == ":feed":
@@ -151,11 +162,12 @@ def _main(instaloader: Instaloader, targetlist: List[str],
                 elif target == ":saved":
                     instaloader.download_saved_posts(fast_update=fast_update, max_count=max_count,
                                                      post_filter=post_filter)
-                else:
+                elif re.match(r"^[A-Za-z0-9._]+$", target):
                     try:
                         profile = instaloader.check_profile_id(target)
                         if instaloader.context.is_logged_in and profile.has_blocked_viewer:
-                            if download_profile_pic or ((download_posts or download_tagged) and not profile.is_private):
+                            if download_profile_pic or ((download_posts or download_tagged or download_igtv)
+                                                        and not profile.is_private):
                                 raise ProfileNotExistsException("{} blocked you; But we download her anonymously."
                                                                 .format(target))
                             else:
@@ -166,7 +178,7 @@ def _main(instaloader: Instaloader, targetlist: List[str],
                         # Not only our profile.has_blocked_viewer condition raises ProfileNotExistsException,
                         # check_profile_id() also does, since access to blocked profile may be responded with 404.
                         if instaloader.context.is_logged_in and (download_profile_pic or download_posts or
-                                                                 download_tagged):
+                                                                 download_tagged or download_igtv):
                             instaloader.context.log(err)
                             instaloader.context.log("Trying again anonymously, helps in case you are just blocked.")
                             with instaloader.anonymous_copy() as anonymous_loader:
@@ -176,20 +188,28 @@ def _main(instaloader: Instaloader, targetlist: List[str],
                                                               .format(target, err))
                         else:
                             raise
+                else:
+                    target_type = {
+                        '#': 'hashtag',
+                        '%': 'location',
+                        '-': 'shortcode',
+                    }.get(target[0], 'username')
+                    raise ProfileNotExistsException('Invalid {} {}'.format(target_type, target))
         if len(profiles) > 1:
             instaloader.context.log("Downloading {} profiles: {}".format(len(profiles),
                                                                          ' '.join([p.username for p in profiles])))
-        if profiles and download_profile_pic and not instaloader.context.is_logged_in:
-            instaloader.context.error("Warning: Use --login to download HD version of profile pictures.")
+        if profiles and (download_profile_pic or download_posts) and not instaloader.context.is_logged_in:
+            instaloader.context.error("Warning: Use --login to download higher-quality versions of pictures.")
         instaloader.download_profiles(profiles,
-                                      download_profile_pic, download_posts, download_tagged, download_highlights,
-                                      download_stories, fast_update, post_filter, storyitem_filter)
+                                      download_profile_pic, download_posts, download_tagged, download_igtv,
+                                      download_highlights, download_stories,
+                                      fast_update, post_filter, storyitem_filter)
         if anonymous_retry_profiles:
             instaloader.context.log("Downloading anonymously: {}"
                                     .format(' '.join([p.username for p in anonymous_retry_profiles])))
             with instaloader.anonymous_copy() as anonymous_loader:
                 anonymous_loader.download_profiles(anonymous_retry_profiles,
-                                                   download_profile_pic, download_posts, download_tagged,
+                                                   download_profile_pic, download_posts, download_tagged, download_igtv,
                                                    fast_update=fast_update, post_filter=post_filter)
     except KeyboardInterrupt:
         print("\nInterrupted by user.", file=sys.stderr)
@@ -202,14 +222,13 @@ def _main(instaloader: Instaloader, targetlist: List[str],
             # Instaloader did at least save a session file
             instaloader.context.log("No targets were specified, thus nothing has been downloaded.")
         else:
-            # Instloader did not do anything
+            # Instaloader did not do anything
             instaloader.context.log("usage:" + usage_string())
 
 
 def main():
     parser = ArgumentParser(description=__doc__, add_help=False, usage=usage_string(),
-                            epilog="Report issues at https://github.com/instaloader/instaloader/issues. "
-                                   "The complete documentation can be found at "
+                            epilog="The complete documentation can be found at "
                                    "https://instaloader.github.io/.",
                             fromfile_prefix_chars='+')
 
@@ -223,7 +242,8 @@ def main():
                            help="Download all followees of profile. Requires --login. "
                                 "Consider using :feed rather than @yourself.")
     g_targets.add_argument('_hashtag', nargs='*', metavar='"#hashtag"', help="Download #hashtag.")
-    g_targets.add_argument('_location', nargs='*', metavar='%location_id', help="Download %%location_id.")
+    g_targets.add_argument('_location', nargs='*', metavar='%location_id',
+                           help="Download %%location_id. Requires --login.")
     g_targets.add_argument('_feed', nargs='*', metavar=":feed",
                            help="Download pictures from your feed. Requires --login.")
     g_targets.add_argument('_stories', nargs='*', metavar=":stories",
@@ -247,6 +267,8 @@ def main():
                         help="Do not download regular posts.")
     g_prof.add_argument('--no-profile-pic', action='store_true',
                         help='Do not download profile picture.')
+    g_post.add_argument('--slide', action='store',
+                        help='Set what image/interval of a sidecar you want to download.')
     g_post.add_argument('--no-pictures', action='store_true',
                         help='Do not download post pictures. Cannot be used together with --fast-update. '
                              'Implies --no-video-thumbnails, does not imply --no-videos.')
@@ -258,7 +280,7 @@ def main():
                         help='Download geotags when available. Geotags are stored as a '
                              'text file with the location\'s name and a Google Maps link. '
                              'This requires an additional request to the Instagram '
-                             'server for each picture, which is why it is disabled by default.')
+                             'server for each picture. Requires --login.')
     g_post.add_argument('-C', '--comments', action='store_true',
                         help='Download and update comments for each post. '
                              'This requires an additional request to the Instagram '
@@ -283,6 +305,8 @@ def main():
                         help='Also download highlights of each profile that is downloaded. Requires --login.')
     g_prof.add_argument('--tagged', action='store_true',
                         help='Also download posts where each profile is tagged.')
+    g_prof.add_argument('--igtv', action='store_true',
+                        help='Also download IGTV videos.')
 
     g_cond = parser.add_argument_group("Which Posts to Download")
 
@@ -328,6 +352,13 @@ def main():
                             '--dirname-pattern. {profile} is replaced by the profile name,'
                             '{target} is replaced by the target you specified, i.e. either :feed'
                             '#hashtag or the profile name. Defaults to \'{date_utc}_UTC\'')
+    g_how.add_argument('--resume-prefix', metavar='PREFIX',
+                       help='Prefix for filenames that are used to save the information to resume an interrupted '
+                            'download.')
+    g_how.add_argument('--no-resume', action='store_true',
+                       help='Do not resume a previously-aborted download iteration, and do not save such information '
+                            'when interrupted.')
+    g_how.add_argument('--use-aged-resume-files', action='store_true', help=SUPPRESS)
     g_how.add_argument('--user-agent',
                        help='User Agent to use for HTTP requests. Defaults to \'{}\'.'.format(default_user_agent()))
     g_how.add_argument('-S', '--no-sleep', action='store_true', help=SUPPRESS)
@@ -335,10 +366,9 @@ def main():
                        help='Maximum number of connection attempts until a request is aborted. Defaults to 3. If a '
                             'connection fails, it can be manually skipped by hitting CTRL+C. Set this to 0 to retry '
                             'infinitely.')
-    g_how.add_argument('--commit-mode', action='store_true',
-                       help='Tries to ensure downloaded images avoid corruption in case of unexpected interruption. '
-                       'If the last picture is corrupted, Instaloader will fix the picture the next time it is run. '
-                       'Requires the JSON metadata to be saved.')
+    g_how.add_argument('--commit-mode', action='store_true', help=SUPPRESS)
+    g_how.add_argument('--request-timeout', metavar='N', type=float, default=300.0,
+                       help='Seconds to wait before timing out a connection request. Defaults to 300.')
 
     g_misc = parser.add_argument_group('Miscellaneous Options')
     g_misc.add_argument('-q', '--quiet', action='store_true',
@@ -372,6 +402,10 @@ def main():
                 raise SystemExit("--no-captions and --post-metadata-txt or --storyitem-metadata-txt given; "
                                  "That contradicts.")
 
+        if args.no_resume and args.resume_prefix:
+            raise SystemExit("--no-resume and --resume-prefix given; That contradicts.")
+        resume_prefix = (args.resume_prefix if args.resume_prefix else 'iterator') if not args.no_resume else None
+
         if args.no_pictures and args.fast_update:
             raise SystemExit('--no-pictures and --fast-update cannot be used together.')
 
@@ -380,9 +414,6 @@ def main():
         download_posts = not (args.no_posts or args.stories_only or args.profile_pic_only)
         download_stories = args.stories or args.stories_only
 
-        if args.commit_mode and args.no_metadata_json:
-            raise SystemExit('--commit-mode requires JSON metadata to be saved.')
-
         loader = Instaloader(sleep=not args.no_sleep, quiet=args.quiet, user_agent=args.user_agent,
                              dirname_pattern=args.dirname_pattern, filename_pattern=args.filename_pattern,
                              download_pictures=not args.no_pictures,
@@ -393,7 +424,10 @@ def main():
                              post_metadata_txt_pattern=post_metadata_txt_pattern,
                              storyitem_metadata_txt_pattern=storyitem_metadata_txt_pattern,
                              max_connection_attempts=args.max_connection_attempts,
-                             commit_mode=args.commit_mode)
+                             request_timeout=args.request_timeout,
+                             resume_prefix=resume_prefix,
+                             check_resume_bbd=not args.use_aged_resume_files,
+                             slide=args.slide)
         _main(loader,
               args.profile,
               username=args.login.lower() if args.login is not None else None,
@@ -404,6 +438,7 @@ def main():
               download_stories=download_stories,
               download_highlights=args.highlights,
               download_tagged=args.tagged,
+              download_igtv=args.igtv,
               fast_update=args.fast_update,
               max_count=int(args.count) if args.count is not None else None,
               post_filter_str=args.post_filter,

instaloader/exceptions.py

@@ -9,10 +9,6 @@ class QueryReturnedBadRequestException(InstaloaderException):
     pass
 
 
-class QueryReturnedNotFoundException(InstaloaderException):
-    pass
-
-
 class QueryReturnedForbiddenException(InstaloaderException):
     pass
 
@@ -62,5 +58,9 @@ class PostChangedException(InstaloaderException):
     pass
 
 
+class QueryReturnedNotFoundException(ConnectionException):
+    pass
+
+
 class TooManyRequestsException(ConnectionException):
     pass

instaloader/instaloadercontext.py

@@ -1,5 +1,6 @@
 import hashlib
 import json
+import os
 import pickle
 import random
 import re
@@ -10,6 +11,7 @@ import time
 import urllib.parse
 from contextlib import contextmanager
 from datetime import datetime, timedelta
+from functools import partial
 from typing import Any, Callable, Dict, Iterator, List, Optional, Union
 
 import requests
@@ -18,11 +20,14 @@ import requests.utils
 from .exceptions import *
 
 
-def copy_session(session: requests.Session) -> requests.Session:
+def copy_session(session: requests.Session, request_timeout: Optional[float] = None) -> requests.Session:
     """Duplicates a requests.Session."""
     new = requests.Session()
     new.cookies = requests.utils.cookiejar_from_dict(requests.utils.dict_from_cookiejar(session.cookies))
-    new.headers = session.headers.copy() # type: ignore
+    new.headers = session.headers.copy()
+    # Override default timeout behavior.
+    # Need to silence mypy bug for this. See: https://github.com/python/mypy/issues/2427
+    new.request = partial(new.request, timeout=request_timeout) # type: ignore
     return new
 
 
@@ -47,9 +52,11 @@ class InstaloaderContext:
     """
 
     def __init__(self, sleep: bool = True, quiet: bool = False, user_agent: Optional[str] = None,
-                 max_connection_attempts: int = 3):
+                 max_connection_attempts: int = 3, request_timeout: float = 300.0,
+                 rate_controller: Optional[Callable[["InstaloaderContext"], "RateController"]] = None):
 
         self.user_agent = user_agent if user_agent is not None else default_user_agent()
+        self.request_timeout = request_timeout
         self._session = self.get_anonymous_session()
         self.username = None
         self.sleep = sleep
@@ -62,9 +69,7 @@ class InstaloaderContext:
         # error log, filled with error() and printed at the end of Instaloader.main()
         self.error_log = []                      # type: List[str]
 
-        # For the adaption of sleep intervals (rate control)
-        self._graphql_query_timestamps = dict()  # type: Dict[str, List[float]]
-        self._graphql_earliest_next_request_time = 0.0
+        self._rate_controller = rate_controller(self) if rate_controller is not None else RateController(self)
 
         # Can be set to True for testing, disables supression of InstaloaderContext._error_catcher
         self.raise_all_errors = False
@@ -107,7 +112,7 @@ class InstaloaderContext:
     def close(self):
         """Print error log and close session"""
         if self.error_log and not self.quiet:
-            print("\nErrors occured:", file=sys.stderr)
+            print("\nErrors or warnings occurred:", file=sys.stderr)
             for err in self.error_log:
                 print(err, file=sys.stderr)
         self._session.close()
@@ -155,6 +160,9 @@ class InstaloaderContext:
                                 'ig_vw': '1920', 'csrftoken': '',
                                 's_network': '', 'ds_user_id': ''})
         session.headers.update(self._default_http_header(empty_session_only=True))
+        # Override default timeout behavior.
+        # Need to silence mypy bug for this. See: https://github.com/python/mypy/issues/2427
+        session.request = partial(session.request, timeout=self.request_timeout) # type: ignore
         return session
 
     def save_session_to_file(self, sessionfile):
@@ -167,6 +175,9 @@ class InstaloaderContext:
         session.cookies = requests.utils.cookiejar_from_dict(pickle.load(sessionfile))
         session.headers.update(self._default_http_header())
         session.headers.update({'X-CSRFToken': session.cookies.get_dict()['csrftoken']})
+        # Override default timeout behavior.
+        # Need to silence mypy bug for this. See: https://github.com/python/mypy/issues/2427
+        session.request = partial(session.request, timeout=self.request_timeout) # type: ignore
         self._session = session
         self.username = username
 
@@ -181,7 +192,9 @@ class InstaloaderContext:
         :raises InvalidArgumentException: If the provided username does not exist.
         :raises BadCredentialsException: If the provided password is wrong.
         :raises ConnectionException: If connection to Instagram failed.
-        :raises TwoFactorAuthRequiredException: First step of 2FA login done, now call :meth:`Instaloader.two_factor_login`."""
+        :raises TwoFactorAuthRequiredException: First step of 2FA login done, now call
+           :meth:`Instaloader.two_factor_login`."""
+        # pylint:disable=import-outside-toplevel
         import http.client
         # pylint:disable=protected-access
         http.client._MAXHEADERS = 200
@@ -190,19 +203,25 @@ class InstaloaderContext:
                                 'ig_vw': '1920', 'ig_cb': '1', 'csrftoken': '',
                                 's_network': '', 'ds_user_id': ''})
         session.headers.update(self._default_http_header())
+        # Override default timeout behavior.
+        # Need to silence mypy bug for this. See: https://github.com/python/mypy/issues/2427
+        session.request = partial(session.request, timeout=self.request_timeout) # type: ignore
         session.get('https://www.instagram.com/web/__mid/')
         csrf_token = session.cookies.get_dict()['csrftoken']
         session.headers.update({'X-CSRFToken': csrf_token})
         # Not using self.get_json() here, because we need to access csrftoken cookie
         self.do_sleep()
+        # Workaround credits to pgrimaud.
+        # See: https://github.com/pgrimaud/instagram-user-feed/commit/96ad4cf54d1ad331b337f325c73e664999a6d066
+        enc_password = '#PWD_INSTAGRAM_BROWSER:0:{}:{}'.format(int(datetime.now().timestamp()), passwd)
         login = session.post('https://www.instagram.com/accounts/login/ajax/',
-                             data={'password': passwd, 'username': user}, allow_redirects=True)
+                             data={'enc_password': enc_password, 'username': user}, allow_redirects=True)
         try:
             resp_json = login.json()
         except json.decoder.JSONDecodeError:
             raise ConnectionException("Login error: JSON decode fail, {} - {}.".format(login.status_code, login.reason))
         if resp_json.get('two_factor_required'):
-            two_factor_session = copy_session(session)
+            two_factor_session = copy_session(session, self.request_timeout)
             two_factor_session.headers.update({'X-CSRFToken': csrf_token})
             two_factor_session.cookies.update({'csrftoken': csrf_token})
             self.two_factor_auth_pending = (two_factor_session,
@@ -211,7 +230,7 @@ class InstaloaderContext:
             raise TwoFactorAuthRequiredException("Login error: two-factor authentication required.")
         if resp_json.get('checkpoint_url'):
             raise ConnectionException("Login: Checkpoint required. Point your browser to "
-                                      "https://www.instagram.com{}, "
+                                      "https://www.instagram.com{} - "
                                       "follow the instructions, then retry.".format(resp_json.get('checkpoint_url')))
         if resp_json['status'] != 'ok':
             if 'message' in resp_json:
@@ -219,6 +238,12 @@ class InstaloaderContext:
                                                                                                resp_json['message']))
             else:
                 raise ConnectionException("Login error: \"{}\" status.".format(resp_json['status']))
+        if 'authenticated' not in resp_json:
+            # Issue #472
+            if 'message' in resp_json:
+                raise ConnectionException("Login error: Unexpected response, \"{}\".".format(resp_json['message']))
+            else:
+                raise ConnectionException("Login error: Unexpected response, this might indicate a blocked IP.")
         if not resp_json['authenticated']:
             if resp_json['user']:
                 # '{"authenticated": false, "user": true, "status": "ok"}'
@@ -265,72 +290,6 @@ class InstaloaderContext:
         if self.sleep:
             time.sleep(min(random.expovariate(0.7), 5.0))
 
-    def _dump_query_timestamps(self, current_time: float):
-        """Output the number of GraphQL queries grouped by their query_hash within the last time."""
-        windows = [10, 11, 15, 20, 30, 60]
-        print("GraphQL requests:", file=sys.stderr)
-        for query_hash, times in self._graphql_query_timestamps.items():
-            print("  {}".format(query_hash), file=sys.stderr)
-            for window in windows:
-                reqs_in_sliding_window = sum(t > current_time - window * 60 for t in times)
-                print("    last {} minutes: {} requests".format(window, reqs_in_sliding_window), file=sys.stderr)
-
-    def _graphql_request_count_per_sliding_window(self, query_hash: str) -> int:
-        """Return how many GraphQL requests can be done within the sliding window."""
-        if self.is_logged_in:
-            max_reqs = {'1cb6ec562846122743b61e492c85999f': 20, '33ba35852cb50da46f5b5e889df7d159': 20}
-        else:
-            max_reqs = {'1cb6ec562846122743b61e492c85999f': 200, '33ba35852cb50da46f5b5e889df7d159': 200}
-        return max_reqs.get(query_hash) or min(max_reqs.values())
-
-    def _graphql_query_waittime(self, query_hash: str, current_time: float, untracked_queries: bool = False) -> float:
-        """Calculate time needed to wait before GraphQL query can be executed."""
-        sliding_window = 660
-        if query_hash not in self._graphql_query_timestamps:
-            self._graphql_query_timestamps[query_hash] = []
-        self._graphql_query_timestamps[query_hash] = list(filter(lambda t: t > current_time - 60 * 60,
-                                                                 self._graphql_query_timestamps[query_hash]))
-        reqs_in_sliding_window = list(filter(lambda t: t > current_time - sliding_window,
-                                             self._graphql_query_timestamps[query_hash]))
-        count_per_sliding_window = self._graphql_request_count_per_sliding_window(query_hash)
-        if len(reqs_in_sliding_window) < count_per_sliding_window and not untracked_queries:
-            return max(0, self._graphql_earliest_next_request_time - current_time)
-        next_request_time = min(reqs_in_sliding_window) + sliding_window + 6
-        if untracked_queries:
-            self._graphql_earliest_next_request_time = next_request_time
-        return round(max(next_request_time, self._graphql_earliest_next_request_time) - current_time)
-
-    def _ratecontrol_graphql_query(self, query_hash: str, untracked_queries: bool = False):
-        """Called before a GraphQL query is made in order to stay within Instagram's rate limits.
-
-        :param query_hash: The query_hash parameter of the query.
-        :param untracked_queries: True, if 429 has been returned to apply 429 logic.
-        """
-        if not untracked_queries:
-            waittime = self._graphql_query_waittime(query_hash, time.monotonic(), untracked_queries)
-            assert waittime >= 0
-            if waittime > 10:
-                self.log('\nToo many queries in the last time. Need to wait {} seconds, until {:%H:%M}.'
-                         .format(waittime, datetime.now() + timedelta(seconds=waittime)))
-            time.sleep(waittime)
-            if query_hash not in self._graphql_query_timestamps:
-                self._graphql_query_timestamps[query_hash] = [time.monotonic()]
-            else:
-                self._graphql_query_timestamps[query_hash].append(time.monotonic())
-        else:
-            text_for_429 = ("HTTP error code 429 was returned because too many queries occured in the last time. "
-                            "Please do not use Instagram in your browser or run multiple instances of Instaloader "
-                            "in parallel.")
-            print(textwrap.fill(text_for_429), file=sys.stderr)
-            current_time = time.monotonic()
-            waittime = self._graphql_query_waittime(query_hash, current_time, untracked_queries)
-            assert waittime >= 0
-            if waittime > 10:
-                self.log('The request will be retried in {} seconds, at {:%H:%M}.'
-                         .format(waittime, datetime.now() + timedelta(seconds=waittime)))
-            self._dump_query_timestamps(current_time)
-            time.sleep(waittime)
-
     def get_json(self, path: str, params: Dict[str, Any], host: str = 'www.instagram.com',
                  session: Optional[requests.Session] = None, _attempt=1) -> Dict[str, Any]:
         """JSON request to Instagram.
@@ -345,15 +304,26 @@ class InstaloaderContext:
         :raises ConnectionException: When query repeatedly failed.
         """
         is_graphql_query = 'query_hash' in params and 'graphql/query' in path
+        is_iphone_query = host == 'i.instagram.com'
+        is_other_query = not is_graphql_query and host == "www.instagram.com"
         sess = session if session else self._session
         try:
             self.do_sleep()
             if is_graphql_query:
-                self._ratecontrol_graphql_query(params['query_hash'])
+                self._rate_controller.wait_before_query(params['query_hash'])
+            if is_iphone_query:
+                self._rate_controller.wait_before_query('iphone')
+            if is_other_query:
+                self._rate_controller.wait_before_query('other')
             resp = sess.get('https://{0}/{1}'.format(host, path), params=params, allow_redirects=False)
             while resp.is_redirect:
                 redirect_url = resp.headers['location']
                 self.log('\nHTTP redirect from https://{0}/{1} to {2}'.format(host, path, redirect_url))
+                if redirect_url.startswith('https://www.instagram.com/accounts/login'):
+                    if not self.is_logged_in:
+                        raise LoginRequiredException("Redirected to login page. Use --login.")
+                    # alternate rate limit exceeded behavior
+                    raise TooManyRequestsException("Redirected to login")
                 if redirect_url.startswith('https://{}/'.format(host)):
                     resp = sess.get(redirect_url if redirect_url.endswith('/') else redirect_url + '/',
                                     params=params, allow_redirects=False)
@@ -371,8 +341,19 @@ class InstaloaderContext:
             if is_html_query:
                 match = re.search(r'window\._sharedData = (.*);</script>', resp.text)
                 if match is None:
-                    raise ConnectionException("Could not find \"window._sharedData\" in html response.")
-                return json.loads(match.group(1))
+                    raise QueryReturnedNotFoundException("Could not find \"window._sharedData\" in html response.")
+                resp_json = json.loads(match.group(1))
+                entry_data = resp_json.get('entry_data')
+                post_or_profile_page = list(entry_data.values())[0] if entry_data is not None else None
+                if post_or_profile_page is None:
+                    raise ConnectionException("\"window._sharedData\" does not contain required keys.")
+                # If GraphQL data is missing in `window._sharedData`, search for it in `__additionalDataLoaded`.
+                if 'graphql' not in post_or_profile_page[0]:
+                    match = re.search(r'window\.__additionalDataLoaded\(.*?({.*"graphql":.*})\);</script>',
+                                      resp.text)
+                    if match is not None:
+                        post_or_profile_page[0]['graphql'] = json.loads(match.group(1))['graphql']
+                return resp_json
             else:
                 resp_json = resp.json()
             if 'status' in resp_json and resp_json['status'] != "ok":
@@ -385,11 +366,19 @@ class InstaloaderContext:
         except (ConnectionException, json.decoder.JSONDecodeError, requests.exceptions.RequestException) as err:
             error_string = "JSON Query to {}: {}".format(path, err)
             if _attempt == self.max_connection_attempts:
-                raise ConnectionException(error_string) from err
+                if isinstance(err, QueryReturnedNotFoundException):
+                    raise QueryReturnedNotFoundException(error_string) from err
+                else:
+                    raise ConnectionException(error_string) from err
             self.error(error_string + " [retrying; skip with ^C]", repeat_at_end=False)
             try:
                 if isinstance(err, TooManyRequestsException):
-                    self._ratecontrol_graphql_query(params['query_hash'], untracked_queries=True)
+                    if is_graphql_query:
+                        self._rate_controller.handle_429(params['query_hash'])
+                    if is_iphone_query:
+                        self._rate_controller.handle_429('iphone')
+                    if is_other_query:
+                        self._rate_controller.handle_429('other')
                 return self.get_json(path=path, params=params, host=host, session=sess, _attempt=_attempt + 1)
             except KeyboardInterrupt:
                 self.error("[skipped by user]", repeat_at_end=False)
@@ -406,7 +395,7 @@ class InstaloaderContext:
         :param rhx_gis: 'rhx_gis' variable as somewhere returned by Instagram, needed to 'sign' request
         :return: The server's response dictionary.
         """
-        with copy_session(self._session) as tmpsession:
+        with copy_session(self._session, self.request_timeout) as tmpsession:
             tmpsession.headers.update(self._default_http_header(empty_session_only=True))
             del tmpsession.headers['Connection']
             del tmpsession.headers['Content-Length']
@@ -437,7 +426,12 @@ class InstaloaderContext:
                           edge_extractor: Callable[[Dict[str, Any]], Dict[str, Any]],
                           rhx_gis: Optional[str] = None,
                           first_data: Optional[Dict[str, Any]] = None) -> Iterator[Dict[str, Any]]:
-        """Retrieve a list of GraphQL nodes."""
+        """
+        Retrieve a list of GraphQL nodes.
+
+        .. deprecated:: 4.5
+           Use :class:`NodeIterator` instead, which provides more functionality.
+        """
 
         def _query():
             query_variables['first'] = self._graphql_page_length
@@ -474,9 +468,9 @@ class InstaloaderContext:
         :raises ConnectionException: When query repeatedly failed.
 
         .. versionadded:: 4.2.1"""
-        with copy_session(self._session) as tempsession:
-            tempsession.headers['User-Agent'] = 'Instagram 10.3.2 (iPhone7,2; iPhone OS 9_3_3; en_US; en-US; ' \
-                                                'scale=2.00; 750x1334) AppleWebKit/420+'
+        with copy_session(self._session, self.request_timeout) as tempsession:
+            tempsession.headers['User-Agent'] = 'Instagram 146.0.0.27.125 (iPhone12,1; iOS 13_3; en_US; en-US; ' \
+                                                'scale=2.00; 1656x3584; 190542906)'
             for header in ['Host', 'Origin', 'X-Instagram-AJAX', 'X-Requested-With']:
                 tempsession.headers.pop(header, None)
             return self.get_json(path, params, 'i.instagram.com', tempsession)
@@ -486,11 +480,12 @@ class InstaloaderContext:
 
         .. versionadded:: 4.2.1"""
         self.log(filename, end=' ', flush=True)
-        with open(filename, 'wb') as file:
+        with open(filename + '.temp', 'wb') as file:
             if isinstance(resp, requests.Response):
                 shutil.copyfileobj(resp.raw, file)
             else:
                 file.write(resp)
+        os.replace(filename + '.temp', filename)
 
     def get_raw(self, url: str, _attempt=1) -> requests.Response:
         """Downloads a file anonymously.
@@ -529,6 +524,138 @@ class InstaloaderContext:
             # At the moment, rhx_gis seems to be required for anonymous requests only. By returning None when logged
             # in, we can save the root_rhx_gis lookup query.
             return None
-        if not self._root_rhx_gis:
-            self._root_rhx_gis = self.get_json('', {})['rhx_gis']
-        return self._root_rhx_gis
+        if self._root_rhx_gis is None:
+            self._root_rhx_gis = self.get_json('', {}).get('rhx_gis', '')
+        return self._root_rhx_gis or None
+
+
+class RateController:
+    """
+    Class providing request tracking and rate controlling to stay within rate limits.
+
+    It can be overridden to change Instaloader's behavior regarding rate limits, for example to raise a custom
+    exception when the rate limit is hit::
+
+       import instaloader
+
+       class MyRateController(instaloader.RateController):
+           def sleep(self, secs):
+               raise MyCustomException()
+
+       L = instaloader.Instaloader(rate_controller=lambda ctx: MyRateController(ctx))
+    """
+
+    def __init__(self, context: InstaloaderContext):
+        self._context = context
+        self._query_timestamps = dict()  # type: Dict[str, List[float]]
+        self._earliest_next_request_time = 0.0
+
+    def sleep(self, secs: float):
+        """Wait given number of seconds."""
+        # Not static, to allow for the behavior of this method to depend on context-inherent properties, such as
+        # whether we are logged in.
+        # pylint:disable=no-self-use
+        time.sleep(secs)
+
+    def _dump_query_timestamps(self, current_time: float, failed_query_type: str):
+        windows = [10, 11, 20, 22, 30, 60]
+        self._context.error("Requests within last {} minutes grouped by type:"
+                            .format('/'.join(str(w) for w in windows)),
+                            repeat_at_end=False)
+        for query_type, times in self._query_timestamps.items():
+            reqs_in_sliding_window = [sum(t > current_time - w * 60 for t in times) for w in windows]
+            self._context.error(" {} {:>32}: {}".format(
+                "*" if query_type == failed_query_type else " ",
+                query_type,
+                " ".join("{:4}".format(reqs) for reqs in reqs_in_sliding_window)
+            ), repeat_at_end=False)
+
+    def count_per_sliding_window(self, query_type: str) -> int:
+        """Return how many requests can be done within the sliding window."""
+        # Not static, to allow for the count_per_sliding_window to depend on context-inherent properties, such as
+        # whether we are logged in.
+        # pylint:disable=no-self-use
+        return 75 if query_type in ['iphone', 'other'] else 200
+
+    def _reqs_in_sliding_window(self, query_type: Optional[str], current_time: float, window: float) -> List[float]:
+        if query_type is not None:
+            # timestamps of type query_type
+            relevant_timestamps = self._query_timestamps[query_type]
+        else:
+            # all GraphQL queries, i.e. not 'iphone' or 'other'
+            graphql_query_timestamps = filter(lambda tp: tp[0] not in ['iphone', 'other'],
+                                              self._query_timestamps.items())
+            relevant_timestamps = [t for times in (tp[1] for tp in graphql_query_timestamps) for t in times]
+        return list(filter(lambda t: t > current_time - window, relevant_timestamps))
+
+    def query_waittime(self, query_type: str, current_time: float, untracked_queries: bool = False) -> float:
+        """Calculate time needed to wait before query can be executed."""
+        per_type_sliding_window = 660
+        if query_type not in self._query_timestamps:
+            self._query_timestamps[query_type] = []
+        self._query_timestamps[query_type] = list(filter(lambda t: t > current_time - 60 * 60,
+                                                         self._query_timestamps[query_type]))
+
+        def per_type_next_request_time():
+            reqs_in_sliding_window = self._reqs_in_sliding_window(query_type, current_time, per_type_sliding_window)
+            if len(reqs_in_sliding_window) < self.count_per_sliding_window(query_type):
+                return 0.0
+            else:
+                return min(reqs_in_sliding_window) + per_type_sliding_window + 6
+
+        def gql_accumulated_next_request_time():
+            if query_type in ['iphone', 'other']:
+                return 0.0
+            gql_accumulated_sliding_window = 600
+            gql_accumulated_max_count = 275
+            reqs_in_sliding_window = self._reqs_in_sliding_window(None, current_time, gql_accumulated_sliding_window)
+            if len(reqs_in_sliding_window) < gql_accumulated_max_count:
+                return 0.0
+            else:
+                return min(reqs_in_sliding_window) + gql_accumulated_sliding_window
+
+        def untracked_next_request_time():
+            if untracked_queries:
+                reqs_in_sliding_window = self._reqs_in_sliding_window(query_type, current_time, per_type_sliding_window)
+                self._earliest_next_request_time = min(reqs_in_sliding_window) + per_type_sliding_window + 6
+            return self._earliest_next_request_time
+
+        return max(0.0,
+                   max(
+                       per_type_next_request_time(),
+                       gql_accumulated_next_request_time(),
+                       untracked_next_request_time(),
+                   ) - current_time)
+
+    def wait_before_query(self, query_type: str) -> None:
+        """This method is called before a query to Instagram. It calls :meth:`RateController.sleep` to wait
+        until the request can be made."""
+        waittime = self.query_waittime(query_type, time.monotonic(), False)
+        assert waittime >= 0
+        if waittime > 15:
+            self._context.log("\nToo many queries in the last time. Need to wait {} seconds, until {:%H:%M}."
+                              .format(round(waittime), datetime.now() + timedelta(seconds=waittime)))
+        if waittime > 0:
+            self.sleep(waittime)
+        if query_type not in self._query_timestamps:
+            self._query_timestamps[query_type] = [time.monotonic()]
+        else:
+            self._query_timestamps[query_type].append(time.monotonic())
+
+    def handle_429(self, query_type: str) -> None:
+        """This method is called to handle a 429 Too Many Requests response. It calls :meth:`RateController.sleep` to
+         wait until we can repeat the same request."""
+        current_time = time.monotonic()
+        waittime = self.query_waittime(query_type, current_time, True)
+        assert waittime >= 0
+        self._dump_query_timestamps(current_time, query_type)
+        text_for_429 = ("Instagram responded with HTTP error \"429 - Too Many Requests\". Please do not run multiple "
+                        "instances of Instaloader in parallel or within short sequence. Also, do not use any Instagram "
+                        "App while Instaloader is running.")
+        self._context.error(textwrap.fill(text_for_429), repeat_at_end=False)
+        if waittime > 1.5:
+            self._context.error("The request will be retried in {} seconds, at {:%H:%M}."
+                                .format(round(waittime), datetime.now() + timedelta(seconds=waittime)),
+                                repeat_at_end=False)
+        if waittime > 0:
+            self.sleep(waittime)

instaloader/nodeiterator.py

@@ -0,0 +1,273 @@
+import base64
+import hashlib
+import json
+import os
+from contextlib import contextmanager
+from datetime import datetime, timedelta
+from lzma import LZMAError
+from typing import Any, Callable, Dict, Iterator, NamedTuple, Optional, Tuple, TypeVar
+
+from .exceptions import InvalidArgumentException, QueryReturnedBadRequestException
+from .instaloadercontext import InstaloaderContext
+
+FrozenNodeIterator = NamedTuple('FrozenNodeIterator',
+                                [('query_hash', str),
+                                 ('query_variables', Dict),
+                                 ('query_referer', Optional[str]),
+                                 ('context_username', Optional[str]),
+                                 ('total_index', int),
+                                 ('best_before', Optional[float]),
+                                 ('remaining_data', Optional[Dict])])
+FrozenNodeIterator.__doc__ = \
+    """A serializable representation of a :class:`NodeIterator` instance, saving its iteration state."""
+FrozenNodeIterator.query_hash.__doc__ = """The GraphQL ``query_hash`` parameter."""
+FrozenNodeIterator.query_variables.__doc__ = """The GraphQL ``query_variables`` parameter."""
+FrozenNodeIterator.query_referer.__doc__ = """The HTTP referer used for the GraphQL query."""
+FrozenNodeIterator.context_username.__doc__ = """The username who created the iterator, or ``None``."""
+FrozenNodeIterator.total_index.__doc__ = """Number of items that have already been returned."""
+FrozenNodeIterator.best_before.__doc__ = """Date when parts of the stored nodes might have expired."""
+FrozenNodeIterator.remaining_data.__doc__ = \
+    """The already-retrieved, yet-unprocessed ``edges`` and the ``page_info`` at time of freezing."""
+
+
+T = TypeVar('T')
+
+
+class NodeIterator(Iterator[T]):
+    """
+    Iterate the nodes within edges in a GraphQL pagination. Instances of this class are returned by many (but not all)
+    of Instaloader's :class:`Post`-returning functions (such as :meth:`Profile.get_posts` etc.).
+
+    What makes this iterator special is its ability to freeze/store its current state, e.g. to interrupt an iteration,
+    and later thaw/resume from where it left off.
+
+    You can freeze a NodeIterator with :meth:`NodeIterator.freeze`::
+
+       post_iterator = profile.get_posts()
+       try:
+           for post in post_iterator:
+               do_something_with(post)
+       except KeyboardInterrupt:
+           save("resume_information.json", post_iterator.freeze())
+
+    and later reuse it with :meth:`NodeIterator.thaw` on an equally-constructed NodeIterator::
+
+       post_iterator = profile.get_posts()
+       post_iterator.thaw(load("resume_information.json"))
+
+    A :class:`FrozenNodeIterator` can only be thawn with a matching NodeIterator, i.e. a NodeIterator instance that has
+    been constructed with the same parameters as the instance that is represented by the :class:`FrozenNodeIterator` in
+    question. This is to ensure that an iteration cannot be resumed in a wrong, unmatching loop. As a quick way to
+    distinguish iterators that are saved e.g. in files, there is the :attr:`NodeIterator.magic` string: Two
+    NodeIterators are matching if and only if they have the same magic.
+
+    See also :func:`resumable_iteration` for a high-level context manager that handles a resumable iteration.
+    """
+
+    _graphql_page_length = 50
+    _shelf_life = timedelta(days=29)
+
+    def __init__(self,
+                 context: InstaloaderContext,
+                 query_hash: str,
+                 edge_extractor: Callable[[Dict[str, Any]], Dict[str, Any]],
+                 node_wrapper: Callable[[Dict], T],
+                 query_variables: Optional[Dict[str, Any]] = None,
+                 query_referer: Optional[str] = None,
+                 first_data: Optional[Dict[str, Any]] = None):
+        self._context = context
+        self._query_hash = query_hash
+        self._edge_extractor = edge_extractor
+        self._node_wrapper = node_wrapper
+        self._query_variables = query_variables if query_variables is not None else {}
+        self._query_referer = query_referer
+        self._page_index = 0
+        self._total_index = 0
+        if first_data is not None:
+            self._data = first_data
+            self._best_before = datetime.now() + NodeIterator._shelf_life
+        else:
+            self._data = self._query()
+
+    def _query(self, after: Optional[str] = None) -> Dict:
+        pagination_variables = {'first': NodeIterator._graphql_page_length}  # type: Dict[str, Any]
+        if after is not None:
+            pagination_variables['after'] = after
+        try:
+            data = self._edge_extractor(
+                self._context.graphql_query(
+                    self._query_hash, {**self._query_variables, **pagination_variables}, self._query_referer
+                )
+            )
+            self._best_before = datetime.now() + NodeIterator._shelf_life
+            return data
+        except QueryReturnedBadRequestException:
+            new_page_length = int(NodeIterator._graphql_page_length / 2)
+            if new_page_length >= 12:
+                NodeIterator._graphql_page_length = new_page_length
+                self._context.error("HTTP Error 400 (Bad Request) on GraphQL Query. Retrying with shorter page length.",
+                                    repeat_at_end=False)
+                return self._query(after)
+            else:
+                raise
+
+    def __iter__(self):
+        return self
+
+    def __next__(self) -> T:
+        if self._page_index < len(self._data['edges']):
+            node = self._data['edges'][self._page_index]['node']
+            page_index, total_index = self._page_index, self._total_index
+            try:
+                self._page_index += 1
+                self._total_index += 1
+            except KeyboardInterrupt:
+                self._page_index, self._total_index = page_index, total_index
+                raise
+            return self._node_wrapper(node)
+        if self._data['page_info']['has_next_page']:
+            query_response = self._query(self._data['page_info']['end_cursor'])
+            page_index, data = self._page_index, self._data
+            try:
+                self._page_index = 0
+                self._data = query_response
+            except KeyboardInterrupt:
+                self._page_index, self._data = page_index, data
+                raise
+            return self.__next__()
+        raise StopIteration()
+
+    @property
+    def count(self) -> Optional[int]:
+        """The ``count`` as returned by Instagram. This is not always the total count this iterator will yield."""
+        return self._data.get('count') if self._data is not None else None
+
+    @property
+    def total_index(self) -> int:
+        """Number of items that have already been returned."""
+        return self._total_index
+
+    @property
+    def magic(self) -> str:
+        """Magic string for easily identifying a matching iterator file for resuming (hash of some parameters)."""
+        if 'blake2b' not in hashlib.algorithms_available:
+            magic_hash = hashlib.new('sha224')
+        else:
+            # Use blake2b when possible, i.e. on Python >= 3.6.
+            magic_hash = hashlib.blake2b(digest_size=6)  # type:ignore  # pylint: disable=no-member
+        magic_hash.update(json.dumps(
+            [self._query_hash, self._query_variables, self._query_referer, self._context.username]
+        ).encode())
+        return base64.urlsafe_b64encode(magic_hash.digest()).decode()
+
+    def freeze(self) -> FrozenNodeIterator:
+        """Freeze the iterator for later resuming."""
+        remaining_data = None
+        if self._data is not None:
+            remaining_data = {**self._data,
+                              'edges': (self._data['edges'][(max(self._page_index - 1, 0)):])}
+        return FrozenNodeIterator(
+            query_hash=self._query_hash,
+            query_variables=self._query_variables,
+            query_referer=self._query_referer,
+            context_username=self._context.username,
+            total_index=max(self.total_index - 1, 0),
+            best_before=self._best_before.timestamp() if self._best_before else None,
+            remaining_data=remaining_data,
+        )
+
+    def thaw(self, frozen: FrozenNodeIterator) -> None:
+        """
+        Use this iterator for resuming from earlier iteration.
+
+        :raises InvalidArgumentException:
+           If
+
+           - the iterator on which this method is called has already been used, or
+           - the given :class:`FrozenNodeIterator` does not match, i.e. belongs to a different iteration.
+        """
+        if self._total_index or self._page_index:
+            raise InvalidArgumentException("thaw() called on already-used iterator.")
+        if (self._query_hash != frozen.query_hash or
+                self._query_variables != frozen.query_variables or
+                self._query_referer != frozen.query_referer or
+                self._context.username != frozen.context_username):
+            raise InvalidArgumentException("Mismatching resume information.")
+        if not frozen.best_before:
+            raise InvalidArgumentException("\"best before\" date missing.")
+        if frozen.remaining_data is None:
+            raise InvalidArgumentException("\"remaining_data\" missing.")
+        self._total_index = frozen.total_index
+        self._best_before = datetime.fromtimestamp(frozen.best_before)
+        self._data = frozen.remaining_data
+
+
+@contextmanager
+def resumable_iteration(context: InstaloaderContext,
+                        iterator: Iterator,
+                        load: Callable[[InstaloaderContext, str], Any],
+                        save: Callable[[FrozenNodeIterator, str], None],
+                        format_path: Callable[[str], str],
+                        check_bbd: bool = True,
+                        enabled: bool = True) -> Iterator[Tuple[bool, int]]:
+    """
+    High-level context manager to handle a resumable iteration that can be interrupted with a KeyboardInterrupt.
+
+    It can be used as follows to automatically load a previously-saved state into the iterator, save the iterator's
+    state when interrupted, and delete the resume file upon completion::
+
+       post_iterator = profile.get_posts()
+       with resumable_iteration(
+               context=L.context,
+               iterator=post_iterator,
+               load=lambda _, path: FrozenNodeIterator(**json.load(open(path))),
+               save=lambda fni, path: json.dump(fni._asdict(), open(path, 'w')),
+               format_path=lambda magic: "resume_info_{}.json".format(magic)
+       ) as (is_resuming, start_index):
+           for post in post_iterator:
+               do_something_with(post)
+
+    It yields a tuple (is_resuming, start_index).
+
+    When the passed iterator is not a :class:`NodeIterator`, it behaves as if ``resumable_iteration`` was not used,
+    just executing the inner body.
+
+    :param context: The :class:`InstaloaderContext`.
+    :param iterator: The fresh :class:`NodeIterator`.
+    :param load: Loads a FrozenNodeIterator from given path. The object is ignored if it has a different type.
+    :param save: Saves the given FrozenNodeIterator to the given path.
+    :param format_path: Returns the path to the resume file for the given magic.
+    :param check_bbd: Whether to check the best before date and reject an expired FrozenNodeIterator.
+    :param enabled: Set to False to disable all functionality and simply execute the inner body.
+    """
+    if not enabled or not isinstance(iterator, NodeIterator):
+        yield False, 0
+        return
+    is_resuming = False
+    start_index = 0
+    resume_file_path = format_path(iterator.magic)
+    resume_file_exists = os.path.isfile(resume_file_path)
+    if resume_file_exists:
+        try:
+            fni = load(context, resume_file_path)
+            if not isinstance(fni, FrozenNodeIterator):
+                raise InvalidArgumentException("Invalid type.")
+            if check_bbd and fni.best_before and datetime.fromtimestamp(fni.best_before) < datetime.now():
+                raise InvalidArgumentException("\"Best before\" date exceeded.")
+            iterator.thaw(fni)
+            is_resuming = True
+            start_index = iterator.total_index
+            context.log("Resuming from {}.".format(resume_file_path))
+        except (InvalidArgumentException, LZMAError, json.decoder.JSONDecodeError) as exc:
+            context.error("Warning: Not resuming from {}: {}".format(resume_file_path, exc))
+    try:
+        yield is_resuming, start_index
+    except KeyboardInterrupt:
+        if os.path.dirname(resume_file_path):
+            os.makedirs(os.path.dirname(resume_file_path), exist_ok=True)
+        save(iterator.freeze(), resume_file_path)
+        context.log("\nSaved resume information to {}.".format(resume_file_path))
+        raise
+    if resume_file_exists:
+        os.unlink(resume_file_path)
+        context.log("Iteration complete, deleted resume information file {}.".format(resume_file_path))

instaloader/structures.py

@@ -9,7 +9,7 @@ from typing import Any, Dict, Iterator, List, Optional, Union
 from . import __version__
 from .exceptions import *
 from .instaloadercontext import InstaloaderContext
-
+from .nodeiterator import FrozenNodeIterator, NodeIterator
 
 PostSidecarNode = namedtuple('PostSidecarNode', ['is_video', 'display_url', 'video_url'])
 PostSidecarNode.__doc__ = "Item of a Sidecar Post."
@@ -17,15 +17,16 @@ PostSidecarNode.is_video.__doc__ = "Whether this node is a video."
 PostSidecarNode.display_url.__doc__ = "URL of image or video thumbnail."
 PostSidecarNode.video_url.__doc__ = "URL of video or None."
 
-PostCommentAnswer = namedtuple('PostCommentAnswer', ['id', 'created_at_utc', 'text', 'owner'])
+PostCommentAnswer = namedtuple('PostCommentAnswer', ['id', 'created_at_utc', 'text', 'owner', 'likes_count'])
 PostCommentAnswer.id.__doc__ = "ID number of comment."
 PostCommentAnswer.created_at_utc.__doc__ = ":class:`~datetime.datetime` when comment was created (UTC)."
 PostCommentAnswer.text.__doc__ = "Comment text."
 PostCommentAnswer.owner.__doc__ = "Owner :class:`Profile` of the comment."
+PostCommentAnswer.likes_count.__doc__ = "Number of likes on comment."
 
 PostComment = namedtuple('PostComment', (*PostCommentAnswer._fields, 'answers')) # type: ignore
 for field in PostCommentAnswer._fields:
-    getattr(PostComment, field).__doc__ = getattr(PostCommentAnswer, field).__doc__
+    getattr(PostComment, field).__doc__ = getattr(PostCommentAnswer, field).__doc__  # pylint: disable=no-member
 PostComment.answers.__doc__ = r"Iterator which yields all :class:`PostCommentAnswer`\ s for the comment." # type: ignore
 
 PostLocation = namedtuple('PostLocation', ['id', 'name', 'slug', 'has_public_page', 'lat', 'lng'])
@@ -68,8 +69,11 @@ class Post:
         self._node = node
         self._owner_profile = owner_profile
         self._full_metadata_dict = None  # type: Optional[Dict[str, Any]]
-        self._rhx_gis_str = None         # type: Optional[str]
         self._location = None            # type: Optional[PostLocation]
+        self._iphone_struct_ = None
+        if 'iphone_struct' in node:
+            # if loaded from JSON with load_structure_from_file()
+            self._iphone_struct_ = node['iphone_struct']
 
     @classmethod
     def from_shortcode(cls, context: InstaloaderContext, shortcode: str):
@@ -97,6 +101,11 @@ class Post:
             raise InvalidArgumentException("Wrong mediaid {0}, unable to convert to shortcode".format(str(mediaid)))
         return b64encode(mediaid.to_bytes(9, 'big'), b'-_').decode().replace('A', ' ').lstrip().replace(' ', 'A')
 
+    @staticmethod
+    def supported_graphql_types() -> List[str]:
+        """The values of __typename fields that the :class:`Post` class can handle."""
+        return ["GraphImage", "GraphVideo", "GraphSidecar"]
+
     def _asdict(self):
         node = self._node
         if self._full_metadata_dict:
@@ -105,6 +114,8 @@ class Post:
             node['owner'] = self.owner_profile._asdict()
         if self._location:
             node['location'] = self._location._asdict()
+        if self._iphone_struct_:
+            node['iphone_struct'] = self._iphone_struct_
         return node
 
     @property
@@ -130,9 +141,18 @@ class Post:
 
     def _obtain_metadata(self):
         if not self._full_metadata_dict:
-            pic_json = self._context.get_json("p/{0}/".format(self.shortcode), params={})
-            self._full_metadata_dict = pic_json['entry_data']['PostPage'][0]['graphql']['shortcode_media']
-            self._rhx_gis_str = pic_json['rhx_gis']
+            if self._context.is_logged_in:
+                pic_json = self._context.graphql_query(
+                    '2b0673e0dc4580674a88d426fe00ea90',
+                    {'shortcode': self.shortcode}
+                )
+                self._full_metadata_dict = pic_json['data']['shortcode_media']
+            else:
+                # if not logged in, above graphql query is responded with a redirect to login page
+                pic_json = self._context.get_json("p/{0}/".format(self.shortcode), params={})
+                self._full_metadata_dict = pic_json['entry_data']['PostPage'][0]['graphql']['shortcode_media']
+            if self._full_metadata_dict is None:
+                raise BadResponseException("Fetching Post metadata failed.")
             if self.shortcode != self._full_metadata_dict['shortcode']:
                 self._node.update(self._full_metadata_dict)
                 raise PostChangedException
@@ -144,10 +164,13 @@ class Post:
         return self._full_metadata_dict
 
     @property
-    def _rhx_gis(self) -> str:
-        self._obtain_metadata()
-        assert self._rhx_gis_str is not None
-        return self._rhx_gis_str
+    def _iphone_struct(self) -> Dict[str, Any]:
+        if not self._context.is_logged_in:
+            raise LoginRequiredException("--login required to access iPhone media info endpoint.")
+        if not self._iphone_struct_:
+            data = self._context.get_iphone_json(path='api/v1/media/{}/info/'.format(self.mediaid), params={})
+            self._iphone_struct_ = data['items'][0]
+        return self._iphone_struct_
 
     def _field(self, *keys) -> Any:
         """Lookups given fields in _node, and if not found in _full_metadata. Raises KeyError if not found anywhere."""
@@ -185,17 +208,27 @@ class Post:
     @property
     def owner_id(self) -> int:
         """The ID of the Post's owner."""
-        return self.owner_profile.userid
+        # The ID may already be available, e.g. if the post instance was created
+        # from an `hashtag.get_posts()` iterator, so no need to make another
+        # http request.
+        if 'owner' in self._node and 'id' in self._node['owner']:
+            return self._node['owner']['id']
+        else:
+            return self.owner_profile.userid
 
     @property
     def date_local(self) -> datetime:
         """Timestamp when the post was created (local time zone)."""
-        return datetime.fromtimestamp(self._node["date"] if "date" in self._node else self._node["taken_at_timestamp"])
+        return datetime.fromtimestamp(self._node["date"]
+                                      if "date" in self._node
+                                      else self._node["taken_at_timestamp"])
 
     @property
     def date_utc(self) -> datetime:
         """Timestamp when the post was created (UTC)."""
-        return datetime.utcfromtimestamp(self._node["date"] if "date" in self._node else self._node["taken_at_timestamp"])
+        return datetime.utcfromtimestamp(self._node["date"]
+                                         if "date" in self._node
+                                         else self._node["taken_at_timestamp"])
 
     @property
     def date(self) -> datetime:
@@ -210,6 +243,13 @@ class Post:
     @property
     def url(self) -> str:
         """URL of the picture / video thumbnail of the post"""
+        if self.typename == "GraphImage" and self._context.is_logged_in:
+            try:
+                orig_url = self._iphone_struct['image_versions2']['candidates'][0]['url']
+                url = re.sub(r'&se=\d+(&?)', r'\1', orig_url)
+                return url
+            except (InstaloaderException, KeyError, IndexError) as err:
+                self._context.error('{} Unable to fetch high quality image version of {}.'.format(err, self))
         return self._node["display_url"] if "display_url" in self._node else self._node["display_src"]
 
     @property
@@ -217,14 +257,49 @@ class Post:
         """Type of post, GraphImage, GraphVideo or GraphSidecar"""
         return self._field('__typename')
 
-    def get_sidecar_nodes(self) -> Iterator[PostSidecarNode]:
-        """Sidecar nodes of a Post with typename==GraphSidecar."""
+    @property
+    def mediacount(self) -> int:
+        """
+        The number of media in a sidecar Post, or 1 if the Post it not a sidecar.
+
+        .. versionadded:: 4.6
+        """
         if self.typename == 'GraphSidecar':
-            for edge in self._field('edge_sidecar_to_children', 'edges'):
-                node = edge['node']
-                is_video = node['is_video']
-                yield PostSidecarNode(is_video=is_video, display_url=node['display_url'],
-                                      video_url=node['video_url'] if is_video else None)
+            edges = self._field('edge_sidecar_to_children', 'edges')
+            return len(edges)
+        return 1
+
+    def get_sidecar_nodes(self, start=0, end=-1) -> Iterator[PostSidecarNode]:
+        """
+        Sidecar nodes of a Post with typename==GraphSidecar.
+
+        .. versionchanged:: 4.6
+           Added parameters *start* and *end* to specify a slice of sidecar media.
+        """
+        if self.typename == 'GraphSidecar':
+            edges = self._field('edge_sidecar_to_children', 'edges')
+            if any(edge['node']['is_video'] for edge in edges):
+                # video_url is only present in full metadata, issue #558.
+                edges = self._full_metadata['edge_sidecar_to_children']['edges']
+            if end < 0:
+                end = len(edges)-1
+            if start < 0:
+                start = len(edges)-1
+            for idx, edge in enumerate(edges):
+                if start <= idx <= end:
+                    node = edge['node']
+                    is_video = node['is_video']
+                    display_url = node['display_url']
+                    if not is_video and self._context.is_logged_in:
+                        try:
+                            carousel_media = self._iphone_struct['carousel_media']
+                            orig_url = carousel_media[idx]['image_versions2']['candidates'][0]['url']
+                            display_url = re.sub(r'&se=\d+(&?)', r'\1', orig_url)
+                        except (InstaloaderException, KeyError, IndexError) as err:
+                            self._context.error('{} Unable to fetch high quality image version of {}.'.format(
+                                err, self))
+                    yield PostSidecarNode(is_video=is_video, display_url=display_url,
+                                          video_url=node['video_url'] if is_video else None)
 
     @property
     def caption(self) -> Optional[str]:
@@ -257,7 +332,9 @@ class Post:
 
     @property
     def pcaption(self) -> str:
-        """Printable caption, useful as a format specifier for --filename-pattern."""
+        """Printable caption, useful as a format specifier for --filename-pattern.
+
+        .. versionadded:: 4.2.6"""
         def _elliptify(caption):
             pcaption = ' '.join([s.replace('/', '\u2215') for s in caption.splitlines() if s]).strip()
             return (pcaption[:30] + u"\u2026") if len(pcaption) > 31 else pcaption
@@ -281,19 +358,29 @@ class Post:
     def video_url(self) -> Optional[str]:
         """URL of the video, or None."""
         if self.is_video:
+            if self._context.is_logged_in:
+                try:
+                    url = self._iphone_struct['video_versions'][0]['url']
+                    return url
+                except (InstaloaderException, KeyError, IndexError) as err:
+                    self._context.error('{} Unable to fetch high quality video version of {}.'.format(err, self))
             return self._field('video_url')
         return None
 
     @property
     def video_view_count(self) -> Optional[int]:
-        """View count of the video, or None."""
+        """View count of the video, or None.
+
+        .. versionadded:: 4.2.6"""
         if self.is_video:
             return self._field('video_view_count')
         return None
 
     @property
     def video_duration(self) -> Optional[float]:
-        """Duration of the video in seconds, or None."""
+        """Duration of the video in seconds, or None.
+
+        .. versionadded:: 4.2.6"""
         if self.is_video:
             return self._field('video_duration')
         return None
@@ -315,6 +402,11 @@ class Post:
     @property
     def comments(self) -> int:
         """Comment count including answers"""
+        # If the count is already present in `self._node`, do not use `self._field` which could trigger fetching the
+        # full metadata dict.
+        comments = self._node.get('edge_media_to_comment')
+        if comments and 'count' in comments:
+            return comments['count']
         try:
             return self._field('edge_media_to_parent_comment', 'count')
         except KeyError:
@@ -331,7 +423,8 @@ class Post:
             return PostCommentAnswer(id=int(node['id']),
                                      created_at_utc=datetime.utcfromtimestamp(node['created_at']),
                                      text=node['text'],
-                                     owner=Profile(self._context, node['owner']))
+                                     owner=Profile(self._context, node['owner']),
+                                     likes_count=node.get('edge_liked_by', {}).get('count', 0))
 
         def _postcommentanswers(node):
             if 'edge_threaded_comments' not in node:
@@ -345,11 +438,14 @@ class Post:
                 # If the answer's metadata already contains all comments, don't do GraphQL requests to obtain them
                 yield from (_postcommentanswer(comment['node']) for comment in answer_edges)
                 return
-            yield from (_postcommentanswer(answer_node) for answer_node in
-                        self._context.graphql_node_list("51fdd02b67508306ad4484ff574a0b62",
-                                                        {'comment_id': node['id']},
-                                                        'https://www.instagram.com/p/' + self.shortcode + '/',
-                                                        lambda d: d['data']['comment']['edge_threaded_comments']))
+            yield from NodeIterator(
+                self._context,
+                '51fdd02b67508306ad4484ff574a0b62',
+                lambda d: d['data']['comment']['edge_threaded_comments'],
+                _postcommentanswer,
+                {'comment_id': node['id']},
+                'https://www.instagram.com/p/{0}/'.format(self.shortcode),
+            )
 
         def _postcomment(node):
             return PostComment(*_postcommentanswer(node),
@@ -357,32 +453,40 @@ class Post:
         if self.comments == 0:
             # Avoid doing additional requests if there are no comments
             return
+
         try:
-            comment_edges = self._field('edge_media_to_parent_comment', 'edges')
-            answers_count = sum([edge['node']['edge_threaded_comments']['count'] for edge in comment_edges])
-            threaded_comments_available = True
-        except KeyError:
+            # this seems to be the structure for logged-in accesses
             comment_edges = self._field('edge_media_to_comment', 'edges')
-            answers_count = 0
-            threaded_comments_available = False
+            answers_count = sum([edge['node'].get('edge_threaded_comments', {}).get('count', 0)
+                                 for edge in comment_edges])
+        except KeyError:
+            # ... and this for anonymous accesses
+            comment_edges = self._field('edge_media_to_parent_comment', 'edges')
+            answers_count = sum([edge['node']['edge_threaded_comments']['count']
+                                 for edge in comment_edges])
 
         if self.comments == len(comment_edges) + answers_count:
             # If the Post's metadata already contains all parent comments, don't do GraphQL requests to obtain them
             yield from (_postcomment(comment['node']) for comment in comment_edges)
             return
-        yield from (_postcomment(node) for node in
-                    self._context.graphql_node_list(
-                        "97b41c52301f77ce508f55e66d17620e" if threaded_comments_available
-                        else "f0986789a5c5d17c2400faebf16efd0d",
-                        {'shortcode': self.shortcode},
-                        'https://www.instagram.com/p/' + self.shortcode + '/',
-                        lambda d:
-                        d['data']['shortcode_media'][
-                            'edge_media_to_parent_comment' if threaded_comments_available else 'edge_media_to_comment'],
-                        self._rhx_gis))
+        yield from NodeIterator(
+            self._context,
+            '97b41c52301f77ce508f55e66d17620e',
+            lambda d: d['data']['shortcode_media']['edge_media_to_parent_comment'],
+            _postcomment,
+            {'shortcode': self.shortcode},
+            'https://www.instagram.com/p/{0}/'.format(self.shortcode),
+        )
 
     def get_likes(self) -> Iterator['Profile']:
-        """Iterate over all likes of the post. A :class:`Profile` instance of each likee is yielded."""
+        """
+        Iterate over all likes of the post. A :class:`Profile` instance of each likee is yielded.
+
+        .. versionchanged:: 4.5.4
+           Require being logged in (as required by Instagram).
+        """
+        if not self._context.is_logged_in:
+            raise LoginRequiredException("--login required to access likes of a post.")
         if self.likes == 0:
             # Avoid doing additional requests if there are no comments
             return
@@ -391,18 +495,52 @@ class Post:
             # If the Post's metadata already contains all likes, don't do GraphQL requests to obtain them
             yield from (Profile(self._context, like['node']) for like in likes_edges)
             return
-        yield from (Profile(self._context, node) for node in
-                    self._context.graphql_node_list("1cb6ec562846122743b61e492c85999f", {'shortcode': self.shortcode},
-                                                    'https://www.instagram.com/p/' + self.shortcode + '/',
-                                                    lambda d: d['data']['shortcode_media']['edge_liked_by'],
-                                                    self._rhx_gis))
+        yield from NodeIterator(
+            self._context,
+            '1cb6ec562846122743b61e492c85999f',
+            lambda d: d['data']['shortcode_media']['edge_liked_by'],
+            lambda n: Profile(self._context, n),
+            {'shortcode': self.shortcode},
+            'https://www.instagram.com/p/{0}/'.format(self.shortcode),
+        )
+
+    @property
+    def is_sponsored(self) -> bool:
+        """
+        Whether Post is a sponsored post, equivalent to non-empty :meth:`Post.sponsor_users`.
+
+        .. versionadded:: 4.4
+        """
+        try:
+            sponsor_edges = self._field('edge_media_to_sponsor_user', 'edges')
+        except KeyError:
+            return False
+        return bool(sponsor_edges)
+
+    @property
+    def sponsor_users(self) -> List['Profile']:
+        """
+        The Post's sponsors.
+
+        .. versionadded:: 4.4
+        """
+        return ([] if not self.is_sponsored else
+                [Profile(self._context, edge['node']['sponsor']) for edge in
+                 self._field('edge_media_to_sponsor_user', 'edges')])
 
     @property
     def location(self) -> Optional[PostLocation]:
-        """If the Post has a location, returns PostLocation namedtuple with fields 'id', 'lat' and 'lng' and 'name'."""
+        """
+        If the Post has a location, returns PostLocation namedtuple with fields 'id', 'lat' and 'lng' and 'name'.
+
+        .. versionchanged:: 4.2.9
+           Require being logged in (as required by Instagram), return None if not logged-in.
+        """
         loc = self._field("location")
         if self._location or not loc:
             return self._location
+        if not self._context.is_logged_in:
+            return None
         location_id = int(loc['id'])
         if any(k not in loc for k in ('name', 'slug', 'has_public_page', 'lat', 'lng')):
             loc = self._context.get_json("explore/locations/{0}/".format(location_id),
@@ -444,7 +582,7 @@ class Profile:
         self._context = context
         self._has_public_story = None  # type: Optional[bool]
         self._node = node
-        self._rhx_gis = None
+        self._has_full_metadata = False
         self._iphone_struct_ = None
         if 'iphone_struct' in node:
             # if loaded from JSON with load_structure_from_file()
@@ -462,7 +600,7 @@ class Profile:
         """
         # pylint:disable=protected-access
         profile = cls(context, {'username': username.lower()})
-        profile._obtain_metadata()  # to raise ProfileNotExistException now in case username is invalid
+        profile._obtain_metadata()  # to raise ProfileNotExistsException now in case username is invalid
         return profile
 
     @classmethod
@@ -492,23 +630,42 @@ class Profile:
         context.profile_id_cache[profile_id] = profile
         return profile
 
+    @classmethod
+    def own_profile(cls, context: InstaloaderContext):
+        """Return own profile if logged-in.
+
+        :param context: :attr:`Instaloader.context`
+
+        .. versionadded:: 4.5.2"""
+        if not context.is_logged_in:
+            raise LoginRequiredException("--login required to access own profile.")
+        return cls(context, context.graphql_query("d6f4427fbe92d846298cf93df0b937d3", {})["data"]["user"])
+
     def _asdict(self):
         json_node = self._node.copy()
-        # remove posts
+        # remove posts to avoid "Circular reference detected" exception
         json_node.pop('edge_media_collections', None)
         json_node.pop('edge_owner_to_timeline_media', None)
         json_node.pop('edge_saved_media', None)
+        json_node.pop('edge_felix_video_timeline', None)
         if self._iphone_struct_:
             json_node['iphone_struct'] = self._iphone_struct_
         return json_node
 
     def _obtain_metadata(self):
         try:
-            if not self._rhx_gis:
-                metadata = self._context.get_json('{}/'.format(self.username), params={})
+            if not self._has_full_metadata:
+                metadata = self._context.get_json('{}/feed/'.format(self.username), params={})
                 self._node = metadata['entry_data']['ProfilePage'][0]['graphql']['user']
-                self._rhx_gis = metadata['rhx_gis']
+                self._has_full_metadata = True
         except (QueryReturnedNotFoundException, KeyError) as err:
+            top_search_results = TopSearchResults(self._context, self.username)
+            similar_profiles = [profile.username for profile in top_search_results.get_profiles()]
+            if similar_profiles:
+                raise ProfileNotExistsException('Profile {} does not exist.\nThe most similar profile{}: {}.'
+                                                .format(self.username,
+                                                        's are' if len(similar_profiles) > 1 else ' is',
+                                                        ', '.join(similar_profiles[0:5]))) from err
             raise ProfileNotExistsException('Profile {} does not exist.'.format(self.username)) from err
 
     def _metadata(self, *keys) -> Any:
@@ -566,6 +723,10 @@ class Profile:
     def mediacount(self) -> int:
         return self._metadata('edge_owner_to_timeline_media', 'count')
 
+    @property
+    def igtvcount(self) -> int:
+        return self._metadata('edge_felix_video_timeline', 'count')
+
     @property
     def followers(self) -> int:
         return self._metadata('edge_followed_by', 'count')
@@ -578,6 +739,16 @@ class Profile:
     def external_url(self) -> Optional[str]:
         return self._metadata('external_url')
 
+    @property
+    def is_business_account(self) -> bool:
+        """.. versionadded:: 4.4"""
+        return self._metadata('is_business_account')
+
+    @property
+    def business_category_name(self) -> str:
+        """.. versionadded:: 4.4"""
+        return self._metadata('business_category_name')
+
     @property
     def biography(self) -> str:
         return self._metadata('biography')
@@ -619,8 +790,7 @@ class Profile:
                                                         'include_reel': False, 'include_suggested_users': False,
                                                         'include_logged_out_extras': True,
                                                         'include_highlight_reels': False},
-                                                       'https://www.instagram.com/{}/'.format(self.username),
-                                                       self._rhx_gis)
+                                                       'https://www.instagram.com/{}/'.format(self.username))
             self._has_public_story = data['data']['user']['has_public_story']
         assert self._has_public_story is not None
         return self._has_public_story
@@ -671,73 +841,124 @@ class Profile:
 	   Use :attr:`profile_pic_url`."""
         return self.profile_pic_url
 
-    def get_posts(self) -> Iterator[Post]:
-        """Retrieve all posts from a profile."""
-        self._obtain_metadata()
-        yield from (Post(self._context, node, self) for node in
-                    self._context.graphql_node_list("472f257a40c653c64c666ce877d59d2b",
-                                                    {'id': self.userid},
-                                                    'https://www.instagram.com/{0}/'.format(self.username),
-                                                    lambda d: d['data']['user']['edge_owner_to_timeline_media'],
-                                                    self._rhx_gis,
-                                                    self._metadata('edge_owner_to_timeline_media')))
+    def get_posts(self) -> NodeIterator[Post]:
+        """Retrieve all posts from a profile.
 
-    def get_saved_posts(self) -> Iterator[Post]:
-        """Get Posts that are marked as saved by the user."""
+        :rtype: NodeIterator[Post]"""
+        self._obtain_metadata()
+        return NodeIterator(
+            self._context,
+            '472f257a40c653c64c666ce877d59d2b',
+            lambda d: d['data']['user']['edge_owner_to_timeline_media'],
+            lambda n: Post(self._context, n, self),
+            {'id': self.userid},
+            'https://www.instagram.com/{0}/'.format(self.username),
+            self._metadata('edge_owner_to_timeline_media'),
+        )
+
+    def get_saved_posts(self) -> NodeIterator[Post]:
+        """Get Posts that are marked as saved by the user.
+
+        :rtype: NodeIterator[Post]"""
 
         if self.username != self._context.username:
             raise LoginRequiredException("--login={} required to get that profile's saved posts.".format(self.username))
 
-        self._obtain_metadata()
-        yield from (Post(self._context, node) for node in
-                    self._context.graphql_node_list("f883d95537fbcd400f466f63d42bd8a1",
-                                                    {'id': self.userid},
-                                                    'https://www.instagram.com/{0}/'.format(self.username),
-                                                    lambda d: d['data']['user']['edge_saved_media'],
-                                                    self._rhx_gis,
-                                                    self._metadata('edge_saved_media')))
+        return NodeIterator(
+            self._context,
+            'f883d95537fbcd400f466f63d42bd8a1',
+            lambda d: d['data']['user']['edge_saved_media'],
+            lambda n: Post(self._context, n),
+            {'id': self.userid},
+            'https://www.instagram.com/{0}/'.format(self.username),
+        )
 
-    def get_tagged_posts(self) -> Iterator[Post]:
+    def get_tagged_posts(self) -> NodeIterator[Post]:
         """Retrieve all posts where a profile is tagged.
 
+        :rtype: NodeIterator[Post]
+
         .. versionadded:: 4.0.7"""
         self._obtain_metadata()
-        yield from (Post(self._context, node, self if int(node['owner']['id']) == self.userid else None) for node in
-                    self._context.graphql_node_list("e31a871f7301132ceaab56507a66bbb7",
-                                                    {'id': self.userid},
-                                                    'https://www.instagram.com/{0}/'.format(self.username),
-                                                    lambda d: d['data']['user']['edge_user_to_photos_of_you'],
-                                                    self._rhx_gis))
+        return NodeIterator(
+            self._context,
+            'e31a871f7301132ceaab56507a66bbb7',
+            lambda d: d['data']['user']['edge_user_to_photos_of_you'],
+            lambda n: Post(self._context, n, self if int(n['owner']['id']) == self.userid else None),
+            {'id': self.userid},
+            'https://www.instagram.com/{0}/'.format(self.username),
+        )
 
-    def get_followers(self) -> Iterator['Profile']:
+    def get_igtv_posts(self) -> NodeIterator[Post]:
+        """Retrieve all IGTV posts.
+
+        :rtype: NodeIterator[Post]
+
+        .. versionadded:: 4.3"""
+        self._obtain_metadata()
+        return NodeIterator(
+            self._context,
+            'bc78b344a68ed16dd5d7f264681c4c76',
+            lambda d: d['data']['user']['edge_felix_video_timeline'],
+            lambda n: Post(self._context, n, self),
+            {'id': self.userid},
+            'https://www.instagram.com/{0}/channel/'.format(self.username),
+            self._metadata('edge_felix_video_timeline'),
+        )
+
+    def get_followers(self) -> NodeIterator['Profile']:
         """
         Retrieve list of followers of given profile.
         To use this, one needs to be logged in and private profiles has to be followed.
+
+        :rtype: NodeIterator[Profile]
         """
         if not self._context.is_logged_in:
             raise LoginRequiredException("--login required to get a profile's followers.")
         self._obtain_metadata()
-        yield from (Profile(self._context, node) for node in
-                    self._context.graphql_node_list("37479f2b8209594dde7facb0d904896a",
-                                                    {'id': str(self.userid)},
-                                                    'https://www.instagram.com/' + self.username + '/',
-                                                    lambda d: d['data']['user']['edge_followed_by'],
-                                                    self._rhx_gis))
+        return NodeIterator(
+            self._context,
+            '37479f2b8209594dde7facb0d904896a',
+            lambda d: d['data']['user']['edge_followed_by'],
+            lambda n: Profile(self._context, n),
+            {'id': str(self.userid)},
+            'https://www.instagram.com/{0}/'.format(self.username),
+        )
 
-    def get_followees(self) -> Iterator['Profile']:
+    def get_followees(self) -> NodeIterator['Profile']:
         """
         Retrieve list of followees (followings) of given profile.
         To use this, one needs to be logged in and private profiles has to be followed.
+
+        :rtype: NodeIterator[Profile]
         """
         if not self._context.is_logged_in:
             raise LoginRequiredException("--login required to get a profile's followees.")
         self._obtain_metadata()
-        yield from (Profile(self._context, node) for node in
-                    self._context.graphql_node_list("58712303d941c6855d4e888c5f0cd22f",
-                                                    {'id': str(self.userid)},
-                                                    'https://www.instagram.com/' + self.username + '/',
-                                                    lambda d: d['data']['user']['edge_follow'],
-                                                    self._rhx_gis))
+        return NodeIterator(
+            self._context,
+            '58712303d941c6855d4e888c5f0cd22f',
+            lambda d: d['data']['user']['edge_follow'],
+            lambda n: Profile(self._context, n),
+            {'id': str(self.userid)},
+            'https://www.instagram.com/{0}/'.format(self.username),
+        )
+
+    def get_similar_accounts(self) -> Iterator['Profile']:
+        """
+        Retrieve list of suggested / similar accounts for this profile.
+        To use this, one needs to be logged in.
+
+        .. versionadded:: 4.4
+        """
+        if not self._context.is_logged_in:
+            raise LoginRequiredException("--login required to get a profile's similar accounts.")
+        self._obtain_metadata()
+        yield from (Profile(self._context, edge["node"]) for edge in
+                    self._context.graphql_query("ad99dd9d3646cc3c0dda65debcd266a7",
+                                                {"user_id": str(self.userid), "include_chaining": True},
+                                                "https://www.instagram.com/{0}/"
+                                                .format(self.username))["data"]["user"]["edge_chaining"]["edges"])
 
 
 class StoryItem:
@@ -755,11 +976,17 @@ class StoryItem:
         self._context = context
         self._node = node
         self._owner_profile = owner_profile
+        self._iphone_struct_ = None
+        if 'iphone_struct' in node:
+            # if loaded from JSON with load_structure_from_file()
+            self._iphone_struct_ = node['iphone_struct']
 
     def _asdict(self):
         node = self._node
         if self._owner_profile:
             node['owner'] = self._owner_profile._asdict()
+        if self._iphone_struct_:
+            node['iphone_struct'] = self._iphone_struct_
         return node
 
     @property
@@ -784,6 +1011,15 @@ class StoryItem:
     def __hash__(self) -> int:
         return hash(self.mediaid)
 
+    @property
+    def _iphone_struct(self) -> Dict[str, Any]:
+        if not self._context.is_logged_in:
+            raise LoginRequiredException("--login required to access iPhone media info endpoint.")
+        if not self._iphone_struct_:
+            data = self._context.get_iphone_json(path='api/v1/media/{}/info/'.format(self.mediaid), params={})
+            self._iphone_struct_ = data['items'][0]
+        return self._iphone_struct_
+
     @property
     def owner_profile(self) -> Profile:
         """:class:`Profile` instance of the story item's owner."""
@@ -835,6 +1071,13 @@ class StoryItem:
     @property
     def url(self) -> str:
         """URL of the picture / video thumbnail of the StoryItem"""
+        if self.typename == "GraphStoryImage" and self._context.is_logged_in:
+            try:
+                orig_url = self._iphone_struct['image_versions2']['candidates'][0]['url']
+                url = re.sub(r'&se=\d+(&?)', r'\1', orig_url)
+                return url
+            except (InstaloaderException, KeyError, IndexError) as err:
+                self._context.error('{} Unable to fetch high quality image version of {}.'.format(err, self))
         return self._node['display_resources'][-1]['src']
 
     @property
@@ -907,14 +1150,14 @@ class Story:
 
     @property
     def last_seen_local(self) -> Optional[datetime]:
-        """Timestamp when the story has last been watched or None (local time zone)."""
+        """Timestamp of the most recent StoryItem that has been watched or None (local time zone)."""
         if self._node['seen']:
             return datetime.fromtimestamp(self._node['seen'])
         return None
 
     @property
     def last_seen_utc(self) -> Optional[datetime]:
-        """Timestamp when the story has last been watched or None (UTC)."""
+        """Timestamp of the most recent StoryItem that has been watched or None (UTC)."""
         if self._node['seen']:
             return datetime.utcfromtimestamp(self._node['seen'])
         return None
@@ -1035,17 +1278,262 @@ class Highlight(Story):
         yield from (StoryItem(self._context, item, self.owner_profile) for item in self._items)
 
 
-JsonExportable = Union[Post, Profile, StoryItem]
+class Hashtag:
+    """
+    An Hashtag.
+
+    Analogous to :class:`Profile`, get an instance with::
+
+       L = Instaloader()
+       hashtag = Hashtag.from_name(L.context, HASHTAG)
+
+    To then download the Hashtag's Posts, do::
+
+       for post in hashtag.get_posts():
+          L.download_post(post, target="#"+hashtag.name)
+
+    Also, this class implements == and is hashable.
+    """
+    def __init__(self, context: InstaloaderContext, node: Dict[str, Any]):
+        assert "name" in node
+        self._context = context
+        self._node = node
+        self._has_full_metadata = False
+
+    @classmethod
+    def from_name(cls, context: InstaloaderContext, name: str):
+        """
+        Create a Hashtag instance from a given hashtag name, without preceeding '#'. Raises an Exception if there is no
+        hashtag with the given name.
+
+        :param context: :attr:`Instaloader.context`
+        :param name: Hashtag, without preceeding '#'
+        :raises: :class:`QueryReturnedNotFoundException`
+        """
+        # pylint:disable=protected-access
+        hashtag = cls(context, {'name': name.lower()})
+        hashtag._obtain_metadata()
+        return hashtag
+
+    @property
+    def name(self):
+        """Hashtag name lowercased, without preceeding '#'"""
+        return self._node["name"].lower()
+
+    def _query(self, params):
+        return self._context.get_json("explore/tags/{0}/".format(self.name),
+                                      params)["graphql"]["hashtag"]
+
+    def _obtain_metadata(self):
+        if not self._has_full_metadata:
+            self._node = self._query({"__a": 1})
+            self._has_full_metadata = True
+
+    def _asdict(self):
+        json_node = self._node.copy()
+        # remove posts
+        json_node.pop("edge_hashtag_to_top_posts", None)
+        json_node.pop("edge_hashtag_to_media", None)
+        return json_node
+
+    def __repr__(self):
+        return "<Hashtag #{}>".format(self.name)
+
+    def __eq__(self, other: object) -> bool:
+        if isinstance(other, Hashtag):
+            return self.name == other.name
+        return NotImplemented
+
+    def __hash__(self) -> int:
+        return hash(self.name)
+
+    def _metadata(self, *keys) -> Any:
+        try:
+            d = self._node
+            for key in keys:
+                d = d[key]
+            return d
+        except KeyError:
+            self._obtain_metadata()
+            d = self._node
+            for key in keys:
+                d = d[key]
+            return d
+
+    @property
+    def hashtagid(self) -> int:
+        return int(self._metadata("id"))
+
+    @property
+    def profile_pic_url(self) -> str:
+        return self._metadata("profile_pic_url")
+
+    @property
+    def description(self) -> str:
+        return self._metadata("description")
+
+    @property
+    def allow_following(self) -> bool:
+        return self._metadata("allow_following")
+
+    @property
+    def is_following(self) -> bool:
+        return self._metadata("is_following")
+
+    @property
+    def is_top_media_only(self) -> bool:
+        return self._metadata("is_top_media_only")
+
+    def get_related_tags(self) -> Iterator["Hashtag"]:
+        """Yields similar hashtags."""
+        yield from (Hashtag(self._context, edge["node"])
+                    for edge in self._metadata("edge_hashtag_to_related_tags", "edges"))
+
+    def get_top_posts(self) -> Iterator[Post]:
+        """Yields the top posts of the hashtag."""
+        yield from (Post(self._context, edge["node"])
+                    for edge in self._metadata("edge_hashtag_to_top_posts", "edges"))
+
+    @property
+    def mediacount(self) -> int:
+        """
+        The count of all media associated with this hashtag.
+
+        The number of posts with a certain hashtag may differ from the number of posts that can actually be accessed, as
+        the hashtag count might include private posts
+        """
+        return self._metadata("edge_hashtag_to_media", "count")
+
+    def get_posts(self) -> Iterator[Post]:
+        """Yields the posts associated with this hashtag."""
+        self._metadata("edge_hashtag_to_media", "edges")
+        self._metadata("edge_hashtag_to_media", "page_info")
+        conn = self._metadata("edge_hashtag_to_media")
+        yield from (Post(self._context, edge["node"]) for edge in conn["edges"])
+        while conn["page_info"]["has_next_page"]:
+            data = self._query({'__a': 1, 'max_id': conn["page_info"]["end_cursor"]})
+            conn = data["edge_hashtag_to_media"]
+            yield from (Post(self._context, edge["node"]) for edge in conn["edges"])
+
+    def get_all_posts(self) -> Iterator[Post]:
+        """Yields all posts, i.e. all most recent posts and the top posts, in almost-chronological order."""
+        sorted_top_posts = iter(sorted(self.get_top_posts(), key=lambda p: p.date_utc, reverse=True))
+        other_posts = self.get_posts()
+        next_top = next(sorted_top_posts, None)
+        next_other = next(other_posts, None)
+        while next_top is not None or next_other is not None:
+            if next_other is None:
+                assert next_top is not None
+                yield next_top
+                yield from sorted_top_posts
+                break
+            if next_top is None:
+                assert next_other is not None
+                yield next_other
+                yield from other_posts
+                break
+            if next_top == next_other:
+                yield next_top
+                next_top = next(sorted_top_posts, None)
+                next_other = next(other_posts, None)
+                continue
+            if next_top.date_utc > next_other.date_utc:
+                yield next_top
+                next_top = next(sorted_top_posts, None)
+            else:
+                yield next_other
+                next_other = next(other_posts, None)
+
+
+class TopSearchResults:
+    """
+    An invocation of this class triggers a search on Instagram for the provided search string.
+
+    Provides methods to access the search results as profiles (:class:`Profile`), locations (:class:`PostLocation`) and
+    hashtags.
+
+    :param context: :attr:`Instaloader.context` used to send the query for the search.
+    :param searchstring: String to search for with Instagram's "top search".
+    """
+
+    def __init__(self, context: InstaloaderContext, searchstring: str):
+        self._context = context
+        self._searchstring = searchstring
+        # The `__a` param is only needed to prevent `get_json()` from searching for 'window._sharedData'.
+        self._node = context.get_json('web/search/topsearch/',
+                                      params={'context': 'blended',
+                                              'query': searchstring,
+                                              'include_reel': False,
+                                              '__a': 1})
+
+    def get_profiles(self) -> Iterator[Profile]:
+        """
+        Provides the :class:`Profile` instances from the search result.
+        """
+        for user in self._node.get('users', []):
+            user_node = user['user']
+            if 'pk' in user_node:
+                user_node['id'] = user_node['pk']
+            yield Profile(self._context, user_node)
+
+    def get_prefixed_usernames(self) -> Iterator[str]:
+        """
+        Provides all profile names from the search result that start with the search string.
+        """
+        for user in self._node.get('users', []):
+            username = user.get('user', {}).get('username', '')
+            if username.startswith(self._searchstring):
+                yield username
+
+    def get_locations(self) -> Iterator[PostLocation]:
+        """
+        Provides instances of :class:`PostLocation` from the search result.
+        """
+        for location in self._node.get('places', []):
+            place = location.get('place', {})
+            slug = place.get('slug')
+            loc = place.get('location', {})
+            yield PostLocation(int(loc['pk']), loc['name'], slug, None, loc['lat'], loc['lng'])
+
+    def get_hashtag_strings(self) -> Iterator[str]:
+        """
+        Provides the hashtags from the search result as strings.
+        """
+        for hashtag in self._node.get('hashtags', []):
+            name = hashtag.get('hashtag', {}).get('name')
+            if name:
+                yield name
+
+    def get_hashtags(self) -> Iterator[Hashtag]:
+        """
+        Provides the hashtags from the search result.
+
+        .. versionadded:: 4.4
+        """
+        for hashtag in self._node.get('hashtags', []):
+            node = hashtag.get('hashtag', {})
+            if 'name' in node:
+                yield Hashtag(self._context, node)
+
+    @property
+    def searchstring(self) -> str:
+        """
+        The string that was searched for on Instagram to produce this :class:`TopSearchResults` instance.
+        """
+        return self._searchstring
+
+
+JsonExportable = Union[Post, Profile, StoryItem, Hashtag, FrozenNodeIterator]
 
 
 def save_structure_to_file(structure: JsonExportable, filename: str) -> None:
-    """Saves a :class:`Post`, :class:`Profile` or :class:`StoryItem` to a '.json' or '.json.xz' file such that it can
-    later be loaded by :func:`load_structure_from_file`.
+    """Saves a :class:`Post`, :class:`Profile`, :class:`StoryItem` or :class:`Hashtag` to a '.json' or '.json.xz' file
+    such that it can later be loaded by :func:`load_structure_from_file`.
 
     If the specified filename ends in '.xz', the file will be LZMA compressed. Otherwise, a pretty-printed JSON file
     will be created.
 
-    :param structure: :class:`Post`, :class:`Profile` or :class:`StoryItem`
+    :param structure: :class:`Post`, :class:`Profile`, :class:`StoryItem` or :class:`Hashtag`
     :param filename: Filename, ends in '.json' or '.json.xz'
     """
     json_structure = {'node': structure._asdict(),
@@ -1060,8 +1548,8 @@ def save_structure_to_file(structure: JsonExportable, filename: str) -> None:
 
 
 def load_structure_from_file(context: InstaloaderContext, filename: str) -> JsonExportable:
-    """Loads a :class:`Post`, :class:`Profile` or :class:`StoryItem` from a '.json' or '.json.xz' file that
-    has been saved by :func:`save_structure_to_file`.
+    """Loads a :class:`Post`, :class:`Profile`, :class:`StoryItem` or :class:`Hashtag` from a '.json' or '.json.xz' file
+    that has been saved by :func:`save_structure_to_file`.
 
     :param context: :attr:`Instaloader.context` linked to the new object, used for additional queries if neccessary.
     :param filename: Filename, ends in '.json' or '.json.xz'
@@ -1082,6 +1570,10 @@ def load_structure_from_file(context: InstaloaderContext, filename: str) -> Json
             return Profile(context, json_structure['node'])
         elif node_type == "StoryItem":
             return StoryItem(context, json_structure['node'])
+        elif node_type == "Hashtag":
+            return Hashtag(context, json_structure['node'])
+        elif node_type == "FrozenNodeIterator":
+            return FrozenNodeIterator(**json_structure['node'])
         else:
             raise InvalidArgumentException("{}: Not an Instaloader JSON.".format(filename))
     elif 'shortcode' in json_structure:

setup.py

@@ -38,6 +38,7 @@ setup(
     name='instaloader',
     version=get_version(),
     packages=['instaloader'],
+    package_data={'instaloader': ['py.typed']},
     url='https://instaloader.github.io/',
     license='MIT',
     author='Alexander Graf, André Koch-Kramer',
@@ -48,7 +49,7 @@ setup(
     install_requires=requirements,
     python_requires='>=3.5',
     entry_points={'console_scripts': ['instaloader=instaloader.__main__:main']},
-    zip_safe=True,
+    zip_safe=False,
     keywords=keywords,
     classifiers=[
         'Development Status :: 5 - Production/Stable',
@@ -60,6 +61,7 @@ setup(
         'Programming Language :: Python :: 3.5',
         'Programming Language :: Python :: 3.6',
         'Programming Language :: Python :: 3.7',
+        'Programming Language :: Python :: 3.8',
         'Programming Language :: Python :: 3 :: Only',
         'Topic :: Internet',
         'Topic :: Multimedia :: Graphics'

test/instaloader_unittests.py

@@ -5,12 +5,14 @@ import shutil
 import tempfile
 import unittest
 from itertools import islice
+from typing import Optional
 
 import instaloader
 
 PROFILE_WITH_HIGHLIGHTS = 325732271
 PUBLIC_PROFILE = "selenagomez"
 PUBLIC_PROFILE_ID = 460563723
+PUBLIC_PROFILE_WITH_IGTV = "natgeo"
 HASHTAG = "kitten"
 LOCATION = "362629379"
 OWN_USERNAME = "aandergr"
@@ -21,8 +23,7 @@ PRIVATE_PROFILE_ID = 1706625676
 EMPTY_PROFILE = "not_public"
 EMPTY_PROFILE_ID = 1928659031
 
-# Preserve query timestamps (rate control) between tests to not get rate limited
-instaloadercontext_query_timestamps = dict()
+ratecontroller = None  # type: Optional[instaloader.RateController]
 
 
 class TestInstaloaderAnonymously(unittest.TestCase):
@@ -35,11 +36,15 @@ class TestInstaloaderAnonymously(unittest.TestCase):
                                          download_comments=True,
                                          save_metadata=True)
         self.L.context.raise_all_errors = True
-        self.L.context.query_timestamps = instaloadercontext_query_timestamps.copy()
+        if ratecontroller is not None:
+            # pylint:disable=protected-access
+            ratecontroller._context = self.L.context
+            self.L.context._rate_controller = ratecontroller
 
     def tearDown(self):
-        global instaloadercontext_query_timestamps
-        instaloadercontext_query_timestamps = self.L.context.query_timestamps.copy()
+        # pylint:disable=global-statement,protected-access
+        global ratecontroller
+        ratecontroller = self.L.context._rate_controller
         self.L.close()
         os.chdir('/')
         print("Removing {}".format(self.dir))
@@ -53,31 +58,23 @@ class TestInstaloaderAnonymously(unittest.TestCase):
                 self.assertTrue(post.date_utc < previous_post.date_utc)
             previous_post = post
 
-    @unittest.SkipTest
+    @unittest.skip('')
     def test_public_profile_download(self):
-        self.L.download_profile(PUBLIC_PROFILE, profile_pic=False, fast_update=True)
-        self.L.download_profile(PUBLIC_PROFILE, profile_pic=False, fast_update=True)
+        profiles = {self.L.check_profile_id(PUBLIC_PROFILE)}
+        self.L.download_profiles(profiles, profile_pic=False, fast_update=True, raise_errors=True)
+        self.L.download_profiles(profiles, profile_pic=False, fast_update=True, raise_errors=True)
 
     def test_public_profile_paging(self):
         self.post_paging_test(instaloader.Profile.from_username(self.L.context, PUBLIC_PROFILE).get_posts())
 
     def test_profile_pic_download(self):
-        self.L.download_profile(PUBLIC_PROFILE, profile_pic_only=True)
+        self.L.download_profiles({self.L.check_profile_id(PUBLIC_PROFILE)}, posts=False, raise_errors=True)
 
     def test_hashtag_download(self):
         self.L.download_hashtag(HASHTAG, NORMAL_MAX_COUNT)
 
     def test_hashtag_paging(self):
-        for count, post in enumerate(self.L.get_hashtag_posts(HASHTAG)):
-            print(post)
-            if count == PAGING_MAX_COUNT:
-                break
-
-    def test_location_download(self):
-        self.L.download_location(LOCATION, NORMAL_MAX_COUNT)
-
-    def test_location_paging(self):
-        for count, post in enumerate(self.L.get_location_posts(LOCATION)):
+        for count, post in enumerate(instaloader.Hashtag.from_name(self.L.context, HASHTAG).get_all_posts()):
             print(post)
             if count == PAGING_MAX_COUNT:
                 break
@@ -98,6 +95,10 @@ class TestInstaloaderAnonymously(unittest.TestCase):
         self.assertEqual(EMPTY_PROFILE.lower(),
                          instaloader.Profile.from_id(self.L.context, EMPTY_PROFILE_ID).username)
 
+    def test_get_username_by_name_empty(self):
+        self.assertEqual(EMPTY_PROFILE_ID,
+                         instaloader.Profile.from_username(self.L.context, EMPTY_PROFILE).userid)
+
     def test_post_from_mediaid(self):
         for post in instaloader.Profile.from_username(self.L.context, PUBLIC_PROFILE).get_posts():
             post2 = instaloader.Post.from_mediaid(self.L.context, post.mediaid)
@@ -109,6 +110,11 @@ class TestInstaloaderAnonymously(unittest.TestCase):
                            PAGING_MAX_COUNT):
             print(post)
 
+    def test_public_profile_igtv(self):
+        for post in islice(instaloader.Profile.from_username(self.L.context, PUBLIC_PROFILE_WITH_IGTV).get_igtv_posts(),
+                           PAGING_MAX_COUNT):
+            print(post)
+
 
 class TestInstaloaderLoggedIn(TestInstaloaderAnonymously):
 
@@ -116,13 +122,13 @@ class TestInstaloaderLoggedIn(TestInstaloaderAnonymously):
         super().setUp()
         self.L.load_session_from_file(OWN_USERNAME)
 
-    @unittest.SkipTest
+    @unittest.skip('')
     def test_stories_download(self):
         self.L.download_stories()
 
-    @unittest.SkipTest
+    @unittest.skip('')
     def test_private_profile_download(self):
-        self.L.download_profile(PRIVATE_PROFILE, download_stories=True)
+        self.L.download_profiles({self.L.check_profile_id(PRIVATE_PROFILE)}, stories=True, raise_errors=True)
 
     def test_stories_paging(self):
         for user_story in self.L.get_stories():
@@ -141,9 +147,6 @@ class TestInstaloaderLoggedIn(TestInstaloaderAnonymously):
     def test_private_profile_paging(self):
         self.post_paging_test(instaloader.Profile.from_username(self.L.context, PRIVATE_PROFILE).get_posts())
 
-    def test_profile_pic_download(self):
-        self.L.download_profile(PUBLIC_PROFILE, profile_pic_only=True)
-
     def test_feed_download(self):
         self.L.download_feed_posts(NORMAL_MAX_COUNT)
 
@@ -167,8 +170,8 @@ class TestInstaloaderLoggedIn(TestInstaloaderAnonymously):
 
     def test_followees_and_stories(self):
         profile = instaloader.Profile.from_username(self.L.context, OWN_USERNAME)
-        for f in islice(profile.get_followees(), PAGING_MAX_COUNT):
-            self.L.download_profile(f.username, profile_pic=False, download_stories_only=True)
+        followees = set(islice(profile.get_followees(), PAGING_MAX_COUNT))
+        self.L.download_profiles(followees, profile_pic=False, stories=True, posts=False, raise_errors=True)
 
     def test_get_followees(self):
         profile = instaloader.Profile.from_username(self.L.context, OWN_USERNAME)
@@ -180,6 +183,11 @@ class TestInstaloaderLoggedIn(TestInstaloaderAnonymously):
         for f in profile.get_followers():
             print(f.username)
 
+    def test_get_similar_accounts(self):
+        profile = instaloader.Profile.from_username(self.L.context, OWN_USERNAME)
+        for f in profile.get_similar_accounts():
+            print(f.username)
+
     def test_get_likes(self):
         for post in instaloader.Profile.from_username(self.L.context, OWN_USERNAME).get_posts():
             for like in post.get_likes():
@@ -192,6 +200,15 @@ class TestInstaloaderLoggedIn(TestInstaloaderAnonymously):
             if count == PAGING_MAX_COUNT:
                 break
 
+    def test_location_download(self):
+        self.L.download_location(LOCATION, NORMAL_MAX_COUNT)
+
+    def test_location_paging(self):
+        for count, post in enumerate(self.L.get_location_posts(LOCATION)):
+            print(post)
+            if count == PAGING_MAX_COUNT:
+                break
+
 
 if __name__ == '__main__':
     unittest.main()