First Release Candidate for Version 4.4.5

* Bypass IP-based redirects to /accounts/login

There are two main changes made:

For users, we request /{username}/feed/ instead of /{username}/. For some
reason, this completely bypasses the login redirect. This page doesn't
work in browser while blocked, but fortunately all the data we need is
just present in the HTML page.

For posts, we change from using the /p/ page to using the graphql
endpoint for the same data, which is still subject to graphql rate
limits, but is not subject to login redirects. The data is identical
between the two pages, apart from the object keys being sorted
differently and rhx_gis being missing on graphql.

Yes, this now unblocks access from VPNs, Tor, cloud servers, etc.

* Apply requested patch to fix comments

* Remove rhx_gis from Post and Profile

Co-authored-by: Alexander Graf <17130992+aandergr@users.noreply.github.com>

.github/ISSUE_TEMPLATE/bug_report.md

@@ -1,6 +1,7 @@
 ---
-name: Bug report
-about: Create a report to help us improve
+name: 🐛 Bug report
+about: Create a report to help us improve 🤕
+labels: bug
 
 ---
 
@@ -22,3 +23,5 @@ If applicable, add error messages and tracebacks to help explain your problem.
 
 **Additional context**
 Add any other context about the problem here.
+
+<!-- please also see https://instaloader.github.io/contributing.html for how to report a bug -->

.github/ISSUE_TEMPLATE/config.yml

@@ -0,0 +1,11 @@
+blank_issues_enabled: false
+contact_links:
+  - name: Contributing to Instaloader
+    url: https://instaloader.github.io/contributing.html
+    about: Guidelines for how to report an issue.
+  - name: Instaloader Troubleshooting
+    url: https://instaloader.github.io/troubleshooting.html
+    about: Many common problems and their workarounds are described here.
+  - name: Instaloader Documentation
+    url: https://instaloader.github.io/
+    about: Many questions might be already answered here.

.github/ISSUE_TEMPLATE/feature_request.md

@@ -1,6 +1,7 @@
 ---
-name: Feature request
-about: Suggest an idea for this project
+name: 🚀 Feature Suggestion
+about: Suggest an idea for this project 💡
+labels: "feature suggestion"
 
 ---
 
@@ -13,5 +14,11 @@ A clear and concise description of what you want to happen.
 **Describe alternatives you've considered**
 A clear and concise description of any alternative solutions or features you've considered.
 
+**If the feature request is accepted, would you be willing to submit a PR?**
+Yes / No
+<!-- Help can be provided if you need assistance submitting a PR -->
+
 **Additional context**
 Add any other context about the feature request here.
+
+<!-- please also see https://instaloader.github.io/contributing.html for how to suggest a feature -->

.github/ISSUE_TEMPLATE/question.md

@@ -0,0 +1,12 @@
+---
+name: ❓ Question
+about: If you have a question about how to use Instaloader that's not well covered by our documentation
+labels: question
+
+---
+
+Your question here...
+
+<!-- Please see our documentation: https://instaloader.github.io/ -->
+
+<!-- Basic questions regarding Instaloader's usage might be more appropriate to ask on Stack Overflow -->

.github/pull_request_template.md

@@ -0,0 +1,17 @@
+
+
+<!--
+Please describe:
+
+- A motivation for this change, e.g.
+  - Fixes # .
+  - More general: What problem does the pull request solve?
+
+- The changes proposed in this pull request
+
+- The completeness of this change
+  - Is it just a proof of concept?
+  - Is the documentation updated (if appropriate)?
+  - Do you consider it ready to be merged or is it a draft?
+  - Can we help you at some point?
+-->

.github/workflows/lint.yml

@@ -0,0 +1,30 @@
+name: PyLint and MyPy
+
+on: [push, pull_request]
+
+jobs:
+  lint:
+    name: PyLint and MyPy
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: [3.8]
+    steps:
+      - name: Checkout Instaloader Repository
+        uses: actions/checkout@v2
+        with:
+          fetch-depth: 0  # needed for building docs
+      - name: Setup Python
+        uses: actions/setup-python@v2
+        with:
+          python-version: ${{ matrix.python-version }}
+      - name: Install Dependencies
+        run: |
+          python -m pip install pipenv==2020.6.2
+          pipenv sync --dev
+      - name: PyLint
+        run: pipenv run pylint instaloader
+      - name: MyPy
+        run: pipenv run mypy -m instaloader
+      - name: Build Documentation
+        run: pipenv run make -C docs html SPHINXOPTS="-W -n"

.github/workflows/stale.yml

@@ -0,0 +1,34 @@
+name: Mark stale issues and pull requests
+
+on:
+  schedule:
+  - cron: "10 14 * * *"
+
+jobs:
+  stale:
+
+    name: Mark stale issues and pull requests
+    runs-on: ubuntu-latest
+    
+    steps:
+    - uses: actions/stale@v3
+      with:
+        repo-token: ${{ secrets.GITHUB_TOKEN }}
+        only-labels: 'question'
+        stale-issue-message: 'There has been no activity on this question for an extended period of time. This issue will be closed after further 14 days of inactivity.'
+        stale-issue-label: 'stale'
+        exempt-issue-labels: 'leave open'
+        days-before-stale: 15
+        days-before-close: -1
+        remove-stale-when-updated: false
+    - uses: actions/stale@v1
+      with:
+        repo-token: ${{ secrets.GITHUB_TOKEN }}
+        stale-issue-message: 'There has been no activity on this issue for an extended period of time. This issue will be closed after further 14 days of inactivity.'
+        stale-pr-message: 'There has been no activity on this Pull Request for an extended period of time. This Pull Request will be closed after further 14 days of inactivity.'
+        stale-issue-label: 'stale'
+        stale-pr-label: 'stale'
+        exempt-issue-label: 'leave open'
+        exempt-pr-label: 'leave open'
+        days-before-stale: 90
+        days-before-close: 14

.github/workflows/windows_exe.yml

@@ -0,0 +1,48 @@
+name: Windows EXE build
+
+on: [push, pull_request]
+
+jobs:
+  build:
+    name: Windows EXE build
+    runs-on: windows-latest
+    steps:
+      - name: Checkout Instaloader repository
+        uses: actions/checkout@v2
+      - name: Setup Python
+        uses: actions/setup-python@v1
+        with:
+          python-version: 3.8
+          architecture: x64
+      - name: Get the tagged version
+        id: get_version
+        env:
+          GITHUB_REF: ${{ github.ref }}
+        run: echo ::set-output name=VERSION::${GITHUB_REF/refs\/tags\/v/}
+        shell: bash
+      - name: Run custom python script for EXE creation
+        env:
+          VERSION_TAG: v${{ steps.get_version.outputs.VERSION }}
+        run: python deploy/windows/create_exe.py
+      - name: Create draft release
+        if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v')
+        id: create_release
+        uses: actions/create-release@v1
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        with:
+          tag_name: v${{ steps.get_version.outputs.VERSION }}
+          release_name: Version ${{ steps.get_version.outputs.VERSION }}
+          draft: true
+          prerelease: contains(steps.get_version.outputs.VERSION, 'a') || contains(steps.get_version.outputs.VERSION, 'b') || contains(steps.get_version.outputs.VERSION, 'c')
+      - name: Upload EXE as release asset
+        if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v')
+        id: upload-release-asset
+        uses: actions/upload-release-asset@v1
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        with:
+          upload_url: ${{ steps.create_release.outputs.upload_url }}
+          asset_path: ./instaloader-v${{ steps.get_version.outputs.VERSION }}-windows-standalone.zip
+          asset_name: instaloader-v${{ steps.get_version.outputs.VERSION }}-windows-standalone.zip
+          asset_content_type: application/zip

.gitignore

@@ -1 +1,2 @@
 __pycache__/
+.mypy_cache/

.pylintrc

@@ -62,9 +62,7 @@ confidence=
 # --disable=W".
 disable=invalid-name,
         missing-docstring,
-        line-too-long,
         too-many-lines,
-        bad-whitespace,
         print-statement,
         parameter-unpacking,
         unpacking-in-except,
@@ -83,7 +81,6 @@ disable=invalid-name,
         useless-suppression,
         deprecated-pragma,
         use-symbolic-message-instead,
-        cyclic-import,
         too-many-instance-attributes,
         too-many-public-methods,
         too-many-branches,
@@ -91,7 +88,6 @@ disable=invalid-name,
         too-many-locals,
         too-many-statements,
         no-else-return,
-        inconsistent-return-statements,
         no-else-raise,
         unnecessary-pass,
         wildcard-import,
@@ -362,7 +358,7 @@ indent-after-paren=4
 indent-string='    '
 
 # Maximum number of characters on a single line.
-max-line-length=100
+max-line-length=120
 
 # Maximum number of lines in a module.
 max-module-lines=1000

.travis.yml

@@ -5,11 +5,10 @@ python:
  - "3.6"
  - "3.7"
 install:
- - pip install pylint~=2.3.1 requests mypy
- - pip install -r docs/requirements.txt
+ - pipenv sync --dev
 script:
- - python3 -m pylint instaloader
- - python3 -m mypy -m instaloader
+ - pylint instaloader
+ - mypy -m instaloader
  - make -C docs html SPHINXOPTS="-W -n"
 addons:
   ssh_known_hosts: aur.archlinux.org

Pipfile

@@ -0,0 +1,16 @@
+[[source]]
+name = "pypi"
+url = "https://pypi.org/simple"
+verify_ssl = true
+
+[dev-packages]
+pylint = "*"
+mypy = "*"
+sphinx = "*"
+pyinstaller = "*"
+pefile = "*"
+pywin32-ctypes = "*"
+psutil = "*"
+
+[packages]
+requests = "*"

Pipfile.lock

@@ -0,0 +1,431 @@
+{
+    "_meta": {
+        "hash": {
+            "sha256": "33408a60fda1a56bf680462fad21643ae0159bc13f3b85d8f10ebf3eef7d8725"
+        },
+        "pipfile-spec": 6,
+        "requires": {},
+        "sources": [
+            {
+                "name": "pypi",
+                "url": "https://pypi.org/simple",
+                "verify_ssl": true
+            }
+        ]
+    },
+    "default": {
+        "certifi": {
+            "hashes": [
+                "sha256:017c25db2a153ce562900032d5bc68e9f191e44e9a0f762f373977de9df1fbb3",
+                "sha256:25b64c7da4cd7479594d035c08c2d809eb4aab3a26e5a990ea98cc450c320f1f"
+            ],
+            "version": "==2019.11.28"
+        },
+        "chardet": {
+            "hashes": [
+                "sha256:84ab92ed1c4d4f16916e05906b6b75a6c0fb5db821cc65e70cbd64a3e2a5eaae",
+                "sha256:fc323ffcaeaed0e0a02bf4d117757b98aed530d9ed4531e3e15460124c106691"
+            ],
+            "version": "==3.0.4"
+        },
+        "idna": {
+            "hashes": [
+                "sha256:c357b3f628cf53ae2c4c05627ecc484553142ca23264e593d327bcde5e9c3407",
+                "sha256:ea8b7f6188e6fa117537c3df7da9fc686d485087abf6ac197f9c46432f7e4a3c"
+            ],
+            "version": "==2.8"
+        },
+        "requests": {
+            "hashes": [
+                "sha256:11e007a8a2aa0323f5a921e9e6a2d7e4e67d9877e85773fba9ba6419025cbeb4",
+                "sha256:9cf5292fcd0f598c671cfc1e0d7d1a7f13bb8085e9a590f48c010551dc6c4b31"
+            ],
+            "index": "pypi",
+            "version": "==2.22.0"
+        },
+        "urllib3": {
+            "hashes": [
+                "sha256:2f3db8b19923a873b3e5256dc9c2dedfa883e33d87c690d9c7913e1f40673cdc",
+                "sha256:87716c2d2a7121198ebcb7ce7cccf6ce5e9ba539041cfbaeecfb641dc0bf6acc"
+            ],
+            "version": "==1.25.8"
+        }
+    },
+    "develop": {
+        "alabaster": {
+            "hashes": [
+                "sha256:446438bdcca0e05bd45ea2de1668c1d9b032e1a9154c2c259092d77031ddd359",
+                "sha256:a661d72d58e6ea8a57f7a86e37d86716863ee5e92788398526d58b26a4e4dc02"
+            ],
+            "version": "==0.7.12"
+        },
+        "altgraph": {
+            "hashes": [
+                "sha256:1f05a47122542f97028caf78775a095fbe6a2699b5089de8477eb583167d69aa",
+                "sha256:c623e5f3408ca61d4016f23a681b9adb100802ca3e3da5e718915a9e4052cebe"
+            ],
+            "version": "==0.17"
+        },
+        "astroid": {
+            "hashes": [
+                "sha256:71ea07f44df9568a75d0f354c49143a4575d90645e9fead6dfb52c26a85ed13a",
+                "sha256:840947ebfa8b58f318d42301cf8c0a20fd794a33b61cc4638e28e9e61ba32f42"
+            ],
+            "version": "==2.3.3"
+        },
+        "babel": {
+            "hashes": [
+                "sha256:1aac2ae2d0d8ea368fa90906567f5c08463d98ade155c0c4bfedd6a0f7160e38",
+                "sha256:d670ea0b10f8b723672d3a6abeb87b565b244da220d76b4dba1b66269ec152d4"
+            ],
+            "version": "==2.8.0"
+        },
+        "certifi": {
+            "hashes": [
+                "sha256:017c25db2a153ce562900032d5bc68e9f191e44e9a0f762f373977de9df1fbb3",
+                "sha256:25b64c7da4cd7479594d035c08c2d809eb4aab3a26e5a990ea98cc450c320f1f"
+            ],
+            "version": "==2019.11.28"
+        },
+        "chardet": {
+            "hashes": [
+                "sha256:84ab92ed1c4d4f16916e05906b6b75a6c0fb5db821cc65e70cbd64a3e2a5eaae",
+                "sha256:fc323ffcaeaed0e0a02bf4d117757b98aed530d9ed4531e3e15460124c106691"
+            ],
+            "version": "==3.0.4"
+        },
+        "docutils": {
+            "hashes": [
+                "sha256:0c5b78adfbf7762415433f5515cd5c9e762339e23369dbe8000d84a4bf4ab3af",
+                "sha256:c2de3a60e9e7d07be26b7f2b00ca0309c207e06c100f9cc2a94931fc75a478fc"
+            ],
+            "version": "==0.16"
+        },
+        "future": {
+            "hashes": [
+                "sha256:b1bead90b70cf6ec3f0710ae53a525360fa360d306a86583adc6bf83a4db537d"
+            ],
+            "version": "==0.18.2"
+        },
+        "idna": {
+            "hashes": [
+                "sha256:c357b3f628cf53ae2c4c05627ecc484553142ca23264e593d327bcde5e9c3407",
+                "sha256:ea8b7f6188e6fa117537c3df7da9fc686d485087abf6ac197f9c46432f7e4a3c"
+            ],
+            "version": "==2.8"
+        },
+        "imagesize": {
+            "hashes": [
+                "sha256:6965f19a6a2039c7d48bca7dba2473069ff854c36ae6f19d2cde309d998228a1",
+                "sha256:b1f6b5a4eab1f73479a50fb79fcf729514a900c341d8503d62a62dbc4127a2b1"
+            ],
+            "version": "==1.2.0"
+        },
+        "isort": {
+            "hashes": [
+                "sha256:54da7e92468955c4fceacd0c86bd0ec997b0e1ee80d97f67c35a78b719dccab1",
+                "sha256:6e811fcb295968434526407adb8796944f1988c5b65e8139058f2014cbe100fd"
+            ],
+            "version": "==4.3.21"
+        },
+        "jinja2": {
+            "hashes": [
+                "sha256:93187ffbc7808079673ef52771baa950426fd664d3aad1d0fa3e95644360e250",
+                "sha256:b0eaf100007721b5c16c1fc1eecb87409464edc10469ddc9a22a27a99123be49"
+            ],
+            "version": "==2.11.1"
+        },
+        "lazy-object-proxy": {
+            "hashes": [
+                "sha256:0c4b206227a8097f05c4dbdd323c50edf81f15db3b8dc064d08c62d37e1a504d",
+                "sha256:194d092e6f246b906e8f70884e620e459fc54db3259e60cf69a4d66c3fda3449",
+                "sha256:1be7e4c9f96948003609aa6c974ae59830a6baecc5376c25c92d7d697e684c08",
+                "sha256:4677f594e474c91da97f489fea5b7daa17b5517190899cf213697e48d3902f5a",
+                "sha256:48dab84ebd4831077b150572aec802f303117c8cc5c871e182447281ebf3ac50",
+                "sha256:5541cada25cd173702dbd99f8e22434105456314462326f06dba3e180f203dfd",
+                "sha256:59f79fef100b09564bc2df42ea2d8d21a64fdcda64979c0fa3db7bdaabaf6239",
+                "sha256:8d859b89baf8ef7f8bc6b00aa20316483d67f0b1cbf422f5b4dc56701c8f2ffb",
+                "sha256:9254f4358b9b541e3441b007a0ea0764b9d056afdeafc1a5569eee1cc6c1b9ea",
+                "sha256:9651375199045a358eb6741df3e02a651e0330be090b3bc79f6d0de31a80ec3e",
+                "sha256:97bb5884f6f1cdce0099f86b907aa41c970c3c672ac8b9c8352789e103cf3156",
+                "sha256:9b15f3f4c0f35727d3a0fba4b770b3c4ebbb1fa907dbcc046a1d2799f3edd142",
+                "sha256:a2238e9d1bb71a56cd710611a1614d1194dc10a175c1e08d75e1a7bcc250d442",
+                "sha256:a6ae12d08c0bf9909ce12385803a543bfe99b95fe01e752536a60af2b7797c62",
+                "sha256:ca0a928a3ddbc5725be2dd1cf895ec0a254798915fb3a36af0964a0a4149e3db",
+                "sha256:cb2c7c57005a6804ab66f106ceb8482da55f5314b7fcb06551db1edae4ad1531",
+                "sha256:d74bb8693bf9cf75ac3b47a54d716bbb1a92648d5f781fc799347cfc95952383",
+                "sha256:d945239a5639b3ff35b70a88c5f2f491913eb94871780ebfabb2568bd58afc5a",
+                "sha256:eba7011090323c1dadf18b3b689845fd96a61ba0a1dfbd7f24b921398affc357",
+                "sha256:efa1909120ce98bbb3777e8b6f92237f5d5c8ea6758efea36a473e1d38f7d3e4",
+                "sha256:f3900e8a5de27447acbf900b4750b0ddfd7ec1ea7fbaf11dfa911141bc522af0"
+            ],
+            "version": "==1.4.3"
+        },
+        "markupsafe": {
+            "hashes": [
+                "sha256:00bc623926325b26bb9605ae9eae8a215691f33cae5df11ca5424f06f2d1f473",
+                "sha256:09027a7803a62ca78792ad89403b1b7a73a01c8cb65909cd876f7fcebd79b161",
+                "sha256:09c4b7f37d6c648cb13f9230d847adf22f8171b1ccc4d5682398e77f40309235",
+                "sha256:1027c282dad077d0bae18be6794e6b6b8c91d58ed8a8d89a89d59693b9131db5",
+                "sha256:13d3144e1e340870b25e7b10b98d779608c02016d5184cfb9927a9f10c689f42",
+                "sha256:24982cc2533820871eba85ba648cd53d8623687ff11cbb805be4ff7b4c971aff",
+                "sha256:29872e92839765e546828bb7754a68c418d927cd064fd4708fab9fe9c8bb116b",
+                "sha256:43a55c2930bbc139570ac2452adf3d70cdbb3cfe5912c71cdce1c2c6bbd9c5d1",
+                "sha256:46c99d2de99945ec5cb54f23c8cd5689f6d7177305ebff350a58ce5f8de1669e",
+                "sha256:500d4957e52ddc3351cabf489e79c91c17f6e0899158447047588650b5e69183",
+                "sha256:535f6fc4d397c1563d08b88e485c3496cf5784e927af890fb3c3aac7f933ec66",
+                "sha256:596510de112c685489095da617b5bcbbac7dd6384aeebeda4df6025d0256a81b",
+                "sha256:62fe6c95e3ec8a7fad637b7f3d372c15ec1caa01ab47926cfdf7a75b40e0eac1",
+                "sha256:6788b695d50a51edb699cb55e35487e430fa21f1ed838122d722e0ff0ac5ba15",
+                "sha256:6dd73240d2af64df90aa7c4e7481e23825ea70af4b4922f8ede5b9e35f78a3b1",
+                "sha256:717ba8fe3ae9cc0006d7c451f0bb265ee07739daf76355d06366154ee68d221e",
+                "sha256:79855e1c5b8da654cf486b830bd42c06e8780cea587384cf6545b7d9ac013a0b",
+                "sha256:7c1699dfe0cf8ff607dbdcc1e9b9af1755371f92a68f706051cc8c37d447c905",
+                "sha256:88e5fcfb52ee7b911e8bb6d6aa2fd21fbecc674eadd44118a9cc3863f938e735",
+                "sha256:8defac2f2ccd6805ebf65f5eeb132adcf2ab57aa11fdf4c0dd5169a004710e7d",
+                "sha256:98c7086708b163d425c67c7a91bad6e466bb99d797aa64f965e9d25c12111a5e",
+                "sha256:9add70b36c5666a2ed02b43b335fe19002ee5235efd4b8a89bfcf9005bebac0d",
+                "sha256:9bf40443012702a1d2070043cb6291650a0841ece432556f784f004937f0f32c",
+                "sha256:ade5e387d2ad0d7ebf59146cc00c8044acbd863725f887353a10df825fc8ae21",
+                "sha256:b00c1de48212e4cc9603895652c5c410df699856a2853135b3967591e4beebc2",
+                "sha256:b1282f8c00509d99fef04d8ba936b156d419be841854fe901d8ae224c59f0be5",
+                "sha256:b2051432115498d3562c084a49bba65d97cf251f5a331c64a12ee7e04dacc51b",
+                "sha256:ba59edeaa2fc6114428f1637ffff42da1e311e29382d81b339c1817d37ec93c6",
+                "sha256:c8716a48d94b06bb3b2524c2b77e055fb313aeb4ea620c8dd03a105574ba704f",
+                "sha256:cd5df75523866410809ca100dc9681e301e3c27567cf498077e8551b6d20e42f",
+                "sha256:cdb132fc825c38e1aeec2c8aa9338310d29d337bebbd7baa06889d09a60a1fa2",
+                "sha256:e249096428b3ae81b08327a63a485ad0878de3fb939049038579ac0ef61e17e7",
+                "sha256:e8313f01ba26fbbe36c7be1966a7b7424942f670f38e666995b88d012765b9be"
+            ],
+            "version": "==1.1.1"
+        },
+        "mccabe": {
+            "hashes": [
+                "sha256:ab8a6258860da4b6677da4bd2fe5dc2c659cff31b3ee4f7f5d64e79735b80d42",
+                "sha256:dd8d182285a0fe56bace7f45b5e7d1a6ebcbf524e8f3bd87eb0f125271b8831f"
+            ],
+            "version": "==0.6.1"
+        },
+        "mypy": {
+            "hashes": [
+                "sha256:0a9a45157e532da06fe56adcfef8a74629566b607fa2c1ac0122d1ff995c748a",
+                "sha256:2c35cae79ceb20d47facfad51f952df16c2ae9f45db6cb38405a3da1cf8fc0a7",
+                "sha256:4b9365ade157794cef9685791032521233729cb00ce76b0ddc78749abea463d2",
+                "sha256:53ea810ae3f83f9c9b452582261ea859828a9ed666f2e1ca840300b69322c474",
+                "sha256:634aef60b4ff0f650d3e59d4374626ca6153fcaff96ec075b215b568e6ee3cb0",
+                "sha256:7e396ce53cacd5596ff6d191b47ab0ea18f8e0ec04e15d69728d530e86d4c217",
+                "sha256:7eadc91af8270455e0d73565b8964da1642fe226665dd5c9560067cd64d56749",
+                "sha256:7f672d02fffcbace4db2b05369142e0506cdcde20cea0e07c7c2171c4fd11dd6",
+                "sha256:85baab8d74ec601e86134afe2bcccd87820f79d2f8d5798c889507d1088287bf",
+                "sha256:87c556fb85d709dacd4b4cb6167eecc5bbb4f0a9864b69136a0d4640fdc76a36",
+                "sha256:a6bd44efee4dc8c3324c13785a9dc3519b3ee3a92cada42d2b57762b7053b49b",
+                "sha256:c6d27bd20c3ba60d5b02f20bd28e20091d6286a699174dfad515636cb09b5a72",
+                "sha256:e2bb577d10d09a2d8822a042a23b8d62bc3b269667c9eb8e60a6edfa000211b1",
+                "sha256:f97a605d7c8bc2c6d1172c2f0d5a65b24142e11a58de689046e62c2d632ca8c1"
+            ],
+            "index": "pypi",
+            "version": "==0.761"
+        },
+        "mypy-extensions": {
+            "hashes": [
+                "sha256:090fedd75945a69ae91ce1303b5824f428daf5a028d2f6ab8a299250a846f15d",
+                "sha256:2d82818f5bb3e369420cb3c4060a7970edba416647068eb4c5343488a6c604a8"
+            ],
+            "version": "==0.4.3"
+        },
+        "packaging": {
+            "hashes": [
+                "sha256:170748228214b70b672c581a3dd610ee51f733018650740e98c7df862a583f73",
+                "sha256:e665345f9eef0c621aa0bf2f8d78cf6d21904eef16a93f020240b704a57f1334"
+            ],
+            "version": "==20.1"
+        },
+        "pefile": {
+            "hashes": [
+                "sha256:a5d6e8305c6b210849b47a6174ddf9c452b2888340b8177874b862ba6c207645"
+            ],
+            "index": "pypi",
+            "version": "==2019.4.18"
+        },
+        "psutil": {
+            "hashes": [
+                "sha256:094f899ac3ef72422b7e00411b4ed174e3c5a2e04c267db6643937ddba67a05b",
+                "sha256:10b7f75cc8bd676cfc6fa40cd7d5c25b3f45a0e06d43becd7c2d2871cbb5e806",
+                "sha256:1b1575240ca9a90b437e5a40db662acd87bbf181f6aa02f0204978737b913c6b",
+                "sha256:21231ef1c1a89728e29b98a885b8e0a8e00d09018f6da5cdc1f43f988471a995",
+                "sha256:28f771129bfee9fc6b63d83a15d857663bbdcae3828e1cb926e91320a9b5b5cd",
+                "sha256:70387772f84fa5c3bb6a106915a2445e20ac8f9821c5914d7cbde148f4d7ff73",
+                "sha256:b560f5cd86cf8df7bcd258a851ca1ad98f0d5b8b98748e877a0aec4e9032b465",
+                "sha256:b74b43fecce384a57094a83d2778cdfc2e2d9a6afaadd1ebecb2e75e0d34e10d",
+                "sha256:e85f727ffb21539849e6012f47b12f6dd4c44965e56591d8dec6e8bc9ab96f4a",
+                "sha256:fd2e09bb593ad9bdd7429e779699d2d47c1268cbde4dda95fcd1bd17544a0217",
+                "sha256:ffad8eb2ac614518bbe3c0b8eb9dffdb3a8d2e3a7d5da51c5b974fb723a5c5aa"
+            ],
+            "index": "pypi",
+            "version": "==5.6.7"
+        },
+        "pygments": {
+            "hashes": [
+                "sha256:2a3fe295e54a20164a9df49c75fa58526d3be48e14aceba6d6b1e8ac0bfd6f1b",
+                "sha256:98c8aa5a9f778fcd1026a17361ddaf7330d1b7c62ae97c3bb0ae73e0b9b6b0fe"
+            ],
+            "version": "==2.5.2"
+        },
+        "pyinstaller": {
+            "hashes": [
+                "sha256:3730fa80d088f8bb7084d32480eb87cbb4ddb64123363763cf8f2a1378c1c4b7"
+            ],
+            "index": "pypi",
+            "version": "==3.6"
+        },
+        "pylint": {
+            "hashes": [
+                "sha256:3db5468ad013380e987410a8d6956226963aed94ecb5f9d3a28acca6d9ac36cd",
+                "sha256:886e6afc935ea2590b462664b161ca9a5e40168ea99e5300935f6591ad467df4"
+            ],
+            "index": "pypi",
+            "version": "==2.4.4"
+        },
+        "pyparsing": {
+            "hashes": [
+                "sha256:4c830582a84fb022400b85429791bc551f1f4871c33f23e44f353119e92f969f",
+                "sha256:c342dccb5250c08d45fd6f8b4a559613ca603b57498511740e65cd11a2e7dcec"
+            ],
+            "version": "==2.4.6"
+        },
+        "pytz": {
+            "hashes": [
+                "sha256:1c557d7d0e871de1f5ccd5833f60fb2550652da6be2693c1e02300743d21500d",
+                "sha256:b02c06db6cf09c12dd25137e563b31700d3b80fcc4ad23abb7a315f2789819be"
+            ],
+            "version": "==2019.3"
+        },
+        "pywin32-ctypes": {
+            "hashes": [
+                "sha256:24ffc3b341d457d48e8922352130cf2644024a4ff09762a2261fd34c36ee5942",
+                "sha256:9dc2d991b3479cc2df15930958b674a48a227d5361d413827a4cfd0b5876fc98"
+            ],
+            "index": "pypi",
+            "version": "==0.2.0"
+        },
+        "requests": {
+            "hashes": [
+                "sha256:11e007a8a2aa0323f5a921e9e6a2d7e4e67d9877e85773fba9ba6419025cbeb4",
+                "sha256:9cf5292fcd0f598c671cfc1e0d7d1a7f13bb8085e9a590f48c010551dc6c4b31"
+            ],
+            "index": "pypi",
+            "version": "==2.22.0"
+        },
+        "six": {
+            "hashes": [
+                "sha256:236bdbdce46e6e6a3d61a337c0f8b763ca1e8717c03b369e87a7ec7ce1319c0a",
+                "sha256:8f3cd2e254d8f793e7f3d6d9df77b92252b52637291d0f0da013c76ea2724b6c"
+            ],
+            "version": "==1.14.0"
+        },
+        "snowballstemmer": {
+            "hashes": [
+                "sha256:209f257d7533fdb3cb73bdbd24f436239ca3b2fa67d56f6ff88e86be08cc5ef0",
+                "sha256:df3bac3df4c2c01363f3dd2cfa78cce2840a79b9f1c2d2de9ce8d31683992f52"
+            ],
+            "version": "==2.0.0"
+        },
+        "sphinx": {
+            "hashes": [
+                "sha256:298537cb3234578b2d954ff18c5608468229e116a9757af3b831c2b2b4819159",
+                "sha256:e6e766b74f85f37a5f3e0773a1e1be8db3fcb799deb58ca6d18b70b0b44542a5"
+            ],
+            "index": "pypi",
+            "version": "==2.3.1"
+        },
+        "sphinxcontrib-applehelp": {
+            "hashes": [
+                "sha256:edaa0ab2b2bc74403149cb0209d6775c96de797dfd5b5e2a71981309efab3897",
+                "sha256:fb8dee85af95e5c30c91f10e7eb3c8967308518e0f7488a2828ef7bc191d0d5d"
+            ],
+            "version": "==1.0.1"
+        },
+        "sphinxcontrib-devhelp": {
+            "hashes": [
+                "sha256:6c64b077937330a9128a4da74586e8c2130262f014689b4b89e2d08ee7294a34",
+                "sha256:9512ecb00a2b0821a146736b39f7aeb90759834b07e81e8cc23a9c70bacb9981"
+            ],
+            "version": "==1.0.1"
+        },
+        "sphinxcontrib-htmlhelp": {
+            "hashes": [
+                "sha256:4670f99f8951bd78cd4ad2ab962f798f5618b17675c35c5ac3b2132a14ea8422",
+                "sha256:d4fd39a65a625c9df86d7fa8a2d9f3cd8299a3a4b15db63b50aac9e161d8eff7"
+            ],
+            "version": "==1.0.2"
+        },
+        "sphinxcontrib-jsmath": {
+            "hashes": [
+                "sha256:2ec2eaebfb78f3f2078e73666b1415417a116cc848b72e5172e596c871103178",
+                "sha256:a9925e4a4587247ed2191a22df5f6970656cb8ca2bd6284309578f2153e0c4b8"
+            ],
+            "version": "==1.0.1"
+        },
+        "sphinxcontrib-qthelp": {
+            "hashes": [
+                "sha256:513049b93031beb1f57d4daea74068a4feb77aa5630f856fcff2e50de14e9a20",
+                "sha256:79465ce11ae5694ff165becda529a600c754f4bc459778778c7017374d4d406f"
+            ],
+            "version": "==1.0.2"
+        },
+        "sphinxcontrib-serializinghtml": {
+            "hashes": [
+                "sha256:c0efb33f8052c04fd7a26c0a07f1678e8512e0faec19f4aa8f2473a8b81d5227",
+                "sha256:db6615af393650bf1151a6cd39120c29abaf93cc60db8c48eb2dddbfdc3a9768"
+            ],
+            "version": "==1.1.3"
+        },
+        "typed-ast": {
+            "hashes": [
+                "sha256:0666aa36131496aed8f7be0410ff974562ab7eeac11ef351def9ea6fa28f6355",
+                "sha256:0c2c07682d61a629b68433afb159376e24e5b2fd4641d35424e462169c0a7919",
+                "sha256:249862707802d40f7f29f6e1aad8d84b5aa9e44552d2cc17384b209f091276aa",
+                "sha256:24995c843eb0ad11a4527b026b4dde3da70e1f2d8806c99b7b4a7cf491612652",
+                "sha256:269151951236b0f9a6f04015a9004084a5ab0d5f19b57de779f908621e7d8b75",
+                "sha256:4083861b0aa07990b619bd7ddc365eb7fa4b817e99cf5f8d9cf21a42780f6e01",
+                "sha256:498b0f36cc7054c1fead3d7fc59d2150f4d5c6c56ba7fb150c013fbc683a8d2d",
+                "sha256:4e3e5da80ccbebfff202a67bf900d081906c358ccc3d5e3c8aea42fdfdfd51c1",
+                "sha256:6daac9731f172c2a22ade6ed0c00197ee7cc1221aa84cfdf9c31defeb059a907",
+                "sha256:715ff2f2df46121071622063fc7543d9b1fd19ebfc4f5c8895af64a77a8c852c",
+                "sha256:73d785a950fc82dd2a25897d525d003f6378d1cb23ab305578394694202a58c3",
+                "sha256:8c8aaad94455178e3187ab22c8b01a3837f8ee50e09cf31f1ba129eb293ec30b",
+                "sha256:8ce678dbaf790dbdb3eba24056d5364fb45944f33553dd5869b7580cdbb83614",
+                "sha256:aaee9905aee35ba5905cfb3c62f3e83b3bec7b39413f0a7f19be4e547ea01ebb",
+                "sha256:bcd3b13b56ea479b3650b82cabd6b5343a625b0ced5429e4ccad28a8973f301b",
+                "sha256:c9e348e02e4d2b4a8b2eedb48210430658df6951fa484e59de33ff773fbd4b41",
+                "sha256:d205b1b46085271b4e15f670058ce182bd1199e56b317bf2ec004b6a44f911f6",
+                "sha256:d43943ef777f9a1c42bf4e552ba23ac77a6351de620aa9acf64ad54933ad4d34",
+                "sha256:d5d33e9e7af3b34a40dc05f498939f0ebf187f07c385fd58d591c533ad8562fe",
+                "sha256:fc0fea399acb12edbf8a628ba8d2312f583bdbdb3335635db062fa98cf71fca4",
+                "sha256:fe460b922ec15dd205595c9b5b99e2f056fd98ae8f9f56b888e7a17dc2b757e7"
+            ],
+            "version": "==1.4.1"
+        },
+        "typing-extensions": {
+            "hashes": [
+                "sha256:091ecc894d5e908ac75209f10d5b4f118fbdb2eb1ede6a63544054bb1edb41f2",
+                "sha256:910f4656f54de5993ad9304959ce9bb903f90aadc7c67a0bef07e678014e892d",
+                "sha256:cf8b63fedea4d89bab840ecbb93e75578af28f76f66c35889bd7065f5af88575"
+            ],
+            "version": "==3.7.4.1"
+        },
+        "urllib3": {
+            "hashes": [
+                "sha256:2f3db8b19923a873b3e5256dc9c2dedfa883e33d87c690d9c7913e1f40673cdc",
+                "sha256:87716c2d2a7121198ebcb7ce7cccf6ce5e9ba539041cfbaeecfb641dc0bf6acc"
+            ],
+            "version": "==1.25.8"
+        },
+        "wrapt": {
+            "hashes": [
+                "sha256:565a021fd19419476b9362b05eeaa094178de64f8361e44468f9e9d7843901e1"
+            ],
+            "version": "==1.11.2"
+        }
+    }
+}

README.rst

@@ -2,7 +2,7 @@
 
 .. badges-start
 
-|travis| |pypi| |pyversion| |license| |aur| |contributors| |downloads| |saythanks|
+|travis| |pypi| |pyversion| |license| |aur| |contributors| |downloads|
 
 .. |travis| image:: https://img.shields.io/travis/instaloader/instaloader/master.svg
    :alt: Travis-CI Build Status
@@ -23,10 +23,6 @@
    :alt: Contributor Count
    :target: https://github.com/instaloader/instaloader/graphs/contributors
 
-.. |saythanks| image:: https://img.shields.io/badge/Say%20Thanks-!-ff7efd.svg
-   :alt: Say Thanks!
-   :target: https://saythanks.io/to/aandergr
-
 .. |aur| image:: https://img.shields.io/aur/version/instaloader.svg
    :alt: Arch User Repository Package
    :target: https://aur.archlinux.org/packages/instaloader/
@@ -57,7 +53,8 @@
 
 ::
 
-    instaloader [--comments] [--geotags] [--stories] [--highlights] [--tagged]
+    instaloader [--comments] [--geotags]
+                [--stories] [--highlights] [--tagged] [--igtv]
                 [--login YOUR-USERNAME] [--fast-update]
                 profile | "#hashtag" | :stories | :feed | :saved
 

deploy/arch/deploy.sh

@@ -1,5 +1,10 @@
 #!/usr/bin/env bash
 
+VERSION=${1:1}
+
+# do not deploy pre-releases
+echo $VERSION | grep -qe "[abc]" && exit 0
+
 cd $(dirname $0)
 
 # decrypt and add ssh key
@@ -12,7 +17,6 @@ ssh-add /tmp/AUR_openssh
 git clone --depth 1 ssh://aur@aur.archlinux.org/instaloader.git
 
 curl -sSfOJ https://codeload.github.com/instaloader/instaloader/tar.gz/$1
-VERSION=${1:1}
 HASH=$(sha512sum instaloader-$VERSION.tar.gz | cut -f1 -d " ")
 sed -e "s/{{version}}/$VERSION/g" -e "s/{{hash}}/$HASH/g" PKGBUILD.template > instaloader/PKGBUILD
 sed -e "s/{{version}}/$VERSION/g" -e "s/{{hash}}/$HASH/g" .SRCINFO.template > instaloader/.SRCINFO

deploy/windows/create_exe.py

@@ -0,0 +1,71 @@
+import hashlib
+import os
+import re
+import shutil
+import subprocess
+import sys
+
+# copy the required files into repo root
+shutil.copy('docs/favicon.ico', '.')
+shutil.copy('deploy/windows/instaloader.spec', '.')
+
+code = """
+import contextlib
+import psutil
+import subprocess
+
+def __main():
+    with contextlib.suppress(AttributeError, psutil.Error):
+        if psutil.Process().parent().parent().name() == "explorer.exe":
+            subprocess.Popen("powershell -NoExit -Command \\\"& {0}\\\"".format(sys.argv[0]))
+            return
+    main()
+
+if __name__ == "__main__":
+    __main()
+"""
+
+with open('instaloader/__main__.py', 'r') as f:
+    # adjust imports for changed file structure
+    regex = re.compile(r'from (?:(\.[^ ]+ )|\.( ))import')
+    lines = [regex.sub(r'from instaloader\1\2import', line) for line in f.readlines()]
+
+    # insert code for magic exe behavior
+    index = lines.index('if __name__ == "__main__":\n')
+    code_lines = [cl + '\n' for cl in code.splitlines()]
+    for i, code_line in enumerate(code_lines):
+        if i + index < len(lines):
+            lines[i + index] = code_line
+        else:
+            lines.extend(code_lines[i:])
+            break
+
+with open('__main__.py', 'w+') as f:
+    f.writelines(lines)
+
+# install dependencies and invoke PyInstaller
+commands = ["pip install pipenv==2018.11.26",
+            "pipenv sync --dev",
+            "pipenv run pyinstaller --log-level=DEBUG instaloader.spec"]
+
+for command in commands:
+    print()
+    print('#' * (len(command) + 6))
+    print('## {} ##'.format(command))
+    print('#' * (len(command) + 6))
+    print(flush=True)
+    err = subprocess.Popen(command).wait()
+    if err != 0:
+        sys.exit(err)
+
+# calculate and store MD5 hash for created executable
+hash_md5 = hashlib.md5()
+with open('dist/instaloader.exe', 'rb') as f:
+    for chunk in iter(lambda: f.read(4096), b''):
+        hash_md5.update(chunk)
+
+with open('dist/instaloader.exe.md5', 'w+') as f:
+    f.write('{} *instaloader.exe\n'.format(hash_md5.hexdigest()))
+
+# Create ZIP file
+shutil.make_archive('instaloader-{}-windows-standalone'.format(os.getenv('VERSION_TAG')), 'zip', 'dist')

deploy/windows/instaloader.spec

@@ -0,0 +1,28 @@
+# -*- mode: python -*-
+
+block_cipher = None
+
+
+a = Analysis(['__main__.py'],
+             binaries=[],
+             datas=[],
+             hiddenimports=[],
+             hookspath=[],
+             runtime_hooks=[],
+             excludes=[],
+             win_no_prefer_redirects=False,
+             win_private_assemblies=False,
+             cipher=block_cipher)
+pyz = PYZ(a.pure, a.zipped_data,
+             cipher=block_cipher)
+exe = EXE(pyz,
+          a.scripts,
+          a.binaries,
+          a.zipfiles,
+          a.datas,
+          name='instaloader',
+          debug=False,
+          strip=False,
+          upx=False,
+          runtime_tmpdir=None,
+          console=True , icon='favicon.ico')

docs/_static/instaloader.css

@@ -18,14 +18,17 @@ code {
     padding: 0.7em;
     color: #fff; }
   .highlight .c1 {
-    color: #008d06; }
+    color: #666; }
   .highlight .k, .highlight .kn, .highlight .ow {
-    color: #f48400;
-    font-weight: bold; }
-  .highlight .nb {
+    color: #008d06; }
+  .highlight .nb, .highlight .ne, .highlight .nf, .highlight .vm {
     color: #f48400; }
-  .highlight .s1, .highlight .s2, .highlight .mi {
+  .highlight .s1, .highlight .s2, .highlight .sa, .highlight .se, .highlight .si, .highlight .mi {
     color: #ff7efd; }
+  .highlight .c1, .highlight .nf, .highlight .sa, .highlight .se, .highlight .si {
+    font-style: italic; }
+  .highlight .k, .highlight .kn, .highlight .ow {
+    font-weight: bold; }
 
 .doc-content {
   padding-top: 0.5em;

docs/_static/instaloader.scss

@@ -37,21 +37,28 @@ code {
   }
 
   .c1 {
-    color: $color_instaloader_main
+    color: #666;
   }
 
   .k, .kn, .ow {
-    color: $color_instaloader_accent;
-    font-weight: bold
+    color: $color_instaloader_main
   }
 
-  .nb {
+  .nb, .ne, .nf, .vm {
     color: $color_instaloader_accent
   }
 
-  .s1, .s2, .mi {
+  .s1, .s2, .sa, .se, .si, .mi {
     color: $color_instaloader_hyper
   }
+
+  .c1, .nf, .sa, .se, .si {
+    font-style: italic;
+  }
+
+  .k, .kn, .ow {
+    font-weight: bold
+  }
 }
 
 .doc-content {

docs/_templates/layout.html

@@ -51,9 +51,6 @@
         <li class="nav-item">
           <a target="_blank" class="nav-link" href="https://github.com/instaloader/instaloader">GitHub</a>
         </li>
-        <li class="nav-item d-sm-none d-lg-block">
-          <a target="_blank" class="nav-link" href="https://saythanks.io/to/aandergr">Say Thanks!</a>
-        </li>
       </ul>
       <form class="form-inline" action="search.html" method="get">
         <div class="input-group my-2 my-sm-0 ml-2">

docs/as-module.rst

@@ -37,7 +37,7 @@ picture, video or sidecar (set of multiple pictures/videos) posted in a user's
 profile. :class:`Instaloader` provides methods to iterate over Posts from a
 certain source::
 
-    for post in L.get_hashtag_posts('cat'):
+    for post in instaloader.Hashtag.from_name(L.context, 'cat').get_posts():
         # post is an instance of instaloader.Post
         L.download_post(post, target='#cat')
 
@@ -63,7 +63,7 @@ certain source::
 - :meth:`Instaloader.get_explore_posts`
    Media that is suggested by Instagram to explore.
 
-- :meth:`Instaloader.get_hashtag_posts`
+- :meth:`Hashtag.get_posts`
    Media associated with given hashtag.
 
 With the :class:`Profile` class, Instaloader also makes it easy to access
@@ -155,10 +155,26 @@ Profiles
 .. autoclass:: Profile
    :no-show-inheritance:
 
+Hashtags
+""""""""
+
+.. autoclass:: Hashtag
+   :no-show-inheritance:
+
+   .. versionadded:: 4.4
+
+TopSearchResults
+""""""""""""""""
+
+.. autoclass:: TopSearchResults
+   :no-show-inheritance:
+
+   .. versionadded:: 4.3
+
 Loading and Saving
 """"""""""""""""""
 
-:class:`Post`, :class:`StoryItem` and :class:`Profile` can be saved and loaded
+:class:`Post`, :class:`StoryItem`, :class:`Profile` and :class:`Hashtag` can be saved and loaded
 to/from JSON files.
 
 .. autofunction:: load_structure_from_file
@@ -179,8 +195,6 @@ Exceptions
 
 .. autoexception:: QueryReturnedBadRequestException
 
-.. autoexception:: QueryReturnedNotFoundException
-
 .. autoexception:: QueryReturnedForbiddenException
 
 .. autoexception:: ProfileNotExistsException
@@ -203,6 +217,13 @@ Exceptions
 
 .. autoexception:: PostChangedException
 
+.. autoexception:: QueryReturnedNotFoundException
+
+   .. versionchanged:: 4.3
+      QueryReturnedNotFoundException now inherits ConnectionException
+      to retry on 404 errors.
+
+
 .. autoexception:: TooManyRequestsException
 
 ``InstaloaderContext`` (Low-level functions)

docs/basic-usage.rst

@@ -51,7 +51,7 @@ invoke it with
     instaloader --login=your_username profile [profile ...]
 
 When logging in, Instaloader **stores the session cookies** in a file in your
-temporary directory, which will be reused later the next time :option:`--login`
+home directory, which will be reused later the next time :option:`--login`
 is given.  So you can download private profiles **non-interactively** when you
 already have a valid session cookie file.
 
@@ -77,10 +77,13 @@ Instaloader supports the following targets:
       instructs Instaloader to also **download the user's stories**,
 
    - :option:`--highlights`
-      to **download highlights of each profile that is downloaded**, and
+      to **download highlights of each profile that is downloaded**,
 
    - :option:`--tagged`
-      to **download posts where the user is tagged**.
+      to **download posts where the user is tagged**, and
+
+   - :option:`--igtv`
+      to **download IGTV videos**.
 
 - ``"#hashtag"``
    Posts with a certain **hashtag** (the quotes are usually necessary),
@@ -89,6 +92,7 @@ Instaloader supports the following targets:
    Posts tagged with a given location; the location ID is the numerical ID
    Instagram labels a location with (e.g.
    \https://www.instagram.com/explore/locations/**362629379**/plymouth-naval-memorial/).
+   Requires :option:`--login`.
 
    .. versionadded:: 4.2
 
@@ -107,8 +111,12 @@ Instaloader supports the following targets:
    ``profile`` (requires :option:`--login`).
 
 - ``-post``
-   The single **post** with the given shortcode. Must be preceeded by ``--`` in
-   the argument list to not be mistaken as an option flag.
+   Replace **post** with the post's shortcode to download single post. Must be preceeded by ``--`` in
+   the argument list to not be mistaken as an option flag::
+
+    instaloader -- -B_K4CykAOtf
+
+
 
    .. versionadded:: 4.1
 
@@ -122,7 +130,7 @@ downloads the pictures and videos and their captions. You can specify
 
 - :option:`--geotags`
    **download geotags** of each post and save them as
-   Google Maps link,
+   Google Maps link (requires :option:`--login`),
 
 For a reference of all supported command line options, see
 :ref:`command-line-options`.
@@ -287,17 +295,18 @@ Instaloader as Cronjob
 Instaloader is suitable for running as a cronjob to periodically update your
 personal Instagram archive. The :option:`--quiet` option disables user
 interactions and logging of non-error messages. To non-interactively use
-Instaloader logged-in, create a session file, e.g. in your home directory::
+Instaloader logged-in, create a session file::
 
-   instaloader --login=your_username --sessionfile=~/.instaloader-session
+   instaloader --login=your_username
 
-Then use the same parameters in your cronjob to load the session and download
+Then use the same parameter in your cronjob to load the session and download
 the given targets::
 
-   instaloader --login=your_username --sessionfile=~/.instaloader-session --quiet <target> [...]
+   instaloader --login=your_username --quiet <target> [...]
 
-Without :option:`--sessionfile` option, Instaloader saves the session file in
-a path within your temporary directory.
+Instaloader saves the session file to
+``~/.config/instaloader/session-YOUR-USERNAME``. See
+:option:`--sessionfile` option for how to override this path.
 
 Programming Instaloader
 ^^^^^^^^^^^^^^^^^^^^^^^

docs/cli-options.rst

@@ -61,8 +61,7 @@ What to Download of each Post
 
    **Download geotags** when available. Geotags are stored as a text file with
    the location's name and a Google Maps link. This requires an additional
-   request to the Instagram server for each picture, which is why it is disabled
-   by default.
+   request to the Instagram server for each picture. Requires :option:`--login`.
 
 .. option:: --comments, -C
 
@@ -130,6 +129,12 @@ What to Download of each Profile
 
    .. versionadded:: 4.1
 
+.. option:: --igtv
+
+   Also download IGTV videos.
+
+   .. versionadded:: 4.3
+
 .. option:: --stories-only
 
    .. deprecated:: 4.1
@@ -188,9 +193,8 @@ Instaloader to login.
 
 .. option:: --sessionfile SESSIONFILE, -f SESSIONFILE
 
-   Path for loading and storing session key file.  Defaults to a path within
-   your temporary directory, encoding your local username and your Instagram
-   profile name.
+   Path for loading and storing session key file.  Defaults to
+   ``~/.config/instaloader/session-YOUR-USERNAME``.
 
 .. option:: --password YOUR-PASSWORD, -p YOUR-PASSWORD
 
@@ -238,6 +242,17 @@ How to Download
 
    .. versionadded:: 4.2
 
+   .. warning::
+      We discourage from using the "commit mode" due to its bugs when being used
+      with other command line options (e.g. :issue:`257` with
+      :option:`--fast-update` and :issue:`483` with :option:`--tagged`).
+
+.. option:: --request-timeout N
+
+   Seconds to wait before timing out a connection request.
+
+   .. versionadded:: 4.3
+
 Miscellaneous Options
 ^^^^^^^^^^^^^^^^^^^^^
 

docs/codesnippets.rst

@@ -28,8 +28,9 @@ Download Posts in a Specific Period
 -----------------------------------
 
 To only download Instagram pictures (and metadata) that are within a specific
-period, you can play around with :func:`~itertools.dropwhile` and
-:func:`~itertools.takewhile` from :mod:`itertools` like in this snippet.
+period, you can simply use :func:`~itertools.dropwhile` and
+:func:`~itertools.takewhile` from :mod:`itertools` on a generator that returns
+Posts in **exact chronological order**, such as :meth:`Profile.get_posts`.
 
 .. literalinclude:: codesnippets/121_since_until.py
 
@@ -37,6 +38,14 @@ See also :class:`Post`, :meth:`Instaloader.download_post`.
 
 Discussed in :issue:`121`.
 
+The code example with :func:`~itertools.dropwhile` and
+:func:`~itertools.takewhile` makes the assumption that the post iterator returns
+posts in exact chronological order.  As discussed in :issue:`666`, the following
+approach fits for an **almost chronological order**, where up to *k* older posts
+are inserted into an otherwise chronological order, such as an Hashtag feed.
+
+.. literalinclude:: codesnippets/666_historical_hashtag_data.py
+
 Likes of a Profile / Ghost Followers
 ------------------------------------
 

docs/codesnippets/113_only_one_per_user.py

@@ -2,13 +2,13 @@ import instaloader
 
 L = instaloader.Instaloader()
 
-posts = L.get_hashtag_posts('milfgarden')
+posts = instaloader.Hashtag.from_name(L.context, 'urbanphotography').get_posts()
 
 users = set()
 
 for post in posts:
     if not post.owner_profile in users:
-        L.download_post(post, '#milfgarden')
+        L.download_post(post, '#urbanphotography')
         users.add(post.owner_profile)
     else:
         print("{} from {} skipped.".format(post, post.owner_profile))

docs/codesnippets/121_since_until.py

@@ -5,13 +5,11 @@ import instaloader
 
 L = instaloader.Instaloader()
 
-posts = L.get_hashtag_posts('milfgarden')
-# or
-# posts = instaloader.Profile.from_username(L.context, PROFILE).get_posts()
+posts = instaloader.Profile.from_username(L.context, "instagram").get_posts()
 
 SINCE = datetime(2015, 5, 1)
 UNTIL = datetime(2015, 3, 1)
 
 for post in takewhile(lambda p: p.date > UNTIL, dropwhile(lambda p: p.date > SINCE, posts)):
     print(post.date)
-    L.download_post(post, '#milfgarden')
+    L.download_post(post, "instagram")

docs/codesnippets/194_top_x_of_user.py

@@ -9,7 +9,9 @@ X_percentage = 10    # percentage of posts that should be downloaded
 L = Instaloader()
 
 profile = Profile.from_username(L.context, PROFILE)
-posts_sorted_by_likes = sorted(profile.get_posts(), key = lambda p: p.likes + p.comments)
+posts_sorted_by_likes = sorted(profile.get_posts(),
+                               key=lambda p: p.likes + p.comments,
+                               reverse=True)
 
 for post in islice(posts_sorted_by_likes, ceil(profile.mediacount * X_percentage / 100)):
     L.download_post(post, PROFILE)

docs/codesnippets/615_import_firefox_session.py

@@ -0,0 +1,53 @@
+from argparse import ArgumentParser
+from glob import glob
+from os.path import expanduser
+from platform import system
+from sqlite3 import OperationalError, connect
+
+try:
+    from instaloader import ConnectionException, Instaloader
+except ModuleNotFoundError:
+    raise SystemExit("Instaloader not found.\n  pip install [--user] instaloader")
+
+
+def get_cookiefile():
+    default_cookiefile = {
+        "Windows": "~/AppData/Roaming/Mozilla/Firefox/Profiles/*/cookies.sqlite",
+        "Darwin": "~/Library/Application Support/Firefox/Profiles/*/cookies.sqlite",
+    }.get(system(), "~/.mozilla/firefox/*/cookies.sqlite")
+    cookiefiles = glob(expanduser(default_cookiefile))
+    if not cookiefiles:
+        raise SystemExit("No Firefox cookies.sqlite file found. Use -c COOKIEFILE.")
+    return cookiefiles[0]
+
+
+def import_session(cookiefile, sessionfile):
+    print("Using cookies from {}.".format(cookiefile))
+    conn = connect(cookiefile)
+    try:
+        cookie_data = conn.execute(
+            "SELECT name, value FROM moz_cookies WHERE baseDomain='instagram.com'"
+        )
+    except OperationalError:
+        cookie_data = conn.execute(
+            "SELECT name, value FROM moz_cookies WHERE host LIKE '%instagram.com'"
+        )
+    instaloader = Instaloader(max_connection_attempts=1)
+    instaloader.context._session.cookies.update(cookie_data)
+    username = instaloader.test_login()
+    if not username:
+        raise SystemExit("Not logged in. Are you logged in successfully in Firefox?")
+    print("Imported session cookie for {}.".format(username))
+    instaloader.context.username = username
+    instaloader.save_session_to_file(sessionfile)
+
+
+if __name__ == "__main__":
+    p = ArgumentParser()
+    p.add_argument("-c", "--cookiefile")
+    p.add_argument("-f", "--sessionfile")
+    args = p.parse_args()
+    try:
+        import_session(args.cookiefile or get_cookiefile(), args.sessionfile)
+    except (ConnectionException, OperationalError) as e:
+        raise SystemExit("Cookie import failed: {}".format(e))

docs/codesnippets/666_historical_hashtag_data.py

@@ -0,0 +1,30 @@
+from datetime import datetime
+import instaloader
+
+L = instaloader.Instaloader()
+
+posts = instaloader.Hashtag.from_name(L.context, "urbanphotography").get_posts()
+
+SINCE = datetime(2020, 5, 10)  # further from today, inclusive
+UNTIL = datetime(2020, 5, 11)  # closer to today, not inclusive
+
+k = 0  # initiate k
+k_list = []  # uncomment this to tune k
+
+for post in posts:
+    postdate = post.date
+
+    if postdate > UNTIL:
+        continue
+    elif postdate <= SINCE:
+        k += 1
+        if k == 50:
+            break
+        else:
+            continue
+    else:
+        L.download_post(post, "#urbanphotography")
+        k = 0  # set k to 0
+        # if you want to tune k, uncomment below to get your k max
+        #k_list.append(k)
+#max(k_list)

docs/codesnippets/92_import_firefox_session.py

@@ -1,23 +0,0 @@
-from glob import glob
-from os.path import expanduser
-from sqlite3 import connect
-
-from instaloader import ConnectionException, Instaloader
-
-# FIREFOXCOOKIEFILE = "/home/alex/.mozilla/firefox/l96w6b90.default/cookies.sqlite"
-FIREFOXCOOKIEFILE = glob(expanduser("~/.mozilla/firefox/*.default/cookies.sqlite"))[0]
-
-instaloader = Instaloader(max_connection_attempts=1)
-instaloader.context._session.cookies.update(connect(FIREFOXCOOKIEFILE)
-                                            .execute("SELECT name, value FROM moz_cookies "
-                                                     "WHERE baseDomain='instagram.com'"))
-
-try:
-    username = instaloader.test_login()
-    if not username:
-        raise ConnectionException()
-except ConnectionException:
-    raise SystemExit("Cookie import failed. Are you logged in successfully in Firefox?")
-
-instaloader.context.username = username
-instaloader.save_session_to_file()

docs/conf.py

@@ -18,6 +18,7 @@
 # documentation root, use os.path.abspath to make it absolute, like shown here.
 #
 import os
+import platform
 import subprocess
 import sys
 import typing
@@ -47,9 +48,25 @@ autodoc_default_flags = ['show-inheritance', 'members', 'undoc-members']
 autodoc_member_order = 'bysource'
 
 intersphinx_mapping = {'python': ('https://docs.python.org/3', None),
-                       'requests': ('http://docs.python-requests.org/en/master', None)}
+                       'requests': ('https://requests.kennethreitz.org/en/master/', None)}
 
-extlinks = {'issue': ('https://github.com/instaloader/instaloader/issues/%s', 'issue #')}
+current_release = subprocess.check_output(["git", "describe", "--abbrev=0"]).decode("ascii")[1:-1]
+date_format = "%e %b %Y" if platform.system() != "Windows" else "%d %b %Y"
+current_release_date = subprocess.check_output(
+    ["git", "log", "-1", "--tags", "--format=%ad", "--date=format:"+date_format]).decode("ascii")[:-1]
+
+html_context = {'current_release': current_release, 'current_release_date': current_release_date}
+
+extlinks = {
+    'issue': (
+        'https://github.com/instaloader/instaloader/issues/%s',
+        'Issue #'
+    ),
+    'example': (
+        'https://raw.githubusercontent.com/instaloader/instaloader/v' + current_release + '/docs/codesnippets/%s',
+        'Example '
+    ),
+}
 
 # Add any paths that contain templates here, relative to this directory.
 templates_path = ['_templates']
@@ -69,7 +86,7 @@ master_doc = 'index'
 
 # General information about the project.
 project = 'Instaloader'
-copyright = '2017, Alexander Graf and Andre Koch-Kramer'
+copyright = '2017-2020, Alexander Graf and Andre Koch-Kramer'
 author = 'Alexander Graf and Andre Koch-Kramer'
 
 # The version info for the project you're documenting, acts as replacement for
@@ -353,12 +370,6 @@ texinfo_documents = [
 # texinfo_no_detailmenu = False
 
 
-current_release = subprocess.check_output(["git", "describe", "--abbrev=0"]).decode("ascii")[1:-1]
-current_release_date = subprocess.check_output(["git", "log", "-1", "--tags", "--format=%ad", "--date=format:%e %b %Y"]).decode("ascii")[:-1]
-
-html_context = {'current_release': current_release, 'current_release_date': current_release_date}
-
-
 def setup(app):
     typing.TYPE_CHECKING = True
     app.add_stylesheet("instaloader.css")

docs/contributing.rst

@@ -15,10 +15,8 @@ Reporting Bugs
 --------------
 
 If you encounter a bug, do not hesitate to report it in our
-`Issue Tracker <https://github.com/instaloader/instaloader/issues>`__. Since
-Instaloader is actively developed, the majority of bugs is fixed within only
-**4 days** after being reported. When reporting a problem, please keep the
-following in mind:
+`Issue Tracker <https://github.com/instaloader/instaloader/issues>`__. When
+reporting a problem, please keep the following in mind:
 
 - Ensure you **use the latest version** of Instaloader. The currently-installed
   version can be found out with ``instaloader --version``.
@@ -37,8 +35,8 @@ following in mind:
 - If not obvious, describe **which behavior you expected**
   instead of what actually happened.
 
-- If you think an issue has been closed accidentally or inappropriately, feel
-  free to reopen it.
+- If we have closed an issue apparently inadvertently or inappropriately, please
+  let us know.
 
 Writing Code or Improving the Documentation
 -------------------------------------------
@@ -47,29 +45,35 @@ Changes of the Instaloader source can be proposed as a
 `Pull Request <https://github.com/instaloader/instaloader/pulls>`__. There are only
 few things to consider:
 
-- Sometimes, the most current code is not in the ``master`` branch. Check that
-  you forked from the most recent branch.
+- Base your Pull Request on the ``master`` branch if it fixes a bug,
+  or the ``upcoming/v4.X`` branch (if it exists at the moment of submitting the PR)
+  otherwise.
 
 - We use `Pylint <https://www.pylint.org/>`__ for error and syntax checking of
-  the source. The file ``.travis.yml`` in the project's root directory
-  shows how it is invoked. Note that sometimes it might be better to disable a
-  warning rather than adapting the code to Pylint's desires.
+  the source and `MyPy <https://github.com/python/mypy>`__ for type checking.
+  Beware that sometimes it might be better to disable a warning rather than
+  adapting the code to a tool's desires.
 
 - The documentation source is located in the ``docs`` folder. The file
   ``cli-options.rst`` is merely an RST-formatted copy of ``instaloader --help``
   output, of which the source is in ``instaloader/__main__.py``.
 
-- Feel free to contact us, even if you "only" have Proof-of-Concepts or
-  not-fully integrated changes. They already might be an advance for the
-  project.
+Proposing Features
+------------------
 
-Suggesting Features
--------------------
+.. goal-start
 
-If you have an idea of how Instaloader should be enhanced, but do not want to
-implement the feature yourself, feel free to open a ticket in our 
-`Issue Tracker <https://github.com/instaloader/instaloader/issues>`__.
-Please consider the following:
+Instaloader's goal is to mimic the browser's behavior to access the data that
+is available through the Instagram web interface, pack this data into complete
+and easily-(re)usable python objects, and provide a user interface for the most
+common downloading and metadata collection tasks, without changing any of the
+online data.
+
+.. goal-end
+
+Prior spending effort on implementing a new feature, it might be appropriate to
+clarify how it could fit into the project's scope or discuss implementation
+details. If you feel the need to do so, please create a "feature suggestion".
 
 - Instaloader already has plenty of features. **Check the documentation**
   beforehand to ensure your desired suggestion is not already implemented.
@@ -78,13 +82,21 @@ Please consider the following:
   an issue suggesting the same or a similar feature, share your thoughts in a
   comment there, instead of opening a new issue.
 
-- If possible, provide us a **use case of the feature**: How could the user
-  invoke the new function? Which problem would it solve? If new information is
-  obtained, how would it be further processed?
+- **Motivate the feature**, i.e.
 
-- **Be patient**. Naturally, bugs and pull requests have a higher priority than
-  feature suggestions. Keep in mind that this is a free software project, and
-  unfortunately we only have limited time to work on it.
+  - Provide us a **use case of the feature**: How could the user
+    invoke the new function? Which problem would it solve? If new information is
+    obtained, how would it be further processed?
+
+  - Describe already-working **alternatives of the feature** and how they
+    compare to your proposed feature.
+
+  - Briefly describe how your suggested feature **conforms with Instaloader's
+    project goal**.
+
+- Explain your **solution ideas**. Describe your ideas on how the feature could
+  be implemented and the underlying problem could be solved. Also **describe
+  alternatives** that you have considered.
 
 Donations
 ---------

docs/index.rst

@@ -45,7 +45,8 @@ See :ref:`install` for more options on how to install Instaloader.
 
 ::
 
-    instaloader [--comments] [--geotags] [--stories] [--highlights] [--tagged]
+    instaloader [--comments] [--geotags]
+                [--stories] [--highlights] [--tagged] [--igtv]
                 [--login YOUR-USERNAME] [--fast-update]
                 profile | "#hashtag" | %location_id |
                 :stories | :feed | :saved

docs/troubleshooting.rst

@@ -27,12 +27,26 @@ expired. In case the retry continuously fails for some reason, which should not
 happen in normal conditions, consider adjusting the
 :option:`--max-connection-attempts` option.
 
+There have been observations that services, that in their nature offer
+promiscious IP addresses, such as cloud, VPN and public proxy services, might be
+subject to significantly stricter limits for anonymous access. However,
+logged-in accesses (see :option:`--login`) do not seem to be affected.
+
+Too many queries in the last time
+---------------------------------
+
 **"Too many queries in the last time"** is not an error. It is a notice that the
 rate limit has almost been reached, according to Instaloader's own rate
 accounting mechanism. We regularly adjust this mechanism to match Instagram's
 current rate limiting.
 
-Login error
+Private but not followed
+------------------------
+
+You have to follow a private account to access most of its associated
+information.
+
+Login Error
 -----------
 
 Instaloader's login *should* work fine, both with and without
@@ -40,21 +54,31 @@ Two-Factor-Authentication. It also supports handling the *checkpoint challenge*,
 issued when Instagram suspects authentication activity on your account, by
 pointing the user to an URL to be opened in a browser.
 
-Nevertheless, in :issue:`92` users report problems with logging in. To still use
-Instaloader's logged-in functionality, you may use the following script to
-workaround login problems by importing the session cookies from Firefox and
-bypassing Instaloader's login.
+Nevertheless, in :issue:`92` and :issue:`615` users reported problems with
+logging in. We recommend to always keep the sessionfile which Instaloader
+creates when using :option:`--login`. If a sessionfile is present,
+:option:`--login` does not make make use of the failure-prone login procedure.
+Also, sessionfiles usually do not expire and can be copied between different
+computers without any problems.
 
-.. literalinclude:: codesnippets/92_import_firefox_session.py
+If you do not have a sessionfile present, you may use the following script
+(:example:`615_import_firefox_session.py`) to workaround login problems by
+importing the session cookies from Firefox and bypassing Instaloader's login and
+so still use Instaloader's logged-in functionality.
 
-To use this,
+.. literalinclude:: codesnippets/615_import_firefox_session.py
 
-#. login to Instagram in Firefox, 
+To use this script,
 
-#. execute the snippet,
+#. Download the script: :example:`615_import_firefox_session.py`,
 
-#. then, ``instaloader -l USERNAME`` should work fine.
+#. Login to Instagram in Firefox,
 
-If you do not use your default firefox profile, or your operating system has the
-paths differently set up, you may have to alter the ``FIREFOXCOOKIEFILE``
-variable first.
+#. Execute the snippet, e.g. with ``python 615_import_firefox_session.py``,
+
+#. Then, ``instaloader -l USERNAME`` should work fine.
+
+This script also supports specifying a cookiefile path, which may be useful if
+you use multiple Firefox profiles or if your operating system has the directory
+structure differently set up. Also, you can specify an alternative sessionfile
+path.

instaloader/__init__.py

@@ -1,7 +1,7 @@
 """Download pictures (or videos) along with their captions and other metadata from Instagram."""
 
 
-__version__ = '4.2.5'
+__version__ = '4.4.5rc1'
 
 
 try:
@@ -15,5 +15,5 @@ else:
 from .exceptions import *
 from .instaloader import Instaloader
 from .instaloadercontext import InstaloaderContext
-from .structures import (Highlight, Post, PostSidecarNode, PostComment, PostCommentAnswer, PostLocation, Profile, Story, StoryItem,
-                         load_structure_from_file, save_structure_to_file)
+from .structures import (Hashtag, Highlight, Post, PostSidecarNode, PostComment, PostCommentAnswer, PostLocation,
+                         Profile, Story, StoryItem, TopSearchResults, load_structure_from_file, save_structure_to_file)

instaloader/__main__.py

@@ -3,6 +3,7 @@
 import ast
 import datetime
 import os
+import re
 import sys
 from argparse import ArgumentParser, SUPPRESS
 from typing import List, Optional
@@ -19,7 +20,8 @@ def usage_string():
     argv0 = os.path.basename(sys.argv[0])
     argv0 = "instaloader" if argv0 == "__main__.py" else argv0
     return """
-{0} [--comments] [--geotags] [--stories] [--highlights] [--tagged]
+{0} [--comments] [--geotags]
+{2:{1}} [--stories] [--highlights] [--tagged] [--igtv]
 {2:{1}} [--login YOUR-USERNAME] [--fast-update]
 {2:{1}} profile | "#hashtag" | %%location_id | :stories | :feed | :saved
 {0} --help""".format(argv0, len(argv0), '')
@@ -61,7 +63,10 @@ def _main(instaloader: Instaloader, targetlist: List[str],
           username: Optional[str] = None, password: Optional[str] = None,
           sessionfile: Optional[str] = None,
           download_profile_pic: bool = True, download_posts=True,
-          download_stories: bool = False, download_highlights: bool = False, download_tagged: bool = False,
+          download_stories: bool = False,
+          download_highlights: bool = False,
+          download_tagged: bool = False,
+          download_igtv: bool = False,
           fast_update: bool = False,
           max_count: Optional[int] = None, post_filter_str: Optional[str] = None,
           storyitem_filter_str: Optional[str] = None) -> None:
@@ -77,6 +82,8 @@ def _main(instaloader: Instaloader, targetlist: List[str],
         instaloader.context.log('Only download storyitems with property "{}".'.format(storyitem_filter_str))
     # Login, if desired
     if username is not None:
+        if not re.match(r"^[A-Za-z0-9._]+$", username):
+            instaloader.context.error("Warning: Parameter \"{}\" for --login is not a valid username.".format(username))
         try:
             instaloader.load_session_from_file(username, sessionfile)
         except FileNotFoundError as err:
@@ -98,6 +105,9 @@ def _main(instaloader: Instaloader, targetlist: List[str],
             else:
                 instaloader.interactive_login(username)
         instaloader.context.log("Logged in as %s." % username)
+    # since 4.2.9 login is required for geotags
+    if instaloader.download_geotags and not instaloader.context.is_logged_in:
+        instaloader.context.error("Warning: Use --login to download geotags of posts.")
     # Try block for KeyboardInterrupt (save session on ^C)
     profiles = set()
     anonymous_retry_profiles = set()
@@ -129,18 +139,19 @@ def _main(instaloader: Instaloader, targetlist: List[str],
             # strip '/' characters to be more shell-autocompletion-friendly
             target = target.rstrip('/')
             with instaloader.context.error_catcher(target):
-                if target[0] == '@':
+                if re.match(r"^@[A-Za-z0-9._]+$", target):
                     instaloader.context.log("Retrieving followees of %s..." % target[1:])
                     profile = Profile.from_username(instaloader.context, target[1:])
                     for followee in profile.get_followees():
                         instaloader.save_profile_id(followee)
                         profiles.add(followee)
-                elif target[0] == '#':
+                elif re.match(r"^#\w+$", target):
                     instaloader.download_hashtag(hashtag=target[1:], max_count=max_count, fast_update=fast_update,
-                                                 post_filter=post_filter)
-                elif target[0] == '-':
+                                                 post_filter=post_filter,
+                                                 profile_pic=download_profile_pic, posts=download_posts)
+                elif re.match(r"^-[A-Za-z0-9-_]+$", target):
                     instaloader.download_post(Post.from_shortcode(instaloader.context, target[1:]), target)
-                elif target[0] == "%":
+                elif re.match(r"^%[0-9]+$", target):
                     instaloader.download_location(location=target[1:], max_count=max_count, fast_update=fast_update,
                                                   post_filter=post_filter)
                 elif target == ":feed":
@@ -151,11 +162,12 @@ def _main(instaloader: Instaloader, targetlist: List[str],
                 elif target == ":saved":
                     instaloader.download_saved_posts(fast_update=fast_update, max_count=max_count,
                                                      post_filter=post_filter)
-                else:
+                elif re.match(r"^[A-Za-z0-9._]+$", target):
                     try:
                         profile = instaloader.check_profile_id(target)
                         if instaloader.context.is_logged_in and profile.has_blocked_viewer:
-                            if download_profile_pic or ((download_posts or download_tagged) and not profile.is_private):
+                            if download_profile_pic or ((download_posts or download_tagged or download_igtv)
+                                                        and not profile.is_private):
                                 raise ProfileNotExistsException("{} blocked you; But we download her anonymously."
                                                                 .format(target))
                             else:
@@ -166,7 +178,7 @@ def _main(instaloader: Instaloader, targetlist: List[str],
                         # Not only our profile.has_blocked_viewer condition raises ProfileNotExistsException,
                         # check_profile_id() also does, since access to blocked profile may be responded with 404.
                         if instaloader.context.is_logged_in and (download_profile_pic or download_posts or
-                                                                 download_tagged):
+                                                                 download_tagged or download_igtv):
                             instaloader.context.log(err)
                             instaloader.context.log("Trying again anonymously, helps in case you are just blocked.")
                             with instaloader.anonymous_copy() as anonymous_loader:
@@ -176,20 +188,28 @@ def _main(instaloader: Instaloader, targetlist: List[str],
                                                               .format(target, err))
                         else:
                             raise
+                else:
+                    target_type = {
+                        '#': 'hashtag',
+                        '%': 'location',
+                        '-': 'shortcode',
+                    }.get(target[0], 'username')
+                    raise ProfileNotExistsException('Invalid {} {}'.format(target_type, target))
         if len(profiles) > 1:
             instaloader.context.log("Downloading {} profiles: {}".format(len(profiles),
                                                                          ' '.join([p.username for p in profiles])))
-        if profiles and download_profile_pic and not instaloader.context.is_logged_in:
-            instaloader.context.error("Warning: Use --login to download HD version of profile pictures.")
+        if profiles and (download_profile_pic or download_posts) and not instaloader.context.is_logged_in:
+            instaloader.context.error("Warning: Use --login to download higher-quality versions of pictures.")
         instaloader.download_profiles(profiles,
-                                      download_profile_pic, download_posts, download_tagged, download_highlights,
-                                      download_stories, fast_update, post_filter, storyitem_filter)
+                                      download_profile_pic, download_posts, download_tagged, download_igtv,
+                                      download_highlights, download_stories,
+                                      fast_update, post_filter, storyitem_filter)
         if anonymous_retry_profiles:
             instaloader.context.log("Downloading anonymously: {}"
                                     .format(' '.join([p.username for p in anonymous_retry_profiles])))
             with instaloader.anonymous_copy() as anonymous_loader:
                 anonymous_loader.download_profiles(anonymous_retry_profiles,
-                                                   download_profile_pic, download_posts, download_tagged,
+                                                   download_profile_pic, download_posts, download_tagged, download_igtv,
                                                    fast_update=fast_update, post_filter=post_filter)
     except KeyboardInterrupt:
         print("\nInterrupted by user.", file=sys.stderr)
@@ -223,7 +243,8 @@ def main():
                            help="Download all followees of profile. Requires --login. "
                                 "Consider using :feed rather than @yourself.")
     g_targets.add_argument('_hashtag', nargs='*', metavar='"#hashtag"', help="Download #hashtag.")
-    g_targets.add_argument('_location', nargs='*', metavar='%location_id', help="Download %%location_id.")
+    g_targets.add_argument('_location', nargs='*', metavar='%location_id',
+                           help="Download %%location_id. Requires --login.")
     g_targets.add_argument('_feed', nargs='*', metavar=":feed",
                            help="Download pictures from your feed. Requires --login.")
     g_targets.add_argument('_stories', nargs='*', metavar=":stories",
@@ -258,7 +279,7 @@ def main():
                         help='Download geotags when available. Geotags are stored as a '
                              'text file with the location\'s name and a Google Maps link. '
                              'This requires an additional request to the Instagram '
-                             'server for each picture, which is why it is disabled by default.')
+                             'server for each picture. Requires --login.')
     g_post.add_argument('-C', '--comments', action='store_true',
                         help='Download and update comments for each post. '
                              'This requires an additional request to the Instagram '
@@ -283,6 +304,8 @@ def main():
                         help='Also download highlights of each profile that is downloaded. Requires --login.')
     g_prof.add_argument('--tagged', action='store_true',
                         help='Also download posts where each profile is tagged.')
+    g_prof.add_argument('--igtv', action='store_true',
+                        help='Also download IGTV videos.')
 
     g_cond = parser.add_argument_group("Which Posts to Download")
 
@@ -339,6 +362,8 @@ def main():
                        help='Tries to ensure downloaded images avoid corruption in case of unexpected interruption. '
                        'If the last picture is corrupted, Instaloader will fix the picture the next time it is run. '
                        'Requires the JSON metadata to be saved.')
+    g_how.add_argument('--request-timeout', metavar='N', type=float,
+                       help='seconds to wait before timing out a connection request')
 
     g_misc = parser.add_argument_group('Miscellaneous Options')
     g_misc.add_argument('-q', '--quiet', action='store_true',
@@ -393,6 +418,7 @@ def main():
                              post_metadata_txt_pattern=post_metadata_txt_pattern,
                              storyitem_metadata_txt_pattern=storyitem_metadata_txt_pattern,
                              max_connection_attempts=args.max_connection_attempts,
+                             request_timeout=args.request_timeout,
                              commit_mode=args.commit_mode)
         _main(loader,
               args.profile,
@@ -404,6 +430,7 @@ def main():
               download_stories=download_stories,
               download_highlights=args.highlights,
               download_tagged=args.tagged,
+              download_igtv=args.igtv,
               fast_update=args.fast_update,
               max_count=int(args.count) if args.count is not None else None,
               post_filter_str=args.post_filter,

instaloader/exceptions.py

@@ -9,10 +9,6 @@ class QueryReturnedBadRequestException(InstaloaderException):
     pass
 
 
-class QueryReturnedNotFoundException(InstaloaderException):
-    pass
-
-
 class QueryReturnedForbiddenException(InstaloaderException):
     pass
 
@@ -62,5 +58,9 @@ class PostChangedException(InstaloaderException):
     pass
 
 
+class QueryReturnedNotFoundException(ConnectionException):
+    pass
+
+
 class TooManyRequestsException(ConnectionException):
     pass

instaloader/instaloader.py

@@ -13,18 +13,34 @@ from datetime import datetime, timezone
 from functools import wraps
 from hashlib import md5
 from io import BytesIO
-from typing import Any, Callable, ContextManager, Iterator, List, Optional, Set, Union, cast
+from pathlib import Path
+from typing import Any, Callable, Iterator, List, Optional, Set, Union
 
 import requests
 import urllib3  # type: ignore
 
 from .exceptions import *
 from .instaloadercontext import InstaloaderContext
-from .structures import Highlight, JsonExportable, Post, PostLocation, Profile, Story, StoryItem, save_structure_to_file, load_structure_from_file
+from .structures import (Hashtag, Highlight, JsonExportable, Post, PostLocation, Profile, Story, StoryItem,
+                         save_structure_to_file, load_structure_from_file)
 
 
 def get_default_session_filename(username: str) -> str:
     """Returns default session filename for given username."""
+    sessionfilename = "session-{}".format(username)
+    if platform.system() == "Windows":
+        # on Windows, use %LOCALAPPDATA%\Instaloader\session-USERNAME
+        localappdata = os.getenv("LOCALAPPDATA")
+        if localappdata is not None:
+            return os.path.join(localappdata, "Instaloader", sessionfilename)
+        # legacy fallback - store in temp dir if %LOCALAPPDATA% is not set
+        return os.path.join(tempfile.gettempdir(), ".instaloader-" + getpass.getuser(), sessionfilename)
+    # on Unix, use ~/.config/instaloader/session-USERNAME
+    return os.path.join(os.getenv("XDG_CONFIG_HOME", os.path.expanduser("~/.config")), "instaloader", sessionfilename)
+
+
+def get_legacy_session_filename(username: str) -> str:
+    """Returns legacy (until v4.4.3) default session filename for given username."""
     dirname = tempfile.gettempdir() + "/" + ".instaloader-" + getpass.getuser()
     filename = dirname + "/" + "session-" + username
     return filename.lower()
@@ -45,8 +61,6 @@ def _requires_login(func: Callable) -> Callable:
         if not instaloader.context.is_logged_in:
             raise LoginRequiredException("--login=USERNAME required.")
         return func(instaloader, *args, **kwargs)
-    docstring_text = ":raises LoginRequiredException: If called without being logged in.\n"
-    call.__doc__ = call.__doc__ + docstring_text if call.__doc__ is not None else docstring_text
     return call
 
 
@@ -103,22 +117,20 @@ class _ArbitraryItemFormatter(string.Formatter):
 
 
 class _PostPathFormatter(_ArbitraryItemFormatter):
-    def vformat(self, format_string, args, kwargs):
-        """Override :meth:`string.Formatter.vformat` for character substitution in paths for Windows, see issue #84."""
-        ret = super().vformat(format_string, args, kwargs)
-        if platform.system() == 'Windows':
-            ret = ret.replace(':', '\ua789').replace('<', '\ufe64').replace('>', '\ufe65').replace('\"', '\uff02')
-            ret = ret.replace('\\', '\ufe68').replace('|', '\uff5c').replace('?', '\ufe16').replace('*', '\uff0a')
-        return ret
-
     def get_value(self, key, args, kwargs):
-        """Replaces '/' with similar looking Division Slash and on windows newline with space"""
         ret = super().get_value(key, args, kwargs)
         if not isinstance(ret, str):
             return ret
+        return self.sanitize_path(ret)
+
+    @staticmethod
+    def sanitize_path(ret: str) -> str:
+        """Replaces '/' with similar looking Division Slash and some other illegal filename characters on Windows."""
         ret = ret.replace('/', '\u2215')
         if platform.system() == 'Windows':
-            ret = ret.replace('\n', ' ')
+            ret = ret.replace(':', '\uff1a').replace('<', '\ufe64').replace('>', '\ufe65').replace('\"', '\uff02')
+            ret = ret.replace('\\', '\ufe68').replace('|', '\uff5c').replace('?', '\ufe16').replace('*', '\uff0a')
+            ret = ret.replace('\n', ' ').replace('\r', ' ')
         return ret
 
 
@@ -142,6 +154,7 @@ class Instaloader:
     :param storyitem_metadata_txt_pattern: :option:`--storyitem-metadata-txt`, default is empty (=none)
     :param max_connection_attempts: :option:`--max-connection-attempts`
     :param commit_mode: :option:`--commit-mode`
+    :param request_timeout: :option:`--request-timeout`, set per-request timeout (seconds)
 
     .. attribute:: context
 
@@ -149,7 +162,8 @@ class Instaloader:
     """
 
     def __init__(self,
-                 sleep: bool = True, quiet: bool = False,
+                 sleep: bool = True,
+                 quiet: bool = False,
                  user_agent: Optional[str] = None,
                  dirname_pattern: Optional[str] = None,
                  filename_pattern: Optional[str] = None,
@@ -163,9 +177,10 @@ class Instaloader:
                  post_metadata_txt_pattern: str = None,
                  storyitem_metadata_txt_pattern: str = None,
                  max_connection_attempts: int = 3,
+                 request_timeout: Optional[float] = None,
                  commit_mode: bool = False):
 
-        self.context = InstaloaderContext(sleep, quiet, user_agent, max_connection_attempts)
+        self.context = InstaloaderContext(sleep, quiet, user_agent, max_connection_attempts, request_timeout)
 
         # configuration parameters
         self.dirname_pattern = dirname_pattern or "{target}"
@@ -191,15 +206,24 @@ class Instaloader:
     @contextmanager
     def anonymous_copy(self):
         """Yield an anonymous, otherwise equally-configured copy of an Instaloader instance; Then copy its error log."""
-        new_loader = Instaloader(self.context.sleep, self.context.quiet, self.context.user_agent, self.dirname_pattern,
-                                 self.filename_pattern, download_pictures=self.download_pictures,
-                                 download_videos=self.download_videos,
-                                 download_video_thumbnails=self.download_video_thumbnails,
-                                 download_geotags=self.download_geotags, download_comments=self.download_comments,
-                                 save_metadata=self.save_metadata, compress_json=self.compress_json,
-                                 post_metadata_txt_pattern=self.post_metadata_txt_pattern,
-                                 storyitem_metadata_txt_pattern=self.storyitem_metadata_txt_pattern,
-                                 max_connection_attempts=self.context.max_connection_attempts)
+        new_loader = Instaloader(
+            sleep=self.context.sleep,
+            quiet=self.context.quiet,
+            user_agent=self.context.user_agent,
+            dirname_pattern=self.dirname_pattern,
+            filename_pattern=self.filename_pattern,
+            download_pictures=self.download_pictures,
+            download_videos=self.download_videos,
+            download_video_thumbnails=self.download_video_thumbnails,
+            download_geotags=self.download_geotags,
+            download_comments=self.download_comments,
+            save_metadata=self.save_metadata,
+            compress_json=self.compress_json,
+            post_metadata_txt_pattern=self.post_metadata_txt_pattern,
+            storyitem_metadata_txt_pattern=self.storyitem_metadata_txt_pattern,
+            max_connection_attempts=self.context.max_connection_attempts,
+            request_timeout=self.context.request_timeout,
+            commit_mode=self.commit_mode)
         yield new_loader
         self.context.error_log.extend(new_loader.context.error_log)
         new_loader.context.error_log = []  # avoid double-printing of errors
@@ -244,6 +268,7 @@ class Instaloader:
             filename += '.json.xz'
         else:
             filename += '.json'
+        os.makedirs(os.path.dirname(filename), exist_ok=True)
         save_structure_to_file(structure, filename)
         if isinstance(structure, (Post, StoryItem)):
             # log 'json ' message when saving Post or StoryItem
@@ -254,7 +279,8 @@ class Instaloader:
             return {'id': comment.id,
                     'created_at': int(comment.created_at_utc.replace(tzinfo=timezone.utc).timestamp()),
                     'text': comment.text,
-                    'owner': comment.owner._asdict()}
+                    'owner': comment.owner._asdict(),
+                    'likes_count': comment.likes_count}
 
         def _postcomment_asdict(comment):
             return {**_postcommentanswer_asdict(comment),
@@ -271,11 +297,13 @@ class Instaloader:
             for x, y in zip(comments_list[:-1], comments_list[1:]):
                 if x['id'] != y['id']:
                     unique_comments_list.append(y)
-                elif combine_answers:
-                    combined_answers = unique_comments_list[-1].get('answers') or list()
-                    if 'answers' in y:
-                        combined_answers.extend(y['answers'])
-                    unique_comments_list[-1]['answers'] = get_unique_comments(combined_answers)
+                else:
+                    unique_comments_list[-1]['likes_count'] = y.get('likes_count')
+                    if combine_answers:
+                        combined_answers = unique_comments_list[-1].get('answers') or list()
+                        if 'answers' in y:
+                            combined_answers.extend(y['answers'])
+                        unique_comments_list[-1]['answers'] = get_unique_comments(combined_answers)
             return unique_comments_list
         filename += '_comments.json'
         try:
@@ -342,45 +370,69 @@ class Instaloader:
         self.context.log('geo', end=' ', flush=True)
 
     @_retry_on_connection_error
-    def download_profilepic(self, profile: Profile, _attempt: int = 1) -> None:
-        """Downloads and saves profile pic."""
+    def download_title_pic(self, url: str, target: Union[str, Path], name_suffix: str, owner_profile: Optional[Profile],
+                           _attempt: int = 1) -> None:
+        """Downloads and saves a picture that does not have an association with a Post or StoryItem, such as a
+        Profile picture or a Highlight cover picture. Modification time is taken from the HTTP response headers.
+
+        .. versionadded:: 4.3"""
 
         def _epoch_to_string(epoch: datetime) -> str:
             return epoch.strftime('%Y-%m-%d_%H-%M-%S_UTC')
 
-        profile_pic_response = self.context.get_raw(profile.profile_pic_url)
+        http_response = self.context.get_raw(url)
         date_object = None  # type: Optional[datetime]
-        if 'Last-Modified' in profile_pic_response.headers:
-            date_object = datetime.strptime(profile_pic_response.headers["Last-Modified"], '%a, %d %b %Y %H:%M:%S GMT')
-            profile_pic_bytes = None
-            profile_pic_identifier = _epoch_to_string(date_object)
+        if 'Last-Modified' in http_response.headers:
+            date_object = datetime.strptime(http_response.headers["Last-Modified"], '%a, %d %b %Y %H:%M:%S GMT')
+            pic_bytes = None
+            pic_identifier = _epoch_to_string(date_object)
         else:
-            profile_pic_bytes = profile_pic_response.content
-            profile_pic_identifier = md5(profile_pic_bytes).hexdigest()[:16]
-        profile_pic_extension = 'jpg'
+            pic_bytes = http_response.content
+            pic_identifier = md5(pic_bytes).hexdigest()[:16]
+        pic_extension = 'jpg'
         if ((format_string_contains_key(self.dirname_pattern, 'profile') or
              format_string_contains_key(self.dirname_pattern, 'target'))):
-            filename = '{0}/{1}_profile_pic.{2}'.format(self.dirname_pattern.format(profile=profile.username.lower(),
-                                                                                    target=profile.username.lower()),
-                                                        profile_pic_identifier, profile_pic_extension)
+            profile_str = owner_profile.username.lower() if owner_profile is not None else target
+            filename = os.path.join(self.dirname_pattern.format(profile=profile_str,
+                                                                target=target),
+                                    '{0}_{1}.{2}'.format(pic_identifier, name_suffix, pic_extension))
         else:
-            filename = '{0}/{1}_{2}_profile_pic.{3}'.format(self.dirname_pattern.format(), profile.username.lower(),
-                                                            profile_pic_identifier, profile_pic_extension)
-        content_length = profile_pic_response.headers.get('Content-Length', None)
+            filename = os.path.join(self.dirname_pattern.format(),
+                                    '{0}_{1}_{2}.{3}'.format(target, pic_identifier, name_suffix, pic_extension))
+        content_length = http_response.headers.get('Content-Length', None)
         if os.path.isfile(filename) and (not self.context.is_logged_in or
-                                         content_length is not None and os.path.getsize(filename) >= int(content_length)):
+                                         (content_length is not None and
+                                          os.path.getsize(filename) >= int(content_length))):
             self.context.log(filename + ' already exists')
             return None
-        self.context.write_raw(profile_pic_bytes if profile_pic_bytes else profile_pic_response, filename)
+        os.makedirs(os.path.dirname(filename), exist_ok=True)
+        self.context.write_raw(pic_bytes if pic_bytes else http_response, filename)
         if date_object:
             os.utime(filename, (datetime.now().timestamp(), date_object.timestamp()))
         self.context.log('')  # log output of _get_and_write_raw() does not produce \n
 
+    def download_profilepic(self, profile: Profile) -> None:
+        """Downloads and saves profile pic."""
+        self.download_title_pic(profile.profile_pic_url, profile.username.lower(), 'profile_pic', profile)
+
+    def download_highlight_cover(self, highlight: Highlight, target: Union[str, Path]) -> None:
+        """Downloads and saves Highlight cover picture.
+
+        .. versionadded:: 4.3"""
+        self.download_title_pic(highlight.cover_url, target, 'cover', highlight.owner_profile)
+
+    def download_hashtag_profilepic(self, hashtag: Hashtag) -> None:
+        """Downloads and saves the profile picture of a Hashtag.
+
+        .. versionadded:: 4.4"""
+        self.download_title_pic(hashtag.profile_pic_url, '#' + hashtag.name, 'profile_pic', None)
+
     @_requires_login
     def save_session_to_file(self, filename: Optional[str] = None) -> None:
         """Saves internally stored :class:`requests.Session` object.
 
         :param filename: Filename, or None to use default filename.
+        :raises LoginRequiredException: If called without being logged in.
         """
         if filename is None:
             assert self.context.username is not None
@@ -403,6 +455,8 @@ class Instaloader:
         """
         if filename is None:
             filename = get_default_session_filename(username)
+            if not os.path.exists(filename):
+                filename = get_legacy_session_filename(username)
         with open(filename, 'rb') as sessionfile:
             self.context.load_session_from_file(username, sessionfile)
             self.context.log("Loaded session from %s." % filename)
@@ -417,7 +471,8 @@ class Instaloader:
         :raises InvalidArgumentException: If the provided username does not exist.
         :raises BadCredentialsException: If the provided password is wrong.
         :raises ConnectionException: If connection to Instagram failed.
-        :raises TwoFactorAuthRequiredException: First step of 2FA login done, now call :meth:`Instaloader.two_factor_login`."""
+        :raises TwoFactorAuthRequiredException: First step of 2FA login done, now call
+           :meth:`Instaloader.two_factor_login`."""
         self.context.login(user, passwd)
 
     def two_factor_login(self, two_factor_code) -> None:
@@ -430,13 +485,13 @@ class Instaloader:
         .. versionadded:: 4.2"""
         self.context.two_factor_login(two_factor_code)
 
-    def format_filename(self, item: Union[Post, StoryItem], target: Optional[str] = None):
+    def format_filename(self, item: Union[Post, StoryItem], target: Optional[Union[str, Path]] = None):
         """Format filename of a :class:`Post` or :class:`StoryItem` according to ``filename-pattern`` parameter.
 
         .. versionadded:: 4.1"""
         return _PostPathFormatter(item).format(self.filename_pattern, target=target)
 
-    def download_post(self, post: Post, target: str) -> bool:
+    def download_post(self, post: Post, target: Union[str, Path]) -> bool:
         """
         Download everything associated with one instagram post node, i.e. picture, caption and video.
 
@@ -446,7 +501,7 @@ class Instaloader:
         """
 
         dirname = _PostPathFormatter(post).format(self.dirname_pattern, target=target)
-        filename = dirname + '/' + self.format_filename(post, target=target)
+        filename = os.path.join(dirname, self.format_filename(post, target=target))
         os.makedirs(os.path.dirname(filename), exist_ok=True)
 
         # Download the image(s) / video thumbnail and videos within sidecars if desired
@@ -504,6 +559,7 @@ class Instaloader:
         To use this, one needs to be logged in
 
         :param userids: List of user IDs to be processed in terms of downloading their stories, or None.
+        :raises LoginRequiredException: If called without being logged in.
         """
 
         if not userids:
@@ -540,6 +596,7 @@ class Instaloader:
         :param filename_target: Replacement for {target} in dirname_pattern and filename_pattern
                or None if profile name should be used instead
         :param storyitem_filter: function(storyitem), which returns True if given StoryItem should be downloaded
+        :raises LoginRequiredException: If called without being logged in.
         """
 
         if not userids:
@@ -563,7 +620,7 @@ class Instaloader:
                     if fast_update and not downloaded:
                         break
 
-    def download_storyitem(self, item: StoryItem, target: str) -> bool:
+    def download_storyitem(self, item: StoryItem, target: Union[str, Path]) -> bool:
         """Download one user story.
 
         :param item: Story item, as in story['items'] for story in :meth:`get_stories`
@@ -573,7 +630,7 @@ class Instaloader:
 
         date_local = item.date_local
         dirname = _PostPathFormatter(item).format(self.dirname_pattern, target=target)
-        filename = dirname + '/' + self.format_filename(item, target=target)
+        filename = os.path.join(dirname, self.format_filename(item, target=target))
         os.makedirs(os.path.dirname(filename), exist_ok=True)
         downloaded = False
         if not item.is_video or self.download_video_thumbnails is True:
@@ -599,6 +656,7 @@ class Instaloader:
         .. versionadded:: 4.1
 
         :param user: ID or Profile of the user whose highlights should get fetched.
+        :raises LoginRequiredException: If called without being logged in.
         """
 
         userid = user if isinstance(user, int) else user.userid
@@ -623,15 +681,24 @@ class Instaloader:
 
         .. versionadded:: 4.1
 
+        .. versionchanged:: 4.3
+           Also downloads and saves the Highlight's cover pictures.
+
         :param user: ID or Profile of the user whose highlights should get downloaded.
         :param fast_update: If true, abort when first already-downloaded picture is encountered
         :param filename_target: Replacement for {target} in dirname_pattern and filename_pattern
                or None if profile name and the highlights' titles should be used instead
         :param storyitem_filter: function(storyitem), which returns True if given StoryItem should be downloaded
+        :raises LoginRequiredException: If called without being logged in.
         """
         for user_highlight in self.get_highlights(user):
             name = user_highlight.owner_username
+            highlight_target = (filename_target
+                                if filename_target
+                                else (Path(_PostPathFormatter.sanitize_path(name)) /
+                                      _PostPathFormatter.sanitize_path(user_highlight.title)))  # type: Union[str, Path]
             self.context.log("Retrieving highlights \"{}\" from profile {}".format(user_highlight.title, name))
+            self.download_highlight_cover(user_highlight, highlight_target)
             totalcount = user_highlight.itemcount
             count = 1
             for item in user_highlight.get_items():
@@ -640,26 +707,88 @@ class Instaloader:
                     continue
                 self.context.log("[%3i/%3i] " % (count, totalcount), end="", flush=True)
                 count += 1
-                with self.context.error_catcher('Download highlights \"{}\" from user {}'.format(user_highlight.title, name)):
-                    downloaded = self.download_storyitem(item, filename_target
-                                                         if filename_target
-                                                         else '{}/{}'.format(name, user_highlight.title))
+                with self.context.error_catcher('Download highlights \"{}\" from user {}'.format(user_highlight.title,
+                                                                                                 name)):
+                    downloaded = self.download_storyitem(item, highlight_target)
                     if fast_update and not downloaded:
                         break
 
+    def posts_download_loop(self,
+                            posts: Iterator[Post],
+                            target: Union[str, Path],
+                            fast_update: bool = False,
+                            post_filter: Optional[Callable[[Post], bool]] = None,
+                            max_count: Optional[int] = None,
+                            total_count: Optional[int] = None) -> None:
+        """
+        Download the Posts returned by given Post Iterator.
+
+        ..versionadded:: 4.4
+
+        :param posts: Post Iterator to loop through.
+        :param target: Target name
+        :param fast_update: :option:`--fast-update`
+        :param post_filter: :option:`--post-filter`
+        :param max_count: Maximum count of Posts to download (:option:`--count`)
+        :param total_count: Total number of posts returned by given iterator
+        """
+        for number, post in enumerate(posts):
+            if max_count is not None and number >= max_count:
+                break
+            if total_count is not None:
+                self.context.log("[{0:{w}d}/{1:{w}d}] ".format(number + 1, total_count,
+                                                               w=len(str(total_count))),
+                                 end="", flush=True)
+            else:
+                if max_count is not None:
+                    self.context.log("[{0:{w}d}/{1:{w}d}] ".format(number + 1, max_count,
+                                                                   w=len(str(max_count))),
+                                     end="", flush=True)
+                else:
+                    self.context.log("[{:3d}] ".format(number + 1), end="", flush=True)
+            if post_filter is not None:
+                try:
+                    if not post_filter(post):
+                        self.context.log("{} skipped".format(post))
+                        continue
+                except (InstaloaderException, KeyError, TypeError) as err:
+                    self.context.error("{} skipped. Filter evaluation failed: {}".format(post, err))
+                    continue
+            with self.context.error_catcher("Download {} of {}".format(post, target)):
+                # The PostChangedException gets raised if the Post's id/shortcode changed while obtaining
+                # additional metadata. This is most likely the case if a HTTP redirect takes place while
+                # resolving the shortcode URL.
+                # The `post_changed` variable keeps the fast-update functionality alive: A Post which is
+                # obained after a redirect has probably already been downloaded as a previous Post of the
+                # same Profile.
+                # Observed in issue #225: https://github.com/instaloader/instaloader/issues/225
+                post_changed = False
+                while True:
+                    try:
+                        downloaded = self.download_post(post, target=target)
+                        break
+                    except PostChangedException:
+                        post_changed = True
+                        continue
+                if fast_update and not downloaded and not post_changed:
+                    break
+
     @_requires_login
     def get_feed_posts(self) -> Iterator[Post]:
         """Get Posts of the user's feed.
 
         :return: Iterator over Posts of the user's feed.
+        :raises LoginRequiredException: If called without being logged in.
         """
 
         data = self.context.graphql_query("d6f4427fbe92d846298cf93df0b937d3", {})["data"]
 
         while True:
             feed = data["user"]["edge_web_feed_timeline"]
-            yield from (Post(self.context, edge["node"]) for edge in feed["edges"]
-                        if not edge["node"]["__typename"] == "GraphSuggestedUserFeedUnit")
+            for edge in feed["edges"]:
+                node = edge["node"]
+                if node.get("__typename") in Post.supported_graphql_types() and node.get("shortcode") is not None:
+                    yield Post(self.context, node)
             if not feed["page_info"]["has_next_page"]:
                 break
             data = self.context.graphql_query("d6f4427fbe92d846298cf93df0b937d3",
@@ -685,22 +814,10 @@ class Instaloader:
         :param max_count: Maximum count of pictures to download
         :param fast_update: If true, abort when first already-downloaded picture is encountered
         :param post_filter: function(post), which returns True if given picture should be downloaded
+        :raises LoginRequiredException: If called without being logged in.
         """
         self.context.log("Retrieving pictures from your feed...")
-        count = 1
-        for post in self.get_feed_posts():
-            if max_count is not None and count > max_count:
-                break
-            name = post.owner_username
-            if post_filter is not None and not post_filter(post):
-                self.context.log("<pic by %s skipped>" % name, flush=True)
-                continue
-            self.context.log("[%3i] %s " % (count, name), end="", flush=True)
-            count += 1
-            with self.context.error_catcher('Download feed'):
-                downloaded = self.download_post(post, target=':feed')
-                if fast_update and not downloaded:
-                    break
+        self.posts_download_loop(self.get_feed_posts(), ":feed", fast_update, post_filter, max_count=max_count)
 
     @_requires_login
     def download_saved_posts(self, max_count: int = None, fast_update: bool = False,
@@ -710,29 +827,24 @@ class Instaloader:
         :param max_count: Maximum count of pictures to download
         :param fast_update: If true, abort when first already-downloaded picture is encountered
         :param post_filter: function(post), which returns True if given picture should be downloaded
+        :raises LoginRequiredException: If called without being logged in.
         """
         self.context.log("Retrieving saved posts...")
-        count = 1
-        assert self.context.username is not None
-        for post in Profile.from_username(self.context, self.context.username).get_saved_posts():
-            if max_count is not None and count > max_count:
-                break
-            if post_filter is not None and not post_filter(post):
-                self.context.log("<{} skipped>".format(post), flush=True)
-                continue
-            self.context.log("[{:>3}] ".format(count), end=str(), flush=True)
-            count += 1
-            with self.context.error_catcher('Download saved posts'):
-                downloaded = self.download_post(post, target=':saved')
-                if fast_update and not downloaded:
-                    break
+        assert self.context.username is not None  # safe due to @_requires_login; required by typechecker
+        self.posts_download_loop(Profile.from_username(self.context, self.context.username).get_saved_posts(), ":saved",
+                                 fast_update, post_filter, max_count=max_count)
 
+    @_requires_login
     def get_location_posts(self, location: str) -> Iterator[Post]:
         """Get Posts which are listed by Instagram for a given Location.
 
         :return:  Iterator over Posts of a location's posts
+        :raises LoginRequiredException: If called without being logged in.
 
         .. versionadded:: 4.2
+
+        .. versionchanged:: 4.2.9
+           Require being logged in (as required by Instagram)
         """
         has_next_page = True
         end_cursor = None
@@ -747,6 +859,7 @@ class Instaloader:
             has_next_page = location_data['page_info']['has_next_page']
             end_cursor = location_data['page_info']['end_cursor']
 
+    @_requires_login
     def download_location(self, location: str,
                           max_count: Optional[int] = None,
                           post_filter: Optional[Callable[[Post], bool]] = None,
@@ -762,56 +875,44 @@ class Instaloader:
         :param max_count: Maximum count of pictures to download
         :param post_filter: function(post), which returns True if given picture should be downloaded
         :param fast_update: If true, abort when first already-downloaded picture is encountered
+        :raises LoginRequiredException: If called without being logged in.
 
         .. versionadded:: 4.2
+
+        .. versionchanged:: 4.2.9
+           Require being logged in (as required by Instagram)
         """
         self.context.log("Retrieving pictures for location {}...".format(location))
-        count = 1
-        for post in self.get_location_posts(location):
-            if max_count is not None and count > max_count:
-                break
-            self.context.log('[{0:3d}] %{1} '.format(count, location), end='', flush=True)
-            if post_filter is not None and not post_filter(post):
-                self.context.log('<skipped>')
-                continue
-            count += 1
-            with self.context.error_catcher('Download location {}'.format(location)):
-                downloaded = self.download_post(post, target='%' + location)
-                if fast_update and not downloaded:
-                    break
+        self.posts_download_loop(self.get_location_posts(location), "%" + location, fast_update, post_filter,
+                                 max_count=max_count)
 
     @_requires_login
     def get_explore_posts(self) -> Iterator[Post]:
         """Get Posts which are worthy of exploring suggested by Instagram.
 
         :return: Iterator over Posts of the user's suggested posts.
+        :raises LoginRequiredException: If called without being logged in.
         """
         data = self.context.get_json('explore/', {})
         yield from (Post(self.context, node)
                     for node in self.context.graphql_node_list("df0dcc250c2b18d9fd27c5581ef33c7c",
                                                                {}, 'https://www.instagram.com/explore/',
                                                                lambda d: d['data']['user']['edge_web_discover_media'],
-                                                               data['rhx_gis']))
+                                                               data.get('rhx_gis')))
 
     def get_hashtag_posts(self, hashtag: str) -> Iterator[Post]:
-        """Get Posts associated with a #hashtag."""
-        has_next_page = True
-        end_cursor = None
-        while has_next_page:
-            if end_cursor:
-                params = {'__a': 1, 'max_id': end_cursor}
-            else:
-                params = {'__a': 1}
-            hashtag_data = self.context.get_json('explore/tags/{0}/'.format(hashtag),
-                                                 params)['graphql']['hashtag']['edge_hashtag_to_media']
-            yield from (Post(self.context, edge['node']) for edge in hashtag_data['edges'])
-            has_next_page = hashtag_data['page_info']['has_next_page']
-            end_cursor = hashtag_data['page_info']['end_cursor']
+        """Get Posts associated with a #hashtag.
 
-    def download_hashtag(self, hashtag: str,
+        .. deprecated:: 4.4
+           Use :meth:`Hashtag.get_posts`."""
+        return Hashtag.from_name(self.context, hashtag).get_posts()
+
+    def download_hashtag(self, hashtag: Union[Hashtag, str],
                          max_count: Optional[int] = None,
                          post_filter: Optional[Callable[[Post], bool]] = None,
-                         fast_update: bool = False) -> None:
+                         fast_update: bool = False,
+                         profile_pic: bool = True,
+                         posts: bool = True) -> None:
         """Download pictures of one hashtag.
 
         To download the last 30 pictures with hashtag #cat, do::
@@ -819,26 +920,34 @@ class Instaloader:
             loader = Instaloader()
             loader.download_hashtag('cat', max_count=30)
 
-        :param hashtag: Hashtag to download, without leading '#'
+        :param hashtag: Hashtag to download, as instance of :class:`Hashtag`, or string without leading '#'
         :param max_count: Maximum count of pictures to download
         :param post_filter: function(post), which returns True if given picture should be downloaded
         :param fast_update: If true, abort when first already-downloaded picture is encountered
+        :param profile_pic: not :option:`--no-profile-pic`.
+        :param posts: not :option:`--no-posts`.
+
+        .. versionchanged:: 4.4
+           Add parameters `profile_pic` and `posts`.
         """
-        hashtag = hashtag.lower()
-        self.context.log("Retrieving pictures with hashtag {}...".format(hashtag))
-        count = 1
-        for post in self.get_hashtag_posts(hashtag):
-            if max_count is not None and count > max_count:
-                break
-            self.context.log('[{0:3d}] #{1} '.format(count, hashtag), end='', flush=True)
-            if post_filter is not None and not post_filter(post):
-                self.context.log('<skipped>')
-                continue
-            count += 1
-            with self.context.error_catcher('Download hashtag #{}'.format(hashtag)):
-                downloaded = self.download_post(post, target='#' + hashtag)
-                if fast_update and not downloaded:
-                    break
+        if isinstance(hashtag, str):
+            with self.context.error_catcher("Get hashtag #{}".format(hashtag)):
+                hashtag = Hashtag.from_name(self.context, hashtag)
+        if not isinstance(hashtag, Hashtag):
+            return
+        target = "#" + hashtag.name
+        if profile_pic:
+            with self.context.error_catcher("Download profile picture of {}".format(target)):
+                self.download_hashtag_profilepic(hashtag)
+        if posts:
+            self.context.log("Retrieving pictures with hashtag #{}...".format(hashtag.name))
+            self.posts_download_loop(hashtag.get_all_posts(), target, fast_update, post_filter,
+                                     max_count=max_count)
+        if self.save_metadata:
+            json_filename = '{0}/{1}'.format(self.dirname_pattern.format(profile=target,
+                                                                         target=target),
+                                             target)
+            self.save_metadata_json(json_filename, hashtag)
 
     def download_tagged(self, profile: Profile, fast_update: bool = False,
                         target: Optional[str] = None,
@@ -846,27 +955,31 @@ class Instaloader:
         """Download all posts where a profile is tagged.
 
         .. versionadded:: 4.1"""
-        if target is None:
-            target = profile.username + '/:tagged'
         self.context.log("Retrieving tagged posts for profile {}.".format(profile.username))
-        count = 1
-        for post in profile.get_tagged_posts():
-            self.context.log("[%3i/???] " % (count), end="", flush=True)
-            count += 1
-            if post_filter is not None and not post_filter(post):
-                self.context.log('<{} skipped>'.format(post))
-            with self.context.error_catcher('Download tagged {}'.format(profile.username)):
-                downloaded = self.download_post(post, target)
-                if fast_update and not downloaded:
-                    break
+        self.posts_download_loop(profile.get_tagged_posts(),
+                                 target if target
+                                 else (Path(_PostPathFormatter.sanitize_path(profile.username)) /
+                                       _PostPathFormatter.sanitize_path(':tagged')),
+                                 fast_update, post_filter)
+
+    def download_igtv(self, profile: Profile, fast_update: bool = False,
+                      post_filter: Optional[Callable[[Post], bool]] = None) -> None:
+        """Download IGTV videos of a profile.
+
+        .. versionadded:: 4.3"""
+        self.context.log("Retrieving IGTV videos for profile {}.".format(profile.username))
+        self.posts_download_loop(profile.get_igtv_posts(), profile.username, fast_update, post_filter,
+                                 total_count=profile.igtvcount)
 
     def _get_id_filename(self, profile_name: str) -> str:
         if ((format_string_contains_key(self.dirname_pattern, 'profile') or
              format_string_contains_key(self.dirname_pattern, 'target'))):
-            return '{0}/id'.format(self.dirname_pattern.format(profile=profile_name.lower(),
-                                                               target=profile_name.lower()))
+            return os.path.join(self.dirname_pattern.format(profile=profile_name.lower(),
+                                                            target=profile_name.lower()),
+                                'id')
         else:
-            return '{0}/{1}_id'.format(self.dirname_pattern.format(), profile_name.lower())
+            return os.path.join(self.dirname_pattern.format(),
+                                '{0}_id'.format(profile_name.lower()))
 
     def save_profile_id(self, profile: Profile):
         """
@@ -889,16 +1002,18 @@ class Instaloader:
         :return: Instance of current profile
         """
         profile = None
-        with suppress(ProfileNotExistsException):
+        profile_name_not_exists_err = None
+        try:
             profile = Profile.from_username(self.context, profile_name)
-        profile_exists = profile is not None
+        except ProfileNotExistsException as err:
+            profile_name_not_exists_err = err
         id_filename = self._get_id_filename(profile_name)
         try:
             with open(id_filename, 'rb') as id_file:
                 profile_id = int(id_file.read())
-            if (not profile_exists) or \
+            if (profile is None) or \
                     (profile_id != profile.userid):
-                if profile_exists:
+                if profile is not None:
                     self.context.log("Profile {0} does not match the stored unique ID {1}.".format(profile_name,
                                                                                                    profile_id))
                 else:
@@ -917,17 +1032,23 @@ class Instaloader:
                     os.rename('{0}/{1}_id'.format(self.dirname_pattern.format(), profile_name.lower()),
                               '{0}/{1}_id'.format(self.dirname_pattern.format(), newname.lower()))
                 return profile_from_id
+            # profile exists and profile id matches saved id
             return profile
         except (FileNotFoundError, ValueError):
             pass
-        if profile_exists:
+        if profile is not None:
             self.save_profile_id(profile)
             return profile
+        if profile_name_not_exists_err:
+            raise profile_name_not_exists_err
         raise ProfileNotExistsException("Profile {0} does not exist.".format(profile_name))
 
     def download_profiles(self, profiles: Set[Profile],
                           profile_pic: bool = True, posts: bool = True,
-                          tagged: bool = False, highlights: bool = False, stories: bool = False,
+                          tagged: bool = False,
+                          igtv: bool = False,
+                          highlights: bool = False,
+                          stories: bool = False,
                           fast_update: bool = False,
                           post_filter: Optional[Callable[[Post], bool]] = None,
                           storyitem_filter: Optional[Callable[[Post], bool]] = None,
@@ -938,6 +1059,7 @@ class Instaloader:
         :param profile_pic: not :option:`--no-profile-pic`.
         :param posts: not :option:`--no-posts`.
         :param tagged: :option:`--tagged`.
+        :param igtv: :option:`--igtv`.
         :param highlights: :option:`--highlights`.
         :param stories: :option:`--stories`.
         :param fast_update: :option:`--fast-update`.
@@ -947,17 +1069,21 @@ class Instaloader:
            Whether :exc:`LoginRequiredException` and :exc:`PrivateProfileNotFollowedException` should be raised or
            catched and printed with :meth:`InstaloaderContext.error_catcher`.
 
-        .. versionadded:: 4.1"""
+        .. versionadded:: 4.1
+
+        .. versionchanged:: 4.3
+           Add `igtv` parameter.
+        """
 
         @contextmanager
         def _error_raiser(_str):
             yield
 
-        error_handler = cast(Callable[[Optional[str]], ContextManager[None]],
-                             _error_raiser if raise_errors else self.context.error_catcher)
+        # error_handler type is Callable[[Optional[str]], ContextManager[None]] (not supported with Python 3.5)
+        error_handler = _error_raiser if raise_errors else self.context.error_catcher
 
         for profile in profiles:
-            with error_handler(profile.username):
+            with error_handler(profile.username):  # type: ignore
                 profile_name = profile.username
 
                 # Download profile picture
@@ -967,13 +1093,13 @@ class Instaloader:
 
                 # Save metadata as JSON if desired.
                 if self.save_metadata:
-                    json_filename = '{0}/{1}_{2}'.format(self.dirname_pattern.format(profile=profile_name,
-                                                                                     target=profile_name),
-                                                         profile_name, profile.userid)
+                    json_filename = os.path.join(self.dirname_pattern.format(profile=profile_name,
+                                                                             target=profile_name),
+                                                 '{0}_{1}'.format(profile_name, profile.userid))
                     self.save_metadata_json(json_filename, profile)
 
                 # Catch some errors
-                if profile.is_private and (tagged or highlights or posts):
+                if profile.is_private and (tagged or igtv or highlights or posts):
                     if not self.context.is_logged_in:
                         raise LoginRequiredException("--login=USERNAME required.")
                     if not profile.followed_by_viewer and self.context.username != profile.username:
@@ -984,6 +1110,11 @@ class Instaloader:
                     with self.context.error_catcher('Download tagged of {}'.format(profile_name)):
                         self.download_tagged(profile, fast_update=fast_update, post_filter=post_filter)
 
+                # Download IGTV, if requested
+                if igtv:
+                    with self.context.error_catcher('Download IGTV of {}'.format(profile_name)):
+                        self.download_igtv(profile, fast_update=fast_update, post_filter=post_filter)
+
                 # Download highlights, if requested
                 if highlights:
                     with self.context.error_catcher('Download highlights of {}'.format(profile_name)):
@@ -992,32 +1123,8 @@ class Instaloader:
                 # Iterate over pictures and download them
                 if posts:
                     self.context.log("Retrieving posts from profile {}.".format(profile_name))
-                    totalcount = profile.mediacount
-                    count = 1
-                    for post in profile.get_posts():
-                        self.context.log("[%3i/%3i] " % (count, totalcount), end="", flush=True)
-                        count += 1
-                        if post_filter is not None and not post_filter(post):
-                            self.context.log('<skipped>')
-                            continue
-                        with self.context.error_catcher("Download {} of {}".format(post, profile_name)):
-                            # The PostChangedException gets raised if the Post's id/shortcode changed while obtaining
-                            # additional metadata. This is most likely the case if a HTTP redirect takes place while
-                            # resolving the shortcode URL.
-                            # The `post_changed` variable keeps the fast-update functionality alive: A Post which is
-                            # obained after a redirect has probably already been downloaded as a previous Post of the
-                            # same Profile.
-                            # Observed in issue #225: https://github.com/instaloader/instaloader/issues/225
-                            post_changed = False
-                            while True:
-                                try:
-                                    downloaded = self.download_post(post, target=profile_name)
-                                    break
-                                except PostChangedException:
-                                    post_changed = True
-                                    continue
-                            if fast_update and not downloaded and not post_changed:
-                                break
+                    self.posts_download_loop(profile.get_posts(), profile_name, fast_update, post_filter,
+                                             total_count=profile.mediacount)
 
         if stories and profiles:
             with self.context.error_catcher("Download stories"):
@@ -1096,18 +1203,8 @@ class Instaloader:
 
         # Iterate over pictures and download them
         self.context.log("Retrieving posts from profile {}.".format(profile_name))
-        totalcount = profile.mediacount
-        count = 1
-        for post in profile.get_posts():
-            self.context.log("[%3i/%3i] " % (count, totalcount), end="", flush=True)
-            count += 1
-            if post_filter is not None and not post_filter(post):
-                self.context.log('<skipped>')
-                continue
-            with self.context.error_catcher('Download profile {}'.format(profile_name)):
-                downloaded = self.download_post(post, target=profile_name)
-                if fast_update and not downloaded:
-                    break
+        self.posts_download_loop(profile.get_posts(), profile_name, fast_update, post_filter,
+                                 total_count=profile.mediacount)
 
     def check_if_committed(self, filename: str) -> bool:
         """Checks to see if the current post has been committed.

instaloader/instaloadercontext.py

@@ -10,6 +10,7 @@ import time
 import urllib.parse
 from contextlib import contextmanager
 from datetime import datetime, timedelta
+from functools import partial
 from typing import Any, Callable, Dict, Iterator, List, Optional, Union
 
 import requests
@@ -18,11 +19,15 @@ import requests.utils
 from .exceptions import *
 
 
-def copy_session(session: requests.Session) -> requests.Session:
+def copy_session(session: requests.Session, request_timeout: Optional[float] = None) -> requests.Session:
     """Duplicates a requests.Session."""
     new = requests.Session()
     new.cookies = requests.utils.cookiejar_from_dict(requests.utils.dict_from_cookiejar(session.cookies))
     new.headers = session.headers.copy() # type: ignore
+    if request_timeout is not None:
+        # Override default timeout behavior.
+        # Need to silence mypy bug for this. See: https://github.com/python/mypy/issues/2427
+        new.request = partial(new.request, timeout=request_timeout) # type: ignore
     return new
 
 
@@ -47,9 +52,10 @@ class InstaloaderContext:
     """
 
     def __init__(self, sleep: bool = True, quiet: bool = False, user_agent: Optional[str] = None,
-                 max_connection_attempts: int = 3):
+                 max_connection_attempts: int = 3, request_timeout: Optional[float] = None):
 
         self.user_agent = user_agent if user_agent is not None else default_user_agent()
+        self.request_timeout = request_timeout
         self._session = self.get_anonymous_session()
         self.username = None
         self.sleep = sleep
@@ -107,7 +113,7 @@ class InstaloaderContext:
     def close(self):
         """Print error log and close session"""
         if self.error_log and not self.quiet:
-            print("\nErrors occured:", file=sys.stderr)
+            print("\nErrors or warnings occurred:", file=sys.stderr)
             for err in self.error_log:
                 print(err, file=sys.stderr)
         self._session.close()
@@ -155,6 +161,10 @@ class InstaloaderContext:
                                 'ig_vw': '1920', 'csrftoken': '',
                                 's_network': '', 'ds_user_id': ''})
         session.headers.update(self._default_http_header(empty_session_only=True))
+        if self.request_timeout is not None:
+            # Override default timeout behavior.
+            # Need to silence mypy bug for this. See: https://github.com/python/mypy/issues/2427
+            session.request = partial(session.request, timeout=self.request_timeout) # type: ignore
         return session
 
     def save_session_to_file(self, sessionfile):
@@ -167,6 +177,10 @@ class InstaloaderContext:
         session.cookies = requests.utils.cookiejar_from_dict(pickle.load(sessionfile))
         session.headers.update(self._default_http_header())
         session.headers.update({'X-CSRFToken': session.cookies.get_dict()['csrftoken']})
+        if self.request_timeout is not None:
+            # Override default timeout behavior.
+            # Need to silence mypy bug for this. See: https://github.com/python/mypy/issues/2427
+            session.request = partial(session.request, timeout=self.request_timeout) # type: ignore
         self._session = session
         self.username = username
 
@@ -181,7 +195,9 @@ class InstaloaderContext:
         :raises InvalidArgumentException: If the provided username does not exist.
         :raises BadCredentialsException: If the provided password is wrong.
         :raises ConnectionException: If connection to Instagram failed.
-        :raises TwoFactorAuthRequiredException: First step of 2FA login done, now call :meth:`Instaloader.two_factor_login`."""
+        :raises TwoFactorAuthRequiredException: First step of 2FA login done, now call
+           :meth:`Instaloader.two_factor_login`."""
+        # pylint:disable=import-outside-toplevel
         import http.client
         # pylint:disable=protected-access
         http.client._MAXHEADERS = 200
@@ -190,19 +206,26 @@ class InstaloaderContext:
                                 'ig_vw': '1920', 'ig_cb': '1', 'csrftoken': '',
                                 's_network': '', 'ds_user_id': ''})
         session.headers.update(self._default_http_header())
+        if self.request_timeout is not None:
+            # Override default timeout behavior.
+            # Need to silence mypy bug for this. See: https://github.com/python/mypy/issues/2427
+            session.request = partial(session.request, timeout=self.request_timeout) # type: ignore
         session.get('https://www.instagram.com/web/__mid/')
         csrf_token = session.cookies.get_dict()['csrftoken']
         session.headers.update({'X-CSRFToken': csrf_token})
         # Not using self.get_json() here, because we need to access csrftoken cookie
         self.do_sleep()
+        # Workaround credits to pgrimaud.
+        # See: https://github.com/pgrimaud/instagram-user-feed/commit/96ad4cf54d1ad331b337f325c73e664999a6d066
+        enc_password = '#PWD_INSTAGRAM_BROWSER:0:{}:{}'.format(int(datetime.now().timestamp()), passwd)
         login = session.post('https://www.instagram.com/accounts/login/ajax/',
-                             data={'password': passwd, 'username': user}, allow_redirects=True)
+                             data={'enc_password': enc_password, 'username': user}, allow_redirects=True)
         try:
             resp_json = login.json()
         except json.decoder.JSONDecodeError:
             raise ConnectionException("Login error: JSON decode fail, {} - {}.".format(login.status_code, login.reason))
         if resp_json.get('two_factor_required'):
-            two_factor_session = copy_session(session)
+            two_factor_session = copy_session(session, self.request_timeout)
             two_factor_session.headers.update({'X-CSRFToken': csrf_token})
             two_factor_session.cookies.update({'csrftoken': csrf_token})
             self.two_factor_auth_pending = (two_factor_session,
@@ -211,7 +234,7 @@ class InstaloaderContext:
             raise TwoFactorAuthRequiredException("Login error: two-factor authentication required.")
         if resp_json.get('checkpoint_url'):
             raise ConnectionException("Login: Checkpoint required. Point your browser to "
-                                      "https://www.instagram.com{}, "
+                                      "https://www.instagram.com{} - "
                                       "follow the instructions, then retry.".format(resp_json.get('checkpoint_url')))
         if resp_json['status'] != 'ok':
             if 'message' in resp_json:
@@ -219,6 +242,12 @@ class InstaloaderContext:
                                                                                                resp_json['message']))
             else:
                 raise ConnectionException("Login error: \"{}\" status.".format(resp_json['status']))
+        if 'authenticated' not in resp_json:
+            # Issue #472
+            if 'message' in resp_json:
+                raise ConnectionException("Login error: Unexpected response, \"{}\".".format(resp_json['message']))
+            else:
+                raise ConnectionException("Login error: Unexpected response, this might indicate a blocked IP.")
         if not resp_json['authenticated']:
             if resp_json['user']:
                 # '{"authenticated": false, "user": true, "status": "ok"}'
@@ -278,9 +307,14 @@ class InstaloaderContext:
     def _graphql_request_count_per_sliding_window(self, query_hash: str) -> int:
         """Return how many GraphQL requests can be done within the sliding window."""
         if self.is_logged_in:
-            max_reqs = {'1cb6ec562846122743b61e492c85999f': 20, '33ba35852cb50da46f5b5e889df7d159': 20}
+            max_reqs = {'1cb6ec562846122743b61e492c85999f': 20,
+                        '33ba35852cb50da46f5b5e889df7d159': 20,
+                        'iphone': 100,
+                        'other': 100}
         else:
-            max_reqs = {'1cb6ec562846122743b61e492c85999f': 200, '33ba35852cb50da46f5b5e889df7d159': 200}
+            max_reqs = {'1cb6ec562846122743b61e492c85999f': 200,
+                        '33ba35852cb50da46f5b5e889df7d159': 200,
+                        'other': 200}
         return max_reqs.get(query_hash) or min(max_reqs.values())
 
     def _graphql_query_waittime(self, query_hash: str, current_time: float, untracked_queries: bool = False) -> float:
@@ -318,7 +352,7 @@ class InstaloaderContext:
             else:
                 self._graphql_query_timestamps[query_hash].append(time.monotonic())
         else:
-            text_for_429 = ("HTTP error code 429 was returned because too many queries occured in the last time. "
+            text_for_429 = ("HTTP error code 429 was returned because too many queries occurred in the last time. "
                             "Please do not use Instagram in your browser or run multiple instances of Instaloader "
                             "in parallel.")
             print(textwrap.fill(text_for_429), file=sys.stderr)
@@ -345,15 +379,24 @@ class InstaloaderContext:
         :raises ConnectionException: When query repeatedly failed.
         """
         is_graphql_query = 'query_hash' in params and 'graphql/query' in path
+        is_iphone_query = host == 'i.instagram.com'
+        is_other_query = not is_graphql_query and host == "www.instagram.com"
         sess = session if session else self._session
         try:
             self.do_sleep()
             if is_graphql_query:
                 self._ratecontrol_graphql_query(params['query_hash'])
+            if is_iphone_query:
+                self._ratecontrol_graphql_query('iphone')
+            if is_other_query:
+                self._ratecontrol_graphql_query('other')
             resp = sess.get('https://{0}/{1}'.format(host, path), params=params, allow_redirects=False)
             while resp.is_redirect:
                 redirect_url = resp.headers['location']
                 self.log('\nHTTP redirect from https://{0}/{1} to {2}'.format(host, path, redirect_url))
+                if redirect_url.startswith('https://www.instagram.com/accounts/login'):
+                    # alternate rate limit exceeded behavior
+                    raise TooManyRequestsException("429 Too Many Requests: redirected to login")
                 if redirect_url.startswith('https://{}/'.format(host)):
                     resp = sess.get(redirect_url if redirect_url.endswith('/') else redirect_url + '/',
                                     params=params, allow_redirects=False)
@@ -371,8 +414,19 @@ class InstaloaderContext:
             if is_html_query:
                 match = re.search(r'window\._sharedData = (.*);</script>', resp.text)
                 if match is None:
-                    raise ConnectionException("Could not find \"window._sharedData\" in html response.")
-                return json.loads(match.group(1))
+                    raise QueryReturnedNotFoundException("Could not find \"window._sharedData\" in html response.")
+                resp_json = json.loads(match.group(1))
+                entry_data = resp_json.get('entry_data')
+                post_or_profile_page = list(entry_data.values())[0] if entry_data is not None else None
+                if post_or_profile_page is None:
+                    raise ConnectionException("\"window._sharedData\" does not contain required keys.")
+                # If GraphQL data is missing in `window._sharedData`, search for it in `__additionalDataLoaded`.
+                if 'graphql' not in post_or_profile_page[0]:
+                    match = re.search(r'window\.__additionalDataLoaded\([^{]+{"graphql":({.*})}\);</script>',
+                                      resp.text)
+                    if match is not None:
+                        post_or_profile_page[0]['graphql'] = json.loads(match.group(1))
+                return resp_json
             else:
                 resp_json = resp.json()
             if 'status' in resp_json and resp_json['status'] != "ok":
@@ -385,11 +439,18 @@ class InstaloaderContext:
         except (ConnectionException, json.decoder.JSONDecodeError, requests.exceptions.RequestException) as err:
             error_string = "JSON Query to {}: {}".format(path, err)
             if _attempt == self.max_connection_attempts:
-                raise ConnectionException(error_string) from err
+                if isinstance(err, QueryReturnedNotFoundException):
+                    raise QueryReturnedNotFoundException(error_string) from err
+                else:
+                    raise ConnectionException(error_string) from err
             self.error(error_string + " [retrying; skip with ^C]", repeat_at_end=False)
             try:
-                if isinstance(err, TooManyRequestsException):
+                if is_graphql_query and isinstance(err, TooManyRequestsException):
                     self._ratecontrol_graphql_query(params['query_hash'], untracked_queries=True)
+                if is_iphone_query and isinstance(err, TooManyRequestsException):
+                    self._ratecontrol_graphql_query('iphone', untracked_queries=True)
+                if is_other_query and isinstance(err, TooManyRequestsException):
+                    self._ratecontrol_graphql_query('other', untracked_queries=True)
                 return self.get_json(path=path, params=params, host=host, session=sess, _attempt=_attempt + 1)
             except KeyboardInterrupt:
                 self.error("[skipped by user]", repeat_at_end=False)
@@ -406,7 +467,7 @@ class InstaloaderContext:
         :param rhx_gis: 'rhx_gis' variable as somewhere returned by Instagram, needed to 'sign' request
         :return: The server's response dictionary.
         """
-        with copy_session(self._session) as tmpsession:
+        with copy_session(self._session, self.request_timeout) as tmpsession:
             tmpsession.headers.update(self._default_http_header(empty_session_only=True))
             del tmpsession.headers['Connection']
             del tmpsession.headers['Content-Length']
@@ -474,9 +535,9 @@ class InstaloaderContext:
         :raises ConnectionException: When query repeatedly failed.
 
         .. versionadded:: 4.2.1"""
-        with copy_session(self._session) as tempsession:
-            tempsession.headers['User-Agent'] = 'Instagram 10.3.2 (iPhone7,2; iPhone OS 9_3_3; en_US; en-US; ' \
-                                                'scale=2.00; 750x1334) AppleWebKit/420+'
+        with copy_session(self._session, self.request_timeout) as tempsession:
+            tempsession.headers['User-Agent'] = 'Instagram 123.1.0.26.115 (iPhone12,1; iOS 13_3; en_US; en-US; ' \
+                                                'scale=2.00; 1656x3584; 190542906)'
             for header in ['Host', 'Origin', 'X-Instagram-AJAX', 'X-Requested-With']:
                 tempsession.headers.pop(header, None)
             return self.get_json(path, params, 'i.instagram.com', tempsession)
@@ -529,6 +590,6 @@ class InstaloaderContext:
             # At the moment, rhx_gis seems to be required for anonymous requests only. By returning None when logged
             # in, we can save the root_rhx_gis lookup query.
             return None
-        if not self._root_rhx_gis:
-            self._root_rhx_gis = self.get_json('', {})['rhx_gis']
-        return self._root_rhx_gis
+        if self._root_rhx_gis is None:
+            self._root_rhx_gis = self.get_json('', {}).get('rhx_gis', '')
+        return self._root_rhx_gis or None

instaloader/structures.py

@@ -17,11 +17,12 @@ PostSidecarNode.is_video.__doc__ = "Whether this node is a video."
 PostSidecarNode.display_url.__doc__ = "URL of image or video thumbnail."
 PostSidecarNode.video_url.__doc__ = "URL of video or None."
 
-PostCommentAnswer = namedtuple('PostCommentAnswer', ['id', 'created_at_utc', 'text', 'owner'])
+PostCommentAnswer = namedtuple('PostCommentAnswer', ['id', 'created_at_utc', 'text', 'owner', 'likes_count'])
 PostCommentAnswer.id.__doc__ = "ID number of comment."
 PostCommentAnswer.created_at_utc.__doc__ = ":class:`~datetime.datetime` when comment was created (UTC)."
 PostCommentAnswer.text.__doc__ = "Comment text."
 PostCommentAnswer.owner.__doc__ = "Owner :class:`Profile` of the comment."
+PostCommentAnswer.likes_count.__doc__ = "Number of likes on comment."
 
 PostComment = namedtuple('PostComment', (*PostCommentAnswer._fields, 'answers')) # type: ignore
 for field in PostCommentAnswer._fields:
@@ -68,8 +69,11 @@ class Post:
         self._node = node
         self._owner_profile = owner_profile
         self._full_metadata_dict = None  # type: Optional[Dict[str, Any]]
-        self._rhx_gis_str = None         # type: Optional[str]
         self._location = None            # type: Optional[PostLocation]
+        self._iphone_struct_ = None
+        if 'iphone_struct' in node:
+            # if loaded from JSON with load_structure_from_file()
+            self._iphone_struct_ = node['iphone_struct']
 
     @classmethod
     def from_shortcode(cls, context: InstaloaderContext, shortcode: str):
@@ -97,6 +101,11 @@ class Post:
             raise InvalidArgumentException("Wrong mediaid {0}, unable to convert to shortcode".format(str(mediaid)))
         return b64encode(mediaid.to_bytes(9, 'big'), b'-_').decode().replace('A', ' ').lstrip().replace(' ', 'A')
 
+    @staticmethod
+    def supported_graphql_types() -> List[str]:
+        """The values of __typename fields that the :class:`Post` class can handle."""
+        return ["GraphImage", "GraphVideo", "GraphSidecar"]
+
     def _asdict(self):
         node = self._node
         if self._full_metadata_dict:
@@ -105,6 +114,8 @@ class Post:
             node['owner'] = self.owner_profile._asdict()
         if self._location:
             node['location'] = self._location._asdict()
+        if self._iphone_struct_:
+            node['iphone_struct'] = self._iphone_struct_
         return node
 
     @property
@@ -130,9 +141,17 @@ class Post:
 
     def _obtain_metadata(self):
         if not self._full_metadata_dict:
-            pic_json = self._context.get_json("p/{0}/".format(self.shortcode), params={})
-            self._full_metadata_dict = pic_json['entry_data']['PostPage'][0]['graphql']['shortcode_media']
-            self._rhx_gis_str = pic_json['rhx_gis']
+            pic_json = self._context.graphql_query(
+                '2b0673e0dc4580674a88d426fe00ea90',
+                {'shortcode': self.shortcode}
+            )
+            self._full_metadata_dict = pic_json['data']['shortcode_media']
+            if self._full_metadata_dict is None:
+                # issue #449
+                self._context.error("Fetching Post metadata failed (issue #449). "
+                                    "The following data has been returned:\n"
+                                    + json.dumps(pic_json['entry_data'], indent=2))
+                raise BadResponseException("Fetching Post metadata failed.")
             if self.shortcode != self._full_metadata_dict['shortcode']:
                 self._node.update(self._full_metadata_dict)
                 raise PostChangedException
@@ -144,10 +163,13 @@ class Post:
         return self._full_metadata_dict
 
     @property
-    def _rhx_gis(self) -> str:
-        self._obtain_metadata()
-        assert self._rhx_gis_str is not None
-        return self._rhx_gis_str
+    def _iphone_struct(self) -> Dict[str, Any]:
+        if not self._context.is_logged_in:
+            raise LoginRequiredException("--login required to access iPhone media info endpoint.")
+        if not self._iphone_struct_:
+            data = self._context.get_iphone_json(path='api/v1/media/{}/info/'.format(self.mediaid), params={})
+            self._iphone_struct_ = data['items'][0]
+        return self._iphone_struct_
 
     def _field(self, *keys) -> Any:
         """Lookups given fields in _node, and if not found in _full_metadata. Raises KeyError if not found anywhere."""
@@ -190,12 +212,16 @@ class Post:
     @property
     def date_local(self) -> datetime:
         """Timestamp when the post was created (local time zone)."""
-        return datetime.fromtimestamp(self._node["date"] if "date" in self._node else self._node["taken_at_timestamp"])
+        return datetime.fromtimestamp(self._node["date"]
+                                      if "date" in self._node
+                                      else self._node["taken_at_timestamp"])
 
     @property
     def date_utc(self) -> datetime:
         """Timestamp when the post was created (UTC)."""
-        return datetime.utcfromtimestamp(self._node["date"] if "date" in self._node else self._node["taken_at_timestamp"])
+        return datetime.utcfromtimestamp(self._node["date"]
+                                         if "date" in self._node
+                                         else self._node["taken_at_timestamp"])
 
     @property
     def date(self) -> datetime:
@@ -210,6 +236,13 @@ class Post:
     @property
     def url(self) -> str:
         """URL of the picture / video thumbnail of the post"""
+        if self.typename == "GraphImage" and self._context.is_logged_in:
+            try:
+                orig_url = self._iphone_struct['image_versions2']['candidates'][0]['url']
+                url = re.sub(r'&se=\d+(&?)', r'\1', orig_url)
+                return url
+            except (InstaloaderException, KeyError, IndexError) as err:
+                self._context.error('{} Unable to fetch high quality image version of {}.'.format(err, self))
         return self._node["display_url"] if "display_url" in self._node else self._node["display_src"]
 
     @property
@@ -220,10 +253,22 @@ class Post:
     def get_sidecar_nodes(self) -> Iterator[PostSidecarNode]:
         """Sidecar nodes of a Post with typename==GraphSidecar."""
         if self.typename == 'GraphSidecar':
-            for edge in self._field('edge_sidecar_to_children', 'edges'):
+            edges = self._field('edge_sidecar_to_children', 'edges')
+            if any(edge['node']['is_video'] for edge in edges):
+                # video_url is only present in full metadata, issue #558.
+                edges = self._full_metadata['edge_sidecar_to_children']['edges']
+            for idx, edge in enumerate(edges):
                 node = edge['node']
                 is_video = node['is_video']
-                yield PostSidecarNode(is_video=is_video, display_url=node['display_url'],
+                display_url = node['display_url']
+                if not is_video and self._context.is_logged_in:
+                    try:
+                        carousel_media = self._iphone_struct['carousel_media']
+                        orig_url = carousel_media[idx]['image_versions2']['candidates'][0]['url']
+                        display_url = re.sub(r'&se=\d+(&?)', r'\1', orig_url)
+                    except (InstaloaderException, KeyError, IndexError) as err:
+                        self._context.error('{} Unable to fetch high quality image version of {}.'.format(err, self))
+                yield PostSidecarNode(is_video=is_video, display_url=display_url,
                                       video_url=node['video_url'] if is_video else None)
 
     @property
@@ -257,7 +302,9 @@ class Post:
 
     @property
     def pcaption(self) -> str:
-        """Printable caption, useful as a format specifier for --filename-pattern."""
+        """Printable caption, useful as a format specifier for --filename-pattern.
+
+        .. versionadded:: 4.2.6"""
         def _elliptify(caption):
             pcaption = ' '.join([s.replace('/', '\u2215') for s in caption.splitlines() if s]).strip()
             return (pcaption[:30] + u"\u2026") if len(pcaption) > 31 else pcaption
@@ -286,14 +333,18 @@ class Post:
 
     @property
     def video_view_count(self) -> Optional[int]:
-        """View count of the video, or None."""
+        """View count of the video, or None.
+
+        .. versionadded:: 4.2.6"""
         if self.is_video:
             return self._field('video_view_count')
         return None
 
     @property
     def video_duration(self) -> Optional[float]:
-        """Duration of the video in seconds, or None."""
+        """Duration of the video in seconds, or None.
+
+        .. versionadded:: 4.2.6"""
         if self.is_video:
             return self._field('video_duration')
         return None
@@ -315,6 +366,11 @@ class Post:
     @property
     def comments(self) -> int:
         """Comment count including answers"""
+        # If the count is already present in `self._node`, do not use `self._field` which could trigger fetching the
+        # full metadata dict.
+        comments = self._node.get('edge_media_to_comment')
+        if comments and 'count' in comments:
+            return comments['count']
         try:
             return self._field('edge_media_to_parent_comment', 'count')
         except KeyError:
@@ -331,7 +387,8 @@ class Post:
             return PostCommentAnswer(id=int(node['id']),
                                      created_at_utc=datetime.utcfromtimestamp(node['created_at']),
                                      text=node['text'],
-                                     owner=Profile(self._context, node['owner']))
+                                     owner=Profile(self._context, node['owner']),
+                                     likes_count=node.get('edge_liked_by', {}).get('count', 0))
 
         def _postcommentanswers(node):
             if 'edge_threaded_comments' not in node:
@@ -357,14 +414,9 @@ class Post:
         if self.comments == 0:
             # Avoid doing additional requests if there are no comments
             return
-        try:
-            comment_edges = self._field('edge_media_to_parent_comment', 'edges')
-            answers_count = sum([edge['node']['edge_threaded_comments']['count'] for edge in comment_edges])
-            threaded_comments_available = True
-        except KeyError:
-            comment_edges = self._field('edge_media_to_comment', 'edges')
-            answers_count = 0
-            threaded_comments_available = False
+
+        comment_edges = self._field('edge_media_to_comment', 'edges')
+        answers_count = sum([edge['node'].get('edge_threaded_comments', {}).get('count', 0) for edge in comment_edges])
 
         if self.comments == len(comment_edges) + answers_count:
             # If the Post's metadata already contains all parent comments, don't do GraphQL requests to obtain them
@@ -372,14 +424,10 @@ class Post:
             return
         yield from (_postcomment(node) for node in
                     self._context.graphql_node_list(
-                        "97b41c52301f77ce508f55e66d17620e" if threaded_comments_available
-                        else "f0986789a5c5d17c2400faebf16efd0d",
+                        "97b41c52301f77ce508f55e66d17620e",
                         {'shortcode': self.shortcode},
                         'https://www.instagram.com/p/' + self.shortcode + '/',
-                        lambda d:
-                        d['data']['shortcode_media'][
-                            'edge_media_to_parent_comment' if threaded_comments_available else 'edge_media_to_comment'],
-                        self._rhx_gis))
+                        lambda d: d['data']['shortcode_media']['edge_media_to_parent_comment']))
 
     def get_likes(self) -> Iterator['Profile']:
         """Iterate over all likes of the post. A :class:`Profile` instance of each likee is yielded."""
@@ -394,15 +442,45 @@ class Post:
         yield from (Profile(self._context, node) for node in
                     self._context.graphql_node_list("1cb6ec562846122743b61e492c85999f", {'shortcode': self.shortcode},
                                                     'https://www.instagram.com/p/' + self.shortcode + '/',
-                                                    lambda d: d['data']['shortcode_media']['edge_liked_by'],
-                                                    self._rhx_gis))
+                                                    lambda d: d['data']['shortcode_media']['edge_liked_by']))
+
+    @property
+    def is_sponsored(self) -> bool:
+        """
+        Whether Post is a sponsored post, equivalent to non-empty :meth:`Post.sponsor_users`.
+
+        .. versionadded:: 4.4
+        """
+        try:
+            sponsor_edges = self._field('edge_media_to_sponsor_user', 'edges')
+        except KeyError:
+            return False
+        return bool(sponsor_edges)
+
+    @property
+    def sponsor_users(self) -> List['Profile']:
+        """
+        The Post's sponsors.
+
+        .. versionadded:: 4.4
+        """
+        return ([] if not self.is_sponsored else
+                [Profile(self._context, edge['node']['sponsor']) for edge in
+                 self._field('edge_media_to_sponsor_user', 'edges')])
 
     @property
     def location(self) -> Optional[PostLocation]:
-        """If the Post has a location, returns PostLocation namedtuple with fields 'id', 'lat' and 'lng' and 'name'."""
+        """
+        If the Post has a location, returns PostLocation namedtuple with fields 'id', 'lat' and 'lng' and 'name'.
+
+        .. versionchanged:: 4.2.9
+           Require being logged in (as required by Instagram), return None if not logged-in.
+        """
         loc = self._field("location")
         if self._location or not loc:
             return self._location
+        if not self._context.is_logged_in:
+            return None
         location_id = int(loc['id'])
         if any(k not in loc for k in ('name', 'slug', 'has_public_page', 'lat', 'lng')):
             loc = self._context.get_json("explore/locations/{0}/".format(location_id),
@@ -444,7 +522,7 @@ class Profile:
         self._context = context
         self._has_public_story = None  # type: Optional[bool]
         self._node = node
-        self._rhx_gis = None
+        self._has_full_metadata = False
         self._iphone_struct_ = None
         if 'iphone_struct' in node:
             # if loaded from JSON with load_structure_from_file()
@@ -494,21 +572,29 @@ class Profile:
 
     def _asdict(self):
         json_node = self._node.copy()
-        # remove posts
+        # remove posts to avoid "Circular reference detected" exception
         json_node.pop('edge_media_collections', None)
         json_node.pop('edge_owner_to_timeline_media', None)
         json_node.pop('edge_saved_media', None)
+        json_node.pop('edge_felix_video_timeline', None)
         if self._iphone_struct_:
             json_node['iphone_struct'] = self._iphone_struct_
         return json_node
 
     def _obtain_metadata(self):
         try:
-            if not self._rhx_gis:
-                metadata = self._context.get_json('{}/'.format(self.username), params={})
+            if not self._has_full_metadata:
+                metadata = self._context.get_json('{}/feed/'.format(self.username), params={})
                 self._node = metadata['entry_data']['ProfilePage'][0]['graphql']['user']
-                self._rhx_gis = metadata['rhx_gis']
+                self._has_full_metadata = True
         except (QueryReturnedNotFoundException, KeyError) as err:
+            top_search_results = TopSearchResults(self._context, self.username)
+            similar_profiles = [profile.username for profile in top_search_results.get_profiles()]
+            if similar_profiles:
+                raise ProfileNotExistsException('Profile {} does not exist.\nThe most similar profile{}: {}.'
+                                                .format(self.username,
+                                                        's are' if len(similar_profiles) > 1 else ' is',
+                                                        ', '.join(similar_profiles[0:5]))) from err
             raise ProfileNotExistsException('Profile {} does not exist.'.format(self.username)) from err
 
     def _metadata(self, *keys) -> Any:
@@ -566,6 +652,10 @@ class Profile:
     def mediacount(self) -> int:
         return self._metadata('edge_owner_to_timeline_media', 'count')
 
+    @property
+    def igtvcount(self) -> int:
+        return self._metadata('edge_felix_video_timeline', 'count')
+
     @property
     def followers(self) -> int:
         return self._metadata('edge_followed_by', 'count')
@@ -578,6 +668,16 @@ class Profile:
     def external_url(self) -> Optional[str]:
         return self._metadata('external_url')
 
+    @property
+    def is_business_account(self) -> bool:
+        """.. versionadded:: 4.4"""
+        return self._metadata('is_business_account')
+
+    @property
+    def business_category_name(self) -> str:
+        """.. versionadded:: 4.4"""
+        return self._metadata('business_category_name')
+
     @property
     def biography(self) -> str:
         return self._metadata('biography')
@@ -619,8 +719,7 @@ class Profile:
                                                         'include_reel': False, 'include_suggested_users': False,
                                                         'include_logged_out_extras': True,
                                                         'include_highlight_reels': False},
-                                                       'https://www.instagram.com/{}/'.format(self.username),
-                                                       self._rhx_gis)
+                                                       'https://www.instagram.com/{}/'.format(self.username))
             self._has_public_story = data['data']['user']['has_public_story']
         assert self._has_public_story is not None
         return self._has_public_story
@@ -679,8 +778,7 @@ class Profile:
                                                     {'id': self.userid},
                                                     'https://www.instagram.com/{0}/'.format(self.username),
                                                     lambda d: d['data']['user']['edge_owner_to_timeline_media'],
-                                                    self._rhx_gis,
-                                                    self._metadata('edge_owner_to_timeline_media')))
+                                                    first_data=self._metadata('edge_owner_to_timeline_media')))
 
     def get_saved_posts(self) -> Iterator[Post]:
         """Get Posts that are marked as saved by the user."""
@@ -694,8 +792,7 @@ class Profile:
                                                     {'id': self.userid},
                                                     'https://www.instagram.com/{0}/'.format(self.username),
                                                     lambda d: d['data']['user']['edge_saved_media'],
-                                                    self._rhx_gis,
-                                                    self._metadata('edge_saved_media')))
+                                                    first_data=self._metadata('edge_saved_media')))
 
     def get_tagged_posts(self) -> Iterator[Post]:
         """Retrieve all posts where a profile is tagged.
@@ -706,8 +803,19 @@ class Profile:
                     self._context.graphql_node_list("e31a871f7301132ceaab56507a66bbb7",
                                                     {'id': self.userid},
                                                     'https://www.instagram.com/{0}/'.format(self.username),
-                                                    lambda d: d['data']['user']['edge_user_to_photos_of_you'],
-                                                    self._rhx_gis))
+                                                    lambda d: d['data']['user']['edge_user_to_photos_of_you']))
+
+    def get_igtv_posts(self) -> Iterator[Post]:
+        """Retrieve all IGTV posts.
+
+        .. versionadded:: 4.3"""
+        self._obtain_metadata()
+        yield from (Post(self._context, node, self) for node in
+                    self._context.graphql_node_list('bc78b344a68ed16dd5d7f264681c4c76',
+                                                    {'id': self.userid},
+                                                    'https://www.instagram.com/{0}/channel/'.format(self.username),
+                                                    lambda d: d['data']['user']['edge_felix_video_timeline'],
+                                                    first_data=self._metadata('edge_felix_video_timeline')))
 
     def get_followers(self) -> Iterator['Profile']:
         """
@@ -721,8 +829,7 @@ class Profile:
                     self._context.graphql_node_list("37479f2b8209594dde7facb0d904896a",
                                                     {'id': str(self.userid)},
                                                     'https://www.instagram.com/' + self.username + '/',
-                                                    lambda d: d['data']['user']['edge_followed_by'],
-                                                    self._rhx_gis))
+                                                    lambda d: d['data']['user']['edge_followed_by']))
 
     def get_followees(self) -> Iterator['Profile']:
         """
@@ -736,8 +843,23 @@ class Profile:
                     self._context.graphql_node_list("58712303d941c6855d4e888c5f0cd22f",
                                                     {'id': str(self.userid)},
                                                     'https://www.instagram.com/' + self.username + '/',
-                                                    lambda d: d['data']['user']['edge_follow'],
-                                                    self._rhx_gis))
+                                                    lambda d: d['data']['user']['edge_follow']))
+
+    def get_similar_accounts(self) -> Iterator['Profile']:
+        """
+        Retrieve list of suggested / similar accounts for this profile.
+        To use this, one needs to be logged in.
+
+        .. versionadded:: 4.4
+        """
+        if not self._context.is_logged_in:
+            raise LoginRequiredException("--login required to get a profile's similar accounts.")
+        self._obtain_metadata()
+        yield from (Profile(self._context, edge["node"]) for edge in
+                    self._context.graphql_query("ad99dd9d3646cc3c0dda65debcd266a7",
+                                                {"user_id": str(self.userid), "include_chaining": True},
+                                                "https://www.instagram.com/{0}/"
+                                                .format(self.username))["data"]["user"]["edge_chaining"]["edges"])
 
 
 class StoryItem:
@@ -1035,17 +1157,258 @@ class Highlight(Story):
         yield from (StoryItem(self._context, item, self.owner_profile) for item in self._items)
 
 
-JsonExportable = Union[Post, Profile, StoryItem]
+class Hashtag:
+    """
+    An Hashtag.
+
+    Analogous to :class:`Profile`, get an instance with::
+
+       L = Instaloader()
+       hashtag = Hashtag.from_name(L.context, HASHTAG)
+
+    To then download the Hashtag's Posts, do::
+
+       for post in hashtag.get_posts():
+          L.download_post(post, target="#"+hashtag.name)
+
+    Also, this class implements == and is hashable.
+    """
+    def __init__(self, context: InstaloaderContext, node: Dict[str, Any]):
+        assert "name" in node
+        self._context = context
+        self._node = node
+        self._has_full_metadata = False
+
+    @classmethod
+    def from_name(cls, context: InstaloaderContext, name: str):
+        """
+        Create a Hashtag instance from a given hashtag name, without preceeding '#'. Raises an Exception if there is no
+        hashtag with the given name.
+
+        :param context: :attr:`Instaloader.context`
+        :param name: Hashtag, without preceeding '#'
+        :raises: :class:`QueryReturnedNotFoundException`
+        """
+        # pylint:disable=protected-access
+        hashtag = cls(context, {'name': name.lower()})
+        hashtag._obtain_metadata()
+        return hashtag
+
+    @property
+    def name(self):
+        """Hashtag name lowercased, without preceeding '#'"""
+        return self._node["name"].lower()
+
+    def _query(self, params):
+        return self._context.get_json("explore/tags/{0}/".format(self.name),
+                                      params)["graphql"]["hashtag"]
+
+    def _obtain_metadata(self):
+        if not self._has_full_metadata:
+            self._node = self._query({"__a": 1})
+            self._has_full_metadata = True
+
+    def _asdict(self):
+        json_node = self._node.copy()
+        # remove posts
+        json_node.pop("edge_hashtag_to_top_posts", None)
+        json_node.pop("edge_hashtag_to_media", None)
+        return json_node
+
+    def __repr__(self):
+        return "<Hashtag #{}>".format(self.name)
+
+    def __eq__(self, other: object) -> bool:
+        if isinstance(other, Hashtag):
+            return self.name == other.name
+        return NotImplemented
+
+    def __hash__(self) -> int:
+        return hash(self.name)
+
+    def _metadata(self, *keys) -> Any:
+        try:
+            d = self._node
+            for key in keys:
+                d = d[key]
+            return d
+        except KeyError:
+            self._obtain_metadata()
+            d = self._node
+            for key in keys:
+                d = d[key]
+            return d
+
+    @property
+    def hashtagid(self) -> int:
+        return int(self._metadata("id"))
+
+    @property
+    def profile_pic_url(self) -> str:
+        return self._metadata("profile_pic_url")
+
+    @property
+    def description(self) -> str:
+        return self._metadata("description")
+
+    @property
+    def allow_following(self) -> bool:
+        return self._metadata("allow_following")
+
+    @property
+    def is_following(self) -> bool:
+        return self._metadata("is_following")
+
+    @property
+    def is_top_media_only(self) -> bool:
+        return self._metadata("is_top_media_only")
+
+    def get_related_tags(self) -> Iterator["Hashtag"]:
+        """Yields similar hashtags."""
+        yield from (Hashtag(self._context, edge["node"])
+                    for edge in self._metadata("edge_hashtag_to_related_tags", "edges"))
+
+    def get_top_posts(self) -> Iterator[Post]:
+        """Yields the top posts of the hashtag."""
+        yield from (Post(self._context, edge["node"])
+                    for edge in self._metadata("edge_hashtag_to_top_posts", "edges"))
+
+    @property
+    def mediacount(self) -> int:
+        """
+        The count of all media associated with this hashtag.
+
+        The number of posts with a certain hashtag may differ from the number of posts that can actually be accessed, as
+        the hashtag count might include private posts
+        """
+        return self._metadata("edge_hashtag_to_media", "count")
+
+    def get_posts(self) -> Iterator[Post]:
+        """Yields the posts associated with this hashtag."""
+        self._metadata("edge_hashtag_to_media", "edges")
+        self._metadata("edge_hashtag_to_media", "page_info")
+        conn = self._metadata("edge_hashtag_to_media")
+        yield from (Post(self._context, edge["node"]) for edge in conn["edges"])
+        while conn["page_info"]["has_next_page"]:
+            data = self._query({'__a': 1, 'max_id': conn["page_info"]["end_cursor"]})
+            conn = data["edge_hashtag_to_media"]
+            yield from (Post(self._context, edge["node"]) for edge in conn["edges"])
+
+    def get_all_posts(self) -> Iterator[Post]:
+        """Yields all posts, i.e. all most recent posts and the top posts, in chronological order."""
+        sorted_top_posts = iter(sorted(self.get_top_posts(), key=lambda p: p.date_utc, reverse=True))
+        other_posts = self.get_posts()
+        next_top = next(sorted_top_posts, None)
+        next_other = next(other_posts, None)
+        while next_top is not None or next_other is not None:
+            if next_other is None:
+                yield from sorted_top_posts
+                break
+            if next_top is None:
+                yield from other_posts
+                break
+            if next_top == next_other:
+                yield next_top
+                next_top = next(sorted_top_posts, None)
+                next_other = next(other_posts, None)
+                continue
+            if next_top.date_utc > next_other.date_utc:
+                yield next_top
+                next_top = next(sorted_top_posts, None)
+            else:
+                yield next_other
+                next_other = next(other_posts, None)
+
+
+class TopSearchResults:
+    """
+    An invocation of this class triggers a search on Instagram for the provided search string.
+
+    Provides methods to access the search results as profiles (:class:`Profile`), locations (:class:`PostLocation`) and
+    hashtags.
+
+    :param context: :attr:`Instaloader.context` used to send the query for the search.
+    :param searchstring: String to search for with Instagram's "top search".
+    """
+
+    def __init__(self, context: InstaloaderContext, searchstring: str):
+        self._context = context
+        self._searchstring = searchstring
+        # The `__a` param is only needed to prevent `get_json()` from searching for 'window._sharedData'.
+        self._node = context.get_json('web/search/topsearch/',
+                                      params={'context': 'blended',
+                                              'query': searchstring,
+                                              'include_reel': False,
+                                              '__a': 1})
+
+    def get_profiles(self) -> Iterator[Profile]:
+        """
+        Provides the :class:`Profile` instances from the search result.
+        """
+        for user in self._node.get('users', []):
+            user_node = user['user']
+            if 'pk' in user_node:
+                user_node['id'] = user_node['pk']
+            yield Profile(self._context, user_node)
+
+    def get_prefixed_usernames(self) -> Iterator[str]:
+        """
+        Provides all profile names from the search result that start with the search string.
+        """
+        for user in self._node.get('users', []):
+            username = user.get('user', {}).get('username', '')
+            if username.startswith(self._searchstring):
+                yield username
+
+    def get_locations(self) -> Iterator[PostLocation]:
+        """
+        Provides instances of :class:`PostLocation` from the search result.
+        """
+        for location in self._node.get('places', []):
+            place = location.get('place', {})
+            slug = place.get('slug')
+            loc = place.get('location', {})
+            yield PostLocation(int(loc['pk']), loc['name'], slug, None, loc['lat'], loc['lng'])
+
+    def get_hashtag_strings(self) -> Iterator[str]:
+        """
+        Provides the hashtags from the search result as strings.
+        """
+        for hashtag in self._node.get('hashtags', []):
+            name = hashtag.get('hashtag', {}).get('name')
+            if name:
+                yield name
+
+    def get_hashtags(self) -> Iterator[Hashtag]:
+        """
+        Provides the hashtags from the search result.
+
+        .. versionadded:: 4.4
+        """
+        for hashtag in self._node.get('hashtags', []):
+            node = hashtag.get('hashtag', {})
+            if 'name' in node:
+                yield Hashtag(self._context, node)
+
+    @property
+    def searchstring(self) -> str:
+        """
+        The string that was searched for on Instagram to produce this :class:`TopSearchResults` instance.
+        """
+        return self._searchstring
+
+
+JsonExportable = Union[Post, Profile, StoryItem, Hashtag]
 
 
 def save_structure_to_file(structure: JsonExportable, filename: str) -> None:
-    """Saves a :class:`Post`, :class:`Profile` or :class:`StoryItem` to a '.json' or '.json.xz' file such that it can
-    later be loaded by :func:`load_structure_from_file`.
+    """Saves a :class:`Post`, :class:`Profile`, :class:`StoryItem` or :class:`Hashtag` to a '.json' or '.json.xz' file
+    such that it can later be loaded by :func:`load_structure_from_file`.
 
     If the specified filename ends in '.xz', the file will be LZMA compressed. Otherwise, a pretty-printed JSON file
     will be created.
 
-    :param structure: :class:`Post`, :class:`Profile` or :class:`StoryItem`
+    :param structure: :class:`Post`, :class:`Profile`, :class:`StoryItem` or :class:`Hashtag`
     :param filename: Filename, ends in '.json' or '.json.xz'
     """
     json_structure = {'node': structure._asdict(),
@@ -1060,8 +1423,8 @@ def save_structure_to_file(structure: JsonExportable, filename: str) -> None:
 
 
 def load_structure_from_file(context: InstaloaderContext, filename: str) -> JsonExportable:
-    """Loads a :class:`Post`, :class:`Profile` or :class:`StoryItem` from a '.json' or '.json.xz' file that
-    has been saved by :func:`save_structure_to_file`.
+    """Loads a :class:`Post`, :class:`Profile`, :class:`StoryItem` or :class:`Hashtag` from a '.json' or '.json.xz' file
+    that has been saved by :func:`save_structure_to_file`.
 
     :param context: :attr:`Instaloader.context` linked to the new object, used for additional queries if neccessary.
     :param filename: Filename, ends in '.json' or '.json.xz'
@@ -1082,6 +1445,8 @@ def load_structure_from_file(context: InstaloaderContext, filename: str) -> Json
             return Profile(context, json_structure['node'])
         elif node_type == "StoryItem":
             return StoryItem(context, json_structure['node'])
+        elif node_type == "Hashtag":
+            return Hashtag(context, json_structure['node'])
         else:
             raise InvalidArgumentException("{}: Not an Instaloader JSON.".format(filename))
     elif 'shortcode' in json_structure:

setup.py

@@ -60,6 +60,7 @@ setup(
         'Programming Language :: Python :: 3.5',
         'Programming Language :: Python :: 3.6',
         'Programming Language :: Python :: 3.7',
+        'Programming Language :: Python :: 3.8',
         'Programming Language :: Python :: 3 :: Only',
         'Topic :: Internet',
         'Topic :: Multimedia :: Graphics'

test/instaloader_unittests.py

@@ -5,12 +5,14 @@ import shutil
 import tempfile
 import unittest
 from itertools import islice
+from typing import Dict, List
 
 import instaloader
 
 PROFILE_WITH_HIGHLIGHTS = 325732271
 PUBLIC_PROFILE = "selenagomez"
 PUBLIC_PROFILE_ID = 460563723
+PUBLIC_PROFILE_WITH_IGTV = "natgeo"
 HASHTAG = "kitten"
 LOCATION = "362629379"
 OWN_USERNAME = "aandergr"
@@ -22,7 +24,7 @@ EMPTY_PROFILE = "not_public"
 EMPTY_PROFILE_ID = 1928659031
 
 # Preserve query timestamps (rate control) between tests to not get rate limited
-instaloadercontext_query_timestamps = dict()
+instaloadercontext_query_timestamps = dict()  # type: Dict[str, List[float]]
 
 
 class TestInstaloaderAnonymously(unittest.TestCase):
@@ -35,11 +37,13 @@ class TestInstaloaderAnonymously(unittest.TestCase):
                                          download_comments=True,
                                          save_metadata=True)
         self.L.context.raise_all_errors = True
-        self.L.context.query_timestamps = instaloadercontext_query_timestamps.copy()
+        # pylint:disable=protected-access
+        self.L.context._graphql_query_timestamps = instaloadercontext_query_timestamps.copy()
 
     def tearDown(self):
+        # pylint:disable=global-statement,protected-access
         global instaloadercontext_query_timestamps
-        instaloadercontext_query_timestamps = self.L.context.query_timestamps.copy()
+        instaloadercontext_query_timestamps = self.L.context._graphql_query_timestamps.copy()
         self.L.close()
         os.chdir('/')
         print("Removing {}".format(self.dir))
@@ -53,31 +57,23 @@ class TestInstaloaderAnonymously(unittest.TestCase):
                 self.assertTrue(post.date_utc < previous_post.date_utc)
             previous_post = post
 
-    @unittest.SkipTest
+    @unittest.skip('')
     def test_public_profile_download(self):
-        self.L.download_profile(PUBLIC_PROFILE, profile_pic=False, fast_update=True)
-        self.L.download_profile(PUBLIC_PROFILE, profile_pic=False, fast_update=True)
+        profiles = {self.L.check_profile_id(PUBLIC_PROFILE)}
+        self.L.download_profiles(profiles, profile_pic=False, fast_update=True, raise_errors=True)
+        self.L.download_profiles(profiles, profile_pic=False, fast_update=True, raise_errors=True)
 
     def test_public_profile_paging(self):
         self.post_paging_test(instaloader.Profile.from_username(self.L.context, PUBLIC_PROFILE).get_posts())
 
     def test_profile_pic_download(self):
-        self.L.download_profile(PUBLIC_PROFILE, profile_pic_only=True)
+        self.L.download_profiles({self.L.check_profile_id(PUBLIC_PROFILE)}, posts=False, raise_errors=True)
 
     def test_hashtag_download(self):
         self.L.download_hashtag(HASHTAG, NORMAL_MAX_COUNT)
 
     def test_hashtag_paging(self):
-        for count, post in enumerate(self.L.get_hashtag_posts(HASHTAG)):
-            print(post)
-            if count == PAGING_MAX_COUNT:
-                break
-
-    def test_location_download(self):
-        self.L.download_location(LOCATION, NORMAL_MAX_COUNT)
-
-    def test_location_paging(self):
-        for count, post in enumerate(self.L.get_location_posts(LOCATION)):
+        for count, post in enumerate(instaloader.Hashtag.from_name(self.L.context, HASHTAG).get_all_posts()):
             print(post)
             if count == PAGING_MAX_COUNT:
                 break
@@ -98,6 +94,10 @@ class TestInstaloaderAnonymously(unittest.TestCase):
         self.assertEqual(EMPTY_PROFILE.lower(),
                          instaloader.Profile.from_id(self.L.context, EMPTY_PROFILE_ID).username)
 
+    def test_get_username_by_name_empty(self):
+        self.assertEqual(EMPTY_PROFILE_ID,
+                         instaloader.Profile.from_username(self.L.context, EMPTY_PROFILE).userid)
+
     def test_post_from_mediaid(self):
         for post in instaloader.Profile.from_username(self.L.context, PUBLIC_PROFILE).get_posts():
             post2 = instaloader.Post.from_mediaid(self.L.context, post.mediaid)
@@ -109,6 +109,11 @@ class TestInstaloaderAnonymously(unittest.TestCase):
                            PAGING_MAX_COUNT):
             print(post)
 
+    def test_public_profile_igtv(self):
+        for post in islice(instaloader.Profile.from_username(self.L.context, PUBLIC_PROFILE_WITH_IGTV).get_igtv_posts(),
+                           PAGING_MAX_COUNT):
+            print(post)
+
 
 class TestInstaloaderLoggedIn(TestInstaloaderAnonymously):
 
@@ -116,13 +121,13 @@ class TestInstaloaderLoggedIn(TestInstaloaderAnonymously):
         super().setUp()
         self.L.load_session_from_file(OWN_USERNAME)
 
-    @unittest.SkipTest
+    @unittest.skip('')
     def test_stories_download(self):
         self.L.download_stories()
 
-    @unittest.SkipTest
+    @unittest.skip('')
     def test_private_profile_download(self):
-        self.L.download_profile(PRIVATE_PROFILE, download_stories=True)
+        self.L.download_profiles({self.L.check_profile_id(PRIVATE_PROFILE)}, stories=True, raise_errors=True)
 
     def test_stories_paging(self):
         for user_story in self.L.get_stories():
@@ -141,9 +146,6 @@ class TestInstaloaderLoggedIn(TestInstaloaderAnonymously):
     def test_private_profile_paging(self):
         self.post_paging_test(instaloader.Profile.from_username(self.L.context, PRIVATE_PROFILE).get_posts())
 
-    def test_profile_pic_download(self):
-        self.L.download_profile(PUBLIC_PROFILE, profile_pic_only=True)
-
     def test_feed_download(self):
         self.L.download_feed_posts(NORMAL_MAX_COUNT)
 
@@ -167,8 +169,8 @@ class TestInstaloaderLoggedIn(TestInstaloaderAnonymously):
 
     def test_followees_and_stories(self):
         profile = instaloader.Profile.from_username(self.L.context, OWN_USERNAME)
-        for f in islice(profile.get_followees(), PAGING_MAX_COUNT):
-            self.L.download_profile(f.username, profile_pic=False, download_stories_only=True)
+        followees = set(islice(profile.get_followees(), PAGING_MAX_COUNT))
+        self.L.download_profiles(followees, profile_pic=False, stories=True, posts=False, raise_errors=True)
 
     def test_get_followees(self):
         profile = instaloader.Profile.from_username(self.L.context, OWN_USERNAME)
@@ -180,6 +182,11 @@ class TestInstaloaderLoggedIn(TestInstaloaderAnonymously):
         for f in profile.get_followers():
             print(f.username)
 
+    def test_get_similar_accounts(self):
+        profile = instaloader.Profile.from_username(self.L.context, OWN_USERNAME)
+        for f in profile.get_similar_accounts():
+            print(f.username)
+
     def test_get_likes(self):
         for post in instaloader.Profile.from_username(self.L.context, OWN_USERNAME).get_posts():
             for like in post.get_likes():
@@ -192,6 +199,15 @@ class TestInstaloaderLoggedIn(TestInstaloaderAnonymously):
             if count == PAGING_MAX_COUNT:
                 break
 
+    def test_location_download(self):
+        self.L.download_location(LOCATION, NORMAL_MAX_COUNT)
+
+    def test_location_paging(self):
+        for count, post in enumerate(self.L.get_location_posts(LOCATION)):
+            print(post)
+            if count == PAGING_MAX_COUNT:
+                break
+
 
 if __name__ == '__main__':
     unittest.main()