Second Release Candidate for Version 4.2

Closes #200.

.github/ISSUE_TEMPLATE/bug_report.md

@@ -0,0 +1,24 @@
+---
+name: Bug report
+about: Create a report to help us improve
+
+---
+
+**Describe the bug**
+A clear and concise description of what the bug is.
+
+**To Reproduce**
+Steps to reproduce the behavior:
+(e.g. Instaloader command line)
+
+**Expected behavior**
+A clear and concise description of what you expected to happen (if not obvious).
+
+**Error messages and tracebacks**
+If applicable, add error messages and tracebacks to help explain your problem.
+
+**Instaloader version**
+`instaloader --version` output.
+
+**Additional context**
+Add any other context about the problem here.

.github/ISSUE_TEMPLATE/feature_request.md

@@ -0,0 +1,17 @@
+---
+name: Feature request
+about: Suggest an idea for this project
+
+---
+
+**Provide us a use case of the feature**
+How could the user invoke the new function? Which problem would it solve? If new information is obtained, how would it be further processed?
+
+**Describe the solution you'd like**
+A clear and concise description of what you want to happen.
+
+**Describe alternatives you've considered**
+A clear and concise description of any alternative solutions or features you've considered.
+
+**Additional context**
+Add any other context about the feature request here.

.travis.yml

@@ -4,17 +4,26 @@ python:
  - "3.5-dev"
  - "3.6"
  - "3.6-dev"
- - "3.7-dev"
- - "nightly"
 install:
  - pip install pylint requests
+ - pip install -r docs/requirements.txt
 script:
- - python3 -m pylint -r n -d bad-whitespace,missing-docstring,too-many-arguments,locally-disabled,line-too-long,too-many-public-methods,too-many-lines,too-many-instance-attributes,too-many-locals,too-many-branches,too-many-statements instaloader
+ - python3 -m pylint -r n -d bad-whitespace,missing-docstring,too-many-arguments,locally-disabled,line-too-long,too-many-public-methods,too-many-lines,too-many-instance-attributes,too-many-locals,too-many-branches,too-many-statements,inconsistent-return-statements,invalid-name,wildcard-import,unused-wildcard-import,no-else-return,cyclic-import,unnecessary-pass instaloader
+ - make -C docs html SPHINXOPTS="-W -n"
 deploy:
-  provider: pypi
-  user: aandergr
-  on:
-    tags: true
-    python: 3.6
-  password:
-    secure: KBsATtXNnEzqr0X2ALdyx2zaYj2EUfK8YjP4jEOilBELpJ4evHYXVgRkn9CKXGsD7LKbWmR3gz2rr8NZ6U42pHw0fpQHizz91MpJSKbNY7RZZh4UVWMDh81izeglchWAF+m3Wdpxpo/GBDbrZIbRQFhPgB+kjSzJo3uiCS+yVXtGG+6csJHzd0WkJjsVHz+XSdxyOiLy/qFF3nmY6ZrVrgw6WbX2YLbx9je6DxK3YRrnBqgMgH7Ui2xWWbgJCk3KKpQTMiMKnTa9mdeMJAd26YMCz1gG/X7ck6VNly/6bs5+Zq7/bU0dSolhPBhNthyccYdS0IeGPdPF3YqatblJ4fLGTfHn8oJs3/T11yIW9pvOT3UftEynPOUX+en2Qnb/3WfgwUfVts8i8DAYgEFLGEzbF5XMGy4gwAD/bZ0JW0pmgAG2PZsdOokAt+ydlNj/uxq1GFW2doZbul6k4t03p1mik6itYscvuhKoEm8Yw6wr1P9aDvMR6FFpSu4gDt2XrKmRmWkSXxpNeCfZujTjyAnISxLVQGSY7BG5K0Z2HbDtqqNz9nj4WgNGzZmngiRofCevWR+vPIDUDW7NCX03csfcFS9RRllBO/BrL7NVUV+Qp9dfTQYp+8SqYLWgHtDoXdgyMREjJaAmbx2u/TYTY8bADZ5F3vGUpv2j5hP3NTU=
+  - provider: pypi
+    user: aandergr
+    on:
+      tags: true
+      python: 3.6
+    password:
+      secure: KBsATtXNnEzqr0X2ALdyx2zaYj2EUfK8YjP4jEOilBELpJ4evHYXVgRkn9CKXGsD7LKbWmR3gz2rr8NZ6U42pHw0fpQHizz91MpJSKbNY7RZZh4UVWMDh81izeglchWAF+m3Wdpxpo/GBDbrZIbRQFhPgB+kjSzJo3uiCS+yVXtGG+6csJHzd0WkJjsVHz+XSdxyOiLy/qFF3nmY6ZrVrgw6WbX2YLbx9je6DxK3YRrnBqgMgH7Ui2xWWbgJCk3KKpQTMiMKnTa9mdeMJAd26YMCz1gG/X7ck6VNly/6bs5+Zq7/bU0dSolhPBhNthyccYdS0IeGPdPF3YqatblJ4fLGTfHn8oJs3/T11yIW9pvOT3UftEynPOUX+en2Qnb/3WfgwUfVts8i8DAYgEFLGEzbF5XMGy4gwAD/bZ0JW0pmgAG2PZsdOokAt+ydlNj/uxq1GFW2doZbul6k4t03p1mik6itYscvuhKoEm8Yw6wr1P9aDvMR6FFpSu4gDt2XrKmRmWkSXxpNeCfZujTjyAnISxLVQGSY7BG5K0Z2HbDtqqNz9nj4WgNGzZmngiRofCevWR+vPIDUDW7NCX03csfcFS9RRllBO/BrL7NVUV+Qp9dfTQYp+8SqYLWgHtDoXdgyMREjJaAmbx2u/TYTY8bADZ5F3vGUpv2j5hP3NTU=
+  - provider: pages
+    skip-cleanup: true
+    github-token: $GITHUB_TOKEN
+    local-dir: docs/_build/html
+    repo: instaloader/instaloader.github.io
+    target-branch: master
+    on:
+      branch: master
+      python: 3.6

AUTHORS.md

@@ -0,0 +1,8 @@
+Authors
+=======
+
+Instaloader is written by
+
+- Alexander Graf (@aandergr)
+- André Koch-Kramer (@Thammus)
+- Lars Lindqvist (@e5150)

LICENSE

@@ -1,6 +1,6 @@
 The MIT License (MIT)
 
-Copyright (c) 2016 Alexander Graf and André Koch-Kramer.
+Copyright (c) 2016-2018 Alexander Graf and André Koch-Kramer.
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

README.rst

@@ -1,39 +1,36 @@
-Instaloader
-===========
-
-Installation
-------------
-
-.. installation-start
-
-Instaloader requires `Python <https://www.python.org/>`__, at least
-version 3.5.  If you have `pip <https://pypi.python.org/pypi/pip>`__
-installed, you may install Instaloader using
+.. image:: https://raw.githubusercontent.com/instaloader/instaloader/master/docs/logo_heading.png
 
 ::
 
-    pip3 install instaloader
+    $ pip3 install instaloader
 
-Alternatively, to get the most current version of Instaloader from our
-`Git repository <https://github.com/Thammus/instaloader>`__:
+    $ instaloader profile [profile ...]
+
+**Instaloader**
+
+- downloads **public and private profiles, hashtags, user stories,
+  feeds and saved media**,
+
+- downloads **comments, geotags and captions** of each post,
+
+- automatically **detects profile name changes** and renames the target
+  directory accordingly,
+
+- allows **fine-grained customization** of filters and where to store
+  downloaded media.
 
 ::
 
-    pip3 install git+https://github.com/Thammus/instaloader
+    instaloader [--comments] [--geotags] [--stories] [--highlights] [--tagged]
+                [--login YOUR-USERNAME] [--fast-update]
+                profile | "#hashtag" | :stories | :feed | :saved
 
-(pass ``--upgrade`` to upgrade if Instaloader is already installed)
+`Instaloader Documentation <https://instaloader.github.io/>`__
 
-Instaloader requires
-`requests <http://python-requests.org/>`__, which
-will be installed automatically, if it is not already installed.
-
-.. installation-end
 
 How to Automatically Download Pictures from Instagram
 -----------------------------------------------------
 
-.. basic-usage-start
-
 To **download all pictures and videos of a profile**, as well as the
 **profile picture**, do
 
@@ -52,8 +49,8 @@ To later **update your local copy** of that profiles, you may run
 
 If ``--fast-update`` is given, Instaloader stops when arriving at the
 first already-downloaded picture. When updating profiles, Instaloader
-automatically **detects profile name changes** and renames the target
-directory accordingly.
+automatically **detects profile name changes** and renames the target directory
+accordingly.
 
 Instaloader can also be used to **download private profiles**. To do so,
 invoke it with
@@ -62,291 +59,13 @@ invoke it with
 
     instaloader --login=your_username profile [profile ...]
 
-When logging in, Instaloader **stores the session cookies** in a file in
-your temporary directory, which will be reused later the next time
-``--login`` is given.  So you can download private profiles
-**non-interactively** when you already have a valid session cookie file.
+When logging in, Instaloader **stores the session cookies** in a file in your
+temporary directory, which will be reused later the next time ``--login``
+is given.  So you can download private profiles **non-interactively** when you
+already have a valid session cookie file.
 
-What to Download
-^^^^^^^^^^^^^^^^
+`Instaloader Documentation <https://instaloader.github.io/basic-usage.html>`__
 
-Instaloader does not only download media by-profile. More generally, you
-may specify the following targets:
-
-- ``profile``: Public profile, or private profile with ``--login``,
-
-- ``"#hashtag"``: Posts with a certain **hashtag** (the quotes are
-  usually neccessary),
-
-- ``:stories``: The currently-visible **stories** of your followees
-  (requires ``--login``),
-
-- ``:feed``: Your **feed** (requires ``--login``),
-
-- ``@profile``: All profiles that are followed by ``profile``, i.e. the
-  *followees* of ``profile`` (requires ``--login``).
-
-Instaloader goes through all media matching the specified targets and
-downloads the pictures and videos and their captions. You can specify
-``--comments`` to also **download comments** of each post and
-``--geotags`` to **download geotags** of each post and save them as
-Google Maps link.  For each profile you download, ``--stories``
-instructs Instaloader to **download the user's stories**.
-
-Filename Specification
-^^^^^^^^^^^^^^^^^^^^^^
-
-For each target, Instaloader creates a directory named after the target,
-i.e. ``profile``, ``#hashtag``, ``:feed``, etc. and therein saves the
-posts in files named after the post's timestamp.
-
-``--dirname-pattern`` allows to configure the directory name of each
-target. The default is ``--dirname-pattern={target}``. In the dirname
-pattern, the token ``{target}`` is replaced by the target name, and
-``{profile}`` is replaced by the owner of the post which is downloaded.
-
-``--filename-pattern`` configures the path of the post's files relative
-to the target directory. The default is ``--filename-pattern={date}``.
-The tokens ``{target}`` and ``{profile}`` are replaced like in the
-dirname pattern. Further, the tokens ``{date}`` and ``{shortcode}`` are
-defined.
-
-For example, encode the poster's profile name in the filenames with:
-
-::
-
-    instaloader --filename-pattern={date}_{profile} "#hashtag"
-
-The pattern string is formatted with Python's string formatter. This
-gives additional flexibilty for pattern specification. For example,
-`strftime-style formatting options <https://docs.python.org/3/library/datetime.html#strftime-and-strptime-behavior>`__
-are supported for the post's
-timestamp. The default for ``{date}`` is ``{date:%Y-%m-%d_%H-%M-%S}``.
-
-Filter Posts
-^^^^^^^^^^^^
-
-The ``--only-if`` option allows to specify criterias that posts have to
-meet to be downloaded. If not given, all posts are downloaded. It must
-be a boolean Python expression where the variables ``likes``,
-``comments``, ``viewer_has_liked``, ``is_video``, and many
-more are defined.
-
-A few examples:
-
-To **download the pictures from your feed that you have liked**:
-
-::
-
-    instaloader --login=your_username --only-if=viewer_has_liked :feed
-
-Or you might only want to download **posts that either you liked or were
-liked by many others**:
-
-::
-
-    instaloader --login=your_username --only-if="likes>100 or viewer_has_liked" profile
-
-Or you may **skip videos**:
-
-::
-
-    instaloader --only-if="not is_video" target
-
-Or you may filter by hashtags that occur in the Post's caption. For
-example, to download posts of kittens that are cute: ::
-
-    instaloader --only-if="'cute' in caption_hashtags" "#kitten"
-
-.. basic-usage-end
-
-(For a more complete description of the ``-only-if`` option, refer to
-the `Instaloader Documentation <https://instaloader.readthedocs.io/basic-usage.html#filter-posts>`__)
-
-
-Advanced Options
-----------------
-
-.. cli-options-start
-
-The following flags can be given to Instaloader to specify how profiles should
-be downloaded.
-
-To get a list of all flags, their abbreviations and their descriptions, you may
-run ``instaloader --help``.
-
-What to Download
-^^^^^^^^^^^^^^^^
-
-Specify a list of profiles or #hashtags. For each of these, Instaloader
-creates a folder and downloads all posts along with the pictures's
-captions and the current **profile picture**. If an already-downloaded profile
-has been renamed, Instaloader automatically **finds it by its unique ID** and
-renames the folder likewise.
-
-Instead of a *profile* or a *#hashtag*, the special targets
-``:feed`` (pictures from your feed) and
-``:stories`` (stories of your followees) can be specified.
-
---profile-pic-only         Only download profile picture.
---no-videos                Do not download videos.
---geotags                  **Download geotags** when available. Geotags are stored as
-                           a text file with the location's name and a Google Maps
-                           link. This requires an additional request to the
-                           Instagram server for each picture, which is why it is
-                           disabled by default.
---no-geotags               Do not store geotags, even if they can be obtained
-                           without any additional request.
---comments                 Download and update comments for each post. This
-                           requires an additional request to the Instagram server
-                           for each post, which is why it is disabled by default.
---no-captions              Do not store media captions, although no additional
-                           request is needed to obtain them.
---stories                  Also **download stories** of each profile that is
-                           downloaded. Requires ``--login``.
---stories-only             Rather than downloading regular posts of each
-                           specified profile, only download stories.
-                           Requires ``--login``.
---only-if filter           Expression that, if given, must evaluate to True for each post to
-                           be downloaded. Must be a syntactically valid python
-                           expression. Variables are evaluated to
-                           ``instaloader.Post`` attributes.
-                           Example: ``--only-if=viewer_has_liked``.
-
-
-When to Stop Downloading
-^^^^^^^^^^^^^^^^^^^^^^^^
-
-If none of these options are given, Instaloader goes through all pictures
-matching the specified targets.
-
---fast-update              For each target, stop when encountering the first
-                           already-downloaded picture. This flag is recommended
-                           when you use Instaloader to update your personal
-                           Instagram archive.
---count COUNT              Do not attempt to download more than COUNT posts.
-                           Applies only to ``#hashtag``, ``:feed-all`` and ``:feed-liked``.
-
-
-Login (Download Private Profiles)
-^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-
-Instaloader can **login to Instagram**. This allows downloading private
-profiles. To login, pass the ``--login`` option. Your session cookie (not your
-password!) will be saved to a local file to be reused next time you want
-Instaloader to login.
-
---login YOUR-USERNAME      Login name (profile name) for your Instagram account.
---sessionfile SESSIONFILE  Path for loading and storing session key file.
-                           Defaults to a path
-                           within your temporary directory, encoding your local
-                           username and your Instagram profile name.
---password YOUR-PASSWORD   Password for your Instagram account. Without this
-                           option, you'll be prompted for your password
-                           interactively if there is not yet a valid session
-                           file.
-
-How to Download
-^^^^^^^^^^^^^^^
-
---dirname-pattern DIRNAME_PATTERN
-                           Name of directory where to store posts. ``{profile}``
-                           is replaced by the profile name, ``{target}`` is replaced
-                           by the target you specified, i.e. either ``:feed``,
-                           ``#hashtag`` or the profile name. Defaults to ``{target}``.
---filename-pattern FILENAME_PATTERN
-                           Prefix of filenames. Posts are stored in the
-                           directory whose pattern is given with ``--dirname-pattern``.
-                           ``{profile}`` is replaced by the profile name,
-                           ``{target}`` is replaced by the target you specified, i.e.
-                           either ``:feed``, ``#hashtag`` or the profile name. Also, the
-                           fields ``{date}`` and ``{shortcode}`` can be specified.
-                           Defaults to ``{date:%Y-%m-%d_%H-%M-%S}``.
---user-agent USER_AGENT    User Agent to use for HTTP requests. Per default,
-                           Instaloader pretends being Chrome/51.
-
-Miscellaneous Options
-^^^^^^^^^^^^^^^^^^^^^
-
---quiet                    Disable user interaction, i.e. do not print messages
-                           (except errors) and fail if login credentials are
-                           needed but not given. This makes Instaloader
-                           **suitable as a cron job**.
-
-.. cli-options-end
-
-Usage as Python module
-----------------------
-
-.. as-module-intro-start
-
-You may also use parts of Instaloader as library to do other interesting
-things.
-
-For example, to get a list of all followees and a list of all followers of a profile, do
-
-.. code:: python
-
-    import instaloader
-
-    # Get instance
-    loader = instaloader.Instaloader()
-
-    # Login
-    loader.interactive_login(USERNAME)
-
-    # Print followees
-    print(PROFILE + " follows these profiles:")
-    for f in loader.get_followees(PROFILE):
-        print("\t%s\t%s" % (f['username'], f['full_name']))
-
-    # Print followers
-    print("Followers of " + PROFILE + ":")
-    for f in loader.get_followers(PROFILE):
-        print("\t%s\t%s" % (f['username'], f['full_name']))
-
-Then, you may download all pictures of all followees with
-
-.. code:: python
-
-    for f in loader.get_followees(PROFILE):
-        loader.download_profile(f['username'])
-
-You could also download your last 20 liked pics with
-
-.. code:: python
-
-    loader.download_feed_posts(max_count=20, fast_update=True,
-                               filter_func=lambda post: post.viewer_has_liked)
-
-To download the last 20 pictures with hashtag #cat, do
-
-.. code:: python
-
-    loader.download_hashtag('cat', max_count=20)
-
-Generally, Instaloader provides methods to iterate over the Posts from
-a certain source.
-
-.. code:: python
-
-    for post in loader.get_hashtag_posts('cat'):
-        # post is an instance of instaloader.Post
-        loader.download_post(post, target='#cat')
-
-Each Instagram profile has its own unique ID which stays unmodified even
-if a user changes his/her username. To get said ID, given the profile's
-name, you may call
-
-.. code:: python
-
-    loader.get_id_by_username(PROFILE_NAME)
-
-.. as-module-intro-end
-
-Refer to the
-`Instaloader Documentation <https://instaloader.readthedocs.io/as-module.html>`__ for
-more information.
 
 Disclaimer
 ----------
@@ -356,6 +75,24 @@ Disclaimer
 Instaloader is in no way affiliated with, authorized, maintained or endorsed by Instagram or any of its affiliates or
 subsidiaries. This is an independent and unofficial project. Use at your own risk.
 
+Instaloader is licensed under an MIT license. Refer to ``LICENSE`` file for more information.
+
 .. disclaimer-end
 
-Instaloader is licensed under an MIT license. Refer to ``LICENSE`` file for more information.
+Contributing
+------------
+
+As an open source project, Instaloader heavily depends on the contributions from
+its community. See
+`contributing <https://instaloader.github.io/contributing.html>`__
+for how you may help Instaloader to become an even greater tool.
+
+It is a pleasure for us to share our Instaloader to the world, and we are proud
+to have attracted such an active and motivating community, with so many users
+who share their suggestions and ideas with us. Buying a community-sponsored beer
+or coffee from time to time is very likely to further raise our passion for the
+development of Instaloader.
+
+| For Donations, we provide a PayPal.Me link and a Bitcoin address.
+|  PayPal: `PayPal.me/aandergr <https://www.paypal.me/aandergr>`__
+|  BTC: 1Nst4LoadeYzrKjJ1DX9CpbLXBYE9RKLwY

docs/Makefile

@@ -50,7 +50,7 @@ clean:
 
 .PHONY: html
 html:
-	$(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html
+	LC_ALL=C $(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html
 	@echo
 	@echo "Build finished. The HTML pages are in $(BUILDDIR)/html."
 
@@ -223,3 +223,9 @@ dummy:
 	$(SPHINXBUILD) -b dummy $(ALLSPHINXOPTS) $(BUILDDIR)/dummy
 	@echo
 	@echo "Build finished. Dummy builder generates no files."
+
+favicon.ico: logo.svg
+	convert -density 384 -background transparent -define icon:auto-resize -colors 256 logo.svg favicon.ico
+
+logo.png: logo.svg
+	inkscape -z -e logo.png -w 100 -h 100 logo.svg

docs/README.md

@@ -0,0 +1,15 @@
+# Instaloader Documentation
+
+This directory contains sources for the Instaloader documentation.
+
+The documentation is hosted at:
+https://instaloader.github.io/
+
+## Build
+
+The documentation is created with [Sphinx](http://www.sphinx-doc.org/). To build it, use
+
+```
+pip3 install -r requirements.txt
+make html
+```

docs/_static/style.css

@@ -0,0 +1,28 @@
+code {
+    color: #222;
+    background-color: #f5f5f5;
+}
+
+dd {
+    padding-left: 8px;
+}
+
+dl.option dd {
+    padding-left: 12px;
+}
+
+a {
+    color: #008d06;
+}
+
+a:hover {
+    color: #f48400;
+}
+
+body {
+    font-size: 16px;
+}
+
+.versionmodified {
+    font-size: 14px;
+}

docs/_templates/caption.html

@@ -1 +0,0 @@
-<h2>Instaloader</h2>

docs/_templates/globaltoc.html

@@ -0,0 +1,14 @@
+<li class="dropdown globaltoc-container">
+  <a role="button"
+     id="dLabelGlobalToc"
+     data-toggle="dropdown"
+     data-target="#"
+     href="{{ pathto(master_doc) }}">{{ theme_navbar_site_name }} <b class="caret"></b></a>
+  <ul class="dropdown-menu globaltoc"
+      role="menu"
+      aria-labelledby="dLabelGlobalToc">
+    {{ toctree(maxdepth=theme_globaltoc_depth|toint, collapse=False, includehidden=theme_globaltoc_includehidden|tobool) }}
+    <li role="separator" class="divider"></li>
+    <li class="toctree-l1"><a class="reference internal" href="genindex.html">Index</a></li>
+  </ul>
+</li>

docs/_templates/layout.html

@@ -1,4 +1,18 @@
 {% extends "!layout.html" %}
+{% block htmltitle %}
+  {% if pagename == "index" %}
+  <title>{{ title|striptags|e }}</title>
+  {% else %}
+  <title>{{ title|striptags|e }}{{ titlesuffix }}</title>
+  {% endif %}
+{% endblock %}
 {% block extrahead %}
-<script async defer src="https://buttons.github.io/buttons.js"></script>
+{{ super() }}
+  <meta name="google-site-verification" content="FNUsDdkOJM9obm9QEdrNBVd9AtAzHzA-aaSMCUrS6C8" />
+  {% if pagename == "index" %}
+  <link rel="canonical" href="https://instaloader.github.io/" />
+  {% else %}
+  <link rel="canonical" href="https://instaloader.github.io/{{ pagename }}.html" />
+  {% endif %}
+  <script async defer src="https://buttons.github.io/buttons.js"></script>
 {% endblock %}

docs/_templates/links.html

@@ -1,18 +0,0 @@
-{% if next %}
-<h3>Next</h3>
-<ul>
-  <li class="toctree-l1"><a class="reference internal" href="{{ next.link|e }}">{{ next.title }}</a></li>
-</ul>
-{% endif %}
-<h3>Links</h3>
-<ul>
-  <li><a href="https://github.com/Thammus/instaloader/issues">Issue Tracker</a></li>
-  <li><a href="https://github.com/Thammus/instaloader/releases">Version History</a></li>
-  <li><a href="https://pypi.python.org/pypi/instaloader">Instaloader on PyPI</a></li>
-  <li><a href="https://github.com/Thammus/instaloader">Instaloader on GitHub</a></li>
-</ul>
-<ul>
-  <li>
-    <a class="github-button" href="https://github.com/Thammus/Instaloader" data-icon="octicon-star" data-size="large" data-show-count="true" aria-label="Star Thammus/Instaloader on GitHub">Star</a>
-  </li>
-</ul>

docs/_templates/navbar.html

@@ -0,0 +1,60 @@
+  <div id="navbar" class="{{ theme_navbar_class }} navbar-default {% if theme_navbar_fixed_top|tobool -%} navbar-fixed-top{%- endif -%}">
+    <div class="container">
+      <div class="navbar-header">
+        <!-- .btn-navbar is used as the toggle for collapsed navbar content -->
+        <button type="button" class="navbar-toggle" data-toggle="collapse" data-target=".nav-collapse">
+          <span class="icon-bar"></span>
+          <span class="icon-bar"></span>
+          <span class="icon-bar"></span>
+        </button>
+        <a class="navbar-brand" href="{{ pathto(master_doc) }}">
+          {%- block sidebarlogo %}
+            {%- if logo %}<span><img src="{{ pathto('_static/' + logo, 1) }}"></span>{%- endif %}
+          {%- endblock %}
+          {% if theme_navbar_title -%}{{ theme_navbar_title|e }}{%- else -%}{{ project|e }}{%- endif -%}
+        </a>
+        <span class="navbar-text navbar-version pull-left">
+          <a href="https://github.com/instaloader/instaloader/releases/tag/v{{current_release}}" title="Released on {{current_release_date}}">v<b>{{current_release}}</b></a>
+        </span>
+      </div>
+
+        <div class="collapse navbar-collapse nav-collapse">
+          <ul class="nav navbar-nav">
+            {% if theme_navbar_links %}
+              {%- for link in theme_navbar_links %}
+                <li><a href="{{ pathto(*link[1:]) }}">{{ link[0] }}</a></li>
+              {%- endfor %}
+            {% endif %}
+            {% block navbartoc %}
+              {% include "globaltoc.html" %}
+              {% if theme_navbar_pagenav and pagename not in ("index", "genindex", "installation") %}
+                {% include "navbartoc.html" %}
+              {% endif %}
+            {% endblock %}
+            {% if theme_navbar_sidebarrel %}
+              {% block sidebarrel %}
+                {% include "relations.html" %}
+              {% endblock %}
+            {% endif %}
+            </ul>
+            <ul class="nav navbar-nav navbar-right">
+              <li class="dropdown">
+                <a role="button" id="dLabelUsefulLinks" data-toggle="dropdown" data-target="#" href="#">
+                  Useful Links
+                  <b class="caret"></b>
+                </a>
+                <ul class="dropdown-menu" role="menu" aria-labelledby="dLabelUsefulLinks">
+                  <li><a href="https://github.com/instaloader/instaloader/issues">Issue Tracker</a></li>
+                  <li><a href="https://github.com/instaloader/instaloader/releases">Version History</a></li>
+                  <li><a href="https://pypi.org/project/instaloader/">Instaloader on PyPI</a></li>
+                  <li><a href="https://github.com/instaloader/instaloader">Instaloader on GitHub</a></li>
+                  <li role="separator" class="divider"></li>
+                  <li style="padding-left: 20px;"><a class="github-button" href="https://github.com/instaloader/instaloader" data-icon="octicon-star" data-size="large" data-show-count="true" aria-label="Star Instaloader on GitHub">Star</a></li>
+                  <li role="separator" class="divider"></li>
+                  <li><a href="https://www.paypal.me/aandergr"><b>Donate (via Paypal)</b></a></li>
+                </ul>
+              </li>
+            </ul>
+        </div>
+    </div>
+  </div>

docs/_templates/page.html

@@ -0,0 +1,10 @@
+{% extends "!page.html" %}
+{% block body %}
+{{ super() }}
+{% if next and prev %}
+<h2>Next Section</h2>
+<p>
+  <a class="reference internal" href="{{ next.link|e }}">{{ next.title }}</a>
+</p>
+{% endif %}
+{% endblock %}

docs/as-module.rst

@@ -1,26 +1,207 @@
-Usage as Python Module
-----------------------
+.. meta::
+   :description:
+      Documentation of Instaloader module, a powerful and intuitive
+      Python library to download Instagram media and metadata.
 
-Introduction
-^^^^^^^^^^^^
+.. _python-module-instaloader:
 
-.. include:: ../README.rst
-   :start-after: as-module-intro-start
-   :end-before: as-module-intro-end
+Python Module :mod:`instaloader`
+--------------------------------
 
+.. module:: instaloader
+
+.. highlight:: python
+
+.. contents::
+  :backlinks: none
+
+Instaloader exposes its internally used methods and structures as a Python
+module, making it a **powerful and intuitive Python API for Instagram**,
+allowing to further customize obtaining media and metadata.
+
+Start with getting an instance of :class:`Instaloader`::
+
+    import instaloader
+
+    # Get instance
+    L = instaloader.Instaloader()
+
+    # Optionally, login or load session
+    L.login(USER, PASSWORD)        # (login)
+    L.interactive_login(USER)      # (ask password on terminal)
+    L.load_session_from_file(USER) # (load session created w/
+                                   #  `instaloader -l USERNAME`)
+
+:mod:`instaloader` provides the :class:`Post` structure, which represents a
+picture, video or sidecar (set of multiple pictures/videos) posted in a user's
+profile. :class:`Instaloader` provides methods to iterate over Posts from a
+certain source::
+
+    for post in L.get_hashtag_posts('cat'):
+        # post is an instance of instaloader.Post
+        L.download_post(post, target='#cat')
+
+:class:`Post` instances can be created with:
+
+- :func:`Post.from_shortcode`
+   Use a Post shortcode (part of the Post URL,
+   ``https://www.instagram.com/p/SHORTCODE/``) to create a Post
+   object::
+
+      post = Post.from_shortcode(L.context, SHORTCODE)
+
+- :meth:`Profile.get_posts`
+   All media of a :class:`Profile`.
+
+- :meth:`Profile.get_saved_posts`
+   Media that the user marked as saved (:class:`Profile` must be own profile
+   for this to work)
+
+- :meth:`Instaloader.get_feed_posts`
+   Media in the user's feed.
+
+- :meth:`Instaloader.get_explore_posts`
+   Media that is suggested by Instagram to explore.
+
+- :meth:`Instaloader.get_hashtag_posts`
+   Media associated with given hashtag.
+
+With the :class:`Profile` class, Instaloader also makes it easy to access
+metadata of a Profile. :class:`Profile` instances can be created with:
+
+- :meth:`Profile.from_username`::
+
+     profile = Profile.from_username(L.context, USERNAME)
+
+- :meth:`Profile.from_id`
+   given its User ID. This allows to easily lookup a Profile's username given
+   its ID::
+
+      Profile.from_id(L.context, USERID).username
+
+- :meth:`Profile.get_followees`
+   Profiles that are followed by given user.
+
+- :meth:`Profile.get_followers`
+   Profiles that follow given user.
+
+- :attr:`Post.owner_profile`, :attr:`Story.owner_profile` and :attr:`StoryItem.owner_profile`
+   Owner profile of particular object.
+
+- :meth:`Post.get_likes`
+   Profiles that liked a given :class:`Post`
+
+- :attr:`PostComment.owner` attribute for comment in :meth:`Post.get_comments`
+   Profile of a Post comment.
+
+A reference of the many methods provided by the :mod:`instaloader` module is
+provided in the remainder of this document.
+
+For a list of real code examples using the Instaloader module for advanced
+tasks, see :ref:`codesnippets`.
 
 ``Instaloader`` (Main Class)
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
-.. autoclass:: instaloader.Instaloader
-   :members:
-   :undoc-members:
+.. autoclass:: Instaloader
+   :no-show-inheritance:
 
-.. _post-class:
+Instagram Structures
+^^^^^^^^^^^^^^^^^^^^
 
-``Post`` Class
-^^^^^^^^^^^^^^
+Posts
+"""""
 
-.. autoclass:: instaloader.Post
-   :members:
-   :undoc-members:
+.. autoclass:: Post
+   :no-show-inheritance:
+
+Additionally, the following trivial structures are defined:
+
+.. autoclass:: PostSidecarNode
+   :no-show-inheritance:
+
+.. autoclass:: PostComment
+   :no-show-inheritance:
+
+.. autoclass:: PostLocation
+   :no-show-inheritance:
+
+User Stories
+""""""""""""
+
+.. autoclass:: Story
+   :no-show-inheritance:
+
+.. autoclass:: StoryItem
+   :no-show-inheritance:
+
+Highlights
+""""""""""
+
+.. autoclass:: Highlight
+   :no-show-inheritance:
+   :inherited-members:
+
+   Bases: :class:`Story`
+
+   .. versionadded:: 4.1
+
+Profiles
+""""""""
+
+.. autoclass:: Profile
+   :no-show-inheritance:
+
+Loading and Saving
+""""""""""""""""""
+
+:class:`Post`, :class:`StoryItem` and :class:`Profile` can be saved and loaded
+to/from JSON files.
+
+.. autofunction:: load_structure_from_file
+
+.. autofunction:: save_structure_to_file
+
+Exceptions
+^^^^^^^^^^
+
+.. currentmodule:: instaloader.exceptions
+
+.. autoexception:: InstaloaderException
+   :no-show-inheritance:
+
+.. autoexception:: ConnectionException
+
+.. currentmodule:: instaloader
+
+.. autoexception:: QueryReturnedBadRequestException
+
+.. autoexception:: QueryReturnedNotFoundException
+
+.. autoexception:: QueryReturnedForbiddenException
+
+.. autoexception:: ProfileNotExistsException
+
+.. autoexception:: ProfileHasNoPicsException
+
+.. autoexception:: PrivateProfileNotFollowedException
+
+.. autoexception:: LoginRequiredException
+
+.. autoexception:: TwoFactorAuthRequiredException
+
+   .. versionadded:: 4.2
+
+.. autoexception:: InvalidArgumentException
+
+.. autoexception:: BadResponseException
+
+.. autoexception:: BadCredentialsException
+
+.. autoexception:: TooManyRequestsException
+
+``InstaloaderContext`` (Low-level functions)
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+.. autoclass:: InstaloaderContext
+   :no-show-inheritance:

docs/basic-usage.rst

@@ -1,15 +1,312 @@
+.. meta::
+   :description:
+      How to download pictures from Instagram. Description of basic
+      usage of Instaloader, free tool to download photos from public
+      and private profiles, hashtags, stories, feeds, saved media, and
+      their metadata, comments and captions.
+
+.. _download-pictures-from-instagram:
+
 Download Pictures from Instagram
 ---------------------------------
 
+.. highlight:: none
+
+Here we describe how to use Instaloader to download pictures from Instagram. If
+you do not have Instaloader installed yet, see :ref:`install`.
+
+.. NOTE that Section "Basic Usage" is duplicated in README.rst.
+
 Basic Usage
 ^^^^^^^^^^^
 
-.. include:: ../README.rst
-   :start-after: basic-usage-start
-   :end-before: basic-usage-end
+To **download all pictures and videos of a profile**, as well as the
+**profile picture**, do
 
-.. (continuation of --only-if explanation)
+::
 
-The given string is evaluated as a
-`Python boolean expression <https://docs.python.org/3/reference/expressions.html#boolean-operations>`__,
-where all occuring variables are properties of the :ref:`post-class`.
+    instaloader profile [profile ...]
+
+where ``profile`` is the name of a profile you want to download. Instead
+of only one profile, you may also specify a list of profiles.
+
+To later **update your local copy** of that profiles, you may run
+
+::
+
+    instaloader --fast-update profile [profile ...]
+
+If :option:`--fast-update` is given, Instaloader stops when arriving at the
+first already-downloaded picture.
+
+When updating profiles, Instaloader
+automatically **detects profile name changes** and renames the target directory
+accordingly.
+
+Instaloader can also be used to **download private profiles**. To do so,
+invoke it with
+
+::
+
+    instaloader --login=your_username profile [profile ...]
+
+When logging in, Instaloader **stores the session cookies** in a file in your
+temporary directory, which will be reused later the next time :option:`--login`
+is given.  So you can download private profiles **non-interactively** when you
+already have a valid session cookie file.
+
+.. _what-to-download:
+
+What to Download
+^^^^^^^^^^^^^^^^
+
+.. targets-start
+
+Instaloader supports the following targets:
+
+- ``profile``
+   Public profile, or private profile with :option:`--login`.
+
+   If an already-downloaded profile has been renamed, Instaloader automatically
+   finds it by its unique ID and renames the folder accordingly.
+
+   Besides the profile's posts, its current profile picture is downloaded. For
+   each profile you download,
+
+   - :option:`--stories`
+      instructs Instaloader to also **download the user's stories**,
+
+   - :option:`--highlights`
+      to **download highlights of each profile that is downloaded**, and
+
+   - :option:`--tagged`
+      to **download posts where the user is tagged**.
+
+- ``"#hashtag"``
+   Posts with a certain **hashtag** (the quotes are usually necessary),
+
+- ``%location id``
+   Posts tagged with a given location; the location ID is the numerical ID
+   Instagram labels a location with (e.g.
+   \https://www.instagram.com/explore/locations/**362629379**/plymouth-naval-memorial/).
+
+   .. versionadded:: 4.2
+
+- ``:stories``
+   The currently-visible **stories** of your followees (requires
+   :option:`--login`),
+
+- ``:feed``
+   Your **feed** (requires :option:`--login`),
+
+- ``:saved``
+   Posts which are marked as **saved** (requires :option:`--login`),
+
+- ``@profile``
+   All profiles that are followed by ``profile``, i.e. the *followees* of
+   ``profile`` (requires :option:`--login`).
+
+- ``-post``
+   The single **post** with the given shortcode. Must be preceeded by ``--`` in
+   the argument list to not be mistaken as an option flag.
+
+   .. versionadded:: 4.1
+
+.. targets-end
+
+Instaloader goes through all media matching the specified targets and
+downloads the pictures and videos and their captions. You can specify
+
+- :option:`--comments`
+   also **download comments** of each post,
+
+- :option:`--geotags`
+   **download geotags** of each post and save them as
+   Google Maps link,
+
+For a reference of all supported command line options, see
+:ref:`command-line-options`.
+
+.. _filename-specification:
+
+Filename Specification
+^^^^^^^^^^^^^^^^^^^^^^
+
+For each target, Instaloader creates a directory named after the target,
+i.e. ``profile``, ``#hashtag``, ``%location id``, ``:feed``, etc. and therein saves the
+posts in files named after the post's timestamp.
+
+:option:`--dirname-pattern` allows to configure the directory name of each
+target. The default is ``--dirname-pattern={target}``. In the dirname
+pattern, the token ``{target}`` is replaced by the target name, and
+``{profile}`` is replaced by the owner of the post which is downloaded.
+
+:option:`--filename-pattern` configures the path of the post's files relative
+to the target directory that is specified with :option:`--dirname-pattern`.
+The default is ``--filename-pattern={date_utc}_UTC``.
+The tokens ``{target}`` and ``{profile}`` are replaced like in the
+dirname pattern. The following tokens are defined for usage with
+:option:`--filename-pattern`:
+
+- ``{target}``
+   Target name (as given in Instaloader command line)
+
+- ``{profile}`` (same as ``{owner_username}``)
+   Owner of the Post / StoryItem.
+
+- ``{owner_id}``
+   Unique integer ID of owner profile.
+
+- ``{shortcode}``
+   Shortcode (identifier string).
+
+- ``{mediaid}``
+   Integer representation of shortcode.
+
+- ``{date_utc}`` (same as ``{date}``)
+   Creation time in UTC timezone.
+   `strftime()-style formatting options <https://docs.python.org/3/library/datetime.html#strftime-and-strptime-behavior>`__
+   are supported as format specifier. The default date format specifier used by
+   Instaloader is::
+
+      {date_utc:%Y-%m-%d_%H-%M-%S}
+
+For example, encode the poster's profile name in the filenames with::
+
+    instaloader --filename-pattern={date_utc}_UTC_{profile} "#hashtag"
+
+As another example, you may instruct Instaloader to store posts in a
+``PROFILE/YEAR/SHORTCODE.jpg`` directory structure::
+
+    instaloader --dirname-pattern={profile} --filename-pattern={date_utc:%Y}/{shortcode} <target> ...
+
+.. _filter-posts:
+
+Filter Posts
+^^^^^^^^^^^^
+
+.. py:currentmodule:: instaloader
+
+The options :option:`--post-filter` and :option:`--storyitem-filter`
+allows to specify criteria that posts or story items have to
+meet to be downloaded. If not given, all posts are downloaded.
+
+The filter string must be a
+`Python boolean expression <https://docs.python.org/3/reference/expressions.html#boolean-operations>`__
+where the attributes from :class:`Post` or
+:class:`StoryItem` respectively are defined.
+
+Id est, the following attributes can be used with both
+:option:`--post-filter` and :option:`--storyitem-filter`:
+
+- :attr:`~Post.owner_username` (str), :attr:`~Post.owner_id` (int)
+   Owner profile username / userid.
+
+- :attr:`~Post.date_utc` (datetime), :attr:`~Post.date_local` (datetime)
+   Creation timestamp. Since :class:`~datetime.datetime` objects can be created
+   inside filter strings, this easily allows filtering by creation date. E.g.::
+
+      instaloader --post-filter="date_utc <= datetime(2018, 5, 31)" target
+
+- :attr:`~Post.is_video` (bool)
+   Post/StoryItem is a video. For example, you may skip videos::
+
+      instaloader --post-filter="not is_video" target
+
+   This is not the same as :option:`--no-videos` and
+   :option:`--no-video-thumbnails`, since sidecar posts (posts that contain
+   multiple pictures/videos in one post) have this attribute set to False.
+
+As :option:`--post-filter`, the following attributes can be used additionally:
+
+- :attr:`~Post.viewer_has_liked` (bool)
+   Whether user (with :option:`--login`) has liked given post. To download the
+   pictures from your feed that you have liked::
+
+      instaloader --login=your_username --post-filter=viewer_has_liked :feed
+
+- :attr:`~Post.likes` (int), :attr:`~Post.comments` (int)
+   Likes count / Comments count. You might only want to download posts that
+   either you liked or were liked by many others**::
+
+      instaloader --login=your_username --post-filter="likes>100 or viewer_has_liked" profile
+
+- :attr:`~Post.caption_hashtags` (list of str) / :attr:`~Post.caption_mentions` (list of str)
+   ``#hashtags`` or ``@mentions`` (lowercased) in the Post's caption. For example, to
+   download posts of kittens that are cute::
+
+       instaloader --post-filter="'cute' in caption_hashtags" "#kitten"
+
+- :attr:`~Post.tagged_users` (list of str)
+   Lowercased usernames that are tagged in the Post.
+
+For :option:`--storyitem-filter`, the following additional attributes are
+defined:
+
+- :attr:`~StoryItem.expiring_utc` (datetime) / :attr:`~StoryItem.expiring_local` (datetime)
+   Timestamp when StoryItem will get unavailable.
+
+.. _metadata-text-files:
+
+Metadata Text Files
+^^^^^^^^^^^^^^^^^^^
+
+Unless :option:`--no-captions` is given, Instaloader creates a ``.txt`` file
+along with each post where the Post's caption is saved.
+
+You can customize what metadata to save for each Post or StoryItem with
+:option:`--post-metadata-txt` and :option:`--storyitem-metadata-txt`. The
+default is ``--post-metadata-txt={caption}`` and no storyitem metadata txt.
+These strings are formatted similar as in the :ref:`filename-specification` and
+the result is saved in text files, unless it is empty.
+
+Specifying these options multiple times results in output having multiple lines,
+in the order they were given to Instaloader.
+
+The field names are evaluated to :class:`Post` or :class:`StoryItem` attributes,
+and as such, the same fields are supported as in :ref:`filename-specification`
+and :ref:`filter-posts`.
+
+For example, to save the current number of likes for each post, rather than
+the post's caption::
+
+   instaloader --post-metadata-txt="{likes} likes." <target>
+
+Note that with this feature, it is possible to easily and fastly extract
+additional metadata of already-downloaded posts, by reimporting their JSON
+files. Say, you now also want to export the number of comments the Posts had
+when they were downloaded::
+
+   instaloader --post-metadata-txt="{likes} likes, {comments} comments." <target>/*.json.xz
+
+.. _instaloader-as-cronjob:
+
+Instaloader as Cronjob
+^^^^^^^^^^^^^^^^^^^^^^
+
+Instaloader is suitable for running as a cronjob to periodically update your
+personal Instagram archive. The :option:`--quiet` option disables user
+interactions and logging of non-error messages. To non-interactively use
+Instaloader logged-in, create a session file, e.g. in your home directory::
+
+   instaloader --login=your_username --sessionfile=~/.instaloader-session
+
+Then use the same parameters in your cronjob to load the session and download
+the given targets::
+
+   instaloader --login=your_username --sessionfile=~/.instaloader-session --quiet <target> [...]
+
+Without :option:`--sessionfile` option, Instaloader saves the session file in
+a path within your temporary directory.
+
+Programming Instaloader
+^^^^^^^^^^^^^^^^^^^^^^^
+
+If your task cannot be done with the command line interface of Instaloader,
+consider taking a look at the :ref:`python-module-instaloader`.
+Instaloader exposes its internally used methods and structures, making it a
+powerful and intuitive Python API for Instagram, allowing to further customize
+obtaining media and metadata.
+
+Also see :ref:`codesnippets`, where we collect example scripts that use
+Instaloader to achieve more complex tasks.

docs/cli-options.rst

@@ -1,6 +1,266 @@
+.. _command-line-options:
+
 Command Line Options
 ====================
 
-.. include:: ../README.rst
-   :start-after: cli-options-start
-   :end-before: cli-options-end
+Instaloader is invoked with::
+
+   $ instaloader [options] target [target ...]
+
+where ``target`` is a ``profile``, a ``"#hashtag"``, ``@profile`` (all profiles
+that *profile* is following), ``%location ID``, or if logged in ``:feed`` (pictures from your
+feed), ``:stories`` (stories of your followees) or ``:saved`` (collection of
+posts marked as saved).
+
+Here we explain the additional options that can be given to Instaloader to
+customize its behavior.  For an
+introduction on how to use Instaloader, see
+:ref:`download-pictures-from-instagram`.
+
+To get a list of all flags, their abbreviations and
+their descriptions, you may also run::
+
+   instaloader --help
+
+Targets
+^^^^^^^
+
+Specify a list of targets. For each of these, Instaloader creates a folder and
+stores all posts along with the pictures's captions there.
+
+.. include:: basic-usage.rst
+   :start-after: targets-start
+   :end-before: targets-end
+
+- ``filename.json[.xz]``
+   Re-Download the given object
+
+- ``+args.txt``
+   Read targets (and options) from given textfile. See :option:`+args.txt`.
+
+What to Download of each Post
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+.. option:: --no-pictures
+
+   Do not download post pictures. Cannot be used together with
+   :option:`--fast-update`. Implies :option:`--no-video-thumbnails`, does not
+   imply :option:`--no-videos`.
+
+   .. versionadded:: 4.1
+
+.. option:: --no-videos, -V
+
+   Do not download videos.
+
+.. option:: --no-video-thumbnails
+
+   Do not download thumbnails of videos.
+
+.. option:: --geotags, -G
+
+   **Download geotags** when available. Geotags are stored as a text file with
+   the location's name and a Google Maps link. This requires an additional
+   request to the Instagram server for each picture, which is why it is disabled
+   by default.
+
+.. option:: --comments, -C
+
+   Download and update comments for each post. This requires an additional
+   request to the Instagram server for each post, which is why it is disabled by
+   default.
+
+.. option:: --no-captions
+
+   Do not create txt files.
+
+.. option:: --post-metadata-txt
+
+   Template to write in txt file for each Post. See :ref:`metadata-text-files`.
+
+.. option:: --storyitem-metadata-txt
+
+   Template to write in txt file for each StoryItem. See
+   :ref:`metadata-text-files`.
+
+.. option:: --no-metadata-json
+
+   Do not create a JSON file containing the metadata of each post.
+
+.. option:: --no-compress-json
+
+   Do not xz compress JSON files, rather create pretty formatted JSONs.
+
+
+What to Download of each Profile
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+.. option:: --profile-pic-only, -P
+
+   .. deprecated:: 4.1
+      Use :option:`--no-posts`.
+
+   Only download profile picture.
+
+.. option:: --no-posts
+
+   Do not download regular posts.
+
+   .. versionadded:: 4.1
+
+.. option:: --no-profile-pic
+
+   Do not download profile picture.
+
+.. option:: --stories, -s
+
+   Also **download stories** of each profile that is downloaded. Requires
+   :option:`--login`.
+
+.. option:: --highlights
+
+   Also **download highlights** of each profile that is downloaded. Requires
+   :option:`--login`.
+
+   .. versionadded:: 4.1
+
+.. option:: --tagged
+
+   Also download posts where each profile is tagged.
+
+   .. versionadded:: 4.1
+
+.. option:: --stories-only
+
+   .. deprecated:: 4.1
+      Use :option:`--stories` :option:`--no-posts`.
+
+   Rather than downloading regular posts of each specified profile, only
+   download stories.  Requires :option:`--login`. Does not imply
+   :option:`--no-profile-pic`.
+
+   .. note::
+
+      If possible, use ``:stories`` target rather than :option:`--stories-only`
+      with all your followees. ``:stories`` uses fewer API requests.
+
+Which Posts to Download
+^^^^^^^^^^^^^^^^^^^^^^^
+
+.. option:: --fast-update, -F
+
+   For each target, stop when encountering the first already-downloaded picture.
+   This flag is recommended when you use Instaloader to update your personal
+   Instagram archive.
+
+.. option:: --post-filter filter, --only-if filter
+
+   Expression that, if given, must evaluate to True for each post to be
+   downloaded.  Must be a syntactically valid Python expression. Variables are
+   evaluated to :class:`instaloader.Post` attributes.  Example:
+   ``--post-filter=viewer_has_liked``. See :ref:`filter-posts` for more
+   examples.
+
+.. option:: --storyitem-filter filter
+
+   Expression that, if given, must evaluate to True for each storyitem to be
+   downloaded.  Must be a syntactically valid Python expression. Variables are
+   evaluated to :class:`instaloader.StoryItem` attributes.
+   See :ref:`filter-posts` for more examples.
+
+.. option:: --count COUNT, -c
+
+   Do not attempt to download more than COUNT posts.  Applies only to
+   ``#hashtag``, ``%location id``, and ``:feed``.
+
+
+Login (Download Private Profiles)
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Instaloader can **login to Instagram**. This allows downloading private
+profiles. To login, pass the :option:`--login` option. Your session cookie (not your
+password!) will be saved to a local file to be reused next time you want
+Instaloader to login.
+
+.. option:: --login YOUR-USERNAME, -l YOUR-USERNAME
+
+   Login name (profile name) for your Instagram account.
+
+.. option:: --sessionfile SESSIONFILE, -f SESSIONFILE
+
+   Path for loading and storing session key file.  Defaults to a path within
+   your temporary directory, encoding your local username and your Instagram
+   profile name.
+
+.. option:: --password YOUR-PASSWORD, -p YOUR-PASSWORD
+
+   Password for your Instagram account.  Without this option, you'll be prompted
+   for your password interactively if there is not yet a valid session file.
+
+   .. warning:: Using :option:`--password` option is discouraged for security
+      reasons.  Enter your password interactively when asked, or use the
+      sessionfile feature (:option:`--sessionfile` to customize path).
+
+How to Download
+^^^^^^^^^^^^^^^
+
+.. option:: --dirname-pattern DIRNAME_PATTERN
+
+   Name of directory where to store posts. ``{profile}`` is replaced by the
+   profile name, ``{target}`` is replaced by the target you specified, i.e.
+   either ``:feed``, ``#hashtag`` or the profile name. Defaults to ``{target}``.
+   See :ref:`filename-specification`.
+
+.. option:: --filename-pattern FILENAME_PATTERN
+
+   Prefix of filenames, relative to the directory given with
+   :option:`--dirname-pattern`. ``{profile}`` is replaced by the profile name,
+   ``{target}`` is replaced by the target you specified, i.e.  either ``:feed``,
+   ``#hashtag`` or the profile name. Defaults to ``{date_utc}_UTC``.
+   See :ref:`filename-specification` for a list of supported tokens.
+
+.. option:: --user-agent USER_AGENT
+
+   User Agent to use for HTTP requests. Per default, Instaloader pretends being
+   Chrome/51.
+
+.. option:: --max-connection-attempts N
+
+   Maximum number of connection attempts until a request is aborted. Defaults
+   to ``3``. If a connection fails, it can be manually skipped by hitting
+   :kbd:`Control-c`. Set this to ``0`` to retry infinitely.
+
+.. option:: --commit-mode
+
+   Tries to ensure downloaded images avoid corruption in case of unexpected
+   interruption. If the last picture is corrupted, Instaloader will fix the
+   picture the next time it is run.
+
+   .. versionadded:: 4.2
+
+Miscellaneous Options
+^^^^^^^^^^^^^^^^^^^^^
+
+.. option:: --quiet, -q
+
+   Disable user interaction, i.e. do not print messages (except errors) and fail
+   if login credentials are needed but not given.
+   This is handy for running :ref:`instaloader-as-cronjob`.
+
+.. option:: +args.txt
+
+   Read arguments from file `args.txt`, a shortcut to provide argument from
+   file rather than command-line. This provide a convient way to hide login
+   info from CLI. and also can use for simplify managment of long arguments.
+
+   .. note::
+
+      text file should separate arg with line break.
+
+      args.txt example::
+
+         --login MYUSENAME
+         --password MYPASSWORD
+         --fast-update
+
+   .. versionadded:: 4.1

docs/codesnippets.rst

@@ -0,0 +1,131 @@
+.. _codesnippets:
+
+Advanced Instaloader Examples
+=============================
+
+.. currentmodule:: instaloader
+
+.. highlight:: python
+
+.. contents::
+   :backlinks: none
+
+Here we present code examples that use the :ref:`python-module-instaloader` for
+more advanced Instagram downloading or metadata mining than what is possible
+with the Instaloader command line interface.
+
+The scripts presented here can be downloaded from our source tree:
+`instaloader/docs/codesnippets/ <https://github.com/instaloader/instaloader/tree/master/docs/codesnippets>`__
+
+.. For each code snippet:
+   - title
+   - brief description of what it does / motivation / how it works
+   - code, or link to code
+   - link to discussion issue
+   - link used methods
+
+Download Posts in a Specific Period
+-----------------------------------
+
+To only download Instagram pictures (and metadata) that are within a specific
+period, you can play around with :func:`~itertools.dropwhile` and
+:func:`~itertools.takewhile` from :mod:`itertools` like in this snippet.
+
+.. literalinclude:: codesnippets/121_since_until.py
+
+See also :class:`Post`, :meth:`Instaloader.download_post`.
+
+Discussed in :issue:`121`.
+
+Likes of a Profile / Ghost Followers
+------------------------------------
+
+To obtain a list of your inactive followers, i.e. followers that did not like
+any of your pictures, into a file you can use this approach.
+
+.. literalinclude:: codesnippets/120_ghost_followers.py
+
+See also :meth:`Profile.get_posts`, :meth:`Post.get_likes`,
+:meth:`Profile.get_followers`, :meth:`Instaloader.load_session_from_file`,
+:meth:`Profile.from_username`.
+
+Discussed in :issue:`120`.
+
+Track Deleted Posts
+-------------------
+
+This script uses Instaloader to obtain a list of currently-online Instagram and
+compares it with the set of posts that you already have downloaded.  It outputs
+a list of posts which are online but not offline (i.e. not yet downloaded) and a
+list of posts which are offline but not online (i.e. deleted in the profile).
+
+.. literalinclude:: codesnippets/56_track_deleted.py
+
+See also :func:`load_structure_from_file`, :meth:`Profile.from_username`,
+:meth:`Profile.get_posts`, :class:`Post`.
+
+Discussed in :issue:`56`.
+
+Only one Post per User
+----------------------
+
+To download only the one most recent post from each user, this snippet creates a
+:class:`set` that contains the users of which a post has already been
+downloaded. While iterating the posts, it checks whether the post's owner
+already is in the set. If not, the post is downloaded from Instagram and the
+user is added to that set.
+
+.. literalinclude:: codesnippets/113_only_one_per_user.py
+
+See also :class:`Post`, :meth:`Instaloader.download_post`,
+:attr:`Post.owner_profile`, :class:`Profile`.
+
+Discussed in :issue:`113`.
+
+Upgrade Images by Local Copies
+------------------------------
+
+The following script finds local versions of images fetched by Instaloader, in
+order to upgrade the downloaded images by locally-found versions with better
+quality. It uses image hashing to identify similar images.
+
+`updgrade-instaloader-images.py <https://gist.github.com/pavelkryukov/15f93d19a99428a284a8bcec27e0187b>`__ (external link to GitHub Gist)
+
+Discussed in :issue:`46`.
+
+Add Captions to Images
+----------------------
+
+Instaloader does not modify the downloaded JPEG file. However, one could combine
+it with an imaging library such as Pillow or PIL to render the caption on
+Instagram pictures.  The following shows an approach.
+
+.. literalinclude:: codesnippets/110_pil_captions.py
+
+See also :attr:`Post.caption`, :attr:`Post.url`, :meth:`Post.from_shortcode`,
+:func:`load_structure_from_file`.
+
+Discussed in :issue:`110`.
+
+Metadata JSON Files
+-------------------
+
+The JSON files Instaloader saves along with each Post contain all the metadata
+that has been retrieved from Instagram while downloading the picture and
+associated required information.
+
+With `jq <https://stedolan.github.io/jq/>`__, a command-line JSON processor, the
+metadata can be easily post-processed. For example, Instaloader's JSON files can
+be pretty-formatted with:
+
+.. code-block:: none
+
+   xzcat 2018-05-13_11-18-45_UTC.json.xz | jq .node
+
+However, Instaloader tries to do as few metadata requests as possible, so,
+depending on how Instaloader has been invoked, it may occur that these files do
+not contain the complete available metadata structure. Nevertheless, the file
+can be loaded into Instaloader with :func:`load_structure_from_file` and the
+required metadata then be accessed via the :class:`Post` or :class:`Profile`
+attributes, which trigger an Instagram request if that particular information is
+not present in the JSON file.

docs/codesnippets/110_pil_captions.py

@@ -0,0 +1,20 @@
+from io import BytesIO
+
+from requests import get
+from PIL import Image, ImageDraw
+from instaloader import *
+
+L = Instaloader()
+
+# Load Post instance
+post = load_structure_from_file(L.context, '2017-10-01_18-53-03_UTC.json.xz')
+# or post = Post.from_shortcode(L.context, SHORTCODE)
+
+# Render caption
+image = Image.open(BytesIO(get(post.url).content))
+draw = ImageDraw.Draw(image)
+color = 'rgb(0, 0, 0)'  # black color
+draw.text((300,100), post.caption.encode('latin1', errors='ignore'), fill=color)
+
+# Save image
+image.save('test.jpg')

docs/codesnippets/113_only_one_per_user.py

@@ -0,0 +1,14 @@
+import instaloader
+
+L = instaloader.Instaloader()
+
+posts = L.get_hashtag_posts('milfgarden')
+
+users = set()
+
+for post in posts:
+    if not post.owner_profile in users:
+        L.download_post(post, '#milfgarden')
+        users.add(post.owner_profile)
+    else:
+        print("{} from {} skipped.".format(post, post.owner_profile))

docs/codesnippets/120_ghost_followers.py

@@ -0,0 +1,27 @@
+import instaloader
+
+L = instaloader.Instaloader()
+
+USER = 'your_account'
+PROFILE = USER
+
+# Your preferred way of logging in:
+L.load_session_from_file(USER)
+
+profile = instaloader.Profile.from_username(L.context, PROFILE)
+
+likes = set()
+print('Fetching likes of all posts of profile {}.'.format(profile.username))
+for post in profile.get_posts():
+    print(post)
+    likes = likes | set(post.get_likes())
+
+print('Fetching followers of profile {}.'.format(profile.username))
+followers = set(profile.get_followers())
+
+ghosts = followers - likes
+
+print('Storing ghosts into file.')
+with open('/YOUR PATH/inactive-users.txt', 'w') as f:
+    for ghost in ghosts:
+        print(ghost.username, file=f)

docs/codesnippets/121_since_until.py

@@ -0,0 +1,17 @@
+from datetime import datetime
+from itertools import dropwhile, takewhile
+
+import instaloader
+
+L = instaloader.Instaloader()
+
+posts = L.get_hashtag_posts('milfgarden')
+# or
+# posts = instaloader.Profile.from_username(L.context, PROFILE).get_posts()
+
+SINCE = datetime(2015, 5, 1)
+UNTIL = datetime(2015, 3, 1)
+
+for post in takewhile(lambda p: p.date > UNTIL, dropwhile(lambda p: p.date > SINCE, posts)):
+    print(post.date)
+    L.download_post(post, '#milfgarden')

docs/codesnippets/56_track_deleted.py

@@ -0,0 +1,34 @@
+from glob import glob
+from sys import argv
+from os import chdir
+
+from instaloader import Instaloader, Post, Profile, load_structure_from_file
+
+# Instaloader instantiation - you may pass additional arguments to the constructor here
+L = Instaloader()
+
+# If desired, load session previously saved with `instaloader -l USERNAME`:
+#L.load_session_from_file(USERNAME)
+
+try:
+    TARGET = argv[1]
+except IndexError:
+    raise SystemExit("Pass profile name as argument!")
+
+# Obtain set of posts that are on hard disk
+chdir(TARGET)
+offline_posts = set(filter(lambda s: isinstance(s, Post),
+                           (load_structure_from_file(L.context, file)
+                            for file in (glob('*.json.xz') + glob('*.json')))))
+
+# Obtain set of posts that are currently online
+post_iterator = Profile.from_username(L.context, TARGET).get_posts()
+online_posts = set(post_iterator)
+
+if online_posts - offline_posts:
+    print("Not yet downloaded posts:")
+    print(" ".join(str(p) for p in (online_posts - offline_posts)))
+
+if offline_posts - online_posts:
+    print("Deleted posts:")
+    print(" ".join(str(p) for p in (offline_posts - online_posts)))

docs/codesnippets/README.md

@@ -0,0 +1,4 @@
+# Advanced Instaloader Examples
+
+This directory contains the scripts that are presented at:
+https://instaloader.github.io/codesnippets.html

docs/conf.py

@@ -18,9 +18,16 @@
 # documentation root, use os.path.abspath to make it absolute, like shown here.
 #
 import os
+import subprocess
 import sys
+import typing
+
+import sphinx_bootstrap_theme
+
 sys.path.insert(0, os.path.abspath('..'))
 
+import docs.sphinx_autodoc_typehints as sphinx_autodoc_typehints
+
 # -- General configuration ------------------------------------------------
 
 # If your documentation needs a minimal Sphinx version, state it here.
@@ -32,10 +39,20 @@ sys.path.insert(0, os.path.abspath('..'))
 # ones.
 extensions = [
     'sphinx.ext.autodoc',
-    'sphinx_autodoc_typehints',
     'sphinx.ext.githubpages',
+    'sphinx.ext.intersphinx',
+    'sphinx.ext.extlinks'
 ]
 
+autodoc_default_flags = ['show-inheritance', 'members', 'undoc-members']
+
+autodoc_member_order = 'bysource'
+
+intersphinx_mapping = {'python': ('https://docs.python.org/3', None),
+                       'requests': ('http://docs.python-requests.org/en/master', None)}
+
+extlinks = {'issue': ('https://github.com/instaloader/instaloader/issues/%s', 'issue #')}
+
 # Add any paths that contain templates here, relative to this directory.
 templates_path = ['_templates']
 
@@ -124,16 +141,20 @@ todo_include_todos = False
 # The theme to use for HTML and HTML Help pages.  See the documentation for
 # a list of builtin themes.
 #
-html_theme = 'alabaster'
+html_theme = 'bootstrap'
+html_theme_path = sphinx_bootstrap_theme.get_html_theme_path()
 
 # Theme options are theme-specific and customize the look and feel of a theme
 # further.  For a list of options available for each theme, see the
 # documentation.
 #
 html_theme_options = {
-    'show_powered_by': False,
-    'sidebar_width': '290px',
-    'page_width': '935px' }
+    'navbar_site_name': 'Site Contents',
+    'navbar_pagenav_name': 'Page Contents',
+    'navbar_pagenav': True,
+    'navbar_sidebarrel': True,
+    'nosidebar': True,
+     }
 
 # Add any paths that contain custom themes here, relative to this directory.
 # html_theme_path = []
@@ -150,13 +171,13 @@ html_theme_options = {
 # The name of an image file (relative to this directory) to place at the top
 # of the sidebar.
 #
-# html_logo = None
+html_logo = "logo.png"
 
 # The name of an image file (relative to this directory) to use as a favicon of
 # the docs.  This file should be a Windows icon file (.ico) being 16x16 or 32x32
 # pixels large.
 #
-# html_favicon = None
+html_favicon = "favicon.ico"
 
 # Add any paths that contain custom static files (such as style sheets) here,
 # relative to this directory. They are copied after the builtin static files,
@@ -182,7 +203,8 @@ html_static_path = ['_static']
 
 # Custom sidebar templates, maps document names to template names.
 #
-html_sidebars = {'**': ["caption.html", "globaltoc.html", "relations.html", "links.html"] }
+#html_sidebars = {'**': ["relations.html", "links.html"] }
+html_sidebars = {}
 
 # Additional templates that should be rendered to pages, maps page names to
 # template names.
@@ -195,7 +217,7 @@ html_domain_indices = False
 
 # If false, no index is generated.
 #
-html_use_index = False
+html_use_index = True
 
 # If true, the index is split into individual pages for each letter.
 #
@@ -344,3 +366,16 @@ texinfo_documents = [
 # If true, do not generate a @detailmenu in the "Top" node's menu.
 #
 # texinfo_no_detailmenu = False
+
+
+current_release = subprocess.check_output(["git", "describe", "--abbrev=0"]).decode("ascii")[1:-1]
+current_release_date = subprocess.check_output(["git", "log", "-1", "--tags", "--format=%ad", "--date=format:%e %b %Y"]).decode("ascii")[:-1]
+
+html_context = {'current_release': current_release, 'current_release_date': current_release_date}
+
+
+def setup(app):
+    typing.TYPE_CHECKING = True
+    app.add_stylesheet("style.css")
+    app.connect('autodoc-process-signature', sphinx_autodoc_typehints.process_signature)
+    app.connect('autodoc-process-docstring', sphinx_autodoc_typehints.process_docstring)

docs/contributing.rst

@@ -0,0 +1,104 @@
+.. _contributing:
+
+Contributing to Instaloader
+===========================
+
+As an open source project, Instaloader heavily depends on the contributions from
+its community.  In this document, we advise on how you may help Instaloader to
+become an even greater tool.
+
+Instaloader's development is organized on 
+`GitHub <https://github.com/instaloader/instaloader>`__, where Issues and Pull
+Requests are discussed.
+
+Reporting Bugs
+--------------
+
+If you encounter a bug, do not hesitate to report it in our
+`Issue Tracker <https://github.com/instaloader/instaloader/issues>`__. Since
+Instaloader is actively developed, the majority of bugs is fixed within only
+**3 days** after being reported. When reporting a problem, please keep the
+following in mind:
+
+- Ensure you **use the latest version** of Instaloader. The currently-installed
+  version can be found out with ``instaloader --version``.
+
+- Briefly **check whether the bug has already been reported**. If you find an
+  issue reporting the same bug you encountered, comment there rather than
+  opening a new issue. However, if unsure, please create a new issue.
+
+- State **how the bug can be reproduced**, i.e. how Instaloader was invoked
+  when the problem occurred (of course, you may anonymize profile names etc.).
+
+- Include all **error messages and tracebacks** in the report.
+
+- If not obvious, describe **which behavior you expected**
+  instead of what actually happened.
+
+- If you think an issue has been closed accidentally or inappropriately, feel
+  free to reopen it.
+
+Writing Code or Improving the Documentation
+-------------------------------------------
+
+Changes of the Instaloader source can be proposed as a
+`Pull Request <https://github.com/instaloader/instaloader/pulls>`__. There are only
+few things to consider:
+
+- Sometimes, the most current code is not in the ``master`` branch. Check that
+  you forked from the most recent branch.
+
+- We use `Pylint <https://www.pylint.org/>`__ for error and syntax checking of
+  the source. The file ``.travis.yml`` in the project's root directory
+  shows how it is invoked. Note that sometimes it might be better to disable a
+  warning rather than adapting the code to Pylint's desires.
+
+- The documentation source is located in the ``docs`` folder. The file
+  ``cli-options.rst`` is merely an RST-formatted copy of ``instaloader --help``
+  output, of which the source is in ``instaloader/__main__.py``.
+
+- Feel free to contact us, even if you "only" have Proof-of-Concepts or
+  not-fully integrated changes. They already might be an advance for the
+  project.
+
+Suggesting Features
+-------------------
+
+If you have an idea of how Instaloader should be enhanced, but do not want to
+implement the feature yourself, feel free to open a ticket in our 
+`Issue Tracker <https://github.com/instaloader/instaloader/issues>`__.
+Please consider the following:
+
+- Instaloader already has plenty of features. **Check the documentation**
+  beforehand to ensure your desired suggestion is not already implemented.
+
+- Briefly **ensure that your idea has not already been suggested**. If you find
+  an issue suggesting the same or a similar feature, share your thoughts in a
+  comment there, instead of opening a new issue.
+
+- If possible, provide us a **use case of the feature**: How could the user
+  invoke the new function? Which problem would it solve? If new information is
+  obtained, how would it be further processed?
+
+- **Be patient**. Naturally, bugs and pull requests have a higher priority than
+  feature suggestions. Keep in mind that this is a free software project, and
+  unfortunately we only have limited time to work on it.
+
+Donations
+---------
+
+.. donations-start
+
+It is a pleasure for us to share our Instaloader to the world, and we are proud
+to have attracted such an active and motivating community, with so many users
+who share their suggestions and ideas with us. Buying a community-sponsored beer
+or coffee from time to time is very likely to further raise our passion for the
+development of Instaloader.
+
+| For donations, we provide a PayPal.Me link and a Bitcoin address.
+|  `PayPal.me/aandergr <https://www.paypal.me/aandergr>`__
+|  BTC: 1Nst4LoadeYzrKjJ1DX9CpbLXBYE9RKLwY
+
+.. donations-end
+
+.. (Discussion in :issue:`130`)

docs/index.rst

@@ -1,36 +1,56 @@
+.. meta::
+   :description:
+      Free command line tool to download photos from Instagram.
+      Scrapes public and private profiles, hashtags, stories, feeds,
+      saved media, and their metadata, comments and captions.
+      Written in Python.
+
+.. title:: Instaloader — Download Instagram Photos and Metadata
+
 Instaloader
 ===========
 
+.. highlight:: none
+
 **Instaloader** is a tool to download pictures (or videos) along with
 their captions and other metadata from Instagram.
 
-With `Python <https://www.python.org/>`__ installed, do:
-
-::
+With `Python <https://www.python.org/>`__ installed, do::
 
     $ pip3 install instaloader
 
     $ instaloader profile [profile ...]
 
+See :ref:`install` for more options on how to install Instaloader.
+
 **Instaloader**
 
-- downloads **public and private profiles, hashtags, user stories and
-  feeds**,
-  
+- downloads **public and private profiles, hashtags, user stories,
+  feeds and saved media**,
+
 - downloads **comments, geotags and captions** of each post,
 
 - automatically **detects profile name changes** and renames the target
   directory accordingly,
-  
+
 - allows **fine-grained customization** of filters and where to store
   downloaded media,
 
-- is free `Open Source <https://github.com/Thammus/instaloader>`__
+- is free `open source <https://github.com/instaloader/instaloader>`__
   software written in Python.
 
+::
 
-Table of Contents
------------------
+    instaloader [--comments] [--geotags] [--stories] [--highlights] [--tagged]
+                [--login YOUR-USERNAME] [--fast-update]
+                profile | "#hashtag" | %location_id |
+                :stories | :feed | :saved
+
+See :ref:`download-pictures-from-instagram` for a detailed introduction on how
+to use Instaloader to download pictures from Instagram.
+
+Instaloader Documentation
+-------------------------
 
 .. toctree::
    :maxdepth: 2
@@ -39,19 +59,34 @@ Table of Contents
    basic-usage
    cli-options
    as-module
+   codesnippets
+   contributing
 
 Useful Links
 ------------
 
-- `Git Repository (on GitHub) <https://github.com/Thammus/instaloader>`__
-- `PyPI Project Page <https://pypi.python.org/pypi/instaloader>`__
-- `Issue Tracker / Bug Tracker <https://github.com/Thammus/instaloader/issues>`__
-- `Version History <https://github.com/Thammus/instaloader/releases>`__
+- `Git Repository (on GitHub) <https://github.com/instaloader/instaloader>`__
+- `PyPI Project Page <https://pypi.org/project/instaloader/>`__
+- `Issue Tracker / Bug Tracker <https://github.com/instaloader/instaloader/issues>`__
+- `Version History <https://github.com/instaloader/instaloader/releases>`__
+
+Disclaimer
+----------
 
 .. include:: ../README.rst
    :start-after: disclaimer-start
    :end-before: disclaimer-end
 
+Contributing
+------------
+
+As an open source project, Instaloader heavily depends on the contributions from
+its community. See :ref:`contributing` for how you may help Instaloader to
+become an even greater tool.
+
+.. include:: contributing.rst
+   :start-after: donations-start
+   :end-before: donations-end
 
 ..
    * :ref:`genindex`

docs/installation.rst

@@ -1,11 +1,38 @@
-Installation
-============
+.. _install:
 
-.. include:: ../README.rst
-   :start-after: installation-start
-   :end-before: installation-end
+Install Instaloader
+===================
 
-If you do not want to use pip, even though it is highly recommended,
-and prefer installing Instaloader manually,
-`Download the Source <https://github.com/Thammus/instaloader/releases>`__,
-extract the Zip or Tarball and execute ``instaloader.py`` from there.
+.. highlight:: none
+
+**To install Instaloader**,
+
+#. Ensure that you have `Python <https://www.python.org/>`__, at least
+   version 3.5 and `pip <https://pypi.python.org/pypi/pip>`__
+   installed
+
+#. Then, install Instaloader using::
+
+      pip3 install instaloader
+
+**To upgrade Instaloader**, do::
+
+   pip3 install --upgrade instaloader
+
+
+**Alternative methods for installing Instaloader:**
+
+- If you do not want to use pip, even though it is highly recommended,
+  and prefer to **install Instaloader manually**,
+  `Download the Source <https://github.com/instaloader/instaloader/releases/latest>`__,
+  extract the Zip or Tarball and execute ``instaloader.py`` from there.
+
+- On **Arch Linux**, you may install Instaloader using the
+  `Instaloader AUR package <https://aur.archlinux.org/packages/instaloader/>`__.
+
+- On **Windows 10**, you may download the standalone executable from the
+  `current release page <https://github.com/instaloader/instaloader/releases/latest>`__.
+
+- To test the most current pre-release or development version of Instaloader::
+
+     pip3 install --pre instaloader

docs/requirements.txt

@@ -1 +1,3 @@
-sphinx-autodoc-typehints
+requests
+sphinx!=1.7.7
+sphinx-bootstrap-theme

docs/sphinx_autodoc_typehints.py

@@ -0,0 +1,251 @@
+# sphinx_autodoc_typehints from Nov 20 2018 (version 1.5.2).
+#  https://github.com/agronholm/sphinx-autodoc-typehints
+# (slightly modified to fix class links, maybe related to agronholm/sphinx-autodoc-typehints#38)
+#
+# This is the MIT license: http://www.opensource.org/licenses/mit-license.php
+#
+# Copyright (c) Alex Grönholm
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy of this
+# software and associated documentation files (the "Software"), to deal in the Software
+# without restriction, including without limitation the rights to use, copy, modify, merge,
+# publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons
+# to whom the Software is furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in all copies or
+# substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
+# INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+# PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE
+# FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
+# OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+# DEALINGS IN THE SOFTWARE.
+#
+
+
+import inspect
+from typing import get_type_hints, TypeVar, Any, AnyStr, Generic, Union
+
+from sphinx.util import logging
+from sphinx.util.inspect import Signature
+
+try:
+    from inspect import unwrap
+except ImportError:
+    def unwrap(func, *, stop=None):
+        """This is the inspect.unwrap() method copied from Python 3.5's standard library."""
+        if stop is None:
+            def _is_wrapper(f):
+                return hasattr(f, '__wrapped__')
+        else:
+            def _is_wrapper(f):
+                return hasattr(f, '__wrapped__') and not stop(f)
+        f = func  # remember the original func for error reporting
+        memo = {id(f)}  # Memoise by id to tolerate non-hashable objects
+        while _is_wrapper(func):
+            func = func.__wrapped__
+            id_func = id(func)
+            if id_func in memo:
+                raise ValueError('wrapper loop when unwrapping {!r}'.format(f))
+            memo.add(id_func)
+        return func
+
+logger = logging.getLogger(__name__)
+
+
+def format_annotation(annotation):
+    if inspect.isclass(annotation) and annotation.__module__ == 'builtins':
+        if annotation.__qualname__ == 'NoneType':
+            return '``None``'
+        else:
+            return ':py:class:`{}`'.format(annotation.__qualname__)
+
+    annotation_cls = annotation if inspect.isclass(annotation) else type(annotation)
+    class_name = None
+    if annotation_cls.__module__ == 'typing':
+        params = None
+        prefix = ':py:class:'
+        module = 'typing'
+        extra = ''
+
+        if inspect.isclass(getattr(annotation, '__origin__', None)):
+            annotation_cls = annotation.__origin__
+            try:
+                if Generic in annotation_cls.mro():
+                    module = annotation_cls.__module__
+            except TypeError:
+                pass  # annotation_cls was either the "type" object or typing.Type
+
+        if annotation is Any:
+            return ':py:data:`~typing.Any`'
+        elif annotation is AnyStr:
+            return ':py:data:`~typing.AnyStr`'
+        elif isinstance(annotation, TypeVar):
+            return '\\%r' % annotation
+        elif (annotation is Union or getattr(annotation, '__origin__', None) is Union or
+              hasattr(annotation, '__union_params__')):
+            prefix = ':py:data:'
+            class_name = 'Union'
+            if hasattr(annotation, '__union_params__'):
+                params = annotation.__union_params__
+            elif hasattr(annotation, '__args__'):
+                params = annotation.__args__
+
+            if params and len(params) == 2 and (hasattr(params[1], '__qualname__') and
+                                                params[1].__qualname__ == 'NoneType'):
+                class_name = 'Optional'
+                params = (params[0],)
+        elif annotation_cls.__qualname__ == 'Tuple' and hasattr(annotation, '__tuple_params__'):
+            params = annotation.__tuple_params__
+            if annotation.__tuple_use_ellipsis__:
+                params += (Ellipsis,)
+        elif annotation_cls.__qualname__ == 'Callable':
+            prefix = ':py:data:'
+            arg_annotations = result_annotation = None
+            if hasattr(annotation, '__result__'):
+                arg_annotations = annotation.__args__
+                result_annotation = annotation.__result__
+            elif getattr(annotation, '__args__', None):
+                arg_annotations = annotation.__args__[:-1]
+                result_annotation = annotation.__args__[-1]
+
+            if arg_annotations in (Ellipsis, (Ellipsis,)):
+                params = [Ellipsis, result_annotation]
+            elif arg_annotations is not None:
+                params = [
+                    '\\[{}]'.format(
+                        ', '.join(format_annotation(param) for param in arg_annotations)),
+                    result_annotation
+                ]
+        elif hasattr(annotation, 'type_var'):
+            # Type alias
+            class_name = annotation.name
+            params = (annotation.type_var,)
+        elif getattr(annotation, '__args__', None) is not None:
+            params = annotation.__args__
+        elif hasattr(annotation, '__parameters__'):
+            params = annotation.__parameters__
+
+        if params:
+            extra = '\\[{}]'.format(', '.join(format_annotation(param) for param in params))
+
+        if not class_name:
+            class_name = annotation_cls.__qualname__.title()
+
+        return '{}`~{}.{}`{}'.format(prefix, module, class_name, extra)
+    elif annotation is Ellipsis:
+        return '...'
+    elif inspect.isclass(annotation) or inspect.isclass(getattr(annotation, '__origin__', None)):
+        if not inspect.isclass(annotation):
+            annotation_cls = annotation.__origin__
+
+        extra = ''
+        if Generic in annotation_cls.mro():
+            params = (getattr(annotation, '__parameters__', None) or
+                      getattr(annotation, '__args__', None))
+            extra = '\\[{}]'.format(', '.join(format_annotation(param) for param in params))
+
+        module = annotation.__module__.split('.')[0]    # hack to 'fix' class linking for Instaloader project
+        return ':py:class:`~{}.{}`{}'.format(module, annotation_cls.__qualname__,
+                                             extra)
+
+    return str(annotation)
+
+
+def process_signature(app, what: str, name: str, obj, options, signature, return_annotation):
+    if not callable(obj):
+        return
+
+    if what in ('class', 'exception'):
+        obj = getattr(obj, '__init__', getattr(obj, '__new__', None))
+
+    if not getattr(obj, '__annotations__', None):
+        return
+
+    obj = unwrap(obj)
+    signature = Signature(obj)
+    parameters = [
+        param.replace(annotation=inspect.Parameter.empty)
+        for param in signature.signature.parameters.values()
+    ]
+
+    if parameters:
+        if what in ('class', 'exception'):
+            del parameters[0]
+        elif what == 'method':
+            outer = inspect.getmodule(obj)
+            for clsname in obj.__qualname__.split('.')[:-1]:
+                outer = getattr(outer, clsname)
+
+            method_name = obj.__name__
+            if method_name.startswith("__") and not method_name.endswith("__"):
+                # If the method starts with double underscore (dunder)
+                # Python applies mangling so we need to prepend the class name.
+                # This doesn't happen if it always ends with double underscore.
+                class_name = obj.__qualname__.split('.')[-2]
+                method_name = "_{c}{m}".format(c=class_name, m=method_name)
+
+            method_object = outer.__dict__[method_name]
+            if not isinstance(method_object, (classmethod, staticmethod)):
+                del parameters[0]
+
+    signature.signature = signature.signature.replace(
+        parameters=parameters,
+        return_annotation=inspect.Signature.empty)
+
+    return signature.format_args().replace('\\', '\\\\'), None
+
+
+def process_docstring(app, what, name, obj, options, lines):
+    if isinstance(obj, property):
+        obj = obj.fget
+
+    if callable(obj):
+        if what in ('class', 'exception'):
+            obj = getattr(obj, '__init__')
+
+        obj = unwrap(obj)
+        try:
+            type_hints = get_type_hints(obj)
+        except (AttributeError, TypeError):
+            # Introspecting a slot wrapper will raise TypeError
+            return
+        except NameError as exc:
+            logger.warning('Cannot resolve forward reference in type annotations of "%s": %s',
+                           name, exc)
+            type_hints = obj.__annotations__
+
+        for argname, annotation in type_hints.items():
+            if argname.endswith('_'):
+                argname = '{}\\_'.format(argname[:-1])
+
+            formatted_annotation = format_annotation(annotation)
+
+            if argname == 'return':
+                if what in ('class', 'exception'):
+                    # Don't add return type None from __init__()
+                    continue
+
+                insert_index = len(lines)
+                for i, line in enumerate(lines):
+                    if line.startswith(':rtype:'):
+                        insert_index = None
+                        break
+                    elif line.startswith(':return:') or line.startswith(':returns:'):
+                        insert_index = i
+
+                if insert_index is not None:
+                    if insert_index == len(lines):
+                        # Ensure that :rtype: doesn't get joined with a paragraph of text, which
+                        # prevents it being interpreted.
+                        lines.append('')
+                        insert_index += 1
+
+                    lines.insert(insert_index, ':rtype: {}'.format(formatted_annotation))
+            else:
+                searchfor = ':param {}:'.format(argname)
+                for i, line in enumerate(lines):
+                    if line.startswith(searchfor):
+                        lines.insert(i, ':type {}: {}'.format(argname, formatted_annotation))
+                        break

instaloader/__init__.py

@@ -0,0 +1,19 @@
+"""Download pictures (or videos) along with their captions and other metadata from Instagram."""
+
+
+__version__ = '4.2rc2'
+
+
+try:
+    # pylint:disable=wrong-import-position
+    import win_unicode_console
+except ImportError:
+    pass
+else:
+    win_unicode_console.enable()
+
+from .exceptions import *
+from .instaloader import Instaloader
+from .instaloadercontext import InstaloaderContext
+from .structures import (Highlight, Post, PostSidecarNode, PostComment, PostLocation, Profile, Story, StoryItem,
+                         load_structure_from_file, save_structure_to_file)

instaloader/__main__.py

@@ -0,0 +1,417 @@
+"""Download pictures (or videos) along with their captions and other metadata from Instagram."""
+
+import ast
+import datetime
+import os
+import sys
+from argparse import ArgumentParser, SUPPRESS
+from typing import List, Optional
+
+from . import (Instaloader, InstaloaderException, InvalidArgumentException, Post, Profile, ProfileNotExistsException,
+               StoryItem, __version__, load_structure_from_file, TwoFactorAuthRequiredException,
+               BadCredentialsException)
+from .instaloader import get_default_session_filename
+from .instaloadercontext import default_user_agent
+
+
+def usage_string():
+    # NOTE: duplicated in README.rst and docs/index.rst
+    argv0 = os.path.basename(sys.argv[0])
+    argv0 = "instaloader" if argv0 == "__main__.py" else argv0
+    return """
+{0} [--comments] [--geotags] [--stories] [--highlights] [--tagged]
+{2:{1}} [--login YOUR-USERNAME] [--fast-update]
+{2:{1}} profile | "#hashtag" | %%location_id | :stories | :feed | :saved
+{0} --help""".format(argv0, len(argv0), '')
+
+
+def filterstr_to_filterfunc(filter_str: str, item_type: type):
+    """Takes an --post-filter=... or --storyitem-filter=... filter
+     specification and makes a filter_func Callable out of it."""
+
+    # The filter_str is parsed, then all names occurring in its AST are replaced by loads to post.<name>. A
+    # function Post->bool is returned which evaluates the filter with the post as 'post' in its namespace.
+
+    class TransformFilterAst(ast.NodeTransformer):
+        def visit_Name(self, node: ast.Name):
+            # pylint:disable=no-self-use
+            if not isinstance(node.ctx, ast.Load):
+                raise InvalidArgumentException("Invalid filter: Modifying variables ({}) not allowed.".format(node.id))
+            if node.id == "datetime":
+                return node
+            if not hasattr(item_type, node.id):
+                raise InvalidArgumentException("Invalid filter: {} not a {} attribute.".format(node.id,
+                                                                                               item_type.__name__))
+            new_node = ast.Attribute(ast.copy_location(ast.Name('item', ast.Load()), node), node.id,
+                                     ast.copy_location(ast.Load(), node))
+            return ast.copy_location(new_node, node)
+
+    input_filename = '<command line filter parameter>'
+    compiled_filter = compile(TransformFilterAst().visit(ast.parse(filter_str, filename=input_filename, mode='eval')),
+                              filename=input_filename, mode='eval')
+
+    def filterfunc(item) -> bool:
+        # pylint:disable=eval-used
+        return bool(eval(compiled_filter, {'item': item, 'datetime': datetime.datetime}))
+
+    return filterfunc
+
+
+def _main(instaloader: Instaloader, targetlist: List[str],
+          username: Optional[str] = None, password: Optional[str] = None,
+          sessionfile: Optional[str] = None,
+          download_profile_pic: bool = True, download_posts=True,
+          download_stories: bool = False, download_highlights: bool = False, download_tagged: bool = False,
+          fast_update: bool = False,
+          max_count: Optional[int] = None, post_filter_str: Optional[str] = None,
+          storyitem_filter_str: Optional[str] = None) -> None:
+    """Download set of profiles, hashtags etc. and handle logging in and session files if desired."""
+    # Parse and generate filter function
+    post_filter = None
+    if post_filter_str is not None:
+        post_filter = filterstr_to_filterfunc(post_filter_str, Post)
+        instaloader.context.log('Only download posts with property "{}".'.format(post_filter_str))
+    storyitem_filter = None
+    if storyitem_filter_str is not None:
+        storyitem_filter = filterstr_to_filterfunc(storyitem_filter_str, StoryItem)
+        instaloader.context.log('Only download storyitems with property "{}".'.format(storyitem_filter_str))
+    # Login, if desired
+    if username is not None:
+        try:
+            instaloader.load_session_from_file(username, sessionfile)
+        except FileNotFoundError as err:
+            if sessionfile is not None:
+                print(err, file=sys.stderr)
+            instaloader.context.log("Session file does not exist yet - Logging in.")
+        if not instaloader.context.is_logged_in or username != instaloader.test_login():
+            if password is not None:
+                try:
+                    instaloader.login(username, password)
+                except TwoFactorAuthRequiredException:
+                    while True:
+                        try:
+                            code = input("Enter 2FA verification code: ")
+                            instaloader.two_factor_login(code)
+                            break
+                        except BadCredentialsException:
+                            pass
+            else:
+                instaloader.interactive_login(username)
+        instaloader.context.log("Logged in as %s." % username)
+    # Try block for KeyboardInterrupt (save session on ^C)
+    profiles = set()
+    anonymous_retry_profiles = set()
+    try:
+        # Generate set of profiles, already downloading non-profile targets
+        for target in targetlist:
+            if (target.endswith('.json') or target.endswith('.json.xz')) and os.path.isfile(target):
+                with instaloader.context.error_catcher(target):
+                    structure = load_structure_from_file(instaloader.context, target)
+                    if isinstance(structure, Post):
+                        if post_filter is not None and not post_filter(structure):
+                            instaloader.context.log("<{} ({}) skipped>".format(structure, target), flush=True)
+                            continue
+                        instaloader.context.log("Downloading {} ({})".format(structure, target))
+                        instaloader.download_post(structure, os.path.dirname(target))
+                    elif isinstance(structure, StoryItem):
+                        if storyitem_filter is not None and not storyitem_filter(structure):
+                            instaloader.context.log("<{} ({}) skipped>".format(structure, target), flush=True)
+                            continue
+                        instaloader.context.log("Attempting to download {} ({})".format(structure, target))
+                        instaloader.download_storyitem(structure, os.path.dirname(target))
+                    elif isinstance(structure, Profile):
+                        raise InvalidArgumentException("Profile JSON are ignored. Pass \"{}\" to download that profile"
+                                                       .format(structure.username))
+                    else:
+                        raise InvalidArgumentException("{} JSON file not supported as target"
+                                                       .format(structure.__class__.__name__))
+                continue
+            # strip '/' characters to be more shell-autocompletion-friendly
+            target = target.rstrip('/')
+            with instaloader.context.error_catcher(target):
+                if target[0] == '@':
+                    instaloader.context.log("Retrieving followees of %s..." % target[1:])
+                    profile = Profile.from_username(instaloader.context, target[1:])
+                    for followee in profile.get_followees():
+                        instaloader.save_profile_id(followee)
+                        profiles.add(followee)
+                elif target[0] == '#':
+                    instaloader.download_hashtag(hashtag=target[1:], max_count=max_count, fast_update=fast_update,
+                                                 post_filter=post_filter)
+                elif target[0] == '-':
+                    instaloader.download_post(Post.from_shortcode(instaloader.context, target[1:]), target)
+                elif target[0] == "%":
+                    instaloader.download_location(location=target[1:], max_count=max_count, fast_update=fast_update,
+                                                  post_filter=post_filter)
+                elif target == ":feed":
+                    instaloader.download_feed_posts(fast_update=fast_update, max_count=max_count,
+                                                    post_filter=post_filter)
+                elif target == ":stories":
+                    instaloader.download_stories(fast_update=fast_update, storyitem_filter=storyitem_filter)
+                elif target == ":saved":
+                    instaloader.download_saved_posts(fast_update=fast_update, max_count=max_count,
+                                                     post_filter=post_filter)
+                else:
+                    try:
+                        profile = instaloader.check_profile_id(target)
+                        if instaloader.context.is_logged_in and profile.has_blocked_viewer:
+                            if download_profile_pic or ((download_posts or download_tagged) and not profile.is_private):
+                                raise ProfileNotExistsException("{} blocked you; But we download her anonymously."
+                                                                .format(target))
+                            else:
+                                instaloader.context.error("{} blocked you.".format(target))
+                        else:
+                            profiles.add(profile)
+                    except ProfileNotExistsException as err:
+                        # Not only our profile.has_blocked_viewer condition raises ProfileNotExistsException,
+                        # check_profile_id() also does, since access to blocked profile may be responded with 404.
+                        if instaloader.context.is_logged_in and (download_profile_pic or download_posts or
+                                                                 download_tagged):
+                            instaloader.context.log(err)
+                            instaloader.context.log("Trying again anonymously, helps in case you are just blocked.")
+                            with instaloader.anonymous_copy() as anonymous_loader:
+                                with instaloader.context.error_catcher():
+                                    anonymous_retry_profiles.add(anonymous_loader.check_profile_id(target))
+                                    instaloader.context.error("Warning: {} will be downloaded anonymously (\"{}\")."
+                                                              .format(target, err))
+                        else:
+                            raise
+        if len(profiles) > 1:
+            instaloader.context.log("Downloading {} profiles: {}".format(len(profiles),
+                                                                         ' '.join([p.username for p in profiles])))
+        instaloader.download_profiles(profiles,
+                                      download_profile_pic, download_posts, download_tagged, download_highlights,
+                                      download_stories, fast_update, post_filter, storyitem_filter)
+        if anonymous_retry_profiles:
+            instaloader.context.log("Downloading anonymously: {}"
+                                    .format(' '.join([p.username for p in anonymous_retry_profiles])))
+            with instaloader.anonymous_copy() as anonymous_loader:
+                anonymous_loader.download_profiles(anonymous_retry_profiles,
+                                                   download_profile_pic, download_posts, download_tagged,
+                                                   fast_update=fast_update, post_filter=post_filter)
+    except KeyboardInterrupt:
+        print("\nInterrupted by user.", file=sys.stderr)
+    # Save session if it is useful
+    if instaloader.context.is_logged_in:
+        instaloader.save_session_to_file(sessionfile)
+    # User might be confused if Instaloader does nothing
+    if not targetlist:
+        if instaloader.context.is_logged_in:
+            # Instaloader did at least save a session file
+            instaloader.context.log("No targets were specified, thus nothing has been downloaded.")
+        else:
+            # Instloader did not do anything
+            instaloader.context.log("usage:" + usage_string())
+
+
+def main():
+    parser = ArgumentParser(description=__doc__, add_help=False, usage=usage_string(),
+                            epilog="Report issues at https://github.com/instaloader/instaloader/issues. "
+                                   "The complete documentation can be found at "
+                                   "https://instaloader.github.io/.",
+                            fromfile_prefix_chars='+')
+
+    g_targets = parser.add_argument_group("What to Download",
+                                          "Specify a list of targets. For each of these, Instaloader creates a folder "
+                                          "and downloads all posts. The following targets are supported:")
+    g_targets.add_argument('profile', nargs='*',
+                           help="Download profile. If an already-downloaded profile has been renamed, Instaloader "
+                                "automatically finds it by its unique ID and renames the folder likewise.")
+    g_targets.add_argument('_at_profile', nargs='*', metavar="@profile",
+                           help="Download all followees of profile. Requires --login. "
+                                "Consider using :feed rather than @yourself.")
+    g_targets.add_argument('_hashtag', nargs='*', metavar='"#hashtag"', help="Download #hashtag.")
+    g_targets.add_argument('_location', nargs='*', metavar='%location_id', help="Download %%location_id.")
+    g_targets.add_argument('_feed', nargs='*', metavar=":feed",
+                           help="Download pictures from your feed. Requires --login.")
+    g_targets.add_argument('_stories', nargs='*', metavar=":stories",
+                           help="Download the stories of your followees. Requires --login.")
+    g_targets.add_argument('_saved', nargs='*', metavar=":saved",
+                           help="Download the posts that you marked as saved. Requires --login.")
+    g_targets.add_argument('_singlepost', nargs='*', metavar="-- -shortcode",
+                           help="Download the post with the given shortcode")
+    g_targets.add_argument('_json', nargs='*', metavar="filename.json[.xz]",
+                           help="Re-Download the given object.")
+    g_targets.add_argument('_fromfile', nargs='*', metavar="+args.txt",
+                           help="Read targets (and options) from given textfile.")
+
+    g_post = parser.add_argument_group("What to Download of each Post")
+
+    g_prof = parser.add_argument_group("What to Download of each Profile")
+
+    g_prof.add_argument('-P', '--profile-pic-only', action='store_true',
+                        help=SUPPRESS)
+    g_prof.add_argument('--no-posts', action='store_true',
+                        help="Do not download regular posts.")
+    g_prof.add_argument('--no-profile-pic', action='store_true',
+                        help='Do not download profile picture.')
+    g_post.add_argument('--no-pictures', action='store_true',
+                        help='Do not download post pictures. Cannot be used together with --fast-update. '
+                             'Implies --no-video-thumbnails, does not imply --no-videos.')
+    g_post.add_argument('-V', '--no-videos', action='store_true',
+                        help='Do not download videos.')
+    g_post.add_argument('--no-video-thumbnails', action='store_true',
+                        help='Do not download thumbnails of videos.')
+    g_post.add_argument('-G', '--geotags', action='store_true',
+                        help='Download geotags when available. Geotags are stored as a '
+                             'text file with the location\'s name and a Google Maps link. '
+                             'This requires an additional request to the Instagram '
+                             'server for each picture, which is why it is disabled by default.')
+    g_post.add_argument('-C', '--comments', action='store_true',
+                        help='Download and update comments for each post. '
+                             'This requires an additional request to the Instagram '
+                             'server for each post, which is why it is disabled by default.')
+    g_post.add_argument('--no-captions', action='store_true',
+                        help='Do not create txt files.')
+    g_post.add_argument('--post-metadata-txt', action='append',
+                        help='Template to write in txt file for each Post.')
+    g_post.add_argument('--storyitem-metadata-txt', action='append',
+                        help='Template to write in txt file for each StoryItem.')
+    g_post.add_argument('--no-metadata-json', action='store_true',
+                        help='Do not create a JSON file containing the metadata of each post.')
+    g_post.add_argument('--metadata-json', action='store_true',
+                        help=SUPPRESS)
+    g_post.add_argument('--no-compress-json', action='store_true',
+                        help='Do not xz compress JSON files, rather create pretty formatted JSONs.')
+    g_prof.add_argument('-s', '--stories', action='store_true',
+                        help='Also download stories of each profile that is downloaded. Requires --login.')
+    g_prof.add_argument('--stories-only', action='store_true',
+                        help=SUPPRESS)
+    g_prof.add_argument('--highlights', action='store_true',
+                        help='Also download highlights of each profile that is downloaded. Requires --login.')
+    g_prof.add_argument('--tagged', action='store_true',
+                        help='Also download posts where each profile is tagged.')
+
+    g_cond = parser.add_argument_group("Which Posts to Download")
+
+    g_cond.add_argument('-F', '--fast-update', action='store_true',
+                        help='For each target, stop when encountering the first already-downloaded picture. This '
+                             'flag is recommended when you use Instaloader to update your personal Instagram archive.')
+
+    g_cond.add_argument('--post-filter', '--only-if', metavar='filter',
+                        help='Expression that, if given, must evaluate to True for each post to be downloaded. Must be '
+                             'a syntactically valid python expression. Variables are evaluated to '
+                             'instaloader.Post attributes. Example: --post-filter=viewer_has_liked.')
+    g_cond.add_argument('--storyitem-filter', metavar='filter',
+                        help='Expression that, if given, must evaluate to True for each storyitem to be downloaded. '
+                             'Must be a syntactically valid python expression. Variables are evaluated to '
+                             'instaloader.StoryItem attributes.')
+
+    g_cond.add_argument('-c', '--count',
+                        help='Do not attempt to download more than COUNT posts. '
+                             'Applies only to #hashtag and :feed.')
+
+    g_login = parser.add_argument_group('Login (Download Private Profiles)',
+                                        'Instaloader can login to Instagram. This allows downloading private profiles. '
+                                        'To login, pass the --login option. Your session cookie (not your password!) '
+                                        'will be saved to a local file to be reused next time you want Instaloader '
+                                        'to login.')
+    g_login.add_argument('-l', '--login', metavar='YOUR-USERNAME',
+                         help='Login name (profile name) for your Instagram account.')
+    g_login.add_argument('-f', '--sessionfile',
+                         help='Path for loading and storing session key file. '
+                              'Defaults to ' + get_default_session_filename("<login_name>"))
+    g_login.add_argument('-p', '--password', metavar='YOUR-PASSWORD',
+                         help='Password for your Instagram account. Without this option, '
+                              'you\'ll be prompted for your password interactively if '
+                              'there is not yet a valid session file.')
+
+    g_how = parser.add_argument_group('How to Download')
+    g_how.add_argument('--dirname-pattern',
+                       help='Name of directory where to store posts. {profile} is replaced by the profile name, '
+                            '{target} is replaced by the target you specified, i.e. either :feed, #hashtag or the '
+                            'profile name. Defaults to \'{target}\'.')
+    g_how.add_argument('--filename-pattern',
+                       help='Prefix of filenames, relative to the directory given with '
+                            '--dirname-pattern. {profile} is replaced by the profile name,'
+                            '{target} is replaced by the target you specified, i.e. either :feed'
+                            '#hashtag or the profile name. Defaults to \'{date_utc}_UTC\'')
+    g_how.add_argument('--user-agent',
+                       help='User Agent to use for HTTP requests. Defaults to \'{}\'.'.format(default_user_agent()))
+    g_how.add_argument('-S', '--no-sleep', action='store_true', help=SUPPRESS)
+    g_how.add_argument('--graphql-rate-limit', type=int, help=SUPPRESS)
+    g_how.add_argument('--max-connection-attempts', metavar='N', type=int, default=3,
+                       help='Maximum number of connection attempts until a request is aborted. Defaults to 3. If a '
+                            'connection fails, it can be manually skipped by hitting CTRL+C. Set this to 0 to retry '
+                            'infinitely.')
+    g_how.add_argument('--commit-mode', action='store_true',
+                       help='Tries to ensure downloaded images avoid corruption in case of unexpected interruption. '
+                       'If the last picture is corrupted, Instaloader will fix the picture the next time it is run. '
+                       'Requires the JSON metadata to be saved.')
+
+    g_misc = parser.add_argument_group('Miscellaneous Options')
+    g_misc.add_argument('-q', '--quiet', action='store_true',
+                        help='Disable user interaction, i.e. do not print messages (except errors) and fail '
+                             'if login credentials are needed but not given. This makes Instaloader suitable as a '
+                             'cron job.')
+    g_misc.add_argument('-h', '--help', action='help', help='Show this help message and exit.')
+    g_misc.add_argument('--version', action='version', help='Show version number and exit.',
+                        version=__version__)
+
+    args = parser.parse_args()
+    try:
+        if args.login is None and (args.stories or args.stories_only):
+            print("--login=USERNAME required to download stories.", file=sys.stderr)
+            args.stories = False
+            if args.stories_only:
+                raise SystemExit(1)
+
+        if ':feed-all' in args.profile or ':feed-liked' in args.profile:
+            raise SystemExit(":feed-all and :feed-liked were removed. Use :feed as target and "
+                             "eventually --post-filter=viewer_has_liked.")
+
+        post_metadata_txt_pattern = '\n'.join(args.post_metadata_txt) if args.post_metadata_txt else None
+        storyitem_metadata_txt_pattern = '\n'.join(args.storyitem_metadata_txt) if args.storyitem_metadata_txt else None
+
+        if args.no_captions:
+            if not (post_metadata_txt_pattern or storyitem_metadata_txt_pattern):
+                post_metadata_txt_pattern = ''
+                storyitem_metadata_txt_pattern = ''
+            else:
+                raise SystemExit("--no-captions and --post-metadata-txt or --storyitem-metadata-txt given; "
+                                 "That contradicts.")
+
+        if args.no_pictures and args.fast_update:
+            raise SystemExit('--no-pictures and --fast-update cannot be used together.')
+
+        # Determine what to download
+        download_profile_pic = not args.no_profile_pic or args.profile_pic_only
+        download_posts = not (args.no_posts or args.stories_only or args.profile_pic_only)
+        download_stories = args.stories or args.stories_only
+
+        if args.commit_mode and args.no_metadata_json:
+            raise SystemExit('--commit-mode requires JSON metadata to be saved.')
+
+        loader = Instaloader(sleep=not args.no_sleep, quiet=args.quiet, user_agent=args.user_agent,
+                             dirname_pattern=args.dirname_pattern, filename_pattern=args.filename_pattern,
+                             download_pictures=not args.no_pictures,
+                             download_videos=not args.no_videos, download_video_thumbnails=not args.no_video_thumbnails,
+                             download_geotags=args.geotags,
+                             download_comments=args.comments, save_metadata=not args.no_metadata_json,
+                             compress_json=not args.no_compress_json,
+                             post_metadata_txt_pattern=post_metadata_txt_pattern,
+                             storyitem_metadata_txt_pattern=storyitem_metadata_txt_pattern,
+                             graphql_rate_limit=args.graphql_rate_limit,
+                             max_connection_attempts=args.max_connection_attempts,
+                             commit_mode=args.commit_mode)
+        _main(loader,
+              args.profile,
+              username=args.login.lower() if args.login is not None else None,
+              password=args.password,
+              sessionfile=args.sessionfile,
+              download_profile_pic=download_profile_pic,
+              download_posts=download_posts,
+              download_stories=download_stories,
+              download_highlights=args.highlights,
+              download_tagged=args.tagged,
+              fast_update=args.fast_update,
+              max_count=int(args.count) if args.count is not None else None,
+              post_filter_str=args.post_filter,
+              storyitem_filter_str=args.storyitem_filter)
+        loader.close()
+    except InstaloaderException as err:
+        raise SystemExit("Fatal error: %s" % err)
+
+
+if __name__ == "__main__":
+    main()

instaloader/exceptions.py

@@ -0,0 +1,57 @@
+class InstaloaderException(Exception):
+    """Base exception for this script.
+
+    :note: This exception should not be raised directly."""
+    pass
+
+
+class QueryReturnedBadRequestException(InstaloaderException):
+    pass
+
+
+class QueryReturnedNotFoundException(InstaloaderException):
+    pass
+
+
+class QueryReturnedForbiddenException(InstaloaderException):
+    pass
+
+
+class ProfileNotExistsException(InstaloaderException):
+    pass
+
+
+class ProfileHasNoPicsException(InstaloaderException):
+    pass
+
+
+class PrivateProfileNotFollowedException(InstaloaderException):
+    pass
+
+
+class LoginRequiredException(InstaloaderException):
+    pass
+
+
+class TwoFactorAuthRequiredException(InstaloaderException):
+    pass
+
+
+class InvalidArgumentException(InstaloaderException):
+    pass
+
+
+class BadResponseException(InstaloaderException):
+    pass
+
+
+class BadCredentialsException(InstaloaderException):
+    pass
+
+
+class ConnectionException(InstaloaderException):
+    pass
+
+
+class TooManyRequestsException(ConnectionException):
+    pass

instaloader/instaloadercontext.py

@@ -0,0 +1,468 @@
+import hashlib
+import json
+import pickle
+import random
+import re
+import shutil
+import sys
+import textwrap
+import time
+import urllib.parse
+from contextlib import contextmanager
+from datetime import datetime, timedelta
+from typing import Any, Callable, Dict, Iterator, Optional
+
+import requests
+import requests.utils
+import urllib3
+
+from .exceptions import *
+
+
+def copy_session(session: requests.Session) -> requests.Session:
+    """Duplicates a requests.Session."""
+    new = requests.Session()
+    new.cookies = requests.utils.cookiejar_from_dict(requests.utils.dict_from_cookiejar(session.cookies))
+    new.headers = session.headers.copy()
+    return new
+
+
+def default_user_agent() -> str:
+    return 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 ' \
+           '(KHTML, like Gecko) Chrome/51.0.2704.79 Safari/537.36'
+
+
+class InstaloaderContext:
+    """Class providing methods for (error) logging and low-level communication with Instagram.
+
+    It is not thought to be instantiated directly, rather :class:`Instaloader` instances maintain a context
+    object.
+
+    For logging, it provides :meth:`log`, :meth:`error`, :meth:`error_catcher`.
+
+    It provides low-level communication routines :meth:`get_json`, :meth:`graphql_query`, :meth:`graphql_node_list`,
+    :meth:`get_and_write_raw` and implements mechanisms for rate controlling and error handling.
+
+    Further, it provides methods for logging in and general session handles, which are used by that routines in
+    class :class:`Instaloader`.
+    """
+
+    def __init__(self, sleep: bool = True, quiet: bool = False, user_agent: Optional[str] = None,
+                 graphql_count_per_slidingwindow: Optional[int] = None, max_connection_attempts: int = 3):
+
+        self.user_agent = user_agent if user_agent is not None else default_user_agent()
+        self._session = self.get_anonymous_session()
+        self.username = None
+        self.sleep = sleep
+        self.quiet = quiet
+        self.max_connection_attempts = max_connection_attempts
+        self._graphql_page_length = 50
+        self.graphql_count_per_slidingwindow = graphql_count_per_slidingwindow or 200
+        self._root_rhx_gis = None
+        self.two_factor_auth_pending = None
+
+        # error log, filled with error() and printed at the end of Instaloader.main()
+        self.error_log = []
+
+        # For the adaption of sleep intervals (rate control)
+        self.query_timestamps = list()
+
+        # Can be set to True for testing, disables supression of InstaloaderContext._error_catcher
+        self.raise_all_errors = False
+
+        # Cache profile from id (mapping from id to Profile)
+        self.profile_id_cache = dict()
+
+    @contextmanager
+    def anonymous_copy(self):
+        session = self._session
+        username = self.username
+        self._session = self.get_anonymous_session()
+        self.username = None
+        try:
+            yield self
+        finally:
+            self._session.close()
+            self.username = username
+            self._session = session
+
+    @property
+    def is_logged_in(self) -> bool:
+        """True, if this Instaloader instance is logged in."""
+        return bool(self.username)
+
+    def log(self, *msg, sep='', end='\n', flush=False):
+        """Log a message to stdout that can be suppressed with --quiet."""
+        if not self.quiet:
+            print(*msg, sep=sep, end=end, flush=flush)
+
+    def error(self, msg, repeat_at_end=True):
+        """Log a non-fatal error message to stderr, which is repeated at program termination.
+
+        :param msg: Message to be printed.
+        :param repeat_at_end: Set to false if the message should be printed, but not repeated at program termination."""
+        print(msg, file=sys.stderr)
+        if repeat_at_end:
+            self.error_log.append(msg)
+
+    def close(self):
+        """Print error log and close session"""
+        if self.error_log and not self.quiet:
+            print("\nErrors occured:", file=sys.stderr)
+            for err in self.error_log:
+                print(err, file=sys.stderr)
+        self._session.close()
+
+    @contextmanager
+    def error_catcher(self, extra_info: Optional[str] = None):
+        """
+        Context manager to catch, print and record InstaloaderExceptions.
+
+        :param extra_info: String to prefix error message with."""
+        try:
+            yield
+        except InstaloaderException as err:
+            if extra_info:
+                self.error('{}: {}'.format(extra_info, err))
+            else:
+                self.error('{}'.format(err))
+            if self.raise_all_errors:
+                raise
+
+    def _default_http_header(self, empty_session_only: bool = False) -> Dict[str, str]:
+        """Returns default HTTP header we use for requests."""
+        header = {'Accept-Encoding': 'gzip, deflate',
+                  'Accept-Language': 'en-US,en;q=0.8',
+                  'Connection': 'keep-alive',
+                  'Content-Length': '0',
+                  'Host': 'www.instagram.com',
+                  'Origin': 'https://www.instagram.com',
+                  'Referer': 'https://www.instagram.com/',
+                  'User-Agent': self.user_agent,
+                  'X-Instagram-AJAX': '1',
+                  'X-Requested-With': 'XMLHttpRequest'}
+        if empty_session_only:
+            del header['Host']
+            del header['Origin']
+            del header['Referer']
+            del header['X-Instagram-AJAX']
+            del header['X-Requested-With']
+        return header
+
+    def get_anonymous_session(self) -> requests.Session:
+        """Returns our default anonymous requests.Session object."""
+        session = requests.Session()
+        session.cookies.update({'sessionid': '', 'mid': '', 'ig_pr': '1',
+                                'ig_vw': '1920', 'csrftoken': '',
+                                's_network': '', 'ds_user_id': ''})
+        session.headers.update(self._default_http_header(empty_session_only=True))
+        return session
+
+    def save_session_to_file(self, sessionfile):
+        """Not meant to be used directly, use :meth:`Instaloader.save_session_to_file`."""
+        pickle.dump(requests.utils.dict_from_cookiejar(self._session.cookies), sessionfile)
+
+    def load_session_from_file(self, username, sessionfile):
+        """Not meant to be used directly, use :meth:`Instaloader.load_session_from_file`."""
+        session = requests.Session()
+        session.cookies = requests.utils.cookiejar_from_dict(pickle.load(sessionfile))
+        session.headers.update(self._default_http_header())
+        session.headers.update({'X-CSRFToken': session.cookies.get_dict()['csrftoken']})
+        self._session = session
+        self.username = username
+
+    def test_login(self) -> Optional[str]:
+        """Not meant to be used directly, use :meth:`Instaloader.test_login`."""
+        data = self.graphql_query("d6f4427fbe92d846298cf93df0b937d3", {})
+        return data["data"]["user"]["username"] if data["data"]["user"] is not None else None
+
+    def login(self, user, passwd):
+        """Not meant to be used directly, use :meth:`Instaloader.login`.
+
+        :raises InvalidArgumentException: If the provided username does not exist.
+        :raises BadCredentialsException: If the provided password is wrong.
+        :raises ConnectionException: If connection to Instagram failed.
+        :raises TwoFactorAuthRequiredException: First step of 2FA login done, now call :meth:`Instaloader.two_factor_login`."""
+        import http.client
+        # pylint:disable=protected-access
+        http.client._MAXHEADERS = 200
+        session = requests.Session()
+        session.cookies.update({'sessionid': '', 'mid': '', 'ig_pr': '1',
+                                'ig_vw': '1920', 'ig_cb': '1', 'csrftoken': '',
+                                's_network': '', 'ds_user_id': ''})
+        session.headers.update(self._default_http_header())
+        session.get('https://www.instagram.com/web/__mid/')
+        csrf_token = session.cookies.get_dict()['csrftoken']
+        session.headers.update({'X-CSRFToken': csrf_token})
+        # Not using self.get_json() here, because we need to access csrftoken cookie
+        self._sleep()
+        login = session.post('https://www.instagram.com/accounts/login/ajax/',
+                             data={'password': passwd, 'username': user}, allow_redirects=True)
+        resp_json = login.json()
+        if resp_json.get('two_factor_required'):
+            two_factor_session = copy_session(session)
+            two_factor_session.headers.update({'X-CSRFToken': csrf_token})
+            two_factor_session.cookies.update({'csrftoken': csrf_token})
+            self.two_factor_auth_pending = (two_factor_session,
+                                            user,
+                                            resp_json['two_factor_info']['two_factor_identifier'])
+            raise TwoFactorAuthRequiredException("Login error: two-factor authentication required.")
+        if resp_json['status'] != 'ok':
+            if 'message' in resp_json:
+                raise ConnectionException("Login error: \"{}\" status, message \"{}\".".format(resp_json['status'],
+                                                                                               resp_json['message']))
+            else:
+                raise ConnectionException("Login error: \"{}\" status.".format(resp_json['status']))
+        if not resp_json['authenticated']:
+            if resp_json['user']:
+                # '{"authenticated": false, "user": true, "status": "ok"}'
+                raise BadCredentialsException('Login error: Wrong password.')
+            else:
+                # '{"authenticated": false, "user": false, "status": "ok"}'
+                # Raise InvalidArgumentException rather than BadCredentialException, because BadCredentialException
+                # triggers re-asking of password in Instaloader.interactive_login(), which makes no sense if the
+                # username is invalid.
+                raise InvalidArgumentException('Login error: User {} does not exist.'.format(user))
+        # '{"authenticated": true, "user": true, "userId": ..., "oneTapPrompt": false, "status": "ok"}'
+        session.headers.update({'X-CSRFToken': login.cookies['csrftoken']})
+        self._session = session
+        self.username = user
+
+    def two_factor_login(self, two_factor_code):
+        """Second step of login if 2FA is enabled.
+        Not meant to be used directly, use :meth:`Instaloader.two_factor_login`.
+
+        :raises InvalidArgumentException: No two-factor authentication pending.
+        :raises BadCredentialsException: 2FA verification code invalid.
+
+        .. versionadded:: 4.2"""
+        if not self.two_factor_auth_pending:
+            raise InvalidArgumentException("No two-factor authentication pending.")
+        (session, user, two_factor_id) = self.two_factor_auth_pending
+
+        login = session.post('https://www.instagram.com/accounts/login/ajax/two_factor/',
+                             data={'username': user, 'verificationCode': two_factor_code, 'identifier': two_factor_id},
+                             allow_redirects=True)
+        resp_json = login.json()
+        if resp_json['status'] != 'ok':
+            if 'message' in resp_json:
+                raise BadCredentialsException("Login error: {}".format(resp_json['message']))
+            else:
+                raise BadCredentialsException("Login error: \"{}\" status.".format(resp_json['status']))
+        session.headers.update({'X-CSRFToken': login.cookies['csrftoken']})
+        self._session = session
+        self.username = user
+        self.two_factor_auth_pending = None
+
+    def _sleep(self):
+        """Sleep a short time if self.sleep is set. Called before each request to instagram.com."""
+        if self.sleep:
+            time.sleep(min(random.expovariate(0.7), 5.0))
+
+    def get_json(self, path: str, params: Dict[str, Any], host: str = 'www.instagram.com',
+                 session: Optional[requests.Session] = None, _attempt=1) -> Dict[str, Any]:
+        """JSON request to Instagram.
+
+        :param path: URL, relative to the given domain which defaults to www.instagram.com/
+        :param params: GET parameters
+        :param host: Domain part of the URL from where to download the requested JSON; defaults to www.instagram.com
+        :param session: Session to use, or None to use self.session
+        :return: Decoded response dictionary
+        :raises QueryReturnedBadRequestException: When the server responds with a 400.
+        :raises QueryReturnedNotFoundException: When the server responds with a 404.
+        :raises ConnectionException: When query repeatedly failed.
+        """
+        def graphql_query_waittime(untracked_queries: bool = False) -> int:
+            sliding_window = 660
+            if not self.query_timestamps:
+                return sliding_window if untracked_queries else 0
+            current_time = time.monotonic()
+            self.query_timestamps = list(filter(lambda t: t > current_time - sliding_window, self.query_timestamps))
+            if len(self.query_timestamps) < self.graphql_count_per_slidingwindow and not untracked_queries:
+                return 0
+            return round(min(self.query_timestamps) + sliding_window - current_time) + 6
+        is_graphql_query = 'query_hash' in params and 'graphql/query' in path
+        # some queries are not rate limited if invoked anonymously:
+        query_not_limited = is_graphql_query and not self.is_logged_in \
+                            and params['query_hash'] in ['9ca88e465c3f866a76f7adee3871bdd8']
+        if is_graphql_query and not query_not_limited:
+            waittime = graphql_query_waittime()
+            if waittime > 0:
+                self.log('\nToo many queries in the last time. Need to wait {} seconds, until {:%H:%M}.'
+                         .format(waittime, datetime.now() + timedelta(seconds=waittime)))
+                time.sleep(waittime)
+            if self.query_timestamps is not None:
+                self.query_timestamps.append(time.monotonic())
+            else:
+                self.query_timestamps = [time.monotonic()]
+        sess = session if session else self._session
+        try:
+            self._sleep()
+            resp = sess.get('https://{0}/{1}'.format(host, path), params=params, allow_redirects=False)
+            while resp.is_redirect:
+                redirect_url = resp.headers['location']
+                self.log('\nHTTP redirect from https://{0}/{1} to {2}'.format(host, path, redirect_url))
+                if redirect_url.startswith('https://{}/'.format(host)):
+                    resp = sess.get(redirect_url if redirect_url.endswith('/') else redirect_url + '/',
+                                    params=params, allow_redirects=False)
+                else:
+                    break
+            if resp.status_code == 400:
+                raise QueryReturnedBadRequestException("400 Bad Request")
+            if resp.status_code == 404:
+                raise QueryReturnedNotFoundException("404 Not Found")
+            if resp.status_code == 429:
+                raise TooManyRequestsException("429 Too Many Requests")
+            if resp.status_code != 200:
+                raise ConnectionException("HTTP error code {}.".format(resp.status_code))
+            is_html_query = not is_graphql_query and not "__a" in params and host == "www.instagram.com"
+            if is_html_query:
+                match = re.search(r'window\._sharedData = (.*);</script>', resp.text)
+                if match is None:
+                    raise ConnectionException("Could not find \"window._sharedData\" in html response.")
+                return json.loads(match.group(1))
+            else:
+                resp_json = resp.json()
+            if 'status' in resp_json and resp_json['status'] != "ok":
+                if 'message' in resp_json:
+                    raise ConnectionException("Returned \"{}\" status, message \"{}\".".format(resp_json['status'],
+                                                                                               resp_json['message']))
+                else:
+                    raise ConnectionException("Returned \"{}\" status.".format(resp_json['status']))
+            return resp_json
+        except (ConnectionException, json.decoder.JSONDecodeError, requests.exceptions.RequestException) as err:
+            error_string = "JSON Query to {}: {}".format(path, err)
+            if _attempt == self.max_connection_attempts:
+                raise ConnectionException(error_string) from err
+            self.error(error_string + " [retrying; skip with ^C]", repeat_at_end=False)
+            text_for_429 = ("HTTP error code 429 was returned because too many queries occured in the last time. "
+                            "Please do not use Instagram in your browser or run multiple instances of Instaloader "
+                            "in parallel.")
+            try:
+                if isinstance(err, TooManyRequestsException):
+                    print(textwrap.fill(text_for_429), file=sys.stderr)
+                    if is_graphql_query:
+                        print("Made {} GraphQL requests.".format(len(self.query_timestamps)), file=sys.stderr)
+                        waittime = graphql_query_waittime(untracked_queries=True)
+                        if waittime > 0:
+                            self.log('The request will be retried in {} seconds, at {:%H:%M}.'
+                                     .format(waittime, datetime.now() + timedelta(seconds=waittime)))
+                            time.sleep(waittime)
+                self._sleep()
+                return self.get_json(path=path, params=params, host=host, session=sess, _attempt=_attempt + 1)
+            except KeyboardInterrupt:
+                self.error("[skipped by user]", repeat_at_end=False)
+                raise ConnectionException(error_string) from err
+
+    def graphql_query(self, query_hash: str, variables: Dict[str, Any],
+                      referer: Optional[str] = None, rhx_gis: Optional[str] = None) -> Dict[str, Any]:
+        """
+        Do a GraphQL Query.
+
+        :param query_hash: Query identifying hash.
+        :param variables: Variables for the Query.
+        :param referer: HTTP Referer, or None.
+        :param rhx_gis: 'rhx_gis' variable as somewhere returned by Instagram, needed to 'sign' request
+        :return: The server's response dictionary.
+        """
+        with copy_session(self._session) as tmpsession:
+            tmpsession.headers.update(self._default_http_header(empty_session_only=True))
+            del tmpsession.headers['Connection']
+            del tmpsession.headers['Content-Length']
+            tmpsession.headers['authority'] = 'www.instagram.com'
+            tmpsession.headers['scheme'] = 'https'
+            tmpsession.headers['accept'] = '*/*'
+            if referer is not None:
+                tmpsession.headers['referer'] = urllib.parse.quote(referer)
+
+            variables_json = json.dumps(variables, separators=(',', ':'))
+
+            if rhx_gis:
+                #self.log("rhx_gis {} query_hash {}".format(rhx_gis, query_hash))
+                values = "{}:{}".format(rhx_gis, variables_json)
+                x_instagram_gis = hashlib.md5(values.encode()).hexdigest()
+                tmpsession.headers['x-instagram-gis'] = x_instagram_gis
+
+            resp_json = self.get_json('graphql/query',
+                                      params={'query_hash': query_hash,
+                                              'variables': variables_json},
+                                      session=tmpsession)
+        if 'status' not in resp_json:
+            self.error("GraphQL response did not contain a \"status\" field.")
+        return resp_json
+
+    def graphql_node_list(self, query_hash: str, query_variables: Dict[str, Any],
+                          query_referer: Optional[str],
+                          edge_extractor: Callable[[Dict[str, Any]], Dict[str, Any]],
+                          rhx_gis: Optional[str] = None,
+                          first_data: Optional[Dict[str, Any]] = None) -> Iterator[Dict[str, Any]]:
+        """Retrieve a list of GraphQL nodes."""
+
+        def _query():
+            query_variables['first'] = self._graphql_page_length
+            try:
+                return edge_extractor(self.graphql_query(query_hash, query_variables, query_referer, rhx_gis))
+            except QueryReturnedBadRequestException:
+                new_page_length = int(self._graphql_page_length / 2)
+                if new_page_length >= 12:
+                    self._graphql_page_length = new_page_length
+                    self.error("HTTP Error 400 (Bad Request) on GraphQL Query. Retrying with shorter page length.",
+                               repeat_at_end=False)
+                    return _query()
+                else:
+                    raise
+
+        if first_data:
+            data = first_data
+        else:
+            data = _query()
+        yield from (edge['node'] for edge in data['edges'])
+        while data['page_info']['has_next_page']:
+            query_variables['after'] = data['page_info']['end_cursor']
+            data = _query()
+            yield from (edge['node'] for edge in data['edges'])
+
+    def get_and_write_raw(self, url: str, filename: str, _attempt=1) -> None:
+        """Downloads raw data.
+
+        :raises QueryReturnedNotFoundException: When the server responds with a 404.
+        :raises QueryReturnedForbiddenException: When the server responds with a 403.
+        :raises ConnectionException: When download repeatedly failed."""
+        try:
+            with self.get_anonymous_session() as anonymous_session:
+                resp = anonymous_session.get(url, stream=True)
+            if resp.status_code == 200:
+                self.log(filename, end=' ', flush=True)
+                with open(filename, 'wb') as file:
+                    resp.raw.decode_content = True
+                    shutil.copyfileobj(resp.raw, file)
+            else:
+                if resp.status_code == 403:
+                    # suspected invalid URL signature
+                    raise QueryReturnedForbiddenException("403 when accessing {}.".format(url))
+                if resp.status_code == 404:
+                    # 404 not worth retrying.
+                    raise QueryReturnedNotFoundException("404 when accessing {}.".format(url))
+                raise ConnectionException("HTTP error code {}.".format(resp.status_code))
+        except (urllib3.exceptions.HTTPError, requests.exceptions.RequestException, ConnectionException) as err:
+            error_string = "URL {}: {}".format(url, err)
+            if _attempt == self.max_connection_attempts:
+                raise ConnectionException(error_string) from err
+            self.error(error_string + " [retrying; skip with ^C]", repeat_at_end=False)
+            try:
+                self._sleep()
+                self.get_and_write_raw(url, filename, _attempt + 1)
+            except KeyboardInterrupt:
+                self.error("[skipped by user]", repeat_at_end=False)
+                raise ConnectionException(error_string) from err
+
+    @property
+    def root_rhx_gis(self) -> Optional[str]:
+        """rhx_gis string returned in the / query."""
+        if self.is_logged_in:
+            # At the moment, rhx_gis seems to be required for anonymous requests only. By returning None when logged
+            # in, we can save the root_rhx_gis lookup query.
+            return None
+        if not self._root_rhx_gis:
+            self._root_rhx_gis = self.get_json('', {})['rhx_gis']
+        return self._root_rhx_gis

instaloader/structures.py

@@ -0,0 +1,988 @@
+import json
+import lzma
+import re
+from base64 import b64decode, b64encode
+from collections import namedtuple
+from datetime import datetime
+from typing import Any, Dict, Iterator, List, Optional, Union
+
+from . import __version__
+from .exceptions import *
+from .instaloadercontext import InstaloaderContext
+
+
+PostSidecarNode = namedtuple('PostSidecarNode', ['is_video', 'display_url', 'video_url'])
+PostSidecarNode.__doc__ = "Item of a Sidecar Post."
+PostSidecarNode.is_video.__doc__ = "Whether this node is a video."
+PostSidecarNode.display_url.__doc__ = "URL of image or video thumbnail."
+PostSidecarNode.video_url.__doc__ = "URL of video or None."
+
+PostComment = namedtuple('PostComment', ['id', 'created_at_utc', 'text', 'owner'])
+PostComment.id.__doc__ = "ID number of comment."
+PostComment.created_at_utc.__doc__ = ":class:`~datetime.datetime` when comment was created (UTC)."
+PostComment.text.__doc__ = "Comment text."
+PostComment.owner.__doc__ = "Owner :class:`Profile` of the comment."
+
+PostLocation = namedtuple('PostLocation', ['id', 'name', 'slug', 'has_public_page', 'lat', 'lng'])
+PostLocation.id.__doc__ = "ID number of location."
+PostLocation.name.__doc__ = "Location name."
+PostLocation.slug.__doc__ = "URL friendly variant of location name."
+PostLocation.has_public_page.__doc__ = "Whether location has a public page."
+PostLocation.lat.__doc__ = "Latitude (:class:`float`)."
+PostLocation.lng.__doc__ = "Longitude (:class:`float`)."
+
+
+class Post:
+    """
+    Structure containing information about an Instagram post.
+
+    Created by methods :meth:`Profile.get_posts`, :meth:`Instaloader.get_hashtag_posts`,
+    :meth:`Instaloader.get_feed_posts` and :meth:`Profile.get_saved_posts`, which return iterators of Posts::
+
+       L = Instaloader()
+       for post in L.get_hashtag_posts(HASHTAG):
+           L.download_post(post, target='#'+HASHTAG)
+
+    Might also be created with::
+
+       post = Post.from_shortcode(L.context, SHORTCODE)
+
+    This class unifies access to the properties associated with a post. It implements == and is
+    hashable.
+
+    :param context: :attr:`Instaloader.context` used for additional queries if neccessary..
+    :param node: Node structure, as returned by Instagram.
+    :param owner_profile: The Profile of the owner, if already known at creation.
+    """
+
+    def __init__(self, context: InstaloaderContext, node: Dict[str, Any],
+                 owner_profile: Optional['Profile'] = None):
+        assert 'shortcode' in node or 'code' in node
+
+        self._context = context
+        self._node = node
+        self._owner_profile = owner_profile
+        self._full_metadata_dict = None
+        self._rhx_gis_str = None
+        self._location = None
+
+    @classmethod
+    def from_shortcode(cls, context: InstaloaderContext, shortcode: str):
+        """Create a post object from a given shortcode"""
+        # pylint:disable=protected-access
+        post = cls(context, {'shortcode': shortcode})
+        post._node = post._full_metadata
+        return post
+
+    @classmethod
+    def from_mediaid(cls, context: InstaloaderContext, mediaid: int):
+        """Create a post object from a given mediaid"""
+        return cls.from_shortcode(context, Post.mediaid_to_shortcode(mediaid))
+
+    @staticmethod
+    def shortcode_to_mediaid(code: str) -> int:
+        if len(code) > 11:
+            raise InvalidArgumentException("Wrong shortcode \"{0}\", unable to convert to mediaid.".format(code))
+        code = 'A' * (12 - len(code)) + code
+        return int.from_bytes(b64decode(code.encode(), b'-_'), 'big')
+
+    @staticmethod
+    def mediaid_to_shortcode(mediaid: int) -> str:
+        if mediaid.bit_length() > 64:
+            raise InvalidArgumentException("Wrong mediaid {0}, unable to convert to shortcode".format(str(mediaid)))
+        return b64encode(mediaid.to_bytes(9, 'big'), b'-_').decode().replace('A', ' ').lstrip().replace(' ', 'A')
+
+    def _asdict(self):
+        node = self._node
+        if self._full_metadata_dict:
+            node.update(self._full_metadata_dict)
+        if self._owner_profile:
+            node['owner'] = self.owner_profile._asdict()
+        if self._location:
+            node['location'] = self._location._asdict()
+        return node
+
+    @property
+    def shortcode(self) -> str:
+        """Media shortcode. URL of the post is instagram.com/p/<shortcode>/."""
+        return self._node['shortcode'] if 'shortcode' in self._node else self._node['code']
+
+    @property
+    def mediaid(self) -> int:
+        """The mediaid is a decimal representation of the media shortcode."""
+        return int(self._node['id'])
+
+    def __repr__(self):
+        return '<Post {}>'.format(self.shortcode)
+
+    def __eq__(self, o: object) -> bool:
+        if isinstance(o, Post):
+            return self.shortcode == o.shortcode
+        return NotImplemented
+
+    def __hash__(self) -> int:
+        return hash(self.shortcode)
+
+    def _obtain_metadata(self):
+        if not self._full_metadata_dict:
+            pic_json = self._context.get_json("p/{0}/".format(self.shortcode), params={})
+            self._full_metadata_dict = pic_json['entry_data']['PostPage'][0]['graphql']['shortcode_media']
+            self._rhx_gis_str = pic_json['rhx_gis']
+
+    @property
+    def _full_metadata(self) -> Dict[str, Any]:
+        self._obtain_metadata()
+        return self._full_metadata_dict
+
+    @property
+    def _rhx_gis(self) -> str:
+        self._obtain_metadata()
+        return self._rhx_gis_str
+
+    def _field(self, *keys) -> Any:
+        """Lookups given fields in _node, and if not found in _full_metadata. Raises KeyError if not found anywhere."""
+        try:
+            d = self._node
+            for key in keys:
+                d = d[key]
+            return d
+        except KeyError:
+            d = self._full_metadata
+            for key in keys:
+                d = d[key]
+            return d
+
+    @property
+    def owner_profile(self) -> 'Profile':
+        """:class:`Profile` instance of the Post's owner."""
+        if not self._owner_profile:
+            if 'username' in self._node['owner']:
+                owner_struct = self._node['owner']
+            else:
+                # Sometimes, the 'owner' structure does not contain the username, only the user's ID.  In that case,
+                # this call triggers downloading of the complete Post metadata struct, where the owner username
+                # is contained.
+                # Note that we cannot use Profile.from_id() here since that would lead us into a recursion.
+                owner_struct = self._full_metadata['owner']
+            self._owner_profile = Profile(self._context, owner_struct)
+        return self._owner_profile
+
+    @property
+    def owner_username(self) -> str:
+        """The Post's lowercase owner name."""
+        return self.owner_profile.username
+
+    @property
+    def owner_id(self) -> int:
+        """The ID of the Post's owner."""
+        return self.owner_profile.userid
+
+    @property
+    def date_local(self) -> datetime:
+        """Timestamp when the post was created (local time zone)."""
+        return datetime.fromtimestamp(self._node["date"] if "date" in self._node else self._node["taken_at_timestamp"])
+
+    @property
+    def date_utc(self) -> datetime:
+        """Timestamp when the post was created (UTC)."""
+        return datetime.utcfromtimestamp(self._node["date"] if "date" in self._node else self._node["taken_at_timestamp"])
+
+    @property
+    def date(self) -> datetime:
+        """Synonym to :attr:`~Post.date_utc`"""
+        return self.date_utc
+
+    @property
+    def profile(self) -> str:
+        """Synonym to :attr:`~Post.owner_username`"""
+        return self.owner_username
+
+    @property
+    def url(self) -> str:
+        """URL of the picture / video thumbnail of the post"""
+        return self._node["display_url"] if "display_url" in self._node else self._node["display_src"]
+
+    @property
+    def typename(self) -> str:
+        """Type of post, GraphImage, GraphVideo or GraphSidecar"""
+        return self._field('__typename')
+
+    def get_sidecar_nodes(self) -> Iterator[PostSidecarNode]:
+        """Sidecar nodes of a Post with typename==GraphSidecar."""
+        if self.typename == 'GraphSidecar':
+            for edge in self._field('edge_sidecar_to_children', 'edges'):
+                node = edge['node']
+                is_video = node['is_video']
+                yield PostSidecarNode(is_video=is_video, display_url=node['display_url'],
+                                      video_url=node['video_url'] if is_video else None)
+
+    @property
+    def caption(self) -> Optional[str]:
+        """Caption."""
+        if "edge_media_to_caption" in self._node and self._node["edge_media_to_caption"]["edges"]:
+            return self._node["edge_media_to_caption"]["edges"][0]["node"]["text"]
+        elif "caption" in self._node:
+            return self._node["caption"]
+
+    @property
+    def caption_hashtags(self) -> List[str]:
+        """List of all lowercased hashtags (without preceeding #) that occur in the Post's caption."""
+        if not self.caption:
+            return []
+        # This regular expression is from jStassen, adjusted to use Python's \w to support Unicode
+        # http://blog.jstassen.com/2016/03/code-regex-for-instagram-username-and-hashtags/
+        hashtag_regex = re.compile(r"(?:#)(\w(?:(?:\w|(?:\.(?!\.))){0,28}(?:\w))?)")
+        return re.findall(hashtag_regex, self.caption.lower())
+
+    @property
+    def caption_mentions(self) -> List[str]:
+        """List of all lowercased profiles that are mentioned in the Post's caption, without preceeding @."""
+        if not self.caption:
+            return []
+        # This regular expression is from jStassen, adjusted to use Python's \w to support Unicode
+        # http://blog.jstassen.com/2016/03/code-regex-for-instagram-username-and-hashtags/
+        mention_regex = re.compile(r"(?:@)(\w(?:(?:\w|(?:\.(?!\.))){0,28}(?:\w))?)")
+        return re.findall(mention_regex, self.caption.lower())
+
+    @property
+    def tagged_users(self) -> List[str]:
+        """List of all lowercased users that are tagged in the Post."""
+        try:
+            return [edge['node']['user']['username'].lower() for edge in self._field('edge_media_to_tagged_user',
+                                                                                     'edges')]
+        except KeyError:
+            return []
+
+    @property
+    def is_video(self) -> bool:
+        """True if the Post is a video."""
+        return self._node['is_video']
+
+    @property
+    def video_url(self) -> Optional[str]:
+        """URL of the video, or None."""
+        if self.is_video:
+            return self._field('video_url')
+
+    @property
+    def viewer_has_liked(self) -> Optional[bool]:
+        """Whether the viewer has liked the post, or None if not logged in."""
+        if not self._context.is_logged_in:
+            return None
+        if 'likes' in self._node and 'viewer_has_liked' in self._node['likes']:
+            return self._node['likes']['viewer_has_liked']
+        return self._field('viewer_has_liked')
+
+    @property
+    def likes(self) -> int:
+        """Likes count"""
+        return self._field('edge_media_preview_like', 'count')
+
+    @property
+    def comments(self) -> int:
+        """Comment count"""
+        return self._field('edge_media_to_comment', 'count')
+
+    def get_comments(self) -> Iterator[PostComment]:
+        """Iterate over all comments of the post.
+
+        Each comment is represented by a PostComment namedtuple with fields text (string), created_at (datetime),
+        id (int) and owner (:class:`Profile`).
+        """
+        def _postcomment(node):
+            return PostComment(id=int(node['id']),
+                               created_at_utc=datetime.utcfromtimestamp(node['created_at']),
+                               text=node['text'],
+                               owner=Profile(self._context, node['owner']))
+        if self.comments == 0:
+            # Avoid doing additional requests if there are no comments
+            return
+        comment_edges = self._field('edge_media_to_comment', 'edges')
+        if self.comments == len(comment_edges):
+            # If the Post's metadata already contains all comments, don't do GraphQL requests to obtain them
+            yield from (_postcomment(comment['node']) for comment in comment_edges)
+            return
+        yield from (_postcomment(node) for node in
+                    self._context.graphql_node_list("33ba35852cb50da46f5b5e889df7d159",
+                                                    {'shortcode': self.shortcode},
+                                                    'https://www.instagram.com/p/' + self.shortcode + '/',
+                                                    lambda d: d['data']['shortcode_media']['edge_media_to_comment'],
+                                                    self._rhx_gis))
+
+    def get_likes(self) -> Iterator['Profile']:
+        """Iterate over all likes of the post. A :class:`Profile` instance of each likee is yielded."""
+        if self.likes == 0:
+            # Avoid doing additional requests if there are no comments
+            return
+        likes_edges = self._field('edge_media_preview_like', 'edges')
+        if self.likes == len(likes_edges):
+            # If the Post's metadata already contains all likes, don't do GraphQL requests to obtain them
+            yield from (Profile(self._context, like['node']) for like in likes_edges)
+            return
+        yield from (Profile(self._context, node) for node in
+                    self._context.graphql_node_list("1cb6ec562846122743b61e492c85999f", {'shortcode': self.shortcode},
+                                                    'https://www.instagram.com/p/' + self.shortcode + '/',
+                                                    lambda d: d['data']['shortcode_media']['edge_liked_by'],
+                                                    self._rhx_gis))
+
+    @property
+    def location(self) -> Optional[PostLocation]:
+        """If the Post has a location, returns PostLocation namedtuple with fields 'id', 'lat' and 'lng' and 'name'."""
+        loc = self._field("location")
+        if self._location or not loc:
+            return self._location
+        location_id = int(loc['id'])
+        if any(k not in loc for k in ('name', 'slug', 'has_public_page', 'lat', 'lng')):
+            loc = self._context.get_json("explore/locations/{0}/".format(location_id),
+                                         params={'__a': 1})['graphql']['location']
+        self._location = PostLocation(location_id, loc['name'], loc['slug'], loc['has_public_page'],
+                                      loc['lat'], loc['lng'])
+        return self._location
+
+
+class Profile:
+    """
+    An Instagram Profile.
+
+    Provides methods for accessing profile properties, as well as :meth:`Profile.get_posts` and for own profile
+    :meth:`Profile.get_saved_posts`.
+
+    Get instances with :meth:`Post.owner_profile`, :meth:`StoryItem.owner_profile`, :meth:`Profile.get_followees`,
+    :meth:`Profile.get_followers` or::
+
+       L = Instaloader()
+       profile = Profile.from_username(L.context, USERNAME)
+
+    Provides :meth:`Profile.get_posts` and for own profile :meth:`Profile.get_saved_posts` to iterate over associated
+    :class:`Post` objects::
+
+       for post in profile.get_posts():
+           L.download_post(post, target=profile.username)
+
+    :meth:`Profile.get_followees` and :meth:`Profile.get_followers`::
+
+       print("{} follows these profiles:".format(profile.username))
+       for followee in profile.get_followees():
+           print(followee.username)
+
+    Also, this class implements == and is hashable.
+    """
+    def __init__(self, context: InstaloaderContext, node: Dict[str, Any]):
+        assert 'username' in node
+        self._context = context
+        self._has_public_story = None
+        self._node = node
+        self._rhx_gis = None
+
+    @classmethod
+    def from_username(cls, context: InstaloaderContext, username: str):
+        """Create a Profile instance from a given username, raise exception if it does not exist.
+
+        See also :meth:`Instaloader.check_profile_id`.
+
+        :param context: :attr:`Instaloader.context`
+        :param username: Username
+        :raises: :class:`ProfileNotExistsException`
+        """
+        # pylint:disable=protected-access
+        profile = cls(context, {'username': username.lower()})
+        profile._obtain_metadata()  # to raise ProfileNotExistException now in case username is invalid
+        return profile
+
+    @classmethod
+    def from_id(cls, context: InstaloaderContext, profile_id: int):
+        """Create a Profile instance from a given userid. If possible, use :meth:`Profile.from_username`
+        or constructor directly rather than this method, since it does many requests.
+
+        :param context: :attr:`Instaloader.context`
+        :param profile_id: userid
+        :raises: :class:`ProfileNotExistsException`, :class:`ProfileHasNoPicsException`
+        """
+        if profile_id in context.profile_id_cache:
+            return context.profile_id_cache[profile_id]
+        data = context.graphql_query("472f257a40c653c64c666ce877d59d2b",
+                                     {'id': str(profile_id), 'first': 1},
+                                     rhx_gis=context.root_rhx_gis)['data']['user']
+        if data:
+            data = data["edge_owner_to_timeline_media"]
+        else:
+            raise ProfileNotExistsException("No profile found, the user may have blocked you (ID: " +
+                                            str(profile_id) + ").")
+        if not data['edges']:
+            if data['count'] == 0:
+                raise ProfileHasNoPicsException("Profile with ID {0}: no pics found.".format(str(profile_id)))
+            else:
+                raise LoginRequiredException("Login required to determine username (ID: " + str(profile_id) + ").")
+        profile = Post(context, data['edges'][0]['node']).owner_profile
+        context.profile_id_cache[profile_id] = profile
+        return profile
+
+    def _asdict(self):
+        json_node = self._node.copy()
+        # remove posts
+        json_node.pop('edge_media_collections', None)
+        json_node.pop('edge_owner_to_timeline_media', None)
+        json_node.pop('edge_saved_media', None)
+        return json_node
+
+    def _obtain_metadata(self):
+        try:
+            if not self._rhx_gis:
+                metadata = self._context.get_json('{}/'.format(self.username), params={})
+                self._node = metadata['entry_data']['ProfilePage'][0]['graphql']['user']
+                self._rhx_gis = metadata['rhx_gis']
+        except (QueryReturnedNotFoundException, KeyError) as err:
+            raise ProfileNotExistsException('Profile {} does not exist.'.format(self.username)) from err
+
+    def _metadata(self, *keys) -> Any:
+        try:
+            d = self._node
+            for key in keys:
+                d = d[key]
+            return d
+        except KeyError:
+            self._obtain_metadata()
+            d = self._node
+            for key in keys:
+                d = d[key]
+            return d
+
+    @property
+    def userid(self) -> int:
+        """User ID"""
+        return int(self._metadata('id'))
+
+    @property
+    def username(self) -> str:
+        """Profile Name"""
+        return self._metadata('username').lower()
+
+    def __repr__(self):
+        return '<Profile {} ({})>'.format(self.username, self.userid)
+
+    def __eq__(self, o: object) -> bool:
+        if isinstance(o, Profile):
+            return self.userid == o.userid
+        return NotImplemented
+
+    def __hash__(self) -> int:
+        return hash(self.userid)
+
+    @property
+    def is_private(self) -> bool:
+        return self._metadata('is_private')
+
+    @property
+    def followed_by_viewer(self) -> bool:
+        return self._metadata('followed_by_viewer')
+
+    @property
+    def mediacount(self) -> int:
+        return self._metadata('edge_owner_to_timeline_media', 'count')
+
+    @property
+    def followers(self) -> int:
+        return self._metadata('edge_followed_by', 'count')
+
+    @property
+    def followees(self) -> int:
+        return self._metadata('edge_follow', 'count')
+
+    @property
+    def external_url(self) -> Optional[str]:
+        return self._metadata('external_url')
+
+    @property
+    def biography(self) -> str:
+        return self._metadata('biography')
+
+    @property
+    def blocked_by_viewer(self) -> bool:
+        return self._metadata('blocked_by_viewer')
+
+    @property
+    def follows_viewer(self) -> bool:
+        return self._metadata('follows_viewer')
+
+    @property
+    def full_name(self) -> str:
+        return self._metadata('full_name')
+
+    @property
+    def has_blocked_viewer(self) -> bool:
+        return self._metadata('has_blocked_viewer')
+
+    @property
+    def has_highlight_reels(self) -> bool:
+        """
+        .. deprecated:: 4.0.6
+           Always returns `True` since :issue:`153`.
+
+        Before broken, this indicated whether the :class:`Profile` had available stories.
+        """
+        return True
+
+    @property
+    def has_public_story(self) -> bool:
+        if not self._has_public_story:
+            self._obtain_metadata()
+            # query not rate limited if invoked anonymously:
+            with self._context.anonymous_copy() as anonymous_context:
+                data = anonymous_context.graphql_query('9ca88e465c3f866a76f7adee3871bdd8',
+                                                       {'user_id': self.userid, 'include_chaining': False,
+                                                        'include_reel': False, 'include_suggested_users': False,
+                                                        'include_logged_out_extras': True,
+                                                        'include_highlight_reels': False},
+                                                       'https://www.instagram.com/{}/'.format(self.username),
+                                                       self._rhx_gis)
+            self._has_public_story = data['data']['user']['has_public_story']
+        return self._has_public_story
+
+    @property
+    def has_viewable_story(self) -> bool:
+        """
+        .. deprecated:: 4.0.6
+
+        Some stories are private. This property determines if the :class:`Profile`
+        has at least one story which can be viewed using the associated :class:`InstaloaderContext`,
+        i.e. the viewer has privileges to view it.
+        """
+        return self.has_public_story or self.followed_by_viewer and self.has_highlight_reels
+
+    @property
+    def has_requested_viewer(self) -> bool:
+        return self._metadata('has_requested_viewer')
+
+    @property
+    def is_verified(self) -> bool:
+        return self._metadata('is_verified')
+
+    @property
+    def requested_by_viewer(self) -> bool:
+        return self._metadata('requested_by_viewer')
+
+    @property
+    def profile_pic_url(self) -> str:
+        """Return URL of profile picture
+
+        .. versionadded:: 4.0.3"""
+        return self._metadata("profile_pic_url_hd")
+
+    def get_profile_pic_url(self) -> str:
+        """.. deprecated:: 4.0.3
+
+	   Use :attr:`profile_pic_url`."""
+        return self.profile_pic_url
+
+    def get_posts(self) -> Iterator[Post]:
+        """Retrieve all posts from a profile."""
+        self._obtain_metadata()
+        yield from (Post(self._context, node, self) for node in
+                    self._context.graphql_node_list("472f257a40c653c64c666ce877d59d2b",
+                                                    {'id': self.userid},
+                                                    'https://www.instagram.com/{0}/'.format(self.username),
+                                                    lambda d: d['data']['user']['edge_owner_to_timeline_media'],
+                                                    self._rhx_gis,
+                                                    self._metadata('edge_owner_to_timeline_media')))
+
+    def get_saved_posts(self) -> Iterator[Post]:
+        """Get Posts that are marked as saved by the user."""
+
+        if self.username != self._context.username:
+            raise LoginRequiredException("--login={} required to get that profile's saved posts.".format(self.username))
+
+        self._obtain_metadata()
+        yield from (Post(self._context, node) for node in
+                    self._context.graphql_node_list("f883d95537fbcd400f466f63d42bd8a1",
+                                                    {'id': self.userid},
+                                                    'https://www.instagram.com/{0}/'.format(self.username),
+                                                    lambda d: d['data']['user']['edge_saved_media'],
+                                                    self._rhx_gis,
+                                                    self._metadata('edge_saved_media')))
+
+    def get_tagged_posts(self) -> Iterator[Post]:
+        """Retrieve all posts where a profile is tagged.
+
+        .. versionadded:: 4.0.7"""
+        self._obtain_metadata()
+        yield from (Post(self._context, node, self if int(node['owner']['id']) == self.userid else None) for node in
+                    self._context.graphql_node_list("e31a871f7301132ceaab56507a66bbb7",
+                                                    {'id': self.userid},
+                                                    'https://www.instagram.com/{0}/'.format(self.username),
+                                                    lambda d: d['data']['user']['edge_user_to_photos_of_you'],
+                                                    self._rhx_gis))
+
+    def get_followers(self) -> Iterator['Profile']:
+        """
+        Retrieve list of followers of given profile.
+        To use this, one needs to be logged in and private profiles has to be followed.
+        """
+        if not self._context.is_logged_in:
+            raise LoginRequiredException("--login required to get a profile's followers.")
+        self._obtain_metadata()
+        yield from (Profile(self._context, node) for node in
+                    self._context.graphql_node_list("37479f2b8209594dde7facb0d904896a",
+                                                    {'id': str(self.userid)},
+                                                    'https://www.instagram.com/' + self.username + '/',
+                                                    lambda d: d['data']['user']['edge_followed_by'],
+                                                    self._rhx_gis))
+
+    def get_followees(self) -> Iterator['Profile']:
+        """
+        Retrieve list of followees (followings) of given profile.
+        To use this, one needs to be logged in and private profiles has to be followed.
+        """
+        if not self._context.is_logged_in:
+            raise LoginRequiredException("--login required to get a profile's followees.")
+        self._obtain_metadata()
+        yield from (Profile(self._context, node) for node in
+                    self._context.graphql_node_list("58712303d941c6855d4e888c5f0cd22f",
+                                                    {'id': str(self.userid)},
+                                                    'https://www.instagram.com/' + self.username + '/',
+                                                    lambda d: d['data']['user']['edge_follow'],
+                                                    self._rhx_gis))
+
+
+class StoryItem:
+    """
+    Structure containing information about a user story item i.e. image or video.
+
+    Created by method :meth:`Story.get_items`. This class implements == and is hashable.
+
+    :param context: :class:`InstaloaderContext` instance used for additional queries if necessary.
+    :param node: Dictionary containing the available information of the story item.
+    :param owner_profile: :class:`Profile` instance representing the story owner.
+    """
+
+    def __init__(self, context: InstaloaderContext, node: Dict[str, Any], owner_profile: Optional[Profile] = None):
+        self._context = context
+        self._node = node
+        self._owner_profile = owner_profile
+
+    def _asdict(self):
+        node = self._node
+        if self._owner_profile:
+            node['owner'] = self._owner_profile._asdict()
+        return node
+
+    @property
+    def mediaid(self) -> int:
+        """The mediaid is a decimal representation of the media shortcode."""
+        return int(self._node['id'])
+
+    @property
+    def shortcode(self) -> str:
+        """Convert :attr:`~StoryItem.mediaid` to a shortcode-like string, allowing ``{shortcode}`` to be used with
+        :option:`--filename-pattern`."""
+        return Post.mediaid_to_shortcode(self.mediaid)
+
+    def __repr__(self):
+        return '<StoryItem {}>'.format(self.mediaid)
+
+    def __eq__(self, o: object) -> bool:
+        if isinstance(o, StoryItem):
+            return self.mediaid == o.mediaid
+        return NotImplemented
+
+    def __hash__(self) -> int:
+        return hash(self.mediaid)
+
+    @property
+    def owner_profile(self) -> Profile:
+        """:class:`Profile` instance of the story item's owner."""
+        if not self._owner_profile:
+            self._owner_profile = Profile.from_id(self._context, self._node['owner']['id'])
+        return self._owner_profile
+
+    @property
+    def owner_username(self) -> str:
+        """The StoryItem owner's lowercase name."""
+        return self.owner_profile.username
+
+    @property
+    def owner_id(self) -> int:
+        """The ID of the StoryItem owner."""
+        return self.owner_profile.userid
+
+    @property
+    def date_local(self) -> datetime:
+        """Timestamp when the StoryItem was created (local time zone)."""
+        return datetime.fromtimestamp(self._node['taken_at_timestamp'])
+
+    @property
+    def date_utc(self) -> datetime:
+        """Timestamp when the StoryItem was created (UTC)."""
+        return datetime.utcfromtimestamp(self._node['taken_at_timestamp'])
+
+    @property
+    def date(self) -> datetime:
+        """Synonym to :attr:`~StoryItem.date_utc`"""
+        return self.date_utc
+
+    @property
+    def profile(self) -> str:
+        """Synonym to :attr:`~StoryItem.owner_username`"""
+        return self.owner_username
+
+    @property
+    def expiring_local(self) -> datetime:
+        """Timestamp when the StoryItem will get unavailable (local time zone)."""
+        return datetime.fromtimestamp(self._node['expiring_at_timestamp'])
+
+    @property
+    def expiring_utc(self) -> datetime:
+        """Timestamp when the StoryItem will get unavailable (UTC)."""
+        return datetime.utcfromtimestamp(self._node['expiring_at_timestamp'])
+
+    @property
+    def url(self) -> str:
+        """URL of the picture / video thumbnail of the StoryItem"""
+        return self._node['display_resources'][-1]['src']
+
+    @property
+    def typename(self) -> str:
+        """Type of post, GraphStoryImage or GraphStoryVideo"""
+        return self._node['__typename']
+
+    @property
+    def is_video(self) -> bool:
+        """True if the StoryItem is a video."""
+        return self._node['is_video']
+
+    @property
+    def video_url(self) -> Optional[str]:
+        """URL of the video, or None."""
+        if self.is_video:
+            return self._node['video_resources'][-1]['src']
+
+
+class Story:
+    """
+    Structure representing a user story with its associated items.
+
+    Provides methods for accessing story properties, as well as :meth:`Story.get_items` to request associated
+    :class:`StoryItem` nodes. Stories are returned by :meth:`Instaloader.get_stories`.
+
+    With a logged-in :class:`Instaloader` instance `L`, you may download all your visible user stories with::
+
+       for story in L.get_stories():
+           # story is a Story object
+           for item in story.get_items():
+               # item is a StoryItem object
+               L.download_storyitem(item, ':stories')
+
+    This class implements == and is hashable.
+
+    :param context: :class:`InstaloaderContext` instance used for additional queries if necessary.
+    :param node: Dictionary containing the available information of the story as returned by Instagram.
+    """
+
+    def __init__(self, context: InstaloaderContext, node: Dict[str, Any]):
+        self._context = context
+        self._node = node
+        self._unique_id = None
+        self._owner_profile = None
+
+    def __repr__(self):
+        return '<Story by {} changed {:%Y-%m-%d_%H-%M-%S_UTC}>'.format(self.owner_username, self.latest_media_utc)
+
+    def __eq__(self, o: object) -> bool:
+        if isinstance(o, Story):
+            return self.unique_id == o.unique_id
+        return NotImplemented
+
+    def __hash__(self) -> int:
+        return hash(self.unique_id)
+
+    @property
+    def unique_id(self) -> str:
+        """
+        This ID only equals amongst :class:`Story` instances which have the same owner and the same set of
+        :class:`StoryItem`. For all other :class:`Story` instances this ID is different.
+        """
+        if not self._unique_id:
+            id_list = [item.mediaid for item in self.get_items()]
+            id_list.sort()
+            self._unique_id = str().join([str(self.owner_id)] + list(map(str, id_list)))
+        return self._unique_id
+
+    @property
+    def last_seen_local(self) -> Optional[datetime]:
+        """Timestamp when the story has last been watched or None (local time zone)."""
+        if self._node['seen']:
+            return datetime.fromtimestamp(self._node['seen'])
+
+    @property
+    def last_seen_utc(self) -> Optional[datetime]:
+        """Timestamp when the story has last been watched or None (UTC)."""
+        if self._node['seen']:
+            return datetime.utcfromtimestamp(self._node['seen'])
+
+    @property
+    def latest_media_local(self) -> datetime:
+        """Timestamp when the last item of the story was created (local time zone)."""
+        return datetime.fromtimestamp(self._node['latest_reel_media'])
+
+    @property
+    def latest_media_utc(self) -> datetime:
+        """Timestamp when the last item of the story was created (UTC)."""
+        return datetime.utcfromtimestamp(self._node['latest_reel_media'])
+
+    @property
+    def itemcount(self) -> int:
+        """Count of items associated with the :class:`Story` instance."""
+        return len(self._node['items'])
+
+    @property
+    def owner_profile(self) -> Profile:
+        """:class:`Profile` instance of the story owner."""
+        if not self._owner_profile:
+            self._owner_profile = Profile(self._context, self._node['user'])
+        return self._owner_profile
+
+    @property
+    def owner_username(self) -> str:
+        """The story owner's lowercase username."""
+        return self.owner_profile.username
+
+    @property
+    def owner_id(self) -> int:
+        """The story owner's ID."""
+        return self.owner_profile.userid
+
+    def get_items(self) -> Iterator[StoryItem]:
+        """Retrieve all items from a story."""
+        yield from (StoryItem(self._context, item, self.owner_profile) for item in reversed(self._node['items']))
+
+
+class Highlight(Story):
+    """
+    Structure representing a user's highlight with its associated story items.
+
+    Provides methods for accessing highlight properties, as well as :meth:`Highlight.get_items` to request associated
+    :class:`StoryItem` nodes. Highlights are returned by :meth:`Instaloader.get_highlights`.
+
+    With a logged-in :class:`Instaloader` instance `L`, you may download all highlights of a :class:`Profile` instance
+    USER with::
+
+       for highlight in L.get_highlights(USER):
+           # highlight is a Highlight object
+           for item in highlight.get_items():
+               # item is a StoryItem object
+               L.download_storyitem(item, '{}/{}'.format(highlight.owner_username, highlight.title))
+
+    This class implements == and is hashable.
+
+    :param context: :class:`InstaloaderContext` instance used for additional queries if necessary.
+    :param node: Dictionary containing the available information of the highlight as returned by Instagram.
+    :param owner: :class:`Profile` instance representing the owner profile of the highlight.
+    """
+
+    def __init__(self, context: InstaloaderContext, node: Dict[str, Any], owner: Optional[Profile] = None):
+        super().__init__(context, node)
+        self._owner_profile = owner
+        self._items = None
+
+    def __repr__(self):
+        return '<Highlight by {}: {}>'.format(self.owner_username, self.title)
+
+    @property
+    def unique_id(self) -> int:
+        """A unique ID identifying this set of highlights."""
+        return int(self._node['id'])
+
+    @property
+    def owner_profile(self) -> Profile:
+        """:class:`Profile` instance of the highlights' owner."""
+        if not self._owner_profile:
+            self._owner_profile = Profile(self._context, self._node['owner'])
+        return self._owner_profile
+
+    @property
+    def title(self) -> str:
+        """The title of these highlights."""
+        return self._node['title']
+
+    @property
+    def cover_url(self) -> str:
+        """URL of the highlights' cover."""
+        return self._node['cover_media']['thumbnail_src']
+
+    @property
+    def cover_cropped_url(self) -> str:
+        """URL of the cropped version of the cover."""
+        return self._node['cover_media_cropped_thumbnail']['url']
+
+    def _fetch_items(self):
+        if not self._items:
+            self._items = self._context.graphql_query("45246d3fe16ccc6577e0bd297a5db1ab",
+                                                      {"reel_ids": [], "tag_names": [], "location_ids": [],
+                                                       "highlight_reel_ids": [str(self.unique_id)],
+                                                       "precomposed_overlay": False})['data']['reels_media'][0]['items']
+
+    @property
+    def itemcount(self) -> int:
+        """Count of items associated with the :class:`Highlight` instance."""
+        self._fetch_items()
+        return len(self._items)
+
+    def get_items(self) -> Iterator[StoryItem]:
+        """Retrieve all associated highlight items."""
+        self._fetch_items()
+        yield from (StoryItem(self._context, item, self.owner_profile) for item in self._items)
+
+
+JsonExportable = Union[Post, Profile, StoryItem]
+
+
+def save_structure_to_file(structure: JsonExportable, filename: str) -> None:
+    """Saves a :class:`Post`, :class:`Profile` or :class:`StoryItem` to a '.json' or '.json.xz' file such that it can
+    later be loaded by :func:`load_structure_from_file`.
+
+    If the specified filename ends in '.xz', the file will be LZMA compressed. Otherwise, a pretty-printed JSON file
+    will be created.
+
+    :param structure: :class:`Post`, :class:`Profile` or :class:`StoryItem`
+    :param filename: Filename, ends in '.json' or '.json.xz'
+    """
+    json_structure = {'node': structure._asdict(),
+                      'instaloader': {'version': __version__, 'node_type': structure.__class__.__name__}}
+    compress = filename.endswith('.xz')
+    if compress:
+        with lzma.open(filename, 'wt', check=lzma.CHECK_NONE) as fp:
+            json.dump(json_structure, fp=fp, separators=(',', ':'))
+    else:
+        with open(filename, 'wt') as fp:
+            json.dump(json_structure, fp=fp, indent=4, sort_keys=True)
+
+
+def load_structure_from_file(context: InstaloaderContext, filename: str) -> JsonExportable:
+    """Loads a :class:`Post`, :class:`Profile` or :class:`StoryItem` from a '.json' or '.json.xz' file that
+    has been saved by :func:`save_structure_to_file`.
+
+    :param context: :attr:`Instaloader.context` linked to the new object, used for additional queries if neccessary.
+    :param filename: Filename, ends in '.json' or '.json.xz'
+    """
+    compressed = filename.endswith('.xz')
+    if compressed:
+        fp = lzma.open(filename, 'rt')
+    else:
+        fp = open(filename, 'rt')
+    json_structure = json.load(fp)
+    fp.close()
+    if 'node' in json_structure and 'instaloader' in json_structure and \
+            'node_type' in json_structure['instaloader']:
+        node_type = json_structure['instaloader']['node_type']
+        if node_type == "Post":
+            return Post(context, json_structure['node'])
+        elif node_type == "Profile":
+            return Profile(context, json_structure['node'])
+        elif node_type == "StoryItem":
+            return StoryItem(context, json_structure['node'])
+        else:
+            raise InvalidArgumentException("{}: Not an Instaloader JSON.".format(filename))
+    elif 'shortcode' in json_structure:
+        # Post JSON created with Instaloader v3
+        return Post.from_shortcode(context, json_structure['shortcode'])
+    else:
+        raise InvalidArgumentException("{}: Not an Instaloader JSON.".format(filename))

setup.py

@@ -11,7 +11,7 @@ SRC = os.path.abspath(os.path.dirname(__file__))
 
 
 def get_version():
-    with open(os.path.join(SRC, 'instaloader.py')) as f:
+    with open(os.path.join(SRC, 'instaloader/__init__.py')) as f:
         for line in f:
             m = re.match("__version__ = '(.*)'", line)
             if m:
@@ -37,8 +37,8 @@ keywords = (['instagram', 'instagram-scraper', 'instagram-client', 'instagram-fe
 setup(
     name='instaloader',
     version=get_version(),
-    py_modules=['instaloader'],
-    url='https://instaloader.readthedocs.io/',
+    packages=['instaloader'],
+    url='https://instaloader.github.io/',
     license='MIT',
     author='Alexander Graf, André Koch-Kramer',
     author_email='mail@agraf.me, koch-kramer@web.de',
@@ -47,17 +47,19 @@ setup(
     long_description=open(os.path.join(SRC, 'README.rst')).read(),
     install_requires=requirements,
     python_requires='>=3.5',
-    entry_points={'console_scripts': ['instaloader=instaloader:main']},
+    entry_points={'console_scripts': ['instaloader=instaloader.__main__:main']},
     zip_safe=True,
     keywords=keywords,
     classifiers=[
         'Development Status :: 5 - Production/Stable',
         'Environment :: Console',
         'Intended Audience :: End Users/Desktop',
+        'Intended Audience :: Developers',
         'License :: OSI Approved :: MIT License',
         'Operating System :: OS Independent',
         'Programming Language :: Python :: 3.5',
         'Programming Language :: Python :: 3.6',
+        'Programming Language :: Python :: 3.7',
         'Programming Language :: Python :: 3 :: Only',
         'Topic :: Internet',
         'Topic :: Multimedia :: Graphics'

test/instaloader_unittests.py

@@ -0,0 +1,187 @@
+"""Unit Tests for Instaloader"""
+
+import os
+import shutil
+import tempfile
+import unittest
+from itertools import islice
+
+import instaloader
+
+PROFILE_WITH_HIGHLIGHTS = 325732271
+PUBLIC_PROFILE = "selenagomez"
+PUBLIC_PROFILE_ID = 460563723
+HASHTAG = "kitten"
+LOCATION = "362629379"
+OWN_USERNAME = "aandergr"
+NORMAL_MAX_COUNT = 2
+PAGING_MAX_COUNT = 15
+PRIVATE_PROFILE = "aandergr"
+
+# Preserve query timestamps (rate control) between tests to not get rate limited
+instaloadercontext_query_timestamps = list()
+
+
+class TestInstaloaderAnonymously(unittest.TestCase):
+
+    def setUp(self):
+        self.dir = tempfile.mkdtemp()
+        print("Testing in {}".format(self.dir))
+        os.chdir(self.dir)
+        self.L = instaloader.Instaloader(download_geotags=True,
+                                         download_comments=True,
+                                         save_metadata=True)
+        self.L.context.raise_all_errors = True
+        self.L.context.query_timestamps = instaloadercontext_query_timestamps
+
+    def tearDown(self):
+        global instaloadercontext_query_timestamps
+        instaloadercontext_query_timestamps = self.L.context.query_timestamps
+        self.L.close()
+        os.chdir('/')
+        print("Removing {}".format(self.dir))
+        shutil.rmtree(self.dir)
+
+    def post_paging_test(self, iterator):
+        previous_post = None
+        for post in islice(iterator, PAGING_MAX_COUNT):
+            print(post)
+            if previous_post:
+                self.assertTrue(post.date_utc < previous_post.date_utc)
+            previous_post = post
+
+    @unittest.SkipTest
+    def test_public_profile_download(self):
+        self.L.download_profile(PUBLIC_PROFILE, profile_pic=False, fast_update=True)
+        self.L.download_profile(PUBLIC_PROFILE, profile_pic=False, fast_update=True)
+
+    def test_public_profile_paging(self):
+        self.post_paging_test(instaloader.Profile.from_username(self.L.context, PUBLIC_PROFILE).get_posts())
+
+    def test_profile_pic_download(self):
+        self.L.download_profile(PUBLIC_PROFILE, profile_pic_only=True)
+
+    def test_hashtag_download(self):
+        self.L.download_hashtag(HASHTAG, NORMAL_MAX_COUNT)
+
+    def test_hashtag_paging(self):
+        for count, post in enumerate(self.L.get_hashtag_posts(HASHTAG)):
+            print(post)
+            if count == PAGING_MAX_COUNT:
+                break
+
+    def test_location_download(self):
+        self.L.download_location(LOCATION, NORMAL_MAX_COUNT)
+
+    def test_location_paging(self):
+        for count, post in enumerate(self.L.get_location_posts(LOCATION)):
+            print(post)
+            if count == PAGING_MAX_COUNT:
+                break
+
+    def test_get_id_by_username(self):
+        self.assertEqual(PUBLIC_PROFILE_ID,
+                         instaloader.Profile.from_username(self.L.context, PUBLIC_PROFILE).userid)
+
+    def test_get_username_by_id(self):
+        self.assertEqual(PUBLIC_PROFILE.lower(),
+                         instaloader.Profile.from_id(self.L.context, PUBLIC_PROFILE_ID).username)
+
+    def test_post_from_mediaid(self):
+        for post in instaloader.Profile.from_username(self.L.context, PUBLIC_PROFILE).get_posts():
+            post2 = instaloader.Post.from_mediaid(self.L.context, post.mediaid)
+            self.assertEqual(post, post2)
+            break
+
+    def test_public_profile_tagged_paging(self):
+        for post in islice(instaloader.Profile.from_username(self.L.context, PUBLIC_PROFILE).get_tagged_posts(),
+                           PAGING_MAX_COUNT):
+            print(post)
+
+
+class TestInstaloaderLoggedIn(TestInstaloaderAnonymously):
+
+    def setUp(self):
+        super().setUp()
+        self.L.load_session_from_file(OWN_USERNAME)
+
+    @unittest.SkipTest
+    def test_stories_download(self):
+        self.L.download_stories()
+
+    @unittest.SkipTest
+    def test_private_profile_download(self):
+        self.L.download_profile(PRIVATE_PROFILE, download_stories=True)
+
+    def test_stories_paging(self):
+        for user_story in self.L.get_stories():
+            print("profile {}.".format(user_story.owner_username))
+            for item in user_story.get_items():
+                print(item)
+
+    def test_highlights_paging(self):
+        for user_highlight in self.L.get_highlights(PROFILE_WITH_HIGHLIGHTS):
+            print("Retrieving {} highlights \"{}\" from profile {}".format(user_highlight.itemcount,
+                                                                           user_highlight.title,
+                                                                           user_highlight.owner_username))
+            for item in user_highlight.get_items():
+                print(item)
+
+    def test_private_profile_paging(self):
+        self.post_paging_test(instaloader.Profile.from_username(self.L.context, PRIVATE_PROFILE).get_posts())
+
+    def test_feed_download(self):
+        self.L.download_feed_posts(NORMAL_MAX_COUNT)
+
+    def test_feed_paging(self):
+        for count, post in enumerate(self.L.get_feed_posts()):
+            print(post)
+            if count == PAGING_MAX_COUNT:
+                break
+
+    def test_saved_download(self):
+        self.L.download_saved_posts(NORMAL_MAX_COUNT)
+
+    def test_saved_paging(self):
+        for count, post in enumerate(instaloader.Profile.from_username(self.L.context, OWN_USERNAME).get_saved_posts()):
+            print(post)
+            if count == PAGING_MAX_COUNT:
+                break
+
+    def test_test_login(self):
+        self.assertEqual(OWN_USERNAME, self.L.test_login())
+
+    def test_followees_and_stories(self):
+        profile = instaloader.Profile.from_username(self.L.context, OWN_USERNAME)
+        for f in islice(profile.get_followees(), PAGING_MAX_COUNT):
+            self.L.download_profile(f.username, profile_pic=False, download_stories_only=True)
+
+    def test_get_followees(self):
+        profile = instaloader.Profile.from_username(self.L.context, OWN_USERNAME)
+        for f in profile.get_followees():
+            print(f.username)
+
+    def test_get_followers(self):
+        profile = instaloader.Profile.from_username(self.L.context, OWN_USERNAME)
+        for f in profile.get_followers():
+            print(f.username)
+
+    def test_get_username_by_id(self):
+        self.assertEqual(PUBLIC_PROFILE.lower(),
+                         instaloader.Profile.from_id(self.L.context, PUBLIC_PROFILE_ID).username)
+
+    def test_get_likes(self):
+        for post in instaloader.Profile.from_username(self.L.context, OWN_USERNAME).get_posts():
+            for like in post.get_likes():
+                print(like.username)
+            break
+
+    def test_explore_paging(self):
+        for count, post in enumerate(self.L.get_explore_posts()):
+            print(post)
+            if count == PAGING_MAX_COUNT:
+                break
+
+
+if __name__ == '__main__':
+    unittest.main()