First Release Candidate for Version 4.0

as-module.rst: List how Post and Profile is created

Describe new filename specification logic

Update filter-posts section

.travis.yml

@@ -8,7 +8,7 @@ install:
  - pip install pylint requests
  - pip install -r docs/requirements.txt
 script:
- - python3 -m pylint -r n -d bad-whitespace,missing-docstring,too-many-arguments,locally-disabled,line-too-long,too-many-public-methods,too-many-lines,too-many-instance-attributes,too-many-locals,too-many-branches,too-many-statements,inconsistent-return-statements instaloader
+ - python3 -m pylint -r n -d bad-whitespace,missing-docstring,too-many-arguments,locally-disabled,line-too-long,too-many-public-methods,too-many-lines,too-many-instance-attributes,too-many-locals,too-many-branches,too-many-statements,inconsistent-return-statements,invalid-name,wildcard-import,unused-wildcard-import instaloader
  - make -C docs html
 deploy:
   - provider: pypi

docs/_static/style.css

@@ -1,3 +1,12 @@
 code {
     color: #222;
-}
+    background-color: #f5f5f5;
+}
+
+dd {
+    padding-left: 8px;
+}
+
+dl.option dd {
+    padding-left: 12px;
+}

docs/_templates/globaltoc.html

@@ -0,0 +1,14 @@
+<li class="dropdown globaltoc-container">
+  <a role="button"
+     id="dLabelGlobalToc"
+     data-toggle="dropdown"
+     data-target="#"
+     href="{{ pathto(master_doc) }}">{{ theme_navbar_site_name }} <b class="caret"></b></a>
+  <ul class="dropdown-menu globaltoc"
+      role="menu"
+      aria-labelledby="dLabelGlobalToc">
+    {{ toctree(maxdepth=theme_globaltoc_depth|toint, collapse=False, includehidden=theme_globaltoc_includehidden|tobool) }}
+    <li role="separator" class="divider"></li>
+    <li class="toctree-l1"><a class="reference internal" href="genindex.html">Index</a></li>
+  </ul>
+</li>

docs/_templates/layout.html

@@ -1,4 +1,11 @@
 {% extends "!layout.html" %}
+{% block htmltitle %}
+  {% if pagename == "index" %}
+  <title>{{ title|striptags|e }}</title>
+  {% else %}
+  <title>{{ title|striptags|e }}{{ titlesuffix }}</title>
+  {% endif %}
+{% endblock %}
 {% block extrahead %}
 {{ super() }}
   <meta name="google-site-verification" content="FNUsDdkOJM9obm9QEdrNBVd9AtAzHzA-aaSMCUrS6C8" />

docs/_templates/navbar.html

@@ -14,7 +14,7 @@
           {% if theme_navbar_title -%}{{ theme_navbar_title|e }}{%- else -%}{{ project|e }}{%- endif -%}
         </a>
         <span class="navbar-text navbar-version pull-left">
-          <a href="https://github.com/instaloader/instaloader/releases/tag/v{{current_release}}" title="Released on {{current_release_date}}.">v<b>{{current_release}}</b></a>
+          <a href="https://github.com/instaloader/instaloader/releases/tag/v{{current_release}}" title="Released on {{current_release_date}}">v<b>{{current_release}}</b></a>
         </span>
       </div>
 
@@ -27,7 +27,7 @@
             {% endif %}
             {% block navbartoc %}
               {% include "globaltoc.html" %}
-              {% if theme_navbar_pagenav %}
+              {% if theme_navbar_pagenav and pagename not in ("index", "genindex", "installation") %}
                 {% include "navbartoc.html" %}
               {% endif %}
             {% endblock %}
@@ -46,7 +46,7 @@
                 <ul class="dropdown-menu" role="menu" aria-labelledby="dLabelUsefulLinks">
                   <li><a href="https://github.com/instaloader/instaloader/issues">Issue Tracker</a></li>
                   <li><a href="https://github.com/instaloader/instaloader/releases">Version History</a></li>
-                  <li><a href="https://pypi.python.org/pypi/instaloader">Instaloader on PyPI</a></li>
+                  <li><a href="https://pypi.org/project/instaloader/">Instaloader on PyPI</a></li>
                   <li><a href="https://github.com/instaloader/instaloader">Instaloader on GitHub</a></li>
                   <li role="separator" class="divider"></li>
                   <li style="padding-left: 20px;"><a class="github-button" href="https://github.com/instaloader/instaloader" data-icon="octicon-star" data-size="large" data-show-count="true" aria-label="Star Instaloader on GitHub">Star</a></li>

docs/as-module.rst

@@ -1,6 +1,6 @@
 .. meta::
    :description:
-      Documentation of Instaloader module, a powerful and easy-to-use
+      Documentation of Instaloader module, a powerful and intuitive
       Python library to download Instagram media and metadata.
 
 Python Module :mod:`instaloader`
@@ -10,9 +10,12 @@ Python Module :mod:`instaloader`
 
 .. highlight:: python
 
-Instaloader exposes its internally used methods as a Python module, making it a
-**powerful and easy-to-use Python API for Instagram**, allowing to further
-customize obtaining media and metadata.
+.. contents::
+  :backlinks: none
+
+Instaloader exposes its internally used methods and structures as a Python
+module, making it a **powerful and intuitive Python API for Instagram**,
+allowing to further customize obtaining media and metadata.
 
 Start with getting an instance of :class:`Instaloader`::
 
@@ -36,40 +39,58 @@ certain source::
         # post is an instance of instaloader.Post
         L.download_post(post, target='#cat')
 
-Besides :func:`Instaloader.get_hashtag_posts`, there is
-:func:`Instaloader.get_feed_posts`, :func:`Instaloader.get_profile_posts` and
-:func:`Instaloader.get_saved_posts`.
-Also, :class:`Post` instances can be created with :func:`Post.from_shortcode`
-and :func:`Post.from_mediaid`.
+:class:`Post` instances can be created with:
 
-Further, information about profiles can be easily obtained. For example, you may
-print a list of all followees and followers of a profile with::
+- :func:`Post.from_shortcode`
+   Use a Post shortcode (part of the Post URL,
+   ``https://www.instagram.com/p/SHORTCODE/``) to create a Post
+   object::
 
-    # Print followees
-    print(PROFILE + " follows these profiles:")
-    for f in L.get_followees(PROFILE):
-        print("\t%s\t%s" % (f['username'], f['full_name']))
+      post = Post.from_shortcode(L.context, SHORTCODE)
 
-    # Print followers
-    print("Followers of " + PROFILE + ":")
-    for f in L.get_followers(PROFILE):
-        print("\t%s\t%s" % (f['username'], f['full_name']))
+- :meth:`Profile.get_posts`
+   All media of a :class:`Profile`.
 
-Then, you may download all pictures of all followees with::
+- :meth:`Profile.get_saved_posts`
+   Media that the user marked as saved (:class:`Profile` must be own profile
+   for this to work)
 
-    for f in L.get_followees(PROFILE):
-        L.download_profile(f['username'])
+- :meth:`Instaloader.get_feed_posts`
+   Media in the user's feed.
 
-Each Instagram profile has its own unique ID which stays unmodified even if a
-user changes his/her username. To get said ID, given the profile's name, you may
-call::
+- :meth:`Instaloader.get_explore_posts`
+   Media that is suggested by Instagram to explore.
 
-    L.get_id_by_username(PROFILE_NAME)
+- :meth:`Instaloader.get_hashtag_posts`
+   Media associated with given hashtag.
+
+With the :class:`Profile` class, Instaloader also makes it easy to access
+metadata of a Profile. :class:`Profile` instances can be created with:
+
+- :meth:`Profile.from_username`::
+
+     profile = Profile.from_username(L.context, USERNAME)
+
+- :meth:`Profile_from_userid`
+   given its User ID (currently requires to be logged in).
+
+- :meth:`Profile.get_followees`
+   Profiles that are followed by given user.
+
+- :meth:`Profile.get_followers`
+   Profiles that follow given user.
+
+- :attr:`Post.owner_profile`, :attr:`Story.owner_profile` and :attr:`StoryItem.owner_profile`
+   Owner profile of particular object.
+
+- :meth:`Post.get_likes`
+   Profiles that liked a given :class:`Post`
+
+- :attr:`PostComment.owner` attribute for comment in :meth:`Post.get_comments`
+   Profile of a Post comment.
 
 A reference of the many methods provided by the :mod:`instaloader` module is
-provided in the remainder of this document. Feel free to direct any issue or
-contribution to
-`Instaloader on Github <https://github.com/instaloader/instaloader>`__.
+provided in the remainder of this document.
 
 ``Instaloader`` (Main Class)
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^
@@ -77,20 +98,53 @@ contribution to
 .. autoclass:: Instaloader
    :no-show-inheritance:
 
-``Post`` Class
-^^^^^^^^^^^^^^
+Instagram Structures
+^^^^^^^^^^^^^^^^^^^^
+
+Posts
+"""""
 
 .. autoclass:: Post
    :no-show-inheritance:
 
-Miscellaneous Functions
-^^^^^^^^^^^^^^^^^^^^^^^
+Additionally, the following trivial structures are defined:
 
-.. autofunction:: shortcode_to_mediaid
+.. autoclass:: PostSidecarNode
+   :no-show-inheritance:
+   :no-members:
 
-.. autofunction:: mediaid_to_shortcode
+.. autoclass:: PostComment
+   :no-show-inheritance:
+   :no-members:
 
-.. autoclass:: Tristate
+.. autoclass:: PostLocation
+   :no-show-inheritance:
+   :no-members:
+
+User Stories
+""""""""""""
+
+.. autoclass:: Story
+   :no-show-inheritance:
+
+.. autoclass:: StoryItem
+   :no-show-inheritance:
+
+Profiles
+""""""""
+
+.. autoclass:: Profile
+   :no-show-inheritance:
+
+Loading and Saving
+""""""""""""""""""
+
+:class:`Post`, :class:`StoryItem` and :class:`Profile` can be saved and loaded
+to/from JSON files.
+
+.. autofunction:: load_structure_from_file
+
+.. autofunction:: save_structure_to_file
 
 Exceptions
 ^^^^^^^^^^
@@ -98,8 +152,12 @@ Exceptions
 .. autoexception:: InstaloaderException
    :no-show-inheritance:
 
+.. autoexception:: QueryReturnedBadRequestException
+
 .. autoexception:: QueryReturnedNotFoundException
 
+.. autoexception:: QueryReturnedForbiddenException
+
 .. autoexception:: ProfileNotExistsException
 
 .. autoexception:: ProfileHasNoPicsException
@@ -115,3 +173,11 @@ Exceptions
 .. autoexception:: BadCredentialsException
 
 .. autoexception:: ConnectionException
+
+.. autoexception:: TooManyRequestsException
+
+``InstaloaderContext`` (Low-level functions)
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+.. autoclass:: InstaloaderContext
+   :no-show-inheritance:

docs/basic-usage.rst

@@ -10,6 +10,9 @@ Download Pictures from Instagram
 
 .. highlight:: none
 
+Here we describe how to use Instaloader to download pictures from Instagram. If
+you do not have Instaloader installed yet, see :ref:`install`.
+
 .. NOTE that Section "Basic Usage" is duplicated in README.rst.
 
 Basic Usage
@@ -55,38 +58,37 @@ What to Download
 
 Instaloader supports the following targets:
 
-``profile``
+- ``profile``
    Public profile, or private profile with :option:`--login`. For each profile
    you download, :option:`--stories` instructs Instaloader to also
    **download the user's stories**.
 
-``"#hashtag"``
+- ``"#hashtag"``
    Posts with a certain **hashtag** (the quotes are usually neccessary),
 
-``:stories``
+- ``:stories``
    The currently-visible **stories** of your followees (requires
    :option:`--login`),
 
-``:feed``
+- ``:feed``
    Your **feed** (requires :option:`--login`),
 
-``:saved``
+- ``:saved``
    Posts which are marked as **saved** (requires :option:`--login`),
 
-``@profile``
+- ``@profile``
    All profiles that are followed by ``profile``, i.e. the *followees* of
    ``profile`` (requires :option:`--login`).
 
 Instaloader goes through all media matching the specified targets and
 downloads the pictures and videos and their captions. You can specify
 
-- :option:`--comments`, to also **download comments** of each post,
+- :option:`--comments`
+   also **download comments** of each post,
 
-- :option:`--geotags`, to **download geotags** of each post and save them as
-  Google Maps link,
-
-- :option:`--metadata-json`, to store further post metadata in a separate JSON
-  file.
+- :option:`--geotags`
+   **download geotags** of each post and save them as
+   Google Maps link,
 
 .. _filename-specification:
 
@@ -103,60 +105,135 @@ pattern, the token ``{target}`` is replaced by the target name, and
 ``{profile}`` is replaced by the owner of the post which is downloaded.
 
 :option:`--filename-pattern` configures the path of the post's files relative
-to the target directory. The default is ``--filename-pattern={date}``.
+to the target directory that is specified with :option:`--dirname-pattern`.
+The default is ``--filename-pattern={date_utc}_UTC``.
 The tokens ``{target}`` and ``{profile}`` are replaced like in the
-dirname pattern. Further, the tokens ``{date}``, ``{date_utc}`` and ``{shortcode}`` are
-defined. Additionally, in case of not downloading stories, the attributes of
-:class:`.Post` can be used, e.g. ``{post.owner_id}`` or ``{post.mediaid}``.
+dirname pattern. The following tokens are defined for usage with
+:option:`--filename-pattern`:
 
-For example, encode the poster's profile name in the filenames with:
+- ``{target}``
+   Target name (as given in Instaloader command line)
 
-::
+- ``{profile}`` (same as ``{owner_username}``)
+   Owner of the Post / StoryItem.
 
-    instaloader --filename-pattern={date}_{profile} "#hashtag"
+- ``{owner_id}``
+   Unique integer ID of owner profile.
 
-The pattern string is formatted with Python's string formatter. This
-gives additional flexibilty for pattern specification. For example,
-`strftime-style formatting options <https://docs.python.org/3/library/datetime.html#strftime-and-strptime-behavior>`__
-are supported for the post's
-timestamp. The default for ``{date}`` is ``{date:%Y-%m-%d_%H-%M-%S}``.
+- ``{shortcode}``
+   Shortcode (identifier string).
+
+- ``{mediaid}``
+   Integer representation of shortcode.
+
+- ``{date_utc}`` (same as ``{date}``)
+   Creation time in UTC timezone.
+   `strftime()-style formatting options <https://docs.python.org/3/library/datetime.html#strftime-and-strptime-behavior>`__
+   are supported as format specifier. The default date format specifier used by
+   Instaloader is::
+
+      {date_utc:%Y-%m-%d_%H-%M-%S}
+
+For example, encode the poster's profile name in the filenames with::
+
+    instaloader --filename-pattern={date_utc}_UTC_{profile} "#hashtag"
+
+As another example, you may instruct Instaloader to store posts in a
+``PROFILE/YEAR/SHORTCODE.jpg`` directory structure::
+
+    instaloader --dirname-pattern={profile} --filename-pattern={date_utc:%Y}/{shortcode} <target> ...
 
 .. _filter-posts:
 
 Filter Posts
 ^^^^^^^^^^^^
 
-The :option:`--only-if` option allows to specify criterias that posts have to
-meet to be downloaded. If not given, all posts are downloaded. It must be a
-boolean Python expression where the variables :attr:`.likes`, :attr:`.comments`,
-:attr:`.viewer_has_liked`, :attr:`.is_video`, and many more are defined.
+.. py:currentmodule:: instaloader
 
-A few examples:
+The options :option:`--post-filter` and :option:`--storyitem-filter`
+allows to specify criterias that posts or story items have to
+meet to be downloaded. If not given, all posts are downloaded.
 
-To **download the pictures from your feed that you have liked**:
+The filter string must be a
+`Python boolean expression <https://docs.python.org/3/reference/expressions.html#boolean-operations>`__
+where the attributes from :class:`Post` or
+:class:`StoryItem` respectively are defined.
 
-::
+Id est, the following attributes can be used with both
+:option:`--post-filter` and :option:`--storyitem-filter`:
 
-    instaloader --login=your_username --only-if=viewer_has_liked :feed
+- :attr:`~Post.is_video` (bool)
+   Post/StoryItem is a video. For example, you may skip videos::
 
-Or you might only want to download **posts that either you liked or were
-liked by many others**:
+      instaloader --post-filter="not is_video" target
 
-::
+- :attr:`~Post.owner_username` (str), :attr:`~Post.owner_id` (int)
+   Owner profile username / userid.
 
-    instaloader --login=your_username --only-if="likes>100 or viewer_has_liked" profile
+- :attr:`~Post.date_utc` (datetime), :attr:`~Post.date_local` (datetime)
+   Creation timestamp. Since :class:`~datetime.datetime` objects can be created
+   inside filter strings, this easily allows filtering by creation date. E.g.::
 
-Or you may **skip videos**:
+      instaloader --post-filter="date_utc <= datetime(2018, 5, 31)" target
 
-::
+As :option:`--post-filter`, the following attributes can be used additionally:
 
-    instaloader --only-if="not is_video" target
+- :attr:`~Post.viewer_has_liked` (bool)
+   Whether user (with :option:`--login`) has liked given post. To download the
+   pictures from your feed that you have liked::
 
-Or you may filter by hashtags that occur in the Post's caption. For
-example, to download posts of kittens that are cute: ::
+      instaloader --login=your_username --post-filter=viewer_has_liked :feed
 
-    instaloader --only-if="'cute' in caption_hashtags" "#kitten"
+- :attr:`~Post.likes` (int), :attr:`~Post.comments` (int)
+   Likes count / Comments count. You might only want to download posts that
+   either you liked or were liked by many others**::
 
-The given string is evaluated as a
-`Python boolean expression <https://docs.python.org/3/reference/expressions.html#boolean-operations>`__,
-where all occuring variables are attributes of the :class:`.Post` class.
+      instaloader --login=your_username --post-filter="likes>100 or viewer_has_liked" profile
+
+- :attr:`~Post.caption_hashtags` (list of str) / :attr:`~Post.caption_mentions` (list of str)
+   ``#hashtags`` or ``@mentions`` (lowercased) in the Post's caption. For example, to
+   download posts of kittens that are cute::
+
+       instaloader --post-filter="'cute' in caption_hashtags" "#kitten"
+
+- :attr:`~Post.tagged_users` (list of str)
+   Lowercased usernames that are tagged in the Post.
+
+For :option:`--storyitem-filter`, the following additional attributes are
+defined:
+
+- :attr:`~StoryItem.expiring_utc` (datetime) / :attr:`~StoryItem.expiring_local` (datetime)
+   Timestamp when StoryItem will get unavailable.
+
+.. _metadata-text-files:
+
+Metadata Text Files
+^^^^^^^^^^^^^^^^^^^
+
+Unless :option:`--no-captions` is given, Instaloader creates a ``.txt`` file
+along with each post where the Post's caption is saved.
+
+You can customize what metadata to save for each Post or StoryItem with
+:option:`--post-metadata-txt` and :option:`--storyitem-metadata-txt`. The
+default is ``--post-metadata-txt={caption}`` and no storyitem metadata txt.
+These strings are formatted similar as in the :ref:`filename-specification` and
+the result is saved in text files, unless it is empty.
+
+Specifying these options multiple times results in output having multiple lines,
+in the order they were given to Instaloader.
+
+The field names are evaluated to :class:`Post` or :class:`StoryItem` attributes,
+and as such, the same fields are supported as in :ref:`filename-specification`
+and :ref:`filter-posts`.
+
+For example, to save the current number of likes for each post, rather than
+the post's caption::
+
+   instaloader --post-metadata-txt="{likes} likes." <target>
+
+Note that with this feature, it is possible to easily and fastly extract
+additional metadata of already-downloaded posts, by reimporting their JSON
+files. Say, you now also want to export the number of comments the Posts had
+when they were downloaded::
+
+   instaloader --post-metadata-txt="{likes} likes, {comments} comments." <target>/*.json.xz

docs/cli-options.rst

@@ -23,7 +23,7 @@ posts along with the pictures's captions and the current **profile picture**
 there. If an already-downloaded profile has been renamed, Instaloader
 automatically **finds it by its unique ID** and renames the folder likewise.
 
-.. option:: --profile-pic-only
+.. option:: --profile-pic-only, -P
 
    Only download profile picture.
 
@@ -31,7 +31,7 @@ automatically **finds it by its unique ID** and renames the folder likewise.
 
    Do not download profile picture.
 
-.. option:: --no-videos
+.. option:: --no-videos, -V
 
    Do not download videos.
 
@@ -39,19 +39,14 @@ automatically **finds it by its unique ID** and renames the folder likewise.
 
    Do not download thumbnails of videos.
 
-.. option:: --geotags
+.. option:: --geotags, -G
 
    **Download geotags** when available. Geotags are stored as a text file with
    the location's name and a Google Maps link. This requires an additional
    request to the Instagram server for each picture, which is why it is disabled
    by default.
 
-.. option:: --no-geotags
-
-   Do not store geotags, even if they can be obtained without any additional
-   request.
-
-.. option:: --comments
+.. option:: --comments, -C
 
    Download and update comments for each post. This requires an additional
    request to the Instagram server for each post, which is why it is disabled by
@@ -59,20 +54,29 @@ automatically **finds it by its unique ID** and renames the folder likewise.
 
 .. option:: --no-captions
 
-   Do not store media captions, although no additional request is needed to
-   obtain them.
+   Do not create txt files.
 
-.. option:: --stories
+.. option:: --post-metadata-txt
+
+   Template to write in txt file for each Post. See :ref:`metadata-text-files`.
+
+.. option:: --storyitem-metadata-txt
+
+   Template to write in txt file for each StoryItem. See
+   :ref:`metadata-text-files`.
+
+.. option:: --stories, -s
 
    Also **download stories** of each profile that is downloaded. Requires
    :option:`--login`.
 
-.. option:: --metadata-json
+.. option:: --no-metadata-json
 
-   Create a JSON file containing the metadata of each post. This does not
-   include comments (see :option:`--comments`) nor geotags (see
-   :option:`--geotags`). The JSON files contain the properties of
-   :class:`instaloader.Post`.
+   Do not create a JSON file containing the metadata of each post.
+
+.. option:: --no-compress-json
+
+   Do not xz compress JSON files, rather create pretty formatted JSONs.
 
 .. option:: --stories-only
 
@@ -80,14 +84,22 @@ automatically **finds it by its unique ID** and renames the folder likewise.
    download stories.  Requires :option:`--login`. Does not imply
    :option:`--no-profile-pic`.
 
-.. option:: --only-if filter
+.. option:: --post-filter filter, --only-if filter
 
    Expression that, if given, must evaluate to True for each post to be
    downloaded.  Must be a syntactically valid Python expression. Variables are
    evaluated to :class:`instaloader.Post` attributes.  Example:
-   ``--only-if=viewer_has_liked``. See :ref:`filter-posts` for more
+   ``--post-filter=viewer_has_liked``. See :ref:`filter-posts` for more
    examples.
 
+.. option:: --storyitem-filter filter
+
+   Expression that, if given, must evaluate to True for each storyitem to be
+   downloaded.  Must be a syntactically valid Python expression. Variables are
+   evaluated to :class:`instaloader.StoryItem` attributes.
+   See :ref:`filter-posts` for more examples.
+
+
 
 When to Stop Downloading
 ^^^^^^^^^^^^^^^^^^^^^^^^
@@ -95,13 +107,13 @@ When to Stop Downloading
 If none of these options are given, Instaloader goes through all pictures
 matching the specified targets.
 
-.. option:: --fast-update
+.. option:: --fast-update, -F
 
    For each target, stop when encountering the first already-downloaded picture.
    This flag is recommended when you use Instaloader to update your personal
    Instagram archive.
 
-.. option:: --count COUNT
+.. option:: --count COUNT, -c
 
    Do not attempt to download more than COUNT posts.  Applies only to
    ``#hashtag`` and ``:feed``.
@@ -115,17 +127,17 @@ profiles. To login, pass the :option:`--login` option. Your session cookie (not
 password!) will be saved to a local file to be reused next time you want
 Instaloader to login.
 
-.. option:: --login YOUR-USERNAME
+.. option:: --login YOUR-USERNAME, -l YOUR-USERNAME
 
    Login name (profile name) for your Instagram account.
 
-.. option:: --sessionfile SESSIONFILE
+.. option:: --sessionfile SESSIONFILE, -f SESSIONFILE
 
    Path for loading and storing session key file.  Defaults to a path within
    your temporary directory, encoding your local username and your Instagram
    profile name.
 
-.. option:: --password YOUR-PASSWORD
+.. option:: --password YOUR-PASSWORD, -p YOUR-PASSWORD
 
    Password for your Instagram account.  Without this option, you'll be prompted
    for your password interactively if there is not yet a valid session file.
@@ -142,15 +154,11 @@ How to Download
 
 .. option:: --filename-pattern FILENAME_PATTERN
 
-   Prefix of filenames. Posts are stored in the directory whose pattern is given
-   with ``--dirname-pattern``.  ``{profile}`` is replaced by the profile name,
+   Prefix of filenames, relative to the directory given with
+   :option:`--dirname-pattern`. ``{profile}`` is replaced by the profile name,
    ``{target}`` is replaced by the target you specified, i.e.  either ``:feed``,
-   ``#hashtag`` or the profile name.  Also, the fields ``{date}``, ``{date_utc}`` and
-   ``{shortcode}`` can be specified.  In case of not downloading stories, the
-   attributes of the :class:`.Post` class can be used in addition, e.g.
-   ``{post.owner_id}`` or ``{post.mediaid}``.
-   Defaults to
-   ``{date:%Y-%m-%d_%H-%M-%S}``.  See :ref:`filename-specification`.
+   ``#hashtag`` or the profile name. Defaults to ``{date_utc}_UTC``.
+   See :ref:`filename-specification` for a list of supported tokens.
 
 .. option:: --user-agent USER_AGENT
 
@@ -166,7 +174,7 @@ How to Download
 Miscellaneous Options
 ^^^^^^^^^^^^^^^^^^^^^
 
-.. option:: --quiet
+.. option:: --quiet, -q
 
    Disable user interaction, i.e. do not print messages (except errors) and fail
    if login credentials are needed but not given. This makes Instaloader

docs/conf.py

@@ -25,6 +25,8 @@ import sphinx_bootstrap_theme
 
 sys.path.insert(0, os.path.abspath('..'))
 
+import docs.sphinx_autodoc_typehints as sphinx_autodoc_typehints
+
 # -- General configuration ------------------------------------------------
 
 # If your documentation needs a minimal Sphinx version, state it here.
@@ -36,7 +38,6 @@ sys.path.insert(0, os.path.abspath('..'))
 # ones.
 extensions = [
     'sphinx.ext.autodoc',
-    'sphinx_autodoc_typehints',
     'sphinx.ext.githubpages',
     'sphinx.ext.intersphinx'
 ]
@@ -212,7 +213,7 @@ html_domain_indices = False
 
 # If false, no index is generated.
 #
-html_use_index = False
+html_use_index = True
 
 # If true, the index is split into individual pages for each letter.
 #
@@ -368,5 +369,8 @@ current_release_date = subprocess.check_output(["git", "log", "-1", "--tags", "-
 
 html_context = {'current_release': current_release, 'current_release_date': current_release_date}
 
+
 def setup(app):
-    app.add_stylesheet("style.css")
+    app.add_stylesheet("style.css")
+    app.connect('autodoc-process-signature', sphinx_autodoc_typehints.process_signature)
+    app.connect('autodoc-process-docstring', sphinx_autodoc_typehints.process_docstring)

docs/index.rst

@@ -5,6 +5,8 @@
       saved media, and their metadata, comments and captions.
       Written in Python.
 
+.. title:: Instaloader — Download Instagram Photos and Metadata
+
 Instaloader
 ===========
 
@@ -43,8 +45,8 @@ With `Python <https://www.python.org/>`__ installed, do::
                 :stories | :feed | :saved
 
 
-Table of Contents
------------------
+Instaloader Documentation
+-------------------------
 
 .. toctree::
    :maxdepth: 2
@@ -59,7 +61,7 @@ Useful Links
 ------------
 
 - `Git Repository (on GitHub) <https://github.com/instaloader/instaloader>`__
-- `PyPI Project Page <https://pypi.python.org/pypi/instaloader>`__
+- `PyPI Project Page <https://pypi.org/project/instaloader/>`__
 - `Issue Tracker / Bug Tracker <https://github.com/instaloader/instaloader/issues>`__
 - `Version History <https://github.com/instaloader/instaloader/releases>`__
 

docs/installation.rst

@@ -1,28 +1,24 @@
-Installation
-============
+.. _install:
+
+Install Instaloader
+===================
 
 .. highlight:: none
 
-Instaloader requires `Python <https://www.python.org/>`__, at least
-version 3.5.  If you have `pip <https://pypi.python.org/pypi/pip>`__
-installed, you may install Instaloader using
+**To install Instaloader**,
 
-::
+#. Ensure that you have `Python <https://www.python.org/>`__, at least
+   version 3.5 and `pip <https://pypi.python.org/pypi/pip>`__
+   installed
 
-    pip3 install instaloader
+#. Then, install Instaloader using::
 
-Alternatively, to get the most current version of Instaloader from our
-`Git repository <https://github.com/instaloader/instaloader>`__:
+      pip3 install instaloader
 
-::
+**To upgrade Instaloader**, do::
 
-    pip3 install git+https://github.com/instaloader/instaloader
+   pip3 install --upgrade instaloader
 
-(pass ``--upgrade`` to upgrade if Instaloader is already installed)
-
-Instaloader requires
-`requests <http://python-requests.org/>`__, which
-will be installed automatically, if it is not already installed.
 
 **Alternative methods for installing Instaloader:**
 
@@ -34,8 +30,10 @@ will be installed automatically, if it is not already installed.
 - On **Arch Linux**, you may install Instaloader using the
   `Instaloader AUR package <https://aur.archlinux.org/packages/instaloader/>`__.
 
-- On **Gentoo Linux**, you may install Instaloader using the
-  `Instaloader Ebuild <https://bugs.gentoo.org/642100>`__.
-
 - On **Windows 10**, you may download the standalone executable from the
   `current release page <https://github.com/instaloader/instaloader/releases/latest>`__.
+
+To get the most current version of Instaloader from our
+`Git repository <https://github.com/instaloader/instaloader>`__::
+
+    pip3 install --upgrade git+https://github.com/instaloader/instaloader

docs/requirements.txt

@@ -1,4 +1,3 @@
 requests
 sphinx
-sphinx-autodoc-typehints
 sphinx-bootstrap-theme

docs/sphinx_autodoc_typehints.py

@@ -0,0 +1,228 @@
+# sphinx_autodoc_typehints from Mar 18 2018.
+#  https://github.com/agronholm/sphinx-autodoc-typehints
+# (slightly modified to fix class links, maybe related to agronholm/sphinx-autodoc-typehints#38)
+#
+# This is the MIT license: http://www.opensource.org/licenses/mit-license.php
+#
+# Copyright (c) Alex Grönholm
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy of this
+# software and associated documentation files (the "Software"), to deal in the Software
+# without restriction, including without limitation the rights to use, copy, modify, merge,
+# publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons
+# to whom the Software is furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in all copies or
+# substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
+# INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+# PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE
+# FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
+# OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+# DEALINGS IN THE SOFTWARE.
+#
+
+
+import inspect
+from typing import get_type_hints, TypeVar, Any, AnyStr, Generic, Union
+
+from sphinx.util.inspect import getargspec
+from sphinx.ext.autodoc import formatargspec
+
+try:
+    from inspect import unwrap
+except ImportError:
+    def unwrap(func, *, stop=None):
+        """This is the inspect.unwrap() method copied from Python 3.5's standard library."""
+        if stop is None:
+            def _is_wrapper(f):
+                return hasattr(f, '__wrapped__')
+        else:
+            def _is_wrapper(f):
+                return hasattr(f, '__wrapped__') and not stop(f)
+        f = func  # remember the original func for error reporting
+        memo = {id(f)}  # Memoise by id to tolerate non-hashable objects
+        while _is_wrapper(func):
+            func = func.__wrapped__
+            id_func = id(func)
+            if id_func in memo:
+                raise ValueError('wrapper loop when unwrapping {!r}'.format(f))
+            memo.add(id_func)
+        return func
+
+
+def format_annotation(annotation):
+    if inspect.isclass(annotation) and annotation.__module__ == 'builtins':
+        if annotation.__qualname__ == 'NoneType':
+            return '``None``'
+        else:
+            return ':py:class:`{}`'.format(annotation.__qualname__)
+
+    annotation_cls = annotation if inspect.isclass(annotation) else type(annotation)
+    class_name = None
+    if annotation_cls.__module__ == 'typing':
+        params = None
+        prefix = ':py:class:'
+        module = 'typing'
+        extra = ''
+
+        if inspect.isclass(getattr(annotation, '__origin__', None)):
+            annotation_cls = annotation.__origin__
+            try:
+                if Generic in annotation_cls.mro():
+                    module = annotation_cls.__module__
+            except TypeError:
+                pass  # annotation_cls was either the "type" object or typing.Type
+
+        if annotation is Any:
+            return ':py:data:`~typing.Any`'
+        elif annotation is AnyStr:
+            return ':py:data:`~typing.AnyStr`'
+        elif isinstance(annotation, TypeVar):
+            return '\\%r' % annotation
+        elif (annotation is Union or getattr(annotation, '__origin__', None) is Union or
+              hasattr(annotation, '__union_params__')):
+            prefix = ':py:data:'
+            class_name = 'Union'
+            if hasattr(annotation, '__union_params__'):
+                params = annotation.__union_params__
+            elif hasattr(annotation, '__args__'):
+                params = annotation.__args__
+
+            if params and len(params) == 2 and (hasattr(params[1], '__qualname__') and
+                                                params[1].__qualname__ == 'NoneType'):
+                class_name = 'Optional'
+                params = (params[0],)
+        elif annotation_cls.__qualname__ == 'Tuple' and hasattr(annotation, '__tuple_params__'):
+            params = annotation.__tuple_params__
+            if annotation.__tuple_use_ellipsis__:
+                params += (Ellipsis,)
+        elif annotation_cls.__qualname__ == 'Callable':
+            prefix = ':py:data:'
+            arg_annotations = result_annotation = None
+            if hasattr(annotation, '__result__'):
+                arg_annotations = annotation.__args__
+                result_annotation = annotation.__result__
+            elif getattr(annotation, '__args__', None):
+                arg_annotations = annotation.__args__[:-1]
+                result_annotation = annotation.__args__[-1]
+
+            if arg_annotations in (Ellipsis, (Ellipsis,)):
+                params = [Ellipsis, result_annotation]
+            elif arg_annotations is not None:
+                params = [
+                    '\\[{}]'.format(
+                        ', '.join(format_annotation(param) for param in arg_annotations)),
+                    result_annotation
+                ]
+        elif hasattr(annotation, 'type_var'):
+            # Type alias
+            class_name = annotation.name
+            params = (annotation.type_var,)
+        elif getattr(annotation, '__args__', None) is not None:
+            params = annotation.__args__
+        elif hasattr(annotation, '__parameters__'):
+            params = annotation.__parameters__
+
+        if params:
+            extra = '\\[{}]'.format(', '.join(format_annotation(param) for param in params))
+
+        if not class_name:
+            class_name = annotation_cls.__qualname__.title()
+
+        return '{}`~{}.{}`{}'.format(prefix, module, class_name, extra)
+    elif annotation is Ellipsis:
+        return '...'
+    elif inspect.isclass(annotation) or inspect.isclass(getattr(annotation, '__origin__', None)):
+        if not inspect.isclass(annotation):
+            annotation_cls = annotation.__origin__
+
+        extra = ''
+        if Generic in annotation_cls.mro():
+            params = (getattr(annotation, '__parameters__', None) or
+                      getattr(annotation, '__args__', None))
+            extra = '\\[{}]'.format(', '.join(format_annotation(param) for param in params))
+
+        module = annotation.__module__.split('.')[0]    # hack to 'fix' class linking for Instaloader project
+        return ':py:class:`~{}.{}`{}'.format(module, annotation_cls.__qualname__,
+                                             extra)
+
+    return str(annotation)
+
+
+def process_signature(app, what: str, name: str, obj, options, signature, return_annotation):
+    if not callable(obj):
+        return
+
+    if what in ('class', 'exception'):
+        obj = getattr(obj, '__init__', getattr(obj, '__new__', None))
+
+    if not getattr(obj, '__annotations__', None):
+        return
+
+    obj = unwrap(obj)
+    try:
+        argspec = getargspec(obj)
+    except (TypeError, ValueError):
+        return
+
+    if argspec.args:
+        if what in ('class', 'exception'):
+            del argspec.args[0]
+        elif what == 'method':
+            outer = inspect.getmodule(obj)
+            for clsname in obj.__qualname__.split('.')[:-1]:
+                outer = getattr(outer, clsname)
+            method_object = outer.__dict__[obj.__name__]
+            if not isinstance(method_object, (classmethod, staticmethod)):
+                del argspec.args[0]
+
+    return formatargspec(obj, *argspec[:-1]), None
+
+
+def process_docstring(app, what, name, obj, options, lines):
+    if isinstance(obj, property):
+        obj = obj.fget
+
+    if callable(obj):
+        if what in ('class', 'exception'):
+            obj = getattr(obj, '__init__')
+
+        obj = unwrap(obj)
+        try:
+            type_hints = get_type_hints(obj)
+        except (AttributeError, TypeError):
+            # Introspecting a slot wrapper will raise TypeError
+            return
+
+        for argname, annotation in type_hints.items():
+            formatted_annotation = format_annotation(annotation)
+
+            if argname == 'return':
+                if what in ('class', 'exception'):
+                    # Don't add return type None from __init__()
+                    continue
+
+                insert_index = len(lines)
+                for i, line in enumerate(lines):
+                    if line.startswith(':rtype:'):
+                        insert_index = None
+                        break
+                    elif line.startswith(':return:') or line.startswith(':returns:'):
+                        insert_index = i
+
+                if insert_index is not None:
+                    if insert_index == len(lines):
+                        # Ensure that :rtype: doesn't get joined with a paragraph of text, which
+                        # prevents it being interpreted.
+                        lines.append('')
+                        insert_index += 1
+
+                    lines.insert(insert_index, ':rtype: {}'.format(formatted_annotation))
+            else:
+                searchfor = ':param {}:'.format(argname)
+                for i, line in enumerate(lines):
+                    if line.startswith(searchfor):
+                        lines.insert(i, ':type {}: {}'.format(argname, formatted_annotation))
+                        break

instaloader/__init__.py

@@ -0,0 +1,19 @@
+"""Download pictures (or videos) along with their captions and other metadata from Instagram."""
+
+
+__version__ = '4.0rc1'
+
+
+try:
+    # pylint:disable=wrong-import-position
+    import win_unicode_console
+except ImportError:
+    pass
+else:
+    win_unicode_console.enable()
+
+from .exceptions import *
+from .instaloader import Instaloader
+from .instaloadercontext import InstaloaderContext
+from .structures import (Post, PostSidecarNode, PostComment, PostLocation, Profile, Story, StoryItem,
+                         load_structure_from_file, save_structure_to_file)

instaloader/__main__.py

@@ -0,0 +1,339 @@
+"""Download pictures (or videos) along with their captions and other metadata from Instagram."""
+
+import ast
+import datetime
+import os
+import sys
+from argparse import ArgumentParser, SUPPRESS
+from typing import List, Optional
+
+from . import (Instaloader, InstaloaderException, InvalidArgumentException, Post, Profile, ProfileNotExistsException,
+               StoryItem, __version__, load_structure_from_file)
+from .instaloader import get_default_session_filename
+from .instaloadercontext import default_user_agent
+
+
+def usage_string():
+    # NOTE: duplicated in README.rst and docs/index.rst
+    argv0 = os.path.basename(sys.argv[0])
+    argv0 = "instaloader" if argv0 == "__main__.py" else argv0
+    return """
+{0} [--comments] [--geotags] [--stories]
+{2:{1}} [--login YOUR-USERNAME] [--fast-update]
+{2:{1}} profile | "#hashtag" | :stories | :feed | :saved
+{0} --help""".format(argv0, len(argv0), '')
+
+
+def filterstr_to_filterfunc(filter_str: str, item_type: type):
+    """Takes an --post-filter=... or --storyitem-filter=... filter
+     specification and makes a filter_func Callable out of it."""
+
+    # The filter_str is parsed, then all names occurring in its AST are replaced by loads to post.<name>. A
+    # function Post->bool is returned which evaluates the filter with the post as 'post' in its namespace.
+
+    class TransformFilterAst(ast.NodeTransformer):
+        def visit_Name(self, node: ast.Name):
+            # pylint:disable=no-self-use
+            if not isinstance(node.ctx, ast.Load):
+                raise InvalidArgumentException("Invalid filter: Modifying variables ({}) not allowed.".format(node.id))
+            if node.id == "datetime":
+                return node
+            if not hasattr(item_type, node.id):
+                raise InvalidArgumentException("Invalid filter: {} not a {} attribute.".format(node.id,
+                                                                                               item_type.__name__))
+            new_node = ast.Attribute(ast.copy_location(ast.Name('item', ast.Load()), node), node.id,
+                                     ast.copy_location(ast.Load(), node))
+            return ast.copy_location(new_node, node)
+
+    input_filename = '<command line filter parameter>'
+    compiled_filter = compile(TransformFilterAst().visit(ast.parse(filter_str, filename=input_filename, mode='eval')),
+                              filename=input_filename, mode='eval')
+
+    def filterfunc(item) -> bool:
+        # pylint:disable=eval-used
+        return bool(eval(compiled_filter, {'item': item, 'datetime': datetime.datetime}))
+
+    return filterfunc
+
+
+def _main(instaloader: Instaloader, targetlist: List[str],
+          username: Optional[str] = None, password: Optional[str] = None,
+          sessionfile: Optional[str] = None, max_count: Optional[int] = None,
+          profile_pic: bool = True, profile_pic_only: bool = False,
+          fast_update: bool = False,
+          stories: bool = False, stories_only: bool = False,
+          post_filter_str: Optional[str] = None,
+          storyitem_filter_str: Optional[str] = None) -> None:
+    """Download set of profiles, hashtags etc. and handle logging in and session files if desired."""
+    # Parse and generate filter function
+    post_filter = None
+    if post_filter_str is not None:
+        post_filter = filterstr_to_filterfunc(post_filter_str, Post)
+        instaloader.context.log('Only download posts with property "{}".'.format(post_filter_str))
+    storyitem_filter = None
+    if storyitem_filter_str is not None:
+        storyitem_filter = filterstr_to_filterfunc(storyitem_filter_str, StoryItem)
+        instaloader.context.log('Only download storyitems with property "{}".'.format(storyitem_filter_str))
+    # Login, if desired
+    if username is not None:
+        try:
+            instaloader.load_session_from_file(username, sessionfile)
+        except FileNotFoundError as err:
+            if sessionfile is not None:
+                print(err, file=sys.stderr)
+            instaloader.context.log("Session file does not exist yet - Logging in.")
+        if not instaloader.context.is_logged_in or username != instaloader.test_login():
+            if password is not None:
+                instaloader.login(username, password)
+            else:
+                instaloader.interactive_login(username)
+        instaloader.context.log("Logged in as %s." % username)
+    # Try block for KeyboardInterrupt (save session on ^C)
+    profiles = set()
+    try:
+        # Generate set of profiles, already downloading non-profile targets
+        for target in targetlist:
+            if (target.endswith('.json') or target.endswith('.json.xz')) and os.path.isfile(target):
+                with instaloader.context.error_catcher(target):
+                    structure = load_structure_from_file(instaloader.context, target)
+                    if isinstance(structure, Post):
+                        if post_filter is not None and not post_filter(structure):
+                            instaloader.context.log("<{} ({}) skipped>".format(structure, target), flush=True)
+                            continue
+                        instaloader.context.log("Downloading {} ({})".format(structure, target))
+                        instaloader.download_post(structure, os.path.dirname(target))
+                    elif isinstance(structure, StoryItem):
+                        if storyitem_filter is not None and not storyitem_filter(structure):
+                            instaloader.context.log("<{} ({}) skipped>".format(structure, target), flush=True)
+                            continue
+                        instaloader.context.log("Attempting to download {} ({})".format(structure, target))
+                        instaloader.download_storyitem(structure, os.path.dirname(target))
+                    elif isinstance(structure, Profile):
+                        raise InvalidArgumentException("Profile JSON are ignored. Pass \"{}\" to download that profile"
+                                                       .format(structure.username))
+                    else:
+                        raise InvalidArgumentException("{} JSON file not supported as target"
+                                                       .format(structure.__class__.__name__))
+                continue
+            # strip '/' characters to be more shell-autocompletion-friendly
+            target = target.rstrip('/')
+            with instaloader.context.error_catcher(target):
+                if target[0] == '@':
+                    instaloader.context.log("Retrieving followees of %s..." % target[1:])
+                    profile = Profile.from_username(instaloader.context, target[1:])
+                    followees = profile.get_followees()
+                    profiles.update([followee.username for followee in followees])
+                elif target[0] == '#':
+                    instaloader.download_hashtag(hashtag=target[1:], max_count=max_count, fast_update=fast_update,
+                                                 post_filter=post_filter)
+                elif target == ":feed":
+                    instaloader.download_feed_posts(fast_update=fast_update, max_count=max_count,
+                                                    post_filter=post_filter)
+                elif target == ":stories":
+                    instaloader.download_stories(fast_update=fast_update, storyitem_filter=storyitem_filter)
+                elif target == ":saved":
+                    instaloader.download_saved_posts(fast_update=fast_update, max_count=max_count,
+                                                     post_filter=post_filter)
+                else:
+                    profiles.add(target)
+        if len(profiles) > 1:
+            instaloader.context.log("Downloading {} profiles: {}".format(len(profiles), ','.join(profiles)))
+        # Iterate through profiles list and download them
+        for target in profiles:
+            with instaloader.context.error_catcher(target):
+                try:
+                    instaloader.download_profile(target, profile_pic, profile_pic_only, fast_update,
+                                                 stories, stories_only, post_filter=post_filter,
+                                                 storyitem_filter=storyitem_filter)
+                except ProfileNotExistsException as err:
+                    if not instaloader.context.is_logged_in:
+                        instaloader.context.log(err)
+                        instaloader.context.log("Trying again anonymously, helps in case you are just blocked.")
+                        with instaloader.anonymous_copy() as anonymous_loader:
+                            with instaloader.context.error_catcher():
+                                anonymous_loader.download_profile(target, profile_pic, profile_pic_only,
+                                                                  fast_update, post_filter=post_filter)
+                    else:
+                        raise
+    except KeyboardInterrupt:
+        print("\nInterrupted by user.", file=sys.stderr)
+    # Save session if it is useful
+    if instaloader.context.is_logged_in:
+        instaloader.save_session_to_file(sessionfile)
+    # User might be confused if Instaloader does nothing
+    if not targetlist:
+        if instaloader.context.is_logged_in:
+            # Instaloader did at least save a session file
+            instaloader.context.log("No targets were specified, thus nothing has been downloaded.")
+        else:
+            # Instloader did not do anything
+            instaloader.context.log("usage:" + usage_string())
+
+
+def main():
+    parser = ArgumentParser(description=__doc__, add_help=False, usage=usage_string(),
+                            epilog="Report issues at https://github.com/instaloader/instaloader/issues. "
+                                   "The complete documentation can be found at "
+                                   "https://instaloader.github.io/.")
+
+    g_what = parser.add_argument_group('What to Download',
+                                       'Specify a list of profiles or #hashtags. For each of these, Instaloader '
+                                       'creates a folder and '
+                                       'downloads all posts along with the pictures\'s '
+                                       'captions and the current profile picture. '
+                                       'If an already-downloaded profile has been renamed, Instaloader automatically '
+                                       'finds it by its unique ID and renames the folder likewise.')
+    g_what.add_argument('profile', nargs='*', metavar='profile|#hashtag',
+                        help='Name of profile or #hashtag to download. '
+                             'Alternatively, if --login is given: @<profile> to download all followees of '
+                             '<profile>; the special targets '
+                             ':feed to download pictures from your feed; '
+                             ':stories to download the stories of your followees; or '
+                             ':saved to download the posts marked as saved.')
+    g_what.add_argument('-P', '--profile-pic-only', action='store_true',
+                        help='Only download profile picture.')
+    g_what.add_argument('--no-profile-pic', action='store_true',
+                        help='Do not download profile picture.')
+    g_what.add_argument('-V', '--no-videos', action='store_true',
+                        help='Do not download videos.')
+    g_what.add_argument('--no-video-thumbnails', action='store_true',
+                        help='Do not download thumbnails of videos.')
+    g_what.add_argument('-G', '--geotags', action='store_true',
+                        help='Download geotags when available. Geotags are stored as a '
+                             'text file with the location\'s name and a Google Maps link. '
+                             'This requires an additional request to the Instagram '
+                             'server for each picture, which is why it is disabled by default.')
+    g_what.add_argument('-C', '--comments', action='store_true',
+                        help='Download and update comments for each post. '
+                             'This requires an additional request to the Instagram '
+                             'server for each post, which is why it is disabled by default.')
+    g_what.add_argument('--no-captions', action='store_true',
+                        help='Do not create txt files.')
+    g_what.add_argument('--post-metadata-txt', action='append',
+                        help='Template to write in txt file for each Post.')
+    g_what.add_argument('--storyitem-metadata-txt', action='append',
+                        help='Template to write in txt file for each StoryItem.')
+    g_what.add_argument('--no-metadata-json', action='store_true',
+                        help='Do not create a JSON file containing the metadata of each post.')
+    g_what.add_argument('--metadata-json', action='store_true',
+                        help=SUPPRESS)
+    g_what.add_argument('--no-compress-json', action='store_true',
+                        help='Do not xz compress JSON files, rather create pretty formatted JSONs.')
+    g_what.add_argument('-s', '--stories', action='store_true',
+                        help='Also download stories of each profile that is downloaded. Requires --login.')
+    g_what.add_argument('--stories-only', action='store_true',
+                        help='Rather than downloading regular posts of each specified profile, only download '
+                             'stories. Requires --login. Does not imply --no-profile-pic.')
+    g_what.add_argument('--post-filter', '--only-if', metavar='filter',
+                        help='Expression that, if given, must evaluate to True for each post to be downloaded. Must be '
+                             'a syntactically valid python expression. Variables are evaluated to '
+                             'instaloader.Post attributes. Example: --post-filter=viewer_has_liked.')
+    g_what.add_argument('--storyitem-filter', metavar='filter',
+                        help='Expression that, if given, must evaluate to True for each storyitem to be downloaded. '
+                             'Must be a syntactically valid python expression. Variables are evaluated to '
+                             'instaloader.StoryItem attributes.')
+
+    g_stop = parser.add_argument_group('When to Stop Downloading',
+                                       'If none of these options are given, Instaloader goes through all pictures '
+                                       'matching the specified targets.')
+    g_stop.add_argument('-F', '--fast-update', action='store_true',
+                        help='For each target, stop when encountering the first already-downloaded picture. This '
+                             'flag is recommended when you use Instaloader to update your personal Instagram archive.')
+    g_stop.add_argument('-c', '--count',
+                        help='Do not attempt to download more than COUNT posts. '
+                             'Applies only to #hashtag and :feed.')
+
+    g_login = parser.add_argument_group('Login (Download Private Profiles)',
+                                        'Instaloader can login to Instagram. This allows downloading private profiles. '
+                                        'To login, pass the --login option. Your session cookie (not your password!) '
+                                        'will be saved to a local file to be reused next time you want Instaloader '
+                                        'to login.')
+    g_login.add_argument('-l', '--login', metavar='YOUR-USERNAME',
+                         help='Login name (profile name) for your Instagram account.')
+    g_login.add_argument('-f', '--sessionfile',
+                         help='Path for loading and storing session key file. '
+                              'Defaults to ' + get_default_session_filename("<login_name>"))
+    g_login.add_argument('-p', '--password', metavar='YOUR-PASSWORD',
+                         help='Password for your Instagram account. Without this option, '
+                              'you\'ll be prompted for your password interactively if '
+                              'there is not yet a valid session file.')
+
+    g_how = parser.add_argument_group('How to Download')
+    g_how.add_argument('--dirname-pattern',
+                       help='Name of directory where to store posts. {profile} is replaced by the profile name, '
+                            '{target} is replaced by the target you specified, i.e. either :feed, #hashtag or the '
+                            'profile name. Defaults to \'{target}\'.')
+    g_how.add_argument('--filename-pattern',
+                       help='Prefix of filenames, relative to the directory given with '
+                            '--dirname-pattern. {profile} is replaced by the profile name,'
+                            '{target} is replaced by the target you specified, i.e. either :feed'
+                            '#hashtag or the profile name. Defaults to \'{date_utc}_UTC\'')
+    g_how.add_argument('--user-agent',
+                       help='User Agent to use for HTTP requests. Defaults to \'{}\'.'.format(default_user_agent()))
+    g_how.add_argument('-S', '--no-sleep', action='store_true', help=SUPPRESS)
+    g_how.add_argument('--max-connection-attempts', metavar='N', type=int, default=3,
+                       help='Maximum number of connection attempts until a request is aborted. Defaults to 3. If a '
+                            'connection fails, it can be manually skipped by hitting CTRL+C. Set this to 0 to retry '
+                            'infinitely.')
+
+    g_misc = parser.add_argument_group('Miscellaneous Options')
+    g_misc.add_argument('-q', '--quiet', action='store_true',
+                        help='Disable user interaction, i.e. do not print messages (except errors) and fail '
+                             'if login credentials are needed but not given. This makes Instaloader suitable as a '
+                             'cron job.')
+    g_misc.add_argument('-h', '--help', action='help', help='Show this help message and exit.')
+    g_misc.add_argument('--version', action='version', help='Show version number and exit.',
+                        version=__version__)
+
+    args = parser.parse_args()
+    try:
+        if args.login is None and (args.stories or args.stories_only):
+            print("--login=USERNAME required to download stories.", file=sys.stderr)
+            args.stories = False
+            if args.stories_only:
+                raise SystemExit(1)
+
+        if ':feed-all' in args.profile or ':feed-liked' in args.profile:
+            raise SystemExit(":feed-all and :feed-liked were removed. Use :feed as target and "
+                             "eventually --post-filter=viewer_has_liked.")
+
+        post_metadata_txt_pattern = '\n'.join(args.post_metadata_txt) if args.post_metadata_txt else None
+        storyitem_metadata_txt_pattern = '\n'.join(args.storyitem_metadata_txt) if args.storyitem_metadata_txt else None
+
+        if args.no_captions:
+            if not (post_metadata_txt_pattern or storyitem_metadata_txt_pattern):
+                post_metadata_txt_pattern = ''
+                storyitem_metadata_txt_pattern = ''
+            else:
+                raise SystemExit("--no-captions and --post-metadata-txt or --storyitem-metadata-txt given; "
+                                 "That contradicts.")
+
+        loader = Instaloader(sleep=not args.no_sleep, quiet=args.quiet, user_agent=args.user_agent,
+                             dirname_pattern=args.dirname_pattern, filename_pattern=args.filename_pattern,
+                             download_videos=not args.no_videos, download_video_thumbnails=not args.no_video_thumbnails,
+                             download_geotags=args.geotags,
+                             download_comments=args.comments, save_metadata=not args.no_metadata_json,
+                             compress_json=not args.no_compress_json,
+                             post_metadata_txt_pattern=post_metadata_txt_pattern,
+                             storyitem_metadata_txt_pattern=storyitem_metadata_txt_pattern,
+                             max_connection_attempts=args.max_connection_attempts)
+        _main(loader,
+              args.profile,
+              username=args.login.lower() if args.login is not None else None,
+              password=args.password,
+              sessionfile=args.sessionfile,
+              max_count=int(args.count) if args.count is not None else None,
+              profile_pic=not args.no_profile_pic,
+              profile_pic_only=args.profile_pic_only,
+              fast_update=args.fast_update,
+              stories=args.stories,
+              stories_only=args.stories_only,
+              post_filter_str=args.post_filter,
+              storyitem_filter_str=args.storyitem_filter)
+        loader.close()
+    except InstaloaderException as err:
+        raise SystemExit("Fatal error: %s" % err)
+
+
+if __name__ == "__main__":
+    main()

instaloader/exceptions.py

@@ -0,0 +1,53 @@
+class InstaloaderException(Exception):
+    """Base exception for this script.
+
+    :note: This exception should not be raised directly."""
+    pass
+
+
+class QueryReturnedBadRequestException(InstaloaderException):
+    pass
+
+
+class QueryReturnedNotFoundException(InstaloaderException):
+    pass
+
+
+class QueryReturnedForbiddenException(InstaloaderException):
+    pass
+
+
+class ProfileNotExistsException(InstaloaderException):
+    pass
+
+
+class ProfileHasNoPicsException(InstaloaderException):
+    pass
+
+
+class PrivateProfileNotFollowedException(InstaloaderException):
+    pass
+
+
+class LoginRequiredException(InstaloaderException):
+    pass
+
+
+class InvalidArgumentException(InstaloaderException):
+    pass
+
+
+class BadResponseException(InstaloaderException):
+    pass
+
+
+class BadCredentialsException(InstaloaderException):
+    pass
+
+
+class ConnectionException(InstaloaderException):
+    pass
+
+
+class TooManyRequestsException(ConnectionException):
+    pass

instaloader/instaloader.py

@@ -0,0 +1,740 @@
+import getpass
+import json
+import os
+import platform
+import re
+import shutil
+import string
+import sys
+import tempfile
+from contextlib import contextmanager, suppress
+from datetime import datetime, timezone
+from functools import wraps
+from io import BytesIO
+from typing import Callable, Iterator, List, Optional, Any
+
+from .exceptions import *
+from .instaloadercontext import InstaloaderContext
+from .structures import JsonExportable, Post, PostLocation, Profile, Story, StoryItem, save_structure_to_file
+
+
+def get_default_session_filename(username: str) -> str:
+    """Returns default session filename for given username."""
+    dirname = tempfile.gettempdir() + "/" + ".instaloader-" + getpass.getuser()
+    filename = dirname + "/" + "session-" + username
+    return filename.lower()
+
+
+def format_string_contains_key(format_string: str, key: str) -> bool:
+    # pylint:disable=unused-variable
+    for literal_text, field_name, format_spec, conversion in string.Formatter().parse(format_string):
+        if field_name and (field_name == key or field_name.startswith(key + '.')):
+            return True
+    return False
+
+
+def _requires_login(func: Callable) -> Callable:
+    """Decorator to raise an exception if herewith-decorated function is called without being logged in"""
+    @wraps(func)
+    def call(instaloader, *args, **kwargs):
+        if not instaloader.context.is_logged_in:
+            raise LoginRequiredException("--login=USERNAME required.")
+        return func(instaloader, *args, **kwargs)
+    # pylint:disable=no-member
+    call.__doc__ += ":raises LoginRequiredException: If called without being logged in.\n"
+    return call
+
+
+class _ArbitraryItemFormatter(string.Formatter):
+    def __init__(self, item: Any):
+        self._item = item
+
+    def get_field(self, field_name, args, kwargs):
+        """Override to substitute {ATTRIBUTE} by attributes of our _item."""
+        if hasattr(self._item, field_name):
+            return self._item.__getattribute__(field_name), None
+        return super().get_field(field_name, args, kwargs)
+
+    def format_field(self, value, format_spec):
+        """Override :meth:`string.Formatter.format_field` to have our
+         default format_spec for :class:`datetime.Datetime` objects, and to
+         let None yield an empty string rather than ``None``."""
+        if isinstance(value, datetime) and not format_spec:
+            return super().format_field(value, '%Y-%m-%d_%H-%M-%S')
+        if value is None:
+            return ''
+        return super().format_field(value, format_spec)
+
+
+class _PostPathFormatter(_ArbitraryItemFormatter):
+    def vformat(self, format_string, args, kwargs):
+        """Override :meth:`string.Formatter.vformat` for character substitution in paths for Windows, see issue #84."""
+        ret = super().vformat(format_string, args, kwargs)
+        return ret.replace(':', '\ua789') if platform.system() == 'Windows' else ret
+
+
+class Instaloader:
+    """Instaloader Class.
+
+    :param quiet: :option:`--quiet`
+    :param user_agent: :option:`--user-agent`
+    :param dirname_pattern: :option:`--dirname-pattern`, default is ``{target}``
+    :param filename_pattern: :option:`--filename-pattern`, default is ``{date_utc}_UTC``
+    :param download_videos: not :option:`--no-videos`
+    :param download_video_thumbnails: not :option:`--no-video-thumbnails`
+    :param download_geotags: :option:`--geotags`
+    :param download_comments: :option:`--comments`
+    :param save_metadata: not :option:`--no-metadata-json`
+    :param compress_json: not :option:`--no-compress-json`
+    :param post_metadata_txt_pattern:
+       :option:`--post-metadata-txt`, default is ``{caption}``. Set to empty string to avoid creation of post metadata
+       txt file.
+    :param storyitem_metadata_txt_pattern: :option:`--storyitem-metadata-txt`, default is empty (=none)
+    :param max_connection_attempts: :option:`--max-connection-attempts`
+    """
+
+    def __init__(self,
+                 sleep: bool = True, quiet: bool = False,
+                 user_agent: Optional[str] = None,
+                 dirname_pattern: Optional[str] = None,
+                 filename_pattern: Optional[str] = None,
+                 download_videos: bool = True,
+                 download_video_thumbnails: bool = True,
+                 download_geotags: bool = True,
+                 download_comments: bool = True,
+                 save_metadata: bool = True,
+                 compress_json: bool = True,
+                 post_metadata_txt_pattern: str = None,
+                 storyitem_metadata_txt_pattern: str = None,
+                 max_connection_attempts: int = 3):
+
+        self.context = InstaloaderContext(sleep, quiet, user_agent, max_connection_attempts)
+
+        # configuration parameters
+        self.dirname_pattern = dirname_pattern or "{target}"
+        self.filename_pattern = filename_pattern or "{date_utc}_UTC"
+        self.download_videos = download_videos
+        self.download_video_thumbnails = download_video_thumbnails
+        self.download_geotags = download_geotags
+        self.download_comments = download_comments
+        self.save_metadata = save_metadata
+        self.compress_json = compress_json
+        self.post_metadata_txt_pattern = '{caption}' if post_metadata_txt_pattern is None \
+            else post_metadata_txt_pattern
+        self.storyitem_metadata_txt_pattern = '' if storyitem_metadata_txt_pattern is None \
+            else storyitem_metadata_txt_pattern
+
+    @contextmanager
+    def anonymous_copy(self):
+        """Yield an anonymous, otherwise equally-configured copy of an Instaloader instance; Then copy its error log."""
+        new_loader = Instaloader(self.context.sleep, self.context.quiet, self.context.user_agent, self.dirname_pattern,
+                                 self.filename_pattern, self.download_videos, self.download_video_thumbnails,
+                                 self.download_geotags, self.download_comments, self.save_metadata,
+                                 self.compress_json, self.post_metadata_txt_pattern,
+                                 self.storyitem_metadata_txt_pattern, self.context.max_connection_attempts)
+        new_loader.context.query_timestamps = self.context.query_timestamps
+        yield new_loader
+        self.context.error_log.extend(new_loader.context.error_log)
+        new_loader.context.error_log = []  # avoid double-printing of errors
+        self.context.query_timestamps = new_loader.context.query_timestamps
+        new_loader.close()
+
+    def close(self):
+        """Close associated session objects and repeat error log."""
+        self.context.close()
+
+    def __enter__(self):
+        return self
+
+    def __exit__(self, *args):
+        self.close()
+
+    def download_pic(self, filename: str, url: str, mtime: datetime, filename_suffix: Optional[str] = None) -> bool:
+        """Downloads and saves picture with given url under given directory with given timestamp.
+        Returns true, if file was actually downloaded, i.e. updated."""
+        urlmatch = re.search('\\.[a-z0-9]*\\?', url)
+        file_extension = url[-3:] if urlmatch is None else urlmatch.group(0)[1:-1]
+        if filename_suffix is not None:
+            filename += '_' + filename_suffix
+        filename += '.' + file_extension
+        if os.path.isfile(filename):
+            self.context.log(filename + ' exists', end=' ', flush=True)
+            return False
+        self.context.get_and_write_raw(url, filename)
+        os.utime(filename, (datetime.now().timestamp(), mtime.timestamp()))
+        return True
+
+    def save_metadata_json(self, filename: str, structure: JsonExportable) -> None:
+        """Saves metadata JSON file of a structure."""
+        if self.compress_json:
+            filename += '.json.xz'
+        else:
+            filename += '.json'
+        save_structure_to_file(structure, filename)
+        if isinstance(structure, (Post, StoryItem)):
+            # log 'json ' message when saving Post or StoryItem
+            self.context.log('json', end=' ', flush=True)
+
+    def update_comments(self, filename: str, post: Post) -> None:
+        def _postcomment_asdict(comment):
+            return {'id': comment.id,
+                    'created_at': comment.created_at_utc.replace(tzinfo=timezone.utc).timestamp(),
+                    'text': comment.text,
+                    'owner': comment.owner._asdict()}
+        filename += '_comments.json'
+        try:
+            comments = json.load(open(filename))
+        except FileNotFoundError:
+            comments = list()
+        comments.extend(_postcomment_asdict(comment) for comment in post.get_comments())
+        if comments:
+            with open(filename, 'w') as file:
+                comments_list = sorted(sorted(list(comments), key=lambda t: t['id']),
+                                       key=lambda t: t['created_at'], reverse=True)
+                unique_comments_list = [comments_list[0]]
+                #for comment in comments_list:
+                #    if unique_comments_list[-1]['id'] != comment['id']:
+                #        unique_comments_list.append(comment)
+                #file.write(json.dumps(unique_comments_list, indent=4))
+                for x, y in zip(comments_list[:-1], comments_list[1:]):
+                    if x['id'] != y['id']:
+                        unique_comments_list.append(y)
+                file.write(json.dumps(unique_comments_list, indent=4))
+            self.context.log('comments', end=' ', flush=True)
+
+    def save_caption(self, filename: str, mtime: datetime, caption: str) -> None:
+        """Updates picture caption / Post metadata info"""
+        def _elliptify(caption):
+            pcaption = caption.replace('\n', ' ').strip()
+            return '[' + ((pcaption[:29] + u"\u2026") if len(pcaption) > 31 else pcaption) + ']'
+        filename += '.txt'
+        caption += '\n'
+        pcaption = _elliptify(caption)
+        caption = caption.encode("UTF-8")
+        with suppress(FileNotFoundError):
+            with open(filename, 'rb') as file:
+                file_caption = file.read()
+            if file_caption.replace(b'\r\n', b'\n') == caption.replace(b'\r\n', b'\n'):
+                try:
+                    self.context.log(pcaption + ' unchanged', end=' ', flush=True)
+                except UnicodeEncodeError:
+                    self.context.log('txt unchanged', end=' ', flush=True)
+                return None
+            else:
+                def get_filename(index):
+                    return filename if index == 0 else (filename[:-4] + '_old_' +
+                                                        (str(0) if index < 10 else str()) + str(index) + filename[-4:])
+
+                i = 0
+                while os.path.isfile(get_filename(i)):
+                    i = i + 1
+                for index in range(i, 0, -1):
+                    os.rename(get_filename(index - 1), get_filename(index))
+                try:
+                    self.context.log(_elliptify(file_caption.decode("UTF-8")) + ' updated', end=' ', flush=True)
+                except UnicodeEncodeError:
+                    self.context.log('txt updated', end=' ', flush=True)
+        try:
+            self.context.log(pcaption, end=' ', flush=True)
+        except UnicodeEncodeError:
+            self.context.log('txt', end=' ', flush=True)
+        with open(filename, 'wb') as text_file:
+            shutil.copyfileobj(BytesIO(caption), text_file)
+        os.utime(filename, (datetime.now().timestamp(), mtime.timestamp()))
+
+    def save_location(self, filename: str, location: PostLocation, mtime: datetime) -> None:
+        """Save post location name and Google Maps link."""
+        filename += '_location.txt'
+        location_string = (location.name + "\n" +
+                           "https://maps.google.com/maps?q={0},{1}&ll={0},{1}\n".format(location.lat,
+                                                                                        location.lng))
+        with open(filename, 'wb') as text_file:
+            shutil.copyfileobj(BytesIO(location_string.encode()), text_file)
+        os.utime(filename, (datetime.now().timestamp(), mtime.timestamp()))
+        self.context.log('geo', end=' ', flush=True)
+
+    def download_profilepic(self, profile: Profile) -> None:
+        """Downloads and saves profile pic."""
+
+        def _epoch_to_string(epoch: datetime) -> str:
+            return epoch.strftime('%Y-%m-%d_%H-%M-%S')
+
+        profile_pic_url = profile.get_profile_pic_url()
+        with self.context.get_anonymous_session() as anonymous_session:
+            date_object = datetime.strptime(anonymous_session.head(profile_pic_url).headers["Last-Modified"],
+                                            '%a, %d %b %Y %H:%M:%S GMT')
+        if ((format_string_contains_key(self.dirname_pattern, 'profile') or
+             format_string_contains_key(self.dirname_pattern, 'target'))):
+            filename = '{0}/{1}_UTC_profile_pic.{2}'.format(self.dirname_pattern.format(profile=profile.username.lower(),
+                                                                                        target=profile.username.lower()),
+                                                            _epoch_to_string(date_object), profile_pic_url[-3:])
+        else:
+            filename = '{0}/{1}_{2}_UTC_profile_pic.{3}'.format(self.dirname_pattern.format(), profile.username.lower(),
+                                                                _epoch_to_string(date_object), profile_pic_url[-3:])
+        if os.path.isfile(filename):
+            self.context.log(filename + ' already exists')
+            return None
+        self.context.get_and_write_raw(profile_pic_url, filename)
+        os.utime(filename, (datetime.now().timestamp(), date_object.timestamp()))
+        self.context.log('')  # log output of _get_and_write_raw() does not produce \n
+
+    @_requires_login
+    def save_session_to_file(self, filename: Optional[str] = None) -> None:
+        """Saves internally stored :class:`requests.Session` object.
+
+        :param filename: Filename, or None to use default filename.
+        """
+        if filename is None:
+            filename = get_default_session_filename(self.context.username)
+        dirname = os.path.dirname(filename)
+        if dirname != '' and not os.path.exists(dirname):
+            os.makedirs(dirname)
+            os.chmod(dirname, 0o700)
+        with open(filename, 'wb') as sessionfile:
+            os.chmod(filename, 0o600)
+            self.context.save_session_to_file(sessionfile)
+            self.context.log("Saved session to %s." % filename)
+
+    def load_session_from_file(self, username: str, filename: Optional[str] = None) -> None:
+        """Internally stores :class:`requests.Session` object loaded from file.
+
+        If filename is None, the file with the default session path is loaded.
+
+        :raises FileNotFoundError: If the file does not exist.
+        """
+        if filename is None:
+            filename = get_default_session_filename(username)
+        with open(filename, 'rb') as sessionfile:
+            self.context.load_session_from_file(username, sessionfile)
+            self.context.log("Loaded session from %s." % filename)
+
+    def test_login(self) -> Optional[str]:
+        """Returns the Instagram username to which given :class:`requests.Session` object belongs, or None."""
+        return self.context.test_login()
+
+    def login(self, user: str, passwd: str) -> None:
+        """Log in to instagram with given username and password and internally store session object"""
+        self.context.login(user, passwd)
+
+    def download_post(self, post: Post, target: str) -> bool:
+        """
+        Download everything associated with one instagram post node, i.e. picture, caption and video.
+
+        :param post: Post to download.
+        :param target: Target name, i.e. profile name, #hashtag, :feed; for filename.
+        :return: True if something was downloaded, False otherwise, i.e. file was already there
+        """
+
+        dirname = _PostPathFormatter(post).format(self.dirname_pattern, target=target)
+        filename = dirname + '/' + _PostPathFormatter(post).format(self.filename_pattern, target=target)
+        os.makedirs(os.path.dirname(filename), exist_ok=True)
+
+        # Download the image(s) / video thumbnail and videos within sidecars if desired
+        downloaded = False
+        if post.typename == 'GraphSidecar':
+            edge_number = 1
+            for sidecar_node in post.get_sidecar_nodes():
+                # Download picture or video thumbnail
+                if not sidecar_node.is_video or self.download_video_thumbnails is True:
+                    downloaded |= self.download_pic(filename=filename, url=sidecar_node.display_url,
+                                                    mtime=post.date_local, filename_suffix=str(edge_number))
+                # Additionally download video if available and desired
+                if sidecar_node.is_video and self.download_videos is True:
+                    downloaded |= self.download_pic(filename=filename, url=sidecar_node.video_url,
+                                                    mtime=post.date_local, filename_suffix=str(edge_number))
+                edge_number += 1
+        elif post.typename == 'GraphImage':
+            downloaded = self.download_pic(filename=filename, url=post.url, mtime=post.date_local)
+        elif post.typename == 'GraphVideo':
+            if self.download_video_thumbnails is True:
+                downloaded = self.download_pic(filename=filename, url=post.url, mtime=post.date_local)
+        else:
+            self.context.error("Warning: {0} has unknown typename: {1}".format(post, post.typename))
+
+        # Save caption if desired
+        metadata_string = _ArbitraryItemFormatter(post).format(self.post_metadata_txt_pattern).strip()
+        if metadata_string:
+            self.save_caption(filename=filename, mtime=post.date_local, caption=metadata_string)
+
+        # Download video if desired
+        if post.is_video and self.download_videos is True:
+            downloaded |= self.download_pic(filename=filename, url=post.video_url, mtime=post.date_local)
+
+        # Download geotags if desired
+        if self.download_geotags and post.location:
+            self.save_location(filename, post.location, post.date_local)
+
+        # Update comments if desired
+        if self.download_comments is True:
+            self.update_comments(filename=filename, post=post)
+
+        # Save metadata as JSON if desired.
+        if self.save_metadata is not False:
+            self.save_metadata_json(filename, post)
+
+        self.context.log()
+        return downloaded
+
+    @_requires_login
+    def get_stories(self, userids: Optional[List[int]] = None) -> Iterator[Story]:
+        """Get available stories from followees or all stories of users whose ID are given.
+        Does not mark stories as seen.
+        To use this, one needs to be logged in
+
+        :param userids: List of user IDs to be processed in terms of downloading their stories, or None.
+        """
+
+        if userids is None:
+            data = self.context.graphql_query("d15efd8c0c5b23f0ef71f18bf363c704",
+                                              {"only_stories": True})["data"]["user"]
+            if data is None:
+                raise BadResponseException('Bad stories reel JSON.')
+            userids = list(edge["node"]["id"] for edge in data["feed_reels_tray"]["edge_reels_tray_to_reel"]["edges"])
+
+        stories = self.context.graphql_query("bf41e22b1c4ba4c9f31b844ebb7d9056",
+                                             {"reel_ids": userids, "precomposed_overlay": False})["data"]
+
+        yield from (Story(self.context, media) for media in stories['reels_media'])
+
+    @_requires_login
+    def download_stories(self,
+                         userids: Optional[List[int]] = None,
+                         fast_update: bool = False,
+                         filename_target: str = ':stories',
+                         storyitem_filter: Optional[Callable[[StoryItem], bool]] = None) -> None:
+        """
+        Download available stories from user followees or all stories of users whose ID are given.
+        Does not mark stories as seen.
+        To use this, one needs to be logged in
+
+        :param userids: List of user IDs to be processed in terms of downloading their stories
+        :param fast_update: If true, abort when first already-downloaded picture is encountered
+        :param filename_target: Replacement for {target} in dirname_pattern and filename_pattern
+        :param storyitem_filter: function(storyitem), which returns True if given StoryItem should be downloaded
+        """
+
+        if not userids:
+            self.context.log("Retrieving all visible stories...")
+
+        for user_story in self.get_stories(userids):
+            name = user_story.owner_username
+            self.context.log("Retrieving stories from profile {}.".format(name))
+            totalcount = user_story.itemcount
+            count = 1
+            for item in user_story.get_items():
+                if storyitem_filter is not None and not storyitem_filter(item):
+                    self.context.log("<{} skipped>".format(item), flush=True)
+                    continue
+                self.context.log("[%3i/%3i] " % (count, totalcount), end="", flush=True)
+                count += 1
+                with self.context.error_catcher('Download story from user {}'.format(name)):
+                    downloaded = self.download_storyitem(item, filename_target)
+                    if fast_update and not downloaded:
+                        break
+
+    def download_storyitem(self, item: StoryItem, target: str) -> bool:
+        """Download one user story.
+
+        :param item: Story item, as in story['items'] for story in :meth:`get_stories`
+        :param target: Replacement for {target} in dirname_pattern and filename_pattern
+        :return: True if something was downloaded, False otherwise, i.e. file was already there
+        """
+
+        date_local = item.date_local
+        dirname = _PostPathFormatter(item).format(self.dirname_pattern, target=target)
+        filename = dirname + '/' + _PostPathFormatter(item).format(self.filename_pattern, target=target)
+        os.makedirs(os.path.dirname(filename), exist_ok=True)
+        downloaded = False
+        if not item.is_video or self.download_video_thumbnails is True:
+            url = item.url
+            downloaded = self.download_pic(filename=filename, url=url, mtime=date_local)
+        if item.is_video and self.download_videos is True:
+            downloaded |= self.download_pic(filename=filename, url=item.video_url, mtime=date_local)
+        # Save caption if desired
+        metadata_string = _ArbitraryItemFormatter(item).format(self.storyitem_metadata_txt_pattern).strip()
+        if metadata_string:
+            self.save_caption(filename=filename, mtime=item.date_local, caption=metadata_string)
+        # Save metadata as JSON if desired.
+        if self.save_metadata is not False:
+            self.save_metadata_json(filename, item)
+        self.context.log()
+        return downloaded
+
+    @_requires_login
+    def get_feed_posts(self) -> Iterator[Post]:
+        """Get Posts of the user's feed.
+
+        :return: Iterator over Posts of the user's feed.
+        """
+
+        data = self.context.graphql_query("d6f4427fbe92d846298cf93df0b937d3", {})["data"]
+
+        while True:
+            feed = data["user"]["edge_web_feed_timeline"]
+            yield from (Post(self.context, edge["node"]) for edge in feed["edges"]
+                        if not edge["node"]["__typename"] == "GraphSuggestedUserFeedUnit")
+            if not feed["page_info"]["has_next_page"]:
+                break
+            data = self.context.graphql_query("d6f4427fbe92d846298cf93df0b937d3",
+                                              {'fetch_media_item_count': 12,
+                                               'fetch_media_item_cursor': feed["page_info"]["end_cursor"],
+                                               'fetch_comment_count': 4,
+                                               'fetch_like': 10,
+                                               'has_stories': False})["data"]
+
+    @_requires_login
+    def download_feed_posts(self, max_count: int = None, fast_update: bool = False,
+                            post_filter: Optional[Callable[[Post], bool]] = None) -> None:
+        """
+        Download pictures from the user's feed.
+
+        Example to download up to the 20 pics the user last liked::
+
+            loader = Instaloader()
+            loader.load_session_from_file('USER')
+            loader.download_feed_posts(max_count=20, fast_update=True,
+                                       post_filter=lambda post: post.viewer_has_liked)
+
+        :param max_count: Maximum count of pictures to download
+        :param fast_update: If true, abort when first already-downloaded picture is encountered
+        :param post_filter: function(post), which returns True if given picture should be downloaded
+        """
+        self.context.log("Retrieving pictures from your feed...")
+        count = 1
+        for post in self.get_feed_posts():
+            if max_count is not None and count > max_count:
+                break
+            name = post.owner_username
+            if post_filter is not None and not post_filter(post):
+                self.context.log("<pic by %s skipped>" % name, flush=True)
+                continue
+            self.context.log("[%3i] %s " % (count, name), end="", flush=True)
+            count += 1
+            with self.context.error_catcher('Download feed'):
+                downloaded = self.download_post(post, target=':feed')
+                if fast_update and not downloaded:
+                    break
+
+    @_requires_login
+    def download_saved_posts(self, max_count: int = None, fast_update: bool = False,
+                             post_filter: Optional[Callable[[Post], bool]] = None) -> None:
+        """Download user's saved pictures.
+
+        :param max_count: Maximum count of pictures to download
+        :param fast_update: If true, abort when first already-downloaded picture is encountered
+        :param post_filter: function(post), which returns True if given picture should be downloaded
+        """
+        self.context.log("Retrieving saved posts...")
+        count = 1
+        for post in Profile.from_username(self.context, self.context.username).get_saved_posts():
+            if max_count is not None and count > max_count:
+                break
+            name = post.owner_username
+            if post_filter is not None and not post_filter(post):
+                self.context.log("<pic by {} skipped".format(name), flush=True)
+                continue
+            self.context.log("[{:>3}] {} ".format(count, name), end=str(), flush=True)
+            count += 1
+            with self.context.error_catcher('Download saved posts'):
+                downloaded = self.download_post(post, target=':saved')
+                if fast_update and not downloaded:
+                    break
+
+    @_requires_login
+    def get_explore_posts(self) -> Iterator[Post]:
+        """Get Posts which are worthy of exploring suggested by Instagram.
+
+        :return: Iterator over Posts of the user's suggested posts.
+        """
+        data = self.context.get_json('explore/', {})
+        yield from (Post(self.context, node)
+                    for node in self.context.graphql_node_list("df0dcc250c2b18d9fd27c5581ef33c7c",
+                                                               {}, 'https://www.instagram.com/explore/',
+                                                               lambda d: d['data']['user']['edge_web_discover_media'],
+                                                               data['rhx_gis']))
+
+    def get_hashtag_posts(self, hashtag: str) -> Iterator[Post]:
+        """Get Posts associated with a #hashtag."""
+        has_next_page = True
+        end_cursor = None
+        while has_next_page:
+            if end_cursor:
+                params = {'__a': 1, 'max_id': end_cursor}
+            else:
+                params = {'__a': 1}
+            hashtag_data = self.context.get_json('explore/tags/{0}/'.format(hashtag),
+                                                 params)['graphql']['hashtag']['edge_hashtag_to_media']
+            yield from (Post(self.context, edge['node']) for edge in hashtag_data['edges'])
+            has_next_page = hashtag_data['page_info']['has_next_page']
+            end_cursor = hashtag_data['page_info']['end_cursor']
+
+    def download_hashtag(self, hashtag: str,
+                         max_count: Optional[int] = None,
+                         post_filter: Optional[Callable[[Post], bool]] = None,
+                         fast_update: bool = False) -> None:
+        """Download pictures of one hashtag.
+
+        To download the last 30 pictures with hashtag #cat, do::
+
+            loader = Instaloader()
+            loader.download_hashtag('cat', max_count=30)
+
+        :param hashtag: Hashtag to download, without leading '#'
+        :param max_count: Maximum count of pictures to download
+        :param post_filter: function(post), which returns True if given picture should be downloaded
+        :param fast_update: If true, abort when first already-downloaded picture is encountered
+        """
+        hashtag = hashtag.lower()
+        self.context.log("Retrieving pictures with hashtag {}...".format(hashtag))
+        count = 1
+        for post in self.get_hashtag_posts(hashtag):
+            if max_count is not None and count > max_count:
+                break
+            self.context.log('[{0:3d}] #{1} '.format(count, hashtag), end='', flush=True)
+            if post_filter is not None and not post_filter(post):
+                self.context.log('<skipped>')
+                continue
+            count += 1
+            with self.context.error_catcher('Download hashtag #{}'.format(hashtag)):
+                downloaded = self.download_post(post, target='#' + hashtag)
+                if fast_update and not downloaded:
+                    break
+
+    def check_profile_id(self, profile_name: str) -> Profile:
+        """
+        Consult locally stored ID of profile with given name, check whether ID matches and whether name
+        has changed and return current name of the profile, and store ID of profile.
+
+        :param profile_name: Profile name
+        :return: Instance of current profile
+        """
+        profile = None
+        with suppress(ProfileNotExistsException):
+            profile = Profile.from_username(self.context, profile_name)
+        profile_exists = profile is not None
+        if ((format_string_contains_key(self.dirname_pattern, 'profile') or
+             format_string_contains_key(self.dirname_pattern, 'target'))):
+            id_filename = '{0}/id'.format(self.dirname_pattern.format(profile=profile_name.lower(),
+                                                                      target=profile_name.lower()))
+        else:
+            id_filename = '{0}/{1}_id'.format(self.dirname_pattern.format(), profile_name.lower())
+        try:
+            with open(id_filename, 'rb') as id_file:
+                profile_id = int(id_file.read())
+            if (not profile_exists) or \
+                    (profile_id != profile.userid):
+                if profile_exists:
+                    self.context.log("Profile {0} does not match the stored unique ID {1}.".format(profile_name,
+                                                                                                   profile_id))
+                else:
+                    self.context.log("Trying to find profile {0} using its unique ID {1}.".format(profile_name,
+                                                                                                  profile_id))
+                if not self.context.is_logged_in:
+                    self.context.error("Profile {} changed its name. "
+                                       "If you use --login=USERNAME, I can find out the new name.")
+                    raise LoginRequiredException("--login=USERNAME required to obtain profile name from its ID number")
+                profile_from_id = Profile.from_id(self.context, profile_id)
+                newname = profile_from_id.username
+                self.context.log("Profile {0} has changed its name to {1}.".format(profile_name, newname))
+                if ((format_string_contains_key(self.dirname_pattern, 'profile') or
+                     format_string_contains_key(self.dirname_pattern, 'target'))):
+                    os.rename(self.dirname_pattern.format(profile=profile_name.lower(),
+                                                          target=profile_name.lower()),
+                              self.dirname_pattern.format(profile=newname.lower(),
+                                                          target=newname.lower()))
+                else:
+                    os.rename('{0}/{1}_id'.format(self.dirname_pattern.format(), profile_name.lower()),
+                              '{0}/{1}_id'.format(self.dirname_pattern.format(), newname.lower()))
+                return profile_from_id
+            return profile
+        except FileNotFoundError:
+            pass
+        if profile_exists:
+            os.makedirs(self.dirname_pattern.format(profile=profile_name.lower(),
+                                                    target=profile_name.lower()), exist_ok=True)
+            with open(id_filename, 'w') as text_file:
+                text_file.write(str(profile.userid) + "\n")
+                self.context.log("Stored ID {0} for profile {1}.".format(profile.userid, profile_name))
+            return profile
+        raise ProfileNotExistsException("Profile {0} does not exist.".format(profile_name))
+
+    def download_profile(self, profile_name: str,
+                         profile_pic: bool = True, profile_pic_only: bool = False,
+                         fast_update: bool = False,
+                         download_stories: bool = False, download_stories_only: bool = False,
+                         post_filter: Optional[Callable[[Post], bool]] = None,
+                         storyitem_filter: Optional[Callable[[StoryItem], bool]] = None) -> None:
+        """Download one profile"""
+
+        # Get profile main page json
+        # check if profile does exist or name has changed since last download
+        # and update name and json data if necessary
+        profile = self.check_profile_id(profile_name.lower())
+
+        profile_name = profile.username
+
+        # Save metadata as JSON if desired.
+        if self.save_metadata is not False:
+            json_filename = '{0}/{1}_{2}'.format(self.dirname_pattern.format(profile=profile_name, target=profile_name),
+                                                 profile_name, profile.userid)
+            self.save_metadata_json(json_filename, profile)
+
+        if self.context.is_logged_in and profile.has_blocked_viewer and not profile.is_private:
+            # raising ProfileNotExistsException invokes "trying again anonymously" logic
+            raise ProfileNotExistsException("Profile {} has blocked you".format(profile_name))
+
+        # Download profile picture
+        if profile_pic or profile_pic_only:
+            with self.context.error_catcher('Download profile picture of {}'.format(profile_name)):
+                self.download_profilepic(profile)
+        if profile_pic_only:
+            return
+
+        # Catch some errors
+        if profile.is_private:
+            if not self.context.is_logged_in:
+                raise LoginRequiredException("profile %s requires login" % profile_name)
+            if not profile.followed_by_viewer and \
+                    self.context.username != profile.username:
+                raise PrivateProfileNotFollowedException("Profile %s: private but not followed." % profile_name)
+        else:
+            if self.context.is_logged_in and not (download_stories or download_stories_only):
+                self.context.log("profile %s could also be downloaded anonymously." % profile_name)
+
+        # Download stories, if requested
+        if download_stories or download_stories_only:
+            with self.context.error_catcher("Download stories of {}".format(profile_name)):
+                self.download_stories(userids=[profile.userid], filename_target=profile_name, fast_update=fast_update,
+                                      storyitem_filter=storyitem_filter)
+        if download_stories_only:
+            return
+
+        # Iterate over pictures and download them
+        self.context.log("Retrieving posts from profile {}.".format(profile_name))
+        totalcount = profile.mediacount
+        count = 1
+        for post in profile.get_posts():
+            self.context.log("[%3i/%3i] " % (count, totalcount), end="", flush=True)
+            count += 1
+            if post_filter is not None and not post_filter(post):
+                self.context.log('<skipped>')
+                continue
+            with self.context.error_catcher('Download profile {}'.format(profile_name)):
+                downloaded = self.download_post(post, target=profile_name)
+                if fast_update and not downloaded:
+                    break
+
+    def interactive_login(self, username: str) -> None:
+        """Logs in and internally stores session, asking user for password interactively.
+
+        :raises LoginRequiredException: when in quiet mode."""
+        if self.context.quiet:
+            raise LoginRequiredException("Quiet mode requires given password or valid session file.")
+        password = None
+        while password is None:
+            password = getpass.getpass(prompt="Enter Instagram password for %s: " % username)
+            try:
+                self.login(username, password)
+            except BadCredentialsException as err:
+                print(err, file=sys.stderr)
+                password = None

instaloader/instaloadercontext.py

@@ -0,0 +1,378 @@
+import hashlib
+import json
+import pickle
+import random
+import re
+import shutil
+import sys
+import textwrap
+import time
+import urllib.parse
+from contextlib import contextmanager
+from typing import Any, Callable, Dict, Iterator, Optional
+
+import requests
+import requests.utils
+import urllib3
+
+from .exceptions import *
+
+
+def copy_session(session: requests.Session) -> requests.Session:
+    """Duplicates a requests.Session."""
+    new = requests.Session()
+    new.cookies = requests.utils.cookiejar_from_dict(requests.utils.dict_from_cookiejar(session.cookies))
+    new.headers = session.headers.copy()
+    return new
+
+
+def default_user_agent() -> str:
+    return 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 ' \
+           '(KHTML, like Gecko) Chrome/51.0.2704.79 Safari/537.36'
+
+
+class InstaloaderContext:
+    """Class providing methods for (error) logging and low-level communication with Instagram.
+
+    It is not thought to be instantiated directly, rather :class:`Instaloader` instances maintain a context
+    object.
+
+    For logging, it provides :meth:`log`, :meth:`error`, :meth:`error_catcher`.
+
+    It provides low-level communication routines :meth:`get_json`, :meth:`graphql_query`, :meth:`graphql_node_list`,
+    :meth:`get_and_write_raw` and implements mechanisms for rate controlling and error handling.
+
+    Further, it provides methods for logging in and general session handles, which are used by that routines in
+    class :class:`Instaloader`.
+    """
+
+    def __init__(self, sleep: bool = True, quiet: bool = False,
+                 user_agent: Optional[str] = None, max_connection_attempts: int = 3):
+
+        self.user_agent = user_agent if user_agent is not None else default_user_agent()
+        self._session = self.get_anonymous_session()
+        self.username = None
+        self.sleep = sleep
+        self.quiet = quiet
+        self.max_connection_attempts = max_connection_attempts
+        self._graphql_page_length = 50
+
+        # error log, filled with error() and printed at the end of Instaloader.main()
+        self.error_log = []
+
+        # For the adaption of sleep intervals (rate control)
+        self.query_timestamps = list()
+
+        # Can be set to True for testing, disables supression of InstaloaderContext._error_catcher
+        self.raise_all_errors = False
+
+    @property
+    def is_logged_in(self) -> bool:
+        """True, if this Instaloader instance is logged in."""
+        return bool(self.username)
+
+    def log(self, *msg, sep='', end='\n', flush=False):
+        """Log a message to stdout that can be suppressed with --quiet."""
+        if not self.quiet:
+            print(*msg, sep=sep, end=end, flush=flush)
+
+    def error(self, msg, repeat_at_end=True):
+        """Log a non-fatal error message to stderr, which is repeated at program termination.
+
+        :param msg: Message to be printed.
+        :param repeat_at_end: Set to false if the message should be printed, but not repeated at program termination."""
+        print(msg, file=sys.stderr)
+        if repeat_at_end:
+            self.error_log.append(msg)
+
+    def close(self):
+        """Print error log and close session"""
+        if self.error_log and not self.quiet:
+            print("\nErrors occured:", file=sys.stderr)
+            for err in self.error_log:
+                print(err, file=sys.stderr)
+        self._session.close()
+
+    @contextmanager
+    def error_catcher(self, extra_info: Optional[str] = None):
+        """
+        Context manager to catch, print and record InstaloaderExceptions.
+
+        :param extra_info: String to prefix error message with."""
+        try:
+            yield
+        except InstaloaderException as err:
+            if extra_info:
+                self.error('{}: {}'.format(extra_info, err))
+            else:
+                self.error('{}'.format(err))
+            if self.raise_all_errors:
+                raise
+
+    def _default_http_header(self, empty_session_only: bool = False) -> Dict[str, str]:
+        """Returns default HTTP header we use for requests."""
+        header = {'Accept-Encoding': 'gzip, deflate',
+                  'Accept-Language': 'en-US,en;q=0.8',
+                  'Connection': 'keep-alive',
+                  'Content-Length': '0',
+                  'Host': 'www.instagram.com',
+                  'Origin': 'https://www.instagram.com',
+                  'Referer': 'https://www.instagram.com/',
+                  'User-Agent': self.user_agent,
+                  'X-Instagram-AJAX': '1',
+                  'X-Requested-With': 'XMLHttpRequest'}
+        if empty_session_only:
+            del header['Host']
+            del header['Origin']
+            del header['Referer']
+            del header['X-Instagram-AJAX']
+            del header['X-Requested-With']
+        return header
+
+    def get_anonymous_session(self) -> requests.Session:
+        """Returns our default anonymous requests.Session object."""
+        session = requests.Session()
+        session.cookies.update({'sessionid': '', 'mid': '', 'ig_pr': '1',
+                                'ig_vw': '1920', 'csrftoken': '',
+                                's_network': '', 'ds_user_id': ''})
+        session.headers.update(self._default_http_header(empty_session_only=True))
+        return session
+
+    def save_session_to_file(self, sessionfile):
+        """Not meant to be used directly, use :meth:`Instaloader.save_session_to_file`."""
+        pickle.dump(requests.utils.dict_from_cookiejar(self._session.cookies), sessionfile)
+
+    def load_session_from_file(self, username, sessionfile):
+        """Not meant to be used directly, use :meth:`Instaloader.load_session_to_file`."""
+        session = requests.Session()
+        session.cookies = requests.utils.cookiejar_from_dict(pickle.load(sessionfile))
+        session.headers.update(self._default_http_header())
+        session.headers.update({'X-CSRFToken': session.cookies.get_dict()['csrftoken']})
+        self._session = session
+        self.username = username
+
+    def test_login(self) -> Optional[str]:
+        """Not meant to be used directly, use :meth:`Instaloader.test_login`."""
+        data = self.graphql_query("d6f4427fbe92d846298cf93df0b937d3", {})
+        return data["data"]["user"]["username"] if data["data"]["user"] is not None else None
+
+    def login(self, user, passwd):
+        """Not meant to be used directly, use :meth:`Instaloader.login`."""
+        session = requests.Session()
+        session.cookies.update({'sessionid': '', 'mid': '', 'ig_pr': '1',
+                                'ig_vw': '1920', 'csrftoken': '',
+                                's_network': '', 'ds_user_id': ''})
+        session.headers.update(self._default_http_header())
+        self._sleep()
+        resp = session.get('https://www.instagram.com/')
+        session.headers.update({'X-CSRFToken': resp.cookies['csrftoken']})
+        self._sleep()
+        login = session.post('https://www.instagram.com/accounts/login/ajax/',
+                             data={'password': passwd, 'username': user}, allow_redirects=True)
+        session.headers.update({'X-CSRFToken': login.cookies['csrftoken']})
+        if login.status_code == 200:
+            self._session = session
+            if user == self.test_login():
+                self.username = user
+            else:
+                self.username = None
+                self._session = None
+                raise BadCredentialsException('Login error! Check your credentials!')
+        else:
+            raise ConnectionException('Login error! Connection error!')
+
+    def _sleep(self):
+        """Sleep a short time if self.sleep is set. Called before each request to instagram.com."""
+        if self.sleep:
+            time.sleep(random.uniform(0.5, 3))
+
+    def get_json(self, path: str, params: Dict[str, Any], host: str = 'www.instagram.com',
+                 session: Optional[requests.Session] = None, _attempt=1) -> Dict[str, Any]:
+        """JSON request to Instagram.
+
+        :param path: URL, relative to the given domain which defaults to www.instagram.com/
+        :param params: GET parameters
+        :param host: Domain part of the URL from where to download the requested JSON; defaults to www.instagram.com
+        :param session: Session to use, or None to use self.session
+        :return: Decoded response dictionary
+        :raises QueryReturnedBadRequestException: When the server responds with a 400.
+        :raises QueryReturnedNotFoundException: When the server responds with a 404.
+        :raises ConnectionException: When query repeatedly failed.
+        """
+        def graphql_query_waittime(untracked_queries: bool = False) -> int:
+            sliding_window = 660
+            if not self.query_timestamps:
+                return sliding_window if untracked_queries else 0
+            current_time = time.monotonic()
+            self.query_timestamps = list(filter(lambda t: t > current_time - sliding_window, self.query_timestamps))
+            if len(self.query_timestamps) < 20 and not untracked_queries:
+                return 0
+            return round(min(self.query_timestamps) + sliding_window - current_time) + 6
+        is_graphql_query = 'query_hash' in params and 'graphql/query' in path
+        if is_graphql_query:
+            waittime = graphql_query_waittime()
+            if waittime > 0:
+                self.log('\nToo many queries in the last time. Need to wait {} seconds.'.format(waittime))
+                time.sleep(waittime)
+            if self.query_timestamps is not None:
+                self.query_timestamps.append(time.monotonic())
+            else:
+                self.query_timestamps = [time.monotonic()]
+        sess = session if session else self._session
+        try:
+            self._sleep()
+            resp = sess.get('https://{0}/{1}'.format(host, path), params=params, allow_redirects=False)
+            while resp.is_redirect:
+                redirect_url = resp.headers['location']
+                self.log('\nHTTP redirect from https://{0}/{1} to {2}'.format(host, path, redirect_url))
+                if redirect_url.index('https://{}/'.format(host)) == 0:
+                    resp = sess.get(redirect_url if redirect_url.endswith('/') else redirect_url + '/',
+                                    params=params, allow_redirects=False)
+                else:
+                    break
+            if resp.status_code == 400:
+                raise QueryReturnedBadRequestException("400 Bad Request")
+            if resp.status_code == 404:
+                raise QueryReturnedNotFoundException("404 Not Found")
+            if resp.status_code == 429:
+                raise TooManyRequestsException("429 Too Many Requests")
+            if resp.status_code != 200:
+                raise ConnectionException("HTTP error code {}.".format(resp.status_code))
+            is_html_query = not is_graphql_query and not "__a" in params and host == "www.instagram.com"
+            if is_html_query:
+                match = re.search(r'window\._sharedData = (.*);</script>', resp.text)
+                if match is None:
+                    raise ConnectionException("Could not find \"window._sharedData\" in html response.")
+                return json.loads(match.group(1))
+            else:
+                resp_json = resp.json()
+            if 'status' in resp_json and resp_json['status'] != "ok":
+                if 'message' in resp_json:
+                    raise ConnectionException("Returned \"{}\" status, message \"{}\".".format(resp_json['status'],
+                                                                                               resp_json['message']))
+                else:
+                    raise ConnectionException("Returned \"{}\" status.".format(resp_json['status']))
+            return resp_json
+        except (ConnectionException, json.decoder.JSONDecodeError, requests.exceptions.RequestException) as err:
+            error_string = "JSON Query to {}: {}".format(path, err)
+            if _attempt == self.max_connection_attempts:
+                raise ConnectionException(error_string) from err
+            self.error(error_string + " [retrying; skip with ^C]", repeat_at_end=False)
+            text_for_429 = ("HTTP error code 429 was returned because too many queries occured in the last time. "
+                            "Please do not use Instagram in your browser or run multiple instances of Instaloader "
+                            "in parallel.")
+            try:
+                if isinstance(err, TooManyRequestsException):
+                    print(textwrap.fill(text_for_429), file=sys.stderr)
+                    if is_graphql_query:
+                        waittime = graphql_query_waittime(untracked_queries=True)
+                        if waittime > 0:
+                            self.log('The request will be retried in {} seconds.'.format(waittime))
+                            time.sleep(waittime)
+                self._sleep()
+                return self.get_json(path=path, params=params, host=host, session=sess, _attempt=_attempt + 1)
+            except KeyboardInterrupt:
+                self.error("[skipped by user]", repeat_at_end=False)
+                raise ConnectionException(error_string) from err
+
+    def graphql_query(self, query_hash: str, variables: Dict[str, Any],
+                      referer: Optional[str] = None, rhx_gis: Optional[str] = None) -> Dict[str, Any]:
+        """
+        Do a GraphQL Query.
+
+        :param query_hash: Query identifying hash.
+        :param variables: Variables for the Query.
+        :param referer: HTTP Referer, or None.
+        :param rhx_gis: 'rhx_gis' variable as somewhere returned by Instagram, needed to 'sign' request
+        :return: The server's response dictionary.
+        """
+        with copy_session(self._session) as tmpsession:
+            tmpsession.headers.update(self._default_http_header(empty_session_only=True))
+            del tmpsession.headers['Connection']
+            del tmpsession.headers['Content-Length']
+            tmpsession.headers['authority'] = 'www.instagram.com'
+            tmpsession.headers['scheme'] = 'https'
+            tmpsession.headers['accept'] = '*/*'
+            if referer is not None:
+                tmpsession.headers['referer'] = urllib.parse.quote(referer)
+
+            variables_json = json.dumps(variables, separators=(',', ':'))
+
+            if rhx_gis:
+                #self.log("rhx_gis {} query_hash {}".format(rhx_gis, query_hash))
+                values = "{}:{}".format(rhx_gis, variables_json)
+                x_instagram_gis = hashlib.md5(values.encode()).hexdigest()
+                tmpsession.headers['x-instagram-gis'] = x_instagram_gis
+
+            resp_json = self.get_json('graphql/query',
+                                      params={'query_hash': query_hash,
+                                              'variables': variables_json},
+                                      session=tmpsession)
+        if 'status' not in resp_json:
+            self.error("GraphQL response did not contain a \"status\" field.")
+        return resp_json
+
+    def graphql_node_list(self, query_hash: str, query_variables: Dict[str, Any],
+                          query_referer: Optional[str],
+                          edge_extractor: Callable[[Dict[str, Any]], Dict[str, Any]],
+                          rhx_gis: Optional[str] = None,
+                          first_data: Optional[Dict[str, Any]] = None) -> Iterator[Dict[str, Any]]:
+        """Retrieve a list of GraphQL nodes."""
+
+        def _query():
+            query_variables['first'] = self._graphql_page_length
+            try:
+                return edge_extractor(self.graphql_query(query_hash, query_variables, query_referer, rhx_gis))
+            except QueryReturnedBadRequestException:
+                new_page_length = int(self._graphql_page_length / 2)
+                if new_page_length >= 12:
+                    self._graphql_page_length = new_page_length
+                    self.error("HTTP Error 400 (Bad Request) on GraphQL Query. Retrying with shorter page length.",
+                               repeat_at_end=False)
+                    return _query()
+                else:
+                    raise
+
+        if first_data:
+            data = first_data
+        else:
+            data = _query()
+        yield from (edge['node'] for edge in data['edges'])
+        while data['page_info']['has_next_page']:
+            query_variables['after'] = data['page_info']['end_cursor']
+            data = _query()
+            yield from (edge['node'] for edge in data['edges'])
+
+    def get_and_write_raw(self, url: str, filename: str, _attempt=1) -> None:
+        """Downloads raw data.
+
+        :raises QueryReturnedNotFoundException: When the server responds with a 404.
+        :raises QueryReturnedForbiddenException: When the server responds with a 403.
+        :raises ConnectionException: When download repeatedly failed."""
+        try:
+            with self.get_anonymous_session() as anonymous_session:
+                resp = anonymous_session.get(url, stream=True)
+            if resp.status_code == 200:
+                self.log(filename, end=' ', flush=True)
+                with open(filename, 'wb') as file:
+                    resp.raw.decode_content = True
+                    shutil.copyfileobj(resp.raw, file)
+            else:
+                if resp.status_code == 403:
+                    # suspected invalid URL signature
+                    raise QueryReturnedForbiddenException("403 when accessing {}.".format(url))
+                if resp.status_code == 404:
+                    # 404 not worth retrying.
+                    raise QueryReturnedNotFoundException("404 when accessing {}.".format(url))
+                raise ConnectionException("HTTP error code {}.".format(resp.status_code))
+        except (urllib3.exceptions.HTTPError, requests.exceptions.RequestException, ConnectionException) as err:
+            error_string = "URL {}: {}".format(url, err)
+            if _attempt == self.max_connection_attempts:
+                raise ConnectionException(error_string) from err
+            self.error(error_string + " [retrying; skip with ^C]", repeat_at_end=False)
+            try:
+                self._sleep()
+                self.get_and_write_raw(url, filename, _attempt + 1)
+            except KeyboardInterrupt:
+                self.error("[skipped by user]", repeat_at_end=False)
+                raise ConnectionException(error_string) from err

instaloader/structures.py

@@ -0,0 +1,834 @@
+import json
+import lzma
+import re
+from base64 import b64decode, b64encode
+from collections import namedtuple
+from datetime import datetime
+from typing import Any, Dict, Iterator, List, Optional, Union
+
+from . import __version__
+from .exceptions import *
+from .instaloadercontext import InstaloaderContext
+
+
+PostSidecarNode = namedtuple('PostSidecarNode', ['is_video', 'display_url', 'video_url'])
+PostComment = namedtuple('PostComment', ['id', 'created_at_utc', 'text', 'owner'])
+PostLocation = namedtuple('PostLocation', ['id', 'name', 'slug', 'has_public_page', 'lat', 'lng'])
+
+
+class Post:
+    """
+    Structure containing information about an Instagram post.
+
+    Created by methods :meth:`Profile.get_posts`, :meth:`Instaloader.get_hashtag_posts`,
+    :meth:`Instaloader.get_feed_posts` and :meth:`Profile.get_saved_posts`, which return iterators of Posts::
+
+       L = Instaloader()
+       for post in L.get_hashtag_posts(HASHTAG):
+           L.download_post(post, target='#'+HASHTAG)
+
+    Might also be created with::
+
+       post = Post.from_shortcode(L.context, SHORTCODE)
+
+    This class unifies access to the properties associated with a post. It implements == and is
+    hashable.
+
+    :param context: :attr:`Instaloader.context` used for additional queries if neccessary..
+    :param node: Node structure, as returned by Instagram.
+    :param owner_profile: The Profile of the owner, if already known at creation.
+    """
+
+    def __init__(self, context: InstaloaderContext, node: Dict[str, Any],
+                 owner_profile: Optional['Profile'] = None):
+        assert 'shortcode' in node or 'code' in node
+
+        self._context = context
+        self._node = node
+        self._owner_profile = owner_profile
+        self._full_metadata_dict = None
+        self._rhx_gis_str = None
+        self._location = None
+
+    @classmethod
+    def from_shortcode(cls, context: InstaloaderContext, shortcode: str):
+        """Create a post object from a given shortcode"""
+        # pylint:disable=protected-access
+        post = cls(context, {'shortcode': shortcode})
+        post._node = post._full_metadata
+        return post
+
+    @classmethod
+    def from_mediaid(cls, context: InstaloaderContext, mediaid: int):
+        """Create a post object from a given mediaid"""
+        return cls.from_shortcode(context, Post.mediaid_to_shortcode(mediaid))
+
+    @staticmethod
+    def shortcode_to_mediaid(code: str) -> int:
+        if len(code) > 11:
+            raise InvalidArgumentException("Wrong shortcode \"{0}\", unable to convert to mediaid.".format(code))
+        code = 'A' * (12 - len(code)) + code
+        return int.from_bytes(b64decode(code.encode(), b'-_'), 'big')
+
+    @staticmethod
+    def mediaid_to_shortcode(mediaid: int) -> str:
+        if mediaid.bit_length() > 64:
+            raise InvalidArgumentException("Wrong mediaid {0}, unable to convert to shortcode".format(str(mediaid)))
+        return b64encode(mediaid.to_bytes(9, 'big'), b'-_').decode().replace('A', ' ').lstrip().replace(' ', 'A')
+
+    def _asdict(self):
+        if self._full_metadata_dict:
+            node = self._full_metadata_dict
+        else:
+            node = self._node
+        if self._owner_profile:
+            node['owner'] = self.owner_profile._asdict()
+        if self._location:
+            node['location'] = self._location._asdict()
+        return node
+
+    @property
+    def shortcode(self) -> str:
+        """Media shortcode. URL of the post is instagram.com/p/<shortcode>/."""
+        return self._node['shortcode'] if 'shortcode' in self._node else self._node['code']
+
+    @property
+    def mediaid(self) -> int:
+        """The mediaid is a decimal representation of the media shortcode."""
+        return int(self._node['id'])
+
+    def __repr__(self):
+        return '<Post {}>'.format(self.shortcode)
+
+    def __eq__(self, o: object) -> bool:
+        if isinstance(o, Post):
+            return self.shortcode == o.shortcode
+        return NotImplemented
+
+    def __hash__(self) -> int:
+        return hash(self.shortcode)
+
+    def _obtain_metadata(self):
+        if not self._full_metadata_dict:
+            pic_json = self._context.get_json("p/{0}/".format(self.shortcode), params={})
+            self._full_metadata_dict = pic_json['entry_data']['PostPage'][0]['graphql']['shortcode_media']
+            self._rhx_gis_str = pic_json['rhx_gis']
+
+    @property
+    def _full_metadata(self) -> Dict[str, Any]:
+        self._obtain_metadata()
+        return self._full_metadata_dict
+
+    @property
+    def _rhx_gis(self) -> str:
+        self._obtain_metadata()
+        return self._rhx_gis_str
+
+    def _field(self, *keys) -> Any:
+        """Lookups given fields in _node, and if not found in _full_metadata. Raises KeyError if not found anywhere."""
+        try:
+            d = self._node
+            for key in keys:
+                d = d[key]
+            return d
+        except KeyError:
+            d = self._full_metadata
+            for key in keys:
+                d = d[key]
+            return d
+
+    @property
+    def owner_profile(self) -> 'Profile':
+        if not self._owner_profile:
+            if 'username' in self._node['owner']:
+                owner_struct = self._node['owner']
+            else:
+                # Sometimes, the 'owner' structure does not contain the username, only the user's ID.  In that case,
+                # this call triggers downloading of the complete Post metadata struct, where the owner username
+                # is contained. This is better than to get the username by user ID, since it is possible anonymously
+                # and gives us other information that is more likely to be usable.
+                owner_struct = self._full_metadata['owner']
+            if 'username' in owner_struct:
+                self._owner_profile = Profile(self._context, owner_struct)
+            else:
+                # Fallback, if we still did not get the owner username
+                self._owner_profile = Profile.from_id(self._context, owner_struct['id'])
+        return self._owner_profile
+
+    @property
+    def owner_username(self) -> str:
+        """The Post's lowercase owner name."""
+        return self.owner_profile.username
+
+    @property
+    def owner_id(self) -> int:
+        """The ID of the Post's owner."""
+        return self.owner_profile.userid
+
+    @property
+    def date_local(self) -> datetime:
+        """Timestamp when the post was created (local time zone)."""
+        return datetime.fromtimestamp(self._node["date"] if "date" in self._node else self._node["taken_at_timestamp"])
+
+    @property
+    def date_utc(self) -> datetime:
+        """Timestamp when the post was created (UTC)."""
+        return datetime.utcfromtimestamp(self._node["date"] if "date" in self._node else self._node["taken_at_timestamp"])
+
+    @property
+    def date(self) -> datetime:
+        """Synonym to :attr:`~Post.date_utc`"""
+        return self.date_utc
+
+    @property
+    def profile(self) -> str:
+        """Synonym to :attr:`~Post.owner_username`"""
+        return self.owner_username
+
+    @property
+    def url(self) -> str:
+        """URL of the picture / video thumbnail of the post"""
+        return self._node["display_url"] if "display_url" in self._node else self._node["display_src"]
+
+    @property
+    def typename(self) -> str:
+        """Type of post, GraphImage, GraphVideo or GraphSidecar"""
+        if '__typename' in self._node:
+            return self._node['__typename']
+        # if __typename is not in node, it is an old image or video
+        return 'GraphImage'
+
+    def get_sidecar_nodes(self) -> Iterator[PostSidecarNode]:
+        """Sidecar nodes of a Post with typename==GraphSidecar."""
+        if self.typename == 'GraphSidecar':
+            for edge in self._field('edge_sidecar_to_children', 'edges'):
+                node = edge['node']
+                is_video = node['is_video']
+                yield PostSidecarNode(is_video=is_video, display_url=node['display_url'],
+                                      video_url=node['video_url'] if is_video else None)
+
+    @property
+    def caption(self) -> Optional[str]:
+        """Caption."""
+        if "edge_media_to_caption" in self._node and self._node["edge_media_to_caption"]["edges"]:
+            return self._node["edge_media_to_caption"]["edges"][0]["node"]["text"]
+        elif "caption" in self._node:
+            return self._node["caption"]
+
+    @property
+    def caption_hashtags(self) -> List[str]:
+        """List of all lowercased hashtags (without preceeding #) that occur in the Post's caption."""
+        if not self.caption:
+            return []
+        # This regular expression is from jStassen, adjusted to use Python's \w to support Unicode
+        # http://blog.jstassen.com/2016/03/code-regex-for-instagram-username-and-hashtags/
+        hashtag_regex = re.compile(r"(?:#)(\w(?:(?:\w|(?:\.(?!\.))){0,28}(?:\w))?)")
+        return re.findall(hashtag_regex, self.caption.lower())
+
+    @property
+    def caption_mentions(self) -> List[str]:
+        """List of all lowercased profiles that are mentioned in the Post's caption, without preceeding @."""
+        if not self.caption:
+            return []
+        # This regular expression is from jStassen, adjusted to use Python's \w to support Unicode
+        # http://blog.jstassen.com/2016/03/code-regex-for-instagram-username-and-hashtags/
+        mention_regex = re.compile(r"(?:@)(\w(?:(?:\w|(?:\.(?!\.))){0,28}(?:\w))?)")
+        return re.findall(mention_regex, self.caption.lower())
+
+    @property
+    def tagged_users(self) -> List[str]:
+        """List of all lowercased users that are tagged in the Post."""
+        try:
+            return [edge['node']['user']['username'].lower() for edge in self._field('edge_media_to_tagged_user',
+                                                                                     'edges')]
+        except KeyError:
+            return []
+
+    @property
+    def is_video(self) -> bool:
+        """True if the Post is a video."""
+        return self._node['is_video']
+
+    @property
+    def video_url(self) -> Optional[str]:
+        """URL of the video, or None."""
+        if self.is_video:
+            return self._field('video_url')
+
+    @property
+    def viewer_has_liked(self) -> Optional[bool]:
+        """Whether the viewer has liked the post, or None if not logged in."""
+        if not self._context.is_logged_in:
+            return None
+        if 'likes' in self._node and 'viewer_has_liked' in self._node['likes']:
+            return self._node['likes']['viewer_has_liked']
+        return self._field('viewer_has_liked')
+
+    @property
+    def likes(self) -> int:
+        """Likes count"""
+        return self._field('edge_media_preview_like', 'count')
+
+    @property
+    def comments(self) -> int:
+        """Comment count"""
+        return self._field('edge_media_to_comment', 'count')
+
+    def get_comments(self) -> Iterator[PostComment]:
+        """Iterate over all comments of the post.
+
+        Each comment is represented by a PostComment namedtuple with fields text (string), created_at (datetime),
+        id (int) and owner (:class:`Profile`).
+        """
+        def _postcomment(node):
+            return PostComment(id=int(node['id']),
+                               created_at_utc=datetime.utcfromtimestamp(node['created_at']),
+                               text=node['text'],
+                               owner=Profile(self._context, node['owner']))
+        if self.comments == 0:
+            # Avoid doing additional requests if there are no comments
+            return
+        comment_edges = self._field('edge_media_to_comment', 'edges')
+        if self.comments == len(comment_edges):
+            # If the Post's metadata already contains all comments, don't do GraphQL requests to obtain them
+            yield from (_postcomment(comment['node']) for comment in comment_edges)
+            return
+        yield from (_postcomment(node) for node in
+                    self._context.graphql_node_list("33ba35852cb50da46f5b5e889df7d159",
+                                                    {'shortcode': self.shortcode},
+                                                    'https://www.instagram.com/p/' + self.shortcode + '/',
+                                                    lambda d: d['data']['shortcode_media']['edge_media_to_comment'],
+                                                    self._rhx_gis))
+
+    def get_likes(self) -> Iterator['Profile']:
+        """Iterate over all likes of the post. A :class:`Profile` instance of each likee is yielded."""
+        if self.likes == 0:
+            # Avoid doing additional requests if there are no comments
+            return
+        likes_edges = self._field('edge_media_preview_like', 'edges')
+        if self.likes == len(likes_edges):
+            # If the Post's metadata already contains all likes, don't do GraphQL requests to obtain them
+            yield from (Profile(self._context, like['node']) for like in likes_edges)
+            return
+        yield from (Profile(self._context, node) for node in
+                    self._context.graphql_node_list("1cb6ec562846122743b61e492c85999f", {'shortcode': self.shortcode},
+                                                    'https://www.instagram.com/p/' + self.shortcode + '/',
+                                                    lambda d: d['data']['shortcode_media']['edge_liked_by'],
+                                                    self._rhx_gis))
+
+    @property
+    def location(self) -> Optional[PostLocation]:
+        """If the Post has a location, returns PostLocation namedtuple with fields 'id', 'lat' and 'lng' and 'name'."""
+        loc = self._field("location")
+        if self._location or not loc:
+            return self._location
+        location_id = int(loc['id'])
+        if any(k not in loc for k in ('name', 'slug', 'has_public_page', 'lat', 'lng')):
+            loc = self._context.get_json("explore/locations/{0}/".format(location_id),
+                                         params={'__a': 1})['graphql']['location']
+        self._location = PostLocation(location_id, loc['name'], loc['slug'], loc['has_public_page'],
+                                      loc['lat'], loc['lng'])
+        return self._location
+
+
+class Profile:
+    """
+    An Instagram Profile.
+
+    Provides methods for accessing profile properties, as well as :meth:`Profile.get_posts` and for own profile
+    :meth:`Profile.get_saved_posts`.
+
+    Get instances with :meth:`Post.owner_profile`, :meth:`StoryItem.owner_profile`, :meth:`Profile.get_followees`,
+    :meth:`Profile.get_followers` or::
+
+       L = Instaloader()
+       profile = Profile.from_username(L.context, USERNAME)
+
+    Provides :meth:`Profile.get_posts` and for own profile :meth:`Profile.get_saved_posts` to iterate over associated
+    :class:`Post` objects::
+
+       for post in profile.get_posts():
+           L.download_post(post, target=profile.username)
+
+    :meth:`Profile.get_followees` and :meth:`Profile.get_followers`::
+
+       print("{} follows these profiles:".format(profile.username))
+       for followee in profile.get_followees():
+           print(followee.username)
+
+    Also, this class implements == and is hashable.
+    """
+    def __init__(self, context: InstaloaderContext, node: Dict[str, Any]):
+        assert 'username' in node
+        self._context = context
+        self._node = node
+        self._rhx_gis = None
+
+    @classmethod
+    def from_username(cls, context: InstaloaderContext, username: str):
+        """Create a Profile instance from a given username, raise exception if it does not exist.
+
+        :param context: :attr:`Instaloader.context`
+        :param username: Username
+        :raises: :class:`ProfileNotExistsException`
+        """
+        # pylint:disable=protected-access
+        profile = cls(context, {'username': username.lower()})
+        profile._obtain_metadata()  # to raise ProfileNotExistException now in case username is invalid
+        return profile
+
+    @classmethod
+    def from_id(cls, context: InstaloaderContext, profile_id: int):
+        """If logged in, create a Profile instance from a given userid. If possible, use :meth:`Profile.from_username`
+        or constructor directly rather than this method, since does many requests.
+
+        :param context: :attr:`Instaloader.context`
+        :param profile_id: userid
+        :raises: :class:`ProfileNotExistsException`, :class:`LoginRequiredException`, :class:`ProfileHasNoPicsException`
+        """
+        if not context.is_logged_in:
+            raise LoginRequiredException("--login=USERNAME required to obtain profile metadata from its ID number.")
+        data = context.graphql_query("472f257a40c653c64c666ce877d59d2b",
+                                     {'id': str(profile_id), 'first': 1})['data']['user']
+        if data:
+            data = data["edge_owner_to_timeline_media"]
+        else:
+            raise ProfileNotExistsException("No profile found, the user may have blocked you (ID: " +
+                                            str(profile_id) + ").")
+        if not data['edges']:
+            if data['count'] == 0:
+                raise ProfileHasNoPicsException("Profile with ID {0}: no pics found.".format(str(profile_id)))
+            else:
+                raise LoginRequiredException("Login required to determine username (ID: " + str(profile_id) + ").")
+        username = Post.from_mediaid(context, int(data['edges'][0]["node"]["id"])).owner_username
+        return cls(context, {'username': username.lower(), 'id': profile_id})
+
+    def _asdict(self):
+        json_node = self._node.copy()
+        # remove posts
+        json_node.pop('edge_media_collections', None)
+        json_node.pop('edge_owner_to_timeline_media', None)
+        json_node.pop('edge_saved_media', None)
+        return json_node
+
+    def _obtain_metadata(self):
+        try:
+            if not self._rhx_gis:
+                metadata = self._context.get_json('{}/'.format(self.username), params={})
+                self._node = metadata['entry_data']['ProfilePage'][0]['graphql']['user']
+                self._rhx_gis = metadata['rhx_gis']
+        except QueryReturnedNotFoundException:
+            raise ProfileNotExistsException('Profile {} does not exist.'.format(self.username))
+
+    def _metadata(self, *keys) -> Any:
+        try:
+            d = self._node
+            for key in keys:
+                d = d[key]
+            return d
+        except KeyError:
+            self._obtain_metadata()
+            d = self._node
+            for key in keys:
+                d = d[key]
+            return d
+
+    @property
+    def userid(self) -> int:
+        """User ID"""
+        return int(self._metadata('id'))
+
+    @property
+    def username(self) -> str:
+        """Profile Name"""
+        return self._metadata('username').lower()
+
+    def __repr__(self):
+        return '<Profile {} ({})>'.format(self.username, self.userid)
+
+    def __eq__(self, o: object) -> bool:
+        if isinstance(o, Profile):
+            return self.userid == o.userid
+        return NotImplemented
+
+    def __hash__(self) -> int:
+        return hash(self.userid)
+
+    @property
+    def is_private(self) -> bool:
+        return self._metadata('is_private')
+
+    @property
+    def followed_by_viewer(self) -> bool:
+        return self._metadata('followed_by_viewer')
+
+    @property
+    def mediacount(self) -> int:
+        return self._metadata('edge_owner_to_timeline_media', 'count')
+
+    @property
+    def biography(self) -> str:
+        return self._metadata('biography')
+
+    @property
+    def blocked_by_viewer(self) -> bool:
+        return self._metadata('blocked_by_viewer')
+
+    @property
+    def follows_viewer(self) -> bool:
+        return self._metadata('follows_viewer')
+
+    @property
+    def full_name(self) -> str:
+        return self._metadata('full_name')
+
+    @property
+    def has_blocked_viewer(self) -> bool:
+        return self._metadata('has_blocked_viewer')
+
+    @property
+    def has_requested_viewer(self) -> bool:
+        return self._metadata('has_requested_viewer')
+
+    @property
+    def is_verified(self) -> bool:
+        return self._metadata('is_verified')
+
+    @property
+    def requested_by_viewer(self) -> bool:
+        return self._metadata('requested_by_viewer')
+
+    def get_profile_pic_url(self) -> str:
+        """Return URL of profile picture"""
+        try:
+            with self._context.get_anonymous_session() as anonymous_session:
+                data = self._context.get_json(path='api/v1/users/{0}/info/'.format(self.userid), params={},
+                                              host='i.instagram.com', session=anonymous_session)
+            return data["user"]["hd_profile_pic_url_info"]["url"]
+        except (InstaloaderException, KeyError) as err:
+            self._context.error('{} Unable to fetch high quality profile pic.'.format(err))
+            return self._metadata("profile_pic_url_hd")
+
+    def get_posts(self) -> Iterator[Post]:
+        """Retrieve all posts from a profile."""
+        self._obtain_metadata()
+        yield from (Post(self._context, node, self) for node in
+                    self._context.graphql_node_list("472f257a40c653c64c666ce877d59d2b",
+                                                    {'id': self.userid},
+                                                    'https://www.instagram.com/{0}/'.format(self.username),
+                                                    lambda d: d['data']['user']['edge_owner_to_timeline_media'],
+                                                    self._rhx_gis,
+                                                    self._metadata('edge_owner_to_timeline_media')))
+
+    def get_saved_posts(self) -> Iterator[Post]:
+        """Get Posts that are marked as saved by the user."""
+
+        if self.username != self._context.username:
+            raise LoginRequiredException("--login={} required to get that profile's saved posts.".format(self.username))
+
+        self._obtain_metadata()
+        yield from (Post(self._context, node) for node in
+                    self._context.graphql_node_list("f883d95537fbcd400f466f63d42bd8a1",
+                                                    {'id': self.userid},
+                                                    'https://www.instagram.com/{0}/'.format(self.username),
+                                                    lambda d: d['data']['user']['edge_saved_media'],
+                                                    self._rhx_gis,
+                                                    self._metadata('edge_saved_media')))
+
+    def get_followers(self) -> Iterator['Profile']:
+        """
+        Retrieve list of followers of given profile.
+        To use this, one needs to be logged in and private profiles has to be followed.
+        """
+        if not self._context.is_logged_in:
+            raise LoginRequiredException("--login required to get a profile's followers.")
+        self._obtain_metadata()
+        yield from (Profile(self._context, node) for node in
+                    self._context.graphql_node_list("37479f2b8209594dde7facb0d904896a",
+                                                    {'id': str(self.userid)},
+                                                    'https://www.instagram.com/' + self.username + '/',
+                                                    lambda d: d['data']['user']['edge_followed_by'],
+                                                    self._rhx_gis))
+
+    def get_followees(self) -> Iterator['Profile']:
+        """
+        Retrieve list of followees (followings) of given profile.
+        To use this, one needs to be logged in and private profiles has to be followed.
+        """
+        if not self._context.is_logged_in:
+            raise LoginRequiredException("--login required to get a profile's followees.")
+        self._obtain_metadata()
+        yield from (Profile(self._context, node) for node in
+                    self._context.graphql_node_list("58712303d941c6855d4e888c5f0cd22f",
+                                                    {'id': str(self.userid)},
+                                                    'https://www.instagram.com/' + self.username + '/',
+                                                    lambda d: d['data']['user']['edge_follow'],
+                                                    self._rhx_gis))
+
+
+class StoryItem:
+    """
+    Structure containing information about a user story item i.e. image or video.
+
+    Created by method :meth:`Story.get_items`. This class implements == and is hashable.
+
+    :param context: :class:`InstaloaderContext` instance used for additional queries if necessary.
+    :param node: Dictionary containing the available information of the story item.
+    :param owner_profile: :class:`Profile` instance representing the story owner.
+    """
+
+    def __init__(self, context: InstaloaderContext, node: Dict[str, Any], owner_profile: Optional[Profile] = None):
+        self._context = context
+        self._node = node
+        self._owner_profile = owner_profile
+
+    def _asdict(self):
+        node = self._node
+        if self._owner_profile:
+            node['owner'] = self._owner_profile._asdict()
+        return node
+
+    @property
+    def mediaid(self) -> int:
+        """The mediaid is a decimal representation of the media shortcode."""
+        return int(self._node['id'])
+
+    @property
+    def shortcode(self) -> str:
+        """Convert :attr:`~StoryItem.mediaid` to a shortcode-like string, allowing ``{shortcode}`` to be used with
+        :option:`--filename-pattern`."""
+        return Post.mediaid_to_shortcode(self.mediaid)
+
+    def __repr__(self):
+        return '<StoryItem {}>'.format(self.mediaid)
+
+    def __eq__(self, o: object) -> bool:
+        if isinstance(o, StoryItem):
+            return self.mediaid == o.mediaid
+        return NotImplemented
+
+    def __hash__(self) -> int:
+        return hash(self.mediaid)
+
+    @property
+    def owner_profile(self) -> Profile:
+        """:class:`Profile` instance of the story item's owner."""
+        if not self._owner_profile:
+            self._owner_profile = Profile.from_id(self._context, self._node['owner']['id'])
+        return self._owner_profile
+
+    @property
+    def owner_username(self) -> str:
+        """The StoryItem owner's lowercase name."""
+        return self.owner_profile.username
+
+    @property
+    def owner_id(self) -> int:
+        """The ID of the StoryItem owner."""
+        return self.owner_profile.userid
+
+    @property
+    def date_local(self) -> datetime:
+        """Timestamp when the StoryItem was created (local time zone)."""
+        return datetime.fromtimestamp(self._node['taken_at_timestamp'])
+
+    @property
+    def date_utc(self) -> datetime:
+        """Timestamp when the StoryItem was created (UTC)."""
+        return datetime.utcfromtimestamp(self._node['taken_at_timestamp'])
+
+    @property
+    def date(self) -> datetime:
+        """Synonym to :attr:`~StoryItem.date_utc`"""
+        return self.date_utc
+
+    @property
+    def profile(self) -> str:
+        """Synonym to :attr:`~StoryItem.owner_username`"""
+        return self.owner_username
+
+    @property
+    def expiring_local(self) -> datetime:
+        """Timestamp when the StoryItem will get unavailable (local time zone)."""
+        return datetime.fromtimestamp(self._node['expiring_at_timestamp'])
+
+    @property
+    def expiring_utc(self) -> datetime:
+        """Timestamp when the StoryItem will get unavailable (UTC)."""
+        return datetime.utcfromtimestamp(self._node['expiring_at_timestamp'])
+
+    @property
+    def url(self) -> str:
+        """URL of the picture / video thumbnail of the StoryItem"""
+        return self._node['display_resources'][-1]['src']
+
+    @property
+    def typename(self) -> str:
+        """Type of post, GraphStoryImage or GraphStoryVideo"""
+        return self._node['__typename']
+
+    @property
+    def is_video(self) -> bool:
+        """True if the StoryItem is a video."""
+        return self._node['is_video']
+
+    @property
+    def video_url(self) -> Optional[str]:
+        """URL of the video, or None."""
+        if self.is_video:
+            return self._node['video_resources'][-1]['src']
+
+
+class Story:
+    """
+    Structure representing a user story with its associated items.
+
+    Provides methods for accessing story properties, as well as :meth:`Story.get_items` to request associated
+    :class:`StoryItem` nodes. Stories are returned by :meth:`Instaloader.get_stories`.
+
+    With a logged-in :class:`Instaloader` instance `L`, you may download all your visible user stories with::
+
+       for story in L.get_stories():
+           # story is a Story object
+           for item in story.get_items():
+               # item is a StoryItem object
+               L.download_storyitem(item, ':stores')
+
+    This class implements == and is hashable.
+
+    :param context: :class:`InstaloaderContext` instance used for additional queries if necessary.
+    :param node: Dictionary containing the available information of the story as returned by Instagram.
+    """
+
+    def __init__(self, context: InstaloaderContext, node: Dict[str, Any]):
+        self._context = context
+        self._node = node
+        self._unique_id = None
+        self._owner_profile = None
+
+    def __repr__(self):
+        return '<Story by {} changed {:%Y-%m-%d_%H-%M-%S_UTC}>'.format(self.owner_username, self.latest_media_utc)
+
+    def __eq__(self, o: object) -> bool:
+        if isinstance(o, Story):
+            return self.unique_id == o.unique_id
+        return NotImplemented
+
+    def __hash__(self) -> int:
+        return hash(self._unique_id)
+
+    @property
+    def unique_id(self) -> str:
+        """
+        This ID only equals amongst :class:`Story` instances which have the same owner and the same set of
+        :class:`StoryItem`s. For all other :class:`Story` instances this ID is different.
+        """
+        if not self._unique_id:
+            id_list = [item.mediaid for item in self.get_items()]
+            id_list.sort()
+            self._unique_id = str().join([str(self.owner_id)] + list(map(str, id_list)))
+        return self._unique_id
+
+    @property
+    def last_seen_local(self) -> Optional[datetime]:
+        """Timestamp when the story has last been watched or None (local time zone)."""
+        if self._node['seen']:
+            return datetime.fromtimestamp(self._node['seen'])
+
+    @property
+    def last_seen_utc(self) -> Optional[datetime]:
+        """Timestamp when the story has last been watched or None (UTC)."""
+        if self._node['seen']:
+            return datetime.utcfromtimestamp(self._node['seen'])
+
+    @property
+    def latest_media_local(self) -> datetime:
+        """Timestamp when the last item of the story was created (local time zone)."""
+        return datetime.fromtimestamp(self._node['latest_reel_media'])
+
+    @property
+    def latest_media_utc(self) -> datetime:
+        """Timestamp when the last item of the story was created (UTC)."""
+        return datetime.utcfromtimestamp(self._node['latest_reel_media'])
+
+    @property
+    def itemcount(self) -> int:
+        """Count of items associated with the :class:`Story` instance."""
+        return len(self._node['items'])
+
+    @property
+    def owner_profile(self) -> Profile:
+        """:class:`Profile` instance of the story owner."""
+        if not self._owner_profile:
+            self._owner_profile = Profile(self._context, self._node['user'])
+        return self._owner_profile
+
+    @property
+    def owner_username(self) -> str:
+        """The story owner's lowercase username."""
+        return self.owner_profile.username
+
+    @property
+    def owner_id(self) -> int:
+        """The story owner's ID."""
+        return self.owner_profile.userid
+
+    def get_items(self) -> Iterator[StoryItem]:
+        """Retrieve all items from a story."""
+        yield from (StoryItem(self._context, item, self.owner_profile) for item in reversed(self._node['items']))
+
+
+JsonExportable = Union[Post, Profile, StoryItem]
+
+
+def save_structure_to_file(structure: JsonExportable, filename: str) -> None:
+    """Saves a :class:`Post`, :class:`Profile` or :class:`StoryItem` to a '.json' or '.json.xz' file such that it can
+    later be loaded by :func:`load_structure_from_file`.
+
+    If the specified filename ends in '.xz', the file will be LZMA compressed. Otherwise, a pretty-printed JSON file
+    will be created.
+
+    :param structure: :class:`Post`, :class:`Profile` or :class:`StoryItem`
+    :param filename: Filename, ends in '.json' or '.json.xz'
+    """
+    json_structure = {'node': structure._asdict(),
+                      'instaloader': {'version': __version__, 'node_type': structure.__class__.__name__}}
+    compress = filename.endswith('.xz')
+    if compress:
+        with lzma.open(filename, 'wt', check=lzma.CHECK_NONE) as fp:
+            json.dump(json_structure, fp=fp, separators=(',', ':'))
+    else:
+        with open(filename, 'wt') as fp:
+            json.dump(json_structure, fp=fp, indent=4, sort_keys=True)
+
+
+def load_structure_from_file(context: InstaloaderContext, filename: str) -> JsonExportable:
+    """Loads a :class:`Post`, :class:`Profile` or :class:`StoryItem` from a '.json' or '.json.xz' file that
+    has been saved by :func:`save_structure_from_file`.
+
+    :param context: :attr:`Instaloader.context` linked to the new object, used for additional queries if neccessary.
+    :param filename: Filename, ends in '.json' or '.json.xz'
+    """
+    compressed = filename.endswith('.xz')
+    if compressed:
+        fp = lzma.open(filename, 'rt')
+    else:
+        fp = open(filename, 'rt')
+    json_structure = json.load(fp)
+    fp.close()
+    if 'node' in json_structure and 'instaloader' in json_structure and \
+            'node_type' in json_structure['instaloader']:
+        node_type = json_structure['instaloader']['node_type']
+        if node_type == "Post":
+            return Post(context, json_structure['node'])
+        elif node_type == "Profile":
+            return Profile(context, json_structure['node'])
+        elif node_type == "StoryItem":
+            return StoryItem(context, json_structure['node'])
+        else:
+            raise InvalidArgumentException("{}: Not an Instaloader JSON.".format(filename))
+    elif 'shortcode' in json_structure:
+        # Post JSON created with Instaloader v3
+        return Post.from_shortcode(context, json_structure['shortcode'])
+    else:
+        raise InvalidArgumentException("{}: Not an Instaloader JSON.".format(filename))

setup.py

@@ -11,7 +11,7 @@ SRC = os.path.abspath(os.path.dirname(__file__))
 
 
 def get_version():
-    with open(os.path.join(SRC, 'instaloader.py')) as f:
+    with open(os.path.join(SRC, 'instaloader/__init__.py')) as f:
         for line in f:
             m = re.match("__version__ = '(.*)'", line)
             if m:
@@ -37,7 +37,7 @@ keywords = (['instagram', 'instagram-scraper', 'instagram-client', 'instagram-fe
 setup(
     name='instaloader',
     version=get_version(),
-    py_modules=['instaloader'],
+    packages=['instaloader'],
     url='https://instaloader.github.io/',
     license='MIT',
     author='Alexander Graf, André Koch-Kramer',
@@ -47,17 +47,19 @@ setup(
     long_description=open(os.path.join(SRC, 'README.rst')).read(),
     install_requires=requirements,
     python_requires='>=3.5',
-    entry_points={'console_scripts': ['instaloader=instaloader:main']},
+    entry_points={'console_scripts': ['instaloader=instaloader.__main__:main']},
     zip_safe=True,
     keywords=keywords,
     classifiers=[
         'Development Status :: 5 - Production/Stable',
         'Environment :: Console',
         'Intended Audience :: End Users/Desktop',
+        'Intended Audience :: Developers',
         'License :: OSI Approved :: MIT License',
         'Operating System :: OS Independent',
         'Programming Language :: Python :: 3.5',
         'Programming Language :: Python :: 3.6',
+        'Programming Language :: Python :: 3.7',
         'Programming Language :: Python :: 3 :: Only',
         'Topic :: Internet',
         'Topic :: Multimedia :: Graphics'

test/instaloader_unittests.py

@@ -0,0 +1,139 @@
+"""Unit Tests for Instaloader"""
+
+import os
+import shutil
+import tempfile
+import unittest
+
+import instaloader
+
+PUBLIC_PROFILE = "selenagomez"
+PUBLIC_PROFILE_ID = 460563723
+HASHTAG = "kitten"
+OWN_USERNAME = "aandergr"
+NORMAL_MAX_COUNT = 2
+PAGING_MAX_COUNT = 15
+PRIVATE_PROFILE = "aandergr"
+
+
+class TestInstaloaderAnonymously(unittest.TestCase):
+
+    def setUp(self):
+        self.dir = tempfile.mkdtemp()
+        print("Testing in {}".format(self.dir))
+        os.chdir(self.dir)
+        self.L = instaloader.Instaloader(download_geotags=True,
+                                         download_comments=True,
+                                         save_metadata=True)
+        self.L.context.raise_all_errors = True
+
+    def tearDown(self):
+        self.L.close()
+        os.chdir('/')
+        print("Removing {}".format(self.dir))
+        shutil.rmtree(self.dir)
+
+    @unittest.SkipTest
+    def test_public_profile_download(self):
+        self.L.download_profile(PUBLIC_PROFILE, profile_pic=False, fast_update=True)
+        self.L.download_profile(PUBLIC_PROFILE, profile_pic=False, fast_update=True)
+
+    def test_public_profile_paging(self):
+        for count, post in enumerate(instaloader.Profile.from_username(self.L.context, PUBLIC_PROFILE).get_posts()):
+            print(post)
+            if count == PAGING_MAX_COUNT:
+                break
+
+    def test_profile_pic_download(self):
+        self.L.download_profile(PUBLIC_PROFILE, profile_pic_only=True)
+
+    def test_hashtag_download(self):
+        self.L.download_hashtag(HASHTAG, NORMAL_MAX_COUNT)
+
+    def test_get_id_by_username(self):
+        self.assertEqual(PUBLIC_PROFILE_ID,
+                         instaloader.Profile.from_username(self.L.context, PUBLIC_PROFILE).userid)
+
+    def test_post_from_mediaid(self):
+        for post in instaloader.Profile.from_username(self.L.context, PUBLIC_PROFILE).get_posts():
+            post2 = instaloader.Post.from_mediaid(self.L.context, post.mediaid)
+            self.assertEqual(post, post2)
+            break
+
+
+class TestInstaloaderLoggedIn(TestInstaloaderAnonymously):
+
+    def setUp(self):
+        super().setUp()
+        self.L.load_session_from_file(OWN_USERNAME)
+
+    @unittest.SkipTest
+    def test_stories_download(self):
+        self.L.download_stories()
+
+    @unittest.SkipTest
+    def test_private_profile_download(self):
+        self.L.download_profile(PRIVATE_PROFILE, download_stories=True)
+
+    def test_stories_paging(self):
+        for user_story in self.L.get_stories():
+            print("profile {}.".format(user_story.owner_username))
+            for item in user_story.get_items():
+                print(item)
+
+    def test_private_profile_paging(self):
+        for count, post in enumerate(instaloader.Profile.from_username(self.L.context, PRIVATE_PROFILE).get_posts()):
+            print(post)
+            if count == PAGING_MAX_COUNT:
+                break
+
+    def test_feed_download(self):
+        self.L.download_feed_posts(NORMAL_MAX_COUNT)
+
+    def test_feed_paging(self):
+        for count, post in enumerate(self.L.get_feed_posts()):
+            print(post)
+            if count == PAGING_MAX_COUNT:
+                break
+
+    def test_saved_download(self):
+        self.L.download_saved_posts(NORMAL_MAX_COUNT)
+
+    def test_saved_paging(self):
+        for count, post in enumerate(instaloader.Profile.from_username(self.L.context, OWN_USERNAME).get_saved_posts()):
+            print(post)
+            if count == PAGING_MAX_COUNT:
+                break
+
+    def test_test_login(self):
+        self.assertEqual(OWN_USERNAME, self.L.test_login())
+
+    def test_get_followees(self):
+        profile = instaloader.Profile.from_username(self.L.context, OWN_USERNAME)
+        for f in profile.get_followees():
+            print(f.username)
+
+    def test_get_followers(self):
+        profile = instaloader.Profile.from_username(self.L.context, OWN_USERNAME)
+        for f in profile.get_followers():
+            print(f.username)
+
+    def test_get_username_by_id(self):
+        self.assertEqual(PUBLIC_PROFILE.lower(),
+                         instaloader.Profile.from_id(self.L.context, PUBLIC_PROFILE_ID).username)
+
+    def test_get_likes(self):
+        for post in instaloader.Profile.from_username(self.L.context, OWN_USERNAME).get_posts():
+            for like in post.get_likes():
+                print(like.username)
+            break
+
+    def test_explore_paging(self):
+        for count, post in enumerate(self.L.get_explore_posts()):
+            print(post)
+            if count == PAGING_MAX_COUNT:
+                break
+
+
+if __name__ == '__main__':
+    unittest.main()