Release of version 4.0.5

Motivated by #140.

.travis.yml

@@ -9,7 +9,7 @@ install:
  - pip install -r docs/requirements.txt
 script:
  - python3 -m pylint -r n -d bad-whitespace,missing-docstring,too-many-arguments,locally-disabled,line-too-long,too-many-public-methods,too-many-lines,too-many-instance-attributes,too-many-locals,too-many-branches,too-many-statements,inconsistent-return-statements,invalid-name,wildcard-import,unused-wildcard-import instaloader
- - make -C docs html
+ - make -C docs html SPHINXOPTS="-W -n"
 deploy:
   - provider: pypi
     user: aandergr

docs/_templates/navbar.html

@@ -50,6 +50,8 @@
                   <li><a href="https://github.com/instaloader/instaloader">Instaloader on GitHub</a></li>
                   <li role="separator" class="divider"></li>
                   <li style="padding-left: 20px;"><a class="github-button" href="https://github.com/instaloader/instaloader" data-icon="octicon-star" data-size="large" data-show-count="true" aria-label="Star Instaloader on GitHub">Star</a></li>
+                  <li role="separator" class="divider"></li>
+                  <li><a href="https://www.paypal.me/aandergr"><b>Donate (via Paypal)</b></a></li>
                 </ul>
               </li>
             </ul>

docs/as-module.rst

@@ -73,7 +73,7 @@ metadata of a Profile. :class:`Profile` instances can be created with:
 
      profile = Profile.from_username(L.context, USERNAME)
 
-- :meth:`Profile_from_userid`
+- :meth:`Profile.from_id`
    given its User ID. This allows to easily lookup a Profile's username given
    its ID::
 
@@ -97,6 +97,9 @@ metadata of a Profile. :class:`Profile` instances can be created with:
 A reference of the many methods provided by the :mod:`instaloader` module is
 provided in the remainder of this document.
 
+For a list of real code examples using the Instaloader module for advanced
+tasks, see :ref:`codesnippets`.
+
 ``Instaloader`` (Main Class)
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
@@ -116,15 +119,12 @@ Additionally, the following trivial structures are defined:
 
 .. autoclass:: PostSidecarNode
    :no-show-inheritance:
-   :no-members:
 
 .. autoclass:: PostComment
    :no-show-inheritance:
-   :no-members:
 
 .. autoclass:: PostLocation
    :no-show-inheritance:
-   :no-members:
 
 User Stories
 """"""""""""
@@ -154,9 +154,15 @@ to/from JSON files.
 Exceptions
 ^^^^^^^^^^
 
+.. currentmodule:: instaloader.exceptions
+
 .. autoexception:: InstaloaderException
    :no-show-inheritance:
 
+.. autoexception:: ConnectionException
+
+.. currentmodule:: instaloader
+
 .. autoexception:: QueryReturnedBadRequestException
 
 .. autoexception:: QueryReturnedNotFoundException
@@ -177,8 +183,6 @@ Exceptions
 
 .. autoexception:: BadCredentialsException
 
-.. autoexception:: ConnectionException
-
 .. autoexception:: TooManyRequestsException
 
 ``InstaloaderContext`` (Low-level functions)

docs/basic-usage.rst

@@ -68,7 +68,7 @@ Instaloader supports the following targets:
    **download the user's stories**.
 
 - ``"#hashtag"``
-   Posts with a certain **hashtag** (the quotes are usually neccessary),
+   Posts with a certain **hashtag** (the quotes are usually necessary),
 
 - ``:stories``
    The currently-visible **stories** of your followees (requires
@@ -158,7 +158,7 @@ Filter Posts
 .. py:currentmodule:: instaloader
 
 The options :option:`--post-filter` and :option:`--storyitem-filter`
-allows to specify criterias that posts or story items have to
+allows to specify criteria that posts or story items have to
 meet to be downloaded. If not given, all posts are downloaded.
 
 The filter string must be a
@@ -273,3 +273,6 @@ consider taking a look at the :ref:`python-module-instaloader`.
 Instaloader exposes its internally used methods and structures, making it a
 powerful and intuitive Python API for Instagram, allowing to further customize
 obtaining media and metadata.
+
+Also see :ref:`codesnippets`, where we collect example scripts that use
+Instaloader to achieve more complex tasks.

docs/cli-options.rst

@@ -88,6 +88,11 @@ automatically **finds it by its unique ID** and renames the folder likewise.
    download stories.  Requires :option:`--login`. Does not imply
    :option:`--no-profile-pic`.
 
+   .. note::
+
+      If possible, use ``:stories`` target rather than :option:`--stories-only`
+      with all your followees. ``:stories`` uses fewer API requests.
+
 .. option:: --post-filter filter, --only-if filter
 
    Expression that, if given, must evaluate to True for each post to be

docs/codesnippets.rst

@@ -0,0 +1,128 @@
+.. _codesnippets:
+
+Advanced Instaloader Examples
+=============================
+
+.. currentmodule:: instaloader
+
+.. highlight:: python
+
+.. contents::
+   :backlinks: none
+
+Here we present code examples that use the :ref:`python-module-instaloader` for
+more advanced Instagram downloading or metadata mining than what is possible
+with the Instaloader command line interface.
+
+.. For each code snippet:
+   - title
+   - brief description of what it does / motivation / how it works
+   - code, or link to code
+   - link to discussion issue
+   - link used methods
+
+Download Posts in a Specific Period
+-----------------------------------
+
+To only download Instagram pictures (and metadata) that are within a specific
+period, you can play around with :func:`~itertools.dropwhile` and
+:func:`~itertools.takewhile` from :mod:`itertools` like in this snippet.
+
+.. literalinclude:: codesnippets/121_since_until.py
+
+See also :class:`Post`, :meth:`Instaloader.download_post`.
+
+Discussed in :issue:`121`.
+
+Likes of a Profile / Ghost Followers
+------------------------------------
+
+To obtain a list of your inactive followers, i.e. followers that did not like
+any of your pictures, into a file you can use this approach.
+
+.. literalinclude:: codesnippets/120_ghost_followers.py
+
+See also :meth:`Profile.get_posts`, :meth:`Post.get_likes`,
+:meth:`Profile.get_followers`, :meth:`Instaloader.load_session_from_file`,
+:meth:`Profile.from_username`.
+
+Discussed in :issue:`120`.
+
+Track Deleted Posts
+-------------------
+
+This script uses Instaloader to obtain a list of currently-online Instagram and
+compares it with the set of posts that you already have downloaded.  It outputs
+a list of posts which are online but not offline (i.e. not yet downloaded) and a
+list of posts which are offline but not online (i.e. deleted in the profile).
+
+.. literalinclude:: codesnippets/56_track_deleted.py
+
+See also :func:`load_structure_from_file`, :meth:`Profile.from_username`,
+:meth:`Profile.get_posts`, :class:`Post`.
+
+Discussed in :issue:`56`.
+
+Only one Post per User
+----------------------
+
+To download only the one most recent post from each user, this snippet creates a
+:class:`set` that contains the users of which a post has already been
+downloaded. While iterating the posts, it checks whether the post's owner
+already is in the set. If not, the post is downloaded from Instagram and the
+user is added to that set.
+
+.. literalinclude:: codesnippets/113_only_one_per_user.py
+
+See also :class:`Post`, :meth:`Instaloader.download_post`,
+:attr:`Post.owner_profile`, :class:`Profile`.
+
+Discussed in :issue:`113`.
+
+Upgrade Images by Local Copies
+------------------------------
+
+The following script finds local versions of images fetched by Instaloader, in
+order to upgrade the downloaded images by locally-found versions with better
+quality. It uses image hashing to identify similar images.
+
+`updgrade-instaloader-images.py <https://gist.github.com/pavelkryukov/15f93d19a99428a284a8bcec27e0187b>`__ (external link to GitHub Gist)
+
+Discussed in :issue:`46`.
+
+Add Captions to Images
+----------------------
+
+Instaloader does not modify the downloaded JPEG file. However, one could combine
+it with an imaging library such as Pillow or PIL to render the caption on
+Instagram pictures.  The following shows an approach.
+
+.. literalinclude:: codesnippets/110_pil_captions.py
+
+See also :attr:`Post.caption`, :attr:`Post.url`, :meth:`Post.from_shortcode`,
+:func:`load_structure_from_file`.
+
+Discussed in :issue:`110`.
+
+Metadata JSON Files
+-------------------
+
+The JSON files Instaloader saves along with each Post contain all the metadata
+that has been retrieved from Instagram while downloading the picture and
+associated required information.
+
+With `jq <https://stedolan.github.io/jq/>`__, a command-line JSON processor, the
+metadata can be easily post-processed. For example, Instaloader's JSON files can
+be pretty-formatted with:
+
+.. code-block:: none
+
+   xzcat 2018-05-13_11-18-45_UTC.json.xz | jq .node
+
+However, Instaloader tries to do as few metadata requests as possible, so,
+depending on how Instaloader has been invoked, it may occur that these files do
+not contain the complete available metadata structure. Nevertheless, the file
+can be loaded into Instaloader with :func:`load_structure_from_file` and the
+required metadata then be accessed via the :class:`Post` or :class:`Profile`
+attributes, which trigger an Instagram request if that particular information is
+not present in the JSON file.

docs/codesnippets/110_pil_captions.py

@@ -0,0 +1,20 @@
+from io import BytesIO
+
+from requests import get
+from PIL import Image, ImageDraw
+from instaloader import *
+
+L = Instaloader()
+
+# Load Post instance
+post = load_structure_from_file(L.context, '2017-10-01_18-53-03_UTC.json.xz')
+# or post = Post.from_shortcode(L.context, SHORTCODE)
+
+# Render caption
+image = Image.open(BytesIO(get(post.url).content))
+draw = ImageDraw.Draw(image)
+color = 'rgb(0, 0, 0)'  # black color
+draw.text((300,100), post.caption.encode('latin1', errors='ignore'), fill=color)
+
+# Save image
+image.save('test.jpg')

docs/codesnippets/113_only_one_per_user.py

@@ -0,0 +1,14 @@
+import instaloader
+
+L = instaloader.Instaloader()
+
+posts = L.get_hashtag_posts('milfgarden')
+
+users = set()
+
+for post in posts:
+    if not post.owner_profile in users:
+        L.download_post(post, '#milfgarden')
+        users.add(post.owner_profile)
+    else:
+        print("{} from {} skipped.".format(post, post.owner_profile))

docs/codesnippets/120_ghost_followers.py

@@ -0,0 +1,27 @@
+import instaloader
+
+L = instaloader.Instaloader()
+
+USER = 'your_account'
+PROFILE = USER
+
+# Your preferred way of logging in:
+L.load_session_from_file(USER)
+
+profile = instaloader.Profile.from_username(L.context, PROFILE)
+
+likes = set()
+print('Fetching likes of all posts of profile {}.'.format(profile.username))
+for post in profile.get_posts():
+    print(post)
+    likes = likes | set(post.get_likes())
+
+print('Fetching followers of profile {}.'.format(profile.username))
+followers = set(profile.get_followers())
+
+ghosts = followers - likes
+
+print('Storing ghosts into file.')
+with open('/YOUR PATH/inactive-users.txt', 'w') as f:
+    for ghost in ghosts:
+        print(ghost.username, file=f)

docs/codesnippets/121_since_until.py

@@ -0,0 +1,17 @@
+from datetime import datetime
+from itertools import dropwhile, takewhile
+
+import instaloader
+
+L = instaloader.Instaloader()
+
+posts = L.get_hashtag_posts('milfgarden')
+# or
+# posts = instaloader.Profile.from_username(L.context, PROFILE).get_posts()
+
+SINCE = datetime(2015, 5, 1)
+UNTIL = datetime(2015, 3, 1)
+
+for post in takewhile(lambda p: p.date > UNTIL, dropwhile(lambda p: p.date > SINCE, posts)):
+    print(post.date)
+    L.download_post(post, '#milfgarden')

docs/codesnippets/56_track_deleted.py

@@ -0,0 +1,34 @@
+from glob import glob
+from sys import argv
+from os import chdir
+
+from instaloader import Instaloader, Post, Profile, load_structure_from_file
+
+# Instaloader instantiation - you may pass additional arguments to the constructor here
+L = Instaloader()
+
+# If desired, load session previously saved with `instaloader -l USERNAME`:
+#L.load_session_from_file(USERNAME)
+
+try:
+    TARGET = argv[1]
+except IndexError:
+    raise SystemExit("Pass profile name as argument!")
+
+# Obtain set of posts that are on hard disk
+chdir(TARGET)
+offline_posts = set(filter(lambda s: isinstance(s, Post),
+                           (load_structure_from_file(L.context, file)
+                            for file in (glob('*.json.xz') + glob('*.json')))))
+
+# Obtain set of posts that are currently online
+post_iterator = Profile.from_username(L.context, TARGET).get_posts()
+online_posts = set(post_iterator)
+
+if online_posts - offline_posts:
+    print("Not yet downloaded posts:")
+    print(" ".join(str(p) for p in (online_posts - offline_posts)))
+
+if offline_posts - online_posts:
+    print("Deleted posts:")
+    print(" ".join(str(p) for p in (offline_posts - online_posts)))

docs/conf.py

@@ -39,7 +39,8 @@ import docs.sphinx_autodoc_typehints as sphinx_autodoc_typehints
 extensions = [
     'sphinx.ext.autodoc',
     'sphinx.ext.githubpages',
-    'sphinx.ext.intersphinx'
+    'sphinx.ext.intersphinx',
+    'sphinx.ext.extlinks'
 ]
 
 autodoc_default_flags = ['show-inheritance', 'members', 'undoc-members']
@@ -49,6 +50,8 @@ autodoc_member_order = 'bysource'
 intersphinx_mapping = {'python': ('https://docs.python.org/3', None),
                        'requests': ('http://docs.python-requests.org/en/master', None)}
 
+extlinks = {'issue': ('https://github.com/instaloader/instaloader/issues/%s', 'issue #')}
+
 # Add any paths that contain templates here, relative to this directory.
 templates_path = ['_templates']
 

docs/contributing.rst

@@ -1,3 +1,5 @@
+.. _contributing:
+
 Contributing to Instaloader
 ===========================
 
@@ -23,10 +25,10 @@ following in mind:
 
 - Briefly **check whether the bug has already been reported**. If you find an
   issue reporting the same bug you encountered, comment there rather than
-  opening a new issue. However, if unsure, it is advised to create a new issue.
+  opening a new issue. However, if unsure, please create a new issue.
 
 - State **how the bug can be reproduced**, i.e. how Instaloader was invoked
-  when the problem occured (of course, you may anonymize profile names etc.).
+  when the problem occurred (of course, you may anonymize profile names etc.).
 
 - Include all **error messages and tracebacks** in the report.
 
@@ -47,13 +49,13 @@ few things to consider:
   you forked from the most recent branch.
 
 - We use `Pylint <https://www.pylint.org/>`__ for error and syntax checking of
-  ``instaloader.py``. The file ``.travis.yml`` in the project's root directory
+  the source. The file ``.travis.yml`` in the project's root directory
   shows how it is invoked. Note that sometimes it might be better to disable a
   warning rather than adapting the code to Pylint's desires.
 
 - The documentation source is located in the ``docs`` folder. The file
-  ``cli-options.rst`` is merely an RST-formatted copy of ``ìnstaloader --help``
-  output, of which the source is near the bottom of ``instaloader.py``.
+  ``cli-options.rst`` is merely an RST-formatted copy of ``instaloader --help``
+  output, of which the source is in ``instaloader/__main__.py``.
 
 - Feel free to contact us, even if you "only" have Proof-of-Concepts or
   not-fully integrated changes. They already might be an advance for the
@@ -81,3 +83,21 @@ Please consider the following:
 - **Be patient**. Naturally, bugs and pull requests have a higher priority than
   feature suggestions. Keep in mind that this is a free software project, and
   unfortunately we only have limited time to work on it.
+
+Donations
+---------
+
+.. donations-start
+
+It is a pleasure for us to share our Instaloader to the world, and we are proud
+to have attracted such an active and motivating community, with so many users
+who share their suggestions and ideas with us. Buying a community-sponsored beer
+or coffee from time to time is very likely to further raise our passion for the
+development of Instaloader.
+
+We have a PayPal.Me link for donations: `PayPal.me/aandergr
+<https://www.paypal.me/aandergr>`__
+
+.. donations-end
+
+.. (Discussion in :issue:`130`)

docs/index.rst

@@ -59,6 +59,7 @@ Instaloader Documentation
    basic-usage
    cli-options
    as-module
+   codesnippets
    contributing
 
 Useful Links
@@ -69,10 +70,23 @@ Useful Links
 - `Issue Tracker / Bug Tracker <https://github.com/instaloader/instaloader/issues>`__
 - `Version History <https://github.com/instaloader/instaloader/releases>`__
 
+Disclaimer
+----------
+
 .. include:: ../README.rst
    :start-after: disclaimer-start
    :end-before: disclaimer-end
 
+Contributing
+------------
+
+As an open source project, Instaloader heavily depends on the contributions from
+its community. See :ref:`contributing` for how you may help Instaloader to
+become an even greater tool.
+
+.. include:: contributing.rst
+   :start-after: donations-start
+   :end-before: donations-end
 
 ..
    * :ref:`genindex`

instaloader/__init__.py

@@ -1,7 +1,7 @@
 """Download pictures (or videos) along with their captions and other metadata from Instagram."""
 
 
-__version__ = '4.0'
+__version__ = '4.0.5'
 
 
 try:

instaloader/__main__.py

@@ -146,7 +146,7 @@ def _main(instaloader: Instaloader, targetlist: List[str],
                                                  stories, stories_only, post_filter=post_filter,
                                                  storyitem_filter=storyitem_filter)
                 except ProfileNotExistsException as err:
-                    if not instaloader.context.is_logged_in:
+                    if instaloader.context.is_logged_in:
                         instaloader.context.log(err)
                         instaloader.context.log("Trying again anonymously, helps in case you are just blocked.")
                         with instaloader.anonymous_copy() as anonymous_loader:
@@ -271,6 +271,7 @@ def main():
     g_how.add_argument('--user-agent',
                        help='User Agent to use for HTTP requests. Defaults to \'{}\'.'.format(default_user_agent()))
     g_how.add_argument('-S', '--no-sleep', action='store_true', help=SUPPRESS)
+    g_how.add_argument('--graphql-rate-limit', type=int, help=SUPPRESS)
     g_how.add_argument('--max-connection-attempts', metavar='N', type=int, default=3,
                        help='Maximum number of connection attempts until a request is aborted. Defaults to 3. If a '
                             'connection fails, it can be manually skipped by hitting CTRL+C. Set this to 0 to retry '
@@ -316,6 +317,7 @@ def main():
                              compress_json=not args.no_compress_json,
                              post_metadata_txt_pattern=post_metadata_txt_pattern,
                              storyitem_metadata_txt_pattern=storyitem_metadata_txt_pattern,
+                             graphql_rate_limit=args.graphql_rate_limit,
                              max_connection_attempts=args.max_connection_attempts)
         _main(loader,
               args.profile,

instaloader/instaloader.py

@@ -91,6 +91,10 @@ class Instaloader:
        txt file.
     :param storyitem_metadata_txt_pattern: :option:`--storyitem-metadata-txt`, default is empty (=none)
     :param max_connection_attempts: :option:`--max-connection-attempts`
+
+    .. attribute:: context
+
+       The associated :class:`InstaloaderContext` with low-level communication functions and logging.
     """
 
     def __init__(self,
@@ -106,9 +110,10 @@ class Instaloader:
                  compress_json: bool = True,
                  post_metadata_txt_pattern: str = None,
                  storyitem_metadata_txt_pattern: str = None,
+                 graphql_rate_limit: Optional[int] = None,
                  max_connection_attempts: int = 3):
 
-        self.context = InstaloaderContext(sleep, quiet, user_agent, max_connection_attempts)
+        self.context = InstaloaderContext(sleep, quiet, user_agent, graphql_rate_limit, max_connection_attempts)
 
         # configuration parameters
         self.dirname_pattern = dirname_pattern or "{target}"
@@ -131,7 +136,8 @@ class Instaloader:
                                  self.filename_pattern, self.download_videos, self.download_video_thumbnails,
                                  self.download_geotags, self.download_comments, self.save_metadata,
                                  self.compress_json, self.post_metadata_txt_pattern,
-                                 self.storyitem_metadata_txt_pattern, self.context.max_connection_attempts)
+                                 self.storyitem_metadata_txt_pattern, self.context.graphql_count_per_slidingwindow,
+                                 self.context.max_connection_attempts)
         new_loader.context.query_timestamps = self.context.query_timestamps
         yield new_loader
         self.context.error_log.extend(new_loader.context.error_log)
@@ -178,27 +184,28 @@ class Instaloader:
     def update_comments(self, filename: str, post: Post) -> None:
         def _postcomment_asdict(comment):
             return {'id': comment.id,
-                    'created_at': comment.created_at_utc.replace(tzinfo=timezone.utc).timestamp(),
+                    'created_at': int(comment.created_at_utc.replace(tzinfo=timezone.utc).timestamp()),
                     'text': comment.text,
                     'owner': comment.owner._asdict()}
         filename += '_comments.json'
         try:
-            comments = json.load(open(filename))
-        except FileNotFoundError:
+            with open(filename) as fp:
+                comments = json.load(fp)
+        except (FileNotFoundError, json.decoder.JSONDecodeError):
             comments = list()
         comments.extend(_postcomment_asdict(comment) for comment in post.get_comments())
         if comments:
+            comments_list = sorted(sorted(list(comments), key=lambda t: int(t['id'])),
+                                   key=lambda t: int(t['created_at']), reverse=True)
+            unique_comments_list = [comments_list[0]]
+            #for comment in comments_list:
+            #    if unique_comments_list[-1]['id'] != comment['id']:
+            #        unique_comments_list.append(comment)
+            #file.write(json.dumps(unique_comments_list, indent=4))
+            for x, y in zip(comments_list[:-1], comments_list[1:]):
+                if x['id'] != y['id']:
+                    unique_comments_list.append(y)
             with open(filename, 'w') as file:
-                comments_list = sorted(sorted(list(comments), key=lambda t: t['id']),
-                                       key=lambda t: t['created_at'], reverse=True)
-                unique_comments_list = [comments_list[0]]
-                #for comment in comments_list:
-                #    if unique_comments_list[-1]['id'] != comment['id']:
-                #        unique_comments_list.append(comment)
-                #file.write(json.dumps(unique_comments_list, indent=4))
-                for x, y in zip(comments_list[:-1], comments_list[1:]):
-                    if x['id'] != y['id']:
-                        unique_comments_list.append(y)
                 file.write(json.dumps(unique_comments_list, indent=4))
             self.context.log('comments', end=' ', flush=True)
 
@@ -259,18 +266,19 @@ class Instaloader:
         def _epoch_to_string(epoch: datetime) -> str:
             return epoch.strftime('%Y-%m-%d_%H-%M-%S')
 
-        profile_pic_url = profile.get_profile_pic_url()
+        profile_pic_url = profile.profile_pic_url
         with self.context.get_anonymous_session() as anonymous_session:
             date_object = datetime.strptime(anonymous_session.head(profile_pic_url).headers["Last-Modified"],
                                             '%a, %d %b %Y %H:%M:%S GMT')
+        profile_pic_extension = 'jpg'
         if ((format_string_contains_key(self.dirname_pattern, 'profile') or
              format_string_contains_key(self.dirname_pattern, 'target'))):
             filename = '{0}/{1}_UTC_profile_pic.{2}'.format(self.dirname_pattern.format(profile=profile.username.lower(),
                                                                                         target=profile.username.lower()),
-                                                            _epoch_to_string(date_object), profile_pic_url[-3:])
+                                                            _epoch_to_string(date_object), profile_pic_extension)
         else:
             filename = '{0}/{1}_{2}_UTC_profile_pic.{3}'.format(self.dirname_pattern.format(), profile.username.lower(),
-                                                                _epoch_to_string(date_object), profile_pic_url[-3:])
+                                                                _epoch_to_string(date_object), profile_pic_extension)
         if os.path.isfile(filename):
             self.context.log(filename + ' already exists')
             return None
@@ -313,7 +321,11 @@ class Instaloader:
         return self.context.test_login()
 
     def login(self, user: str, passwd: str) -> None:
-        """Log in to instagram with given username and password and internally store session object"""
+        """Log in to instagram with given username and password and internally store session object.
+
+        :raises InvalidArgumentException: If the provided username does not exist.
+        :raises BadCredentialsException: If the provided password is wrong.
+        :raises ConnectionException: If connection to Instagram failed."""
         self.context.login(user, passwd)
 
     def download_post(self, post: Post, target: str) -> bool:
@@ -698,9 +710,12 @@ class Instaloader:
 
         # Download stories, if requested
         if download_stories or download_stories_only:
-            with self.context.error_catcher("Download stories of {}".format(profile_name)):
-                self.download_stories(userids=[profile.userid], filename_target=profile_name, fast_update=fast_update,
-                                      storyitem_filter=storyitem_filter)
+            if profile.has_viewable_story:
+                with self.context.error_catcher("Download stories of {}".format(profile_name)):
+                    self.download_stories(userids=[profile.userid], filename_target=profile_name,
+                                          fast_update=fast_update, storyitem_filter=storyitem_filter)
+            else:
+                self.context.log("{} does not have any stories.".format(profile_name))
         if download_stories_only:
             return
 
@@ -722,7 +737,9 @@ class Instaloader:
     def interactive_login(self, username: str) -> None:
         """Logs in and internally stores session, asking user for password interactively.
 
-        :raises LoginRequiredException: when in quiet mode."""
+        :raises LoginRequiredException: when in quiet mode.
+        :raises InvalidArgumentException: If the provided username does not exist.
+        :raises ConnectionException: If connection to Instagram failed."""
         if self.context.quiet:
             raise LoginRequiredException("Quiet mode requires given password or valid session file.")
         password = None

instaloader/instaloadercontext.py

@@ -46,8 +46,8 @@ class InstaloaderContext:
     class :class:`Instaloader`.
     """
 
-    def __init__(self, sleep: bool = True, quiet: bool = False,
-                 user_agent: Optional[str] = None, max_connection_attempts: int = 3):
+    def __init__(self, sleep: bool = True, quiet: bool = False, user_agent: Optional[str] = None,
+                 graphql_count_per_slidingwindow: Optional[int] = None, max_connection_attempts: int = 3):
 
         self.user_agent = user_agent if user_agent is not None else default_user_agent()
         self._session = self.get_anonymous_session()
@@ -56,6 +56,7 @@ class InstaloaderContext:
         self.quiet = quiet
         self.max_connection_attempts = max_connection_attempts
         self._graphql_page_length = 50
+        self.graphql_count_per_slidingwindow = graphql_count_per_slidingwindow or 20
         self._root_rhx_gis = None
 
         # error log, filled with error() and printed at the end of Instaloader.main()
@@ -67,6 +68,17 @@ class InstaloaderContext:
         # Can be set to True for testing, disables supression of InstaloaderContext._error_catcher
         self.raise_all_errors = False
 
+    @contextmanager
+    def anonymous_copy(self):
+        session = self._session
+        username = self.username
+        self._session = self.get_anonymous_session()
+        self.username = None
+        yield self
+        self._session.close()
+        self.username = username
+        self._session = session
+
     @property
     def is_logged_in(self) -> bool:
         """True, if this Instaloader instance is logged in."""
@@ -144,7 +156,7 @@ class InstaloaderContext:
         pickle.dump(requests.utils.dict_from_cookiejar(self._session.cookies), sessionfile)
 
     def load_session_from_file(self, username, sessionfile):
-        """Not meant to be used directly, use :meth:`Instaloader.load_session_to_file`."""
+        """Not meant to be used directly, use :meth:`Instaloader.load_session_from_file`."""
         session = requests.Session()
         session.cookies = requests.utils.cookiejar_from_dict(pickle.load(sessionfile))
         session.headers.update(self._default_http_header())
@@ -158,34 +170,49 @@ class InstaloaderContext:
         return data["data"]["user"]["username"] if data["data"]["user"] is not None else None
 
     def login(self, user, passwd):
-        """Not meant to be used directly, use :meth:`Instaloader.login`."""
+        """Not meant to be used directly, use :meth:`Instaloader.login`.
+
+        :raises InvalidArgumentException: If the provided username does not exist.
+        :raises BadCredentialsException: If the provided password is wrong.
+        :raises ConnectionException: If connection to Instagram failed."""
         session = requests.Session()
         session.cookies.update({'sessionid': '', 'mid': '', 'ig_pr': '1',
                                 'ig_vw': '1920', 'csrftoken': '',
                                 's_network': '', 'ds_user_id': ''})
         session.headers.update(self._default_http_header())
-        self._sleep()
-        resp = session.get('https://www.instagram.com/')
-        session.headers.update({'X-CSRFToken': resp.cookies['csrftoken']})
+        session.headers.update({'X-CSRFToken': self.get_json('', {})['config']['csrf_token']})
+        # Not using self.get_json() here, because we need to access csrftoken cookie
         self._sleep()
         login = session.post('https://www.instagram.com/accounts/login/ajax/',
                              data={'password': passwd, 'username': user}, allow_redirects=True)
-        session.headers.update({'X-CSRFToken': login.cookies['csrftoken']})
-        if login.status_code == 200:
-            self._session = session
-            if user == self.test_login():
-                self.username = user
+        if login.status_code != 200:
+            raise ConnectionException("Login error: {} {}".format(login.status_code, login.reason))
+        resp_json = login.json()
+        if resp_json['status'] != 'ok':
+            if 'message' in resp_json:
+                raise ConnectionException("Login error: \"{}\" status, message \"{}\".".format(resp_json['status'],
+                                                                                               resp_json['message']))
             else:
-                self.username = None
-                self._session = None
-                raise BadCredentialsException('Login error! Check your credentials!')
-        else:
-            raise ConnectionException('Login error! Connection error!')
+                raise ConnectionException("Login error: \"{}\" status.".format(resp_json['status']))
+        if not resp_json['authenticated']:
+            if resp_json['user']:
+                # '{"authenticated": false, "user": true, "status": "ok"}'
+                raise BadCredentialsException('Login error: Wrong password.')
+            else:
+                # '{"authenticated": false, "user": false, "status": "ok"}'
+                # Raise InvalidArgumentException rather than BadCredentialException, because BadCredentialException
+                # triggers re-asking of password in Instaloader.interactive_login(), which makes no sense if the
+                # username is invalid.
+                raise InvalidArgumentException('Login error: User {} does not exist.'.format(user))
+        # '{"authenticated": true, "user": true, "userId": ..., "oneTapPrompt": false, "status": "ok"}'
+        session.headers.update({'X-CSRFToken': login.cookies['csrftoken']})
+        self._session = session
+        self.username = user
 
     def _sleep(self):
         """Sleep a short time if self.sleep is set. Called before each request to instagram.com."""
         if self.sleep:
-            time.sleep(random.uniform(0.5, 3))
+            time.sleep(min(random.expovariate(0.6), 5.0))
 
     def get_json(self, path: str, params: Dict[str, Any], host: str = 'www.instagram.com',
                  session: Optional[requests.Session] = None, _attempt=1) -> Dict[str, Any]:
@@ -206,11 +233,14 @@ class InstaloaderContext:
                 return sliding_window if untracked_queries else 0
             current_time = time.monotonic()
             self.query_timestamps = list(filter(lambda t: t > current_time - sliding_window, self.query_timestamps))
-            if len(self.query_timestamps) < 20 and not untracked_queries:
+            if len(self.query_timestamps) < self.graphql_count_per_slidingwindow and not untracked_queries:
                 return 0
             return round(min(self.query_timestamps) + sliding_window - current_time) + 6
         is_graphql_query = 'query_hash' in params and 'graphql/query' in path
-        if is_graphql_query:
+        # some queries are not rate limited if invoked anonymously:
+        query_not_limited = is_graphql_query and not self.is_logged_in \
+                            and params['query_hash'] in ['9ca88e465c3f866a76f7adee3871bdd8']
+        if is_graphql_query and not query_not_limited:
             waittime = graphql_query_waittime()
             if waittime > 0:
                 self.log('\nToo many queries in the last time. Need to wait {} seconds.'.format(waittime))
@@ -226,7 +256,7 @@ class InstaloaderContext:
             while resp.is_redirect:
                 redirect_url = resp.headers['location']
                 self.log('\nHTTP redirect from https://{0}/{1} to {2}'.format(host, path, redirect_url))
-                if redirect_url.index('https://{}/'.format(host)) == 0:
+                if redirect_url.startswith('https://{}/'.format(host)):
                     resp = sess.get(redirect_url if redirect_url.endswith('/') else redirect_url + '/',
                                     params=params, allow_redirects=False)
                 else:

instaloader/structures.py

@@ -12,8 +12,24 @@ from .instaloadercontext import InstaloaderContext
 
 
 PostSidecarNode = namedtuple('PostSidecarNode', ['is_video', 'display_url', 'video_url'])
+PostSidecarNode.__doc__ = "Item of a Sidecar Post."
+PostSidecarNode.is_video.__doc__ = "Whether this node is a video."
+PostSidecarNode.display_url.__doc__ = "URL of image or video thumbnail."
+PostSidecarNode.video_url.__doc__ = "URL of video or None."
+
 PostComment = namedtuple('PostComment', ['id', 'created_at_utc', 'text', 'owner'])
+PostComment.id.__doc__ = "ID number of comment."
+PostComment.created_at_utc.__doc__ = ":class:`~datetime.datetime` when comment was created (UTC)."
+PostComment.text.__doc__ = "Comment text."
+PostComment.owner.__doc__ = "Owner :class:`Profile` of the comment."
+
 PostLocation = namedtuple('PostLocation', ['id', 'name', 'slug', 'has_public_page', 'lat', 'lng'])
+PostLocation.id.__doc__ = "ID number of location."
+PostLocation.name.__doc__ = "Location name."
+PostLocation.slug.__doc__ = "URL friendly variant of location name."
+PostLocation.has_public_page.__doc__ = "Whether location has a public page."
+PostLocation.lat.__doc__ = "Latitude (:class:`float`)."
+PostLocation.lng.__doc__ = "Longitude (:class:`float`)."
 
 
 class Post:
@@ -77,10 +93,9 @@ class Post:
         return b64encode(mediaid.to_bytes(9, 'big'), b'-_').decode().replace('A', ' ').lstrip().replace(' ', 'A')
 
     def _asdict(self):
+        node = self._node
         if self._full_metadata_dict:
-            node = self._full_metadata_dict
-        else:
-            node = self._node
+            node.update(self._full_metadata_dict)
         if self._owner_profile:
             node['owner'] = self.owner_profile._asdict()
         if self._location:
@@ -190,10 +205,7 @@ class Post:
     @property
     def typename(self) -> str:
         """Type of post, GraphImage, GraphVideo or GraphSidecar"""
-        if '__typename' in self._node:
-            return self._node['__typename']
-        # if __typename is not in node, it is an old image or video
-        return 'GraphImage'
+        return self._field('__typename')
 
     def get_sidecar_nodes(self) -> Iterator[PostSidecarNode]:
         """Sidecar nodes of a Post with typename==GraphSidecar."""
@@ -358,13 +370,20 @@ class Profile:
     def __init__(self, context: InstaloaderContext, node: Dict[str, Any]):
         assert 'username' in node
         self._context = context
+        self._has_public_story = None
         self._node = node
         self._rhx_gis = None
+        self._iphone_struct_ = None
+        if 'iphone_struct' in node:
+            # if loaded from JSON with load_structure_from_file()
+            self._iphone_struct_ = node['iphone_struct']
 
     @classmethod
     def from_username(cls, context: InstaloaderContext, username: str):
         """Create a Profile instance from a given username, raise exception if it does not exist.
 
+        See also :meth:`Instaloader.check_profile_id`.
+
         :param context: :attr:`Instaloader.context`
         :param username: Username
         :raises: :class:`ProfileNotExistsException`
@@ -404,6 +423,8 @@ class Profile:
         json_node.pop('edge_media_collections', None)
         json_node.pop('edge_owner_to_timeline_media', None)
         json_node.pop('edge_saved_media', None)
+        if self._iphone_struct_:
+            json_node['iphone_struct'] = self._iphone_struct_
         return json_node
 
     def _obtain_metadata(self):
@@ -412,8 +433,8 @@ class Profile:
                 metadata = self._context.get_json('{}/'.format(self.username), params={})
                 self._node = metadata['entry_data']['ProfilePage'][0]['graphql']['user']
                 self._rhx_gis = metadata['rhx_gis']
-        except QueryReturnedNotFoundException:
-            raise ProfileNotExistsException('Profile {} does not exist.'.format(self.username))
+        except (QueryReturnedNotFoundException, KeyError) as err:
+            raise ProfileNotExistsException('Profile {} does not exist.'.format(self.username)) from err
 
     def _metadata(self, *keys) -> Any:
         try:
@@ -428,6 +449,15 @@ class Profile:
                 d = d[key]
             return d
 
+    @property
+    def _iphone_struct(self) -> Dict[str, Any]:
+        if not self._iphone_struct_:
+            with self._context.anonymous_copy() as anonymous_context:
+                data = anonymous_context.get_json(path='api/v1/users/{}/info/'.format(self.userid),
+                                                  params={}, host='i.instagram.com')
+            self._iphone_struct_ = data['user']
+        return self._iphone_struct_
+
     @property
     def userid(self) -> int:
         """User ID"""
@@ -461,6 +491,18 @@ class Profile:
     def mediacount(self) -> int:
         return self._metadata('edge_owner_to_timeline_media', 'count')
 
+    @property
+    def followers(self) -> int:
+        return self._metadata('edge_followed_by', 'count')
+
+    @property
+    def followees(self) -> int:
+        return self._metadata('edge_follow', 'count')
+
+    @property
+    def external_url(self) -> Optional[str]:
+        return self._metadata('external_url')
+
     @property
     def biography(self) -> str:
         return self._metadata('biography')
@@ -481,6 +523,39 @@ class Profile:
     def has_blocked_viewer(self) -> bool:
         return self._metadata('has_blocked_viewer')
 
+    @property
+    def has_highlight_reels(self) -> bool:
+        """
+        This becomes `True` if the :class:`Profile` has any stories currently available,
+        even if not viewable by the viewer.
+        """
+        return self._iphone_struct['has_highlight_reels']
+
+    @property
+    def has_public_story(self) -> bool:
+        if not self._has_public_story:
+            self._obtain_metadata()
+            # query not rate limited if invoked anonymously:
+            with self._context.anonymous_copy() as anonymous_context:
+                data = anonymous_context.graphql_query('9ca88e465c3f866a76f7adee3871bdd8',
+                                                       {'user_id': self.userid, 'include_chaining': False,
+                                                        'include_reel': False, 'include_suggested_users': False,
+                                                        'include_logged_out_extras': True,
+                                                        'include_highlight_reels': False},
+                                                       'https://www.instagram.com/{}/'.format(self.username),
+                                                       self._rhx_gis)
+            self._has_public_story = data['data']['user']['has_public_story']
+        return self._has_public_story
+
+    @property
+    def has_viewable_story(self) -> bool:
+        """
+        Some stories are private. This property determines if the :class:`Profile`
+        has at least one story which can be viewed using the associated :class:`InstaloaderContext`,
+        i.e. the viewer has privileges to view it.
+        """
+        return self.has_public_story or self.followed_by_viewer and self.has_highlight_reels
+
     @property
     def has_requested_viewer(self) -> bool:
         return self._metadata('has_requested_viewer')
@@ -493,17 +568,21 @@ class Profile:
     def requested_by_viewer(self) -> bool:
         return self._metadata('requested_by_viewer')
 
-    def get_profile_pic_url(self) -> str:
+    @property
+    def profile_pic_url(self) -> str:
         """Return URL of profile picture"""
         try:
-            with self._context.get_anonymous_session() as anonymous_session:
-                data = self._context.get_json(path='api/v1/users/{0}/info/'.format(self.userid), params={},
-                                              host='i.instagram.com', session=anonymous_session)
-            return data["user"]["hd_profile_pic_url_info"]["url"]
+            return self._iphone_struct['hd_profile_pic_url_info']['url']
         except (InstaloaderException, KeyError) as err:
             self._context.error('{} Unable to fetch high quality profile pic.'.format(err))
             return self._metadata("profile_pic_url_hd")
 
+    def get_profile_pic_url(self) -> str:
+        """.. deprecated:: 4.0.3
+
+	   Use :attr:`profile_pic_url`."""
+        return self.profile_pic_url
+
     def get_posts(self) -> Iterator[Post]:
         """Retrieve all posts from a profile."""
         self._obtain_metadata()
@@ -716,7 +795,7 @@ class Story:
     def unique_id(self) -> str:
         """
         This ID only equals amongst :class:`Story` instances which have the same owner and the same set of
-        :class:`StoryItem`s. For all other :class:`Story` instances this ID is different.
+        :class:`StoryItem`. For all other :class:`Story` instances this ID is different.
         """
         if not self._unique_id:
             id_list = [item.mediaid for item in self.get_items()]
@@ -799,7 +878,7 @@ def save_structure_to_file(structure: JsonExportable, filename: str) -> None:
 
 def load_structure_from_file(context: InstaloaderContext, filename: str) -> JsonExportable:
     """Loads a :class:`Post`, :class:`Profile` or :class:`StoryItem` from a '.json' or '.json.xz' file that
-    has been saved by :func:`save_structure_from_file`.
+    has been saved by :func:`save_structure_to_file`.
 
     :param context: :attr:`Instaloader.context` linked to the new object, used for additional queries if neccessary.
     :param filename: Filename, ends in '.json' or '.json.xz'

test/instaloader_unittests.py

@@ -4,6 +4,7 @@ import os
 import shutil
 import tempfile
 import unittest
+from itertools import islice
 
 import instaloader
 
@@ -15,6 +16,9 @@ NORMAL_MAX_COUNT = 2
 PAGING_MAX_COUNT = 15
 PRIVATE_PROFILE = "aandergr"
 
+# Preserve query timestamps (rate control) between tests to not get rate limited
+instaloadercontext_query_timestamps = list()
+
 
 class TestInstaloaderAnonymously(unittest.TestCase):
 
@@ -26,23 +30,31 @@ class TestInstaloaderAnonymously(unittest.TestCase):
                                          download_comments=True,
                                          save_metadata=True)
         self.L.context.raise_all_errors = True
+        self.L.context.query_timestamps = instaloadercontext_query_timestamps
 
     def tearDown(self):
+        global instaloadercontext_query_timestamps
+        instaloadercontext_query_timestamps = self.L.context.query_timestamps
         self.L.close()
         os.chdir('/')
         print("Removing {}".format(self.dir))
         shutil.rmtree(self.dir)
 
+    def post_paging_test(self, iterator):
+        previous_post = None
+        for post in islice(iterator, PAGING_MAX_COUNT):
+            print(post)
+            if previous_post:
+                self.assertTrue(post.date_utc < previous_post.date_utc)
+            previous_post = post
+
     @unittest.SkipTest
     def test_public_profile_download(self):
         self.L.download_profile(PUBLIC_PROFILE, profile_pic=False, fast_update=True)
         self.L.download_profile(PUBLIC_PROFILE, profile_pic=False, fast_update=True)
 
     def test_public_profile_paging(self):
-        for count, post in enumerate(instaloader.Profile.from_username(self.L.context, PUBLIC_PROFILE).get_posts()):
-            print(post)
-            if count == PAGING_MAX_COUNT:
-                break
+        self.post_paging_test(instaloader.Profile.from_username(self.L.context, PUBLIC_PROFILE).get_posts())
 
     def test_profile_pic_download(self):
         self.L.download_profile(PUBLIC_PROFILE, profile_pic_only=True)
@@ -86,10 +98,7 @@ class TestInstaloaderLoggedIn(TestInstaloaderAnonymously):
                 print(item)
 
     def test_private_profile_paging(self):
-        for count, post in enumerate(instaloader.Profile.from_username(self.L.context, PRIVATE_PROFILE).get_posts()):
-            print(post)
-            if count == PAGING_MAX_COUNT:
-                break
+        self.post_paging_test(instaloader.Profile.from_username(self.L.context, PRIVATE_PROFILE).get_posts())
 
     def test_feed_download(self):
         self.L.download_feed_posts(NORMAL_MAX_COUNT)
@@ -112,6 +121,11 @@ class TestInstaloaderLoggedIn(TestInstaloaderAnonymously):
     def test_test_login(self):
         self.assertEqual(OWN_USERNAME, self.L.test_login())
 
+    def test_followees_and_stories(self):
+        profile = instaloader.Profile.from_username(self.L.context, OWN_USERNAME)
+        for f in islice(profile.get_followees(), PAGING_MAX_COUNT):
+            self.L.download_profile(f.username, profile_pic=False, download_stories_only=True)
+
     def test_get_followees(self):
         profile = instaloader.Profile.from_username(self.L.context, OWN_USERNAME)
         for f in profile.get_followees():