Release of version 4.0.2

Motivated by #112, #118.

.travis.yml

@@ -9,7 +9,7 @@ install:
  - pip install -r docs/requirements.txt
 script:
  - python3 -m pylint -r n -d bad-whitespace,missing-docstring,too-many-arguments,locally-disabled,line-too-long,too-many-public-methods,too-many-lines,too-many-instance-attributes,too-many-locals,too-many-branches,too-many-statements,inconsistent-return-statements,invalid-name,wildcard-import,unused-wildcard-import instaloader
- - make -C docs html
+ - make -C docs html SPHINXOPTS="-W -n"
 deploy:
   - provider: pypi
     user: aandergr

docs/as-module.rst

@@ -73,7 +73,7 @@ metadata of a Profile. :class:`Profile` instances can be created with:
 
      profile = Profile.from_username(L.context, USERNAME)
 
-- :meth:`Profile_from_userid`
+- :meth:`Profile.from_id`
    given its User ID. This allows to easily lookup a Profile's username given
    its ID::
 
@@ -116,15 +116,12 @@ Additionally, the following trivial structures are defined:
 
 .. autoclass:: PostSidecarNode
    :no-show-inheritance:
-   :no-members:
 
 .. autoclass:: PostComment
    :no-show-inheritance:
-   :no-members:
 
 .. autoclass:: PostLocation
    :no-show-inheritance:
-   :no-members:
 
 User Stories
 """"""""""""
@@ -154,9 +151,15 @@ to/from JSON files.
 Exceptions
 ^^^^^^^^^^
 
+.. currentmodule:: instaloader.exceptions
+
 .. autoexception:: InstaloaderException
    :no-show-inheritance:
 
+.. autoexception:: ConnectionException
+
+.. currentmodule:: instaloader
+
 .. autoexception:: QueryReturnedBadRequestException
 
 .. autoexception:: QueryReturnedNotFoundException
@@ -177,8 +180,6 @@ Exceptions
 
 .. autoexception:: BadCredentialsException
 
-.. autoexception:: ConnectionException
-
 .. autoexception:: TooManyRequestsException
 
 ``InstaloaderContext`` (Low-level functions)

docs/cli-options.rst

@@ -88,6 +88,11 @@ automatically **finds it by its unique ID** and renames the folder likewise.
    download stories.  Requires :option:`--login`. Does not imply
    :option:`--no-profile-pic`.
 
+   .. note::
+
+      If possible, use ``:stories`` target rather than :option:`--stories-only`
+      with all your followees. ``:stories`` uses fewer API requests.
+
 .. option:: --post-filter filter, --only-if filter
 
    Expression that, if given, must evaluate to True for each post to be

instaloader/__init__.py

@@ -1,7 +1,7 @@
 """Download pictures (or videos) along with their captions and other metadata from Instagram."""
 
 
-__version__ = '4.0.1'
+__version__ = '4.0.2'
 
 
 try:

instaloader/__main__.py

@@ -271,6 +271,7 @@ def main():
     g_how.add_argument('--user-agent',
                        help='User Agent to use for HTTP requests. Defaults to \'{}\'.'.format(default_user_agent()))
     g_how.add_argument('-S', '--no-sleep', action='store_true', help=SUPPRESS)
+    g_how.add_argument('--graphql-rate-limit', type=int, help=SUPPRESS)
     g_how.add_argument('--max-connection-attempts', metavar='N', type=int, default=3,
                        help='Maximum number of connection attempts until a request is aborted. Defaults to 3. If a '
                             'connection fails, it can be manually skipped by hitting CTRL+C. Set this to 0 to retry '
@@ -316,6 +317,7 @@ def main():
                              compress_json=not args.no_compress_json,
                              post_metadata_txt_pattern=post_metadata_txt_pattern,
                              storyitem_metadata_txt_pattern=storyitem_metadata_txt_pattern,
+                             graphql_rate_limit=args.graphql_rate_limit,
                              max_connection_attempts=args.max_connection_attempts)
         _main(loader,
               args.profile,

instaloader/instaloader.py

@@ -91,6 +91,10 @@ class Instaloader:
        txt file.
     :param storyitem_metadata_txt_pattern: :option:`--storyitem-metadata-txt`, default is empty (=none)
     :param max_connection_attempts: :option:`--max-connection-attempts`
+
+    .. attribute:: context
+
+       The associated :class:`InstaloaderContext` with low-level communication functions and logging.
     """
 
     def __init__(self,
@@ -106,9 +110,10 @@ class Instaloader:
                  compress_json: bool = True,
                  post_metadata_txt_pattern: str = None,
                  storyitem_metadata_txt_pattern: str = None,
+                 graphql_rate_limit: Optional[int] = None,
                  max_connection_attempts: int = 3):
 
-        self.context = InstaloaderContext(sleep, quiet, user_agent, max_connection_attempts)
+        self.context = InstaloaderContext(sleep, quiet, user_agent, graphql_rate_limit, max_connection_attempts)
 
         # configuration parameters
         self.dirname_pattern = dirname_pattern or "{target}"
@@ -131,7 +136,8 @@ class Instaloader:
                                  self.filename_pattern, self.download_videos, self.download_video_thumbnails,
                                  self.download_geotags, self.download_comments, self.save_metadata,
                                  self.compress_json, self.post_metadata_txt_pattern,
-                                 self.storyitem_metadata_txt_pattern, self.context.max_connection_attempts)
+                                 self.storyitem_metadata_txt_pattern, self.context.graphql_count_per_slidingwindow,
+                                 self.context.max_connection_attempts)
         new_loader.context.query_timestamps = self.context.query_timestamps
         yield new_loader
         self.context.error_log.extend(new_loader.context.error_log)
@@ -699,7 +705,7 @@ class Instaloader:
 
         # Download stories, if requested
         if download_stories or download_stories_only:
-            if profile.has_highlight_reel:
+            if profile.has_viewable_story:
                 with self.context.error_catcher("Download stories of {}".format(profile_name)):
                     self.download_stories(userids=[profile.userid], filename_target=profile_name,
                                           fast_update=fast_update, storyitem_filter=storyitem_filter)

instaloader/instaloadercontext.py

@@ -46,8 +46,8 @@ class InstaloaderContext:
     class :class:`Instaloader`.
     """
 
-    def __init__(self, sleep: bool = True, quiet: bool = False,
-                 user_agent: Optional[str] = None, max_connection_attempts: int = 3):
+    def __init__(self, sleep: bool = True, quiet: bool = False, user_agent: Optional[str] = None,
+                 graphql_count_per_slidingwindow: Optional[int] = None, max_connection_attempts: int = 3):
 
         self.user_agent = user_agent if user_agent is not None else default_user_agent()
         self._session = self.get_anonymous_session()
@@ -56,7 +56,7 @@ class InstaloaderContext:
         self.quiet = quiet
         self.max_connection_attempts = max_connection_attempts
         self._graphql_page_length = 50
-        self._graphql_count_per_slidingwindow = 25
+        self.graphql_count_per_slidingwindow = graphql_count_per_slidingwindow or 20
         self._root_rhx_gis = None
 
         # error log, filled with error() and printed at the end of Instaloader.main()
@@ -68,6 +68,17 @@ class InstaloaderContext:
         # Can be set to True for testing, disables supression of InstaloaderContext._error_catcher
         self.raise_all_errors = False
 
+    @contextmanager
+    def anonymous_copy(self):
+        session = self._session
+        username = self.username
+        self._session = self.get_anonymous_session()
+        self.username = None
+        yield self
+        self._session.close()
+        self.username = username
+        self._session = session
+
     @property
     def is_logged_in(self) -> bool:
         """True, if this Instaloader instance is logged in."""
@@ -145,7 +156,7 @@ class InstaloaderContext:
         pickle.dump(requests.utils.dict_from_cookiejar(self._session.cookies), sessionfile)
 
     def load_session_from_file(self, username, sessionfile):
-        """Not meant to be used directly, use :meth:`Instaloader.load_session_to_file`."""
+        """Not meant to be used directly, use :meth:`Instaloader.load_session_from_file`."""
         session = requests.Session()
         session.cookies = requests.utils.cookiejar_from_dict(pickle.load(sessionfile))
         session.headers.update(self._default_http_header())
@@ -207,11 +218,14 @@ class InstaloaderContext:
                 return sliding_window if untracked_queries else 0
             current_time = time.monotonic()
             self.query_timestamps = list(filter(lambda t: t > current_time - sliding_window, self.query_timestamps))
-            if len(self.query_timestamps) < self._graphql_count_per_slidingwindow and not untracked_queries:
+            if len(self.query_timestamps) < self.graphql_count_per_slidingwindow and not untracked_queries:
                 return 0
             return round(min(self.query_timestamps) + sliding_window - current_time) + 6
         is_graphql_query = 'query_hash' in params and 'graphql/query' in path
-        if is_graphql_query:
+        # some queries are not rate limited if invoked anonymously:
+        query_not_limited = is_graphql_query and not self.is_logged_in \
+                            and params['query_hash'] in ['9ca88e465c3f866a76f7adee3871bdd8']
+        if is_graphql_query and not query_not_limited:
             waittime = graphql_query_waittime()
             if waittime > 0:
                 self.log('\nToo many queries in the last time. Need to wait {} seconds.'.format(waittime))
@@ -227,7 +241,7 @@ class InstaloaderContext:
             while resp.is_redirect:
                 redirect_url = resp.headers['location']
                 self.log('\nHTTP redirect from https://{0}/{1} to {2}'.format(host, path, redirect_url))
-                if redirect_url.index('https://{}/'.format(host)) == 0:
+                if redirect_url.startswith('https://{}/'.format(host)):
                     resp = sess.get(redirect_url if redirect_url.endswith('/') else redirect_url + '/',
                                     params=params, allow_redirects=False)
                 else:

instaloader/structures.py

@@ -12,8 +12,24 @@ from .instaloadercontext import InstaloaderContext
 
 
 PostSidecarNode = namedtuple('PostSidecarNode', ['is_video', 'display_url', 'video_url'])
+PostSidecarNode.__doc__ = "Item of a Sidecar Post."
+PostSidecarNode.is_video.__doc__ = "Whether this node is a video."
+PostSidecarNode.display_url.__doc__ = "URL of image or video thumbnail."
+PostSidecarNode.video_url.__doc__ = "URL of video or None."
+
 PostComment = namedtuple('PostComment', ['id', 'created_at_utc', 'text', 'owner'])
+PostComment.id.__doc__ = "ID number of comment."
+PostComment.created_at_utc.__doc__ = ":class:`~datetime.datetime` when comment was created (UTC)."
+PostComment.text.__doc__ = "Comment text."
+PostComment.owner.__doc__ = "Owner :class:`Profile` of the comment."
+
 PostLocation = namedtuple('PostLocation', ['id', 'name', 'slug', 'has_public_page', 'lat', 'lng'])
+PostLocation.id.__doc__ = "ID number of location."
+PostLocation.name.__doc__ = "Location name."
+PostLocation.slug.__doc__ = "URL friendly variant of location name."
+PostLocation.has_public_page.__doc__ = "Whether location has a public page."
+PostLocation.lat.__doc__ = "Latitude (:class:`float`)."
+PostLocation.lng.__doc__ = "Longitude (:class:`float`)."
 
 
 class Post:
@@ -358,6 +374,8 @@ class Profile:
     def __init__(self, context: InstaloaderContext, node: Dict[str, Any]):
         assert 'username' in node
         self._context = context
+        self._has_highlight_reels = None
+        self._has_public_story = None
         self._node = node
         self._rhx_gis = None
 
@@ -463,6 +481,18 @@ class Profile:
     def mediacount(self) -> int:
         return self._metadata('edge_owner_to_timeline_media', 'count')
 
+    @property
+    def followers(self) -> int:
+        return self._metadata('edge_followed_by', 'count')
+
+    @property
+    def followees(self) -> int:
+        return self._metadata('edge_follow', 'count')
+
+    @property
+    def external_url(self) -> Optional[str]:
+        return self._metadata('external_url')
+
     @property
     def biography(self) -> str:
         return self._metadata('biography')
@@ -484,8 +514,42 @@ class Profile:
         return self._metadata('has_blocked_viewer')
 
     @property
-    def has_highlight_reel(self) -> bool:
-        return self._metadata('has_highlight_reel')
+    def has_highlight_reels(self) -> bool:
+        """
+        This becomes `True` if the :class:`Profile` has any stories currently available,
+        even if not viewable by the viewer.
+        """
+        if not self._has_highlight_reels:
+            with self._context.anonymous_copy() as anonymous_context:
+                data = anonymous_context.get_json(path='api/v1/users/{}/info/'.format(self.userid),
+                                                  params={}, host='i.instagram.com')
+            self._has_highlight_reels = data['user']['has_highlight_reels']
+        return self._has_highlight_reels
+
+    @property
+    def has_public_story(self) -> bool:
+        if not self._has_public_story:
+            self._obtain_metadata()
+            # query not rate limited if invoked anonymously:
+            with self._context.anonymous_copy() as anonymous_context:
+                data = anonymous_context.graphql_query('9ca88e465c3f866a76f7adee3871bdd8',
+                                                       {'user_id': self.userid, 'include_chaining': False,
+                                                        'include_reel': False, 'include_suggested_users': False,
+                                                        'include_logged_out_extras': True,
+                                                        'include_highlight_reels': False},
+                                                       'https://www.instagram.com/{}/'.format(self.username),
+                                                       self._rhx_gis)
+            self._has_public_story = data['data']['user']['has_public_story']
+        return self._has_public_story
+
+    @property
+    def has_viewable_story(self) -> bool:
+        """
+        Some stories are private. This property determines if the :class:`Profile`
+        has at least one story which can be viewed using the associated :class:`InstaloaderContext`,
+        i.e. the viewer has privileges to view it.
+        """
+        return self.has_public_story or self.followed_by_viewer and self.has_highlight_reels
 
     @property
     def has_requested_viewer(self) -> bool:
@@ -722,7 +786,7 @@ class Story:
     def unique_id(self) -> str:
         """
         This ID only equals amongst :class:`Story` instances which have the same owner and the same set of
-        :class:`StoryItem`s. For all other :class:`Story` instances this ID is different.
+        :class:`StoryItem`. For all other :class:`Story` instances this ID is different.
         """
         if not self._unique_id:
             id_list = [item.mediaid for item in self.get_items()]
@@ -805,7 +869,7 @@ def save_structure_to_file(structure: JsonExportable, filename: str) -> None:
 
 def load_structure_from_file(context: InstaloaderContext, filename: str) -> JsonExportable:
     """Loads a :class:`Post`, :class:`Profile` or :class:`StoryItem` from a '.json' or '.json.xz' file that
-    has been saved by :func:`save_structure_from_file`.
+    has been saved by :func:`save_structure_to_file`.
 
     :param context: :attr:`Instaloader.context` linked to the new object, used for additional queries if neccessary.
     :param filename: Filename, ends in '.json' or '.json.xz'

test/instaloader_unittests.py

@@ -4,6 +4,7 @@ import os
 import shutil
 import tempfile
 import unittest
+from itertools import islice
 
 import instaloader
 
@@ -112,6 +113,11 @@ class TestInstaloaderLoggedIn(TestInstaloaderAnonymously):
     def test_test_login(self):
         self.assertEqual(OWN_USERNAME, self.L.test_login())
 
+    def test_followees_and_stories(self):
+        profile = instaloader.Profile.from_username(self.L.context, OWN_USERNAME)
+        for f in islice(profile.get_followees(), PAGING_MAX_COUNT):
+            self.L.download_profile(f.username, profile_pic=False, download_stories_only=True)
+
     def test_get_followees(self):
         profile = instaloader.Profile.from_username(self.L.context, OWN_USERNAME)
         for f in profile.get_followees():