Release of version 3.3.10

When fetching comments, likes, followees or followers, output a warning
that the used queries are rate limited.

docs/_static/style.css

@@ -1,3 +1,12 @@
 code {
     color: #222;
-}
+    background-color: #f5f5f5;
+}
+
+dd {
+    padding-left: 8px;
+}
+
+dl.option dd {
+    padding-left: 12px;
+}

docs/_templates/globaltoc.html

@@ -0,0 +1,14 @@
+<li class="dropdown globaltoc-container">
+  <a role="button"
+     id="dLabelGlobalToc"
+     data-toggle="dropdown"
+     data-target="#"
+     href="{{ pathto(master_doc) }}">{{ theme_navbar_site_name }} <b class="caret"></b></a>
+  <ul class="dropdown-menu globaltoc"
+      role="menu"
+      aria-labelledby="dLabelGlobalToc">
+    {{ toctree(maxdepth=theme_globaltoc_depth|toint, collapse=False, includehidden=theme_globaltoc_includehidden|tobool) }}
+    <li role="separator" class="divider"></li>
+    <li class="toctree-l1"><a class="reference internal" href="genindex.html">Index</a></li>
+  </ul>
+</li>

docs/_templates/layout.html

@@ -1,4 +1,11 @@
 {% extends "!layout.html" %}
+{% block htmltitle %}
+  {% if pagename == "index" %}
+  <title>{{ title|striptags|e }}</title>
+  {% else %}
+  <title>{{ title|striptags|e }}{{ titlesuffix }}</title>
+  {% endif %}
+{% endblock %}
 {% block extrahead %}
 {{ super() }}
   <meta name="google-site-verification" content="FNUsDdkOJM9obm9QEdrNBVd9AtAzHzA-aaSMCUrS6C8" />

docs/_templates/navbar.html

@@ -14,7 +14,7 @@
           {% if theme_navbar_title -%}{{ theme_navbar_title|e }}{%- else -%}{{ project|e }}{%- endif -%}
         </a>
         <span class="navbar-text navbar-version pull-left">
-          <a href="https://github.com/instaloader/instaloader/releases/tag/v{{current_release}}" title="Released on {{current_release_date}}.">v<b>{{current_release}}</b></a>
+          <a href="https://github.com/instaloader/instaloader/releases/tag/v{{current_release}}" title="Released on {{current_release_date}}">v<b>{{current_release}}</b></a>
         </span>
       </div>
 
@@ -27,7 +27,7 @@
             {% endif %}
             {% block navbartoc %}
               {% include "globaltoc.html" %}
-              {% if theme_navbar_pagenav %}
+              {% if theme_navbar_pagenav and pagename not in ("index", "genindex", "installation") %}
                 {% include "navbartoc.html" %}
               {% endif %}
             {% endblock %}
@@ -46,7 +46,7 @@
                 <ul class="dropdown-menu" role="menu" aria-labelledby="dLabelUsefulLinks">
                   <li><a href="https://github.com/instaloader/instaloader/issues">Issue Tracker</a></li>
                   <li><a href="https://github.com/instaloader/instaloader/releases">Version History</a></li>
-                  <li><a href="https://pypi.python.org/pypi/instaloader">Instaloader on PyPI</a></li>
+                  <li><a href="https://pypi.org/project/instaloader/">Instaloader on PyPI</a></li>
                   <li><a href="https://github.com/instaloader/instaloader">Instaloader on GitHub</a></li>
                   <li role="separator" class="divider"></li>
                   <li style="padding-left: 20px;"><a class="github-button" href="https://github.com/instaloader/instaloader" data-icon="octicon-star" data-size="large" data-show-count="true" aria-label="Star Instaloader on GitHub">Star</a></li>

docs/conf.py

@@ -212,7 +212,7 @@ html_domain_indices = False
 
 # If false, no index is generated.
 #
-html_use_index = False
+html_use_index = True
 
 # If true, the index is split into individual pages for each letter.
 #

docs/index.rst

@@ -5,6 +5,8 @@
       saved media, and their metadata, comments and captions.
       Written in Python.
 
+.. title:: Instaloader — Download Instagram Photos and Metadata
+
 Instaloader
 ===========
 
@@ -43,8 +45,8 @@ With `Python <https://www.python.org/>`__ installed, do::
                 :stories | :feed | :saved
 
 
-Table of Contents
------------------
+Instaloader Documentation
+-------------------------
 
 .. toctree::
    :maxdepth: 2
@@ -59,7 +61,7 @@ Useful Links
 ------------
 
 - `Git Repository (on GitHub) <https://github.com/instaloader/instaloader>`__
-- `PyPI Project Page <https://pypi.python.org/pypi/instaloader>`__
+- `PyPI Project Page <https://pypi.org/project/instaloader/>`__
 - `Issue Tracker / Bug Tracker <https://github.com/instaloader/instaloader/issues>`__
 - `Version History <https://github.com/instaloader/instaloader/releases>`__
 

instaloader.py

@@ -3,6 +3,7 @@
 """Download pictures (or videos) along with their captions and other metadata from Instagram."""
 import ast
 import getpass
+import hashlib
 import json
 import os
 import pickle
@@ -30,7 +31,7 @@ import requests.utils
 import urllib3
 
 
-__version__ = '3.3.4'
+__version__ = '3.3.10'
 
 # NOTE: duplicated in README.rst and docs/index.rst
 USAGE_STRING = """
@@ -201,6 +202,7 @@ class Post:
         self._profile = profile
         self._profile_id = profile_id
         self._full_metadata_dict = None
+        self._rhx_gis = None
 
     @classmethod
     def from_shortcode(cls, instaloader: 'Instaloader', shortcode: str):
@@ -239,11 +241,9 @@ class Post:
     @property
     def _full_metadata(self) -> Dict[str, Any]:
         if not self._full_metadata_dict:
-            pic_json = self._instaloader.get_json("p/{0}/".format(self.shortcode), params={'__a': 1})
-            if "graphql" in pic_json:
-                self._full_metadata_dict = pic_json["graphql"]["shortcode_media"]
-            else:
-                self._full_metadata_dict = pic_json["media"]
+            pic_json = self._instaloader.get_json("p/{0}/".format(self.shortcode), params={})
+            self._full_metadata_dict = pic_json['entry_data']['PostPage'][0]['graphql']['shortcode_media']
+            self._rhx_gis = pic_json['rhx_gis']
         return self._full_metadata_dict
 
     def _field(self, *keys) -> Any:
@@ -377,6 +377,7 @@ class Post:
         Each comment is represented by a dictionary having the keys text, created_at, id and owner, which is a
         dictionary with keys username, profile_pic_url and id.
         """
+        # pylint:disable=protected-access
         if self.comments == 0:
             # Avoid doing additional requests if there are no comments
             return
@@ -384,9 +385,12 @@ class Post:
         if self.comments == len(comment_edges):
             # If the Post's metadata already contains all comments, don't do GraphQL requests to obtain them
             yield from (comment['node'] for comment in comment_edges)
+            return
+        self._instaloader._log('(Warning: Fetching comments uses many queries, which are rate limited.)')
         yield from self._instaloader.graphql_node_list(17852405266163336, {'shortcode': self.shortcode},
                                                        'https://www.instagram.com/p/' + self.shortcode + '/',
-                                                       lambda d: d['data']['shortcode_media']['edge_media_to_comment'])
+                                                       lambda d: d['data']['shortcode_media']['edge_media_to_comment'],
+                                                       rhx_gis=self._rhx_gis)
 
     def get_likes(self) -> Iterator[Dict[str, Any]]:
         """Iterate over all likes of the post.
@@ -394,6 +398,7 @@ class Post:
         Each like is represented by a dictionary having the keys username, followed_by_viewer, id, is_verified,
         requested_by_viewer, followed_by_viewer, profile_pic_url.
         """
+        # pylint:disable=protected-access
         if self.likes == 0:
             # Avoid doing additional requests if there are no comments
             return
@@ -401,9 +406,12 @@ class Post:
         if self.likes == len(likes_edges):
             # If the Post's metadata already contains all likes, don't do GraphQL requests to obtain them
             yield from (like['node'] for like in likes_edges)
+            return
+        self._instaloader._log('(Warning: Fetching likes uses many queries, which are rate limited.)')
         yield from self._instaloader.graphql_node_list("1cb6ec562846122743b61e492c85999f", {'shortcode': self.shortcode},
                                                        'https://www.instagram.com/p/' + self.shortcode + '/',
-                                                       lambda d: d['data']['shortcode_media']['edge_liked_by'])
+                                                       lambda d: d['data']['shortcode_media']['edge_liked_by'],
+                                                       rhx_gis=self._rhx_gis)
 
     def get_location(self) -> Optional[Dict[str, str]]:
         """If the Post has a location, returns a dictionary with fields 'lat' and 'lng'."""
@@ -456,7 +464,7 @@ class _PathPattern(str):
 
 
 class Instaloader:
-    GRAPHQL_PAGE_LENGTH = 200
+    GRAPHQL_PAGE_LENGTH = 50
 
     def __init__(self,
                  sleep: bool = True, quiet: bool = False,
@@ -502,7 +510,7 @@ class Instaloader:
         self.error_log = []
 
         # For the adaption of sleep intervals (rate control)
-        self.previous_queries = dict()
+        self.query_timestamps = list()
 
     @property
     def is_logged_in(self) -> bool:
@@ -519,10 +527,10 @@ class Instaloader:
                                  self.download_geotags,
                                  self.save_captions, self.download_comments,
                                  self.save_metadata, self.max_connection_attempts)
-        new_loader.previous_queries = self.previous_queries
+        new_loader.query_timestamps = self.query_timestamps
         yield new_loader
         self.error_log.extend(new_loader.error_log)
-        self.previous_queries = new_loader.previous_queries
+        self.query_timestamps = new_loader.query_timestamps
 
     def _log(self, *msg, sep='', end='\n', flush=False):
         """Log a message to stdout that can be suppressed with --quiet."""
@@ -600,29 +608,25 @@ class Instaloader:
         :raises QueryReturnedNotFoundException: When the server responds with a 404.
         :raises ConnectionException: When query repeatedly failed.
         """
-        def graphql_query_waittime(query_id: int, untracked_queries: bool = False) -> int:
+        def graphql_query_waittime(untracked_queries: bool = False) -> int:
             sliding_window = 660
-            timestamps = self.previous_queries.get(query_id)
-            if not timestamps:
+            if not self.query_timestamps:
                 return sliding_window if untracked_queries else 0
             current_time = time.monotonic()
-            timestamps = list(filter(lambda t: t > current_time - sliding_window, timestamps))
-            self.previous_queries[query_id] = timestamps
-            if len(timestamps) < 100 and not untracked_queries:
+            self.query_timestamps = list(filter(lambda t: t > current_time - sliding_window, self.query_timestamps))
+            if len(self.query_timestamps) < 20 and not untracked_queries:
                 return 0
-            return round(min(timestamps) + sliding_window - current_time) + 6
-        is_graphql_query = 'query_id' in params and 'graphql/query' in path
+            return round(min(self.query_timestamps) + sliding_window - current_time) + 6
+        is_graphql_query = 'graphql/query' in path
         if is_graphql_query:
-            query_id = params['query_id']
-            waittime = graphql_query_waittime(query_id)
+            waittime = graphql_query_waittime()
             if waittime > 0:
                 self._log('\nToo many queries in the last time. Need to wait {} seconds.'.format(waittime))
                 time.sleep(waittime)
-            timestamp_list = self.previous_queries.get(query_id)
-            if timestamp_list is not None:
-                timestamp_list.append(time.monotonic())
+            if self.query_timestamps is not None:
+                self.query_timestamps.append(time.monotonic())
             else:
-                self.previous_queries[query_id] = [time.monotonic()]
+                self.query_timestamps = [time.monotonic()]
         sess = session if session else self.session
         try:
             self._sleep()
@@ -633,7 +637,13 @@ class Instaloader:
                 raise TooManyRequests("429 - Too Many Requests")
             if resp.status_code != 200:
                 raise ConnectionException("HTTP error code {}.".format(resp.status_code))
-            resp_json = resp.json()
+            if not is_graphql_query and not "__a" in params and host == "www.instagram.com":
+                match = re.search(r'window\._sharedData = (.*);</script>', resp.text)
+                if match is None:
+                    raise ConnectionException("Could not find \"window._sharedData\" in html response.")
+                return json.loads(match.group(1))
+            else:
+                resp_json = resp.json()
             if 'status' in resp_json and resp_json['status'] != "ok":
                 if 'message' in resp_json:
                     raise ConnectionException("Returned \"{}\" status, message \"{}\".".format(resp_json['status'],
@@ -653,7 +663,7 @@ class Instaloader:
                 if isinstance(err, TooManyRequests):
                     print(textwrap.fill(text_for_429), file=sys.stderr)
                     if is_graphql_query:
-                        waittime = graphql_query_waittime(query_id=params['query_id'], untracked_queries=True)
+                        waittime = graphql_query_waittime(untracked_queries=True)
                         if waittime > 0:
                             self._log('The request will be retried in {} seconds.'.format(waittime))
                             time.sleep(waittime)
@@ -693,7 +703,7 @@ class Instaloader:
         return session
 
     def graphql_query(self, query_identifier: Union[int, str], variables: Dict[str, Any],
-                      referer: Optional[str] = None) -> Dict[str, Any]:
+                      referer: Optional[str] = None, rhx_gis: Optional[str] = None) -> Dict[str, Any]:
         """
         Do a GraphQL Query.
 
@@ -711,9 +721,18 @@ class Instaloader:
         tmpsession.headers['accept'] = '*/*'
         if referer is not None:
             tmpsession.headers['referer'] = urllib.parse.quote(referer)
+
+        variables_json = json.dumps(variables, separators=(',', ':'))
+
+        if rhx_gis:
+            values = "{}:{}".format(rhx_gis, variables_json)
+            x_instagram_gis = hashlib.md5(values.encode()).hexdigest()
+            tmpsession.cookies.set('ig_pr', '2')
+            tmpsession.headers['x-instagram-gis'] = x_instagram_gis
+
         resp_json = self.get_json('graphql/query',
                                   params={'query_id' if isinstance(query_identifier, int) else 'query_hash': query_identifier,
-                                          'variables': json.dumps(variables, separators=(',', ':'))},
+                                          'variables': variables_json},
                                   session=tmpsession)
         if 'status' not in resp_json:
             self.error("GraphQL response did not contain a \"status\" field.")
@@ -738,20 +757,21 @@ class Instaloader:
     def get_id_by_username(self, profile: str) -> int:
         """Each Instagram profile has its own unique ID which stays unmodified even if a user changes
         his/her username. To get said ID, given the profile's name, you may call this function."""
-        return int(self.get_profile_metadata(profile)['user']['id'])
+        return int(self.get_profile_metadata(profile)[0]['user']['id'])
 
     def graphql_node_list(self, query_identifier: Union[int, str], query_variables: Dict[str, Any],
                           query_referer: Optional[str],
-                          edge_extractor: Callable[[Dict[str, Any]], Dict[str, Any]]) -> Iterator[Dict[str, Any]]:
+                          edge_extractor: Callable[[Dict[str, Any]], Dict[str, Any]],
+                          rhx_gis: Optional[str] = None) -> Iterator[Dict[str, Any]]:
         """Retrieve a list of GraphQL nodes."""
         query_variables['first'] = Instaloader.GRAPHQL_PAGE_LENGTH
-        data = self.graphql_query(query_identifier, query_variables, query_referer)
+        data = self.graphql_query(query_identifier, query_variables, query_referer, rhx_gis)
         while True:
             edge_struct = edge_extractor(data)
             yield from [edge['node'] for edge in edge_struct['edges']]
             if edge_struct['page_info']['has_next_page']:
                 query_variables['after'] = edge_struct['page_info']['end_cursor']
-                data = self.graphql_query(query_identifier, query_variables, query_referer)
+                data = self.graphql_query(query_identifier, query_variables, query_referer, rhx_gis)
             else:
                 break
 
@@ -763,6 +783,7 @@ class Instaloader:
 
         :param profile: Name of profile to lookup followers.
         """
+        self._log('(Warning: Fetching followers uses many queries, which are rate limited.)')
         yield from self.graphql_node_list(17851374694183129, {'id': str(self.get_id_by_username(profile))},
                                           'https://www.instagram.com/' + profile + '/',
                                           lambda d: d['data']['user']['edge_followed_by'])
@@ -775,6 +796,7 @@ class Instaloader:
 
         :param profile: Name of profile to lookup followers.
         """
+        self._log('(Warning: Fetching followees uses many queries, which are rate limited.)')
         yield from self.graphql_node_list(17874545323001329, {'id': str(self.get_id_by_username(profile))},
                                           'https://www.instagram.com/' + profile + '/',
                                           lambda d: d['data']['user']['edge_follow'])
@@ -1141,7 +1163,7 @@ class Instaloader:
             self._log("Retrieving stories from profile {}.".format(name))
             totalcount = len(user_stories["items"])
             count = 1
-            for item in user_stories["items"]:
+            for item in reversed(user_stories["items"]):
                 self._log("[%3i/%3i] " % (count, totalcount), end="", flush=True)
                 count += 1
                 with self._error_catcher('Download story from user {}'.format(name)):
@@ -1255,7 +1277,7 @@ class Instaloader:
 
         if not self.is_logged_in:
             return
-        data = self.get_profile_metadata(self.username)
+        data, full_metadata = self.get_profile_metadata(self.username)
         user_id = data["user"]["id"]
 
         while True:
@@ -1275,7 +1297,8 @@ class Instaloader:
                 break
             data = self.graphql_query("f883d95537fbcd400f466f63d42bd8a1",
                                       {'id': user_id, 'first': Instaloader.GRAPHQL_PAGE_LENGTH,
-                                       'after': saved_media["page_info"]["end_cursor"]})['data']
+                                       'after': saved_media["page_info"]["end_cursor"]},
+                                      rhx_gis=full_metadata['rhx_gis'])['data']
 
     def download_saved_posts(self, max_count: int = None, fast_update: bool = False,
                              filter_func: Optional[Callable[[Post], bool]] = None) -> None:
@@ -1302,10 +1325,19 @@ class Instaloader:
 
     def get_hashtag_posts(self, hashtag: str) -> Iterator[Post]:
         """Get Posts associated with a #hashtag."""
-        yield from (Post(self, node) for node in
-                    self.graphql_node_list(17875800862117404, {'tag_name': hashtag},
-                                           'https://www.instagram.com/explore/tags/{0}/'.format(hashtag),
-                                           lambda d: d['data']['hashtag']['edge_hashtag_to_media']))
+        has_next_page = True
+        end_cursor = None
+        while has_next_page:
+            if end_cursor:
+                params = {'__a': 1, 'max_id': end_cursor}
+            else:
+                params = {'__a': 1}
+            hashtag_data = self.get_json('explore/tags/{0}/'.format(hashtag),
+                                         params)['graphql']['hashtag']['edge_hashtag_to_media']
+            yield from (Post(self, edge['node']) for edge in
+                        hashtag_data['edges'])
+            has_next_page = hashtag_data['page_info']['has_next_page']
+            end_cursor = hashtag_data['page_info']['end_cursor']
 
     def download_hashtag(self, hashtag: str,
                          max_count: Optional[int] = None,
@@ -1364,6 +1396,9 @@ class Instaloader:
                     self._log("Profile {0} does not match the stored unique ID {1}.".format(profile, profile_id))
                 else:
                     self._log("Trying to find profile {0} using its unique ID {1}.".format(profile, profile_id))
+                if not self.is_logged_in:
+                    self.error("Profile {} changed its name. If you use --login=USERNAME, I can find out the new name.")
+                    raise LoginRequiredException("Login required to obtain username from userid")
                 newname = self.get_username_by_id(profile_id)
                 self._log("Profile {0} has changed its name to {1}.".format(profile, newname))
                 if ((format_string_contains_key(self.dirname_pattern, 'profile') or
@@ -1389,15 +1424,15 @@ class Instaloader:
             return profile, profile_id
         raise ProfileNotExistsException("Profile {0} does not exist.".format(profile))
 
-    def get_profile_metadata(self, profile_name: str) -> Dict[str, Any]:
+    def get_profile_metadata(self, profile_name: str) -> Tuple[Dict[str, Any], Dict[str, Any]]:
         """Retrieves a profile's metadata, for use with e.g. :meth:`get_profile_posts` and :meth:`check_profile_id`."""
         try:
-            metadata = self.get_json('{}/'.format(profile_name), params={'__a': 1})
-            return metadata['graphql'] if 'graphql' in metadata else metadata
+            metadata = self.get_json('{}/'.format(profile_name), params={})
+            return metadata['entry_data']['ProfilePage'][0]['graphql'], metadata
         except QueryReturnedNotFoundException:
             raise ProfileNotExistsException('Profile {} does not exist.'.format(profile_name))
 
-    def get_profile_posts(self, profile_metadata: Dict[str, Any]) -> Iterator[Post]:
+    def get_profile_posts(self, profile_metadata: Dict[str, Any], rhx_gis: str) -> Iterator[Post]:
         """Retrieve all posts from a profile."""
         profile_name = profile_metadata['user']['username']
         profile_id = int(profile_metadata['user']['id'])
@@ -1418,7 +1453,7 @@ class Instaloader:
             data = self.graphql_query(17888483320059182, {'id': profile_metadata['user']['id'],
                                                           'first': Instaloader.GRAPHQL_PAGE_LENGTH,
                                                           'after': end_cursor},
-                                      'https://www.instagram.com/{0}/'.format(profile_name))
+                                      referer='https://www.instagram.com/{0}/'.format(profile_name), rhx_gis=rhx_gis)
             media = data['data']['user']['edge_owner_to_timeline_media']
             yield from (Post(self, edge['node'], profile=profile_name, profile_id=profile_id)
                         for edge in media['edges'])
@@ -1438,14 +1473,14 @@ class Instaloader:
         with suppress(ProfileNotExistsException):
             # ProfileNotExistsException is raised again later in check_profile_id() when we search the profile, so we
             # must suppress it here.
-            profile_metadata = self.get_profile_metadata(name)
+            profile_metadata, full_metadata = self.get_profile_metadata(name)
 
         # check if profile does exist or name has changed since last download
         # and update name and json data if necessary
         name_updated, profile_id = self.check_profile_id(name, profile_metadata)
         if name_updated != name:
             name = name_updated
-            profile_metadata = self.get_profile_metadata(name)
+            profile_metadata, full_metadata = self.get_profile_metadata(name)
 
         # Download profile picture
         if profile_pic or profile_pic_only:
@@ -1480,7 +1515,7 @@ class Instaloader:
         else:
             totalcount = profile_metadata["user"]["edge_owner_to_timeline_media"]["count"]
         count = 1
-        for post in self.get_profile_posts(profile_metadata):
+        for post in self.get_profile_posts(profile_metadata, rhx_gis=full_metadata['rhx_gis']):
             self._log("[%3i/%3i] " % (count, totalcount), end="", flush=True)
             count += 1
             if filter_func is not None and not filter_func(post):