Release of version 3.3.10

When fetching comments, likes, followees or followers, output a warning
that the used queries are rate limited.

docs/_static/style.css

@@ -0,0 +1,12 @@
+code {
+    color: #222;
+    background-color: #f5f5f5;
+}
+
+dd {
+    padding-left: 8px;
+}
+
+dl.option dd {
+    padding-left: 12px;
+}

docs/_templates/caption.html

@@ -1,2 +0,0 @@
-<h2><a href="index.html">Instaloader</a></h2>
-<p>Download pictures (or videos) along with their captions and other metadata from Instagram.</p>

docs/_templates/globaltoc.html

@@ -0,0 +1,14 @@
+<li class="dropdown globaltoc-container">
+  <a role="button"
+     id="dLabelGlobalToc"
+     data-toggle="dropdown"
+     data-target="#"
+     href="{{ pathto(master_doc) }}">{{ theme_navbar_site_name }} <b class="caret"></b></a>
+  <ul class="dropdown-menu globaltoc"
+      role="menu"
+      aria-labelledby="dLabelGlobalToc">
+    {{ toctree(maxdepth=theme_globaltoc_depth|toint, collapse=False, includehidden=theme_globaltoc_includehidden|tobool) }}
+    <li role="separator" class="divider"></li>
+    <li class="toctree-l1"><a class="reference internal" href="genindex.html">Index</a></li>
+  </ul>
+</li>

docs/_templates/layout.html

@@ -1,5 +1,18 @@
 {% extends "!layout.html" %}
+{% block htmltitle %}
+  {% if pagename == "index" %}
+  <title>{{ title|striptags|e }}</title>
+  {% else %}
+  <title>{{ title|striptags|e }}{{ titlesuffix }}</title>
+  {% endif %}
+{% endblock %}
 {% block extrahead %}
-<meta name="google-site-verification" content="FNUsDdkOJM9obm9QEdrNBVd9AtAzHzA-aaSMCUrS6C8" />
-<script async defer src="https://buttons.github.io/buttons.js"></script>
+{{ super() }}
+  <meta name="google-site-verification" content="FNUsDdkOJM9obm9QEdrNBVd9AtAzHzA-aaSMCUrS6C8" />
+  {% if pagename == "index" %}
+  <link rel="canonical" href="https://instaloader.github.io/" />
+  {% else %}
+  <link rel="canonical" href="https://instaloader.github.io/{{ pagename }}.html" />
+  {% endif %}
+  <script async defer src="https://buttons.github.io/buttons.js"></script>
 {% endblock %}

docs/_templates/links.html

@@ -1,23 +0,0 @@
-{% if next %}
-<h3>Next</h3>
-<ul>
-  <li class="toctree-l1"><a class="reference internal" href="{{ next.link|e }}">{{ next.title }}</a></li>
-</ul>
-{% endif %}
-<h3>Current Release</h3>
-<ul>
-  <li class="toctree-l1"><a href="https://github.com/instaloader/instaloader/releases/tag/v{{current_release}}">Version <b>{{current_release}}</b></a></li>
-  <li>Released on <b>{{current_release_date}}</b>.</li>
-</ul>
-<h3>Links</h3>
-<ul>
-  <li><a href="https://github.com/instaloader/instaloader/issues">Issue Tracker</a></li>
-  <li><a href="https://github.com/instaloader/instaloader/releases">Version History</a></li>
-  <li><a href="https://pypi.python.org/pypi/instaloader">Instaloader on PyPI</a></li>
-  <li><a href="https://github.com/instaloader/instaloader">Instaloader on GitHub</a></li>
-</ul>
-<ul>
-  <li>
-    <a class="github-button" href="https://github.com/instaloader/Instaloader" data-icon="octicon-star" data-size="large" data-show-count="true" aria-label="Star instaloader/Instaloader on GitHub">Star</a>
-  </li>
-</ul>

docs/_templates/navbar.html

@@ -0,0 +1,58 @@
+  <div id="navbar" class="{{ theme_navbar_class }} navbar-default {% if theme_navbar_fixed_top|tobool -%} navbar-fixed-top{%- endif -%}">
+    <div class="container">
+      <div class="navbar-header">
+        <!-- .btn-navbar is used as the toggle for collapsed navbar content -->
+        <button type="button" class="navbar-toggle" data-toggle="collapse" data-target=".nav-collapse">
+          <span class="icon-bar"></span>
+          <span class="icon-bar"></span>
+          <span class="icon-bar"></span>
+        </button>
+        <a class="navbar-brand" href="{{ pathto(master_doc) }}">
+          {%- block sidebarlogo %}
+            {%- if logo %}<span><img src="{{ pathto('_static/' + logo, 1) }}"></span>{%- endif %}
+          {%- endblock %}
+          {% if theme_navbar_title -%}{{ theme_navbar_title|e }}{%- else -%}{{ project|e }}{%- endif -%}
+        </a>
+        <span class="navbar-text navbar-version pull-left">
+          <a href="https://github.com/instaloader/instaloader/releases/tag/v{{current_release}}" title="Released on {{current_release_date}}">v<b>{{current_release}}</b></a>
+        </span>
+      </div>
+
+        <div class="collapse navbar-collapse nav-collapse">
+          <ul class="nav navbar-nav">
+            {% if theme_navbar_links %}
+              {%- for link in theme_navbar_links %}
+                <li><a href="{{ pathto(*link[1:]) }}">{{ link[0] }}</a></li>
+              {%- endfor %}
+            {% endif %}
+            {% block navbartoc %}
+              {% include "globaltoc.html" %}
+              {% if theme_navbar_pagenav and pagename not in ("index", "genindex", "installation") %}
+                {% include "navbartoc.html" %}
+              {% endif %}
+            {% endblock %}
+            {% if theme_navbar_sidebarrel %}
+              {% block sidebarrel %}
+                {% include "relations.html" %}
+              {% endblock %}
+            {% endif %}
+            </ul>
+            <ul class="nav navbar-nav navbar-right">
+              <li class="dropdown">
+                <a role="button" id="dLabelUsefulLinks" data-toggle="dropdown" data-target="#" href="#">
+                  Useful Links
+                  <b class="caret"></b>
+                </a>
+                <ul class="dropdown-menu" role="menu" aria-labelledby="dLabelUsefulLinks">
+                  <li><a href="https://github.com/instaloader/instaloader/issues">Issue Tracker</a></li>
+                  <li><a href="https://github.com/instaloader/instaloader/releases">Version History</a></li>
+                  <li><a href="https://pypi.org/project/instaloader/">Instaloader on PyPI</a></li>
+                  <li><a href="https://github.com/instaloader/instaloader">Instaloader on GitHub</a></li>
+                  <li role="separator" class="divider"></li>
+                  <li style="padding-left: 20px;"><a class="github-button" href="https://github.com/instaloader/instaloader" data-icon="octicon-star" data-size="large" data-show-count="true" aria-label="Star Instaloader on GitHub">Star</a></li>
+                </ul>
+              </li>
+            </ul>
+        </div>
+    </div>
+  </div>

docs/_templates/rtdmessage.html

@@ -1,3 +0,0 @@
-<h3>New Documentation URL</h3>
-<div class="danger"><p style="margin-left: 1em"><b>Instaloader's Documentation has been moved to
-    <a class="reference external" href="https://instaloader.github.io/">https://instaloader.github.io</a>.</b></p></div>

docs/as-module.rst

@@ -1,7 +1,7 @@
 .. meta::
    :description:
-      Instaloader can also be used as a powerful and easy-to-use
-      Python API for Instagram, allowing to download media and metadata.
+      Documentation of Instaloader module, a powerful and easy-to-use
+      Python library to download Instagram media and metadata.
 
 Python Module :mod:`instaloader`
 --------------------------------

docs/basic-usage.rst

@@ -1,3 +1,10 @@
+.. meta::
+   :description:
+      How to download pictures from Instagram. Description of basic
+      usage of Instaloader, free tool to download photos from public
+      and private profiles, hashtags, stories, feeds, saved media, and
+      their metadata, comments and captions.
+
 Download Pictures from Instagram
 ---------------------------------
 

docs/conf.py

@@ -20,6 +20,9 @@
 import os
 import subprocess
 import sys
+
+import sphinx_bootstrap_theme
+
 sys.path.insert(0, os.path.abspath('..'))
 
 # -- General configuration ------------------------------------------------
@@ -133,16 +136,20 @@ todo_include_todos = False
 # The theme to use for HTML and HTML Help pages.  See the documentation for
 # a list of builtin themes.
 #
-html_theme = 'alabaster'
+html_theme = 'bootstrap'
+html_theme_path = sphinx_bootstrap_theme.get_html_theme_path()
 
 # Theme options are theme-specific and customize the look and feel of a theme
 # further.  For a list of options available for each theme, see the
 # documentation.
 #
 html_theme_options = {
-    'show_powered_by': False,
-    'sidebar_width': '290px',
-    'page_width': '935px' }
+    'navbar_site_name': 'Site Contents',
+    'navbar_pagenav_name': 'Page Contents',
+    'navbar_pagenav': True,
+    'navbar_sidebarrel': True,
+    'nosidebar': True,
+     }
 
 # Add any paths that contain custom themes here, relative to this directory.
 # html_theme_path = []
@@ -191,10 +198,8 @@ html_static_path = ['_static']
 
 # Custom sidebar templates, maps document names to template names.
 #
-if not os.environ.get("READTHEDOCS"):
-    html_sidebars = {'**': ["caption.html", "globaltoc.html", "relations.html", "links.html"] }
-else:
-    html_sidebars = {'**': ["caption.html", "rtdmessage.html", "globaltoc.html", "relations.html", "links.html"] }
+#html_sidebars = {'**': ["relations.html", "links.html"] }
+html_sidebars = {}
 
 # Additional templates that should be rendered to pages, maps page names to
 # template names.
@@ -207,7 +212,7 @@ html_domain_indices = False
 
 # If false, no index is generated.
 #
-html_use_index = False
+html_use_index = True
 
 # If true, the index is split into individual pages for each letter.
 #
@@ -362,3 +367,6 @@ current_release = subprocess.check_output(["git", "describe", "--abbrev=0"]).dec
 current_release_date = subprocess.check_output(["git", "log", "-1", "--tags", "--format=%ad", "--date=format:%e %b %Y"]).decode("ascii")[:-1]
 
 html_context = {'current_release': current_release, 'current_release_date': current_release_date}
+
+def setup(app):
+    app.add_stylesheet("style.css")

docs/index.rst

@@ -1,8 +1,11 @@
 .. meta::
    :description:
-      Command line tool to download pictures (and videos) from Instagram.
-      Instaloader downloads public and private profiles, hashtags, user stories,
-      feeds, saved media, comments, geotags, captions and other metadata of each post.
+      Free command line tool to download photos from Instagram.
+      Scrapes public and private profiles, hashtags, stories, feeds,
+      saved media, and their metadata, comments and captions.
+      Written in Python.
+
+.. title:: Instaloader — Download Instagram Photos and Metadata
 
 Instaloader
 ===========
@@ -42,8 +45,8 @@ With `Python <https://www.python.org/>`__ installed, do::
                 :stories | :feed | :saved
 
 
-Table of Contents
------------------
+Instaloader Documentation
+-------------------------
 
 .. toctree::
    :maxdepth: 2
@@ -58,7 +61,7 @@ Useful Links
 ------------
 
 - `Git Repository (on GitHub) <https://github.com/instaloader/instaloader>`__
-- `PyPI Project Page <https://pypi.python.org/pypi/instaloader>`__
+- `PyPI Project Page <https://pypi.org/project/instaloader/>`__
 - `Issue Tracker / Bug Tracker <https://github.com/instaloader/instaloader/issues>`__
 - `Version History <https://github.com/instaloader/instaloader/releases>`__
 

docs/installation.rst

@@ -28,7 +28,7 @@ will be installed automatically, if it is not already installed.
 
 - If you do not want to use pip, even though it is highly recommended,
   and prefer to **install Instaloader manually**,
-  `Download the Source <https://github.com/instaloader/instaloader/releases>`__,
+  `Download the Source <https://github.com/instaloader/instaloader/releases/latest>`__,
   extract the Zip or Tarball and execute ``instaloader.py`` from there.
 
 - On **Arch Linux**, you may install Instaloader using the
@@ -36,3 +36,6 @@ will be installed automatically, if it is not already installed.
 
 - On **Gentoo Linux**, you may install Instaloader using the
   `Instaloader Ebuild <https://bugs.gentoo.org/642100>`__.
+
+- On **Windows 10**, you may download the standalone executable from the
+  `current release page <https://github.com/instaloader/instaloader/releases/latest>`__.

docs/requirements.txt

@@ -1,3 +1,4 @@
 requests
 sphinx
 sphinx-autodoc-typehints
+sphinx-bootstrap-theme

instaloader.py

@@ -3,9 +3,11 @@
 """Download pictures (or videos) along with their captions and other metadata from Instagram."""
 import ast
 import getpass
+import hashlib
 import json
 import os
 import pickle
+import platform
 import random
 import re
 import shutil
@@ -29,8 +31,14 @@ import requests.utils
 import urllib3
 
 
-__version__ = '3.3.2'
+__version__ = '3.3.10'
 
+# NOTE: duplicated in README.rst and docs/index.rst
+USAGE_STRING = """
+{0} [--comments] [--geotags] [--stories]
+{2:{1}} [--login YOUR-USERNAME] [--fast-update]
+{2:{1}} profile | "#hashtag" | :stories | :feed | :saved
+{0} --help""".format(sys.argv[0], len(sys.argv[0]), '')
 
 try:
     # pylint:disable=wrong-import-position
@@ -127,10 +135,10 @@ def mediaid_to_shortcode(mediaid: int) -> str:
     return b64encode(mediaid.to_bytes(9, 'big'), b'-_').decode().replace('A', ' ').lstrip().replace(' ','A')
 
 
-def format_string_contains_key(format_string: str, key: str) -> bool:
+def format_string_contains_key(format_string: '_PathPattern', key: str) -> bool:
     # pylint:disable=unused-variable
     for literal_text, field_name, format_spec, conversion in string.Formatter().parse(format_string):
-        if field_name == key or field_name.startswith(key + '.'):
+        if field_name and (field_name == key or field_name.startswith(key + '.')):
             return True
     return False
 
@@ -194,6 +202,7 @@ class Post:
         self._profile = profile
         self._profile_id = profile_id
         self._full_metadata_dict = None
+        self._rhx_gis = None
 
     @classmethod
     def from_shortcode(cls, instaloader: 'Instaloader', shortcode: str):
@@ -232,11 +241,9 @@ class Post:
     @property
     def _full_metadata(self) -> Dict[str, Any]:
         if not self._full_metadata_dict:
-            pic_json = self._instaloader.get_json("p/{0}/".format(self.shortcode), params={'__a': 1})
-            if "graphql" in pic_json:
-                self._full_metadata_dict = pic_json["graphql"]["shortcode_media"]
-            else:
-                self._full_metadata_dict = pic_json["media"]
+            pic_json = self._instaloader.get_json("p/{0}/".format(self.shortcode), params={})
+            self._full_metadata_dict = pic_json['entry_data']['PostPage'][0]['graphql']['shortcode_media']
+            self._rhx_gis = pic_json['rhx_gis']
         return self._full_metadata_dict
 
     def _field(self, *keys) -> Any:
@@ -370,6 +377,7 @@ class Post:
         Each comment is represented by a dictionary having the keys text, created_at, id and owner, which is a
         dictionary with keys username, profile_pic_url and id.
         """
+        # pylint:disable=protected-access
         if self.comments == 0:
             # Avoid doing additional requests if there are no comments
             return
@@ -377,9 +385,12 @@ class Post:
         if self.comments == len(comment_edges):
             # If the Post's metadata already contains all comments, don't do GraphQL requests to obtain them
             yield from (comment['node'] for comment in comment_edges)
+            return
+        self._instaloader._log('(Warning: Fetching comments uses many queries, which are rate limited.)')
         yield from self._instaloader.graphql_node_list(17852405266163336, {'shortcode': self.shortcode},
                                                        'https://www.instagram.com/p/' + self.shortcode + '/',
-                                                       lambda d: d['data']['shortcode_media']['edge_media_to_comment'])
+                                                       lambda d: d['data']['shortcode_media']['edge_media_to_comment'],
+                                                       rhx_gis=self._rhx_gis)
 
     def get_likes(self) -> Iterator[Dict[str, Any]]:
         """Iterate over all likes of the post.
@@ -387,6 +398,7 @@ class Post:
         Each like is represented by a dictionary having the keys username, followed_by_viewer, id, is_verified,
         requested_by_viewer, followed_by_viewer, profile_pic_url.
         """
+        # pylint:disable=protected-access
         if self.likes == 0:
             # Avoid doing additional requests if there are no comments
             return
@@ -394,9 +406,12 @@ class Post:
         if self.likes == len(likes_edges):
             # If the Post's metadata already contains all likes, don't do GraphQL requests to obtain them
             yield from (like['node'] for like in likes_edges)
+            return
+        self._instaloader._log('(Warning: Fetching likes uses many queries, which are rate limited.)')
         yield from self._instaloader.graphql_node_list("1cb6ec562846122743b61e492c85999f", {'shortcode': self.shortcode},
                                                        'https://www.instagram.com/p/' + self.shortcode + '/',
-                                                       lambda d: d['data']['shortcode_media']['edge_liked_by'])
+                                                       lambda d: d['data']['shortcode_media']['edge_liked_by'],
+                                                       rhx_gis=self._rhx_gis)
 
     def get_location(self) -> Optional[Dict[str, str]]:
         """If the Post has a location, returns a dictionary with fields 'lat' and 'lng'."""
@@ -404,7 +419,7 @@ class Post:
         if loc_dict is not None:
             location_json = self._instaloader.get_json("explore/locations/{0}/".format(loc_dict["id"]),
                                                        params={'__a': 1})
-            return location_json["location"]
+            return location_json["location"] if "location" in location_json else location_json['graphql']['location']
 
     @staticmethod
     def json_encoder(obj) -> Dict[str, Any]:
@@ -440,8 +455,16 @@ class Tristate(Enum):
     always = 2
 
 
+class _PathPattern(str):
+    """Class overriding :meth:`str.format` for character substitution in paths for Windows, see issue #84."""
+
+    def format(self, *args: Any, **kwargs: Any) -> str:
+        ret = super().format(*args, **kwargs)
+        return ret.replace(':', '\ua789') if platform.system() == 'Windows' else ret
+
+
 class Instaloader:
-    GRAPHQL_PAGE_LENGTH = 200
+    GRAPHQL_PAGE_LENGTH = 50
 
     def __init__(self,
                  sleep: bool = True, quiet: bool = False,
@@ -462,18 +485,19 @@ class Instaloader:
         self.username = None
         self.sleep = sleep
         self.quiet = quiet
-        self.dirname_pattern = dirname_pattern if dirname_pattern is not None else '{target}'
+        self.dirname_pattern = _PathPattern(dirname_pattern if dirname_pattern is not None else '{target}')
         if filename_pattern is not None:
             filename_pattern = re.sub(r"({(?:post\.)?date)([:}])", r"\1_utc\2", filename_pattern)
             self.filename_pattern_old = filename_pattern.replace('{date_utc}', '{date_utc:%Y-%m-%d_%H-%M-%S}')
-            self.filename_pattern_old = re.sub(r"(?i)({(?:post\.)?date_utc:[^}]*?)_UTC",
-                                               r"\1", self.filename_pattern_old)
+            self.filename_pattern_old = _PathPattern(re.sub(r"(?i)({(?:post\.)?date_utc:[^}]*?)_UTC",
+                                                            r"\1", self.filename_pattern_old))
             filename_pattern = re.sub(r"(?i)({(date_utc|post\.date_utc):(?![^}]*UTC[^}]*).*?)}",
                                       r"\1_UTC}", filename_pattern)
-            self.filename_pattern = filename_pattern.replace('{date_utc}', '{date_utc:%Y-%m-%d_%H-%M-%S_UTC}')
+            self.filename_pattern = _PathPattern(filename_pattern.replace('{date_utc}',
+                                                                          '{date_utc:%Y-%m-%d_%H-%M-%S_UTC}'))
         else:
-            self.filename_pattern = '{date_utc:%Y-%m-%d_%H-%M-%S_UTC}'
-            self.filename_pattern_old = '{date_utc:%Y-%m-%d_%H-%M-%S}'
+            self.filename_pattern = _PathPattern('{date_utc:%Y-%m-%d_%H-%M-%S_UTC}')
+            self.filename_pattern_old = _PathPattern('{date_utc:%Y-%m-%d_%H-%M-%S}')
         self.download_videos = download_videos
         self.download_video_thumbnails = download_video_thumbnails
         self.download_geotags = download_geotags
@@ -486,7 +510,7 @@ class Instaloader:
         self.error_log = []
 
         # For the adaption of sleep intervals (rate control)
-        self.previous_queries = dict()
+        self.query_timestamps = list()
 
     @property
     def is_logged_in(self) -> bool:
@@ -503,10 +527,10 @@ class Instaloader:
                                  self.download_geotags,
                                  self.save_captions, self.download_comments,
                                  self.save_metadata, self.max_connection_attempts)
-        new_loader.previous_queries = self.previous_queries
+        new_loader.query_timestamps = self.query_timestamps
         yield new_loader
         self.error_log.extend(new_loader.error_log)
-        self.previous_queries = new_loader.previous_queries
+        self.query_timestamps = new_loader.query_timestamps
 
     def _log(self, *msg, sep='', end='\n', flush=False):
         """Log a message to stdout that can be suppressed with --quiet."""
@@ -572,51 +596,54 @@ class Instaloader:
                 self.error("[skipped by user]", repeat_at_end=False)
                 raise ConnectionException(error_string)
 
-    def get_json(self, url: str, params: Dict[str, Any],
+    def get_json(self, path: str, params: Dict[str, Any], host: str = 'www.instagram.com',
                  session: Optional[requests.Session] = None, _attempt = 1) -> Dict[str, Any]:
         """JSON request to Instagram.
 
-        :param url: URL, relative to www.instagram.com/
+        :param path: URL, relative to the given domain which defaults to www.instagram.com/
         :param params: GET parameters
+        :param host: Domain part of the URL from where to download the requested JSON; defaults to www.instagram.com
         :param session: Session to use, or None to use self.session
         :return: Decoded response dictionary
         :raises QueryReturnedNotFoundException: When the server responds with a 404.
         :raises ConnectionException: When query repeatedly failed.
         """
-        def graphql_query_waittime(query_id: int, untracked_queries: bool = False) -> int:
+        def graphql_query_waittime(untracked_queries: bool = False) -> int:
             sliding_window = 660
-            timestamps = self.previous_queries.get(query_id)
-            if not timestamps:
+            if not self.query_timestamps:
                 return sliding_window if untracked_queries else 0
             current_time = time.monotonic()
-            timestamps = list(filter(lambda t: t > current_time - sliding_window, timestamps))
-            self.previous_queries[query_id] = timestamps
-            if len(timestamps) < 100 and not untracked_queries:
+            self.query_timestamps = list(filter(lambda t: t > current_time - sliding_window, self.query_timestamps))
+            if len(self.query_timestamps) < 20 and not untracked_queries:
                 return 0
-            return round(min(timestamps) + sliding_window - current_time) + 6
-        is_graphql_query = 'query_id' in params and 'graphql/query' in url
+            return round(min(self.query_timestamps) + sliding_window - current_time) + 6
+        is_graphql_query = 'graphql/query' in path
         if is_graphql_query:
-            query_id = params['query_id']
-            waittime = graphql_query_waittime(query_id)
+            waittime = graphql_query_waittime()
             if waittime > 0:
                 self._log('\nToo many queries in the last time. Need to wait {} seconds.'.format(waittime))
                 time.sleep(waittime)
-            timestamp_list = self.previous_queries.get(query_id)
-            if timestamp_list is not None:
-                timestamp_list.append(time.monotonic())
+            if self.query_timestamps is not None:
+                self.query_timestamps.append(time.monotonic())
             else:
-                self.previous_queries[query_id] = [time.monotonic()]
+                self.query_timestamps = [time.monotonic()]
         sess = session if session else self.session
         try:
             self._sleep()
-            resp = sess.get('https://www.instagram.com/' + url, params=params)
+            resp = sess.get('https://{0}/{1}'.format(host, path), params=params)
             if resp.status_code == 404:
                 raise QueryReturnedNotFoundException("404")
             if resp.status_code == 429:
                 raise TooManyRequests("429 - Too Many Requests")
             if resp.status_code != 200:
                 raise ConnectionException("HTTP error code {}.".format(resp.status_code))
-            resp_json = resp.json()
+            if not is_graphql_query and not "__a" in params and host == "www.instagram.com":
+                match = re.search(r'window\._sharedData = (.*);</script>', resp.text)
+                if match is None:
+                    raise ConnectionException("Could not find \"window._sharedData\" in html response.")
+                return json.loads(match.group(1))
+            else:
+                resp_json = resp.json()
             if 'status' in resp_json and resp_json['status'] != "ok":
                 if 'message' in resp_json:
                     raise ConnectionException("Returned \"{}\" status, message \"{}\".".format(resp_json['status'],
@@ -625,7 +652,7 @@ class Instaloader:
                     raise ConnectionException("Returned \"{}\" status.".format(resp_json['status']))
             return resp_json
         except (ConnectionException, json.decoder.JSONDecodeError, requests.exceptions.RequestException) as err:
-            error_string = "JSON Query to {}: {}".format(url, err)
+            error_string = "JSON Query to {}: {}".format(path, err)
             if _attempt == self.max_connection_attempts:
                 raise ConnectionException(error_string)
             self.error(error_string + " [retrying; skip with ^C]", repeat_at_end=False)
@@ -636,12 +663,12 @@ class Instaloader:
                 if isinstance(err, TooManyRequests):
                     print(textwrap.fill(text_for_429), file=sys.stderr)
                     if is_graphql_query:
-                        waittime = graphql_query_waittime(query_id=params['query_id'], untracked_queries=True)
+                        waittime = graphql_query_waittime(untracked_queries=True)
                         if waittime > 0:
                             self._log('The request will be retried in {} seconds.'.format(waittime))
                             time.sleep(waittime)
                 self._sleep()
-                return self.get_json(url, params, sess, _attempt + 1)
+                return self.get_json(path=path, params=params, host=host, session=sess, _attempt=_attempt + 1)
             except KeyboardInterrupt:
                 self.error("[skipped by user]", repeat_at_end=False)
                 raise ConnectionException(error_string)
@@ -676,7 +703,7 @@ class Instaloader:
         return session
 
     def graphql_query(self, query_identifier: Union[int, str], variables: Dict[str, Any],
-                      referer: Optional[str] = None) -> Dict[str, Any]:
+                      referer: Optional[str] = None, rhx_gis: Optional[str] = None) -> Dict[str, Any]:
         """
         Do a GraphQL Query.
 
@@ -694,9 +721,18 @@ class Instaloader:
         tmpsession.headers['accept'] = '*/*'
         if referer is not None:
             tmpsession.headers['referer'] = urllib.parse.quote(referer)
+
+        variables_json = json.dumps(variables, separators=(',', ':'))
+
+        if rhx_gis:
+            values = "{}:{}".format(rhx_gis, variables_json)
+            x_instagram_gis = hashlib.md5(values.encode()).hexdigest()
+            tmpsession.cookies.set('ig_pr', '2')
+            tmpsession.headers['x-instagram-gis'] = x_instagram_gis
+
         resp_json = self.get_json('graphql/query',
                                   params={'query_id' if isinstance(query_identifier, int) else 'query_hash': query_identifier,
-                                          'variables': json.dumps(variables, separators=(',', ':'))},
+                                          'variables': variables_json},
                                   session=tmpsession)
         if 'status' not in resp_json:
             self.error("GraphQL response did not contain a \"status\" field.")
@@ -721,20 +757,21 @@ class Instaloader:
     def get_id_by_username(self, profile: str) -> int:
         """Each Instagram profile has its own unique ID which stays unmodified even if a user changes
         his/her username. To get said ID, given the profile's name, you may call this function."""
-        return int(self.get_profile_metadata(profile)['user']['id'])
+        return int(self.get_profile_metadata(profile)[0]['user']['id'])
 
     def graphql_node_list(self, query_identifier: Union[int, str], query_variables: Dict[str, Any],
                           query_referer: Optional[str],
-                          edge_extractor: Callable[[Dict[str, Any]], Dict[str, Any]]) -> Iterator[Dict[str, Any]]:
+                          edge_extractor: Callable[[Dict[str, Any]], Dict[str, Any]],
+                          rhx_gis: Optional[str] = None) -> Iterator[Dict[str, Any]]:
         """Retrieve a list of GraphQL nodes."""
         query_variables['first'] = Instaloader.GRAPHQL_PAGE_LENGTH
-        data = self.graphql_query(query_identifier, query_variables, query_referer)
+        data = self.graphql_query(query_identifier, query_variables, query_referer, rhx_gis)
         while True:
             edge_struct = edge_extractor(data)
             yield from [edge['node'] for edge in edge_struct['edges']]
             if edge_struct['page_info']['has_next_page']:
                 query_variables['after'] = edge_struct['page_info']['end_cursor']
-                data = self.graphql_query(query_identifier, query_variables, query_referer)
+                data = self.graphql_query(query_identifier, query_variables, query_referer, rhx_gis)
             else:
                 break
 
@@ -746,6 +783,7 @@ class Instaloader:
 
         :param profile: Name of profile to lookup followers.
         """
+        self._log('(Warning: Fetching followers uses many queries, which are rate limited.)')
         yield from self.graphql_node_list(17851374694183129, {'id': str(self.get_id_by_username(profile))},
                                           'https://www.instagram.com/' + profile + '/',
                                           lambda d: d['data']['user']['edge_followed_by'])
@@ -758,6 +796,7 @@ class Instaloader:
 
         :param profile: Name of profile to lookup followers.
         """
+        self._log('(Warning: Fetching followees uses many queries, which are rate limited.)')
         yield from self.graphql_node_list(17874545323001329, {'id': str(self.get_id_by_username(profile))},
                                           'https://www.instagram.com/' + profile + '/',
                                           lambda d: d['data']['user']['edge_follow'])
@@ -884,8 +923,14 @@ class Instaloader:
     def download_profilepic(self, name: str, profile_metadata: Dict[str, Any]) -> None:
         """Downloads and saves profile pic."""
 
-        url = profile_metadata["user"]["profile_pic_url_hd"] if "profile_pic_url_hd" in profile_metadata["user"] \
-            else profile_metadata["user"]["profile_pic_url"]
+        try:
+            data = self.get_json(path='api/v1/users/{0}/info/'.format(profile_metadata["user"]["id"]), params={},
+                                 host='i.instagram.com', session=self._get_anonymous_session())
+            url = data["user"]["hd_profile_pic_url_info"]["url"]
+        except (InstaloaderException, KeyError) as err:
+            self.error('{} Unable to fetch high quality profile pic.'.format(err))
+            url = profile_metadata["user"]["profile_pic_url_hd"] if "profile_pic_url_hd" in profile_metadata["user"] \
+                else profile_metadata["user"]["profile_pic_url"]
 
         def _epoch_to_string(epoch: datetime) -> str:
             return epoch.strftime('%Y-%m-%d_%H-%M-%S')
@@ -903,13 +948,7 @@ class Instaloader:
         if os.path.isfile(filename):
             self._log(filename + ' already exists')
             return None
-        url_best = re.sub(r'/s([1-9][0-9]{2})x\1/', '/s2048x2048/', url)
-        url_best = re.sub(r'/vp/[a-f0-9]{32}/[A-F0-9]{8}/', '/', url_best)      # remove signature
-        try:
-            self._get_and_write_raw(url_best, filename)
-        except (QueryReturnedForbiddenException, QueryReturnedNotFoundException) as err:
-            self.error('{} Retrying with lower quality version.'.format(err))
-            self._get_and_write_raw(url, filename)
+        self._get_and_write_raw(url, filename)
         os.utime(filename, (datetime.now().timestamp(), date_object.timestamp()))
         self._log('') # log output of _get_and_write_raw() does not produce \n
 
@@ -1124,7 +1163,7 @@ class Instaloader:
             self._log("Retrieving stories from profile {}.".format(name))
             totalcount = len(user_stories["items"])
             count = 1
-            for item in user_stories["items"]:
+            for item in reversed(user_stories["items"]):
                 self._log("[%3i/%3i] " % (count, totalcount), end="", flush=True)
                 count += 1
                 with self._error_catcher('Download story from user {}'.format(name)):
@@ -1238,7 +1277,7 @@ class Instaloader:
 
         if not self.is_logged_in:
             return
-        data = self.get_profile_metadata(self.username)
+        data, full_metadata = self.get_profile_metadata(self.username)
         user_id = data["user"]["id"]
 
         while True:
@@ -1258,7 +1297,8 @@ class Instaloader:
                 break
             data = self.graphql_query("f883d95537fbcd400f466f63d42bd8a1",
                                       {'id': user_id, 'first': Instaloader.GRAPHQL_PAGE_LENGTH,
-                                       'after': saved_media["page_info"]["end_cursor"]})['data']
+                                       'after': saved_media["page_info"]["end_cursor"]},
+                                      rhx_gis=full_metadata['rhx_gis'])['data']
 
     def download_saved_posts(self, max_count: int = None, fast_update: bool = False,
                              filter_func: Optional[Callable[[Post], bool]] = None) -> None:
@@ -1285,10 +1325,19 @@ class Instaloader:
 
     def get_hashtag_posts(self, hashtag: str) -> Iterator[Post]:
         """Get Posts associated with a #hashtag."""
-        yield from (Post(self, node) for node in
-                    self.graphql_node_list(17875800862117404, {'tag_name': hashtag},
-                                           'https://www.instagram.com/explore/tags/{0}/'.format(hashtag),
-                                           lambda d: d['data']['hashtag']['edge_hashtag_to_media']))
+        has_next_page = True
+        end_cursor = None
+        while has_next_page:
+            if end_cursor:
+                params = {'__a': 1, 'max_id': end_cursor}
+            else:
+                params = {'__a': 1}
+            hashtag_data = self.get_json('explore/tags/{0}/'.format(hashtag),
+                                         params)['graphql']['hashtag']['edge_hashtag_to_media']
+            yield from (Post(self, edge['node']) for edge in
+                        hashtag_data['edges'])
+            has_next_page = hashtag_data['page_info']['has_next_page']
+            end_cursor = hashtag_data['page_info']['end_cursor']
 
     def download_hashtag(self, hashtag: str,
                          max_count: Optional[int] = None,
@@ -1347,6 +1396,9 @@ class Instaloader:
                     self._log("Profile {0} does not match the stored unique ID {1}.".format(profile, profile_id))
                 else:
                     self._log("Trying to find profile {0} using its unique ID {1}.".format(profile, profile_id))
+                if not self.is_logged_in:
+                    self.error("Profile {} changed its name. If you use --login=USERNAME, I can find out the new name.")
+                    raise LoginRequiredException("Login required to obtain username from userid")
                 newname = self.get_username_by_id(profile_id)
                 self._log("Profile {0} has changed its name to {1}.".format(profile, newname))
                 if ((format_string_contains_key(self.dirname_pattern, 'profile') or
@@ -1372,15 +1424,15 @@ class Instaloader:
             return profile, profile_id
         raise ProfileNotExistsException("Profile {0} does not exist.".format(profile))
 
-    def get_profile_metadata(self, profile_name: str) -> Dict[str, Any]:
+    def get_profile_metadata(self, profile_name: str) -> Tuple[Dict[str, Any], Dict[str, Any]]:
         """Retrieves a profile's metadata, for use with e.g. :meth:`get_profile_posts` and :meth:`check_profile_id`."""
         try:
-            metadata = self.get_json('{}/'.format(profile_name), params={'__a': 1})
-            return metadata['graphql'] if 'graphql' in metadata else metadata
+            metadata = self.get_json('{}/'.format(profile_name), params={})
+            return metadata['entry_data']['ProfilePage'][0]['graphql'], metadata
         except QueryReturnedNotFoundException:
             raise ProfileNotExistsException('Profile {} does not exist.'.format(profile_name))
 
-    def get_profile_posts(self, profile_metadata: Dict[str, Any]) -> Iterator[Post]:
+    def get_profile_posts(self, profile_metadata: Dict[str, Any], rhx_gis: str) -> Iterator[Post]:
         """Retrieve all posts from a profile."""
         profile_name = profile_metadata['user']['username']
         profile_id = int(profile_metadata['user']['id'])
@@ -1401,7 +1453,7 @@ class Instaloader:
             data = self.graphql_query(17888483320059182, {'id': profile_metadata['user']['id'],
                                                           'first': Instaloader.GRAPHQL_PAGE_LENGTH,
                                                           'after': end_cursor},
-                                      'https://www.instagram.com/{0}/'.format(profile_name))
+                                      referer='https://www.instagram.com/{0}/'.format(profile_name), rhx_gis=rhx_gis)
             media = data['data']['user']['edge_owner_to_timeline_media']
             yield from (Post(self, edge['node'], profile=profile_name, profile_id=profile_id)
                         for edge in media['edges'])
@@ -1421,14 +1473,14 @@ class Instaloader:
         with suppress(ProfileNotExistsException):
             # ProfileNotExistsException is raised again later in check_profile_id() when we search the profile, so we
             # must suppress it here.
-            profile_metadata = self.get_profile_metadata(name)
+            profile_metadata, full_metadata = self.get_profile_metadata(name)
 
         # check if profile does exist or name has changed since last download
         # and update name and json data if necessary
         name_updated, profile_id = self.check_profile_id(name, profile_metadata)
         if name_updated != name:
             name = name_updated
-            profile_metadata = self.get_profile_metadata(name)
+            profile_metadata, full_metadata = self.get_profile_metadata(name)
 
         # Download profile picture
         if profile_pic or profile_pic_only:
@@ -1463,7 +1515,7 @@ class Instaloader:
         else:
             totalcount = profile_metadata["user"]["edge_owner_to_timeline_media"]["count"]
         count = 1
-        for post in self.get_profile_posts(profile_metadata):
+        for post in self.get_profile_posts(profile_metadata, rhx_gis=full_metadata['rhx_gis']):
             self._log("[%3i/%3i] " % (count, totalcount), end="", flush=True)
             count += 1
             if filter_func is not None and not filter_func(post):
@@ -1581,6 +1633,14 @@ class Instaloader:
         # Save session if it is useful
         if username is not None:
             self.save_session_to_file(sessionfile)
+        # User might be confused if Instaloader does nothing
+        if not profilelist:
+            if self.is_logged_in:
+                # Instaloader did at least save a session file
+                self._log("No targets were specified, thus nothing has been downloaded.")
+            else:
+                # Instloader did not do anything
+                self._log("usage:"+USAGE_STRING)
         if self.error_log:
             print("\nErrors occured:", file=sys.stderr)
             for err in self.error_log:
@@ -1588,7 +1648,7 @@ class Instaloader:
 
 
 def main():
-    parser = ArgumentParser(description=__doc__, add_help=False,
+    parser = ArgumentParser(description=__doc__, add_help=False, usage=USAGE_STRING,
                             epilog="Report issues at https://github.com/instaloader/instaloader/issues. "
                                    "The complete documentation can be found at "
                                    "https://instaloader.github.io/.")