Release of version 3.3.6

Closes #94.

LICENSE

@@ -1,6 +1,6 @@
 The MIT License (MIT)
 
-Copyright (c) 2016 Alexander Graf and André Koch-Kramer.
+Copyright (c) 2016-2018 Alexander Graf and André Koch-Kramer.
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

README.rst

@@ -1,5 +1,4 @@
-Instaloader
+.. image:: https://raw.githubusercontent.com/instaloader/instaloader/master/docs/logo_heading.png
-===========
 
 ::
 

docs/Makefile

@@ -50,7 +50,7 @@ clean:
 
 .PHONY: html
 html:
-	$(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html
+	LC_ALL=C $(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html
 	@echo
 	@echo "Build finished. The HTML pages are in $(BUILDDIR)/html."
 
@@ -223,3 +223,9 @@ dummy:
 	$(SPHINXBUILD) -b dummy $(ALLSPHINXOPTS) $(BUILDDIR)/dummy
 	@echo
 	@echo "Build finished. Dummy builder generates no files."
+
+favicon.ico: logo.svg
+	convert -density 384 -background transparent -define icon:auto-resize -colors 256 logo.svg favicon.ico
+
+logo.png: logo.svg
+	inkscape -z -e logo.png -w 100 -h 100 logo.svg

docs/_static/style.css

@@ -0,0 +1,12 @@
+code {
+    color: #222;
+    background-color: #f5f5f5;
+}
+
+dd {
+    padding-left: 8px;
+}
+
+dl.option dd {
+    padding-left: 12px;
+}

docs/_templates/caption.html

@@ -1,2 +0,0 @@
-<h2><a href="index.html">Instaloader</a></h2>
-<p>Download pictures (or videos) along with their captions and other metadata from Instagram.</p>

docs/_templates/globaltoc.html

@@ -0,0 +1,14 @@
+<li class="dropdown globaltoc-container">
+  <a role="button"
+     id="dLabelGlobalToc"
+     data-toggle="dropdown"
+     data-target="#"
+     href="{{ pathto(master_doc) }}">{{ theme_navbar_site_name }} <b class="caret"></b></a>
+  <ul class="dropdown-menu globaltoc"
+      role="menu"
+      aria-labelledby="dLabelGlobalToc">
+    {{ toctree(maxdepth=theme_globaltoc_depth|toint, collapse=False, includehidden=theme_globaltoc_includehidden|tobool) }}
+    <li role="separator" class="divider"></li>
+    <li class="toctree-l1"><a class="reference internal" href="genindex.html">Index</a></li>
+  </ul>
+</li>

docs/_templates/layout.html

@@ -1,5 +1,18 @@
 {% extends "!layout.html" %}
+{% block htmltitle %}
+  {% if pagename == "index" %}
+  <title>{{ title|striptags|e }}</title>
+  {% else %}
+  <title>{{ title|striptags|e }}{{ titlesuffix }}</title>
+  {% endif %}
+{% endblock %}
 {% block extrahead %}
-<meta name="google-site-verification" content="FNUsDdkOJM9obm9QEdrNBVd9AtAzHzA-aaSMCUrS6C8" />
+{{ super() }}
-<script async defer src="https://buttons.github.io/buttons.js"></script>
+  <meta name="google-site-verification" content="FNUsDdkOJM9obm9QEdrNBVd9AtAzHzA-aaSMCUrS6C8" />
+  {% if pagename == "index" %}
+  <link rel="canonical" href="https://instaloader.github.io/" />
+  {% else %}
+  <link rel="canonical" href="https://instaloader.github.io/{{ pagename }}.html" />
+  {% endif %}
+  <script async defer src="https://buttons.github.io/buttons.js"></script>
 {% endblock %}

docs/_templates/links.html

@@ -1,23 +0,0 @@
-{% if next %}
-<h3>Next</h3>
-<ul>
-  <li class="toctree-l1"><a class="reference internal" href="{{ next.link|e }}">{{ next.title }}</a></li>
-</ul>
-{% endif %}
-<h3>Current Release</h3>
-<ul>
-  <li class="toctree-l1"><a href="https://github.com/instaloader/instaloader/releases/tag/v{{current_release}}">Version <b>{{current_release}}</b></a></li>
-  <li>Released on <b>{{current_release_date}}</b>.</li>
-</ul>
-<h3>Links</h3>
-<ul>
-  <li><a href="https://github.com/instaloader/instaloader/issues">Issue Tracker</a></li>
-  <li><a href="https://github.com/instaloader/instaloader/releases">Version History</a></li>
-  <li><a href="https://pypi.python.org/pypi/instaloader">Instaloader on PyPI</a></li>
-  <li><a href="https://github.com/instaloader/instaloader">Instaloader on GitHub</a></li>
-</ul>
-<ul>
-  <li>
-    <a class="github-button" href="https://github.com/instaloader/Instaloader" data-icon="octicon-star" data-size="large" data-show-count="true" aria-label="Star instaloader/Instaloader on GitHub">Star</a>
-  </li>
-</ul>

docs/_templates/navbar.html

@@ -0,0 +1,58 @@
+  <div id="navbar" class="{{ theme_navbar_class }} navbar-default {% if theme_navbar_fixed_top|tobool -%} navbar-fixed-top{%- endif -%}">
+    <div class="container">
+      <div class="navbar-header">
+        <!-- .btn-navbar is used as the toggle for collapsed navbar content -->
+        <button type="button" class="navbar-toggle" data-toggle="collapse" data-target=".nav-collapse">
+          <span class="icon-bar"></span>
+          <span class="icon-bar"></span>
+          <span class="icon-bar"></span>
+        </button>
+        <a class="navbar-brand" href="{{ pathto(master_doc) }}">
+          {%- block sidebarlogo %}
+            {%- if logo %}<span><img src="{{ pathto('_static/' + logo, 1) }}"></span>{%- endif %}
+          {%- endblock %}
+          {% if theme_navbar_title -%}{{ theme_navbar_title|e }}{%- else -%}{{ project|e }}{%- endif -%}
+        </a>
+        <span class="navbar-text navbar-version pull-left">
+          <a href="https://github.com/instaloader/instaloader/releases/tag/v{{current_release}}" title="Released on {{current_release_date}}">v<b>{{current_release}}</b></a>
+        </span>
+      </div>
+
+        <div class="collapse navbar-collapse nav-collapse">
+          <ul class="nav navbar-nav">
+            {% if theme_navbar_links %}
+              {%- for link in theme_navbar_links %}
+                <li><a href="{{ pathto(*link[1:]) }}">{{ link[0] }}</a></li>
+              {%- endfor %}
+            {% endif %}
+            {% block navbartoc %}
+              {% include "globaltoc.html" %}
+              {% if theme_navbar_pagenav and pagename not in ("index", "genindex", "installation") %}
+                {% include "navbartoc.html" %}
+              {% endif %}
+            {% endblock %}
+            {% if theme_navbar_sidebarrel %}
+              {% block sidebarrel %}
+                {% include "relations.html" %}
+              {% endblock %}
+            {% endif %}
+            </ul>
+            <ul class="nav navbar-nav navbar-right">
+              <li class="dropdown">
+                <a role="button" id="dLabelUsefulLinks" data-toggle="dropdown" data-target="#" href="#">
+                  Useful Links
+                  <b class="caret"></b>
+                </a>
+                <ul class="dropdown-menu" role="menu" aria-labelledby="dLabelUsefulLinks">
+                  <li><a href="https://github.com/instaloader/instaloader/issues">Issue Tracker</a></li>
+                  <li><a href="https://github.com/instaloader/instaloader/releases">Version History</a></li>
+                  <li><a href="https://pypi.org/project/instaloader/">Instaloader on PyPI</a></li>
+                  <li><a href="https://github.com/instaloader/instaloader">Instaloader on GitHub</a></li>
+                  <li role="separator" class="divider"></li>
+                  <li style="padding-left: 20px;"><a class="github-button" href="https://github.com/instaloader/instaloader" data-icon="octicon-star" data-size="large" data-show-count="true" aria-label="Star Instaloader on GitHub">Star</a></li>
+                </ul>
+              </li>
+            </ul>
+        </div>
+    </div>
+  </div>

docs/as-module.rst

@@ -1,7 +1,7 @@
 .. meta::
    :description:
-      Instaloader can also be used as a powerful and easy-to-use
+      Documentation of Instaloader module, a powerful and easy-to-use
-      Python API for Instagram, allowing to download media and metadata.
+      Python library to download Instagram media and metadata.
 
 Python Module :mod:`instaloader`
 --------------------------------
@@ -37,7 +37,8 @@ certain source::
         L.download_post(post, target='#cat')
 
 Besides :func:`Instaloader.get_hashtag_posts`, there is
-:func:`Instaloader.get_feed_posts` and :func:`Instaloader.get_profile_posts`.
+:func:`Instaloader.get_feed_posts`, :func:`Instaloader.get_profile_posts` and
+:func:`Instaloader.get_saved_posts`.
 Also, :class:`Post` instances can be created with :func:`Post.from_shortcode`
 and :func:`Post.from_mediaid`.
 

docs/basic-usage.rst

@@ -1,3 +1,10 @@
+.. meta::
+   :description:
+      How to download pictures from Instagram. Description of basic
+      usage of Instaloader, free tool to download photos from public
+      and private profiles, hashtags, stories, feeds, saved media, and
+      their metadata, comments and captions.
+
 Download Pictures from Instagram
 ---------------------------------
 

docs/conf.py

@@ -20,6 +20,9 @@
 import os
 import subprocess
 import sys
+
+import sphinx_bootstrap_theme
+
 sys.path.insert(0, os.path.abspath('..'))
 
 # -- General configuration ------------------------------------------------
@@ -133,16 +136,20 @@ todo_include_todos = False
 # The theme to use for HTML and HTML Help pages.  See the documentation for
 # a list of builtin themes.
 #
-html_theme = 'alabaster'
+html_theme = 'bootstrap'
+html_theme_path = sphinx_bootstrap_theme.get_html_theme_path()
 
 # Theme options are theme-specific and customize the look and feel of a theme
 # further.  For a list of options available for each theme, see the
 # documentation.
 #
 html_theme_options = {
-    'show_powered_by': False,
+    'navbar_site_name': 'Site Contents',
-    'sidebar_width': '290px',
+    'navbar_pagenav_name': 'Page Contents',
-    'page_width': '935px' }
+    'navbar_pagenav': True,
+    'navbar_sidebarrel': True,
+    'nosidebar': True,
+     }
 
 # Add any paths that contain custom themes here, relative to this directory.
 # html_theme_path = []
@@ -159,13 +166,13 @@ html_theme_options = {
 # The name of an image file (relative to this directory) to place at the top
 # of the sidebar.
 #
-# html_logo = None
+html_logo = "logo.png"
 
 # The name of an image file (relative to this directory) to use as a favicon of
 # the docs.  This file should be a Windows icon file (.ico) being 16x16 or 32x32
 # pixels large.
 #
-# html_favicon = None
+html_favicon = "favicon.ico"
 
 # Add any paths that contain custom static files (such as style sheets) here,
 # relative to this directory. They are copied after the builtin static files,
@@ -191,7 +198,8 @@ html_static_path = ['_static']
 
 # Custom sidebar templates, maps document names to template names.
 #
-html_sidebars = {'**': ["caption.html", "globaltoc.html", "relations.html", "links.html"] }
+#html_sidebars = {'**': ["relations.html", "links.html"] }
+html_sidebars = {}
 
 # Additional templates that should be rendered to pages, maps page names to
 # template names.
@@ -204,7 +212,7 @@ html_domain_indices = False
 
 # If false, no index is generated.
 #
-html_use_index = False
+html_use_index = True
 
 # If true, the index is split into individual pages for each letter.
 #
@@ -359,3 +367,6 @@ current_release = subprocess.check_output(["git", "describe", "--abbrev=0"]).dec
 current_release_date = subprocess.check_output(["git", "log", "-1", "--tags", "--format=%ad", "--date=format:%e %b %Y"]).decode("ascii")[:-1]
 
 html_context = {'current_release': current_release, 'current_release_date': current_release_date}
+
+def setup(app):
+    app.add_stylesheet("style.css")

docs/index.rst

@@ -1,8 +1,11 @@
 .. meta::
    :description:
-      Command line tool to download pictures (and videos) from Instagram.
+      Free command line tool to download photos from Instagram.
-      Instaloader downloads public and private profiles, hashtags, user stories,
+      Scrapes public and private profiles, hashtags, stories, feeds,
-      feeds, saved media, comments, geotags, captions and other metadata of each post.
+      saved media, and their metadata, comments and captions.
+      Written in Python.
+
+.. title:: Instaloader — Download Instagram Photos and Metadata
 
 Instaloader
 ===========
@@ -42,8 +45,8 @@ With `Python <https://www.python.org/>`__ installed, do::
                 :stories | :feed | :saved
 
 
-Table of Contents
+Instaloader Documentation
------------------
+-------------------------
 
 .. toctree::
    :maxdepth: 2
@@ -58,7 +61,7 @@ Useful Links
 ------------
 
 - `Git Repository (on GitHub) <https://github.com/instaloader/instaloader>`__
-- `PyPI Project Page <https://pypi.python.org/pypi/instaloader>`__
+- `PyPI Project Page <https://pypi.org/project/instaloader/>`__
 - `Issue Tracker / Bug Tracker <https://github.com/instaloader/instaloader/issues>`__
 - `Version History <https://github.com/instaloader/instaloader/releases>`__
 

docs/installation.rst

@@ -28,7 +28,7 @@ will be installed automatically, if it is not already installed.
 
 - If you do not want to use pip, even though it is highly recommended,
   and prefer to **install Instaloader manually**,
-  `Download the Source <https://github.com/instaloader/instaloader/releases>`__,
+  `Download the Source <https://github.com/instaloader/instaloader/releases/latest>`__,
   extract the Zip or Tarball and execute ``instaloader.py`` from there.
 
 - On **Arch Linux**, you may install Instaloader using the
@@ -36,3 +36,6 @@ will be installed automatically, if it is not already installed.
 
 - On **Gentoo Linux**, you may install Instaloader using the
   `Instaloader Ebuild <https://bugs.gentoo.org/642100>`__.
+
+- On **Windows 10**, you may download the standalone executable from the
+  `current release page <https://github.com/instaloader/instaloader/releases/latest>`__.

docs/requirements.txt

@@ -1,3 +1,4 @@
 requests
 sphinx
 sphinx-autodoc-typehints
+sphinx-bootstrap-theme

instaloader.py

@@ -3,9 +3,11 @@
 """Download pictures (or videos) along with their captions and other metadata from Instagram."""
 import ast
 import getpass
+import hashlib
 import json
 import os
 import pickle
+import platform
 import random
 import re
 import shutil
@@ -29,8 +31,14 @@ import requests.utils
 import urllib3
 
 
-__version__ = '3.3'
+__version__ = '3.3.6'
 
+# NOTE: duplicated in README.rst and docs/index.rst
+USAGE_STRING = """
+{0} [--comments] [--geotags] [--stories]
+{2:{1}} [--login YOUR-USERNAME] [--fast-update]
+{2:{1}} profile | "#hashtag" | :stories | :feed | :saved
+{0} --help""".format(sys.argv[0], len(sys.argv[0]), '')
 
 try:
     # pylint:disable=wrong-import-position
@@ -127,10 +135,10 @@ def mediaid_to_shortcode(mediaid: int) -> str:
     return b64encode(mediaid.to_bytes(9, 'big'), b'-_').decode().replace('A', ' ').lstrip().replace(' ','A')
 
 
-def format_string_contains_key(format_string: str, key: str) -> bool:
+def format_string_contains_key(format_string: '_PathPattern', key: str) -> bool:
     # pylint:disable=unused-variable
     for literal_text, field_name, format_spec, conversion in string.Formatter().parse(format_string):
-        if field_name == key or field_name.startswith(key + '.'):
+        if field_name and (field_name == key or field_name.startswith(key + '.')):
             return True
     return False
 
@@ -169,7 +177,8 @@ class Post:
     """
     Structure containing information about an Instagram post.
 
-    Created by Instaloader methods :meth:`.get_profile_posts`, :meth:`.get_hashtag_posts`, :meth:`.get_feed_posts`.
+    Created by Instaloader methods :meth:`.get_profile_posts`, :meth:`.get_hashtag_posts`, :meth:`.get_feed_posts` and
+    :meth:`.get_saved_posts`.
     Posts are linked to an :class:`Instaloader` instance which is used for error logging and obtaining of additional
     metadata, if required. This class unifies access to the properties associated with a post. It implements == and is
     hashable.
@@ -193,6 +202,7 @@ class Post:
         self._profile = profile
         self._profile_id = profile_id
         self._full_metadata_dict = None
+        self._rhx_gis = None
 
     @classmethod
     def from_shortcode(cls, instaloader: 'Instaloader', shortcode: str):
@@ -231,11 +241,9 @@ class Post:
     @property
     def _full_metadata(self) -> Dict[str, Any]:
         if not self._full_metadata_dict:
-            pic_json = self._instaloader.get_json("p/{0}/".format(self.shortcode), params={'__a': 1})
+            pic_json = self._instaloader.get_json("p/{0}/".format(self.shortcode), params={})
-            if "graphql" in pic_json:
+            self._full_metadata_dict = pic_json['entry_data']['PostPage'][0]['graphql']['shortcode_media']
-                self._full_metadata_dict = pic_json["graphql"]["shortcode_media"]
+            self._rhx_gis = pic_json['rhx_gis']
-            else:
-                self._full_metadata_dict = pic_json["media"]
         return self._full_metadata_dict
 
     def _field(self, *keys) -> Any:
@@ -376,9 +384,11 @@ class Post:
         if self.comments == len(comment_edges):
             # If the Post's metadata already contains all comments, don't do GraphQL requests to obtain them
             yield from (comment['node'] for comment in comment_edges)
+            return
         yield from self._instaloader.graphql_node_list(17852405266163336, {'shortcode': self.shortcode},
                                                        'https://www.instagram.com/p/' + self.shortcode + '/',
-                                                       lambda d: d['data']['shortcode_media']['edge_media_to_comment'])
+                                                       lambda d: d['data']['shortcode_media']['edge_media_to_comment'],
+                                                       rhx_gis=self._rhx_gis)
 
     def get_likes(self) -> Iterator[Dict[str, Any]]:
         """Iterate over all likes of the post.
@@ -393,9 +403,11 @@ class Post:
         if self.likes == len(likes_edges):
             # If the Post's metadata already contains all likes, don't do GraphQL requests to obtain them
             yield from (like['node'] for like in likes_edges)
+            return
         yield from self._instaloader.graphql_node_list("1cb6ec562846122743b61e492c85999f", {'shortcode': self.shortcode},
                                                        'https://www.instagram.com/p/' + self.shortcode + '/',
-                                                       lambda d: d['data']['shortcode_media']['edge_liked_by'])
+                                                       lambda d: d['data']['shortcode_media']['edge_liked_by'],
+                                                       rhx_gis=self._rhx_gis)
 
     def get_location(self) -> Optional[Dict[str, str]]:
         """If the Post has a location, returns a dictionary with fields 'lat' and 'lng'."""
@@ -403,7 +415,7 @@ class Post:
         if loc_dict is not None:
             location_json = self._instaloader.get_json("explore/locations/{0}/".format(loc_dict["id"]),
                                                        params={'__a': 1})
-            return location_json["location"]
+            return location_json["location"] if "location" in location_json else location_json['graphql']['location']
 
     @staticmethod
     def json_encoder(obj) -> Dict[str, Any]:
@@ -439,6 +451,14 @@ class Tristate(Enum):
     always = 2
 
 
+class _PathPattern(str):
+    """Class overriding :meth:`str.format` for character substitution in paths for Windows, see issue #84."""
+
+    def format(self, *args: Any, **kwargs: Any) -> str:
+        ret = super().format(*args, **kwargs)
+        return ret.replace(':', '\ua789') if platform.system() == 'Windows' else ret
+
+
 class Instaloader:
     GRAPHQL_PAGE_LENGTH = 200
 
@@ -461,18 +481,19 @@ class Instaloader:
         self.username = None
         self.sleep = sleep
         self.quiet = quiet
-        self.dirname_pattern = dirname_pattern if dirname_pattern is not None else '{target}'
+        self.dirname_pattern = _PathPattern(dirname_pattern if dirname_pattern is not None else '{target}')
         if filename_pattern is not None:
             filename_pattern = re.sub(r"({(?:post\.)?date)([:}])", r"\1_utc\2", filename_pattern)
             self.filename_pattern_old = filename_pattern.replace('{date_utc}', '{date_utc:%Y-%m-%d_%H-%M-%S}')
-            self.filename_pattern_old = re.sub(r"(?i)({(?:post\.)?date_utc:[^}]*?)_UTC",
+            self.filename_pattern_old = _PathPattern(re.sub(r"(?i)({(?:post\.)?date_utc:[^}]*?)_UTC",
-                                               r"\1", self.filename_pattern_old)
+                                                            r"\1", self.filename_pattern_old))
             filename_pattern = re.sub(r"(?i)({(date_utc|post\.date_utc):(?![^}]*UTC[^}]*).*?)}",
                                       r"\1_UTC}", filename_pattern)
-            self.filename_pattern = filename_pattern.replace('{date_utc}', '{date_utc:%Y-%m-%d_%H-%M-%S_UTC}')
+            self.filename_pattern = _PathPattern(filename_pattern.replace('{date_utc}',
+                                                                          '{date_utc:%Y-%m-%d_%H-%M-%S_UTC}'))
         else:
-            self.filename_pattern = '{date_utc:%Y-%m-%d_%H-%M-%S_UTC}'
+            self.filename_pattern = _PathPattern('{date_utc:%Y-%m-%d_%H-%M-%S_UTC}')
-            self.filename_pattern_old = '{date_utc:%Y-%m-%d_%H-%M-%S}'
+            self.filename_pattern_old = _PathPattern('{date_utc:%Y-%m-%d_%H-%M-%S}')
         self.download_videos = download_videos
         self.download_video_thumbnails = download_video_thumbnails
         self.download_geotags = download_geotags
@@ -571,18 +592,19 @@ class Instaloader:
                 self.error("[skipped by user]", repeat_at_end=False)
                 raise ConnectionException(error_string)
 
-    def get_json(self, url: str, params: Dict[str, Any],
+    def get_json(self, path: str, params: Dict[str, Any], host: str = 'www.instagram.com',
                  session: Optional[requests.Session] = None, _attempt = 1) -> Dict[str, Any]:
         """JSON request to Instagram.
 
-        :param url: URL, relative to www.instagram.com/
+        :param path: URL, relative to the given domain which defaults to www.instagram.com/
         :param params: GET parameters
+        :param host: Domain part of the URL from where to download the requested JSON; defaults to www.instagram.com
         :param session: Session to use, or None to use self.session
         :return: Decoded response dictionary
         :raises QueryReturnedNotFoundException: When the server responds with a 404.
         :raises ConnectionException: When query repeatedly failed.
         """
-        def graphql_query_waittime(query_id: int, untracked_queries: bool = False) -> int:
+        def graphql_query_waittime(query_id: Union[int, str], untracked_queries: bool = False) -> int:
             sliding_window = 660
             timestamps = self.previous_queries.get(query_id)
             if not timestamps:
@@ -593,9 +615,9 @@ class Instaloader:
             if len(timestamps) < 100 and not untracked_queries:
                 return 0
             return round(min(timestamps) + sliding_window - current_time) + 6
-        is_graphql_query = 'query_id' in params and 'graphql/query' in url
+        is_graphql_query = 'graphql/query' in path
         if is_graphql_query:
-            query_id = params['query_id']
+            query_id = params['query_id'] if 'query_id' in params else params['query_hash']
             waittime = graphql_query_waittime(query_id)
             if waittime > 0:
                 self._log('\nToo many queries in the last time. Need to wait {} seconds.'.format(waittime))
@@ -608,14 +630,20 @@ class Instaloader:
         sess = session if session else self.session
         try:
             self._sleep()
-            resp = sess.get('https://www.instagram.com/' + url, params=params)
+            resp = sess.get('https://{0}/{1}'.format(host, path), params=params)
             if resp.status_code == 404:
                 raise QueryReturnedNotFoundException("404")
             if resp.status_code == 429:
                 raise TooManyRequests("429 - Too Many Requests")
             if resp.status_code != 200:
                 raise ConnectionException("HTTP error code {}.".format(resp.status_code))
-            resp_json = resp.json()
+            if not is_graphql_query and not "__a" in params and host == "www.instagram.com":
+                match = re.search(r'window\._sharedData = (.*);</script>', resp.text)
+                if match is None:
+                    raise ConnectionException("Could not find \"window._sharedData\" in html response.")
+                return json.loads(match.group(1))
+            else:
+                resp_json = resp.json()
             if 'status' in resp_json and resp_json['status'] != "ok":
                 if 'message' in resp_json:
                     raise ConnectionException("Returned \"{}\" status, message \"{}\".".format(resp_json['status'],
@@ -624,7 +652,7 @@ class Instaloader:
                     raise ConnectionException("Returned \"{}\" status.".format(resp_json['status']))
             return resp_json
         except (ConnectionException, json.decoder.JSONDecodeError, requests.exceptions.RequestException) as err:
-            error_string = "JSON Query to {}: {}".format(url, err)
+            error_string = "JSON Query to {}: {}".format(path, err)
             if _attempt == self.max_connection_attempts:
                 raise ConnectionException(error_string)
             self.error(error_string + " [retrying; skip with ^C]", repeat_at_end=False)
@@ -640,7 +668,7 @@ class Instaloader:
                             self._log('The request will be retried in {} seconds.'.format(waittime))
                             time.sleep(waittime)
                 self._sleep()
-                return self.get_json(url, params, sess, _attempt + 1)
+                return self.get_json(path=path, params=params, host=host, session=sess, _attempt=_attempt + 1)
             except KeyboardInterrupt:
                 self.error("[skipped by user]", repeat_at_end=False)
                 raise ConnectionException(error_string)
@@ -675,7 +703,7 @@ class Instaloader:
         return session
 
     def graphql_query(self, query_identifier: Union[int, str], variables: Dict[str, Any],
-                      referer: Optional[str] = None) -> Dict[str, Any]:
+                      referer: Optional[str] = None, rhx_gis: Optional[str] = None) -> Dict[str, Any]:
         """
         Do a GraphQL Query.
 
@@ -693,9 +721,18 @@ class Instaloader:
         tmpsession.headers['accept'] = '*/*'
         if referer is not None:
             tmpsession.headers['referer'] = urllib.parse.quote(referer)
+
+        variables_json = json.dumps(variables, separators=(',', ':'))
+
+        if rhx_gis:
+            values = "{}:{}:{}:{}".format(rhx_gis, tmpsession.cookies['csrftoken'], self.user_agent, variables_json)
+            x_instagram_gis = hashlib.md5(values.encode()).hexdigest()
+            tmpsession.cookies.set('ig_pr', '2')
+            tmpsession.headers['x-instagram-gis'] = x_instagram_gis
+
         resp_json = self.get_json('graphql/query',
                                   params={'query_id' if isinstance(query_identifier, int) else 'query_hash': query_identifier,
-                                          'variables': json.dumps(variables, separators=(',', ':'))},
+                                          'variables': variables_json},
                                   session=tmpsession)
         if 'status' not in resp_json:
             self.error("GraphQL response did not contain a \"status\" field.")
@@ -720,20 +757,21 @@ class Instaloader:
     def get_id_by_username(self, profile: str) -> int:
         """Each Instagram profile has its own unique ID which stays unmodified even if a user changes
         his/her username. To get said ID, given the profile's name, you may call this function."""
-        return int(self.get_profile_metadata(profile)['user']['id'])
+        return int(self.get_profile_metadata(profile)[0]['user']['id'])
 
     def graphql_node_list(self, query_identifier: Union[int, str], query_variables: Dict[str, Any],
                           query_referer: Optional[str],
-                          edge_extractor: Callable[[Dict[str, Any]], Dict[str, Any]]) -> Iterator[Dict[str, Any]]:
+                          edge_extractor: Callable[[Dict[str, Any]], Dict[str, Any]],
+                          rhx_gis: Optional[str] = None) -> Iterator[Dict[str, Any]]:
         """Retrieve a list of GraphQL nodes."""
         query_variables['first'] = Instaloader.GRAPHQL_PAGE_LENGTH
-        data = self.graphql_query(query_identifier, query_variables, query_referer)
+        data = self.graphql_query(query_identifier, query_variables, query_referer, rhx_gis)
         while True:
             edge_struct = edge_extractor(data)
             yield from [edge['node'] for edge in edge_struct['edges']]
             if edge_struct['page_info']['has_next_page']:
                 query_variables['after'] = edge_struct['page_info']['end_cursor']
-                data = self.graphql_query(query_identifier, query_variables, query_referer)
+                data = self.graphql_query(query_identifier, query_variables, query_referer, rhx_gis)
             else:
                 break
 
@@ -883,8 +921,14 @@ class Instaloader:
     def download_profilepic(self, name: str, profile_metadata: Dict[str, Any]) -> None:
         """Downloads and saves profile pic."""
 
-        url = profile_metadata["user"]["profile_pic_url_hd"] if "profile_pic_url_hd" in profile_metadata["user"] \
+        try:
-            else profile_metadata["user"]["profile_pic_url"]
+            data = self.get_json(path='api/v1/users/{0}/info/'.format(profile_metadata["user"]["id"]), params={},
+                                 host='i.instagram.com', session=self._get_anonymous_session())
+            url = data["user"]["hd_profile_pic_url_info"]["url"]
+        except (InstaloaderException, KeyError) as err:
+            self.error('{} Unable to fetch high quality profile pic.'.format(err))
+            url = profile_metadata["user"]["profile_pic_url_hd"] if "profile_pic_url_hd" in profile_metadata["user"] \
+                else profile_metadata["user"]["profile_pic_url"]
 
         def _epoch_to_string(epoch: datetime) -> str:
             return epoch.strftime('%Y-%m-%d_%H-%M-%S')
@@ -902,13 +946,7 @@ class Instaloader:
         if os.path.isfile(filename):
             self._log(filename + ' already exists')
             return None
-        url_best = re.sub(r'/s([1-9][0-9]{2})x\1/', '/s2048x2048/', url)
+        self._get_and_write_raw(url, filename)
-        url_best = re.sub(r'/vp/[a-f0-9]{32}/[A-F0-9]{8}/', '/', url_best)      # remove signature
-        try:
-            self._get_and_write_raw(url_best, filename)
-        except (QueryReturnedForbiddenException, QueryReturnedNotFoundException) as err:
-            self.error('{} Retrying with lower quality version.'.format(err))
-            self._get_and_write_raw(url, filename)
         os.utime(filename, (datetime.now().timestamp(), date_object.timestamp()))
         self._log('') # log output of _get_and_write_raw() does not produce \n
 
@@ -946,7 +984,7 @@ class Instaloader:
     def test_login(self) -> Optional[str]:
         """Returns the Instagram username to which given :class:`requests.Session` object belongs, or None."""
         data = self.graphql_query("d6f4427fbe92d846298cf93df0b937d3", {})
-        return data["data"]["user"]["username"] if "username" in data["data"]["user"] else None
+        return data["data"]["user"]["username"] if data["data"]["user"] is not None else None
 
     def login(self, user: str, passwd: str) -> None:
         """Log in to instagram with given username and password and internally store session object"""
@@ -1184,30 +1222,22 @@ class Instaloader:
     def get_feed_posts(self) -> Iterator[Post]:
         """Get Posts of the user's feed."""
 
-        data = self.get_json('', params={'__a': 1})
+        if not self.is_logged_in:
+            return
+        data = self.graphql_query("d6f4427fbe92d846298cf93df0b937d3", {})["data"]
 
         while True:
-            if "graphql" in data:
+            feed = data["user"]["edge_web_feed_timeline"]
-                is_edge = True
+            yield from (Post(self, edge["node"]) for edge in feed["edges"]
-                feed = data["graphql"]["user"]["edge_web_feed_timeline"]
+                        if not edge["node"]["__typename"] == "GraphSuggestedUserFeedUnit")
-            elif "data" in data:
-                is_edge = True
-                feed = data["data"]["user"]["edge_web_feed_timeline"]
-            else:
-                is_edge = False
-                feed = data["feed"]["media"]
-
-            if is_edge:
-                yield from (Post(self, edge["node"]) for edge in feed["edges"])
-            else:
-                yield from (Post(self, node) for node in feed["nodes"])
-
             if not feed["page_info"]["has_next_page"]:
                 break
-            data = self.graphql_query(17863003771166879, {'fetch_media_item_count': 12,
+            data = self.graphql_query("d6f4427fbe92d846298cf93df0b937d3",
-                                                          'fetch_media_item_cursor': feed["page_info"]["end_cursor"],
+                                      {'fetch_media_item_count': 12,
-                                                          'fetch_comment_count': 4,
+                                       'fetch_media_item_cursor': feed["page_info"]["end_cursor"],
-                                                          'fetch_like': 10})
+                                       'fetch_comment_count': 4,
+                                       'fetch_like': 10,
+                                       'has_stories': False})["data"]
 
     def download_feed_posts(self, max_count: int = None, fast_update: bool = False,
                             filter_func: Optional[Callable[[Post], bool]] = None) -> None:
@@ -1243,16 +1273,15 @@ class Instaloader:
     def get_saved_posts(self) -> Iterator[Post]:
         """Get Posts that are marked as saved by the user."""
 
-        data = self.get_profile_metadata(self.username)
+        if not self.is_logged_in:
+            return
+        data, full_metadata = self.get_profile_metadata(self.username)
         user_id = data["user"]["id"]
 
         while True:
-            if "graphql" in data:
+            if "edge_saved_media" in data["user"]:
                 is_edge = True
-                saved_media = data["graphql"]["user"]["edge_saved_media"]
+                saved_media = data["user"]["edge_saved_media"]
-            elif "data" in data:
-                is_edge = True
-                saved_media = data["data"]["user"]["edge_saved_media"]
             else:
                 is_edge = False
                 saved_media = data["user"]["saved_media"]
@@ -1266,7 +1295,8 @@ class Instaloader:
                 break
             data = self.graphql_query("f883d95537fbcd400f466f63d42bd8a1",
                                       {'id': user_id, 'first': Instaloader.GRAPHQL_PAGE_LENGTH,
-                                       'after': saved_media["page_info"]["end_cursor"]})
+                                       'after': saved_media["page_info"]["end_cursor"]},
+                                      rhx_gis=full_metadata['rhx_gis'])['data']
 
     def download_saved_posts(self, max_count: int = None, fast_update: bool = False,
                              filter_func: Optional[Callable[[Post], bool]] = None) -> None:
@@ -1293,10 +1323,19 @@ class Instaloader:
 
     def get_hashtag_posts(self, hashtag: str) -> Iterator[Post]:
         """Get Posts associated with a #hashtag."""
-        yield from (Post(self, node) for node in
+        has_next_page = True
-                    self.graphql_node_list(17875800862117404, {'tag_name': hashtag},
+        end_cursor = None
-                                           'https://www.instagram.com/explore/tags/{0}/'.format(hashtag),
+        while has_next_page:
-                                           lambda d: d['data']['hashtag']['edge_hashtag_to_media']))
+            if end_cursor:
+                params = {'__a': 1, 'max_id': end_cursor}
+            else:
+                params = {'__a': 1}
+            hashtag_data = self.get_json('explore/tags/{0}/'.format(hashtag),
+                                         params)['graphql']['hashtag']['edge_hashtag_to_media']
+            yield from (Post(self, edge['node']) for edge in
+                        hashtag_data['edges'])
+            has_next_page = hashtag_data['page_info']['has_next_page']
+            end_cursor = hashtag_data['page_info']['end_cursor']
 
     def download_hashtag(self, hashtag: str,
                          max_count: Optional[int] = None,
@@ -1355,6 +1394,9 @@ class Instaloader:
                     self._log("Profile {0} does not match the stored unique ID {1}.".format(profile, profile_id))
                 else:
                     self._log("Trying to find profile {0} using its unique ID {1}.".format(profile, profile_id))
+                if not self.is_logged_in:
+                    self.error("Profile {} changed its name. If you use --login=USERNAME, I can find out the new name.")
+                    raise LoginRequiredException("Login required to obtain username from userid")
                 newname = self.get_username_by_id(profile_id)
                 self._log("Profile {0} has changed its name to {1}.".format(profile, newname))
                 if ((format_string_contains_key(self.dirname_pattern, 'profile') or
@@ -1380,28 +1422,36 @@ class Instaloader:
             return profile, profile_id
         raise ProfileNotExistsException("Profile {0} does not exist.".format(profile))
 
-    def get_profile_metadata(self, profile_name: str) -> Dict[str, Any]:
+    def get_profile_metadata(self, profile_name: str) -> Tuple[Dict[str, Any], Dict[str, Any]]:
         """Retrieves a profile's metadata, for use with e.g. :meth:`get_profile_posts` and :meth:`check_profile_id`."""
         try:
-            return self.get_json('{}/'.format(profile_name), params={'__a': 1})
+            metadata = self.get_json('{}/'.format(profile_name), params={})
+            return metadata['entry_data']['ProfilePage'][0]['graphql'], metadata
         except QueryReturnedNotFoundException:
             raise ProfileNotExistsException('Profile {} does not exist.'.format(profile_name))
 
-    def get_profile_posts(self, profile_metadata: Dict[str, Any]) -> Iterator[Post]:
+    def get_profile_posts(self, profile_metadata: Dict[str, Any], rhx_gis: str) -> Iterator[Post]:
         """Retrieve all posts from a profile."""
         profile_name = profile_metadata['user']['username']
         profile_id = int(profile_metadata['user']['id'])
-        yield from (Post(self, node, profile=profile_name, profile_id=profile_id)
+        if 'media' in profile_metadata['user']:
-                    for node in profile_metadata['user']['media']['nodes'])
+            # backwards compatibility with old non-graphql structure
-        has_next_page = profile_metadata['user']['media']['page_info']['has_next_page']
+            yield from (Post(self, node, profile=profile_name, profile_id=profile_id)
-        end_cursor = profile_metadata['user']['media']['page_info']['end_cursor']
+                        for node in profile_metadata['user']['media']['nodes'])
+            has_next_page = profile_metadata['user']['media']['page_info']['has_next_page']
+            end_cursor = profile_metadata['user']['media']['page_info']['end_cursor']
+        else:
+            yield from (Post(self, edge['node'], profile=profile_name, profile_id=profile_id)
+                        for edge in profile_metadata['user']['edge_owner_to_timeline_media']['edges'])
+            has_next_page = profile_metadata['user']['edge_owner_to_timeline_media']['page_info']['has_next_page']
+            end_cursor = profile_metadata['user']['edge_owner_to_timeline_media']['page_info']['end_cursor']
         while has_next_page:
             # We do not use self.graphql_node_list() here, because profile_metadata
             # lets us obtain the first 12 nodes 'for free'
             data = self.graphql_query(17888483320059182, {'id': profile_metadata['user']['id'],
                                                           'first': Instaloader.GRAPHQL_PAGE_LENGTH,
                                                           'after': end_cursor},
-                                      'https://www.instagram.com/{0}/'.format(profile_name))
+                                      referer='https://www.instagram.com/{0}/'.format(profile_name), rhx_gis=rhx_gis)
             media = data['data']['user']['edge_owner_to_timeline_media']
             yield from (Post(self, edge['node'], profile=profile_name, profile_id=profile_id)
                         for edge in media['edges'])
@@ -1421,14 +1471,14 @@ class Instaloader:
         with suppress(ProfileNotExistsException):
             # ProfileNotExistsException is raised again later in check_profile_id() when we search the profile, so we
             # must suppress it here.
-            profile_metadata = self.get_profile_metadata(name)
+            profile_metadata, full_metadata = self.get_profile_metadata(name)
 
         # check if profile does exist or name has changed since last download
         # and update name and json data if necessary
         name_updated, profile_id = self.check_profile_id(name, profile_metadata)
         if name_updated != name:
             name = name_updated
-            profile_metadata = self.get_profile_metadata(name)
+            profile_metadata, full_metadata = self.get_profile_metadata(name)
 
         # Download profile picture
         if profile_pic or profile_pic_only:
@@ -1455,15 +1505,15 @@ class Instaloader:
         if download_stories_only:
             return
 
-        if ("nodes" not in profile_metadata["user"]["media"] or
-                not profile_metadata["user"]["media"]["nodes"]):
-            raise ProfileHasNoPicsException("Profile %s: no pics found." % name)
-
         # Iterate over pictures and download them
         self._log("Retrieving posts from profile {}.".format(name))
-        totalcount = profile_metadata["user"]["media"]["count"]
+        if "media" in profile_metadata["user"]:
+            # backwards compatibility with old non-graphql structure
+            totalcount = profile_metadata["user"]["media"]["count"]
+        else:
+            totalcount = profile_metadata["user"]["edge_owner_to_timeline_media"]["count"]
         count = 1
-        for post in self.get_profile_posts(profile_metadata):
+        for post in self.get_profile_posts(profile_metadata, rhx_gis=full_metadata['rhx_gis']):
             self._log("[%3i/%3i] " % (count, totalcount), end="", flush=True)
             count += 1
             if filter_func is not None and not filter_func(post):
@@ -1581,6 +1631,14 @@ class Instaloader:
         # Save session if it is useful
         if username is not None:
             self.save_session_to_file(sessionfile)
+        # User might be confused if Instaloader does nothing
+        if not profilelist:
+            if self.is_logged_in:
+                # Instaloader did at least save a session file
+                self._log("No targets were specified, thus nothing has been downloaded.")
+            else:
+                # Instloader did not do anything
+                self._log("usage:"+USAGE_STRING)
         if self.error_log:
             print("\nErrors occured:", file=sys.stderr)
             for err in self.error_log:
@@ -1588,7 +1646,7 @@ class Instaloader:
 
 
 def main():
-    parser = ArgumentParser(description=__doc__, add_help=False,
+    parser = ArgumentParser(description=__doc__, add_help=False, usage=USAGE_STRING,
                             epilog="Report issues at https://github.com/instaloader/instaloader/issues. "
                                    "The complete documentation can be found at "
                                    "https://instaloader.github.io/.")