Release of version 3.3.4

Use 'i.instagram.com' API to retrieve the best quality version of the
profile pic.
Related to #85.

docs/_static/style.css

@@ -0,0 +1,3 @@
+code {
+    color: #222;
+}

docs/_templates/caption.html

@@ -1,2 +0,0 @@
-<h2><a href="index.html">Instaloader</a></h2>
-<p>Download pictures (or videos) along with their captions and other metadata from Instagram.</p>

docs/_templates/layout.html

@@ -1,5 +1,11 @@
 {% extends "!layout.html" %}
 {% block extrahead %}
-<meta name="google-site-verification" content="FNUsDdkOJM9obm9QEdrNBVd9AtAzHzA-aaSMCUrS6C8" />
-<script async defer src="https://buttons.github.io/buttons.js"></script>
+{{ super() }}
+  <meta name="google-site-verification" content="FNUsDdkOJM9obm9QEdrNBVd9AtAzHzA-aaSMCUrS6C8" />
+  {% if pagename == "index" %}
+  <link rel="canonical" href="https://instaloader.github.io/" />
+  {% else %}
+  <link rel="canonical" href="https://instaloader.github.io/{{ pagename }}.html" />
+  {% endif %}
+  <script async defer src="https://buttons.github.io/buttons.js"></script>
 {% endblock %}

docs/_templates/links.html

@@ -1,23 +0,0 @@
-{% if next %}
-<h3>Next</h3>
-<ul>
-  <li class="toctree-l1"><a class="reference internal" href="{{ next.link|e }}">{{ next.title }}</a></li>
-</ul>
-{% endif %}
-<h3>Current Release</h3>
-<ul>
-  <li class="toctree-l1"><a href="https://github.com/instaloader/instaloader/releases/tag/v{{current_release}}">Version <b>{{current_release}}</b></a></li>
-  <li>Released on <b>{{current_release_date}}</b>.</li>
-</ul>
-<h3>Links</h3>
-<ul>
-  <li><a href="https://github.com/instaloader/instaloader/issues">Issue Tracker</a></li>
-  <li><a href="https://github.com/instaloader/instaloader/releases">Version History</a></li>
-  <li><a href="https://pypi.python.org/pypi/instaloader">Instaloader on PyPI</a></li>
-  <li><a href="https://github.com/instaloader/instaloader">Instaloader on GitHub</a></li>
-</ul>
-<ul>
-  <li>
-    <a class="github-button" href="https://github.com/instaloader/Instaloader" data-icon="octicon-star" data-size="large" data-show-count="true" aria-label="Star instaloader/Instaloader on GitHub">Star</a>
-  </li>
-</ul>

docs/_templates/navbar.html

@@ -0,0 +1,58 @@
+  <div id="navbar" class="{{ theme_navbar_class }} navbar-default {% if theme_navbar_fixed_top|tobool -%} navbar-fixed-top{%- endif -%}">
+    <div class="container">
+      <div class="navbar-header">
+        <!-- .btn-navbar is used as the toggle for collapsed navbar content -->
+        <button type="button" class="navbar-toggle" data-toggle="collapse" data-target=".nav-collapse">
+          <span class="icon-bar"></span>
+          <span class="icon-bar"></span>
+          <span class="icon-bar"></span>
+        </button>
+        <a class="navbar-brand" href="{{ pathto(master_doc) }}">
+          {%- block sidebarlogo %}
+            {%- if logo %}<span><img src="{{ pathto('_static/' + logo, 1) }}"></span>{%- endif %}
+          {%- endblock %}
+          {% if theme_navbar_title -%}{{ theme_navbar_title|e }}{%- else -%}{{ project|e }}{%- endif -%}
+        </a>
+        <span class="navbar-text navbar-version pull-left">
+          <a href="https://github.com/instaloader/instaloader/releases/tag/v{{current_release}}" title="Released on {{current_release_date}}.">v<b>{{current_release}}</b></a>
+        </span>
+      </div>
+
+        <div class="collapse navbar-collapse nav-collapse">
+          <ul class="nav navbar-nav">
+            {% if theme_navbar_links %}
+              {%- for link in theme_navbar_links %}
+                <li><a href="{{ pathto(*link[1:]) }}">{{ link[0] }}</a></li>
+              {%- endfor %}
+            {% endif %}
+            {% block navbartoc %}
+              {% include "globaltoc.html" %}
+              {% if theme_navbar_pagenav %}
+                {% include "navbartoc.html" %}
+              {% endif %}
+            {% endblock %}
+            {% if theme_navbar_sidebarrel %}
+              {% block sidebarrel %}
+                {% include "relations.html" %}
+              {% endblock %}
+            {% endif %}
+            </ul>
+            <ul class="nav navbar-nav navbar-right">
+              <li class="dropdown">
+                <a role="button" id="dLabelUsefulLinks" data-toggle="dropdown" data-target="#" href="#">
+                  Useful Links
+                  <b class="caret"></b>
+                </a>
+                <ul class="dropdown-menu" role="menu" aria-labelledby="dLabelUsefulLinks">
+                  <li><a href="https://github.com/instaloader/instaloader/issues">Issue Tracker</a></li>
+                  <li><a href="https://github.com/instaloader/instaloader/releases">Version History</a></li>
+                  <li><a href="https://pypi.python.org/pypi/instaloader">Instaloader on PyPI</a></li>
+                  <li><a href="https://github.com/instaloader/instaloader">Instaloader on GitHub</a></li>
+                  <li role="separator" class="divider"></li>
+                  <li style="padding-left: 20px;"><a class="github-button" href="https://github.com/instaloader/instaloader" data-icon="octicon-star" data-size="large" data-show-count="true" aria-label="Star Instaloader on GitHub">Star</a></li>
+                </ul>
+              </li>
+            </ul>
+        </div>
+    </div>
+  </div>

docs/_templates/rtdmessage.html

@@ -1,3 +0,0 @@
-<h3>New Documentation URL</h3>
-<div class="danger"><p style="margin-left: 1em"><b>Instaloader's Documentation has been moved to
-    <a class="reference external" href="https://instaloader.github.io/">https://instaloader.github.io</a>.</b></p></div>

docs/as-module.rst

@@ -1,7 +1,7 @@
 .. meta::
    :description:
-      Instaloader can also be used as a powerful and easy-to-use
-      Python API for Instagram, allowing to download media and metadata.
+      Documentation of Instaloader module, a powerful and easy-to-use
+      Python library to download Instagram media and metadata.
 
 Python Module :mod:`instaloader`
 --------------------------------

docs/basic-usage.rst

@@ -1,3 +1,10 @@
+.. meta::
+   :description:
+      How to download pictures from Instagram. Description of basic
+      usage of Instaloader, free tool to download photos from public
+      and private profiles, hashtags, stories, feeds, saved media, and
+      their metadata, comments and captions.
+
 Download Pictures from Instagram
 ---------------------------------
 

docs/conf.py

@@ -20,6 +20,9 @@
 import os
 import subprocess
 import sys
+
+import sphinx_bootstrap_theme
+
 sys.path.insert(0, os.path.abspath('..'))
 
 # -- General configuration ------------------------------------------------
@@ -133,16 +136,20 @@ todo_include_todos = False
 # The theme to use for HTML and HTML Help pages.  See the documentation for
 # a list of builtin themes.
 #
-html_theme = 'alabaster'
+html_theme = 'bootstrap'
+html_theme_path = sphinx_bootstrap_theme.get_html_theme_path()
 
 # Theme options are theme-specific and customize the look and feel of a theme
 # further.  For a list of options available for each theme, see the
 # documentation.
 #
 html_theme_options = {
-    'show_powered_by': False,
-    'sidebar_width': '290px',
-    'page_width': '935px' }
+    'navbar_site_name': 'Site Contents',
+    'navbar_pagenav_name': 'Page Contents',
+    'navbar_pagenav': True,
+    'navbar_sidebarrel': True,
+    'nosidebar': True,
+     }
 
 # Add any paths that contain custom themes here, relative to this directory.
 # html_theme_path = []
@@ -191,10 +198,8 @@ html_static_path = ['_static']
 
 # Custom sidebar templates, maps document names to template names.
 #
-if not os.environ.get("READTHEDOCS"):
-    html_sidebars = {'**': ["caption.html", "globaltoc.html", "relations.html", "links.html"] }
-else:
-    html_sidebars = {'**': ["caption.html", "rtdmessage.html", "globaltoc.html", "relations.html", "links.html"] }
+#html_sidebars = {'**': ["relations.html", "links.html"] }
+html_sidebars = {}
 
 # Additional templates that should be rendered to pages, maps page names to
 # template names.
@@ -362,3 +367,6 @@ current_release = subprocess.check_output(["git", "describe", "--abbrev=0"]).dec
 current_release_date = subprocess.check_output(["git", "log", "-1", "--tags", "--format=%ad", "--date=format:%e %b %Y"]).decode("ascii")[:-1]
 
 html_context = {'current_release': current_release, 'current_release_date': current_release_date}
+
+def setup(app):
+    app.add_stylesheet("style.css")

docs/index.rst

@@ -1,8 +1,9 @@
 .. meta::
    :description:
-      Command line tool to download pictures (and videos) from Instagram.
-      Instaloader downloads public and private profiles, hashtags, user stories,
-      feeds, saved media, comments, geotags, captions and other metadata of each post.
+      Free command line tool to download photos from Instagram.
+      Scrapes public and private profiles, hashtags, stories, feeds,
+      saved media, and their metadata, comments and captions.
+      Written in Python.
 
 Instaloader
 ===========

docs/installation.rst

@@ -28,7 +28,7 @@ will be installed automatically, if it is not already installed.
 
 - If you do not want to use pip, even though it is highly recommended,
   and prefer to **install Instaloader manually**,
-  `Download the Source <https://github.com/instaloader/instaloader/releases>`__,
+  `Download the Source <https://github.com/instaloader/instaloader/releases/latest>`__,
   extract the Zip or Tarball and execute ``instaloader.py`` from there.
 
 - On **Arch Linux**, you may install Instaloader using the
@@ -36,3 +36,6 @@ will be installed automatically, if it is not already installed.
 
 - On **Gentoo Linux**, you may install Instaloader using the
   `Instaloader Ebuild <https://bugs.gentoo.org/642100>`__.
+
+- On **Windows 10**, you may download the standalone executable from the
+  `current release page <https://github.com/instaloader/instaloader/releases/latest>`__.

docs/requirements.txt

@@ -1,3 +1,4 @@
 requests
 sphinx
 sphinx-autodoc-typehints
+sphinx-bootstrap-theme

instaloader.py

@@ -30,14 +30,14 @@ import requests.utils
 import urllib3
 
 
-__version__ = '3.3.3'
+__version__ = '3.3.4'
 
 # NOTE: duplicated in README.rst and docs/index.rst
 USAGE_STRING = """
 {0} [--comments] [--geotags] [--stories]
-            [--login YOUR-USERNAME] [--fast-update]
-            profile | "#hashtag" | :stories | :feed | :saved
-{0} --help""".format(sys.argv[0])
+{2:{1}} [--login YOUR-USERNAME] [--fast-update]
+{2:{1}} profile | "#hashtag" | :stories | :feed | :saved
+{0} --help""".format(sys.argv[0], len(sys.argv[0]), '')
 
 try:
     # pylint:disable=wrong-import-position
@@ -411,7 +411,7 @@ class Post:
         if loc_dict is not None:
             location_json = self._instaloader.get_json("explore/locations/{0}/".format(loc_dict["id"]),
                                                        params={'__a': 1})
-            return location_json["location"]
+            return location_json["location"] if "location" in location_json else location_json['graphql']['location']
 
     @staticmethod
     def json_encoder(obj) -> Dict[str, Any]:
@@ -588,12 +588,13 @@ class Instaloader:
                 self.error("[skipped by user]", repeat_at_end=False)
                 raise ConnectionException(error_string)
 
-    def get_json(self, url: str, params: Dict[str, Any],
+    def get_json(self, path: str, params: Dict[str, Any], host: str = 'www.instagram.com',
                  session: Optional[requests.Session] = None, _attempt = 1) -> Dict[str, Any]:
         """JSON request to Instagram.
 
-        :param url: URL, relative to www.instagram.com/
+        :param path: URL, relative to the given domain which defaults to www.instagram.com/
         :param params: GET parameters
+        :param host: Domain part of the URL from where to download the requested JSON; defaults to www.instagram.com
         :param session: Session to use, or None to use self.session
         :return: Decoded response dictionary
         :raises QueryReturnedNotFoundException: When the server responds with a 404.
@@ -610,7 +611,7 @@ class Instaloader:
             if len(timestamps) < 100 and not untracked_queries:
                 return 0
             return round(min(timestamps) + sliding_window - current_time) + 6
-        is_graphql_query = 'query_id' in params and 'graphql/query' in url
+        is_graphql_query = 'query_id' in params and 'graphql/query' in path
         if is_graphql_query:
             query_id = params['query_id']
             waittime = graphql_query_waittime(query_id)
@@ -625,7 +626,7 @@ class Instaloader:
         sess = session if session else self.session
         try:
             self._sleep()
-            resp = sess.get('https://www.instagram.com/' + url, params=params)
+            resp = sess.get('https://{0}/{1}'.format(host, path), params=params)
             if resp.status_code == 404:
                 raise QueryReturnedNotFoundException("404")
             if resp.status_code == 429:
@@ -641,7 +642,7 @@ class Instaloader:
                     raise ConnectionException("Returned \"{}\" status.".format(resp_json['status']))
             return resp_json
         except (ConnectionException, json.decoder.JSONDecodeError, requests.exceptions.RequestException) as err:
-            error_string = "JSON Query to {}: {}".format(url, err)
+            error_string = "JSON Query to {}: {}".format(path, err)
             if _attempt == self.max_connection_attempts:
                 raise ConnectionException(error_string)
             self.error(error_string + " [retrying; skip with ^C]", repeat_at_end=False)
@@ -657,7 +658,7 @@ class Instaloader:
                             self._log('The request will be retried in {} seconds.'.format(waittime))
                             time.sleep(waittime)
                 self._sleep()
-                return self.get_json(url, params, sess, _attempt + 1)
+                return self.get_json(path=path, params=params, host=host, session=sess, _attempt=_attempt + 1)
             except KeyboardInterrupt:
                 self.error("[skipped by user]", repeat_at_end=False)
                 raise ConnectionException(error_string)
@@ -900,8 +901,14 @@ class Instaloader:
     def download_profilepic(self, name: str, profile_metadata: Dict[str, Any]) -> None:
         """Downloads and saves profile pic."""
 
-        url = profile_metadata["user"]["profile_pic_url_hd"] if "profile_pic_url_hd" in profile_metadata["user"] \
-            else profile_metadata["user"]["profile_pic_url"]
+        try:
+            data = self.get_json(path='api/v1/users/{0}/info/'.format(profile_metadata["user"]["id"]), params={},
+                                 host='i.instagram.com', session=self._get_anonymous_session())
+            url = data["user"]["hd_profile_pic_url_info"]["url"]
+        except (InstaloaderException, KeyError) as err:
+            self.error('{} Unable to fetch high quality profile pic.'.format(err))
+            url = profile_metadata["user"]["profile_pic_url_hd"] if "profile_pic_url_hd" in profile_metadata["user"] \
+                else profile_metadata["user"]["profile_pic_url"]
 
         def _epoch_to_string(epoch: datetime) -> str:
             return epoch.strftime('%Y-%m-%d_%H-%M-%S')
@@ -919,13 +926,7 @@ class Instaloader:
         if os.path.isfile(filename):
             self._log(filename + ' already exists')
             return None
-        url_best = re.sub(r'/s([1-9][0-9]{2})x\1/', '/s2048x2048/', url)
-        url_best = re.sub(r'/vp/[a-f0-9]{32}/[A-F0-9]{8}/', '/', url_best)      # remove signature
-        try:
-            self._get_and_write_raw(url_best, filename)
-        except (QueryReturnedForbiddenException, QueryReturnedNotFoundException) as err:
-            self.error('{} Retrying with lower quality version.'.format(err))
-            self._get_and_write_raw(url, filename)
+        self._get_and_write_raw(url, filename)
         os.utime(filename, (datetime.now().timestamp(), date_object.timestamp()))
         self._log('') # log output of _get_and_write_raw() does not produce \n