Release of version 3.3.5

It seems it is not possible at the moment to get username by userid
anonymously.

docs/_static/style.css

@@ -0,0 +1,12 @@
+code {
+    color: #222;
+    background-color: #f5f5f5;
+}
+
+dd {
+    padding-left: 8px;
+}
+
+dl.option dd {
+    padding-left: 12px;
+}

docs/_templates/caption.html

@@ -1,2 +0,0 @@
-<h2><a href="index.html">Instaloader</a></h2>
-<p>Download pictures (or videos) along with their captions and other metadata from Instagram.</p>

docs/_templates/globaltoc.html

@@ -0,0 +1,14 @@
+<li class="dropdown globaltoc-container">
+  <a role="button"
+     id="dLabelGlobalToc"
+     data-toggle="dropdown"
+     data-target="#"
+     href="{{ pathto(master_doc) }}">{{ theme_navbar_site_name }} <b class="caret"></b></a>
+  <ul class="dropdown-menu globaltoc"
+      role="menu"
+      aria-labelledby="dLabelGlobalToc">
+    {{ toctree(maxdepth=theme_globaltoc_depth|toint, collapse=False, includehidden=theme_globaltoc_includehidden|tobool) }}
+    <li role="separator" class="divider"></li>
+    <li class="toctree-l1"><a class="reference internal" href="genindex.html">Index</a></li>
+  </ul>
+</li>

docs/_templates/layout.html

@@ -1,5 +1,18 @@
 {% extends "!layout.html" %}
+{% block htmltitle %}
+  {% if pagename == "index" %}
+  <title>{{ title|striptags|e }}</title>
+  {% else %}
+  <title>{{ title|striptags|e }}{{ titlesuffix }}</title>
+  {% endif %}
+{% endblock %}
 {% block extrahead %}
-<meta name="google-site-verification" content="FNUsDdkOJM9obm9QEdrNBVd9AtAzHzA-aaSMCUrS6C8" />
-<script async defer src="https://buttons.github.io/buttons.js"></script>
+{{ super() }}
+  <meta name="google-site-verification" content="FNUsDdkOJM9obm9QEdrNBVd9AtAzHzA-aaSMCUrS6C8" />
+  {% if pagename == "index" %}
+  <link rel="canonical" href="https://instaloader.github.io/" />
+  {% else %}
+  <link rel="canonical" href="https://instaloader.github.io/{{ pagename }}.html" />
+  {% endif %}
+  <script async defer src="https://buttons.github.io/buttons.js"></script>
 {% endblock %}

docs/_templates/links.html

@@ -1,23 +0,0 @@
-{% if next %}
-<h3>Next</h3>
-<ul>
-  <li class="toctree-l1"><a class="reference internal" href="{{ next.link|e }}">{{ next.title }}</a></li>
-</ul>
-{% endif %}
-<h3>Current Release</h3>
-<ul>
-  <li class="toctree-l1"><a href="https://github.com/instaloader/instaloader/releases/tag/v{{current_release}}">Version <b>{{current_release}}</b></a></li>
-  <li>Released on <b>{{current_release_date}}</b>.</li>
-</ul>
-<h3>Links</h3>
-<ul>
-  <li><a href="https://github.com/instaloader/instaloader/issues">Issue Tracker</a></li>
-  <li><a href="https://github.com/instaloader/instaloader/releases">Version History</a></li>
-  <li><a href="https://pypi.python.org/pypi/instaloader">Instaloader on PyPI</a></li>
-  <li><a href="https://github.com/instaloader/instaloader">Instaloader on GitHub</a></li>
-</ul>
-<ul>
-  <li>
-    <a class="github-button" href="https://github.com/instaloader/Instaloader" data-icon="octicon-star" data-size="large" data-show-count="true" aria-label="Star instaloader/Instaloader on GitHub">Star</a>
-  </li>
-</ul>

docs/_templates/navbar.html

@@ -0,0 +1,58 @@
+  <div id="navbar" class="{{ theme_navbar_class }} navbar-default {% if theme_navbar_fixed_top|tobool -%} navbar-fixed-top{%- endif -%}">
+    <div class="container">
+      <div class="navbar-header">
+        <!-- .btn-navbar is used as the toggle for collapsed navbar content -->
+        <button type="button" class="navbar-toggle" data-toggle="collapse" data-target=".nav-collapse">
+          <span class="icon-bar"></span>
+          <span class="icon-bar"></span>
+          <span class="icon-bar"></span>
+        </button>
+        <a class="navbar-brand" href="{{ pathto(master_doc) }}">
+          {%- block sidebarlogo %}
+            {%- if logo %}<span><img src="{{ pathto('_static/' + logo, 1) }}"></span>{%- endif %}
+          {%- endblock %}
+          {% if theme_navbar_title -%}{{ theme_navbar_title|e }}{%- else -%}{{ project|e }}{%- endif -%}
+        </a>
+        <span class="navbar-text navbar-version pull-left">
+          <a href="https://github.com/instaloader/instaloader/releases/tag/v{{current_release}}" title="Released on {{current_release_date}}">v<b>{{current_release}}</b></a>
+        </span>
+      </div>
+
+        <div class="collapse navbar-collapse nav-collapse">
+          <ul class="nav navbar-nav">
+            {% if theme_navbar_links %}
+              {%- for link in theme_navbar_links %}
+                <li><a href="{{ pathto(*link[1:]) }}">{{ link[0] }}</a></li>
+              {%- endfor %}
+            {% endif %}
+            {% block navbartoc %}
+              {% include "globaltoc.html" %}
+              {% if theme_navbar_pagenav and pagename not in ("index", "genindex", "installation") %}
+                {% include "navbartoc.html" %}
+              {% endif %}
+            {% endblock %}
+            {% if theme_navbar_sidebarrel %}
+              {% block sidebarrel %}
+                {% include "relations.html" %}
+              {% endblock %}
+            {% endif %}
+            </ul>
+            <ul class="nav navbar-nav navbar-right">
+              <li class="dropdown">
+                <a role="button" id="dLabelUsefulLinks" data-toggle="dropdown" data-target="#" href="#">
+                  Useful Links
+                  <b class="caret"></b>
+                </a>
+                <ul class="dropdown-menu" role="menu" aria-labelledby="dLabelUsefulLinks">
+                  <li><a href="https://github.com/instaloader/instaloader/issues">Issue Tracker</a></li>
+                  <li><a href="https://github.com/instaloader/instaloader/releases">Version History</a></li>
+                  <li><a href="https://pypi.org/project/instaloader/">Instaloader on PyPI</a></li>
+                  <li><a href="https://github.com/instaloader/instaloader">Instaloader on GitHub</a></li>
+                  <li role="separator" class="divider"></li>
+                  <li style="padding-left: 20px;"><a class="github-button" href="https://github.com/instaloader/instaloader" data-icon="octicon-star" data-size="large" data-show-count="true" aria-label="Star Instaloader on GitHub">Star</a></li>
+                </ul>
+              </li>
+            </ul>
+        </div>
+    </div>
+  </div>

docs/_templates/rtdmessage.html

@@ -1,3 +0,0 @@
-<h3>New Documentation URL</h3>
-<div class="danger"><p style="margin-left: 1em"><b>Instaloader's Documentation has been moved to
-    <a class="reference external" href="https://instaloader.github.io/">https://instaloader.github.io</a>.</b></p></div>

docs/as-module.rst

@@ -1,7 +1,7 @@
 .. meta::
    :description:
-      Instaloader can also be used as a powerful and easy-to-use
-      Python API for Instagram, allowing to download media and metadata.
+      Documentation of Instaloader module, a powerful and easy-to-use
+      Python library to download Instagram media and metadata.
 
 Python Module :mod:`instaloader`
 --------------------------------

docs/basic-usage.rst

@@ -1,3 +1,10 @@
+.. meta::
+   :description:
+      How to download pictures from Instagram. Description of basic
+      usage of Instaloader, free tool to download photos from public
+      and private profiles, hashtags, stories, feeds, saved media, and
+      their metadata, comments and captions.
+
 Download Pictures from Instagram
 ---------------------------------
 

docs/conf.py

@@ -20,6 +20,9 @@
 import os
 import subprocess
 import sys
+
+import sphinx_bootstrap_theme
+
 sys.path.insert(0, os.path.abspath('..'))
 
 # -- General configuration ------------------------------------------------
@@ -133,16 +136,20 @@ todo_include_todos = False
 # The theme to use for HTML and HTML Help pages.  See the documentation for
 # a list of builtin themes.
 #
-html_theme = 'alabaster'
+html_theme = 'bootstrap'
+html_theme_path = sphinx_bootstrap_theme.get_html_theme_path()
 
 # Theme options are theme-specific and customize the look and feel of a theme
 # further.  For a list of options available for each theme, see the
 # documentation.
 #
 html_theme_options = {
-    'show_powered_by': False,
-    'sidebar_width': '290px',
-    'page_width': '935px' }
+    'navbar_site_name': 'Site Contents',
+    'navbar_pagenav_name': 'Page Contents',
+    'navbar_pagenav': True,
+    'navbar_sidebarrel': True,
+    'nosidebar': True,
+     }
 
 # Add any paths that contain custom themes here, relative to this directory.
 # html_theme_path = []
@@ -191,10 +198,8 @@ html_static_path = ['_static']
 
 # Custom sidebar templates, maps document names to template names.
 #
-if not os.environ.get("READTHEDOCS"):
-    html_sidebars = {'**': ["caption.html", "globaltoc.html", "relations.html", "links.html"] }
-else:
-    html_sidebars = {'**': ["caption.html", "rtdmessage.html", "globaltoc.html", "relations.html", "links.html"] }
+#html_sidebars = {'**': ["relations.html", "links.html"] }
+html_sidebars = {}
 
 # Additional templates that should be rendered to pages, maps page names to
 # template names.
@@ -207,7 +212,7 @@ html_domain_indices = False
 
 # If false, no index is generated.
 #
-html_use_index = False
+html_use_index = True
 
 # If true, the index is split into individual pages for each letter.
 #
@@ -362,3 +367,6 @@ current_release = subprocess.check_output(["git", "describe", "--abbrev=0"]).dec
 current_release_date = subprocess.check_output(["git", "log", "-1", "--tags", "--format=%ad", "--date=format:%e %b %Y"]).decode("ascii")[:-1]
 
 html_context = {'current_release': current_release, 'current_release_date': current_release_date}
+
+def setup(app):
+    app.add_stylesheet("style.css")

docs/index.rst

@@ -1,8 +1,11 @@
 .. meta::
    :description:
-      Command line tool to download pictures (and videos) from Instagram.
-      Instaloader downloads public and private profiles, hashtags, user stories,
-      feeds, saved media, comments, geotags, captions and other metadata of each post.
+      Free command line tool to download photos from Instagram.
+      Scrapes public and private profiles, hashtags, stories, feeds,
+      saved media, and their metadata, comments and captions.
+      Written in Python.
+
+.. title:: Instaloader — Download Instagram Photos and Metadata
 
 Instaloader
 ===========
@@ -42,8 +45,8 @@ With `Python <https://www.python.org/>`__ installed, do::
                 :stories | :feed | :saved
 
 
-Table of Contents
------------------
+Instaloader Documentation
+-------------------------
 
 .. toctree::
    :maxdepth: 2
@@ -58,7 +61,7 @@ Useful Links
 ------------
 
 - `Git Repository (on GitHub) <https://github.com/instaloader/instaloader>`__
-- `PyPI Project Page <https://pypi.python.org/pypi/instaloader>`__
+- `PyPI Project Page <https://pypi.org/project/instaloader/>`__
 - `Issue Tracker / Bug Tracker <https://github.com/instaloader/instaloader/issues>`__
 - `Version History <https://github.com/instaloader/instaloader/releases>`__
 

docs/installation.rst

@@ -28,7 +28,7 @@ will be installed automatically, if it is not already installed.
 
 - If you do not want to use pip, even though it is highly recommended,
   and prefer to **install Instaloader manually**,
-  `Download the Source <https://github.com/instaloader/instaloader/releases>`__,
+  `Download the Source <https://github.com/instaloader/instaloader/releases/latest>`__,
   extract the Zip or Tarball and execute ``instaloader.py`` from there.
 
 - On **Arch Linux**, you may install Instaloader using the
@@ -36,3 +36,6 @@ will be installed automatically, if it is not already installed.
 
 - On **Gentoo Linux**, you may install Instaloader using the
   `Instaloader Ebuild <https://bugs.gentoo.org/642100>`__.
+
+- On **Windows 10**, you may download the standalone executable from the
+  `current release page <https://github.com/instaloader/instaloader/releases/latest>`__.

docs/requirements.txt

@@ -1,3 +1,4 @@
 requests
 sphinx
 sphinx-autodoc-typehints
+sphinx-bootstrap-theme

instaloader.py

@@ -6,6 +6,7 @@ import getpass
 import json
 import os
 import pickle
+import platform
 import random
 import re
 import shutil
@@ -29,8 +30,14 @@ import requests.utils
 import urllib3
 
 
-__version__ = '3.3.2'
+__version__ = '3.3.5'
 
+# NOTE: duplicated in README.rst and docs/index.rst
+USAGE_STRING = """
+{0} [--comments] [--geotags] [--stories]
+{2:{1}} [--login YOUR-USERNAME] [--fast-update]
+{2:{1}} profile | "#hashtag" | :stories | :feed | :saved
+{0} --help""".format(sys.argv[0], len(sys.argv[0]), '')
 
 try:
     # pylint:disable=wrong-import-position
@@ -127,10 +134,10 @@ def mediaid_to_shortcode(mediaid: int) -> str:
     return b64encode(mediaid.to_bytes(9, 'big'), b'-_').decode().replace('A', ' ').lstrip().replace(' ','A')
 
 
-def format_string_contains_key(format_string: str, key: str) -> bool:
+def format_string_contains_key(format_string: '_PathPattern', key: str) -> bool:
     # pylint:disable=unused-variable
     for literal_text, field_name, format_spec, conversion in string.Formatter().parse(format_string):
-        if field_name == key or field_name.startswith(key + '.'):
+        if field_name and (field_name == key or field_name.startswith(key + '.')):
             return True
     return False
 
@@ -404,7 +411,7 @@ class Post:
         if loc_dict is not None:
             location_json = self._instaloader.get_json("explore/locations/{0}/".format(loc_dict["id"]),
                                                        params={'__a': 1})
-            return location_json["location"]
+            return location_json["location"] if "location" in location_json else location_json['graphql']['location']
 
     @staticmethod
     def json_encoder(obj) -> Dict[str, Any]:
@@ -440,6 +447,14 @@ class Tristate(Enum):
     always = 2
 
 
+class _PathPattern(str):
+    """Class overriding :meth:`str.format` for character substitution in paths for Windows, see issue #84."""
+
+    def format(self, *args: Any, **kwargs: Any) -> str:
+        ret = super().format(*args, **kwargs)
+        return ret.replace(':', '\ua789') if platform.system() == 'Windows' else ret
+
+
 class Instaloader:
     GRAPHQL_PAGE_LENGTH = 200
 
@@ -462,18 +477,19 @@ class Instaloader:
         self.username = None
         self.sleep = sleep
         self.quiet = quiet
-        self.dirname_pattern = dirname_pattern if dirname_pattern is not None else '{target}'
+        self.dirname_pattern = _PathPattern(dirname_pattern if dirname_pattern is not None else '{target}')
         if filename_pattern is not None:
             filename_pattern = re.sub(r"({(?:post\.)?date)([:}])", r"\1_utc\2", filename_pattern)
             self.filename_pattern_old = filename_pattern.replace('{date_utc}', '{date_utc:%Y-%m-%d_%H-%M-%S}')
-            self.filename_pattern_old = re.sub(r"(?i)({(?:post\.)?date_utc:[^}]*?)_UTC",
-                                               r"\1", self.filename_pattern_old)
+            self.filename_pattern_old = _PathPattern(re.sub(r"(?i)({(?:post\.)?date_utc:[^}]*?)_UTC",
+                                                            r"\1", self.filename_pattern_old))
             filename_pattern = re.sub(r"(?i)({(date_utc|post\.date_utc):(?![^}]*UTC[^}]*).*?)}",
                                       r"\1_UTC}", filename_pattern)
-            self.filename_pattern = filename_pattern.replace('{date_utc}', '{date_utc:%Y-%m-%d_%H-%M-%S_UTC}')
+            self.filename_pattern = _PathPattern(filename_pattern.replace('{date_utc}',
+                                                                          '{date_utc:%Y-%m-%d_%H-%M-%S_UTC}'))
         else:
-            self.filename_pattern = '{date_utc:%Y-%m-%d_%H-%M-%S_UTC}'
-            self.filename_pattern_old = '{date_utc:%Y-%m-%d_%H-%M-%S}'
+            self.filename_pattern = _PathPattern('{date_utc:%Y-%m-%d_%H-%M-%S_UTC}')
+            self.filename_pattern_old = _PathPattern('{date_utc:%Y-%m-%d_%H-%M-%S}')
         self.download_videos = download_videos
         self.download_video_thumbnails = download_video_thumbnails
         self.download_geotags = download_geotags
@@ -572,12 +588,13 @@ class Instaloader:
                 self.error("[skipped by user]", repeat_at_end=False)
                 raise ConnectionException(error_string)
 
-    def get_json(self, url: str, params: Dict[str, Any],
+    def get_json(self, path: str, params: Dict[str, Any], host: str = 'www.instagram.com',
                  session: Optional[requests.Session] = None, _attempt = 1) -> Dict[str, Any]:
         """JSON request to Instagram.
 
-        :param url: URL, relative to www.instagram.com/
+        :param path: URL, relative to the given domain which defaults to www.instagram.com/
         :param params: GET parameters
+        :param host: Domain part of the URL from where to download the requested JSON; defaults to www.instagram.com
         :param session: Session to use, or None to use self.session
         :return: Decoded response dictionary
         :raises QueryReturnedNotFoundException: When the server responds with a 404.
@@ -594,7 +611,7 @@ class Instaloader:
             if len(timestamps) < 100 and not untracked_queries:
                 return 0
             return round(min(timestamps) + sliding_window - current_time) + 6
-        is_graphql_query = 'query_id' in params and 'graphql/query' in url
+        is_graphql_query = 'query_id' in params and 'graphql/query' in path
         if is_graphql_query:
             query_id = params['query_id']
             waittime = graphql_query_waittime(query_id)
@@ -609,7 +626,7 @@ class Instaloader:
         sess = session if session else self.session
         try:
             self._sleep()
-            resp = sess.get('https://www.instagram.com/' + url, params=params)
+            resp = sess.get('https://{0}/{1}'.format(host, path), params=params)
             if resp.status_code == 404:
                 raise QueryReturnedNotFoundException("404")
             if resp.status_code == 429:
@@ -625,7 +642,7 @@ class Instaloader:
                     raise ConnectionException("Returned \"{}\" status.".format(resp_json['status']))
             return resp_json
         except (ConnectionException, json.decoder.JSONDecodeError, requests.exceptions.RequestException) as err:
-            error_string = "JSON Query to {}: {}".format(url, err)
+            error_string = "JSON Query to {}: {}".format(path, err)
             if _attempt == self.max_connection_attempts:
                 raise ConnectionException(error_string)
             self.error(error_string + " [retrying; skip with ^C]", repeat_at_end=False)
@@ -641,7 +658,7 @@ class Instaloader:
                             self._log('The request will be retried in {} seconds.'.format(waittime))
                             time.sleep(waittime)
                 self._sleep()
-                return self.get_json(url, params, sess, _attempt + 1)
+                return self.get_json(path=path, params=params, host=host, session=sess, _attempt=_attempt + 1)
             except KeyboardInterrupt:
                 self.error("[skipped by user]", repeat_at_end=False)
                 raise ConnectionException(error_string)
@@ -884,8 +901,14 @@ class Instaloader:
     def download_profilepic(self, name: str, profile_metadata: Dict[str, Any]) -> None:
         """Downloads and saves profile pic."""
 
-        url = profile_metadata["user"]["profile_pic_url_hd"] if "profile_pic_url_hd" in profile_metadata["user"] \
-            else profile_metadata["user"]["profile_pic_url"]
+        try:
+            data = self.get_json(path='api/v1/users/{0}/info/'.format(profile_metadata["user"]["id"]), params={},
+                                 host='i.instagram.com', session=self._get_anonymous_session())
+            url = data["user"]["hd_profile_pic_url_info"]["url"]
+        except (InstaloaderException, KeyError) as err:
+            self.error('{} Unable to fetch high quality profile pic.'.format(err))
+            url = profile_metadata["user"]["profile_pic_url_hd"] if "profile_pic_url_hd" in profile_metadata["user"] \
+                else profile_metadata["user"]["profile_pic_url"]
 
         def _epoch_to_string(epoch: datetime) -> str:
             return epoch.strftime('%Y-%m-%d_%H-%M-%S')
@@ -903,13 +926,7 @@ class Instaloader:
         if os.path.isfile(filename):
             self._log(filename + ' already exists')
             return None
-        url_best = re.sub(r'/s([1-9][0-9]{2})x\1/', '/s2048x2048/', url)
-        url_best = re.sub(r'/vp/[a-f0-9]{32}/[A-F0-9]{8}/', '/', url_best)      # remove signature
-        try:
-            self._get_and_write_raw(url_best, filename)
-        except (QueryReturnedForbiddenException, QueryReturnedNotFoundException) as err:
-            self.error('{} Retrying with lower quality version.'.format(err))
-            self._get_and_write_raw(url, filename)
+        self._get_and_write_raw(url, filename)
         os.utime(filename, (datetime.now().timestamp(), date_object.timestamp()))
         self._log('') # log output of _get_and_write_raw() does not produce \n
 
@@ -1285,10 +1302,19 @@ class Instaloader:
 
     def get_hashtag_posts(self, hashtag: str) -> Iterator[Post]:
         """Get Posts associated with a #hashtag."""
-        yield from (Post(self, node) for node in
-                    self.graphql_node_list(17875800862117404, {'tag_name': hashtag},
-                                           'https://www.instagram.com/explore/tags/{0}/'.format(hashtag),
-                                           lambda d: d['data']['hashtag']['edge_hashtag_to_media']))
+        has_next_page = True
+        end_cursor = None
+        while has_next_page:
+            if end_cursor:
+                params = {'__a': 1, 'max_id': end_cursor}
+            else:
+                params = {'__a': 1}
+            hashtag_data = self.get_json('explore/tags/{0}/'.format(hashtag),
+                                         params)['graphql']['hashtag']['edge_hashtag_to_media']
+            yield from (Post(self, edge['node']) for edge in
+                        hashtag_data['edges'])
+            has_next_page = hashtag_data['page_info']['has_next_page']
+            end_cursor = hashtag_data['page_info']['end_cursor']
 
     def download_hashtag(self, hashtag: str,
                          max_count: Optional[int] = None,
@@ -1347,6 +1373,9 @@ class Instaloader:
                     self._log("Profile {0} does not match the stored unique ID {1}.".format(profile, profile_id))
                 else:
                     self._log("Trying to find profile {0} using its unique ID {1}.".format(profile, profile_id))
+                if not self.is_logged_in:
+                    self.error("Profile {} changed its name. If you use --login=USERNAME, I can find out the new name.")
+                    raise LoginRequiredException("Login required to obtain username from userid")
                 newname = self.get_username_by_id(profile_id)
                 self._log("Profile {0} has changed its name to {1}.".format(profile, newname))
                 if ((format_string_contains_key(self.dirname_pattern, 'profile') or
@@ -1581,6 +1610,14 @@ class Instaloader:
         # Save session if it is useful
         if username is not None:
             self.save_session_to_file(sessionfile)
+        # User might be confused if Instaloader does nothing
+        if not profilelist:
+            if self.is_logged_in:
+                # Instaloader did at least save a session file
+                self._log("No targets were specified, thus nothing has been downloaded.")
+            else:
+                # Instloader did not do anything
+                self._log("usage:"+USAGE_STRING)
         if self.error_log:
             print("\nErrors occured:", file=sys.stderr)
             for err in self.error_log:
@@ -1588,7 +1625,7 @@ class Instaloader:
 
 
 def main():
-    parser = ArgumentParser(description=__doc__, add_help=False,
+    parser = ArgumentParser(description=__doc__, add_help=False, usage=USAGE_STRING,
                             epilog="Report issues at https://github.com/instaloader/instaloader/issues. "
                                    "The complete documentation can be found at "
                                    "https://instaloader.github.io/.")