Release of version 3.3.3

Workaround for #84.

LICENSE

@@ -1,6 +1,6 @@
 The MIT License (MIT)
 
-Copyright (c) 2016 Alexander Graf and André Koch-Kramer.
+Copyright (c) 2016-2018 Alexander Graf and André Koch-Kramer.
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

README.rst

@@ -1,5 +1,4 @@
-Instaloader
-===========
+.. image:: https://raw.githubusercontent.com/instaloader/instaloader/master/docs/logo_heading.png
 
 ::
 

docs/Makefile

@@ -50,7 +50,7 @@ clean:
 
 .PHONY: html
 html:
-	$(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html
+	LC_ALL=C $(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html
 	@echo
 	@echo "Build finished. The HTML pages are in $(BUILDDIR)/html."
 
@@ -223,3 +223,9 @@ dummy:
 	$(SPHINXBUILD) -b dummy $(ALLSPHINXOPTS) $(BUILDDIR)/dummy
 	@echo
 	@echo "Build finished. Dummy builder generates no files."
+
+favicon.ico: logo.svg
+	convert -density 384 -background transparent -define icon:auto-resize -colors 256 logo.svg favicon.ico
+
+logo.png: logo.svg
+	inkscape -z -e logo.png -w 100 -h 100 logo.svg

docs/_templates/rtdmessage.html

@@ -0,0 +1,3 @@
+<h3>New Documentation URL</h3>
+<div class="danger"><p style="margin-left: 1em"><b>Instaloader's Documentation has been moved to
+    <a class="reference external" href="https://instaloader.github.io/">https://instaloader.github.io</a>.</b></p></div>

docs/as-module.rst

@@ -37,7 +37,8 @@ certain source::
         L.download_post(post, target='#cat')
 
 Besides :func:`Instaloader.get_hashtag_posts`, there is
-:func:`Instaloader.get_feed_posts` and :func:`Instaloader.get_profile_posts`.
+:func:`Instaloader.get_feed_posts`, :func:`Instaloader.get_profile_posts` and
+:func:`Instaloader.get_saved_posts`.
 Also, :class:`Post` instances can be created with :func:`Post.from_shortcode`
 and :func:`Post.from_mediaid`.
 

docs/conf.py

@@ -159,13 +159,13 @@ html_theme_options = {
 # The name of an image file (relative to this directory) to place at the top
 # of the sidebar.
 #
-# html_logo = None
+html_logo = "logo.png"
 
 # The name of an image file (relative to this directory) to use as a favicon of
 # the docs.  This file should be a Windows icon file (.ico) being 16x16 or 32x32
 # pixels large.
 #
-# html_favicon = None
+html_favicon = "favicon.ico"
 
 # Add any paths that contain custom static files (such as style sheets) here,
 # relative to this directory. They are copied after the builtin static files,
@@ -191,7 +191,10 @@ html_static_path = ['_static']
 
 # Custom sidebar templates, maps document names to template names.
 #
-html_sidebars = {'**': ["caption.html", "globaltoc.html", "relations.html", "links.html"] }
+if not os.environ.get("READTHEDOCS"):
+    html_sidebars = {'**': ["caption.html", "globaltoc.html", "relations.html", "links.html"] }
+else:
+    html_sidebars = {'**': ["caption.html", "rtdmessage.html", "globaltoc.html", "relations.html", "links.html"] }
 
 # Additional templates that should be rendered to pages, maps page names to
 # template names.

instaloader.py

@@ -6,6 +6,7 @@ import getpass
 import json
 import os
 import pickle
+import platform
 import random
 import re
 import shutil
@@ -29,8 +30,14 @@ import requests.utils
 import urllib3
 
 
-__version__ = '3.3'
+__version__ = '3.3.3'
 
+# NOTE: duplicated in README.rst and docs/index.rst
+USAGE_STRING = """
+{0} [--comments] [--geotags] [--stories]
+            [--login YOUR-USERNAME] [--fast-update]
+            profile | "#hashtag" | :stories | :feed | :saved
+{0} --help""".format(sys.argv[0])
 
 try:
     # pylint:disable=wrong-import-position
@@ -127,10 +134,10 @@ def mediaid_to_shortcode(mediaid: int) -> str:
     return b64encode(mediaid.to_bytes(9, 'big'), b'-_').decode().replace('A', ' ').lstrip().replace(' ','A')
 
 
-def format_string_contains_key(format_string: str, key: str) -> bool:
+def format_string_contains_key(format_string: '_PathPattern', key: str) -> bool:
     # pylint:disable=unused-variable
     for literal_text, field_name, format_spec, conversion in string.Formatter().parse(format_string):
-        if field_name == key or field_name.startswith(key + '.'):
+        if field_name and (field_name == key or field_name.startswith(key + '.')):
             return True
     return False
 
@@ -169,7 +176,8 @@ class Post:
     """
     Structure containing information about an Instagram post.
 
-    Created by Instaloader methods :meth:`.get_profile_posts`, :meth:`.get_hashtag_posts`, :meth:`.get_feed_posts`.
+    Created by Instaloader methods :meth:`.get_profile_posts`, :meth:`.get_hashtag_posts`, :meth:`.get_feed_posts` and
+    :meth:`.get_saved_posts`.
     Posts are linked to an :class:`Instaloader` instance which is used for error logging and obtaining of additional
     metadata, if required. This class unifies access to the properties associated with a post. It implements == and is
     hashable.
@@ -439,6 +447,14 @@ class Tristate(Enum):
     always = 2
 
 
+class _PathPattern(str):
+    """Class overriding :meth:`str.format` for character substitution in paths for Windows, see issue #84."""
+
+    def format(self, *args: Any, **kwargs: Any) -> str:
+        ret = super().format(*args, **kwargs)
+        return ret.replace(':', '\ua789') if platform.system() == 'Windows' else ret
+
+
 class Instaloader:
     GRAPHQL_PAGE_LENGTH = 200
 
@@ -461,18 +477,19 @@ class Instaloader:
         self.username = None
         self.sleep = sleep
         self.quiet = quiet
-        self.dirname_pattern = dirname_pattern if dirname_pattern is not None else '{target}'
+        self.dirname_pattern = _PathPattern(dirname_pattern if dirname_pattern is not None else '{target}')
         if filename_pattern is not None:
             filename_pattern = re.sub(r"({(?:post\.)?date)([:}])", r"\1_utc\2", filename_pattern)
             self.filename_pattern_old = filename_pattern.replace('{date_utc}', '{date_utc:%Y-%m-%d_%H-%M-%S}')
-            self.filename_pattern_old = re.sub(r"(?i)({(?:post\.)?date_utc:[^}]*?)_UTC",
-                                               r"\1", self.filename_pattern_old)
+            self.filename_pattern_old = _PathPattern(re.sub(r"(?i)({(?:post\.)?date_utc:[^}]*?)_UTC",
+                                                            r"\1", self.filename_pattern_old))
             filename_pattern = re.sub(r"(?i)({(date_utc|post\.date_utc):(?![^}]*UTC[^}]*).*?)}",
                                       r"\1_UTC}", filename_pattern)
-            self.filename_pattern = filename_pattern.replace('{date_utc}', '{date_utc:%Y-%m-%d_%H-%M-%S_UTC}')
+            self.filename_pattern = _PathPattern(filename_pattern.replace('{date_utc}',
+                                                                          '{date_utc:%Y-%m-%d_%H-%M-%S_UTC}'))
         else:
-            self.filename_pattern = '{date_utc:%Y-%m-%d_%H-%M-%S_UTC}'
-            self.filename_pattern_old = '{date_utc:%Y-%m-%d_%H-%M-%S}'
+            self.filename_pattern = _PathPattern('{date_utc:%Y-%m-%d_%H-%M-%S_UTC}')
+            self.filename_pattern_old = _PathPattern('{date_utc:%Y-%m-%d_%H-%M-%S}')
         self.download_videos = download_videos
         self.download_video_thumbnails = download_video_thumbnails
         self.download_geotags = download_geotags
@@ -946,7 +963,7 @@ class Instaloader:
     def test_login(self) -> Optional[str]:
         """Returns the Instagram username to which given :class:`requests.Session` object belongs, or None."""
         data = self.graphql_query("d6f4427fbe92d846298cf93df0b937d3", {})
-        return data["data"]["user"]["username"] if "username" in data["data"]["user"] else None
+        return data["data"]["user"]["username"] if data["data"]["user"] is not None else None
 
     def login(self, user: str, passwd: str) -> None:
         """Log in to instagram with given username and password and internally store session object"""
@@ -1184,30 +1201,22 @@ class Instaloader:
     def get_feed_posts(self) -> Iterator[Post]:
         """Get Posts of the user's feed."""
 
-        data = self.get_json('', params={'__a': 1})
+        if not self.is_logged_in:
+            return
+        data = self.graphql_query("d6f4427fbe92d846298cf93df0b937d3", {})["data"]
 
         while True:
-            if "graphql" in data:
-                is_edge = True
-                feed = data["graphql"]["user"]["edge_web_feed_timeline"]
-            elif "data" in data:
-                is_edge = True
-                feed = data["data"]["user"]["edge_web_feed_timeline"]
-            else:
-                is_edge = False
-                feed = data["feed"]["media"]
-
-            if is_edge:
-                yield from (Post(self, edge["node"]) for edge in feed["edges"])
-            else:
-                yield from (Post(self, node) for node in feed["nodes"])
-
+            feed = data["user"]["edge_web_feed_timeline"]
+            yield from (Post(self, edge["node"]) for edge in feed["edges"]
+                        if not edge["node"]["__typename"] == "GraphSuggestedUserFeedUnit")
             if not feed["page_info"]["has_next_page"]:
                 break
-            data = self.graphql_query(17863003771166879, {'fetch_media_item_count': 12,
-                                                          'fetch_media_item_cursor': feed["page_info"]["end_cursor"],
-                                                          'fetch_comment_count': 4,
-                                                          'fetch_like': 10})
+            data = self.graphql_query("d6f4427fbe92d846298cf93df0b937d3",
+                                      {'fetch_media_item_count': 12,
+                                       'fetch_media_item_cursor': feed["page_info"]["end_cursor"],
+                                       'fetch_comment_count': 4,
+                                       'fetch_like': 10,
+                                       'has_stories': False})["data"]
 
     def download_feed_posts(self, max_count: int = None, fast_update: bool = False,
                             filter_func: Optional[Callable[[Post], bool]] = None) -> None:
@@ -1243,16 +1252,15 @@ class Instaloader:
     def get_saved_posts(self) -> Iterator[Post]:
         """Get Posts that are marked as saved by the user."""
 
+        if not self.is_logged_in:
+            return
         data = self.get_profile_metadata(self.username)
         user_id = data["user"]["id"]
 
         while True:
-            if "graphql" in data:
+            if "edge_saved_media" in data["user"]:
                 is_edge = True
-                saved_media = data["graphql"]["user"]["edge_saved_media"]
-            elif "data" in data:
-                is_edge = True
-                saved_media = data["data"]["user"]["edge_saved_media"]
+                saved_media = data["user"]["edge_saved_media"]
             else:
                 is_edge = False
                 saved_media = data["user"]["saved_media"]
@@ -1266,7 +1274,7 @@ class Instaloader:
                 break
             data = self.graphql_query("f883d95537fbcd400f466f63d42bd8a1",
                                       {'id': user_id, 'first': Instaloader.GRAPHQL_PAGE_LENGTH,
-                                       'after': saved_media["page_info"]["end_cursor"]})
+                                       'after': saved_media["page_info"]["end_cursor"]})['data']
 
     def download_saved_posts(self, max_count: int = None, fast_update: bool = False,
                              filter_func: Optional[Callable[[Post], bool]] = None) -> None:
@@ -1383,7 +1391,8 @@ class Instaloader:
     def get_profile_metadata(self, profile_name: str) -> Dict[str, Any]:
         """Retrieves a profile's metadata, for use with e.g. :meth:`get_profile_posts` and :meth:`check_profile_id`."""
         try:
-            return self.get_json('{}/'.format(profile_name), params={'__a': 1})
+            metadata = self.get_json('{}/'.format(profile_name), params={'__a': 1})
+            return metadata['graphql'] if 'graphql' in metadata else metadata
         except QueryReturnedNotFoundException:
             raise ProfileNotExistsException('Profile {} does not exist.'.format(profile_name))
 
@@ -1391,10 +1400,17 @@ class Instaloader:
         """Retrieve all posts from a profile."""
         profile_name = profile_metadata['user']['username']
         profile_id = int(profile_metadata['user']['id'])
-        yield from (Post(self, node, profile=profile_name, profile_id=profile_id)
-                    for node in profile_metadata['user']['media']['nodes'])
-        has_next_page = profile_metadata['user']['media']['page_info']['has_next_page']
-        end_cursor = profile_metadata['user']['media']['page_info']['end_cursor']
+        if 'media' in profile_metadata['user']:
+            # backwards compatibility with old non-graphql structure
+            yield from (Post(self, node, profile=profile_name, profile_id=profile_id)
+                        for node in profile_metadata['user']['media']['nodes'])
+            has_next_page = profile_metadata['user']['media']['page_info']['has_next_page']
+            end_cursor = profile_metadata['user']['media']['page_info']['end_cursor']
+        else:
+            yield from (Post(self, edge['node'], profile=profile_name, profile_id=profile_id)
+                        for edge in profile_metadata['user']['edge_owner_to_timeline_media']['edges'])
+            has_next_page = profile_metadata['user']['edge_owner_to_timeline_media']['page_info']['has_next_page']
+            end_cursor = profile_metadata['user']['edge_owner_to_timeline_media']['page_info']['end_cursor']
         while has_next_page:
             # We do not use self.graphql_node_list() here, because profile_metadata
             # lets us obtain the first 12 nodes 'for free'
@@ -1455,13 +1471,13 @@ class Instaloader:
         if download_stories_only:
             return
 
-        if ("nodes" not in profile_metadata["user"]["media"] or
-                not profile_metadata["user"]["media"]["nodes"]):
-            raise ProfileHasNoPicsException("Profile %s: no pics found." % name)
-
         # Iterate over pictures and download them
         self._log("Retrieving posts from profile {}.".format(name))
-        totalcount = profile_metadata["user"]["media"]["count"]
+        if "media" in profile_metadata["user"]:
+            # backwards compatibility with old non-graphql structure
+            totalcount = profile_metadata["user"]["media"]["count"]
+        else:
+            totalcount = profile_metadata["user"]["edge_owner_to_timeline_media"]["count"]
         count = 1
         for post in self.get_profile_posts(profile_metadata):
             self._log("[%3i/%3i] " % (count, totalcount), end="", flush=True)
@@ -1581,6 +1597,14 @@ class Instaloader:
         # Save session if it is useful
         if username is not None:
             self.save_session_to_file(sessionfile)
+        # User might be confused if Instaloader does nothing
+        if not profilelist:
+            if self.is_logged_in:
+                # Instaloader did at least save a session file
+                self._log("No targets were specified, thus nothing has been downloaded.")
+            else:
+                # Instloader did not do anything
+                self._log("usage:"+USAGE_STRING)
         if self.error_log:
             print("\nErrors occured:", file=sys.stderr)
             for err in self.error_log:
@@ -1588,7 +1612,7 @@ class Instaloader:
 
 
 def main():
-    parser = ArgumentParser(description=__doc__, add_help=False,
+    parser = ArgumentParser(description=__doc__, add_help=False, usage=USAGE_STRING,
                             epilog="Report issues at https://github.com/instaloader/instaloader/issues. "
                                    "The complete documentation can be found at "
                                    "https://instaloader.github.io/.")