Release of version 4.2.7

Closes #297.

.pylintrc

@@ -62,9 +62,7 @@ confidence=
 # --disable=W".
 disable=invalid-name,
         missing-docstring,
-        line-too-long,
         too-many-lines,
-        bad-whitespace,
         print-statement,
         parameter-unpacking,
         unpacking-in-except,
@@ -83,7 +81,6 @@ disable=invalid-name,
         useless-suppression,
         deprecated-pragma,
         use-symbolic-message-instead,
-        cyclic-import,
         too-many-instance-attributes,
         too-many-public-methods,
         too-many-branches,
@@ -91,7 +88,6 @@ disable=invalid-name,
         too-many-locals,
         too-many-statements,
         no-else-return,
-        inconsistent-return-statements,
         no-else-raise,
         unnecessary-pass,
         wildcard-import,
@@ -362,7 +358,7 @@ indent-after-paren=4
 indent-string='    '
 
 # Maximum number of characters on a single line.
-max-line-length=100
+max-line-length=120
 
 # Maximum number of lines in a module.
 max-module-lines=1000

README.rst

@@ -1,5 +1,41 @@
 .. image:: https://raw.githubusercontent.com/instaloader/instaloader/master/docs/logo_heading.png
 
+.. badges-start
+
+|travis| |pypi| |pyversion| |license| |aur| |contributors| |downloads| |saythanks|
+
+.. |travis| image:: https://img.shields.io/travis/instaloader/instaloader/master.svg
+   :alt: Travis-CI Build Status
+   :target: https://travis-ci.org/instaloader/instaloader
+
+.. |pypi| image:: https://img.shields.io/pypi/v/instaloader.svg
+   :alt: Instaloader PyPI Project Page
+   :target: https://pypi.org/project/instaloader/
+
+.. |license| image:: https://img.shields.io/github/license/instaloader/instaloader.svg
+   :alt: MIT License
+   :target: https://github.com/instaloader/instaloader/blob/master/LICENSE
+
+.. |pyversion| image:: https://img.shields.io/pypi/pyversions/instaloader.svg
+   :alt: Supported Python Versions
+
+.. |contributors| image:: https://img.shields.io/github/contributors/instaloader/instaloader.svg
+   :alt: Contributor Count
+   :target: https://github.com/instaloader/instaloader/graphs/contributors
+
+.. |saythanks| image:: https://img.shields.io/badge/Say%20Thanks-!-ff7efd.svg
+   :alt: Say Thanks!
+   :target: https://saythanks.io/to/aandergr
+
+.. |aur| image:: https://img.shields.io/aur/version/instaloader.svg
+   :alt: Arch User Repository Package
+   :target: https://aur.archlinux.org/packages/instaloader/
+
+.. |downloads| image:: https://img.shields.io/pypi/dm/instaloader.svg
+   :alt: PyPI Download Count
+
+.. badges-end
+
 ::
 
     $ pip3 install instaloader

deploy/arch/deploy.sh

@@ -1,5 +1,10 @@
 #!/usr/bin/env bash
 
+VERSION=${1:1}
+
+# do not deploy pre-releases
+echo $VERSION | grep -qe "[abc]" && exit 0
+
 cd $(dirname $0)
 
 # decrypt and add ssh key
@@ -12,7 +17,6 @@ ssh-add /tmp/AUR_openssh
 git clone --depth 1 ssh://aur@aur.archlinux.org/instaloader.git
 
 curl -sSfOJ https://codeload.github.com/instaloader/instaloader/tar.gz/$1
-VERSION=${1:1}
 HASH=$(sha512sum instaloader-$VERSION.tar.gz | cut -f1 -d " ")
 sed -e "s/{{version}}/$VERSION/g" -e "s/{{hash}}/$HASH/g" PKGBUILD.template > instaloader/PKGBUILD
 sed -e "s/{{version}}/$VERSION/g" -e "s/{{hash}}/$HASH/g" .SRCINFO.template > instaloader/.SRCINFO

docs/index.rst

@@ -15,6 +15,10 @@ Instaloader
 **Instaloader** is a tool to download pictures (or videos) along with
 their captions and other metadata from Instagram.
 
+.. include:: ../README.rst
+   :start-after: badges-start
+   :end-before: badges-end
+
 With `Python <https://www.python.org/>`__ installed, do::
 
     $ pip3 install instaloader

instaloader/__init__.py

@@ -1,7 +1,7 @@
 """Download pictures (or videos) along with their captions and other metadata from Instagram."""
 
 
-__version__ = '4.2.5rc1'
+__version__ = '4.2.7'
 
 
 try:
@@ -15,5 +15,5 @@ else:
 from .exceptions import *
 from .instaloader import Instaloader
 from .instaloadercontext import InstaloaderContext
-from .structures import (Highlight, Post, PostSidecarNode, PostComment, PostCommentAnswer, PostLocation, Profile, Story, StoryItem,
+from .structures import (Highlight, Post, PostSidecarNode, PostComment, PostCommentAnswer, PostLocation, Profile, Story,
-                         load_structure_from_file, save_structure_to_file)
+                         StoryItem, load_structure_from_file, save_structure_to_file)

instaloader/instaloader.py

@@ -13,14 +13,16 @@ from datetime import datetime, timezone
 from functools import wraps
 from hashlib import md5
 from io import BytesIO
-from typing import Any, Callable, ContextManager, Iterator, List, Optional, Set, Union, cast
+from pathlib import Path
+from typing import Any, Callable, Iterator, List, Optional, Set, Union
 
 import requests
 import urllib3  # type: ignore
 
 from .exceptions import *
 from .instaloadercontext import InstaloaderContext
-from .structures import Highlight, JsonExportable, Post, PostLocation, Profile, Story, StoryItem, save_structure_to_file, load_structure_from_file
+from .structures import (Highlight, JsonExportable, Post, PostLocation, Profile, Story, StoryItem,
+                         save_structure_to_file, load_structure_from_file)
 
 
 def get_default_session_filename(username: str) -> str:
@@ -368,7 +370,8 @@ class Instaloader:
                                                             profile_pic_identifier, profile_pic_extension)
         content_length = profile_pic_response.headers.get('Content-Length', None)
         if os.path.isfile(filename) and (not self.context.is_logged_in or
-                                         content_length is not None and os.path.getsize(filename) >= int(content_length)):
+                                         (content_length is not None and
+                                          os.path.getsize(filename) >= int(content_length))):
             self.context.log(filename + ' already exists')
             return None
         self.context.write_raw(profile_pic_bytes if profile_pic_bytes else profile_pic_response, filename)
@@ -417,7 +420,8 @@ class Instaloader:
         :raises InvalidArgumentException: If the provided username does not exist.
         :raises BadCredentialsException: If the provided password is wrong.
         :raises ConnectionException: If connection to Instagram failed.
-        :raises TwoFactorAuthRequiredException: First step of 2FA login done, now call :meth:`Instaloader.two_factor_login`."""
+        :raises TwoFactorAuthRequiredException: First step of 2FA login done, now call
+           :meth:`Instaloader.two_factor_login`."""
         self.context.login(user, passwd)
 
     def two_factor_login(self, two_factor_code) -> None:
@@ -430,13 +434,13 @@ class Instaloader:
         .. versionadded:: 4.2"""
         self.context.two_factor_login(two_factor_code)
 
-    def format_filename(self, item: Union[Post, StoryItem], target: Optional[str] = None):
+    def format_filename(self, item: Union[Post, StoryItem], target: Optional[Union[str, Path]] = None):
         """Format filename of a :class:`Post` or :class:`StoryItem` according to ``filename-pattern`` parameter.
 
         .. versionadded:: 4.1"""
         return _PostPathFormatter(item).format(self.filename_pattern, target=target)
 
-    def download_post(self, post: Post, target: str) -> bool:
+    def download_post(self, post: Post, target: Union[str, Path]) -> bool:
         """
         Download everything associated with one instagram post node, i.e. picture, caption and video.
 
@@ -563,7 +567,7 @@ class Instaloader:
                     if fast_update and not downloaded:
                         break
 
-    def download_storyitem(self, item: StoryItem, target: str) -> bool:
+    def download_storyitem(self, item: StoryItem, target: Union[str, Path]) -> bool:
         """Download one user story.
 
         :param item: Story item, as in story['items'] for story in :meth:`get_stories`
@@ -640,10 +644,11 @@ class Instaloader:
                     continue
                 self.context.log("[%3i/%3i] " % (count, totalcount), end="", flush=True)
                 count += 1
-                with self.context.error_catcher('Download highlights \"{}\" from user {}'.format(user_highlight.title, name)):
+                with self.context.error_catcher('Download highlights \"{}\" from user {}'.format(user_highlight.title,
+                                                                                                 name)):
                     downloaded = self.download_storyitem(item, filename_target
                                                          if filename_target
-                                                         else '{}/{}'.format(name, user_highlight.title))
+                                                         else Path(name) / Path(user_highlight.title))
                     if fast_update and not downloaded:
                         break
 
@@ -791,7 +796,7 @@ class Instaloader:
                     for node in self.context.graphql_node_list("df0dcc250c2b18d9fd27c5581ef33c7c",
                                                                {}, 'https://www.instagram.com/explore/',
                                                                lambda d: d['data']['user']['edge_web_discover_media'],
-                                                               data['rhx_gis']))
+                                                               data.get('rhx_gis')))
 
     def get_hashtag_posts(self, hashtag: str) -> Iterator[Post]:
         """Get Posts associated with a #hashtag."""
@@ -846,8 +851,6 @@ class Instaloader:
         """Download all posts where a profile is tagged.
 
         .. versionadded:: 4.1"""
-        if target is None:
-            target = profile.username + '/:tagged'
         self.context.log("Retrieving tagged posts for profile {}.".format(profile.username))
         count = 1
         for post in profile.get_tagged_posts():
@@ -856,7 +859,7 @@ class Instaloader:
             if post_filter is not None and not post_filter(post):
                 self.context.log('<{} skipped>'.format(post))
             with self.context.error_catcher('Download tagged {}'.format(profile.username)):
-                downloaded = self.download_post(post, target)
+                downloaded = self.download_post(post, target if target else Path(profile.username) / Path(':tagged'))
                 if fast_update and not downloaded:
                     break
 
@@ -953,11 +956,11 @@ class Instaloader:
         def _error_raiser(_str):
             yield
 
-        error_handler = cast(Callable[[Optional[str]], ContextManager[None]],
+        # error_handler type is Callable[[Optional[str]], ContextManager[None]] (not supported with Python 3.5)
-                             _error_raiser if raise_errors else self.context.error_catcher)
+        error_handler = _error_raiser if raise_errors else self.context.error_catcher
 
         for profile in profiles:
-            with error_handler(profile.username):
+            with error_handler(profile.username):  # type: ignore
                 profile_name = profile.username
 
                 # Download profile picture

instaloader/instaloadercontext.py

@@ -181,7 +181,8 @@ class InstaloaderContext:
         :raises InvalidArgumentException: If the provided username does not exist.
         :raises BadCredentialsException: If the provided password is wrong.
         :raises ConnectionException: If connection to Instagram failed.
-        :raises TwoFactorAuthRequiredException: First step of 2FA login done, now call :meth:`Instaloader.two_factor_login`."""
+        :raises TwoFactorAuthRequiredException: First step of 2FA login done, now call
+           :meth:`Instaloader.two_factor_login`."""
         import http.client
         # pylint:disable=protected-access
         http.client._MAXHEADERS = 200
@@ -529,6 +530,6 @@ class InstaloaderContext:
             # At the moment, rhx_gis seems to be required for anonymous requests only. By returning None when logged
             # in, we can save the root_rhx_gis lookup query.
             return None
-        if not self._root_rhx_gis:
+        if self._root_rhx_gis is None:
-            self._root_rhx_gis = self.get_json('', {})['rhx_gis']
+            self._root_rhx_gis = self.get_json('', {}).get('rhx_gis', '')
-        return self._root_rhx_gis
+        return self._root_rhx_gis or None

instaloader/structures.py

@@ -132,7 +132,7 @@ class Post:
         if not self._full_metadata_dict:
             pic_json = self._context.get_json("p/{0}/".format(self.shortcode), params={})
             self._full_metadata_dict = pic_json['entry_data']['PostPage'][0]['graphql']['shortcode_media']
-            self._rhx_gis_str = pic_json['rhx_gis']
+            self._rhx_gis_str = pic_json.get('rhx_gis')
             if self.shortcode != self._full_metadata_dict['shortcode']:
                 self._node.update(self._full_metadata_dict)
                 raise PostChangedException
@@ -144,9 +144,8 @@ class Post:
         return self._full_metadata_dict
 
     @property
-    def _rhx_gis(self) -> str:
+    def _rhx_gis(self) -> Optional[str]:
         self._obtain_metadata()
-        assert self._rhx_gis_str is not None
         return self._rhx_gis_str
 
     def _field(self, *keys) -> Any:
@@ -190,12 +189,16 @@ class Post:
     @property
     def date_local(self) -> datetime:
         """Timestamp when the post was created (local time zone)."""
-        return datetime.fromtimestamp(self._node["date"] if "date" in self._node else self._node["taken_at_timestamp"])
+        return datetime.fromtimestamp(self._node["date"]
+                                      if "date" in self._node
+                                      else self._node["taken_at_timestamp"])
 
     @property
     def date_utc(self) -> datetime:
         """Timestamp when the post was created (UTC)."""
-        return datetime.utcfromtimestamp(self._node["date"] if "date" in self._node else self._node["taken_at_timestamp"])
+        return datetime.utcfromtimestamp(self._node["date"]
+                                         if "date" in self._node
+                                         else self._node["taken_at_timestamp"])
 
     @property
     def date(self) -> datetime:
@@ -257,7 +260,9 @@ class Post:
 
     @property
     def pcaption(self) -> str:
-        """Printable caption, useful as a format specifier for --filename-pattern."""
+        """Printable caption, useful as a format specifier for --filename-pattern.
+
+        .. versionadded:: 4.2.6"""
         def _elliptify(caption):
             pcaption = ' '.join([s.replace('/', '\u2215') for s in caption.splitlines() if s]).strip()
             return (pcaption[:30] + u"\u2026") if len(pcaption) > 31 else pcaption
@@ -286,14 +291,18 @@ class Post:
 
     @property
     def video_view_count(self) -> Optional[int]:
-        """View count of the video, or None."""
+        """View count of the video, or None.
+
+        .. versionadded:: 4.2.6"""
         if self.is_video:
             return self._field('video_view_count')
         return None
 
     @property
     def video_duration(self) -> Optional[float]:
-        """Duration of the video in seconds, or None."""
+        """Duration of the video in seconds, or None.
+
+        .. versionadded:: 4.2.6"""
         if self.is_video:
             return self._field('video_duration')
         return None
@@ -444,6 +453,7 @@ class Profile:
         self._context = context
         self._has_public_story = None  # type: Optional[bool]
         self._node = node
+        self._has_full_metadata = False
         self._rhx_gis = None
         self._iphone_struct_ = None
         if 'iphone_struct' in node:
@@ -504,10 +514,11 @@ class Profile:
 
     def _obtain_metadata(self):
         try:
-            if not self._rhx_gis:
+            if not self._has_full_metadata:
                 metadata = self._context.get_json('{}/'.format(self.username), params={})
                 self._node = metadata['entry_data']['ProfilePage'][0]['graphql']['user']
-                self._rhx_gis = metadata['rhx_gis']
+                self._has_full_metadata = True
+                self._rhx_gis = metadata.get('rhx_gis')
         except (QueryReturnedNotFoundException, KeyError) as err:
             raise ProfileNotExistsException('Profile {} does not exist.'.format(self.username)) from err