Release of version 4.2.2

This fixes #235. The retry logic has been broken with commit
4bc0a94e12 in version 4.2.1.

LICENSE

@@ -1,6 +1,6 @@
 The MIT License (MIT)
 
-Copyright (c) 2016-2018 Alexander Graf and André Koch-Kramer.
+Copyright (c) 2016-2019 Alexander Graf and André Koch-Kramer.
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

docs/_templates/layout.html

@@ -51,6 +51,9 @@
         <li class="nav-item">
           <a target="_blank" class="nav-link" href="https://github.com/instaloader/instaloader">GitHub</a>
         </li>
+        <li class="nav-item d-sm-none d-lg-block">
+          <a target="_blank" class="nav-link" href="https://saythanks.io/to/aandergr">Say Thanks!</a>
+        </li>
       </ul>
       <form class="form-inline" action="search.html" method="get">
         <div class="input-group my-2 my-sm-0 ml-2">

docs/as-module.rst

@@ -198,6 +198,8 @@ Exceptions
 
 .. autoexception:: BadCredentialsException
 
+.. autoexception:: PostChangedException
+
 .. autoexception:: TooManyRequestsException
 
 ``InstaloaderContext`` (Low-level functions)

docs/conf.py

@@ -86,7 +86,7 @@ author = 'Alexander Graf and Andre Koch-Kramer'
 #
 # This is also used if you do content translation via gettext catalogs.
 # Usually you set "language" from the command line for these cases.
-language = None
+language = "en"
 
 # There are two options for replacing |today|: either, you set today to some
 # non-false value, then it is used:

instaloader/__init__.py

@@ -1,7 +1,7 @@
 """Download pictures (or videos) along with their captions and other metadata from Instagram."""
 
 
-__version__ = '4.2.1'
+__version__ = '4.2.2'
 
 
 try:

instaloader/exceptions.py

@@ -22,6 +22,10 @@ class ProfileNotExistsException(InstaloaderException):
 
 
 class ProfileHasNoPicsException(InstaloaderException):
+    """
+    .. deprecated:: 4.2.2
+       Not raised anymore.
+    """
     pass
 
 
@@ -53,5 +57,10 @@ class ConnectionException(InstaloaderException):
     pass
 
 
+class PostChangedException(InstaloaderException):
+    """.. versionadded:: 4.2.2"""
+    pass
+
+
 class TooManyRequestsException(ConnectionException):
     pass

instaloader/instaloader.py

@@ -15,6 +15,9 @@ from hashlib import md5
 from io import BytesIO
 from typing import Any, Callable, Iterator, List, Optional, Set, Union
 
+import requests
+import urllib3
+
 from .exceptions import *
 from .instaloadercontext import InstaloaderContext
 from .structures import Highlight, JsonExportable, Post, PostLocation, Profile, Story, StoryItem, save_structure_to_file, load_structure_from_file
@@ -47,6 +50,37 @@ def _requires_login(func: Callable) -> Callable:
     return call
 
 
+def _retry_on_connection_error(func: Callable) -> Callable:
+    """Decorator to retry the function max_connection_attemps number of times.
+
+    Herewith-decorated functions need an ``_attempt`` keyword argument.
+
+    This is to decorate functions that do network requests that may fail. Note that
+    :meth:`.get_json`, :meth:`.get_iphone_json`, :meth:`.graphql_query` and :meth:`.graphql_node_list` already have
+    their own logic for retrying, hence functions that only use these for network access must not be decorated with this
+    decorator."""
+    @wraps(func)
+    def call(instaloader, *args, **kwargs):
+        try:
+            return func(instaloader, *args, **kwargs)
+        except (urllib3.exceptions.HTTPError, requests.exceptions.RequestException, ConnectionException) as err:
+            error_string = "{}({}): {}".format(func.__name__, ', '.join([repr(arg) for arg in args]), err)
+            if (kwargs.get('_attempt') or 1) == instaloader.context.max_connection_attempts:
+                raise ConnectionException(error_string) from None
+            instaloader.context.error(error_string + " [retrying; skip with ^C]", repeat_at_end=False)
+            try:
+                if kwargs.get('_attempt'):
+                    kwargs['_attempt'] += 1
+                else:
+                    kwargs['_attempt'] = 2
+                instaloader.context.do_sleep()
+                return call(instaloader, *args, **kwargs)
+            except KeyboardInterrupt:
+                instaloader.context.error("[skipped by user]", repeat_at_end=False)
+                raise ConnectionException(error_string) from None
+    return call
+
+
 class _ArbitraryItemFormatter(string.Formatter):
     def __init__(self, item: Any):
         self._item = item
@@ -175,7 +209,9 @@ class Instaloader:
     def __exit__(self, *args):
         self.close()
 
-    def download_pic(self, filename: str, url: str, mtime: datetime, filename_suffix: Optional[str] = None) -> bool:
+    @_retry_on_connection_error
+    def download_pic(self, filename: str, url: str, mtime: datetime,
+                     filename_suffix: Optional[str] = None, _attempt: int = 1) -> bool:
         """Downloads and saves picture with given url under given directory with given timestamp.
         Returns true, if file was actually downloaded, i.e. updated."""
         urlmatch = re.search('\\.[a-z0-9]*\\?', url)
@@ -285,7 +321,8 @@ class Instaloader:
         os.utime(filename, (datetime.now().timestamp(), mtime.timestamp()))
         self.context.log('geo', end=' ', flush=True)
 
-    def download_profilepic(self, profile: Profile) -> None:
+    @_retry_on_connection_error
+    def download_profilepic(self, profile: Profile, _attempt: int = 1) -> None:
         """Downloads and saves profile pic."""
 
         def _epoch_to_string(epoch: datetime) -> str:
@@ -898,6 +935,11 @@ class Instaloader:
             with error_handler(profile.username):
                 profile_name = profile.username
 
+                # Download profile picture
+                if profile_pic:
+                    with self.context.error_catcher('Download profile picture of {}'.format(profile_name)):
+                        self.download_profilepic(profile)
+
                 # Save metadata as JSON if desired.
                 if self.save_metadata:
                     json_filename = '{0}/{1}_{2}'.format(self.dirname_pattern.format(profile=profile_name,
@@ -905,11 +947,6 @@ class Instaloader:
                                                          profile_name, profile.userid)
                     self.save_metadata_json(json_filename, profile)
 
-                # Download profile picture
-                if profile_pic:
-                    with self.context.error_catcher('Download profile picture of {}'.format(profile_name)):
-                        self.download_profilepic(profile)
-
                 # Catch some errors
                 if profile.is_private and (tagged or highlights or posts):
                     if not self.context.is_logged_in:
@@ -939,8 +976,22 @@ class Instaloader:
                             self.context.log('<skipped>')
                             continue
                         with self.context.error_catcher("Download {} of {}".format(post, profile_name)):
-                            downloaded = self.download_post(post, target=profile_name)
-                            if fast_update and not downloaded:
+                            # The PostChangedException gets raised if the Post's id/shortcode changed while obtaining
+                            # additional metadata. This is most likely the case if a HTTP redirect takes place while
+                            # resolving the shortcode URL.
+                            # The `post_changed` variable keeps the fast-update functionality alive: A Post which is
+                            # obained after a redirect has probably already been downloaded as a previous Post of the
+                            # same Profile.
+                            # Observed in issue #225: https://github.com/instaloader/instaloader/issues/225
+                            post_changed = False
+                            while True:
+                                try:
+                                    downloaded = self.download_post(post, target=profile_name)
+                                    break
+                                except PostChangedException:
+                                    post_changed = True
+                                    continue
+                            if fast_update and not downloaded and not post_changed:
                                 break
 
         if stories and profiles:

instaloader/instaloadercontext.py

@@ -14,7 +14,6 @@ from typing import Any, Callable, Dict, Iterator, Optional, Union
 
 import requests
 import requests.utils
-import urllib3
 
 from .exceptions import *
 
@@ -195,7 +194,7 @@ class InstaloaderContext:
         csrf_token = session.cookies.get_dict()['csrftoken']
         session.headers.update({'X-CSRFToken': csrf_token})
         # Not using self.get_json() here, because we need to access csrftoken cookie
-        self._sleep()
+        self.do_sleep()
         login = session.post('https://www.instagram.com/accounts/login/ajax/',
                              data={'password': passwd, 'username': user}, allow_redirects=True)
         try:
@@ -261,7 +260,7 @@ class InstaloaderContext:
         self.username = user
         self.two_factor_auth_pending = None
 
-    def _sleep(self):
+    def do_sleep(self):
         """Sleep a short time if self.sleep is set. Called before each request to instagram.com."""
         if self.sleep:
             time.sleep(min(random.expovariate(0.7), 5.0))
@@ -304,7 +303,7 @@ class InstaloaderContext:
                 self.query_timestamps = [time.monotonic()]
         sess = session if session else self._session
         try:
-            self._sleep()
+            self.do_sleep()
             resp = sess.get('https://{0}/{1}'.format(host, path), params=params, allow_redirects=False)
             while resp.is_redirect:
                 redirect_url = resp.headers['location']
@@ -355,7 +354,7 @@ class InstaloaderContext:
                             self.log('The request will be retried in {} seconds, at {:%H:%M}.'
                                      .format(waittime, datetime.now() + timedelta(seconds=waittime)))
                             time.sleep(waittime)
-                self._sleep()
+                self.do_sleep()
                 return self.get_json(path=path, params=params, host=host, session=sess, _attempt=_attempt + 1)
             except KeyboardInterrupt:
                 self.error("[skipped by user]", repeat_at_end=False)
@@ -440,15 +439,17 @@ class InstaloaderContext:
         :raises ConnectionException: When query repeatedly failed.
 
         .. versionadded:: 4.2.1"""
-        tempsession = copy_session(self._session)
-        tempsession.headers['User-Agent'] = 'Instagram 10.3.2 (iPhone7,2; iPhone OS 9_3_3; en_US; en-US; ' \
-                                            'scale=2.00; 750x1334) AppleWebKit/420+'
-        for header in ['Host', 'Origin', 'X-Instagram-AJAX', 'X-Requested-With']:
-            tempsession.headers.pop(header, None)
-        return self.get_json(path, params, 'i.instagram.com', tempsession)
+        with copy_session(self._session) as tempsession:
+            tempsession.headers['User-Agent'] = 'Instagram 10.3.2 (iPhone7,2; iPhone OS 9_3_3; en_US; en-US; ' \
+                                                'scale=2.00; 750x1334) AppleWebKit/420+'
+            for header in ['Host', 'Origin', 'X-Instagram-AJAX', 'X-Requested-With']:
+                tempsession.headers.pop(header, None)
+            return self.get_json(path, params, 'i.instagram.com', tempsession)
 
     def write_raw(self, resp: Union[bytes, requests.Response], filename: str) -> None:
-        """Write raw response data into a file."""
+        """Write raw response data into a file.
+
+        .. versionadded:: 4.2.1"""
         self.log(filename, end=' ', flush=True)
         with open(filename, 'wb') as file:
             if isinstance(resp, requests.Response):
@@ -461,32 +462,22 @@ class InstaloaderContext:
 
         :raises QueryReturnedNotFoundException: When the server responds with a 404.
         :raises QueryReturnedForbiddenException: When the server responds with a 403.
-        :raises ConnectionException: When download repeatedly failed."""
-        try:
-            with self.get_anonymous_session() as anonymous_session:
-                resp = anonymous_session.get(url, stream=True)
-            if resp.status_code == 200:
-                resp.raw.decode_content = True
-                return resp
-            else:
-                if resp.status_code == 403:
-                    # suspected invalid URL signature
-                    raise QueryReturnedForbiddenException("403 when accessing {}.".format(url))
-                if resp.status_code == 404:
-                    # 404 not worth retrying.
-                    raise QueryReturnedNotFoundException("404 when accessing {}.".format(url))
-                raise ConnectionException("HTTP error code {}.".format(resp.status_code))
-        except (urllib3.exceptions.HTTPError, requests.exceptions.RequestException, ConnectionException) as err:
-            error_string = "URL {}: {}".format(url, err)
-            if _attempt == self.max_connection_attempts:
-                raise ConnectionException(error_string) from err
-            self.error(error_string + " [retrying; skip with ^C]", repeat_at_end=False)
-            try:
-                self._sleep()
-                return self.get_raw(url, _attempt + 1)
-            except KeyboardInterrupt:
-                self.error("[skipped by user]", repeat_at_end=False)
-                raise ConnectionException(error_string) from err
+        :raises ConnectionException: When download failed.
+
+        .. versionadded:: 4.2.1"""
+        with self.get_anonymous_session() as anonymous_session:
+            resp = anonymous_session.get(url, stream=True)
+        if resp.status_code == 200:
+            resp.raw.decode_content = True
+            return resp
+        else:
+            if resp.status_code == 403:
+                # suspected invalid URL signature
+                raise QueryReturnedForbiddenException("403 when accessing {}.".format(url))
+            if resp.status_code == 404:
+                # 404 not worth retrying.
+                raise QueryReturnedNotFoundException("404 when accessing {}.".format(url))
+            raise ConnectionException("HTTP error code {}.".format(resp.status_code))
 
     def get_and_write_raw(self, url: str, filename: str) -> None:
         """Downloads and writes anonymously-requested raw data into a file.

instaloader/structures.py

@@ -128,6 +128,9 @@ class Post:
             pic_json = self._context.get_json("p/{0}/".format(self.shortcode), params={})
             self._full_metadata_dict = pic_json['entry_data']['PostPage'][0]['graphql']['shortcode_media']
             self._rhx_gis_str = pic_json['rhx_gis']
+            if self.shortcode != self._full_metadata_dict['shortcode']:
+                self._node.update(self._full_metadata_dict)
+                raise PostChangedException
 
     @property
     def _full_metadata(self) -> Dict[str, Any]:
@@ -396,28 +399,27 @@ class Profile:
     @classmethod
     def from_id(cls, context: InstaloaderContext, profile_id: int):
         """Create a Profile instance from a given userid. If possible, use :meth:`Profile.from_username`
-        or constructor directly rather than this method, since it does many requests.
+        or constructor directly rather than this method, since it requires more requests.
 
         :param context: :attr:`Instaloader.context`
         :param profile_id: userid
-        :raises: :class:`ProfileNotExistsException`, :class:`ProfileHasNoPicsException`
+        :raises: :class:`ProfileNotExistsException`
         """
         if profile_id in context.profile_id_cache:
             return context.profile_id_cache[profile_id]
-        data = context.graphql_query("472f257a40c653c64c666ce877d59d2b",
-                                     {'id': str(profile_id), 'first': 1},
+        data = context.graphql_query('7c16654f22c819fb63d1183034a5162f',
+                                     {'user_id': str(profile_id),
+                                      'include_chaining': False,
+                                      'include_reel': True,
+                                      'include_suggested_users': False,
+                                      'include_logged_out_extras': False,
+                                      'include_highlight_reels': False},
                                      rhx_gis=context.root_rhx_gis)['data']['user']
         if data:
-            data = data["edge_owner_to_timeline_media"]
+            profile = cls(context, data['reel']['owner'])
         else:
             raise ProfileNotExistsException("No profile found, the user may have blocked you (ID: " +
                                             str(profile_id) + ").")
-        if not data['edges']:
-            if data['count'] == 0:
-                raise ProfileHasNoPicsException("Profile with ID {0}: no pics found.".format(str(profile_id)))
-            else:
-                raise LoginRequiredException("Login required to determine username (ID: " + str(profile_id) + ").")
-        profile = Post(context, data['edges'][0]['node']).owner_profile
         context.profile_id_cache[profile_id] = profile
         return profile
 

test/instaloader_unittests.py

@@ -17,6 +17,9 @@ OWN_USERNAME = "aandergr"
 NORMAL_MAX_COUNT = 2
 PAGING_MAX_COUNT = 15
 PRIVATE_PROFILE = "aandergr"
+PRIVATE_PROFILE_ID = 1706625676
+EMPTY_PROFILE = "not_public"
+EMPTY_PROFILE_ID = 1928659031
 
 # Preserve query timestamps (rate control) between tests to not get rate limited
 instaloadercontext_query_timestamps = list()
@@ -83,10 +86,18 @@ class TestInstaloaderAnonymously(unittest.TestCase):
         self.assertEqual(PUBLIC_PROFILE_ID,
                          instaloader.Profile.from_username(self.L.context, PUBLIC_PROFILE).userid)
 
-    def test_get_username_by_id(self):
+    def test_get_username_by_id_private(self):
+        self.assertEqual(PRIVATE_PROFILE.lower(),
+                         instaloader.Profile.from_id(self.L.context, PRIVATE_PROFILE_ID).username)
+
+    def test_get_username_by_id_public(self):
         self.assertEqual(PUBLIC_PROFILE.lower(),
                          instaloader.Profile.from_id(self.L.context, PUBLIC_PROFILE_ID).username)
 
+    def test_get_username_by_id_empty(self):
+        self.assertEqual(EMPTY_PROFILE.lower(),
+                         instaloader.Profile.from_id(self.L.context, EMPTY_PROFILE_ID).username)
+
     def test_post_from_mediaid(self):
         for post in instaloader.Profile.from_username(self.L.context, PUBLIC_PROFILE).get_posts():
             post2 = instaloader.Post.from_mediaid(self.L.context, post.mediaid)
@@ -130,6 +141,9 @@ class TestInstaloaderLoggedIn(TestInstaloaderAnonymously):
     def test_private_profile_paging(self):
         self.post_paging_test(instaloader.Profile.from_username(self.L.context, PRIVATE_PROFILE).get_posts())
 
+    def test_profile_pic_download(self):
+        self.L.download_profile(PUBLIC_PROFILE, profile_pic_only=True)
+
     def test_feed_download(self):
         self.L.download_feed_posts(NORMAL_MAX_COUNT)
 
@@ -166,10 +180,6 @@ class TestInstaloaderLoggedIn(TestInstaloaderAnonymously):
         for f in profile.get_followers():
             print(f.username)
 
-    def test_get_username_by_id(self):
-        self.assertEqual(PUBLIC_PROFILE.lower(),
-                         instaloader.Profile.from_id(self.L.context, PUBLIC_PROFILE_ID).username)
-
     def test_get_likes(self):
         for post in instaloader.Profile.from_username(self.L.context, OWN_USERNAME).get_posts():
             for like in post.get_likes():