Release of version 4.0.8

This reverts commit 8423e3bd00. The channel did
not have as much activity as expected and an IRC channel does not work if there
are not at least a few people in it.

.travis.yml

@@ -8,7 +8,7 @@ install:
  - pip install pylint requests
  - pip install -r docs/requirements.txt
 script:
- - python3 -m pylint -r n -d bad-whitespace,missing-docstring,too-many-arguments,locally-disabled,line-too-long,too-many-public-methods,too-many-lines,too-many-instance-attributes,too-many-locals,too-many-branches,too-many-statements,inconsistent-return-statements,invalid-name,wildcard-import,unused-wildcard-import instaloader
+ - python3 -m pylint -r n -d bad-whitespace,missing-docstring,too-many-arguments,locally-disabled,line-too-long,too-many-public-methods,too-many-lines,too-many-instance-attributes,too-many-locals,too-many-branches,too-many-statements,inconsistent-return-statements,invalid-name,wildcard-import,unused-wildcard-import,no-else-return instaloader
  - make -C docs html SPHINXOPTS="-W -n"
 deploy:
   - provider: pypi

AUTHORS.md

@@ -0,0 +1,8 @@
+Authors
+=======
+
+Instaloader is written by
+
+- Alexander Graf (@aandergr)
+- André Koch-Kramer (@Thammus)
+- Lars Lindqvist (@e5150)

README.rst

@@ -75,6 +75,24 @@ Disclaimer
 Instaloader is in no way affiliated with, authorized, maintained or endorsed by Instagram or any of its affiliates or
 subsidiaries. This is an independent and unofficial project. Use at your own risk.
 
+Instaloader is licensed under an MIT license. Refer to ``LICENSE`` file for more information.
+
 .. disclaimer-end
 
-Instaloader is licensed under an MIT license. Refer to ``LICENSE`` file for more information.
+Contributing
+------------
+
+As an open source project, Instaloader heavily depends on the contributions from
+its community. See
+`contributing <https://instaloader.github.io/contributing.html>`__
+for how you may help Instaloader to become an even greater tool.
+
+It is a pleasure for us to share our Instaloader to the world, and we are proud
+to have attracted such an active and motivating community, with so many users
+who share their suggestions and ideas with us. Buying a community-sponsored beer
+or coffee from time to time is very likely to further raise our passion for the
+development of Instaloader.
+
+| For Donations, we provide a PayPal.Me link and a Bitcoin address.
+|  PayPal: `PayPal.me/aandergr <https://www.paypal.me/aandergr>`__
+|  BTC: 1Nst4LoadeYzrKjJ1DX9CpbLXBYE9RKLwY

docs/_static/style.css

@@ -10,3 +10,11 @@ dd {
 dl.option dd {
     padding-left: 12px;
 }
+
+a {
+    color: #008d06;
+}
+
+a:hover {
+    color: #f48400;
+}

docs/contributing.rst

@@ -17,7 +17,7 @@ Reporting Bugs
 If you encounter a bug, do not hesitate to report it in our
 `Issue Tracker <https://github.com/instaloader/instaloader/issues>`__. Since
 Instaloader is actively developed, the majority of bugs is fixed within only
-**4 days** after being reported. When reporting a problem, please keep the
+**3 days** after being reported. When reporting a problem, please keep the
 following in mind:
 
 - Ensure you **use the latest version** of Instaloader. The currently-installed
@@ -95,8 +95,9 @@ who share their suggestions and ideas with us. Buying a community-sponsored beer
 or coffee from time to time is very likely to further raise our passion for the
 development of Instaloader.
 
-We have a PayPal.Me link for donations: `PayPal.me/aandergr
-<https://www.paypal.me/aandergr>`__
+| For donations, we provide a PayPal.Me link and a Bitcoin address.
+|  `PayPal.me/aandergr <https://www.paypal.me/aandergr>`__
+|  BTC: 1Nst4LoadeYzrKjJ1DX9CpbLXBYE9RKLwY
 
 .. donations-end
 

instaloader/__init__.py

@@ -1,7 +1,7 @@
 """Download pictures (or videos) along with their captions and other metadata from Instagram."""
 
 
-__version__ = '4.0.5'
+__version__ = '4.0.8'
 
 
 try:

instaloader/__main__.py

@@ -90,6 +90,7 @@ def _main(instaloader: Instaloader, targetlist: List[str],
         instaloader.context.log("Logged in as %s." % username)
     # Try block for KeyboardInterrupt (save session on ^C)
     profiles = set()
+    anonymous_retry_profiles = set()
     try:
         # Generate set of profiles, already downloading non-profile targets
         for target in targetlist:
@@ -121,8 +122,9 @@ def _main(instaloader: Instaloader, targetlist: List[str],
                 if target[0] == '@':
                     instaloader.context.log("Retrieving followees of %s..." % target[1:])
                     profile = Profile.from_username(instaloader.context, target[1:])
-                    followees = profile.get_followees()
-                    profiles.update([followee.username for followee in followees])
+                    for followee in profile.get_followees():
+                        instaloader.save_profile_id(followee)
+                        profiles.add(followee)
                 elif target[0] == '#':
                     instaloader.download_hashtag(hashtag=target[1:], max_count=max_count, fast_update=fast_update,
                                                  post_filter=post_filter)
@@ -135,26 +137,50 @@ def _main(instaloader: Instaloader, targetlist: List[str],
                     instaloader.download_saved_posts(fast_update=fast_update, max_count=max_count,
                                                      post_filter=post_filter)
                 else:
-                    profiles.add(target)
+                    try:
+                        profile = instaloader.check_profile_id(target)
+                        if instaloader.context.is_logged_in and profile.has_blocked_viewer:
+                            if not stories_only and not profile.is_private:
+                                raise ProfileNotExistsException("{} blocked you; But she's public, "
+                                                                "so we just download her anonymously.".format(target))
+                            else:
+                                instaloader.context.error("{} blocked you.".format(target))
+                        else:
+                            profiles.add(profile)
+                    except ProfileNotExistsException as err:
+                        # Not only our profile.has_blocked_viewer condition raises ProfileNotExistsException,
+                        # check_profile_id() also does, since access to blocked profile may be responded with 404.
+                        if instaloader.context.is_logged_in and not stories_only:
+                            instaloader.context.log(err)
+                            instaloader.context.log("Trying again anonymously, helps in case you are just blocked.")
+                            with instaloader.anonymous_copy() as anonymous_loader:
+                                with instaloader.context.error_catcher():
+                                    anonymous_retry_profiles.add(anonymous_loader.check_profile_id(target))
+                                    instaloader.context.log("Looks good.")
+                        else:
+                            raise
         if len(profiles) > 1:
-            instaloader.context.log("Downloading {} profiles: {}".format(len(profiles), ' '.join(profiles)))
-        # Iterate through profiles list and download them
-        for target in profiles:
-            with instaloader.context.error_catcher(target):
-                try:
-                    instaloader.download_profile(target, profile_pic, profile_pic_only, fast_update,
-                                                 stories, stories_only, post_filter=post_filter,
-                                                 storyitem_filter=storyitem_filter)
-                except ProfileNotExistsException as err:
-                    if instaloader.context.is_logged_in:
-                        instaloader.context.log(err)
-                        instaloader.context.log("Trying again anonymously, helps in case you are just blocked.")
-                        with instaloader.anonymous_copy() as anonymous_loader:
-                            with instaloader.context.error_catcher():
-                                anonymous_loader.download_profile(target, profile_pic, profile_pic_only,
-                                                                  fast_update, post_filter=post_filter)
-                    else:
-                        raise
+            instaloader.context.log("Downloading {} profiles: {}".format(len(profiles),
+                                                                         ' '.join([p.username for p in profiles])))
+        if not stories_only:
+            # Iterate through profiles list and download them
+            for target in profiles:
+                with instaloader.context.error_catcher(target):
+                    instaloader.download_profile(target, profile_pic, profile_pic_only,
+                                                 fast_update, post_filter=post_filter)
+            if anonymous_retry_profiles:
+                instaloader.context.log("Downloading anonymously: {}"
+                                        .format(' '.join([p.username for p in anonymous_retry_profiles])))
+                with instaloader.anonymous_copy() as anonymous_loader:
+                    for target in anonymous_retry_profiles:
+                        with instaloader.context.error_catcher(target):
+                            anonymous_loader.download_profile(target, profile_pic, profile_pic_only,
+                                                              fast_update, post_filter=post_filter)
+        if stories or stories_only:
+            with instaloader.context.error_catcher("Download stories"):
+                instaloader.context.log("Downloading stories")
+                instaloader.download_stories(userids=list(profiles), fast_update=fast_update,
+                                             filename_target=None, storyitem_filter=storyitem_filter)
     except KeyboardInterrupt:
         print("\nInterrupted by user.", file=sys.stderr)
     # Save session if it is useful

instaloader/instaloader.py

@@ -11,7 +11,7 @@ from contextlib import contextmanager, suppress
 from datetime import datetime, timezone
 from functools import wraps
 from io import BytesIO
-from typing import Callable, Iterator, List, Optional, Any
+from typing import Any, Callable, Iterator, List, Optional, Union
 
 from .exceptions import *
 from .instaloadercontext import InstaloaderContext
@@ -49,11 +49,11 @@ class _ArbitraryItemFormatter(string.Formatter):
     def __init__(self, item: Any):
         self._item = item
 
-    def get_field(self, field_name, args, kwargs):
+    def get_value(self, key, args, kwargs):
         """Override to substitute {ATTRIBUTE} by attributes of our _item."""
-        if hasattr(self._item, field_name):
-            return self._item.__getattribute__(field_name), None
-        return super().get_field(field_name, args, kwargs)
+        if hasattr(self._item, key):
+            return getattr(self._item, key)
+        return super().get_value(key, args, kwargs)
 
     def format_field(self, value, format_spec):
         """Override :meth:`string.Formatter.format_field` to have our
@@ -396,37 +396,45 @@ class Instaloader:
         :param userids: List of user IDs to be processed in terms of downloading their stories, or None.
         """
 
-        if userids is None:
+        if not userids:
             data = self.context.graphql_query("d15efd8c0c5b23f0ef71f18bf363c704",
                                               {"only_stories": True})["data"]["user"]
             if data is None:
                 raise BadResponseException('Bad stories reel JSON.')
             userids = list(edge["node"]["id"] for edge in data["feed_reels_tray"]["edge_reels_tray_to_reel"]["edges"])
 
-        stories = self.context.graphql_query("bf41e22b1c4ba4c9f31b844ebb7d9056",
-                                             {"reel_ids": userids, "precomposed_overlay": False})["data"]
+        def _userid_chunks():
+            userids_per_query = 100
+            for i in range(0, len(userids), userids_per_query):
+                yield userids[i:i + userids_per_query]
 
-        yield from (Story(self.context, media) for media in stories['reels_media'])
+        for userid_chunk in _userid_chunks():
+            stories = self.context.graphql_query("bf41e22b1c4ba4c9f31b844ebb7d9056",
+                                                 {"reel_ids": userid_chunk, "precomposed_overlay": False})["data"]
+            yield from (Story(self.context, media) for media in stories['reels_media'])
 
     @_requires_login
     def download_stories(self,
-                         userids: Optional[List[int]] = None,
+                         userids: Optional[List[Union[int, Profile]]] = None,
                          fast_update: bool = False,
-                         filename_target: str = ':stories',
+                         filename_target: Optional[str] = ':stories',
                          storyitem_filter: Optional[Callable[[StoryItem], bool]] = None) -> None:
         """
         Download available stories from user followees or all stories of users whose ID are given.
         Does not mark stories as seen.
         To use this, one needs to be logged in
 
-        :param userids: List of user IDs to be processed in terms of downloading their stories
+        :param userids: List of user IDs or Profiles to be processed in terms of downloading their stories
         :param fast_update: If true, abort when first already-downloaded picture is encountered
         :param filename_target: Replacement for {target} in dirname_pattern and filename_pattern
+               or None if profile name should be used instead
         :param storyitem_filter: function(storyitem), which returns True if given StoryItem should be downloaded
         """
 
         if not userids:
             self.context.log("Retrieving all visible stories...")
+        else:
+            userids = [p if isinstance(p, int) else p.userid for p in userids]
 
         for user_story in self.get_stories(userids):
             name = user_story.owner_username
@@ -440,7 +448,7 @@ class Instaloader:
                 self.context.log("[%3i/%3i] " % (count, totalcount), end="", flush=True)
                 count += 1
                 with self.context.error_catcher('Download story from user {}'.format(name)):
-                    downloaded = self.download_storyitem(item, filename_target)
+                    downloaded = self.download_storyitem(item, filename_target if filename_target else name)
                     if fast_update and not downloaded:
                         break
 
@@ -611,6 +619,24 @@ class Instaloader:
                 if fast_update and not downloaded:
                     break
 
+    def _get_id_filename(self, profile_name: str) -> str:
+        if ((format_string_contains_key(self.dirname_pattern, 'profile') or
+             format_string_contains_key(self.dirname_pattern, 'target'))):
+            return '{0}/id'.format(self.dirname_pattern.format(profile=profile_name.lower(),
+                                                               target=profile_name.lower()))
+        else:
+            return '{0}/{1}_id'.format(self.dirname_pattern.format(), profile_name.lower())
+
+    def save_profile_id(self, profile: Profile):
+        """
+        Store ID of profile locally.
+        """
+        os.makedirs(self.dirname_pattern.format(profile=profile.username,
+                                                target=profile.username), exist_ok=True)
+        with open(self._get_id_filename(profile.username), 'w') as text_file:
+            text_file.write(str(profile.userid) + "\n")
+            self.context.log("Stored ID {0} for profile {1}.".format(profile.userid, profile.username))
+
     def check_profile_id(self, profile_name: str) -> Profile:
         """
         Consult locally stored ID of profile with given name, check whether ID matches and whether name
@@ -623,12 +649,7 @@ class Instaloader:
         with suppress(ProfileNotExistsException):
             profile = Profile.from_username(self.context, profile_name)
         profile_exists = profile is not None
-        if ((format_string_contains_key(self.dirname_pattern, 'profile') or
-             format_string_contains_key(self.dirname_pattern, 'target'))):
-            id_filename = '{0}/id'.format(self.dirname_pattern.format(profile=profile_name.lower(),
-                                                                      target=profile_name.lower()))
-        else:
-            id_filename = '{0}/{1}_id'.format(self.dirname_pattern.format(), profile_name.lower())
+        id_filename = self._get_id_filename(profile_name)
         try:
             with open(id_filename, 'rb') as id_file:
                 profile_id = int(id_file.read())
@@ -657,15 +678,11 @@ class Instaloader:
         except (FileNotFoundError, ValueError):
             pass
         if profile_exists:
-            os.makedirs(self.dirname_pattern.format(profile=profile_name.lower(),
-                                                    target=profile_name.lower()), exist_ok=True)
-            with open(id_filename, 'w') as text_file:
-                text_file.write(str(profile.userid) + "\n")
-                self.context.log("Stored ID {0} for profile {1}.".format(profile.userid, profile_name))
+            self.save_profile_id(profile)
             return profile
         raise ProfileNotExistsException("Profile {0} does not exist.".format(profile_name))
 
-    def download_profile(self, profile_name: str,
+    def download_profile(self, profile_name: Union[str, Profile],
                          profile_pic: bool = True, profile_pic_only: bool = False,
                          fast_update: bool = False,
                          download_stories: bool = False, download_stories_only: bool = False,
@@ -676,7 +693,10 @@ class Instaloader:
         # Get profile main page json
         # check if profile does exist or name has changed since last download
         # and update name and json data if necessary
-        profile = self.check_profile_id(profile_name.lower())
+        if isinstance(profile_name, str):
+            profile = self.check_profile_id(profile_name.lower())
+        else:
+            profile = profile_name
 
         profile_name = profile.username
 

instaloader/instaloadercontext.py

@@ -9,6 +9,7 @@ import textwrap
 import time
 import urllib.parse
 from contextlib import contextmanager
+from datetime import datetime, timedelta
 from typing import Any, Callable, Dict, Iterator, Optional
 
 import requests
@@ -74,10 +75,12 @@ class InstaloaderContext:
         username = self.username
         self._session = self.get_anonymous_session()
         self.username = None
-        yield self
-        self._session.close()
-        self.username = username
-        self._session = session
+        try:
+            yield self
+        finally:
+            self._session.close()
+            self.username = username
+            self._session = session
 
     @property
     def is_logged_in(self) -> bool:
@@ -175,6 +178,9 @@ class InstaloaderContext:
         :raises InvalidArgumentException: If the provided username does not exist.
         :raises BadCredentialsException: If the provided password is wrong.
         :raises ConnectionException: If connection to Instagram failed."""
+        import http.client
+        # pylint:disable=protected-access
+        http.client._MAXHEADERS = 200
         session = requests.Session()
         session.cookies.update({'sessionid': '', 'mid': '', 'ig_pr': '1',
                                 'ig_vw': '1920', 'csrftoken': '',
@@ -243,7 +249,8 @@ class InstaloaderContext:
         if is_graphql_query and not query_not_limited:
             waittime = graphql_query_waittime()
             if waittime > 0:
-                self.log('\nToo many queries in the last time. Need to wait {} seconds.'.format(waittime))
+                self.log('\nToo many queries in the last time. Need to wait {} seconds, until {:%H:%M}.'
+                         .format(waittime, datetime.now() + timedelta(seconds=waittime)))
                 time.sleep(waittime)
             if self.query_timestamps is not None:
                 self.query_timestamps.append(time.monotonic())
@@ -298,7 +305,8 @@ class InstaloaderContext:
                     if is_graphql_query:
                         waittime = graphql_query_waittime(untracked_queries=True)
                         if waittime > 0:
-                            self.log('The request will be retried in {} seconds.'.format(waittime))
+                            self.log('The request will be retried in {} seconds, at {:%H:%M}.'
+                                     .format(waittime, datetime.now() + timedelta(seconds=waittime)))
                             time.sleep(waittime)
                 self._sleep()
                 return self.get_json(path=path, params=params, host=host, session=sess, _attempt=_attempt + 1)

instaloader/structures.py

@@ -526,10 +526,12 @@ class Profile:
     @property
     def has_highlight_reels(self) -> bool:
         """
-        This becomes `True` if the :class:`Profile` has any stories currently available,
-        even if not viewable by the viewer.
+        .. deprecated:: 4.0.6
+           Always returns `True` since :issue:`153`.
+
+        Before broken, this indicated whether the :class:`Profile` had available stories.
         """
-        return self._iphone_struct['has_highlight_reels']
+        return True
 
     @property
     def has_public_story(self) -> bool:
@@ -550,6 +552,8 @@ class Profile:
     @property
     def has_viewable_story(self) -> bool:
         """
+        .. deprecated:: 4.0.6
+
         Some stories are private. This property determines if the :class:`Profile`
         has at least one story which can be viewed using the associated :class:`InstaloaderContext`,
         i.e. the viewer has privileges to view it.
@@ -609,6 +613,16 @@ class Profile:
                                                     self._rhx_gis,
                                                     self._metadata('edge_saved_media')))
 
+    def get_tagged_posts(self) -> Iterator[Post]:
+        """Retrieve all posts where a profile is tagged."""
+        self._obtain_metadata()
+        yield from (Post(self._context, node, self if int(node['owner']['id']) == self.userid else None) for node in
+                    self._context.graphql_node_list("e31a871f7301132ceaab56507a66bbb7",
+                                                    {'id': self.userid},
+                                                    'https://www.instagram.com/{0}/'.format(self.username),
+                                                    lambda d: d['data']['user']['edge_user_to_photos_of_you'],
+                                                    self._rhx_gis))
+
     def get_followers(self) -> Iterator['Profile']:
         """
         Retrieve list of followers of given profile.

test/instaloader_unittests.py

@@ -76,6 +76,11 @@ class TestInstaloaderAnonymously(unittest.TestCase):
             self.assertEqual(post, post2)
             break
 
+    def test_public_profile_tagged_paging(self):
+        for post in islice(instaloader.Profile.from_username(self.L.context, PUBLIC_PROFILE).get_tagged_posts(),
+                           PAGING_MAX_COUNT):
+            print(post)
+
 
 class TestInstaloaderLoggedIn(TestInstaloaderAnonymously):