Release of version 4.2.10

Closes #380.
Closes #394.

Co-Authored-By: Dāvis <davispuh@users.noreply.github.com>
Co-Authored-By: Alexander Graf <mail@agraf.me>
Co-Authored-By: Thammus <koch-kramer@web.de>

.github/ISSUE_TEMPLATE/bug_report.md

@@ -1,6 +1,7 @@
 ---
-name: Bug report
-about: Create a report to help us improve
+name: 🐛 Bug report
+about: Create a report to help us improve 🤕
+labels: bug
 
 ---
 
@@ -22,3 +23,5 @@ If applicable, add error messages and tracebacks to help explain your problem.
 
 **Additional context**
 Add any other context about the problem here.
+
+(please also see https://instaloader.github.io/contributing.html for how to report a bug)

.github/ISSUE_TEMPLATE/feature_request.md

@@ -1,6 +1,7 @@
 ---
-name: Feature request
-about: Suggest an idea for this project
+name: 🚀 Feature Suggestion
+about: Suggest an idea for this project 💡
+labels: "feature suggestion"
 
 ---
 
@@ -13,5 +14,11 @@ A clear and concise description of what you want to happen.
 **Describe alternatives you've considered**
 A clear and concise description of any alternative solutions or features you've considered.
 
+**If the feature request is accepted, would you be willing to submit a PR?**
+
+Yes / No _(Help can be provided if you need assistance submitting a PR)_
+
 **Additional context**
 Add any other context about the feature request here.
+
+(please also see https://instaloader.github.io/contributing.html for how to suggest a feature)

.github/ISSUE_TEMPLATE/question.md

@@ -0,0 +1,8 @@
+---
+name: ❓ Question
+about: If you have a question about how to use Instaloader that's not well covered by our documentation
+labels: question
+
+---
+
+Your question here...

.github/workflows/stale.yml

@@ -0,0 +1,24 @@
+name: Mark stale issues and pull requests
+
+on:
+  schedule:
+  - cron: "15 16 * * *"
+
+jobs:
+  stale:
+
+    runs-on: ubuntu-latest
+    
+    steps:
+    - uses: actions/stale@v1
+      with:
+        repo-token: ${{ secrets.GITHUB_TOKEN }}
+        stale-issue-message: 'There has been no activity on this issue for an extended period of time. This issue will be closed after further 14 days of inactivity.'
+        stale-pr-message: 'There has been no activity on this Pull Request for an extended period of time. This Pull Request will be closed after further 14 days of inactivity.'
+        stale-issue-label: 'stale'
+        stale-pr-label: 'stale'
+        exempt-issue-label: 'leave open'
+        exempt-pr-label: 'leave open'
+        days-before-stale: 90
+        days-before-close: 14
+        operations-per-run: 7

.travis.yml

@@ -4,7 +4,6 @@ python:
  - "3.5"
  - "3.6"
  - "3.7"
- - "3.8-dev"
 install:
  - pipenv sync --dev
 script:

docs/basic-usage.rst

@@ -89,6 +89,7 @@ Instaloader supports the following targets:
    Posts tagged with a given location; the location ID is the numerical ID
    Instagram labels a location with (e.g.
    \https://www.instagram.com/explore/locations/**362629379**/plymouth-naval-memorial/).
+   Requires :option:`--login`.
 
    .. versionadded:: 4.2
 
@@ -122,7 +123,7 @@ downloads the pictures and videos and their captions. You can specify
 
 - :option:`--geotags`
    **download geotags** of each post and save them as
-   Google Maps link,
+   Google Maps link (requires :option:`--login`),
 
 For a reference of all supported command line options, see
 :ref:`command-line-options`.

docs/cli-options.rst

@@ -61,8 +61,7 @@ What to Download of each Post
 
    **Download geotags** when available. Geotags are stored as a text file with
    the location's name and a Google Maps link. This requires an additional
-   request to the Instagram server for each picture, which is why it is disabled
-   by default.
+   request to the Instagram server for each picture. Requires :option:`--login`.
 
 .. option:: --comments, -C
 

docs/codesnippets/113_only_one_per_user.py

@@ -2,13 +2,13 @@ import instaloader
 
 L = instaloader.Instaloader()
 
-posts = L.get_hashtag_posts('milfgarden')
+posts = L.get_hashtag_posts('urbanphotography')
 
 users = set()
 
 for post in posts:
     if not post.owner_profile in users:
-        L.download_post(post, '#milfgarden')
+        L.download_post(post, '#urbanphotography')
         users.add(post.owner_profile)
     else:
         print("{} from {} skipped.".format(post, post.owner_profile))

docs/codesnippets/121_since_until.py

@@ -5,7 +5,7 @@ import instaloader
 
 L = instaloader.Instaloader()
 
-posts = L.get_hashtag_posts('milfgarden')
+posts = L.get_hashtag_posts('urbanphotography')
 # or
 # posts = instaloader.Profile.from_username(L.context, PROFILE).get_posts()
 
@@ -14,4 +14,4 @@ UNTIL = datetime(2015, 3, 1)
 
 for post in takewhile(lambda p: p.date > UNTIL, dropwhile(lambda p: p.date > SINCE, posts)):
     print(post.date)
-    L.download_post(post, '#milfgarden')
+    L.download_post(post, '#urbanphotography')

docs/codesnippets/194_top_x_of_user.py

@@ -9,7 +9,9 @@ X_percentage = 10    # percentage of posts that should be downloaded
 L = Instaloader()
 
 profile = Profile.from_username(L.context, PROFILE)
-posts_sorted_by_likes = sorted(profile.get_posts(), key = lambda p: p.likes + p.comments)
+posts_sorted_by_likes = sorted(profile.get_posts(),
+                               key=lambda p: p.likes + p.comments,
+                               reverse=True)
 
 for post in islice(posts_sorted_by_likes, ceil(profile.mediacount * X_percentage / 100)):
     L.download_post(post, PROFILE)

docs/conf.py

@@ -47,7 +47,7 @@ autodoc_default_flags = ['show-inheritance', 'members', 'undoc-members']
 autodoc_member_order = 'bysource'
 
 intersphinx_mapping = {'python': ('https://docs.python.org/3', None),
-                       'requests': ('http://docs.python-requests.org/en/master', None)}
+                       'requests': ('https://requests.kennethreitz.org/en/master/', None)}
 
 extlinks = {'issue': ('https://github.com/instaloader/instaloader/issues/%s', 'issue #')}
 

docs/contributing.rst

@@ -15,10 +15,8 @@ Reporting Bugs
 --------------
 
 If you encounter a bug, do not hesitate to report it in our
-`Issue Tracker <https://github.com/instaloader/instaloader/issues>`__. Since
-Instaloader is actively developed, the majority of bugs is fixed within only
-**4 days** after being reported. When reporting a problem, please keep the
-following in mind:
+`Issue Tracker <https://github.com/instaloader/instaloader/issues>`__. When
+reporting a problem, please keep the following in mind:
 
 - Ensure you **use the latest version** of Instaloader. The currently-installed
   version can be found out with ``instaloader --version``.
@@ -66,6 +64,16 @@ few things to consider:
 Suggesting Features
 -------------------
 
+.. goal-start
+
+Instaloader's goal is to mimick the browser's behavior to access the data that
+is available through the Instagram web interface, pack this data into complete
+and easily-(re)usable python objects, and provide a user interface for the most
+common downloading and metadata collection tasks, without changing any of the
+online data.
+
+.. goal-end
+
 If you have an idea of how Instaloader should be enhanced, but do not want to
 implement the feature yourself, feel free to open a ticket in our 
 `Issue Tracker <https://github.com/instaloader/instaloader/issues>`__.
@@ -82,6 +90,9 @@ Please consider the following:
   invoke the new function? Which problem would it solve? If new information is
   obtained, how would it be further processed?
 
+- If not obvious, briefly motivate how your suggested feature **conforms with
+  Instaloader's project goal**.
+
 - **Be patient**. Naturally, bugs and pull requests have a higher priority than
   feature suggestions. Keep in mind that this is a free software project, and
   unfortunately we only have limited time to work on it.

instaloader/__init__.py

@@ -1,7 +1,7 @@
 """Download pictures (or videos) along with their captions and other metadata from Instagram."""
 
 
-__version__ = '4.2.8'
+__version__ = '4.2.10'
 
 
 try:

instaloader/__main__.py

@@ -98,6 +98,9 @@ def _main(instaloader: Instaloader, targetlist: List[str],
             else:
                 instaloader.interactive_login(username)
         instaloader.context.log("Logged in as %s." % username)
+    # since 4.2.9 login is required for geotags
+    if instaloader.download_geotags and not instaloader.context.is_logged_in:
+        instaloader.context.error("Warning: Use --login to download geotags of posts.")
     # Try block for KeyboardInterrupt (save session on ^C)
     profiles = set()
     anonymous_retry_profiles = set()
@@ -223,7 +226,8 @@ def main():
                            help="Download all followees of profile. Requires --login. "
                                 "Consider using :feed rather than @yourself.")
     g_targets.add_argument('_hashtag', nargs='*', metavar='"#hashtag"', help="Download #hashtag.")
-    g_targets.add_argument('_location', nargs='*', metavar='%location_id', help="Download %%location_id.")
+    g_targets.add_argument('_location', nargs='*', metavar='%location_id',
+                           help="Download %%location_id. Requires --login.")
     g_targets.add_argument('_feed', nargs='*', metavar=":feed",
                            help="Download pictures from your feed. Requires --login.")
     g_targets.add_argument('_stories', nargs='*', metavar=":stories",
@@ -258,7 +262,7 @@ def main():
                         help='Download geotags when available. Geotags are stored as a '
                              'text file with the location\'s name and a Google Maps link. '
                              'This requires an additional request to the Instagram '
-                             'server for each picture, which is why it is disabled by default.')
+                             'server for each picture. Requires --login.')
     g_post.add_argument('-C', '--comments', action='store_true',
                         help='Download and update comments for each post. '
                              'This requires an additional request to the Instagram '

instaloader/instaloader.py

@@ -47,8 +47,6 @@ def _requires_login(func: Callable) -> Callable:
         if not instaloader.context.is_logged_in:
             raise LoginRequiredException("--login=USERNAME required.")
         return func(instaloader, *args, **kwargs)
-    docstring_text = ":raises LoginRequiredException: If called without being logged in.\n"
-    call.__doc__ = call.__doc__ + docstring_text if call.__doc__ is not None else docstring_text
     return call
 
 
@@ -246,6 +244,7 @@ class Instaloader:
             filename += '.json.xz'
         else:
             filename += '.json'
+        os.makedirs(os.path.dirname(filename), exist_ok=True)
         save_structure_to_file(structure, filename)
         if isinstance(structure, (Post, StoryItem)):
             # log 'json ' message when saving Post or StoryItem
@@ -256,7 +255,8 @@ class Instaloader:
             return {'id': comment.id,
                     'created_at': int(comment.created_at_utc.replace(tzinfo=timezone.utc).timestamp()),
                     'text': comment.text,
-                    'owner': comment.owner._asdict()}
+                    'owner': comment.owner._asdict(),
+                    'likes_count': comment.likes_count}
 
         def _postcomment_asdict(comment):
             return {**_postcommentanswer_asdict(comment),
@@ -273,11 +273,13 @@ class Instaloader:
             for x, y in zip(comments_list[:-1], comments_list[1:]):
                 if x['id'] != y['id']:
                     unique_comments_list.append(y)
-                elif combine_answers:
-                    combined_answers = unique_comments_list[-1].get('answers') or list()
-                    if 'answers' in y:
-                        combined_answers.extend(y['answers'])
-                    unique_comments_list[-1]['answers'] = get_unique_comments(combined_answers)
+                else:
+                    unique_comments_list[-1]['likes_count'] = y.get('likes_count')
+                    if combine_answers:
+                        combined_answers = unique_comments_list[-1].get('answers') or list()
+                        if 'answers' in y:
+                            combined_answers.extend(y['answers'])
+                        unique_comments_list[-1]['answers'] = get_unique_comments(combined_answers)
             return unique_comments_list
         filename += '_comments.json'
         try:
@@ -374,6 +376,7 @@ class Instaloader:
                                           os.path.getsize(filename) >= int(content_length))):
             self.context.log(filename + ' already exists')
             return None
+        os.makedirs(os.path.dirname(filename), exist_ok=True)
         self.context.write_raw(profile_pic_bytes if profile_pic_bytes else profile_pic_response, filename)
         if date_object:
             os.utime(filename, (datetime.now().timestamp(), date_object.timestamp()))
@@ -384,6 +387,7 @@ class Instaloader:
         """Saves internally stored :class:`requests.Session` object.
 
         :param filename: Filename, or None to use default filename.
+        :raises LoginRequiredException: If called without being logged in.
         """
         if filename is None:
             assert self.context.username is not None
@@ -508,6 +512,7 @@ class Instaloader:
         To use this, one needs to be logged in
 
         :param userids: List of user IDs to be processed in terms of downloading their stories, or None.
+        :raises LoginRequiredException: If called without being logged in.
         """
 
         if not userids:
@@ -544,6 +549,7 @@ class Instaloader:
         :param filename_target: Replacement for {target} in dirname_pattern and filename_pattern
                or None if profile name should be used instead
         :param storyitem_filter: function(storyitem), which returns True if given StoryItem should be downloaded
+        :raises LoginRequiredException: If called without being logged in.
         """
 
         if not userids:
@@ -603,6 +609,7 @@ class Instaloader:
         .. versionadded:: 4.1
 
         :param user: ID or Profile of the user whose highlights should get fetched.
+        :raises LoginRequiredException: If called without being logged in.
         """
 
         userid = user if isinstance(user, int) else user.userid
@@ -632,6 +639,7 @@ class Instaloader:
         :param filename_target: Replacement for {target} in dirname_pattern and filename_pattern
                or None if profile name and the highlights' titles should be used instead
         :param storyitem_filter: function(storyitem), which returns True if given StoryItem should be downloaded
+        :raises LoginRequiredException: If called without being logged in.
         """
         for user_highlight in self.get_highlights(user):
             name = user_highlight.owner_username
@@ -657,6 +665,7 @@ class Instaloader:
         """Get Posts of the user's feed.
 
         :return: Iterator over Posts of the user's feed.
+        :raises LoginRequiredException: If called without being logged in.
         """
 
         data = self.context.graphql_query("d6f4427fbe92d846298cf93df0b937d3", {})["data"]
@@ -690,6 +699,7 @@ class Instaloader:
         :param max_count: Maximum count of pictures to download
         :param fast_update: If true, abort when first already-downloaded picture is encountered
         :param post_filter: function(post), which returns True if given picture should be downloaded
+        :raises LoginRequiredException: If called without being logged in.
         """
         self.context.log("Retrieving pictures from your feed...")
         count = 1
@@ -715,6 +725,7 @@ class Instaloader:
         :param max_count: Maximum count of pictures to download
         :param fast_update: If true, abort when first already-downloaded picture is encountered
         :param post_filter: function(post), which returns True if given picture should be downloaded
+        :raises LoginRequiredException: If called without being logged in.
         """
         self.context.log("Retrieving saved posts...")
         count = 1
@@ -732,12 +743,17 @@ class Instaloader:
                 if fast_update and not downloaded:
                     break
 
+    @_requires_login
     def get_location_posts(self, location: str) -> Iterator[Post]:
         """Get Posts which are listed by Instagram for a given Location.
 
         :return:  Iterator over Posts of a location's posts
+        :raises LoginRequiredException: If called without being logged in.
 
         .. versionadded:: 4.2
+
+        .. versionchanged:: 4.2.9
+           Require being logged in (as required by Instagram)
         """
         has_next_page = True
         end_cursor = None
@@ -752,6 +768,7 @@ class Instaloader:
             has_next_page = location_data['page_info']['has_next_page']
             end_cursor = location_data['page_info']['end_cursor']
 
+    @_requires_login
     def download_location(self, location: str,
                           max_count: Optional[int] = None,
                           post_filter: Optional[Callable[[Post], bool]] = None,
@@ -767,8 +784,12 @@ class Instaloader:
         :param max_count: Maximum count of pictures to download
         :param post_filter: function(post), which returns True if given picture should be downloaded
         :param fast_update: If true, abort when first already-downloaded picture is encountered
+        :raises LoginRequiredException: If called without being logged in.
 
         .. versionadded:: 4.2
+
+        .. versionchanged:: 4.2.9
+           Require being logged in (as required by Instagram)
         """
         self.context.log("Retrieving pictures for location {}...".format(location))
         count = 1
@@ -790,6 +811,7 @@ class Instaloader:
         """Get Posts which are worthy of exploring suggested by Instagram.
 
         :return: Iterator over Posts of the user's suggested posts.
+        :raises LoginRequiredException: If called without being logged in.
         """
         data = self.context.get_json('explore/', {})
         yield from (Post(self.context, node)

instaloader/instaloadercontext.py

@@ -212,7 +212,7 @@ class InstaloaderContext:
             raise TwoFactorAuthRequiredException("Login error: two-factor authentication required.")
         if resp_json.get('checkpoint_url'):
             raise ConnectionException("Login: Checkpoint required. Point your browser to "
-                                      "https://www.instagram.com{}, "
+                                      "https://www.instagram.com{} - "
                                       "follow the instructions, then retry.".format(resp_json.get('checkpoint_url')))
         if resp_json['status'] != 'ok':
             if 'message' in resp_json:
@@ -358,6 +358,9 @@ class InstaloaderContext:
             while resp.is_redirect:
                 redirect_url = resp.headers['location']
                 self.log('\nHTTP redirect from https://{0}/{1} to {2}'.format(host, path, redirect_url))
+                if redirect_url.startswith('https://www.instagram.com/accounts/login'):
+                    # alternate rate limit exceeded behavior
+                    raise TooManyRequestsException("429 Too Many Requests: redirected to login")
                 if redirect_url.startswith('https://{}/'.format(host)):
                     resp = sess.get(redirect_url if redirect_url.endswith('/') else redirect_url + '/',
                                     params=params, allow_redirects=False)
@@ -376,7 +379,18 @@ class InstaloaderContext:
                 match = re.search(r'window\._sharedData = (.*);</script>', resp.text)
                 if match is None:
                     raise ConnectionException("Could not find \"window._sharedData\" in html response.")
-                return json.loads(match.group(1))
+                resp_json = json.loads(match.group(1))
+                entry_data = resp_json.get('entry_data')
+                post_or_profile_page = list(entry_data.values())[0] if entry_data is not None else None
+                if post_or_profile_page is None:
+                    raise ConnectionException("\"window._sharedData\" does not contain required keys.")
+                # If GraphQL data is missing in `window._sharedData`, search for it in `__additionalDataLoaded`.
+                if 'graphql' not in post_or_profile_page[0]:
+                    match = re.search(r'window\.__additionalDataLoaded\([^{]+{"graphql":({.*})}\);</script>',
+                                      resp.text)
+                    if match is not None:
+                        post_or_profile_page[0]['graphql'] = json.loads(match.group(1))
+                return resp_json
             else:
                 resp_json = resp.json()
             if 'status' in resp_json and resp_json['status'] != "ok":

instaloader/structures.py

@@ -17,11 +17,12 @@ PostSidecarNode.is_video.__doc__ = "Whether this node is a video."
 PostSidecarNode.display_url.__doc__ = "URL of image or video thumbnail."
 PostSidecarNode.video_url.__doc__ = "URL of video or None."
 
-PostCommentAnswer = namedtuple('PostCommentAnswer', ['id', 'created_at_utc', 'text', 'owner'])
+PostCommentAnswer = namedtuple('PostCommentAnswer', ['id', 'created_at_utc', 'text', 'owner', 'likes_count'])
 PostCommentAnswer.id.__doc__ = "ID number of comment."
 PostCommentAnswer.created_at_utc.__doc__ = ":class:`~datetime.datetime` when comment was created (UTC)."
 PostCommentAnswer.text.__doc__ = "Comment text."
 PostCommentAnswer.owner.__doc__ = "Owner :class:`Profile` of the comment."
+PostCommentAnswer.likes_count.__doc__ = "Number of likes on comment."
 
 PostComment = namedtuple('PostComment', (*PostCommentAnswer._fields, 'answers')) # type: ignore
 for field in PostCommentAnswer._fields:
@@ -340,7 +341,8 @@ class Post:
             return PostCommentAnswer(id=int(node['id']),
                                      created_at_utc=datetime.utcfromtimestamp(node['created_at']),
                                      text=node['text'],
-                                     owner=Profile(self._context, node['owner']))
+                                     owner=Profile(self._context, node['owner']),
+                                     likes_count=node['edge_liked_by']['count'])
 
         def _postcommentanswers(node):
             if 'edge_threaded_comments' not in node:
@@ -408,10 +410,17 @@ class Post:
 
     @property
     def location(self) -> Optional[PostLocation]:
-        """If the Post has a location, returns PostLocation namedtuple with fields 'id', 'lat' and 'lng' and 'name'."""
+        """
+        If the Post has a location, returns PostLocation namedtuple with fields 'id', 'lat' and 'lng' and 'name'.
+
+        .. versionchanged:: 4.2.9
+           Require being logged in (as required by Instagram), return None if not logged-in.
+        """
         loc = self._field("location")
         if self._location or not loc:
             return self._location
+        if not self._context.is_logged_in:
+            return None
         location_id = int(loc['id'])
         if any(k not in loc for k in ('name', 'slug', 'has_public_page', 'lat', 'lng')):
             loc = self._context.get_json("explore/locations/{0}/".format(location_id),

test/instaloader_unittests.py

@@ -77,15 +77,6 @@ class TestInstaloaderAnonymously(unittest.TestCase):
             if count == PAGING_MAX_COUNT:
                 break
 
-    def test_location_download(self):
-        self.L.download_location(LOCATION, NORMAL_MAX_COUNT)
-
-    def test_location_paging(self):
-        for count, post in enumerate(self.L.get_location_posts(LOCATION)):
-            print(post)
-            if count == PAGING_MAX_COUNT:
-                break
-
     def test_get_id_by_username(self):
         self.assertEqual(PUBLIC_PROFILE_ID,
                          instaloader.Profile.from_username(self.L.context, PUBLIC_PROFILE).userid)
@@ -193,6 +184,15 @@ class TestInstaloaderLoggedIn(TestInstaloaderAnonymously):
             if count == PAGING_MAX_COUNT:
                 break
 
+    def test_location_download(self):
+        self.L.download_location(LOCATION, NORMAL_MAX_COUNT)
+
+    def test_location_paging(self):
+        for count, post in enumerate(self.L.get_location_posts(LOCATION)):
+            print(post)
+            if count == PAGING_MAX_COUNT:
+                break
+
 
 if __name__ == '__main__':
     unittest.main()