Release of version 4.10.3

Closes issue #2125 by using an iphone endpoint for posts that have more
than 50 comments.

.github/workflows/lint.yml

@@ -9,7 +9,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: ["3.8", "3.9", "3.10", "3.11"]
+        python-version: ["3.8", "3.9", "3.10"]
     steps:
       - name: Checkout Instaloader Repository
         uses: actions/checkout@v2

AUTHORS.md

@@ -6,4 +6,5 @@ Instaloader is written by
 - Alexander Graf (@aandergr)
 - André Koch-Kramer (@Thammus)
 - Lars Lindqvist (@e5150)
+- Eduardo M Kalinowksi (@ekalin)
 - ... and many more, see https://github.com/instaloader/instaloader/graphs/contributors

README.rst

@@ -124,6 +124,7 @@ Supporters
 .. current-sponsors-start
 
 | Instaloader is proudly sponsored by
+|  `@rocketapi-io <https://github.com/rocketapi-io>`__
 |  `@socialmethod <https://github.com/socialmethod>`__
 
 See `Alex' GitHub Sponsors <https://github.com/sponsors/aandergr>`__ page for

instaloader/__init__.py

@@ -1,7 +1,7 @@
 """Download pictures (or videos) along with their captions and other metadata from Instagram."""
 
 
-__version__ = '4.10b1'
+__version__ = '4.10.3'
 
 
 try:

instaloader/instaloader.py

@@ -991,7 +991,8 @@ class Instaloader:
                             max_count: Optional[int] = None,
                             total_count: Optional[int] = None,
                             owner_profile: Optional[Profile] = None,
-                            takewhile: Optional[Callable[[Post], bool]] = None) -> None:
+                            takewhile: Optional[Callable[[Post], bool]] = None,
+                            possibly_pinned: int = 0) -> None:
         """
         Download the Posts returned by given Post Iterator.
 
@@ -1003,6 +1004,9 @@ class Instaloader:
         .. versionchanged:: 4.8
            Add `takewhile` parameter.
 
+        .. versionchanged:: 4.10.3
+           Add `possibly_pinned` parameter.
+
         :param posts: Post Iterator to loop through.
         :param target: Target name.
         :param fast_update: :option:`--fast-update`.
@@ -1011,6 +1015,8 @@ class Instaloader:
         :param total_count: Total number of posts returned by given iterator.
         :param owner_profile: Associated profile, if any.
         :param takewhile: Expression evaluated for each post. Once it returns false, downloading stops.
+        :param possibly_pinned: Number of posts that might be pinned. These posts do not cause download
+               to stop even if they've already been downloaded.
         """
         displayed_count = (max_count if total_count is None or max_count is not None and max_count < total_count
                            else total_count)
@@ -1032,7 +1038,7 @@ class Instaloader:
         ) as (is_resuming, start_index):
             for number, post in enumerate(posts, start=start_index + 1):
                 should_stop = not takewhile(post)
-                if should_stop and post.is_pinned:
+                if should_stop and number <= possibly_pinned:
                     continue
                 if (max_count is not None and number > max_count) or should_stop:
                     break
@@ -1066,7 +1072,7 @@ class Instaloader:
                         except PostChangedException:
                             post_changed = True
                             continue
-                    if fast_update and not downloaded and not post_changed and not post.is_pinned:
+                    if fast_update and not downloaded and not post_changed and number > possibly_pinned:
                         # disengage fast_update for first post when resuming
                         if not is_resuming or number > 0:
                             break
@@ -1488,7 +1494,7 @@ class Instaloader:
                     posts_to_download = profile.get_posts()
                     self.posts_download_loop(posts_to_download, profile_name, fast_update, post_filter,
                                              total_count=profile.mediacount, owner_profile=profile,
-                                             takewhile=posts_takewhile)
+                                             takewhile=posts_takewhile, possibly_pinned=3)
                     if latest_stamps is not None and posts_to_download.first_item is not None:
                         latest_stamps.set_last_post_timestamp(profile_name,
                                                               posts_to_download.first_item.date_local)

instaloader/instaloadercontext.py

@@ -3,7 +3,6 @@ import json
 import os
 import pickle
 import random
-import re
 import shutil
 import sys
 import textwrap
@@ -256,18 +255,23 @@ class InstaloaderContext:
         # Override default timeout behavior.
         # Need to silence mypy bug for this. See: https://github.com/python/mypy/issues/2427
         session.request = partial(session.request, timeout=self.request_timeout) # type: ignore
-        csrf_json = self.get_json('accounts/login/', {}, session=session)
-        csrf_token = csrf_json['config']['csrf_token']
+
+        # Make a request to Instagram's root URL, which will set the session's csrftoken cookie
+        # Not using self.get_json() here, because we need to access the cookie
+        session.get('https://www.instagram.com/')
+        # Add session's csrftoken cookie to session headers
+        csrf_token = session.cookies.get_dict()['csrftoken']
         session.headers.update({'X-CSRFToken': csrf_token})
-        # Not using self.get_json() here, because we need to access csrftoken cookie
+
         self.do_sleep()
         # Workaround credits to pgrimaud.
         # See: https://github.com/pgrimaud/instagram-user-feed/commit/96ad4cf54d1ad331b337f325c73e664999a6d066
         enc_password = '#PWD_INSTAGRAM_BROWSER:0:{}:{}'.format(int(datetime.now().timestamp()), passwd)
-        login = session.post('https://www.instagram.com/accounts/login/ajax/',
+        login = session.post('https://www.instagram.com/api/v1/web/accounts/login/ajax/',
                              data={'enc_password': enc_password, 'username': user}, allow_redirects=True)
         try:
             resp_json = login.json()
+
         except json.decoder.JSONDecodeError as err:
             raise ConnectionException(
                 "Login error: JSON decode fail, {} - {}.".format(login.status_code, login.reason)
@@ -372,8 +376,11 @@ class InstaloaderContext:
             resp = sess.get('https://{0}/{1}'.format(host, path), params=params, allow_redirects=False)
             if resp.status_code in self.fatal_status_codes:
                 redirect = " redirect to {}".format(resp.headers['location']) if 'location' in resp.headers else ""
-                raise AbortDownloadException("Query to https://{}/{} responded with \"{} {}\"{}".format(
-                    host, path, resp.status_code, resp.reason, redirect
+                body = ""
+                if resp.headers['Content-Type'].startswith('application/json'):
+                    body = ': ' + resp.text[:500] + ('…' if len(resp.text) > 501 else '')
+                raise AbortDownloadException("Query to https://{}/{} responded with \"{} {}\"{}{}".format(
+                    host, path, resp.status_code, resp.reason, redirect, body
                 ))
             while resp.is_redirect:
                 redirect_url = resp.headers['location']
@@ -400,23 +407,6 @@ class InstaloaderContext:
                 raise TooManyRequestsException("429 Too Many Requests")
             if resp.status_code != 200:
                 raise ConnectionException("HTTP error code {}.".format(resp.status_code))
-            is_html_query = not is_graphql_query and not "__a" in params and host == "www.instagram.com"
-            if is_html_query:
-                match = re.search(r'window\._sharedData = (.*);</script>', resp.text)
-                if match is None:
-                    raise QueryReturnedNotFoundException("Could not find \"window._sharedData\" in html response.")
-                resp_json = json.loads(match.group(1))
-                entry_data = resp_json.get('entry_data')
-                post_or_profile_page = list(entry_data.values())[0] if entry_data is not None else None
-                if post_or_profile_page is None:
-                    raise ConnectionException("\"window._sharedData\" does not contain required keys.")
-                # If GraphQL data is missing in `window._sharedData`, search for it in `__additionalDataLoaded`.
-                if 'graphql' not in post_or_profile_page[0]:
-                    match = re.search(r'window\.__additionalDataLoaded\(.*?({.*"graphql":.*})\);</script>',
-                                      resp.text)
-                    if match is not None:
-                        post_or_profile_page[0]['graphql'] = json.loads(match.group(1))['graphql']
-                return resp_json
             else:
                 resp_json = resp.json()
             if 'status' in resp_json and resp_json['status'] != "ok":

instaloader/lateststamps.py

@@ -27,7 +27,8 @@ class LatestStamps:
         self.data.read(latest_stamps_file)
 
     def _save(self):
-        makedirs(dirname(self.file), exist_ok=True)
+        if dn := dirname(self.file):
+            makedirs(dn, exist_ok=True)
         with open(self.file, 'w') as f:
             self.data.write(f)
 

instaloader/nodeiterator.py

@@ -137,7 +137,7 @@ class NodeIterator(Iterator[T]):
                 if self._first_node is None:
                     self._first_node = node
             return item
-        if self._data['page_info']['has_next_page']:
+        if self._data.get('page_info', {}).get('has_next_page'):
             query_response = self._query(self._data['page_info']['end_cursor'])
             if self._data['edges'] != query_response['edges'] and len(query_response['edges']) > 0:
                 page_index, data = self._page_index, self._data
@@ -184,6 +184,10 @@ class NodeIterator(Iterator[T]):
         """
         return self._node_wrapper(self._first_node) if self._first_node is not None else None
 
+    @staticmethod
+    def page_length() -> int:
+        return NodeIterator._graphql_page_length
+
     def freeze(self) -> FrozenNodeIterator:
         """Freeze the iterator for later resuming."""
         remaining_data = None
@@ -289,7 +293,7 @@ def resumable_iteration(context: InstaloaderContext,
             is_resuming = True
             start_index = iterator.total_index
             context.log("Resuming from {}.".format(resume_file_path))
-        except (InvalidArgumentException, LZMAError, json.decoder.JSONDecodeError) as exc:
+        except (InvalidArgumentException, LZMAError, json.decoder.JSONDecodeError, EOFError) as exc:
             context.error("Warning: Not resuming from {}: {}".format(resume_file_path, exc))
     try:
         yield is_resuming, start_index

instaloader/structures.py

@@ -545,11 +545,74 @@ class Post:
         except KeyError:
             return self._field('edge_media_to_comment', 'count')
 
+    def _get_comments_via_iphone_endpoint(self) -> Iterable[PostComment]:
+        """
+        Iterate over all comments of the post via an iPhone endpoint.
+
+        .. versionadded:: 4.10.3
+           fallback for :issue:`2125`.
+        """
+        def _query(min_id=None):
+            pagination_params = {"min_id": min_id} if min_id is not None else {}
+            return self._context.get_iphone_json(
+                f"api/v1/media/{self.mediaid}/comments/",
+                {
+                    "can_support_threading": "true",
+                    "permalink_enabled": "false",
+                    **pagination_params,
+                },
+            )
+
+        def _answers(comment_node):
+            def _answer(child_comment):
+                return PostCommentAnswer(
+                    id=int(child_comment["pk"]),
+                    created_at_utc=datetime.utcfromtimestamp(child_comment["created_at"]),
+                    text=child_comment["text"],
+                    owner=Profile.from_iphone_struct(self._context, child_comment["user"]),
+                    likes_count=child_comment["comment_like_count"],
+                )
+
+            child_comment_count = comment_node["child_comment_count"]
+            if child_comment_count == 0:
+                return
+            preview_child_comments = comment_node["preview_child_comments"]
+            if child_comment_count == len(preview_child_comments):
+                yield from (
+                    _answer(child_comment) for child_comment in preview_child_comments
+                )
+                return
+            pk = comment_node["pk"]
+            answers_json = self._context.get_iphone_json(
+                f"api/v1/media/{self.mediaid}/comments/{pk}/child_comments/",
+                {"max_id": ""},
+            )
+            yield from (
+                _answer(child_comment) for child_comment in answers_json["child_comments"]
+            )
+
+        def _paginated_comments(comments_json):
+            for comment_node in comments_json.get("comments", []):
+                yield PostComment(
+                    id=int(comment_node["pk"]),
+                    created_at_utc=datetime.utcfromtimestamp(comment_node["created_at"]),
+                    text=comment_node["text"],
+                    owner=Profile.from_iphone_struct(self._context, comment_node["user"]),
+                    likes_count=comment_node["comment_like_count"],
+                    answers=_answers(comment_node),
+                )
+
+            next_min_id = comments_json.get("next_min_id")
+            if next_min_id:
+                yield from _paginated_comments(_query(next_min_id))
+
+        return _paginated_comments(_query())
+
     def get_comments(self) -> Iterable[PostComment]:
-        r"""Iterate over all comments of the post.
+        """Iterate over all comments of the post.
 
         Each comment is represented by a PostComment NamedTuple with fields text (string), created_at (datetime),
-        id (int), owner (:class:`Profile`) and answers (:class:`~typing.Iterator`\ [:class:`PostCommentAnswer`])
+        id (int), owner (:class:`Profile`) and answers (:class:`~typing.Iterator` [:class:`PostCommentAnswer`])
         if available.
 
         .. versionchanged:: 4.7
@@ -596,6 +659,12 @@ class Post:
         if self.comments == len(comment_edges) + answers_count:
             # If the Post's metadata already contains all parent comments, don't do GraphQL requests to obtain them
             return [_postcomment(comment['node']) for comment in comment_edges]
+
+        if self.comments > NodeIterator.page_length():
+            # comments pagination via our graphql query does not work reliably anymore (issue #2125), fallback to an
+            # iphone endpoint if needed.
+            return self._get_comments_via_iphone_endpoint()
+
         return NodeIterator(
             self._context,
             '97b41c52301f77ce508f55e66d17620e',
@@ -678,7 +747,11 @@ class Post:
 
     @property
     def is_pinned(self) -> bool:
-        """True if this Post has been pinned by at least one user.
+        """
+        .. deprecated: 4.10.3
+           This information is not returned by IG anymore
+
+        Used to return True if this Post has been pinned by at least one user, now likely returns always false.
 
         .. versionadded: 4.9.2"""
         return 'pinned_for_users' in self._node and bool(self._node['pinned_for_users'])
@@ -1097,6 +1170,27 @@ class Profile:
     def _make_is_newest_checker() -> Callable[[Post, Optional[Post]], bool]:
         return lambda post, first: first is None or post.date_local > first.date_local
 
+    def get_followed_hashtags(self) -> NodeIterator['Hashtag']:
+        """
+        Retrieve list of hashtags followed by given profile.
+        To use this, one needs to be logged in and private profiles has to be followed.
+
+        :rtype: NodeIterator[Hashtag]
+
+        .. versionadded:: 4.10
+        """
+        if not self._context.is_logged_in:
+            raise LoginRequiredException("--login required to get a profile's followers.")
+        self._obtain_metadata()
+        return NodeIterator(
+            self._context,
+            'e6306cc3dbe69d6a82ef8b5f8654c50b',
+            lambda d: d["data"]["user"]["edge_following_hashtag"],
+            lambda n: Hashtag(self._context, n),
+            {'id': str(self.userid)},
+            'https://www.instagram.com/{0}/'.format(self.username),
+        )
+
     def get_followers(self) -> NodeIterator['Profile']:
         """
         Retrieve list of followers of given profile.