attempt to fix 2FA

.github/workflows/lint.yml

@@ -9,7 +9,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: ["3.8", "3.9", "3.10"]
+        python-version: ["3.8", "3.9", "3.10", "3.11"]
     steps:
       - name: Checkout Instaloader Repository
         uses: actions/checkout@v2

AUTHORS.md

@@ -6,5 +6,4 @@ Instaloader is written by
 - Alexander Graf (@aandergr)
 - André Koch-Kramer (@Thammus)
 - Lars Lindqvist (@e5150)
-- Eduardo M Kalinowksi (@ekalin)
 - ... and many more, see https://github.com/instaloader/instaloader/graphs/contributors

instaloader/__init__.py

@@ -1,7 +1,7 @@
 """Download pictures (or videos) along with their captions and other metadata from Instagram."""
 
 
-__version__ = '4.10.3'
+__version__ = '4.10'
 
 
 try:

instaloader/instaloader.py

@@ -991,8 +991,7 @@ class Instaloader:
                             max_count: Optional[int] = None,
                             total_count: Optional[int] = None,
                             owner_profile: Optional[Profile] = None,
-                            takewhile: Optional[Callable[[Post], bool]] = None,
-                            possibly_pinned: int = 0) -> None:
+                            takewhile: Optional[Callable[[Post], bool]] = None) -> None:
         """
         Download the Posts returned by given Post Iterator.
 
@@ -1004,9 +1003,6 @@ class Instaloader:
         .. versionchanged:: 4.8
            Add `takewhile` parameter.
 
-        .. versionchanged:: 4.10.3
-           Add `possibly_pinned` parameter.
-
         :param posts: Post Iterator to loop through.
         :param target: Target name.
         :param fast_update: :option:`--fast-update`.
@@ -1015,8 +1011,6 @@ class Instaloader:
         :param total_count: Total number of posts returned by given iterator.
         :param owner_profile: Associated profile, if any.
         :param takewhile: Expression evaluated for each post. Once it returns false, downloading stops.
-        :param possibly_pinned: Number of posts that might be pinned. These posts do not cause download
-               to stop even if they've already been downloaded.
         """
         displayed_count = (max_count if total_count is None or max_count is not None and max_count < total_count
                            else total_count)
@@ -1038,7 +1032,7 @@ class Instaloader:
         ) as (is_resuming, start_index):
             for number, post in enumerate(posts, start=start_index + 1):
                 should_stop = not takewhile(post)
-                if should_stop and number <= possibly_pinned:
+                if should_stop and post.is_pinned:
                     continue
                 if (max_count is not None and number > max_count) or should_stop:
                     break
@@ -1072,7 +1066,7 @@ class Instaloader:
                         except PostChangedException:
                             post_changed = True
                             continue
-                    if fast_update and not downloaded and not post_changed and number > possibly_pinned:
+                    if fast_update and not downloaded and not post_changed and not post.is_pinned:
                         # disengage fast_update for first post when resuming
                         if not is_resuming or number > 0:
                             break
@@ -1494,7 +1488,7 @@ class Instaloader:
                     posts_to_download = profile.get_posts()
                     self.posts_download_loop(posts_to_download, profile_name, fast_update, post_filter,
                                              total_count=profile.mediacount, owner_profile=profile,
-                                             takewhile=posts_takewhile, possibly_pinned=3)
+                                             takewhile=posts_takewhile)
                     if latest_stamps is not None and posts_to_download.first_item is not None:
                         latest_stamps.set_last_post_timestamp(profile_name,
                                                               posts_to_download.first_item.date_local)

instaloader/instaloadercontext.py

@@ -3,6 +3,7 @@ import json
 import os
 import pickle
 import random
+import re
 import shutil
 import sys
 import textwrap
@@ -252,26 +253,23 @@ class InstaloaderContext:
                                 'ig_vw': '1920', 'ig_cb': '1', 'csrftoken': '',
                                 's_network': '', 'ds_user_id': ''})
         session.headers.update(self._default_http_header())
+        # Getting mid cookie value
+        session.get("https://i.instagram.com/api/v1/web/login_page/")
         # Override default timeout behavior.
         # Need to silence mypy bug for this. See: https://github.com/python/mypy/issues/2427
         session.request = partial(session.request, timeout=self.request_timeout) # type: ignore
-
-        # Make a request to Instagram's root URL, which will set the session's csrftoken cookie
-        # Not using self.get_json() here, because we need to access the cookie
-        session.get('https://www.instagram.com/')
-        # Add session's csrftoken cookie to session headers
-        csrf_token = session.cookies.get_dict()['csrftoken']
+        csrf_json = self.get_json('accounts/login/', {}, session=session)
+        csrf_token = csrf_json['config']['csrf_token']
         session.headers.update({'X-CSRFToken': csrf_token})
-
+        # Not using self.get_json() here, because we need to access csrftoken cookie
         self.do_sleep()
         # Workaround credits to pgrimaud.
         # See: https://github.com/pgrimaud/instagram-user-feed/commit/96ad4cf54d1ad331b337f325c73e664999a6d066
         enc_password = '#PWD_INSTAGRAM_BROWSER:0:{}:{}'.format(int(datetime.now().timestamp()), passwd)
-        login = session.post('https://www.instagram.com/api/v1/web/accounts/login/ajax/',
+        login = session.post('https://www.instagram.com/accounts/login/ajax/',
                              data={'enc_password': enc_password, 'username': user}, allow_redirects=True)
         try:
             resp_json = login.json()
-
         except json.decoder.JSONDecodeError as err:
             raise ConnectionException(
                 "Login error: JSON decode fail, {} - {}.".format(login.status_code, login.reason)
@@ -311,7 +309,8 @@ class InstaloaderContext:
                 # username is invalid.
                 raise InvalidArgumentException('Login error: User {} does not exist.'.format(user))
         # '{"authenticated": true, "user": true, "userId": ..., "oneTapPrompt": false, "status": "ok"}'
-        session.headers.update({'X-CSRFToken': login.cookies['csrftoken']})
+        if 'csrftoken' in login.cookies:
+            session.headers.update({'X-CSRFToken': login.cookies['csrftoken']})
         self._session = session
         self.username = user
         self.user_id = resp_json['userId']
@@ -328,8 +327,15 @@ class InstaloaderContext:
             raise InvalidArgumentException("No two-factor authentication pending.")
         (session, user, two_factor_id) = self.two_factor_auth_pending
 
-        login = session.post('https://www.instagram.com/accounts/login/ajax/two_factor/',
-                             data={'username': user, 'verificationCode': two_factor_code, 'identifier': two_factor_id},
+        login = session.post('https://www.instagram.com/api/v1/web/accounts/login/ajax/two_factor/',
+                             data={
+                                 'identifier': two_factor_id,
+                                 'queryParams': '{"next":"/"}',
+                                 'trust_signal': 'true',
+                                 'username': user,
+                                 'verification_method': '1',
+                                 'verificationCode': two_factor_code,
+                             },
                              allow_redirects=True)
         resp_json = login.json()
         if resp_json['status'] != 'ok':
@@ -337,7 +343,8 @@ class InstaloaderContext:
                 raise BadCredentialsException("2FA error: {}".format(resp_json['message']))
             else:
                 raise BadCredentialsException("2FA error: \"{}\" status.".format(resp_json['status']))
-        session.headers.update({'X-CSRFToken': login.cookies['csrftoken']})
+        if 'csrftoken' in login.cookies:
+            session.headers.update({'X-CSRFToken': login.cookies['csrftoken']})
         self._session = session
         self.username = user
         self.two_factor_auth_pending = None
@@ -407,6 +414,16 @@ class InstaloaderContext:
                 raise TooManyRequestsException("429 Too Many Requests")
             if resp.status_code != 200:
                 raise ConnectionException("HTTP error code {}.".format(resp.status_code))
+            is_html_query = not is_graphql_query and not "__a" in params and host == "www.instagram.com"
+            if is_html_query:
+                # Extract JSON from HTML response
+                match = re.search('(?<={"raw":").*?(?<!\\\\)(?=")', resp.text)
+                if match is None:
+                    raise QueryReturnedNotFoundException("Could not find JSON data in html response.")
+                # Unescape escaped JSON string
+                unescaped_string = match.group(0).encode("utf-8").decode("unicode_escape")
+                resp_json = json.loads(unescaped_string)
+                return resp_json
             else:
                 resp_json = resp.json()
             if 'status' in resp_json and resp_json['status'] != "ok":

instaloader/nodeiterator.py

@@ -184,10 +184,6 @@ class NodeIterator(Iterator[T]):
         """
         return self._node_wrapper(self._first_node) if self._first_node is not None else None
 
-    @staticmethod
-    def page_length() -> int:
-        return NodeIterator._graphql_page_length
-
     def freeze(self) -> FrozenNodeIterator:
         """Freeze the iterator for later resuming."""
         remaining_data = None

instaloader/structures.py

@@ -545,74 +545,11 @@ class Post:
         except KeyError:
             return self._field('edge_media_to_comment', 'count')
 
-    def _get_comments_via_iphone_endpoint(self) -> Iterable[PostComment]:
-        """
-        Iterate over all comments of the post via an iPhone endpoint.
-
-        .. versionadded:: 4.10.3
-           fallback for :issue:`2125`.
-        """
-        def _query(min_id=None):
-            pagination_params = {"min_id": min_id} if min_id is not None else {}
-            return self._context.get_iphone_json(
-                f"api/v1/media/{self.mediaid}/comments/",
-                {
-                    "can_support_threading": "true",
-                    "permalink_enabled": "false",
-                    **pagination_params,
-                },
-            )
-
-        def _answers(comment_node):
-            def _answer(child_comment):
-                return PostCommentAnswer(
-                    id=int(child_comment["pk"]),
-                    created_at_utc=datetime.utcfromtimestamp(child_comment["created_at"]),
-                    text=child_comment["text"],
-                    owner=Profile.from_iphone_struct(self._context, child_comment["user"]),
-                    likes_count=child_comment["comment_like_count"],
-                )
-
-            child_comment_count = comment_node["child_comment_count"]
-            if child_comment_count == 0:
-                return
-            preview_child_comments = comment_node["preview_child_comments"]
-            if child_comment_count == len(preview_child_comments):
-                yield from (
-                    _answer(child_comment) for child_comment in preview_child_comments
-                )
-                return
-            pk = comment_node["pk"]
-            answers_json = self._context.get_iphone_json(
-                f"api/v1/media/{self.mediaid}/comments/{pk}/child_comments/",
-                {"max_id": ""},
-            )
-            yield from (
-                _answer(child_comment) for child_comment in answers_json["child_comments"]
-            )
-
-        def _paginated_comments(comments_json):
-            for comment_node in comments_json.get("comments", []):
-                yield PostComment(
-                    id=int(comment_node["pk"]),
-                    created_at_utc=datetime.utcfromtimestamp(comment_node["created_at"]),
-                    text=comment_node["text"],
-                    owner=Profile.from_iphone_struct(self._context, comment_node["user"]),
-                    likes_count=comment_node["comment_like_count"],
-                    answers=_answers(comment_node),
-                )
-
-            next_min_id = comments_json.get("next_min_id")
-            if next_min_id:
-                yield from _paginated_comments(_query(next_min_id))
-
-        return _paginated_comments(_query())
-
     def get_comments(self) -> Iterable[PostComment]:
-        """Iterate over all comments of the post.
+        r"""Iterate over all comments of the post.
 
         Each comment is represented by a PostComment NamedTuple with fields text (string), created_at (datetime),
-        id (int), owner (:class:`Profile`) and answers (:class:`~typing.Iterator` [:class:`PostCommentAnswer`])
+        id (int), owner (:class:`Profile`) and answers (:class:`~typing.Iterator`\ [:class:`PostCommentAnswer`])
         if available.
 
         .. versionchanged:: 4.7
@@ -659,12 +596,6 @@ class Post:
         if self.comments == len(comment_edges) + answers_count:
             # If the Post's metadata already contains all parent comments, don't do GraphQL requests to obtain them
             return [_postcomment(comment['node']) for comment in comment_edges]
-
-        if self.comments > NodeIterator.page_length():
-            # comments pagination via our graphql query does not work reliably anymore (issue #2125), fallback to an
-            # iphone endpoint if needed.
-            return self._get_comments_via_iphone_endpoint()
-
         return NodeIterator(
             self._context,
             '97b41c52301f77ce508f55e66d17620e',
@@ -747,11 +678,7 @@ class Post:
 
     @property
     def is_pinned(self) -> bool:
-        """
-        .. deprecated: 4.10.3
-           This information is not returned by IG anymore
-
-        Used to return True if this Post has been pinned by at least one user, now likely returns always false.
+        """True if this Post has been pinned by at least one user.
 
         .. versionadded: 4.9.2"""
         return 'pinned_for_users' in self._node and bool(self._node['pinned_for_users'])