Release of version 4.15.5

The NodeIterator is sometimes invocated with node wrappers that are heavy to
invocate, e.g. because they trigger requests. Storing the resulting item
besides the node allows to avoid those extra requests.

In the FrozenNodeIterator, the node is still saved, rather than the item.

.github/pull_request_template.md

@@ -20,5 +20,8 @@ describe:
 - Further considerations
   - Is it based on master (bugfix), upcoming/v4.X (new feature), or another pull request?
   - Does it preserve backward compatibility and existing behavior?
+  - If generative AI was involved, ensure it follows our AI policy.
+
+(also see https://instaloader.github.io/contributing.html)
 
 -->

.pylintrc

@@ -93,6 +93,10 @@ py-version=3.8
 # Discover python modules and packages in the file system subtree.
 recursive=no
 
+# When enabled, pylint would attempt to guess common misconfiguration and emit
+# user-friendly hints instead of false-positive error messages.
+suggestion-mode=yes
+
 # Allow loading of arbitrary C extensions. Extensions are imported into the
 # active Python interpreter and may run arbitrary code.
 unsafe-load-any-extension=no

LICENSE

@@ -1,6 +1,7 @@
 The MIT License (MIT)
 
-Copyright (c) 2016-2020 Alexander Graf and André Koch-Kramer.
+Copyright (c) 2016-2026 Alexander Graf, André Koch-Kramer and the Instaloader
+contributors.
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

docs/conf.py

@@ -15,7 +15,7 @@ sys.path.insert(0, os.path.abspath('..'))
 # https://www.sphinx-doc.org/en/master/usage/configuration.html#project-information
 
 project = 'Instaloader'
-copyright = '2023, Alexander Graf and Andre Koch-Kramer'
+copyright = '2016-2026, Alexander Graf, Andre Koch-Kramer and the Instaloader contributors'
 author = 'Alexander Graf and Andre Koch-Kramer'
 
 # -- General configuration ---------------------------------------------------

docs/contributing.rst

@@ -80,7 +80,7 @@ Improvements of the Instaloader source or its documentation can be proposed as a
   review, validate, or selectively merge them.
 
 - Be mindful of backward compatibility: changes should preserve existing intended
-  behavior whenever possible.
+  behavior whenever feasible.
 
 - All Pull Requests are analyzed by `Pylint <https://www.pylint.org/>`__ for
   error and syntax checking of the source and
@@ -100,6 +100,19 @@ Improvements of the Instaloader source or its documentation can be proposed as a
   agrees that the change might be an improvement, or if you want to discuss
   your basic proposal, before working on a pull request.
 
+AI Policy
+---------
+
+- If generative AI was used to create notable portions of a contribution, the
+  author must disclose which parts were AI-assisted (e.g. research, testing,
+  coding, or documentation).
+
+- The author must carefully review and understand the contribution before
+  submitting it.
+
+- If the maintainers need more effort to review the contribution than it took
+  to create it, please do not submit it.
+
 Proposing Features
 ------------------
 

instaloader/__init__.py

@@ -1,7 +1,7 @@
 """Download pictures (or videos) along with their captions and other metadata from Instagram."""
 
 
-__version__ = '4.16b1'
+__version__ = '4.15.5'
 
 
 try:

instaloader/__main__.py

@@ -16,7 +16,7 @@ from .instaloader import (get_default_session_filename, get_default_stamps_filen
 from .instaloadercontext import default_user_agent
 from .lateststamps import LatestStamps
 try:
-    import browser_cookie3  # type: ignore[import-untyped]
+    import browser_cookie3
     bc3_library = True
 except ImportError:
     bc3_library = False

instaloader/instaloader.py

@@ -85,36 +85,31 @@ def _requires_login(func: Callable) -> Callable:
 def _retry_on_connection_error(func: Callable) -> Callable:
     """Decorator to retry the function max_connection_attempts number of times.
 
+    Herewith-decorated functions need an ``_attempt`` keyword argument.
+
     This is to decorate functions that do network requests that may fail. Note that
     :meth:`.get_json`, :meth:`.get_iphone_json`, :meth:`.graphql_query` and :meth:`.graphql_node_list` already have
     their own logic for retrying, hence functions that only use these for network access must not be decorated with this
-    decorator.
-
-    .. versionchanged:: 4.16
-       Decorated functions no longer need to accept an ``_attempt`` keyword argument.
-    """
+    decorator."""
     @wraps(func)
-    def call(instaloader, *args, **kwargs) -> Any:
-        # The failure we are currently retrying, if any. It tells us whether a ^C means "skip this
-        # download" rather than "abort everything", and whether to sleep before the next attempt.
-        pending_error_string: Optional[str] = None
-        for attempt in range(1, instaloader.context.max_connection_attempts + 1):
+    def call(instaloader, *args, **kwargs):
+        try:
+            return func(instaloader, *args, **kwargs)
+        except (urllib3.exceptions.HTTPError, requests.exceptions.RequestException, ConnectionException) as err:
+            error_string = "{}({}): {}".format(func.__name__, ', '.join([repr(arg) for arg in args]), err)
+            if (kwargs.get('_attempt') or 1) == instaloader.context.max_connection_attempts:
+                raise ConnectionException(error_string) from None
+            instaloader.context.error(error_string + " [retrying; skip with ^C]", repeat_at_end=False)
             try:
-                if pending_error_string is not None:
-                    instaloader.context.do_sleep()
-                return func(instaloader, *args, **kwargs)
-            except (urllib3.exceptions.HTTPError, requests.exceptions.RequestException, ConnectionException) as err:
-                error_string = "{}({}): {}".format(func.__name__, ', '.join([repr(arg) for arg in args]), err)
-                if attempt == instaloader.context.max_connection_attempts:
-                    raise ConnectionException(error_string) from None
-                instaloader.context.error(error_string + " [retrying; skip with ^C]", repeat_at_end=False)
-                pending_error_string = error_string
+                if kwargs.get('_attempt'):
+                    kwargs['_attempt'] += 1
+                else:
+                    kwargs['_attempt'] = 2
+                instaloader.context.do_sleep()
+                return call(instaloader, *args, **kwargs)
             except KeyboardInterrupt:
-                if pending_error_string is None:
-                    raise
                 instaloader.context.error("[skipped by user]", repeat_at_end=False)
-                raise ConnectionException(pending_error_string) from None
-        raise AssertionError("_retry_on_connection_error() retry loop exited without result")
+                raise ConnectionException(error_string) from None
     return call
 
 
@@ -340,7 +335,7 @@ class Instaloader:
 
     @_retry_on_connection_error
     def download_pic(self, filename: str, url: str, mtime: datetime,
-                     filename_suffix: Optional[str] = None) -> bool:
+                     filename_suffix: Optional[str] = None, _attempt: int = 1) -> bool:
         """Downloads and saves picture with given url under given directory with given timestamp.
         Returns true, if file was actually downloaded, i.e. updated."""
         if filename_suffix is not None:
@@ -524,9 +519,8 @@ class Instaloader:
                                 '{0}_{1}_{2}.{3}'.format(target, identifier, name_suffix, extension))
 
     @_retry_on_connection_error
-    def download_title_pic(
-            self, url: str, target: Union[str, Path], name_suffix: str, owner_profile: Optional[Profile]
-    ) -> None:
+    def download_title_pic(self, url: str, target: Union[str, Path], name_suffix: str, owner_profile: Optional[Profile],
+                           _attempt: int = 1) -> None:
         """Downloads and saves a picture that does not have an association with a Post or StoryItem, such as a
         Profile picture or a Highlight cover picture. Modification time is taken from the HTTP response headers.
 
@@ -969,11 +963,6 @@ class Instaloader:
         .. versionchanged:: 4.3
            Also downloads and saves the Highlight's cover pictures.
 
-        .. versionchanged:: 4.16
-           Add progress output for highlight retrieval, displaying a counter of the current
-           highlight number and the total number of highlights being processed. This provides
-           feedback similar to the per-item download counter already present in this function.
-
         :param user: ID or Profile of the user whose highlights should get downloaded.
         :param fast_update: If true, abort when first already-downloaded picture is encountered
         :param filename_target: Replacement for {target} in dirname_pattern and filename_pattern
@@ -981,20 +970,14 @@ class Instaloader:
         :param storyitem_filter: function(storyitem), which returns True if given StoryItem should be downloaded
         :raises LoginRequiredException: If called without being logged in.
         """
-        user_highlights = list(self.get_highlights(user))
-        hl_size = len(user_highlights)
-        for hl_number, user_highlight in enumerate(user_highlights, start=1):
+        for user_highlight in self.get_highlights(user):
             name = user_highlight.owner_username
             highlight_target: Union[str, Path] = (filename_target
                                 if filename_target
                                 else (Path(_PostPathFormatter.sanitize_path(name, self.sanitize_paths)) /
                                       _PostPathFormatter.sanitize_path(user_highlight.title,
                                                                        self.sanitize_paths)))
-            self.context.log(
-                "[{0:{w}d}/{1:{w}d}] Retrieving highlights \"{2}\" from profile {3}".format(
-                    hl_number, hl_size, user_highlight.title, name, w=len(str(hl_size))
-                )
-            )
+            self.context.log("Retrieving highlights \"{}\" from profile {}".format(user_highlight.title, name))
             self.download_highlight_cover(user_highlight, highlight_target)
             totalcount = user_highlight.itemcount
             count = 1

instaloader/instaloadercontext.py

@@ -101,7 +101,7 @@ class InstaloaderContext:
         self.user_agent = user_agent if user_agent is not None else default_user_agent()
         self.request_timeout = request_timeout
         self._session = self.get_anonymous_session()
-        self.username: Optional[str] = None
+        self.username = None
         self.user_id = None
         self.sleep = sleep
         self.quiet = quiet
@@ -125,9 +125,6 @@ class InstaloaderContext:
         # Cache profile from id (mapping from id to Profile)
         self.profile_id_cache: Dict[int, Any] = dict()
 
-        # Tokens of the logged-in web page for api/graphql queries, by sessionid cookie
-        self._web_api_tokens_cache: Dict[str, Dict[str, str]] = dict()
-
     @contextmanager
     def anonymous_copy(self):
         session = self._session
@@ -404,102 +401,8 @@ class InstaloaderContext:
             f" when accessing {resp.url}"
         )
 
-    @staticmethod
-    def _rate_limit_key(path: str, params: Dict[str, Any], host: str) -> str:
-        """Determine the :class:`RateController` bucket that a request is accounted to."""
-        if 'graphql/query' in path or path == 'api/graphql':
-            if 'query_hash' in params:
-                return params['query_hash']
-            if 'doc_id' in params:
-                return params['doc_id']
-        if host == 'i.instagram.com':
-            return 'iphone'
-        return 'other'
-
-    def _raise_if_fatal_status(self, resp: requests.Response, host: str, path: str) -> None:
-        if resp.status_code not in self.fatal_status_codes:
-            return
-        redirect = " redirect to {}".format(resp.headers['location']) if 'location' in resp.headers else ""
-        body = ""
-        if resp.headers.get('Content-Type', '').startswith('application/json'):
-            body = ': ' + resp.text[:500] + ('…' if len(resp.text) > 501 else '')
-        raise AbortDownloadException("Query to https://{}/{} responded with \"{} {}\"{}{}".format(
-            host, path, resp.status_code, resp.reason, redirect, body
-        ))
-
-    def _raise_if_login_redirect(self, resp: requests.Response) -> None:
-        """Turn a redirect to the login page into a meaningful exception.
-
-        Instagram answers requests it will not serve anonymously with a redirect to the login page
-        rather than with an error status, so this needs to be checked wherever we do not follow
-        redirects automatically.
-        """
-        if not resp.is_redirect:
-            return
-        redirect_url = resp.headers.get('location', '')
-        if not (redirect_url.startswith('https://www.instagram.com/accounts/login') or
-                redirect_url.startswith('https://i.instagram.com/accounts/login')):
-            return
-        if not self.is_logged_in:
-            raise LoginRequiredException("Redirected to login page. Use --login or --load-cookies.")
-        raise AbortDownloadException("Redirected to login page. You've been logged out, please wait " +
-                                     "some time, recreate the session and try again")
-
-    def _follow_redirects(self, sess: requests.Session, resp: requests.Response,
-                          host: str, path: str, params: Dict[str, Any], use_post: bool) -> requests.Response:
-        """Follow redirects that stay on *host*, and reject those to the login page."""
-        # Redirects are followed with GET. For a POST request, `params` is the request body and must not be
-        # turned into query arguments.
-        redirect_params = None if use_post else params
-        while resp.is_redirect:
-            redirect_url = resp.headers['location']
-            self.log('\nHTTP redirect from https://{0}/{1} to {2}'.format(host, path, redirect_url))
-            self._raise_if_login_redirect(resp)
-            if not redirect_url.startswith('https://{}/'.format(host)):
-                break
-            resp = sess.get(redirect_url if redirect_url.endswith('/') else redirect_url + '/',
-                            params=redirect_params, allow_redirects=False)
-        return resp
-
-    def _raise_for_status(self, resp: requests.Response) -> None:
-        """Translate an unsuccessful HTTP status code into the corresponding exception."""
-        if resp.status_code == 400:
-            with suppress(json.decoder.JSONDecodeError):
-                if resp.json().get("message") in [
-                    "feedback_required",
-                    "checkpoint_required",
-                    "challenge_required",
-                ]:
-                    # Raise AbortDownloadException in case of substantial Instagram
-                    # requirements to stop producing more requests
-                    raise AbortDownloadException(self._response_error(resp))
-            raise QueryReturnedBadRequestException(self._response_error(resp))
-        if resp.status_code == 404:
-            raise QueryReturnedNotFoundException(self._response_error(resp))
-        if resp.status_code == 429:
-            raise TooManyRequestsException(self._response_error(resp))
-        if resp.status_code != 200:
-            raise ConnectionException(self._response_error(resp))
-
-    def _raise_for_download_status(self, resp: requests.Response) -> None:
-        """Translate an unsuccessful status code of an anonymous download request into an exception.
-
-        This is deliberately a different policy from :meth:`_raise_for_status`, which applies to the
-        JSON API: a download may be answered with 403 for an invalid URL signature, and is never
-        answered with the 400 or 429 that the API uses.
-        """
-        if resp.status_code == 200:
-            return
-        if resp.status_code == 403:
-            # suspected invalid URL signature
-            raise QueryReturnedForbiddenException(self._response_error(resp))
-        if resp.status_code == 404:
-            # 404 not worth retrying.
-            raise QueryReturnedNotFoundException(self._response_error(resp))
-        raise ConnectionException(self._response_error(resp))
-
     def get_json(self, path: str, params: Dict[str, Any], host: str = 'www.instagram.com',
-                 session: Optional[requests.Session] = None,
+                 session: Optional[requests.Session] = None, _attempt=1,
                  response_headers: Optional[Dict[str, Any]] = None,
                  use_post: bool = False) -> Dict[str, Any]:
         """JSON request to Instagram.
@@ -519,110 +422,96 @@ class InstaloaderContext:
 
         .. versionchanged:: 4.13
            Added `use_post` parameter.
-
-        .. versionchanged:: 4.16
-           Removed the internal `_attempt` parameter; retries are no longer implemented by recursion.
         """
-        sess = session or self._session
-        rate_limit_key = self._rate_limit_key(path, params, host)
-        url = 'https://{0}/{1}'.format(host, path)
-
-        # The failure we are currently retrying, if any. It drives the 429 backoff performed before the
-        # next attempt, and tells us whether a ^C means "skip this query" rather than "abort everything".
-        pending_error: Optional[BaseException] = None
-        pending_error_string = ""
-
-        for attempt in range(1, self.max_connection_attempts + 1):
-            try:
-                if isinstance(pending_error, TooManyRequestsException):
-                    self._rate_controller.handle_429(rate_limit_key)
-                self.do_sleep()
-                self._rate_controller.wait_before_query(rate_limit_key)
-
-                if use_post:
-                    resp = sess.post(url, data=params, allow_redirects=False)
+        is_graphql_query = 'query_hash' in params and 'graphql/query' in path
+        is_doc_id_query = 'doc_id' in params and 'graphql/query' in path
+        is_iphone_query = host == 'i.instagram.com'
+        is_other_query = not is_graphql_query and not is_doc_id_query and host == "www.instagram.com"
+        sess = session if session else self._session
+        try:
+            self.do_sleep()
+            if is_graphql_query:
+                self._rate_controller.wait_before_query(params['query_hash'])
+            if is_doc_id_query:
+                self._rate_controller.wait_before_query(params['doc_id'])
+            if is_iphone_query:
+                self._rate_controller.wait_before_query('iphone')
+            if is_other_query:
+                self._rate_controller.wait_before_query('other')
+            if use_post:
+                resp = sess.post('https://{0}/{1}'.format(host, path), data=params, allow_redirects=False)
+            else:
+                resp = sess.get('https://{0}/{1}'.format(host, path), params=params, allow_redirects=False)
+            if resp.status_code in self.fatal_status_codes:
+                redirect = " redirect to {}".format(resp.headers['location']) if 'location' in resp.headers else ""
+                body = ""
+                if resp.headers['Content-Type'].startswith('application/json'):
+                    body = ': ' + resp.text[:500] + ('…' if len(resp.text) > 501 else '')
+                raise AbortDownloadException("Query to https://{}/{} responded with \"{} {}\"{}{}".format(
+                    host, path, resp.status_code, resp.reason, redirect, body
+                ))
+            while resp.is_redirect:
+                redirect_url = resp.headers['location']
+                self.log('\nHTTP redirect from https://{0}/{1} to {2}'.format(host, path, redirect_url))
+                if (redirect_url.startswith('https://www.instagram.com/accounts/login') or
+                    redirect_url.startswith('https://i.instagram.com/accounts/login')):
+                    if not self.is_logged_in:
+                        raise LoginRequiredException("Redirected to login page. Use --login or --load-cookies.")
+                    raise AbortDownloadException("Redirected to login page. You've been logged out, please wait " +
+                                                 "some time, recreate the session and try again")
+                if redirect_url.startswith('https://{}/'.format(host)):
+                    resp = sess.get(redirect_url if redirect_url.endswith('/') else redirect_url + '/',
+                                    params=params, allow_redirects=False)
                 else:
-                    resp = sess.get(url, params=params, allow_redirects=False)
-
-                self._raise_if_fatal_status(resp, host, path)
-                resp = self._follow_redirects(sess, resp, host, path, params, use_post)
-
-                if response_headers is not None:
-                    response_headers.clear()
-                    response_headers.update(resp.headers)
-
-                self._raise_for_status(resp)
+                    break
+            if response_headers is not None:
+                response_headers.clear()
+                response_headers.update(resp.headers)
+            if resp.status_code == 400:
+                with suppress(json.decoder.JSONDecodeError):
+                    if resp.json().get("message") in [
+                        "feedback_required",
+                        "checkpoint_required",
+                        "challenge_required",
+                    ]:
+                        # Raise AbortDownloadException in case of substantial Instagram
+                        # requirements to stop producing more requests
+                        raise AbortDownloadException(self._response_error(resp))
+                raise QueryReturnedBadRequestException(self._response_error(resp))
+            if resp.status_code == 404:
+                raise QueryReturnedNotFoundException(self._response_error(resp))
+            if resp.status_code == 429:
+                raise TooManyRequestsException(self._response_error(resp))
+            if resp.status_code != 200:
+                raise ConnectionException(self._response_error(resp))
+            else:
                 resp_json = resp.json()
-                if 'status' in resp_json and resp_json['status'] != "ok":
-                    raise ConnectionException(self._response_error(resp))
-                return resp_json
-            except (ConnectionException, json.decoder.JSONDecodeError, requests.exceptions.RequestException) as err:
-                error_string = "JSON Query to {}: {}".format(path, err)
-                if attempt == self.max_connection_attempts:
-                    if isinstance(err, QueryReturnedNotFoundException):
-                        raise QueryReturnedNotFoundException(error_string) from err
+            if 'status' in resp_json and resp_json['status'] != "ok":
+                raise ConnectionException(self._response_error(resp))
+            return resp_json
+        except (ConnectionException, json.decoder.JSONDecodeError, requests.exceptions.RequestException) as err:
+            error_string = "JSON Query to {}: {}".format(path, err)
+            if _attempt == self.max_connection_attempts:
+                if isinstance(err, QueryReturnedNotFoundException):
+                    raise QueryReturnedNotFoundException(error_string) from err
+                else:
                     raise ConnectionException(error_string) from err
-                self.error(error_string + " [retrying; skip with ^C]", repeat_at_end=False)
-                pending_error, pending_error_string = err, error_string
+            self.error(error_string + " [retrying; skip with ^C]", repeat_at_end=False)
+            try:
+                if isinstance(err, TooManyRequestsException):
+                    if is_graphql_query:
+                        self._rate_controller.handle_429(params['query_hash'])
+                    if is_doc_id_query:
+                        self._rate_controller.handle_429(params['doc_id'])
+                    if is_iphone_query:
+                        self._rate_controller.handle_429('iphone')
+                    if is_other_query:
+                        self._rate_controller.handle_429('other')
+                return self.get_json(path=path, params=params, host=host, session=sess, _attempt=_attempt + 1,
+                                     response_headers=response_headers)
             except KeyboardInterrupt:
-                if pending_error is None:
-                    raise
                 self.error("[skipped by user]", repeat_at_end=False)
-                raise ConnectionException(pending_error_string) from pending_error
-
-        raise AssertionError("get_json() retry loop exited without result")
-
-    @contextmanager
-    def _graphql_session(self, referer: Optional[str], csrf: Optional[str] = None,
-                         lsd: Optional[str] = None) -> Iterator[requests.Session]:
-        """Short-lived session carrying the headers that the GraphQL endpoint expects.
-
-        Pass the page's `lsd` token for the web app's ``api/graphql`` endpoint."""
-        with copy_session(self._session, self.request_timeout) as tmpsession:
-            tmpsession.headers.update(self._default_http_header(empty_session_only=True))
-            del tmpsession.headers['Connection']
-            del tmpsession.headers['Content-Length']
-            tmpsession.headers['authority'] = 'www.instagram.com'
-            tmpsession.headers['scheme'] = 'https'
-            tmpsession.headers['accept'] = '*/*'
-            if csrf is not None:
-                tmpsession.headers['x-csrftoken'] = csrf
-            if referer is not None:
-                tmpsession.headers['referer'] = urllib.parse.quote(referer)
-            if lsd is not None:
-                tmpsession.headers.update({'x-fb-lsd': lsd, 'sec-fetch-dest': 'empty', 'sec-fetch-mode': 'cors',
-                                           'sec-fetch-site': 'same-origin'})
-            yield tmpsession
-
-    def _csrf_token(self) -> str:
-        """The csrftoken cookie value, requesting the start page once if we do not have one yet."""
-        def token_from_cookies() -> Optional[str]:
-            return next((c.value for c in self._session.cookies if c.name == 'csrftoken' and c.value), None)
-        csrf = token_from_cookies()
-        if csrf:
-            return csrf
-        self._session.get('https://www.instagram.com/', timeout=self.request_timeout)
-        return token_from_cookies() or ''
-
-    def _web_api_tokens(self) -> Dict[str, str]:
-        """The tokens that the web app sends along with its api/graphql queries, from the logged-in start page."""
-        sessionid = self._session.cookies.get_dict().get("sessionid", "")
-        if sessionid not in self._web_api_tokens_cache:
-            html = self._get_page(self._session, "")
-            tokens = {}
-            for name, pattern in [
-                ("fb_dtsg", r'"DTSGInitialData",\[\],\{[^{}]*"token":"([^"]+)"'),
-                ("lsd", r'"LSD",\[\],\{[^{}]*"token":"([^"]+)"'),
-                ("av", r'"IG_USER_EIMU":"?(\d+)'),
-            ]:
-                match = re.search(pattern, html)
-                if match is None:
-                    raise BadResponseException(
-                        "The start page did not contain the token {}.".format(name)
-                    )
-                tokens[name] = match.group(1)
-            self._web_api_tokens_cache[sessionid] = tokens
-        return self._web_api_tokens_cache[sessionid]
+                raise ConnectionException(error_string) from err
 
     def graphql_query(self, query_hash: str, variables: Dict[str, Any],
                       referer: Optional[str] = None) -> Dict[str, Any]:
@@ -637,8 +526,18 @@ class InstaloaderContext:
         .. versionchanged:: 4.13.1
            Removed the `rhx_gis` parameter.
         """
-        variables_json = json.dumps(variables, separators=(',', ':'))
-        with self._graphql_session(referer) as tmpsession:
+        with copy_session(self._session, self.request_timeout) as tmpsession:
+            tmpsession.headers.update(self._default_http_header(empty_session_only=True))
+            del tmpsession.headers['Connection']
+            del tmpsession.headers['Content-Length']
+            tmpsession.headers['authority'] = 'www.instagram.com'
+            tmpsession.headers['scheme'] = 'https'
+            tmpsession.headers['accept'] = '*/*'
+            if referer is not None:
+                tmpsession.headers['referer'] = urllib.parse.quote(referer)
+
+            variables_json = json.dumps(variables, separators=(',', ':'))
+
             resp_json = self.get_json('graphql/query',
                                       params={'query_hash': query_hash,
                                               'variables': variables_json},
@@ -648,7 +547,7 @@ class InstaloaderContext:
         return resp_json
 
     def doc_id_graphql_query(self, doc_id: str, variables: Dict[str, Any],
-                             referer: Optional[str] = None, web_api: bool = False) -> Dict[str, Any]:
+                             referer: Optional[str] = None) -> Dict[str, Any]:
         """
         Do a doc_id-based GraphQL Query using method POST.
 
@@ -657,23 +556,35 @@ class InstaloaderContext:
         :param doc_id: doc_id for the query.
         :param variables: Variables for the Query.
         :param referer: HTTP Referer, or None.
-        :param web_api: Query the web app's ``api/graphql`` endpoint, as the web app does, rather than
-           ``graphql/query``. Requires login.
         :return: The server's response dictionary.
-
-        .. versionchanged:: 4.16
-           Added `web_api` parameter.
         """
-        variables_json = json.dumps(variables, separators=(',', ':'))
-        params = {'variables': variables_json,
-                  'doc_id': doc_id,
-                  'server_timestamps': 'true',
-                  **(self._web_api_tokens() if web_api else {})}
-        with self._graphql_session(referer, csrf=self._csrf_token(), lsd=params.get('lsd')) as tmpsession:
-            resp_json = self.get_json('api/graphql' if web_api else 'graphql/query', params=params,
-                                      session=tmpsession, use_post=True)
-        # api/graphql does not respond with a "status" field.
-        if not web_api and 'status' not in resp_json:
+        csrf = next((c.value for c in self._session.cookies
+                     if c.name == 'csrftoken' and c.value), None)
+        if not csrf:
+            self._session.get('https://www.instagram.com/', timeout=self.request_timeout)
+            csrf = next((c.value for c in self._session.cookies
+                         if c.name == 'csrftoken' and c.value), '')
+
+        with copy_session(self._session, self.request_timeout) as tmpsession:
+            tmpsession.headers.update(self._default_http_header(empty_session_only=True))
+            del tmpsession.headers['Connection']
+            del tmpsession.headers['Content-Length']
+            tmpsession.headers['authority'] = 'www.instagram.com'
+            tmpsession.headers['scheme'] = 'https'
+            tmpsession.headers['accept'] = '*/*'
+            tmpsession.headers['x-csrftoken'] = csrf
+            if referer is not None:
+                tmpsession.headers['referer'] = urllib.parse.quote(referer)
+
+            variables_json = json.dumps(variables, separators=(',', ':'))
+
+            resp_json = self.get_json('graphql/query',
+                                      params={'variables': variables_json,
+                                              'doc_id': doc_id,
+                                              'server_timestamps': 'true'},
+                                      session=tmpsession,
+                                      use_post=True)
+        if 'status' not in resp_json:
             self.error("GraphQL response did not contain a \"status\" field.")
         return resp_json
 
@@ -777,25 +688,6 @@ class InstaloaderContext:
 
             return response
 
-    def _get_page(self, session: requests.Session, path: str) -> str:
-        """Load a page of ``www.instagram.com`` and return its HTML."""
-        self.do_sleep()
-        self._rate_controller.wait_before_query('other')
-        url = 'https://www.instagram.com/{0}'.format(path)
-        # Instagram only embeds the data when the request looks like a browser's page navigation.
-        headers = {'Accept': 'text/html', 'Sec-Fetch-Dest': 'document', 'Sec-Fetch-Mode': 'navigate',
-                   'Sec-Fetch-Site': 'none'}
-        try:
-            resp = session.get(url, headers=headers, allow_redirects=False)
-        except requests.exceptions.RequestException as err:
-            raise ConnectionException("Page request to {}: {}".format(url, err)) from err
-        self._raise_if_login_redirect(resp)
-        if resp.status_code == 404:
-            raise QueryReturnedNotFoundException(self._response_error(resp))
-        if resp.status_code != 200:
-            raise ConnectionException(self._response_error(resp))
-        return resp.text
-
     def get_page_data(self, path: str) -> List[Dict[str, Any]]:
         """Load a page of ``www.instagram.com`` as a logged-out visitor and return the data it embeds.
 
@@ -807,10 +699,23 @@ class InstaloaderContext:
         :raises QueryReturnedNotFoundException: When the server responds with a 404.
         :raises ConnectionException: When the request failed.
         """
-        with self.get_anonymous_session() as anonymous_session:
-            html = self._get_page(anonymous_session, path)
+        self.do_sleep()
+        self._rate_controller.wait_before_query('other')
+        url = 'https://www.instagram.com/{0}'.format(path)
+        # Instagram only embeds the data when the request looks like a browser's page navigation.
+        headers = {'Accept': 'text/html', 'Sec-Fetch-Dest': 'document', 'Sec-Fetch-Mode': 'navigate',
+                   'Sec-Fetch-Site': 'none'}
+        try:
+            with self.get_anonymous_session() as anonymous_session:
+                resp = anonymous_session.get(url, headers=headers, allow_redirects=False)
+        except requests.exceptions.RequestException as err:
+            raise ConnectionException("Page request to {}: {}".format(url, err)) from err
+        if resp.status_code == 404:
+            raise QueryReturnedNotFoundException(self._response_error(resp))
+        if resp.status_code != 200:
+            raise ConnectionException(self._response_error(resp))
         data: List[Dict[str, Any]] = []
-        for script in re.finditer(r'<script type="application/json"[^>]*>(.*?)</script>', html, re.DOTALL):
+        for script in re.finditer(r'<script type="application/json"[^>]*>(.*?)</script>', resp.text, re.DOTALL):
             with suppress(json.decoder.JSONDecodeError):
                 data.extend(_embedded_query_data(json.loads(script.group(1))))
         return data
@@ -827,26 +732,27 @@ class InstaloaderContext:
                 file.write(resp)
         os.replace(filename + '.temp', filename)
 
-    def get_raw(self, url: str) -> requests.Response:
+    def get_raw(self, url: str, _attempt=1) -> requests.Response:
         """Downloads a file anonymously.
 
-        This does not retry on its own; callers that need retrying are decorated with
-        ``_retry_on_connection_error``.
-
         :raises QueryReturnedNotFoundException: When the server responds with a 404.
         :raises QueryReturnedForbiddenException: When the server responds with a 403.
         :raises ConnectionException: When download failed.
 
-        .. versionadded:: 4.2.1
-
-        .. versionchanged:: 4.16
-           Removed the `_attempt` parameter.
-        """
+        .. versionadded:: 4.2.1"""
         with self.get_anonymous_session() as anonymous_session:
             resp = anonymous_session.get(url, stream=True)
-        self._raise_for_download_status(resp)
-        resp.raw.decode_content = True
-        return resp
+        if resp.status_code == 200:
+            resp.raw.decode_content = True
+            return resp
+        else:
+            if resp.status_code == 403:
+                # suspected invalid URL signature
+                raise QueryReturnedForbiddenException(self._response_error(resp))
+            if resp.status_code == 404:
+                # 404 not worth retrying.
+                raise QueryReturnedNotFoundException(self._response_error(resp))
+            raise ConnectionException(self._response_error(resp))
 
     def get_and_write_raw(self, url: str, filename: str) -> None:
         """Downloads and writes anonymously-requested raw data into a file.
@@ -867,8 +773,16 @@ class InstaloaderContext:
         """
         with self.get_anonymous_session() as anonymous_session:
             resp = anonymous_session.head(url, allow_redirects=allow_redirects)
-        self._raise_for_download_status(resp)
-        return resp
+        if resp.status_code == 200:
+            return resp
+        else:
+            if resp.status_code == 403:
+                # suspected invalid URL signature
+                raise QueryReturnedForbiddenException(self._response_error(resp))
+            if resp.status_code == 404:
+                # 404 not worth retrying.
+                raise QueryReturnedNotFoundException(self._response_error(resp))
+            raise ConnectionException(self._response_error(resp))
 
 
 class RateController:

instaloader/nodeiterator.py

@@ -69,9 +69,6 @@ class NodeIterator(Iterator[T]):
 
     .. versionchanged: 4.13
        Included support for `doc_id`-based queries (using POST method).
-
-    .. versionchanged: 4.16
-       Added `web_api` parameter, see :meth:`InstaloaderContext.doc_id_graphql_query`.
     """
 
     _graphql_page_length = 12
@@ -86,12 +83,10 @@ class NodeIterator(Iterator[T]):
                  query_referer: Optional[str] = None,
                  first_data: Optional[Dict[str, Any]] = None,
                  is_first: Optional[Callable[[T, Optional[T]], bool]] = None,
-                 doc_id: Optional[str] = None,
-                 web_api: bool = False):
+                 doc_id: Optional[str] = None):
         self._context = context
         self._query_hash = query_hash
         self._doc_id = doc_id
-        self._web_api = web_api
         self._edge_extractor = edge_extractor
         self._node_wrapper = node_wrapper
         self._query_variables = query_variables if query_variables is not None else {}
@@ -104,6 +99,7 @@ class NodeIterator(Iterator[T]):
         else:
             self._data = self._query()
         self._first_node: Optional[Dict] = None
+        self._first_item: Optional[T] = None
         self._is_first = is_first
 
     def _query(self, after: Optional[str] = None) -> Dict:
@@ -122,7 +118,7 @@ class NodeIterator(Iterator[T]):
             pagination_variables['last'] = None
         data = self._edge_extractor(
             self._context.doc_id_graphql_query(
-                doc_id, {**self._query_variables, **pagination_variables}, self._query_referer, self._web_api
+                doc_id, {**self._query_variables, **pagination_variables}, self._query_referer
             )
         )
         self._best_before = datetime.now() + NodeIterator._shelf_life
@@ -156,10 +152,10 @@ class NodeIterator(Iterator[T]):
             item = self._node_wrapper(node)
             if self._is_first is not None:
                 if self._is_first(item, self.first_item):
-                    self._first_node = node
+                    self._first_node, self._first_item = node, item
             else:
                 if self._first_node is None:
-                    self._first_node = node
+                    self._first_node, self._first_item = node, item
             return item
         if self._data.get('page_info', {}).get('has_next_page'):
             query_response = self._query(self._data['page_info']['end_cursor'])
@@ -206,7 +202,15 @@ class NodeIterator(Iterator[T]):
         .. versionchanged:: 4.9.2
            What is considered the first item can be overridden.
         """
-        return self._node_wrapper(self._first_node) if self._first_node is not None else None
+        if self._first_item is None:
+            # Restore _first_item from _first_node. Needed after thawing, since
+            # FrozenNodeIterator stores the node, not the item.
+            self._first_item = (
+                self._node_wrapper(self._first_node)
+                if self._first_node is not None
+                else None
+            )
+        return self._first_item
 
     @staticmethod
     def page_length() -> int:

instaloader/structures.py

@@ -879,19 +879,9 @@ class Post:
         if self.likes == 0:
             # Avoid doing additional requests if there are no comments
             return
-        likes_edges = self._field('edge_media_preview_like', 'edges')
-        if self.likes == len(likes_edges):
-            # If the Post's metadata already contains all likes, don't do GraphQL requests to obtain them
-            yield from (Profile(self._context, like['node']) for like in likes_edges)
-            return
-        yield from NodeIterator(
-            self._context,
-            '1cb6ec562846122743b61e492c85999f',
-            lambda d: d['data']['shortcode_media']['edge_liked_by'],
-            lambda n: Profile(self._context, n),
-            {'shortcode': self.shortcode},
-            'https://www.instagram.com/p/{0}/'.format(self.shortcode),
-        )
+        data = self._context.get_json(f"api/v1/media/{self.mediaid}/likers/", {})
+        for user in data["users"]:
+            yield Profile(self._context, user)
 
     @property
     def is_sponsored(self) -> bool:
@@ -1384,8 +1374,7 @@ class Profile:
     def get_posts(self) -> NodeIterator[Post]:
         """Retrieve all posts from a profile.
 
-        :rtype: NodeIterator[Post]
-        :raises PrivateProfileNotFollowedException: When logged in and the profile is private and not followed."""
+        :rtype: NodeIterator[Post]"""
         self._obtain_metadata()
         if not self._context.is_logged_in:
             return NodeIterator(
@@ -1407,20 +1396,13 @@ class Profile:
                 query_hash=None,
                 first_data=self._metadata("edge_owner_to_timeline_media"),
             )
-        # Instagram answers with an empty timeline rather than an error in that case. Unlike followed_by_viewer,
-        # which defaults to False, friendship_status tells whether the follow state is actually known.
-        if (self.is_private and self.username != (self._context.username or '').lower() and
-                (self._node.get('friendship_status') or {}).get('following') is False):
-            raise PrivateProfileNotFollowedException("Profile {} is private but not followed.".format(self.username))
         return NodeIterator(
             context=self._context,
             edge_extractor=lambda d: d["data"]["xdt_api__v1__feed__user_timeline_graphql_connection"],
             node_wrapper=lambda n: Post.from_iphone_struct(self._context, n),
             # The query and variables that the web app sends for a profile's posts tab
             # (PolarisProfilePostsTabContentQuery_connection). Instagram answers the
-            # previous queries, 7898261790222653 ("execution error") and 28975909992013618
-            # (400 "feedback_required"), with errors.
-            # It must be sent to the web app's api/graphql endpoint.
+            # previous query, 7898261790222653, with "execution error".
             query_variables={
                 "data": {
                     "count": 12,
@@ -1433,16 +1415,13 @@ class Profile:
                 "include_multi_captions": True,
                 "username": self.username,
                 "__relay_internal__pv__PolarisMultiCaptionCarouselEnabledrelayprovider": True,
-                "__relay_internal__pv__PolarisShortDramaEnabledrelayprovider": True,
-                "__relay_internal__pv__PolarisSEOIsCrawlerrelayprovider": False,
-                "__relay_internal__pv__PolarisThreadsStyleCommentsEnabledrelayprovider": False,
+                "__relay_internal__pv__PolarisShortDramaEnabledrelayprovider": False,
                 "__relay_internal__pv__PolarisReelsRecoDebugOverlayEnabledrelayprovider": False,
             },
             query_referer="https://www.instagram.com/{0}/".format(self.username),
             is_first=Profile._make_is_newest_checker(),
-            doc_id="29104886249145907",
+            doc_id="28975909992013618",
             query_hash=None,
-            web_api=True,
         )
 
     def get_saved_posts(self) -> NodeIterator[Post]: