Use new doc_id based query to get posts (#2330)

Fixes #2319.

instaloader/instaloadercontext.py

@@ -387,31 +387,42 @@ class InstaloaderContext:
 
     def get_json(self, path: str, params: Dict[str, Any], host: str = 'www.instagram.com',
                  session: Optional[requests.Session] = None, _attempt=1,
-                 response_headers: Optional[Dict[str, Any]] = None) -> Dict[str, Any]:
+                 response_headers: Optional[Dict[str, Any]] = None,
+                 use_post: bool = False) -> Dict[str, Any]:
         """JSON request to Instagram.
 
         :param path: URL, relative to the given domain which defaults to www.instagram.com/
-        :param params: GET parameters
+        :param params: request parameters
         :param host: Domain part of the URL from where to download the requested JSON; defaults to www.instagram.com
         :param session: Session to use, or None to use self.session
+        :param use_post: Use POST instead of GET to make the request
         :return: Decoded response dictionary
         :raises QueryReturnedBadRequestException: When the server responds with a 400.
         :raises QueryReturnedNotFoundException: When the server responds with a 404.
         :raises ConnectionException: When query repeatedly failed.
+
+        .. versionchanged:: 4.13
+           Added `use_post` parameter.
         """
         is_graphql_query = 'query_hash' in params and 'graphql/query' in path
+        is_doc_id_query = 'doc_id' in params and 'graphql/query' in path
         is_iphone_query = host == 'i.instagram.com'
-        is_other_query = not is_graphql_query and host == "www.instagram.com"
+        is_other_query = not is_graphql_query and not is_doc_id_query and host == "www.instagram.com"
         sess = session if session else self._session
         try:
             self.do_sleep()
             if is_graphql_query:
                 self._rate_controller.wait_before_query(params['query_hash'])
+            if is_doc_id_query:
+                self._rate_controller.wait_before_query(params['doc_id'])
             if is_iphone_query:
                 self._rate_controller.wait_before_query('iphone')
             if is_other_query:
                 self._rate_controller.wait_before_query('other')
-            resp = sess.get('https://{0}/{1}'.format(host, path), params=params, allow_redirects=False)
+            if use_post:
+                resp = sess.post('https://{0}/{1}'.format(host, path), data=params, allow_redirects=False)
+            else:
+                resp = sess.get('https://{0}/{1}'.format(host, path), params=params, allow_redirects=False)
             if resp.status_code in self.fatal_status_codes:
                 redirect = " redirect to {}".format(resp.headers['location']) if 'location' in resp.headers else ""
                 body = ""
@@ -462,6 +473,8 @@ class InstaloaderContext:
                 if isinstance(err, TooManyRequestsException):
                     if is_graphql_query:
                         self._rate_controller.handle_429(params['query_hash'])
+                    if is_doc_id_query:
+                        self._rate_controller.handle_429(params['doc_id'])
                     if is_iphone_query:
                         self._rate_controller.handle_429('iphone')
                     if is_other_query:
@@ -509,6 +522,40 @@ class InstaloaderContext:
             self.error("GraphQL response did not contain a \"status\" field.")
         return resp_json
 
+    def doc_id_graphql_query(self, doc_id: str, variables: Dict[str, Any],
+                             referer: Optional[str] = None) -> Dict[str, Any]:
+        """
+        Do a doc_id-based GraphQL Query using method POST.
+
+        .. versionadded:: 4.13
+
+        :param doc_id: doc_id for the query.
+        :param variables: Variables for the Query.
+        :param referer: HTTP Referer, or None.
+        :return: The server's response dictionary.
+        """
+        with copy_session(self._session, self.request_timeout) as tmpsession:
+            tmpsession.headers.update(self._default_http_header(empty_session_only=True))
+            del tmpsession.headers['Connection']
+            del tmpsession.headers['Content-Length']
+            tmpsession.headers['authority'] = 'www.instagram.com'
+            tmpsession.headers['scheme'] = 'https'
+            tmpsession.headers['accept'] = '*/*'
+            if referer is not None:
+                tmpsession.headers['referer'] = urllib.parse.quote(referer)
+
+            variables_json = json.dumps(variables, separators=(',', ':'))
+
+            resp_json = self.get_json('graphql/query',
+                                      params={'variables': variables_json,
+                                              'doc_id': doc_id,
+                                              'server_timestamps': 'true'},
+                                      session=tmpsession,
+                                      use_post=True)
+        if 'status' not in resp_json:
+            self.error("GraphQL response did not contain a \"status\" field.")
+        return resp_json
+
     def graphql_node_list(self, query_hash: str, query_variables: Dict[str, Any],
                           query_referer: Optional[str],
                           edge_extractor: Callable[[Dict[str, Any]], Dict[str, Any]],

instaloader/nodeiterator.py

@@ -11,7 +11,7 @@ from .exceptions import AbortDownloadException, InvalidArgumentException
 from .instaloadercontext import InstaloaderContext
 
 class FrozenNodeIterator(NamedTuple):
-    query_hash: str
+    query_hash: Optional[str]
     query_variables: Dict
     query_referer: Optional[str]
     context_username: Optional[str]
@@ -19,6 +19,7 @@ class FrozenNodeIterator(NamedTuple):
     best_before: Optional[float]
     remaining_data: Optional[Dict]
     first_node: Optional[Dict]
+    doc_id: Optional[str]
 FrozenNodeIterator.query_hash.__doc__ = """The GraphQL ``query_hash`` parameter."""
 FrozenNodeIterator.query_variables.__doc__ = """The GraphQL ``query_variables`` parameter."""
 FrozenNodeIterator.query_referer.__doc__ = """The HTTP referer used for the GraphQL query."""
@@ -28,6 +29,7 @@ FrozenNodeIterator.best_before.__doc__ = """Date when parts of the stored nodes
 FrozenNodeIterator.remaining_data.__doc__ = \
     """The already-retrieved, yet-unprocessed ``edges`` and the ``page_info`` at time of freezing."""
 FrozenNodeIterator.first_node.__doc__ = """Node data of the first item, if an item has been produced."""
+FrozenNodeIterator.doc_id.__doc__ = """The GraphQL ``doc_id`` parameter."""
 
 T = TypeVar('T')
 
@@ -64,6 +66,9 @@ class NodeIterator(Iterator[T]):
     NodeIterators are matching if and only if they have the same magic.
 
     See also :func:`resumable_iteration` for a high-level context manager that handles a resumable iteration.
+
+    .. versionchanged: 4.13
+       Included support for `doc_id`-based queries (using POST method).
     """
 
     _graphql_page_length = 12
@@ -71,15 +76,17 @@ class NodeIterator(Iterator[T]):
 
     def __init__(self,
                  context: InstaloaderContext,
-                 query_hash: str,
+                 query_hash: Optional[str],
                  edge_extractor: Callable[[Dict[str, Any]], Dict[str, Any]],
                  node_wrapper: Callable[[Dict], T],
                  query_variables: Optional[Dict[str, Any]] = None,
                  query_referer: Optional[str] = None,
                  first_data: Optional[Dict[str, Any]] = None,
-                 is_first: Optional[Callable[[T, Optional[T]], bool]] = None):
+                 is_first: Optional[Callable[[T, Optional[T]], bool]] = None,
+                 doc_id: Optional[str] = None):
         self._context = context
         self._query_hash = query_hash
+        self._doc_id = doc_id
         self._edge_extractor = edge_extractor
         self._node_wrapper = node_wrapper
         self._query_variables = query_variables if query_variables is not None else {}
@@ -95,12 +102,34 @@ class NodeIterator(Iterator[T]):
         self._is_first = is_first
 
     def _query(self, after: Optional[str] = None) -> Dict:
+        if self._doc_id is not None:
+            return self._query_doc_id(self._doc_id, after)
+        else:
+            assert self._query_hash is not None
+            return self._query_query_hash(self._query_hash, after)
+
+    def _query_doc_id(self, doc_id: str, after: Optional[str] = None) -> Dict:
+        pagination_variables: Dict[str, Any] = {'__relay_internal__pv__PolarisFeedShareMenurelayprovider': False}
+        if after is not None:
+            pagination_variables['after'] = after
+            pagination_variables['before'] = None
+            pagination_variables['first'] = 12
+            pagination_variables['last'] = None
+        data = self._edge_extractor(
+            self._context.doc_id_graphql_query(
+                doc_id, {**self._query_variables, **pagination_variables}, self._query_referer
+            )
+        )
+        self._best_before = datetime.now() + NodeIterator._shelf_life
+        return data
+
+    def _query_query_hash(self, query_hash: str, after: Optional[str] = None) -> Dict:
         pagination_variables: Dict[str, Any] = {'first': NodeIterator._graphql_page_length}
         if after is not None:
             pagination_variables['after'] = after
         data = self._edge_extractor(
             self._context.graphql_query(
-                self._query_hash, {**self._query_variables, **pagination_variables}, self._query_referer
+                query_hash, {**self._query_variables, **pagination_variables}, self._query_referer
             )
         )
         self._best_before = datetime.now() + NodeIterator._shelf_life
@@ -193,6 +222,7 @@ class NodeIterator(Iterator[T]):
             best_before=self._best_before.timestamp() if self._best_before else None,
             remaining_data=remaining_data,
             first_node=self._first_node,
+            doc_id=self._doc_id,
         )
 
     def thaw(self, frozen: FrozenNodeIterator) -> None:
@@ -210,7 +240,8 @@ class NodeIterator(Iterator[T]):
         if (self._query_hash != frozen.query_hash or
                 self._query_variables != frozen.query_variables or
                 self._query_referer != frozen.query_referer or
-                self._context.username != frozen.context_username):
+                self._context.username != frozen.context_username or
+                self._doc_id != frozen.doc_id):
             raise InvalidArgumentException("Mismatching resume information.")
         if not frozen.best_before:
             raise InvalidArgumentException("\"best before\" date missing.")

instaloader/structures.py

@@ -232,21 +232,32 @@ class Post:
             "title": media.get("title"),
             "viewer_has_liked": media["has_liked"],
             "edge_media_preview_like": {"count": media["like_count"]},
+            "accessibility_caption": media.get("accessibility_caption"),
+            "comments": media.get("comment_count"),
             "iphone_struct": media,
         }
         with suppress(KeyError):
             fake_node["display_url"] = media['image_versions2']['candidates'][0]['url']
-        with suppress(KeyError):
+        with suppress(KeyError, TypeError):
             fake_node["video_url"] = media['video_versions'][-1]['url']
             fake_node["video_duration"] = media["video_duration"]
             fake_node["video_view_count"] = media["view_count"]
-        with suppress(KeyError):
+        with suppress(KeyError, TypeError):
-            fake_node["edge_sidecar_to_children"] = {"edges": [{"node": {
+            fake_node["edge_sidecar_to_children"] = {"edges": [{"node":
-                "display_url": node['image_versions2']['candidates'][0]['url'],
+                Post._convert_iphone_carousel(node, media_types)}
-                "is_video": media_types[node["media_type"]] == "GraphVideo",
+                for node in media["carousel_media"]]}
-            }} for node in media["carousel_media"]]}
         return cls(context, fake_node, Profile.from_iphone_struct(context, media["user"]) if "user" in media else None)
 
+    @staticmethod
+    def _convert_iphone_carousel(iphone_node: Dict[str, Any], media_types: Dict[int, str]) -> Dict[str, Any]:
+        fake_node = {
+            "display_url": iphone_node["image_versions2"]["candidates"][0]["url"],
+            "is_video": media_types[iphone_node["media_type"]] == "GraphVideo",
+        }
+        if "video_versions" in iphone_node and iphone_node["video_versions"] is not None:
+            fake_node["video_url"] = iphone_node["video_versions"][0]["url"]
+        return fake_node
+
     @staticmethod
     def shortcode_to_mediaid(code: str) -> int:
         if len(code) > 11:
@@ -1174,14 +1185,17 @@ class Profile:
         :rtype: NodeIterator[Post]"""
         self._obtain_metadata()
         return NodeIterator(
-            self._context,
+            context = self._context,
-            '003056d32c2554def87228bc3fd9668a',
+            edge_extractor = lambda d: d['data']['xdt_api__v1__feed__user_timeline_graphql_connection'],
-            lambda d: d['data']['user']['edge_owner_to_timeline_media'],
+            node_wrapper = lambda n: Post.from_iphone_struct(self._context, n),
-            lambda n: Post(self._context, n, self),
+            query_variables = {'data': {
-            {'id': self.userid},
+                'count': 12, 'include_relationship_info': True,
-            'https://www.instagram.com/{0}/'.format(self.username),
+                'latest_besties_reel_media': True, 'latest_reel_media': True},
-            self._metadata('edge_owner_to_timeline_media'),
+             'username': self.username},
-            Profile._make_is_newest_checker()
+            query_referer = 'https://www.instagram.com/{0}/'.format(self.username),
+            is_first = Profile._make_is_newest_checker(),
+            doc_id = '7898261790222653',
+            query_hash = None,
         )
 
     def get_saved_posts(self) -> NodeIterator[Post]: