From ed21752365ef4cc0b2891198b449059e2089fef4 Mon Sep 17 00:00:00 2001 From: Eduardo Kalinowski Date: Sat, 10 Aug 2024 12:27:07 -0300 Subject: [PATCH] Use new doc_id based query to get posts (#2330) Fixes #2319. --- instaloader/instaloadercontext.py | 55 ++++++++++++++++++++++++++++--- instaloader/nodeiterator.py | 41 ++++++++++++++++++++--- instaloader/structures.py | 42 +++++++++++++++-------- 3 files changed, 115 insertions(+), 23 deletions(-) diff --git a/instaloader/instaloadercontext.py b/instaloader/instaloadercontext.py index 4508a5f..59101ed 100644 --- a/instaloader/instaloadercontext.py +++ b/instaloader/instaloadercontext.py @@ -387,31 +387,42 @@ class InstaloaderContext: def get_json(self, path: str, params: Dict[str, Any], host: str = 'www.instagram.com', session: Optional[requests.Session] = None, _attempt=1, - response_headers: Optional[Dict[str, Any]] = None) -> Dict[str, Any]: + response_headers: Optional[Dict[str, Any]] = None, + use_post: bool = False) -> Dict[str, Any]: """JSON request to Instagram. :param path: URL, relative to the given domain which defaults to www.instagram.com/ - :param params: GET parameters + :param params: request parameters :param host: Domain part of the URL from where to download the requested JSON; defaults to www.instagram.com :param session: Session to use, or None to use self.session + :param use_post: Use POST instead of GET to make the request :return: Decoded response dictionary :raises QueryReturnedBadRequestException: When the server responds with a 400. :raises QueryReturnedNotFoundException: When the server responds with a 404. :raises ConnectionException: When query repeatedly failed. + + .. versionchanged:: 4.13 + Added `use_post` parameter. """ is_graphql_query = 'query_hash' in params and 'graphql/query' in path + is_doc_id_query = 'doc_id' in params and 'graphql/query' in path is_iphone_query = host == 'i.instagram.com' - is_other_query = not is_graphql_query and host == "www.instagram.com" + is_other_query = not is_graphql_query and not is_doc_id_query and host == "www.instagram.com" sess = session if session else self._session try: self.do_sleep() if is_graphql_query: self._rate_controller.wait_before_query(params['query_hash']) + if is_doc_id_query: + self._rate_controller.wait_before_query(params['doc_id']) if is_iphone_query: self._rate_controller.wait_before_query('iphone') if is_other_query: self._rate_controller.wait_before_query('other') - resp = sess.get('https://{0}/{1}'.format(host, path), params=params, allow_redirects=False) + if use_post: + resp = sess.post('https://{0}/{1}'.format(host, path), data=params, allow_redirects=False) + else: + resp = sess.get('https://{0}/{1}'.format(host, path), params=params, allow_redirects=False) if resp.status_code in self.fatal_status_codes: redirect = " redirect to {}".format(resp.headers['location']) if 'location' in resp.headers else "" body = "" @@ -462,6 +473,8 @@ class InstaloaderContext: if isinstance(err, TooManyRequestsException): if is_graphql_query: self._rate_controller.handle_429(params['query_hash']) + if is_doc_id_query: + self._rate_controller.handle_429(params['doc_id']) if is_iphone_query: self._rate_controller.handle_429('iphone') if is_other_query: @@ -509,6 +522,40 @@ class InstaloaderContext: self.error("GraphQL response did not contain a \"status\" field.") return resp_json + def doc_id_graphql_query(self, doc_id: str, variables: Dict[str, Any], + referer: Optional[str] = None) -> Dict[str, Any]: + """ + Do a doc_id-based GraphQL Query using method POST. + + .. versionadded:: 4.13 + + :param doc_id: doc_id for the query. + :param variables: Variables for the Query. + :param referer: HTTP Referer, or None. + :return: The server's response dictionary. + """ + with copy_session(self._session, self.request_timeout) as tmpsession: + tmpsession.headers.update(self._default_http_header(empty_session_only=True)) + del tmpsession.headers['Connection'] + del tmpsession.headers['Content-Length'] + tmpsession.headers['authority'] = 'www.instagram.com' + tmpsession.headers['scheme'] = 'https' + tmpsession.headers['accept'] = '*/*' + if referer is not None: + tmpsession.headers['referer'] = urllib.parse.quote(referer) + + variables_json = json.dumps(variables, separators=(',', ':')) + + resp_json = self.get_json('graphql/query', + params={'variables': variables_json, + 'doc_id': doc_id, + 'server_timestamps': 'true'}, + session=tmpsession, + use_post=True) + if 'status' not in resp_json: + self.error("GraphQL response did not contain a \"status\" field.") + return resp_json + def graphql_node_list(self, query_hash: str, query_variables: Dict[str, Any], query_referer: Optional[str], edge_extractor: Callable[[Dict[str, Any]], Dict[str, Any]], diff --git a/instaloader/nodeiterator.py b/instaloader/nodeiterator.py index 2c8b583..b573287 100644 --- a/instaloader/nodeiterator.py +++ b/instaloader/nodeiterator.py @@ -11,7 +11,7 @@ from .exceptions import AbortDownloadException, InvalidArgumentException from .instaloadercontext import InstaloaderContext class FrozenNodeIterator(NamedTuple): - query_hash: str + query_hash: Optional[str] query_variables: Dict query_referer: Optional[str] context_username: Optional[str] @@ -19,6 +19,7 @@ class FrozenNodeIterator(NamedTuple): best_before: Optional[float] remaining_data: Optional[Dict] first_node: Optional[Dict] + doc_id: Optional[str] FrozenNodeIterator.query_hash.__doc__ = """The GraphQL ``query_hash`` parameter.""" FrozenNodeIterator.query_variables.__doc__ = """The GraphQL ``query_variables`` parameter.""" FrozenNodeIterator.query_referer.__doc__ = """The HTTP referer used for the GraphQL query.""" @@ -28,6 +29,7 @@ FrozenNodeIterator.best_before.__doc__ = """Date when parts of the stored nodes FrozenNodeIterator.remaining_data.__doc__ = \ """The already-retrieved, yet-unprocessed ``edges`` and the ``page_info`` at time of freezing.""" FrozenNodeIterator.first_node.__doc__ = """Node data of the first item, if an item has been produced.""" +FrozenNodeIterator.doc_id.__doc__ = """The GraphQL ``doc_id`` parameter.""" T = TypeVar('T') @@ -64,6 +66,9 @@ class NodeIterator(Iterator[T]): NodeIterators are matching if and only if they have the same magic. See also :func:`resumable_iteration` for a high-level context manager that handles a resumable iteration. + + .. versionchanged: 4.13 + Included support for `doc_id`-based queries (using POST method). """ _graphql_page_length = 12 @@ -71,15 +76,17 @@ class NodeIterator(Iterator[T]): def __init__(self, context: InstaloaderContext, - query_hash: str, + query_hash: Optional[str], edge_extractor: Callable[[Dict[str, Any]], Dict[str, Any]], node_wrapper: Callable[[Dict], T], query_variables: Optional[Dict[str, Any]] = None, query_referer: Optional[str] = None, first_data: Optional[Dict[str, Any]] = None, - is_first: Optional[Callable[[T, Optional[T]], bool]] = None): + is_first: Optional[Callable[[T, Optional[T]], bool]] = None, + doc_id: Optional[str] = None): self._context = context self._query_hash = query_hash + self._doc_id = doc_id self._edge_extractor = edge_extractor self._node_wrapper = node_wrapper self._query_variables = query_variables if query_variables is not None else {} @@ -95,12 +102,34 @@ class NodeIterator(Iterator[T]): self._is_first = is_first def _query(self, after: Optional[str] = None) -> Dict: + if self._doc_id is not None: + return self._query_doc_id(self._doc_id, after) + else: + assert self._query_hash is not None + return self._query_query_hash(self._query_hash, after) + + def _query_doc_id(self, doc_id: str, after: Optional[str] = None) -> Dict: + pagination_variables: Dict[str, Any] = {'__relay_internal__pv__PolarisFeedShareMenurelayprovider': False} + if after is not None: + pagination_variables['after'] = after + pagination_variables['before'] = None + pagination_variables['first'] = 12 + pagination_variables['last'] = None + data = self._edge_extractor( + self._context.doc_id_graphql_query( + doc_id, {**self._query_variables, **pagination_variables}, self._query_referer + ) + ) + self._best_before = datetime.now() + NodeIterator._shelf_life + return data + + def _query_query_hash(self, query_hash: str, after: Optional[str] = None) -> Dict: pagination_variables: Dict[str, Any] = {'first': NodeIterator._graphql_page_length} if after is not None: pagination_variables['after'] = after data = self._edge_extractor( self._context.graphql_query( - self._query_hash, {**self._query_variables, **pagination_variables}, self._query_referer + query_hash, {**self._query_variables, **pagination_variables}, self._query_referer ) ) self._best_before = datetime.now() + NodeIterator._shelf_life @@ -193,6 +222,7 @@ class NodeIterator(Iterator[T]): best_before=self._best_before.timestamp() if self._best_before else None, remaining_data=remaining_data, first_node=self._first_node, + doc_id=self._doc_id, ) def thaw(self, frozen: FrozenNodeIterator) -> None: @@ -210,7 +240,8 @@ class NodeIterator(Iterator[T]): if (self._query_hash != frozen.query_hash or self._query_variables != frozen.query_variables or self._query_referer != frozen.query_referer or - self._context.username != frozen.context_username): + self._context.username != frozen.context_username or + self._doc_id != frozen.doc_id): raise InvalidArgumentException("Mismatching resume information.") if not frozen.best_before: raise InvalidArgumentException("\"best before\" date missing.") diff --git a/instaloader/structures.py b/instaloader/structures.py index 3670df4..1dd43d9 100644 --- a/instaloader/structures.py +++ b/instaloader/structures.py @@ -232,21 +232,32 @@ class Post: "title": media.get("title"), "viewer_has_liked": media["has_liked"], "edge_media_preview_like": {"count": media["like_count"]}, + "accessibility_caption": media.get("accessibility_caption"), + "comments": media.get("comment_count"), "iphone_struct": media, } with suppress(KeyError): fake_node["display_url"] = media['image_versions2']['candidates'][0]['url'] - with suppress(KeyError): + with suppress(KeyError, TypeError): fake_node["video_url"] = media['video_versions'][-1]['url'] fake_node["video_duration"] = media["video_duration"] fake_node["video_view_count"] = media["view_count"] - with suppress(KeyError): - fake_node["edge_sidecar_to_children"] = {"edges": [{"node": { - "display_url": node['image_versions2']['candidates'][0]['url'], - "is_video": media_types[node["media_type"]] == "GraphVideo", - }} for node in media["carousel_media"]]} + with suppress(KeyError, TypeError): + fake_node["edge_sidecar_to_children"] = {"edges": [{"node": + Post._convert_iphone_carousel(node, media_types)} + for node in media["carousel_media"]]} return cls(context, fake_node, Profile.from_iphone_struct(context, media["user"]) if "user" in media else None) + @staticmethod + def _convert_iphone_carousel(iphone_node: Dict[str, Any], media_types: Dict[int, str]) -> Dict[str, Any]: + fake_node = { + "display_url": iphone_node["image_versions2"]["candidates"][0]["url"], + "is_video": media_types[iphone_node["media_type"]] == "GraphVideo", + } + if "video_versions" in iphone_node and iphone_node["video_versions"] is not None: + fake_node["video_url"] = iphone_node["video_versions"][0]["url"] + return fake_node + @staticmethod def shortcode_to_mediaid(code: str) -> int: if len(code) > 11: @@ -1174,14 +1185,17 @@ class Profile: :rtype: NodeIterator[Post]""" self._obtain_metadata() return NodeIterator( - self._context, - '003056d32c2554def87228bc3fd9668a', - lambda d: d['data']['user']['edge_owner_to_timeline_media'], - lambda n: Post(self._context, n, self), - {'id': self.userid}, - 'https://www.instagram.com/{0}/'.format(self.username), - self._metadata('edge_owner_to_timeline_media'), - Profile._make_is_newest_checker() + context = self._context, + edge_extractor = lambda d: d['data']['xdt_api__v1__feed__user_timeline_graphql_connection'], + node_wrapper = lambda n: Post.from_iphone_struct(self._context, n), + query_variables = {'data': { + 'count': 12, 'include_relationship_info': True, + 'latest_besties_reel_media': True, 'latest_reel_media': True}, + 'username': self.username}, + query_referer = 'https://www.instagram.com/{0}/'.format(self.username), + is_first = Profile._make_is_newest_checker(), + doc_id = '7898261790222653', + query_hash = None, ) def get_saved_posts(self) -> NodeIterator[Post]: