mirror/instaloader
1
0
Fork 0
mirror of https://github.com/instaloader/instaloader.git synced 2026-10-16 13:48:03 -07:00
Code Issues Packages Projects Releases Wiki Activity

Use new doc_id based query to get posts (#2330)

Browse Source 
Fixes #2319.
This commit is contained in:
Eduardo Kalinowski
2024-08-10 17:27:07 +02:00
committed by GitHub
parent b0157d9bc1
commit ed21752365
3 changed files with 115 additions and 23 deletions
Download Patch File Download Diff File Expand all files Collapse all files
instaloader/instaloadercontext.py
+51 -4
View File
@@ -387,31 +387,42 @@ class InstaloaderContext:
def get_json(self, path: str, params: Dict[str, Any], host: str = 'www.instagram.com',
session: Optional[requests.Session] = None, _attempt=1,
response_headers: Optional[Dict[str, Any]] = None) -> Dict[str, Any]:
response_headers: Optional[Dict[str, Any]] = None,
use_post: bool = False) -> Dict[str, Any]:
"""JSON request to Instagram.
:param path: URL, relative to the given domain which defaults to www.instagram.com/
:param params: GET parameters
:param params: request parameters
:param host: Domain part of the URL from where to download the requested JSON; defaults to www.instagram.com
:param session: Session to use, or None to use self.session
:param use_post: Use POST instead of GET to make the request
:return: Decoded response dictionary
:raises QueryReturnedBadRequestException: When the server responds with a 400.
:raises QueryReturnedNotFoundException: When the server responds with a 404.
:raises ConnectionException: When query repeatedly failed.
.. versionchanged:: 4.13
Added `use_post` parameter.
"""
is_graphql_query = 'query_hash' in params and 'graphql/query' in path
is_doc_id_query = 'doc_id' in params and 'graphql/query' in path
is_iphone_query = host == 'i.instagram.com'
is_other_query = not is_graphql_query and host == "www.instagram.com"
is_other_query = not is_graphql_query and not is_doc_id_query and host == "www.instagram.com"
sess = session if session else self._session
try:
self.do_sleep()
if is_graphql_query:
self._rate_controller.wait_before_query(params['query_hash'])
if is_doc_id_query:
self._rate_controller.wait_before_query(params['doc_id'])
if is_iphone_query:
self._rate_controller.wait_before_query('iphone')
if is_other_query:
self._rate_controller.wait_before_query('other')
resp = sess.get('https://{0}/{1}'.format(host, path), params=params, allow_redirects=False)
if use_post:
resp = sess.post('https://{0}/{1}'.format(host, path), data=params, allow_redirects=False)
else:
resp = sess.get('https://{0}/{1}'.format(host, path), params=params, allow_redirects=False)
if resp.status_code in self.fatal_status_codes:
redirect = " redirect to {}".format(resp.headers['location']) if 'location' in resp.headers else ""
body = ""
@@ -462,6 +473,8 @@ class InstaloaderContext:
if isinstance(err, TooManyRequestsException):
if is_graphql_query:
self._rate_controller.handle_429(params['query_hash'])
if is_doc_id_query:
self._rate_controller.handle_429(params['doc_id'])
if is_iphone_query:
self._rate_controller.handle_429('iphone')
if is_other_query:
@@ -509,6 +522,40 @@ class InstaloaderContext:
self.error("GraphQL response did not contain a \"status\" field.")
return resp_json
def doc_id_graphql_query(self, doc_id: str, variables: Dict[str, Any],
referer: Optional[str] = None) -> Dict[str, Any]:
"""
Do a doc_id-based GraphQL Query using method POST.
.. versionadded:: 4.13
:param doc_id: doc_id for the query.
:param variables: Variables for the Query.
:param referer: HTTP Referer, or None.
:return: The server's response dictionary.
"""
with copy_session(self._session, self.request_timeout) as tmpsession:
tmpsession.headers.update(self._default_http_header(empty_session_only=True))
del tmpsession.headers['Connection']
del tmpsession.headers['Content-Length']
tmpsession.headers['authority'] = 'www.instagram.com'
tmpsession.headers['scheme'] = 'https'
tmpsession.headers['accept'] = '*/*'
if referer is not None:
tmpsession.headers['referer'] = urllib.parse.quote(referer)
variables_json = json.dumps(variables, separators=(',', ':'))
resp_json = self.get_json('graphql/query',
params={'variables': variables_json,
'doc_id': doc_id,
'server_timestamps': 'true'},
session=tmpsession,
use_post=True)
if 'status' not in resp_json:
self.error("GraphQL response did not contain a \"status\" field.")
return resp_json
def graphql_node_list(self, query_hash: str, query_variables: Dict[str, Any],
query_referer: Optional[str],
edge_extractor: Callable[[Dict[str, Any]], Dict[str, Any]],
instaloader/nodeiterator.py
+36 -5
View File
@@ -11,7 +11,7 @@ from .exceptions import AbortDownloadException, InvalidArgumentException
from .instaloadercontext import InstaloaderContext
class FrozenNodeIterator(NamedTuple):
query_hash: str
query_hash: Optional[str]
query_variables: Dict
query_referer: Optional[str]
context_username: Optional[str]
@@ -19,6 +19,7 @@ class FrozenNodeIterator(NamedTuple):
best_before: Optional[float]
remaining_data: Optional[Dict]
first_node: Optional[Dict]
doc_id: Optional[str]
FrozenNodeIterator.query_hash.__doc__ = """The GraphQL ``query_hash`` parameter."""
FrozenNodeIterator.query_variables.__doc__ = """The GraphQL ``query_variables`` parameter."""
FrozenNodeIterator.query_referer.__doc__ = """The HTTP referer used for the GraphQL query."""
@@ -28,6 +29,7 @@ FrozenNodeIterator.best_before.__doc__ = """Date when parts of the stored nodes
FrozenNodeIterator.remaining_data.__doc__ = \
"""The already-retrieved, yet-unprocessed ``edges`` and the ``page_info`` at time of freezing."""
FrozenNodeIterator.first_node.__doc__ = """Node data of the first item, if an item has been produced."""
FrozenNodeIterator.doc_id.__doc__ = """The GraphQL ``doc_id`` parameter."""
T = TypeVar('T')
@@ -64,6 +66,9 @@ class NodeIterator(Iterator[T]):
NodeIterators are matching if and only if they have the same magic.
See also :func:`resumable_iteration` for a high-level context manager that handles a resumable iteration.
.. versionchanged: 4.13
Included support for `doc_id`-based queries (using POST method).
"""
_graphql_page_length = 12
@@ -71,15 +76,17 @@ class NodeIterator(Iterator[T]):
def __init__(self,
context: InstaloaderContext,
query_hash: str,
query_hash: Optional[str],
edge_extractor: Callable[[Dict[str, Any]], Dict[str, Any]],
node_wrapper: Callable[[Dict], T],
query_variables: Optional[Dict[str, Any]] = None,
query_referer: Optional[str] = None,
first_data: Optional[Dict[str, Any]] = None,
is_first: Optional[Callable[[T, Optional[T]], bool]] = None):
is_first: Optional[Callable[[T, Optional[T]], bool]] = None,
doc_id: Optional[str] = None):
self._context = context
self._query_hash = query_hash
self._doc_id = doc_id
self._edge_extractor = edge_extractor
self._node_wrapper = node_wrapper
self._query_variables = query_variables if query_variables is not None else {}
@@ -95,12 +102,34 @@ class NodeIterator(Iterator[T]):
self._is_first = is_first
def _query(self, after: Optional[str] = None) -> Dict:
if self._doc_id is not None:
return self._query_doc_id(self._doc_id, after)
else:
assert self._query_hash is not None
return self._query_query_hash(self._query_hash, after)
def _query_doc_id(self, doc_id: str, after: Optional[str] = None) -> Dict:
pagination_variables: Dict[str, Any] = {'__relay_internal__pv__PolarisFeedShareMenurelayprovider': False}
if after is not None:
pagination_variables['after'] = after
pagination_variables['before'] = None
pagination_variables['first'] = 12
pagination_variables['last'] = None
data = self._edge_extractor(
self._context.doc_id_graphql_query(
doc_id, {**self._query_variables, **pagination_variables}, self._query_referer
)
)
self._best_before = datetime.now() + NodeIterator._shelf_life
return data
def _query_query_hash(self, query_hash: str, after: Optional[str] = None) -> Dict:
pagination_variables: Dict[str, Any] = {'first': NodeIterator._graphql_page_length}
if after is not None:
pagination_variables['after'] = after
data = self._edge_extractor(
self._context.graphql_query(
self._query_hash, {**self._query_variables, **pagination_variables}, self._query_referer
query_hash, {**self._query_variables, **pagination_variables}, self._query_referer
)
)
self._best_before = datetime.now() + NodeIterator._shelf_life
@@ -193,6 +222,7 @@ class NodeIterator(Iterator[T]):
best_before=self._best_before.timestamp() if self._best_before else None,
remaining_data=remaining_data,
first_node=self._first_node,
doc_id=self._doc_id,
)
def thaw(self, frozen: FrozenNodeIterator) -> None:
@@ -210,7 +240,8 @@ class NodeIterator(Iterator[T]):
if (self._query_hash != frozen.query_hash or
self._query_variables != frozen.query_variables or
self._query_referer != frozen.query_referer or
self._context.username != frozen.context_username):
self._context.username != frozen.context_username or
self._doc_id != frozen.doc_id):
raise InvalidArgumentException("Mismatching resume information.")
if not frozen.best_before:
raise InvalidArgumentException("\"best before\" date missing.")
instaloader/structures.py
+28 -14
View File
@@ -232,21 +232,32 @@ class Post:
"title": media.get("title"),
"viewer_has_liked": media["has_liked"],
"edge_media_preview_like": {"count": media["like_count"]},
"accessibility_caption": media.get("accessibility_caption"),
"comments": media.get("comment_count"),
"iphone_struct": media,
}
with suppress(KeyError):
fake_node["display_url"] = media['image_versions2']['candidates'][0]['url']
with suppress(KeyError):
with suppress(KeyError, TypeError):
fake_node["video_url"] = media['video_versions'][-1]['url']
fake_node["video_duration"] = media["video_duration"]
fake_node["video_view_count"] = media["view_count"]
with suppress(KeyError):
fake_node["edge_sidecar_to_children"] = {"edges": [{"node": {
"display_url": node['image_versions2']['candidates'][0]['url'],
"is_video": media_types[node["media_type"]] == "GraphVideo",
}} for node in media["carousel_media"]]}
with suppress(KeyError, TypeError):
fake_node["edge_sidecar_to_children"] = {"edges": [{"node":
Post._convert_iphone_carousel(node, media_types)}
for node in media["carousel_media"]]}
return cls(context, fake_node, Profile.from_iphone_struct(context, media["user"]) if "user" in media else None)
@staticmethod
def _convert_iphone_carousel(iphone_node: Dict[str, Any], media_types: Dict[int, str]) -> Dict[str, Any]:
fake_node = {
"display_url": iphone_node["image_versions2"]["candidates"][0]["url"],
"is_video": media_types[iphone_node["media_type"]] == "GraphVideo",
}
if "video_versions" in iphone_node and iphone_node["video_versions"] is not None:
fake_node["video_url"] = iphone_node["video_versions"][0]["url"]
return fake_node
@staticmethod
def shortcode_to_mediaid(code: str) -> int:
if len(code) > 11:
@@ -1174,14 +1185,17 @@ class Profile:
:rtype: NodeIterator[Post]"""
self._obtain_metadata()
return NodeIterator(
self._context,
'003056d32c2554def87228bc3fd9668a',
lambda d: d['data']['user']['edge_owner_to_timeline_media'],
lambda n: Post(self._context, n, self),
{'id': self.userid},
'https://www.instagram.com/{0}/'.format(self.username),
self._metadata('edge_owner_to_timeline_media'),
Profile._make_is_newest_checker()
context = self._context,
edge_extractor = lambda d: d['data']['xdt_api__v1__feed__user_timeline_graphql_connection'],
node_wrapper = lambda n: Post.from_iphone_struct(self._context, n),
query_variables = {'data': {
'count': 12, 'include_relationship_info': True,
'latest_besties_reel_media': True, 'latest_reel_media': True},
'username': self.username},
query_referer = 'https://www.instagram.com/{0}/'.format(self.username),
is_first = Profile._make_is_newest_checker(),
doc_id = '7898261790222653',
query_hash = None,
)
def get_saved_posts(self) -> NodeIterator[Post]: