Use new doc_id based query to get posts (#2330)

Fixes #2319.

instaloader/nodeiterator.py

@@ -11,7 +11,7 @@ from .exceptions import AbortDownloadException, InvalidArgumentException
 from .instaloadercontext import InstaloaderContext
 
 class FrozenNodeIterator(NamedTuple):
-    query_hash: str
+    query_hash: Optional[str]
     query_variables: Dict
     query_referer: Optional[str]
     context_username: Optional[str]
@@ -19,6 +19,7 @@ class FrozenNodeIterator(NamedTuple):
     best_before: Optional[float]
     remaining_data: Optional[Dict]
     first_node: Optional[Dict]
+    doc_id: Optional[str]
 FrozenNodeIterator.query_hash.__doc__ = """The GraphQL ``query_hash`` parameter."""
 FrozenNodeIterator.query_variables.__doc__ = """The GraphQL ``query_variables`` parameter."""
 FrozenNodeIterator.query_referer.__doc__ = """The HTTP referer used for the GraphQL query."""
@@ -28,6 +29,7 @@ FrozenNodeIterator.best_before.__doc__ = """Date when parts of the stored nodes
 FrozenNodeIterator.remaining_data.__doc__ = \
     """The already-retrieved, yet-unprocessed ``edges`` and the ``page_info`` at time of freezing."""
 FrozenNodeIterator.first_node.__doc__ = """Node data of the first item, if an item has been produced."""
+FrozenNodeIterator.doc_id.__doc__ = """The GraphQL ``doc_id`` parameter."""
 
 T = TypeVar('T')
 
@@ -64,6 +66,9 @@ class NodeIterator(Iterator[T]):
     NodeIterators are matching if and only if they have the same magic.
 
     See also :func:`resumable_iteration` for a high-level context manager that handles a resumable iteration.
+
+    .. versionchanged: 4.13
+       Included support for `doc_id`-based queries (using POST method).
     """
 
     _graphql_page_length = 12
@@ -71,15 +76,17 @@ class NodeIterator(Iterator[T]):
 
     def __init__(self,
                  context: InstaloaderContext,
-                 query_hash: str,
+                 query_hash: Optional[str],
                  edge_extractor: Callable[[Dict[str, Any]], Dict[str, Any]],
                  node_wrapper: Callable[[Dict], T],
                  query_variables: Optional[Dict[str, Any]] = None,
                  query_referer: Optional[str] = None,
                  first_data: Optional[Dict[str, Any]] = None,
-                 is_first: Optional[Callable[[T, Optional[T]], bool]] = None):
+                 is_first: Optional[Callable[[T, Optional[T]], bool]] = None,
+                 doc_id: Optional[str] = None):
         self._context = context
         self._query_hash = query_hash
+        self._doc_id = doc_id
         self._edge_extractor = edge_extractor
         self._node_wrapper = node_wrapper
         self._query_variables = query_variables if query_variables is not None else {}
@@ -95,12 +102,34 @@ class NodeIterator(Iterator[T]):
         self._is_first = is_first
 
     def _query(self, after: Optional[str] = None) -> Dict:
+        if self._doc_id is not None:
+            return self._query_doc_id(self._doc_id, after)
+        else:
+            assert self._query_hash is not None
+            return self._query_query_hash(self._query_hash, after)
+
+    def _query_doc_id(self, doc_id: str, after: Optional[str] = None) -> Dict:
+        pagination_variables: Dict[str, Any] = {'__relay_internal__pv__PolarisFeedShareMenurelayprovider': False}
+        if after is not None:
+            pagination_variables['after'] = after
+            pagination_variables['before'] = None
+            pagination_variables['first'] = 12
+            pagination_variables['last'] = None
+        data = self._edge_extractor(
+            self._context.doc_id_graphql_query(
+                doc_id, {**self._query_variables, **pagination_variables}, self._query_referer
+            )
+        )
+        self._best_before = datetime.now() + NodeIterator._shelf_life
+        return data
+
+    def _query_query_hash(self, query_hash: str, after: Optional[str] = None) -> Dict:
         pagination_variables: Dict[str, Any] = {'first': NodeIterator._graphql_page_length}
         if after is not None:
             pagination_variables['after'] = after
         data = self._edge_extractor(
             self._context.graphql_query(
-                self._query_hash, {**self._query_variables, **pagination_variables}, self._query_referer
+                query_hash, {**self._query_variables, **pagination_variables}, self._query_referer
             )
         )
         self._best_before = datetime.now() + NodeIterator._shelf_life
@@ -193,6 +222,7 @@ class NodeIterator(Iterator[T]):
             best_before=self._best_before.timestamp() if self._best_before else None,
             remaining_data=remaining_data,
             first_node=self._first_node,
+            doc_id=self._doc_id,
         )
 
     def thaw(self, frozen: FrozenNodeIterator) -> None:
@@ -210,7 +240,8 @@ class NodeIterator(Iterator[T]):
         if (self._query_hash != frozen.query_hash or
                 self._query_variables != frozen.query_variables or
                 self._query_referer != frozen.query_referer or
-                self._context.username != frozen.context_username):
+                self._context.username != frozen.context_username or
+                self._doc_id != frozen.doc_id):
             raise InvalidArgumentException("Mismatching resume information.")
         if not frozen.best_before:
             raise InvalidArgumentException("\"best before\" date missing.")