diff --git a/instaloader/instaloadercontext.py b/instaloader/instaloadercontext.py index 61a6084..369c635 100644 --- a/instaloader/instaloadercontext.py +++ b/instaloader/instaloadercontext.py @@ -742,6 +742,25 @@ class InstaloaderContext: return response + def _get_page(self, session: requests.Session, path: str) -> str: + """Load a page of ``www.instagram.com`` and return its HTML.""" + self.do_sleep() + self._rate_controller.wait_before_query('other') + url = 'https://www.instagram.com/{0}'.format(path) + # Instagram only embeds the data when the request looks like a browser's page navigation. + headers = {'Accept': 'text/html', 'Sec-Fetch-Dest': 'document', 'Sec-Fetch-Mode': 'navigate', + 'Sec-Fetch-Site': 'none'} + try: + resp = session.get(url, headers=headers, allow_redirects=False) + except requests.exceptions.RequestException as err: + raise ConnectionException("Page request to {}: {}".format(url, err)) from err + self._raise_if_login_redirect(resp) + if resp.status_code == 404: + raise QueryReturnedNotFoundException(self._response_error(resp)) + if resp.status_code != 200: + raise ConnectionException(self._response_error(resp)) + return resp.text + def get_page_data(self, path: str) -> List[Dict[str, Any]]: """Load a page of ``www.instagram.com`` as a logged-out visitor and return the data it embeds. @@ -753,24 +772,10 @@ class InstaloaderContext: :raises QueryReturnedNotFoundException: When the server responds with a 404. :raises ConnectionException: When the request failed. """ - self.do_sleep() - self._rate_controller.wait_before_query('other') - url = 'https://www.instagram.com/{0}'.format(path) - # Instagram only embeds the data when the request looks like a browser's page navigation. - headers = {'Accept': 'text/html', 'Sec-Fetch-Dest': 'document', 'Sec-Fetch-Mode': 'navigate', - 'Sec-Fetch-Site': 'none'} - try: - with self.get_anonymous_session() as anonymous_session: - resp = anonymous_session.get(url, headers=headers, allow_redirects=False) - except requests.exceptions.RequestException as err: - raise ConnectionException("Page request to {}: {}".format(url, err)) from err - self._raise_if_login_redirect(resp) - if resp.status_code == 404: - raise QueryReturnedNotFoundException(self._response_error(resp)) - if resp.status_code != 200: - raise ConnectionException(self._response_error(resp)) + with self.get_anonymous_session() as anonymous_session: + html = self._get_page(anonymous_session, path) data: List[Dict[str, Any]] = [] - for script in re.finditer(r'', resp.text, re.DOTALL): + for script in re.finditer(r'', html, re.DOTALL): with suppress(json.decoder.JSONDecodeError): data.extend(_embedded_query_data(json.loads(script.group(1)))) return data