Compare commits

..
7 changed files with 120 additions and 147 deletions
+1 -1
View File
@@ -13,7 +13,7 @@ Steps to reproduce the behavior:
(e.g. Instaloader command line)
**Expected behavior**
A clear and concise description of what you expected to happen (even if it seems obvious).
A clear and concise description of what you expected to happen (if not obvious).
**Error messages and tracebacks**
If applicable, add error messages and tracebacks to help explain your problem.
+1 -1
View File
@@ -32,7 +32,7 @@ reporting a problem, please keep the following in mind:
- Include all **error messages and tracebacks** in the report.
- Even if it seems obvious, describe **which behavior you expected**
- If not obvious, describe **which behavior you expected**
instead of what actually happened.
- If we have closed an issue apparently inadvertently or inappropriately, please
+1 -1
View File
@@ -1,7 +1,7 @@
"""Download pictures (or videos) along with their captions and other metadata from Instagram."""
__version__ = '4.5.3'
__version__ = '4.5'
try:
+29 -25
View File
@@ -13,7 +13,7 @@ from functools import wraps
from hashlib import md5
from io import BytesIO
from pathlib import Path
from typing import Any, Callable, IO, Iterator, List, Optional, Set, Union, cast
from typing import Any, Callable, ContextManager, IO, Iterator, List, Optional, Set, Union, cast
import requests
import urllib3 # type: ignore
@@ -470,7 +470,13 @@ class Instaloader:
def test_login(self) -> Optional[str]:
"""Returns the Instagram username to which given :class:`requests.Session` object belongs, or None."""
return self.context.test_login()
data = self.context.graphql_query("d6f4427fbe92d846298cf93df0b937d3", {}).get('data', {}).get('user')
if data is not None:
profile = Profile.from_node(self.context, {'id': data.get('id'),
'username': data.get('username'),
'profile_pic_url_hd': data.get('profile_pic_url')})
return profile.username
return None
def login(self, user: str, passwd: str) -> None:
"""Log in to instagram with given username and password and internally store session object.
@@ -513,28 +519,26 @@ class Instaloader:
# Download the image(s) / video thumbnail and videos within sidecars if desired
downloaded = True
if post.typename == 'GraphSidecar':
if self.download_pictures or self.download_videos:
for edge_number, sidecar_node in enumerate(post.get_sidecar_nodes(), start=1):
if self.download_pictures and (not sidecar_node.is_video or self.download_video_thumbnails):
# Download sidecar picture or video thumbnail (--no-pictures implies --no-video-thumbnails)
if self.download_pictures:
if post.typename == 'GraphSidecar':
edge_number = 1
for sidecar_node in post.get_sidecar_nodes():
# Download picture or video thumbnail
if not sidecar_node.is_video or self.download_video_thumbnails is True:
downloaded &= self.download_pic(filename=filename, url=sidecar_node.display_url,
mtime=post.date_local, filename_suffix=str(edge_number))
if sidecar_node.is_video and self.download_videos:
# Download sidecar video if desired
# Additionally download video if available and desired
if sidecar_node.is_video and self.download_videos is True:
downloaded &= self.download_pic(filename=filename, url=sidecar_node.video_url,
mtime=post.date_local, filename_suffix=str(edge_number))
elif post.typename == 'GraphImage':
# Download picture
if self.download_pictures:
edge_number += 1
elif post.typename == 'GraphImage':
downloaded = self.download_pic(filename=filename, url=post.url, mtime=post.date_local)
elif post.typename == 'GraphVideo':
# Download video thumbnail (--no-pictures implies --no-video-thumbnails)
if self.download_pictures and self.download_video_thumbnails:
with self.context.error_catcher("Video thumbnail of {}".format(post)):
elif post.typename == 'GraphVideo':
if self.download_video_thumbnails is True:
downloaded = self.download_pic(filename=filename, url=post.url, mtime=post.date_local)
else:
self.context.error("Warning: {0} has unknown typename: {1}".format(post, post.typename))
else:
self.context.error("Warning: {0} has unknown typename: {1}".format(post, post.typename))
# Save caption if desired
metadata_string = _ArbitraryItemFormatter(post).format(self.post_metadata_txt_pattern).strip()
@@ -542,7 +546,7 @@ class Instaloader:
self.save_caption(filename=filename, mtime=post.date_local, caption=metadata_string)
# Download video if desired
if post.is_video and self.download_videos:
if post.is_video and self.download_videos is True:
downloaded &= self.download_pic(filename=filename, url=post.video_url, mtime=post.date_local)
# Download geotags if desired
@@ -550,11 +554,11 @@ class Instaloader:
self.save_location(filename, post.location, post.date_local)
# Update comments if desired
if self.download_comments:
if self.download_comments is True:
self.update_comments(filename=filename, post=post)
# Save metadata as JSON if desired.
if self.save_metadata:
if self.save_metadata is not False:
self.save_metadata_json(filename, post)
self.context.log()
@@ -857,7 +861,7 @@ class Instaloader:
"""
self.context.log("Retrieving saved posts...")
assert self.context.username is not None # safe due to @_requires_login; required by typechecker
node_iterator = Profile.own_profile(self.context).get_saved_posts()
node_iterator = Profile.from_username(self.context, self.context.username).get_saved_posts()
self.posts_download_loop(node_iterator, ":saved",
fast_update, post_filter,
max_count=max_count, total_count=node_iterator.count)
@@ -1109,11 +1113,11 @@ class Instaloader:
def _error_raiser(_str):
yield
# error_handler type is Callable[[Optional[str]], ContextManager[None]] (not supported with Python 3.5.0..3.5.3)
error_handler = _error_raiser if raise_errors else self.context.error_catcher
error_handler = cast(Callable[[Optional[str]], ContextManager[None]],
_error_raiser if raise_errors else self.context.error_catcher)
for profile in profiles:
with error_handler(profile.username): # type: ignore # (ignore type for Python 3.5 support)
with error_handler(profile.username):
profile_name = profile.username
# Download profile picture
+51 -81
View File
@@ -78,6 +78,9 @@ class InstaloaderContext:
# Cache profile from id (mapping from id to Profile)
self.profile_id_cache = dict() # type: Dict[int, Any]
# Cache profile from name (mapping from username to Profile)
self.profile_name_cache = dict() # type: Dict[str, Any]
@contextmanager
def anonymous_copy(self):
session = self._session
@@ -185,7 +188,12 @@ class InstaloaderContext:
self.username = username
def test_login(self) -> Optional[str]:
"""Not meant to be used directly, use :meth:`Instaloader.test_login`."""
"""Not meant to be used directly, use :meth:`Instaloader.test_login`.
.. deprecated:: 4.5
:meth:`Instaloader.test_login` now calls :meth:`InstaloaderContext.graphql_query` directly
without using this function.
"""
data = self.graphql_query("d6f4427fbe92d846298cf93df0b937d3", {})
return data["data"]["user"]["username"] if data["data"]["user"] is not None else None
@@ -351,10 +359,10 @@ class InstaloaderContext:
raise ConnectionException("\"window._sharedData\" does not contain required keys.")
# If GraphQL data is missing in `window._sharedData`, search for it in `__additionalDataLoaded`.
if 'graphql' not in post_or_profile_page[0]:
match = re.search(r'window\.__additionalDataLoaded\(.*?({.*"graphql":.*})\);</script>',
match = re.search(r'window\.__additionalDataLoaded\([^{]+{"graphql":({.*})}\);</script>',
resp.text)
if match is not None:
post_or_profile_page[0]['graphql'] = json.loads(match.group(1))['graphql']
post_or_profile_page[0]['graphql'] = json.loads(match.group(1))
return resp_json
else:
resp_json = resp.json()
@@ -487,7 +495,7 @@ class InstaloaderContext:
shutil.copyfileobj(resp.raw, file)
else:
file.write(resp)
os.replace(filename + '.temp', filename)
os.rename(filename + '.temp', filename)
def get_raw(self, url: str, _attempt=1) -> requests.Response:
"""Downloads a file anonymously.
@@ -549,8 +557,8 @@ class RateController:
def __init__(self, context: InstaloaderContext):
self._context = context
self._query_timestamps = dict() # type: Dict[str, List[float]]
self._earliest_next_request_time = 0.0
self._graphql_query_timestamps = dict() # type: Dict[str, List[float]]
self._graphql_earliest_next_request_time = 0.0
def sleep(self, secs: float):
"""Wait given number of seconds."""
@@ -559,75 +567,38 @@ class RateController:
# pylint:disable=no-self-use
time.sleep(secs)
def _dump_query_timestamps(self, current_time: float, failed_query_type: str):
windows = [10, 11, 20, 22, 30, 60]
self._context.error("Requests within last {} minutes grouped by type:"
.format('/'.join(str(w) for w in windows)),
repeat_at_end=False)
for query_type, times in self._query_timestamps.items():
reqs_in_sliding_window = [sum(t > current_time - w * 60 for t in times) for w in windows]
self._context.error(" {} {:>32}: {}".format(
"*" if query_type == failed_query_type else " ",
query_type,
" ".join("{:4}".format(reqs) for reqs in reqs_in_sliding_window)
), repeat_at_end=False)
def _dump_query_timestamps(self, current_time: float):
windows = [10, 11, 15, 20, 30, 60]
print("GraphQL requests:", file=sys.stderr)
for query_hash, times in self._graphql_query_timestamps.items():
print(" {}".format(query_hash), file=sys.stderr)
for window in windows:
reqs_in_sliding_window = sum(t > current_time - window * 60 for t in times)
print(" last {} minutes: {} requests".format(window, reqs_in_sliding_window), file=sys.stderr)
def count_per_sliding_window(self, query_type: str) -> int:
"""Return how many requests can be done within the sliding window."""
"""Return how many GraphQL requests can be done within the sliding window."""
# Not static, to allow for the count_per_sliding_window to depend on context-inherent properties, such as
# whether we are logged in.
# pylint:disable=no-self-use
return 75 if query_type in ['iphone', 'other'] else 200
def _reqs_in_sliding_window(self, query_type: Optional[str], current_time: float, window: float) -> List[float]:
if query_type is not None:
# timestamps of type query_type
relevant_timestamps = self._query_timestamps[query_type]
else:
# all GraphQL queries, i.e. not 'iphone' or 'other'
graphql_query_timestamps = filter(lambda tp: tp[0] not in ['iphone', 'other'],
self._query_timestamps.items())
relevant_timestamps = [t for times in (tp[1] for tp in graphql_query_timestamps) for t in times]
return list(filter(lambda t: t > current_time - window, relevant_timestamps))
# pylint:disable=no-self-use,unused-argument
return 200
def query_waittime(self, query_type: str, current_time: float, untracked_queries: bool = False) -> float:
"""Calculate time needed to wait before query can be executed."""
per_type_sliding_window = 660
if query_type not in self._query_timestamps:
self._query_timestamps[query_type] = []
self._query_timestamps[query_type] = list(filter(lambda t: t > current_time - 60 * 60,
self._query_timestamps[query_type]))
def per_type_next_request_time():
reqs_in_sliding_window = self._reqs_in_sliding_window(query_type, current_time, per_type_sliding_window)
if len(reqs_in_sliding_window) < self.count_per_sliding_window(query_type):
return 0.0
else:
return min(reqs_in_sliding_window) + per_type_sliding_window + 6
def gql_accumulated_next_request_time():
if query_type in ['iphone', 'other']:
return 0.0
gql_accumulated_sliding_window = 600
gql_accumulated_max_count = 275
reqs_in_sliding_window = self._reqs_in_sliding_window(None, current_time, gql_accumulated_sliding_window)
if len(reqs_in_sliding_window) < gql_accumulated_max_count:
return 0.0
else:
return min(reqs_in_sliding_window) + gql_accumulated_sliding_window
def untracked_next_request_time():
if untracked_queries:
reqs_in_sliding_window = self._reqs_in_sliding_window(query_type, current_time, per_type_sliding_window)
self._earliest_next_request_time = min(reqs_in_sliding_window) + per_type_sliding_window + 6
return self._earliest_next_request_time
return max(0.0,
max(
per_type_next_request_time(),
gql_accumulated_next_request_time(),
untracked_next_request_time(),
) - current_time)
"""Calculate time needed to wait before GraphQL query can be executed."""
sliding_window = 660
if query_type not in self._graphql_query_timestamps:
self._graphql_query_timestamps[query_type] = []
self._graphql_query_timestamps[query_type] = list(filter(lambda t: t > current_time - 60 * 60,
self._graphql_query_timestamps[query_type]))
reqs_in_sliding_window = list(filter(lambda t: t > current_time - sliding_window,
self._graphql_query_timestamps[query_type]))
count_per_sliding_window = self.count_per_sliding_window(query_type)
if len(reqs_in_sliding_window) < count_per_sliding_window and not untracked_queries:
return max(0.0, self._graphql_earliest_next_request_time - current_time)
next_request_time = min(reqs_in_sliding_window) + sliding_window + 6
if untracked_queries:
self._graphql_earliest_next_request_time = next_request_time
return max(next_request_time, self._graphql_earliest_next_request_time) - current_time
def wait_before_query(self, query_type: str) -> None:
"""This method is called before a query to Instagram. It calls :meth:`RateController.sleep` to wait
@@ -639,25 +610,24 @@ class RateController:
.format(round(waittime), datetime.now() + timedelta(seconds=waittime)))
if waittime > 0:
self.sleep(waittime)
if query_type not in self._query_timestamps:
self._query_timestamps[query_type] = [time.monotonic()]
if query_type not in self._graphql_query_timestamps:
self._graphql_query_timestamps[query_type] = [time.monotonic()]
else:
self._query_timestamps[query_type].append(time.monotonic())
self._graphql_query_timestamps[query_type].append(time.monotonic())
def handle_429(self, query_type: str) -> None:
"""This method is called to handle a 429 Too Many Requests response. It calls :meth:`RateController.sleep` to
wait until we can repeat the same request."""
text_for_429 = ("HTTP error code 429 was returned because too many queries occurred in the last time. "
"Please do not use Instagram in your browser or run multiple instances of Instaloader "
"in parallel.")
print(textwrap.fill(text_for_429), file=sys.stderr)
current_time = time.monotonic()
waittime = self.query_waittime(query_type, current_time, True)
assert waittime >= 0
self._dump_query_timestamps(current_time, query_type)
text_for_429 = ("Instagram responded with HTTP error \"429 - Too Many Requests\". Please do not run multiple "
"instances of Instaloader in parallel or within short sequence. Also, do not use any Instagram "
"App while Instaloader is running.")
self._context.error(textwrap.fill(text_for_429), repeat_at_end=False)
if waittime > 1.5:
self._context.error("The request will be retried in {} seconds, at {:%H:%M}."
.format(round(waittime), datetime.now() + timedelta(seconds=waittime)),
repeat_at_end=False)
if waittime > 15:
self._context.log("The request will be retried in {} seconds, at {:%H:%M}."
.format(round(waittime), datetime.now() + timedelta(seconds=waittime)))
self._dump_query_timestamps(current_time)
if waittime > 0:
self.sleep(waittime)
+6 -10
View File
@@ -81,13 +81,11 @@ class NodeIterator(Iterator[T]):
self._node_wrapper = node_wrapper
self._query_variables = query_variables if query_variables is not None else {}
self._query_referer = query_referer
self._data = first_data
self._page_index = 0
self._total_index = 0
if first_data is not None:
self._data = first_data
self._best_before = datetime.now() + NodeIterator._shelf_life
else:
self._data = self._query()
self._best_before = (None if first_data is None else
datetime.now() + NodeIterator._shelf_life)
def _query(self, after: Optional[str] = None) -> Dict:
pagination_variables = {'first': NodeIterator._graphql_page_length} # type: Dict[str, Any]
@@ -115,6 +113,8 @@ class NodeIterator(Iterator[T]):
return self
def __next__(self) -> T:
if self._data is None:
self._data = self._query()
if self._page_index < len(self._data['edges']):
node = self._data['edges'][self._page_index]['node']
page_index, total_index = self._page_index, self._total_index
@@ -193,12 +193,8 @@ class NodeIterator(Iterator[T]):
self._query_referer != frozen.query_referer or
self._context.username != frozen.context_username):
raise InvalidArgumentException("Mismatching resume information.")
if not frozen.best_before:
raise InvalidArgumentException("\"best before\" date missing.")
if frozen.remaining_data is None:
raise InvalidArgumentException("\"remaining_data\" missing.")
self._total_index = frozen.total_index
self._best_before = datetime.fromtimestamp(frozen.best_before)
self._best_before = datetime.fromtimestamp(frozen.best_before) if frozen.best_before else None
self._data = frozen.remaining_data
+31 -28
View File
@@ -196,7 +196,7 @@ class Post:
# is contained.
# Note that we cannot use Profile.from_id() here since that would lead us into a recursion.
owner_struct = self._full_metadata['owner']
self._owner_profile = Profile(self._context, owner_struct)
self._owner_profile = Profile.from_node(self._context, owner_struct)
return self._owner_profile
@property
@@ -387,7 +387,7 @@ class Post:
return PostCommentAnswer(id=int(node['id']),
created_at_utc=datetime.utcfromtimestamp(node['created_at']),
text=node['text'],
owner=Profile(self._context, node['owner']),
owner=Profile.from_node(self._context, node['owner']),
likes_count=node.get('edge_liked_by', {}).get('count', 0))
def _postcommentanswers(node):
@@ -442,13 +442,13 @@ class Post:
likes_edges = self._field('edge_media_preview_like', 'edges')
if self.likes == len(likes_edges):
# If the Post's metadata already contains all likes, don't do GraphQL requests to obtain them
yield from (Profile(self._context, like['node']) for like in likes_edges)
yield from (Profile.from_node(self._context, like['node']) for like in likes_edges)
return
yield from NodeIterator(
self._context,
'1cb6ec562846122743b61e492c85999f',
lambda d: d['data']['shortcode_media']['edge_liked_by'],
lambda n: Profile(self._context, n),
lambda n: Profile.from_node(self._context, n),
{'shortcode': self.shortcode},
'https://www.instagram.com/p/{0}/'.format(self.shortcode),
)
@@ -474,7 +474,7 @@ class Post:
.. versionadded:: 4.4
"""
return ([] if not self.is_sponsored else
[Profile(self._context, edge['node']['sponsor']) for edge in
[Profile.from_node(self._context, edge['node']['sponsor']) for edge in
self._field('edge_media_to_sponsor_user', 'edges')])
@property
@@ -537,6 +537,17 @@ class Profile:
# if loaded from JSON with load_structure_from_file()
self._iphone_struct_ = node['iphone_struct']
@classmethod
def from_node(cls, context: InstaloaderContext, node: Dict[str, Any]) -> 'Profile':
assert 'username' in node
if node['username'] in context.profile_name_cache:
return context.profile_name_cache[node['username']]
profile = cls(context, node)
if 'id' in node:
context.profile_id_cache[node['id']] = profile
context.profile_name_cache[profile.username] = profile
return profile
@classmethod
def from_username(cls, context: InstaloaderContext, username: str):
"""Create a Profile instance from a given username, raise exception if it does not exist.
@@ -548,8 +559,11 @@ class Profile:
:raises: :class:`ProfileNotExistsException`
"""
# pylint:disable=protected-access
profile = cls(context, {'username': username.lower()})
profile._obtain_metadata() # to raise ProfileNotExistsException now in case username is invalid
if username.lower() in context.profile_name_cache:
return context.profile_name_cache[username.lower()]
profile = cls.from_node(context, {'username': username.lower()})
profile._obtain_metadata() # to raise ProfileNotExistException now in case username is invalid
context.profile_id_cache[profile.userid] = profile
return profile
@classmethod
@@ -572,23 +586,10 @@ class Profile:
'include_highlight_reels': False},
rhx_gis=context.root_rhx_gis)['data']['user']
if data:
profile = cls(context, data['reel']['owner'])
return cls.from_node(context, data['reel']['owner'])
else:
raise ProfileNotExistsException("No profile found, the user may have blocked you (ID: " +
str(profile_id) + ").")
context.profile_id_cache[profile_id] = profile
return profile
@classmethod
def own_profile(cls, context: InstaloaderContext):
"""Return own profile if logged-in.
:param context: :attr:`Instaloader.context`
.. versionadded:: 4.5.2"""
if not context.is_logged_in:
raise LoginRequiredException("--login required to access own profile.")
return cls(context, context.graphql_query("d6f4427fbe92d846298cf93df0b937d3", {})["data"]["user"])
def _asdict(self):
json_node = self._node.copy()
@@ -813,6 +814,7 @@ class Profile:
if self.username != self._context.username:
raise LoginRequiredException("--login={} required to get that profile's saved posts.".format(self.username))
self._obtain_metadata()
return NodeIterator(
self._context,
'f883d95537fbcd400f466f63d42bd8a1',
@@ -820,6 +822,7 @@ class Profile:
lambda n: Post(self._context, n),
{'id': self.userid},
'https://www.instagram.com/{0}/'.format(self.username),
self._metadata('edge_saved_media'),
)
def get_tagged_posts(self) -> NodeIterator[Post]:
@@ -869,7 +872,7 @@ class Profile:
self._context,
'37479f2b8209594dde7facb0d904896a',
lambda d: d['data']['user']['edge_followed_by'],
lambda n: Profile(self._context, n),
lambda n: Profile.from_node(self._context, n),
{'id': str(self.userid)},
'https://www.instagram.com/{0}/'.format(self.username),
)
@@ -888,7 +891,7 @@ class Profile:
self._context,
'58712303d941c6855d4e888c5f0cd22f',
lambda d: d['data']['user']['edge_follow'],
lambda n: Profile(self._context, n),
lambda n: Profile.from_node(self._context, n),
{'id': str(self.userid)},
'https://www.instagram.com/{0}/'.format(self.username),
)
@@ -903,7 +906,7 @@ class Profile:
if not self._context.is_logged_in:
raise LoginRequiredException("--login required to get a profile's similar accounts.")
self._obtain_metadata()
yield from (Profile(self._context, edge["node"]) for edge in
yield from (Profile.from_node(self._context, edge["node"]) for edge in
self._context.graphql_query("ad99dd9d3646cc3c0dda65debcd266a7",
{"user_id": str(self.userid), "include_chaining": True},
"https://www.instagram.com/{0}/"
@@ -1108,7 +1111,7 @@ class Story:
def owner_profile(self) -> Profile:
""":class:`Profile` instance of the story owner."""
if not self._owner_profile:
self._owner_profile = Profile(self._context, self._node['user'])
self._owner_profile = Profile.from_node(self._context, self._node['user'])
return self._owner_profile
@property
@@ -1166,7 +1169,7 @@ class Highlight(Story):
def owner_profile(self) -> Profile:
""":class:`Profile` instance of the highlights' owner."""
if not self._owner_profile:
self._owner_profile = Profile(self._context, self._node['owner'])
self._owner_profile = Profile.from_node(self._context, self._node['owner'])
return self._owner_profile
@property
@@ -1397,7 +1400,7 @@ class TopSearchResults:
user_node = user['user']
if 'pk' in user_node:
user_node['id'] = user_node['pk']
yield Profile(self._context, user_node)
yield Profile.from_node(self._context, user_node)
def get_prefixed_usernames(self) -> Iterator[str]:
"""
@@ -1490,7 +1493,7 @@ def load_structure_from_file(context: InstaloaderContext, filename: str) -> Json
if node_type == "Post":
return Post(context, json_structure['node'])
elif node_type == "Profile":
return Profile(context, json_structure['node'])
return Profile.from_node(context, json_structure['node'])
elif node_type == "StoryItem":
return StoryItem(context, json_structure['node'])
elif node_type == "Hashtag":