Compare commits

..
36 Commits
Author SHA1 Message Date
Alexander Graf 65462b01ec Release of Version 4.4.1 2020-05-14 16:22:42 +02:00
Alexander Graf 9857c5fa24 Regex target matching: Fix hashtag regex
Unicode characters are allowed in hashtags, so instead of just
[A-Za-z0-9], we now match all \w.  Fixes #614.
2020-05-14 16:17:07 +02:00
Alexander Graf 83f013f6ca Release of Version 4.4 2020-05-11 20:25:43 +02:00
Alexander Graf e7437310ea Add versionadded doc annotation for is_business_account and business_category_name 2020-05-11 20:24:04 +02:00
yolo dcba25a40e [Issue #601] Add is_business_account and business_category_name properties to Profile structure 2020-05-06 08:45:31 +02:00
Alexander Graf 5e4b5fd877 First Beta Release for Version 4.4 2020-04-21 10:22:03 +02:00
Lars Lindqvist 657e76e2d9 Simplify username regex
Without passing re.ASCII, \w matches non-ascii letter, such as ö, π
and я, which to my knowledge isn't allowed in usernames. Now it's
explicit which character ranges are expected in each target.
2020-04-21 10:15:24 +02:00
Lars Lindqvist 78023b3c98 Regex target matching 2020-04-21 10:15:24 +02:00
Alexander Graf ab67ca30e5 Merge branch 'master' into v4.4-dev 2020-04-21 09:53:01 +02:00
Alexander Graf 0e13c73b62 Include top posts in download_hashtag() 2020-04-19 12:21:27 +02:00
Alexander Graf b98e9941ae Omit media when saving Hashtag JSON 2020-04-19 11:30:19 +02:00
Alexander Graf c9e4f68597 Download Hashtag profilepic and save metadata JSON 2020-04-19 11:16:39 +02:00
Alexander Graf 1338cf0fcd Fix 'occured' typo in error messages
Mentioned in #573 - thanks!
2020-04-18 11:16:55 +02:00
Alexander Graf 83a0f52c60 Hashtag class 2020-04-17 18:49:38 +02:00
Alexander Graf 5c5c633bee Catch Exceptions raised during filter evaluation
Closes #192.
2020-04-10 14:39:40 +02:00
Alexander Graf e268f9e625 Release of Version 4.3.2 2020-04-10 12:08:24 +02:00
Alexander Graf 793b7e82ba Limit feed node processing to Post nodes
Fixes #478.
2020-04-10 12:05:15 +02:00
Alexander Graf 7329ee10d8 Catch KeyError: 'authenticated' when Logging In
Fixes #472.
2020-04-10 12:05:15 +02:00
Alexander Graf 57d2ce9538 Alleviate #449 by catching bad metadata response
Instead of ungracefully failing with a TypeError, the Post is now skipped with
an error message containing the invalid JSON.
2020-04-10 11:57:36 +02:00
Alexander Graf dc458a3f3c Limit feed node processing to Post nodes
Fixes #478.
2020-04-10 11:13:07 +02:00
Alexander Graf 08a359408c Improve Log Output of Skipped Posts 2020-04-10 11:12:09 +02:00
Alexander Graf a794089c4f Deduplicate Post Download Loop 2020-04-07 10:06:53 +02:00
Alexander Graf c56657855d Release of Version 4.3.1 2020-04-04 10:54:04 +02:00
Alexander Graf 8c7561c5d7 Fix Downloading of Videos within a Sidecar Post
Fixes #558.
2020-04-04 10:51:55 +02:00
Alexander Graf c76385bde2 Unit Test for Profile.get_similar_accounts() 2020-03-25 19:06:05 +01:00
Alexander Graf b5bf8a0560 Implement Profile.get_similar_accounts()
Fixes #489.
2020-03-25 19:00:03 +01:00
Alexander Graf 2790342a09 Catch KeyError: 'authenticated' when Logging In
Fixes #472.
2020-03-25 18:17:02 +01:00
Alexander Graf e725ca2640 Post properties is_sponsored and sponsor_users
This allows to access the Paid Partnership With metadata of a Post.

Closes #295.
2020-03-25 17:13:46 +01:00
Alexander Graf 9c2e7c70ac Fix number output in download_igtv() 2020-03-16 13:38:13 +01:00
Alexander Graf ffbf5b517c Release of Version 4.3 2020-03-16 09:13:25 +01:00
Alexander Graf 0843e10187 First Release Candidate for Version 4.3 2020-03-15 14:28:16 +01:00
Alexander Graf 78a93c945e Revert "Revert "Remove nonworking saythanks.io link""
saythanks.io does not work again and the link is now removed.
This reverts commit b2acbcf0e0.
2020-03-14 17:56:44 +01:00
Alexander Graf bb3e309195 First Beta Release for Version 4.3 2020-03-14 17:21:07 +01:00
Alexander Graf 6cc2ed54ea Download IGTV Videos of a Profile (--igtv) 2020-03-14 16:56:51 +01:00
Alexander Graf 41cf260fdd Fix post filter for downloading tagged posts 2020-03-14 16:55:37 +01:00
Alexander Graf f910705c51 docs: Fix formatting of request_timeout param 2020-03-07 11:14:07 +01:00
14 changed files with 496 additions and 173 deletions
+3 -6
View File
@@ -2,7 +2,7 @@
.. badges-start
|travis| |pypi| |pyversion| |license| |aur| |contributors| |downloads| |saythanks|
|travis| |pypi| |pyversion| |license| |aur| |contributors| |downloads|
.. |travis| image:: https://img.shields.io/travis/instaloader/instaloader/master.svg
:alt: Travis-CI Build Status
@@ -23,10 +23,6 @@
:alt: Contributor Count
:target: https://github.com/instaloader/instaloader/graphs/contributors
.. |saythanks| image:: https://img.shields.io/badge/Say%20Thanks-!-ff7efd.svg
:alt: Say Thanks!
:target: https://saythanks.io/to/aandergr
.. |aur| image:: https://img.shields.io/aur/version/instaloader.svg
:alt: Arch User Repository Package
:target: https://aur.archlinux.org/packages/instaloader/
@@ -57,7 +53,8 @@
::
instaloader [--comments] [--geotags] [--stories] [--highlights] [--tagged]
instaloader [--comments] [--geotags]
[--stories] [--highlights] [--tagged] [--igtv]
[--login YOUR-USERNAME] [--fast-update]
profile | "#hashtag" | :stories | :feed | :saved
-3
View File
@@ -51,9 +51,6 @@
<li class="nav-item">
<a target="_blank" class="nav-link" href="https://github.com/instaloader/instaloader">GitHub</a>
</li>
<li class="nav-item d-sm-none d-lg-block">
<a target="_blank" class="nav-link" href="https://saythanks.io/to/aandergr">Say Thanks!</a>
</li>
</ul>
<form class="form-inline" action="search.html" method="get">
<div class="input-group my-2 my-sm-0 ml-2">
+11 -3
View File
@@ -37,7 +37,7 @@ picture, video or sidecar (set of multiple pictures/videos) posted in a user's
profile. :class:`Instaloader` provides methods to iterate over Posts from a
certain source::
for post in L.get_hashtag_posts('cat'):
for post in instaloader.Hashtag.from_name(L.context, 'cat').get_posts():
# post is an instance of instaloader.Post
L.download_post(post, target='#cat')
@@ -63,7 +63,7 @@ certain source::
- :meth:`Instaloader.get_explore_posts`
Media that is suggested by Instagram to explore.
- :meth:`Instaloader.get_hashtag_posts`
- :meth:`Hashtag.get_posts`
Media associated with given hashtag.
With the :class:`Profile` class, Instaloader also makes it easy to access
@@ -155,6 +155,14 @@ Profiles
.. autoclass:: Profile
:no-show-inheritance:
Hashtags
""""""""
.. autoclass:: Hashtag
:no-show-inheritance:
.. versionadded:: 4.4
TopSearchResults
""""""""""""""""
@@ -166,7 +174,7 @@ TopSearchResults
Loading and Saving
""""""""""""""""""
:class:`Post`, :class:`StoryItem` and :class:`Profile` can be saved and loaded
:class:`Post`, :class:`StoryItem`, :class:`Profile` and :class:`Hashtag` can be saved and loaded
to/from JSON files.
.. autofunction:: load_structure_from_file
+5 -2
View File
@@ -77,10 +77,13 @@ Instaloader supports the following targets:
instructs Instaloader to also **download the user's stories**,
- :option:`--highlights`
to **download highlights of each profile that is downloaded**, and
to **download highlights of each profile that is downloaded**,
- :option:`--tagged`
to **download posts where the user is tagged**.
to **download posts where the user is tagged**, and
- :option:`--igtv`
to **download IGTV videos**.
- ``"#hashtag"``
Posts with a certain **hashtag** (the quotes are usually necessary),
+6
View File
@@ -129,6 +129,12 @@ What to Download of each Profile
.. versionadded:: 4.1
.. option:: --igtv
Also download IGTV videos.
.. versionadded:: 4.3
.. option:: --stories-only
.. deprecated:: 4.1
+1 -1
View File
@@ -2,7 +2,7 @@ import instaloader
L = instaloader.Instaloader()
posts = L.get_hashtag_posts('urbanphotography')
posts = instaloader.Hashtag.from_name(L.context, 'urbanphotography').get_posts()
users = set()
+1 -1
View File
@@ -5,7 +5,7 @@ import instaloader
L = instaloader.Instaloader()
posts = L.get_hashtag_posts('urbanphotography')
posts = instaloader.Hashtag.from_name(L.context, 'urbanphotography').get_posts()
# or
# posts = instaloader.Profile.from_username(L.context, PROFILE).get_posts()
+2 -1
View File
@@ -45,7 +45,8 @@ See :ref:`install` for more options on how to install Instaloader.
::
instaloader [--comments] [--geotags] [--stories] [--highlights] [--tagged]
instaloader [--comments] [--geotags]
[--stories] [--highlights] [--tagged] [--igtv]
[--login YOUR-USERNAME] [--fast-update]
profile | "#hashtag" | %location_id |
:stories | :feed | :saved
+3 -3
View File
@@ -1,7 +1,7 @@
"""Download pictures (or videos) along with their captions and other metadata from Instagram."""
__version__ = '4.3a2'
__version__ = '4.4.1'
try:
@@ -15,5 +15,5 @@ else:
from .exceptions import *
from .instaloader import Instaloader
from .instaloadercontext import InstaloaderContext
from .structures import (Highlight, Post, PostSidecarNode, PostComment, PostCommentAnswer, PostLocation, Profile, Story,
StoryItem, TopSearchResults, load_structure_from_file, save_structure_to_file)
from .structures import (Hashtag, Highlight, Post, PostSidecarNode, PostComment, PostCommentAnswer, PostLocation,
Profile, Story, StoryItem, TopSearchResults, load_structure_from_file, save_structure_to_file)
+31 -13
View File
@@ -3,6 +3,7 @@
import ast
import datetime
import os
import re
import sys
from argparse import ArgumentParser, SUPPRESS
from typing import List, Optional
@@ -19,7 +20,8 @@ def usage_string():
argv0 = os.path.basename(sys.argv[0])
argv0 = "instaloader" if argv0 == "__main__.py" else argv0
return """
{0} [--comments] [--geotags] [--stories] [--highlights] [--tagged]
{0} [--comments] [--geotags]
{2:{1}} [--stories] [--highlights] [--tagged] [--igtv]
{2:{1}} [--login YOUR-USERNAME] [--fast-update]
{2:{1}} profile | "#hashtag" | %%location_id | :stories | :feed | :saved
{0} --help""".format(argv0, len(argv0), '')
@@ -61,7 +63,10 @@ def _main(instaloader: Instaloader, targetlist: List[str],
username: Optional[str] = None, password: Optional[str] = None,
sessionfile: Optional[str] = None,
download_profile_pic: bool = True, download_posts=True,
download_stories: bool = False, download_highlights: bool = False, download_tagged: bool = False,
download_stories: bool = False,
download_highlights: bool = False,
download_tagged: bool = False,
download_igtv: bool = False,
fast_update: bool = False,
max_count: Optional[int] = None, post_filter_str: Optional[str] = None,
storyitem_filter_str: Optional[str] = None) -> None:
@@ -132,18 +137,19 @@ def _main(instaloader: Instaloader, targetlist: List[str],
# strip '/' characters to be more shell-autocompletion-friendly
target = target.rstrip('/')
with instaloader.context.error_catcher(target):
if target[0] == '@':
if re.match(r"^@[A-Za-z0-9._]+$", target):
instaloader.context.log("Retrieving followees of %s..." % target[1:])
profile = Profile.from_username(instaloader.context, target[1:])
for followee in profile.get_followees():
instaloader.save_profile_id(followee)
profiles.add(followee)
elif target[0] == '#':
elif re.match(r"^#\w+$", target):
instaloader.download_hashtag(hashtag=target[1:], max_count=max_count, fast_update=fast_update,
post_filter=post_filter)
elif target[0] == '-':
post_filter=post_filter,
profile_pic=download_profile_pic, posts=download_posts)
elif re.match(r"^-[A-Za-z0-9-_]+$", target):
instaloader.download_post(Post.from_shortcode(instaloader.context, target[1:]), target)
elif target[0] == "%":
elif re.match(r"^%[0-9]+$", target):
instaloader.download_location(location=target[1:], max_count=max_count, fast_update=fast_update,
post_filter=post_filter)
elif target == ":feed":
@@ -154,11 +160,12 @@ def _main(instaloader: Instaloader, targetlist: List[str],
elif target == ":saved":
instaloader.download_saved_posts(fast_update=fast_update, max_count=max_count,
post_filter=post_filter)
else:
elif re.match(r"^[A-Za-z0-9._]+$", target):
try:
profile = instaloader.check_profile_id(target)
if instaloader.context.is_logged_in and profile.has_blocked_viewer:
if download_profile_pic or ((download_posts or download_tagged) and not profile.is_private):
if download_profile_pic or ((download_posts or download_tagged or download_igtv)
and not profile.is_private):
raise ProfileNotExistsException("{} blocked you; But we download her anonymously."
.format(target))
else:
@@ -169,7 +176,7 @@ def _main(instaloader: Instaloader, targetlist: List[str],
# Not only our profile.has_blocked_viewer condition raises ProfileNotExistsException,
# check_profile_id() also does, since access to blocked profile may be responded with 404.
if instaloader.context.is_logged_in and (download_profile_pic or download_posts or
download_tagged):
download_tagged or download_igtv):
instaloader.context.log(err)
instaloader.context.log("Trying again anonymously, helps in case you are just blocked.")
with instaloader.anonymous_copy() as anonymous_loader:
@@ -179,20 +186,28 @@ def _main(instaloader: Instaloader, targetlist: List[str],
.format(target, err))
else:
raise
else:
target_type = {
'#': 'hashtag',
'%': 'location',
'-': 'shortcode',
}.get(target[0], 'username')
raise ProfileNotExistsException('Invalid {} {}'.format(target_type, target))
if len(profiles) > 1:
instaloader.context.log("Downloading {} profiles: {}".format(len(profiles),
' '.join([p.username for p in profiles])))
if profiles and download_profile_pic and not instaloader.context.is_logged_in:
instaloader.context.error("Warning: Use --login to download HD version of profile pictures.")
instaloader.download_profiles(profiles,
download_profile_pic, download_posts, download_tagged, download_highlights,
download_stories, fast_update, post_filter, storyitem_filter)
download_profile_pic, download_posts, download_tagged, download_igtv,
download_highlights, download_stories,
fast_update, post_filter, storyitem_filter)
if anonymous_retry_profiles:
instaloader.context.log("Downloading anonymously: {}"
.format(' '.join([p.username for p in anonymous_retry_profiles])))
with instaloader.anonymous_copy() as anonymous_loader:
anonymous_loader.download_profiles(anonymous_retry_profiles,
download_profile_pic, download_posts, download_tagged,
download_profile_pic, download_posts, download_tagged, download_igtv,
fast_update=fast_update, post_filter=post_filter)
except KeyboardInterrupt:
print("\nInterrupted by user.", file=sys.stderr)
@@ -287,6 +302,8 @@ def main():
help='Also download highlights of each profile that is downloaded. Requires --login.')
g_prof.add_argument('--tagged', action='store_true',
help='Also download posts where each profile is tagged.')
g_prof.add_argument('--igtv', action='store_true',
help='Also download IGTV videos.')
g_cond = parser.add_argument_group("Which Posts to Download")
@@ -411,6 +428,7 @@ def main():
download_stories=download_stories,
download_highlights=args.highlights,
download_tagged=args.tagged,
download_igtv=args.igtv,
fast_update=args.fast_update,
max_count=int(args.count) if args.count is not None else None,
post_filter_str=args.post_filter,
+146 -129
View File
@@ -21,7 +21,7 @@ import urllib3 # type: ignore
from .exceptions import *
from .instaloadercontext import InstaloaderContext
from .structures import (Highlight, JsonExportable, Post, PostLocation, Profile, Story, StoryItem,
from .structures import (Hashtag, Highlight, JsonExportable, Post, PostLocation, Profile, Story, StoryItem,
save_structure_to_file, load_structure_from_file)
@@ -142,7 +142,7 @@ class Instaloader:
:param storyitem_metadata_txt_pattern: :option:`--storyitem-metadata-txt`, default is empty (=none)
:param max_connection_attempts: :option:`--max-connection-attempts`
:param commit_mode: :option:`--commit-mode`
:param request_timeout: "option:`--request-timeout`, set per-request timeout (seconds)
:param request_timeout: :option:`--request-timeout`, set per-request timeout (seconds)
.. attribute:: context
@@ -358,7 +358,7 @@ class Instaloader:
self.context.log('geo', end=' ', flush=True)
@_retry_on_connection_error
def download_title_pic(self, url: str, target: Union[str, Path], name_suffix: str, owner_profile: Profile,
def download_title_pic(self, url: str, target: Union[str, Path], name_suffix: str, owner_profile: Optional[Profile],
_attempt: int = 1) -> None:
"""Downloads and saves a picture that does not have an association with a Post or StoryItem, such as a
Profile picture or a Highlight cover picture. Modification time is taken from the HTTP response headers.
@@ -380,7 +380,8 @@ class Instaloader:
pic_extension = 'jpg'
if ((format_string_contains_key(self.dirname_pattern, 'profile') or
format_string_contains_key(self.dirname_pattern, 'target'))):
filename = '{0}/{1}_{2}.{3}'.format(self.dirname_pattern.format(profile=owner_profile.username.lower(),
profile_str = owner_profile.username.lower() if owner_profile is not None else target
filename = '{0}/{1}_{2}.{3}'.format(self.dirname_pattern.format(profile=profile_str,
target=target),
pic_identifier, name_suffix, pic_extension)
else:
@@ -408,6 +409,12 @@ class Instaloader:
.. versionadded:: 4.3"""
self.download_title_pic(highlight.cover_url, target, 'cover', highlight.owner_profile)
def download_hashtag_profilepic(self, hashtag: Hashtag) -> None:
"""Downloads and saves the profile picture of a Hashtag.
.. versionadded:: 4.4"""
self.download_title_pic(hashtag.profile_pic_url, '#' + hashtag.name, 'profile_pic', None)
@_requires_login
def save_session_to_file(self, filename: Optional[str] = None) -> None:
"""Saves internally stored :class:`requests.Session` object.
@@ -691,6 +698,66 @@ class Instaloader:
if fast_update and not downloaded:
break
def posts_download_loop(self,
posts: Iterator[Post],
target: Union[str, Path],
fast_update: bool = False,
post_filter: Optional[Callable[[Post], bool]] = None,
max_count: Optional[int] = None,
total_count: Optional[int] = None) -> None:
"""
Download the Posts returned by given Post Iterator.
..versionadded:: 4.4
:param posts: Post Iterator to loop through.
:param target: Target name
:param fast_update: :option:`--fast-update`
:param post_filter: :option:`--post-filter`
:param max_count: Maximum count of Posts to download (:option:`--count`)
:param total_count: Total number of posts returned by given iterator
"""
for number, post in enumerate(posts):
if max_count is not None and number >= max_count:
break
if total_count is not None:
self.context.log("[{0:{w}d}/{1:{w}d}] ".format(number + 1, total_count,
w=len(str(total_count))),
end="", flush=True)
else:
if max_count is not None:
self.context.log("[{0:{w}d}/{1:{w}d}] ".format(number + 1, max_count,
w=len(str(max_count))),
end="", flush=True)
else:
self.context.log("[{:3d}] ".format(number + 1), end="", flush=True)
if post_filter is not None:
try:
if not post_filter(post):
self.context.log("{} skipped".format(post))
continue
except (InstaloaderException, KeyError, TypeError) as err:
self.context.error("{} skipped. Filter evaluation failed: {}".format(post, err))
continue
with self.context.error_catcher("Download {} of {}".format(post, target)):
# The PostChangedException gets raised if the Post's id/shortcode changed while obtaining
# additional metadata. This is most likely the case if a HTTP redirect takes place while
# resolving the shortcode URL.
# The `post_changed` variable keeps the fast-update functionality alive: A Post which is
# obained after a redirect has probably already been downloaded as a previous Post of the
# same Profile.
# Observed in issue #225: https://github.com/instaloader/instaloader/issues/225
post_changed = False
while True:
try:
downloaded = self.download_post(post, target=target)
break
except PostChangedException:
post_changed = True
continue
if fast_update and not downloaded and not post_changed:
break
@_requires_login
def get_feed_posts(self) -> Iterator[Post]:
"""Get Posts of the user's feed.
@@ -703,8 +770,10 @@ class Instaloader:
while True:
feed = data["user"]["edge_web_feed_timeline"]
yield from (Post(self.context, edge["node"]) for edge in feed["edges"]
if not edge["node"]["__typename"] == "GraphSuggestedUserFeedUnit")
for edge in feed["edges"]:
node = edge["node"]
if node.get("__typename") in Post.supported_graphql_types() and node.get("shortcode") is not None:
yield Post(self.context, node)
if not feed["page_info"]["has_next_page"]:
break
data = self.context.graphql_query("d6f4427fbe92d846298cf93df0b937d3",
@@ -733,20 +802,7 @@ class Instaloader:
:raises LoginRequiredException: If called without being logged in.
"""
self.context.log("Retrieving pictures from your feed...")
count = 1
for post in self.get_feed_posts():
if max_count is not None and count > max_count:
break
name = post.owner_username
if post_filter is not None and not post_filter(post):
self.context.log("<pic by %s skipped>" % name, flush=True)
continue
self.context.log("[%3i] %s " % (count, name), end="", flush=True)
count += 1
with self.context.error_catcher('Download feed'):
downloaded = self.download_post(post, target=':feed')
if fast_update and not downloaded:
break
self.posts_download_loop(self.get_feed_posts(), ":feed", fast_update, post_filter, max_count=max_count)
@_requires_login
def download_saved_posts(self, max_count: int = None, fast_update: bool = False,
@@ -759,20 +815,9 @@ class Instaloader:
:raises LoginRequiredException: If called without being logged in.
"""
self.context.log("Retrieving saved posts...")
count = 1
assert self.context.username is not None
for post in Profile.from_username(self.context, self.context.username).get_saved_posts():
if max_count is not None and count > max_count:
break
if post_filter is not None and not post_filter(post):
self.context.log("<{} skipped>".format(post), flush=True)
continue
self.context.log("[{:>3}] ".format(count), end=str(), flush=True)
count += 1
with self.context.error_catcher('Download saved posts'):
downloaded = self.download_post(post, target=':saved')
if fast_update and not downloaded:
break
assert self.context.username is not None # safe due to @_requires_login; required by typechecker
self.posts_download_loop(Profile.from_username(self.context, self.context.username).get_saved_posts(), ":saved",
fast_update, post_filter, max_count=max_count)
@_requires_login
def get_location_posts(self, location: str) -> Iterator[Post]:
@@ -823,19 +868,8 @@ class Instaloader:
Require being logged in (as required by Instagram)
"""
self.context.log("Retrieving pictures for location {}...".format(location))
count = 1
for post in self.get_location_posts(location):
if max_count is not None and count > max_count:
break
self.context.log('[{0:3d}] %{1} '.format(count, location), end='', flush=True)
if post_filter is not None and not post_filter(post):
self.context.log('<skipped>')
continue
count += 1
with self.context.error_catcher('Download location {}'.format(location)):
downloaded = self.download_post(post, target='%' + location)
if fast_update and not downloaded:
break
self.posts_download_loop(self.get_location_posts(location), "%" + location, fast_update, post_filter,
max_count=max_count)
@_requires_login
def get_explore_posts(self) -> Iterator[Post]:
@@ -852,24 +886,18 @@ class Instaloader:
data.get('rhx_gis')))
def get_hashtag_posts(self, hashtag: str) -> Iterator[Post]:
"""Get Posts associated with a #hashtag."""
has_next_page = True
end_cursor = None
while has_next_page:
if end_cursor:
params = {'__a': 1, 'max_id': end_cursor}
else:
params = {'__a': 1}
hashtag_data = self.context.get_json('explore/tags/{0}/'.format(hashtag),
params)['graphql']['hashtag']['edge_hashtag_to_media']
yield from (Post(self.context, edge['node']) for edge in hashtag_data['edges'])
has_next_page = hashtag_data['page_info']['has_next_page']
end_cursor = hashtag_data['page_info']['end_cursor']
"""Get Posts associated with a #hashtag.
def download_hashtag(self, hashtag: str,
.. deprecated:: 4.4
Use :meth:`Hashtag.get_posts`."""
return Hashtag.from_name(self.context, hashtag).get_posts()
def download_hashtag(self, hashtag: Union[Hashtag, str],
max_count: Optional[int] = None,
post_filter: Optional[Callable[[Post], bool]] = None,
fast_update: bool = False) -> None:
fast_update: bool = False,
profile_pic: bool = True,
posts: bool = True) -> None:
"""Download pictures of one hashtag.
To download the last 30 pictures with hashtag #cat, do::
@@ -877,26 +905,34 @@ class Instaloader:
loader = Instaloader()
loader.download_hashtag('cat', max_count=30)
:param hashtag: Hashtag to download, without leading '#'
:param hashtag: Hashtag to download, as instance of :class:`Hashtag`, or string without leading '#'
:param max_count: Maximum count of pictures to download
:param post_filter: function(post), which returns True if given picture should be downloaded
:param fast_update: If true, abort when first already-downloaded picture is encountered
:param profile_pic: not :option:`--no-profile-pic`.
:param posts: not :option:`--no-posts`.
.. versionchanged:: 4.4
Add parameters `profile_pic` and `posts`.
"""
hashtag = hashtag.lower()
self.context.log("Retrieving pictures with hashtag {}...".format(hashtag))
count = 1
for post in self.get_hashtag_posts(hashtag):
if max_count is not None and count > max_count:
break
self.context.log('[{0:3d}] #{1} '.format(count, hashtag), end='', flush=True)
if post_filter is not None and not post_filter(post):
self.context.log('<skipped>')
continue
count += 1
with self.context.error_catcher('Download hashtag #{}'.format(hashtag)):
downloaded = self.download_post(post, target='#' + hashtag)
if fast_update and not downloaded:
break
if isinstance(hashtag, str):
with self.context.error_catcher("Get hashtag #{}".format(hashtag)):
hashtag = Hashtag.from_name(self.context, hashtag)
if not isinstance(hashtag, Hashtag):
return
target = "#" + hashtag.name
if profile_pic:
with self.context.error_catcher("Download profile picture of {}".format(target)):
self.download_hashtag_profilepic(hashtag)
if posts:
self.context.log("Retrieving pictures with hashtag #{}...".format(hashtag.name))
self.posts_download_loop(hashtag.get_all_posts(), target, fast_update, post_filter,
max_count=max_count)
if self.save_metadata:
json_filename = '{0}/{1}'.format(self.dirname_pattern.format(profile=target,
target=target),
target)
self.save_metadata_json(json_filename, hashtag)
def download_tagged(self, profile: Profile, fast_update: bool = False,
target: Optional[str] = None,
@@ -905,16 +941,18 @@ class Instaloader:
.. versionadded:: 4.1"""
self.context.log("Retrieving tagged posts for profile {}.".format(profile.username))
count = 1
for post in profile.get_tagged_posts():
self.context.log("[%3i/???] " % (count), end="", flush=True)
count += 1
if post_filter is not None and not post_filter(post):
self.context.log('<{} skipped>'.format(post))
with self.context.error_catcher('Download tagged {}'.format(profile.username)):
downloaded = self.download_post(post, target if target else Path(profile.username) / Path(':tagged'))
if fast_update and not downloaded:
break
self.posts_download_loop(profile.get_tagged_posts(),
target if target else Path(profile.username) / Path(':tagged'), fast_update,
post_filter)
def download_igtv(self, profile: Profile, fast_update: bool = False,
post_filter: Optional[Callable[[Post], bool]] = None) -> None:
"""Download IGTV videos of a profile.
.. versionadded:: 4.3"""
self.context.log("Retrieving IGTV videos for profile {}.".format(profile.username))
self.posts_download_loop(profile.get_igtv_posts(), profile.username, fast_update, post_filter,
total_count=profile.igtvcount)
def _get_id_filename(self, profile_name: str) -> str:
if ((format_string_contains_key(self.dirname_pattern, 'profile') or
@@ -988,7 +1026,10 @@ class Instaloader:
def download_profiles(self, profiles: Set[Profile],
profile_pic: bool = True, posts: bool = True,
tagged: bool = False, highlights: bool = False, stories: bool = False,
tagged: bool = False,
igtv: bool = False,
highlights: bool = False,
stories: bool = False,
fast_update: bool = False,
post_filter: Optional[Callable[[Post], bool]] = None,
storyitem_filter: Optional[Callable[[Post], bool]] = None,
@@ -999,6 +1040,7 @@ class Instaloader:
:param profile_pic: not :option:`--no-profile-pic`.
:param posts: not :option:`--no-posts`.
:param tagged: :option:`--tagged`.
:param igtv: :option:`--igtv`.
:param highlights: :option:`--highlights`.
:param stories: :option:`--stories`.
:param fast_update: :option:`--fast-update`.
@@ -1008,7 +1050,11 @@ class Instaloader:
Whether :exc:`LoginRequiredException` and :exc:`PrivateProfileNotFollowedException` should be raised or
catched and printed with :meth:`InstaloaderContext.error_catcher`.
.. versionadded:: 4.1"""
.. versionadded:: 4.1
.. versionchanged:: 4.3
Add `igtv` parameter.
"""
@contextmanager
def _error_raiser(_str):
@@ -1034,7 +1080,7 @@ class Instaloader:
self.save_metadata_json(json_filename, profile)
# Catch some errors
if profile.is_private and (tagged or highlights or posts):
if profile.is_private and (tagged or igtv or highlights or posts):
if not self.context.is_logged_in:
raise LoginRequiredException("--login=USERNAME required.")
if not profile.followed_by_viewer and self.context.username != profile.username:
@@ -1045,6 +1091,11 @@ class Instaloader:
with self.context.error_catcher('Download tagged of {}'.format(profile_name)):
self.download_tagged(profile, fast_update=fast_update, post_filter=post_filter)
# Download IGTV, if requested
if igtv:
with self.context.error_catcher('Download IGTV of {}'.format(profile_name)):
self.download_igtv(profile, fast_update=fast_update, post_filter=post_filter)
# Download highlights, if requested
if highlights:
with self.context.error_catcher('Download highlights of {}'.format(profile_name)):
@@ -1053,32 +1104,8 @@ class Instaloader:
# Iterate over pictures and download them
if posts:
self.context.log("Retrieving posts from profile {}.".format(profile_name))
totalcount = profile.mediacount
count = 1
for post in profile.get_posts():
self.context.log("[%3i/%3i] " % (count, totalcount), end="", flush=True)
count += 1
if post_filter is not None and not post_filter(post):
self.context.log('<skipped>')
continue
with self.context.error_catcher("Download {} of {}".format(post, profile_name)):
# The PostChangedException gets raised if the Post's id/shortcode changed while obtaining
# additional metadata. This is most likely the case if a HTTP redirect takes place while
# resolving the shortcode URL.
# The `post_changed` variable keeps the fast-update functionality alive: A Post which is
# obained after a redirect has probably already been downloaded as a previous Post of the
# same Profile.
# Observed in issue #225: https://github.com/instaloader/instaloader/issues/225
post_changed = False
while True:
try:
downloaded = self.download_post(post, target=profile_name)
break
except PostChangedException:
post_changed = True
continue
if fast_update and not downloaded and not post_changed:
break
self.posts_download_loop(profile.get_posts(), profile_name, fast_update, post_filter,
total_count=profile.mediacount)
if stories and profiles:
with self.context.error_catcher("Download stories"):
@@ -1157,18 +1184,8 @@ class Instaloader:
# Iterate over pictures and download them
self.context.log("Retrieving posts from profile {}.".format(profile_name))
totalcount = profile.mediacount
count = 1
for post in profile.get_posts():
self.context.log("[%3i/%3i] " % (count, totalcount), end="", flush=True)
count += 1
if post_filter is not None and not post_filter(post):
self.context.log('<skipped>')
continue
with self.context.error_catcher('Download profile {}'.format(profile_name)):
downloaded = self.download_post(post, target=profile_name)
if fast_update and not downloaded:
break
self.posts_download_loop(profile.get_posts(), profile_name, fast_update, post_filter,
total_count=profile.mediacount)
def check_if_committed(self, filename: str) -> bool:
"""Checks to see if the current post has been committed.
+8 -2
View File
@@ -113,7 +113,7 @@ class InstaloaderContext:
def close(self):
"""Print error log and close session"""
if self.error_log and not self.quiet:
print("\nErrors occured:", file=sys.stderr)
print("\nErrors occurred:", file=sys.stderr)
for err in self.error_log:
print(err, file=sys.stderr)
self._session.close()
@@ -239,6 +239,12 @@ class InstaloaderContext:
resp_json['message']))
else:
raise ConnectionException("Login error: \"{}\" status.".format(resp_json['status']))
if 'authenticated' not in resp_json:
# Issue #472
if 'message' in resp_json:
raise ConnectionException("Login error: Unexpected response, \"{}\".".format(resp_json['message']))
else:
raise ConnectionException("Login error: Unexpected response, this might indicate a blocked IP.")
if not resp_json['authenticated']:
if resp_json['user']:
# '{"authenticated": false, "user": true, "status": "ok"}'
@@ -343,7 +349,7 @@ class InstaloaderContext:
else:
self._graphql_query_timestamps[query_hash].append(time.monotonic())
else:
text_for_429 = ("HTTP error code 429 was returned because too many queries occured in the last time. "
text_for_429 = ("HTTP error code 429 was returned because too many queries occurred in the last time. "
"Please do not use Instagram in your browser or run multiple instances of Instaloader "
"in parallel.")
print(textwrap.fill(text_for_429), file=sys.stderr)
+267 -8
View File
@@ -98,6 +98,11 @@ class Post:
raise InvalidArgumentException("Wrong mediaid {0}, unable to convert to shortcode".format(str(mediaid)))
return b64encode(mediaid.to_bytes(9, 'big'), b'-_').decode().replace('A', ' ').lstrip().replace(' ', 'A')
@staticmethod
def supported_graphql_types() -> List[str]:
"""The values of __typename fields that the :class:`Post` class can handle."""
return ["GraphImage", "GraphVideo", "GraphSidecar"]
def _asdict(self):
node = self._node
if self._full_metadata_dict:
@@ -134,6 +139,12 @@ class Post:
pic_json = self._context.get_json("p/{0}/".format(self.shortcode), params={})
self._full_metadata_dict = pic_json['entry_data']['PostPage'][0]['graphql']['shortcode_media']
self._rhx_gis_str = pic_json.get('rhx_gis')
if self._full_metadata_dict is None:
# issue #449
self._context.error("Fetching Post metadata failed (issue #449). "
"The following data has been returned:\n"
+ json.dumps(pic_json['entry_data'], indent=2))
raise BadResponseException("Fetching Post metadata failed.")
if self.shortcode != self._full_metadata_dict['shortcode']:
self._node.update(self._full_metadata_dict)
raise PostChangedException
@@ -224,7 +235,11 @@ class Post:
def get_sidecar_nodes(self) -> Iterator[PostSidecarNode]:
"""Sidecar nodes of a Post with typename==GraphSidecar."""
if self.typename == 'GraphSidecar':
for edge in self._field('edge_sidecar_to_children', 'edges'):
edges = self._field('edge_sidecar_to_children', 'edges')
if any(edge['node']['is_video'] for edge in edges):
# video_url is only present in full metadata, issue #558.
edges = self._full_metadata['edge_sidecar_to_children']['edges']
for edge in edges:
node = edge['node']
is_video = node['is_video']
yield PostSidecarNode(is_video=is_video, display_url=node['display_url'],
@@ -413,6 +428,30 @@ class Post:
lambda d: d['data']['shortcode_media']['edge_liked_by'],
self._rhx_gis))
@property
def is_sponsored(self) -> bool:
"""
Whether Post is a sponsored post, equivalent to non-empty :meth:`Post.sponsor_users`.
.. versionadded:: 4.4
"""
try:
sponsor_edges = self._field('edge_media_to_sponsor_user', 'edges')
except KeyError:
return False
return bool(sponsor_edges)
@property
def sponsor_users(self) -> List['Profile']:
"""
The Post's sponsors.
.. versionadded:: 4.4
"""
return ([] if not self.is_sponsored else
[Profile(self._context, edge['node']['sponsor']) for edge in
self._field('edge_media_to_sponsor_user', 'edges')])
@property
def location(self) -> Optional[PostLocation]:
"""
@@ -518,10 +557,11 @@ class Profile:
def _asdict(self):
json_node = self._node.copy()
# remove posts
# remove posts to avoid "Circular reference detected" exception
json_node.pop('edge_media_collections', None)
json_node.pop('edge_owner_to_timeline_media', None)
json_node.pop('edge_saved_media', None)
json_node.pop('edge_felix_video_timeline', None)
if self._iphone_struct_:
json_node['iphone_struct'] = self._iphone_struct_
return json_node
@@ -598,6 +638,10 @@ class Profile:
def mediacount(self) -> int:
return self._metadata('edge_owner_to_timeline_media', 'count')
@property
def igtvcount(self) -> int:
return self._metadata('edge_felix_video_timeline', 'count')
@property
def followers(self) -> int:
return self._metadata('edge_followed_by', 'count')
@@ -610,6 +654,16 @@ class Profile:
def external_url(self) -> Optional[str]:
return self._metadata('external_url')
@property
def is_business_account(self) -> bool:
""".. versionadded:: 4.4"""
return self._metadata('is_business_account')
@property
def business_category_name(self) -> str:
""".. versionadded:: 4.4"""
return self._metadata('business_category_name')
@property
def biography(self) -> str:
return self._metadata('biography')
@@ -741,6 +795,19 @@ class Profile:
lambda d: d['data']['user']['edge_user_to_photos_of_you'],
self._rhx_gis))
def get_igtv_posts(self) -> Iterator[Post]:
"""Retrieve all IGTV posts.
.. versionadded:: 4.3"""
self._obtain_metadata()
yield from (Post(self._context, node, self) for node in
self._context.graphql_node_list('bc78b344a68ed16dd5d7f264681c4c76',
{'id': self.userid},
'https://www.instagram.com/{0}/channel/'.format(self.username),
lambda d: d['data']['user']['edge_felix_video_timeline'],
self._rhx_gis,
self._metadata('edge_felix_video_timeline')))
def get_followers(self) -> Iterator['Profile']:
"""
Retrieve list of followers of given profile.
@@ -771,6 +838,22 @@ class Profile:
lambda d: d['data']['user']['edge_follow'],
self._rhx_gis))
def get_similar_accounts(self) -> Iterator['Profile']:
"""
Retrieve list of suggested / similar accounts for this profile.
To use this, one needs to be logged in.
.. versionadded:: 4.4
"""
if not self._context.is_logged_in:
raise LoginRequiredException("--login required to get a profile's similar accounts.")
self._obtain_metadata()
yield from (Profile(self._context, edge["node"]) for edge in
self._context.graphql_query("ad99dd9d3646cc3c0dda65debcd266a7",
{"user_id": str(self.userid), "include_chaining": True},
"https://www.instagram.com/{0}/".format(self.username),
self._rhx_gis)["data"]["user"]["edge_chaining"]["edges"])
class StoryItem:
"""
@@ -1067,6 +1150,169 @@ class Highlight(Story):
yield from (StoryItem(self._context, item, self.owner_profile) for item in self._items)
class Hashtag:
"""
An Hashtag.
Analogous to :class:`Profile`, get an instance with::
L = Instaloader()
hashtag = Hashtag.from_name(L.context, HASHTAG)
To then download the Hashtag's Posts, do::
for post in hashtag.get_posts():
L.download_post(post, target="#"+hashtag.name)
Also, this class implements == and is hashable.
"""
def __init__(self, context: InstaloaderContext, node: Dict[str, Any]):
assert "name" in node
self._context = context
self._node = node
self._has_full_metadata = False
@classmethod
def from_name(cls, context: InstaloaderContext, name: str):
"""
Create a Hashtag instance from a given hashtag name, without preceeding '#'. Raises an Exception if there is no
hashtag with the given name.
:param context: :attr:`Instaloader.context`
:param name: Hashtag, without preceeding '#'
:raises: :class:`QueryReturnedNotFoundException`
"""
# pylint:disable=protected-access
hashtag = cls(context, {'name': name.lower()})
hashtag._obtain_metadata()
return hashtag
@property
def name(self):
"""Hashtag name lowercased, without preceeding '#'"""
return self._node["name"].lower()
def _query(self, params):
return self._context.get_json("explore/tags/{0}/".format(self.name),
params)["graphql"]["hashtag"]
def _obtain_metadata(self):
if not self._has_full_metadata:
self._node = self._query({"__a": 1})
self._has_full_metadata = True
def _asdict(self):
json_node = self._node.copy()
# remove posts
json_node.pop("edge_hashtag_to_top_posts", None)
json_node.pop("edge_hashtag_to_media", None)
return json_node
def __repr__(self):
return "<Hashtag #{}>".format(self.name)
def __eq__(self, other: object) -> bool:
if isinstance(other, Hashtag):
return self.name == other.name
return NotImplemented
def __hash__(self) -> int:
return hash(self.name)
def _metadata(self, *keys) -> Any:
try:
d = self._node
for key in keys:
d = d[key]
return d
except KeyError:
self._obtain_metadata()
d = self._node
for key in keys:
d = d[key]
return d
@property
def hashtagid(self) -> int:
return int(self._metadata("id"))
@property
def profile_pic_url(self) -> str:
return self._metadata("profile_pic_url")
@property
def description(self) -> str:
return self._metadata("description")
@property
def allow_following(self) -> bool:
return self._metadata("allow_following")
@property
def is_following(self) -> bool:
return self._metadata("is_following")
@property
def is_top_media_only(self) -> bool:
return self._metadata("is_top_media_only")
def get_related_tags(self) -> Iterator["Hashtag"]:
"""Yields similar hashtags."""
yield from (Hashtag(self._context, edge["node"])
for edge in self._metadata("edge_hashtag_to_related_tags", "edges"))
def get_top_posts(self) -> Iterator[Post]:
"""Yields the top posts of the hashtag."""
yield from (Post(self._context, edge["node"])
for edge in self._metadata("edge_hashtag_to_top_posts", "edges"))
@property
def mediacount(self) -> int:
"""
The count of all media associated with this hashtag.
The number of posts with a certain hashtag may differ from the number of posts that can actually be accessed, as
the hashtag count might include private posts
"""
return self._metadata("edge_hashtag_to_media", "count")
def get_posts(self) -> Iterator[Post]:
"""Yields the posts associated with this hashtag."""
self._metadata("edge_hashtag_to_media", "edges")
self._metadata("edge_hashtag_to_media", "page_info")
conn = self._metadata("edge_hashtag_to_media")
yield from (Post(self._context, edge["node"]) for edge in conn["edges"])
while conn["page_info"]["has_next_page"]:
data = self._query({'__a': 1, 'max_id': conn["page_info"]["end_cursor"]})
conn = data["edge_hashtag_to_media"]
yield from (Post(self._context, edge["node"]) for edge in conn["edges"])
def get_all_posts(self) -> Iterator[Post]:
"""Yields all posts, i.e. all most recent posts and the top posts, in chronological order."""
sorted_top_posts = iter(sorted(self.get_top_posts(), key=lambda p: p.date_utc, reverse=True))
other_posts = self.get_posts()
next_top = next(sorted_top_posts, None)
next_other = next(other_posts, None)
while next_top is not None or next_other is not None:
if next_other is None:
yield from sorted_top_posts
break
if next_top is None:
yield from other_posts
break
if next_top == next_other:
yield next_top
next_top = next(sorted_top_posts, None)
next_other = next(other_posts, None)
continue
if next_top.date_utc > next_other.date_utc:
yield next_top
next_top = next(sorted_top_posts, None)
else:
yield next_other
next_other = next(other_posts, None)
class TopSearchResults:
"""
An invocation of this class triggers a search on Instagram for the provided search string.
@@ -1126,6 +1372,17 @@ class TopSearchResults:
if name:
yield name
def get_hashtags(self) -> Iterator[Hashtag]:
"""
Provides the hashtags from the search result.
.. versionadded:: 4.4
"""
for hashtag in self._node.get('hashtags', []):
node = hashtag.get('hashtag', {})
if 'name' in node:
yield Hashtag(self._context, node)
@property
def searchstring(self) -> str:
"""
@@ -1134,17 +1391,17 @@ class TopSearchResults:
return self._searchstring
JsonExportable = Union[Post, Profile, StoryItem]
JsonExportable = Union[Post, Profile, StoryItem, Hashtag]
def save_structure_to_file(structure: JsonExportable, filename: str) -> None:
"""Saves a :class:`Post`, :class:`Profile` or :class:`StoryItem` to a '.json' or '.json.xz' file such that it can
later be loaded by :func:`load_structure_from_file`.
"""Saves a :class:`Post`, :class:`Profile`, :class:`StoryItem` or :class:`Hashtag` to a '.json' or '.json.xz' file
such that it can later be loaded by :func:`load_structure_from_file`.
If the specified filename ends in '.xz', the file will be LZMA compressed. Otherwise, a pretty-printed JSON file
will be created.
:param structure: :class:`Post`, :class:`Profile` or :class:`StoryItem`
:param structure: :class:`Post`, :class:`Profile`, :class:`StoryItem` or :class:`Hashtag`
:param filename: Filename, ends in '.json' or '.json.xz'
"""
json_structure = {'node': structure._asdict(),
@@ -1159,8 +1416,8 @@ def save_structure_to_file(structure: JsonExportable, filename: str) -> None:
def load_structure_from_file(context: InstaloaderContext, filename: str) -> JsonExportable:
"""Loads a :class:`Post`, :class:`Profile` or :class:`StoryItem` from a '.json' or '.json.xz' file that
has been saved by :func:`save_structure_to_file`.
"""Loads a :class:`Post`, :class:`Profile`, :class:`StoryItem` or :class:`Hashtag` from a '.json' or '.json.xz' file
that has been saved by :func:`save_structure_to_file`.
:param context: :attr:`Instaloader.context` linked to the new object, used for additional queries if neccessary.
:param filename: Filename, ends in '.json' or '.json.xz'
@@ -1181,6 +1438,8 @@ def load_structure_from_file(context: InstaloaderContext, filename: str) -> Json
return Profile(context, json_structure['node'])
elif node_type == "StoryItem":
return StoryItem(context, json_structure['node'])
elif node_type == "Hashtag":
return Hashtag(context, json_structure['node'])
else:
raise InvalidArgumentException("{}: Not an Instaloader JSON.".format(filename))
elif 'shortcode' in json_structure:
+12 -1
View File
@@ -12,6 +12,7 @@ import instaloader
PROFILE_WITH_HIGHLIGHTS = 325732271
PUBLIC_PROFILE = "selenagomez"
PUBLIC_PROFILE_ID = 460563723
PUBLIC_PROFILE_WITH_IGTV = "natgeo"
HASHTAG = "kitten"
LOCATION = "362629379"
OWN_USERNAME = "aandergr"
@@ -72,7 +73,7 @@ class TestInstaloaderAnonymously(unittest.TestCase):
self.L.download_hashtag(HASHTAG, NORMAL_MAX_COUNT)
def test_hashtag_paging(self):
for count, post in enumerate(self.L.get_hashtag_posts(HASHTAG)):
for count, post in enumerate(instaloader.Hashtag.from_name(self.L.context, HASHTAG).get_all_posts()):
print(post)
if count == PAGING_MAX_COUNT:
break
@@ -104,6 +105,11 @@ class TestInstaloaderAnonymously(unittest.TestCase):
PAGING_MAX_COUNT):
print(post)
def test_public_profile_igtv(self):
for post in islice(instaloader.Profile.from_username(self.L.context, PUBLIC_PROFILE_WITH_IGTV).get_igtv_posts(),
PAGING_MAX_COUNT):
print(post)
class TestInstaloaderLoggedIn(TestInstaloaderAnonymously):
@@ -172,6 +178,11 @@ class TestInstaloaderLoggedIn(TestInstaloaderAnonymously):
for f in profile.get_followers():
print(f.username)
def test_get_similar_accounts(self):
profile = instaloader.Profile.from_username(self.L.context, OWN_USERNAME)
for f in profile.get_similar_accounts():
print(f.username)
def test_get_likes(self):
for post in instaloader.Profile.from_username(self.L.context, OWN_USERNAME).get_posts():
for like in post.get_likes():