Release of Version 4.4.1

Unicode characters are allowed in hashtags, so instead of just
[A-Za-z0-9], we now match all \w.  Fixes #614.

docs/as-module.rst

@@ -37,7 +37,7 @@ picture, video or sidecar (set of multiple pictures/videos) posted in a user's
 profile. :class:`Instaloader` provides methods to iterate over Posts from a
 certain source::
 
-    for post in L.get_hashtag_posts('cat'):
+    for post in instaloader.Hashtag.from_name(L.context, 'cat').get_posts():
         # post is an instance of instaloader.Post
         L.download_post(post, target='#cat')
 
@@ -63,7 +63,7 @@ certain source::
 - :meth:`Instaloader.get_explore_posts`
    Media that is suggested by Instagram to explore.
 
-- :meth:`Instaloader.get_hashtag_posts`
+- :meth:`Hashtag.get_posts`
    Media associated with given hashtag.
 
 With the :class:`Profile` class, Instaloader also makes it easy to access
@@ -155,6 +155,14 @@ Profiles
 .. autoclass:: Profile
    :no-show-inheritance:
 
+Hashtags
+""""""""
+
+.. autoclass:: Hashtag
+   :no-show-inheritance:
+
+   .. versionadded:: 4.4
+
 TopSearchResults
 """"""""""""""""
 
@@ -166,7 +174,7 @@ TopSearchResults
 Loading and Saving
 """"""""""""""""""
 
-:class:`Post`, :class:`StoryItem` and :class:`Profile` can be saved and loaded
+:class:`Post`, :class:`StoryItem`, :class:`Profile` and :class:`Hashtag` can be saved and loaded
 to/from JSON files.
 
 .. autofunction:: load_structure_from_file

docs/codesnippets/113_only_one_per_user.py

@@ -2,7 +2,7 @@ import instaloader
 
 L = instaloader.Instaloader()
 
-posts = L.get_hashtag_posts('urbanphotography')
+posts = instaloader.Hashtag.from_name(L.context, 'urbanphotography').get_posts()
 
 users = set()
 

docs/codesnippets/121_since_until.py

@@ -5,7 +5,7 @@ import instaloader
 
 L = instaloader.Instaloader()
 
-posts = L.get_hashtag_posts('urbanphotography')
+posts = instaloader.Hashtag.from_name(L.context, 'urbanphotography').get_posts()
 # or
 # posts = instaloader.Profile.from_username(L.context, PROFILE).get_posts()
 

instaloader/__init__.py

@@ -1,7 +1,7 @@
 """Download pictures (or videos) along with their captions and other metadata from Instagram."""
 
 
-__version__ = '4.3'
+__version__ = '4.4.1'
 
 
 try:
@@ -15,5 +15,5 @@ else:
 from .exceptions import *
 from .instaloader import Instaloader
 from .instaloadercontext import InstaloaderContext
-from .structures import (Highlight, Post, PostSidecarNode, PostComment, PostCommentAnswer, PostLocation, Profile, Story,
-                         StoryItem, TopSearchResults, load_structure_from_file, save_structure_to_file)
+from .structures import (Hashtag, Highlight, Post, PostSidecarNode, PostComment, PostCommentAnswer, PostLocation,
+                         Profile, Story, StoryItem, TopSearchResults, load_structure_from_file, save_structure_to_file)

instaloader/__main__.py

@@ -3,6 +3,7 @@
 import ast
 import datetime
 import os
+import re
 import sys
 from argparse import ArgumentParser, SUPPRESS
 from typing import List, Optional
@@ -136,18 +137,19 @@ def _main(instaloader: Instaloader, targetlist: List[str],
             # strip '/' characters to be more shell-autocompletion-friendly
             target = target.rstrip('/')
             with instaloader.context.error_catcher(target):
-                if target[0] == '@':
+                if re.match(r"^@[A-Za-z0-9._]+$", target):
                     instaloader.context.log("Retrieving followees of %s..." % target[1:])
                     profile = Profile.from_username(instaloader.context, target[1:])
                     for followee in profile.get_followees():
                         instaloader.save_profile_id(followee)
                         profiles.add(followee)
-                elif target[0] == '#':
+                elif re.match(r"^#\w+$", target):
                     instaloader.download_hashtag(hashtag=target[1:], max_count=max_count, fast_update=fast_update,
-                                                 post_filter=post_filter)
-                elif target[0] == '-':
+                                                 post_filter=post_filter,
+                                                 profile_pic=download_profile_pic, posts=download_posts)
+                elif re.match(r"^-[A-Za-z0-9-_]+$", target):
                     instaloader.download_post(Post.from_shortcode(instaloader.context, target[1:]), target)
-                elif target[0] == "%":
+                elif re.match(r"^%[0-9]+$", target):
                     instaloader.download_location(location=target[1:], max_count=max_count, fast_update=fast_update,
                                                   post_filter=post_filter)
                 elif target == ":feed":
@@ -158,7 +160,7 @@ def _main(instaloader: Instaloader, targetlist: List[str],
                 elif target == ":saved":
                     instaloader.download_saved_posts(fast_update=fast_update, max_count=max_count,
                                                      post_filter=post_filter)
-                else:
+                elif re.match(r"^[A-Za-z0-9._]+$", target):
                     try:
                         profile = instaloader.check_profile_id(target)
                         if instaloader.context.is_logged_in and profile.has_blocked_viewer:
@@ -184,6 +186,13 @@ def _main(instaloader: Instaloader, targetlist: List[str],
                                                               .format(target, err))
                         else:
                             raise
+                else:
+                    target_type = {
+                        '#': 'hashtag',
+                        '%': 'location',
+                        '-': 'shortcode',
+                    }.get(target[0], 'username')
+                    raise ProfileNotExistsException('Invalid {} {}'.format(target_type, target))
         if len(profiles) > 1:
             instaloader.context.log("Downloading {} profiles: {}".format(len(profiles),
                                                                          ' '.join([p.username for p in profiles])))

instaloader/instaloader.py

@@ -21,7 +21,7 @@ import urllib3  # type: ignore
 
 from .exceptions import *
 from .instaloadercontext import InstaloaderContext
-from .structures import (Highlight, JsonExportable, Post, PostLocation, Profile, Story, StoryItem,
+from .structures import (Hashtag, Highlight, JsonExportable, Post, PostLocation, Profile, Story, StoryItem,
                          save_structure_to_file, load_structure_from_file)
 
 
@@ -358,7 +358,7 @@ class Instaloader:
         self.context.log('geo', end=' ', flush=True)
 
     @_retry_on_connection_error
-    def download_title_pic(self, url: str, target: Union[str, Path], name_suffix: str, owner_profile: Profile,
+    def download_title_pic(self, url: str, target: Union[str, Path], name_suffix: str, owner_profile: Optional[Profile],
                            _attempt: int = 1) -> None:
         """Downloads and saves a picture that does not have an association with a Post or StoryItem, such as a
         Profile picture or a Highlight cover picture. Modification time is taken from the HTTP response headers.
@@ -380,7 +380,8 @@ class Instaloader:
         pic_extension = 'jpg'
         if ((format_string_contains_key(self.dirname_pattern, 'profile') or
              format_string_contains_key(self.dirname_pattern, 'target'))):
-            filename = '{0}/{1}_{2}.{3}'.format(self.dirname_pattern.format(profile=owner_profile.username.lower(),
+            profile_str = owner_profile.username.lower() if owner_profile is not None else target
+            filename = '{0}/{1}_{2}.{3}'.format(self.dirname_pattern.format(profile=profile_str,
                                                                             target=target),
                                                 pic_identifier, name_suffix, pic_extension)
         else:
@@ -408,6 +409,12 @@ class Instaloader:
         .. versionadded:: 4.3"""
         self.download_title_pic(highlight.cover_url, target, 'cover', highlight.owner_profile)
 
+    def download_hashtag_profilepic(self, hashtag: Hashtag) -> None:
+        """Downloads and saves the profile picture of a Hashtag.
+
+        .. versionadded:: 4.4"""
+        self.download_title_pic(hashtag.profile_pic_url, '#' + hashtag.name, 'profile_pic', None)
+
     @_requires_login
     def save_session_to_file(self, filename: Optional[str] = None) -> None:
         """Saves internally stored :class:`requests.Session` object.
@@ -691,6 +698,66 @@ class Instaloader:
                     if fast_update and not downloaded:
                         break
 
+    def posts_download_loop(self,
+                            posts: Iterator[Post],
+                            target: Union[str, Path],
+                            fast_update: bool = False,
+                            post_filter: Optional[Callable[[Post], bool]] = None,
+                            max_count: Optional[int] = None,
+                            total_count: Optional[int] = None) -> None:
+        """
+        Download the Posts returned by given Post Iterator.
+
+        ..versionadded:: 4.4
+
+        :param posts: Post Iterator to loop through.
+        :param target: Target name
+        :param fast_update: :option:`--fast-update`
+        :param post_filter: :option:`--post-filter`
+        :param max_count: Maximum count of Posts to download (:option:`--count`)
+        :param total_count: Total number of posts returned by given iterator
+        """
+        for number, post in enumerate(posts):
+            if max_count is not None and number >= max_count:
+                break
+            if total_count is not None:
+                self.context.log("[{0:{w}d}/{1:{w}d}] ".format(number + 1, total_count,
+                                                               w=len(str(total_count))),
+                                 end="", flush=True)
+            else:
+                if max_count is not None:
+                    self.context.log("[{0:{w}d}/{1:{w}d}] ".format(number + 1, max_count,
+                                                                   w=len(str(max_count))),
+                                     end="", flush=True)
+                else:
+                    self.context.log("[{:3d}] ".format(number + 1), end="", flush=True)
+            if post_filter is not None:
+                try:
+                    if not post_filter(post):
+                        self.context.log("{} skipped".format(post))
+                        continue
+                except (InstaloaderException, KeyError, TypeError) as err:
+                    self.context.error("{} skipped. Filter evaluation failed: {}".format(post, err))
+                    continue
+            with self.context.error_catcher("Download {} of {}".format(post, target)):
+                # The PostChangedException gets raised if the Post's id/shortcode changed while obtaining
+                # additional metadata. This is most likely the case if a HTTP redirect takes place while
+                # resolving the shortcode URL.
+                # The `post_changed` variable keeps the fast-update functionality alive: A Post which is
+                # obained after a redirect has probably already been downloaded as a previous Post of the
+                # same Profile.
+                # Observed in issue #225: https://github.com/instaloader/instaloader/issues/225
+                post_changed = False
+                while True:
+                    try:
+                        downloaded = self.download_post(post, target=target)
+                        break
+                    except PostChangedException:
+                        post_changed = True
+                        continue
+                if fast_update and not downloaded and not post_changed:
+                    break
+
     @_requires_login
     def get_feed_posts(self) -> Iterator[Post]:
         """Get Posts of the user's feed.
@@ -703,8 +770,10 @@ class Instaloader:
 
         while True:
             feed = data["user"]["edge_web_feed_timeline"]
-            yield from (Post(self.context, edge["node"]) for edge in feed["edges"]
-                        if not edge["node"]["__typename"] == "GraphSuggestedUserFeedUnit")
+            for edge in feed["edges"]:
+                node = edge["node"]
+                if node.get("__typename") in Post.supported_graphql_types() and node.get("shortcode") is not None:
+                    yield Post(self.context, node)
             if not feed["page_info"]["has_next_page"]:
                 break
             data = self.context.graphql_query("d6f4427fbe92d846298cf93df0b937d3",
@@ -733,20 +802,7 @@ class Instaloader:
         :raises LoginRequiredException: If called without being logged in.
         """
         self.context.log("Retrieving pictures from your feed...")
-        count = 1
-        for post in self.get_feed_posts():
-            if max_count is not None and count > max_count:
-                break
-            name = post.owner_username
-            if post_filter is not None and not post_filter(post):
-                self.context.log("<pic by %s skipped>" % name, flush=True)
-                continue
-            self.context.log("[%3i] %s " % (count, name), end="", flush=True)
-            count += 1
-            with self.context.error_catcher('Download feed'):
-                downloaded = self.download_post(post, target=':feed')
-                if fast_update and not downloaded:
-                    break
+        self.posts_download_loop(self.get_feed_posts(), ":feed", fast_update, post_filter, max_count=max_count)
 
     @_requires_login
     def download_saved_posts(self, max_count: int = None, fast_update: bool = False,
@@ -759,20 +815,9 @@ class Instaloader:
         :raises LoginRequiredException: If called without being logged in.
         """
         self.context.log("Retrieving saved posts...")
-        count = 1
-        assert self.context.username is not None
-        for post in Profile.from_username(self.context, self.context.username).get_saved_posts():
-            if max_count is not None and count > max_count:
-                break
-            if post_filter is not None and not post_filter(post):
-                self.context.log("<{} skipped>".format(post), flush=True)
-                continue
-            self.context.log("[{:>3}] ".format(count), end=str(), flush=True)
-            count += 1
-            with self.context.error_catcher('Download saved posts'):
-                downloaded = self.download_post(post, target=':saved')
-                if fast_update and not downloaded:
-                    break
+        assert self.context.username is not None  # safe due to @_requires_login; required by typechecker
+        self.posts_download_loop(Profile.from_username(self.context, self.context.username).get_saved_posts(), ":saved",
+                                 fast_update, post_filter, max_count=max_count)
 
     @_requires_login
     def get_location_posts(self, location: str) -> Iterator[Post]:
@@ -823,19 +868,8 @@ class Instaloader:
            Require being logged in (as required by Instagram)
         """
         self.context.log("Retrieving pictures for location {}...".format(location))
-        count = 1
-        for post in self.get_location_posts(location):
-            if max_count is not None and count > max_count:
-                break
-            self.context.log('[{0:3d}] %{1} '.format(count, location), end='', flush=True)
-            if post_filter is not None and not post_filter(post):
-                self.context.log('<skipped>')
-                continue
-            count += 1
-            with self.context.error_catcher('Download location {}'.format(location)):
-                downloaded = self.download_post(post, target='%' + location)
-                if fast_update and not downloaded:
-                    break
+        self.posts_download_loop(self.get_location_posts(location), "%" + location, fast_update, post_filter,
+                                 max_count=max_count)
 
     @_requires_login
     def get_explore_posts(self) -> Iterator[Post]:
@@ -852,24 +886,18 @@ class Instaloader:
                                                                data.get('rhx_gis')))
 
     def get_hashtag_posts(self, hashtag: str) -> Iterator[Post]:
-        """Get Posts associated with a #hashtag."""
-        has_next_page = True
-        end_cursor = None
-        while has_next_page:
-            if end_cursor:
-                params = {'__a': 1, 'max_id': end_cursor}
-            else:
-                params = {'__a': 1}
-            hashtag_data = self.context.get_json('explore/tags/{0}/'.format(hashtag),
-                                                 params)['graphql']['hashtag']['edge_hashtag_to_media']
-            yield from (Post(self.context, edge['node']) for edge in hashtag_data['edges'])
-            has_next_page = hashtag_data['page_info']['has_next_page']
-            end_cursor = hashtag_data['page_info']['end_cursor']
+        """Get Posts associated with a #hashtag.
 
-    def download_hashtag(self, hashtag: str,
+        .. deprecated:: 4.4
+           Use :meth:`Hashtag.get_posts`."""
+        return Hashtag.from_name(self.context, hashtag).get_posts()
+
+    def download_hashtag(self, hashtag: Union[Hashtag, str],
                          max_count: Optional[int] = None,
                          post_filter: Optional[Callable[[Post], bool]] = None,
-                         fast_update: bool = False) -> None:
+                         fast_update: bool = False,
+                         profile_pic: bool = True,
+                         posts: bool = True) -> None:
         """Download pictures of one hashtag.
 
         To download the last 30 pictures with hashtag #cat, do::
@@ -877,26 +905,34 @@ class Instaloader:
             loader = Instaloader()
             loader.download_hashtag('cat', max_count=30)
 
-        :param hashtag: Hashtag to download, without leading '#'
+        :param hashtag: Hashtag to download, as instance of :class:`Hashtag`, or string without leading '#'
         :param max_count: Maximum count of pictures to download
         :param post_filter: function(post), which returns True if given picture should be downloaded
         :param fast_update: If true, abort when first already-downloaded picture is encountered
+        :param profile_pic: not :option:`--no-profile-pic`.
+        :param posts: not :option:`--no-posts`.
+
+        .. versionchanged:: 4.4
+           Add parameters `profile_pic` and `posts`.
         """
-        hashtag = hashtag.lower()
-        self.context.log("Retrieving pictures with hashtag {}...".format(hashtag))
-        count = 1
-        for post in self.get_hashtag_posts(hashtag):
-            if max_count is not None and count > max_count:
-                break
-            self.context.log('[{0:3d}] #{1} '.format(count, hashtag), end='', flush=True)
-            if post_filter is not None and not post_filter(post):
-                self.context.log('<skipped>')
-                continue
-            count += 1
-            with self.context.error_catcher('Download hashtag #{}'.format(hashtag)):
-                downloaded = self.download_post(post, target='#' + hashtag)
-                if fast_update and not downloaded:
-                    break
+        if isinstance(hashtag, str):
+            with self.context.error_catcher("Get hashtag #{}".format(hashtag)):
+                hashtag = Hashtag.from_name(self.context, hashtag)
+        if not isinstance(hashtag, Hashtag):
+            return
+        target = "#" + hashtag.name
+        if profile_pic:
+            with self.context.error_catcher("Download profile picture of {}".format(target)):
+                self.download_hashtag_profilepic(hashtag)
+        if posts:
+            self.context.log("Retrieving pictures with hashtag #{}...".format(hashtag.name))
+            self.posts_download_loop(hashtag.get_all_posts(), target, fast_update, post_filter,
+                                     max_count=max_count)
+        if self.save_metadata:
+            json_filename = '{0}/{1}'.format(self.dirname_pattern.format(profile=target,
+                                                                         target=target),
+                                             target)
+            self.save_metadata_json(json_filename, hashtag)
 
     def download_tagged(self, profile: Profile, fast_update: bool = False,
                         target: Optional[str] = None,
@@ -905,17 +941,9 @@ class Instaloader:
 
         .. versionadded:: 4.1"""
         self.context.log("Retrieving tagged posts for profile {}.".format(profile.username))
-        count = 1
-        for post in profile.get_tagged_posts():
-            self.context.log("[%3i/???] " % (count), end="", flush=True)
-            count += 1
-            if post_filter is not None and not post_filter(post):
-                self.context.log('<{} skipped>'.format(post))
-                continue
-            with self.context.error_catcher('Download tagged {}'.format(profile.username)):
-                downloaded = self.download_post(post, target if target else Path(profile.username) / Path(':tagged'))
-                if fast_update and not downloaded:
-                    break
+        self.posts_download_loop(profile.get_tagged_posts(),
+                                 target if target else Path(profile.username) / Path(':tagged'), fast_update,
+                                 post_filter)
 
     def download_igtv(self, profile: Profile, fast_update: bool = False,
                       post_filter: Optional[Callable[[Post], bool]] = None) -> None:
@@ -923,16 +951,8 @@ class Instaloader:
 
         .. versionadded:: 4.3"""
         self.context.log("Retrieving IGTV videos for profile {}.".format(profile.username))
-        for number, post in enumerate(profile.get_igtv_posts()):
-            self.context.log("[{0:{w}d}/{1:{w}d}] ".format(number, profile.igtvcount, w=len(str(profile.igtvcount))),
-                             end="", flush=True)
-            if post_filter is not None and not post_filter(post):
-                self.context.log('<{} skipped>'.format(post))
-                continue
-            with self.context.error_catcher('Download IGTV {}'.format(post.shortcode)):
-                downloaded = self.download_post(post, target=profile.username)
-                if fast_update and not downloaded:
-                    break
+        self.posts_download_loop(profile.get_igtv_posts(), profile.username, fast_update, post_filter,
+                                 total_count=profile.igtvcount)
 
     def _get_id_filename(self, profile_name: str) -> str:
         if ((format_string_contains_key(self.dirname_pattern, 'profile') or
@@ -1084,32 +1104,8 @@ class Instaloader:
                 # Iterate over pictures and download them
                 if posts:
                     self.context.log("Retrieving posts from profile {}.".format(profile_name))
-                    totalcount = profile.mediacount
-                    count = 1
-                    for post in profile.get_posts():
-                        self.context.log("[%3i/%3i] " % (count, totalcount), end="", flush=True)
-                        count += 1
-                        if post_filter is not None and not post_filter(post):
-                            self.context.log('<skipped>')
-                            continue
-                        with self.context.error_catcher("Download {} of {}".format(post, profile_name)):
-                            # The PostChangedException gets raised if the Post's id/shortcode changed while obtaining
-                            # additional metadata. This is most likely the case if a HTTP redirect takes place while
-                            # resolving the shortcode URL.
-                            # The `post_changed` variable keeps the fast-update functionality alive: A Post which is
-                            # obained after a redirect has probably already been downloaded as a previous Post of the
-                            # same Profile.
-                            # Observed in issue #225: https://github.com/instaloader/instaloader/issues/225
-                            post_changed = False
-                            while True:
-                                try:
-                                    downloaded = self.download_post(post, target=profile_name)
-                                    break
-                                except PostChangedException:
-                                    post_changed = True
-                                    continue
-                            if fast_update and not downloaded and not post_changed:
-                                break
+                    self.posts_download_loop(profile.get_posts(), profile_name, fast_update, post_filter,
+                                             total_count=profile.mediacount)
 
         if stories and profiles:
             with self.context.error_catcher("Download stories"):
@@ -1188,18 +1184,8 @@ class Instaloader:
 
         # Iterate over pictures and download them
         self.context.log("Retrieving posts from profile {}.".format(profile_name))
-        totalcount = profile.mediacount
-        count = 1
-        for post in profile.get_posts():
-            self.context.log("[%3i/%3i] " % (count, totalcount), end="", flush=True)
-            count += 1
-            if post_filter is not None and not post_filter(post):
-                self.context.log('<skipped>')
-                continue
-            with self.context.error_catcher('Download profile {}'.format(profile_name)):
-                downloaded = self.download_post(post, target=profile_name)
-                if fast_update and not downloaded:
-                    break
+        self.posts_download_loop(profile.get_posts(), profile_name, fast_update, post_filter,
+                                 total_count=profile.mediacount)
 
     def check_if_committed(self, filename: str) -> bool:
         """Checks to see if the current post has been committed.

instaloader/instaloadercontext.py

@@ -113,7 +113,7 @@ class InstaloaderContext:
     def close(self):
         """Print error log and close session"""
         if self.error_log and not self.quiet:
-            print("\nErrors occured:", file=sys.stderr)
+            print("\nErrors occurred:", file=sys.stderr)
             for err in self.error_log:
                 print(err, file=sys.stderr)
         self._session.close()
@@ -239,6 +239,12 @@ class InstaloaderContext:
                                                                                                resp_json['message']))
             else:
                 raise ConnectionException("Login error: \"{}\" status.".format(resp_json['status']))
+        if 'authenticated' not in resp_json:
+            # Issue #472
+            if 'message' in resp_json:
+                raise ConnectionException("Login error: Unexpected response, \"{}\".".format(resp_json['message']))
+            else:
+                raise ConnectionException("Login error: Unexpected response, this might indicate a blocked IP.")
         if not resp_json['authenticated']:
             if resp_json['user']:
                 # '{"authenticated": false, "user": true, "status": "ok"}'
@@ -343,7 +349,7 @@ class InstaloaderContext:
             else:
                 self._graphql_query_timestamps[query_hash].append(time.monotonic())
         else:
-            text_for_429 = ("HTTP error code 429 was returned because too many queries occured in the last time. "
+            text_for_429 = ("HTTP error code 429 was returned because too many queries occurred in the last time. "
                             "Please do not use Instagram in your browser or run multiple instances of Instaloader "
                             "in parallel.")
             print(textwrap.fill(text_for_429), file=sys.stderr)

instaloader/structures.py

@@ -98,6 +98,11 @@ class Post:
             raise InvalidArgumentException("Wrong mediaid {0}, unable to convert to shortcode".format(str(mediaid)))
         return b64encode(mediaid.to_bytes(9, 'big'), b'-_').decode().replace('A', ' ').lstrip().replace(' ', 'A')
 
+    @staticmethod
+    def supported_graphql_types() -> List[str]:
+        """The values of __typename fields that the :class:`Post` class can handle."""
+        return ["GraphImage", "GraphVideo", "GraphSidecar"]
+
     def _asdict(self):
         node = self._node
         if self._full_metadata_dict:
@@ -134,6 +139,12 @@ class Post:
             pic_json = self._context.get_json("p/{0}/".format(self.shortcode), params={})
             self._full_metadata_dict = pic_json['entry_data']['PostPage'][0]['graphql']['shortcode_media']
             self._rhx_gis_str = pic_json.get('rhx_gis')
+            if self._full_metadata_dict is None:
+                # issue #449
+                self._context.error("Fetching Post metadata failed (issue #449). "
+                                    "The following data has been returned:\n"
+                                    + json.dumps(pic_json['entry_data'], indent=2))
+                raise BadResponseException("Fetching Post metadata failed.")
             if self.shortcode != self._full_metadata_dict['shortcode']:
                 self._node.update(self._full_metadata_dict)
                 raise PostChangedException
@@ -224,7 +235,11 @@ class Post:
     def get_sidecar_nodes(self) -> Iterator[PostSidecarNode]:
         """Sidecar nodes of a Post with typename==GraphSidecar."""
         if self.typename == 'GraphSidecar':
-            for edge in self._field('edge_sidecar_to_children', 'edges'):
+            edges = self._field('edge_sidecar_to_children', 'edges')
+            if any(edge['node']['is_video'] for edge in edges):
+                # video_url is only present in full metadata, issue #558.
+                edges = self._full_metadata['edge_sidecar_to_children']['edges']
+            for edge in edges:
                 node = edge['node']
                 is_video = node['is_video']
                 yield PostSidecarNode(is_video=is_video, display_url=node['display_url'],
@@ -413,6 +428,30 @@ class Post:
                                                     lambda d: d['data']['shortcode_media']['edge_liked_by'],
                                                     self._rhx_gis))
 
+    @property
+    def is_sponsored(self) -> bool:
+        """
+        Whether Post is a sponsored post, equivalent to non-empty :meth:`Post.sponsor_users`.
+
+        .. versionadded:: 4.4
+        """
+        try:
+            sponsor_edges = self._field('edge_media_to_sponsor_user', 'edges')
+        except KeyError:
+            return False
+        return bool(sponsor_edges)
+
+    @property
+    def sponsor_users(self) -> List['Profile']:
+        """
+        The Post's sponsors.
+
+        .. versionadded:: 4.4
+        """
+        return ([] if not self.is_sponsored else
+                [Profile(self._context, edge['node']['sponsor']) for edge in
+                 self._field('edge_media_to_sponsor_user', 'edges')])
+
     @property
     def location(self) -> Optional[PostLocation]:
         """
@@ -615,6 +654,16 @@ class Profile:
     def external_url(self) -> Optional[str]:
         return self._metadata('external_url')
 
+    @property
+    def is_business_account(self) -> bool:
+        """.. versionadded:: 4.4"""
+        return self._metadata('is_business_account')
+
+    @property
+    def business_category_name(self) -> str:
+        """.. versionadded:: 4.4"""
+        return self._metadata('business_category_name')
+
     @property
     def biography(self) -> str:
         return self._metadata('biography')
@@ -789,6 +838,22 @@ class Profile:
                                                     lambda d: d['data']['user']['edge_follow'],
                                                     self._rhx_gis))
 
+    def get_similar_accounts(self) -> Iterator['Profile']:
+        """
+        Retrieve list of suggested / similar accounts for this profile.
+        To use this, one needs to be logged in.
+
+        .. versionadded:: 4.4
+        """
+        if not self._context.is_logged_in:
+            raise LoginRequiredException("--login required to get a profile's similar accounts.")
+        self._obtain_metadata()
+        yield from (Profile(self._context, edge["node"]) for edge in
+                    self._context.graphql_query("ad99dd9d3646cc3c0dda65debcd266a7",
+                                                {"user_id": str(self.userid), "include_chaining": True},
+                                                "https://www.instagram.com/{0}/".format(self.username),
+                                                self._rhx_gis)["data"]["user"]["edge_chaining"]["edges"])
+
 
 class StoryItem:
     """
@@ -1085,6 +1150,169 @@ class Highlight(Story):
         yield from (StoryItem(self._context, item, self.owner_profile) for item in self._items)
 
 
+class Hashtag:
+    """
+    An Hashtag.
+
+    Analogous to :class:`Profile`, get an instance with::
+
+       L = Instaloader()
+       hashtag = Hashtag.from_name(L.context, HASHTAG)
+
+    To then download the Hashtag's Posts, do::
+
+       for post in hashtag.get_posts():
+          L.download_post(post, target="#"+hashtag.name)
+
+    Also, this class implements == and is hashable.
+    """
+    def __init__(self, context: InstaloaderContext, node: Dict[str, Any]):
+        assert "name" in node
+        self._context = context
+        self._node = node
+        self._has_full_metadata = False
+
+    @classmethod
+    def from_name(cls, context: InstaloaderContext, name: str):
+        """
+        Create a Hashtag instance from a given hashtag name, without preceeding '#'. Raises an Exception if there is no
+        hashtag with the given name.
+
+        :param context: :attr:`Instaloader.context`
+        :param name: Hashtag, without preceeding '#'
+        :raises: :class:`QueryReturnedNotFoundException`
+        """
+        # pylint:disable=protected-access
+        hashtag = cls(context, {'name': name.lower()})
+        hashtag._obtain_metadata()
+        return hashtag
+
+    @property
+    def name(self):
+        """Hashtag name lowercased, without preceeding '#'"""
+        return self._node["name"].lower()
+
+    def _query(self, params):
+        return self._context.get_json("explore/tags/{0}/".format(self.name),
+                                      params)["graphql"]["hashtag"]
+
+    def _obtain_metadata(self):
+        if not self._has_full_metadata:
+            self._node = self._query({"__a": 1})
+            self._has_full_metadata = True
+
+    def _asdict(self):
+        json_node = self._node.copy()
+        # remove posts
+        json_node.pop("edge_hashtag_to_top_posts", None)
+        json_node.pop("edge_hashtag_to_media", None)
+        return json_node
+
+    def __repr__(self):
+        return "<Hashtag #{}>".format(self.name)
+
+    def __eq__(self, other: object) -> bool:
+        if isinstance(other, Hashtag):
+            return self.name == other.name
+        return NotImplemented
+
+    def __hash__(self) -> int:
+        return hash(self.name)
+
+    def _metadata(self, *keys) -> Any:
+        try:
+            d = self._node
+            for key in keys:
+                d = d[key]
+            return d
+        except KeyError:
+            self._obtain_metadata()
+            d = self._node
+            for key in keys:
+                d = d[key]
+            return d
+
+    @property
+    def hashtagid(self) -> int:
+        return int(self._metadata("id"))
+
+    @property
+    def profile_pic_url(self) -> str:
+        return self._metadata("profile_pic_url")
+
+    @property
+    def description(self) -> str:
+        return self._metadata("description")
+
+    @property
+    def allow_following(self) -> bool:
+        return self._metadata("allow_following")
+
+    @property
+    def is_following(self) -> bool:
+        return self._metadata("is_following")
+
+    @property
+    def is_top_media_only(self) -> bool:
+        return self._metadata("is_top_media_only")
+
+    def get_related_tags(self) -> Iterator["Hashtag"]:
+        """Yields similar hashtags."""
+        yield from (Hashtag(self._context, edge["node"])
+                    for edge in self._metadata("edge_hashtag_to_related_tags", "edges"))
+
+    def get_top_posts(self) -> Iterator[Post]:
+        """Yields the top posts of the hashtag."""
+        yield from (Post(self._context, edge["node"])
+                    for edge in self._metadata("edge_hashtag_to_top_posts", "edges"))
+
+    @property
+    def mediacount(self) -> int:
+        """
+        The count of all media associated with this hashtag.
+
+        The number of posts with a certain hashtag may differ from the number of posts that can actually be accessed, as
+        the hashtag count might include private posts
+        """
+        return self._metadata("edge_hashtag_to_media", "count")
+
+    def get_posts(self) -> Iterator[Post]:
+        """Yields the posts associated with this hashtag."""
+        self._metadata("edge_hashtag_to_media", "edges")
+        self._metadata("edge_hashtag_to_media", "page_info")
+        conn = self._metadata("edge_hashtag_to_media")
+        yield from (Post(self._context, edge["node"]) for edge in conn["edges"])
+        while conn["page_info"]["has_next_page"]:
+            data = self._query({'__a': 1, 'max_id': conn["page_info"]["end_cursor"]})
+            conn = data["edge_hashtag_to_media"]
+            yield from (Post(self._context, edge["node"]) for edge in conn["edges"])
+
+    def get_all_posts(self) -> Iterator[Post]:
+        """Yields all posts, i.e. all most recent posts and the top posts, in chronological order."""
+        sorted_top_posts = iter(sorted(self.get_top_posts(), key=lambda p: p.date_utc, reverse=True))
+        other_posts = self.get_posts()
+        next_top = next(sorted_top_posts, None)
+        next_other = next(other_posts, None)
+        while next_top is not None or next_other is not None:
+            if next_other is None:
+                yield from sorted_top_posts
+                break
+            if next_top is None:
+                yield from other_posts
+                break
+            if next_top == next_other:
+                yield next_top
+                next_top = next(sorted_top_posts, None)
+                next_other = next(other_posts, None)
+                continue
+            if next_top.date_utc > next_other.date_utc:
+                yield next_top
+                next_top = next(sorted_top_posts, None)
+            else:
+                yield next_other
+                next_other = next(other_posts, None)
+
+
 class TopSearchResults:
     """
     An invocation of this class triggers a search on Instagram for the provided search string.
@@ -1144,6 +1372,17 @@ class TopSearchResults:
             if name:
                 yield name
 
+    def get_hashtags(self) -> Iterator[Hashtag]:
+        """
+        Provides the hashtags from the search result.
+
+        .. versionadded:: 4.4
+        """
+        for hashtag in self._node.get('hashtags', []):
+            node = hashtag.get('hashtag', {})
+            if 'name' in node:
+                yield Hashtag(self._context, node)
+
     @property
     def searchstring(self) -> str:
         """
@@ -1152,17 +1391,17 @@ class TopSearchResults:
         return self._searchstring
 
 
-JsonExportable = Union[Post, Profile, StoryItem]
+JsonExportable = Union[Post, Profile, StoryItem, Hashtag]
 
 
 def save_structure_to_file(structure: JsonExportable, filename: str) -> None:
-    """Saves a :class:`Post`, :class:`Profile` or :class:`StoryItem` to a '.json' or '.json.xz' file such that it can
-    later be loaded by :func:`load_structure_from_file`.
+    """Saves a :class:`Post`, :class:`Profile`, :class:`StoryItem` or :class:`Hashtag` to a '.json' or '.json.xz' file
+    such that it can later be loaded by :func:`load_structure_from_file`.
 
     If the specified filename ends in '.xz', the file will be LZMA compressed. Otherwise, a pretty-printed JSON file
     will be created.
 
-    :param structure: :class:`Post`, :class:`Profile` or :class:`StoryItem`
+    :param structure: :class:`Post`, :class:`Profile`, :class:`StoryItem` or :class:`Hashtag`
     :param filename: Filename, ends in '.json' or '.json.xz'
     """
     json_structure = {'node': structure._asdict(),
@@ -1177,8 +1416,8 @@ def save_structure_to_file(structure: JsonExportable, filename: str) -> None:
 
 
 def load_structure_from_file(context: InstaloaderContext, filename: str) -> JsonExportable:
-    """Loads a :class:`Post`, :class:`Profile` or :class:`StoryItem` from a '.json' or '.json.xz' file that
-    has been saved by :func:`save_structure_to_file`.
+    """Loads a :class:`Post`, :class:`Profile`, :class:`StoryItem` or :class:`Hashtag` from a '.json' or '.json.xz' file
+    that has been saved by :func:`save_structure_to_file`.
 
     :param context: :attr:`Instaloader.context` linked to the new object, used for additional queries if neccessary.
     :param filename: Filename, ends in '.json' or '.json.xz'
@@ -1199,6 +1438,8 @@ def load_structure_from_file(context: InstaloaderContext, filename: str) -> Json
             return Profile(context, json_structure['node'])
         elif node_type == "StoryItem":
             return StoryItem(context, json_structure['node'])
+        elif node_type == "Hashtag":
+            return Hashtag(context, json_structure['node'])
         else:
             raise InvalidArgumentException("{}: Not an Instaloader JSON.".format(filename))
     elif 'shortcode' in json_structure:

test/instaloader_unittests.py

@@ -73,7 +73,7 @@ class TestInstaloaderAnonymously(unittest.TestCase):
         self.L.download_hashtag(HASHTAG, NORMAL_MAX_COUNT)
 
     def test_hashtag_paging(self):
-        for count, post in enumerate(self.L.get_hashtag_posts(HASHTAG)):
+        for count, post in enumerate(instaloader.Hashtag.from_name(self.L.context, HASHTAG).get_all_posts()):
             print(post)
             if count == PAGING_MAX_COUNT:
                 break
@@ -178,6 +178,11 @@ class TestInstaloaderLoggedIn(TestInstaloaderAnonymously):
         for f in profile.get_followers():
             print(f.username)
 
+    def test_get_similar_accounts(self):
+        profile = instaloader.Profile.from_username(self.L.context, OWN_USERNAME)
+        for f in profile.get_similar_accounts():
+            print(f.username)
+
     def test_get_likes(self):
         for post in instaloader.Profile.from_username(self.L.context, OWN_USERNAME).get_posts():
             for like in post.get_likes():