Third Release Candidate for Version 4.4.4

Addresses a suggestion in #630.

.github/workflows/stale.yml

@@ -2,7 +2,7 @@ name: Mark stale issues and pull requests
 
 on:
   schedule:
-  - cron: "15 16 * * *"
+  - cron: "10 14 * * *"
 
 jobs:
   stale:
@@ -10,6 +10,16 @@ jobs:
     runs-on: ubuntu-latest
     
     steps:
+    - uses: actions/stale@v3
+      with:
+        repo-token: ${{ secrets.GITHUB_TOKEN }}
+        only-labels: 'question'
+        stale-issue-message: 'There has been no activity on this question for an extended period of time. This issue will be closed after further 14 days of inactivity.'
+        stale-issue-label: 'stale'
+        exempt-issue-labels: 'leave open'
+        days-before-stale: 15
+        days-before-close: -1
+        remove-stale-when-updated: false
     - uses: actions/stale@v1
       with:
         repo-token: ${{ secrets.GITHUB_TOKEN }}
@@ -21,4 +31,3 @@ jobs:
         exempt-pr-label: 'leave open'
         days-before-stale: 90
         days-before-close: 14
-        operations-per-run: 7

README.rst

@@ -2,7 +2,7 @@
 
 .. badges-start
 
-|travis| |pypi| |pyversion| |license| |aur| |contributors| |downloads| |saythanks|
+|travis| |pypi| |pyversion| |license| |aur| |contributors| |downloads|
 
 .. |travis| image:: https://img.shields.io/travis/instaloader/instaloader/master.svg
    :alt: Travis-CI Build Status
@@ -23,10 +23,6 @@
    :alt: Contributor Count
    :target: https://github.com/instaloader/instaloader/graphs/contributors
 
-.. |saythanks| image:: https://img.shields.io/badge/Say%20Thanks-!-ff7efd.svg
-   :alt: Say Thanks!
-   :target: https://saythanks.io/to/aandergr
-
 .. |aur| image:: https://img.shields.io/aur/version/instaloader.svg
    :alt: Arch User Repository Package
    :target: https://aur.archlinux.org/packages/instaloader/

deploy/windows/create_exe.py

@@ -8,33 +8,18 @@ import sys
 # copy the required files into repo root
 shutil.copy('docs/favicon.ico', '.')
 shutil.copy('deploy/windows/instaloader.spec', '.')
-shutil.unpack_archive('deploy/windows/ps', '.', 'xztar')
 
 code = """
+import contextlib
 import psutil
 import subprocess
 
 def __main():
-    grandparentpid = psutil.Process(os.getppid()).ppid()
-    grandparentpidsearchstring = ' ' + str(grandparentpid) + ' '
-    if hasattr(sys, "_MEIPASS"):
-        ps = os.path.join(sys._MEIPASS, 'tasklist.exe')
-    else:
-        ps = 'tasklist'
-    popen = subprocess.Popen(ps, stdout=subprocess.PIPE, universal_newlines=True)
-    for examine in iter(popen.stdout.readline, ""):
-        if grandparentpidsearchstring in examine:
-            pname = examine
-            break
-    popen.stdout.close()
-    return_code = popen.wait()
-    if return_code:
-        raise subprocess.CalledProcessError(return_code, ps)
-    if pname[0:12] == 'explorer.exe':
-        subprocess.Popen("cmd /K \\\"{0}\\\"".format(os.path.splitext(os.path.basename(sys.argv[0]))[0]))
-    else:
-        main()
-
+    with contextlib.suppress(AttributeError, psutil.Error):
+        if psutil.Process().parent().parent().name() == "explorer.exe":
+            subprocess.Popen("powershell -NoExit -Command \\\"& {0}\\\"".format(sys.argv[0]))
+            return
+    main()
 
 if __name__ == "__main__":
     __main()

deploy/windows/instaloader.spec

@@ -5,7 +5,7 @@ block_cipher = None
 
 a = Analysis(['__main__.py'],
              binaries=[],
-             datas=[('tasklist.exe', '.'), ('framedyn.dll', '.')],
+             datas=[],
              hiddenimports=[],
              hookspath=[],
              runtime_hooks=[],

docs/_static/instaloader.css

@@ -17,15 +17,16 @@ code {
   .highlight pre {
     padding: 0.7em;
     color: #fff; }
-  .highlight .c1 {
+  .highlight .c1, .highlight .k, .highlight .kn, .highlight .ow {
     color: #008d06; }
-  .highlight .k, .highlight .kn, .highlight .ow {
-    color: #f48400;
-    font-weight: bold; }
-  .highlight .nb {
+  .highlight .nb, .highlight .ne, .highlight .nf, .highlight .vm {
     color: #f48400; }
-  .highlight .s1, .highlight .s2, .highlight .mi {
+  .highlight .s1, .highlight .s2, .highlight .sa, .highlight .se, .highlight .si, .highlight .mi {
     color: #ff7efd; }
+  .highlight .c1, .highlight .nf, .highlight .sa, .highlight .se, .highlight .si {
+    font-style: italic; }
+  .highlight .k, .highlight .kn, .highlight .ow {
+    font-weight: bold; }
 
 .doc-content {
   padding-top: 0.5em;

docs/_static/instaloader.scss

@@ -36,22 +36,25 @@ code {
     color: #fff;
   }
 
-  .c1 {
+  .c1, .k, .kn, .ow {
     color: $color_instaloader_main
   }
 
-  .k, .kn, .ow {
-    color: $color_instaloader_accent;
-    font-weight: bold
-  }
-
-  .nb {
+  .nb, .ne, .nf, .vm {
     color: $color_instaloader_accent
   }
 
-  .s1, .s2, .mi {
+  .s1, .s2, .sa, .se, .si, .mi {
     color: $color_instaloader_hyper
   }
+
+  .c1, .nf, .sa, .se, .si {
+    font-style: italic;
+  }
+
+  .k, .kn, .ow {
+    font-weight: bold
+  }
 }
 
 .doc-content {

docs/_templates/layout.html

@@ -51,9 +51,6 @@
         <li class="nav-item">
           <a target="_blank" class="nav-link" href="https://github.com/instaloader/instaloader">GitHub</a>
         </li>
-        <li class="nav-item d-sm-none d-lg-block">
-          <a target="_blank" class="nav-link" href="https://saythanks.io/to/aandergr">Say Thanks!</a>
-        </li>
       </ul>
       <form class="form-inline" action="search.html" method="get">
         <div class="input-group my-2 my-sm-0 ml-2">

docs/as-module.rst

@@ -37,7 +37,7 @@ picture, video or sidecar (set of multiple pictures/videos) posted in a user's
 profile. :class:`Instaloader` provides methods to iterate over Posts from a
 certain source::
 
-    for post in L.get_hashtag_posts('cat'):
+    for post in instaloader.Hashtag.from_name(L.context, 'cat').get_posts():
         # post is an instance of instaloader.Post
         L.download_post(post, target='#cat')
 
@@ -63,7 +63,7 @@ certain source::
 - :meth:`Instaloader.get_explore_posts`
    Media that is suggested by Instagram to explore.
 
-- :meth:`Instaloader.get_hashtag_posts`
+- :meth:`Hashtag.get_posts`
    Media associated with given hashtag.
 
 With the :class:`Profile` class, Instaloader also makes it easy to access
@@ -155,6 +155,14 @@ Profiles
 .. autoclass:: Profile
    :no-show-inheritance:
 
+Hashtags
+""""""""
+
+.. autoclass:: Hashtag
+   :no-show-inheritance:
+
+   .. versionadded:: 4.4
+
 TopSearchResults
 """"""""""""""""
 
@@ -166,7 +174,7 @@ TopSearchResults
 Loading and Saving
 """"""""""""""""""
 
-:class:`Post`, :class:`StoryItem` and :class:`Profile` can be saved and loaded
+:class:`Post`, :class:`StoryItem`, :class:`Profile` and :class:`Hashtag` can be saved and loaded
 to/from JSON files.
 
 .. autofunction:: load_structure_from_file

docs/basic-usage.rst

@@ -51,7 +51,7 @@ invoke it with
     instaloader --login=your_username profile [profile ...]
 
 When logging in, Instaloader **stores the session cookies** in a file in your
-temporary directory, which will be reused later the next time :option:`--login`
+home directory, which will be reused later the next time :option:`--login`
 is given.  So you can download private profiles **non-interactively** when you
 already have a valid session cookie file.
 
@@ -291,17 +291,18 @@ Instaloader as Cronjob
 Instaloader is suitable for running as a cronjob to periodically update your
 personal Instagram archive. The :option:`--quiet` option disables user
 interactions and logging of non-error messages. To non-interactively use
-Instaloader logged-in, create a session file, e.g. in your home directory::
+Instaloader logged-in, create a session file::
 
-   instaloader --login=your_username --sessionfile=~/.instaloader-session
+   instaloader --login=your_username
 
-Then use the same parameters in your cronjob to load the session and download
+Then use the same parameter in your cronjob to load the session and download
 the given targets::
 
-   instaloader --login=your_username --sessionfile=~/.instaloader-session --quiet <target> [...]
+   instaloader --login=your_username --quiet <target> [...]
 
-Without :option:`--sessionfile` option, Instaloader saves the session file in
-a path within your temporary directory.
+Instaloader saves the session file to
+``~/.config/instaloader/session-YOUR-USERNAME``. See
+:option:`--sessionfile` option for how to override this path.
 
 Programming Instaloader
 ^^^^^^^^^^^^^^^^^^^^^^^

docs/cli-options.rst

@@ -193,9 +193,8 @@ Instaloader to login.
 
 .. option:: --sessionfile SESSIONFILE, -f SESSIONFILE
 
-   Path for loading and storing session key file.  Defaults to a path within
-   your temporary directory, encoding your local username and your Instagram
-   profile name.
+   Path for loading and storing session key file.  Defaults to
+   ``~/.config/instaloader/session-YOUR-USERNAME``.
 
 .. option:: --password YOUR-PASSWORD, -p YOUR-PASSWORD
 
@@ -243,6 +242,11 @@ How to Download
 
    .. versionadded:: 4.2
 
+   .. warning::
+      We discourage from using the "commit mode" due to its bugs when being used
+      with other command line options (e.g. :issue:`257` with
+      :option:`--fast-update` and :issue:`483` with :option:`--tagged`).
+
 .. option:: --request-timeout N
 
    Seconds to wait before timing out a connection request.

docs/codesnippets/113_only_one_per_user.py

@@ -2,7 +2,7 @@ import instaloader
 
 L = instaloader.Instaloader()
 
-posts = L.get_hashtag_posts('urbanphotography')
+posts = instaloader.Hashtag.from_name(L.context, 'urbanphotography').get_posts()
 
 users = set()
 

docs/codesnippets/121_since_until.py

@@ -5,7 +5,7 @@ import instaloader
 
 L = instaloader.Instaloader()
 
-posts = L.get_hashtag_posts('urbanphotography')
+posts = instaloader.Hashtag.from_name(L.context, 'urbanphotography').get_posts()
 # or
 # posts = instaloader.Profile.from_username(L.context, PROFILE).get_posts()
 

docs/codesnippets/615_import_firefox_session.py

@@ -0,0 +1,53 @@
+from argparse import ArgumentParser
+from glob import glob
+from os.path import expanduser
+from platform import system
+from sqlite3 import OperationalError, connect
+
+try:
+    from instaloader import ConnectionException, Instaloader
+except ModuleNotFoundError:
+    raise SystemExit("Instaloader not found.\n  pip install [--user] instaloader")
+
+
+def get_cookiefile():
+    default_cookiefile = {
+        "Windows": "~/AppData/Roaming/Mozilla/Firefox/Profiles/*/cookies.sqlite",
+        "Darwin": "~/Library/Application Support/Firefox/Profiles/*/cookies.sqlite",
+    }.get(system(), "~/.mozilla/firefox/*/cookies.sqlite")
+    cookiefiles = glob(expanduser(default_cookiefile))
+    if not cookiefiles:
+        raise SystemExit("No Firefox cookies.sqlite file found. Use -c COOKIEFILE.")
+    return cookiefiles[0]
+
+
+def import_session(cookiefile, sessionfile):
+    print("Using cookies from {}.".format(cookiefile))
+    conn = connect(cookiefile)
+    try:
+        cookie_data = conn.execute(
+            "SELECT name, value FROM moz_cookies WHERE baseDomain='instagram.com'"
+        )
+    except OperationalError:
+        cookie_data = conn.execute(
+            "SELECT name, value FROM moz_cookies WHERE host LIKE '%instagram.com'"
+        )
+    instaloader = Instaloader(max_connection_attempts=1)
+    instaloader.context._session.cookies.update(cookie_data)
+    username = instaloader.test_login()
+    if not username:
+        raise SystemExit("Not logged in. Are you logged in successfully in Firefox?")
+    print("Imported session cookie for {}.".format(username))
+    instaloader.context.username = username
+    instaloader.save_session_to_file(sessionfile)
+
+
+if __name__ == "__main__":
+    p = ArgumentParser()
+    p.add_argument("-c", "--cookiefile")
+    p.add_argument("-f", "--sessionfile")
+    args = p.parse_args()
+    try:
+        import_session(args.cookiefile or get_cookiefile(), args.sessionfile)
+    except (ConnectionException, OperationalError) as e:
+        raise SystemExit("Cookie import failed: {}".format(e))

docs/codesnippets/92_import_firefox_session.py

@@ -1,23 +0,0 @@
-from glob import glob
-from os.path import expanduser
-from sqlite3 import connect
-
-from instaloader import ConnectionException, Instaloader
-
-# FIREFOXCOOKIEFILE = "/home/alex/.mozilla/firefox/l96w6b90.default/cookies.sqlite"
-FIREFOXCOOKIEFILE = glob(expanduser("~/.mozilla/firefox/*.default/cookies.sqlite"))[0]
-
-instaloader = Instaloader(max_connection_attempts=1)
-instaloader.context._session.cookies.update(connect(FIREFOXCOOKIEFILE)
-                                            .execute("SELECT name, value FROM moz_cookies "
-                                                     "WHERE baseDomain='instagram.com'"))
-
-try:
-    username = instaloader.test_login()
-    if not username:
-        raise ConnectionException()
-except ConnectionException:
-    raise SystemExit("Cookie import failed. Are you logged in successfully in Firefox?")
-
-instaloader.context.username = username
-instaloader.save_session_to_file()

docs/conf.py

@@ -49,7 +49,22 @@ autodoc_member_order = 'bysource'
 intersphinx_mapping = {'python': ('https://docs.python.org/3', None),
                        'requests': ('https://requests.kennethreitz.org/en/master/', None)}
 
-extlinks = {'issue': ('https://github.com/instaloader/instaloader/issues/%s', 'issue #')}
+current_release = subprocess.check_output(["git", "describe", "--abbrev=0"]).decode("ascii")[1:-1]
+current_release_date = subprocess.check_output(
+    ["git", "log", "-1", "--tags", "--format=%ad", "--date=format:%e %b %Y"]).decode("ascii")[:-1]
+
+html_context = {'current_release': current_release, 'current_release_date': current_release_date}
+
+extlinks = {
+    'issue': (
+        'https://github.com/instaloader/instaloader/issues/%s',
+        'Issue #'
+    ),
+    'example': (
+        'https://raw.githubusercontent.com/instaloader/instaloader/v' + current_release + '/docs/codesnippets/%s',
+        'Example '
+    ),
+}
 
 # Add any paths that contain templates here, relative to this directory.
 templates_path = ['_templates']
@@ -69,7 +84,7 @@ master_doc = 'index'
 
 # General information about the project.
 project = 'Instaloader'
-copyright = '2017, Alexander Graf and Andre Koch-Kramer'
+copyright = '2017-2020, Alexander Graf and Andre Koch-Kramer'
 author = 'Alexander Graf and Andre Koch-Kramer'
 
 # The version info for the project you're documenting, acts as replacement for
@@ -353,12 +368,6 @@ texinfo_documents = [
 # texinfo_no_detailmenu = False
 
 
-current_release = subprocess.check_output(["git", "describe", "--abbrev=0"]).decode("ascii")[1:-1]
-current_release_date = subprocess.check_output(["git", "log", "-1", "--tags", "--format=%ad", "--date=format:%e %b %Y"]).decode("ascii")[:-1]
-
-html_context = {'current_release': current_release, 'current_release_date': current_release_date}
-
-
 def setup(app):
     typing.TYPE_CHECKING = True
     app.add_stylesheet("instaloader.css")

docs/troubleshooting.rst

@@ -32,7 +32,7 @@ rate limit has almost been reached, according to Instaloader's own rate
 accounting mechanism. We regularly adjust this mechanism to match Instagram's
 current rate limiting.
 
-Login error
+Login Error
 -----------
 
 Instaloader's login *should* work fine, both with and without
@@ -40,21 +40,31 @@ Two-Factor-Authentication. It also supports handling the *checkpoint challenge*,
 issued when Instagram suspects authentication activity on your account, by
 pointing the user to an URL to be opened in a browser.
 
-Nevertheless, in :issue:`92` users report problems with logging in. To still use
-Instaloader's logged-in functionality, you may use the following script to
-workaround login problems by importing the session cookies from Firefox and
-bypassing Instaloader's login.
+Nevertheless, in :issue:`92` and :issue:`615` users reported problems with
+logging in. We recommend to always keep the sessionfile which Instaloader
+creates when using :option:`--login`. If a sessionfile is present,
+:option:`--login` does not make make use of the failure-prone login procedure.
+Also, sessionfiles usually do not expire and can be copied between different
+computers without any problems.
 
-.. literalinclude:: codesnippets/92_import_firefox_session.py
+If you do not have a sessionfile present, you may use the following script
+(:example:`615_import_firefox_session.py`) to workaround login problems by
+importing the session cookies from Firefox and bypassing Instaloader's login and
+so still use Instaloader's logged-in functionality.
 
-To use this,
+.. literalinclude:: codesnippets/615_import_firefox_session.py
 
-#. login to Instagram in Firefox, 
+To use this script,
 
-#. execute the snippet,
+#. Download the script: :example:`615_import_firefox_session.py`,
 
-#. then, ``instaloader -l USERNAME`` should work fine.
+#. Login to Instagram in Firefox,
 
-If you do not use your default firefox profile, or your operating system has the
-paths differently set up, you may have to alter the ``FIREFOXCOOKIEFILE``
-variable first.
+#. Execute the snippet, e.g. with ``python 615_import_firefox_session.py``,
+
+#. Then, ``instaloader -l USERNAME`` should work fine.
+
+This script also supports specifying a cookiefile path, which may be useful if
+you use multiple Firefox profiles or if your operating system has the directory
+structure differently set up. Also, you can specify an alternative sessionfile
+path.

instaloader/__init__.py

@@ -1,7 +1,7 @@
 """Download pictures (or videos) along with their captions and other metadata from Instagram."""
 
 
-__version__ = '4.3b1'
+__version__ = '4.4.4rc3'
 
 
 try:
@@ -15,5 +15,5 @@ else:
 from .exceptions import *
 from .instaloader import Instaloader
 from .instaloadercontext import InstaloaderContext
-from .structures import (Highlight, Post, PostSidecarNode, PostComment, PostCommentAnswer, PostLocation, Profile, Story,
-                         StoryItem, TopSearchResults, load_structure_from_file, save_structure_to_file)
+from .structures import (Hashtag, Highlight, Post, PostSidecarNode, PostComment, PostCommentAnswer, PostLocation,
+                         Profile, Story, StoryItem, TopSearchResults, load_structure_from_file, save_structure_to_file)

instaloader/__main__.py

@@ -3,6 +3,7 @@
 import ast
 import datetime
 import os
+import re
 import sys
 from argparse import ArgumentParser, SUPPRESS
 from typing import List, Optional
@@ -136,18 +137,19 @@ def _main(instaloader: Instaloader, targetlist: List[str],
             # strip '/' characters to be more shell-autocompletion-friendly
             target = target.rstrip('/')
             with instaloader.context.error_catcher(target):
-                if target[0] == '@':
+                if re.match(r"^@[A-Za-z0-9._]+$", target):
                     instaloader.context.log("Retrieving followees of %s..." % target[1:])
                     profile = Profile.from_username(instaloader.context, target[1:])
                     for followee in profile.get_followees():
                         instaloader.save_profile_id(followee)
                         profiles.add(followee)
-                elif target[0] == '#':
+                elif re.match(r"^#\w+$", target):
                     instaloader.download_hashtag(hashtag=target[1:], max_count=max_count, fast_update=fast_update,
-                                                 post_filter=post_filter)
-                elif target[0] == '-':
+                                                 post_filter=post_filter,
+                                                 profile_pic=download_profile_pic, posts=download_posts)
+                elif re.match(r"^-[A-Za-z0-9-_]+$", target):
                     instaloader.download_post(Post.from_shortcode(instaloader.context, target[1:]), target)
-                elif target[0] == "%":
+                elif re.match(r"^%[0-9]+$", target):
                     instaloader.download_location(location=target[1:], max_count=max_count, fast_update=fast_update,
                                                   post_filter=post_filter)
                 elif target == ":feed":
@@ -158,7 +160,7 @@ def _main(instaloader: Instaloader, targetlist: List[str],
                 elif target == ":saved":
                     instaloader.download_saved_posts(fast_update=fast_update, max_count=max_count,
                                                      post_filter=post_filter)
-                else:
+                elif re.match(r"^[A-Za-z0-9._]+$", target):
                     try:
                         profile = instaloader.check_profile_id(target)
                         if instaloader.context.is_logged_in and profile.has_blocked_viewer:
@@ -184,11 +186,18 @@ def _main(instaloader: Instaloader, targetlist: List[str],
                                                               .format(target, err))
                         else:
                             raise
+                else:
+                    target_type = {
+                        '#': 'hashtag',
+                        '%': 'location',
+                        '-': 'shortcode',
+                    }.get(target[0], 'username')
+                    raise ProfileNotExistsException('Invalid {} {}'.format(target_type, target))
         if len(profiles) > 1:
             instaloader.context.log("Downloading {} profiles: {}".format(len(profiles),
                                                                          ' '.join([p.username for p in profiles])))
-        if profiles and download_profile_pic and not instaloader.context.is_logged_in:
-            instaloader.context.error("Warning: Use --login to download HD version of profile pictures.")
+        if profiles and (download_profile_pic or download_posts) and not instaloader.context.is_logged_in:
+            instaloader.context.error("Warning: Use --login to download higher-quality versions of pictures.")
         instaloader.download_profiles(profiles,
                                       download_profile_pic, download_posts, download_tagged, download_igtv,
                                       download_highlights, download_stories,

instaloader/instaloader.py

@@ -21,12 +21,26 @@ import urllib3  # type: ignore
 
 from .exceptions import *
 from .instaloadercontext import InstaloaderContext
-from .structures import (Highlight, JsonExportable, Post, PostLocation, Profile, Story, StoryItem,
+from .structures import (Hashtag, Highlight, JsonExportable, Post, PostLocation, Profile, Story, StoryItem,
                          save_structure_to_file, load_structure_from_file)
 
 
 def get_default_session_filename(username: str) -> str:
     """Returns default session filename for given username."""
+    sessionfilename = "session-{}".format(username)
+    if platform.system() == "Windows":
+        # on Windows, use %LOCALAPPDATA%\Instaloader\session-USERNAME
+        localappdata = os.getenv("LOCALAPPDATA")
+        if localappdata is not None:
+            return os.path.join(localappdata, "Instaloader", sessionfilename)
+        # legacy fallback - store in temp dir if %LOCALAPPDATA% is not set
+        return os.path.join(tempfile.gettempdir(), ".instaloader-" + getpass.getuser(), sessionfilename)
+    # on Unix, use ~/.config/instaloader/session-USERNAME
+    return os.path.join(os.getenv("XDG_CONFIG_HOME", os.path.expanduser("~/.config")), "instaloader", sessionfilename)
+
+
+def get_legacy_session_filename(username: str) -> str:
+    """Returns legacy (until v4.4.3) default session filename for given username."""
     dirname = tempfile.gettempdir() + "/" + ".instaloader-" + getpass.getuser()
     filename = dirname + "/" + "session-" + username
     return filename.lower()
@@ -103,22 +117,20 @@ class _ArbitraryItemFormatter(string.Formatter):
 
 
 class _PostPathFormatter(_ArbitraryItemFormatter):
-    def vformat(self, format_string, args, kwargs):
-        """Override :meth:`string.Formatter.vformat` for character substitution in paths for Windows, see issue #84."""
-        ret = super().vformat(format_string, args, kwargs)
-        if platform.system() == 'Windows':
-            ret = ret.replace(':', '\ua789').replace('<', '\ufe64').replace('>', '\ufe65').replace('\"', '\uff02')
-            ret = ret.replace('\\', '\ufe68').replace('|', '\uff5c').replace('?', '\ufe16').replace('*', '\uff0a')
-        return ret
-
     def get_value(self, key, args, kwargs):
-        """Replaces '/' with similar looking Division Slash and on windows newline with space"""
         ret = super().get_value(key, args, kwargs)
         if not isinstance(ret, str):
             return ret
+        return self.sanitize_path(ret)
+
+    @staticmethod
+    def sanitize_path(ret: str) -> str:
+        """Replaces '/' with similar looking Division Slash and some other illegal filename characters on Windows."""
         ret = ret.replace('/', '\u2215')
         if platform.system() == 'Windows':
-            ret = ret.replace('\n', ' ')
+            ret = ret.replace(':', '\uff1a').replace('<', '\ufe64').replace('>', '\ufe65').replace('\"', '\uff02')
+            ret = ret.replace('\\', '\ufe68').replace('|', '\uff5c').replace('?', '\ufe16').replace('*', '\uff0a')
+            ret = ret.replace('\n', ' ').replace('\r', ' ')
         return ret
 
 
@@ -358,7 +370,7 @@ class Instaloader:
         self.context.log('geo', end=' ', flush=True)
 
     @_retry_on_connection_error
-    def download_title_pic(self, url: str, target: Union[str, Path], name_suffix: str, owner_profile: Profile,
+    def download_title_pic(self, url: str, target: Union[str, Path], name_suffix: str, owner_profile: Optional[Profile],
                            _attempt: int = 1) -> None:
         """Downloads and saves a picture that does not have an association with a Post or StoryItem, such as a
         Profile picture or a Highlight cover picture. Modification time is taken from the HTTP response headers.
@@ -380,7 +392,8 @@ class Instaloader:
         pic_extension = 'jpg'
         if ((format_string_contains_key(self.dirname_pattern, 'profile') or
              format_string_contains_key(self.dirname_pattern, 'target'))):
-            filename = '{0}/{1}_{2}.{3}'.format(self.dirname_pattern.format(profile=owner_profile.username.lower(),
+            profile_str = owner_profile.username.lower() if owner_profile is not None else target
+            filename = '{0}/{1}_{2}.{3}'.format(self.dirname_pattern.format(profile=profile_str,
                                                                             target=target),
                                                 pic_identifier, name_suffix, pic_extension)
         else:
@@ -408,6 +421,12 @@ class Instaloader:
         .. versionadded:: 4.3"""
         self.download_title_pic(highlight.cover_url, target, 'cover', highlight.owner_profile)
 
+    def download_hashtag_profilepic(self, hashtag: Hashtag) -> None:
+        """Downloads and saves the profile picture of a Hashtag.
+
+        .. versionadded:: 4.4"""
+        self.download_title_pic(hashtag.profile_pic_url, '#' + hashtag.name, 'profile_pic', None)
+
     @_requires_login
     def save_session_to_file(self, filename: Optional[str] = None) -> None:
         """Saves internally stored :class:`requests.Session` object.
@@ -436,6 +455,8 @@ class Instaloader:
         """
         if filename is None:
             filename = get_default_session_filename(username)
+            if not os.path.exists(filename):
+                filename = get_legacy_session_filename(username)
         with open(filename, 'rb') as sessionfile:
             self.context.load_session_from_file(username, sessionfile)
             self.context.log("Loaded session from %s." % filename)
@@ -674,7 +695,8 @@ class Instaloader:
             name = user_highlight.owner_username
             highlight_target = (filename_target
                                 if filename_target
-                                else Path(name) / Path(user_highlight.title))  # type: Union[str, Path]
+                                else (Path(_PostPathFormatter.sanitize_path(name)) /
+                                      _PostPathFormatter.sanitize_path(user_highlight.title)))  # type: Union[str, Path]
             self.context.log("Retrieving highlights \"{}\" from profile {}".format(user_highlight.title, name))
             self.download_highlight_cover(user_highlight, highlight_target)
             totalcount = user_highlight.itemcount
@@ -691,6 +713,66 @@ class Instaloader:
                     if fast_update and not downloaded:
                         break
 
+    def posts_download_loop(self,
+                            posts: Iterator[Post],
+                            target: Union[str, Path],
+                            fast_update: bool = False,
+                            post_filter: Optional[Callable[[Post], bool]] = None,
+                            max_count: Optional[int] = None,
+                            total_count: Optional[int] = None) -> None:
+        """
+        Download the Posts returned by given Post Iterator.
+
+        ..versionadded:: 4.4
+
+        :param posts: Post Iterator to loop through.
+        :param target: Target name
+        :param fast_update: :option:`--fast-update`
+        :param post_filter: :option:`--post-filter`
+        :param max_count: Maximum count of Posts to download (:option:`--count`)
+        :param total_count: Total number of posts returned by given iterator
+        """
+        for number, post in enumerate(posts):
+            if max_count is not None and number >= max_count:
+                break
+            if total_count is not None:
+                self.context.log("[{0:{w}d}/{1:{w}d}] ".format(number + 1, total_count,
+                                                               w=len(str(total_count))),
+                                 end="", flush=True)
+            else:
+                if max_count is not None:
+                    self.context.log("[{0:{w}d}/{1:{w}d}] ".format(number + 1, max_count,
+                                                                   w=len(str(max_count))),
+                                     end="", flush=True)
+                else:
+                    self.context.log("[{:3d}] ".format(number + 1), end="", flush=True)
+            if post_filter is not None:
+                try:
+                    if not post_filter(post):
+                        self.context.log("{} skipped".format(post))
+                        continue
+                except (InstaloaderException, KeyError, TypeError) as err:
+                    self.context.error("{} skipped. Filter evaluation failed: {}".format(post, err))
+                    continue
+            with self.context.error_catcher("Download {} of {}".format(post, target)):
+                # The PostChangedException gets raised if the Post's id/shortcode changed while obtaining
+                # additional metadata. This is most likely the case if a HTTP redirect takes place while
+                # resolving the shortcode URL.
+                # The `post_changed` variable keeps the fast-update functionality alive: A Post which is
+                # obained after a redirect has probably already been downloaded as a previous Post of the
+                # same Profile.
+                # Observed in issue #225: https://github.com/instaloader/instaloader/issues/225
+                post_changed = False
+                while True:
+                    try:
+                        downloaded = self.download_post(post, target=target)
+                        break
+                    except PostChangedException:
+                        post_changed = True
+                        continue
+                if fast_update and not downloaded and not post_changed:
+                    break
+
     @_requires_login
     def get_feed_posts(self) -> Iterator[Post]:
         """Get Posts of the user's feed.
@@ -703,8 +785,10 @@ class Instaloader:
 
         while True:
             feed = data["user"]["edge_web_feed_timeline"]
-            yield from (Post(self.context, edge["node"]) for edge in feed["edges"]
-                        if not edge["node"]["__typename"] == "GraphSuggestedUserFeedUnit")
+            for edge in feed["edges"]:
+                node = edge["node"]
+                if node.get("__typename") in Post.supported_graphql_types() and node.get("shortcode") is not None:
+                    yield Post(self.context, node)
             if not feed["page_info"]["has_next_page"]:
                 break
             data = self.context.graphql_query("d6f4427fbe92d846298cf93df0b937d3",
@@ -733,20 +817,7 @@ class Instaloader:
         :raises LoginRequiredException: If called without being logged in.
         """
         self.context.log("Retrieving pictures from your feed...")
-        count = 1
-        for post in self.get_feed_posts():
-            if max_count is not None and count > max_count:
-                break
-            name = post.owner_username
-            if post_filter is not None and not post_filter(post):
-                self.context.log("<pic by %s skipped>" % name, flush=True)
-                continue
-            self.context.log("[%3i] %s " % (count, name), end="", flush=True)
-            count += 1
-            with self.context.error_catcher('Download feed'):
-                downloaded = self.download_post(post, target=':feed')
-                if fast_update and not downloaded:
-                    break
+        self.posts_download_loop(self.get_feed_posts(), ":feed", fast_update, post_filter, max_count=max_count)
 
     @_requires_login
     def download_saved_posts(self, max_count: int = None, fast_update: bool = False,
@@ -759,20 +830,9 @@ class Instaloader:
         :raises LoginRequiredException: If called without being logged in.
         """
         self.context.log("Retrieving saved posts...")
-        count = 1
-        assert self.context.username is not None
-        for post in Profile.from_username(self.context, self.context.username).get_saved_posts():
-            if max_count is not None and count > max_count:
-                break
-            if post_filter is not None and not post_filter(post):
-                self.context.log("<{} skipped>".format(post), flush=True)
-                continue
-            self.context.log("[{:>3}] ".format(count), end=str(), flush=True)
-            count += 1
-            with self.context.error_catcher('Download saved posts'):
-                downloaded = self.download_post(post, target=':saved')
-                if fast_update and not downloaded:
-                    break
+        assert self.context.username is not None  # safe due to @_requires_login; required by typechecker
+        self.posts_download_loop(Profile.from_username(self.context, self.context.username).get_saved_posts(), ":saved",
+                                 fast_update, post_filter, max_count=max_count)
 
     @_requires_login
     def get_location_posts(self, location: str) -> Iterator[Post]:
@@ -823,19 +883,8 @@ class Instaloader:
            Require being logged in (as required by Instagram)
         """
         self.context.log("Retrieving pictures for location {}...".format(location))
-        count = 1
-        for post in self.get_location_posts(location):
-            if max_count is not None and count > max_count:
-                break
-            self.context.log('[{0:3d}] %{1} '.format(count, location), end='', flush=True)
-            if post_filter is not None and not post_filter(post):
-                self.context.log('<skipped>')
-                continue
-            count += 1
-            with self.context.error_catcher('Download location {}'.format(location)):
-                downloaded = self.download_post(post, target='%' + location)
-                if fast_update and not downloaded:
-                    break
+        self.posts_download_loop(self.get_location_posts(location), "%" + location, fast_update, post_filter,
+                                 max_count=max_count)
 
     @_requires_login
     def get_explore_posts(self) -> Iterator[Post]:
@@ -852,24 +901,18 @@ class Instaloader:
                                                                data.get('rhx_gis')))
 
     def get_hashtag_posts(self, hashtag: str) -> Iterator[Post]:
-        """Get Posts associated with a #hashtag."""
-        has_next_page = True
-        end_cursor = None
-        while has_next_page:
-            if end_cursor:
-                params = {'__a': 1, 'max_id': end_cursor}
-            else:
-                params = {'__a': 1}
-            hashtag_data = self.context.get_json('explore/tags/{0}/'.format(hashtag),
-                                                 params)['graphql']['hashtag']['edge_hashtag_to_media']
-            yield from (Post(self.context, edge['node']) for edge in hashtag_data['edges'])
-            has_next_page = hashtag_data['page_info']['has_next_page']
-            end_cursor = hashtag_data['page_info']['end_cursor']
+        """Get Posts associated with a #hashtag.
 
-    def download_hashtag(self, hashtag: str,
+        .. deprecated:: 4.4
+           Use :meth:`Hashtag.get_posts`."""
+        return Hashtag.from_name(self.context, hashtag).get_posts()
+
+    def download_hashtag(self, hashtag: Union[Hashtag, str],
                          max_count: Optional[int] = None,
                          post_filter: Optional[Callable[[Post], bool]] = None,
-                         fast_update: bool = False) -> None:
+                         fast_update: bool = False,
+                         profile_pic: bool = True,
+                         posts: bool = True) -> None:
         """Download pictures of one hashtag.
 
         To download the last 30 pictures with hashtag #cat, do::
@@ -877,26 +920,34 @@ class Instaloader:
             loader = Instaloader()
             loader.download_hashtag('cat', max_count=30)
 
-        :param hashtag: Hashtag to download, without leading '#'
+        :param hashtag: Hashtag to download, as instance of :class:`Hashtag`, or string without leading '#'
         :param max_count: Maximum count of pictures to download
         :param post_filter: function(post), which returns True if given picture should be downloaded
         :param fast_update: If true, abort when first already-downloaded picture is encountered
+        :param profile_pic: not :option:`--no-profile-pic`.
+        :param posts: not :option:`--no-posts`.
+
+        .. versionchanged:: 4.4
+           Add parameters `profile_pic` and `posts`.
         """
-        hashtag = hashtag.lower()
-        self.context.log("Retrieving pictures with hashtag {}...".format(hashtag))
-        count = 1
-        for post in self.get_hashtag_posts(hashtag):
-            if max_count is not None and count > max_count:
-                break
-            self.context.log('[{0:3d}] #{1} '.format(count, hashtag), end='', flush=True)
-            if post_filter is not None and not post_filter(post):
-                self.context.log('<skipped>')
-                continue
-            count += 1
-            with self.context.error_catcher('Download hashtag #{}'.format(hashtag)):
-                downloaded = self.download_post(post, target='#' + hashtag)
-                if fast_update and not downloaded:
-                    break
+        if isinstance(hashtag, str):
+            with self.context.error_catcher("Get hashtag #{}".format(hashtag)):
+                hashtag = Hashtag.from_name(self.context, hashtag)
+        if not isinstance(hashtag, Hashtag):
+            return
+        target = "#" + hashtag.name
+        if profile_pic:
+            with self.context.error_catcher("Download profile picture of {}".format(target)):
+                self.download_hashtag_profilepic(hashtag)
+        if posts:
+            self.context.log("Retrieving pictures with hashtag #{}...".format(hashtag.name))
+            self.posts_download_loop(hashtag.get_all_posts(), target, fast_update, post_filter,
+                                     max_count=max_count)
+        if self.save_metadata:
+            json_filename = '{0}/{1}'.format(self.dirname_pattern.format(profile=target,
+                                                                         target=target),
+                                             target)
+            self.save_metadata_json(json_filename, hashtag)
 
     def download_tagged(self, profile: Profile, fast_update: bool = False,
                         target: Optional[str] = None,
@@ -905,17 +956,11 @@ class Instaloader:
 
         .. versionadded:: 4.1"""
         self.context.log("Retrieving tagged posts for profile {}.".format(profile.username))
-        count = 1
-        for post in profile.get_tagged_posts():
-            self.context.log("[%3i/???] " % (count), end="", flush=True)
-            count += 1
-            if post_filter is not None and not post_filter(post):
-                self.context.log('<{} skipped>'.format(post))
-                continue
-            with self.context.error_catcher('Download tagged {}'.format(profile.username)):
-                downloaded = self.download_post(post, target if target else Path(profile.username) / Path(':tagged'))
-                if fast_update and not downloaded:
-                    break
+        self.posts_download_loop(profile.get_tagged_posts(),
+                                 target if target
+                                 else (Path(_PostPathFormatter.sanitize_path(profile.username)) /
+                                       _PostPathFormatter.sanitize_path(':tagged')),
+                                 fast_update, post_filter)
 
     def download_igtv(self, profile: Profile, fast_update: bool = False,
                       post_filter: Optional[Callable[[Post], bool]] = None) -> None:
@@ -923,16 +968,8 @@ class Instaloader:
 
         .. versionadded:: 4.3"""
         self.context.log("Retrieving IGTV videos for profile {}.".format(profile.username))
-        for number, post in enumerate(profile.get_igtv_posts()):
-            self.context.log("[{0:{w}d}/{1:{w}d}] ".format(number, profile.igtvcount, w=len(str(profile.igtvcount))),
-                             end="", flush=True)
-            if post_filter is not None and not post_filter(post):
-                self.context.log('<{} skipped>'.format(post))
-                continue
-            with self.context.error_catcher('Download IGTV {}'.format(post.shortcode)):
-                downloaded = self.download_post(post, target=profile.username)
-                if fast_update and not downloaded:
-                    break
+        self.posts_download_loop(profile.get_igtv_posts(), profile.username, fast_update, post_filter,
+                                 total_count=profile.igtvcount)
 
     def _get_id_filename(self, profile_name: str) -> str:
         if ((format_string_contains_key(self.dirname_pattern, 'profile') or
@@ -1084,32 +1121,8 @@ class Instaloader:
                 # Iterate over pictures and download them
                 if posts:
                     self.context.log("Retrieving posts from profile {}.".format(profile_name))
-                    totalcount = profile.mediacount
-                    count = 1
-                    for post in profile.get_posts():
-                        self.context.log("[%3i/%3i] " % (count, totalcount), end="", flush=True)
-                        count += 1
-                        if post_filter is not None and not post_filter(post):
-                            self.context.log('<skipped>')
-                            continue
-                        with self.context.error_catcher("Download {} of {}".format(post, profile_name)):
-                            # The PostChangedException gets raised if the Post's id/shortcode changed while obtaining
-                            # additional metadata. This is most likely the case if a HTTP redirect takes place while
-                            # resolving the shortcode URL.
-                            # The `post_changed` variable keeps the fast-update functionality alive: A Post which is
-                            # obained after a redirect has probably already been downloaded as a previous Post of the
-                            # same Profile.
-                            # Observed in issue #225: https://github.com/instaloader/instaloader/issues/225
-                            post_changed = False
-                            while True:
-                                try:
-                                    downloaded = self.download_post(post, target=profile_name)
-                                    break
-                                except PostChangedException:
-                                    post_changed = True
-                                    continue
-                            if fast_update and not downloaded and not post_changed:
-                                break
+                    self.posts_download_loop(profile.get_posts(), profile_name, fast_update, post_filter,
+                                             total_count=profile.mediacount)
 
         if stories and profiles:
             with self.context.error_catcher("Download stories"):
@@ -1188,18 +1201,8 @@ class Instaloader:
 
         # Iterate over pictures and download them
         self.context.log("Retrieving posts from profile {}.".format(profile_name))
-        totalcount = profile.mediacount
-        count = 1
-        for post in profile.get_posts():
-            self.context.log("[%3i/%3i] " % (count, totalcount), end="", flush=True)
-            count += 1
-            if post_filter is not None and not post_filter(post):
-                self.context.log('<skipped>')
-                continue
-            with self.context.error_catcher('Download profile {}'.format(profile_name)):
-                downloaded = self.download_post(post, target=profile_name)
-                if fast_update and not downloaded:
-                    break
+        self.posts_download_loop(profile.get_posts(), profile_name, fast_update, post_filter,
+                                 total_count=profile.mediacount)
 
     def check_if_committed(self, filename: str) -> bool:
         """Checks to see if the current post has been committed.

instaloader/instaloadercontext.py

@@ -113,7 +113,7 @@ class InstaloaderContext:
     def close(self):
         """Print error log and close session"""
         if self.error_log and not self.quiet:
-            print("\nErrors occured:", file=sys.stderr)
+            print("\nErrors or warnings occurred:", file=sys.stderr)
             for err in self.error_log:
                 print(err, file=sys.stderr)
         self._session.close()
@@ -215,8 +215,11 @@ class InstaloaderContext:
         session.headers.update({'X-CSRFToken': csrf_token})
         # Not using self.get_json() here, because we need to access csrftoken cookie
         self.do_sleep()
+        # Workaround credits to pgrimaud.
+        # See: https://github.com/pgrimaud/instagram-user-feed/commit/96ad4cf54d1ad331b337f325c73e664999a6d066
+        enc_password = '#PWD_INSTAGRAM_BROWSER:0:{}:{}'.format(int(datetime.now().timestamp()), passwd)
         login = session.post('https://www.instagram.com/accounts/login/ajax/',
-                             data={'password': passwd, 'username': user}, allow_redirects=True)
+                             data={'enc_password': enc_password, 'username': user}, allow_redirects=True)
         try:
             resp_json = login.json()
         except json.decoder.JSONDecodeError:
@@ -239,6 +242,12 @@ class InstaloaderContext:
                                                                                                resp_json['message']))
             else:
                 raise ConnectionException("Login error: \"{}\" status.".format(resp_json['status']))
+        if 'authenticated' not in resp_json:
+            # Issue #472
+            if 'message' in resp_json:
+                raise ConnectionException("Login error: Unexpected response, \"{}\".".format(resp_json['message']))
+            else:
+                raise ConnectionException("Login error: Unexpected response, this might indicate a blocked IP.")
         if not resp_json['authenticated']:
             if resp_json['user']:
                 # '{"authenticated": false, "user": true, "status": "ok"}'
@@ -343,7 +352,7 @@ class InstaloaderContext:
             else:
                 self._graphql_query_timestamps[query_hash].append(time.monotonic())
         else:
-            text_for_429 = ("HTTP error code 429 was returned because too many queries occured in the last time. "
+            text_for_429 = ("HTTP error code 429 was returned because too many queries occurred in the last time. "
                             "Please do not use Instagram in your browser or run multiple instances of Instaloader "
                             "in parallel.")
             print(textwrap.fill(text_for_429), file=sys.stderr)
@@ -527,8 +536,8 @@ class InstaloaderContext:
 
         .. versionadded:: 4.2.1"""
         with copy_session(self._session, self.request_timeout) as tempsession:
-            tempsession.headers['User-Agent'] = 'Instagram 10.3.2 (iPhone7,2; iPhone OS 9_3_3; en_US; en-US; ' \
-                                                'scale=2.00; 750x1334) AppleWebKit/420+'
+            tempsession.headers['User-Agent'] = 'Instagram 123.1.0.26.115 (iPhone12,1; iOS 13_3; en_US; en-US; ' \
+                                                'scale=2.00; 1656x3584; 190542906)'
             for header in ['Host', 'Origin', 'X-Instagram-AJAX', 'X-Requested-With']:
                 tempsession.headers.pop(header, None)
             return self.get_json(path, params, 'i.instagram.com', tempsession)

instaloader/structures.py

@@ -71,6 +71,10 @@ class Post:
         self._full_metadata_dict = None  # type: Optional[Dict[str, Any]]
         self._rhx_gis_str = None         # type: Optional[str]
         self._location = None            # type: Optional[PostLocation]
+        self._iphone_struct_ = None
+        if 'iphone_struct' in node:
+            # if loaded from JSON with load_structure_from_file()
+            self._iphone_struct_ = node['iphone_struct']
 
     @classmethod
     def from_shortcode(cls, context: InstaloaderContext, shortcode: str):
@@ -98,6 +102,11 @@ class Post:
             raise InvalidArgumentException("Wrong mediaid {0}, unable to convert to shortcode".format(str(mediaid)))
         return b64encode(mediaid.to_bytes(9, 'big'), b'-_').decode().replace('A', ' ').lstrip().replace(' ', 'A')
 
+    @staticmethod
+    def supported_graphql_types() -> List[str]:
+        """The values of __typename fields that the :class:`Post` class can handle."""
+        return ["GraphImage", "GraphVideo", "GraphSidecar"]
+
     def _asdict(self):
         node = self._node
         if self._full_metadata_dict:
@@ -106,6 +115,8 @@ class Post:
             node['owner'] = self.owner_profile._asdict()
         if self._location:
             node['location'] = self._location._asdict()
+        if self._iphone_struct_:
+            node['iphone_struct'] = self._iphone_struct_
         return node
 
     @property
@@ -134,6 +145,12 @@ class Post:
             pic_json = self._context.get_json("p/{0}/".format(self.shortcode), params={})
             self._full_metadata_dict = pic_json['entry_data']['PostPage'][0]['graphql']['shortcode_media']
             self._rhx_gis_str = pic_json.get('rhx_gis')
+            if self._full_metadata_dict is None:
+                # issue #449
+                self._context.error("Fetching Post metadata failed (issue #449). "
+                                    "The following data has been returned:\n"
+                                    + json.dumps(pic_json['entry_data'], indent=2))
+                raise BadResponseException("Fetching Post metadata failed.")
             if self.shortcode != self._full_metadata_dict['shortcode']:
                 self._node.update(self._full_metadata_dict)
                 raise PostChangedException
@@ -149,6 +166,15 @@ class Post:
         self._obtain_metadata()
         return self._rhx_gis_str
 
+    @property
+    def _iphone_struct(self) -> Dict[str, Any]:
+        if not self._context.is_logged_in:
+            raise LoginRequiredException("--login required to access iPhone media info endpoint.")
+        if not self._iphone_struct_:
+            data = self._context.get_iphone_json(path='api/v1/media/{}/info/'.format(self.mediaid), params={})
+            self._iphone_struct_ = data['items'][0]
+        return self._iphone_struct_
+
     def _field(self, *keys) -> Any:
         """Lookups given fields in _node, and if not found in _full_metadata. Raises KeyError if not found anywhere."""
         try:
@@ -214,6 +240,13 @@ class Post:
     @property
     def url(self) -> str:
         """URL of the picture / video thumbnail of the post"""
+        if self.typename == "GraphImage" and self._context.is_logged_in:
+            try:
+                orig_url = self._iphone_struct['image_versions2']['candidates'][0]['url']
+                url = re.sub(r'&se=\d+(&?)', r'\1', orig_url)
+                return url
+            except (InstaloaderException, KeyError, IndexError) as err:
+                self._context.error('{} Unable to fetch high quality image version of {}.'.format(err, self))
         return self._node["display_url"] if "display_url" in self._node else self._node["display_src"]
 
     @property
@@ -224,10 +257,22 @@ class Post:
     def get_sidecar_nodes(self) -> Iterator[PostSidecarNode]:
         """Sidecar nodes of a Post with typename==GraphSidecar."""
         if self.typename == 'GraphSidecar':
-            for edge in self._field('edge_sidecar_to_children', 'edges'):
+            edges = self._field('edge_sidecar_to_children', 'edges')
+            if any(edge['node']['is_video'] for edge in edges):
+                # video_url is only present in full metadata, issue #558.
+                edges = self._full_metadata['edge_sidecar_to_children']['edges']
+            for idx, edge in enumerate(edges):
                 node = edge['node']
                 is_video = node['is_video']
-                yield PostSidecarNode(is_video=is_video, display_url=node['display_url'],
+                display_url = node['display_url']
+                if not is_video and self._context.is_logged_in:
+                    try:
+                        carousel_media = self._iphone_struct['carousel_media']
+                        orig_url = carousel_media[idx]['image_versions2']['candidates'][0]['url']
+                        display_url = re.sub(r'&se=\d+(&?)', r'\1', orig_url)
+                    except (InstaloaderException, KeyError, IndexError) as err:
+                        self._context.error('{} Unable to fetch high quality image version of {}.'.format(err, self))
+                yield PostSidecarNode(is_video=is_video, display_url=display_url,
                                       video_url=node['video_url'] if is_video else None)
 
     @property
@@ -413,6 +458,30 @@ class Post:
                                                     lambda d: d['data']['shortcode_media']['edge_liked_by'],
                                                     self._rhx_gis))
 
+    @property
+    def is_sponsored(self) -> bool:
+        """
+        Whether Post is a sponsored post, equivalent to non-empty :meth:`Post.sponsor_users`.
+
+        .. versionadded:: 4.4
+        """
+        try:
+            sponsor_edges = self._field('edge_media_to_sponsor_user', 'edges')
+        except KeyError:
+            return False
+        return bool(sponsor_edges)
+
+    @property
+    def sponsor_users(self) -> List['Profile']:
+        """
+        The Post's sponsors.
+
+        .. versionadded:: 4.4
+        """
+        return ([] if not self.is_sponsored else
+                [Profile(self._context, edge['node']['sponsor']) for edge in
+                 self._field('edge_media_to_sponsor_user', 'edges')])
+
     @property
     def location(self) -> Optional[PostLocation]:
         """
@@ -615,6 +684,16 @@ class Profile:
     def external_url(self) -> Optional[str]:
         return self._metadata('external_url')
 
+    @property
+    def is_business_account(self) -> bool:
+        """.. versionadded:: 4.4"""
+        return self._metadata('is_business_account')
+
+    @property
+    def business_category_name(self) -> str:
+        """.. versionadded:: 4.4"""
+        return self._metadata('business_category_name')
+
     @property
     def biography(self) -> str:
         return self._metadata('biography')
@@ -789,6 +868,22 @@ class Profile:
                                                     lambda d: d['data']['user']['edge_follow'],
                                                     self._rhx_gis))
 
+    def get_similar_accounts(self) -> Iterator['Profile']:
+        """
+        Retrieve list of suggested / similar accounts for this profile.
+        To use this, one needs to be logged in.
+
+        .. versionadded:: 4.4
+        """
+        if not self._context.is_logged_in:
+            raise LoginRequiredException("--login required to get a profile's similar accounts.")
+        self._obtain_metadata()
+        yield from (Profile(self._context, edge["node"]) for edge in
+                    self._context.graphql_query("ad99dd9d3646cc3c0dda65debcd266a7",
+                                                {"user_id": str(self.userid), "include_chaining": True},
+                                                "https://www.instagram.com/{0}/".format(self.username),
+                                                self._rhx_gis)["data"]["user"]["edge_chaining"]["edges"])
+
 
 class StoryItem:
     """
@@ -1085,6 +1180,169 @@ class Highlight(Story):
         yield from (StoryItem(self._context, item, self.owner_profile) for item in self._items)
 
 
+class Hashtag:
+    """
+    An Hashtag.
+
+    Analogous to :class:`Profile`, get an instance with::
+
+       L = Instaloader()
+       hashtag = Hashtag.from_name(L.context, HASHTAG)
+
+    To then download the Hashtag's Posts, do::
+
+       for post in hashtag.get_posts():
+          L.download_post(post, target="#"+hashtag.name)
+
+    Also, this class implements == and is hashable.
+    """
+    def __init__(self, context: InstaloaderContext, node: Dict[str, Any]):
+        assert "name" in node
+        self._context = context
+        self._node = node
+        self._has_full_metadata = False
+
+    @classmethod
+    def from_name(cls, context: InstaloaderContext, name: str):
+        """
+        Create a Hashtag instance from a given hashtag name, without preceeding '#'. Raises an Exception if there is no
+        hashtag with the given name.
+
+        :param context: :attr:`Instaloader.context`
+        :param name: Hashtag, without preceeding '#'
+        :raises: :class:`QueryReturnedNotFoundException`
+        """
+        # pylint:disable=protected-access
+        hashtag = cls(context, {'name': name.lower()})
+        hashtag._obtain_metadata()
+        return hashtag
+
+    @property
+    def name(self):
+        """Hashtag name lowercased, without preceeding '#'"""
+        return self._node["name"].lower()
+
+    def _query(self, params):
+        return self._context.get_json("explore/tags/{0}/".format(self.name),
+                                      params)["graphql"]["hashtag"]
+
+    def _obtain_metadata(self):
+        if not self._has_full_metadata:
+            self._node = self._query({"__a": 1})
+            self._has_full_metadata = True
+
+    def _asdict(self):
+        json_node = self._node.copy()
+        # remove posts
+        json_node.pop("edge_hashtag_to_top_posts", None)
+        json_node.pop("edge_hashtag_to_media", None)
+        return json_node
+
+    def __repr__(self):
+        return "<Hashtag #{}>".format(self.name)
+
+    def __eq__(self, other: object) -> bool:
+        if isinstance(other, Hashtag):
+            return self.name == other.name
+        return NotImplemented
+
+    def __hash__(self) -> int:
+        return hash(self.name)
+
+    def _metadata(self, *keys) -> Any:
+        try:
+            d = self._node
+            for key in keys:
+                d = d[key]
+            return d
+        except KeyError:
+            self._obtain_metadata()
+            d = self._node
+            for key in keys:
+                d = d[key]
+            return d
+
+    @property
+    def hashtagid(self) -> int:
+        return int(self._metadata("id"))
+
+    @property
+    def profile_pic_url(self) -> str:
+        return self._metadata("profile_pic_url")
+
+    @property
+    def description(self) -> str:
+        return self._metadata("description")
+
+    @property
+    def allow_following(self) -> bool:
+        return self._metadata("allow_following")
+
+    @property
+    def is_following(self) -> bool:
+        return self._metadata("is_following")
+
+    @property
+    def is_top_media_only(self) -> bool:
+        return self._metadata("is_top_media_only")
+
+    def get_related_tags(self) -> Iterator["Hashtag"]:
+        """Yields similar hashtags."""
+        yield from (Hashtag(self._context, edge["node"])
+                    for edge in self._metadata("edge_hashtag_to_related_tags", "edges"))
+
+    def get_top_posts(self) -> Iterator[Post]:
+        """Yields the top posts of the hashtag."""
+        yield from (Post(self._context, edge["node"])
+                    for edge in self._metadata("edge_hashtag_to_top_posts", "edges"))
+
+    @property
+    def mediacount(self) -> int:
+        """
+        The count of all media associated with this hashtag.
+
+        The number of posts with a certain hashtag may differ from the number of posts that can actually be accessed, as
+        the hashtag count might include private posts
+        """
+        return self._metadata("edge_hashtag_to_media", "count")
+
+    def get_posts(self) -> Iterator[Post]:
+        """Yields the posts associated with this hashtag."""
+        self._metadata("edge_hashtag_to_media", "edges")
+        self._metadata("edge_hashtag_to_media", "page_info")
+        conn = self._metadata("edge_hashtag_to_media")
+        yield from (Post(self._context, edge["node"]) for edge in conn["edges"])
+        while conn["page_info"]["has_next_page"]:
+            data = self._query({'__a': 1, 'max_id': conn["page_info"]["end_cursor"]})
+            conn = data["edge_hashtag_to_media"]
+            yield from (Post(self._context, edge["node"]) for edge in conn["edges"])
+
+    def get_all_posts(self) -> Iterator[Post]:
+        """Yields all posts, i.e. all most recent posts and the top posts, in chronological order."""
+        sorted_top_posts = iter(sorted(self.get_top_posts(), key=lambda p: p.date_utc, reverse=True))
+        other_posts = self.get_posts()
+        next_top = next(sorted_top_posts, None)
+        next_other = next(other_posts, None)
+        while next_top is not None or next_other is not None:
+            if next_other is None:
+                yield from sorted_top_posts
+                break
+            if next_top is None:
+                yield from other_posts
+                break
+            if next_top == next_other:
+                yield next_top
+                next_top = next(sorted_top_posts, None)
+                next_other = next(other_posts, None)
+                continue
+            if next_top.date_utc > next_other.date_utc:
+                yield next_top
+                next_top = next(sorted_top_posts, None)
+            else:
+                yield next_other
+                next_other = next(other_posts, None)
+
+
 class TopSearchResults:
     """
     An invocation of this class triggers a search on Instagram for the provided search string.
@@ -1144,6 +1402,17 @@ class TopSearchResults:
             if name:
                 yield name
 
+    def get_hashtags(self) -> Iterator[Hashtag]:
+        """
+        Provides the hashtags from the search result.
+
+        .. versionadded:: 4.4
+        """
+        for hashtag in self._node.get('hashtags', []):
+            node = hashtag.get('hashtag', {})
+            if 'name' in node:
+                yield Hashtag(self._context, node)
+
     @property
     def searchstring(self) -> str:
         """
@@ -1152,17 +1421,17 @@ class TopSearchResults:
         return self._searchstring
 
 
-JsonExportable = Union[Post, Profile, StoryItem]
+JsonExportable = Union[Post, Profile, StoryItem, Hashtag]
 
 
 def save_structure_to_file(structure: JsonExportable, filename: str) -> None:
-    """Saves a :class:`Post`, :class:`Profile` or :class:`StoryItem` to a '.json' or '.json.xz' file such that it can
-    later be loaded by :func:`load_structure_from_file`.
+    """Saves a :class:`Post`, :class:`Profile`, :class:`StoryItem` or :class:`Hashtag` to a '.json' or '.json.xz' file
+    such that it can later be loaded by :func:`load_structure_from_file`.
 
     If the specified filename ends in '.xz', the file will be LZMA compressed. Otherwise, a pretty-printed JSON file
     will be created.
 
-    :param structure: :class:`Post`, :class:`Profile` or :class:`StoryItem`
+    :param structure: :class:`Post`, :class:`Profile`, :class:`StoryItem` or :class:`Hashtag`
     :param filename: Filename, ends in '.json' or '.json.xz'
     """
     json_structure = {'node': structure._asdict(),
@@ -1177,8 +1446,8 @@ def save_structure_to_file(structure: JsonExportable, filename: str) -> None:
 
 
 def load_structure_from_file(context: InstaloaderContext, filename: str) -> JsonExportable:
-    """Loads a :class:`Post`, :class:`Profile` or :class:`StoryItem` from a '.json' or '.json.xz' file that
-    has been saved by :func:`save_structure_to_file`.
+    """Loads a :class:`Post`, :class:`Profile`, :class:`StoryItem` or :class:`Hashtag` from a '.json' or '.json.xz' file
+    that has been saved by :func:`save_structure_to_file`.
 
     :param context: :attr:`Instaloader.context` linked to the new object, used for additional queries if neccessary.
     :param filename: Filename, ends in '.json' or '.json.xz'
@@ -1199,6 +1468,8 @@ def load_structure_from_file(context: InstaloaderContext, filename: str) -> Json
             return Profile(context, json_structure['node'])
         elif node_type == "StoryItem":
             return StoryItem(context, json_structure['node'])
+        elif node_type == "Hashtag":
+            return Hashtag(context, json_structure['node'])
         else:
             raise InvalidArgumentException("{}: Not an Instaloader JSON.".format(filename))
     elif 'shortcode' in json_structure:

test/instaloader_unittests.py

@@ -73,7 +73,7 @@ class TestInstaloaderAnonymously(unittest.TestCase):
         self.L.download_hashtag(HASHTAG, NORMAL_MAX_COUNT)
 
     def test_hashtag_paging(self):
-        for count, post in enumerate(self.L.get_hashtag_posts(HASHTAG)):
+        for count, post in enumerate(instaloader.Hashtag.from_name(self.L.context, HASHTAG).get_all_posts()):
             print(post)
             if count == PAGING_MAX_COUNT:
                 break
@@ -178,6 +178,11 @@ class TestInstaloaderLoggedIn(TestInstaloaderAnonymously):
         for f in profile.get_followers():
             print(f.username)
 
+    def test_get_similar_accounts(self):
+        profile = instaloader.Profile.from_username(self.L.context, OWN_USERNAME)
+        for f in profile.get_similar_accounts():
+            print(f.username)
+
     def test_get_likes(self):
         for post in instaloader.Profile.from_username(self.L.context, OWN_USERNAME).get_posts():
             for like in post.get_likes():