Third Release Candidate for Version 4.4.4

Addresses a suggestion in #630.

.github/workflows/stale.yml

@@ -2,7 +2,7 @@ name: Mark stale issues and pull requests
 
 on:
   schedule:
-  - cron: "15 16 * * *"
+  - cron: "10 14 * * *"
 
 jobs:
   stale:
@@ -10,6 +10,16 @@ jobs:
     runs-on: ubuntu-latest
     
     steps:
+    - uses: actions/stale@v3
+      with:
+        repo-token: ${{ secrets.GITHUB_TOKEN }}
+        only-labels: 'question'
+        stale-issue-message: 'There has been no activity on this question for an extended period of time. This issue will be closed after further 14 days of inactivity.'
+        stale-issue-label: 'stale'
+        exempt-issue-labels: 'leave open'
+        days-before-stale: 15
+        days-before-close: -1
+        remove-stale-when-updated: false
     - uses: actions/stale@v1
       with:
         repo-token: ${{ secrets.GITHUB_TOKEN }}
@@ -21,4 +31,3 @@ jobs:
         exempt-pr-label: 'leave open'
         days-before-stale: 90
         days-before-close: 14
-        operations-per-run: 7

deploy/windows/create_exe.py

@@ -8,33 +8,18 @@ import sys
 # copy the required files into repo root
 shutil.copy('docs/favicon.ico', '.')
 shutil.copy('deploy/windows/instaloader.spec', '.')
-shutil.unpack_archive('deploy/windows/ps', '.', 'xztar')
 
 code = """
+import contextlib
 import psutil
 import subprocess
 
 def __main():
-    grandparentpid = psutil.Process(os.getppid()).ppid()
-    grandparentpidsearchstring = ' ' + str(grandparentpid) + ' '
-    if hasattr(sys, "_MEIPASS"):
-        ps = os.path.join(sys._MEIPASS, 'tasklist.exe')
-    else:
-        ps = 'tasklist'
-    popen = subprocess.Popen(ps, stdout=subprocess.PIPE, universal_newlines=True)
-    for examine in iter(popen.stdout.readline, ""):
-        if grandparentpidsearchstring in examine:
-            pname = examine
-            break
-    popen.stdout.close()
-    return_code = popen.wait()
-    if return_code:
-        raise subprocess.CalledProcessError(return_code, ps)
-    if pname[0:12] == 'explorer.exe':
-        subprocess.Popen("cmd /K \\\"{0}\\\"".format(os.path.splitext(os.path.basename(sys.argv[0]))[0]))
-    else:
-        main()
-
+    with contextlib.suppress(AttributeError, psutil.Error):
+        if psutil.Process().parent().parent().name() == "explorer.exe":
+            subprocess.Popen("powershell -NoExit -Command \\\"& {0}\\\"".format(sys.argv[0]))
+            return
+    main()
 
 if __name__ == "__main__":
     __main()

deploy/windows/instaloader.spec

@@ -5,7 +5,7 @@ block_cipher = None
 
 a = Analysis(['__main__.py'],
              binaries=[],
-             datas=[('tasklist.exe', '.'), ('framedyn.dll', '.')],
+             datas=[],
              hiddenimports=[],
              hookspath=[],
              runtime_hooks=[],

docs/_static/instaloader.css

@@ -17,15 +17,16 @@ code {
   .highlight pre {
     padding: 0.7em;
     color: #fff; }
-  .highlight .c1 {
+  .highlight .c1, .highlight .k, .highlight .kn, .highlight .ow {
     color: #008d06; }
-  .highlight .k, .highlight .kn, .highlight .ow {
-    color: #f48400;
-    font-weight: bold; }
-  .highlight .nb {
+  .highlight .nb, .highlight .ne, .highlight .nf, .highlight .vm {
     color: #f48400; }
-  .highlight .s1, .highlight .s2, .highlight .mi {
+  .highlight .s1, .highlight .s2, .highlight .sa, .highlight .se, .highlight .si, .highlight .mi {
     color: #ff7efd; }
+  .highlight .c1, .highlight .nf, .highlight .sa, .highlight .se, .highlight .si {
+    font-style: italic; }
+  .highlight .k, .highlight .kn, .highlight .ow {
+    font-weight: bold; }
 
 .doc-content {
   padding-top: 0.5em;

docs/_static/instaloader.scss

@@ -36,22 +36,25 @@ code {
     color: #fff;
   }
 
-  .c1 {
+  .c1, .k, .kn, .ow {
     color: $color_instaloader_main
   }
 
-  .k, .kn, .ow {
-    color: $color_instaloader_accent;
-    font-weight: bold
-  }
-
-  .nb {
+  .nb, .ne, .nf, .vm {
     color: $color_instaloader_accent
   }
 
-  .s1, .s2, .mi {
+  .s1, .s2, .sa, .se, .si, .mi {
     color: $color_instaloader_hyper
   }
+
+  .c1, .nf, .sa, .se, .si {
+    font-style: italic;
+  }
+
+  .k, .kn, .ow {
+    font-weight: bold
+  }
 }
 
 .doc-content {

docs/basic-usage.rst

@@ -51,7 +51,7 @@ invoke it with
     instaloader --login=your_username profile [profile ...]
 
 When logging in, Instaloader **stores the session cookies** in a file in your
-temporary directory, which will be reused later the next time :option:`--login`
+home directory, which will be reused later the next time :option:`--login`
 is given.  So you can download private profiles **non-interactively** when you
 already have a valid session cookie file.
 
@@ -291,17 +291,18 @@ Instaloader as Cronjob
 Instaloader is suitable for running as a cronjob to periodically update your
 personal Instagram archive. The :option:`--quiet` option disables user
 interactions and logging of non-error messages. To non-interactively use
-Instaloader logged-in, create a session file, e.g. in your home directory::
+Instaloader logged-in, create a session file::
 
-   instaloader --login=your_username --sessionfile=~/.instaloader-session
+   instaloader --login=your_username
 
-Then use the same parameters in your cronjob to load the session and download
+Then use the same parameter in your cronjob to load the session and download
 the given targets::
 
-   instaloader --login=your_username --sessionfile=~/.instaloader-session --quiet <target> [...]
+   instaloader --login=your_username --quiet <target> [...]
 
-Without :option:`--sessionfile` option, Instaloader saves the session file in
-a path within your temporary directory.
+Instaloader saves the session file to
+``~/.config/instaloader/session-YOUR-USERNAME``. See
+:option:`--sessionfile` option for how to override this path.
 
 Programming Instaloader
 ^^^^^^^^^^^^^^^^^^^^^^^

docs/cli-options.rst

@@ -193,9 +193,8 @@ Instaloader to login.
 
 .. option:: --sessionfile SESSIONFILE, -f SESSIONFILE
 
-   Path for loading and storing session key file.  Defaults to a path within
-   your temporary directory, encoding your local username and your Instagram
-   profile name.
+   Path for loading and storing session key file.  Defaults to
+   ``~/.config/instaloader/session-YOUR-USERNAME``.
 
 .. option:: --password YOUR-PASSWORD, -p YOUR-PASSWORD
 
@@ -243,6 +242,11 @@ How to Download
 
    .. versionadded:: 4.2
 
+   .. warning::
+      We discourage from using the "commit mode" due to its bugs when being used
+      with other command line options (e.g. :issue:`257` with
+      :option:`--fast-update` and :issue:`483` with :option:`--tagged`).
+
 .. option:: --request-timeout N
 
    Seconds to wait before timing out a connection request.

docs/codesnippets/615_import_firefox_session.py

@@ -0,0 +1,53 @@
+from argparse import ArgumentParser
+from glob import glob
+from os.path import expanduser
+from platform import system
+from sqlite3 import OperationalError, connect
+
+try:
+    from instaloader import ConnectionException, Instaloader
+except ModuleNotFoundError:
+    raise SystemExit("Instaloader not found.\n  pip install [--user] instaloader")
+
+
+def get_cookiefile():
+    default_cookiefile = {
+        "Windows": "~/AppData/Roaming/Mozilla/Firefox/Profiles/*/cookies.sqlite",
+        "Darwin": "~/Library/Application Support/Firefox/Profiles/*/cookies.sqlite",
+    }.get(system(), "~/.mozilla/firefox/*/cookies.sqlite")
+    cookiefiles = glob(expanduser(default_cookiefile))
+    if not cookiefiles:
+        raise SystemExit("No Firefox cookies.sqlite file found. Use -c COOKIEFILE.")
+    return cookiefiles[0]
+
+
+def import_session(cookiefile, sessionfile):
+    print("Using cookies from {}.".format(cookiefile))
+    conn = connect(cookiefile)
+    try:
+        cookie_data = conn.execute(
+            "SELECT name, value FROM moz_cookies WHERE baseDomain='instagram.com'"
+        )
+    except OperationalError:
+        cookie_data = conn.execute(
+            "SELECT name, value FROM moz_cookies WHERE host LIKE '%instagram.com'"
+        )
+    instaloader = Instaloader(max_connection_attempts=1)
+    instaloader.context._session.cookies.update(cookie_data)
+    username = instaloader.test_login()
+    if not username:
+        raise SystemExit("Not logged in. Are you logged in successfully in Firefox?")
+    print("Imported session cookie for {}.".format(username))
+    instaloader.context.username = username
+    instaloader.save_session_to_file(sessionfile)
+
+
+if __name__ == "__main__":
+    p = ArgumentParser()
+    p.add_argument("-c", "--cookiefile")
+    p.add_argument("-f", "--sessionfile")
+    args = p.parse_args()
+    try:
+        import_session(args.cookiefile or get_cookiefile(), args.sessionfile)
+    except (ConnectionException, OperationalError) as e:
+        raise SystemExit("Cookie import failed: {}".format(e))

docs/codesnippets/92_import_firefox_session.py

@@ -1,23 +0,0 @@
-from glob import glob
-from os.path import expanduser
-from sqlite3 import connect
-
-from instaloader import ConnectionException, Instaloader
-
-# FIREFOXCOOKIEFILE = "/home/alex/.mozilla/firefox/l96w6b90.default/cookies.sqlite"
-FIREFOXCOOKIEFILE = glob(expanduser("~/.mozilla/firefox/*.default/cookies.sqlite"))[0]
-
-instaloader = Instaloader(max_connection_attempts=1)
-instaloader.context._session.cookies.update(connect(FIREFOXCOOKIEFILE)
-                                            .execute("SELECT name, value FROM moz_cookies "
-                                                     "WHERE baseDomain='instagram.com'"))
-
-try:
-    username = instaloader.test_login()
-    if not username:
-        raise ConnectionException()
-except ConnectionException:
-    raise SystemExit("Cookie import failed. Are you logged in successfully in Firefox?")
-
-instaloader.context.username = username
-instaloader.save_session_to_file()

docs/conf.py

@@ -49,7 +49,22 @@ autodoc_member_order = 'bysource'
 intersphinx_mapping = {'python': ('https://docs.python.org/3', None),
                        'requests': ('https://requests.kennethreitz.org/en/master/', None)}
 
-extlinks = {'issue': ('https://github.com/instaloader/instaloader/issues/%s', 'issue #')}
+current_release = subprocess.check_output(["git", "describe", "--abbrev=0"]).decode("ascii")[1:-1]
+current_release_date = subprocess.check_output(
+    ["git", "log", "-1", "--tags", "--format=%ad", "--date=format:%e %b %Y"]).decode("ascii")[:-1]
+
+html_context = {'current_release': current_release, 'current_release_date': current_release_date}
+
+extlinks = {
+    'issue': (
+        'https://github.com/instaloader/instaloader/issues/%s',
+        'Issue #'
+    ),
+    'example': (
+        'https://raw.githubusercontent.com/instaloader/instaloader/v' + current_release + '/docs/codesnippets/%s',
+        'Example '
+    ),
+}
 
 # Add any paths that contain templates here, relative to this directory.
 templates_path = ['_templates']
@@ -69,7 +84,7 @@ master_doc = 'index'
 
 # General information about the project.
 project = 'Instaloader'
-copyright = '2017, Alexander Graf and Andre Koch-Kramer'
+copyright = '2017-2020, Alexander Graf and Andre Koch-Kramer'
 author = 'Alexander Graf and Andre Koch-Kramer'
 
 # The version info for the project you're documenting, acts as replacement for
@@ -353,12 +368,6 @@ texinfo_documents = [
 # texinfo_no_detailmenu = False
 
 
-current_release = subprocess.check_output(["git", "describe", "--abbrev=0"]).decode("ascii")[1:-1]
-current_release_date = subprocess.check_output(["git", "log", "-1", "--tags", "--format=%ad", "--date=format:%e %b %Y"]).decode("ascii")[:-1]
-
-html_context = {'current_release': current_release, 'current_release_date': current_release_date}
-
-
 def setup(app):
     typing.TYPE_CHECKING = True
     app.add_stylesheet("instaloader.css")

docs/troubleshooting.rst

@@ -32,7 +32,7 @@ rate limit has almost been reached, according to Instaloader's own rate
 accounting mechanism. We regularly adjust this mechanism to match Instagram's
 current rate limiting.
 
-Login error
+Login Error
 -----------
 
 Instaloader's login *should* work fine, both with and without
@@ -40,21 +40,31 @@ Two-Factor-Authentication. It also supports handling the *checkpoint challenge*,
 issued when Instagram suspects authentication activity on your account, by
 pointing the user to an URL to be opened in a browser.
 
-Nevertheless, in :issue:`92` users report problems with logging in. To still use
-Instaloader's logged-in functionality, you may use the following script to
-workaround login problems by importing the session cookies from Firefox and
-bypassing Instaloader's login.
+Nevertheless, in :issue:`92` and :issue:`615` users reported problems with
+logging in. We recommend to always keep the sessionfile which Instaloader
+creates when using :option:`--login`. If a sessionfile is present,
+:option:`--login` does not make make use of the failure-prone login procedure.
+Also, sessionfiles usually do not expire and can be copied between different
+computers without any problems.
 
-.. literalinclude:: codesnippets/92_import_firefox_session.py
+If you do not have a sessionfile present, you may use the following script
+(:example:`615_import_firefox_session.py`) to workaround login problems by
+importing the session cookies from Firefox and bypassing Instaloader's login and
+so still use Instaloader's logged-in functionality.
 
-To use this,
+.. literalinclude:: codesnippets/615_import_firefox_session.py
 
-#. login to Instagram in Firefox, 
+To use this script,
 
-#. execute the snippet,
+#. Download the script: :example:`615_import_firefox_session.py`,
 
-#. then, ``instaloader -l USERNAME`` should work fine.
+#. Login to Instagram in Firefox,
 
-If you do not use your default firefox profile, or your operating system has the
-paths differently set up, you may have to alter the ``FIREFOXCOOKIEFILE``
-variable first.
+#. Execute the snippet, e.g. with ``python 615_import_firefox_session.py``,
+
+#. Then, ``instaloader -l USERNAME`` should work fine.
+
+This script also supports specifying a cookiefile path, which may be useful if
+you use multiple Firefox profiles or if your operating system has the directory
+structure differently set up. Also, you can specify an alternative sessionfile
+path.

instaloader/__init__.py

@@ -1,7 +1,7 @@
 """Download pictures (or videos) along with their captions and other metadata from Instagram."""
 
 
-__version__ = '4.4.3'
+__version__ = '4.4.4rc3'
 
 
 try:

instaloader/__main__.py

@@ -196,8 +196,8 @@ def _main(instaloader: Instaloader, targetlist: List[str],
         if len(profiles) > 1:
             instaloader.context.log("Downloading {} profiles: {}".format(len(profiles),
                                                                          ' '.join([p.username for p in profiles])))
-        if profiles and download_profile_pic and not instaloader.context.is_logged_in:
-            instaloader.context.error("Warning: Use --login to download HD version of profile pictures.")
+        if profiles and (download_profile_pic or download_posts) and not instaloader.context.is_logged_in:
+            instaloader.context.error("Warning: Use --login to download higher-quality versions of pictures.")
         instaloader.download_profiles(profiles,
                                       download_profile_pic, download_posts, download_tagged, download_igtv,
                                       download_highlights, download_stories,

instaloader/instaloader.py

@@ -27,6 +27,20 @@ from .structures import (Hashtag, Highlight, JsonExportable, Post, PostLocation,
 
 def get_default_session_filename(username: str) -> str:
     """Returns default session filename for given username."""
+    sessionfilename = "session-{}".format(username)
+    if platform.system() == "Windows":
+        # on Windows, use %LOCALAPPDATA%\Instaloader\session-USERNAME
+        localappdata = os.getenv("LOCALAPPDATA")
+        if localappdata is not None:
+            return os.path.join(localappdata, "Instaloader", sessionfilename)
+        # legacy fallback - store in temp dir if %LOCALAPPDATA% is not set
+        return os.path.join(tempfile.gettempdir(), ".instaloader-" + getpass.getuser(), sessionfilename)
+    # on Unix, use ~/.config/instaloader/session-USERNAME
+    return os.path.join(os.getenv("XDG_CONFIG_HOME", os.path.expanduser("~/.config")), "instaloader", sessionfilename)
+
+
+def get_legacy_session_filename(username: str) -> str:
+    """Returns legacy (until v4.4.3) default session filename for given username."""
     dirname = tempfile.gettempdir() + "/" + ".instaloader-" + getpass.getuser()
     filename = dirname + "/" + "session-" + username
     return filename.lower()
@@ -441,6 +455,8 @@ class Instaloader:
         """
         if filename is None:
             filename = get_default_session_filename(username)
+            if not os.path.exists(filename):
+                filename = get_legacy_session_filename(username)
         with open(filename, 'rb') as sessionfile:
             self.context.load_session_from_file(username, sessionfile)
             self.context.log("Loaded session from %s." % filename)

instaloader/instaloadercontext.py

@@ -113,7 +113,7 @@ class InstaloaderContext:
     def close(self):
         """Print error log and close session"""
         if self.error_log and not self.quiet:
-            print("\nErrors occurred:", file=sys.stderr)
+            print("\nErrors or warnings occurred:", file=sys.stderr)
             for err in self.error_log:
                 print(err, file=sys.stderr)
         self._session.close()
@@ -536,8 +536,8 @@ class InstaloaderContext:
 
         .. versionadded:: 4.2.1"""
         with copy_session(self._session, self.request_timeout) as tempsession:
-            tempsession.headers['User-Agent'] = 'Instagram 10.3.2 (iPhone7,2; iPhone OS 9_3_3; en_US; en-US; ' \
-                                                'scale=2.00; 750x1334) AppleWebKit/420+'
+            tempsession.headers['User-Agent'] = 'Instagram 123.1.0.26.115 (iPhone12,1; iOS 13_3; en_US; en-US; ' \
+                                                'scale=2.00; 1656x3584; 190542906)'
             for header in ['Host', 'Origin', 'X-Instagram-AJAX', 'X-Requested-With']:
                 tempsession.headers.pop(header, None)
             return self.get_json(path, params, 'i.instagram.com', tempsession)

instaloader/structures.py

@@ -71,6 +71,10 @@ class Post:
         self._full_metadata_dict = None  # type: Optional[Dict[str, Any]]
         self._rhx_gis_str = None         # type: Optional[str]
         self._location = None            # type: Optional[PostLocation]
+        self._iphone_struct_ = None
+        if 'iphone_struct' in node:
+            # if loaded from JSON with load_structure_from_file()
+            self._iphone_struct_ = node['iphone_struct']
 
     @classmethod
     def from_shortcode(cls, context: InstaloaderContext, shortcode: str):
@@ -111,6 +115,8 @@ class Post:
             node['owner'] = self.owner_profile._asdict()
         if self._location:
             node['location'] = self._location._asdict()
+        if self._iphone_struct_:
+            node['iphone_struct'] = self._iphone_struct_
         return node
 
     @property
@@ -160,6 +166,15 @@ class Post:
         self._obtain_metadata()
         return self._rhx_gis_str
 
+    @property
+    def _iphone_struct(self) -> Dict[str, Any]:
+        if not self._context.is_logged_in:
+            raise LoginRequiredException("--login required to access iPhone media info endpoint.")
+        if not self._iphone_struct_:
+            data = self._context.get_iphone_json(path='api/v1/media/{}/info/'.format(self.mediaid), params={})
+            self._iphone_struct_ = data['items'][0]
+        return self._iphone_struct_
+
     def _field(self, *keys) -> Any:
         """Lookups given fields in _node, and if not found in _full_metadata. Raises KeyError if not found anywhere."""
         try:
@@ -225,6 +240,13 @@ class Post:
     @property
     def url(self) -> str:
         """URL of the picture / video thumbnail of the post"""
+        if self.typename == "GraphImage" and self._context.is_logged_in:
+            try:
+                orig_url = self._iphone_struct['image_versions2']['candidates'][0]['url']
+                url = re.sub(r'&se=\d+(&?)', r'\1', orig_url)
+                return url
+            except (InstaloaderException, KeyError, IndexError) as err:
+                self._context.error('{} Unable to fetch high quality image version of {}.'.format(err, self))
         return self._node["display_url"] if "display_url" in self._node else self._node["display_src"]
 
     @property
@@ -239,10 +261,18 @@ class Post:
             if any(edge['node']['is_video'] for edge in edges):
                 # video_url is only present in full metadata, issue #558.
                 edges = self._full_metadata['edge_sidecar_to_children']['edges']
-            for edge in edges:
+            for idx, edge in enumerate(edges):
                 node = edge['node']
                 is_video = node['is_video']
-                yield PostSidecarNode(is_video=is_video, display_url=node['display_url'],
+                display_url = node['display_url']
+                if not is_video and self._context.is_logged_in:
+                    try:
+                        carousel_media = self._iphone_struct['carousel_media']
+                        orig_url = carousel_media[idx]['image_versions2']['candidates'][0]['url']
+                        display_url = re.sub(r'&se=\d+(&?)', r'\1', orig_url)
+                    except (InstaloaderException, KeyError, IndexError) as err:
+                        self._context.error('{} Unable to fetch high quality image version of {}.'.format(err, self))
+                yield PostSidecarNode(is_video=is_video, display_url=display_url,
                                       video_url=node['video_url'] if is_video else None)
 
     @property