Release of Version 4.9.6

We are proud to announce @socialmethod as our new sponsor.

.github/workflows/documentation.yml

@@ -17,10 +17,10 @@ jobs:
           python-version: 3.8
       - name: "Install Dependencies"
         run: |
-          python -m pip install pipenv==2022.1.8
-          pipenv sync --dev
+          python -m pip install pipenv==2023.2.4
+          pipenv --python `python --version | grep -Eo '3\.[0-9]+'` sync --dev
       - name: "Build Documentation"
-        run: pipenv run make -C docs html SPHINXOPTS="-W -n"
+        run: pipenv --python `python --version | grep -Eo '3\.[0-9]+'` run make -C docs html SPHINXOPTS="-W -n"
       - name: "Deploy Documentation"
         if: github.event_name == 'push' && github.ref == 'refs/heads/master'
         uses: JamesIves/github-pages-deploy-action@v4.2.5

.github/workflows/lint.yml

@@ -7,8 +7,9 @@ jobs:
     name: PyLint and MyPy
     runs-on: ubuntu-latest
     strategy:
+      fail-fast: false
       matrix:
-        python-version: ["3.6", "3.7", "3.8", "3.9", "3.10"]
+        python-version: ["3.8", "3.9", "3.10", "3.11"]
     steps:
       - name: Checkout Instaloader Repository
         uses: actions/checkout@v2
@@ -18,9 +19,9 @@ jobs:
           python-version: ${{ matrix.python-version }}
       - name: Install Dependencies
         run: |
-          python -m pip install pipenv==2022.1.8
-          pipenv sync --dev
+          python -m pip install pipenv==2023.2.4
+          pipenv --python `python --version | grep -Eo '3\.[0-9]+'` sync --dev
       - name: PyLint
-        run: pipenv run pylint instaloader
+        run: pipenv --python `python --version | grep -Eo '3\.[0-9]+'` run pylint instaloader
       - name: MyPy
-        run: pipenv run mypy -m instaloader
+        run: pipenv --python `python --version | grep -Eo '3\.[0-9]+'` run mypy -m instaloader

.github/workflows/stale.yml

@@ -18,7 +18,7 @@ jobs:
         stale-issue-message: 'There has been no activity on this question for an extended period of time. This issue will be closed after further 14 days of inactivity.'
         stale-issue-label: 'stale'
         exempt-issue-labels: 'leave open'
-        days-before-stale: 15
+        days-before-stale: 21
         days-before-close: -1
         remove-stale-when-updated: false
     - uses: actions/stale@v1
@@ -30,5 +30,5 @@ jobs:
         stale-pr-label: 'stale'
         exempt-issue-label: 'leave open'
         exempt-pr-label: 'leave open'
-        days-before-stale: 135
+        days-before-stale: 189
         days-before-close: 14

Pipfile

@@ -14,6 +14,10 @@ psutil = "*"
 typing-extensions = "*"
 types-requests = "*"
 typed-ast = "*"
+jinja2 = "~=3.0.1"
+wrapt = "*"  # Needed for Python<3.11
+tomli = "*"  # Needed for Python<3.11
+dill = "*"  # Needed for Python<3.11
 
 [packages]
 requests = "*"

README.rst

@@ -127,6 +127,9 @@ Supporters
 
 .. current-sponsors-start
 
+| Instaloader is proudly sponsored by
+|  `@socialmethod <https://github.com/socialmethod>`__
+
 See `Alex' GitHub Sponsors <https://github.com/sponsors/aandergr>`__ page for
 how you can sponsor the development of Instaloader!
 

deploy/arch/.SRCINFO.template

@@ -6,7 +6,7 @@ pkgbase = instaloader
 	arch = any
 	license = MIT
 	makedepends = python-setuptools
-	depends = python>=3.6
+	depends = python>=3.8
 	depends = python-requests>=2.4
 	options = !emptydirs
 	source = instaloader-{{version}}.tar.gz::https://codeload.github.com/instaloader/instaloader/tar.gz/v{{version}}

deploy/arch/PKGBUILD.template

@@ -8,7 +8,7 @@ arch=('any')
 url="https://instaloader.github.io/"
 license=('MIT')
 groups=()
-depends=('python>=3.6' 'python-requests>=2.4')
+depends=('python>=3.8' 'python-requests>=2.4')
 makedepends=('python-setuptools')
 options=('!emptydirs')
 source=($pkgname-$pkgver.tar.gz::https://codeload.github.com/instaloader/instaloader/tar.gz/v$pkgver)

deploy/windows/create_exe.py

@@ -44,7 +44,7 @@ with open('__main__.py', 'w+') as f:
     f.writelines(lines)
 
 # install dependencies and invoke PyInstaller
-commands = ["pip install pipenv==2022.1.8",
+commands = ["pip install pipenv==2023.2.4",
             f"pipenv --python {sys.version_info.major}.{sys.version_info.minor} sync --dev",
             "pipenv run pyinstaller --log-level=DEBUG instaloader.spec"]
 

docs/installation.rst

@@ -8,7 +8,7 @@ Install Instaloader
 To **install Instaloader**,
 
 #. Ensure that you have `Python <https://www.python.org/>`__, at least
-   version 3.6, and `pip <https://pypi.python.org/pypi/pip>`__
+   version 3.8, and `pip <https://pypi.python.org/pypi/pip>`__
    installed.
 
 #. Then, install Instaloader using::

instaloader/__init__.py

@@ -1,7 +1,7 @@
 """Download pictures (or videos) along with their captions and other metadata from Instagram."""
 
 
-__version__ = '4.9.3'
+__version__ = '4.9.6'
 
 
 try:

instaloader/__main__.py

@@ -45,7 +45,6 @@ def filterstr_to_filterfunc(filter_str: str, item_type: type):
 
     class TransformFilterAst(ast.NodeTransformer):
         def visit_Name(self, node: ast.Name):
-            # pylint:disable=no-self-use
             if not isinstance(node.ctx, ast.Load):
                 raise InvalidArgumentException("Invalid filter: Modifying variables ({}) not allowed.".format(node.id))
             if node.id == "datetime":

instaloader/instaloader.py

@@ -221,8 +221,8 @@ class Instaloader:
                  download_comments: bool = False,
                  save_metadata: bool = True,
                  compress_json: bool = True,
-                 post_metadata_txt_pattern: str = None,
-                 storyitem_metadata_txt_pattern: str = None,
+                 post_metadata_txt_pattern: Optional[str] = None,
+                 storyitem_metadata_txt_pattern: Optional[str] = None,
                  max_connection_attempts: int = 3,
                  request_timeout: float = 300.0,
                  rate_controller: Optional[Callable[[InstaloaderContext], RateController]] = None,
@@ -447,7 +447,7 @@ class Instaloader:
         """Updates picture caption / Post metadata info"""
         def _elliptify(caption):
             pcaption = caption.replace('\n', ' ').strip()
-            return '[' + ((pcaption[:29] + u"\u2026") if len(pcaption) > 31 else pcaption) + ']'
+            return '[' + ((pcaption[:29] + "\u2026") if len(pcaption) > 31 else pcaption) + ']'
         filename += '.txt'
         caption += '\n'
         pcaption = _elliptify(caption)
@@ -878,13 +878,18 @@ class Instaloader:
         filename_template = os.path.join(dirname, self.format_filename(item, target=target))
         filename = self.__prepare_filename(filename_template, lambda: item.url)
         downloaded = False
-        if not item.is_video or self.download_video_thumbnails is True:
+        video_url_fetch_failed = False
+        if item.is_video and self.download_videos is True:
+            video_url = item.video_url
+            if video_url:
+                filename = self.__prepare_filename(filename_template, lambda: str(video_url))
+                downloaded |= (not _already_downloaded(filename + ".mp4") and
+                               self.download_pic(filename=filename, url=video_url, mtime=date_local))
+            else:
+                video_url_fetch_failed = True
+        if video_url_fetch_failed or not item.is_video or self.download_video_thumbnails is True:
             downloaded = (not _already_downloaded(filename + ".jpg") and
                           self.download_pic(filename=filename, url=item.url, mtime=date_local))
-        if item.is_video and self.download_videos is True:
-            filename = self.__prepare_filename(filename_template, lambda: str(item.video_url))
-            downloaded |= (not _already_downloaded(filename + ".mp4") and
-                           self.download_pic(filename=filename, url=item.video_url, mtime=date_local))
         # Save caption if desired
         metadata_string = _ArbitraryItemFormatter(item).format(self.storyitem_metadata_txt_pattern).strip()
         if metadata_string:
@@ -1075,7 +1080,7 @@ class Instaloader:
                                                'has_stories': False})["data"]
 
     @_requires_login
-    def download_feed_posts(self, max_count: int = None, fast_update: bool = False,
+    def download_feed_posts(self, max_count: Optional[int] = None, fast_update: bool = False,
                             post_filter: Optional[Callable[[Post], bool]] = None) -> None:
         """
         Download pictures from the user's feed.
@@ -1096,7 +1101,7 @@ class Instaloader:
         self.posts_download_loop(self.get_feed_posts(), ":feed", fast_update, post_filter, max_count=max_count)
 
     @_requires_login
-    def download_saved_posts(self, max_count: int = None, fast_update: bool = False,
+    def download_saved_posts(self, max_count: Optional[int] = None, fast_update: bool = False,
                              post_filter: Optional[Callable[[Post], bool]] = None) -> None:
         """Download user's saved pictures.
 

instaloader/instaloadercontext.py

@@ -24,10 +24,10 @@ def copy_session(session: requests.Session, request_timeout: Optional[float] = N
     """Duplicates a requests.Session."""
     new = requests.Session()
     new.cookies = requests.utils.cookiejar_from_dict(requests.utils.dict_from_cookiejar(session.cookies))
-    new.headers = session.headers.copy()
+    new.headers = session.headers.copy()  # type: ignore
     # Override default timeout behavior.
     # Need to silence mypy bug for this. See: https://github.com/python/mypy/issues/2427
-    new.request = partial(new.request, timeout=request_timeout) # type: ignore
+    new.request = partial(new.request, timeout=request_timeout)  # type: ignore
     return new
 
 
@@ -331,7 +331,8 @@ class InstaloaderContext:
             while resp.is_redirect:
                 redirect_url = resp.headers['location']
                 self.log('\nHTTP redirect from https://{0}/{1} to {2}'.format(host, path, redirect_url))
-                if redirect_url.startswith('https://www.instagram.com/accounts/login'):
+                if (redirect_url.startswith('https://www.instagram.com/accounts/login') or
+                    redirect_url.startswith('https://i.instagram.com/accounts/login')):
                     if not self.is_logged_in:
                         raise LoginRequiredException("Redirected to login page. Use --login.")
                     # alternate rate limit exceeded behavior
@@ -589,7 +590,6 @@ class RateController:
         """Wait given number of seconds."""
         # Not static, to allow for the behavior of this method to depend on context-inherent properties, such as
         # whether we are logged in.
-        # pylint:disable=no-self-use
         time.sleep(secs)
 
     def _dump_query_timestamps(self, current_time: float, failed_query_type: str):
@@ -613,7 +613,6 @@ class RateController:
         control."""
         # Not static, to allow for the count_per_sliding_window to depend on context-inherent properties, such as
         # whether we are logged in.
-        # pylint:disable=no-self-use
         return 75 if query_type == 'other' else 200
 
     def _reqs_in_sliding_window(self, query_type: Optional[str], current_time: float, window: float) -> List[float]:

instaloader/nodeiterator.py

@@ -77,7 +77,7 @@ class NodeIterator(Iterator[T]):
                  query_variables: Optional[Dict[str, Any]] = None,
                  query_referer: Optional[str] = None,
                  first_data: Optional[Dict[str, Any]] = None,
-                 is_first: Optional[Callable[[T], bool]] = None):
+                 is_first: Optional[Callable[[T, Optional[T]], bool]] = None):
         self._context = context
         self._query_hash = query_hash
         self._edge_extractor = edge_extractor
@@ -131,7 +131,7 @@ class NodeIterator(Iterator[T]):
                 raise
             item = self._node_wrapper(node)
             if self._is_first is not None:
-                if self._is_first(item):
+                if self._is_first(item, self.first_item):
                     self._first_node = node
             else:
                 if self._first_node is None:

instaloader/sectioniterator.py

@@ -29,7 +29,7 @@ class SectionIterator(Iterator[T]):
     def _query(self, max_id: Optional[str] = None) -> Dict[str, Any]:
         pagination_variables = {"max_id": max_id} if max_id is not None else {}
         return self._sections_extractor(
-            self._context.get_json(self._query_path, params={"__a": 1, **pagination_variables})
+            self._context.get_json(self._query_path, params={"__a": 1, "__d": "dis", **pagination_variables})
         )
 
     def __next__(self) -> T:

instaloader/structures.py

@@ -405,7 +405,7 @@ class Post:
         .. versionadded:: 4.2.6"""
         def _elliptify(caption):
             pcaption = ' '.join([s.replace('/', '\u2215') for s in caption.splitlines() if s]).strip()
-            return (pcaption[:30] + u"\u2026") if len(pcaption) > 31 else pcaption
+            return (pcaption[:30] + "\u2026") if len(pcaption) > 31 else pcaption
         return _elliptify(self.caption) if self.caption else ''
 
     @property
@@ -558,7 +558,7 @@ class Post:
             return []
 
         comment_edges = self._field('edge_media_to_comment', 'edges')
-        answers_count = sum([edge['node'].get('edge_threaded_comments', {}).get('count', 0) for edge in comment_edges])
+        answers_count = sum(edge['node'].get('edge_threaded_comments', {}).get('count', 0) for edge in comment_edges)
 
         if self.comments == len(comment_edges) + answers_count:
             # If the Post's metadata already contains all parent comments, don't do GraphQL requests to obtain them
@@ -638,7 +638,7 @@ class Post:
         location_id = int(loc['id'])
         if any(k not in loc for k in ('name', 'slug', 'has_public_page', 'lat', 'lng')):
             loc.update(self._context.get_json("explore/locations/{0}/".format(location_id),
-                                              params={'__a': 1})['native_location_data']['location_info'])
+                                              params={'__a': 1, '__d': 'dis'})['native_location_data']['location_info'])
         self._location = PostLocation(location_id, loc['name'], loc['slug'], loc['has_public_page'],
                                       loc.get('lat'), loc.get('lng'))
         return self._location
@@ -1039,17 +1039,8 @@ class Profile:
         )
 
     @staticmethod
-    def _make_is_newest_checker() -> Callable[[Post], bool]:
-        newest_date: Optional[datetime] = None
-        def is_newest(p: Post) -> bool:
-            nonlocal newest_date
-            post_date = p.date_local
-            if newest_date is None or post_date > newest_date:
-                newest_date = post_date
-                return True
-            else:
-                return False
-        return is_newest
+    def _make_is_newest_checker() -> Callable[[Post, Optional[Post]], bool]:
+        return lambda post, first: first is None or post.date_local > first.date_local
 
     def get_followers(self) -> NodeIterator['Profile']:
         """
@@ -1559,7 +1550,7 @@ class Hashtag:
 
     def _obtain_metadata(self):
         if not self._has_full_metadata:
-            self._node = self._query({"__a": 1})
+            self._node = self._query({"__a": 1, "__d": "dis"})
             self._has_full_metadata = True
 
     def _asdict(self):

setup.py

@@ -18,8 +18,8 @@ def get_version():
     raise SystemExit("Could not find version string.")
 
 
-if sys.version_info < (3, 6):
-    sys.exit('Instaloader requires Python >= 3.6.')
+if sys.version_info < (3, 8):
+    sys.exit('Instaloader requires Python >= 3.8.')
 
 requirements = ['requests>=2.4']
 
@@ -43,7 +43,7 @@ setup(
                 'from Instagram.',
     long_description=open(os.path.join(SRC, 'README.rst')).read(),
     install_requires=requirements,
-    python_requires='>=3.6',
+    python_requires='>=3.8',
     entry_points={'console_scripts': ['instaloader=instaloader.__main__:main']},
     zip_safe=False,
     keywords=keywords,
@@ -54,10 +54,10 @@ setup(
         'Intended Audience :: Developers',
         'License :: OSI Approved :: MIT License',
         'Operating System :: OS Independent',
-        'Programming Language :: Python :: 3.6',
-        'Programming Language :: Python :: 3.7',
         'Programming Language :: Python :: 3.8',
         'Programming Language :: Python :: 3.9',
+        'Programming Language :: Python :: 3.10',
+        'Programming Language :: Python :: 3.11',
         'Programming Language :: Python :: 3 :: Only',
         'Topic :: Internet',
         'Topic :: Multimedia :: Graphics'