Release of Version 4.8.5

Fixes #1379.

Revert "Make {Post,StoryItem}.date_utc timezone aware"
d864ce08ff.

Revert "Remove unnecessary conversion to timezone aware timestamp (#1372)"
555c86633c.

Revert "Timezone option for datetime style formatting (#1316)"
21365ec668.

.github/ISSUE_TEMPLATE/feature_request.md

@@ -6,13 +6,15 @@ labels: "feature suggestion"
 ---
 
 **Provide us a use case of the feature**
-How could the user invoke the new function? Which problem would it solve? If new information is obtained, how would it be further processed?
+How could the user invoke the new function? Which problem would it solve?
+If new information is obtained, how would it be further processed?
 
 **Describe the solution you'd like**
 A clear and concise description of what you want to happen.
 
 **Describe alternatives you've considered**
 A clear and concise description of any alternative solutions or features you've considered.
+How do you achieve the here-described goal without the feature you are requesting?
 
 **If the feature request is accepted, would you be willing to submit a pull request?**
 Yes / No
@@ -21,4 +23,9 @@ Yes / No
 **Additional context**
 Add any other context about the feature request here.
 
-<!-- please also see https://instaloader.github.io/contributing.html for how to suggest a feature -->
+<!-- 
+Also see https://instaloader.github.io/contributing.html for how to suggest a feature.
+
+Please understand that a feature suggestion issue that lacks the required information cannot be
+processed.
+-->

.github/workflows/documentation.yml

@@ -0,0 +1,31 @@
+name: "Documentation"
+
+on: [push, pull_request]
+
+jobs:
+  docs:
+    name: "Documentation"
+    runs-on: ubuntu-latest
+    steps:
+      - name: "Checkout Instaloader Repository"
+        uses: actions/checkout@v2
+        with:
+          fetch-depth: 0  # needed for building docs
+      - name: "Setup Python"
+        uses: actions/setup-python@v2
+        with:
+          python-version: 3.8
+      - name: "Install Dependencies"
+        run: |
+          python -m pip install pipenv==2022.1.8
+          pipenv sync --dev
+      - name: "Build Documentation"
+        run: pipenv run make -C docs html SPHINXOPTS="-W -n"
+      - name: "Deploy Documentation"
+        if: github.event_name == 'push' && github.ref == 'refs/heads/master'
+        uses: JamesIves/github-pages-deploy-action@v4.2.5
+        with:
+          branch: master
+          repository-name: instaloader/instaloader.github.io
+          folder: docs/_build/html
+          token: ${{ secrets.PAGES_DEPLOYMENT_TOKEN }}

.github/workflows/lint.yml

@@ -8,24 +8,19 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        python-version: [3.8, 3.9]
+        python-version: ["3.6", "3.7", "3.8", "3.9", "3.10"]
     steps:
       - name: Checkout Instaloader Repository
         uses: actions/checkout@v2
-        with:
-          fetch-depth: 0  # needed for building docs
       - name: Setup Python
         uses: actions/setup-python@v2
         with:
           python-version: ${{ matrix.python-version }}
       - name: Install Dependencies
         run: |
-          python -m pip install pipenv==2020.11.15
+          python -m pip install pipenv==2022.1.8
           pipenv sync --dev
       - name: PyLint
         run: pipenv run pylint instaloader
       - name: MyPy
         run: pipenv run mypy -m instaloader
-      - name: Build Documentation
-        if: matrix.python-version == '3.8'
-        run: pipenv run make -C docs html SPHINXOPTS="-W -n"

.github/workflows/packages.yml

@@ -0,0 +1,40 @@
+name: "Publish Packages"
+
+on:
+  push:
+    tags:
+      - v*
+
+jobs:
+  publish:
+    name: "Publish Package"
+    runs-on: ubuntu-latest
+    steps:
+      - name: "Checkout Repository"
+        uses: actions/checkout@v2
+      - name: "Setup Python"
+        uses: actions/setup-python@v2
+        with:
+          python-version: 3.8
+      - name: "Get the tagged version"
+        id: get_version
+        env:
+          GITHUB_REF: ${{ github.ref }}
+        run: echo ::set-output name=VERSION::${GITHUB_REF/refs\/tags\/v/}
+        shell: bash
+      - name: "Install Dependencies"
+        run: |
+          python -m pip install pipenv==2022.1.8
+          pipenv sync --dev
+      - name: "Generate Python Package"
+        run: python setup.py sdist
+      - name: "Publish Distribution to PyPI"
+        uses: pypa/gh-action-pypi-publish@master
+        with:
+          password: ${{ secrets.PYPI_PASSWORD }}
+      - name: "Publish Distribution to AUR"
+        run: deploy/arch/deploy.sh $VERSION_TAG
+        env:
+          VERSION_TAG: v${{ steps.get_version.outputs.VERSION }}
+          AUR_KEY: ${{ secrets.AUR_KEY }}
+          AUR_IV: ${{ secrets.AUR_IV }}

.github/workflows/windows_exe.yml

@@ -10,9 +10,9 @@ jobs:
       - name: Checkout Instaloader repository
         uses: actions/checkout@v2
       - name: Setup Python
-        uses: actions/setup-python@v1
+        uses: actions/setup-python@v2
         with:
-          python-version: 3.9
+          python-version: "3.9"
           architecture: x64
       - name: Get the tagged version
         id: get_version

.travis.yml

@@ -1,35 +0,0 @@
-dist: xenial
-language: python
-python:
- - "3.6"
- - "3.7"
-install:
- - pipenv sync --dev
-script:
- - pylint instaloader
- - mypy -m instaloader
- - make -C docs html SPHINXOPTS="-W -n"
-addons:
-  ssh_known_hosts: aur.archlinux.org
-deploy:
-  - provider: pypi
-    user: aandergr
-    on:
-      tags: true
-      python: 3.6
-    password:
-      secure: KBsATtXNnEzqr0X2ALdyx2zaYj2EUfK8YjP4jEOilBELpJ4evHYXVgRkn9CKXGsD7LKbWmR3gz2rr8NZ6U42pHw0fpQHizz91MpJSKbNY7RZZh4UVWMDh81izeglchWAF+m3Wdpxpo/GBDbrZIbRQFhPgB+kjSzJo3uiCS+yVXtGG+6csJHzd0WkJjsVHz+XSdxyOiLy/qFF3nmY6ZrVrgw6WbX2YLbx9je6DxK3YRrnBqgMgH7Ui2xWWbgJCk3KKpQTMiMKnTa9mdeMJAd26YMCz1gG/X7ck6VNly/6bs5+Zq7/bU0dSolhPBhNthyccYdS0IeGPdPF3YqatblJ4fLGTfHn8oJs3/T11yIW9pvOT3UftEynPOUX+en2Qnb/3WfgwUfVts8i8DAYgEFLGEzbF5XMGy4gwAD/bZ0JW0pmgAG2PZsdOokAt+ydlNj/uxq1GFW2doZbul6k4t03p1mik6itYscvuhKoEm8Yw6wr1P9aDvMR6FFpSu4gDt2XrKmRmWkSXxpNeCfZujTjyAnISxLVQGSY7BG5K0Z2HbDtqqNz9nj4WgNGzZmngiRofCevWR+vPIDUDW7NCX03csfcFS9RRllBO/BrL7NVUV+Qp9dfTQYp+8SqYLWgHtDoXdgyMREjJaAmbx2u/TYTY8bADZ5F3vGUpv2j5hP3NTU=
-  - provider: pages
-    skip-cleanup: true
-    github-token: $GITHUB_TOKEN
-    local-dir: docs/_build/html
-    repo: instaloader/instaloader.github.io
-    target-branch: master
-    on:
-      branch: master
-      python: 3.6
-  - provider: script
-    script: deploy/arch/deploy.sh $TRAVIS_TAG
-    on:
-      tags: true
-      python: 3.6

deploy/arch/deploy.sh

@@ -8,10 +8,12 @@ echo $VERSION | grep -qe "[abc]" && exit 0
 cd $(dirname $0)
 
 # decrypt and add ssh key
-openssl aes-256-cbc -K $encrypted_354637631c28_key -iv $encrypted_354637631c28_iv -in id_rsa_AUR.enc -out /tmp/AUR_openssh -d
+openssl aes-256-cbc -K $AUR_KEY -iv $AUR_IV -in id_rsa_AUR.enc -out /tmp/AUR_openssh -d
 eval "$(ssh-agent -s)"
 chmod 600 /tmp/AUR_openssh
 ssh-add /tmp/AUR_openssh
+mkdir -p ~/.ssh
+ssh-keyscan -t rsa aur.archlinux.org >> ~/.ssh/known_hosts
 
 # clone and modify AUR repo
 git clone --depth 1 ssh://aur@aur.archlinux.org/instaloader.git

deploy/windows/create_exe.py

@@ -44,8 +44,8 @@ with open('__main__.py', 'w+') as f:
     f.writelines(lines)
 
 # install dependencies and invoke PyInstaller
-commands = ["pip install pipenv==2020.11.15",
-            "pipenv sync --dev",
+commands = ["pip install pipenv==2022.1.8",
+            f"pipenv --python {sys.version_info.major}.{sys.version_info.minor} sync --dev",
             "pipenv run pyinstaller --log-level=DEBUG instaloader.spec"]
 
 for command in commands:

instaloader/__init__.py

@@ -1,7 +1,7 @@
 """Download pictures (or videos) along with their captions and other metadata from Instagram."""
 
 
-__version__ = '4.8rc4'
+__version__ = '4.8.5'
 
 
 try:

instaloader/__main__.py

@@ -93,6 +93,7 @@ def _main(instaloader: Instaloader, targetlist: List[str],
     latest_stamps = None
     if latest_stamps_file is not None:
         latest_stamps = LatestStamps(latest_stamps_file)
+        instaloader.context.log(f"Using latest stamps from {latest_stamps_file}.")
     # Login, if desired
     if username is not None:
         if not re.match(r"^[A-Za-z0-9._]+$", username):

instaloader/instaloader.py

@@ -312,15 +312,25 @@ class Instaloader:
                      filename_suffix: Optional[str] = None, _attempt: int = 1) -> bool:
         """Downloads and saves picture with given url under given directory with given timestamp.
         Returns true, if file was actually downloaded, i.e. updated."""
-        urlmatch = re.search('\\.[a-z0-9]*\\?', url)
-        file_extension = url[-3:] if urlmatch is None else urlmatch.group(0)[1:-1]
         if filename_suffix is not None:
             filename += '_' + filename_suffix
-        filename += '.' + file_extension
-        if os.path.isfile(filename):
+        urlmatch = re.search('\\.[a-z0-9]*\\?', url)
+        file_extension = url[-3:] if urlmatch is None else urlmatch.group(0)[1:-1]
+        nominal_filename = filename + '.' + file_extension
+        if os.path.isfile(nominal_filename):
+            self.context.log(nominal_filename + ' exists', end=' ', flush=True)
+            return False
+        resp = self.context.get_raw(url)
+        if 'Content-Type' in resp.headers and resp.headers['Content-Type']:
+            header_extension = '.' + resp.headers['Content-Type'].split(';')[0].split('/')[-1]
+            header_extension = header_extension.lower().replace('jpeg', 'jpg')
+            filename += header_extension
+        else:
+            filename = nominal_filename
+        if filename != nominal_filename and os.path.isfile(filename):
             self.context.log(filename + ' exists', end=' ', flush=True)
             return False
-        self.context.get_and_write_raw(url, filename)
+        self.context.write_raw(resp, filename)
         os.utime(filename, (datetime.now().timestamp(), mtime.timestamp()))
         return True
 
@@ -453,9 +463,12 @@ class Instaloader:
     def save_location(self, filename: str, location: PostLocation, mtime: datetime) -> None:
         """Save post location name and Google Maps link."""
         filename += '_location.txt'
-        location_string = (location.name + "\n" +
-                           "https://maps.google.com/maps?q={0},{1}&ll={0},{1}\n".format(location.lat,
-                                                                                        location.lng))
+        if location.lat is not None and location.lng is not None:
+            location_string = (location.name + "\n" +
+                               "https://maps.google.com/maps?q={0},{1}&ll={0},{1}\n".format(location.lat,
+                                                                                            location.lng))
+        else:
+            location_string = location.name
         with open(filename, 'wb') as text_file:
             with BytesIO(location_string.encode()) as bio:
                 shutil.copyfileobj(cast(IO, bio), text_file)
@@ -788,6 +801,7 @@ class Instaloader:
 
         if not userids:
             self.context.log("Retrieving all visible stories...")
+            profile_count = None
         else:
             userids = [p if isinstance(p, int) else p.userid for p in userids]
             profile_count = len(userids)

instaloader/instaloadercontext.py

@@ -529,6 +529,28 @@ class InstaloaderContext:
         :raises ConnectionException: When download repeatedly failed."""
         self.write_raw(self.get_raw(url), filename)
 
+    def head(self, url: str, allow_redirects: bool = False) -> requests.Response:
+        """HEAD a URL anonymously.
+
+        :raises QueryReturnedNotFoundException: When the server responds with a 404.
+        :raises QueryReturnedForbiddenException: When the server responds with a 403.
+        :raises ConnectionException: When request failed.
+
+        .. versionadded:: 4.7.6
+        """
+        with self.get_anonymous_session() as anonymous_session:
+            resp = anonymous_session.head(url, allow_redirects=allow_redirects)
+        if resp.status_code == 200:
+            return resp
+        else:
+            if resp.status_code == 403:
+                # suspected invalid URL signature
+                raise QueryReturnedForbiddenException("403 when accessing {}.".format(url))
+            if resp.status_code == 404:
+                # 404 not worth retrying.
+                raise QueryReturnedNotFoundException("404 when accessing {}.".format(url))
+            raise ConnectionException("HTTP error code {}.".format(resp.status_code))
+
     @property
     def root_rhx_gis(self) -> Optional[str]:
         """rhx_gis string returned in the / query."""

instaloader/nodeiterator.py

@@ -133,14 +133,15 @@ class NodeIterator(Iterator[T]):
             return item
         if self._data['page_info']['has_next_page']:
             query_response = self._query(self._data['page_info']['end_cursor'])
-            page_index, data = self._page_index, self._data
-            try:
-                self._page_index = 0
-                self._data = query_response
-            except KeyboardInterrupt:
-                self._page_index, self._data = page_index, data
-                raise
-            return self.__next__()
+            if self._data['edges'] != query_response['edges']:
+                page_index, data = self._page_index, self._data
+                try:
+                    self._page_index = 0
+                    self._data = query_response
+                except KeyboardInterrupt:
+                    self._page_index, self._data = page_index, data
+                    raise
+                return self.__next__()
         raise StopIteration()
 
     @property

instaloader/structures.py

@@ -5,7 +5,8 @@ from base64 import b64decode, b64encode
 from collections import namedtuple
 from datetime import datetime
 from pathlib import Path
-from typing import Any, Dict, Iterable, Iterator, List, Optional, Union
+from typing import Any, Dict, Iterable, Iterator, List, Optional, Tuple, Union
+from unicodedata import normalize
 
 from . import __version__
 from .exceptions import *
@@ -35,8 +36,8 @@ PostLocation.id.__doc__ = "ID number of location."
 PostLocation.name.__doc__ = "Location name."
 PostLocation.slug.__doc__ = "URL friendly variant of location name."
 PostLocation.has_public_page.__doc__ = "Whether location has a public page."
-PostLocation.lat.__doc__ = "Latitude (:class:`float`)."
-PostLocation.lng.__doc__ = "Longitude (:class:`float`)."
+PostLocation.lat.__doc__ = "Latitude (:class:`float` or None)."
+PostLocation.lng.__doc__ = "Longitude (:class:`float` or None)."
 
 
 class Post:
@@ -322,9 +323,9 @@ class Post:
     def caption(self) -> Optional[str]:
         """Caption."""
         if "edge_media_to_caption" in self._node and self._node["edge_media_to_caption"]["edges"]:
-            return self._node["edge_media_to_caption"]["edges"][0]["node"]["text"]
+            return normalize("NFC", self._node["edge_media_to_caption"]["edges"][0]["node"]["text"])
         elif "caption" in self._node:
-            return self._node["caption"]
+            return normalize("NFC", self._node["caption"])
         return None
 
     @property
@@ -346,7 +347,7 @@ class Post:
         # support Unicode and a word/beginning of string delimiter at the beginning to ensure
         # that no email addresses join the list of mentions.
         # http://blog.jstassen.com/2016/03/code-regex-for-instagram-username-and-hashtags/
-        mention_regex = re.compile(r"(?:^|\W|_)(?:@)(\w(?:(?:\w|(?:\.(?!\.))){0,28}(?:\w))?)")
+        mention_regex = re.compile(r"(?:^|\W|_)(?:@)(\w(?:(?:\w|(?:\.(?!\.))){0,28}(?:\w))?)", re.ASCII)
         return re.findall(mention_regex, self.caption.lower())
 
     @property
@@ -377,13 +378,35 @@ class Post:
     def video_url(self) -> Optional[str]:
         """URL of the video, or None."""
         if self.is_video:
+            version_urls = []
+            try:
+                version_urls.append(self._field('video_url'))
+            except (InstaloaderException, KeyError, IndexError) as err:
+                self._context.error(f"Warning: Unable to fetch video from graphql of {self}: {err}")
             if self._context.iphone_support and self._context.is_logged_in:
                 try:
-                    url = self._iphone_struct['video_versions'][0]['url']
-                    return url
+                    version_urls.extend(version['url'] for version in self._iphone_struct['video_versions'])
                 except (InstaloaderException, KeyError, IndexError) as err:
-                    self._context.error('{} Unable to fetch high quality video version of {}.'.format(err, self))
-            return self._field('video_url')
+                    self._context.error(f"Unable to fetch high-quality video version of {self}: {err}")
+            version_urls = list(dict.fromkeys(version_urls))
+            if len(version_urls) == 0:
+                return None
+            if len(version_urls) == 1:
+                return version_urls[0]
+            url_candidates: List[Tuple[int, str]] = []
+            for idx, version_url in enumerate(version_urls):
+                try:
+                    url_candidates.append((
+                        int(self._context.head(version_url, allow_redirects=True).headers.get('Content-Length', 0)),
+                        version_url
+                    ))
+                except (InstaloaderException, KeyError, IndexError) as err:
+                    self._context.error(f"Video URL candidate {idx+1}/{len(version_urls)} for {self}: {err}")
+            if not url_candidates:
+                # All candidates fail: Fallback to default URL and handle errors later at the actual download attempt
+                return version_urls[0]
+            url_candidates.sort()
+            return url_candidates[-1][1]
         return None
 
     @property
@@ -556,10 +579,10 @@ class Post:
             return None
         location_id = int(loc['id'])
         if any(k not in loc for k in ('name', 'slug', 'has_public_page', 'lat', 'lng')):
-            loc = self._context.get_json("explore/locations/{0}/".format(location_id),
-                                         params={'__a': 1})['graphql']['location']
+            loc.update(self._context.get_json("explore/locations/{0}/".format(location_id),
+                                              params={'__a': 1})['native_location_data']['location_info'])
         self._location = PostLocation(location_id, loc['name'], loc['slug'], loc['has_public_page'],
-                                      loc['lat'], loc['lng'])
+                                      loc.get('lat'), loc.get('lng'))
         return self._location
 
 
@@ -1110,7 +1133,35 @@ class StoryItem:
     def video_url(self) -> Optional[str]:
         """URL of the video, or None."""
         if self.is_video:
-            return self._node['video_resources'][-1]['src']
+            version_urls = []
+            try:
+                version_urls.append(self._node['video_resources'][-1]['src'])
+            except (InstaloaderException, KeyError, IndexError) as err:
+                self._context.error(f"Warning: Unable to fetch video from graphql of {self}: {err}")
+            if self._context.iphone_support and self._context.is_logged_in:
+                try:
+                    version_urls.extend(version['url'] for version in self._iphone_struct['video_versions'])
+                except (InstaloaderException, KeyError, IndexError) as err:
+                    self._context.error(f"Unable to fetch high-quality video version of {self}: {err}")
+            version_urls = list(dict.fromkeys(version_urls))
+            if len(version_urls) == 0:
+                return None
+            if len(version_urls) == 1:
+                return version_urls[0]
+            url_candidates: List[Tuple[int, str]] = []
+            for idx, version_url in enumerate(version_urls):
+                try:
+                    url_candidates.append((
+                        int(self._context.head(version_url, allow_redirects=True).headers.get('Content-Length', 0)),
+                        version_url
+                    ))
+                except (InstaloaderException, KeyError, IndexError) as err:
+                    self._context.error(f"Video URL candidate {idx+1}/{len(version_urls)} for {self}: {err}")
+            if not url_candidates:
+                # All candidates fail: Fallback to default URL and handle errors later at the actual download attempt
+                return version_urls[0]
+            url_candidates.sort()
+            return url_candidates[-1][1]
         return None
 
 
@@ -1509,7 +1560,7 @@ class TopSearchResults:
             place = location.get('place', {})
             slug = place.get('slug')
             loc = place.get('location', {})
-            yield PostLocation(int(loc['pk']), loc['name'], slug, None, loc['lat'], loc['lng'])
+            yield PostLocation(int(loc['pk']), loc['name'], slug, None, loc.get('lat'), loc.get('lng'))
 
     def get_hashtag_strings(self) -> Iterator[str]:
         """