Release of Version 4.8.5

Fixes #1379.

Revert "Make {Post,StoryItem}.date_utc timezone aware"
d864ce08ff.

Revert "Remove unnecessary conversion to timezone aware timestamp (#1372)"
555c86633c.

Revert "Timezone option for datetime style formatting (#1316)"
21365ec668.

.github/workflows/documentation.yml

@@ -0,0 +1,31 @@
+name: "Documentation"
+
+on: [push, pull_request]
+
+jobs:
+  docs:
+    name: "Documentation"
+    runs-on: ubuntu-latest
+    steps:
+      - name: "Checkout Instaloader Repository"
+        uses: actions/checkout@v2
+        with:
+          fetch-depth: 0  # needed for building docs
+      - name: "Setup Python"
+        uses: actions/setup-python@v2
+        with:
+          python-version: 3.8
+      - name: "Install Dependencies"
+        run: |
+          python -m pip install pipenv==2022.1.8
+          pipenv sync --dev
+      - name: "Build Documentation"
+        run: pipenv run make -C docs html SPHINXOPTS="-W -n"
+      - name: "Deploy Documentation"
+        if: github.event_name == 'push' && github.ref == 'refs/heads/master'
+        uses: JamesIves/github-pages-deploy-action@v4.2.5
+        with:
+          branch: master
+          repository-name: instaloader/instaloader.github.io
+          folder: docs/_build/html
+          token: ${{ secrets.PAGES_DEPLOYMENT_TOKEN }}

.github/workflows/lint.yml

@@ -8,24 +8,19 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        python-version: ["3.8", "3.9", "3.10"]
+        python-version: ["3.6", "3.7", "3.8", "3.9", "3.10"]
     steps:
       - name: Checkout Instaloader Repository
         uses: actions/checkout@v2
-        with:
-          fetch-depth: 0  # needed for building docs
       - name: Setup Python
         uses: actions/setup-python@v2
         with:
           python-version: ${{ matrix.python-version }}
       - name: Install Dependencies
         run: |
-          python -m pip install pipenv==2020.11.15
+          python -m pip install pipenv==2022.1.8
           pipenv sync --dev
       - name: PyLint
         run: pipenv run pylint instaloader
       - name: MyPy
         run: pipenv run mypy -m instaloader
-      - name: Build Documentation
-        if: matrix.python-version == '3.8'
-        run: pipenv run make -C docs html SPHINXOPTS="-W -n"

.github/workflows/packages.yml

@@ -0,0 +1,40 @@
+name: "Publish Packages"
+
+on:
+  push:
+    tags:
+      - v*
+
+jobs:
+  publish:
+    name: "Publish Package"
+    runs-on: ubuntu-latest
+    steps:
+      - name: "Checkout Repository"
+        uses: actions/checkout@v2
+      - name: "Setup Python"
+        uses: actions/setup-python@v2
+        with:
+          python-version: 3.8
+      - name: "Get the tagged version"
+        id: get_version
+        env:
+          GITHUB_REF: ${{ github.ref }}
+        run: echo ::set-output name=VERSION::${GITHUB_REF/refs\/tags\/v/}
+        shell: bash
+      - name: "Install Dependencies"
+        run: |
+          python -m pip install pipenv==2022.1.8
+          pipenv sync --dev
+      - name: "Generate Python Package"
+        run: python setup.py sdist
+      - name: "Publish Distribution to PyPI"
+        uses: pypa/gh-action-pypi-publish@master
+        with:
+          password: ${{ secrets.PYPI_PASSWORD }}
+      - name: "Publish Distribution to AUR"
+        run: deploy/arch/deploy.sh $VERSION_TAG
+        env:
+          VERSION_TAG: v${{ steps.get_version.outputs.VERSION }}
+          AUR_KEY: ${{ secrets.AUR_KEY }}
+          AUR_IV: ${{ secrets.AUR_IV }}

.travis.yml

@@ -1,35 +0,0 @@
-dist: xenial
-language: python
-python:
- - "3.6"
- - "3.7"
-install:
- - pipenv sync --dev
-script:
- - pylint instaloader
- - mypy -m instaloader
- - make -C docs html SPHINXOPTS="-W -n"
-addons:
-  ssh_known_hosts: aur.archlinux.org
-deploy:
-  - provider: pypi
-    user: aandergr
-    on:
-      tags: true
-      python: 3.6
-    password:
-      secure: KBsATtXNnEzqr0X2ALdyx2zaYj2EUfK8YjP4jEOilBELpJ4evHYXVgRkn9CKXGsD7LKbWmR3gz2rr8NZ6U42pHw0fpQHizz91MpJSKbNY7RZZh4UVWMDh81izeglchWAF+m3Wdpxpo/GBDbrZIbRQFhPgB+kjSzJo3uiCS+yVXtGG+6csJHzd0WkJjsVHz+XSdxyOiLy/qFF3nmY6ZrVrgw6WbX2YLbx9je6DxK3YRrnBqgMgH7Ui2xWWbgJCk3KKpQTMiMKnTa9mdeMJAd26YMCz1gG/X7ck6VNly/6bs5+Zq7/bU0dSolhPBhNthyccYdS0IeGPdPF3YqatblJ4fLGTfHn8oJs3/T11yIW9pvOT3UftEynPOUX+en2Qnb/3WfgwUfVts8i8DAYgEFLGEzbF5XMGy4gwAD/bZ0JW0pmgAG2PZsdOokAt+ydlNj/uxq1GFW2doZbul6k4t03p1mik6itYscvuhKoEm8Yw6wr1P9aDvMR6FFpSu4gDt2XrKmRmWkSXxpNeCfZujTjyAnISxLVQGSY7BG5K0Z2HbDtqqNz9nj4WgNGzZmngiRofCevWR+vPIDUDW7NCX03csfcFS9RRllBO/BrL7NVUV+Qp9dfTQYp+8SqYLWgHtDoXdgyMREjJaAmbx2u/TYTY8bADZ5F3vGUpv2j5hP3NTU=
-  - provider: pages
-    skip-cleanup: true
-    github-token: $GITHUB_TOKEN
-    local-dir: docs/_build/html
-    repo: instaloader/instaloader.github.io
-    target-branch: master
-    on:
-      branch: master
-      python: 3.6
-  - provider: script
-    script: deploy/arch/deploy.sh $TRAVIS_TAG
-    on:
-      tags: true
-      python: 3.6

deploy/arch/deploy.sh

@@ -8,10 +8,12 @@ echo $VERSION | grep -qe "[abc]" && exit 0
 cd $(dirname $0)
 
 # decrypt and add ssh key
-openssl aes-256-cbc -K $encrypted_354637631c28_key -iv $encrypted_354637631c28_iv -in id_rsa_AUR.enc -out /tmp/AUR_openssh -d
+openssl aes-256-cbc -K $AUR_KEY -iv $AUR_IV -in id_rsa_AUR.enc -out /tmp/AUR_openssh -d
 eval "$(ssh-agent -s)"
 chmod 600 /tmp/AUR_openssh
 ssh-add /tmp/AUR_openssh
+mkdir -p ~/.ssh
+ssh-keyscan -t rsa aur.archlinux.org >> ~/.ssh/known_hosts
 
 # clone and modify AUR repo
 git clone --depth 1 ssh://aur@aur.archlinux.org/instaloader.git

deploy/windows/create_exe.py

@@ -44,7 +44,7 @@ with open('__main__.py', 'w+') as f:
     f.writelines(lines)
 
 # install dependencies and invoke PyInstaller
-commands = ["pip install pipenv==2020.11.15",
+commands = ["pip install pipenv==2022.1.8",
             f"pipenv --python {sys.version_info.major}.{sys.version_info.minor} sync --dev",
             "pipenv run pyinstaller --log-level=DEBUG instaloader.spec"]
 

instaloader/__init__.py

@@ -1,7 +1,7 @@
 """Download pictures (or videos) along with their captions and other metadata from Instagram."""
 
 
-__version__ = '4.8.3'
+__version__ = '4.8.5'
 
 
 try:

instaloader/instaloader.py

@@ -312,15 +312,25 @@ class Instaloader:
                      filename_suffix: Optional[str] = None, _attempt: int = 1) -> bool:
         """Downloads and saves picture with given url under given directory with given timestamp.
         Returns true, if file was actually downloaded, i.e. updated."""
-        urlmatch = re.search('\\.[a-z0-9]*\\?', url)
-        file_extension = url[-3:] if urlmatch is None else urlmatch.group(0)[1:-1]
         if filename_suffix is not None:
             filename += '_' + filename_suffix
-        filename += '.' + file_extension
-        if os.path.isfile(filename):
+        urlmatch = re.search('\\.[a-z0-9]*\\?', url)
+        file_extension = url[-3:] if urlmatch is None else urlmatch.group(0)[1:-1]
+        nominal_filename = filename + '.' + file_extension
+        if os.path.isfile(nominal_filename):
+            self.context.log(nominal_filename + ' exists', end=' ', flush=True)
+            return False
+        resp = self.context.get_raw(url)
+        if 'Content-Type' in resp.headers and resp.headers['Content-Type']:
+            header_extension = '.' + resp.headers['Content-Type'].split(';')[0].split('/')[-1]
+            header_extension = header_extension.lower().replace('jpeg', 'jpg')
+            filename += header_extension
+        else:
+            filename = nominal_filename
+        if filename != nominal_filename and os.path.isfile(filename):
             self.context.log(filename + ' exists', end=' ', flush=True)
             return False
-        self.context.get_and_write_raw(url, filename)
+        self.context.write_raw(resp, filename)
         os.utime(filename, (datetime.now().timestamp(), mtime.timestamp()))
         return True
 
@@ -811,7 +821,7 @@ class Instaloader:
                 last_scraped = latest_stamps.get_last_story_timestamp(name)
                 scraped_timestamp = datetime.now().astimezone()
             for item in user_story.get_items():
-                if latest_stamps is not None and item.date_local <= last_scraped:
+                if latest_stamps is not None and item.date_utc.replace(tzinfo=timezone.utc) <= last_scraped:
                     break
                 if storyitem_filter is not None and not storyitem_filter(item):
                     self.context.log("<{} skipped>".format(item), flush=True)
@@ -1206,7 +1216,7 @@ class Instaloader:
         posts_takewhile: Optional[Callable[[Post], bool]] = None
         if latest_stamps is not None:
             last_scraped = latest_stamps.get_last_tagged_timestamp(profile.username)
-            posts_takewhile = lambda p: p.date_local > last_scraped
+            posts_takewhile = lambda p: p.date_utc.replace(tzinfo=timezone.utc) > last_scraped
         tagged_posts = profile.get_tagged_posts()
         self.posts_download_loop(tagged_posts,
                                  target if target
@@ -1214,7 +1224,7 @@ class Instaloader:
                                        _PostPathFormatter.sanitize_path(':tagged')),
                                  fast_update, post_filter, takewhile=posts_takewhile)
         if latest_stamps is not None and tagged_posts.first_item is not None:
-            latest_stamps.set_last_tagged_timestamp(profile.username, tagged_posts.first_item.date_local)
+            latest_stamps.set_last_tagged_timestamp(profile.username, tagged_posts.first_item.date_local.astimezone())
 
     def download_igtv(self, profile: Profile, fast_update: bool = False,
                       post_filter: Optional[Callable[[Post], bool]] = None,
@@ -1229,12 +1239,12 @@ class Instaloader:
         posts_takewhile: Optional[Callable[[Post], bool]] = None
         if latest_stamps is not None:
             last_scraped = latest_stamps.get_last_igtv_timestamp(profile.username)
-            posts_takewhile = lambda p: p.date_local > last_scraped
+            posts_takewhile = lambda p: p.date_utc.replace(tzinfo=timezone.utc) > last_scraped
         igtv_posts = profile.get_igtv_posts()
         self.posts_download_loop(igtv_posts, profile.username, fast_update, post_filter,
                                  total_count=profile.igtvcount, owner_profile=profile, takewhile=posts_takewhile)
         if latest_stamps is not None and igtv_posts.first_item is not None:
-            latest_stamps.set_last_igtv_timestamp(profile.username, igtv_posts.first_item.date_local)
+            latest_stamps.set_last_igtv_timestamp(profile.username, igtv_posts.first_item.date_local.astimezone())
 
     def _get_id_filename(self, profile_name: str) -> str:
         if ((format_string_contains_key(self.dirname_pattern, 'profile') or
@@ -1427,14 +1437,14 @@ class Instaloader:
                     if latest_stamps is not None:
                         # pylint:disable=cell-var-from-loop
                         last_scraped = latest_stamps.get_last_post_timestamp(profile_name)
-                        posts_takewhile = lambda p: p.date_local > last_scraped
+                        posts_takewhile = lambda p: p.date_utc.replace(tzinfo=timezone.utc) > last_scraped
                     posts_to_download = profile.get_posts()
                     self.posts_download_loop(posts_to_download, profile_name, fast_update, post_filter,
                                              total_count=profile.mediacount, owner_profile=profile,
                                              takewhile=posts_takewhile)
                     if latest_stamps is not None and posts_to_download.first_item is not None:
                         latest_stamps.set_last_post_timestamp(profile_name,
-                                                              posts_to_download.first_item.date_local)
+                                                              posts_to_download.first_item.date_local.astimezone())
 
         if stories and profiles:
             with self.context.error_catcher("Download stories"):

instaloader/structures.py

@@ -3,9 +3,10 @@ import lzma
 import re
 from base64 import b64decode, b64encode
 from collections import namedtuple
-from datetime import datetime, timezone
+from datetime import datetime
 from pathlib import Path
 from typing import Any, Dict, Iterable, Iterator, List, Optional, Tuple, Union
+from unicodedata import normalize
 
 from . import __version__
 from .exceptions import *
@@ -225,12 +226,16 @@ class Post:
     @property
     def date_local(self) -> datetime:
         """Timestamp when the post was created (local time zone)."""
-        return datetime.fromtimestamp(self.get_timestamp_date_created()).astimezone()
+        return datetime.fromtimestamp(self._node["date"]
+                                      if "date" in self._node
+                                      else self._node["taken_at_timestamp"])
 
     @property
     def date_utc(self) -> datetime:
         """Timestamp when the post was created (UTC)."""
-        return datetime.utcfromtimestamp(self.get_timestamp_date_created()).replace(tzinfo=timezone.utc)
+        return datetime.utcfromtimestamp(self._node["date"]
+                                         if "date" in self._node
+                                         else self._node["taken_at_timestamp"])
 
     @property
     def date(self) -> datetime:
@@ -271,12 +276,6 @@ class Post:
             return len(edges)
         return 1
 
-    def get_timestamp_date_created(self) -> float:
-        """Timestamp when the post was created"""
-        return (self._node["date"]
-                if "date" in self._node
-                else self._node["taken_at_timestamp"])
-
     def get_is_videos(self) -> List[bool]:
         """
         Return a list containing the ``is_video`` property for each media in the post.
@@ -324,9 +323,9 @@ class Post:
     def caption(self) -> Optional[str]:
         """Caption."""
         if "edge_media_to_caption" in self._node and self._node["edge_media_to_caption"]["edges"]:
-            return self._node["edge_media_to_caption"]["edges"][0]["node"]["text"]
+            return normalize("NFC", self._node["edge_media_to_caption"]["edges"][0]["node"]["text"])
         elif "caption" in self._node:
-            return self._node["caption"]
+            return normalize("NFC", self._node["caption"])
         return None
 
     @property
@@ -348,7 +347,7 @@ class Post:
         # support Unicode and a word/beginning of string delimiter at the beginning to ensure
         # that no email addresses join the list of mentions.
         # http://blog.jstassen.com/2016/03/code-regex-for-instagram-username-and-hashtags/
-        mention_regex = re.compile(r"(?:^|\W|_)(?:@)(\w(?:(?:\w|(?:\.(?!\.))){0,28}(?:\w))?)")
+        mention_regex = re.compile(r"(?:^|\W|_)(?:@)(\w(?:(?:\w|(?:\.(?!\.))){0,28}(?:\w))?)", re.ASCII)
         return re.findall(mention_regex, self.caption.lower())
 
     @property
@@ -1081,12 +1080,12 @@ class StoryItem:
     @property
     def date_local(self) -> datetime:
         """Timestamp when the StoryItem was created (local time zone)."""
-        return datetime.fromtimestamp(self._node['taken_at_timestamp']).astimezone()
+        return datetime.fromtimestamp(self._node['taken_at_timestamp'])
 
     @property
     def date_utc(self) -> datetime:
         """Timestamp when the StoryItem was created (UTC)."""
-        return datetime.utcfromtimestamp(self._node['taken_at_timestamp']).replace(tzinfo=timezone.utc)
+        return datetime.utcfromtimestamp(self._node['taken_at_timestamp'])
 
     @property
     def date(self) -> datetime: