mirror of
https://github.com/instaloader/instaloader.git
synced 2026-10-16 21:52:17 -07:00
Compare commits
1
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
 Alexander Graf
|
a9cf78655c |
@@ -6,15 +6,13 @@ labels: "feature suggestion"
|
||||
---
|
||||
|
||||
**Provide us a use case of the feature**
|
||||
How could the user invoke the new function? Which problem would it solve?
|
||||
If new information is obtained, how would it be further processed?
|
||||
How could the user invoke the new function? Which problem would it solve? If new information is obtained, how would it be further processed?
|
||||
|
||||
**Describe the solution you'd like**
|
||||
A clear and concise description of what you want to happen.
|
||||
|
||||
**Describe alternatives you've considered**
|
||||
A clear and concise description of any alternative solutions or features you've considered.
|
||||
How do you achieve the here-described goal without the feature you are requesting?
|
||||
|
||||
**If the feature request is accepted, would you be willing to submit a pull request?**
|
||||
Yes / No
|
||||
@@ -23,9 +21,4 @@ Yes / No
|
||||
**Additional context**
|
||||
Add any other context about the feature request here.
|
||||
|
||||
<!--
|
||||
Also see https://instaloader.github.io/contributing.html for how to suggest a feature.
|
||||
|
||||
Please understand that a feature suggestion issue that lacks the required information cannot be
|
||||
processed.
|
||||
-->
|
||||
<!-- please also see https://instaloader.github.io/contributing.html for how to suggest a feature -->
|
||||
|
||||
+1
-2
@@ -1,2 +1 @@
|
||||
github: [aandergr]
|
||||
custom: "https://instaloader.github.io/sponsors.html"
|
||||
custom: "https://instaloader.github.io/contributing.html#donations"
|
||||
|
||||
@@ -1,31 +0,0 @@
|
||||
name: "Documentation"
|
||||
|
||||
on: [push, pull_request]
|
||||
|
||||
jobs:
|
||||
docs:
|
||||
name: "Documentation"
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: "Checkout Instaloader Repository"
|
||||
uses: actions/checkout@v2
|
||||
with:
|
||||
fetch-depth: 0 # needed for building docs
|
||||
- name: "Setup Python"
|
||||
uses: actions/setup-python@v2
|
||||
with:
|
||||
python-version: 3.12
|
||||
- name: "Install Dependencies"
|
||||
run: |
|
||||
python -m pip install pipenv==2023.12.1
|
||||
pipenv --python `python --version | grep -Eo '3\.[0-9]+'` sync --dev
|
||||
- name: "Build Documentation"
|
||||
run: pipenv --python `python --version | grep -Eo '3\.[0-9]+'` run make -C docs html SPHINXOPTS="-W -n"
|
||||
- name: "Deploy Documentation"
|
||||
if: github.event_name == 'push' && github.ref == 'refs/heads/master'
|
||||
uses: JamesIves/github-pages-deploy-action@v4.2.5
|
||||
with:
|
||||
branch: master
|
||||
repository-name: instaloader/instaloader.github.io
|
||||
folder: docs/_build/html
|
||||
token: ${{ secrets.PAGES_DEPLOYMENT_TOKEN }}
|
||||
@@ -7,21 +7,24 @@ jobs:
|
||||
name: PyLint and MyPy
|
||||
runs-on: ubuntu-latest
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"]
|
||||
python-version: [3.8]
|
||||
steps:
|
||||
- name: Checkout Instaloader Repository
|
||||
uses: actions/checkout@v2
|
||||
with:
|
||||
fetch-depth: 0 # needed for building docs
|
||||
- name: Setup Python
|
||||
uses: actions/setup-python@v2
|
||||
with:
|
||||
python-version: ${{ matrix.python-version }}
|
||||
- name: Install Dependencies
|
||||
run: |
|
||||
python -m pip install pipenv==2023.12.1
|
||||
pipenv --python `python --version | grep -Eo '3\.[0-9]+'` sync --dev
|
||||
python -m pip install pipenv==2020.6.2
|
||||
pipenv sync --dev
|
||||
- name: PyLint
|
||||
run: pipenv --python `python --version | grep -Eo '3\.[0-9]+'` run pylint instaloader
|
||||
run: pipenv run pylint instaloader
|
||||
- name: MyPy
|
||||
run: pipenv --python `python --version | grep -Eo '3\.[0-9]+'` run mypy -m instaloader
|
||||
run: pipenv run mypy -m instaloader
|
||||
- name: Build Documentation
|
||||
run: pipenv run make -C docs html SPHINXOPTS="-W -n"
|
||||
|
||||
@@ -1,38 +0,0 @@
|
||||
name: "Publish Packages"
|
||||
|
||||
on:
|
||||
push:
|
||||
tags:
|
||||
- v*
|
||||
|
||||
jobs:
|
||||
publish:
|
||||
name: "Publish Package"
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: "Checkout Repository"
|
||||
uses: actions/checkout@v2
|
||||
- name: "Setup Python"
|
||||
uses: actions/setup-python@v2
|
||||
with:
|
||||
python-version: 3.12
|
||||
- name: "Get the tagged version"
|
||||
id: get_version
|
||||
env:
|
||||
GITHUB_REF: ${{ github.ref }}
|
||||
run: echo ::set-output name=VERSION::${GITHUB_REF/refs\/tags\/v/}
|
||||
shell: bash
|
||||
- name: "Install Dependencies"
|
||||
run: python -m pip install setuptools
|
||||
- name: "Generate Python Package"
|
||||
run: python setup.py sdist
|
||||
- name: "Publish Distribution to PyPI"
|
||||
uses: pypa/gh-action-pypi-publish@master
|
||||
with:
|
||||
password: ${{ secrets.PYPI_PASSWORD }}
|
||||
- name: "Publish Distribution to AUR"
|
||||
run: deploy/arch/deploy.sh $VERSION_TAG
|
||||
env:
|
||||
VERSION_TAG: v${{ steps.get_version.outputs.VERSION }}
|
||||
AUR_KEY: ${{ secrets.AUR_KEY }}
|
||||
AUR_IV: ${{ secrets.AUR_IV }}
|
||||
@@ -18,7 +18,7 @@ jobs:
|
||||
stale-issue-message: 'There has been no activity on this question for an extended period of time. This issue will be closed after further 14 days of inactivity.'
|
||||
stale-issue-label: 'stale'
|
||||
exempt-issue-labels: 'leave open'
|
||||
days-before-stale: 21
|
||||
days-before-stale: 15
|
||||
days-before-close: -1
|
||||
remove-stale-when-updated: false
|
||||
- uses: actions/stale@v1
|
||||
@@ -30,5 +30,5 @@ jobs:
|
||||
stale-pr-label: 'stale'
|
||||
exempt-issue-label: 'leave open'
|
||||
exempt-pr-label: 'leave open'
|
||||
days-before-stale: 189
|
||||
days-before-stale: 90
|
||||
days-before-close: 14
|
||||
|
||||
@@ -10,9 +10,9 @@ jobs:
|
||||
- name: Checkout Instaloader repository
|
||||
uses: actions/checkout@v2
|
||||
- name: Setup Python
|
||||
uses: actions/setup-python@v2
|
||||
uses: actions/setup-python@v1
|
||||
with:
|
||||
python-version: "3.12"
|
||||
python-version: 3.8
|
||||
architecture: x64
|
||||
- name: Get the tagged version
|
||||
id: get_version
|
||||
|
||||
+36
@@ -0,0 +1,36 @@
|
||||
dist: xenial
|
||||
language: python
|
||||
python:
|
||||
- "3.5"
|
||||
- "3.6"
|
||||
- "3.7"
|
||||
install:
|
||||
- pipenv sync --dev
|
||||
script:
|
||||
- pylint instaloader
|
||||
- mypy -m instaloader
|
||||
- make -C docs html SPHINXOPTS="-W -n"
|
||||
addons:
|
||||
ssh_known_hosts: aur.archlinux.org
|
||||
deploy:
|
||||
- provider: pypi
|
||||
user: aandergr
|
||||
on:
|
||||
tags: true
|
||||
python: 3.6
|
||||
password:
|
||||
secure: KBsATtXNnEzqr0X2ALdyx2zaYj2EUfK8YjP4jEOilBELpJ4evHYXVgRkn9CKXGsD7LKbWmR3gz2rr8NZ6U42pHw0fpQHizz91MpJSKbNY7RZZh4UVWMDh81izeglchWAF+m3Wdpxpo/GBDbrZIbRQFhPgB+kjSzJo3uiCS+yVXtGG+6csJHzd0WkJjsVHz+XSdxyOiLy/qFF3nmY6ZrVrgw6WbX2YLbx9je6DxK3YRrnBqgMgH7Ui2xWWbgJCk3KKpQTMiMKnTa9mdeMJAd26YMCz1gG/X7ck6VNly/6bs5+Zq7/bU0dSolhPBhNthyccYdS0IeGPdPF3YqatblJ4fLGTfHn8oJs3/T11yIW9pvOT3UftEynPOUX+en2Qnb/3WfgwUfVts8i8DAYgEFLGEzbF5XMGy4gwAD/bZ0JW0pmgAG2PZsdOokAt+ydlNj/uxq1GFW2doZbul6k4t03p1mik6itYscvuhKoEm8Yw6wr1P9aDvMR6FFpSu4gDt2XrKmRmWkSXxpNeCfZujTjyAnISxLVQGSY7BG5K0Z2HbDtqqNz9nj4WgNGzZmngiRofCevWR+vPIDUDW7NCX03csfcFS9RRllBO/BrL7NVUV+Qp9dfTQYp+8SqYLWgHtDoXdgyMREjJaAmbx2u/TYTY8bADZ5F3vGUpv2j5hP3NTU=
|
||||
- provider: pages
|
||||
skip-cleanup: true
|
||||
github-token: $GITHUB_TOKEN
|
||||
local-dir: docs/_build/html
|
||||
repo: instaloader/instaloader.github.io
|
||||
target-branch: master
|
||||
on:
|
||||
branch: master
|
||||
python: 3.6
|
||||
- provider: script
|
||||
script: deploy/arch/deploy.sh $TRAVIS_TAG
|
||||
on:
|
||||
tags: true
|
||||
python: 3.6
|
||||
@@ -6,5 +6,4 @@ Instaloader is written by
|
||||
- Alexander Graf (@aandergr)
|
||||
- André Koch-Kramer (@Thammus)
|
||||
- Lars Lindqvist (@e5150)
|
||||
- Eduardo M Kalinowksi (@ekalin)
|
||||
- ... and many more, see https://github.com/instaloader/instaloader/graphs/contributors
|
||||
|
||||
@@ -6,17 +6,11 @@ verify_ssl = true
|
||||
[dev-packages]
|
||||
pylint = "*"
|
||||
mypy = "*"
|
||||
sphinx = "*"
|
||||
sphinx = "~=2.3"
|
||||
pyinstaller = "*"
|
||||
pefile = "*"
|
||||
pywin32-ctypes = "*"
|
||||
psutil = "*"
|
||||
types-requests = "*"
|
||||
sphinx-autodoc-typehints = "*"
|
||||
sphinxcontrib-jquery = "*"
|
||||
tomli = "*" # Needed for Pylint for "python_version < '3.11'"
|
||||
dill = "*" # Needed for Pylint for "python_version < '3.11'"
|
||||
pywin32-ctypes = "*" # Needed for pyinstaller on Windows
|
||||
pefile = "*" # Needed for pyinstaller on Windows
|
||||
|
||||
[packages]
|
||||
requests = "*"
|
||||
browser-cookie3 = "*"
|
||||
|
||||
Generated
+249
-597
File diff suppressed because it is too large
Load Diff
+25
-46
@@ -2,7 +2,11 @@
|
||||
|
||||
.. badges-start
|
||||
|
||||
|pypi| |pyversion| |license| |aur| |contributors| |downloads|
|
||||
|travis| |pypi| |pyversion| |license| |aur| |contributors| |downloads|
|
||||
|
||||
.. |travis| image:: https://img.shields.io/travis/instaloader/instaloader/master.svg
|
||||
:alt: Travis-CI Build Status
|
||||
:target: https://travis-ci.org/instaloader/instaloader
|
||||
|
||||
.. |pypi| image:: https://img.shields.io/pypi/v/instaloader.svg
|
||||
:alt: Instaloader PyPI Project Page
|
||||
@@ -25,7 +29,7 @@
|
||||
|
||||
.. |downloads| image:: https://pepy.tech/badge/instaloader/month
|
||||
:alt: PyPI Download Count
|
||||
:target: https://pepy.tech/project/instaloader
|
||||
:target: https://pepy.tech/project/instaloader/month
|
||||
|
||||
.. badges-end
|
||||
|
||||
@@ -80,19 +84,7 @@ To later **update your local copy** of that profiles, you may run
|
||||
instaloader --fast-update profile [profile ...]
|
||||
|
||||
If ``--fast-update`` is given, Instaloader stops when arriving at the
|
||||
first already-downloaded picture.
|
||||
|
||||
Alternatively, you can use ``--latest-stamps`` to have Instaloader store
|
||||
the time each profile was last downloaded and only download newer media:
|
||||
|
||||
::
|
||||
|
||||
instaloader --latest-stamps -- profile [profile ...]
|
||||
|
||||
With this option it's possible to move or delete downloaded media and still keep
|
||||
the archive updated.
|
||||
|
||||
When updating profiles, Instaloader
|
||||
first already-downloaded picture. When updating profiles, Instaloader
|
||||
automatically **detects profile name changes** and renames the target directory
|
||||
accordingly.
|
||||
|
||||
@@ -110,37 +102,6 @@ already have a valid session cookie file.
|
||||
|
||||
`Instaloader Documentation <https://instaloader.github.io/basic-usage.html>`__
|
||||
|
||||
Contributing
|
||||
------------
|
||||
|
||||
As an open source project, Instaloader heavily depends on the contributions from
|
||||
its community. See
|
||||
`contributing <https://instaloader.github.io/contributing.html>`__
|
||||
for how you may help Instaloader to become an even greater tool.
|
||||
|
||||
Supporters
|
||||
----------
|
||||
|
||||
.. current-sponsors-start
|
||||
|
||||
| Instaloader is proudly sponsored by
|
||||
| `@rocketapi-io <https://github.com/rocketapi-io>`__
|
||||
|
||||
See `Alex' GitHub Sponsors <https://github.com/sponsors/aandergr>`__ page for
|
||||
how you can sponsor the development of Instaloader!
|
||||
|
||||
.. current-sponsors-end
|
||||
|
||||
It is a pleasure for us to share our Instaloader to the world, and we are proud
|
||||
to have attracted such an active and motivating community, with so many users
|
||||
who share their suggestions and ideas with us. Buying a community-sponsored beer
|
||||
or coffee from time to time is very likely to further raise our passion for the
|
||||
development of Instaloader.
|
||||
|
||||
| For Donations, we provide GitHub Sponsors page, a PayPal.Me link and a Bitcoin address.
|
||||
| GitHub Sponsors: `Sponsor @aandergr on GitHub Sponsors <https://github.com/sponsors/aandergr>`__
|
||||
| PayPal: `PayPal.me/aandergr <https://www.paypal.me/aandergr>`__
|
||||
| BTC: 1Nst4LoadeYzrKjJ1DX9CpbLXBYE9RKLwY
|
||||
|
||||
Disclaimer
|
||||
----------
|
||||
@@ -153,3 +114,21 @@ subsidiaries. This is an independent and unofficial project. Use at your own ris
|
||||
Instaloader is licensed under an MIT license. Refer to ``LICENSE`` file for more information.
|
||||
|
||||
.. disclaimer-end
|
||||
|
||||
Contributing
|
||||
------------
|
||||
|
||||
As an open source project, Instaloader heavily depends on the contributions from
|
||||
its community. See
|
||||
`contributing <https://instaloader.github.io/contributing.html>`__
|
||||
for how you may help Instaloader to become an even greater tool.
|
||||
|
||||
It is a pleasure for us to share our Instaloader to the world, and we are proud
|
||||
to have attracted such an active and motivating community, with so many users
|
||||
who share their suggestions and ideas with us. Buying a community-sponsored beer
|
||||
or coffee from time to time is very likely to further raise our passion for the
|
||||
development of Instaloader.
|
||||
|
||||
| For Donations, we provide a PayPal.Me link and a Bitcoin address.
|
||||
| PayPal: `PayPal.me/aandergr <https://www.paypal.me/aandergr>`__
|
||||
| BTC: 1Nst4LoadeYzrKjJ1DX9CpbLXBYE9RKLwY
|
||||
|
||||
@@ -6,7 +6,7 @@ pkgbase = instaloader
|
||||
arch = any
|
||||
license = MIT
|
||||
makedepends = python-setuptools
|
||||
depends = python>=3.8
|
||||
depends = python>=3.5
|
||||
depends = python-requests>=2.4
|
||||
options = !emptydirs
|
||||
source = instaloader-{{version}}.tar.gz::https://codeload.github.com/instaloader/instaloader/tar.gz/v{{version}}
|
||||
|
||||
@@ -8,7 +8,7 @@ arch=('any')
|
||||
url="https://instaloader.github.io/"
|
||||
license=('MIT')
|
||||
groups=()
|
||||
depends=('python>=3.8' 'python-requests>=2.4')
|
||||
depends=('python>=3.5' 'python-requests>=2.4')
|
||||
makedepends=('python-setuptools')
|
||||
options=('!emptydirs')
|
||||
source=($pkgname-$pkgver.tar.gz::https://codeload.github.com/instaloader/instaloader/tar.gz/v$pkgver)
|
||||
|
||||
@@ -8,12 +8,10 @@ echo $VERSION | grep -qe "[abc]" && exit 0
|
||||
cd $(dirname $0)
|
||||
|
||||
# decrypt and add ssh key
|
||||
openssl aes-256-cbc -K $AUR_KEY -iv $AUR_IV -in id_rsa_AUR.enc -out /tmp/AUR_openssh -d
|
||||
openssl aes-256-cbc -K $encrypted_354637631c28_key -iv $encrypted_354637631c28_iv -in id_rsa_AUR.enc -out /tmp/AUR_openssh -d
|
||||
eval "$(ssh-agent -s)"
|
||||
chmod 600 /tmp/AUR_openssh
|
||||
ssh-add /tmp/AUR_openssh
|
||||
mkdir -p ~/.ssh
|
||||
ssh-keyscan -t rsa aur.archlinux.org >> ~/.ssh/known_hosts
|
||||
|
||||
# clone and modify AUR repo
|
||||
git clone --depth 1 ssh://aur@aur.archlinux.org/instaloader.git
|
||||
|
||||
Binary file not shown.
@@ -44,8 +44,8 @@ with open('__main__.py', 'w+') as f:
|
||||
f.writelines(lines)
|
||||
|
||||
# install dependencies and invoke PyInstaller
|
||||
commands = ["pip install pipenv==2023.12.1",
|
||||
f"pipenv --python {sys.version_info.major}.{sys.version_info.minor} sync --dev",
|
||||
commands = ["pip install pipenv==2020.6.2",
|
||||
"pipenv sync --dev",
|
||||
"pipenv run pyinstaller --log-level=DEBUG instaloader.spec"]
|
||||
|
||||
for command in commands:
|
||||
|
||||
Vendored
+1
-1
File diff suppressed because one or more lines are too long
Vendored
+1
-1
@@ -21,7 +21,7 @@
|
||||
{% block header %}
|
||||
<nav class="navbar navbar-light navbar-expand bg-light border-bottom">
|
||||
<a class="navbar-brand" href="{{ pathto(master_doc) }}">
|
||||
<img src="{{ pathto('_static/logo.png', 1) }}" width="30" height="30" class="d-inline-block align-top" alt="logo">
|
||||
<img src="{{ pathto('_static/' + logo, 1) }}" width="30" height="30" class="d-inline-block align-top" alt="logo">
|
||||
<span class="ml-3">Instaloader</span>
|
||||
</a>
|
||||
<ul class="navbar-nav mr-auto">
|
||||
|
||||
+7
-31
@@ -39,16 +39,6 @@ To later **update your local copy** of that profiles, you may run
|
||||
If :option:`--fast-update` is given, Instaloader stops when arriving at the
|
||||
first already-downloaded picture.
|
||||
|
||||
Alternatively, you can use :option:`--latest-stamps` to have Instaloader store
|
||||
the time each profile was last downloaded and only download newer media:
|
||||
|
||||
::
|
||||
|
||||
instaloader --latest-stamps -- profile [profile ...]
|
||||
|
||||
With this option it's possible to move or delete downloaded media and still keep
|
||||
the archive updated.
|
||||
|
||||
When updating profiles, Instaloader
|
||||
automatically **detects profile name changes** and renames the target directory
|
||||
accordingly.
|
||||
@@ -159,37 +149,27 @@ target. The default is ``--dirname-pattern={target}``. In the dirname
|
||||
pattern, the token ``{target}`` is replaced by the target name, and
|
||||
``{profile}`` is replaced by the owner of the post which is downloaded.
|
||||
|
||||
:option:`--filename-pattern` configures the path of the post and story's files relative
|
||||
:option:`--filename-pattern` configures the path of the post's files relative
|
||||
to the target directory that is specified with :option:`--dirname-pattern`.
|
||||
The default is ``--filename-pattern={date_utc}_UTC``.
|
||||
The tokens ``{target}`` and ``{profile}`` are replaced like in the
|
||||
dirname pattern.
|
||||
|
||||
:option:`--title-pattern` is similar to :option:`--filename-pattern`, but for profile
|
||||
pics, hashtag profile pics, and highlight covers. The default is
|
||||
``{date_utc}_UTC_{typename}`` if :option:`--dirname-pattern` contains ``{target}`` or
|
||||
``{profile}``, or ``{target}_{date_utc}_UTC_{typename}`` if it does not. Some tokens
|
||||
are not supported for this option, see below for details.
|
||||
|
||||
The following tokens are defined for usage with
|
||||
:option:`--filename-pattern` and :option:`--title-pattern`:
|
||||
dirname pattern. The following tokens are defined for usage with
|
||||
:option:`--filename-pattern`:
|
||||
|
||||
- ``{target}``
|
||||
Target name (as given in Instaloader command line)
|
||||
|
||||
- ``{profile}`` (same as ``{owner_username}``)
|
||||
Owner of the Post / StoryItem / ProfilePic. For hashtag profile pics and
|
||||
highlight covers, equivalent to ``{target}``.
|
||||
Owner of the Post / StoryItem.
|
||||
|
||||
- ``{owner_id}``
|
||||
Unique integer ID of owner profile. For hashtag profile pics, equivalent to
|
||||
``{target}``.
|
||||
Unique integer ID of owner profile.
|
||||
|
||||
- ``{shortcode}``
|
||||
Shortcode (identifier string). Not available for :option:`--title-pattern`.
|
||||
Shortcode (identifier string).
|
||||
|
||||
- ``{mediaid}``
|
||||
Integer representation of shortcode. Not available for :option:`--title-pattern`.
|
||||
Integer representation of shortcode.
|
||||
|
||||
- ``{filename}``
|
||||
Instagram's internal filename.
|
||||
@@ -202,10 +182,6 @@ The following tokens are defined for usage with
|
||||
|
||||
{date_utc:%Y-%m-%d_%H-%M-%S}
|
||||
|
||||
- ``{typename}``
|
||||
Type of media being saved, such as GraphImage, GraphStoryVideo, profile_pic,
|
||||
etc.
|
||||
|
||||
For example, encode the poster's profile name in the filenames with::
|
||||
|
||||
instaloader --filename-pattern={date_utc}_UTC_{profile} "#hashtag"
|
||||
|
||||
+9
-89
@@ -100,6 +100,7 @@ What to Download of each Post
|
||||
|
||||
Do not xz compress JSON files, rather create pretty formatted JSONs.
|
||||
|
||||
|
||||
What to Download of each Profile
|
||||
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
|
||||
@@ -146,23 +147,6 @@ Which Posts to Download
|
||||
This flag is recommended when you use Instaloader to update your personal
|
||||
Instagram archive.
|
||||
|
||||
.. option:: --latest-stamps [STAMPSFILE]
|
||||
|
||||
Works similarly to :option:`--fast-update`, but instead of relying on already
|
||||
downloaded media, the time each profile was downloaded is stored, and only
|
||||
media newer than the last download is fetched. This allows updating your
|
||||
personal Instagram archive while emptying the target directories.
|
||||
|
||||
Only works for media associated with a specific profile, and that is returned
|
||||
in chronological order: profile posts, profile stories, profile IGTV posts
|
||||
and profile tagged posts.
|
||||
|
||||
By default, the information is stored in
|
||||
``~/.config/instaloader/latest-stamps.ini``, but you can specify an
|
||||
alternative location.
|
||||
|
||||
.. versionadded:: 4.8
|
||||
|
||||
.. option:: --post-filter filter, --only-if filter
|
||||
|
||||
Expression that, if given, must evaluate to True for each post to be
|
||||
@@ -180,8 +164,8 @@ Which Posts to Download
|
||||
|
||||
.. option:: --count COUNT, -c
|
||||
|
||||
Do not attempt to download more than COUNT posts. Applies to
|
||||
``#hashtag``, ``%location_id``, ``:feed``, and ``:saved``.
|
||||
Do not attempt to download more than COUNT posts. Applies only to
|
||||
``#hashtag``, ``%location id``, and ``:feed``.
|
||||
|
||||
|
||||
Login (Download Private Profiles)
|
||||
@@ -196,25 +180,6 @@ Instaloader to login.
|
||||
|
||||
Login name (profile name) for your Instagram account.
|
||||
|
||||
.. option:: --load-cookies BROWSER-NAME, -b BROWSER-NAME
|
||||
|
||||
Use Instagram cookie in your browser to login.
|
||||
This feature requires the browser_cookie3 library.
|
||||
Compatible with :option:`--cookiefile` if you want to load cookies from browser profiles.
|
||||
Incompatible with :option:`--login` due to potential username mismatch between user input and browser login.
|
||||
Supported browsers: Chrome, Firefox, Edge, Opera, Safari, Brave.
|
||||
|
||||
After loading the cookies run the :option:`--login` option as it is required to download high quality media
|
||||
and to make full use of Instaloader's features.
|
||||
|
||||
.. versionadded:: 4.11
|
||||
|
||||
.. option:: --cookiefile COOKIE-FILE, -B COOKIE-FILE
|
||||
|
||||
Cookie file path of a browser profile to load cookies from.
|
||||
|
||||
.. versionadded:: 4.11
|
||||
|
||||
.. option:: --sessionfile SESSIONFILE, -f SESSIONFILE
|
||||
|
||||
Path for loading and storing session key file. Defaults to
|
||||
@@ -241,30 +206,12 @@ How to Download
|
||||
|
||||
.. option:: --filename-pattern FILENAME_PATTERN
|
||||
|
||||
Prefix of filenames for posts and stories, relative to the directory given with
|
||||
Prefix of filenames, relative to the directory given with
|
||||
:option:`--dirname-pattern`. ``{profile}`` is replaced by the profile name,
|
||||
``{target}`` is replaced by the target you specified, i.e. either ``:feed``,
|
||||
``#hashtag`` or the profile name. Defaults to ``{date_utc}_UTC``.
|
||||
See :ref:`filename-specification` for a list of supported tokens.
|
||||
|
||||
.. option:: --title-pattern TITLE_PATTERN
|
||||
|
||||
Prefix of filenames for profile pics, hashtag profile pics, and highlight
|
||||
covers, relative to the directory given with :option:`--dirname-pattern`.
|
||||
Defaults to ``{date_utc}_UTC_{typename}`` if :option:`--dirname-pattern`
|
||||
contains ``{target}`` or ``{profile}``, otherwise defaults to
|
||||
``{target}_{date_utc}_UTC_{typename}``.
|
||||
See :ref:`filename-specification` for a list of supported tokens.
|
||||
|
||||
.. versionadded:: 4.8
|
||||
|
||||
.. option:: --sanitize-paths
|
||||
|
||||
Force sanitization of paths so that the resulting file and directory names
|
||||
are valid on both Windows and Unix.
|
||||
|
||||
.. versionadded:: 4.9
|
||||
|
||||
.. option:: --resume-prefix prefix
|
||||
|
||||
For many targets, Instaloader is capable of resuming a previously-aborted
|
||||
@@ -273,21 +220,12 @@ How to Download
|
||||
used to save the information to resume an interrupted download. The default
|
||||
prefix is ``iterator``.
|
||||
|
||||
Resuming an interrupted download is supported for the following targets:
|
||||
- Profile posts,
|
||||
- Profile IGTV posts (:option:`--igtv`),
|
||||
- Profile tagged posts (:option:`--tagged`),
|
||||
- Saved posts (``:saved``),
|
||||
- Hashtags.
|
||||
|
||||
This feature is enabled by default for targets where it is supported;
|
||||
:option:`--resume-prefix` only changes the name of the iterator files.
|
||||
|
||||
To turn this feature off, use :option:`--no-resume`.
|
||||
|
||||
Resuming an interrupted download is supported for most, but not all targets.
|
||||
JSON files with resume information are always compressed, regardless of
|
||||
:option:`--no-compress-json`.
|
||||
|
||||
This feature is turned off entirely with :option:`--no-resume`.
|
||||
|
||||
.. versionadded:: 4.5
|
||||
|
||||
.. option:: --no-resume
|
||||
@@ -300,7 +238,7 @@ How to Download
|
||||
.. option:: --user-agent USER_AGENT
|
||||
|
||||
User Agent to use for HTTP requests. Per default, Instaloader pretends being
|
||||
Chrome/120 on Linux.
|
||||
Chrome/51.
|
||||
|
||||
.. option:: --max-connection-attempts N
|
||||
|
||||
@@ -317,23 +255,6 @@ How to Download
|
||||
.. versionchanged:: 4.6
|
||||
Enabled this option by default with a timeout of 300 seconds.
|
||||
|
||||
.. option:: --abort-on STATUS_CODE_LIST
|
||||
|
||||
Comma-separated list of HTTP status codes that cause Instaloader to abort,
|
||||
bypassing all retry logic.
|
||||
|
||||
For example, with ``--abort-on=302,400,429``, Instaloader will stop if a
|
||||
request is responded with a 302 redirect, a Bad Request error, or a Too Many
|
||||
Requests error.
|
||||
|
||||
.. versionadded:: 4.7
|
||||
|
||||
.. option:: --no-iphone
|
||||
|
||||
Do not attempt to download iPhone version of images and videos.
|
||||
|
||||
.. versionadded:: 4.8
|
||||
|
||||
Miscellaneous Options
|
||||
^^^^^^^^^^^^^^^^^^^^^
|
||||
|
||||
@@ -348,13 +269,12 @@ Miscellaneous Options
|
||||
Read arguments from file `args.txt`, a shortcut to provide arguments from
|
||||
file rather than command-line. This provides a convenient way to hide login
|
||||
info from CLI, and can also be used to simplify management of long arguments.
|
||||
You can provide more than one file at once, e.g.: ``+args1.txt +args2.txt``.
|
||||
|
||||
.. note::
|
||||
|
||||
Text file should separate arguments with line breaks.
|
||||
|
||||
`args.txt` example::
|
||||
args.txt example::
|
||||
|
||||
--login=MYUSERNAME
|
||||
--password=MYPASSWORD
|
||||
|
||||
+354
-63
@@ -1,84 +1,56 @@
|
||||
# Configuration file for the Sphinx documentation builder.
|
||||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
#
|
||||
# For the full list of built-in configuration values, see the documentation:
|
||||
# https://www.sphinx-doc.org/en/master/usage/configuration.html
|
||||
# Instaloader documentation build configuration file, created by
|
||||
# sphinx-quickstart on Fri Aug 25 15:38:26 2017.
|
||||
#
|
||||
# This file is execfile()d with the current directory set to its
|
||||
# containing dir.
|
||||
#
|
||||
# Note that not all possible configuration values are present in this
|
||||
# autogenerated file.
|
||||
#
|
||||
# All configuration values have a default; values that are commented out
|
||||
# serve to show the default.
|
||||
|
||||
# If extensions (or modules to document with autodoc) are in another directory,
|
||||
# add these directories to sys.path here. If the directory is relative to the
|
||||
# documentation root, use os.path.abspath to make it absolute, like shown here.
|
||||
#
|
||||
import os
|
||||
import platform
|
||||
import subprocess
|
||||
import sys
|
||||
import typing
|
||||
|
||||
sys.path.insert(0, os.path.abspath('..'))
|
||||
|
||||
import docs.sphinx_autodoc_typehints as sphinx_autodoc_typehints
|
||||
|
||||
# -- Project information -----------------------------------------------------
|
||||
# https://www.sphinx-doc.org/en/master/usage/configuration.html#project-information
|
||||
# -- General configuration ------------------------------------------------
|
||||
|
||||
project = 'Instaloader'
|
||||
copyright = '2023, Alexander Graf and Andre Koch-Kramer'
|
||||
author = 'Alexander Graf and Andre Koch-Kramer'
|
||||
|
||||
# -- General configuration ---------------------------------------------------
|
||||
# https://www.sphinx-doc.org/en/master/usage/configuration.html#general-configuration
|
||||
# If your documentation needs a minimal Sphinx version, state it here.
|
||||
#
|
||||
# needs_sphinx = '1.0'
|
||||
|
||||
# Add any Sphinx extension module names here, as strings. They can be
|
||||
# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
|
||||
# ones.
|
||||
extensions = [
|
||||
"sphinx.ext.autodoc",
|
||||
"sphinx.ext.extlinks",
|
||||
"sphinx.ext.githubpages",
|
||||
"sphinx.ext.intersphinx",
|
||||
"sphinxcontrib.jquery",
|
||||
'sphinx.ext.autodoc',
|
||||
'sphinx.ext.githubpages',
|
||||
'sphinx.ext.intersphinx',
|
||||
'sphinx.ext.extlinks'
|
||||
]
|
||||
|
||||
# autodoc options
|
||||
autodoc_default_flags = ['show-inheritance', 'members', 'undoc-members']
|
||||
|
||||
autodoc_default_options = {
|
||||
'show-inheritance': True,
|
||||
'members': True,
|
||||
'undoc-members': True,
|
||||
}
|
||||
autodoc_member_order = 'bysource'
|
||||
|
||||
# extlinks options
|
||||
intersphinx_mapping = {'python': ('https://docs.python.org/3', None),
|
||||
'requests': ('https://requests.kennethreitz.org/en/master/', None)}
|
||||
|
||||
extlinks = {
|
||||
'issue': (
|
||||
'https://github.com/instaloader/instaloader/issues/%s',
|
||||
'Issue #%s'
|
||||
),
|
||||
'example': (
|
||||
'https://raw.githubusercontent.com/instaloader/instaloader/master/docs/codesnippets/%s',
|
||||
'Example %s'
|
||||
),
|
||||
}
|
||||
|
||||
# intersphinx options
|
||||
|
||||
intersphinx_mapping = {
|
||||
'python': ('https://docs.python.org/3', None),
|
||||
'requests': ('https://requests.readthedocs.io/en/latest/', None),
|
||||
}
|
||||
|
||||
# other options
|
||||
|
||||
templates_path = ['_templates']
|
||||
exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store']
|
||||
|
||||
# ignore TypeVar as it is currently unsupported by Sphinx
|
||||
nitpick_ignore = [('py:class', 'instaloader.nodeiterator.T')]
|
||||
|
||||
# -- Options for HTML output -------------------------------------------------
|
||||
# https://www.sphinx-doc.org/en/master/usage/configuration.html#options-for-html-output
|
||||
|
||||
pygments_style = 'fruity'
|
||||
|
||||
html_theme = 'basic'
|
||||
html_static_path = ['_static']
|
||||
|
||||
html_logo = "logo.png"
|
||||
|
||||
html_css_files = ['instaloaderdoc.css']
|
||||
html_js_files = ['instaloaderdoc.js']
|
||||
|
||||
# get release number
|
||||
nitpick_ignore = [('py:class', 'typing.Tuple')]
|
||||
|
||||
current_release = subprocess.check_output(["git", "describe", "--abbrev=0"]).decode("ascii")[1:-1]
|
||||
date_format = "%e %b %Y" if platform.system() != "Windows" else "%d %b %Y"
|
||||
@@ -86,3 +58,322 @@ current_release_date = subprocess.check_output(
|
||||
["git", "log", "-1", "--tags", "--format=%ad", "--date=format:"+date_format]).decode("ascii")[:-1]
|
||||
|
||||
html_context = {'current_release': current_release, 'current_release_date': current_release_date}
|
||||
|
||||
extlinks = {
|
||||
'issue': (
|
||||
'https://github.com/instaloader/instaloader/issues/%s',
|
||||
'Issue #'
|
||||
),
|
||||
'example': (
|
||||
'https://raw.githubusercontent.com/instaloader/instaloader/master/docs/codesnippets/%s',
|
||||
'Example '
|
||||
),
|
||||
}
|
||||
|
||||
# Add any paths that contain templates here, relative to this directory.
|
||||
templates_path = ['_templates']
|
||||
|
||||
# The suffix(es) of source filenames.
|
||||
# You can specify multiple suffix as a list of string:
|
||||
#
|
||||
# source_suffix = ['.rst', '.md']
|
||||
source_suffix = '.rst'
|
||||
|
||||
# The encoding of source files.
|
||||
#
|
||||
# source_encoding = 'utf-8-sig'
|
||||
|
||||
# The master toctree document.
|
||||
master_doc = 'index'
|
||||
|
||||
# General information about the project.
|
||||
project = 'Instaloader'
|
||||
copyright = '2017-2020, Alexander Graf and Andre Koch-Kramer'
|
||||
author = 'Alexander Graf and Andre Koch-Kramer'
|
||||
|
||||
# The version info for the project you're documenting, acts as replacement for
|
||||
# |version| and |release|, also used in various other places throughout the
|
||||
# built documents.
|
||||
#
|
||||
# The short X.Y version.
|
||||
#version = '3.0'
|
||||
# The full version, including alpha/beta/rc tags.
|
||||
#release = '3.0rc0'
|
||||
|
||||
# The language for content autogenerated by Sphinx. Refer to documentation
|
||||
# for a list of supported languages.
|
||||
#
|
||||
# This is also used if you do content translation via gettext catalogs.
|
||||
# Usually you set "language" from the command line for these cases.
|
||||
language = "en"
|
||||
|
||||
# There are two options for replacing |today|: either, you set today to some
|
||||
# non-false value, then it is used:
|
||||
#
|
||||
# today = ''
|
||||
#
|
||||
# Else, today_fmt is used as the format for a strftime call.
|
||||
#
|
||||
# today_fmt = '%B %d, %Y'
|
||||
|
||||
# List of patterns, relative to source directory, that match files and
|
||||
# directories to ignore when looking for source files.
|
||||
# This patterns also effect to html_static_path and html_extra_path
|
||||
exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store']
|
||||
|
||||
# The reST default role (used for this markup: `text`) to use for all
|
||||
# documents.
|
||||
#
|
||||
# default_role = None
|
||||
|
||||
# If true, '()' will be appended to :func: etc. cross-reference text.
|
||||
#
|
||||
# add_function_parentheses = True
|
||||
|
||||
# If true, the current module name will be prepended to all description
|
||||
# unit titles (such as .. function::).
|
||||
#
|
||||
add_module_names = False
|
||||
|
||||
# If true, sectionauthor and moduleauthor directives will be shown in the
|
||||
# output. They are ignored by default.
|
||||
#
|
||||
# show_authors = False
|
||||
|
||||
# The name of the Pygments (syntax highlighting) style to use.
|
||||
pygments_style = 'fruity'
|
||||
|
||||
# A list of ignored prefixes for module index sorting.
|
||||
# modindex_common_prefix = []
|
||||
|
||||
# If true, keep warnings as "system message" paragraphs in the built documents.
|
||||
# keep_warnings = False
|
||||
|
||||
# If true, `todo` and `todoList` produce output, else they produce nothing.
|
||||
todo_include_todos = False
|
||||
|
||||
|
||||
# -- Options for HTML output ----------------------------------------------
|
||||
|
||||
# The theme to use for HTML and HTML Help pages. See the documentation for
|
||||
# a list of builtin themes.
|
||||
#
|
||||
html_theme = 'basic'
|
||||
|
||||
# Add any paths that contain custom themes here, relative to this directory.
|
||||
# html_theme_path = []
|
||||
|
||||
# The name for this set of Sphinx documents.
|
||||
# "<project> v<release> documentation" by default.
|
||||
#
|
||||
# html_title = 'Instaloader v3.0rc0'
|
||||
|
||||
# A shorter title for the navigation bar. Default is the same as html_title.
|
||||
#
|
||||
# html_short_title = None
|
||||
|
||||
# The name of an image file (relative to this directory) to place at the top
|
||||
# of the sidebar.
|
||||
#
|
||||
html_logo = "logo.png"
|
||||
|
||||
# The name of an image file (relative to this directory) to use as a favicon of
|
||||
# the docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32
|
||||
# pixels large.
|
||||
#
|
||||
html_favicon = "favicon.ico"
|
||||
|
||||
# Add any paths that contain custom static files (such as style sheets) here,
|
||||
# relative to this directory. They are copied after the builtin static files,
|
||||
# so a file named "default.css" will overwrite the builtin "default.css".
|
||||
html_static_path = ['_static']
|
||||
|
||||
# Add any extra paths that contain custom files (such as robots.txt or
|
||||
# .htaccess) here, relative to this directory. These files are copied
|
||||
# directly to the root of the documentation.
|
||||
#
|
||||
# html_extra_path = []
|
||||
|
||||
# If not None, a 'Last updated on:' timestamp is inserted at every page
|
||||
# bottom, using the given strftime format.
|
||||
# The empty string is equivalent to '%b %d, %Y'.
|
||||
#
|
||||
# html_last_updated_fmt = None
|
||||
|
||||
# If true, SmartyPants will be used to convert quotes and dashes to
|
||||
# typographically correct entities.
|
||||
#
|
||||
# html_use_smartypants = True
|
||||
|
||||
# Custom sidebar templates, maps document names to template names.
|
||||
#
|
||||
#html_sidebars = {'**': ["relations.html", "links.html"] }
|
||||
html_sidebars = {}
|
||||
|
||||
# Additional templates that should be rendered to pages, maps page names to
|
||||
# template names.
|
||||
#
|
||||
# html_additional_pages = {}
|
||||
|
||||
# If false, no module index is generated.
|
||||
#
|
||||
html_domain_indices = False
|
||||
|
||||
# If false, no index is generated.
|
||||
#
|
||||
html_use_index = True
|
||||
|
||||
# If true, the index is split into individual pages for each letter.
|
||||
#
|
||||
# html_split_index = False
|
||||
|
||||
# If true, links to the reST sources are added to the pages.
|
||||
#
|
||||
html_show_sourcelink = False
|
||||
|
||||
# If true, "Created using Sphinx" is shown in the HTML footer. Default is True.
|
||||
#
|
||||
html_show_sphinx = False
|
||||
|
||||
# If true, "(C) Copyright ..." is shown in the HTML footer. Default is True.
|
||||
#
|
||||
html_show_copyright = False
|
||||
|
||||
# If true, an OpenSearch description file will be output, and all pages will
|
||||
# contain a <link> tag referring to it. The value of this option must be the
|
||||
# base URL from which the finished HTML is served.
|
||||
#
|
||||
# html_use_opensearch = ''
|
||||
|
||||
# This is the file name suffix for HTML files (e.g. ".xhtml").
|
||||
# html_file_suffix = None
|
||||
|
||||
# Language to be used for generating the HTML full-text search index.
|
||||
# Sphinx supports the following languages:
|
||||
# 'da', 'de', 'en', 'es', 'fi', 'fr', 'h', 'it', 'ja'
|
||||
# 'nl', 'no', 'pt', 'ro', 'r', 'sv', 'tr', 'zh'
|
||||
#
|
||||
# html_search_language = 'en'
|
||||
|
||||
# A dictionary with options for the search language support, empty by default.
|
||||
# 'ja' uses this config value.
|
||||
# 'zh' user can custom change `jieba` dictionary path.
|
||||
#
|
||||
# html_search_options = {'type': 'default'}
|
||||
|
||||
# The name of a javascript file (relative to the configuration directory) that
|
||||
# implements a search results scorer. If empty, the default will be used.
|
||||
#
|
||||
# html_search_scorer = 'scorer.js'
|
||||
|
||||
# Output file base name for HTML help builder.
|
||||
htmlhelp_basename = 'Instaloaderdoc'
|
||||
|
||||
# -- Options for LaTeX output ---------------------------------------------
|
||||
|
||||
latex_elements = {
|
||||
# The paper size ('letterpaper' or 'a4paper').
|
||||
#
|
||||
# 'papersize': 'letterpaper',
|
||||
|
||||
# The font size ('10pt', '11pt' or '12pt').
|
||||
#
|
||||
# 'pointsize': '10pt',
|
||||
|
||||
# Additional stuff for the LaTeX preamble.
|
||||
#
|
||||
# 'preamble': '',
|
||||
|
||||
# Latex figure (float) alignment
|
||||
#
|
||||
# 'figure_align': 'htbp',
|
||||
}
|
||||
|
||||
# Grouping the document tree into LaTeX files. List of tuples
|
||||
# (source start file, target name, title,
|
||||
# author, documentclass [howto, manual, or own class]).
|
||||
latex_documents = [
|
||||
(master_doc, 'Instaloader.tex', 'Instaloader Documentation',
|
||||
'Alexander Graf, André Koch-Kramer', 'manual'),
|
||||
]
|
||||
|
||||
# The name of an image file (relative to this directory) to place at the top of
|
||||
# the title page.
|
||||
#
|
||||
# latex_logo = None
|
||||
|
||||
# For "manual" documents, if this is true, then toplevel headings are parts,
|
||||
# not chapters.
|
||||
#
|
||||
# latex_use_parts = False
|
||||
|
||||
# If true, show page references after internal links.
|
||||
#
|
||||
# latex_show_pagerefs = False
|
||||
|
||||
# If true, show URL addresses after external links.
|
||||
#
|
||||
# latex_show_urls = False
|
||||
|
||||
# Documents to append as an appendix to all manuals.
|
||||
#
|
||||
# latex_appendices = []
|
||||
|
||||
# It false, will not define \strong, \code, itleref, \crossref ... but only
|
||||
# \sphinxstrong, ..., \sphinxtitleref, ... To help avoid clash with user added
|
||||
# packages.
|
||||
#
|
||||
# latex_keep_old_macro_names = True
|
||||
|
||||
# If false, no module index is generated.
|
||||
#
|
||||
# latex_domain_indices = True
|
||||
|
||||
|
||||
# -- Options for manual page output ---------------------------------------
|
||||
|
||||
# One entry per manual page. List of tuples
|
||||
# (source start file, name, description, authors, manual section).
|
||||
man_pages = [
|
||||
(master_doc, 'instaloader', 'Instaloader Documentation',
|
||||
[author], 1)
|
||||
]
|
||||
|
||||
# If true, show URL addresses after external links.
|
||||
#
|
||||
# man_show_urls = False
|
||||
|
||||
|
||||
# -- Options for Texinfo output -------------------------------------------
|
||||
|
||||
# Grouping the document tree into Texinfo files. List of tuples
|
||||
# (source start file, target name, title, author,
|
||||
# dir menu entry, description, category)
|
||||
texinfo_documents = [
|
||||
(master_doc, 'Instaloader', 'Instaloader Documentation',
|
||||
author, 'Instaloader', 'One line description of project.',
|
||||
'Miscellaneous'),
|
||||
]
|
||||
|
||||
# Documents to append as an appendix to all manuals.
|
||||
#
|
||||
# texinfo_appendices = []
|
||||
|
||||
# If false, no module index is generated.
|
||||
#
|
||||
# texinfo_domain_indices = True
|
||||
|
||||
# How to display URL addresses: 'footnote', 'no', or 'inline'.
|
||||
#
|
||||
# texinfo_show_urls = 'footnote'
|
||||
|
||||
# If true, do not generate a @detailmenu in the "Top" node's menu.
|
||||
#
|
||||
# texinfo_no_detailmenu = False
|
||||
|
||||
|
||||
def setup(app):
|
||||
typing.TYPE_CHECKING = True
|
||||
app.add_stylesheet("instaloaderdoc.css")
|
||||
app.connect('autodoc-process-signature', sphinx_autodoc_typehints.process_signature)
|
||||
app.connect('autodoc-process-docstring', sphinx_autodoc_typehints.process_docstring)
|
||||
|
||||
+17
-5
@@ -132,9 +132,21 @@ details. If you feel the need to do so, please create a "feature suggestion".
|
||||
be implemented and the underlying problem could be solved. Also **describe
|
||||
alternatives** that you have considered.
|
||||
|
||||
Sponsoring
|
||||
----------
|
||||
Donations
|
||||
---------
|
||||
|
||||
.. include:: sponsors.rst
|
||||
:start-after: donations-start
|
||||
:end-before: donations-end
|
||||
.. donations-start
|
||||
|
||||
It is a pleasure for us to share our Instaloader to the world, and we are proud
|
||||
to have attracted such an active and motivating community, with so many users
|
||||
who share their suggestions and ideas with us. Buying a community-sponsored beer
|
||||
or coffee from time to time is very likely to further raise our passion for the
|
||||
development of Instaloader.
|
||||
|
||||
| For donations, we provide a PayPal.Me link and a Bitcoin address.
|
||||
| `PayPal.me/aandergr <https://www.paypal.me/aandergr>`__
|
||||
| BTC: 1Nst4LoadeYzrKjJ1DX9CpbLXBYE9RKLwY
|
||||
|
||||
.. donations-end
|
||||
|
||||
.. (Discussion in :issue:`130`)
|
||||
|
||||
+11
-14
@@ -78,20 +78,6 @@ Useful Links
|
||||
- `Issue Tracker / Bug Tracker <https://github.com/instaloader/instaloader/issues>`__
|
||||
- `Version History <https://github.com/instaloader/instaloader/releases>`__
|
||||
|
||||
Contributing
|
||||
------------
|
||||
|
||||
As an open source project, Instaloader heavily depends on the contributions from
|
||||
its community. See :ref:`contributing` for how you may help Instaloader to
|
||||
become an even greater tool.
|
||||
|
||||
Supporters
|
||||
----------
|
||||
|
||||
.. include:: sponsors.rst
|
||||
:start-after: donations-start
|
||||
:end-before: donations-end
|
||||
|
||||
Disclaimer
|
||||
----------
|
||||
|
||||
@@ -99,6 +85,17 @@ Disclaimer
|
||||
:start-after: disclaimer-start
|
||||
:end-before: disclaimer-end
|
||||
|
||||
Contributing
|
||||
------------
|
||||
|
||||
As an open source project, Instaloader heavily depends on the contributions from
|
||||
its community. See :ref:`contributing` for how you may help Instaloader to
|
||||
become an even greater tool.
|
||||
|
||||
.. include:: contributing.rst
|
||||
:start-after: donations-start
|
||||
:end-before: donations-end
|
||||
|
||||
..
|
||||
* :ref:`genindex`
|
||||
* :ref:`modindex`
|
||||
|
||||
@@ -8,7 +8,7 @@ Install Instaloader
|
||||
To **install Instaloader**,
|
||||
|
||||
#. Ensure that you have `Python <https://www.python.org/>`__, at least
|
||||
version 3.8, and `pip <https://pypi.python.org/pypi/pip>`__
|
||||
version 3.5, and `pip <https://pypi.python.org/pypi/pip>`__
|
||||
installed.
|
||||
|
||||
#. Then, install Instaloader using::
|
||||
|
||||
@@ -47,5 +47,3 @@ Exceptions
|
||||
|
||||
|
||||
.. autoexception:: TooManyRequestsException
|
||||
|
||||
.. autoexception:: AbortDownloadException
|
||||
|
||||
@@ -25,13 +25,6 @@ Iterator :class:`NodeIterator` and a context manager
|
||||
.. autoclass:: FrozenNodeIterator
|
||||
:no-show-inheritance:
|
||||
|
||||
A serializable representation of a :class:`NodeIterator` instance, saving
|
||||
its iteration state.
|
||||
|
||||
It can be serialized and deserialized with :func:`save_structure_to_file`
|
||||
and :func:`load_structure_from_file`, as well as with :mod:`json` and
|
||||
:mod:`pickle` thanks to being a :class:`~typing.NamedTuple`.
|
||||
|
||||
``resumable_iteration``
|
||||
"""""""""""""""""""""""
|
||||
|
||||
|
||||
@@ -71,30 +71,12 @@ TopSearchResults
|
||||
|
||||
.. versionadded:: 4.3
|
||||
|
||||
TitlePic
|
||||
""""""""
|
||||
|
||||
.. autoclass:: TitlePic
|
||||
:no-show-inheritance:
|
||||
|
||||
.. versionadded:: 4.8
|
||||
|
||||
Loading and Saving
|
||||
""""""""""""""""""
|
||||
|
||||
:class:`Post`, :class:`StoryItem`, :class:`Profile`, :class:`Hashtag` and
|
||||
:class:`FrozenNodeIterator` can be saved and loaded to/from JSON files.
|
||||
|
||||
.. autofunction:: load_structure
|
||||
|
||||
.. autofunction:: load_structure_from_file
|
||||
|
||||
.. autofunction:: get_json_structure
|
||||
|
||||
.. autofunction:: save_structure_to_file
|
||||
|
||||
LatestStamps
|
||||
""""""""""""
|
||||
|
||||
.. autoclass:: LatestStamps
|
||||
:no-show-inheritance:
|
||||
|
||||
@@ -0,0 +1,251 @@
|
||||
# sphinx_autodoc_typehints from Nov 20 2018 (version 1.5.2).
|
||||
# https://github.com/agronholm/sphinx-autodoc-typehints
|
||||
# (slightly modified to fix class links, maybe related to agronholm/sphinx-autodoc-typehints#38)
|
||||
#
|
||||
# This is the MIT license: http://www.opensource.org/licenses/mit-license.php
|
||||
#
|
||||
# Copyright (c) Alex Grönholm
|
||||
#
|
||||
# Permission is hereby granted, free of charge, to any person obtaining a copy of this
|
||||
# software and associated documentation files (the "Software"), to deal in the Software
|
||||
# without restriction, including without limitation the rights to use, copy, modify, merge,
|
||||
# publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons
|
||||
# to whom the Software is furnished to do so, subject to the following conditions:
|
||||
#
|
||||
# The above copyright notice and this permission notice shall be included in all copies or
|
||||
# substantial portions of the Software.
|
||||
#
|
||||
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
|
||||
# INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
|
||||
# PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE
|
||||
# FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
|
||||
# OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
|
||||
# DEALINGS IN THE SOFTWARE.
|
||||
#
|
||||
|
||||
|
||||
import inspect
|
||||
from typing import get_type_hints, TypeVar, Any, AnyStr, Generic, Union
|
||||
|
||||
from sphinx.util import logging
|
||||
from sphinx.util.inspect import Signature
|
||||
|
||||
try:
|
||||
from inspect import unwrap
|
||||
except ImportError:
|
||||
def unwrap(func, *, stop=None):
|
||||
"""This is the inspect.unwrap() method copied from Python 3.5's standard library."""
|
||||
if stop is None:
|
||||
def _is_wrapper(f):
|
||||
return hasattr(f, '__wrapped__')
|
||||
else:
|
||||
def _is_wrapper(f):
|
||||
return hasattr(f, '__wrapped__') and not stop(f)
|
||||
f = func # remember the original func for error reporting
|
||||
memo = {id(f)} # Memoise by id to tolerate non-hashable objects
|
||||
while _is_wrapper(func):
|
||||
func = func.__wrapped__
|
||||
id_func = id(func)
|
||||
if id_func in memo:
|
||||
raise ValueError('wrapper loop when unwrapping {!r}'.format(f))
|
||||
memo.add(id_func)
|
||||
return func
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def format_annotation(annotation):
|
||||
if inspect.isclass(annotation) and annotation.__module__ == 'builtins':
|
||||
if annotation.__qualname__ == 'NoneType':
|
||||
return '``None``'
|
||||
else:
|
||||
return ':py:class:`{}`'.format(annotation.__qualname__)
|
||||
|
||||
annotation_cls = annotation if inspect.isclass(annotation) else type(annotation)
|
||||
class_name = None
|
||||
if annotation_cls.__module__ == 'typing':
|
||||
params = None
|
||||
prefix = ':py:class:'
|
||||
module = 'typing'
|
||||
extra = ''
|
||||
|
||||
if inspect.isclass(getattr(annotation, '__origin__', None)):
|
||||
annotation_cls = annotation.__origin__
|
||||
try:
|
||||
if Generic in annotation_cls.mro():
|
||||
module = annotation_cls.__module__
|
||||
except TypeError:
|
||||
pass # annotation_cls was either the "type" object or typing.Type
|
||||
|
||||
if annotation is Any:
|
||||
return ':py:data:`~typing.Any`'
|
||||
elif annotation is AnyStr:
|
||||
return ':py:data:`~typing.AnyStr`'
|
||||
elif isinstance(annotation, TypeVar):
|
||||
return '\\%r' % annotation
|
||||
elif (annotation is Union or getattr(annotation, '__origin__', None) is Union or
|
||||
hasattr(annotation, '__union_params__')):
|
||||
prefix = ':py:data:'
|
||||
class_name = 'Union'
|
||||
if hasattr(annotation, '__union_params__'):
|
||||
params = annotation.__union_params__
|
||||
elif hasattr(annotation, '__args__'):
|
||||
params = annotation.__args__
|
||||
|
||||
if params and len(params) == 2 and (hasattr(params[1], '__qualname__') and
|
||||
params[1].__qualname__ == 'NoneType'):
|
||||
class_name = 'Optional'
|
||||
params = (params[0],)
|
||||
elif annotation_cls.__qualname__ == 'Tuple' and hasattr(annotation, '__tuple_params__'):
|
||||
params = annotation.__tuple_params__
|
||||
if annotation.__tuple_use_ellipsis__:
|
||||
params += (Ellipsis,)
|
||||
elif annotation_cls.__qualname__ == 'Callable':
|
||||
prefix = ':py:data:'
|
||||
arg_annotations = result_annotation = None
|
||||
if hasattr(annotation, '__result__'):
|
||||
arg_annotations = annotation.__args__
|
||||
result_annotation = annotation.__result__
|
||||
elif getattr(annotation, '__args__', None):
|
||||
arg_annotations = annotation.__args__[:-1]
|
||||
result_annotation = annotation.__args__[-1]
|
||||
|
||||
if arg_annotations in (Ellipsis, (Ellipsis,)):
|
||||
params = [Ellipsis, result_annotation]
|
||||
elif arg_annotations is not None:
|
||||
params = [
|
||||
'\\[{}]'.format(
|
||||
', '.join(format_annotation(param) for param in arg_annotations)),
|
||||
result_annotation
|
||||
]
|
||||
elif hasattr(annotation, 'type_var'):
|
||||
# Type alias
|
||||
class_name = annotation.name
|
||||
params = (annotation.type_var,)
|
||||
elif getattr(annotation, '__args__', None) is not None:
|
||||
params = annotation.__args__
|
||||
elif hasattr(annotation, '__parameters__'):
|
||||
params = annotation.__parameters__
|
||||
|
||||
if params:
|
||||
extra = '\\[{}]'.format(', '.join(format_annotation(param) for param in params))
|
||||
|
||||
if not class_name:
|
||||
class_name = annotation_cls.__qualname__.title()
|
||||
|
||||
return '{}`~{}.{}`{}'.format(prefix, module, class_name, extra)
|
||||
elif annotation is Ellipsis:
|
||||
return '...'
|
||||
elif inspect.isclass(annotation) or inspect.isclass(getattr(annotation, '__origin__', None)):
|
||||
if not inspect.isclass(annotation):
|
||||
annotation_cls = annotation.__origin__
|
||||
|
||||
extra = ''
|
||||
if Generic in annotation_cls.mro():
|
||||
params = (getattr(annotation, '__parameters__', None) or
|
||||
getattr(annotation, '__args__', None))
|
||||
extra = '\\[{}]'.format(', '.join(format_annotation(param) for param in params))
|
||||
|
||||
module = annotation.__module__.split('.')[0] # hack to 'fix' class linking for Instaloader project
|
||||
return ':py:class:`~{}.{}`{}'.format(module, annotation_cls.__qualname__,
|
||||
extra)
|
||||
|
||||
return str(annotation)
|
||||
|
||||
|
||||
def process_signature(app, what: str, name: str, obj, options, signature, return_annotation):
|
||||
if not callable(obj):
|
||||
return
|
||||
|
||||
if what in ('class', 'exception'):
|
||||
obj = getattr(obj, '__init__', getattr(obj, '__new__', None))
|
||||
|
||||
if not getattr(obj, '__annotations__', None):
|
||||
return
|
||||
|
||||
obj = unwrap(obj)
|
||||
signature = Signature(obj)
|
||||
parameters = [
|
||||
param.replace(annotation=inspect.Parameter.empty)
|
||||
for param in signature.signature.parameters.values()
|
||||
]
|
||||
|
||||
if parameters:
|
||||
if what in ('class', 'exception'):
|
||||
del parameters[0]
|
||||
elif what == 'method':
|
||||
outer = inspect.getmodule(obj)
|
||||
for clsname in obj.__qualname__.split('.')[:-1]:
|
||||
outer = getattr(outer, clsname)
|
||||
|
||||
method_name = obj.__name__
|
||||
if method_name.startswith("__") and not method_name.endswith("__"):
|
||||
# If the method starts with double underscore (dunder)
|
||||
# Python applies mangling so we need to prepend the class name.
|
||||
# This doesn't happen if it always ends with double underscore.
|
||||
class_name = obj.__qualname__.split('.')[-2]
|
||||
method_name = "_{c}{m}".format(c=class_name, m=method_name)
|
||||
|
||||
method_object = outer.__dict__[method_name]
|
||||
if not isinstance(method_object, (classmethod, staticmethod)):
|
||||
del parameters[0]
|
||||
|
||||
signature.signature = signature.signature.replace(
|
||||
parameters=parameters,
|
||||
return_annotation=inspect.Signature.empty)
|
||||
|
||||
return signature.format_args().replace('\\', '\\\\'), None
|
||||
|
||||
|
||||
def process_docstring(app, what, name, obj, options, lines):
|
||||
if isinstance(obj, property):
|
||||
obj = obj.fget
|
||||
|
||||
if callable(obj):
|
||||
if what in ('class', 'exception'):
|
||||
obj = getattr(obj, '__init__')
|
||||
|
||||
obj = unwrap(obj)
|
||||
try:
|
||||
type_hints = get_type_hints(obj)
|
||||
except (AttributeError, TypeError):
|
||||
# Introspecting a slot wrapper will raise TypeError
|
||||
return
|
||||
except NameError as exc:
|
||||
logger.warning('Cannot resolve forward reference in type annotations of "%s": %s',
|
||||
name, exc)
|
||||
type_hints = obj.__annotations__
|
||||
|
||||
for argname, annotation in type_hints.items():
|
||||
if argname.endswith('_'):
|
||||
argname = '{}\\_'.format(argname[:-1])
|
||||
|
||||
formatted_annotation = format_annotation(annotation)
|
||||
|
||||
if argname == 'return':
|
||||
if what in ('class', 'exception'):
|
||||
# Don't add return type None from __init__()
|
||||
continue
|
||||
|
||||
insert_index = len(lines)
|
||||
for i, line in enumerate(lines):
|
||||
if line.startswith(':rtype:'):
|
||||
insert_index = None
|
||||
break
|
||||
elif line.startswith(':return:') or line.startswith(':returns:'):
|
||||
insert_index = i
|
||||
|
||||
if insert_index is not None:
|
||||
if insert_index == len(lines):
|
||||
# Ensure that :rtype: doesn't get joined with a paragraph of text, which
|
||||
# prevents it being interpreted.
|
||||
lines.append('')
|
||||
insert_index += 1
|
||||
|
||||
lines.insert(insert_index, ':rtype: {}'.format(formatted_annotation))
|
||||
else:
|
||||
searchfor = ':param {}:'.format(argname)
|
||||
for i, line in enumerate(lines):
|
||||
if line.startswith(searchfor):
|
||||
lines.insert(i, ':type {}: {}'.format(argname, formatted_annotation))
|
||||
break
|
||||
@@ -1,23 +0,0 @@
|
||||
.. _sponsors:
|
||||
|
||||
Sponsoring the development of Instaloader
|
||||
=========================================
|
||||
|
||||
.. donations-start
|
||||
|
||||
.. include:: ../README.rst
|
||||
:start-after: current-sponsors-start
|
||||
:end-before: current-sponsors-end
|
||||
|
||||
It is a pleasure for us to share our Instaloader to the world, and we are proud
|
||||
to have attracted such an active and motivating community, with so many users
|
||||
who share their suggestions and ideas with us. Buying a community-sponsored beer
|
||||
or coffee from time to time is very likely to further raise our passion for the
|
||||
development of Instaloader.
|
||||
|
||||
| For Donations, we provide GitHub Sponsors page, a PayPal.Me link and a Bitcoin address.
|
||||
| GitHub Sponsors: `Sponsor @aandergr on GitHub Sponsors <https://github.com/sponsors/aandergr>`__
|
||||
| PayPal: `PayPal.me/aandergr <https://www.paypal.me/aandergr>`__
|
||||
| BTC: 1Nst4LoadeYzrKjJ1DX9CpbLXBYE9RKLwY
|
||||
|
||||
.. donations-end
|
||||
@@ -62,8 +62,7 @@ Two-Factor-Authentication. It also supports handling the *checkpoint challenge*,
|
||||
issued when Instagram suspects authentication activity on your account, by
|
||||
pointing the user to an URL to be opened in a browser.
|
||||
|
||||
Nevertheless, in :issue:`92`, :issue:`615`, :issue:`1150` and :issue:`1217`,
|
||||
users reported problems with
|
||||
Nevertheless, in :issue:`92` and :issue:`615` users reported problems with
|
||||
logging in. We recommend to always keep the session file which Instaloader
|
||||
creates when using :option:`--login`. If a session file is present,
|
||||
:option:`--login` does not make use of the failure-prone login procedure.
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
"""Download pictures (or videos) along with their captions and other metadata from Instagram."""
|
||||
|
||||
|
||||
__version__ = '4.11'
|
||||
__version__ = '4.6'
|
||||
|
||||
|
||||
try:
|
||||
@@ -15,9 +15,6 @@ else:
|
||||
from .exceptions import *
|
||||
from .instaloader import Instaloader
|
||||
from .instaloadercontext import InstaloaderContext, RateController
|
||||
from .lateststamps import LatestStamps
|
||||
from .nodeiterator import NodeIterator, FrozenNodeIterator, resumable_iteration
|
||||
from .structures import (Hashtag, Highlight, Post, PostSidecarNode, PostComment, PostCommentAnswer, PostLocation,
|
||||
Profile, Story, StoryItem, TopSearchResults, TitlePic,
|
||||
load_structure_from_file, save_structure_to_file,
|
||||
load_structure, get_json_structure)
|
||||
Profile, Story, StoryItem, TopSearchResults, load_structure_from_file, save_structure_to_file)
|
||||
|
||||
+20
-133
@@ -5,21 +5,14 @@ import datetime
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
import textwrap
|
||||
from argparse import ArgumentParser, ArgumentTypeError, SUPPRESS
|
||||
from argparse import ArgumentParser, SUPPRESS
|
||||
from typing import List, Optional
|
||||
|
||||
from . import (AbortDownloadException, BadCredentialsException, Instaloader, InstaloaderException,
|
||||
InvalidArgumentException, Post, Profile, ProfileNotExistsException, StoryItem,
|
||||
TwoFactorAuthRequiredException, __version__, load_structure_from_file)
|
||||
from .instaloader import (get_default_session_filename, get_default_stamps_filename)
|
||||
from . import (Instaloader, InstaloaderException, InvalidArgumentException, Post, Profile, ProfileNotExistsException,
|
||||
StoryItem, __version__, load_structure_from_file, TwoFactorAuthRequiredException,
|
||||
BadCredentialsException)
|
||||
from .instaloader import get_default_session_filename
|
||||
from .instaloadercontext import default_user_agent
|
||||
from .lateststamps import LatestStamps
|
||||
try:
|
||||
import browser_cookie3
|
||||
bc3_library = True
|
||||
except ImportError:
|
||||
bc3_library = False
|
||||
|
||||
|
||||
def usage_string():
|
||||
@@ -34,14 +27,6 @@ def usage_string():
|
||||
{0} --help""".format(argv0, len(argv0), '')
|
||||
|
||||
|
||||
def http_status_code_list(code_list_str: str) -> List[int]:
|
||||
codes = [int(s) for s in code_list_str.split(',')]
|
||||
for code in codes:
|
||||
if not 100 <= code <= 599:
|
||||
raise ArgumentTypeError("Invalid HTTP status code: {}".format(code))
|
||||
return codes
|
||||
|
||||
|
||||
def filterstr_to_filterfunc(filter_str: str, item_type: type):
|
||||
"""Takes an --post-filter=... or --storyitem-filter=... filter
|
||||
specification and makes a filter_func Callable out of it."""
|
||||
@@ -51,6 +36,7 @@ def filterstr_to_filterfunc(filter_str: str, item_type: type):
|
||||
|
||||
class TransformFilterAst(ast.NodeTransformer):
|
||||
def visit_Name(self, node: ast.Name):
|
||||
# pylint:disable=no-self-use
|
||||
if not isinstance(node.ctx, ast.Load):
|
||||
raise InvalidArgumentException("Invalid filter: Modifying variables ({}) not allowed.".format(node.id))
|
||||
if node.id == "datetime":
|
||||
@@ -73,53 +59,6 @@ def filterstr_to_filterfunc(filter_str: str, item_type: type):
|
||||
return filterfunc
|
||||
|
||||
|
||||
def get_cookies_from_instagram(domain, browser, cookie_file='', cookie_name=''):
|
||||
supported_browsers = {
|
||||
"chrome": browser_cookie3.chrome,
|
||||
"firefox": browser_cookie3.firefox,
|
||||
"edge": browser_cookie3.edge,
|
||||
"brave": browser_cookie3.brave,
|
||||
"opera": browser_cookie3.opera,
|
||||
"safari": browser_cookie3.safari
|
||||
}
|
||||
|
||||
if browser not in supported_browsers:
|
||||
print("Loading cookies from the specified browser failed")
|
||||
print("Supported browsers are Chrome, Firefox, Edge, Brave, Opera and Safari")
|
||||
return {}
|
||||
|
||||
cookies = {}
|
||||
browser_cookies = list(supported_browsers[browser](cookie_file=cookie_file))
|
||||
|
||||
for cookie in browser_cookies:
|
||||
if domain in cookie.domain:
|
||||
cookies[cookie.name] = cookie.value
|
||||
|
||||
if cookies:
|
||||
print(f"Cookies loaded successfully from {browser}")
|
||||
else:
|
||||
print(f"No cookies found for Instagram in {browser}, Are you logged in succesfully in {browser}?")
|
||||
|
||||
if cookie_name:
|
||||
return cookies.get(cookie_name, {})
|
||||
else:
|
||||
return cookies
|
||||
|
||||
|
||||
def import_session(browser, instaloader, cookiefile):
|
||||
cookie = get_cookies_from_instagram('instagram', browser, cookiefile)
|
||||
if cookie is not None:
|
||||
instaloader.context.update_cookies(cookie)
|
||||
username = instaloader.test_login()
|
||||
if not username:
|
||||
raise SystemExit(f"Not logged in. Are you logged in successfully in {browser}?")
|
||||
instaloader.context.username = username
|
||||
print(f"{username} has been successfully logged in.")
|
||||
next_step_text = (f"Next: Run instaloader --login={username} as it is required to download high quality media "
|
||||
"and to make full use of instaloader's features.")
|
||||
print(textwrap.fill(next_step_text))
|
||||
|
||||
|
||||
def _main(instaloader: Instaloader, targetlist: List[str],
|
||||
username: Optional[str] = None, password: Optional[str] = None,
|
||||
sessionfile: Optional[str] = None,
|
||||
@@ -129,11 +68,8 @@ def _main(instaloader: Instaloader, targetlist: List[str],
|
||||
download_tagged: bool = False,
|
||||
download_igtv: bool = False,
|
||||
fast_update: bool = False,
|
||||
latest_stamps_file: Optional[str] = None,
|
||||
max_count: Optional[int] = None, post_filter_str: Optional[str] = None,
|
||||
storyitem_filter_str: Optional[str] = None,
|
||||
browser: Optional[str] = None,
|
||||
cookiefile: Optional[str] = None) -> None:
|
||||
storyitem_filter_str: Optional[str] = None) -> None:
|
||||
"""Download set of profiles, hashtags etc. and handle logging in and session files if desired."""
|
||||
# Parse and generate filter function
|
||||
post_filter = None
|
||||
@@ -144,15 +80,6 @@ def _main(instaloader: Instaloader, targetlist: List[str],
|
||||
if storyitem_filter_str is not None:
|
||||
storyitem_filter = filterstr_to_filterfunc(storyitem_filter_str, StoryItem)
|
||||
instaloader.context.log('Only download storyitems with property "{}".'.format(storyitem_filter_str))
|
||||
latest_stamps = None
|
||||
if latest_stamps_file is not None:
|
||||
latest_stamps = LatestStamps(latest_stamps_file)
|
||||
instaloader.context.log(f"Using latest stamps from {latest_stamps_file}.")
|
||||
# load cookies if browser is not None
|
||||
if browser and bc3_library:
|
||||
import_session(browser.lower(), instaloader, cookiefile)
|
||||
elif browser and not bc3_library:
|
||||
raise SystemExit("browser_cookie3 library is needed to load cookies from browsers")
|
||||
# Login, if desired
|
||||
if username is not None:
|
||||
if not re.match(r"^[A-Za-z0-9._]+$", username):
|
||||
@@ -168,17 +95,12 @@ def _main(instaloader: Instaloader, targetlist: List[str],
|
||||
try:
|
||||
instaloader.login(username, password)
|
||||
except TwoFactorAuthRequiredException:
|
||||
# https://github.com/instaloader/instaloader/issues/1217
|
||||
instaloader.context.error("Warning: There have been reports of 2FA currently not working. "
|
||||
"Consider importing session cookies from your browser with "
|
||||
"--load-cookies.")
|
||||
while True:
|
||||
try:
|
||||
code = input("Enter 2FA verification code: ")
|
||||
instaloader.two_factor_login(code)
|
||||
break
|
||||
except BadCredentialsException as err:
|
||||
print(err, file=sys.stderr)
|
||||
except BadCredentialsException:
|
||||
pass
|
||||
else:
|
||||
instaloader.interactive_login(username)
|
||||
@@ -242,7 +164,7 @@ def _main(instaloader: Instaloader, targetlist: List[str],
|
||||
post_filter=post_filter)
|
||||
elif re.match(r"^[A-Za-z0-9._]+$", target):
|
||||
try:
|
||||
profile = instaloader.check_profile_id(target, latest_stamps)
|
||||
profile = instaloader.check_profile_id(target)
|
||||
if instaloader.context.is_logged_in and profile.has_blocked_viewer:
|
||||
if download_profile_pic or ((download_posts or download_tagged or download_igtv)
|
||||
and not profile.is_private):
|
||||
@@ -261,8 +183,7 @@ def _main(instaloader: Instaloader, targetlist: List[str],
|
||||
instaloader.context.log("Trying again anonymously, helps in case you are just blocked.")
|
||||
with instaloader.anonymous_copy() as anonymous_loader:
|
||||
with instaloader.context.error_catcher():
|
||||
anonymous_retry_profiles.add(anonymous_loader.check_profile_id(target,
|
||||
latest_stamps))
|
||||
anonymous_retry_profiles.add(anonymous_loader.check_profile_id(target))
|
||||
instaloader.context.error("Warning: {} will be downloaded anonymously (\"{}\")."
|
||||
.format(target, err))
|
||||
else:
|
||||
@@ -277,25 +198,21 @@ def _main(instaloader: Instaloader, targetlist: List[str],
|
||||
if len(profiles) > 1:
|
||||
instaloader.context.log("Downloading {} profiles: {}".format(len(profiles),
|
||||
' '.join([p.username for p in profiles])))
|
||||
if instaloader.context.iphone_support and profiles and (download_profile_pic or download_posts) and \
|
||||
not instaloader.context.is_logged_in:
|
||||
instaloader.context.log("Hint: Use --login to download higher-quality versions of pictures.")
|
||||
if profiles and (download_profile_pic or download_posts) and not instaloader.context.is_logged_in:
|
||||
instaloader.context.error("Warning: Use --login to download higher-quality versions of pictures.")
|
||||
instaloader.download_profiles(profiles,
|
||||
download_profile_pic, download_posts, download_tagged, download_igtv,
|
||||
download_highlights, download_stories,
|
||||
fast_update, post_filter, storyitem_filter, latest_stamps=latest_stamps)
|
||||
fast_update, post_filter, storyitem_filter)
|
||||
if anonymous_retry_profiles:
|
||||
instaloader.context.log("Downloading anonymously: {}"
|
||||
.format(' '.join([p.username for p in anonymous_retry_profiles])))
|
||||
with instaloader.anonymous_copy() as anonymous_loader:
|
||||
anonymous_loader.download_profiles(anonymous_retry_profiles,
|
||||
download_profile_pic, download_posts, download_tagged, download_igtv,
|
||||
fast_update=fast_update, post_filter=post_filter,
|
||||
latest_stamps=latest_stamps)
|
||||
fast_update=fast_update, post_filter=post_filter)
|
||||
except KeyboardInterrupt:
|
||||
print("\nInterrupted by user.", file=sys.stderr)
|
||||
except AbortDownloadException as exc:
|
||||
print("\nDownload aborted: {}.".format(exc), file=sys.stderr)
|
||||
# Save session if it is useful
|
||||
if instaloader.context.is_logged_in:
|
||||
instaloader.save_session_to_file(sessionfile)
|
||||
@@ -396,10 +313,7 @@ def main():
|
||||
g_cond.add_argument('-F', '--fast-update', action='store_true',
|
||||
help='For each target, stop when encountering the first already-downloaded picture. This '
|
||||
'flag is recommended when you use Instaloader to update your personal Instagram archive.')
|
||||
g_cond.add_argument('--latest-stamps', nargs='?', metavar='STAMPSFILE', const=get_default_stamps_filename(),
|
||||
help='Store the timestamps of latest media scraped for each profile. This allows updating '
|
||||
'your personal Instagram archive even if you delete the destination directories. '
|
||||
'If STAMPSFILE is not provided, defaults to ' + get_default_stamps_filename())
|
||||
|
||||
g_cond.add_argument('--post-filter', '--only-if', metavar='filter',
|
||||
help='Expression that, if given, must evaluate to True for each post to be downloaded. Must be '
|
||||
'a syntactically valid python expression. Variables are evaluated to '
|
||||
@@ -411,7 +325,7 @@ def main():
|
||||
|
||||
g_cond.add_argument('-c', '--count',
|
||||
help='Do not attempt to download more than COUNT posts. '
|
||||
'Applies to #hashtag, %%location_id, :feed, and :saved.')
|
||||
'Applies only to #hashtag and :feed.')
|
||||
|
||||
g_login = parser.add_argument_group('Login (Download Private Profiles)',
|
||||
'Instaloader can login to Instagram. This allows downloading private profiles. '
|
||||
@@ -420,10 +334,6 @@ def main():
|
||||
'to login.')
|
||||
g_login.add_argument('-l', '--login', metavar='YOUR-USERNAME',
|
||||
help='Login name (profile name) for your Instagram account.')
|
||||
g_login.add_argument('-b', '--load-cookies', metavar='BROWSER-NAME',
|
||||
help='Browser name to load cookies from Instagram')
|
||||
g_login.add_argument('-B', '--cookiefile', metavar='COOKIE-FILE',
|
||||
help='Cookie file of a profile to load cookies')
|
||||
g_login.add_argument('-f', '--sessionfile',
|
||||
help='Path for loading and storing session key file. '
|
||||
'Defaults to ' + get_default_session_filename("<login_name>"))
|
||||
@@ -438,21 +348,13 @@ def main():
|
||||
'{target} is replaced by the target you specified, i.e. either :feed, #hashtag or the '
|
||||
'profile name. Defaults to \'{target}\'.')
|
||||
g_how.add_argument('--filename-pattern',
|
||||
help='Prefix of filenames for posts and stories, relative to the directory given with '
|
||||
help='Prefix of filenames, relative to the directory given with '
|
||||
'--dirname-pattern. {profile} is replaced by the profile name,'
|
||||
'{target} is replaced by the target you specified, i.e. either :feed'
|
||||
'#hashtag or the profile name. Defaults to \'{date_utc}_UTC\'')
|
||||
g_how.add_argument('--title-pattern',
|
||||
help='Prefix of filenames for profile pics, hashtag profile pics, and highlight covers. '
|
||||
'Defaults to \'{date_utc}_UTC_{typename}\' if --dirname-pattern contains \'{target}\' '
|
||||
'or \'{dirname}\', or if --dirname-pattern is not specified. Otherwise defaults to '
|
||||
'\'{target}_{date_utc}_UTC_{typename}\'.')
|
||||
g_how.add_argument('--resume-prefix', metavar='PREFIX',
|
||||
help='Prefix for filenames that are used to save the information to resume an interrupted '
|
||||
'download.')
|
||||
g_how.add_argument('--sanitize-paths', action='store_true',
|
||||
help='Sanitize paths so that the resulting file and directory names are valid on both '
|
||||
'Windows and Unix.')
|
||||
g_how.add_argument('--no-resume', action='store_true',
|
||||
help='Do not resume a previously-aborted download iteration, and do not save such information '
|
||||
'when interrupted.')
|
||||
@@ -467,11 +369,6 @@ def main():
|
||||
g_how.add_argument('--commit-mode', action='store_true', help=SUPPRESS)
|
||||
g_how.add_argument('--request-timeout', metavar='N', type=float, default=300.0,
|
||||
help='Seconds to wait before timing out a connection request. Defaults to 300.')
|
||||
g_how.add_argument('--abort-on', type=http_status_code_list, metavar="STATUS_CODES",
|
||||
help='Comma-separated list of HTTP status codes that cause Instaloader to abort, bypassing all '
|
||||
'retry logic.')
|
||||
g_how.add_argument('--no-iphone', action='store_true',
|
||||
help='Do not attempt to download iPhone version of images and videos.')
|
||||
|
||||
g_misc = parser.add_argument_group('Miscellaneous Options')
|
||||
g_misc.add_argument('-q', '--quiet', action='store_true',
|
||||
@@ -512,9 +409,6 @@ def main():
|
||||
if args.no_pictures and args.fast_update:
|
||||
raise SystemExit('--no-pictures and --fast-update cannot be used together.')
|
||||
|
||||
if args.login and args.load_cookies:
|
||||
raise SystemExit('--load-cookies and --login cannot be used together.')
|
||||
|
||||
# Determine what to download
|
||||
download_profile_pic = not args.no_profile_pic or args.profile_pic_only
|
||||
download_posts = not (args.no_posts or args.stories_only or args.profile_pic_only)
|
||||
@@ -533,11 +427,7 @@ def main():
|
||||
request_timeout=args.request_timeout,
|
||||
resume_prefix=resume_prefix,
|
||||
check_resume_bbd=not args.use_aged_resume_files,
|
||||
slide=args.slide,
|
||||
fatal_status_codes=args.abort_on,
|
||||
iphone_support=not args.no_iphone,
|
||||
title_pattern=args.title_pattern,
|
||||
sanitize_paths=args.sanitize_paths)
|
||||
slide=args.slide)
|
||||
_main(loader,
|
||||
args.profile,
|
||||
username=args.login.lower() if args.login is not None else None,
|
||||
@@ -550,15 +440,12 @@ def main():
|
||||
download_tagged=args.tagged,
|
||||
download_igtv=args.igtv,
|
||||
fast_update=args.fast_update,
|
||||
latest_stamps_file=args.latest_stamps,
|
||||
max_count=int(args.count) if args.count is not None else None,
|
||||
post_filter_str=args.post_filter,
|
||||
storyitem_filter_str=args.storyitem_filter,
|
||||
browser=args.load_cookies,
|
||||
cookiefile=args.cookiefile)
|
||||
storyitem_filter_str=args.storyitem_filter)
|
||||
loader.close()
|
||||
except InstaloaderException as err:
|
||||
raise SystemExit("Fatal error: %s" % err) from err
|
||||
raise SystemExit("Fatal error: %s" % err)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
@@ -64,17 +64,3 @@ class QueryReturnedNotFoundException(ConnectionException):
|
||||
|
||||
class TooManyRequestsException(ConnectionException):
|
||||
pass
|
||||
|
||||
class IPhoneSupportDisabledException(InstaloaderException):
|
||||
pass
|
||||
|
||||
class AbortDownloadException(Exception):
|
||||
"""
|
||||
Exception that is not catched in the error catchers inside the download loop and so aborts the
|
||||
download loop.
|
||||
|
||||
This exception is not a subclass of ``InstaloaderException``.
|
||||
|
||||
.. versionadded:: 4.7
|
||||
"""
|
||||
pass
|
||||
|
||||
+143
-426
@@ -10,6 +10,7 @@ import tempfile
|
||||
from contextlib import contextmanager, suppress
|
||||
from datetime import datetime, timezone
|
||||
from functools import wraps
|
||||
from hashlib import md5
|
||||
from io import BytesIO
|
||||
from pathlib import Path
|
||||
from typing import Any, Callable, IO, Iterator, List, Optional, Set, Union, cast
|
||||
@@ -20,30 +21,23 @@ import urllib3 # type: ignore
|
||||
|
||||
from .exceptions import *
|
||||
from .instaloadercontext import InstaloaderContext, RateController
|
||||
from .lateststamps import LatestStamps
|
||||
from .nodeiterator import NodeIterator, resumable_iteration
|
||||
from .sectioniterator import SectionIterator
|
||||
from .structures import (Hashtag, Highlight, JsonExportable, Post, PostLocation, Profile, Story, StoryItem,
|
||||
load_structure_from_file, save_structure_to_file, PostSidecarNode, TitlePic)
|
||||
|
||||
|
||||
def _get_config_dir() -> str:
|
||||
if platform.system() == "Windows":
|
||||
# on Windows, use %LOCALAPPDATA%\Instaloader
|
||||
localappdata = os.getenv("LOCALAPPDATA")
|
||||
if localappdata is not None:
|
||||
return os.path.join(localappdata, "Instaloader")
|
||||
# legacy fallback - store in temp dir if %LOCALAPPDATA% is not set
|
||||
return os.path.join(tempfile.gettempdir(), ".instaloader-" + getpass.getuser())
|
||||
# on Unix, use ~/.config/instaloader
|
||||
return os.path.join(os.getenv("XDG_CONFIG_HOME", os.path.expanduser("~/.config")), "instaloader")
|
||||
load_structure_from_file, save_structure_to_file, PostSidecarNode)
|
||||
|
||||
|
||||
def get_default_session_filename(username: str) -> str:
|
||||
"""Returns default session filename for given username."""
|
||||
configdir = _get_config_dir()
|
||||
sessionfilename = "session-{}".format(username)
|
||||
return os.path.join(configdir, sessionfilename)
|
||||
if platform.system() == "Windows":
|
||||
# on Windows, use %LOCALAPPDATA%\Instaloader\session-USERNAME
|
||||
localappdata = os.getenv("LOCALAPPDATA")
|
||||
if localappdata is not None:
|
||||
return os.path.join(localappdata, "Instaloader", sessionfilename)
|
||||
# legacy fallback - store in temp dir if %LOCALAPPDATA% is not set
|
||||
return os.path.join(tempfile.gettempdir(), ".instaloader-" + getpass.getuser(), sessionfilename)
|
||||
# on Unix, use ~/.config/instaloader/session-USERNAME
|
||||
return os.path.join(os.getenv("XDG_CONFIG_HOME", os.path.expanduser("~/.config")), "instaloader", sessionfilename)
|
||||
|
||||
|
||||
def get_legacy_session_filename(username: str) -> str:
|
||||
@@ -53,17 +47,6 @@ def get_legacy_session_filename(username: str) -> str:
|
||||
return filename.lower()
|
||||
|
||||
|
||||
def get_default_stamps_filename() -> str:
|
||||
"""
|
||||
Returns default filename for latest stamps database.
|
||||
|
||||
.. versionadded:: 4.8
|
||||
|
||||
"""
|
||||
configdir = _get_config_dir()
|
||||
return os.path.join(configdir, "latest-stamps.ini")
|
||||
|
||||
|
||||
def format_string_contains_key(format_string: str, key: str) -> bool:
|
||||
# pylint:disable=unused-variable
|
||||
for literal_text, field_name, format_spec, conversion in string.Formatter().parse(format_string):
|
||||
@@ -119,7 +102,7 @@ class _ArbitraryItemFormatter(string.Formatter):
|
||||
|
||||
def get_value(self, key, args, kwargs):
|
||||
"""Override to substitute {ATTRIBUTE} by attributes of our _item."""
|
||||
if key == 'filename' and isinstance(self._item, (Post, StoryItem, PostSidecarNode, TitlePic)):
|
||||
if key == 'filename' and isinstance(self._item, (Post, StoryItem, PostSidecarNode)):
|
||||
return "{filename}"
|
||||
if hasattr(self._item, key):
|
||||
return getattr(self._item, key)
|
||||
@@ -137,38 +120,20 @@ class _ArbitraryItemFormatter(string.Formatter):
|
||||
|
||||
|
||||
class _PostPathFormatter(_ArbitraryItemFormatter):
|
||||
RESERVED: set = {'CON', 'PRN', 'AUX', 'NUL',
|
||||
'COM1', 'COM2', 'COM3', 'COM4', 'COM5', 'COM6', 'COM7', 'COM8', 'COM9',
|
||||
'LPT1', 'LPT2', 'LPT3', 'LPT4', 'LPT5', 'LPT6', 'LPT7', 'LPT8', 'LPT9'}
|
||||
|
||||
def __init__(self, item: Any, force_windows_path: bool = False):
|
||||
super().__init__(item)
|
||||
self.force_windows_path = force_windows_path
|
||||
|
||||
def get_value(self, key, args, kwargs):
|
||||
ret = super().get_value(key, args, kwargs)
|
||||
if not isinstance(ret, str):
|
||||
return ret
|
||||
return self.sanitize_path(ret, self.force_windows_path)
|
||||
return self.sanitize_path(ret)
|
||||
|
||||
@staticmethod
|
||||
def sanitize_path(ret: str, force_windows_path: bool = False) -> str:
|
||||
def sanitize_path(ret: str) -> str:
|
||||
"""Replaces '/' with similar looking Division Slash and some other illegal filename characters on Windows."""
|
||||
ret = ret.replace('/', '\u2215')
|
||||
|
||||
if ret.startswith('.'):
|
||||
ret = ret.replace('.', '\u2024', 1)
|
||||
|
||||
if force_windows_path or platform.system() == 'Windows':
|
||||
if platform.system() == 'Windows':
|
||||
ret = ret.replace(':', '\uff1a').replace('<', '\ufe64').replace('>', '\ufe65').replace('\"', '\uff02')
|
||||
ret = ret.replace('\\', '\ufe68').replace('|', '\uff5c').replace('?', '\ufe16').replace('*', '\uff0a')
|
||||
ret = ret.replace('\n', ' ').replace('\r', ' ')
|
||||
root, ext = os.path.splitext(ret)
|
||||
if root.upper() in _PostPathFormatter.RESERVED:
|
||||
root += '_'
|
||||
if ext == '.':
|
||||
ext = '\u2024'
|
||||
ret = root + ext
|
||||
return ret
|
||||
|
||||
|
||||
@@ -179,9 +144,6 @@ class Instaloader:
|
||||
:param user_agent: :option:`--user-agent`
|
||||
:param dirname_pattern: :option:`--dirname-pattern`, default is ``{target}``
|
||||
:param filename_pattern: :option:`--filename-pattern`, default is ``{date_utc}_UTC``
|
||||
:param title_pattern:
|
||||
:option:`--title-pattern`, default is ``{date_utc}_UTC_{typename}`` if ``dirname_pattern`` contains
|
||||
``{target}`` or ``{profile}``, ``{target}_{date_utc}_UTC_{typename}`` otherwise.
|
||||
:param download_pictures: not :option:`--no-pictures`
|
||||
:param download_videos: not :option:`--no-videos`
|
||||
:param download_video_thumbnails: not :option:`--no-video-thumbnails`
|
||||
@@ -199,9 +161,6 @@ class Instaloader:
|
||||
:param resume_prefix: :option:`--resume-prefix`, or None for :option:`--no-resume`.
|
||||
:param check_resume_bbd: Whether to check the date of expiry of resume files and reject them if expired.
|
||||
:param slide: :option:`--slide`
|
||||
:param fatal_status_codes: :option:`--abort-on`
|
||||
:param iphone_support: not :option:`--no-iphone`
|
||||
:param sanitize_paths: :option:`--sanitize-paths`
|
||||
|
||||
.. attribute:: context
|
||||
|
||||
@@ -217,39 +176,25 @@ class Instaloader:
|
||||
download_pictures=True,
|
||||
download_videos: bool = True,
|
||||
download_video_thumbnails: bool = True,
|
||||
download_geotags: bool = False,
|
||||
download_comments: bool = False,
|
||||
download_geotags: bool = True,
|
||||
download_comments: bool = True,
|
||||
save_metadata: bool = True,
|
||||
compress_json: bool = True,
|
||||
post_metadata_txt_pattern: Optional[str] = None,
|
||||
storyitem_metadata_txt_pattern: Optional[str] = None,
|
||||
post_metadata_txt_pattern: str = None,
|
||||
storyitem_metadata_txt_pattern: str = None,
|
||||
max_connection_attempts: int = 3,
|
||||
request_timeout: float = 300.0,
|
||||
rate_controller: Optional[Callable[[InstaloaderContext], RateController]] = None,
|
||||
resume_prefix: Optional[str] = "iterator",
|
||||
check_resume_bbd: bool = True,
|
||||
slide: Optional[str] = None,
|
||||
fatal_status_codes: Optional[List[int]] = None,
|
||||
iphone_support: bool = True,
|
||||
title_pattern: Optional[str] = None,
|
||||
sanitize_paths: bool = False):
|
||||
slide: Optional[str] = None):
|
||||
|
||||
self.context = InstaloaderContext(sleep, quiet, user_agent, max_connection_attempts,
|
||||
request_timeout, rate_controller, fatal_status_codes,
|
||||
iphone_support)
|
||||
request_timeout, rate_controller)
|
||||
|
||||
# configuration parameters
|
||||
self.dirname_pattern = dirname_pattern or "{target}"
|
||||
self.filename_pattern = filename_pattern or "{date_utc}_UTC"
|
||||
if title_pattern is not None:
|
||||
self.title_pattern = title_pattern
|
||||
else:
|
||||
if (format_string_contains_key(self.dirname_pattern, 'profile') or
|
||||
format_string_contains_key(self.dirname_pattern, 'target')):
|
||||
self.title_pattern = '{date_utc}_UTC_{typename}'
|
||||
else:
|
||||
self.title_pattern = '{target}_{date_utc}_UTC_{typename}'
|
||||
self.sanitize_paths = sanitize_paths
|
||||
self.download_pictures = download_pictures
|
||||
self.download_videos = download_videos
|
||||
self.download_video_thumbnails = download_video_thumbnails
|
||||
@@ -311,10 +256,7 @@ class Instaloader:
|
||||
request_timeout=self.context.request_timeout,
|
||||
resume_prefix=self.resume_prefix,
|
||||
check_resume_bbd=self.check_resume_bbd,
|
||||
slide=self.slide,
|
||||
fatal_status_codes=self.context.fatal_status_codes,
|
||||
iphone_support=self.context.iphone_support,
|
||||
sanitize_paths=self.sanitize_paths)
|
||||
slide=self.slide)
|
||||
yield new_loader
|
||||
self.context.error_log.extend(new_loader.context.error_log)
|
||||
new_loader.context.error_log = [] # avoid double-printing of errors
|
||||
@@ -335,25 +277,15 @@ class Instaloader:
|
||||
filename_suffix: Optional[str] = None, _attempt: int = 1) -> bool:
|
||||
"""Downloads and saves picture with given url under given directory with given timestamp.
|
||||
Returns true, if file was actually downloaded, i.e. updated."""
|
||||
if filename_suffix is not None:
|
||||
filename += '_' + filename_suffix
|
||||
urlmatch = re.search('\\.[a-z0-9]*\\?', url)
|
||||
file_extension = url[-3:] if urlmatch is None else urlmatch.group(0)[1:-1]
|
||||
nominal_filename = filename + '.' + file_extension
|
||||
if os.path.isfile(nominal_filename):
|
||||
self.context.log(nominal_filename + ' exists', end=' ', flush=True)
|
||||
return False
|
||||
resp = self.context.get_raw(url)
|
||||
if 'Content-Type' in resp.headers and resp.headers['Content-Type']:
|
||||
header_extension = '.' + resp.headers['Content-Type'].split(';')[0].split('/')[-1]
|
||||
header_extension = header_extension.lower().replace('jpeg', 'jpg')
|
||||
filename += header_extension
|
||||
else:
|
||||
filename = nominal_filename
|
||||
if filename != nominal_filename and os.path.isfile(filename):
|
||||
if filename_suffix is not None:
|
||||
filename += '_' + filename_suffix
|
||||
filename += '.' + file_extension
|
||||
if os.path.isfile(filename):
|
||||
self.context.log(filename + ' exists', end=' ', flush=True)
|
||||
return False
|
||||
self.context.write_raw(resp, filename)
|
||||
self.context.get_and_write_raw(url, filename)
|
||||
os.utime(filename, (datetime.now().timestamp(), mtime.timestamp()))
|
||||
return True
|
||||
|
||||
@@ -400,54 +332,25 @@ class Instaloader:
|
||||
combined_answers.extend(y['answers'])
|
||||
unique_comments_list[-1]['answers'] = get_unique_comments(combined_answers)
|
||||
return unique_comments_list
|
||||
|
||||
def get_new_comments(new_comments, start):
|
||||
for idx, comment in enumerate(new_comments, start=start+1):
|
||||
if idx % 250 == 0:
|
||||
self.context.log('{}'.format(idx), end='…', flush=True)
|
||||
yield comment
|
||||
|
||||
def save_comments(extended_comments):
|
||||
unique_comments = get_unique_comments(extended_comments, combine_answers=True)
|
||||
answer_ids = set(int(answer['id']) for comment in unique_comments for answer in comment.get('answers', []))
|
||||
with open(filename, 'w') as file:
|
||||
file.write(json.dumps(list(filter(lambda t: int(t['id']) not in answer_ids, unique_comments)),
|
||||
indent=4))
|
||||
|
||||
base_filename = filename
|
||||
filename += '_comments.json'
|
||||
try:
|
||||
with open(filename) as fp:
|
||||
comments = json.load(fp)
|
||||
except (FileNotFoundError, json.decoder.JSONDecodeError):
|
||||
comments = list()
|
||||
|
||||
comments_iterator = post.get_comments()
|
||||
try:
|
||||
with resumable_iteration(
|
||||
context=self.context,
|
||||
iterator=comments_iterator,
|
||||
load=load_structure_from_file,
|
||||
save=save_structure_to_file,
|
||||
format_path=lambda magic: "{}_{}_{}.json.xz".format(base_filename, self.resume_prefix, magic),
|
||||
check_bbd=self.check_resume_bbd,
|
||||
enabled=self.resume_prefix is not None
|
||||
) as (_is_resuming, start_index):
|
||||
comments.extend(_postcomment_asdict(comment)
|
||||
for comment in get_new_comments(comments_iterator, start_index))
|
||||
except (KeyboardInterrupt, AbortDownloadException):
|
||||
if comments:
|
||||
save_comments(comments)
|
||||
raise
|
||||
comments.extend(_postcomment_asdict(comment) for comment in post.get_comments())
|
||||
if comments:
|
||||
save_comments(comments)
|
||||
comments = get_unique_comments(comments, combine_answers=True)
|
||||
answer_ids = set(int(answer['id']) for comment in comments for answer in comment.get('answers', []))
|
||||
with open(filename, 'w') as file:
|
||||
file.write(json.dumps(list(filter(lambda t: int(t['id']) not in answer_ids, comments)), indent=4))
|
||||
self.context.log('comments', end=' ', flush=True)
|
||||
|
||||
def save_caption(self, filename: str, mtime: datetime, caption: str) -> None:
|
||||
"""Updates picture caption / Post metadata info"""
|
||||
def _elliptify(caption):
|
||||
pcaption = caption.replace('\n', ' ').strip()
|
||||
return '[' + ((pcaption[:29] + "\u2026") if len(pcaption) > 31 else pcaption) + ']'
|
||||
return '[' + ((pcaption[:29] + u"\u2026") if len(pcaption) > 31 else pcaption) + ']'
|
||||
filename += '.txt'
|
||||
caption += '\n'
|
||||
pcaption = _elliptify(caption)
|
||||
@@ -478,19 +381,17 @@ class Instaloader:
|
||||
self.context.log(pcaption, end=' ', flush=True)
|
||||
except UnicodeEncodeError:
|
||||
self.context.log('txt', end=' ', flush=True)
|
||||
with open(filename, 'w', encoding='UTF-8') as fio:
|
||||
fio.write(caption)
|
||||
with open(filename, 'wb') as text_file:
|
||||
with BytesIO(bcaption) as bio:
|
||||
shutil.copyfileobj(cast(IO, bio), text_file)
|
||||
os.utime(filename, (datetime.now().timestamp(), mtime.timestamp()))
|
||||
|
||||
def save_location(self, filename: str, location: PostLocation, mtime: datetime) -> None:
|
||||
"""Save post location name and Google Maps link."""
|
||||
filename += '_location.txt'
|
||||
if location.lat is not None and location.lng is not None:
|
||||
location_string = (location.name + "\n" +
|
||||
"https://maps.google.com/maps?q={0},{1}&ll={0},{1}\n".format(location.lat,
|
||||
location.lng))
|
||||
else:
|
||||
location_string = location.name
|
||||
location_string = (location.name + "\n" +
|
||||
"https://maps.google.com/maps?q={0},{1}&ll={0},{1}\n".format(location.lat,
|
||||
location.lng))
|
||||
with open(filename, 'wb') as text_file:
|
||||
with BytesIO(location_string.encode()) as bio:
|
||||
shutil.copyfileobj(cast(IO, bio), text_file)
|
||||
@@ -523,52 +424,31 @@ class Instaloader:
|
||||
|
||||
.. versionadded:: 4.3"""
|
||||
|
||||
def _epoch_to_string(epoch: datetime) -> str:
|
||||
return epoch.strftime('%Y-%m-%d_%H-%M-%S_UTC')
|
||||
|
||||
http_response = self.context.get_raw(url)
|
||||
date_object: Optional[datetime] = None
|
||||
date_object = None # type: Optional[datetime]
|
||||
if 'Last-Modified' in http_response.headers:
|
||||
date_object = datetime.strptime(http_response.headers["Last-Modified"], '%a, %d %b %Y %H:%M:%S GMT')
|
||||
date_object = date_object.replace(tzinfo=timezone.utc)
|
||||
pic_bytes = None
|
||||
pic_identifier = _epoch_to_string(date_object)
|
||||
else:
|
||||
pic_bytes = http_response.content
|
||||
ig_filename = url.split('/')[-1].split('?')[0]
|
||||
pic_data = TitlePic(owner_profile, target, name_suffix, ig_filename, date_object)
|
||||
dirname = _PostPathFormatter(pic_data, self.sanitize_paths).format(self.dirname_pattern, target=target)
|
||||
filename_template = os.path.join(
|
||||
dirname,
|
||||
_PostPathFormatter(pic_data, self.sanitize_paths).format(self.title_pattern, target=target))
|
||||
filename = self.__prepare_filename(filename_template, lambda: url) + ".jpg"
|
||||
pic_identifier = md5(pic_bytes).hexdigest()[:16]
|
||||
filename = self.format_filename_within_target_path(target, owner_profile, pic_identifier, name_suffix, 'jpg')
|
||||
content_length = http_response.headers.get('Content-Length', None)
|
||||
if os.path.isfile(filename) and (not self.context.is_logged_in or
|
||||
(content_length is not None and
|
||||
os.path.getsize(filename) >= int(content_length))):
|
||||
self.context.log(filename + ' already exists')
|
||||
return
|
||||
return None
|
||||
os.makedirs(os.path.dirname(filename), exist_ok=True)
|
||||
self.context.write_raw(pic_bytes if pic_bytes else http_response, filename)
|
||||
if date_object:
|
||||
os.utime(filename, (datetime.now().timestamp(), date_object.timestamp()))
|
||||
self.context.log('') # log output of _get_and_write_raw() does not produce \n
|
||||
|
||||
def download_profilepic_if_new(self, profile: Profile, latest_stamps: Optional[LatestStamps]) -> None:
|
||||
"""
|
||||
Downloads and saves profile pic if it has not been downloaded before.
|
||||
|
||||
:param latest_stamps: Database with the last downloaded data. If not present,
|
||||
the profile pic is downloaded unless it already exists
|
||||
|
||||
.. versionadded:: 4.8
|
||||
"""
|
||||
if latest_stamps is None:
|
||||
self.download_profilepic(profile)
|
||||
return
|
||||
profile_pic_basename = profile.profile_pic_url_no_iphone.split('/')[-1].split('?')[0]
|
||||
saved_basename = latest_stamps.get_profile_pic(profile.username)
|
||||
if saved_basename == profile_pic_basename:
|
||||
return
|
||||
self.download_profilepic(profile)
|
||||
latest_stamps.set_profile_pic(profile.username, profile_pic_basename)
|
||||
|
||||
def download_profilepic(self, profile: Profile) -> None:
|
||||
"""Downloads and saves profile pic."""
|
||||
self.download_title_pic(profile.profile_pic_url, profile.username.lower(), 'profile_pic', profile)
|
||||
@@ -585,23 +465,6 @@ class Instaloader:
|
||||
.. versionadded:: 4.4"""
|
||||
self.download_title_pic(hashtag.profile_pic_url, '#' + hashtag.name, 'profile_pic', None)
|
||||
|
||||
@_requires_login
|
||||
def save_session(self) -> dict:
|
||||
"""Saves internally stored :class:`requests.Session` object to :class:`dict`.
|
||||
|
||||
:raises LoginRequiredException: If called without being logged in.
|
||||
|
||||
.. versionadded:: 4.10
|
||||
"""
|
||||
return self.context.save_session()
|
||||
|
||||
def load_session(self, username: str, session_data: dict) -> None:
|
||||
"""Internally stores :class:`requests.Session` object from :class:`dict`.
|
||||
|
||||
.. versionadded:: 4.10
|
||||
"""
|
||||
self.context.load_session(username, session_data)
|
||||
|
||||
@_requires_login
|
||||
def save_session_to_file(self, filename: Optional[str] = None) -> None:
|
||||
"""Saves internally stored :class:`requests.Session` object.
|
||||
@@ -673,12 +536,11 @@ class Instaloader:
|
||||
os.makedirs(os.path.dirname(filename), exist_ok=True)
|
||||
return filename
|
||||
|
||||
def format_filename(self, item: Union[Post, StoryItem, PostSidecarNode, TitlePic],
|
||||
target: Optional[Union[str, Path]] = None):
|
||||
def format_filename(self, item: Union[Post, StoryItem, PostSidecarNode], target: Optional[Union[str, Path]] = None):
|
||||
"""Format filename of a :class:`Post` or :class:`StoryItem` according to ``filename-pattern`` parameter.
|
||||
|
||||
.. versionadded:: 4.1"""
|
||||
return _PostPathFormatter(item, self.sanitize_paths).format(self.filename_pattern, target=target)
|
||||
return _PostPathFormatter(item).format(self.filename_pattern, target=target)
|
||||
|
||||
def download_post(self, post: Post, target: Union[str, Path]) -> bool:
|
||||
"""
|
||||
@@ -689,28 +551,7 @@ class Instaloader:
|
||||
:return: True if something was downloaded, False otherwise, i.e. file was already there
|
||||
"""
|
||||
|
||||
def _already_downloaded(path: str) -> bool:
|
||||
if not os.path.isfile(path):
|
||||
return False
|
||||
else:
|
||||
self.context.log(path + ' exists', end=' ', flush=True)
|
||||
return True
|
||||
|
||||
def _all_already_downloaded(path_base, is_videos_enumerated) -> bool:
|
||||
if '{filename}' in self.filename_pattern:
|
||||
# full URL needed to evaluate actual filename, cannot determine at
|
||||
# this point if all sidecar nodes were already downloaded.
|
||||
return False
|
||||
for idx, is_video in is_videos_enumerated:
|
||||
if self.download_pictures and (not is_video or self.download_video_thumbnails):
|
||||
if not _already_downloaded("{0}_{1}.jpg".format(path_base, idx)):
|
||||
return False
|
||||
if is_video and self.download_videos:
|
||||
if not _already_downloaded("{0}_{1}.mp4".format(path_base, idx)):
|
||||
return False
|
||||
return True
|
||||
|
||||
dirname = _PostPathFormatter(post, self.sanitize_paths).format(self.dirname_pattern, target=target)
|
||||
dirname = _PostPathFormatter(post).format(self.dirname_pattern, target=target)
|
||||
filename_template = os.path.join(dirname, self.format_filename(post, target=target))
|
||||
filename = self.__prepare_filename(filename_template, lambda: post.url)
|
||||
|
||||
@@ -718,47 +559,37 @@ class Instaloader:
|
||||
downloaded = True
|
||||
if post.typename == 'GraphSidecar':
|
||||
if self.download_pictures or self.download_videos:
|
||||
if not _all_already_downloaded(
|
||||
filename_template, enumerate(
|
||||
(post.get_is_videos()[i]
|
||||
for i in range(self.slide_start % post.mediacount, self.slide_end % post.mediacount + 1)),
|
||||
start=self.slide_start % post.mediacount + 1
|
||||
)
|
||||
for edge_number, sidecar_node in enumerate(
|
||||
post.get_sidecar_nodes(self.slide_start, self.slide_end),
|
||||
start=post.mediacount if self.slide_start < 0 else self.slide_start + 1
|
||||
):
|
||||
for edge_number, sidecar_node in enumerate(
|
||||
post.get_sidecar_nodes(self.slide_start, self.slide_end),
|
||||
start=self.slide_start % post.mediacount + 1
|
||||
):
|
||||
suffix: Optional[str] = str(edge_number)
|
||||
if self.download_pictures and (not sidecar_node.is_video or self.download_video_thumbnails):
|
||||
suffix = str(edge_number)
|
||||
if '{filename}' in self.filename_pattern:
|
||||
suffix = None
|
||||
if self.download_pictures and (not sidecar_node.is_video or self.download_video_thumbnails):
|
||||
# pylint:disable=cell-var-from-loop
|
||||
sidecar_filename = self.__prepare_filename(filename_template,
|
||||
lambda: sidecar_node.display_url)
|
||||
# Download sidecar picture or video thumbnail (--no-pictures implies --no-video-thumbnails)
|
||||
downloaded &= self.download_pic(filename=sidecar_filename, url=sidecar_node.display_url,
|
||||
mtime=post.date_local, filename_suffix=suffix)
|
||||
if sidecar_node.is_video and self.download_videos:
|
||||
# pylint:disable=cell-var-from-loop
|
||||
sidecar_filename = self.__prepare_filename(filename_template,
|
||||
lambda: sidecar_node.video_url)
|
||||
# Download sidecar video if desired
|
||||
downloaded &= self.download_pic(filename=sidecar_filename, url=sidecar_node.video_url,
|
||||
mtime=post.date_local, filename_suffix=suffix)
|
||||
else:
|
||||
downloaded = False
|
||||
suffix = ''
|
||||
# pylint:disable=cell-var-from-loop
|
||||
filename = self.__prepare_filename(filename_template, lambda: sidecar_node.display_url)
|
||||
# Download sidecar picture or video thumbnail (--no-pictures implies --no-video-thumbnails)
|
||||
downloaded &= self.download_pic(filename=filename, url=sidecar_node.display_url,
|
||||
mtime=post.date_local, filename_suffix=suffix)
|
||||
if sidecar_node.is_video and self.download_videos:
|
||||
suffix = str(edge_number)
|
||||
if '{filename}' in self.filename_pattern:
|
||||
suffix = ''
|
||||
# pylint:disable=cell-var-from-loop
|
||||
filename = self.__prepare_filename(filename_template, lambda: sidecar_node.video_url)
|
||||
# Download sidecar video if desired
|
||||
downloaded &= self.download_pic(filename=filename, url=sidecar_node.video_url,
|
||||
mtime=post.date_local, filename_suffix=suffix)
|
||||
elif post.typename == 'GraphImage':
|
||||
# Download picture
|
||||
if self.download_pictures:
|
||||
downloaded = (not _already_downloaded(filename + ".jpg") and
|
||||
self.download_pic(filename=filename, url=post.url, mtime=post.date_local))
|
||||
downloaded = self.download_pic(filename=filename, url=post.url, mtime=post.date_local)
|
||||
elif post.typename == 'GraphVideo':
|
||||
# Download video thumbnail (--no-pictures implies --no-video-thumbnails)
|
||||
if self.download_pictures and self.download_video_thumbnails:
|
||||
with self.context.error_catcher("Video thumbnail of {}".format(post)):
|
||||
downloaded = (not _already_downloaded(filename + ".jpg") and
|
||||
self.download_pic(filename=filename, url=post.url, mtime=post.date_local))
|
||||
downloaded = self.download_pic(filename=filename, url=post.url, mtime=post.date_local)
|
||||
else:
|
||||
self.context.error("Warning: {0} has unknown typename: {1}".format(post, post.typename))
|
||||
|
||||
@@ -769,8 +600,7 @@ class Instaloader:
|
||||
|
||||
# Download video if desired
|
||||
if post.is_video and self.download_videos:
|
||||
downloaded &= (not _already_downloaded(filename + ".mp4") and
|
||||
self.download_pic(filename=filename, url=post.video_url, mtime=post.date_local))
|
||||
downloaded &= self.download_pic(filename=filename, url=post.video_url, mtime=post.date_local)
|
||||
|
||||
# Download geotags if desired
|
||||
if self.download_geotags and post.location:
|
||||
@@ -820,8 +650,7 @@ class Instaloader:
|
||||
userids: Optional[List[Union[int, Profile]]] = None,
|
||||
fast_update: bool = False,
|
||||
filename_target: Optional[str] = ':stories',
|
||||
storyitem_filter: Optional[Callable[[StoryItem], bool]] = None,
|
||||
latest_stamps: Optional[LatestStamps] = None) -> None:
|
||||
storyitem_filter: Optional[Callable[[StoryItem], bool]] = None) -> None:
|
||||
"""
|
||||
Download available stories from user followees or all stories of users whose ID are given.
|
||||
Does not mark stories as seen.
|
||||
@@ -832,37 +661,20 @@ class Instaloader:
|
||||
:param filename_target: Replacement for {target} in dirname_pattern and filename_pattern
|
||||
or None if profile name should be used instead
|
||||
:param storyitem_filter: function(storyitem), which returns True if given StoryItem should be downloaded
|
||||
:param latest_stamps: Database with the last times each user was scraped
|
||||
:raises LoginRequiredException: If called without being logged in.
|
||||
|
||||
.. versionchanged:: 4.8
|
||||
Add `latest_stamps` parameter.
|
||||
"""
|
||||
|
||||
if not userids:
|
||||
self.context.log("Retrieving all visible stories...")
|
||||
profile_count = None
|
||||
else:
|
||||
userids = [p if isinstance(p, int) else p.userid for p in userids]
|
||||
profile_count = len(userids)
|
||||
|
||||
for i, user_story in enumerate(self.get_stories(userids), start=1):
|
||||
for user_story in self.get_stories(userids):
|
||||
name = user_story.owner_username
|
||||
if profile_count is not None:
|
||||
msg = "[{0:{w}d}/{1:{w}d}] Retrieving stories from profile {2}.".format(i, profile_count, name,
|
||||
w=len(str(profile_count)))
|
||||
else:
|
||||
msg = "[{:3d}] Retrieving stories from profile {}.".format(i, name)
|
||||
self.context.log(msg)
|
||||
self.context.log("Retrieving stories from profile {}.".format(name))
|
||||
totalcount = user_story.itemcount
|
||||
count = 1
|
||||
if latest_stamps is not None:
|
||||
# pylint:disable=cell-var-from-loop
|
||||
last_scraped = latest_stamps.get_last_story_timestamp(name)
|
||||
scraped_timestamp = datetime.now().astimezone()
|
||||
for item in user_story.get_items():
|
||||
if latest_stamps is not None and item.date_local <= last_scraped:
|
||||
break
|
||||
if storyitem_filter is not None and not storyitem_filter(item):
|
||||
self.context.log("<{} skipped>".format(item), flush=True)
|
||||
continue
|
||||
@@ -872,8 +684,6 @@ class Instaloader:
|
||||
downloaded = self.download_storyitem(item, filename_target if filename_target else name)
|
||||
if fast_update and not downloaded:
|
||||
break
|
||||
if latest_stamps is not None:
|
||||
latest_stamps.set_last_story_timestamp(name, scraped_timestamp)
|
||||
|
||||
def download_storyitem(self, item: StoryItem, target: Union[str, Path]) -> bool:
|
||||
"""Download one user story.
|
||||
@@ -883,30 +693,17 @@ class Instaloader:
|
||||
:return: True if something was downloaded, False otherwise, i.e. file was already there
|
||||
"""
|
||||
|
||||
def _already_downloaded(path: str) -> bool:
|
||||
if not os.path.isfile(path):
|
||||
return False
|
||||
else:
|
||||
self.context.log(path + ' exists', end=' ', flush=True)
|
||||
return True
|
||||
|
||||
date_local = item.date_local
|
||||
dirname = _PostPathFormatter(item, self.sanitize_paths).format(self.dirname_pattern, target=target)
|
||||
dirname = _PostPathFormatter(item).format(self.dirname_pattern, target=target)
|
||||
filename_template = os.path.join(dirname, self.format_filename(item, target=target))
|
||||
filename = self.__prepare_filename(filename_template, lambda: item.url)
|
||||
downloaded = False
|
||||
video_url_fetch_failed = False
|
||||
if not item.is_video or self.download_video_thumbnails is True:
|
||||
url = item.url
|
||||
downloaded = self.download_pic(filename=filename, url=url, mtime=date_local)
|
||||
if item.is_video and self.download_videos is True:
|
||||
video_url = item.video_url
|
||||
if video_url:
|
||||
filename = self.__prepare_filename(filename_template, lambda: str(video_url))
|
||||
downloaded |= (not _already_downloaded(filename + ".mp4") and
|
||||
self.download_pic(filename=filename, url=video_url, mtime=date_local))
|
||||
else:
|
||||
video_url_fetch_failed = True
|
||||
if video_url_fetch_failed or not item.is_video or self.download_video_thumbnails is True:
|
||||
downloaded = (not _already_downloaded(filename + ".jpg") and
|
||||
self.download_pic(filename=filename, url=item.url, mtime=date_local))
|
||||
filename = self.__prepare_filename(filename_template, lambda: str(item.video_url))
|
||||
downloaded |= self.download_pic(filename=filename, url=item.video_url, mtime=date_local)
|
||||
# Save caption if desired
|
||||
metadata_string = _ArbitraryItemFormatter(item).format(self.storyitem_metadata_txt_pattern).strip()
|
||||
if metadata_string:
|
||||
@@ -962,11 +759,10 @@ class Instaloader:
|
||||
"""
|
||||
for user_highlight in self.get_highlights(user):
|
||||
name = user_highlight.owner_username
|
||||
highlight_target: Union[str, Path] = (filename_target
|
||||
highlight_target = (filename_target
|
||||
if filename_target
|
||||
else (Path(_PostPathFormatter.sanitize_path(name, self.sanitize_paths)) /
|
||||
_PostPathFormatter.sanitize_path(user_highlight.title,
|
||||
self.sanitize_paths)))
|
||||
else (Path(_PostPathFormatter.sanitize_path(name)) /
|
||||
_PostPathFormatter.sanitize_path(user_highlight.title))) # type: Union[str, Path]
|
||||
self.context.log("Retrieving highlights \"{}\" from profile {}".format(user_highlight.title, name))
|
||||
self.download_highlight_cover(user_highlight, highlight_target)
|
||||
totalcount = user_highlight.itemcount
|
||||
@@ -990,9 +786,7 @@ class Instaloader:
|
||||
post_filter: Optional[Callable[[Post], bool]] = None,
|
||||
max_count: Optional[int] = None,
|
||||
total_count: Optional[int] = None,
|
||||
owner_profile: Optional[Profile] = None,
|
||||
takewhile: Optional[Callable[[Post], bool]] = None,
|
||||
possibly_pinned: int = 0) -> None:
|
||||
owner_profile: Optional[Profile] = None) -> None:
|
||||
"""
|
||||
Download the Posts returned by given Post Iterator.
|
||||
|
||||
@@ -1001,12 +795,6 @@ class Instaloader:
|
||||
.. versionchanged:: 4.5
|
||||
Transparently resume an aborted operation if `posts` is a :class:`NodeIterator`.
|
||||
|
||||
.. versionchanged:: 4.8
|
||||
Add `takewhile` parameter.
|
||||
|
||||
.. versionchanged:: 4.10.3
|
||||
Add `possibly_pinned` parameter.
|
||||
|
||||
:param posts: Post Iterator to loop through.
|
||||
:param target: Target name.
|
||||
:param fast_update: :option:`--fast-update`.
|
||||
@@ -1014,17 +802,12 @@ class Instaloader:
|
||||
:param max_count: Maximum count of Posts to download (:option:`--count`).
|
||||
:param total_count: Total number of posts returned by given iterator.
|
||||
:param owner_profile: Associated profile, if any.
|
||||
:param takewhile: Expression evaluated for each post. Once it returns false, downloading stops.
|
||||
:param possibly_pinned: Number of posts that might be pinned. These posts do not cause download
|
||||
to stop even if they've already been downloaded.
|
||||
"""
|
||||
displayed_count = (max_count if total_count is None or max_count is not None and max_count < total_count
|
||||
else total_count)
|
||||
sanitized_target = target
|
||||
if isinstance(target, str):
|
||||
sanitized_target = _PostPathFormatter.sanitize_path(target, self.sanitize_paths)
|
||||
if takewhile is None:
|
||||
takewhile = lambda _: True
|
||||
sanitized_target = _PostPathFormatter.sanitize_path(target)
|
||||
with resumable_iteration(
|
||||
context=self.context,
|
||||
iterator=posts,
|
||||
@@ -1037,10 +820,7 @@ class Instaloader:
|
||||
enabled=self.resume_prefix is not None
|
||||
) as (is_resuming, start_index):
|
||||
for number, post in enumerate(posts, start=start_index + 1):
|
||||
should_stop = not takewhile(post)
|
||||
if should_stop and number <= possibly_pinned:
|
||||
continue
|
||||
if (max_count is not None and number > max_count) or should_stop:
|
||||
if max_count is not None and number > max_count:
|
||||
break
|
||||
if displayed_count is not None:
|
||||
self.context.log("[{0:{w}d}/{1:{w}d}] ".format(number, displayed_count,
|
||||
@@ -1072,7 +852,7 @@ class Instaloader:
|
||||
except PostChangedException:
|
||||
post_changed = True
|
||||
continue
|
||||
if fast_update and not downloaded and not post_changed and number > possibly_pinned:
|
||||
if fast_update and not downloaded and not post_changed:
|
||||
# disengage fast_update for first post when resuming
|
||||
if not is_resuming or number > 0:
|
||||
break
|
||||
@@ -1103,7 +883,7 @@ class Instaloader:
|
||||
'has_stories': False})["data"]
|
||||
|
||||
@_requires_login
|
||||
def download_feed_posts(self, max_count: Optional[int] = None, fast_update: bool = False,
|
||||
def download_feed_posts(self, max_count: int = None, fast_update: bool = False,
|
||||
post_filter: Optional[Callable[[Post], bool]] = None) -> None:
|
||||
"""
|
||||
Download pictures from the user's feed.
|
||||
@@ -1124,7 +904,7 @@ class Instaloader:
|
||||
self.posts_download_loop(self.get_feed_posts(), ":feed", fast_update, post_filter, max_count=max_count)
|
||||
|
||||
@_requires_login
|
||||
def download_saved_posts(self, max_count: Optional[int] = None, fast_update: bool = False,
|
||||
def download_saved_posts(self, max_count: int = None, fast_update: bool = False,
|
||||
post_filter: Optional[Callable[[Post], bool]] = None) -> None:
|
||||
"""Download user's saved pictures.
|
||||
|
||||
@@ -1152,12 +932,18 @@ class Instaloader:
|
||||
.. versionchanged:: 4.2.9
|
||||
Require being logged in (as required by Instagram)
|
||||
"""
|
||||
yield from SectionIterator(
|
||||
self.context,
|
||||
lambda d: d["native_location_data"]["recent"],
|
||||
lambda m: Post.from_iphone_struct(self.context, m),
|
||||
f"explore/locations/{location}/",
|
||||
)
|
||||
has_next_page = True
|
||||
end_cursor = None
|
||||
while has_next_page:
|
||||
if end_cursor:
|
||||
params = {'__a': 1, 'max_id': end_cursor}
|
||||
else:
|
||||
params = {'__a': 1}
|
||||
location_data = self.context.get_json('explore/locations/{0}/'.format(location),
|
||||
params)['graphql']['location']['edge_location_to_media']
|
||||
yield from (Post(self.context, edge['node']) for edge in location_data['edges'])
|
||||
has_next_page = location_data['page_info']['has_next_page']
|
||||
end_cursor = location_data['page_info']['end_cursor']
|
||||
|
||||
@_requires_login
|
||||
def download_location(self, location: str,
|
||||
@@ -1206,8 +992,8 @@ class Instaloader:
|
||||
"""Get Posts associated with a #hashtag.
|
||||
|
||||
.. deprecated:: 4.4
|
||||
Use :meth:`Hashtag.get_posts_resumable`."""
|
||||
return Hashtag.from_name(self.context, hashtag).get_posts_resumable()
|
||||
Use :meth:`Hashtag.get_posts`."""
|
||||
return Hashtag.from_name(self.context, hashtag).get_posts()
|
||||
|
||||
def download_hashtag(self, hashtag: Union[Hashtag, str],
|
||||
max_count: Optional[int] = None,
|
||||
@@ -1243,7 +1029,7 @@ class Instaloader:
|
||||
self.download_hashtag_profilepic(hashtag)
|
||||
if posts:
|
||||
self.context.log("Retrieving pictures with hashtag #{}...".format(hashtag.name))
|
||||
self.posts_download_loop(hashtag.get_posts_resumable(), target, fast_update, post_filter,
|
||||
self.posts_download_loop(hashtag.get_all_posts(), target, fast_update, post_filter,
|
||||
max_count=max_count)
|
||||
if self.save_metadata:
|
||||
json_filename = '{0}/{1}'.format(self.dirname_pattern.format(profile=target,
|
||||
@@ -1253,47 +1039,25 @@ class Instaloader:
|
||||
|
||||
def download_tagged(self, profile: Profile, fast_update: bool = False,
|
||||
target: Optional[str] = None,
|
||||
post_filter: Optional[Callable[[Post], bool]] = None,
|
||||
latest_stamps: Optional[LatestStamps] = None) -> None:
|
||||
post_filter: Optional[Callable[[Post], bool]] = None) -> None:
|
||||
"""Download all posts where a profile is tagged.
|
||||
|
||||
.. versionadded:: 4.1
|
||||
|
||||
.. versionchanged:: 4.8
|
||||
Add `latest_stamps` parameter."""
|
||||
.. versionadded:: 4.1"""
|
||||
self.context.log("Retrieving tagged posts for profile {}.".format(profile.username))
|
||||
posts_takewhile: Optional[Callable[[Post], bool]] = None
|
||||
if latest_stamps is not None:
|
||||
last_scraped = latest_stamps.get_last_tagged_timestamp(profile.username)
|
||||
posts_takewhile = lambda p: p.date_local > last_scraped
|
||||
tagged_posts = profile.get_tagged_posts()
|
||||
self.posts_download_loop(tagged_posts,
|
||||
self.posts_download_loop(profile.get_tagged_posts(),
|
||||
target if target
|
||||
else (Path(_PostPathFormatter.sanitize_path(profile.username, self.sanitize_paths)) /
|
||||
_PostPathFormatter.sanitize_path(':tagged', self.sanitize_paths)),
|
||||
fast_update, post_filter, takewhile=posts_takewhile)
|
||||
if latest_stamps is not None and tagged_posts.first_item is not None:
|
||||
latest_stamps.set_last_tagged_timestamp(profile.username, tagged_posts.first_item.date_local)
|
||||
else (Path(_PostPathFormatter.sanitize_path(profile.username)) /
|
||||
_PostPathFormatter.sanitize_path(':tagged')),
|
||||
fast_update, post_filter)
|
||||
|
||||
def download_igtv(self, profile: Profile, fast_update: bool = False,
|
||||
post_filter: Optional[Callable[[Post], bool]] = None,
|
||||
latest_stamps: Optional[LatestStamps] = None) -> None:
|
||||
post_filter: Optional[Callable[[Post], bool]] = None) -> None:
|
||||
"""Download IGTV videos of a profile.
|
||||
|
||||
.. versionadded:: 4.3
|
||||
|
||||
.. versionchanged:: 4.8
|
||||
Add `latest_stamps` parameter."""
|
||||
.. versionadded:: 4.3"""
|
||||
self.context.log("Retrieving IGTV videos for profile {}.".format(profile.username))
|
||||
posts_takewhile: Optional[Callable[[Post], bool]] = None
|
||||
if latest_stamps is not None:
|
||||
last_scraped = latest_stamps.get_last_igtv_timestamp(profile.username)
|
||||
posts_takewhile = lambda p: p.date_local > last_scraped
|
||||
igtv_posts = profile.get_igtv_posts()
|
||||
self.posts_download_loop(igtv_posts, profile.username, fast_update, post_filter,
|
||||
total_count=profile.igtvcount, owner_profile=profile, takewhile=posts_takewhile)
|
||||
if latest_stamps is not None and igtv_posts.first_item is not None:
|
||||
latest_stamps.set_last_igtv_timestamp(profile.username, igtv_posts.first_item.date_local)
|
||||
self.posts_download_loop(profile.get_igtv_posts(), profile.username, fast_update, post_filter,
|
||||
total_count=profile.igtvcount, owner_profile=profile)
|
||||
|
||||
def _get_id_filename(self, profile_name: str) -> str:
|
||||
if ((format_string_contains_key(self.dirname_pattern, 'profile') or
|
||||
@@ -1305,22 +1069,9 @@ class Instaloader:
|
||||
return os.path.join(self.dirname_pattern.format(),
|
||||
'{0}_id'.format(profile_name.lower()))
|
||||
|
||||
def load_profile_id(self, profile_name: str) -> Optional[int]:
|
||||
"""
|
||||
Load ID of profile from profile directory.
|
||||
|
||||
.. versionadded:: 4.8
|
||||
"""
|
||||
id_filename = self._get_id_filename(profile_name)
|
||||
try:
|
||||
with open(id_filename, 'rb') as id_file:
|
||||
return int(id_file.read())
|
||||
except (FileNotFoundError, ValueError):
|
||||
return None
|
||||
|
||||
def save_profile_id(self, profile: Profile):
|
||||
"""
|
||||
Store ID of profile on profile directory.
|
||||
Store ID of profile locally.
|
||||
|
||||
.. versionadded:: 4.0.6
|
||||
"""
|
||||
@@ -1330,18 +1081,13 @@ class Instaloader:
|
||||
text_file.write(str(profile.userid) + "\n")
|
||||
self.context.log("Stored ID {0} for profile {1}.".format(profile.userid, profile.username))
|
||||
|
||||
def check_profile_id(self, profile_name: str, latest_stamps: Optional[LatestStamps] = None) -> Profile:
|
||||
def check_profile_id(self, profile_name: str) -> Profile:
|
||||
"""
|
||||
Consult locally stored ID of profile with given name, check whether ID matches and whether name
|
||||
has changed and return current name of the profile, and store ID of profile.
|
||||
|
||||
:param profile_name: Profile name
|
||||
:param latest_stamps: Database of downloaded data. If present, IDs are retrieved from it,
|
||||
otherwise from the target directory
|
||||
:return: Instance of current profile
|
||||
|
||||
.. versionchanged:: 4.8
|
||||
Add `latest_stamps` parameter.
|
||||
"""
|
||||
profile = None
|
||||
profile_name_not_exists_err = None
|
||||
@@ -1349,11 +1095,10 @@ class Instaloader:
|
||||
profile = Profile.from_username(self.context, profile_name)
|
||||
except ProfileNotExistsException as err:
|
||||
profile_name_not_exists_err = err
|
||||
if latest_stamps is None:
|
||||
profile_id = self.load_profile_id(profile_name)
|
||||
else:
|
||||
profile_id = latest_stamps.get_profile_id(profile_name)
|
||||
if profile_id is not None:
|
||||
id_filename = self._get_id_filename(profile_name)
|
||||
try:
|
||||
with open(id_filename, 'rb') as id_file:
|
||||
profile_id = int(id_file.read())
|
||||
if (profile is None) or \
|
||||
(profile_id != profile.userid):
|
||||
if profile is not None:
|
||||
@@ -1364,31 +1109,23 @@ class Instaloader:
|
||||
profile_id))
|
||||
profile_from_id = Profile.from_id(self.context, profile_id)
|
||||
newname = profile_from_id.username
|
||||
if profile_name == newname:
|
||||
self.context.error(
|
||||
f"Warning: Profile {profile_name} could not be retrieved by its name, but by its ID.")
|
||||
return profile_from_id
|
||||
self.context.error("Profile {0} has changed its name to {1}.".format(profile_name, newname))
|
||||
if latest_stamps is None:
|
||||
if ((format_string_contains_key(self.dirname_pattern, 'profile') or
|
||||
format_string_contains_key(self.dirname_pattern, 'target'))):
|
||||
os.rename(self.dirname_pattern.format(profile=profile_name.lower(),
|
||||
target=profile_name.lower()),
|
||||
self.dirname_pattern.format(profile=newname.lower(),
|
||||
target=newname.lower()))
|
||||
else:
|
||||
os.rename('{0}/{1}_id'.format(self.dirname_pattern.format(), profile_name.lower()),
|
||||
'{0}/{1}_id'.format(self.dirname_pattern.format(), newname.lower()))
|
||||
self.context.log("Profile {0} has changed its name to {1}.".format(profile_name, newname))
|
||||
if ((format_string_contains_key(self.dirname_pattern, 'profile') or
|
||||
format_string_contains_key(self.dirname_pattern, 'target'))):
|
||||
os.rename(self.dirname_pattern.format(profile=profile_name.lower(),
|
||||
target=profile_name.lower()),
|
||||
self.dirname_pattern.format(profile=newname.lower(),
|
||||
target=newname.lower()))
|
||||
else:
|
||||
latest_stamps.rename_profile(profile_name, newname)
|
||||
os.rename('{0}/{1}_id'.format(self.dirname_pattern.format(), profile_name.lower()),
|
||||
'{0}/{1}_id'.format(self.dirname_pattern.format(), newname.lower()))
|
||||
return profile_from_id
|
||||
# profile exists and profile id matches saved id
|
||||
return profile
|
||||
except (FileNotFoundError, ValueError):
|
||||
pass
|
||||
if profile is not None:
|
||||
if latest_stamps is None:
|
||||
self.save_profile_id(profile)
|
||||
else:
|
||||
latest_stamps.save_profile_id(profile.username, profile.userid)
|
||||
self.save_profile_id(profile)
|
||||
return profile
|
||||
if profile_name_not_exists_err:
|
||||
raise profile_name_not_exists_err
|
||||
@@ -1403,8 +1140,7 @@ class Instaloader:
|
||||
fast_update: bool = False,
|
||||
post_filter: Optional[Callable[[Post], bool]] = None,
|
||||
storyitem_filter: Optional[Callable[[Post], bool]] = None,
|
||||
raise_errors: bool = False,
|
||||
latest_stamps: Optional[LatestStamps] = None):
|
||||
raise_errors: bool = False):
|
||||
"""High-level method to download set of profiles.
|
||||
|
||||
:param profiles: Set of profiles to download.
|
||||
@@ -1420,15 +1156,11 @@ class Instaloader:
|
||||
:param raise_errors:
|
||||
Whether :exc:`LoginRequiredException` and :exc:`PrivateProfileNotFollowedException` should be raised or
|
||||
catched and printed with :meth:`InstaloaderContext.error_catcher`.
|
||||
:param latest_stamps: :option:`--latest-stamps`.
|
||||
|
||||
.. versionadded:: 4.1
|
||||
|
||||
.. versionchanged:: 4.3
|
||||
Add `igtv` parameter.
|
||||
|
||||
.. versionchanged:: 4.8
|
||||
Add `latest_stamps` parameter.
|
||||
"""
|
||||
|
||||
@contextmanager
|
||||
@@ -1438,16 +1170,14 @@ class Instaloader:
|
||||
# error_handler type is Callable[[Optional[str]], ContextManager[None]] (not supported with Python 3.5.0..3.5.3)
|
||||
error_handler = _error_raiser if raise_errors else self.context.error_catcher
|
||||
|
||||
for i, profile in enumerate(profiles, start=1):
|
||||
self.context.log("[{0:{w}d}/{1:{w}d}] Downloading profile {2}".format(i, len(profiles), profile.username,
|
||||
w=len(str(len(profiles)))))
|
||||
for profile in profiles:
|
||||
with error_handler(profile.username): # type: ignore # (ignore type for Python 3.5 support)
|
||||
profile_name = profile.username
|
||||
|
||||
# Download profile picture
|
||||
if profile_pic:
|
||||
with self.context.error_catcher('Download profile picture of {}'.format(profile_name)):
|
||||
self.download_profilepic_if_new(profile, latest_stamps)
|
||||
self.download_profilepic(profile)
|
||||
|
||||
# Save metadata as JSON if desired.
|
||||
if self.save_metadata:
|
||||
@@ -1469,14 +1199,12 @@ class Instaloader:
|
||||
# Download tagged, if requested
|
||||
if tagged:
|
||||
with self.context.error_catcher('Download tagged of {}'.format(profile_name)):
|
||||
self.download_tagged(profile, fast_update=fast_update, post_filter=post_filter,
|
||||
latest_stamps=latest_stamps)
|
||||
self.download_tagged(profile, fast_update=fast_update, post_filter=post_filter)
|
||||
|
||||
# Download IGTV, if requested
|
||||
if igtv:
|
||||
with self.context.error_catcher('Download IGTV of {}'.format(profile_name)):
|
||||
self.download_igtv(profile, fast_update=fast_update, post_filter=post_filter,
|
||||
latest_stamps=latest_stamps)
|
||||
self.download_igtv(profile, fast_update=fast_update, post_filter=post_filter)
|
||||
|
||||
# Download highlights, if requested
|
||||
if highlights:
|
||||
@@ -1486,24 +1214,14 @@ class Instaloader:
|
||||
# Iterate over pictures and download them
|
||||
if posts:
|
||||
self.context.log("Retrieving posts from profile {}.".format(profile_name))
|
||||
posts_takewhile: Optional[Callable[[Post], bool]] = None
|
||||
if latest_stamps is not None:
|
||||
# pylint:disable=cell-var-from-loop
|
||||
last_scraped = latest_stamps.get_last_post_timestamp(profile_name)
|
||||
posts_takewhile = lambda p: p.date_local > last_scraped
|
||||
posts_to_download = profile.get_posts()
|
||||
self.posts_download_loop(posts_to_download, profile_name, fast_update, post_filter,
|
||||
total_count=profile.mediacount, owner_profile=profile,
|
||||
takewhile=posts_takewhile, possibly_pinned=3)
|
||||
if latest_stamps is not None and posts_to_download.first_item is not None:
|
||||
latest_stamps.set_last_post_timestamp(profile_name,
|
||||
posts_to_download.first_item.date_local)
|
||||
self.posts_download_loop(profile.get_posts(), profile_name, fast_update, post_filter,
|
||||
total_count=profile.mediacount, owner_profile=profile)
|
||||
|
||||
if stories and profiles:
|
||||
with self.context.error_catcher("Download stories"):
|
||||
self.context.log("Downloading stories")
|
||||
self.download_stories(userids=list(profiles), fast_update=fast_update, filename_target=None,
|
||||
storyitem_filter=storyitem_filter, latest_stamps=latest_stamps)
|
||||
storyitem_filter=storyitem_filter)
|
||||
|
||||
def download_profile(self, profile_name: Union[str, Profile],
|
||||
profile_pic: bool = True, profile_pic_only: bool = False,
|
||||
@@ -1602,6 +1320,5 @@ class Instaloader:
|
||||
code = input("Enter 2FA verification code: ")
|
||||
self.two_factor_login(code)
|
||||
break
|
||||
except BadCredentialsException as err:
|
||||
print(err, file=sys.stderr)
|
||||
except BadCredentialsException:
|
||||
pass
|
||||
|
||||
@@ -3,12 +3,12 @@ import json
|
||||
import os
|
||||
import pickle
|
||||
import random
|
||||
import re
|
||||
import shutil
|
||||
import sys
|
||||
import textwrap
|
||||
import time
|
||||
import urllib.parse
|
||||
import uuid
|
||||
from contextlib import contextmanager
|
||||
from datetime import datetime, timedelta
|
||||
from functools import partial
|
||||
@@ -24,44 +24,16 @@ def copy_session(session: requests.Session, request_timeout: Optional[float] = N
|
||||
"""Duplicates a requests.Session."""
|
||||
new = requests.Session()
|
||||
new.cookies = requests.utils.cookiejar_from_dict(requests.utils.dict_from_cookiejar(session.cookies))
|
||||
new.headers = session.headers.copy() # type: ignore
|
||||
new.headers = session.headers.copy()
|
||||
# Override default timeout behavior.
|
||||
# Need to silence mypy bug for this. See: https://github.com/python/mypy/issues/2427
|
||||
new.request = partial(new.request, timeout=request_timeout) # type: ignore
|
||||
new.request = partial(new.request, timeout=request_timeout) # type: ignore
|
||||
return new
|
||||
|
||||
|
||||
def default_user_agent() -> str:
|
||||
return ('Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 '
|
||||
'(KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36')
|
||||
|
||||
|
||||
def default_iphone_headers() -> Dict[str, Any]:
|
||||
return {'User-Agent': 'Instagram 273.0.0.16.70 (iPad13,8; iOS 16_3; en_US; en-US; ' \
|
||||
'scale=2.00; 2048x2732; 452417278) AppleWebKit/420+',
|
||||
'x-ads-opt-out': '1',
|
||||
'x-bloks-is-panorama-enabled': 'true',
|
||||
'x-bloks-version-id': '01507c21540f73e2216b6f62a11a5b5e51aa85491b72475c080da35b1228ddd6',
|
||||
'x-fb-client-ip': 'True',
|
||||
'x-fb-connection-type': 'wifi',
|
||||
'x-fb-http-engine': 'Liger',
|
||||
'x-fb-server-cluster': 'True',
|
||||
'x-fb': '1',
|
||||
'x-ig-abr-connection-speed-kbps': '2',
|
||||
'x-ig-app-id': '124024574287414',
|
||||
'x-ig-app-locale': 'en-US',
|
||||
'x-ig-app-startup-country': 'US',
|
||||
'x-ig-bandwidth-speed-kbps': '0.000',
|
||||
'x-ig-capabilities': '36r/F/8=',
|
||||
'x-ig-connection-speed': '{}kbps'.format(random.randint(1000, 20000)),
|
||||
'x-ig-connection-type': 'WiFi',
|
||||
'x-ig-device-locale': 'en-US',
|
||||
'x-ig-mapped-locale': 'en-US',
|
||||
'x-ig-timezone-offset': str((datetime.now().astimezone().utcoffset() or timedelta(seconds=0)).seconds),
|
||||
'x-ig-www-claim': '0',
|
||||
'x-pigeon-session-id': str(uuid.uuid4()),
|
||||
'x-tigon-is-retry': 'False',
|
||||
'x-whatsapp': '0'}
|
||||
return 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 ' \
|
||||
'(KHTML, like Gecko) Chrome/51.0.2704.79 Safari/537.36'
|
||||
|
||||
|
||||
class InstaloaderContext:
|
||||
@@ -81,56 +53,42 @@ class InstaloaderContext:
|
||||
|
||||
def __init__(self, sleep: bool = True, quiet: bool = False, user_agent: Optional[str] = None,
|
||||
max_connection_attempts: int = 3, request_timeout: float = 300.0,
|
||||
rate_controller: Optional[Callable[["InstaloaderContext"], "RateController"]] = None,
|
||||
fatal_status_codes: Optional[List[int]] = None,
|
||||
iphone_support: bool = True):
|
||||
rate_controller: Optional[Callable[["InstaloaderContext"], "RateController"]] = None):
|
||||
|
||||
self.user_agent = user_agent if user_agent is not None else default_user_agent()
|
||||
self.request_timeout = request_timeout
|
||||
self._session = self.get_anonymous_session()
|
||||
self.username = None
|
||||
self.user_id = None
|
||||
self.sleep = sleep
|
||||
self.quiet = quiet
|
||||
self.max_connection_attempts = max_connection_attempts
|
||||
self._graphql_page_length = 50
|
||||
self._root_rhx_gis = None
|
||||
self.two_factor_auth_pending = None
|
||||
self.iphone_support = iphone_support
|
||||
self.iphone_headers = default_iphone_headers()
|
||||
|
||||
# error log, filled with error() and printed at the end of Instaloader.main()
|
||||
self.error_log: List[str] = []
|
||||
self.error_log = [] # type: List[str]
|
||||
|
||||
self._rate_controller = rate_controller(self) if rate_controller is not None else RateController(self)
|
||||
|
||||
# Can be set to True for testing, disables supression of InstaloaderContext._error_catcher
|
||||
self.raise_all_errors = False
|
||||
|
||||
# HTTP status codes that should cause an AbortDownloadException
|
||||
self.fatal_status_codes = fatal_status_codes or []
|
||||
|
||||
# Cache profile from id (mapping from id to Profile)
|
||||
self.profile_id_cache: Dict[int, Any] = dict()
|
||||
self.profile_id_cache = dict() # type: Dict[int, Any]
|
||||
|
||||
@contextmanager
|
||||
def anonymous_copy(self):
|
||||
session = self._session
|
||||
username = self.username
|
||||
user_id = self.user_id
|
||||
iphone_headers = self.iphone_headers
|
||||
self._session = self.get_anonymous_session()
|
||||
self.username = None
|
||||
self.user_id = None
|
||||
self.iphone_headers = default_iphone_headers()
|
||||
try:
|
||||
yield self
|
||||
finally:
|
||||
self._session.close()
|
||||
self.username = username
|
||||
self._session = session
|
||||
self.user_id = user_id
|
||||
self.iphone_headers = iphone_headers
|
||||
|
||||
@property
|
||||
def is_logged_in(self) -> bool:
|
||||
@@ -190,6 +148,7 @@ class InstaloaderContext:
|
||||
if empty_session_only:
|
||||
del header['Host']
|
||||
del header['Origin']
|
||||
del header['Referer']
|
||||
del header['X-Instagram-AJAX']
|
||||
del header['X-Requested-With']
|
||||
return header
|
||||
@@ -206,34 +165,22 @@ class InstaloaderContext:
|
||||
session.request = partial(session.request, timeout=self.request_timeout) # type: ignore
|
||||
return session
|
||||
|
||||
def save_session(self):
|
||||
"""Not meant to be used directly, use :meth:`Instaloader.save_session`."""
|
||||
return requests.utils.dict_from_cookiejar(self._session.cookies)
|
||||
def save_session_to_file(self, sessionfile):
|
||||
"""Not meant to be used directly, use :meth:`Instaloader.save_session_to_file`."""
|
||||
pickle.dump(requests.utils.dict_from_cookiejar(self._session.cookies), sessionfile)
|
||||
|
||||
def update_cookies(self, cookie):
|
||||
""".. versionadded:: 4.11"""
|
||||
self._session.cookies.update(cookie)
|
||||
|
||||
def load_session(self, username, sessiondata):
|
||||
"""Not meant to be used directly, use :meth:`Instaloader.load_session`."""
|
||||
def load_session_from_file(self, username, sessionfile):
|
||||
"""Not meant to be used directly, use :meth:`Instaloader.load_session_from_file`."""
|
||||
session = requests.Session()
|
||||
session.cookies = requests.utils.cookiejar_from_dict(sessiondata)
|
||||
session.cookies = requests.utils.cookiejar_from_dict(pickle.load(sessionfile))
|
||||
session.headers.update(self._default_http_header())
|
||||
session.headers.update({'X-CSRFToken': session.cookies.get_dict()['csrftoken']})
|
||||
# Override default timeout behavior.
|
||||
# Need to silence mypy bug for this. See: https://github.com/python/mypy/issues/2427
|
||||
session.request = partial(session.request, timeout=self.request_timeout) # type: ignore
|
||||
session.request = partial(session.request, timeout=self.request_timeout) # type: ignore
|
||||
self._session = session
|
||||
self.username = username
|
||||
|
||||
def save_session_to_file(self, sessionfile):
|
||||
"""Not meant to be used directly, use :meth:`Instaloader.save_session_to_file`."""
|
||||
pickle.dump(self.save_session(), sessionfile)
|
||||
|
||||
def load_session_from_file(self, username, sessionfile):
|
||||
"""Not meant to be used directly, use :meth:`Instaloader.load_session_from_file`."""
|
||||
self.load_session(username, pickle.load(sessionfile))
|
||||
|
||||
def test_login(self) -> Optional[str]:
|
||||
"""Not meant to be used directly, use :meth:`Instaloader.test_login`."""
|
||||
data = self.graphql_query("d6f4427fbe92d846298cf93df0b937d3", {})
|
||||
@@ -259,27 +206,20 @@ class InstaloaderContext:
|
||||
# Override default timeout behavior.
|
||||
# Need to silence mypy bug for this. See: https://github.com/python/mypy/issues/2427
|
||||
session.request = partial(session.request, timeout=self.request_timeout) # type: ignore
|
||||
|
||||
# Make a request to Instagram's root URL, which will set the session's csrftoken cookie
|
||||
# Not using self.get_json() here, because we need to access the cookie
|
||||
session.get('https://www.instagram.com/')
|
||||
# Add session's csrftoken cookie to session headers
|
||||
session.get('https://www.instagram.com/web/__mid/')
|
||||
csrf_token = session.cookies.get_dict()['csrftoken']
|
||||
session.headers.update({'X-CSRFToken': csrf_token})
|
||||
|
||||
# Not using self.get_json() here, because we need to access csrftoken cookie
|
||||
self.do_sleep()
|
||||
# Workaround credits to pgrimaud.
|
||||
# See: https://github.com/pgrimaud/instagram-user-feed/commit/96ad4cf54d1ad331b337f325c73e664999a6d066
|
||||
enc_password = '#PWD_INSTAGRAM_BROWSER:0:{}:{}'.format(int(datetime.now().timestamp()), passwd)
|
||||
login = session.post('https://www.instagram.com/api/v1/web/accounts/login/ajax/',
|
||||
login = session.post('https://www.instagram.com/accounts/login/ajax/',
|
||||
data={'enc_password': enc_password, 'username': user}, allow_redirects=True)
|
||||
try:
|
||||
resp_json = login.json()
|
||||
|
||||
except json.decoder.JSONDecodeError as err:
|
||||
raise ConnectionException(
|
||||
"Login error: JSON decode fail, {} - {}.".format(login.status_code, login.reason)
|
||||
) from err
|
||||
except json.decoder.JSONDecodeError:
|
||||
raise ConnectionException("Login error: JSON decode fail, {} - {}.".format(login.status_code, login.reason))
|
||||
if resp_json.get('two_factor_required'):
|
||||
two_factor_session = copy_session(session, self.request_timeout)
|
||||
two_factor_session.headers.update({'X-CSRFToken': csrf_token})
|
||||
@@ -318,7 +258,6 @@ class InstaloaderContext:
|
||||
session.headers.update({'X-CSRFToken': login.cookies['csrftoken']})
|
||||
self._session = session
|
||||
self.username = user
|
||||
self.user_id = resp_json['userId']
|
||||
|
||||
def two_factor_login(self, two_factor_code):
|
||||
"""Second step of login if 2FA is enabled.
|
||||
@@ -338,9 +277,9 @@ class InstaloaderContext:
|
||||
resp_json = login.json()
|
||||
if resp_json['status'] != 'ok':
|
||||
if 'message' in resp_json:
|
||||
raise BadCredentialsException("2FA error: {}".format(resp_json['message']))
|
||||
raise BadCredentialsException("Login error: {}".format(resp_json['message']))
|
||||
else:
|
||||
raise BadCredentialsException("2FA error: \"{}\" status.".format(resp_json['status']))
|
||||
raise BadCredentialsException("Login error: \"{}\" status.".format(resp_json['status']))
|
||||
session.headers.update({'X-CSRFToken': login.cookies['csrftoken']})
|
||||
self._session = session
|
||||
self.username = user
|
||||
@@ -349,11 +288,10 @@ class InstaloaderContext:
|
||||
def do_sleep(self):
|
||||
"""Sleep a short time if self.sleep is set. Called before each request to instagram.com."""
|
||||
if self.sleep:
|
||||
time.sleep(min(random.expovariate(0.6), 15.0))
|
||||
time.sleep(min(random.expovariate(0.7), 5.0))
|
||||
|
||||
def get_json(self, path: str, params: Dict[str, Any], host: str = 'www.instagram.com',
|
||||
session: Optional[requests.Session] = None, _attempt=1,
|
||||
response_headers: Optional[Dict[str, Any]] = None) -> Dict[str, Any]:
|
||||
session: Optional[requests.Session] = None, _attempt=1) -> Dict[str, Any]:
|
||||
"""JSON request to Instagram.
|
||||
|
||||
:param path: URL, relative to the given domain which defaults to www.instagram.com/
|
||||
@@ -378,31 +316,19 @@ class InstaloaderContext:
|
||||
if is_other_query:
|
||||
self._rate_controller.wait_before_query('other')
|
||||
resp = sess.get('https://{0}/{1}'.format(host, path), params=params, allow_redirects=False)
|
||||
if resp.status_code in self.fatal_status_codes:
|
||||
redirect = " redirect to {}".format(resp.headers['location']) if 'location' in resp.headers else ""
|
||||
body = ""
|
||||
if resp.headers['Content-Type'].startswith('application/json'):
|
||||
body = ': ' + resp.text[:500] + ('…' if len(resp.text) > 501 else '')
|
||||
raise AbortDownloadException("Query to https://{}/{} responded with \"{} {}\"{}{}".format(
|
||||
host, path, resp.status_code, resp.reason, redirect, body
|
||||
))
|
||||
while resp.is_redirect:
|
||||
redirect_url = resp.headers['location']
|
||||
self.log('\nHTTP redirect from https://{0}/{1} to {2}'.format(host, path, redirect_url))
|
||||
if (redirect_url.startswith('https://www.instagram.com/accounts/login') or
|
||||
redirect_url.startswith('https://i.instagram.com/accounts/login')):
|
||||
if redirect_url.startswith('https://www.instagram.com/accounts/login'):
|
||||
if not self.is_logged_in:
|
||||
raise LoginRequiredException("Redirected to login page. Use --login.")
|
||||
raise AbortDownloadException("Redirected to login page. You've been logged out, please wait " +
|
||||
"some time, recreate the session and try again")
|
||||
# alternate rate limit exceeded behavior
|
||||
raise TooManyRequestsException("Redirected to login")
|
||||
if redirect_url.startswith('https://{}/'.format(host)):
|
||||
resp = sess.get(redirect_url if redirect_url.endswith('/') else redirect_url + '/',
|
||||
params=params, allow_redirects=False)
|
||||
else:
|
||||
break
|
||||
if response_headers is not None:
|
||||
response_headers.clear()
|
||||
response_headers.update(resp.headers)
|
||||
if resp.status_code == 400:
|
||||
raise QueryReturnedBadRequestException("400 Bad Request")
|
||||
if resp.status_code == 404:
|
||||
@@ -411,6 +337,23 @@ class InstaloaderContext:
|
||||
raise TooManyRequestsException("429 Too Many Requests")
|
||||
if resp.status_code != 200:
|
||||
raise ConnectionException("HTTP error code {}.".format(resp.status_code))
|
||||
is_html_query = not is_graphql_query and not "__a" in params and host == "www.instagram.com"
|
||||
if is_html_query:
|
||||
match = re.search(r'window\._sharedData = (.*);</script>', resp.text)
|
||||
if match is None:
|
||||
raise QueryReturnedNotFoundException("Could not find \"window._sharedData\" in html response.")
|
||||
resp_json = json.loads(match.group(1))
|
||||
entry_data = resp_json.get('entry_data')
|
||||
post_or_profile_page = list(entry_data.values())[0] if entry_data is not None else None
|
||||
if post_or_profile_page is None:
|
||||
raise ConnectionException("\"window._sharedData\" does not contain required keys.")
|
||||
# If GraphQL data is missing in `window._sharedData`, search for it in `__additionalDataLoaded`.
|
||||
if 'graphql' not in post_or_profile_page[0]:
|
||||
match = re.search(r'window\.__additionalDataLoaded\(.*?({.*"graphql":.*})\);</script>',
|
||||
resp.text)
|
||||
if match is not None:
|
||||
post_or_profile_page[0]['graphql'] = json.loads(match.group(1))['graphql']
|
||||
return resp_json
|
||||
else:
|
||||
resp_json = resp.json()
|
||||
if 'status' in resp_json and resp_json['status'] != "ok":
|
||||
@@ -436,8 +379,7 @@ class InstaloaderContext:
|
||||
self._rate_controller.handle_429('iphone')
|
||||
if is_other_query:
|
||||
self._rate_controller.handle_429('other')
|
||||
return self.get_json(path=path, params=params, host=host, session=sess, _attempt=_attempt + 1,
|
||||
response_headers=response_headers)
|
||||
return self.get_json(path=path, params=params, host=host, session=sess, _attempt=_attempt + 1)
|
||||
except KeyboardInterrupt:
|
||||
self.error("[skipped by user]", repeat_at_end=False)
|
||||
raise ConnectionException(error_string) from err
|
||||
@@ -527,57 +469,11 @@ class InstaloaderContext:
|
||||
|
||||
.. versionadded:: 4.2.1"""
|
||||
with copy_session(self._session, self.request_timeout) as tempsession:
|
||||
# Set headers to simulate an API request from iPad
|
||||
tempsession.headers['ig-intended-user-id'] = str(self.user_id)
|
||||
tempsession.headers['x-pigeon-rawclienttime'] = '{:.6f}'.format(time.time())
|
||||
|
||||
# Add headers obtained from previous iPad request
|
||||
tempsession.headers.update(self.iphone_headers)
|
||||
|
||||
# Extract key information from cookies if we haven't got it already from a previous request
|
||||
header_cookies_mapping = {'x-mid': 'mid',
|
||||
'ig-u-ds-user-id': 'ds_user_id',
|
||||
'x-ig-device-id': 'ig_did',
|
||||
'x-ig-family-device-id': 'ig_did',
|
||||
'family_device_id': 'ig_did'}
|
||||
|
||||
# Map the cookie value to the matching HTTP request header
|
||||
cookies = tempsession.cookies.get_dict().copy()
|
||||
for key, value in header_cookies_mapping.items():
|
||||
if value in cookies:
|
||||
if key not in tempsession.headers:
|
||||
tempsession.headers[key] = cookies[value]
|
||||
else:
|
||||
# Remove the cookie value if it's already specified as a header
|
||||
tempsession.cookies.pop(value, None)
|
||||
|
||||
# Edge case for ig-u-rur header due to special string encoding in cookie
|
||||
if 'rur' in cookies:
|
||||
if 'ig-u-rur' not in tempsession.headers:
|
||||
tempsession.headers['ig-u-rur'] = cookies['rur'].strip('\"').encode('utf-8') \
|
||||
.decode('unicode_escape')
|
||||
else:
|
||||
tempsession.cookies.pop('rur', None)
|
||||
|
||||
# Remove headers specific to Desktop version
|
||||
for header in ['Host', 'Origin', 'X-Instagram-AJAX', 'X-Requested-With', 'Referer']:
|
||||
tempsession.headers['User-Agent'] = 'Instagram 146.0.0.27.125 (iPhone12,1; iOS 13_3; en_US; en-US; ' \
|
||||
'scale=2.00; 1656x3584; 190542906)'
|
||||
for header in ['Host', 'Origin', 'X-Instagram-AJAX', 'X-Requested-With']:
|
||||
tempsession.headers.pop(header, None)
|
||||
|
||||
# No need for cookies if we have a bearer token
|
||||
if 'authorization' in tempsession.headers:
|
||||
tempsession.cookies.clear()
|
||||
|
||||
response_headers = dict() # type: Dict[str, Any]
|
||||
response = self.get_json(path, params, 'i.instagram.com', tempsession, response_headers=response_headers)
|
||||
|
||||
# Extract the ig-set-* headers and use them in the next request
|
||||
for key, value in response_headers.items():
|
||||
if key.startswith('ig-set-'):
|
||||
self.iphone_headers[key.replace('ig-set-', '')] = value
|
||||
elif key.startswith('x-ig-set-'):
|
||||
self.iphone_headers[key.replace('x-ig-set-', 'x-ig-')] = value
|
||||
|
||||
return response
|
||||
return self.get_json(path, params, 'i.instagram.com', tempsession)
|
||||
|
||||
def write_raw(self, resp: Union[bytes, requests.Response], filename: str) -> None:
|
||||
"""Write raw response data into a file.
|
||||
@@ -621,28 +517,6 @@ class InstaloaderContext:
|
||||
:raises ConnectionException: When download repeatedly failed."""
|
||||
self.write_raw(self.get_raw(url), filename)
|
||||
|
||||
def head(self, url: str, allow_redirects: bool = False) -> requests.Response:
|
||||
"""HEAD a URL anonymously.
|
||||
|
||||
:raises QueryReturnedNotFoundException: When the server responds with a 404.
|
||||
:raises QueryReturnedForbiddenException: When the server responds with a 403.
|
||||
:raises ConnectionException: When request failed.
|
||||
|
||||
.. versionadded:: 4.7.6
|
||||
"""
|
||||
with self.get_anonymous_session() as anonymous_session:
|
||||
resp = anonymous_session.head(url, allow_redirects=allow_redirects)
|
||||
if resp.status_code == 200:
|
||||
return resp
|
||||
else:
|
||||
if resp.status_code == 403:
|
||||
# suspected invalid URL signature
|
||||
raise QueryReturnedForbiddenException("403 when accessing {}.".format(url))
|
||||
if resp.status_code == 404:
|
||||
# 404 not worth retrying.
|
||||
raise QueryReturnedNotFoundException("404 when accessing {}.".format(url))
|
||||
raise ConnectionException("HTTP error code {}.".format(resp.status_code))
|
||||
|
||||
@property
|
||||
def root_rhx_gis(self) -> Optional[str]:
|
||||
"""rhx_gis string returned in the / query."""
|
||||
@@ -673,19 +547,19 @@ class RateController:
|
||||
|
||||
def __init__(self, context: InstaloaderContext):
|
||||
self._context = context
|
||||
self._query_timestamps: Dict[str, List[float]] = dict()
|
||||
self._query_timestamps = dict() # type: Dict[str, List[float]]
|
||||
self._earliest_next_request_time = 0.0
|
||||
self._iphone_earliest_next_request_time = 0.0
|
||||
|
||||
def sleep(self, secs: float):
|
||||
"""Wait given number of seconds."""
|
||||
# Not static, to allow for the behavior of this method to depend on context-inherent properties, such as
|
||||
# whether we are logged in.
|
||||
# pylint:disable=no-self-use
|
||||
time.sleep(secs)
|
||||
|
||||
def _dump_query_timestamps(self, current_time: float, failed_query_type: str):
|
||||
windows = [10, 11, 20, 22, 30, 60]
|
||||
self._context.error("Number of requests within last {} minutes grouped by type:"
|
||||
self._context.error("Requests within last {} minutes grouped by type:"
|
||||
.format('/'.join(str(w) for w in windows)),
|
||||
repeat_at_end=False)
|
||||
for query_type, times in self._query_timestamps.items():
|
||||
@@ -697,14 +571,11 @@ class RateController:
|
||||
), repeat_at_end=False)
|
||||
|
||||
def count_per_sliding_window(self, query_type: str) -> int:
|
||||
"""Return how many requests of the given type can be done within a sliding window of 11 minutes.
|
||||
|
||||
This is called by :meth:`RateController.query_waittime` and allows to simply customize wait times before queries
|
||||
at query_type granularity. Consider overriding :meth:`RateController.query_waittime` directly if you need more
|
||||
control."""
|
||||
"""Return how many requests can be done within the sliding window."""
|
||||
# Not static, to allow for the count_per_sliding_window to depend on context-inherent properties, such as
|
||||
# whether we are logged in.
|
||||
return 75 if query_type == 'other' else 200
|
||||
# pylint:disable=no-self-use
|
||||
return 75 if query_type in ['iphone', 'other'] else 200
|
||||
|
||||
def _reqs_in_sliding_window(self, query_type: Optional[str], current_time: float, window: float) -> List[float]:
|
||||
if query_type is not None:
|
||||
@@ -720,7 +591,6 @@ class RateController:
|
||||
def query_waittime(self, query_type: str, current_time: float, untracked_queries: bool = False) -> float:
|
||||
"""Calculate time needed to wait before query can be executed."""
|
||||
per_type_sliding_window = 660
|
||||
iphone_sliding_window = 1800
|
||||
if query_type not in self._query_timestamps:
|
||||
self._query_timestamps[query_type] = []
|
||||
self._query_timestamps[query_type] = list(filter(lambda t: t > current_time - 60 * 60,
|
||||
@@ -746,43 +616,25 @@ class RateController:
|
||||
|
||||
def untracked_next_request_time():
|
||||
if untracked_queries:
|
||||
if query_type == "iphone":
|
||||
reqs_in_sliding_window = self._reqs_in_sliding_window(query_type, current_time,
|
||||
iphone_sliding_window)
|
||||
self._iphone_earliest_next_request_time = min(reqs_in_sliding_window) + iphone_sliding_window + 18
|
||||
else:
|
||||
reqs_in_sliding_window = self._reqs_in_sliding_window(query_type, current_time,
|
||||
per_type_sliding_window)
|
||||
self._earliest_next_request_time = min(reqs_in_sliding_window) + per_type_sliding_window + 6
|
||||
return max(self._iphone_earliest_next_request_time, self._earliest_next_request_time)
|
||||
|
||||
def iphone_next_request():
|
||||
if query_type == "iphone":
|
||||
reqs_in_sliding_window = self._reqs_in_sliding_window(query_type, current_time, iphone_sliding_window)
|
||||
if len(reqs_in_sliding_window) >= 199:
|
||||
return min(reqs_in_sliding_window) + iphone_sliding_window + 18
|
||||
return 0.0
|
||||
reqs_in_sliding_window = self._reqs_in_sliding_window(query_type, current_time, per_type_sliding_window)
|
||||
self._earliest_next_request_time = min(reqs_in_sliding_window) + per_type_sliding_window + 6
|
||||
return self._earliest_next_request_time
|
||||
|
||||
return max(0.0,
|
||||
max(
|
||||
per_type_next_request_time(),
|
||||
gql_accumulated_next_request_time(),
|
||||
untracked_next_request_time(),
|
||||
iphone_next_request(),
|
||||
) - current_time)
|
||||
|
||||
def wait_before_query(self, query_type: str) -> None:
|
||||
"""This method is called before a query to Instagram.
|
||||
|
||||
It calls :meth:`RateController.query_waittime` to determine the time needed to wait and then calls
|
||||
:meth:`RateController.sleep` to wait until the request can be made."""
|
||||
"""This method is called before a query to Instagram. It calls :meth:`RateController.sleep` to wait
|
||||
until the request can be made."""
|
||||
waittime = self.query_waittime(query_type, time.monotonic(), False)
|
||||
assert waittime >= 0
|
||||
if waittime > 15:
|
||||
formatted_waittime = ("{} seconds".format(round(waittime)) if waittime <= 666 else
|
||||
"{} minutes".format(round(waittime / 60)))
|
||||
self._context.log("\nToo many queries in the last time. Need to wait {}, until {:%H:%M}."
|
||||
.format(formatted_waittime, datetime.now() + timedelta(seconds=waittime)))
|
||||
self._context.log("\nToo many queries in the last time. Need to wait {} seconds, until {:%H:%M}."
|
||||
.format(round(waittime), datetime.now() + timedelta(seconds=waittime)))
|
||||
if waittime > 0:
|
||||
self.sleep(waittime)
|
||||
if query_type not in self._query_timestamps:
|
||||
@@ -791,10 +643,8 @@ class RateController:
|
||||
self._query_timestamps[query_type].append(time.monotonic())
|
||||
|
||||
def handle_429(self, query_type: str) -> None:
|
||||
"""This method is called to handle a 429 Too Many Requests response.
|
||||
|
||||
It calls :meth:`RateController.query_waittime` to determine the time needed to wait and then calls
|
||||
:meth:`RateController.sleep` to wait until we can repeat the same request."""
|
||||
"""This method is called to handle a 429 Too Many Requests response. It calls :meth:`RateController.sleep` to
|
||||
wait until we can repeat the same request."""
|
||||
current_time = time.monotonic()
|
||||
waittime = self.query_waittime(query_type, current_time, True)
|
||||
assert waittime >= 0
|
||||
@@ -804,10 +654,8 @@ class RateController:
|
||||
"App while Instaloader is running.")
|
||||
self._context.error(textwrap.fill(text_for_429), repeat_at_end=False)
|
||||
if waittime > 1.5:
|
||||
formatted_waittime = ("{} seconds".format(round(waittime)) if waittime <= 666 else
|
||||
"{} minutes".format(round(waittime / 60)))
|
||||
self._context.error("The request will be retried in {}, at {:%H:%M}."
|
||||
.format(formatted_waittime, datetime.now() + timedelta(seconds=waittime)),
|
||||
self._context.error("The request will be retried in {} seconds, at {:%H:%M}."
|
||||
.format(round(waittime), datetime.now() + timedelta(seconds=waittime)),
|
||||
repeat_at_end=False)
|
||||
if waittime > 0:
|
||||
self.sleep(waittime)
|
||||
|
||||
@@ -1,117 +0,0 @@
|
||||
import configparser
|
||||
from datetime import datetime, timezone
|
||||
from typing import Optional
|
||||
from os.path import dirname
|
||||
from os import makedirs
|
||||
|
||||
|
||||
class LatestStamps:
|
||||
"""LatestStamps class.
|
||||
|
||||
Convenience class for retrieving and storing data from the :option:`--latest-stamps` file.
|
||||
|
||||
:param latest_stamps_file: path to file.
|
||||
|
||||
.. versionadded:: 4.8"""
|
||||
PROFILE_ID = 'profile-id'
|
||||
PROFILE_PIC = 'profile-pic'
|
||||
POST_TIMESTAMP = 'post-timestamp'
|
||||
TAGGED_TIMESTAMP = 'tagged-timestamp'
|
||||
IGTV_TIMESTAMP = 'igtv-timestamp'
|
||||
STORY_TIMESTAMP = 'story-timestamp'
|
||||
ISO_FORMAT = '%Y-%m-%dT%H:%M:%S.%f%z'
|
||||
|
||||
def __init__(self, latest_stamps_file):
|
||||
self.file = latest_stamps_file
|
||||
self.data = configparser.ConfigParser()
|
||||
self.data.read(latest_stamps_file)
|
||||
|
||||
def _save(self):
|
||||
if dn := dirname(self.file):
|
||||
makedirs(dn, exist_ok=True)
|
||||
with open(self.file, 'w') as f:
|
||||
self.data.write(f)
|
||||
|
||||
def _ensure_section(self, section: str):
|
||||
if not self.data.has_section(section):
|
||||
self.data.add_section(section)
|
||||
|
||||
def get_profile_id(self, profile_name: str) -> Optional[int]:
|
||||
"""Returns stored ID of profile."""
|
||||
try:
|
||||
return self.data.getint(profile_name, self.PROFILE_ID)
|
||||
except (configparser.Error, ValueError):
|
||||
return None
|
||||
|
||||
def save_profile_id(self, profile_name: str, profile_id: int):
|
||||
"""Stores ID of profile."""
|
||||
self._ensure_section(profile_name)
|
||||
self.data.set(profile_name, self.PROFILE_ID, str(profile_id))
|
||||
self._save()
|
||||
|
||||
def rename_profile(self, old_profile: str, new_profile: str):
|
||||
"""Renames a profile."""
|
||||
self._ensure_section(new_profile)
|
||||
for option in [self.PROFILE_ID, self.PROFILE_PIC, self.POST_TIMESTAMP,
|
||||
self.TAGGED_TIMESTAMP, self.IGTV_TIMESTAMP, self.STORY_TIMESTAMP]:
|
||||
if self.data.has_option(old_profile, option):
|
||||
value = self.data.get(old_profile, option)
|
||||
self.data.set(new_profile, option, value)
|
||||
self.data.remove_section(old_profile)
|
||||
self._save()
|
||||
|
||||
def _get_timestamp(self, section: str, key: str) -> datetime:
|
||||
try:
|
||||
return datetime.strptime(self.data.get(section, key), self.ISO_FORMAT)
|
||||
except (configparser.Error, ValueError):
|
||||
return datetime.fromtimestamp(0, timezone.utc)
|
||||
|
||||
def _set_timestamp(self, section: str, key: str, timestamp: datetime):
|
||||
self._ensure_section(section)
|
||||
self.data.set(section, key, timestamp.strftime(self.ISO_FORMAT))
|
||||
self._save()
|
||||
|
||||
def get_last_post_timestamp(self, profile_name: str) -> datetime:
|
||||
"""Returns timestamp of last download of a profile's posts."""
|
||||
return self._get_timestamp(profile_name, self.POST_TIMESTAMP)
|
||||
|
||||
def set_last_post_timestamp(self, profile_name: str, timestamp: datetime):
|
||||
"""Sets timestamp of last download of a profile's posts."""
|
||||
self._set_timestamp(profile_name, self.POST_TIMESTAMP, timestamp)
|
||||
|
||||
def get_last_tagged_timestamp(self, profile_name: str) -> datetime:
|
||||
"""Returns timestamp of last download of a profile's tagged posts."""
|
||||
return self._get_timestamp(profile_name, self.TAGGED_TIMESTAMP)
|
||||
|
||||
def set_last_tagged_timestamp(self, profile_name: str, timestamp: datetime):
|
||||
"""Sets timestamp of last download of a profile's tagged posts."""
|
||||
self._set_timestamp(profile_name, self.TAGGED_TIMESTAMP, timestamp)
|
||||
|
||||
def get_last_igtv_timestamp(self, profile_name: str) -> datetime:
|
||||
"""Returns timestamp of last download of a profile's igtv posts."""
|
||||
return self._get_timestamp(profile_name, self.IGTV_TIMESTAMP)
|
||||
|
||||
def set_last_igtv_timestamp(self, profile_name: str, timestamp: datetime):
|
||||
"""Sets timestamp of last download of a profile's igtv posts."""
|
||||
self._set_timestamp(profile_name, self.IGTV_TIMESTAMP, timestamp)
|
||||
|
||||
def get_last_story_timestamp(self, profile_name: str) -> datetime:
|
||||
"""Returns timestamp of last download of a profile's stories."""
|
||||
return self._get_timestamp(profile_name, self.STORY_TIMESTAMP)
|
||||
|
||||
def set_last_story_timestamp(self, profile_name: str, timestamp: datetime):
|
||||
"""Sets timestamp of last download of a profile's stories."""
|
||||
self._set_timestamp(profile_name, self.STORY_TIMESTAMP, timestamp)
|
||||
|
||||
def get_profile_pic(self, profile_name: str) -> str:
|
||||
"""Returns filename of profile's last downloaded profile pic."""
|
||||
try:
|
||||
return self.data.get(profile_name, self.PROFILE_PIC)
|
||||
except configparser.Error:
|
||||
return ""
|
||||
|
||||
def set_profile_pic(self, profile_name: str, profile_pic: str):
|
||||
"""Sets filename of profile's last downloaded profile pic."""
|
||||
self._ensure_section(profile_name)
|
||||
self.data.set(profile_name, self.PROFILE_PIC, profile_pic)
|
||||
self._save()
|
||||
+34
-69
@@ -5,20 +5,21 @@ import os
|
||||
from contextlib import contextmanager
|
||||
from datetime import datetime, timedelta
|
||||
from lzma import LZMAError
|
||||
from typing import Any, Callable, Dict, Iterable, Iterator, NamedTuple, Optional, Tuple, TypeVar
|
||||
from typing import Any, Callable, Dict, Iterator, NamedTuple, Optional, Tuple, TypeVar
|
||||
|
||||
from .exceptions import AbortDownloadException, InvalidArgumentException, QueryReturnedBadRequestException
|
||||
from .exceptions import InvalidArgumentException, QueryReturnedBadRequestException
|
||||
from .instaloadercontext import InstaloaderContext
|
||||
|
||||
class FrozenNodeIterator(NamedTuple):
|
||||
query_hash: str
|
||||
query_variables: Dict
|
||||
query_referer: Optional[str]
|
||||
context_username: Optional[str]
|
||||
total_index: int
|
||||
best_before: Optional[float]
|
||||
remaining_data: Optional[Dict]
|
||||
first_node: Optional[Dict]
|
||||
FrozenNodeIterator = NamedTuple('FrozenNodeIterator',
|
||||
[('query_hash', str),
|
||||
('query_variables', Dict),
|
||||
('query_referer', Optional[str]),
|
||||
('context_username', Optional[str]),
|
||||
('total_index', int),
|
||||
('best_before', Optional[float]),
|
||||
('remaining_data', Optional[Dict])])
|
||||
FrozenNodeIterator.__doc__ = \
|
||||
"""A serializable representation of a :class:`NodeIterator` instance, saving its iteration state."""
|
||||
FrozenNodeIterator.query_hash.__doc__ = """The GraphQL ``query_hash`` parameter."""
|
||||
FrozenNodeIterator.query_variables.__doc__ = """The GraphQL ``query_variables`` parameter."""
|
||||
FrozenNodeIterator.query_referer.__doc__ = """The HTTP referer used for the GraphQL query."""
|
||||
@@ -27,7 +28,7 @@ FrozenNodeIterator.total_index.__doc__ = """Number of items that have already be
|
||||
FrozenNodeIterator.best_before.__doc__ = """Date when parts of the stored nodes might have expired."""
|
||||
FrozenNodeIterator.remaining_data.__doc__ = \
|
||||
"""The already-retrieved, yet-unprocessed ``edges`` and the ``page_info`` at time of freezing."""
|
||||
FrozenNodeIterator.first_node.__doc__ = """Node data of the first item, if an item has been produced."""
|
||||
|
||||
|
||||
T = TypeVar('T')
|
||||
|
||||
@@ -54,9 +55,6 @@ class NodeIterator(Iterator[T]):
|
||||
post_iterator = profile.get_posts()
|
||||
post_iterator.thaw(load("resume_information.json"))
|
||||
|
||||
(an appropriate method to load and save the :class:`FrozenNodeIterator` is e.g.
|
||||
:func:`load_structure_from_file` and :func:`save_structure_to_file`.)
|
||||
|
||||
A :class:`FrozenNodeIterator` can only be thawn with a matching NodeIterator, i.e. a NodeIterator instance that has
|
||||
been constructed with the same parameters as the instance that is represented by the :class:`FrozenNodeIterator` in
|
||||
question. This is to ensure that an iteration cannot be resumed in a wrong, unmatching loop. As a quick way to
|
||||
@@ -76,8 +74,7 @@ class NodeIterator(Iterator[T]):
|
||||
node_wrapper: Callable[[Dict], T],
|
||||
query_variables: Optional[Dict[str, Any]] = None,
|
||||
query_referer: Optional[str] = None,
|
||||
first_data: Optional[Dict[str, Any]] = None,
|
||||
is_first: Optional[Callable[[T, Optional[T]], bool]] = None):
|
||||
first_data: Optional[Dict[str, Any]] = None):
|
||||
self._context = context
|
||||
self._query_hash = query_hash
|
||||
self._edge_extractor = edge_extractor
|
||||
@@ -91,11 +88,9 @@ class NodeIterator(Iterator[T]):
|
||||
self._best_before = datetime.now() + NodeIterator._shelf_life
|
||||
else:
|
||||
self._data = self._query()
|
||||
self._first_node: Optional[Dict] = None
|
||||
self._is_first = is_first
|
||||
|
||||
def _query(self, after: Optional[str] = None) -> Dict:
|
||||
pagination_variables: Dict[str, Any] = {'first': NodeIterator._graphql_page_length}
|
||||
pagination_variables = {'first': NodeIterator._graphql_page_length} # type: Dict[str, Any]
|
||||
if after is not None:
|
||||
pagination_variables['after'] = after
|
||||
try:
|
||||
@@ -129,25 +124,17 @@ class NodeIterator(Iterator[T]):
|
||||
except KeyboardInterrupt:
|
||||
self._page_index, self._total_index = page_index, total_index
|
||||
raise
|
||||
item = self._node_wrapper(node)
|
||||
if self._is_first is not None:
|
||||
if self._is_first(item, self.first_item):
|
||||
self._first_node = node
|
||||
else:
|
||||
if self._first_node is None:
|
||||
self._first_node = node
|
||||
return item
|
||||
if self._data.get('page_info', {}).get('has_next_page'):
|
||||
return self._node_wrapper(node)
|
||||
if self._data['page_info']['has_next_page']:
|
||||
query_response = self._query(self._data['page_info']['end_cursor'])
|
||||
if self._data['edges'] != query_response['edges'] and len(query_response['edges']) > 0:
|
||||
page_index, data = self._page_index, self._data
|
||||
try:
|
||||
self._page_index = 0
|
||||
self._data = query_response
|
||||
except KeyboardInterrupt:
|
||||
self._page_index, self._data = page_index, data
|
||||
raise
|
||||
return self.__next__()
|
||||
page_index, data = self._page_index, self._data
|
||||
try:
|
||||
self._page_index = 0
|
||||
self._data = query_response
|
||||
except KeyboardInterrupt:
|
||||
self._page_index, self._data = page_index, data
|
||||
raise
|
||||
return self.__next__()
|
||||
raise StopIteration()
|
||||
|
||||
@property
|
||||
@@ -163,31 +150,16 @@ class NodeIterator(Iterator[T]):
|
||||
@property
|
||||
def magic(self) -> str:
|
||||
"""Magic string for easily identifying a matching iterator file for resuming (hash of some parameters)."""
|
||||
magic_hash = hashlib.blake2b(digest_size=6)
|
||||
if 'blake2b' not in hashlib.algorithms_available:
|
||||
magic_hash = hashlib.new('sha224')
|
||||
else:
|
||||
# Use blake2b when possible, i.e. on Python >= 3.6.
|
||||
magic_hash = hashlib.blake2b(digest_size=6) # type:ignore # pylint: disable=no-member
|
||||
magic_hash.update(json.dumps(
|
||||
[self._query_hash, self._query_variables, self._query_referer, self._context.username]
|
||||
).encode())
|
||||
return base64.urlsafe_b64encode(magic_hash.digest()).decode()
|
||||
|
||||
@property
|
||||
def first_item(self) -> Optional[T]:
|
||||
"""
|
||||
If this iterator has produced any items, returns the first item produced.
|
||||
|
||||
It is possible to override what is considered the first item (for example, to consider the
|
||||
newest item in case items are not in strict chronological order) by passing a callback
|
||||
function as the `is_first` parameter when creating the class.
|
||||
|
||||
.. versionadded:: 4.8
|
||||
.. versionchanged:: 4.9.2
|
||||
What is considered the first item can be overridden.
|
||||
"""
|
||||
return self._node_wrapper(self._first_node) if self._first_node is not None else None
|
||||
|
||||
@staticmethod
|
||||
def page_length() -> int:
|
||||
return NodeIterator._graphql_page_length
|
||||
|
||||
def freeze(self) -> FrozenNodeIterator:
|
||||
"""Freeze the iterator for later resuming."""
|
||||
remaining_data = None
|
||||
@@ -202,7 +174,6 @@ class NodeIterator(Iterator[T]):
|
||||
total_index=max(self.total_index - 1, 0),
|
||||
best_before=self._best_before.timestamp() if self._best_before else None,
|
||||
remaining_data=remaining_data,
|
||||
first_node=self._first_node,
|
||||
)
|
||||
|
||||
def thaw(self, frozen: FrozenNodeIterator) -> None:
|
||||
@@ -229,21 +200,18 @@ class NodeIterator(Iterator[T]):
|
||||
self._total_index = frozen.total_index
|
||||
self._best_before = datetime.fromtimestamp(frozen.best_before)
|
||||
self._data = frozen.remaining_data
|
||||
if frozen.first_node is not None:
|
||||
self._first_node = frozen.first_node
|
||||
|
||||
|
||||
@contextmanager
|
||||
def resumable_iteration(context: InstaloaderContext,
|
||||
iterator: Iterable,
|
||||
iterator: Iterator,
|
||||
load: Callable[[InstaloaderContext, str], Any],
|
||||
save: Callable[[FrozenNodeIterator, str], None],
|
||||
format_path: Callable[[str], str],
|
||||
check_bbd: bool = True,
|
||||
enabled: bool = True) -> Iterator[Tuple[bool, int]]:
|
||||
"""
|
||||
High-level context manager to handle a resumable iteration that can be interrupted
|
||||
with a :class:`KeyboardInterrupt` or an :class:`AbortDownloadException`.
|
||||
High-level context manager to handle a resumable iteration that can be interrupted with a KeyboardInterrupt.
|
||||
|
||||
It can be used as follows to automatically load a previously-saved state into the iterator, save the iterator's
|
||||
state when interrupted, and delete the resume file upon completion::
|
||||
@@ -271,9 +239,6 @@ def resumable_iteration(context: InstaloaderContext,
|
||||
:param format_path: Returns the path to the resume file for the given magic.
|
||||
:param check_bbd: Whether to check the best before date and reject an expired FrozenNodeIterator.
|
||||
:param enabled: Set to False to disable all functionality and simply execute the inner body.
|
||||
|
||||
.. versionchanged:: 4.7
|
||||
Also interrupt on :class:`AbortDownloadException`.
|
||||
"""
|
||||
if not enabled or not isinstance(iterator, NodeIterator):
|
||||
yield False, 0
|
||||
@@ -293,11 +258,11 @@ def resumable_iteration(context: InstaloaderContext,
|
||||
is_resuming = True
|
||||
start_index = iterator.total_index
|
||||
context.log("Resuming from {}.".format(resume_file_path))
|
||||
except (InvalidArgumentException, LZMAError, json.decoder.JSONDecodeError, EOFError) as exc:
|
||||
except (InvalidArgumentException, LZMAError, json.decoder.JSONDecodeError) as exc:
|
||||
context.error("Warning: Not resuming from {}: {}".format(resume_file_path, exc))
|
||||
try:
|
||||
yield is_resuming, start_index
|
||||
except (KeyboardInterrupt, AbortDownloadException):
|
||||
except KeyboardInterrupt:
|
||||
if os.path.dirname(resume_file_path):
|
||||
os.makedirs(os.path.dirname(resume_file_path), exist_ok=True)
|
||||
save(iterator.freeze(), resume_file_path)
|
||||
|
||||
@@ -1,46 +0,0 @@
|
||||
from typing import Any, Callable, Dict, Iterator, Optional, TypeVar
|
||||
|
||||
from .instaloadercontext import InstaloaderContext
|
||||
|
||||
T = TypeVar('T')
|
||||
|
||||
|
||||
class SectionIterator(Iterator[T]):
|
||||
"""Iterator for the new 'sections'-style responses.
|
||||
|
||||
.. versionadded:: 4.9"""
|
||||
def __init__(self,
|
||||
context: InstaloaderContext,
|
||||
sections_extractor: Callable[[Dict[str, Any]], Dict[str, Any]],
|
||||
media_wrapper: Callable[[Dict], T],
|
||||
query_path: str,
|
||||
first_data: Optional[Dict[str, Any]] = None):
|
||||
self._context = context
|
||||
self._sections_extractor = sections_extractor
|
||||
self._media_wrapper = media_wrapper
|
||||
self._query_path = query_path
|
||||
self._data = first_data or self._query()
|
||||
self._page_index = 0
|
||||
self._section_index = 0
|
||||
|
||||
def __iter__(self):
|
||||
return self
|
||||
|
||||
def _query(self, max_id: Optional[str] = None) -> Dict[str, Any]:
|
||||
pagination_variables = {"max_id": max_id} if max_id is not None else {}
|
||||
return self._sections_extractor(
|
||||
self._context.get_json(self._query_path, params={"__a": 1, "__d": "dis", **pagination_variables})
|
||||
)
|
||||
|
||||
def __next__(self) -> T:
|
||||
if self._page_index < len(self._data['sections']):
|
||||
media = self._data['sections'][self._page_index]['layout_content']['medias'][self._section_index]['media']
|
||||
self._section_index += 1
|
||||
if self._section_index >= len(self._data['sections'][self._page_index]['layout_content']['medias']):
|
||||
self._section_index = 0
|
||||
self._page_index += 1
|
||||
return self._media_wrapper(media)
|
||||
if self._data['more_available']:
|
||||
self._page_index, self._section_index, self._data = 0, 0, self._query(self._data["next_max_id"])
|
||||
return self.__next__()
|
||||
raise StopIteration()
|
||||
+156
-748
File diff suppressed because it is too large
Load Diff
@@ -3,6 +3,7 @@
|
||||
import re
|
||||
import sys
|
||||
import os
|
||||
import platform
|
||||
from setuptools import setup
|
||||
|
||||
|
||||
@@ -18,13 +19,13 @@ def get_version():
|
||||
raise SystemExit("Could not find version string.")
|
||||
|
||||
|
||||
if sys.version_info < (3, 8):
|
||||
sys.exit('Instaloader requires Python >= 3.8.')
|
||||
if sys.version_info < (3, 5):
|
||||
sys.exit('Instaloader requires Python >= 3.5.')
|
||||
|
||||
requirements = ['requests>=2.4']
|
||||
optional_requirements = {
|
||||
'browser_cookie3': ['browser_cookie3>=0.19.1'],
|
||||
}
|
||||
|
||||
if platform.system() == 'Windows' and sys.version_info < (3, 6):
|
||||
requirements.append('win_unicode_console')
|
||||
|
||||
keywords = (['instagram', 'instagram-scraper', 'instagram-client', 'instagram-feed', 'downloader', 'videos', 'photos',
|
||||
'pictures', 'instagram-user-photos', 'instagram-photos', 'instagram-metadata', 'instagram-downloader',
|
||||
@@ -46,8 +47,7 @@ setup(
|
||||
'from Instagram.',
|
||||
long_description=open(os.path.join(SRC, 'README.rst')).read(),
|
||||
install_requires=requirements,
|
||||
python_requires='>=3.8',
|
||||
extras_require=optional_requirements,
|
||||
python_requires='>=3.5',
|
||||
entry_points={'console_scripts': ['instaloader=instaloader.__main__:main']},
|
||||
zip_safe=False,
|
||||
keywords=keywords,
|
||||
@@ -58,11 +58,10 @@ setup(
|
||||
'Intended Audience :: Developers',
|
||||
'License :: OSI Approved :: MIT License',
|
||||
'Operating System :: OS Independent',
|
||||
'Programming Language :: Python :: 3.5',
|
||||
'Programming Language :: Python :: 3.6',
|
||||
'Programming Language :: Python :: 3.7',
|
||||
'Programming Language :: Python :: 3.8',
|
||||
'Programming Language :: Python :: 3.9',
|
||||
'Programming Language :: Python :: 3.10',
|
||||
'Programming Language :: Python :: 3.11',
|
||||
'Programming Language :: Python :: 3.12',
|
||||
'Programming Language :: Python :: 3 :: Only',
|
||||
'Topic :: Internet',
|
||||
'Topic :: Multimedia :: Graphics'
|
||||
|
||||
@@ -23,7 +23,7 @@ PRIVATE_PROFILE_ID = 1706625676
|
||||
EMPTY_PROFILE = "not_public"
|
||||
EMPTY_PROFILE_ID = 1928659031
|
||||
|
||||
ratecontroller: Optional[instaloader.RateController] = None
|
||||
ratecontroller = None # type: Optional[instaloader.RateController]
|
||||
|
||||
|
||||
class TestInstaloaderAnonymously(unittest.TestCase):
|
||||
|
||||
Reference in New Issue
Block a user