Release of version 4.0.4

This was noticeable when downloading hashtags.
Fixes #133

docs/as-module.rst

@@ -97,6 +97,9 @@ metadata of a Profile. :class:`Profile` instances can be created with:
 A reference of the many methods provided by the :mod:`instaloader` module is
 provided in the remainder of this document.
 
+For a list of real code examples using the Instaloader module for advanced
+tasks, see :ref:`codesnippets`.
+
 ``Instaloader`` (Main Class)
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 

docs/basic-usage.rst

@@ -273,3 +273,6 @@ consider taking a look at the :ref:`python-module-instaloader`.
 Instaloader exposes its internally used methods and structures, making it a
 powerful and intuitive Python API for Instagram, allowing to further customize
 obtaining media and metadata.
+
+Also see :ref:`codesnippets`, where we collect example scripts that use
+Instaloader to achieve more complex tasks.

docs/codesnippets.rst

@@ -0,0 +1,105 @@
+.. _codesnippets:
+
+Advanced Instaloader Examples
+=============================
+
+.. currentmodule:: instaloader
+
+.. highlight:: python
+
+.. contents::
+   :backlinks: none
+
+Here we present code examples that use the :ref:`python-module-instaloader` for
+more advanced tasks than what is possible with the Instaloader command line
+interface.
+
+.. For each code snippet:
+   - title
+   - brief description of what it does / motivation / how it works
+   - code, or link to code
+   - link to discussion issue
+   - link used methods
+
+Download Posts in a specific period
+-----------------------------------
+
+To collect pictures (and metadata) only from a specific period, you can play
+around with :func:`~itertools.dropwhile` and :func:`~itertools.takewhile` from
+:mod:`itertools` like in this snippet.
+
+.. literalinclude:: codesnippets/121_since_until.py
+
+See also :class:`Post`, :meth:`Instaloader.download_post`.
+
+Discussed in :issue:`121`.
+
+Likes of a Profile / Ghost Followers
+------------------------------------
+
+To store inactive followers, i.e. followers that did not like any of your
+pictures, into a file you can use this approach.
+
+.. literalinclude:: codesnippets/120_ghost_followers.py
+
+See also :meth:`Profile.get_posts`, :meth:`Post.get_likes`,
+:meth:`Profile.get_followers`, :meth:`Instaloader.load_session_from_file`,
+:meth:`Profile.from_username`.
+
+Discussed in :issue:`120`.
+
+Track Deleted Posts
+-------------------
+
+This script uses Instaloader to obtain a list of currently-online posts, and
+generates the matching filename of each post. It outputs a list of posts which
+are online but not offline (i.e. not yet downloaded) and a list of posts which
+are offline but not online (i.e. deleted in the profile).
+
+.. literalinclude:: codesnippets/56_track_deleted.py
+
+See also :func:`load_structure_from_file`, :meth:`Profile.from_username`,
+:meth:`Profile.get_posts`, :class:`Post`.
+
+Discussed in :issue:`56`.
+
+Only one Post per User
+----------------------
+
+To download only the one most recent post from each user, this snippet creates a
+:class:`set` that contains the users of which a post has already been
+downloaded. When iterating the posts, check whether the post's owner already is
+in the set. If so, skip the post. Otherwise, download it and add the user to
+that set.
+
+.. literalinclude:: codesnippets/113_only_one_per_user.py
+
+See also :class:`Post`, :meth:`Instaloader.download_post`,
+:attr:`Post.owner_profile`, :class:`Profile`.
+
+Discussed in :issue:`113`.
+
+Upgrade Images by local Copies
+------------------------------
+
+The following script finds local versions of images fetched by Instaloader, in
+order to ugprade the downloaded images by locally-found versions with better
+quality. It uses image hashing to identify similar images.
+
+`updgrade-instaloader-images.py <https://gist.github.com/pavelkryukov/15f93d19a99428a284a8bcec27e0187b>`__ (external link to GitHub Gist)
+
+Discussed in :issue:`46`.
+
+Render Captions to downloaded Images
+------------------------------------
+
+Instaloader does not modify the downloaded JPEG file. However, one could combine
+it with an imaging library such as Pillow or PIL to render the
+:attr:`Post.caption` on pictures.  The following shows an approach.
+
+.. literalinclude:: codesnippets/110_pil_captions.py
+
+See also :attr:`Post.caption`, :attr:`Post.url`, :meth:`Post.from_shortcode`,
+:func:`load_structure_from_file`.
+
+Discussed in :issue:`110`.

docs/codesnippets/110_pil_captions.py

@@ -0,0 +1,20 @@
+from io import BytesIO
+
+from requests import get
+from PIL import Image, ImageDraw
+from instaloader import *
+
+L = Instaloader()
+
+# Load Post instance
+post = load_structure_from_file(L.context, '2017-10-01_18-53-03_UTC.json.xz')
+# or post = Post.from_shortcode(L.context, SHORTCODE)
+
+# Render caption
+image = Image.open(BytesIO(get(post.url).content))
+draw = ImageDraw.Draw(image)
+color = 'rgb(0, 0, 0)'  # black color
+draw.text((300,100), post.caption.encode('latin1', errors='ignore'), fill=color)
+
+# Save image
+image.save('test.jpg')

docs/codesnippets/113_only_one_per_user.py

@@ -0,0 +1,14 @@
+import instaloader
+
+L = instaloader.Instaloader()
+
+posts = L.get_hashtag_posts('milfgarden')
+
+users = set()
+
+for post in posts:
+    if not post.owner_profile in users:
+        L.download_post(post, '#milfgarden')
+        users.add(post.owner_profile)
+    else:
+        print("{} from {} skipped.".format(post, post.owner_profile))

docs/codesnippets/120_ghost_followers.py

@@ -0,0 +1,27 @@
+import instaloader
+
+L = instaloader.Instaloader()
+
+USER = 'your_account'
+PROFILE = USER
+
+# Your preferred way of logging in:
+L.load_session_from_file(USER)
+
+profile = instaloader.Profile.from_username(L.context, PROFILE)
+
+likes = set()
+print('Fetching likes of all posts of profile {}.'.format(profile.username))
+for post in profile.get_posts():
+    print(post)
+    likes = likes | set(post.get_likes())
+
+print('Fetching followers of profile {}.'.format(profile.username))
+followers = set(profile.get_followers())
+
+ghosts = followers - likes
+
+print('Storing ghosts into file.')
+with open('/YOUR PATH/inactive-users.txt', 'w') as f:
+    for ghost in ghosts:
+        print(ghost.username, file=f)

docs/codesnippets/121_since_until.py

@@ -0,0 +1,17 @@
+from datetime import datetime
+from itertools import dropwhile, takewhile
+
+import instaloader
+
+L = instaloader.Instaloader()
+
+posts = L.get_hashtag_posts('milfgarden')
+# or
+# posts = instaloader.Profile.from_username(L.context, PROFILE).get_posts()
+
+SINCE = datetime(2015, 5, 1)
+UNTIL = datetime(2015, 3, 1)
+
+for post in takewhile(lambda p: p.date > UNTIL, dropwhile(lambda p: p.date > SINCE, posts)):
+    print(post.date)
+    L.download_post(post, '#milfgarden')

docs/codesnippets/56_track_deleted.py

@@ -0,0 +1,34 @@
+from glob import glob
+from sys import argv
+from os import chdir
+
+from instaloader import Instaloader, Post, Profile, load_structure_from_file
+
+# Instaloader instantiation - you may pass additional arguments to the constructor here
+L = Instaloader()
+
+# If desired, load session previously saved with `instaloader -l USERNAME`:
+#L.load_session_from_file(USERNAME)
+
+try:
+    TARGET = argv[1]
+except IndexError:
+    raise SystemExit("Pass profile name as argument!")
+
+# Obtain set of posts that are on hard disk
+chdir(TARGET)
+offline_posts = set(filter(lambda s: isinstance(s, Post),
+                           (load_structure_from_file(L.context, file)
+                            for file in (glob('*.json.xz') + glob('*.json')))))
+
+# Obtain set of posts that are currently online
+post_iterator = Profile.from_username(L.context, TARGET).get_posts()
+online_posts = set(post_iterator)
+
+if online_posts - offline_posts:
+    print("Not yet downloaded posts:")
+    print(" ".join(str(p) for p in (online_posts - offline_posts)))
+
+if offline_posts - online_posts:
+    print("Deleted posts:")
+    print(" ".join(str(p) for p in (offline_posts - online_posts)))

docs/conf.py

@@ -39,7 +39,8 @@ import docs.sphinx_autodoc_typehints as sphinx_autodoc_typehints
 extensions = [
     'sphinx.ext.autodoc',
     'sphinx.ext.githubpages',
-    'sphinx.ext.intersphinx'
+    'sphinx.ext.intersphinx',
+    'sphinx.ext.extlinks'
 ]
 
 autodoc_default_flags = ['show-inheritance', 'members', 'undoc-members']
@@ -49,6 +50,8 @@ autodoc_member_order = 'bysource'
 intersphinx_mapping = {'python': ('https://docs.python.org/3', None),
                        'requests': ('http://docs.python-requests.org/en/master', None)}
 
+extlinks = {'issue': ('https://github.com/instaloader/instaloader/issues/%s', 'issue #')}
+
 # Add any paths that contain templates here, relative to this directory.
 templates_path = ['_templates']
 

docs/index.rst

@@ -59,6 +59,7 @@ Instaloader Documentation
    basic-usage
    cli-options
    as-module
+   codesnippets
    contributing
 
 Useful Links

instaloader/__init__.py

@@ -1,7 +1,7 @@
 """Download pictures (or videos) along with their captions and other metadata from Instagram."""
 
 
-__version__ = '4.0.2'
+__version__ = '4.0.4'
 
 
 try:

instaloader/instaloader.py

@@ -266,7 +266,7 @@ class Instaloader:
         def _epoch_to_string(epoch: datetime) -> str:
             return epoch.strftime('%Y-%m-%d_%H-%M-%S')
 
-        profile_pic_url = profile.get_profile_pic_url()
+        profile_pic_url = profile.profile_pic_url
         with self.context.get_anonymous_session() as anonymous_session:
             date_object = datetime.strptime(anonymous_session.head(profile_pic_url).headers["Last-Modified"],
                                             '%a, %d %b %Y %H:%M:%S GMT')
@@ -320,7 +320,11 @@ class Instaloader:
         return self.context.test_login()
 
     def login(self, user: str, passwd: str) -> None:
-        """Log in to instagram with given username and password and internally store session object"""
+        """Log in to instagram with given username and password and internally store session object.
+
+        :raises InvalidArgumentException: If the provided username does not exist.
+        :raises BadCredentialsException: If the provided password is wrong.
+        :raises ConnectionException: If connection to Instagram failed."""
         self.context.login(user, passwd)
 
     def download_post(self, post: Post, target: str) -> bool:
@@ -732,7 +736,9 @@ class Instaloader:
     def interactive_login(self, username: str) -> None:
         """Logs in and internally stores session, asking user for password interactively.
 
-        :raises LoginRequiredException: when in quiet mode."""
+        :raises LoginRequiredException: when in quiet mode.
+        :raises InvalidArgumentException: If the provided username does not exist.
+        :raises ConnectionException: If connection to Instagram failed."""
         if self.context.quiet:
             raise LoginRequiredException("Quiet mode requires given password or valid session file.")
         password = None

instaloader/instaloadercontext.py

@@ -170,29 +170,44 @@ class InstaloaderContext:
         return data["data"]["user"]["username"] if data["data"]["user"] is not None else None
 
     def login(self, user, passwd):
-        """Not meant to be used directly, use :meth:`Instaloader.login`."""
+        """Not meant to be used directly, use :meth:`Instaloader.login`.
+
+        :raises InvalidArgumentException: If the provided username does not exist.
+        :raises BadCredentialsException: If the provided password is wrong.
+        :raises ConnectionException: If connection to Instagram failed."""
         session = requests.Session()
         session.cookies.update({'sessionid': '', 'mid': '', 'ig_pr': '1',
                                 'ig_vw': '1920', 'csrftoken': '',
                                 's_network': '', 'ds_user_id': ''})
         session.headers.update(self._default_http_header())
-        self._sleep()
-        resp = session.get('https://www.instagram.com/')
-        session.headers.update({'X-CSRFToken': resp.cookies['csrftoken']})
+        session.headers.update({'X-CSRFToken': self.get_json('', {})['config']['csrf_token']})
+        # Not using self.get_json() here, because we need to access csrftoken cookie
         self._sleep()
         login = session.post('https://www.instagram.com/accounts/login/ajax/',
                              data={'password': passwd, 'username': user}, allow_redirects=True)
-        session.headers.update({'X-CSRFToken': login.cookies['csrftoken']})
-        if login.status_code == 200:
-            self._session = session
-            if user == self.test_login():
-                self.username = user
+        if login.status_code != 200:
+            raise ConnectionException("Login error: {} {}".format(login.status_code, login.reason))
+        resp_json = login.json()
+        if resp_json['status'] != 'ok':
+            if 'message' in resp_json:
+                raise ConnectionException("Login error: \"{}\" status, message \"{}\".".format(resp_json['status'],
+                                                                                               resp_json['message']))
             else:
-                self.username = None
-                self._session = None
-                raise BadCredentialsException('Login error! Check your credentials!')
-        else:
-            raise ConnectionException('Login error! Connection error!')
+                raise ConnectionException("Login error: \"{}\" status.".format(resp_json['status']))
+        if not resp_json['authenticated']:
+            if resp_json['user']:
+                # '{"authenticated": false, "user": true, "status": "ok"}'
+                raise BadCredentialsException('Login error: Wrong password.')
+            else:
+                # '{"authenticated": false, "user": false, "status": "ok"}'
+                # Raise InvalidArgumentException rather than BadCredentialException, because BadCredentialException
+                # triggers re-asking of password in Instaloader.interactive_login(), which makes no sense if the
+                # username is invalid.
+                raise InvalidArgumentException('Login error: User {} does not exist.'.format(user))
+        # '{"authenticated": true, "user": true, "userId": ..., "oneTapPrompt": false, "status": "ok"}'
+        session.headers.update({'X-CSRFToken': login.cookies['csrftoken']})
+        self._session = session
+        self.username = user
 
     def _sleep(self):
         """Sleep a short time if self.sleep is set. Called before each request to instagram.com."""

instaloader/structures.py

@@ -206,10 +206,7 @@ class Post:
     @property
     def typename(self) -> str:
         """Type of post, GraphImage, GraphVideo or GraphSidecar"""
-        if '__typename' in self._node:
-            return self._node['__typename']
-        # if __typename is not in node, it is an old image or video
-        return 'GraphImage'
+        return self._field('__typename')
 
     def get_sidecar_nodes(self) -> Iterator[PostSidecarNode]:
         """Sidecar nodes of a Post with typename==GraphSidecar."""
@@ -374,10 +371,13 @@ class Profile:
     def __init__(self, context: InstaloaderContext, node: Dict[str, Any]):
         assert 'username' in node
         self._context = context
-        self._has_highlight_reels = None
         self._has_public_story = None
         self._node = node
         self._rhx_gis = None
+        self._iphone_struct_ = None
+        if 'iphone_struct' in node:
+            # if loaded from JSON with load_structure_from_file()
+            self._iphone_struct_ = node['iphone_struct']
 
     @classmethod
     def from_username(cls, context: InstaloaderContext, username: str):
@@ -424,6 +424,8 @@ class Profile:
         json_node.pop('edge_media_collections', None)
         json_node.pop('edge_owner_to_timeline_media', None)
         json_node.pop('edge_saved_media', None)
+        if self._iphone_struct_:
+            json_node['iphone_struct'] = self._iphone_struct_
         return json_node
 
     def _obtain_metadata(self):
@@ -448,6 +450,15 @@ class Profile:
                 d = d[key]
             return d
 
+    @property
+    def _iphone_struct(self) -> Dict[str, Any]:
+        if not self._iphone_struct_:
+            with self._context.anonymous_copy() as anonymous_context:
+                data = anonymous_context.get_json(path='api/v1/users/{}/info/'.format(self.userid),
+                                                  params={}, host='i.instagram.com')
+            self._iphone_struct_ = data['user']
+        return self._iphone_struct_
+
     @property
     def userid(self) -> int:
         """User ID"""
@@ -519,12 +530,7 @@ class Profile:
         This becomes `True` if the :class:`Profile` has any stories currently available,
         even if not viewable by the viewer.
         """
-        if not self._has_highlight_reels:
-            with self._context.anonymous_copy() as anonymous_context:
-                data = anonymous_context.get_json(path='api/v1/users/{}/info/'.format(self.userid),
-                                                  params={}, host='i.instagram.com')
-            self._has_highlight_reels = data['user']['has_highlight_reels']
-        return self._has_highlight_reels
+        return self._iphone_struct['has_highlight_reels']
 
     @property
     def has_public_story(self) -> bool:
@@ -563,17 +569,21 @@ class Profile:
     def requested_by_viewer(self) -> bool:
         return self._metadata('requested_by_viewer')
 
-    def get_profile_pic_url(self) -> str:
+    @property
+    def profile_pic_url(self) -> str:
         """Return URL of profile picture"""
         try:
-            with self._context.get_anonymous_session() as anonymous_session:
-                data = self._context.get_json(path='api/v1/users/{0}/info/'.format(self.userid), params={},
-                                              host='i.instagram.com', session=anonymous_session)
-            return data["user"]["hd_profile_pic_url_info"]["url"]
+            return self._iphone_struct['hd_profile_pic_url_info']['url']
         except (InstaloaderException, KeyError) as err:
             self._context.error('{} Unable to fetch high quality profile pic.'.format(err))
             return self._metadata("profile_pic_url_hd")
 
+    def get_profile_pic_url(self) -> str:
+        """.. deprecated:: 4.0.3
+
+	   Use :attr:`profile_pic_url`."""
+        return self.profile_pic_url
+
     def get_posts(self) -> Iterator[Post]:
         """Retrieve all posts from a profile."""
         self._obtain_metadata()

test/instaloader_unittests.py

@@ -16,6 +16,9 @@ NORMAL_MAX_COUNT = 2
 PAGING_MAX_COUNT = 15
 PRIVATE_PROFILE = "aandergr"
 
+# Preserve query timestamps (rate control) between tests to not get rate limited
+instaloadercontext_query_timestamps = list()
+
 
 class TestInstaloaderAnonymously(unittest.TestCase):
 
@@ -27,8 +30,11 @@ class TestInstaloaderAnonymously(unittest.TestCase):
                                          download_comments=True,
                                          save_metadata=True)
         self.L.context.raise_all_errors = True
+        self.L.context.query_timestamps = instaloadercontext_query_timestamps
 
     def tearDown(self):
+        global instaloadercontext_query_timestamps
+        instaloadercontext_query_timestamps = self.L.context.query_timestamps
         self.L.close()
         os.chdir('/')
         print("Removing {}".format(self.dir))