Release of Version 4.3.2

Fixes #478.

README.rst

@@ -2,7 +2,7 @@
 
 .. badges-start
 
-|travis| |pypi| |pyversion| |license| |aur| |contributors| |downloads| |saythanks|
+|travis| |pypi| |pyversion| |license| |aur| |contributors| |downloads|
 
 .. |travis| image:: https://img.shields.io/travis/instaloader/instaloader/master.svg
    :alt: Travis-CI Build Status
@@ -23,10 +23,6 @@
    :alt: Contributor Count
    :target: https://github.com/instaloader/instaloader/graphs/contributors
 
-.. |saythanks| image:: https://img.shields.io/badge/Say%20Thanks-!-ff7efd.svg
-   :alt: Say Thanks!
-   :target: https://saythanks.io/to/aandergr
-
 .. |aur| image:: https://img.shields.io/aur/version/instaloader.svg
    :alt: Arch User Repository Package
    :target: https://aur.archlinux.org/packages/instaloader/
@@ -57,7 +53,8 @@
 
 ::
 
-    instaloader [--comments] [--geotags] [--stories] [--highlights] [--tagged]
+    instaloader [--comments] [--geotags]
+                [--stories] [--highlights] [--tagged] [--igtv]
                 [--login YOUR-USERNAME] [--fast-update]
                 profile | "#hashtag" | :stories | :feed | :saved
 

docs/_templates/layout.html

@@ -51,9 +51,6 @@
         <li class="nav-item">
           <a target="_blank" class="nav-link" href="https://github.com/instaloader/instaloader">GitHub</a>
         </li>
-        <li class="nav-item d-sm-none d-lg-block">
-          <a target="_blank" class="nav-link" href="https://saythanks.io/to/aandergr">Say Thanks!</a>
-        </li>
       </ul>
       <form class="form-inline" action="search.html" method="get">
         <div class="input-group my-2 my-sm-0 ml-2">

docs/basic-usage.rst

@@ -77,10 +77,13 @@ Instaloader supports the following targets:
       instructs Instaloader to also **download the user's stories**,
 
    - :option:`--highlights`
-      to **download highlights of each profile that is downloaded**, and
+      to **download highlights of each profile that is downloaded**,
 
    - :option:`--tagged`
-      to **download posts where the user is tagged**.
+      to **download posts where the user is tagged**, and
+
+   - :option:`--igtv`
+      to **download IGTV videos**.
 
 - ``"#hashtag"``
    Posts with a certain **hashtag** (the quotes are usually necessary),

docs/cli-options.rst

@@ -129,6 +129,12 @@ What to Download of each Profile
 
    .. versionadded:: 4.1
 
+.. option:: --igtv
+
+   Also download IGTV videos.
+
+   .. versionadded:: 4.3
+
 .. option:: --stories-only
 
    .. deprecated:: 4.1
@@ -237,6 +243,12 @@ How to Download
 
    .. versionadded:: 4.2
 
+.. option:: --request-timeout N
+
+   Seconds to wait before timing out a connection request.
+
+   .. versionadded:: 4.3
+
 Miscellaneous Options
 ^^^^^^^^^^^^^^^^^^^^^
 
@@ -246,12 +258,6 @@ Miscellaneous Options
    if login credentials are needed but not given.
    This is handy for running :ref:`instaloader-as-cronjob`.
 
-.. option:: --request-timeout N
-
-   Seconds to wait before timing out a connection request.
-
-   .. versionadded:: 4.3
-
 .. option:: +args.txt
 
    Read arguments from file `args.txt`, a shortcut to provide arguments from

docs/index.rst

@@ -45,7 +45,8 @@ See :ref:`install` for more options on how to install Instaloader.
 
 ::
 
-    instaloader [--comments] [--geotags] [--stories] [--highlights] [--tagged]
+    instaloader [--comments] [--geotags]
+                [--stories] [--highlights] [--tagged] [--igtv]
                 [--login YOUR-USERNAME] [--fast-update]
                 profile | "#hashtag" | %location_id |
                 :stories | :feed | :saved

instaloader/__init__.py

@@ -1,7 +1,7 @@
 """Download pictures (or videos) along with their captions and other metadata from Instagram."""
 
 
-__version__ = '4.3a1'
+__version__ = '4.3.2'
 
 
 try:

instaloader/__main__.py

@@ -19,7 +19,8 @@ def usage_string():
     argv0 = os.path.basename(sys.argv[0])
     argv0 = "instaloader" if argv0 == "__main__.py" else argv0
     return """
-{0} [--comments] [--geotags] [--stories] [--highlights] [--tagged]
+{0} [--comments] [--geotags]
+{2:{1}} [--stories] [--highlights] [--tagged] [--igtv]
 {2:{1}} [--login YOUR-USERNAME] [--fast-update]
 {2:{1}} profile | "#hashtag" | %%location_id | :stories | :feed | :saved
 {0} --help""".format(argv0, len(argv0), '')
@@ -61,7 +62,10 @@ def _main(instaloader: Instaloader, targetlist: List[str],
           username: Optional[str] = None, password: Optional[str] = None,
           sessionfile: Optional[str] = None,
           download_profile_pic: bool = True, download_posts=True,
-          download_stories: bool = False, download_highlights: bool = False, download_tagged: bool = False,
+          download_stories: bool = False,
+          download_highlights: bool = False,
+          download_tagged: bool = False,
+          download_igtv: bool = False,
           fast_update: bool = False,
           max_count: Optional[int] = None, post_filter_str: Optional[str] = None,
           storyitem_filter_str: Optional[str] = None) -> None:
@@ -158,7 +162,8 @@ def _main(instaloader: Instaloader, targetlist: List[str],
                     try:
                         profile = instaloader.check_profile_id(target)
                         if instaloader.context.is_logged_in and profile.has_blocked_viewer:
-                            if download_profile_pic or ((download_posts or download_tagged) and not profile.is_private):
+                            if download_profile_pic or ((download_posts or download_tagged or download_igtv)
+                                                        and not profile.is_private):
                                 raise ProfileNotExistsException("{} blocked you; But we download her anonymously."
                                                                 .format(target))
                             else:
@@ -169,7 +174,7 @@ def _main(instaloader: Instaloader, targetlist: List[str],
                         # Not only our profile.has_blocked_viewer condition raises ProfileNotExistsException,
                         # check_profile_id() also does, since access to blocked profile may be responded with 404.
                         if instaloader.context.is_logged_in and (download_profile_pic or download_posts or
-                                                                 download_tagged):
+                                                                 download_tagged or download_igtv):
                             instaloader.context.log(err)
                             instaloader.context.log("Trying again anonymously, helps in case you are just blocked.")
                             with instaloader.anonymous_copy() as anonymous_loader:
@@ -185,14 +190,15 @@ def _main(instaloader: Instaloader, targetlist: List[str],
         if profiles and download_profile_pic and not instaloader.context.is_logged_in:
             instaloader.context.error("Warning: Use --login to download HD version of profile pictures.")
         instaloader.download_profiles(profiles,
-                                      download_profile_pic, download_posts, download_tagged, download_highlights,
-                                      download_stories, fast_update, post_filter, storyitem_filter)
+                                      download_profile_pic, download_posts, download_tagged, download_igtv,
+                                      download_highlights, download_stories,
+                                      fast_update, post_filter, storyitem_filter)
         if anonymous_retry_profiles:
             instaloader.context.log("Downloading anonymously: {}"
                                     .format(' '.join([p.username for p in anonymous_retry_profiles])))
             with instaloader.anonymous_copy() as anonymous_loader:
                 anonymous_loader.download_profiles(anonymous_retry_profiles,
-                                                   download_profile_pic, download_posts, download_tagged,
+                                                   download_profile_pic, download_posts, download_tagged, download_igtv,
                                                    fast_update=fast_update, post_filter=post_filter)
     except KeyboardInterrupt:
         print("\nInterrupted by user.", file=sys.stderr)
@@ -287,6 +293,8 @@ def main():
                         help='Also download highlights of each profile that is downloaded. Requires --login.')
     g_prof.add_argument('--tagged', action='store_true',
                         help='Also download posts where each profile is tagged.')
+    g_prof.add_argument('--igtv', action='store_true',
+                        help='Also download IGTV videos.')
 
     g_cond = parser.add_argument_group("Which Posts to Download")
 
@@ -411,6 +419,7 @@ def main():
               download_stories=download_stories,
               download_highlights=args.highlights,
               download_tagged=args.tagged,
+              download_igtv=args.igtv,
               fast_update=args.fast_update,
               max_count=int(args.count) if args.count is not None else None,
               post_filter_str=args.post_filter,

instaloader/instaloader.py

@@ -142,7 +142,7 @@ class Instaloader:
     :param storyitem_metadata_txt_pattern: :option:`--storyitem-metadata-txt`, default is empty (=none)
     :param max_connection_attempts: :option:`--max-connection-attempts`
     :param commit_mode: :option:`--commit-mode`
-    :param request_timeout: "option:`--request-timeout`, set per-request timeout (seconds)
+    :param request_timeout: :option:`--request-timeout`, set per-request timeout (seconds)
 
     .. attribute:: context
 
@@ -358,42 +358,56 @@ class Instaloader:
         self.context.log('geo', end=' ', flush=True)
 
     @_retry_on_connection_error
-    def download_profilepic(self, profile: Profile, _attempt: int = 1) -> None:
-        """Downloads and saves profile pic."""
+    def download_title_pic(self, url: str, target: Union[str, Path], name_suffix: str, owner_profile: Profile,
+                           _attempt: int = 1) -> None:
+        """Downloads and saves a picture that does not have an association with a Post or StoryItem, such as a
+        Profile picture or a Highlight cover picture. Modification time is taken from the HTTP response headers.
+
+        .. versionadded:: 4.3"""
 
         def _epoch_to_string(epoch: datetime) -> str:
             return epoch.strftime('%Y-%m-%d_%H-%M-%S_UTC')
 
-        profile_pic_response = self.context.get_raw(profile.profile_pic_url)
+        http_response = self.context.get_raw(url)
         date_object = None  # type: Optional[datetime]
-        if 'Last-Modified' in profile_pic_response.headers:
-            date_object = datetime.strptime(profile_pic_response.headers["Last-Modified"], '%a, %d %b %Y %H:%M:%S GMT')
-            profile_pic_bytes = None
-            profile_pic_identifier = _epoch_to_string(date_object)
+        if 'Last-Modified' in http_response.headers:
+            date_object = datetime.strptime(http_response.headers["Last-Modified"], '%a, %d %b %Y %H:%M:%S GMT')
+            pic_bytes = None
+            pic_identifier = _epoch_to_string(date_object)
         else:
-            profile_pic_bytes = profile_pic_response.content
-            profile_pic_identifier = md5(profile_pic_bytes).hexdigest()[:16]
-        profile_pic_extension = 'jpg'
+            pic_bytes = http_response.content
+            pic_identifier = md5(pic_bytes).hexdigest()[:16]
+        pic_extension = 'jpg'
         if ((format_string_contains_key(self.dirname_pattern, 'profile') or
              format_string_contains_key(self.dirname_pattern, 'target'))):
-            filename = '{0}/{1}_profile_pic.{2}'.format(self.dirname_pattern.format(profile=profile.username.lower(),
-                                                                                    target=profile.username.lower()),
-                                                        profile_pic_identifier, profile_pic_extension)
+            filename = '{0}/{1}_{2}.{3}'.format(self.dirname_pattern.format(profile=owner_profile.username.lower(),
+                                                                            target=target),
+                                                pic_identifier, name_suffix, pic_extension)
         else:
-            filename = '{0}/{1}_{2}_profile_pic.{3}'.format(self.dirname_pattern.format(), profile.username.lower(),
-                                                            profile_pic_identifier, profile_pic_extension)
-        content_length = profile_pic_response.headers.get('Content-Length', None)
+            filename = '{0}/{1}_{2}_{3}.{4}'.format(self.dirname_pattern.format(), target,
+                                                    pic_identifier, name_suffix, pic_extension)
+        content_length = http_response.headers.get('Content-Length', None)
         if os.path.isfile(filename) and (not self.context.is_logged_in or
                                          (content_length is not None and
                                           os.path.getsize(filename) >= int(content_length))):
             self.context.log(filename + ' already exists')
             return None
         os.makedirs(os.path.dirname(filename), exist_ok=True)
-        self.context.write_raw(profile_pic_bytes if profile_pic_bytes else profile_pic_response, filename)
+        self.context.write_raw(pic_bytes if pic_bytes else http_response, filename)
         if date_object:
             os.utime(filename, (datetime.now().timestamp(), date_object.timestamp()))
         self.context.log('')  # log output of _get_and_write_raw() does not produce \n
 
+    def download_profilepic(self, profile: Profile) -> None:
+        """Downloads and saves profile pic."""
+        self.download_title_pic(profile.profile_pic_url, profile.username.lower(), 'profile_pic', profile)
+
+    def download_highlight_cover(self, highlight: Highlight, target: Union[str, Path]) -> None:
+        """Downloads and saves Highlight cover picture.
+
+        .. versionadded:: 4.3"""
+        self.download_title_pic(highlight.cover_url, target, 'cover', highlight.owner_profile)
+
     @_requires_login
     def save_session_to_file(self, filename: Optional[str] = None) -> None:
         """Saves internally stored :class:`requests.Session` object.
@@ -646,6 +660,9 @@ class Instaloader:
 
         .. versionadded:: 4.1
 
+        .. versionchanged:: 4.3
+           Also downloads and saves the Highlight's cover pictures.
+
         :param user: ID or Profile of the user whose highlights should get downloaded.
         :param fast_update: If true, abort when first already-downloaded picture is encountered
         :param filename_target: Replacement for {target} in dirname_pattern and filename_pattern
@@ -655,7 +672,11 @@ class Instaloader:
         """
         for user_highlight in self.get_highlights(user):
             name = user_highlight.owner_username
+            highlight_target = (filename_target
+                                if filename_target
+                                else Path(name) / Path(user_highlight.title))  # type: Union[str, Path]
             self.context.log("Retrieving highlights \"{}\" from profile {}".format(user_highlight.title, name))
+            self.download_highlight_cover(user_highlight, highlight_target)
             totalcount = user_highlight.itemcount
             count = 1
             for item in user_highlight.get_items():
@@ -666,9 +687,7 @@ class Instaloader:
                 count += 1
                 with self.context.error_catcher('Download highlights \"{}\" from user {}'.format(user_highlight.title,
                                                                                                  name)):
-                    downloaded = self.download_storyitem(item, filename_target
-                                                         if filename_target
-                                                         else Path(name) / Path(user_highlight.title))
+                    downloaded = self.download_storyitem(item, highlight_target)
                     if fast_update and not downloaded:
                         break
 
@@ -684,8 +703,10 @@ class Instaloader:
 
         while True:
             feed = data["user"]["edge_web_feed_timeline"]
-            yield from (Post(self.context, edge["node"]) for edge in feed["edges"]
-                        if not edge["node"]["__typename"] == "GraphSuggestedUserFeedUnit")
+            for edge in feed["edges"]:
+                node = edge["node"]
+                if node.get("__typename") in Post.supported_graphql_types() and node.get("shortcode") is not None:
+                    yield Post(self.context, node)
             if not feed["page_info"]["has_next_page"]:
                 break
             data = self.context.graphql_query("d6f4427fbe92d846298cf93df0b937d3",
@@ -892,11 +913,29 @@ class Instaloader:
             count += 1
             if post_filter is not None and not post_filter(post):
                 self.context.log('<{} skipped>'.format(post))
+                continue
             with self.context.error_catcher('Download tagged {}'.format(profile.username)):
                 downloaded = self.download_post(post, target if target else Path(profile.username) / Path(':tagged'))
                 if fast_update and not downloaded:
                     break
 
+    def download_igtv(self, profile: Profile, fast_update: bool = False,
+                      post_filter: Optional[Callable[[Post], bool]] = None) -> None:
+        """Download IGTV videos of a profile.
+
+        .. versionadded:: 4.3"""
+        self.context.log("Retrieving IGTV videos for profile {}.".format(profile.username))
+        for number, post in enumerate(profile.get_igtv_posts()):
+            self.context.log("[{0:{w}d}/{1:{w}d}] ".format(number+1, profile.igtvcount, w=len(str(profile.igtvcount))),
+                             end="", flush=True)
+            if post_filter is not None and not post_filter(post):
+                self.context.log('<{} skipped>'.format(post))
+                continue
+            with self.context.error_catcher('Download IGTV {}'.format(post.shortcode)):
+                downloaded = self.download_post(post, target=profile.username)
+                if fast_update and not downloaded:
+                    break
+
     def _get_id_filename(self, profile_name: str) -> str:
         if ((format_string_contains_key(self.dirname_pattern, 'profile') or
              format_string_contains_key(self.dirname_pattern, 'target'))):
@@ -969,7 +1008,10 @@ class Instaloader:
 
     def download_profiles(self, profiles: Set[Profile],
                           profile_pic: bool = True, posts: bool = True,
-                          tagged: bool = False, highlights: bool = False, stories: bool = False,
+                          tagged: bool = False,
+                          igtv: bool = False,
+                          highlights: bool = False,
+                          stories: bool = False,
                           fast_update: bool = False,
                           post_filter: Optional[Callable[[Post], bool]] = None,
                           storyitem_filter: Optional[Callable[[Post], bool]] = None,
@@ -980,6 +1022,7 @@ class Instaloader:
         :param profile_pic: not :option:`--no-profile-pic`.
         :param posts: not :option:`--no-posts`.
         :param tagged: :option:`--tagged`.
+        :param igtv: :option:`--igtv`.
         :param highlights: :option:`--highlights`.
         :param stories: :option:`--stories`.
         :param fast_update: :option:`--fast-update`.
@@ -989,7 +1032,11 @@ class Instaloader:
            Whether :exc:`LoginRequiredException` and :exc:`PrivateProfileNotFollowedException` should be raised or
            catched and printed with :meth:`InstaloaderContext.error_catcher`.
 
-        .. versionadded:: 4.1"""
+        .. versionadded:: 4.1
+
+        .. versionchanged:: 4.3
+           Add `igtv` parameter.
+        """
 
         @contextmanager
         def _error_raiser(_str):
@@ -1015,7 +1062,7 @@ class Instaloader:
                     self.save_metadata_json(json_filename, profile)
 
                 # Catch some errors
-                if profile.is_private and (tagged or highlights or posts):
+                if profile.is_private and (tagged or igtv or highlights or posts):
                     if not self.context.is_logged_in:
                         raise LoginRequiredException("--login=USERNAME required.")
                     if not profile.followed_by_viewer and self.context.username != profile.username:
@@ -1026,6 +1073,11 @@ class Instaloader:
                     with self.context.error_catcher('Download tagged of {}'.format(profile_name)):
                         self.download_tagged(profile, fast_update=fast_update, post_filter=post_filter)
 
+                # Download IGTV, if requested
+                if igtv:
+                    with self.context.error_catcher('Download IGTV of {}'.format(profile_name)):
+                        self.download_igtv(profile, fast_update=fast_update, post_filter=post_filter)
+
                 # Download highlights, if requested
                 if highlights:
                     with self.context.error_catcher('Download highlights of {}'.format(profile_name)):

instaloader/instaloadercontext.py

@@ -239,6 +239,12 @@ class InstaloaderContext:
                                                                                                resp_json['message']))
             else:
                 raise ConnectionException("Login error: \"{}\" status.".format(resp_json['status']))
+        if 'authenticated' not in resp_json:
+            # Issue #472
+            if 'message' in resp_json:
+                raise ConnectionException("Login error: Unexpected response, \"{}\".".format(resp_json['message']))
+            else:
+                raise ConnectionException("Login error: Unexpected response, this might indicate a blocked IP.")
         if not resp_json['authenticated']:
             if resp_json['user']:
                 # '{"authenticated": false, "user": true, "status": "ok"}'

instaloader/structures.py

@@ -98,6 +98,11 @@ class Post:
             raise InvalidArgumentException("Wrong mediaid {0}, unable to convert to shortcode".format(str(mediaid)))
         return b64encode(mediaid.to_bytes(9, 'big'), b'-_').decode().replace('A', ' ').lstrip().replace(' ', 'A')
 
+    @staticmethod
+    def supported_graphql_types() -> List[str]:
+        """The values of __typename fields that the :class:`Post` class can handle."""
+        return ["GraphImage", "GraphVideo", "GraphSidecar"]
+
     def _asdict(self):
         node = self._node
         if self._full_metadata_dict:
@@ -134,6 +139,12 @@ class Post:
             pic_json = self._context.get_json("p/{0}/".format(self.shortcode), params={})
             self._full_metadata_dict = pic_json['entry_data']['PostPage'][0]['graphql']['shortcode_media']
             self._rhx_gis_str = pic_json.get('rhx_gis')
+            if self._full_metadata_dict is None:
+                # issue #449
+                self._context.error("Fetching Post metadata failed (issue #449). "
+                                    "The following data has been returned:\n"
+                                    + json.dumps(pic_json['entry_data'], indent=2))
+                raise BadResponseException("Fetching Post metadata failed.")
             if self.shortcode != self._full_metadata_dict['shortcode']:
                 self._node.update(self._full_metadata_dict)
                 raise PostChangedException
@@ -224,7 +235,11 @@ class Post:
     def get_sidecar_nodes(self) -> Iterator[PostSidecarNode]:
         """Sidecar nodes of a Post with typename==GraphSidecar."""
         if self.typename == 'GraphSidecar':
-            for edge in self._field('edge_sidecar_to_children', 'edges'):
+            edges = self._field('edge_sidecar_to_children', 'edges')
+            if any(edge['node']['is_video'] for edge in edges):
+                # video_url is only present in full metadata, issue #558.
+                edges = self._full_metadata['edge_sidecar_to_children']['edges']
+            for edge in edges:
                 node = edge['node']
                 is_video = node['is_video']
                 yield PostSidecarNode(is_video=is_video, display_url=node['display_url'],
@@ -518,10 +533,11 @@ class Profile:
 
     def _asdict(self):
         json_node = self._node.copy()
-        # remove posts
+        # remove posts to avoid "Circular reference detected" exception
         json_node.pop('edge_media_collections', None)
         json_node.pop('edge_owner_to_timeline_media', None)
         json_node.pop('edge_saved_media', None)
+        json_node.pop('edge_felix_video_timeline', None)
         if self._iphone_struct_:
             json_node['iphone_struct'] = self._iphone_struct_
         return json_node
@@ -598,6 +614,10 @@ class Profile:
     def mediacount(self) -> int:
         return self._metadata('edge_owner_to_timeline_media', 'count')
 
+    @property
+    def igtvcount(self) -> int:
+        return self._metadata('edge_felix_video_timeline', 'count')
+
     @property
     def followers(self) -> int:
         return self._metadata('edge_followed_by', 'count')
@@ -741,6 +761,19 @@ class Profile:
                                                     lambda d: d['data']['user']['edge_user_to_photos_of_you'],
                                                     self._rhx_gis))
 
+    def get_igtv_posts(self) -> Iterator[Post]:
+        """Retrieve all IGTV posts.
+
+        .. versionadded:: 4.3"""
+        self._obtain_metadata()
+        yield from (Post(self._context, node, self) for node in
+                    self._context.graphql_node_list('bc78b344a68ed16dd5d7f264681c4c76',
+                                                    {'id': self.userid},
+                                                    'https://www.instagram.com/{0}/channel/'.format(self.username),
+                                                    lambda d: d['data']['user']['edge_felix_video_timeline'],
+                                                    self._rhx_gis,
+                                                    self._metadata('edge_felix_video_timeline')))
+
     def get_followers(self) -> Iterator['Profile']:
         """
         Retrieve list of followers of given profile.

test/instaloader_unittests.py

@@ -12,6 +12,7 @@ import instaloader
 PROFILE_WITH_HIGHLIGHTS = 325732271
 PUBLIC_PROFILE = "selenagomez"
 PUBLIC_PROFILE_ID = 460563723
+PUBLIC_PROFILE_WITH_IGTV = "natgeo"
 HASHTAG = "kitten"
 LOCATION = "362629379"
 OWN_USERNAME = "aandergr"
@@ -104,6 +105,11 @@ class TestInstaloaderAnonymously(unittest.TestCase):
                            PAGING_MAX_COUNT):
             print(post)
 
+    def test_public_profile_igtv(self):
+        for post in islice(instaloader.Profile.from_username(self.L.context, PUBLIC_PROFILE_WITH_IGTV).get_igtv_posts(),
+                           PAGING_MAX_COUNT):
+            print(post)
+
 
 class TestInstaloaderLoggedIn(TestInstaloaderAnonymously):