Release of version 2.1

Unless --no-sleep is given, Instaloader waits between requests to the
Instagram servers.

This commit fixes and enhances this behavior. Now, --no-sleep is always
obeyed. Between requests to the instagram.com servers, there is now a
delay of 250 ms ~ 2000 ms. Requests to the file servers do not cause a
delay.

.travis.yml

@@ -9,4 +9,4 @@ python:
 install:
  - pip install pylint requests
 script:
- - python3 -m pylint -r n -d bad-whitespace,missing-docstring,too-many-arguments,locally-disabled,line-too-long,too-many-public-methods,too-many-lines instaloader
+ - python3 -m pylint -r n -d bad-whitespace,missing-docstring,too-many-arguments,locally-disabled,line-too-long,too-many-public-methods,too-many-lines,too-many-instance-attributes instaloader

README.rst

@@ -1,7 +1,7 @@
 Instaloader
 ===========
 
-Tool to automatically download pictures (or videos) of given profiles
+Download pictures (or videos) along with their captions and other metadata
 from Instagram.
 
 Installation
@@ -75,6 +75,15 @@ You may also download
 
     instaloader "#hashtag"
 
+This downloads the requested posts into a directory named according to the specified
+hashtag and the filenames correspond to the timestamp of the posts.
+As with all download tasks, this behavior can easily be customized, for example
+encode the poster's profile name in the filenames:
+
+::
+
+    instaloader --filename-pattern={date}_{profile} "#hashtag"
+
 If you want to **download all followees of a given profile**, call
 
 ::
@@ -157,15 +166,19 @@ Instaloader to login.
 How to Download
 ^^^^^^^^^^^^^^^
 
---no-profile-subdir        Instead of creating a subdirectory for each profile
-                           and storing pictures there, store pictures in files
-                           named ``PROFILE__DATE_TIME.jpg``.
---hashtag-username         Lookup username of pictures when downloading by
-                           #hashtag and encode it in the downlaoded file's path
-                           or filename (if ``--no-profile-subdir``). Without this
-                           option, the #hashtag is used instead. This requires an
-                           additional request to the Instagram server for each
-                           picture, which is why it is disabled by default.
+--dirname-pattern DIRNAME_PATTERN
+                           Name of directory where to store posts. ``{profile}``
+                           is replaced by the profile name, ``{target}`` is replaced
+                           by the target you specified, i.e. either ``:feed``,
+                           ``#hashtag`` or the profile name. Defaults to ``{target}``.
+--filename-pattern FILENAME_PATTERN
+                           Prefix of filenames. Posts are stored in the
+                           directory whose pattern is given with ``--dirname-pattern``.
+                           ``{profile}`` is replaced by the profile name,
+                           ``{target}`` is replaced by the target you specified, i.e.
+                           either ``:feed``, ``#hashtag`` or the profile name. Also, the
+                           fields ``{date}`` and ``{shortcode}`` can be specified.
+                           Defaults to ``{date:%Y-%m-%d_%H-%M-%S}``.
 --user-agent USER_AGENT    User Agent to use for HTTP requests. Per default,
                            Instaloader pretends being Chrome/51.
 --no-sleep                 Do not sleep between requests to Instagram's servers.

instaloader.py

@@ -10,6 +10,7 @@ import pickle
 import random
 import re
 import shutil
+import string
 import sys
 import tempfile
 import time
@@ -120,33 +121,49 @@ def mediaid_to_shortcode(mediaid: int) -> str:
     return b64encode(mediaid.to_bytes(9, 'big'), b'-_').decode().replace('A', ' ').lstrip().replace(' ','A')
 
 
+def format_string_contains_key(format_string: str, key: str) -> bool:
+    # pylint:disable=unused-variable
+    for literal_text, field_name, format_spec, conversion in string.Formatter().parse(format_string):
+        if field_name == key:
+            return True
+    return False
+
+
 class Instaloader:
     def __init__(self,
-                 sleep: bool = True, quiet: bool = False, shorter_output: bool = False, profile_subdirs: bool = True,
-                 user_agent: Optional[str] = None):
+                 sleep: bool = True, quiet: bool = False, shorter_output: bool = False,
+                 user_agent: Optional[str] = None,
+                 dirname_pattern: Optional[str] = None,
+                 filename_pattern: Optional[str] = None):
         self.user_agent = user_agent if user_agent is not None else default_user_agent()
         self.session = self.get_anonymous_session()
         self.username = None
         self.sleep = sleep
         self.quiet = quiet
         self.shorter_output = shorter_output
-        self.profile_subdirs = profile_subdirs
+        self.dirname_pattern = dirname_pattern if dirname_pattern is not None else '{target}'
+        self.filename_pattern = filename_pattern.replace('{date}', '{date:%Y-%m-%d_%H-%M-%S}') \
+            if filename_pattern is not None else '{date:%Y-%m-%d_%H-%M-%S}'
 
     def _log(self, *msg, sep='', end='\n', flush=False):
         if not self.quiet:
             print(*msg, sep=sep, end=end, flush=flush)
 
+    def _sleep(self):
+        """Sleep a short, random time if self.sleep is set. Called before each request to the instagram.com."""
+        if self.sleep:
+            time.sleep(random.uniform(0.25, 2.0))
+
     def get_json(self, name: str, session: requests.Session = None,
                  max_id: Optional[str] = None) -> Optional[Dict[str, Any]]:
         """Return JSON of a profile"""
         if session is None:
             session = self.session
+        self._sleep()
         if not max_id:
             resp = session.get('https://www.instagram.com/' + name)
         else:
             resp = session.get('https://www.instagram.com/' + name, params={'max_id': max_id})
-        if self.sleep:
-            time.sleep(4 * random.random() + 1)
         match = re.search('window\\._sharedData = .*<', resp.text)
         if match is not None:
             return json.loads(match.group(0)[21:-2])
@@ -180,34 +197,50 @@ class Instaloader:
         session.headers.update(self.default_http_header(empty_session_only=True))
         return session
 
+    def graphql_query(self, query_id: int, variables: Dict[str, Any],
+                      referer: Optional[str] = None) -> Dict[str, Any]:
+        """
+        Do a GraphQL Query.
+
+        :param query_id: Query ID.
+        :param variables: Variables for the Query.
+        :param referer: HTTP Referer, or None.
+        :return: The server's response dictionary.
+        """
+        tmpsession = copy_session(self.session)
+        tmpsession.headers.update(self.default_http_header(empty_session_only=True))
+        del tmpsession.headers['Connection']
+        del tmpsession.headers['Content-Length']
+        tmpsession.headers['authority'] = 'www.instagram.com'
+        tmpsession.headers['scheme'] = 'https'
+        tmpsession.headers['accept'] = '*/*'
+        if referer is not None:
+            tmpsession.headers['referer'] = referer
+        self._sleep()
+        response = tmpsession.get('https://www.instagram.com/graphql/query',
+                                  params={'query_id': query_id,
+                                          'variables': json.dumps(variables, separators=(',', ':'))})
+        if response.status_code != 200:
+            raise ConnectionException("GraphQL query returned HTTP error code {}.".format(response.status_code))
+        return response.json()
+
     def get_username_by_id(self, profile_id: int) -> str:
         """To get the current username of a profile, given its unique ID, this function can be used."""
-        tmpsession = copy_session(self.session)
-        tmpsession.headers.update(self.default_http_header())
-        del tmpsession.headers['Referer']
-        del tmpsession.headers['X-Instagram-AJAX']
-        resp = tmpsession.get('https://www.instagram.com/graphql/query/',
-                              params={'query_id': '17862015703145017',
-                                      'variables': '{"id":"' + str(profile_id) + '","first":1}'})
-        if resp.status_code == 200:
-            data = json.loads(resp.text)["data"]["user"]
-            if data:
-                data = data["edge_owner_to_timeline_media"]
-            else:
-                raise ProfileNotExistsException("No profile found, the user may have blocked you (ID: " +
-                                                str(profile_id) + ").")
-            if not data['edges']:
-                if data['count'] == 0:
-                    raise ProfileHasNoPicsException("Profile with ID {0}: no pics found.".format(str(profile_id)))
-                else:
-                    raise LoginRequiredException("Login required to determine username (ID: " + str(profile_id) + ").")
-            else:
-                shortcode = mediaid_to_shortcode(int(data['edges'][0]["node"]["id"]))
-                data = self.get_json("p/" + shortcode)
-                return data['entry_data']['PostPage'][0]['graphql']['shortcode_media']['owner']['username']
+        data = self.graphql_query(17862015703145017, {'id': str(profile_id), 'first': 1})['data']['user']
+        if data:
+            data = data["edge_owner_to_timeline_media"]
         else:
-            raise ProfileAccessDeniedException("Username could not be determined due to error {0} (ID: {1})."
-                                               .format(str(resp.status_code), str(profile_id)))
+            raise ProfileNotExistsException("No profile found, the user may have blocked you (ID: " +
+                                            str(profile_id) + ").")
+        if not data['edges']:
+            if data['count'] == 0:
+                raise ProfileHasNoPicsException("Profile with ID {0}: no pics found.".format(str(profile_id)))
+            else:
+                raise LoginRequiredException("Login required to determine username (ID: " + str(profile_id) + ").")
+        else:
+            shortcode = mediaid_to_shortcode(int(data['edges'][0]["node"]["id"]))
+            data = self.get_json("p/" + shortcode)
+            return data['entry_data']['PostPage'][0]['graphql']['shortcode_media']['owner']['username']
 
     def get_id_by_username(self, profile: str) -> int:
         """Each Instagram profile has its own unique ID which stays unmodified even if a user changes
@@ -226,36 +259,22 @@ class Instaloader:
         :param profile: Name of profile to lookup followers.
         :return: List of followers (list of dictionaries).
         """
-        tmpsession = copy_session(self.session)
-        header = self.default_http_header(empty_session_only=True)
-        del header['Connection']
-        del header['Content-Length']
-        header['authority'] = 'www.instagram.com'
-        header['scheme'] = 'https'
-        header['accept'] = '*/*'
-        header['referer'] = 'https://www.instagram.com/' + profile + '/'
-        tmpsession.headers = header
         profile_id = self.get_id_by_username(profile)
-        resp = tmpsession.get('https://www.instagram.com/graphql/query/',
-                              params={'query_id': 17851374694183129,
-                                      'variables': '{"id":"' + str(profile_id) + '","first":500}'})
-        if resp.status_code == 200:
-            data = resp.json()
-            followers = []
-            while True:
-                edge_followed_by = data['data']['user']['edge_followed_by']
-                followers.extend([follower['node'] for follower in edge_followed_by['edges']])
-                page_info = edge_followed_by['page_info']
-                if page_info['has_next_page']:
-                    resp = tmpsession.get('https://www.instagram.com/graphql/query/',
-                                          params={'query_id': 17851374694183129,
-                                                  'variables': '{"id":"' + str(profile_id) + '","first":500,"after":"' +
-                                                               page_info['end_cursor'] + '"}'})
-                    data = resp.json()
-                else:
-                    break
-        else:
-            raise ConnectionException("ConnectionError({0}): unable to gather followers.".format(resp.status_code))
+        data = self.graphql_query(17851374694183129, {'id': str(profile_id),
+                                                      'first': 500},
+                                  referer='https://www.instagram.com/' + profile + '/')
+        followers = []
+        while True:
+            edge_followed_by = data['data']['user']['edge_followed_by']
+            followers.extend([follower['node'] for follower in edge_followed_by['edges']])
+            page_info = edge_followed_by['page_info']
+            if page_info['has_next_page']:
+                data = self.graphql_query(17851374694183129, {'id': str(profile_id),
+                                                              'first': 500,
+                                                              'after': page_info['end_cursor']},
+                                          referer='https://www.instagram.com/' + profile + '/')
+            else:
+                break
         return followers
 
     def get_followees(self, profile: str) -> List[Dict[str, Any]]:
@@ -267,93 +286,58 @@ class Instaloader:
         :param profile: Name of profile to lookup followers.
         :return: List of followees (list of dictionaries).
         """
-        tmpsession = copy_session(self.session)
-        header = self.default_http_header(empty_session_only=True)
-        del header['Connection']
-        del header['Content-Length']
-        header['authority'] = 'www.instagram.com'
-        header['scheme'] = 'https'
-        header['accept'] = '*/*'
-        header['referer'] = 'https://www.instagram.com/' + profile + '/'
-        tmpsession.headers = header
         profile_id = self.get_id_by_username(profile)
-        resp = tmpsession.get('https://www.instagram.com/graphql/query/',
-                              params={'query_id': 17874545323001329,
-                                      'variables': '{"id":"' + str(profile_id) + '","first":500}'})
-        if resp.status_code == 200:
-            data = resp.json()
-            followees = []
-            while True:
-                edge_follow = data['data']['user']['edge_follow']
-                followees.extend([followee['node'] for followee in edge_follow['edges']])
-                page_info = edge_follow['page_info']
-                if page_info['has_next_page']:
-                    resp = tmpsession.get('https://www.instagram.com/graphql/query/',
-                                          params={'query_id': 17874545323001329,
-                                                  'variables': '{"id":"' + str(profile_id) + '","first":500,"after":"' +
-                                                               page_info['end_cursor'] + '"}'})
-                    data = resp.json()
-                else:
-                    break
-        else:
-            raise ConnectionException("ConnectionError({0}): unable to gather followees.".format(resp.status_code))
+        data = self.graphql_query(17874545323001329, {'id': profile_id,
+                                                      'first': 500},
+                                  referer='https://www.instagram.com/' + profile + '/')
+        followees = []
+        while True:
+            edge_follow = data['data']['user']['edge_follow']
+            followees.extend([followee['node'] for followee in edge_follow['edges']])
+            page_info = edge_follow['page_info']
+            if page_info['has_next_page']:
+                data = self.graphql_query(17874545323001329, {'id': profile_id,
+                                                              'first': 500,
+                                                              'after': page_info['end_cursor']},
+                                          referer='https://www.instagram.com/' + profile + '/')
+            else:
+                break
         return followees
 
     def get_comments(self, shortcode: str) -> List[Dict[str, Any]]:
-        tmpsession = copy_session(self.session)
-        header = self.default_http_header(empty_session_only=True)
-        del header['Connection']
-        del header['Content-Length']
-        header['authority'] = 'www.instagram.com'
-        header['scheme'] = 'https'
-        header['accept'] = '*/*'
-        header['referer'] = 'https://www.instagram.com/p/' + shortcode + '/'
-        tmpsession.headers = header
-        resp = tmpsession.get('https://www.instagram.com/graphql/query/',
-                              params={'query_id': 17852405266163336,
-                                      'variables': '{"shortcode":"' + shortcode + '","first":500}'})
-        if resp.status_code == 200:
-            data = resp.json()
-            comments = []
-            while True:
-                edge_media_to_comment = data['data']['shortcode_media']['edge_media_to_comment']
-                comments.extend([comment['node'] for comment in edge_media_to_comment['edges']])
-                page_info = edge_media_to_comment['page_info']
-                if page_info['has_next_page']:
-                    resp = tmpsession.get('https://www.instagram.com/graphql/query/',
-                                          params={'query_id': 17852405266163336,
-                                                  'variables': '{"shortcode":"' + shortcode + '","first":500,"after":"'
-                                                               + page_info['end_cursor'] + '"}'})
-                    data = resp.json()
-                else:
-                    break
-        else:
-            raise ConnectionException("ConnectionError({0}): unable to gather comments.".format(resp.status_code))
+        """Retrieve comments of node with given shortcode."""
+        data = self.graphql_query(17852405266163336, {'shortcode': shortcode,
+                                                      'first': 500},
+                                  referer='https://www.instagram.com/p/' + shortcode + '/')
+        comments = []
+        while True:
+            edge_media_to_comment = data['data']['shortcode_media']['edge_media_to_comment']
+            comments.extend([comment['node'] for comment in edge_media_to_comment['edges']])
+            page_info = edge_media_to_comment['page_info']
+            if page_info['has_next_page']:
+                data = self.graphql_query(17852405266163336, {'shortcode': shortcode,
+                                                              'first': 500,
+                                                              'after': page_info['end_cursor']},
+                                          referer='https://www.instagram.com/p/' + shortcode + '/')
+            else:
+                break
         return comments
 
-    def download_pic(self, name: str, url: str, date_epoch: float, outputlabel: Optional[str] = None,
+    def download_pic(self, filename: str, url: str, date_epoch: float,
                      filename_suffix: Optional[str] = None) -> bool:
         """Downloads and saves picture with given url under given directory with given timestamp.
         Returns true, if file was actually downloaded, i.e. updated."""
-        if outputlabel is None:
-            outputlabel = _epoch_to_string(date_epoch)
         urlmatch = re.search('\\.[a-z]*\\?', url)
         file_extension = url[-3:] if urlmatch is None else urlmatch.group(0)[1:-1]
-        if self.profile_subdirs:
-            filename = name.lower() + '/' + _epoch_to_string(date_epoch)
-        else:
-            filename = name.lower() + '__' + _epoch_to_string(date_epoch)
         if filename_suffix is not None:
             filename += '_' + filename_suffix
         filename += '.' + file_extension
         if os.path.isfile(filename):
-            self._log(outputlabel + ' exists', end=' ', flush=True)
+            self._log(filename + ' exists', end=' ', flush=True)
             return False
         resp = self.get_anonymous_session().get(url, stream=True)
         if resp.status_code == 200:
-            self._log(outputlabel, end=' ', flush=True)
-            if self.profile_subdirs:
-                os.makedirs(name.lower(), exist_ok=True)
+            self._log(filename, end=' ', flush=True)
             with open(filename, 'wb') as file:
                 resp.raw.decode_content = True
                 shutil.copyfileobj(resp.raw, file)
@@ -362,11 +346,8 @@ class Instaloader:
         else:
             raise ConnectionException("File \'" + url + "\' could not be downloaded.")
 
-    def update_comments(self, name: str, shortcode: str, date_epoch: float) -> None:
-        if self.profile_subdirs:
-            filename = name.lower() + '/' + _epoch_to_string(date_epoch) + '_comments.json'
-        else:
-            filename = name.lower() + '__' + _epoch_to_string(date_epoch) + '_comments.json'
+    def update_comments(self, filename: str, shortcode: str) -> None:
+        filename += '_comments.json'
         try:
             comments = json.load(open(filename))
         except FileNotFoundError:
@@ -388,13 +369,10 @@ class Instaloader:
                 file.write(json.dumps(unique_comments_list, indent=4))
             self._log('comments', end=' ', flush=True)
 
-    def save_caption(self, name: str, date_epoch: float, caption: str) -> None:
+    def save_caption(self, filename: str, date_epoch: float, caption: str) -> None:
         """Updates picture caption"""
         # pylint:disable=too-many-branches
-        if self.profile_subdirs:
-            filename = name.lower() + '/' + _epoch_to_string(date_epoch) + '.txt'
-        else:
-            filename = name.lower() + '__' + _epoch_to_string(date_epoch) + '.txt'
+        filename += '.txt'
         pcaption = caption.replace('\n', ' ').strip()
         caption = caption.encode("UTF-8")
         if self.shorter_output:
@@ -430,22 +408,15 @@ class Instaloader:
             self._log(pcaption, end=' ', flush=True)
         except UnicodeEncodeError:
             self._log('txt', end=' ', flush=True)
-        if self.profile_subdirs:
-            os.makedirs(name.lower(), exist_ok=True)
         with open(filename, 'wb') as text_file:
             shutil.copyfileobj(BytesIO(caption), text_file)
         os.utime(filename, (datetime.datetime.now().timestamp(), date_epoch))
 
-    def save_location(self, name: str, location_json: Dict[str, str], date_epoch: float) -> None:
-        if self.profile_subdirs:
-            filename = name.lower() + '/' + _epoch_to_string(date_epoch) + '_location.txt'
-        else:
-            filename = name.lower() + '__' + _epoch_to_string(date_epoch) + '_location.txt'
+    def save_location(self, filename: str, location_json: Dict[str, str], date_epoch: float) -> None:
+        filename += '_location.txt'
         location_string = (location_json["name"] + "\n" +
                            "https://maps.google.com/maps?q={0},{1}&ll={0},{1}\n".format(location_json["lat"],
                                                                                         location_json["lng"]))
-        if self.profile_subdirs:
-            os.makedirs(name.lower(), exist_ok=True)
         with open(filename, 'wb') as text_file:
             shutil.copyfileobj(BytesIO(location_string.encode()), text_file)
         os.utime(filename, (datetime.datetime.now().timestamp(), date_epoch))
@@ -455,10 +426,14 @@ class Instaloader:
         """Downloads and saves profile pic with given url."""
         date_object = datetime.datetime.strptime(requests.head(url).headers["Last-Modified"],
                                                  '%a, %d %b %Y %H:%M:%S GMT')
-        if self.profile_subdirs:
-            filename = name.lower() + '/' + _epoch_to_string(date_object.timestamp()) + '_UTC_profile_pic.' + url[-3:]
+        if ((format_string_contains_key(self.dirname_pattern, 'profile') or
+             format_string_contains_key(self.dirname_pattern, 'target'))):
+            filename = '{0}/{1}_UTC_profile_pic.{2}'.format(self.dirname_pattern.format(profile=name.lower(),
+                                                                                        target=name.lower()),
+                                                            _epoch_to_string(date_object.timestamp()), url[-3:])
         else:
-            filename = name.lower() + '__' + _epoch_to_string(date_object.timestamp()) + '_UTC_profile_pic.' + url[-3:]
+            filename = '{0}/{1}_{2}_UTC_profile_pic.{3}'.format(self.dirname_pattern.format(), name.lower(),
+                                                                _epoch_to_string(date_object.timestamp()), url[-3:])
         if os.path.isfile(filename):
             self._log(filename + ' already exists')
             return None
@@ -471,8 +446,6 @@ class Instaloader:
         resp = self.get_anonymous_session().get(url, stream=True)
         if resp.status_code == 200:
             self._log(filename)
-            if self.profile_subdirs:
-                os.makedirs(name.lower(), exist_ok=True)
             with open(filename, 'wb') as file:
                 resp.raw.decode_content = True
                 shutil.copyfileobj(resp.raw, file)
@@ -518,7 +491,6 @@ class Instaloader:
         data = self.get_json(str(), session=session)
         if data['config']['viewer'] is None:
             return
-        time.sleep(4 * random.random() + 1)
         return data['config']['viewer']['username']
 
     def login(self, user: str, passwd: str) -> None:
@@ -528,13 +500,13 @@ class Instaloader:
                                 'ig_vw': '1920', 'csrftoken': '',
                                 's_network': '', 'ds_user_id': ''})
         session.headers.update(self.default_http_header())
+        self._sleep()
         resp = session.get('https://www.instagram.com/')
         session.headers.update({'X-CSRFToken': resp.cookies['csrftoken']})
-        time.sleep(9 * random.random() + 3)
+        self._sleep()
         login = session.post('https://www.instagram.com/accounts/login/ajax/',
                              data={'password': passwd, 'username': user}, allow_redirects=True)
         session.headers.update({'X-CSRFToken': login.cookies['csrftoken']})
-        time.sleep(5 * random.random())
         if login.status_code == 200:
             if user == self.test_login(session):
                 self.username = user
@@ -553,38 +525,10 @@ class Instaloader:
         """
         if end_cursor is None:
             return self.get_json(str())["entry_data"]["FeedPage"][0]
-        tmpsession = copy_session(self.session)
-        query = "q=ig_me()+%7B%0A++feed+%7B%0A++++media.after(" + end_cursor + "%2C+12)+%7B%0A" + \
-                "++++++nodes+%7B%0A++++++++id%2C%0A++++++++caption%2C%0A++++++++code%2C%0A++++++++" + \
-                "comments.last(4)+%7B%0A++++++++++count%2C%0A++++++++++nodes+%7B%0A++++++++++++" + \
-                "id%2C%0A++++++++++++created_at%2C%0A++++++++++++text%2C%0A++++++++++++" + \
-                "user+%7B%0A++++++++++++++id%2C%0A++++++++++++++profile_pic_url%2C%0A++++++++++++++" + \
-                "username%0A++++++++++++%7D%0A++++++++++%7D%2C%0A++++++++++" + \
-                "page_info%0A++++++++%7D%2C%0A++++++++comments_disabled%2C%0A++++++++" + \
-                "date%2C%0A++++++++dimensions+%7B%0A++++++++++height%2C%0A++++++++++" + \
-                "width%0A++++++++%7D%2C%0A++++++++display_src%2C%0A++++++++is_video%2C%0A++++++++" + \
-                "likes+%7B%0A++++++++++count%2C%0A++++++++++nodes+%7B%0A++++++++++++" + \
-                "user+%7B%0A++++++++++++++id%2C%0A++++++++++++++profile_pic_url%2C%0A++++++++++++++" + \
-                "username%0A++++++++++++%7D%0A++++++++++%7D%2C%0A++++++++++" + \
-                "viewer_has_liked%0A++++++++%7D%2C%0A++++++++location+%7B%0A++++++++++" + \
-                "id%2C%0A++++++++++has_public_page%2C%0A++++++++++name%0A++++++++%7D%2C%0A++++++++" + \
-                "owner+%7B%0A++++++++++id%2C%0A++++++++++blocked_by_viewer%2C%0A++++++++++" + \
-                "followed_by_viewer%2C%0A++++++++++full_name%2C%0A++++++++++" + \
-                "has_blocked_viewer%2C%0A++++++++++is_private%2C%0A++++++++++" + \
-                "profile_pic_url%2C%0A++++++++++requested_by_viewer%2C%0A++++++++++" + \
-                "username%0A++++++++%7D%2C%0A++++++++usertags+%7B%0A++++++++++" + \
-                "nodes+%7B%0A++++++++++++user+%7B%0A++++++++++++++" + \
-                "username%0A++++++++++++%7D%2C%0A++++++++++++x%2C%0A++++++++++++y%0A++++++++++" + \
-                "%7D%0A++++++++%7D%2C%0A++++++++video_url%2C%0A++++++++" + \
-                "video_views%0A++++++%7D%2C%0A++++++page_info%0A++++%7D%0A++%7D%2C%0A++id%2C%0A++" + \
-                "profile_pic_url%2C%0A++username%0A%7D%0A&ref=feed::show"
-        tmpsession.headers.update(self.default_http_header())
-        tmpsession.headers.update({'Referer': 'https://www.instagram.com/'})
-        tmpsession.headers.update({'Content-Type': 'application/x-www-form-urlencoded'})
-        resp = tmpsession.post('https://www.instagram.com/query/', data=query)
-        if self.sleep:
-            time.sleep(4 * random.random() + 1)
-        return json.loads(resp.text)
+        return self.graphql_query(17863003771166879, {'fetch_media_item_count': 12,
+                                                      'fetch_media_item_cursor': end_cursor,
+                                                      'fetch_comment_count': 4,
+                                                      'fetch_like': 10})
 
     def get_node_metadata(self, node_code: str) -> Dict[str, Any]:
         pic_json = self.get_json("p/" + node_code)
@@ -600,68 +544,83 @@ class Instaloader:
                                           media["location"]["id"])
             return location_json["entry_data"]["LocationsPage"][0]["location"]
 
-    def download_node(self, node: Dict[str, Any], name: str,
+    def download_node(self, node: Dict[str, Any], profile: Optional[str], target: str,
                       download_videos: bool = True, geotags: bool = False, download_comments: bool = False) -> bool:
         """
         Download everything associated with one instagram node, i.e. picture, caption and video.
 
         :param node: Node, as from media->nodes list in instagram's JSONs
-        :param name: Name of profile to which this node belongs
+        :param profile: Name of profile to which this node belongs
+        :param target: Target name, i.e. profile name, #hashtag, :feed; for filename.
         :param download_videos: True, if videos should be downloaded
         :param geotags: Download geotags
         :param download_comments: Update comments
         :return: True if something was downloaded, False otherwise, i.e. file was already there
         """
-        # pylint:disable=too-many-branches,too-many-locals
+        # pylint:disable=too-many-branches,too-many-locals,too-many-statements
+        already_has_profilename = profile is not None or ('owner' in node and 'username' in node['owner'])
+        needs_profilename = (format_string_contains_key(self.dirname_pattern, 'profile') or
+                             format_string_contains_key(self.filename_pattern, 'profile'))
+        shortcode = node['shortcode'] if 'shortcode' in node else node['code']
+        if needs_profilename:
+            if already_has_profilename:
+                profilename = profile if profile is not None else node['owner']['username']
+            else:
+                metadata = self.get_node_metadata(shortcode)
+                profilename = metadata['owner']['username']
+        else:
+            profilename = None
+        profilename = profilename.lower() if profilename else None
         date = node["date"] if "date" in node else node["taken_at_timestamp"]
+        dirname = self.dirname_pattern.format(profile=profilename, target=target.lower())
+        filename = dirname + '/' + self.filename_pattern.format(profile=profilename, target=target.lower(),
+                                                                date=datetime.datetime.fromtimestamp(date),
+                                                                shortcode=shortcode)
+        os.makedirs(dirname, exist_ok=True)
         if '__typename' in node:
             if node['__typename'] == 'GraphSidecar':
-                sidecar_data = self.session.get('https://www.instagram.com/p/' + node['code'] + '/',
+                self._sleep()
+                sidecar_data = self.session.get('https://www.instagram.com/p/' + shortcode + '/',
                                                 params={'__a': 1}).json()
                 edge_number = 1
                 downloaded = True
                 media = sidecar_data["graphql"]["shortcode_media"] if "graphql" in sidecar_data else sidecar_data[
                     "media"]
                 for edge in media['edge_sidecar_to_children']['edges']:
-                    edge_downloaded = self.download_pic(name, edge['node']['display_url'], date,
-                                                        filename_suffix=str(edge_number),
-                                                        outputlabel=(str(edge_number) if edge_number != 1 else None))
+                    edge_downloaded = self.download_pic(filename=filename,
+                                                        url=edge['node']['display_url'],
+                                                        date_epoch=date,
+                                                        filename_suffix=str(edge_number))
                     downloaded = downloaded and edge_downloaded
                     edge_number += 1
-                    if self.sleep:
-                        time.sleep(1.75 * random.random() + 0.25)
             elif node['__typename'] in ['GraphImage', 'GraphVideo']:
-                downloaded = self.download_pic(name,
-                                               node["display_url"] if "display_url" in node else node["display_src"],
-                                               date)
-                if self.sleep:
-                    time.sleep(1.75 * random.random() + 0.25)
+                url = node["display_url"] if "display_url" in node else node["display_src"]
+                downloaded = self.download_pic(filename=filename,
+                                               url=url,
+                                               date_epoch=date)
             else:
                 self._log("Warning: Unknown typename discovered:" + node['__typename'])
                 downloaded = False
         else:
             # Node is an old image or video.
-            downloaded = self.download_pic(name, node["display_src"], date)
-            if self.sleep:
-                time.sleep(1.75 * random.random() + 0.25)
+            downloaded = self.download_pic(filename=filename, url=node["display_src"], date_epoch=date)
         if "edge_media_to_caption" in node and node["edge_media_to_caption"]["edges"]:
-            self.save_caption(name, date, node["edge_media_to_caption"]["edges"][0]["node"]["text"])
+            self.save_caption(filename, date, node["edge_media_to_caption"]["edges"][0]["node"]["text"])
         elif "caption" in node:
-            self.save_caption(name, date, node["caption"])
+            self.save_caption(filename, date, node["caption"])
         else:
             self._log("<no caption>", end=' ', flush=True)
-        node_code = node['shortcode'] if 'shortcode' in node else node['code']
         if node["is_video"] and download_videos:
-            video_data = self.get_json('p/' + node_code)
-            self.download_pic(name,
-                              video_data['entry_data']['PostPage'][0]['graphql']['shortcode_media']['video_url'],
-                              date, 'mp4')
+            video_data = self.get_json('p/' + shortcode)
+            self.download_pic(filename=filename,
+                              url=video_data['entry_data']['PostPage'][0]['graphql']['shortcode_media']['video_url'],
+                              date_epoch=date)
         if geotags:
-            location = self.get_location(node_code)
+            location = self.get_location(shortcode)
             if location:
-                self.save_location(name, location, date)
+                self.save_location(filename, location, date)
         if download_comments:
-            self.update_comments(name, node_code, date)
+            self.update_comments(filename, shortcode)
         self._log()
         return downloaded
 
@@ -695,6 +654,9 @@ class Instaloader:
             if "graphql" in data:
                 is_edge = True
                 feed = data["graphql"]["user"]["edge_web_feed_timeline"]
+            elif "data" in data:
+                is_edge = True
+                feed = data["data"]["user"]["edge_web_feed_timeline"]
             else:
                 is_edge = False
                 feed = data["feed"]["media"]
@@ -708,7 +670,7 @@ class Instaloader:
                     continue
                 self._log("[%3i] %s " % (count, name), end="", flush=True)
                 count += 1
-                downloaded = self.download_node(node, name,
+                downloaded = self.download_node(node, profile=name, target=':feed',
                                                 download_videos=download_videos, geotags=geotags,
                                                 download_comments=download_comments)
                 if fast_update and not downloaded:
@@ -726,8 +688,7 @@ class Instaloader:
                          max_count: Optional[int] = None,
                          filter_func: Optional[Callable[[Dict[str, Dict[str, Any]]], bool]] = None,
                          fast_update: bool = False, download_videos: bool = True, geotags: bool = False,
-                         download_comments: bool = False,
-                         lookup_username: bool = False) -> None:
+                         download_comments: bool = False) -> None:
         """Download pictures of one hashtag.
 
         To download the last 30 pictures with hashtag #cat, do
@@ -741,7 +702,6 @@ class Instaloader:
         :param download_videos: True, if videos should be downloaded
         :param geotags: Download geotags
         :param download_comments: Update comments
-        :param lookup_username: Lookup username to encode it in the downloaded file's path, rather than the hashtag
         """
         data = self.get_hashtag_json(hashtag)
         count = 1
@@ -749,17 +709,12 @@ class Instaloader:
             for node in data['entry_data']['TagPage'][0]['tag']['media']['nodes']:
                 if max_count is not None and count > max_count:
                     return
-                if lookup_username:
-                    metadata = self.get_node_metadata(node['shortcode'] if 'shortcode' in node else node['code'])
-                    pathname = metadata['owner']['username']
-                else:
-                    pathname = '#{0}'.format(hashtag)
-                self._log('[{0:3d}] #{1} {2}/'.format(count, hashtag, pathname), end='', flush=True)
+                self._log('[{0:3d}] #{1} '.format(count, hashtag), end='', flush=True)
                 if filter_func is not None and filter_func(node):
                     self._log('<skipped>')
                     continue
                 count += 1
-                downloaded = self.download_node(node, pathname,
+                downloaded = self.download_node(node=node, profile=None, target='#'+hashtag,
                                                 download_videos=download_videos, geotags=geotags,
                                                 download_comments=download_comments)
                 if fast_update and not downloaded:
@@ -776,11 +731,13 @@ class Instaloader:
         Consult locally stored ID of profile with given name, check whether ID matches and whether name
         has changed and return current name of the profile, and store ID of profile.
         """
-        profile_exists = len(json_data["entry_data"]) > 0 and "ProfilePage" in json_data["entry_data"]
-        if self.profile_subdirs:
-            id_filename = profile.lower() + "/id"
+        profile_exists = "ProfilePage" in json_data["entry_data"]
+        if ((format_string_contains_key(self.dirname_pattern, 'profile') or
+             format_string_contains_key(self.dirname_pattern, 'target'))):
+            id_filename = '{0}/id'.format(self.dirname_pattern.format(profile=profile.lower(),
+                                                                      target=profile.lower()))
         else:
-            id_filename = profile.lower() + "__id"
+            id_filename = '{0}/{1}_id'.format(self.dirname_pattern.format(), profile.lower())
         try:
             with open(id_filename, 'rb') as id_file:
                 profile_id = int(id_file.read())
@@ -792,14 +749,22 @@ class Instaloader:
                     self._log("Trying to find profile {0} using its unique ID {1}.".format(profile, profile_id))
                 newname = self.get_username_by_id(profile_id)
                 self._log("Profile {0} has changed its name to {1}.".format(profile, newname))
-                os.rename(profile, newname)
+                if ((format_string_contains_key(self.dirname_pattern, 'profile') or
+                     format_string_contains_key(self.dirname_pattern, 'target'))):
+                    os.rename(self.dirname_pattern.format(profile=profile.lower(),
+                                                          target=profile.lower()),
+                              self.dirname_pattern.format(profile=newname.lower(),
+                                                          target=newname.lower()))
+                else:
+                    os.rename('{0}/{1}_id'.format(self.dirname_pattern.format(), profile.lower()),
+                              '{0}/{1}_id'.format(self.dirname_pattern.format(), newname.lower()))
                 return newname
             return profile
         except FileNotFoundError:
             pass
         if profile_exists:
-            if self.profile_subdirs:
-                os.makedirs(profile.lower(), exist_ok=True)
+            os.makedirs(self.dirname_pattern.format(profile=profile.lower(),
+                                                    target=profile.lower()), exist_ok=True)
             with open(id_filename, 'w') as text_file:
                 profile_id = json_data['entry_data']['ProfilePage'][0]['user']['id']
                 text_file.write(profile_id + "\n")
@@ -822,8 +787,6 @@ class Instaloader:
             data = self.get_json(name)
         # Download profile picture
         self.download_profilepic(name, data["entry_data"]["ProfilePage"][0]["user"]["profile_pic_url"])
-        if self.sleep:
-            time.sleep(1.75 * random.random() + 0.25)
         if profile_pic_only:
             return
         # Catch some errors
@@ -851,7 +814,7 @@ class Instaloader:
             for node in data["entry_data"]["ProfilePage"][0]["user"]["media"]["nodes"]:
                 self._log("[%3i/%3i] " % (count, totalcount), end="", flush=True)
                 count += 1
-                downloaded = self.download_node(node, name,
+                downloaded = self.download_node(node=node, profile=name, target=name,
                                                 download_videos=download_videos, geotags=geotags,
                                                 download_comments=download_comments)
                 if fast_update and not downloaded:
@@ -877,7 +840,7 @@ class Instaloader:
                           sessionfile: Optional[str] = None, max_count: Optional[int] = None,
                           profile_pic_only: bool = False, download_videos: bool = True, geotags: bool = False,
                           download_comments: bool = False,
-                          fast_update: bool = False, hashtag_lookup_username: bool = False) -> None:
+                          fast_update: bool = False) -> None:
         """Download set of profiles and handle sessions"""
         # pylint:disable=too-many-branches,too-many-locals,too-many-statements
         # Login, if desired
@@ -904,7 +867,7 @@ class Instaloader:
                     self._log("Retrieving pictures with hashtag {0}".format(pentry))
                     self.download_hashtag(hashtag=pentry[1:], max_count=max_count, fast_update=fast_update,
                                           download_videos=download_videos, geotags=geotags,
-                                          download_comments=download_comments, lookup_username=hashtag_lookup_username)
+                                          download_comments=download_comments)
                 elif pentry[0] == '@':
                     if username is not None:
                         self._log("Retrieving followees of %s..." % pentry[1:])
@@ -947,7 +910,7 @@ class Instaloader:
                             self._log(err)
                             self._log("Trying again anonymously, helps in case you are just blocked.")
                             anonymous_loader = Instaloader(self.sleep, self.quiet, self.shorter_output,
-                                                           self.profile_subdirs, self.user_agent)
+                                                           self.user_agent, self.dirname_pattern, self.filename_pattern)
                             anonymous_loader.download(target, profile_pic_only, download_videos,
                                                       geotags, download_comments, fast_update)
                         else:
@@ -1022,14 +985,16 @@ def main():
                               'there is not yet a valid session file.')
 
     g_how = parser.add_argument_group('How to Download')
-    g_how.add_argument('--no-profile-subdir', action='store_true',
-                       help='Instead of creating a subdirectory for each profile and storing pictures there, store '
-                            'pictures in files named PROFILE__DATE_TIME.jpg.')
-    g_how.add_argument('--hashtag-username', action='store_true',
-                       help='Lookup username of pictures when downloading by #hashtag and encode it in the downlaoded '
-                            'file\'s path or filename (if --no-profile-subdir). Without this option, the #hashtag is '
-                            'used instead. This requires an additional request to the Instagram server for each '
-                            'picture, which is why it is disabled by default.')
+    g_how.add_argument('--dirname-pattern',
+                       help='Name of directory where to store posts. {profile} is replaced by the profile name, '
+                            '{target} is replaced by the target you specified, i.e. either :feed, #hashtag or the '
+                            'profile name. Defaults to \'{target}\'.')
+    g_how.add_argument('--filename-pattern',
+                       help='Prefix of filenames. Posts are stored in the directory whose pattern is given with '
+                            '--dirname-pattern. {profile} is replaced by the profile name, '
+                            '{target} is replaced by the target you specified, i.e. either :feed, #hashtag or the '
+                            'profile name. Also, the fields date and shortcode can be specified. Defaults to '
+                            '\'{date:%%Y-%%m-%%d_%%H-%%M-%%S}\'.')
     g_how.add_argument('--user-agent',
                        help='User Agent to use for HTTP requests. Defaults to \'{}\'.'.format(default_user_agent()))
     g_how.add_argument('-S', '--no-sleep', action='store_true',
@@ -1050,11 +1015,12 @@ def main():
     args = parser.parse_args()
     try:
         loader = Instaloader(sleep=not args.no_sleep, quiet=args.quiet, shorter_output=args.shorter_output,
-                             profile_subdirs=not args.no_profile_subdir, user_agent=args.user_agent)
+                             user_agent=args.user_agent,
+                             dirname_pattern=args.dirname_pattern, filename_pattern=args.filename_pattern)
         loader.download_profiles(args.profile, args.login, args.password, args.sessionfile,
                                  int(args.count) if args.count is not None else None,
                                  args.profile_pic_only, not args.skip_videos, args.geotags, args.comments,
-                                 args.fast_update, args.hashtag_username)
+                                 args.fast_update)
     except InstaloaderException as err:
         raise SystemExit("Fatal error: %s" % err)
 

setup.py

@@ -8,15 +8,14 @@ if sys.version_info < (3, 5):
 
 setup(
     name='instaloader',
-    version='2.0',
+    version='2.1',
     py_modules=['instaloader'],
     url='https://github.com/Thammus/instaloader',
     license='MIT',
     author='Alexander Graf, André Koch-Kramer',
     author_email='mail@agraf.me, koch-kramer@web.de',
-    description='Tool to download Instagram pictures (or videos) and captions from a given set of '
-                'profiles (even if private), from your feed or from all followees of a given '
-                'profile.',
+    description='Download pictures (or videos) along with their captions and other metadata '
+                'from Instagram.',
     long_description=open('README.rst').read(),
     install_requires=['requests>=2.4'],
     python_requires='>=3.5',