Release of version 1.0.1

Closes #11.

instaloader now tries to use win-unicode-console when recognizing
windows.

README.md

@@ -7,6 +7,9 @@ Simple downloader to fetch all Instagram pictures and captions from a given prof
 Ensure having [Python](https://www.python.org/) (at least version 3.3) and
 [python3-requests](https://pypi.python.org/pypi/requests/) installed.
 
+If you intend to use this tool under Windows, it is highly recommended to first install
+[win-unicode-console](https://github.com/Drekin/win-unicode-console).
+
 After having [downloaded instaloader.py](https://github.com/Thammus/instaloader/releases), you invoke it with
 ```
 ./instaloader.py profile [profile ...]
@@ -29,6 +32,11 @@ When invoked like this, it also **stores the session cookies** in a file in `/tm
 reused later when `--login` is given. So you can download private profiles **non-interactively**
 when you already have a valid session cookies file.
 
+If you want to **download all followees of a given profile**, call
+```
+./instaloader.py --login=your_username @profile
+```
+
 The `--quiet` option makes it also **suitable as a cron job**.
 
 To get a list of other helpful flags, run `./instaloader.py --help`.

instaloader.py

@@ -3,6 +3,15 @@
 import re, json, datetime, shutil, os, time, random, sys, pickle, getpass, tempfile
 from argparse import ArgumentParser
 import requests, requests.utils
+from io import BytesIO
+
+try:
+    import win_unicode_console
+except ImportError:
+    WINUNICODE = False
+else:
+    win_unicode_console.enable()
+    WINUNICODE = True
 
 class InstaloaderException(Exception):
     """Base exception for this script"""
@@ -62,14 +71,14 @@ def get_username_by_id(session, profile_id):
         data = json.loads(resp.text)
         if 'username' in data:
             return json.loads(resp.text)['username']
-        raise ProfileNotExistsException("no profile found, the user may have blocked " +
-                                            "you (id: " + str(profile_id) + ")")
+        raise ProfileNotExistsException("No profile found, the user may have blocked " +
+                                            "you (id: " + str(profile_id) + ").")
     else:
         if test_login(session):
-            raise ConnectionException("username could not be determined due to ConnectionError" +
-                                        " (id: "+ str(profile_id) +")")
+            raise ConnectionException("Username could not be determined due to connection error" +
+                                        " (id: "+ str(profile_id) +").")
         raise LoginRequiredException("Login required to determine username (id: " +
-                                        str(profile_id) + ")")
+                                        str(profile_id) + ").")
 
 def epoch_to_string(epoch):
     return datetime.datetime.fromtimestamp(epoch).strftime('%Y-%m-%d_%H-%M-%S')
@@ -127,8 +136,8 @@ def get_followees(profile, session):
         return followees
     if test_login(tmpsession):
         raise ConnectionException("ConnectionError("+str(resp.status_code)+"): "
-                                  "unable to gather followees")
-    raise LoginRequiredException("Login required to gather followees")
+                                  "unable to gather followees.")
+    raise LoginRequiredException("Login required to gather followees.")
 
 def download_pic(name, url, date_epoch, outputlabel=None, quiet=False):
     # Returns true, if file was actually downloaded, i.e. updated
@@ -136,11 +145,11 @@ def download_pic(name, url, date_epoch, outputlabel=None, quiet=False):
         outputlabel = epoch_to_string(date_epoch)
     filename = name.lower() + '/' + epoch_to_string(date_epoch) + '.' + get_file_extension(url)
     if os.path.isfile(filename):
-        log(outputlabel + ' exists', end='  ', flush=True, quiet=quiet)
+        log(outputlabel + ' exists', end=' ', flush=True, quiet=quiet)
         return False
     resp = get_anonymous_session().get(url, stream=True)
     if resp.status_code == 200:
-        log(outputlabel, end='  ', flush=True, quiet=quiet)
+        log(outputlabel, end=' ', flush=True, quiet=quiet)
         os.makedirs(name.lower(), exist_ok=True)
         with open(filename, 'wb') as file:
             resp.raw.decode_content = True
@@ -148,15 +157,22 @@ def download_pic(name, url, date_epoch, outputlabel=None, quiet=False):
         os.utime(filename, (datetime.datetime.now().timestamp(), date_epoch))
         return True
     else:
-        raise ConnectionException("file \'" + url + "\' could not be downloaded")
+        raise ConnectionException("File \'" + url + "\' could not be downloaded.")
 
 def save_caption(name, date_epoch, caption, quiet=False):
     filename = name.lower() + '/' + epoch_to_string(date_epoch) + '.txt'
+    pcaption = caption.replace('\n', ' ').strip()
+    caption = caption.encode("UTF-8")
+    if os.name in ['nt', 'ce'] and not WINUNICODE:
+        output = str()
+    else:
+        output = '[' + ((pcaption[:26]+"…") if len(pcaption)>28 else pcaption) + ']'
     if os.path.isfile(filename):
-        with open(filename, 'r') as file:
+        with open(filename, 'rb') as file:
             file_caption = file.read()
-        if file_caption == caption:
-            log('txt unchanged', end=' ', flush=True, quiet=quiet)
+        if file_caption.replace(b'\r\n', b'\n') == caption.replace(b'\r\n', b'\n'):
+            output = output + ' unchanged'
+            log(output, end=' ', flush=True, quiet=quiet)
             return None
         else:
             def get_filename(index):
@@ -167,11 +183,11 @@ def save_caption(name, date_epoch, caption, quiet=False):
                 i = i + 1
             for index in range(i, 0, -1):
                 os.rename(get_filename(index-1), get_filename(index))
-            log('txt updated', end=' ', flush=True, quiet=quiet)
-    log('txt', end=' ', flush=True, quiet=quiet)
+            output = output + ' updated'
+    log(output, end=' ', flush=True, quiet=quiet)
     os.makedirs(name.lower(), exist_ok=True)
-    with open(filename, 'w') as text_file:
-        text_file.write(caption)
+    with open(filename, 'wb') as text_file:
+        shutil.copyfileobj(BytesIO(caption), text_file)
     os.utime(filename, (datetime.datetime.now().timestamp(), date_epoch))
 
 def download_profilepic(name, url, quiet=False):
@@ -184,7 +200,7 @@ def download_profilepic(name, url, quiet=False):
         return None
     match = re.search('http.*://.*instagram.*[^/]*\.(com|net)/[^/]+/.', url)
     if match is None:
-        raise ConnectionException("url \'" + url + "\' could not be processed")
+        raise ConnectionException("URL \'" + url + "\' could not be processed.")
     index = len(match.group(0))-1
     offset = 8 if match.group(0)[-1:] == 's' else 0
     url = url[:index] + 's2048x2048' + ('/' if offset == 0 else str()) + url[index+offset:]
@@ -197,7 +213,7 @@ def download_profilepic(name, url, quiet=False):
             shutil.copyfileobj(resp.raw, file)
         os.utime(filename, (datetime.datetime.now().timestamp(), date_object.timestamp()))
     else:
-        raise ConnectionException("file \'" + url + "\' could not be downloaded")
+        raise ConnectionException("File \'" + url + "\' could not be downloaded.")
 
 def get_default_session_filename(username):
     dirname = tempfile.gettempdir() + "/" + ".instaloader-" + getpass.getuser()
@@ -300,11 +316,11 @@ def get_session(user, passwd):
 def download(name, session, profile_pic_only=False, download_videos=True,
         fast_update=False, sleep_min_max=[0.25,2], quiet=False):
     """Download one profile"""
-    # pylint:disable=too-many-arguments
+    # pylint:disable=too-many-arguments,too-many-branches
     # Get profile main page json
     data = get_json(name, session)
     if len(data["entry_data"]) == 0 or "ProfilePage" not in data["entry_data"]:
-        raise ProfileNotExistsException("profile %s does not exist" % name)
+        raise ProfileNotExistsException("Profile %s does not exist." % name)
     # Download profile picture
     download_profilepic(name, data["entry_data"]["ProfilePage"][0]["user"]["profile_pic_url"],
             quiet=quiet)
@@ -316,14 +332,14 @@ def download(name, session, profile_pic_only=False, download_videos=True,
         if data["config"]["viewer"] is None:
             raise LoginRequiredException("profile %s requires login" % name)
         if not data["entry_data"]["ProfilePage"][0]["user"]["followed_by_viewer"]:
-            raise PrivateProfileNotFollowedException("user %s: private but not followed" % name)
+            raise PrivateProfileNotFollowedException("Profile %s: private but not followed." % name)
     else:
         if data["config"]["viewer"] is not None:
             log("profile %s could also be downloaded anonymously." % name, quiet=quiet)
     if ("nodes" not in data["entry_data"]["ProfilePage"][0]["user"]["media"] or
             len(data["entry_data"]["ProfilePage"][0]["user"]["media"]["nodes"]) == 0) \
                     and not profile_pic_only:
-        raise ProfileHasNoPicsException("profile %s: no pics found" % name)
+        raise ProfileHasNoPicsException("Profile %s: no pics found." % name)
     # Iterate over pictures and download them
     totalcount = data["entry_data"]["ProfilePage"][0]["user"]["media"]["count"]
     count = 1
@@ -336,6 +352,8 @@ def download(name, session, profile_pic_only=False, download_videos=True,
                        abs(sleep_min_max[0]))
             if "caption" in node:
                 save_caption(name, node["date"], node["caption"], quiet=quiet)
+            else:
+                log("<no caption>", end=' ', flush=True, quiet=quiet)
             if node["is_video"] and download_videos:
                 video_data = get_json('p/' + node["code"], session)
                 download_pic(name, \
@@ -362,11 +380,11 @@ def get_logged_in_session(username, password=None, quiet=False):
             print(err, file=sys.stderr)
             password = None
 
-def download_profiles(targets, username=None, password=None, sessionfile=None,
+def download_profiles(profilelist, username=None, password=None, sessionfile=None,
         profile_pic_only=False, download_videos=True, fast_update=False,
         sleep_min_max=[0.25,2], quiet=False):
     """Download set of profiles and handle sessions"""
-    # pylint:disable=too-many-arguments
+    # pylint:disable=too-many-arguments,too-many-branches,too-many-locals
     # Login, if desired
     if username is not None:
         session = load_session(username, sessionfile, quiet=quiet)
@@ -375,9 +393,23 @@ def download_profiles(targets, username=None, password=None, sessionfile=None,
         log("Logged in as %s." % username, quiet=quiet)
     else:
         session = get_anonymous_session()
-    # Iterate through targets list and download them
+    # Try block for KeyboardInterrupt (save session on ^C)
     failedtargets = []
+    targets = set()
     try:
+        # Generate set of targets
+        for pentry in profilelist:
+            if pentry[0] == '@':
+                log("Retrieving followees of %s..." % pentry[1:], quiet=quiet)
+                followees = get_followees(pentry[1:], session)
+                targets.update([followee['username'] for followee in followees])
+            else:
+                targets.add(pentry)
+        if len(targets) == 0:
+            log("No profiles to download given.", quiet=quiet)
+        elif len(targets) > 1:
+            log("Downloading %i profiles..." % len(targets), quiet=quiet)
+        # Iterate through targets list and download them
         for target in targets:
             try:
                 download(target, session, profile_pic_only, download_videos,
@@ -388,7 +420,7 @@ def download_profiles(targets, username=None, password=None, sessionfile=None,
     except KeyboardInterrupt:
         print("\nInterrupted by user.", file=sys.stderr)
     if len(targets) > 1 and len(failedtargets) > 0:
-        print("Errors occured (see above) while downloading profiles: %s" %
+        print("Errors occured (see above) while downloading profiles: %s." %
                 ", ".join(failedtargets), file=sys.stderr)
     # Save session if it is useful
     if username is not None:
@@ -397,13 +429,20 @@ def download_profiles(targets, username=None, password=None, sessionfile=None,
 def main():
     parser = ArgumentParser(description='Simple downloader to fetch all Instagram pics and '\
                                         'captions from a given profile')
-    parser.add_argument('targets', nargs='+', help='Names of profiles to download')
+    parser.add_argument('profile', nargs='*',
+            help='Name of profile to download, or @<profile> to download all followees of '\
+                    '<profile>')
     parser.add_argument('--version', action='version',
-                        version='1.0')
-    parser.add_argument('-l', '--login', metavar='login_name',
-            help='Provide login name for your Instagram account')
-    parser.add_argument('-p', '--password',
-            help='Provide password for your Instagram account')
+                        version='1.0.1')
+    parser.add_argument('-l', '--login', metavar='YOUR-USERNAME',
+            help='Login name for your Instagram account. Not needed to download public '\
+                    'profiles, but if you want to download private profiles or all followees of '\
+                    'some profile, you have to specify a username used to login.')
+    parser.add_argument('-p', '--password', metavar='YOUR-PASSWORD',
+            help='Password for your Instagram account. If --login is given and there is '\
+                    'not yet a valid session file, you\'ll be prompted for your password if '\
+                    '--password is not given. Specifying this option without --login has no '\
+                    'effect.')
     parser.add_argument('-f', '--sessionfile',
             help='File to store session key, defaults to '+ \
             get_default_session_filename("<login_name>"))
@@ -420,7 +459,7 @@ def main():
                     'if login credentials are needed but not given.')
     args = parser.parse_args()
     try:
-        download_profiles(args.targets, args.login, args.password, args.sessionfile,
+        download_profiles(args.profile, args.login, args.password, args.sessionfile,
                 args.profile_pic_only, not args.skip_videos, args.fast_update,
                 [0,0] if args.no_sleep else [0.25,2], args.quiet)
     except InstaloaderException as err: