[utils] Add posix expanduser implementation and clarify the original source

youtube_dl/utils.py

@@ -204,18 +204,50 @@ def compat_ord(c):
     else: return ord(c)
 
 
-# Environment variables should be decoded with filesystem encoding
-# otherwise this results in issues like #3854 #2918 #3217
 if sys.version_info >= (3, 0):
     compat_getenv = os.getenv
     compat_expanduser = os.path.expanduser
 else:
+    # Environment variables should be decoded with filesystem encoding.
+    # Otherwise it will fail if any non-ASCII characters present (see #3854 #3217 #2918)
+
     def compat_getenv(key, default=None):
         env = os.getenv(key, default)
         if env:
             env = env.decode(get_filesystem_encoding())
         return env
 
+    # HACK: The default implementations of os.path.expanduser from cpython do not decode
+    # environment variables with filesystem encoding. We will work around this by
+    # providing adjusted implementations.
+    # The following are os.path.expanduser implementations from cpython 2.7.8 stdlib
+    # for different platforms with correct environment variables decoding.
+
+    if os.name == 'posix':
+        def compat_expanduser(path):
+            """Expand ~ and ~user constructions.  If user or $HOME is unknown,
+            do nothing."""
+            if not path.startswith('~'):
+                return path
+            i = path.find('/', 1)
+            if i < 0:
+                i = len(path)
+            if i == 1:
+                if 'HOME' not in os.environ:
+                    import pwd
+                    userhome = pwd.getpwuid(os.getuid()).pw_dir
+                else:
+                    userhome = compat_getenv('HOME')
+            else:
+                import pwd
+                try:
+                    pwent = pwd.getpwnam(path[1:i])
+                except KeyError:
+                    return path
+                userhome = pwent.pw_dir
+            userhome = userhome.rstrip('/')
+            return (userhome + path[i:]) or '/'
+    elif os.name == 'nt' or os.name == 'ce':
         def compat_expanduser(path):
             """Expand ~ and ~user constructs.
 
@@ -224,7 +256,7 @@ else:
                 return path
             i, n = 1, len(path)
             while i < n and path[i] not in '/\\':
-            i += 1
+                i = i + 1
 
             if 'HOME' in os.environ:
                 userhome = compat_getenv('HOME')
@@ -243,6 +275,8 @@ else:
                 userhome = os.path.join(os.path.dirname(userhome), path[1:i])
 
             return userhome + path[i:]
+    else:
+        compat_expanduser = os.path.expanduser
 
 
 # This is not clearly defined otherwise