Small fixes to utils and compat and test

4 files changed, 19 insertions, 10 deletions

diff --git a/test/test_compat.py b/test/test_compat.py
index 86ff389fd..04581e6ca 100644
--- a/test/test_compat.py
+++ b/test/test_compat.py
@@ -39,8 +39,12 @@ class TestCompat(unittest.TestCase):
         self.assertEqual(compat_getenv(test_var), test_str)
 
     def test_compat_expanduser(self):
+        from youtube_dl.compat import compat_os_name
         old_home = os.environ.get('HOME')
-        test_str = r'C:\Documents and Settings\тест\Application Data'
+        if compat_os_name in ('nt', 'ce'):
+            test_str = r'C:\Documents and Settings\тест\Application Data'
+        else:
+            test_str = '/home/тест'
         compat_setenv('HOME', test_str)
         self.assertEqual(compat_expanduser('~'), test_str)
         compat_setenv('HOME', old_home or '')
diff --git a/test/test_utils.py b/test/test_utils.py
index 259c4763e..6f8945792 100644
--- a/test/test_utils.py
+++ b/test/test_utils.py
@@ -480,7 +480,7 @@ class TestUtil(unittest.TestCase):
         args = ['ffmpeg', '-i', encodeFilename('ñ€ß\'.mp4')]
         self.assertEqual(
             shell_quote(args),
-            """ffmpeg -i 'ñ€ß'"'"'.mp4'""" if compat_os_name != 'nt' else '''ffmpeg -i "ñ€ß'.mp4"''')
+            """ffmpeg -i 'ñ€ß'"'"'.mp4'""" if not(compat_os_name in ('nt', 'ce')) else '''ffmpeg -i "ñ€ß'.mp4"''')
 
     def test_float_or_none(self):
         self.assertEqual(float_or_none('42.42'), 42.42)
@@ -1085,7 +1085,7 @@ class TestUtil(unittest.TestCase):
     def test_args_to_str(self):
         self.assertEqual(
             args_to_str(['foo', 'ba/r', '-baz', '2 be', '']),
-            'foo ba/r -baz \'2 be\' \'\'' if compat_os_name != 'nt' else 'foo ba/r -baz "2 be" ""'
+            'foo ba/r -baz \'2 be\' \'\'' if not(compat_os_name in ('nt', 'ce')) else 'foo ba/r -baz "2 be" ""'
         )
 
     def test_parse_filesize(self):
diff --git a/youtube_dl/compat.py b/youtube_dl/compat.py
index 9e45c454b..90dd20457 100644
--- a/youtube_dl/compat.py
+++ b/youtube_dl/compat.py
@@ -2700,16 +2700,19 @@ else:
     # Otherwise it will fail if any non-ASCII characters present (see #3854 #3217 #2918)
 
     def compat_getenv(key, default=None):
-        from .utils import get_filesystem_encoding
         env = os.getenv(key, default)
         if env:
-            env = env.decode(get_filesystem_encoding())
+            from .utils import get_filesystem_encoding
+            encoding = get_filesystem_encoding()
+            env = env.decode(encoding)
+            if not encoding.lower().startswith('ut'):
+                env = env.encode('utf-8').decode('unicode-escape')
         return env
 
     def compat_setenv(key, value, env=os.environ):
         def encode(v):
             from .utils import get_filesystem_encoding
-            return v.encode(get_filesystem_encoding()) if isinstance(v, compat_str) else v
+            return v.encode(get_filesystem_encoding(), 'backslashreplace') if isinstance(v, compat_str) else v
         env[encode(key)] = encode(value)
 
     # HACK: The default implementations of os.path.expanduser from cpython do not decode
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index e722eed58..eaf86bb44 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -2245,7 +2245,8 @@ def encodeFilename(s, for_subprocess=False):
     if sys.platform.startswith('java'):
         return s
 
-    return s.encode(get_subprocess_encoding(), 'ignore')
+    # If encoding is (eg) 'ascii', use escape sequences (allows round-trip test)
+    return s.encode(get_subprocess_encoding(), 'backslashreplace')
 
 
 def decodeFilename(b, for_subprocess=False):
@@ -3354,8 +3355,7 @@ class locked_file(object):
 
 
 def get_filesystem_encoding():
-    encoding = sys.getfilesystemencoding()
-    return encoding if encoding is not None else 'utf-8'
+    return sys.getfilesystemencoding() or sys.getdefaultencoding() or 'utf-8'
 
 
 def shell_quote(args):
@@ -3365,6 +3365,8 @@ def shell_quote(args):
         if isinstance(a, bytes):
             # We may get a filename encoded with 'encodeFilename'
             a = a.decode(encoding)
+            if not encoding.lower().startswith('ut'):
+                a = a.encode('utf-8').decode('unicode-escape')
         quoted_args.append(compat_shlex_quote(a))
     return ' '.join(quoted_args)
 
@@ -4610,7 +4612,7 @@ def dfxp2srt(dfxp_data):
             continue
         default_style.update(style)
 
-    for para, index in zip(paras, itertools.count(1)):
+    for index, para in enumerate(paras, 1):
         begin_time = parse_dfxp_time_expr(para.attrib.get('begin'))
         end_time = parse_dfxp_time_expr(para.attrib.get('end'))
         dur = parse_dfxp_time_expr(para.attrib.get('dur'))