ncoghlan · Jun 11, 2017 · Mar 5, 2017 · Mar 11, 2017 · Mar 11, 2017 · Mar 13, 2017
diff --git a/Doc/using/cmdline.rst b/Doc/using/cmdline.rst
@@ -713,6 +713,35 @@ conflict.
 
    .. versionadded:: 3.6
 
+
+.. envvar:: PYTHONCOERCECLOCALE
+
+   If set to a non-empty string, causes the main Python command line application
+   to skip coercing the legacy ASCII-based C locale to a more capable UTF-8
+   based alternative. Note that this setting is checked even when the
+   :option:`-E` or :option:`-I` options are used, as it is handled prior to
+   the processing of command line options.
+
+   If this variable is *not* set, and the current locale reported for the
+   ``LC_CTYPE`` category is the default ``C`` locale, then the Python CLI will
+   attempt to configure one of the following locales for the given locale
+   categories before loading the interpreter runtime:
+
+   * ``C.UTF-8` (``LC_ALL``)
+   * ``C.utf8` (``LC_ALL``)
+   * ``UTF-8` (``LC_CTYPE``)
+
+   If setting one of these locale categories succeeds, then the matching
+   environment variables will be set (both ``LC_ALL` and ``LANG`` for the
+   ``LC_ALL`` category, and ``LC_CTYPE`` for the ``LC_CTYPE`` category),
+   and (if not already set to a non-empty string) :envvar:`PYTHONIOENCODING`
+   will be set to ``utf-8:surrogateescape``.
+
+   Availability: \*nix
+
+   .. versionadded:: 3.7
+      See :pep:`538` for more details.
+
 Debug-mode variables
 ~~~~~~~~~~~~~~~~~~~~
 

diff --git a/Lib/test/support/script_helper.py b/Lib/test/support/script_helper.py
@@ -48,8 +48,35 @@ def interpreter_requires_environment():
     return __cached_interp_requires_environment
 
 
-_PythonRunResult = collections.namedtuple("_PythonRunResult",
-                                          ("rc", "out", "err"))
+class _PythonRunResult(collections.namedtuple("_PythonRunResult",
+                                          ("rc", "out", "err"))):
+    """Helper for reporting Python subprocess run results"""
+    def fail(self, cmd_line):
+        """Provide helpful details about failed subcommand runs"""
+        # Limit to 80 lines to ASCII characters
+        maxlen = 80 * 100
+        out, err = self.out, self.err
+        if len(out) > maxlen:
+            out = b'(... truncated stdout ...)' + out[-maxlen:]
+        if len(err) > maxlen:
+            err = b'(... truncated stderr ...)' + err[-maxlen:]
+        out = out.decode('ascii', 'replace').rstrip()
+        err = err.decode('ascii', 'replace').rstrip()
+        raise AssertionError("Process return code is %d\n"
+                             "command line: %r\n"
+                             "\n"
+                             "stdout:\n"
+                             "---\n"
+                             "%s\n"
+                             "---\n"
+                             "\n"
+                             "stderr:\n"
+                             "---\n"
+                             "%s\n"
+                             "---"
+                             % (self.rc, cmd_line,
+                                out,
+                                err))
 
 
 # Executing the interpreter in a subprocess
@@ -107,30 +134,7 @@ def run_python_until_end(*args, **env_vars):
 def _assert_python(expected_success, *args, **env_vars):
     res, cmd_line = run_python_until_end(*args, **env_vars)
     if (res.rc and expected_success) or (not res.rc and not expected_success):
-        # Limit to 80 lines to ASCII characters
-        maxlen = 80 * 100
-        out, err = res.out, res.err
-        if len(out) > maxlen:
-            out = b'(... truncated stdout ...)' + out[-maxlen:]
-        if len(err) > maxlen:
-            err = b'(... truncated stderr ...)' + err[-maxlen:]
-        out = out.decode('ascii', 'replace').rstrip()
-        err = err.decode('ascii', 'replace').rstrip()
-        raise AssertionError("Process return code is %d\n"
-                             "command line: %r\n"
-                             "\n"
-                             "stdout:\n"
-                             "---\n"
-                             "%s\n"
-                             "---\n"
-                             "\n"
-                             "stderr:\n"
-                             "---\n"
-                             "%s\n"
-                             "---"
-                             % (res.rc, cmd_line,
-                                out,
-                                err))
+        res.fail(cmd_line)
     return res
 
 def assert_python_ok(*args, **env_vars):

diff --git a/Lib/test/test_capi.py b/Lib/test/test_capi.py
@@ -386,7 +386,7 @@ def run_embedded_interpreter(self, *args):
     def test_subinterps(self):
         # This is just a "don't crash" test
         out, err = self.run_embedded_interpreter("repeated_init_and_subinterpreters")
-        if support.verbose:
+        if support.verbose > 1:
             print()
             print(out)
             print(err)
@@ -404,14 +404,15 @@ def _get_default_pipe_encoding():
     def test_forced_io_encoding(self):
         # Checks forced configuration of embedded interpreter IO streams
         out, err = self.run_embedded_interpreter("forced_io_encoding")
-        if support.verbose:
+        if support.verbose > 1:
             print()
             print(out)
             print(err)
-        expected_errors = sys.__stdout__.errors
-        expected_stdin_encoding = sys.__stdin__.encoding
+        expected_errors = "surrogateescape"
+        expected_stdin_encoding = "UTF-8"
         expected_pipe_encoding = self._get_default_pipe_encoding()
         expected_output = '\n'.join([
+        "Setting PYTHONIOENCODING=UTF-8:surrogateescape",
         "--- Use defaults ---",
         "Expected encoding: default",
         "Expected errors: default",

diff --git a/Lib/test/test_cmd_line.py b/Lib/test/test_cmd_line.py
@@ -8,8 +8,9 @@
 import subprocess
 import tempfile
 from test.support import script_helper, is_android
-from test.support.script_helper import (spawn_python, kill_python, assert_python_ok,
-    assert_python_failure)
+from test.support.script_helper import (
+    spawn_python, kill_python, assert_python_ok, assert_python_failure
+)
 
 
 # XXX (ncoghlan): Move to script_helper and make consistent with run_python
@@ -150,6 +151,7 @@ def test_undecodable_code(self):
         env = os.environ.copy()
         # Use C locale to get ascii for the locale encoding
         env['LC_ALL'] = 'C'
+        env['PYTHONCOERCECLOCALE'] = '0'
         code = (
             b'import locale; '
             b'print(ascii("' + undecodable + b'"), '

diff --git a/Lib/test/test_locale_coercion.py b/Lib/test/test_locale_coercion.py
@@ -0,0 +1,199 @@
+# Tests the attempted automatic coercion of the C locale to a UTF-8 locale
+
+import unittest
+import os
+import sys
+import sysconfig
+import shutil
+import subprocess
+import test.support
+from test.support.script_helper import (
+    run_python_until_end,
+    interpreter_requires_environment,
+)
+
+# In order to get the warning messages to match up as expected, the candidate
+# order here must much the target locale order in Programs/python.c
+_C_UTF8_LOCALES = (
+    # Entries: (Target locale, target category, expected env var updates)
+    ("C.UTF-8", "LC_ALL", "LC_ALL & LANG"),
+    ("C.utf8", "LC_ALL", "LC_ALL & LANG"),
+    ("UTF-8", "LC_CTYPE", "LC_CTYPE"),
+)
+
+# There's no reliable cross-platform way of checking locale alias
+# lists, so the only way of knowing which of these locales will work
+# is to try them with locale.setlocale(). We do that in a subprocess
+# to avoid altering the locale of the test runner.
+def _set_locale_in_subprocess(locale_name, category):
+    cmd_fmt = "import locale; print(locale.setlocale(locale.{}, '{}'))"
+    cmd = cmd_fmt.format(category, locale_name)
+    result, py_cmd = run_python_until_end("-c", cmd, __isolated=True)
+    return result.rc == 0
+
+# Details of the CLI warning emitted at runtime
+CLI_COERCION_WARNING_FMT = (
+    "Python detected LC_CTYPE=C: {} coerced to {} (set another locale "
+    "or PYTHONCOERCECLOCALE=0 to disable this locale coercion behaviour)."
+)
+
+@test.support.cpython_only
+@unittest.skipUnless(sysconfig.get_config_var("PY_COERCE_C_LOCALE"),
+                     "C locale coercion disabled at build time")
+class LocaleOverrideTests(unittest.TestCase):
+
+    @classmethod
+    def setUpClass(cls):
+        for target_locale, target_category, env_updates in _C_UTF8_LOCALES:
+            if _set_locale_in_subprocess(target_locale, target_category):
+                break
+        else:
+            raise unittest.SkipTest("No C-with-UTF-8 locale available")
+        cls.EXPECTED_COERCION_WARNING = CLI_COERCION_WARNING_FMT.format(
+            env_updates, target_locale
+        )
+
+    def _get_child_fsencoding(self, env_vars):
+        """Retrieves sys.getfilesystemencoding() from a child process
+
+        Returns (fsencoding, stderr_lines):
+
+        - fsencoding: a lowercase str value with the child's fsencoding
+        - stderr_lines: result of calling splitlines() on the stderr output
+
+        The child is run in isolated mode if the current interpreter supports
+        that.
+        """
+        cmd = "import sys; print(sys.getfilesystemencoding().lower())"
+        result, py_cmd = run_python_until_end(
+            "-c", cmd,
+            __isolated=True,
+            **env_vars
+        )
+        if not result.rc == 0:
+            result.fail(py_cmd)
+        # All subprocess outputs in this test case should be pure ASCII
+        child_fsencoding = result.out.decode("ascii").rstrip()
+        child_stderr_lines = result.err.decode("ascii").rstrip().splitlines()
+        return child_fsencoding, child_stderr_lines
+
+
+    def test_C_utf8_locale(self):
+        # Ensure the C.UTF-8 locale is accepted entirely without complaint
+        base_var_dict = {
+            "LANG": "",
+            "LC_CTYPE": "",
+            "LC_ALL": "",
+        }
+        for env_var in ("LC_ALL", "LC_CTYPE", "LANG"):
+            with self.subTest(env_var=env_var):
+                var_dict = base_var_dict.copy()
+                var_dict[env_var] = "C.UTF-8"
+                fsencoding, stderr_lines = self._get_child_fsencoding(var_dict)
+                self.assertEqual(fsencoding, "utf-8")
+                self.assertFalse(stderr_lines)
+
+
+    def _check_c_locale_coercion(self, expected_fsencoding, coerce_c_locale):
+        """Check the handling of the C locale for various configurations
+
+        Parameters:
+            expected_fsencoding: the encoding the child is expected to report
+            allow_c_locale: setting to use for PYTHONALLOWCLOCALE
+              None: don't set the variable at all
+              str: the value set in the child's environment
+        """
+        expected_warning = []
+        if coerce_c_locale != "0":
+            # Check C locale is coerced with a warning on stderr
+            expected_warning.append(self.EXPECTED_COERCION_WARNING)
+        base_var_dict = {
+            "LANG": "",
+            "LC_CTYPE": "",
+            "LC_ALL": "",
+        }
+        for env_var in ("LC_ALL", "LC_CTYPE", "LANG"):
+            for locale_to_set in ("", "C", "POSIX", "invalid.ascii"):
+                with self.subTest(env_var=env_var,
+                                  nominal_locale=locale_to_set,
+                                  PYTHONCOERCECLOCALE=coerce_c_locale):
+                    var_dict = base_var_dict.copy()
+                    var_dict[env_var] = locale_to_set
+                    if coerce_c_locale is not None:
+                        var_dict["PYTHONCOERCECLOCALE"] = coerce_c_locale
+                    fsencoding, stderr_lines = self._get_child_fsencoding(var_dict)
+                    self.assertEqual(fsencoding, expected_fsencoding)
+                    self.assertEqual(stderr_lines, expected_warning)
+
+
+    def test_test_PYTHONCOERCECLOCALE_not_set(self):
+        # This should coerce to the C.UTF-8 locale by default
+        self._check_c_locale_coercion("utf-8", coerce_c_locale=None)
+
+    def test_PYTHONCOERCECLOCALE_not_zero(self):
+        # *Any* string other that "0" is considered "set" for our purposes
+        # and hence should result in the locale coercion being enabled
+        for setting in ("", "1", "true", "false"):
+            self._check_c_locale_coercion("utf-8", coerce_c_locale=setting)
+
+    def test_PYTHONCOERCECLOCALE_set_to_zero(self):
+        # The setting "0" should result in the locale coercion being disabled
+        self._check_c_locale_coercion("ascii", coerce_c_locale="0")
+
+
+# Details of the shared library warning emitted at runtime
+LIBRARY_C_LOCALE_WARNING = (
+    "Python runtime initialized with LC_CTYPE=C (a locale with default ASCII "
+    "encoding), which may cause Unicode compatibility problems. Using C.UTF-8, "
+    "C.utf8, or UTF-8 (if available) as alternative Unicode-compatible "
+    "locales is recommended.\n"
+)
+
+@unittest.skipUnless(sysconfig.get_config_var("PY_WARN_ON_C_LOCALE"),
+                     "C locale runtime warning disabled at build time")
+class EmbeddingTests(unittest.TestCase):
+    def setUp(self):
+        here = os.path.abspath(__file__)
+        basepath = os.path.dirname(os.path.dirname(os.path.dirname(here)))
+        self.test_exe = exe = os.path.join(basepath, "Programs", "_testembed")
+        if not os.path.exists(exe):
+            self.skipTest("%r doesn't exist" % exe)
+        # This is needed otherwise we get a fatal error:
+        # "Py_Initialize: Unable to get the locale encoding
+        # LookupError: no codec search functions registered: can't find encoding"
+        self.oldcwd = os.getcwd()
+        os.chdir(basepath)
+
+    def tearDown(self):
+        os.chdir(self.oldcwd)
+
+    def run_embedded_interpreter(self, *args):
+        """Runs a test in the embedded interpreter"""
+        cmd = [self.test_exe]
+        cmd.extend(args)
+        p = subprocess.Popen(cmd,
+                             stdout=subprocess.PIPE,
+                             stderr=subprocess.PIPE,
+                             universal_newlines=True)
+        (out, err) = p.communicate()
+        self.assertEqual(p.returncode, 0,
+                         "bad returncode %d, stderr is %r" %
+                         (p.returncode, err))
+        return out, err
+
+    def test_library_c_locale_warning(self):
+        # Checks forced configuration of embedded interpreter IO streams
+        out, err = self.run_embedded_interpreter("c_locale_warning")
+        if test.support.verbose > 1:
+            print()
+            print(out)
+            print(err)
+        self.assertEqual(out, "")
+        self.assertEqual(err, LIBRARY_C_LOCALE_WARNING)
+
+def test_main():
+    test.support.run_unittest(LocaleOverrideTests, EmbeddingTests)
+    test.support.reap_children()
+
+if __name__ == "__main__":
+    test_main()
diff --git a/Lib/test/test_sys.py b/Lib/test/test_sys.py
@@ -680,6 +680,7 @@ def c_locale_get_error_handler(self, isolated=False, encoding=None):
         # Force the POSIX locale
         env = os.environ.copy()
         env["LC_ALL"] = "C"
+        env["PYTHONCOERCECLOCALE"] = "0"
         code = '\n'.join((
             'import sys',
             'def dump(name):',