Update to latest version of PEP 538

- move all required logic inside the shared library - explicitly setting one of the coercion target locales now also automatically enables "surrogateescape" on sys.stdin and sys.stdout
ncoghlan · Jun 11, 2017 · Mar 5, 2017 · Mar 11, 2017 · Mar 11, 2017 · Mar 13, 2017
commit 188e7807b6d9e49377aacbb287c074e5cabf70c5
diff --git a/Doc/using/cmdline.rst b/Doc/using/cmdline.rst
@@ -734,10 +734,13 @@ conflict.
 
    If setting one of these locale categories succeeds, then the matching
    environment variables will be set (both ``LC_ALL`` and ``LANG`` for the
-   ``LC_ALL`` category, and ``LC_CTYPE`` for the ``LC_CTYPE`` category),
-   and (if :envvar:`PYTHONIOENCODING` is not explicitly set), the text encoding
-   and error handling for Python's standard streams in the current process
-   will be set to ``utf-8`` and ``surrogateescape`` respectively.
+   ``LC_ALL`` category, and ``LC_CTYPE`` for the ``LC_CTYPE`` category) in
+   the current process environment before the Python runtime is initialized.
+
+   Configuring one of these locales (either explicitly or via the above
+   implicit locale coercion) will automatically set the error handler for
+   :data:`sys.stdin` and :data:`sys.stdout` to ``surrogateescape``. This
+   behavior can be overridden using :envvar:`PYTHONIOENCODING` as usual.
 
    Availability: \*nix
 

diff --git a/Lib/test/test_c_locale_coercion.py b/Lib/test/test_c_locale_coercion.py
@@ -15,7 +15,7 @@
 )
 
 # In order to get the warning messages to match up as expected, the candidate
-# order here must much the target locale order in Programs/python.c
+# order here must much the target locale order in Python/pylifecycle.c
 _C_UTF8_LOCALES = (
     # Entries: (Target locale, target category, expected env var updates)
     ("C.UTF-8", "LC_ALL", "LC_ALL & LANG"),
@@ -63,9 +63,13 @@ def get_expected_details(cls, expected_fsencoding):
         return dict(cls(expected_fsencoding, *stream_info)._asdict())
 
     @staticmethod
-    def _replace_ascii_alias(data):
-        """ASCII may be reported as ANSI_X3.4-1968, so replace it in output"""
-        return data.replace(b"ANSI_X3.4-1968", b"ascii")
+    def _handle_output_variations(data):
+        """Adjust the output to handle platform specific idiosyncrasies
+
+        * Some platforms report ASCII as ANSI_X3.4-1968
+        * Some platforms report UTF-8 instead of utf-8
+        """
+        return data.replace(b"ANSI_X3.4-1968", b"ascii").lower()
 
     @classmethod
     def get_child_details(cls, env_vars):
@@ -87,7 +91,7 @@ def get_child_details(cls, env_vars):
         if not result.rc == 0:
             result.fail(py_cmd)
         # All subprocess outputs in this test case should be pure ASCII
-        adjusted_output = cls._replace_ascii_alias(result.out)
+        adjusted_output = cls._handle_output_variations(result.out)
         stdout_lines = adjusted_output.decode("ascii").rstrip().splitlines()
         child_encoding_details = dict(cls(*stdout_lines)._asdict())
         stderr_lines = result.err.decode("ascii").rstrip().splitlines()
@@ -99,15 +103,24 @@ def get_child_details(cls, env_vars):
                      "C locale coercion disabled at build time")
 class LocaleOverrideTests(unittest.TestCase):
 
+    available_targets = []
+
     @classmethod
     def setUpClass(cls):
+        first_target_locale = first_env_updates = None
+        available_targets = cls.available_targets
+        # Find the target locales available in the current system
         for target_locale, target_category, env_updates in _C_UTF8_LOCALES:
             if _set_locale_in_subprocess(target_locale, target_category):
-                break
-        else:
+                available_targets.append(target_locale)
+                if first_target_locale is None:
+                    first_target_locale = target_locale
+                    first_env_updates = env_updates
+        if not available_targets:
             raise unittest.SkipTest("No C-with-UTF-8 locale available")
+        # Expect coercion to use the first available locale
         cls.EXPECTED_COERCION_WARNING = CLI_COERCION_WARNING_FMT.format(
-            env_updates, target_locale
+            first_env_updates, first_target_locale
         )
 
     def _check_child_encoding_details(self,
@@ -141,11 +154,12 @@ def _check_c_locale_coercion(self, expected_fsencoding, coerce_c_locale):
         """
 
         # Check for expected warning on stderr if C locale is coerced
+        self.maxDiff = None
+
         expected_warning = []
         if coerce_c_locale != "0":
             expected_warning.append(self.EXPECTED_COERCION_WARNING)
 
-        self.maxDiff = None
         base_var_dict = {
             "LANG": "",
             "LC_CTYPE": "",
@@ -166,7 +180,7 @@ def _check_c_locale_coercion(self, expected_fsencoding, coerce_c_locale):
 
 
     def test_test_PYTHONCOERCECLOCALE_not_set(self):
-        # This should coerce to the C.UTF-8 locale by default
+        # This should coerce to the first available target locale by default
         self._check_c_locale_coercion("utf-8", coerce_c_locale=None)
 
     def test_PYTHONCOERCECLOCALE_not_zero(self):
@@ -179,6 +193,29 @@ def test_PYTHONCOERCECLOCALE_set_to_zero(self):
         # The setting "0" should result in the locale coercion being disabled
         self._check_c_locale_coercion("ascii", coerce_c_locale="0")
 
+    def test_external_target_locale_configuration(self):
+        # Explicitly setting a target locale should give the same behaviour as
+        # is seen when implicitly coercing to that target locale
+        self.maxDiff = None
+
+        expected_warning = []
+        expected_fsencoding = "utf-8"
+
+        base_var_dict = {
+            "LANG": "",
+            "LC_CTYPE": "",
+            "LC_ALL": "",
+        }
+        for env_var in base_var_dict:
+            for locale_to_set in self.available_targets:
+                with self.subTest(env_var=env_var,
+                                  configured_locale=locale_to_set):
+                    var_dict = base_var_dict.copy()
+                    var_dict[env_var] = locale_to_set
+                    self._check_child_encoding_details(var_dict,
+                                                       expected_fsencoding,
+                                                       expected_warning)
+
 
 # Details of the shared library warning emitted at runtime
 LIBRARY_C_LOCALE_WARNING = (

diff --git a/Programs/python.c b/Programs/python.c
@@ -15,7 +15,7 @@ wmain(int argc, wchar_t **argv)
 }
 #else
 
-/* Helpers to better handle the legacy C locale
+/* Access private pylifecycle helper API to better handle the legacy C locale
  *
  * The legacy C locale assumes ASCII as the default text encoding, which
  * causes problems not only for the CPython runtime, but also other
@@ -27,107 +27,8 @@ wmain(int argc, wchar_t **argv)
  * See the documentation of the PYTHONCOERCECLOCALE setting for more details.
  *
  */
-
-#ifdef PY_COERCE_C_LOCALE
-/* Access private pylifecycle API to check PYTHONCOERCECLOCALE */
-extern int _Py_CLocaleCoercionIsExpected(void);
-
-static const char *_C_LOCALE_COERCION_WARNING =
-    "Python detected LC_CTYPE=C: %.20s coerced to %.20s (set another locale "
-    "or PYTHONCOERCECLOCALE=0 to disable this locale coercion behavior).\n";
-
-typedef struct _CandidateLocale {
-    const char *locale_name;
-    int category;
-} _LocaleCoercionTarget;
-
-static _LocaleCoercionTarget _TARGET_LOCALES[] = {
-    { "C.UTF-8", LC_ALL },
-    { "C.utf8", LC_ALL },
-    { "UTF-8", LC_CTYPE },
-    { NULL, 0 }
-};
-
-void
-_coerce_default_locale_settings(const _LocaleCoercionTarget *target)
-{
-    const char *newloc = target->locale_name;
-    int category = target->category;
-
-    /* Reset locale back to currently configured defaults */
-    setlocale(LC_ALL, "");
-
-    /* Set the relevant locale environment variables */
-    if (category == LC_ALL) {
-        const char *env_vars_updated = "LC_ALL & LANG";
-        if (setenv("LC_ALL", newloc, 1)) {
-            fprintf(stderr,
-                    "Error setting LC_ALL, skipping C locale coercion\n");
-            return;
-        }
-        if (setenv("LANG", newloc, 1)) {
-            fprintf(stderr,
-                    "Error setting LANG during C locale coercion\n");
-            env_vars_updated = "LC_ALL";
-        }
-        fprintf(stderr, _C_LOCALE_COERCION_WARNING, env_vars_updated, newloc);
-    } else if (category == LC_CTYPE) {
-        if (setenv("LC_CTYPE", newloc, 1)) {
-            fprintf(stderr,
-                    "Error setting LC_CTYPE, skipping C locale coercion\n");
-            return;
-        }
-        fprintf(stderr, _C_LOCALE_COERCION_WARNING, "LC_CTYPE", newloc);
-    } else {
-        fprintf(stderr, "Locale coercion must target LC_ALL or LC_CTYPE\n");
-        return;
-    }
-
-    /* Set standard stream encoding if PYTHONIOENCODING is not set
-     *
-     * We avoid setting PYTHONIOENCODING, as that can confuse Python 2
-     * instances in subprocesses that inherit the environment (as Python
-     * 2 has no 'surrogateescape' error handler).
-     *
-     * If PEP 540 is also implemented, this check will be replaced with
-     * unconditionally setting PYTHONUTF8=1
-     */
-    const char *io_encoding = getenv("PYTHONIOENCODING");
-    if ((io_encoding == NULL) || (strnlen(io_encoding, 1) == 0)) {
-        Py_SetStandardStreamEncoding("utf-8", "surrogateescape");
-    }
-
-    /* Reconfigure with the overridden environment variables */
-    setlocale(LC_ALL, "");
-}
-
-void
-_handle_legacy_c_locale(void)
-{
-    /* We ignore the Python -E and -I flags here, as we need to sort out
-     * the locale settings *before* we try to do anything with the command
-     * line arguments. For cross-platform debugging purposes, we also need
-     * to give end users a way to force even scripts that are otherwise
-     * isolated from their environment to use the legacy ASCII-centric C
-     * locale.
-    */
-    if (_Py_CLocaleCoercionIsExpected()) {
-        /* PYTHONCOERCECLOCALE is not set, or is not set to exactly "0" */
-        const _LocaleCoercionTarget *target = NULL;
-        for (target = _TARGET_LOCALES; target->locale_name; target++) {
-            const char *reconfigured_locale = setlocale(target->category,
-                                                        target->locale_name);
-            if (reconfigured_locale != NULL) {
-                /* Successfully configured locale, so make it the default */
-                _coerce_default_locale_settings(target);
-                return;
-            }
-        }
-
-    }
-    /* No C locale warning here, as Py_Initialize will emit one later */
-}
-#endif
+extern int _Py_LegacyLocaleDetected(void);
+extern void _Py_CoerceLegacyLocale(void);
 
 int
 main(int argc, char **argv)
@@ -177,14 +78,9 @@ main(int argc, char **argv)
     setlocale(LC_ALL, "");
 #endif
 
-#ifdef PY_COERCE_C_LOCALE
-    /* When the LC_CTYPE category still claims to be using the C locale,
-       assume configuration error and try for a UTF-8 based locale instead */
-    const char *ctype_loc = setlocale(LC_CTYPE, NULL);
-    if (ctype_loc != NULL && strcmp(ctype_loc, "C") == 0) {
-        _handle_legacy_c_locale();
+    if (_Py_LegacyLocaleDetected()) {
+        _Py_CoerceLegacyLocale();
     }
-#endif
 
     /* Convert from char to wchar_t based on the locale settings */
     for (i = 0; i < argc; i++) {