Skip to content
Merged
Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
43 commits
Select commit Hold shift + click to select a range
b5d125b
WIP: PEP 538 reference implementation
ncoghlan Mar 5, 2017
78c17a7
Fix test case failures
ncoghlan Mar 11, 2017
721b27f
Merge remote-tracking branch 'origin/master' into pep538-coerce-c-locale
ncoghlan Mar 11, 2017
16a4415
Merge branch 'master' into pep538-coerce-c-locale
ncoghlan Mar 13, 2017
d283de1
Clarify locale coercion warnings
ncoghlan Mar 13, 2017
f7a03fe
Avoid -Wformat-security warning
ncoghlan Mar 13, 2017
fe92a29
Support running tests under 'LANG=C'
ncoghlan Mar 13, 2017
64d9d2f
Suppress locale warning for PYTHONCOERCECLOCALE=0
ncoghlan Mar 13, 2017
384a146
Add test case for library runtime warning
ncoghlan Mar 13, 2017
b4f3a34
Add C locale coercion and warning build flags
ncoghlan Mar 13, 2017
4d684a6
Always use C.UTF-8 on Android
ncoghlan Mar 13, 2017
1c3a270
Fix PYTHONCOERCECLOCALE docs
ncoghlan Mar 15, 2017
7626fcf
Use Py_SetStandardStreamEncoding instead of PYTHONIOENCODING
ncoghlan Mar 15, 2017
d12b412
Some test cleanups suggested by Barry
ncoghlan Mar 15, 2017
ec4f2ea
Use more precise name for test file
ncoghlan Mar 15, 2017
4e6d502
Check standard stream settings in locale coercion tests
ncoghlan Mar 15, 2017
ccfc83f
Use US spelling
ncoghlan Mar 15, 2017
b173af3
Helper function to query PYTHONCOERCECLOCALE
ncoghlan Mar 15, 2017
501a829
Merge remote-tracking branch 'origin/master' into pep538-coerce-c-locale
ncoghlan Mar 15, 2017
d099a52
Fix ReST markup
ncoghlan Mar 15, 2017
762a09b
Fix Py_DEBUG/Py_SetStandardStreamEncoding compatibility problem
ncoghlan Mar 15, 2017
820bfad
Restore Windows _testembed compatibility
ncoghlan Mar 17, 2017
6a00ce6
Merge remote-tracking branch 'origin/master' into pep538-coerce-c-locale
ncoghlan May 6, 2017
188e780
Update to latest version of PEP 538
ncoghlan May 6, 2017
476a781
Change locale coercion to always respect LC_ALL
ncoghlan May 9, 2017
123ba24
Merge remote-tracking branch 'origin/master' into pep538-coerce-c-locale
ncoghlan May 27, 2017
939ba0a
Don't set LANG during locale coercion
ncoghlan May 27, 2017
6d564c9
Update docs to match current behaviour
ncoghlan Jun 3, 2017
53bd6da
Address CI failure and review comments
ncoghlan Jun 3, 2017
cad0669
OK, two-use function :)
ncoghlan Jun 3, 2017
421516f
Still check for the C locale in Windows
ncoghlan Jun 3, 2017
e48a378
Check actual control flow on Appveyor
ncoghlan Jun 3, 2017
f62dbd8
Merge remote-tracking branch 'origin/master' into pep538-coerce-c-locale
ncoghlan Jun 3, 2017
d181b92
Use correct reference type in docs
ncoghlan Jun 3, 2017
8cf0590
More Appveyor debugging
ncoghlan Jun 3, 2017
cea7970
New theory regarding the Windows problem
ncoghlan Jun 3, 2017
c63d5fa
Locale coercion may inject LC_CTYPE into environment
ncoghlan Jun 3, 2017
8e0e1ca
Ensure SYSTEMROOT is set in Windows embedding tests
ncoghlan Jun 3, 2017
7379398
Don't use the default pipe encoding in test_capi
ncoghlan Jun 3, 2017
89759b5
stdin encoding ends up normalised on Windows
ncoghlan Jun 3, 2017
5a56a3f
PEP 538: Add What's New entry
ncoghlan Jun 4, 2017
0036bea
Merge remote-tracking branch 'origin/master' into pep538-coerce-c-locale
ncoghlan Jun 11, 2017
5288662
Add NEWS entry
ncoghlan Jun 11, 2017
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
Don't set LANG during locale coercion
It turns out setting onlyLC_CTYPE is sufficient, which
improves cross-platform consistency and reduces the chance
of unintended side effects.
  • Loading branch information
ncoghlan committed May 27, 2017
commit 939ba0a77d4b52a04315c129f9db89b90c0532cd
19 changes: 6 additions & 13 deletions Lib/test/test_c_locale_coercion.py
Original file line number Diff line number Diff line change
Expand Up @@ -16,12 +16,7 @@

# In order to get the warning messages to match up as expected, the candidate
# order here must much the target locale order in Python/pylifecycle.c
_C_UTF8_LOCALES = (
# Entries: (Target locale, expected env var updates)
("C.UTF-8", "LC_CTYPE & LANG"),
("C.utf8", "LC_CTYPE & LANG"),
("UTF-8", "LC_CTYPE"),
)
_C_UTF8_LOCALES = ("C.UTF-8", "C.utf8", "UTF-8")

# There's no reliable cross-platform way of checking locale alias
# lists, so the only way of knowing which of these locales will work
Expand Down Expand Up @@ -144,7 +139,7 @@ def test_library_c_locale_warning(self):

# Details of the CLI locale coercion warning emitted at runtime
CLI_COERCION_WARNING_FMT = (
"Python detected LC_CTYPE=C: {} coerced to {} (set another locale "
"Python detected LC_CTYPE=C: LC_CTYPE coerced to {} (set another locale "
"or PYTHONCOERCECLOCALE=0 to disable this locale coercion behavior)."
)

Expand All @@ -156,21 +151,19 @@ class _LocaleCoercionTargetsTestCase(_ChildProcessEncodingTestCase):

@classmethod
def setUpClass(cls):
first_target_locale = first_env_updates = None
first_target_locale = None
available_targets = cls.available_targets
# Find the target locales available in the current system
for target_locale, env_updates in _C_UTF8_LOCALES:
for target_locale in _C_UTF8_LOCALES:
if _set_locale_in_subprocess(target_locale):
available_targets.append(target_locale)
if first_target_locale is None:
first_target_locale = target_locale
first_env_updates = env_updates
if cls.targets_required and not available_targets:
raise unittest.SkipTest("No C-with-UTF-8 locale available")
# Expect coercion to use the first available locale
cls.EXPECTED_COERCION_WARNING = CLI_COERCION_WARNING_FMT.format(
first_env_updates, first_target_locale
)
warning_msg = CLI_COERCION_WARNING_FMT.format(first_target_locale)
cls.EXPECTED_COERCION_WARNING = warning_msg


class LocaleConfigurationTests(_LocaleCoercionTargetsTestCase):
Expand Down
24 changes: 7 additions & 17 deletions Python/pylifecycle.c
Original file line number Diff line number Diff line change
Expand Up @@ -357,14 +357,13 @@ _Py_LegacyLocaleDetected(void)

typedef struct _CandidateLocale {
const char *locale_name; /* The locale to try as a coercion target */
int set_LANG; /* Whether to set LANG in addition to LC_CTYPE */
} _LocaleCoercionTarget;

static _LocaleCoercionTarget _TARGET_LOCALES[] = {
{ "C.UTF-8", 1 },
{ "C.utf8", 1},
{ "UTF-8", 0 },
{ NULL, 0 }
{"C.UTF-8"},
{"C.utf8"},
{"UTF-8"},
{NULL}
};

static char *
Expand Down Expand Up @@ -392,33 +391,24 @@ get_default_standard_stream_error_handler(void)

#ifdef PY_COERCE_C_LOCALE
static const char *_C_LOCALE_COERCION_WARNING =
"Python detected LC_CTYPE=C: %.20s coerced to %.20s (set another locale "
"Python detected LC_CTYPE=C: LC_CTYPE coerced to %.20s (set another locale "
"or PYTHONCOERCECLOCALE=0 to disable this locale coercion behavior).\n";

static void
_coerce_default_locale_settings(const _LocaleCoercionTarget *target)
{
const char *env_vars_updated = "LC_CTYPE";
const char *newloc = target->locale_name;

/* Reset locale back to currently configured defaults */
setlocale(LC_ALL, "");

/* Set the relevant locale environment variables */
/* Set the relevant locale environment variable */
if (setenv("LC_CTYPE", newloc, 1)) {
fprintf(stderr,
"Error setting LC_CTYPE, skipping C locale coercion\n");
return;
}
if (target->set_LANG) {
if (setenv("LANG", newloc, 1) == 0) {
env_vars_updated = "LC_CTYPE & LANG";
} else {
fprintf(stderr,
"Error setting LANG during C locale coercion\n");
}
}
fprintf(stderr, _C_LOCALE_COERCION_WARNING, env_vars_updated, newloc);
fprintf(stderr, _C_LOCALE_COERCION_WARNING, newloc);

/* Reconfigure with the overridden environment variables */
setlocale(LC_ALL, "");
Expand Down