changeset:   69587:d3af2a2b621b
user:        Victor Stinner <victor.stinner@haypocalc.com>
date:        Wed Apr 27 00:24:21 2011 +0200
files:       Include/pystate.h Misc/NEWS Objects/unicodeobject.c Python/pystate.c Python/pythonrun.c
description:
Issue #10914: Initialize correctly the filesystem codec when creating a new
subinterpreter to fix a bootstrap issue with codecs implemented in Python, as
the ISO-8859-15 codec.

Add fscodec_initialized attribute to the PyInterpreterState structure.


diff -r c8338cfa3578 -r d3af2a2b621b Include/pystate.h
--- a/Include/pystate.h	Wed Apr 27 00:20:27 2011 +0200
+++ b/Include/pystate.h	Wed Apr 27 00:24:21 2011 +0200
@@ -31,6 +31,7 @@
     PyObject *codec_search_cache;
     PyObject *codec_error_registry;
     int codecs_initialized;
+    int fscodec_initialized;
 
 #ifdef HAVE_DLOPEN
     int dlopenflags;
diff -r c8338cfa3578 -r d3af2a2b621b Misc/NEWS
--- a/Misc/NEWS	Wed Apr 27 00:20:27 2011 +0200
+++ b/Misc/NEWS	Wed Apr 27 00:24:21 2011 +0200
@@ -10,6 +10,10 @@
 Core and Builtins
 -----------------
 
+- Issue #10914: Initialize correctly the filesystem codec when creating a new
+  subinterpreter to fix a bootstrap issue with codecs implemented in Python, as
+  the ISO-8859-15 codec.
+
 - Issue #11918: OS/2 and VMS are no more supported because of the lack of
   maintainer.
 
diff -r c8338cfa3578 -r d3af2a2b621b Objects/unicodeobject.c
--- a/Objects/unicodeobject.c	Wed Apr 27 00:20:27 2011 +0200
+++ b/Objects/unicodeobject.c	Wed Apr 27 00:24:21 2011 +0200
@@ -1653,7 +1653,17 @@
                                 PyUnicode_GET_SIZE(unicode),
                                 "surrogateescape");
 #else
-    if (Py_FileSystemDefaultEncoding) {
+    PyInterpreterState *interp = PyThreadState_GET()->interp;
+    /* Bootstrap check: if the filesystem codec is implemented in Python, we
+       cannot use it to encode and decode filenames before it is loaded. Load
+       the Python codec requires to encode at least its own filename. Use the C
+       version of the locale codec until the codec registry is initialized and
+       the Python codec is loaded.
+
+       Py_FileSystemDefaultEncoding is shared between all interpreters, we
+       cannot only rely on it: check also interp->fscodec_initialized for
+       subinterpreters. */
+    if (Py_FileSystemDefaultEncoding && interp->fscodec_initialized) {
         return PyUnicode_AsEncodedString(unicode,
                                          Py_FileSystemDefaultEncoding,
                                          "surrogateescape");
@@ -1843,12 +1853,17 @@
 #elif defined(__APPLE__)
     return PyUnicode_DecodeUTF8(s, size, "surrogateescape");
 #else
-    /* During the early bootstrapping process, Py_FileSystemDefaultEncoding
-       can be undefined. If it is case, decode using UTF-8. The following assumes
-       that Py_FileSystemDefaultEncoding is set to a built-in encoding during the
-       bootstrapping process where the codecs aren't ready yet.
-    */
-    if (Py_FileSystemDefaultEncoding) {
+    PyInterpreterState *interp = PyThreadState_GET()->interp;
+    /* Bootstrap check: if the filesystem codec is implemented in Python, we
+       cannot use it to encode and decode filenames before it is loaded. Load
+       the Python codec requires to encode at least its own filename. Use the C
+       version of the locale codec until the codec registry is initialized and
+       the Python codec is loaded.
+
+       Py_FileSystemDefaultEncoding is shared between all interpreters, we
+       cannot only rely on it: check also interp->fscodec_initialized for
+       subinterpreters. */
+    if (Py_FileSystemDefaultEncoding && interp->fscodec_initialized) {
         return PyUnicode_Decode(s, size,
                                 Py_FileSystemDefaultEncoding,
                                 "surrogateescape");
diff -r c8338cfa3578 -r d3af2a2b621b Python/pystate.c
--- a/Python/pystate.c	Wed Apr 27 00:20:27 2011 +0200
+++ b/Python/pystate.c	Wed Apr 27 00:24:21 2011 +0200
@@ -79,6 +79,7 @@
         interp->codec_search_cache = NULL;
         interp->codec_error_registry = NULL;
         interp->codecs_initialized = 0;
+        interp->fscodec_initialized = 0;
 #ifdef HAVE_DLOPEN
 #ifdef RTLD_NOW
         interp->dlopenflags = RTLD_NOW;
diff -r c8338cfa3578 -r d3af2a2b621b Python/pythonrun.c
--- a/Python/pythonrun.c	Wed Apr 27 00:20:27 2011 +0200
+++ b/Python/pythonrun.c	Wed Apr 27 00:24:21 2011 +0200
@@ -53,7 +53,7 @@
 
 /* Forward */
 static void initmain(void);
-static void initfsencoding(void);
+static int initfsencoding(PyInterpreterState *interp);
 static void initsite(void);
 static int initstdio(void);
 static void flush_io(void);
@@ -298,7 +298,8 @@
 
     _PyTime_Init();
 
-    initfsencoding();
+    if (initfsencoding(interp) < 0)
+        Py_FatalError("Py_Initialize: unable to load the file system codec");
 
     if (install_sigs)
         initsigs(); /* Signal handling stuff, including initintr() */
@@ -618,6 +619,10 @@
         Py_DECREF(pstderr);
 
         _PyImportHooks_Init();
+
+        if (initfsencoding(interp) < 0)
+            goto handle_error;
+
         if (initstdio() < 0)
             Py_FatalError(
             "Py_Initialize: can't initialize sys standard streams");
@@ -730,8 +735,8 @@
     }
 }
 
-static void
-initfsencoding(void)
+static int
+initfsencoding(PyInterpreterState *interp)
 {
     PyObject *codec;
 #if defined(HAVE_LANGINFO_H) && defined(CODESET)
@@ -748,7 +753,8 @@
 
         Py_FileSystemDefaultEncoding = codeset;
         Py_HasFileSystemDefaultEncoding = 0;
-        return;
+        interp->fscodec_initialized = 1;
+        return 0;
     }
 #endif
 
@@ -758,10 +764,11 @@
         /* Such error can only occurs in critical situations: no more
          * memory, import a module of the standard library failed,
          * etc. */
-        Py_FatalError("Py_Initialize: unable to load the file system codec");
-    } else {
-        Py_DECREF(codec);
+        return -1;
     }
+    Py_DECREF(codec);
+    interp->fscodec_initialized = 1;
+    return 0;
 }
 
 /* Import the site module (not into __main__ though) */