Skip to content

Commit 00bc0e0

Browse files
gvanrossumgvanrossum
committed
Patch python#1272, by Christian Heimes and Alexandre Vassalotti.
Changes to make __file__ a proper Unicode object, using the default filesystem encoding. This is a bit tricky because the default filesystem encoding isn't set by the time we import the first modules; at that point we fudge things a bit. This is okay since __file__ isn't really used much except for error reporting. Tested on OSX and Linux only so far.
1 parent cdadf24 commit 00bc0e0

17 files changed

Lines changed: 96 additions & 41 deletions

File tree

‎Include/code.h‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -21,8 +21,8 @@ typedef struct {
2121
PyObject *co_freevars; /* tuple of strings (free variable names) */
2222
PyObject *co_cellvars; /* tuple of strings (cell variable names) */
2323
/* The rest doesn't count for hash/cmp */
24-
PyObject *co_filename; /* string (where it was loaded from) */
25-
PyObject *co_name; /* string (name, for reference) */
24+
PyObject *co_filename; /* unicode (where it was loaded from) */
25+
PyObject *co_name; /* unicode (name, for reference) */
2626
int co_firstlineno; /* first source line number */
2727
PyObject *co_lnotab; /* string (encoding addr<->lineno mapping) */
2828
void *co_zombieframe; /* for optimization only (see frameobject.c) */

‎Include/unicodeobject.h‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -154,6 +154,7 @@ typedef PY_UNICODE_TYPE Py_UNICODE;
154154
# define PyUnicode_DecodeASCII PyUnicodeUCS2_DecodeASCII
155155
# define PyUnicode_DecodeCharmap PyUnicodeUCS2_DecodeCharmap
156156
# define PyUnicode_DecodeLatin1 PyUnicodeUCS2_DecodeLatin1
157+
# define PyUnicode_DecodeFSDefault PyUnicodeUCS2_DecodeFSDefault
157158
# define PyUnicode_DecodeRawUnicodeEscape PyUnicodeUCS2_DecodeRawUnicodeEscape
158159
# define PyUnicode_DecodeUTF32 PyUnicodeUCS2_DecodeUTF32
159160
# define PyUnicode_DecodeUTF32Stateful PyUnicodeUCS2_DecodeUTF32Stateful
@@ -245,6 +246,7 @@ typedef PY_UNICODE_TYPE Py_UNICODE;
245246
# define PyUnicode_DecodeASCII PyUnicodeUCS4_DecodeASCII
246247
# define PyUnicode_DecodeCharmap PyUnicodeUCS4_DecodeCharmap
247248
# define PyUnicode_DecodeLatin1 PyUnicodeUCS4_DecodeLatin1
249+
# define PyUnicode_DecodeFSDefault PyUnicodeUCS4_DecodeFSDefault
248250
# define PyUnicode_DecodeRawUnicodeEscape PyUnicodeUCS4_DecodeRawUnicodeEscape
249251
# define PyUnicode_DecodeUTF32 PyUnicodeUCS4_DecodeUTF32
250252
# define PyUnicode_DecodeUTF32Stateful PyUnicodeUCS4_DecodeUTF32Stateful
@@ -641,6 +643,20 @@ PyAPI_FUNC(PyObject*) PyUnicode_FromOrdinal(int ordinal);
641643
PyAPI_FUNC(PyObject *) _PyUnicode_AsDefaultEncodedString(
642644
PyObject *, const char *);
643645

646+
/* Decode a null-terminated string using Py_FileSystemDefaultEncoding.
647+
648+
If the encoding is supported by one of the built-in codecs (i.e., UTF-8,
649+
UTF-16, UTF-32, Latin-1 or MBCS), otherwise fallback to UTF-8 and replace
650+
invalid characters with '?'.
651+
652+
The function is intended to be used for paths and file names only
653+
during bootstrapping process where the codecs are not set up.
654+
*/
655+
656+
PyAPI_FUNC(PyObject*) PyUnicode_DecodeFSDefault(
657+
const char *s /* encoded string */
658+
);
659+
644660
/* Return a char* holding the UTF-8 encoded value of the
645661
Unicode object.
646662

‎Misc/ACKS‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -273,6 +273,7 @@ Larry Hastings
273273
Shane Hathaway
274274
Rycharde Hawkes
275275
Jochen Hayek
276+
Christian Heimes
276277
Thomas Heller
277278
Lance Finn Helsten
278279
Jonathan Hendry
@@ -667,6 +668,7 @@ Michael Urman
667668
Hector Urtubia
668669
Atul Varma
669670
Dmitry Vasiliev
671+
Alexandre Vassalotti
670672
Frank Vercruesse
671673
Mike Verdone
672674
Jaap Vermeulen

‎Modules/_ctypes/callbacks.c‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -34,9 +34,9 @@ void _AddTraceback(char *funcname, char *filename, int lineno)
3434
PyCodeObject *py_code = 0;
3535
PyFrameObject *py_frame = 0;
3636

37-
py_srcfile = PyString_FromString(filename);
37+
py_srcfile = PyUnicode_DecodeFSDefault(filename);
3838
if (!py_srcfile) goto bad;
39-
py_funcname = PyString_FromString(funcname);
39+
py_funcname = PyUnicode_FromString(funcname);
4040
if (!py_funcname) goto bad;
4141
py_globals = PyDict_New();
4242
if (!py_globals) goto bad;

‎Modules/posixmodule.c‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -5370,7 +5370,7 @@ posix_tempnam(PyObject *self, PyObject *args)
53705370
#endif
53715371
if (name == NULL)
53725372
return PyErr_NoMemory();
5373-
result = PyString_FromString(name);
5373+
result = PyUnicode_DecodeFSDefault(name);
53745374
free(name);
53755375
return result;
53765376
}
@@ -5428,7 +5428,7 @@ posix_tmpnam(PyObject *self, PyObject *noargs)
54285428
Py_XDECREF(err);
54295429
return NULL;
54305430
}
5431-
return PyString_FromString(buffer);
5431+
return PyUnicode_DecodeFSDefault(buffer);
54325432
}
54335433
#endif
54345434

‎Modules/pyexpat.c‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -232,13 +232,13 @@ getcode(enum HandlerTypes slot, char* func_name, int lineno)
232232
code = PyString_FromString("");
233233
if (code == NULL)
234234
goto failed;
235-
name = PyString_FromString(func_name);
235+
name = PyUnicode_FromString(func_name);
236236
if (name == NULL)
237237
goto failed;
238238
nulltuple = PyTuple_New(0);
239239
if (nulltuple == NULL)
240240
goto failed;
241-
filename = PyString_FromString(__FILE__);
241+
filename = PyUnicode_DecodeFSDefault(__FILE__);
242242
handler_info[slot].tb_code =
243243
PyCode_New(0, /* argcount */
244244
0, /* kwonlyargcount */

‎Objects/codeobject.c‎

Lines changed: 8 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -50,6 +50,7 @@ PyCode_New(int argcount, int kwonlyargcount,
5050
{
5151
PyCodeObject *co;
5252
Py_ssize_t i;
53+
5354
/* Check argument types */
5455
if (argcount < 0 || nlocals < 0 ||
5556
code == NULL ||
@@ -58,20 +59,16 @@ PyCode_New(int argcount, int kwonlyargcount,
5859
varnames == NULL || !PyTuple_Check(varnames) ||
5960
freevars == NULL || !PyTuple_Check(freevars) ||
6061
cellvars == NULL || !PyTuple_Check(cellvars) ||
61-
name == NULL || (!PyString_Check(name) && !PyUnicode_Check(name)) ||
62-
filename == NULL || !PyString_Check(filename) ||
62+
name == NULL || !PyUnicode_Check(name) ||
63+
filename == NULL || !PyUnicode_Check(filename) ||
6364
lnotab == NULL || !PyString_Check(lnotab) ||
6465
!PyObject_CheckReadBuffer(code)) {
6566
PyErr_BadInternalCall();
6667
return NULL;
6768
}
68-
if (PyString_Check(name)) {
69-
name = PyUnicode_FromString(PyString_AS_STRING(name));
70-
if (name == NULL)
71-
return NULL;
72-
} else {
73-
Py_INCREF(name);
74-
}
69+
Py_INCREF(name);
70+
Py_INCREF(filename);
71+
7572
intern_strings(names);
7673
intern_strings(varnames);
7774
intern_strings(freevars);
@@ -299,8 +296,8 @@ code_repr(PyCodeObject *co)
299296

300297
if (co->co_firstlineno != 0)
301298
lineno = co->co_firstlineno;
302-
if (co->co_filename && PyString_Check(co->co_filename))
303-
filename = PyString_AS_STRING(co->co_filename);
299+
if (co->co_filename && PyUnicode_Check(co->co_filename))
300+
filename = PyUnicode_AsString(co->co_filename);
304301
return PyUnicode_FromFormat(
305302
"<code object %.100U at %p, file \"%.300s\", line %d>",
306303
co->co_name, co, filename, lineno);

‎Objects/moduleobject.c‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -86,12 +86,12 @@ PyModule_GetFilename(PyObject *m)
8686
d = ((PyModuleObject *)m)->md_dict;
8787
if (d == NULL ||
8888
(fileobj = PyDict_GetItemString(d, "__file__")) == NULL ||
89-
!PyString_Check(fileobj))
89+
!PyUnicode_Check(fileobj))
9090
{
9191
PyErr_SetString(PyExc_SystemError, "module filename missing");
9292
return NULL;
9393
}
94-
return PyString_AsString(fileobj);
94+
return PyUnicode_AsString(fileobj);
9595
}
9696

9797
void

‎Objects/unicodeobject.c‎

Lines changed: 34 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -117,7 +117,11 @@ static PyUnicodeObject *unicode_latin1[256];
117117

118118
/* Default encoding to use and assume when NULL is passed as encoding
119119
parameter; it is fixed to "utf-8". Always use the
120-
PyUnicode_GetDefaultEncoding() API to access this global. */
120+
PyUnicode_GetDefaultEncoding() API to access this global.
121+
122+
Don't forget to alter Py_FileSystemDefaultEncoding() if you change the
123+
hard coded default!
124+
*/
121125
static const char unicode_default_encoding[] = "utf-8";
122126

123127
Py_UNICODE
@@ -1231,6 +1235,35 @@ PyObject *_PyUnicode_AsDefaultEncodedString(PyObject *unicode,
12311235
return v;
12321236
}
12331237

1238+
PyObject*
1239+
PyUnicode_DecodeFSDefault(const char *s)
1240+
{
1241+
Py_ssize_t size = (Py_ssize_t)strlen(s);
1242+
1243+
/* During the early bootstrapping process, Py_FileSystemDefaultEncoding
1244+
can be undefined. If it is case, decode using UTF-8. The following assumes
1245+
that Py_FileSystemDefaultEncoding is set to a built-in encoding during the
1246+
bootstrapping process where the codecs aren't ready yet.
1247+
*/
1248+
if (Py_FileSystemDefaultEncoding) {
1249+
#if defined(MS_WINDOWS) && defined(HAVE_USABLE_WCHAR_T)
1250+
if (strcmp(Py_FileSystemDefaultEncoding, "mbcs")) {
1251+
return PyUnicode_DecodeMBCS(s, size, "replace");
1252+
}
1253+
#elif defined(__APPLE__)
1254+
if (strcmp(Py_FileSystemDefaultEncoding, "utf-8")) {
1255+
return PyUnicode_DecodeUTF8(s, size, "replace");
1256+
}
1257+
#endif
1258+
return PyUnicode_Decode(s, size,
1259+
Py_FileSystemDefaultEncoding,
1260+
"replace");
1261+
}
1262+
else {
1263+
return PyUnicode_DecodeUTF8(s, size, "replace");
1264+
}
1265+
}
1266+
12341267
char*
12351268
PyUnicode_AsStringAndSize(PyObject *unicode, Py_ssize_t *psize)
12361269
{

‎Python/bltinmodule.c‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,9 @@
1010

1111
/* The default encoding used by the platform file system APIs
1212
Can remain NULL for all platforms that don't have such a concept
13+
14+
Don't forget to modify PyUnicode_DecodeFSDefault() if you touch any of the
15+
values for Py_FileSystemDefaultEncoding!
1316
*/
1417
#if defined(MS_WINDOWS) && defined(HAVE_USABLE_WCHAR_T)
1518
const char *Py_FileSystemDefaultEncoding = "mbcs";

0 commit comments

Comments
 (0)