Skip to content

Commit 5b22213

Browse files
loewisloewis
committed
Make identifiers str (not str8) objects throughout.
This affects the parser, various object implementations, and all places that put identifiers into C string literals. In testing, a number of crashes occurred as code would fail when the recursion limit was reached (such as the Unicode interning dictionary having key/value pairs where key is not value). To solve these, I added an overflowed flag, which allows for 50 more recursions after the limit was reached and the exception was raised, and a recursion_critical flag, which indicates that recursion absolutely must be allowed, i.e. that a certain call must not cause a stack overflow exception. There are still some places where both str and str8 are accepted as identifiers; these should eventually be removed.
1 parent 38e43c2 commit 5b22213

40 files changed

Lines changed: 462 additions & 289 deletions

‎Include/ceval.h‎

Lines changed: 12 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -50,7 +50,10 @@ PyAPI_FUNC(int) Py_GetRecursionLimit(void);
5050
(_Py_MakeRecCheck(PyThreadState_GET()->recursion_depth) && \
5151
_Py_CheckRecursiveCall(where))
5252
#define Py_LeaveRecursiveCall() \
53-
(--PyThreadState_GET()->recursion_depth)
53+
do{ if((--PyThreadState_GET()->recursion_depth) < \
54+
_Py_CheckRecursionLimit - 50); \
55+
PyThreadState_GET()->overflowed = 0; \
56+
} while(0)
5457
PyAPI_FUNC(int) _Py_CheckRecursiveCall(char *where);
5558
PyAPI_DATA(int) _Py_CheckRecursionLimit;
5659
#ifdef USE_STACKCHECK
@@ -59,6 +62,14 @@ PyAPI_DATA(int) _Py_CheckRecursionLimit;
5962
# define _Py_MakeRecCheck(x) (++(x) > _Py_CheckRecursionLimit)
6063
#endif
6164

65+
#define Py_ALLOW_RECURSION \
66+
do { unsigned char _old = PyThreadState_GET()->recursion_critical;\
67+
PyThreadState_GET()->recursion_critical = 1;
68+
69+
#define Py_END_ALLOW_RECURSION \
70+
PyThreadState_GET()->recursion_critical = _old; \
71+
} while(0);
72+
6273
PyAPI_FUNC(const char *) PyEval_GetFuncName(PyObject *);
6374
PyAPI_FUNC(const char *) PyEval_GetFuncDesc(PyObject *);
6475

‎Include/pystate.h‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -61,6 +61,10 @@ typedef struct _ts {
6161

6262
struct _frame *frame;
6363
int recursion_depth;
64+
char overflowed; /* The stack has overflowed. Allow 50 more calls
65+
to handle the runtime error. */
66+
char recursion_critical; /* The current calls must not cause
67+
a stack overflow. */
6468
/* 'tracing' keeps track of the execution depth when tracing/profiling.
6569
This is to prevent the actual trace/profile code from being recorded in
6670
the trace/profile. */

‎Include/stringobject.h‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -84,8 +84,8 @@ PyAPI_FUNC(void) _Py_ReleaseInternedStrings(void);
8484
#define PyString_CHECK_INTERNED(op) (((PyStringObject *)(op))->ob_sstate)
8585

8686
/* Macro, trading safety for speed */
87-
#define PyString_AS_STRING(op) (((PyStringObject *)(op))->ob_sval)
88-
#define PyString_GET_SIZE(op) (((PyStringObject *)(op))->ob_size)
87+
#define PyString_AS_STRING(op) (assert(PyString_Check(op)),(((PyStringObject *)(op))->ob_sval))
88+
#define PyString_GET_SIZE(op) (assert(PyString_Check(op)),(((PyStringObject *)(op))->ob_size))
8989

9090
/* _PyString_Join(sep, x) is like sep.join(x). sep must be PyStringObject*,
9191
x must be an iterable object. */

‎Include/unicodeobject.h‎

Lines changed: 32 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -410,13 +410,13 @@ PyAPI_DATA(PyTypeObject) PyUnicode_Type;
410410

411411
/* Fast access macros */
412412
#define PyUnicode_GET_SIZE(op) \
413-
(((PyUnicodeObject *)(op))->length)
413+
(assert(PyUnicode_Check(op)),(((PyUnicodeObject *)(op))->length))
414414
#define PyUnicode_GET_DATA_SIZE(op) \
415-
(((PyUnicodeObject *)(op))->length * sizeof(Py_UNICODE))
415+
(assert(PyUnicode_Check(op)),(((PyUnicodeObject *)(op))->length * sizeof(Py_UNICODE)))
416416
#define PyUnicode_AS_UNICODE(op) \
417-
(((PyUnicodeObject *)(op))->str)
417+
(assert(PyUnicode_Check(op)),(((PyUnicodeObject *)(op))->str))
418418
#define PyUnicode_AS_DATA(op) \
419-
((const char *)((PyUnicodeObject *)(op))->str)
419+
(assert(PyUnicode_Check(op)),((const char *)((PyUnicodeObject *)(op))->str))
420420

421421
/* --- Constants ---------------------------------------------------------- */
422422

@@ -627,6 +627,13 @@ PyAPI_FUNC(PyObject*) PyUnicode_FromOrdinal(int ordinal);
627627
PyAPI_FUNC(PyObject *) _PyUnicode_AsDefaultEncodedString(
628628
PyObject *, const char *);
629629

630+
/* Return a char* holding the default encoded value of the
631+
Unicode object.
632+
*/
633+
634+
PyAPI_FUNC(char *) PyUnicode_AsString(PyObject*);
635+
636+
630637
/* Returns the currently active default encoding.
631638
632639
The default encoding is currently implemented as run-time settable
@@ -1193,6 +1200,11 @@ PyAPI_FUNC(int) PyUnicode_Compare(
11931200
PyObject *right /* Right string */
11941201
);
11951202

1203+
PyAPI_FUNC(int) PyUnicode_CompareWithASCIIString(
1204+
PyObject *left,
1205+
const char *right
1206+
);
1207+
11961208
/* Rich compare two strings and return one of the following:
11971209
11981210
- NULL in case an exception was raised
@@ -1310,6 +1322,22 @@ PyAPI_FUNC(int) _PyUnicode_IsAlpha(
13101322
Py_UNICODE ch /* Unicode character */
13111323
);
13121324

1325+
PyAPI_FUNC(size_t) Py_UNICODE_strlen(const Py_UNICODE *u);
1326+
1327+
PyAPI_FUNC(Py_UNICODE*) Py_UNICODE_strcpy(
1328+
Py_UNICODE *s1, const Py_UNICODE *s2);
1329+
1330+
PyAPI_FUNC(Py_UNICODE*) Py_UNICODE_strncpy(
1331+
Py_UNICODE *s1, const Py_UNICODE *s2, size_t n);
1332+
1333+
PyAPI_FUNC(int) Py_UNICODE_strcmp(
1334+
const Py_UNICODE *s1, const Py_UNICODE *s2);
1335+
1336+
PyAPI_FUNC(Py_UNICODE*) Py_UNICODE_strchr(
1337+
const Py_UNICODE *s, Py_UNICODE c
1338+
);
1339+
1340+
13131341
#ifdef __cplusplus
13141342
}
13151343
#endif

‎Lib/test/test_frozen.py‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,8 @@
1010
from test.test_support import TestFailed
1111
import sys, os
1212

13+
raise TestFailed, "test currently causes assertion in debug mode"
14+
1315
try:
1416
import __hello__
1517
except ImportError as x:

‎Lib/test/test_new.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -143,7 +143,7 @@ def f(a): pass
143143
firstlineno, lnotab)
144144

145145
# new.code used to be a way to mutate a tuple...
146-
class S(str8):
146+
class S(str):
147147
pass
148148
t = (S("ab"),)
149149
d = new.code(argcount, kwonlyargcount, nlocals, stacksize,

‎Lib/test/test_sys.py‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -321,12 +321,6 @@ def __hash__(self):
321321

322322
self.assertRaises(TypeError, sys.intern, S("abc"))
323323

324-
# It's still safe to pass these strings to routines that
325-
# call intern internally, e.g. PyObject_SetAttr().
326-
s = S("abc")
327-
setattr(s, s, s)
328-
self.assertEqual(getattr(s, s), s)
329-
330324
s = "never interned as unicode before"
331325
self.assert_(sys.intern(s) is s)
332326
s2 = s.swapcase().swapcase()
@@ -338,6 +332,12 @@ def __hash__(self):
338332

339333
self.assertRaises(TypeError, sys.intern, U("abc"))
340334

335+
# It's still safe to pass these strings to routines that
336+
# call intern internally, e.g. PyObject_SetAttr().
337+
s = U("abc")
338+
setattr(s, s, s)
339+
self.assertEqual(getattr(s, s), s)
340+
341341

342342
def test_main():
343343
test.test_support.run_unittest(SysModuleTest)

‎Modules/_codecsmodule.c‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -172,7 +172,7 @@ escape_encode(PyObject *self,
172172
&PyString_Type, &str, &errors))
173173
return NULL;
174174

175-
size = PyUnicode_GET_SIZE(str);
175+
size = PyString_GET_SIZE(str);
176176
newsize = 4*size;
177177
if (newsize > PY_SSIZE_T_MAX || newsize / 4 != size) {
178178
PyErr_SetString(PyExc_OverflowError,

‎Modules/_hotshot.c‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -810,7 +810,7 @@ get_fileno(ProfilerObject *self, PyCodeObject *fcode)
810810
PyObject *name = PyDict_GetItem(dict, obj);
811811
if (name == NULL) {
812812
if (pack_define_func(self, fileno, fcode->co_firstlineno,
813-
PyString_AS_STRING(fcode->co_name)) < 0) {
813+
PyUnicode_AsString(fcode->co_name)) < 0) {
814814
Py_DECREF(obj);
815815
return -1;
816816
}

‎Modules/cPickle.c‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1829,8 +1829,8 @@ save_global(Picklerobject *self, PyObject *args, PyObject *name)
18291829
(name_size = PyString_Size(global_name)) < 0)
18301830
goto finally;
18311831

1832-
module_str = PyString_AS_STRING((PyStringObject *)module);
1833-
name_str = PyString_AS_STRING((PyStringObject *)global_name);
1832+
module_str = PyUnicode_AsString(module);
1833+
name_str = PyUnicode_AsString(global_name);
18341834

18351835
/* XXX This can be doing a relative import. Clearly it shouldn't,
18361836
but I don't know how to stop it. :-( */
@@ -1842,7 +1842,7 @@ save_global(Picklerobject *self, PyObject *args, PyObject *name)
18421842
"OS", args, module);
18431843
goto finally;
18441844
}
1845-
klass = PyObject_GetAttrString(mod, name_str);
1845+
klass = PyObject_GetAttr(mod, global_name);
18461846
if (klass == NULL) {
18471847
cPickle_ErrFormat(PicklingError,
18481848
"Can't pickle %s: attribute lookup %s.%s "
@@ -2223,7 +2223,7 @@ save(Picklerobject *self, PyObject *args, int pers_save)
22232223
res = save_string(self, args, 0);
22242224
goto finally;
22252225
}
2226-
if ((type == &PyUnicode_Type) && (PyString_GET_SIZE(args) < 2)) {
2226+
if ((type == &PyUnicode_Type) && (PyUnicode_GET_SIZE(args) < 2)) {
22272227
res = save_unicode(self, args, 0);
22282228
goto finally;
22292229
}
@@ -3584,7 +3584,7 @@ load_global(Unpicklerobject *self)
35843584
Py_DECREF(module_name);
35853585
return bad_readline();
35863586
}
3587-
if ((class_name = PyString_FromStringAndSize(s, len - 1))) {
3587+
if ((class_name = PyUnicode_FromStringAndSize(s, len - 1))) {
35883588
class = find_class(module_name, class_name,
35893589
self->find_class);
35903590
Py_DECREF(class_name);
@@ -5379,7 +5379,7 @@ init_stuff(PyObject *module_dict)
53795379
{
53805380
PyObject *copy_reg, *t, *r;
53815381

5382-
#define INIT_STR(S) if (!( S ## _str=PyString_InternFromString(#S))) return -1;
5382+
#define INIT_STR(S) if (!( S ## _str=PyUnicode_InternFromString(#S))) return -1;
53835383

53845384
if (PyType_Ready(&Unpicklertype) < 0)
53855385
return -1;

0 commit comments

Comments
 (0)