Skip to content

Commit 1c04e79

Browse files
akruisakruis
authored
Stackless issue python#268: Add magic number to pickled code objects.
Pickled code objects now contain importlib.util.MAGIC_NUMBER. Unpickling code objects pickled with an incompatible version of Python now creates a RuntimeWarning. The resulting code object starts with an invalid opcode and renders unpickled frames invalid. Unpickling a function whose code is invalid causes a RuntimeWarning too.
1 parent 3eec437 commit 1c04e79

4 files changed

Lines changed: 262 additions & 4 deletions

File tree

‎Doc/library/stackless/pickling.rst‎

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -147,6 +147,19 @@ types:
147147
C-types PyAsyncGenASend and PyAsyncGenAThrow (see :pep:`525`) as well as
148148
all kinds of :ref:`Dictionary view objects <dict-views>`.
149149

150+
Code
151+
====
152+
153+
|SLP| can pickle :data:`~types.CodeType` objects.
154+
155+
.. versionchanged:: 3.8
156+
The pickled representation of a code object contains the bytecode version number (:data:`~importlib.util.MAGIC_NUMBER`).
157+
If a program tries to unpickle a code object with a wrong bytecode version number, then |SLP|
158+
159+
* emits a ``RuntimeWarning('Unpickling code object with invalid magic number %ld')`` and
160+
* prepends the *co_code* attribute of the unpickled code object with an invalid |PY| bytecode instruction. This way any attempt
161+
to execute the code object raises :exc:`SystemError`.
162+
150163
Frames
151164
======
152165

@@ -156,6 +169,25 @@ generator. |SLP| does not register a "reduction" function for
156169
:data:`~types.FrameType`. This way |SLP| stays compatible with application
157170
code that registers its own "reduction" function for :data:`~types.FrameType`.
158171

172+
It is not possible to execute an unpickled frame, if the tasklet the original frame belonged to was
173+
not :attr:`~tasklet.restorable`. In this case the frame is marked as invalid and any attempt
174+
to execute it raises
175+
176+
.. versionchanged:: 3.8
177+
If a program tries to unpickle a frame using a code object whose first bytecode instruction is invalid, then |SLP|
178+
marks the frame as invalid. Any attempt to execute the frame raises :exc:`RuntimeError`.
179+
180+
181+
Functions
182+
=========
183+
184+
|SLP| can pickle functions including lambda-objects objects by value.
185+
186+
.. versionchanged:: 3.8
187+
If a program tries to unpickle a function using a code object whose first bytecode instruction is invalid, then |SLP|
188+
emits a ``RuntimeWarning('Unpickling function with invalid code object: %V')``. Any attempt
189+
to execute the function raises :exc:`SystemError`.
190+
159191
.. _slp_pickling_asyncgen:
160192

161193
Asynchronous Generators

‎Stackless/changelog.txt‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,14 @@ What's New in Stackless 3.X.X?
99

1010
*Release date: 20XX-XX-XX*
1111

12+
- https://github.com/stackless-dev/stackless/issues/268
13+
Pickled code objects now contain importlib.util.MAGIC_NUMBER. Unpickling
14+
code objects pickled with an incompatible version of Python now creates a
15+
RuntimeWarning("Unpickling code object with invalid magic number"). The
16+
resulting code object starts with an invalid opcode and renders unpickled
17+
frames invalid. Unpickling a function whose code is invalid causes a
18+
RuntimeWarning("Unpickling function with invalid code object").
19+
1220
- https://github.com/stackless-dev/stackless/issues/270
1321
Stackless now uses an unmodified PyFrameObject structure. Stackless now
1422
stores more state information in the field f->f_executing than C-Python.

‎Stackless/pickling/prickelpit.c‎

Lines changed: 116 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -381,8 +381,7 @@ slp_cannot_execute(PyCFrameObject *f, const char *exec_name, PyObject *retval)
381381
if (retval != NULL) {
382382
Py_DECREF(retval);
383383
PyErr_Format(PyExc_RuntimeError, "cannot execute invalid frame with "
384-
"'%.100s': frame had a C state that"
385-
" can't be restored.",
384+
"'%.100s': frame had a C state that can't be restored or an invalid code object.",
386385
exec_name);
387386
}
388387

@@ -610,16 +609,36 @@ slp_from_tuple_with_nulls(PyObject **start, PyObject *tup)
610609
611610
******************************************************/
612611

613-
#define codetuplefmt "iiiiiSOOOSSiSOO"
612+
#define codetuplefmt "liiiiiSOOOSSiSOO"
613+
/* Index of co_code in the tuple given to code_new */
614+
static const Py_ssize_t code_co_code_index = 5;
615+
616+
/*
617+
* An unused (invalid) opcode. See opcode.h for a list of used opcodes.
618+
* If Stackless unpickles a code object with an invalid magic number, it prefixes
619+
* co_code with this opcode.
620+
*
621+
* frame_setstate tests if the first opcode of the code of the frame is CODE_INVALID_OPCODE
622+
* and eventually marks a frame as invalid.
623+
*/
624+
static const char CODE_INVALID_OPCODE = 0;
614625

615626
static struct _typeobject wrap_PyCode_Type;
627+
static long bytecode_magic = 0;
616628

617629
static PyObject *
618630
code_reduce(PyCodeObject * co, PyObject *unused)
619631
{
632+
if (0 >= bytecode_magic) {
633+
bytecode_magic = PyImport_GetMagicNumber();
634+
if (-1 == bytecode_magic)
635+
return NULL;
636+
}
637+
620638
PyObject *tup = Py_BuildValue(
621639
"(O(" codetuplefmt ")())",
622640
&wrap_PyCode_Type,
641+
bytecode_magic,
623642
co->co_argcount,
624643
co->co_kwonlyargcount,
625644
co->co_nlocals,
@@ -640,7 +659,73 @@ code_reduce(PyCodeObject * co, PyObject *unused)
640659
return tup;
641660
}
642661

643-
MAKE_WRAPPERTYPE(PyCode_Type, code, "code", code_reduce, generic_new,
662+
static PyObject *
663+
code_new(PyTypeObject *type, PyObject *args, PyObject *kwds)
664+
{
665+
long magic = 0;
666+
667+
if (0 >= bytecode_magic) {
668+
bytecode_magic = PyImport_GetMagicNumber();
669+
if (-1 == bytecode_magic)
670+
return NULL;
671+
}
672+
673+
assert(PyTuple_CheckExact(args));
674+
if (PyTuple_GET_SIZE(args) == sizeof(codetuplefmt) - 1) {
675+
/* */
676+
magic = PyLong_AsLong(PyTuple_GET_ITEM(args, 0));
677+
if (-1 == magic && PyErr_Occurred()) {
678+
return NULL;
679+
}
680+
args = PyTuple_GetSlice(args, 1, sizeof(codetuplefmt) - 1);
681+
} else if (PyTuple_GET_SIZE(args) == sizeof(codetuplefmt) - 2) {
682+
/* Format used by Stackless versions up to 3.7 */
683+
args = PyTuple_GetSlice(args, 0, sizeof(codetuplefmt) - 2);
684+
} else {
685+
PyErr_SetString(PyExc_IndexError, "Argument tuple has wrong size.");
686+
return NULL;
687+
}
688+
if (NULL == args)
689+
return NULL;
690+
691+
if (bytecode_magic != magic) {
692+
if (PyErr_WarnFormat(PyExc_RuntimeWarning, 1, "Unpickling code object with invalid magic number %ld", magic)) {
693+
Py_DECREF(args);
694+
return NULL;
695+
}
696+
697+
PyObject *code = PyTuple_GET_ITEM(args, code_co_code_index);
698+
if (NULL == code) {
699+
Py_DECREF(args);
700+
return NULL;
701+
}
702+
if (!PyBytes_Check(code)) {
703+
Py_DECREF(args);
704+
PyErr_SetString(PyExc_TypeError,
705+
"Unpickling code object: code is not a bytes object");
706+
return NULL;
707+
}
708+
Py_ssize_t code_len = PyBytes_Size(code);
709+
assert(code_len <= INT_MAX);
710+
assert(code_len % sizeof(_Py_CODEUNIT) == 0);
711+
712+
/* Now prepend an invalid opcode to the code.
713+
*/
714+
PyObject *code2 = PyBytes_FromStringAndSize(NULL, code_len + sizeof(_Py_CODEUNIT));
715+
assert(_Py_IS_ALIGNED(PyBytes_AS_STRING(code), sizeof(_Py_CODEUNIT)));
716+
char *p = PyBytes_AS_STRING(code2);
717+
p[0] = Py_BUILD_ASSERT_EXPR(sizeof(_Py_CODEUNIT) == 2) + CODE_INVALID_OPCODE;
718+
p[1] = 0; /* Argument */
719+
memcpy(p + sizeof(_Py_CODEUNIT), PyBytes_AS_STRING(code), code_len);
720+
PyTuple_SET_ITEM(args, code_co_code_index, code2);
721+
}
722+
723+
PyObject *retval = generic_new(type, args, kwds);
724+
Py_DECREF(args);
725+
return retval;
726+
}
727+
728+
MAKE_WRAPPERTYPE(PyCode_Type, code, "code", code_reduce, code_new,
644729
generic_setstate)
645730

646731
static int init_codetype(PyObject * mod)
@@ -779,12 +864,33 @@ func_setstate(PyObject *self, PyObject *args)
779864
{
780865
PyFunctionObject *fu;
781866
PyObject *args2;
867+
char *pcode;
782868

783869
if (is_wrong_type(Py_TYPE(self))) return NULL;
784870
Py_TYPE(self) = Py_TYPE(self)->tp_base;
871+
872+
/* Test for an invalid code object */
873+
args2 = PyTuple_GetItem(args, 0);
874+
if (NULL==args2)
875+
return NULL;
876+
if (! PyCode_Check(args2)) {
877+
PyErr_SetString(PyExc_TypeError, "func_setstate: value for func_code is not a code object");
878+
return NULL;
879+
}
880+
pcode = PyBytes_AsString(((PyCodeObject *) args2)->co_code);
881+
if (NULL == pcode)
882+
return NULL;
883+
if (*pcode == CODE_INVALID_OPCODE) {
884+
/* invalid code object, was pickled with a different version of python */
885+
if (PyErr_WarnFormat(PyExc_RuntimeWarning, 1, "Unpickling function with invalid code object: %V",
886+
PyTuple_GetItem(args, 2), "~ name is missing ~"))
887+
return NULL;
888+
}
889+
785890
args2 = PyTuple_GetSlice(args, 0, 5);
786891
if (args2 == NULL)
787892
return NULL;
893+
788894
fu = (PyFunctionObject *)
789895
Py_TYPE(self)->tp_new(Py_TYPE(self), args2, NULL);
790896
Py_DECREF(args2);
@@ -958,6 +1064,7 @@ frame_setstate(PyFrameObject *f, PyObject *args)
9581064
int valid, have_locals;
9591065
char f_executing;
9601066
Py_ssize_t tmp;
1067+
char *pcode;
9611068

9621069
if (is_wrong_type(Py_TYPE(f))) return NULL;
9631070

@@ -982,6 +1089,11 @@ frame_setstate(PyFrameObject *f, PyObject *args)
9821089
"invalid code object for frame_setstate");
9831090
return NULL;
9841091
}
1092+
pcode = PyBytes_AsString(((PyCodeObject *) f_code)->co_code);
1093+
if (NULL == pcode)
1094+
return NULL;
1095+
if (*pcode == CODE_INVALID_OPCODE)
1096+
valid = 0; /* invalid code object, was pickled with a different version of python */
9851097

9861098
if (have_locals) {
9871099
Py_INCREF(f_locals);

‎Stackless/unittests/test_pickle.py‎

Lines changed: 106 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,8 +8,13 @@
88
import threading
99
import contextvars
1010
import ctypes
11+
import importlib.util
12+
import struct
13+
import warnings
14+
import subprocess
1115
import stackless
1216

17+
from textwrap import dedent
1318
from stackless import schedule, tasklet
1419

1520
from support import test_main # @UnusedImport
@@ -20,6 +25,7 @@
2025
# we need to make it appear that pickling them is ok, otherwise we will fail when pickling
2126
# closures that refer to test runner instances
2227
import copyreg
28+
from _warnings import warn
2329

2430

2531
def reduce(obj):
@@ -1346,6 +1352,106 @@ def run():
13461352
self.assertEqual(stackless.pickle_flags(), current)
13471353

13481354

1355+
class TestCodePickling(unittest.TestCase):
1356+
def test_reduce_magic(self):
1357+
code = (lambda :None).__code__
1358+
reduce = stackless._stackless._wrap.code.__reduce__
1359+
reduced = reduce(code)
1360+
self.assertIsInstance(reduced, tuple)
1361+
self.assertEqual(len(reduced), 3)
1362+
self.assertIsInstance(reduced[1], tuple)
1363+
self.assertGreater(len(reduced[1]), 0)
1364+
self.assertIsInstance(reduced[1][0], int)
1365+
# see Python C-API documentation for PyImport_GetMagicNumber()
1366+
self.assertEqual(reduced[1][0], struct.unpack("<l", importlib.util.MAGIC_NUMBER)[0])
1367+
1368+
def test_new_with_wrong_magic_error(self):
1369+
code = (lambda :None).__code__
1370+
reduce = stackless._stackless._wrap.code.__reduce__
1371+
reduced = reduce(code)
1372+
args = (reduced[1][0] + 1,) + reduced[1][1:]
1373+
self.assertIsInstance(reduced[0](*reduced[1]), type(code))
1374+
with self.assertRaisesRegex(RuntimeWarning, "Unpickling code object with invalid magic number"):
1375+
with stackless.atomic():
1376+
with warnings.catch_warnings():
1377+
warnings.simplefilter("error", RuntimeWarning)
1378+
reduced[0](*args)
1379+
1380+
def test_new_without_magic(self):
1381+
code = (lambda :None).__code__
1382+
reduce = stackless._stackless._wrap.code.__reduce__
1383+
reduced = reduce(code)
1384+
args = reduced[1][1:]
1385+
with self.assertRaisesRegex(RuntimeWarning, "Unpickling code object with invalid magic number 0"):
1386+
with stackless.atomic():
1387+
with warnings.catch_warnings():
1388+
warnings.simplefilter("error", RuntimeWarning)
1389+
reduced[0](*args)
1390+
1391+
def test_func_with_wrong_magic(self):
1392+
l = (lambda :None)
1393+
code = l.__code__
1394+
reduce = stackless._stackless._wrap.code.__reduce__
1395+
reduced = reduce(code)
1396+
args = (reduced[1][0] + 1,) + reduced[1][1:]
1397+
self.assertIsInstance(reduced[0](*reduced[1]), type(code))
1398+
with stackless.atomic():
1399+
with warnings.catch_warnings():
1400+
warnings.simplefilter("ignore", RuntimeWarning)
1401+
code2 = reduced[0](*args)
1402+
code2.__setstate__(())
1403+
self.assertIs(type(code2), type(code))
1404+
1405+
reduce = stackless._stackless._wrap.function.__reduce__
1406+
reduced_func = reduce(l)
1407+
f = reduced_func[0](*reduced_func[1])
1408+
args = (code2,) + reduced_func[2][1:]
1409+
with self.assertRaisesRegex(RuntimeWarning, "Unpickling function with invalid code object:"):
1410+
with stackless.atomic():
1411+
with warnings.catch_warnings():
1412+
warnings.simplefilter("error", RuntimeWarning)
1413+
f = f.__setstate__(args)
1414+
1415+
def test_run_with_wrong_magic(self):
1416+
# run this test as a subprocess, because it triggers a fprintf(stderr, ...) in ceval.c
1417+
# and I don't like this output in our test suite.
1418+
args = []
1419+
if not stackless.enable_softswitch(None):
1420+
args.append("--hard")
1421+
1422+
rc = subprocess.call([sys.executable, "-s", "-S", "-E", "-c", dedent("""
1423+
import stackless
1424+
import warnings
1425+
import sys
1426+
1427+
sys.stderr = sys.stdout
1428+
if "--hard" in sys.argv:
1429+
stackless.enable_softswitch(False)
1430+
l = (lambda :None)
1431+
code = l.__code__
1432+
reduce = stackless._stackless._wrap.code.__reduce__
1433+
reduced = reduce(code)
1434+
args = (reduced[1][0] + 1,) + reduced[1][1:]
1435+
with stackless.atomic():
1436+
with warnings.catch_warnings():
1437+
warnings.simplefilter("ignore", RuntimeWarning)
1438+
code2 = reduced[0](*args)
1439+
code2.__setstate__(())
1440+
assert(type(code2) is type(code))
1441+
# now execute code 2, first create a function from code2
1442+
f = type(l)(code2, globals())
1443+
# f should raise
1444+
try:
1445+
f()
1446+
except SystemError as e:
1447+
assert(str(e) == 'unknown opcode')
1448+
else:
1449+
assert(0, "expected exception not raised")
1450+
sys.exit(42)
1451+
""")] + args, stderr=subprocess.DEVNULL)
1452+
self.assertEqual(rc, 42)
1453+
1454+
13491455
if __name__ == '__main__':
13501456
if not sys.argv[1:]:
13511457
sys.argv.append('-v')

0 commit comments

Comments
 (0)