Skip to content

Commit e634391

Browse files
methaneDinoV
methane
authored and
DinoV
committed
bpo-26219: per opcode cache for LOAD_GLOBAL (pythonGH-12884)
This patch implements per opcode cache mechanism, and use it in only LOAD_GLOBAL opcode. Based on Yury's opcache3.patch in bpo-26219.
1 parent faf33da commit e634391

13 files changed

Lines changed: 285 additions & 16 deletions

File tree

‎Doc/whatsnew/3.8.rst‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -860,6 +860,10 @@ Optimizations
860860
methods up to 20--50%. (Contributed by Serhiy Storchaka in :issue:`23867`,
861861
:issue:`35582` and :issue:`36127`.)
862862

863+
* ``LOAD_GLOBAL`` instruction now uses new "per opcode cache" mechanism.
864+
It is about 40% faster now. (Contributed by Yury Selivanov and Inada Naoki in
865+
:issue:`26219`.)
866+
863867

864868
Build and C API Changes
865869
=======================

‎Include/code.h‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,8 @@ typedef uint16_t _Py_CODEUNIT;
1717
# define _Py_OPARG(word) ((word) >> 8)
1818
#endif
1919

20+
typedef struct _PyOpcache _PyOpcache;
21+
2022
/* Bytecode object */
2123
typedef struct {
2224
PyObject_HEAD
@@ -49,6 +51,21 @@ typedef struct {
4951
Type is a void* to keep the format private in codeobject.c to force
5052
people to go through the proper APIs. */
5153
void *co_extra;
54+
55+
/* Per opcodes just-in-time cache
56+
*
57+
* To reduce cache size, we use indirect mapping from opcode index to
58+
* cache object:
59+
* cache = co_opcache[co_opcache_map[next_instr - first_instr] - 1]
60+
*/
61+
62+
// co_opcache_map is indexed by (next_instr - first_instr).
63+
// * 0 means there is no cache for this opcode.
64+
// * n > 0 means there is cache in co_opcache[n-1].
65+
unsigned char *co_opcache_map;
66+
_PyOpcache *co_opcache;
67+
int co_opcache_flag; // used to determine when create a cache.
68+
unsigned char co_opcache_size; // length of co_opcache.
5269
} PyCodeObject;
5370

5471
/* Masks for co_flags above */

‎Include/internal/pycore_ceval.h‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,9 @@ PyAPI_FUNC(void) _PyEval_SignalAsyncExc(
3131
PyAPI_FUNC(void) _PyEval_ReInitThreads(
3232
_PyRuntimeState *runtime);
3333

34+
/* Private function */
35+
void _PyEval_Fini(void);
36+
3437
#ifdef __cplusplus
3538
}
3639
#endif

‎Include/internal/pycore_code.h‎

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,27 @@
1+
#ifndef Py_INTERNAL_CODE_H
2+
#define Py_INTERNAL_CODE_H
3+
#ifdef __cplusplus
4+
extern "C" {
5+
#endif
6+
7+
typedef struct {
8+
PyObject *ptr; /* Cached pointer (borrowed reference) */
9+
uint64_t globals_ver; /* ma_version of global dict */
10+
uint64_t builtins_ver; /* ma_version of builtin dict */
11+
} _PyOpcache_LoadGlobal;
12+
13+
struct _PyOpcache {
14+
union {
15+
_PyOpcache_LoadGlobal lg;
16+
} u;
17+
char optimized;
18+
};
19+
20+
/* Private API */
21+
int _PyCode_InitOpcache(PyCodeObject *co);
22+
23+
24+
#ifdef __cplusplus
25+
}
26+
#endif
27+
#endif /* !Py_INTERNAL_CODE_H */

‎Lib/test/test_dict_version.py‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -80,14 +80,14 @@ def test_setitem_same_value(self):
8080

8181
# setting a key to the same value with dict.__setitem__
8282
# must change the version
83-
self.check_version_changed(d, d.__setitem__, 'key', value)
83+
self.check_version_dont_change(d, d.__setitem__, 'key', value)
8484

8585
# setting a key to the same value with dict.update
8686
# must change the version
87-
self.check_version_changed(d, d.update, key=value)
87+
self.check_version_dont_change(d, d.update, key=value)
8888

8989
d2 = self.new_dict(key=value)
90-
self.check_version_changed(d, d.update, d2)
90+
self.check_version_dont_change(d, d.update, d2)
9191

9292
def test_setitem_equal(self):
9393
class AlwaysEqual:

‎Makefile.pre.in‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1070,6 +1070,7 @@ PYTHON_HEADERS= \
10701070
$(srcdir)/Include/internal/pycore_accu.h \
10711071
$(srcdir)/Include/internal/pycore_atomic.h \
10721072
$(srcdir)/Include/internal/pycore_ceval.h \
1073+
$(srcdir)/Include/internal/pycore_code.h \
10731074
$(srcdir)/Include/internal/pycore_condvar.h \
10741075
$(srcdir)/Include/internal/pycore_context.h \
10751076
$(srcdir)/Include/internal/pycore_fileutils.h \

‎Misc/NEWS.d/next/Core and Builtins/2019-05-29-22-03-09.bpo-26219.Ovf1Qs.rst‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
Implemented per opcode cache mechanism and ``LOAD_GLOBAL`` instruction use
2+
it. ``LOAD_GLOBAL`` is now about 40% faster. Contributed by Yury Selivanov,
3+
and Inada Naoki.

‎Objects/codeobject.c‎

Lines changed: 65 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,9 @@
22

33
#include "Python.h"
44
#include "code.h"
5+
#include "opcode.h"
56
#include "structmember.h"
7+
#include "pycore_code.h"
68
#include "pycore_pystate.h"
79
#include "pycore_tupleobject.h"
810
#include "clinic/codeobject.c.h"
@@ -252,9 +254,56 @@ PyCode_New(int argcount, int posonlyargcount, int kwonlyargcount,
252254
co->co_zombieframe = NULL;
253255
co->co_weakreflist = NULL;
254256
co->co_extra = NULL;
257+
258+
co->co_opcache_map = NULL;
259+
co->co_opcache = NULL;
260+
co->co_opcache_flag = 0;
261+
co->co_opcache_size = 0;
255262
return co;
256263
}
257264

265+
int
266+
_PyCode_InitOpcache(PyCodeObject *co)
267+
{
268+
Py_ssize_t co_size = PyBytes_Size(co->co_code) / sizeof(_Py_CODEUNIT);
269+
co->co_opcache_map = (unsigned char *)PyMem_Calloc(co_size, 1);
270+
if (co->co_opcache_map == NULL) {
271+
return -1;
272+
}
273+
274+
_Py_CODEUNIT *opcodes = (_Py_CODEUNIT*)PyBytes_AS_STRING(co->co_code);
275+
Py_ssize_t opts = 0;
276+
277+
for (Py_ssize_t i = 0; i < co_size;) {
278+
unsigned char opcode = _Py_OPCODE(opcodes[i]);
279+
i++; // 'i' is now aligned to (next_instr - first_instr)
280+
281+
// TODO: LOAD_METHOD, LOAD_ATTR
282+
if (opcode == LOAD_GLOBAL) {
283+
co->co_opcache_map[i] = ++opts;
284+
if (opts > 254) {
285+
break;
286+
}
287+
}
288+
}
289+
290+
if (opts) {
291+
co->co_opcache = (_PyOpcache *)PyMem_Calloc(opts, sizeof(_PyOpcache));
292+
if (co->co_opcache == NULL) {
293+
PyMem_FREE(co->co_opcache_map);
294+
return -1;
295+
}
296+
}
297+
else {
298+
PyMem_FREE(co->co_opcache_map);
299+
co->co_opcache_map = NULL;
300+
co->co_opcache = NULL;
301+
}
302+
303+
co->co_opcache_size = opts;
304+
return 0;
305+
}
306+
258307
PyCodeObject *
259308
PyCode_NewEmpty(const char *filename, const char *funcname, int firstlineno)
260309
{
@@ -485,6 +534,15 @@ code_new(PyTypeObject *type, PyObject *args, PyObject *kw)
485534
static void
486535
code_dealloc(PyCodeObject *co)
487536
{
537+
if (co->co_opcache != NULL) {
538+
PyMem_FREE(co->co_opcache);
539+
}
540+
if (co->co_opcache_map != NULL) {
541+
PyMem_FREE(co->co_opcache_map);
542+
}
543+
co->co_opcache_flag = 0;
544+
co->co_opcache_size = 0;
545+
488546
if (co->co_extra != NULL) {
489547
PyInterpreterState *interp = _PyInterpreterState_GET_UNSAFE();
490548
_PyCodeObjectExtra *co_extra = co->co_extra;
@@ -531,6 +589,13 @@ code_sizeof(PyCodeObject *co, PyObject *Py_UNUSED(args))
531589
res += sizeof(_PyCodeObjectExtra) +
532590
(co_extra->ce_size-1) * sizeof(co_extra->ce_extras[0]);
533591
}
592+
if (co->co_opcache != NULL) {
593+
assert(co->co_opcache_map != NULL);
594+
// co_opcache_map
595+
res += PyBytes_GET_SIZE(co->co_code) / sizeof(_Py_CODEUNIT);
596+
// co_opcache
597+
res += co->co_opcache_size * sizeof(_PyOpcache);
598+
}
534599
return PyLong_FromSsize_t(res);
535600
}
536601

‎Objects/dictobject.c‎

Lines changed: 13 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -1080,20 +1080,21 @@ insertdict(PyDictObject *mp, PyObject *key, Py_hash_t hash, PyObject *value)
10801080
return 0;
10811081
}
10821082

1083-
if (_PyDict_HasSplitTable(mp)) {
1084-
mp->ma_values[ix] = value;
1085-
if (old_value == NULL) {
1086-
/* pending state */
1087-
assert(ix == mp->ma_used);
1088-
mp->ma_used++;
1083+
if (old_value != value) {
1084+
if (_PyDict_HasSplitTable(mp)) {
1085+
mp->ma_values[ix] = value;
1086+
if (old_value == NULL) {
1087+
/* pending state */
1088+
assert(ix == mp->ma_used);
1089+
mp->ma_used++;
1090+
}
10891091
}
1092+
else {
1093+
assert(old_value != NULL);
1094+
DK_ENTRIES(mp->ma_keys)[ix].me_value = value;
1095+
}
1096+
mp->ma_version_tag = DICT_NEXT_VERSION();
10901097
}
1091-
else {
1092-
assert(old_value != NULL);
1093-
DK_ENTRIES(mp->ma_keys)[ix].me_value = value;
1094-
}
1095-
1096-
mp->ma_version_tag = DICT_NEXT_VERSION();
10971098
Py_XDECREF(old_value); /* which **CAN** re-enter (see issue #22653) */
10981099
ASSERT_CONSISTENT(mp);
10991100
Py_DECREF(key);

‎PCbuild/pythoncore.vcxproj‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -158,6 +158,7 @@
158158
<ClInclude Include="..\Include\import.h" />
159159
<ClInclude Include="..\Include\internal\pycore_accu.h" />
160160
<ClInclude Include="..\Include\internal\pycore_atomic.h" />
161+
<ClInclude Include="..\Include\internal\pycore_code.h" />
161162
<ClInclude Include="..\Include\internal\pycore_ceval.h" />
162163
<ClInclude Include="..\Include\internal\pycore_condvar.h" />
163164
<ClInclude Include="..\Include\internal\pycore_context.h" />

0 commit comments

Comments
 (0)