Skip to content

gh-143421: Use new buffer to save optimized uops #143682

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
Fidget-Spinner merged 12 commits into from
Jan 17, 2026
Merged

gh-143421: Use new buffer to save optimized uops #143682

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
12 commits
Select commit Hold shift + click to select a range
e771fd0
use add_op to save optimized uops
cocolato Jan 11, 2026
4a0db15
optimize code struct
cocolato Jan 11, 2026
25797cf
optimize code struct
cocolato Jan 12, 2026
618cb4f
Merge branch 'main' into feat/ADD_OP
cocolato Jan 12, 2026
57a22e9
regen cases
cocolato Jan 12, 2026
031d6e3
optimize code
cocolato Jan 12, 2026
45f8ce0
remove out_len
cocolato Jan 12, 2026
2969891
Merge branch 'main' into feat/ADD_OP
cocolato Jan 12, 2026
0d07c5b
regen cases
cocolato Jan 12, 2026
15f32b9
move out_len to JitOptContext
cocolato Jan 12, 2026
543971c
remove cur_len
cocolato Jan 12, 2026
184196e
Merge branch 'main' into feat/ADD_OP
cocolato Jan 17, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
remove out_len
  • Loading branch information
@cocolato
cocolato committed Jan 12, 2026
commit 45f8ce0cbd0b9e3ad2194817a3b0ccfe4bdb3eef
4 changes: 1 addition & 3 deletions Include/internal/pycore_optimizer_types.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,8 +10,6 @@ extern "C" {

#include "pycore_uop.h" // UOP_MAX_TRACE_LENGTH

typedef struct _PyJitTracerState _PyJitTracerState;

// Holds locals, stack, locals, stack ... (in that order)
#define MAX_ABSTRACT_INTERP_SIZE 512

Expand Down Expand Up @@ -130,7 +128,7 @@ typedef struct _JitOptContext {
JitOptRef *n_consumed;
JitOptRef *limit;
JitOptRef locals_and_stack[MAX_ABSTRACT_INTERP_SIZE];
_PyJitTracerState *tracer;
_PyUOpInstruction *out_buffer;
} JitOptContext;


Expand Down
1 change: 0 additions & 1 deletion Include/internal/pycore_tstate.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -60,7 +60,6 @@ typedef struct _PyJitTracerState {
JitOptContext opt_context;
_PyUOpInstruction code_buffer[UOP_MAX_TRACE_LENGTH];
_PyUOpInstruction out_buffer[UOP_MAX_TRACE_LENGTH];
int out_len;
} _PyJitTracerState;

#endif
Expand Down
39 changes: 20 additions & 19 deletions Python/optimizer_analysis.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -164,20 +164,20 @@ is_terminator_uop(const _PyUOpInstruction *uop)
(INST)->oparg = ARG; \
(INST)->operand0 = OPERAND;

#define ADD_OP(OP, ARG, OPERAND) add_op(ctx, this_instr, (OP), (ARG), (OPERAND))
#define ADD_OP(OP, ARG, OPERAND) add_op(ctx, this_instr, &out_len, (OP), (ARG), (OPERAND))

static inline void
add_op(JitOptContext *ctx, _PyUOpInstruction *this_instr,
add_op(JitOptContext *ctx, _PyUOpInstruction *this_instr, int *out_len,
uint16_t opcode, uint16_t oparg, uintptr_t operand0)
{
_PyUOpInstruction *out = &ctx->tracer->out_buffer[ctx->tracer->out_len];
_PyUOpInstruction *out = &ctx->out_buffer[*out_len];
out->opcode = (opcode);
out->format = this_instr->format;
out->oparg = (oparg);
out->target = this_instr->target;
out->operand0 = (operand0);
out->operand1 = this_instr->operand1;
ctx->tracer->out_len++;
(*out_len)++;
}

/* Shortened forms for convenience, used in optimizer_bytecodes.c */
Expand Down Expand Up @@ -242,12 +242,13 @@ static int
optimize_to_bool(
_PyUOpInstruction *this_instr,
JitOptContext *ctx,
int *out_len,
JitOptRef value,
JitOptRef *result_ptr,
bool insert_mode)
{
if (sym_matches_type(value, &PyBool_Type)) {
ADD_OP(_NOP, 0, 0);
add_op(ctx, this_instr, out_len, _NOP, 0, 0);
*result_ptr = value;
return 1;
}
Expand All @@ -257,17 +258,17 @@ optimize_to_bool(
int opcode = insert_mode ?
_INSERT_1_LOAD_CONST_INLINE_BORROW :
_POP_TOP_LOAD_CONST_INLINE_BORROW;
ADD_OP(opcode, 0, (uintptr_t)load);
add_op(ctx, this_instr, out_len, opcode, 0, (uintptr_t)load);
*result_ptr = sym_new_const(ctx, load);
return 1;
}
return 0;
}

static void
eliminate_pop_guard(_PyUOpInstruction *this_instr, JitOptContext *ctx, bool exit)
eliminate_pop_guard(_PyUOpInstruction *this_instr, JitOptContext *ctx, int *out_len, bool exit)
{
ADD_OP(_POP_TOP, 0, 0);
add_op(ctx, this_instr, out_len, _POP_TOP, 0, 0);
if (exit) {
REPLACE_OP((this_instr+1), _EXIT_TRACE, 0, 0);
this_instr[1].target = this_instr->target;
Expand All @@ -276,15 +277,15 @@ eliminate_pop_guard(_PyUOpInstruction *this_instr, JitOptContext *ctx, bool exit

static JitOptRef
lookup_attr(JitOptContext *ctx, _PyBloomFilter *dependencies, _PyUOpInstruction *this_instr,
PyTypeObject *type, PyObject *name, uint16_t immortal,
int *out_len, PyTypeObject *type, PyObject *name, uint16_t immortal,
uint16_t mortal)
{
// The cached value may be dead, so we need to do the lookup again... :(
if (type && PyType_Check(type)) {
PyObject *lookup = _PyType_Lookup(type, name);
if (lookup) {
int opcode = _Py_IsImmortal(lookup) ? immortal : mortal;
ADD_OP(opcode, 0, (uintptr_t)lookup);
add_op(ctx, this_instr, out_len, opcode, 0, (uintptr_t)lookup);
PyType_Watch(TYPE_WATCHER_ID, (PyObject *)type);
_Py_BloomFilter_Add(dependencies, type);
return sym_new_const(ctx, lookup);
Expand Down Expand Up @@ -377,7 +378,7 @@ optimize_uops(
JitOptContext *ctx = &tstate->jit_tracer_state->opt_context;
uint32_t opcode = UINT16_MAX;

ctx->tracer = tstate->jit_tracer_state;
ctx->out_buffer = tstate->jit_tracer_state->out_buffer;

// Make sure that watchers are set up
PyInterpreterState *interp = _PyInterpreterState_GET();
Expand All @@ -394,7 +395,7 @@ optimize_uops(
frame->func = func;
ctx->curr_frame_depth++;
ctx->frame = frame;
ctx->tracer->out_len = 0;
int out_len = 0;

_PyUOpInstruction *this_instr = NULL;
JitOptRef *stack_pointer = ctx->frame->stack_pointer;
Expand Down Expand Up @@ -427,8 +428,8 @@ optimize_uops(
Py_UNREACHABLE();
}
// If no ADD_OP was called during this iteration, copy the original instruction
if (ctx->tracer->out_len == i) {
ctx->tracer->out_buffer[ctx->tracer->out_len++] = *this_instr;
if (out_len == i) {
ctx->out_buffer[out_len++] = *this_instr;
}
assert(ctx->frame != NULL);
if (!CURRENT_FRAME_IS_INIT_SHIM()) {
Expand Down Expand Up @@ -459,20 +460,20 @@ optimize_uops(
* would be no benefit in retrying later */
_Py_uop_abstractcontext_fini(ctx);
// Check that the trace ends with a proper terminator
if (ctx->tracer->out_len > 0) {
_PyUOpInstruction *last_uop = &ctx->tracer->out_buffer[ctx->tracer->out_len - 1];
if (out_len > 0) {
_PyUOpInstruction *last_uop = &ctx->out_buffer[out_len - 1];
if (!is_terminator_uop(last_uop)) {
// Copy remaining uops from original trace until we find a terminator
for (int i = ctx->tracer->out_len; i < trace_len; i++) {
ctx->tracer->out_buffer[ctx->tracer->out_len++] = trace[i];
for (int i = out_len; i < trace_len; i++) {
ctx->out_buffer[out_len++] = trace[i];
if (is_terminator_uop(&trace[i])) {
break;
}
}
}
}

return ctx->tracer->out_len;
return out_len;

error:
DPRINTF(3, "\n");
Expand Down
46 changes: 24 additions & 22 deletions Python/optimizer_bytecodes.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -43,12 +43,13 @@ extern int
optimize_to_bool(
_PyUOpInstruction *this_instr,
JitOptContext *ctx,
int *out_len,
JitOptSymbol *value,
JitOptSymbol **result_ptr,
bool insert_mode);

extern void
eliminate_pop_guard(_PyUOpInstruction *this_instr, JitOptContext *ctx, bool exit);
eliminate_pop_guard(_PyUOpInstruction *this_instr, JitOptContext *ctx, int *out_len, bool exit);

extern PyCodeObject *get_code(_PyUOpInstruction *op);

Expand All @@ -57,6 +58,7 @@ dummy_func(void) {

PyCodeObject *co;
int oparg;
int out_len;
JitOptSymbol *flag;
JitOptSymbol *left;
JitOptSymbol *right;
Expand Down Expand Up @@ -381,36 +383,36 @@ dummy_func(void) {
}

op(_TO_BOOL, (value -- res)) {
int already_bool = optimize_to_bool(this_instr, ctx, value, &res, false);
int already_bool = optimize_to_bool(this_instr, ctx, &out_len, value, &res, false);
if (!already_bool) {
res = sym_new_truthiness(ctx, value, true);
}
}

op(_TO_BOOL_BOOL, (value -- value)) {
int already_bool = optimize_to_bool(this_instr, ctx, value, &value, false);
int already_bool = optimize_to_bool(this_instr, ctx, &out_len, value, &value, false);
if (!already_bool) {
sym_set_type(value, &PyBool_Type);
}
}

op(_TO_BOOL_INT, (value -- res)) {
int already_bool = optimize_to_bool(this_instr, ctx, value, &res, false);
int already_bool = optimize_to_bool(this_instr, ctx, &out_len, value, &res, false);
if (!already_bool) {
sym_set_type(value, &PyLong_Type);
res = sym_new_truthiness(ctx, value, true);
}
}

op(_TO_BOOL_LIST, (value -- res)) {
int already_bool = optimize_to_bool(this_instr, ctx, value, &res, false);
int already_bool = optimize_to_bool(this_instr, ctx, &out_len, value, &res, false);
if (!already_bool) {
res = sym_new_type(ctx, &PyBool_Type);
}
}

op(_TO_BOOL_NONE, (value -- res)) {
int already_bool = optimize_to_bool(this_instr, ctx, value, &res, false);
int already_bool = optimize_to_bool(this_instr, ctx, &out_len, value, &res, false);
if (!already_bool) {
sym_set_const(value, Py_None);
res = sym_new_const(ctx, Py_False);
Expand All @@ -436,7 +438,7 @@ dummy_func(void) {
}

op(_TO_BOOL_STR, (value -- res, v)) {
int already_bool = optimize_to_bool(this_instr, ctx, value, &res, true);
int already_bool = optimize_to_bool(this_instr, ctx, &out_len, value, &res, true);
v = value;
if (!already_bool) {
res = sym_new_truthiness(ctx, value, true);
Expand Down Expand Up @@ -693,7 +695,7 @@ dummy_func(void) {
(void)descr;
PyTypeObject *type = (PyTypeObject *)sym_get_const(ctx, owner);
PyObject *name = get_co_name(ctx, oparg >> 1);
attr = lookup_attr(ctx, dependencies, this_instr, type, name,
attr = lookup_attr(ctx, dependencies, this_instr, &out_len, type, name,
_POP_TOP_LOAD_CONST_INLINE_BORROW,
_POP_TOP_LOAD_CONST_INLINE);
}
Expand All @@ -702,7 +704,7 @@ dummy_func(void) {
(void)descr;
PyTypeObject *type = sym_get_type(owner);
PyObject *name = get_co_name(ctx, oparg >> 1);
attr = lookup_attr(ctx, dependencies, this_instr, type, name,
attr = lookup_attr(ctx, dependencies, this_instr, &out_len, type, name,
_POP_TOP_LOAD_CONST_INLINE_BORROW,
_POP_TOP_LOAD_CONST_INLINE);
}
Expand All @@ -711,7 +713,7 @@ dummy_func(void) {
(void)descr;
PyTypeObject *type = sym_get_type(owner);
PyObject *name = get_co_name(ctx, oparg >> 1);
attr = lookup_attr(ctx, dependencies, this_instr, type, name,
attr = lookup_attr(ctx, dependencies, this_instr, &out_len, type, name,
_POP_TOP_LOAD_CONST_INLINE_BORROW,
_POP_TOP_LOAD_CONST_INLINE);
}
Expand All @@ -720,7 +722,7 @@ dummy_func(void) {
(void)descr;
PyTypeObject *type = sym_get_type(owner);
PyObject *name = get_co_name(ctx, oparg >> 1);
attr = lookup_attr(ctx, dependencies, this_instr, type, name,
attr = lookup_attr(ctx, dependencies, this_instr, &out_len, type, name,
_LOAD_CONST_UNDER_INLINE_BORROW,
_LOAD_CONST_UNDER_INLINE);
self = owner;
Expand All @@ -730,7 +732,7 @@ dummy_func(void) {
(void)descr;
PyTypeObject *type = sym_get_type(owner);
PyObject *name = get_co_name(ctx, oparg >> 1);
attr = lookup_attr(ctx, dependencies, this_instr, type, name,
attr = lookup_attr(ctx, dependencies, this_instr, &out_len, type, name,
_LOAD_CONST_UNDER_INLINE_BORROW,
_LOAD_CONST_UNDER_INLINE);
self = owner;
Expand All @@ -740,7 +742,7 @@ dummy_func(void) {
(void)descr;
PyTypeObject *type = sym_get_type(owner);
PyObject *name = get_co_name(ctx, oparg >> 1);
attr = lookup_attr(ctx, dependencies, this_instr, type, name,
attr = lookup_attr(ctx, dependencies, this_instr, &out_len, type, name,
_LOAD_CONST_UNDER_INLINE_BORROW,
_LOAD_CONST_UNDER_INLINE);
self = owner;
Expand All @@ -761,7 +763,7 @@ dummy_func(void) {
if (sym_is_const(ctx, callable) && sym_matches_type(callable, &PyFunction_Type)) {
assert(PyFunction_Check(sym_get_const(ctx, callable)));
ADD_OP(_CHECK_FUNCTION_VERSION_INLINE, 0, func_version);
ctx->tracer->out_buffer[ctx->tracer->out_len - 1].operand1 = (uintptr_t)sym_get_const(ctx, callable);
ctx->out_buffer[out_len - 1].operand1 = (uintptr_t)sym_get_const(ctx, callable);
}
sym_set_type(callable, &PyFunction_Type);
}
Expand All @@ -771,7 +773,7 @@ dummy_func(void) {
PyMethodObject *method = (PyMethodObject *)sym_get_const(ctx, callable);
assert(PyMethod_Check(method));
ADD_OP(_CHECK_FUNCTION_VERSION_INLINE, 0, func_version);
ctx->tracer->out_buffer[ctx->tracer->out_len - 1].operand1 = (uintptr_t)method->im_func;
ctx->out_buffer[out_len - 1].operand1 = (uintptr_t)method->im_func;
}
sym_set_type(callable, &PyMethod_Type);
}
Expand Down Expand Up @@ -1105,7 +1107,7 @@ dummy_func(void) {
if (sym_is_const(ctx, flag)) {
PyObject *value = sym_get_const(ctx, flag);
assert(value != NULL);
eliminate_pop_guard(this_instr, ctx, value != Py_True);
eliminate_pop_guard(this_instr, ctx, &out_len, value != Py_True);
}
sym_set_const(flag, Py_True);
}
Expand Down Expand Up @@ -1150,7 +1152,7 @@ dummy_func(void) {
if (sym_is_const(ctx, flag)) {
PyObject *value = sym_get_const(ctx, flag);
assert(value != NULL);
eliminate_pop_guard(this_instr, ctx, value != Py_False);
eliminate_pop_guard(this_instr, ctx, &out_len, value != Py_False);
}
sym_set_const(flag, Py_False);
}
Expand All @@ -1159,11 +1161,11 @@ dummy_func(void) {
if (sym_is_const(ctx, val)) {
PyObject *value = sym_get_const(ctx, val);
assert(value != NULL);
eliminate_pop_guard(this_instr, ctx, !Py_IsNone(value));
eliminate_pop_guard(this_instr, ctx, &out_len, !Py_IsNone(value));
}
else if (sym_has_type(val)) {
assert(!sym_matches_type(val, &_PyNone_Type));
eliminate_pop_guard(this_instr, ctx, true);
eliminate_pop_guard(this_instr, ctx, &out_len, true);
}
sym_set_const(val, Py_None);
}
Expand All @@ -1172,11 +1174,11 @@ dummy_func(void) {
if (sym_is_const(ctx, val)) {
PyObject *value = sym_get_const(ctx, val);
assert(value != NULL);
eliminate_pop_guard(this_instr, ctx, Py_IsNone(value));
eliminate_pop_guard(this_instr, ctx, &out_len, Py_IsNone(value));
}
else if (sym_has_type(val)) {
assert(!sym_matches_type(val, &_PyNone_Type));
eliminate_pop_guard(this_instr, ctx, false);
eliminate_pop_guard(this_instr, ctx, &out_len, false);
}
}

Expand Down Expand Up @@ -1511,7 +1513,7 @@ dummy_func(void) {
ctx->frame->globals_watched = true;
}
if (ctx->frame->globals_checked_version != version && this_instr[-1].opcode == _NOP) {
REPLACE_OP(&ctx->tracer->out_buffer[ctx->tracer->out_len - 1], _GUARD_GLOBALS_VERSION, 0, version);
REPLACE_OP(&ctx->out_buffer[out_len - 1], _GUARD_GLOBALS_VERSION, 0, version);
ctx->frame->globals_checked_version = version;
}
if (ctx->frame->globals_checked_version == version) {
Expand Down
Loading