Skip to content

gh-146640: Optimize int operations by mutating uniquely-referenced operands in place (JIT only) #146641

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
Fidget-Spinner merged 15 commits into from
Apr 3, 2026
Merged

gh-146640: Optimize int operations by mutating uniquely-referenced operands in place (JIT only) #146641

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
15 commits
Select commit Hold shift + click to select a range
befeded
Add inplace binary operations for type int-int
eendebakpt Mar 30, 2026
964bb34
rework
eendebakpt Mar 30, 2026
f33b452
refactor
eendebakpt Mar 30, 2026
654ae93
fix tag assignment
eendebakpt Mar 30, 2026
af5b4c9
fix result refs
eendebakpt Mar 30, 2026
9d13a23
review comments
eendebakpt Mar 31, 2026
46b6cca
news entry
eendebakpt Mar 31, 2026
63c1317
fix
eendebakpt Mar 31, 2026
4d6288b
move dup into macro
eendebakpt Mar 31, 2026
73b9a45
Merge branch 'main' into int_int_inplace
eendebakpt Mar 31, 2026
3fb1e6c
Apply suggestion from @eendebakpt
eendebakpt Mar 31, 2026
7710226
revert
eendebakpt Mar 31, 2026
60b81be
review comments
eendebakpt Apr 1, 2026
e0bae62
Merge branch 'main' into int_int_inplace
eendebakpt Apr 1, 2026
76e0785
Apply suggestion from @eendebakpt
eendebakpt Apr 3, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
rework
  • Loading branch information
@eendebakpt
eendebakpt committed Mar 30, 2026
commit 964bb34be583a9bd3cf7b97e2d4c97329ee04cc4
84 changes: 63 additions & 21 deletions Python/bytecodes.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -708,60 +708,102 @@ dummy_func(
macro(BINARY_OP_SUBTRACT_INT) =
_GUARD_TOS_INT + _GUARD_NOS_INT + unused/5 + _BINARY_OP_SUBTRACT_INT + _POP_TOP_INT + _POP_TOP_INT;

// Inplace compact int ops: mutate the uniquely-referenced operand
// when the result is a non-small medium int. For small int results,
// return the cached singleton. Deopt on overflow. Tier 2 only.
// Inplace compact int ops: try to mutate the uniquely-referenced
// operand. If TARGET is immortal (small int at runtime) or the
// result doesn't fit, fall back to _PyCompactLong_*. Tier 2 only.
tier2 op(_BINARY_OP_ADD_INT_INPLACE, (left, right -- res, l, r)) {
INT_INPLACE_OP(left, right, left, +);

@Fidget-Spinner Fidget-Spinner Mar 31, 2026

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think you can factor out all the common code here into a function that takes a function pointer (the operation to do).

For each op, pass the function pointer e.g. _PyCompactLong_Add or whatever, and then the CPU/inliner can fix it up/constant-fold it away in the compiler.

EXIT_IF(_int_inplace_status == 2);
res = _int_inplace_status == 1 ? _int_inplace_smallref : left;
l = _int_inplace_status == 0 ? PyStackRef_NULL : left;
if (_int_inplace_ok) {
res = left;
l = PyStackRef_NULL;
}
else {
res = _PyCompactLong_Add((PyLongObject *)PyStackRef_AsPyObjectBorrow(left),
(PyLongObject *)PyStackRef_AsPyObjectBorrow(right));
EXIT_IF(PyStackRef_IsNull(res));
l = left;
}
r = right;
INPUTS_DEAD();
}

tier2 op(_BINARY_OP_SUBTRACT_INT_INPLACE, (left, right -- res, l, r)) {
INT_INPLACE_OP(left, right, left, -);
EXIT_IF(_int_inplace_status == 2);
res = _int_inplace_status == 1 ? _int_inplace_smallref : left;
l = _int_inplace_status == 0 ? PyStackRef_NULL : left;
if (_int_inplace_ok) {
res = left;
l = PyStackRef_NULL;
}
else {
res = _PyCompactLong_Subtract((PyLongObject *)PyStackRef_AsPyObjectBorrow(left),
(PyLongObject *)PyStackRef_AsPyObjectBorrow(right));
EXIT_IF(PyStackRef_IsNull(res));
l = left;
}
r = right;
INPUTS_DEAD();
}

tier2 op(_BINARY_OP_MULTIPLY_INT_INPLACE, (left, right -- res, l, r)) {
INT_INPLACE_OP(left, right, left, *);
EXIT_IF(_int_inplace_status == 2);
res = _int_inplace_status == 1 ? _int_inplace_smallref : left;
l = _int_inplace_status == 0 ? PyStackRef_NULL : left;
if (_int_inplace_ok) {
res = left;
l = PyStackRef_NULL;
}
else {
res = _PyCompactLong_Multiply((PyLongObject *)PyStackRef_AsPyObjectBorrow(left),
(PyLongObject *)PyStackRef_AsPyObjectBorrow(right));
EXIT_IF(PyStackRef_IsNull(res));
l = left;
}
r = right;
INPUTS_DEAD();
}

tier2 op(_BINARY_OP_ADD_INT_INPLACE_RIGHT, (left, right -- res, l, r)) {
INT_INPLACE_OP(left, right, right, +);
EXIT_IF(_int_inplace_status == 2);
res = _int_inplace_status == 1 ? _int_inplace_smallref : right;
if (_int_inplace_ok) {
res = right;
r = PyStackRef_NULL;
}
else {
res = _PyCompactLong_Add((PyLongObject *)PyStackRef_AsPyObjectBorrow(left),
(PyLongObject *)PyStackRef_AsPyObjectBorrow(right));
EXIT_IF(PyStackRef_IsNull(res));
r = right;
}
l = left;
r = _int_inplace_status == 0 ? PyStackRef_NULL : right;
INPUTS_DEAD();
}

tier2 op(_BINARY_OP_SUBTRACT_INT_INPLACE_RIGHT, (left, right -- res, l, r)) {
INT_INPLACE_OP(left, right, right, -);
EXIT_IF(_int_inplace_status == 2);
res = _int_inplace_status == 1 ? _int_inplace_smallref : right;
if (_int_inplace_ok) {
res = right;
r = PyStackRef_NULL;
}
else {
res = _PyCompactLong_Subtract((PyLongObject *)PyStackRef_AsPyObjectBorrow(left),
(PyLongObject *)PyStackRef_AsPyObjectBorrow(right));
EXIT_IF(PyStackRef_IsNull(res));
r = right;
}
l = left;
r = _int_inplace_status == 0 ? PyStackRef_NULL : right;
INPUTS_DEAD();
}

tier2 op(_BINARY_OP_MULTIPLY_INT_INPLACE_RIGHT, (left, right -- res, l, r)) {
INT_INPLACE_OP(left, right, right, *);
EXIT_IF(_int_inplace_status == 2);
res = _int_inplace_status == 1 ? _int_inplace_smallref : right;
if (_int_inplace_ok) {
res = right;
r = PyStackRef_NULL;
}
else {
res = _PyCompactLong_Multiply((PyLongObject *)PyStackRef_AsPyObjectBorrow(left),
(PyLongObject *)PyStackRef_AsPyObjectBorrow(right));
EXIT_IF(PyStackRef_IsNull(res));
r = right;
}
l = left;
r = _int_inplace_status == 0 ? PyStackRef_NULL : right;
INPUTS_DEAD();
}

Expand Down
47 changes: 22 additions & 25 deletions Python/ceval_macros.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -564,52 +564,49 @@ gen_try_set_executing(PyGenObject *gen)
} while (0)

// Inplace compact int operation. Computes left OP right and attempts to
// store the result in TARGET (which must be uniquely referenced).
// After the macro:
// _int_inplace_ok = 1: TARGET was mutated in place, res = TARGET
// _int_inplace_ok = 0: result is a small int or overflow; caller
// must fall back to _PyCompactLong_* and EXIT_IF
// _int_inplace_status after macro:
// 0 = mutated TARGET in place (res = TARGET)
// 1 = result is a small int stored in _int_inplace_smallref
// 2 = overflow, caller must EXIT_IF
// Inplace compact int operation. TARGET is expected to be uniquely
// referenced at the optimizer level, but at runtime it may be a
// cached small int singleton. We check _Py_IsImmortal on TARGET
// to decide whether inplace mutation is safe.
//
// _int_inplace_ok after macro:
// 1 = mutated TARGET in place (caller should set res = TARGET)
// 0 = TARGET was immortal (small int); caller should use the
// standard _PyCompactLong_* function instead
#define INT_INPLACE_OP(left, right, TARGET, OP) \
int _int_inplace_status = 0; \
_PyStackRef _int_inplace_smallref = PyStackRef_NULL; \
int _int_inplace_ok = 0; \
do { \
PyObject *_target_o = PyStackRef_AsPyObjectBorrow(TARGET); \
Comment thread
vstinner marked this conversation as resolved.
Outdated
if (_Py_IsImmortal(_target_o)) { \
break; \
} \
PyObject *_left_o = PyStackRef_AsPyObjectBorrow(left); \
PyObject *_right_o = PyStackRef_AsPyObjectBorrow(right); \
assert(PyLong_CheckExact(_left_o)); \
assert(PyLong_CheckExact(_right_o)); \
assert(_PyLong_BothAreCompact((PyLongObject *)_left_o, \
(PyLongObject *)_right_o)); \
assert(_PyObject_IsUniquelyReferenced( \
PyStackRef_AsPyObjectBorrow(TARGET))); \
assert(_PyObject_IsUniquelyReferenced(_target_o)); \
STAT_INC(BINARY_OP, hit); \
Py_ssize_t _left_val = _PyLong_CompactValue( \
(PyLongObject *)_left_o); \
Py_ssize_t _right_val = _PyLong_CompactValue( \
(PyLongObject *)_right_o); \
Py_ssize_t _result = _left_val OP _right_val; \
Comment thread
eendebakpt marked this conversation as resolved.
Outdated
if (_result >= -_PY_NSMALLNEGINTS \
&& _result < _PY_NSMALLPOSINTS) { \
_int_inplace_smallref = PyStackRef_FromPyObjectBorrow( \
_PyLong_GetSmallInt(_result)); \
_int_inplace_status = 1; \
} \
else if ((twodigits)((stwodigits)_result) + PyLong_MASK \
/* Result must fit in a single digit and not be a small int */ \
if (((twodigits)((stwodigits)_result) + PyLong_MASK \
< (twodigits)PyLong_MASK + PyLong_BASE) \
&& (_result < -_PY_NSMALLNEGINTS \
|| _result >= _PY_NSMALLPOSINTS)) \
{ \
PyLongObject *_target = (PyLongObject *) \
PyStackRef_AsPyObjectBorrow(TARGET); \
PyLongObject *_target = (PyLongObject *)_target_o; \
Py_ssize_t _sign = _result < 0 ? -1 : 1; \
digit _abs = (digit)(_result < 0 ? -_result : _result); \
_target->long_value.lv_tag = \
(Py_ssize_t)_sign << _PyLong_NON_SIZE_BITS; \
_target->long_value.ob_digit[0] = _abs; \
_int_inplace_ok = 1; \
} \
else { \
_int_inplace_status = 2; \
} \
/* else: result is small int or overflow — fall back */ \
} while (0)

Loading