#include "Python.h"

#include "opcode.h"

#include "pycore_bytesobject.h" // _PyBytes_Concat

#include "pycore_call.h" // _Py_METH_CALL_FLAGS

#include "pycore_code.h"

#include "pycore_critical_section.h"

#include "pycore_descrobject.h" // _PyMethodWrapper_Type

#include "pycore_dict.h" // DICT_KEYS_UNICODE

#include "pycore_function.h" // _PyFunction_GetVersionForCurrentState()

#include "pycore_interpframe.h" // FRAME_SPECIALS_SIZE

#include "pycore_lazyimportobject.h" // PyLazyImport_CheckExact

#include "pycore_list.h" // _PyListIterObject, _PyList_Concat

#include "pycore_tuple.h" // _PyTuple_Concat

#include "pycore_long.h" // _PyLong_IsNonNegativeCompact()

#include "pycore_moduleobject.h"

#include "pycore_object.h"

#include "pycore_opcode_metadata.h" // _PyOpcode_Caches

#include "pycore_uop_metadata.h" // _PyOpcode_uop_name

#include "pycore_uop_ids.h" // MAX_UOP_ID

#include "pycore_opcode_utils.h" // RESUME_AT_FUNC_START

#include "pycore_pylifecycle.h" // _PyOS_URandomNonblock()

#include "pycore_runtime.h" // _Py_ID()

#include "pycore_unicodeobject.h" // _PyUnicodeASCIIIter_Type

#include <stdlib.h> // rand()

#if Py_STATS

#define SPECIALIZATION_FAIL(opcode, kind) \

do { \

PyStats *s = _PyStats_GET(); \

if (s) { \

int _kind = (kind); \

assert(_kind < SPECIALIZATION_FAILURE_KINDS); \

s->opcode_stats[opcode].specialization.failure_kinds[_kind]++; \

} \

} while (0)

# define SPECIALIZATION_FAIL(opcode, kind) ((void)0)

#endif // Py_STATS

static void

fixup_getiter(_Py_CODEUNIT *instruction, int flags)

{

// Compiler can't know if types.coroutine() will be called,

// so fix up here

if (instruction->op.arg) {

if (flags & (CO_COROUTINE | CO_ITERABLE_COROUTINE)) {

instruction->op.arg = GET_ITER_YIELD_FROM_NO_CHECK;

}

else {

instruction->op.arg = GET_ITER_YIELD_FROM_CORO_CHECK;

}

}

}

_PyCode_Quicken(_Py_CODEUNIT *instructions, Py_ssize_t size, int enable_counters, int flags)

{

#if ENABLE_SPECIALIZATION

_Py_BackoffCounter jump_counter, adaptive_counter, resume_counter;

if (enable_counters) {

PyThreadState *tstate = _PyThreadState_GET();

PyInterpreterState *interp = tstate->interp;

jump_counter = initial_jump_backoff_counter(&interp->opt_config);

adaptive_counter = adaptive_counter_warmup();

resume_counter = initial_resume_backoff_counter(&interp->opt_config);

}

else {

jump_counter = initial_unreachable_backoff_counter();

adaptive_counter = initial_unreachable_backoff_counter();

resume_counter = initial_unreachable_backoff_counter();

}

int opcode = 0;

int oparg = 0;

/* The last code unit cannot have a cache, so we don't need to check it */

for (Py_ssize_t i = 0; i < size-1; i++) {

opcode = instructions[i].op.code;

int caches = _PyOpcode_Caches[opcode];

oparg = (oparg << 8) | instructions[i].op.arg;

if (opcode == GET_ITER) {

fixup_getiter(&instructions[i], flags);

}

if (caches) {

// The initial value depends on the opcode

switch (opcode) {

case JUMP_BACKWARD:

instructions[i + 1].counter = jump_counter;

break;

case RESUME:

instructions[i + 1].counter = resume_counter;

break;

case POP_JUMP_IF_FALSE:

case POP_JUMP_IF_TRUE:

case POP_JUMP_IF_NONE:

case POP_JUMP_IF_NOT_NONE:

instructions[i + 1].cache = 0x5555; // Alternating 0, 1 bits

break;

default:

instructions[i + 1].counter = adaptive_counter;

break;

}

i += caches;

}

if (opcode != EXTENDED_ARG) {

oparg = 0;

}

}

#else

for (Py_ssize_t i = 0; i < size-1; i++) {

int opcode = instructions[i].op.code;

if (opcode == GET_ITER) {

fixup_getiter(&instructions[i], flags);

}

i += _PyOpcode_Caches[opcode];

}

#endif /* ENABLE_SPECIALIZATION */

}

#define SIMPLE_FUNCTION 0

#define SPEC_FAIL_OTHER 0

#define SPEC_FAIL_NO_DICT 1

#define SPEC_FAIL_OVERRIDDEN 2

#define SPEC_FAIL_OUT_OF_VERSIONS 3

#define SPEC_FAIL_OUT_OF_RANGE 4

#define SPEC_FAIL_EXPECTED_ERROR 5

#define SPEC_FAIL_WRONG_NUMBER_ARGUMENTS 6

#define SPEC_FAIL_CODE_COMPLEX_PARAMETERS 7

#define SPEC_FAIL_CODE_NOT_OPTIMIZED 8

#define SPEC_FAIL_LOAD_GLOBAL_NON_DICT 17

#define SPEC_FAIL_LOAD_GLOBAL_NON_STRING_OR_SPLIT 18

#define SPEC_FAIL_SUPER_BAD_CLASS 9

#define SPEC_FAIL_SUPER_SHADOWED 10

#define SPEC_FAIL_ATTR_OVERRIDING_DESCRIPTOR 9

#define SPEC_FAIL_ATTR_NON_OVERRIDING_DESCRIPTOR 10

#define SPEC_FAIL_ATTR_NOT_DESCRIPTOR 11

#define SPEC_FAIL_ATTR_METHOD 12

#define SPEC_FAIL_ATTR_MUTABLE_CLASS 13

#define SPEC_FAIL_ATTR_PROPERTY 14

#define SPEC_FAIL_ATTR_NON_OBJECT_SLOT 15

#define SPEC_FAIL_ATTR_READ_ONLY 16

#define SPEC_FAIL_ATTR_AUDITED_SLOT 17

#define SPEC_FAIL_ATTR_NOT_MANAGED_DICT 18

#define SPEC_FAIL_ATTR_NON_STRING 19

#define SPEC_FAIL_ATTR_MODULE_ATTR_NOT_FOUND 20

#define SPEC_FAIL_ATTR_SHADOWED 21

#define SPEC_FAIL_ATTR_BUILTIN_CLASS_METHOD 22

#define SPEC_FAIL_ATTR_CLASS_METHOD_OBJ 23

#define SPEC_FAIL_ATTR_OBJECT_SLOT 24

#define SPEC_FAIL_ATTR_MODULE_LAZY_VALUE 25

#define SPEC_FAIL_ATTR_INSTANCE_ATTRIBUTE 26

#define SPEC_FAIL_ATTR_METACLASS_ATTRIBUTE 27

#define SPEC_FAIL_ATTR_PROPERTY_NOT_PY_FUNCTION 28

#define SPEC_FAIL_ATTR_NOT_IN_KEYS 29

#define SPEC_FAIL_ATTR_NOT_IN_DICT 30

#define SPEC_FAIL_ATTR_CLASS_ATTR_SIMPLE 31

#define SPEC_FAIL_ATTR_CLASS_ATTR_DESCRIPTOR 32

#define SPEC_FAIL_ATTR_BUILTIN_CLASS_METHOD_OBJ 33

#define SPEC_FAIL_ATTR_METACLASS_OVERRIDDEN 34

#define SPEC_FAIL_ATTR_SPLIT_DICT 35

#define SPEC_FAIL_ATTR_DESCR_NOT_DEFERRED 36

#define SPEC_FAIL_ATTR_SLOT_AFTER_ITEMS 37

#define SPEC_FAIL_SUBSCR_ARRAY_INT 9

#define SPEC_FAIL_SUBSCR_ARRAY_SLICE 10

#define SPEC_FAIL_SUBSCR_LIST_SLICE 11

#define SPEC_FAIL_SUBSCR_BUFFER_INT 12

#define SPEC_FAIL_SUBSCR_BUFFER_SLICE 13

#define SPEC_FAIL_SUBSCR_BYTEARRAY_INT 18

#define SPEC_FAIL_SUBSCR_BYTEARRAY_SLICE 19

#define SPEC_FAIL_SUBSCR_PY_SIMPLE 20

#define SPEC_FAIL_SUBSCR_PY_OTHER 21

#define SPEC_FAIL_SUBSCR_DICT_SUBCLASS_NO_OVERRIDE 22

#define SPEC_FAIL_SUBSCR_NOT_HEAP_TYPE 23

#define SPEC_FAIL_BINARY_OP_ADD_DIFFERENT_TYPES 9

#define SPEC_FAIL_BINARY_OP_ADD_OTHER 10

#define SPEC_FAIL_BINARY_OP_AND_DIFFERENT_TYPES 11

#define SPEC_FAIL_BINARY_OP_AND_INT 12

#define SPEC_FAIL_BINARY_OP_AND_OTHER 13

#define SPEC_FAIL_BINARY_OP_FLOOR_DIVIDE 14

#define SPEC_FAIL_BINARY_OP_LSHIFT 15

#define SPEC_FAIL_BINARY_OP_MATRIX_MULTIPLY 16

#define SPEC_FAIL_BINARY_OP_MULTIPLY_DIFFERENT_TYPES 17

#define SPEC_FAIL_BINARY_OP_MULTIPLY_OTHER 18

#define SPEC_FAIL_BINARY_OP_OR 19

#define SPEC_FAIL_BINARY_OP_POWER 20

#define SPEC_FAIL_BINARY_OP_REMAINDER 21

#define SPEC_FAIL_BINARY_OP_RSHIFT 22

#define SPEC_FAIL_BINARY_OP_SUBTRACT_DIFFERENT_TYPES 23

#define SPEC_FAIL_BINARY_OP_SUBTRACT_OTHER 24

#define SPEC_FAIL_BINARY_OP_TRUE_DIVIDE_DIFFERENT_TYPES 25

#define SPEC_FAIL_BINARY_OP_TRUE_DIVIDE_FLOAT 26

#define SPEC_FAIL_BINARY_OP_TRUE_DIVIDE_OTHER 27

#define SPEC_FAIL_BINARY_OP_XOR 28

#define SPEC_FAIL_BINARY_OP_OR_INT 29

#define SPEC_FAIL_BINARY_OP_OR_DIFFERENT_TYPES 30

#define SPEC_FAIL_BINARY_OP_XOR_INT 31

#define SPEC_FAIL_BINARY_OP_XOR_DIFFERENT_TYPES 32

#define SPEC_FAIL_BINARY_OP_SUBSCR 33

#define SPEC_FAIL_BINARY_OP_SUBSCR_LIST_SLICE 34

#define SPEC_FAIL_BINARY_OP_SUBSCR_TUPLE_SLICE 35

#define SPEC_FAIL_BINARY_OP_SUBSCR_STRING_SLICE 36

#define SPEC_FAIL_BINARY_OP_SUBSCR_NOT_HEAP_TYPE 37

#define SPEC_FAIL_BINARY_OP_SUBSCR_OTHER_SLICE 38

#define SPEC_FAIL_BINARY_OP_SUBSCR_MAPPINGPROXY 39

#define SPEC_FAIL_BINARY_OP_SUBSCR_RE_MATCH 40

#define SPEC_FAIL_BINARY_OP_SUBSCR_ARRAY 41

#define SPEC_FAIL_BINARY_OP_SUBSCR_DEQUE 42

#define SPEC_FAIL_BINARY_OP_SUBSCR_ENUMDICT 43

#define SPEC_FAIL_BINARY_OP_SUBSCR_STACKSUMMARY 44

#define SPEC_FAIL_BINARY_OP_SUBSCR_DEFAULTDICT 45

#define SPEC_FAIL_BINARY_OP_SUBSCR_COUNTER 46

#define SPEC_FAIL_BINARY_OP_SUBSCR_ORDEREDDICT 47

#define SPEC_FAIL_BINARY_OP_SUBSCR_BYTES 48

#define SPEC_FAIL_BINARY_OP_SUBSCR_STRUCTTIME 49

#define SPEC_FAIL_BINARY_OP_SUBSCR_RANGE 50

#define SPEC_FAIL_CALL_INSTANCE_METHOD 11

#define SPEC_FAIL_CALL_CMETHOD 12

#define SPEC_FAIL_CALL_CFUNC_VARARGS 13

#define SPEC_FAIL_CALL_CFUNC_VARARGS_KEYWORDS 14

#define SPEC_FAIL_CALL_CFUNC_NOARGS 15

#define SPEC_FAIL_CALL_CFUNC_METHOD_FASTCALL_KEYWORDS 16

#define SPEC_FAIL_CALL_METH_DESCR_VARARGS 17

#define SPEC_FAIL_CALL_METH_DESCR_VARARGS_KEYWORDS 18

#define SPEC_FAIL_CALL_METH_DESCR_METHOD_FASTCALL_KEYWORDS 19

#define SPEC_FAIL_CALL_BAD_CALL_FLAGS 20

#define SPEC_FAIL_CALL_INIT_NOT_PYTHON 21

#define SPEC_FAIL_CALL_PEP_523 22

#define SPEC_FAIL_CALL_BOUND_METHOD 23

#define SPEC_FAIL_CALL_VECTORCALL 24

#define SPEC_FAIL_CALL_CLASS_MUTABLE 26

#define SPEC_FAIL_CALL_METHOD_WRAPPER 28

#define SPEC_FAIL_CALL_OPERATOR_WRAPPER 29

#define SPEC_FAIL_CALL_INIT_NOT_SIMPLE 30

#define SPEC_FAIL_CALL_METACLASS 31

#define SPEC_FAIL_CALL_INIT_NOT_INLINE_VALUES 32

#define SPEC_FAIL_COMPARE_OP_DIFFERENT_TYPES 12

#define SPEC_FAIL_COMPARE_OP_STRING 13

#define SPEC_FAIL_COMPARE_OP_BIG_INT 14

#define SPEC_FAIL_COMPARE_OP_BYTES 15

#define SPEC_FAIL_COMPARE_OP_TUPLE 16

#define SPEC_FAIL_COMPARE_OP_LIST 17

#define SPEC_FAIL_COMPARE_OP_SET 18

#define SPEC_FAIL_COMPARE_OP_BOOL 19

#define SPEC_FAIL_COMPARE_OP_BASEOBJECT 20

#define SPEC_FAIL_COMPARE_OP_FLOAT_LONG 21

#define SPEC_FAIL_COMPARE_OP_LONG_FLOAT 22

#define SPEC_FAIL_ITER_GENERATOR 10

#define SPEC_FAIL_ITER_COROUTINE 11

#define SPEC_FAIL_ITER_ASYNC_GENERATOR 12

#define SPEC_FAIL_ITER_LIST 13

#define SPEC_FAIL_ITER_TUPLE 14

#define SPEC_FAIL_ITER_SET 15

#define SPEC_FAIL_ITER_STRING 16

#define SPEC_FAIL_ITER_BYTES 17

#define SPEC_FAIL_ITER_RANGE 18

#define SPEC_FAIL_ITER_ITERTOOLS 19

#define SPEC_FAIL_ITER_DICT_KEYS 20

#define SPEC_FAIL_ITER_DICT_ITEMS 21

#define SPEC_FAIL_ITER_DICT_VALUES 22

#define SPEC_FAIL_ITER_ENUMERATE 23

#define SPEC_FAIL_ITER_MAP 24

#define SPEC_FAIL_ITER_ZIP 25

#define SPEC_FAIL_ITER_SEQ_ITER 26

#define SPEC_FAIL_ITER_REVERSED_LIST 27

#define SPEC_FAIL_ITER_CALLABLE 28

#define SPEC_FAIL_ITER_ASCII_STRING 29

#define SPEC_FAIL_ITER_ASYNC_GENERATOR_SEND 30

#define SPEC_FAIL_ITER_SELF 31

#define SPEC_FAIL_UNPACK_SEQUENCE_ITERATOR 9

#define SPEC_FAIL_UNPACK_SEQUENCE_SEQUENCE 10

#define SPEC_FAIL_TO_BOOL_BYTEARRAY 9

#define SPEC_FAIL_TO_BOOL_BYTES 10

#define SPEC_FAIL_TO_BOOL_DICT 11

#define SPEC_FAIL_TO_BOOL_FLOAT 12

#define SPEC_FAIL_TO_BOOL_MAPPING 13

#define SPEC_FAIL_TO_BOOL_MEMORY_VIEW 14

#define SPEC_FAIL_TO_BOOL_NUMBER 15

#define SPEC_FAIL_TO_BOOL_SEQUENCE 16

#define SPEC_FAIL_TO_BOOL_SET 17

#define SPEC_FAIL_TO_BOOL_TUPLE 18

#define SPEC_FAIL_CONTAINS_OP_STR 9

#define SPEC_FAIL_CONTAINS_OP_TUPLE 10

#define SPEC_FAIL_CONTAINS_OP_LIST 11

#define SPEC_FAIL_CONTAINS_OP_USER_CLASS 12

static inline int

set_opcode(_Py_CODEUNIT *instr, uint8_t opcode)

{

#ifdef Py_GIL_DISABLED

uint8_t old_op = _Py_atomic_load_uint8_relaxed(&instr->op.code);

if (old_op >= MIN_INSTRUMENTED_OPCODE) {

/* Lost race with instrumentation */

return 0;

}

if (!_Py_atomic_compare_exchange_uint8(&instr->op.code, &old_op, opcode)) {

/* Lost race with instrumentation */

assert(old_op >= MIN_INSTRUMENTED_OPCODE);

return 0;

}

return 1;

instr->op.code = opcode;

return 1;

}

static inline void

set_counter(_Py_BackoffCounter *counter, _Py_BackoffCounter value)

{

FT_ATOMIC_STORE_UINT16_RELAXED(counter->value_and_backoff,

value.value_and_backoff);

}

static inline _Py_BackoffCounter

load_counter(_Py_BackoffCounter *counter)

{

_Py_BackoffCounter result = {

.value_and_backoff =

FT_ATOMIC_LOAD_UINT16_RELAXED(counter->value_and_backoff)};

return result;

}

static inline void

specialize(_Py_CODEUNIT *instr, uint8_t specialized_opcode)

{

assert(!PyErr_Occurred());

if (!set_opcode(instr, specialized_opcode)) {

STAT_INC(_PyOpcode_Deopt[specialized_opcode], failure);

SPECIALIZATION_FAIL(_PyOpcode_Deopt[specialized_opcode],

SPEC_FAIL_OTHER);

return;

}

STAT_INC(_PyOpcode_Deopt[specialized_opcode], success);

set_counter((_Py_BackoffCounter *)instr + 1, adaptive_counter_cooldown());

}

static inline void

unspecialize(_Py_CODEUNIT *instr)

{

assert(!PyErr_Occurred());

uint8_t opcode = FT_ATOMIC_LOAD_UINT8_RELAXED(instr->op.code);

uint8_t generic_opcode = _PyOpcode_Deopt[opcode];

STAT_INC(generic_opcode, failure);

if (!set_opcode(instr, generic_opcode)) {

SPECIALIZATION_FAIL(generic_opcode, SPEC_FAIL_OTHER);

return;

}

_Py_BackoffCounter *counter = (_Py_BackoffCounter *)instr + 1;

_Py_BackoffCounter cur = load_counter(counter);

set_counter(counter, adaptive_counter_backoff(cur));

}

static int function_kind(PyCodeObject *code);

static bool function_check_args(PyObject *o, int expected_argcount, int opcode);

static uint32_t function_get_version(PyObject *o, int opcode);

#ifdef Py_GIL_DISABLED

static void

maybe_enable_deferred_ref_count(PyObject *op)

{

if (!_Py_IsOwnedByCurrentThread(op) && _PyObject_GC_IS_TRACKED(op)) {

// For module level variables that are heavily used from multiple

// threads, deferred reference counting provides good scaling

// benefits. The downside is that the object will only be deallocated

// by a GC run.

PyUnstable_Object_EnableDeferredRefcount(op);

}

}

static int

specialize_module_load_attr_lock_held(PyDictObject *dict, _Py_CODEUNIT *instr, PyObject *name)

{

_PyAttrCache *cache = (_PyAttrCache *)(instr + 1);

if (dict->ma_keys->dk_kind != DICT_KEYS_UNICODE) {

SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_ATTR_NON_STRING);

return -1;

}

PyObject *value;

Py_ssize_t index = _PyDict_LookupIndexAndValue(dict, name, &value);

if (value != NULL && PyLazyImport_CheckExact(value)) {

SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_ATTR_MODULE_LAZY_VALUE);

return -1;

}

assert(index != DKIX_ERROR);

if (index != (uint16_t)index) {

SPECIALIZATION_FAIL(LOAD_ATTR,

index == DKIX_EMPTY ?

SPEC_FAIL_ATTR_MODULE_ATTR_NOT_FOUND :

SPEC_FAIL_OUT_OF_RANGE);

return -1;

}

uint32_t keys_version = _PyDict_GetKeysVersionForCurrentState(

_PyInterpreterState_GET(), dict);

if (keys_version == 0) {

SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_OUT_OF_VERSIONS);

return -1;

}

#ifdef Py_GIL_DISABLED

maybe_enable_deferred_ref_count(value);

write_u32(cache->version, keys_version);

cache->index = (uint16_t)index;

specialize(instr, LOAD_ATTR_MODULE);

return 0;

}

static int

specialize_module_load_attr(

PyObject *owner, _Py_CODEUNIT *instr, PyObject *name)

{

PyModuleObject *m = (PyModuleObject *)owner;

assert((Py_TYPE(owner)->tp_flags & Py_TPFLAGS_MANAGED_DICT) == 0);

PyDictObject *dict = (PyDictObject *)m->md_dict;

if (dict == NULL) {

SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_NO_DICT);

return -1;

}

int result;

Py_BEGIN_CRITICAL_SECTION(dict);

result = specialize_module_load_attr_lock_held(dict, instr, name);

Py_END_CRITICAL_SECTION();

return result;

}

Py_NO_INLINE void

_Py_Specialize_LoadSuperAttr(_PyStackRef global_super_st, _PyStackRef cls_st, _Py_CODEUNIT *instr, int load_method) {

PyObject *global_super = PyStackRef_AsPyObjectBorrow(global_super_st);

PyObject *cls = PyStackRef_AsPyObjectBorrow(cls_st);

assert(ENABLE_SPECIALIZATION);

assert(_PyOpcode_Caches[LOAD_SUPER_ATTR] == INLINE_CACHE_ENTRIES_LOAD_SUPER_ATTR);

if (global_super != (PyObject *)&PySuper_Type) {

SPECIALIZATION_FAIL(LOAD_SUPER_ATTR, SPEC_FAIL_SUPER_SHADOWED);

goto fail;

}

if (!PyType_Check(cls)) {

SPECIALIZATION_FAIL(LOAD_SUPER_ATTR, SPEC_FAIL_SUPER_BAD_CLASS);

goto fail;

}

uint8_t load_code = load_method ? LOAD_SUPER_ATTR_METHOD : LOAD_SUPER_ATTR_ATTR;

specialize(instr, load_code);

return;

fail:

unspecialize(instr);

}

typedef enum {

OVERRIDING, /* Is an overriding descriptor, and will remain so. */

METHOD, /* Attribute has Py_TPFLAGS_METHOD_DESCRIPTOR set */

PROPERTY, /* Is a property */

OBJECT_SLOT, /* Is an object slot descriptor */

OTHER_SLOT, /* Is a slot descriptor of another type */

NON_OVERRIDING, /* Is another non-overriding descriptor, and is an instance of an immutable class*/

BUILTIN_CLASSMETHOD, /* Builtin methods with METH_CLASS */

PYTHON_CLASSMETHOD, /* Python classmethod(func) object */

NON_DESCRIPTOR, /* Is not a descriptor, and is an instance of an immutable class */

MUTABLE, /* Instance of a mutable class; might, or might not, be a descriptor */

ABSENT, /* Attribute is not present on the class */

DUNDER_CLASS, /* __class__ attribute */

GETSET_OVERRIDDEN, /* __getattribute__ or __setattr__ has been overridden */

GETATTRIBUTE_IS_PYTHON_FUNCTION /* Descriptor requires calling a Python __getattribute__ */

} DescriptorClassification;

static DescriptorClassification

classify_descriptor(PyObject *descriptor, bool has_getattr)

{

if (descriptor == NULL) {

return ABSENT;

}

PyTypeObject *desc_cls = Py_TYPE(descriptor);

if (!(desc_cls->tp_flags & Py_TPFLAGS_IMMUTABLETYPE)) {

return MUTABLE;

}

if (desc_cls->tp_descr_set) {

if (desc_cls == &PyMemberDescr_Type) {

PyMemberDescrObject *member = (PyMemberDescrObject *)descriptor;

struct PyMemberDef *dmem = member->d_member;

if (dmem->type == Py_T_OBJECT_EX || dmem->type == _Py_T_OBJECT) {

return OBJECT_SLOT;

}

return OTHER_SLOT;

}

if (desc_cls == &PyProperty_Type) {

/* We can't detect at runtime whether an attribute exists

return has_getattr ? GETSET_OVERRIDDEN : PROPERTY;

}

return OVERRIDING;

}

if (desc_cls->tp_descr_get) {

if (desc_cls->tp_flags & Py_TPFLAGS_METHOD_DESCRIPTOR) {

return METHOD;

}

if (Py_IS_TYPE(descriptor, &PyClassMethodDescr_Type)) {

return BUILTIN_CLASSMETHOD;

}

if (Py_IS_TYPE(descriptor, &PyClassMethod_Type)) {

return PYTHON_CLASSMETHOD;

}

return NON_OVERRIDING;

}

return NON_DESCRIPTOR;

}

static bool

descriptor_is_class(PyObject *descriptor, PyObject *name)

{

return ((PyUnicode_CompareWithASCIIString(name, "__class__") == 0) &&

(descriptor == _PyType_Lookup(&PyBaseObject_Type, name)));

}

static DescriptorClassification

analyze_descriptor_load(PyTypeObject *type, PyObject *name, PyObject **descr, unsigned int *tp_version) {

bool has_getattr = false;

bool have_ga_version = false;

unsigned int ga_version;

getattrofunc getattro_slot = type->tp_getattro;

if (getattro_slot == PyObject_GenericGetAttr) {

/* Normal attribute lookup; */

has_getattr = false;

}

else if (getattro_slot == _Py_slot_tp_getattr_hook ||

getattro_slot == _Py_slot_tp_getattro) {

/* One or both of __getattribute__ or __getattr__ may have been

PyObject *getattribute = _PyType_LookupRefAndVersion(type,

&_Py_ID(__getattribute__), &ga_version);

have_ga_version = true;

PyInterpreterState *interp = _PyInterpreterState_GET();

bool has_custom_getattribute = getattribute != NULL &&

getattribute != interp->callable_cache.object__getattribute__;

PyObject *getattr = _PyType_Lookup(type, &_Py_ID(__getattr__));

has_getattr = getattr != NULL;

if (has_custom_getattribute) {

if (!has_getattr &&

Py_IS_TYPE(getattribute, &PyFunction_Type)) {

*descr = getattribute;

*tp_version = ga_version;

return GETATTRIBUTE_IS_PYTHON_FUNCTION;

}

/* Potentially both __getattr__ and __getattribute__ are set.

Py_DECREF(getattribute);

*descr = NULL;

*tp_version = ga_version;

return GETSET_OVERRIDDEN;

}

/* Potentially has __getattr__ but no custom __getattribute__.

Py_XDECREF(getattribute);

}

else {

*descr = NULL;

*tp_version = FT_ATOMIC_LOAD_UINT_RELAXED(type->tp_version_tag);

return GETSET_OVERRIDDEN;

}

unsigned int descr_version;

PyObject *descriptor = _PyType_LookupRefAndVersion(type, name, &descr_version);

*descr = descriptor;

*tp_version = have_ga_version ? ga_version : descr_version;

if (descriptor_is_class(descriptor, name)) {

return DUNDER_CLASS;

}

return classify_descriptor(descriptor, has_getattr);

}

static DescriptorClassification

analyze_descriptor_store(PyTypeObject *type, PyObject *name, PyObject **descr, unsigned int *tp_version)

{

if (type->tp_setattro != PyObject_GenericSetAttr) {

*descr = NULL;

return GETSET_OVERRIDDEN;

}

PyObject *descriptor = _PyType_LookupRefAndVersion(type, name, tp_version);

*descr = descriptor;

if (descriptor_is_class(descriptor, name)) {

return DUNDER_CLASS;

}

return classify_descriptor(descriptor, false);

}

static int

specialize_dict_access_inline(

PyObject *owner, _Py_CODEUNIT *instr, PyTypeObject *type,

PyObject *name, unsigned int tp_version,

int base_op, int values_op)

{

_PyAttrCache *cache = (_PyAttrCache *)(instr + 1);

PyDictKeysObject *keys = ((PyHeapTypeObject *)type)->ht_cached_keys;

assert(PyUnicode_CheckExact(name));

_Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(owner);

Py_ssize_t index = _PyDictKeys_StringLookupSplit(keys, name);

assert (index != DKIX_ERROR);

if (index == DKIX_EMPTY) {

SPECIALIZATION_FAIL(base_op, SPEC_FAIL_ATTR_NOT_IN_KEYS);

return 0;

}

assert(index >= 0);

assert(_PyObject_InlineValues(owner)->valid);

char *value_addr = (char *)&_PyObject_InlineValues(owner)->values[index];

Py_ssize_t offset = value_addr - (char *)owner;

if (offset != (uint16_t)offset) {

SPECIALIZATION_FAIL(base_op, SPEC_FAIL_OUT_OF_RANGE);

return 0;

}

cache->index = (uint16_t)offset;

write_u32(cache->version, tp_version);

specialize(instr, values_op);

return 1;

}

static int

specialize_dict_access_hint(

PyDictObject *dict, _Py_CODEUNIT *instr, PyTypeObject *type,

PyObject *name, unsigned int tp_version,

int base_op, int hint_op)

{

_PyAttrCache *cache = (_PyAttrCache *)(instr + 1);

_Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(dict);

#ifdef Py_GIL_DISABLED

_PyDict_EnsureSharedOnRead(dict);

// We found an instance with a __dict__.

if (_PyDict_HasSplitTable(dict)) {

SPECIALIZATION_FAIL(base_op, SPEC_FAIL_ATTR_SPLIT_DICT);

return 0;

}

Py_ssize_t index = _PyDict_LookupIndex(dict, name);

if (index != (uint16_t)index) {

SPECIALIZATION_FAIL(base_op,

index == DKIX_EMPTY ?

SPEC_FAIL_ATTR_NOT_IN_DICT :

SPEC_FAIL_OUT_OF_RANGE);

return 0;

}

cache->index = (uint16_t)index;

write_u32(cache->version, tp_version);

specialize(instr, hint_op);

return 1;

}

static int

specialize_dict_access(

PyObject *owner, _Py_CODEUNIT *instr, PyTypeObject *type,

DescriptorClassification kind, PyObject *name, unsigned int tp_version,

int base_op, int values_op, int hint_op)

{

assert(kind == NON_OVERRIDING || kind == NON_DESCRIPTOR || kind == ABSENT ||

kind == BUILTIN_CLASSMETHOD || kind == PYTHON_CLASSMETHOD ||

kind == METHOD);

// No descriptor, or non overriding.

if ((type->tp_flags & Py_TPFLAGS_MANAGED_DICT) == 0) {

SPECIALIZATION_FAIL(base_op, SPEC_FAIL_ATTR_NOT_MANAGED_DICT);

return 0;

}

if (type->tp_flags & Py_TPFLAGS_INLINE_VALUES &&

FT_ATOMIC_LOAD_UINT8(_PyObject_InlineValues(owner)->valid) &&

!(base_op == STORE_ATTR && _PyObject_GetManagedDict(owner) != NULL))

{

int res;

Py_BEGIN_CRITICAL_SECTION(owner);

PyDictObject *dict = _PyObject_GetManagedDict(owner);

if (dict == NULL) {

// managed dict, not materialized, inline values valid

res = specialize_dict_access_inline(owner, instr, type, name,

tp_version, base_op, values_op);

}

else {

// lost race and dict was created, fail specialization

SPECIALIZATION_FAIL(STORE_ATTR, SPEC_FAIL_OTHER);

res = 0;

}

Py_END_CRITICAL_SECTION();

return res;

}

else {

PyDictObject *dict = _PyObject_GetManagedDict(owner);

if (dict == NULL || !PyDict_CheckExact(dict)) {

SPECIALIZATION_FAIL(base_op, SPEC_FAIL_NO_DICT);

return 0;

}

int res;

Py_BEGIN_CRITICAL_SECTION(dict);

// materialized managed dict

res = specialize_dict_access_hint(dict, instr, type, name,

tp_version, base_op, hint_op);

Py_END_CRITICAL_SECTION();

return res;

}

}

static int

specialize_attr_loadclassattr(PyObject *owner, _Py_CODEUNIT *instr,

uint32_t shared_keys_version);

static int specialize_class_load_attr(PyObject* owner, _Py_CODEUNIT* instr, PyObject* name);

static bool

instance_has_key(PyObject *obj, PyObject *name, uint32_t *shared_keys_version)

{

PyTypeObject *cls = Py_TYPE(obj);

if ((cls->tp_flags & Py_TPFLAGS_MANAGED_DICT) == 0) {

return false;

}

if (cls->tp_flags & Py_TPFLAGS_INLINE_VALUES) {

PyDictKeysObject *keys = ((PyHeapTypeObject *)cls)->ht_cached_keys;

Py_ssize_t index =

_PyDictKeys_StringLookupAndVersion(keys, name, shared_keys_version);

return index >= 0;

}

PyDictObject *dict = _PyObject_GetManagedDict(obj);

if (dict == NULL || !PyDict_CheckExact(dict)) {

return false;

}

bool result;

Py_BEGIN_CRITICAL_SECTION(dict);

if (dict->ma_values) {

result = false;

}

else {

result = (_PyDict_LookupIndex(dict, name) >= 0);

}

Py_END_CRITICAL_SECTION();

return result;

}

static int

do_specialize_instance_load_attr(PyObject* owner, _Py_CODEUNIT* instr, PyObject* name,

bool shadow, uint32_t shared_keys_version,

DescriptorClassification kind, PyObject *descr, unsigned int tp_version)

{

_PyAttrCache *cache = (_PyAttrCache *)(instr + 1);

PyTypeObject *type = Py_TYPE(owner);

if (tp_version == 0) {

SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_OUT_OF_VERSIONS);

return -1;

}

uint8_t oparg = FT_ATOMIC_LOAD_UINT8_RELAXED(instr->op.arg);

switch(kind) {

case OVERRIDING:

SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_ATTR_OVERRIDING_DESCRIPTOR);

return -1;

case METHOD:

{

if (shadow) {

goto try_instance;

}

if (oparg & 1) {

if (specialize_attr_loadclassattr(owner, instr, name, descr,

tp_version, kind, true,

shared_keys_version)) {

return 0;

}

else {

return -1;

}

}

SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_ATTR_METHOD);

return -1;

}

case PROPERTY:

{

_PyLoadMethodCache *lm_cache = (_PyLoadMethodCache *)(instr + 1);

assert(Py_TYPE(descr) == &PyProperty_Type);

PyObject *fget = ((_PyPropertyObject *)descr)->prop_get;

if (fget == NULL) {

SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_EXPECTED_ERROR);

return -1;

}

if (!Py_IS_TYPE(fget, &PyFunction_Type)) {

SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_ATTR_PROPERTY_NOT_PY_FUNCTION);

return -1;

}

if (!function_check_args(fget, 1, LOAD_ATTR)) {

return -1;

}

if (oparg & 1) {

SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_ATTR_METHOD);

return -1;

}

/* Don't specialize if PEP 523 is active */

if (!_PyInterpreterState_IsSpecializationEnabled(_PyInterpreterState_GET())) {

SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_OTHER);

return -1;

}

#ifdef Py_GIL_DISABLED

if (!_PyObject_HasDeferredRefcount(fget)) {

SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_ATTR_DESCR_NOT_DEFERRED);

return -1;

}

#endif

uint32_t func_version = function_get_version(fget, LOAD_ATTR);

if (func_version == 0) {

return -1;

}

assert(tp_version != 0);

write_u32(lm_cache->type_version, tp_version);

write_u32(lm_cache->keys_version, func_version);

/* borrowed */

write_ptr(lm_cache->descr, fget);

specialize(instr, LOAD_ATTR_PROPERTY);

return 0;

}

case OBJECT_SLOT:

{

PyMemberDescrObject *member = (PyMemberDescrObject *)descr;

struct PyMemberDef *dmem = member->d_member;

Py_ssize_t offset = dmem->offset;

if (!PyObject_TypeCheck(owner, member->d_common.d_type)) {

SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_EXPECTED_ERROR);

return -1;

}

if (dmem->flags & _Py_AFTER_ITEMS) {

SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_ATTR_SLOT_AFTER_ITEMS);

return -1;

}

if (dmem->flags & Py_AUDIT_READ) {

SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_ATTR_AUDITED_SLOT);

return -1;

}

if (offset != (uint16_t)offset) {

SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_OUT_OF_RANGE);

return -1;

}

assert(dmem->type == Py_T_OBJECT_EX || dmem->type == _Py_T_OBJECT);

assert(offset > 0);

cache->index = (uint16_t)offset;

write_u32(cache->version, tp_version);

specialize(instr, LOAD_ATTR_SLOT);

return 0;

}

case DUNDER_CLASS:

{

Py_ssize_t offset = offsetof(PyObject, ob_type);

assert(offset == (uint16_t)offset);

cache->index = (uint16_t)offset;

write_u32(cache->version, tp_version);

specialize(instr, LOAD_ATTR_SLOT);

return 0;

}

case OTHER_SLOT:

SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_ATTR_NON_OBJECT_SLOT);

return -1;

case MUTABLE:

SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_ATTR_MUTABLE_CLASS);

return -1;

case GETSET_OVERRIDDEN:

SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_OVERRIDDEN);

return -1;

case GETATTRIBUTE_IS_PYTHON_FUNCTION:

{

assert(Py_IS_TYPE(descr, &PyFunction_Type));

_PyLoadMethodCache *lm_cache = (_PyLoadMethodCache *)(instr + 1);

if (!function_check_args(descr, 2, LOAD_ATTR)) {

return -1;

}

if (oparg & 1) {

SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_ATTR_METHOD);

return -1;

}

uint32_t version = function_get_version(descr, LOAD_ATTR);

if (version == 0) {

return -1;

}

/* Don't specialize if PEP 523 is active */

if (!_PyInterpreterState_IsSpecializationEnabled(_PyInterpreterState_GET())) {

SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_OTHER);

return -1;

}

#ifdef Py_GIL_DISABLED

if (!_PyObject_HasDeferredRefcount(descr)) {

SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_ATTR_DESCR_NOT_DEFERRED);

return -1;

}

#endif

write_u32(lm_cache->keys_version, version);

/* borrowed */

write_ptr(lm_cache->descr, descr);

write_u32(lm_cache->type_version, tp_version);

specialize(instr, LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN);

return 0;

}

case BUILTIN_CLASSMETHOD:

case PYTHON_CLASSMETHOD:

case NON_OVERRIDING:

if (shadow) {

goto try_instance;

}

return -1;

case NON_DESCRIPTOR:

if (shadow) {

goto try_instance;

}

if ((oparg & 1) == 0) {

if (specialize_attr_loadclassattr(owner, instr, name, descr,

tp_version, kind, false,

shared_keys_version)) {

return 0;

}

}

return -1;

case ABSENT:

if (shadow) {

goto try_instance;

}

set_counter((_Py_BackoffCounter*)instr + 1, adaptive_counter_cooldown());

return 0;

}

Py_UNREACHABLE();

try_instance:

if (specialize_dict_access(owner, instr, type, kind, name, tp_version,

LOAD_ATTR, LOAD_ATTR_INSTANCE_VALUE, LOAD_ATTR_WITH_HINT))

{

return 0;

}

return -1;

}

static int

specialize_instance_load_attr(PyObject* owner, _Py_CODEUNIT* instr, PyObject* name)

{

// 0 is not a valid version

PyObject *descr = NULL;

unsigned int tp_version = 0;

PyTypeObject *type = Py_TYPE(owner);

// Read the type version before the keys version, we could have a concurrent

// modification of the split keys in which case we update the keys version and

// then the type version, this ensures we will still deopt if that happens.

DescriptorClassification kind = analyze_descriptor_load(type, name, &descr, &tp_version);

uint32_t shared_keys_version = 0;

bool shadow = instance_has_key(owner, name, &shared_keys_version);

int result = do_specialize_instance_load_attr(owner, instr, name, shadow, shared_keys_version, kind, descr, tp_version);

Py_XDECREF(descr);

return result;

}

Py_NO_INLINE void

_Py_Specialize_LoadAttr(_PyStackRef owner_st, _Py_CODEUNIT *instr, PyObject *name)

{