Rip out the old quickening code

brandtbucher · Mar 7, 2022 · Mar 4, 2022 · Mar 4, 2022 · Mar 4, 2022 · Mar 4, 2022
commit 59ad3edaa1dadd2aa97bc400b1490cd78a56ac90
diff --git a/Include/cpython/code.h b/Include/cpython/code.h
@@ -105,7 +105,7 @@ struct PyCodeObject {
     /* Quickened instructions and cache, or NULL
      This should be treated as opaque by all code except the specializer and
      interpreter. */
-    union _cache_or_instruction *co_quickened;
+    _Py_CODEUNIT *co_quickened;
 
 };
 

diff --git a/Include/internal/pycore_code.h b/Include/internal/pycore_code.h
@@ -8,27 +8,6 @@ extern "C" {
  * Specialization and quickening structs and helper functions
  */
 
-typedef struct {
-    int32_t cache_count;
-    int32_t _; /* Force 8 byte size */
-} _PyEntryZero;
-
-
-/* Add specialized versions of entries to this union.
- *
- * Do not break the invariant: sizeof(SpecializedCacheEntry) == 8
- * Preserving this invariant is necessary because:
-    - If any one form uses more space, then all must and on 64 bit machines
-      this is likely to double the memory consumption of caches
-    - The function for calculating the offset of caches assumes a 4:1
-      cache:instruction size ratio. Changing that would need careful
-      analysis to choose a new function.
- */
-typedef union {
-    _PyEntryZero zero;
-} SpecializedCacheEntry;
-
-#define INSTRUCTIONS_PER_ENTRY (sizeof(SpecializedCacheEntry)/sizeof(_Py_CODEUNIT))
 
 /* Inline caches */
 
@@ -110,72 +89,7 @@ typedef struct {
 #define INLINE_CACHE_ENTRIES_PRECALL CACHE_ENTRIES(_PyPrecallCache)
 
 /* Maximum size of code to quicken, in code units. */
-#define MAX_SIZE_TO_QUICKEN 5000
-
-typedef union _cache_or_instruction {
-    _Py_CODEUNIT code[1];
-    SpecializedCacheEntry entry;
-} SpecializedCacheOrInstruction;
-
-/* Get pointer to the nth cache entry, from the first instruction and n.
- * Cache entries are indexed backwards, with [count-1] first in memory, and [0] last.
- * The zeroth entry immediately precedes the instructions.
- */
-static inline SpecializedCacheEntry *
-_GetSpecializedCacheEntry(const _Py_CODEUNIT *first_instr, Py_ssize_t n)
-{
-    SpecializedCacheOrInstruction *last_cache_plus_one = (SpecializedCacheOrInstruction *)first_instr;
-    assert(&last_cache_plus_one->code[0] == first_instr);
-    return &last_cache_plus_one[-1-n].entry;
-}
-
-/* Following two functions form a pair.
- *
- * oparg_from_offset_and_index() is used to compute the oparg
- * when quickening, so that offset_from_oparg_and_nexti()
- * can be used at runtime to compute the offset.
- *
- * The relationship between the three values is currently
- *     offset == (index>>1) + oparg
- * This relation is chosen based on the following observations:
- * 1. typically 1 in 4 instructions need a cache
- * 2. instructions that need a cache typically use 2 entries
- *  These observations imply:  offset ≈ index/2
- *  We use the oparg to fine tune the relation to avoid wasting space
- * and allow consecutive instructions to use caches.
- *
- * If the number of cache entries < number of instructions/2 we will waste
- * some small amoount of space.
- * If the number of cache entries > (number of instructions/2) + 255, then
- * some instructions will not be able to use a cache.
- * In practice, we expect some small amount of wasted space in a shorter functions
- * and only functions exceeding a 1000 lines or more not to have enugh cache space.
- *
- */
-static inline int
-oparg_from_offset_and_nexti(int offset, int nexti)
-{
-    return offset-(nexti>>1);
-}
-
-static inline int
-offset_from_oparg_and_nexti(int oparg, int nexti)
-{
-    return (nexti>>1)+oparg;
-}
-
-/* Get pointer to the cache entry associated with an instruction.
- * nexti is the index of the instruction plus one.
- * nexti is used as it corresponds to the instruction pointer in the interpreter.
- * This doesn't check that an entry has been allocated for that instruction. */
-static inline SpecializedCacheEntry *
-_GetSpecializedCacheEntryForInstruction(const _Py_CODEUNIT *first_instr, int nexti, int oparg)
-{
-    return _GetSpecializedCacheEntry(
-        first_instr,
-        offset_from_oparg_and_nexti(oparg, nexti)
-    );
-}
+#define MAX_SIZE_TO_QUICKEN 10000
 
 #define QUICKENING_WARMUP_DELAY 8
 

@@ -1571,10 +1571,7 @@ code_sizeof(PyCodeObject *co, PyObject *Py_UNUSED(args))
     }
 
     if (co->co_quickened != NULL) {
-        Py_ssize_t count = co->co_quickened[0].entry.zero.cache_count;
-        count += (PyBytes_GET_SIZE(co->co_code)+sizeof(SpecializedCacheEntry)-1)/
-            sizeof(SpecializedCacheEntry);
-        res += count * sizeof(SpecializedCacheEntry);
+        res += PyBytes_GET_SIZE(co->co_code);
     }
 
     return PyLong_FromSsize_t(res);

diff --git a/Python/specialize.c b/Python/specialize.c
@@ -56,11 +56,6 @@ static uint8_t adaptive_opcodes[256] = {
     [UNPACK_SEQUENCE] = UNPACK_SEQUENCE_ADAPTIVE,
 };
 
-/* The number of cache entries required for a "family" of instructions. */
-static uint8_t cache_requirements[256] = {
-    [STORE_SUBSCR] = 0,
-};
-
 Py_ssize_t _Py_QuickenedCount = 0;
 #ifdef Py_STATS
 PyStats _py_stats = { 0 };
@@ -280,137 +275,38 @@ _Py_PrintSpecializationStats(int to_file)
 #define SPECIALIZATION_FAIL(opcode, kind) ((void)0)
 #endif
 
-static SpecializedCacheOrInstruction *
-allocate(int cache_count, int instruction_count)
+static _Py_CODEUNIT *
+allocate(int instruction_count)
 {
-    assert(sizeof(SpecializedCacheOrInstruction) == 2*sizeof(int32_t));
-    assert(sizeof(SpecializedCacheEntry) == 2*sizeof(int32_t));
-    assert(cache_count > 0);
     assert(instruction_count > 0);
-    int count = cache_count + (instruction_count + INSTRUCTIONS_PER_ENTRY -1)/INSTRUCTIONS_PER_ENTRY;
-    SpecializedCacheOrInstruction *array = (SpecializedCacheOrInstruction *)
-        PyMem_Malloc(sizeof(SpecializedCacheOrInstruction) * count);
+    void *array = PyMem_Malloc(sizeof(_Py_CODEUNIT) * instruction_count);
     if (array == NULL) {
         PyErr_NoMemory();
         return NULL;
     }
     _Py_QuickenedCount++;
-    array[0].entry.zero.cache_count = cache_count;
-    return array;
-}
-
-static int
-get_cache_count(SpecializedCacheOrInstruction *quickened) {
-    return quickened[0].entry.zero.cache_count;
+    return (_Py_CODEUNIT *)array;
 }
 
-/* Return the oparg for the cache_offset and instruction index.
- *
- * If no cache is needed then return the original oparg.
- * If a cache is needed, but cannot be accessed because
- * oparg would be too large, then return -1.
- *
- * Also updates the cache_offset, as it may need to be incremented by
- * more than the cache requirements, if many instructions do not need caches.
- *
- * See pycore_code.h for details of how the cache offset,
- * instruction index and oparg are related */
-static int
-oparg_from_instruction_and_update_offset(int index, int opcode, int original_oparg, int *cache_offset) {
-    /* The instruction pointer in the interpreter points to the next
-     * instruction, so we compute the offset using nexti (index + 1) */
-    int nexti = index + 1;
-    uint8_t need = cache_requirements[opcode];
-    if (need == 0) {
-        return original_oparg;
-    }
-    assert(adaptive_opcodes[opcode] != 0);
-    int oparg = oparg_from_offset_and_nexti(*cache_offset, nexti);
-    assert(*cache_offset == offset_from_oparg_and_nexti(oparg, nexti));
-    /* Some cache space is wasted here as the minimum possible offset is (nexti>>1) */
-    if (oparg < 0) {
-        oparg = 0;
-        *cache_offset = offset_from_oparg_and_nexti(oparg, nexti);
-    }
-    else if (oparg > 255) {
-        return -1;
-    }
-    *cache_offset += need;
-    return oparg;
-}
 
-static int
-entries_needed(const _Py_CODEUNIT *code, int len)
-{
-    int cache_offset = 0;
-    int previous_opcode = -1;
-    for (int i = 0; i < len; i++) {
-        uint8_t opcode = _Py_OPCODE(code[i]);
-        if (previous_opcode != EXTENDED_ARG) {
-            oparg_from_instruction_and_update_offset(i, opcode, 0, &cache_offset);
-        }
-        previous_opcode = opcode;
-    }
-    return cache_offset + 1;   // One extra for the count entry
-}
-
-static inline _Py_CODEUNIT *
-first_instruction(SpecializedCacheOrInstruction *quickened)
-{
-    return &quickened[get_cache_count(quickened)].code[0];
-}
-
-/** Insert adaptive instructions and superinstructions.
- *
- * Skip instruction preceded by EXTENDED_ARG for adaptive
- * instructions as those are both very rare and tricky
- * to handle.
- */
+// Insert adaptive instructions and superinstructions.
 static void
-optimize(SpecializedCacheOrInstruction *quickened, int len)
+optimize(_Py_CODEUNIT *instructions, int len)
 {
-    _Py_CODEUNIT *instructions = first_instruction(quickened);
-    int cache_offset = 0;
     int previous_opcode = -1;
-    int previous_oparg = 0;
+    int previous_oparg = -1;
     for(int i = 0; i < len; i++) {
         int opcode = _Py_OPCODE(instructions[i]);
         int oparg = _Py_OPARG(instructions[i]);
         uint8_t adaptive_opcode = adaptive_opcodes[opcode];
         if (adaptive_opcode) {
-            if (_PyOpcode_InlineCacheEntries[opcode]) {
-                instructions[i] = _Py_MAKECODEUNIT(adaptive_opcode, oparg);
-                previous_opcode = -1;
-                i += _PyOpcode_InlineCacheEntries[opcode];
-            }
-            else if (previous_opcode != EXTENDED_ARG) {
-                int new_oparg = oparg_from_instruction_and_update_offset(
-                    i, opcode, oparg, &cache_offset
-                );
-                if (new_oparg < 0) {
-                    /* Not possible to allocate a cache for this instruction */
-                    previous_opcode = opcode;
-                    continue;
-                }
-                previous_opcode = adaptive_opcode;
-                // int entries_needed = cache_requirements[opcode];
-                // if (entries_needed) {
-                //     /* Initialize the adpative cache entry */
-                //     int cache0_offset = cache_offset-entries_needed;
-                //     SpecializedCacheEntry *cache =
-                //         _GetSpecializedCacheEntry(instructions, cache0_offset);
-                //     cache->adaptive.original_oparg = oparg;
-                //     cache->adaptive.counter = 0;
-                // } else {
-                //     // oparg is the adaptive cache counter
-                //     new_oparg = 0;
-                // }
-                instructions[i] = _Py_MAKECODEUNIT(adaptive_opcode, new_oparg);
-            }
+            instructions[i] = _Py_MAKECODEUNIT(adaptive_opcode, oparg);
+            i += _PyOpcode_InlineCacheEntries[opcode];
+            previous_opcode = -1;
+            previous_oparg = -1;
         }
         else {
-            /* Super instructions don't use the cache,
-             * so no need to update the offset. */
+            assert(!_PyOpcode_InlineCacheEntries[opcode]);
             switch (opcode) {
                 case JUMP_ABSOLUTE:
                     instructions[i] = _Py_MAKECODEUNIT(JUMP_ABSOLUTE_QUICK, oparg);
@@ -421,23 +317,28 @@ optimize(SpecializedCacheOrInstruction *quickened, int len)
                 case LOAD_FAST:
                     switch(previous_opcode) {
                         case LOAD_FAST:
+                            assert(0 <= previous_oparg);
                             instructions[i-1] = _Py_MAKECODEUNIT(LOAD_FAST__LOAD_FAST, previous_oparg);
                             break;
                         case STORE_FAST:
+                            assert(0 <= previous_oparg);
                             instructions[i-1] = _Py_MAKECODEUNIT(STORE_FAST__LOAD_FAST, previous_oparg);
                             break;
                         case LOAD_CONST:
+                            assert(0 <= previous_oparg);
                             instructions[i-1] = _Py_MAKECODEUNIT(LOAD_CONST__LOAD_FAST, previous_oparg);
                             break;
                     }
                     break;
                 case STORE_FAST:
                     if (previous_opcode == STORE_FAST) {
+                        assert(0 <= previous_oparg);
                         instructions[i-1] = _Py_MAKECODEUNIT(STORE_FAST__STORE_FAST, previous_oparg);
                     }
                     break;
                 case LOAD_CONST:
                     if (previous_opcode == LOAD_FAST) {
+                        assert(0 <= previous_oparg);
                         instructions[i-1] = _Py_MAKECODEUNIT(LOAD_FAST__LOAD_CONST, previous_oparg);
                     }
                     break;
@@ -446,7 +347,6 @@ optimize(SpecializedCacheOrInstruction *quickened, int len)
             previous_oparg = oparg;
         }
     }
-    assert(cache_offset+1 == get_cache_count(quickened));
 }
 
 int
@@ -460,16 +360,14 @@ _Py_Quicken(PyCodeObject *code) {
         code->co_warmup = QUICKENING_WARMUP_COLDEST;
         return 0;
     }
-    int entry_count = entries_needed(code->co_firstinstr, instr_count);
-    SpecializedCacheOrInstruction *quickened = allocate(entry_count, instr_count);
+    _Py_CODEUNIT *quickened = allocate(instr_count);
     if (quickened == NULL) {
         return -1;
     }
-    _Py_CODEUNIT *new_instructions = first_instruction(quickened);
-    memcpy(new_instructions, code->co_firstinstr, size);
+    memcpy(quickened, code->co_firstinstr, size);
     optimize(quickened, instr_count);
     code->co_quickened = quickened;
-    code->co_firstinstr = new_instructions;
+    code->co_firstinstr = quickened;
     return 0;
 }