#include "Python.h"

#include "pycore_ast.h" // _PyAST_GetDocString()

#include "pycore_compile.h" // _PyFuture_FromAST()

#include "pycore_pymem.h" // _PyMem_IsPtrFreed()

#include "pycore_long.h" // _PyLong_GetZero()

#include "pycore_symtable.h" // PySTEntryObject

#define NEED_OPCODE_JUMP_TABLES

#include "opcode.h" // EXTENDED_ARG

#include "wordcode_helpers.h" // instrsize()

#define DEFAULT_BLOCK_SIZE 16

#define DEFAULT_BLOCKS 8

#define DEFAULT_CODE_SIZE 128

#define DEFAULT_LNOTAB_SIZE 16

#define COMP_GENEXP 0

#define COMP_LISTCOMP 1

#define COMP_SETCOMP 2

#define COMP_DICTCOMP 3

#define STACK_USE_GUIDELINE 30

#define MAX_ALLOWED_STACK_USE (STACK_USE_GUIDELINE * 100)

#define SETUP_FINALLY 255

#define SETUP_CLEANUP 254

#define SETUP_WITH 253

#define POP_BLOCK 252

#define IS_TOP_LEVEL_AWAIT(c) ( \

(c->c_flags->cf_flags & PyCF_ALLOW_TOP_LEVEL_AWAIT) \

&& (c->u->u_ste->ste_type == ModuleBlock))

struct instr {

};

typedef struct excepthandler {

struct instr *setup;

int offset;

} ExceptHandler;

typedef struct exceptstack {

struct basicblock_ *handlers[CO_MAXBLOCKS+1];

int depth;

} ExceptStack;

#define LOG_BITS_PER_INT 5

#define MASK_LOW_LOG_BITS 31

static inline int

is_bit_set_in_table(uint32_t *table, int bitindex) {

/* Is the relevant bit set in the relevant word? */

/* 256 bits fit into 8 32-bits words.

uint32_t word = table[bitindex >> LOG_BITS_PER_INT];

return (word >> (bitindex & MASK_LOW_LOG_BITS)) & 1;

}

static inline int

is_relative_jump(struct instr *i)

{

return is_bit_set_in_table(_PyOpcode_RelativeJump, i->i_opcode);

}

static inline int

is_jump(struct instr *i)

{

return i->i_opcode >= SETUP_WITH || is_bit_set_in_table(_PyOpcode_Jump, i->i_opcode);

}

typedef struct basicblock_ {

} basicblock;

enum fblocktype { WHILE_LOOP, FOR_LOOP, TRY_EXCEPT, FINALLY_TRY, FINALLY_END,

WITH, ASYNC_WITH, HANDLER_CLEANUP, POP_VALUE, EXCEPTION_HANDLER,

ASYNC_COMPREHENSION_GENERATOR };

struct fblockinfo {

};

enum {

COMPILER_SCOPE_MODULE,

COMPILER_SCOPE_CLASS,

COMPILER_SCOPE_FUNCTION,

COMPILER_SCOPE_ASYNC_FUNCTION,

COMPILER_SCOPE_LAMBDA,

COMPILER_SCOPE_COMPREHENSION,

};

struct compiler_unit {

};

struct compiler {

};

typedef struct {

// A list of strings corresponding to name captures. It is used to track:

// - Repeated name assignments in the same pattern.

// - Different name assignments in alternatives.

// - The order of name assignments in alternatives.

PyObject *stores;

// If 0, any name captures against our subject will raise.

int allow_irrefutable;

// An array of blocks to jump to on failure. Jumping to fail_pop[i] will pop

// i items off of the stack. The end result looks like this (with each block

// falling through to the next):

// fail_pop[4]: POP_TOP

// fail_pop[3]: POP_TOP

// fail_pop[2]: POP_TOP

// fail_pop[1]: POP_TOP

// fail_pop[0]: NOP

basicblock **fail_pop;

// The current length of fail_pop.

Py_ssize_t fail_pop_size;

// The number of items on top of the stack that need to *stay* on top of the

// stack. Variable captures go beneath these. All of them will be popped on

// failure.

Py_ssize_t on_top;

} pattern_context;

static int compiler_enter_scope(struct compiler *, identifier, int, void *, int);

static void compiler_free(struct compiler *);

static basicblock *compiler_new_block(struct compiler *);

static int compiler_next_instr(basicblock *);

static int compiler_addop(struct compiler *, int);

static int compiler_addop_i(struct compiler *, int, Py_ssize_t);

static int compiler_addop_j(struct compiler *, int, basicblock *);

static int compiler_addop_j_noline(struct compiler *, int, basicblock *);

static int compiler_error(struct compiler *, const char *, ...);

static int compiler_warn(struct compiler *, const char *, ...);

static int compiler_nameop(struct compiler *, identifier, expr_context_ty);

static PyCodeObject *compiler_mod(struct compiler *, mod_ty);

static int compiler_visit_stmt(struct compiler *, stmt_ty);

static int compiler_visit_keyword(struct compiler *, keyword_ty);

static int compiler_visit_expr(struct compiler *, expr_ty);

static int compiler_augassign(struct compiler *, stmt_ty);

static int compiler_annassign(struct compiler *, stmt_ty);

static int compiler_subscript(struct compiler *, expr_ty);

static int compiler_slice(struct compiler *, expr_ty);

static int inplace_binop(operator_ty);

static int are_all_items_const(asdl_expr_seq *, Py_ssize_t, Py_ssize_t);

static int compiler_with(struct compiler *, stmt_ty, int);

static int compiler_async_with(struct compiler *, stmt_ty, int);

static int compiler_async_for(struct compiler *, stmt_ty);

static int compiler_call_helper(struct compiler *c, int n,

asdl_expr_seq *args,

asdl_keyword_seq *keywords);

static int compiler_try_except(struct compiler *, stmt_ty);

static int compiler_set_qualname(struct compiler *);

static int compiler_sync_comprehension_generator(

expr_ty elt, expr_ty val, int type);

static int compiler_async_comprehension_generator(

expr_ty elt, expr_ty val, int type);

static int compiler_pattern(struct compiler *, pattern_ty, pattern_context *);

static int compiler_match(struct compiler *, stmt_ty);

static int compiler_pattern_subpattern(struct compiler *, pattern_ty,

pattern_context *);

static void clean_basic_block(basicblock *bb);

static PyCodeObject *assemble(struct compiler *, int addNone);

static PyObject *__doc__, *__annotations__;

#define CAPSULE_NAME "compile.c compiler unit"

PyObject *

_Py_Mangle(PyObject *privateobj, PyObject *ident)

{

/* Name mangling: __private becomes _classname__private.

PyObject *result;

size_t nlen, plen, ipriv;

Py_UCS4 maxchar;

if (privateobj == NULL || !PyUnicode_Check(privateobj) ||

PyUnicode_READ_CHAR(ident, 0) != '_' ||

PyUnicode_READ_CHAR(ident, 1) != '_') {

Py_INCREF(ident);

return ident;

}

nlen = PyUnicode_GET_LENGTH(ident);

plen = PyUnicode_GET_LENGTH(privateobj);

/* Don't mangle __id__ or names with dots.

if ((PyUnicode_READ_CHAR(ident, nlen-1) == '_' &&

PyUnicode_READ_CHAR(ident, nlen-2) == '_') ||

PyUnicode_FindChar(ident, '.', 0, nlen, 1) != -1) {

Py_INCREF(ident);

return ident; /* Don't mangle __whatever__ */

}

/* Strip leading underscores from class name */

ipriv = 0;

while (PyUnicode_READ_CHAR(privateobj, ipriv) == '_')

ipriv++;

if (ipriv == plen) {

Py_INCREF(ident);

return ident; /* Don't mangle if class is just underscores */

}

plen -= ipriv;

if (plen + nlen >= PY_SSIZE_T_MAX - 1) {

PyErr_SetString(PyExc_OverflowError,

"private identifier too large to be mangled");

return NULL;

}

maxchar = PyUnicode_MAX_CHAR_VALUE(ident);

if (PyUnicode_MAX_CHAR_VALUE(privateobj) > maxchar)

maxchar = PyUnicode_MAX_CHAR_VALUE(privateobj);

result = PyUnicode_New(1 + nlen + plen, maxchar);

if (!result)

return 0;

/* ident = "_" + priv[ipriv:] + ident # i.e. 1+plen+nlen bytes */

PyUnicode_WRITE(PyUnicode_KIND(result), PyUnicode_DATA(result), 0, '_');

if (PyUnicode_CopyCharacters(result, 1, privateobj, ipriv, plen) < 0) {

Py_DECREF(result);

return NULL;

}

if (PyUnicode_CopyCharacters(result, plen+1, ident, 0, nlen) < 0) {

Py_DECREF(result);

return NULL;

}

assert(_PyUnicode_CheckConsistency(result, 1));

return result;

}

static int

compiler_init(struct compiler *c)

{

memset(c, 0, sizeof(struct compiler));

c->c_const_cache = PyDict_New();

if (!c->c_const_cache) {

return 0;

}

c->c_stack = PyList_New(0);

if (!c->c_stack) {

Py_CLEAR(c->c_const_cache);

return 0;

}

return 1;

}

PyCodeObject *

_PyAST_Compile(mod_ty mod, PyObject *filename, PyCompilerFlags *flags,

int optimize, PyArena *arena)

{

struct compiler c;

PyCodeObject *co = NULL;

PyCompilerFlags local_flags = _PyCompilerFlags_INIT;

int merged;

if (!__doc__) {

__doc__ = PyUnicode_InternFromString("__doc__");

if (!__doc__)

return NULL;

}

if (!__annotations__) {

__annotations__ = PyUnicode_InternFromString("__annotations__");

if (!__annotations__)

return NULL;

}

if (!compiler_init(&c))

return NULL;

Py_INCREF(filename);

c.c_filename = filename;

c.c_arena = arena;

c.c_future = _PyFuture_FromAST(mod, filename);

if (c.c_future == NULL)

goto finally;

if (!flags) {

flags = &local_flags;

}

merged = c.c_future->ff_features | flags->cf_flags;

c.c_future->ff_features = merged;

flags->cf_flags = merged;

c.c_flags = flags;

c.c_optimize = (optimize == -1) ? _Py_GetConfig()->optimization_level : optimize;

c.c_nestlevel = 0;

_PyASTOptimizeState state;

state.optimize = c.c_optimize;

state.ff_features = merged;

if (!_PyAST_Optimize(mod, arena, &state)) {

goto finally;

}

c.c_st = _PySymtable_Build(mod, filename, c.c_future);

if (c.c_st == NULL) {

if (!PyErr_Occurred())

PyErr_SetString(PyExc_SystemError, "no symtable");

goto finally;

}

co = compiler_mod(&c, mod);

finally:

compiler_free(&c);

assert(co || PyErr_Occurred());

return co;

}

static void

compiler_free(struct compiler *c)

{

if (c->c_st)

_PySymtable_Free(c->c_st);

if (c->c_future)

PyObject_Free(c->c_future);

Py_XDECREF(c->c_filename);

Py_DECREF(c->c_const_cache);

Py_DECREF(c->c_stack);

}

static PyObject *

list2dict(PyObject *list)

{

Py_ssize_t i, n;

PyObject *v, *k;

PyObject *dict = PyDict_New();

if (!dict) return NULL;

n = PyList_Size(list);

for (i = 0; i < n; i++) {

v = PyLong_FromSsize_t(i);

if (!v) {

Py_DECREF(dict);

return NULL;

}

k = PyList_GET_ITEM(list, i);

if (PyDict_SetItem(dict, k, v) < 0) {

Py_DECREF(v);

Py_DECREF(dict);

return NULL;

}

Py_DECREF(v);

}

return dict;

}

static PyObject *

dictbytype(PyObject *src, int scope_type, int flag, Py_ssize_t offset)

{

Py_ssize_t i = offset, scope, num_keys, key_i;

PyObject *k, *v, *dest = PyDict_New();

PyObject *sorted_keys;

assert(offset >= 0);

if (dest == NULL)

return NULL;

/* Sort the keys so that we have a deterministic order on the indexes

sorted_keys = PyDict_Keys(src);

if (sorted_keys == NULL)

return NULL;

if (PyList_Sort(sorted_keys) != 0) {

Py_DECREF(sorted_keys);

return NULL;

}

num_keys = PyList_GET_SIZE(sorted_keys);

for (key_i = 0; key_i < num_keys; key_i++) {

/* XXX this should probably be a macro in symtable.h */

long vi;

k = PyList_GET_ITEM(sorted_keys, key_i);

v = PyDict_GetItemWithError(src, k);

assert(v && PyLong_Check(v));

vi = PyLong_AS_LONG(v);

scope = (vi >> SCOPE_OFFSET) & SCOPE_MASK;

if (scope == scope_type || vi & flag) {

PyObject *item = PyLong_FromSsize_t(i);

if (item == NULL) {

Py_DECREF(sorted_keys);

Py_DECREF(dest);

return NULL;

}

i++;

if (PyDict_SetItem(dest, k, item) < 0) {

Py_DECREF(sorted_keys);

Py_DECREF(item);

Py_DECREF(dest);

return NULL;

}

Py_DECREF(item);

}

}

Py_DECREF(sorted_keys);

return dest;

}

static void

compiler_unit_check(struct compiler_unit *u)

{

basicblock *block;

for (block = u->u_blocks; block != NULL; block = block->b_list) {

assert(!_PyMem_IsPtrFreed(block));

if (block->b_instr != NULL) {

assert(block->b_ialloc > 0);

assert(block->b_iused >= 0);

assert(block->b_ialloc >= block->b_iused);

}

else {

assert (block->b_iused == 0);

assert (block->b_ialloc == 0);

}

}

}

static void

compiler_unit_free(struct compiler_unit *u)

{

basicblock *b, *next;

compiler_unit_check(u);

b = u->u_blocks;

while (b != NULL) {

if (b->b_instr)

PyObject_Free((void *)b->b_instr);

next = b->b_list;

PyObject_Free((void *)b);

b = next;

}

Py_CLEAR(u->u_ste);

Py_CLEAR(u->u_name);

Py_CLEAR(u->u_qualname);

Py_CLEAR(u->u_consts);

Py_CLEAR(u->u_names);

Py_CLEAR(u->u_varnames);

Py_CLEAR(u->u_freevars);

Py_CLEAR(u->u_cellvars);

Py_CLEAR(u->u_private);

PyObject_Free(u);

}

static int

compiler_enter_scope(struct compiler *c, identifier name,

int scope_type, void *key, int lineno)

{

struct compiler_unit *u;

basicblock *block;

u = (struct compiler_unit *)PyObject_Calloc(1, sizeof(

struct compiler_unit));

if (!u) {

PyErr_NoMemory();

return 0;

}

u->u_scope_type = scope_type;

u->u_argcount = 0;

u->u_posonlyargcount = 0;

u->u_kwonlyargcount = 0;

u->u_ste = PySymtable_Lookup(c->c_st, key);

if (!u->u_ste) {

compiler_unit_free(u);

return 0;

}

Py_INCREF(name);

u->u_name = name;

u->u_varnames = list2dict(u->u_ste->ste_varnames);

u->u_cellvars = dictbytype(u->u_ste->ste_symbols, CELL, 0, 0);

if (!u->u_varnames || !u->u_cellvars) {

compiler_unit_free(u);

return 0;

}

if (u->u_ste->ste_needs_class_closure) {

/* Cook up an implicit __class__ cell. */

_Py_IDENTIFIER(__class__);

PyObject *name;

int res;

assert(u->u_scope_type == COMPILER_SCOPE_CLASS);

assert(PyDict_GET_SIZE(u->u_cellvars) == 0);

name = _PyUnicode_FromId(&PyId___class__);

if (!name) {

compiler_unit_free(u);

return 0;

}

res = PyDict_SetItem(u->u_cellvars, name, _PyLong_GetZero());

if (res < 0) {

compiler_unit_free(u);

return 0;

}

}

u->u_freevars = dictbytype(u->u_ste->ste_symbols, FREE, DEF_FREE_CLASS,

PyDict_GET_SIZE(u->u_cellvars));

if (!u->u_freevars) {

compiler_unit_free(u);

return 0;

}

u->u_blocks = NULL;

u->u_nfblocks = 0;

u->u_firstlineno = lineno;

u->u_lineno = 0;

u->u_col_offset = 0;

u->u_end_lineno = 0;

u->u_end_col_offset = 0;

u->u_consts = PyDict_New();

if (!u->u_consts) {

compiler_unit_free(u);

return 0;

}

u->u_names = PyDict_New();

if (!u->u_names) {

compiler_unit_free(u);

return 0;

}

u->u_private = NULL;

/* Push the old compiler_unit on the stack. */

if (c->u) {

PyObject *capsule = PyCapsule_New(c->u, CAPSULE_NAME, NULL);

if (!capsule || PyList_Append(c->c_stack, capsule) < 0) {

Py_XDECREF(capsule);

compiler_unit_free(u);

return 0;

}

Py_DECREF(capsule);

u->u_private = c->u->u_private;

Py_XINCREF(u->u_private);

}

c->u = u;

c->c_nestlevel++;

block = compiler_new_block(c);

if (block == NULL)

return 0;

c->u->u_curblock = block;

if (u->u_scope_type != COMPILER_SCOPE_MODULE) {

if (!compiler_set_qualname(c))

return 0;

}

return 1;

}

static void

compiler_exit_scope(struct compiler *c)

{

// Don't call PySequence_DelItem() with an exception raised

PyObject *exc_type, *exc_val, *exc_tb;

PyErr_Fetch(&exc_type, &exc_val, &exc_tb);

c->c_nestlevel--;

compiler_unit_free(c->u);

/* Restore c->u to the parent unit. */

Py_ssize_t n = PyList_GET_SIZE(c->c_stack) - 1;

if (n >= 0) {

PyObject *capsule = PyList_GET_ITEM(c->c_stack, n);

c->u = (struct compiler_unit *)PyCapsule_GetPointer(capsule, CAPSULE_NAME);

assert(c->u);

/* we are deleting from a list so this really shouldn't fail */

if (PySequence_DelItem(c->c_stack, n) < 0) {

_PyErr_WriteUnraisableMsg("on removing the last compiler "

"stack item", NULL);

}

compiler_unit_check(c->u);

}

else {

c->u = NULL;

}

PyErr_Restore(exc_type, exc_val, exc_tb);

}

static int

compiler_set_qualname(struct compiler *c)

{

_Py_static_string(dot, ".");

_Py_static_string(dot_locals, ".<locals>");

Py_ssize_t stack_size;

struct compiler_unit *u = c->u;

PyObject *name, *base, *dot_str, *dot_locals_str;

base = NULL;

stack_size = PyList_GET_SIZE(c->c_stack);

assert(stack_size >= 1);

if (stack_size > 1) {

int scope, force_global = 0;

struct compiler_unit *parent;

PyObject *mangled, *capsule;

capsule = PyList_GET_ITEM(c->c_stack, stack_size - 1);

parent = (struct compiler_unit *)PyCapsule_GetPointer(capsule, CAPSULE_NAME);

assert(parent);

if (u->u_scope_type == COMPILER_SCOPE_FUNCTION

|| u->u_scope_type == COMPILER_SCOPE_ASYNC_FUNCTION

|| u->u_scope_type == COMPILER_SCOPE_CLASS) {

assert(u->u_name);

mangled = _Py_Mangle(parent->u_private, u->u_name);

if (!mangled)

return 0;

scope = _PyST_GetScope(parent->u_ste, mangled);

Py_DECREF(mangled);

assert(scope != GLOBAL_IMPLICIT);

if (scope == GLOBAL_EXPLICIT)

force_global = 1;

}

if (!force_global) {

if (parent->u_scope_type == COMPILER_SCOPE_FUNCTION

|| parent->u_scope_type == COMPILER_SCOPE_ASYNC_FUNCTION

|| parent->u_scope_type == COMPILER_SCOPE_LAMBDA) {

dot_locals_str = _PyUnicode_FromId(&dot_locals);

if (dot_locals_str == NULL)

return 0;

base = PyUnicode_Concat(parent->u_qualname, dot_locals_str);

if (base == NULL)

return 0;

}

else {

Py_INCREF(parent->u_qualname);

base = parent->u_qualname;

}

}

}

if (base != NULL) {

dot_str = _PyUnicode_FromId(&dot);

if (dot_str == NULL) {

Py_DECREF(base);

return 0;

}

name = PyUnicode_Concat(base, dot_str);

Py_DECREF(base);

if (name == NULL)

return 0;

PyUnicode_Append(&name, u->u_name);

if (name == NULL)

return 0;

}

else {

Py_INCREF(u->u_name);

name = u->u_name;

}

u->u_qualname = name;

return 1;

}

static basicblock *

compiler_new_block(struct compiler *c)

{

basicblock *b;

struct compiler_unit *u;

u = c->u;

b = (basicblock *)PyObject_Calloc(1, sizeof(basicblock));

if (b == NULL) {

PyErr_NoMemory();

return NULL;

}

/* Extend the singly linked list of blocks with new block. */

b->b_list = u->u_blocks;

u->u_blocks = b;

return b;

}

static basicblock *

compiler_next_block(struct compiler *c)

{

basicblock *block = compiler_new_block(c);

if (block == NULL)

return NULL;

c->u->u_curblock->b_next = block;

c->u->u_curblock = block;

return block;

}

static basicblock *

compiler_use_next_block(struct compiler *c, basicblock *block)

{

assert(block != NULL);

c->u->u_curblock->b_next = block;

c->u->u_curblock = block;

return block;

}

static basicblock *

compiler_copy_block(struct compiler *c, basicblock *block)

{

/* Cannot copy a block if it has a fallthrough, since

assert(block->b_nofallthrough);

basicblock *result = compiler_next_block(c);

if (result == NULL) {

return NULL;

}

for (int i = 0; i < block->b_iused; i++) {

int n = compiler_next_instr(result);

if (n < 0) {

return NULL;

}

result->b_instr[n] = block->b_instr[i];

}

result->b_exit = block->b_exit;

result->b_nofallthrough = 1;

return result;

}

static int

compiler_next_instr(basicblock *b)

{

assert(b != NULL);

if (b->b_instr == NULL) {

b->b_instr = (struct instr *)PyObject_Calloc(

DEFAULT_BLOCK_SIZE, sizeof(struct instr));

if (b->b_instr == NULL) {

PyErr_NoMemory();

return -1;

}

b->b_ialloc = DEFAULT_BLOCK_SIZE;

}

else if (b->b_iused == b->b_ialloc) {

struct instr *tmp;

size_t oldsize, newsize;

oldsize = b->b_ialloc * sizeof(struct instr);

newsize = oldsize << 1;

if (oldsize > (SIZE_MAX >> 1)) {

PyErr_NoMemory();

return -1;

}

if (newsize == 0) {

PyErr_NoMemory();

return -1;

}

b->b_ialloc <<= 1;

tmp = (struct instr *)PyObject_Realloc(

(void *)b->b_instr, newsize);

if (tmp == NULL) {

PyErr_NoMemory();

return -1;

}

b->b_instr = tmp;

memset((char *)b->b_instr + oldsize, 0, newsize - oldsize);

}

return b->b_iused++;

}

#define SET_LOC(c, x) \

(c)->u->u_lineno = (x)->lineno; \

(c)->u->u_col_offset = (x)->col_offset; \

(c)->u->u_end_lineno = (x)->end_lineno; \

(c)->u->u_end_col_offset = (x)->end_col_offset;

static int

stack_effect(int opcode, int oparg, int jump)

{

switch (opcode) {

case NOP:

case EXTENDED_ARG:

return 0;

/* Stack manipulation */

case POP_TOP:

return -1;

case ROT_TWO:

case ROT_THREE:

case ROT_FOUR:

return 0;

case DUP_TOP:

return 1;

case DUP_TOP_TWO:

return 2;

/* Unary operators */