#include "Python.h" /* general Python API */

#include "Python-ast.h" /* mod_ty */

#include "graminit.h" /* symbols defined in the grammar */

#include "node.h" /* internal parser structure */

#include "errcode.h" /* error codes for PyNode_*() */

#include "token.h" /* token definitions */

#include "grammar.h"

#include "parsetok.h"

/* ISTERMINAL() / ISNONTERMINAL() */

#include "compile.h"

#undef Yield

#include "ast.h"

#include "pyarena.h"

extern grammar _PyParser_Grammar; /* From graminit.c */

#ifdef lint

#include <note.h>

#define NOTE(x)

static char parser_copyright_string[] =

Centrum, Amsterdam, The Netherlands.";

PyDoc_STRVAR(parser_doc_string,

"This is an interface to Python's internal parser.");

static char parser_version_string[] = "0.5";

typedef PyObject* (*SeqMaker) (Py_ssize_t length);

typedef int (*SeqInserter) (PyObject* sequence,

Py_ssize_t index,

PyObject* element);

static PyObject*

node2tuple(node *n, /* node to convert */

int col_offset) /* include column offsets? */

{

if (n == NULL) {

Py_INCREF(Py_None);

return (Py_None);

}

if (ISNONTERMINAL(TYPE(n))) {

int i;

PyObject *v;

PyObject *w;

v = mkseq(1 + NCH(n) + (TYPE(n) == encoding_decl));

if (v == NULL)

return (v);

w = PyInt_FromLong(TYPE(n));

if (w == NULL) {

Py_DECREF(v);

return ((PyObject*) NULL);

}

(void) addelem(v, 0, w);

for (i = 0; i < NCH(n); i++) {

w = node2tuple(CHILD(n, i), mkseq, addelem, lineno, col_offset);

if (w == NULL) {

Py_DECREF(v);

return ((PyObject*) NULL);

}

(void) addelem(v, i+1, w);

}

if (TYPE(n) == encoding_decl)

(void) addelem(v, i+1, PyString_FromString(STR(n)));

return (v);

}

else if (ISTERMINAL(TYPE(n))) {

PyObject *result = mkseq(2 + lineno + col_offset);

if (result != NULL) {

(void) addelem(result, 0, PyInt_FromLong(TYPE(n)));

(void) addelem(result, 1, PyString_FromString(STR(n)));

if (lineno)

(void) addelem(result, 2, PyInt_FromLong(n->n_lineno));

if (col_offset)

(void) addelem(result, 2 + lineno, PyInt_FromLong(n->n_col_offset));

}

return (result);

}

else {

PyErr_SetString(PyExc_SystemError,

"unrecognized parse tree node type");

return ((PyObject*) NULL);

}

}

#define PyST_EXPR 1

#define PyST_SUITE 2

static PyObject*

parser_error = 0;

typedef struct {

PyObject_HEAD /* standard object header */

node* st_node; /* the node* returned by the parser */

int st_type; /* EXPR or SUITE ? */

PyCompilerFlags st_flags; /* Parser and compiler flags */

} PyST_Object;

static void parser_free(PyST_Object *st);

static PyObject* parser_sizeof(PyST_Object *, void *);

static int parser_compare(PyST_Object *left, PyST_Object *right);

static PyObject *parser_getattr(PyObject *self, char *name);

static PyObject* parser_compilest(PyST_Object *, PyObject *, PyObject *);

static PyObject* parser_isexpr(PyST_Object *, PyObject *, PyObject *);

static PyObject* parser_issuite(PyST_Object *, PyObject *, PyObject *);

static PyObject* parser_st2list(PyST_Object *, PyObject *, PyObject *);

static PyObject* parser_st2tuple(PyST_Object *, PyObject *, PyObject *);

#define PUBLIC_METHOD_TYPE (METH_VARARGS|METH_KEYWORDS)

static PyMethodDef

parser_methods[] = {

{"compile", (PyCFunction)parser_compilest, PUBLIC_METHOD_TYPE,

PyDoc_STR("Compile this ST object into a code object.")},

{"isexpr", (PyCFunction)parser_isexpr, PUBLIC_METHOD_TYPE,

PyDoc_STR("Determines if this ST object was created from an expression.")},

{"issuite", (PyCFunction)parser_issuite, PUBLIC_METHOD_TYPE,

PyDoc_STR("Determines if this ST object was created from a suite.")},

{"tolist", (PyCFunction)parser_st2list, PUBLIC_METHOD_TYPE,

PyDoc_STR("Creates a list-tree representation of this ST.")},

{"totuple", (PyCFunction)parser_st2tuple, PUBLIC_METHOD_TYPE,

PyDoc_STR("Creates a tuple-tree representation of this ST.")},

{"__sizeof__", (PyCFunction)parser_sizeof, METH_NOARGS,

PyDoc_STR("Returns size in memory, in bytes.")},

{NULL, NULL, 0, NULL}

};

PyTypeObject PyST_Type = {

PyVarObject_HEAD_INIT(NULL, 0)

"parser.st", /* tp_name */

(int) sizeof(PyST_Object), /* tp_basicsize */

0, /* tp_itemsize */

(destructor)parser_free, /* tp_dealloc */

0, /* tp_print */

parser_getattr, /* tp_getattr */

0, /* tp_setattr */

(cmpfunc)parser_compare, /* tp_compare */

0, /* tp_repr */

0, /* tp_as_number */

0, /* tp_as_sequence */

0, /* tp_as_mapping */

0, /* tp_hash */

0, /* tp_call */

0, /* tp_str */

0, /* tp_getattro */

0, /* tp_setattro */

/* Functions to access object as input/output buffer */

0, /* tp_as_buffer */

Py_TPFLAGS_DEFAULT, /* tp_flags */

/* __doc__ */

"Intermediate representation of a Python parse tree.",

0, /* tp_traverse */

0, /* tp_clear */

0, /* tp_richcompare */

0, /* tp_weaklistoffset */

0, /* tp_iter */

0, /* tp_iternext */

parser_methods, /* tp_methods */

}; /* PyST_Type */

static int

parser_compare_nodes(node *left, node *right)

{

int j;

if (TYPE(left) < TYPE(right))

return (-1);

if (TYPE(right) < TYPE(left))

return (1);

if (ISTERMINAL(TYPE(left)))

return (strcmp(STR(left), STR(right)));

if (NCH(left) < NCH(right))

return (-1);

if (NCH(right) < NCH(left))

return (1);

for (j = 0; j < NCH(left); ++j) {

int v = parser_compare_nodes(CHILD(left, j), CHILD(right, j));

if (v != 0)

return (v);

}

return (0);

}

static int

parser_compare(PyST_Object *left, PyST_Object *right)

{

if (left == right)

return (0);

if ((left == 0) || (right == 0))

return (-1);

return (parser_compare_nodes(left->st_node, right->st_node));

}

static PyObject*

parser_newstobject(node *st, int type)

{

PyST_Object* o = PyObject_New(PyST_Object, &PyST_Type);

if (o != 0) {

o->st_node = st;

o->st_type = type;

o->st_flags.cf_flags = 0;

}

else {

PyNode_Free(st);

}

return ((PyObject*)o);

}

static void

parser_free(PyST_Object *st)

{

PyNode_Free(st->st_node);

PyObject_Del(st);

}

static PyObject*

parser_st2tuple(PyST_Object *self, PyObject *args, PyObject *kw)

{

PyObject *line_option = 0;

PyObject *col_option = 0;

PyObject *res = 0;

int ok;

static char *keywords[] = {"ast", "line_info", "col_info", NULL};

if (self == NULL) {

ok = PyArg_ParseTupleAndKeywords(args, kw, "O!|OO:st2tuple", keywords,

&PyST_Type, &self, &line_option,

&col_option);

}

else

ok = PyArg_ParseTupleAndKeywords(args, kw, "|OO:totuple", &keywords[1],

&line_option, &col_option);

if (ok != 0) {

int lineno = 0;

int col_offset = 0;

if (line_option != NULL) {

lineno = PyObject_IsTrue(line_option);

if (lineno < 0)

return NULL;

}

if (col_option != NULL) {

col_offset = PyObject_IsTrue(col_option);

if (col_offset < 0)

return NULL;

}

/*

res = node2tuple(((PyST_Object*)self)->st_node,

PyTuple_New, PyTuple_SetItem, lineno, col_offset);

}

return (res);

}

static PyObject*

parser_ast2tuple(PyST_Object *self, PyObject *args, PyObject *kw)

{

if (PyErr_WarnPy3k("ast2tuple is removed in 3.x; use st2tuple", 1) < 0)

return NULL;

return parser_st2tuple(self, args, kw);

}

static PyObject*

parser_st2list(PyST_Object *self, PyObject *args, PyObject *kw)

{

PyObject *line_option = 0;

PyObject *col_option = 0;

PyObject *res = 0;

int ok;

static char *keywords[] = {"ast", "line_info", "col_info", NULL};

if (self == NULL)

ok = PyArg_ParseTupleAndKeywords(args, kw, "O!|OO:st2list", keywords,

&PyST_Type, &self, &line_option,

&col_option);

else

ok = PyArg_ParseTupleAndKeywords(args, kw, "|OO:tolist", &keywords[1],

&line_option, &col_option);

if (ok) {

int lineno = 0;

int col_offset = 0;

if (line_option != 0) {

lineno = PyObject_IsTrue(line_option);

if (lineno < 0)

return NULL;

}

if (col_option != 0) {

col_offset = PyObject_IsTrue(col_option);

if (col_offset < 0)

return NULL;

}

/*

res = node2tuple(self->st_node,

PyList_New, PyList_SetItem, lineno, col_offset);

}

return (res);

}

static PyObject*

parser_ast2list(PyST_Object *self, PyObject *args, PyObject *kw)

{

if (PyErr_WarnPy3k("ast2list is removed in 3.x; use st2list", 1) < 0)

return NULL;

return parser_st2list(self, args, kw);

}

static PyObject*

parser_compilest(PyST_Object *self, PyObject *args, PyObject *kw)

{

PyObject* res = 0;

PyArena* arena;

mod_ty mod;

char* str = "<syntax-tree>";

int ok;

static char *keywords[] = {"ast", "filename", NULL};

if (self == NULL)

ok = PyArg_ParseTupleAndKeywords(args, kw, "O!|s:compilest", keywords,

&PyST_Type, &self, &str);

else

ok = PyArg_ParseTupleAndKeywords(args, kw, "|s:compile", &keywords[1],

&str);

if (ok) {

arena = PyArena_New();

if (arena) {

mod = PyAST_FromNode(self->st_node, &(self->st_flags), str, arena);

if (mod) {

res = (PyObject *)PyAST_Compile(mod, str, &(self->st_flags), arena);

}

PyArena_Free(arena);

}

}

return (res);

}

static PyObject*

parser_compileast(PyST_Object *self, PyObject *args, PyObject *kw)

{

if (PyErr_WarnPy3k("compileast is removed in 3.x; use compilest", 1) < 0)

return NULL;

return parser_compilest(self, args, kw);

}

static PyObject*

parser_isexpr(PyST_Object *self, PyObject *args, PyObject *kw)

{

PyObject* res = 0;

int ok;

static char *keywords[] = {"ast", NULL};

if (self == NULL)

ok = PyArg_ParseTupleAndKeywords(args, kw, "O!:isexpr", keywords,

&PyST_Type, &self);

else

ok = PyArg_ParseTupleAndKeywords(args, kw, ":isexpr", &keywords[1]);

if (ok) {

/* Check to see if the ST represents an expression or not. */

res = (self->st_type == PyST_EXPR) ? Py_True : Py_False;

Py_INCREF(res);

}

return (res);

}

static PyObject*

parser_issuite(PyST_Object *self, PyObject *args, PyObject *kw)

{

PyObject* res = 0;

int ok;

static char *keywords[] = {"ast", NULL};

if (self == NULL)

ok = PyArg_ParseTupleAndKeywords(args, kw, "O!:issuite", keywords,

&PyST_Type, &self);

else

ok = PyArg_ParseTupleAndKeywords(args, kw, ":issuite", &keywords[1]);

if (ok) {

/* Check to see if the ST represents an expression or not. */

res = (self->st_type == PyST_EXPR) ? Py_False : Py_True;

Py_INCREF(res);

}

return (res);

}

static PyObject*

parser_getattr(PyObject *self, char *name)

{

return (Py_FindMethod(parser_methods, self, name));

}

static void

err_string(char *message)

{

PyErr_SetString(parser_error, message);

}

static PyObject*

parser_do_parse(PyObject *args, PyObject *kw, char *argspec, int type)

{

char* string = 0;

PyObject* res = 0;

int flags = 0;

perrdetail err;

static char *keywords[] = {"source", NULL};

if (PyArg_ParseTupleAndKeywords(args, kw, argspec, keywords, &string)) {

node* n = PyParser_ParseStringFlagsFilenameEx(string, NULL,

&_PyParser_Grammar,

(type == PyST_EXPR)

? eval_input : file_input,

&err, &flags);

if (n) {

res = parser_newstobject(n, type);

if (res)

((PyST_Object *)res)->st_flags.cf_flags = flags & PyCF_MASK;

}

else

PyParser_SetError(&err);

}

return (res);

}

static PyObject*

parser_expr(PyST_Object *self, PyObject *args, PyObject *kw)

{

NOTE(ARGUNUSED(self))

return (parser_do_parse(args, kw, "s:expr", PyST_EXPR));

}

static PyObject*

parser_suite(PyST_Object *self, PyObject *args, PyObject *kw)

{

NOTE(ARGUNUSED(self))

return (parser_do_parse(args, kw, "s:suite", PyST_SUITE));

}

static node* build_node_tree(PyObject *tuple);

static int validate_expr_tree(node *tree);

static int validate_file_input(node *tree);

static int validate_encoding_decl(node *tree);

static PyObject*

parser_tuple2st(PyST_Object *self, PyObject *args, PyObject *kw)

{

NOTE(ARGUNUSED(self))

PyObject *st = 0;

PyObject *tuple;

node *tree;

static char *keywords[] = {"sequence", NULL};

if (!PyArg_ParseTupleAndKeywords(args, kw, "O:sequence2st", keywords,

&tuple))

return (0);

if (!PySequence_Check(tuple)) {

PyErr_SetString(PyExc_ValueError,

"sequence2st() requires a single sequence argument");

return (0);

}

/*

tree = build_node_tree(tuple);

if (tree != 0) {

int start_sym = TYPE(tree);

if (start_sym == eval_input) {

/* Might be an eval form. */

if (validate_expr_tree(tree))

st = parser_newstobject(tree, PyST_EXPR);

else

PyNode_Free(tree);

}

else if (start_sym == file_input) {

/* This looks like an exec form so far. */

if (validate_file_input(tree))

st = parser_newstobject(tree, PyST_SUITE);

else

PyNode_Free(tree);

}

else if (start_sym == encoding_decl) {

/* This looks like an encoding_decl so far. */

if (validate_encoding_decl(tree))

st = parser_newstobject(tree, PyST_SUITE);

else

PyNode_Free(tree);

}

else {

/* This is a fragment, at best. */

PyNode_Free(tree);

err_string("parse tree does not use a valid start symbol");

}

}

/* Make sure we raise an exception on all errors. We should never

if (st == NULL && !PyErr_Occurred())

err_string("unspecified ST error occurred");

return st;

}

static PyObject*

parser_tuple2ast(PyST_Object *self, PyObject *args, PyObject *kw)

{

if (PyErr_WarnPy3k("tuple2ast is removed in 3.x; use tuple2st", 1) < 0)

return NULL;

return parser_tuple2st(self, args, kw);

}

static PyObject *

parser_sizeof(PyST_Object *st, void *unused)

{

Py_ssize_t res;

res = _PyObject_SIZE(Py_TYPE(st)) + _PyNode_SizeOf(st->st_node);

return PyLong_FromSsize_t(res);

}

static node*

build_node_children(PyObject *tuple, node *root, int *line_num)

{

Py_ssize_t len = PyObject_Size(tuple);

Py_ssize_t i;

int err;

if (len < 0) {

return NULL;

}

for (i = 1; i < len; ++i) {

/* elem must always be a sequence, however simple */

PyObject* elem = PySequence_GetItem(tuple, i);

int ok = elem != NULL;

long type = 0;

char *strn = 0;

if (ok)

ok = PySequence_Check(elem);

if (ok) {

PyObject *temp = PySequence_GetItem(elem, 0);

if (temp == NULL)

ok = 0;

else {

ok = PyInt_Check(temp);

if (ok)

type = PyInt_AS_LONG(temp);

Py_DECREF(temp);

}

}

if (!ok) {

PyObject *err = Py_BuildValue("os", elem,

"Illegal node construct.");

PyErr_SetObject(parser_error, err);

Py_XDECREF(err);

Py_XDECREF(elem);

return NULL;

}

if (ISTERMINAL(type)) {

Py_ssize_t len = PyObject_Size(elem);

PyObject *temp;

if ((len != 2) && (len != 3)) {

err_string("terminal nodes must have 2 or 3 entries");

Py_DECREF(elem);

return NULL;

}

temp = PySequence_GetItem(elem, 1);

if (temp == NULL) {

Py_DECREF(elem);

return NULL;

}

if (!PyString_Check(temp)) {

PyErr_Format(parser_error,

"second item in terminal node must be a string,"

" found %s",

Py_TYPE(temp)->tp_name);

Py_DECREF(temp);

Py_DECREF(elem);

return NULL;

}

if (len == 3) {

PyObject *o = PySequence_GetItem(elem, 2);

if (o == NULL) {

Py_DECREF(temp);

Py_DECREF(elem);

return NULL;

}

if (PyInt_Check(o))

*line_num = PyInt_AS_LONG(o);

else {

PyErr_Format(parser_error,

"third item in terminal node must be an"

" integer, found %s",

Py_TYPE(temp)->tp_name);

Py_DECREF(o);

Py_DECREF(temp);

Py_DECREF(elem);

return NULL;

}

Py_DECREF(o);

}

len = PyString_GET_SIZE(temp) + 1;

strn = (char *)PyObject_MALLOC(len);

if (strn == NULL) {

Py_DECREF(temp);

Py_DECREF(elem);

PyErr_NoMemory();

return NULL;

}

(void) memcpy(strn, PyString_AS_STRING(temp), len);

Py_DECREF(temp);

}

else if (!ISNONTERMINAL(type)) {

/*

PyObject *err = Py_BuildValue("Os", elem, "unknown node type.");

PyErr_SetObject(parser_error, err);

Py_XDECREF(err);

Py_DECREF(elem);

return NULL;

}

err = PyNode_AddChild(root, type, strn, *line_num, 0);

if (err == E_NOMEM) {

Py_DECREF(elem);

PyObject_FREE(strn);

PyErr_NoMemory();

return NULL;

}

if (err == E_OVERFLOW) {

Py_DECREF(elem);

PyObject_FREE(strn);

PyErr_SetString(PyExc_ValueError,

"unsupported number of child nodes");

return NULL;

}

if (ISNONTERMINAL(type)) {

node* new_child = CHILD(root, i - 1);

if (new_child != build_node_children(elem, new_child, line_num)) {

Py_DECREF(elem);

return NULL;

}

}

else if (type == NEWLINE) { /* It's true: we increment the */

++(*line_num); /* line number *after* the newline! */

}

Py_DECREF(elem);

}

return root;

}

static node*

build_node_tree(PyObject *tuple)

{

node* res = 0;

PyObject *temp = PySequence_GetItem(tuple, 0);

long num = -1;

if (temp != NULL)

num = PyInt_AsLong(temp);

Py_XDECREF(temp);

if (ISTERMINAL(num)) {

/*

tuple = Py_BuildValue("os", tuple,

"Illegal syntax-tree; cannot start with terminal symbol.");

PyErr_SetObject(parser_error, tuple);

Py_XDECREF(tuple);

}

else if (ISNONTERMINAL(num)) {

/*

int line_num = 0;

PyObject *encoding = NULL;

if (num == encoding_decl) {

encoding = PySequence_GetItem(tuple, 2);

if (encoding == NULL) {

PyErr_SetString(parser_error, "missed encoding");

return NULL;

}

if (!PyString_Check(encoding)) {

PyErr_Format(parser_error,

"encoding must be a string, found %.200s",

Py_TYPE(encoding)->tp_name);

Py_DECREF(encoding);

return NULL;

}

/* tuple isn't borrowed anymore here, need to DECREF */

tuple = PySequence_GetSlice(tuple, 0, 2);

if (tuple == NULL) {

Py_DECREF(encoding);

return NULL;

}

}

res = PyNode_New(num);

if (res != NULL) {

if (res != build_node_children(tuple, res, &line_num)) {

PyNode_Free(res);

res = NULL;

}

if (res && encoding) {

Py_ssize_t len;

len = PyString_GET_SIZE(encoding) + 1;

res->n_str = (char *)PyObject_MALLOC(len);

if (res->n_str == NULL) {

PyNode_Free(res);

Py_DECREF(encoding);

Py_DECREF(tuple);

PyErr_NoMemory();

return NULL;

}

(void) memcpy(res->n_str, PyString_AS_STRING(encoding), len);

}

}

if (encoding != NULL) {

Py_DECREF(encoding);

Py_DECREF(tuple);

}

}

else {

/* The tuple is illegal -- if the number is neither TERMINAL nor

PyObject *err = Py_BuildValue("Os", tuple,

"Illegal component tuple.");

PyErr_SetObject(parser_error, err);

Py_XDECREF(err);

}

return (res);

}

static int validate_terminal(node *terminal, int type, char *string);

#define validate_ampersand(ch) validate_terminal(ch, AMPER, "&")

#define validate_circumflex(ch) validate_terminal(ch, CIRCUMFLEX, "^")

#define validate_colon(ch) validate_terminal(ch, COLON, ":")

#define validate_comma(ch) validate_terminal(ch, COMMA, ",")

#define validate_dedent(ch) validate_terminal(ch, DEDENT, "")

#define validate_equal(ch) validate_terminal(ch, EQUAL, "=")

#define validate_indent(ch) validate_terminal(ch, INDENT, (char*)NULL)

#define validate_lparen(ch) validate_terminal(ch, LPAR, "(")

#define validate_newline(ch) validate_terminal(ch, NEWLINE, (char*)NULL)

#define validate_rparen(ch) validate_terminal(ch, RPAR, ")")

#define validate_semi(ch) validate_terminal(ch, SEMI, ";")

#define validate_star(ch) validate_terminal(ch, STAR, "*")

#define validate_vbar(ch) validate_terminal(ch, VBAR, "|")

#define validate_doublestar(ch) validate_terminal(ch, DOUBLESTAR, "**")

#define validate_dot(ch) validate_terminal(ch, DOT, ".")

#define validate_at(ch) validate_terminal(ch, AT, "@")

#define validate_name(ch, str) validate_terminal(ch, NAME, str)

#define VALIDATER(n) static int validate_##n(node *tree)

VALIDATER(node); VALIDATER(small_stmt);

VALIDATER(class); VALIDATER(node);

VALIDATER(parameters); VALIDATER(suite);

VALIDATER(testlist); VALIDATER(varargslist);

VALIDATER(fpdef); VALIDATER(fplist);

VALIDATER(stmt); VALIDATER(simple_stmt);

VALIDATER(expr_stmt); VALIDATER(power);

VALIDATER(print_stmt); VALIDATER(del_stmt);

VALIDATER(return_stmt); VALIDATER(list_iter);

VALIDATER(raise_stmt); VALIDATER(import_stmt);

VALIDATER(import_name); VALIDATER(import_from);

VALIDATER(global_stmt); VALIDATER(list_if);

VALIDATER(assert_stmt); VALIDATER(list_for);

VALIDATER(exec_stmt); VALIDATER(compound_stmt);

VALIDATER(while); VALIDATER(for);

VALIDATER(try); VALIDATER(except_clause);

VALIDATER(test); VALIDATER(and_test);