From 5bf649f37065ac0855f55f99496adb4cce63557d Mon Sep 17 00:00:00 2001 From: Damien George Date: Tue, 13 Oct 2015 16:39:10 +0100 Subject: [PATCH 01/24] py: Expose vstr_in_blank_bytes as public. --- py/misc.h | 1 + py/vstr.c | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/py/misc.h b/py/misc.h index 79a4c1c6ef723..56ec84ee365b8 100644 --- a/py/misc.h +++ b/py/misc.h @@ -167,6 +167,7 @@ void vstr_add_byte(vstr_t *vstr, byte v); void vstr_add_char(vstr_t *vstr, unichar chr); void vstr_add_str(vstr_t *vstr, const char *str); void vstr_add_strn(vstr_t *vstr, const char *str, size_t len); +char *vstr_ins_blank_bytes(vstr_t *vstr, size_t byte_pos, size_t byte_len); void vstr_ins_byte(vstr_t *vstr, size_t byte_pos, byte b); void vstr_ins_char(vstr_t *vstr, size_t char_pos, unichar chr); void vstr_cut_head_bytes(vstr_t *vstr, size_t bytes_to_cut); diff --git a/py/vstr.c b/py/vstr.c index 3367ae581d16c..55c597bb125f0 100644 --- a/py/vstr.c +++ b/py/vstr.c @@ -256,7 +256,7 @@ void vstr_add_strn(vstr_t *vstr, const char *str, size_t len) { vstr->len += len; } -STATIC char *vstr_ins_blank_bytes(vstr_t *vstr, size_t byte_pos, size_t byte_len) { +char *vstr_ins_blank_bytes(vstr_t *vstr, size_t byte_pos, size_t byte_len) { if (vstr->had_error) { return NULL; } From 61398ab45a2ef27ce44769b4259ee97b583f2978 Mon Sep 17 00:00:00 2001 From: Damien George Date: Tue, 13 Oct 2015 16:39:46 +0100 Subject: [PATCH 02/24] py: Implement parse bytecode. --- py/compile2.c | 3343 +++++++++++++++++++++++++++++++++++++++++++++++++ py/emit.h | 4 +- py/grammar.h | 7 +- py/parse.h | 1 + py/parse2.c | 1400 +++++++++++++++++++++ py/parse2.h | 133 ++ py/py.mk | 4 +- py/scope.c | 6 +- py/scope.h | 5 +- 9 files changed, 4889 insertions(+), 14 deletions(-) create mode 100644 py/compile2.c create mode 100644 py/parse2.c create mode 100644 py/parse2.h diff --git a/py/compile2.c b/py/compile2.c new file mode 100644 index 0000000000000..9db0c2f04d6ee --- /dev/null +++ b/py/compile2.c @@ -0,0 +1,3343 @@ +/* + * This file is part of the Micro Python project, http://micropython.org/ + * + * The MIT License (MIT) + * + * Copyright (c) 2013-2015 Damien P. George + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ + +#include +#include +#include +#include +#include + +#include "py/scope.h" +#include "py/emit.h" +#include "py/compile.h" +#include "py/smallint.h" +#include "py/runtime.h" +#include "py/builtin.h" + +// TODO need to mangle __attr names + +typedef enum { +#define DEF_RULE(rule, comp, kind, ...) PN_##rule, +#include "py/grammar.h" +#undef DEF_RULE + PN_maximum_number_of, +} pn_kind_t; + +#define NEED_METHOD_TABLE MICROPY_EMIT_NATIVE + +#if NEED_METHOD_TABLE + +// we need a method table to do the lookup for the emitter functions +#define EMIT(fun) (comp->emit_method_table->fun(comp->emit)) +#define EMIT_ARG(fun, ...) (comp->emit_method_table->fun(comp->emit, __VA_ARGS__)) +#define EMIT_LOAD_FAST(qst, local_num) (comp->emit_method_table->load_id.fast(comp->emit, qst, local_num)) +#define EMIT_LOAD_GLOBAL(qst) (comp->emit_method_table->load_id.global(comp->emit, qst)) + +#else + +// if we only have the bytecode emitter enabled then we can do a direct call to the functions +#define EMIT(fun) (mp_emit_bc_##fun(comp->emit)) +#define EMIT_ARG(fun, ...) (mp_emit_bc_##fun(comp->emit, __VA_ARGS__)) +#define EMIT_LOAD_FAST(qst, local_num) (mp_emit_bc_load_fast(comp->emit, qst, local_num)) +#define EMIT_LOAD_GLOBAL(qst) (mp_emit_bc_load_global(comp->emit, qst)) + +#endif + +#define EMIT_INLINE_ASM(fun) (comp->emit_inline_asm_method_table->fun(comp->emit_inline_asm)) +#define EMIT_INLINE_ASM_ARG(fun, ...) (comp->emit_inline_asm_method_table->fun(comp->emit_inline_asm, __VA_ARGS__)) + +// elements in this struct are ordered to make it compact +typedef struct _compiler_t { + qstr source_file; + + uint8_t is_repl; + uint8_t pass; // holds enum type pass_kind_t + uint8_t func_arg_is_super; // used to compile special case of super() function call + uint8_t have_star; + + // try to keep compiler clean from nlr + mp_obj_t compile_error; // set to an exception object if there's an error + mp_uint_t compile_error_line; // set to best guess of line of error + + uint next_label; + + uint16_t num_dict_params; + uint16_t num_default_params; + + uint16_t break_label; // highest bit set indicates we are breaking out of a for loop + uint16_t continue_label; + uint16_t cur_except_level; // increased for SETUP_EXCEPT, SETUP_FINALLY; decreased for POP_BLOCK, POP_EXCEPT + uint16_t break_continue_except_level; + + mp_uint_t *co_data; + + mp_uint_t num_scopes; + scope_t **scopes; + scope_t *scope_cur; + + emit_t *emit; // current emitter + #if NEED_METHOD_TABLE + const emit_method_table_t *emit_method_table; // current emit method table + #endif + + #if MICROPY_EMIT_INLINE_THUMB + emit_inline_asm_t *emit_inline_asm; // current emitter for inline asm + const emit_inline_asm_method_table_t *emit_inline_asm_method_table; // current emit method table for inline asm + #endif +} compiler_t; + +STATIC void compile_error_set_line(compiler_t *comp, const byte *p) { + // if the line of the error is unknown then try to update it from the parse data + if (comp->compile_error_line == 0 && p != NULL && pt_is_any_rule(p)) { + mp_uint_t rule_id, src_line; + const byte *ptop; + pt_rule_extract(p, &rule_id, &src_line, &ptop); + comp->compile_error_line = src_line; + } +} + +STATIC void compile_syntax_error(compiler_t *comp, const byte *p, const char *msg) { + // only register the error if there has been no other error + if (comp->compile_error == MP_OBJ_NULL) { + comp->compile_error = mp_obj_new_exception_msg(&mp_type_SyntaxError, msg); + compile_error_set_line(comp, p); + } +} + +STATIC void compile_trailer_paren_helper(compiler_t *comp, const byte *p_arglist, bool is_method_call, int n_positional_extra); +STATIC void compile_comprehension(compiler_t *comp, const byte *p, scope_kind_t kind); +STATIC const byte *compile_node(compiler_t *comp, const byte *p); + +STATIC uint comp_next_label(compiler_t *comp) { + return comp->next_label++; +} + +STATIC void compile_increase_except_level(compiler_t *comp) { + comp->cur_except_level += 1; + if (comp->cur_except_level > comp->scope_cur->exc_stack_size) { + comp->scope_cur->exc_stack_size = comp->cur_except_level; + } +} + +STATIC void compile_decrease_except_level(compiler_t *comp) { + assert(comp->cur_except_level > 0); + comp->cur_except_level -= 1; +} + +STATIC void scope_new_and_link(compiler_t *comp, mp_uint_t scope_idx, scope_kind_t kind, const byte *p, uint emit_options) { + scope_t *scope = scope_new(kind, p, comp->source_file, emit_options); + scope->parent = comp->scope_cur; + comp->scopes[scope_idx] = scope; +} + +typedef void (*apply_list_fun_t)(compiler_t *comp, const byte *p); + +STATIC void apply_to_single_or_list(compiler_t *comp, const byte *p, pn_kind_t pn_list_kind, apply_list_fun_t f) { + if (pt_is_rule(p, pn_list_kind)) { + const byte *ptop; + p = pt_rule_extract_top(p, &ptop); + while (p != ptop) { + f(comp, p); + p = pt_next(p); + } + } else if (!pt_is_null(p)) { + f(comp, p); + } +} + +STATIC void compile_generic_all_nodes(compiler_t *comp, const byte *p, const byte *ptop) { + while (p != ptop) { + //printf("NODE: %02x %02x %02x %02x\n", p[0], p[1], p[2], p[3]); + p = compile_node(comp, p); + } +} + +STATIC void compile_load_id(compiler_t *comp, qstr qst) { + if (comp->pass == MP_PASS_SCOPE) { + mp_emit_common_get_id_for_load(comp->scope_cur, qst); + } else { + #if NEED_METHOD_TABLE + mp_emit_common_id_op(comp->emit, &comp->emit_method_table->load_id, comp->scope_cur, qst); + #else + mp_emit_common_id_op(comp->emit, &mp_emit_bc_method_table_load_id_ops, comp->scope_cur, qst); + #endif + } +} + +STATIC void compile_store_id(compiler_t *comp, qstr qst) { + if (comp->pass == MP_PASS_SCOPE) { + mp_emit_common_get_id_for_modification(comp->scope_cur, qst); + } else { + #if NEED_METHOD_TABLE + mp_emit_common_id_op(comp->emit, &comp->emit_method_table->store_id, comp->scope_cur, qst); + #else + mp_emit_common_id_op(comp->emit, &mp_emit_bc_method_table_store_id_ops, comp->scope_cur, qst); + #endif + } +} + +STATIC void compile_delete_id(compiler_t *comp, qstr qst) { + if (comp->pass == MP_PASS_SCOPE) { + mp_emit_common_get_id_for_modification(comp->scope_cur, qst); + } else { + #if NEED_METHOD_TABLE + mp_emit_common_id_op(comp->emit, &comp->emit_method_table->delete_id, comp->scope_cur, qst); + #else + mp_emit_common_id_op(comp->emit, &mp_emit_bc_method_table_delete_id_ops, comp->scope_cur, qst); + #endif + } +} + +STATIC void c_tuple(compiler_t *comp, const byte *p, const byte *p_list, const byte *p_list_top) { + int total = 0; + if (p != NULL) { + compile_node(comp, p); + total += 1; + } + while (p_list != p_list_top) { + p_list = compile_node(comp, p_list); + total += 1; + } + EMIT_ARG(build_tuple, total); +} + +STATIC void compile_generic_tuple(compiler_t *comp, const byte *p, const byte *ptop) { + // a simple tuple expression + c_tuple(comp, NULL, p, ptop); +} + +STATIC bool node_is_const_false(const byte *p) { + return pt_is_tok(p, MP_TOKEN_KW_FALSE) + || (pt_is_small_int(p) && pt_small_int_value(p) == 0); +} + +STATIC bool node_is_const_true(const byte *p) { + return pt_is_tok(p, MP_TOKEN_KW_TRUE) + || (pt_is_small_int(p) && pt_small_int_value(p) != 0); +} + +STATIC const byte *c_if_cond(compiler_t *comp, const byte *p, bool jump_if, int label) { + if (node_is_const_false(p)) { + if (jump_if == false) { + EMIT_ARG(jump, label); + } + return pt_next(p); + } else if (node_is_const_true(p)) { + if (jump_if == true) { + EMIT_ARG(jump, label); + } + return pt_next(p); + } else if (pt_is_any_rule(p)) { + const byte *ptop; + const byte *p2 = pt_rule_extract_top(p, &ptop); + if (pt_is_rule(p, PN_or_test)) { + if (jump_if == false) { + and_or_logic1:; + uint label2 = comp_next_label(comp); + while (pt_next(p2) != ptop) { + p2 = c_if_cond(comp, p2, !jump_if, label2); + } + p2 = c_if_cond(comp, p2, jump_if, label); + EMIT_ARG(label_assign, label2); + } else { + and_or_logic2: + while (p2 != ptop) { + p2 = c_if_cond(comp, p2, jump_if, label); + } + } + return p2; + } else if (pt_is_rule(p, PN_and_test)) { + if (jump_if == false) { + goto and_or_logic2; + } else { + goto and_or_logic1; + } + } else if (pt_is_rule(p, PN_not_test_2)) { + return c_if_cond(comp, p2, !jump_if, label); + } else if (pt_is_rule(p, PN_atom_paren)) { + // cond is something in parenthesis + if (pt_is_rule_empty(p)) { + // empty tuple, acts as false for the condition + if (jump_if == false) { + EMIT_ARG(jump, label); + } + } else if (pt_is_rule(pt_rule_first(p), PN_testlist_comp)) { + // non-empty tuple, acts as true for the condition + if (jump_if == true) { + EMIT_ARG(jump, label); + } + } else { + // parenthesis around 1 item, is just that item + c_if_cond(comp, pt_rule_first(p), jump_if, label); + } + return pt_next(p); + } + } + + // nothing special, fall back to default compiling for node and jump + p = compile_node(comp, p); + EMIT_ARG(pop_jump_if, jump_if, label); + return p; +} + +typedef enum { ASSIGN_STORE, ASSIGN_AUG_LOAD, ASSIGN_AUG_STORE } assign_kind_t; +STATIC void c_assign(compiler_t *comp, const byte *p, assign_kind_t kind); + +STATIC void c_assign_power(compiler_t *comp, const byte *p_orig, assign_kind_t assign_kind) { + const byte *ptop; + const byte *p0 = pt_rule_extract_top(p_orig, &ptop); + + if (assign_kind != ASSIGN_AUG_STORE) { + compile_node(comp, p0); + } + + const byte *p1 = pt_next(p0); + + if (pt_is_null_with_top(p1, ptop)) { + cannot_assign: + compile_syntax_error(comp, p_orig, "can't assign to expression"); + return; + } + + if (pt_is_rule(p1, PN_power_trailers)) { + const byte *p1top; + p1 = pt_rule_extract_top(p1, &p1top); + for (;;) { + const byte *p1next = pt_next(p1); + if (p1next >= p1top) { + break; + } + if (assign_kind != ASSIGN_AUG_STORE) { + compile_node(comp, p1); + } + p1 = p1next; + } + // p1 now points to final trailer for store + } + + if (pt_is_rule(p1, PN_trailer_bracket)) { + if (assign_kind == ASSIGN_AUG_STORE) { + EMIT(rot_three); + EMIT(store_subscr); + } else { + compile_node(comp, pt_rule_first(p1)); + if (assign_kind == ASSIGN_AUG_LOAD) { + EMIT(dup_top_two); + EMIT(load_subscr); + } else { + EMIT(store_subscr); + } + } + } else if (pt_is_rule(p1, PN_trailer_period)) { + qstr attr; + pt_extract_id(pt_rule_first(p1), &attr); + if (assign_kind == ASSIGN_AUG_LOAD) { + EMIT(dup_top); + EMIT_ARG(load_attr, attr); + } else { + if (assign_kind == ASSIGN_AUG_STORE) { + EMIT(rot_two); + } + EMIT_ARG(store_attr, attr); + } + } else { + goto cannot_assign; + } + + if (!pt_is_null_with_top(pt_next(p1), ptop)) { + goto cannot_assign; + } +} + +// we need to allow for a caller passing in 1 initial node followed by an array of nodes +STATIC void c_assign_tuple(compiler_t *comp, const byte *p_head, const byte *p_tail, const byte *p_tail_top) { + uint num_head = (p_head == NULL) ? 0 : 1; + uint num_tail = pt_num_nodes(p_tail, p_tail_top); + + // look for star expression + const byte *p_star = NULL; + if (num_head != 0 && pt_is_rule(p_head, PN_star_expr)) { + EMIT_ARG(unpack_ex, 0, num_tail); + p_star = p_head; + } + uint i = 0; + for (const byte *p = p_tail; p != p_tail_top; p = pt_next(p), ++i) { + if (pt_is_rule(p, PN_star_expr)) { + if (p_star == NULL) { + EMIT_ARG(unpack_ex, num_head + i, num_tail - i - 1); + p_star = p; + } else { + compile_syntax_error(comp, p, "multiple *x in assignment"); + return; + } + } + } + if (p_star == NULL) { + EMIT_ARG(unpack_sequence, num_head + num_tail); + } + if (num_head != 0) { + if (p_head == p_star) { + c_assign(comp, pt_rule_first(p_head), ASSIGN_STORE); + } else { + c_assign(comp, p_head, ASSIGN_STORE); + } + } + for (const byte *p = p_tail; p != p_tail_top; p = pt_next(p)) { + if (p == p_star) { + c_assign(comp, pt_rule_first(p), ASSIGN_STORE); + } else { + c_assign(comp, p, ASSIGN_STORE); + } + } +} + +// assigns top of stack to pn +STATIC void c_assign(compiler_t *comp, const byte *p, assign_kind_t assign_kind) { + tail_recursion: + assert(!pt_is_null(p)); + if (pt_is_any_id(p)) { + qstr arg; + p = pt_extract_id(p, &arg); + switch (assign_kind) { + case ASSIGN_STORE: + case ASSIGN_AUG_STORE: + compile_store_id(comp, arg); + break; + case ASSIGN_AUG_LOAD: + default: + compile_load_id(comp, arg); + break; + } + } else if (!pt_is_any_rule(p)) { + compile_syntax_error(comp, p, "can't assign to literal"); + } else { + switch (pt_rule_extract_rule_id(p)) { + case PN_power: + // lhs is an index or attribute + c_assign_power(comp, p, assign_kind); + break; + + case PN_testlist_star_expr: + case PN_exprlist: { + // lhs is a tuple + if (assign_kind != ASSIGN_STORE) { + goto bad_aug; + } + const byte *ptop; + const byte *p0 = pt_rule_extract_top(p, &ptop); + c_assign_tuple(comp, NULL, p0, ptop); + break; + } + + case PN_atom_paren: { + // lhs is something in parenthesis + const byte *ptop; + const byte *p0 = pt_rule_extract_top(p, &ptop); + if (pt_is_null_with_top(p0, ptop)) { + // empty tuple + goto cannot_assign; + } else if (pt_is_rule(p0, PN_testlist_comp)) { + if (assign_kind != ASSIGN_STORE) { + goto bad_aug; + } + p = p0; + goto testlist_comp; + } else { + // parenthesis around 1 item, is just that item + p = p0; + goto tail_recursion; + } + break; + } + + case PN_atom_bracket: { + // lhs is something in brackets + if (assign_kind != ASSIGN_STORE) { + goto bad_aug; + } + const byte *ptop; + const byte *p0 = pt_rule_extract_top(p, &ptop); // skip rule header + if (pt_is_null_with_top(p0, ptop)) { + // empty list, assignment allowed + c_assign_tuple(comp, NULL, NULL, NULL); + } else if (pt_is_rule(p0, PN_testlist_comp)) { + p = p0; + goto testlist_comp; + } else { + // brackets around 1 item + c_assign_tuple(comp, p0, NULL, NULL); + } + break; + } + + default: + goto cannot_assign; + } + return; + + testlist_comp:; + // lhs is a sequence + const byte *ptop; + const byte *p0 = pt_rule_extract_top(p, &ptop); + const byte *p1 = pt_next(p0); + if (pt_is_rule(p1, PN_testlist_comp_3b)) { + // sequence of one item, with trailing comma + assert(pt_is_rule_empty(p1)); + c_assign_tuple(comp, p0, NULL, NULL); + } else if (pt_is_rule(p1, PN_testlist_comp_3c)) { + // sequence of many items + p1 = pt_rule_extract_top(p1, &ptop); + c_assign_tuple(comp, p0, p1, ptop); + } else if (pt_is_rule(p1, PN_comp_for)) { + // TODO can we ever get here? can it be compiled? + goto cannot_assign; + } else { + // sequence with 2 items + c_assign_tuple(comp, NULL, p0, pt_next(p1)); + } + } + return; + + cannot_assign: + compile_syntax_error(comp, p, "can't assign to expression"); + return; + + bad_aug: + compile_syntax_error(comp, p, "illegal expression for augmented assignment"); +} + +// stuff for lambda and comprehensions and generators: +// if n_pos_defaults > 0 then there is a tuple on the stack with the positional defaults +// if n_kw_defaults > 0 then there is a dictionary on the stack with the keyword defaults +// if both exist, the tuple is above the dictionary (ie the first pop gets the tuple) +STATIC void close_over_variables_etc(compiler_t *comp, scope_t *this_scope, int n_pos_defaults, int n_kw_defaults) { + assert(n_pos_defaults >= 0); + assert(n_kw_defaults >= 0); + + // set flags + if (n_kw_defaults > 0) { + this_scope->scope_flags |= MP_SCOPE_FLAG_DEFKWARGS; + } + this_scope->num_def_pos_args = n_pos_defaults; + + // make closed over variables, if any + // ensure they are closed over in the order defined in the outer scope (mainly to agree with CPython) + int nfree = 0; + if (comp->scope_cur->kind != SCOPE_MODULE) { + for (int i = 0; i < comp->scope_cur->id_info_len; i++) { + id_info_t *id = &comp->scope_cur->id_info[i]; + if (id->kind == ID_INFO_KIND_CELL || id->kind == ID_INFO_KIND_FREE) { + for (int j = 0; j < this_scope->id_info_len; j++) { + id_info_t *id2 = &this_scope->id_info[j]; + if (id2->kind == ID_INFO_KIND_FREE && id->qst == id2->qst) { + // in Micro Python we load closures using LOAD_FAST + EMIT_LOAD_FAST(id->qst, id->local_num); + nfree += 1; + } + } + } + } + } + + // make the function/closure + if (nfree == 0) { + EMIT_ARG(make_function, this_scope, n_pos_defaults, n_kw_defaults); + } else { + EMIT_ARG(make_closure, this_scope, nfree, n_pos_defaults, n_kw_defaults); + } +} + +STATIC void compile_funcdef_lambdef_param(compiler_t *comp, const byte *p) { + const byte *p_orig = p; + + if (pt_is_rule(p, PN_typedargslist_star) + || pt_is_rule(p, PN_varargslist_star)) { + comp->have_star = true; + /* don't need to distinguish bare from named star + mp_parse_node_struct_t *pns = (mp_parse_node_struct_t*)pn; + if (MP_PARSE_NODE_IS_NULL(pns->nodes[0])) { + // bare star + } else { + // named star + } + */ + + } else if (pt_is_rule(p, PN_typedargslist_dbl_star) + || pt_is_rule(p, PN_varargslist_dbl_star)) { + // named double star + // TODO do we need to do anything with this? + + } else { + const byte *p_id; + const byte *p_colon = NULL; + const byte *p_equal = NULL; + if (pt_is_any_id(p)) { + // this parameter is just an id + + p_id = p; + + } else if (pt_is_rule(p, PN_typedargslist_name)) { + // this parameter has a colon and/or equal specifier + + const byte *ptop; + p = pt_rule_extract_top(p, &ptop); + + p_id = p; + p = pt_next(p); + if (p != ptop) { + p_colon = p; + p = pt_next(p); + if (p != ptop) { + p_equal = p; + } + } + + } else { + assert(pt_is_rule(p, PN_varargslist_name)); // should be + // this parameter has an equal specifier + + p_id = pt_rule_first(p); + p_equal = pt_next(p_id); + } + + qstr q_id; + pt_extract_id(p_id, &q_id); + + if (p_equal == NULL || pt_is_null(p_equal)) { + // this parameter does not have a default value + + // check for non-default parameters given after default parameters (allowed by parser, but not syntactically valid) + if (!comp->have_star && comp->num_default_params != 0) { + compile_syntax_error(comp, p_orig, "non-default argument follows default argument"); + return; + } + + } else { + // this parameter has a default value + // in CPython, None (and True, False?) as default parameters are loaded with LOAD_NAME; don't understandy why + + if (comp->have_star) { + comp->num_dict_params += 1; + // in Micro Python we put the default dict parameters into a dictionary using the bytecode + if (comp->num_dict_params == 1) { + // in Micro Python we put the default positional parameters into a tuple using the bytecode + // we need to do this here before we start building the map for the default keywords + if (comp->num_default_params > 0) { + EMIT_ARG(build_tuple, comp->num_default_params); + } else { + EMIT(load_null); // sentinel indicating empty default positional args + } + // first default dict param, so make the map + EMIT_ARG(build_map, 0); + } + + // compile value then key, then store it to the dict + compile_node(comp, p_equal); + EMIT_ARG(load_const_str, q_id); + EMIT(store_map); + } else { + comp->num_default_params += 1; + compile_node(comp, p_equal); + } + } + + // TODO p_colon not implemented + (void)p_colon; + } +} + +STATIC void compile_funcdef_lambdef(compiler_t *comp, scope_t *scope, const byte *p, pn_kind_t pn_list_kind) { + // compile default parameters + comp->have_star = false; + comp->num_dict_params = 0; + comp->num_default_params = 0; + apply_to_single_or_list(comp, p, pn_list_kind, compile_funcdef_lambdef_param); + + if (comp->compile_error != MP_OBJ_NULL) { + return; + } + + // in Micro Python we put the default positional parameters into a tuple using the bytecode + // the default keywords args may have already made the tuple; if not, do it now + if (comp->num_default_params > 0 && comp->num_dict_params == 0) { + EMIT_ARG(build_tuple, comp->num_default_params); + EMIT(load_null); // sentinel indicating empty default keyword args + } + + // make the function + close_over_variables_etc(comp, scope, comp->num_default_params, comp->num_dict_params); +} + +// leaves function object on stack +// returns function name +STATIC qstr compile_funcdef_helper(compiler_t *comp, const byte *p, uint emit_options) { + mp_int_t scope_idx; + p = pt_get_small_int(p, &scope_idx); + + if (comp->pass == MP_PASS_SCOPE) { + // create a new scope for this function + scope_new_and_link(comp, scope_idx, SCOPE_FUNCTION, p, emit_options); + } + + p = pt_next(p); // skip function name + + // get the scope for this function + scope_t *fscope = comp->scopes[scope_idx]; + + // compile the function definition + compile_funcdef_lambdef(comp, fscope, p, PN_typedargslist); + + // return its name (the 'f' in "def f(...):") + return fscope->simple_name; +} + +// leaves class object on stack +// returns class name +STATIC qstr compile_classdef_helper(compiler_t *comp, const byte *p, uint emit_options) { + mp_int_t scope_idx; + p = pt_get_small_int(p, &scope_idx); + + if (comp->pass == MP_PASS_SCOPE) { + // create a new scope for this class + scope_new_and_link(comp, scope_idx, SCOPE_CLASS, p, emit_options); + } + + EMIT(load_build_class); + + // scope for this class + scope_t *cscope = comp->scopes[scope_idx]; + + // compile the class + close_over_variables_etc(comp, cscope, 0, 0); + + // get its name + EMIT_ARG(load_const_str, cscope->simple_name); + + // second node has parent classes, if any + // empty parenthesis (eg class C():) gets here as an empty PN_classdef_2 and needs special handling + const byte *p_parents = pt_next(p); + if (pt_is_rule(p_parents, PN_classdef_2)) { + p_parents = NULL; + } + comp->func_arg_is_super = false; + compile_trailer_paren_helper(comp, p_parents, false, 2); + + // return its name (the 'C' in class C(...):") + return cscope->simple_name; +} + +// returns true if it was a built-in decorator (even if the built-in had an error) +STATIC bool compile_built_in_decorator(compiler_t *comp, const byte *p, const byte *ptop, uint *emit_options) { + qstr qst; + p = pt_extract_id(p, &qst); + if (qst != MP_QSTR_micropython) { + return false; + } + + if (p >= ptop || pt_next(p) != ptop) { + compile_syntax_error(comp, NULL, "invalid micropython decorator"); + return true; + } + + qstr attr; + p = pt_extract_id(p, &attr); + if (attr == MP_QSTR_bytecode) { + *emit_options = MP_EMIT_OPT_BYTECODE; +#if MICROPY_EMIT_NATIVE + } else if (attr == MP_QSTR_native) { + *emit_options = MP_EMIT_OPT_NATIVE_PYTHON; + } else if (attr == MP_QSTR_viper) { + *emit_options = MP_EMIT_OPT_VIPER; +#endif +#if MICROPY_EMIT_INLINE_THUMB + } else if (attr == MP_QSTR_asm_thumb) { + *emit_options = MP_EMIT_OPT_ASM_THUMB; +#endif + } else { + compile_syntax_error(comp, NULL, "invalid micropython decorator"); + } + + return true; +} + +STATIC void compile_decorated(compiler_t *comp, const byte *p, const byte *ptop) { + // get the list of decorators + ptop = mp_parse_node_extract_list(&p, PN_decorators); + + // inherit emit options for this function/class definition + uint emit_options = comp->scope_cur->emit_options; + + // compile each decorator + int num_non_built_in_decorators = 0; + while (p != ptop) { + assert(pt_is_rule(p, PN_decorator)); // should be + + const byte *ptop_decorator; + p = pt_rule_extract_top(p, &ptop_decorator); + + // first node contains the decorator function, which is a dotted name + const byte *ptop_dotted_name = mp_parse_node_extract_list(&p, PN_dotted_name); + + // check for built-in decorators + if (compile_built_in_decorator(comp, p, ptop_dotted_name, &emit_options)) { + // this was a built-in + + } else { + // not a built-in, compile normally + + num_non_built_in_decorators += 1; + + // compile the decorator function + p = compile_node(comp, p); + while (p != ptop_dotted_name) { + assert(pt_is_any_id(p)); // should be + qstr qst; + p = pt_extract_id(p, &qst); + EMIT_ARG(load_attr, qst); + } + + // nodes[1] contains arguments to the decorator function, if any + if (!pt_is_null_with_top(p, ptop_decorator)) { + // call the decorator function with the arguments in nodes[1] + comp->func_arg_is_super = false; + compile_node(comp, p); + } + } + + p = ptop_decorator; + } + + // compile the body (funcdef or classdef) and get its name + qstr body_name = 0; + p = pt_rule_first(ptop); // skip the rule header + if (pt_is_rule(ptop, PN_funcdef)) { + body_name = compile_funcdef_helper(comp, p, emit_options); + } else { + assert(pt_is_rule(ptop, PN_classdef)); // should be + body_name = compile_classdef_helper(comp, p, emit_options); + } + + // call each decorator + while (num_non_built_in_decorators-- > 0) { + EMIT_ARG(call_function, 1, 0, 0); + } + + // store func/class object into name + compile_store_id(comp, body_name); +} + +STATIC void compile_funcdef(compiler_t *comp, const byte *p, const byte *ptop) { + (void)ptop; + qstr fname = compile_funcdef_helper(comp, p, comp->scope_cur->emit_options); + // store function object into function name + compile_store_id(comp, fname); +} + +STATIC void c_del_stmt(compiler_t *comp, const byte *p) { + if (pt_is_any_id(p)) { + qstr id; + pt_extract_id(p, &id); + compile_delete_id(comp, id); + } else if (pt_is_rule(p, PN_power)) { + const byte *ptop; + const byte *p0 = pt_rule_extract_top(p, &ptop); + + const byte *p1 = compile_node(comp, p0); // base of the power node + + if (pt_is_rule(p1, PN_power_trailers)) { + const byte *p1top; + p1 = pt_rule_extract_top(p1, &p1top); + for (;;) { + const byte *p1next = pt_next(p1); + if (p1next == p1top) { + break; + } + compile_node(comp, p1); + p1 = p1next; + } + // p1 now points to final trailer for delete + } + + const byte *p2; + if (pt_is_rule(p1, PN_trailer_bracket)) { + p2 = compile_node(comp, pt_rule_first(p1)); + EMIT(delete_subscr); + } else if (pt_is_rule(p1, PN_trailer_period)) { + qstr id; + p2 = pt_extract_id(pt_rule_first(p1), &id); + EMIT_ARG(delete_attr, id); + } else { + goto cannot_delete; + } + + if (!pt_is_null_with_top(p2, ptop)) { + goto cannot_delete; + } + } else if (pt_is_rule(p, PN_atom_paren)) { + p = pt_rule_first(p); + if (pt_is_rule(p, PN_testlist_comp)) { + // TODO perhaps factorise testlist_comp code with other uses of PN_testlist_comp + // or, simplify the logic here my making the parser simplify everything to a list + const byte *p0 = pt_rule_first(p); + c_del_stmt(comp, p0); + + const byte *p1 = pt_next(p0); + if (pt_is_rule(p1, PN_testlist_comp_3b)) { + // sequence of one item, with trailing comma + assert(pt_is_rule_empty(p1)); + } else if (pt_is_rule(p1, PN_testlist_comp_3c)) { + // sequence of many items + const byte *ptop; + p1 = pt_rule_extract_top(p1, &ptop); + while (p1 != ptop) { + c_del_stmt(comp, p1); + p1 = pt_next(p1); + } + } else if (pt_is_rule(p1, PN_comp_for)) { + // TODO not implemented; can't del comprehension? can we get here? + goto cannot_delete; + } else { + // sequence with 2 items + c_del_stmt(comp, p1); + } + } else { + // tuple with 1 element + c_del_stmt(comp, p); + } + } else { + // TODO is there anything else to implement? + goto cannot_delete; + } + + return; + +cannot_delete: + compile_syntax_error(comp, p, "can't delete expression"); +} + +STATIC void compile_del_stmt(compiler_t *comp, const byte *p, const byte *ptop) { + (void)ptop; + apply_to_single_or_list(comp, p, PN_exprlist, c_del_stmt); +} + +STATIC void compile_break_stmt(compiler_t *comp, const byte *p, const byte *ptop) { + (void)ptop; + if (comp->break_label == 0) { + compile_syntax_error(comp, p, "'break' outside loop"); + } + assert(comp->cur_except_level >= comp->break_continue_except_level); + EMIT_ARG(break_loop, comp->break_label, comp->cur_except_level - comp->break_continue_except_level); +} + +STATIC void compile_continue_stmt(compiler_t *comp, const byte *p, const byte *ptop) { + (void)ptop; + if (comp->continue_label == 0) { + compile_syntax_error(comp, p, "'continue' outside loop"); + } + assert(comp->cur_except_level >= comp->break_continue_except_level); + EMIT_ARG(continue_loop, comp->continue_label, comp->cur_except_level - comp->break_continue_except_level); +} + +STATIC void compile_return_stmt(compiler_t *comp, const byte *p, const byte *ptop) { + if (comp->scope_cur->kind != SCOPE_FUNCTION) { + compile_syntax_error(comp, NULL, "'return' outside function"); + return; + } + if (pt_is_null_with_top(p, ptop)) { + // no argument to 'return', so return None + EMIT_ARG(load_const_tok, MP_TOKEN_KW_NONE); + #if 0 + // TODO do we need this optimisation? i guess it's hardly used + } else if (pt_is_rule(p, PN_test_if_expr)) { + // special case when returning an if-expression; to match CPython optimisation + mp_parse_node_struct_t *pns_test_if_expr = (mp_parse_node_struct_t*)pns->nodes[0]; + mp_parse_node_struct_t *pns_test_if_else = (mp_parse_node_struct_t*)pns_test_if_expr->nodes[1]; + + uint l_fail = comp_next_label(comp); + c_if_cond(comp, pns_test_if_else->nodes[0], false, l_fail); // condition + compile_node(comp, pns_test_if_expr->nodes[0]); // success value + EMIT(return_value); + EMIT_ARG(label_assign, l_fail); + compile_node(comp, pns_test_if_else->nodes[1]); // failure value + #endif + } else { + compile_node(comp, p); + } + EMIT(return_value); +} + +STATIC void compile_yield_stmt(compiler_t *comp, const byte *p, const byte *ptop) { + (void)ptop; + compile_node(comp, p); + EMIT(pop_top); +} + +STATIC void compile_raise_stmt(compiler_t *comp, const byte *p, const byte *ptop) { + if (pt_is_null_with_top(p, ptop)) { + // raise + EMIT_ARG(raise_varargs, 0); + } else if (pt_is_rule(p, PN_raise_stmt_arg)) { + // raise x from y + p = pt_rule_first(p); + p = compile_node(comp, p); + compile_node(comp, p); + EMIT_ARG(raise_varargs, 2); + } else { + // raise x + compile_node(comp, p); + EMIT_ARG(raise_varargs, 1); + } +} + +// q_base holds the base of the name +// eg a -> q_base=a +// a.b.c -> q_base=a +STATIC void do_import_name(compiler_t *comp, const byte *p, qstr *q_base) { + bool is_as = false; + if (p != NULL && pt_is_rule(p, PN_dotted_as_name)) { + // a name of the form x as y; unwrap it + p = pt_rule_first(p); // point to 'x' + pt_extract_id(pt_next(p), q_base); // extract 'y' + is_as = true; + } + if (p == NULL || pt_is_null(p)) { + // empty name (eg, from . import x) + *q_base = MP_QSTR_; + EMIT_ARG(import_name, MP_QSTR_); // import the empty string + } else if (pt_is_any_id(p)) { + // just a simple name + qstr q_full; + pt_extract_id(p, &q_full); + if (!is_as) { + *q_base = q_full; + } + EMIT_ARG(import_name, q_full); + } else { + // a name of the form a.b.c + assert(pt_is_rule(p, PN_dotted_name)); // should be + const byte *ptop; + p = pt_rule_extract_top(p, &ptop); + + if (!is_as) { + pt_extract_id(p, q_base); + } + + // work out string length + int len = -1; + for (const byte *p2 = p; p2 != ptop;) { + qstr qst; + p2 = pt_extract_id(p2, &qst); + len += 1 + qstr_len(qst); + } + + // build string + byte *q_ptr; + byte *str_dest = qstr_build_start(len, &q_ptr); + for (const byte *p2 = p; p2 != ptop;) { + if (p2 > p) { + *str_dest++ = '.'; + } + qstr qst; + p2 = pt_extract_id(p2, &qst); + mp_uint_t str_src_len; + const byte *str_src = qstr_data(qst, &str_src_len); + memcpy(str_dest, str_src, str_src_len); + str_dest += str_src_len; + } + qstr q_full = qstr_build_end(q_ptr); + EMIT_ARG(import_name, q_full); + if (is_as) { + for (const byte *p2 = pt_next(p); p2 != ptop;) { + qstr qst; + p2 = pt_extract_id(p2, &qst); + EMIT_ARG(load_attr, qst); + } + } + } +} + +STATIC void compile_dotted_as_name(compiler_t *comp, const byte *p) { + EMIT_ARG(load_const_small_int, 0); // level 0 import + EMIT_ARG(load_const_tok, MP_TOKEN_KW_NONE); // not importing from anything + qstr q_base; + do_import_name(comp, p, &q_base); + compile_store_id(comp, q_base); +} + +STATIC void compile_import_name(compiler_t *comp, const byte *p, const byte *ptop) { + (void)ptop; + apply_to_single_or_list(comp, p, PN_dotted_as_names, compile_dotted_as_name); +} + +STATIC void compile_import_from(compiler_t *comp, const byte *p, const byte *ptop) { + const byte *p_import_source = p; + + // extract the preceeding .'s (if any) for a relative import, to compute the import level + uint import_level = 0; + do { + const byte *p_rel; + if (pt_is_any_tok(p_import_source) || pt_is_rule(p_import_source, PN_one_or_more_period_or_ellipsis)) { + // This covers relative imports with dots only like "from .. import" + p_rel = p_import_source; + p_import_source = NULL; + } else if (pt_is_rule(p_import_source, PN_import_from_2b)) { + // This covers relative imports starting with dot(s) like "from .foo import" + p_rel = pt_rule_first(p_import_source); + p_import_source = pt_next(p_rel); + } else { + // Not a relative import + break; + } + + // get the list of . and/or ...'s + const byte *p_rel_top = mp_parse_node_extract_list(&p_rel, PN_one_or_more_period_or_ellipsis); + + // count the total number of .'s + while (p_rel != p_rel_top) { + if (pt_is_tok(p_rel, MP_TOKEN_DEL_PERIOD)) { + import_level++; + } else { + // should be an MP_TOKEN_ELLIPSIS + import_level += 3; + } + p_rel = pt_next(p_rel); + } + } while (0); + + p = pt_next(p); + + if (pt_is_tok(p, MP_TOKEN_OP_STAR)) { + EMIT_ARG(load_const_small_int, import_level); + + // build the "fromlist" tuple + EMIT_ARG(load_const_str, MP_QSTR__star_); + EMIT_ARG(build_tuple, 1); + + // do the import + qstr dummy_q; + do_import_name(comp, p_import_source, &dummy_q); + EMIT(import_star); + + } else { + EMIT_ARG(load_const_small_int, import_level); + + // build the "fromlist" tuple + ptop = mp_parse_node_extract_list(&p, PN_import_as_names); + uint n = 0; + for (const byte *p_list = p; p_list < ptop; p_list = pt_next(p_list), ++n) { + assert(pt_is_rule(p_list, PN_import_as_name)); + qstr id2; + pt_extract_id(pt_rule_first(p_list), &id2); + EMIT_ARG(load_const_str, id2); + } + EMIT_ARG(build_tuple, n); + + // do the import + qstr dummy_q; + do_import_name(comp, p_import_source, &dummy_q); + for (const byte *p_list = p; p_list < ptop;) { + assert(pt_is_rule(p_list, PN_import_as_name)); + const byte *p_list_top; + p_list = pt_rule_extract_top(p_list, &p_list_top); + qstr id2; + p_list = pt_extract_id(p_list, &id2); + EMIT_ARG(import_from, id2); + if (p_list == p_list_top) { + compile_store_id(comp, id2); + } else { + qstr id3; + p_list = pt_extract_id(p_list, &id3); + compile_store_id(comp, id3); + } + } + EMIT(pop_top); + } +} + +STATIC void compile_declare_global(compiler_t *comp, const byte *p_for_err, qstr qst) { + bool added; + id_info_t *id_info = scope_find_or_add_id(comp->scope_cur, qst, &added); + if (!added && id_info->kind != ID_INFO_KIND_GLOBAL_EXPLICIT) { + compile_syntax_error(comp, p_for_err, "identifier redefined as global"); + return; + } + id_info->kind = ID_INFO_KIND_GLOBAL_EXPLICIT; + + // if the id exists in the global scope, set its kind to EXPLICIT_GLOBAL + id_info = scope_find_global(comp->scope_cur, qst); + if (id_info != NULL) { + id_info->kind = ID_INFO_KIND_GLOBAL_EXPLICIT; + } +} + +STATIC void compile_global_stmt(compiler_t *comp, const byte *p, const byte *ptop) { + if (comp->pass == MP_PASS_SCOPE) { + const byte *p_orig = p; + ptop = mp_parse_node_extract_list(&p, PN_name_list); + while (p != ptop) { + qstr qst; + p = pt_extract_id(p, &qst); + compile_declare_global(comp, p_orig, qst); + } + } +} + +STATIC void compile_declare_nonlocal(compiler_t *comp, const byte *p_for_err, qstr qst) { + bool added; + id_info_t *id_info = scope_find_or_add_id(comp->scope_cur, qst, &added); + if (!added && id_info->kind != ID_INFO_KIND_FREE) { + compile_syntax_error(comp, p_for_err, "identifier redefined as nonlocal"); + return; + } + id_info_t *id_info2 = scope_find_local_in_parent(comp->scope_cur, qst); + if (id_info2 == NULL || !(id_info2->kind == ID_INFO_KIND_LOCAL + || id_info2->kind == ID_INFO_KIND_CELL || id_info2->kind == ID_INFO_KIND_FREE)) { + compile_syntax_error(comp, p_for_err, "no binding for nonlocal found"); + return; + } + id_info->kind = ID_INFO_KIND_FREE; + scope_close_over_in_parents(comp->scope_cur, qst); +} + +STATIC void compile_nonlocal_stmt(compiler_t *comp, const byte *p, const byte *ptop) { + if (comp->pass == MP_PASS_SCOPE) { + if (comp->scope_cur->kind == SCOPE_MODULE) { + compile_syntax_error(comp, p, "can't declare nonlocal in outer code"); + return; + } + const byte *p_orig = p; + ptop = mp_parse_node_extract_list(&p, PN_name_list); + while (p != ptop) { + qstr qst; + p = pt_extract_id(p, &qst); + compile_declare_nonlocal(comp, p_orig, qst); + } + } +} + +STATIC void compile_assert_stmt(compiler_t *comp, const byte *p, const byte *ptop) { + uint l_end = comp_next_label(comp); + p = c_if_cond(comp, p, true, l_end); + EMIT_LOAD_GLOBAL(MP_QSTR_AssertionError); // we load_global instead of load_id, to be consistent with CPython + if (!pt_is_null_with_top(p, ptop)) { + // assertion message + compile_node(comp, p); + EMIT_ARG(call_function, 1, 0, 0); + } + EMIT_ARG(raise_varargs, 1); + EMIT_ARG(label_assign, l_end); +} + +STATIC void compile_if_stmt(compiler_t *comp, const byte *p, const byte *ptop) { + // TODO proper and/or short circuiting + + uint l_end = comp_next_label(comp); + + // optimisation: don't emit anything when "if False" + if (node_is_const_false(p)) { + p = pt_next(p); // skip if condition + p = pt_next(p); // skip if block + } else { + uint l_fail = comp_next_label(comp); + bool if_true = node_is_const_true(p); + p = c_if_cond(comp, p, false, l_fail); // if condition + + p = compile_node(comp, p); // if block + + // optimisation: skip everything else when "if True" + if (if_true) { + goto done; + } + + if ( + // optimisation: don't jump over non-existent elif/else blocks + !(pt_is_null_with_top(p, ptop) && pt_is_null_with_top(pt_next(p), ptop)) + // optimisation: don't jump if last instruction was return + && !EMIT(last_emit_was_return_value) + ) { + // jump over elif/else blocks + EMIT_ARG(jump, l_end); + } + + EMIT_ARG(label_assign, l_fail); + } + + // at this point p points to elif node (which may not exist) + + // compile elif blocks (if any) + if (p != ptop) { + const byte *p_else_top = mp_parse_node_extract_list(&p, PN_if_stmt_elif_list); + while (p != p_else_top) { + assert(pt_is_rule(p, PN_if_stmt_elif)); // should be + p = pt_rule_first(p); + + // optimisation: don't emit anything when "if False" + if (node_is_const_false(p)) { + p = pt_next(p); // skip elif condition + p = pt_next(p); // skip elif block + } else { + uint l_fail = comp_next_label(comp); + bool elif_true = node_is_const_true(p); + p = c_if_cond(comp, p, false, l_fail); // elif condition + + p = compile_node(comp, p); // elif block + + // optimisation: skip everything else when "elif True" + if (elif_true) { + goto done; + } + + // optimisation: don't jump if last instruction was return + if (!EMIT(last_emit_was_return_value)) { + EMIT_ARG(jump, l_end); + } + EMIT_ARG(label_assign, l_fail); + } + } + + // compile else block (if any) + if (p != ptop) { + compile_node(comp, p); + } + } + +done: + EMIT_ARG(label_assign, l_end); +} + +#define START_BREAK_CONTINUE_BLOCK \ + uint16_t old_break_label = comp->break_label; \ + uint16_t old_continue_label = comp->continue_label; \ + uint16_t old_break_continue_except_level = comp->break_continue_except_level; \ + uint break_label = comp_next_label(comp); \ + uint continue_label = comp_next_label(comp); \ + comp->break_label = break_label; \ + comp->continue_label = continue_label; \ + comp->break_continue_except_level = comp->cur_except_level; + +#define END_BREAK_CONTINUE_BLOCK \ + comp->break_label = old_break_label; \ + comp->continue_label = old_continue_label; \ + comp->break_continue_except_level = old_break_continue_except_level; + +STATIC void compile_while_stmt(compiler_t *comp, const byte *p, const byte *ptop) { + START_BREAK_CONTINUE_BLOCK + + const byte *p_body = pt_next(p); + const byte *p_else = pt_next(p_body); + + if (!node_is_const_false(p)) { // optimisation: don't emit anything for "while False" + uint top_label = comp_next_label(comp); + if (!node_is_const_true(p)) { // optimisation: don't jump to cond for "while True" + EMIT_ARG(jump, continue_label); + } + EMIT_ARG(label_assign, top_label); + compile_node(comp, p_body); // body + EMIT_ARG(label_assign, continue_label); + c_if_cond(comp, p, true, top_label); // condition + } + + // break/continue apply to outer loop (if any) in the else block + END_BREAK_CONTINUE_BLOCK + + if (p_else != ptop) { + compile_node(comp, p_else); // else + } + + EMIT_ARG(label_assign, break_label); +} + +// This function compiles an optimised for-loop of the form: +// for in range(, , ): +// +// else: +// +// must be an identifier and must be a small-int. +// +// Semantics of for-loop require: +// - final failing value should not be stored in the loop variable +// - if the loop never runs, the loop variable should never be assigned +// - assignments to , or in the body do not alter the loop +// ( is a constant for us, so no need to worry about it changing) +// +// If is a small-int, then the stack during the for-loop contains just +// the current value of . Otherwise, the stack contains then the +// current value of . +STATIC void compile_for_stmt_optimised_range(compiler_t *comp, const byte *pn_var, + const byte *pn_start, const byte *pn_end, mp_int_t step, + const byte *pn_body, const byte *pn_else) { + + START_BREAK_CONTINUE_BLOCK + + uint top_label = comp_next_label(comp); + uint entry_label = comp_next_label(comp); + + // put the end value on the stack if it's not a small-int constant + bool end_on_stack = !pt_is_small_int(pn_end); + if (end_on_stack) { + compile_node(comp, pn_end); + } + + // compile: start + compile_node(comp, pn_start); + + EMIT_ARG(jump, entry_label); + EMIT_ARG(label_assign, top_label); + + // duplicate next value and store it to var + EMIT(dup_top); + c_assign(comp, pn_var, ASSIGN_STORE); + + // compile body + compile_node(comp, pn_body); + + EMIT_ARG(label_assign, continue_label); + + // compile: var + step + EMIT_ARG(load_const_small_int, step); + EMIT_ARG(binary_op, MP_BINARY_OP_INPLACE_ADD); + + EMIT_ARG(label_assign, entry_label); + + // compile: if var end: goto top + if (end_on_stack) { + EMIT(dup_top_two); + EMIT(rot_two); + } else { + EMIT(dup_top); + compile_node(comp, pn_end); + } + if (step >= 0) { + EMIT_ARG(binary_op, MP_BINARY_OP_LESS); + } else { + EMIT_ARG(binary_op, MP_BINARY_OP_MORE); + } + EMIT_ARG(pop_jump_if, true, top_label); + + // break/continue apply to outer loop (if any) in the else block + END_BREAK_CONTINUE_BLOCK + + if (pn_else != NULL) { + compile_node(comp, pn_else); + } + + EMIT_ARG(label_assign, break_label); + + // discard final value of var that failed the loop condition + EMIT(pop_top); + + // discard value if it's on the stack + if (end_on_stack) { + EMIT(pop_top); + } +} + +STATIC void compile_for_stmt(compiler_t *comp, const byte *p, const byte *ptop) { + // this bit optimises: for in range(...), turning it into an explicitly incremented variable + // this is actually slower, but uses no heap memory + // for viper it will be much, much faster + if (/*comp->scope_cur->emit_options == MP_EMIT_OPT_VIPER &&*/ pt_is_any_id(p) + && pt_is_rule(pt_next(p), PN_power)) { + const byte *p_it_top; + const byte *p_it0 = pt_rule_extract_top(pt_next(p), &p_it_top); + if (!pt_is_id(p_it0, MP_QSTR_range)) { + goto optimise_fail; + } + const byte *p_it1 = pt_next(p_it0); + if (pt_is_rule(p_it1, PN_trailer_paren) + && !pt_is_rule_empty(p_it1) + && pt_next(p_it1) == p_it_top) { + // iterator is of the form range(...) with at least 1 arg + const byte *p_range_args = pt_rule_first(p_it1); + const byte *p_range_args_top = mp_parse_node_extract_list(&p_range_args, PN_arglist); + const byte *p_start = pt_const_int0; + const byte *p_end = p_range_args; + mp_int_t step = 1; + p_range_args = pt_next(p_range_args); + if (p_range_args != p_range_args_top) { + // range has at least 2 args + p_start = p_end; + p_end = p_range_args; + p_range_args = pt_next(p_range_args); + if (p_range_args != p_range_args_top) { + // range has at least 3 args + // We need to know sign of step. This is possible only if it's constant + if (!pt_is_small_int(p_range_args)) { + goto optimise_fail; + } + p_range_args = pt_get_small_int(p_range_args, &step); + if (p_range_args != p_range_args_top) { + // range has at least 4 args, so don't know how to optimise it + goto optimise_fail; + } + } + } + // can optimise + const byte *p_body = p_it_top; + const byte *p_else = pt_next(p_body); + if (p_else == ptop) { + p_else = NULL; + } + compile_for_stmt_optimised_range(comp, p, p_start, p_end, step, p_body, p_else); + return; + } + } +optimise_fail:; + + START_BREAK_CONTINUE_BLOCK + comp->break_label |= MP_EMIT_BREAK_FROM_FOR; + + uint pop_label = comp_next_label(comp); + + const byte *p_it = pt_next(p); + const byte *p_body = compile_node(comp, p_it); // iterator + EMIT(get_iter); + EMIT_ARG(label_assign, continue_label); + EMIT_ARG(for_iter, pop_label); + c_assign(comp, p, ASSIGN_STORE); // variable + const byte *p_else = compile_node(comp, p_body); // body + if (!EMIT(last_emit_was_return_value)) { + EMIT_ARG(jump, continue_label); + } + EMIT_ARG(label_assign, pop_label); + EMIT(for_iter_end); + + // break/continue apply to outer loop (if any) in the else block + END_BREAK_CONTINUE_BLOCK + + if (p_else != ptop) { + compile_node(comp, p_else); // else + } + + EMIT_ARG(label_assign, break_label); +} + +STATIC void compile_try_except(compiler_t *comp, const byte *p_body, const byte *p_except, const byte *p_except_top, const byte *p_else) { + // setup code + uint l1 = comp_next_label(comp); + uint success_label = comp_next_label(comp); + + EMIT_ARG(setup_except, l1); + compile_increase_except_level(comp); + + compile_node(comp, p_body); // body + EMIT(pop_block); + EMIT_ARG(jump, success_label); // jump over exception handler + + EMIT_ARG(label_assign, l1); // start of exception handler + EMIT(start_except_handler); + + uint l2 = comp_next_label(comp); + + while (p_except != p_except_top) { + assert(pt_is_rule(p_except, PN_try_stmt_except)); // should be + p_except = pt_rule_first(p_except); + + qstr qstr_exception_local = 0; + uint end_finally_label = comp_next_label(comp); + + if (pt_is_null(p_except)) { + // this is a catch all exception handler + if (pt_next(pt_next(p_except)) != p_except_top) { + compile_syntax_error(comp, p_except, "default 'except:' must be last"); + compile_decrease_except_level(comp); + return; + } + } else { + // this exception handler requires a match to a certain type of exception + const byte *p_exception_expr = p_except; + if (pt_is_rule(p_exception_expr, PN_try_stmt_as_name)) { + // handler binds the exception to a local + p_exception_expr = pt_rule_first(p_exception_expr); + pt_extract_id(pt_next(p_exception_expr), &qstr_exception_local); + } + EMIT(dup_top); + compile_node(comp, p_exception_expr); + EMIT_ARG(binary_op, MP_BINARY_OP_EXCEPTION_MATCH); + EMIT_ARG(pop_jump_if, false, end_finally_label); + } + + p_except = pt_next(p_except); + + EMIT(pop_top); + + if (qstr_exception_local == 0) { + EMIT(pop_top); + } else { + compile_store_id(comp, qstr_exception_local); + } + + EMIT(pop_top); + + uint l3 = 0; + if (qstr_exception_local != 0) { + l3 = comp_next_label(comp); + EMIT_ARG(setup_finally, l3); + compile_increase_except_level(comp); + } + p_except = compile_node(comp, p_except); + if (qstr_exception_local != 0) { + EMIT(pop_block); + } + EMIT(pop_except); + if (qstr_exception_local != 0) { + EMIT_ARG(load_const_tok, MP_TOKEN_KW_NONE); + EMIT_ARG(label_assign, l3); + EMIT_ARG(load_const_tok, MP_TOKEN_KW_NONE); + compile_store_id(comp, qstr_exception_local); + compile_delete_id(comp, qstr_exception_local); + + compile_decrease_except_level(comp); + EMIT(end_finally); + } + EMIT_ARG(jump, l2); + EMIT_ARG(label_assign, end_finally_label); + EMIT_ARG(adjust_stack_size, 3); // stack adjust for the 3 exception items + } + + compile_decrease_except_level(comp); + EMIT(end_finally); + EMIT(end_except_handler); + + EMIT_ARG(label_assign, success_label); + if (p_else != NULL) { + compile_node(comp, p_else); // else block + } + EMIT_ARG(label_assign, l2); +} + +STATIC void compile_try_finally(compiler_t *comp, const byte *p_body, const byte *p_except, const byte *p_except_top, const byte *p_else, const byte *p_finally) { + uint l_finally_block = comp_next_label(comp); + + EMIT_ARG(setup_finally, l_finally_block); + compile_increase_except_level(comp); + + if (p_except == NULL) { + assert(p_else == NULL); + EMIT_ARG(adjust_stack_size, 3); // stack adjust for possible UNWIND_JUMP state + compile_node(comp, p_body); + EMIT_ARG(adjust_stack_size, -3); + } else { + compile_try_except(comp, p_body, p_except, p_except_top, p_else); + } + EMIT(pop_block); + EMIT_ARG(load_const_tok, MP_TOKEN_KW_NONE); + EMIT_ARG(label_assign, l_finally_block); + compile_node(comp, p_finally); + + compile_decrease_except_level(comp); + EMIT(end_finally); +} + +STATIC void compile_try_stmt(compiler_t *comp, const byte *p, const byte *ptop) { + (void)ptop; + const byte* p1 = pt_next(p); + if (pt_is_rule(p1, PN_try_stmt_except) || pt_is_rule(p1, PN_try_stmt_except_list)) { + // just try-except + const byte *p1_top = mp_parse_node_extract_list(&p1, PN_try_stmt_except_list); + compile_try_except(comp, p, p1, p1_top, NULL); + } else if (pt_is_rule(p1, PN_try_stmt_except_and_more)) { + // try-except and possibly else and/or finally + const byte *p1_top; + const byte *p1_p0 = pt_rule_extract_top(p1, &p1_top); + const byte *p1_p1 = mp_parse_node_extract_list(&p1_p0, PN_try_stmt_except_list); + if (pt_next(p1_p1) == p1_top) { + // no finally, but have else + compile_try_except(comp, p, p1_p0, p1_p1, p1_p1); + } else { + // have finally, may or may not have else + compile_try_finally(comp, p, p1_p0, p1_p1, p1_p1, pt_next(p1_p1)); + } + } else { + // just try-finally + compile_try_finally(comp, p, NULL, NULL, NULL, p1); + } +} + +STATIC void compile_with_stmt_helper(compiler_t *comp, const byte *n_pre, const byte *p_body) { + if (n_pre >= p_body) { + // no more pre-bits, compile the body of the with + compile_node(comp, p_body); + } else { + uint l_end = comp_next_label(comp); + if (pt_is_rule(n_pre, PN_with_item)) { + // this pre-bit is of the form "a as b" + const byte *p = pt_rule_first(n_pre); + p = compile_node(comp, p); + EMIT_ARG(setup_with, l_end); + c_assign(comp, p, ASSIGN_STORE); + n_pre = pt_next(n_pre); + } else { + // this pre-bit is just an expression + n_pre = compile_node(comp, n_pre); + EMIT_ARG(setup_with, l_end); + EMIT(pop_top); + } + compile_increase_except_level(comp); + // compile additional pre-bits and the body + compile_with_stmt_helper(comp, n_pre, p_body); + // finish this with block + EMIT(pop_block); + EMIT_ARG(load_const_tok, MP_TOKEN_KW_NONE); + EMIT_ARG(label_assign, l_end); + EMIT(with_cleanup); + compile_decrease_except_level(comp); + EMIT(end_finally); + } +} + +STATIC void compile_with_stmt(compiler_t *comp, const byte *p, const byte *ptop) { + // get the nodes for the pre-bit of the with (the a as b, c as d, ... bit) + ptop = mp_parse_node_extract_list(&p, PN_with_stmt_list); + + // compile in a nested fashion + compile_with_stmt_helper(comp, p, ptop); +} + +STATIC void compile_expr_stmt(compiler_t *comp, const byte *p, const byte *ptop) { + const byte *p_n1 = pt_next(p); + + if (pt_is_null_with_top(p_n1, ptop)) { + if (comp->is_repl && comp->scope_cur->kind == SCOPE_MODULE) { + // for REPL, evaluate then print the expression + compile_load_id(comp, MP_QSTR___repl_print__); + compile_node(comp, p); + EMIT_ARG(call_function, 1, 0, 0); + EMIT(pop_top); + + } else { + #if 0 + // for non-REPL, evaluate then discard the expression + if ((MP_PARSE_NODE_IS_LEAF(pns->nodes[0]) && !MP_PARSE_NODE_IS_ID(pns->nodes[0])) + || MP_PARSE_NODE_IS_STRUCT_KIND(pns->nodes[0], PN_string) + || MP_PARSE_NODE_IS_STRUCT_KIND(pns->nodes[0], PN_bytes) + || MP_PARSE_NODE_IS_STRUCT_KIND(pns->nodes[0], PN_const_object)) { + // do nothing with a lonely constant + } else + #endif + { + compile_node(comp, p); // just an expression + EMIT(pop_top); // discard last result since this is a statement and leaves nothing on the stack + } + } + } else if (pt_is_rule(p_n1, PN_expr_stmt_augassign)) { + c_assign(comp, p, ASSIGN_AUG_LOAD); // lhs load for aug assign + p_n1 = pt_rule_first(p_n1); + assert(pt_is_any_tok(p_n1)); + byte tok; + p_n1 = pt_tok_extract(p_n1, &tok); + mp_binary_op_t op; + switch (tok) { + case MP_TOKEN_DEL_PIPE_EQUAL: op = MP_BINARY_OP_INPLACE_OR; break; + case MP_TOKEN_DEL_CARET_EQUAL: op = MP_BINARY_OP_INPLACE_XOR; break; + case MP_TOKEN_DEL_AMPERSAND_EQUAL: op = MP_BINARY_OP_INPLACE_AND; break; + case MP_TOKEN_DEL_DBL_LESS_EQUAL: op = MP_BINARY_OP_INPLACE_LSHIFT; break; + case MP_TOKEN_DEL_DBL_MORE_EQUAL: op = MP_BINARY_OP_INPLACE_RSHIFT; break; + case MP_TOKEN_DEL_PLUS_EQUAL: op = MP_BINARY_OP_INPLACE_ADD; break; + case MP_TOKEN_DEL_MINUS_EQUAL: op = MP_BINARY_OP_INPLACE_SUBTRACT; break; + case MP_TOKEN_DEL_STAR_EQUAL: op = MP_BINARY_OP_INPLACE_MULTIPLY; break; + case MP_TOKEN_DEL_DBL_SLASH_EQUAL: op = MP_BINARY_OP_INPLACE_FLOOR_DIVIDE; break; + case MP_TOKEN_DEL_SLASH_EQUAL: op = MP_BINARY_OP_INPLACE_TRUE_DIVIDE; break; + case MP_TOKEN_DEL_PERCENT_EQUAL: op = MP_BINARY_OP_INPLACE_MODULO; break; + case MP_TOKEN_DEL_DBL_STAR_EQUAL: default: op = MP_BINARY_OP_INPLACE_POWER; break; + } + compile_node(comp, p_n1); // rhs + EMIT_ARG(binary_op, op); + c_assign(comp, p, ASSIGN_AUG_STORE); // lhs store for aug assign + } else if (pt_is_rule(p_n1, PN_expr_stmt_assign_list)) { + const byte *p_n1_top; + p_n1 = pt_rule_extract_top(p_n1, &p_n1_top); + const byte *p_rhs = NULL; + for (const byte *pp = p_n1; pp != p_n1_top; pp = pt_next(pp)) { + p_rhs = pp; + } + compile_node(comp, p_rhs); // rhs + // following CPython, we store left-most first + //if (num rhs > 1) { always true? + EMIT(dup_top); + //} + c_assign(comp, p, ASSIGN_STORE); // lhs store + for (const byte *pp = p_n1; pp != p_rhs;) { + const byte *pp_next = pt_next(pp); + if (pp_next != p_rhs) { + EMIT(dup_top); + } + c_assign(comp, pp, ASSIGN_STORE); // middle store + pp = pp_next; + } + } else { + // single assignment + #if 0 + if (MICROPY_COMP_DOUBLE_TUPLE_ASSIGN + && MP_PARSE_NODE_IS_STRUCT_KIND(pns1->nodes[0], PN_testlist_star_expr) + && MP_PARSE_NODE_IS_STRUCT_KIND(pns->nodes[0], PN_testlist_star_expr) + && MP_PARSE_NODE_STRUCT_NUM_NODES((mp_parse_node_struct_t*)pns1->nodes[0]) == 2 + && MP_PARSE_NODE_STRUCT_NUM_NODES((mp_parse_node_struct_t*)pns->nodes[0]) == 2) { + // optimisation for a, b = c, d + mp_parse_node_struct_t *pns10 = (mp_parse_node_struct_t*)pns1->nodes[0]; + mp_parse_node_struct_t *pns0 = (mp_parse_node_struct_t*)pns->nodes[0]; + if (MP_PARSE_NODE_IS_STRUCT_KIND(pns0->nodes[0], PN_star_expr) + || MP_PARSE_NODE_IS_STRUCT_KIND(pns0->nodes[1], PN_star_expr)) { + // can't optimise when it's a star expression on the lhs + goto no_optimisation; + } + compile_node(comp, pns10->nodes[0]); // rhs + compile_node(comp, pns10->nodes[1]); // rhs + EMIT(rot_two); + c_assign(comp, pns0->nodes[0], ASSIGN_STORE); // lhs store + c_assign(comp, pns0->nodes[1], ASSIGN_STORE); // lhs store + } else if (MICROPY_COMP_TRIPLE_TUPLE_ASSIGN + && MP_PARSE_NODE_IS_STRUCT_KIND(pns1->nodes[0], PN_testlist_star_expr) + && MP_PARSE_NODE_IS_STRUCT_KIND(pns->nodes[0], PN_testlist_star_expr) + && MP_PARSE_NODE_STRUCT_NUM_NODES((mp_parse_node_struct_t*)pns1->nodes[0]) == 3 + && MP_PARSE_NODE_STRUCT_NUM_NODES((mp_parse_node_struct_t*)pns->nodes[0]) == 3) { + // optimisation for a, b, c = d, e, f + mp_parse_node_struct_t *pns10 = (mp_parse_node_struct_t*)pns1->nodes[0]; + mp_parse_node_struct_t *pns0 = (mp_parse_node_struct_t*)pns->nodes[0]; + if (MP_PARSE_NODE_IS_STRUCT_KIND(pns0->nodes[0], PN_star_expr) + || MP_PARSE_NODE_IS_STRUCT_KIND(pns0->nodes[1], PN_star_expr) + || MP_PARSE_NODE_IS_STRUCT_KIND(pns0->nodes[2], PN_star_expr)) { + // can't optimise when it's a star expression on the lhs + goto no_optimisation; + } + compile_node(comp, pns10->nodes[0]); // rhs + compile_node(comp, pns10->nodes[1]); // rhs + compile_node(comp, pns10->nodes[2]); // rhs + EMIT(rot_three); + EMIT(rot_two); + c_assign(comp, pns0->nodes[0], ASSIGN_STORE); // lhs store + c_assign(comp, pns0->nodes[1], ASSIGN_STORE); // lhs store + c_assign(comp, pns0->nodes[2], ASSIGN_STORE); // lhs store + } else + #endif + { + //no_optimisation: + compile_node(comp, p_n1); // rhs + c_assign(comp, p, ASSIGN_STORE); // lhs store + } + } +} + +STATIC void compile_test_if_expr(compiler_t *comp, const byte *p, const byte *ptop) { + (void)ptop; + const byte *p_test_if_else = pt_next(p); + assert(p_test_if_else != ptop && pt_is_rule(p_test_if_else, PN_test_if_else)); + p_test_if_else = pt_rule_first(p_test_if_else); + + uint l_fail = comp_next_label(comp); + uint l_end = comp_next_label(comp); + p_test_if_else = c_if_cond(comp, p_test_if_else, false, l_fail); // condition + compile_node(comp, p); // success value + EMIT_ARG(jump, l_end); + EMIT_ARG(label_assign, l_fail); + EMIT_ARG(adjust_stack_size, -1); // adjust stack size + compile_node(comp, p_test_if_else); // failure value + EMIT_ARG(label_assign, l_end); +} + +STATIC void compile_lambdef(compiler_t *comp, const byte *p, const byte *ptop) { + (void)ptop; + mp_int_t scope_idx; + p = pt_get_small_int(p, &scope_idx); + + if (comp->pass == MP_PASS_SCOPE) { + // create a new scope for this lambda + scope_new_and_link(comp, scope_idx, SCOPE_LAMBDA, p, comp->scope_cur->emit_options); + } + + // get the scope for this lambda + scope_t *this_scope = comp->scopes[scope_idx]; + + // compile the lambda definition + compile_funcdef_lambdef(comp, this_scope, p, PN_varargslist); +} + +STATIC void compile_or_and_test(compiler_t *comp, const byte *p, const byte *ptop, bool cond) { + uint l_end = comp_next_label(comp); + while (p != ptop) { + p = compile_node(comp, p); + if (p != ptop) { + EMIT_ARG(jump_if_or_pop, cond, l_end); + } + } + EMIT_ARG(label_assign, l_end); +} + +STATIC void compile_or_test(compiler_t *comp, const byte *p, const byte *ptop) { + compile_or_and_test(comp, p, ptop, true); +} + +STATIC void compile_and_test(compiler_t *comp, const byte *p, const byte *ptop) { + compile_or_and_test(comp, p, ptop, false); +} + +STATIC void compile_not_test_2(compiler_t *comp, const byte *p, const byte *ptop) { + (void)ptop; + compile_node(comp, p); + EMIT_ARG(unary_op, MP_UNARY_OP_NOT); +} + +STATIC void compile_comparison(compiler_t *comp, const byte *p, const byte *ptop) { + int num_nodes = pt_num_nodes(p, ptop); + p = compile_node(comp, p); + bool multi = (num_nodes > 3); + uint l_fail = 0; + if (multi) { + l_fail = comp_next_label(comp); + } + for (int i = 1; i + 1 < num_nodes; i += 2) { + mp_binary_op_t op; + if (pt_is_any_tok(p)) { + byte tok; + p = pt_tok_extract(p, &tok); + switch (tok) { + case MP_TOKEN_OP_LESS: op = MP_BINARY_OP_LESS; break; + case MP_TOKEN_OP_MORE: op = MP_BINARY_OP_MORE; break; + case MP_TOKEN_OP_DBL_EQUAL: op = MP_BINARY_OP_EQUAL; break; + case MP_TOKEN_OP_LESS_EQUAL: op = MP_BINARY_OP_LESS_EQUAL; break; + case MP_TOKEN_OP_MORE_EQUAL: op = MP_BINARY_OP_MORE_EQUAL; break; + case MP_TOKEN_OP_NOT_EQUAL: op = MP_BINARY_OP_NOT_EQUAL; break; + case MP_TOKEN_KW_IN: default: op = MP_BINARY_OP_IN; break; + } + } else { + if (pt_is_rule(p, PN_comp_op_not_in)) { + op = MP_BINARY_OP_NOT_IN; + } else { + assert(pt_is_rule(p, PN_comp_op_is)); // should be + if (pt_is_rule_empty(p)) { + op = MP_BINARY_OP_IS; + } else { + op = MP_BINARY_OP_IS_NOT; + } + } + p = pt_next(p); + } + + p = compile_node(comp, p); + + if (i + 2 < num_nodes) { + EMIT(dup_top); + EMIT(rot_three); + } + + EMIT_ARG(binary_op, op); + + if (i + 2 < num_nodes) { + EMIT_ARG(jump_if_or_pop, false, l_fail); + } + } + if (multi) { + uint l_end = comp_next_label(comp); + EMIT_ARG(jump, l_end); + EMIT_ARG(label_assign, l_fail); + EMIT_ARG(adjust_stack_size, 1); + EMIT(rot_two); + EMIT(pop_top); + EMIT_ARG(label_assign, l_end); + } +} + +STATIC void compile_star_expr(compiler_t *comp, const byte *p, const byte *ptop) { + (void)ptop; + compile_syntax_error(comp, p, "*x must be assignment target"); +} + +STATIC void c_binary_op(compiler_t *comp, const byte *p, const byte *ptop, mp_binary_op_t binary_op) { + p = compile_node(comp, p); + while (p != ptop) { + p = compile_node(comp, p); + EMIT_ARG(binary_op, binary_op); + } +} + +STATIC void compile_expr(compiler_t *comp, const byte *p, const byte *ptop) { + c_binary_op(comp, p, ptop, MP_BINARY_OP_OR); +} + +STATIC void compile_xor_expr(compiler_t *comp, const byte *p, const byte *ptop) { + c_binary_op(comp, p, ptop, MP_BINARY_OP_XOR); +} + +STATIC void compile_and_expr(compiler_t *comp, const byte *p, const byte *ptop) { + c_binary_op(comp, p, ptop, MP_BINARY_OP_AND); +} + +STATIC void compile_shift_expr(compiler_t *comp, const byte *p, const byte *ptop) { + p = compile_node(comp, p); + while (p != ptop) { + byte tok; + p = pt_tok_extract(p, &tok); + p = compile_node(comp, p); + if (tok == MP_TOKEN_OP_DBL_LESS) { + EMIT_ARG(binary_op, MP_BINARY_OP_LSHIFT); + } else { + assert(tok == MP_TOKEN_OP_DBL_MORE); // should be + EMIT_ARG(binary_op, MP_BINARY_OP_RSHIFT); + } + } +} + +STATIC void compile_arith_expr(compiler_t *comp, const byte *p, const byte *ptop) { + p = compile_node(comp, p); + while (p != ptop) { + byte tok; + p = pt_tok_extract(p, &tok); + p = compile_node(comp, p); + if (tok == MP_TOKEN_OP_PLUS) { + EMIT_ARG(binary_op, MP_BINARY_OP_ADD); + } else { + assert(tok == MP_TOKEN_OP_MINUS); // should be + EMIT_ARG(binary_op, MP_BINARY_OP_SUBTRACT); + } + } +} + +STATIC void compile_term(compiler_t *comp, const byte *p, const byte *ptop) { + p = compile_node(comp, p); + while (p != ptop) { + byte tok; + p = pt_tok_extract(p, &tok); + p = compile_node(comp, p); + if (tok == MP_TOKEN_OP_STAR) { + EMIT_ARG(binary_op, MP_BINARY_OP_MULTIPLY); + } else if (tok == MP_TOKEN_OP_DBL_SLASH) { + EMIT_ARG(binary_op, MP_BINARY_OP_FLOOR_DIVIDE); + } else if (tok == MP_TOKEN_OP_SLASH) { + EMIT_ARG(binary_op, MP_BINARY_OP_TRUE_DIVIDE); + } else { + assert(tok == MP_TOKEN_OP_PERCENT); // should be + EMIT_ARG(binary_op, MP_BINARY_OP_MODULO); + } + } +} + +STATIC void compile_factor_2(compiler_t *comp, const byte *p, const byte *ptop) { + (void)ptop; + byte tok; + p = pt_tok_extract(p, &tok); + compile_node(comp, p); + if (tok == MP_TOKEN_OP_PLUS) { + EMIT_ARG(unary_op, MP_UNARY_OP_POSITIVE); + } else if (tok == MP_TOKEN_OP_MINUS) { + EMIT_ARG(unary_op, MP_UNARY_OP_NEGATIVE); + } else { + assert(tok == MP_TOKEN_OP_TILDE); // should be + EMIT_ARG(unary_op, MP_UNARY_OP_INVERT); + } +} + +STATIC void compile_power(compiler_t *comp, const byte *p, const byte *ptop) { + // this is to handle special super() call + comp->func_arg_is_super = pt_is_id(p, MP_QSTR_super); + + compile_generic_all_nodes(comp, p, ptop); +} + +// if p_arglist==NULL then there are no arguments +STATIC void compile_trailer_paren_helper(compiler_t *comp, const byte *p_arglist, bool is_method_call, int n_positional_extra) { + // function to call is on top of stack + + // this is to handle special super() call + if (p_arglist == NULL && comp->func_arg_is_super && comp->scope_cur->kind == SCOPE_FUNCTION) { + compile_load_id(comp, MP_QSTR___class__); + // look for first argument to function (assumes it's "self") + for (int i = 0; i < comp->scope_cur->id_info_len; i++) { + if (comp->scope_cur->id_info[i].flags & ID_FLAG_IS_PARAM) { + // first argument found; load it and call super + EMIT_LOAD_FAST(MP_QSTR_, comp->scope_cur->id_info[i].local_num); + EMIT_ARG(call_function, 2, 0, 0); + return; + } + } + compile_syntax_error(comp, NULL, "super() call cannot find self"); // really a TypeError + return; + } + + // get the list of arguments + const byte *ptop; + if (p_arglist == NULL) { + ptop = NULL; + } else { + ptop = mp_parse_node_extract_list(&p_arglist, PN_arglist); + } + + // compile the arguments + // Rather than calling compile_node on the list, we go through the list of args + // explicitly here so that we can count the number of arguments and give sensible + // error messages. + int n_positional = n_positional_extra; + uint n_keyword = 0; + uint star_flags = 0; + const byte *p_star_args = NULL, *p_dblstar_args = NULL; + for (const byte *p = p_arglist; p != ptop;) { + if (pt_is_rule(p, PN_arglist_star)) { + if (star_flags & MP_EMIT_STAR_FLAG_SINGLE) { + compile_syntax_error(comp, p, "can't have multiple *x"); + return; + } + star_flags |= MP_EMIT_STAR_FLAG_SINGLE; + p_star_args = pt_rule_first(p); + p = pt_next(p); + } else if (pt_is_rule(p, PN_arglist_dbl_star)) { + if (star_flags & MP_EMIT_STAR_FLAG_DOUBLE) { + compile_syntax_error(comp, p, "can't have multiple **x"); + return; + } + star_flags |= MP_EMIT_STAR_FLAG_DOUBLE; + p_dblstar_args = pt_rule_first(p); + p = pt_next(p); + } else if (pt_is_rule(p, PN_argument)) { + p = pt_rule_first(p); // skip rule header + const byte *p2 = pt_next(p); // get second node + if (pt_is_rule(p2, PN_comp_for)) { + // list comprehension argument + compile_comprehension(comp, p, SCOPE_GEN_EXPR); + n_positional++; + p = pt_next(pt_next(p)); + } else { + // keyword argument + if (!pt_is_any_id(p)) { + compile_syntax_error(comp, p, "LHS of keyword arg must be an id"); + return; + } + qstr kw; + p = pt_extract_id(p, &kw); + EMIT_ARG(load_const_str, kw); + p = compile_node(comp, p); + n_keyword += 1; + } + } else { + if (n_keyword > 0) { + compile_syntax_error(comp, p, "non-keyword arg after keyword arg"); + return; + } + p = compile_node(comp, p); + n_positional++; + } + } + + // compile the star/double-star arguments if we had them + // if we had one but not the other then we load "null" as a place holder + if (star_flags != 0) { + if (p_star_args == NULL) { + EMIT(load_null); + } else { + compile_node(comp, p_star_args); + } + if (p_dblstar_args == NULL) { + EMIT(load_null); + } else { + compile_node(comp, p_dblstar_args); + } + } + + // emit the function/method call + if (is_method_call) { + EMIT_ARG(call_method, n_positional, n_keyword, star_flags); + } else { + EMIT_ARG(call_function, n_positional, n_keyword, star_flags); + } +} + +STATIC void compile_power_trailers(compiler_t *comp, const byte *p, const byte *ptop) { + while (p != ptop) { + const byte *p_next = pt_next(p); + if (p_next != ptop && pt_is_rule(p, PN_trailer_period) && pt_is_rule(p_next, PN_trailer_paren)) { + // optimisation for method calls a.f(...), following PyPy + const byte *p_period = pt_rule_first(p); + const byte *p_paren; + if (pt_is_rule_empty(p_next)) { + p_paren = NULL; + } else { + p_paren = pt_rule_first(p_next); + } + qstr method_name; + pt_extract_id(p_period, &method_name); + EMIT_ARG(load_method, method_name); + compile_trailer_paren_helper(comp, p_paren, true, 0); + p = pt_next(p_next); + } else { + p = compile_node(comp, p); + } + comp->func_arg_is_super = false; + } +} + +STATIC void compile_power_dbl_star(compiler_t *comp, const byte *p, const byte *ptop) { + (void)ptop; + compile_node(comp, p); + EMIT_ARG(binary_op, MP_BINARY_OP_POWER); +} + +// p needs to point to 2 successive nodes, first is lhs of comprehension, second is PN_comp_for node +STATIC void compile_comprehension(compiler_t *comp, const byte *p, scope_kind_t kind) { + const byte *p_comp_for = pt_next(p); + assert(pt_is_rule(p_comp_for, PN_comp_for)); + p_comp_for = pt_rule_first(p_comp_for); + + mp_int_t scope_idx; + p_comp_for = pt_get_small_int(p_comp_for, &scope_idx); + + if (comp->pass == MP_PASS_SCOPE) { + // create a new scope for this comprehension + scope_new_and_link(comp, scope_idx, kind, p, comp->scope_cur->emit_options); + } + + // get the scope for this comprehension + scope_t *this_scope = comp->scopes[scope_idx]; + + // compile the comprehension + close_over_variables_etc(comp, this_scope, 0, 0); + + compile_node(comp, pt_next(p_comp_for)); // source of the iterator + EMIT(get_iter); + EMIT_ARG(call_function, 1, 0, 0); +} + +STATIC void compile_atom_paren(compiler_t *comp, const byte *p, const byte *ptop) { + if (pt_is_null_with_top(p, ptop)) { + // an empty tuple + c_tuple(comp, NULL, NULL, NULL); + } else if (pt_is_rule(p, PN_testlist_comp)) { + p = pt_rule_first(p); + const byte *p1 = pt_next(p); + if (pt_is_rule(p1, PN_testlist_comp_3b) || pt_is_rule(p1, PN_testlist_comp_3c)) { + // tuple of one item with trailing comma (3b); or tuple of many items (3c) + c_tuple(comp, p, pt_rule_first(p1), ptop); + } else if (pt_is_rule(p1, PN_comp_for)) { + // generator expression + compile_comprehension(comp, p, SCOPE_GEN_EXPR); + } else { + // tuple with 2 items + c_tuple(comp, NULL, p, ptop); + } + } else { + // parenthesis around a single item, is just that item + compile_node(comp, p); + } +} + +STATIC void compile_atom_bracket(compiler_t *comp, const byte *p, const byte *ptop) { + if (pt_is_null_with_top(p, ptop)) { + // empty list + EMIT_ARG(build_list, 0); + } else if (pt_is_rule(p, PN_testlist_comp)) { + p = pt_rule_first(p); + const byte *p3 = pt_next(p); + if (pt_is_rule(p3, PN_testlist_comp_3b) || pt_is_rule(p3, PN_testlist_comp_3c)) { + // list of one item with trailing comma (3b); or list of many items (3c) + p3 = pt_rule_first(p3); + compile_node(comp, p); + compile_generic_all_nodes(comp, p3, ptop); + EMIT_ARG(build_list, 1 + pt_num_nodes(p3, ptop)); + } else if (pt_is_rule(p3, PN_comp_for)) { + // list comprehension + compile_comprehension(comp, p, SCOPE_LIST_COMP); + } else { + // list with 2 items + p = compile_node(comp, p); + compile_node(comp, p); + EMIT_ARG(build_list, 2); + } + } else { + // list with 1 item + compile_node(comp, p); + EMIT_ARG(build_list, 1); + } +} + +STATIC void compile_atom_brace(compiler_t *comp, const byte *p, const byte *ptop) { + if (pt_is_null_with_top(p, ptop)) { + // empty dict + EMIT_ARG(build_map, 0); + } else if (pt_is_rule(p, PN_dictorsetmaker_item)) { + // dict with one element + EMIT_ARG(build_map, 1); + compile_node(comp, p); + EMIT(store_map); + } else if (pt_is_rule(p, PN_dictorsetmaker)) { + p = pt_rule_first(p); + const byte *p1 = pt_next(p); + if (pt_is_rule(p1, PN_dictorsetmaker_list)) { + // dict/set with multiple elements + const byte *p1_top; + p1 = pt_rule_extract_top(p1, &p1_top); + + // get tail elements (2nd, 3rd, ...) + if (p1 != p1_top) { + mp_parse_node_extract_list(&p1, PN_dictorsetmaker_list2); + } + + // first element sets whether it's a dict or set + bool is_dict; + if (!MICROPY_PY_BUILTINS_SET || pt_is_rule(p, PN_dictorsetmaker_item)) { + // a dictionary + EMIT_ARG(build_map, 1 + pt_num_nodes(p1, p1_top)); + compile_node(comp, p); + EMIT(store_map); + is_dict = true; + } else { + // a set + compile_node(comp, p); // 1st value of set + is_dict = false; + } + + // process rest of elements + for (const byte *p_elem = p1; p_elem != p1_top;) { + bool is_key_value = pt_is_rule(p_elem, PN_dictorsetmaker_item); + p_elem = compile_node(comp, p_elem); + if (is_dict) { + if (!is_key_value) { + // TODO what is the correct p for error node? + compile_syntax_error(comp, p, "expecting key:value for dictionary"); + return; + } + EMIT(store_map); + } else { + if (is_key_value) { + // TODO what is the correct p for error node? + compile_syntax_error(comp, p, "expecting just a value for set"); + return; + } + } + } + + #if MICROPY_PY_BUILTINS_SET + // if it's a set, build it + if (!is_dict) { + EMIT_ARG(build_set, 1 + pt_num_nodes(p1, p1_top)); + } + #endif + } else { + assert(pt_is_rule(p1, PN_comp_for)); // should be + // dict/set comprehension + if (!MICROPY_PY_BUILTINS_SET || pt_is_rule(p, PN_dictorsetmaker_item)) { + // a dictionary comprehension + compile_comprehension(comp, p, SCOPE_DICT_COMP); + } else { + // a set comprehension + compile_comprehension(comp, p, SCOPE_SET_COMP); + } + } + } else { + // set with one element + #if MICROPY_PY_BUILTINS_SET + compile_node(comp, p); + EMIT_ARG(build_set, 1); + #else + assert(0); + #endif + } +} + +STATIC void compile_trailer_paren(compiler_t *comp, const byte *p, const byte *ptop) { + if (p >= ptop) { + p = NULL; + } + compile_trailer_paren_helper(comp, p, false, 0); +} + +STATIC void compile_trailer_bracket(compiler_t *comp, const byte *p, const byte *ptop) { + (void)ptop; + // object who's index we want is on top of stack + compile_node(comp, p); // the index + EMIT(load_subscr); +} + +STATIC void compile_trailer_period(compiler_t *comp, const byte *p, const byte *ptop) { + (void)ptop; + // object who's attribute we want is on top of stack + qstr attr; + p = pt_extract_id(p, &attr); + EMIT_ARG(load_attr, attr); +} + +#if MICROPY_PY_BUILTINS_SLICE +// p,ptop should be the args of subscript_3 +STATIC void compile_subscript_3_helper(compiler_t *comp, const byte *p, const byte *ptop) { + if (p == ptop) { + // [?:] + EMIT_ARG(load_const_tok, MP_TOKEN_KW_NONE); + EMIT_ARG(build_slice, 2); + } else if (pt_is_rule(p, PN_subscript_3c)) { + EMIT_ARG(load_const_tok, MP_TOKEN_KW_NONE); + if (pt_is_rule_empty(p)) { + // [?::] + EMIT_ARG(build_slice, 2); + } else { + // [?::x] + compile_node(comp, pt_rule_first(p)); + EMIT_ARG(build_slice, 3); + } + } else if (pt_is_rule(p, PN_subscript_3d)) { + p = pt_rule_first(p); + p = compile_node(comp, p); + if (pt_is_rule(p, PN_sliceop)) { + // [?:x:] + EMIT_ARG(build_slice, 2); + } else { + // [?:x:x] + compile_node(comp, p); + EMIT_ARG(build_slice, 3); + } + } else { + // [?:x] + compile_node(comp, p); + EMIT_ARG(build_slice, 2); + } +} + +STATIC void compile_subscript_2(compiler_t *comp, const byte *p, const byte *ptop) { + p = compile_node(comp, p); // start of slice + p = pt_rule_first(p); // skip header of subscript_3 + compile_subscript_3_helper(comp, p, ptop); +} + +STATIC void compile_subscript_3(compiler_t *comp, const byte *p, const byte *ptop) { + EMIT_ARG(load_const_tok, MP_TOKEN_KW_NONE); + compile_subscript_3_helper(comp, p, ptop); +} +#endif // MICROPY_PY_BUILTINS_SLICE + +STATIC void compile_dictorsetmaker_item(compiler_t *comp, const byte *p, const byte *ptop) { + (void)ptop; + // if this is called then we are compiling a dict key:value pair + compile_node(comp, pt_next(p)); // value + compile_node(comp, p); // key +} + +STATIC void compile_classdef(compiler_t *comp, const byte *p, const byte *ptop) { + (void)ptop; + qstr cname = compile_classdef_helper(comp, p, comp->scope_cur->emit_options); + // store class object into class name + compile_store_id(comp, cname); +} + +STATIC void compile_yield_expr(compiler_t *comp, const byte *p, const byte *ptop) { + if (comp->scope_cur->kind != SCOPE_FUNCTION && comp->scope_cur->kind != SCOPE_LAMBDA) { + compile_syntax_error(comp, NULL, "'yield' outside function"); + return; + } + if (pt_is_null_with_top(p, ptop)) { + EMIT_ARG(load_const_tok, MP_TOKEN_KW_NONE); + EMIT(yield_value); + } else if (pt_is_rule(p, PN_yield_arg_from)) { + p = pt_rule_first(p); + compile_node(comp, p); + EMIT(get_iter); + EMIT_ARG(load_const_tok, MP_TOKEN_KW_NONE); + EMIT(yield_from); + } else { + compile_node(comp, p); + EMIT(yield_value); + } +} + +typedef void (*compile_function_t)(compiler_t*, const byte*, const byte*); +STATIC compile_function_t compile_function[] = { +#define nc NULL +#define c(f) compile_##f +#define DEF_RULE(rule, comp, kind, ...) comp, +#include "py/grammar.h" +#undef nc +#undef c +#undef DEF_RULE + NULL, +}; + +STATIC const byte *compile_node(compiler_t *comp, const byte *p) { + //printf("CN %p %02x %02x %02x\n", p, p[0], p[1], p[2]); + if (pt_is_null(p)) { + // pass + return p + 1; + } else if (pt_is_small_int(p)) { + mp_int_t arg; + p = pt_get_small_int(p, &arg); + EMIT_ARG(load_const_small_int, arg); + return p; + } else if (pt_is_any_tok(p)) { + byte tok; + p = pt_tok_extract(p, &tok); + if (tok == MP_TOKEN_NEWLINE) { + // this can occur when file_input lets through a NEWLINE (eg if file starts with a newline) + // or when single_input lets through a NEWLINE (user enters a blank line) + // do nothing + } else { + EMIT_ARG(load_const_tok, tok); + } + return p; + } else if (*p == MP_PT_STRING) { + qstr qst = p[1] | (p[2] << 8); + EMIT_ARG(load_const_str, qst); + return pt_next(p); + } else if (*p == MP_PT_BYTES) { + // only create and load the actual bytes object on the last pass + if (comp->pass != MP_PASS_EMIT) { + EMIT_ARG(load_const_obj, mp_const_none); + } else { + qstr qst = p[1] | (p[2] << 8); + mp_uint_t len; + const byte *data = qstr_data(qst, &len); + EMIT_ARG(load_const_obj, mp_obj_new_bytes(data, len)); + } + return pt_next(p); + } else if (pt_is_any_id(p)) { + qstr qst; + p = pt_extract_id(p, &qst); + compile_load_id(comp, qst); + return p; + } else if (*p == MP_PT_CONST_OBJECT) { + mp_uint_t idx; + p = pt_extract_const_obj(p, &idx); + EMIT_ARG(load_const_obj, (mp_obj_t)comp->co_data[idx]); + return p; + } else { + assert(*p >= MP_PT_RULE_BASE); + mp_uint_t rule_id, src_line; + const byte *ptop; + p = pt_rule_extract(p, &rule_id, &src_line, &ptop); + EMIT_ARG(set_source_line, src_line); + compile_function_t f = compile_function[rule_id]; + if (f == NULL) { + #if MICROPY_DEBUG_PRINTERS + printf("node %u cannot be compiled\n", (uint)rule_id); + //mp_parse_node_print(pn, 0); + #endif + compile_syntax_error(comp, p, "internal compiler error"); + assert(0); + return ptop; + } else { + f(comp, p, ptop); + if (comp->compile_error != MP_OBJ_NULL && comp->compile_error_line == 0) { + // add line info for the error in case it didn't have a line number + comp->compile_error_line = src_line; + } + return ptop; + } + } +} + +STATIC void compile_scope_func_lambda_param(compiler_t *comp, const byte *p, pn_kind_t pn_name, pn_kind_t pn_star, pn_kind_t pn_dbl_star) { + (void)pn_dbl_star; + // TODO verify that *k and **k are last etc + qstr param_name = MP_QSTR_NULL; + uint param_flag = ID_FLAG_IS_PARAM; + if (pt_is_any_id(p)) { + pt_extract_id(p, ¶m_name); + if (comp->have_star) { + // comes after a star, so counts as a keyword-only parameter + comp->scope_cur->num_kwonly_args += 1; + } else { + // comes before a star, so counts as a positional parameter + comp->scope_cur->num_pos_args += 1; + } + } else { + if (pt_is_rule(p, pn_name)) { + pt_extract_id(pt_rule_first(p), ¶m_name); + if (comp->have_star) { + // comes after a star, so counts as a keyword-only parameter + comp->scope_cur->num_kwonly_args += 1; + } else { + // comes before a star, so counts as a positional parameter + comp->scope_cur->num_pos_args += 1; + } + } else if (pt_is_rule(p, pn_star)) { + comp->have_star = true; + param_flag = ID_FLAG_IS_PARAM | ID_FLAG_IS_STAR_PARAM; + if (pt_is_rule_empty(p)) { + // bare star + // TODO see http://www.python.org/dev/peps/pep-3102/ + //assert(comp->scope_cur->num_dict_params == 0); + } else if (pt_is_any_id(pt_rule_first(p))) { + // named star + comp->scope_cur->scope_flags |= MP_SCOPE_FLAG_VARARGS; + pt_extract_id(pt_rule_first(p), ¶m_name); + } else { + assert(pt_is_rule(pt_rule_first(p), PN_tfpdef)); // should be + // named star with possible annotation + comp->scope_cur->scope_flags |= MP_SCOPE_FLAG_VARARGS; + pt_extract_id(pt_rule_first(pt_rule_first(p)), ¶m_name); + } + } else { + assert(pt_is_rule(p, pn_dbl_star)); // should be + pt_extract_id(pt_rule_first(p), ¶m_name); + param_flag = ID_FLAG_IS_PARAM | ID_FLAG_IS_DBL_STAR_PARAM; + comp->scope_cur->scope_flags |= MP_SCOPE_FLAG_VARKEYWORDS; + } + } + + if (param_name != MP_QSTR_NULL) { + bool added; + id_info_t *id_info = scope_find_or_add_id(comp->scope_cur, param_name, &added); + if (!added) { + compile_syntax_error(comp, p, "name reused for argument"); + return; + } + id_info->kind = ID_INFO_KIND_LOCAL; + id_info->flags = param_flag; + } +} + +STATIC void compile_scope_func_param(compiler_t *comp, const byte *p) { + compile_scope_func_lambda_param(comp, p, PN_typedargslist_name, PN_typedargslist_star, PN_typedargslist_dbl_star); +} + +STATIC void compile_scope_lambda_param(compiler_t *comp, const byte *p) { + compile_scope_func_lambda_param(comp, p, PN_varargslist_name, PN_varargslist_star, PN_varargslist_dbl_star); +} + +#if MICROPY_EMIT_NATIVE +STATIC void compile_scope_func_annotations(compiler_t *comp, const byte *p) { + if (pt_is_rule(p, PN_typedargslist_name)) { + // named parameter with possible annotation + // fallthrough + } else if (pt_is_rule(p, PN_typedargslist_star)) { + const byte *p0 = pt_rule_first(p); + if (pt_is_rule(p0, PN_tfpdef)) { + // named star with possible annotation + p = p0; + // fallthrough + } else { + // no annotation + return; + } + } else if (pt_is_rule(p, PN_typedargslist_dbl_star)) { + // double star with possible annotation + // fallthrough + } else { + // no annotation + return; + } + + // p should be a rule whose first node is an identifier and second may be the annotation + + const byte *ptop; + p = pt_rule_extract_top(p, &ptop); + + qstr param_name; + p = pt_extract_id(p, ¶m_name); + + if (!pt_is_null_with_top(p, ptop)) { + id_info_t *id_info = scope_find(comp->scope_cur, param_name); + assert(id_info != NULL); + + if (pt_is_any_id(p)) { + qstr arg_type; + pt_extract_id(p, &arg_type); + EMIT_ARG(set_native_type, MP_EMIT_NATIVE_TYPE_ARG, id_info->local_num, arg_type); + } else { + compile_syntax_error(comp, p, "parameter annotation must be an identifier"); + } + } +} +#endif // MICROPY_EMIT_NATIVE + +STATIC void compile_scope_comp_iter(compiler_t *comp, const byte *p_iter, const byte *p_inner_expr, int l_top, int for_depth) { + tail_recursion: + if (p_iter == NULL) { + // no more nested if/for; compile inner expression + compile_node(comp, p_inner_expr); + if (comp->scope_cur->kind == SCOPE_LIST_COMP) { + EMIT_ARG(list_append, for_depth + 2); + } else if (comp->scope_cur->kind == SCOPE_DICT_COMP) { + EMIT_ARG(map_add, for_depth + 2); + #if MICROPY_PY_BUILTINS_SET + } else if (comp->scope_cur->kind == SCOPE_SET_COMP) { + EMIT_ARG(set_add, for_depth + 2); + #endif + } else { + EMIT(yield_value); + EMIT(pop_top); + } + } else if (pt_is_rule(p_iter, PN_comp_if)) { + // if condition + const byte *ptop; + const byte *p0 = pt_rule_extract_top(p_iter, &ptop); + p_iter = c_if_cond(comp, p0, false, l_top); + if (p_iter == ptop) { + p_iter = NULL; + } + goto tail_recursion; + } else { + assert(pt_is_rule(p_iter, PN_comp_for)); // should be + // for loop + const byte *ptop; + const byte *p0 = pt_rule_extract_top(p_iter, &ptop); + p0 = pt_next(p0); // skip scope index + const byte *p2 = compile_node(comp, pt_next(p0)); + uint l_end2 = comp_next_label(comp); + uint l_top2 = comp_next_label(comp); + EMIT(get_iter); + EMIT_ARG(label_assign, l_top2); + EMIT_ARG(for_iter, l_end2); + c_assign(comp, p0, ASSIGN_STORE); + compile_scope_comp_iter(comp, p2 == ptop ? NULL : p2, p_inner_expr, l_top2, for_depth + 1); + EMIT_ARG(jump, l_top2); + EMIT_ARG(label_assign, l_end2); + EMIT(for_iter_end); + } +} + +#if 0 +STATIC void check_for_doc_string(compiler_t *comp, mp_parse_node_t pn) { +#if MICROPY_ENABLE_DOC_STRING + // see http://www.python.org/dev/peps/pep-0257/ + + // look for the first statement + if (MP_PARSE_NODE_IS_STRUCT_KIND(pn, PN_expr_stmt)) { + // a statement; fall through + } else if (MP_PARSE_NODE_IS_STRUCT_KIND(pn, PN_file_input_2)) { + // file input; find the first non-newline node + mp_parse_node_struct_t *pns = (mp_parse_node_struct_t*)pn; + int num_nodes = MP_PARSE_NODE_STRUCT_NUM_NODES(pns); + for (int i = 0; i < num_nodes; i++) { + pn = pns->nodes[i]; + if (!(MP_PARSE_NODE_IS_LEAF(pn) && MP_PARSE_NODE_LEAF_KIND(pn) == MP_PARSE_NODE_TOKEN && MP_PARSE_NODE_LEAF_ARG(pn) == MP_TOKEN_NEWLINE)) { + // not a newline, so this is the first statement; finish search + break; + } + } + // if we didn't find a non-newline then it's okay to fall through; pn will be a newline and so doc-string test below will fail gracefully + } else if (MP_PARSE_NODE_IS_STRUCT_KIND(pn, PN_suite_block_stmts)) { + // a list of statements; get the first one + pn = ((mp_parse_node_struct_t*)pn)->nodes[0]; + } else { + return; + } + + // check the first statement for a doc string + if (MP_PARSE_NODE_IS_STRUCT_KIND(pn, PN_expr_stmt)) { + mp_parse_node_struct_t *pns = (mp_parse_node_struct_t*)pn; + if ((MP_PARSE_NODE_IS_LEAF(pns->nodes[0]) + && MP_PARSE_NODE_LEAF_KIND(pns->nodes[0]) == MP_PARSE_NODE_STRING) + || MP_PARSE_NODE_IS_STRUCT_KIND(pns->nodes[0], PN_string)) { + // compile the doc string + compile_node(comp, pns->nodes[0]); + // store the doc string + compile_store_id(comp, MP_QSTR___doc__); + } + } +#else + (void)comp; + (void)pn; +#endif +} +#endif + +STATIC void compile_scope(compiler_t *comp, scope_t *scope, pass_kind_t pass) { + comp->pass = pass; + comp->scope_cur = scope; + comp->next_label = 1; + EMIT_ARG(start_pass, pass, scope); + + if (comp->pass == MP_PASS_SCOPE) { + // reset maximum stack sizes in scope + // they will be computed in this first pass + scope->stack_size = 0; + scope->exc_stack_size = 0; + } + + // compile + if (pt_is_rule(scope->pn, PN_eval_input)) { + assert(scope->kind == SCOPE_MODULE); + compile_node(comp, pt_rule_first(scope->pn)); // compile the expression + EMIT(return_value); + } else if (scope->kind == SCOPE_MODULE) { + if (!comp->is_repl) { + //check_for_doc_string(comp, scope->pn); + } + compile_node(comp, scope->pn); + EMIT_ARG(load_const_tok, MP_TOKEN_KW_NONE); + EMIT(return_value); + } else if (scope->kind == SCOPE_FUNCTION) { + const byte *p = scope->pn; + + p = pt_next(p); // skip func name + + // work out number of parameters, keywords and default parameters, and add them to the id_info array + // must be done before compiling the body so that arguments are numbered first (for LOAD_FAST etc) + if (comp->pass == MP_PASS_SCOPE) { + comp->have_star = false; + apply_to_single_or_list(comp, p, PN_typedargslist, compile_scope_func_param); + } + + #if MICROPY_EMIT_NATIVE + else if (scope->emit_options == MP_EMIT_OPT_VIPER) { + // compile annotations; only needed on latter compiler passes + // only needed for viper emitter + + // argument annotations + apply_to_single_or_list(comp, p, PN_typedargslist, compile_scope_func_annotations); + + const byte *p_ret = pt_next(p); // skip arg list + + // next node is return/whole function annotation + if (pt_is_any_id(p_ret)) { + qstr ret_type; + pt_extract_id(p_ret, &ret_type); + EMIT_ARG(set_native_type, MP_EMIT_NATIVE_TYPE_RETURN, 0, ret_type); + } else if (!pt_is_null(p_ret)) { + compile_syntax_error(comp, p_ret, "return annotation must be an identifier"); + } + } + #endif // MICROPY_EMIT_NATIVE + + p = pt_next(p); // skip arg list + p = pt_next(p); // skip return annotation + + compile_node(comp, p); // function body + + // emit return if it wasn't the last opcode + if (!EMIT(last_emit_was_return_value)) { + EMIT_ARG(load_const_tok, MP_TOKEN_KW_NONE); + EMIT(return_value); + } + } else if (scope->kind == SCOPE_LAMBDA) { + const byte *p = scope->pn; + + // work out number of parameters, keywords and default parameters, and add them to the id_info array + // must be done before compiling the body so that arguments are numbered first (for LOAD_FAST etc) + if (comp->pass == MP_PASS_SCOPE) { + comp->have_star = false; + apply_to_single_or_list(comp, p, PN_varargslist, compile_scope_lambda_param); + } + + p = pt_next(p); // skip arg list + + compile_node(comp, p); // lambda body + + // if the lambda is a generator, then we return None, not the result of the expression of the lambda + if (scope->scope_flags & MP_SCOPE_FLAG_GENERATOR) { + EMIT(pop_top); + EMIT_ARG(load_const_tok, MP_TOKEN_KW_NONE); + } + EMIT(return_value); + } else if (scope->kind == SCOPE_LIST_COMP || scope->kind == SCOPE_DICT_COMP || scope->kind == SCOPE_SET_COMP || scope->kind == SCOPE_GEN_EXPR) { + const byte *p = scope->pn; + const byte *p_comp_for = pt_next(p); + const byte *p_comp_for_top; + p_comp_for = pt_rule_extract_top(p_comp_for, &p_comp_for_top); + p_comp_for = pt_next(p_comp_for); // skip scope index + + // We need a unique name for the comprehension argument (the iterator). + // CPython uses .0, but we should be able to use anything that won't + // clash with a user defined variable. Best to use an existing qstr, + // so we use the blank qstr. + qstr qstr_arg = MP_QSTR_; + if (comp->pass == MP_PASS_SCOPE) { + bool added; + id_info_t *id_info = scope_find_or_add_id(comp->scope_cur, qstr_arg, &added); + assert(added); + id_info->kind = ID_INFO_KIND_LOCAL; + scope->num_pos_args = 1; + } + + if (scope->kind == SCOPE_LIST_COMP) { + EMIT_ARG(build_list, 0); + } else if (scope->kind == SCOPE_DICT_COMP) { + EMIT_ARG(build_map, 0); + #if MICROPY_PY_BUILTINS_SET + } else if (scope->kind == SCOPE_SET_COMP) { + EMIT_ARG(build_set, 0); + #endif + } + + uint l_end = comp_next_label(comp); + uint l_top = comp_next_label(comp); + compile_load_id(comp, qstr_arg); + EMIT_ARG(label_assign, l_top); + EMIT_ARG(for_iter, l_end); + c_assign(comp, p_comp_for, ASSIGN_STORE); + const byte *p_comp_for_p2 = pt_next(pt_next(p_comp_for)); + if (p_comp_for_p2 == p_comp_for_top) { + p_comp_for_p2 = NULL; + } + compile_scope_comp_iter(comp, p_comp_for_p2, p, l_top, 0); + EMIT_ARG(jump, l_top); + EMIT_ARG(label_assign, l_end); + EMIT(for_iter_end); + + if (scope->kind == SCOPE_GEN_EXPR) { + EMIT_ARG(load_const_tok, MP_TOKEN_KW_NONE); + } + EMIT(return_value); + } else { + assert(scope->kind == SCOPE_CLASS); + + if (comp->pass == MP_PASS_SCOPE) { + bool added; + id_info_t *id_info = scope_find_or_add_id(scope, MP_QSTR___class__, &added); + assert(added); + id_info->kind = ID_INFO_KIND_LOCAL; + } + + // just to check, should remove this code + qstr class_name; + pt_extract_id(scope->pn, &class_name); + assert(class_name == scope->simple_name); + + compile_load_id(comp, MP_QSTR___name__); + compile_store_id(comp, MP_QSTR___module__); + EMIT_ARG(load_const_str, scope->simple_name); + compile_store_id(comp, MP_QSTR___qualname__); + + const byte *p = pt_next(pt_next(scope->pn)); // skip name, bases + //check_for_doc_string(comp, p); + compile_node(comp, p); // class body + + id_info_t *id = scope_find(scope, MP_QSTR___class__); + assert(id != NULL); + if (id->kind == ID_INFO_KIND_LOCAL) { + EMIT_ARG(load_const_tok, MP_TOKEN_KW_NONE); + } else { + EMIT_LOAD_FAST(MP_QSTR___class__, id->local_num); + } + EMIT(return_value); + } + + EMIT(end_pass); + + // make sure we match all the exception levels + assert(comp->cur_except_level == 0); +} + +#if MICROPY_EMIT_INLINE_THUMB +// requires 3 passes: SCOPE, CODE_SIZE, EMIT +STATIC void compile_scope_inline_asm(compiler_t *comp, scope_t *scope, pass_kind_t pass) { + comp->pass = pass; + comp->scope_cur = scope; + comp->next_label = 1; + + if (scope->kind != SCOPE_FUNCTION) { + compile_syntax_error(comp, MP_PARSE_NODE_NULL, "inline assembler must be a function"); + return; + } + + if (comp->pass > MP_PASS_SCOPE) { + EMIT_INLINE_ASM_ARG(start_pass, comp->pass, comp->scope_cur, &comp->compile_error); + } + + // get the function definition parse node + assert(MP_PARSE_NODE_IS_STRUCT(scope->pn)); + mp_parse_node_struct_t *pns = (mp_parse_node_struct_t*)scope->pn; + assert(MP_PARSE_NODE_STRUCT_KIND(pns) == PN_funcdef); + + //qstr f_id = MP_PARSE_NODE_LEAF_ARG(pns->nodes[0]); // function name + + // parameters are in pns->nodes[1] + if (comp->pass == MP_PASS_CODE_SIZE) { + mp_parse_node_t *pn_params; + int n_params = mp_parse_node_extract_list(&pns->nodes[1], PN_typedargslist, &pn_params); + scope->num_pos_args = EMIT_INLINE_ASM_ARG(count_params, n_params, pn_params); + if (comp->compile_error != MP_OBJ_NULL) { + goto inline_asm_error; + } + } + + assert(MP_PARSE_NODE_IS_NULL(pns->nodes[2])); // type + + mp_parse_node_t pn_body = pns->nodes[3]; // body + mp_parse_node_t *nodes; + int num = mp_parse_node_extract_list(&pn_body, PN_suite_block_stmts, &nodes); + + for (int i = 0; i < num; i++) { + assert(MP_PARSE_NODE_IS_STRUCT(nodes[i])); + mp_parse_node_struct_t *pns2 = (mp_parse_node_struct_t*)nodes[i]; + if (MP_PARSE_NODE_STRUCT_KIND(pns2) == PN_pass_stmt) { + // no instructions + continue; + } else if (MP_PARSE_NODE_STRUCT_KIND(pns2) != PN_expr_stmt) { + // not an instruction; error + not_an_instruction: + compile_syntax_error(comp, nodes[i], "expecting an assembler instruction"); + return; + } + + // check structure of parse node + assert(MP_PARSE_NODE_IS_STRUCT(pns2->nodes[0])); + if (!MP_PARSE_NODE_IS_NULL(pns2->nodes[1])) { + goto not_an_instruction; + } + pns2 = (mp_parse_node_struct_t*)pns2->nodes[0]; + if (MP_PARSE_NODE_STRUCT_KIND(pns2) != PN_power) { + goto not_an_instruction; + } + if (!MP_PARSE_NODE_IS_ID(pns2->nodes[0])) { + goto not_an_instruction; + } + if (!MP_PARSE_NODE_IS_STRUCT_KIND(pns2->nodes[1], PN_trailer_paren)) { + goto not_an_instruction; + } + assert(MP_PARSE_NODE_IS_NULL(pns2->nodes[2])); + + // parse node looks like an instruction + // get instruction name and args + qstr op = MP_PARSE_NODE_LEAF_ARG(pns2->nodes[0]); + pns2 = (mp_parse_node_struct_t*)pns2->nodes[1]; // PN_trailer_paren + mp_parse_node_t *pn_arg; + int n_args = mp_parse_node_extract_list(&pns2->nodes[0], PN_arglist, &pn_arg); + + // emit instructions + if (op == MP_QSTR_label) { + if (!(n_args == 1 && MP_PARSE_NODE_IS_ID(pn_arg[0]))) { + compile_syntax_error(comp, nodes[i], "'label' requires 1 argument"); + return; + } + uint lab = comp_next_label(comp); + if (pass > MP_PASS_SCOPE) { + if (!EMIT_INLINE_ASM_ARG(label, lab, MP_PARSE_NODE_LEAF_ARG(pn_arg[0]))) { + compile_syntax_error(comp, nodes[i], "label redefined"); + return; + } + } + } else if (op == MP_QSTR_align) { + if (!(n_args == 1 && MP_PARSE_NODE_IS_SMALL_INT(pn_arg[0]))) { + compile_syntax_error(comp, nodes[i], "'align' requires 1 argument"); + return; + } + if (pass > MP_PASS_SCOPE) { + EMIT_INLINE_ASM_ARG(align, MP_PARSE_NODE_LEAF_SMALL_INT(pn_arg[0])); + } + } else if (op == MP_QSTR_data) { + if (!(n_args >= 2 && MP_PARSE_NODE_IS_SMALL_INT(pn_arg[0]))) { + compile_syntax_error(comp, nodes[i], "'data' requires at least 2 arguments"); + return; + } + if (pass > MP_PASS_SCOPE) { + mp_int_t bytesize = MP_PARSE_NODE_LEAF_SMALL_INT(pn_arg[0]); + for (uint j = 1; j < n_args; j++) { + if (!MP_PARSE_NODE_IS_SMALL_INT(pn_arg[j])) { + compile_syntax_error(comp, nodes[i], "'data' requires integer arguments"); + return; + } + EMIT_INLINE_ASM_ARG(data, bytesize, MP_PARSE_NODE_LEAF_SMALL_INT(pn_arg[j])); + } + } + } else { + if (pass > MP_PASS_SCOPE) { + EMIT_INLINE_ASM_ARG(op, op, n_args, pn_arg); + } + } + + if (comp->compile_error != MP_OBJ_NULL) { + pns = pns2; // this is the parse node that had the error + goto inline_asm_error; + } + } + + if (comp->pass > MP_PASS_SCOPE) { + EMIT_INLINE_ASM(end_pass); + } + + if (comp->compile_error != MP_OBJ_NULL) { + // inline assembler had an error; set line for its exception + inline_asm_error: + comp->compile_error_line = pns->source_line; + } +} +#endif + +STATIC void scope_compute_things(scope_t *scope) { + // in Micro Python we put the *x parameter after all other parameters (except **y) + if (scope->scope_flags & MP_SCOPE_FLAG_VARARGS) { + id_info_t *id_param = NULL; + for (int i = scope->id_info_len - 1; i >= 0; i--) { + id_info_t *id = &scope->id_info[i]; + if (id->flags & ID_FLAG_IS_STAR_PARAM) { + if (id_param != NULL) { + // swap star param with last param + id_info_t temp = *id_param; *id_param = *id; *id = temp; + } + break; + } else if (id_param == NULL && id->flags == ID_FLAG_IS_PARAM) { + id_param = id; + } + } + } + + // in functions, turn implicit globals into explicit globals + // compute the index of each local + scope->num_locals = 0; + for (int i = 0; i < scope->id_info_len; i++) { + id_info_t *id = &scope->id_info[i]; + if (scope->kind == SCOPE_CLASS && id->qst == MP_QSTR___class__) { + // __class__ is not counted as a local; if it's used then it becomes a ID_INFO_KIND_CELL + continue; + } + if (scope->kind >= SCOPE_FUNCTION && scope->kind <= SCOPE_GEN_EXPR && id->kind == ID_INFO_KIND_GLOBAL_IMPLICIT) { + id->kind = ID_INFO_KIND_GLOBAL_EXPLICIT; + } + // params always count for 1 local, even if they are a cell + if (id->kind == ID_INFO_KIND_LOCAL || (id->flags & ID_FLAG_IS_PARAM)) { + id->local_num = scope->num_locals++; + } + } + + // compute the index of cell vars + for (int i = 0; i < scope->id_info_len; i++) { + id_info_t *id = &scope->id_info[i]; + // in Micro Python the cells come right after the fast locals + // parameters are not counted here, since they remain at the start + // of the locals, even if they are cell vars + if (id->kind == ID_INFO_KIND_CELL && !(id->flags & ID_FLAG_IS_PARAM)) { + id->local_num = scope->num_locals; + scope->num_locals += 1; + } + } + + // compute the index of free vars + // make sure they are in the order of the parent scope + if (scope->parent != NULL) { + int num_free = 0; + for (int i = 0; i < scope->parent->id_info_len; i++) { + id_info_t *id = &scope->parent->id_info[i]; + if (id->kind == ID_INFO_KIND_CELL || id->kind == ID_INFO_KIND_FREE) { + for (int j = 0; j < scope->id_info_len; j++) { + id_info_t *id2 = &scope->id_info[j]; + if (id2->kind == ID_INFO_KIND_FREE && id->qst == id2->qst) { + assert(!(id2->flags & ID_FLAG_IS_PARAM)); // free vars should not be params + // in Micro Python the frees come first, before the params + id2->local_num = num_free; + num_free += 1; + } + } + } + } + // in Micro Python shift all other locals after the free locals + if (num_free > 0) { + for (int i = 0; i < scope->id_info_len; i++) { + id_info_t *id = &scope->id_info[i]; + if (id->kind != ID_INFO_KIND_FREE || (id->flags & ID_FLAG_IS_PARAM)) { + id->local_num += num_free; + } + } + scope->num_pos_args += num_free; // free vars are counted as params for passing them into the function + scope->num_locals += num_free; + } + } +} + +mp_obj_t mp_compile(mp_parse_tree_t *parse_tree, qstr source_file, uint emit_opt, bool is_repl) { + // put compiler state on the stack, it's relatively small + compiler_t comp_state = {0}; + compiler_t *comp = &comp_state; + + comp->source_file = source_file; + comp->is_repl = is_repl; + comp->co_data = parse_tree->co_data; + + // create the array of scopes + comp->num_scopes = pt_small_int_value(pt_next(parse_tree->root)); + comp->scopes = m_new0(scope_t*, comp->num_scopes); + + // create the module scope + scope_new_and_link(comp, 0, SCOPE_MODULE, parse_tree->root, emit_opt); + + // create standard emitter; it's used at least for MP_PASS_SCOPE + emit_t *emit_bc = emit_bc_new(); + + // compile pass 1 + comp->emit = emit_bc; + #if MICROPY_EMIT_NATIVE + comp->emit_method_table = &emit_bc_method_table; + #endif + uint max_num_labels = 0; + + // grrr: scope for nested comp_for's are not used, unless they are parenthesised + // and become individual generators; in this case they are parsed in the wrong + // direction for allocation of scope id + bool keep_going = true; + while (keep_going) { + keep_going = false; + + for (uint i = 0; i < comp->num_scopes && comp->compile_error == MP_OBJ_NULL; ++i) { + scope_t *s = comp->scopes[i]; + if (s == NULL) { continue; } // no scope (yet?) + if (s->raw_code != NULL) { continue; } // scope already did pass 1 + keep_going = true; + s->raw_code = mp_emit_glue_new_raw_code(); + if (false) { +#if MICROPY_EMIT_INLINE_THUMB + } else if (s->emit_options == MP_EMIT_OPT_ASM_THUMB) { + compile_scope_inline_asm(comp, s, MP_PASS_SCOPE); +#endif + } else { + compile_scope(comp, s, MP_PASS_SCOPE); + } + + // update maximim number of labels needed + if (comp->next_label > max_num_labels) { + max_num_labels = comp->next_label; + } + } + } + + // compute some things related to scope and identifiers + for (uint i = 0; i < comp->num_scopes && comp->compile_error == MP_OBJ_NULL; ++i) { + scope_t *s = comp->scopes[i]; + if (s == NULL) { continue; } // TODO scope for nested comp_for's are not used + scope_compute_things(s); + } + + // set max number of labels now that it's calculated + emit_bc_set_max_num_labels(emit_bc, max_num_labels); + + // compile pass 2 and 3 +#if MICROPY_EMIT_NATIVE + emit_t *emit_native = NULL; +#endif +#if MICROPY_EMIT_INLINE_THUMB + emit_inline_asm_t *emit_inline_thumb = NULL; +#endif + for (uint i = 0; i < comp->num_scopes && comp->compile_error == MP_OBJ_NULL; ++i) { + scope_t *s = comp->scopes[i]; + if (s == NULL) { continue; } + if (false) { + // dummy + +#if MICROPY_EMIT_INLINE_THUMB + } else if (s->emit_options == MP_EMIT_OPT_ASM_THUMB) { + // inline assembly for thumb + if (emit_inline_thumb == NULL) { + emit_inline_thumb = emit_inline_thumb_new(max_num_labels); + } + comp->emit = NULL; + comp->emit_inline_asm = emit_inline_thumb; + comp->emit_inline_asm_method_table = &emit_inline_thumb_method_table; + compile_scope_inline_asm(comp, s, MP_PASS_CODE_SIZE); + if (comp->compile_error == MP_OBJ_NULL) { + compile_scope_inline_asm(comp, s, MP_PASS_EMIT); + } +#endif + + } else { + + // choose the emit type + + switch (s->emit_options) { + +#if MICROPY_EMIT_NATIVE + case MP_EMIT_OPT_NATIVE_PYTHON: + case MP_EMIT_OPT_VIPER: +#if MICROPY_EMIT_X64 + if (emit_native == NULL) { + emit_native = emit_native_x64_new(&comp->compile_error, max_num_labels); + } + comp->emit_method_table = &emit_native_x64_method_table; +#elif MICROPY_EMIT_X86 + if (emit_native == NULL) { + emit_native = emit_native_x86_new(&comp->compile_error, max_num_labels); + } + comp->emit_method_table = &emit_native_x86_method_table; +#elif MICROPY_EMIT_THUMB + if (emit_native == NULL) { + emit_native = emit_native_thumb_new(&comp->compile_error, max_num_labels); + } + comp->emit_method_table = &emit_native_thumb_method_table; +#elif MICROPY_EMIT_ARM + if (emit_native == NULL) { + emit_native = emit_native_arm_new(&comp->compile_error, max_num_labels); + } + comp->emit_method_table = &emit_native_arm_method_table; +#endif + comp->emit = emit_native; + EMIT_ARG(set_native_type, MP_EMIT_NATIVE_TYPE_ENABLE, s->emit_options == MP_EMIT_OPT_VIPER, 0); + break; +#endif // MICROPY_EMIT_NATIVE + + default: + comp->emit = emit_bc; + #if MICROPY_EMIT_NATIVE + comp->emit_method_table = &emit_bc_method_table; + #endif + break; + } + + // need a pass to compute stack size + compile_scope(comp, s, MP_PASS_STACK_SIZE); + + // second last pass: compute code size + if (comp->compile_error == MP_OBJ_NULL) { + compile_scope(comp, s, MP_PASS_CODE_SIZE); + } + + // final pass: emit code + if (comp->compile_error == MP_OBJ_NULL) { + compile_scope(comp, s, MP_PASS_EMIT); + } + } + } + + if (comp->compile_error != MP_OBJ_NULL) { + // if there is no line number for the error then use the line + // number for the start of this scope + compile_error_set_line(comp, comp->scope_cur->pn); + // add a traceback to the exception using relevant source info + mp_obj_exception_add_traceback(comp->compile_error, comp->source_file, + comp->compile_error_line, comp->scope_cur->simple_name); + } + + // free the emitters + + emit_bc_free(emit_bc); +#if MICROPY_EMIT_NATIVE + if (emit_native != NULL) { +#if MICROPY_EMIT_X64 + emit_native_x64_free(emit_native); +#elif MICROPY_EMIT_X86 + emit_native_x86_free(emit_native); +#elif MICROPY_EMIT_THUMB + emit_native_thumb_free(emit_native); +#elif MICROPY_EMIT_ARM + emit_native_arm_free(emit_native); +#endif + } +#endif +#if MICROPY_EMIT_INLINE_THUMB + if (emit_inline_thumb != NULL) { + emit_inline_thumb_free(emit_inline_thumb); + } +#endif + + // free the parse tree + mp_parse_tree_clear(parse_tree); + + mp_raw_code_t *outer_raw_code = comp->scopes[0]->raw_code; + + // free the scopes + for (uint i = 0; i < comp->num_scopes; ++i) { + if (comp->scopes[i] == NULL) { continue; } // TODO scope for nested comp_for's are not used + scope_free(comp->scopes[i]); + } + m_del(scope_t*, comp->scopes, comp->num_scopes); + + if (comp->compile_error != MP_OBJ_NULL) { + nlr_raise(comp->compile_error); + } else { + // return function that executes the outer module + return mp_make_function_from_raw_code(outer_raw_code, MP_OBJ_NULL, MP_OBJ_NULL); + } +} diff --git a/py/emit.h b/py/emit.h index 9121e719f78b5..d65864b50305d 100644 --- a/py/emit.h +++ b/py/emit.h @@ -270,10 +270,12 @@ typedef struct _emit_inline_asm_method_table_t { void (*start_pass)(emit_inline_asm_t *emit, pass_kind_t pass, scope_t *scope, mp_obj_t *error_slot); void (*end_pass)(emit_inline_asm_t *emit, mp_uint_t type_sig); mp_uint_t (*count_params)(emit_inline_asm_t *emit, mp_uint_t n_params, mp_parse_node_t *pn_params); + void (*end_pass)(emit_inline_asm_t *emit, mp_uint_t type_sig); + mp_uint_t (*count_params)(emit_inline_asm_t *emit, const byte *p, const byte *ptop); bool (*label)(emit_inline_asm_t *emit, mp_uint_t label_num, qstr label_id); void (*align)(emit_inline_asm_t *emit, mp_uint_t align); void (*data)(emit_inline_asm_t *emit, mp_uint_t bytesize, mp_uint_t val); - void (*op)(emit_inline_asm_t *emit, qstr op, mp_uint_t n_args, mp_parse_node_t *pn_args); + void (*op)(emit_inline_asm_t *emit, qstr op, mp_uint_t n_args, const byte **pn_args); } emit_inline_asm_method_table_t; extern const emit_inline_asm_method_table_t emit_inline_thumb_method_table; diff --git a/py/grammar.h b/py/grammar.h index a7d1b41e6cbff..004e14826f3c3 100644 --- a/py/grammar.h +++ b/py/grammar.h @@ -180,7 +180,7 @@ DEF_RULE(try_stmt_except_and_more, nc, and(3), rule(try_stmt_except_list), opt_r DEF_RULE(try_stmt_except, nc, and(4), tok(KW_EXCEPT), opt_rule(try_stmt_as_name), tok(DEL_COLON), rule(suite)) DEF_RULE(try_stmt_as_name, nc, and(2), rule(test), opt_rule(as_name)) DEF_RULE(try_stmt_except_list, nc, one_or_more, rule(try_stmt_except)) -DEF_RULE(try_stmt_finally, nc, and(3), tok(KW_FINALLY), tok(DEL_COLON), rule(suite)) +DEF_RULE(try_stmt_finally, nc, ident | and(3), tok(KW_FINALLY), tok(DEL_COLON), rule(suite)) DEF_RULE(else_stmt, nc, ident | and(3), tok(KW_ELSE), tok(DEL_COLON), rule(suite)) DEF_RULE(with_stmt, c(with_stmt), and(4), tok(KW_WITH), rule(with_stmt_list), tok(DEL_COLON), rule(suite)) DEF_RULE(with_stmt_list, nc, list, rule(with_item), tok(DEL_COMMA)) @@ -247,8 +247,7 @@ DEF_RULE(power_dbl_star, nc, ident | and(2), tok(OP_DBL_STAR), rule(factor)) // testlist_comp: (test|star_expr) ( comp_for | (',' (test|star_expr))* [','] ) // trailer: '(' [arglist] ')' | '[' subscriptlist ']' | '.' NAME -DEF_RULE(atom, nc, or(11), tok(NAME), tok(INTEGER), tok(FLOAT_OR_IMAG), rule(atom_string), tok(ELLIPSIS), tok(KW_NONE), tok(KW_TRUE), tok(KW_FALSE), rule(atom_paren), rule(atom_bracket), rule(atom_brace)) -DEF_RULE(atom_string, c(atom_string), one_or_more, rule(string_or_bytes)) +DEF_RULE(atom, nc, or(12), tok(NAME), tok(INTEGER), tok(FLOAT_OR_IMAG), tok(STRING), tok(BYTES), tok(ELLIPSIS), tok(KW_NONE), tok(KW_TRUE), tok(KW_FALSE), rule(atom_paren), rule(atom_bracket), rule(atom_brace)) DEF_RULE(string_or_bytes, nc, or(2), tok(STRING), tok(BYTES)) DEF_RULE(atom_paren, c(atom_paren), and(3), tok(DEL_PAREN_OPEN), opt_rule(atom_2b), tok(DEL_PAREN_CLOSE)) DEF_RULE(atom_2b, nc, or(2), rule(yield_expr), rule(testlist_comp)) @@ -276,7 +275,7 @@ DEF_RULE(subscript_3, c(subscript_3), and(2), tok(DEL_COLON), opt_rule(subscript DEF_RULE(subscript_3b, nc, or(2), rule(subscript_3c), rule(subscript_3d)) DEF_RULE(subscript_3c, nc, and(2), tok(DEL_COLON), opt_rule(test)) DEF_RULE(subscript_3d, nc, and(2), rule(test), opt_rule(sliceop)) -DEF_RULE(sliceop, nc, and(2), tok(DEL_COLON), opt_rule(test)) +DEF_RULE(sliceop, nc, ident | and(2), tok(DEL_COLON), opt_rule(test)) #else DEF_RULE(subscriptlist, c(generic_tuple), list_with_end, rule(test), tok(DEL_COMMA)) #endif diff --git a/py/parse.h b/py/parse.h index f518b9bb66ae0..c4ac15ffcf381 100644 --- a/py/parse.h +++ b/py/parse.h @@ -23,6 +23,7 @@ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN * THE SOFTWARE. */ +#include "py/parse2.h" #ifndef __MICROPY_INCLUDED_PY_PARSE_H__ #define __MICROPY_INCLUDED_PY_PARSE_H__ diff --git a/py/parse2.c b/py/parse2.c new file mode 100644 index 0000000000000..ce649270c0e50 --- /dev/null +++ b/py/parse2.c @@ -0,0 +1,1400 @@ +/* + * This file is part of the Micro Python project, http://micropython.org/ + * + * The MIT License (MIT) + * + * Copyright (c) 2013-2015 Damien P. George + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ + +#include +#include +#include +#include +#include + +#include "py/nlr.h" +#include "py/lexer.h" +#include "py/parse.h" +#include "py/parsenum.h" +#include "py/smallint.h" + +#define RULE_ACT_ARG_MASK (0x0f) +#define RULE_ACT_KIND_MASK (0x30) +#define RULE_ACT_ALLOW_IDENT (0x40) +#define RULE_ACT_ADD_BLANK (0x80) +#define RULE_ACT_OR (0x10) +#define RULE_ACT_AND (0x20) +#define RULE_ACT_LIST (0x30) + +#define RULE_ARG_KIND_MASK (0xf000) +#define RULE_ARG_ARG_MASK (0x0fff) +#define RULE_ARG_TOK (0x1000) +#define RULE_ARG_RULE (0x2000) +#define RULE_ARG_OPT_RULE (0x3000) + +#define ADD_BLANK_NODE(rule) ((rule->act & RULE_ACT_ADD_BLANK) != 0) + +// (un)comment to use rule names; for debugging +//#define USE_RULE_NAME (1) + +typedef struct _rule_t { + byte rule_id; + byte act; +#ifdef USE_RULE_NAME + const char *rule_name; +#endif + uint16_t arg[]; +} rule_t; + +enum { +#define DEF_RULE(rule, comp, kind, ...) RULE_##rule, +#include "py/grammar.h" +#undef DEF_RULE + RULE_maximum_number_of, +}; + +#define ident (RULE_ACT_ALLOW_IDENT) +#define blank (RULE_ACT_ADD_BLANK) +#define or(n) (RULE_ACT_OR | n) +#define and(n) (RULE_ACT_AND | n) +#define one_or_more (RULE_ACT_LIST | 2) +#define list (RULE_ACT_LIST | 1) +#define list_with_end (RULE_ACT_LIST | 3) +#define tok(t) (RULE_ARG_TOK | MP_TOKEN_##t) +#define rule(r) (RULE_ARG_RULE | RULE_##r) +#define opt_rule(r) (RULE_ARG_OPT_RULE | RULE_##r) +#ifdef USE_RULE_NAME +#define DEF_RULE(rule, comp, kind, ...) static const rule_t rule_##rule = { RULE_##rule, kind, #rule, { __VA_ARGS__ } }; +#else +#define DEF_RULE(rule, comp, kind, ...) static const rule_t rule_##rule = { RULE_##rule, kind, { __VA_ARGS__ } }; +#endif +#include "py/grammar.h" +#undef or +#undef and +#undef list +#undef list_with_end +#undef tok +#undef rule +#undef opt_rule +#undef one_or_more +#undef DEF_RULE + +STATIC const rule_t *rules[] = { +#define DEF_RULE(rule, comp, kind, ...) &rule_##rule, +#include "py/grammar.h" +#undef DEF_RULE +}; + +typedef struct _rule_stack_t { + mp_uint_t src_line : BITS_PER_WORD - 8; // maximum bits storing source line number + mp_uint_t rule_id : 8; // this must be large enough to fit largest rule number + mp_uint_t arg_i; // this dictates the maximum nodes in a "list" of things + size_t pt_off; +} rule_stack_t; + +typedef struct _mp_parse_chunk_t { + mp_uint_t alloc; + union { + mp_uint_t used; + struct _mp_parse_chunk_t *next; + } union_; + byte data[]; +} mp_parse_chunk_t; + +typedef enum { + PARSE_ERROR_NONE = 0, + PARSE_ERROR_MEMORY, + PARSE_ERROR_CONST, +} parse_error_t; + +typedef struct _parser_t { + parse_error_t parse_error; + + mp_uint_t rule_stack_alloc; + mp_uint_t rule_stack_top; + rule_stack_t *rule_stack; + + mp_uint_t cur_scope_id; + + mp_uint_t co_alloc; + mp_uint_t co_used; + mp_uint_t *co_data; + + mp_lexer_t *lexer; + + mp_parse_tree_t tree; + + #if MICROPY_COMP_CONST + mp_map_t consts; + #endif +} parser_t; + +STATIC void push_rule(parser_t *parser, mp_uint_t src_line, const rule_t *rule, mp_uint_t arg_i, size_t pt_off) { + if (parser->parse_error) { + return; + } + if (parser->rule_stack_top >= parser->rule_stack_alloc) { + rule_stack_t *rs = m_renew_maybe(rule_stack_t, parser->rule_stack, parser->rule_stack_alloc, parser->rule_stack_alloc + MICROPY_ALLOC_PARSE_RULE_INC, true); + if (rs == NULL) { + parser->parse_error = PARSE_ERROR_MEMORY; + return; + } + parser->rule_stack = rs; + parser->rule_stack_alloc += MICROPY_ALLOC_PARSE_RULE_INC; + } + rule_stack_t *rs = &parser->rule_stack[parser->rule_stack_top++]; + rs->src_line = src_line; + rs->rule_id = rule->rule_id; + rs->arg_i = arg_i; + rs->pt_off = pt_off; +} + +STATIC void push_rule_from_arg(parser_t *parser, mp_uint_t arg) { + assert((arg & RULE_ARG_KIND_MASK) == RULE_ARG_RULE || (arg & RULE_ARG_KIND_MASK) == RULE_ARG_OPT_RULE); + mp_uint_t rule_id = arg & RULE_ARG_ARG_MASK; + assert(rule_id < RULE_maximum_number_of); + push_rule(parser, parser->lexer->tok_line, rules[rule_id], 0, 0); +} + +STATIC void pop_rule(parser_t *parser, const rule_t **rule, mp_uint_t *arg_i, mp_uint_t *src_line, size_t *pt_off) { + assert(!parser->parse_error); + parser->rule_stack_top -= 1; + *rule = rules[parser->rule_stack[parser->rule_stack_top].rule_id]; + *arg_i = parser->rule_stack[parser->rule_stack_top].arg_i; + *src_line = parser->rule_stack[parser->rule_stack_top].src_line; + *pt_off = parser->rule_stack[parser->rule_stack_top].pt_off; +} + +typedef struct _pt_t { + vstr_t vv; +} pt_t; + +STATIC pt_t *pt_new(void) { + pt_t *pt = m_new_obj(pt_t); + vstr_init(&pt->vv, 16); + return pt; +} + +STATIC byte *pt_raw_add_blank(pt_t *pt, size_t nbytes) { + return (byte*)vstr_add_len(&pt->vv, nbytes); +} + +STATIC byte *pt_raw_ins_blank(pt_t *pt, size_t pt_off, size_t nbytes) { + return (byte*)vstr_ins_blank_bytes(&pt->vv, pt_off, nbytes); +} + +STATIC void pt_raw_truncate_at(pt_t *pt, size_t pt_off) { + pt->vv.len = pt_off; +} + +STATIC int vuint_nbytes(mp_uint_t val) { + int n = 0; + do { + n += 1; + val >>= 7; + } while (val != 0); + return n; +} + +STATIC void vuint_store(byte *p, int nbytes, mp_uint_t val) { + p += nbytes; + *--p = val & 0x7f; + for (--nbytes; nbytes > 0; --nbytes) { + val >>= 7; + *--p = 0x80 | (val & 0x7f); + } +} + +STATIC mp_uint_t vuint_load(const byte **p_in) { + const byte *p = *p_in; + mp_uint_t val = 0; + do { + val = (val << 7) + (*p & 0x7f); + } while ((*p++ & 0x80) != 0); + *p_in = p; + return val; +} + +STATIC byte *pt_advance(const byte *p, bool full_rule) { + switch (*p++) { + case MP_PT_NULL: + break; + case MP_PT_TOKEN: + p += 1; + break; + case MP_PT_SMALL_INT: + p += BYTES_PER_WORD; + break; + case MP_PT_STRING: + p += 2; + break; + case MP_PT_BYTES: + p += 2; + break; + case MP_PT_CONST_OBJECT: + vuint_load(&p); + break; + default: + if (p[-1] < MP_PT_RULE_BASE) { + // MP_PT_ID_BASE + p += 1; + } else { + // MP_PT_RULE_BASE + vuint_load(&p); + uint32_t n = vuint_load(&p); + if (full_rule) { + p += n; + } + } + break; + } + return (byte*)p; +} + +// TODO this could perhaps allow *p to be null and in that case return null? +const byte *mp_parse_node_extract_list(const byte **p, mp_uint_t pn_kind) { + if (pt_is_null(*p)) { + *p += 1; + return *p; + } else if (!pt_is_any_rule(*p)) { + return pt_advance(*p, true); + } else { + if (!pt_is_rule(*p, pn_kind)) { + return pt_advance(*p, true); + } else { + const byte *ptop; + *p = pt_rule_extract_top(*p, &ptop); + return ptop; + } + } +} + +/* +const byte *pt_extract_id(const byte *p, qstr *qst) { + //assert(*p == MP_PT_ID_BASE); + *qst = p[1] | ((p[0] - MP_PT_ID_BASE) << 8); + return p + 2; +} +*/ + +const byte *pt_extract_const_obj(const byte *p, mp_uint_t *idx) { + assert(*p == MP_PT_CONST_OBJECT); + p += 1; + *idx = vuint_load(&p); + return p; +} + +const byte *pt_get_small_int(const byte *p, mp_int_t *val) { + assert(*p == MP_PT_SMALL_INT); + *val = 0; + for (size_t i = 0; i < BYTES_PER_WORD; i++) { + *val |= (mp_int_t)*++p << (8 * i); + } + return p + 1; +} + +mp_int_t pt_small_int_value(const byte *p) { + mp_int_t val; + pt_get_small_int(p, &val); + return val; +} + +int pt_num_nodes(const byte *p, const byte *ptop) { + int n = 0; + while (p < ptop) { + n += 1; + p = pt_advance(p, true); + } + return n; +} + +const byte *pt_next(const byte *p) { + return pt_advance(p, true); +} + +const byte *pt_rule_first(const byte *p) { + return pt_advance(p, false); +} + +#if 0 +void pt_show(pt_t *pt) { + const byte *top = (byte*)pt->vv.buf + pt->vv.len; + /* + for (const byte *p = (byte*)pt->buf; p < top; ++p) { + if ((p - (byte*)pt->buf) % 20 == 0) { + printf("\n"); + } + printf("%02x ", *p); + } + printf("\n"); + */ + for (const byte *p = (byte*)pt->vv.buf; p < top;) { + printf("%04u ", (uint)(p - (byte*)pt->vv.buf)); + const byte *p2 = pt_advance(p, false); + for (const byte *p3 = p; p3 < p2; ++p3) { + printf("%02x ", *p3); + } + for (int i = 8 - (p2 - p); i > 0; --i) { + printf(" "); + } + switch (*p) { + case MP_PT_NULL: + printf("NULL\n"); + break; + case MP_PT_TOKEN: + printf("TOKEN %u\n", p[1]); + break; + case MP_PT_SMALL_INT: + printf("SMALL_INT " INT_FMT "\n", pt_small_int_value(p)); + break; + case MP_PT_STRING: + printf("STRING %s\n", qstr_str(p[1] | (p[2] << 8))); + break; + case MP_PT_BYTES: + printf("BYTES %s\n", qstr_str(p[1] | (p[2] << 8))); + break; + case MP_PT_CONST_OBJECT: + printf("CONST_OBJECT\n"); + break; + default: + if (p[0] < MP_PT_RULE_BASE) { + // MP_PT_ID_BASE + printf("ID %s\n", qstr_str(p[1] | ((p[0] - MP_PT_ID_BASE) << 8))); + } else { + // MP_PT_RULE_BASE + byte rule_id = p[0] - MP_PT_RULE_BASE; + const byte *p4 = p + 1; + uint32_t src_line = vuint_load(&p4); + uint32_t n = vuint_load(&p4); + #if USE_RULE_NAME + printf("RULE %s line=%u bytes=%u\n", rules[rule_id]->rule_name, src_line, n); + #else + printf("RULE %d line=%u bytes=%u\n", rule_id, src_line, n); + #endif + } + break; + } + p = p2; + } +} +#endif + +STATIC void pt_add_null(pt_t *pt) { + *pt_raw_add_blank(pt, 1) = MP_PT_NULL; +} + +STATIC void pt_add_kind_byte(pt_t *pt, byte kind, byte b) { + byte *buf = pt_raw_add_blank(pt, 2); + buf[0] = kind; + buf[1] = b; +} + +STATIC void pt_add_kind_qstr(pt_t *pt, byte kind, qstr qst) { + if (kind == MP_PT_ID_BASE) { + assert((qst >> 12) == 0); + byte *buf = pt_raw_add_blank(pt, 2); + buf[0] = MP_PT_ID_BASE + (qst >> 8); + buf[1] = qst; + } else { + assert((qst >> 16) == 0); + byte *buf = pt_raw_add_blank(pt, 3); + buf[0] = kind; + buf[1] = qst; + buf[2] = qst >> 8; + } +} + +// valid for up to BYTES_PER_WORD=8 +const byte pt_const_int0[] = {MP_PT_SMALL_INT, 0, 0, 0, 0, 0, 0, 0, 0}; + +STATIC void pt_add_kind_int(pt_t *pt, byte kind, mp_int_t val) { + byte *buf = pt_raw_add_blank(pt, 1 + BYTES_PER_WORD); + buf[0] = kind; + for (size_t i = 0; i < BYTES_PER_WORD; ++i) { + buf[i + 1] = val; + val >>= 8; + } +} + +STATIC void pt_del_tail_bytes(pt_t *pt, size_t nbytes) { + vstr_cut_tail_bytes(&pt->vv, nbytes); +} + +STATIC const byte *pt_del_byte(pt_t *pt, const byte *p) { + vstr_cut_out_bytes(&pt->vv, p - (byte*)pt->vv.buf, 1); + return p; +} + +#if MICROPY_COMP_MODULE_CONST +#include "py/builtin.h" +STATIC const mp_map_elem_t mp_constants_table[] = { + #if MICROPY_PY_UCTYPES + { MP_OBJ_NEW_QSTR(MP_QSTR_uctypes), (mp_obj_t)&mp_module_uctypes }, + #endif + // Extra constants as defined by a port + MICROPY_PORT_CONSTANTS +}; +STATIC MP_DEFINE_CONST_MAP(mp_constants_map, mp_constants_table); +#endif + +#if MICROPY_COMP_CONST_FOLDING +STATIC bool fold_constants(parser_t *parser, pt_t *pt, size_t pt_off, const rule_t *rule) { + (void)parser; + + // this code does folding of arbitrary integer expressions, eg 1 + 2 * 3 + 4 + // it does not do partial folding, eg 1 + 2 + x -> 3 + x + + mp_int_t arg0; + if (rule->rule_id == RULE_expr + || rule->rule_id == RULE_xor_expr + || rule->rule_id == RULE_and_expr) { + // folding for binary ops: | ^ & + const byte *p = (byte*)pt->vv.buf + pt_off; + const byte *ptop = (byte*)pt->vv.buf + pt->vv.len; + if (*p != MP_PT_SMALL_INT) { + return false; + } + p = pt_get_small_int(p, &arg0); + while (p != ptop) { + if (*p != MP_PT_SMALL_INT) { + return false; + } + mp_int_t arg1; + p = pt_get_small_int(p, &arg1); + if (rule->rule_id == RULE_expr) { + // int | int + arg0 |= arg1; + } else if (rule->rule_id == RULE_xor_expr) { + // int ^ int + arg0 ^= arg1; + } else if (rule->rule_id == RULE_and_expr) { + // int & int + arg0 &= arg1; + } + } + } else if (rule->rule_id == RULE_shift_expr + || rule->rule_id == RULE_arith_expr + || rule->rule_id == RULE_term) { + // folding for binary ops: << >> + - * / % // + const byte *p = (byte*)pt->vv.buf + pt_off; + const byte *ptop = (byte*)pt->vv.buf + pt->vv.len; + if (*p != MP_PT_SMALL_INT) { + return false; + } + p = pt_get_small_int(p, &arg0); + while (p != ptop) { + p += 1; // it's a token + byte tok = *p++; + if (*p != MP_PT_SMALL_INT) { + return false; + } + mp_int_t arg1; + p = pt_get_small_int(p, &arg1); + if (tok == MP_TOKEN_OP_DBL_LESS) { + // int << int + if (arg1 >= (mp_int_t)BITS_PER_WORD + || arg0 > (MP_SMALL_INT_MAX >> arg1) + || arg0 < (MP_SMALL_INT_MIN >> arg1)) { + return false; + } + arg0 <<= arg1; + } else if (tok == MP_TOKEN_OP_DBL_MORE) { + // int >> int + if (arg1 >= (mp_int_t)BITS_PER_WORD) { + // Shifting to big amounts is underfined behavior + // in C and is CPU-dependent; propagate sign bit. + arg1 = BITS_PER_WORD - 1; + } + arg0 >>= arg1; + } else if (tok == MP_TOKEN_OP_PLUS) { + // int + int + arg0 += arg1; + } else if (tok == MP_TOKEN_OP_MINUS) { + // int - int + arg0 -= arg1; + } else if (tok == MP_TOKEN_OP_STAR) { + // int * int + if (mp_small_int_mul_overflow(arg0, arg1)) { + return false; + } + arg0 *= arg1; + } else if (tok == MP_TOKEN_OP_SLASH) { + // int / int + return false; + } else if (tok == MP_TOKEN_OP_PERCENT) { + // int % int + if (arg1 == 0) { + return false; + } + arg0 = mp_small_int_modulo(arg0, arg1); + } else { + assert(tok == MP_TOKEN_OP_DBL_SLASH); // should be + // int // int + if (arg1 == 0) { + return false; + } + arg0 = mp_small_int_floor_divide(arg0, arg1); + } + if (!MP_SMALL_INT_FITS(arg0)) { + return false; + } + } + } else if (rule->rule_id == RULE_factor_2) { + // folding for unary ops: + - ~ + const byte *p = (byte*)pt->vv.buf + pt_off; + p += 1; // it's a token + byte tok = *p++; + if (*p != MP_PT_SMALL_INT) { + return false; + } + arg0 = pt_small_int_value(p); + if (tok == MP_TOKEN_OP_PLUS) { + // +int + } else if (tok == MP_TOKEN_OP_MINUS) { + // -int + arg0 = -arg0; + if (!MP_SMALL_INT_FITS(arg0)) { + return false; + } + } else { + assert(tok == MP_TOKEN_OP_TILDE); // should be + // ~int + arg0 = ~arg0; + } + + #if 0&&MICROPY_COMP_CONST + } else if (rule->rule_id == RULE_expr_stmt) { + mp_parse_node_t pn1 = peek_result(parser, 0); + if (!MP_PARSE_NODE_IS_NULL(pn1) + && !(MP_PARSE_NODE_IS_STRUCT_KIND(pn1, RULE_expr_stmt_augassign) + || MP_PARSE_NODE_IS_STRUCT_KIND(pn1, RULE_expr_stmt_assign_list))) { + // this node is of the form = + mp_parse_node_t pn0 = peek_result(parser, 1); + if (MP_PARSE_NODE_IS_ID(pn0) + && MP_PARSE_NODE_IS_STRUCT_KIND(pn1, RULE_power) + && MP_PARSE_NODE_IS_ID(((mp_parse_node_struct_t*)pn1)->nodes[0]) + && MP_PARSE_NODE_LEAF_ARG(((mp_parse_node_struct_t*)pn1)->nodes[0]) == MP_QSTR_const + && MP_PARSE_NODE_IS_STRUCT_KIND(((mp_parse_node_struct_t*)pn1)->nodes[1], RULE_trailer_paren) + && MP_PARSE_NODE_IS_NULL(((mp_parse_node_struct_t*)pn1)->nodes[2]) + ) { + // code to assign dynamic constants: id = const(value) + + // get the id + qstr id = MP_PARSE_NODE_LEAF_ARG(pn0); + + // get the value + mp_parse_node_t pn_value = ((mp_parse_node_struct_t*)((mp_parse_node_struct_t*)pn1)->nodes[1])->nodes[0]; + if (!MP_PARSE_NODE_IS_SMALL_INT(pn_value)) { + parser->parse_error = PARSE_ERROR_CONST; + return false; + } + mp_int_t value = MP_PARSE_NODE_LEAF_SMALL_INT(pn_value); + + // store the value in the table of dynamic constants + mp_map_elem_t *elem = mp_map_lookup(&parser->consts, MP_OBJ_NEW_QSTR(id), MP_MAP_LOOKUP_ADD_IF_NOT_FOUND); + assert(elem->value == MP_OBJ_NULL); + elem->value = MP_OBJ_NEW_SMALL_INT(value); + + // replace const(value) with value + pop_result(parser); + push_result_node(parser, pn_value); + + // finished folding this assignment, but we still want it to be part of the tree + return false; + } + } + return false; + #endif + + #if 0&&MICROPY_COMP_MODULE_CONST + } else if (rule->rule_id == RULE_power) { + mp_parse_node_t pn0 = peek_result(parser, 2); + mp_parse_node_t pn1 = peek_result(parser, 1); + mp_parse_node_t pn2 = peek_result(parser, 0); + if (!(MP_PARSE_NODE_IS_ID(pn0) + && MP_PARSE_NODE_IS_STRUCT_KIND(pn1, RULE_trailer_period) + && MP_PARSE_NODE_IS_NULL(pn2))) { + return false; + } + // id1.id2 + // look it up in constant table, see if it can be replaced with an integer + mp_parse_node_struct_t *pns1 = (mp_parse_node_struct_t*)pn1; + assert(MP_PARSE_NODE_IS_ID(pns1->nodes[0])); + qstr q_base = MP_PARSE_NODE_LEAF_ARG(pn0); + qstr q_attr = MP_PARSE_NODE_LEAF_ARG(pns1->nodes[0]); + mp_map_elem_t *elem = mp_map_lookup((mp_map_t*)&mp_constants_map, MP_OBJ_NEW_QSTR(q_base), MP_MAP_LOOKUP); + if (elem == NULL) { + return false; + } + mp_obj_t dest[2]; + mp_load_method_maybe(elem->value, q_attr, dest); + if (!(MP_OBJ_IS_SMALL_INT(dest[0]) && dest[1] == NULL)) { + return false; + } + arg0 = MP_OBJ_SMALL_INT_VALUE(dest[0]); + #endif + + } else { + return false; + } + + // success folding this rule + + pt_raw_truncate_at(pt, pt_off); + pt_add_kind_int(pt, MP_PT_SMALL_INT, arg0); + + return true; +} +#endif + +STATIC void pt_ins_rule(parser_t *parser, pt_t *pt, size_t pt_off, mp_uint_t src_line, const rule_t *rule, mp_uint_t num_args) { + (void)num_args; + + #if MICROPY_COMP_CONST_FOLDING + if (fold_constants(parser, pt, pt_off, rule)) { + // we folded this rule so return straight away + return; + } + #endif + +#if 0 + // TODO partial folding, eg 1 + 2 + x -> 3 + x + mp_int_t arg0; + if (rule->rule_id == RULE_expr + || rule->rule_id == RULE_xor_expr + || rule->rule_id == RULE_and_expr) { + // combined node folding for these rules + const byte *p = (byte*)pt->vv.buf + pt_off; + const byte *ptop = (byte*)pt->vv.buf + pt->vv.len; + if (*p != MP_PT_SMALL_INT) { + goto folding_fail; + } + p = pt_get_small_int(p, &arg0); + while (p != ptop) { + if (*p != MP_PT_SMALL_INT) { + goto folding_fail; + } + mp_int_t arg1; + p = pt_get_small_int(p, &arg1); + if (rule->rule_id == RULE_expr) { + // int | int + arg0 |= arg1; + } else if (rule->rule_id == RULE_xor_expr) { + // int ^ int + arg0 ^= arg1; + } else if (rule->rule_id == RULE_and_expr) { + // int & int + arg0 &= arg1; + } + if (!MP_SMALL_INT_FITS(arg0)) { // check needed? + goto folding_fail; + } + } + } else if (rule->rule_id == RULE_shift_expr + || rule->rule_id == RULE_arith_expr + || rule->rule_id == RULE_term) { + // combined node folding for these rules + const byte *p = (byte*)pt->vv.buf + pt_off; + const byte *ptop = (byte*)pt->vv.buf + pt->vv.len; + if (*p != MP_PT_SMALL_INT) { + goto folding_fail; + } + p = pt_get_small_int(p, &arg0); + while (p != ptop) { + p += 1; // it's a token + byte tok = *p++; + if (*p != MP_PT_SMALL_INT) { + goto folding_fail; + } + mp_int_t arg1; + p = pt_get_small_int(p, &arg1); + if (tok == MP_TOKEN_OP_DBL_LESS) { + // int << int + if (arg1 >= (mp_int_t)BITS_PER_WORD + || arg0 > (MP_SMALL_INT_MAX >> arg1) + || arg0 < (MP_SMALL_INT_MIN >> arg1)) { + goto folding_fail; + } + arg0 <<= arg1; + } else if (tok == MP_TOKEN_OP_DBL_MORE) { + // int >> int + if (arg1 >= (mp_int_t)BITS_PER_WORD) { + // Shifting to big amounts is underfined behavior + // in C and is CPU-dependent; propagate sign bit. + arg1 = BITS_PER_WORD - 1; + } + arg0 >>= arg1; + } else if (tok == MP_TOKEN_OP_PLUS) { + // int + int + arg0 += arg1; + } else if (tok == MP_TOKEN_OP_MINUS) { + // int - int + arg0 -= arg1; + } else if (tok == MP_TOKEN_OP_STAR) { + // int * int + if (mp_small_int_mul_overflow(arg0, arg1)) { + goto folding_fail; + } + arg0 *= arg1; + } else if (tok == MP_TOKEN_OP_SLASH) { + // int / int + goto folding_fail; + } else if (tok == MP_TOKEN_OP_PERCENT) { + // int % int + if (arg1 == 0) { + goto folding_fail; + } + arg0 = mp_small_int_modulo(arg0, arg1); + } else { + assert(tok == MP_TOKEN_OP_DBL_SLASH); // should be + // int // int + if (arg1 == 0) { + goto folding_fail; + } + arg0 = mp_small_int_floor_divide(arg0, arg1); + } + if (!MP_SMALL_INT_FITS(arg0)) { + goto folding_fail; + } + } + } else if (rule->rule_id == RULE_factor_2) { + const byte *p = (byte*)pt->vv.buf + pt_off; + p += 1; // it's a token + byte tok = *p++; + if (*p != MP_PT_SMALL_INT) { + goto folding_fail; + } + arg0 = pt_small_int_value(p); + if (tok == MP_TOKEN_OP_PLUS) { + // +int + } else if (tok == MP_TOKEN_OP_MINUS) { + // -int + arg0 = -arg0; + if (!MP_SMALL_INT_FITS(arg0)) { + goto folding_fail; + } + } else { + assert(tok == MP_TOKEN_OP_TILDE); // should be + // ~int + arg0 = ~arg0; + } + } else { + goto folding_fail; + } + + // success folding this rule + pt_raw_truncate_at(pt, pt_off); + pt_add_kind_int(pt, MP_PT_SMALL_INT, arg0); + return; + +folding_fail:; +#endif + + int extra_node = 0; + /* + if (ADD_BLANK_NODE(rule)) { + extra_node = 1 + BYTES_PER_WORD; // for small int node + } + */ + + size_t nbytes = pt->vv.len + extra_node - pt_off; + int nb1 = vuint_nbytes(src_line); + int nb2 = vuint_nbytes(nbytes); + byte *dest = (byte*)pt_raw_ins_blank(pt, pt_off, 1 + nb1 + nb2 + extra_node); + dest[0] = MP_PT_RULE_BASE + rule->rule_id; + vuint_store(dest + 1, nb1, src_line); + vuint_store(dest + 1 + nb1, nb2, nbytes); + + // insert small int node for scope index + if (extra_node != 0) { + dest[1 + nb1 + nb2] = MP_PT_SMALL_INT; + mp_uint_t val = ++parser->cur_scope_id; + for (size_t i = 0; i < BYTES_PER_WORD; ++i) { + dest[1 + nb1 + nb2 + 1 + i] = val; + val >>= 8; + } + } +} + +STATIC void make_node_const_object(parser_t *parser, pt_t *pt, mp_obj_t obj) { + int nb = vuint_nbytes(parser->co_used); + byte *buf = pt_raw_add_blank(pt, 1 + nb); + buf[0] = MP_PT_CONST_OBJECT; + vuint_store(buf + 1, nb, parser->co_used); + if (parser->co_used >= parser->co_alloc) { + // TODO use m_renew_maybe + size_t alloc = parser->co_alloc + 8; + parser->co_data = m_renew(mp_uint_t, parser->co_data, parser->co_alloc, alloc); + parser->co_alloc = alloc; + } + parser->co_data[parser->co_used++] = (mp_uint_t)obj; +} + +STATIC void make_node_string_bytes(parser_t *parser, pt_t *pt, mp_token_kind_t tok, const char *str, mp_uint_t len) { + mp_obj_t o; + if (tok == MP_TOKEN_STRING) { + o = mp_obj_new_str(str, len, false); + } else { + o = mp_obj_new_bytes((const byte*)str, len); + } + make_node_const_object(parser, pt, o); +} + +STATIC bool pt_add_token(parser_t *parser, pt_t *pt) { + mp_lexer_t *lex = parser->lexer; + if (lex->tok_kind == MP_TOKEN_NAME) { + qstr id = qstr_from_strn(lex->vstr.buf, lex->vstr.len); + #if MICROPY_COMP_CONST + // lookup identifier in table of dynamic constants + mp_map_elem_t *elem = mp_map_lookup(&parser->consts, MP_OBJ_NEW_QSTR(id), MP_MAP_LOOKUP); + if (elem != NULL) { + pt_add_kind_int(pt, MP_PT_SMALL_INT, MP_OBJ_SMALL_INT_VALUE(elem->value)); + } else + #endif + { + pt_add_kind_qstr(pt, MP_PT_ID_BASE, id); + } + } else if (lex->tok_kind == MP_TOKEN_INTEGER) { + mp_obj_t o = mp_parse_num_integer(lex->vstr.buf, lex->vstr.len, 0, lex); + if (MP_OBJ_IS_SMALL_INT(o)) { + pt_add_kind_int(pt, MP_PT_SMALL_INT, MP_OBJ_SMALL_INT_VALUE(o)); + } else { + make_node_const_object(parser, pt, o); + } + } else if (lex->tok_kind == MP_TOKEN_FLOAT_OR_IMAG) { + mp_obj_t o = mp_parse_num_decimal(lex->vstr.buf, lex->vstr.len, true, false, lex); + make_node_const_object(parser, pt, o); + } else if (lex->tok_kind == MP_TOKEN_STRING || lex->tok_kind == MP_TOKEN_BYTES) { + // join adjacent string/bytes literals + mp_token_kind_t tok_kind = lex->tok_kind; + vstr_t vstr; + vstr_init(&vstr, lex->vstr.len); + do { + vstr_add_strn(&vstr, lex->vstr.buf, lex->vstr.len); + mp_lexer_to_next(lex); + } while (lex->tok_kind == tok_kind); + + if (lex->tok_kind == MP_TOKEN_STRING || lex->tok_kind == MP_TOKEN_BYTES) { + return false; + } + + // Don't automatically intern all strings/bytes. doc strings (which are usually large) + // will be discarded by the compiler, and so we shouldn't intern them. + qstr qst = MP_QSTR_NULL; + if (vstr.len <= MICROPY_ALLOC_PARSE_INTERN_STRING_LEN) { + // intern short strings + qst = qstr_from_strn(vstr.buf, vstr.len); + } else { + // check if this string is already interned + qst = qstr_find_strn(vstr.buf, vstr.len); + } + if (qst != MP_QSTR_NULL) { + // qstr exists, make a leaf node + pt_add_kind_qstr(pt, tok_kind == MP_TOKEN_STRING ? MP_PT_STRING : MP_PT_BYTES, qst); + } else { + // not interned, make a node holding a pointer to the string/bytes data + make_node_string_bytes(parser, pt, tok_kind, vstr.buf, vstr.len); + } + vstr_clear(&vstr); + return true; + } else { + pt_add_kind_byte(pt, MP_PT_TOKEN, lex->tok_kind); + } + mp_lexer_to_next(lex); + return true; +} + +const byte *pt_rule_extract_top(const byte *p, const byte **ptop) { + assert(*p >= MP_PT_RULE_BASE); + p++; + vuint_load(&p); + mp_uint_t nbytes = vuint_load(&p); + *ptop = p + nbytes; + return p; +} + +const byte *pt_rule_extract(const byte *p, mp_uint_t *rule_id, mp_uint_t *src_line, const byte **ptop) { + assert(*p >= MP_PT_RULE_BASE); + *rule_id = *p++ - MP_PT_RULE_BASE; + *src_line = vuint_load(&p); + mp_uint_t nbytes = vuint_load(&p); + *ptop = p + nbytes; + return p; +} + +bool pt_is_rule_empty(const byte *p) { + const byte *ptop; + p = pt_rule_extract_top(p, &ptop); + return p == ptop; +} + +mp_parse_tree_t mp_parse(mp_lexer_t *lex, mp_parse_input_kind_t input_kind) { + + // initialise parser and allocate memory for its stacks + parser_t parser; + + parser.parse_error = PARSE_ERROR_NONE; + + parser.rule_stack_alloc = MICROPY_ALLOC_PARSE_RULE_INIT; + parser.rule_stack_top = 0; + parser.rule_stack = m_new_maybe(rule_stack_t, parser.rule_stack_alloc); + + parser.cur_scope_id = 0; + + parser.co_alloc = 0; + parser.co_used = 0; + parser.co_data = NULL; + + parser.lexer = lex; + + parser.tree.chunk = NULL; + + #if MICROPY_COMP_CONST + mp_map_init(&parser.consts, 0); + #endif + + // check if we could allocate the stacks + if (parser.rule_stack == NULL) { + goto memory_error; + } + + // work out the top-level rule to use, and push it on the stack + mp_uint_t top_level_rule; + switch (input_kind) { + case MP_PARSE_SINGLE_INPUT: top_level_rule = RULE_single_input; break; + case MP_PARSE_EVAL_INPUT: top_level_rule = RULE_eval_input; break; + default: top_level_rule = RULE_file_input; + } + push_rule(&parser, lex->tok_line, rules[top_level_rule], 0, 0); + + // parse! + + mp_uint_t n, i; // state for the current rule + size_t pt_off = 0; // state for the current rule + mp_uint_t rule_src_line; // source line for the first token matched by the current rule + bool backtrack = false; + const rule_t *rule = NULL; + pt_t *pt = pt_new(); + + for (;;) { + next_rule: + if (parser.rule_stack_top == 0 || parser.parse_error) { + break; + } + + pop_rule(&parser, &rule, &i, &rule_src_line, &pt_off); + n = rule->act & RULE_ACT_ARG_MASK; + + if (i == 0) { + pt_off = pt->vv.len; + } + + /* + // debugging + printf("depth=%d ", parser.rule_stack_top); + for (int j = 0; j < parser.rule_stack_top; ++j) { + printf(" "); + } + printf("%s n=%d i=%d bt=%d\n", rule->rule_name, n, i, backtrack); + */ + + switch (rule->act & RULE_ACT_KIND_MASK) { + case RULE_ACT_OR: + if (i > 0 && !backtrack) { + goto next_rule; + } else { + backtrack = false; + } + for (; i < n; ++i) { + uint16_t kind = rule->arg[i] & RULE_ARG_KIND_MASK; + if (kind == RULE_ARG_TOK) { + if (lex->tok_kind == (rule->arg[i] & RULE_ARG_ARG_MASK)) { + if (!pt_add_token(&parser, pt)) { + goto syntax_error; + } + goto next_rule; + } + } else { + assert(kind == RULE_ARG_RULE); + if (i + 1 < n) { + push_rule(&parser, rule_src_line, rule, i + 1, pt_off); // save this or-rule + } + push_rule_from_arg(&parser, rule->arg[i]); // push child of or-rule + goto next_rule; + } + } + backtrack = true; + break; + + case RULE_ACT_AND: { + + // failed, backtrack if we can, else syntax error + if (backtrack) { + assert(i > 0); + if ((rule->arg[i - 1] & RULE_ARG_KIND_MASK) == RULE_ARG_OPT_RULE) { + // an optional rule that failed, so continue with next arg + pt_add_null(pt); + backtrack = false; + } else { + // a mandatory rule that failed, so propagate backtrack + if (i > 1) { + // already eaten tokens so can't backtrack + goto syntax_error; + } else { + goto next_rule; + } + } + } + + // progress through the rule + for (; i < n; ++i) { + switch (rule->arg[i] & RULE_ARG_KIND_MASK) { + case RULE_ARG_TOK: { + // need to match a token + mp_token_kind_t tok_kind = rule->arg[i] & RULE_ARG_ARG_MASK; + if (lex->tok_kind == tok_kind) { + // matched token + if (tok_kind == MP_TOKEN_NAME) { + pt_add_kind_qstr(pt, MP_PT_ID_BASE, qstr_from_strn(lex->vstr.buf, lex->vstr.len)); + } + if (i == 0 && ADD_BLANK_NODE(rule)) { + pt_add_kind_int(pt, MP_PT_SMALL_INT, ++parser.cur_scope_id); + } + mp_lexer_to_next(lex); + } else { + // failed to match token + if (i > 0) { + // already eaten tokens so can't backtrack + goto syntax_error; + } else { + // this rule failed, so backtrack + backtrack = true; + goto next_rule; + } + } + break; + } + case RULE_ARG_RULE: + case RULE_ARG_OPT_RULE: + default: + push_rule(&parser, rule_src_line, rule, i + 1, pt_off); // save this and-rule + push_rule_from_arg(&parser, rule->arg[i]); // push child of and-rule + goto next_rule; + } + } + + assert(i == n); + + // matched the rule, so now build the corresponding parse_node + + // count number of arguments for the parse_node + i = 0; + bool emit_rule = false; + /* + for (mp_uint_t x = 0; x < n; ++x) { + if ((rule->arg[x] & RULE_ARG_KIND_MASK) == RULE_ARG_TOK) { + mp_token_kind_t tok_kind = rule->arg[x] & RULE_ARG_ARG_MASK; + if (tok_kind >= MP_TOKEN_NAME) { + emit_rule = true; + } + if (tok_kind == MP_TOKEN_NAME) { + // only tokens which were names are pushed to stack + i += 1; + } + } else { + // rules are always pushed + i += 1; + } + } + */ + for (mp_uint_t x = 0; x < n; ++x) { + if ((rule->arg[x] & RULE_ARG_KIND_MASK) == RULE_ARG_TOK) { + mp_token_kind_t tok_kind = rule->arg[x] & RULE_ARG_ARG_MASK; + if (tok_kind >= MP_TOKEN_NAME) { + emit_rule = true; + } + } + } + for (const byte *p = (byte*)pt->vv.buf + pt_off; p < (byte*)pt->vv.buf + pt->vv.len;) { + i += 1; + p = pt_advance(p, true); + } + + #if 0 && !MICROPY_ENABLE_DOC_STRING + // this code discards lonely statements, such as doc strings + if (input_kind != MP_PARSE_SINGLE_INPUT && rule->rule_id == RULE_expr_stmt && peek_result(&parser, 0) == MP_PARSE_NODE_NULL) { + mp_parse_node_t p = peek_result(&parser, 1); + if ((MP_PARSE_NODE_IS_LEAF(p) && !MP_PARSE_NODE_IS_ID(p)) || MP_PARSE_NODE_IS_STRUCT_KIND(p, RULE_string)) { + pop_result(&parser); // MP_PARSE_NODE_NULL + mp_parse_node_t pn = pop_result(&parser); // possibly RULE_string + if (MP_PARSE_NODE_IS_STRUCT(pn)) { + mp_parse_node_struct_t *pns = (mp_parse_node_struct_t *)pn; + if (MP_PARSE_NODE_STRUCT_KIND(pns) == RULE_string) { + m_del(char, (char*)pns->nodes[0], (mp_uint_t)pns->nodes[1]); + } + } + push_result_rule(&parser, rule_src_line, rules[RULE_pass_stmt], 0); + break; + } + } + #endif + + // always emit these rules, even if they have only 1 argument + if (rule->rule_id == RULE_expr_stmt || rule->rule_id == RULE_yield_stmt) { + emit_rule = true; + } + + // if a rule has the RULE_ACT_ALLOW_IDENT bit set then this + // rule should not be emitted if it has only 1 argument + // NOTE: can't set this flag for atom_paren because we need it + // to distinguish, for example, [a,b] from [(a,b)] + if (rule->act & RULE_ACT_ALLOW_IDENT) { + emit_rule = false; + } + + // always emit these rules, and add an extra blank node at the end (to be used by the compiler to store data) + if (ADD_BLANK_NODE(rule)) { + emit_rule = true; + // TODO + //add_result_node(&parser, MP_PARSE_NODE_NULL); + //i += 1; + } + + // count number of non-null nodes + mp_uint_t num_not_null = 0; + mp_uint_t num_trail_null = 0; + { const byte *p = (byte*)pt->vv.buf + pt_off; + for (mp_uint_t x = 0; x < i; ++x) { + if (*p != MP_PT_NULL) { + num_not_null += 1; + num_trail_null = 0; + } else { + num_trail_null += 1; + } + p = pt_advance(p, true); + }} + + if (emit_rule || num_not_null != 1) { + // need to add rule when num_not_null == 0 for, eg, atom_paren, testlist_comp_3b + pt_del_tail_bytes(pt, num_trail_null); // remove trailing null nodes, they are store implicitly + pt_ins_rule(&parser, pt, pt_off, rule_src_line, rule, i - num_trail_null); + } else { + // single result, leave it on stack + const byte *p = (byte*)pt->vv.buf + pt_off; + for (mp_uint_t x = 0; x < i; ++x) { + if (*p == MP_PT_NULL) { + p = pt_del_byte(pt, p); + } else { + p = pt_advance(p, true); + } + } + } + break; + } + + case RULE_ACT_LIST: + default: // nothing else + { + // n=2 is: item item* + // n=1 is: item (sep item)* + // n=3 is: item (sep item)* [sep] + bool had_trailing_sep; + if (backtrack) { + list_backtrack: + had_trailing_sep = false; + if (n == 2) { + if (i == 1) { + // fail on item, first time round; propagate backtrack + goto next_rule; + } else { + // fail on item, in later rounds; finish with this rule + backtrack = false; + } + } else { + if (i == 1) { + // fail on item, first time round; propagate backtrack + goto next_rule; + } else if ((i & 1) == 1) { + // fail on item, in later rounds; have eaten tokens so can't backtrack + if (n == 3) { + // list allows trailing separator; finish parsing list + had_trailing_sep = true; + backtrack = false; + } else { + // list doesn't allowing trailing separator; fail + goto syntax_error; + } + } else { + // fail on separator; finish parsing list + backtrack = false; + } + } + } else { + for (;;) { + mp_uint_t arg = rule->arg[i & 1 & n]; + switch (arg & RULE_ARG_KIND_MASK) { + case RULE_ARG_TOK: + if (lex->tok_kind == (arg & RULE_ARG_ARG_MASK)) { + if (i & 1 & n) { + // separators which are tokens are not pushed to result stack + mp_lexer_to_next(lex); + } else { + pt_add_token(&parser, pt); + } + // got element of list, so continue parsing list + i += 1; + } else { + // couldn't get element of list + i += 1; + backtrack = true; + goto list_backtrack; + } + break; + case RULE_ARG_RULE: + rule_list_no_other_choice: + push_rule(&parser, rule_src_line, rule, i + 1, pt_off); // save this list-rule + push_rule_from_arg(&parser, arg); // push child of list-rule + goto next_rule; + default: + assert(0); + goto rule_list_no_other_choice; // to help flow control analysis + } + } + } + assert(i >= 1); + + // compute number of elements in list, result in i + i -= 1; + if ((n & 1) && (rule->arg[1] & RULE_ARG_KIND_MASK) == RULE_ARG_TOK) { + // don't count separators when they are tokens + i = (i + 1) / 2; + } + + if (i == 1) { + // list matched single item + if (had_trailing_sep) { + // if there was a trailing separator, make a list of a single item + pt_ins_rule(&parser, pt, pt_off, rule_src_line, rule, i); + } else { + // just leave single item on stack (ie don't wrap in a list) + } + } else { + pt_ins_rule(&parser, pt, pt_off, rule_src_line, rule, i); + } + break; + } + } + } + + #if MICROPY_COMP_CONST + mp_map_deinit(&parser.consts); + #endif + + #if 0 + pt_show(pt); + + { + mp_uint_t n_pool, n_qstr, n_str_data_bytes, n_total_bytes; + qstr_pool_info(&n_pool, &n_qstr, &n_str_data_bytes, &n_total_bytes); + printf("qstr pool: n_pool=" UINT_FMT ", n_qstr=" UINT_FMT ", n_str_data_bytes=" + UINT_FMT ", n_total_bytes=" UINT_FMT "\n", + n_pool, n_qstr, n_str_data_bytes, n_total_bytes); + } + #endif + + mp_obj_t exc; + + if (parser.parse_error) { + #if MICROPY_COMP_CONST + if (parser.parse_error == PARSE_ERROR_CONST) { + exc = mp_obj_new_exception_msg(&mp_type_SyntaxError, + "constant must be an integer"); + } else + #endif + { + assert(parser.parse_error == PARSE_ERROR_MEMORY); + memory_error: + exc = mp_obj_new_exception_msg(&mp_type_MemoryError, + "parser could not allocate enough memory"); + } + parser.tree.root = NULL; + } else if ( + lex->tok_kind != MP_TOKEN_END // check we are at the end of the token stream + || pt->vv.len == 0 // check that we got a node (can fail on empty input) + ) { + syntax_error: + if (lex->tok_kind == MP_TOKEN_INDENT) { + exc = mp_obj_new_exception_msg(&mp_type_IndentationError, + "unexpected indent"); + } else if (lex->tok_kind == MP_TOKEN_DEDENT_MISMATCH) { + exc = mp_obj_new_exception_msg(&mp_type_IndentationError, + "unindent does not match any outer indentation level"); + } else if (lex->tok_kind == MP_TOKEN_STRING || lex->tok_kind == MP_TOKEN_BYTES) { + exc = mp_obj_new_exception_msg(&mp_type_SyntaxError, + "cannot mix bytes and nonbytes literals"); + } else { + exc = mp_obj_new_exception_msg(&mp_type_SyntaxError, + "invalid syntax"); + } + parser.tree.root = NULL; + } else { + // no errors + + //result_stack_show(parser); + //printf("rule stack alloc: %d\n", parser.rule_stack_alloc); + //printf("result stack alloc: %d\n", parser.result_stack_alloc); + //printf("number of parse nodes allocated: %d\n", num_parse_nodes_allocated); + + // add number of scopes + pt_add_kind_int(pt, MP_PT_SMALL_INT, parser.cur_scope_id + 1); + + // get the root parse node that we created + //assert(parser.result_stack_top == 1); + exc = MP_OBJ_NULL; + parser.tree.root = (byte*)pt->vv.buf; + parser.tree.co_data = parser.co_data; + } + + // free the memory that we don't need anymore + m_del(rule_stack_t, parser.rule_stack, parser.rule_stack_alloc); + // we also free the lexer on behalf of the caller (see below) + + if (exc != MP_OBJ_NULL) { + // had an error so raise the exception + // add traceback to give info about file name and location + // we don't have a 'block' name, so just pass the NULL qstr to indicate this + mp_obj_exception_add_traceback(exc, lex->source_name, lex->tok_line, MP_QSTR_NULL); + mp_lexer_free(lex); + nlr_raise(exc); + } else { + mp_lexer_free(lex); + return parser.tree; + } +} + +void mp_parse_tree_clear(mp_parse_tree_t *tree) { + mp_parse_chunk_t *chunk = tree->chunk; + while (chunk != NULL) { + mp_parse_chunk_t *next = chunk->union_.next; + m_del(byte, chunk, sizeof(mp_parse_chunk_t) + chunk->alloc); + chunk = next; + } +} diff --git a/py/parse2.h b/py/parse2.h new file mode 100644 index 0000000000000..5090cc00c79f6 --- /dev/null +++ b/py/parse2.h @@ -0,0 +1,133 @@ +/* + * This file is part of the Micro Python project, http://micropython.org/ + * + * The MIT License (MIT) + * + * Copyright (c) 2013, 2014 Damien P. George + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ +#ifndef __MICROPY_INCLUDED_PY_PARSE_H__ +#define __MICROPY_INCLUDED_PY_PARSE_H__ + +#include + +#include "py/mpconfig.h" +#include "py/misc.h" +#include "py/qstr.h" + +struct _mp_lexer_t; + +#define MP_PT_NULL (0) +#define MP_PT_TOKEN (1) +#define MP_PT_SMALL_INT (3) +#define MP_PT_STRING (4) +#define MP_PT_BYTES (5) +#define MP_PT_CONST_OBJECT (8) +#define MP_PT_ID_BASE (10) // +16 +#define MP_PT_RULE_BASE (26) // +173-ish + +extern const byte pt_const_int0[]; + +static inline const byte *pt_tok_extract(const byte *p, byte *tok) { + //assert(*p == MP_PT_TOKEN); + p += 1; + *tok = *p++; + return p; +} + +static inline bool pt_is_null(const byte *p) { + return *p == MP_PT_NULL; +} + +static inline bool pt_is_null_with_top(const byte *p, const byte *ptop) { + return p == ptop || *p == MP_PT_NULL; +} + +static inline bool pt_is_small_int(const byte *p) { + return *p == MP_PT_SMALL_INT; +} + +static inline bool pt_is_any_rule(const byte *p) { + return *p >= MP_PT_RULE_BASE; +} + +static inline mp_uint_t pt_rule_extract_rule_id(const byte *p) { + return *p - MP_PT_RULE_BASE; +} + +static inline bool pt_is_any_id(const byte *p) { + return *p >= MP_PT_ID_BASE && *p < MP_PT_RULE_BASE; +} + +static inline bool pt_is_id(const byte *p, qstr qst) { + //assert(*p == MP_PT_ID_BASE); + return qst == ((mp_uint_t)p[1] | (((mp_uint_t)p[0] - MP_PT_ID_BASE) << 8)); +} + +static inline bool pt_is_any_tok(const byte *p) { + return p[0] == MP_PT_TOKEN; +} + +static inline bool pt_is_tok(const byte *p, int tok) { + return p[0] == MP_PT_TOKEN && p[1] == tok; +} + +static inline bool pt_is_rule(const byte *p, int rule) { + return *p == MP_PT_RULE_BASE + rule; +} + +int pt_num_nodes(const byte *p, const byte *ptop); +const byte *pt_next(const byte *p); + +//const byte *pt_extract_id(const byte *p, qstr *qst); +static inline const byte *pt_extract_id(const byte *p, qstr *qst) { + //assert(*p == MP_PT_ID_BASE); + *qst = p[1] | ((p[0] - MP_PT_ID_BASE) << 8); + return p + 2; +} + +const byte *pt_extract_const_obj(const byte *p, mp_uint_t *idx); +mp_int_t pt_small_int_value(const byte *p); +const byte *pt_get_small_int(const byte *p, mp_int_t *val); +const byte *pt_rule_first(const byte *p); +const byte *pt_rule_extract_top(const byte *p, const byte **ptop); +const byte *pt_rule_extract(const byte *p, mp_uint_t *rule_id, mp_uint_t *src_line, const byte **ptop); +bool pt_is_rule_empty(const byte *p); + +const byte *mp_parse_node_extract_list(const byte **p, mp_uint_t pn_kind); + +typedef enum { + MP_PARSE_SINGLE_INPUT, + MP_PARSE_FILE_INPUT, + MP_PARSE_EVAL_INPUT, +} mp_parse_input_kind_t; + +typedef struct _mp_parse_t { + const byte *root; + mp_uint_t *co_data; + struct _mp_parse_chunk_t *chunk; +} mp_parse_tree_t; + +// the parser will raise an exception if an error occurred +// the parser will free the lexer before it returns +mp_parse_tree_t mp_parse(struct _mp_lexer_t *lex, mp_parse_input_kind_t input_kind); +void mp_parse_tree_clear(mp_parse_tree_t *tree); + +#endif // __MICROPY_INCLUDED_PY_PARSE_H__ diff --git a/py/py.mk b/py/py.mk index 899f333c62d16..fff20b9cd8fb7 100644 --- a/py/py.mk +++ b/py/py.mk @@ -75,9 +75,9 @@ PY_O_BASENAME = \ lexer.o \ lexerstr.o \ lexerunix.o \ - parse.o \ + parse2.o \ scope.o \ - compile.o \ + compile2.o \ emitcommon.o \ emitbc.o \ asmx64.o \ diff --git a/py/scope.c b/py/scope.c index 632e5275218bc..e408251733d25 100644 --- a/py/scope.c +++ b/py/scope.c @@ -30,7 +30,7 @@ #if MICROPY_ENABLE_COMPILER -scope_t *scope_new(scope_kind_t kind, mp_parse_node_t pn, qstr source_file, mp_uint_t emit_options) { +scope_t *scope_new(scope_kind_t kind, const byte *pn, qstr source_file, mp_uint_t emit_options) { scope_t *scope = m_new0(scope_t, 1); scope->kind = kind; scope->pn = pn; @@ -41,8 +41,7 @@ scope_t *scope_new(scope_kind_t kind, mp_parse_node_t pn, qstr source_file, mp_u break; case SCOPE_FUNCTION: case SCOPE_CLASS: - assert(MP_PARSE_NODE_IS_STRUCT(pn)); - scope->simple_name = MP_PARSE_NODE_LEAF_ARG(((mp_parse_node_struct_t*)pn)->nodes[0]); + pt_extract_id(pn, &scope->simple_name); // function name break; case SCOPE_LAMBDA: scope->simple_name = MP_QSTR__lt_lambda_gt_; @@ -62,7 +61,6 @@ scope_t *scope_new(scope_kind_t kind, mp_parse_node_t pn, qstr source_file, mp_u default: assert(0); } - scope->raw_code = mp_emit_glue_new_raw_code(); scope->emit_options = emit_options; scope->id_info_alloc = MICROPY_ALLOC_SCOPE_ID_INIT; scope->id_info = m_new(id_info_t, scope->id_info_alloc); diff --git a/py/scope.h b/py/scope.h index fac936a729d70..e210a5a9cf92f 100644 --- a/py/scope.h +++ b/py/scope.h @@ -57,8 +57,7 @@ typedef enum { SCOPE_MODULE, SCOPE_FUNCTION, SCOPE_LAMBDA, SCOPE_LIST_COMP, SCOP typedef struct _scope_t { scope_kind_t kind; struct _scope_t *parent; - struct _scope_t *next; - mp_parse_node_t pn; + const byte *pn; // points to the node after the scope index node qstr source_file; qstr simple_name; mp_raw_code_t *raw_code; @@ -75,7 +74,7 @@ typedef struct _scope_t { id_info_t *id_info; } scope_t; -scope_t *scope_new(scope_kind_t kind, mp_parse_node_t pn, qstr source_file, mp_uint_t emit_options); +scope_t *scope_new(scope_kind_t kind, const byte *pn, qstr source_file, mp_uint_t emit_options); void scope_free(scope_t *scope); id_info_t *scope_find_or_add_id(scope_t *scope, qstr qstr, bool *added); id_info_t *scope_find(scope_t *scope, qstr qstr); From 127d1f5a2b3b64dd4ca658fea8582ab5af8555a7 Mon Sep 17 00:00:00 2001 From: Damien George Date: Tue, 13 Oct 2015 17:55:23 +0100 Subject: [PATCH 03/24] fixing inline assembler (compile2.c) --- py/compile2.c | 106 ++++++++++++++++++++++++++++---------------------- 1 file changed, 60 insertions(+), 46 deletions(-) diff --git a/py/compile2.c b/py/compile2.c index 9db0c2f04d6ee..188d0f3f5e18d 100644 --- a/py/compile2.c +++ b/py/compile2.c @@ -2932,7 +2932,7 @@ STATIC void compile_scope_inline_asm(compiler_t *comp, scope_t *scope, pass_kind comp->next_label = 1; if (scope->kind != SCOPE_FUNCTION) { - compile_syntax_error(comp, MP_PARSE_NODE_NULL, "inline assembler must be a function"); + compile_syntax_error(comp, NULL, "inline assembler must be a function"); return; } @@ -2941,109 +2941,123 @@ STATIC void compile_scope_inline_asm(compiler_t *comp, scope_t *scope, pass_kind } // get the function definition parse node - assert(MP_PARSE_NODE_IS_STRUCT(scope->pn)); - mp_parse_node_struct_t *pns = (mp_parse_node_struct_t*)scope->pn; - assert(MP_PARSE_NODE_STRUCT_KIND(pns) == PN_funcdef); + const byte *p = scope->pn; + assert(pt_is_any_id(p)); + p = pt_next(p); // skip the function name - //qstr f_id = MP_PARSE_NODE_LEAF_ARG(pns->nodes[0]); // function name - - // parameters are in pns->nodes[1] + // parameters are in next node if (comp->pass == MP_PASS_CODE_SIZE) { - mp_parse_node_t *pn_params; - int n_params = mp_parse_node_extract_list(&pns->nodes[1], PN_typedargslist, &pn_params); - scope->num_pos_args = EMIT_INLINE_ASM_ARG(count_params, n_params, pn_params); + const byte *pp = p; + const byte *pptop = mp_parse_node_extract_list(&pp, PN_typedargslist); + scope->num_pos_args = EMIT_INLINE_ASM_ARG(count_params, pp, pptop); if (comp->compile_error != MP_OBJ_NULL) { goto inline_asm_error; } } - assert(MP_PARSE_NODE_IS_NULL(pns->nodes[2])); // type + p = pt_next(p); // skip the parameter list + p = pt_next(p); // skip the return type - mp_parse_node_t pn_body = pns->nodes[3]; // body - mp_parse_node_t *nodes; - int num = mp_parse_node_extract_list(&pn_body, PN_suite_block_stmts, &nodes); + // get the list of statements within the body of the function + const byte *ptop = mp_parse_node_extract_list(&p, PN_suite_block_stmts); - for (int i = 0; i < num; i++) { - assert(MP_PARSE_NODE_IS_STRUCT(nodes[i])); - mp_parse_node_struct_t *pns2 = (mp_parse_node_struct_t*)nodes[i]; - if (MP_PARSE_NODE_STRUCT_KIND(pns2) == PN_pass_stmt) { + for (const byte *p_instr = p; p_instr != ptop; p_instr = pt_next(p_instr)) { + p = p_instr; + if (pt_is_rule(p, PN_pass_stmt)) { // no instructions continue; - } else if (MP_PARSE_NODE_STRUCT_KIND(pns2) != PN_expr_stmt) { + } else if (!pt_is_rule(p, PN_expr_stmt)) { // not an instruction; error not_an_instruction: - compile_syntax_error(comp, nodes[i], "expecting an assembler instruction"); + compile_syntax_error(comp, p, "expecting an assembler instruction"); return; } // check structure of parse node - assert(MP_PARSE_NODE_IS_STRUCT(pns2->nodes[0])); - if (!MP_PARSE_NODE_IS_NULL(pns2->nodes[1])) { + const byte *p_expr_top; + const byte *p_expr = pt_rule_extract_top(p, &p_expr_top); + if (!pt_is_rule(p_expr, PN_power)) { + goto not_an_instruction; + } + if (pt_next(p_expr) != p_expr_top) { goto not_an_instruction; } - pns2 = (mp_parse_node_struct_t*)pns2->nodes[0]; - if (MP_PARSE_NODE_STRUCT_KIND(pns2) != PN_power) { + p_expr = pt_rule_extract_top(p_expr, &p_expr_top); + if (!pt_is_any_id(p_expr)) { goto not_an_instruction; } - if (!MP_PARSE_NODE_IS_ID(pns2->nodes[0])) { + const byte *p_expr_paren = pt_next(p_expr); + if (p_expr_paren == p_expr_top || !pt_is_rule(p_expr_paren, PN_trailer_paren)) { goto not_an_instruction; } - if (!MP_PARSE_NODE_IS_STRUCT_KIND(pns2->nodes[1], PN_trailer_paren)) { + if (pt_next(p_expr_paren) != p_expr_top) { goto not_an_instruction; } - assert(MP_PARSE_NODE_IS_NULL(pns2->nodes[2])); // parse node looks like an instruction // get instruction name and args - qstr op = MP_PARSE_NODE_LEAF_ARG(pns2->nodes[0]); - pns2 = (mp_parse_node_struct_t*)pns2->nodes[1]; // PN_trailer_paren - mp_parse_node_t *pn_arg; - int n_args = mp_parse_node_extract_list(&pns2->nodes[0], PN_arglist, &pn_arg); + qstr op; + pt_extract_id(p_expr, &op); + + const byte *p_args = pt_rule_first(p_expr_paren); + const byte *p_args_top = mp_parse_node_extract_list(&p_args, PN_arglist); + uint n_args = pt_num_nodes(p_args, p_args_top); // emit instructions if (op == MP_QSTR_label) { - if (!(n_args == 1 && MP_PARSE_NODE_IS_ID(pn_arg[0]))) { - compile_syntax_error(comp, nodes[i], "'label' requires 1 argument"); + if (!(n_args == 1 && pt_is_any_id(p_args))) { + compile_syntax_error(comp, p, "'label' requires 1 argument"); return; } uint lab = comp_next_label(comp); if (pass > MP_PASS_SCOPE) { - if (!EMIT_INLINE_ASM_ARG(label, lab, MP_PARSE_NODE_LEAF_ARG(pn_arg[0]))) { - compile_syntax_error(comp, nodes[i], "label redefined"); + qstr id; + pt_extract_id(p_args, &id); + if (!EMIT_INLINE_ASM_ARG(label, lab, id)) { + compile_syntax_error(comp, p, "label redefined"); return; } } } else if (op == MP_QSTR_align) { - if (!(n_args == 1 && MP_PARSE_NODE_IS_SMALL_INT(pn_arg[0]))) { - compile_syntax_error(comp, nodes[i], "'align' requires 1 argument"); + if (!(n_args == 1 && pt_is_small_int(p_args))) { + compile_syntax_error(comp, p, "'align' requires 1 argument"); return; } if (pass > MP_PASS_SCOPE) { - EMIT_INLINE_ASM_ARG(align, MP_PARSE_NODE_LEAF_SMALL_INT(pn_arg[0])); + EMIT_INLINE_ASM_ARG(align, pt_small_int_value(p_args)); } } else if (op == MP_QSTR_data) { - if (!(n_args >= 2 && MP_PARSE_NODE_IS_SMALL_INT(pn_arg[0]))) { - compile_syntax_error(comp, nodes[i], "'data' requires at least 2 arguments"); + if (!(n_args >= 2 && pt_is_small_int(p_args))) { + compile_syntax_error(comp, p, "'data' requires at least 2 arguments"); return; } if (pass > MP_PASS_SCOPE) { - mp_int_t bytesize = MP_PARSE_NODE_LEAF_SMALL_INT(pn_arg[0]); + mp_int_t bytesize; + p_args = pt_get_small_int(p_args, &bytesize); for (uint j = 1; j < n_args; j++) { - if (!MP_PARSE_NODE_IS_SMALL_INT(pn_arg[j])) { - compile_syntax_error(comp, nodes[i], "'data' requires integer arguments"); + if (!pt_is_small_int(p_args)) { + compile_syntax_error(comp, p, "'data' requires integer arguments"); return; } - EMIT_INLINE_ASM_ARG(data, bytesize, MP_PARSE_NODE_LEAF_SMALL_INT(pn_arg[j])); + mp_int_t val; + p_args = pt_get_small_int(p_args, &val); + EMIT_INLINE_ASM_ARG(data, bytesize, val); } } } else { if (pass > MP_PASS_SCOPE) { + if (n_args > 3) { + goto not_an_instruction; + } + const byte *pn_arg[3]; + pn_arg[0] = p_args; + pn_arg[1] = pt_next(pn_arg[0]); + pn_arg[2] = pt_next(pn_arg[1]); EMIT_INLINE_ASM_ARG(op, op, n_args, pn_arg); } } if (comp->compile_error != MP_OBJ_NULL) { - pns = pns2; // this is the parse node that had the error goto inline_asm_error; } } @@ -3055,7 +3069,7 @@ STATIC void compile_scope_inline_asm(compiler_t *comp, scope_t *scope, pass_kind if (comp->compile_error != MP_OBJ_NULL) { // inline assembler had an error; set line for its exception inline_asm_error: - comp->compile_error_line = pns->source_line; + compile_error_set_line(comp, p); } } #endif From 6cdf8167f70bf6c782d91a4b1bb5fa90e4bcf944 Mon Sep 17 00:00:00 2001 From: Damien George Date: Tue, 13 Oct 2015 17:55:30 +0100 Subject: [PATCH 04/24] fixing inline assembler (emitinlinethumb.c) --- py/emitinlinethumb.c | 132 +++++++++++++++++++++++-------------------- 1 file changed, 70 insertions(+), 62 deletions(-) diff --git a/py/emitinlinethumb.c b/py/emitinlinethumb.c index 90a68caa6da36..9ee1ab170d96e 100644 --- a/py/emitinlinethumb.c +++ b/py/emitinlinethumb.c @@ -95,18 +95,21 @@ STATIC void emit_inline_thumb_end_pass(emit_inline_asm_t *emit, mp_uint_t type_s } } -STATIC mp_uint_t emit_inline_thumb_count_params(emit_inline_asm_t *emit, mp_uint_t n_params, mp_parse_node_t *pn_params) { - if (n_params > 4) { - emit_inline_thumb_error_msg(emit, "can only have up to 4 parameters to Thumb assembly"); - return 0; - } - for (mp_uint_t i = 0; i < n_params; i++) { - if (!MP_PARSE_NODE_IS_ID(pn_params[i])) { +STATIC mp_uint_t emit_inline_thumb_count_params(emit_inline_asm_t *emit, const byte *p, const byte *ptop) { + mp_uint_t n_params = 0; + while (p != ptop) { + if (++n_params > 4) { + emit_inline_thumb_error_msg(emit, "can only have up to 4 parameters to Thumb assembly"); + return 0; + } + if (!pt_is_any_id(p)) { emit_inline_thumb_error_msg(emit, "parameters must be registers in sequence r0 to r3"); return 0; } - const char *p = qstr_str(MP_PARSE_NODE_LEAF_ARG(pn_params[i])); - if (!(strlen(p) == 2 && p[0] == 'r' && p[1] == '0' + i)) { + qstr qst; + p = pt_extract_id(p, &qst); + const char *param = qstr_str(qst); + if (!(strlen(param) == 2 && param[0] == 'r' && param[1] == '0' + n_params - 1)) { emit_inline_thumb_error_msg(emit, "parameters must be registers in sequence r0 to r3"); return 0; } @@ -171,16 +174,17 @@ STATIC const special_reg_name_t special_reg_name_table[] = { // return empty string in case of error, so we can attempt to parse the string // without a special check if it was in fact a string -STATIC const char *get_arg_str(mp_parse_node_t pn) { - if (MP_PARSE_NODE_IS_ID(pn)) { - qstr qst = MP_PARSE_NODE_LEAF_ARG(pn); +STATIC const char *get_arg_str(const byte *pn) { + if (pt_is_any_id(pn)) { + qstr qst; + pt_extract_id(pn, &qst); return qstr_str(qst); } else { return ""; } } -STATIC mp_uint_t get_arg_reg(emit_inline_asm_t *emit, const char *op, mp_parse_node_t pn, mp_uint_t max_reg) { +STATIC mp_uint_t get_arg_reg(emit_inline_asm_t *emit, const char *op, const byte *pn, mp_uint_t max_reg) { const char *reg_str = get_arg_str(pn); for (mp_uint_t i = 0; i < MP_ARRAY_SIZE(reg_name_table); i++) { const reg_name_t *r = ®_name_table[i]; @@ -204,7 +208,7 @@ STATIC mp_uint_t get_arg_reg(emit_inline_asm_t *emit, const char *op, mp_parse_n return 0; } -STATIC mp_uint_t get_arg_special_reg(emit_inline_asm_t *emit, const char *op, mp_parse_node_t pn) { +STATIC mp_uint_t get_arg_special_reg(emit_inline_asm_t *emit, const char *op, const byte *pn) { const char *reg_str = get_arg_str(pn); for (mp_uint_t i = 0; i < MP_ARRAY_SIZE(special_reg_name_table); i++) { const special_reg_name_t *r = &special_reg_name_table[i]; @@ -219,7 +223,7 @@ STATIC mp_uint_t get_arg_special_reg(emit_inline_asm_t *emit, const char *op, mp } #if MICROPY_EMIT_INLINE_THUMB_FLOAT -STATIC mp_uint_t get_arg_vfpreg(emit_inline_asm_t *emit, const char *op, mp_parse_node_t pn) { +STATIC mp_uint_t get_arg_vfpreg(emit_inline_asm_t *emit, const char *op, const byte *pn) { const char *reg_str = get_arg_str(pn); if (reg_str[0] == 's' && reg_str[1] != '\0') { mp_uint_t regno = 0; @@ -247,43 +251,42 @@ STATIC mp_uint_t get_arg_vfpreg(emit_inline_asm_t *emit, const char *op, mp_pars } #endif -STATIC mp_uint_t get_arg_reglist(emit_inline_asm_t *emit, const char *op, mp_parse_node_t pn) { +STATIC mp_uint_t get_arg_reglist(emit_inline_asm_t *emit, const char *op, const byte *p) { // a register list looks like {r0, r1, r2} and is parsed as a Python set - if (!MP_PARSE_NODE_IS_STRUCT_KIND(pn, PN_atom_brace)) { + if (!pt_is_rule(p, PN_atom_brace)) { goto bad_arg; } - mp_parse_node_struct_t *pns = (mp_parse_node_struct_t*)pn; - assert(MP_PARSE_NODE_STRUCT_NUM_NODES(pns) == 1); // should always be - pn = pns->nodes[0]; + const byte *ptop; + p = pt_rule_extract_top(p, &ptop); mp_uint_t reglist = 0; - if (MP_PARSE_NODE_IS_ID(pn)) { + if (p == ptop) { + goto bad_arg; + } else if (pt_is_any_id(p)) { // set with one element - reglist |= 1 << get_arg_reg(emit, op, pn, 15); - } else if (MP_PARSE_NODE_IS_STRUCT(pn)) { - pns = (mp_parse_node_struct_t*)pn; - if (MP_PARSE_NODE_STRUCT_KIND(pns) == PN_dictorsetmaker) { - assert(MP_PARSE_NODE_IS_STRUCT(pns->nodes[1])); // should succeed - mp_parse_node_struct_t *pns1 = (mp_parse_node_struct_t*)pns->nodes[1]; - if (MP_PARSE_NODE_STRUCT_KIND(pns1) == PN_dictorsetmaker_list) { - // set with multiple elements - - // get first element of set (we rely on get_arg_reg to catch syntax errors) - reglist |= 1 << get_arg_reg(emit, op, pns->nodes[0], 15); - - // get tail elements (2nd, 3rd, ...) - mp_parse_node_t *nodes; - int n = mp_parse_node_extract_list(&pns1->nodes[0], PN_dictorsetmaker_list2, &nodes); - - // process rest of elements - for (int i = 0; i < n; i++) { - reglist |= 1 << get_arg_reg(emit, op, nodes[i], 15); - } - } else { - goto bad_arg; + reglist |= 1 << get_arg_reg(emit, op, p, 15); + } else if (pt_is_rule(p, PN_dictorsetmaker)) { + p = pt_rule_first(p); + const byte *p1 = pt_next(p); + if (pt_is_rule(p1, PN_dictorsetmaker_list)) { + // set with multiple elements + + // get first element of set (we rely on get_arg_reg to catch syntax errors) + reglist |= 1 << get_arg_reg(emit, op, p, 15); + + // get tail elements (2nd, 3rd, ...) + const byte *p1_top; + p1 = pt_rule_extract_top(p1, &p1_top); + if (p1 != p1_top) { + mp_parse_node_extract_list(&p1, PN_dictorsetmaker_list2); + } + + // process rest of elements + for (; p1 != p1_top; p1 = pt_next(p1)) { + reglist |= 1 << get_arg_reg(emit, op, p1, 15); } } else { goto bad_arg; @@ -299,7 +302,7 @@ STATIC mp_uint_t get_arg_reglist(emit_inline_asm_t *emit, const char *op, mp_par return 0; } -STATIC uint32_t get_arg_i(emit_inline_asm_t *emit, const char *op, mp_parse_node_t pn, uint32_t fit_mask) { +STATIC uint32_t get_arg_i(emit_inline_asm_t *emit, const char *op, const byte *pn, uint32_t fit_mask) { mp_obj_t o; if (!mp_parse_node_get_int_maybe(pn, &o)) { emit_inline_thumb_error_exc(emit, mp_obj_new_exception_msg_varg(&mp_type_SyntaxError, "'%s' expects an integer", op)); @@ -313,21 +316,25 @@ STATIC uint32_t get_arg_i(emit_inline_asm_t *emit, const char *op, mp_parse_node return i; } -STATIC bool get_arg_addr(emit_inline_asm_t *emit, const char *op, mp_parse_node_t pn, mp_parse_node_t *pn_base, mp_parse_node_t *pn_offset) { - if (!MP_PARSE_NODE_IS_STRUCT_KIND(pn, PN_atom_bracket)) { +STATIC bool get_arg_addr(emit_inline_asm_t *emit, const char *op, const byte *p, const byte **p_base, const byte **p_offset) { + if (!pt_is_rule(p, PN_atom_bracket)) { + goto bad_arg; + } + if (pt_is_rule_empty(p)) { goto bad_arg; } - mp_parse_node_struct_t *pns = (mp_parse_node_struct_t*)pn; - if (!MP_PARSE_NODE_IS_STRUCT_KIND(pns->nodes[0], PN_testlist_comp)) { + p = pt_rule_first(p); + if (!pt_is_rule(p, PN_testlist_comp)) { goto bad_arg; } - pns = (mp_parse_node_struct_t*)pns->nodes[0]; - if (MP_PARSE_NODE_STRUCT_NUM_NODES(pns) != 2) { + const byte *ptop; + p = pt_rule_extract_top(p, &ptop); + if (pt_num_nodes(p, ptop) != 2) { goto bad_arg; } - *pn_base = pns->nodes[0]; - *pn_offset = pns->nodes[1]; + *p_base = p; + *p_offset = pt_next(p); return true; bad_arg: @@ -335,12 +342,13 @@ STATIC bool get_arg_addr(emit_inline_asm_t *emit, const char *op, mp_parse_node_ return false; } -STATIC int get_arg_label(emit_inline_asm_t *emit, const char *op, mp_parse_node_t pn) { - if (!MP_PARSE_NODE_IS_ID(pn)) { +STATIC int get_arg_label(emit_inline_asm_t *emit, const char *op, const byte *p) { + if (!pt_is_any_id(p)) { emit_inline_thumb_error_exc(emit, mp_obj_new_exception_msg_varg(&mp_type_SyntaxError, "'%s' expects a label", op)); return 0; } - qstr label_qstr = MP_PARSE_NODE_LEAF_ARG(pn); + qstr label_qstr; + pt_extract_id(p, &label_qstr); for (uint i = 0; i < emit->max_num_labels; i++) { if (emit->label_lookup[i] == label_qstr) { return i; @@ -419,7 +427,7 @@ STATIC const format_vfp_op_t format_vfp_op_table[] = { // shorthand alias for whether we allow ARMv7-M instructions #define ARMV7M MICROPY_EMIT_INLINE_THUMB_ARMV7M -STATIC void emit_inline_thumb_op(emit_inline_asm_t *emit, qstr op, mp_uint_t n_args, mp_parse_node_t *pn_args) { +STATIC void emit_inline_thumb_op(emit_inline_asm_t *emit, qstr op, mp_uint_t n_args, const byte **pn_args) { // TODO perhaps make two tables: // one_args = // "b", LAB, asm_thumb_b_n, @@ -493,7 +501,7 @@ STATIC void emit_inline_thumb_op(emit_inline_asm_t *emit, qstr op, mp_uint_t n_a op_code_hi = 0xed90; op_vldr_vstr:; mp_uint_t vd = get_arg_vfpreg(emit, op_str, pn_args[0]); - mp_parse_node_t pn_base, pn_offset; + const byte *pn_base, *pn_offset; if (get_arg_addr(emit, op_str, pn_args[1], &pn_base, &pn_offset)) { mp_uint_t rlo_base = get_arg_reg(emit, op_str, pn_base, 7); mp_uint_t i8; @@ -632,7 +640,7 @@ STATIC void emit_inline_thumb_op(emit_inline_asm_t *emit, qstr op, mp_uint_t n_a } } else if (n_args == 2) { - if (MP_PARSE_NODE_IS_ID(pn_args[1])) { + if (pt_is_any_id(pn_args[1])) { // second arg is a register (or should be) mp_uint_t op_code, op_code_hi; if (op == MP_QSTR_mov) { @@ -711,7 +719,7 @@ STATIC void emit_inline_thumb_op(emit_inline_asm_t *emit, qstr op, mp_uint_t n_a asm_thumb_mov_reg_i16(emit->as, ASM_THUMB_OP_MOVT, reg_dest, (i_src >> 16) & 0xffff); } else if (ARMV7M && op == MP_QSTR_ldrex) { mp_uint_t r_dest = get_arg_reg(emit, op_str, pn_args[0], 15); - mp_parse_node_t pn_base, pn_offset; + const byte *pn_base, *pn_offset; if (get_arg_addr(emit, op_str, pn_args[1], &pn_base, &pn_offset)) { mp_uint_t r_base = get_arg_reg(emit, op_str, pn_base, 15); mp_uint_t i8 = get_arg_i(emit, op_str, pn_offset, 0xff) >> 2; @@ -722,7 +730,7 @@ STATIC void emit_inline_thumb_op(emit_inline_asm_t *emit, qstr op, mp_uint_t n_a for (mp_uint_t i = 0; i < MP_ARRAY_SIZE(format_9_10_op_table); i++) { if (op == format_9_10_op_table[i].name) { op_code = format_9_10_op_table[i].op; - mp_parse_node_t pn_base, pn_offset; + const byte *pn_base, *pn_offset; mp_uint_t rlo_dest = get_arg_reg(emit, op_str, pn_args[0], 7); if (get_arg_addr(emit, op_str, pn_args[1], &pn_base, &pn_offset)) { mp_uint_t rlo_base = get_arg_reg(emit, op_str, pn_base, 7); @@ -767,7 +775,7 @@ STATIC void emit_inline_thumb_op(emit_inline_asm_t *emit, qstr op, mp_uint_t n_a rlo_dest = get_arg_reg(emit, op_str, pn_args[0], 7); rlo_src = get_arg_reg(emit, op_str, pn_args[1], 7); int src_b; - if (MP_PARSE_NODE_IS_ID(pn_args[2])) { + if (pt_is_any_id(pn_args[2])) { op_code |= ASM_THUMB_FORMAT_2_REG_OPERAND; src_b = get_arg_reg(emit, op_str, pn_args[2], 7); } else { @@ -792,7 +800,7 @@ STATIC void emit_inline_thumb_op(emit_inline_asm_t *emit, qstr op, mp_uint_t n_a } else if (ARMV7M && op == MP_QSTR_strex) { mp_uint_t r_dest = get_arg_reg(emit, op_str, pn_args[0], 15); mp_uint_t r_src = get_arg_reg(emit, op_str, pn_args[1], 15); - mp_parse_node_t pn_base, pn_offset; + const byte *pn_base, *pn_offset; if (get_arg_addr(emit, op_str, pn_args[2], &pn_base, &pn_offset)) { mp_uint_t r_base = get_arg_reg(emit, op_str, pn_base, 15); mp_uint_t i8 = get_arg_i(emit, op_str, pn_offset, 0xff) >> 2; From c23a2eabf5e22ca7d3a07dbbe6dae62025558e5b Mon Sep 17 00:00:00 2001 From: Damien George Date: Wed, 20 Jan 2016 23:51:33 +0000 Subject: [PATCH 05/24] update parser and compiler to align with latest master --- py/compile2.c | 158 +++++++++++++++++++++++++++++--------------------- py/emit.h | 2 - py/parse2.c | 115 +++++++++++++++++++++++------------- py/parse2.h | 12 ++-- 4 files changed, 171 insertions(+), 116 deletions(-) diff --git a/py/compile2.c b/py/compile2.c index 188d0f3f5e18d..cb034c6f3672d 100644 --- a/py/compile2.c +++ b/py/compile2.c @@ -1,9 +1,9 @@ /* - * This file is part of the Micro Python project, http://micropython.org/ + * This file is part of the MicroPython project, http://micropython.org/ * * The MIT License (MIT) * - * Copyright (c) 2013-2015 Damien P. George + * Copyright (c) 2013-2016 Damien P. George * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -33,9 +33,9 @@ #include "py/scope.h" #include "py/emit.h" #include "py/compile.h" -#include "py/smallint.h" #include "py/runtime.h" -#include "py/builtin.h" + +#if MICROPY_ENABLE_COMPILER // TODO need to mangle __attr names @@ -80,7 +80,7 @@ typedef struct _compiler_t { // try to keep compiler clean from nlr mp_obj_t compile_error; // set to an exception object if there's an error - mp_uint_t compile_error_line; // set to best guess of line of error + size_t compile_error_line; // set to best guess of line of error uint next_label; @@ -94,7 +94,7 @@ typedef struct _compiler_t { mp_uint_t *co_data; - mp_uint_t num_scopes; + size_t num_scopes; scope_t **scopes; scope_t *scope_cur; @@ -112,7 +112,7 @@ typedef struct _compiler_t { STATIC void compile_error_set_line(compiler_t *comp, const byte *p) { // if the line of the error is unknown then try to update it from the parse data if (comp->compile_error_line == 0 && p != NULL && pt_is_any_rule(p)) { - mp_uint_t rule_id, src_line; + size_t rule_id, src_line; const byte *ptop; pt_rule_extract(p, &rule_id, &src_line, &ptop); comp->compile_error_line = src_line; @@ -147,7 +147,7 @@ STATIC void compile_decrease_except_level(compiler_t *comp) { comp->cur_except_level -= 1; } -STATIC void scope_new_and_link(compiler_t *comp, mp_uint_t scope_idx, scope_kind_t kind, const byte *p, uint emit_options) { +STATIC void scope_new_and_link(compiler_t *comp, size_t scope_idx, scope_kind_t kind, const byte *p, uint emit_options) { scope_t *scope = scope_new(kind, p, comp->source_file, emit_options); scope->parent = comp->scope_cur; comp->scopes[scope_idx] = scope; @@ -284,14 +284,12 @@ STATIC const byte *c_if_cond(compiler_t *comp, const byte *p, bool jump_if, int if (jump_if == false) { EMIT_ARG(jump, label); } - } else if (pt_is_rule(pt_rule_first(p), PN_testlist_comp)) { + } else { + assert(pt_is_rule(pt_rule_first(p), PN_testlist_comp)); // non-empty tuple, acts as true for the condition if (jump_if == true) { EMIT_ARG(jump, label); } - } else { - // parenthesis around 1 item, is just that item - c_if_cond(comp, pt_rule_first(p), jump_if, label); } return pt_next(p); } @@ -416,7 +414,6 @@ STATIC void c_assign_tuple(compiler_t *comp, const byte *p_head, const byte *p_t // assigns top of stack to pn STATIC void c_assign(compiler_t *comp, const byte *p, assign_kind_t assign_kind) { - tail_recursion: assert(!pt_is_null(p)); if (pt_is_any_id(p)) { qstr arg; @@ -459,16 +456,13 @@ STATIC void c_assign(compiler_t *comp, const byte *p, assign_kind_t assign_kind) if (pt_is_null_with_top(p0, ptop)) { // empty tuple goto cannot_assign; - } else if (pt_is_rule(p0, PN_testlist_comp)) { + } else { + assert(pt_is_rule(p0, PN_testlist_comp)); if (assign_kind != ASSIGN_STORE) { goto bad_aug; } p = p0; goto testlist_comp; - } else { - // parenthesis around 1 item, is just that item - p = p0; - goto tail_recursion; } break; } @@ -670,6 +664,13 @@ STATIC void compile_funcdef_lambdef_param(compiler_t *comp, const byte *p) { } STATIC void compile_funcdef_lambdef(compiler_t *comp, scope_t *scope, const byte *p, pn_kind_t pn_list_kind) { + // When we call compile_funcdef_lambdef_param below it can compile an arbitrary + // expression for default arguments, which may contain a lambda. The lambda will + // call here in a nested way, so we must save and restore the relevant state. + bool orig_have_star = comp->have_star; + uint16_t orig_num_dict_params = comp->num_dict_params; + uint16_t orig_num_default_params = comp->num_default_params; + // compile default parameters comp->have_star = false; comp->num_dict_params = 0; @@ -689,6 +690,11 @@ STATIC void compile_funcdef_lambdef(compiler_t *comp, scope_t *scope, const byte // make the function close_over_variables_etc(comp, scope, comp->num_default_params, comp->num_dict_params); + + // restore state + comp->have_star = orig_have_star; + comp->num_dict_params = orig_num_dict_params; + comp->num_default_params = orig_num_default_params; } // leaves function object on stack @@ -897,8 +903,11 @@ STATIC void c_del_stmt(compiler_t *comp, const byte *p) { goto cannot_delete; } } else if (pt_is_rule(p, PN_atom_paren)) { - p = pt_rule_first(p); - if (pt_is_rule(p, PN_testlist_comp)) { + if (pt_is_rule_empty(p)) { + goto cannot_delete; + } else { + p = pt_rule_first(p); + assert(pt_is_rule(p, PN_testlist_comp)); // TODO perhaps factorise testlist_comp code with other uses of PN_testlist_comp // or, simplify the logic here my making the parser simplify everything to a list const byte *p0 = pt_rule_first(p); @@ -923,12 +932,9 @@ STATIC void c_del_stmt(compiler_t *comp, const byte *p) { // sequence with 2 items c_del_stmt(comp, p1); } - } else { - // tuple with 1 element - c_del_stmt(comp, p); } } else { - // TODO is there anything else to implement? + // some arbitrary statment that we can't delete (eg del 1) goto cannot_delete; } @@ -1062,7 +1068,7 @@ STATIC void do_import_name(compiler_t *comp, const byte *p, qstr *q_base) { } qstr qst; p2 = pt_extract_id(p2, &qst); - mp_uint_t str_src_len; + size_t str_src_len; const byte *str_src = qstr_data(qst, &str_src_len); memcpy(str_dest, str_src, str_src_len); str_dest += str_src_len; @@ -1495,6 +1501,19 @@ STATIC void compile_for_stmt(compiler_t *comp, const byte *p, const byte *ptop) } } } + // arguments must be able to be compiled as standard expressions + if (pt_is_any_rule(p_start)) { + int k = pt_rule_extract_rule_id(p_start); + if (k == PN_arglist_star || k == PN_arglist_dbl_star || k == PN_argument) { + goto optimise_fail; + } + } + if (pt_is_any_rule(p_end)) { + int k = pt_rule_extract_rule_id(p_end); + if (k == PN_arglist_star || k == PN_arglist_dbl_star || k == PN_argument) { + goto optimise_fail; + } + } // can optimise const byte *p_body = p_it_top; const byte *p_else = pt_next(p_body); @@ -2221,7 +2240,8 @@ STATIC void compile_atom_paren(compiler_t *comp, const byte *p, const byte *ptop if (pt_is_null_with_top(p, ptop)) { // an empty tuple c_tuple(comp, NULL, NULL, NULL); - } else if (pt_is_rule(p, PN_testlist_comp)) { + } else { + assert(pt_is_rule(p, PN_testlist_comp)); p = pt_rule_first(p); const byte *p1 = pt_next(p); if (pt_is_rule(p1, PN_testlist_comp_3b) || pt_is_rule(p1, PN_testlist_comp_3c)) { @@ -2234,9 +2254,6 @@ STATIC void compile_atom_paren(compiler_t *comp, const byte *p, const byte *ptop // tuple with 2 items c_tuple(comp, NULL, p, ptop); } - } else { - // parenthesis around a single item, is just that item - compile_node(comp, p); } } @@ -2499,7 +2516,7 @@ STATIC const byte *compile_node(compiler_t *comp, const byte *p) { EMIT_ARG(load_const_obj, mp_const_none); } else { qstr qst = p[1] | (p[2] << 8); - mp_uint_t len; + size_t len; const byte *data = qstr_data(qst, &len); EMIT_ARG(load_const_obj, mp_obj_new_bytes(data, len)); } @@ -2510,13 +2527,13 @@ STATIC const byte *compile_node(compiler_t *comp, const byte *p) { compile_load_id(comp, qst); return p; } else if (*p == MP_PT_CONST_OBJECT) { - mp_uint_t idx; + size_t idx; p = pt_extract_const_obj(p, &idx); EMIT_ARG(load_const_obj, (mp_obj_t)comp->co_data[idx]); return p; } else { assert(*p >= MP_PT_RULE_BASE); - mp_uint_t rule_id, src_line; + size_t rule_id, src_line; const byte *ptop; p = pt_rule_extract(p, &rule_id, &src_line, &ptop); EMIT_ARG(set_source_line, src_line); @@ -2542,7 +2559,12 @@ STATIC const byte *compile_node(compiler_t *comp, const byte *p) { STATIC void compile_scope_func_lambda_param(compiler_t *comp, const byte *p, pn_kind_t pn_name, pn_kind_t pn_star, pn_kind_t pn_dbl_star) { (void)pn_dbl_star; - // TODO verify that *k and **k are last etc + // check that **kw is last + if ((comp->scope_cur->scope_flags & MP_SCOPE_FLAG_VARKEYWORDS) != 0) { + compile_syntax_error(comp, p, "invalid syntax"); + return; + } + qstr param_name = MP_QSTR_NULL; uint param_flag = ID_FLAG_IS_PARAM; if (pt_is_any_id(p)) { @@ -2565,6 +2587,11 @@ STATIC void compile_scope_func_lambda_param(compiler_t *comp, const byte *p, pn_ comp->scope_cur->num_pos_args += 1; } } else if (pt_is_rule(p, pn_star)) { + if (comp->have_star) { + // more than one star + compile_syntax_error(comp, p, "invalid syntax"); + return; + } comp->have_star = true; param_flag = ID_FLAG_IS_PARAM | ID_FLAG_IS_STAR_PARAM; if (pt_is_rule_empty(p)) { @@ -2655,9 +2682,16 @@ STATIC void compile_scope_func_annotations(compiler_t *comp, const byte *p) { } #endif // MICROPY_EMIT_NATIVE -STATIC void compile_scope_comp_iter(compiler_t *comp, const byte *p_iter, const byte *p_inner_expr, int l_top, int for_depth) { +STATIC void compile_scope_comp_iter(compiler_t *comp, const byte *p_comp_for, const byte *p_comp_for_top, const byte *p_inner_expr, int for_depth) { + uint l_top = comp_next_label(comp); + uint l_end = comp_next_label(comp); + EMIT_ARG(label_assign, l_top); + EMIT_ARG(for_iter, l_end); + c_assign(comp, p_comp_for, ASSIGN_STORE); + const byte *p_iter = pt_next(pt_next(p_comp_for)); + tail_recursion: - if (p_iter == NULL) { + if (p_iter == p_comp_for_top) { // no more nested if/for; compile inner expression compile_node(comp, p_inner_expr); if (comp->scope_cur->kind == SCOPE_LIST_COMP) { @@ -2674,12 +2708,8 @@ STATIC void compile_scope_comp_iter(compiler_t *comp, const byte *p_iter, const } } else if (pt_is_rule(p_iter, PN_comp_if)) { // if condition - const byte *ptop; - const byte *p0 = pt_rule_extract_top(p_iter, &ptop); + const byte *p0 = pt_rule_extract_top(p_iter, &p_comp_for_top); p_iter = c_if_cond(comp, p0, false, l_top); - if (p_iter == ptop) { - p_iter = NULL; - } goto tail_recursion; } else { assert(pt_is_rule(p_iter, PN_comp_for)); // should be @@ -2687,18 +2717,14 @@ STATIC void compile_scope_comp_iter(compiler_t *comp, const byte *p_iter, const const byte *ptop; const byte *p0 = pt_rule_extract_top(p_iter, &ptop); p0 = pt_next(p0); // skip scope index - const byte *p2 = compile_node(comp, pt_next(p0)); - uint l_end2 = comp_next_label(comp); - uint l_top2 = comp_next_label(comp); + compile_node(comp, pt_next(p0)); EMIT(get_iter); - EMIT_ARG(label_assign, l_top2); - EMIT_ARG(for_iter, l_end2); - c_assign(comp, p0, ASSIGN_STORE); - compile_scope_comp_iter(comp, p2 == ptop ? NULL : p2, p_inner_expr, l_top2, for_depth + 1); - EMIT_ARG(jump, l_top2); - EMIT_ARG(label_assign, l_end2); - EMIT(for_iter_end); + compile_scope_comp_iter(comp, p0, ptop, p_inner_expr, for_depth + 1); } + + EMIT_ARG(jump, l_top); + EMIT_ARG(label_assign, l_end); + EMIT(for_iter_end); } #if 0 @@ -2865,20 +2891,8 @@ STATIC void compile_scope(compiler_t *comp, scope_t *scope, pass_kind_t pass) { #endif } - uint l_end = comp_next_label(comp); - uint l_top = comp_next_label(comp); compile_load_id(comp, qstr_arg); - EMIT_ARG(label_assign, l_top); - EMIT_ARG(for_iter, l_end); - c_assign(comp, p_comp_for, ASSIGN_STORE); - const byte *p_comp_for_p2 = pt_next(pt_next(p_comp_for)); - if (p_comp_for_p2 == p_comp_for_top) { - p_comp_for_p2 = NULL; - } - compile_scope_comp_iter(comp, p_comp_for_p2, p, l_top, 0); - EMIT_ARG(jump, l_top); - EMIT_ARG(label_assign, l_end); - EMIT(for_iter_end); + compile_scope_comp_iter(comp, p_comp_for, p_comp_for_top, p, 0); if (scope->kind == SCOPE_GEN_EXPR) { EMIT_ARG(load_const_tok, MP_TOKEN_KW_NONE); @@ -3063,7 +3077,7 @@ STATIC void compile_scope_inline_asm(compiler_t *comp, scope_t *scope, pass_kind } if (comp->pass > MP_PASS_SCOPE) { - EMIT_INLINE_ASM(end_pass); + EMIT_INLINE_ASM_ARG(end_pass, 0); } if (comp->compile_error != MP_OBJ_NULL) { @@ -3154,7 +3168,10 @@ STATIC void scope_compute_things(scope_t *scope) { } } -mp_obj_t mp_compile(mp_parse_tree_t *parse_tree, qstr source_file, uint emit_opt, bool is_repl) { +#if !MICROPY_PERSISTENT_CODE_SAVE +STATIC +#endif +mp_raw_code_t *mp_compile_to_raw_code(mp_parse_tree_t *parse_tree, qstr source_file, uint emit_opt, bool is_repl) { // put compiler state on the stack, it's relatively small compiler_t comp_state = {0}; compiler_t *comp = &comp_state; @@ -3351,7 +3368,14 @@ mp_obj_t mp_compile(mp_parse_tree_t *parse_tree, qstr source_file, uint emit_opt if (comp->compile_error != MP_OBJ_NULL) { nlr_raise(comp->compile_error); } else { - // return function that executes the outer module - return mp_make_function_from_raw_code(outer_raw_code, MP_OBJ_NULL, MP_OBJ_NULL); + return outer_raw_code; } } + +mp_obj_t mp_compile(mp_parse_tree_t *parse_tree, qstr source_file, uint emit_opt, bool is_repl) { + mp_raw_code_t *rc = mp_compile_to_raw_code(parse_tree, source_file, emit_opt, is_repl); + // return function that executes the outer module + return mp_make_function_from_raw_code(rc, MP_OBJ_NULL, MP_OBJ_NULL); +} + +#endif // MICROPY_ENABLE_COMPILER diff --git a/py/emit.h b/py/emit.h index d65864b50305d..652e6118fbd31 100644 --- a/py/emit.h +++ b/py/emit.h @@ -269,8 +269,6 @@ typedef struct _emit_inline_asm_t emit_inline_asm_t; typedef struct _emit_inline_asm_method_table_t { void (*start_pass)(emit_inline_asm_t *emit, pass_kind_t pass, scope_t *scope, mp_obj_t *error_slot); void (*end_pass)(emit_inline_asm_t *emit, mp_uint_t type_sig); - mp_uint_t (*count_params)(emit_inline_asm_t *emit, mp_uint_t n_params, mp_parse_node_t *pn_params); - void (*end_pass)(emit_inline_asm_t *emit, mp_uint_t type_sig); mp_uint_t (*count_params)(emit_inline_asm_t *emit, const byte *p, const byte *ptop); bool (*label)(emit_inline_asm_t *emit, mp_uint_t label_num, qstr label_id); void (*align)(emit_inline_asm_t *emit, mp_uint_t align); diff --git a/py/parse2.c b/py/parse2.c index ce649270c0e50..06b518784f60f 100644 --- a/py/parse2.c +++ b/py/parse2.c @@ -1,9 +1,9 @@ /* - * This file is part of the Micro Python project, http://micropython.org/ + * This file is part of the MicroPython project, http://micropython.org/ * * The MIT License (MIT) * - * Copyright (c) 2013-2015 Damien P. George + * Copyright (c) 2013-2016 Damien P. George * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -104,16 +104,16 @@ STATIC const rule_t *rules[] = { }; typedef struct _rule_stack_t { - mp_uint_t src_line : BITS_PER_WORD - 8; // maximum bits storing source line number - mp_uint_t rule_id : 8; // this must be large enough to fit largest rule number - mp_uint_t arg_i; // this dictates the maximum nodes in a "list" of things + size_t src_line : 8 * sizeof(size_t) - 8; // maximum bits storing source line number + size_t rule_id : 8; // this must be large enough to fit largest rule number + size_t arg_i; // this dictates the maximum nodes in a "list" of things size_t pt_off; } rule_stack_t; typedef struct _mp_parse_chunk_t { - mp_uint_t alloc; + size_t alloc; union { - mp_uint_t used; + size_t used; struct _mp_parse_chunk_t *next; } union_; byte data[]; @@ -128,14 +128,14 @@ typedef enum { typedef struct _parser_t { parse_error_t parse_error; - mp_uint_t rule_stack_alloc; - mp_uint_t rule_stack_top; + size_t rule_stack_alloc; + size_t rule_stack_top; rule_stack_t *rule_stack; mp_uint_t cur_scope_id; - mp_uint_t co_alloc; - mp_uint_t co_used; + size_t co_alloc; + size_t co_used; mp_uint_t *co_data; mp_lexer_t *lexer; @@ -147,7 +147,7 @@ typedef struct _parser_t { #endif } parser_t; -STATIC void push_rule(parser_t *parser, mp_uint_t src_line, const rule_t *rule, mp_uint_t arg_i, size_t pt_off) { +STATIC void push_rule(parser_t *parser, size_t src_line, const rule_t *rule, size_t arg_i, size_t pt_off) { if (parser->parse_error) { return; } @@ -167,14 +167,14 @@ STATIC void push_rule(parser_t *parser, mp_uint_t src_line, const rule_t *rule, rs->pt_off = pt_off; } -STATIC void push_rule_from_arg(parser_t *parser, mp_uint_t arg) { +STATIC void push_rule_from_arg(parser_t *parser, size_t arg) { assert((arg & RULE_ARG_KIND_MASK) == RULE_ARG_RULE || (arg & RULE_ARG_KIND_MASK) == RULE_ARG_OPT_RULE); - mp_uint_t rule_id = arg & RULE_ARG_ARG_MASK; + size_t rule_id = arg & RULE_ARG_ARG_MASK; assert(rule_id < RULE_maximum_number_of); push_rule(parser, parser->lexer->tok_line, rules[rule_id], 0, 0); } -STATIC void pop_rule(parser_t *parser, const rule_t **rule, mp_uint_t *arg_i, mp_uint_t *src_line, size_t *pt_off) { +STATIC void pop_rule(parser_t *parser, const rule_t **rule, size_t *arg_i, size_t *src_line, size_t *pt_off) { assert(!parser->parse_error); parser->rule_stack_top -= 1; *rule = rules[parser->rule_stack[parser->rule_stack_top].rule_id]; @@ -205,7 +205,7 @@ STATIC void pt_raw_truncate_at(pt_t *pt, size_t pt_off) { pt->vv.len = pt_off; } -STATIC int vuint_nbytes(mp_uint_t val) { +STATIC int vuint_nbytes(size_t val) { int n = 0; do { n += 1; @@ -214,7 +214,7 @@ STATIC int vuint_nbytes(mp_uint_t val) { return n; } -STATIC void vuint_store(byte *p, int nbytes, mp_uint_t val) { +STATIC void vuint_store(byte *p, int nbytes, size_t val) { p += nbytes; *--p = val & 0x7f; for (--nbytes; nbytes > 0; --nbytes) { @@ -223,9 +223,9 @@ STATIC void vuint_store(byte *p, int nbytes, mp_uint_t val) { } } -STATIC mp_uint_t vuint_load(const byte **p_in) { +STATIC size_t vuint_load(const byte **p_in) { const byte *p = *p_in; - mp_uint_t val = 0; + size_t val = 0; do { val = (val << 7) + (*p & 0x7f); } while ((*p++ & 0x80) != 0); @@ -269,8 +269,28 @@ STATIC byte *pt_advance(const byte *p, bool full_rule) { return (byte*)p; } +bool mp_parse_node_get_int_maybe(const byte *p, mp_obj_t *o) { + if (pt_is_small_int(p)) { + *o = MP_OBJ_NEW_SMALL_INT(pt_small_int_value(p)); + return true; + #if 0 // TODO + } else if (MP_PARSE_NODE_IS_STRUCT_KIND(pn, RULE_const_object)) { + mp_parse_node_struct_t *pns = (mp_parse_node_struct_t*)pn; + #if MICROPY_OBJ_REPR == MICROPY_OBJ_REPR_D + // nodes are 32-bit pointers, but need to extract 64-bit object + *o = (uint64_t)pns->nodes[0] | ((uint64_t)pns->nodes[1] << 32); + #else + *o = (mp_obj_t)pns->nodes[0]; + #endif + return MP_OBJ_IS_INT(*o); + #endif + } else { + return false; + } +} + // TODO this could perhaps allow *p to be null and in that case return null? -const byte *mp_parse_node_extract_list(const byte **p, mp_uint_t pn_kind) { +const byte *mp_parse_node_extract_list(const byte **p, size_t pn_kind) { if (pt_is_null(*p)) { *p += 1; return *p; @@ -295,7 +315,7 @@ const byte *pt_extract_id(const byte *p, qstr *qst) { } */ -const byte *pt_extract_const_obj(const byte *p, mp_uint_t *idx) { +const byte *pt_extract_const_obj(const byte *p, size_t *idx) { assert(*p == MP_PT_CONST_OBJECT); p += 1; *idx = vuint_load(&p); @@ -646,7 +666,7 @@ STATIC bool fold_constants(parser_t *parser, pt_t *pt, size_t pt_off, const rule } mp_obj_t dest[2]; mp_load_method_maybe(elem->value, q_attr, dest); - if (!(MP_OBJ_IS_SMALL_INT(dest[0]) && dest[1] == NULL)) { + if (!(MP_OBJ_IS_SMALL_INT(dest[0]) && dest[1] == MP_OBJ_NULL)) { return false; } arg0 = MP_OBJ_SMALL_INT_VALUE(dest[0]); @@ -665,9 +685,24 @@ STATIC bool fold_constants(parser_t *parser, pt_t *pt, size_t pt_off, const rule } #endif -STATIC void pt_ins_rule(parser_t *parser, pt_t *pt, size_t pt_off, mp_uint_t src_line, const rule_t *rule, mp_uint_t num_args) { +STATIC void pt_ins_rule(parser_t *parser, pt_t *pt, size_t pt_off, size_t src_line, const rule_t *rule, size_t num_args) { (void)num_args; + // optimise away parenthesis around an expression if possible + if (rule->rule_id == RULE_atom_paren) { + // there should be just 1 arg for this rule + const byte *p = (byte*)pt->vv.buf + pt_off; + if (pt_is_null(p)) { + // need to keep parenthesis for () + } else if (pt_is_rule(p, RULE_testlist_comp)) { + // need to keep parenthesis for (a, b, ...) + } else { + // parenthesis around a single expression, so it's just the expression + //printf("opt!\n"); + return; + } + } + #if MICROPY_COMP_CONST_FOLDING if (fold_constants(parser, pt, pt_off, rule)) { // we folded this rule so return straight away @@ -826,7 +861,7 @@ folding_fail:; // insert small int node for scope index if (extra_node != 0) { dest[1 + nb1 + nb2] = MP_PT_SMALL_INT; - mp_uint_t val = ++parser->cur_scope_id; + size_t val = ++parser->cur_scope_id; for (size_t i = 0; i < BYTES_PER_WORD; ++i) { dest[1 + nb1 + nb2 + 1 + i] = val; val >>= 8; @@ -848,7 +883,7 @@ STATIC void make_node_const_object(parser_t *parser, pt_t *pt, mp_obj_t obj) { parser->co_data[parser->co_used++] = (mp_uint_t)obj; } -STATIC void make_node_string_bytes(parser_t *parser, pt_t *pt, mp_token_kind_t tok, const char *str, mp_uint_t len) { +STATIC void make_node_string_bytes(parser_t *parser, pt_t *pt, mp_token_kind_t tok, const char *str, size_t len) { mp_obj_t o; if (tok == MP_TOKEN_STRING) { o = mp_obj_new_str(str, len, false); @@ -926,16 +961,16 @@ const byte *pt_rule_extract_top(const byte *p, const byte **ptop) { assert(*p >= MP_PT_RULE_BASE); p++; vuint_load(&p); - mp_uint_t nbytes = vuint_load(&p); + size_t nbytes = vuint_load(&p); *ptop = p + nbytes; return p; } -const byte *pt_rule_extract(const byte *p, mp_uint_t *rule_id, mp_uint_t *src_line, const byte **ptop) { +const byte *pt_rule_extract(const byte *p, size_t *rule_id, size_t *src_line, const byte **ptop) { assert(*p >= MP_PT_RULE_BASE); *rule_id = *p++ - MP_PT_RULE_BASE; *src_line = vuint_load(&p); - mp_uint_t nbytes = vuint_load(&p); + size_t nbytes = vuint_load(&p); *ptop = p + nbytes; return p; } @@ -977,7 +1012,7 @@ mp_parse_tree_t mp_parse(mp_lexer_t *lex, mp_parse_input_kind_t input_kind) { } // work out the top-level rule to use, and push it on the stack - mp_uint_t top_level_rule; + size_t top_level_rule; switch (input_kind) { case MP_PARSE_SINGLE_INPUT: top_level_rule = RULE_single_input; break; case MP_PARSE_EVAL_INPUT: top_level_rule = RULE_eval_input; break; @@ -987,9 +1022,9 @@ mp_parse_tree_t mp_parse(mp_lexer_t *lex, mp_parse_input_kind_t input_kind) { // parse! - mp_uint_t n, i; // state for the current rule + size_t n, i; // state for the current rule size_t pt_off = 0; // state for the current rule - mp_uint_t rule_src_line; // source line for the first token matched by the current rule + size_t rule_src_line; // source line for the first token matched by the current rule bool backtrack = false; const rule_t *rule = NULL; pt_t *pt = pt_new(); @@ -1109,7 +1144,7 @@ mp_parse_tree_t mp_parse(mp_lexer_t *lex, mp_parse_input_kind_t input_kind) { i = 0; bool emit_rule = false; /* - for (mp_uint_t x = 0; x < n; ++x) { + for (size_t x = 0; x < n; ++x) { if ((rule->arg[x] & RULE_ARG_KIND_MASK) == RULE_ARG_TOK) { mp_token_kind_t tok_kind = rule->arg[x] & RULE_ARG_ARG_MASK; if (tok_kind >= MP_TOKEN_NAME) { @@ -1125,7 +1160,7 @@ mp_parse_tree_t mp_parse(mp_lexer_t *lex, mp_parse_input_kind_t input_kind) { } } */ - for (mp_uint_t x = 0; x < n; ++x) { + for (size_t x = 0; x < n; ++x) { if ((rule->arg[x] & RULE_ARG_KIND_MASK) == RULE_ARG_TOK) { mp_token_kind_t tok_kind = rule->arg[x] & RULE_ARG_ARG_MASK; if (tok_kind >= MP_TOKEN_NAME) { @@ -1164,8 +1199,6 @@ mp_parse_tree_t mp_parse(mp_lexer_t *lex, mp_parse_input_kind_t input_kind) { // if a rule has the RULE_ACT_ALLOW_IDENT bit set then this // rule should not be emitted if it has only 1 argument - // NOTE: can't set this flag for atom_paren because we need it - // to distinguish, for example, [a,b] from [(a,b)] if (rule->act & RULE_ACT_ALLOW_IDENT) { emit_rule = false; } @@ -1179,10 +1212,10 @@ mp_parse_tree_t mp_parse(mp_lexer_t *lex, mp_parse_input_kind_t input_kind) { } // count number of non-null nodes - mp_uint_t num_not_null = 0; - mp_uint_t num_trail_null = 0; + size_t num_not_null = 0; + size_t num_trail_null = 0; { const byte *p = (byte*)pt->vv.buf + pt_off; - for (mp_uint_t x = 0; x < i; ++x) { + for (size_t x = 0; x < i; ++x) { if (*p != MP_PT_NULL) { num_not_null += 1; num_trail_null = 0; @@ -1199,7 +1232,7 @@ mp_parse_tree_t mp_parse(mp_lexer_t *lex, mp_parse_input_kind_t input_kind) { } else { // single result, leave it on stack const byte *p = (byte*)pt->vv.buf + pt_off; - for (mp_uint_t x = 0; x < i; ++x) { + for (size_t x = 0; x < i; ++x) { if (*p == MP_PT_NULL) { p = pt_del_byte(pt, p); } else { @@ -1249,7 +1282,7 @@ mp_parse_tree_t mp_parse(mp_lexer_t *lex, mp_parse_input_kind_t input_kind) { } } else { for (;;) { - mp_uint_t arg = rule->arg[i & 1 & n]; + size_t arg = rule->arg[i & 1 & n]; switch (arg & RULE_ARG_KIND_MASK) { case RULE_ARG_TOK: if (lex->tok_kind == (arg & RULE_ARG_ARG_MASK)) { @@ -1312,7 +1345,7 @@ mp_parse_tree_t mp_parse(mp_lexer_t *lex, mp_parse_input_kind_t input_kind) { pt_show(pt); { - mp_uint_t n_pool, n_qstr, n_str_data_bytes, n_total_bytes; + size_t n_pool, n_qstr, n_str_data_bytes, n_total_bytes; qstr_pool_info(&n_pool, &n_qstr, &n_str_data_bytes, &n_total_bytes); printf("qstr pool: n_pool=" UINT_FMT ", n_qstr=" UINT_FMT ", n_str_data_bytes=" UINT_FMT ", n_total_bytes=" UINT_FMT "\n", diff --git a/py/parse2.h b/py/parse2.h index 5090cc00c79f6..f50af0df45214 100644 --- a/py/parse2.h +++ b/py/parse2.h @@ -1,9 +1,9 @@ /* - * This file is part of the Micro Python project, http://micropython.org/ + * This file is part of the MicroPython project, http://micropython.org/ * * The MIT License (MIT) * - * Copyright (c) 2013, 2014 Damien P. George + * Copyright (c) 2013-2016 Damien P. George * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -26,11 +26,10 @@ #ifndef __MICROPY_INCLUDED_PY_PARSE_H__ #define __MICROPY_INCLUDED_PY_PARSE_H__ +#include #include -#include "py/mpconfig.h" -#include "py/misc.h" -#include "py/qstr.h" +#include "py/obj.h" struct _mp_lexer_t; @@ -108,9 +107,10 @@ mp_int_t pt_small_int_value(const byte *p); const byte *pt_get_small_int(const byte *p, mp_int_t *val); const byte *pt_rule_first(const byte *p); const byte *pt_rule_extract_top(const byte *p, const byte **ptop); -const byte *pt_rule_extract(const byte *p, mp_uint_t *rule_id, mp_uint_t *src_line, const byte **ptop); +const byte *pt_rule_extract(const byte *p, mp_uint_t *rule_id, size_t *src_line, const byte **ptop); bool pt_is_rule_empty(const byte *p); +bool mp_parse_node_get_int_maybe(const byte *p, mp_obj_t *o); const byte *mp_parse_node_extract_list(const byte **p, mp_uint_t pn_kind); typedef enum { From 238b5f5f891838de04faa1bd0718c069e14cc1fa Mon Sep 17 00:00:00 2001 From: Damien George Date: Thu, 21 Jan 2016 00:10:33 +0000 Subject: [PATCH 06/24] py/emitglue: Suppress warning for unused parameter. --- py/emitglue.c | 1 + 1 file changed, 1 insertion(+) diff --git a/py/emitglue.c b/py/emitglue.c index d2baaa8459c1c..e327d9d0cb63d 100644 --- a/py/emitglue.c +++ b/py/emitglue.c @@ -83,6 +83,7 @@ void mp_emit_glue_assign_bytecode(mp_raw_code_t *rc, const byte *code, mp_uint_t uint16_t n_obj, uint16_t n_raw_code, #endif mp_uint_t scope_flags) { + (void)len; // possibly unused rc->kind = MP_CODE_BYTECODE; rc->scope_flags = scope_flags; From 2d9531a777e01c982ce8f2e5867b05bbac74373c Mon Sep 17 00:00:00 2001 From: Damien George Date: Mon, 1 Feb 2016 21:30:18 +0000 Subject: [PATCH 07/24] py/vstr: Change allocation policy from *2 to +64. --- py/vstr.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/py/vstr.c b/py/vstr.c index 55c597bb125f0..181b1c49b95db 100644 --- a/py/vstr.c +++ b/py/vstr.c @@ -151,7 +151,7 @@ STATIC bool vstr_ensure_extra(vstr_t *vstr, size_t size) { if (vstr->fixed_buf) { return false; } - size_t new_alloc = ROUND_ALLOC((vstr->len + size) * 2); + size_t new_alloc = ROUND_ALLOC((vstr->len + size) + 64); char *new_buf = m_renew(char, vstr->buf, vstr->alloc, new_alloc); if (new_buf == NULL) { vstr->had_error = true; From 1c0343f9d991c241d335712593f3a63858dc91b6 Mon Sep 17 00:00:00 2001 From: Damien George Date: Mon, 1 Feb 2016 21:32:05 +0000 Subject: [PATCH 08/24] py/gc: Zero out all newly allocated memory to prevent stale pointers. --- py/gc.c | 8 ++++---- py/malloc.c | 5 +---- 2 files changed, 5 insertions(+), 8 deletions(-) diff --git a/py/gc.c b/py/gc.c index 4e4cd9f303d96..41526c8b095d7 100644 --- a/py/gc.c +++ b/py/gc.c @@ -409,12 +409,12 @@ void *gc_alloc(size_t n_bytes, bool has_finaliser) { void *ret_ptr = (void*)(MP_STATE_MEM(gc_pool_start) + start_block * BYTES_PER_BLOCK); DEBUG_printf("gc_alloc(%p)\n", ret_ptr); - // zero out the additional bytes of the newly allocated blocks + // Zero out all the bytes of the newly allocated blocks. // This is needed because the blocks may have previously held pointers // to the heap and will not be set to something else if the caller // doesn't actually use the entire block. As such they will continue // to point to the heap and may prevent other blocks from being reclaimed. - memset((byte*)ret_ptr + n_bytes, 0, (end_block - start_block + 1) * BYTES_PER_BLOCK - n_bytes); + memset((byte*)ret_ptr, 0, (end_block - start_block + 1) * BYTES_PER_BLOCK); #if MICROPY_ENABLE_FINALISER if (has_finaliser) { @@ -620,8 +620,8 @@ void *gc_realloc(void *ptr_in, size_t n_bytes, bool allow_move) { ATB_FREE_TO_TAIL(bl); } - // zero out the additional bytes of the newly allocated blocks (see comment above in gc_alloc) - memset((byte*)ptr_in + n_bytes, 0, new_blocks * BYTES_PER_BLOCK - n_bytes); + // zero out the bytes of the newly allocated blocks (see comment above in gc_alloc) + memset((byte*)ptr_in + n_blocks * BYTES_PER_BLOCK, 0, (new_blocks - n_blocks) * BYTES_PER_BLOCK); #if EXTENSIVE_HEAP_PROFILING gc_dump_alloc_table(); diff --git a/py/malloc.c b/py/malloc.c index b0493d9341e19..c837ed5735bf8 100644 --- a/py/malloc.c +++ b/py/malloc.c @@ -114,10 +114,7 @@ void *m_malloc_with_finaliser(size_t num_bytes) { void *m_malloc0(size_t num_bytes) { void *ptr = m_malloc(num_bytes); - if (ptr == NULL && num_bytes != 0) { - return m_malloc_fail(num_bytes); - } - memset(ptr, 0, num_bytes); + // memory is already cleared by gc_alloc return ptr; } From be020eb2c3c7e8421ee0989a7a09498a91b974fb Mon Sep 17 00:00:00 2001 From: Damien George Date: Mon, 18 Apr 2016 16:17:50 +0100 Subject: [PATCH 09/24] update compiler to align with latest changes on master, v1.7 --- py/compile2.c | 58 ++++++++++++++++++++++++++++++++++++++++----------- 1 file changed, 46 insertions(+), 12 deletions(-) diff --git a/py/compile2.c b/py/compile2.c index cb034c6f3672d..5ef00feaa29c5 100644 --- a/py/compile2.c +++ b/py/compile2.c @@ -1702,6 +1702,11 @@ STATIC void compile_with_stmt_helper(compiler_t *comp, const byte *n_pre, const compile_node(comp, p_body); } else { uint l_end = comp_next_label(comp); + if (MICROPY_EMIT_NATIVE && comp->scope_cur->emit_options != MP_EMIT_OPT_BYTECODE) { + // we need to allocate an extra label for the native emitter + // it will use l_end+1 as an auxiliary label + comp_next_label(comp); + } if (pt_is_rule(n_pre, PN_with_item)) { // this pre-bit is of the form "a as b" const byte *p = pt_rule_first(n_pre); @@ -1719,10 +1724,7 @@ STATIC void compile_with_stmt_helper(compiler_t *comp, const byte *n_pre, const // compile additional pre-bits and the body compile_with_stmt_helper(comp, n_pre, p_body); // finish this with block - EMIT(pop_block); - EMIT_ARG(load_const_tok, MP_TOKEN_KW_NONE); - EMIT_ARG(label_assign, l_end); - EMIT(with_cleanup); + EMIT_ARG(with_cleanup, l_end); compile_decrease_except_level(comp); EMIT(end_finally); } @@ -2073,6 +2075,10 @@ STATIC void compile_power(compiler_t *comp, const byte *p, const byte *ptop) { comp->func_arg_is_super = pt_is_id(p, MP_QSTR_super); compile_generic_all_nodes(comp, p, ptop); + + if (pt_num_nodes(p, ptop) == 3) { + EMIT_ARG(binary_op, MP_BINARY_OP_POWER); + } } // if p_arglist==NULL then there are no arguments @@ -2205,12 +2211,6 @@ STATIC void compile_power_trailers(compiler_t *comp, const byte *p, const byte * } } -STATIC void compile_power_dbl_star(compiler_t *comp, const byte *p, const byte *ptop) { - (void)ptop; - compile_node(comp, p); - EMIT_ARG(binary_op, MP_BINARY_OP_POWER); -} - // p needs to point to 2 successive nodes, first is lhs of comprehension, second is PN_comp_for node STATIC void compile_comprehension(compiler_t *comp, const byte *p, scope_kind_t kind) { const byte *p_comp_for = pt_next(p); @@ -2493,7 +2493,23 @@ STATIC const byte *compile_node(compiler_t *comp, const byte *p) { } else if (pt_is_small_int(p)) { mp_int_t arg; p = pt_get_small_int(p, &arg); + #if MICROPY_DYNAMIC_COMPILER + mp_uint_t sign_mask = -(1 << (mp_dynamic_compiler.small_int_bits - 1)); + if ((arg & sign_mask) == 0 || (arg & sign_mask) == sign_mask) { + // integer fits in target runtime's small-int + EMIT_ARG(load_const_small_int, arg); + } else { + // integer doesn't fit, so create a multi-precision int object + // (but only create the actual object on the last pass) + if (comp->pass != MP_PASS_EMIT) { + EMIT_ARG(load_const_obj, mp_const_none); + } else { + EMIT_ARG(load_const_obj, mp_obj_new_int_from_ll(arg)); + } + } + #else EMIT_ARG(load_const_small_int, arg); + #endif return p; } else if (pt_is_any_tok(p)) { byte tok; @@ -2970,7 +2986,25 @@ STATIC void compile_scope_inline_asm(compiler_t *comp, scope_t *scope, pass_kind } p = pt_next(p); // skip the parameter list - p = pt_next(p); // skip the return type + + // function return annotation is in the next node + mp_uint_t type_sig = MP_NATIVE_TYPE_INT; + if (!pt_is_null(p)) { + if (pt_is_any_id(p)) { + qstr ret_type; + pt_extract_id(p, &ret_type); + switch (ret_type) { + case MP_QSTR_object: type_sig = MP_NATIVE_TYPE_OBJ; break; + case MP_QSTR_bool: type_sig = MP_NATIVE_TYPE_BOOL; break; + case MP_QSTR_int: type_sig = MP_NATIVE_TYPE_INT; break; + case MP_QSTR_uint: type_sig = MP_NATIVE_TYPE_UINT; break; + default: compile_syntax_error(comp, p, "unknown type"); return; + } + } else { + compile_syntax_error(comp, p, "return annotation must be an identifier"); + } + } + p = pt_next(p); // move past function return annotation // get the list of statements within the body of the function const byte *ptop = mp_parse_node_extract_list(&p, PN_suite_block_stmts); @@ -3077,7 +3111,7 @@ STATIC void compile_scope_inline_asm(compiler_t *comp, scope_t *scope, pass_kind } if (comp->pass > MP_PASS_SCOPE) { - EMIT_INLINE_ASM_ARG(end_pass, 0); + EMIT_INLINE_ASM_ARG(end_pass, type_sig); } if (comp->compile_error != MP_OBJ_NULL) { From 72912c753a31301efb7d6b2f35dd1e3e64b98cd4 Mon Sep 17 00:00:00 2001 From: Damien George Date: Sun, 13 Aug 2017 21:23:02 +1000 Subject: [PATCH 10/24] py: Deoptimise try-finally and subscript parse nodes to match master. This is a small de-optimisation so that it's easier to merge master into this branch. --- py/compile2.c | 8 ++++++-- py/grammar.h | 4 ++-- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/py/compile2.c b/py/compile2.c index 5ef00feaa29c5..ef9e434723b0c 100644 --- a/py/compile2.c +++ b/py/compile2.c @@ -1649,6 +1649,8 @@ STATIC void compile_try_except(compiler_t *comp, const byte *p_body, const byte } STATIC void compile_try_finally(compiler_t *comp, const byte *p_body, const byte *p_except, const byte *p_except_top, const byte *p_else, const byte *p_finally) { + assert(pt_is_rule(p_finally, PN_try_stmt_finally)); + uint l_finally_block = comp_next_label(comp); EMIT_ARG(setup_finally, l_finally_block); @@ -1665,7 +1667,7 @@ STATIC void compile_try_finally(compiler_t *comp, const byte *p_body, const byte EMIT(pop_block); EMIT_ARG(load_const_tok, MP_TOKEN_KW_NONE); EMIT_ARG(label_assign, l_finally_block); - compile_node(comp, p_finally); + compile_node(comp, pt_rule_first(p_finally)); compile_decrease_except_level(comp); EMIT(end_finally); @@ -2412,7 +2414,9 @@ STATIC void compile_subscript_3_helper(compiler_t *comp, const byte *p, const by } else if (pt_is_rule(p, PN_subscript_3d)) { p = pt_rule_first(p); p = compile_node(comp, p); - if (pt_is_rule(p, PN_sliceop)) { + assert(pt_is_rule(p, PN_sliceop)); // should always be + p = pt_rule_first(p); + if (p == ptop) { // [?:x:] EMIT_ARG(build_slice, 2); } else { diff --git a/py/grammar.h b/py/grammar.h index 004e14826f3c3..89a5a06537f8d 100644 --- a/py/grammar.h +++ b/py/grammar.h @@ -180,7 +180,7 @@ DEF_RULE(try_stmt_except_and_more, nc, and(3), rule(try_stmt_except_list), opt_r DEF_RULE(try_stmt_except, nc, and(4), tok(KW_EXCEPT), opt_rule(try_stmt_as_name), tok(DEL_COLON), rule(suite)) DEF_RULE(try_stmt_as_name, nc, and(2), rule(test), opt_rule(as_name)) DEF_RULE(try_stmt_except_list, nc, one_or_more, rule(try_stmt_except)) -DEF_RULE(try_stmt_finally, nc, ident | and(3), tok(KW_FINALLY), tok(DEL_COLON), rule(suite)) +DEF_RULE(try_stmt_finally, nc, and(3), tok(KW_FINALLY), tok(DEL_COLON), rule(suite)) DEF_RULE(else_stmt, nc, ident | and(3), tok(KW_ELSE), tok(DEL_COLON), rule(suite)) DEF_RULE(with_stmt, c(with_stmt), and(4), tok(KW_WITH), rule(with_stmt_list), tok(DEL_COLON), rule(suite)) DEF_RULE(with_stmt_list, nc, list, rule(with_item), tok(DEL_COMMA)) @@ -275,7 +275,7 @@ DEF_RULE(subscript_3, c(subscript_3), and(2), tok(DEL_COLON), opt_rule(subscript DEF_RULE(subscript_3b, nc, or(2), rule(subscript_3c), rule(subscript_3d)) DEF_RULE(subscript_3c, nc, and(2), tok(DEL_COLON), opt_rule(test)) DEF_RULE(subscript_3d, nc, and(2), rule(test), opt_rule(sliceop)) -DEF_RULE(sliceop, nc, ident | and(2), tok(DEL_COLON), opt_rule(test)) +DEF_RULE(sliceop, nc, and(2), tok(DEL_COLON), opt_rule(test)) #else DEF_RULE(subscriptlist, c(generic_tuple), list_with_end, rule(test), tok(DEL_COMMA)) #endif From be2387885bf251fb44b6b64aeadff147a4ae7266 Mon Sep 17 00:00:00 2001 From: Damien George Date: Sun, 13 Aug 2017 21:32:08 +1000 Subject: [PATCH 11/24] py: Update compiler/parser to align with latest changes on master, v1.8. --- py/compile2.c | 31 ++++++++++++++++++------------- py/parse2.c | 4 ++-- 2 files changed, 20 insertions(+), 15 deletions(-) diff --git a/py/compile2.c b/py/compile2.c index ef9e434723b0c..387d6b4bdeccc 100644 --- a/py/compile2.c +++ b/py/compile2.c @@ -37,6 +37,10 @@ #if MICROPY_ENABLE_COMPILER +#if MICROPY_PY_ASYNC_AWAIT +#error "async/await syntax not implemented with this parser/compiler" +#endif + // TODO need to mangle __attr names typedef enum { @@ -304,7 +308,7 @@ STATIC const byte *c_if_cond(compiler_t *comp, const byte *p, bool jump_if, int typedef enum { ASSIGN_STORE, ASSIGN_AUG_LOAD, ASSIGN_AUG_STORE } assign_kind_t; STATIC void c_assign(compiler_t *comp, const byte *p, assign_kind_t kind); -STATIC void c_assign_power(compiler_t *comp, const byte *p_orig, assign_kind_t assign_kind) { +STATIC void c_assign_atom_expr(compiler_t *comp, const byte *p_orig, assign_kind_t assign_kind) { const byte *ptop; const byte *p0 = pt_rule_extract_top(p_orig, &ptop); @@ -320,7 +324,7 @@ STATIC void c_assign_power(compiler_t *comp, const byte *p_orig, assign_kind_t a return; } - if (pt_is_rule(p1, PN_power_trailers)) { + if (pt_is_rule(p1, PN_atom_expr_trailers)) { const byte *p1top; p1 = pt_rule_extract_top(p1, &p1top); for (;;) { @@ -432,9 +436,9 @@ STATIC void c_assign(compiler_t *comp, const byte *p, assign_kind_t assign_kind) compile_syntax_error(comp, p, "can't assign to literal"); } else { switch (pt_rule_extract_rule_id(p)) { - case PN_power: + case PN_atom_expr_normal: // lhs is an index or attribute - c_assign_power(comp, p, assign_kind); + c_assign_atom_expr(comp, p, assign_kind); break; case PN_testlist_star_expr: @@ -867,13 +871,13 @@ STATIC void c_del_stmt(compiler_t *comp, const byte *p) { qstr id; pt_extract_id(p, &id); compile_delete_id(comp, id); - } else if (pt_is_rule(p, PN_power)) { + } else if (pt_is_rule(p, PN_atom_expr_normal)) { const byte *ptop; const byte *p0 = pt_rule_extract_top(p, &ptop); const byte *p1 = compile_node(comp, p0); // base of the power node - if (pt_is_rule(p1, PN_power_trailers)) { + if (pt_is_rule(p1, PN_atom_expr_trailers)) { const byte *p1top; p1 = pt_rule_extract_top(p1, &p1top); for (;;) { @@ -1466,7 +1470,7 @@ STATIC void compile_for_stmt(compiler_t *comp, const byte *p, const byte *ptop) // this is actually slower, but uses no heap memory // for viper it will be much, much faster if (/*comp->scope_cur->emit_options == MP_EMIT_OPT_VIPER &&*/ pt_is_any_id(p) - && pt_is_rule(pt_next(p), PN_power)) { + && pt_is_rule(pt_next(p), PN_atom_expr_normal)) { const byte *p_it_top; const byte *p_it0 = pt_rule_extract_top(pt_next(p), &p_it_top); if (!pt_is_id(p_it0, MP_QSTR_range)) { @@ -2072,15 +2076,16 @@ STATIC void compile_factor_2(compiler_t *comp, const byte *p, const byte *ptop) } } -STATIC void compile_power(compiler_t *comp, const byte *p, const byte *ptop) { +STATIC void compile_atom_expr_normal(compiler_t *comp, const byte *p, const byte *ptop) { // this is to handle special super() call comp->func_arg_is_super = pt_is_id(p, MP_QSTR_super); compile_generic_all_nodes(comp, p, ptop); +} - if (pt_num_nodes(p, ptop) == 3) { - EMIT_ARG(binary_op, MP_BINARY_OP_POWER); - } +STATIC void compile_power(compiler_t *comp, const byte *p, const byte *ptop) { + compile_generic_all_nodes(comp, p, ptop); // 2 nodes, arguments of power + EMIT_ARG(binary_op, MP_BINARY_OP_POWER); } // if p_arglist==NULL then there are no arguments @@ -2189,7 +2194,7 @@ STATIC void compile_trailer_paren_helper(compiler_t *comp, const byte *p_arglist } } -STATIC void compile_power_trailers(compiler_t *comp, const byte *p, const byte *ptop) { +STATIC void compile_atom_expr_trailers(compiler_t *comp, const byte *p, const byte *ptop) { while (p != ptop) { const byte *p_next = pt_next(p); if (p_next != ptop && pt_is_rule(p, PN_trailer_period) && pt_is_rule(p_next, PN_trailer_paren)) { @@ -3028,7 +3033,7 @@ STATIC void compile_scope_inline_asm(compiler_t *comp, scope_t *scope, pass_kind // check structure of parse node const byte *p_expr_top; const byte *p_expr = pt_rule_extract_top(p, &p_expr_top); - if (!pt_is_rule(p_expr, PN_power)) { + if (!pt_is_rule(p_expr, PN_atom_expr_normal)) { goto not_an_instruction; } if (pt_next(p_expr) != p_expr_top) { diff --git a/py/parse2.c b/py/parse2.c index 06b518784f60f..0e729c6dde4cd 100644 --- a/py/parse2.c +++ b/py/parse2.c @@ -71,10 +71,10 @@ enum { RULE_maximum_number_of, }; -#define ident (RULE_ACT_ALLOW_IDENT) -#define blank (RULE_ACT_ADD_BLANK) #define or(n) (RULE_ACT_OR | n) #define and(n) (RULE_ACT_AND | n) +#define and_ident(n) (RULE_ACT_AND | n | RULE_ACT_ALLOW_IDENT) +#define and_blank(n) (RULE_ACT_AND | n | RULE_ACT_ADD_BLANK) #define one_or_more (RULE_ACT_LIST | 2) #define list (RULE_ACT_LIST | 1) #define list_with_end (RULE_ACT_LIST | 3) From b4078cbbf3354229fc81ecfd26295237d808caa9 Mon Sep 17 00:00:00 2001 From: Damien George Date: Mon, 14 Aug 2017 12:14:05 +1000 Subject: [PATCH 12/24] Revert "py/gc: Zero out all newly allocated memory to prevent stale pointers." This reverts commit 1c0343f9d991c241d335712593f3a63858dc91b6. Reason: to make merging simpler. This feature will anyway be provided by master in a later version. --- py/gc.c | 8 ++++---- py/malloc.c | 5 ++++- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/py/gc.c b/py/gc.c index ca332860f5acd..a1ba45a2bfc71 100644 --- a/py/gc.c +++ b/py/gc.c @@ -409,12 +409,12 @@ void *gc_alloc(size_t n_bytes, bool has_finaliser) { void *ret_ptr = (void*)(MP_STATE_MEM(gc_pool_start) + start_block * BYTES_PER_BLOCK); DEBUG_printf("gc_alloc(%p)\n", ret_ptr); - // Zero out all the bytes of the newly allocated blocks. + // zero out the additional bytes of the newly allocated blocks // This is needed because the blocks may have previously held pointers // to the heap and will not be set to something else if the caller // doesn't actually use the entire block. As such they will continue // to point to the heap and may prevent other blocks from being reclaimed. - memset((byte*)ret_ptr, 0, (end_block - start_block + 1) * BYTES_PER_BLOCK); + memset((byte*)ret_ptr + n_bytes, 0, (end_block - start_block + 1) * BYTES_PER_BLOCK - n_bytes); #if MICROPY_ENABLE_FINALISER if (has_finaliser) { @@ -620,8 +620,8 @@ void *gc_realloc(void *ptr_in, size_t n_bytes, bool allow_move) { ATB_FREE_TO_TAIL(bl); } - // zero out the bytes of the newly allocated blocks (see comment above in gc_alloc) - memset((byte*)ptr_in + n_blocks * BYTES_PER_BLOCK, 0, (new_blocks - n_blocks) * BYTES_PER_BLOCK); + // zero out the additional bytes of the newly allocated blocks (see comment above in gc_alloc) + memset((byte*)ptr_in + n_bytes, 0, new_blocks * BYTES_PER_BLOCK - n_bytes); #if EXTENSIVE_HEAP_PROFILING gc_dump_alloc_table(); diff --git a/py/malloc.c b/py/malloc.c index c837ed5735bf8..b0493d9341e19 100644 --- a/py/malloc.c +++ b/py/malloc.c @@ -114,7 +114,10 @@ void *m_malloc_with_finaliser(size_t num_bytes) { void *m_malloc0(size_t num_bytes) { void *ptr = m_malloc(num_bytes); - // memory is already cleared by gc_alloc + if (ptr == NULL && num_bytes != 0) { + return m_malloc_fail(num_bytes); + } + memset(ptr, 0, num_bytes); return ptr; } From d9d9b0a3005a810543724622374cc26a5399bd0e Mon Sep 17 00:00:00 2001 From: Damien George Date: Mon, 14 Aug 2017 12:26:13 +1000 Subject: [PATCH 13/24] py/compile2: Don't compile assert statement when optimisations enabled. --- py/compile2.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/py/compile2.c b/py/compile2.c index 387d6b4bdeccc..ff496d8fc2d39 100644 --- a/py/compile2.c +++ b/py/compile2.c @@ -1249,6 +1249,11 @@ STATIC void compile_nonlocal_stmt(compiler_t *comp, const byte *p, const byte *p } STATIC void compile_assert_stmt(compiler_t *comp, const byte *p, const byte *ptop) { + // with optimisations enabled we don't compile assertions + if (MP_STATE_VM(mp_optimise_value) != 0) { + return; + } + uint l_end = comp_next_label(comp); p = c_if_cond(comp, p, true, l_end); EMIT_LOAD_GLOBAL(MP_QSTR_AssertionError); // we load_global instead of load_id, to be consistent with CPython From a7f5c64d2157356ff4457d32c662403cea0ddcb7 Mon Sep 17 00:00:00 2001 From: Damien George Date: Mon, 14 Aug 2017 16:24:46 +1000 Subject: [PATCH 14/24] py/compile2: Track changes made between v1.8.4 and v1.8.5. --- py/compile2.c | 62 +++++++++++++++++---------------------------------- 1 file changed, 20 insertions(+), 42 deletions(-) diff --git a/py/compile2.c b/py/compile2.c index ff496d8fc2d39..cdd3a0ccc244a 100644 --- a/py/compile2.c +++ b/py/compile2.c @@ -510,7 +510,6 @@ STATIC void c_assign(compiler_t *comp, const byte *p, assign_kind_t assign_kind) p1 = pt_rule_extract_top(p1, &ptop); c_assign_tuple(comp, p0, p1, ptop); } else if (pt_is_rule(p1, PN_comp_for)) { - // TODO can we ever get here? can it be compiled? goto cannot_assign; } else { // sequence with 2 items @@ -930,7 +929,6 @@ STATIC void c_del_stmt(compiler_t *comp, const byte *p) { p1 = pt_next(p1); } } else if (pt_is_rule(p1, PN_comp_for)) { - // TODO not implemented; can't del comprehension? can we get here? goto cannot_delete; } else { // sequence with 2 items @@ -1218,18 +1216,14 @@ STATIC void compile_global_stmt(compiler_t *comp, const byte *p, const byte *pto STATIC void compile_declare_nonlocal(compiler_t *comp, const byte *p_for_err, qstr qst) { bool added; id_info_t *id_info = scope_find_or_add_id(comp->scope_cur, qst, &added); - if (!added && id_info->kind != ID_INFO_KIND_FREE) { + if (added) { + scope_find_local_and_close_over(comp->scope_cur, id_info, qst); + if (id_info->kind == ID_INFO_KIND_GLOBAL_IMPLICIT) { + compile_syntax_error(comp, p_for_err, "no binding for nonlocal found"); + } + } else if (id_info->kind != ID_INFO_KIND_FREE) { compile_syntax_error(comp, p_for_err, "identifier redefined as nonlocal"); - return; - } - id_info_t *id_info2 = scope_find_local_in_parent(comp->scope_cur, qst); - if (id_info2 == NULL || !(id_info2->kind == ID_INFO_KIND_LOCAL - || id_info2->kind == ID_INFO_KIND_CELL || id_info2->kind == ID_INFO_KIND_FREE)) { - compile_syntax_error(comp, p_for_err, "no binding for nonlocal found"); - return; } - id_info->kind = ID_INFO_KIND_FREE; - scope_close_over_in_parents(comp->scope_cur, qst); } STATIC void compile_nonlocal_stmt(compiler_t *comp, const byte *p, const byte *ptop) { @@ -1578,6 +1572,8 @@ STATIC void compile_try_except(compiler_t *comp, const byte *p_body, const byte EMIT_ARG(label_assign, l1); // start of exception handler EMIT(start_except_handler); + // at this point the top of the stack contains the exception instance that was raised + uint l2 = comp_next_label(comp); while (p_except != p_except_top) { @@ -1610,16 +1606,13 @@ STATIC void compile_try_except(compiler_t *comp, const byte *p_body, const byte p_except = pt_next(p_except); - EMIT(pop_top); - + // either discard or store the exception instance if (qstr_exception_local == 0) { EMIT(pop_top); } else { compile_store_id(comp, qstr_exception_local); } - EMIT(pop_top); - uint l3 = 0; if (qstr_exception_local != 0) { l3 = comp_next_label(comp); @@ -1643,7 +1636,7 @@ STATIC void compile_try_except(compiler_t *comp, const byte *p_body, const byte } EMIT_ARG(jump, l2); EMIT_ARG(label_assign, end_finally_label); - EMIT_ARG(adjust_stack_size, 3); // stack adjust for the 3 exception items + EMIT_ARG(adjust_stack_size, 1); // stack adjust for the exception instance } compile_decrease_except_level(comp); @@ -2568,22 +2561,13 @@ STATIC const byte *compile_node(compiler_t *comp, const byte *p) { p = pt_rule_extract(p, &rule_id, &src_line, &ptop); EMIT_ARG(set_source_line, src_line); compile_function_t f = compile_function[rule_id]; - if (f == NULL) { - #if MICROPY_DEBUG_PRINTERS - printf("node %u cannot be compiled\n", (uint)rule_id); - //mp_parse_node_print(pn, 0); - #endif - compile_syntax_error(comp, p, "internal compiler error"); - assert(0); - return ptop; - } else { - f(comp, p, ptop); - if (comp->compile_error != MP_OBJ_NULL && comp->compile_error_line == 0) { - // add line info for the error in case it didn't have a line number - comp->compile_error_line = src_line; - } - return ptop; + assert(f != NULL); + f(comp, p, ptop); + if (comp->compile_error != MP_OBJ_NULL && comp->compile_error_line == 0) { + // add line info for the error in case it didn't have a line number + comp->compile_error_line = src_line; } + return ptop; } } @@ -2724,17 +2708,11 @@ STATIC void compile_scope_comp_iter(compiler_t *comp, const byte *p_comp_for, co if (p_iter == p_comp_for_top) { // no more nested if/for; compile inner expression compile_node(comp, p_inner_expr); - if (comp->scope_cur->kind == SCOPE_LIST_COMP) { - EMIT_ARG(list_append, for_depth + 2); - } else if (comp->scope_cur->kind == SCOPE_DICT_COMP) { - EMIT_ARG(map_add, for_depth + 2); - #if MICROPY_PY_BUILTINS_SET - } else if (comp->scope_cur->kind == SCOPE_SET_COMP) { - EMIT_ARG(set_add, for_depth + 2); - #endif - } else { + if (comp->scope_cur->kind == SCOPE_GEN_EXPR) { EMIT(yield_value); EMIT(pop_top); + } else { + EMIT_ARG(store_comp, comp->scope_cur->kind, for_depth + 2); } } else if (pt_is_rule(p_iter, PN_comp_if)) { // if condition @@ -3163,7 +3141,7 @@ STATIC void scope_compute_things(scope_t *scope) { // __class__ is not counted as a local; if it's used then it becomes a ID_INFO_KIND_CELL continue; } - if (scope->kind >= SCOPE_FUNCTION && scope->kind <= SCOPE_GEN_EXPR && id->kind == ID_INFO_KIND_GLOBAL_IMPLICIT) { + if (SCOPE_IS_FUNC_LIKE(scope->kind) && id->kind == ID_INFO_KIND_GLOBAL_IMPLICIT) { id->kind = ID_INFO_KIND_GLOBAL_EXPLICIT; } // params always count for 1 local, even if they are a cell From 3417b28eafe5b02af64e4d59635c6334df97ac50 Mon Sep 17 00:00:00 2001 From: Damien George Date: Mon, 14 Aug 2017 18:20:00 +1000 Subject: [PATCH 15/24] py/{parse2,compile2}: Update to work with merged master, v1.9. Changes inclued: - grammar.h now has DEF_RULE and DEF_RULE_NC - iterators can be on the (Python) stack - super() handling is refactored - supe().foo(...) can be executed without the heap --- py/compile2.c | 184 +++++++++++++++++++++++++++++++++++--------------- py/parse2.c | 27 +++++++- 2 files changed, 154 insertions(+), 57 deletions(-) diff --git a/py/compile2.c b/py/compile2.c index cdd3a0ccc244a..f9636bd3c7b65 100644 --- a/py/compile2.c +++ b/py/compile2.c @@ -44,10 +44,19 @@ // TODO need to mangle __attr names typedef enum { +// define rules with a compile function #define DEF_RULE(rule, comp, kind, ...) PN_##rule, +#define DEF_RULE_NC(rule, kind, ...) #include "py/grammar.h" #undef DEF_RULE - PN_maximum_number_of, +#undef DEF_RULE_NC + PN_const_object, // special node for a constant, generic Python object +// define rules without a compile function +#define DEF_RULE(rule, comp, kind, ...) +#define DEF_RULE_NC(rule, kind, ...) PN_##rule, +#include "py/grammar.h" +#undef DEF_RULE +#undef DEF_RULE_NC } pn_kind_t; #define NEED_METHOD_TABLE MICROPY_EMIT_NATIVE @@ -79,7 +88,6 @@ typedef struct _compiler_t { uint8_t is_repl; uint8_t pass; // holds enum type pass_kind_t - uint8_t func_arg_is_super; // used to compile special case of super() function call uint8_t have_star; // try to keep compiler clean from nlr @@ -751,7 +759,6 @@ STATIC qstr compile_classdef_helper(compiler_t *comp, const byte *p, uint emit_o if (pt_is_rule(p_parents, PN_classdef_2)) { p_parents = NULL; } - comp->func_arg_is_super = false; compile_trailer_paren_helper(comp, p_parents, false, 2); // return its name (the 'C' in class C(...):") @@ -831,7 +838,6 @@ STATIC void compile_decorated(compiler_t *comp, const byte *p, const byte *ptop) // nodes[1] contains arguments to the decorator function, if any if (!pt_is_null_with_top(p, ptop_decorator)) { // call the decorator function with the arguments in nodes[1] - comp->func_arg_is_super = false; compile_node(comp, p); } } @@ -1498,6 +1504,10 @@ STATIC void compile_for_stmt(compiler_t *comp, const byte *p, const byte *ptop) goto optimise_fail; } p_range_args = pt_get_small_int(p_range_args, &step); + // the step must be non-zero + if (step == 0) { + goto optimise_fail; + } if (p_range_args != p_range_args_top) { // range has at least 4 args, so don't know how to optimise it goto optimise_fail; @@ -1536,7 +1546,7 @@ optimise_fail:; const byte *p_it = pt_next(p); const byte *p_body = compile_node(comp, p_it); // iterator - EMIT(get_iter); + EMIT_ARG(get_iter, true); EMIT_ARG(label_assign, continue_label); EMIT_ARG(for_iter, pop_label); c_assign(comp, p, ASSIGN_STORE); // variable @@ -2075,10 +2085,89 @@ STATIC void compile_factor_2(compiler_t *comp, const byte *p, const byte *ptop) } STATIC void compile_atom_expr_normal(compiler_t *comp, const byte *p, const byte *ptop) { - // this is to handle special super() call - comp->func_arg_is_super = pt_is_id(p, MP_QSTR_super); + const byte *p_start = p; + + // compile the subject of the expression + p = compile_node(comp, p); + + // get the array of trailers, it may be a single item or a list + if (pt_is_rule(p, PN_atom_expr_trailers)) { + p = pt_rule_extract_top(p, &ptop); + } + + // handle special super() call + if (comp->scope_cur->kind == SCOPE_FUNCTION + && pt_is_id(p_start, MP_QSTR_super) + && pt_is_rule(p, PN_trailer_paren) + && pt_is_rule_empty(p)) { + // at this point we have matched "super()" within a function + + // load the class for super to search for a parent + compile_load_id(comp, MP_QSTR___class__); - compile_generic_all_nodes(comp, p, ptop); + // look for first argument to function (assumes it's "self") + bool found = false; + id_info_t *id = &comp->scope_cur->id_info[0]; + for (size_t n = comp->scope_cur->id_info_len; n > 0; --n, ++id) { + if (id->flags & ID_FLAG_IS_PARAM) { + // first argument found; load it + compile_load_id(comp, id->qst); + found = true; + break; + } + } + if (!found) { + compile_syntax_error(comp, p, + "super() can't find self"); // really a TypeError + return; + } + + if (pt_num_nodes(p, ptop) >= 3 + && pt_is_rule(pt_next(p), PN_trailer_period) + && pt_is_rule(pt_next(pt_next(p)), PN_trailer_paren)) { + // optimisation for method calls super().f(...), to eliminate heap allocation + const byte *p_period = pt_next(p); + const byte *p_paren = pt_next(p_period); + qstr method_name; + pt_extract_id(pt_rule_first(p_period), &method_name); + EMIT_ARG(load_method, method_name, true); + if (pt_is_rule_empty(p_paren)) { + p_paren = NULL; + } else { + p_paren = pt_rule_first(p_paren); + } + compile_trailer_paren_helper(comp, p_paren, true, 0); + p = pt_next(p); + p = pt_next(p); + p = pt_next(p); + } else { + // a super() call + EMIT_ARG(call_function, 2, 0, 0); + p = pt_next(p); + } + } + + while (p != ptop) { + const byte *p_next = pt_next(p); + if (p_next != ptop && pt_is_rule(p, PN_trailer_period) && pt_is_rule(p_next, PN_trailer_paren)) { + // optimisation for method calls a.f(...), following PyPy + const byte *p_period = pt_rule_first(p); + const byte *p_paren; + if (pt_is_rule_empty(p_next)) { + p_paren = NULL; + } else { + p_paren = pt_rule_first(p_next); + } + qstr method_name; + pt_extract_id(p_period, &method_name); + EMIT_ARG(load_method, method_name, false); + compile_trailer_paren_helper(comp, p_paren, true, 0); + p = pt_next(p_next); + } else { + // node is one of: trailer_paren, trailer_bracket, trailer_period + p = compile_node(comp, p); + } + } } STATIC void compile_power(compiler_t *comp, const byte *p, const byte *ptop) { @@ -2090,22 +2179,6 @@ STATIC void compile_power(compiler_t *comp, const byte *p, const byte *ptop) { STATIC void compile_trailer_paren_helper(compiler_t *comp, const byte *p_arglist, bool is_method_call, int n_positional_extra) { // function to call is on top of stack - // this is to handle special super() call - if (p_arglist == NULL && comp->func_arg_is_super && comp->scope_cur->kind == SCOPE_FUNCTION) { - compile_load_id(comp, MP_QSTR___class__); - // look for first argument to function (assumes it's "self") - for (int i = 0; i < comp->scope_cur->id_info_len; i++) { - if (comp->scope_cur->id_info[i].flags & ID_FLAG_IS_PARAM) { - // first argument found; load it and call super - EMIT_LOAD_FAST(MP_QSTR_, comp->scope_cur->id_info[i].local_num); - EMIT_ARG(call_function, 2, 0, 0); - return; - } - } - compile_syntax_error(comp, NULL, "super() call cannot find self"); // really a TypeError - return; - } - // get the list of arguments const byte *ptop; if (p_arglist == NULL) { @@ -2192,30 +2265,6 @@ STATIC void compile_trailer_paren_helper(compiler_t *comp, const byte *p_arglist } } -STATIC void compile_atom_expr_trailers(compiler_t *comp, const byte *p, const byte *ptop) { - while (p != ptop) { - const byte *p_next = pt_next(p); - if (p_next != ptop && pt_is_rule(p, PN_trailer_period) && pt_is_rule(p_next, PN_trailer_paren)) { - // optimisation for method calls a.f(...), following PyPy - const byte *p_period = pt_rule_first(p); - const byte *p_paren; - if (pt_is_rule_empty(p_next)) { - p_paren = NULL; - } else { - p_paren = pt_rule_first(p_next); - } - qstr method_name; - pt_extract_id(p_period, &method_name); - EMIT_ARG(load_method, method_name); - compile_trailer_paren_helper(comp, p_paren, true, 0); - p = pt_next(p_next); - } else { - p = compile_node(comp, p); - } - comp->func_arg_is_super = false; - } -} - // p needs to point to 2 successive nodes, first is lhs of comprehension, second is PN_comp_for node STATIC void compile_comprehension(compiler_t *comp, const byte *p, scope_kind_t kind) { const byte *p_comp_for = pt_next(p); @@ -2237,7 +2286,9 @@ STATIC void compile_comprehension(compiler_t *comp, const byte *p, scope_kind_t close_over_variables_etc(comp, this_scope, 0, 0); compile_node(comp, pt_next(p_comp_for)); // source of the iterator - EMIT(get_iter); + if (kind == SCOPE_GEN_EXPR) { + EMIT_ARG(get_iter, false); + } EMIT_ARG(call_function, 1, 0, 0); } @@ -2471,7 +2522,7 @@ STATIC void compile_yield_expr(compiler_t *comp, const byte *p, const byte *ptop } else if (pt_is_rule(p, PN_yield_arg_from)) { p = pt_rule_first(p); compile_node(comp, p); - EMIT(get_iter); + EMIT_ARG(get_iter, false); EMIT_ARG(load_const_tok, MP_TOKEN_KW_NONE); EMIT(yield_from); } else { @@ -2480,16 +2531,27 @@ STATIC void compile_yield_expr(compiler_t *comp, const byte *p, const byte *ptop } } +STATIC mp_obj_t get_const_object(compiler_t *comp, const byte *p) { + size_t idx; + p = pt_extract_const_obj(p, &idx); + return (mp_obj_t)comp->co_data[idx]; +} + +STATIC void compile_const_object(compiler_t *comp, const byte *p, const byte *ptop) { + EMIT_ARG(load_const_obj, get_const_object(comp, p)); +} + typedef void (*compile_function_t)(compiler_t*, const byte*, const byte*); STATIC compile_function_t compile_function[] = { #define nc NULL #define c(f) compile_##f #define DEF_RULE(rule, comp, kind, ...) comp, +#define DEF_RULE_NC(rule, kind, ...) #include "py/grammar.h" -#undef nc #undef c #undef DEF_RULE - NULL, +#undef DEF_RULE_NC + compile_const_object, }; STATIC const byte *compile_node(compiler_t *comp, const byte *p) { @@ -2712,7 +2774,7 @@ STATIC void compile_scope_comp_iter(compiler_t *comp, const byte *p_comp_for, co EMIT(yield_value); EMIT(pop_top); } else { - EMIT_ARG(store_comp, comp->scope_cur->kind, for_depth + 2); + EMIT_ARG(store_comp, comp->scope_cur->kind, 4 * for_depth + 5); } } else if (pt_is_rule(p_iter, PN_comp_if)) { // if condition @@ -2726,7 +2788,7 @@ STATIC void compile_scope_comp_iter(compiler_t *comp, const byte *p_comp_for, co const byte *p0 = pt_rule_extract_top(p_iter, &ptop); p0 = pt_next(p0); // skip scope index compile_node(comp, pt_next(p0)); - EMIT(get_iter); + EMIT_ARG(get_iter, true); compile_scope_comp_iter(comp, p0, ptop, p_inner_expr, for_depth + 1); } @@ -2899,7 +2961,19 @@ STATIC void compile_scope(compiler_t *comp, scope_t *scope, pass_kind_t pass) { #endif } - compile_load_id(comp, qstr_arg); + // There are 4 slots on the stack for the iterator, and the first one is + // NULL to indicate that the second one points to the iterator object. + if (scope->kind == SCOPE_GEN_EXPR) { + // TODO static assert that MP_OBJ_ITER_BUF_NSLOTS == 4 + EMIT(load_null); + compile_load_id(comp, qstr_arg); + EMIT(load_null); + EMIT(load_null); + } else { + compile_load_id(comp, qstr_arg); + EMIT_ARG(get_iter, true); + } + compile_scope_comp_iter(comp, p_comp_for, p_comp_for_top, p, 0); if (scope->kind == SCOPE_GEN_EXPR) { diff --git a/py/parse2.c b/py/parse2.c index 0e729c6dde4cd..09eda60a7b035 100644 --- a/py/parse2.c +++ b/py/parse2.c @@ -65,10 +65,20 @@ typedef struct _rule_t { } rule_t; enum { +// define rules with a compile function #define DEF_RULE(rule, comp, kind, ...) RULE_##rule, +#define DEF_RULE_NC(rule, kind, ...) #include "py/grammar.h" #undef DEF_RULE - RULE_maximum_number_of, +#undef DEF_RULE_NC + RULE_const_object, // special node for a constant, generic Python object + +// define rules without a compile function +#define DEF_RULE(rule, comp, kind, ...) +#define DEF_RULE_NC(rule, kind, ...) RULE_##rule, +#include "py/grammar.h" +#undef DEF_RULE +#undef DEF_RULE_NC }; #define or(n) (RULE_ACT_OR | n) @@ -83,8 +93,10 @@ enum { #define opt_rule(r) (RULE_ARG_OPT_RULE | RULE_##r) #ifdef USE_RULE_NAME #define DEF_RULE(rule, comp, kind, ...) static const rule_t rule_##rule = { RULE_##rule, kind, #rule, { __VA_ARGS__ } }; +#define DEF_RULE_NC(rule, kind, ...) static const rule_t rule_##rule = { RULE_##rule, kind, #rule, { __VA_ARGS__ } }; #else #define DEF_RULE(rule, comp, kind, ...) static const rule_t rule_##rule = { RULE_##rule, kind, { __VA_ARGS__ } }; +#define DEF_RULE_NC(rule, kind, ...) static const rule_t rule_##rule = { RULE_##rule, kind, { __VA_ARGS__ } }; #endif #include "py/grammar.h" #undef or @@ -96,11 +108,23 @@ enum { #undef opt_rule #undef one_or_more #undef DEF_RULE +#undef DEF_RULE_NC STATIC const rule_t *rules[] = { +// define rules with a compile function #define DEF_RULE(rule, comp, kind, ...) &rule_##rule, +#define DEF_RULE_NC(rule, kind, ...) +#include "py/grammar.h" +#undef DEF_RULE +#undef DEF_RULE_NC + NULL, // RULE_const_object + +// define rules without a compile function +#define DEF_RULE(rule, comp, kind, ...) +#define DEF_RULE_NC(rule, kind, ...) &rule_##rule, #include "py/grammar.h" #undef DEF_RULE +#undef DEF_RULE_NC }; typedef struct _rule_stack_t { @@ -170,7 +194,6 @@ STATIC void push_rule(parser_t *parser, size_t src_line, const rule_t *rule, siz STATIC void push_rule_from_arg(parser_t *parser, size_t arg) { assert((arg & RULE_ARG_KIND_MASK) == RULE_ARG_RULE || (arg & RULE_ARG_KIND_MASK) == RULE_ARG_OPT_RULE); size_t rule_id = arg & RULE_ARG_ARG_MASK; - assert(rule_id < RULE_maximum_number_of); push_rule(parser, parser->lexer->tok_line, rules[rule_id], 0, 0); } From 022d6a979851461432fdec6c6855a73d462baf4d Mon Sep 17 00:00:00 2001 From: Damien George Date: Mon, 14 Aug 2017 18:21:25 +1000 Subject: [PATCH 16/24] py/parse2: Make pt_show function work. --- py/parse2.c | 19 +++++-------------- 1 file changed, 5 insertions(+), 14 deletions(-) diff --git a/py/parse2.c b/py/parse2.c index 09eda60a7b035..0394340a7b077 100644 --- a/py/parse2.c +++ b/py/parse2.c @@ -378,19 +378,10 @@ const byte *pt_rule_first(const byte *p) { } #if 0 -void pt_show(pt_t *pt) { - const byte *top = (byte*)pt->vv.buf + pt->vv.len; - /* - for (const byte *p = (byte*)pt->buf; p < top; ++p) { - if ((p - (byte*)pt->buf) % 20 == 0) { - printf("\n"); - } - printf("%02x ", *p); - } - printf("\n"); - */ - for (const byte *p = (byte*)pt->vv.buf; p < top;) { - printf("%04u ", (uint)(p - (byte*)pt->vv.buf)); +void pt_show(const byte *p, const byte *ptop) { + const byte *start = p; + while (p < ptop) { + printf("%04u ", (uint)(p - (byte*)start)); const byte *p2 = pt_advance(p, false); for (const byte *p3 = p; p3 < p2; ++p3) { printf("%02x ", *p3); @@ -1365,7 +1356,7 @@ mp_parse_tree_t mp_parse(mp_lexer_t *lex, mp_parse_input_kind_t input_kind) { #endif #if 0 - pt_show(pt); + pt_show((const byte*)pt->vv.buf, (const byte*)pt->vv.buf + pt->vv.len); { size_t n_pool, n_qstr, n_str_data_bytes, n_total_bytes; From d88eab7445a905ea04fd55e75e40934a9f9acd80 Mon Sep 17 00:00:00 2001 From: Damien George Date: Wed, 30 Aug 2017 11:23:11 +1000 Subject: [PATCH 17/24] py/parse2.h: Update to work with new version, v1.9.2. --- py/parse2.h | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/py/parse2.h b/py/parse2.h index f50af0df45214..82d9639455533 100644 --- a/py/parse2.h +++ b/py/parse2.h @@ -23,8 +23,8 @@ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN * THE SOFTWARE. */ -#ifndef __MICROPY_INCLUDED_PY_PARSE_H__ -#define __MICROPY_INCLUDED_PY_PARSE_H__ +#ifndef MICROPY_INCLUDED_PY_PARSE_H +#define MICROPY_INCLUDED_PY_PARSE_H #include #include @@ -130,4 +130,4 @@ typedef struct _mp_parse_t { mp_parse_tree_t mp_parse(struct _mp_lexer_t *lex, mp_parse_input_kind_t input_kind); void mp_parse_tree_clear(mp_parse_tree_t *tree); -#endif // __MICROPY_INCLUDED_PY_PARSE_H__ +#endif // MICROPY_INCLUDED_PY_PARSE_H From d7576c4ba798f5067d9f07c745a4ab059b1121ea Mon Sep 17 00:00:00 2001 From: Damien George Date: Wed, 30 Aug 2017 11:24:29 +1000 Subject: [PATCH 18/24] py/compile2: Fix bug with break/continue in else of optimised for-range. A port of 4c5f108321a8fd3f67f597ca918427eda813c12e. --- py/compile2.c | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/py/compile2.c b/py/compile2.c index f9636bd3c7b65..94a7591703d78 100644 --- a/py/compile2.c +++ b/py/compile2.c @@ -1455,8 +1455,19 @@ STATIC void compile_for_stmt_optimised_range(compiler_t *comp, const byte *pn_va // break/continue apply to outer loop (if any) in the else block END_BREAK_CONTINUE_BLOCK + // Compile the else block. We must pop the iterator variables before + // executing the else code because it may contain break/continue statements. + uint end_label = 0; if (pn_else != NULL) { + // discard final value of "var", and possible "end" value + EMIT(pop_top); + if (end_on_stack) { + EMIT(pop_top); + } compile_node(comp, pn_else); + end_label = comp_next_label(comp); + EMIT_ARG(jump, end_label); + EMIT_ARG(adjust_stack_size, 1 + end_on_stack); } EMIT_ARG(label_assign, break_label); @@ -1468,6 +1479,10 @@ STATIC void compile_for_stmt_optimised_range(compiler_t *comp, const byte *pn_va if (end_on_stack) { EMIT(pop_top); } + + if (pn_else != NULL) { + EMIT_ARG(label_assign, end_label); + } } STATIC void compile_for_stmt(compiler_t *comp, const byte *p, const byte *ptop) { From 4ddb838f8b03bfbbce0b7c1ffec4cf1dd1f34ec6 Mon Sep 17 00:00:00 2001 From: Damien George Date: Wed, 30 Aug 2017 11:26:05 +1000 Subject: [PATCH 19/24] py/compile2: Raise SyntaxError if positional args are given after */**. A port of 1e70fda69fcb4991eb60ed43e610f664ea1319e6 --- py/compile2.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/py/compile2.c b/py/compile2.c index 94a7591703d78..b22ae8b940ed3 100644 --- a/py/compile2.c +++ b/py/compile2.c @@ -2248,6 +2248,10 @@ STATIC void compile_trailer_paren_helper(compiler_t *comp, const byte *p_arglist n_keyword += 1; } } else { + if (star_flags) { + compile_syntax_error(comp, p, "non-keyword arg after */**"); + return; + } if (n_keyword > 0) { compile_syntax_error(comp, p, "non-keyword arg after keyword arg"); return; From 20f1bd0a14181873ffce446d90c6125b66883926 Mon Sep 17 00:00:00 2001 From: Damien George Date: Wed, 30 Aug 2017 11:26:29 +1000 Subject: [PATCH 20/24] py/compile2: Combine arith and bit-shift ops into 1 compile routine. A port of a040fb89e7b8507aa775b0620de1770642b0f5ee --- py/compile2.c | 53 ++++++++++++++------------------------------------- 1 file changed, 14 insertions(+), 39 deletions(-) diff --git a/py/compile2.c b/py/compile2.c index b22ae8b940ed3..189fa890849f1 100644 --- a/py/compile2.c +++ b/py/compile2.c @@ -2035,52 +2035,27 @@ STATIC void compile_and_expr(compiler_t *comp, const byte *p, const byte *ptop) c_binary_op(comp, p, ptop, MP_BINARY_OP_AND); } -STATIC void compile_shift_expr(compiler_t *comp, const byte *p, const byte *ptop) { - p = compile_node(comp, p); - while (p != ptop) { - byte tok; - p = pt_tok_extract(p, &tok); - p = compile_node(comp, p); - if (tok == MP_TOKEN_OP_DBL_LESS) { - EMIT_ARG(binary_op, MP_BINARY_OP_LSHIFT); - } else { - assert(tok == MP_TOKEN_OP_DBL_MORE); // should be - EMIT_ARG(binary_op, MP_BINARY_OP_RSHIFT); - } - } -} - -STATIC void compile_arith_expr(compiler_t *comp, const byte *p, const byte *ptop) { - p = compile_node(comp, p); - while (p != ptop) { - byte tok; - p = pt_tok_extract(p, &tok); - p = compile_node(comp, p); - if (tok == MP_TOKEN_OP_PLUS) { - EMIT_ARG(binary_op, MP_BINARY_OP_ADD); - } else { - assert(tok == MP_TOKEN_OP_MINUS); // should be - EMIT_ARG(binary_op, MP_BINARY_OP_SUBTRACT); - } - } -} - STATIC void compile_term(compiler_t *comp, const byte *p, const byte *ptop) { p = compile_node(comp, p); while (p != ptop) { byte tok; p = pt_tok_extract(p, &tok); p = compile_node(comp, p); - if (tok == MP_TOKEN_OP_STAR) { - EMIT_ARG(binary_op, MP_BINARY_OP_MULTIPLY); - } else if (tok == MP_TOKEN_OP_DBL_SLASH) { - EMIT_ARG(binary_op, MP_BINARY_OP_FLOOR_DIVIDE); - } else if (tok == MP_TOKEN_OP_SLASH) { - EMIT_ARG(binary_op, MP_BINARY_OP_TRUE_DIVIDE); - } else { - assert(tok == MP_TOKEN_OP_PERCENT); // should be - EMIT_ARG(binary_op, MP_BINARY_OP_MODULO); + mp_binary_op_t op; + switch (tok) { + case MP_TOKEN_OP_PLUS: op = MP_BINARY_OP_ADD; break; + case MP_TOKEN_OP_MINUS: op = MP_BINARY_OP_SUBTRACT; break; + case MP_TOKEN_OP_STAR: op = MP_BINARY_OP_MULTIPLY; break; + case MP_TOKEN_OP_DBL_SLASH: op = MP_BINARY_OP_FLOOR_DIVIDE; break; + case MP_TOKEN_OP_SLASH: op = MP_BINARY_OP_TRUE_DIVIDE; break; + case MP_TOKEN_OP_PERCENT: op = MP_BINARY_OP_MODULO; break; + case MP_TOKEN_OP_DBL_LESS: op = MP_BINARY_OP_LSHIFT; break; + default: + assert(tok == MP_TOKEN_OP_DBL_MORE); + op = MP_BINARY_OP_RSHIFT; + break; } + EMIT_ARG(binary_op, op); } } From 39b465ca831d0ec866c2a137dccbfec7f7badfba Mon Sep 17 00:00:00 2001 From: Damien George Date: Wed, 30 Aug 2017 11:29:52 +1000 Subject: [PATCH 21/24] py/compile2: Optimise emitter label indices to save a word of heap. A port of d94bc675e8f3bcf37acf2d88d7c0ce0d9de68672 --- py/compile2.c | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/py/compile2.c b/py/compile2.c index 189fa890849f1..6d162ca53e972 100644 --- a/py/compile2.c +++ b/py/compile2.c @@ -43,6 +43,8 @@ // TODO need to mangle __attr names +#define INVALID_LABEL (0xffff) + typedef enum { // define rules with a compile function #define DEF_RULE(rule, comp, kind, ...) PN_##rule, @@ -959,7 +961,7 @@ STATIC void compile_del_stmt(compiler_t *comp, const byte *p, const byte *ptop) STATIC void compile_break_stmt(compiler_t *comp, const byte *p, const byte *ptop) { (void)ptop; - if (comp->break_label == 0) { + if (comp->break_label == INVALID_LABEL) { compile_syntax_error(comp, p, "'break' outside loop"); } assert(comp->cur_except_level >= comp->break_continue_except_level); @@ -968,7 +970,7 @@ STATIC void compile_break_stmt(compiler_t *comp, const byte *p, const byte *ptop STATIC void compile_continue_stmt(compiler_t *comp, const byte *p, const byte *ptop) { (void)ptop; - if (comp->continue_label == 0) { + if (comp->continue_label == INVALID_LABEL) { compile_syntax_error(comp, p, "'continue' outside loop"); } assert(comp->cur_except_level >= comp->break_continue_except_level); @@ -2840,7 +2842,7 @@ STATIC void check_for_doc_string(compiler_t *comp, mp_parse_node_t pn) { STATIC void compile_scope(compiler_t *comp, scope_t *scope, pass_kind_t pass) { comp->pass = pass; comp->scope_cur = scope; - comp->next_label = 1; + comp->next_label = 0; EMIT_ARG(start_pass, pass, scope); if (comp->pass == MP_PASS_SCOPE) { @@ -3019,7 +3021,7 @@ STATIC void compile_scope(compiler_t *comp, scope_t *scope, pass_kind_t pass) { STATIC void compile_scope_inline_asm(compiler_t *comp, scope_t *scope, pass_kind_t pass) { comp->pass = pass; comp->scope_cur = scope; - comp->next_label = 1; + comp->next_label = 0; if (scope->kind != SCOPE_FUNCTION) { compile_syntax_error(comp, NULL, "inline assembler must be a function"); @@ -3273,6 +3275,8 @@ mp_raw_code_t *mp_compile_to_raw_code(mp_parse_tree_t *parse_tree, qstr source_f comp->source_file = source_file; comp->is_repl = is_repl; comp->co_data = parse_tree->co_data; + comp->break_label = INVALID_LABEL; + comp->continue_label = INVALID_LABEL; // create the array of scopes comp->num_scopes = pt_small_int_value(pt_next(parse_tree->root)); From 414537711dd05531c63eb830bfc9c3eecf657b1e Mon Sep 17 00:00:00 2001 From: Damien George Date: Wed, 30 Aug 2017 11:51:25 +1000 Subject: [PATCH 22/24] py: Add MICROPY_USE_SMALL_HEAP_COMPILER option, disabled by default. This new option allows the original and new parser/compiler to coexist. --- py/compile.c | 4 ++-- py/compile2.c | 4 ++-- py/mpconfig.h | 10 +++++++++- py/parse.c | 4 ++-- py/parse.h | 6 +++++- py/parse2.c | 4 ++++ py/parse2.h | 12 +++++++++--- py/py.mk | 2 ++ py/scope.c | 9 ++++++++- py/scope.h | 7 +++++-- 10 files changed, 48 insertions(+), 14 deletions(-) diff --git a/py/compile.c b/py/compile.c index 4e704abfbbadd..c856bde96fb24 100644 --- a/py/compile.c +++ b/py/compile.c @@ -36,7 +36,7 @@ #include "py/runtime.h" #include "py/asmbase.h" -#if MICROPY_ENABLE_COMPILER +#if MICROPY_ENABLE_COMPILER && !MICROPY_USE_SMALL_HEAP_COMPILER // TODO need to mangle __attr names @@ -3514,4 +3514,4 @@ mp_obj_t mp_compile(mp_parse_tree_t *parse_tree, qstr source_file, uint emit_opt return mp_make_function_from_raw_code(rc, MP_OBJ_NULL, MP_OBJ_NULL); } -#endif // MICROPY_ENABLE_COMPILER +#endif // MICROPY_ENABLE_COMPILER && !MICROPY_USE_SMALL_HEAP_COMPILER diff --git a/py/compile2.c b/py/compile2.c index 6d162ca53e972..935c50f52fe88 100644 --- a/py/compile2.c +++ b/py/compile2.c @@ -35,7 +35,7 @@ #include "py/compile.h" #include "py/runtime.h" -#if MICROPY_ENABLE_COMPILER +#if MICROPY_ENABLE_COMPILER && MICROPY_USE_SMALL_HEAP_COMPILER #if MICROPY_PY_ASYNC_AWAIT #error "async/await syntax not implemented with this parser/compiler" @@ -3476,4 +3476,4 @@ mp_obj_t mp_compile(mp_parse_tree_t *parse_tree, qstr source_file, uint emit_opt return mp_make_function_from_raw_code(rc, MP_OBJ_NULL, MP_OBJ_NULL); } -#endif // MICROPY_ENABLE_COMPILER +#endif // MICROPY_ENABLE_COMPILER && MICROPY_USE_SMALL_HEAP_COMPILER diff --git a/py/mpconfig.h b/py/mpconfig.h index dac8a903c94aa..3c2962f9a4989 100644 --- a/py/mpconfig.h +++ b/py/mpconfig.h @@ -358,6 +358,14 @@ #define MICROPY_COMP_RETURN_IF_EXPR (0) #endif +// Whether to use an alternate parser and compiler optimised for small heaps. +// This parser/compiler uses more code space but a lot less heap when building +// the parse tree. But it has the disadvantage that the entire parse tree must +// fit in a contiguous chunk of memory on the heap. +#ifndef MICROPY_USE_SMALL_HEAP_COMPILER +#define MICROPY_USE_SMALL_HEAP_COMPILER (0) +#endif + /*****************************************************************************/ /* Internal debugging stuff */ @@ -678,7 +686,7 @@ typedef double mp_float_t; // Support for async/await/async for/async with #ifndef MICROPY_PY_ASYNC_AWAIT -#define MICROPY_PY_ASYNC_AWAIT (1) +#define MICROPY_PY_ASYNC_AWAIT (!MICROPY_USE_SMALL_HEAP_COMPILER) #endif // Issue a warning when comparing str and bytes objects diff --git a/py/parse.c b/py/parse.c index e399aac53540a..694b319b3995d 100644 --- a/py/parse.c +++ b/py/parse.c @@ -41,7 +41,7 @@ #include "py/objstr.h" #include "py/builtin.h" -#if MICROPY_ENABLE_COMPILER +#if MICROPY_ENABLE_COMPILER && !MICROPY_USE_SMALL_HEAP_COMPILER #define RULE_ACT_ARG_MASK (0x0f) #define RULE_ACT_KIND_MASK (0x30) @@ -1081,4 +1081,4 @@ void mp_parse_tree_clear(mp_parse_tree_t *tree) { } } -#endif // MICROPY_ENABLE_COMPILER +#endif // MICROPY_ENABLE_COMPILER && !MICROPY_USE_SMALL_HEAP_COMPILER diff --git a/py/parse.h b/py/parse.h index 2087a2d1a27d4..4916d463bb9db 100644 --- a/py/parse.h +++ b/py/parse.h @@ -23,15 +23,17 @@ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN * THE SOFTWARE. */ -#include "py/parse2.h" #ifndef MICROPY_INCLUDED_PY_PARSE_H #define MICROPY_INCLUDED_PY_PARSE_H #include #include +#include "py/parse2.h" #include "py/obj.h" +#if !MICROPY_USE_SMALL_HEAP_COMPILER + struct _mp_lexer_t; // a mp_parse_node_t is: @@ -105,4 +107,6 @@ typedef struct _mp_parse_t { mp_parse_tree_t mp_parse(struct _mp_lexer_t *lex, mp_parse_input_kind_t input_kind); void mp_parse_tree_clear(mp_parse_tree_t *tree); +#endif // !MICROPY_USE_SMALL_HEAP_COMPILER + #endif // MICROPY_INCLUDED_PY_PARSE_H diff --git a/py/parse2.c b/py/parse2.c index 0394340a7b077..bcd35bd9d082f 100644 --- a/py/parse2.c +++ b/py/parse2.c @@ -36,6 +36,8 @@ #include "py/parsenum.h" #include "py/smallint.h" +#if MICROPY_ENABLE_COMPILER && MICROPY_USE_SMALL_HEAP_COMPILER + #define RULE_ACT_ARG_MASK (0x0f) #define RULE_ACT_KIND_MASK (0x30) #define RULE_ACT_ALLOW_IDENT (0x40) @@ -1445,3 +1447,5 @@ void mp_parse_tree_clear(mp_parse_tree_t *tree) { chunk = next; } } + +#endif // MICROPY_ENABLE_COMPILER && MICROPY_USE_SMALL_HEAP_COMPILER diff --git a/py/parse2.h b/py/parse2.h index 82d9639455533..55b05e062de8f 100644 --- a/py/parse2.h +++ b/py/parse2.h @@ -23,14 +23,16 @@ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN * THE SOFTWARE. */ -#ifndef MICROPY_INCLUDED_PY_PARSE_H -#define MICROPY_INCLUDED_PY_PARSE_H +#ifndef MICROPY_INCLUDED_PY_PARSE2_H +#define MICROPY_INCLUDED_PY_PARSE2_H #include #include #include "py/obj.h" +#if MICROPY_USE_SMALL_HEAP_COMPILER + struct _mp_lexer_t; #define MP_PT_NULL (0) @@ -42,6 +44,8 @@ struct _mp_lexer_t; #define MP_PT_ID_BASE (10) // +16 #define MP_PT_RULE_BASE (26) // +173-ish +typedef const byte *mp_parse_node_t; + extern const byte pt_const_int0[]; static inline const byte *pt_tok_extract(const byte *p, byte *tok) { @@ -130,4 +134,6 @@ typedef struct _mp_parse_t { mp_parse_tree_t mp_parse(struct _mp_lexer_t *lex, mp_parse_input_kind_t input_kind); void mp_parse_tree_clear(mp_parse_tree_t *tree); -#endif // MICROPY_INCLUDED_PY_PARSE_H +#endif // MICROPY_USE_SMALL_HEAP_COMPILER + +#endif // MICROPY_INCLUDED_PY_PARSE2_H diff --git a/py/py.mk b/py/py.mk index 55ce98f12705b..fe2c02eb05e51 100644 --- a/py/py.mk +++ b/py/py.mk @@ -117,8 +117,10 @@ PY_O_BASENAME = \ mpz.o \ reader.o \ lexer.o \ + parse.o \ parse2.o \ scope.o \ + compile.o \ compile2.o \ emitcommon.o \ emitbc.o \ diff --git a/py/scope.c b/py/scope.c index 19ab3727b7dd2..70ff93166668b 100644 --- a/py/scope.c +++ b/py/scope.c @@ -40,18 +40,25 @@ STATIC const uint8_t scope_simple_name_table[] = { [SCOPE_GEN_EXPR] = MP_QSTR__lt_genexpr_gt_, }; -scope_t *scope_new(scope_kind_t kind, const byte *pn, qstr source_file, mp_uint_t emit_options) { +scope_t *scope_new(scope_kind_t kind, mp_parse_node_t pn, qstr source_file, mp_uint_t emit_options) { scope_t *scope = m_new0(scope_t, 1); scope->kind = kind; scope->pn = pn; scope->source_file = source_file; if (kind == SCOPE_FUNCTION || kind == SCOPE_CLASS) { + #if MICROPY_USE_SMALL_HEAP_COMPILER qstr id; pt_extract_id(pn, &id); // function name scope->simple_name = id; + #else + scope->simple_name = MP_PARSE_NODE_LEAF_ARG(((mp_parse_node_struct_t*)pn)->nodes[0]); + #endif } else { scope->simple_name = scope_simple_name_table[kind]; } + #if !MICROPY_USE_SMALL_HEAP_COMPILER + scope->raw_code = mp_emit_glue_new_raw_code(); + #endif scope->emit_options = emit_options; scope->id_info_alloc = MICROPY_ALLOC_SCOPE_ID_INIT; scope->id_info = m_new(id_info_t, scope->id_info_alloc); diff --git a/py/scope.h b/py/scope.h index 370de533569b7..19df1c3d1a4aa 100644 --- a/py/scope.h +++ b/py/scope.h @@ -69,7 +69,10 @@ typedef enum { typedef struct _scope_t { scope_kind_t kind; struct _scope_t *parent; - const byte *pn; // points to the node after the scope index node + #if !MICROPY_USE_SMALL_HEAP_COMPILER + struct _scope_t *next; + #endif + mp_parse_node_t pn; // for small-heap compiler, points to the node after the scope index node uint16_t source_file; // a qstr uint16_t simple_name; // a qstr mp_raw_code_t *raw_code; @@ -86,7 +89,7 @@ typedef struct _scope_t { id_info_t *id_info; } scope_t; -scope_t *scope_new(scope_kind_t kind, const byte *pn, qstr source_file, mp_uint_t emit_options); +scope_t *scope_new(scope_kind_t kind, mp_parse_node_t pn, qstr source_file, mp_uint_t emit_options); void scope_free(scope_t *scope); id_info_t *scope_find_or_add_id(scope_t *scope, qstr qstr, bool *added); id_info_t *scope_find(scope_t *scope, qstr qstr); From d64154c730b55606e6212959f976b6fd7f4bd41b Mon Sep 17 00:00:00 2001 From: Damien George Date: Fri, 1 Sep 2017 14:55:44 +1000 Subject: [PATCH 23/24] py/emitinlinethumb: Update to work with new small-heap compiler. Note that the inline assembler only works with the small-heap compiler enabled. --- py/compile2.c | 79 +++++++++++++++++++++++------------ py/emit.h | 2 +- py/emitinlinethumb.c | 99 +++++++++++++++++++++++--------------------- py/parse2.c | 22 ++++------ py/parse2.h | 5 ++- 5 files changed, 116 insertions(+), 91 deletions(-) diff --git a/py/compile2.c b/py/compile2.c index 935c50f52fe88..8e6d76edbe1f0 100644 --- a/py/compile2.c +++ b/py/compile2.c @@ -34,6 +34,7 @@ #include "py/emit.h" #include "py/compile.h" #include "py/runtime.h" +#include "py/asmbase.h" #if MICROPY_ENABLE_COMPILER && MICROPY_USE_SMALL_HEAP_COMPILER @@ -81,6 +82,19 @@ typedef enum { #endif +#if MICROPY_EMIT_INLINE_ASM +// define macros for inline assembler +#if MICROPY_EMIT_INLINE_THUMB +#define ASM_DECORATOR_QSTR MP_QSTR_asm_thumb +#define ASM_EMITTER(f) emit_inline_thumb_##f +#elif MICROPY_EMIT_INLINE_XTENSA +#define ASM_DECORATOR_QSTR MP_QSTR_asm_xtensa +#define ASM_EMITTER(f) emit_inline_xtensa_##f +#else +#error "unknown asm emitter" +#endif +#endif + #define EMIT_INLINE_ASM(fun) (comp->emit_inline_asm_method_table->fun(comp->emit_inline_asm)) #define EMIT_INLINE_ASM_ARG(fun, ...) (comp->emit_inline_asm_method_table->fun(comp->emit_inline_asm, __VA_ARGS__)) @@ -117,7 +131,7 @@ typedef struct _compiler_t { const emit_method_table_t *emit_method_table; // current emit method table #endif - #if MICROPY_EMIT_INLINE_THUMB + #if MICROPY_EMIT_INLINE_ASM emit_inline_asm_t *emit_inline_asm; // current emitter for inline asm const emit_inline_asm_method_table_t *emit_inline_asm_method_table; // current emit method table for inline asm #endif @@ -790,10 +804,10 @@ STATIC bool compile_built_in_decorator(compiler_t *comp, const byte *p, const by } else if (attr == MP_QSTR_viper) { *emit_options = MP_EMIT_OPT_VIPER; #endif -#if MICROPY_EMIT_INLINE_THUMB - } else if (attr == MP_QSTR_asm_thumb) { - *emit_options = MP_EMIT_OPT_ASM_THUMB; -#endif + #if MICROPY_EMIT_INLINE_ASM + } else if (attr == ASM_DECORATOR_QSTR) { + *emit_options = MP_EMIT_OPT_ASM; + #endif } else { compile_syntax_error(comp, NULL, "invalid micropython decorator"); } @@ -3016,7 +3030,7 @@ STATIC void compile_scope(compiler_t *comp, scope_t *scope, pass_kind_t pass) { assert(comp->cur_except_level == 0); } -#if MICROPY_EMIT_INLINE_THUMB +#if MICROPY_EMIT_INLINE_ASM // requires 3 passes: SCOPE, CODE_SIZE, EMIT STATIC void compile_scope_inline_asm(compiler_t *comp, scope_t *scope, pass_kind_t pass) { comp->pass = pass; @@ -3029,7 +3043,7 @@ STATIC void compile_scope_inline_asm(compiler_t *comp, scope_t *scope, pass_kind } if (comp->pass > MP_PASS_SCOPE) { - EMIT_INLINE_ASM_ARG(start_pass, comp->pass, comp->scope_cur, &comp->compile_error); + EMIT_INLINE_ASM_ARG(start_pass, comp->pass, &comp->compile_error); } // get the function definition parse node @@ -3134,7 +3148,8 @@ STATIC void compile_scope_inline_asm(compiler_t *comp, scope_t *scope, pass_kind return; } if (pass > MP_PASS_SCOPE) { - EMIT_INLINE_ASM_ARG(align, pt_small_int_value(p_args)); + mp_asm_base_align((mp_asm_base_t*)comp->emit_inline_asm, + pt_small_int_value(p_args)); } } else if (op == MP_QSTR_data) { if (!(n_args >= 2 && pt_is_small_int(p_args))) { @@ -3151,7 +3166,8 @@ STATIC void compile_scope_inline_asm(compiler_t *comp, scope_t *scope, pass_kind } mp_int_t val; p_args = pt_get_small_int(p_args, &val); - EMIT_INLINE_ASM_ARG(data, bytesize, val); + mp_asm_base_data((mp_asm_base_t*)comp->emit_inline_asm, + bytesize, val); } } } else { @@ -3174,6 +3190,13 @@ STATIC void compile_scope_inline_asm(compiler_t *comp, scope_t *scope, pass_kind if (comp->pass > MP_PASS_SCOPE) { EMIT_INLINE_ASM_ARG(end_pass, type_sig); + + if (comp->pass == MP_PASS_EMIT) { + void *f = mp_asm_base_get_code((mp_asm_base_t*)comp->emit_inline_asm); + mp_emit_glue_assign_native(comp->scope_cur->raw_code, MP_CODE_NATIVE_ASM, + f, mp_asm_base_get_code_size((mp_asm_base_t*)comp->emit_inline_asm), + NULL, comp->scope_cur->num_pos_args, 0, type_sig); + } } if (comp->compile_error != MP_OBJ_NULL) { @@ -3309,10 +3332,10 @@ mp_raw_code_t *mp_compile_to_raw_code(mp_parse_tree_t *parse_tree, qstr source_f keep_going = true; s->raw_code = mp_emit_glue_new_raw_code(); if (false) { -#if MICROPY_EMIT_INLINE_THUMB - } else if (s->emit_options == MP_EMIT_OPT_ASM_THUMB) { + #if MICROPY_EMIT_INLINE_ASM + } else if (s->emit_options == MP_EMIT_OPT_ASM) { compile_scope_inline_asm(comp, s, MP_PASS_SCOPE); -#endif + #endif } else { compile_scope(comp, s, MP_PASS_SCOPE); } @@ -3337,9 +3360,6 @@ mp_raw_code_t *mp_compile_to_raw_code(mp_parse_tree_t *parse_tree, qstr source_f // compile pass 2 and 3 #if MICROPY_EMIT_NATIVE emit_t *emit_native = NULL; -#endif -#if MICROPY_EMIT_INLINE_THUMB - emit_inline_asm_t *emit_inline_thumb = NULL; #endif for (uint i = 0; i < comp->num_scopes && comp->compile_error == MP_OBJ_NULL; ++i) { scope_t *s = comp->scopes[i]; @@ -3347,20 +3367,25 @@ mp_raw_code_t *mp_compile_to_raw_code(mp_parse_tree_t *parse_tree, qstr source_f if (false) { // dummy -#if MICROPY_EMIT_INLINE_THUMB - } else if (s->emit_options == MP_EMIT_OPT_ASM_THUMB) { - // inline assembly for thumb - if (emit_inline_thumb == NULL) { - emit_inline_thumb = emit_inline_thumb_new(max_num_labels); + #if MICROPY_EMIT_INLINE_ASM + } else if (s->emit_options == MP_EMIT_OPT_ASM) { + // inline assembly + if (comp->emit_inline_asm == NULL) { + comp->emit_inline_asm = ASM_EMITTER(new)(comp->co_data, max_num_labels); } comp->emit = NULL; - comp->emit_inline_asm = emit_inline_thumb; - comp->emit_inline_asm_method_table = &emit_inline_thumb_method_table; + comp->emit_inline_asm_method_table = &ASM_EMITTER(method_table); + compile_scope_inline_asm(comp, s, MP_PASS_CODE_SIZE); + #if MICROPY_EMIT_INLINE_XTENSA + // Xtensa requires an extra pass to compute size of l32r const table + // TODO this can be improved by calculating it during SCOPE pass + // but that requires some other structural changes to the asm emitters compile_scope_inline_asm(comp, s, MP_PASS_CODE_SIZE); + #endif if (comp->compile_error == MP_OBJ_NULL) { compile_scope_inline_asm(comp, s, MP_PASS_EMIT); } -#endif + #endif } else { @@ -3445,11 +3470,11 @@ mp_raw_code_t *mp_compile_to_raw_code(mp_parse_tree_t *parse_tree, qstr source_f #endif } #endif -#if MICROPY_EMIT_INLINE_THUMB - if (emit_inline_thumb != NULL) { - emit_inline_thumb_free(emit_inline_thumb); + #if MICROPY_EMIT_INLINE_ASM + if (comp->emit_inline_asm != NULL) { + ASM_EMITTER(free)(comp->emit_inline_asm); } -#endif + #endif // free the parse tree mp_parse_tree_clear(parse_tree); diff --git a/py/emit.h b/py/emit.h index 9d5726e6eff9c..cdeedd5b5557a 100644 --- a/py/emit.h +++ b/py/emit.h @@ -271,7 +271,7 @@ typedef struct _emit_inline_asm_method_table_t { extern const emit_inline_asm_method_table_t emit_inline_thumb_method_table; extern const emit_inline_asm_method_table_t emit_inline_xtensa_method_table; -emit_inline_asm_t *emit_inline_thumb_new(mp_uint_t max_num_labels); +emit_inline_asm_t *emit_inline_thumb_new(mp_uint_t *co_data, mp_uint_t max_num_labels); emit_inline_asm_t *emit_inline_xtensa_new(mp_uint_t max_num_labels); void emit_inline_thumb_free(emit_inline_asm_t *emit); diff --git a/py/emitinlinethumb.c b/py/emitinlinethumb.c index a695b92b858a2..4e5b545e1af2c 100644 --- a/py/emitinlinethumb.c +++ b/py/emitinlinethumb.c @@ -55,6 +55,7 @@ struct _emit_inline_asm_t { asm_thumb_t as; uint16_t pass; mp_obj_t *error_slot; + mp_uint_t *co_data; mp_uint_t max_num_labels; qstr *label_lookup; }; @@ -67,10 +68,11 @@ STATIC void emit_inline_thumb_error_exc(emit_inline_asm_t *emit, mp_obj_t exc) { *emit->error_slot = exc; } -emit_inline_asm_t *emit_inline_thumb_new(mp_uint_t max_num_labels) { +emit_inline_asm_t *emit_inline_thumb_new(mp_uint_t *co_data, mp_uint_t max_num_labels) { emit_inline_asm_t *emit = m_new_obj(emit_inline_asm_t); memset(&emit->as, 0, sizeof(emit->as)); mp_asm_base_init(&emit->as.base, max_num_labels); + emit->co_data = co_data; emit->max_num_labels = max_num_labels; emit->label_lookup = m_new(qstr, max_num_labels); return emit; @@ -97,21 +99,21 @@ STATIC void emit_inline_thumb_end_pass(emit_inline_asm_t *emit, mp_uint_t type_s asm_thumb_end_pass(&emit->as); } -STATIC mp_uint_t emit_inline_thumb_count_params(emit_inline_asm_t *emit, const byte *p, const byte *ptop) { - mp_uint_t n_params = 0; - while (p != ptop) { - if (++n_params > 4) { - emit_inline_thumb_error_msg(emit, "can only have up to 4 parameters to Thumb assembly"); - return 0; - } - if (!pt_is_any_id(p)) { +STATIC mp_uint_t emit_inline_thumb_count_params(emit_inline_asm_t *emit, const byte *pn, const byte *ptop) { + mp_uint_t n_params = pt_num_nodes(pn, ptop); + if (n_params > 4) { + emit_inline_thumb_error_msg(emit, "can only have up to 4 parameters to Thumb assembly"); + return 0; + } + for (mp_uint_t i = 0; i < n_params; i++) { + if (!MP_PARSE_NODE_IS_ID(pn)) { emit_inline_thumb_error_msg(emit, "parameters must be registers in sequence r0 to r3"); return 0; } qstr qst; - p = pt_extract_id(p, &qst); - const char *param = qstr_str(qst); - if (!(strlen(param) == 2 && param[0] == 'r' && param[1] == '0' + n_params - 1)) { + pn = pt_extract_id(pn, &qst); + const char *p = qstr_str(qst); + if (!(strlen(p) == 2 && p[0] == 'r' && p[1] == '0' + i)) { emit_inline_thumb_error_msg(emit, "parameters must be registers in sequence r0 to r3"); return 0; } @@ -168,8 +170,8 @@ STATIC const special_reg_name_t special_reg_name_table[] = { // return empty string in case of error, so we can attempt to parse the string // without a special check if it was in fact a string -STATIC const char *get_arg_str(const byte *pn) { - if (pt_is_any_id(pn)) { +STATIC const char *get_arg_str(mp_parse_node_t pn) { + if (MP_PARSE_NODE_IS_ID(pn)) { qstr qst; pt_extract_id(pn, &qst); return qstr_str(qst); @@ -178,7 +180,7 @@ STATIC const char *get_arg_str(const byte *pn) { } } -STATIC mp_uint_t get_arg_reg(emit_inline_asm_t *emit, const char *op, const byte *pn, mp_uint_t max_reg) { +STATIC mp_uint_t get_arg_reg(emit_inline_asm_t *emit, const char *op, mp_parse_node_t pn, mp_uint_t max_reg) { const char *reg_str = get_arg_str(pn); for (mp_uint_t i = 0; i < MP_ARRAY_SIZE(reg_name_table); i++) { const reg_name_t *r = ®_name_table[i]; @@ -202,7 +204,7 @@ STATIC mp_uint_t get_arg_reg(emit_inline_asm_t *emit, const char *op, const byte return 0; } -STATIC mp_uint_t get_arg_special_reg(emit_inline_asm_t *emit, const char *op, const byte *pn) { +STATIC mp_uint_t get_arg_special_reg(emit_inline_asm_t *emit, const char *op, mp_parse_node_t pn) { const char *reg_str = get_arg_str(pn); for (mp_uint_t i = 0; i < MP_ARRAY_SIZE(special_reg_name_table); i++) { const special_reg_name_t *r = &special_reg_name_table[i]; @@ -217,7 +219,7 @@ STATIC mp_uint_t get_arg_special_reg(emit_inline_asm_t *emit, const char *op, co } #if MICROPY_EMIT_INLINE_THUMB_FLOAT -STATIC mp_uint_t get_arg_vfpreg(emit_inline_asm_t *emit, const char *op, const byte *pn) { +STATIC mp_uint_t get_arg_vfpreg(emit_inline_asm_t *emit, const char *op, mp_parse_node_t pn) { const char *reg_str = get_arg_str(pn); if (reg_str[0] == 's' && reg_str[1] != '\0') { mp_uint_t regno = 0; @@ -245,31 +247,32 @@ STATIC mp_uint_t get_arg_vfpreg(emit_inline_asm_t *emit, const char *op, const b } #endif -STATIC mp_uint_t get_arg_reglist(emit_inline_asm_t *emit, const char *op, const byte *p) { +STATIC mp_uint_t get_arg_reglist(emit_inline_asm_t *emit, const char *op, mp_parse_node_t pn) { // a register list looks like {r0, r1, r2} and is parsed as a Python set - if (!pt_is_rule(p, PN_atom_brace)) { + if (!pt_is_rule(pn, PN_atom_brace)) { goto bad_arg; } const byte *ptop; - p = pt_rule_extract_top(p, &ptop); + pn = pt_rule_extract_top(pn, &ptop); mp_uint_t reglist = 0; - if (p == ptop) { + if (pn == ptop) { goto bad_arg; - } else if (pt_is_any_id(p)) { + } + if (MP_PARSE_NODE_IS_ID(pn)) { // set with one element - reglist |= 1 << get_arg_reg(emit, op, p, 15); - } else if (pt_is_rule(p, PN_dictorsetmaker)) { - p = pt_rule_first(p); - const byte *p1 = pt_next(p); + reglist |= 1 << get_arg_reg(emit, op, pn, 15); + } else if (pt_is_rule(pn, PN_dictorsetmaker)) { + pn = pt_rule_first(pn); + const byte *p1 = pt_next(pn); if (pt_is_rule(p1, PN_dictorsetmaker_list)) { // set with multiple elements // get first element of set (we rely on get_arg_reg to catch syntax errors) - reglist |= 1 << get_arg_reg(emit, op, p, 15); + reglist |= 1 << get_arg_reg(emit, op, pn, 15); // get tail elements (2nd, 3rd, ...) const byte *p1_top; @@ -296,9 +299,9 @@ STATIC mp_uint_t get_arg_reglist(emit_inline_asm_t *emit, const char *op, const return 0; } -STATIC uint32_t get_arg_i(emit_inline_asm_t *emit, const char *op, const byte *pn, uint32_t fit_mask) { +STATIC uint32_t get_arg_i(emit_inline_asm_t *emit, const char *op, mp_parse_node_t pn, uint32_t fit_mask) { mp_obj_t o; - if (!mp_parse_node_get_int_maybe(pn, &o)) { + if (!mp_parse_node_get_int_maybe(pn, &o, emit->co_data)) { emit_inline_thumb_error_exc(emit, mp_obj_new_exception_msg_varg(&mp_type_SyntaxError, "'%s' expects an integer", op)); return 0; } @@ -310,25 +313,25 @@ STATIC uint32_t get_arg_i(emit_inline_asm_t *emit, const char *op, const byte *p return i; } -STATIC bool get_arg_addr(emit_inline_asm_t *emit, const char *op, const byte *p, const byte **p_base, const byte **p_offset) { - if (!pt_is_rule(p, PN_atom_bracket)) { +STATIC bool get_arg_addr(emit_inline_asm_t *emit, const char *op, mp_parse_node_t pn, mp_parse_node_t *pn_base, mp_parse_node_t *pn_offset) { + if (!pt_is_rule(pn, PN_atom_bracket)) { goto bad_arg; } - if (pt_is_rule_empty(p)) { + if (pt_is_rule_empty(pn)) { goto bad_arg; } - p = pt_rule_first(p); - if (!pt_is_rule(p, PN_testlist_comp)) { + pn = pt_rule_first(pn); + if (!pt_is_rule(pn, PN_testlist_comp)) { goto bad_arg; } const byte *ptop; - p = pt_rule_extract_top(p, &ptop); - if (pt_num_nodes(p, ptop) != 2) { + pn = pt_rule_extract_top(pn, &ptop); + if (pt_num_nodes(pn, ptop) != 2) { goto bad_arg; } - *p_base = p; - *p_offset = pt_next(p); + *pn_base = pn; + *pn_offset = pt_next(pn); return true; bad_arg: @@ -336,13 +339,13 @@ STATIC bool get_arg_addr(emit_inline_asm_t *emit, const char *op, const byte *p, return false; } -STATIC int get_arg_label(emit_inline_asm_t *emit, const char *op, const byte *p) { - if (!pt_is_any_id(p)) { +STATIC int get_arg_label(emit_inline_asm_t *emit, const char *op, mp_parse_node_t pn) { + if (!MP_PARSE_NODE_IS_ID(pn)) { emit_inline_thumb_error_exc(emit, mp_obj_new_exception_msg_varg(&mp_type_SyntaxError, "'%s' expects a label", op)); return 0; } qstr label_qstr; - pt_extract_id(p, &label_qstr); + pt_extract_id(pn, &label_qstr); for (uint i = 0; i < emit->max_num_labels; i++) { if (emit->label_lookup[i] == label_qstr) { return i; @@ -421,7 +424,7 @@ STATIC const format_vfp_op_t format_vfp_op_table[] = { // shorthand alias for whether we allow ARMv7-M instructions #define ARMV7M MICROPY_EMIT_INLINE_THUMB_ARMV7M -STATIC void emit_inline_thumb_op(emit_inline_asm_t *emit, qstr op, mp_uint_t n_args, const byte **pn_args) { +STATIC void emit_inline_thumb_op(emit_inline_asm_t *emit, qstr op, mp_uint_t n_args, mp_parse_node_t *pn_args) { // TODO perhaps make two tables: // one_args = // "b", LAB, asm_thumb_b_n, @@ -495,7 +498,7 @@ STATIC void emit_inline_thumb_op(emit_inline_asm_t *emit, qstr op, mp_uint_t n_a op_code_hi = 0xed90; op_vldr_vstr:; mp_uint_t vd = get_arg_vfpreg(emit, op_str, pn_args[0]); - const byte *pn_base, *pn_offset; + mp_parse_node_t pn_base, pn_offset; if (get_arg_addr(emit, op_str, pn_args[1], &pn_base, &pn_offset)) { mp_uint_t rlo_base = get_arg_reg(emit, op_str, pn_base, 7); mp_uint_t i8; @@ -634,7 +637,7 @@ STATIC void emit_inline_thumb_op(emit_inline_asm_t *emit, qstr op, mp_uint_t n_a } } else if (n_args == 2) { - if (pt_is_any_id(pn_args[1])) { + if (MP_PARSE_NODE_IS_ID(pn_args[1])) { // second arg is a register (or should be) mp_uint_t op_code, op_code_hi; if (op == MP_QSTR_mov) { @@ -713,7 +716,7 @@ STATIC void emit_inline_thumb_op(emit_inline_asm_t *emit, qstr op, mp_uint_t n_a asm_thumb_mov_reg_i16(&emit->as, ASM_THUMB_OP_MOVT, reg_dest, (i_src >> 16) & 0xffff); } else if (ARMV7M && op == MP_QSTR_ldrex) { mp_uint_t r_dest = get_arg_reg(emit, op_str, pn_args[0], 15); - const byte *pn_base, *pn_offset; + mp_parse_node_t pn_base, pn_offset; if (get_arg_addr(emit, op_str, pn_args[1], &pn_base, &pn_offset)) { mp_uint_t r_base = get_arg_reg(emit, op_str, pn_base, 15); mp_uint_t i8 = get_arg_i(emit, op_str, pn_offset, 0xff) >> 2; @@ -724,7 +727,7 @@ STATIC void emit_inline_thumb_op(emit_inline_asm_t *emit, qstr op, mp_uint_t n_a for (mp_uint_t i = 0; i < MP_ARRAY_SIZE(format_9_10_op_table); i++) { if (op == format_9_10_op_table[i].name) { op_code = format_9_10_op_table[i].op; - const byte *pn_base, *pn_offset; + mp_parse_node_t pn_base, pn_offset; mp_uint_t rlo_dest = get_arg_reg(emit, op_str, pn_args[0], 7); if (get_arg_addr(emit, op_str, pn_args[1], &pn_base, &pn_offset)) { mp_uint_t rlo_base = get_arg_reg(emit, op_str, pn_base, 7); @@ -769,7 +772,7 @@ STATIC void emit_inline_thumb_op(emit_inline_asm_t *emit, qstr op, mp_uint_t n_a rlo_dest = get_arg_reg(emit, op_str, pn_args[0], 7); rlo_src = get_arg_reg(emit, op_str, pn_args[1], 7); int src_b; - if (pt_is_any_id(pn_args[2])) { + if (MP_PARSE_NODE_IS_ID(pn_args[2])) { op_code |= ASM_THUMB_FORMAT_2_REG_OPERAND; src_b = get_arg_reg(emit, op_str, pn_args[2], 7); } else { @@ -794,7 +797,7 @@ STATIC void emit_inline_thumb_op(emit_inline_asm_t *emit, qstr op, mp_uint_t n_a } else if (ARMV7M && op == MP_QSTR_strex) { mp_uint_t r_dest = get_arg_reg(emit, op_str, pn_args[0], 15); mp_uint_t r_src = get_arg_reg(emit, op_str, pn_args[1], 15); - const byte *pn_base, *pn_offset; + mp_parse_node_t pn_base, pn_offset; if (get_arg_addr(emit, op_str, pn_args[2], &pn_base, &pn_offset)) { mp_uint_t r_base = get_arg_reg(emit, op_str, pn_base, 15); mp_uint_t i8 = get_arg_i(emit, op_str, pn_offset, 0xff) >> 2; diff --git a/py/parse2.c b/py/parse2.c index bcd35bd9d082f..0356274ded624 100644 --- a/py/parse2.c +++ b/py/parse2.c @@ -294,21 +294,15 @@ STATIC byte *pt_advance(const byte *p, bool full_rule) { return (byte*)p; } -bool mp_parse_node_get_int_maybe(const byte *p, mp_obj_t *o) { +bool mp_parse_node_get_int_maybe(const byte *p, mp_obj_t *o, mp_uint_t *co_data) { if (pt_is_small_int(p)) { *o = MP_OBJ_NEW_SMALL_INT(pt_small_int_value(p)); return true; - #if 0 // TODO - } else if (MP_PARSE_NODE_IS_STRUCT_KIND(pn, RULE_const_object)) { - mp_parse_node_struct_t *pns = (mp_parse_node_struct_t*)pn; - #if MICROPY_OBJ_REPR == MICROPY_OBJ_REPR_D - // nodes are 32-bit pointers, but need to extract 64-bit object - *o = (uint64_t)pns->nodes[0] | ((uint64_t)pns->nodes[1] << 32); - #else - *o = (mp_obj_t)pns->nodes[0]; - #endif - return MP_OBJ_IS_INT(*o); - #endif + } else if (*p == MP_PT_CONST_OBJECT) { + size_t idx; + p = pt_extract_const_obj(p, &idx); + *o = (mp_obj_t)co_data[idx]; + return true; } else { return false; } @@ -481,9 +475,9 @@ STATIC const byte *pt_del_byte(pt_t *pt, const byte *p) { #if MICROPY_COMP_MODULE_CONST #include "py/builtin.h" -STATIC const mp_map_elem_t mp_constants_table[] = { +STATIC const mp_rom_map_elem_t mp_constants_table[] = { #if MICROPY_PY_UCTYPES - { MP_OBJ_NEW_QSTR(MP_QSTR_uctypes), (mp_obj_t)&mp_module_uctypes }, + { MP_ROM_QSTR(MP_QSTR_uctypes), MP_ROM_PTR(&mp_module_uctypes) }, #endif // Extra constants as defined by a port MICROPY_PORT_CONSTANTS diff --git a/py/parse2.h b/py/parse2.h index 55b05e062de8f..2e363a7795c83 100644 --- a/py/parse2.h +++ b/py/parse2.h @@ -44,6 +44,9 @@ struct _mp_lexer_t; #define MP_PT_ID_BASE (10) // +16 #define MP_PT_RULE_BASE (26) // +173-ish +// macro to eliminate differences between original parser and this one +#define MP_PARSE_NODE_IS_ID(pn) pt_is_any_id(pn) + typedef const byte *mp_parse_node_t; extern const byte pt_const_int0[]; @@ -114,7 +117,7 @@ const byte *pt_rule_extract_top(const byte *p, const byte **ptop); const byte *pt_rule_extract(const byte *p, mp_uint_t *rule_id, size_t *src_line, const byte **ptop); bool pt_is_rule_empty(const byte *p); -bool mp_parse_node_get_int_maybe(const byte *p, mp_obj_t *o); +bool mp_parse_node_get_int_maybe(const byte *p, mp_obj_t *o, mp_uint_t *co_data); const byte *mp_parse_node_extract_list(const byte **p, mp_uint_t pn_kind); typedef enum { From 949b55691c598f5432c459e19e79b61443aa434b Mon Sep 17 00:00:00 2001 From: Damien George Date: Wed, 6 Sep 2017 12:46:24 +1000 Subject: [PATCH 24/24] py/compile2: Rename "Micro Python" to "MicroPython". --- py/compile2.c | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/py/compile2.c b/py/compile2.c index 8e6d76edbe1f0..e203ac5918b56 100644 --- a/py/compile2.c +++ b/py/compile2.c @@ -574,7 +574,7 @@ STATIC void close_over_variables_etc(compiler_t *comp, scope_t *this_scope, int for (int j = 0; j < this_scope->id_info_len; j++) { id_info_t *id2 = &this_scope->id_info[j]; if (id2->kind == ID_INFO_KIND_FREE && id->qst == id2->qst) { - // in Micro Python we load closures using LOAD_FAST + // in MicroPython we load closures using LOAD_FAST EMIT_LOAD_FAST(id->qst, id->local_num); nfree += 1; } @@ -662,9 +662,9 @@ STATIC void compile_funcdef_lambdef_param(compiler_t *comp, const byte *p) { if (comp->have_star) { comp->num_dict_params += 1; - // in Micro Python we put the default dict parameters into a dictionary using the bytecode + // in MicroPython we put the default dict parameters into a dictionary using the bytecode if (comp->num_dict_params == 1) { - // in Micro Python we put the default positional parameters into a tuple using the bytecode + // in MicroPython we put the default positional parameters into a tuple using the bytecode // we need to do this here before we start building the map for the default keywords if (comp->num_default_params > 0) { EMIT_ARG(build_tuple, comp->num_default_params); @@ -708,7 +708,7 @@ STATIC void compile_funcdef_lambdef(compiler_t *comp, scope_t *scope, const byte return; } - // in Micro Python we put the default positional parameters into a tuple using the bytecode + // in MicroPython we put the default positional parameters into a tuple using the bytecode // the default keywords args may have already made the tuple; if not, do it now if (comp->num_default_params > 0 && comp->num_dict_params == 0) { EMIT_ARG(build_tuple, comp->num_default_params); @@ -3208,7 +3208,7 @@ STATIC void compile_scope_inline_asm(compiler_t *comp, scope_t *scope, pass_kind #endif STATIC void scope_compute_things(scope_t *scope) { - // in Micro Python we put the *x parameter after all other parameters (except **y) + // in MicroPython we put the *x parameter after all other parameters (except **y) if (scope->scope_flags & MP_SCOPE_FLAG_VARARGS) { id_info_t *id_param = NULL; for (int i = scope->id_info_len - 1; i >= 0; i--) { @@ -3246,7 +3246,7 @@ STATIC void scope_compute_things(scope_t *scope) { // compute the index of cell vars for (int i = 0; i < scope->id_info_len; i++) { id_info_t *id = &scope->id_info[i]; - // in Micro Python the cells come right after the fast locals + // in MicroPython the cells come right after the fast locals // parameters are not counted here, since they remain at the start // of the locals, even if they are cell vars if (id->kind == ID_INFO_KIND_CELL && !(id->flags & ID_FLAG_IS_PARAM)) { @@ -3266,14 +3266,14 @@ STATIC void scope_compute_things(scope_t *scope) { id_info_t *id2 = &scope->id_info[j]; if (id2->kind == ID_INFO_KIND_FREE && id->qst == id2->qst) { assert(!(id2->flags & ID_FLAG_IS_PARAM)); // free vars should not be params - // in Micro Python the frees come first, before the params + // in MicroPython the frees come first, before the params id2->local_num = num_free; num_free += 1; } } } } - // in Micro Python shift all other locals after the free locals + // in MicroPython shift all other locals after the free locals if (num_free > 0) { for (int i = 0; i < scope->id_info_len; i++) { id_info_t *id = &scope->id_info[i]; pFad - Phonifier reborn

Pfad - The Proxy pFad of © 2024 Garber Painting. All rights reserved.

Note: This service is not intended for secure transactions such as banking, social media, email, or purchasing. Use at your own risk. We assume no liability whatsoever for broken pages.


Alternative Proxies:

Alternative Proxy

pFad Proxy

pFad v3 Proxy

pFad v4 Proxy