From b02e79f92471b8912c92fbab3e00d3115106cbc8 Mon Sep 17 00:00:00 2001 From: Andrew Leech Date: Thu, 26 Oct 2023 09:02:16 +1100 Subject: [PATCH 1/7] unix/main: Use standard pyexec/repl for unix & windows ports. This improves repl usage consistency across ports. Only enabled when MICROPY_USE_READLINE == 1 (default). Signed-off-by: Andrew Leech --- ports/unix/Makefile | 1 + ports/unix/main.c | 94 ++++--------------- ports/windows/Makefile | 1 + ports/windows/micropython.vcxproj | 1 + shared/runtime/pyexec.c | 18 +++- tests/cmdline/cmd_sys_exit_0.py | 5 + tests/cmdline/cmd_sys_exit_0.py.exp | 1 + tests/cmdline/cmd_sys_exit_error.py | 5 + tests/cmdline/cmd_sys_exit_error.py.exp | 1 + tests/cmdline/cmd_sys_exit_none.py | 5 + tests/cmdline/cmd_sys_exit_none.py.exp | 1 + tests/cmdline/repl_autocomplete.py.exp | 2 +- .../repl_autocomplete_underscore.py.exp | 3 +- tests/cmdline/repl_autoindent.py.exp | 2 +- tests/cmdline/repl_basic.py.exp | 2 +- tests/cmdline/repl_cont.py.exp | 2 +- tests/cmdline/repl_emacs_keys.py.exp | 2 +- tests/cmdline/repl_inspect.py.exp | 2 +- tests/cmdline/repl_micropyinspect.py.exp | 2 +- tests/cmdline/repl_paste.py.exp | 9 +- tests/cmdline/repl_sys_ps1_ps2.py.exp | 2 +- tests/cmdline/repl_words_move.py.exp | 2 +- 22 files changed, 72 insertions(+), 91 deletions(-) create mode 100644 tests/cmdline/cmd_sys_exit_0.py create mode 100644 tests/cmdline/cmd_sys_exit_0.py.exp create mode 100644 tests/cmdline/cmd_sys_exit_error.py create mode 100644 tests/cmdline/cmd_sys_exit_error.py.exp create mode 100644 tests/cmdline/cmd_sys_exit_none.py create mode 100644 tests/cmdline/cmd_sys_exit_none.py.exp diff --git a/ports/unix/Makefile b/ports/unix/Makefile index 8bd58a2542683..1a419b594da44 100644 --- a/ports/unix/Makefile +++ b/ports/unix/Makefile @@ -223,6 +223,7 @@ SRC_C += \ SHARED_SRC_C += $(addprefix shared/,\ runtime/gchelper_generic.c \ + runtime/pyexec.c \ timeutils/timeutils.c \ $(SHARED_SRC_C_EXTRA) \ ) diff --git a/ports/unix/main.c b/ports/unix/main.c index 530e20a3863b4..8ee3da1c946b1 100644 --- a/ports/unix/main.c +++ b/ports/unix/main.c @@ -54,6 +54,7 @@ #include "extmod/vfs_posix.h" #include "genhdr/mpversion.h" #include "input.h" +#include "shared/runtime/pyexec.h" // Command line options, with their defaults static bool compile_only = false; @@ -194,91 +195,27 @@ static char *strjoin(const char *s1, int sep_char, const char *s2) { #endif static int do_repl(void) { - mp_hal_stdout_tx_str(MICROPY_BANNER_NAME_AND_VERSION); - mp_hal_stdout_tx_str("; " MICROPY_BANNER_MACHINE); - mp_hal_stdout_tx_str("\nUse Ctrl-D to exit, Ctrl-E for paste mode\n"); - + int ret = 0; #if MICROPY_USE_READLINE == 1 - - // use MicroPython supplied readline - - vstr_t line; - vstr_init(&line, 16); + // use MicroPython supplied readline based repl + mp_hal_stdio_mode_raw(); for (;;) { - mp_hal_stdio_mode_raw(); - - input_restart: - vstr_reset(&line); - int ret = readline(&line, mp_repl_get_ps1()); - mp_parse_input_kind_t parse_input_kind = MP_PARSE_SINGLE_INPUT; - - if (ret == CHAR_CTRL_C) { - // cancel input - mp_hal_stdout_tx_str("\r\n"); - goto input_restart; - } else if (ret == CHAR_CTRL_D) { - // EOF - printf("\n"); - mp_hal_stdio_mode_orig(); - vstr_clear(&line); - return 0; - } else if (ret == CHAR_CTRL_E) { - // paste mode - mp_hal_stdout_tx_str("\npaste mode; Ctrl-C to cancel, Ctrl-D to finish\n=== "); - vstr_reset(&line); - for (;;) { - char c = mp_hal_stdin_rx_chr(); - if (c == CHAR_CTRL_C) { - // cancel everything - mp_hal_stdout_tx_str("\n"); - goto input_restart; - } else if (c == CHAR_CTRL_D) { - // end of input - mp_hal_stdout_tx_str("\n"); - break; - } else { - // add char to buffer and echo - vstr_add_byte(&line, c); - if (c == '\r') { - mp_hal_stdout_tx_str("\n=== "); - } else { - mp_hal_stdout_tx_strn(&c, 1); - } - } - } - parse_input_kind = MP_PARSE_FILE_INPUT; - } else if (line.len == 0) { - if (ret != 0) { - printf("\n"); + if (pyexec_mode_kind == PYEXEC_MODE_RAW_REPL) { + if ((ret = pyexec_raw_repl()) != 0) { + break; } - goto input_restart; } else { - // got a line with non-zero length, see if it needs continuing - while (mp_repl_continue_with_input(vstr_null_terminated_str(&line))) { - vstr_add_byte(&line, '\n'); - ret = readline(&line, mp_repl_get_ps2()); - if (ret == CHAR_CTRL_C) { - // cancel everything - printf("\n"); - goto input_restart; - } else if (ret == CHAR_CTRL_D) { - // stop entering compound statement - break; - } + if ((ret = pyexec_friendly_repl()) != 0) { + break; } } - - mp_hal_stdio_mode_orig(); - - ret = execute_from_lexer(LEX_SRC_VSTR, &line, parse_input_kind, true); - if (ret & FORCED_EXIT) { - return ret; - } } - + mp_hal_stdio_mode_orig(); #else - // use simple readline + mp_hal_stdout_tx_str(MICROPY_BANNER_NAME_AND_VERSION); + mp_hal_stdout_tx_str("; " MICROPY_BANNER_MACHINE); + mp_hal_stdout_tx_str("\nUse Ctrl-D to exit, Ctrl-E for paste mode\n"); for (;;) { char *line = prompt((char *)mp_repl_get_ps1()); @@ -297,16 +234,17 @@ static int do_repl(void) { line = line3; } - int ret = execute_from_lexer(LEX_SRC_STR, line, MP_PARSE_SINGLE_INPUT, true); + ret = execute_from_lexer(LEX_SRC_STR, line, MP_PARSE_SINGLE_INPUT, true); free(line); if (ret & FORCED_EXIT) { return ret; } } - #endif + return ret; } + static int do_file(const char *file) { return execute_from_lexer(LEX_SRC_FILENAME, file, MP_PARSE_FILE_INPUT, false); } diff --git a/ports/windows/Makefile b/ports/windows/Makefile index 9eee98cdd4538..4129b7fe2cc44 100644 --- a/ports/windows/Makefile +++ b/ports/windows/Makefile @@ -83,6 +83,7 @@ OBJ += $(addprefix $(BUILD)/, $(LIB_SRC_C:.c=.o)) ifeq ($(MICROPY_USE_READLINE),1) CFLAGS += -DMICROPY_USE_READLINE=1 SRC_C += shared/readline/readline.c +SRC_C += shared/runtime/pyexec.c endif LIB += -lws2_32 diff --git a/ports/windows/micropython.vcxproj b/ports/windows/micropython.vcxproj index 9326f3f4cde18..f8bbec82cfdc6 100644 --- a/ports/windows/micropython.vcxproj +++ b/ports/windows/micropython.vcxproj @@ -89,6 +89,7 @@ + diff --git a/shared/runtime/pyexec.c b/shared/runtime/pyexec.c index c828c75817940..aefb878fa50e8 100644 --- a/shared/runtime/pyexec.c +++ b/shared/runtime/pyexec.c @@ -41,6 +41,7 @@ #endif #include "shared/readline/readline.h" #include "shared/runtime/pyexec.h" +#include "extmod/modplatform.h" #include "genhdr/mpversion.h" pyexec_mode_kind_t pyexec_mode_kind = PYEXEC_MODE_FRIENDLY_REPL; @@ -103,6 +104,14 @@ static int parse_compile_execute(const void *source, mp_parse_input_kind_t input // source is a lexer, parse and compile the script qstr source_name = lex->source_name; mp_parse_tree_t parse_tree = mp_parse(lex, input_kind); + #if defined(MICROPY_UNIX_COVERAGE) + // allow to print the parse tree in the coverage build + if (mp_verbose_flag >= 3) { + printf("----------------\n"); + mp_parse_node_print(&mp_plat_print, parse_tree.root, 0); + printf("----------------\n"); + } + #endif module_fun = mp_compile(&parse_tree, source_name, exec_flags & EXEC_FLAG_IS_REPL); #else mp_raise_msg(&mp_type_RuntimeError, MP_ERROR_TEXT("script compilation not supported")); @@ -141,8 +150,13 @@ static int parse_compile_execute(const void *source, mp_parse_input_kind_t input // check for SystemExit if (mp_obj_is_subclass_fast(MP_OBJ_FROM_PTR(((mp_obj_base_t *)nlr.ret_val)->type), MP_OBJ_FROM_PTR(&mp_type_SystemExit))) { - // at the moment, the value of SystemExit is unused - ret = PYEXEC_FORCED_EXIT; + // Extract SystemExit value + mp_obj_t exit_val = mp_obj_exception_get_value(MP_OBJ_FROM_PTR(nlr.ret_val)); + mp_int_t val = 0; + if (exit_val != mp_const_none && !mp_obj_get_int_maybe(exit_val, &val)) { + val = 1; + } + ret = PYEXEC_FORCED_EXIT | (val & 255); } else { mp_obj_print_exception(&mp_plat_print, MP_OBJ_FROM_PTR(nlr.ret_val)); ret = 0; diff --git a/tests/cmdline/cmd_sys_exit_0.py b/tests/cmdline/cmd_sys_exit_0.py new file mode 100644 index 0000000000000..1294b739e8ff1 --- /dev/null +++ b/tests/cmdline/cmd_sys_exit_0.py @@ -0,0 +1,5 @@ +# cmdline: +# test sys.exit(0) - success exit code +import sys + +sys.exit(0) diff --git a/tests/cmdline/cmd_sys_exit_0.py.exp b/tests/cmdline/cmd_sys_exit_0.py.exp new file mode 100644 index 0000000000000..8b137891791fe --- /dev/null +++ b/tests/cmdline/cmd_sys_exit_0.py.exp @@ -0,0 +1 @@ + diff --git a/tests/cmdline/cmd_sys_exit_error.py b/tests/cmdline/cmd_sys_exit_error.py new file mode 100644 index 0000000000000..ecac15e94f1bf --- /dev/null +++ b/tests/cmdline/cmd_sys_exit_error.py @@ -0,0 +1,5 @@ +# cmdline: +# test sys.exit() functionality and exit codes +import sys + +sys.exit(123) diff --git a/tests/cmdline/cmd_sys_exit_error.py.exp b/tests/cmdline/cmd_sys_exit_error.py.exp new file mode 100644 index 0000000000000..3911f71d6244d --- /dev/null +++ b/tests/cmdline/cmd_sys_exit_error.py.exp @@ -0,0 +1 @@ +CRASH \ No newline at end of file diff --git a/tests/cmdline/cmd_sys_exit_none.py b/tests/cmdline/cmd_sys_exit_none.py new file mode 100644 index 0000000000000..66e19666589ed --- /dev/null +++ b/tests/cmdline/cmd_sys_exit_none.py @@ -0,0 +1,5 @@ +# cmdline: +# test sys.exit(None) - should exit with code 0 +import sys + +sys.exit(None) diff --git a/tests/cmdline/cmd_sys_exit_none.py.exp b/tests/cmdline/cmd_sys_exit_none.py.exp new file mode 100644 index 0000000000000..8b137891791fe --- /dev/null +++ b/tests/cmdline/cmd_sys_exit_none.py.exp @@ -0,0 +1 @@ + diff --git a/tests/cmdline/repl_autocomplete.py.exp b/tests/cmdline/repl_autocomplete.py.exp index 75002985e3c63..8cf71bb447d43 100644 --- a/tests/cmdline/repl_autocomplete.py.exp +++ b/tests/cmdline/repl_autocomplete.py.exp @@ -1,5 +1,5 @@ MicroPython \.\+ version -Use \.\+ +Type "help()" for more information. >>> # tests for autocompletion >>> import sys >>> not_exist. diff --git a/tests/cmdline/repl_autocomplete_underscore.py.exp b/tests/cmdline/repl_autocomplete_underscore.py.exp index 35617554f57b7..3960fd1a3c984 100644 --- a/tests/cmdline/repl_autocomplete_underscore.py.exp +++ b/tests/cmdline/repl_autocomplete_underscore.py.exp @@ -1,5 +1,5 @@ MicroPython \.\+ version -Use Ctrl-D to exit, Ctrl-E for paste mode +Type "help()" for more information. >>> # Test REPL autocompletion filtering of underscore attributes >>> >>> # Start paste mode @@ -27,6 +27,7 @@ paste mode; Ctrl-C to cancel, Ctrl-D to finish === return 99 === === +>>> >>> # Paste executed >>> >>> # Create an instance diff --git a/tests/cmdline/repl_autoindent.py.exp b/tests/cmdline/repl_autoindent.py.exp index 9127a7d31d903..f45bf840f092a 100644 --- a/tests/cmdline/repl_autoindent.py.exp +++ b/tests/cmdline/repl_autoindent.py.exp @@ -1,5 +1,5 @@ MicroPython \.\+ version -Use \.\+ +Type "help()" for more information. >>> # tests for autoindent >>> if 1: ... print(1) diff --git a/tests/cmdline/repl_basic.py.exp b/tests/cmdline/repl_basic.py.exp index 2b390ea98bb75..a190684743285 100644 --- a/tests/cmdline/repl_basic.py.exp +++ b/tests/cmdline/repl_basic.py.exp @@ -1,5 +1,5 @@ MicroPython \.\+ version -Use \.\+ +Type "help()" for more information. >>> # basic REPL tests >>> print(1) 1 diff --git a/tests/cmdline/repl_cont.py.exp b/tests/cmdline/repl_cont.py.exp index 834c18a4d3699..d0d20adc49617 100644 --- a/tests/cmdline/repl_cont.py.exp +++ b/tests/cmdline/repl_cont.py.exp @@ -1,5 +1,5 @@ MicroPython \.\+ version -Use \.\+ +Type "help()" for more information. >>> # check REPL allows to continue input >>> 1 \\\\ ... + 2 diff --git a/tests/cmdline/repl_emacs_keys.py.exp b/tests/cmdline/repl_emacs_keys.py.exp index 6102c19639a8a..b8b7b794f2d7c 100644 --- a/tests/cmdline/repl_emacs_keys.py.exp +++ b/tests/cmdline/repl_emacs_keys.py.exp @@ -1,5 +1,5 @@ MicroPython \.\+ version -Use \.\+ +Type "help()" for more information. >>> # REPL tests of GNU-ish readline navigation >>> # history buffer navigation >>> 1 diff --git a/tests/cmdline/repl_inspect.py.exp b/tests/cmdline/repl_inspect.py.exp index 051acfd153a61..89ae142019b8e 100644 --- a/tests/cmdline/repl_inspect.py.exp +++ b/tests/cmdline/repl_inspect.py.exp @@ -1,6 +1,6 @@ test MicroPython \.\+ version -Use \.\+ +Type "help()" for more information. >>> # cmdline: -i -c print("test") >>> # -c option combined with -i option results in REPL >>> diff --git a/tests/cmdline/repl_micropyinspect.py.exp b/tests/cmdline/repl_micropyinspect.py.exp index 93ff43546eace..504bb07d7d45e 100644 --- a/tests/cmdline/repl_micropyinspect.py.exp +++ b/tests/cmdline/repl_micropyinspect.py.exp @@ -1,5 +1,5 @@ MicroPython \.\+ version -Use \.\+ +Type "help()" for more information. >>> # cmdline: cmdline/repl_micropyinspect >>> # setting MICROPYINSPECT environment variable before program exit triggers REPL >>> diff --git a/tests/cmdline/repl_paste.py.exp b/tests/cmdline/repl_paste.py.exp index 22d9bd574006a..ecf144e5c11ba 100644 --- a/tests/cmdline/repl_paste.py.exp +++ b/tests/cmdline/repl_paste.py.exp @@ -1,5 +1,5 @@ MicroPython \.\+ version -Use Ctrl-D to exit, Ctrl-E for paste mode +Type "help()" for more information. >>> # Test REPL paste mode functionality >>> >>> # Basic paste mode with a simple function @@ -12,6 +12,7 @@ paste mode; Ctrl-C to cancel, Ctrl-D to finish === Hello from paste mode! >>> +>>> >>> # Paste mode with multiple indentation levels >>> paste mode; Ctrl-C to cancel, Ctrl-D to finish @@ -34,6 +35,7 @@ Even: 2 Odd: 3 Even: 4 >>> +>>> >>> # Paste mode with blank lines >>> paste mode; Ctrl-C to cancel, Ctrl-D to finish @@ -52,6 +54,7 @@ First line After blank line After two blank lines >>> +>>> >>> # Paste mode with class definition and multiple methods >>> paste mode; Ctrl-C to cancel, Ctrl-D to finish @@ -76,6 +79,7 @@ Value is: 21 Doubled: 42 Value is: 42 >>> +>>> >>> # Paste mode with exception handling >>> paste mode; Ctrl-C to cancel, Ctrl-D to finish @@ -90,6 +94,7 @@ paste mode; Ctrl-C to cancel, Ctrl-D to finish Caught division by zero Finally block executed >>> +>>> >>> # Cancel paste mode with Ctrl-C >>> paste mode; Ctrl-C to cancel, Ctrl-D to finish @@ -113,6 +118,7 @@ Traceback (most recent call last): File "", line 2 SyntaxError: invalid syntax >>> +>>> >>> # Paste mode with runtime error >>> paste mode; Ctrl-C to cancel, Ctrl-D to finish @@ -127,6 +133,7 @@ Traceback (most recent call last): File "", line 3, in will_error NameError: name 'undefined_variable' isn't defined >>> +>>> >>> # Final test to show REPL is still functioning >>> 1 + 2 + 3 6 diff --git a/tests/cmdline/repl_sys_ps1_ps2.py.exp b/tests/cmdline/repl_sys_ps1_ps2.py.exp index 9e82db5e313e4..6781660bf337d 100644 --- a/tests/cmdline/repl_sys_ps1_ps2.py.exp +++ b/tests/cmdline/repl_sys_ps1_ps2.py.exp @@ -1,5 +1,5 @@ MicroPython \.\+ version -Use \.\+ +Type "help()" for more information. >>> # test changing ps1/ps2 >>> import sys >>> sys.ps1 = "PS1" diff --git a/tests/cmdline/repl_words_move.py.exp b/tests/cmdline/repl_words_move.py.exp index 86f6b7788989e..c4d22a0d9a7fc 100644 --- a/tests/cmdline/repl_words_move.py.exp +++ b/tests/cmdline/repl_words_move.py.exp @@ -1,5 +1,5 @@ MicroPython \.\+ version -Use \.\+ +Type "help()" for more information. >>> # word movement >>> # backward-word, start in word >>> \.\+ From 03fe1aac2b98b9e00ca40451632dd8bcd7b9f007 Mon Sep 17 00:00:00 2001 From: Andrew Leech Date: Mon, 26 May 2025 19:28:40 +1000 Subject: [PATCH 2/7] shared/runtime/pyexec: Set __file__ for file input when enabled. When MICROPY_PY___FILE__ is enabled and parsing file input, set the global __file__ variable to the source filename. This matches the behavior of the unix port and provides the current filename to the executing script. Signed-off-by: Andrew Leech --- shared/runtime/pyexec.c | 5 +++++ tests/cmdline/cmd_file_variable.py | 5 +++++ tests/cmdline/cmd_file_variable.py.exp | 1 + 3 files changed, 11 insertions(+) create mode 100644 tests/cmdline/cmd_file_variable.py create mode 100644 tests/cmdline/cmd_file_variable.py.exp diff --git a/shared/runtime/pyexec.c b/shared/runtime/pyexec.c index aefb878fa50e8..1be2bc7afaff3 100644 --- a/shared/runtime/pyexec.c +++ b/shared/runtime/pyexec.c @@ -103,6 +103,11 @@ static int parse_compile_execute(const void *source, mp_parse_input_kind_t input } // source is a lexer, parse and compile the script qstr source_name = lex->source_name; + #if MICROPY_PY___FILE__ + if (input_kind == MP_PARSE_FILE_INPUT) { + mp_store_global(MP_QSTR___file__, MP_OBJ_NEW_QSTR(source_name)); + } + #endif mp_parse_tree_t parse_tree = mp_parse(lex, input_kind); #if defined(MICROPY_UNIX_COVERAGE) // allow to print the parse tree in the coverage build diff --git a/tests/cmdline/cmd_file_variable.py b/tests/cmdline/cmd_file_variable.py new file mode 100644 index 0000000000000..6cac6744d904e --- /dev/null +++ b/tests/cmdline/cmd_file_variable.py @@ -0,0 +1,5 @@ +# Test that __file__ is set correctly for script execution +try: + print("__file__ =", __file__) +except NameError: + print("__file__ not defined") diff --git a/tests/cmdline/cmd_file_variable.py.exp b/tests/cmdline/cmd_file_variable.py.exp new file mode 100644 index 0000000000000..0fac9137b0701 --- /dev/null +++ b/tests/cmdline/cmd_file_variable.py.exp @@ -0,0 +1 @@ +__file__ = cmdline/cmd_file_variable.py \ No newline at end of file From 3a77db61427cc1bb1b0cd0c2a44f865c08240141 Mon Sep 17 00:00:00 2001 From: Andrew Leech Date: Mon, 26 May 2025 19:28:02 +1000 Subject: [PATCH 3/7] shared/runtime/pyexec: Provide support for compile-only mode. When MICROPY_PYEXEC_COMPILE_ONLY is enabled and global mp_compile_only is True code is compiled but not executed. Also add comprehensive tests for compile-only functionality covering both successful compilation and syntax error detection. Signed-off-by: Andrew Leech --- pyproject.toml | 8 +++++++- shared/runtime/pyexec.c | 8 +++++++- tests/cmdline/cmd_compile_only.py | 13 +++++++++++++ tests/cmdline/cmd_compile_only.py.exp | 1 + tests/cmdline/cmd_compile_only_error.py | 6 ++++++ tests/cmdline/cmd_compile_only_error.py.exp | 1 + 6 files changed, 35 insertions(+), 2 deletions(-) create mode 100644 tests/cmdline/cmd_compile_only.py create mode 100644 tests/cmdline/cmd_compile_only.py.exp create mode 100644 tests/cmdline/cmd_compile_only_error.py create mode 100644 tests/cmdline/cmd_compile_only_error.py.exp diff --git a/pyproject.toml b/pyproject.toml index 0dd15d06c7bcc..ddf6164cc907f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -28,6 +28,7 @@ target-version = "py37" [tool.ruff.lint] exclude = [ # Ruff finds Python SyntaxError in these files + "tests/cmdline/cmd_compile_only_error.py", "tests/cmdline/repl_autocomplete.py", "tests/cmdline/repl_autocomplete_underscore.py", "tests/cmdline/repl_autoindent.py", @@ -67,5 +68,10 @@ mccabe.max-complexity = 40 # basics: needs careful attention before applying automatic formatting # repl_: not real python files # viper_args: uses f(*) -exclude = ["tests/basics/*.py", "tests/*/repl_*.py", "tests/micropython/viper_args.py"] +exclude = [ + "tests/basics/*.py", + "tests/*/repl_*.py", + "tests/cmdline/cmd_compile_only_error.py", + "tests/micropython/viper_args.py", +] quote-style = "preserve" diff --git a/shared/runtime/pyexec.c b/shared/runtime/pyexec.c index 1be2bc7afaff3..9d8c77ef14c66 100644 --- a/shared/runtime/pyexec.c +++ b/shared/runtime/pyexec.c @@ -130,7 +130,12 @@ static int parse_compile_execute(const void *source, mp_parse_input_kind_t input #if MICROPY_REPL_INFO start = mp_hal_ticks_ms(); #endif - mp_call_function_0(module_fun); + #if MICROPY_PYEXEC_COMPILE_ONLY + if (!mp_compile_only) + #endif + { + mp_call_function_0(module_fun); + } mp_hal_set_interrupt_char(-1); // disable interrupt mp_handle_pending(true); // handle any pending exceptions (and any callbacks) nlr_pop(); @@ -701,6 +706,7 @@ int pyexec_file(const char *filename) { return parse_compile_execute(filename, MP_PARSE_FILE_INPUT, EXEC_FLAG_SOURCE_IS_FILENAME); } + int pyexec_file_if_exists(const char *filename) { #if MICROPY_MODULE_FROZEN if (mp_find_frozen_module(filename, NULL, NULL) == MP_IMPORT_STAT_FILE) { diff --git a/tests/cmdline/cmd_compile_only.py b/tests/cmdline/cmd_compile_only.py new file mode 100644 index 0000000000000..89964c1b5bdf2 --- /dev/null +++ b/tests/cmdline/cmd_compile_only.py @@ -0,0 +1,13 @@ +# cmdline: -X compile-only +# test compile-only functionality +print("This should not be printed") +x = 1 + 2 + + +def hello(): + return "world" + + +class TestClass: + def __init__(self): + self.value = 42 diff --git a/tests/cmdline/cmd_compile_only.py.exp b/tests/cmdline/cmd_compile_only.py.exp new file mode 100644 index 0000000000000..8b137891791fe --- /dev/null +++ b/tests/cmdline/cmd_compile_only.py.exp @@ -0,0 +1 @@ + diff --git a/tests/cmdline/cmd_compile_only_error.py b/tests/cmdline/cmd_compile_only_error.py new file mode 100644 index 0000000000000..326937a5c07ef --- /dev/null +++ b/tests/cmdline/cmd_compile_only_error.py @@ -0,0 +1,6 @@ +# cmdline: -X compile-only +# test compile-only with syntax error +print("This should not be printed") +def broken_syntax( + # Missing closing parenthesis + return "error" diff --git a/tests/cmdline/cmd_compile_only_error.py.exp b/tests/cmdline/cmd_compile_only_error.py.exp new file mode 100644 index 0000000000000..3911f71d6244d --- /dev/null +++ b/tests/cmdline/cmd_compile_only_error.py.exp @@ -0,0 +1 @@ +CRASH \ No newline at end of file From e2cd5a02727484f180f5214b6f8aec14cd9104d2 Mon Sep 17 00:00:00 2001 From: Andrew Leech Date: Mon, 26 May 2025 20:25:01 +1000 Subject: [PATCH 4/7] unix: Enable compile-only mode with shared pyexec repl. Provides support for command line -X compile-only option on unix port. Signed-off-by: Andrew Leech --- ports/unix/main.c | 17 +++++++++++++---- ports/unix/mpconfigport.h | 3 +++ ports/unix/mphalport.h | 4 ++++ ports/windows/mpconfigport.h | 3 +++ 4 files changed, 23 insertions(+), 4 deletions(-) diff --git a/ports/unix/main.c b/ports/unix/main.c index 8ee3da1c946b1..2ac8bed267914 100644 --- a/ports/unix/main.c +++ b/ports/unix/main.c @@ -57,7 +57,7 @@ #include "shared/runtime/pyexec.h" // Command line options, with their defaults -static bool compile_only = false; +bool mp_compile_only = false; static uint emit_opt = MP_EMIT_OPT_NONE; #if MICROPY_ENABLE_GC @@ -158,7 +158,7 @@ static int execute_from_lexer(int source_kind, const void *source, mp_parse_inpu mp_obj_t module_fun = mp_compile(&parse_tree, source_name, is_repl); - if (!compile_only) { + if (!mp_compile_only) { // execute it mp_call_function_0(module_fun); } @@ -246,7 +246,16 @@ static int do_repl(void) { static int do_file(const char *file) { - return execute_from_lexer(LEX_SRC_FILENAME, file, MP_PARSE_FILE_INPUT, false); + int ret = pyexec_file(file); + // pyexec returns 1 for success, 0 for exception, PYEXEC_FORCED_EXIT for SystemExit + // Convert to unix port's expected codes: 0 for success, 1 for exception, FORCED_EXIT|val for SystemExit + if (ret == 1) { + return 0; // success + } else if (ret & PYEXEC_FORCED_EXIT) { + return ret; // SystemExit with exit value in lower 8 bits + } else { + return 1; // exception + } } static int do_str(const char *str) { @@ -319,7 +328,7 @@ static void pre_process_options(int argc, char **argv) { } if (0) { } else if (strcmp(argv[a + 1], "compile-only") == 0) { - compile_only = true; + mp_compile_only = true; } else if (strcmp(argv[a + 1], "emit=bytecode") == 0) { emit_opt = MP_EMIT_OPT_BYTECODE; #if MICROPY_EMIT_NATIVE diff --git a/ports/unix/mpconfigport.h b/ports/unix/mpconfigport.h index 973b5e74ce10d..c0b7c25843f7a 100644 --- a/ports/unix/mpconfigport.h +++ b/ports/unix/mpconfigport.h @@ -160,6 +160,9 @@ typedef long mp_off_t; // Enable sys.executable. #define MICROPY_PY_SYS_EXECUTABLE (1) +// Enable support for compile-only mode. +#define MICROPY_PYEXEC_COMPILE_ONLY (1) + #define MICROPY_PY_SOCKET_LISTEN_BACKLOG_DEFAULT (SOMAXCONN < 128 ? SOMAXCONN : 128) // Bare-metal ports don't have stderr. Printing debug to stderr may give tests diff --git a/ports/unix/mphalport.h b/ports/unix/mphalport.h index 02b60d8a873b9..018427546ea27 100644 --- a/ports/unix/mphalport.h +++ b/ports/unix/mphalport.h @@ -25,6 +25,7 @@ */ #include #include +#include #ifndef CHAR_CTRL_C #define CHAR_CTRL_C (3) @@ -112,3 +113,6 @@ enum { void mp_hal_get_mac(int idx, uint8_t buf[6]); #endif + +// Global variable to control compile-only mode. +extern bool mp_compile_only; diff --git a/ports/windows/mpconfigport.h b/ports/windows/mpconfigport.h index 4e140d5edb7d0..a66a57d48c0da 100644 --- a/ports/windows/mpconfigport.h +++ b/ports/windows/mpconfigport.h @@ -161,6 +161,9 @@ #define MICROPY_MACHINE_MEM_GET_READ_ADDR mod_machine_mem_get_addr #define MICROPY_MACHINE_MEM_GET_WRITE_ADDR mod_machine_mem_get_addr +// Enable support for compile-only mode. +#define MICROPY_PYEXEC_COMPILE_ONLY (1) + #define MICROPY_ERROR_REPORTING (MICROPY_ERROR_REPORTING_DETAILED) #define MICROPY_ERROR_PRINTER (&mp_stderr_print) #define MICROPY_WARNINGS (1) From dbda9284b8b2d7cf6a2f970f3b7195e1773e89b8 Mon Sep 17 00:00:00 2001 From: Andrew Leech Date: Mon, 26 May 2025 19:31:30 +1000 Subject: [PATCH 5/7] tests/run-tests: Add general newline normalization function. Add a general normalize_newlines() function that handles newline variations (\\r\\r\\n, \\r\\n) to \\n while preserving literal \\r characters that are part of test content. This provides a robust solution for cross-platform test compatibility, particularly addressing PTY double-newline issues that can occur with some terminal implementations. The function is applied to all test output before comparison, eliminating platform-specific newline issues. Includes a unit test to verify the normalization behavior. Signed-off-by: Andrew Leech --- .gitattributes | 1 + pyproject.toml | 1 + tests/micropython/test_normalize_newlines.py | 14 ++++++++++++++ .../test_normalize_newlines.py.exp | 8 ++++++++ tests/run-tests.py | 19 ++++++++++++++++++- 5 files changed, 42 insertions(+), 1 deletion(-) create mode 100644 tests/micropython/test_normalize_newlines.py create mode 100644 tests/micropython/test_normalize_newlines.py.exp diff --git a/.gitattributes b/.gitattributes index 2d8496db50488..c14a61b0d97d7 100644 --- a/.gitattributes +++ b/.gitattributes @@ -18,6 +18,7 @@ # These should also not be modified by git. tests/basics/string_cr_conversion.py -text tests/basics/string_crlf_conversion.py -text +tests/micropython/test_normalize_newlines.py.exp -text ports/stm32/pybcdc.inf_template -text ports/stm32/usbhost/** -text ports/cc3200/hal/aes.c -text diff --git a/pyproject.toml b/pyproject.toml index ddf6164cc907f..6f61cac15d8a0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -72,6 +72,7 @@ exclude = [ "tests/basics/*.py", "tests/*/repl_*.py", "tests/cmdline/cmd_compile_only_error.py", + "tests/micropython/test_normalize_newlines.py", "tests/micropython/viper_args.py", ] quote-style = "preserve" diff --git a/tests/micropython/test_normalize_newlines.py b/tests/micropython/test_normalize_newlines.py new file mode 100644 index 0000000000000..f19aaa69a3f75 --- /dev/null +++ b/tests/micropython/test_normalize_newlines.py @@ -0,0 +1,14 @@ +# Test for normalize_newlines functionality +# This test verifies that test framework handles various newline combinations correctly + +# Note: This is more of an integration test since normalize_newlines is in the test framework +# The actual testing happens when this test is run through run-tests.py + +print("Testing newline handling") +print("Line 1\r\nLine 2") # Windows-style line ending - should be normalized +print("Line 3") # Normal line +print("Line 4") # Normal line +print("Line 5\nLine 6") # Unix-style line ending - already normalized + +# Test that literal \r in strings is preserved +print(repr("test\rstring")) # Should show 'test\rstring' not 'test\nstring' diff --git a/tests/micropython/test_normalize_newlines.py.exp b/tests/micropython/test_normalize_newlines.py.exp new file mode 100644 index 0000000000000..c4395468cf109 --- /dev/null +++ b/tests/micropython/test_normalize_newlines.py.exp @@ -0,0 +1,8 @@ +Testing newline handling +Line 1 +Line 2 +Line 3 +Line 4 +Line 5 +Line 6 +'test\rstring' diff --git a/tests/run-tests.py b/tests/run-tests.py index 0eaee5278e0b0..d3db9a3c91236 100755 --- a/tests/run-tests.py +++ b/tests/run-tests.py @@ -58,6 +58,23 @@ def base_path(*p): # Set PYTHONIOENCODING so that CPython will use utf-8 on systems which set another encoding in the locale os.environ["PYTHONIOENCODING"] = "utf-8" + +def normalize_newlines(data): + """Normalize newline variations to \\n. + + Only normalizes actual line endings, not literal \\r characters in strings. + Handles \\r\\r\\n and \\r\\n cases to ensure consistent comparison + across different platforms and terminals. + """ + if isinstance(data, bytes): + # Handle PTY double-newline issue first + data = data.replace(b'\r\r\n', b'\n') + # Then handle standard Windows line endings + data = data.replace(b'\r\n', b'\n') + # Don't convert standalone \r as it might be literal content + return data + + # Code to allow a target MicroPython to import an .mpy from RAM # Note: the module is named `__injected_test` but it needs to have `__name__` set to # `__main__` so that the test sees itself as the main module, eg so unittest works. @@ -490,7 +507,7 @@ def send_get(what): ) # canonical form for all ports/platforms is to use \n for end-of-line - output_mupy = output_mupy.replace(b"\r\n", b"\n") + output_mupy = normalize_newlines(output_mupy) # don't try to convert the output if we should skip this test if had_crash or output_mupy in (b"SKIP\n", b"SKIP-TOO-LARGE\n", b"CRASH"): From 11241dae20fc1d453b7ea81f1f269b60c78c272a Mon Sep 17 00:00:00 2001 From: Andrew Leech Date: Tue, 22 Jul 2025 15:28:28 +1000 Subject: [PATCH 6/7] unix/main: Replace execute_from_lexer with pyexec_vstr in do_str. Consolidates string execution to use the standard pyexec interface for consistency with other ports. Simplify execute_from_lexer for remaining usage: Remove unused LEX_SRC_VSTR and LEX_SRC_FILENAME cases, keeping only LEX_SRC_STR for REPL and LEX_SRC_STDIN for stdin execution. Signed-off-by: Andrew Leech --- ports/unix/main.c | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/ports/unix/main.c b/ports/unix/main.c index 2ac8bed267914..9ce5a242c3385 100644 --- a/ports/unix/main.c +++ b/ports/unix/main.c @@ -110,8 +110,6 @@ static int handle_uncaught_exception(mp_obj_base_t *exc) { } #define LEX_SRC_STR (1) -#define LEX_SRC_VSTR (2) -#define LEX_SRC_FILENAME (3) #define LEX_SRC_STDIN (4) // Returns standard error codes: 0 for success, 1 for all other errors, @@ -127,12 +125,6 @@ static int execute_from_lexer(int source_kind, const void *source, mp_parse_inpu if (source_kind == LEX_SRC_STR) { const char *line = source; lex = mp_lexer_new_from_str_len(MP_QSTR__lt_stdin_gt_, line, strlen(line), false); - } else if (source_kind == LEX_SRC_VSTR) { - const vstr_t *vstr = source; - lex = mp_lexer_new_from_str_len(MP_QSTR__lt_stdin_gt_, vstr->buf, vstr->len, false); - } else if (source_kind == LEX_SRC_FILENAME) { - const char *filename = (const char *)source; - lex = mp_lexer_new_from_file(qstr_from_str(filename)); } else { // LEX_SRC_STDIN lex = mp_lexer_new_from_fd(MP_QSTR__lt_stdin_gt_, 0, false); } @@ -259,7 +251,18 @@ static int do_file(const char *file) { } static int do_str(const char *str) { - return execute_from_lexer(LEX_SRC_STR, str, MP_PARSE_FILE_INPUT, false); + vstr_t vstr; + vstr_init(&vstr, strlen(str)); + vstr_add_strn(&vstr, str, strlen(str)); + int ret = pyexec_vstr(&vstr, false); + vstr_clear(&vstr); + if (ret == 1) { + return 0; // success + } else if (ret & PYEXEC_FORCED_EXIT) { + return ret; // SystemExit with exit value in lower 8 bits + } else { + return 1; // exception + } } static void print_help(char **argv) { From b1917e3a231fd362b8c607503d7a65ac4b3f7a39 Mon Sep 17 00:00:00 2001 From: Andrew Leech Date: Tue, 22 Jul 2025 14:13:37 +1000 Subject: [PATCH 7/7] shared/runtime/pyexec: Add __file__ support for frozen modules. Set __file__ attribute for frozen boot.py/main.py modules during execution. Uses temporary storage to provide the module name to frozen MPY modules which bypass normal lexer-based __file__ setting. Signed-off-by: Andrew Leech --- shared/runtime/pyexec.c | 31 ++++++++++++++++++++----------- 1 file changed, 20 insertions(+), 11 deletions(-) diff --git a/shared/runtime/pyexec.c b/shared/runtime/pyexec.c index 9d8c77ef14c66..d0e101cddff8b 100644 --- a/shared/runtime/pyexec.c +++ b/shared/runtime/pyexec.c @@ -64,7 +64,7 @@ static bool repl_display_debugging_info = 0; // EXEC_FLAG_PRINT_EOF prints 2 EOF chars: 1 after normal output, 1 after exception output // EXEC_FLAG_ALLOW_DEBUGGING allows debugging info to be printed after executing the code // EXEC_FLAG_IS_REPL is used for REPL inputs (flag passed on to mp_compile) -static int parse_compile_execute(const void *source, mp_parse_input_kind_t input_kind, mp_uint_t exec_flags) { +static int parse_compile_execute(const void *source, mp_parse_input_kind_t input_kind, mp_uint_t exec_flags, const char *frozen_module_name) { int ret = 0; #if MICROPY_REPL_INFO uint32_t start = 0; @@ -86,6 +86,14 @@ static int parse_compile_execute(const void *source, mp_parse_input_kind_t input ctx->module.globals = mp_globals_get(); ctx->constants = frozen->constants; module_fun = mp_make_function_from_proto_fun(frozen->proto_fun, ctx, NULL); + + #if MICROPY_PY___FILE__ + // Set __file__ for frozen MPY modules + if (input_kind == MP_PARSE_FILE_INPUT && frozen_module_name != NULL) { + qstr source_name = qstr_from_str(frozen_module_name); + mp_store_global(MP_QSTR___file__, MP_OBJ_NEW_QSTR(source_name)); + } + #endif } else #endif { @@ -161,6 +169,7 @@ static int parse_compile_execute(const void *source, mp_parse_input_kind_t input // check for SystemExit if (mp_obj_is_subclass_fast(MP_OBJ_FROM_PTR(((mp_obj_base_t *)nlr.ret_val)->type), MP_OBJ_FROM_PTR(&mp_type_SystemExit))) { // Extract SystemExit value + // None is an exit value of 0; an int is its value; anything else is 1 mp_obj_t exit_val = mp_obj_exception_get_value(MP_OBJ_FROM_PTR(nlr.ret_val)); mp_int_t val = 0; if (exit_val != mp_const_none && !mp_obj_get_int_maybe(exit_val, &val)) { @@ -297,7 +306,7 @@ static int do_reader_stdin(int c) { mp_reader_stdin_t reader_stdin; mp_reader_new_stdin(&reader, &reader_stdin, MICROPY_REPL_STDIN_BUFFER_MAX); int exec_flags = EXEC_FLAG_PRINT_EOF | EXEC_FLAG_SOURCE_IS_READER; - return parse_compile_execute(&reader, MP_PARSE_FILE_INPUT, exec_flags); + return parse_compile_execute(&reader, MP_PARSE_FILE_INPUT, exec_flags, NULL); } #if MICROPY_REPL_EVENT_DRIVEN @@ -372,7 +381,7 @@ static int pyexec_raw_repl_process_char(int c) { return PYEXEC_FORCED_EXIT; } - int ret = parse_compile_execute(MP_STATE_VM(repl_line), MP_PARSE_FILE_INPUT, EXEC_FLAG_PRINT_EOF | EXEC_FLAG_SOURCE_IS_VSTR); + int ret = parse_compile_execute(MP_STATE_VM(repl_line), MP_PARSE_FILE_INPUT, EXEC_FLAG_PRINT_EOF | EXEC_FLAG_SOURCE_IS_VSTR, NULL); if (ret & PYEXEC_FORCED_EXIT) { return ret; } @@ -393,7 +402,7 @@ static int pyexec_friendly_repl_process_char(int c) { } else if (c == CHAR_CTRL_D) { // end of input mp_hal_stdout_tx_str("\r\n"); - int ret = parse_compile_execute(MP_STATE_VM(repl_line), MP_PARSE_FILE_INPUT, EXEC_FLAG_ALLOW_DEBUGGING | EXEC_FLAG_IS_REPL | EXEC_FLAG_SOURCE_IS_VSTR); + int ret = parse_compile_execute(MP_STATE_VM(repl_line), MP_PARSE_FILE_INPUT, EXEC_FLAG_ALLOW_DEBUGGING | EXEC_FLAG_IS_REPL | EXEC_FLAG_SOURCE_IS_VSTR, NULL); if (ret & PYEXEC_FORCED_EXIT) { return ret; } @@ -484,7 +493,7 @@ static int pyexec_friendly_repl_process_char(int c) { } exec:; - int ret = parse_compile_execute(MP_STATE_VM(repl_line), MP_PARSE_SINGLE_INPUT, EXEC_FLAG_ALLOW_DEBUGGING | EXEC_FLAG_IS_REPL | EXEC_FLAG_SOURCE_IS_VSTR); + int ret = parse_compile_execute(MP_STATE_VM(repl_line), MP_PARSE_SINGLE_INPUT, EXEC_FLAG_ALLOW_DEBUGGING | EXEC_FLAG_IS_REPL | EXEC_FLAG_SOURCE_IS_VSTR, NULL); if (ret & PYEXEC_FORCED_EXIT) { return ret; } @@ -567,7 +576,7 @@ int pyexec_raw_repl(void) { return PYEXEC_FORCED_EXIT; } - int ret = parse_compile_execute(&line, MP_PARSE_FILE_INPUT, EXEC_FLAG_PRINT_EOF | EXEC_FLAG_SOURCE_IS_VSTR); + int ret = parse_compile_execute(&line, MP_PARSE_FILE_INPUT, EXEC_FLAG_PRINT_EOF | EXEC_FLAG_SOURCE_IS_VSTR, NULL); if (ret & PYEXEC_FORCED_EXIT) { return ret; } @@ -692,7 +701,7 @@ int pyexec_friendly_repl(void) { } } - ret = parse_compile_execute(&line, parse_input_kind, EXEC_FLAG_ALLOW_DEBUGGING | EXEC_FLAG_IS_REPL | EXEC_FLAG_SOURCE_IS_VSTR); + ret = parse_compile_execute(&line, parse_input_kind, EXEC_FLAG_ALLOW_DEBUGGING | EXEC_FLAG_IS_REPL | EXEC_FLAG_SOURCE_IS_VSTR, NULL); if (ret & PYEXEC_FORCED_EXIT) { return ret; } @@ -703,7 +712,7 @@ int pyexec_friendly_repl(void) { #endif // MICROPY_ENABLE_COMPILER int pyexec_file(const char *filename) { - return parse_compile_execute(filename, MP_PARSE_FILE_INPUT, EXEC_FLAG_SOURCE_IS_FILENAME); + return parse_compile_execute(filename, MP_PARSE_FILE_INPUT, EXEC_FLAG_SOURCE_IS_FILENAME, NULL); } @@ -729,13 +738,13 @@ int pyexec_frozen_module(const char *name, bool allow_keyboard_interrupt) { switch (frozen_type) { #if MICROPY_MODULE_FROZEN_STR case MP_FROZEN_STR: - return parse_compile_execute(frozen_data, MP_PARSE_FILE_INPUT, exec_flags); + return parse_compile_execute(frozen_data, MP_PARSE_FILE_INPUT, exec_flags, NULL); #endif #if MICROPY_MODULE_FROZEN_MPY case MP_FROZEN_MPY: return parse_compile_execute(frozen_data, MP_PARSE_FILE_INPUT, exec_flags | - EXEC_FLAG_SOURCE_IS_RAW_CODE); + EXEC_FLAG_SOURCE_IS_RAW_CODE, name); #endif default: @@ -747,7 +756,7 @@ int pyexec_frozen_module(const char *name, bool allow_keyboard_interrupt) { int pyexec_vstr(vstr_t *str, bool allow_keyboard_interrupt) { mp_uint_t exec_flags = allow_keyboard_interrupt ? 0 : EXEC_FLAG_NO_INTERRUPT; - return parse_compile_execute(str, MP_PARSE_FILE_INPUT, exec_flags | EXEC_FLAG_SOURCE_IS_VSTR); + return parse_compile_execute(str, MP_PARSE_FILE_INPUT, exec_flags | EXEC_FLAG_SOURCE_IS_VSTR, NULL); } #if MICROPY_REPL_INFO pFad - Phonifier reborn

Pfad - The Proxy pFad of © 2024 Garber Painting. All rights reserved.

Note: This service is not intended for secure transactions such as banking, social media, email, or purchasing. Use at your own risk. We assume no liability whatsoever for broken pages.


Alternative Proxies:

Alternative Proxy

pFad Proxy

pFad v3 Proxy

pFad v4 Proxy