micropython · andrewleech · Oct 25, 2023 · May 26, 2025 · May 26, 2025 · May 26, 2025
diff --git a/.gitattributes b/.gitattributes
@@ -18,6 +18,7 @@
 # These should also not be modified by git.
 tests/basics/string_cr_conversion.py -text
 tests/basics/string_crlf_conversion.py -text
+tests/micropython/test_normalize_newlines.py.exp -text
 ports/stm32/pybcdc.inf_template -text
 ports/stm32/usbhost/** -text
 ports/cc3200/hal/aes.c -text

diff --git a/ports/unix/Makefile b/ports/unix/Makefile
@@ -217,6 +217,7 @@ SRC_C += \
 
 SHARED_SRC_C += $(addprefix shared/,\
 	runtime/gchelper_generic.c \
+	runtime/pyexec.c \
 	timeutils/timeutils.c \
 	$(SHARED_SRC_C_EXTRA) \
 	)

diff --git a/ports/unix/main.c b/ports/unix/main.c
@@ -54,9 +54,10 @@
 #include "extmod/vfs_posix.h"
 #include "genhdr/mpversion.h"
 #include "input.h"
+#include "shared/runtime/pyexec.h"
 
 // Command line options, with their defaults
-static bool compile_only = false;
+bool mp_compile_only = false;
 static uint emit_opt = MP_EMIT_OPT_NONE;
 
 #if MICROPY_ENABLE_GC
@@ -108,72 +109,6 @@ static int handle_uncaught_exception(mp_obj_base_t *exc) {
     return 1;
 }
 
-#define LEX_SRC_STR (1)
-#define LEX_SRC_VSTR (2)
-#define LEX_SRC_FILENAME (3)
-#define LEX_SRC_STDIN (4)
-
-// Returns standard error codes: 0 for success, 1 for all other errors,
-// except if FORCED_EXIT bit is set then script raised SystemExit and the
-// value of the exit is in the lower 8 bits of the return value
-static int execute_from_lexer(int source_kind, const void *source, mp_parse_input_kind_t input_kind, bool is_repl) {
-    mp_hal_set_interrupt_char(CHAR_CTRL_C);
-
-    nlr_buf_t nlr;
-    if (nlr_push(&nlr) == 0) {
-        // create lexer based on source kind
-        mp_lexer_t *lex;
-        if (source_kind == LEX_SRC_STR) {
-            const char *line = source;
-            lex = mp_lexer_new_from_str_len(MP_QSTR__lt_stdin_gt_, line, strlen(line), false);
-        } else if (source_kind == LEX_SRC_VSTR) {
-            const vstr_t *vstr = source;
-            lex = mp_lexer_new_from_str_len(MP_QSTR__lt_stdin_gt_, vstr->buf, vstr->len, false);
-        } else if (source_kind == LEX_SRC_FILENAME) {
-            const char *filename = (const char *)source;
-            lex = mp_lexer_new_from_file(qstr_from_str(filename));
-        } else { // LEX_SRC_STDIN
-            lex = mp_lexer_new_from_fd(MP_QSTR__lt_stdin_gt_, 0, false);
-        }
-
-        qstr source_name = lex->source_name;
-
-        #if MICROPY_PY___FILE__
-        if (input_kind == MP_PARSE_FILE_INPUT) {
-            mp_store_global(MP_QSTR___file__, MP_OBJ_NEW_QSTR(source_name));
-        }
-        #endif
-
-        mp_parse_tree_t parse_tree = mp_parse(lex, input_kind);
-
-        #if defined(MICROPY_UNIX_COVERAGE)
-        // allow to print the parse tree in the coverage build
-        if (mp_verbose_flag >= 3) {
-            printf("----------------\n");
-            mp_parse_node_print(&mp_plat_print, parse_tree.root, 0);
-            printf("----------------\n");
-        }
-        #endif
-
-        mp_obj_t module_fun = mp_compile(&parse_tree, source_name, is_repl);
-
-        if (!compile_only) {
-            // execute it
-            mp_call_function_0(module_fun);
-        }
-
-        mp_hal_set_interrupt_char(-1);
-        mp_handle_pending(true);
-        nlr_pop();
-        return 0;
-
-    } else {
-        // uncaught exception
-        mp_hal_set_interrupt_char(-1);
-        mp_handle_pending(false);
-        return handle_uncaught_exception(nlr.ret_val);
-    }
-}
 
 #if MICROPY_USE_READLINE == 1
 #include "shared/readline/readline.h"
@@ -194,91 +129,27 @@ static char *strjoin(const char *s1, int sep_char, const char *s2) {
 #endif
 
 static int do_repl(void) {
-    mp_hal_stdout_tx_str(MICROPY_BANNER_NAME_AND_VERSION);
-    mp_hal_stdout_tx_str("; " MICROPY_BANNER_MACHINE);
-    mp_hal_stdout_tx_str("\nUse Ctrl-D to exit, Ctrl-E for paste mode\n");
-
+    int ret = 0;
     #if MICROPY_USE_READLINE == 1
-
-    // use MicroPython supplied readline
-
-    vstr_t line;
-    vstr_init(&line, 16);
+    // use MicroPython supplied readline based repl
+    mp_hal_stdio_mode_raw();
     for (;;) {
-        mp_hal_stdio_mode_raw();
-
-    input_restart:
-        vstr_reset(&line);
-        int ret = readline(&line, mp_repl_get_ps1());
-        mp_parse_input_kind_t parse_input_kind = MP_PARSE_SINGLE_INPUT;
-
-        if (ret == CHAR_CTRL_C) {
-            // cancel input
-            mp_hal_stdout_tx_str("\r\n");
-            goto input_restart;
-        } else if (ret == CHAR_CTRL_D) {
-            // EOF
-            printf("\n");
-            mp_hal_stdio_mode_orig();
-            vstr_clear(&line);
-            return 0;
-        } else if (ret == CHAR_CTRL_E) {
-            // paste mode
-            mp_hal_stdout_tx_str("\npaste mode; Ctrl-C to cancel, Ctrl-D to finish\n=== ");
-            vstr_reset(&line);
-            for (;;) {
-                char c = mp_hal_stdin_rx_chr();
-                if (c == CHAR_CTRL_C) {
-                    // cancel everything
-                    mp_hal_stdout_tx_str("\n");
-                    goto input_restart;
-                } else if (c == CHAR_CTRL_D) {
-                    // end of input
-                    mp_hal_stdout_tx_str("\n");
-                    break;
-                } else {
-                    // add char to buffer and echo
-                    vstr_add_byte(&line, c);
-                    if (c == '\r') {
-                        mp_hal_stdout_tx_str("\n=== ");
-                    } else {
-                        mp_hal_stdout_tx_strn(&c, 1);
-                    }
-                }
-            }
-            parse_input_kind = MP_PARSE_FILE_INPUT;
-        } else if (line.len == 0) {
-            if (ret != 0) {
-                printf("\n");
+        if (pyexec_mode_kind == PYEXEC_MODE_RAW_REPL) {
+            if ((ret = pyexec_raw_repl()) != 0) {
+                break;
             }
-            goto input_restart;
         } else {
-            // got a line with non-zero length, see if it needs continuing
-            while (mp_repl_continue_with_input(vstr_null_terminated_str(&line))) {
-                vstr_add_byte(&line, '\n');
-                ret = readline(&line, mp_repl_get_ps2());
-                if (ret == CHAR_CTRL_C) {
-                    // cancel everything
-                    printf("\n");
-                    goto input_restart;
-                } else if (ret == CHAR_CTRL_D) {
-                    // stop entering compound statement
-                    break;
-                }
+            if ((ret = pyexec_friendly_repl()) != 0) {
+                break;
             }
         }
-
-        mp_hal_stdio_mode_orig();
-
-        ret = execute_from_lexer(LEX_SRC_VSTR, &line, parse_input_kind, true);
-        if (ret & FORCED_EXIT) {
-            return ret;
-        }
     }
-
+    mp_hal_stdio_mode_orig();
     #else
-
     // use simple readline
+    mp_hal_stdout_tx_str(MICROPY_BANNER_NAME_AND_VERSION);
+    mp_hal_stdout_tx_str("; " MICROPY_BANNER_MACHINE);
+    mp_hal_stdout_tx_str("\nUse Ctrl-D to exit, Ctrl-E for paste mode\n");
 
     for (;;) {
         char *line = prompt((char *)mp_repl_get_ps1());
@@ -297,22 +168,40 @@ static int do_repl(void) {
             line = line3;
         }
 
-        int ret = execute_from_lexer(LEX_SRC_STR, line, MP_PARSE_SINGLE_INPUT, true);
+        ret = convert_pyexec_result(pyexec_str_single(line, true));
         free(line);
         if (ret & FORCED_EXIT) {
             return ret;
         }
     }
-
     #endif
+    return ret;
+}
+
+
+// Convert pyexec return codes to unix port's expected codes
+// pyexec returns 1 for success, 0 for exception, PYEXEC_FORCED_EXIT for SystemExit
+// Convert to unix port's expected codes: 0 for success, 1 for exception, FORCED_EXIT|val for SystemExit
+static int convert_pyexec_result(int ret) {
+    if (ret == 1) {
+        return 0; // success
+    } else if (ret & PYEXEC_FORCED_EXIT) {
+        return ret; // SystemExit with exit value in lower 8 bits
+    } else {
+        return 1; // exception
+    }
 }
 
 static int do_file(const char *file) {
-    return execute_from_lexer(LEX_SRC_FILENAME, file, MP_PARSE_FILE_INPUT, false);
+    return convert_pyexec_result(pyexec_file(file));
 }
 
 static int do_str(const char *str) {
-    return execute_from_lexer(LEX_SRC_STR, str, MP_PARSE_FILE_INPUT, false);
+    vstr_t vstr;
+    size_t len = strlen(str);
+    vstr_init_fixed_buf(&vstr, len, (char *)str);
+    vstr.len = len;
+    return convert_pyexec_result(pyexec_vstr(&vstr, false));
 }
 
 static void print_help(char **argv) {
@@ -381,7 +270,7 @@ static void pre_process_options(int argc, char **argv) {
                 }
                 if (0) {
                 } else if (strcmp(argv[a + 1], "compile-only") == 0) {
-                    compile_only = true;
+                    mp_compile_only = true;
                 } else if (strcmp(argv[a + 1], "emit=bytecode") == 0) {
                     emit_opt = MP_EMIT_OPT_BYTECODE;
                 #if MICROPY_EMIT_NATIVE
@@ -753,7 +642,7 @@ MP_NOINLINE int main_(int argc, char **argv) {
             ret = do_repl();
             prompt_write_history();
         } else {
-            ret = execute_from_lexer(LEX_SRC_STDIN, NULL, MP_PARSE_FILE_INPUT, false);
+            ret = convert_pyexec_result(pyexec_stdin());
         }
     }
 

diff --git a/ports/unix/mpconfigport.h b/ports/unix/mpconfigport.h
@@ -166,6 +166,12 @@ typedef long mp_off_t;
 // Enable sys.executable.
 #define MICROPY_PY_SYS_EXECUTABLE (1)
 
+// Enable support for compile-only mode.
+#define MICROPY_PYEXEC_COMPILE_ONLY (1)
+
+// Enable POSIX-specific pyexec functions.
+#define MICROPY_PYEXEC_POSIX_FUNCTIONS (1)
+
 #define MICROPY_PY_SOCKET_LISTEN_BACKLOG_DEFAULT (SOMAXCONN < 128 ? SOMAXCONN : 128)
 
 // Bare-metal ports don't have stderr. Printing debug to stderr may give tests

diff --git a/ports/unix/mphalport.h b/ports/unix/mphalport.h
@@ -25,6 +25,7 @@
  */
 #include <errno.h>
 #include <unistd.h>
+#include <stdbool.h>
 
 #ifndef CHAR_CTRL_C
 #define CHAR_CTRL_C (3)
@@ -117,3 +118,6 @@ enum {
 
 void mp_hal_get_mac(int idx, uint8_t buf[6]);
 #endif
+
+// Global variable to control compile-only mode.
+extern bool mp_compile_only;
diff --git a/ports/windows/Makefile b/ports/windows/Makefile
@@ -83,6 +83,7 @@ OBJ += $(addprefix $(BUILD)/, $(LIB_SRC_C:.c=.o))
 ifeq ($(MICROPY_USE_READLINE),1)
 CFLAGS += -DMICROPY_USE_READLINE=1
 SRC_C += shared/readline/readline.c
+SRC_C += shared/runtime/pyexec.c
 endif
 
 LIB += -lws2_32

diff --git a/ports/windows/micropython.vcxproj b/ports/windows/micropython.vcxproj
@@ -89,6 +89,7 @@
     <ClCompile Include="@(PyExtModSource)" />
     <ClCompile Include="$(PyBaseDir)shared\readline\*.c" />
     <ClCompile Include="$(PyBaseDir)shared\runtime\gchelper_generic.c" />
+    <ClCompile Include="$(PyBaseDir)shared\runtime\pyexec.c" />
     <ClCompile Include="$(PyBaseDir)ports\windows\*.c" />
     <ClCompile Include="$(PyBaseDir)ports\windows\msvc\*.c" />
     <ClCompile Include="$(PyBaseDir)ports\unix\gccollect.c"/>

diff --git a/ports/windows/mpconfigport.h b/ports/windows/mpconfigport.h
@@ -161,6 +161,12 @@
 #define MICROPY_MACHINE_MEM_GET_READ_ADDR   mod_machine_mem_get_addr
 #define MICROPY_MACHINE_MEM_GET_WRITE_ADDR  mod_machine_mem_get_addr
 
+// Enable support for compile-only mode.
+#define MICROPY_PYEXEC_COMPILE_ONLY (1)
+
+// Enable POSIX-specific pyexec functions.
+#define MICROPY_PYEXEC_POSIX_FUNCTIONS (1)
+
 #define MICROPY_ERROR_REPORTING     (MICROPY_ERROR_REPORTING_DETAILED)
 #define MICROPY_ERROR_PRINTER       (&mp_stderr_print)
 #define MICROPY_WARNINGS            (1)

diff --git a/pyproject.toml b/pyproject.toml
@@ -28,6 +28,7 @@ target-version = "py37"
 
 [tool.ruff.lint]
 exclude = [  # Ruff finds Python SyntaxError in these files
+  "tests/cmdline/cmd_compile_only_error.py",
   "tests/cmdline/repl_autocomplete.py",
   "tests/cmdline/repl_autocomplete_underscore.py",
   "tests/cmdline/repl_autoindent.py",
@@ -67,4 +68,10 @@ mccabe.max-complexity = 40
 #   basics: needs careful attention before applying automatic formatting
 #   repl_: not real python files
 #   viper_args: uses f(*)
-exclude = ["tests/basics/*.py", "tests/*/repl_*.py", "tests/micropython/viper_args.py"]
+exclude = [
+  "tests/basics/*.py", 
+  "tests/*/repl_*.py", 
+  "tests/cmdline/cmd_compile_only_error.py",
+  "tests/micropython/test_normalize_newlines.py",
+  "tests/micropython/viper_args.py",
+]