From 86392384894cf65a4130c28145478e7c9eb7a343 Mon Sep 17 00:00:00 2001 From: Angus Gratton Date: Thu, 9 Jun 2022 13:08:50 +1000 Subject: [PATCH 1/4] mpremote: Support bytecode raw paste for 'mpremote run module.mpy'. - Adds a new raw paste command 'B' for 'raw paste bytecode' ('A' is 'paste source') - Adds an escaping mechanism (Ctrl-F ) during raw paste so bytes less than 8 can be sent without triggering Ctrl-C or Ctrl-D handlers. The two-byte escape sequence still counts as one byte in the paste window. - Adds relevant support to mpremote.py Signed-off-by: Angus Gratton --- shared/runtime/pyexec.c | 43 +++++++++++++++++++++++++++------ tools/mpremote/mpremote/main.py | 14 +++++++---- tools/pyboard.py | 24 +++++++++++++++--- 3 files changed, 64 insertions(+), 17 deletions(-) diff --git a/shared/runtime/pyexec.c b/shared/runtime/pyexec.c index 2763319c01b75..9b0e7b2759f49 100644 --- a/shared/runtime/pyexec.c +++ b/shared/runtime/pyexec.c @@ -35,6 +35,7 @@ #include "py/gc.h" #include "py/frozenmod.h" #include "py/mphal.h" +#include "py/persistentcode.h" #if MICROPY_HW_ENABLE_USB #include "irq.h" #include "usb.h" @@ -58,6 +59,8 @@ STATIC bool repl_display_debugging_info = 0; #define EXEC_FLAG_SOURCE_IS_FILENAME (1 << 5) #define EXEC_FLAG_SOURCE_IS_READER (1 << 6) +#define NUM_ESCAPED 8 // This value has to match the value in tools/pyboard.py + // parses, compiles and executes the code in the lexer // frees the lexer before returning // EXEC_FLAG_PRINT_EOF prints 2 EOF chars: 1 after normal output, 1 after exception output @@ -80,16 +83,26 @@ STATIC int parse_compile_execute(const void *source, mp_parse_input_kind_t input nlr.ret_val = NULL; if (nlr_push(&nlr) == 0) { mp_obj_t module_fun; - #if MICROPY_MODULE_FROZEN_MPY + #if MICROPY_MODULE_FROZEN_MPY || MICROPY_PERSISTENT_CODE_LOAD if (exec_flags & EXEC_FLAG_SOURCE_IS_RAW_CODE) { - // source is a raw_code object, create the function - const mp_frozen_module_t *frozen = source; mp_module_context_t *ctx = m_new_obj(mp_module_context_t); ctx->module.globals = mp_globals_get(); - ctx->constants = frozen->constants; - module_fun = mp_make_function_from_raw_code(frozen->rc, ctx, NULL); + + #if MICROPY_PERSISTENT_CODE_LOAD + if (exec_flags & EXEC_FLAG_SOURCE_IS_READER) { + // source is a reader that will give us raw code (mpy file equivalent) + mp_compiled_module_t cm = mp_raw_code_load((mp_reader_t *)source, ctx); + module_fun = mp_make_function_from_raw_code(cm.rc, ctx, NULL); + } else + #endif // MICROPY_PERSISTENT_CODE_LOAD + { + // source is a raw_code object, create the module function from it + const mp_frozen_module_t *frozen = source; + ctx->constants = frozen->constants; + module_fun = mp_make_function_from_raw_code(frozen->rc, ctx, NULL); + } } else - #endif + #endif // MICROPY_MODULE_FROZEN_PY || MICROPY_PERSISTENT_CODE_LOAD { #if MICROPY_ENABLE_COMPILER mp_lexer_t *lex; @@ -109,7 +122,7 @@ STATIC int parse_compile_execute(const void *source, mp_parse_input_kind_t input module_fun = mp_compile(&parse_tree, source_name, exec_flags & EXEC_FLAG_IS_REPL); #else mp_raise_msg(&mp_type_RuntimeError, MP_ERROR_TEXT("script compilation not supported")); - #endif + #endif // MICROPY_ENABLE_COMPILER } // execute code @@ -220,6 +233,10 @@ STATIC mp_uint_t mp_reader_stdin_readbyte(void *data) { } else { return MP_READER_EOF; } + } else if (c == CHAR_CTRL_F) { + // escape sequence, next character is escaped by adding NUM_ESCAPED to it + int e = mp_hal_stdin_rx_chr(); + c = e - NUM_ESCAPED; } if (--reader->window_remain == 0) { @@ -261,7 +278,12 @@ STATIC void mp_reader_new_stdin(mp_reader_t *reader, mp_reader_stdin_t *reader_s } STATIC int do_reader_stdin(int c) { - if (c != 'A') { + bool supported_command = c == 'A'; + #if MICROPY_PERSISTENT_CODE_LOAD + supported_command = (c == 'B') || supported_command; + #endif + + if (!supported_command) { // Unsupported command. mp_hal_stdout_tx_strn("R\x00", 2); return 0; @@ -270,10 +292,15 @@ STATIC int do_reader_stdin(int c) { // Indicate reception of command. mp_hal_stdout_tx_strn("R\x01", 2); + // Entering raw paste mode + // c == 'A' input is source, c == 'B' input is bytecode mp_reader_t reader; mp_reader_stdin_t reader_stdin; mp_reader_new_stdin(&reader, &reader_stdin, MICROPY_REPL_STDIN_BUFFER_MAX); int exec_flags = EXEC_FLAG_PRINT_EOF | EXEC_FLAG_SOURCE_IS_READER; + if (c == 'B') { + exec_flags |= EXEC_FLAG_SOURCE_IS_RAW_CODE; + } return parse_compile_execute(&reader, MP_PARSE_FILE_INPUT, exec_flags); } diff --git a/tools/mpremote/mpremote/main.py b/tools/mpremote/mpremote/main.py index 4d74743aa349a..e450c5db97dc5 100644 --- a/tools/mpremote/mpremote/main.py +++ b/tools/mpremote/mpremote/main.py @@ -352,8 +352,10 @@ def do_repl_main_loop(pyb, console_in, console_out_write, *, code_to_inject, fil pyb.enter_raw_repl(soft_reset=False) with open(file_to_inject, "rb") as f: pyfile = f.read() + + is_bytecode = pyfile[0] == ord("M") and file_to_inject.endswith(".mpy") try: - pyb.exec_raw_no_follow(pyfile) + pyb.exec_raw_no_follow(pyfile, is_bytecode) except pyboard.PyboardError as er: console_out_write(b"Error:\r\n") console_out_write(er) @@ -430,10 +432,10 @@ def console_out_write(b): capture_file.close() -def execbuffer(pyb, buf, follow): +def execbuffer(pyb, buf, follow, is_bytecode=False): ret_val = 0 try: - pyb.exec_raw_no_follow(buf) + pyb.exec_raw_no_follow(buf, is_bytecode) if follow: ret, ret_err = pyb.follow(timeout=None, data_consumer=pyboard.stdout_write_bytes) if ret_err: @@ -546,6 +548,7 @@ def main(): elif cmd == "umount": pyb.umount_local() elif cmd in ("exec", "eval", "run"): + is_bytecode = False follow = True if args[0] == "--no-follow": args.pop(0) @@ -554,15 +557,16 @@ def main(): buf = args.pop(0) elif cmd == "eval": buf = "print(" + args.pop(0) + ")" - else: + else: # run filename = args.pop(0) try: with open(filename, "rb") as f: buf = f.read() + is_bytecode = buf[0] == ord("M") and filename.endswith(".mpy") except OSError: print(f"{_PROG}: could not read file '{filename}'") return 1 - ret = execbuffer(pyb, buf, follow) + ret = execbuffer(pyb, buf, follow, is_bytecode) if ret: return ret elif cmd == "fs": diff --git a/tools/pyboard.py b/tools/pyboard.py index a9a4cbff87155..a4408443180be 100755 --- a/tools/pyboard.py +++ b/tools/pyboard.py @@ -70,6 +70,7 @@ import sys import time import os +import re import ast try: @@ -399,10 +400,22 @@ def raw_paste_write(self, command_bytes): raise PyboardError("unexpected read during raw paste: {}".format(data)) # Send out as much data as possible that fits within the allowed window. b = command_bytes[i : min(i + window_remain, len(command_bytes))] - self.serial.write(b) + window_remain -= len(b) i += len(b) + # escape any characters that need to be escaped. Note this doesn't + # count towards the window size, as unescaping happens before filling + # the window buffer in the device + NUM_ESCAPED = 8 # this values has to match value in pyexec.c + b = re.sub( + rb"[" + bytes(range(NUM_ESCAPED)) + rb"]", + lambda c: bytes((0x06, c.group()[0] + NUM_ESCAPED)), + b, + ) + + self.serial.write(b) + # Indicate end of data. self.serial.write(b"\x04") @@ -411,7 +424,7 @@ def raw_paste_write(self, command_bytes): if not data.endswith(b"\x04"): raise PyboardError("could not complete raw paste: {}".format(data)) - def exec_raw_no_follow(self, command): + def exec_raw_no_follow(self, command, is_bytecode=False): if isinstance(command, bytes): command_bytes = command else: @@ -424,7 +437,8 @@ def exec_raw_no_follow(self, command): if self.use_raw_paste: # Try to enter raw-paste mode. - self.serial.write(b"\x05A\x01") + raw_paste_cmd = b"\x05A\x01" if not is_bytecode else b"\x05B\x01" + self.serial.write(raw_paste_cmd) data = self.serial.read(2) if data == b"R\x00": # Device understood raw-paste command but doesn't support it. @@ -432,12 +446,14 @@ def exec_raw_no_follow(self, command): elif data == b"R\x01": # Device supports raw-paste mode, write out the command using this mode. return self.raw_paste_write(command_bytes) - else: + elif not is_bytecode: # Device doesn't support raw-paste, fall back to normal raw REPL. data = self.read_until(1, b"w REPL; CTRL-B to exit\r\n>") if not data.endswith(b"w REPL; CTRL-B to exit\r\n>"): print(data) raise PyboardError("could not enter raw repl") + else: + raise NotImplementedError() # sending bytecode currently requires raw paste # Don't try to use raw-paste mode again for this connection. self.use_raw_paste = False From 95382304a430f61c6e470b02f8e973bf243eaf9f Mon Sep 17 00:00:00 2001 From: Angus Gratton Date: Thu, 16 Jun 2022 15:42:04 +1000 Subject: [PATCH 2/4] mpremote: Allow 'run .mpy' for devices which don't support raw paste. Signed-off-by: Angus Gratton --- tools/pyboard.py | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/tools/pyboard.py b/tools/pyboard.py index a4408443180be..5f70551e1f335 100755 --- a/tools/pyboard.py +++ b/tools/pyboard.py @@ -446,16 +446,24 @@ def exec_raw_no_follow(self, command, is_bytecode=False): elif data == b"R\x01": # Device supports raw-paste mode, write out the command using this mode. return self.raw_paste_write(command_bytes) - elif not is_bytecode: + else: # Device doesn't support raw-paste, fall back to normal raw REPL. data = self.read_until(1, b"w REPL; CTRL-B to exit\r\n>") if not data.endswith(b"w REPL; CTRL-B to exit\r\n>"): print(data) raise PyboardError("could not enter raw repl") - else: - raise NotImplementedError() # sending bytecode currently requires raw paste - # Don't try to use raw-paste mode again for this connection. - self.use_raw_paste = False + + if is_bytecode: + # if we can't raw paste bytecode then use the injected import hook to load it instead + command_bytes = "_injected_buf={!r}\n{}\n".format( + command_bytes, _injected_import_hook_code + ) + if self.use_raw_paste and data == b"R\x00": + # the device did understand raw-paste, so try again as a plaintext raw paste + return self.exec_raw_no_follow(command_bytes, is_bytecode=False) + + # Don't try to use raw-paste mode again for this connection. + self.use_raw_paste = False # Write command using standard raw REPL, 256 bytes every 10ms. for i in range(0, len(command_bytes), 256): From 54e5d290ccbc96fcf43114f347c0ee92cd4bc383 Mon Sep 17 00:00:00 2001 From: Angus Gratton Date: Thu, 16 Jun 2022 15:42:36 +1000 Subject: [PATCH 3/4] pyboard: Clean up if import fails when injecting mpy in a variable. Signed-off-by: Angus Gratton --- tools/pyboard.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/tools/pyboard.py b/tools/pyboard.py index 5f70551e1f335..db7ae7b54d7aa 100755 --- a/tools/pyboard.py +++ b/tools/pyboard.py @@ -659,9 +659,11 @@ def open(self, path, mode): return self.File() uos.mount(_FS(), '/_') uos.chdir('/_') -from _injected import * -uos.umount('/_') -del _injected_buf, _FS +try: + from _injected import * +finally: + uos.umount('/_') + del _injected_buf, _FS """ From 02826f098b3997c683bc3a5b09f509cc1cd73410 Mon Sep 17 00:00:00 2001 From: Angus Gratton Date: Mon, 20 Jun 2022 12:34:02 +1000 Subject: [PATCH 4/4] pyexec: Add named constants for raw repl control characters. The raw repl language is gradually getting more complex. To structure the code a bit more, introduce some names for the different control characters and the "init command" sequence which is currently only used for starting a paste. Signed-off-by: Angus Gratton --- shared/runtime/pyexec.c | 73 ++++++++++++++++++++++++++--------------- tools/pyboard.py | 6 ++-- 2 files changed, 49 insertions(+), 30 deletions(-) diff --git a/shared/runtime/pyexec.c b/shared/runtime/pyexec.c index 9b0e7b2759f49..6caa49af3d3bc 100644 --- a/shared/runtime/pyexec.c +++ b/shared/runtime/pyexec.c @@ -59,7 +59,23 @@ STATIC bool repl_display_debugging_info = 0; #define EXEC_FLAG_SOURCE_IS_FILENAME (1 << 5) #define EXEC_FLAG_SOURCE_IS_READER (1 << 6) -#define NUM_ESCAPED 8 // This value has to match the value in tools/pyboard.py +#define RAWCODE_PASTE_NUM_ESCAPED 8 // This value has to match the same constant in tools/pyboard.py + +// Raw REPL serial protocol control sequences +#define RAW_REPL_CTRL_INIT CHAR_CTRL_A +#define RAW_REPL_CTRL_EXIT_TO_FRIENDLY CHAR_CTRL_B +#define RAW_REPL_CTRL_CLEAR_LINE CHAR_CTRL_C +#define RAW_REPL_CTRL_EOF CHAR_CTRL_D +#define RAW_REPL_CTRL_INIT_CMD CHAR_CTRL_E +// CHAR_CTRL_F is recognised in raw paste mode (as an escape sequence), but not in raw REPL mode + +// Sequence ^A ^E (RAW_REPL_CTRL_INIT, RAW_REPL_CTRL_INIT_CMD) can initiate one or more "init commands" based on the next +// character in the sequence: +#define RAW_REPL_INIT_CMD_PASTE_SOURCE 'A' +#define RAW_REPL_INIT_CMD_PASTE_RAWCODE 'B' + +#define RAW_REPL_INIT_CMD_RESP_UNSUPPORTED "R\x00" +#define RAW_REPL_INIT_CMD_RESP_OK "R\x01" // parses, compiles and executes the code in the lexer // frees the lexer before returning @@ -234,9 +250,9 @@ STATIC mp_uint_t mp_reader_stdin_readbyte(void *data) { return MP_READER_EOF; } } else if (c == CHAR_CTRL_F) { - // escape sequence, next character is escaped by adding NUM_ESCAPED to it + // escape sequence, next character is escaped by adding RAWCODE_PASTE_NUM_ESCAPED to it int e = mp_hal_stdin_rx_chr(); - c = e - NUM_ESCAPED; + c = e - RAWCODE_PASTE_NUM_ESCAPED; } if (--reader->window_remain == 0) { @@ -277,30 +293,33 @@ STATIC void mp_reader_new_stdin(mp_reader_t *reader, mp_reader_stdin_t *reader_s reader->close = mp_reader_stdin_close; } -STATIC int do_reader_stdin(int c) { - bool supported_command = c == 'A'; +STATIC int handle_raw_repl_init_cmd(int c) { + int exec_flags = EXEC_FLAG_PRINT_EOF | EXEC_FLAG_SOURCE_IS_READER; + bool supported_command = false; + + if (c == RAW_REPL_INIT_CMD_PASTE_SOURCE) { + supported_command = true; + } #if MICROPY_PERSISTENT_CODE_LOAD - supported_command = (c == 'B') || supported_command; + if (c == RAW_REPL_INIT_CMD_PASTE_RAWCODE) { + exec_flags |= EXEC_FLAG_SOURCE_IS_RAW_CODE; + supported_command = true; + } #endif if (!supported_command) { // Unsupported command. - mp_hal_stdout_tx_strn("R\x00", 2); + mp_hal_stdout_tx_strn(RAW_REPL_INIT_CMD_RESP_UNSUPPORTED, 2); return 0; } // Indicate reception of command. - mp_hal_stdout_tx_strn("R\x01", 2); + mp_hal_stdout_tx_strn(RAW_REPL_INIT_CMD_RESP_OK, 2); // Entering raw paste mode - // c == 'A' input is source, c == 'B' input is bytecode mp_reader_t reader; mp_reader_stdin_t reader_stdin; mp_reader_new_stdin(&reader, &reader_stdin, MICROPY_REPL_STDIN_BUFFER_MAX); - int exec_flags = EXEC_FLAG_PRINT_EOF | EXEC_FLAG_SOURCE_IS_READER; - if (c == 'B') { - exec_flags |= EXEC_FLAG_SOURCE_IS_RAW_CODE; - } return parse_compile_execute(&reader, MP_PARSE_FILE_INPUT, exec_flags); } @@ -335,10 +354,10 @@ void pyexec_event_repl_init(void) { } STATIC int pyexec_raw_repl_process_char(int c) { - if (c == CHAR_CTRL_A) { + if (c == RAW_REPL_CTRL_INIT) { // reset raw REPL - if (vstr_len(MP_STATE_VM(repl_line)) == 2 && vstr_str(MP_STATE_VM(repl_line))[0] == CHAR_CTRL_E) { - int ret = do_reader_stdin(vstr_str(MP_STATE_VM(repl_line))[1]); + if (vstr_len(MP_STATE_VM(repl_line)) == 2 && vstr_str(MP_STATE_VM(repl_line))[0] == RAW_REPL_CTRL_INIT_CMD) { + int ret = handle_raw_repl_init_cmd(vstr_str(MP_STATE_VM(repl_line))[1]); if (ret & PYEXEC_FORCED_EXIT) { return ret; } @@ -346,7 +365,7 @@ STATIC int pyexec_raw_repl_process_char(int c) { } mp_hal_stdout_tx_str("raw REPL; CTRL-B to exit\r\n"); goto reset; - } else if (c == CHAR_CTRL_B) { + } else if (c == RAW_REPL_CTRL_EXIT_TO_FRIENDLY) { // change to friendly REPL pyexec_mode_kind = PYEXEC_MODE_FRIENDLY_REPL; vstr_reset(MP_STATE_VM(repl_line)); @@ -354,11 +373,11 @@ STATIC int pyexec_raw_repl_process_char(int c) { repl.paste_mode = false; pyexec_friendly_repl_process_char(CHAR_CTRL_B); return 0; - } else if (c == CHAR_CTRL_C) { + } else if (c == RAW_REPL_CTRL_CLEAR_LINE) { // clear line vstr_reset(MP_STATE_VM(repl_line)); return 0; - } else if (c == CHAR_CTRL_D) { + } else if (c == RAW_REPL_CTRL_EOF) { // input finished } else { // let through any other raw 8-bit value @@ -419,7 +438,7 @@ STATIC int pyexec_friendly_repl_process_char(int c) { if (!repl.cont_line) { - if (ret == CHAR_CTRL_A) { + if (ret == RAW_REPL_CTRL_INIT) { // change to raw REPL pyexec_mode_kind = PYEXEC_MODE_RAW_REPL; mp_hal_stdout_tx_str("\r\n"); @@ -529,10 +548,10 @@ int pyexec_raw_repl(void) { mp_hal_stdout_tx_str(">"); for (;;) { int c = mp_hal_stdin_rx_chr(); - if (c == CHAR_CTRL_A) { + if (c == RAW_REPL_CTRL_INIT) { // reset raw REPL - if (vstr_len(&line) == 2 && vstr_str(&line)[0] == CHAR_CTRL_E) { - int ret = do_reader_stdin(vstr_str(&line)[1]); + if (vstr_len(&line) == 2 && vstr_str(&line)[0] == RAW_REPL_CTRL_INIT_CMD) { + int ret = handle_raw_repl_init_cmd(vstr_str(&line)[1]); if (ret & PYEXEC_FORCED_EXIT) { return ret; } @@ -541,16 +560,16 @@ int pyexec_raw_repl(void) { continue; } goto raw_repl_reset; - } else if (c == CHAR_CTRL_B) { + } else if (c == RAW_REPL_CTRL_EXIT_TO_FRIENDLY) { // change to friendly REPL mp_hal_stdout_tx_str("\r\n"); vstr_clear(&line); pyexec_mode_kind = PYEXEC_MODE_FRIENDLY_REPL; return 0; - } else if (c == CHAR_CTRL_C) { + } else if (c == RAW_REPL_CTRL_CLEAR_LINE) { // clear line vstr_reset(&line); - } else if (c == CHAR_CTRL_D) { + } else if (c == RAW_REPL_CTRL_EOF) { // input finished break; } else { @@ -632,7 +651,7 @@ int pyexec_friendly_repl(void) { int ret = readline(&line, mp_repl_get_ps1()); mp_parse_input_kind_t parse_input_kind = MP_PARSE_SINGLE_INPUT; - if (ret == CHAR_CTRL_A) { + if (ret == RAW_REPL_CTRL_INIT) { // change to raw REPL mp_hal_stdout_tx_str("\r\n"); vstr_clear(&line); diff --git a/tools/pyboard.py b/tools/pyboard.py index db7ae7b54d7aa..91b98c2326d9d 100755 --- a/tools/pyboard.py +++ b/tools/pyboard.py @@ -407,10 +407,10 @@ def raw_paste_write(self, command_bytes): # escape any characters that need to be escaped. Note this doesn't # count towards the window size, as unescaping happens before filling # the window buffer in the device - NUM_ESCAPED = 8 # this values has to match value in pyexec.c + RAWCODE_PASTE_NUM_ESCAPED = 8 # value has to match the same constant in pyexec.c b = re.sub( - rb"[" + bytes(range(NUM_ESCAPED)) + rb"]", - lambda c: bytes((0x06, c.group()[0] + NUM_ESCAPED)), + rb"[" + bytes(range(RAWCODE_PASTE_NUM_ESCAPED)) + rb"]", + lambda c: bytes((0x06, c.group()[0] + RAWCODE_PASTE_NUM_ESCAPED)), b, ) pFad - Phonifier reborn

Pfad - The Proxy pFad of © 2024 Garber Painting. All rights reserved.

Note: This service is not intended for secure transactions such as banking, social media, email, or purchasing. Use at your own risk. We assume no liability whatsoever for broken pages.


Alternative Proxies:

Alternative Proxy

pFad Proxy

pFad v3 Proxy

pFad v4 Proxy