From 2b8e88c563e4bab53e3183d15f04795d645eea28 Mon Sep 17 00:00:00 2001 From: Damien George Date: Sat, 24 Jun 2023 17:02:58 +1000 Subject: [PATCH 01/19] py/compile: Add option to allow compiling top-level await. Enabled by MICROPY_COMPILE_ALLOW_TOP_LEVEL_AWAIT. When enabled, this means that scope such as module-level functions and REPL statements can yield. The outer C code must then handle this yielded generator. Signed-off-by: Damien George --- py/compile.c | 13 +++++++++++-- py/compile.h | 5 +++++ py/mpconfig.h | 5 +++++ 3 files changed, 21 insertions(+), 2 deletions(-) diff --git a/py/compile.c b/py/compile.c index 7a359e662e731..62757de3c083c 100644 --- a/py/compile.c +++ b/py/compile.c @@ -196,6 +196,10 @@ typedef struct _compiler_t { mp_emit_common_t emit_common; } compiler_t; +#if MICROPY_COMP_ALLOW_TOP_LEVEL_AWAIT +bool mp_compile_allow_top_level_await = false; +#endif + /******************************************************************************/ // mp_emit_common_t helper functions // These are defined here so they can be inlined, to reduce code size. @@ -2759,8 +2763,13 @@ static void compile_yield_expr(compiler_t *comp, mp_parse_node_struct_t *pns) { #if MICROPY_PY_ASYNC_AWAIT static void compile_atom_expr_await(compiler_t *comp, mp_parse_node_struct_t *pns) { if (comp->scope_cur->kind != SCOPE_FUNCTION && comp->scope_cur->kind != SCOPE_LAMBDA) { - compile_syntax_error(comp, (mp_parse_node_t)pns, MP_ERROR_TEXT("'await' outside function")); - return; + #if MICROPY_COMP_ALLOW_TOP_LEVEL_AWAIT + if (!mp_compile_allow_top_level_await) + #endif + { + compile_syntax_error(comp, (mp_parse_node_t)pns, MP_ERROR_TEXT("'await' outside function")); + return; + } } compile_atom_expr_normal(comp, pns); compile_yield_from(comp); diff --git a/py/compile.h b/py/compile.h index 5e0fd8b31c4a8..f9970a521d644 100644 --- a/py/compile.h +++ b/py/compile.h @@ -30,6 +30,11 @@ #include "py/parse.h" #include "py/emitglue.h" +#if MICROPY_COMP_ALLOW_TOP_LEVEL_AWAIT +// set to `true` to allow top-level await expressions +extern bool mp_compile_allow_top_level_await; +#endif + // the compiler will raise an exception if an error occurred // the compiler will clear the parse tree before it returns // mp_globals_get() will be used for the context diff --git a/py/mpconfig.h b/py/mpconfig.h index d9cff930d1485..af2480266bcab 100644 --- a/py/mpconfig.h +++ b/py/mpconfig.h @@ -446,6 +446,11 @@ #define MICROPY_DYNAMIC_COMPILER (0) #endif +// Whether the compiler allows compiling top-level await expressions +#ifndef MICROPY_COMP_ALLOW_TOP_LEVEL_AWAIT +#define MICROPY_COMP_ALLOW_TOP_LEVEL_AWAIT (0) +#endif + // Whether to enable constant folding; eg 1+2 rewritten as 3 #ifndef MICROPY_COMP_CONST_FOLDING #define MICROPY_COMP_CONST_FOLDING (MICROPY_CONFIG_ROM_LEVEL_AT_LEAST_CORE_FEATURES) From ff15dfcaa84dc7d29b9e791f01eaba21dbc2fd72 Mon Sep 17 00:00:00 2001 From: Damien George Date: Sat, 15 Apr 2023 09:27:43 +1000 Subject: [PATCH 02/19] webassembly: Include lib in sys.path. Following other ports. Signed-off-by: Damien George --- ports/webassembly/main.c | 1 + ports/webassembly/qstrdefsport.h | 1 + 2 files changed, 2 insertions(+) diff --git a/ports/webassembly/main.c b/ports/webassembly/main.c index 0aacf1ee095be..1b053046bfb99 100644 --- a/ports/webassembly/main.c +++ b/ports/webassembly/main.c @@ -105,6 +105,7 @@ void mp_js_init(int heap_size) { mp_vfs_mount(2, args, (mp_map_t *)&mp_const_empty_map); MP_STATE_VM(vfs_cur) = MP_STATE_VM(vfs_mount_table); } + mp_obj_list_append(mp_sys_path, MP_OBJ_NEW_QSTR(MP_QSTR__slash_lib)); #endif } diff --git a/ports/webassembly/qstrdefsport.h b/ports/webassembly/qstrdefsport.h index 00d3e2ae3c555..472d05f4375f6 100644 --- a/ports/webassembly/qstrdefsport.h +++ b/ports/webassembly/qstrdefsport.h @@ -1,2 +1,3 @@ // qstrs specific to this port // *FORMAT-OFF* +Q(/lib) From 8282bd93a2381dcc235ee89093832bb5e76e8102 Mon Sep 17 00:00:00 2001 From: Damien George Date: Tue, 30 May 2023 13:13:39 +1000 Subject: [PATCH 03/19] webassembly: Move MP_JS_EPOCH init to library postset. This eliminates the need for wrapper.js to run to set up the time. Signed-off-by: Damien George --- ports/webassembly/library.js | 7 ++++--- ports/webassembly/wrapper.js | 2 -- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/ports/webassembly/library.js b/ports/webassembly/library.js index 009b0a4dcc20c..f554f681c9332 100644 --- a/ports/webassembly/library.js +++ b/ports/webassembly/library.js @@ -35,9 +35,10 @@ mergeInto(LibraryManager.library, { } }, - mp_js_ticks_ms: function() { - return Date.now() - MP_JS_EPOCH; - }, + // This string will be emitted directly into the output file by Emscripten. + mp_js_ticks_ms__postset: "var MP_JS_EPOCH = Date.now()", + + mp_js_ticks_ms: () => Date.now() - MP_JS_EPOCH, mp_js_hook: function() { if (ENVIRONMENT_IS_NODE) { diff --git a/ports/webassembly/wrapper.js b/ports/webassembly/wrapper.js index e63abfffe4ae9..399801b7afcb9 100644 --- a/ports/webassembly/wrapper.js +++ b/ports/webassembly/wrapper.js @@ -33,8 +33,6 @@ var mainProgram = function() mp_js_init_repl = Module.cwrap('mp_js_init_repl', 'null', ['null']); mp_js_process_char = Module.cwrap('mp_js_process_char', 'number', ['number'], {async: true}); - MP_JS_EPOCH = Date.now(); - if (typeof window === 'undefined' && require.main === module) { var fs = require('fs'); var heap_size = 128 * 1024; From 76898cbfa1b8c48ec5f2ac75bc060675895bb67a Mon Sep 17 00:00:00 2001 From: Damien George Date: Wed, 13 Dec 2023 13:10:03 +1100 Subject: [PATCH 04/19] webassembly: Implement MICROPY_PY_RANDOM_SEED_INIT_FUNC. Signed-off-by: Damien George --- ports/webassembly/library.h | 1 + ports/webassembly/library.js | 7 +++++++ ports/webassembly/mpconfigport.h | 4 ++++ 3 files changed, 12 insertions(+) diff --git a/ports/webassembly/library.h b/ports/webassembly/library.h index 47982e064007a..21027c1372f0e 100644 --- a/ports/webassembly/library.h +++ b/ports/webassembly/library.h @@ -29,3 +29,4 @@ extern void mp_js_write(const char *str, mp_uint_t len); extern int mp_js_ticks_ms(void); extern void mp_js_hook(void); +extern uint32_t mp_js_random_u32(void); diff --git a/ports/webassembly/library.js b/ports/webassembly/library.js index f554f681c9332..4a17942bb4e83 100644 --- a/ports/webassembly/library.js +++ b/ports/webassembly/library.js @@ -63,4 +63,11 @@ mergeInto(LibraryManager.library, { } } }, + + // Node prior to v19 did not expose "crypto" as a global, so make sure it exists. + mp_js_random_u32__postset: + "if (globalThis.crypto === undefined) { globalThis.crypto = require('crypto'); }", + + mp_js_random_u32: () => + globalThis.crypto.getRandomValues(new Uint32Array(1))[0], }); diff --git a/ports/webassembly/mpconfigport.h b/ports/webassembly/mpconfigport.h index e0d1af0cce2bd..43a029c530ca8 100644 --- a/ports/webassembly/mpconfigport.h +++ b/ports/webassembly/mpconfigport.h @@ -49,6 +49,8 @@ #define MICROPY_FLOAT_IMPL (MICROPY_FLOAT_IMPL_DOUBLE) #define MICROPY_USE_INTERNAL_ERRNO (1) #define MICROPY_USE_INTERNAL_PRINTF (0) + +#define MICROPY_PY_RANDOM_SEED_INIT_FUNC (mp_js_random_u32()) #ifndef MICROPY_VFS #define MICROPY_VFS (1) #endif @@ -95,3 +97,5 @@ typedef long mp_off_t; // _GNU_SOURCE must be defined to get definitions of DT_xxx symbols from dirent.h. #define _GNU_SOURCE #endif + +uint32_t mp_js_random_u32(void); From 8e3b701dee944582739adfa9a97de8ea76911a80 Mon Sep 17 00:00:00 2001 From: Damien George Date: Wed, 13 Dec 2023 13:48:32 +1100 Subject: [PATCH 05/19] webassembly: Enable time localtime, gmtime, time, time_ns. Signed-off-by: Damien George --- ports/webassembly/Makefile | 1 + ports/webassembly/library.h | 1 + ports/webassembly/library.js | 2 ++ ports/webassembly/modtime.c | 51 ++++++++++++++++++++++++++++++++ ports/webassembly/mpconfigport.h | 4 +++ ports/webassembly/mphalport.c | 9 ++++++ ports/webassembly/mphalport.h | 1 + 7 files changed, 69 insertions(+) create mode 100644 ports/webassembly/modtime.c diff --git a/ports/webassembly/Makefile b/ports/webassembly/Makefile index c5ee80fe0046f..4e1d53b0ccfdf 100644 --- a/ports/webassembly/Makefile +++ b/ports/webassembly/Makefile @@ -23,6 +23,7 @@ SRC_SHARED = $(addprefix shared/,\ runtime/stdout_helpers.c \ runtime/pyexec.c \ readline/readline.c \ + timeutils/timeutils.c \ ) SRC_C = \ diff --git a/ports/webassembly/library.h b/ports/webassembly/library.h index 21027c1372f0e..04b408d71a25a 100644 --- a/ports/webassembly/library.h +++ b/ports/webassembly/library.h @@ -29,4 +29,5 @@ extern void mp_js_write(const char *str, mp_uint_t len); extern int mp_js_ticks_ms(void); extern void mp_js_hook(void); +extern double mp_js_time_ms(void); extern uint32_t mp_js_random_u32(void); diff --git a/ports/webassembly/library.js b/ports/webassembly/library.js index 4a17942bb4e83..d1266598da93e 100644 --- a/ports/webassembly/library.js +++ b/ports/webassembly/library.js @@ -64,6 +64,8 @@ mergeInto(LibraryManager.library, { } }, + mp_js_time_ms: () => Date.now(), + // Node prior to v19 did not expose "crypto" as a global, so make sure it exists. mp_js_random_u32__postset: "if (globalThis.crypto === undefined) { globalThis.crypto = require('crypto'); }", diff --git a/ports/webassembly/modtime.c b/ports/webassembly/modtime.c new file mode 100644 index 0000000000000..1b1e63d4ddf3a --- /dev/null +++ b/ports/webassembly/modtime.c @@ -0,0 +1,51 @@ +/* + * This file is part of the MicroPython project, http://micropython.org/ + * + * The MIT License (MIT) + * + * Copyright (c) 2023 Damien P. George + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ + +#include "py/obj.h" +#include "shared/timeutils/timeutils.h" +#include "library.h" + +// Return the localtime as an 8-tuple. +static mp_obj_t mp_time_localtime_get(void) { + timeutils_struct_time_t tm; + timeutils_seconds_since_epoch_to_struct_time(mp_hal_time_ms() / 1000, &tm); + mp_obj_t tuple[8] = { + mp_obj_new_int(tm.tm_year), + mp_obj_new_int(tm.tm_mon), + mp_obj_new_int(tm.tm_mday), + mp_obj_new_int(tm.tm_hour), + mp_obj_new_int(tm.tm_min), + mp_obj_new_int(tm.tm_sec), + mp_obj_new_int(tm.tm_wday), + mp_obj_new_int(tm.tm_yday), + }; + return mp_obj_new_tuple(8, tuple); +} + +// Returns the number of seconds, as a float, since the Epoch. +static mp_obj_t mp_time_time_get(void) { + return mp_obj_new_float((mp_float_t)mp_hal_time_ms() / 1000); +} diff --git a/ports/webassembly/mpconfigport.h b/ports/webassembly/mpconfigport.h index 43a029c530ca8..79537dd72de4c 100644 --- a/ports/webassembly/mpconfigport.h +++ b/ports/webassembly/mpconfigport.h @@ -50,7 +50,11 @@ #define MICROPY_USE_INTERNAL_ERRNO (1) #define MICROPY_USE_INTERNAL_PRINTF (0) +#define MICROPY_EPOCH_IS_1970 (1) #define MICROPY_PY_RANDOM_SEED_INIT_FUNC (mp_js_random_u32()) +#define MICROPY_PY_TIME_GMTIME_LOCALTIME_MKTIME (1) +#define MICROPY_PY_TIME_TIME_TIME_NS (1) +#define MICROPY_PY_TIME_INCLUDEFILE "ports/webassembly/modtime.c" #ifndef MICROPY_VFS #define MICROPY_VFS (1) #endif diff --git a/ports/webassembly/mphalport.c b/ports/webassembly/mphalport.c index f91a509013a3c..72a326e30d1f9 100644 --- a/ports/webassembly/mphalport.c +++ b/ports/webassembly/mphalport.c @@ -56,6 +56,15 @@ mp_uint_t mp_hal_ticks_cpu(void) { return 0; } +uint64_t mp_hal_time_ms(void) { + double mm = mp_js_time_ms(); + return (uint64_t)mm; +} + +uint64_t mp_hal_time_ns(void) { + return mp_hal_time_ms() * 1000000ULL; +} + extern int mp_interrupt_char; int mp_hal_get_interrupt_char(void) { diff --git a/ports/webassembly/mphalport.h b/ports/webassembly/mphalport.h index 1b3179698644b..a90de8ec5fb0e 100644 --- a/ports/webassembly/mphalport.h +++ b/ports/webassembly/mphalport.h @@ -35,6 +35,7 @@ void mp_hal_delay_us(mp_uint_t us); mp_uint_t mp_hal_ticks_ms(void); mp_uint_t mp_hal_ticks_us(void); mp_uint_t mp_hal_ticks_cpu(void); +uint64_t mp_hal_time_ms(void); int mp_hal_get_interrupt_char(void); From ae6bcc9d2345d73016fd22f9cc22b56c73bae432 Mon Sep 17 00:00:00 2001 From: Damien George Date: Wed, 31 May 2023 11:44:45 +1000 Subject: [PATCH 06/19] webassembly: Use POSIX write for output and add stderr. All output is now handled by Emscripten's stdio facility. Signed-off-by: Damien George --- ports/webassembly/library.js | 10 ---------- ports/webassembly/mpconfigport.h | 4 +++- ports/webassembly/mphalport.c | 11 +++++++++-- 3 files changed, 12 insertions(+), 13 deletions(-) diff --git a/ports/webassembly/library.js b/ports/webassembly/library.js index d1266598da93e..db5bac2691443 100644 --- a/ports/webassembly/library.js +++ b/ports/webassembly/library.js @@ -25,16 +25,6 @@ */ mergeInto(LibraryManager.library, { - mp_js_write: function(ptr, len) { - const buffer = HEAPU8.subarray(ptr, ptr + len) - if (ENVIRONMENT_IS_NODE) { - process.stdout.write(buffer); - } else { - const printEvent = new CustomEvent('micropython-print', { detail: buffer }); - document.dispatchEvent(printEvent); - } - }, - // This string will be emitted directly into the output file by Emscripten. mp_js_ticks_ms__postset: "var MP_JS_EPOCH = Date.now()", diff --git a/ports/webassembly/mpconfigport.h b/ports/webassembly/mpconfigport.h index 79537dd72de4c..6b0f2753a960a 100644 --- a/ports/webassembly/mpconfigport.h +++ b/ports/webassembly/mpconfigport.h @@ -46,6 +46,7 @@ #define MICROPY_LONGINT_IMPL (MICROPY_LONGINT_IMPL_MPZ) #define MICROPY_ENABLE_DOC_STRING (1) #define MICROPY_WARNINGS (1) +#define MICROPY_ERROR_PRINTER (&mp_stderr_print) #define MICROPY_FLOAT_IMPL (MICROPY_FLOAT_IMPL_DOUBLE) #define MICROPY_USE_INTERNAL_ERRNO (1) #define MICROPY_USE_INTERNAL_PRINTF (0) @@ -60,7 +61,6 @@ #endif #define MICROPY_VFS_POSIX (MICROPY_VFS) #define MICROPY_PY_SYS_PLATFORM "webassembly" -#define MICROPY_PY_SYS_STDFILES (0) #define MICROPY_EVENT_POLL_HOOK \ do { \ @@ -102,4 +102,6 @@ typedef long mp_off_t; #define _GNU_SOURCE #endif +extern const struct _mp_print_t mp_stderr_print; + uint32_t mp_js_random_u32(void); diff --git a/ports/webassembly/mphalport.c b/ports/webassembly/mphalport.c index 72a326e30d1f9..9ab47762e3e97 100644 --- a/ports/webassembly/mphalport.c +++ b/ports/webassembly/mphalport.c @@ -24,12 +24,19 @@ * THE SOFTWARE. */ +#include #include "library.h" #include "mphalport.h" +static void stderr_print_strn(void *env, const char *str, size_t len) { + (void)env; + write(2, str, len); +} + +const mp_print_t mp_stderr_print = {NULL, stderr_print_strn}; + mp_uint_t mp_hal_stdout_tx_strn(const char *str, size_t len) { - mp_js_write(str, len); - return len; + return write(1, str, len); } void mp_hal_delay_ms(mp_uint_t ms) { From 98a8ff7a1a42211acc62e5c41a445a9d6f256e70 Mon Sep 17 00:00:00 2001 From: Damien George Date: Thu, 22 Jun 2023 17:31:40 +1000 Subject: [PATCH 07/19] webassembly: Add support for enabling MICROPY_GC_SPLIT_HEAP_AUTO. When enabled the GC will not reclaim any memory on a call to `gc_collect()`. Instead it will grow the heap. Signed-off-by: Damien George --- ports/webassembly/main.c | 15 +++++++++++++++ ports/webassembly/mpconfigport.h | 1 + 2 files changed, 16 insertions(+) diff --git a/ports/webassembly/main.c b/ports/webassembly/main.c index 1b053046bfb99..ebde8ac7008ba 100644 --- a/ports/webassembly/main.c +++ b/ports/webassembly/main.c @@ -113,6 +113,19 @@ void mp_js_init_repl() { pyexec_event_repl_init(); } +#if MICROPY_GC_SPLIT_HEAP_AUTO + +// The largest new region that is available to become Python heap. +size_t gc_get_max_new_split(void) { + return 128 * 1024 * 1024; +} + +// Don't collect anything. Instead require the heap to grow. +void gc_collect(void) { +} + +#else + static void gc_scan_func(void *begin, void *end) { gc_collect_root((void **)begin, (void **)end - (void **)begin + 1); } @@ -124,6 +137,8 @@ void gc_collect(void) { gc_collect_end(); } +#endif + #if !MICROPY_VFS mp_lexer_t *mp_lexer_new_from_file(qstr filename) { mp_raise_OSError(MP_ENOENT); diff --git a/ports/webassembly/mpconfigport.h b/ports/webassembly/mpconfigport.h index 6b0f2753a960a..d0261362390fa 100644 --- a/ports/webassembly/mpconfigport.h +++ b/ports/webassembly/mpconfigport.h @@ -26,6 +26,7 @@ */ #include +#include // for malloc, for MICROPY_GC_SPLIT_HEAP_AUTO // options to control how MicroPython is built From 691cd3a56d74354bf2cb3d6f46dc226760134297 Mon Sep 17 00:00:00 2001 From: Damien George Date: Tue, 30 May 2023 13:15:58 +1000 Subject: [PATCH 08/19] webassembly: Clean up Makefile and add variant support. This commit cleans up and generalises the Makefile, adds support for variants (following the unix port) and adds the "standard" variant as the default variant. Signed-off-by: Damien George --- ports/webassembly/Makefile | 78 +++++++++++++++---- ports/webassembly/mpconfigport.h | 21 +++-- .../variants/standard/mpconfigvariant.h | 1 + .../variants/standard/mpconfigvariant.mk | 1 + 4 files changed, 82 insertions(+), 19 deletions(-) create mode 100644 ports/webassembly/variants/standard/mpconfigvariant.h create mode 100644 ports/webassembly/variants/standard/mpconfigvariant.mk diff --git a/ports/webassembly/Makefile b/ports/webassembly/Makefile index 4e1d53b0ccfdf..e8f27d862bb8a 100644 --- a/ports/webassembly/Makefile +++ b/ports/webassembly/Makefile @@ -1,23 +1,67 @@ -include ../../py/mkenv.mk +################################################################################ +# Initial setup of Makefile environment. + +# Select the variant to build for: +ifdef VARIANT_DIR +# Custom variant path - remove trailing slash and get the final component of +# the path as the variant name. +VARIANT ?= $(notdir $(VARIANT_DIR:/=)) +else +# If not given on the command line, then default to standard. +VARIANT ?= standard +VARIANT_DIR ?= variants/$(VARIANT) +endif + +ifeq ($(wildcard $(VARIANT_DIR)/.),) +$(error Invalid VARIANT specified: $(VARIANT_DIR)) +endif + +# If the build directory is not given, make it reflect the variant name. +BUILD ?= build-$(VARIANT) -CROSS = 0 +include ../../py/mkenv.mk +include $(VARIANT_DIR)/mpconfigvariant.mk +# Qstr definitions (must come before including py.mk). QSTR_DEFS = qstrdefsport.h +# Include py core make definitions. include $(TOP)/py/py.mk include $(TOP)/extmod/extmod.mk +################################################################################ +# Project specific settings and compiler/linker flags. + CC = emcc LD = emcc +TERSER ?= npx terser INC += -I. INC += -I$(TOP) INC += -I$(BUILD) +INC += -I$(VARIANT_DIR) CFLAGS += -std=c99 -Wall -Werror -Wdouble-promotion -Wfloat-conversion CFLAGS += -Os -DNDEBUG CFLAGS += $(INC) +JSFLAGS += -s EXPORTED_FUNCTIONS="\ + _mp_js_init,\ + _mp_js_init_repl,\ + _mp_js_do_str,\ + _mp_js_process_char,\ + _mp_hal_get_interrupt_char,\ + _mp_sched_keyboard_interrupt$(EXPORTED_FUNCTIONS_EXTRA)" +JSFLAGS += -s EXPORTED_RUNTIME_METHODS="\ + ccall,\ + cwrap,\ + FS$(EXPORTED_RUNTIME_METHODS_EXTRA)" +JSFLAGS += --js-library library.js +JSFLAGS += -s SUPPORT_LONGJMP=emscripten + +################################################################################ +# Source files and libraries. + SRC_SHARED = $(addprefix shared/,\ runtime/interrupt_char.c \ runtime/stdout_helpers.c \ @@ -26,33 +70,41 @@ SRC_SHARED = $(addprefix shared/,\ timeutils/timeutils.c \ ) -SRC_C = \ +SRC_C += \ main.c \ mphalport.c \ +# List of sources for qstr extraction. SRC_QSTR += $(SRC_C) $(SRC_SHARED) +SRC_JS ?= wrapper.js + OBJ += $(PY_O) OBJ += $(addprefix $(BUILD)/, $(SRC_SHARED:.c=.o)) OBJ += $(addprefix $(BUILD)/, $(SRC_C:.c=.o)) -JSFLAGS += -s ASYNCIFY -JSFLAGS += -s EXPORTED_FUNCTIONS="['_mp_js_init', '_mp_js_init_repl', '_mp_js_do_str', '_mp_js_process_char', '_mp_hal_get_interrupt_char', '_mp_sched_keyboard_interrupt']" -JSFLAGS += -s EXPORTED_RUNTIME_METHODS="['ccall', 'cwrap', 'FS']" -JSFLAGS += --js-library library.js +################################################################################ +# Main targets. + +.PHONY: all min test all: $(BUILD)/micropython.js -$(BUILD)/micropython.js: $(OBJ) library.js wrapper.js - $(ECHO) "LINK $(BUILD)/firmware.js" - $(Q)emcc $(LDFLAGS) -o $(BUILD)/firmware.js $(OBJ) $(JSFLAGS) - cat wrapper.js $(BUILD)/firmware.js > $@ +$(BUILD)/micropython.js: $(OBJ) library.js $(SRC_JS) + $(ECHO) "LINK $@" + $(Q)emcc $(LDFLAGS) -o $@ $(OBJ) $(JSFLAGS) + $(Q)cat $(SRC_JS) >> $@ -min: $(BUILD)/micropython.js - uglifyjs $< -c -o $(BUILD)/micropython.min.js +$(BUILD)/micropython.min.js: $(BUILD)/micropython.js + $(TERSER) $< --compress --module -o $@ + +min: $(BUILD)/micropython.min.js test: $(BUILD)/micropython.js $(TOP)/tests/run-tests.py $(eval DIRNAME=ports/$(notdir $(CURDIR))) cd $(TOP)/tests && MICROPY_MICROPYTHON=../ports/webassembly/node_run.sh ./run-tests.py -j1 +################################################################################ +# Remaining make rules. + include $(TOP)/py/mkrules.mk diff --git a/ports/webassembly/mpconfigport.h b/ports/webassembly/mpconfigport.h index d0261362390fa..abfbbca7941f8 100644 --- a/ports/webassembly/mpconfigport.h +++ b/ports/webassembly/mpconfigport.h @@ -25,17 +25,18 @@ * THE SOFTWARE. */ +// Options to control how MicroPython is built for this port, overriding +// defaults in py/mpconfig.h. + #include #include // for malloc, for MICROPY_GC_SPLIT_HEAP_AUTO -// options to control how MicroPython is built +// Variant-specific definitions. +#include "mpconfigvariant.h" +#ifndef MICROPY_CONFIG_ROM_LEVEL #define MICROPY_CONFIG_ROM_LEVEL (MICROPY_CONFIG_ROM_LEVEL_EXTRA_FEATURES) - -// You can disable the built-in MicroPython compiler by setting the following -// config option to 0. If you do this then you won't get a REPL prompt, but you -// will still be able to execute pre-compiled scripts, compiled with mpy-cross. -#define MICROPY_ENABLE_COMPILER (1) +#endif #define MICROPY_ALLOC_PATH_MAX (256) #define MICROPY_READER_VFS (MICROPY_VFS) @@ -69,6 +70,13 @@ mp_handle_pending(true); \ } while (0); +// Whether the VM will periodically call mp_js_hook(), which checks for +// interrupt characters on stdin (or equivalent input). +#ifndef MICROPY_VARIANT_ENABLE_JS_HOOK +#define MICROPY_VARIANT_ENABLE_JS_HOOK (0) +#endif + +#if MICROPY_VARIANT_ENABLE_JS_HOOK #define MICROPY_VM_HOOK_COUNT (10) #define MICROPY_VM_HOOK_INIT static uint vm_hook_divisor = MICROPY_VM_HOOK_COUNT; #define MICROPY_VM_HOOK_POLL if (--vm_hook_divisor == 0) { \ @@ -78,6 +86,7 @@ } #define MICROPY_VM_HOOK_LOOP MICROPY_VM_HOOK_POLL #define MICROPY_VM_HOOK_RETURN MICROPY_VM_HOOK_POLL +#endif // type definitions for the specific machine diff --git a/ports/webassembly/variants/standard/mpconfigvariant.h b/ports/webassembly/variants/standard/mpconfigvariant.h new file mode 100644 index 0000000000000..7be62ea7f4f3f --- /dev/null +++ b/ports/webassembly/variants/standard/mpconfigvariant.h @@ -0,0 +1 @@ +#define MICROPY_VARIANT_ENABLE_JS_HOOK (1) diff --git a/ports/webassembly/variants/standard/mpconfigvariant.mk b/ports/webassembly/variants/standard/mpconfigvariant.mk new file mode 100644 index 0000000000000..62ee161907de2 --- /dev/null +++ b/ports/webassembly/variants/standard/mpconfigvariant.mk @@ -0,0 +1 @@ +JSFLAGS += -s ASYNCIFY From 39bd0b8a0a18d7bfc499acc7a11028da12f43edc Mon Sep 17 00:00:00 2001 From: Damien George Date: Wed, 31 May 2023 11:45:34 +1000 Subject: [PATCH 09/19] webassembly: Add JavaScript proxying, and js and jsffi modules. This commit improves the webassembly port by adding: - Proxying of Python objects to JavaScript with a PyProxy type that lives on the JavaScript side. PyProxy implements JavaScript Proxy traps such as has, get, set and ownKeys, to make Python objects have functionality on the JavaScript side. - Proxying of JavaScript objects to Python with a JsProxy type that lives on the Python side. JsProxy passes through calls, attributes, subscription and iteration from Python to JavaScript. - A top-level API on the JavaScript side to construct a MicroPython interpreter instance via `loadMicroPython()`. That function returns an object that can be used to execute Python code, access the Python globals dict, access the Emscripten filesystem, and other things. This API is based on the API provided by Pyodide (https://pyodide.org/). As part of this, the top-level file is changed from `micropython.js` to `micropython.mjs`. - A Python `js` module which can be used to access all JavaScript-side symbols, for example the DOM when run within a browser. - A Python `jsffi` module with various helper functions like `create_proxy()` and `to_js()`. - A dedenting lexer which automatically dedents Python source code if every non-empty line in that source starts with a common whitespace prefix. This is very helpful when Python source code is indented within a string within HTML or JavaScript for formatting reasons. Signed-off-by: Damien George --- ports/webassembly/Makefile | 56 +++++- ports/webassembly/api.js | 146 ++++++++++++++ ports/webassembly/lexer_dedent.c | 105 ++++++++++ ports/webassembly/lexer_dedent.h | 36 ++++ ports/webassembly/main.c | 56 ++++++ ports/webassembly/modjs.c | 55 ++++++ ports/webassembly/modjsffi.c | 80 ++++++++ ports/webassembly/mpconfigport.h | 8 + ports/webassembly/objjsproxy.c | 330 +++++++++++++++++++++++++++++++ ports/webassembly/objpyproxy.js | 191 ++++++++++++++++++ ports/webassembly/proxy_c.c | 281 ++++++++++++++++++++++++++ ports/webassembly/proxy_c.h | 58 ++++++ ports/webassembly/proxy_js.js | 222 +++++++++++++++++++++ 13 files changed, 1615 insertions(+), 9 deletions(-) create mode 100644 ports/webassembly/api.js create mode 100644 ports/webassembly/lexer_dedent.c create mode 100644 ports/webassembly/lexer_dedent.h create mode 100644 ports/webassembly/modjs.c create mode 100644 ports/webassembly/modjsffi.c create mode 100644 ports/webassembly/objjsproxy.c create mode 100644 ports/webassembly/objpyproxy.js create mode 100644 ports/webassembly/proxy_c.c create mode 100644 ports/webassembly/proxy_c.h create mode 100644 ports/webassembly/proxy_js.js diff --git a/ports/webassembly/Makefile b/ports/webassembly/Makefile index e8f27d862bb8a..2a5669392eb41 100644 --- a/ports/webassembly/Makefile +++ b/ports/webassembly/Makefile @@ -45,7 +45,34 @@ CFLAGS += -std=c99 -Wall -Werror -Wdouble-promotion -Wfloat-conversion CFLAGS += -Os -DNDEBUG CFLAGS += $(INC) +EXPORTED_FUNCTIONS_EXTRA += ,\ + _mp_js_do_exec,\ + _mp_js_do_import,\ + _mp_js_register_js_module,\ + _proxy_c_init,\ + _proxy_c_to_js_call,\ + _proxy_c_to_js_delete_attr,\ + _proxy_c_to_js_dir,\ + _proxy_c_to_js_get_array,\ + _proxy_c_to_js_get_dict,\ + _proxy_c_to_js_get_type,\ + _proxy_c_to_js_has_attr,\ + _proxy_c_to_js_lookup_attr,\ + _proxy_c_to_js_store_attr,\ + _proxy_convert_mp_to_js_obj_cside + +EXPORTED_RUNTIME_METHODS_EXTRA += ,\ + PATH,\ + PATH_FS,\ + UTF8ToString,\ + getValue,\ + lengthBytesUTF8,\ + setValue,\ + stringToUTF8 + JSFLAGS += -s EXPORTED_FUNCTIONS="\ + _free,\ + _malloc,\ _mp_js_init,\ _mp_js_init_repl,\ _mp_js_do_str,\ @@ -58,6 +85,7 @@ JSFLAGS += -s EXPORTED_RUNTIME_METHODS="\ FS$(EXPORTED_RUNTIME_METHODS_EXTRA)" JSFLAGS += --js-library library.js JSFLAGS += -s SUPPORT_LONGJMP=emscripten +JSFLAGS += -s MODULARIZE -s EXPORT_NAME=_createMicroPythonModule ################################################################################ # Source files and libraries. @@ -71,13 +99,21 @@ SRC_SHARED = $(addprefix shared/,\ ) SRC_C += \ + lexer_dedent.c \ main.c \ + modjs.c \ + modjsffi.c \ mphalport.c \ + objjsproxy.c \ + proxy_c.c \ # List of sources for qstr extraction. SRC_QSTR += $(SRC_C) $(SRC_SHARED) -SRC_JS ?= wrapper.js +SRC_JS += \ + api.js \ + objpyproxy.js \ + proxy_js.js \ OBJ += $(PY_O) OBJ += $(addprefix $(BUILD)/, $(SRC_SHARED:.c=.o)) @@ -86,23 +122,25 @@ OBJ += $(addprefix $(BUILD)/, $(SRC_C:.c=.o)) ################################################################################ # Main targets. -.PHONY: all min test +.PHONY: all min test test_min -all: $(BUILD)/micropython.js +all: $(BUILD)/micropython.mjs -$(BUILD)/micropython.js: $(OBJ) library.js $(SRC_JS) +$(BUILD)/micropython.mjs: $(OBJ) library.js $(SRC_JS) $(ECHO) "LINK $@" $(Q)emcc $(LDFLAGS) -o $@ $(OBJ) $(JSFLAGS) $(Q)cat $(SRC_JS) >> $@ -$(BUILD)/micropython.min.js: $(BUILD)/micropython.js +$(BUILD)/micropython.min.mjs: $(BUILD)/micropython.mjs $(TERSER) $< --compress --module -o $@ -min: $(BUILD)/micropython.min.js +min: $(BUILD)/micropython.min.mjs + +test: $(BUILD)/micropython.mjs $(TOP)/tests/run-tests.py + cd $(TOP)/tests && MICROPY_MICROPYTHON_MJS=../ports/webassembly/$< ./run-tests.py --target webassembly -test: $(BUILD)/micropython.js $(TOP)/tests/run-tests.py - $(eval DIRNAME=ports/$(notdir $(CURDIR))) - cd $(TOP)/tests && MICROPY_MICROPYTHON=../ports/webassembly/node_run.sh ./run-tests.py -j1 +test_min: $(BUILD)/micropython.min.mjs $(TOP)/tests/run-tests.py + cd $(TOP)/tests && MICROPY_MICROPYTHON_MJS=../ports/webassembly/$< ./run-tests.py --target webassembly ################################################################################ # Remaining make rules. diff --git a/ports/webassembly/api.js b/ports/webassembly/api.js new file mode 100644 index 0000000000000..dfe7561768a4f --- /dev/null +++ b/ports/webassembly/api.js @@ -0,0 +1,146 @@ +/* + * This file is part of the MicroPython project, http://micropython.org/ + * + * The MIT License (MIT) + * + * Copyright (c) 2023-2024 Damien P. George + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ + +// Options: +// - heapsize: size in bytes of the MicroPython GC heap. +// - url: location to load `micropython.mjs`. +// - stdin: function to return input characters. +// - stdout: function that takes one argument, and is passed lines of stdout +// output as they are produced. By default this is handled by Emscripten +// and in a browser goes to console, in node goes to process.stdout.write. +// - stderr: same behaviour as stdout but for error output. +// - linebuffer: whether to buffer line-by-line to stdout/stderr. +export async function loadMicroPython(options) { + const { heapsize, url, stdin, stdout, stderr, linebuffer } = Object.assign( + { heapsize: 1024 * 1024, linebuffer: true }, + options, + ); + const Module = {}; + Module.locateFile = (path, scriptDirectory) => + url || scriptDirectory + path; + Module._textDecoder = new TextDecoder(); + if (stdin !== undefined) { + Module.stdin = stdin; + } + if (stdout !== undefined) { + if (linebuffer) { + Module._stdoutBuffer = []; + Module.stdout = (c) => { + if (c === 10) { + stdout( + Module._textDecoder.decode( + new Uint8Array(Module._stdoutBuffer), + ), + ); + Module._stdoutBuffer = []; + } else { + Module._stdoutBuffer.push(c); + } + }; + } else { + Module.stdout = (c) => stdout(new Uint8Array([c])); + } + } + if (stderr !== undefined) { + if (linebuffer) { + Module._stderrBuffer = []; + Module.stderr = (c) => { + if (c === 10) { + stderr( + Module._textDecoder.decode( + new Uint8Array(Module._stderrBuffer), + ), + ); + Module._stderrBuffer = []; + } else { + Module._stderrBuffer.push(c); + } + }; + } else { + Module.stderr = (c) => stderr(new Uint8Array([c])); + } + } + const moduleLoaded = new Promise((r) => { + Module.postRun = r; + }); + _createMicroPythonModule(Module); + await moduleLoaded; + globalThis.Module = Module; + proxy_js_init(); + const pyimport = (name) => { + const value = Module._malloc(3 * 4); + Module.ccall( + "mp_js_do_import", + "null", + ["string", "pointer"], + [name, value], + ); + return proxy_convert_mp_to_js_obj_jsside_with_free(value); + }; + Module.ccall("mp_js_init", "null", ["number"], [heapsize]); + Module.ccall("proxy_c_init", "null", [], []); + return { + _module: Module, + PyProxy: PyProxy, + FS: Module.FS, + globals: { + __dict__: pyimport("__main__").__dict__, + get(key) { + return this.__dict__[key]; + }, + set(key, value) { + this.__dict__[key] = value; + }, + delete(key) { + delete this.__dict__[key]; + }, + }, + registerJsModule(name, module) { + const value = Module._malloc(3 * 4); + proxy_convert_js_to_mp_obj_jsside(module, value); + Module.ccall( + "mp_js_register_js_module", + "null", + ["string", "pointer"], + [name, value], + ); + Module._free(value); + }, + pyimport: pyimport, + runPython(code) { + const value = Module._malloc(3 * 4); + Module.ccall( + "mp_js_do_exec", + "number", + ["string", "pointer"], + [code, value], + ); + return proxy_convert_mp_to_js_obj_jsside_with_free(value); + }, + }; +} + +globalThis.loadMicroPython = loadMicroPython; diff --git a/ports/webassembly/lexer_dedent.c b/ports/webassembly/lexer_dedent.c new file mode 100644 index 0000000000000..555caea89692b --- /dev/null +++ b/ports/webassembly/lexer_dedent.c @@ -0,0 +1,105 @@ +/* + * This file is part of the MicroPython project, http://micropython.org/ + * + * The MIT License (MIT) + * + * Copyright (c) 2023 Damien P. George + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ + +#include "lexer_dedent.h" + +typedef struct _mp_reader_mem_dedent_t { + size_t free_len; // if >0 mem is freed on close by: m_free(beg, free_len) + const byte *beg; + const byte *cur; + const byte *end; + size_t dedent_prefix; +} mp_reader_mem_dedent_t; + +// Work out the amount of common whitespace among all non-empty lines. +static size_t dedent(const byte *text, size_t len) { + size_t min_prefix = -1; + size_t cur_prefix = 0; + bool start_of_line = true; + for (const byte *t = text; t < text + len; ++t) { + if (*t == '\n') { + start_of_line = true; + cur_prefix = 0; + } else if (start_of_line) { + if (unichar_isspace(*t)) { + ++cur_prefix; + } else { + if (cur_prefix < min_prefix) { + min_prefix = cur_prefix; + if (min_prefix == 0) { + return min_prefix; + } + } + start_of_line = false; + } + } + } + return min_prefix; +} + +static mp_uint_t mp_reader_mem_dedent_readbyte(void *data) { + mp_reader_mem_dedent_t *reader = (mp_reader_mem_dedent_t *)data; + if (reader->cur < reader->end) { + byte c = *reader->cur++; + if (c == '\n') { + for (size_t i = 0; i < reader->dedent_prefix; ++i) { + if (*reader->cur == '\n') { + break; + } + ++reader->cur; + } + } + return c; + } else { + return MP_READER_EOF; + } +} + +static void mp_reader_mem_dedent_close(void *data) { + mp_reader_mem_dedent_t *reader = (mp_reader_mem_dedent_t *)data; + if (reader->free_len > 0) { + m_del(char, (char *)reader->beg, reader->free_len); + } + m_del_obj(mp_reader_mem_dedent_t, reader); +} + +static void mp_reader_new_mem_dedent(mp_reader_t *reader, const byte *buf, size_t len, size_t free_len) { + mp_reader_mem_dedent_t *rm = m_new_obj(mp_reader_mem_dedent_t); + rm->free_len = free_len; + rm->beg = buf; + rm->cur = buf; + rm->end = buf + len; + rm->dedent_prefix = dedent(buf, len); + reader->data = rm; + reader->readbyte = mp_reader_mem_dedent_readbyte; + reader->close = mp_reader_mem_dedent_close; +} + +mp_lexer_t *mp_lexer_new_from_str_len_dedent(qstr src_name, const char *str, size_t len, size_t free_len) { + mp_reader_t reader; + mp_reader_new_mem_dedent(&reader, (const byte *)str, len, free_len); + return mp_lexer_new(src_name, reader); +} diff --git a/ports/webassembly/lexer_dedent.h b/ports/webassembly/lexer_dedent.h new file mode 100644 index 0000000000000..a8cc2526b4f53 --- /dev/null +++ b/ports/webassembly/lexer_dedent.h @@ -0,0 +1,36 @@ +/* + * This file is part of the MicroPython project, http://micropython.org/ + * + * The MIT License (MIT) + * + * Copyright (c) 2023-2024 Damien P. George + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ +#ifndef MICROPY_INCLUDED_WEBASSEMBLY_LEXER_DEDENT_H +#define MICROPY_INCLUDED_WEBASSEMBLY_LEXER_DEDENT_H + +#include "py/lexer.h" + +// This function creates a new "dedenting lexer" which automatically dedents the input +// source code if every non-empty line in that source starts with a common whitespace +// prefix. It does this dedenting inplace as the memory is read. +mp_lexer_t *mp_lexer_new_from_str_len_dedent(qstr src_name, const char *str, size_t len, size_t free_len); + +#endif // MICROPY_INCLUDED_WEBASSEMBLY_LEXER_DEDENT_H diff --git a/ports/webassembly/main.c b/ports/webassembly/main.c index ebde8ac7008ba..c1c7a88840ec3 100644 --- a/ports/webassembly/main.c +++ b/ports/webassembly/main.c @@ -40,7 +40,9 @@ #include "shared/runtime/pyexec.h" #include "emscripten.h" +#include "lexer_dedent.h" #include "library.h" +#include "proxy_c.h" #if MICROPY_ENABLE_COMPILER int do_str(const char *src, mp_parse_input_kind_t input_kind) { @@ -113,6 +115,60 @@ void mp_js_init_repl() { pyexec_event_repl_init(); } +void mp_js_register_js_module(const char *name, uint32_t *value) { + mp_obj_t module_name = MP_OBJ_NEW_QSTR(qstr_from_str(name)); + mp_obj_t module = proxy_convert_js_to_mp_obj_cside(value); + mp_map_t *mp_loaded_modules_map = &MP_STATE_VM(mp_loaded_modules_dict).map; + mp_map_lookup(mp_loaded_modules_map, module_name, MP_MAP_LOOKUP_ADD_IF_NOT_FOUND)->value = module; +} + +void mp_js_do_import(const char *name, uint32_t *out) { + nlr_buf_t nlr; + if (nlr_push(&nlr) == 0) { + mp_obj_t ret = mp_import_name(qstr_from_str(name), mp_const_none, MP_OBJ_NEW_SMALL_INT(0)); + // Return the leaf of the import, eg for "a.b.c" return "c". + const char *m = name; + const char *n = name; + for (;; ++n) { + if (*n == '\0' || *n == '.') { + if (m != name) { + ret = mp_load_attr(ret, qstr_from_strn(m, n - m)); + } + m = n + 1; + if (*n == '\0') { + break; + } + } + } + nlr_pop(); + proxy_convert_mp_to_js_obj_cside(ret, out); + } else { + // uncaught exception + proxy_convert_mp_to_js_exc_cside(nlr.ret_val, out); + } +} + +void mp_js_do_exec(const char *src, uint32_t *out) { + // Collect at the top-level, where there are no root pointers from stack/registers. + gc_collect_start(); + gc_collect_end(); + + mp_parse_input_kind_t input_kind = MP_PARSE_FILE_INPUT; + nlr_buf_t nlr; + if (nlr_push(&nlr) == 0) { + mp_lexer_t *lex = mp_lexer_new_from_str_len_dedent(MP_QSTR__lt_stdin_gt_, src, strlen(src), 0); + qstr source_name = lex->source_name; + mp_parse_tree_t parse_tree = mp_parse(lex, input_kind); + mp_obj_t module_fun = mp_compile(&parse_tree, source_name, false); + mp_obj_t ret = mp_call_function_0(module_fun); + nlr_pop(); + proxy_convert_mp_to_js_obj_cside(ret, out); + } else { + // uncaught exception + proxy_convert_mp_to_js_exc_cside(nlr.ret_val, out); + } +} + #if MICROPY_GC_SPLIT_HEAP_AUTO // The largest new region that is available to become Python heap. diff --git a/ports/webassembly/modjs.c b/ports/webassembly/modjs.c new file mode 100644 index 0000000000000..bed09086ab7cf --- /dev/null +++ b/ports/webassembly/modjs.c @@ -0,0 +1,55 @@ +/* + * This file is part of the MicroPython project, http://micropython.org/ + * + * The MIT License (MIT) + * + * Copyright (c) 2023-2024 Damien P. George + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ + +#include "py/objmodule.h" +#include "py/runtime.h" +#include "proxy_c.h" + +#if MICROPY_PY_JS + +/******************************************************************************/ +// js module + +void mp_module_js_attr(mp_obj_t self_in, qstr attr, mp_obj_t *dest) { + mp_obj_jsproxy_t global_this; + global_this.ref = 0; + mp_obj_jsproxy_attr(MP_OBJ_FROM_PTR(&global_this), attr, dest); +} + +static const mp_rom_map_elem_t mp_module_js_globals_table[] = { + { MP_ROM_QSTR(MP_QSTR___name__), MP_ROM_QSTR(MP_QSTR_js) }, +}; +static MP_DEFINE_CONST_DICT(mp_module_js_globals, mp_module_js_globals_table); + +const mp_obj_module_t mp_module_js = { + .base = { &mp_type_module }, + .globals = (mp_obj_dict_t *)&mp_module_js_globals, +}; + +MP_REGISTER_MODULE(MP_QSTR_js, mp_module_js); +MP_REGISTER_MODULE_DELEGATION(mp_module_js, mp_module_js_attr); + +#endif // MICROPY_PY_JS diff --git a/ports/webassembly/modjsffi.c b/ports/webassembly/modjsffi.c new file mode 100644 index 0000000000000..d4e61e368f009 --- /dev/null +++ b/ports/webassembly/modjsffi.c @@ -0,0 +1,80 @@ +/* + * This file is part of the MicroPython project, http://micropython.org/ + * + * The MIT License (MIT) + * + * Copyright (c) 2023-2024 Damien P. George + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ + +#include "emscripten.h" +#include "py/objmodule.h" +#include "py/runtime.h" +#include "proxy_c.h" + +#if MICROPY_PY_JSFFI + +/******************************************************************************/ +// jsffi module + +EM_JS(void, proxy_convert_mp_to_js_then_js_to_mp_obj_jsside, (uint32_t * out), { + const ret = proxy_convert_mp_to_js_obj_jsside(out); + proxy_convert_js_to_mp_obj_jsside_force_double_proxy(ret, out); +}); + +static mp_obj_t mp_jsffi_create_proxy(mp_obj_t arg) { + uint32_t out[3]; + proxy_convert_mp_to_js_obj_cside(arg, out); + proxy_convert_mp_to_js_then_js_to_mp_obj_jsside(out); + return proxy_convert_js_to_mp_obj_cside(out); +} +static MP_DEFINE_CONST_FUN_OBJ_1(mp_jsffi_create_proxy_obj, mp_jsffi_create_proxy); + +EM_JS(void, proxy_convert_mp_to_js_then_js_to_js_then_js_to_mp_obj_jsside, (uint32_t * out), { + const ret = proxy_convert_mp_to_js_obj_jsside(out); + const js_obj = PyProxy.toJs(ret); + proxy_convert_js_to_mp_obj_jsside(js_obj, out); +}); + +static mp_obj_t mp_jsffi_to_js(mp_obj_t arg) { + uint32_t out[3]; + proxy_convert_mp_to_js_obj_cside(arg, out); + proxy_convert_mp_to_js_then_js_to_js_then_js_to_mp_obj_jsside(out); + return proxy_convert_js_to_mp_obj_cside(out); +} +static MP_DEFINE_CONST_FUN_OBJ_1(mp_jsffi_to_js_obj, mp_jsffi_to_js); + +static const mp_rom_map_elem_t mp_module_jsffi_globals_table[] = { + { MP_ROM_QSTR(MP_QSTR___name__), MP_ROM_QSTR(MP_QSTR_jsffi) }, + + { MP_ROM_QSTR(MP_QSTR_JsProxy), MP_ROM_PTR(&mp_type_jsproxy) }, + { MP_ROM_QSTR(MP_QSTR_create_proxy), MP_ROM_PTR(&mp_jsffi_create_proxy_obj) }, + { MP_ROM_QSTR(MP_QSTR_to_js), MP_ROM_PTR(&mp_jsffi_to_js_obj) }, +}; +static MP_DEFINE_CONST_DICT(mp_module_jsffi_globals, mp_module_jsffi_globals_table); + +const mp_obj_module_t mp_module_jsffi = { + .base = { &mp_type_module }, + .globals = (mp_obj_dict_t *)&mp_module_jsffi_globals, +}; + +MP_REGISTER_MODULE(MP_QSTR_jsffi, mp_module_jsffi); + +#endif // MICROPY_PY_JSFFI diff --git a/ports/webassembly/mpconfigport.h b/ports/webassembly/mpconfigport.h index abfbbca7941f8..fc7ba2f82d33c 100644 --- a/ports/webassembly/mpconfigport.h +++ b/ports/webassembly/mpconfigport.h @@ -64,6 +64,14 @@ #define MICROPY_VFS_POSIX (MICROPY_VFS) #define MICROPY_PY_SYS_PLATFORM "webassembly" +#ifndef MICROPY_PY_JS +#define MICROPY_PY_JS (MICROPY_CONFIG_ROM_LEVEL_AT_LEAST_EXTRA_FEATURES) +#endif + +#ifndef MICROPY_PY_JSFFI +#define MICROPY_PY_JSFFI (MICROPY_CONFIG_ROM_LEVEL_AT_LEAST_EXTRA_FEATURES) +#endif + #define MICROPY_EVENT_POLL_HOOK \ do { \ extern void mp_handle_pending(bool); \ diff --git a/ports/webassembly/objjsproxy.c b/ports/webassembly/objjsproxy.c new file mode 100644 index 0000000000000..a28b791cf23d4 --- /dev/null +++ b/ports/webassembly/objjsproxy.c @@ -0,0 +1,330 @@ +/* + * This file is part of the MicroPython project, http://micropython.org/ + * + * The MIT License (MIT) + * + * Copyright (c) 2023-2024 Damien P. George + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ + +#include +#include + +#include "emscripten.h" +#include "py/objmodule.h" +#include "py/runtime.h" +#include "proxy_c.h" + +// *FORMAT-OFF* +EM_JS(bool, lookup_attr, (int jsref, const char *str, uint32_t * out), { + const base = proxy_js_ref[jsref]; + const attr = UTF8ToString(str); + if (attr in base) { + let value = base[attr]; + if (typeof value == "function") { + if (base !== globalThis) { + value = value.bind(base); + } + } + proxy_convert_js_to_mp_obj_jsside(value, out); + return true; + } else { + return false; + } +}); +// *FORMAT-ON* + +EM_JS(void, store_attr, (int jsref, const char *attr_ptr, uint32_t * value_ref), { + const attr = UTF8ToString(attr_ptr); + const value = proxy_convert_mp_to_js_obj_jsside(value_ref); + proxy_js_ref[jsref][attr] = value; +}); + +EM_JS(void, call0, (int f_ref, uint32_t * out), { + // Because of JavaScript "this" semantics, we must extract the target function + // to a variable before calling it, so "this" is bound to the correct value. + // + // In detail: + // In JavaScript, proxy_js_ref[f_ref] acts like a function call + // proxy_js_ref.at(f_ref), and "this" will be bound to proxy_js_ref if + // there is a chain of calls, such as proxy_js_ref.at(f_ref)(). + // But proxy_js_ref is not "this" in the context of the call, so we + // must extract the function to an independent variable and then call + // that variable, so that "this" is correct (it will be "undefined"). + + const f = proxy_js_ref[f_ref]; + const ret = f(); + proxy_convert_js_to_mp_obj_jsside(ret, out); +}); + +EM_JS(int, call1, (int f_ref, uint32_t * a0, uint32_t * out), { + const a0_js = proxy_convert_mp_to_js_obj_jsside(a0); + const f = proxy_js_ref[f_ref]; + const ret = f(a0_js); + proxy_convert_js_to_mp_obj_jsside(ret, out); +}); + +EM_JS(int, call2, (int f_ref, uint32_t * a0, uint32_t * a1, uint32_t * out), { + const a0_js = proxy_convert_mp_to_js_obj_jsside(a0); + const a1_js = proxy_convert_mp_to_js_obj_jsside(a1); + const f = proxy_js_ref[f_ref]; + const ret = f(a0_js, a1_js); + proxy_convert_js_to_mp_obj_jsside(ret, out); +}); + +EM_JS(int, calln, (int f_ref, uint32_t n_args, uint32_t * value, uint32_t * out), { + const f = proxy_js_ref[f_ref]; + const a = []; + for (let i = 0; i < n_args; ++i) { + const v = proxy_convert_mp_to_js_obj_jsside(value + i * 3 * 4); + a.push(v); + } + const ret = f(... a); + proxy_convert_js_to_mp_obj_jsside(ret, out); +}); + +EM_JS(void, call0_kwarg, (int f_ref, uint32_t n_kw, uint32_t * key, uint32_t * value, uint32_t * out), { + const f = proxy_js_ref[f_ref]; + const a = {}; + for (let i = 0; i < n_kw; ++i) { + const k = UTF8ToString(getValue(key + i * 4, "i32")); + const v = proxy_convert_mp_to_js_obj_jsside(value + i * 3 * 4); + a[k] = v; + } + const ret = f(a); + proxy_convert_js_to_mp_obj_jsside(ret, out); +}); + +EM_JS(void, call1_kwarg, (int f_ref, uint32_t * arg0, uint32_t n_kw, uint32_t * key, uint32_t * value, uint32_t * out), { + const f = proxy_js_ref[f_ref]; + const a0 = proxy_convert_mp_to_js_obj_jsside(arg0); + const a = {}; + for (let i = 0; i < n_kw; ++i) { + const k = UTF8ToString(getValue(key + i * 4, "i32")); + const v = proxy_convert_mp_to_js_obj_jsside(value + i * 3 * 4); + a[k] = v; + } + const ret = f(a0, a); + proxy_convert_js_to_mp_obj_jsside(ret, out); +}); + +EM_JS(void, js_reflect_construct, (int f_ref, uint32_t n_args, uint32_t * args, uint32_t * out), { + const f = proxy_js_ref[f_ref]; + const as = []; + for (let i = 0; i < n_args; ++i) { + as.push(proxy_convert_mp_to_js_obj_jsside(args + i * 4)); + } + const ret = Reflect.construct(f, as); + proxy_convert_js_to_mp_obj_jsside(ret, out); +}); + +EM_JS(int, js_get_len, (int f_ref), { + return proxy_js_ref[f_ref].length; +}); + +EM_JS(void, js_subscr_int, (int f_ref, int idx, uint32_t * out), { + const f = proxy_js_ref[f_ref]; + const ret = f[idx]; + proxy_convert_js_to_mp_obj_jsside(ret, out); +}); + +EM_JS(void, js_subscr_load, (int f_ref, uint32_t * index_ref, uint32_t * out), { + const target = proxy_js_ref[f_ref]; + const index = python_index_semantics(target, proxy_convert_mp_to_js_obj_jsside(index_ref)); + const ret = target[index]; + proxy_convert_js_to_mp_obj_jsside(ret, out); +}); + +EM_JS(void, js_subscr_store, (int f_ref, uint32_t * idx, uint32_t * value), { + const f = proxy_js_ref[f_ref]; + f[proxy_convert_mp_to_js_obj_jsside(idx)] = proxy_convert_mp_to_js_obj_jsside(value); +}); + +static void jsproxy_print(const mp_print_t *print, mp_obj_t self_in, mp_print_kind_t kind) { + mp_obj_jsproxy_t *self = MP_OBJ_TO_PTR(self_in); + mp_printf(print, "", self->ref); +} + +static mp_obj_t jsproxy_call(mp_obj_t self_in, size_t n_args, size_t n_kw, const mp_obj_t *args) { + mp_obj_jsproxy_t *self = MP_OBJ_TO_PTR(self_in); + + if (n_kw == 0) { + mp_arg_check_num(n_args, n_kw, 0, MP_OBJ_FUN_ARGS_MAX, false); + } else { + mp_arg_check_num(n_args, n_kw, 0, 1, true); + uint32_t key[n_kw]; + uint32_t value[PVN * n_kw]; + for (int i = 0; i < n_kw; ++i) { + key[i] = (uintptr_t)mp_obj_str_get_str(args[n_args + i * 2]); + proxy_convert_mp_to_js_obj_cside(args[n_args + i * 2 + 1], &value[i * PVN]); + } + uint32_t out[3]; + if (n_args == 0) { + call0_kwarg(self->ref, n_kw, key, value, out); + } else { + // n_args == 1 + uint32_t arg0[PVN]; + proxy_convert_mp_to_js_obj_cside(args[0], arg0); + call1_kwarg(self->ref, arg0, n_kw, key, value, out); + } + return proxy_convert_js_to_mp_obj_cside(out); + } + + if (n_args == 0) { + uint32_t out[3]; + call0(self->ref, out); + return proxy_convert_js_to_mp_obj_cside(out); + } else if (n_args == 1) { + uint32_t arg0[PVN]; + uint32_t out[PVN]; + proxy_convert_mp_to_js_obj_cside(args[0], arg0); + call1(self->ref, arg0, out); + return proxy_convert_js_to_mp_obj_cside(out); + } else if (n_args == 2) { + uint32_t arg0[PVN]; + proxy_convert_mp_to_js_obj_cside(args[0], arg0); + uint32_t arg1[PVN]; + proxy_convert_mp_to_js_obj_cside(args[1], arg1); + uint32_t out[3]; + call2(self->ref, arg0, arg1, out); + return proxy_convert_js_to_mp_obj_cside(out); + } else { + uint32_t value[PVN * n_args]; + for (int i = 0; i < n_args; ++i) { + proxy_convert_mp_to_js_obj_cside(args[i], &value[i * PVN]); + } + uint32_t out[3]; + calln(self->ref, n_args, value, out); + return proxy_convert_js_to_mp_obj_cside(out); + } +} + +static mp_obj_t jsproxy_reflect_construct(size_t n_args, const mp_obj_t *args) { + int arg0 = mp_obj_jsproxy_get_ref(args[0]); + n_args -= 1; + args += 1; + uint32_t args_conv[n_args]; + for (unsigned int i = 0; i < n_args; ++i) { + proxy_convert_mp_to_js_obj_cside(args[i], &args_conv[i * PVN]); + } + uint32_t out[3]; + js_reflect_construct(arg0, n_args, args_conv, out); + return proxy_convert_js_to_mp_obj_cside(out); +} +static MP_DEFINE_CONST_FUN_OBJ_VAR(jsproxy_reflect_construct_obj, 1, jsproxy_reflect_construct); + +static mp_obj_t jsproxy_subscr(mp_obj_t self_in, mp_obj_t index, mp_obj_t value) { + mp_obj_jsproxy_t *self = MP_OBJ_TO_PTR(self_in); + if (value == MP_OBJ_SENTINEL) { + // Load subscript. + uint32_t idx[PVN], out[PVN]; + proxy_convert_mp_to_js_obj_cside(index, idx); + js_subscr_load(self->ref, idx, out); + return proxy_convert_js_to_mp_obj_cside(out); + } else if (value == MP_OBJ_NULL) { + // Delete subscript. + return MP_OBJ_NULL; // not supported + } else { + // Store subscript. + uint32_t idx[PVN], val[PVN]; + proxy_convert_mp_to_js_obj_cside(index, idx); + proxy_convert_mp_to_js_obj_cside(value, val); + js_subscr_store(self->ref, idx, val); + return mp_const_none; + } +} + +void mp_obj_jsproxy_attr(mp_obj_t self_in, qstr attr, mp_obj_t *dest) { + mp_obj_jsproxy_t *self = MP_OBJ_TO_PTR(self_in); + if (dest[0] == MP_OBJ_NULL) { + // Load attribute. + uint32_t out[PVN]; + if (lookup_attr(self->ref, qstr_str(attr), out)) { + dest[0] = proxy_convert_js_to_mp_obj_cside(out); + } else if (attr == MP_QSTR_new) { + // Special case to handle construction of JS objects. + // JS objects don't have a ".new" attribute, doing "Obj.new" is a Pyodide idiom for "new Obj". + // It translates to the JavaScript "Reflect.construct(Obj, Array(...args))". + dest[0] = MP_OBJ_FROM_PTR(&jsproxy_reflect_construct_obj); + dest[1] = self_in; + } + } else if (dest[1] == MP_OBJ_NULL) { + // Delete attribute. + } else { + // Store attribute. + uint32_t value[PVN]; + proxy_convert_mp_to_js_obj_cside(dest[1], value); + store_attr(self->ref, qstr_str(attr), value); + dest[0] = MP_OBJ_NULL; + } +} + +/******************************************************************************/ +// jsproxy iterator + +typedef struct _jsproxy_it_t { + mp_obj_base_t base; + mp_fun_1_t iternext; + int ref; + uint16_t cur; + uint16_t len; +} jsproxy_it_t; + +static mp_obj_t jsproxy_it_iternext(mp_obj_t self_in) { + jsproxy_it_t *self = MP_OBJ_TO_PTR(self_in); + if (self->cur < self->len) { + uint32_t out[3]; + js_subscr_int(self->ref, self->cur, out); + self->cur += 1; + return proxy_convert_js_to_mp_obj_cside(out); + } else { + return MP_OBJ_STOP_ITERATION; + } +} + +static mp_obj_t jsproxy_getiter(mp_obj_t o_in, mp_obj_iter_buf_t *iter_buf) { + assert(sizeof(jsproxy_it_t) <= sizeof(mp_obj_iter_buf_t)); + jsproxy_it_t *o = (jsproxy_it_t *)iter_buf; + o->base.type = &mp_type_polymorph_iter; + o->iternext = jsproxy_it_iternext; + o->ref = mp_obj_jsproxy_get_ref(o_in); + o->cur = 0; + o->len = js_get_len(o->ref); + return MP_OBJ_FROM_PTR(o); +} + +/******************************************************************************/ + +MP_DEFINE_CONST_OBJ_TYPE( + mp_type_jsproxy, + MP_QSTR_JsProxy, + MP_TYPE_FLAG_ITER_IS_GETITER, + print, jsproxy_print, + call, jsproxy_call, + attr, mp_obj_jsproxy_attr, + subscr, jsproxy_subscr, + iter, jsproxy_getiter + ); + +mp_obj_t mp_obj_new_jsproxy(int ref) { + mp_obj_jsproxy_t *o = mp_obj_malloc(mp_obj_jsproxy_t, &mp_type_jsproxy); + o->ref = ref; + return MP_OBJ_FROM_PTR(o); +} diff --git a/ports/webassembly/objpyproxy.js b/ports/webassembly/objpyproxy.js new file mode 100644 index 0000000000000..52670b66e9542 --- /dev/null +++ b/ports/webassembly/objpyproxy.js @@ -0,0 +1,191 @@ +/* + * This file is part of the MicroPython project, http://micropython.org/ + * + * The MIT License (MIT) + * + * Copyright (c) 2023-2024 Damien P. George + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ + +class PyProxy { + constructor(ref) { + this._ref = ref; + } + + // Convert js_obj -- which is possibly a PyProxy -- to a JavaScript object. + static toJs(js_obj) { + if (!(js_obj instanceof PyProxy)) { + return js_obj; + } + + const type = Module.ccall( + "proxy_c_to_js_get_type", + "number", + ["number"], + [js_obj._ref], + ); + + if (type === 1 || type === 2) { + // List or tuple. + const array_ref = Module._malloc(2 * 4); + const item = Module._malloc(3 * 4); + Module.ccall( + "proxy_c_to_js_get_array", + "null", + ["number", "pointer"], + [js_obj._ref, array_ref], + ); + const len = Module.getValue(array_ref, "i32"); + const items_ptr = Module.getValue(array_ref + 4, "i32"); + const js_array = []; + for (let i = 0; i < len; ++i) { + Module.ccall( + "proxy_convert_mp_to_js_obj_cside", + "null", + ["pointer", "pointer"], + [Module.getValue(items_ptr + i * 4, "i32"), item], + ); + const js_item = proxy_convert_mp_to_js_obj_jsside(item); + js_array.push(PyProxy.toJs(js_item)); + } + Module._free(array_ref); + Module._free(item); + return js_array; + } + + if (type === 3) { + // Dict. + const map_ref = Module._malloc(2 * 4); + const item = Module._malloc(3 * 4); + Module.ccall( + "proxy_c_to_js_get_dict", + "null", + ["number", "pointer"], + [js_obj._ref, map_ref], + ); + const alloc = Module.getValue(map_ref, "i32"); + const table_ptr = Module.getValue(map_ref + 4, "i32"); + const js_dict = {}; + for (let i = 0; i < alloc; ++i) { + const mp_key = Module.getValue(table_ptr + i * 8, "i32"); + if (mp_key > 8) { + // Convert key to JS object. + Module.ccall( + "proxy_convert_mp_to_js_obj_cside", + "null", + ["pointer", "pointer"], + [mp_key, item], + ); + const js_key = proxy_convert_mp_to_js_obj_jsside(item); + + // Convert value to JS object. + const mp_value = Module.getValue( + table_ptr + i * 8 + 4, + "i32", + ); + Module.ccall( + "proxy_convert_mp_to_js_obj_cside", + "null", + ["pointer", "pointer"], + [mp_value, item], + ); + const js_value = proxy_convert_mp_to_js_obj_jsside(item); + + // Populate JS dict. + js_dict[js_key] = PyProxy.toJs(js_value); + } + } + Module._free(map_ref); + Module._free(item); + return js_dict; + } + + // Cannot convert to JS, leave as a PyProxy. + return js_obj; + } +} + +// This handler's goal is to allow minimal introspection +// of Python references from the JS world/utilities. +const py_proxy_handler = { + isExtensible() { + return true; + }, + ownKeys(target) { + const value = Module._malloc(3 * 4); + Module.ccall( + "proxy_c_to_js_dir", + "null", + ["number", "pointer"], + [target._ref, value], + ); + const dir = proxy_convert_mp_to_js_obj_jsside_with_free(value); + return PyProxy.toJs(dir).filter((attr) => !attr.startsWith("__")); + }, + getOwnPropertyDescriptor(target, prop) { + return { + value: target[prop], + enumerable: true, + writable: true, + configurable: true, + }; + }, + has(target, prop) { + return Module.ccall( + "proxy_c_to_js_has_attr", + "number", + ["number", "string"], + [target._ref, prop], + ); + }, + get(target, prop) { + if (prop === "_ref") { + return target._ref; + } + const value = Module._malloc(3 * 4); + Module.ccall( + "proxy_c_to_js_lookup_attr", + "number", + ["number", "string", "pointer"], + [target._ref, prop, value], + ); + return proxy_convert_mp_to_js_obj_jsside_with_free(value); + }, + set(target, prop, value) { + const value_conv = Module._malloc(3 * 4); + proxy_convert_js_to_mp_obj_jsside(value, value_conv); + const ret = Module.ccall( + "proxy_c_to_js_store_attr", + "number", + ["number", "string", "number"], + [target._ref, prop, value_conv], + ); + Module._free(value_conv); + return ret; + }, + deleteProperty(target, prop) { + return Module.ccall( + "proxy_c_to_js_delete_attr", + "number", + ["number", "string"], + [target._ref, prop], + ); + }, +}; diff --git a/ports/webassembly/proxy_c.c b/ports/webassembly/proxy_c.c new file mode 100644 index 0000000000000..809dd44ddefd1 --- /dev/null +++ b/ports/webassembly/proxy_c.c @@ -0,0 +1,281 @@ +/* + * This file is part of the MicroPython project, http://micropython.org/ + * + * The MIT License (MIT) + * + * Copyright (c) 2023-2024 Damien P. George + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ + +#include +#include + +#include "py/builtin.h" +#include "py/runtime.h" +#include "proxy_c.h" + +// These constants should match the constants in proxy_js.js. + +enum { + PROXY_KIND_MP_EXCEPTION = -1, + PROXY_KIND_MP_NULL = 0, + PROXY_KIND_MP_NONE = 1, + PROXY_KIND_MP_BOOL = 2, + PROXY_KIND_MP_INT = 3, + PROXY_KIND_MP_FLOAT = 4, + PROXY_KIND_MP_STR = 5, + PROXY_KIND_MP_CALLABLE = 6, + PROXY_KIND_MP_OBJECT = 7, + PROXY_KIND_MP_JSPROXY = 8, +}; + +enum { + PROXY_KIND_JS_NULL = 1, + PROXY_KIND_JS_BOOLEAN = 2, + PROXY_KIND_JS_INTEGER = 3, + PROXY_KIND_JS_DOUBLE = 4, + PROXY_KIND_JS_STRING = 5, + PROXY_KIND_JS_OBJECT = 6, + PROXY_KIND_JS_PYPROXY = 7, +}; + +void proxy_c_init(void) { + MP_STATE_PORT(proxy_c_ref) = mp_obj_new_list(0, NULL); + mp_obj_list_append(MP_STATE_PORT(proxy_c_ref), MP_OBJ_NULL); +} + +MP_REGISTER_ROOT_POINTER(mp_obj_t proxy_c_ref); + +static inline mp_obj_t proxy_c_get_obj(uint32_t c_ref) { + return ((mp_obj_list_t *)MP_OBJ_TO_PTR(MP_STATE_PORT(proxy_c_ref)))->items[c_ref]; +} + +mp_obj_t proxy_convert_js_to_mp_obj_cside(uint32_t *value) { + if (value[0] == PROXY_KIND_JS_NULL) { + return mp_const_none; + } else if (value[0] == PROXY_KIND_JS_BOOLEAN) { + return mp_obj_new_bool(value[1]); + } else if (value[0] == PROXY_KIND_JS_INTEGER) { + return mp_obj_new_int(value[1]); + } else if (value[0] == PROXY_KIND_JS_DOUBLE) { + return mp_obj_new_float_from_d(*(double *)&value[1]); + } else if (value[0] == PROXY_KIND_JS_STRING) { + mp_obj_t s = mp_obj_new_str((void *)value[2], value[1]); + free((void *)value[2]); + return s; + } else if (value[0] == PROXY_KIND_JS_PYPROXY) { + return proxy_c_get_obj(value[1]); + } else { + // PROXY_KIND_JS_OBJECT + return mp_obj_new_jsproxy(value[1]); + } +} + +void proxy_convert_mp_to_js_obj_cside(mp_obj_t obj, uint32_t *out) { + uint32_t kind; + if (obj == MP_OBJ_NULL) { + kind = PROXY_KIND_MP_NULL; + } else if (obj == mp_const_none) { + kind = PROXY_KIND_MP_NONE; + } else if (mp_obj_is_bool(obj)) { + kind = PROXY_KIND_MP_BOOL; + out[1] = mp_obj_is_true(obj); + } else if (mp_obj_is_int(obj)) { + kind = PROXY_KIND_MP_INT; + out[1] = mp_obj_get_int_truncated(obj); // TODO support big int + } else if (mp_obj_is_float(obj)) { + kind = PROXY_KIND_MP_FLOAT; + *(double *)&out[1] = mp_obj_get_float(obj); + } else if (mp_obj_is_str(obj)) { + kind = PROXY_KIND_MP_STR; + size_t len; + const char *str = mp_obj_str_get_data(obj, &len); + out[1] = len; + out[2] = (uintptr_t)str; + } else if (mp_obj_is_jsproxy(obj)) { + kind = PROXY_KIND_MP_JSPROXY; + out[1] = mp_obj_jsproxy_get_ref(obj); + } else { + if (mp_obj_is_callable(obj)) { + kind = PROXY_KIND_MP_CALLABLE; + } else { + kind = PROXY_KIND_MP_OBJECT; + } + size_t id = ((mp_obj_list_t *)MP_OBJ_TO_PTR(MP_STATE_PORT(proxy_c_ref)))->len; + mp_obj_list_append(MP_STATE_PORT(proxy_c_ref), obj); + out[1] = id; + } + out[0] = kind; +} + +void proxy_convert_mp_to_js_exc_cside(void *exc, uint32_t *out) { + out[0] = PROXY_KIND_MP_EXCEPTION; + vstr_t vstr; + mp_print_t print; + vstr_init_print(&vstr, 64, &print); + vstr_add_str(&vstr, qstr_str(mp_obj_get_type(MP_OBJ_FROM_PTR(exc))->name)); + vstr_add_char(&vstr, '\x04'); + mp_obj_print_exception(&print, MP_OBJ_FROM_PTR(exc)); + char *s = malloc(vstr_len(&vstr) + 1); + memcpy(s, vstr_str(&vstr), vstr_len(&vstr)); + out[1] = vstr_len(&vstr); + out[2] = (uintptr_t)s; + vstr_clear(&vstr); +} + +void proxy_c_to_js_call(uint32_t c_ref, uint32_t n_args, uint32_t *args_value, uint32_t *out) { + nlr_buf_t nlr; + if (nlr_push(&nlr) == 0) { + mp_obj_t args[4] = { mp_const_none, mp_const_none, mp_const_none, mp_const_none }; + for (size_t i = 0; i < n_args; ++i) { + args[i] = proxy_convert_js_to_mp_obj_cside(args_value + i * 3); + } + mp_obj_t obj = proxy_c_get_obj(c_ref); + mp_obj_t member = mp_call_function_n_kw(obj, n_args, 0, args); + nlr_pop(); + proxy_convert_mp_to_js_obj_cside(member, out); + } else { + // uncaught exception + proxy_convert_mp_to_js_exc_cside(nlr.ret_val, out); + } +} + +void proxy_c_to_js_dir(uint32_t c_ref, uint32_t *out) { + nlr_buf_t nlr; + if (nlr_push(&nlr) == 0) { + mp_obj_t obj = proxy_c_get_obj(c_ref); + mp_obj_t dir; + if (mp_obj_is_dict_or_ordereddict(obj)) { + mp_map_t *map = mp_obj_dict_get_map(obj); + dir = mp_obj_new_list(0, NULL); + for (size_t i = 0; i < map->alloc; i++) { + if (mp_map_slot_is_filled(map, i)) { + mp_obj_list_append(dir, map->table[i].key); + } + } + } else { + mp_obj_t args[1] = { obj }; + dir = mp_builtin_dir_obj.fun.var(1, args); + } + nlr_pop(); + return proxy_convert_mp_to_js_obj_cside(dir, out); + } else { + // uncaught exception + return proxy_convert_mp_to_js_exc_cside(nlr.ret_val, out); + } +} + +bool proxy_c_to_js_has_attr(uint32_t c_ref, const char *attr_in) { + mp_obj_t obj = proxy_c_get_obj(c_ref); + qstr attr = qstr_from_str(attr_in); + if (mp_obj_is_dict_or_ordereddict(obj)) { + mp_map_t *map = mp_obj_dict_get_map(obj); + mp_map_elem_t *elem = mp_map_lookup(map, MP_OBJ_NEW_QSTR(attr), MP_MAP_LOOKUP); + return elem != NULL; + } else { + mp_obj_t dest[2]; + mp_load_method_protected(obj, attr, dest, true); + if (dest[0] != MP_OBJ_NULL) { + return true; + } + } + return false; +} + +void proxy_c_to_js_lookup_attr(uint32_t c_ref, const char *attr_in, uint32_t *out) { + nlr_buf_t nlr; + if (nlr_push(&nlr) == 0) { + mp_obj_t obj = proxy_c_get_obj(c_ref); + qstr attr = qstr_from_str(attr_in); + mp_obj_t member; + if (mp_obj_is_dict_or_ordereddict(obj)) { + member = mp_obj_dict_get(obj, MP_OBJ_NEW_QSTR(attr)); + } else { + member = mp_load_attr(obj, attr); + } + nlr_pop(); + return proxy_convert_mp_to_js_obj_cside(member, out); + } else { + // uncaught exception + return proxy_convert_mp_to_js_exc_cside(nlr.ret_val, out); + } +} + +static bool proxy_c_to_js_store_helper(uint32_t c_ref, const char *attr_in, mp_obj_t value) { + mp_obj_t obj = proxy_c_get_obj(c_ref); + qstr attr = qstr_from_str(attr_in); + + nlr_buf_t nlr; + if (nlr_push(&nlr) == 0) { + if (mp_obj_is_dict_or_ordereddict(obj)) { + if (value == MP_OBJ_NULL) { + mp_obj_dict_delete(obj, MP_OBJ_NEW_QSTR(attr)); + } else { + mp_obj_dict_store(obj, MP_OBJ_NEW_QSTR(attr), value); + } + } else { + mp_store_attr(obj, attr, value); + } + nlr_pop(); + return true; + } else { + // uncaught exception + return false; + } +} + +bool proxy_c_to_js_store_attr(uint32_t c_ref, const char *attr_in, uint32_t *value_in) { + mp_obj_t value = proxy_convert_js_to_mp_obj_cside(value_in); + return proxy_c_to_js_store_helper(c_ref, attr_in, value); +} + +bool proxy_c_to_js_delete_attr(uint32_t c_ref, const char *attr_in) { + return proxy_c_to_js_store_helper(c_ref, attr_in, MP_OBJ_NULL); +} + +uint32_t proxy_c_to_js_get_type(uint32_t c_ref) { + mp_obj_t obj = proxy_c_get_obj(c_ref); + const mp_obj_type_t *type = mp_obj_get_type(obj); + if (type == &mp_type_tuple) { + return 1; + } else if (type == &mp_type_list) { + return 2; + } else if (type == &mp_type_dict) { + return 3; + } else { + return 4; + } +} + +void proxy_c_to_js_get_array(uint32_t c_ref, uint32_t *out) { + mp_obj_t obj = proxy_c_get_obj(c_ref); + size_t len; + mp_obj_t *items; + mp_obj_get_array(obj, &len, &items); + out[0] = len; + out[1] = (uintptr_t)items; +} + +void proxy_c_to_js_get_dict(uint32_t c_ref, uint32_t *out) { + mp_obj_t obj = proxy_c_get_obj(c_ref); + mp_map_t *map = mp_obj_dict_get_map(obj); + out[0] = map->alloc; + out[1] = (uintptr_t)map->table; +} diff --git a/ports/webassembly/proxy_c.h b/ports/webassembly/proxy_c.h new file mode 100644 index 0000000000000..3e68d2504958c --- /dev/null +++ b/ports/webassembly/proxy_c.h @@ -0,0 +1,58 @@ +/* + * This file is part of the MicroPython project, http://micropython.org/ + * + * The MIT License (MIT) + * + * Copyright (c) 2023-2024 Damien P. George + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ +#ifndef MICROPY_INCLUDED_WEBASSEMBLY_PROXY_C_H +#define MICROPY_INCLUDED_WEBASSEMBLY_PROXY_C_H + +#include "py/obj.h" + +// proxy value number of items +#define PVN (3) + +typedef struct _mp_obj_jsproxy_t { + mp_obj_base_t base; + int ref; +} mp_obj_jsproxy_t; + +extern const mp_obj_type_t mp_type_jsproxy; + +void proxy_c_init(void); +mp_obj_t proxy_convert_js_to_mp_obj_cside(uint32_t *value); +void proxy_convert_mp_to_js_obj_cside(mp_obj_t obj, uint32_t *out); +void proxy_convert_mp_to_js_exc_cside(void *exc, uint32_t *out); + +mp_obj_t mp_obj_new_jsproxy(int ref); +void mp_obj_jsproxy_attr(mp_obj_t self_in, qstr attr, mp_obj_t *dest); + +static inline bool mp_obj_is_jsproxy(mp_obj_t o) { + return mp_obj_get_type(o) == &mp_type_jsproxy; +} + +static inline int mp_obj_jsproxy_get_ref(mp_obj_t o) { + mp_obj_jsproxy_t *self = MP_OBJ_TO_PTR(o); + return self->ref; +} + +#endif // MICROPY_INCLUDED_WEBASSEMBLY_PROXY_C_H diff --git a/ports/webassembly/proxy_js.js b/ports/webassembly/proxy_js.js new file mode 100644 index 0000000000000..1835bdfdfadaa --- /dev/null +++ b/ports/webassembly/proxy_js.js @@ -0,0 +1,222 @@ +/* + * This file is part of the MicroPython project, http://micropython.org/ + * + * The MIT License (MIT) + * + * Copyright (c) 2023-2024 Damien P. George + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ + +// These constants should match the constants in proxy_c.c. + +const PROXY_KIND_MP_EXCEPTION = -1; +const PROXY_KIND_MP_NULL = 0; +const PROXY_KIND_MP_NONE = 1; +const PROXY_KIND_MP_BOOL = 2; +const PROXY_KIND_MP_INT = 3; +const PROXY_KIND_MP_FLOAT = 4; +const PROXY_KIND_MP_STR = 5; +const PROXY_KIND_MP_CALLABLE = 6; +const PROXY_KIND_MP_OBJECT = 7; +const PROXY_KIND_MP_JSPROXY = 8; + +const PROXY_KIND_JS_NULL = 1; +const PROXY_KIND_JS_BOOLEAN = 2; +const PROXY_KIND_JS_INTEGER = 3; +const PROXY_KIND_JS_DOUBLE = 4; +const PROXY_KIND_JS_STRING = 5; +const PROXY_KIND_JS_OBJECT = 6; +const PROXY_KIND_JS_PYPROXY = 7; + +class PythonError extends Error { + constructor(exc_type, exc_details) { + super(exc_details); + this.name = "PythonError"; + this.type = exc_type; + } +} + +function proxy_js_init() { + globalThis.proxy_js_ref = [globalThis]; +} + +function proxy_call_python(target, argumentsList) { + let args = 0; + + // Strip trailing "undefined" arguments. + while ( + argumentsList.length > 0 && + argumentsList[argumentsList.length - 1] === undefined + ) { + argumentsList.pop(); + } + + if (argumentsList.length > 0) { + // TODO use stackAlloc/stackRestore? + args = Module._malloc(argumentsList.length * 3 * 4); + for (const i in argumentsList) { + proxy_convert_js_to_mp_obj_jsside( + argumentsList[i], + args + i * 3 * 4, + ); + } + } + const value = Module._malloc(3 * 4); + Module.ccall( + "proxy_c_to_js_call", + "null", + ["number", "number", "number", "pointer"], + [target, argumentsList.length, args, value], + ); + if (argumentsList.length > 0) { + Module._free(args); + } + return proxy_convert_mp_to_js_obj_jsside_with_free(value); +} + +function proxy_convert_js_to_mp_obj_jsside(js_obj, out) { + let kind; + if (js_obj === null) { + kind = PROXY_KIND_JS_NULL; + } else if (typeof js_obj === "boolean") { + kind = PROXY_KIND_JS_BOOLEAN; + Module.setValue(out + 4, js_obj, "i32"); + } else if (typeof js_obj === "number") { + if (Number.isInteger(js_obj)) { + kind = PROXY_KIND_JS_INTEGER; + Module.setValue(out + 4, js_obj, "i32"); + } else { + kind = PROXY_KIND_JS_DOUBLE; + // double must be stored to an address that's a multiple of 8 + const temp = (out + 4) & ~7; + Module.setValue(temp, js_obj, "double"); + const double_lo = Module.getValue(temp, "i32"); + const double_hi = Module.getValue(temp + 4, "i32"); + Module.setValue(out + 4, double_lo, "i32"); + Module.setValue(out + 8, double_hi, "i32"); + } + } else if (typeof js_obj === "string") { + kind = PROXY_KIND_JS_STRING; + const len = Module.lengthBytesUTF8(js_obj); + const buf = Module._malloc(len + 1); + Module.stringToUTF8(js_obj, buf, len + 1); + Module.setValue(out + 4, len, "i32"); + Module.setValue(out + 8, buf, "i32"); + } else if (js_obj instanceof PyProxy) { + kind = PROXY_KIND_JS_PYPROXY; + Module.setValue(out + 4, js_obj._ref, "i32"); + } else { + kind = PROXY_KIND_JS_OBJECT; + const id = proxy_js_ref.length; + proxy_js_ref[id] = js_obj; + Module.setValue(out + 4, id, "i32"); + } + Module.setValue(out + 0, kind, "i32"); +} + +function proxy_convert_js_to_mp_obj_jsside_force_double_proxy(js_obj, out) { + if (js_obj instanceof PyProxy) { + const kind = PROXY_KIND_JS_OBJECT; + const id = proxy_js_ref.length; + proxy_js_ref[id] = js_obj; + Module.setValue(out + 4, id, "i32"); + Module.setValue(out + 0, kind, "i32"); + } else { + proxy_convert_js_to_mp_obj_jsside(js_obj, out); + } +} + +function proxy_convert_mp_to_js_obj_jsside(value) { + const kind = Module.getValue(value, "i32"); + let obj; + if (kind === PROXY_KIND_MP_EXCEPTION) { + // Exception + const str_len = Module.getValue(value + 4, "i32"); + const str_ptr = Module.getValue(value + 8, "i32"); + const str = Module.UTF8ToString(str_ptr, str_len); + Module._free(str_ptr); + const str_split = str.split("\x04"); + throw new PythonError(str_split[0], str_split[1]); + } + if (kind === PROXY_KIND_MP_NULL) { + // MP_OBJ_NULL + throw new Error("NULL object"); + } + if (kind === PROXY_KIND_MP_NONE) { + // None + obj = null; + } else if (kind === PROXY_KIND_MP_BOOL) { + // bool + obj = Module.getValue(value + 4, "i32") ? true : false; + } else if (kind === PROXY_KIND_MP_INT) { + // int + obj = Module.getValue(value + 4, "i32"); + } else if (kind === PROXY_KIND_MP_FLOAT) { + // float + // double must be loaded from an address that's a multiple of 8 + const temp = (value + 4) & ~7; + const double_lo = Module.getValue(value + 4, "i32"); + const double_hi = Module.getValue(value + 8, "i32"); + Module.setValue(temp, double_lo, "i32"); + Module.setValue(temp + 4, double_hi, "i32"); + obj = Module.getValue(temp, "double"); + } else if (kind === PROXY_KIND_MP_STR) { + // str + const str_len = Module.getValue(value + 4, "i32"); + const str_ptr = Module.getValue(value + 8, "i32"); + obj = Module.UTF8ToString(str_ptr, str_len); + } else if (kind === PROXY_KIND_MP_JSPROXY) { + // js proxy + const id = Module.getValue(value + 4, "i32"); + obj = proxy_js_ref[id]; + } else { + // obj + const id = Module.getValue(value + 4, "i32"); + if (kind === PROXY_KIND_MP_CALLABLE) { + obj = (...args) => { + return proxy_call_python(id, args); + }; + } else { + // PROXY_KIND_MP_OBJECT + const target = new PyProxy(id); + obj = new Proxy(target, py_proxy_handler); + } + } + return obj; +} + +function proxy_convert_mp_to_js_obj_jsside_with_free(value) { + const ret = proxy_convert_mp_to_js_obj_jsside(value); + Module._free(value); + return ret; +} + +function python_index_semantics(target, index_in) { + let index = index_in; + if (typeof index === "number") { + if (index < 0) { + index += target.length; + } + if (index < 0 || index >= target.length) { + throw new PythonError("IndexError", "index out of range"); + } + } + return index; +} From 9b090603a04e69ec710e14f6c8bd93011516c5a1 Mon Sep 17 00:00:00 2001 From: Damien George Date: Sat, 24 Jun 2023 17:19:05 +1000 Subject: [PATCH 10/19] webassembly: Implement runPythonAsync() for top-level async code. With this commit, `interpreter.runPythonAsync(code)` can now be used to run Python code that uses `await` at the top level. That will yield up to JavaScript and produce a thenable, which the JavaScript runtime can then resume. Also implemented is the ability for Python code to await on JavaScript promises/thenables. For example, outer JavaScript code can await on `runPythonAsync(code)` which then runs Python code that does `await js.fetch(url)`. The entire chain of calls will be suspended until the fetch completes. Signed-off-by: Damien George --- ports/webassembly/Makefile | 2 + ports/webassembly/api.js | 10 ++ ports/webassembly/main.c | 6 ++ ports/webassembly/mpconfigport.h | 1 + ports/webassembly/objjsproxy.c | 163 ++++++++++++++++++++++++++++++- ports/webassembly/objpyproxy.js | 23 +++++ ports/webassembly/proxy_c.c | 83 +++++++++++++++- ports/webassembly/proxy_js.js | 10 +- 8 files changed, 291 insertions(+), 7 deletions(-) diff --git a/ports/webassembly/Makefile b/ports/webassembly/Makefile index 2a5669392eb41..05e0c1f0f9085 100644 --- a/ports/webassembly/Makefile +++ b/ports/webassembly/Makefile @@ -47,6 +47,7 @@ CFLAGS += $(INC) EXPORTED_FUNCTIONS_EXTRA += ,\ _mp_js_do_exec,\ + _mp_js_do_exec_async,\ _mp_js_do_import,\ _mp_js_register_js_module,\ _proxy_c_init,\ @@ -58,6 +59,7 @@ EXPORTED_FUNCTIONS_EXTRA += ,\ _proxy_c_to_js_get_type,\ _proxy_c_to_js_has_attr,\ _proxy_c_to_js_lookup_attr,\ + _proxy_c_to_js_resume,\ _proxy_c_to_js_store_attr,\ _proxy_convert_mp_to_js_obj_cside diff --git a/ports/webassembly/api.js b/ports/webassembly/api.js index dfe7561768a4f..ec0601c612ade 100644 --- a/ports/webassembly/api.js +++ b/ports/webassembly/api.js @@ -140,6 +140,16 @@ export async function loadMicroPython(options) { ); return proxy_convert_mp_to_js_obj_jsside_with_free(value); }, + runPythonAsync(code) { + const value = Module._malloc(3 * 4); + Module.ccall( + "mp_js_do_exec_async", + "number", + ["string", "pointer"], + [code, value], + ); + return proxy_convert_mp_to_js_obj_jsside_with_free(value); + }, }; } diff --git a/ports/webassembly/main.c b/ports/webassembly/main.c index c1c7a88840ec3..5bb4222aaa726 100644 --- a/ports/webassembly/main.c +++ b/ports/webassembly/main.c @@ -169,6 +169,12 @@ void mp_js_do_exec(const char *src, uint32_t *out) { } } +void mp_js_do_exec_async(const char *src, uint32_t *out) { + mp_compile_allow_top_level_await = true; + mp_js_do_exec(src, out); + mp_compile_allow_top_level_await = false; +} + #if MICROPY_GC_SPLIT_HEAP_AUTO // The largest new region that is available to become Python heap. diff --git a/ports/webassembly/mpconfigport.h b/ports/webassembly/mpconfigport.h index fc7ba2f82d33c..ae5dfa6fa50b8 100644 --- a/ports/webassembly/mpconfigport.h +++ b/ports/webassembly/mpconfigport.h @@ -39,6 +39,7 @@ #endif #define MICROPY_ALLOC_PATH_MAX (256) +#define MICROPY_COMP_ALLOW_TOP_LEVEL_AWAIT (1) #define MICROPY_READER_VFS (MICROPY_VFS) #define MICROPY_ENABLE_GC (1) #define MICROPY_ENABLE_PYSTACK (1) diff --git a/ports/webassembly/objjsproxy.c b/ports/webassembly/objjsproxy.c index a28b791cf23d4..5e2aeb6a36e4b 100644 --- a/ports/webassembly/objjsproxy.c +++ b/ports/webassembly/objjsproxy.c @@ -32,6 +32,16 @@ #include "py/runtime.h" #include "proxy_c.h" +EM_JS(bool, has_attr, (int jsref, const char *str), { + const base = proxy_js_ref[jsref]; + const attr = UTF8ToString(str); + if (attr in base) { + return true; + } else { + return false; + } +}); + // *FORMAT-OFF* EM_JS(bool, lookup_attr, (int jsref, const char *str, uint32_t * out), { const base = proxy_js_ref[jsref]; @@ -299,19 +309,166 @@ static mp_obj_t jsproxy_it_iternext(mp_obj_t self_in) { } } -static mp_obj_t jsproxy_getiter(mp_obj_t o_in, mp_obj_iter_buf_t *iter_buf) { +static mp_obj_t jsproxy_new_it(mp_obj_t self_in, mp_obj_iter_buf_t *iter_buf) { assert(sizeof(jsproxy_it_t) <= sizeof(mp_obj_iter_buf_t)); + mp_obj_jsproxy_t *self = MP_OBJ_TO_PTR(self_in); jsproxy_it_t *o = (jsproxy_it_t *)iter_buf; o->base.type = &mp_type_polymorph_iter; o->iternext = jsproxy_it_iternext; - o->ref = mp_obj_jsproxy_get_ref(o_in); + o->ref = self->ref; o->cur = 0; - o->len = js_get_len(o->ref); + o->len = js_get_len(self->ref); + return MP_OBJ_FROM_PTR(o); +} + +/******************************************************************************/ +// jsproxy generator + +enum { + JSOBJ_GEN_STATE_WAITING, + JSOBJ_GEN_STATE_COMPLETED, + JSOBJ_GEN_STATE_EXHAUSTED, +}; + +typedef struct _jsproxy_gen_t { + mp_obj_base_t base; + mp_obj_t thenable; + int state; +} jsproxy_gen_t; + +mp_vm_return_kind_t jsproxy_gen_resume(mp_obj_t self_in, mp_obj_t send_value, mp_obj_t throw_value, mp_obj_t *ret_val) { + jsproxy_gen_t *self = MP_OBJ_TO_PTR(self_in); + switch (self->state) { + case JSOBJ_GEN_STATE_WAITING: + self->state = JSOBJ_GEN_STATE_COMPLETED; + *ret_val = self->thenable; + return MP_VM_RETURN_YIELD; + + case JSOBJ_GEN_STATE_COMPLETED: + self->state = JSOBJ_GEN_STATE_EXHAUSTED; + *ret_val = send_value; + return MP_VM_RETURN_NORMAL; + + case JSOBJ_GEN_STATE_EXHAUSTED: + default: + // Trying to resume an already stopped generator. + // This is an optimised "raise StopIteration(None)". + *ret_val = mp_const_none; + return MP_VM_RETURN_NORMAL; + } +} + +static mp_obj_t jsproxy_gen_resume_and_raise(mp_obj_t self_in, mp_obj_t send_value, mp_obj_t throw_value, bool raise_stop_iteration) { + mp_obj_t ret; + switch (jsproxy_gen_resume(self_in, send_value, throw_value, &ret)) { + case MP_VM_RETURN_NORMAL: + default: + // A normal return is a StopIteration, either raise it or return + // MP_OBJ_STOP_ITERATION as an optimisation. + if (ret == mp_const_none) { + ret = MP_OBJ_NULL; + } + if (raise_stop_iteration) { + mp_raise_StopIteration(ret); + } else { + return mp_make_stop_iteration(ret); + } + + case MP_VM_RETURN_YIELD: + return ret; + + case MP_VM_RETURN_EXCEPTION: + nlr_raise(ret); + } +} + +static mp_obj_t jsproxy_gen_instance_iternext(mp_obj_t self_in) { + return jsproxy_gen_resume_and_raise(self_in, mp_const_none, MP_OBJ_NULL, false); +} + +static mp_obj_t jsproxy_gen_instance_send(mp_obj_t self_in, mp_obj_t send_value) { + return jsproxy_gen_resume_and_raise(self_in, send_value, MP_OBJ_NULL, true); +} +static MP_DEFINE_CONST_FUN_OBJ_2(jsproxy_gen_instance_send_obj, jsproxy_gen_instance_send); + +static mp_obj_t jsproxy_gen_instance_throw(size_t n_args, const mp_obj_t *args) { + // The signature of this function is: throw(type[, value[, traceback]]) + // CPython will pass all given arguments through the call chain and process them + // at the point they are used (native generators will handle them differently to + // user-defined generators with a throw() method). To save passing multiple + // values, MicroPython instead does partial processing here to reduce it down to + // one argument and passes that through: + // - if only args[1] is given, or args[2] is given but is None, args[1] is + // passed through (in the standard case it is an exception class or instance) + // - if args[2] is given and not None it is passed through (in the standard + // case it would be an exception instance and args[1] its corresponding class) + // - args[3] is always ignored + + mp_obj_t exc = args[1]; + if (n_args > 2 && args[2] != mp_const_none) { + exc = args[2]; + } + + return jsproxy_gen_resume_and_raise(args[0], mp_const_none, exc, true); +} +static MP_DEFINE_CONST_FUN_OBJ_VAR_BETWEEN(jsproxy_gen_instance_throw_obj, 2, 4, jsproxy_gen_instance_throw); + +static mp_obj_t jsproxy_gen_instance_close(mp_obj_t self_in) { + mp_obj_t ret; + switch (jsproxy_gen_resume(self_in, mp_const_none, MP_OBJ_FROM_PTR(&mp_const_GeneratorExit_obj), &ret)) { + case MP_VM_RETURN_YIELD: + mp_raise_msg(&mp_type_RuntimeError, MP_ERROR_TEXT("generator ignored GeneratorExit")); + + // Swallow GeneratorExit (== successful close), and re-raise any other + case MP_VM_RETURN_EXCEPTION: + // ret should always be an instance of an exception class + if (mp_obj_is_subclass_fast(MP_OBJ_FROM_PTR(mp_obj_get_type(ret)), MP_OBJ_FROM_PTR(&mp_type_GeneratorExit))) { + return mp_const_none; + } + nlr_raise(ret); + + default: + // The only choice left is MP_VM_RETURN_NORMAL which is successful close + return mp_const_none; + } +} +static MP_DEFINE_CONST_FUN_OBJ_1(jsproxy_gen_instance_close_obj, jsproxy_gen_instance_close); + +static const mp_rom_map_elem_t jsproxy_gen_instance_locals_dict_table[] = { + { MP_ROM_QSTR(MP_QSTR_close), MP_ROM_PTR(&jsproxy_gen_instance_close_obj) }, + { MP_ROM_QSTR(MP_QSTR_send), MP_ROM_PTR(&jsproxy_gen_instance_send_obj) }, + { MP_ROM_QSTR(MP_QSTR_throw), MP_ROM_PTR(&jsproxy_gen_instance_throw_obj) }, +}; +static MP_DEFINE_CONST_DICT(jsproxy_gen_instance_locals_dict, jsproxy_gen_instance_locals_dict_table); + +MP_DEFINE_CONST_OBJ_TYPE( + mp_type_jsproxy_gen, + MP_QSTR_generator, + MP_TYPE_FLAG_ITER_IS_ITERNEXT, + iter, jsproxy_gen_instance_iternext, + locals_dict, &jsproxy_gen_instance_locals_dict + ); + +static mp_obj_t jsproxy_new_gen(mp_obj_t self_in, mp_obj_iter_buf_t *iter_buf) { + assert(sizeof(jsproxy_gen_t) <= sizeof(mp_obj_iter_buf_t)); + jsproxy_gen_t *o = (jsproxy_gen_t *)iter_buf; + o->base.type = &mp_type_jsproxy_gen; + o->thenable = self_in; + o->state = JSOBJ_GEN_STATE_WAITING; return MP_OBJ_FROM_PTR(o); } /******************************************************************************/ +static mp_obj_t jsproxy_getiter(mp_obj_t self_in, mp_obj_iter_buf_t *iter_buf) { + mp_obj_jsproxy_t *self = MP_OBJ_TO_PTR(self_in); + if (has_attr(self->ref, "then")) { + return jsproxy_new_gen(self_in, iter_buf); + } else { + return jsproxy_new_it(self_in, iter_buf); + } +} + MP_DEFINE_CONST_OBJ_TYPE( mp_type_jsproxy, MP_QSTR_JsProxy, diff --git a/ports/webassembly/objpyproxy.js b/ports/webassembly/objpyproxy.js index 52670b66e9542..9ba06283ea2b0 100644 --- a/ports/webassembly/objpyproxy.js +++ b/ports/webassembly/objpyproxy.js @@ -159,6 +159,9 @@ const py_proxy_handler = { if (prop === "_ref") { return target._ref; } + if (prop === "then") { + return null; + } const value = Module._malloc(3 * 4); Module.ccall( "proxy_c_to_js_lookup_attr", @@ -189,3 +192,23 @@ const py_proxy_handler = { ); }, }; + +// PyProxy of a Python generator, that implements the thenable interface. +class PyProxyThenable { + constructor(ref) { + this._ref = ref; + } + + then(resolve, reject) { + const values = Module._malloc(3 * 3 * 4); + proxy_convert_js_to_mp_obj_jsside(resolve, values + 3 * 4); + proxy_convert_js_to_mp_obj_jsside(reject, values + 2 * 3 * 4); + Module.ccall( + "proxy_c_to_js_resume", + "null", + ["number", "pointer"], + [this._ref, values], + ); + return proxy_convert_mp_to_js_obj_jsside_with_free(values); + } +} diff --git a/ports/webassembly/proxy_c.c b/ports/webassembly/proxy_c.c index 809dd44ddefd1..1e4573ce0ba55 100644 --- a/ports/webassembly/proxy_c.c +++ b/ports/webassembly/proxy_c.c @@ -27,6 +27,7 @@ #include #include +#include "emscripten.h" #include "py/builtin.h" #include "py/runtime.h" #include "proxy_c.h" @@ -42,8 +43,9 @@ enum { PROXY_KIND_MP_FLOAT = 4, PROXY_KIND_MP_STR = 5, PROXY_KIND_MP_CALLABLE = 6, - PROXY_KIND_MP_OBJECT = 7, - PROXY_KIND_MP_JSPROXY = 8, + PROXY_KIND_MP_GENERATOR = 7, + PROXY_KIND_MP_OBJECT = 8, + PROXY_KIND_MP_JSPROXY = 9, }; enum { @@ -115,6 +117,8 @@ void proxy_convert_mp_to_js_obj_cside(mp_obj_t obj, uint32_t *out) { } else { if (mp_obj_is_callable(obj)) { kind = PROXY_KIND_MP_CALLABLE; + } else if (mp_obj_is_type(obj, &mp_type_gen_instance)) { + kind = PROXY_KIND_MP_GENERATOR; } else { kind = PROXY_KIND_MP_OBJECT; } @@ -279,3 +283,78 @@ void proxy_c_to_js_get_dict(uint32_t c_ref, uint32_t *out) { out[0] = map->alloc; out[1] = (uintptr_t)map->table; } + +/******************************************************************************/ +// Bridge Python generator to JavaScript thenable. + +static const mp_obj_fun_builtin_var_t resume_obj; + +EM_JS(void, js_then_resolve, (uint32_t * resolve, uint32_t * reject), { + const resolve_js = proxy_convert_mp_to_js_obj_jsside(resolve); + const reject_js = proxy_convert_mp_to_js_obj_jsside(reject); + resolve_js(null); +}); + +EM_JS(void, js_then_reject, (uint32_t * resolve, uint32_t * reject), { + const resolve_js = proxy_convert_mp_to_js_obj_jsside(resolve); + const reject_js = proxy_convert_mp_to_js_obj_jsside(reject); + reject_js(null); +}); + +// *FORMAT-OFF* +EM_JS(void, js_then_continue, (int jsref, uint32_t * py_resume, uint32_t * resolve, uint32_t * reject, uint32_t * out), { + const py_resume_js = proxy_convert_mp_to_js_obj_jsside(py_resume); + const resolve_js = proxy_convert_mp_to_js_obj_jsside(resolve); + const reject_js = proxy_convert_mp_to_js_obj_jsside(reject); + const ret = proxy_js_ref[jsref].then((x) => {py_resume_js(x, resolve_js, reject_js);}, reject_js); + proxy_convert_js_to_mp_obj_jsside(ret, out); +}); +// *FORMAT-ON* + +static mp_obj_t proxy_resume_execute(mp_obj_t self_in, mp_obj_t value, mp_obj_t resolve, mp_obj_t reject) { + mp_obj_t ret_value; + mp_vm_return_kind_t ret_kind = mp_resume(self_in, value, MP_OBJ_NULL, &ret_value); + + uint32_t out_resolve[PVN]; + uint32_t out_reject[PVN]; + proxy_convert_mp_to_js_obj_cside(resolve, out_resolve); + proxy_convert_mp_to_js_obj_cside(reject, out_reject); + + if (ret_kind == MP_VM_RETURN_NORMAL) { + js_then_resolve(out_resolve, out_reject); + return mp_const_none; + } else if (ret_kind == MP_VM_RETURN_YIELD) { + // ret_value should be a JS thenable + mp_obj_t py_resume = mp_obj_new_bound_meth(MP_OBJ_FROM_PTR(&resume_obj), self_in); + int ref = mp_obj_jsproxy_get_ref(ret_value); + uint32_t out_py_resume[PVN]; + proxy_convert_mp_to_js_obj_cside(py_resume, out_py_resume); + uint32_t out[PVN]; + js_then_continue(ref, out_py_resume, out_resolve, out_reject, out); + return proxy_convert_js_to_mp_obj_cside(out); + } else { + // MP_VM_RETURN_EXCEPTION; + js_then_reject(out_resolve, out_reject); + nlr_raise(ret_value); + } +} + +static mp_obj_t resume_fun(size_t n_args, const mp_obj_t *args) { + return proxy_resume_execute(args[0], args[1], args[2], args[3]); +} +static MP_DEFINE_CONST_FUN_OBJ_VAR_BETWEEN(resume_obj, 4, 4, resume_fun); + +void proxy_c_to_js_resume(uint32_t c_ref, uint32_t *args) { + nlr_buf_t nlr; + if (nlr_push(&nlr) == 0) { + mp_obj_t obj = proxy_c_get_obj(c_ref); + mp_obj_t resolve = proxy_convert_js_to_mp_obj_cside(args + 1 * 3); + mp_obj_t reject = proxy_convert_js_to_mp_obj_cside(args + 2 * 3); + mp_obj_t ret = proxy_resume_execute(obj, mp_const_none, resolve, reject); + nlr_pop(); + return proxy_convert_mp_to_js_obj_cside(ret, args); + } else { + // uncaught exception + return proxy_convert_mp_to_js_exc_cside(nlr.ret_val, args); + } +} diff --git a/ports/webassembly/proxy_js.js b/ports/webassembly/proxy_js.js index 1835bdfdfadaa..7a0a1bbe8932d 100644 --- a/ports/webassembly/proxy_js.js +++ b/ports/webassembly/proxy_js.js @@ -34,8 +34,9 @@ const PROXY_KIND_MP_INT = 3; const PROXY_KIND_MP_FLOAT = 4; const PROXY_KIND_MP_STR = 5; const PROXY_KIND_MP_CALLABLE = 6; -const PROXY_KIND_MP_OBJECT = 7; -const PROXY_KIND_MP_JSPROXY = 8; +const PROXY_KIND_MP_GENERATOR = 7; +const PROXY_KIND_MP_OBJECT = 8; +const PROXY_KIND_MP_JSPROXY = 9; const PROXY_KIND_JS_NULL = 1; const PROXY_KIND_JS_BOOLEAN = 2; @@ -122,6 +123,9 @@ function proxy_convert_js_to_mp_obj_jsside(js_obj, out) { } else if (js_obj instanceof PyProxy) { kind = PROXY_KIND_JS_PYPROXY; Module.setValue(out + 4, js_obj._ref, "i32"); + } else if (js_obj instanceof PyProxyThenable) { + kind = PROXY_KIND_JS_PYPROXY; + Module.setValue(out + 4, js_obj._ref, "i32"); } else { kind = PROXY_KIND_JS_OBJECT; const id = proxy_js_ref.length; @@ -193,6 +197,8 @@ function proxy_convert_mp_to_js_obj_jsside(value) { obj = (...args) => { return proxy_call_python(id, args); }; + } else if (kind === PROXY_KIND_MP_GENERATOR) { + obj = new PyProxyThenable(id); } else { // PROXY_KIND_MP_OBJECT const target = new PyProxy(id); From 625b17a410f9e26d6c1ab0b81e475848d70af236 Mon Sep 17 00:00:00 2001 From: Damien George Date: Thu, 1 Feb 2024 17:46:34 +1100 Subject: [PATCH 11/19] webassembly: Implement runCLI() for a Node-based CLI. This allows running MicroPython webassembly from the command line using: node micropython.mjs Signed-off-by: Damien George --- ports/webassembly/api.js | 92 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 92 insertions(+) diff --git a/ports/webassembly/api.js b/ports/webassembly/api.js index ec0601c612ade..2a5522dfb0d9e 100644 --- a/ports/webassembly/api.js +++ b/ports/webassembly/api.js @@ -154,3 +154,95 @@ export async function loadMicroPython(options) { } globalThis.loadMicroPython = loadMicroPython; + +async function runCLI() { + const fs = await import("fs"); + let heap_size = 128 * 1024; + let contents = ""; + let repl = true; + + for (let i = 2; i < process.argv.length; i++) { + if (process.argv[i] === "-X" && i < process.argv.length - 1) { + if (process.argv[i + 1].includes("heapsize=")) { + heap_size = parseInt(process.argv[i + 1].split("heapsize=")[1]); + const suffix = process.argv[i + 1].substr(-1).toLowerCase(); + if (suffix === "k") { + heap_size *= 1024; + } else if (suffix === "m") { + heap_size *= 1024 * 1024; + } + ++i; + } + } else { + contents += fs.readFileSync(process.argv[i], "utf8"); + repl = false; + } + } + + if (process.stdin.isTTY === false) { + contents = fs.readFileSync(0, "utf8"); + repl = false; + } + + const mp = await loadMicroPython({ + heapsize: heap_size, + stdout: (data) => process.stdout.write(data), + linebuffer: false, + }); + + if (repl) { + mp_js_init_repl(); + process.stdin.setRawMode(true); + process.stdin.on("data", (data) => { + for (let i = 0; i < data.length; i++) { + mp_js_process_char(data[i]).then((result) => { + if (result) { + process.exit(); + } + }); + } + }); + } else { + try { + mp.runPython(contents); + } catch (error) { + if (error.name === "PythonError") { + if (error.type === "SystemExit") { + // SystemExit, this is a valid exception to successfully end a script. + } else { + // An unhandled Python exception, print in out. + console.error(error.message); + } + } else { + // A non-Python exception. Re-raise it. + throw error; + } + } + } +} + +// Check if Node is running (equivalent to ENVIRONMENT_IS_NODE). +if ( + typeof process === "object" && + typeof process.versions === "object" && + typeof process.versions.node === "string" +) { + // Check if this module is ron from the command line. + // + // See https://stackoverflow.com/questions/6398196/detect-if-called-through-require-or-directly-by-command-line/66309132#66309132 + // + // Note: + // - `resolve()` is used to handle symlinks + // - `includes()` is used to handle cases where the file extension was omitted when passed to node + + const path = await import("path"); + const url = await import("url"); + + const pathToThisFile = path.resolve(url.fileURLToPath(import.meta.url)); + const pathPassedToNode = path.resolve(process.argv[1]); + const isThisFileBeingRunViaCLI = pathToThisFile.includes(pathPassedToNode); + + if (isThisFileBeingRunViaCLI) { + runCLI(); + } +} From 6ff3e356e2cef310ea5e6481d60163a8138c3d66 Mon Sep 17 00:00:00 2001 From: Damien George Date: Wed, 6 Mar 2024 10:16:52 +1100 Subject: [PATCH 12/19] webassembly: Implement replInit() and replProcessChar(). This is the JavaScript API for starting and running a REPL. Signed-off-by: Damien George --- ports/webassembly/Makefile | 11 +++-- ports/webassembly/api.js | 25 ++++++++++- ports/webassembly/main.c | 52 ++++------------------ ports/webassembly/wrapper.js | 85 ------------------------------------ 4 files changed, 38 insertions(+), 135 deletions(-) delete mode 100644 ports/webassembly/wrapper.js diff --git a/ports/webassembly/Makefile b/ports/webassembly/Makefile index 05e0c1f0f9085..d0a8aa99243ca 100644 --- a/ports/webassembly/Makefile +++ b/ports/webassembly/Makefile @@ -34,6 +34,7 @@ include $(TOP)/extmod/extmod.mk CC = emcc LD = emcc +NODE ?= node TERSER ?= npx terser INC += -I. @@ -76,9 +77,8 @@ JSFLAGS += -s EXPORTED_FUNCTIONS="\ _free,\ _malloc,\ _mp_js_init,\ - _mp_js_init_repl,\ - _mp_js_do_str,\ - _mp_js_process_char,\ + _mp_js_repl_init,\ + _mp_js_repl_process_char,\ _mp_hal_get_interrupt_char,\ _mp_sched_keyboard_interrupt$(EXPORTED_FUNCTIONS_EXTRA)" JSFLAGS += -s EXPORTED_RUNTIME_METHODS="\ @@ -124,7 +124,7 @@ OBJ += $(addprefix $(BUILD)/, $(SRC_C:.c=.o)) ################################################################################ # Main targets. -.PHONY: all min test test_min +.PHONY: all repl min test test_min all: $(BUILD)/micropython.mjs @@ -136,6 +136,9 @@ $(BUILD)/micropython.mjs: $(OBJ) library.js $(SRC_JS) $(BUILD)/micropython.min.mjs: $(BUILD)/micropython.mjs $(TERSER) $< --compress --module -o $@ +repl: $(BUILD)/micropython.mjs + $(NODE) $< + min: $(BUILD)/micropython.min.mjs test: $(BUILD)/micropython.mjs $(TOP)/tests/run-tests.py diff --git a/ports/webassembly/api.js b/ports/webassembly/api.js index 2a5522dfb0d9e..7d1832af4f2ba 100644 --- a/ports/webassembly/api.js +++ b/ports/webassembly/api.js @@ -150,6 +150,27 @@ export async function loadMicroPython(options) { ); return proxy_convert_mp_to_js_obj_jsside_with_free(value); }, + replInit() { + Module.ccall("mp_js_repl_init", "null", ["null"]); + }, + replProcessChar(chr) { + return Module.ccall( + "mp_js_repl_process_char", + "number", + ["number"], + [chr], + ); + }, + // Needed if the GC/asyncify is enabled. + async replProcessCharWithAsyncify(chr) { + return Module.ccall( + "mp_js_repl_process_char", + "number", + ["number"], + [chr], + { async: true }, + ); + }, }; } @@ -191,11 +212,11 @@ async function runCLI() { }); if (repl) { - mp_js_init_repl(); + mp.replInit(); process.stdin.setRawMode(true); process.stdin.on("data", (data) => { for (let i = 0; i < data.length; i++) { - mp_js_process_char(data[i]).then((result) => { + mp.replProcessCharWithAsyncify(data[i]).then((result) => { if (result) { process.exit(); } diff --git a/ports/webassembly/main.c b/ports/webassembly/main.c index 5bb4222aaa726..441c6d67e350a 100644 --- a/ports/webassembly/main.c +++ b/ports/webassembly/main.c @@ -44,46 +44,6 @@ #include "library.h" #include "proxy_c.h" -#if MICROPY_ENABLE_COMPILER -int do_str(const char *src, mp_parse_input_kind_t input_kind) { - int ret = 0; - nlr_buf_t nlr; - if (nlr_push(&nlr) == 0) { - mp_lexer_t *lex = mp_lexer_new_from_str_len(MP_QSTR__lt_stdin_gt_, src, strlen(src), 0); - qstr source_name = lex->source_name; - mp_parse_tree_t parse_tree = mp_parse(lex, input_kind); - mp_obj_t module_fun = mp_compile(&parse_tree, source_name, false); - mp_call_function_0(module_fun); - nlr_pop(); - } else { - // uncaught exception - if (mp_obj_is_subclass_fast(mp_obj_get_type((mp_obj_t)nlr.ret_val), &mp_type_SystemExit)) { - mp_obj_t exit_val = mp_obj_exception_get_value(MP_OBJ_FROM_PTR(nlr.ret_val)); - if (exit_val != mp_const_none) { - mp_int_t int_val; - if (mp_obj_get_int_maybe(exit_val, &int_val)) { - ret = int_val & 255; - } else { - ret = 1; - } - } - } else { - mp_obj_print_exception(&mp_plat_print, (mp_obj_t)nlr.ret_val); - ret = 1; - } - } - return ret; -} -#endif - -int mp_js_do_str(const char *code) { - return do_str(code, MP_PARSE_FILE_INPUT); -} - -int mp_js_process_char(int c) { - return pyexec_event_repl_process_char(c); -} - void mp_js_init(int heap_size) { #if MICROPY_ENABLE_GC char *heap = (char *)malloc(heap_size * sizeof(char)); @@ -111,10 +71,6 @@ void mp_js_init(int heap_size) { #endif } -void mp_js_init_repl() { - pyexec_event_repl_init(); -} - void mp_js_register_js_module(const char *name, uint32_t *value) { mp_obj_t module_name = MP_OBJ_NEW_QSTR(qstr_from_str(name)); mp_obj_t module = proxy_convert_js_to_mp_obj_cside(value); @@ -175,6 +131,14 @@ void mp_js_do_exec_async(const char *src, uint32_t *out) { mp_compile_allow_top_level_await = false; } +void mp_js_repl_init(void) { + pyexec_event_repl_init(); +} + +int mp_js_repl_process_char(int c) { + return pyexec_event_repl_process_char(c); +} + #if MICROPY_GC_SPLIT_HEAP_AUTO // The largest new region that is available to become Python heap. diff --git a/ports/webassembly/wrapper.js b/ports/webassembly/wrapper.js deleted file mode 100644 index 399801b7afcb9..0000000000000 --- a/ports/webassembly/wrapper.js +++ /dev/null @@ -1,85 +0,0 @@ -/* - * This file is part of the MicroPython project, http://micropython.org/ - * - * The MIT License (MIT) - * - * Copyright (c) 2017, 2018 Rami Ali - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ - -var Module = {}; - -var mainProgram = function() -{ - mp_js_init = Module.cwrap('mp_js_init', 'null', ['number']); - mp_js_do_str = Module.cwrap('mp_js_do_str', 'number', ['string'], {async: true}); - mp_js_init_repl = Module.cwrap('mp_js_init_repl', 'null', ['null']); - mp_js_process_char = Module.cwrap('mp_js_process_char', 'number', ['number'], {async: true}); - - if (typeof window === 'undefined' && require.main === module) { - var fs = require('fs'); - var heap_size = 128 * 1024; - var contents = ''; - var repl = true; - - for (var i = 0; i < process.argv.length; i++) { - if (process.argv[i] === '-X' && i < process.argv.length - 1) { - if (process.argv[i + 1].includes('heapsize=')) { - heap_size = parseInt(process.argv[i + 1].split('heapsize=')[1]); - if (process.argv[i + 1].substr(-1).toLowerCase() === 'k') { - heap_size *= 1024; - } else if (process.argv[i + 1].substr(-1).toLowerCase() === 'm') { - heap_size *= 1024 * 1024; - } - } - } else if (process.argv[i].includes('.py')) { - contents += fs.readFileSync(process.argv[i], 'utf8'); - repl = false;; - } - } - - if (process.stdin.isTTY === false) { - contents = fs.readFileSync(0, 'utf8'); - repl = 0; - } - - mp_js_init(heap_size); - - if (repl) { - mp_js_init_repl(); - process.stdin.setRawMode(true); - process.stdin.on('data', function (data) { - for (var i = 0; i < data.length; i++) { - mp_js_process_char(data[i]).then(result => { - if (result) { - process.exit() - } - }) - } - }); - } else { - mp_js_do_str(contents).then(exitCode => { - process.exitCode = exitCode - }) - } - } -} - -Module["onRuntimeInitialized"] = mainProgram; From b9eb74e73b8ad7be7c3d3d2eca2b1755041f03e6 Mon Sep 17 00:00:00 2001 From: Damien George Date: Wed, 31 May 2023 11:45:34 +1000 Subject: [PATCH 13/19] webassembly/variants/pyscript: Add pyscript variant. This commit adds a pyscript variant for use in https://pyscript.net/. The configuration is: - No ASYNCIFY, in order to keep the WASM size down and have good performance. - MICROPY_CONFIG_ROM_LEVEL_FULL_FEATURES to enable most features. - Custom manifest that includes many of the python-stdlib libraries. - MICROPY_GC_SPLIT_HEAP_AUTO to increase GC heap size instead of doing a collection when memory is exhausted. This is needed because ASYNCIFY is disabled. Instead the GC collection is run at the top-level before executing any Python code. - No MICROPY_VARIANT_ENABLE_JS_HOOK because there is no asynchronous keyboard input to interrupt a running script. Signed-off-by: Damien George --- .../webassembly/variants/pyscript/manifest.py | 27 +++++++++++++++++++ .../variants/pyscript/mpconfigvariant.h | 3 +++ .../variants/pyscript/mpconfigvariant.mk | 3 +++ 3 files changed, 33 insertions(+) create mode 100644 ports/webassembly/variants/pyscript/manifest.py create mode 100644 ports/webassembly/variants/pyscript/mpconfigvariant.h create mode 100644 ports/webassembly/variants/pyscript/mpconfigvariant.mk diff --git a/ports/webassembly/variants/pyscript/manifest.py b/ports/webassembly/variants/pyscript/manifest.py new file mode 100644 index 0000000000000..0646e1d897f0c --- /dev/null +++ b/ports/webassembly/variants/pyscript/manifest.py @@ -0,0 +1,27 @@ +require("abc") +require("base64") +require("collections") +require("collections-defaultdict") +require("copy") +require("datetime") +require("fnmatch") +require("functools") +require("gzip") +require("hmac") +require("html") +require("inspect") +require("io") +require("itertools") +require("locale") +require("logging") +require("operator") +require("os") +require("os-path") +require("pathlib") +require("stat") +require("tarfile") +require("tarfile-write") +require("time") +require("unittest") +require("uu") +require("zlib") diff --git a/ports/webassembly/variants/pyscript/mpconfigvariant.h b/ports/webassembly/variants/pyscript/mpconfigvariant.h new file mode 100644 index 0000000000000..ed8e812803533 --- /dev/null +++ b/ports/webassembly/variants/pyscript/mpconfigvariant.h @@ -0,0 +1,3 @@ +#define MICROPY_CONFIG_ROM_LEVEL (MICROPY_CONFIG_ROM_LEVEL_FULL_FEATURES) +#define MICROPY_GC_SPLIT_HEAP (1) +#define MICROPY_GC_SPLIT_HEAP_AUTO (1) diff --git a/ports/webassembly/variants/pyscript/mpconfigvariant.mk b/ports/webassembly/variants/pyscript/mpconfigvariant.mk new file mode 100644 index 0000000000000..016b96a99af0a --- /dev/null +++ b/ports/webassembly/variants/pyscript/mpconfigvariant.mk @@ -0,0 +1,3 @@ +JSFLAGS += -s ALLOW_MEMORY_GROWTH + +FROZEN_MANIFEST ?= variants/pyscript/manifest.py From 26d6969fefa18c1dbbb7bc852588fa00ae253d74 Mon Sep 17 00:00:00 2001 From: Damien George Date: Wed, 6 Mar 2024 10:27:54 +1100 Subject: [PATCH 14/19] webassembly: Update README.md to describe latest changes. Signed-off-by: Damien George --- ports/webassembly/README.md | 147 +++++++++++++++++++++++------------- 1 file changed, 96 insertions(+), 51 deletions(-) diff --git a/ports/webassembly/README.md b/ports/webassembly/README.md index abd2864a56cb3..af395778a748e 100644 --- a/ports/webassembly/README.md +++ b/ports/webassembly/README.md @@ -6,22 +6,22 @@ MicroPython for [WebAssembly](https://webassembly.org/). Dependencies ------------ -Building webassembly port bears the same requirements as the standard -MicroPython ports with the addition of Emscripten (and uglify-js for the -minified file). +Building the webassembly port bears the same requirements as the standard +MicroPython ports with the addition of Emscripten, and optionally terser for +the minified file. -The output includes `micropython.js` (a JavaScript wrapper for the -MicroPython runtime) and `firmware.wasm` (actual MicroPython compiled to +The output includes `micropython.mjs` (a JavaScript wrapper for the +MicroPython runtime) and `micropython.wasm` (actual MicroPython compiled to WASM). Build instructions ------------------ -In order to build micropython.js, run: +In order to build `micropython.mjs`, run: $ make -To generate the minified file micropython.min.js, run: +To generate the minified file `micropython.min.mjs`, run: $ make min @@ -30,55 +30,90 @@ Running with Node.js Access the repl with: - $ node build/micropython.js + $ make repl -Stack size may be modified using: +This is the same as running: - $ node build/micropython.js -X stack=64K + $ node build-standard/micropython.mjs -Where stack size may be represented in Bytes, KiB or MiB. +The initial MicroPython GC heap size may be modified using: + + $ node build-standard/micropython.mjs -X heapsize=64k + +Where stack size may be represented in bytes, or have a `k` or `m` suffix. MicroPython scripts may be executed using: - $ node build/micropython.js hello.py + $ node build-standard/micropython.mjs hello.py -Alternatively micropython.js may by accessed by other javascript programs in node +Alternatively `micropython.mjs` may by accessed by other JavaScript programs in node using the require command and the general API outlined below. For example: ```javascript -var mp_js = require('./build/micropython.js'); +const mp_mjs = await import("micropython.mjs"); +const mp = await mp_mjs.loadMicroPython(); + +mp.runPython("print('hello world')"); +``` -mp_js_init(64 * 1024); -await mp_js_do_str("print('hello world')\n"); +Or without await notation: + +```javascript +import("micropython.mjs").then((mp_mjs) => { + mp_mjs.loadMicroPython().then((mp) => { + mp.runPython("print('hello world')"); + }); +}); ``` Running with HTML ----------------- -The prerequisite for browser operation of micropython.js is to listen to the -`micropython-print` event, which is passed data when MicroPython code prints -something to stdout. The following code demonstrates basic functionality: +The following code demonstrates the simplest way to load `micropython.mjs` in a +browser, create an interpreter context, and run some Python code: + +```html + + + + + + + + + +``` + +The output in the above example will go to the JavaScript console. It's possible +to instead capture the output and print it somewhere else, for example in an +HTML element. The following example shows how to do this, and also demonstrates +the use of top-level await and the `js` module: ```html - + 

-    
   
 
@@ -98,31 +133,41 @@ Run the test suite using:
 API
 ---
 
-The following functions have been exposed to javascript.
+The following functions have been exposed to JavaScript through the interpreter
+context, created and returned by `loadMicroPython()`.
 
-```
-mp_js_init(stack_size)
-```
+- `PyProxy`: the type of the object that proxies Python objects.
 
-Initialize MicroPython with the given stack size in bytes. This must be
-called before attempting to interact with MicroPython.
+- `FS`: the Emscripten filesystem object.
 
-```
-await mp_js_do_str(code)
-```
+- `globals`: an object exposing the globals from the Python `__main__` module,
+  with methods `get(key)`, `set(key, value)` and `delete(key)`.
 
-Execute the input code. `code` must be a `string`.
+- `registerJsModule(name, module)`: register a JavaScript object as importable
+  from Python with the given name.
 
-```
-mp_js_init_repl()
-```
+- `pyimport`: import a Python module and return it.
 
-Initialize MicroPython repl. Must be called before entering characters into
-the repl.
+- `runPython(code)`: execute Python code and return the result.
 
-```
-await mp_js_process_char(char)
-```
+- `runPythonAsync(code)`: execute Python code and return the result, allowing for
+  top-level await expressions (this call must be await'ed on the JavaScript side).
+
+- `replInit()`: initialise the REPL.
+
+- `replProcessChar(chr)`: process an incoming character at the REPL.
+
+- `replProcessCharWithAsyncify(chr)`: process an incoming character at the REPL,
+  for use when ASYNCIFY is enabled.
+
+Proxying
+--------
+
+A Python `dict` instance is proxied such that:
+
+    for (const key in dict) {
+        print(key, dict[key]);
+    }
 
-Input character into MicroPython repl. `char` must be of type `number`. This
-will execute MicroPython code when necessary.
+works as expected on the JavaScript side and iterates through the keys of the
+Python `dict`.

From c2cf58befcfd54d886fca879f05d8af2368d6cbe Mon Sep 17 00:00:00 2001
From: Damien George 
Date: Mon, 11 Mar 2024 15:02:41 +1100
Subject: [PATCH 15/19] webassembly/library: Fix formatting and style for
 Biome.

Signed-off-by: Damien George 
---
 ports/webassembly/library.js | 26 ++++++++++++++++++--------
 1 file changed, 18 insertions(+), 8 deletions(-)

diff --git a/ports/webassembly/library.js b/ports/webassembly/library.js
index db5bac2691443..3f6c9cb61f189 100644
--- a/ports/webassembly/library.js
+++ b/ports/webassembly/library.js
@@ -30,23 +30,33 @@ mergeInto(LibraryManager.library, {
 
     mp_js_ticks_ms: () => Date.now() - MP_JS_EPOCH,
 
-    mp_js_hook: function() {
+    mp_js_hook: () => {
         if (ENVIRONMENT_IS_NODE) {
-            var mp_interrupt_char = Module.ccall('mp_hal_get_interrupt_char', 'number', ['number'], ['null']);
-            var fs = require('fs');
+            const mp_interrupt_char = Module.ccall(
+                "mp_hal_get_interrupt_char",
+                "number",
+                ["number"],
+                ["null"],
+            );
+            const fs = require("fs");
 
-            var buf = Buffer.alloc(1);
+            const buf = Buffer.alloc(1);
             try {
-                var n = fs.readSync(process.stdin.fd, buf, 0, 1);
+                const n = fs.readSync(process.stdin.fd, buf, 0, 1);
                 if (n > 0) {
-                    if (buf[0] == mp_interrupt_char) {
-                        Module.ccall('mp_sched_keyboard_interrupt', 'null', ['null'], ['null']);
+                    if (buf[0] === mp_interrupt_char) {
+                        Module.ccall(
+                            "mp_sched_keyboard_interrupt",
+                            "null",
+                            ["null"],
+                            ["null"],
+                        );
                     } else {
                         process.stdout.write(String.fromCharCode(buf[0]));
                     }
                 }
             } catch (e) {
-                if (e.code === 'EAGAIN') {
+                if (e.code === "EAGAIN") {
                 } else {
                     throw e;
                 }

From e41b571a297bd1682edf6fd98121512bc14ed60d Mon Sep 17 00:00:00 2001
From: Damien George 
Date: Thu, 1 Feb 2024 17:40:59 +1100
Subject: [PATCH 16/19] tests/run-tests.py: Support running webassembly tests
 via node.

This allows running tests with a .js/.mjs suffix, and also .py tests using
node and the webassembly port.

Signed-off-by: Damien George 
---
 tests/run-tests.py | 77 ++++++++++++++++++++++++++++++++++++++++++++--
 1 file changed, 75 insertions(+), 2 deletions(-)

diff --git a/tests/run-tests.py b/tests/run-tests.py
index 70279d379df60..83af61c83d20b 100755
--- a/tests/run-tests.py
+++ b/tests/run-tests.py
@@ -308,7 +308,9 @@ def send_get(what):
 
     else:
         # run via pyboard interface
-        had_crash, output_mupy = run_script_on_remote_target(pyb, args, test_file, is_special)
+        had_crash, output_mupy = pyb.run_script_on_remote_target(
+            args, test_file_abspath, is_special
+        )
 
     # canonical form for all ports/platforms is to use \n for end-of-line
     output_mupy = output_mupy.replace(b"\r\n", b"\n")
@@ -393,6 +395,51 @@ def value(self):
         return self._value
 
 
+class PyboardNodeRunner:
+    def __init__(self):
+        mjs = os.getenv("MICROPY_MICROPYTHON_MJS")
+        if mjs is None:
+            mjs = base_path("../ports/webassembly/build-standard/micropython.mjs")
+        else:
+            mjs = os.path.abspath(mjs)
+        self.micropython_mjs = mjs
+
+    def close(self):
+        pass
+
+    def run_script_on_remote_target(self, args, test_file, is_special):
+        cwd = os.path.dirname(test_file)
+
+        # Create system command list.
+        cmdlist = ["node"]
+        if test_file.endswith(".py"):
+            # Run a Python script indirectly via "node micropython.mjs ".
+            cmdlist.append(self.micropython_mjs)
+            if args.heapsize is not None:
+                cmdlist.extend(["-X", "heapsize=" + args.heapsize])
+            cmdlist.append(test_file)
+        else:
+            # Run a js/mjs script directly with Node, passing in the path to micropython.mjs.
+            cmdlist.append(test_file)
+            cmdlist.append(self.micropython_mjs)
+
+        # Run the script.
+        try:
+            had_crash = False
+            output_mupy = subprocess.check_output(
+                cmdlist, stderr=subprocess.STDOUT, timeout=TEST_TIMEOUT, cwd=cwd
+            )
+        except subprocess.CalledProcessError as er:
+            had_crash = True
+            output_mupy = er.output + b"CRASH"
+        except subprocess.TimeoutExpired as er:
+            had_crash = True
+            output_mupy = (er.output or b"") + b"TIMEOUT"
+
+        # Return the results.
+        return had_crash, output_mupy
+
+
 def run_tests(pyb, tests, args, result_dir, num_threads=1):
     test_count = ThreadSafeCounter()
     testcase_count = ThreadSafeCounter()
@@ -631,6 +678,20 @@ def run_tests(pyb, tests, args, result_dir, num_threads=1):
             )  # RA fsp rtc function doesn't support nano sec info
         elif args.target == "qemu-arm":
             skip_tests.add("misc/print_exception.py")  # requires sys stdfiles
+        elif args.target == "webassembly":
+            skip_tests.add("basics/string_format_modulo.py")  # can't print nulls to stdout
+            skip_tests.add("basics/string_strip.py")  # can't print nulls to stdout
+            skip_tests.add("extmod/binascii_a2b_base64.py")
+            skip_tests.add("extmod/re_stack_overflow.py")
+            skip_tests.add("extmod/time_res.py")
+            skip_tests.add("extmod/vfs_posix.py")
+            skip_tests.add("extmod/vfs_posix_enoent.py")
+            skip_tests.add("extmod/vfs_posix_paths.py")
+            skip_tests.add("extmod/vfs_userfs.py")
+            skip_tests.add("micropython/emg_exc.py")
+            skip_tests.add("micropython/extreme_exc.py")
+            skip_tests.add("micropython/heapalloc_exc_compressed_emg_exc.py")
+            skip_tests.add("micropython/import_mpy_invalid.py")
 
     # Some tests are known to fail on 64-bit machines
     if pyb is None and platform.architecture()[0] == "64bit":
@@ -977,6 +1038,7 @@ def main():
     LOCAL_TARGETS = (
         "unix",
         "qemu-arm",
+        "webassembly",
     )
     EXTERNAL_TARGETS = (
         "pyboard",
@@ -997,6 +1059,8 @@ def main():
                 args.mpy_cross_flags = "-march=host"
             elif args.target == "qemu-arm":
                 args.mpy_cross_flags = "-march=armv7m"
+        if args.target == "webassembly":
+            pyb = PyboardNodeRunner()
     elif args.target in EXTERNAL_TARGETS:
         global pyboard
         sys.path.append(base_path("../tools"))
@@ -1015,6 +1079,7 @@ def main():
                 args.mpy_cross_flags = "-march=armv7m"
 
         pyb = pyboard.Pyboard(args.device, args.baudrate, args.user, args.password)
+        pyboard.Pyboard.run_script_on_remote_target = run_script_on_remote_target
         pyb.enter_raw_repl()
     else:
         raise ValueError("target must be one of %s" % ", ".join(LOCAL_TARGETS + EXTERNAL_TARGETS))
@@ -1032,6 +1097,10 @@ def main():
         else:
             tests = []
     elif len(args.files) == 0:
+        test_extensions = ("*.py",)
+        if args.target == "webassembly":
+            test_extensions += ("*.js", "*.mjs")
+
         if args.test_dirs is None:
             test_dirs = (
                 "basics",
@@ -1072,12 +1141,16 @@ def main():
                     "inlineasm",
                     "ports/qemu-arm",
                 )
+            elif args.target == "webassembly":
+                test_dirs += ("float",)
         else:
             # run tests from these directories
             test_dirs = args.test_dirs
         tests = sorted(
             test_file
-            for test_files in (glob("{}/*.py".format(dir)) for dir in test_dirs)
+            for test_files in (
+                glob(os.path.join(dir, ext)) for dir in test_dirs for ext in test_extensions
+            )
             for test_file in test_files
         )
     else:

From c1513a078dcde3eba74c4572b62b354aed404e21 Mon Sep 17 00:00:00 2001
From: Damien George 
Date: Thu, 1 Feb 2024 17:43:25 +1100
Subject: [PATCH 17/19] tests/ports/webassembly: Add webassembly JS tests.

Signed-off-by: Damien George 
---
 tests/ports/webassembly/basic.js              |   8 ++
 tests/ports/webassembly/basic.js.exp          |   3 +
 tests/ports/webassembly/filesystem.mjs        |   9 ++
 tests/ports/webassembly/filesystem.mjs.exp    |   3 +
 tests/ports/webassembly/float.mjs             |  43 +++++++
 tests/ports/webassembly/float.mjs.exp         |  14 +++
 tests/ports/webassembly/fun_call.mjs          |  17 +++
 tests/ports/webassembly/fun_call.mjs.exp      |   7 ++
 tests/ports/webassembly/globals.mjs           |  13 ++
 tests/ports/webassembly/globals.mjs.exp       |   4 +
 tests/ports/webassembly/heap_expand.mjs       |  15 +++
 tests/ports/webassembly/heap_expand.mjs.exp   |  48 ++++++++
 .../ports/webassembly/jsffi_create_proxy.mjs  |  15 +++
 .../webassembly/jsffi_create_proxy.mjs.exp    |   6 +
 tests/ports/webassembly/jsffi_to_js.mjs       |  28 +++++
 tests/ports/webassembly/jsffi_to_js.mjs.exp   |  11 ++
 tests/ports/webassembly/override_new.mjs      |  33 +++++
 tests/ports/webassembly/override_new.mjs.exp  |   7 ++
 .../webassembly/promise_with_resolvers.mjs    |  23 ++++
 .../promise_with_resolvers.mjs.exp            |   1 +
 tests/ports/webassembly/py_proxy_delete.mjs   |  30 +++++
 .../ports/webassembly/py_proxy_delete.mjs.exp |   2 +
 tests/ports/webassembly/py_proxy_dict.mjs     |  34 ++++++
 tests/ports/webassembly/py_proxy_dict.mjs.exp |  11 ++
 tests/ports/webassembly/py_proxy_has.mjs      |  11 ++
 tests/ports/webassembly/py_proxy_has.mjs.exp  |   2 +
 tests/ports/webassembly/py_proxy_own_keys.mjs |  11 ++
 .../webassembly/py_proxy_own_keys.mjs.exp     |   9 ++
 tests/ports/webassembly/py_proxy_set.mjs      |  27 ++++
 tests/ports/webassembly/py_proxy_set.mjs.exp  |   2 +
 tests/ports/webassembly/py_proxy_to_js.mjs    |  20 +++
 .../ports/webassembly/py_proxy_to_js.mjs.exp  |   4 +
 tests/ports/webassembly/register_js_module.js |   6 +
 .../webassembly/register_js_module.js.exp     |   2 +
 tests/ports/webassembly/run_python_async.mjs  | 115 ++++++++++++++++++
 .../webassembly/run_python_async.mjs.exp      |  38 ++++++
 tests/ports/webassembly/run_python_async2.mjs |  42 +++++++
 .../webassembly/run_python_async2.mjs.exp     |   8 ++
 tests/ports/webassembly/this_behaviour.mjs    |  24 ++++
 .../ports/webassembly/this_behaviour.mjs.exp  |   4 +
 tests/ports/webassembly/various.js            |  38 ++++++
 tests/ports/webassembly/various.js.exp        |   8 ++
 tests/run-tests.py                            |   2 +-
 43 files changed, 757 insertions(+), 1 deletion(-)
 create mode 100644 tests/ports/webassembly/basic.js
 create mode 100644 tests/ports/webassembly/basic.js.exp
 create mode 100644 tests/ports/webassembly/filesystem.mjs
 create mode 100644 tests/ports/webassembly/filesystem.mjs.exp
 create mode 100644 tests/ports/webassembly/float.mjs
 create mode 100644 tests/ports/webassembly/float.mjs.exp
 create mode 100644 tests/ports/webassembly/fun_call.mjs
 create mode 100644 tests/ports/webassembly/fun_call.mjs.exp
 create mode 100644 tests/ports/webassembly/globals.mjs
 create mode 100644 tests/ports/webassembly/globals.mjs.exp
 create mode 100644 tests/ports/webassembly/heap_expand.mjs
 create mode 100644 tests/ports/webassembly/heap_expand.mjs.exp
 create mode 100644 tests/ports/webassembly/jsffi_create_proxy.mjs
 create mode 100644 tests/ports/webassembly/jsffi_create_proxy.mjs.exp
 create mode 100644 tests/ports/webassembly/jsffi_to_js.mjs
 create mode 100644 tests/ports/webassembly/jsffi_to_js.mjs.exp
 create mode 100644 tests/ports/webassembly/override_new.mjs
 create mode 100644 tests/ports/webassembly/override_new.mjs.exp
 create mode 100644 tests/ports/webassembly/promise_with_resolvers.mjs
 create mode 100644 tests/ports/webassembly/promise_with_resolvers.mjs.exp
 create mode 100644 tests/ports/webassembly/py_proxy_delete.mjs
 create mode 100644 tests/ports/webassembly/py_proxy_delete.mjs.exp
 create mode 100644 tests/ports/webassembly/py_proxy_dict.mjs
 create mode 100644 tests/ports/webassembly/py_proxy_dict.mjs.exp
 create mode 100644 tests/ports/webassembly/py_proxy_has.mjs
 create mode 100644 tests/ports/webassembly/py_proxy_has.mjs.exp
 create mode 100644 tests/ports/webassembly/py_proxy_own_keys.mjs
 create mode 100644 tests/ports/webassembly/py_proxy_own_keys.mjs.exp
 create mode 100644 tests/ports/webassembly/py_proxy_set.mjs
 create mode 100644 tests/ports/webassembly/py_proxy_set.mjs.exp
 create mode 100644 tests/ports/webassembly/py_proxy_to_js.mjs
 create mode 100644 tests/ports/webassembly/py_proxy_to_js.mjs.exp
 create mode 100644 tests/ports/webassembly/register_js_module.js
 create mode 100644 tests/ports/webassembly/register_js_module.js.exp
 create mode 100644 tests/ports/webassembly/run_python_async.mjs
 create mode 100644 tests/ports/webassembly/run_python_async.mjs.exp
 create mode 100644 tests/ports/webassembly/run_python_async2.mjs
 create mode 100644 tests/ports/webassembly/run_python_async2.mjs.exp
 create mode 100644 tests/ports/webassembly/this_behaviour.mjs
 create mode 100644 tests/ports/webassembly/this_behaviour.mjs.exp
 create mode 100644 tests/ports/webassembly/various.js
 create mode 100644 tests/ports/webassembly/various.js.exp

diff --git a/tests/ports/webassembly/basic.js b/tests/ports/webassembly/basic.js
new file mode 100644
index 0000000000000..19b1881faf5ca
--- /dev/null
+++ b/tests/ports/webassembly/basic.js
@@ -0,0 +1,8 @@
+import(process.argv[2]).then((mp) => {
+    mp.loadMicroPython().then((py) => {
+        py.runPython("1");
+        py.runPython("print('hello')");
+        py.runPython("import sys; print(f'hello from {sys.platform}')");
+        py.runPython("import collections; print(collections.OrderedDict)");
+    });
+});
diff --git a/tests/ports/webassembly/basic.js.exp b/tests/ports/webassembly/basic.js.exp
new file mode 100644
index 0000000000000..6459b2492cf86
--- /dev/null
+++ b/tests/ports/webassembly/basic.js.exp
@@ -0,0 +1,3 @@
+hello
+hello from webassembly
+
diff --git a/tests/ports/webassembly/filesystem.mjs b/tests/ports/webassembly/filesystem.mjs
new file mode 100644
index 0000000000000..e9e2920a16105
--- /dev/null
+++ b/tests/ports/webassembly/filesystem.mjs
@@ -0,0 +1,9 @@
+const mp = await (await import(process.argv[2])).loadMicroPython();
+
+mp.FS.mkdir("/lib/");
+mp.FS.writeFile("/lib/testmod.py", "x = 1; print(__name__, x)");
+mp.runPython("import testmod");
+
+mp.runPython("import sys; sys.modules.clear()");
+const testmod = mp.pyimport("testmod");
+console.log("testmod:", testmod, testmod.x);
diff --git a/tests/ports/webassembly/filesystem.mjs.exp b/tests/ports/webassembly/filesystem.mjs.exp
new file mode 100644
index 0000000000000..25a48b10832c5
--- /dev/null
+++ b/tests/ports/webassembly/filesystem.mjs.exp
@@ -0,0 +1,3 @@
+testmod 1
+testmod 1
+testmod: PyProxy { _ref: 3 } 1
diff --git a/tests/ports/webassembly/float.mjs b/tests/ports/webassembly/float.mjs
new file mode 100644
index 0000000000000..53bb8b1c4d0ad
--- /dev/null
+++ b/tests/ports/webassembly/float.mjs
@@ -0,0 +1,43 @@
+// Test passing floats between JavaScript and Python.
+
+const mp = await (await import(process.argv[2])).loadMicroPython();
+
+globalThis.a = 1 / 2;
+globalThis.b = Infinity;
+globalThis.c = NaN;
+
+mp.runPython(`
+import js
+
+# Test retrieving floats from JS.
+print(js.a)
+print(js.b)
+print(js.c)
+
+# Test calling JS which returns a float.
+r = js.Math.random()
+print(type(r), 0 < r < 1)
+
+x = 1 / 2
+y = float("inf")
+z = float("nan")
+
+# Test passing floats to a JS function.
+js.console.log(x)
+js.console.log(x, y)
+js.console.log(x, y, z)
+`);
+
+// Test retrieving floats from Python.
+console.log(mp.globals.get("x"));
+console.log(mp.globals.get("y"));
+console.log(mp.globals.get("z"));
+
+// Test passing floats to a Python function.
+const mp_print = mp.pyimport("builtins").print;
+mp_print(globalThis.a);
+mp_print(globalThis.a, globalThis.b);
+mp_print(globalThis.a, globalThis.b, globalThis.c);
+
+// Test calling Python which returns a float.
+console.log(mp.pyimport("math").sqrt(0.16));
diff --git a/tests/ports/webassembly/float.mjs.exp b/tests/ports/webassembly/float.mjs.exp
new file mode 100644
index 0000000000000..57eff74acd8af
--- /dev/null
+++ b/tests/ports/webassembly/float.mjs.exp
@@ -0,0 +1,14 @@
+0.5
+inf
+nan
+ True
+0.5
+0.5 Infinity
+0.5 Infinity NaN
+0.5
+Infinity
+NaN
+0.5
+0.5 inf
+0.5 inf nan
+0.4
diff --git a/tests/ports/webassembly/fun_call.mjs b/tests/ports/webassembly/fun_call.mjs
new file mode 100644
index 0000000000000..295745d2e2e34
--- /dev/null
+++ b/tests/ports/webassembly/fun_call.mjs
@@ -0,0 +1,17 @@
+// Test calling JavaScript functions from Python.
+
+const mp = await (await import(process.argv[2])).loadMicroPython();
+
+globalThis.f = (a, b, c, d, e) => {
+    console.log(a, b, c, d, e);
+};
+mp.runPython(`
+import js
+js.f()
+js.f(1)
+js.f(1, 2)
+js.f(1, 2, 3)
+js.f(1, 2, 3, 4)
+js.f(1, 2, 3, 4, 5)
+js.f(1, 2, 3, 4, 5, 6)
+`);
diff --git a/tests/ports/webassembly/fun_call.mjs.exp b/tests/ports/webassembly/fun_call.mjs.exp
new file mode 100644
index 0000000000000..e9ed5f6ddf9de
--- /dev/null
+++ b/tests/ports/webassembly/fun_call.mjs.exp
@@ -0,0 +1,7 @@
+undefined undefined undefined undefined undefined
+1 undefined undefined undefined undefined
+1 2 undefined undefined undefined
+1 2 3 undefined undefined
+1 2 3 4 undefined
+1 2 3 4 5
+1 2 3 4 5
diff --git a/tests/ports/webassembly/globals.mjs b/tests/ports/webassembly/globals.mjs
new file mode 100644
index 0000000000000..3d5ecb4167878
--- /dev/null
+++ b/tests/ports/webassembly/globals.mjs
@@ -0,0 +1,13 @@
+// Test accessing Python globals dict via mp.globals.
+
+const mp = await (await import(process.argv[2])).loadMicroPython();
+
+mp.runPython("x = 1");
+console.log(mp.globals.get("x"));
+
+mp.globals.set("y", 2);
+mp.runPython("print(y)");
+
+mp.runPython("print('y' in globals())");
+mp.globals.delete("y");
+mp.runPython("print('y' in globals())");
diff --git a/tests/ports/webassembly/globals.mjs.exp b/tests/ports/webassembly/globals.mjs.exp
new file mode 100644
index 0000000000000..a118c13fecba9
--- /dev/null
+++ b/tests/ports/webassembly/globals.mjs.exp
@@ -0,0 +1,4 @@
+1
+2
+True
+False
diff --git a/tests/ports/webassembly/heap_expand.mjs b/tests/ports/webassembly/heap_expand.mjs
new file mode 100644
index 0000000000000..2cf7c07b0b823
--- /dev/null
+++ b/tests/ports/webassembly/heap_expand.mjs
@@ -0,0 +1,15 @@
+// Test expanding the MicroPython GC heap.
+
+const mp = await (await import(process.argv[2])).loadMicroPython();
+
+mp.runPython(`
+import gc
+bs = []
+for i in range(24):
+    b = bytearray(1 << i)
+    bs.append(b)
+    gc.collect()
+    print(gc.mem_free())
+for b in bs:
+    print(len(b))
+`);
diff --git a/tests/ports/webassembly/heap_expand.mjs.exp b/tests/ports/webassembly/heap_expand.mjs.exp
new file mode 100644
index 0000000000000..ee14908409d86
--- /dev/null
+++ b/tests/ports/webassembly/heap_expand.mjs.exp
@@ -0,0 +1,48 @@
+135241360
+135241328
+135241296
+135241264
+135241216
+135241168
+135241088
+135240944
+135240640
+135240112
+135239072
+135237008
+135232896
+135224688
+135208288
+135175504
+135109888
+134978800
+134716640
+135216848
+136217216
+138218032
+142219616
+150222864
+1
+2
+4
+8
+16
+32
+64
+128
+256
+512
+1024
+2048
+4096
+8192
+16384
+32768
+65536
+131072
+262144
+524288
+1048576
+2097152
+4194304
+8388608
diff --git a/tests/ports/webassembly/jsffi_create_proxy.mjs b/tests/ports/webassembly/jsffi_create_proxy.mjs
new file mode 100644
index 0000000000000..5f04bf44d7e21
--- /dev/null
+++ b/tests/ports/webassembly/jsffi_create_proxy.mjs
@@ -0,0 +1,15 @@
+// Test jsffi.create_proxy().
+
+const mp = await (await import(process.argv[2])).loadMicroPython();
+
+mp.runPython(`
+import jsffi
+x = jsffi.create_proxy(1)
+print(x)
+y = jsffi.create_proxy([2])
+print(y)
+`);
+console.log(mp.globals.get("x"));
+console.log(mp.PyProxy.toJs(mp.globals.get("x")));
+console.log(mp.globals.get("y"));
+console.log(mp.PyProxy.toJs(mp.globals.get("y")));
diff --git a/tests/ports/webassembly/jsffi_create_proxy.mjs.exp b/tests/ports/webassembly/jsffi_create_proxy.mjs.exp
new file mode 100644
index 0000000000000..a3b38a78b1132
--- /dev/null
+++ b/tests/ports/webassembly/jsffi_create_proxy.mjs.exp
@@ -0,0 +1,6 @@
+1
+
+1
+1
+PyProxy { _ref: 3 }
+[ 2 ]
diff --git a/tests/ports/webassembly/jsffi_to_js.mjs b/tests/ports/webassembly/jsffi_to_js.mjs
new file mode 100644
index 0000000000000..714af6b629937
--- /dev/null
+++ b/tests/ports/webassembly/jsffi_to_js.mjs
@@ -0,0 +1,28 @@
+// Test jsffi.to_js().
+
+const mp = await (await import(process.argv[2])).loadMicroPython();
+
+mp.runPython(`
+import jsffi
+x = jsffi.to_js(1)
+print(x)
+y = jsffi.to_js([2])
+print(y)
+z = jsffi.to_js({"three":3})
+print(z)
+`);
+
+const x = mp.globals.get("x");
+const y = mp.globals.get("y");
+const z = mp.globals.get("z");
+
+console.log(Array.isArray(x));
+console.log(x);
+
+console.log(Array.isArray(y));
+console.log(y);
+console.log(Reflect.ownKeys(y));
+
+console.log(Array.isArray(z));
+console.log(z);
+console.log(Reflect.ownKeys(z));
diff --git a/tests/ports/webassembly/jsffi_to_js.mjs.exp b/tests/ports/webassembly/jsffi_to_js.mjs.exp
new file mode 100644
index 0000000000000..399dd0aa8c9cb
--- /dev/null
+++ b/tests/ports/webassembly/jsffi_to_js.mjs.exp
@@ -0,0 +1,11 @@
+1
+
+
+false
+1
+true
+[ 2 ]
+[ '0', 'length' ]
+false
+{ three: 3 }
+[ 'three' ]
diff --git a/tests/ports/webassembly/override_new.mjs b/tests/ports/webassembly/override_new.mjs
new file mode 100644
index 0000000000000..f5d64d7d11379
--- /dev/null
+++ b/tests/ports/webassembly/override_new.mjs
@@ -0,0 +1,33 @@
+// Test overriding .new() on a JavaScript class.
+
+const mp = await (await import(process.argv[2])).loadMicroPython();
+
+globalThis.MyClass1 = class {
+    new() {
+        console.log("MyClass1 new");
+        return 1;
+    }
+};
+
+globalThis.MyClass2 = class {
+    static new() {
+        console.log("MyClass2 static new");
+        return 2;
+    }
+    new() {
+        console.log("MyClass2 new");
+        return 3;
+    }
+};
+
+globalThis.myClass2Instance = new globalThis.MyClass2();
+
+mp.runPython(`
+    import js
+
+    print(type(js.MyClass1.new()))
+    print(js.MyClass1.new().new())
+
+    print(js.MyClass2.new())
+    print(js.myClass2Instance.new())
+`);
diff --git a/tests/ports/webassembly/override_new.mjs.exp b/tests/ports/webassembly/override_new.mjs.exp
new file mode 100644
index 0000000000000..2efb669714ed4
--- /dev/null
+++ b/tests/ports/webassembly/override_new.mjs.exp
@@ -0,0 +1,7 @@
+
+MyClass1 new
+1
+MyClass2 static new
+2
+MyClass2 new
+3
diff --git a/tests/ports/webassembly/promise_with_resolvers.mjs b/tests/ports/webassembly/promise_with_resolvers.mjs
new file mode 100644
index 0000000000000..a2c6d509a6095
--- /dev/null
+++ b/tests/ports/webassembly/promise_with_resolvers.mjs
@@ -0,0 +1,23 @@
+// Test polyfill of a method on a built-in.
+
+// Implement Promise.withResolvers, and make sure it has a unique name so
+// the test below is guaranteed to use this version.
+Promise.withResolversCustom = function withResolversCustom() {
+    let a;
+    let b;
+    const c = new this((resolve, reject) => {
+        a = resolve;
+        b = reject;
+    });
+    return { resolve: a, reject: b, promise: c };
+};
+
+const mp = await (await import(process.argv[2])).loadMicroPython();
+
+mp.runPython(`
+    from js import Promise
+
+    deferred = Promise.withResolversCustom()
+    deferred.promise.then(print)
+    deferred.resolve('OK')
+`);
diff --git a/tests/ports/webassembly/promise_with_resolvers.mjs.exp b/tests/ports/webassembly/promise_with_resolvers.mjs.exp
new file mode 100644
index 0000000000000..d86bac9de59ab
--- /dev/null
+++ b/tests/ports/webassembly/promise_with_resolvers.mjs.exp
@@ -0,0 +1 @@
+OK
diff --git a/tests/ports/webassembly/py_proxy_delete.mjs b/tests/ports/webassembly/py_proxy_delete.mjs
new file mode 100644
index 0000000000000..2afea0b9f2221
--- /dev/null
+++ b/tests/ports/webassembly/py_proxy_delete.mjs
@@ -0,0 +1,30 @@
+// Test `delete .` on the JavaScript side, which tests PyProxy.deleteProperty.
+
+const mp = await (await import(process.argv[2])).loadMicroPython();
+
+mp.runPython(`
+class A:
+    pass
+x = A()
+x.foo = 1
+y = []
+`);
+
+const x = mp.globals.get("x");
+const y = mp.globals.get("y");
+
+// Should pass.
+// biome-ignore lint/performance/noDelete: test delete statement
+delete x.foo;
+
+mp.runPython(`
+print(hasattr(x, "foo"))
+`);
+
+// Should fail, can't delete attributes on MicroPython lists.
+try {
+    // biome-ignore lint/performance/noDelete: test delete statement
+    delete y.sort;
+} catch (error) {
+    console.log(error.message);
+}
diff --git a/tests/ports/webassembly/py_proxy_delete.mjs.exp b/tests/ports/webassembly/py_proxy_delete.mjs.exp
new file mode 100644
index 0000000000000..8eb9ad1501419
--- /dev/null
+++ b/tests/ports/webassembly/py_proxy_delete.mjs.exp
@@ -0,0 +1,2 @@
+False
+'deleteProperty' on proxy: trap returned falsish for property 'sort'
diff --git a/tests/ports/webassembly/py_proxy_dict.mjs b/tests/ports/webassembly/py_proxy_dict.mjs
new file mode 100644
index 0000000000000..201f179ef377b
--- /dev/null
+++ b/tests/ports/webassembly/py_proxy_dict.mjs
@@ -0,0 +1,34 @@
+// Test passing a Python dict into JavaScript, it should act like a JS object.
+
+const mp = await (await import(process.argv[2])).loadMicroPython();
+
+mp.runPython(`
+x = {"a": 1, "b": 2}
+`);
+
+const x = mp.globals.get("x");
+
+// Test has, get, keys/iteration.
+console.log("a" in x, "b" in x, "c" in x);
+console.log(x.a, x.b);
+for (const k in x) {
+    console.log(k, x[k]);
+}
+console.log(Object.keys(x));
+console.log(Reflect.ownKeys(x));
+
+// Test set.
+x.c = 3;
+console.log(Object.keys(x));
+
+// Test delete.
+// biome-ignore lint/performance/noDelete: test delete statement
+delete x.b;
+console.log(Object.keys(x));
+
+// Make sure changes on the JavaScript side are reflected in Python.
+mp.runPython(`
+print(x["a"])
+print("b" in x)
+print(x["c"])
+`);
diff --git a/tests/ports/webassembly/py_proxy_dict.mjs.exp b/tests/ports/webassembly/py_proxy_dict.mjs.exp
new file mode 100644
index 0000000000000..f0e15034bd71c
--- /dev/null
+++ b/tests/ports/webassembly/py_proxy_dict.mjs.exp
@@ -0,0 +1,11 @@
+true true false
+1 2
+a 1
+b 2
+[ 'a', 'b' ]
+[ 'a', 'b' ]
+[ 'a', 'c', 'b' ]
+[ 'a', 'c' ]
+1
+False
+3
diff --git a/tests/ports/webassembly/py_proxy_has.mjs b/tests/ports/webassembly/py_proxy_has.mjs
new file mode 100644
index 0000000000000..8881776fdbe51
--- /dev/null
+++ b/tests/ports/webassembly/py_proxy_has.mjs
@@ -0,0 +1,11 @@
+// Test ` in ` on the JavaScript side, which tests PyProxy.has.
+
+const mp = await (await import(process.argv[2])).loadMicroPython();
+
+mp.runPython(`
+x = []
+`);
+
+const x = mp.globals.get("x");
+console.log("no_exist" in x);
+console.log("sort" in x);
diff --git a/tests/ports/webassembly/py_proxy_has.mjs.exp b/tests/ports/webassembly/py_proxy_has.mjs.exp
new file mode 100644
index 0000000000000..1d474d5255713
--- /dev/null
+++ b/tests/ports/webassembly/py_proxy_has.mjs.exp
@@ -0,0 +1,2 @@
+false
+true
diff --git a/tests/ports/webassembly/py_proxy_own_keys.mjs b/tests/ports/webassembly/py_proxy_own_keys.mjs
new file mode 100644
index 0000000000000..bbf5b4f01c056
--- /dev/null
+++ b/tests/ports/webassembly/py_proxy_own_keys.mjs
@@ -0,0 +1,11 @@
+// Test `Reflect.ownKeys()` on the JavaScript side, which tests PyProxy.ownKeys.
+
+const mp = await (await import(process.argv[2])).loadMicroPython();
+
+mp.runPython(`
+x = []
+y = {"a": 1}
+`);
+
+console.log(Reflect.ownKeys(mp.globals.get("x")));
+console.log(Reflect.ownKeys(mp.globals.get("y")));
diff --git a/tests/ports/webassembly/py_proxy_own_keys.mjs.exp b/tests/ports/webassembly/py_proxy_own_keys.mjs.exp
new file mode 100644
index 0000000000000..313d06d4d75da
--- /dev/null
+++ b/tests/ports/webassembly/py_proxy_own_keys.mjs.exp
@@ -0,0 +1,9 @@
+[
+  'append', 'clear',
+  'copy',   'count',
+  'extend', 'index',
+  'insert', 'pop',
+  'remove', 'reverse',
+  'sort'
+]
+[ 'a' ]
diff --git a/tests/ports/webassembly/py_proxy_set.mjs b/tests/ports/webassembly/py_proxy_set.mjs
new file mode 100644
index 0000000000000..30360a847d806
--- /dev/null
+++ b/tests/ports/webassembly/py_proxy_set.mjs
@@ -0,0 +1,27 @@
+// Test `. = ` on the JavaScript side, which tests PyProxy.set.
+
+const mp = await (await import(process.argv[2])).loadMicroPython();
+
+mp.runPython(`
+class A:
+    pass
+x = A()
+y = []
+`);
+
+const x = mp.globals.get("x");
+const y = mp.globals.get("y");
+
+// Should pass.
+x.foo = 1;
+
+mp.runPython(`
+print(x.foo)
+`);
+
+// Should fail, can't set attributes on MicroPython lists.
+try {
+    y.bar = 1;
+} catch (error) {
+    console.log(error.message);
+}
diff --git a/tests/ports/webassembly/py_proxy_set.mjs.exp b/tests/ports/webassembly/py_proxy_set.mjs.exp
new file mode 100644
index 0000000000000..e1d995156341e
--- /dev/null
+++ b/tests/ports/webassembly/py_proxy_set.mjs.exp
@@ -0,0 +1,2 @@
+1
+'set' on proxy: trap returned falsish for property 'bar'
diff --git a/tests/ports/webassembly/py_proxy_to_js.mjs b/tests/ports/webassembly/py_proxy_to_js.mjs
new file mode 100644
index 0000000000000..f9fce606f1fdf
--- /dev/null
+++ b/tests/ports/webassembly/py_proxy_to_js.mjs
@@ -0,0 +1,20 @@
+// Test PyProxy.toJs().
+
+const mp = await (await import(process.argv[2])).loadMicroPython();
+
+mp.runPython(`
+a = 1
+b = (1, 2, 3)
+c = [None, True, 1.2]
+d = {"one": 1, "tuple": b, "list": c}
+`);
+
+const py_a = mp.globals.get("a");
+const py_b = mp.globals.get("b");
+const py_c = mp.globals.get("c");
+const py_d = mp.globals.get("d");
+
+console.log(py_a instanceof mp.PyProxy, mp.PyProxy.toJs(py_a));
+console.log(py_b instanceof mp.PyProxy, mp.PyProxy.toJs(py_b));
+console.log(py_c instanceof mp.PyProxy, mp.PyProxy.toJs(py_c));
+console.log(py_d instanceof mp.PyProxy, mp.PyProxy.toJs(py_d));
diff --git a/tests/ports/webassembly/py_proxy_to_js.mjs.exp b/tests/ports/webassembly/py_proxy_to_js.mjs.exp
new file mode 100644
index 0000000000000..279df7bdfbb84
--- /dev/null
+++ b/tests/ports/webassembly/py_proxy_to_js.mjs.exp
@@ -0,0 +1,4 @@
+false 1
+true [ 1, 2, 3 ]
+true [ null, true, 1.2 ]
+true { tuple: [ 1, 2, 3 ], one: 1, list: [ null, true, 1.2 ] }
diff --git a/tests/ports/webassembly/register_js_module.js b/tests/ports/webassembly/register_js_module.js
new file mode 100644
index 0000000000000..b512f2c0dd465
--- /dev/null
+++ b/tests/ports/webassembly/register_js_module.js
@@ -0,0 +1,6 @@
+import(process.argv[2]).then((mp) => {
+    mp.loadMicroPython().then((py) => {
+        py.registerJsModule("js_module", { y: 2 });
+        py.runPython("import js_module; print(js_module); print(js_module.y)");
+    });
+});
diff --git a/tests/ports/webassembly/register_js_module.js.exp b/tests/ports/webassembly/register_js_module.js.exp
new file mode 100644
index 0000000000000..bb45f4ce00285
--- /dev/null
+++ b/tests/ports/webassembly/register_js_module.js.exp
@@ -0,0 +1,2 @@
+
+2
diff --git a/tests/ports/webassembly/run_python_async.mjs b/tests/ports/webassembly/run_python_async.mjs
new file mode 100644
index 0000000000000..44f2a90ab175d
--- /dev/null
+++ b/tests/ports/webassembly/run_python_async.mjs
@@ -0,0 +1,115 @@
+// Test runPythonAsync() and top-level await in Python.
+
+const mp = await (await import(process.argv[2])).loadMicroPython();
+
+/**********************************************************/
+// Using only promise objects, no await's.
+
+console.log("= TEST 1 ==========");
+
+globalThis.p = new Promise((resolve, reject) => {
+    resolve(123);
+});
+
+console.log(1);
+
+mp.runPython(`
+import js
+print(js.p)
+print("py 1")
+print(js.p.then(lambda x: print("resolved", x)))
+print("py 2")
+`);
+
+console.log(2);
+
+// Let the promise resolve.
+await globalThis.p;
+
+console.log(3);
+
+/**********************************************************/
+// Using setTimeout to resolve the promise.
+
+console.log("= TEST 2 ==========");
+
+globalThis.p = new Promise((resolve, reject) => {
+    setTimeout(() => {
+        resolve(123);
+        console.log("setTimeout resolved");
+    }, 100);
+});
+
+console.log(1);
+
+mp.runPython(`
+import js
+print(js.p)
+print("py 1")
+print(js.p.then(lambda x: print("resolved", x)))
+print("py 2")
+`);
+
+console.log(2);
+
+// Let the promise resolve.
+await globalThis.p;
+
+console.log(3);
+
+/**********************************************************/
+// Using setTimeout and await within Python.
+
+console.log("= TEST 3 ==========");
+
+globalThis.p = new Promise((resolve, reject) => {
+    setTimeout(() => {
+        resolve(123);
+        console.log("setTimeout resolved");
+    }, 100);
+});
+
+console.log(1);
+
+const ret3 = await mp.runPythonAsync(`
+import js
+print("py 1")
+print("resolved value:", await js.p)
+print("py 2")
+`);
+
+console.log(2, ret3);
+
+/**********************************************************/
+// Multiple setTimeout's and await's within Python.
+
+console.log("= TEST 4 ==========");
+
+globalThis.p1 = new Promise((resolve, reject) => {
+    setTimeout(() => {
+        resolve(123);
+        console.log("setTimeout A resolved");
+    }, 100);
+});
+
+globalThis.p2 = new Promise((resolve, reject) => {
+    setTimeout(() => {
+        resolve(456);
+        console.log("setTimeout B resolved");
+    }, 200);
+});
+
+console.log(1);
+
+const ret4 = await mp.runPythonAsync(`
+import js
+print("py 1")
+print("resolved value:", await js.p1)
+print("py 2")
+print("resolved value:", await js.p1)
+print("py 3")
+print("resolved value:", await js.p2)
+print("py 4")
+`);
+
+console.log(2, ret4);
diff --git a/tests/ports/webassembly/run_python_async.mjs.exp b/tests/ports/webassembly/run_python_async.mjs.exp
new file mode 100644
index 0000000000000..f441bc5cf1db6
--- /dev/null
+++ b/tests/ports/webassembly/run_python_async.mjs.exp
@@ -0,0 +1,38 @@
+= TEST 1 ==========
+1
+
+py 1
+
+py 2
+2
+resolved 123
+3
+= TEST 2 ==========
+1
+
+py 1
+
+py 2
+2
+setTimeout resolved
+resolved 123
+3
+= TEST 3 ==========
+1
+py 1
+setTimeout resolved
+resolved value: 123
+py 2
+2 null
+= TEST 4 ==========
+1
+py 1
+setTimeout A resolved
+resolved value: 123
+py 2
+resolved value: 123
+py 3
+setTimeout B resolved
+resolved value: 456
+py 4
+2 null
diff --git a/tests/ports/webassembly/run_python_async2.mjs b/tests/ports/webassembly/run_python_async2.mjs
new file mode 100644
index 0000000000000..87067e6e8c403
--- /dev/null
+++ b/tests/ports/webassembly/run_python_async2.mjs
@@ -0,0 +1,42 @@
+// Test runPythonAsync() and top-level await in Python, with multi-level awaits.
+
+const mp = await (await import(process.argv[2])).loadMicroPython();
+
+// simulate a 2-step resolution of the string "OK".
+await mp.runPythonAsync(`
+import js
+
+def _timeout(resolve, _):
+    js.setTimeout(resolve, 100)
+
+def _fetch():
+    return js.Promise.new(_timeout)
+
+async def _text(promise):
+    if not promise._response:
+        print("_text await start")
+        await promise
+        print("_text awaited end")
+    ret = await promise._response.text()
+    return ret
+
+class _Response:
+    async def text(self):
+        print('_Response.text start')
+        await js.Promise.new(_timeout)
+        print('_Response.text end')
+        return "OK"
+
+def _response(promise):
+    promise._response = _Response()
+    return promise._response
+
+def fetch(url):
+    promise = _fetch().then(lambda *_: _response(promise))
+    promise._response = None
+    promise.text = lambda: _text(promise)
+    return promise
+
+print(await fetch("config.json").text())
+print(await (await fetch("config.json")).text())
+`);
diff --git a/tests/ports/webassembly/run_python_async2.mjs.exp b/tests/ports/webassembly/run_python_async2.mjs.exp
new file mode 100644
index 0000000000000..60d68c5d3b55c
--- /dev/null
+++ b/tests/ports/webassembly/run_python_async2.mjs.exp
@@ -0,0 +1,8 @@
+_text await start
+_text awaited end
+_Response.text start
+_Response.text end
+OK
+_Response.text start
+_Response.text end
+OK
diff --git a/tests/ports/webassembly/this_behaviour.mjs b/tests/ports/webassembly/this_behaviour.mjs
new file mode 100644
index 0000000000000..6411b6ce63485
--- /dev/null
+++ b/tests/ports/webassembly/this_behaviour.mjs
@@ -0,0 +1,24 @@
+// Test "this" behaviour.
+
+const mp = await (await import(process.argv[2])).loadMicroPython();
+
+// "this" should be undefined.
+globalThis.func0 = function () {
+    console.log("func0", this);
+};
+mp.runPython("import js; js.func0()");
+
+globalThis.func1 = function (a) {
+    console.log("func1", a, this);
+};
+mp.runPython("import js; js.func1(123)");
+
+globalThis.func2 = function (a, b) {
+    console.log("func2", a, b, this);
+};
+mp.runPython("import js; js.func2(123, 456)");
+
+globalThis.func3 = function (a, b, c) {
+    console.log("func3", a, b, c, this);
+};
+mp.runPython("import js; js.func3(123, 456, 789)");
diff --git a/tests/ports/webassembly/this_behaviour.mjs.exp b/tests/ports/webassembly/this_behaviour.mjs.exp
new file mode 100644
index 0000000000000..4762026ab2027
--- /dev/null
+++ b/tests/ports/webassembly/this_behaviour.mjs.exp
@@ -0,0 +1,4 @@
+func0 undefined
+func1 123 undefined
+func2 123 456 undefined
+func3 123 456 789 undefined
diff --git a/tests/ports/webassembly/various.js b/tests/ports/webassembly/various.js
new file mode 100644
index 0000000000000..e2fa9362c4f3d
--- /dev/null
+++ b/tests/ports/webassembly/various.js
@@ -0,0 +1,38 @@
+import(process.argv[2]).then((mp) => {
+    mp.loadMicroPython().then((py) => {
+        globalThis.jsadd = (x, y) => {
+            return x + y;
+        };
+        py.runPython("import js; print(js); print(js.jsadd(4, 9))");
+
+        py.runPython(
+            "def set_timeout_callback():\n print('set_timeout_callback')",
+        );
+        py.runPython("import js; js.setTimeout(set_timeout_callback, 100)");
+
+        py.runPython("obj = js.Object(a=1)");
+        console.log("main", py.pyimport("__main__").obj);
+
+        console.log("=======");
+        py.runPython(`
+            from js import Array, Promise, Reflect
+
+            def callback(resolve, reject):
+                resolve('OK1')
+
+            p = Reflect.construct(Promise, Array(callback))
+            p.then(print)
+        `);
+
+        console.log("=======");
+        py.runPython(`
+            from js import Promise
+
+            def callback(resolve, reject):
+                resolve('OK2')
+
+            p = Promise.new(callback)
+            p.then(print)
+        `);
+    });
+});
diff --git a/tests/ports/webassembly/various.js.exp b/tests/ports/webassembly/various.js.exp
new file mode 100644
index 0000000000000..502ab2cccf000
--- /dev/null
+++ b/tests/ports/webassembly/various.js.exp
@@ -0,0 +1,8 @@
+
+13
+main { a: 1 }
+=======
+=======
+OK1
+OK2
+set_timeout_callback
diff --git a/tests/run-tests.py b/tests/run-tests.py
index 83af61c83d20b..4f55cdd39842e 100755
--- a/tests/run-tests.py
+++ b/tests/run-tests.py
@@ -1142,7 +1142,7 @@ def main():
                     "ports/qemu-arm",
                 )
             elif args.target == "webassembly":
-                test_dirs += ("float",)
+                test_dirs += ("float", "ports/webassembly")
         else:
             # run tests from these directories
             test_dirs = args.test_dirs

From badc0106bdc20e95d427184395c43ffd9d9beea8 Mon Sep 17 00:00:00 2001
From: Damien George 
Date: Fri, 2 Feb 2024 12:03:25 +1100
Subject: [PATCH 18/19] tools/ci.sh: Update webassembly CI tests.

Signed-off-by: Damien George 
---
 tools/ci.sh | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/tools/ci.sh b/tools/ci.sh
index f94f23893b710..ad5e79c4ad07c 100755
--- a/tools/ci.sh
+++ b/tools/ci.sh
@@ -178,18 +178,19 @@ function ci_esp8266_build {
 # ports/webassembly
 
 function ci_webassembly_setup {
+    npm install terser
     git clone https://github.com/emscripten-core/emsdk.git
     (cd emsdk && ./emsdk install latest && ./emsdk activate latest)
 }
 
 function ci_webassembly_build {
     source emsdk/emsdk_env.sh
-    make ${MAKEOPTS} -C ports/webassembly
+    make ${MAKEOPTS} -C ports/webassembly VARIANT=pyscript submodules
+    make ${MAKEOPTS} -C ports/webassembly VARIANT=pyscript
 }
 
 function ci_webassembly_run_tests {
-    # This port is very slow at running, so only run a few of the tests.
-    (cd tests && MICROPY_MICROPYTHON=../ports/webassembly/node_run.sh ./run-tests.py -j1 basics/builtin_*.py)
+    make -C ports/webassembly VARIANT=pyscript test_min
 }
 
 ########################################################################################

From 35b2edfc240050fc5310093db29927f6226c3157 Mon Sep 17 00:00:00 2001
From: Damien George 
Date: Wed, 21 Feb 2024 13:00:50 +1100
Subject: [PATCH 19/19] github/workflows: Add Biome workflow for JavaScript
 formatting/linting.

Enable Biome on all of webassembly port and tests.

Signed-off-by: Damien George 
---
 .github/workflows/biome.yml | 16 ++++++++++++++++
 1 file changed, 16 insertions(+)
 create mode 100644 .github/workflows/biome.yml

diff --git a/.github/workflows/biome.yml b/.github/workflows/biome.yml
new file mode 100644
index 0000000000000..88744f16ca7d6
--- /dev/null
+++ b/.github/workflows/biome.yml
@@ -0,0 +1,16 @@
+name: JavaScript code lint and formatting with Biome
+
+on: [push, pull_request]
+
+jobs:
+  eslint:
+    runs-on: ubuntu-latest
+    steps:
+    - name: Checkout
+      uses: actions/checkout@v4
+    - name: Setup Biome
+      uses: biomejs/setup-biome@v2
+      with:
+        version: 1.5.3
+    - name: Run Biome
+      run: biome ci --indent-style=space --indent-width=4 tests/ ports/webassembly




pFad - Phonifier reborn




Pfad - The Proxy pFad of © 2024 Garber Painting. All rights reserved.





Note: This service is not intended for secure transactions such as banking, social media, email, or purchasing. Use at your own risk. We assume no liability whatsoever for broken pages.




Alternative Proxies:
Alternative Proxy
pFad Proxy
pFad v3 Proxy
pFad v4 Proxy