diff --git a/.codespell/ignore-words.txt b/.codespell/ignore-words.txt index fc8feaca9780b..48bee0f30ba7a 100644 --- a/.codespell/ignore-words.txt +++ b/.codespell/ignore-words.txt @@ -26,3 +26,4 @@ ftbfs straightaway ftbs ftb +curren diff --git a/LICENSE b/LICENSE index df12f02238a51..90d19f757b2db 100644 --- a/LICENSE +++ b/LICENSE @@ -1,6 +1,6 @@ The MIT License (MIT) -Copyright (c) 2013-2024 Damien P. George +Copyright (c) 2013-2025 Damien P. George Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/Makefile b/Makefile index e6ddeffbe8604..a6599a2b09fd6 100644 --- a/Makefile +++ b/Makefile @@ -271,20 +271,20 @@ check-translate: .PHONY: stubs stubs: - @rm -rf circuitpython-stubs - @mkdir circuitpython-stubs - @$(PYTHON) tools/extract_pyi.py shared-bindings/ $(STUBDIR) - @$(PYTHON) tools/extract_pyi.py extmod/ulab/code/ $(STUBDIR)/ulab - @for d in ports/*/bindings; do \ + rm -rf circuitpython-stubs + mkdir circuitpython-stubs + $(PYTHON) tools/extract_pyi.py shared-bindings/ $(STUBDIR) + $(PYTHON) tools/extract_pyi.py extmod/ulab/code/ $(STUBDIR)/ulab + for d in ports/*/bindings; do \ $(PYTHON) tools/extract_pyi.py "$$d" $(STUBDIR); done - @sed -e "s,__version__,`python -msetuptools_scm`," < setup.py-stubs > circuitpython-stubs/setup.py - @cp README.rst-stubs circuitpython-stubs/README.rst - @cp MANIFEST.in-stubs circuitpython-stubs/MANIFEST.in - @$(PYTHON) tools/board_stubs/build_board_specific_stubs/board_stub_builder.py - @cp -r tools/board_stubs/circuitpython_setboard circuitpython-stubs/circuitpython_setboard - @$(PYTHON) -m build circuitpython-stubs - @touch circuitpython-stubs/board/__init__.py - @touch circuitpython-stubs/board_definitions/__init__.py + sed -e "s,__version__,`python -msetuptools_scm`," < setup.py-stubs > circuitpython-stubs/setup.py + cp README.rst-stubs circuitpython-stubs/README.rst + cp MANIFEST.in-stubs circuitpython-stubs/MANIFEST.in + $(PYTHON) tools/board_stubs/build_board_specific_stubs/board_stub_builder.py + cp -r tools/board_stubs/circuitpython_setboard circuitpython-stubs/circuitpython_setboard + $(PYTHON) -m build circuitpython-stubs + touch circuitpython-stubs/board/__init__.py + touch circuitpython-stubs/board_definitions/__init__.py .PHONY: check-stubs check-stubs: stubs diff --git a/docs/library/array.rst b/docs/library/array.rst index a7a3b5952e150..63b86a806c3d3 100644 --- a/docs/library/array.rst +++ b/docs/library/array.rst @@ -19,6 +19,10 @@ Classes array are given by an `iterable`. If it is not provided, an empty array is created. + In addition to the methods below, array objects also implement the buffer + protocol. This means the contents of the entire array can be accessed as raw + bytes via a `memoryview` or other interfaces which use this protocol. + .. method:: append(val) Append new element ``val`` to the end of array, growing it. diff --git a/docs/library/binascii.rst b/docs/library/binascii.rst index 1eba46652d7fc..0797692b02030 100644 --- a/docs/library/binascii.rst +++ b/docs/library/binascii.rst @@ -39,6 +39,6 @@ Functions .. function:: crc32(data, value=0, /) - Compute CRC-32, the 32-bit checksum of the bytes in *data* starting with an + Compute CRC-32, the 32-bit checksum of the bytes in ``data`` starting with an initial CRC of *value*. The default initial CRC is 0. The algorithm is consistent with the ZIP file checksum. diff --git a/docs/library/builtins.rst b/docs/library/builtins.rst index 97ca0044ec389..148c1513f3d4e 100644 --- a/docs/library/builtins.rst +++ b/docs/library/builtins.rst @@ -31,6 +31,8 @@ Functions and types .. class:: bytearray() + |see_cpython| `python:bytearray`. + .. class:: bytes() |see_cpython| `python:bytes`. diff --git a/docs/library/index.rst b/docs/library/index.rst index 18b2530a9b596..33d02115cc940 100644 --- a/docs/library/index.rst +++ b/docs/library/index.rst @@ -33,8 +33,8 @@ These libraries are not currently enabled in any CircuitPython build, but may be json.rst platform.rst re.rst - sys.rst select.rst + sys.rst Omitted ``string`` functions ---------------------------- diff --git a/extmod/extmod.mk b/extmod/extmod.mk index a64c1d20388ac..daa6f267342ae 100644 --- a/extmod/extmod.mk +++ b/extmod/extmod.mk @@ -24,7 +24,6 @@ SRC_EXTMOD_C += \ extmod/vfs_posix.c \ extmod/vfs_posix_file.c \ extmod/vfs_reader.c \ - extmod/virtpin.c \ shared/libc/abort_.c \ shared/libc/printf.c \ @@ -253,6 +252,7 @@ SRC_THIRDPARTY_C += $(addprefix $(MBEDTLS_DIR)/library/,\ pkcs12.c \ pkcs5.c \ pkparse.c \ + pk_ecc.c \ pk_wrap.c \ pkwrite.c \ platform.c \ @@ -411,15 +411,14 @@ CYW43_DIR = lib/cyw43-driver GIT_SUBMODULES += $(CYW43_DIR) CFLAGS_EXTMOD += -DMICROPY_PY_NETWORK_CYW43=1 SRC_THIRDPARTY_C += $(addprefix $(CYW43_DIR)/src/,\ + cyw43_bthci_uart.c \ cyw43_ctrl.c \ cyw43_lwip.c \ cyw43_ll.c \ cyw43_sdio.c \ + cyw43_spi.c \ cyw43_stats.c \ ) -ifeq ($(MICROPY_PY_BLUETOOTH),1) -DRIVERS_SRC_C += drivers/cyw43/cywbt.c -endif $(BUILD)/$(CYW43_DIR)/src/cyw43_%.o: CFLAGS += -std=c11 endif # MICROPY_PY_NETWORK_CYW43 @@ -577,6 +576,6 @@ $(BUILD)/$(OPENAMP_DIR)/lib/virtio_mmio/virtio_mmio_drv.o: CFLAGS += -Wno-unused # We need to have generated libmetal before compiling OpenAMP. $(addprefix $(BUILD)/, $(SRC_OPENAMP_C:.c=.o)): $(BUILD)/openamp/metal/config.h -SRC_THIRDPARTY_C += $(SRC_LIBMETAL_C) $(SRC_OPENAMP_C) +SRC_THIRDPARTY_C += $(SRC_OPENAMP_C) $(SRC_LIBMETAL_C:$(BUILD)/%=%) endif # MICROPY_PY_OPENAMP diff --git a/extmod/lwip-include/lwipopts_common.h b/extmod/lwip-include/lwipopts_common.h new file mode 100644 index 0000000000000..3e4230909499e --- /dev/null +++ b/extmod/lwip-include/lwipopts_common.h @@ -0,0 +1,111 @@ +/* + * This file is part of the MicroPython project, http://micropython.org/ + * + * The MIT License (MIT) + * + * Copyright (c) 2025 Damien P. George + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ +#ifndef MICROPY_INCLUDED_LWIPOPTS_COMMON_H +#define MICROPY_INCLUDED_LWIPOPTS_COMMON_H + +#include "py/mpconfig.h" + +// This sys-arch protection is not needed. +// Ports either protect lwIP code with flags, or run it at PendSV priority. +#define SYS_ARCH_DECL_PROTECT(lev) do { } while (0) +#define SYS_ARCH_PROTECT(lev) do { } while (0) +#define SYS_ARCH_UNPROTECT(lev) do { } while (0) + +#define NO_SYS 1 +#define SYS_LIGHTWEIGHT_PROT 1 +#define MEM_ALIGNMENT 4 + +#define LWIP_CHKSUM_ALGORITHM 3 +#define LWIP_CHECKSUM_CTRL_PER_NETIF 1 + +#define LWIP_ARP 1 +#define LWIP_ETHERNET 1 +#define LWIP_RAW 1 +#define LWIP_NETCONN 0 +#define LWIP_SOCKET 0 +#define LWIP_STATS 0 +#define LWIP_NETIF_HOSTNAME 1 + +#define LWIP_DHCP 1 +#define LWIP_DHCP_CHECK_LINK_UP 1 +#define LWIP_DHCP_DOES_ACD_CHECK 0 // to speed DHCP up +#define LWIP_DNS 1 +#define LWIP_DNS_SUPPORT_MDNS_QUERIES 1 +#define LWIP_MDNS_RESPONDER 1 +#define LWIP_IGMP 1 + +#if MICROPY_PY_LWIP_PPP +#define PPP_SUPPORT 1 +#define PAP_SUPPORT 1 +#define CHAP_SUPPORT 1 +#endif + +#define LWIP_NUM_NETIF_CLIENT_DATA LWIP_MDNS_RESPONDER +#define MEMP_NUM_UDP_PCB (4 + LWIP_MDNS_RESPONDER) + +// The mDNS responder requires 5 timers per IP version plus 2 others. Not having enough silently breaks it. +#define MEMP_NUM_SYS_TIMEOUT (LWIP_NUM_SYS_TIMEOUT_INTERNAL + (LWIP_MDNS_RESPONDER * (2 + (5 * (LWIP_IPV4 + LWIP_IPV6))))) + +#define SO_REUSE 1 +#define TCP_LISTEN_BACKLOG 1 + +// TCP memory settings. +// Default lwIP settings takes 15800 bytes; TCP d/l: 380k/s local, 7.2k/s remote; TCP u/l is very slow. +#ifndef MEM_SIZE + +#if 0 +// lwIP takes 19159 bytes; TCP d/l and u/l are around 320k/s on local network. +#define MEM_SIZE (5000) +#define TCP_WND (4 * TCP_MSS) +#define TCP_SND_BUF (4 * TCP_MSS) +#endif + +#if 1 +// lwIP takes 26700 bytes; TCP dl/ul are around 750/600 k/s on local network. +#define MEM_SIZE (8000) +#define TCP_MSS (800) +#define TCP_WND (8 * TCP_MSS) +#define TCP_SND_BUF (8 * TCP_MSS) +#define MEMP_NUM_TCP_SEG (32) +#endif + +#if 0 +// lwIP takes 45600 bytes; TCP dl/ul are around 1200/1000 k/s on local network. +#define MEM_SIZE (16000) +#define TCP_MSS (1460) +#define TCP_WND (8 * TCP_MSS) +#define TCP_SND_BUF (8 * TCP_MSS) +#define MEMP_NUM_TCP_SEG (32) +#endif + +#endif // MEM_SIZE + +// Needed for PPP. +#define sys_jiffies sys_now + +typedef uint32_t sys_prot_t; + +#endif // MICROPY_INCLUDED_LWIPOPTS_COMMON_H diff --git a/extmod/moddeflate.c b/extmod/moddeflate.c index c0c3bb26a8b12..920b898b2ccc2 100644 --- a/extmod/moddeflate.c +++ b/extmod/moddeflate.c @@ -142,7 +142,7 @@ static bool deflateio_init_read(mp_obj_deflateio_t *self) { } } - size_t window_len = 1 << wbits; + size_t window_len = (size_t)1 << wbits; self->read->window = m_new(uint8_t, window_len); uzlib_uncompress_init(&self->read->decomp, self->read->window, window_len); @@ -168,16 +168,20 @@ static bool deflateio_init_write(mp_obj_deflateio_t *self) { const mp_stream_p_t *stream = mp_get_stream_raise(self->stream, MP_STREAM_OP_WRITE); - self->write = m_new_obj(mp_obj_deflateio_write_t); - self->write->input_len = 0; - int wbits = self->window_bits; if (wbits == 0) { // Same default wbits for all formats. wbits = DEFLATEIO_DEFAULT_WBITS; } + + // Allocate the large window before allocating the mp_obj_deflateio_write_t, in case the + // window allocation fails the mp_obj_deflateio_t object will remain in a consistent state. size_t window_len = 1 << wbits; - self->write->window = m_new(uint8_t, window_len); + uint8_t *window = m_new(uint8_t, window_len); + + self->write = m_new_obj(mp_obj_deflateio_write_t); + self->write->window = window; + self->write->input_len = 0; uzlib_lz77_init(&self->write->lz77, self->write->window, window_len); self->write->lz77.dest_write_data = self; diff --git a/extmod/modos.c b/extmod/modos.c index e7f7fc818cf08..69fdc3fac0a06 100644 --- a/extmod/modos.c +++ b/extmod/modos.c @@ -171,13 +171,15 @@ static const mp_rom_map_elem_t os_module_globals_table[] = { { MP_ROM_QSTR(MP_QSTR_chdir), MP_ROM_PTR(&mp_vfs_chdir_obj) }, { MP_ROM_QSTR(MP_QSTR_getcwd), MP_ROM_PTR(&mp_vfs_getcwd_obj) }, { MP_ROM_QSTR(MP_QSTR_listdir), MP_ROM_PTR(&mp_vfs_listdir_obj) }, + #if MICROPY_VFS_WRITABLE { MP_ROM_QSTR(MP_QSTR_mkdir), MP_ROM_PTR(&mp_vfs_mkdir_obj) }, { MP_ROM_QSTR(MP_QSTR_remove), MP_ROM_PTR(&mp_vfs_remove_obj) }, { MP_ROM_QSTR(MP_QSTR_rename), MP_ROM_PTR(&mp_vfs_rename_obj) }, { MP_ROM_QSTR(MP_QSTR_rmdir), MP_ROM_PTR(&mp_vfs_rmdir_obj) }, + { MP_ROM_QSTR(MP_QSTR_unlink), MP_ROM_PTR(&mp_vfs_remove_obj) }, // unlink aliases to remove + #endif { MP_ROM_QSTR(MP_QSTR_stat), MP_ROM_PTR(&mp_vfs_stat_obj) }, { MP_ROM_QSTR(MP_QSTR_statvfs), MP_ROM_PTR(&mp_vfs_statvfs_obj) }, - { MP_ROM_QSTR(MP_QSTR_unlink), MP_ROM_PTR(&mp_vfs_remove_obj) }, // unlink aliases to remove #endif // The following are MicroPython extensions. diff --git a/extmod/modplatform.h b/extmod/modplatform.h index b932551c7ccb2..a155f071cba1b 100644 --- a/extmod/modplatform.h +++ b/extmod/modplatform.h @@ -36,7 +36,11 @@ // See: https://sourceforge.net/p/predef/wiki/Home/ #if defined(__ARM_ARCH) +#if defined(__ARM_ARCH_ISA_A64) +#define MICROPY_PLATFORM_ARCH "aarch64" +#else #define MICROPY_PLATFORM_ARCH "arm" +#endif #elif defined(__x86_64__) || defined(_M_X64) #define MICROPY_PLATFORM_ARCH "x86_64" #elif defined(__i386__) || defined(_M_IX86) @@ -44,12 +48,22 @@ #elif defined(__xtensa__) #define MICROPY_PLATFORM_ARCH "xtensa" #elif defined(__riscv) +#if __riscv_xlen == 64 +#define MICROPY_PLATFORM_ARCH "riscv64" +#else #define MICROPY_PLATFORM_ARCH "riscv" +#endif #else #define MICROPY_PLATFORM_ARCH "" #endif -#if defined(__GNUC__) +#if defined(__clang__) +#define MICROPY_PLATFORM_COMPILER \ + "Clang " \ + MP_STRINGIFY(__clang_major__) "." \ + MP_STRINGIFY(__clang_minor__) "." \ + MP_STRINGIFY(__clang_patchlevel__) +#elif defined(__GNUC__) #define MICROPY_PLATFORM_COMPILER \ "GCC " \ MP_STRINGIFY(__GNUC__) "." \ @@ -86,12 +100,17 @@ #elif defined(_PICOLIBC__) #define MICROPY_PLATFORM_LIBC_LIB "picolibc" #define MICROPY_PLATFORM_LIBC_VER _PICOLIBC_VERSION +#elif defined(__ANDROID__) +#define MICROPY_PLATFORM_LIBC_LIB "bionic" +#define MICROPY_PLATFORM_LIBC_VER MP_STRINGIFY(__ANDROID_API__) #else #define MICROPY_PLATFORM_LIBC_LIB "" #define MICROPY_PLATFORM_LIBC_VER "" #endif -#if defined(__linux) +#if defined(__ANDROID__) +#define MICROPY_PLATFORM_SYSTEM "Android" +#elif defined(__linux) #define MICROPY_PLATFORM_SYSTEM "Linux" #elif defined(__unix__) #define MICROPY_PLATFORM_SYSTEM "Unix" diff --git a/extmod/vfs.c b/extmod/vfs.c index 3d44f623703c2..0cf93c3966c90 100644 --- a/extmod/vfs.c +++ b/extmod/vfs.c @@ -42,8 +42,7 @@ #include "extmod/vfs_lfs.h" #endif -// CIRCUITPY-CHANGE: handle undefined without a warning -#if defined(MICROPY_VFS_POSIX) && MICROPY_VFS_POSIX +#if MICROPY_VFS_POSIX #include "extmod/vfs_posix.h" #endif @@ -212,22 +211,36 @@ static mp_obj_t mp_vfs_autodetect(mp_obj_t bdev_obj) { } mp_obj_t mp_vfs_mount(size_t n_args, const mp_obj_t *pos_args, mp_map_t *kw_args) { - enum { ARG_readonly, ARG_mkfs }; + if (n_args == 0) { + // zero-args, output a table of all current mountpoints + mp_obj_t mount_list = mp_obj_new_list(0, NULL); + mp_vfs_mount_t *vfsp = MP_STATE_VM(vfs_mount_table); + while (vfsp != NULL) { + mp_obj_t items[] = { vfsp->obj, mp_obj_new_str(vfsp->str, vfsp->len) }; + mp_obj_list_append(mount_list, mp_obj_new_tuple(MP_ARRAY_SIZE(items), items)); + vfsp = vfsp->next; + } + return mount_list; + } + + enum { ARG_fsobj, ARG_mount_point, ARG_readonly, ARG_mkfs }; static const mp_arg_t allowed_args[] = { + { MP_QSTR_, MP_ARG_REQUIRED | MP_ARG_OBJ, {.u_obj = MP_OBJ_NULL} }, + { MP_QSTR_, MP_ARG_REQUIRED | MP_ARG_OBJ, {.u_obj = MP_OBJ_NULL} }, { MP_QSTR_readonly, MP_ARG_KW_ONLY | MP_ARG_OBJ, {.u_rom_obj = MP_ROM_FALSE} }, { MP_QSTR_mkfs, MP_ARG_KW_ONLY | MP_ARG_OBJ, {.u_rom_obj = MP_ROM_FALSE} }, }; // parse args mp_arg_val_t args[MP_ARRAY_SIZE(allowed_args)]; - mp_arg_parse_all(n_args - 2, pos_args + 2, kw_args, MP_ARRAY_SIZE(allowed_args), allowed_args, args); + mp_arg_parse_all(n_args, pos_args, kw_args, MP_ARRAY_SIZE(allowed_args), allowed_args, args); // get the mount point size_t mnt_len; - const char *mnt_str = mp_obj_str_get_data(pos_args[1], &mnt_len); + const char *mnt_str = mp_obj_str_get_data(args[ARG_mount_point].u_obj, &mnt_len); // see if we need to auto-detect and create the filesystem - mp_obj_t vfs_obj = pos_args[0]; + mp_obj_t vfs_obj = args[ARG_fsobj].u_obj; mp_obj_t dest[2]; mp_load_method_maybe(vfs_obj, MP_QSTR_mount, dest); if (dest[0] == MP_OBJ_NULL) { @@ -244,11 +257,13 @@ mp_obj_t mp_vfs_mount(size_t n_args, const mp_obj_t *pos_args, mp_map_t *kw_args vfs->next = NULL; // call the underlying object to do any mounting operation - mp_vfs_proxy_call(vfs, MP_QSTR_mount, 2, (mp_obj_t *)&args); + mp_arg_val_t *proxy_args = &args[ARG_readonly]; + size_t proxy_args_len = MP_ARRAY_SIZE(args) - ARG_readonly; + mp_vfs_proxy_call(vfs, MP_QSTR_mount, proxy_args_len, (mp_obj_t *)proxy_args); // check that the destination mount point is unused const char *path_out; - mp_vfs_mount_t *existing_mount = mp_vfs_lookup_path(mp_obj_str_get_str(pos_args[1]), &path_out); + mp_vfs_mount_t *existing_mount = mp_vfs_lookup_path(mp_obj_str_get_str(args[ARG_mount_point].u_obj), &path_out); if (existing_mount != MP_VFS_NONE && existing_mount != MP_VFS_ROOT) { if (vfs->len != 1 && existing_mount->len == 1) { // if root dir is mounted, still allow to mount something within a subdir of root @@ -272,7 +287,7 @@ mp_obj_t mp_vfs_mount(size_t n_args, const mp_obj_t *pos_args, mp_map_t *kw_args return mp_const_none; } -MP_DEFINE_CONST_FUN_OBJ_KW(mp_vfs_mount_obj, 2, mp_vfs_mount); +MP_DEFINE_CONST_FUN_OBJ_KW(mp_vfs_mount_obj, 0, mp_vfs_mount); mp_obj_t mp_vfs_umount(mp_obj_t mnt_in) { // remove vfs from the mount table @@ -320,8 +335,7 @@ mp_obj_t mp_vfs_open(size_t n_args, const mp_obj_t *pos_args, mp_map_t *kw_args) mp_arg_val_t args[MP_ARRAY_SIZE(allowed_args)]; mp_arg_parse_all(n_args, pos_args, kw_args, MP_ARRAY_SIZE(allowed_args), allowed_args, args); - // CIRCUITPY-CHANGE: handle undefined without a warning - #if defined(MICROPY_VFS_POSIX) && MICROPY_VFS_POSIX + #if MICROPY_VFS_POSIX // If the file is an integer then delegate straight to the POSIX handler if (mp_obj_is_small_int(args[ARG_file].u_obj)) { return mp_vfs_posix_file_open(&mp_type_vfs_posix_textio, args[ARG_file].u_obj, args[ARG_mode].u_obj); @@ -454,6 +468,8 @@ mp_obj_t mp_vfs_listdir(size_t n_args, const mp_obj_t *args) { } MP_DEFINE_CONST_FUN_OBJ_VAR_BETWEEN(mp_vfs_listdir_obj, 0, 1, mp_vfs_listdir); +#if MICROPY_VFS_WRITABLE + mp_obj_t mp_vfs_mkdir(mp_obj_t path_in) { // CIRCUITPY-CHANGE: initialize mp_obj_t path_out = mp_const_none; @@ -491,6 +507,8 @@ mp_obj_t mp_vfs_rmdir(mp_obj_t path_in) { } MP_DEFINE_CONST_FUN_OBJ_1(mp_vfs_rmdir_obj, mp_vfs_rmdir); +#endif // MICROPY_VFS_WRITABLE + mp_obj_t mp_vfs_stat(mp_obj_t path_in) { mp_obj_t path_out; mp_vfs_mount_t *vfs = lookup_path(path_in, &path_out); @@ -560,6 +578,32 @@ int mp_vfs_mount_and_chdir_protected(mp_obj_t bdev, mp_obj_t mount_point) { return ret; } +#if MICROPY_VFS_ROM && MICROPY_VFS_ROM_IOCTL + +int mp_vfs_mount_romfs_protected(void) { + int ret; + nlr_buf_t nlr; + if (nlr_push(&nlr) == 0) { + mp_obj_t args[2] = { MP_OBJ_NEW_SMALL_INT(MP_VFS_ROM_IOCTL_GET_SEGMENT), MP_OBJ_NEW_SMALL_INT(0) }; + mp_obj_t rom = mp_vfs_rom_ioctl(2, args); + mp_obj_t romfs = mp_call_function_1(MP_OBJ_FROM_PTR(&mp_type_vfs_rom), rom); + mp_obj_t mount_point = MP_OBJ_NEW_QSTR(MP_QSTR__slash_rom); + mp_call_function_2(MP_OBJ_FROM_PTR(&mp_vfs_mount_obj), romfs, mount_point); + #if MICROPY_PY_SYS_PATH_ARGV_DEFAULTS + // Add "/rom" and "/rom/lib" to `sys.path`. + mp_obj_list_append(mp_sys_path, MP_OBJ_NEW_QSTR(MP_QSTR__slash_rom)); + mp_obj_list_append(mp_sys_path, MP_OBJ_NEW_QSTR(MP_QSTR__slash_rom_slash_lib)); + #endif + ret = 0; // success + nlr_pop(); + } else { + ret = -MP_EIO; + } + return ret; +} + +#endif + MP_REGISTER_ROOT_POINTER(struct _mp_vfs_mount_t *vfs_cur); MP_REGISTER_ROOT_POINTER(struct _mp_vfs_mount_t *vfs_mount_table); diff --git a/extmod/vfs.h b/extmod/vfs.h index e75801db901dd..67d5d9239a33e 100644 --- a/extmod/vfs.h +++ b/extmod/vfs.h @@ -61,6 +61,13 @@ #define MP_BLOCKDEV_IOCTL_BLOCK_SIZE (5) #define MP_BLOCKDEV_IOCTL_BLOCK_ERASE (6) +// Constants for vfs.rom_ioctl() function. +#define MP_VFS_ROM_IOCTL_GET_NUMBER_OF_SEGMENTS (1) // rom_ioctl(1) +#define MP_VFS_ROM_IOCTL_GET_SEGMENT (2) // rom_ioctl(2, ) +#define MP_VFS_ROM_IOCTL_WRITE_PREPARE (3) // rom_ioctl(3, , ) +#define MP_VFS_ROM_IOCTL_WRITE (4) // rom_ioctl(4, , , ) +#define MP_VFS_ROM_IOCTL_WRITE_COMPLETE (5) // rom_ioctl(5, ) + // At the moment the VFS protocol just has import_stat, but could be extended to other methods typedef struct _mp_vfs_proto_t { // CIRCUITPY-CHANGE @@ -110,14 +117,19 @@ mp_obj_t mp_vfs_chdir(mp_obj_t path_in); mp_obj_t mp_vfs_getcwd(void); mp_obj_t mp_vfs_ilistdir(size_t n_args, const mp_obj_t *args); mp_obj_t mp_vfs_listdir(size_t n_args, const mp_obj_t *args); +#if MICROPY_VFS_WRITABLE mp_obj_t mp_vfs_mkdir(mp_obj_t path_in); mp_obj_t mp_vfs_remove(mp_obj_t path_in); mp_obj_t mp_vfs_rename(mp_obj_t old_path_in, mp_obj_t new_path_in); mp_obj_t mp_vfs_rmdir(mp_obj_t path_in); +#endif mp_obj_t mp_vfs_stat(mp_obj_t path_in); mp_obj_t mp_vfs_statvfs(mp_obj_t path_in); int mp_vfs_mount_and_chdir_protected(mp_obj_t bdev, mp_obj_t mount_point); +#if MICROPY_VFS_ROM && MICROPY_VFS_ROM_IOCTL +int mp_vfs_mount_romfs_protected(void); +#endif MP_DECLARE_CONST_FUN_OBJ_KW(mp_vfs_mount_obj); MP_DECLARE_CONST_FUN_OBJ_1(mp_vfs_umount_obj); @@ -126,11 +138,21 @@ MP_DECLARE_CONST_FUN_OBJ_1(mp_vfs_chdir_obj); MP_DECLARE_CONST_FUN_OBJ_0(mp_vfs_getcwd_obj); MP_DECLARE_CONST_FUN_OBJ_VAR_BETWEEN(mp_vfs_ilistdir_obj); MP_DECLARE_CONST_FUN_OBJ_VAR_BETWEEN(mp_vfs_listdir_obj); +#if MICROPY_VFS_WRITABLE MP_DECLARE_CONST_FUN_OBJ_1(mp_vfs_mkdir_obj); MP_DECLARE_CONST_FUN_OBJ_1(mp_vfs_remove_obj); MP_DECLARE_CONST_FUN_OBJ_2(mp_vfs_rename_obj); MP_DECLARE_CONST_FUN_OBJ_1(mp_vfs_rmdir_obj); +#endif MP_DECLARE_CONST_FUN_OBJ_1(mp_vfs_stat_obj); MP_DECLARE_CONST_FUN_OBJ_1(mp_vfs_statvfs_obj); +#if MICROPY_VFS_ROM_IOCTL +// When MICROPY_VFS_ROM_IOCTL is enabled a port must define the following function. +// This is a generic interface to allow querying and modifying the user-accessible, +// read-only memory area of a device, if it is configured with such an area. +// Supported ioctl commands are given by MP_VFS_ROM_IOCTL_xxx. +mp_obj_t mp_vfs_rom_ioctl(size_t n_args, const mp_obj_t *args); +#endif + #endif // MICROPY_INCLUDED_EXTMOD_VFS_H diff --git a/extmod/vfs_reader.c b/extmod/vfs_reader.c index 80d0fa6344aa1..de5c4e03d3c13 100644 --- a/extmod/vfs_reader.c +++ b/extmod/vfs_reader.c @@ -85,6 +85,17 @@ void mp_reader_new_file(mp_reader_t *reader, qstr filename) { const mp_stream_p_t *stream_p = mp_get_stream(file); int errcode = 0; + + #if MICROPY_VFS_ROM + // Check if the stream can be memory mapped. + mp_buffer_info_t bufinfo; + if (mp_get_buffer(file, &bufinfo, MP_BUFFER_READ)) { + mp_reader_new_mem(reader, bufinfo.buf, bufinfo.len, MP_READER_IS_ROM); + return; + } + #endif + + // Determine how big the input buffer should be, if the stream requests a certain size or not. mp_uint_t bufsize = stream_p->ioctl(file, MP_STREAM_GET_BUFFER_SIZE, 0, &errcode); if (bufsize == MP_STREAM_ERROR || bufsize == 0) { // bufsize == 0 is included here to support mpremote v1.21 and older where mount file ioctl @@ -94,6 +105,7 @@ void mp_reader_new_file(mp_reader_t *reader, qstr filename) { bufsize = MIN(MICROPY_READER_VFS_MAX_BUFFER_SIZE, MAX(MICROPY_READER_VFS_MIN_BUFFER_SIZE, bufsize)); } + // Create the reader. mp_reader_vfs_t *rf = m_new_obj_var(mp_reader_vfs_t, buf, byte, bufsize); rf->file = file; rf->bufsize = bufsize; diff --git a/extmod/vfs_rom.c b/extmod/vfs_rom.c new file mode 100644 index 0000000000000..7d814cb980524 --- /dev/null +++ b/extmod/vfs_rom.c @@ -0,0 +1,471 @@ +/* + * This file is part of the MicroPython project, http://micropython.org/ + * + * The MIT License (MIT) + * + * Copyright (c) 2022 Damien P. George + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ + +// ROMFS filesystem format +// ======================= +// +// ROMFS is a flexible and extensible filesystem format designed to represent a +// directory hierarchy with files, where those files are read-only and their data +// can be memory mapped. +// +// Concepts: +// - varuint: An unsigned integer that is encoded in a variable number of bytes. It is +// stored big-endian with the high bit of the byte set if there are following bytes. +// - record: A variable sized element with a type. It is stored as two varuint's and then +// a payload. The first varuint is the record kind and the second varuint is the +// payload length (which may be zero bytes long). +// +// A ROMFS filesystem is a record with record kind 0x14a6b1, chosen so the encoded value +// is 0xd2-0xcd-0x31 which is "RM1" with the first two bytes having their high bit set. +// If the ROMFS record's payload is non-empty then it contains records. +// +// Record types: +// - 0 = unused, can be used to detect corruption of the filesystem. +// - 1 = padding/comments, can contain any data in their payload. +// - 2 = verbatim data, used to store file data. +// - 3 = indirect data, pointer to offset within the ROMFS payload. +// - 4 = a directory: payload contains a varuint which is the length of the directory +// name in bytes, then the name, then optional nested records for the contents +// of the directory (including optional metadata). +// - 5 = a file: payload contains a varuint which is the length of the filename in bytes +// then the name, then optional nested records. +// +// Remarks: +// - A varuint can be padded if needed by prepending with one or more 0x80 bytes. This +// padding does not change any semantics. +// - The size of the ROMFS record (including kind and length and payload) must be a +// multiple of 2 (because it's not possible to add a padding record of one byte). +// - File data can be optionally aligned using padding records and/or indirect data +// records. +// - There is no limit to the size of directory/file names or file data. +// +// Unknown record types must be skipped over. They may in the future add optional +// features, while still retaining backwards compatibility. Such features may be: +// - Alignment requirements of the ROMFS record. +// - Timestamps on directories/files. +// - A precomputed hash of a file, or other metadata. +// - An optimised lookup table indexing the directory hierarchy. + +#include + +#include "py/bc.h" +#include "py/runtime.h" +#include "py/mperrno.h" +#include "extmod/vfs.h" +#include "extmod/vfs_rom.h" + +#if MICROPY_VFS_ROM + +#define ROMFS_SIZE_MIN (4) +#define ROMFS_HEADER_BYTE0 (0x80 | 'R') +#define ROMFS_HEADER_BYTE1 (0x80 | 'M') +#define ROMFS_HEADER_BYTE2 (0x00 | '1') + +// Values for `record_kind_t`. +#define ROMFS_RECORD_KIND_UNUSED (0) +#define ROMFS_RECORD_KIND_PADDING (1) +#define ROMFS_RECORD_KIND_DATA_VERBATIM (2) +#define ROMFS_RECORD_KIND_DATA_POINTER (3) +#define ROMFS_RECORD_KIND_DIRECTORY (4) +#define ROMFS_RECORD_KIND_FILE (5) +#define ROMFS_RECORD_KIND_FILESYSTEM (0x14a6b1) + +typedef mp_uint_t record_kind_t; + +struct _mp_obj_vfs_rom_t { + mp_obj_base_t base; + mp_obj_t memory; + const uint8_t *filesystem; + const uint8_t *filesystem_end; +}; + +// Returns 0 for success, -1 for failure. +static int mp_decode_uint_checked(const uint8_t **ptr, const uint8_t *ptr_max, mp_uint_t *value_out) { + mp_uint_t unum = 0; + byte val; + const uint8_t *p = *ptr; + do { + if (p >= ptr_max) { + return -1; + } + val = *p++; + unum = (unum << 7) | (val & 0x7f); + } while ((val & 0x80) != 0); + *ptr = p; + *value_out = unum; + return 0; +} + +static record_kind_t extract_record(const uint8_t **fs, const uint8_t **fs_next, const uint8_t *fs_max) { + mp_uint_t record_kind; + if (mp_decode_uint_checked(fs, fs_max, &record_kind) != 0) { + return ROMFS_RECORD_KIND_UNUSED; + } + mp_uint_t record_len; + if (mp_decode_uint_checked(fs, fs_max, &record_len) != 0) { + return ROMFS_RECORD_KIND_UNUSED; + } + *fs_next = *fs + record_len; + return record_kind; +} + +// Returns 0 for success, a negative integer for failure. +static int extract_data(mp_obj_vfs_rom_t *self, const uint8_t *fs, const uint8_t *fs_top, size_t *size_out, const uint8_t **data_out) { + while (fs < fs_top) { + const uint8_t *fs_next; + record_kind_t record_kind = extract_record(&fs, &fs_next, fs_top); + if (record_kind == ROMFS_RECORD_KIND_UNUSED) { + // Corrupt filesystem. + break; + } else if (record_kind == ROMFS_RECORD_KIND_DATA_VERBATIM) { + // Verbatim data. + if (size_out != NULL) { + *size_out = fs_next - fs; + *data_out = fs; + } + return 0; + } else if (record_kind == ROMFS_RECORD_KIND_DATA_POINTER) { + // Pointer to data. + mp_uint_t size; + if (mp_decode_uint_checked(&fs, fs_next, &size) != 0) { + break; + } + mp_uint_t offset; + if (mp_decode_uint_checked(&fs, fs_next, &offset) != 0) { + break; + } + if (size_out != NULL) { + *size_out = size; + *data_out = self->filesystem + offset; + } + return 0; + } else { + // Skip this record. + fs = fs_next; + } + } + return -MP_EIO; +} + +// Searches for `path` in the filesystem. +// `path` must be null-terminated. +mp_import_stat_t mp_vfs_rom_search_filesystem(mp_obj_vfs_rom_t *self, const char *path, size_t *size_out, const uint8_t **data_out) { + const uint8_t *fs = self->filesystem; + const uint8_t *fs_top = self->filesystem_end; + size_t path_len = strlen(path); + if (*path == '/') { + // An optional slash at the start of the path enters the top-level filesystem. + ++path; + --path_len; + } + while (path_len > 0 && fs < fs_top) { + const uint8_t *fs_next; + record_kind_t record_kind = extract_record(&fs, &fs_next, fs_top); + if (record_kind == ROMFS_RECORD_KIND_UNUSED) { + // Corrupt filesystem. + return MP_IMPORT_STAT_NO_EXIST; + } else if (record_kind == ROMFS_RECORD_KIND_DIRECTORY || record_kind == ROMFS_RECORD_KIND_FILE) { + // A directory or file record. + mp_uint_t name_len; + if (mp_decode_uint_checked(&fs, fs_next, &name_len) != 0) { + // Corrupt filesystem. + return MP_IMPORT_STAT_NO_EXIST; + } + if ((name_len == path_len + || (name_len < path_len && path[name_len] == '/')) + && memcmp(path, fs, name_len) == 0) { + // Name matches, so enter this record. + fs += name_len; + fs_top = fs_next; + path += name_len; + path_len -= name_len; + if (record_kind == ROMFS_RECORD_KIND_DIRECTORY) { + // Continue searching in this directory. + if (*path == '/') { + ++path; + --path_len; + } + } else { + // Return this file. + if (path_len != 0) { + return MP_IMPORT_STAT_NO_EXIST; + } + if (extract_data(self, fs, fs_top, size_out, data_out) != 0) { + // Corrupt filesystem. + return MP_IMPORT_STAT_NO_EXIST; + } + return MP_IMPORT_STAT_FILE; + } + } else { + // Skip this directory/file record. + fs = fs_next; + } + } else { + // Skip this record. + fs = fs_next; + } + } + if (path_len == 0) { + if (size_out != NULL) { + *size_out = fs_top - fs; + *data_out = fs; + } + return MP_IMPORT_STAT_DIR; + } + return MP_IMPORT_STAT_NO_EXIST; +} + +static mp_obj_t vfs_rom_make_new(const mp_obj_type_t *type, size_t n_args, size_t n_kw, const mp_obj_t *args) { + mp_arg_check_num(n_args, n_kw, 1, 1, false); + + mp_obj_vfs_rom_t *self = m_new_obj(mp_obj_vfs_rom_t); + self->base.type = type; + self->memory = args[0]; + + // Get the ROMFS memory region. + mp_buffer_info_t bufinfo; + mp_get_buffer_raise(self->memory, &bufinfo, MP_BUFFER_READ); + if (bufinfo.len < ROMFS_SIZE_MIN) { + mp_raise_OSError(MP_ENODEV); + } + self->filesystem = bufinfo.buf; + + // Verify it is a ROMFS. + if (!(self->filesystem[0] == ROMFS_HEADER_BYTE0 + && self->filesystem[1] == ROMFS_HEADER_BYTE1 + && self->filesystem[2] == ROMFS_HEADER_BYTE2)) { + mp_raise_OSError(MP_ENODEV); + } + + // The ROMFS is a record itself, so enter into it and compute its limit. + record_kind_t record_kind = extract_record(&self->filesystem, &self->filesystem_end, self->filesystem + bufinfo.len); + if (record_kind != ROMFS_RECORD_KIND_FILESYSTEM) { + mp_raise_OSError(MP_ENODEV); + } + + // Check the filesystem is within the limits of the input buffer. + if (self->filesystem_end > (const uint8_t *)bufinfo.buf + bufinfo.len) { + mp_raise_OSError(MP_ENODEV); + } + + return MP_OBJ_FROM_PTR(self); +} + +static mp_obj_t vfs_rom_mount(mp_obj_t self_in, mp_obj_t readonly, mp_obj_t mkfs) { + (void)self_in; + (void)readonly; + if (mp_obj_is_true(mkfs)) { + mp_raise_OSError(MP_EPERM); + } + return mp_const_none; +} +static MP_DEFINE_CONST_FUN_OBJ_3(vfs_rom_mount_obj, vfs_rom_mount); + +// mp_vfs_rom_file_open is implemented in vfs_rom_file.c. +static MP_DEFINE_CONST_FUN_OBJ_3(vfs_rom_open_obj, mp_vfs_rom_file_open); + +static mp_obj_t vfs_rom_chdir(mp_obj_t self_in, mp_obj_t path_in) { + mp_obj_vfs_rom_t *self = MP_OBJ_TO_PTR(self_in); + const char *path = mp_vfs_rom_get_path_str(self, path_in); + if (path[0] == '/' && path[1] == '\0') { + // Allow chdir to the root of the filesystem. + } else { + // Don't allow chdir to any subdirectory (not currently implemented). + mp_raise_OSError(MP_EOPNOTSUPP); + } + return mp_const_none; +} +static MP_DEFINE_CONST_FUN_OBJ_2(vfs_rom_chdir_obj, vfs_rom_chdir); + +static mp_obj_t vfs_rom_getcwd(mp_obj_t self_in) { + (void)self_in; + // The current directory is always the root of the ROMFS. + return MP_OBJ_NEW_QSTR(MP_QSTR_); +} +static MP_DEFINE_CONST_FUN_OBJ_1(vfs_rom_getcwd_obj, vfs_rom_getcwd); + +typedef struct _vfs_rom_ilistdir_it_t { + mp_obj_base_t base; + mp_fun_1_t iternext; + mp_obj_vfs_rom_t *vfs_rom; + bool is_str; + const uint8_t *index; + const uint8_t *index_top; +} vfs_rom_ilistdir_it_t; + +static mp_obj_t vfs_rom_ilistdir_it_iternext(mp_obj_t self_in) { + vfs_rom_ilistdir_it_t *self = MP_OBJ_TO_PTR(self_in); + + while (self->index < self->index_top) { + const uint8_t *index_next; + record_kind_t record_kind = extract_record(&self->index, &index_next, self->index_top); + uint32_t type; + mp_uint_t name_len; + size_t data_len; + if (record_kind == ROMFS_RECORD_KIND_UNUSED) { + // Corrupt filesystem. + self->index = self->index_top; + break; + } else if (record_kind == ROMFS_RECORD_KIND_DIRECTORY || record_kind == ROMFS_RECORD_KIND_FILE) { + // A directory or file record. + if (mp_decode_uint_checked(&self->index, index_next, &name_len) != 0) { + // Corrupt filesystem. + self->index = self->index_top; + break; + } + if (record_kind == ROMFS_RECORD_KIND_DIRECTORY) { + // A directory. + type = MP_S_IFDIR; + data_len = index_next - self->index - name_len; + } else { + // A file. + type = MP_S_IFREG; + const uint8_t *data_value; + if (extract_data(self->vfs_rom, self->index + name_len, index_next, &data_len, &data_value) != 0) { + // Corrupt filesystem. + break; + } + } + } else { + // Skip this record. + self->index = index_next; + continue; + } + + const uint8_t *name_str = self->index; + self->index = index_next; + + // Make 4-tuple with info about this entry: (name, attr, inode, size) + mp_obj_tuple_t *t = MP_OBJ_TO_PTR(mp_obj_new_tuple(4, NULL)); + + if (self->is_str) { + t->items[0] = mp_obj_new_str((const char *)name_str, name_len); + } else { + t->items[0] = mp_obj_new_bytes(name_str, name_len); + } + + t->items[1] = MP_OBJ_NEW_SMALL_INT(type); + t->items[2] = MP_OBJ_NEW_SMALL_INT(0); + t->items[3] = mp_obj_new_int(data_len); + + return MP_OBJ_FROM_PTR(t); + } + + return MP_OBJ_STOP_ITERATION; +} + +static mp_obj_t vfs_rom_ilistdir(mp_obj_t self_in, mp_obj_t path_in) { + mp_obj_vfs_rom_t *self = MP_OBJ_TO_PTR(self_in); + vfs_rom_ilistdir_it_t *iter = m_new_obj(vfs_rom_ilistdir_it_t); + iter->base.type = &mp_type_polymorph_iter; + iter->iternext = vfs_rom_ilistdir_it_iternext; + iter->vfs_rom = self; + iter->is_str = mp_obj_get_type(path_in) == &mp_type_str; + const char *path = mp_vfs_rom_get_path_str(self, path_in); + size_t size; + if (mp_vfs_rom_search_filesystem(self, path, &size, &iter->index) != MP_IMPORT_STAT_DIR) { + mp_raise_OSError(MP_ENOENT); + } + iter->index_top = iter->index + size; + return MP_OBJ_FROM_PTR(iter); +} +static MP_DEFINE_CONST_FUN_OBJ_2(vfs_rom_ilistdir_obj, vfs_rom_ilistdir); + +static mp_obj_t vfs_rom_stat(mp_obj_t self_in, mp_obj_t path_in) { + mp_obj_vfs_rom_t *self = MP_OBJ_TO_PTR(self_in); + const char *path = mp_vfs_rom_get_path_str(self, path_in); + size_t file_size; + const uint8_t *file_data; + mp_import_stat_t stat = mp_vfs_rom_search_filesystem(self, path, &file_size, &file_data); + if (stat == MP_IMPORT_STAT_NO_EXIST) { + mp_raise_OSError(MP_ENOENT); + } + mp_obj_tuple_t *t = MP_OBJ_TO_PTR(mp_obj_new_tuple(10, NULL)); + t->items[0] = MP_OBJ_NEW_SMALL_INT(stat == MP_IMPORT_STAT_FILE ? MP_S_IFREG : MP_S_IFDIR); // st_mode + t->items[1] = MP_OBJ_NEW_SMALL_INT(0); // st_ino + t->items[2] = MP_OBJ_NEW_SMALL_INT(0); // st_dev + t->items[3] = MP_OBJ_NEW_SMALL_INT(0); // st_nlink + t->items[4] = MP_OBJ_NEW_SMALL_INT(0); // st_uid + t->items[5] = MP_OBJ_NEW_SMALL_INT(0); // st_gid + t->items[6] = MP_OBJ_NEW_SMALL_INT(file_size); // st_size + t->items[7] = MP_OBJ_NEW_SMALL_INT(0); // st_atime + t->items[8] = MP_OBJ_NEW_SMALL_INT(0); // st_mtime + t->items[9] = MP_OBJ_NEW_SMALL_INT(0); // st_ctime + return MP_OBJ_FROM_PTR(t); +} +static MP_DEFINE_CONST_FUN_OBJ_2(vfs_rom_stat_obj, vfs_rom_stat); + +static mp_obj_t vfs_rom_statvfs(mp_obj_t self_in, mp_obj_t path_in) { + mp_obj_vfs_rom_t *self = MP_OBJ_TO_PTR(self_in); + (void)path_in; + size_t filesystem_len = self->filesystem_end - self->filesystem; + mp_obj_tuple_t *t = MP_OBJ_TO_PTR(mp_obj_new_tuple(10, NULL)); + t->items[0] = MP_OBJ_NEW_SMALL_INT(1); // f_bsize + t->items[1] = MP_OBJ_NEW_SMALL_INT(0); // f_frsize + t->items[2] = mp_obj_new_int_from_uint(filesystem_len); // f_blocks + t->items[3] = MP_OBJ_NEW_SMALL_INT(0); // f_bfree + t->items[4] = MP_OBJ_NEW_SMALL_INT(0); // f_bavail + t->items[5] = MP_OBJ_NEW_SMALL_INT(0); // f_files + t->items[6] = MP_OBJ_NEW_SMALL_INT(0); // f_ffree + t->items[7] = MP_OBJ_NEW_SMALL_INT(0); // f_favail + t->items[8] = MP_OBJ_NEW_SMALL_INT(0); // f_flags + t->items[9] = MP_OBJ_NEW_SMALL_INT(32767); // f_namemax + return MP_OBJ_FROM_PTR(t); +} +static MP_DEFINE_CONST_FUN_OBJ_2(vfs_rom_statvfs_obj, vfs_rom_statvfs); + +static const mp_rom_map_elem_t vfs_rom_locals_dict_table[] = { + { MP_ROM_QSTR(MP_QSTR_mount), MP_ROM_PTR(&vfs_rom_mount_obj) }, + { MP_ROM_QSTR(MP_QSTR_umount), MP_ROM_PTR(&mp_identity_obj) }, + { MP_ROM_QSTR(MP_QSTR_open), MP_ROM_PTR(&vfs_rom_open_obj) }, + + { MP_ROM_QSTR(MP_QSTR_chdir), MP_ROM_PTR(&vfs_rom_chdir_obj) }, + { MP_ROM_QSTR(MP_QSTR_getcwd), MP_ROM_PTR(&vfs_rom_getcwd_obj) }, + { MP_ROM_QSTR(MP_QSTR_ilistdir), MP_ROM_PTR(&vfs_rom_ilistdir_obj) }, + { MP_ROM_QSTR(MP_QSTR_stat), MP_ROM_PTR(&vfs_rom_stat_obj) }, + { MP_ROM_QSTR(MP_QSTR_statvfs), MP_ROM_PTR(&vfs_rom_statvfs_obj) }, +}; +static MP_DEFINE_CONST_DICT(vfs_rom_locals_dict, vfs_rom_locals_dict_table); + +static mp_import_stat_t mp_vfs_rom_import_stat(void *self_in, const char *path) { + mp_obj_vfs_rom_t *self = MP_OBJ_TO_PTR(self_in); + return mp_vfs_rom_search_filesystem(self, path, NULL, NULL); +} + +static const mp_vfs_proto_t vfs_rom_proto = { + .import_stat = mp_vfs_rom_import_stat, +}; + +MP_DEFINE_CONST_OBJ_TYPE( + mp_type_vfs_rom, + MP_QSTR_VfsRom, + MP_TYPE_FLAG_NONE, + make_new, vfs_rom_make_new, + protocol, &vfs_rom_proto, + locals_dict, &vfs_rom_locals_dict + ); + +#endif // MICROPY_VFS_ROM diff --git a/extmod/virtpin.c b/extmod/vfs_rom.h similarity index 62% rename from extmod/virtpin.c rename to extmod/vfs_rom.h index cd0b9f92f830e..d8e2b911ba0ca 100644 --- a/extmod/virtpin.c +++ b/extmod/vfs_rom.h @@ -3,7 +3,7 @@ * * The MIT License (MIT) * - * Copyright (c) 2016 Paul Sokolovsky + * Copyright (c) 2022 Damien P. George * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -23,17 +23,25 @@ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN * THE SOFTWARE. */ +#ifndef MICROPY_INCLUDED_EXTMOD_VFS_ROM_H +#define MICROPY_INCLUDED_EXTMOD_VFS_ROM_H -#include "extmod/virtpin.h" +#include "py/builtin.h" +#include "py/obj.h" -int mp_virtual_pin_read(mp_obj_t pin) { - mp_obj_base_t *s = (mp_obj_base_t *)MP_OBJ_TO_PTR(pin); - mp_pin_p_t *pin_p = (mp_pin_p_t *)MP_OBJ_TYPE_GET_SLOT(s->type, protocol); - return pin_p->ioctl(pin, MP_PIN_READ, 0, NULL); -} +#if MICROPY_VFS_ROM + +typedef struct _mp_obj_vfs_rom_t mp_obj_vfs_rom_t; + +extern const mp_obj_type_t mp_type_vfs_rom; -void mp_virtual_pin_write(mp_obj_t pin, int value) { - mp_obj_base_t *s = (mp_obj_base_t *)MP_OBJ_TO_PTR(pin); - mp_pin_p_t *pin_p = (mp_pin_p_t *)MP_OBJ_TYPE_GET_SLOT(s->type, protocol); - pin_p->ioctl(pin, MP_PIN_WRITE, value, NULL); +static inline const char *mp_vfs_rom_get_path_str(mp_obj_vfs_rom_t *self, mp_obj_t path) { + return mp_obj_str_get_str(path); } + +mp_import_stat_t mp_vfs_rom_search_filesystem(mp_obj_vfs_rom_t *self, const char *path, size_t *size_out, const uint8_t **data_out); +mp_obj_t mp_vfs_rom_file_open(mp_obj_t self_in, mp_obj_t path_in, mp_obj_t mode_in); + +#endif // MICROPY_VFS_ROM + +#endif // MICROPY_INCLUDED_EXTMOD_VFS_ROM_H diff --git a/extmod/vfs_rom_file.c b/extmod/vfs_rom_file.c new file mode 100644 index 0000000000000..57aca8c5dc7d3 --- /dev/null +++ b/extmod/vfs_rom_file.c @@ -0,0 +1,180 @@ +/* + * This file is part of the MicroPython project, http://micropython.org/ + * + * The MIT License (MIT) + * + * Copyright (c) 2022 Damien P. George + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ + +#include + +#include "py/reader.h" +#include "py/runtime.h" +#include "py/stream.h" +#include "extmod/vfs_rom.h" + +#if MICROPY_VFS_ROM + +typedef struct _mp_obj_vfs_rom_file_t { + mp_obj_base_t base; + size_t file_size; + size_t file_offset; + const uint8_t *file_data; +} mp_obj_vfs_rom_file_t; + +static const mp_obj_type_t mp_type_vfs_rom_fileio; +static const mp_obj_type_t mp_type_vfs_rom_textio; + +mp_obj_t mp_vfs_rom_file_open(mp_obj_t self_in, mp_obj_t path_in, mp_obj_t mode_in) { + mp_obj_vfs_rom_t *self = MP_OBJ_TO_PTR(self_in); + + const char *mode_s = mp_obj_str_get_str(mode_in); + const mp_obj_type_t *type = &mp_type_vfs_rom_textio; + while (*mode_s) { + switch (*mode_s++) { + case 'r': + break; + case 'w': + case 'a': + case '+': + mp_raise_OSError(MP_EROFS); + case 'b': + type = &mp_type_vfs_rom_fileio; + break; + case 't': + type = &mp_type_vfs_rom_textio; + break; + } + } + + mp_obj_vfs_rom_file_t *o = m_new_obj(mp_obj_vfs_rom_file_t); + o->base.type = type; + o->file_offset = 0; + + const char *path = mp_vfs_rom_get_path_str(self, path_in); + mp_import_stat_t stat = mp_vfs_rom_search_filesystem(self, path, &o->file_size, &o->file_data); + if (stat == MP_IMPORT_STAT_NO_EXIST) { + mp_raise_OSError(MP_ENOENT); + } else if (stat == MP_IMPORT_STAT_DIR) { + mp_raise_OSError(MP_EISDIR); + } + + return MP_OBJ_FROM_PTR(o); +} + +static mp_int_t vfs_rom_file_get_buffer(mp_obj_t self_in, mp_buffer_info_t *bufinfo, mp_uint_t flags) { + mp_obj_vfs_rom_file_t *self = MP_OBJ_TO_PTR(self_in); + if (flags == MP_BUFFER_READ) { + bufinfo->buf = (void *)self->file_data; + bufinfo->len = self->file_size; + bufinfo->typecode = 'B'; + return 0; + } else { + // Can't write to a ROM file. + return 1; + } +} + +static mp_uint_t vfs_rom_file_read(mp_obj_t o_in, void *buf, mp_uint_t size, int *errcode) { + mp_obj_vfs_rom_file_t *self = MP_OBJ_TO_PTR(o_in); + size_t remain = self->file_size - self->file_offset; + if (size > remain) { + size = remain; + } + memcpy(buf, self->file_data + self->file_offset, size); + self->file_offset += size; + return size; +} + +static mp_uint_t vfs_rom_file_ioctl(mp_obj_t o_in, mp_uint_t request, uintptr_t arg, int *errcode) { + mp_obj_vfs_rom_file_t *self = MP_OBJ_TO_PTR(o_in); + + switch (request) { + case MP_STREAM_SEEK: { + struct mp_stream_seek_t *s = (struct mp_stream_seek_t *)arg; + if (s->whence == 0) { // SEEK_SET + self->file_offset = (size_t)s->offset; + } else if (s->whence == 1) { // SEEK_CUR + self->file_offset += s->offset; + } else { // SEEK_END + self->file_offset = self->file_size + s->offset; + } + if (self->file_offset > self->file_size) { + if (s->offset < 0) { + // Seek to before the start of the file. + *errcode = MP_EINVAL; + return MP_STREAM_ERROR; + } + self->file_offset = self->file_size; + } + s->offset = self->file_offset; + return 0; + } + case MP_STREAM_CLOSE: + return 0; + default: + *errcode = MP_EINVAL; + return MP_STREAM_ERROR; + } +} + +static const mp_rom_map_elem_t vfs_rom_rawfile_locals_dict_table[] = { + { MP_ROM_QSTR(MP_QSTR_read), MP_ROM_PTR(&mp_stream_read_obj) }, + { MP_ROM_QSTR(MP_QSTR_readinto), MP_ROM_PTR(&mp_stream_readinto_obj) }, + { MP_ROM_QSTR(MP_QSTR_readline), MP_ROM_PTR(&mp_stream_unbuffered_readline_obj) }, + { MP_ROM_QSTR(MP_QSTR_readlines), MP_ROM_PTR(&mp_stream_unbuffered_readlines_obj) }, + { MP_ROM_QSTR(MP_QSTR_seek), MP_ROM_PTR(&mp_stream_seek_obj) }, + { MP_ROM_QSTR(MP_QSTR_tell), MP_ROM_PTR(&mp_stream_tell_obj) }, + { MP_ROM_QSTR(MP_QSTR_close), MP_ROM_PTR(&mp_stream_close_obj) }, + { MP_ROM_QSTR(MP_QSTR___enter__), MP_ROM_PTR(&mp_identity_obj) }, + { MP_ROM_QSTR(MP_QSTR___exit__), MP_ROM_PTR(&mp_stream___exit___obj) }, +}; +static MP_DEFINE_CONST_DICT(vfs_rom_rawfile_locals_dict, vfs_rom_rawfile_locals_dict_table); + +static const mp_stream_p_t vfs_rom_fileio_stream_p = { + .read = vfs_rom_file_read, + .ioctl = vfs_rom_file_ioctl, +}; + +static MP_DEFINE_CONST_OBJ_TYPE( + mp_type_vfs_rom_fileio, + MP_QSTR_FileIO, + MP_TYPE_FLAG_ITER_IS_STREAM, + buffer, vfs_rom_file_get_buffer, + protocol, &vfs_rom_fileio_stream_p, + locals_dict, &vfs_rom_rawfile_locals_dict + ); + +static const mp_stream_p_t vfs_rom_textio_stream_p = { + .read = vfs_rom_file_read, + .ioctl = vfs_rom_file_ioctl, + .is_text = true, +}; + +static MP_DEFINE_CONST_OBJ_TYPE( + mp_type_vfs_rom_textio, + MP_QSTR_TextIOWrapper, + MP_TYPE_FLAG_ITER_IS_STREAM, + protocol, &vfs_rom_textio_stream_p, + locals_dict, &vfs_rom_rawfile_locals_dict + ); + +#endif // MICROPY_VFS_ROM diff --git a/extmod/virtpin.h b/extmod/virtpin.h deleted file mode 100644 index 81094f21cf16e..0000000000000 --- a/extmod/virtpin.h +++ /dev/null @@ -1,51 +0,0 @@ -/* - * This file is part of the MicroPython project, http://micropython.org/ - * - * The MIT License (MIT) - * - * Copyright (c) 2016 Paul Sokolovsky - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MICROPY_INCLUDED_EXTMOD_VIRTPIN_H -#define MICROPY_INCLUDED_EXTMOD_VIRTPIN_H - -#include "py/obj.h" -// CIRCUITPY-CHANGE -#include "py/proto.h" - -#define MP_PIN_READ (1) -#define MP_PIN_WRITE (2) -#define MP_PIN_INPUT (3) -#define MP_PIN_OUTPUT (4) - -// Pin protocol -typedef struct _mp_pin_p_t { - // CIRCUITPY-CHANGE - MP_PROTOCOL_HEAD - mp_uint_t (*ioctl)(mp_obj_t obj, mp_uint_t request, uintptr_t arg, int *errcode); -} mp_pin_p_t; - -int mp_virtual_pin_read(mp_obj_t pin); -void mp_virtual_pin_write(mp_obj_t pin, int value); - -// If a port exposes a Pin object, it's constructor should be like this -mp_obj_t mp_pin_make_new(const mp_obj_type_t *type, size_t n_args, size_t n_kw, const mp_obj_t *args); - -#endif // MICROPY_INCLUDED_EXTMOD_VIRTPIN_H diff --git a/lib/mbedtls_config/mbedtls_config.h b/lib/mbedtls_config/mbedtls_config.h index 3791924a13c65..7943994e1575e 100644 --- a/lib/mbedtls_config/mbedtls_config.h +++ b/lib/mbedtls_config/mbedtls_config.h @@ -93,6 +93,7 @@ #define MBEDTLS_PKCS5_C #define MBEDTLS_PEM_PARSE_C #define MBEDTLS_PK_C +#define MBEDTLS_PK_HAVE_ECC_KEYS #define MBEDTLS_PK_PARSE_C #define MBEDTLS_PLATFORM_C #define MBEDTLS_RSA_C @@ -100,6 +101,7 @@ #define MBEDTLS_SHA256_C #define MBEDTLS_SHA512_C #define MBEDTLS_SSL_CLI_C +#define MBEDTLS_SSL_PROTO_DTLS #define MBEDTLS_SSL_SRV_C #define MBEDTLS_SSL_TLS_C #define MBEDTLS_TLS_DEFAULT_ALLOW_SHA1_IN_KEY_EXCHANGE diff --git a/lib/micropython-lib b/lib/micropython-lib index 68e3e07bc7ab6..5b496e944ec04 160000 --- a/lib/micropython-lib +++ b/lib/micropython-lib @@ -1 +1 @@ -Subproject commit 68e3e07bc7ab63931cead3854b2a114e9a084248 +Subproject commit 5b496e944ec045177afa1620920a168410b7f60b diff --git a/locale/circuitpython.pot b/locale/circuitpython.pot index 9e8043764288f..262b5de769c89 100644 --- a/locale/circuitpython.pot +++ b/locale/circuitpython.pot @@ -158,10 +158,6 @@ msgstr "" msgid "%q length must be >= %d" msgstr "" -#: py/runtime.c -msgid "%q moved from %q to %q" -msgstr "" - #: py/argcheck.c msgid "%q must be %d" msgstr "" @@ -250,7 +246,7 @@ msgstr "" msgid "%q out of range" msgstr "" -#: py/objmodule.c py/runtime.c +#: py/objmodule.c msgid "%q renamed %q" msgstr "" @@ -2698,6 +2694,10 @@ msgstr "" msgid "can only have one parent" msgstr "" +#: py/emitinlinerv32.c +msgid "can only have up to 4 parameters for RV32 assembly" +msgstr "" + #: py/emitinlinethumb.c msgid "can only have up to 4 parameters to Thumb assembly" msgstr "" @@ -3867,6 +3867,30 @@ msgstr "" msgid "opcode" msgstr "" +#: py/emitinlinerv32.c +msgid "opcode '%q' argument %d: expecting %q" +msgstr "" + +#: py/emitinlinerv32.c +msgid "opcode '%q' argument %d: must not be zero" +msgstr "" + +#: py/emitinlinerv32.c +msgid "opcode '%q' argument %d: out of range" +msgstr "" + +#: py/emitinlinerv32.c +msgid "opcode '%q' argument %d: undefined label '%q'" +msgstr "" + +#: py/emitinlinerv32.c +msgid "opcode '%q' argument %d: unknown register" +msgstr "" + +#: py/emitinlinerv32.c +msgid "opcode '%q': expecting %d arguments" +msgstr "" + #: extmod/ulab/code/ndarray.c extmod/ulab/code/numpy/bitwise.c #: extmod/ulab/code/numpy/compare.c extmod/ulab/code/numpy/vector.c msgid "operands could not be broadcast together" @@ -3958,6 +3982,10 @@ msgstr "" msgid "palette must be 32 bytes long" msgstr "" +#: py/emitinlinerv32.c +msgid "parameters must be registers in sequence a0 to a3" +msgstr "" + #: py/emitinlinextensa.c msgid "parameters must be registers in sequence a2 to a5" msgstr "" @@ -4147,10 +4175,6 @@ msgstr "" msgid "splitting with sub-captures" msgstr "" -#: py/objstr.c -msgid "start/end indices" -msgstr "" - #: shared-bindings/random/__init__.c msgid "stop not reachable from start" msgstr "" @@ -4313,6 +4337,10 @@ msgstr "" msgid "unindent doesn't match any outer indent level" msgstr "" +#: py/emitinlinerv32.c +msgid "unknown RV32 instruction '%q'" +msgstr "" + #: py/objstr.c #, c-format msgid "unknown conversion specifier %c" diff --git a/mpy-cross/mpconfigport.h b/mpy-cross/mpconfigport.h index a8aaceb796135..edbd9f87c0558 100644 --- a/mpy-cross/mpconfigport.h +++ b/mpy-cross/mpconfigport.h @@ -51,6 +51,7 @@ #define MICROPY_EMIT_INLINE_XTENSA (1) #define MICROPY_EMIT_XTENSAWIN (1) #define MICROPY_EMIT_RV32 (1) +#define MICROPY_EMIT_INLINE_RV32 (1) #define MICROPY_EMIT_NATIVE_DEBUG (1) #define MICROPY_EMIT_NATIVE_DEBUG_PRINTER (&mp_stdout_print) diff --git a/ports/litex/common-hal/microcontroller/Pin.h b/ports/litex/common-hal/microcontroller/Pin.h index 542f3e979b5bc..860a8c9ccadb6 100644 --- a/ports/litex/common-hal/microcontroller/Pin.h +++ b/ports/litex/common-hal/microcontroller/Pin.h @@ -7,6 +7,7 @@ #pragma once #include "py/mphal.h" +#include "py/obj.h" typedef struct { diff --git a/ports/nordic/peripherals/nrf/pins.h b/ports/nordic/peripherals/nrf/pins.h index 096568e87c3da..96810764de99d 100644 --- a/ports/nordic/peripherals/nrf/pins.h +++ b/ports/nordic/peripherals/nrf/pins.h @@ -12,6 +12,8 @@ #include #include +#include "py/obj.h" + #include "nrf_gpio.h" typedef struct { diff --git a/ports/raspberrypi/boards/wiznet_w5100s_evb_pico/mpconfigboard.mk b/ports/raspberrypi/boards/wiznet_w5100s_evb_pico/mpconfigboard.mk index 70011c41b0cb7..4753e28eb0d0d 100644 --- a/ports/raspberrypi/boards/wiznet_w5100s_evb_pico/mpconfigboard.mk +++ b/ports/raspberrypi/boards/wiznet_w5100s_evb_pico/mpconfigboard.mk @@ -9,5 +9,6 @@ CHIP_FAMILY = rp2 EXTERNAL_FLASH_DEVICES = "W25Q16JVxQ" CIRCUITPY__EVE = 1 +CIRCUITPY_FLOPPYIO = 0 CIRCUITPY_SSL = 1 CIRCUITPY_USB_HOST = 0 diff --git a/ports/raspberrypi/boards/wiznet_w5500_evb_pico/mpconfigboard.mk b/ports/raspberrypi/boards/wiznet_w5500_evb_pico/mpconfigboard.mk index b3ac9854d3d96..837246198a42c 100644 --- a/ports/raspberrypi/boards/wiznet_w5500_evb_pico/mpconfigboard.mk +++ b/ports/raspberrypi/boards/wiznet_w5500_evb_pico/mpconfigboard.mk @@ -9,5 +9,6 @@ CHIP_FAMILY = rp2 EXTERNAL_FLASH_DEVICES = "W25Q16JVxQ" CIRCUITPY__EVE = 1 +CIRCUITPY_FLOPPYIO = 0 CIRCUITPY_SSL = 1 CIRCUITPY_USB_HOST = 0 diff --git a/ports/silabs/common-hal/digitalio/DigitalInOut.h b/ports/silabs/common-hal/digitalio/DigitalInOut.h index 23a5a235378fb..a69eb48df33c1 100644 --- a/ports/silabs/common-hal/digitalio/DigitalInOut.h +++ b/ports/silabs/common-hal/digitalio/DigitalInOut.h @@ -1,29 +1,6 @@ /* * This file is part of Adafruit for EFR32 project - * - * The MIT License (MIT) - * - * Copyright 2023 Silicon Laboratories Inc. www.silabs.com - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. */ - #ifndef MICROPY_INCLUDED_EFR32_COMMON_HAL_DIGITALIO_DIGITALINOUT_H #define MICROPY_INCLUDED_EFR32_COMMON_HAL_DIGITALIO_DIGITALINOUT_H diff --git a/ports/unix/Makefile b/ports/unix/Makefile index e7d63d514d968..bb1f6a4740a35 100644 --- a/ports/unix/Makefile +++ b/ports/unix/Makefile @@ -49,6 +49,10 @@ CWARN = -Wall -Werror CWARN += -Wextra -Wno-unused-parameter -Wpointer-arith -Wdouble-promotion -Wfloat-conversion CFLAGS += $(INC) $(CWARN) -std=gnu99 -DUNIX $(COPT) -I$(VARIANT_DIR) $(CFLAGS_EXTRA) +# Force the use of 64-bits for file sizes in C library functions on 32-bit platforms. +# This option has no effect on 64-bit builds. +CFLAGS += -D_FILE_OFFSET_BITS=64 + # Debugging/Optimization ifdef DEBUG COPT ?= -Og diff --git a/ports/unix/main.c b/ports/unix/main.c index 73e61597050f6..ce8b89136370b 100644 --- a/ports/unix/main.c +++ b/ports/unix/main.c @@ -45,6 +45,7 @@ #include "py/gc.h" #include "py/objstr.h" #include "py/cstack.h" +#include "py/mperrno.h" #include "py/mphal.h" #include "py/mpthread.h" #include "extmod/misc.h" @@ -580,7 +581,14 @@ MP_NOINLINE int main_(int argc, char **argv) { MP_OBJ_NEW_QSTR(MP_QSTR__slash_), }; mp_vfs_mount(2, args, (mp_map_t *)&mp_const_empty_map); + + // Make sure the root that was just mounted is the current VFS (it's always at + // the end of the linked list). Can't use chdir('/') because that will change + // the current path within the VfsPosix object. MP_STATE_VM(vfs_cur) = MP_STATE_VM(vfs_mount_table); + while (MP_STATE_VM(vfs_cur)->next != NULL) { + MP_STATE_VM(vfs_cur) = MP_STATE_VM(vfs_cur)->next; + } } #endif @@ -840,3 +848,22 @@ void nlr_jump_fail(void *val) { fprintf(stderr, "FATAL: uncaught NLR %p\n", val); exit(1); } + +#if MICROPY_VFS_ROM_IOCTL + +static uint8_t romfs_buf[4] = { 0xd2, 0xcd, 0x31, 0x00 }; // empty ROMFS +static const MP_DEFINE_MEMORYVIEW_OBJ(romfs_obj, 'B', 0, sizeof(romfs_buf), romfs_buf); + +mp_obj_t mp_vfs_rom_ioctl(size_t n_args, const mp_obj_t *args) { + switch (mp_obj_get_int(args[0])) { + case MP_VFS_ROM_IOCTL_GET_NUMBER_OF_SEGMENTS: + return MP_OBJ_NEW_SMALL_INT(1); + + case MP_VFS_ROM_IOCTL_GET_SEGMENT: + return MP_OBJ_FROM_PTR(&romfs_obj); + } + + return MP_OBJ_NEW_SMALL_INT(-MP_EINVAL); +} + +#endif diff --git a/ports/unix/mbedtls/mbedtls_config_port.h b/ports/unix/mbedtls/mbedtls_config_port.h index c619de9b8b1b9..aec65e6581e73 100644 --- a/ports/unix/mbedtls/mbedtls_config_port.h +++ b/ports/unix/mbedtls/mbedtls_config_port.h @@ -32,7 +32,18 @@ // Enable mbedtls modules #define MBEDTLS_TIMING_C +#if defined(MICROPY_UNIX_COVERAGE) +// Test the "bare metal" memory management in the coverage build +#define MICROPY_MBEDTLS_CONFIG_BARE_METAL (1) +#endif + // Include common mbedtls configuration. #include "extmod/mbedtls/mbedtls_config_common.h" +#if defined(MICROPY_UNIX_COVERAGE) +// See comment above, but fall back to the default platform entropy functions +#undef MBEDTLS_ENTROPY_HARDWARE_ALT +#undef MBEDTLS_NO_PLATFORM_ENTROPY +#endif + #endif /* MICROPY_INCLUDED_MBEDTLS_CONFIG_H */ diff --git a/ports/unix/mpthreadport.c b/ports/unix/mpthreadport.c index 16ac4da8bf56f..5172645bc147a 100644 --- a/ports/unix/mpthreadport.c +++ b/ports/unix/mpthreadport.c @@ -65,7 +65,7 @@ static pthread_key_t tls_key; // The mutex is used for any code in this port that needs to be thread safe. // Specifically for thread management, access to the linked list is one example. // But also, e.g. scheduler state. -static pthread_mutex_t thread_mutex; +static mp_thread_recursive_mutex_t thread_mutex; static mp_thread_t *thread; // this is used to synchronise the signal handler of the thread @@ -78,11 +78,11 @@ static sem_t thread_signal_done; #endif void mp_thread_unix_begin_atomic_section(void) { - pthread_mutex_lock(&thread_mutex); + mp_thread_recursive_mutex_lock(&thread_mutex, true); } void mp_thread_unix_end_atomic_section(void) { - pthread_mutex_unlock(&thread_mutex); + mp_thread_recursive_mutex_unlock(&thread_mutex); } // this signal handler is used to scan the regs and stack of a thread @@ -113,10 +113,7 @@ void mp_thread_init(void) { // Needs to be a recursive mutex to emulate the behavior of // BEGIN_ATOMIC_SECTION on bare metal. - pthread_mutexattr_t thread_mutex_attr; - pthread_mutexattr_init(&thread_mutex_attr); - pthread_mutexattr_settype(&thread_mutex_attr, PTHREAD_MUTEX_RECURSIVE); - pthread_mutex_init(&thread_mutex, &thread_mutex_attr); + mp_thread_recursive_mutex_init(&thread_mutex); // create first entry in linked list of all threads thread = malloc(sizeof(mp_thread_t)); @@ -321,6 +318,26 @@ void mp_thread_mutex_unlock(mp_thread_mutex_t *mutex) { // TODO check return value } +#if MICROPY_PY_THREAD_RECURSIVE_MUTEX + +void mp_thread_recursive_mutex_init(mp_thread_recursive_mutex_t *mutex) { + pthread_mutexattr_t attr; + pthread_mutexattr_init(&attr); + pthread_mutexattr_settype(&attr, PTHREAD_MUTEX_RECURSIVE); + pthread_mutex_init(mutex, &attr); + pthread_mutexattr_destroy(&attr); +} + +int mp_thread_recursive_mutex_lock(mp_thread_recursive_mutex_t *mutex, int wait) { + return mp_thread_mutex_lock(mutex, wait); +} + +void mp_thread_recursive_mutex_unlock(mp_thread_recursive_mutex_t *mutex) { + mp_thread_mutex_unlock(mutex); +} + +#endif // MICROPY_PY_THREAD_RECURSIVE_MUTEX + #endif // MICROPY_PY_THREAD // this is used even when MICROPY_PY_THREAD is disabled diff --git a/ports/unix/mpthreadport.h b/ports/unix/mpthreadport.h index b365f200edf97..a38223e720b45 100644 --- a/ports/unix/mpthreadport.h +++ b/ports/unix/mpthreadport.h @@ -28,6 +28,7 @@ #include typedef pthread_mutex_t mp_thread_mutex_t; +typedef pthread_mutex_t mp_thread_recursive_mutex_t; void mp_thread_init(void); void mp_thread_deinit(void); diff --git a/ports/unix/variants/coverage/mpconfigvariant.h b/ports/unix/variants/coverage/mpconfigvariant.h index 12823f3c2b5fc..ca79d3d0d2b61 100644 --- a/ports/unix/variants/coverage/mpconfigvariant.h +++ b/ports/unix/variants/coverage/mpconfigvariant.h @@ -41,10 +41,16 @@ #define MICROPY_DEBUG_PARSE_RULE_NAME (1) #define MICROPY_TRACKED_ALLOC (1) #define MICROPY_WARNINGS_CATEGORY (1) +#undef MICROPY_VFS_ROM_IOCTL +#define MICROPY_VFS_ROM_IOCTL (1) +#define MICROPY_PY_CRYPTOLIB_CTR (1) // CIRCUITPY-CHANGE: Disable things never used in circuitpython -#define MICROPY_PY_CRYPTOLIB (0) -#define MICROPY_PY_CRYPTOLIB_CTR (0) -#define MICROPY_PY_MICROPYTHON_RINGIO (0) +#define MICROPY_PY_CRYPTOLIB (0) +#undef MICROPY_PY_CRYPTOLIB_CTR +#define MICROPY_PY_CRYPTOLIB_CTR (0) +#define MICROPY_PY_MICROPYTHON_RINGIO (0) // CircuitPython uses shared-bindings struct #define MICROPY_PY_STRUCT (0) +#undef MICROPY_VFS_ROM_IOCTL +#define MICROPY_VFS_ROM_IOCTL (0) diff --git a/ports/unix/variants/coverage/mpconfigvariant.mk b/ports/unix/variants/coverage/mpconfigvariant.mk index 579e42cc05cf9..6ec0e85377ccf 100644 --- a/ports/unix/variants/coverage/mpconfigvariant.mk +++ b/ports/unix/variants/coverage/mpconfigvariant.mk @@ -12,7 +12,8 @@ CFLAGS += \ LDFLAGS += -fprofile-arcs -ftest-coverage FROZEN_MANIFEST ?= $(VARIANT_DIR)/manifest.py -USER_C_MODULES = $(TOP)/examples/usercmodule +# CIRCUITPY-CHANGE: don't include user C modules +# USER_C_MODULES = $(TOP)/examples/usercmodule # CIRCUITPY-CHANGE: use CircuitPython bindings and implementations SRC_QRIO := $(patsubst ../../%,%,$(wildcard ../../shared-bindings/qrio/*.c ../../shared-module/qrio/*.c ../../lib/quirc/lib/*.c)) diff --git a/ports/unix/variants/mpconfigvariant_common.h b/ports/unix/variants/mpconfigvariant_common.h index cea0397414325..9eeed8797366c 100644 --- a/ports/unix/variants/mpconfigvariant_common.h +++ b/ports/unix/variants/mpconfigvariant_common.h @@ -121,3 +121,6 @@ #define MICROPY_PY_MACHINE (1) #define MICROPY_PY_MACHINE_PULSE (1) #define MICROPY_PY_MACHINE_PIN_BASE (1) + +#define MICROPY_VFS_ROM (1) +#define MICROPY_VFS_ROM_IOCTL (0) diff --git a/ports/zephyr-cp/common-hal/microcontroller/Pin.h b/ports/zephyr-cp/common-hal/microcontroller/Pin.h index eb0304dc72fde..d38ab9bd2009c 100644 --- a/ports/zephyr-cp/common-hal/microcontroller/Pin.h +++ b/ports/zephyr-cp/common-hal/microcontroller/Pin.h @@ -7,6 +7,7 @@ #pragma once #include "py/mphal.h" +#include "py/obj.h" #include diff --git a/py/asmarm.c b/py/asmarm.c index 6006490701251..6fa751b32eb7c 100644 --- a/py/asmarm.c +++ b/py/asmarm.c @@ -168,13 +168,23 @@ void asm_arm_entry(asm_arm_t *as, int num_locals) { emit_al(as, asm_arm_op_push(as->push_reglist | 1 << ASM_ARM_REG_LR)); if (as->stack_adjust > 0) { - emit_al(as, asm_arm_op_sub_imm(ASM_ARM_REG_SP, ASM_ARM_REG_SP, as->stack_adjust)); + if (as->stack_adjust < 0x100) { + emit_al(as, asm_arm_op_sub_imm(ASM_ARM_REG_SP, ASM_ARM_REG_SP, as->stack_adjust)); + } else { + asm_arm_mov_reg_i32_optimised(as, ASM_ARM_REG_R8, as->stack_adjust); + emit_al(as, asm_arm_op_sub_reg(ASM_ARM_REG_SP, ASM_ARM_REG_SP, ASM_ARM_REG_R8)); + } } } void asm_arm_exit(asm_arm_t *as) { if (as->stack_adjust > 0) { - emit_al(as, asm_arm_op_add_imm(ASM_ARM_REG_SP, ASM_ARM_REG_SP, as->stack_adjust)); + if (as->stack_adjust < 0x100) { + emit_al(as, asm_arm_op_add_imm(ASM_ARM_REG_SP, ASM_ARM_REG_SP, as->stack_adjust)); + } else { + asm_arm_mov_reg_i32_optimised(as, ASM_ARM_REG_R8, as->stack_adjust); + emit_al(as, asm_arm_op_add_reg(ASM_ARM_REG_SP, ASM_ARM_REG_SP, ASM_ARM_REG_R8)); + } } emit_al(as, asm_arm_op_pop(as->push_reglist | (1 << ASM_ARM_REG_PC))); @@ -282,8 +292,15 @@ void asm_arm_orr_reg_reg_reg(asm_arm_t *as, uint rd, uint rn, uint rm) { } void asm_arm_mov_reg_local_addr(asm_arm_t *as, uint rd, int local_num) { - // add rd, sp, #local_num*4 - emit_al(as, asm_arm_op_add_imm(rd, ASM_ARM_REG_SP, local_num << 2)); + if (local_num >= 0x40) { + // mov r8, #local_num*4 + // add rd, sp, r8 + asm_arm_mov_reg_i32_optimised(as, ASM_ARM_REG_R8, local_num << 2); + emit_al(as, asm_arm_op_add_reg(rd, ASM_ARM_REG_SP, ASM_ARM_REG_R8)); + } else { + // add rd, sp, #local_num*4 + emit_al(as, asm_arm_op_add_imm(rd, ASM_ARM_REG_SP, local_num << 2)); + } } void asm_arm_mov_reg_pcrel(asm_arm_t *as, uint reg_dest, uint label) { @@ -327,8 +344,15 @@ void asm_arm_ldrh_reg_reg(asm_arm_t *as, uint rd, uint rn) { } void asm_arm_ldrh_reg_reg_offset(asm_arm_t *as, uint rd, uint rn, uint byte_offset) { - // ldrh rd, [rn, #off] - emit_al(as, 0x1f000b0 | (rn << 16) | (rd << 12) | ((byte_offset & 0xf0) << 4) | (byte_offset & 0xf)); + if (byte_offset < 0x100) { + // ldrh rd, [rn, #off] + emit_al(as, 0x1d000b0 | (rn << 16) | (rd << 12) | ((byte_offset & 0xf0) << 4) | (byte_offset & 0xf)); + } else { + // mov r8, #off + // ldrh rd, [rn, r8] + asm_arm_mov_reg_i32_optimised(as, ASM_ARM_REG_R8, byte_offset); + emit_al(as, 0x19000b0 | (rn << 16) | (rd << 12) | ASM_ARM_REG_R8); + } } void asm_arm_ldrb_reg_reg(asm_arm_t *as, uint rd, uint rn) { diff --git a/py/asmrv32.c b/py/asmrv32.c index 3f3395842efb9..c24d05a1384d4 100644 --- a/py/asmrv32.c +++ b/py/asmrv32.c @@ -29,6 +29,7 @@ #include #include "py/emit.h" +#include "py/misc.h" #include "py/mpconfig.h" // wrapper around everything in this file @@ -43,34 +44,7 @@ #define DEBUG_printf(...) (void)0 #endif -#ifndef MP_POPCOUNT -#ifdef _MSC_VER -#include -#define MP_POPCOUNT __popcnt -#else -#if defined __has_builtin -#if __has_builtin(__builtin_popcount) -#define MP_POPCOUNT __builtin_popcount -#endif -#else -static uint32_t fallback_popcount(uint32_t value) { - value = value - ((value >> 1) & 0x55555555); - value = (value & 0x33333333) + ((value >> 2) & 0x33333333); - value = (value + (value >> 4)) & 0x0F0F0F0F; - return value * 0x01010101; -} -#define MP_POPCOUNT fallback_popcount -#endif -#endif -#endif - #define INTERNAL_TEMPORARY ASM_RV32_REG_S0 -#define AVAILABLE_REGISTERS_COUNT 32 - -#define IS_IN_C_REGISTER_WINDOW(register_number) \ - (((register_number) >= ASM_RV32_REG_X8) && ((register_number) <= ASM_RV32_REG_X15)) -#define MAP_IN_C_REGISTER_WINDOW(register_number) \ - ((register_number) - ASM_RV32_REG_X8) #define FIT_UNSIGNED(value, bits) (((value) & ~((1U << (bits)) - 1)) == 0) #define FIT_SIGNED(value, bits) \ @@ -126,7 +100,6 @@ static void split_immediate(mp_int_t immediate, mp_uint_t *upper, mp_uint_t *low // Turn the lower half from unsigned to signed. if ((*lower & 0x800) != 0) { *upper += 0x1000; - *lower -= 0x1000; } } @@ -200,7 +173,7 @@ void asm_rv32_emit_optimised_load_immediate(asm_rv32_t *state, mp_uint_t rd, mp_ static void emit_registers_store(asm_rv32_t *state, mp_uint_t registers_mask) { mp_uint_t offset = 0; - for (mp_uint_t register_index = 0; register_index < AVAILABLE_REGISTERS_COUNT; register_index++) { + for (mp_uint_t register_index = 0; register_index < RV32_AVAILABLE_REGISTERS_COUNT; register_index++) { if (registers_mask & (1U << register_index)) { assert(FIT_UNSIGNED(offset >> 2, 6) && "Registers save stack offset out of range."); // c.swsp register, offset @@ -212,7 +185,7 @@ static void emit_registers_store(asm_rv32_t *state, mp_uint_t registers_mask) { static void emit_registers_load(asm_rv32_t *state, mp_uint_t registers_mask) { mp_uint_t offset = 0; - for (mp_uint_t register_index = 0; register_index < AVAILABLE_REGISTERS_COUNT; register_index++) { + for (mp_uint_t register_index = 0; register_index < RV32_AVAILABLE_REGISTERS_COUNT; register_index++) { if (registers_mask & (1U << register_index)) { assert(FIT_UNSIGNED(offset >> 2, 6) && "Registers load stack offset out of range."); // c.lwsp register, offset @@ -249,7 +222,7 @@ static void adjust_stack(asm_rv32_t *state, mp_int_t stack_size) { // stack to hold all the tainted registers and an arbitrary amount of space // for locals. static void emit_function_prologue(asm_rv32_t *state, mp_uint_t registers) { - mp_uint_t registers_count = MP_POPCOUNT(registers); + mp_uint_t registers_count = mp_popcount(registers); state->stack_size = (registers_count + state->locals_count) * sizeof(uint32_t); mp_uint_t old_saved_registers_mask = state->saved_registers_mask; // Move stack pointer up. @@ -282,7 +255,7 @@ static bool calculate_displacement_for_label(asm_rv32_t *state, mp_uint_t label, void asm_rv32_entry(asm_rv32_t *state, mp_uint_t locals) { state->saved_registers_mask |= (1U << REG_FUN_TABLE) | (1U << REG_LOCAL_1) | \ - (1U << REG_LOCAL_2) | (1U << REG_LOCAL_3) | (1U << INTERNAL_TEMPORARY); + (1U << REG_LOCAL_2) | (1U << REG_LOCAL_3); state->locals_count = locals; emit_function_prologue(state, state->saved_registers_mask); } @@ -301,10 +274,11 @@ void asm_rv32_emit_call_ind(asm_rv32_t *state, mp_uint_t index) { mp_uint_t offset = index * ASM_WORD_SIZE; state->saved_registers_mask |= (1U << ASM_RV32_REG_RA); - if (IS_IN_C_REGISTER_WINDOW(REG_FUN_TABLE) && IS_IN_C_REGISTER_WINDOW(INTERNAL_TEMPORARY) && FIT_UNSIGNED(offset, 6)) { + if (RV32_IS_IN_C_REGISTER_WINDOW(REG_FUN_TABLE) && RV32_IS_IN_C_REGISTER_WINDOW(INTERNAL_TEMPORARY) && FIT_UNSIGNED(offset, 6)) { + state->saved_registers_mask |= (1U << INTERNAL_TEMPORARY); // c.lw temporary, offset(fun_table) // c.jalr temporary - asm_rv32_opcode_clw(state, MAP_IN_C_REGISTER_WINDOW(INTERNAL_TEMPORARY), MAP_IN_C_REGISTER_WINDOW(REG_FUN_TABLE), offset); + asm_rv32_opcode_clw(state, RV32_MAP_IN_C_REGISTER_WINDOW(INTERNAL_TEMPORARY), RV32_MAP_IN_C_REGISTER_WINDOW(REG_FUN_TABLE), offset); asm_rv32_opcode_cjalr(state, INTERNAL_TEMPORARY); return; } @@ -361,9 +335,9 @@ void asm_rv32_emit_jump_if_reg_nonzero(asm_rv32_t *state, mp_uint_t rs, mp_uint_ ptrdiff_t displacement = 0; bool can_emit_short_jump = calculate_displacement_for_label(state, label, &displacement); - if (can_emit_short_jump && FIT_SIGNED(displacement, 8) && IS_IN_C_REGISTER_WINDOW(rs)) { + if (can_emit_short_jump && FIT_SIGNED(displacement, 8) && RV32_IS_IN_C_REGISTER_WINDOW(rs)) { // c.bnez rs', displacement - asm_rv32_opcode_cbnez(state, MAP_IN_C_REGISTER_WINDOW(rs), displacement); + asm_rv32_opcode_cbnez(state, RV32_MAP_IN_C_REGISTER_WINDOW(rs), displacement); return; } @@ -384,8 +358,8 @@ void asm_rv32_emit_jump_if_reg_nonzero(asm_rv32_t *state, mp_uint_t rs, mp_uint_ // jalr zero, temporary, LO(displacement) ; PC + 8 // ... ; PC + 12 - if (can_emit_short_jump && IS_IN_C_REGISTER_WINDOW(rs)) { - asm_rv32_opcode_cbeqz(state, MAP_IN_C_REGISTER_WINDOW(rs), 10); + if (can_emit_short_jump && RV32_IS_IN_C_REGISTER_WINDOW(rs)) { + asm_rv32_opcode_cbeqz(state, RV32_MAP_IN_C_REGISTER_WINDOW(rs), 10); // Compensate for the C.BEQZ opcode. displacement -= ASM_HALFWORD_SIZE; } else { @@ -458,9 +432,9 @@ void asm_rv32_emit_mov_reg_local(asm_rv32_t *state, mp_uint_t rd, mp_uint_t loca void asm_rv32_emit_mov_reg_local_addr(asm_rv32_t *state, mp_uint_t rd, mp_uint_t local) { mp_uint_t offset = state->locals_stack_offset + (local * ASM_WORD_SIZE); - if (FIT_UNSIGNED(offset, 10) && offset != 0 && IS_IN_C_REGISTER_WINDOW(rd)) { + if (FIT_UNSIGNED(offset, 10) && offset != 0 && RV32_IS_IN_C_REGISTER_WINDOW(rd)) { // c.addi4spn rd', offset - asm_rv32_opcode_caddi4spn(state, MAP_IN_C_REGISTER_WINDOW(rd), offset); + asm_rv32_opcode_caddi4spn(state, RV32_MAP_IN_C_REGISTER_WINDOW(rd), offset); return; } @@ -479,9 +453,9 @@ void asm_rv32_emit_mov_reg_local_addr(asm_rv32_t *state, mp_uint_t rd, mp_uint_t void asm_rv32_emit_load_reg_reg_offset(asm_rv32_t *state, mp_uint_t rd, mp_uint_t rs, mp_int_t offset) { mp_int_t scaled_offset = offset * sizeof(ASM_WORD_SIZE); - if (scaled_offset >= 0 && IS_IN_C_REGISTER_WINDOW(rd) && IS_IN_C_REGISTER_WINDOW(rs) && FIT_UNSIGNED(scaled_offset, 6)) { + if (scaled_offset >= 0 && RV32_IS_IN_C_REGISTER_WINDOW(rd) && RV32_IS_IN_C_REGISTER_WINDOW(rs) && FIT_UNSIGNED(scaled_offset, 6)) { // c.lw rd', offset(rs') - asm_rv32_opcode_clw(state, MAP_IN_C_REGISTER_WINDOW(rd), MAP_IN_C_REGISTER_WINDOW(rs), scaled_offset); + asm_rv32_opcode_clw(state, RV32_MAP_IN_C_REGISTER_WINDOW(rd), RV32_MAP_IN_C_REGISTER_WINDOW(rs), scaled_offset); return; } diff --git a/py/asmrv32.h b/py/asmrv32.h index 775cf1ffc9630..b09f48eb12f66 100644 --- a/py/asmrv32.h +++ b/py/asmrv32.h @@ -74,9 +74,6 @@ #define ASM_RV32_REG_SP (ASM_RV32_REG_X2) #define ASM_RV32_REG_GP (ASM_RV32_REG_X3) #define ASM_RV32_REG_TP (ASM_RV32_REG_X4) -#define ASM_RV32_REG_T0 (ASM_RV32_REG_X5) -#define ASM_RV32_REG_T1 (ASM_RV32_REG_X6) -#define ASM_RV32_REG_T2 (ASM_RV32_REG_X7) #define ASM_RV32_REG_A0 (ASM_RV32_REG_X10) #define ASM_RV32_REG_A1 (ASM_RV32_REG_X11) #define ASM_RV32_REG_A2 (ASM_RV32_REG_X12) @@ -85,6 +82,9 @@ #define ASM_RV32_REG_A5 (ASM_RV32_REG_X15) #define ASM_RV32_REG_A6 (ASM_RV32_REG_X16) #define ASM_RV32_REG_A7 (ASM_RV32_REG_X17) +#define ASM_RV32_REG_T0 (ASM_RV32_REG_X5) +#define ASM_RV32_REG_T1 (ASM_RV32_REG_X6) +#define ASM_RV32_REG_T2 (ASM_RV32_REG_X7) #define ASM_RV32_REG_T3 (ASM_RV32_REG_X28) #define ASM_RV32_REG_T4 (ASM_RV32_REG_X29) #define ASM_RV32_REG_T5 (ASM_RV32_REG_X30) @@ -103,6 +103,12 @@ #define ASM_RV32_REG_S10 (ASM_RV32_REG_X26) #define ASM_RV32_REG_S11 (ASM_RV32_REG_X27) +#define RV32_AVAILABLE_REGISTERS_COUNT 32 +#define RV32_MAP_IN_C_REGISTER_WINDOW(register_number) \ + ((register_number) - ASM_RV32_REG_X8) +#define RV32_IS_IN_C_REGISTER_WINDOW(register_number) \ + (((register_number) >= ASM_RV32_REG_X8) && ((register_number) <= ASM_RV32_REG_X15)) + typedef struct _asm_rv32_t { // Opaque emitter state. mp_asm_base_t base; @@ -127,6 +133,10 @@ void asm_rv32_end_pass(asm_rv32_t *state); ((imm & 0x1E) << 7) | ((rs1 & 0x1F) << 15) | ((rs2 & 0x1F) << 20) | \ ((imm & 0x7E0) << 20) | ((imm & 0x1000) << 19)) +#define RV32_ENCODE_TYPE_CSRI(op, ft3, rd, csr, imm) \ + ((op & 0x7F) | ((rd & 0x1F) << 7) | ((ft3 & 0x07) << 12) | \ + ((csr & 0xFFF) << 20) | ((imm & 0x1F) << 15)) + #define RV32_ENCODE_TYPE_I(op, ft3, rd, rs, imm) \ ((op & 0x7F) | ((rd & 0x1F) << 7) | ((ft3 & 0x07) << 12) | \ ((rs & 0x1F) << 15) | ((imm & 0xFFF) << 20)) @@ -143,13 +153,16 @@ void asm_rv32_end_pass(asm_rv32_t *state); ((op & 0x7F) | ((imm & 0x1F) << 7) | ((ft3 & 0x07) << 12) | \ ((rs1 & 0x1F) << 15) | ((rs2 & 0x1F) << 20) | ((imm & 0xFE0) << 20)) +#define RV32_ENCODE_TYPE_CA(op, ft6, ft2, rd, rs) \ + ((op & 0x03) | ((ft6 & 0x3F) << 10) | ((ft2 & 0x03) << 5) | \ + ((rd & 0x03) << 7) | ((rs & 0x03) << 2)) + #define RV32_ENCODE_TYPE_U(op, rd, imm) \ ((op & 0x7F) | ((rd & 0x1F) << 7) | (imm & 0xFFFFF000)) #define RV32_ENCODE_TYPE_CB(op, ft3, rs, imm) \ ((op & 0x03) | ((ft3 & 0x07) << 13) | ((rs & 0x07) << 7) | \ - (((imm) & 0x100) << 4) | (((imm) & 0xC0) >> 1) | (((imm) & 0x20) >> 3) | \ - (((imm) & 0x18) << 7) | (((imm) & 0x06) << 2)) + (((imm) & 0xE0) << 5) | (((imm) & 0x1F) << 2)) #define RV32_ENCODE_TYPE_CI(op, ft3, rd, imm) \ ((op & 0x03) | ((ft3 & 0x07) << 13) | ((rd & 0x1F) << 7) | \ @@ -174,6 +187,11 @@ void asm_rv32_end_pass(asm_rv32_t *state); #define RV32_ENCODE_TYPE_CR(op, ft4, rs1, rs2) \ ((op & 0x03) | ((rs2 & 0x1F) << 2) | ((rs1 & 0x1F) << 7) | ((ft4 & 0x0F) << 12)) +#define RV32_ENCODE_TYPE_CS(op, ft3, rd, rs, imm) \ + ((op & 0x03) | ((ft3 & 0x07) << 13) | ((rd & 0x07) << 2) | \ + ((rs & 0x07) << 7) | ((imm & 0x40) >> 1) | ((imm & 0x38) << 7) | \ + ((imm & 0x04) << 4)) + #define RV32_ENCODE_TYPE_CSS(op, ft3, rs, imm) \ ((op & 0x03) | ((ft3 & 0x07) << 13) | ((rs & 0x1F) << 2) | ((imm) & 0x3F) << 7) @@ -198,6 +216,12 @@ static inline void asm_rv32_opcode_and(asm_rv32_t *state, mp_uint_t rd, mp_uint_ asm_rv32_emit_word_opcode(state, RV32_ENCODE_TYPE_R(0x33, 0x07, 0x00, rd, rs1, rs2)); } +// ANDI RD, RS, IMMEDIATE +static inline void asm_rv32_opcode_andi(asm_rv32_t *state, mp_uint_t rd, mp_uint_t rs, mp_int_t immediate) { + // I: ............ ..... 111 ..... 0010011 + asm_rv32_emit_word_opcode(state, RV32_ENCODE_TYPE_I(0x13, 0x07, rd, rs, immediate)); +} + // AUIPC RD, offset static inline void asm_rv32_opcode_auipc(asm_rv32_t *state, mp_uint_t rd, mp_int_t offset) { // U: .................... ..... 0010111 @@ -210,6 +234,30 @@ static inline void asm_rv32_opcode_beq(asm_rv32_t *state, mp_uint_t rs1, mp_uint asm_rv32_emit_word_opcode(state, RV32_ENCODE_TYPE_B(0x63, 0x00, rs1, rs2, offset)); } +// BGE RS1, RS2, OFFSET +static inline void asm_rv32_opcode_bge(asm_rv32_t *state, mp_uint_t rs1, mp_uint_t rs2, mp_int_t offset) { + // B: . ...... ..... ..... 101 .... . 1100011 + asm_rv32_emit_word_opcode(state, RV32_ENCODE_TYPE_B(0x63, 0x05, rs1, rs2, offset)); +} + +// BGEU RS1, RS2, OFFSET +static inline void asm_rv32_opcode_bgeu(asm_rv32_t *state, mp_uint_t rs1, mp_uint_t rs2, mp_int_t offset) { + // B: . ...... ..... ..... 111 .... . 1100011 + asm_rv32_emit_word_opcode(state, RV32_ENCODE_TYPE_B(0x63, 0x07, rs1, rs2, offset)); +} + +// BLT RS1, RS2, OFFSET +static inline void asm_rv32_opcode_blt(asm_rv32_t *state, mp_uint_t rs1, mp_uint_t rs2, mp_int_t offset) { + // B: . ...... ..... ..... 100 .... . 1100011 + asm_rv32_emit_word_opcode(state, RV32_ENCODE_TYPE_B(0x63, 0x04, rs1, rs2, offset)); +} + +// BLTU RS1, RS2, OFFSET +static inline void asm_rv32_opcode_bltu(asm_rv32_t *state, mp_uint_t rs1, mp_uint_t rs2, mp_int_t offset) { + // B: . ...... ..... ..... 110 .... . 1100011 + asm_rv32_emit_word_opcode(state, RV32_ENCODE_TYPE_B(0x63, 0x06, rs1, rs2, offset)); +} + // BNE RS1, RS2, OFFSET static inline void asm_rv32_opcode_bne(asm_rv32_t *state, mp_uint_t rs1, mp_uint_t rs2, mp_int_t offset) { // B: . ...... ..... ..... 001 .... . 1100011 @@ -234,16 +282,39 @@ static inline void asm_rv32_opcode_caddi4spn(asm_rv32_t *state, mp_uint_t rd, mp asm_rv32_emit_halfword_opcode(state, RV32_ENCODE_TYPE_CIW(0x00, 0x00, rd, immediate)); } +// C.AND RD', RS' +static inline void asm_rv32_opcode_cand(asm_rv32_t *state, mp_uint_t rd, mp_uint_t rs) { + // CA: 100011 ... 11 ... 01 + asm_rv32_emit_halfword_opcode(state, RV32_ENCODE_TYPE_CA(0x01, 0x23, 0x03, rd, rs)); +} + +// C.ANDI RD', IMMEDIATE +static inline void asm_rv32_opcode_candi(asm_rv32_t *state, mp_uint_t rd, mp_int_t immediate) { + // CB: 100 . 10 ... ..... 01 + asm_rv32_emit_halfword_opcode(state, RV32_ENCODE_TYPE_CB(0x01, 0x04, rd, + (((immediate & 0x20) << 2) | (immediate & 0x1F) | 0x40))); +} + // C.BEQZ RS', IMMEDIATE static inline void asm_rv32_opcode_cbeqz(asm_rv32_t *state, mp_uint_t rs, mp_int_t offset) { // CB: 110 ... ... ..... 01 - asm_rv32_emit_halfword_opcode(state, RV32_ENCODE_TYPE_CB(0x01, 0x06, rs, offset)); + asm_rv32_emit_halfword_opcode(state, RV32_ENCODE_TYPE_CB(0x01, 0x06, rs, + (((offset & 0x100) >> 1) | ((offset & 0xC0) >> 3) | ((offset & 0x20) >> 5) | + ((offset & 0x18) << 2) | (offset & 0x06)))); } // C.BNEZ RS', IMMEDIATE static inline void asm_rv32_opcode_cbnez(asm_rv32_t *state, mp_uint_t rs, mp_int_t offset) { // CB: 111 ... ... ..... 01 - asm_rv32_emit_halfword_opcode(state, RV32_ENCODE_TYPE_CB(0x01, 0x07, rs, offset)); + asm_rv32_emit_halfword_opcode(state, RV32_ENCODE_TYPE_CB(0x01, 0x07, rs, + (((offset & 0x100) >> 1) | ((offset & 0xC0) >> 3) | ((offset & 0x20) >> 5) | + ((offset & 0x18) << 2) | (offset & 0x06)))); +} + +// C.EBREAK +static inline void asm_rv32_opcode_cebreak(asm_rv32_t *state) { + // CA: 100 1 00000 00000 10 + asm_rv32_emit_halfword_opcode(state, 0x9002); } // C.J OFFSET @@ -252,6 +323,12 @@ static inline void asm_rv32_opcode_cj(asm_rv32_t *state, mp_int_t offset) { asm_rv32_emit_halfword_opcode(state, RV32_ENCODE_TYPE_CJ(0x01, 0x05, offset)); } +// C.JAL OFFSET +static inline void asm_rv32_opcode_cjal(asm_rv32_t *state, mp_int_t offset) { + // CJ: 001 ........... 01 + asm_rv32_emit_halfword_opcode(state, RV32_ENCODE_TYPE_CJ(0x01, 0x01, offset)); +} + // C.JALR RS static inline void asm_rv32_opcode_cjalr(asm_rv32_t *state, mp_uint_t rs) { // CR: 1001 ..... 00000 10 @@ -294,31 +371,159 @@ static inline void asm_rv32_opcode_cmv(asm_rv32_t *state, mp_uint_t rd, mp_uint_ asm_rv32_emit_halfword_opcode(state, RV32_ENCODE_TYPE_CR(0x02, 0x08, rd, rs)); } +// C.NOP +static inline void asm_rv32_opcode_cnop(asm_rv32_t *state) { + // CI: 000 . 00000 ..... 01 + asm_rv32_emit_halfword_opcode(state, 0x0001); +} + +// C.OR RD', RS' +static inline void asm_rv32_opcode_cor(asm_rv32_t *state, mp_uint_t rd, mp_uint_t rs) { + // CA: 100011 ... 10 ... 01 + asm_rv32_emit_halfword_opcode(state, RV32_ENCODE_TYPE_CA(0x01, 0x23, 0x02, rd, rs)); +} + +// C.SLLI RD, IMMEDIATE +static inline void asm_rv32_opcode_cslli(asm_rv32_t *state, mp_uint_t rd, mp_int_t immediate) { + // CI: 000 . ..... ..... 10 + asm_rv32_emit_halfword_opcode(state, RV32_ENCODE_TYPE_CI(0x02, 0x00, rd, immediate)); +} + +// C.SRAI RD, IMMEDIATE +static inline void asm_rv32_opcode_csrai(asm_rv32_t *state, mp_uint_t rd, mp_int_t immediate) { + // CB: 100 . 01 ... ..... 01 + asm_rv32_emit_halfword_opcode(state, RV32_ENCODE_TYPE_CB(0x01, 0x04, rd, + (((immediate & 0x20) << 2) | (immediate & 0x1F) | 0x20))); +} + +// C.SRLI RD, IMMEDIATE +static inline void asm_rv32_opcode_csrli(asm_rv32_t *state, mp_uint_t rd, mp_int_t immediate) { + // CB: 100 . 00 ... ..... 01 + asm_rv32_emit_halfword_opcode(state, RV32_ENCODE_TYPE_CB(0x01, 0x04, rd, + (((immediate & 0x20) << 2) | (immediate & 0x1F)))); +} + +// C.SUB RD', RS' +static inline void asm_rv32_opcode_csub(asm_rv32_t *state, mp_uint_t rd, mp_uint_t rs) { + // CA: 100011 ... 00 ... 01 + asm_rv32_emit_halfword_opcode(state, RV32_ENCODE_TYPE_CA(0x01, 0x23, 0x00, rd, rs)); +} + +// C.SW RS1', OFFSET(RS2') +static inline void asm_rv32_opcode_csw(asm_rv32_t *state, mp_uint_t rs1, mp_uint_t rs2, mp_int_t offset) { + // CS: 110 ... ... .. ... 00 + asm_rv32_emit_halfword_opcode(state, RV32_ENCODE_TYPE_CL(0x00, 0x06, rs1, rs2, offset)); +} + // C.SWSP RS, OFFSET static inline void asm_rv32_opcode_cswsp(asm_rv32_t *state, mp_uint_t rs, mp_uint_t offset) { // CSS: 010 ...... ..... 10 asm_rv32_emit_halfword_opcode(state, RV32_ENCODE_TYPE_CSS(0x02, 0x06, rs, ((offset & 0xC0) >> 6) | (offset & 0x3C))); } -// JALR RD, RS, offset +// C.XOR RD', RS' +static inline void asm_rv32_opcode_cxor(asm_rv32_t *state, mp_uint_t rd, mp_uint_t rs) { + // CA: 100011 ... 01 ... 01 + asm_rv32_emit_halfword_opcode(state, RV32_ENCODE_TYPE_CA(0x01, 0x23, 0x01, rd, rs)); +} + +// CSRRC RD, RS, IMMEDIATE +static inline void asm_rv32_opcode_csrrc(asm_rv32_t *state, mp_uint_t rd, mp_uint_t rs, mp_int_t immediate) { + // I: ............ ..... 011 ..... 1110011 + asm_rv32_emit_word_opcode(state, RV32_ENCODE_TYPE_I(0x73, 0x03, rd, rs, immediate)); +} + +// CSRRS RD, RS, IMMEDIATE +static inline void asm_rv32_opcode_csrrs(asm_rv32_t *state, mp_uint_t rd, mp_uint_t rs, mp_int_t immediate) { + // I: ............ ..... 010 ..... 1110011 + asm_rv32_emit_word_opcode(state, RV32_ENCODE_TYPE_I(0x73, 0x02, rd, rs, immediate)); +} + +// CSRRW RD, RS, IMMEDIATE +static inline void asm_rv32_opcode_csrrw(asm_rv32_t *state, mp_uint_t rd, mp_uint_t rs, mp_int_t immediate) { + // I: ............ ..... 001 ..... 1110011 + asm_rv32_emit_word_opcode(state, RV32_ENCODE_TYPE_I(0x73, 0x01, rd, rs, immediate)); +} + +// CSRRCI RD, CSR, IMMEDIATE +static inline void asm_rv32_opcode_csrrci(asm_rv32_t *state, mp_uint_t rd, mp_uint_t csr, mp_int_t immediate) { + // CSRI: ............ ..... 111 ..... 1110011 + asm_rv32_emit_word_opcode(state, RV32_ENCODE_TYPE_CSRI(0x73, 0x07, rd, csr, immediate)); +} + +// CSRRSI RD, CSR, IMMEDIATE +static inline void asm_rv32_opcode_csrrsi(asm_rv32_t *state, mp_uint_t rd, mp_uint_t csr, mp_int_t immediate) { + // CSRI: ............ ..... 110 ..... 1110011 + asm_rv32_emit_word_opcode(state, RV32_ENCODE_TYPE_CSRI(0x73, 0x06, rd, csr, immediate)); +} + +// CSRRWI RD, CSR, IMMEDIATE +static inline void asm_rv32_opcode_csrrwi(asm_rv32_t *state, mp_uint_t rd, mp_uint_t csr, mp_int_t immediate) { + // CSRI: ............ ..... 101 ..... 1110011 + asm_rv32_emit_word_opcode(state, RV32_ENCODE_TYPE_CSRI(0x73, 0x05, rd, csr, immediate)); +} + +// DIV RD, RS1, RS2 +static inline void asm_rv32_opcode_div(asm_rv32_t *state, mp_uint_t rd, mp_uint_t rs1, mp_uint_t rs2) { + // R: 0000001 ..... ..... 100 ..... 0110011 + asm_rv32_emit_word_opcode(state, RV32_ENCODE_TYPE_R(0x33, 0x04, 0x01, rd, rs1, rs2)); +} + +// DIVU RD, RS1, RS2 +static inline void asm_rv32_opcode_divu(asm_rv32_t *state, mp_uint_t rd, mp_uint_t rs1, mp_uint_t rs2) { + // R: 0000001 ..... ..... 101 ..... 0110011 + asm_rv32_emit_word_opcode(state, RV32_ENCODE_TYPE_R(0x33, 0x05, 0x01, rd, rs1, rs2)); +} + +// EBREAK +static inline void asm_rv32_opcode_ebreak(asm_rv32_t *state) { + // I: 000000000001 00000 000 00000 1110011 + asm_rv32_emit_word_opcode(state, 0x100073); +} + +// ECALL +static inline void asm_rv32_opcode_ecall(asm_rv32_t *state) { + // I: 000000000000 00000 000 00000 1110011 + asm_rv32_emit_word_opcode(state, 0x73); +} + +// JAL RD, OFFSET +static inline void asm_rv32_opcode_jal(asm_rv32_t *state, mp_uint_t rd, mp_int_t offset) { + // J: ......................... 1101111 + asm_rv32_emit_word_opcode(state, RV32_ENCODE_TYPE_J(0x6F, rd, offset)); +} + +// JALR RD, RS, OFFSET static inline void asm_rv32_opcode_jalr(asm_rv32_t *state, mp_uint_t rd, mp_uint_t rs, mp_int_t offset) { // I: ............ ..... 000 ..... 1100111 asm_rv32_emit_word_opcode(state, RV32_ENCODE_TYPE_I(0x67, 0x00, rd, rs, offset)); } +// LB RD, OFFSET(RS) +static inline void asm_rv32_opcode_lb(asm_rv32_t *state, mp_uint_t rd, mp_uint_t rs, mp_int_t offset) { + // I: ............ ..... 000 ..... 0000011 + asm_rv32_emit_word_opcode(state, RV32_ENCODE_TYPE_I(0x03, 0x00, rd, rs, offset)); +} + // LBU RD, OFFSET(RS) static inline void asm_rv32_opcode_lbu(asm_rv32_t *state, mp_uint_t rd, mp_uint_t rs, mp_int_t offset) { // I: ............ ..... 100 ..... 0000011 asm_rv32_emit_word_opcode(state, RV32_ENCODE_TYPE_I(0x03, 0x04, rd, rs, offset)); } +// LH RD, OFFSET(RS) +static inline void asm_rv32_opcode_lh(asm_rv32_t *state, mp_uint_t rd, mp_uint_t rs, mp_int_t offset) { + // I: ............ ..... 001 ..... 0000011 + asm_rv32_emit_word_opcode(state, RV32_ENCODE_TYPE_I(0x03, 0x01, rd, rs, offset)); +} + // LHU RD, OFFSET(RS) static inline void asm_rv32_opcode_lhu(asm_rv32_t *state, mp_uint_t rd, mp_uint_t rs, mp_int_t offset) { // I: ............ ..... 101 ..... 0000011 asm_rv32_emit_word_opcode(state, RV32_ENCODE_TYPE_I(0x03, 0x05, rd, rs, offset)); } -// LUI RD, immediate +// LUI RD, IMMEDIATE static inline void asm_rv32_opcode_lui(asm_rv32_t *state, mp_uint_t rd, mp_int_t immediate) { // U: .................... ..... 0110111 asm_rv32_emit_word_opcode(state, RV32_ENCODE_TYPE_U(0x37, rd, immediate)); @@ -336,12 +541,48 @@ static inline void asm_rv32_opcode_mul(asm_rv32_t *state, mp_uint_t rd, mp_uint_ asm_rv32_emit_word_opcode(state, RV32_ENCODE_TYPE_R(0x33, 0x00, 0x01, rd, rs1, rs2)); } +// MULH RD, RS1, RS2 +static inline void asm_rv32_opcode_mulh(asm_rv32_t *state, mp_uint_t rd, mp_uint_t rs1, mp_uint_t rs2) { + // R: 0000001 ..... ..... 001 ..... 0110011 + asm_rv32_emit_word_opcode(state, RV32_ENCODE_TYPE_R(0x33, 0x01, 0x01, rd, rs1, rs2)); +} + +// MULHSU RD, RS1, RS2 +static inline void asm_rv32_opcode_mulhsu(asm_rv32_t *state, mp_uint_t rd, mp_uint_t rs1, mp_uint_t rs2) { + // R: 0000001 ..... ..... 010 ..... 0110011 + asm_rv32_emit_word_opcode(state, RV32_ENCODE_TYPE_R(0x33, 0x02, 0x01, rd, rs1, rs2)); +} + +// MULHU RD, RS1, RS2 +static inline void asm_rv32_opcode_mulhu(asm_rv32_t *state, mp_uint_t rd, mp_uint_t rs1, mp_uint_t rs2) { + // R: 0000001 ..... ..... 011 ..... 0110011 + asm_rv32_emit_word_opcode(state, RV32_ENCODE_TYPE_R(0x33, 0x03, 0x01, rd, rs1, rs2)); +} + // OR RD, RS1, RS2 static inline void asm_rv32_opcode_or(asm_rv32_t *state, mp_uint_t rd, mp_uint_t rs1, mp_uint_t rs2) { // R: 0000000 ..... ..... 110 ..... 0110011 asm_rv32_emit_word_opcode(state, RV32_ENCODE_TYPE_R(0x33, 0x06, 0x00, rd, rs1, rs2)); } +// ORI RD, RS, IMMEDIATE +static inline void asm_rv32_opcode_ori(asm_rv32_t *state, mp_uint_t rd, mp_uint_t rs, mp_int_t immediate) { + // I: ............ ..... 110 ..... 0010011 + asm_rv32_emit_word_opcode(state, RV32_ENCODE_TYPE_I(0x13, 0x06, rd, rs, immediate)); +} + +// REM RD, RS1, RS2 +static inline void asm_rv32_opcode_rem(asm_rv32_t *state, mp_uint_t rd, mp_uint_t rs1, mp_uint_t rs2) { + // R: 0000001 ..... ..... 110 ..... 0110011 + asm_rv32_emit_word_opcode(state, RV32_ENCODE_TYPE_R(0x33, 0x06, 0x01, rd, rs1, rs2)); +} + +// REMU RD, RS1, RS2 +static inline void asm_rv32_opcode_remu(asm_rv32_t *state, mp_uint_t rd, mp_uint_t rs1, mp_uint_t rs2) { + // R: 0000001 ..... ..... 111 ..... 0110011 + asm_rv32_emit_word_opcode(state, RV32_ENCODE_TYPE_R(0x33, 0x07, 0x01, rd, rs1, rs2)); +} + // SLL RD, RS1, RS2 static inline void asm_rv32_opcode_sll(asm_rv32_t *state, mp_uint_t rd, mp_uint_t rs1, mp_uint_t rs2) { // R: 0000000 ..... ..... 001 ..... 0110011 @@ -349,23 +590,29 @@ static inline void asm_rv32_opcode_sll(asm_rv32_t *state, mp_uint_t rd, mp_uint_ } // SLLI RD, RS, IMMEDIATE -static inline void asm_rv32_opcode_slli(asm_rv32_t *state, mp_uint_t rd, mp_uint_t rs, mp_uint_t immediate) { +static inline void asm_rv32_opcode_slli(asm_rv32_t *state, mp_uint_t rd, mp_uint_t rs, mp_int_t immediate) { // I: 0000000..... ..... 001 ..... 0010011 asm_rv32_emit_word_opcode(state, RV32_ENCODE_TYPE_I(0x13, 0x01, rd, rs, immediate & 0x1F)); } -// SRL RD, RS1, RS2 -static inline void asm_rv32_opcode_srl(asm_rv32_t *state, mp_uint_t rd, mp_uint_t rs1, mp_uint_t rs2) { - // R: 0000000 ..... ..... 101 ..... 0110011 - asm_rv32_emit_word_opcode(state, RV32_ENCODE_TYPE_R(0x33, 0x05, 0x00, rd, rs1, rs2)); -} - // SLT RD, RS1, RS2 static inline void asm_rv32_opcode_slt(asm_rv32_t *state, mp_uint_t rd, mp_uint_t rs1, mp_uint_t rs2) { // R: 0000000 ..... ..... 010 ..... 0110011 asm_rv32_emit_word_opcode(state, RV32_ENCODE_TYPE_R(0x33, 0x02, 0x00, rd, rs1, rs2)); } +// SLTI RD, RS, IMMEDIATE +static inline void asm_rv32_opcode_slti(asm_rv32_t *state, mp_uint_t rd, mp_uint_t rs, mp_int_t immediate) { + // I: ............ ..... 010 ..... 0010011 + asm_rv32_emit_word_opcode(state, RV32_ENCODE_TYPE_I(0x13, 0x02, rd, rs, immediate)); +} + +// SLTIU RD, RS, IMMEDIATE +static inline void asm_rv32_opcode_sltiu(asm_rv32_t *state, mp_uint_t rd, mp_uint_t rs, mp_int_t immediate) { + // I: ............ ..... 011 ..... 0010011 + asm_rv32_emit_word_opcode(state, RV32_ENCODE_TYPE_I(0x13, 0x03, rd, rs, immediate)); +} + // SLTU RD, RS1, RS2 static inline void asm_rv32_opcode_sltu(asm_rv32_t *state, mp_uint_t rd, mp_uint_t rs1, mp_uint_t rs2) { // R: 0000000 ..... ..... 011 ..... 0110011 @@ -378,6 +625,24 @@ static inline void asm_rv32_opcode_sra(asm_rv32_t *state, mp_uint_t rd, mp_uint_ asm_rv32_emit_word_opcode(state, RV32_ENCODE_TYPE_R(0x33, 0x05, 0x20, rd, rs1, rs2)); } +// SRAI RD, RS, IMMEDIATE +static inline void asm_rv32_opcode_srai(asm_rv32_t *state, mp_uint_t rd, mp_uint_t rs, mp_int_t immediate) { + // I: 0100000..... ..... 101 ..... 0010011 + asm_rv32_emit_word_opcode(state, RV32_ENCODE_TYPE_I(0x13, 0x05, rd, rs, ((immediate & 0x1F) | 0x400))); +} + +// SRL RD, RS1, RS2 +static inline void asm_rv32_opcode_srl(asm_rv32_t *state, mp_uint_t rd, mp_uint_t rs1, mp_uint_t rs2) { + // R: 0000000 ..... ..... 101 ..... 0110011 + asm_rv32_emit_word_opcode(state, RV32_ENCODE_TYPE_R(0x33, 0x05, 0x00, rd, rs1, rs2)); +} + +// SRLI RD, RS, IMMEDIATE +static inline void asm_rv32_opcode_srli(asm_rv32_t *state, mp_uint_t rd, mp_uint_t rs, mp_int_t immediate) { + // I: 0000000..... ..... 101 ..... 0010011 + asm_rv32_emit_word_opcode(state, RV32_ENCODE_TYPE_I(0x13, 0x05, rd, rs, immediate & 0x1F)); +} + // SUB RD, RS1, RS2 static inline void asm_rv32_opcode_sub(asm_rv32_t *state, mp_uint_t rd, mp_uint_t rs1, mp_uint_t rs2) { // R: 0100000 ..... ..... 000 ..... 0110011 @@ -429,12 +694,15 @@ static inline void asm_rv32_opcode_xori(asm_rv32_t *state, mp_uint_t rd, mp_uint #define REG_LOCAL_1 ASM_RV32_REG_S3 #define REG_LOCAL_2 ASM_RV32_REG_S4 #define REG_LOCAL_3 ASM_RV32_REG_S5 +#define REG_ZERO ASM_RV32_REG_ZERO void asm_rv32_meta_comparison_eq(asm_rv32_t *state, mp_uint_t rs1, mp_uint_t rs2, mp_uint_t rd); void asm_rv32_meta_comparison_ne(asm_rv32_t *state, mp_uint_t rs1, mp_uint_t rs2, mp_uint_t rd); void asm_rv32_meta_comparison_lt(asm_rv32_t *state, mp_uint_t rs1, mp_uint_t rs2, mp_uint_t rd, bool unsigned_comparison); void asm_rv32_meta_comparison_le(asm_rv32_t *state, mp_uint_t rs1, mp_uint_t rs2, mp_uint_t rd, bool unsigned_comparison); +void asm_rv32_emit_optimised_load_immediate(asm_rv32_t *state, mp_uint_t rd, mp_int_t immediate); + #ifdef GENERIC_ASM_API void asm_rv32_emit_call_ind(asm_rv32_t *state, mp_uint_t index); @@ -444,10 +712,9 @@ void asm_rv32_emit_jump_if_reg_nonzero(asm_rv32_t *state, mp_uint_t rs, mp_uint_ void asm_rv32_emit_load16_reg_reg_offset(asm_rv32_t *state, mp_uint_t rd, mp_uint_t rs, mp_int_t offset); void asm_rv32_emit_load_reg_reg_offset(asm_rv32_t *state, mp_uint_t rd, mp_uint_t rs, mp_int_t offset); void asm_rv32_emit_mov_local_reg(asm_rv32_t *state, mp_uint_t local, mp_uint_t rs); -void asm_rv32_emit_mov_reg_local(asm_rv32_t *state, mp_uint_t rd, mp_uint_t local); void asm_rv32_emit_mov_reg_local_addr(asm_rv32_t *state, mp_uint_t rd, mp_uint_t local); +void asm_rv32_emit_mov_reg_local(asm_rv32_t *state, mp_uint_t rd, mp_uint_t local); void asm_rv32_emit_mov_reg_pcrel(asm_rv32_t *state, mp_uint_t rd, mp_uint_t label); -void asm_rv32_emit_optimised_load_immediate(asm_rv32_t *state, mp_uint_t rd, mp_int_t immediate); void asm_rv32_emit_optimised_xor(asm_rv32_t *state, mp_uint_t rd, mp_uint_t rs); void asm_rv32_emit_store_reg_reg_offset(asm_rv32_t *state, mp_uint_t source, mp_uint_t base, mp_int_t offset); @@ -490,6 +757,7 @@ void asm_rv32_emit_store_reg_reg_offset(asm_rv32_t *state, mp_uint_t source, mp_ #define ASM_STORE_REG_REG(state, rs1, rs2) ASM_STORE32_REG_REG(state, rs1, rs2) #define ASM_SUB_REG_REG(state, rd, rs) asm_rv32_opcode_sub(state, rd, rd, rs) #define ASM_XOR_REG_REG(state, rd, rs) asm_rv32_emit_optimised_xor(state, rd, rs) +#define ASM_CLR_REG(state, rd) #endif diff --git a/py/asmxtensa.h b/py/asmxtensa.h index c3c8f225f3744..a8c39206bd008 100644 --- a/py/asmxtensa.h +++ b/py/asmxtensa.h @@ -143,6 +143,14 @@ static inline void asm_xtensa_op_addi(asm_xtensa_t *as, uint reg_dest, uint reg_ asm_xtensa_op24(as, ASM_XTENSA_ENCODE_RRI8(2, 12, reg_src, reg_dest, imm8 & 0xff)); } +static inline void asm_xtensa_op_addx2(asm_xtensa_t *as, uint reg_dest, uint reg_src_a, uint reg_src_b) { + asm_xtensa_op24(as, ASM_XTENSA_ENCODE_RRR(0, 0, 9, reg_dest, reg_src_a, reg_src_b)); +} + +static inline void asm_xtensa_op_addx4(asm_xtensa_t *as, uint reg_dest, uint reg_src_a, uint reg_src_b) { + asm_xtensa_op24(as, ASM_XTENSA_ENCODE_RRR(0, 0, 10, reg_dest, reg_src_a, reg_src_b)); +} + static inline void asm_xtensa_op_and(asm_xtensa_t *as, uint reg_dest, uint reg_src_a, uint reg_src_b) { asm_xtensa_op24(as, ASM_XTENSA_ENCODE_RRR(0, 0, 1, reg_dest, reg_src_a, reg_src_b)); } diff --git a/py/builtinevex.c b/py/builtinevex.c index e25cbd4d08502..74a4640492674 100644 --- a/py/builtinevex.c +++ b/py/builtinevex.c @@ -26,6 +26,7 @@ #include +#include "py/objcode.h" #include "py/objfun.h" #include "py/compile.h" #include "py/runtime.h" @@ -33,17 +34,6 @@ #if MICROPY_PY_BUILTINS_COMPILE -typedef struct _mp_obj_code_t { - mp_obj_base_t base; - mp_obj_t module_fun; -} mp_obj_code_t; - -static MP_DEFINE_CONST_OBJ_TYPE( - mp_type_code, - MP_QSTR_code, - MP_TYPE_FLAG_NONE - ); - static mp_obj_t code_execute(mp_obj_code_t *self, mp_obj_dict_t *globals, mp_obj_dict_t *locals) { // save context nlr_jump_callback_node_globals_locals_t ctx; @@ -57,19 +47,28 @@ static mp_obj_t code_execute(mp_obj_code_t *self, mp_obj_dict_t *globals, mp_obj // set exception handler to restore context if an exception is raised nlr_push_jump_callback(&ctx.callback, mp_globals_locals_set_from_nlr_jump_callback); + #if MICROPY_PY_BUILTINS_CODE >= MICROPY_PY_BUILTINS_CODE_BASIC + mp_module_context_t *module_context = m_new_obj(mp_module_context_t); + module_context->module.base.type = &mp_type_module; + module_context->module.globals = globals; + module_context->constants = *mp_code_get_constants(self); + mp_obj_t module_fun = mp_make_function_from_proto_fun(mp_code_get_proto_fun(self), module_context, NULL); + #else // The call to mp_parse_compile_execute() in mp_builtin_compile() below passes // NULL for the globals, so repopulate that entry now with the correct globals. + mp_obj_t module_fun = self->module_fun; if (mp_obj_is_type(self->module_fun, &mp_type_fun_bc) #if MICROPY_EMIT_NATIVE || mp_obj_is_type(self->module_fun, &mp_type_fun_native) #endif ) { - mp_obj_fun_bc_t *fun_bc = MP_OBJ_TO_PTR(self->module_fun); + mp_obj_fun_bc_t *fun_bc = MP_OBJ_TO_PTR(module_fun); ((mp_module_context_t *)fun_bc->context)->module.globals = globals; } + #endif // execute code - mp_obj_t ret = mp_call_function_0(self->module_fun); + mp_obj_t ret = mp_call_function_0(module_fun); // deregister exception handler and restore context nlr_pop_jump_callback(true); @@ -108,9 +107,29 @@ static mp_obj_t mp_builtin_compile(size_t n_args, const mp_obj_t *args) { mp_raise_ValueError(MP_ERROR_TEXT("bad compile mode")); } - mp_obj_code_t *code = mp_obj_malloc(mp_obj_code_t, &mp_type_code); - code->module_fun = mp_parse_compile_execute(lex, parse_input_kind, NULL, NULL); - return MP_OBJ_FROM_PTR(code); + #if MICROPY_PY_BUILTINS_CODE >= MICROPY_PY_BUILTINS_CODE_BASIC + + mp_parse_tree_t parse_tree = mp_parse(lex, parse_input_kind); + mp_module_context_t ctx; + ctx.module.globals = NULL; + mp_compiled_module_t cm; + cm.context = &ctx; + mp_compile_to_raw_code(&parse_tree, lex->source_name, parse_input_kind == MP_PARSE_SINGLE_INPUT, &cm); + + #if MICROPY_PY_BUILTINS_CODE >= MICROPY_PY_BUILTINS_CODE_FULL + mp_module_context_t *ctx_ptr = m_new_obj(mp_module_context_t); + *ctx_ptr = ctx; + return mp_obj_new_code(ctx_ptr, cm.rc, true); + #else + return mp_obj_new_code(ctx.constants, cm.rc); + #endif + + #else + + mp_obj_t module_fun = mp_parse_compile_execute(lex, parse_input_kind, NULL, NULL); + return mp_obj_new_code(module_fun); + + #endif } MP_DEFINE_CONST_FUN_OBJ_VAR_BETWEEN(mp_builtin_compile_obj, 3, 6, mp_builtin_compile); diff --git a/py/circuitpy_mpconfig.h b/py/circuitpy_mpconfig.h index bc894e98606eb..3ca06f7a0fd3a 100644 --- a/py/circuitpy_mpconfig.h +++ b/py/circuitpy_mpconfig.h @@ -37,6 +37,7 @@ extern void common_hal_mcu_enable_interrupts(void); // MicroPython-only options not used by CircuitPython, but present in various files // inherited from MicroPython, especially in extmod/ #define MICROPY_ENABLE_DYNRUNTIME (0) +#define MICROPY_HW_ENABLE_USB (0) #define MICROPY_HW_ENABLE_USB_RUNTIME_DEVICE (0) #define MICROPY_PY_BLUETOOTH (0) #define MICROPY_PY_LWIP_SLIP (0) diff --git a/py/compile.c b/py/compile.c index 01904155d67c0..c7dd2fd476136 100644 --- a/py/compile.c +++ b/py/compile.c @@ -147,6 +147,7 @@ static const emit_inline_asm_method_table_t *emit_asm_table[] = { &emit_inline_thumb_method_table, &emit_inline_xtensa_method_table, NULL, + &emit_inline_rv32_method_table, }; #elif MICROPY_EMIT_INLINE_ASM @@ -157,6 +158,9 @@ static const emit_inline_asm_method_table_t *emit_asm_table[] = { #elif MICROPY_EMIT_INLINE_XTENSA #define ASM_DECORATOR_QSTR MP_QSTR_asm_xtensa #define ASM_EMITTER(f) emit_inline_xtensa_##f +#elif MICROPY_EMIT_INLINE_RV32 +#define ASM_DECORATOR_QSTR MP_QSTR_asm_rv32 +#define ASM_EMITTER(f) emit_inline_rv32_##f #else #error "unknown asm emitter" #endif @@ -855,6 +859,8 @@ static bool compile_built_in_decorator(compiler_t *comp, size_t name_len, mp_par *emit_options = MP_EMIT_OPT_ASM; } else if (attr == MP_QSTR_asm_xtensa) { *emit_options = MP_EMIT_OPT_ASM; + } else if (attr == MP_QSTR_asm_rv32) { + *emit_options = MP_EMIT_OPT_ASM; #else } else if (attr == ASM_DECORATOR_QSTR) { *emit_options = MP_EMIT_OPT_ASM; @@ -3479,7 +3485,7 @@ static void scope_compute_things(scope_t *scope) { } } -#if !MICROPY_PERSISTENT_CODE_SAVE +#if !MICROPY_EXPOSE_MP_COMPILE_TO_RAW_CODE static #endif void mp_compile_to_raw_code(mp_parse_tree_t *parse_tree, qstr source_file, bool is_repl, mp_compiled_module_t *cm) { diff --git a/py/compile.h b/py/compile.h index f9970a521d644..64afc487d7c3d 100644 --- a/py/compile.h +++ b/py/compile.h @@ -30,6 +30,9 @@ #include "py/parse.h" #include "py/emitglue.h" +// Whether mp_compile_to_raw_code is exposed as a public function. +#define MICROPY_EXPOSE_MP_COMPILE_TO_RAW_CODE (MICROPY_PY_BUILTINS_CODE >= MICROPY_PY_BUILTINS_CODE_BASIC || MICROPY_PERSISTENT_CODE_SAVE) + #if MICROPY_COMP_ALLOW_TOP_LEVEL_AWAIT // set to `true` to allow top-level await expressions extern bool mp_compile_allow_top_level_await; @@ -40,7 +43,7 @@ extern bool mp_compile_allow_top_level_await; // mp_globals_get() will be used for the context mp_obj_t mp_compile(mp_parse_tree_t *parse_tree, qstr source_file, bool is_repl); -#if MICROPY_PERSISTENT_CODE_SAVE +#if MICROPY_EXPOSE_MP_COMPILE_TO_RAW_CODE // this has the same semantics as mp_compile void mp_compile_to_raw_code(mp_parse_tree_t *parse_tree, qstr source_file, bool is_repl, mp_compiled_module_t *cm); #endif diff --git a/py/dynruntime.h b/py/dynruntime.h index 44f6d05ccb6f5..e87cf6591c4b6 100644 --- a/py/dynruntime.h +++ b/py/dynruntime.h @@ -28,6 +28,14 @@ // This header file contains definitions to dynamically implement the static // MicroPython runtime API defined in py/obj.h and py/runtime.h. +// +// All of the symbols made available in this header are overriding those defined +// in py/obj.h and py/runtime.h. This is done with macros. For macros that +// would be too complicated (usually more than a single expression), they call a +// static-inline function for the implementation. This function has the same +// name as the macro (hence the same name as a public API function) but with +// "_dyn" appended. For example, the m_malloc() macro calls the m_malloc_dyn() +// static-inline function. #include "py/binary.h" #include "py/nativeglue.h" @@ -39,6 +47,10 @@ #error "dynruntime.h included in non-dynamic-module build." #endif +#if MICROPY_MALLOC_USES_ALLOCATED_SIZE +#error "MICROPY_MALLOC_USES_ALLOCATED_SIZE must be disable in a dynamic-module build." +#endif + #undef MP_ROM_QSTR #undef MP_OBJ_QSTR_VALUE #undef MP_OBJ_NEW_QSTR @@ -52,13 +64,32 @@ /******************************************************************************/ // Memory allocation +#define m_malloc_fail(num_bytes) (m_malloc_fail_dyn((num_bytes))) #define m_malloc(n) (m_malloc_dyn((n))) #define m_free(ptr) (m_free_dyn((ptr))) #define m_realloc(ptr, new_num_bytes) (m_realloc_dyn((ptr), (new_num_bytes))) +#define m_realloc_maybe(ptr, new_num_bytes, allow_move) (m_realloc_maybe_dyn((ptr), (new_num_bytes), (allow_move))) + +static NORETURN inline void m_malloc_fail_dyn(size_t num_bytes) { + mp_fun_table.raise_msg( + mp_fun_table.load_global(MP_QSTR_MemoryError), + "memory allocation failed"); +} + +static inline void *m_realloc_maybe_dyn(void *ptr, size_t new_num_bytes, bool allow_move) { + return mp_fun_table.realloc_(ptr, new_num_bytes, allow_move); +} + +static inline void *m_realloc_checked_dyn(void *ptr, size_t new_num_bytes, bool allow_move) { + ptr = m_realloc_maybe(ptr, new_num_bytes, allow_move); + if (ptr == NULL && new_num_bytes != 0) { + m_malloc_fail(new_num_bytes); + } + return ptr; +} static inline void *m_malloc_dyn(size_t n) { - // TODO won't raise on OOM - return mp_fun_table.realloc_(NULL, n, false); + return m_realloc_checked_dyn(NULL, n, false); } static inline void m_free_dyn(void *ptr) { @@ -66,8 +97,7 @@ static inline void m_free_dyn(void *ptr) { } static inline void *m_realloc_dyn(void *ptr, size_t new_num_bytes) { - // TODO won't raise on OOM - return mp_fun_table.realloc_(ptr, new_num_bytes, true); + return m_realloc_checked_dyn(ptr, new_num_bytes, true); } /******************************************************************************/ diff --git a/py/dynruntime.mk b/py/dynruntime.mk index 62db43ad149ca..807befb464a84 100644 --- a/py/dynruntime.mk +++ b/py/dynruntime.mk @@ -29,15 +29,18 @@ CFLAGS += -Wall -Werror -DNDEBUG CFLAGS += -DNO_QSTR CFLAGS += -DMICROPY_ENABLE_DYNRUNTIME CFLAGS += -DMP_CONFIGFILE='<$(CONFIG_H)>' -CFLAGS += -fpic -fno-common -CFLAGS += -U _FORTIFY_SOURCE # prevent use of __*_chk libc functions -#CFLAGS += -fdata-sections -ffunction-sections + +CFLAGS_ARCH += -fpic -fno-common +CFLAGS_ARCH += -U_FORTIFY_SOURCE # prevent use of __*_chk libc functions +#CFLAGS_ARCH += -fdata-sections -ffunction-sections MPY_CROSS_FLAGS += -march=$(ARCH) SRC_O += $(addprefix $(BUILD)/, $(patsubst %.c,%.o,$(filter %.c,$(SRC))) $(patsubst %.S,%.o,$(filter %.S,$(SRC)))) SRC_MPY += $(addprefix $(BUILD)/, $(patsubst %.py,%.mpy,$(filter %.py,$(SRC)))) +CLEAN_EXTRA += $(MOD).mpy .mpy_ld_cache + ################################################################################ # Architecture configuration @@ -45,66 +48,124 @@ ifeq ($(ARCH),x86) # x86 CROSS = -CFLAGS += -m32 -fno-stack-protector +CFLAGS_ARCH += -m32 -fno-stack-protector MICROPY_FLOAT_IMPL ?= double else ifeq ($(ARCH),x64) # x64 CROSS = -CFLAGS += -fno-stack-protector +CFLAGS_ARCH += -fno-stack-protector MICROPY_FLOAT_IMPL ?= double else ifeq ($(ARCH),armv6m) # thumb CROSS = arm-none-eabi- -CFLAGS += -mthumb -mcpu=cortex-m0 +CFLAGS_ARCH += -mthumb -mcpu=cortex-m0 MICROPY_FLOAT_IMPL ?= none else ifeq ($(ARCH),armv7m) # thumb CROSS = arm-none-eabi- -CFLAGS += -mthumb -mcpu=cortex-m3 +CFLAGS_ARCH += -mthumb -mcpu=cortex-m3 MICROPY_FLOAT_IMPL ?= none else ifeq ($(ARCH),armv7emsp) # thumb CROSS = arm-none-eabi- -CFLAGS += -mthumb -mcpu=cortex-m4 -CFLAGS += -mfpu=fpv4-sp-d16 -mfloat-abi=hard +CFLAGS_ARCH += -mthumb -mcpu=cortex-m4 +CFLAGS_ARCH += -mfpu=fpv4-sp-d16 -mfloat-abi=hard MICROPY_FLOAT_IMPL ?= float else ifeq ($(ARCH),armv7emdp) # thumb CROSS = arm-none-eabi- -CFLAGS += -mthumb -mcpu=cortex-m7 -CFLAGS += -mfpu=fpv5-d16 -mfloat-abi=hard +CFLAGS_ARCH += -mthumb -mcpu=cortex-m7 +CFLAGS_ARCH += -mfpu=fpv5-d16 -mfloat-abi=hard MICROPY_FLOAT_IMPL ?= double else ifeq ($(ARCH),xtensa) # xtensa CROSS = xtensa-lx106-elf- -CFLAGS += -mforce-l32 +CFLAGS_ARCH += -mforce-l32 MICROPY_FLOAT_IMPL ?= none else ifeq ($(ARCH),xtensawin) # xtensawin CROSS = xtensa-esp32-elf- -CFLAGS += MICROPY_FLOAT_IMPL ?= float +else ifeq ($(ARCH),rv32imc) + +# rv32imc +CROSS = riscv64-unknown-elf- +CFLAGS_ARCH += -march=rv32imac -mabi=ilp32 -mno-relax +# If Picolibc is available then select it explicitly. Ubuntu 22.04 ships its +# bare metal RISC-V toolchain with Picolibc rather than Newlib, and the default +# is "nosys" so a value must be provided. To avoid having per-distro +# workarounds, always select Picolibc if available. +PICOLIBC_SPECS := $(shell $(CROSS)gcc --print-file-name=picolibc.specs) +ifneq ($(PICOLIBC_SPECS),picolibc.specs) +CFLAGS_ARCH += -specs=$(PICOLIBC_SPECS) +USE_PICOLIBC := 1 +PICOLIBC_ARCH := rv32imac +PICOLIBC_ABI := ilp32 +endif + +MICROPY_FLOAT_IMPL ?= none + else $(error architecture '$(ARCH)' not supported) endif MICROPY_FLOAT_IMPL_UPPER = $(shell echo $(MICROPY_FLOAT_IMPL) | tr '[:lower:]' '[:upper:]') -CFLAGS += -DMICROPY_FLOAT_IMPL=MICROPY_FLOAT_IMPL_$(MICROPY_FLOAT_IMPL_UPPER) +CFLAGS += $(CFLAGS_ARCH) -DMICROPY_FLOAT_IMPL=MICROPY_FLOAT_IMPL_$(MICROPY_FLOAT_IMPL_UPPER) + +ifeq ($(LINK_RUNTIME),1) +# All of these picolibc-specific directives are here to work around a +# limitation of Ubuntu 22.04's RISC-V bare metal toolchain. In short, the +# specific version of GCC in use (10.2.0) does not seem to take into account +# extra paths provided by an explicitly passed specs file when performing name +# resolution via `--print-file-name`. +# +# If Picolibc is used and libc.a fails to resolve, then said file's path will +# be computed by searching the Picolibc libraries root for a libc.a file in a +# subdirectory whose path is built using the current `-march` and `-mabi` +# flags that are passed to GCC. The `PICOLIBC_ROOT` environment variable is +# checked to override the starting point for the library file search, and if +# it is not set then the default value is used, assuming that this is running +# on an Ubuntu 22.04 machine. +# +# This should be revised when the CI base image is updated to a newer Ubuntu +# version (that hopefully contains a newer RISC-V compiler) or to another Linux +# distribution. +ifeq ($(USE_PICOLIBC),1) +LIBM_NAME := libc.a +else +LIBM_NAME := libm.a +endif +LIBGCC_PATH := $(realpath $(shell $(CROSS)gcc $(CFLAGS) --print-libgcc-file-name)) +LIBM_PATH := $(realpath $(shell $(CROSS)gcc $(CFLAGS) --print-file-name=$(LIBM_NAME))) +ifeq ($(USE_PICOLIBC),1) +ifeq ($(LIBM_PATH),) +# The CROSS toolchain prefix usually ends with a dash, but that may not be +# always the case. If the prefix ends with a dash it has to be taken out as +# Picolibc's architecture directory won't have it in its name. GNU Make does +# not have any facility to perform character-level text manipulation so we +# shell out to sed. +CROSS_PREFIX := $(shell echo $(CROSS) | sed -e 's/-$$//') +PICOLIBC_ROOT ?= /usr/lib/picolibc/$(CROSS_PREFIX)/lib +LIBM_PATH := $(PICOLIBC_ROOT)/$(PICOLIBC_ARCH)/$(PICOLIBC_ABI)/$(LIBM_NAME) +endif +endif +MPY_LD_FLAGS += $(addprefix -l, $(LIBGCC_PATH) $(LIBM_PATH)) +endif CFLAGS += $(CFLAGS_EXTRA) @@ -147,7 +208,7 @@ $(BUILD)/%.mpy: %.py # Build native .mpy from object files $(BUILD)/$(MOD).native.mpy: $(SRC_O) $(ECHO) "LINK $<" - $(Q)$(MPY_LD) --arch $(ARCH) --qstrs $(CONFIG_H) -o $@ $^ + $(Q)$(MPY_LD) --arch $(ARCH) --qstrs $(CONFIG_H) $(MPY_LD_FLAGS) -o $@ $^ # Build final .mpy from all intermediate .mpy files $(MOD).mpy: $(BUILD)/$(MOD).native.mpy $(SRC_MPY) diff --git a/py/emit.h b/py/emit.h index 623b163490164..033ac9c763b07 100644 --- a/py/emit.h +++ b/py/emit.h @@ -307,12 +307,15 @@ typedef struct _emit_inline_asm_method_table_t { void (*op)(emit_inline_asm_t *emit, qstr op, mp_uint_t n_args, mp_parse_node_t *pn_args); } emit_inline_asm_method_table_t; +extern const emit_inline_asm_method_table_t emit_inline_rv32_method_table; extern const emit_inline_asm_method_table_t emit_inline_thumb_method_table; extern const emit_inline_asm_method_table_t emit_inline_xtensa_method_table; +emit_inline_asm_t *emit_inline_rv32_new(mp_uint_t max_num_labels); emit_inline_asm_t *emit_inline_thumb_new(mp_uint_t max_num_labels); emit_inline_asm_t *emit_inline_xtensa_new(mp_uint_t max_num_labels); +void emit_inline_rv32_free(emit_inline_asm_t *emit); void emit_inline_thumb_free(emit_inline_asm_t *emit); void emit_inline_xtensa_free(emit_inline_asm_t *emit); diff --git a/py/emitglue.c b/py/emitglue.c index e5d9465ed7c11..8cc14bdc54eb5 100644 --- a/py/emitglue.c +++ b/py/emitglue.c @@ -122,7 +122,7 @@ void mp_emit_glue_assign_native(mp_raw_code_t *rc, mp_raw_code_kind_t kind, cons #endif #elif MICROPY_EMIT_ARM #if (defined(__linux__) && defined(__GNUC__)) || __ARM_ARCH == 7 - __builtin___clear_cache((void *)fun_data, (uint8_t *)fun_data + fun_len); + __builtin___clear_cache((void *)fun_data, (char *)fun_data + fun_len); #elif defined(__arm__) // Flush I-cache and D-cache. asm volatile ( diff --git a/py/emitinlinerv32.c b/py/emitinlinerv32.c new file mode 100644 index 0000000000000..a539242b84d95 --- /dev/null +++ b/py/emitinlinerv32.c @@ -0,0 +1,851 @@ +/* + * This file is part of the MicroPython project, https://micropython.org/ + * + * The MIT License (MIT) + * + * Copyright (c) 2024 Alessandro Gatti + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ + +#include +#include +#include +#include +#include + +#include "py/emit.h" +#include "py/misc.h" + +#if MICROPY_EMIT_INLINE_RV32 + +#include "py/asmrv32.h" + +typedef enum { +// define rules with a compile function +#define DEF_RULE(rule, comp, kind, ...) PN_##rule, +#define DEF_RULE_NC(rule, kind, ...) + #include "py/grammar.h" +#undef DEF_RULE +#undef DEF_RULE_NC + PN_const_object, // special node for a constant, generic Python object +// define rules without a compile function +#define DEF_RULE(rule, comp, kind, ...) +#define DEF_RULE_NC(rule, kind, ...) PN_##rule, + #include "py/grammar.h" +#undef DEF_RULE +#undef DEF_RULE_NC +} pn_kind_t; + +struct _emit_inline_asm_t { + asm_rv32_t as; + uint16_t pass; + mp_obj_t *error_slot; + mp_uint_t max_num_labels; + qstr *label_lookup; +}; + +static const qstr_short_t REGISTERS_QSTR_TABLE[] = { + MP_QSTR_zero, MP_QSTR_ra, MP_QSTR_sp, MP_QSTR_gp, MP_QSTR_tp, MP_QSTR_t0, MP_QSTR_t1, MP_QSTR_t2, + MP_QSTR_s0, MP_QSTR_s1, MP_QSTR_a0, MP_QSTR_a1, MP_QSTR_a2, MP_QSTR_a3, MP_QSTR_a4, MP_QSTR_a5, + MP_QSTR_a6, MP_QSTR_a7, MP_QSTR_s2, MP_QSTR_s3, MP_QSTR_s4, MP_QSTR_s5, MP_QSTR_s6, MP_QSTR_s7, + MP_QSTR_s8, MP_QSTR_s9, MP_QSTR_s10, MP_QSTR_s11, MP_QSTR_t3, MP_QSTR_t4, MP_QSTR_t5, MP_QSTR_t6, + MP_QSTR_x0, MP_QSTR_x1, MP_QSTR_x2, MP_QSTR_x3, MP_QSTR_x4, MP_QSTR_x5, MP_QSTR_x6, MP_QSTR_x7, + MP_QSTR_x8, MP_QSTR_x9, MP_QSTR_x10, MP_QSTR_x11, MP_QSTR_x12, MP_QSTR_x13, MP_QSTR_x14, MP_QSTR_x15, + MP_QSTR_x16, MP_QSTR_x17, MP_QSTR_x18, MP_QSTR_x19, MP_QSTR_x20, MP_QSTR_x21, MP_QSTR_x22, MP_QSTR_x23, + MP_QSTR_x24, MP_QSTR_x25, MP_QSTR_x26, MP_QSTR_x27, MP_QSTR_x28, MP_QSTR_x29, MP_QSTR_x30, MP_QSTR_x31, +}; + +//////////////////////////////////////////////////////////////////////////////// + +static inline void emit_inline_rv32_error_msg(emit_inline_asm_t *emit, mp_rom_error_text_t msg) { + *emit->error_slot = mp_obj_new_exception_msg(&mp_type_SyntaxError, msg); +} + +static inline void emit_inline_rv32_error_exc(emit_inline_asm_t *emit, mp_obj_t exc) { + *emit->error_slot = exc; +} + +emit_inline_asm_t *emit_inline_rv32_new(mp_uint_t max_num_labels) { + emit_inline_asm_t *emit = m_new_obj(emit_inline_asm_t); + memset(&emit->as, 0, sizeof(emit->as)); + mp_asm_base_init(&emit->as.base, max_num_labels); + emit->max_num_labels = max_num_labels; + emit->label_lookup = m_new(qstr, max_num_labels); + return emit; +} + +void emit_inline_rv32_free(emit_inline_asm_t *emit) { + m_del(qstr, emit->label_lookup, emit->max_num_labels); + mp_asm_base_deinit(&emit->as.base, false); + m_del_obj(emit_inline_asm_t, emit); +} + +static void emit_inline_rv32_start_pass(emit_inline_asm_t *emit, pass_kind_t pass, mp_obj_t *error_slot) { + emit->pass = pass; + emit->error_slot = error_slot; + if (emit->pass == MP_PASS_CODE_SIZE) { + memset(emit->label_lookup, 0, emit->max_num_labels * sizeof(qstr)); + } + mp_asm_base_start_pass(&emit->as.base, pass == MP_PASS_EMIT ? MP_ASM_PASS_EMIT : MP_ASM_PASS_COMPUTE); +} + +static void emit_inline_rv32_end_pass(emit_inline_asm_t *emit, mp_uint_t type_sig) { + // c.jr ra + asm_rv32_opcode_cjr(&emit->as, ASM_RV32_REG_RA); + asm_rv32_end_pass(&emit->as); +} + +static bool parse_register_node(mp_parse_node_t node, mp_uint_t *register_number, bool compressed) { + assert(register_number != NULL && "Register number pointer is NULL."); + + if (!MP_PARSE_NODE_IS_ID(node)) { + return false; + } + + qstr node_qstr = MP_PARSE_NODE_LEAF_ARG(node); + for (mp_uint_t index = 0; index < MP_ARRAY_SIZE(REGISTERS_QSTR_TABLE); index++) { + if (node_qstr == REGISTERS_QSTR_TABLE[index]) { + mp_uint_t number = index % RV32_AVAILABLE_REGISTERS_COUNT; + if (!compressed || (compressed && RV32_IS_IN_C_REGISTER_WINDOW(number))) { + *register_number = compressed ? RV32_MAP_IN_C_REGISTER_WINDOW(number) : number; + return true; + } + break; + } + } + + return false; +} + +static mp_uint_t lookup_label(emit_inline_asm_t *emit, mp_parse_node_t node, qstr *qstring) { + assert(qstring && "qstring pointer is NULL"); + + *qstring = MP_PARSE_NODE_LEAF_ARG(node); + for (mp_uint_t label = 0; label < emit->max_num_labels; label++) { + if (emit->label_lookup[label] == *qstring) { + return label; + } + } + + return emit->max_num_labels; +} + +static inline ptrdiff_t label_code_offset(emit_inline_asm_t *emit, mp_uint_t label_index) { + return emit->as.base.label_offsets[label_index] - emit->as.base.code_offset; +} + +static mp_uint_t emit_inline_rv32_count_params(emit_inline_asm_t *emit, mp_uint_t parameters_count, mp_parse_node_t *parameter_nodes) { + // TODO: Raise this up to 8? RV32I has 8 A-registers that are meant to + // be used for passing arguments. + + if (parameters_count > 4) { + emit_inline_rv32_error_msg(emit, MP_ERROR_TEXT("can only have up to 4 parameters for RV32 assembly")); + return 0; + } + + mp_uint_t register_index = 0; + for (mp_uint_t index = 0; index < parameters_count; index++) { + bool valid_register = parse_register_node(parameter_nodes[index], ®ister_index, false); + if (!valid_register || (register_index != (ASM_RV32_REG_A0 + index))) { + emit_inline_rv32_error_msg(emit, MP_ERROR_TEXT("parameters must be registers in sequence a0 to a3")); + return 0; + } + } + + return parameters_count; +} + +static bool emit_inline_rv32_label(emit_inline_asm_t *emit, mp_uint_t label_num, qstr label_id) { + assert(label_num < emit->max_num_labels); + if (emit->pass == MP_PASS_CODE_SIZE) { + for (mp_uint_t index = 0; index < emit->max_num_labels; index++) { + if (emit->label_lookup[index] == label_id) { + return false; + } + } + } + emit->label_lookup[label_num] = label_id; + mp_asm_base_label_assign(&emit->as.base, label_num); + return true; +} + +typedef enum { + CALL_RRR, // Opcode Register, Register, Register + CALL_RR, // Opcode Register, Register + CALL_RRI, // Opcode Register, Register, Immediate + CALL_RRL, // Opcode Register, Register, Label + CALL_RI, // Opcode Register, Immediate + CALL_L, // Opcode Label + CALL_R, // Opcode Register + CALL_RL, // Opcode Register, Label + CALL_N, // Opcode + CALL_I, // Opcode Immediate + CALL_RII, // Opcode Register, Register, Immediate + CALL_RIR, // Opcode Register, Immediate(Register) + CALL_COUNT +} call_convention_t; + +#define N 0 // No argument +#define R 1 // Register +#define I 2 // Immediate +#define L 3 // Label +#define C (1 << 2) // Compressed register +#define U (1 << 2) // Unsigned immediate +#define Z (1 << 3) // Non-zero + +typedef void (*call_l_t)(asm_rv32_t *state, mp_uint_t label_index); +typedef void (*call_ri_t)(asm_rv32_t *state, mp_uint_t rd, mp_int_t immediate); +typedef void (*call_rri_t)(asm_rv32_t *state, mp_uint_t rd, mp_uint_t rs1, mp_int_t immediate); +typedef void (*call_rii_t)(asm_rv32_t *state, mp_uint_t rd, mp_uint_t immediate1, mp_int_t immediate2); +typedef void (*call_rrr_t)(asm_rv32_t *state, mp_uint_t rd, mp_uint_t rs1, mp_uint_t rs2); +typedef void (*call_rr_t)(asm_rv32_t *state, mp_uint_t rd, mp_uint_t rs); +typedef void (*call_i_t)(asm_rv32_t *state, mp_int_t immediate); +typedef void (*call_r_t)(asm_rv32_t *state, mp_uint_t rd); +typedef void (*call_n_t)(asm_rv32_t *state); + +typedef struct _opcode_t { + qstr_short_t qstring; + uint16_t argument1_mask : 4; + uint16_t argument2_mask : 4; + uint16_t argument3_mask : 4; + uint16_t arguments_count : 2; + // 2 bits available here + uint32_t calling_convention : 4; + uint32_t argument1_kind : 4; + uint32_t argument1_shift : 4; + uint32_t argument2_kind : 4; + uint32_t argument2_shift : 4; + uint32_t argument3_kind : 4; + uint32_t argument3_shift : 4; + // 4 bits available here + void *emitter; +} opcode_t; + +#define opcode_li asm_rv32_emit_optimised_load_immediate + +static void opcode_la(asm_rv32_t *state, mp_uint_t rd, mp_int_t displacement) { + // This cannot be optimised for size, otherwise label addresses would move around. + mp_uint_t upper = (mp_uint_t)displacement & 0xFFFFF000; + mp_uint_t lower = (mp_uint_t)displacement & 0x00000FFF; + if ((lower & 0x800) != 0) { + upper += 0x1000; + } + asm_rv32_opcode_auipc(state, rd, upper); + asm_rv32_opcode_addi(state, rd, rd, lower); +} + +#define RC (R | C) +#define IU (I | U) +#define IZ (I | Z) +#define IUZ (I | U | Z) + +#define MASK_NOT_USED 0 + +enum { + MASK_FFFFFFFF, + MASK_00000FFF, + MASK_FFFFF000, + MASK_00001FFE, + MASK_0000001F, + MASK_FFFFFFFE, + MASK_0000003F, + MASK_0000FF00, + MASK_000003FC, + MASK_000001FE, + MASK_00000FFE, + MASK_FFFFFFFA, + MASK_0001F800, + MASK_0000007C, + MASK_000000FC, + MASK_001FFFFE, +}; + +static const uint32_t OPCODE_MASKS[] = { + [MASK_FFFFFFFF] = 0xFFFFFFFF, + [MASK_00000FFF] = 0x00000FFF, + [MASK_FFFFF000] = 0xFFFFF000, + [MASK_00001FFE] = 0x00001FFE, + [MASK_0000001F] = 0x0000001F, + [MASK_FFFFFFFE] = 0xFFFFFFFE, + [MASK_0000003F] = 0x0000003F, + [MASK_0000FF00] = 0x0000FF00, + [MASK_000003FC] = 0x000003FC, + [MASK_000001FE] = 0x000001FE, + [MASK_00000FFE] = 0x00000FFE, + [MASK_FFFFFFFA] = 0xFFFFFFFA, + [MASK_0001F800] = 0x0001F800, + [MASK_0000007C] = 0x0000007C, + [MASK_000000FC] = 0x000000FC, + [MASK_001FFFFE] = 0x001FFFFE, +}; + +static const opcode_t OPCODES[] = { + { MP_QSTR_add, MASK_FFFFFFFF, MASK_FFFFFFFF, MASK_FFFFFFFF, 3, CALL_RRR, R, 0, R, 0, R, 0, asm_rv32_opcode_add }, + { MP_QSTR_addi, MASK_FFFFFFFF, MASK_FFFFFFFF, MASK_00000FFF, 3, CALL_RRI, R, 0, R, 0, I, 0, asm_rv32_opcode_addi }, + { MP_QSTR_and_, MASK_FFFFFFFF, MASK_FFFFFFFF, MASK_FFFFFFFF, 3, CALL_RRR, R, 0, R, 0, R, 0, asm_rv32_opcode_and }, + { MP_QSTR_andi, MASK_FFFFFFFF, MASK_FFFFFFFF, MASK_00000FFF, 3, CALL_RRI, R, 0, R, 0, I, 0, asm_rv32_opcode_andi }, + { MP_QSTR_auipc, MASK_FFFFFFFF, MASK_FFFFF000, MASK_NOT_USED, 2, CALL_RI, R, 0, I, 12, N, 0, asm_rv32_opcode_auipc }, + { MP_QSTR_beq, MASK_FFFFFFFF, MASK_FFFFFFFF, MASK_00001FFE, 3, CALL_RRL, R, 0, R, 0, L, 0, asm_rv32_opcode_beq }, + { MP_QSTR_bge, MASK_FFFFFFFF, MASK_FFFFFFFF, MASK_00001FFE, 3, CALL_RRL, R, 0, R, 0, L, 0, asm_rv32_opcode_bge }, + { MP_QSTR_bgeu, MASK_FFFFFFFF, MASK_FFFFFFFF, MASK_00001FFE, 3, CALL_RRL, R, 0, R, 0, L, 0, asm_rv32_opcode_bgeu }, + { MP_QSTR_blt, MASK_FFFFFFFF, MASK_FFFFFFFF, MASK_00001FFE, 3, CALL_RRL, R, 0, R, 0, L, 0, asm_rv32_opcode_blt }, + { MP_QSTR_bltu, MASK_FFFFFFFF, MASK_FFFFFFFF, MASK_00001FFE, 3, CALL_RRL, R, 0, R, 0, L, 0, asm_rv32_opcode_bltu }, + { MP_QSTR_bne, MASK_FFFFFFFF, MASK_FFFFFFFF, MASK_00001FFE, 3, CALL_RRL, R, 0, R, 0, L, 0, asm_rv32_opcode_bne }, + { MP_QSTR_csrrc, MASK_FFFFFFFF, MASK_FFFFFFFF, MASK_00000FFF, 3, CALL_RRI, R, 0, R, 0, IU, 0, asm_rv32_opcode_csrrc }, + { MP_QSTR_csrrs, MASK_FFFFFFFF, MASK_FFFFFFFF, MASK_00000FFF, 3, CALL_RRI, R, 0, R, 0, IU, 0, asm_rv32_opcode_csrrs }, + { MP_QSTR_csrrw, MASK_FFFFFFFF, MASK_FFFFFFFF, MASK_00000FFF, 3, CALL_RRI, R, 0, R, 0, IU, 0, asm_rv32_opcode_csrrw }, + { MP_QSTR_csrrci, MASK_FFFFFFFF, MASK_00000FFF, MASK_0000001F, 3, CALL_RII, R, 0, IU, 0, IU, 0, asm_rv32_opcode_csrrci }, + { MP_QSTR_csrrsi, MASK_FFFFFFFF, MASK_00000FFF, MASK_0000001F, 3, CALL_RII, R, 0, IU, 0, IU, 0, asm_rv32_opcode_csrrsi }, + { MP_QSTR_csrrwi, MASK_FFFFFFFF, MASK_00000FFF, MASK_0000001F, 3, CALL_RII, R, 0, IU, 0, IU, 0, asm_rv32_opcode_csrrwi }, + { MP_QSTR_c_add, MASK_FFFFFFFE, MASK_FFFFFFFE, MASK_NOT_USED, 2, CALL_RR, R, 0, R, 0, N, 0, asm_rv32_opcode_cadd }, + { MP_QSTR_c_addi, MASK_FFFFFFFE, MASK_0000003F, MASK_NOT_USED, 2, CALL_RI, R, 0, IZ, 0, N, 0, asm_rv32_opcode_caddi }, + { MP_QSTR_c_addi4spn, MASK_0000FF00, MASK_000003FC, MASK_NOT_USED, 2, CALL_RI, R, 0, IUZ, 0, N, 0, asm_rv32_opcode_caddi4spn }, + { MP_QSTR_c_and, MASK_0000FF00, MASK_0000FF00, MASK_NOT_USED, 2, CALL_RR, RC, 0, RC, 0, N, 0, asm_rv32_opcode_cand }, + { MP_QSTR_c_andi, MASK_0000FF00, MASK_0000003F, MASK_NOT_USED, 2, CALL_RI, RC, 0, I, 0, N, 0, asm_rv32_opcode_candi }, + { MP_QSTR_c_beqz, MASK_0000FF00, MASK_000001FE, MASK_NOT_USED, 2, CALL_RL, RC, 0, L, 0, N, 0, asm_rv32_opcode_cbeqz }, + { MP_QSTR_c_bnez, MASK_0000FF00, MASK_000001FE, MASK_NOT_USED, 2, CALL_RL, RC, 0, L, 0, N, 0, asm_rv32_opcode_cbnez }, + { MP_QSTR_c_ebreak, MASK_NOT_USED, MASK_NOT_USED, MASK_NOT_USED, 0, CALL_N, N, 0, N, 0, N, 0, asm_rv32_opcode_cebreak }, + { MP_QSTR_c_j, MASK_00000FFE, MASK_NOT_USED, MASK_NOT_USED, 1, CALL_L, L, 0, N, 0, N, 0, asm_rv32_opcode_cj }, + { MP_QSTR_c_jal, MASK_00000FFE, MASK_NOT_USED, MASK_NOT_USED, 1, CALL_L, L, 0, N, 0, N, 0, asm_rv32_opcode_cjal }, + { MP_QSTR_c_jalr, MASK_FFFFFFFE, MASK_NOT_USED, MASK_NOT_USED, 1, CALL_R, R, 0, N, 0, N, 0, asm_rv32_opcode_cjalr }, + { MP_QSTR_c_jr, MASK_FFFFFFFE, MASK_NOT_USED, MASK_NOT_USED, 1, CALL_R, R, 0, N, 0, N, 0, asm_rv32_opcode_cjr }, + { MP_QSTR_c_li, MASK_FFFFFFFE, MASK_0000003F, MASK_NOT_USED, 2, CALL_RI, R, 0, I, 0, N, 0, asm_rv32_opcode_cli }, + { MP_QSTR_c_lui, MASK_FFFFFFFA, MASK_0001F800, MASK_NOT_USED, 2, CALL_RI, R, 0, IUZ, 12, N, 0, asm_rv32_opcode_clui }, + { MP_QSTR_c_lw, MASK_0000FF00, MASK_0000007C, MASK_0000FF00, 3, CALL_RIR, RC, 0, I, 0, RC, 0, asm_rv32_opcode_clw }, + { MP_QSTR_c_lwsp, MASK_FFFFFFFE, MASK_000000FC, MASK_NOT_USED, 2, CALL_RI, R, 0, I, 0, N, 0, asm_rv32_opcode_clwsp }, + { MP_QSTR_c_mv, MASK_FFFFFFFE, MASK_FFFFFFFE, MASK_NOT_USED, 2, CALL_RR, R, 0, R, 0, N, 0, asm_rv32_opcode_cmv }, + { MP_QSTR_c_nop, MASK_NOT_USED, MASK_NOT_USED, MASK_NOT_USED, 0, CALL_N, N, 0, N, 0, N, 0, asm_rv32_opcode_cnop }, + { MP_QSTR_c_or, MASK_0000FF00, MASK_0000FF00, MASK_NOT_USED, 2, CALL_RR, RC, 0, RC, 0, N, 0, asm_rv32_opcode_cor }, + { MP_QSTR_c_slli, MASK_FFFFFFFE, MASK_0000001F, MASK_NOT_USED, 2, CALL_RI, R, 0, IU, 0, N, 0, asm_rv32_opcode_cslli }, + { MP_QSTR_c_srai, MASK_0000FF00, MASK_0000001F, MASK_NOT_USED, 2, CALL_RI, RC, 0, IU, 0, N, 0, asm_rv32_opcode_csrai }, + { MP_QSTR_c_srli, MASK_0000FF00, MASK_0000001F, MASK_NOT_USED, 2, CALL_RI, RC, 0, IU, 0, N, 0, asm_rv32_opcode_csrli }, + { MP_QSTR_c_sub, MASK_0000FF00, MASK_0000FF00, MASK_NOT_USED, 2, CALL_RR, RC, 0, RC, 0, N, 0, asm_rv32_opcode_csub }, + { MP_QSTR_c_sw, MASK_0000FF00, MASK_0000007C, MASK_0000FF00, 3, CALL_RIR, RC, 0, I, 0, RC, 0, asm_rv32_opcode_csw }, + { MP_QSTR_c_swsp, MASK_FFFFFFFF, MASK_000000FC, MASK_NOT_USED, 2, CALL_RI, R, 0, I, 0, N, 0, asm_rv32_opcode_cswsp }, + { MP_QSTR_c_xor, MASK_0000FF00, MASK_0000FF00, MASK_NOT_USED, 2, CALL_RR, RC, 0, RC, 0, N, 0, asm_rv32_opcode_cxor }, + { MP_QSTR_div, MASK_FFFFFFFF, MASK_FFFFFFFF, MASK_FFFFFFFF, 3, CALL_RRR, R, 0, R, 0, R, 0, asm_rv32_opcode_div }, + { MP_QSTR_divu, MASK_FFFFFFFF, MASK_FFFFFFFF, MASK_FFFFFFFF, 3, CALL_RRR, R, 0, R, 0, R, 0, asm_rv32_opcode_divu }, + { MP_QSTR_ebreak, MASK_NOT_USED, MASK_NOT_USED, MASK_NOT_USED, 0, CALL_N, N, 0, N, 0, N, 0, asm_rv32_opcode_ebreak }, + { MP_QSTR_ecall, MASK_NOT_USED, MASK_NOT_USED, MASK_NOT_USED, 0, CALL_N, N, 0, N, 0, N, 0, asm_rv32_opcode_ecall }, + { MP_QSTR_jal, MASK_FFFFFFFF, MASK_001FFFFE, MASK_NOT_USED, 2, CALL_RL, R, 0, L, 0, N, 0, asm_rv32_opcode_jal }, + { MP_QSTR_jalr, MASK_FFFFFFFF, MASK_FFFFFFFF, MASK_00000FFF, 3, CALL_RRI, R, 0, R, 0, I, 0, asm_rv32_opcode_jalr }, + { MP_QSTR_la, MASK_FFFFFFFF, MASK_FFFFFFFF, MASK_NOT_USED, 2, CALL_RL, R, 0, L, 0, N, 0, opcode_la }, + { MP_QSTR_lb, MASK_FFFFFFFF, MASK_00000FFF, MASK_FFFFFFFF, 3, CALL_RIR, R, 0, I, 0, R, 0, asm_rv32_opcode_lb }, + { MP_QSTR_lbu, MASK_FFFFFFFF, MASK_00000FFF, MASK_FFFFFFFF, 3, CALL_RIR, R, 0, I, 0, R, 0, asm_rv32_opcode_lbu }, + { MP_QSTR_lh, MASK_FFFFFFFF, MASK_00000FFF, MASK_FFFFFFFF, 3, CALL_RIR, R, 0, I, 0, R, 0, asm_rv32_opcode_lh }, + { MP_QSTR_lhu, MASK_FFFFFFFF, MASK_00000FFF, MASK_FFFFFFFF, 3, CALL_RIR, R, 0, I, 0, R, 0, asm_rv32_opcode_lhu }, + { MP_QSTR_li, MASK_FFFFFFFF, MASK_FFFFFFFF, MASK_NOT_USED, 2, CALL_RI, R, 0, I, 0, N, 0, opcode_li }, + { MP_QSTR_lui, MASK_FFFFFFFF, MASK_FFFFF000, MASK_NOT_USED, 2, CALL_RI, R, 0, I, 12, N, 0, asm_rv32_opcode_lui }, + { MP_QSTR_lw, MASK_FFFFFFFF, MASK_00000FFF, MASK_FFFFFFFF, 3, CALL_RIR, R, 0, I, 0, R, 0, asm_rv32_opcode_lw }, + { MP_QSTR_mv, MASK_FFFFFFFF, MASK_FFFFFFFF, MASK_NOT_USED, 2, CALL_RR, R, 0, R, 0, N, 0, asm_rv32_opcode_cmv }, + { MP_QSTR_mul, MASK_FFFFFFFF, MASK_FFFFFFFF, MASK_FFFFFFFF, 3, CALL_RRR, R, 0, R, 0, R, 0, asm_rv32_opcode_mul }, + { MP_QSTR_mulh, MASK_FFFFFFFF, MASK_FFFFFFFF, MASK_FFFFFFFF, 3, CALL_RRR, R, 0, R, 0, R, 0, asm_rv32_opcode_mulh }, + { MP_QSTR_mulhsu, MASK_FFFFFFFF, MASK_FFFFFFFF, MASK_FFFFFFFF, 3, CALL_RRR, R, 0, R, 0, R, 0, asm_rv32_opcode_mulhsu }, + { MP_QSTR_mulhu, MASK_FFFFFFFF, MASK_FFFFFFFF, MASK_FFFFFFFF, 3, CALL_RRR, R, 0, R, 0, R, 0, asm_rv32_opcode_mulhu }, + { MP_QSTR_or_, MASK_FFFFFFFF, MASK_FFFFFFFF, MASK_FFFFFFFF, 3, CALL_RRR, R, 0, R, 0, R, 0, asm_rv32_opcode_or }, + { MP_QSTR_ori, MASK_FFFFFFFF, MASK_FFFFFFFF, MASK_00000FFF, 3, CALL_RRI, R, 0, R, 0, I, 0, asm_rv32_opcode_ori }, + { MP_QSTR_rem, MASK_FFFFFFFF, MASK_FFFFFFFF, MASK_FFFFFFFF, 3, CALL_RRR, R, 0, R, 0, R, 0, asm_rv32_opcode_rem }, + { MP_QSTR_remu, MASK_FFFFFFFF, MASK_FFFFFFFF, MASK_FFFFFFFF, 3, CALL_RRR, R, 0, R, 0, R, 0, asm_rv32_opcode_remu }, + { MP_QSTR_sb, MASK_FFFFFFFF, MASK_00000FFF, MASK_FFFFFFFF, 3, CALL_RIR, R, 0, I, 0, R, 0, asm_rv32_opcode_sb }, + { MP_QSTR_sh, MASK_FFFFFFFF, MASK_00000FFF, MASK_FFFFFFFF, 3, CALL_RIR, R, 0, I, 0, R, 0, asm_rv32_opcode_sh }, + { MP_QSTR_sll, MASK_FFFFFFFF, MASK_FFFFFFFF, MASK_FFFFFFFF, 3, CALL_RRR, R, 0, R, 0, R, 0, asm_rv32_opcode_sll }, + { MP_QSTR_slli, MASK_FFFFFFFF, MASK_FFFFFFFF, MASK_0000001F, 3, CALL_RRI, R, 0, R, 0, IU, 0, asm_rv32_opcode_slli }, + { MP_QSTR_slt, MASK_FFFFFFFF, MASK_FFFFFFFF, MASK_FFFFFFFF, 3, CALL_RRR, R, 0, R, 0, R, 0, asm_rv32_opcode_slt }, + { MP_QSTR_slti, MASK_FFFFFFFF, MASK_FFFFFFFF, MASK_00000FFF, 3, CALL_RRI, R, 0, R, 0, I, 0, asm_rv32_opcode_slti }, + { MP_QSTR_sltiu, MASK_FFFFFFFF, MASK_FFFFFFFF, MASK_00000FFF, 3, CALL_RRI, R, 0, R, 0, I, 0, asm_rv32_opcode_sltiu }, + { MP_QSTR_sltu, MASK_FFFFFFFF, MASK_FFFFFFFF, MASK_FFFFFFFF, 3, CALL_RRR, R, 0, R, 0, R, 0, asm_rv32_opcode_sltu }, + { MP_QSTR_sra, MASK_FFFFFFFF, MASK_FFFFFFFF, MASK_FFFFFFFF, 3, CALL_RRR, R, 0, R, 0, R, 0, asm_rv32_opcode_sra }, + { MP_QSTR_srai, MASK_FFFFFFFF, MASK_FFFFFFFF, MASK_0000001F, 3, CALL_RRI, R, 0, R, 0, IU, 0, asm_rv32_opcode_srai }, + { MP_QSTR_srl, MASK_FFFFFFFF, MASK_FFFFFFFF, MASK_FFFFFFFF, 3, CALL_RRR, R, 0, R, 0, R, 0, asm_rv32_opcode_srl }, + { MP_QSTR_srli, MASK_FFFFFFFF, MASK_FFFFFFFF, MASK_0000001F, 3, CALL_RRI, R, 0, R, 0, IU, 0, asm_rv32_opcode_srli }, + { MP_QSTR_sub, MASK_FFFFFFFF, MASK_FFFFFFFF, MASK_FFFFFFFF, 3, CALL_RRR, R, 0, R, 0, R, 0, asm_rv32_opcode_sub }, + { MP_QSTR_sw, MASK_FFFFFFFF, MASK_00000FFF, MASK_FFFFFFFF, 3, CALL_RIR, R, 0, I, 0, R, 0, asm_rv32_opcode_sw }, + { MP_QSTR_xor, MASK_FFFFFFFF, MASK_FFFFFFFF, MASK_FFFFFFFF, 3, CALL_RRR, R, 0, R, 0, R, 0, asm_rv32_opcode_xor }, + { MP_QSTR_xori, MASK_FFFFFFFF, MASK_FFFFFFFF, MASK_00000FFF, 3, CALL_RRI, R, 0, R, 0, I, 0, asm_rv32_opcode_xori }, +}; + +#undef RC +#undef IU +#undef IZ +#undef IUZ + +// These two checks assume the bitmasks are contiguous. + +static bool is_in_signed_mask(mp_uint_t mask, mp_uint_t value) { + mp_uint_t leading_zeroes = mp_clz(mask); + if (leading_zeroes == 0 || leading_zeroes > 32) { + return true; + } + mp_uint_t positive_mask = ~(mask & ~(1U << (31 - leading_zeroes))); + if ((value & positive_mask) == 0) { + return true; + } + mp_uint_t negative_mask = ~(mask >> 1); + mp_uint_t trailing_zeroes = mp_ctz(mask); + if (trailing_zeroes > 0) { + mp_uint_t trailing_mask = (1U << trailing_zeroes) - 1; + if ((value & trailing_mask) != 0) { + return false; + } + negative_mask &= ~trailing_mask; + } + return (value & negative_mask) == negative_mask; +} + +static inline bool is_in_unsigned_mask(mp_uint_t mask, mp_uint_t value) { + return (value & ~mask) == 0; +} + +static bool validate_integer(mp_uint_t value, mp_uint_t mask, mp_uint_t flags) { + if (flags & U) { + if (!is_in_unsigned_mask(mask, value)) { + return false; + } + } else { + if (!is_in_signed_mask(mask, value)) { + return false; + } + } + + if ((flags & Z) && (value == 0)) { + return false; + } + + return true; +} + +#define ET_WRONG_ARGUMENT_KIND MP_ERROR_TEXT("opcode '%q' argument %d: expecting %q") +#define ET_WRONG_ARGUMENTS_COUNT MP_ERROR_TEXT("opcode '%q': expecting %d arguments") +#define ET_OUT_OF_RANGE MP_ERROR_TEXT("opcode '%q' argument %d: out of range") + +static bool validate_argument(emit_inline_asm_t *emit, qstr opcode_qstr, + const opcode_t *opcode, mp_parse_node_t node, mp_uint_t node_index) { + assert((node_index < 3) && "Invalid argument node number."); + + uint32_t kind = 0; + uint32_t shift = 0; + uint32_t mask = 0; + + switch (node_index) { + case 0: + kind = opcode->argument1_kind; + shift = opcode->argument1_shift; + mask = OPCODE_MASKS[opcode->argument1_mask]; + break; + + case 1: + kind = opcode->argument2_kind; + shift = opcode->argument2_shift; + mask = OPCODE_MASKS[opcode->argument2_mask]; + break; + + case 2: + kind = opcode->argument3_kind; + shift = opcode->argument3_shift; + mask = OPCODE_MASKS[opcode->argument3_mask]; + break; + + default: + break; + } + + switch (kind & 0x03) { + case N: + assert(mask == OPCODE_MASKS[MASK_NOT_USED] && "Invalid mask index for missing operand."); + return true; + + case R: { + mp_uint_t register_index; + if (!parse_register_node(node, ®ister_index, false)) { + emit_inline_rv32_error_exc(emit, + mp_obj_new_exception_msg_varg(&mp_type_SyntaxError, + ET_WRONG_ARGUMENT_KIND, opcode_qstr, node_index + 1, MP_QSTR_register)); + return false; + } + + if ((mask & (1U << register_index)) == 0) { + emit_inline_rv32_error_exc(emit, + mp_obj_new_exception_msg_varg(&mp_type_SyntaxError, + MP_ERROR_TEXT("opcode '%q' argument %d: unknown register"), + opcode_qstr, node_index + 1)); + return false; + } + + return true; + } + break; + + case I: { + mp_obj_t object; + if (!mp_parse_node_get_int_maybe(node, &object)) { + emit_inline_rv32_error_exc(emit, + mp_obj_new_exception_msg_varg(&mp_type_SyntaxError, + ET_WRONG_ARGUMENT_KIND, opcode_qstr, node_index + 1, MP_QSTR_integer)); + return false; + } + + mp_uint_t immediate = mp_obj_get_int_truncated(object) << shift; + if (kind & U) { + if (!is_in_unsigned_mask(mask, immediate)) { + goto out_of_range; + } + } else { + if (!is_in_signed_mask(mask, immediate)) { + goto out_of_range; + } + } + + if ((kind & Z) && (immediate == 0)) { + goto zero_immediate; + } + + return true; + } + break; + + case L: { + if (!MP_PARSE_NODE_IS_ID(node)) { + emit_inline_rv32_error_exc(emit, + mp_obj_new_exception_msg_varg(&mp_type_SyntaxError, + ET_WRONG_ARGUMENT_KIND, opcode_qstr, node_index + 1, MP_QSTR_label)); + return false; + } + + qstr qstring; + mp_uint_t label_index = lookup_label(emit, node, &qstring); + if (label_index >= emit->max_num_labels && emit->pass == MP_PASS_EMIT) { + emit_inline_rv32_error_exc(emit, + mp_obj_new_exception_msg_varg(&mp_type_SyntaxError, + MP_ERROR_TEXT("opcode '%q' argument %d: undefined label '%q'"), + opcode_qstr, node_index + 1, qstring)); + return false; + } + + mp_uint_t displacement = (mp_uint_t)(label_code_offset(emit, label_index)); + if (kind & U) { + if (!is_in_unsigned_mask(mask, displacement)) { + goto out_of_range; + } + } else { + if (!is_in_signed_mask(mask, displacement)) { + goto out_of_range; + } + } + return true; + } + break; + + default: + assert(!"Unknown argument kind"); + break; + } + + return false; + +out_of_range: + emit_inline_rv32_error_exc(emit, + mp_obj_new_exception_msg_varg(&mp_type_SyntaxError, ET_OUT_OF_RANGE, opcode_qstr, node_index + 1)); + return false; + +zero_immediate: + emit_inline_rv32_error_exc(emit, + mp_obj_new_exception_msg_varg(&mp_type_SyntaxError, + MP_ERROR_TEXT("opcode '%q' argument %d: must not be zero"), + opcode_qstr, node_index + 1)); + return false; +} + +static bool parse_register_offset_node(emit_inline_asm_t *emit, qstr opcode_qstr, const opcode_t *opcode_data, mp_parse_node_t node, mp_uint_t node_index, mp_parse_node_t *register_node, mp_parse_node_t *offset_node, bool *negative) { + assert(register_node != NULL && "Register node pointer is NULL."); + assert(offset_node != NULL && "Offset node pointer is NULL."); + assert(negative != NULL && "Negative pointer is NULL."); + + if (!MP_PARSE_NODE_IS_STRUCT_KIND(node, PN_atom_expr_normal) && !MP_PARSE_NODE_IS_STRUCT_KIND(node, PN_factor_2)) { + goto invalid_structure; + } + mp_parse_node_struct_t *node_struct = (mp_parse_node_struct_t *)node; + *negative = false; + if (MP_PARSE_NODE_IS_STRUCT_KIND(node, PN_factor_2)) { + if (MP_PARSE_NODE_IS_TOKEN_KIND(node_struct->nodes[0], MP_TOKEN_OP_MINUS)) { + *negative = true; + } else { + if (!MP_PARSE_NODE_IS_TOKEN_KIND(node_struct->nodes[0], MP_TOKEN_OP_PLUS)) { + goto invalid_structure; + } + } + if (!MP_PARSE_NODE_IS_STRUCT_KIND(node_struct->nodes[1], PN_atom_expr_normal)) { + goto invalid_structure; + } + node_struct = (mp_parse_node_struct_t *)node_struct->nodes[1]; + } + + if (*negative) { + // If the value is negative, RULE_atom_expr_normal's first token will be the + // offset stripped of its negative marker; range check will then fail if the + // default method is used, so a custom check is used instead. + mp_obj_t object; + if (!mp_parse_node_get_int_maybe(node_struct->nodes[0], &object)) { + emit_inline_rv32_error_exc(emit, + mp_obj_new_exception_msg_varg(&mp_type_SyntaxError, ET_WRONG_ARGUMENT_KIND, opcode_qstr, 2, MP_QSTR_integer)); + return false; + } + mp_uint_t value = mp_obj_get_int_truncated(object); + value = (~value + 1) & (mp_uint_t)-1; + if (!validate_integer(value << opcode_data->argument2_shift, OPCODE_MASKS[opcode_data->argument2_mask], opcode_data->argument2_kind)) { + emit_inline_rv32_error_exc(emit, + mp_obj_new_exception_msg_varg(&mp_type_SyntaxError, ET_OUT_OF_RANGE, opcode_qstr, 2)); + return false; + } + } else { + if (!validate_argument(emit, opcode_qstr, opcode_data, node_struct->nodes[0], 1)) { + return false; + } + } + + *offset_node = node_struct->nodes[0]; + node_struct = (mp_parse_node_struct_t *)node_struct->nodes[1]; + if (!validate_argument(emit, opcode_qstr, opcode_data, node_struct->nodes[0], 2)) { + return false; + } + *register_node = node_struct->nodes[0]; + return true; + +invalid_structure: + emit_inline_rv32_error_exc(emit, + mp_obj_new_exception_msg_varg(&mp_type_SyntaxError, + ET_WRONG_ARGUMENT_KIND, opcode_qstr, node_index + 1, MP_QSTR_offset)); + return false; +} + +static void handle_opcode(emit_inline_asm_t *emit, qstr opcode, const opcode_t *opcode_data, mp_parse_node_t *arguments) { + mp_uint_t rd = 0; + mp_uint_t rs1 = 0; + mp_uint_t rs2 = 0; + + switch (opcode_data->calling_convention) { + case CALL_RRR: { + parse_register_node(arguments[0], &rd, opcode_data->argument1_kind & C); + parse_register_node(arguments[1], &rs1, opcode_data->argument2_kind & C); + parse_register_node(arguments[2], &rs2, opcode_data->argument3_kind & C); + ((call_rrr_t)opcode_data->emitter)(&emit->as, rd, rs1, rs2); + break; + } + + case CALL_RR: { + parse_register_node(arguments[0], &rd, opcode_data->argument1_kind & C); + parse_register_node(arguments[1], &rs1, opcode_data->argument2_kind & C); + ((call_rr_t)opcode_data->emitter)(&emit->as, rd, rs1); + break; + } + + case CALL_RRI: { + parse_register_node(arguments[0], &rd, opcode_data->argument1_kind & C); + parse_register_node(arguments[1], &rs1, opcode_data->argument2_kind & C); + mp_obj_t object; + mp_parse_node_get_int_maybe(arguments[2], &object); + mp_uint_t immediate = mp_obj_get_int_truncated(object) << opcode_data->argument3_shift; + ((call_rri_t)opcode_data->emitter)(&emit->as, rd, rs1, immediate); + break; + } + + case CALL_RI: { + parse_register_node(arguments[0], &rd, opcode_data->argument1_kind & C); + mp_obj_t object; + mp_parse_node_get_int_maybe(arguments[1], &object); + mp_uint_t immediate = mp_obj_get_int_truncated(object) << opcode_data->argument2_shift; + ((call_ri_t)opcode_data->emitter)(&emit->as, rd, immediate); + break; + } + + case CALL_R: { + parse_register_node(arguments[0], &rd, opcode_data->argument1_kind & C); + ((call_r_t)opcode_data->emitter)(&emit->as, rd); + break; + } + + case CALL_RRL: { + parse_register_node(arguments[0], &rd, opcode_data->argument1_kind & C); + parse_register_node(arguments[1], &rs1, opcode_data->argument2_kind & C); + qstr qstring; + mp_uint_t label_index = lookup_label(emit, arguments[2], &qstring); + ptrdiff_t displacement = label_code_offset(emit, label_index); + ((call_rri_t)opcode_data->emitter)(&emit->as, rd, rs1, displacement); + break; + } + + case CALL_RL: { + parse_register_node(arguments[0], &rd, opcode_data->argument1_kind & C); + qstr qstring; + mp_uint_t label_index = lookup_label(emit, arguments[1], &qstring); + ptrdiff_t displacement = label_code_offset(emit, label_index); + ((call_ri_t)opcode_data->emitter)(&emit->as, rd, displacement); + break; + } + + case CALL_L: { + qstr qstring; + mp_uint_t label_index = lookup_label(emit, arguments[0], &qstring); + ptrdiff_t displacement = label_code_offset(emit, label_index); + ((call_i_t)opcode_data->emitter)(&emit->as, displacement); + break; + } + + case CALL_N: + ((call_n_t)opcode_data->emitter)(&emit->as); + break; + + case CALL_I: { + mp_obj_t object; + mp_parse_node_get_int_maybe(arguments[0], &object); + mp_uint_t immediate = mp_obj_get_int_truncated(object) << opcode_data->argument1_shift; + ((call_i_t)opcode_data->emitter)(&emit->as, immediate); + break; + } + + case CALL_RII: { + parse_register_node(arguments[0], &rd, opcode_data->argument1_kind & C); + mp_obj_t object; + mp_parse_node_get_int_maybe(arguments[1], &object); + mp_uint_t immediate1 = mp_obj_get_int_truncated(object) << opcode_data->argument2_shift; + mp_parse_node_get_int_maybe(arguments[2], &object); + mp_uint_t immediate2 = mp_obj_get_int_truncated(object) << opcode_data->argument3_shift; + ((call_rii_t)opcode_data->emitter)(&emit->as, rd, immediate1, immediate2); + break; + } + + case CALL_RIR: + assert(!"Should not get here."); + break; + + default: + assert(!"Unhandled call convention."); + break; + } +} + +static bool handle_load_store_opcode_with_offset(emit_inline_asm_t *emit, qstr opcode, const opcode_t *opcode_data, mp_parse_node_t *argument_nodes) { + mp_parse_node_t nodes[3] = {0}; + if (!validate_argument(emit, opcode, opcode_data, argument_nodes[0], 0)) { + return false; + } + nodes[0] = argument_nodes[0]; + bool negative = false; + if (!parse_register_offset_node(emit, opcode, opcode_data, argument_nodes[1], 1, &nodes[1], &nodes[2], &negative)) { + return false; + } + + mp_uint_t rd = 0; + mp_uint_t rs1 = 0; + if (!parse_register_node(nodes[0], &rd, opcode_data->argument1_kind & C)) { + return false; + } + if (!parse_register_node(nodes[1], &rs1, opcode_data->argument3_kind & C)) { + return false; + } + + mp_obj_t object; + mp_parse_node_get_int_maybe(nodes[2], &object); + mp_uint_t immediate = mp_obj_get_int_truncated(object) << opcode_data->argument2_shift; + if (negative) { + immediate = (~immediate + 1) & (mp_uint_t)-1; + } + if (!is_in_signed_mask(OPCODE_MASKS[opcode_data->argument2_mask], immediate)) { + emit_inline_rv32_error_exc(emit, + mp_obj_new_exception_msg_varg(&mp_type_SyntaxError, ET_OUT_OF_RANGE, opcode, 2)); + return false; + } + + ((call_rri_t)opcode_data->emitter)(&emit->as, rd, rs1, immediate); + return true; +} + +static void emit_inline_rv32_opcode(emit_inline_asm_t *emit, qstr opcode, mp_uint_t arguments_count, mp_parse_node_t *argument_nodes) { + const opcode_t *opcode_data = NULL; + for (mp_uint_t index = 0; index < MP_ARRAY_SIZE(OPCODES); index++) { + if (OPCODES[index].qstring == opcode) { + opcode_data = &OPCODES[index]; + break; + } + } + + if (!opcode_data) { + emit_inline_rv32_error_exc(emit, + mp_obj_new_exception_msg_varg(&mp_type_SyntaxError, + MP_ERROR_TEXT("unknown RV32 instruction '%q'"), opcode)); + return; + } + + assert((opcode_data->argument1_mask < MP_ARRAY_SIZE(OPCODE_MASKS)) && "Argument #1 opcode mask index out of bounds."); + assert((opcode_data->argument2_mask < MP_ARRAY_SIZE(OPCODE_MASKS)) && "Argument #2 opcode mask index out of bounds."); + assert((opcode_data->argument3_mask < MP_ARRAY_SIZE(OPCODE_MASKS)) && "Argument #3 opcode mask index out of bounds."); + assert((opcode_data->calling_convention < CALL_COUNT) && "Calling convention index out of bounds."); + if (opcode_data->calling_convention != CALL_RIR) { + if (opcode_data->arguments_count != arguments_count) { + emit_inline_rv32_error_exc(emit, + mp_obj_new_exception_msg_varg(&mp_type_SyntaxError, + ET_WRONG_ARGUMENTS_COUNT, opcode, opcode_data->arguments_count)); + return; + } + if (opcode_data->arguments_count >= 1 && !validate_argument(emit, opcode, opcode_data, argument_nodes[0], 0)) { + return; + } + if (opcode_data->arguments_count >= 2 && !validate_argument(emit, opcode, opcode_data, argument_nodes[1], 1)) { + return; + } + if (opcode_data->arguments_count >= 3 && !validate_argument(emit, opcode, opcode_data, argument_nodes[2], 2)) { + return; + } + handle_opcode(emit, opcode, opcode_data, argument_nodes); + return; + } + + assert((opcode_data->argument2_kind & U) == 0 && "Offset must not be unsigned."); + assert((opcode_data->argument2_kind & Z) == 0 && "Offset can be zero."); + + if (arguments_count != 2) { + emit_inline_rv32_error_exc(emit, + mp_obj_new_exception_msg_varg(&mp_type_SyntaxError, ET_WRONG_ARGUMENTS_COUNT, opcode, 2)); + return; + } + + handle_load_store_opcode_with_offset(emit, opcode, opcode_data, argument_nodes); +} + +#undef N +#undef R +#undef I +#undef L +#undef C +#undef U + +const emit_inline_asm_method_table_t emit_inline_rv32_method_table = { + #if MICROPY_DYNAMIC_COMPILER + emit_inline_rv32_new, + emit_inline_rv32_free, + #endif + + emit_inline_rv32_start_pass, + emit_inline_rv32_end_pass, + emit_inline_rv32_count_params, + emit_inline_rv32_label, + emit_inline_rv32_opcode, +}; + +#endif // MICROPY_EMIT_INLINE_RV32 diff --git a/py/emitinlinextensa.c b/py/emitinlinextensa.c index 57056d597aab7..fed259cfc6b20 100644 --- a/py/emitinlinextensa.c +++ b/py/emitinlinextensa.c @@ -115,50 +115,21 @@ static bool emit_inline_xtensa_label(emit_inline_asm_t *emit, mp_uint_t label_nu return true; } -typedef struct _reg_name_t { byte reg; - byte name[3]; -} reg_name_t; -static const reg_name_t reg_name_table[] = { - {0, "a0\0"}, - {1, "a1\0"}, - {2, "a2\0"}, - {3, "a3\0"}, - {4, "a4\0"}, - {5, "a5\0"}, - {6, "a6\0"}, - {7, "a7\0"}, - {8, "a8\0"}, - {9, "a9\0"}, - {10, "a10"}, - {11, "a11"}, - {12, "a12"}, - {13, "a13"}, - {14, "a14"}, - {15, "a15"}, +static const qstr_short_t REGISTERS[16] = { + MP_QSTR_a0, MP_QSTR_a1, MP_QSTR_a2, MP_QSTR_a3, MP_QSTR_a4, MP_QSTR_a5, MP_QSTR_a6, MP_QSTR_a7, + MP_QSTR_a8, MP_QSTR_a9, MP_QSTR_a10, MP_QSTR_a11, MP_QSTR_a12, MP_QSTR_a13, MP_QSTR_a14, MP_QSTR_a15 }; -// return empty string in case of error, so we can attempt to parse the string -// without a special check if it was in fact a string -static const char *get_arg_str(mp_parse_node_t pn) { - if (MP_PARSE_NODE_IS_ID(pn)) { - qstr qst = MP_PARSE_NODE_LEAF_ARG(pn); - return qstr_str(qst); - } else { - return ""; - } -} - static mp_uint_t get_arg_reg(emit_inline_asm_t *emit, const char *op, mp_parse_node_t pn) { - const char *reg_str = get_arg_str(pn); - for (mp_uint_t i = 0; i < MP_ARRAY_SIZE(reg_name_table); i++) { - const reg_name_t *r = ®_name_table[i]; - if (reg_str[0] == r->name[0] - && reg_str[1] == r->name[1] - && reg_str[2] == r->name[2] - && (reg_str[2] == '\0' || reg_str[3] == '\0')) { - return r->reg; + if (MP_PARSE_NODE_IS_ID(pn)) { + qstr node_qstr = MP_PARSE_NODE_LEAF_ARG(pn); + for (size_t i = 0; i < MP_ARRAY_SIZE(REGISTERS); i++) { + if (node_qstr == REGISTERS[i]) { + return i; + } } } + emit_inline_xtensa_error_exc(emit, mp_obj_new_exception_msg_varg(&mp_type_SyntaxError, MP_ERROR_TEXT("'%s' expects a register"), op)); diff --git a/py/emitnative.c b/py/emitnative.c index 6b589b54731aa..a888418e5dfed 100644 --- a/py/emitnative.c +++ b/py/emitnative.c @@ -93,7 +93,7 @@ #endif #ifndef N_XTENSAWIN -#define N_XTENSWIN (0) +#define N_XTENSAWIN (0) #endif #ifndef N_PRELUDE_AS_BYTES_OBJ @@ -330,6 +330,11 @@ struct _emit_t { ASM_T *as; }; +#ifndef REG_ZERO +#define REG_ZERO REG_TEMP0 +#define ASM_CLR_REG(state, rd) ASM_XOR_REG_REG(state, rd, rd) +#endif + static void emit_load_reg_with_object(emit_t *emit, int reg, mp_obj_t obj); static void emit_native_global_exc_entry(emit_t *emit); static void emit_native_global_exc_exit(emit_t *emit); @@ -1242,12 +1247,12 @@ static void emit_native_global_exc_entry(emit_t *emit) { ASM_JUMP_IF_REG_ZERO(emit->as, REG_RET, start_label, true); } else { // Clear the unwind state - ASM_XOR_REG_REG(emit->as, REG_TEMP0, REG_TEMP0); - ASM_MOV_LOCAL_REG(emit->as, LOCAL_IDX_EXC_HANDLER_UNWIND(emit), REG_TEMP0); + ASM_CLR_REG(emit->as, REG_ZERO); + ASM_MOV_LOCAL_REG(emit->as, LOCAL_IDX_EXC_HANDLER_UNWIND(emit), REG_ZERO); // clear nlr.ret_val, because it's passed to mp_native_raise regardless // of whether there was an exception or not - ASM_MOV_LOCAL_REG(emit->as, LOCAL_IDX_EXC_VAL(emit), REG_TEMP0); + ASM_MOV_LOCAL_REG(emit->as, LOCAL_IDX_EXC_VAL(emit), REG_ZERO); // Put PC of start code block into REG_LOCAL_1 ASM_MOV_REG_PCREL(emit->as, REG_LOCAL_1, start_label); @@ -1263,8 +1268,8 @@ static void emit_native_global_exc_entry(emit_t *emit) { ASM_JUMP_IF_REG_NONZERO(emit->as, REG_RET, global_except_label, true); // Clear PC of current code block, and jump there to resume execution - ASM_XOR_REG_REG(emit->as, REG_TEMP0, REG_TEMP0); - ASM_MOV_LOCAL_REG(emit->as, LOCAL_IDX_EXC_HANDLER_PC(emit), REG_TEMP0); + ASM_CLR_REG(emit->as, REG_ZERO); + ASM_MOV_LOCAL_REG(emit->as, LOCAL_IDX_EXC_HANDLER_PC(emit), REG_ZERO); ASM_JUMP_REG(emit->as, REG_LOCAL_1); // Global exception handler: check for valid exception handler @@ -1587,6 +1592,11 @@ static void emit_native_load_subscr(emit_t *emit) { asm_rv32_opcode_lbu(emit->as, REG_RET, reg_base, index_value); break; } + #elif N_XTENSA || N_XTENSAWIN + if (index_value > 0 && index_value < 256) { + asm_xtensa_op_l8ui(emit->as, REG_RET, reg_base, index_value); + break; + } #endif need_reg_single(emit, reg_index, 0); ASM_MOV_REG_IMM(emit->as, reg_index, index_value); @@ -1610,6 +1620,11 @@ static void emit_native_load_subscr(emit_t *emit) { asm_rv32_opcode_lhu(emit->as, REG_RET, reg_base, index_value << 1); break; } + #elif N_XTENSA || N_XTENSAWIN + if (index_value > 0 && index_value < 256) { + asm_xtensa_op_l16ui(emit->as, REG_RET, reg_base, index_value); + break; + } #endif need_reg_single(emit, reg_index, 0); ASM_MOV_REG_IMM(emit->as, reg_index, index_value << 1); @@ -1633,6 +1648,11 @@ static void emit_native_load_subscr(emit_t *emit) { asm_rv32_opcode_lw(emit->as, REG_RET, reg_base, index_value << 2); break; } + #elif N_XTENSA || N_XTENSAWIN + if (index_value > 0 && index_value < 256) { + asm_xtensa_l32i_optimised(emit->as, REG_RET, reg_base, index_value); + break; + } #endif need_reg_single(emit, reg_index, 0); ASM_MOV_REG_IMM(emit->as, reg_index, index_value << 2); @@ -1667,6 +1687,11 @@ static void emit_native_load_subscr(emit_t *emit) { } case VTYPE_PTR16: { // pointer to 16-bit memory + #if N_XTENSA || N_XTENSAWIN + asm_xtensa_op_addx2(emit->as, REG_ARG_1, reg_index, REG_ARG_1); + asm_xtensa_op_l16ui(emit->as, REG_RET, REG_ARG_1, 0); + break; + #endif ASM_ADD_REG_REG(emit->as, REG_ARG_1, reg_index); // add index to base ASM_ADD_REG_REG(emit->as, REG_ARG_1, reg_index); // add index to base ASM_LOAD16_REG_REG(emit->as, REG_RET, REG_ARG_1); // load from (base+2*index) @@ -1679,6 +1704,10 @@ static void emit_native_load_subscr(emit_t *emit) { asm_rv32_opcode_cadd(emit->as, REG_ARG_1, REG_TEMP2); asm_rv32_opcode_lw(emit->as, REG_RET, REG_ARG_1, 0); break; + #elif N_XTENSA || N_XTENSAWIN + asm_xtensa_op_addx4(emit->as, REG_ARG_1, reg_index, REG_ARG_1); + asm_xtensa_op_l32i_n(emit->as, REG_RET, REG_ARG_1, 0); + break; #endif ASM_ADD_REG_REG(emit->as, REG_ARG_1, reg_index); // add index to base ASM_ADD_REG_REG(emit->as, REG_ARG_1, reg_index); // add index to base @@ -1840,6 +1869,11 @@ static void emit_native_store_subscr(emit_t *emit) { asm_rv32_opcode_sb(emit->as, reg_value, reg_base, index_value); break; } + #elif N_XTENSA || N_XTENSAWIN + if (index_value > 0 && index_value < 256) { + asm_xtensa_op_s8i(emit->as, REG_RET, reg_base, index_value); + break; + } #endif ASM_MOV_REG_IMM(emit->as, reg_index, index_value); #if N_ARM @@ -1866,6 +1900,11 @@ static void emit_native_store_subscr(emit_t *emit) { asm_rv32_opcode_sh(emit->as, reg_value, reg_base, index_value << 1); break; } + #elif N_XTENSA || N_XTENSAWIN + if (index_value > 0 && index_value < 256) { + asm_xtensa_op_s16i(emit->as, REG_RET, reg_base, index_value); + break; + } #endif ASM_MOV_REG_IMM(emit->as, reg_index, index_value << 1); ASM_ADD_REG_REG(emit->as, reg_index, reg_base); // add 2*index to base @@ -1888,6 +1927,11 @@ static void emit_native_store_subscr(emit_t *emit) { asm_rv32_opcode_sw(emit->as, reg_value, reg_base, index_value << 2); break; } + #elif N_XTENSA || N_XTENSAWIN + if (index_value > 0 && index_value < 256) { + asm_xtensa_s32i_optimised(emit->as, REG_RET, reg_base, index_value); + break; + } #elif N_ARM ASM_MOV_REG_IMM(emit->as, reg_index, index_value); asm_arm_str_reg_reg_reg(emit->as, reg_value, reg_base, reg_index); @@ -1942,6 +1986,10 @@ static void emit_native_store_subscr(emit_t *emit) { #if N_ARM asm_arm_strh_reg_reg_reg(emit->as, reg_value, REG_ARG_1, reg_index); break; + #elif N_XTENSA || N_XTENSAWIN + asm_xtensa_op_addx2(emit->as, REG_ARG_1, reg_index, REG_ARG_1); + asm_xtensa_op_s16i(emit->as, reg_value, REG_ARG_1, 0); + break; #endif ASM_ADD_REG_REG(emit->as, REG_ARG_1, reg_index); // add index to base ASM_ADD_REG_REG(emit->as, REG_ARG_1, reg_index); // add index to base @@ -1958,6 +2006,10 @@ static void emit_native_store_subscr(emit_t *emit) { asm_rv32_opcode_cadd(emit->as, REG_ARG_1, REG_TEMP2); asm_rv32_opcode_sw(emit->as, reg_value, REG_ARG_1, 0); break; + #elif N_XTENSA || N_XTENSAWIN + asm_xtensa_op_addx4(emit->as, REG_ARG_1, reg_index, REG_ARG_1); + asm_xtensa_op_s32i_n(emit->as, reg_value, REG_ARG_1, 0); + break; #endif ASM_ADD_REG_REG(emit->as, REG_ARG_1, reg_index); // add index to base ASM_ADD_REG_REG(emit->as, REG_ARG_1, reg_index); // add index to base @@ -2533,7 +2585,7 @@ static void emit_native_binary_op(emit_t *emit, mp_binary_op_t op) { #if N_X64 asm_x64_xor_r64_r64(emit->as, REG_RET, REG_RET); asm_x64_cmp_r64_with_r64(emit->as, reg_rhs, REG_ARG_2); - static byte ops[6 + 6] = { + static const byte ops[6 + 6] = { // unsigned ASM_X64_CC_JB, ASM_X64_CC_JA, @@ -2553,7 +2605,7 @@ static void emit_native_binary_op(emit_t *emit, mp_binary_op_t op) { #elif N_X86 asm_x86_xor_r32_r32(emit->as, REG_RET, REG_RET); asm_x86_cmp_r32_with_r32(emit->as, reg_rhs, REG_ARG_2); - static byte ops[6 + 6] = { + static const byte ops[6 + 6] = { // unsigned ASM_X86_CC_JB, ASM_X86_CC_JA, @@ -2573,7 +2625,7 @@ static void emit_native_binary_op(emit_t *emit, mp_binary_op_t op) { #elif N_THUMB asm_thumb_cmp_rlo_rlo(emit->as, REG_ARG_2, reg_rhs); if (asm_thumb_allow_armv7m(emit->as)) { - static uint16_t ops[6 + 6] = { + static const uint16_t ops[6 + 6] = { // unsigned ASM_THUMB_OP_ITE_CC, ASM_THUMB_OP_ITE_HI, @@ -2593,7 +2645,7 @@ static void emit_native_binary_op(emit_t *emit, mp_binary_op_t op) { asm_thumb_mov_rlo_i8(emit->as, REG_RET, 1); asm_thumb_mov_rlo_i8(emit->as, REG_RET, 0); } else { - static uint16_t ops[6 + 6] = { + static const uint16_t ops[6 + 6] = { // unsigned ASM_THUMB_CC_CC, ASM_THUMB_CC_HI, @@ -2616,7 +2668,7 @@ static void emit_native_binary_op(emit_t *emit, mp_binary_op_t op) { } #elif N_ARM asm_arm_cmp_reg_reg(emit->as, REG_ARG_2, reg_rhs); - static uint ccs[6 + 6] = { + static const uint ccs[6 + 6] = { // unsigned ASM_ARM_CC_CC, ASM_ARM_CC_HI, @@ -2634,7 +2686,7 @@ static void emit_native_binary_op(emit_t *emit, mp_binary_op_t op) { }; asm_arm_setcc_reg(emit->as, REG_RET, ccs[op_idx]); #elif N_XTENSA || N_XTENSAWIN - static uint8_t ccs[6 + 6] = { + static const uint8_t ccs[6 + 6] = { // unsigned ASM_XTENSA_CC_LTU, 0x80 | ASM_XTENSA_CC_LTU, // for GTU we'll swap args diff --git a/py/gc.c b/py/gc.c index c74aba9e8b5c6..338803dc245cc 100644 --- a/py/gc.c +++ b/py/gc.c @@ -136,9 +136,12 @@ #define CTB_CLEAR(area, block) do { area->gc_collect_table_start[(block) / BLOCKS_PER_CTB] &= (~(1 << ((block) & 7))); } while (0) #if MICROPY_PY_THREAD && !MICROPY_PY_THREAD_GIL -#define GC_ENTER() mp_thread_mutex_lock(&MP_STATE_MEM(gc_mutex), 1) -#define GC_EXIT() mp_thread_mutex_unlock(&MP_STATE_MEM(gc_mutex)) +#define GC_MUTEX_INIT() mp_thread_recursive_mutex_init(&MP_STATE_MEM(gc_mutex)) +#define GC_ENTER() mp_thread_recursive_mutex_lock(&MP_STATE_MEM(gc_mutex), 1) +#define GC_EXIT() mp_thread_recursive_mutex_unlock(&MP_STATE_MEM(gc_mutex)) #else +// Either no threading, or assume callers to gc_collect() hold the GIL +#define GC_MUTEX_INIT() #define GC_ENTER() #define GC_EXIT() #endif @@ -155,6 +158,17 @@ void __attribute__ ((noinline)) gc_log_change(uint32_t start_block, uint32_t len #pragma GCC pop_options #endif +// Static functions for individual steps of the GC mark/sweep sequence +static void gc_collect_start_common(void); +static void *gc_get_ptr(void **ptrs, int i); +#if MICROPY_GC_SPLIT_HEAP +static void gc_mark_subtree(mp_state_mem_area_t *area, size_t block); +#else +static void gc_mark_subtree(size_t block); +#endif +static void gc_deal_with_stack_overflow(void); +static void gc_sweep_run_finalisers(void); +static void gc_sweep_free_blocks(void); // TODO waste less memory; currently requires that all entries in alloc_table have a corresponding block in pool static void gc_setup_area(mp_state_mem_area_t *area, void *start, void *end) { @@ -269,9 +283,7 @@ void gc_init(void *start, void *end) { MP_STATE_MEM(gc_alloc_amount) = 0; #endif - #if MICROPY_PY_THREAD && !MICROPY_PY_THREAD_GIL - mp_thread_mutex_init(&MP_STATE_MEM(gc_mutex)); - #endif + GC_MUTEX_INIT(); } #if MICROPY_GC_SPLIT_HEAP @@ -406,7 +418,7 @@ static bool gc_try_add_heap(size_t failed_alloc) { #endif -// CIRCUITPY-CHANGE +// CIRCUITPY-CHANGE: additional function void gc_deinit(void) { // Run any finalisers before we stop using the heap. This will also free // any additional heap areas (but not the first.) @@ -419,19 +431,19 @@ void gc_lock(void) { // - each thread has its own gc_lock_depth so there are no races between threads; // - a hard interrupt will only change gc_lock_depth during its execution, and // upon return will restore the value of gc_lock_depth. - MP_STATE_THREAD(gc_lock_depth)++; + MP_STATE_THREAD(gc_lock_depth) += (1 << GC_LOCK_DEPTH_SHIFT); } void gc_unlock(void) { // This does not need to be atomic, See comment above in gc_lock. - MP_STATE_THREAD(gc_lock_depth)--; + MP_STATE_THREAD(gc_lock_depth) -= (1 << GC_LOCK_DEPTH_SHIFT); } bool gc_is_locked(void) { return MP_STATE_THREAD(gc_lock_depth) != 0; } -// CIRCUITPY-CHANGE +// CIRCUITPY-CHANGE: additional function bool gc_ptr_on_heap(void *ptr) { for (mp_state_mem_area_t *area = &MP_STATE_MEM(area); area != NULL; area = NEXT_AREA(area)) { if (ptr >= (void *)area->gc_pool_start // must be above start of pool @@ -474,11 +486,70 @@ static inline mp_state_mem_area_t *gc_get_ptr_area(const void *ptr) { #endif #endif +void gc_collect_start(void) { + gc_collect_start_common(); + #if MICROPY_GC_ALLOC_THRESHOLD + MP_STATE_MEM(gc_alloc_amount) = 0; + #endif + + // Trace root pointers. This relies on the root pointers being organised + // correctly in the mp_state_ctx structure. We scan nlr_top, dict_locals, + // dict_globals, then the root pointer section of mp_state_vm. + void **ptrs = (void **)(void *)&mp_state_ctx; + size_t root_start = offsetof(mp_state_ctx_t, thread.dict_locals); + size_t root_end = offsetof(mp_state_ctx_t, vm.qstr_last_chunk); + gc_collect_root(ptrs + root_start / sizeof(void *), (root_end - root_start) / sizeof(void *)); + + #if MICROPY_ENABLE_PYSTACK + // Trace root pointers from the Python stack. + ptrs = (void **)(void *)MP_STATE_THREAD(pystack_start); + gc_collect_root(ptrs, (MP_STATE_THREAD(pystack_cur) - MP_STATE_THREAD(pystack_start)) / sizeof(void *)); + #endif +} + +static void gc_collect_start_common(void) { + GC_ENTER(); + assert((MP_STATE_THREAD(gc_lock_depth) & GC_COLLECT_FLAG) == 0); + MP_STATE_THREAD(gc_lock_depth) |= GC_COLLECT_FLAG; + MP_STATE_MEM(gc_stack_overflow) = 0; +} + +void gc_collect_root(void **ptrs, size_t len) { + #if !MICROPY_GC_SPLIT_HEAP + mp_state_mem_area_t *area = &MP_STATE_MEM(area); + #endif + for (size_t i = 0; i < len; i++) { + MICROPY_GC_HOOK_LOOP(i); + void *ptr = gc_get_ptr(ptrs, i); + #if MICROPY_GC_SPLIT_HEAP + mp_state_mem_area_t *area = gc_get_ptr_area(ptr); + if (!area) { + continue; + } + #else + if (!VERIFY_PTR(ptr)) { + continue; + } + #endif + size_t block = BLOCK_FROM_PTR(area, ptr); + if (ATB_GET_KIND(area, block) == AT_HEAD) { + // An unmarked head: mark it, and mark all its children + ATB_HEAD_TO_MARK(area, block); + #if MICROPY_GC_SPLIT_HEAP + gc_mark_subtree(area, block); + #else + gc_mark_subtree(block); + #endif + } + } +} + // Take the given block as the topmost block on the stack. Check all it's // children: mark the unmarked child blocks and put those newly marked // blocks on the stack. When all children have been checked, pop off the // topmost block on the stack and repeat with that one. // CIRCUITPY-CHANGE: We don't instrument these functions because they occur a lot during GC and +// fill up the output buffer quickly. #if MICROPY_GC_SPLIT_HEAP static void MP_NO_INSTRUMENT PLACE_IN_ITCM(gc_mark_subtree)(mp_state_mem_area_t * area, size_t block) #else @@ -564,6 +635,25 @@ static void MP_NO_INSTRUMENT PLACE_IN_ITCM(gc_mark_subtree)(size_t block) } } +void gc_sweep_all(void) { + gc_collect_start_common(); + gc_collect_end(); +} + +void gc_collect_end(void) { + gc_deal_with_stack_overflow(); + gc_sweep_run_finalisers(); + gc_sweep_free_blocks(); + #if MICROPY_GC_SPLIT_HEAP + MP_STATE_MEM(gc_last_free_area) = &MP_STATE_MEM(area); + #endif + for (mp_state_mem_area_t *area = &MP_STATE_MEM(area); area != NULL; area = NEXT_AREA(area)) { + area->gc_last_free_atb_index = 0; + } + MP_STATE_THREAD(gc_lock_depth) &= ~GC_COLLECT_FLAG; + GC_EXIT(); +} + static void gc_deal_with_stack_overflow(void) { while (MP_STATE_MEM(gc_stack_overflow)) { MP_STATE_MEM(gc_stack_overflow) = 0; @@ -585,29 +675,20 @@ static void gc_deal_with_stack_overflow(void) { } } -static void gc_sweep(void) { - #if MICROPY_PY_GC_COLLECT_RETVAL - MP_STATE_MEM(gc_collected) = 0; - #endif - // free unmarked heads and their tails - int free_tail = 0; - #if MICROPY_GC_SPLIT_HEAP_AUTO - mp_state_mem_area_t *prev_area = NULL; - #endif - for (mp_state_mem_area_t *area = &MP_STATE_MEM(area); area != NULL; area = NEXT_AREA(area)) { - size_t end_block = area->gc_alloc_table_byte_len * BLOCKS_PER_ATB; - if (area->gc_last_used_block < end_block) { - end_block = area->gc_last_used_block + 1; - } - - size_t last_used_block = 0; - - for (size_t block = 0; block < end_block; block++) { - MICROPY_GC_HOOK_LOOP(block); - switch (ATB_GET_KIND(area, block)) { - case AT_HEAD: - #if MICROPY_ENABLE_FINALISER - if (FTB_GET(area, block)) { +// Run finalisers for all to-be-freed blocks +static void gc_sweep_run_finalisers(void) { + #if MICROPY_ENABLE_FINALISER + for (const mp_state_mem_area_t *area = &MP_STATE_MEM(area); area != NULL; area = NEXT_AREA(area)) { + assert(area->gc_last_used_block <= area->gc_alloc_table_byte_len * BLOCKS_PER_ATB); + // Small speed optimisation: skip over empty FTB blocks + size_t ftb_end = area->gc_last_used_block / BLOCKS_PER_FTB; // index is inclusive + for (size_t ftb_idx = 0; ftb_idx <= ftb_end; ftb_idx++) { + byte ftb = area->gc_finaliser_table_start[ftb_idx]; + size_t block = ftb_idx * BLOCKS_PER_FTB; + while (ftb) { + MICROPY_GC_HOOK_LOOP(block); + if (ftb & 1) { // FTB_GET(area, block) shortcut + if (ATB_GET_KIND(area, block) == AT_HEAD) { mp_obj_base_t *obj = (mp_obj_base_t *)PTR_FROM_BLOCK(area, block); if (obj->type != NULL) { // if the object has a type then see if it has a __del__ method @@ -627,9 +708,35 @@ static void gc_sweep(void) { // clear finaliser flag FTB_CLEAR(area, block); } - #endif + } + ftb >>= 1; + block++; + } + } + } + #endif // MICROPY_ENABLE_FINALISER +} + +// Free unmarked heads and their tails +static void gc_sweep_free_blocks(void) { + #if MICROPY_PY_GC_COLLECT_RETVAL + MP_STATE_MEM(gc_collected) = 0; + #endif + int free_tail = 0; + #if MICROPY_GC_SPLIT_HEAP_AUTO + mp_state_mem_area_t *prev_area = NULL; + #endif + + for (mp_state_mem_area_t *area = &MP_STATE_MEM(area); area != NULL; area = NEXT_AREA(area)) { + size_t last_used_block = 0; + assert(area->gc_last_used_block <= area->gc_alloc_table_byte_len * BLOCKS_PER_ATB); + + for (size_t block = 0; block <= area->gc_last_used_block; block++) { + MICROPY_GC_HOOK_LOOP(block); + switch (ATB_GET_KIND(area, block)) { + case AT_HEAD: free_tail = 1; - DEBUG_printf("gc_sweep(%p)\n", (void *)PTR_FROM_BLOCK(area, block)); + DEBUG_printf("gc_sweep_free_blocks(%p)\n", (void *)PTR_FROM_BLOCK(area, block)); #if MICROPY_PY_GC_COLLECT_RETVAL MP_STATE_MEM(gc_collected)++; #endif @@ -660,7 +767,7 @@ static void gc_sweep(void) { #if MICROPY_GC_SPLIT_HEAP_AUTO // Free any empty area, aside from the first one if (last_used_block == 0 && prev_area != NULL) { - DEBUG_printf("gc_sweep free empty area %p\n", area); + DEBUG_printf("gc_sweep_free_blocks free empty area %p\n", area); NEXT_AREA(prev_area) = NEXT_AREA(area); MP_PLAT_FREE_HEAP(area); area = prev_area; @@ -670,30 +777,7 @@ static void gc_sweep(void) { } } -void gc_collect_start(void) { - GC_ENTER(); - MP_STATE_THREAD(gc_lock_depth)++; - #if MICROPY_GC_ALLOC_THRESHOLD - MP_STATE_MEM(gc_alloc_amount) = 0; - #endif - MP_STATE_MEM(gc_stack_overflow) = 0; - - // Trace root pointers. This relies on the root pointers being organised - // correctly in the mp_state_ctx structure. We scan nlr_top, dict_locals, - // dict_globals, then the root pointer section of mp_state_vm. - void **ptrs = (void **)(void *)&mp_state_ctx; - size_t root_start = offsetof(mp_state_ctx_t, thread.dict_locals); - size_t root_end = offsetof(mp_state_ctx_t, vm.qstr_last_chunk); - gc_collect_root(ptrs + root_start / sizeof(void *), (root_end - root_start) / sizeof(void *)); - - #if MICROPY_ENABLE_PYSTACK - // Trace root pointers from the Python stack. - ptrs = (void **)(void *)MP_STATE_THREAD(pystack_start); - gc_collect_root(ptrs, (MP_STATE_THREAD(pystack_cur) - MP_STATE_THREAD(pystack_start)) / sizeof(void *)); - #endif -} - -// CIRCUITPY-CHANGE +// CIRCUITPY-CHANGE: add function void gc_collect_ptr(void *ptr) { void *ptrs[1] = { ptr }; gc_collect_root(ptrs, 1); @@ -715,56 +799,6 @@ static void *MP_NO_INSTRUMENT PLACE_IN_ITCM(gc_get_ptr)(void **ptrs, int i) { return ptrs[i]; } -void gc_collect_root(void **ptrs, size_t len) { - #if !MICROPY_GC_SPLIT_HEAP - mp_state_mem_area_t *area = &MP_STATE_MEM(area); - #endif - for (size_t i = 0; i < len; i++) { - MICROPY_GC_HOOK_LOOP(i); - void *ptr = gc_get_ptr(ptrs, i); - #if MICROPY_GC_SPLIT_HEAP - mp_state_mem_area_t *area = gc_get_ptr_area(ptr); - if (!area) { - continue; - } - #else - if (!VERIFY_PTR(ptr)) { - continue; - } - #endif - size_t block = BLOCK_FROM_PTR(area, ptr); - if (ATB_GET_KIND(area, block) == AT_HEAD) { - // An unmarked head: mark it, and mark all its children - ATB_HEAD_TO_MARK(area, block); - #if MICROPY_GC_SPLIT_HEAP - gc_mark_subtree(area, block); - #else - gc_mark_subtree(block); - #endif - } - } -} - -void gc_collect_end(void) { - gc_deal_with_stack_overflow(); - gc_sweep(); - #if MICROPY_GC_SPLIT_HEAP - MP_STATE_MEM(gc_last_free_area) = &MP_STATE_MEM(area); - #endif - for (mp_state_mem_area_t *area = &MP_STATE_MEM(area); area != NULL; area = NEXT_AREA(area)) { - area->gc_last_free_atb_index = 0; - } - MP_STATE_THREAD(gc_lock_depth)--; - GC_EXIT(); -} - -void gc_sweep_all(void) { - GC_ENTER(); - MP_STATE_THREAD(gc_lock_depth)++; - MP_STATE_MEM(gc_stack_overflow) = 0; - gc_collect_end(); -} - void gc_info(gc_info_t *info) { GC_ENTER(); info->total = 0; @@ -838,7 +872,8 @@ void gc_info(gc_info_t *info) { GC_EXIT(); } -// CIRCUITPY-CHANGE: C code may be used when the VM heap isn't active. This +// CIRCUITPY-CHANGE: New function. +// C code may be used when the VM heap isn't active. This function // allows that code to test if it is. It can use the outer pool if needed. bool gc_alloc_possible(void) { return MP_STATE_MEM(area).gc_pool_start != 0; @@ -1038,10 +1073,13 @@ void *gc_alloc(size_t n_bytes, unsigned int alloc_flags) { // force the freeing of a piece of memory // TODO: freeing here does not call finaliser void gc_free(void *ptr) { - if (MP_STATE_THREAD(gc_lock_depth) > 0) { - // Cannot free while the GC is locked. However free is an optimisation - // to reclaim the memory immediately, this means it will now be left - // until the next collection. + // Cannot free while the GC is locked, unless we're only doing a gc sweep. + // However free is an optimisation to reclaim the memory immediately, this + // means it will now be left until the next collection. + // + // (We have the optimisation to free immediately from inside a gc sweep so + // that finalisers can free more memory when trying to avoid MemoryError.) + if (MP_STATE_THREAD(gc_lock_depth) & ~GC_COLLECT_FLAG) { return; } @@ -1071,7 +1109,8 @@ void gc_free(void *ptr) { #endif size_t block = BLOCK_FROM_PTR(area, ptr); - assert(ATB_GET_KIND(area, block) == AT_HEAD); + assert(ATB_GET_KIND(area, block) == AT_HEAD + || (ATB_GET_KIND(area, block) == AT_MARK && (MP_STATE_THREAD(gc_lock_depth) & GC_COLLECT_FLAG))); #if MICROPY_ENABLE_FINALISER FTB_CLEAR(area, block); @@ -1291,12 +1330,13 @@ void *gc_realloc(void *ptr_in, size_t n_bytes, bool allow_move) { uint8_t alloc_flags = 0; #if MICROPY_ENABLE_FINALISER + // CIRCUITPY-CHANGE for selective collect if (FTB_GET(area, block)) { alloc_flags |= GC_ALLOC_FLAG_HAS_FINALISER; } #endif - // CIRCUITPY-CHANGE + // CIRCUITPY-CHANGE for selective collect #if MICROPY_ENABLE_SELECTIVE_COLLECT if (!CTB_GET(area, block)) { alloc_flags |= GC_ALLOC_FLAG_DO_NOT_COLLECT; diff --git a/py/makeqstrdata.py b/py/makeqstrdata.py index d56417b68596b..45cc896f9252e 100644 --- a/py/makeqstrdata.py +++ b/py/makeqstrdata.py @@ -21,15 +21,52 @@ # Python 2/3 compatibility: # - iterating through bytes is different -# - codepoint2name lives in a different module -import platform - -if platform.python_version_tuple()[0] == "2": +# - codepoint2name from html.entities is hard-coded +if sys.version_info[0] == 2: bytes_cons = lambda val, enc=None: bytearray(val) - from htmlentitydefs import codepoint2name -elif platform.python_version_tuple()[0] == "3": +elif sys.version_info[0] == 3: # Also handles MicroPython bytes_cons = bytes - from html.entities import codepoint2name + +# fmt: off +codepoint2name = { + 198: "AElig", 193: "Aacute", 194: "Acirc", 192: "Agrave", 913: "Alpha", 197: "Aring", 195: "Atilde", + 196: "Auml", 914: "Beta", 199: "Ccedil", 935: "Chi", 8225: "Dagger", 916: "Delta", 208: "ETH", + 201: "Eacute", 202: "Ecirc", 200: "Egrave", 917: "Epsilon", 919: "Eta", 203: "Euml", 915: "Gamma", + 205: "Iacute", 206: "Icirc", 204: "Igrave", 921: "Iota", 207: "Iuml", 922: "Kappa", 923: "Lambda", + 924: "Mu", 209: "Ntilde", 925: "Nu", 338: "OElig", 211: "Oacute", 212: "Ocirc", 210: "Ograve", + 937: "Omega", 927: "Omicron", 216: "Oslash", 213: "Otilde", 214: "Ouml", 934: "Phi", 928: "Pi", + 8243: "Prime", 936: "Psi", 929: "Rho", 352: "Scaron", 931: "Sigma", 222: "THORN", 932: "Tau", + 920: "Theta", 218: "Uacute", 219: "Ucirc", 217: "Ugrave", 933: "Upsilon", 220: "Uuml", 926: "Xi", + 221: "Yacute", 376: "Yuml", 918: "Zeta", 225: "aacute", 226: "acirc", 180: "acute", 230: "aelig", + 224: "agrave", 8501: "alefsym", 945: "alpha", 38: "amp", 8743: "and", 8736: "ang", 229: "aring", + 8776: "asymp", 227: "atilde", 228: "auml", 8222: "bdquo", 946: "beta", 166: "brvbar", 8226: "bull", + 8745: "cap", 231: "ccedil", 184: "cedil", 162: "cent", 967: "chi", 710: "circ", 9827: "clubs", + 8773: "cong", 169: "copy", 8629: "crarr", 8746: "cup", 164: "curren", 8659: "dArr", 8224: "dagger", + 8595: "darr", 176: "deg", 948: "delta", 9830: "diams", 247: "divide", 233: "eacute", 234: "ecirc", + 232: "egrave", 8709: "empty", 8195: "emsp", 8194: "ensp", 949: "epsilon", 8801: "equiv", 951: "eta", + 240: "eth", 235: "euml", 8364: "euro", 8707: "exist", 402: "fnof", 8704: "forall", 189: "frac12", + 188: "frac14", 190: "frac34", 8260: "frasl", 947: "gamma", 8805: "ge", 62: "gt", 8660: "hArr", + 8596: "harr", 9829: "hearts", 8230: "hellip", 237: "iacute", 238: "icirc", 161: "iexcl", 236: "igrave", + 8465: "image", 8734: "infin", 8747: "int", 953: "iota", 191: "iquest", 8712: "isin", 239: "iuml", + 954: "kappa", 8656: "lArr", 955: "lambda", 9001: "lang", 171: "laquo", 8592: "larr", 8968: "lceil", + 8220: "ldquo", 8804: "le", 8970: "lfloor", 8727: "lowast", 9674: "loz", 8206: "lrm", 8249: "lsaquo", + 8216: "lsquo", 60: "lt", 175: "macr", 8212: "mdash", 181: "micro", 183: "middot", 8722: "minus", + 956: "mu", 8711: "nabla", 160: "nbsp", 8211: "ndash", 8800: "ne", 8715: "ni", 172: "not", 8713: "notin", + 8836: "nsub", 241: "ntilde", 957: "nu", 243: "oacute", 244: "ocirc", 339: "oelig", 242: "ograve", + 8254: "oline", 969: "omega", 959: "omicron", 8853: "oplus", 8744: "or", 170: "ordf", 186: "ordm", + 248: "oslash", 245: "otilde", 8855: "otimes", 246: "ouml", 182: "para", 8706: "part", 8240: "permil", + 8869: "perp", 966: "phi", 960: "pi", 982: "piv", 177: "plusmn", 163: "pound", 8242: "prime", + 8719: "prod", 8733: "prop", 968: "psi", 34: "quot", 8658: "rArr", 8730: "radic", 9002: "rang", + 187: "raquo", 8594: "rarr", 8969: "rceil", 8221: "rdquo", 8476: "real", 174: "reg", 8971: "rfloor", + 961: "rho", 8207: "rlm", 8250: "rsaquo", 8217: "rsquo", 8218: "sbquo", 353: "scaron", 8901: "sdot", + 167: "sect", 173: "shy", 963: "sigma", 962: "sigmaf", 8764: "sim", 9824: "spades", 8834: "sub", + 8838: "sube", 8721: "sum", 8835: "sup", 185: "sup1", 178: "sup2", 179: "sup3", 8839: "supe", + 223: "szlig", 964: "tau", 8756: "there4", 952: "theta", 977: "thetasym", 8201: "thinsp", 254: "thorn", + 732: "tilde", 215: "times", 8482: "trade", 8657: "uArr", 250: "uacute", 8593: "uarr", 251: "ucirc", + 249: "ugrave", 168: "uml", 978: "upsih", 965: "upsilon", 252: "uuml", 8472: "weierp", 958: "xi", + 253: "yacute", 165: "yen", 255: "yuml", 950: "zeta", 8205: "zwj", 8204: "zwnj" +} +# fmt: on # end compatibility code codepoint2name[ord("-")] = "hyphen" @@ -305,6 +342,9 @@ "", } +# Matches any string that needs no escaping (alphanum + _ only) +RE_NO_ESCAPE = re.compile(r"^[a-zA-Z0-9_]$") + # this must match the equivalent function in qstr.c def compute_hash(qstr, bytes_hash): @@ -317,15 +357,17 @@ def compute_hash(qstr, bytes_hash): def qstr_escape(qst): - def esc_char(m): - c = ord(m.group(0)) + def esc_char(c): + if RE_NO_ESCAPE.match(c): + return c + c = ord(c) try: name = codepoint2name[c] except KeyError: name = "0x%02x" % c return "_" + name + "_" - return re.sub(r"[^A-Za-z0-9_]", esc_char, qst) + return "".join(map(esc_char, qst)) static_qstr_list_ident = list(map(qstr_escape, static_qstr_list)) diff --git a/py/misc.h b/py/misc.h index 3624efaa52220..eb7fc54be6fec 100644 --- a/py/misc.h +++ b/py/misc.h @@ -386,7 +386,7 @@ static inline uint32_t mp_clzll(unsigned long long x) { // Microsoft don't ship _BitScanReverse64 on Win32, so emulate it static inline uint32_t mp_clzll(unsigned long long x) { unsigned long h = x >> 32; - return h ? mp_clzl(h) : (mp_clzl(x) + 32); + return h ? mp_clzl(h) : (mp_clzl((unsigned long)x) + 32); } #endif @@ -399,12 +399,29 @@ static inline uint32_t mp_ctz(uint32_t x) { static inline bool mp_check(bool value) { return value; } + +static inline uint32_t mp_popcount(uint32_t x) { + return __popcnt(x); +} #else #define mp_clz(x) __builtin_clz(x) #define mp_clzl(x) __builtin_clzl(x) #define mp_clzll(x) __builtin_clzll(x) #define mp_ctz(x) __builtin_ctz(x) #define mp_check(x) (x) +#if defined __has_builtin +#if __has_builtin(__builtin_popcount) +#define mp_popcount(x) __builtin_popcount(x) +#endif +#endif +#if !defined(mp_popcount) +static inline uint32_t mp_popcount(uint32_t x) { + x = x - ((x >> 1) & 0x55555555); + x = (x & 0x33333333) + ((x >> 2) & 0x33333333); + x = (x + (x >> 4)) & 0x0F0F0F0F; + return x * 0x01010101; +} +#endif #endif // mp_int_t can be larger than long, i.e. Windows 64-bit, nan-box variants diff --git a/py/mkrules.cmake b/py/mkrules.cmake index bfc56abfe80b7..4374b8b4da3cb 100644 --- a/py/mkrules.cmake +++ b/py/mkrules.cmake @@ -19,6 +19,19 @@ if(NOT MICROPY_PREVIEW_VERSION_2) set(MICROPY_PREVIEW_VERSION_2 0) endif() +# Set the board name. +if(MICROPY_BOARD) + if(MICROPY_BOARD_VARIANT) + set(MICROPY_BOARD_BUILD_NAME ${MICROPY_BOARD}-${MICROPY_BOARD_VARIANT}) + else() + set(MICROPY_BOARD_BUILD_NAME ${MICROPY_BOARD}) + endif() + + target_compile_definitions(${MICROPY_TARGET} PRIVATE + MICROPY_BOARD_BUILD_NAME="${MICROPY_BOARD_BUILD_NAME}" + ) +endif() + # Need to do this before extracting MICROPY_CPP_DEF below. Rest of frozen # manifest handling is at the end of this file. if(MICROPY_FROZEN_MANIFEST) @@ -53,6 +66,15 @@ foreach(_arg ${MICROPY_CPP_DEF}) endforeach() list(APPEND MICROPY_CPP_FLAGS ${MICROPY_CPP_FLAGS_EXTRA}) +# Include anything passed in via CFLAGS_EXTRA +# in both MICROPY_CPP_FLAGS and CMAKE_C_FLAGS +if(DEFINED ENV{CFLAGS_EXTRA}) + set(CFLAGS_EXTRA $ENV{CFLAGS_EXTRA}) + string(APPEND CMAKE_C_FLAGS " ${CFLAGS_EXTRA}") # ... not a list + separate_arguments(CFLAGS_EXTRA) + list(APPEND MICROPY_CPP_FLAGS ${CFLAGS_EXTRA}) # ... a list +endif() + find_package(Python3 REQUIRED COMPONENTS Interpreter) target_sources(${MICROPY_TARGET} PRIVATE @@ -187,16 +209,11 @@ if(MICROPY_FROZEN_MANIFEST) # Note: target_compile_definitions already added earlier. if(NOT MICROPY_LIB_DIR) - string(CONCAT GIT_SUBMODULES "${GIT_SUBMODULES} " lib/micropython-lib) + list(APPEND GIT_SUBMODULES lib/micropython-lib) set(MICROPY_LIB_DIR ${MICROPY_DIR}/lib/micropython-lib) endif() - if(ECHO_SUBMODULES) - # No-op, we're just doing submodule/variant discovery. - # Note: All the following rules are safe to run in discovery mode even - # though the submodule might not be available as they do not directly depend - # on anything from the submodule. - elseif(NOT EXISTS ${MICROPY_LIB_DIR}/README.md) + if(NOT UPDATE_SUBMODULES AND NOT EXISTS ${MICROPY_LIB_DIR}/README.md) message(FATAL_ERROR " micropython-lib not initialized.\n Run 'make BOARD=${MICROPY_BOARD} submodules'") endif() @@ -211,7 +228,7 @@ if(MICROPY_FROZEN_MANIFEST) endif() add_custom_command( OUTPUT ${MICROPY_MPYCROSS_DEPENDENCY} - COMMAND ${MICROPY_MAKE_EXECUTABLE} -C ${MICROPY_DIR}/mpy-cross + COMMAND ${MICROPY_MAKE_EXECUTABLE} -C ${MICROPY_DIR}/mpy-cross USER_C_MODULES= ) endif() @@ -250,12 +267,29 @@ if(MICROPY_FROZEN_MANIFEST) ) endif() -# Update submodules -if(ECHO_SUBMODULES) - # If cmake is run with GIT_SUBMODULES defined on command line, process the port / board - # settings then print the final GIT_SUBMODULES variable and exit. - # Note: the GIT_SUBMODULES is done via echo rather than message, as message splits - # the output onto multiple lines - execute_process(COMMAND ${CMAKE_COMMAND} -E echo "GIT_SUBMODULES=${GIT_SUBMODULES}") - message(FATAL_ERROR "Done") +# Update submodules, this is invoked on some ports via 'make submodules'. +# +# Note: This logic has a Makefile equivalent in py/mkrules.mk +if(UPDATE_SUBMODULES AND GIT_SUBMODULES) + macro(run_git) + execute_process(COMMAND git ${ARGV} WORKING_DIRECTORY ${MICROPY_DIR} + RESULT_VARIABLE RES) + endmacro() + + list(JOIN GIT_SUBMODULES " " GIT_SUBMODULES_MSG) + message("Updating submodules: ${GIT_SUBMODULES_MSG}") + run_git(submodule sync ${GIT_SUBMODULES}) + if(RES EQUAL 0) + # If available, do blobless partial clones of submodules to save time and space. + # A blobless partial clone lazily fetches data as needed, but has all the metadata available (tags, etc.). + run_git(submodule update --init --filter=blob:none ${GIT_SUBMODULES}) + # Fallback to standard submodule update if blobless isn't available (earlier than git 2.36.0) + if (NOT RES EQUAL 0) + run_git(submodule update --init ${GIT_SUBMODULES}) + endif() + endif() + + if (NOT RES EQUAL 0) + message(FATAL_ERROR "Submodule update failed") + endif() endif() diff --git a/py/mkrules.mk b/py/mkrules.mk index e0c2cfd979425..f364297f0f209 100644 --- a/py/mkrules.mk +++ b/py/mkrules.mk @@ -27,6 +27,17 @@ OBJ_EXTRA_ORDER_DEPS += $(HEADER_BUILD)/compressed.data.h CFLAGS += -DMICROPY_ROM_TEXT_COMPRESSION=1 endif +# Set the variant or board name. +ifneq ($(VARIANT),) +CFLAGS += -DMICROPY_BOARD_BUILD_NAME=\"$(VARIANT)\" +else ifneq ($(BOARD),) +ifeq ($(BOARD_VARIANT),) +CFLAGS += -DMICROPY_BOARD_BUILD_NAME=\"$(BOARD)\" +else +CFLAGS += -DMICROPY_BOARD_BUILD_NAME=\"$(BOARD)-$(BOARD_VARIANT)\" +endif +endif + # QSTR generation uses the same CFLAGS, with these modifications. QSTR_GEN_FLAGS = -DNO_QSTR # Note: := to force evaluation immediately. @@ -178,7 +189,7 @@ $(HEADER_BUILD): ifneq ($(MICROPY_MPYCROSS_DEPENDENCY),) # to automatically build mpy-cross, if needed $(MICROPY_MPYCROSS_DEPENDENCY): - $(MAKE) -C "$(abspath $(dir $@)..)" + $(MAKE) -C "$(abspath $(dir $@)..)" USER_C_MODULES= endif ifneq ($(FROZEN_DIR),) @@ -240,11 +251,17 @@ clean-prog: .PHONY: clean-prog endif +# If available, do blobless partial clones of submodules to save time and space. +# A blobless partial clone lazily fetches data as needed, but has all the metadata available (tags, etc.). +# Fallback to standard submodule update if blobless isn't available (earlier than 2.36.0) +# +# Note: This target has a CMake equivalent in py/mkrules.cmake submodules: $(ECHO) "Updating submodules: $(GIT_SUBMODULES)" ifneq ($(GIT_SUBMODULES),) $(Q)cd $(TOP) && git submodule sync $(GIT_SUBMODULES) - $(Q)cd $(TOP) && git submodule update --init $(GIT_SUBMODULES) + $(Q)cd $(TOP) && git submodule update --init --filter=blob:none $(GIT_SUBMODULES) || \ + git submodule update --init $(GIT_SUBMODULES) endif .PHONY: submodules diff --git a/py/modmicropython.c b/py/modmicropython.c index e48b033648d0f..ff25af8ff7eec 100644 --- a/py/modmicropython.c +++ b/py/modmicropython.c @@ -136,13 +136,13 @@ static MP_DEFINE_CONST_FUN_OBJ_0(mp_micropython_heap_lock_obj, mp_micropython_he static mp_obj_t mp_micropython_heap_unlock(void) { gc_unlock(); - return MP_OBJ_NEW_SMALL_INT(MP_STATE_THREAD(gc_lock_depth)); + return MP_OBJ_NEW_SMALL_INT(MP_STATE_THREAD(gc_lock_depth) >> GC_LOCK_DEPTH_SHIFT); } static MP_DEFINE_CONST_FUN_OBJ_0(mp_micropython_heap_unlock_obj, mp_micropython_heap_unlock); #if MICROPY_PY_MICROPYTHON_HEAP_LOCKED static mp_obj_t mp_micropython_heap_locked(void) { - return MP_OBJ_NEW_SMALL_INT(MP_STATE_THREAD(gc_lock_depth)); + return MP_OBJ_NEW_SMALL_INT(MP_STATE_THREAD(gc_lock_depth) >> GC_LOCK_DEPTH_SHIFT); } static MP_DEFINE_CONST_FUN_OBJ_0(mp_micropython_heap_locked_obj, mp_micropython_heap_locked); #endif diff --git a/py/modsys.c b/py/modsys.c index ff63f1e2f88cc..2adbc0b7bd66f 100644 --- a/py/modsys.c +++ b/py/modsys.c @@ -102,6 +102,17 @@ static const MP_DEFINE_STR_OBJ(mp_sys_implementation_machine_obj, MICROPY_BANNER #endif #if MICROPY_PY_ATTRTUPLE + +#if defined(MICROPY_BOARD_BUILD_NAME) +static const MP_DEFINE_STR_OBJ(mp_sys_implementation__build_obj, MICROPY_BOARD_BUILD_NAME); +#define MICROPY_BOARD_BUILD (1) +#define SYS_IMPLEMENTATION_ELEMS__BUILD \ + , MP_ROM_PTR(&mp_sys_implementation__build_obj) +#else +#define MICROPY_BOARD_BUILD (0) +#define SYS_IMPLEMENTATION_ELEMS__BUILD +#endif + #if MICROPY_PREVIEW_VERSION_2 #define SYS_IMPLEMENTATION_ELEMS__V2 \ , MP_ROM_TRUE @@ -116,6 +127,9 @@ static const qstr impl_fields[] = { #if MICROPY_PERSISTENT_CODE_LOAD MP_QSTR__mpy, #endif + #if defined(MICROPY_BOARD_BUILD_NAME) + MP_QSTR__build, + #endif #if MICROPY_PREVIEW_VERSION_2 MP_QSTR__v2, #endif @@ -123,19 +137,20 @@ static const qstr impl_fields[] = { static MP_DEFINE_ATTRTUPLE( mp_sys_implementation_obj, impl_fields, - 3 + MICROPY_PERSISTENT_CODE_LOAD + MICROPY_PREVIEW_VERSION_2, + 3 + MICROPY_PERSISTENT_CODE_LOAD + MICROPY_BOARD_BUILD + MICROPY_PREVIEW_VERSION_2, SYS_IMPLEMENTATION_ELEMS_BASE SYS_IMPLEMENTATION_ELEMS__MPY + SYS_IMPLEMENTATION_ELEMS__BUILD SYS_IMPLEMENTATION_ELEMS__V2 ); #else static const mp_rom_obj_tuple_t mp_sys_implementation_obj = { {&mp_type_tuple}, 3 + MICROPY_PERSISTENT_CODE_LOAD, - // Do not include SYS_IMPLEMENTATION_ELEMS__V2 because - // SYS_IMPLEMENTATION_ELEMS__MPY may be empty if + // Do not include SYS_IMPLEMENTATION_ELEMS__BUILD or SYS_IMPLEMENTATION_ELEMS__V2 + // because SYS_IMPLEMENTATION_ELEMS__MPY may be empty if // MICROPY_PERSISTENT_CODE_LOAD is disabled, which means they'll share - // the same index. Cannot query _v2 if MICROPY_PY_ATTRTUPLE is + // the same index. Cannot query _build or _v2 if MICROPY_PY_ATTRTUPLE is // disabled. { SYS_IMPLEMENTATION_ELEMS_BASE diff --git a/py/mpconfig.h b/py/mpconfig.h index 20e237cc1cca5..a48958200616b 100644 --- a/py/mpconfig.h +++ b/py/mpconfig.h @@ -40,8 +40,8 @@ // as well as a fallback to generate MICROPY_GIT_TAG if the git repo or tags // are unavailable. #define MICROPY_VERSION_MAJOR 1 -#define MICROPY_VERSION_MINOR 24 -#define MICROPY_VERSION_MICRO 1 +#define MICROPY_VERSION_MINOR 25 +#define MICROPY_VERSION_MICRO 0 #define MICROPY_VERSION_PRERELEASE 0 // Combined version as a 32-bit number for convenience to allow version @@ -354,6 +354,11 @@ #define MICROPY_PERSISTENT_CODE_SAVE_FILE (0) #endif +// Whether to support converting functions to persistent code (bytes) +#ifndef MICROPY_PERSISTENT_CODE_SAVE_FUN +#define MICROPY_PERSISTENT_CODE_SAVE_FUN (MICROPY_PY_MARSHAL) +#endif + // Whether generated code can persist independently of the VM/runtime instance // This is enabled automatically when needed by other features #ifndef MICROPY_PERSISTENT_CODE @@ -428,6 +433,11 @@ #define MICROPY_EMIT_NATIVE_DEBUG (0) #endif +// Whether to enable the RISC-V RV32 inline assembler +#ifndef MICROPY_EMIT_INLINE_RV32 +#define MICROPY_EMIT_INLINE_RV32 (0) +#endif + // Convenience definition for whether any native emitter is enabled #define MICROPY_EMIT_NATIVE (MICROPY_EMIT_X64 || MICROPY_EMIT_X86 || MICROPY_EMIT_THUMB || MICROPY_EMIT_ARM || MICROPY_EMIT_XTENSA || MICROPY_EMIT_XTENSAWIN || MICROPY_EMIT_RV32 || MICROPY_EMIT_NATIVE_DEBUG) @@ -437,7 +447,7 @@ #define MICROPY_EMIT_NATIVE_PRELUDE_SEPARATE_FROM_MACHINE_CODE (MICROPY_EMIT_XTENSAWIN) // Convenience definition for whether any inline assembler emitter is enabled -#define MICROPY_EMIT_INLINE_ASM (MICROPY_EMIT_INLINE_THUMB || MICROPY_EMIT_INLINE_XTENSA) +#define MICROPY_EMIT_INLINE_ASM (MICROPY_EMIT_INLINE_THUMB || MICROPY_EMIT_INLINE_XTENSA || MICROPY_EMIT_INLINE_RV32) // Convenience definition for whether any native or inline assembler emitter is enabled #define MICROPY_EMIT_MACHINE_CODE (MICROPY_EMIT_NATIVE || MICROPY_EMIT_INLINE_ASM) @@ -1040,6 +1050,16 @@ typedef double mp_float_t; #define MICROPY_VFS (0) #endif +// Whether to include support for writable filesystems. +#ifndef MICROPY_VFS_WRITABLE +#define MICROPY_VFS_WRITABLE (1) +#endif + +// Whether to enable the mp_vfs_rom_ioctl C function, and vfs.rom_ioctl Python function +#ifndef MICROPY_VFS_ROM_IOCTL +#define MICROPY_VFS_ROM_IOCTL (MICROPY_VFS_ROM) +#endif + // Support for VFS POSIX component, to mount a POSIX filesystem within VFS #ifndef MICROPY_VFS_POSIX #define MICROPY_VFS_POSIX (0) @@ -1060,6 +1080,11 @@ typedef double mp_float_t; #define MICROPY_VFS_LFS2 (0) #endif +// Support for ROMFS. +#ifndef MICROPY_VFS_ROM +#define MICROPY_VFS_ROM (0) +#endif + /*****************************************************************************/ /* Fine control over Python builtins, classes, modules, etc */ @@ -1075,6 +1100,11 @@ typedef double mp_float_t; #define MICROPY_PY_FUNCTION_ATTRS (MICROPY_CONFIG_ROM_LEVEL_AT_LEAST_EXTRA_FEATURES) #endif +// Whether to implement the __code__ attribute on functions, and function constructor +#ifndef MICROPY_PY_FUNCTION_ATTRS_CODE +#define MICROPY_PY_FUNCTION_ATTRS_CODE (MICROPY_CONFIG_ROM_LEVEL_AT_LEAST_FULL_FEATURES) +#endif + // Whether to support the descriptors __get__, __set__, __delete__ // This costs some code size and makes load/store/delete of instance // attributes slower for the classes that use this feature @@ -1163,6 +1193,15 @@ typedef double mp_float_t; #define MICROPY_PY_BUILTINS_BYTEARRAY (MICROPY_CONFIG_ROM_LEVEL_AT_LEAST_CORE_FEATURES) #endif +// Whether to support code objects, and how many features they have +#define MICROPY_PY_BUILTINS_CODE_NONE (0) +#define MICROPY_PY_BUILTINS_CODE_MINIMUM (1) +#define MICROPY_PY_BUILTINS_CODE_BASIC (2) +#define MICROPY_PY_BUILTINS_CODE_FULL (3) +#ifndef MICROPY_PY_BUILTINS_CODE +#define MICROPY_PY_BUILTINS_CODE (MICROPY_PY_SYS_SETTRACE ? MICROPY_PY_BUILTINS_CODE_FULL : (MICROPY_PY_FUNCTION_ATTRS_CODE ? MICROPY_PY_BUILTINS_CODE_BASIC : (MICROPY_PY_BUILTINS_COMPILE ? MICROPY_PY_BUILTINS_CODE_MINIMUM : MICROPY_PY_BUILTINS_CODE_NONE))) +#endif + // Whether to support dict.fromkeys() class method #ifndef MICROPY_PY_BUILTINS_DICT_FROMKEYS #define MICROPY_PY_BUILTINS_DICT_FROMKEYS (MICROPY_CONFIG_ROM_LEVEL_AT_LEAST_CORE_FEATURES) @@ -1225,7 +1264,7 @@ typedef double mp_float_t; // Support for calling next() with second argument #ifndef MICROPY_PY_BUILTINS_NEXT2 -#define MICROPY_PY_BUILTINS_NEXT2 (MICROPY_CONFIG_ROM_LEVEL_AT_LEAST_EVERYTHING) +#define MICROPY_PY_BUILTINS_NEXT2 (MICROPY_CONFIG_ROM_LEVEL_AT_LEAST_BASIC_FEATURES) #endif // Whether to support rounding of integers (incl bignum); eg round(123,-1)=120 @@ -1397,6 +1436,11 @@ typedef double mp_float_t; #define MICROPY_PY_COLLECTIONS_NAMEDTUPLE__ASDICT (MICROPY_CONFIG_ROM_LEVEL_AT_LEAST_EVERYTHING) #endif +// Whether to provide "marshal" module +#ifndef MICROPY_PY_MARSHAL +#define MICROPY_PY_MARSHAL (MICROPY_CONFIG_ROM_LEVEL_AT_LEAST_EVERYTHING) +#endif + // Whether to provide "math" module #ifndef MICROPY_PY_MATH #define MICROPY_PY_MATH (MICROPY_CONFIG_ROM_LEVEL_AT_LEAST_CORE_FEATURES) @@ -1669,6 +1713,11 @@ typedef double mp_float_t; #define MICROPY_PY_THREAD_GIL_VM_DIVISOR (32) #endif +// Is a recursive mutex type in use? +#ifndef MICROPY_PY_THREAD_RECURSIVE_MUTEX +#define MICROPY_PY_THREAD_RECURSIVE_MUTEX (MICROPY_PY_THREAD && !MICROPY_PY_THREAD_GIL) +#endif + // Extended modules #ifndef MICROPY_PY_ASYNCIO @@ -1780,6 +1829,7 @@ typedef double mp_float_t; #endif // CIRCUITPY-CHANGE: does not depend on MICROPY_PY_DEFLATE +// Depends on MICROPY_PY_DEFLATE #ifndef MICROPY_PY_BINASCII_CRC32 #define MICROPY_PY_BINASCII_CRC32 (MICROPY_CONFIG_ROM_LEVEL_AT_LEAST_EXTRA_FEATURES) #endif diff --git a/py/mphal.h b/py/mphal.h index a4f222d0b1e11..95289ac856cb2 100644 --- a/py/mphal.h +++ b/py/mphal.h @@ -88,6 +88,8 @@ mp_uint_t mp_hal_ticks_cpu(void); uint64_t mp_hal_time_ns(void); #endif +// CIRCUITPY-CHANGE: extmod/virtpin.* not used by CircuitPython +#if 0 // If port HAL didn't define its own pin API, use generic // "virtual pin" API from the core. #ifndef mp_hal_pin_obj_t @@ -97,6 +99,7 @@ uint64_t mp_hal_time_ns(void); #define mp_hal_pin_write(pin, v) mp_virtual_pin_write(pin, v) #include "extmod/virtpin.h" #endif +#endif // Event handling and wait-for-event functions. diff --git a/py/mpstate.h b/py/mpstate.h index aa85bd450cd55..4c48e9edaf4bd 100644 --- a/py/mpstate.h +++ b/py/mpstate.h @@ -88,6 +88,18 @@ typedef struct _mp_sched_item_t { mp_obj_t arg; } mp_sched_item_t; +// gc_lock_depth field is a combination of the GC_COLLECT_FLAG +// bit and a lock depth shifted GC_LOCK_DEPTH_SHIFT bits left. +#if MICROPY_ENABLE_FINALISER +#define GC_COLLECT_FLAG 1 +#define GC_LOCK_DEPTH_SHIFT 1 +#else +// If finalisers are disabled then this check doesn't matter, as gc_lock() +// is called anywhere else that heap can't be changed. So save some code size. +#define GC_COLLECT_FLAG 0 +#define GC_LOCK_DEPTH_SHIFT 0 +#endif + // This structure holds information about a single contiguous area of // memory reserved for the memory manager. typedef struct _mp_state_mem_area_t { @@ -148,7 +160,7 @@ typedef struct _mp_state_mem_t { #if MICROPY_PY_THREAD && !MICROPY_PY_THREAD_GIL // This is a global mutex used to make the GC thread-safe. - mp_thread_mutex_t gc_mutex; + mp_thread_recursive_mutex_t gc_mutex; #endif } mp_state_mem_t; @@ -296,6 +308,7 @@ typedef struct _mp_state_thread_t { #endif // Locking of the GC is done per thread. + // See GC_LOCK_DEPTH_SHIFT for an explanation of this field. uint16_t gc_lock_depth; //////////////////////////////////////////////////////////// diff --git a/py/mpthread.h b/py/mpthread.h index f335cc02911fc..795f230bb4a0c 100644 --- a/py/mpthread.h +++ b/py/mpthread.h @@ -48,6 +48,12 @@ void mp_thread_mutex_init(mp_thread_mutex_t *mutex); int mp_thread_mutex_lock(mp_thread_mutex_t *mutex, int wait); void mp_thread_mutex_unlock(mp_thread_mutex_t *mutex); +#if MICROPY_PY_THREAD_RECURSIVE_MUTEX +void mp_thread_recursive_mutex_init(mp_thread_recursive_mutex_t *mutex); +int mp_thread_recursive_mutex_lock(mp_thread_recursive_mutex_t *mutex, int wait); +void mp_thread_recursive_mutex_unlock(mp_thread_recursive_mutex_t *mutex); +#endif + #endif // MICROPY_PY_THREAD #if MICROPY_PY_THREAD && MICROPY_PY_THREAD_GIL diff --git a/py/obj.h b/py/obj.h index 1a7c3f60fb5ba..38db17cba449b 100644 --- a/py/obj.h +++ b/py/obj.h @@ -184,12 +184,13 @@ static inline bool mp_obj_is_small_int(mp_const_obj_t o) { #define MP_OBJ_NEW_SMALL_INT(small_int) ((mp_obj_t)((((mp_uint_t)(small_int)) << 1) | 1)) #if MICROPY_PY_BUILTINS_FLOAT -#define mp_const_float_e MP_ROM_PTR((mp_obj_t)(((0x402df854 & ~3) | 2) + 0x80800000)) -#define mp_const_float_pi MP_ROM_PTR((mp_obj_t)(((0x40490fdb & ~3) | 2) + 0x80800000)) +#define MP_OBJ_NEW_CONST_FLOAT(f) MP_ROM_PTR((mp_obj_t)((((((uint64_t)f) & ~3) | 2) + 0x80800000) & 0xffffffff)) +#define mp_const_float_e MP_OBJ_NEW_CONST_FLOAT(0x402df854) +#define mp_const_float_pi MP_OBJ_NEW_CONST_FLOAT(0x40490fdb) #if MICROPY_PY_MATH_CONSTANTS -#define mp_const_float_tau MP_ROM_PTR((mp_obj_t)(((0x40c90fdb & ~3) | 2) + 0x80800000)) -#define mp_const_float_inf MP_ROM_PTR((mp_obj_t)(((0x7f800000 & ~3) | 2) + 0x80800000)) -#define mp_const_float_nan MP_ROM_PTR((mp_obj_t)(((0xffc00000 & ~3) | 2) + 0x80800000)) +#define mp_const_float_tau MP_OBJ_NEW_CONST_FLOAT(0x40c90fdb) +#define mp_const_float_inf MP_OBJ_NEW_CONST_FLOAT(0x7f800000) +#define mp_const_float_nan MP_OBJ_NEW_CONST_FLOAT(0xffc00000) #endif static inline bool mp_obj_is_float(mp_const_obj_t o) { @@ -202,7 +203,7 @@ static inline mp_float_t mp_obj_float_get(mp_const_obj_t o) { union { mp_float_t f; mp_uint_t u; - } num = {.u = ((mp_uint_t)o - 0x80800000) & ~3}; + } num = {.u = ((mp_uint_t)o - 0x80800000u) & ~3u}; return num.f; } static inline mp_obj_t mp_obj_new_float(mp_float_t f) { @@ -210,7 +211,7 @@ static inline mp_obj_t mp_obj_new_float(mp_float_t f) { mp_float_t f; mp_uint_t u; } num = {.f = f}; - return (mp_obj_t)(((num.u & ~0x3) | 2) + 0x80800000); + return (mp_obj_t)(((num.u & ~0x3u) | 2u) + 0x80800000u); } #endif @@ -881,6 +882,7 @@ extern const mp_obj_type_t mp_type_fun_bc; extern const mp_obj_type_t mp_type_fun_native; extern const mp_obj_type_t mp_type_fun_viper; extern const mp_obj_type_t mp_type_fun_asm; +extern const mp_obj_type_t mp_type_code; extern const mp_obj_type_t mp_type_module; extern const mp_obj_type_t mp_type_staticmethod; extern const mp_obj_type_t mp_type_classmethod; diff --git a/py/objarray.h b/py/objarray.h index 4a0e8a983fe77..bb7a514b97913 100644 --- a/py/objarray.h +++ b/py/objarray.h @@ -53,6 +53,10 @@ typedef struct _mp_obj_array_t { } mp_obj_array_t; #if MICROPY_PY_BUILTINS_MEMORYVIEW + +#define MP_DEFINE_MEMORYVIEW_OBJ(obj_name, typecode, offset, len, ptr) \ + mp_obj_array_t obj_name = {{&mp_type_memoryview}, (typecode), (offset), (len), (ptr)} + static inline void mp_obj_memoryview_init(mp_obj_array_t *self, size_t typecode, size_t offset, size_t len, void *items) { self->base.type = &mp_type_memoryview; self->typecode = typecode; @@ -60,6 +64,7 @@ static inline void mp_obj_memoryview_init(mp_obj_array_t *self, size_t typecode, self->len = len; self->items = items; } + #endif #if MICROPY_PY_ARRAY || MICROPY_PY_BUILTINS_BYTEARRAY diff --git a/py/objcode.c b/py/objcode.c new file mode 100644 index 0000000000000..9b98a696798d4 --- /dev/null +++ b/py/objcode.c @@ -0,0 +1,175 @@ +/* + * This file is part of the MicroPython project, http://micropython.org/ + * + * The MIT License (MIT) + * + * Copyright (c) 2025 Damien P. George + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ + +#include "py/objcode.h" + +#if MICROPY_PY_BUILTINS_CODE == MICROPY_PY_BUILTINS_CODE_NONE + +// Code object not implemented at this configuration level. + +#elif MICROPY_PY_BUILTINS_CODE <= MICROPY_PY_BUILTINS_CODE_BASIC + +MP_DEFINE_CONST_OBJ_TYPE( + mp_type_code, + MP_QSTR_code, + MP_TYPE_FLAG_NONE + ); + +#elif MICROPY_PY_BUILTINS_CODE <= MICROPY_PY_BUILTINS_CODE_FULL + +#include "py/profile.h" + +static void code_print(const mp_print_t *print, mp_obj_t o_in, mp_print_kind_t kind) { + (void)kind; + mp_obj_code_t *o = MP_OBJ_TO_PTR(o_in); + const mp_raw_code_t *rc = o->rc; + const mp_bytecode_prelude_t *prelude = &rc->prelude; + mp_printf(print, + "", + MP_CODE_QSTR_MAP(o->context, prelude->qstr_block_name_idx), + o, + MP_CODE_QSTR_MAP(o->context, 0), + rc->line_of_definition + ); +} + +static mp_obj_tuple_t *code_consts(const mp_module_context_t *context, const mp_raw_code_t *rc) { + mp_obj_tuple_t *consts = MP_OBJ_TO_PTR(mp_obj_new_tuple(rc->n_children + 1, NULL)); + + size_t const_no = 0; + for (size_t i = 0; i < rc->n_children; ++i) { + mp_obj_t code = mp_obj_new_code(context, rc->children[i], true); + consts->items[const_no++] = code; + } + consts->items[const_no++] = mp_const_none; + + return consts; +} + +static mp_obj_t raw_code_lnotab(const mp_raw_code_t *rc) { + // const mp_bytecode_prelude_t *prelude = &rc->prelude; + uint start = 0; + uint stop = rc->fun_data_len - start; + + uint last_lineno = mp_prof_bytecode_lineno(rc, start); + uint lasti = 0; + + const uint buffer_chunk_size = (stop - start) >> 2; // heuristic magic + uint buffer_size = buffer_chunk_size; + byte *buffer = m_new(byte, buffer_size); + uint buffer_index = 0; + + for (uint i = start; i < stop; ++i) { + uint lineno = mp_prof_bytecode_lineno(rc, i); + size_t line_diff = lineno - last_lineno; + if (line_diff > 0) { + uint instr_diff = (i - start) - lasti; + + assert(instr_diff < 256); + assert(line_diff < 256); + + if (buffer_index + 2 > buffer_size) { + buffer = m_renew(byte, buffer, buffer_size, buffer_size + buffer_chunk_size); + buffer_size = buffer_size + buffer_chunk_size; + } + last_lineno = lineno; + lasti = i - start; + buffer[buffer_index++] = instr_diff; + buffer[buffer_index++] = line_diff; + } + } + + mp_obj_t o = mp_obj_new_bytes(buffer, buffer_index); + m_del(byte, buffer, buffer_size); + return o; +} + +static void code_attr(mp_obj_t self_in, qstr attr, mp_obj_t *dest) { + if (dest[0] != MP_OBJ_NULL) { + // not load attribute + return; + } + mp_obj_code_t *o = MP_OBJ_TO_PTR(self_in); + const mp_raw_code_t *rc = o->rc; + const mp_bytecode_prelude_t *prelude = &rc->prelude; + switch (attr) { + case MP_QSTR_co_code: + dest[0] = mp_obj_new_bytes( + (void *)prelude->opcodes, + rc->fun_data_len - (prelude->opcodes - (const byte *)rc->fun_data) + ); + break; + case MP_QSTR_co_consts: + dest[0] = MP_OBJ_FROM_PTR(code_consts(o->context, rc)); + break; + case MP_QSTR_co_filename: + dest[0] = MP_OBJ_NEW_QSTR(MP_CODE_QSTR_MAP(o->context, 0)); + break; + case MP_QSTR_co_firstlineno: + dest[0] = MP_OBJ_NEW_SMALL_INT(mp_prof_bytecode_lineno(rc, 0)); + break; + case MP_QSTR_co_name: + dest[0] = MP_OBJ_NEW_QSTR(MP_CODE_QSTR_MAP(o->context, prelude->qstr_block_name_idx)); + break; + case MP_QSTR_co_names: + dest[0] = MP_OBJ_FROM_PTR(o->dict_locals); + break; + case MP_QSTR_co_lnotab: + if (!o->lnotab) { + o->lnotab = raw_code_lnotab(rc); + } + dest[0] = o->lnotab; + break; + } +} + +MP_DEFINE_CONST_OBJ_TYPE( + mp_type_code, + MP_QSTR_code, + MP_TYPE_FLAG_NONE, + print, code_print, + attr, code_attr + ); + +mp_obj_t mp_obj_new_code(const mp_module_context_t *context, const mp_raw_code_t *rc, bool result_required) { + mp_obj_code_t *o; + if (result_required) { + o = m_new_obj(mp_obj_code_t); + } else { + o = m_new_obj_maybe(mp_obj_code_t); + if (o == NULL) { + return MP_OBJ_NULL; + } + } + o->base.type = &mp_type_code; + o->context = context; + o->rc = rc; + o->dict_locals = mp_locals_get(); // this is a wrong! how to do this properly? + o->lnotab = MP_OBJ_NULL; + return MP_OBJ_FROM_PTR(o); +} + +#endif diff --git a/py/objcode.h b/py/objcode.h new file mode 100644 index 0000000000000..8db9a34b6e1c9 --- /dev/null +++ b/py/objcode.h @@ -0,0 +1,99 @@ +/* + * This file is part of the MicroPython project, http://micropython.org/ + * + * The MIT License (MIT) + * + * Copyright (c) 2025 Damien P. George + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ +#ifndef MICROPY_INCLUDED_PY_OBJCODE_H +#define MICROPY_INCLUDED_PY_OBJCODE_H + +#include "py/bc.h" + +#if MICROPY_PY_BUILTINS_CODE == MICROPY_PY_BUILTINS_CODE_NONE + +// Code object not implemented at this configuration level. + +#elif MICROPY_PY_BUILTINS_CODE <= MICROPY_PY_BUILTINS_CODE_MINIMUM + +typedef struct _mp_obj_code_t { + mp_obj_base_t base; + mp_obj_t module_fun; +} mp_obj_code_t; + +static inline mp_obj_t mp_obj_new_code(mp_obj_t module_fun) { + mp_obj_code_t *code = mp_obj_malloc(mp_obj_code_t, &mp_type_code); + code->module_fun = module_fun; + return MP_OBJ_FROM_PTR(code); +} + +#elif MICROPY_PY_BUILTINS_CODE <= MICROPY_PY_BUILTINS_CODE_BASIC + +typedef struct _mp_obj_code_t { + mp_obj_base_t base; + mp_module_constants_t constants; + const void *proto_fun; +} mp_obj_code_t; + +static inline mp_obj_t mp_obj_new_code(const mp_module_constants_t constants, const void *proto_fun) { + mp_obj_code_t *code = mp_obj_malloc(mp_obj_code_t, &mp_type_code); + code->constants = constants; + code->proto_fun = proto_fun; + return MP_OBJ_FROM_PTR(code); +} + +static inline const mp_module_constants_t *mp_code_get_constants(mp_obj_code_t *self) { + return &self->constants; +} + +static inline const void *mp_code_get_proto_fun(mp_obj_code_t *self) { + return self->proto_fun; +} + +#elif MICROPY_PY_BUILTINS_CODE <= MICROPY_PY_BUILTINS_CODE_FULL + +#include "py/emitglue.h" + +#define MP_CODE_QSTR_MAP(context, idx) (context->constants.qstr_table[idx]) + +typedef struct _mp_obj_code_t { + // TODO this was 4 words + mp_obj_base_t base; + const mp_module_context_t *context; + const mp_raw_code_t *rc; + mp_obj_dict_t *dict_locals; + mp_obj_t lnotab; +} mp_obj_code_t; + +mp_obj_t mp_obj_new_code(const mp_module_context_t *context, const mp_raw_code_t *rc, bool result_required); + +static inline const mp_module_constants_t *mp_code_get_constants(mp_obj_code_t *self) { + return &self->context->constants; +} + +static inline const void *mp_code_get_proto_fun(mp_obj_code_t *self) { + // A mp_raw_code_t is always a proto_fun (but not the other way around). + return self->rc; +} + +#endif + +#endif // MICROPY_INCLUDED_PY_OBJCODE_H diff --git a/py/objfun.c b/py/objfun.c index 6e635a43418b3..e6a923d59e886 100644 --- a/py/objfun.c +++ b/py/objfun.c @@ -28,6 +28,8 @@ #include #include +#include "py/emitglue.h" +#include "py/objcode.h" #include "py/objtuple.h" #include "py/objfun.h" #include "py/runtime.h" @@ -154,6 +156,30 @@ qstr mp_obj_fun_get_name(mp_const_obj_t fun_in) { return name; } +#if MICROPY_PY_FUNCTION_ATTRS_CODE +static mp_obj_t fun_bc_make_new(const mp_obj_type_t *type, size_t n_args, size_t n_kw, const mp_obj_t *args) { + (void)type; + mp_arg_check_num(n_args, n_kw, 2, 2, false); + + if (!mp_obj_is_type(args[0], &mp_type_code)) { + mp_raise_TypeError(NULL); + } + if (!mp_obj_is_type(args[1], &mp_type_dict)) { + mp_raise_TypeError(NULL); + } + + mp_obj_code_t *code = MP_OBJ_TO_PTR(args[0]); + mp_obj_t globals = args[1]; + + mp_module_context_t *module_context = m_new_obj(mp_module_context_t); + module_context->module.base.type = &mp_type_module; + module_context->module.globals = MP_OBJ_TO_PTR(globals); + module_context->constants = *mp_code_get_constants(code); + + return mp_make_function_from_proto_fun(mp_code_get_proto_fun(code), module_context, NULL); +} +#endif + #if MICROPY_CPYTHON_COMPAT static void fun_bc_print(const mp_print_t *print, mp_obj_t o_in, mp_print_kind_t kind) { (void)kind; @@ -344,9 +370,29 @@ void mp_obj_fun_bc_attr(mp_obj_t self_in, qstr attr, mp_obj_t *dest) { mp_obj_fun_bc_t *self = MP_OBJ_TO_PTR(self_in); dest[0] = MP_OBJ_FROM_PTR(self->context->module.globals); } + #if MICROPY_PY_FUNCTION_ATTRS_CODE + if (attr == MP_QSTR___code__) { + const mp_obj_fun_bc_t *self = MP_OBJ_TO_PTR(self_in); + if ((self->base.type == &mp_type_fun_bc + || self->base.type == &mp_type_gen_wrap) + && self->child_table == NULL) { + #if MICROPY_PY_BUILTINS_CODE <= MICROPY_PY_BUILTINS_CODE_BASIC + dest[0] = mp_obj_new_code(self->context->constants, self->bytecode); + #else + dest[0] = mp_obj_new_code(self->context, self->rc, true); + #endif + } + } + #endif } #endif +#if MICROPY_PY_FUNCTION_ATTRS_CODE +#define FUN_BC_MAKE_NEW make_new, fun_bc_make_new, +#else +#define FUN_BC_MAKE_NEW +#endif + #if MICROPY_CPYTHON_COMPAT #define FUN_BC_TYPE_PRINT print, fun_bc_print, #else @@ -363,6 +409,7 @@ MP_DEFINE_CONST_OBJ_TYPE( mp_type_fun_bc, MP_QSTR_function, MP_TYPE_FLAG_BINDS_SELF, + FUN_BC_MAKE_NEW FUN_BC_TYPE_PRINT FUN_BC_TYPE_ATTR call, fun_bc_call diff --git a/py/objint.c b/py/objint.c index bf5b677df3be8..b12f09c9d38a7 100644 --- a/py/objint.c +++ b/py/objint.c @@ -55,7 +55,7 @@ static mp_obj_t mp_obj_int_make_new(const mp_obj_type_t *type_in, size_t n_args, return o; } else if (mp_get_buffer(args[0], &bufinfo, MP_BUFFER_READ)) { // a textual representation, parse it - return mp_parse_num_integer(bufinfo.buf, bufinfo.len, 0, NULL); + return mp_parse_num_integer(bufinfo.buf, bufinfo.len, 10, NULL); #if MICROPY_PY_BUILTINS_FLOAT } else if (mp_obj_is_float(args[0])) { return mp_obj_new_int_from_float(mp_obj_float_get(args[0])); diff --git a/py/objstr.c b/py/objstr.c index d1e0d58b6ef10..1874cdb01d03d 100644 --- a/py/objstr.c +++ b/py/objstr.c @@ -74,6 +74,26 @@ static void check_is_str_or_bytes(mp_obj_t self_in) { mp_check_self(mp_obj_is_str_or_bytes(self_in)); } +static const byte *get_substring_data(const mp_obj_t obj, size_t n_args, const mp_obj_t *args, size_t *len) { + // Get substring data from obj, using args[0,1] to specify start and end indices. + GET_STR_DATA_LEN(obj, str, str_len); + if (n_args > 0) { + const mp_obj_type_t *self_type = mp_obj_get_type(obj); + const byte *end = str + str_len; + if (n_args > 1 && args[1] != mp_const_none) { + end = str_index_to_ptr(self_type, str, str_len, args[1], true); + } + if (args[0] != mp_const_none) { + str = str_index_to_ptr(self_type, str, str_len, args[0], true); + } + str_len = MAX(end - str, 0); + } + if (len) { + *len = str_len; + } + return str; +} + /******************************************************************************/ /* str */ @@ -819,37 +839,34 @@ static mp_obj_t str_rindex(size_t n_args, const mp_obj_t *args) { } MP_DEFINE_CONST_FUN_OBJ_VAR_BETWEEN(str_rindex_obj, 2, 4, str_rindex); -// TODO: (Much) more variety in args -static mp_obj_t str_startswith(size_t n_args, const mp_obj_t *args) { - const mp_obj_type_t *self_type = mp_obj_get_type(args[0]); - GET_STR_DATA_LEN(args[0], str, str_len); - size_t prefix_len; - const char *prefix = mp_obj_str_get_data(args[1], &prefix_len); - const byte *start = str; - if (n_args > 2) { - start = str_index_to_ptr(self_type, str, str_len, args[2], true); +static mp_obj_t str_startendswith(size_t n_args, const mp_obj_t *args, bool ends_with) { + size_t str_len; + const byte *str = get_substring_data(args[0], n_args - 2, args + 2, &str_len); + mp_obj_t *prefixes = (mp_obj_t *)&args[1]; + size_t n_prefixes = 1; + if (mp_obj_is_type(args[1], &mp_type_tuple)) { + mp_obj_tuple_get(args[1], &n_prefixes, &prefixes); } - if (prefix_len + (start - str) > str_len) { - return mp_const_false; + size_t prefix_len; + for (size_t i = 0; i < n_prefixes; i++) { + const char *prefix = mp_obj_str_get_data(prefixes[i], &prefix_len); + const byte *s = str + (ends_with ? str_len - prefix_len : 0); + if (prefix_len <= str_len && memcmp(s, prefix, prefix_len) == 0) { + return mp_const_true; + } } - return mp_obj_new_bool(memcmp(start, prefix, prefix_len) == 0); + return mp_const_false; } -MP_DEFINE_CONST_FUN_OBJ_VAR_BETWEEN(str_startswith_obj, 2, 3, str_startswith); -static mp_obj_t str_endswith(size_t n_args, const mp_obj_t *args) { - GET_STR_DATA_LEN(args[0], str, str_len); - size_t suffix_len; - const char *suffix = mp_obj_str_get_data(args[1], &suffix_len); - if (n_args > 2) { - mp_raise_NotImplementedError(MP_ERROR_TEXT("start/end indices")); - } +static mp_obj_t str_startswith(size_t n_args, const mp_obj_t *args) { + return str_startendswith(n_args, args, false); +} +MP_DEFINE_CONST_FUN_OBJ_VAR_BETWEEN(str_startswith_obj, 2, 4, str_startswith); - if (suffix_len > str_len) { - return mp_const_false; - } - return mp_obj_new_bool(memcmp(str + (str_len - suffix_len), suffix, suffix_len) == 0); +static mp_obj_t str_endswith(size_t n_args, const mp_obj_t *args) { + return str_startendswith(n_args, args, true); } -MP_DEFINE_CONST_FUN_OBJ_VAR_BETWEEN(str_endswith_obj, 2, 3, str_endswith); +MP_DEFINE_CONST_FUN_OBJ_VAR_BETWEEN(str_endswith_obj, 2, 4, str_endswith); enum { LSTRIP, RSTRIP, STRIP }; diff --git a/py/parsenum.c b/py/parsenum.c index 67ac12d190728..b7179acdf36ae 100644 --- a/py/parsenum.c +++ b/py/parsenum.c @@ -152,13 +152,13 @@ mp_obj_t mp_parse_num_integer(const char *restrict str_, size_t len, int base, m raise_exc(exc, lex); #elif MICROPY_ERROR_REPORTING == MICROPY_ERROR_REPORTING_NORMAL mp_obj_t exc = mp_obj_new_exception_msg_varg(&mp_type_ValueError, - MP_ERROR_TEXT("invalid syntax for integer with base %d"), base); + MP_ERROR_TEXT("invalid syntax for integer with base %d"), base == 1 ? 0 : base); raise_exc(exc, lex); #else vstr_t vstr; mp_print_t print; vstr_init_print(&vstr, 50, &print); - mp_printf(&print, "invalid syntax for integer with base %d: ", base); + mp_printf(&print, "invalid syntax for integer with base %d: ", base == 1 ? 0 : base); mp_str_print_quoted(&print, str_val_start, top - str_val_start, true); mp_obj_t exc = mp_obj_new_exception_arg1(&mp_type_ValueError, mp_obj_new_str_from_utf8_vstr(&vstr)); @@ -180,39 +180,40 @@ typedef enum { } parse_dec_in_t; #if MICROPY_PY_BUILTINS_FLOAT -// DEC_VAL_MAX only needs to be rough and is used to retain precision while not overflowing +// MANTISSA_MAX is used to retain precision while not overflowing mantissa // SMALL_NORMAL_VAL is the smallest power of 10 that is still a normal float // EXACT_POWER_OF_10 is the largest value of x so that 10^x can be stored exactly in a float // Note: EXACT_POWER_OF_10 is at least floor(log_5(2^mantissa_length)). Indeed, 10^n = 2^n * 5^n // so we only have to store the 5^n part in the mantissa (the 2^n part will go into the float's // exponent). #if MICROPY_FLOAT_IMPL == MICROPY_FLOAT_IMPL_FLOAT -#define DEC_VAL_MAX 1e20F +#define MANTISSA_MAX 0x19999998U #define SMALL_NORMAL_VAL (1e-37F) #define SMALL_NORMAL_EXP (-37) #define EXACT_POWER_OF_10 (9) #elif MICROPY_FLOAT_IMPL == MICROPY_FLOAT_IMPL_DOUBLE -#define DEC_VAL_MAX 1e200 +#define MANTISSA_MAX 0x1999999999999998ULL #define SMALL_NORMAL_VAL (1e-307) #define SMALL_NORMAL_EXP (-307) #define EXACT_POWER_OF_10 (22) #endif // Break out inner digit accumulation routine to ease trailing zero deferral. -static void accept_digit(mp_float_t *p_dec_val, int dig, int *p_exp_extra, int in) { +static mp_float_uint_t accept_digit(mp_float_uint_t p_mantissa, unsigned int dig, int *p_exp_extra, int in) { // Core routine to ingest an additional digit. - if (*p_dec_val < DEC_VAL_MAX) { + if (p_mantissa < MANTISSA_MAX) { // dec_val won't overflow so keep accumulating - *p_dec_val = 10 * *p_dec_val + dig; if (in == PARSE_DEC_IN_FRAC) { --(*p_exp_extra); } + return 10u * p_mantissa + dig; } else { // dec_val might overflow and we anyway can't represent more digits // of precision, so ignore the digit and just adjust the exponent if (in == PARSE_DEC_IN_INTG) { ++(*p_exp_extra); } + return p_mantissa; } } #endif // MICROPY_PY_BUILTINS_FLOAT @@ -274,6 +275,7 @@ mp_obj_t mp_parse_num_float(const char *str, size_t len, bool allow_imag, mp_lex // string should be a decimal number parse_dec_in_t in = PARSE_DEC_IN_INTG; bool exp_neg = false; + mp_float_uint_t mantissa = 0; int exp_val = 0; int exp_extra = 0; int trailing_zeros_intg = 0, trailing_zeros_frac = 0; @@ -289,9 +291,9 @@ mp_obj_t mp_parse_num_float(const char *str, size_t len, bool allow_imag, mp_lex exp_val = 10 * exp_val + dig; } } else { - if (dig == 0 || dec_val >= DEC_VAL_MAX) { + if (dig == 0 || mantissa >= MANTISSA_MAX) { // Defer treatment of zeros in fractional part. If nothing comes afterwards, ignore them. - // Also, once we reach DEC_VAL_MAX, treat every additional digit as a trailing zero. + // Also, once we reach MANTISSA_MAX, treat every additional digit as a trailing zero. if (in == PARSE_DEC_IN_INTG) { ++trailing_zeros_intg; } else { @@ -300,14 +302,14 @@ mp_obj_t mp_parse_num_float(const char *str, size_t len, bool allow_imag, mp_lex } else { // Time to un-defer any trailing zeros. Intg zeros first. while (trailing_zeros_intg) { - accept_digit(&dec_val, 0, &exp_extra, PARSE_DEC_IN_INTG); + mantissa = accept_digit(mantissa, 0, &exp_extra, PARSE_DEC_IN_INTG); --trailing_zeros_intg; } while (trailing_zeros_frac) { - accept_digit(&dec_val, 0, &exp_extra, PARSE_DEC_IN_FRAC); + mantissa = accept_digit(mantissa, 0, &exp_extra, PARSE_DEC_IN_FRAC); --trailing_zeros_frac; } - accept_digit(&dec_val, dig, &exp_extra, in); + mantissa = accept_digit(mantissa, dig, &exp_extra, in); } } } else if (in == PARSE_DEC_IN_INTG && dig == '.') { @@ -341,6 +343,7 @@ mp_obj_t mp_parse_num_float(const char *str, size_t len, bool allow_imag, mp_lex // apply the exponent, making sure it's not a subnormal value exp_val += exp_extra + trailing_zeros_intg; + dec_val = (mp_float_t)mantissa; if (exp_val < SMALL_NORMAL_EXP) { exp_val -= SMALL_NORMAL_EXP; dec_val *= SMALL_NORMAL_VAL; diff --git a/py/parsenumbase.c b/py/parsenumbase.c index 94523a666d325..fbf07a119584f 100644 --- a/py/parsenumbase.c +++ b/py/parsenumbase.c @@ -30,35 +30,28 @@ // find real radix base, and strip preceding '0x', '0o' and '0b' // puts base in *base, and returns number of bytes to skip the prefix +// in base-0, puts 1 in *base to indicate a number that starts with 0, to provoke a +// ValueError if it's not all-digits-zero. size_t mp_parse_num_base(const char *str, size_t len, int *base) { const byte *p = (const byte *)str; if (len <= 1) { goto no_prefix; } unichar c = *(p++); - if ((*base == 0 || *base == 16) && c == '0') { - c = *(p++); - if ((c | 32) == 'x') { + if (c == '0') { + c = *(p++) | 32; + int b = *base; + if (c == 'x' && (b == 0 || b == 16)) { *base = 16; - } else if (*base == 0 && (c | 32) == 'o') { + } else if (c == 'o' && (b == 0 || b == 8)) { *base = 8; - } else if (*base == 0 && (c | 32) == 'b') { + } else if (c == 'b' && (b == 0 || b == 2)) { *base = 2; } else { - if (*base == 0) { - *base = 10; - } - p -= 2; - } - } else if (*base == 8 && c == '0') { - c = *(p++); - if ((c | 32) != 'o') { - p -= 2; - } - } else if (*base == 2 && c == '0') { - c = *(p++); - if ((c | 32) != 'b') { p -= 2; + if (b == 0) { + *base = 1; + } } } else { p--; diff --git a/py/persistentcode.c b/py/persistentcode.c index 692d8f5dd6cfe..1976729f4d879 100644 --- a/py/persistentcode.c +++ b/py/persistentcode.c @@ -188,6 +188,15 @@ static qstr load_qstr(mp_reader_t *reader) { return len >> 1; } len >>= 1; + + #if MICROPY_VFS_ROM + // If possible, create the qstr from the memory-mapped string data. + const uint8_t *memmap = mp_reader_try_read_rom(reader, len + 1); + if (memmap != NULL) { + return qstr_from_strn_static((const char *)memmap, len); + } + #endif + char *str = m_new(char, len); read_bytes(reader, (byte *)str, len); read_byte(reader); // read and discard null terminator @@ -196,6 +205,24 @@ static qstr load_qstr(mp_reader_t *reader) { return qst; } +#if MICROPY_VFS_ROM +// Create a str/bytes object that can forever reference the given data. +static mp_obj_t mp_obj_new_str_static(const mp_obj_type_t *type, const byte *data, size_t len) { + if (type == &mp_type_str) { + qstr q = qstr_find_strn((const char *)data, len); + if (q != MP_QSTRnull) { + return MP_OBJ_NEW_QSTR(q); + } + } + assert(data[len] == '\0'); + mp_obj_str_t *o = mp_obj_malloc(mp_obj_str_t, type); + o->len = len; + o->hash = qstr_compute_hash(data, len); + o->data = data; + return MP_OBJ_FROM_PTR(o); +} +#endif + static mp_obj_t load_obj(mp_reader_t *reader) { byte obj_type = read_byte(reader); #if MICROPY_EMIT_MACHINE_CODE @@ -213,6 +240,8 @@ static mp_obj_t load_obj(mp_reader_t *reader) { return MP_OBJ_FROM_PTR(&mp_const_ellipsis_obj); } else { size_t len = read_uint(reader); + + // Handle empty bytes object, and tuple objects. if (len == 0 && obj_type == MP_PERSISTENT_OBJ_BYTES) { read_byte(reader); // skip null terminator return mp_const_empty_bytes; @@ -223,11 +252,31 @@ static mp_obj_t load_obj(mp_reader_t *reader) { } return MP_OBJ_FROM_PTR(tuple); } + + // Read in the object's data, either from ROM or into RAM. + const uint8_t *memmap = NULL; vstr_t vstr; - vstr_init_len(&vstr, len); - read_bytes(reader, (byte *)vstr.buf, len); + #if MICROPY_VFS_ROM + memmap = mp_reader_try_read_rom(reader, len); + vstr.buf = (void *)memmap; + vstr.len = len; + #endif + if (memmap == NULL) { + // Data could not be memory-mapped, so allocate it in RAM and read it in. + vstr_init_len(&vstr, len); + read_bytes(reader, (byte *)vstr.buf, len); + } + + // Create and return the object. if (obj_type == MP_PERSISTENT_OBJ_STR || obj_type == MP_PERSISTENT_OBJ_BYTES) { - read_byte(reader); // skip null terminator + read_byte(reader); // skip null terminator (it needs to be there for ROM str objects) + #if MICROPY_VFS_ROM + if (memmap != NULL) { + // Create a str/bytes that references the memory-mapped data. + const mp_obj_type_t *t = obj_type == MP_PERSISTENT_OBJ_STR ? &mp_type_str : &mp_type_bytes; + return mp_obj_new_str_static(t, memmap, len); + } + #endif if (obj_type == MP_PERSISTENT_OBJ_STR) { return mp_obj_new_str_from_utf8_vstr(&vstr); } else { @@ -264,10 +313,17 @@ static mp_raw_code_t *load_raw_code(mp_reader_t *reader, mp_module_context_t *co #endif if (kind == MP_CODE_BYTECODE) { - // Allocate memory for the bytecode - fun_data = m_new(uint8_t, fun_data_len); - // Load bytecode - read_bytes(reader, fun_data, fun_data_len); + #if MICROPY_VFS_ROM + // Try to reference memory-mapped data for the bytecode. + fun_data = (uint8_t *)mp_reader_try_read_rom(reader, fun_data_len); + #endif + + if (fun_data == NULL) { + // Allocate memory for the bytecode. + fun_data = m_new(uint8_t, fun_data_len); + // Load bytecode. + read_bytes(reader, fun_data, fun_data_len); + } #if MICROPY_EMIT_MACHINE_CODE } else { @@ -353,7 +409,7 @@ static mp_raw_code_t *load_raw_code(mp_reader_t *reader, mp_module_context_t *co #if MICROPY_EMIT_MACHINE_CODE } else { - const uint8_t *prelude_ptr; + const uint8_t *prelude_ptr = NULL; #if MICROPY_EMIT_NATIVE_PRELUDE_SEPARATE_FROM_MACHINE_CODE if (kind == MP_CODE_NATIVE_PY) { // Executable code cannot be accessed byte-wise on this architecture, so copy @@ -478,7 +534,7 @@ void mp_raw_code_load_file(qstr filename, mp_compiled_module_t *context) { #endif // MICROPY_PERSISTENT_CODE_LOAD -#if MICROPY_PERSISTENT_CODE_SAVE +#if MICROPY_PERSISTENT_CODE_SAVE || MICROPY_PERSISTENT_CODE_SAVE_FUN #include "py/objstr.h" @@ -576,6 +632,10 @@ static void save_obj(mp_print_t *print, mp_obj_t o) { } } +#endif // MICROPY_PERSISTENT_CODE_SAVE || MICROPY_PERSISTENT_CODE_SAVE_FUN + +#if MICROPY_PERSISTENT_CODE_SAVE + static void save_raw_code(mp_print_t *print, const mp_raw_code_t *rc) { // Save function kind and data length mp_print_uint(print, (rc->fun_data_len << 3) | ((rc->n_children != 0) << 2) | (rc->kind - MP_CODE_BYTECODE)); @@ -646,6 +706,8 @@ void mp_raw_code_save(mp_compiled_module_t *cm, mp_print_t *print) { save_raw_code(print, cm->rc); } +#endif // MICROPY_PERSISTENT_CODE_SAVE + #if MICROPY_PERSISTENT_CODE_SAVE_FILE #include @@ -676,4 +738,184 @@ void mp_raw_code_save_file(mp_compiled_module_t *cm, qstr filename) { #endif // MICROPY_PERSISTENT_CODE_SAVE_FILE -#endif // MICROPY_PERSISTENT_CODE_SAVE +#if MICROPY_PERSISTENT_CODE_SAVE_FUN + +#include "py/bc0.h" +#include "py/objfun.h" +#include "py/smallint.h" +#include "py/gc.h" + +#define MP_BC_OPCODE_HAS_SIGNED_OFFSET(opcode) (MP_BC_UNWIND_JUMP <= (opcode) && (opcode) <= MP_BC_POP_JUMP_IF_FALSE) + +typedef struct _bit_vector_t { + size_t max_bit_set; + size_t alloc; + uintptr_t *bits; +} bit_vector_t; + +static void bit_vector_init(bit_vector_t *self) { + self->max_bit_set = 0; + self->alloc = 1; + self->bits = m_new(uintptr_t, self->alloc); +} + +static void bit_vector_clear(bit_vector_t *self) { + m_del(uintptr_t, self->bits, self->alloc); +} + +static bool bit_vector_is_set(bit_vector_t *self, size_t index) { + const size_t bits_size = sizeof(*self->bits) * MP_BITS_PER_BYTE; + return index / bits_size < self->alloc + && (self->bits[index / bits_size] & (1 << (index % bits_size))) != 0; +} + +static void bit_vector_set(bit_vector_t *self, size_t index) { + const size_t bits_size = sizeof(*self->bits) * MP_BITS_PER_BYTE; + self->max_bit_set = MAX(self->max_bit_set, index); + if (index / bits_size >= self->alloc) { + size_t new_alloc = self->alloc * 2; + self->bits = m_renew(uintptr_t, self->bits, self->alloc, new_alloc); + self->alloc = new_alloc; + } + self->bits[index / bits_size] |= 1 << (index % bits_size); +} + +typedef struct _mp_opcode_t { + uint8_t opcode; + uint8_t format; + uint8_t size; + mp_int_t arg; + uint8_t extra_arg; +} mp_opcode_t; + +static mp_opcode_t mp_opcode_decode(const uint8_t *ip) { + const uint8_t *ip_start = ip; + uint8_t opcode = *ip++; + uint8_t opcode_format = MP_BC_FORMAT(opcode); + mp_uint_t arg = 0; + uint8_t extra_arg = 0; + if (opcode_format == MP_BC_FORMAT_QSTR || opcode_format == MP_BC_FORMAT_VAR_UINT) { + arg = *ip & 0x7f; + if (opcode == MP_BC_LOAD_CONST_SMALL_INT && (arg & 0x40) != 0) { + arg |= (mp_uint_t)(-1) << 7; + } + while ((*ip & 0x80) != 0) { + arg = (arg << 7) | (*++ip & 0x7f); + } + ++ip; + } else if (opcode_format == MP_BC_FORMAT_OFFSET) { + if ((*ip & 0x80) == 0) { + arg = *ip++; + if (MP_BC_OPCODE_HAS_SIGNED_OFFSET(opcode)) { + arg -= 0x40; + } + } else { + arg = (ip[0] & 0x7f) | (ip[1] << 7); + ip += 2; + if (MP_BC_OPCODE_HAS_SIGNED_OFFSET(opcode)) { + arg -= 0x4000; + } + } + } + if ((opcode & MP_BC_MASK_EXTRA_BYTE) == 0) { + extra_arg = *ip++; + } + + mp_opcode_t op = { opcode, opcode_format, ip - ip_start, arg, extra_arg }; + return op; +} + +mp_obj_t mp_raw_code_save_fun_to_bytes(const mp_module_constants_t *consts, const uint8_t *bytecode) { + const uint8_t *fun_data = bytecode; + const uint8_t *fun_data_top = fun_data + gc_nbytes(fun_data); + + // Extract function information. + const byte *ip = fun_data; + MP_BC_PRELUDE_SIG_DECODE(ip); + MP_BC_PRELUDE_SIZE_DECODE(ip); + + // Track the qstrs used by the function. + bit_vector_t qstr_table_used; + bit_vector_init(&qstr_table_used); + + // Track the objects used by the function. + bit_vector_t obj_table_used; + bit_vector_init(&obj_table_used); + + const byte *ip_names = ip; + mp_uint_t simple_name = mp_decode_uint(&ip_names); + bit_vector_set(&qstr_table_used, simple_name); + for (size_t i = 0; i < n_pos_args + n_kwonly_args; ++i) { + mp_uint_t arg_name = mp_decode_uint(&ip_names); + bit_vector_set(&qstr_table_used, arg_name); + } + + // Skip pass source code info and cell info. + // Then ip points to the start of the opcodes. + ip += n_info + n_cell; + + // Decode bytecode. + while (ip < fun_data_top) { + mp_opcode_t op = mp_opcode_decode(ip); + if (op.opcode == MP_BC_BASE_RESERVED) { + // End of opcodes. + fun_data_top = ip; + } else if (op.opcode == MP_BC_LOAD_CONST_OBJ) { + bit_vector_set(&obj_table_used, op.arg); + } else if (op.format == MP_BC_FORMAT_QSTR) { + bit_vector_set(&qstr_table_used, op.arg); + } + ip += op.size; + } + + mp_uint_t fun_data_len = fun_data_top - fun_data; + + mp_print_t print; + vstr_t vstr; + vstr_init_print(&vstr, 64, &print); + + // Start with .mpy header. + const uint8_t header[4] = { 'M', MPY_VERSION, 0, MP_SMALL_INT_BITS }; + mp_print_bytes(&print, header, sizeof(header)); + + // Number of entries in constant table. + mp_print_uint(&print, qstr_table_used.max_bit_set + 1); + mp_print_uint(&print, obj_table_used.max_bit_set + 1); + + // Save qstrs. + for (size_t i = 0; i <= qstr_table_used.max_bit_set; ++i) { + if (bit_vector_is_set(&qstr_table_used, i)) { + save_qstr(&print, consts->qstr_table[i]); + } else { + save_qstr(&print, MP_QSTR_); + } + } + + // Save constant objects. + for (size_t i = 0; i <= obj_table_used.max_bit_set; ++i) { + if (bit_vector_is_set(&obj_table_used, i)) { + save_obj(&print, consts->obj_table[i]); + } else { + save_obj(&print, mp_const_none); + } + } + + bit_vector_clear(&qstr_table_used); + bit_vector_clear(&obj_table_used); + + // Save function kind and data length. + mp_print_uint(&print, fun_data_len << 3); + + // Save function code. + mp_print_bytes(&print, fun_data, fun_data_len); + + // Create and return bytes representing the .mpy data. + return mp_obj_new_bytes_from_vstr(&vstr); +} + +#endif // MICROPY_PERSISTENT_CODE_SAVE_FUN + +#if MICROPY_PERSISTENT_CODE_TRACK_RELOC_CODE +// An mp_obj_list_t that tracks relocated native code to prevent the GC from reclaiming them. +MP_REGISTER_ROOT_POINTER(mp_obj_t track_reloc_code_list); +#endif diff --git a/py/persistentcode.h b/py/persistentcode.h index f0b7f70f7d7fa..46b474e57f707 100644 --- a/py/persistentcode.h +++ b/py/persistentcode.h @@ -30,6 +30,11 @@ #include "py/reader.h" #include "py/emitglue.h" +// CIRCUITPY-CHANGE: Avoid undefined warnings +#ifndef MICROPY_PERSISTENT_CODE_TRACK_RELOC_CODE +#define MICROPY_PERSISTENT_CODE_TRACK_RELOC_CODE (0) +#endif + // The current version of .mpy files. A bytecode-only .mpy file can be loaded // as long as MPY_VERSION matches, but a native .mpy (i.e. one with an arch // set) must also match MPY_SUB_VERSION. This allows 3 additional updates to @@ -121,6 +126,7 @@ void mp_raw_code_load_file(qstr filename, mp_compiled_module_t *ctx); void mp_raw_code_save(mp_compiled_module_t *cm, mp_print_t *print); void mp_raw_code_save_file(mp_compiled_module_t *cm, qstr filename); +mp_obj_t mp_raw_code_save_fun_to_bytes(const mp_module_constants_t *consts, const uint8_t *bytecode); void mp_native_relocate(void *reloc, uint8_t *text, uintptr_t reloc_text); diff --git a/py/profile.c b/py/profile.c index 92f414ace7c92..397d0291f9fad 100644 --- a/py/profile.c +++ b/py/profile.c @@ -38,9 +38,8 @@ #endif #define prof_trace_cb MP_STATE_THREAD(prof_trace_callback) -#define QSTR_MAP(context, idx) (context->constants.qstr_table[idx]) -static uint mp_prof_bytecode_lineno(const mp_raw_code_t *rc, size_t bc) { +uint mp_prof_bytecode_lineno(const mp_raw_code_t *rc, size_t bc) { const mp_bytecode_prelude_t *prelude = &rc->prelude; return mp_bytecode_get_source_line(prelude->line_info, prelude->line_info_top, bc); } @@ -68,137 +67,6 @@ void mp_prof_extract_prelude(const byte *bytecode, mp_bytecode_prelude_t *prelud prelude->line_info = ip; } -/******************************************************************************/ -// code object - -static void code_print(const mp_print_t *print, mp_obj_t o_in, mp_print_kind_t kind) { - (void)kind; - mp_obj_code_t *o = MP_OBJ_TO_PTR(o_in); - const mp_raw_code_t *rc = o->rc; - const mp_bytecode_prelude_t *prelude = &rc->prelude; - mp_printf(print, - "", - QSTR_MAP(o->context, prelude->qstr_block_name_idx), - o, - QSTR_MAP(o->context, 0), - rc->line_of_definition - ); -} - -static mp_obj_tuple_t *code_consts(const mp_module_context_t *context, const mp_raw_code_t *rc) { - mp_obj_tuple_t *consts = MP_OBJ_TO_PTR(mp_obj_new_tuple(rc->n_children + 1, NULL)); - - size_t const_no = 0; - for (size_t i = 0; i < rc->n_children; ++i) { - mp_obj_t code = mp_obj_new_code(context, rc->children[i]); - if (code == MP_OBJ_NULL) { - m_malloc_fail(sizeof(mp_obj_code_t)); - } - consts->items[const_no++] = code; - } - consts->items[const_no++] = mp_const_none; - - return consts; -} - -static mp_obj_t raw_code_lnotab(const mp_raw_code_t *rc) { - // const mp_bytecode_prelude_t *prelude = &rc->prelude; - uint start = 0; - uint stop = rc->fun_data_len - start; - - uint last_lineno = mp_prof_bytecode_lineno(rc, start); - uint lasti = 0; - - const uint buffer_chunk_size = (stop - start) >> 2; // heuristic magic - uint buffer_size = buffer_chunk_size; - byte *buffer = m_new(byte, buffer_size); - uint buffer_index = 0; - - for (uint i = start; i < stop; ++i) { - uint lineno = mp_prof_bytecode_lineno(rc, i); - size_t line_diff = lineno - last_lineno; - if (line_diff > 0) { - uint instr_diff = (i - start) - lasti; - - assert(instr_diff < 256); - assert(line_diff < 256); - - if (buffer_index + 2 > buffer_size) { - buffer = m_renew(byte, buffer, buffer_size, buffer_size + buffer_chunk_size); - buffer_size = buffer_size + buffer_chunk_size; - } - last_lineno = lineno; - lasti = i - start; - buffer[buffer_index++] = instr_diff; - buffer[buffer_index++] = line_diff; - } - } - - mp_obj_t o = mp_obj_new_bytes(buffer, buffer_index); - m_del(byte, buffer, buffer_size); - return o; -} - -static void code_attr(mp_obj_t self_in, qstr attr, mp_obj_t *dest) { - if (dest[0] != MP_OBJ_NULL) { - // not load attribute - return; - } - mp_obj_code_t *o = MP_OBJ_TO_PTR(self_in); - const mp_raw_code_t *rc = o->rc; - const mp_bytecode_prelude_t *prelude = &rc->prelude; - switch (attr) { - case MP_QSTR_co_code: - dest[0] = mp_obj_new_bytes( - (void *)prelude->opcodes, - rc->fun_data_len - (prelude->opcodes - (const byte *)rc->fun_data) - ); - break; - case MP_QSTR_co_consts: - dest[0] = MP_OBJ_FROM_PTR(code_consts(o->context, rc)); - break; - case MP_QSTR_co_filename: - dest[0] = MP_OBJ_NEW_QSTR(QSTR_MAP(o->context, 0)); - break; - case MP_QSTR_co_firstlineno: - dest[0] = MP_OBJ_NEW_SMALL_INT(mp_prof_bytecode_lineno(rc, 0)); - break; - case MP_QSTR_co_name: - dest[0] = MP_OBJ_NEW_QSTR(QSTR_MAP(o->context, prelude->qstr_block_name_idx)); - break; - case MP_QSTR_co_names: - dest[0] = MP_OBJ_FROM_PTR(o->dict_locals); - break; - case MP_QSTR_co_lnotab: - if (!o->lnotab) { - o->lnotab = raw_code_lnotab(rc); - } - dest[0] = o->lnotab; - break; - } -} - -MP_DEFINE_CONST_OBJ_TYPE( - mp_type_settrace_codeobj, - MP_QSTR_code, - MP_TYPE_FLAG_NONE, - print, code_print, - attr, code_attr - ); - -mp_obj_t mp_obj_new_code(const mp_module_context_t *context, const mp_raw_code_t *rc) { - mp_obj_code_t *o = m_new_obj_maybe(mp_obj_code_t); - if (o == NULL) { - return MP_OBJ_NULL; - } - o->base.type = &mp_type_settrace_codeobj; - o->context = context; - o->rc = rc; - o->dict_locals = mp_locals_get(); // this is a wrong! how to do this properly? - o->lnotab = MP_OBJ_NULL; - return MP_OBJ_FROM_PTR(o); -} - /******************************************************************************/ // frame object @@ -211,9 +79,9 @@ static void frame_print(const mp_print_t *print, mp_obj_t o_in, mp_print_kind_t mp_printf(print, "", frame, - QSTR_MAP(code->context, 0), + MP_CODE_QSTR_MAP(code->context, 0), frame->lineno, - QSTR_MAP(code->context, prelude->qstr_block_name_idx) + MP_CODE_QSTR_MAP(code->context, prelude->qstr_block_name_idx) ); } @@ -265,7 +133,7 @@ mp_obj_t mp_obj_new_frame(const mp_code_state_t *code_state) { return MP_OBJ_NULL; } - mp_obj_code_t *code = o->code = MP_OBJ_TO_PTR(mp_obj_new_code(code_state->fun_bc->context, code_state->fun_bc->rc)); + mp_obj_code_t *code = o->code = MP_OBJ_TO_PTR(mp_obj_new_code(code_state->fun_bc->context, code_state->fun_bc->rc, false)); if (code == NULL) { return MP_OBJ_NULL; } diff --git a/py/profile.h b/py/profile.h index 7f3f914034623..db72b9f076818 100644 --- a/py/profile.h +++ b/py/profile.h @@ -28,20 +28,12 @@ #define MICROPY_INCLUDED_PY_PROFILING_H #include "py/emitglue.h" +#include "py/objcode.h" #if MICROPY_PY_SYS_SETTRACE #define mp_prof_is_executing MP_STATE_THREAD(prof_callback_is_executing) -typedef struct _mp_obj_code_t { - // TODO this was 4 words - mp_obj_base_t base; - const mp_module_context_t *context; - const mp_raw_code_t *rc; - mp_obj_dict_t *dict_locals; - mp_obj_t lnotab; -} mp_obj_code_t; - typedef struct _mp_obj_frame_t { mp_obj_base_t base; const mp_code_state_t *code_state; @@ -53,9 +45,9 @@ typedef struct _mp_obj_frame_t { bool trace_opcodes; } mp_obj_frame_t; +uint mp_prof_bytecode_lineno(const mp_raw_code_t *rc, size_t bc); void mp_prof_extract_prelude(const byte *bytecode, mp_bytecode_prelude_t *prelude); -mp_obj_t mp_obj_new_code(const mp_module_context_t *mc, const mp_raw_code_t *rc); mp_obj_t mp_obj_new_frame(const mp_code_state_t *code_state); // This is the implementation for the sys.settrace diff --git a/py/py.cmake b/py/py.cmake index d7f4ffe6df62b..6c180ae53e6f2 100644 --- a/py/py.cmake +++ b/py/py.cmake @@ -24,6 +24,7 @@ set(MICROPY_SOURCE_PY ${MICROPY_PY_DIR}/emitbc.c ${MICROPY_PY_DIR}/emitcommon.c ${MICROPY_PY_DIR}/emitglue.c + ${MICROPY_PY_DIR}/emitinlinerv32.c ${MICROPY_PY_DIR}/emitinlinethumb.c ${MICROPY_PY_DIR}/emitinlinextensa.c ${MICROPY_PY_DIR}/emitnarm.c @@ -71,6 +72,7 @@ set(MICROPY_SOURCE_PY ${MICROPY_PY_DIR}/objattrtuple.c ${MICROPY_PY_DIR}/objbool.c ${MICROPY_PY_DIR}/objboundmeth.c + ${MICROPY_PY_DIR}/objcode.c ${MICROPY_PY_DIR}/objcell.c ${MICROPY_PY_DIR}/objclosure.c ${MICROPY_PY_DIR}/objcomplex.c diff --git a/py/py.mk b/py/py.mk index 67c51bff9f971..c05327bf81d55 100644 --- a/py/py.mk +++ b/py/py.mk @@ -36,6 +36,9 @@ ifneq ($(USER_C_MODULES),) # pre-define USERMOD variables as expanded so that variables are immediate # expanded as they're added to them +# Confirm the provided path exists, show abspath if not to make it clearer to fix. +$(if $(wildcard $(USER_C_MODULES)/.),,$(error USER_C_MODULES doesn't exist: $(abspath $(USER_C_MODULES)))) + # C/C++ files that are included in the QSTR/module build SRC_USERMOD_C := SRC_USERMOD_CXX := @@ -137,6 +140,7 @@ PY_CORE_O_BASENAME = $(addprefix py/,\ emitnxtensawin.o \ asmrv32.o \ emitnrv32.o \ + emitinlinerv32.o \ emitndebug.o \ formatfloat.o \ parsenumbase.o \ @@ -163,6 +167,7 @@ PY_CORE_O_BASENAME = $(addprefix py/,\ objboundmeth.o \ objcell.o \ objclosure.o \ + objcode.o \ objcomplex.o \ objdeque.o \ objdict.o \ diff --git a/py/qstr.c b/py/qstr.c index b8bf7360e9271..ab9ff4dd67db1 100644 --- a/py/qstr.c +++ b/py/qstr.c @@ -338,7 +338,7 @@ qstr qstr_from_str(const char *str) { return qstr_from_strn(str, strlen(str)); } -qstr qstr_from_strn(const char *str, size_t len) { +static qstr qstr_from_strn_helper(const char *str, size_t len, bool data_is_static) { QSTR_ENTER(); qstr q = qstr_find_strn(str, len); if (q == 0) { @@ -350,6 +350,12 @@ qstr qstr_from_strn(const char *str, size_t len) { mp_raise_msg(&mp_type_RuntimeError, MP_ERROR_TEXT("name too long")); } + if (data_is_static) { + // Given string data will be forever available so use it directly. + assert(str[len] == '\0'); + goto add; + } + // compute number of bytes needed to intern this string size_t n_bytes = len + 1; @@ -394,12 +400,26 @@ qstr qstr_from_strn(const char *str, size_t len) { // store the interned strings' data memcpy(q_ptr, str, len); q_ptr[len] = '\0'; - q = qstr_add(len, q_ptr); + str = q_ptr; + + add: + q = qstr_add(len, str); } QSTR_EXIT(); return q; } +qstr qstr_from_strn(const char *str, size_t len) { + return qstr_from_strn_helper(str, len, false); +} + +#if MICROPY_VFS_ROM +// Create a new qstr that can forever reference the given string data. +qstr qstr_from_strn_static(const char *str, size_t len) { + return qstr_from_strn_helper(str, len, true); +} +#endif + mp_uint_t qstr_hash(qstr q) { const qstr_pool_t *pool = find_qstr(&q); #if MICROPY_QSTR_BYTES_IN_HASH diff --git a/py/qstr.h b/py/qstr.h index 967990beea398..7dc04a8accb39 100644 --- a/py/qstr.h +++ b/py/qstr.h @@ -107,6 +107,9 @@ qstr qstr_find_strn(const char *str, size_t str_len); // returns MP_QSTRnull if qstr qstr_from_str(const char *str); qstr qstr_from_strn(const char *str, size_t len); +#if MICROPY_VFS_ROM +qstr qstr_from_strn_static(const char *str, size_t len); +#endif mp_uint_t qstr_hash(qstr q); const char *qstr_str(qstr q); diff --git a/py/reader.c b/py/reader.c index 151e04cac2c71..8feb6d75275f1 100644 --- a/py/reader.c +++ b/py/reader.c @@ -50,7 +50,7 @@ static mp_uint_t mp_reader_mem_readbyte(void *data) { static void mp_reader_mem_close(void *data) { mp_reader_mem_t *reader = (mp_reader_mem_t *)data; - if (reader->free_len > 0) { + if (reader->free_len > 0 && reader->free_len != MP_READER_IS_ROM) { m_del(char, (char *)reader->beg, reader->free_len); } m_del_obj(mp_reader_mem_t, reader); @@ -67,6 +67,19 @@ void mp_reader_new_mem(mp_reader_t *reader, const byte *buf, size_t len, size_t reader->close = mp_reader_mem_close; } +const uint8_t *mp_reader_try_read_rom(mp_reader_t *reader, size_t len) { + if (reader->readbyte != mp_reader_mem_readbyte) { + return NULL; + } + mp_reader_mem_t *m = reader->data; + if (m->free_len != MP_READER_IS_ROM) { + return NULL; + } + const uint8_t *data = m->cur; + m->cur += len; + return data; +} + #if MICROPY_READER_POSIX #include diff --git a/py/reader.h b/py/reader.h index 5cb1e67966c61..6378457007cb3 100644 --- a/py/reader.h +++ b/py/reader.h @@ -28,6 +28,10 @@ #include "py/obj.h" +// Pass to the `free_len` argument to `mp_reader_new_mem` to indicate that the data is in ROM. +// This means that the data is addressable and will remain valid at least until a soft reset. +#define MP_READER_IS_ROM ((size_t)-1) + // the readbyte function must return the next byte in the input stream // it must return MP_READER_EOF if end of stream // it can be called again after returning MP_READER_EOF, and in that case must return MP_READER_EOF @@ -43,4 +47,9 @@ void mp_reader_new_mem(mp_reader_t *reader, const byte *buf, size_t len, size_t void mp_reader_new_file(mp_reader_t *reader, qstr filename); void mp_reader_new_file_from_fd(mp_reader_t *reader, int fd, bool close_fd); +// Try to efficiently read the given number of bytes from a ROM-based reader. +// Returns a valid, non-NULL pointer to the requested data if the reader points to ROM. +// Returns NULL if the reader does not point to ROM. +const uint8_t *mp_reader_try_read_rom(mp_reader_t *reader, size_t len); + #endif // MICROPY_INCLUDED_PY_READER_H diff --git a/py/runtime.c b/py/runtime.c index 79a6fc4ed7b45..9def98380fa45 100644 --- a/py/runtime.c +++ b/py/runtime.c @@ -48,6 +48,10 @@ #include "py/cstack.h" #include "py/gc.h" +#if MICROPY_VFS_ROM && MICROPY_VFS_ROM_IOCTL +#include "extmod/vfs.h" +#endif + // CIRCUITPY-CHANGE #if CIRCUITPY_WARNINGS #include "shared-module/warnings/__init__.h" @@ -197,6 +201,11 @@ void mp_init(void) { #endif MP_THREAD_GIL_ENTER(); + + #if MICROPY_VFS_ROM && MICROPY_VFS_ROM_IOCTL + // Mount ROMFS if it exists. + mp_vfs_mount_romfs_protected(); + #endif } void mp_deinit(void) { @@ -1681,26 +1690,6 @@ mp_obj_t __attribute__((noinline, )) mp_import_from(mp_obj_t module, qstr name) void mp_import_all(mp_obj_t module) { DEBUG_printf("import all %p\n", module); - // CIRCUITPY-CHANGE: displayio name changes; remove in 10.0 - #if CIRCUITPY_DISPLAYIO && CIRCUITPY_WARNINGS - if (module == &displayio_module) { - #if CIRCUITPY_BUSDISPLAY - warnings_warn(&mp_type_FutureWarning, MP_ERROR_TEXT("%q moved from %q to %q"), MP_QSTR_Display, MP_QSTR_displayio, MP_QSTR_busdisplay); - warnings_warn(&mp_type_FutureWarning, MP_ERROR_TEXT("%q renamed %q"), MP_QSTR_Display, MP_QSTR_BusDisplay); - #endif - #if CIRCUITPY_EPAPERDISPLAY - warnings_warn(&mp_type_FutureWarning, MP_ERROR_TEXT("%q moved from %q to %q"), MP_QSTR_EPaperDisplay, MP_QSTR_displayio, MP_QSTR_epaperdisplay); - #endif - #if CIRCUITPY_FOURWIRE - warnings_warn(&mp_type_FutureWarning, MP_ERROR_TEXT("%q moved from %q to %q"), MP_QSTR_FourWire, MP_QSTR_displayio, MP_QSTR_fourwire); - #endif - #if CIRCUITPY_I2CDISPLAYBUS - warnings_warn(&mp_type_FutureWarning, MP_ERROR_TEXT("%q moved from %q to %q"), MP_QSTR_I2CDisplay, MP_QSTR_displayio, MP_QSTR_i2cdisplaybus); - warnings_warn(&mp_type_FutureWarning, MP_ERROR_TEXT("%q renamed %q"), MP_QSTR_I2CDisplay, MP_QSTR_I2CDisplayBus); - #endif - } - #endif - // TODO: Support __all__ mp_map_t *map = &mp_obj_module_get_globals(module)->map; for (size_t i = 0; i < map->alloc; i++) { @@ -1737,10 +1726,13 @@ mp_obj_t mp_parse_compile_execute(mp_lexer_t *lex, mp_parse_input_kind_t parse_i mp_obj_t module_fun = mp_compile(&parse_tree, source_name, parse_input_kind == MP_PARSE_SINGLE_INPUT); mp_obj_t ret; - if (MICROPY_PY_BUILTINS_COMPILE && globals == NULL) { + #if MICROPY_PY_BUILTINS_COMPILE && MICROPY_PY_BUILTINS_CODE == MICROPY_PY_BUILTINS_CODE_MINIMUM + if (globals == NULL) { // for compile only, return value is the module function ret = module_fun; - } else { + } else + #endif + { // execute module function and get return value ret = mp_call_function_0(module_fun); } @@ -1939,7 +1931,6 @@ NORETURN MP_COLD void mp_raise_type_arg(const mp_obj_type_t *exc_type, mp_obj_t nlr_raise(mp_obj_new_exception_arg1(exc_type, arg)); } -// CIRCUITPY-CHANGE: MP_COLD NORETURN void mp_raise_StopIteration(mp_obj_t arg) { if (arg == MP_OBJ_NULL) { mp_raise_type(&mp_type_StopIteration); diff --git a/py/usermod.cmake b/py/usermod.cmake index c814369a4865e..4a8b99ff31b46 100644 --- a/py/usermod.cmake +++ b/py/usermod.cmake @@ -42,6 +42,16 @@ endfunction() # Include CMake files for user modules. if (USER_C_MODULES) foreach(USER_C_MODULE_PATH ${USER_C_MODULES}) + # If a directory is given, append the micropython.cmake to it. + if (IS_DIRECTORY ${USER_C_MODULE_PATH}) + set(USER_C_MODULE_PATH "${USER_C_MODULE_PATH}/micropython.cmake") + endif() + # Confirm the provided path exists, show abspath if not to make it clearer to fix. + if (NOT EXISTS ${USER_C_MODULE_PATH}) + get_filename_component(USER_C_MODULES_ABS "${USER_C_MODULE_PATH}" ABSOLUTE) + message(FATAL_ERROR "USER_C_MODULES doesn't exist: ${USER_C_MODULES_ABS}") + endif() + message("Including User C Module(s) from ${USER_C_MODULE_PATH}") include(${USER_C_MODULE_PATH}) endforeach() diff --git a/py/verbose.mk b/py/verbose.mk new file mode 100644 index 0000000000000..734623a21e80f --- /dev/null +++ b/py/verbose.mk @@ -0,0 +1,16 @@ +# Turn on increased build verbosity by defining BUILD_VERBOSE in your main +# Makefile or in your environment. You can also use V=1 on the make command +# line. + +ifeq ("$(origin V)", "command line") +BUILD_VERBOSE=$(V) +endif +ifndef BUILD_VERBOSE +$(info Use make V=1 or set BUILD_VERBOSE in your environment to increase build verbosity.) +BUILD_VERBOSE = 0 +endif +ifeq ($(BUILD_VERBOSE),0) +Q = @ +else +Q = +endif diff --git a/shared/runtime/gchelper_generic.c b/shared/runtime/gchelper_generic.c index 0937231374057..45b2e4f7d848a 100644 --- a/shared/runtime/gchelper_generic.c +++ b/shared/runtime/gchelper_generic.c @@ -101,6 +101,10 @@ static void gc_helper_get_regs(gc_helper_regs_t arr) { // Fallback implementation, prefer gchelper_thumb1.s or gchelper_thumb2.s static void gc_helper_get_regs(gc_helper_regs_t arr) { + #ifdef __clang__ + #pragma clang diagnostic push + #pragma clang diagnostic ignored "-Wuninitialized" + #endif register long r4 asm ("r4"); register long r5 asm ("r5"); register long r6 asm ("r6"); @@ -121,6 +125,9 @@ static void gc_helper_get_regs(gc_helper_regs_t arr) { arr[7] = r11; arr[8] = r12; arr[9] = r13; + #ifdef __clang__ + #pragma clang diagnostic pop + #endif } #elif defined(__aarch64__) diff --git a/shared/runtime/pyexec.c b/shared/runtime/pyexec.c index 4ea8f434db96a..06680ff2dd125 100644 --- a/shared/runtime/pyexec.c +++ b/shared/runtime/pyexec.c @@ -31,14 +31,14 @@ // CIRCUITPY-CHANGE: add #include "py/mphal.h" + #include "py/compile.h" #include "py/runtime.h" #include "py/repl.h" #include "py/gc.h" #include "py/frozenmod.h" #include "py/mphal.h" -// CIRCUITPY-CHANGE: prevent undefined warning -#if defined(MICROPY_HW_ENABLE_USB) && MICROPY_HW_ENABLE_USB +#if MICROPY_HW_ENABLE_USB #include "irq.h" #include "usb.h" #endif @@ -662,8 +662,7 @@ int pyexec_friendly_repl(void) { for (;;) { input_restart: - // CIRCUITPY-CHANGE: prevent undef warning - #if defined(MICROPY_HW_ENABLE_USB) && MICROPY_HW_ENABLE_USB + #if MICROPY_HW_ENABLE_USB if (usb_vcp_is_enabled()) { // If the user gets to here and interrupts are disabled then // they'll never see the prompt, traceback etc. The USB REPL needs @@ -830,6 +829,11 @@ int pyexec_exit_handler(const void *source, pyexec_result_t *result) { } #endif +int pyexec_vstr(vstr_t *str, bool allow_keyboard_interrupt, pyexec_result_t *result) { + mp_uint_t exec_flags = allow_keyboard_interrupt ? 0 : EXEC_FLAG_NO_INTERRUPT; + return parse_compile_execute(str, MP_PARSE_FILE_INPUT, exec_flags | EXEC_FLAG_SOURCE_IS_VSTR, result); +} + #if MICROPY_REPL_INFO mp_obj_t pyb_set_repl_info(mp_obj_t o_value) { repl_display_debugging_info = mp_obj_get_int(o_value); diff --git a/shared/runtime/pyexec.h b/shared/runtime/pyexec.h index dffdc24772e7c..55a3003791cb2 100644 --- a/shared/runtime/pyexec.h +++ b/shared/runtime/pyexec.h @@ -57,6 +57,7 @@ int pyexec_friendly_repl(void); int pyexec_file(const char *filename, pyexec_result_t *result); int pyexec_file_if_exists(const char *filename, pyexec_result_t *result); int pyexec_frozen_module(const char *name, bool allow_keyboard_interrupt, pyexec_result_t *result); +int pyexec_vstr(vstr_t *str, bool allow_keyboard_interrupt, pyexec_result_t *result); void pyexec_event_repl_init(void); int pyexec_event_repl_process_char(int c); extern uint8_t pyexec_repl_active; diff --git a/shared/timeutils/timeutils.h b/shared/timeutils/timeutils.h index 01f54586e1315..b41903d3b2777 100644 --- a/shared/timeutils/timeutils.h +++ b/shared/timeutils/timeutils.h @@ -65,7 +65,7 @@ mp_uint_t timeutils_mktime_2000(mp_uint_t year, mp_int_t month, mp_int_t mday, static inline void timeutils_seconds_since_epoch_to_struct_time(uint64_t t, timeutils_struct_time_t *tm) { // TODO this will give incorrect results for dates before 2000/1/1 - timeutils_seconds_since_2000_to_struct_time(t - TIMEUTILS_SECONDS_1970_TO_2000, tm); + timeutils_seconds_since_2000_to_struct_time((mp_uint_t)(t - TIMEUTILS_SECONDS_1970_TO_2000), tm); } // Year is absolute, month/mday are 1-based, hours/minutes/seconds are 0-based. @@ -81,7 +81,7 @@ static inline uint64_t timeutils_seconds_since_epoch(mp_uint_t year, mp_uint_t m } static inline mp_uint_t timeutils_seconds_since_epoch_from_nanoseconds_since_1970(uint64_t ns) { - return ns / 1000000000ULL; + return (mp_uint_t)(ns / 1000000000ULL); } static inline uint64_t timeutils_nanoseconds_since_epoch_to_nanoseconds_since_1970(uint64_t ns) { diff --git a/tests/README.md b/tests/README.md index 7bd7eb587177b..21e14eee5e128 100644 --- a/tests/README.md +++ b/tests/README.md @@ -1,7 +1,45 @@ # MicroPython Test Suite -This directory contains tests for various functionality areas of MicroPython. -To run all stable tests, run "run-tests.py" script in this directory. +This directory contains tests for most parts of MicroPython. + +To run all stable tests, run the "run-tests.py" script in this directory. By default +that will run the test suite against the unix port of MicroPython. + +To run the test suite against a bare-metal target (a board running MicroPython firmware) +use the `-t` option to specify the serial port. This will automatically detect the +target platform and run the appropriate set of tests for that platform. For example: + + $ ./run-tests.py -t /dev/ttyACM0 + +That will run tests on the `/dev/ttyACM0` serial port. You can also use shortcut +device names like `a` for `/dev/ttyACM` and `c` for `COM`. Use +`./run-tests.py --help` to see all of the device possibilities, and other options. + +There are three kinds of tests: + +* Tests that use `unittest`: these tests require `unittest` to be installed on the + target (eg via `mpremote mip install unittest`), and are used to test things that are + MicroPython-specific, such as behaviour that is different to CPython, modules that + aren't available in CPython, and hardware tests. These tests are run only under + MicroPython and the test passes if the `unittest` runner prints "OK" at the end of the + run. Other output may be printed, eg for use as diagnostics, and this output does not + affect the result of the test. + +* Tests with a corresponding `.exp` file: similar to the `unittest` tests, these tests + are for features that generally cannot be run under CPython. In this case the test is + run under MicroPython only and the output from MicroPython is compared against the + provided `.exp` file. The test passes if the output matches exactly. + +* Tests without a corresponding `.exp` file (and don't use `unittest`): these tests are + used to test MicroPython behaviour that should precisely match CPython. These tests + are first run under CPython and the output captured, and then run under MicroPython + and the output compared to the CPython output. The test passes if the output matches + exactly. If the output differs then the test fails and the outputs are saved in a + `.exp` and a `.out` file respectively. + +In all three cases above, the test can usually be run directly on the target MicroPython +instance, either using the unix port with `micropython `, or on a board with +`mpremote run `. This is useful for creating and debugging tests. Tests of capabilities not supported on all platforms should be written to check for the capability being present. If it is not, the test @@ -15,15 +53,6 @@ condition a test. The run-tests.py script uses small scripts in the feature_check directory to check whether each such feature is present, and skips the relevant tests if not. -Tests are generally verified by running the test both in MicroPython and -in CPython and comparing the outputs. If the output differs the test fails -and the outputs are saved in a .out and a .exp file respectively. -For tests that cannot be run in CPython, for example because they use -the machine module, a .exp file can be provided next to the test's .py -file. A convenient way to generate that is to run the test, let it fail -(because CPython cannot run it) and then copy the .out file (but not -before checking it manually!) - When creating new tests, anything that relies on float support should go in the float/ subdirectory. Anything that relies on import x, where x is not a built-in module, should go in the import/ subdirectory. @@ -193,7 +222,7 @@ need to be re-created by end users. This section is included here for reference A new self-signed RSA key/cert pair can be created with openssl: ``` -$ openssl req -x509 -newkey rsa:2048 -keyout rsa_key.pem -out rsa_cert.pem -days 365 -nodes -subj '/CN=micropython.local/O=MicroPython/C=AU' +$ openssl req -x509 -newkey rsa:2048 -keyout rsa_key.pem -out rsa_cert.pem -days 3650 -nodes -subj '/CN=micropython.local/O=MicroPython/C=AU' ``` In this case CN is: micropython.local @@ -206,6 +235,6 @@ $ openssl x509 -in rsa_cert.pem -out rsa_cert.der -outform DER For elliptic curve tests using key/cert pairs, create a key then a certificate using: ``` $ openssl ecparam -name prime256v1 -genkey -noout -out ec_key.pem -$ openssl x509 -in ec_key.pem -out ec_key.der -outform DER -$ openssl req -new -x509 -key ec_key.pem -out ec_cert.der -outform DER -days 365 -nodes -subj '/CN=micropython.local/O=MicroPython/C=AU' +$ openssl pkey -in ec_key.pem -out ec_key.der -outform DER +$ openssl req -new -x509 -key ec_key.pem -out ec_cert.der -outform DER -days 3650 -nodes -subj '/CN=micropython.local/O=MicroPython/C=AU' ``` diff --git a/tests/basics/fun_code.py b/tests/basics/fun_code.py new file mode 100644 index 0000000000000..59e1f7ec0483d --- /dev/null +++ b/tests/basics/fun_code.py @@ -0,0 +1,36 @@ +# Test function.__code__ attribute. + +try: + (lambda: 0).__code__ +except AttributeError: + print("SKIP") + raise SystemExit + + +def f(): + return a + + +ftype = type(f) + +# Test __code__ access and function constructor. +code = f.__code__ +print(type(ftype(code, {})) is ftype) + +# Test instantiating multiple code's with different globals dicts. +code = f.__code__ +f1 = ftype(code, {"a": 1}) +f2 = ftype(code, {"a": 2}) +print(f1(), f2()) + +# Test bad first argument type. +try: + ftype(None, {}) +except TypeError: + print("TypeError") + +# Test bad second argument type. +try: + ftype(f.__code__, None) +except TypeError: + print("TypeError") diff --git a/tests/basics/fun_code_micropython.py b/tests/basics/fun_code_micropython.py new file mode 100644 index 0000000000000..2c319a2db8c75 --- /dev/null +++ b/tests/basics/fun_code_micropython.py @@ -0,0 +1,19 @@ +# Test MicroPython-specific restrictions of function.__code__ attribute. + +try: + (lambda: 0).__code__ +except AttributeError: + print("SKIP") + raise SystemExit + + +def f_with_children(): + def g(): + pass + + +# Can't access __code__ when function has children. +try: + f_with_children.__code__ +except AttributeError: + print("AttributeError") diff --git a/tests/basics/fun_code_micropython.py.exp b/tests/basics/fun_code_micropython.py.exp new file mode 100644 index 0000000000000..d169edffb4cfb --- /dev/null +++ b/tests/basics/fun_code_micropython.py.exp @@ -0,0 +1 @@ +AttributeError diff --git a/tests/basics/int1.py b/tests/basics/int1.py index 2d92105c73e88..94723af4d00b4 100644 --- a/tests/basics/int1.py +++ b/tests/basics/int1.py @@ -13,6 +13,7 @@ print(int('+1')) print(int('-1')) print(int('01')) +print(int('00')) print(int('9')) print(int('10')) print(int('+10')) @@ -31,6 +32,7 @@ print(int('0', 10)) print(int('1', 10)) print(int(' \t 1 \t ', 10)) +print(int(' \t 00 \t ', 10)) print(int('11', 10)) print(int('11', 16)) print(int('11', 8)) @@ -52,6 +54,17 @@ print(int('0o12 \t ', 8)) print(int(b"12", 10)) print(int(b"12")) +print(int('000 ', 0)) +print(int('000 ', 2)) +print(int('000 ', 8)) +print(int('000 ', 10)) +print(int('000 ', 16)) +print(int('000 ', 36)) +print(int('010 ', 2)) +print(int('010 ', 8)) +print(int('010 ', 10)) +print(int('010 ', 16)) +print(int('010 ', 36)) def test(value, base): @@ -79,6 +92,8 @@ def test(value, base): test('0xg', 16) test('1 1', 16) test('123', 37) +test('01', 0) +test('01 ', 0) # check that we don't parse this as a floating point number print(0x1e+1) diff --git a/tests/basics/lexer.py b/tests/basics/lexer.py index 181d62db1aadb..addb8a13df36a 100644 --- a/tests/basics/lexer.py +++ b/tests/basics/lexer.py @@ -83,3 +83,11 @@ def a(x): exec(r"'\U0000000'") except SyntaxError: print("SyntaxError") + +# Properly formed integer literals +print(eval("00")) +# badly formed integer literals +try: + eval("01") +except SyntaxError: + print("SyntaxError") diff --git a/tests/basics/nanbox_smallint.py b/tests/basics/nanbox_smallint.py index b3a502e447e3a..9451ab3284661 100644 --- a/tests/basics/nanbox_smallint.py +++ b/tests/basics/nanbox_smallint.py @@ -23,17 +23,17 @@ raise SystemExit micropython.heap_lock() -print(int("0x80000000")) +print(int("0x80000000", 16)) micropython.heap_unlock() # This is the most positive small integer. micropython.heap_lock() -print(int("0x3fffffffffff")) +print(int("0x3fffffffffff", 16)) micropython.heap_unlock() # This is the most negative small integer. micropython.heap_lock() -print(int("-0x3fffffffffff") - 1) +print(int("-0x3fffffffffff", 16) - 1) micropython.heap_unlock() x = 1 diff --git a/tests/basics/string_endswith.py b/tests/basics/string_endswith.py index 683562d10c32e..2b0a063988b45 100644 --- a/tests/basics/string_endswith.py +++ b/tests/basics/string_endswith.py @@ -5,11 +5,28 @@ print("foobar".endswith("")) print("foobar".endswith("foobarbaz")) -#print("1foobar".startswith("foo", 1)) -#print("1foo".startswith("foo", 1)) -#print("1foo".startswith("1foo", 1)) -#print("1fo".startswith("foo", 1)) -#print("1fo".startswith("foo", 10)) +print("foobar".endswith("bar", 3)) +print("foobar".endswith("bar", 4)) +print("foobar".endswith("foo", 0, 3)) +print("foobar".endswith("foo", 0, 4)) +print("foobar".endswith("foo", 1, 3)) +print("foobar".endswith("foo", 1, 3)) +print("foobar".endswith("oo", 1, 3)) +print("foobar".endswith("o", 2, 3)) +print("foobar".endswith("o", 3, 3)) +print("foobar".endswith("o", 4, 3)) + +print("foobar".endswith("bar", None, None)) +print("foobar".endswith("bar", None, 3)) +print("foobar".endswith("bar", 3, None)) +print("foobar".endswith("bar", 2, None)) +print("foobar".endswith("foo", None, 3)) + +print("foobar".endswith(("bar", "foo"))) +print("foobar".endswith(("foo", "bar"))) +print("foobar".endswith(("foo", "bar1"))) +print("foobar".endswith(("bar", ))) +print("foobar".endswith(("foo", ))) try: "foobar".endswith(1) diff --git a/tests/basics/string_endswith_upy.py b/tests/basics/string_endswith_upy.py deleted file mode 100644 index 06a4e71d2c927..0000000000000 --- a/tests/basics/string_endswith_upy.py +++ /dev/null @@ -1,6 +0,0 @@ -# MicroPython doesn't support tuple argument - -try: - "foobar".endswith(("bar", "sth")) -except TypeError: - print("TypeError") diff --git a/tests/basics/string_endswith_upy.py.exp b/tests/basics/string_endswith_upy.py.exp deleted file mode 100644 index 6002b71c56ea0..0000000000000 --- a/tests/basics/string_endswith_upy.py.exp +++ /dev/null @@ -1 +0,0 @@ -TypeError diff --git a/tests/basics/string_startswith.py b/tests/basics/string_startswith.py index e63ae3c1866df..eefdea82198f1 100644 --- a/tests/basics/string_startswith.py +++ b/tests/basics/string_startswith.py @@ -10,6 +10,25 @@ print("1fo".startswith("foo", 1)) print("1fo".startswith("foo", 10)) +print("1foobar".startswith("foo", 1, 5)) +print("1foobar".startswith("foo", 1, 4)) +print("1foobar".startswith("foo", 1, 3)) +print("1foobar".startswith("oo", 2, 4)) +print("1foobar".startswith("o", 3, 4)) +print("1foobar".startswith("o", 4, 4)) +print("1foobar".startswith("o", 5, 4)) + +print("foobar".startswith("foo", None, None)) +print("foobar".startswith("foo", None, 3)) +print("foobar".startswith("foo", None, 2)) +print("foobar".startswith("bar", 3, None)) + + +print("foobar".startswith(("foo", "sth"))) +print("foobar".startswith(("sth", "foo"))) +print("foobar".startswith(("sth", "foo2"))) +print("foobar".startswith(("foo", ))) + try: "foobar".startswith(1) except TypeError: diff --git a/tests/basics/string_startswith_upy.py b/tests/basics/string_startswith_upy.py deleted file mode 100644 index 9ea1796c218fd..0000000000000 --- a/tests/basics/string_startswith_upy.py +++ /dev/null @@ -1,6 +0,0 @@ -# MicroPython doesn't support tuple argument - -try: - "foobar".startswith(("foo", "sth")) -except TypeError: - print("TypeError") diff --git a/tests/basics/string_startswith_upy.py.exp b/tests/basics/string_startswith_upy.py.exp deleted file mode 100644 index 6002b71c56ea0..0000000000000 --- a/tests/basics/string_startswith_upy.py.exp +++ /dev/null @@ -1 +0,0 @@ -TypeError diff --git a/tests/basics/subclass_native1.py b/tests/basics/subclass_native1.py index 288a686d1a756..74b377eac91b9 100644 --- a/tests/basics/subclass_native1.py +++ b/tests/basics/subclass_native1.py @@ -21,11 +21,9 @@ class mylist(list): # TODO: Faults #print(a + a) -def foo(): - print("hello from foo") - +# subclassing a type that doesn't have make_new at the C level (not allowed) try: - class myfunc(type(foo)): + class myfunc(type([].append)): pass except TypeError: print("TypeError") diff --git a/tests/basics/sys1.py b/tests/basics/sys1.py index 6266440e353ca..7f261aa96459d 100644 --- a/tests/basics/sys1.py +++ b/tests/basics/sys1.py @@ -25,6 +25,12 @@ # Effectively skip subtests print(int) +if hasattr(sys.implementation, '_build'): + print(type(sys.implementation._build)) +else: + # Effectively skip subtests + print(str) + try: print(sys.intern('micropython') == 'micropython') has_intern = True diff --git a/tests/cpydiff/builtin_next_arg2.py b/tests/cpydiff/builtin_next_arg2.py deleted file mode 100644 index ed9565fe0fe77..0000000000000 --- a/tests/cpydiff/builtin_next_arg2.py +++ /dev/null @@ -1,13 +0,0 @@ -""" -categories: Modules,builtins -description: Second argument to next() is not implemented -cause: MicroPython is optimised for code space. -workaround: Instead of ``val = next(it, deflt)`` use:: - - try: - val = next(it) - except StopIteration: - val = deflt -""" - -print(next(iter(range(0)), 42)) diff --git a/tests/cpydiff/core_fstring_concat.py b/tests/cpydiff/core_fstring_concat.py index 1ef016d5230b6..3daa13d75360e 100644 --- a/tests/cpydiff/core_fstring_concat.py +++ b/tests/cpydiff/core_fstring_concat.py @@ -6,7 +6,9 @@ """ x, y = 1, 2 -print(f"aa{x}") # works -print(f"{x}ab") # works -print(f"a{{}}a{x}") # fails -print(f"{x}a{{}}b") # fails +# fmt: off +print("aa" f"{x}") # works +print(f"{x}" "ab") # works +print("a{}a" f"{x}") # fails +print(f"{x}" "a{}b") # fails +# fmt: on diff --git a/tests/cpydiff/modules_json_nonserializable.py b/tests/cpydiff/modules_json_nonserializable.py index 1adc13b26b1eb..d83e7beca2d5a 100644 --- a/tests/cpydiff/modules_json_nonserializable.py +++ b/tests/cpydiff/modules_json_nonserializable.py @@ -7,10 +7,7 @@ import json -a = bytes(x for x in range(256)) try: - z = json.dumps(a) - x = json.loads(z) - print("Should not get here") + print(json.dumps(b"shouldn't be able to serialise bytes")) except TypeError: print("TypeError") diff --git a/tests/cpydiff/syntax_assign_expr.py b/tests/cpydiff/syntax_assign_expr.py index 58f57ca1fbe02..704c5c3ecab37 100644 --- a/tests/cpydiff/syntax_assign_expr.py +++ b/tests/cpydiff/syntax_assign_expr.py @@ -1,8 +1,8 @@ """ categories: Syntax,Operators -description: MicroPython allows using := to assign to the variable of a comprehension, CPython raises a SyntaxError. -cause: MicroPython is optimised for code size and doesn't check this case. -workaround: Do not rely on this behaviour if writing CPython compatible code. +description: MicroPython allows := to assign to the iteration variable in nested comprehensions, CPython does not. +cause: MicroPython is optimised for code size. Although it is a syntax error to assign to the iteration variable in a standard comprehension (same as CPython), it doesn't check if an inner nested comprehension assigns to the iteration variable of the outer comprehension. +workaround: Do not use := to assign to the iteration variable of a comprehension. """ -print([i := -1 for i in range(4)]) +print([[(j := i) for i in range(2)] for j in range(2)]) diff --git a/tests/cpydiff/types_str_endswith.py b/tests/cpydiff/types_str_endswith.py deleted file mode 100644 index 890c7ba5ef47f..0000000000000 --- a/tests/cpydiff/types_str_endswith.py +++ /dev/null @@ -1,8 +0,0 @@ -""" -categories: Types,str -description: Start/end indices such as str.endswith(s, start) not implemented -cause: Unknown -workaround: Unknown -""" - -print("abc".endswith("c", 1)) diff --git a/tests/extmod/asyncio_new_event_loop.py b/tests/extmod/asyncio_new_event_loop.py index bebc3bf70cc57..3f05ffdd551d7 100644 --- a/tests/extmod/asyncio_new_event_loop.py +++ b/tests/extmod/asyncio_new_event_loop.py @@ -12,6 +12,16 @@ asyncio.set_event_loop(asyncio.new_event_loop()) +def exception_handler(loop, context): + # This is a workaround for a difference between CPython and MicroPython: if + # a CPython event loop is closed while there are tasks pending (i.e. not finished) + # on it, then the task will log an error. MicroPython does not log this error. + if context.get("message", "") == "Task was destroyed but it is pending!": + pass + else: + loop.default_exception_handler(context) + + async def task(): for i in range(4): print("task", i) @@ -22,17 +32,21 @@ async def task(): async def main(): print("start") loop.create_task(task()) - await asyncio.sleep(0) + await asyncio.sleep(0) # yields, meaning new task will run once print("stop") loop.stop() # Use default event loop to run some tasks loop = asyncio.get_event_loop() +loop.set_exception_handler(exception_handler) loop.create_task(main()) loop.run_forever() +loop.close() # Create new event loop, old one should not keep running loop = asyncio.new_event_loop() +loop.set_exception_handler(exception_handler) loop.create_task(main()) loop.run_forever() +loop.close() diff --git a/tests/extmod/deflate_compress_memory_error.py b/tests/extmod/deflate_compress_memory_error.py new file mode 100644 index 0000000000000..19bef87bff3da --- /dev/null +++ b/tests/extmod/deflate_compress_memory_error.py @@ -0,0 +1,39 @@ +# Test deflate.DeflateIO compression, with out-of-memory errors. + +try: + # Check if deflate is available. + import deflate + import io +except ImportError: + print("SKIP") + raise SystemExit + +# Check if compression is enabled. +if not hasattr(deflate.DeflateIO, "write"): + print("SKIP") + raise SystemExit + +# Create a compressor object. +b = io.BytesIO() +g = deflate.DeflateIO(b, deflate.RAW, 15) + +# Then, use up most of the heap. +l = [] +while True: + try: + l.append(bytearray(1000)) + except: + break +l.pop() + +# Try to compress. This will try to allocate a large window and fail. +try: + g.write("test") +except MemoryError: + print("MemoryError") + +# Should still be able to close the stream. +g.close() + +# The underlying output stream should be unchanged. +print(b.getvalue()) diff --git a/tests/extmod/deflate_compress_memory_error.py.exp b/tests/extmod/deflate_compress_memory_error.py.exp new file mode 100644 index 0000000000000..606315c14604c --- /dev/null +++ b/tests/extmod/deflate_compress_memory_error.py.exp @@ -0,0 +1,2 @@ +MemoryError +b'' diff --git a/tests/extmod/marshal_basic.py b/tests/extmod/marshal_basic.py new file mode 100644 index 0000000000000..9e7b70be4829d --- /dev/null +++ b/tests/extmod/marshal_basic.py @@ -0,0 +1,38 @@ +# Test the marshal module, basic functionality. + +try: + import marshal + + (lambda: 0).__code__ +except (AttributeError, ImportError): + print("SKIP") + raise SystemExit + +ftype = type(lambda: 0) + +# Test basic dumps and loads. +print(ftype(marshal.loads(marshal.dumps((lambda: a).__code__)), {"a": 4})()) + +# Test dumps of a result from compile(). +ftype(marshal.loads(marshal.dumps(compile("print(a)", "", "exec"))), {"print": print, "a": 5})() + +# Test marshalling a function with arguments. +print(ftype(marshal.loads(marshal.dumps((lambda x, y: x + y).__code__)), {})(1, 2)) + +# Test marshalling a function with default arguments. +print(ftype(marshal.loads(marshal.dumps((lambda x=0: x).__code__)), {})("arg")) + +# Test marshalling a function containing constant objects (a tuple). +print(ftype(marshal.loads(marshal.dumps((lambda: (None, ...)).__code__)), {})()) + +# Test instantiating multiple code's with different globals dicts. +code = marshal.loads(marshal.dumps((lambda: a).__code__)) +f1 = ftype(code, {"a": 1}) +f2 = ftype(code, {"a": 2}) +print(f1(), f2()) + +# Test unmarshallable object. +try: + marshal.dumps(type) +except ValueError: + print("ValueError") diff --git a/tests/extmod/marshal_micropython.py b/tests/extmod/marshal_micropython.py new file mode 100644 index 0000000000000..213b3bf31895d --- /dev/null +++ b/tests/extmod/marshal_micropython.py @@ -0,0 +1,21 @@ +# Test the marshal module, MicroPython-specific functionality. + +try: + import marshal +except ImportError: + print("SKIP") + raise SystemExit + +import unittest + + +class Test(unittest.TestCase): + def test_function_with_children(self): + # Can't marshal a function with children (in this case the module has a child function f). + code = compile("def f(): pass", "", "exec") + with self.assertRaises(ValueError): + marshal.dumps(code) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/extmod/marshal_stress.py b/tests/extmod/marshal_stress.py new file mode 100644 index 0000000000000..b52475c0361dc --- /dev/null +++ b/tests/extmod/marshal_stress.py @@ -0,0 +1,122 @@ +# Test the marshal module, stressing edge cases. + +try: + import marshal + + (lambda: 0).__code__ +except (AttributeError, ImportError): + print("SKIP") + raise SystemExit + +ftype = type(lambda: 0) + +# Test a large function. + + +def large_function(arg0, arg1, arg2, arg3): + # Arguments. + print(arg0, arg1, arg2, arg3) + + # Positive medium-sized integer (still a small-int though). + print(1234) + + # Negative small-ish integer. + print(-20) + + # More than 64 constant objects. + x = (0,) + x = (1,) + x = (2,) + x = (3,) + x = (4,) + x = (5,) + x = (6,) + x = (7,) + x = (8,) + x = (9,) + x = (10,) + x = (11,) + x = (12,) + x = (13,) + x = (14,) + x = (15,) + x = (16,) + x = (17,) + x = (18,) + x = (19,) + x = (20,) + x = (21,) + x = (22,) + x = (23,) + x = (24,) + x = (25,) + x = (26,) + x = (27,) + x = (28,) + x = (29,) + x = (30,) + x = (31,) + x = (32,) + x = (33,) + x = (34,) + x = (35,) + x = (36,) + x = (37,) + x = (38,) + x = (39,) + x = (40,) + x = (41,) + x = (42,) + x = (43,) + x = (44,) + x = (45,) + x = (46,) + x = (47,) + x = (48,) + x = (49,) + x = (50,) + x = (51,) + x = (52,) + x = (53,) + x = (54,) + x = (55,) + x = (56,) + x = (57,) + x = (58,) + x = (59,) + x = (60,) + x = (61,) + x = (62,) + x = (63,) + x = (64,) + + # Small jump. + x = 0 + while x < 2: + print("loop", x) + x += 1 + + # Large jump. + x = 0 + while x < 2: + try: + try: + try: + print + except Exception as e: + print + finally: + print + except Exception as e: + print + finally: + print + except Exception as e: + print + finally: + print("loop", x) + x += 1 + + +code = marshal.dumps(large_function.__code__) +ftype(marshal.loads(code), {"print": print})(0, 1, 2, 3) diff --git a/tests/extmod/re_sub.py b/tests/extmod/re_sub.py index 2c7c6c10f1a49..3959949724d98 100644 --- a/tests/extmod/re_sub.py +++ b/tests/extmod/re_sub.py @@ -10,6 +10,8 @@ print("SKIP") raise SystemExit +import sys + def multiply(m): return str(int(m.group(0)) * 2) @@ -47,7 +49,12 @@ def A(): print(re.sub("a", "b", "c")) # with maximum substitution count specified -print(re.sub("a", "b", "1a2a3a", 2)) +# CIRCUITPY-CHANGE: was "micropython" +if sys.implementation.name != "circuitpython": + # On CPython 3.13 and later the substitution count must be a keyword argument. + print(re.sub("a", "b", "1a2a3a", count=2)) +else: + print(re.sub("a", "b", "1a2a3a", 2)) # invalid group try: diff --git a/tests/extmod/ssl_noleak.py b/tests/extmod/ssl_noleak.py new file mode 100644 index 0000000000000..870032d58e63e --- /dev/null +++ b/tests/extmod/ssl_noleak.py @@ -0,0 +1,50 @@ +# Ensure that SSLSockets can be allocated sequentially +# without running out of available memory. +try: + import io + import tls +except ImportError: + print("SKIP") + raise SystemExit + +import unittest + + +class TestSocket(io.IOBase): + def write(self, buf): + return len(buf) + + def readinto(self, buf): + return 0 + + def ioctl(self, cmd, arg): + return 0 + + def setblocking(self, value): + pass + + +ITERS = 128 + + +class TLSNoLeaks(unittest.TestCase): + def test_unique_context(self): + for n in range(ITERS): + print(n) + s = TestSocket() + ctx = tls.SSLContext(tls.PROTOCOL_TLS_CLIENT) + ctx.verify_mode = tls.CERT_NONE + s = ctx.wrap_socket(s, do_handshake_on_connect=False) + + def test_shared_context(self): + # Single SSLContext, multiple sockets + ctx = tls.SSLContext(tls.PROTOCOL_TLS_CLIENT) + ctx.verify_mode = tls.CERT_NONE + for n in range(ITERS): + print(n) + s = TestSocket() + s = ctx.wrap_socket(s, do_handshake_on_connect=False) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/extmod/tls_dtls.py b/tests/extmod/tls_dtls.py new file mode 100644 index 0000000000000..b2d716769d3f7 --- /dev/null +++ b/tests/extmod/tls_dtls.py @@ -0,0 +1,51 @@ +# Test DTLS functionality including timeout handling + +try: + from tls import PROTOCOL_DTLS_CLIENT, PROTOCOL_DTLS_SERVER, SSLContext, CERT_NONE + import io +except ImportError: + print("SKIP") + raise SystemExit + + +class DummySocket(io.IOBase): + def __init__(self): + self.write_buffer = bytearray() + self.read_buffer = bytearray() + + def write(self, data): + return len(data) + + def readinto(self, buf): + # This is a placeholder socket that doesn't actually read anything + # so the read buffer is always empty. + return None + + def ioctl(self, req, arg): + if req == 4: # MP_STREAM_CLOSE + return 0 + return -1 + + +# Create dummy sockets for testing +server_socket = DummySocket() +client_socket = DummySocket() + +# Wrap the DTLS Server +dtls_server_ctx = SSLContext(PROTOCOL_DTLS_SERVER) +dtls_server_ctx.verify_mode = CERT_NONE +dtls_server = dtls_server_ctx.wrap_socket(server_socket, do_handshake_on_connect=False) +print("Wrapped DTLS Server") + +# Wrap the DTLS Client +dtls_client_ctx = SSLContext(PROTOCOL_DTLS_CLIENT) +dtls_client_ctx.verify_mode = CERT_NONE +dtls_client = dtls_client_ctx.wrap_socket(client_socket, do_handshake_on_connect=False) +print("Wrapped DTLS Client") + +# Trigger the timing check multiple times with different elapsed times +for i in range(10): # Try multiple iterations to hit the timing window + dtls_client.write(b"test") + data = dtls_server.read(1024) # This should eventually hit the timing condition + +print("OK") diff --git a/tests/extmod/tls_dtls.py.exp b/tests/extmod/tls_dtls.py.exp new file mode 100644 index 0000000000000..78d72bff18816 --- /dev/null +++ b/tests/extmod/tls_dtls.py.exp @@ -0,0 +1,3 @@ +Wrapped DTLS Server +Wrapped DTLS Client +OK diff --git a/tests/extmod/uctypes_addressof.py b/tests/extmod/uctypes_addressof.py new file mode 100644 index 0000000000000..c83089d0f72af --- /dev/null +++ b/tests/extmod/uctypes_addressof.py @@ -0,0 +1,16 @@ +# Test uctypes.addressof(). + +try: + from sys import maxsize + import uctypes +except ImportError: + print("SKIP") + raise SystemExit + +# Test small addresses. +for i in range(8): + print(uctypes.addressof(uctypes.bytearray_at(1 << i, 8))) + +# Test address that is bigger than the greatest small-int but still within the address range. +large_addr = maxsize + 1 +print(uctypes.addressof(uctypes.bytearray_at(large_addr, 8)) == large_addr) diff --git a/tests/extmod/uctypes_addressof.py.exp b/tests/extmod/uctypes_addressof.py.exp new file mode 100644 index 0000000000000..5b48569d0185d --- /dev/null +++ b/tests/extmod/uctypes_addressof.py.exp @@ -0,0 +1,9 @@ +1 +2 +4 +8 +16 +32 +64 +128 +True diff --git a/tests/extmod/vfs_mountinfo.py b/tests/extmod/vfs_mountinfo.py new file mode 100644 index 0000000000000..f674e80763409 --- /dev/null +++ b/tests/extmod/vfs_mountinfo.py @@ -0,0 +1,66 @@ +# test VFS functionality without any particular filesystem type + +try: + import os, vfs +except ImportError: + print("SKIP") + raise SystemExit +import errno + + +class Filesystem: + def __init__(self, id, paths=[]): + self.id = id + self.paths = paths + + def mount(self, readonly, mksfs): + print("mount", self) + + def umount(self): + print("umount", self) + + def stat(self, path): + if path in self.paths: + return (0, 0, 0, 0, 0, 0, 0, 0, 0, 0) + else: + raise OSError + + statvfs = stat + + def open(self, path, mode): + pass + + def __repr__(self): + return "Filesystem(%d)" % self.id + + +# first we umount any existing mount points the target may have +try: + vfs.umount("/") +except OSError: + pass +for path in os.listdir("/"): + vfs.umount("/" + path) + + +print(vfs.mount()) + +vfs.mount(Filesystem(1), "/foo") + +print(vfs.mount()) + +vfs.mount(Filesystem(2), "/bar/baz") + +print(vfs.mount()) + +vfs.mount(Filesystem(3), "/bar") + +print(vfs.mount()) + +vfs.umount("/bar/baz") + +print(vfs.mount()) + +vfs.mount(Filesystem(4), "/") + +print(vfs.mount()) diff --git a/tests/extmod/vfs_mountinfo.py.exp b/tests/extmod/vfs_mountinfo.py.exp new file mode 100644 index 0000000000000..4ddf06c8c976f --- /dev/null +++ b/tests/extmod/vfs_mountinfo.py.exp @@ -0,0 +1,11 @@ +[] +mount Filesystem(1) +[(Filesystem(1), '/foo')] +mount Filesystem(2) +[(Filesystem(1), '/foo'), (Filesystem(2), '/bar/baz')] +mount Filesystem(3) +[(Filesystem(1), '/foo'), (Filesystem(2), '/bar/baz'), (Filesystem(3), '/bar')] +umount Filesystem(2) +[(Filesystem(1), '/foo'), (Filesystem(3), '/bar')] +mount Filesystem(4) +[(Filesystem(1), '/foo'), (Filesystem(3), '/bar'), (Filesystem(4), '/')] diff --git a/tests/extmod/vfs_rom.py b/tests/extmod/vfs_rom.py new file mode 100644 index 0000000000000..770b6863b9c43 --- /dev/null +++ b/tests/extmod/vfs_rom.py @@ -0,0 +1,488 @@ +# Test VfsRom filesystem. + +try: + import errno, sys, struct, os, uctypes, vfs + + vfs.VfsRom +except (ImportError, AttributeError): + print("SKIP") + raise SystemExit + +try: + import select +except ImportError: + select = None + +import unittest + +IFDIR = 0x4000 +IFREG = 0x8000 + +SEEK_SET = 0 +SEEK_CUR = 1 +SEEK_END = 2 + +# An mpy file with four constant objects: str, bytes, long-int, float. +test_mpy = ( + # header + b"M\x06\x00\x1f" # mpy file header + b"\x06" # n_qstr + b"\x05" # n_obj + # qstrs + b"\x0etest.py\x00" # qstr0 = "test.py" + b"\x0f" # qstr1 = "" + b"\x0estr_obj\x00" # qstr2 = "str_obj" + b"\x12bytes_obj\x00" # qstr3 = "bytes_obj" + b"\x0eint_obj\x00" # qstr4 = "int_obj" + b"\x12float_obj\x00" # qstr5 = "float_obj" + # objects + b"\x05\x14this is a str object\x00" + b"\x06\x16this is a bytes object\x00" + b"\x07\x0a1234567890" # long-int object + b"\x08\x041.23" # float object + b"\x05\x07str_obj\x00" # str object of existing qstr + # bytecode + b"\x81\x28" # 21 bytes, no children, bytecode + b"\x00\x02" # prelude + b"\x01" # simple name () + b"\x23\x00" # LOAD_CONST_OBJ(0) + b"\x16\x02" # STORE_NAME(str_obj) + b"\x23\x01" # LOAD_CONST_OBJ(1) + b"\x16\x03" # STORE_NAME(bytes_obj) + b"\x23\x02" # LOAD_CONST_OBJ(2) + b"\x16\x04" # STORE_NAME(int_obj) + b"\x23\x03" # LOAD_CONST_OBJ(3) + b"\x16\x05" # STORE_NAME(float_obj) + b"\x51" # LOAD_CONST_NONE + b"\x63" # RETURN_VALUE +) + + +class VfsRomWriter: + ROMFS_HEADER = b"\xd2\xcd\x31" + + ROMFS_RECORD_KIND_UNUSED = 0 + ROMFS_RECORD_KIND_PADDING = 1 + ROMFS_RECORD_KIND_DATA_VERBATIM = 2 + ROMFS_RECORD_KIND_DATA_POINTER = 3 + ROMFS_RECORD_KIND_DIRECTORY = 4 + ROMFS_RECORD_KIND_FILE = 5 + + def __init__(self): + self._dir_stack = [(None, bytearray())] + + def _encode_uint(self, value): + encoded = [value & 0x7F] + value >>= 7 + while value != 0: + encoded.insert(0, 0x80 | (value & 0x7F)) + value >>= 7 + return bytes(encoded) + + def _pack(self, kind, payload): + return self._encode_uint(kind) + self._encode_uint(len(payload)) + payload + + def _extend(self, data): + buf = self._dir_stack[-1][1] + buf.extend(data) + return len(buf) + + def finalise(self): + _, data = self._dir_stack.pop() + encoded_kind = VfsRomWriter.ROMFS_HEADER + encoded_len = self._encode_uint(len(data)) + if (len(encoded_kind) + len(encoded_len) + len(data)) % 2 == 1: + encoded_len = b"\x80" + encoded_len + data = encoded_kind + encoded_len + data + return data + + def opendir(self, dirname): + self._dir_stack.append((dirname, bytearray())) + + def closedir(self): + dirname, dirdata = self._dir_stack.pop() + dirdata = self._encode_uint(len(dirname)) + bytes(dirname, "ascii") + dirdata + self._extend(self._pack(VfsRomWriter.ROMFS_RECORD_KIND_DIRECTORY, dirdata)) + + def mkdata(self, data): + assert len(self._dir_stack) == 1 + return self._extend(self._pack(VfsRomWriter.ROMFS_RECORD_KIND_DATA_VERBATIM, data)) - len( + data + ) + + def mkfile(self, filename, filedata, extra_payload=b""): + filename = bytes(filename, "ascii") + payload = self._encode_uint(len(filename)) + payload += filename + payload += extra_payload + if isinstance(filedata, tuple): + sub_payload = self._encode_uint(filedata[0]) + sub_payload += self._encode_uint(filedata[1]) + payload += self._pack(VfsRomWriter.ROMFS_RECORD_KIND_DATA_POINTER, sub_payload) + else: + payload += self._pack(VfsRomWriter.ROMFS_RECORD_KIND_DATA_VERBATIM, filedata) + self._dir_stack[-1][1].extend(self._pack(VfsRomWriter.ROMFS_RECORD_KIND_FILE, payload)) + + +def _make_romfs(fs, files, data_map): + for filename, contents in files: + if isinstance(contents, tuple): + fs.opendir(filename) + _make_romfs(fs, contents, data_map) + fs.closedir() + elif isinstance(contents, int): + fs.mkfile(filename, data_map[contents]) + else: + fs.mkfile(filename, contents) + + +def make_romfs(files, data=None): + fs = VfsRomWriter() + data_map = {} + if data: + for k, v in data.items(): + data_map[k] = len(v), fs.mkdata(v) + _make_romfs(fs, files, data_map) + return fs.finalise() + + +# A class to test if a value is within a range, needed because MicroPython's range +# doesn't support arbitrary objects. +class Range: + def __init__(self, lower, upper): + self._lower = lower + self._upper = upper + + def __repr__(self): + return "Range({}, {})".format(self._lower, self._upper) + + def __contains__(self, value): + return self._lower <= value < self._upper + + +class TestBase(unittest.TestCase): + @classmethod + def setUpClass(cls): + fs_inner = make_romfs((("test_inner.txt", b"contents_inner"), ("c.py", b""))) + cls.romfs = make_romfs( + ( + ("fs.romfs", 0), + ("test.txt", b"contents"), + ( + "dir", + ( + ("a.py", b"x = 1"), + ("b.py", b"x = 2"), + ("test.mpy", test_mpy), + ), + ), + ), + {0: fs_inner}, + ) + cls.romfs_ilistdir = [ + ("fs.romfs", IFREG, 0, 46), + ("test.txt", IFREG, 0, 8), + ("dir", IFDIR, 0, 198), + ] + cls.romfs_listdir = [x[0] for x in cls.romfs_ilistdir] + cls.romfs_listdir_dir = ["a.py", "b.py", "test.mpy"] + cls.romfs_listdir_bytes = [bytes(x, "ascii") for x in cls.romfs_listdir] + cls.romfs_addr = uctypes.addressof(cls.romfs) + cls.romfs_addr_range = Range(cls.romfs_addr, cls.romfs_addr + len(cls.romfs)) + + +class TestEdgeCases(unittest.TestCase): + def test_empty_romfs(self): + empty_romfs = make_romfs(()) + self.assertEqual(empty_romfs, bytes([0x80 | ord("R"), 0x80 | ord("M"), ord("1"), 0])) + fs = vfs.VfsRom(empty_romfs) + self.assertIsInstance(fs, vfs.VfsRom) + fs.mount(True, False) + self.assertEqual(list(fs.ilistdir("")), []) + self.assertEqual(fs.stat(""), (IFDIR, 0, 0, 0, 0, 0, 0, 0, 0, 0)) + self.assertEqual(fs.statvfs(""), (1, 0, 0, 0, 0, 0, 0, 0, 0, 32767)) + + def test_error(self): + for bad_romfs in (b"", b"xxx", b"not a romfs"): + with self.assertRaises(OSError) as ctx: + vfs.VfsRom(bad_romfs) + self.assertEqual(ctx.exception.errno, errno.ENODEV) + + def test_unknown_record(self): + fs = VfsRomWriter() + fs._extend(fs._pack(VfsRomWriter.ROMFS_RECORD_KIND_PADDING, b"payload")) + fs.mkfile( + "test", + b"contents", + extra_payload=fs._pack(VfsRomWriter.ROMFS_RECORD_KIND_PADDING, b"pad"), + ) + romfs = fs.finalise() + fs = vfs.VfsRom(romfs) + self.assertEqual(list(fs.ilistdir("")), [("test", IFREG, 0, 8)]) + with fs.open("test", "rb") as f: + self.assertEqual(f.read(), b"contents") + + +class TestCorrupt(unittest.TestCase): + def test_corrupt_filesystem(self): + # Make the filesystem length bigger than the buffer. + romfs = bytearray(make_romfs(())) + romfs[3] = 0x01 + with self.assertRaises(OSError): + vfs.VfsRom(romfs) + + # Corrupt the filesystem length. + romfs = bytearray(make_romfs(())) + romfs[3] = 0xFF + with self.assertRaises(OSError): + vfs.VfsRom(romfs) + + # Corrupt the contents of the filesystem. + romfs = bytearray(make_romfs(())) + romfs[3] = 0x01 + romfs.extend(b"\xff\xff") + fs = vfs.VfsRom(romfs) + with self.assertRaises(OSError): + fs.stat("a") + self.assertEqual(list(fs.ilistdir("")), []) + + def test_corrupt_file_entry(self): + romfs = make_romfs((("file", b"data"),)) + + # Corrupt the length of filename. + romfs_corrupt = bytearray(romfs) + romfs_corrupt[7:] = b"\xff" * (len(romfs) - 7) + fs = vfs.VfsRom(romfs_corrupt) + with self.assertRaises(OSError): + fs.stat("file") + self.assertEqual(list(fs.ilistdir("")), []) + + # Erase the data record (change it to a padding record). + romfs_corrupt = bytearray(romfs) + romfs_corrupt[12] = VfsRomWriter.ROMFS_RECORD_KIND_PADDING + fs = vfs.VfsRom(romfs_corrupt) + with self.assertRaises(OSError): + fs.stat("file") + self.assertEqual(list(fs.ilistdir("")), []) + + # Corrupt the header of the data record. + romfs_corrupt = bytearray(romfs) + romfs_corrupt[12:] = b"\xff" * (len(romfs) - 12) + fs = vfs.VfsRom(romfs_corrupt) + with self.assertRaises(OSError): + fs.stat("file") + + # Corrupt the interior of the data record. + romfs_corrupt = bytearray(romfs) + romfs_corrupt[13:] = b"\xff" * (len(romfs) - 13) + fs = vfs.VfsRom(romfs_corrupt) + with self.assertRaises(OSError): + fs.stat("file") + + # Change the data record to an indirect pointer and corrupt the length. + romfs_corrupt = bytearray(romfs) + romfs_corrupt[12] = VfsRomWriter.ROMFS_RECORD_KIND_DATA_POINTER + romfs_corrupt[14:18] = b"\xff\xff\xff\xff" + fs = vfs.VfsRom(romfs_corrupt) + with self.assertRaises(OSError): + fs.stat("file") + + # Change the data record to an indirect pointer and corrupt the offset. + romfs_corrupt = bytearray(romfs) + romfs_corrupt[12] = VfsRomWriter.ROMFS_RECORD_KIND_DATA_POINTER + romfs_corrupt[14:18] = b"\x00\xff\xff\xff" + fs = vfs.VfsRom(romfs_corrupt) + with self.assertRaises(OSError): + fs.stat("file") + + +class TestStandalone(TestBase): + def test_constructor(self): + self.assertIsInstance(vfs.VfsRom(self.romfs), vfs.VfsRom) + with self.assertRaises(TypeError): + vfs.VfsRom(self.romfs_addr) + + def test_mount(self): + vfs.VfsRom(self.romfs).mount(True, False) + with self.assertRaises(OSError): + vfs.VfsRom(self.romfs).mount(True, True) + + def test_ilistdir(self): + fs = vfs.VfsRom(self.romfs) + self.assertEqual(list(fs.ilistdir("")), self.romfs_ilistdir) + self.assertEqual(list(fs.ilistdir("/")), self.romfs_ilistdir) + with self.assertRaises(OSError): + fs.ilistdir("does not exist") + + def test_stat(self): + fs = vfs.VfsRom(self.romfs) + self.assertEqual(fs.stat(""), (IFDIR, 0, 0, 0, 0, 0, 289, 0, 0, 0)) + self.assertEqual(fs.stat("/"), (IFDIR, 0, 0, 0, 0, 0, 289, 0, 0, 0)) + self.assertEqual(fs.stat("/test.txt"), (IFREG, 0, 0, 0, 0, 0, 8, 0, 0, 0)) + self.assertEqual(fs.stat("/dir"), (IFDIR, 0, 0, 0, 0, 0, 198, 0, 0, 0)) + with self.assertRaises(OSError): + fs.stat("/does-not-exist") + + def test_statvfs(self): + fs = vfs.VfsRom(self.romfs) + self.assertEqual(fs.statvfs(""), (1, 0, 289, 0, 0, 0, 0, 0, 0, 32767)) + + def test_open(self): + fs = vfs.VfsRom(self.romfs) + + with fs.open("/test.txt", "") as f: + self.assertEqual(f.read(), "contents") + with fs.open("/test.txt", "rt") as f: + self.assertEqual(f.read(), "contents") + with fs.open("/test.txt", "rb") as f: + self.assertEqual(f.read(), b"contents") + + with self.assertRaises(OSError) as ctx: + fs.open("/file-does-not-exist", "") + self.assertEqual(ctx.exception.errno, errno.ENOENT) + + with self.assertRaises(OSError) as ctx: + fs.open("/dir", "rb") + self.assertEqual(ctx.exception.errno, errno.EISDIR) + + with self.assertRaises(OSError): + fs.open("/test.txt", "w") + with self.assertRaises(OSError): + fs.open("/test.txt", "a") + with self.assertRaises(OSError): + fs.open("/test.txt", "+") + + def test_file_seek(self): + fs = vfs.VfsRom(self.romfs) + with fs.open("/test.txt", "") as f: + self.assertEqual(f.seek(0, SEEK_SET), 0) + self.assertEqual(f.seek(3, SEEK_SET), 3) + self.assertEqual(f.read(), "tents") + self.assertEqual(f.seek(0, SEEK_SET), 0) + self.assertEqual(f.seek(100, SEEK_CUR), 8) + self.assertEqual(f.seek(-1, SEEK_END), 7) + self.assertEqual(f.read(), "s") + self.assertEqual(f.seek(1, SEEK_END), 8) + with self.assertRaises(OSError): + f.seek(-1, SEEK_SET) + f.seek(0, SEEK_SET) + with self.assertRaises(OSError): + f.seek(-1, SEEK_CUR) + with self.assertRaises(OSError): + f.seek(-100, SEEK_END) + + @unittest.skipIf(select is None, "no select module") + def test_file_ioctl_invalid(self): + fs = vfs.VfsRom(self.romfs) + with fs.open("/test.txt", "") as f: + p = select.poll() + p.register(f) + with self.assertRaises(OSError): + p.poll(0) + + def test_memory_mapping(self): + fs = vfs.VfsRom(self.romfs) + with fs.open("/test.txt", "rb") as f: + addr = uctypes.addressof(f) + data = memoryview(f) + self.assertIn(addr, self.romfs_addr_range) + self.assertIn(addr + len(data), self.romfs_addr_range) + self.assertEqual(bytes(data), b"contents") + + +class TestMounted(TestBase): + def setUp(self): + self.orig_sys_path = list(sys.path) + self.orig_cwd = os.getcwd() + vfs.mount(vfs.VfsRom(self.romfs), "/test_rom") + + def tearDown(self): + vfs.umount("/test_rom") + os.chdir(self.orig_cwd) + sys.path = self.orig_sys_path + + def test_listdir(self): + self.assertEqual(os.listdir("/test_rom"), self.romfs_listdir) + self.assertEqual(os.listdir("/test_rom/dir"), self.romfs_listdir_dir) + self.assertEqual(os.listdir(b"/test_rom"), self.romfs_listdir_bytes) + + def test_chdir(self): + os.chdir("/test_rom") + self.assertEqual(os.getcwd(), "/test_rom") + self.assertEqual(os.listdir(), self.romfs_listdir) + + os.chdir("/test_rom/") + self.assertEqual(os.getcwd(), "/test_rom") + self.assertEqual(os.listdir(), self.romfs_listdir) + + # chdir within the romfs is not implemented. + with self.assertRaises(OSError): + os.chdir("/test_rom/dir") + + def test_stat(self): + self.assertEqual(os.stat("/test_rom"), (IFDIR, 0, 0, 0, 0, 0, 289, 0, 0, 0)) + self.assertEqual(os.stat("/test_rom/"), (IFDIR, 0, 0, 0, 0, 0, 289, 0, 0, 0)) + self.assertEqual(os.stat("/test_rom/test.txt"), (IFREG, 0, 0, 0, 0, 0, 8, 0, 0, 0)) + self.assertEqual(os.stat("/test_rom/dir"), (IFDIR, 0, 0, 0, 0, 0, 198, 0, 0, 0)) + with self.assertRaises(OSError): + os.stat("/test_rom/does-not-exist") + + def test_open(self): + with open("/test_rom/test.txt") as f: + self.assertEqual(f.read(), "contents") + with open("/test_rom/test.txt", "b") as f: + self.assertEqual(f.read(), b"contents") + + with self.assertRaises(OSError) as ctx: + open("/test_rom/file-does-not-exist") + self.assertEqual(ctx.exception.errno, errno.ENOENT) + + with self.assertRaises(OSError) as ctx: + open("/test_rom/dir") + self.assertEqual(ctx.exception.errno, errno.EISDIR) + + def test_import_py(self): + sys.path.append("/test_rom/dir") + a = __import__("a") + b = __import__("b") + self.assertEqual(a.__file__, "/test_rom/dir/a.py") + self.assertEqual(a.x, 1) + self.assertEqual(b.__file__, "/test_rom/dir/b.py") + self.assertEqual(b.x, 2) + + def test_import_mpy(self): + sys.path.append("/test_rom/dir") + test = __import__("test") + self.assertEqual(test.__file__, "/test_rom/dir/test.mpy") + self.assertEqual(test.str_obj, "this is a str object") + self.assertEqual(test.bytes_obj, b"this is a bytes object") + self.assertEqual(test.int_obj, 1234567890) + self.assertEqual(test.float_obj, 1.23) + self.assertIn(uctypes.addressof(test.str_obj), self.romfs_addr_range) + self.assertIn(uctypes.addressof(test.bytes_obj), self.romfs_addr_range) + + def test_romfs_inner(self): + with open("/test_rom/fs.romfs", "rb") as f: + romfs_inner = vfs.VfsRom(memoryview(f)) + + vfs.mount(romfs_inner, "/test_rom2") + + self.assertEqual(os.listdir("/test_rom2"), ["test_inner.txt", "c.py"]) + + sys.path.append("/test_rom2") + self.assertEqual(__import__("c").__file__, "/test_rom2/c.py") + + with open("/test_rom2/test_inner.txt") as f: + self.assertEqual(f.read(), "contents_inner") + + with open("/test_rom2/test_inner.txt", "rb") as f: + addr = uctypes.addressof(f) + data = memoryview(f) + self.assertIn(addr, self.romfs_addr_range) + self.assertIn(addr + len(data), self.romfs_addr_range) + + vfs.umount("/test_rom2") + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/extmod_hardware/machine_pwm.py b/tests/extmod_hardware/machine_pwm.py new file mode 100644 index 0000000000000..e27da32548659 --- /dev/null +++ b/tests/extmod_hardware/machine_pwm.py @@ -0,0 +1,166 @@ +# Test machine.PWM, frequency and duty cycle (using machine.time_pulse_us). +# +# IMPORTANT: This test requires hardware connections: the PWM-output and pulse-input +# pins must be wired together (see the variable `pwm_pulse_pins`). + +import sys +import time + +try: + from machine import time_pulse_us, Pin, PWM +except ImportError: + print("SKIP") + raise SystemExit + +import unittest + +pwm_freq_limit = 1000000 +freq_margin_per_thousand = 0 +duty_margin_per_thousand = 0 +timing_margin_us = 5 + +# Configure pins based on the target. +if "esp32" in sys.platform: + pwm_pulse_pins = ((4, 5),) + freq_margin_per_thousand = 2 + duty_margin_per_thousand = 1 + timing_margin_us = 20 +elif "esp8266" in sys.platform: + pwm_pulse_pins = ((4, 5),) + pwm_freq_limit = 1_000 + duty_margin_per_thousand = 3 + timing_margin_us = 50 +elif "mimxrt" in sys.platform: + if "Teensy" in sys.implementation._machine: + # Teensy 4.x + pwm_pulse_pins = ( + ("D0", "D1"), # FLEXPWM X and UART 1 + ("D2", "D3"), # FLEXPWM A/B + ("D11", "D12"), # QTMR and MOSI/MISO of SPI 0 + ) + else: + pwm_pulse_pins = (("D0", "D1"),) +elif "rp2" in sys.platform: + pwm_pulse_pins = (("GPIO0", "GPIO1"),) +elif "samd" in sys.platform: + pwm_pulse_pins = (("D0", "D1"),) + if "SAMD21" in sys.implementation._machine: + # MCU is too slow to capture short pulses. + pwm_freq_limit = 2_000 +else: + print("Please add support for this test on this platform.") + raise SystemExit + + +# Test a specific frequency and duty cycle. +def _test_freq_duty(self, pulse_in, pwm, freq, duty_u16): + print("freq={:<5} duty_u16={:<5} :".format(freq, duty_u16), end="") + + # Check configured freq/duty_u16 is within error bound. + freq_error = abs(pwm.freq() - freq) * 1000 // freq + duty_error = abs(pwm.duty_u16() - duty_u16) * 1000 // (duty_u16 or 1) + print(" freq={} freq_er={}".format(pwm.freq(), freq_error), end="") + print(" duty={} duty_er={}".format(pwm.duty_u16(), duty_error), end="") + print(" :", end="") + self.assertLessEqual(freq_error, freq_margin_per_thousand) + self.assertLessEqual(duty_error, duty_margin_per_thousand) + + # Calculate expected timing. + expected_total_us = 1_000_000 // freq + expected_high_us = expected_total_us * duty_u16 // 65535 + expected_low_us = expected_total_us - expected_high_us + expected_us = (expected_low_us, expected_high_us) + timeout = 2 * expected_total_us + + # Wait for output to settle. + time_pulse_us(pulse_in, 0, timeout) + time_pulse_us(pulse_in, 1, timeout) + + if duty_u16 == 0 or duty_u16 == 65535: + # Expect a constant output level. + no_pulse = ( + time_pulse_us(pulse_in, 0, timeout) < 0 and time_pulse_us(pulse_in, 1, timeout) < 0 + ) + self.assertTrue(no_pulse) + if expected_high_us == 0: + # Expect a constant low level. + self.assertEqual(pulse_in(), 0) + else: + # Expect a constant high level. + self.assertEqual(pulse_in(), 1) + else: + # Test timing of low and high pulse. + n_averaging = 10 + for level in (0, 1): + t = 0 + time_pulse_us(pulse_in, level, timeout) + for _ in range(n_averaging): + t += time_pulse_us(pulse_in, level, timeout) + t //= n_averaging + expected = expected_us[level] + print(" level={} timing_er={}".format(level, abs(t - expected)), end="") + self.assertLessEqual(abs(t - expected), timing_margin_us) + + print() + + +# Test a specific frequency with multiple duty cycles. +def _test_freq(self, freq): + print() + self.pwm.freq(freq) + for duty in (0, 10, 25, 50, 75, 90, 100): + duty_u16 = duty * 65535 // 100 + if sys.platform == "esp32": + # TODO why is this bit needed to get it working on esp32? + self.pwm.init(freq=freq, duty_u16=duty_u16) + time.sleep(0.1) + self.pwm.duty_u16(duty_u16) + _test_freq_duty(self, self.pulse_in, self.pwm, freq, duty_u16) + + +# Given a set of pins, this test class will test multiple frequencies and duty cycles. +class TestBase: + @classmethod + def setUpClass(cls): + print("set up pins:", cls.pwm_pin, cls.pulse_pin) + cls.pwm = PWM(cls.pwm_pin) + cls.pulse_in = Pin(cls.pulse_pin, Pin.IN) + + @classmethod + def tearDownClass(cls): + cls.pwm.deinit() + + def test_freq_50(self): + _test_freq(self, 50) + + def test_freq_100(self): + _test_freq(self, 100) + + def test_freq_500(self): + _test_freq(self, 500) + + def test_freq_1000(self): + _test_freq(self, 1000) + + @unittest.skipIf(pwm_freq_limit < 2000, "frequency too high") + def test_freq_2000(self): + _test_freq(self, 2000) + + @unittest.skipIf(pwm_freq_limit < 5000, "frequency too high") + def test_freq_5000(self): + _test_freq(self, 5000) + + @unittest.skipIf(pwm_freq_limit < 10000, "frequency too high") + def test_freq_10000(self): + _test_freq(self, 10000) + + +# Generate test classes, one for each set of pins to test. +for pwm, pulse in pwm_pulse_pins: + cls_name = "Test_{}_{}".format(pwm, pulse) + globals()[cls_name] = type( + cls_name, (TestBase, unittest.TestCase), {"pwm_pin": pwm, "pulse_pin": pulse} + ) + +if __name__ == "__main__": + unittest.main() diff --git a/tests/feature_check/inlineasm_rv32.py b/tests/feature_check/inlineasm_rv32.py new file mode 100644 index 0000000000000..21dd103b6c3fe --- /dev/null +++ b/tests/feature_check/inlineasm_rv32.py @@ -0,0 +1,9 @@ +# check if RISC-V 32 inline asm is supported + + +@micropython.asm_rv32 +def f(): + add(a0, a0, a0) + + +print("rv32") diff --git a/tests/feature_check/inlineasm_rv32.py.exp b/tests/feature_check/inlineasm_rv32.py.exp new file mode 100644 index 0000000000000..5eecf09c22403 --- /dev/null +++ b/tests/feature_check/inlineasm_rv32.py.exp @@ -0,0 +1 @@ +rv32 diff --git a/tests/feature_check/inlineasm_thumb.py b/tests/feature_check/inlineasm_thumb.py new file mode 100644 index 0000000000000..321eab0e2f87a --- /dev/null +++ b/tests/feature_check/inlineasm_thumb.py @@ -0,0 +1,9 @@ +# check if Thumb inline asm is supported + + +@micropython.asm_thumb +def f(): + nop() + + +print("thumb") diff --git a/tests/feature_check/inlineasm_thumb.py.exp b/tests/feature_check/inlineasm_thumb.py.exp new file mode 100644 index 0000000000000..bb48e1a2f03c2 --- /dev/null +++ b/tests/feature_check/inlineasm_thumb.py.exp @@ -0,0 +1 @@ +thumb diff --git a/tests/feature_check/inlineasm_xtensa.py b/tests/feature_check/inlineasm_xtensa.py new file mode 100644 index 0000000000000..2a24d39973c8c --- /dev/null +++ b/tests/feature_check/inlineasm_xtensa.py @@ -0,0 +1,9 @@ +# check if Xtensa inline asm is supported + + +@micropython.asm_xtensa +def f(): + ret_n() + + +print("xtensa") diff --git a/tests/feature_check/inlineasm_xtensa.py.exp b/tests/feature_check/inlineasm_xtensa.py.exp new file mode 100644 index 0000000000000..036142c5097b5 --- /dev/null +++ b/tests/feature_check/inlineasm_xtensa.py.exp @@ -0,0 +1 @@ +xtensa diff --git a/tests/feature_check/target_info.py b/tests/feature_check/target_info.py index df89496708aa9..f60f3b319192e 100644 --- a/tests/feature_check/target_info.py +++ b/tests/feature_check/target_info.py @@ -4,6 +4,7 @@ import sys +platform = getattr(sys, "platform", "minimal") sys_mpy = getattr(sys.implementation, "_mpy", 0) arch = [ None, @@ -19,4 +20,4 @@ "xtensawin", "rv32imc", ][sys_mpy >> 10] -print(arch) +print(platform, arch) diff --git a/tests/float/float_parse.py b/tests/float/float_parse.py index de27c33e7be57..6131da0a63ab9 100644 --- a/tests/float/float_parse.py +++ b/tests/float/float_parse.py @@ -31,6 +31,9 @@ print(float("1e18446744073709551621")) print(float("1e-18446744073709551621")) +# mantissa overflow while processing deferred trailing zeros +print(float("10000000000000000000001")) + # check small decimals are as close to their true value as possible for n in range(1, 10): print(float("0.%u" % n) == n / 10) diff --git a/tests/inlineasm/rv32/asmargs.py b/tests/inlineasm/rv32/asmargs.py new file mode 100644 index 0000000000000..78afd511150d6 --- /dev/null +++ b/tests/inlineasm/rv32/asmargs.py @@ -0,0 +1,44 @@ +# test passing arguments + + +@micropython.asm_rv32 +def arg0(): + c_li(a0, 1) + + +print(arg0()) + + +@micropython.asm_rv32 +def arg1(a0): + addi(a0, a0, 1) + + +print(arg1(1)) + + +@micropython.asm_rv32 +def arg2(a0, a1): + add(a0, a0, a1) + + +print(arg2(1, 2)) + + +@micropython.asm_rv32 +def arg3(a0, a1, a2): + add(a0, a0, a1) + add(a0, a0, a2) + + +print(arg3(1, 2, 3)) + + +@micropython.asm_rv32 +def arg4(a0, a1, a2, a3): + add(a0, a0, a1) + add(a0, a0, a2) + add(a0, a0, a3) + + +print(arg4(1, 2, 3, 4)) diff --git a/tests/inlineasm/asmargs.py.exp b/tests/inlineasm/rv32/asmargs.py.exp similarity index 100% rename from tests/inlineasm/asmargs.py.exp rename to tests/inlineasm/rv32/asmargs.py.exp diff --git a/tests/inlineasm/rv32/asmarith.py b/tests/inlineasm/rv32/asmarith.py new file mode 100644 index 0000000000000..8b864c0b3b289 --- /dev/null +++ b/tests/inlineasm/rv32/asmarith.py @@ -0,0 +1,79 @@ +# test arithmetic opcodes + + +@micropython.asm_rv32 +def f1(): + li(a0, 0x100) + li(a1, 1) + add(a0, a0, a1) + addi(a0, a0, 1) + addi(a0, a0, -2) + sub(a0, a0, a1) + c_add(a0, a1) + c_addi(a0, -1) + c_sub(a0, a1) + + +print(hex(f1())) + + +@micropython.asm_rv32 +def f2(): + li(a0, 0x10FF) + li(a1, 1) + and_(a2, a0, a1) + andi(a3, a0, 0x10) + or_(a2, a2, a3) + ori(a2, a2, 8) + li(a1, 0x200) + c_or(a2, a1) + li(a1, 0xF0) + mv(a0, a2) + c_and(a0, a1) + li(a1, 0x101) + xor(a0, a0, a1) + xori(a0, a0, 0x101) + c_xor(a0, a1) + + +print(hex(f2())) + + +@micropython.asm_rv32 +def f3(a0, a1): + slt(a0, a0, a1) + + +print(f3(0xFFFFFFF0, 0xFFFFFFF1)) +print(f3(0x0, 0xFFFFFFF1)) +print(f3(0xFFFFFFF1, 0xFFFFFFF1)) +print(f3(0xFFFFFFF1, 0xFFFFFFF0)) + + +@micropython.asm_rv32 +def f4(a0, a1): + sltu(a0, a0, a1) + + +print(f3(0xFFFFFFF0, 0xFFFFFFF1)) +print(f3(0x0, 0xFFFFFFF1)) +print(f3(0xFFFFFFF1, 0xFFFFFFF1)) +print(f3(0xFFFFFFF1, 0xFFFFFFF0)) + + +@micropython.asm_rv32 +def f5(a0): + slti(a0, a0, -2) + + +print(f5(-1)) +print(f5(-3)) + + +@micropython.asm_rv32 +def f6(a0): + sltiu(a0, a0, -2) + + +print(f6(-1)) +print(f6(-3)) diff --git a/tests/inlineasm/rv32/asmarith.py.exp b/tests/inlineasm/rv32/asmarith.py.exp new file mode 100644 index 0000000000000..7da4dd5c93c40 --- /dev/null +++ b/tests/inlineasm/rv32/asmarith.py.exp @@ -0,0 +1,14 @@ +0xfe +0x111 +1 +0 +0 +0 +1 +0 +0 +0 +0 +1 +0 +1 diff --git a/tests/inlineasm/rv32/asmbranch.py b/tests/inlineasm/rv32/asmbranch.py new file mode 100644 index 0000000000000..d7d059d4067c6 --- /dev/null +++ b/tests/inlineasm/rv32/asmbranch.py @@ -0,0 +1,161 @@ +# test branch instructions + + +@micropython.asm_rv32 +def tbeq(a0): + mv(a1, a0) + + li(a0, 10) + li(a2, 1) + beq(a1, a2, end) + + li(a0, 20) + li(a2, 2) + beq(a1, a2, end) + + li(a0, 30) + li(a2, 3) + beq(a1, a2, end) + + li(a0, 0) + + label(end) + + +print(tbeq(0)) +print(tbeq(1)) +print(tbeq(2)) +print(tbeq(3)) + + +@micropython.asm_rv32 +def tbne(a0): + mv(a1, a0) + + li(a0, 10) + li(a2, 1) + bne(a1, a2, end) + + li(a0, 20) + li(a2, 2) + bne(a1, a2, end) + + li(a0, 30) + li(a2, 3) + bne(a1, a2, end) + + li(a0, 0) + + label(end) + + +print(tbne(0)) +print(tbne(1)) +print(tbne(2)) +print(tbne(3)) + + +@micropython.asm_rv32 +def tbgeu(a0): + mv(a1, a0) + + li(a0, 1) + li(a2, 2) + bgeu(a1, a2, end) + li(a0, 0) + + label(end) + + +print(tbgeu(0)) +print(tbgeu(1)) +print(tbgeu(2)) +print(tbgeu(3)) + + +@micropython.asm_rv32 +def tbltu(a0): + mv(a1, a0) + + li(a0, 1) + li(a2, 2) + bltu(a1, a2, end) + li(a0, 0) + + label(end) + + +print(tbltu(0)) +print(tbltu(1)) +print(tbltu(2)) +print(tbltu(3)) + + +@micropython.asm_rv32 +def tbge(a0): + mv(a1, a0) + + li(a0, 1) + li(a2, -2) + bge(a1, a2, end) + li(a0, 0) + + label(end) + + +print(tbge(-3)) +print(tbge(-2)) +print(tbge(-1)) +print(tbge(0)) + + +@micropython.asm_rv32 +def tblt(a0): + mv(a1, a0) + + li(a0, 1) + li(a2, -2) + blt(a1, a2, end) + li(a0, 0) + + label(end) + + +print(tblt(-3)) +print(tblt(-2)) +print(tblt(-1)) +print(tblt(0)) + + +@micropython.asm_rv32 +def tcbeqz(a0): + mv(a1, a0) + + li(a0, 1) + c_beqz(a1, end) + li(a0, 0) + + label(end) + + +print(tcbeqz(0)) +print(tcbeqz(1)) +print(tcbeqz(2)) +print(tcbeqz(3)) + + +@micropython.asm_rv32 +def tcbnez(a0): + mv(a1, a0) + + li(a0, 1) + c_bnez(a1, end) + li(a0, 0) + + label(end) + + +print(tcbnez(0)) +print(tcbnez(1)) +print(tcbnez(2)) +print(tcbnez(3)) diff --git a/tests/inlineasm/rv32/asmbranch.py.exp b/tests/inlineasm/rv32/asmbranch.py.exp new file mode 100644 index 0000000000000..baae69149538e --- /dev/null +++ b/tests/inlineasm/rv32/asmbranch.py.exp @@ -0,0 +1,32 @@ +0 +10 +20 +30 +10 +20 +10 +10 +0 +0 +1 +1 +1 +1 +0 +0 +0 +1 +1 +1 +1 +0 +0 +0 +1 +0 +0 +0 +0 +1 +1 +1 diff --git a/tests/inlineasm/rv32/asmconst.py b/tests/inlineasm/rv32/asmconst.py new file mode 100644 index 0000000000000..2b6363a43db95 --- /dev/null +++ b/tests/inlineasm/rv32/asmconst.py @@ -0,0 +1,49 @@ +# test constants in assembler + + +@micropython.asm_rv32 +def c1(): + li(a0, 0xFFFFFFFF) + li(a1, 0xF0000000) + sub(a0, a0, a1) + + +print(hex(c1())) + + +@micropython.asm_rv32 +def c2(): + lui(a0, 0x12345) + li(a1, 0x678) + add(a0, a0, a1) + + +print(hex(c2())) + + +@micropython.asm_rv32 +def c3() -> uint: + lui(a0, 0) + addi(a0, a0, 0x7FF) + + +print(hex(c3())) + + +@micropython.asm_rv32 +def c4() -> uint: + lui(a0, 0) + addi(a0, a0, -1) + + +print(hex(c4())) + + +@micropython.asm_rv32 +def c5(): + c_lui(a0, 1) + c_li(a1, 1) + c_add(a0, a1) + + +print(hex(c5())) diff --git a/tests/inlineasm/rv32/asmconst.py.exp b/tests/inlineasm/rv32/asmconst.py.exp new file mode 100644 index 0000000000000..0c713a841486b --- /dev/null +++ b/tests/inlineasm/rv32/asmconst.py.exp @@ -0,0 +1,5 @@ +0xfffffff +0x12345678 +0x7ff +0xffffffff +0x1001 diff --git a/tests/inlineasm/rv32/asmcsr.py b/tests/inlineasm/rv32/asmcsr.py new file mode 100644 index 0000000000000..f27e2aa5e34ec --- /dev/null +++ b/tests/inlineasm/rv32/asmcsr.py @@ -0,0 +1,65 @@ +# test csr instructions + +# CSR 0x340 is `mscratch`. This test suite is only safe to run on a system +# where it is known that there is no other code that can read from or write +# to that register. The qemu port is one such system, as the CSR is only +# accessed when a machine exception occurs, and at that point it doesn't matter +# anymore whether these tests are running or not. + + +@micropython.asm_rv32 +def csr(): + li(a0, 0) + csrrw(zero, zero, 0x340) # All zeroes + csrrs(a1, zero, 0x340) # Read zeroes + c_bnez(a1, end) + addi(a0, a0, 1) + li(a1, 0xA5A5A5A5) + li(a2, 0x5A5A5A5A) + csrrs(a2, a1, 0x340) # Read zeroes, set 0xA5A5A5A5 + c_bnez(a2, end) + addi(a0, a0, 1) + csrrs(a3, zero, 0x340) # Read 0xA5A5A5A5 + bne(a3, a1, end) + addi(a0, a0, 1) + li(a2, 0xF0F0F0F0) + csrrc(zero, a2, 0x340) # Clear upper half + csrrs(a3, zero, 0x340) # Read 0x05050505 + xori(a2, a2, -1) + and_(a2, a1, a2) + bne(a2, a3, end) + addi(a0, a0, 1) + label(end) + + +print(csr()) + + +@micropython.asm_rv32 +def csri(): + li(a0, 0) + csrrwi(zero, 0x340, 15) # Write 0xF + csrrs(a1, zero, 0x340) # Read 0xF + csrrsi(a2, 0x340, 0) # Read + bne(a1, a2, end) + addi(a0, a0, 1) + csrrci(a2, 0x340, 0) # Read + bne(a1, a2, end) + addi(a0, a0, 1) + li(a2, 15) + bne(a1, a2, end) + addi(a0, a0, 1) + csrrci(zero, 0x340, 1) # Clear bit 1 + csrrs(a1, zero, 0x340) # Read 0xE + li(a2, 14) + bne(a1, a2, end) + addi(a0, a0, 1) + csrrsi(zero, 0x340, 1) # Set bit 1 + csrrs(a1, zero, 0x340) # Read 0xF + li(a2, 15) + bne(a1, a2, end) + addi(a0, a0, 1) + label(end) + + +print(csri()) diff --git a/tests/inlineasm/rv32/asmcsr.py.exp b/tests/inlineasm/rv32/asmcsr.py.exp new file mode 100644 index 0000000000000..61c83cba41ce3 --- /dev/null +++ b/tests/inlineasm/rv32/asmcsr.py.exp @@ -0,0 +1,2 @@ +4 +5 diff --git a/tests/inlineasm/rv32/asmdata.py b/tests/inlineasm/rv32/asmdata.py new file mode 100644 index 0000000000000..5e555ef4bf465 --- /dev/null +++ b/tests/inlineasm/rv32/asmdata.py @@ -0,0 +1,33 @@ +# test the "data" directive + + +@micropython.asm_rv32 +def ret_num(a0) -> uint: + slli(a0, a0, 2) + addi(a0, a0, 16) + auipc(a1, 0) + add(a1, a1, a0) + lw(a0, 0(a1)) + jal(zero, HERE) + data(4, 0x12345678, 0x20000000, 0x40000000, 0x7FFFFFFF + 1, (1 << 32) - 2) + label(HERE) + + +for i in range(5): + print(hex(ret_num(i))) + + +@micropython.asm_rv32 +def ret_num_la(a0) -> uint: + slli(a0, a0, 2) + la(a1, DATA) + add(a1, a1, a0) + lw(a0, 0(a1)) + jal(zero, HERE) + label(DATA) + data(4, 0x12345678, 0x20000000, 0x40000000, 0x7FFFFFFF + 1, (1 << 32) - 2) + label(HERE) + + +for i in range(5): + print(hex(ret_num_la(i))) diff --git a/tests/inlineasm/rv32/asmdata.py.exp b/tests/inlineasm/rv32/asmdata.py.exp new file mode 100644 index 0000000000000..79e92bdfa5de5 --- /dev/null +++ b/tests/inlineasm/rv32/asmdata.py.exp @@ -0,0 +1,10 @@ +0x12345678 +0x20000000 +0x40000000 +0x80000000 +0xfffffffe +0x12345678 +0x20000000 +0x40000000 +0x80000000 +0xfffffffe diff --git a/tests/inlineasm/rv32/asmdivmul.py b/tests/inlineasm/rv32/asmdivmul.py new file mode 100644 index 0000000000000..e1120c6f63cef --- /dev/null +++ b/tests/inlineasm/rv32/asmdivmul.py @@ -0,0 +1,63 @@ +@micropython.asm_rv32 +def sdiv(a0, a1): + div(a0, a0, a1) + + +@micropython.asm_rv32 +def udiv(a0, a1): + divu(a0, a0, a1) + + +@micropython.asm_rv32 +def srem(a0, a1): + rem(a0, a0, a1) + + +@micropython.asm_rv32 +def urem(a0, a1): + remu(a0, a0, a1) + + +print(sdiv(1234, 3)) +print(sdiv(-1234, 3)) +print(sdiv(1234, -3)) +print(sdiv(-1234, -3)) + +print(udiv(1234, 3)) +print(udiv(0xFFFFFFFF, 0x7FFFFFFF)) +print(udiv(0xFFFFFFFF, 0xFFFFFFFF)) + +print(srem(1234, 3)) +print(srem(-1234, 3)) +print(srem(1234, -3)) +print(srem(-1234, -3)) + +print(urem(1234, 3)) +print(urem(0xFFFFFFFF, 0x7FFFFFFF)) +print(urem(0xFFFFFFFF, 0xFFFFFFFF)) + + +@micropython.asm_rv32 +def m1(a0, a1): + mul(a0, a0, a1) + + +@micropython.asm_rv32 +def m2(a0, a1): + mulh(a0, a0, a1) + + +@micropython.asm_rv32 +def m3(a0, a1): + mulhu(a0, a0, a1) + + +@micropython.asm_rv32 +def m4(a0, a1): + mulhsu(a0, a0, a1) + + +print(m1(0xFFFFFFFF, 2)) +print(m2(0xFFFFFFFF, 0xFFFFFFF0)) +print(m3(0xFFFFFFFF, 0xFFFFFFF0)) +print(m4(0xFFFFFFFF, 0xFFFFFFF0)) diff --git a/tests/inlineasm/rv32/asmdivmul.py.exp b/tests/inlineasm/rv32/asmdivmul.py.exp new file mode 100644 index 0000000000000..60d28635f792b --- /dev/null +++ b/tests/inlineasm/rv32/asmdivmul.py.exp @@ -0,0 +1,18 @@ +411 +-411 +-411 +411 +411 +2 +1 +1 +-1 +1 +-1 +1 +1 +0 +-2 +0 +-17 +-1 diff --git a/tests/inlineasm/rv32/asmjump.py b/tests/inlineasm/rv32/asmjump.py new file mode 100644 index 0000000000000..fe87d3f968b37 --- /dev/null +++ b/tests/inlineasm/rv32/asmjump.py @@ -0,0 +1,115 @@ +@micropython.asm_rv32 +def f1(): + li(a0, 0) + la(a1, END) + c_jr(a1) + c_addi(a0, 1) + c_addi(a0, 1) + c_addi(a0, 1) + c_addi(a0, 1) + c_addi(a0, 1) + label(END) + + +print(f1()) + + +@micropython.asm_rv32 +def f2(): + addi(sp, sp, -4) + c_swsp(ra, 0) + li(ra, 0) + li(a0, 0) + c_jal(END) + c_addi(a0, 1) + c_addi(a0, 1) + c_addi(a0, 1) + c_addi(a0, 1) + c_addi(a0, 1) + label(END) + bne(ra, zero, SUCCESS) + c_addi(a0, 2) + label(SUCCESS) + c_lwsp(ra, 0) + addi(sp, sp, 4) + + +print(f2()) + + +@micropython.asm_rv32 +def f3(): + li(a0, 0) + c_j(END) + c_addi(a0, 1) + c_addi(a0, 1) + c_addi(a0, 1) + c_addi(a0, 1) + c_addi(a0, 1) + label(END) + + +print(f3()) + + +@micropython.asm_rv32 +def f4(): + addi(sp, sp, -4) + c_swsp(ra, 0) + li(ra, 0) + li(a0, 0) + la(a1, END) + c_jalr(a1) + c_addi(a0, 1) + c_addi(a0, 1) + c_addi(a0, 1) + c_addi(a0, 1) + c_addi(a0, 1) + label(END) + bne(ra, zero, SUCCESS) + c_addi(a0, 2) + label(SUCCESS) + c_lwsp(ra, 0) + addi(sp, sp, 4) + + +print(f4()) + + +@micropython.asm_rv32 +def f5(): + li(a0, 0) + li(a1, 0) + jal(a1, END) + c_addi(a0, 1) + c_addi(a0, 1) + c_addi(a0, 1) + c_addi(a0, 1) + c_addi(a0, 1) + label(END) + bne(a1, zero, SUCCESS) + c_addi(a0, 2) + label(SUCCESS) + + +print(f5()) + + +@micropython.asm_rv32 +def f6(): + li(a0, 0) + la(a1, JUMP) + li(a2, 0) + jalr(a2, a1, 10) + label(JUMP) + c_addi(a0, 1) + c_addi(a0, 1) + c_addi(a0, 1) + c_addi(a0, 1) + c_addi(a0, 1) + bne(a2, zero, SUCCESS) + c_addi(a0, 2) + label(SUCCESS) + + +print(f6()) diff --git a/tests/inlineasm/rv32/asmjump.py.exp b/tests/inlineasm/rv32/asmjump.py.exp new file mode 100644 index 0000000000000..f7eb44d66e0b1 --- /dev/null +++ b/tests/inlineasm/rv32/asmjump.py.exp @@ -0,0 +1,6 @@ +0 +0 +0 +0 +0 +0 diff --git a/tests/inlineasm/rv32/asmloadstore.py b/tests/inlineasm/rv32/asmloadstore.py new file mode 100644 index 0000000000000..2c49e07b41a5c --- /dev/null +++ b/tests/inlineasm/rv32/asmloadstore.py @@ -0,0 +1,86 @@ +# test load/store opcodes + + +@micropython.asm_rv32 +def l(): + li(a5, 4) + addi(sp, sp, -12) + li(a0, 0x123) + c_swsp(a0, 0) + addi(a1, a0, 0x111) + c_swsp(a1, 4) + addi(a2, a1, 0x111) + c_swsp(a2, 8) + mv(a4, sp) + c_lw(a3, 0(a4)) + bne(a3, a0, END) + addi(a5, a5, -1) + lw(a3, 4(a4)) + bne(a3, a1, END) + addi(a5, a5, -1) + lhu(a3, 8(a4)) + bne(a3, a2, END) + addi(a5, a5, -1) + lbu(a0, 8(a4)) + addi(a0, a0, 0x300) + bne(a0, a2, END) + addi(a5, a5, -1) + label(END) + addi(sp, sp, 12) + mv(a0, a5) + + +print(l()) + + +@micropython.asm_rv32 +def s(): + li(a5, 4) + addi(sp, sp, -12) + c_swsp(zero, 0) + c_swsp(zero, 4) + c_swsp(zero, 8) + li(a0, 0x12345) + mv(a4, sp) + c_sw(a0, 0(a4)) + sh(a0, 4(a4)) + sb(a0, 8(a4)) + li(a1, 0xFFFF) + and_(a1, a0, a1) + andi(a2, a0, 0xFF) + lw(a3, 0(sp)) + bne(a3, a0, END) + addi(a5, a5, -1) + lw(a3, 4(sp)) + bne(a3, a1, END) + addi(a5, a5, -1) + lw(a3, 8(sp)) + bne(a3, a2, END) + addi(a5, a5, -1) + label(END) + addi(sp, sp, 12) + mv(a0, a5) + + +print(s()) + + +@micropython.asm_rv32 +def lu(): + li(a5, 4) + addi(sp, sp, -8) + li(a0, 0xF1234567) + c_swsp(a0, 0) + c_swsp(a0, 4) + lh(a1, 0(sp)) + blt(a1, zero, END) + addi(a5, a5, -1) + lb(a2, 4(sp)) + blt(a2, zero, END) + addi(a5, a5, -1) + label(END) + addi(sp, sp, 8) + mv(a0, a5) + + +print(lu()) diff --git a/tests/inlineasm/rv32/asmloadstore.py.exp b/tests/inlineasm/rv32/asmloadstore.py.exp new file mode 100644 index 0000000000000..4539bbf2d22d5 --- /dev/null +++ b/tests/inlineasm/rv32/asmloadstore.py.exp @@ -0,0 +1,3 @@ +0 +1 +2 diff --git a/tests/inlineasm/rv32/asmrettype.py b/tests/inlineasm/rv32/asmrettype.py new file mode 100644 index 0000000000000..fc7ae61d15225 --- /dev/null +++ b/tests/inlineasm/rv32/asmrettype.py @@ -0,0 +1,33 @@ +# test return type of inline asm + + +@micropython.asm_rv32 +def ret_obj(a0) -> object: + pass + + +ret_obj(print)(1) + + +@micropython.asm_rv32 +def ret_bool(a0) -> bool: + pass + + +print(ret_bool(0), ret_bool(1)) + + +@micropython.asm_rv32 +def ret_int(a0) -> int: + slli(a0, a0, 29) + + +print(ret_int(0), hex(ret_int(1)), hex(ret_int(2)), hex(ret_int(4))) + + +@micropython.asm_rv32 +def ret_uint(a0) -> uint: + slli(a0, a0, 29) + + +print(ret_uint(0), hex(ret_uint(1)), hex(ret_uint(2)), hex(ret_uint(4))) diff --git a/tests/inlineasm/asmrettype.py.exp b/tests/inlineasm/rv32/asmrettype.py.exp similarity index 100% rename from tests/inlineasm/asmrettype.py.exp rename to tests/inlineasm/rv32/asmrettype.py.exp diff --git a/tests/inlineasm/rv32/asmsanity.py b/tests/inlineasm/rv32/asmsanity.py new file mode 100644 index 0000000000000..1a16d3504dbfe --- /dev/null +++ b/tests/inlineasm/rv32/asmsanity.py @@ -0,0 +1,204 @@ +TEMPLATE3 = """ +@micropython.asm_rv32 +def f(): + {}({}, {}, {}) +""" + +TEMPLATE2 = """ +@micropython.asm_rv32 +def f(): + {}({}, {}) +""" + +TEMPLATE1 = """ +@micropython.asm_rv32 +def f(): + {}({}) +""" + + +REGISTERS = [ + "zero", + "s0", + "s1", + "s2", + "s3", + "s4", + "s5", + "s6", + "s7", + "s8", + "s9", + "s10", + "s11", + "a0", + "a1", + "a2", + "a3", + "a4", + "a5", + "a6", + "a7", + "tp", + "gp", + "sp", + "ra", + "t0", + "t1", + "t2", + "t3", + "t4", + "t5", + "t6", + "x0", + "x1", + "x2", + "x3", + "x4", + "x5", + "x6", + "x7", + "x8", + "x9", + "x10", + "x11", + "x12", + "x13", + "x14", + "x15", + "x16", + "x17", + "x18", + "x19", + "x20", + "x21", + "x22", + "x23", + "x24", + "x25", + "x26", + "x27", + "x28", + "x29", + "x30", + "x31", +] + + +def harness(opcode, fragment, tag): + try: + exec(fragment) + except SyntaxError: + print(tag, opcode) + + +for opcode in ("slli", "srli", "srai"): + harness(opcode, TEMPLATE3.format(opcode, "a0", "a0", -1), "-") + harness(opcode, TEMPLATE3.format(opcode, "a0", "a0", 33), "+") + +for opcode in ("c_slli", "c_srli", "c_srai"): + harness(opcode, TEMPLATE2.format(opcode, "a0", -1), "-") + harness(opcode, TEMPLATE2.format(opcode, "a0", 33), "+") + +harness("c_slli", TEMPLATE2.format("c_slli", "zero", 0), "0") +harness("c_slli", TEMPLATE2.format("c_slli", "x0", 0), "0") + +for opcode in ("c_srli", "c_srai"): + for register in REGISTERS: + harness(opcode, TEMPLATE2.format(opcode, register, 0), register) + +for opcode in ("c_mv", "c_add"): + harness(opcode, TEMPLATE2.format(opcode, "a0", "zero"), "0l") + harness(opcode, TEMPLATE2.format(opcode, "zero", "a0"), "0r") + harness(opcode, TEMPLATE2.format(opcode, "zero", "zero"), "0b") + +harness("c_jr", TEMPLATE1.format("c_jr", "zero"), "0") + +for opcode in ("addi", "andi", "ori", "slti", "sltiu", "xori"): + harness(opcode, TEMPLATE3.format(opcode, "a0", "a0", 0x7FF), ">=s") + harness(opcode, TEMPLATE3.format(opcode, "a0", "a0", 0x800), ">s") + harness(opcode, TEMPLATE3.format(opcode, "a0", "a0", -2048), "<=s") + harness(opcode, TEMPLATE3.format(opcode, "a0", "a0", -2049), "=s") + harness(opcode, TEMPLATE.format(opcode, 0x800), ">s") + harness(opcode, TEMPLATE.format(opcode, -2048), "<=s") + harness(opcode, TEMPLATE.format(opcode, -2049), "0") +harness("c_addi", TEMPLATE2.format("c_andi", "zero", -512), "<0") +harness("c_addi", TEMPLATE2.format("c_andi", "s0", 0), "s0") +harness("c_addi", TEMPLATE2.format("c_andi", "s0", -100), "s") + +harness("c_andi", TEMPLATE2.format("c_andi", "zero", 0), "00") +harness("c_andi", TEMPLATE2.format("c_andi", "zero", 512), ">0") +harness("c_andi", TEMPLATE2.format("c_andi", "zero", -512), "<0") +harness("c_andi", TEMPLATE2.format("c_andi", "s0", 0), "s0") +harness("c_andi", TEMPLATE2.format("c_andi", "s0", -100), "s") + +C_REGISTERS = ( + "a0", + "a1", + "a2", + "a3", + "a4", + "a5", + "s0", + "s1", + "x8", + "x9", + "x10", + "x11", + "x12", + "x13", + "x14", + "x15", +) + +for opcode in ("c_and", "c_or", "c_xor"): + for source in REGISTERS: + for destination in REGISTERS: + if source in C_REGISTERS and destination in C_REGISTERS: + try: + exec( + """ +@micropython.asm_rv32 +def f(): + {}({}, {}) +""".format(opcode, source, destination) + ) + except SyntaxError: + print(source, destination, opcode) + else: + try: + exec( + """ +@micropython.asm_rv32 +def f(): + {}({}, {}) +""".format(opcode, source, destination) + ) + print(source, destination, opcode) + except SyntaxError: + pass + print(opcode) + +for opcode in ("c_lw", "c_sw"): + TEMPLATE = """ +@micropython.asm_rv32 +def f(): + {}(a0, {}(a0)) +""" + harness(opcode, TEMPLATE.format(opcode, 60), ">=s") + harness(opcode, TEMPLATE.format(opcode, 61), ">s") + harness(opcode, TEMPLATE.format(opcode, -60), "<=s") + harness(opcode, TEMPLATE.format(opcode, -61), "s addi +s andi +s ori +s slti +s sltiu +s xori +s lb +s lbu +s lh +s lhu +s lw +s sb +s sh +s sw +0 c_addi +<0 c_addi +s c_addi +00 c_andi +>0 c_andi +<0 c_andi +s c_andi +c_and +c_or +c_xor +>s c_lw +s c_sw + 4 + and output_mupy_lines[-4] == b"-" * 70 + and output_mupy_lines[-2] == b"" + ): + # look for unittest summary + unittest_ran_match = re.match(rb"Ran (\d+) tests$", output_mupy_lines[-3]) + unittest_result_match = re.match( + b"(" + rb"(OK)( \(skipped=(\d+)\))?" + b"|" + rb"(FAILED) \(failures=(\d+), errors=(\d+)\)" + b")$", + output_mupy_lines[-1], + ) + uses_unittest = unittest_ran_match and unittest_result_match + + # Determine the expected output. + if uses_unittest: + # Expected output is result of running unittest. + output_expected = None + else: + test_file_expected = test_file + ".exp" + if os.path.isfile(test_file_expected): + # Expected output given by a file, so read that in. + with open(test_file_expected, "rb") as f: + output_expected = f.read() + else: + # CIRCUITPY-CHANGE: set language & make sure testlib is available for `skip_ok`. + e = { + "PYTHONPATH": base_path("testlib"), + "PATH": os.environ["PATH"], + "LANG": "en_US.UTF-8", + } + # CIRCUITPY-CHANGE: --keep-path applies to PYTHONPATH as well + if args.keep_path and os.getenv("PYTHONPATH"): + e["PYTHONPATH"] += ":" + os.getenv("PYTHONPATH") + + # Run CPython to work out expected output. + try: + output_expected = subprocess.check_output( + CPYTHON3_CMD + [test_file_abspath], + cwd=os.path.dirname(test_file), + stderr=subprocess.STDOUT, + # CIRCUITPY-CHANGE: pass environment + env=e, + ) + except subprocess.CalledProcessError as er: + output_expected = b"CPYTHON3 CRASH:\n" + er.output + + # Canonical form for all host platforms is to use \n for end-of-line. + output_expected = output_expected.replace(b"\r\n", b"\n") + + # Work out if test passed or not. + test_passed = False + extra_info = "" + if uses_unittest: + test_passed = unittest_result_match.group(2) == b"OK" + num_test_cases = int(unittest_ran_match.group(1)) + extra_info = "unittest: {} ran".format(num_test_cases) + if test_passed and unittest_result_match.group(4) is not None: + num_skipped = int(unittest_result_match.group(4)) + num_test_cases -= num_skipped + extra_info += ", {} skipped".format(num_skipped) + elif not test_passed: + num_failures = int(unittest_result_match.group(6)) + num_errors = int(unittest_result_match.group(7)) + extra_info += ", {} failures, {} errors".format(num_failures, num_errors) + extra_info = "(" + extra_info + ")" + testcase_count.add(num_test_cases) + else: + testcase_count.add(len(output_expected.splitlines())) + test_passed = output_expected == output_mupy filename_expected = os.path.join(result_dir, test_basename + ".exp") filename_mupy = os.path.join(result_dir, test_basename + ".out") - if output_expected == output_mupy: - print("pass ", test_file) + # Print test summary, update counters, and save .exp/.out files if needed. + if test_passed: + print("pass ", test_file, extra_info) passed_count.increment() rm_f(filename_expected) rm_f(filename_mupy) else: - with open(filename_expected, "wb") as f: - f.write(output_expected) + print("FAIL ", test_file, extra_info) + if output_expected is not None: + with open(filename_expected, "wb") as f: + f.write(output_expected) + else: + rm_f(filename_expected) # in case left over from previous failed run with open(filename_mupy, "wb") as f: f.write(output_mupy) - print("FAIL ", test_file) failed_tests.append((test_name, test_file)) test_count.increment() + # Print a note if this looks like it might have been a misfired unittest + if not uses_unittest and not test_passed: + with open(test_file, "r") as f: + if any(re.match("^import.+unittest", l) for l in f.readlines()): + print( + "NOTE: {} may be a unittest that doesn't run unittest.main()".format( + test_file + ) + ) + if pyb: num_threads = 1 - if num_threads > 1: - pool = ThreadPool(num_threads) - pool.map(run_one_test, tests) - else: - for test in tests: - run_one_test(test) + try: + if num_threads > 1: + pool = ThreadPool(num_threads) + pool.map(run_one_test, tests) + else: + for test in tests: + run_one_test(test) + except TestError as er: + for line in er.args[0]: + print(line) + sys.exit(1) print( "{} tests performed ({} individual testcases)".format( @@ -951,17 +1109,38 @@ def main(): formatter_class=argparse.RawDescriptionHelpFormatter, description="""Run and manage tests for MicroPython. +By default the tests are run against the unix port of MicroPython. To run it +against something else, use the -t option. See below for details. + Tests are discovered by scanning test directories for .py files or using the specified test files. If test files nor directories are specified, the script expects to be ran in the tests directory (where this file is located) and the builtin tests suitable for the target platform are ran. + When running tests, run-tests.py compares the MicroPython output of the test with the output produced by running the test through CPython unless a .exp file is found, in which case it is used as comparison. + If a test fails, run-tests.py produces a pair of .out and .exp files in the result directory with the MicroPython output and the expectations, respectively. """, epilog="""\ +The -t option accepts the following for the test instance: +- unix - use the unix port of MicroPython, specified by the MICROPY_MICROPYTHON + environment variable (which defaults to the standard variant of either the unix + or windows ports, depending on the host platform) +- webassembly - use the webassembly port of MicroPython, specified by the + MICROPY_MICROPYTHON_MJS environment variable (which defaults to the standard + variant of the webassembly port) +- port: - connect to and use the given serial port device +- a - connect to and use /dev/ttyACM +- u - connect to and use /dev/ttyUSB +- c - connect to and use COM +- exec: - execute a command and attach to its stdin/stdout +- execpty: - execute a command and attach to the printed /dev/pts/ device +- ... - connect to the given IPv4 address +- anything else specifies a serial port + Options -i and -e can be multiple and processed in the order given. Regex "search" (vs "match") operation is used. An action (include/exclude) of the last matching regex is used: @@ -970,11 +1149,8 @@ def main(): run-tests.py -e async -i async_foo - include all, exclude async, yet still include async_foo """, ) - cmd_parser.add_argument("--target", default="unix", help="the target platform") cmd_parser.add_argument( - "--device", - default="/dev/ttyACM0", - help="the serial device or the IP address of the pyboard", + "-t", "--test-instance", default="unix", help="the MicroPython instance to test" ) cmd_parser.add_argument( "-b", "--baudrate", default=115200, help="the baud rate of the serial device" @@ -1041,11 +1217,18 @@ def main(): args = cmd_parser.parse_args() if args.print_failures: - for exp in glob(os.path.join(args.result_dir, "*.exp")): - testbase = exp[:-4] + for out in glob(os.path.join(args.result_dir, "*.out")): + testbase = out[:-4] print() print("FAILURE {0}".format(testbase)) - os.system("{0} {1}.exp {1}.out".format(DIFF, testbase)) + if os.path.exists(testbase + ".exp"): + # Show diff of expected and actual output. + os.system("{0} {1}.exp {1}.out".format(DIFF, testbase)) + else: + # No expected output, just show the actual output (eg from a unittest). + with open(out) as f: + for line in f: + print(line, end="") sys.exit(0) @@ -1058,43 +1241,11 @@ def main(): sys.exit(0) - LOCAL_TARGETS = ( - "unix", - "webassembly", - ) - EXTERNAL_TARGETS = ( - "pyboard", - "wipy", - "esp8266", - "esp32", - "minimal", - "nrf", - "qemu", - "renesas-ra", - "rp2", - "zephyr", - ) - if args.target in LOCAL_TARGETS: - pyb = None - if args.target == "webassembly": - pyb = PyboardNodeRunner() - elif args.target in EXTERNAL_TARGETS: - global pyboard - sys.path.append(base_path("../tools")) - import pyboard - - pyb = pyboard.Pyboard(args.device, args.baudrate, args.user, args.password) - pyboard.Pyboard.run_script_on_remote_target = run_script_on_remote_target - pyb.enter_raw_repl() - else: - raise ValueError("target must be one of %s" % ", ".join(LOCAL_TARGETS + EXTERNAL_TARGETS)) + # Get the test instance to run on. + pyb = get_test_instance(args.test_instance, args.baudrate, args.user, args.password) - # Automatically detect the native architecture for mpy-cross if not given. - if not args.mpy_cross_flags: - output = run_feature_check(pyb, args, "target_info.py") - arch = str(output, "ascii").strip() - if arch != "None": - args.mpy_cross_flags = "-march=" + arch + # Automatically detect the platform. + detect_test_platform(pyb, args) if args.run_failures and (any(args.files) or args.test_dirs is not None): raise ValueError( @@ -1110,7 +1261,7 @@ def main(): tests = [] elif len(args.files) == 0: test_extensions = ("*.py",) - if args.target == "webassembly": + if args.platform == "webassembly": test_extensions += ("*.js", "*.mjs") if args.test_dirs is None: @@ -1121,23 +1272,25 @@ def main(): "misc", "extmod", ) - if args.target == "pyboard": + if args.inlineasm_arch is not None: + test_dirs += ("inlineasm/{}".format(args.inlineasm_arch),) + if args.platform == "pyboard": # run pyboard tests - test_dirs += ("float", "stress", "inlineasm", "ports/stm32") - elif args.target in ("renesas-ra"): - test_dirs += ("float", "inlineasm", "ports/renesas-ra") - elif args.target == "rp2": + test_dirs += ("float", "stress", "ports/stm32") + elif args.platform == "mimxrt": + test_dirs += ("float", "stress") + elif args.platform == "renesas-ra": + test_dirs += ("float", "ports/renesas-ra") + elif args.platform == "rp2": test_dirs += ("float", "stress", "thread", "ports/rp2") - if "arm" in args.mpy_cross_flags: - test_dirs += ("inlineasm",) - elif args.target == "esp32": + elif args.platform == "esp32": test_dirs += ("float", "stress", "thread") - elif args.target in ("esp8266", "minimal", "nrf"): + elif args.platform in ("esp8266", "minimal", "samd", "nrf"): test_dirs += ("float",) - elif args.target == "wipy": + elif args.platform == "WiPy": # run WiPy tests test_dirs += ("ports/cc3200",) - elif args.target == "unix": + elif args.platform in PC_PLATFORMS: # run PC tests test_dirs += ( "float", @@ -1148,12 +1301,13 @@ def main(): "cmdline", "ports/unix", ) - elif args.target == "qemu": + elif args.platform == "qemu": test_dirs += ( "float", - "inlineasm", "ports/qemu", ) + elif args.platform == "webassembly": + test_dirs += ("float", "ports/webassembly") else: # run tests from these directories test_dirs = args.test_dirs @@ -1169,15 +1323,21 @@ def main(): tests = args.files if not args.keep_path: - # clear search path to make sure tests use only builtin modules and those that can be frozen + # Clear search path to make sure tests use only builtin modules, those in + # extmod, and a path to unittest in case it's needed. # CIRCUITPY-CHANGE: Add testlib for skip_if and our async stuff. - os.environ["MICROPYPATH"] = os.pathsep.join( - [ - ".frozen", - base_path("testlib"), - base_path("../frozen/Adafruit_CircuitPython_asyncio"), - base_path("../frozen/Adafruit_CircuitPython_Ticks"), - ] + os.environ["MICROPYPATH"] = ( + ".frozen" + + os.pathsep + + base_path("testlib") + + os.pathsep + + base_path("../frozen/Adafruit_CircuitPython_asyncio") + + os.pathsep + + base_path("../frozen/Adafruit_CircuitPython_Ticks") + + os.pathsep + + base_path("../extmod") + + os.pathsep + + base_path("../lib/micropython-lib/python-stdlib/unittest") ) try: diff --git a/tests/unix/extra_coverage.py.exp b/tests/unix/extra_coverage.py.exp index 743823dcf4a55..4cd8b8666d270 100644 --- a/tests/unix/extra_coverage.py.exp +++ b/tests/unix/extra_coverage.py.exp @@ -212,6 +212,8 @@ ZeroDivisionError X '\x1b' b'\x00\xff' +frzmpy4 1 +frzmpy4 2 NULL uPy a long string that is not interned diff --git a/tools/ar_util.py b/tools/ar_util.py new file mode 100644 index 0000000000000..b90d379031467 --- /dev/null +++ b/tools/ar_util.py @@ -0,0 +1,236 @@ +#!/usr/bin/env python3 +# +# This file is part of the MicroPython project, http://micropython.org/ +# +# The MIT License (MIT) +# +# Copyright (c) 2024 Volodymyr Shymanskyy +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +# THE SOFTWARE. + +import os +import re +import hashlib +import functools +import pickle + +from elftools.elf import elffile +from collections import defaultdict + +try: + from ar import Archive +except: + Archive = None + + +class PickleCache: + def __init__(self, path, prefix=""): + self.path = path + self._get_fn = lambda key: os.path.join(path, prefix + key[:24]) + + def store(self, key, data): + os.makedirs(self.path, exist_ok=True) + # See also https://bford.info/cachedir/ + cachedir_tag_path = os.path.join(self.path, "CACHEDIR.TAG") + if not os.path.exists(cachedir_tag_path): + with open(cachedir_tag_path, "w") as f: + f.write( + "Signature: 8a477f597d28d172789f06886806bc55\n" + "# This file is a cache directory tag created by MicroPython.\n" + "# For information about cache directory tags see https://bford.info/cachedir/\n" + ) + with open(self._get_fn(key), "wb") as f: + pickle.dump(data, f) + + def load(self, key): + with open(self._get_fn(key), "rb") as f: + return pickle.load(f) + + +def cached(key, cache): + def decorator(func): + @functools.wraps(func) + def wrapper(*args, **kwargs): + cache_key = key(*args, **kwargs) + try: + d = cache.load(cache_key) + if d["key"] != cache_key: + raise Exception("Cache key mismatch") + return d["data"] + except Exception: + res = func(*args, **kwargs) + try: + cache.store( + cache_key, + { + "key": cache_key, + "data": res, + }, + ) + except Exception: + pass + return res + + return wrapper + + return decorator + + +class CachedArFile: + def __init__(self, fn): + if not Archive: + raise RuntimeError("Please run 'pip install ar' to link .a files") + self.fn = fn + self._archive = Archive(open(fn, "rb")) + info = self.load_symbols() + self.objs = info["objs"] + self.symbols = info["symbols"] + + def open(self, obj): + return self._archive.open(obj, "rb") + + def _cache_key(self): + sha = hashlib.sha256() + with open(self.fn, "rb") as f: + for chunk in iter(lambda: f.read(4096), b""): + sha.update(chunk) + # Change this salt if the cache data format changes + sha.update(bytes.fromhex("00000000000000000000000000000001")) + return sha.hexdigest() + + @cached(key=_cache_key, cache=PickleCache(path=".mpy_ld_cache", prefix="ar_")) + def load_symbols(self): + print("Loading", self.fn) + objs = defaultdict(lambda: {"def": set(), "undef": set(), "weak": set()}) + symbols = {} + for entry in self._archive: + obj_name = entry.name + elf = elffile.ELFFile(self.open(obj_name)) + symtab = elf.get_section_by_name(".symtab") + if not symtab: + continue + + obj = objs[obj_name] + + for symbol in symtab.iter_symbols(): + sym_name = symbol.name + sym_bind = symbol["st_info"]["bind"] + + if sym_bind in ("STB_GLOBAL", "STB_WEAK"): + if symbol.entry["st_shndx"] != "SHN_UNDEF": + obj["def"].add(sym_name) + symbols[sym_name] = obj_name + else: + obj["undef"].add(sym_name) + + if sym_bind == "STB_WEAK": + obj["weak"].add(sym_name) + + return {"objs": dict(objs), "symbols": symbols} + + +def resolve(archives, symbols): + resolved_objs = [] # Object files needed to resolve symbols + unresolved_symbols = set() + provided_symbols = {} # Which symbol is provided by which object + symbol_stack = list(symbols) + + # A helper function to handle symbol resolution from a particular object + def add_obj(archive, symbol): + obj_name = archive.symbols[symbol] + obj_info = archive.objs[obj_name] + + obj_tuple = (archive, obj_name) + if obj_tuple in resolved_objs: + return # Already processed this object + + resolved_objs.append(obj_tuple) + + # Add the symbols this object defines + for defined_symbol in obj_info["def"]: + if defined_symbol in provided_symbols and not defined_symbol.startswith( + "__x86.get_pc_thunk." + ): + if defined_symbol in obj_info["weak"]: + continue + else: + raise RuntimeError(f"Multiple definitions for {defined_symbol}") + provided_symbols[defined_symbol] = obj_name # TODO: mark weak if needed + + # Recursively add undefined symbols from this object + for undef_symbol in obj_info["undef"]: + if undef_symbol in obj_info["weak"]: + print(f"Skippping weak dependency: {undef_symbol}") + continue + if undef_symbol not in provided_symbols: + symbol_stack.append(undef_symbol) # Add undefined symbol to resolve + + while symbol_stack: + symbol = symbol_stack.pop(0) + + if symbol in provided_symbols: + continue # Symbol is already resolved + + found = False + for archive in archives: + if symbol in archive.symbols: + add_obj(archive, symbol) + found = True + break + + if not found: + unresolved_symbols.add(symbol) + + return resolved_objs, list(unresolved_symbols) + + +def expand_ld_script(fn): + # This function parses a subset of ld scripts + # Typically these are just groups of static lib references + group_pattern = re.compile(r"GROUP\s*\(\s*([^\)]+)\s*\)", re.MULTILINE) + output_format_pattern = re.compile(r"OUTPUT_FORMAT\s*\(\s*([^\)]+)\s*\)", re.MULTILINE) + comment_pattern = re.compile(r"/\*.*?\*/", re.MULTILINE | re.DOTALL) + + with open(fn, "r") as f: + content = f.read() + content = comment_pattern.sub("", content).strip() + + # Ensure no unrecognized instructions + leftovers = content + for pattern in (group_pattern, output_format_pattern): + leftovers = pattern.sub("", leftovers) + if leftovers.strip(): + raise ValueError("Invalid instruction found in the ld script:" + leftovers) + + # Extract files from GROUP instructions + files = [] + for match in group_pattern.findall(content): + files.extend([file.strip() for file in re.split(r"[,\s]+", match) if file.strip()]) + + return files + + +def load_archive(fn): + ar_header = b"!\012" + with open(fn, "rb") as f: + is_ar_file = f.read(len(ar_header)) == ar_header + if is_ar_file: + return [CachedArFile(fn)] + else: + return [CachedArFile(item) for item in expand_ld_script(fn)] diff --git a/tools/boardgen.py b/tools/boardgen.py index 41a8e9f2e597b..39bedf71cd0c8 100644 --- a/tools/boardgen.py +++ b/tools/boardgen.py @@ -172,6 +172,8 @@ def __init__(self, pin_type, enable_af=False): self._pins = [] self._pin_type = pin_type self._enable_af = enable_af + self._pin_cpu_num_entries = 0 + self._pin_board_num_entries = 0 # Allows a port to define a known cpu pin (without relying on it being in the # csv file). @@ -298,6 +300,9 @@ def print_board_locals_dict(self, out_source): # Don't include hidden pins in Pins.board. continue + # Keep track of the total number of Pin.board entries. + self._pin_board_num_entries += 1 + # We don't use the enable macro for board pins, because they # shouldn't be referenced in pins.csv unless they're # available. @@ -322,6 +327,9 @@ def print_cpu_locals_dict(self, out_source): file=out_source, ) for pin in self.available_pins(exclude_hidden=True): + # Keep track of the total number of Pin.cpu entries. + self._pin_cpu_num_entries += 1 + m = pin.enable_macro() if m: print(" #if {}".format(m), file=out_source) @@ -351,6 +359,20 @@ def board_name_define_prefix(self): # Print the pin_CPUNAME and pin_BOARDNAME macros. def print_defines(self, out_header, cpu=True, board=True): + # Provide #defines for the number of cpu and board pins. + print( + "#define MICROPY_PY_MACHINE_PIN_CPU_NUM_ENTRIES ({})".format( + self._pin_cpu_num_entries + ), + file=out_header, + ) + print( + "#define MICROPY_PY_MACHINE_PIN_BOARD_NUM_ENTRIES ({})".format( + self._pin_board_num_entries + ), + file=out_header, + ) + # Provide #defines for each cpu pin. for pin in self.available_pins(): print(file=out_header) diff --git a/tools/ci.sh b/tools/ci.sh index 6f0f23a1c4810..cfc9754837f76 100755 --- a/tools/ci.sh +++ b/tools/ci.sh @@ -22,6 +22,15 @@ function ci_gcc_riscv_setup { riscv64-unknown-elf-gcc --version } +function ci_picotool_setup { + # Manually installing picotool ensures we use a release version, and speeds up the build. + git clone https://github.com/raspberrypi/pico-sdk.git + (cd pico-sdk && git submodule update --init lib/mbedtls) + git clone https://github.com/raspberrypi/picotool.git + (cd picotool && mkdir build && cd build && cmake -DPICO_SDK_PATH=../../pico-sdk .. && make && sudo make install) + picotool version +} + ######################################################################################## # c code formatting @@ -39,12 +48,18 @@ function ci_c_code_formatting_run { # commit formatting function ci_commit_formatting_run { - git remote add upstream https://github.com/micropython/micropython.git - git fetch --depth=100 upstream master + # Default GitHub Actions checkout for a PR is a generated merge commit where + # the parents are the head of base branch (i.e. master) and the head of the + # PR branch, respectively. Use these parents to find the merge-base (i.e. + # where the PR branch head was branched) + # If the common ancestor commit hasn't been found, fetch more. - git merge-base upstream/master HEAD || git fetch --unshallow upstream master - # For a PR, upstream/master..HEAD ends with a merge commit into master, exclude that one. - tools/verifygitlog.py -v upstream/master..HEAD --no-merges + git merge-base HEAD^1 HEAD^2 || git fetch --unshallow origin + + MERGE_BASE=$(git merge-base HEAD^1 HEAD^2) + HEAD=$(git rev-parse HEAD^2) + echo "Checking commits between merge base ${MERGE_BASE} and PR head ${HEAD}..." + tools/verifygitlog.py -v "${MERGE_BASE}..${HEAD}" } ######################################################################################## @@ -56,6 +71,7 @@ function ci_code_size_setup { gcc --version ci_gcc_arm_setup ci_gcc_riscv_setup + ci_picotool_setup } function ci_code_size_build { @@ -63,42 +79,68 @@ function ci_code_size_build { PORTS_TO_CHECK=bmusxpdv SUBMODULES="lib/asf4 lib/berkeley-db-1.xx lib/btstack lib/cyw43-driver lib/lwip lib/mbedtls lib/micropython-lib lib/nxp_driver lib/pico-sdk lib/stm32lib lib/tinyusb" - # starts off at either the ref/pull/N/merge FETCH_HEAD, or the current branch HEAD - git checkout -b pull_request # save the current location - git remote add upstream https://github.com/micropython/micropython.git - git fetch --depth=100 upstream master - # If the common ancestor commit hasn't been found, fetch more. - git merge-base upstream/master HEAD || git fetch --unshallow upstream master + # Default GitHub pull request sets HEAD to a generated merge commit + # between PR branch (HEAD^2) and base branch (i.e. master) (HEAD^1). + # + # We want to compare this generated commit with the base branch, to see what + # the code size impact would be if we merged this PR. + REFERENCE=$(git rev-parse --short HEAD^1) + COMPARISON=$(git rev-parse --short HEAD) + + echo "Comparing sizes of reference ${REFERENCE} to ${COMPARISON}..." + git log --oneline $REFERENCE..$COMPARISON + + function code_size_build_step { + COMMIT=$1 + OUTFILE=$2 + IGNORE_ERRORS=$3 + + echo "Building ${COMMIT}..." + git checkout --detach $COMMIT + git submodule update --init $SUBMODULES + git show -s + tools/metrics.py clean $PORTS_TO_CHECK + tools/metrics.py build $PORTS_TO_CHECK | tee $OUTFILE || $IGNORE_ERRORS + } + # build reference, save to size0 # ignore any errors with this build, in case master is failing - git checkout `git merge-base --fork-point upstream/master pull_request` - git submodule update --init $SUBMODULES - git show -s - tools/metrics.py clean $PORTS_TO_CHECK - tools/metrics.py build $PORTS_TO_CHECK | tee ~/size0 || true + code_size_build_step $REFERENCE ~/size0 true # build PR/branch, save to size1 - git checkout pull_request - git submodule update --init $SUBMODULES - git log upstream/master..HEAD - tools/metrics.py clean $PORTS_TO_CHECK - tools/metrics.py build $PORTS_TO_CHECK | tee ~/size1 + code_size_build_step $COMPARISON ~/size1 false + + unset -f code_size_build_step } ######################################################################################## # .mpy file format function ci_mpy_format_setup { + sudo apt-get update + sudo apt-get install python2.7 sudo pip3 install pyelftools + python2.7 --version + python3 --version } function ci_mpy_format_test { # Test mpy-tool.py dump feature on bytecode - python2 ./tools/mpy-tool.py -xd tests/frozen/frozentest.mpy + python2.7 ./tools/mpy-tool.py -xd tests/frozen/frozentest.mpy python3 ./tools/mpy-tool.py -xd tests/frozen/frozentest.mpy + # Build MicroPython + ci_unix_standard_build + micropython=./ports/unix/build-standard/micropython + $micropython -m mip install --target . argparse __future__ + export MICROPYPATH=. + + # Test mpy-tool.py running under MicroPython + $micropython ./tools/mpy-tool.py -x -d tests/frozen/frozentest.mpy + # Test mpy-tool.py dump feature on native code make -C examples/natmod/features1 ./tools/mpy-tool.py -xd examples/natmod/features1/features1.mpy + $micropython ./tools/mpy-tool.py -x -d examples/natmod/features1/features1.mpy } ######################################################################################## @@ -118,16 +160,21 @@ function ci_cc3200_build { # GitHub tag of ESP-IDF to use for CI (note: must be a tag or a branch) IDF_VER=v5.2.2 +PYTHON=$(command -v python3 2> /dev/null) +PYTHON_VER=$(${PYTHON:-python} --version | cut -d' ' -f2) export IDF_CCACHE_ENABLE=1 function ci_esp32_idf_setup { - pip3 install pyelftools git clone --depth 1 --branch $IDF_VER https://github.com/espressif/esp-idf.git # doing a treeless clone isn't quite as good as --shallow-submodules, but it # is smaller than full clones and works when the submodule commit isn't a head. git -C esp-idf submodule update --init --recursive --filter=tree:0 ./esp-idf/install.sh + # Install additional packages for mpy_ld into the IDF env + source esp-idf/export.sh + pip3 install pyelftools + pip3 install ar } function ci_esp32_build_common { @@ -161,7 +208,7 @@ function ci_esp32_build_s3_c3 { # ports/esp8266 function ci_esp8266_setup { - sudo pip install pyserial esptool==3.3.1 + sudo pip3 install pyserial esptool==3.3.1 pyelftools ar wget https://github.com/jepler/esp-open-sdk/releases/download/2018-06-10/xtensa-lx106-elf-standalone.tar.gz zcat xtensa-lx106-elf-standalone.tar.gz | tar x # Remove this esptool.py so pip version is used instead @@ -178,6 +225,9 @@ function ci_esp8266_build { make ${MAKEOPTS} -C ports/esp8266 BOARD=ESP8266_GENERIC make ${MAKEOPTS} -C ports/esp8266 BOARD=ESP8266_GENERIC BOARD_VARIANT=FLASH_512K make ${MAKEOPTS} -C ports/esp8266 BOARD=ESP8266_GENERIC BOARD_VARIANT=FLASH_1M + + # Test building native .mpy with xtensa architecture. + ci_native_mpy_modules_build xtensa } ######################################################################################## @@ -212,6 +262,8 @@ function ci_mimxrt_build { make ${MAKEOPTS} -C ports/mimxrt BOARD=MIMXRT1020_EVK make ${MAKEOPTS} -C ports/mimxrt BOARD=TEENSY40 submodules make ${MAKEOPTS} -C ports/mimxrt BOARD=TEENSY40 + make ${MAKEOPTS} -C ports/mimxrt BOARD=MIMXRT1060_EVK submodules + make ${MAKEOPTS} -C ports/mimxrt BOARD=MIMXRT1060_EVK CFLAGS_EXTRA=-DMICROPY_HW_USB_MSC=1 } ######################################################################################## @@ -251,6 +303,8 @@ function ci_qemu_setup_arm { ci_gcc_arm_setup sudo apt-get update sudo apt-get install qemu-system + sudo pip3 install pyelftools + sudo pip3 install ar qemu-system-arm --version } @@ -258,6 +312,8 @@ function ci_qemu_setup_rv32 { ci_gcc_riscv_setup sudo apt-get update sudo apt-get install qemu-system + sudo pip3 install pyelftools + sudo pip3 install ar qemu-system-riscv32 --version } @@ -266,14 +322,22 @@ function ci_qemu_build_arm { make ${MAKEOPTS} -C ports/qemu submodules make ${MAKEOPTS} -C ports/qemu CFLAGS_EXTRA=-DMP_ENDIANNESS_BIG=1 make ${MAKEOPTS} -C ports/qemu clean - make ${MAKEOPTS} -C ports/qemu test - make ${MAKEOPTS} -C ports/qemu BOARD=SABRELITE test + make ${MAKEOPTS} -C ports/qemu test_full + make ${MAKEOPTS} -C ports/qemu BOARD=SABRELITE test_full + + # Test building and running native .mpy with armv7m architecture. + ci_native_mpy_modules_build armv7m + make ${MAKEOPTS} -C ports/qemu test_natmod } function ci_qemu_build_rv32 { make ${MAKEOPTS} -C mpy-cross make ${MAKEOPTS} -C ports/qemu BOARD=VIRT_RV32 submodules - make ${MAKEOPTS} -C ports/qemu BOARD=VIRT_RV32 test + make ${MAKEOPTS} -C ports/qemu BOARD=VIRT_RV32 test_full + + # Test building and running native .mpy with rv32imc architecture. + ci_native_mpy_modules_build rv32imc + make ${MAKEOPTS} -C ports/qemu BOARD=VIRT_RV32 test_natmod } ######################################################################################## @@ -301,6 +365,7 @@ function ci_renesas_ra_board_build { function ci_rp2_setup { ci_gcc_arm_setup + ci_picotool_setup } function ci_rp2_build { @@ -312,7 +377,8 @@ function ci_rp2_build { make ${MAKEOPTS} -C ports/rp2 BOARD=RPI_PICO2 submodules make ${MAKEOPTS} -C ports/rp2 BOARD=RPI_PICO2 make ${MAKEOPTS} -C ports/rp2 BOARD=W5100S_EVB_PICO submodules - make ${MAKEOPTS} -C ports/rp2 BOARD=W5100S_EVB_PICO + # This build doubles as a build test for disabling threads in the config + make ${MAKEOPTS} -C ports/rp2 BOARD=W5100S_EVB_PICO CFLAGS_EXTRA=-DMICROPY_PY_THREAD=0 # Test building ninaw10 driver and NIC interface. make ${MAKEOPTS} -C ports/rp2 BOARD=ARDUINO_NANO_RP2040_CONNECT submodules @@ -339,6 +405,7 @@ function ci_samd_build { function ci_stm32_setup { ci_gcc_arm_setup pip3 install pyelftools + pip3 install ar pip3 install pyhy } @@ -457,16 +524,40 @@ function ci_native_mpy_modules_build { else arch=$1 fi - make -C examples/natmod/features1 ARCH=$arch - make -C examples/natmod/features2 ARCH=$arch - make -C examples/natmod/features3 ARCH=$arch - make -C examples/natmod/features4 ARCH=$arch - make -C examples/natmod/btree ARCH=$arch - make -C examples/natmod/deflate ARCH=$arch - make -C examples/natmod/framebuf ARCH=$arch - make -C examples/natmod/heapq ARCH=$arch - make -C examples/natmod/random ARCH=$arch - make -C examples/natmod/re ARCH=$arch + for natmod in features1 features3 features4 heapq re + do + make -C examples/natmod/$natmod clean + make -C examples/natmod/$natmod ARCH=$arch + done + + # deflate, framebuf, and random currently cannot build on xtensa due to + # some symbols that have been removed from the compiler's runtime, in + # favour of being provided from ROM. + if [ $arch != "xtensa" ]; then + for natmod in deflate framebuf random + do + make -C examples/natmod/$natmod clean + make -C examples/natmod/$natmod ARCH=$arch + done + fi + + # features2 requires soft-float on armv7m, rv32imc, and xtensa. On armv6m + # the compiler generates absolute relocations in the object file + # referencing soft-float functions, which is not supported at the moment. + make -C examples/natmod/features2 clean + if [ $arch = "rv32imc" ] || [ $arch = "armv7m" ] || [ $arch = "xtensa" ]; then + make -C examples/natmod/features2 ARCH=$arch MICROPY_FLOAT_IMPL=float + elif [ $arch != "armv6m" ]; then + make -C examples/natmod/features2 ARCH=$arch + fi + + # btree requires thread local storage support on rv32imc, whilst on xtensa + # it relies on symbols that are provided from ROM but not exposed to + # natmods at the moment. + if [ $arch != "rv32imc" ] && [ $arch != "xtensa" ]; then + make -C examples/natmod/btree clean + make -C examples/natmod/btree ARCH=$arch + fi } function ci_native_mpy_modules_32bit_build { @@ -502,6 +593,7 @@ function ci_unix_standard_v2_run_tests { function ci_unix_coverage_setup { sudo pip3 install setuptools sudo pip3 install pyelftools + sudo pip3 install ar gcc --version python3 --version } @@ -547,10 +639,12 @@ function ci_unix_coverage_run_native_mpy_tests { function ci_unix_32bit_setup { sudo dpkg --add-architecture i386 sudo apt-get update - sudo apt-get install gcc-multilib g++-multilib libffi-dev:i386 + sudo apt-get install gcc-multilib g++-multilib libffi-dev:i386 python2.7 sudo pip3 install setuptools sudo pip3 install pyelftools + sudo pip3 install ar gcc --version + python2.7 --version python3 --version } @@ -569,12 +663,12 @@ function ci_unix_coverage_32bit_run_native_mpy_tests { function ci_unix_nanbox_build { # Use Python 2 to check that it can run the build scripts - ci_unix_build_helper PYTHON=python2 VARIANT=nanbox CFLAGS_EXTRA="-DMICROPY_PY_MATH_CONSTANTS=1" + ci_unix_build_helper PYTHON=python2.7 VARIANT=nanbox CFLAGS_EXTRA="-DMICROPY_PY_MATH_CONSTANTS=1" ci_unix_build_ffi_lib_helper gcc -m32 } function ci_unix_nanbox_run_tests { - ci_unix_run_tests_full_helper nanbox PYTHON=python2 + ci_unix_run_tests_full_helper nanbox PYTHON=python2.7 } function ci_unix_float_build { @@ -633,9 +727,6 @@ function ci_unix_settrace_stackless_run_tests { } function ci_unix_macos_build { - # Install pkg-config to configure libffi paths. - brew install pkg-config - make ${MAKEOPTS} -C mpy-cross make ${MAKEOPTS} -C ports/unix submodules #make ${MAKEOPTS} -C ports/unix deplibs @@ -667,10 +758,8 @@ function ci_unix_qemu_mips_build { } function ci_unix_qemu_mips_run_tests { - # Issues with MIPS tests: - # - (i)listdir does not work, it always returns the empty list (it's an issue with the underlying C call) file ./ports/unix/build-coverage/micropython - (cd tests && MICROPY_MICROPYTHON=../ports/unix/build-coverage/micropython ./run-tests.py --exclude 'vfs_posix.*\.py') + (cd tests && MICROPY_MICROPYTHON=../ports/unix/build-coverage/micropython ./run-tests.py) } function ci_unix_qemu_arm_setup { @@ -734,14 +823,27 @@ ZEPHYR_SDK_VERSION=0.16.8 ZEPHYR_VERSION=v3.7.0 function ci_zephyr_setup { - docker pull zephyrprojectrtos/ci:${ZEPHYR_DOCKER_VERSION} + IMAGE=ghcr.io/zephyrproject-rtos/ci:${ZEPHYR_DOCKER_VERSION} + + docker pull ${IMAGE} + + # Directories cached by GitHub Actions, mounted + # into the container + ZEPHYRPROJECT_DIR="$(pwd)/zephyrproject" + CCACHE_DIR="$(pwd)/.ccache" + + mkdir -p "${ZEPHYRPROJECT_DIR}" + mkdir -p "${CCACHE_DIR}" + docker run --name zephyr-ci -d -it \ -v "$(pwd)":/micropython \ + -v "${ZEPHYRPROJECT_DIR}":/zephyrproject \ + -v "${CCACHE_DIR}":/root/.cache/ccache \ -e ZEPHYR_SDK_INSTALL_DIR=/opt/toolchains/zephyr-sdk-${ZEPHYR_SDK_VERSION} \ -e ZEPHYR_TOOLCHAIN_VARIANT=zephyr \ -e ZEPHYR_BASE=/zephyrproject/zephyr \ -w /micropython/ports/zephyr \ - zephyrprojectrtos/ci:${ZEPHYR_DOCKER_VERSION} + ${IMAGE} docker ps -a # qemu-system-arm is needed to run the test suite. @@ -767,5 +869,20 @@ function ci_zephyr_run_tests { docker exec zephyr-ci west build -p auto -b qemu_cortex_m3 -- -DCONF_FILE=prj_minimal.conf # Issues with zephyr tests: # - inf_nan_arith fails pow(-1, nan) test - (cd tests && ./run-tests.py --target minimal --device execpty:"qemu-system-arm -cpu cortex-m3 -machine lm3s6965evb -nographic -monitor null -serial pty -kernel ../ports/zephyr/build/zephyr/zephyr.elf" -d basics float --exclude inf_nan_arith) + (cd tests && ./run-tests.py -t execpty:"qemu-system-arm -cpu cortex-m3 -machine lm3s6965evb -nographic -monitor null -serial pty -kernel ../ports/zephyr/build/zephyr/zephyr.elf" -d basics float --exclude inf_nan_arith) +} + +######################################################################################## +# ports/alif + +function ci_alif_setup { + ci_gcc_arm_setup +} + +function ci_alif_ae3_build { + make ${MAKEOPTS} -C mpy-cross + make ${MAKEOPTS} -C ports/alif BOARD=OPENMV_AE3 MCU_CORE=M55_HP submodules + make ${MAKEOPTS} -C ports/alif BOARD=OPENMV_AE3 MCU_CORE=M55_HE submodules + make ${MAKEOPTS} -C ports/alif BOARD=OPENMV_AE3 MCU_CORE=M55_DUAL + make ${MAKEOPTS} -C ports/alif BOARD=ALIF_ENSEMBLE MCU_CORE=M55_DUAL } diff --git a/tools/merge_micropython.py b/tools/merge_micropython.py index 97dd53fd33a25..7c4d76507930b 100644 --- a/tools/merge_micropython.py +++ b/tools/merge_micropython.py @@ -8,7 +8,7 @@ I add a sys.exit(0) after a section, and once a section runs, I delete it temporarily and move on to the next section. -- dhalbert -Updated for v1.24.1 merge - dhalbert +Updated for v1.25.0 merge - dhalbert """ @@ -47,6 +47,7 @@ def checkout_ours(always_ours): rm_paths( "ports", [ + "alif", "bare-arm", "cc3200", "embed", @@ -100,6 +101,7 @@ def checkout_ours(always_ours): "library/time.rst", "library/uasyncio.rst", "library/uctypes.rst", + "library/vfs.rst", "library/wipy.rst", "library/wm8960.rst", "library/zephyr*.rst", @@ -124,8 +126,11 @@ def checkout_ours(always_ours): "multi_bluetooth", "multi_espnow", "multi_net", + "multi_pyb_can", + "multi_wlan", "net_hosted", "net_inet", + "ports", "pyb", "wipy", ], @@ -135,11 +140,11 @@ def checkout_ours(always_ours): rm_paths( "lib", [ + "alif*", "asf4", "btstack", "libhydrogen", "lwip", - "micropython-lib", "mynewt-nimble", "nrfx", "nxp_driver", @@ -162,17 +167,20 @@ def checkout_ours(always_ours): "modbtree.*", "modframebuf.*", "modlwip.*", + "modmachine.*", "modnetwork.*", "modonewire.*", "moducryptolib.*", "modsocket.*", - "modssl_*.*", + "modssl_*", + "modtls_*", "modtimeq.*", "modwebsocket.*", "modwebrepl.*", "mpbthci.*", - "network_*.*", + "network_*", "nimble", + "virtpin.*", ], ) @@ -180,6 +188,7 @@ def checkout_ours(always_ours): rm_paths( "shared", [ + "netutils", "tinyusb", "runtime/softtimer.*", ], diff --git a/tools/mpy-tool.py b/tools/mpy-tool.py index b56be0f05b648..8849f2f1e5956 100755 --- a/tools/mpy-tool.py +++ b/tools/mpy-tool.py @@ -24,11 +24,11 @@ # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN # THE SOFTWARE. -# Python 2/3 compatibility code +# Python 2/3/MicroPython compatibility code from __future__ import print_function -import platform +import sys -if platform.python_version_tuple()[0] == "2": +if sys.version_info[0] == 2: from binascii import hexlify as hexlify_py2 str_cons = lambda val, enc=None: str(val) @@ -41,7 +41,7 @@ def hexlify_to_str(b): x = hexlify_py2(b) return ":".join(x[i : i + 2] for i in range(0, len(x), 2)) -else: +elif sys.version_info[0] == 3: # Also handles MicroPython from binascii import hexlify str_cons = str @@ -1771,7 +1771,7 @@ def copy_section(file, offset, offset2): f.write(merged_mpy) -def main(): +def main(args=None): global global_qstrs import argparse @@ -1803,7 +1803,7 @@ def main(): ) cmd_parser.add_argument("-o", "--output", default=None, help="output file") cmd_parser.add_argument("files", nargs="+", help="input .mpy files") - args = cmd_parser.parse_args() + args = cmd_parser.parse_args(args) # set config values relevant to target machine config.MICROPY_LONGINT_IMPL = { diff --git a/tools/mpy_ld.py b/tools/mpy_ld.py index f3ccd1f119e83..219cd1a7468bc 100755 --- a/tools/mpy_ld.py +++ b/tools/mpy_ld.py @@ -30,6 +30,7 @@ import sys, os, struct, re from elftools.elf import elffile +import ar_util sys.path.append(os.path.dirname(__file__) + "/../py") import makeqstrdata as qstrutil @@ -47,6 +48,7 @@ MP_NATIVE_ARCH_ARMV7EMDP = 8 MP_NATIVE_ARCH_XTENSA = 9 MP_NATIVE_ARCH_XTENSAWIN = 10 +MP_NATIVE_ARCH_RV32IMC = 11 MP_PERSISTENT_OBJ_STR = 5 MP_SCOPE_FLAG_VIPERRELOC = 0x10 MP_SCOPE_FLAG_VIPERRODATA = 0x20 @@ -56,6 +58,7 @@ # ELF constants R_386_32 = 1 +R_RISCV_32 = 1 R_X86_64_64 = 1 R_XTENSA_32 = 1 R_386_PC32 = 2 @@ -70,15 +73,57 @@ R_386_GOTPC = 10 R_ARM_THM_CALL = 10 R_XTENSA_ASM_EXPAND = 11 +R_RISCV_BRANCH = 16 +R_RISCV_JAL = 17 +R_RISCV_CALL = 18 +R_RISCV_CALL_PLT = 19 R_XTENSA_DIFF32 = 19 R_XTENSA_SLOT0_OP = 20 +R_RISCV_GOT_HI20 = 20 +R_RISCV_TLS_GD_HI20 = 22 +R_RISCV_PCREL_HI20 = 23 +R_RISCV_PCREL_LO12_I = 24 +R_RISCV_PCREL_LO12_S = 25 R_ARM_BASE_PREL = 25 # aka R_ARM_GOTPC R_ARM_GOT_BREL = 26 # aka R_ARM_GOT32 R_ARM_THM_JUMP24 = 30 +R_RISCV_HI20 = 26 +R_RISCV_LO12_I = 27 +R_RISCV_LO12_S = 28 +R_RISCV_TPREL_HI20 = 29 +R_RISCV_TPREL_LO12_I = 30 +R_RISCV_TPREL_LO12_S = 31 +R_RISCV_TPREL_ADD = 32 +R_RISCV_ADD8 = 33 +R_RISCV_ADD16 = 34 +R_RISCV_ADD32 = 35 +R_RISCV_ADD64 = 36 +R_RISCV_SUB8 = 37 +R_RISCV_SUB16 = 38 +R_RISCV_SUB32 = 39 +R_RISCV_SUB64 = 40 +R_RISCV_GOT32_PCREL = 41 R_X86_64_GOTPCREL = 9 R_X86_64_REX_GOTPCRELX = 42 R_386_GOT32X = 43 +R_RISCV_ALIGN = 43 +R_RISCV_RVC_BRANCH = 44 +R_RISCV_RVC_JUMP = 45 +R_RISCV_RELAX = 51 +R_RISCV_SUB6 = 52 +R_RISCV_SET6 = 53 +R_RISCV_SET8 = 54 +R_RISCV_SET16 = 55 +R_RISCV_SET32 = 56 +R_RISCV_32_PCREL = 57 +R_RISCV_PLT32 = 59 R_XTENSA_PDIFF32 = 59 +R_RISCV_SET_ULEB128 = 60 +R_RISCV_SUB_ULEB128 = 61 +R_RISCV_TLSDESC_HI20 = 62 +R_RISCC_TLSDESC_LOAD_LO12 = 63 +R_RISCV_TLSDESC_ADD_LO12 = 64 +R_RISCV_TLSDESC_CALL = 65 ################################################################################ # Architecture configuration @@ -130,6 +175,18 @@ def asm_jump_xtensa(entry): return struct.pack("> 8) +def asm_jump_rv32(entry): + # This could be 6 bytes shorter, but the code currently cannot + # support a trampoline with varying length depending on the offset. + + # auipc t6, HI(entry) + # jalr zero, t6, LO(entry) + upper, lower = split_riscv_address(entry) + return struct.pack( + "> 16 & 0xFF +def split_riscv_address(value): + # The address can be represented with just the lowest 12 bits + if value < 0 and value > -2048: + value = 4096 + value + return 0, value + # 2s complement + if value < 0: + value = 0x100000000 + value + upper, lower = (value & 0xFFFFF000), (value & 0xFFF) + if lower & 0x800 != 0: + # Reverse lower part sign extension + upper += 0x1000 + return upper & 0xFFFFFFFF, lower & 0xFFFFFFFF + + def xxd(text): for i in range(0, len(text), 16): print("{:08x}:".format(i), end="") @@ -346,7 +425,7 @@ def build_got_generic(env): for r in sec.reloc: s = r.sym if not ( - s.entry["st_info"]["bind"] == "STB_GLOBAL" + s.entry["st_info"]["bind"] in ("STB_GLOBAL", "STB_WEAK") and r["r_info_type"] in env.arch.arch_got ): continue @@ -487,6 +566,8 @@ def do_relocation_text(env, text_addr, r): # Default relocation type and name for logging reloc_type = "le32" log_name = None + addr = None + value = None if ( env.arch.name == "EM_386" @@ -584,18 +665,56 @@ def do_relocation_text(env, text_addr, r): R_XTENSA_PDIFF32, R_XTENSA_ASM_EXPAND, ): - if s.section.name.startswith(".text"): + if not hasattr(s, "section") or s.section.name.startswith(".text"): # it looks like R_XTENSA_[P]DIFF32 into .text is already correctly relocated, # and expand relaxations cannot occur in non-executable sections. return assert 0 + elif env.arch.name == "EM_RISCV" and r_info_type in ( + R_RISCV_TLS_GD_HI20, + R_RISCV_TLSDESC_HI20, + R_RISCV_TLSDESC_ADD_LO12, + R_RISCV_TLSDESC_CALL, + ): + # TLS relocations are not supported. + raise LinkError("{}: RISC-V TLS relocation: {}".format(s.filename, s.name)) + + elif env.arch.name == "EM_RISCV" and r_info_type in ( + R_RISCV_TPREL_HI20, + R_RISCV_TPREL_LO12_I, + R_RISCV_TPREL_LO12_S, + R_RISCV_TPREL_ADD, + ): + # ThreadPointer-relative relocations are not supported. + raise LinkError("{}: RISC-V TP-relative relocation: {}".format(s.filename, s.name)) + + elif env.arch.name == "EM_RISCV" and r_info_type in (R_RISCV_SET_ULEB128, R_RISCV_SUB_ULEB128): + # 128-bit value relocations are not supported + raise LinkError("{}: RISC-V ULEB128 relocation: {}".format(s.filename, s.name)) + + elif env.arch.name == "EM_RISCV" and r_info_type in (R_RISCV_RELAX, R_RISCV_ALIGN): + # To keep things simple, no relocations are relaxed and thus no + # size optimisation is performed even if there is the chance, along + # with no offsets to fix up. + return + + elif env.arch.name == "EM_RISCV": + (addr, value) = process_riscv32_relocation(env, text_addr, r) + + elif env.arch.name == "EM_ARM" and r_info_type == R_ARM_ABS32: + # happens for soft-float on armv6m + raise ValueError("Absolute relocations not supported on ARM") + else: # Unknown/unsupported relocation - assert 0, r_info_type + assert 0, (r_info_type, s.name, s.entry, env.arch.name) # Write relocation - if reloc_type == "le32": + if env.arch.name == "EM_RISCV": + # This case is already handled by `process_riscv_relocation`. + pass + elif reloc_type == "le32": (existing,) = struct.unpack_from(" {:08x}".format(r_offset, log_name, addr)) + if addr is not None: + log(LOG_LEVEL_3, " {:08x} {} -> {:08x}".format(r_offset, log_name, addr)) + else: + log(LOG_LEVEL_3, " {:08x} {} == {:08x}".format(r_offset, log_name, value)) def do_relocation_data(env, text_addr, r): @@ -646,12 +768,16 @@ def do_relocation_data(env, text_addr, r): and r_info_type == R_ARM_ABS32 or env.arch.name == "EM_XTENSA" and r_info_type == R_XTENSA_32 + or env.arch.name == "EM_RISCV" + and r_info_type == R_RISCV_32 ): # Relocation in data.rel.ro to internal/external symbol if env.arch.word_size == 4: struct_type = "> 1) + | ((reloc & 0x20) >> 3) + | ((reloc & 0x18) << 7) + | ((reloc & 0x06) << 2) + ) + & 0xFFFF, + ) + elif reloc_type == "riscv_cj": + # Patch the target of a compressed jump opcode + (existing,) = struct.unpack_from("> 2) + | ((reloc & 0x300) << 1) + | ((reloc & 0x80) >> 1) + | ((reloc & 0x40) << 1) + | ((reloc & 0x20) >> 3) + | ((reloc & 0x10) << 7) + | ((reloc & 0x0E) << 2) + ) + & 0xFFFF, + ) + elif reloc_type == "riscv_call": + # Patch a pair of opcodes forming a call operation + upper, lower = split_riscv_address(reloc) + (existing,) = struct.unpack_from("> 4) + | ((reloc & 0x7E0) << 20) + | ((reloc & 0x1E) << 7) + ) + & 0xFFFFFFFF, + ) + elif reloc_type == "riscv_j": + # Patch a jump/jump with link opcode + (existing,) = struct.unpack_from(" pFad - Phonifier reborn

Pfad - The Proxy pFad of © 2024 Garber Painting. All rights reserved.

Note: This service is not intended for secure transactions such as banking, social media, email, or purchasing. Use at your own risk. We assume no liability whatsoever for broken pages.


Alternative Proxies:

pFad Proxy

pFad v3 Proxy

pFad v4 Proxy

Alternative Proxy