From a96ab0ce367dac5c8d276e5c3b58ee847a66333c Mon Sep 17 00:00:00 2001 From: Jim Mussared Date: Fri, 3 Feb 2023 16:42:03 +1100 Subject: [PATCH 1/6] py/qstr: Add support for MICROPY_QSTR_BYTES_IN_HASH=0. This disables using qstr hashes altogether, which saves RAM and flash (two bytes per interned string on a typical build) as well as code size. On PYBV11 this is worth over 3k flash. qstr comparison will now be done just by length then data. This affects qstr_find_strn although this has a negligible performance impact as, for a given comparison, the length and first character will ~usually be different anyway. String hashing (e.g. builtin `hash()` and map.c) now need to compute the hash dynamically, and for the map case this does come at a performance cost. This work was funded through GitHub Sponsors. Signed-off-by: Jim Mussared --- extmod/modssl_axtls.c | 2 ++ extmod/modssl_mbedtls.c | 2 ++ extmod/vfs_lfsx.c | 2 +- ports/esp32/mphalport.c | 2 ++ ports/stm32/usb.c | 8 +++++ py/map.c | 6 ++++ py/objexcept.c | 8 +++++ py/objstr.c | 24 ++++++++++++++- py/objstr.h | 6 ++++ py/qstr.c | 58 +++++++++++++++++++++++++++++++----- py/qstr.h | 8 ++++- py/runtime.c | 5 ++++ tests/unix/extra_coverage.py | 11 ++++--- tools/mpy-tool.py | 31 +++++++++---------- 14 files changed, 144 insertions(+), 29 deletions(-) diff --git a/extmod/modssl_axtls.c b/extmod/modssl_axtls.c index 6cb999c13b69f..5ceffbb6a4c96 100644 --- a/extmod/modssl_axtls.c +++ b/extmod/modssl_axtls.c @@ -128,7 +128,9 @@ STATIC NORETURN void ssl_raise_error(int err) { o_str->base.type = &mp_type_str; o_str->data = (const byte *)errstr; o_str->len = strlen((char *)o_str->data); + #if MICROPY_QSTR_BYTES_IN_HASH o_str->hash = qstr_compute_hash(o_str->data, o_str->len); + #endif // Raise OSError(err, str). mp_obj_t args[2] = { MP_OBJ_NEW_SMALL_INT(err), MP_OBJ_FROM_PTR(o_str)}; diff --git a/extmod/modssl_mbedtls.c b/extmod/modssl_mbedtls.c index 449952594c2db..be56d1512b0fa 100644 --- a/extmod/modssl_mbedtls.c +++ b/extmod/modssl_mbedtls.c @@ -128,7 +128,9 @@ STATIC NORETURN void mbedtls_raise_error(int err) { o_str->base.type = &mp_type_str; o_str->data = o_str_buf; o_str->len = len; + #if MICROPY_QSTR_BYTES_IN_HASH o_str->hash = qstr_compute_hash(o_str->data, o_str->len); + #endif // raise mp_obj_t args[2] = { MP_OBJ_NEW_SMALL_INT(err), MP_OBJ_FROM_PTR(o_str)}; nlr_raise(mp_obj_exception_make_new(&mp_type_OSError, 2, 0, args)); diff --git a/extmod/vfs_lfsx.c b/extmod/vfs_lfsx.c index fe0731eced2a8..3442133340d03 100644 --- a/extmod/vfs_lfsx.c +++ b/extmod/vfs_lfsx.c @@ -488,7 +488,7 @@ STATIC MP_DEFINE_CONST_DICT(MP_VFS_LFSx(locals_dict), MP_VFS_LFSx(locals_dict_ta STATIC mp_import_stat_t MP_VFS_LFSx(import_stat)(void *self_in, const char *path) { MP_OBJ_VFS_LFSx *self = self_in; struct LFSx_API (info) info; - mp_obj_str_t path_obj = { { &mp_type_str }, 0, 0, (const byte *)path }; + MP_DEFINE_STR_OBJ(path_obj, path); path = MP_VFS_LFSx(make_path)(self, MP_OBJ_FROM_PTR(&path_obj)); int ret = LFSx_API(stat)(&self->lfs, path, &info); if (ret == 0) { diff --git a/ports/esp32/mphalport.c b/ports/esp32/mphalport.c index 538b3e405c905..8aff55f95ccbf 100644 --- a/ports/esp32/mphalport.c +++ b/ports/esp32/mphalport.c @@ -91,7 +91,9 @@ void check_esp_err_(esp_err_t code, const char *func, const int line, const char o_str->data = (const byte *)esp_err_to_name(code); // esp_err_to_name ret's ptr to const str #endif o_str->len = strlen((char *)o_str->data); + #if MICROPY_QSTR_BYTES_IN_HASH o_str->hash = qstr_compute_hash(o_str->data, o_str->len); + #endif // raise mp_obj_t args[2] = { MP_OBJ_NEW_SMALL_INT(pcode), MP_OBJ_FROM_PTR(o_str)}; nlr_raise(mp_obj_exception_make_new(&mp_type_OSError, 2, 0, args)); diff --git a/ports/stm32/usb.c b/ports/stm32/usb.c index df6b2cf62a1ff..19e2fb18b744f 100644 --- a/ports/stm32/usb.c +++ b/ports/stm32/usb.c @@ -185,7 +185,11 @@ STATIC const uint8_t usbd_fifo_size_cdc3_msc_hid[] = { // predefined hid mouse data STATIC const mp_obj_str_t pyb_usb_hid_mouse_desc_obj = { {&mp_type_bytes}, + #if MICROPY_QSTR_BYTES_IN_HASH 0, // hash not valid + #else + 0, + #endif USBD_HID_MOUSE_REPORT_DESC_SIZE, USBD_HID_MOUSE_ReportDesc, }; @@ -204,7 +208,11 @@ const mp_rom_obj_tuple_t pyb_usb_hid_mouse_obj = { // predefined hid keyboard data STATIC const mp_obj_str_t pyb_usb_hid_keyboard_desc_obj = { {&mp_type_bytes}, + #if MICROPY_QSTR_BYTES_IN_HASH 0, // hash not valid + #else + 0, + #endif USBD_HID_KEYBOARD_REPORT_DESC_SIZE, USBD_HID_KEYBOARD_ReportDesc, }; diff --git a/py/map.c b/py/map.c index c18df5a9f333c..89d238818737e 100644 --- a/py/map.c +++ b/py/map.c @@ -32,6 +32,7 @@ #include "py/mpconfig.h" #include "py/misc.h" #include "py/runtime.h" +#include "py/objstr.h" #if MICROPY_DEBUG_VERBOSE // print debugging info #define DEBUG_PRINT (1) @@ -244,7 +245,12 @@ mp_map_elem_t *MICROPY_WRAP_MP_MAP_LOOKUP(mp_map_lookup)(mp_map_t * map, mp_obj_ // get hash of index, with fast path for common case of qstr mp_uint_t hash; if (mp_obj_is_qstr(index)) { + #if MICROPY_QSTR_BYTES_IN_HASH hash = qstr_hash(MP_OBJ_QSTR_VALUE(index)); + #else + GET_STR_DATA_LEN(index, data, len); + hash = qstr_compute_hash(data, len); + #endif } else { hash = MP_OBJ_SMALL_INT_VALUE(mp_unary_op(MP_UNARY_OP_HASH, index)); } diff --git a/py/objexcept.c b/py/objexcept.c index a90405c525484..e2647e78139fa 100644 --- a/py/objexcept.c +++ b/py/objexcept.c @@ -148,12 +148,16 @@ STATIC void decompress_error_text_maybe(mp_obj_exception_t *o) { mp_decompress_rom_string(buf, (mp_rom_error_text_t)o_str->data); o_str->data = buf; o_str->len = strlen((const char *)buf); + #if MICROPY_QSTR_BYTES_IN_HASH o_str->hash = 0; + #endif } + #if MICROPY_QSTR_BYTES_IN_HASH // Lazily compute the string hash. if (o_str->hash == 0) { o_str->hash = qstr_compute_hash(o_str->data, o_str->len); } + #endif } #endif } @@ -417,11 +421,13 @@ mp_obj_t mp_obj_new_exception_msg(const mp_obj_type_t *exc_type, mp_rom_error_te o_str->base.type = &mp_type_str; o_str->len = strlen((const char *)msg); o_str->data = (const byte *)msg; + #if MICROPY_QSTR_BYTES_IN_HASH #if MICROPY_ROM_TEXT_COMPRESSION o_str->hash = 0; // will be computed only if string object is accessed #else o_str->hash = qstr_compute_hash(o_str->data, o_str->len); #endif + #endif mp_obj_t arg = MP_OBJ_FROM_PTR(o_str); return mp_obj_exception_make_new(exc_type, 1, 0, &arg); } @@ -526,11 +532,13 @@ mp_obj_t mp_obj_new_exception_msg_vlist(const mp_obj_type_t *exc_type, mp_rom_er // Create the string object and call mp_obj_exception_make_new to create the exception o_str->base.type = &mp_type_str; + #if MICROPY_QSTR_BYTES_IN_HASH #if MICROPY_ROM_TEXT_COMPRESSION o_str->hash = 0; // will be computed only if string object is accessed #else o_str->hash = qstr_compute_hash(o_str->data, o_str->len); #endif + #endif mp_obj_t arg = MP_OBJ_FROM_PTR(o_str); return mp_obj_exception_make_new(exc_type, 1, 0, &arg); } diff --git a/py/objstr.c b/py/objstr.c index 5dfe94ac4fe1c..a86432e8ce82b 100644 --- a/py/objstr.c +++ b/py/objstr.c @@ -183,10 +183,12 @@ mp_obj_t mp_obj_str_make_new(const mp_obj_type_t *type, size_t n_args, size_t n_ // TODO: validate 2nd/3rd args if (mp_obj_is_type(args[0], &mp_type_bytes)) { GET_STR_DATA_LEN(args[0], str_data, str_len); + #if MICROPY_QSTR_BYTES_IN_HASH GET_STR_HASH(args[0], str_hash); if (str_hash == 0) { str_hash = qstr_compute_hash(str_data, str_len); } + #endif #if MICROPY_PY_BUILTINS_STR_UNICODE_CHECK if (!utf8_check(str_data, str_len)) { mp_raise_msg(&mp_type_UnicodeError, NULL); @@ -201,7 +203,9 @@ mp_obj_t mp_obj_str_make_new(const mp_obj_type_t *type, size_t n_args, size_t n_ mp_obj_str_t *o = MP_OBJ_TO_PTR(mp_obj_new_str_copy(type, NULL, str_len)); o->data = str_data; + #if MICROPY_QSTR_BYTES_IN_HASH o->hash = str_hash; + #endif return MP_OBJ_FROM_PTR(o); } else { mp_buffer_info_t bufinfo; @@ -240,13 +244,17 @@ STATIC mp_obj_t bytes_make_new(const mp_obj_type_t *type_in, size_t n_args, size #endif } GET_STR_DATA_LEN(args[0], str_data, str_len); + #if MICROPY_QSTR_BYTES_IN_HASH GET_STR_HASH(args[0], str_hash); if (str_hash == 0) { str_hash = qstr_compute_hash(str_data, str_len); } + #endif mp_obj_str_t *o = MP_OBJ_TO_PTR(mp_obj_new_str_copy(&mp_type_bytes, NULL, str_len)); o->data = str_data; + #if MICROPY_QSTR_BYTES_IN_HASH o->hash = str_hash; + #endif return MP_OBJ_FROM_PTR(o); } @@ -2063,7 +2071,9 @@ mp_int_t mp_obj_str_get_buffer(mp_obj_t self_in, mp_buffer_info_t *bufinfo, mp_u void mp_obj_str_set_data(mp_obj_str_t *str, const byte *data, size_t len) { str->data = data; str->len = len; + #if MICROPY_QSTR_BYTES_IN_HASH str->hash = qstr_compute_hash(data, len); + #endif } // This locals table is used for the following types: str, bytes, bytearray, array.array. @@ -2199,7 +2209,13 @@ MP_DEFINE_CONST_OBJ_TYPE( ); // The zero-length bytes object, with data that includes a null-terminating byte -const mp_obj_str_t mp_const_empty_bytes_obj = {{&mp_type_bytes}, 0, 0, (const byte *)""}; +const mp_obj_str_t mp_const_empty_bytes_obj = {{&mp_type_bytes}, + #if MICROPY_QSTR_BYTES_IN_HASH + 0, + #else + 0, + #endif + 0, (const byte *)""}; // Create a str/bytes object using the given data. New memory is allocated and // the data is copied across. This function should only be used if the type is bytes, @@ -2208,7 +2224,9 @@ mp_obj_t mp_obj_new_str_copy(const mp_obj_type_t *type, const byte *data, size_t mp_obj_str_t *o = mp_obj_malloc(mp_obj_str_t, type); o->len = len; if (data) { + #if MICROPY_QSTR_BYTES_IN_HASH o->hash = qstr_compute_hash(data, len); + #endif byte *p = m_new(byte, len + 1); o->data = p; memcpy(p, data, len * sizeof(byte)); @@ -2267,7 +2285,9 @@ STATIC mp_obj_t mp_obj_new_str_type_from_vstr(const mp_obj_type_t *type, vstr_t #endif mp_obj_str_t *o = mp_obj_malloc(mp_obj_str_t, type); o->len = vstr->len; + #if MICROPY_QSTR_BYTES_IN_HASH o->hash = qstr_compute_hash(data, vstr->len); + #endif o->data = data; return MP_OBJ_FROM_PTR(o); } @@ -2327,12 +2347,14 @@ bool mp_obj_str_equal(mp_obj_t s1, mp_obj_t s2) { if (mp_obj_is_qstr(s1) && mp_obj_is_qstr(s2)) { return s1 == s2; } else { + #if MICROPY_QSTR_BYTES_IN_HASH GET_STR_HASH(s1, h1); GET_STR_HASH(s2, h2); // If any of hashes is 0, it means it's not valid if (h1 != 0 && h2 != 0 && h1 != h2) { return false; } + #endif GET_STR_DATA_LEN(s1, d1, l1); GET_STR_DATA_LEN(s2, d2, l2); if (l1 != l2) { diff --git a/py/objstr.h b/py/objstr.h index 72fe1cfef01a6..df739c9474ac7 100644 --- a/py/objstr.h +++ b/py/objstr.h @@ -31,7 +31,11 @@ typedef struct _mp_obj_str_t { mp_obj_base_t base; + #if MICROPY_QSTR_BYTES_IN_HASH size_t hash; + #else + size_t _placeholder; // TODO: e.g. byte inline_data[sizeof(size_t)]; + #endif // len == number of bytes used in data, alloc = len + 1 because (at the moment) we also append a null byte size_t len; const byte *data; @@ -46,6 +50,7 @@ typedef struct _mp_obj_str_t { #define MP_DEFINE_STR_OBJ(obj_name, str) mp_obj_str_t obj_name = {{&mp_type_str}, 0, sizeof(str) - 1, (const byte *)str} +#if MICROPY_QSTR_BYTES_IN_HASH // use this macro to extract the string hash // warning: the hash can be 0, meaning invalid, and must then be explicitly computed from the data #define GET_STR_HASH(str_obj_in, str_hash) \ @@ -55,6 +60,7 @@ typedef struct _mp_obj_str_t { } else { \ str_hash = ((mp_obj_str_t *)MP_OBJ_TO_PTR(str_obj_in))->hash; \ } +#endif // use this macro to extract the string length #define GET_STR_LEN(str_obj_in, str_len) \ diff --git a/py/qstr.c b/py/qstr.c index ea700566f4f7d..aa9909d636046 100644 --- a/py/qstr.c +++ b/py/qstr.c @@ -45,7 +45,9 @@ // A qstr is an index into the qstr pool. // The data for a qstr is \0 terminated (so they can be printed using printf) +#if MICROPY_QSTR_BYTES_IN_HASH #define Q_HASH_MASK ((1 << (8 * MICROPY_QSTR_BYTES_IN_HASH)) - 1) +#endif #if MICROPY_PY_THREAD && !MICROPY_PY_THREAD_GIL #define QSTR_ENTER() mp_thread_mutex_lock(&MP_STATE_VM(qstr_mutex), 1) @@ -66,14 +68,17 @@ size_t qstr_compute_hash(const byte *data, size_t len) { for (const byte *top = data + len; data < top; data++) { hash = ((hash << 5) + hash) ^ (*data); // hash * 33 ^ data } + #if MICROPY_QSTR_BYTES_IN_HASH hash &= Q_HASH_MASK; // Make sure that valid hash is never zero, zero means "hash not computed" if (hash == 0) { hash++; } + #endif return hash; } +#if MICROPY_QSTR_BYTES_IN_HASH const qstr_hash_t mp_qstr_const_hashes[] = { #ifndef NO_QSTR #define QDEF(id, hash, len, str) hash, @@ -81,6 +86,7 @@ const qstr_hash_t mp_qstr_const_hashes[] = { #undef QDEF #endif }; +#endif const qstr_len_t mp_qstr_const_lengths[] = { #ifndef NO_QSTR @@ -95,7 +101,9 @@ const qstr_pool_t mp_qstr_const_pool = { 0, // no previous pool MICROPY_ALLOC_QSTR_ENTRIES_INIT, MP_QSTRnumber_of, // corresponds to number of strings in array just below + #if MICROPY_QSTR_BYTES_IN_HASH (qstr_hash_t *)mp_qstr_const_hashes, + #endif (qstr_len_t *)mp_qstr_const_lengths, { #ifndef NO_QSTR @@ -135,8 +143,16 @@ STATIC const qstr_pool_t *find_qstr(qstr *q) { } // qstr_mutex must be taken while in this function -STATIC qstr qstr_add(mp_uint_t hash, mp_uint_t len, const char *q_ptr) { +STATIC qstr qstr_add( + #if MICROPY_QSTR_BYTES_IN_HASH + mp_uint_t hash, + #endif + mp_uint_t len, const char *q_ptr) { + #if MICROPY_QSTR_BYTES_IN_HASH DEBUG_printf("QSTR: add hash=%d len=%d data=%.*s\n", hash, len, len, q_ptr); + #else + DEBUG_printf("QSTR: add len=%d data=%.*s\n", hash, len, len, q_ptr); + #endif // make sure we have room in the pool for a new qstr if (MP_STATE_VM(last_pool)->len >= MP_STATE_VM(last_pool)->alloc) { @@ -146,7 +162,11 @@ STATIC qstr qstr_add(mp_uint_t hash, mp_uint_t len, const char *q_ptr) { new_alloc = MAX(MICROPY_ALLOC_QSTR_ENTRIES_INIT, new_alloc); #endif mp_uint_t pool_size = sizeof(qstr_pool_t) - + (sizeof(const char *) + sizeof(qstr_hash_t) + sizeof(qstr_len_t)) * new_alloc; + + (sizeof(const char *) + #if MICROPY_QSTR_BYTES_IN_HASH + + sizeof(qstr_hash_t) + #endif + + sizeof(qstr_len_t)) * new_alloc; qstr_pool_t *pool = (qstr_pool_t *)m_malloc_maybe(pool_size); if (pool == NULL) { // Keep qstr_last_chunk consistent with qstr_pool_t: qstr_last_chunk is not scanned @@ -158,8 +178,12 @@ STATIC qstr qstr_add(mp_uint_t hash, mp_uint_t len, const char *q_ptr) { QSTR_EXIT(); m_malloc_fail(new_alloc); } + #if MICROPY_QSTR_BYTES_IN_HASH pool->hashes = (qstr_hash_t *)(pool->qstrs + new_alloc); pool->lengths = (qstr_len_t *)(pool->hashes + new_alloc); + #else + pool->lengths = (qstr_len_t *)(pool->qstrs + new_alloc); + #endif pool->prev = MP_STATE_VM(last_pool); pool->total_prev_len = MP_STATE_VM(last_pool)->total_prev_len + MP_STATE_VM(last_pool)->len; pool->alloc = new_alloc; @@ -170,7 +194,9 @@ STATIC qstr qstr_add(mp_uint_t hash, mp_uint_t len, const char *q_ptr) { // add the new qstr mp_uint_t at = MP_STATE_VM(last_pool)->len; + #if MICROPY_QSTR_BYTES_IN_HASH MP_STATE_VM(last_pool)->hashes[at] = hash; + #endif MP_STATE_VM(last_pool)->lengths[at] = len; MP_STATE_VM(last_pool)->qstrs[at] = q_ptr; MP_STATE_VM(last_pool)->len++; @@ -181,12 +207,18 @@ STATIC qstr qstr_add(mp_uint_t hash, mp_uint_t len, const char *q_ptr) { qstr qstr_find_strn(const char *str, size_t str_len) { // work out hash of str + #if MICROPY_QSTR_BYTES_IN_HASH size_t str_hash = qstr_compute_hash((const byte *)str, str_len); + #endif // search pools for the data for (const qstr_pool_t *pool = MP_STATE_VM(last_pool); pool != NULL; pool = pool->prev) { - for (mp_uint_t at = 0, top = pool->len; at < top; at++) { - if (pool->hashes[at] == str_hash && pool->lengths[at] == str_len + for (mp_uint_t at = pool->prev ? 0 : 1, top = pool->len; at < top; at++) { + if ( + #if MICROPY_QSTR_BYTES_IN_HASH + pool->hashes[at] == str_hash && + #endif + pool->lengths[at] == str_len && memcmp(pool->qstrs[at], str, str_len) == 0) { return pool->total_prev_len + at; } @@ -194,7 +226,7 @@ qstr qstr_find_strn(const char *str, size_t str_len) { } // not found; return null qstr - return 0; + return MP_QSTRnull; } qstr qstr_from_str(const char *str) { @@ -254,19 +286,27 @@ qstr qstr_from_strn(const char *str, size_t len) { MP_STATE_VM(qstr_last_used) += n_bytes; // store the interned strings' data + #if MICROPY_QSTR_BYTES_IN_HASH size_t hash = qstr_compute_hash((const byte *)str, len); + #endif memcpy(q_ptr, str, len); q_ptr[len] = '\0'; - q = qstr_add(hash, len, q_ptr); + q = qstr_add( + #if MICROPY_QSTR_BYTES_IN_HASH + hash, + #endif + len, q_ptr); } QSTR_EXIT(); return q; } +#if MICROPY_QSTR_BYTES_IN_HASH mp_uint_t qstr_hash(qstr q) { const qstr_pool_t *pool = find_qstr(&q); return pool->hashes[q]; } +#endif size_t qstr_len(qstr q) { const qstr_pool_t *pool = find_qstr(&q); @@ -300,7 +340,11 @@ void qstr_pool_info(size_t *n_pool, size_t *n_qstr, size_t *n_str_data_bytes, si *n_total_bytes += gc_nbytes(pool); // this counts actual bytes used in heap #else *n_total_bytes += sizeof(qstr_pool_t) - + (sizeof(const char *) + sizeof(qstr_hash_t) + sizeof(qstr_len_t)) * pool->alloc; + + (sizeof(const char *) + #if MICROPY_QSTR_BYTES_IN_HASH + + sizeof(qstr_hash_t) + #endif + + sizeof(qstr_len_t)) * pool->alloc; #endif } *n_total_bytes += *n_str_data_bytes; diff --git a/py/qstr.h b/py/qstr.h index 0ef861f33e8d5..c8925380869e9 100644 --- a/py/qstr.h +++ b/py/qstr.h @@ -48,7 +48,8 @@ enum { typedef size_t qstr; typedef uint16_t qstr_short_t; -#if MICROPY_QSTR_BYTES_IN_HASH == 1 +#if MICROPY_QSTR_BYTES_IN_HASH == 0 +#elif MICROPY_QSTR_BYTES_IN_HASH == 1 typedef uint8_t qstr_hash_t; #elif MICROPY_QSTR_BYTES_IN_HASH == 2 typedef uint16_t qstr_hash_t; @@ -69,7 +70,9 @@ typedef struct _qstr_pool_t { size_t total_prev_len; size_t alloc; size_t len; + #if MICROPY_QSTR_BYTES_IN_HASH qstr_hash_t *hashes; + #endif qstr_len_t *lengths; const char *qstrs[]; } qstr_pool_t; @@ -79,12 +82,15 @@ typedef struct _qstr_pool_t { void qstr_init(void); size_t qstr_compute_hash(const byte *data, size_t len); + qstr qstr_find_strn(const char *str, size_t str_len); // returns MP_QSTRnull if not found qstr qstr_from_str(const char *str); qstr qstr_from_strn(const char *str, size_t len); +#if MICROPY_QSTR_BYTES_IN_HASH mp_uint_t qstr_hash(qstr q); +#endif const char *qstr_str(qstr q); size_t qstr_len(qstr q); const byte *qstr_data(qstr q, size_t *len); diff --git a/py/runtime.c b/py/runtime.c index b4d5250e83a6f..3098a3f62270f 100644 --- a/py/runtime.c +++ b/py/runtime.c @@ -315,12 +315,17 @@ mp_obj_t mp_unary_op(mp_unary_op_t op, mp_obj_t arg) { } } else if (op == MP_UNARY_OP_HASH && mp_obj_is_str_or_bytes(arg)) { // fast path for hashing str/bytes + #if MICROPY_QSTR_BYTES_IN_HASH GET_STR_HASH(arg, h); if (h == 0) { GET_STR_DATA_LEN(arg, data, len); h = qstr_compute_hash(data, len); } return MP_OBJ_NEW_SMALL_INT(h); + #else + GET_STR_DATA_LEN(arg, data, len); + return MP_OBJ_NEW_SMALL_INT(qstr_compute_hash(data, len)); + #endif } else { const mp_obj_type_t *type = mp_obj_get_type(arg); if (MP_OBJ_TYPE_HAS_SLOT(type, unary_op)) { diff --git a/tests/unix/extra_coverage.py b/tests/unix/extra_coverage.py index 0ea8f7886bfff..8d8641ca9336d 100644 --- a/tests/unix/extra_coverage.py +++ b/tests/unix/extra_coverage.py @@ -10,11 +10,14 @@ data = extra_coverage() # test hashing of str/bytes that have an invalid hash +# if MICROPY_QSTR_BYTES_IN_HASH>0, then the hash will be mod 2**(n*8), whereas +# with MICROPY_QSTR_BYTES_IN_HASH==0 the hash will be the full range of +# size_t. so always % 65536 so this test works with both configurations. print(data[0], data[1]) -print(hash(data[0])) -print(hash(data[1])) -print(hash(bytes(data[0], "utf8"))) -print(hash(str(data[1], "utf8"))) +print(hash(data[0]) % 65536) +print(hash(data[1]) % 65536) +print(hash(bytes(data[0], "utf8")) % 65536) +print(hash(str(data[1], "utf8")) % 65536) # test streams stream = data[2] # has set_error and set_buf. Write always returns error diff --git a/tools/mpy-tool.py b/tools/mpy-tool.py index 5ef69b267aec7..7f9c09b7a892b 100755 --- a/tools/mpy-tool.py +++ b/tools/mpy-tool.py @@ -714,11 +714,13 @@ def freeze_constant_obj(self, obj_name, obj): else: obj_type = "mp_type_bytes" print( - 'static const mp_obj_str_t %s = {{&%s}, %u, %u, (const byte*)"%s"};' + 'static const mp_obj_str_t %s = {{&%s}, %s, %u, (const byte*)"%s"};' % ( obj_name, obj_type, - qstrutil.compute_hash(obj, config.MICROPY_QSTR_BYTES_IN_HASH), + str(qstrutil.compute_hash(obj, config.MICROPY_QSTR_BYTES_IN_HASH)) + if config.MICROPY_QSTR_BYTES_IN_HASH + else "0", len(obj), "".join(("\\x%02x" % b) for b in obj), ) @@ -1464,20 +1466,20 @@ def freeze_mpy(base_qstrs, compiled_modules): raw_code_count = 0 raw_code_content = 0 - print() - print("const qstr_hash_t mp_qstr_frozen_const_hashes[] = {") qstr_size = {"metadata": 0, "data": 0} - for _, _, _, qbytes in new: - qhash = qstrutil.compute_hash(qbytes, config.MICROPY_QSTR_BYTES_IN_HASH) - print(" %d," % qhash) - print("};") + if config.MICROPY_QSTR_BYTES_IN_HASH: + print() + print("const qstr_hash_t mp_qstr_frozen_const_hashes[] = {") + for _, _, _, qbytes in new: + qhash = qstrutil.compute_hash(qbytes, config.MICROPY_QSTR_BYTES_IN_HASH) + print(" %d," % qhash) + qstr_size["metadata"] += config.MICROPY_QSTR_BYTES_IN_HASH + print("};") print() print("const qstr_len_t mp_qstr_frozen_const_lengths[] = {") for _, _, _, qbytes in new: print(" %d," % len(qbytes)) - qstr_size["metadata"] += ( - config.MICROPY_QSTR_BYTES_IN_LEN + config.MICROPY_QSTR_BYTES_IN_HASH - ) + qstr_size["metadata"] += config.MICROPY_QSTR_BYTES_IN_LEN qstr_size["data"] += len(qbytes) print("};") print() @@ -1487,14 +1489,13 @@ def freeze_mpy(base_qstrs, compiled_modules): print(" MP_QSTRnumber_of, // previous pool size") print(" %u, // allocated entries" % qstr_pool_alloc) print(" %u, // used entries" % len(new)) - print(" (qstr_hash_t *)mp_qstr_frozen_const_hashes,") + if config.MICROPY_QSTR_BYTES_IN_HASH: + print(" (qstr_hash_t *)mp_qstr_frozen_const_hashes,") print(" (qstr_len_t *)mp_qstr_frozen_const_lengths,") print(" {") for _, _, qstr, qbytes in new: print(' "%s",' % qstrutil.escape_bytes(qstr, qbytes)) - qstr_content += ( - config.MICROPY_QSTR_BYTES_IN_LEN + config.MICROPY_QSTR_BYTES_IN_HASH + len(qbytes) + 1 - ) + qstr_content = qstr_size["metadata"] + qstr_size["data"] + 1 print(" },") print("};") From 3d5a92f13b18a887f0f610f55fd4fc869af3c077 Mon Sep 17 00:00:00 2001 From: Jim Mussared Date: Fri, 3 Feb 2023 16:42:03 +1100 Subject: [PATCH 2/6] py/map: Fast map qstr hashing when MP_QSTR_BYTES_IN_HASH=0. mp_map_lookup now uses the qstr ID directly as the hash. This has the benefit of being a very good hash (guaranteed unique) and free to compute. As mp_map_lookup is a very hot function (e.g. regular dictionaries as well as class locals and globals), this has a significant performance benefit. However, we also need to deal with the case where a map might contain a mixture of str (hashed the regular way) and qstr keys (hashed with ID), so it needs to detect and fall back to a linear search. This work was funded through GitHub Sponsors. Signed-off-by: Jim Mussared --- py/map.c | 59 +++++++++++++++++++++++++++++++++++++++----------------- 1 file changed, 41 insertions(+), 18 deletions(-) diff --git a/py/map.c b/py/map.c index 89d238818737e..3660b7e544c0b 100644 --- a/py/map.c +++ b/py/map.c @@ -32,7 +32,6 @@ #include "py/mpconfig.h" #include "py/misc.h" #include "py/runtime.h" -#include "py/objstr.h" #if MICROPY_DEBUG_VERBOSE // print debugging info #define DEBUG_PRINT (1) @@ -242,17 +241,46 @@ mp_map_elem_t *MICROPY_WRAP_MP_MAP_LOOKUP(mp_map_lookup)(mp_map_t * map, mp_obj_ } } + // When MICROPY_QSTR_BYTES_IN_HASH is zero, for qstr keys we use the qstr + // value (i.e. the index in the qstr pool) as the hash value as it is + // free to compute and also a very effective hash as it is unique across + // all qstrs. + // + // However, non-qstr strings (i.e. str) still use their "true" hash + // (i.e. computed using qstr_hash) which will be different. So this means + // that identical strings will hash differently depending on whether they + // are qstr or str. It should be rare for a program to be working with + // both but it is possible. + // + // So this means that in certain situations (see below), we cannot rely on + // the linear probing finding an unused slot as the signal that the key + // does not exist, as it's possible the key was inserted using + // its "other" hash, and must fall back to searching the entire table. We + // can still use the hash as a good hint for the starting location + // though. This flag must be set to false in these situations. + bool stop_at_empty = true; + // get hash of index, with fast path for common case of qstr mp_uint_t hash; if (mp_obj_is_qstr(index)) { #if MICROPY_QSTR_BYTES_IN_HASH hash = qstr_hash(MP_OBJ_QSTR_VALUE(index)); #else - GET_STR_DATA_LEN(index, data, len); - hash = qstr_compute_hash(data, len); + // Optimisation -- use the qstr index directly as the hash. + hash = MP_OBJ_QSTR_VALUE(index); + // If there are non-qstr keys in this map, we must assume that there + // may possibly be str keys. + stop_at_empty = map->all_keys_are_qstrs; #endif } else { hash = MP_OBJ_SMALL_INT_VALUE(mp_unary_op(MP_UNARY_OP_HASH, index)); + + #if MICROPY_QSTR_BYTES_IN_HASH == 0 + if (mp_obj_is_exact_type(index, &mp_type_str)) { + // We must assume there might be qstr keys. + stop_at_empty = false; + } + #endif } size_t pos = hash % map->alloc; @@ -261,20 +289,14 @@ mp_map_elem_t *MICROPY_WRAP_MP_MAP_LOOKUP(mp_map_lookup)(mp_map_t * map, mp_obj_ for (;;) { mp_map_elem_t *slot = &map->table[pos]; if (slot->key == MP_OBJ_NULL) { - // found NULL slot, so index is not in table - if (lookup_kind == MP_MAP_LOOKUP_ADD_IF_NOT_FOUND) { - map->used += 1; - if (avail_slot == NULL) { - avail_slot = slot; - } - avail_slot->key = index; - avail_slot->value = MP_OBJ_NULL; - if (!mp_obj_is_qstr(index)) { - map->all_keys_are_qstrs = 0; - } - return avail_slot; - } else { - return NULL; + // found an empty slot, remember for later + if (avail_slot == NULL) { + avail_slot = slot; + } + if (stop_at_empty) { + // safe to assume that the key doesn't exist, so pretend like + // we searched the entire table + goto search_over; } } else if (slot->key == MP_OBJ_SENTINEL) { // found deleted slot, remember for later @@ -303,7 +325,8 @@ mp_map_elem_t *MICROPY_WRAP_MP_MAP_LOOKUP(mp_map_lookup)(mp_map_t * map, mp_obj_ pos = (pos + 1) % map->alloc; if (pos == start_pos) { - // search got back to starting position, so index is not in table + search_over: + // search got back to starting position (or found empty slot), so index is not in table if (lookup_kind == MP_MAP_LOOKUP_ADD_IF_NOT_FOUND) { if (avail_slot != NULL) { // there was an available slot, so use that From 51cb2aedbf07db020562f289ddb6474a4f503664 Mon Sep 17 00:00:00 2001 From: Jim Mussared Date: Sat, 4 Feb 2023 03:41:51 +1100 Subject: [PATCH 3/6] py/qstr: Add support for sorted qstr pools. This provides a significant performance boost for qstr_find_strn, which is called a lot during parsing and loading of .mpy files, as well as interning of string objects (which happens in most string methods that return new strings). This work was funded through GitHub Sponsors. Signed-off-by: Jim Mussared --- py/makeqstrdata.py | 104 ++++++++++++++++++++++++++++++++------------- py/qstr.c | 102 ++++++++++++++++++++++++++++++++++++++++---- py/qstr.h | 17 +++++++- tools/mpy-tool.py | 7 ++- 4 files changed, 188 insertions(+), 42 deletions(-) diff --git a/py/makeqstrdata.py b/py/makeqstrdata.py index 71f529fb62900..cc8f6739ea23b 100644 --- a/py/makeqstrdata.py +++ b/py/makeqstrdata.py @@ -220,6 +220,66 @@ "values", "write", "zip", + # Additional QSTRs that must have index <255 (these are not part of + # the .mpy compatibility list though). + "__bool__", + "__pos__", + "__neg__", + "__invert__", + "__abs__", + "__float__", + "__complex__", + "__sizeof__", + "__lt__", + "__gt__", + "__eq__", + "__le__", + "__ge__", + "__ne__", + "__contains__", + "__iadd__", + "__isub__", + "__imul__", + "__imatmul__", + "__ifloordiv__", + "__itruediv__", + "__imod__", + "__ipow__", + "__ior__", + "__ixor__", + "__iand__", + "__ilshift__", + "__irshift__", + "__add__", + "__sub__", + "__mul__", + "__matmul__", + "__floordiv__", + "__truediv__", + "__mod__", + "__divmod__", + "__pow__", + "__or__", + "__xor__", + "__and__", + "__lshift__", + "__rshift__", + "__radd__", + "__rsub__", + "__rmul__", + "__rmatmul__", + "__rfloordiv__", + "__rtruediv__", + "__rmod__", + "__rpow__", + "__ror__", + "__rxor__", + "__rand__", + "__rlshift__", + "__rrshift__", + "__get__", + "__set__", + "__delete__", ] @@ -244,22 +304,13 @@ def esc_char(m): return re.sub(r"[^A-Za-z0-9_]", esc_char, qst) +static_qstr_list_ident = list(map(qstr_escape, static_qstr_list)) + + def parse_input_headers(infiles): qcfgs = {} qstrs = {} - # add static qstrs - for qstr in static_qstr_list: - # work out the corresponding qstr name - ident = qstr_escape(qstr) - - # don't add duplicates - assert ident not in qstrs - - # add the qstr to the list, with order number to retain original order in file - order = len(qstrs) - 300000 - qstrs[ident] = (order, ident, qstr) - # read the qstrs in from the input files for infile in infiles: with open(infile, "rt") as f: @@ -294,22 +345,12 @@ def parse_input_headers(infiles): ident = qstr_escape(qstr) # don't add duplicates + if ident in static_qstr_list_ident: + continue if ident in qstrs: continue - # add the qstr to the list, with order number to retain original order in file - order = len(qstrs) - # but put special method names like __add__ at the top of list, so - # that their id's fit into a byte - if ident == "": - # Sort empty qstr above all still - order = -200000 - elif ident == "__dir__": - # Put __dir__ after empty qstr for builtin dir() to work - order = -190000 - elif ident.startswith("__"): - order -= 100000 - qstrs[ident] = (order, ident, qstr) + qstrs[ident] = (ident, qstr) if not qcfgs: sys.stderr.write("ERROR: Empty preprocessor output - check for errors above\n") @@ -348,12 +389,17 @@ def print_qstr_data(qcfgs, qstrs): print("") # add NULL qstr with no hash or data - print('QDEF(MP_QSTRnull, 0, 0, "")') + print('QDEF0(MP_QSTRnull, 0, 0, "")') + + # add static qstrs to the first unsorted pool + for qstr in static_qstr_list: + qbytes = make_bytes(cfg_bytes_len, cfg_bytes_hash, qstr) + print("QDEF0(MP_QSTR_%s, %s)" % (qstr_escape(qstr), qbytes)) - # go through each qstr and print it out - for order, ident, qstr in sorted(qstrs.values(), key=lambda x: x[0]): + # add remaining qstrs to the sorted (by value) pool + for ident, qstr in sorted(qstrs.values(), key=lambda x: x[1]): qbytes = make_bytes(cfg_bytes_len, cfg_bytes_hash, qstr) - print("QDEF(MP_QSTR_%s, %s)" % (ident, qbytes)) + print("QDEF1(MP_QSTR_%s, %s)" % (ident, qbytes)) def do_work(infiles): diff --git a/py/qstr.c b/py/qstr.c index aa9909d636046..cfcf127c03f28 100644 --- a/py/qstr.c +++ b/py/qstr.c @@ -79,37 +79,83 @@ size_t qstr_compute_hash(const byte *data, size_t len) { } #if MICROPY_QSTR_BYTES_IN_HASH +const qstr_hash_t mp_qstr_const_hashes_special[] = { + #ifndef NO_QSTR +#define QDEF0(id, hash, len, str) hash, +#define QDEF1(id, hash, len, str) + #include "genhdr/qstrdefs.generated.h" +#undef QDEF0 +#undef QDEF1 + #endif +}; const qstr_hash_t mp_qstr_const_hashes[] = { #ifndef NO_QSTR -#define QDEF(id, hash, len, str) hash, +#define QDEF0(id, hash, len, str) +#define QDEF1(id, hash, len, str) hash, #include "genhdr/qstrdefs.generated.h" -#undef QDEF +#undef QDEF0 +#undef QDEF1 #endif }; #endif +const qstr_len_t mp_qstr_const_lengths_special[] = { + #ifndef NO_QSTR +#define QDEF0(id, hash, len, str) len, +#define QDEF1(id, hash, len, str) + #include "genhdr/qstrdefs.generated.h" +#undef QDEF0 +#undef QDEF1 + #endif +}; const qstr_len_t mp_qstr_const_lengths[] = { #ifndef NO_QSTR -#define QDEF(id, hash, len, str) len, +#define QDEF0(id, hash, len, str) +#define QDEF1(id, hash, len, str) len, #include "genhdr/qstrdefs.generated.h" -#undef QDEF +#undef QDEF0 +#undef QDEF1 #endif }; -const qstr_pool_t mp_qstr_const_pool = { +const qstr_pool_t mp_qstr_const_pool_special = { NULL, // no previous pool 0, // no previous pool MICROPY_ALLOC_QSTR_ENTRIES_INIT, - MP_QSTRnumber_of, // corresponds to number of strings in array just below + MP_QSTRnumber_of_special, // corresponds to number of strings in array just below + false, // not sorted + #if MICROPY_QSTR_BYTES_IN_HASH + (qstr_hash_t *)mp_qstr_const_hashes_special, + #endif + (qstr_len_t *)mp_qstr_const_lengths_special, + { + #ifndef NO_QSTR +#define QDEF0(id, hash, len, str) str, +#define QDEF1(id, hash, len, str) + #include "genhdr/qstrdefs.generated.h" +#undef QDEF0 +#undef QDEF1 + #endif + }, +}; + +const qstr_pool_t mp_qstr_const_pool = { + &mp_qstr_const_pool_special, + MP_QSTRnumber_of_special, + MICROPY_ALLOC_QSTR_ENTRIES_INIT, + MP_QSTRnumber_of - MP_QSTRnumber_of_special, // corresponds to number of strings in array just below + true, // sorted #if MICROPY_QSTR_BYTES_IN_HASH (qstr_hash_t *)mp_qstr_const_hashes, #endif (qstr_len_t *)mp_qstr_const_lengths, { #ifndef NO_QSTR -#define QDEF(id, hash, len, str) str, +#define QDEF0(id, hash, len, str) +#define QDEF1(id, hash, len, str) str, #include "genhdr/qstrdefs.generated.h" -#undef QDEF +#undef QDEF0 +#undef QDEF1 #endif }, }; @@ -205,6 +251,24 @@ STATIC qstr qstr_add( return MP_STATE_VM(last_pool)->total_prev_len + at; } +STATIC inline int qstr_strncmp(const char *a, size_t a_len, const char *b) { + while (a_len && *b) { + if (*a != *b) { + return (int)*a - (int)*b; + } + a++; + b++; + a_len--; + } + if (a_len == 0 && *b) { + return -1; + } + if (a_len && !*b) { + return 1; + } + return 0; +} + qstr qstr_find_strn(const char *str, size_t str_len) { // work out hash of str #if MICROPY_QSTR_BYTES_IN_HASH @@ -213,7 +277,27 @@ qstr qstr_find_strn(const char *str, size_t str_len) { // search pools for the data for (const qstr_pool_t *pool = MP_STATE_VM(last_pool); pool != NULL; pool = pool->prev) { - for (mp_uint_t at = pool->prev ? 0 : 1, top = pool->len; at < top; at++) { + size_t low = pool->prev ? 0 : 1; // skip MP_QSTRnull at the start of the first pool + size_t high = pool->len - 1; + + // binary search inside the pool + if (pool->sorted) { + while (high - low > 1) { + size_t mid = (low + high) / 2; + int cmp = qstr_strncmp(str, str_len, pool->qstrs[mid]); + if (cmp < 0) { + high = mid; + } else { + low = mid; + if (MP_UNLIKELY(cmp == 0)) { + break; + } + } + } + } + + // sequential search for the remaining strings + for (mp_uint_t at = low; at < high + 1; at++) { if ( #if MICROPY_QSTR_BYTES_IN_HASH pool->hashes[at] == str_hash && diff --git a/py/qstr.h b/py/qstr.h index c8925380869e9..6dcde390fb0fd 100644 --- a/py/qstr.h +++ b/py/qstr.h @@ -38,9 +38,21 @@ // first entry in enum will be MP_QSTRnull=0, which indicates invalid/no qstr enum { #ifndef NO_QSTR -#define QDEF(id, hash, len, str) id, +#define QDEF0(id, hash, len, str) id, +#define QDEF1(id, hash, len, str) #include "genhdr/qstrdefs.generated.h" -#undef QDEF +#undef QDEF0 +#undef QDEF1 + #endif + MP_QSTRnumber_of_special, + MP_QSTRstart_of_main = MP_QSTRnumber_of_special - 1, + + #ifndef NO_QSTR +#define QDEF0(id, hash, len, str) +#define QDEF1(id, hash, len, str) id, + #include "genhdr/qstrdefs.generated.h" +#undef QDEF0 +#undef QDEF1 #endif MP_QSTRnumber_of, // no underscore so it can't clash with any of the above }; @@ -70,6 +82,7 @@ typedef struct _qstr_pool_t { size_t total_prev_len; size_t alloc; size_t len; + bool sorted; #if MICROPY_QSTR_BYTES_IN_HASH qstr_hash_t *hashes; #endif diff --git a/tools/mpy-tool.py b/tools/mpy-tool.py index 7f9c09b7a892b..f100ceab96ab0 100755 --- a/tools/mpy-tool.py +++ b/tools/mpy-tool.py @@ -1406,7 +1406,7 @@ def freeze_mpy(base_qstrs, compiled_modules): if q is None or q.qstr_esc in base_qstrs or q.qstr_esc in new: continue new[q.qstr_esc] = (len(new), q.qstr_esc, q.str, bytes_cons(q.str, "utf8")) - new = sorted(new.values(), key=lambda x: x[0]) + new = sorted(new.values(), key=lambda x: x[2]) print('#include "py/mpconfig.h"') print('#include "py/objint.h"') @@ -1489,6 +1489,7 @@ def freeze_mpy(base_qstrs, compiled_modules): print(" MP_QSTRnumber_of, // previous pool size") print(" %u, // allocated entries" % qstr_pool_alloc) print(" %u, // used entries" % len(new)) + print(" true, // sorted") if config.MICROPY_QSTR_BYTES_IN_HASH: print(" (qstr_hash_t *)mp_qstr_frozen_const_hashes,") print(" (qstr_len_t *)mp_qstr_frozen_const_lengths,") @@ -1781,7 +1782,9 @@ def main(): # set config values for qstrs, and get the existing base set of qstrs if args.qstr_header: - qcfgs, base_qstrs = qstrutil.parse_input_headers([args.qstr_header]) + special_qstrs = qstrutil.static_qstr_list_ident + qcfgs, extra_qstrs = qstrutil.parse_input_headers([args.qstr_header]) + base_qstrs = set(special_qstrs) | set(extra_qstrs.keys()) config.MICROPY_QSTR_BYTES_IN_LEN = int(qcfgs["BYTES_IN_LEN"]) config.MICROPY_QSTR_BYTES_IN_HASH = int(qcfgs["BYTES_IN_HASH"]) else: From e0a9f82cbbbd885d986b1169b2a85fc6b8233396 Mon Sep 17 00:00:00 2001 From: Jim Mussared Date: Wed, 15 Feb 2023 16:09:04 +1100 Subject: [PATCH 4/6] py/persistentcode.c: Explain the "special" fixed QSTRs. These are part of the .mpy ABI and avoid needing to duplicate string data for QSTRs known to already be in the firmware. This work was funded through GitHub Sponsors. Signed-off-by: Jim Mussared --- py/makeqstrdata.py | 3 ++- py/persistentcode.c | 5 +++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/py/makeqstrdata.py b/py/makeqstrdata.py index cc8f6739ea23b..88785c35fe708 100644 --- a/py/makeqstrdata.py +++ b/py/makeqstrdata.py @@ -52,7 +52,8 @@ codepoint2name[ord("|")] = "pipe" codepoint2name[ord("~")] = "tilde" -# static qstrs, should be sorted +# static qstrs, these must maintain a specific order for .mpy compatibility +# See QSTR_LAST_STATIC at the top of py/persistentcode.c static_qstr_list = [ "", diff --git a/py/persistentcode.c b/py/persistentcode.c index e1218e0b8a577..6f11c96098e2e 100644 --- a/py/persistentcode.c +++ b/py/persistentcode.c @@ -40,6 +40,11 @@ #include "py/smallint.h" +// makeqstrdata.py has a fixed list of qstrs at the start that we can assume +// are available with know indices on all MicroPython implementations, and +// avoid needing to duplicate the string data in the .mpy file. This is the +// last one in that list (anything with a qstr less than or equal to this is +// assumed to be in the list). #define QSTR_LAST_STATIC MP_QSTR_zip #if MICROPY_DYNAMIC_COMPILER From 27f39cb164110fdc874b2276bbc29496cf57dee9 Mon Sep 17 00:00:00 2001 From: Jim Mussared Date: Wed, 15 Feb 2023 23:21:44 +1100 Subject: [PATCH 5/6] top: Set MICROPY_QSTR_BYTES_IN_HASH=0 as default. This work was funded through GitHub Sponsors. Signed-off-by: Jim Mussared --- ports/cc3200/mpconfigport.h | 1 - ports/nrf/mpconfigport.h | 1 - ports/pic16bit/mpconfigport.h | 1 - ports/powerpc/mpconfigport.h | 1 - ports/rp2/mpconfigport.h | 1 - ports/samd/mpconfigport.h | 1 - ports/unix/variants/minimal/mpconfigvariant.h | 3 --- py/mpconfig.h | 8 ++------ 8 files changed, 2 insertions(+), 15 deletions(-) diff --git a/ports/cc3200/mpconfigport.h b/ports/cc3200/mpconfigport.h index b6b412f178f3e..9e89e86df1b61 100644 --- a/ports/cc3200/mpconfigport.h +++ b/ports/cc3200/mpconfigport.h @@ -59,7 +59,6 @@ #else #define MICROPY_CPYTHON_COMPAT (0) #endif -#define MICROPY_QSTR_BYTES_IN_HASH (1) // fatfs configuration used in ffconf.h #define MICROPY_FATFS_ENABLE_LFN (2) diff --git a/ports/nrf/mpconfigport.h b/ports/nrf/mpconfigport.h index 3a311fe5d9abc..b81bec9c62dcb 100644 --- a/ports/nrf/mpconfigport.h +++ b/ports/nrf/mpconfigport.h @@ -260,7 +260,6 @@ #define MICROPY_PY_SYS (1) #define MICROPY_PY_SYS_PATH_ARGV_DEFAULTS (1) #define MICROPY_PY___FILE__ (1) -#define MICROPY_QSTR_BYTES_IN_HASH (2) #endif #ifndef MICROPY_PY_UBLUEPY diff --git a/ports/pic16bit/mpconfigport.h b/ports/pic16bit/mpconfigport.h index f1bfa4cd6f16d..a2d607fb29325 100644 --- a/ports/pic16bit/mpconfigport.h +++ b/ports/pic16bit/mpconfigport.h @@ -29,7 +29,6 @@ // options to control how MicroPython is built #define MICROPY_OBJ_REPR (MICROPY_OBJ_REPR_B) #define MICROPY_ALLOC_PATH_MAX (64) -#define MICROPY_QSTR_BYTES_IN_HASH (1) #define MICROPY_EMIT_X64 (0) #define MICROPY_EMIT_THUMB (0) #define MICROPY_EMIT_INLINE_THUMB (0) diff --git a/ports/powerpc/mpconfigport.h b/ports/powerpc/mpconfigport.h index 06200a9969017..b74f374e7f9f5 100644 --- a/ports/powerpc/mpconfigport.h +++ b/ports/powerpc/mpconfigport.h @@ -35,7 +35,6 @@ // #define MICROPY_DEBUG_VERBOSE (1) -#define MICROPY_QSTR_BYTES_IN_HASH (1) #define MICROPY_QSTR_EXTRA_POOL mp_qstr_frozen_const_pool #define MICROPY_ALLOC_PATH_MAX (256) #define MICROPY_EMIT_X64 (0) diff --git a/ports/rp2/mpconfigport.h b/ports/rp2/mpconfigport.h index 056e2df4fe203..7676d3cb79402 100644 --- a/ports/rp2/mpconfigport.h +++ b/ports/rp2/mpconfigport.h @@ -61,7 +61,6 @@ // Memory allocation policies #define MICROPY_GC_STACK_ENTRY_TYPE uint16_t #define MICROPY_ALLOC_PATH_MAX (128) -#define MICROPY_QSTR_BYTES_IN_HASH (1) // MicroPython emitters #define MICROPY_PERSISTENT_CODE_LOAD (1) diff --git a/ports/samd/mpconfigport.h b/ports/samd/mpconfigport.h index 011119fd11719..5931e1b5595f0 100644 --- a/ports/samd/mpconfigport.h +++ b/ports/samd/mpconfigport.h @@ -35,7 +35,6 @@ #define MICROPY_GC_STACK_ENTRY_TYPE uint16_t #define MICROPY_GC_ALLOC_THRESHOLD (0) #define MICROPY_ALLOC_PATH_MAX (256) -#define MICROPY_QSTR_BYTES_IN_HASH (1) // MicroPython emitters #define MICROPY_PERSISTENT_CODE_LOAD (1) diff --git a/ports/unix/variants/minimal/mpconfigvariant.h b/ports/unix/variants/minimal/mpconfigvariant.h index 0dbfbb3d1cd43..97ed786b8f409 100644 --- a/ports/unix/variants/minimal/mpconfigvariant.h +++ b/ports/unix/variants/minimal/mpconfigvariant.h @@ -64,6 +64,3 @@ // Enable just the sys and os built-in modules. #define MICROPY_PY_SYS (1) #define MICROPY_PY_OS (1) - -// The minimum sets this to 1 to save flash. -#define MICROPY_QSTR_BYTES_IN_HASH (2) diff --git a/py/mpconfig.h b/py/mpconfig.h index a36f9658fbddf..e9fa187af1a8d 100644 --- a/py/mpconfig.h +++ b/py/mpconfig.h @@ -282,13 +282,9 @@ #define MICROPY_QSTR_BYTES_IN_LEN (1) #endif -// Number of bytes used to store qstr hash +// Number of bytes used to store qstr hash, disabled by default #ifndef MICROPY_QSTR_BYTES_IN_HASH -#if MICROPY_CONFIG_ROM_LEVEL_AT_LEAST_CORE_FEATURES -#define MICROPY_QSTR_BYTES_IN_HASH (2) -#else -#define MICROPY_QSTR_BYTES_IN_HASH (1) -#endif +#define MICROPY_QSTR_BYTES_IN_HASH (0) #endif // Avoid using C stack when making Python function calls. C stack still From 8e28db817ccc0b48a15bf1c3648619f8995edcda Mon Sep 17 00:00:00 2001 From: Jim Mussared Date: Mon, 9 Oct 2023 14:07:28 +1100 Subject: [PATCH 6/6] extmod/asyncio_as_uasyncio.py: Fix qstr order dependency. This test depends on the order in which qstrs are stored in ROM, which affects the order in which `dir()` will probe the object to see what it supports. Because of the lazy-loading in asyncio/__init__.py, if it tries to do e.g. `wait_for_ms` before `funcs` then it will import funcs, making `funcs` later succeed. But in the other way around, `funcs` will initially not be found. This work was funded through GitHub Sponsors. Signed-off-by: Jim Mussared --- tests/extmod/asyncio_as_uasyncio.py | 31 +++++++++++++++++++++---- tests/extmod/asyncio_as_uasyncio.py.exp | 2 -- 2 files changed, 26 insertions(+), 7 deletions(-) diff --git a/tests/extmod/asyncio_as_uasyncio.py b/tests/extmod/asyncio_as_uasyncio.py index 612292299c172..b021980590cdc 100644 --- a/tests/extmod/asyncio_as_uasyncio.py +++ b/tests/extmod/asyncio_as_uasyncio.py @@ -1,12 +1,33 @@ try: import uasyncio - import asyncio except ImportError: print("SKIP") raise SystemExit -x = set(dir(uasyncio)) -y = set(dir(asyncio)) - set(["event", "lock", "stream", "funcs"]) -print(x - y) -print(y - x) +# Sample of public symbols we expect to see from `asyncio`. Verify they're all +# available on `uasyncio`. +expected = [ + "CancelledError", + "create_task", + "current_task", + "Event", + "gather", + "get_event_loop", + "Lock", + "Loop", + "open_connection", + "run", + "run_until_complete", + "sleep", + "sleep_ms", + "start_server", + "StreamReader", + "StreamWriter", + "Task", + "ThreadSafeFlag", + "wait_for", +] + +for e in expected: + getattr(uasyncio, e) diff --git a/tests/extmod/asyncio_as_uasyncio.py.exp b/tests/extmod/asyncio_as_uasyncio.py.exp index 9405b80109126..e69de29bb2d1d 100644 --- a/tests/extmod/asyncio_as_uasyncio.py.exp +++ b/tests/extmod/asyncio_as_uasyncio.py.exp @@ -1,2 +0,0 @@ -set() -set() pFad - Phonifier reborn

Pfad - The Proxy pFad of © 2024 Garber Painting. All rights reserved.

Note: This service is not intended for secure transactions such as banking, social media, email, or purchasing. Use at your own risk. We assume no liability whatsoever for broken pages.


Alternative Proxies:

Alternative Proxy

pFad Proxy

pFad v3 Proxy

pFad v4 Proxy