From 030d0dc587e18f6eb1df927bf245c7fdc7ab7f36 Mon Sep 17 00:00:00 2001 From: Amir Gonnen Date: Sun, 14 Feb 2021 00:25:48 +0200 Subject: [PATCH 1/7] py: Faster qstr search. Today qstr implementation scans strings sequntially. In cases there are many strings this can become very inefficient. This change improves qstr search performance by using binary search in sorted qstr pools, when possible. This change introduces an option to create a sorted string pool, which is then searched by a binary search instead of sequential search. qstr pool can be either "sorted" or "unsorted", whereas the unsorted is searched sequentally as today. Native modules (MP_ROM_QSTR) and frozen modules generate sorted pools. Currently runtime strings are unsorted. The constant string pools is split into two and a new pool is introduced, "special_const_pool". This is required because the first sequence of strings already requires special ordering therefore created unsorted, while the rest of the constants are generated sorted. qstr_find_strn searches strings in each pool. If the pool is sorted and larger than a threshold, it will be search using binary search instead of sequential search, significantly improving performance. Signed-off-by: Amir Gonnen --- py/makeqstrdata.py | 17 +++++-- py/qstr.c | 108 ++++++++++++++++++++++++++++++++++++------ py/qstr.h | 17 ++++++- tools/makemanifest.py | 2 +- tools/mpy-tool.py | 3 +- 5 files changed, 125 insertions(+), 22 deletions(-) diff --git a/py/makeqstrdata.py b/py/makeqstrdata.py index e332ab94ed565..78e829e3e79af 100644 --- a/py/makeqstrdata.py +++ b/py/makeqstrdata.py @@ -347,12 +347,21 @@ def print_qstr_data(qcfgs, qstrs): print("") # add NULL qstr with no hash or data - print('QDEF(MP_QSTRnull, 0, 0, "")') + print('QDEF0(MP_QSTRnull, 0, 0, "")') - # go through each qstr and print it out - for order, ident, qstr in sorted(qstrs.values(), key=lambda x: x[0]): + # split qstr values into two pools. static consts first. + q0_values = [q for q in qstrs.values() if q[0] < 0] + q1_values = [q for q in qstrs.values() if q[0] >= 0] + + # go through each qstr in pool 0 and print it out. pool0 has special sort. + for order, ident, qstr in sorted(q0_values, key=lambda x: x[0]): + qbytes = make_bytes(cfg_bytes_len, cfg_bytes_hash, qstr) + print("QDEF0(MP_QSTR_%s, %s)" % (ident, qbytes)) + + # go through each qstr in pool 1 and print it out. pool1 is regularly sorted. + for order, ident, qstr in sorted(q1_values, key=lambda x: x[2]): qbytes = make_bytes(cfg_bytes_len, cfg_bytes_hash, qstr) - print("QDEF(MP_QSTR_%s, %s)" % (ident, qbytes)) + print("QDEF1(MP_QSTR_%s, %s)" % (ident, qbytes)) def do_work(infiles): diff --git a/py/qstr.c b/py/qstr.c index ea700566f4f7d..d2d25ddeac44b 100644 --- a/py/qstr.c +++ b/py/qstr.c @@ -74,34 +74,82 @@ size_t qstr_compute_hash(const byte *data, size_t len) { return hash; } -const qstr_hash_t mp_qstr_const_hashes[] = { +const qstr_hash_t mp_qstr_const_hashes0[] = { #ifndef NO_QSTR -#define QDEF(id, hash, len, str) hash, +#define QDEF0(id, hash, len, str) hash, +#define QDEF1(id, hash, len, str) #include "genhdr/qstrdefs.generated.h" -#undef QDEF +#undef QDEF0 +#undef QDEF1 #endif }; -const qstr_len_t mp_qstr_const_lengths[] = { +const qstr_hash_t mp_qstr_const_hashes1[] = { #ifndef NO_QSTR -#define QDEF(id, hash, len, str) len, +#define QDEF0(id, hash, len, str) +#define QDEF1(id, hash, len, str) hash, #include "genhdr/qstrdefs.generated.h" -#undef QDEF +#undef QDEF0 +#undef QDEF1 #endif }; +const qstr_len_t mp_qstr_const_lengths0[] = { + #ifndef NO_QSTR +#define QDEF0(id, hash, len, str) len, +#define QDEF1(id, hash, len, str) + #include "genhdr/qstrdefs.generated.h" +#undef QDEF0 +#undef QDEF1 + #endif +}; + +const qstr_len_t mp_qstr_const_lengths1[] = { + #ifndef NO_QSTR +#define QDEF0(id, hash, len, str) +#define QDEF1(id, hash, len, str) len, + #include "genhdr/qstrdefs.generated.h" +#undef QDEF0 +#undef QDEF1 + #endif +}; + +const qstr_pool_t mp_qstr_special_const_pool = { + NULL, // no previous pool + 0, // no previous pool + MICROPY_ALLOC_QSTR_ENTRIES_INIT, + MP_QSTRspecial_const_number_of + 1, // corresponds to number of strings in array just below + (qstr_hash_t *)mp_qstr_const_hashes0, + (qstr_len_t *)mp_qstr_const_lengths0, + false, // special constant qstrs are not sorted + { + #ifndef NO_QSTR +#define QDEF0(id, hash, len, str) str, +#define QDEF1(id, hash, len, str) + #include "genhdr/qstrdefs.generated.h" +#undef QDEF0 +#undef QDEF1 + #endif + (const char *)"", // spacer for MP_QSTRspecial_const_number_of + }, +}; + const qstr_pool_t mp_qstr_const_pool = { - NULL, // no previous pool - 0, // no previous pool + (qstr_pool_t *)&mp_qstr_special_const_pool, + MP_QSTRspecial_const_number_of + 1, MICROPY_ALLOC_QSTR_ENTRIES_INIT, - MP_QSTRnumber_of, // corresponds to number of strings in array just below - (qstr_hash_t *)mp_qstr_const_hashes, - (qstr_len_t *)mp_qstr_const_lengths, + MP_QSTRnumber_of - + (MP_QSTRspecial_const_number_of + 1), // corresponds to number of strings in array just below + (qstr_hash_t *)mp_qstr_const_hashes1, + (qstr_len_t *)mp_qstr_const_lengths1, + true, // constant qstrs are sorted { #ifndef NO_QSTR -#define QDEF(id, hash, len, str) str, +#define QDEF0(id, hash, len, str) +#define QDEF1(id, hash, len, str) str, #include "genhdr/qstrdefs.generated.h" -#undef QDEF +#undef QDEF0 +#undef QDEF1 #endif }, }; @@ -164,6 +212,7 @@ STATIC qstr qstr_add(mp_uint_t hash, mp_uint_t len, const char *q_ptr) { pool->total_prev_len = MP_STATE_VM(last_pool)->total_prev_len + MP_STATE_VM(last_pool)->len; pool->alloc = new_alloc; pool->len = 0; + pool->sorted = false; MP_STATE_VM(last_pool) = pool; DEBUG_printf("QSTR: allocate new pool of size %d\n", MP_STATE_VM(last_pool)->alloc); } @@ -179,13 +228,44 @@ STATIC qstr qstr_add(mp_uint_t hash, mp_uint_t len, const char *q_ptr) { return MP_STATE_VM(last_pool)->total_prev_len + at; } +#define MP_QSTR_SEARCH_THRESHOLD 10 + qstr qstr_find_strn(const char *str, size_t str_len) { // work out hash of str size_t str_hash = qstr_compute_hash((const byte *)str, str_len); // search pools for the data for (const qstr_pool_t *pool = MP_STATE_VM(last_pool); pool != NULL; pool = pool->prev) { - for (mp_uint_t at = 0, top = pool->len; at < top; at++) { + size_t low = 0; + size_t high = pool->len - 1; + + // binary search inside the pool + if (pool->sorted) { + while (high - low > MP_QSTR_SEARCH_THRESHOLD) { + size_t mid = (low + high + 1) / 2; + size_t len = pool->lengths[mid]; + if (len > str_len) { + len = str_len; + } + int cmp = memcmp(pool->qstrs[mid], str, str_len); + if (cmp < 0) { + low = mid; + } else if (cmp > 0) { + high = mid; + } else { + if (pool->lengths[mid] < str_len) { + low = mid; + } else if (pool->lengths[mid] > str_len) { + high = mid; + } else { + return pool->total_prev_len + mid; + } + } + } + } + + // sequential search for the remaining strings + for (mp_uint_t at = low; at < high + 1; at++) { if (pool->hashes[at] == str_hash && pool->lengths[at] == str_len && memcmp(pool->qstrs[at], str, str_len) == 0) { return pool->total_prev_len + at; diff --git a/py/qstr.h b/py/qstr.h index 0ef861f33e8d5..22c234e0c4932 100644 --- a/py/qstr.h +++ b/py/qstr.h @@ -38,9 +38,21 @@ // first entry in enum will be MP_QSTRnull=0, which indicates invalid/no qstr enum { #ifndef NO_QSTR -#define QDEF(id, hash, len, str) id, + +#define QDEF0(id, hash, len, str) id, +#define QDEF1(id, hash, len, str) #include "genhdr/qstrdefs.generated.h" -#undef QDEF +#undef QDEF0 +#undef QDEF1 + + MP_QSTRspecial_const_number_of, // no underscore so it can't clash with any of the above + +#define QDEF0(id, hash, len, str) +#define QDEF1(id, hash, len, str) id, + #include "genhdr/qstrdefs.generated.h" +#undef QDEF0 +#undef QDEF1 + #endif MP_QSTRnumber_of, // no underscore so it can't clash with any of the above }; @@ -71,6 +83,7 @@ typedef struct _qstr_pool_t { size_t len; qstr_hash_t *hashes; qstr_len_t *lengths; + bool sorted; const char *qstrs[]; } qstr_pool_t; diff --git a/tools/makemanifest.py b/tools/makemanifest.py index e69698d3f2340..f442eef24c727 100644 --- a/tools/makemanifest.py +++ b/tools/makemanifest.py @@ -415,7 +415,7 @@ def main(): b'#include "py/emitglue.h"\n' b"extern const qstr_pool_t mp_qstr_const_pool;\n" b"const qstr_pool_t mp_qstr_frozen_const_pool = {\n" - b" (qstr_pool_t*)&mp_qstr_const_pool, MP_QSTRnumber_of, 0, 0\n" + b" (qstr_pool_t*)&mp_qstr_const_pool, MP_QSTRnumber_of, 0, false, 0\n" b"};\n" b'const char mp_frozen_names[] = { MP_FROZEN_STR_NAMES "\\0"};\n' b"const mp_raw_code_t *const mp_frozen_mpy_content[] = {NULL};\n" diff --git a/tools/mpy-tool.py b/tools/mpy-tool.py index 31212fd5bdda8..8a6baba6caf3c 100755 --- a/tools/mpy-tool.py +++ b/tools/mpy-tool.py @@ -1397,7 +1397,7 @@ def freeze_mpy(base_qstrs, compiled_modules): if q is None or q.qstr_esc in base_qstrs or q.qstr_esc in new: continue new[q.qstr_esc] = (len(new), q.qstr_esc, q.str, bytes_cons(q.str, "utf8")) - new = sorted(new.values(), key=lambda x: x[0]) + new = sorted(new.values(), key=lambda x: x[2]) print('#include "py/mpconfig.h"') print('#include "py/objint.h"') @@ -1482,6 +1482,7 @@ def freeze_mpy(base_qstrs, compiled_modules): print(" %u, // used entries" % len(new)) print(" (qstr_hash_t *)mp_qstr_frozen_const_hashes,") print(" (qstr_len_t *)mp_qstr_frozen_const_lengths,") + print(" true, // entries are sorted") print(" {") for _, _, qstr, qbytes in new: print(' "%s",' % qstrutil.escape_bytes(qstr, qbytes)) From af147f4af1c2f42eedf0f4ae7cb1ae9c81cb3b30 Mon Sep 17 00:00:00 2001 From: Amir Gonnen Date: Sun, 20 Feb 2022 00:10:25 +0200 Subject: [PATCH 2/7] py/makeqstrdata.py: Refactor qstr class. Use Qstr class instead of tuple, where properties are calculated only when accessed. This is needed as preparation to using hash as the sort key instead the qstr string. It also makes the code more readable when referring to a qstr in py/makeqstrdata.py and tools/mpy-tool.py (for example, refer to q.order instead of q[0], or q.qstr instead of q[2]) Signed-off-by: Amir Gonnen --- py/makeqstrdata.py | 80 ++++++++++++++++++++++++++++------------------ tools/mpy-tool.py | 31 ++++++++++-------- 2 files changed, 67 insertions(+), 44 deletions(-) diff --git a/py/makeqstrdata.py b/py/makeqstrdata.py index 78e829e3e79af..0e5a7a03efcb9 100644 --- a/py/makeqstrdata.py +++ b/py/makeqstrdata.py @@ -223,6 +223,8 @@ ] # this must match the equivalent function in qstr.c + + def compute_hash(qstr, bytes_hash): hash = 5381 for b in qstr: @@ -257,7 +259,7 @@ def parse_input_headers(infiles): # add the qstr to the list, with order number to retain original order in file order = len(qstrs) - 300000 - qstrs[ident] = (order, ident, qstr) + qstrs[ident] = Qstr(order, ident, qstr) # read the qstrs in from the input files for infile in infiles: @@ -308,7 +310,7 @@ def parse_input_headers(infiles): order = -190000 elif ident.startswith("__"): order -= 100000 - qstrs[ident] = (order, ident, qstr) + qstrs[ident] = Qstr(order, ident, qstr) if not qcfgs: sys.stderr.write("ERROR: Empty preprocessor output - check for errors above\n") @@ -317,31 +319,44 @@ def parse_input_headers(infiles): return qcfgs, qstrs -def escape_bytes(qstr, qbytes): - if all(32 <= ord(c) <= 126 and c != "\\" and c != '"' for c in qstr): - # qstr is all printable ASCII so render it as-is (for easier debugging) - return qstr - else: - # qstr contains non-printable codes so render entire thing as hex pairs - return "".join(("\\x%02x" % b) for b in qbytes) +class Qstr: + cfg_bytes_len = 0 + cfg_bytes_hash = 0 + def __init__(self, order, ident, qstr): + self.order = order + self.ident = ident + self.qstr = qstr -def make_bytes(cfg_bytes_len, cfg_bytes_hash, qstr): - qbytes = bytes_cons(qstr, "utf8") - qlen = len(qbytes) - qhash = compute_hash(qbytes, cfg_bytes_hash) - if qlen >= (1 << (8 * cfg_bytes_len)): - print("qstr is too long:", qstr) - assert False - qdata = escape_bytes(qstr, qbytes) - return '%d, %d, "%s"' % (qhash, qlen, qdata) + @property + def qbytes(self): + return bytes_cons(self.qstr, "utf8") + @property + def qlen(self): + if len(self.qbytes) >= (1 << (8 * Qstr.cfg_bytes_len)): + print("qstr is too long:", self.qstr) + assert False + return len(self.qbytes) -def print_qstr_data(qcfgs, qstrs): - # get config variables - cfg_bytes_len = int(qcfgs["BYTES_IN_LEN"]) - cfg_bytes_hash = int(qcfgs["BYTES_IN_HASH"]) + @property + def qhash(self): + return compute_hash(self.qbytes, Qstr.cfg_bytes_hash) + + def _escape_bytes(self): + if all(32 <= ord(c) <= 126 and c != "\\" and c != '"' for c in self.qstr): + # qstr is all printable ASCII so render it as-is (for easier debugging) + return self.qstr + else: + # qstr contains non-printable codes so render entire thing as hex pairs + return "".join(("\\x%02x" % b) for b in self.qbytes) + + @property + def qdata(self): + return self._escape_bytes() + +def print_qstr_data(qstrs): # print out the starter of the generated C header file print("// This file was automatically generated by makeqstrdata.py") print("") @@ -350,23 +365,26 @@ def print_qstr_data(qcfgs, qstrs): print('QDEF0(MP_QSTRnull, 0, 0, "")') # split qstr values into two pools. static consts first. - q0_values = [q for q in qstrs.values() if q[0] < 0] - q1_values = [q for q in qstrs.values() if q[0] >= 0] + q0_values = [q for q in qstrs.values() if q.order < 0] + q1_values = [q for q in qstrs.values() if q.order >= 0] # go through each qstr in pool 0 and print it out. pool0 has special sort. - for order, ident, qstr in sorted(q0_values, key=lambda x: x[0]): - qbytes = make_bytes(cfg_bytes_len, cfg_bytes_hash, qstr) - print("QDEF0(MP_QSTR_%s, %s)" % (ident, qbytes)) + for q in sorted(q0_values, key=lambda x: x.order): + print('QDEF0(MP_QSTR_%s, %d, %d, "%s")' % (q.ident, q.qhash, q.qlen, q.qdata)) # go through each qstr in pool 1 and print it out. pool1 is regularly sorted. - for order, ident, qstr in sorted(q1_values, key=lambda x: x[2]): - qbytes = make_bytes(cfg_bytes_len, cfg_bytes_hash, qstr) - print("QDEF1(MP_QSTR_%s, %s)" % (ident, qbytes)) + for q in sorted(q1_values, key=lambda x: x.qstr): + print('QDEF1(MP_QSTR_%s, %d, %d, "%s")' % (q.ident, q.qhash, q.qlen, q.qdata)) def do_work(infiles): qcfgs, qstrs = parse_input_headers(infiles) - print_qstr_data(qcfgs, qstrs) + + # get config variables + Qstr.cfg_bytes_len = int(qcfgs["BYTES_IN_LEN"]) + Qstr.cfg_bytes_hash = int(qcfgs["BYTES_IN_HASH"]) + + print_qstr_data(qstrs) if __name__ == "__main__": diff --git a/tools/mpy-tool.py b/tools/mpy-tool.py index 8a6baba6caf3c..8a3fa74488f6b 100755 --- a/tools/mpy-tool.py +++ b/tools/mpy-tool.py @@ -1396,8 +1396,8 @@ def freeze_mpy(base_qstrs, compiled_modules): # don't add duplicates if q is None or q.qstr_esc in base_qstrs or q.qstr_esc in new: continue - new[q.qstr_esc] = (len(new), q.qstr_esc, q.str, bytes_cons(q.str, "utf8")) - new = sorted(new.values(), key=lambda x: x[2]) + new[q.qstr_esc] = qstrutil.Qstr(len(new), q.qstr_esc, q.str) + new = sorted(new.values(), key=lambda x: x.qstr) print('#include "py/mpconfig.h"') print('#include "py/objint.h"') @@ -1438,9 +1438,9 @@ def freeze_mpy(base_qstrs, compiled_modules): print("enum {") for i in range(len(new)): if i == 0: - print(" MP_QSTR_%s = MP_QSTRnumber_of," % new[i][1]) + print(" MP_QSTR_%s = MP_QSTRnumber_of," % new[i].ident) else: - print(" MP_QSTR_%s," % new[i][1]) + print(" MP_QSTR_%s," % new[i].ident) print("};") # As in qstr.c, set so that the first dynamically allocated pool is twice this size; must be <= the len @@ -1460,18 +1460,17 @@ def freeze_mpy(base_qstrs, compiled_modules): print() print("const qstr_hash_t mp_qstr_frozen_const_hashes[] = {") qstr_size = {"metadata": 0, "data": 0} - for _, _, _, qbytes in new: - qhash = qstrutil.compute_hash(qbytes, config.MICROPY_QSTR_BYTES_IN_HASH) - print(" %d," % qhash) + for q in new: + print(" %d," % q.qhash) print("};") print() print("const qstr_len_t mp_qstr_frozen_const_lengths[] = {") - for _, _, _, qbytes in new: - print(" %d," % len(qbytes)) + for q in new: + print(" %d," % len(q.qbytes)) qstr_size["metadata"] += ( config.MICROPY_QSTR_BYTES_IN_LEN + config.MICROPY_QSTR_BYTES_IN_HASH ) - qstr_size["data"] += len(qbytes) + qstr_size["data"] += len(q.qbytes) print("};") print() print("extern const qstr_pool_t mp_qstr_const_pool;") @@ -1484,10 +1483,13 @@ def freeze_mpy(base_qstrs, compiled_modules): print(" (qstr_len_t *)mp_qstr_frozen_const_lengths,") print(" true, // entries are sorted") print(" {") - for _, _, qstr, qbytes in new: - print(' "%s",' % qstrutil.escape_bytes(qstr, qbytes)) + for q in new: + print(' "%s",' % q.qdata) qstr_content += ( - config.MICROPY_QSTR_BYTES_IN_LEN + config.MICROPY_QSTR_BYTES_IN_HASH + len(qbytes) + 1 + config.MICROPY_QSTR_BYTES_IN_LEN + + config.MICROPY_QSTR_BYTES_IN_HASH + + len(q.qbytes) + + 1 ) print(" },") print("};") @@ -1782,6 +1784,8 @@ def main(): # Create initial list of global qstrs. global_qstrs = GlobalQStrList() + qstrutil.Qstr.cfg_bytes_len = config.MICROPY_QSTR_BYTES_IN_LEN + qstrutil.Qstr.cfg_bytes_hash = config.MICROPY_QSTR_BYTES_IN_HASH # Load all .mpy files. try: @@ -1789,6 +1793,7 @@ def main(): except MPYReadError as er: print(er, file=sys.stderr) sys.exit(1) + base_qstrs = {} if args.hexdump: hexdump_mpy(compiled_modules) From dfcd5cdf17356fe7fed34deafbf083833804f304 Mon Sep 17 00:00:00 2001 From: Amir Gonnen Date: Wed, 23 Feb 2022 22:23:43 +0200 Subject: [PATCH 3/7] py/qstr: Sort qstrs by hash. Instead of sorting by string, sort qstrs by (hash, len). This allows faster binary serach on qstr_find_strn, since it's faster to compare hashes than strings. A few strings needed to be moved to special string pool (QDEF0) because their qstr is assumed to be small (8 bit) on py/scope.c Signed-off-by: Amir Gonnen --- py/makeqstrdata.py | 7 ++++++- py/qstr.c | 26 ++++++++++++++------------ tools/mpy-tool.py | 2 +- 3 files changed, 21 insertions(+), 14 deletions(-) diff --git a/py/makeqstrdata.py b/py/makeqstrdata.py index 0e5a7a03efcb9..cb33e91d637f9 100644 --- a/py/makeqstrdata.py +++ b/py/makeqstrdata.py @@ -61,7 +61,12 @@ " ", "*", "/", + "", + "", + "", + "", "", + "", "_", "__call__", "__class__", @@ -373,7 +378,7 @@ def print_qstr_data(qstrs): print('QDEF0(MP_QSTR_%s, %d, %d, "%s")' % (q.ident, q.qhash, q.qlen, q.qdata)) # go through each qstr in pool 1 and print it out. pool1 is regularly sorted. - for q in sorted(q1_values, key=lambda x: x.qstr): + for q in sorted(q1_values, key=lambda x: (x.qhash, x.qlen)): print('QDEF1(MP_QSTR_%s, %d, %d, "%s")' % (q.ident, q.qhash, q.qlen, q.qdata)) diff --git a/py/qstr.c b/py/qstr.c index d2d25ddeac44b..2a1131c9147ac 100644 --- a/py/qstr.c +++ b/py/qstr.c @@ -243,23 +243,25 @@ qstr qstr_find_strn(const char *str, size_t str_len) { if (pool->sorted) { while (high - low > MP_QSTR_SEARCH_THRESHOLD) { size_t mid = (low + high + 1) / 2; - size_t len = pool->lengths[mid]; - if (len > str_len) { - len = str_len; - } - int cmp = memcmp(pool->qstrs[mid], str, str_len); + int cmp = pool->hashes[mid] - str_hash; + if (cmp == 0) cmp = pool->lengths[mid] - str_len; if (cmp < 0) { low = mid; } else if (cmp > 0) { high = mid; } else { - if (pool->lengths[mid] < str_len) { - low = mid; - } else if (pool->lengths[mid] > str_len) { - high = mid; - } else { - return pool->total_prev_len + mid; - } + // avoid a rare (hash,len) collisions + while (MP_UNLIKELY( + pool->lengths[mid] != str_len || + memcmp(pool->qstrs[mid], str, str_len) != 0)) { + mid++; + if (mid > high || + pool->hashes[mid] != str_hash || + pool->lengths[mid] != str_len) { + return 0; + } + } + return pool->total_prev_len + mid; } } } diff --git a/tools/mpy-tool.py b/tools/mpy-tool.py index 8a3fa74488f6b..015a2b98e6486 100755 --- a/tools/mpy-tool.py +++ b/tools/mpy-tool.py @@ -1397,7 +1397,7 @@ def freeze_mpy(base_qstrs, compiled_modules): if q is None or q.qstr_esc in base_qstrs or q.qstr_esc in new: continue new[q.qstr_esc] = qstrutil.Qstr(len(new), q.qstr_esc, q.str) - new = sorted(new.values(), key=lambda x: x.qstr) + new = sorted(new.values(), key=lambda x: (x.qhash, x.qlen)) print('#include "py/mpconfig.h"') print('#include "py/objint.h"') From 9696f4375a48f27de1593886569e7c20e0f32da6 Mon Sep 17 00:00:00 2001 From: Amir Gonnen Date: Fri, 25 Feb 2022 23:20:32 +0200 Subject: [PATCH 4/7] py/qstr: Optimize hash sort. In case cmp == 0 there could be a hash collision. To save program size Instead of checking for hash collisions, simply revert to linear search which also compares strings. Signed-off-by: Amir Gonnen --- py/qstr.c | 24 ++++++++---------------- 1 file changed, 8 insertions(+), 16 deletions(-) diff --git a/py/qstr.c b/py/qstr.c index 2a1131c9147ac..1d4239280db68 100644 --- a/py/qstr.c +++ b/py/qstr.c @@ -244,24 +244,16 @@ qstr qstr_find_strn(const char *str, size_t str_len) { while (high - low > MP_QSTR_SEARCH_THRESHOLD) { size_t mid = (low + high + 1) / 2; int cmp = pool->hashes[mid] - str_hash; - if (cmp == 0) cmp = pool->lengths[mid] - str_len; - if (cmp < 0) { - low = mid; - } else if (cmp > 0) { + if (cmp == 0) { + cmp = pool->lengths[mid] - str_len; + } + if (cmp > 0) { high = mid; } else { - // avoid a rare (hash,len) collisions - while (MP_UNLIKELY( - pool->lengths[mid] != str_len || - memcmp(pool->qstrs[mid], str, str_len) != 0)) { - mid++; - if (mid > high || - pool->hashes[mid] != str_hash || - pool->lengths[mid] != str_len) { - return 0; - } - } - return pool->total_prev_len + mid; + low = mid; + if (cmp == 0) { + break; + } } } } From 06c45905d95a4df8e5b3e8cdbf4165e7c5411545 Mon Sep 17 00:00:00 2001 From: Amir Gonnen Date: Sat, 6 Aug 2022 01:41:19 +0300 Subject: [PATCH 5/7] py/qstr: Fix hash collision corner case. This fixes a corner case where both hash and lengths of two qstrs are equal. In such case, if the binary search lower bound is the second instance of the same hash, the search will miss the first instance. To overcome this, when lower bound hash equals the requested hash, we move the lower bound backwards to cover all previous hashes which are also equal to the requested hash. Also removed length comparison since it's no longer needed. Related: https://github.com/lvgl/lv_binding_micropython/issues/224 Signed-off-by: Amir Gonnen --- py/qstr.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/py/qstr.c b/py/qstr.c index 1d4239280db68..04faaacd6ce20 100644 --- a/py/qstr.c +++ b/py/qstr.c @@ -244,14 +244,14 @@ qstr qstr_find_strn(const char *str, size_t str_len) { while (high - low > MP_QSTR_SEARCH_THRESHOLD) { size_t mid = (low + high + 1) / 2; int cmp = pool->hashes[mid] - str_hash; - if (cmp == 0) { - cmp = pool->lengths[mid] - str_len; - } if (cmp > 0) { high = mid; } else { low = mid; if (cmp == 0) { + while (low > 0 && pool->hashes[low - 1] == str_hash) { + low--; + } break; } } From b3b957dba9df710dcd39909d82e746ca7410ec2c Mon Sep 17 00:00:00 2001 From: Amir Gonnen Date: Sun, 7 Aug 2022 00:59:21 +0300 Subject: [PATCH 6/7] py/qstr: Remove MP_QSTR_SEARCH_THRESHOLD. MP_QSTR_SEARCH_THRESHOLD doesn't provide improvement with sorted hashes. Remove MP_QSTR_SEARCH_THRESHOLD and simlify the binary search. Sequential search is still needed in case of hash collisions, or unsorted pools. Signed-off-by: Amir Gonnen --- py/qstr.c | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/py/qstr.c b/py/qstr.c index 04faaacd6ce20..5b12723ff4184 100644 --- a/py/qstr.c +++ b/py/qstr.c @@ -228,8 +228,6 @@ STATIC qstr qstr_add(mp_uint_t hash, mp_uint_t len, const char *q_ptr) { return MP_STATE_VM(last_pool)->total_prev_len + at; } -#define MP_QSTR_SEARCH_THRESHOLD 10 - qstr qstr_find_strn(const char *str, size_t str_len) { // work out hash of str size_t str_hash = qstr_compute_hash((const byte *)str, str_len); @@ -241,8 +239,8 @@ qstr qstr_find_strn(const char *str, size_t str_len) { // binary search inside the pool if (pool->sorted) { - while (high - low > MP_QSTR_SEARCH_THRESHOLD) { - size_t mid = (low + high + 1) / 2; + while (high - low > 1) { + size_t mid = (low + high) / 2; int cmp = pool->hashes[mid] - str_hash; if (cmp > 0) { high = mid; From 311b33c817b05eea00b128fac0d6444d5e6b91a1 Mon Sep 17 00:00:00 2001 From: Amir Gonnen Date: Tue, 9 Aug 2022 23:33:51 +0300 Subject: [PATCH 7/7] py/qstr: Sort the first QSTR pool. Signed-off-by: Amir Gonnen --- py/makeqstrdata.py | 76 ++++++++++++++++++++++++++++++++++--------- py/qstr.c | 4 +-- tools/makemanifest.py | 2 +- tools/mpy-tool.py | 1 - 4 files changed, 64 insertions(+), 19 deletions(-) diff --git a/py/makeqstrdata.py b/py/makeqstrdata.py index cb33e91d637f9..f834dac38684e 100644 --- a/py/makeqstrdata.py +++ b/py/makeqstrdata.py @@ -52,7 +52,7 @@ codepoint2name[ord("|")] = "pipe" codepoint2name[ord("~")] = "tilde" -# static qstrs, should be sorted +# static qstrs, unsorted. static_qstr_list = [ "", @@ -89,6 +89,64 @@ "__repr__", "__setitem__", "__str__", + "__bool__", + "__pos__", + "__neg__", + "__invert__", + "__abs__", + "__float__", + "__complex__", + "__sizeof__", + "__lt__", + "__gt__", + "__eq__", + "__le__", + "__ge__", + "__ne__", + "__contains__", + "__iadd__", + "__isub__", + "__imul__", + "__imatmul__", + "__ifloordiv__", + "__itruediv__", + "__imod__", + "__ipow__", + "__ior__", + "__ixor__", + "__iand__", + "__ilshift__", + "__irshift__", + "__add__", + "__sub__", + "__mul__", + "__matmul__", + "__floordiv__", + "__truediv__", + "__mod__", + "__divmod__", + "__pow__", + "__or__", + "__xor__", + "__and__", + "__lshift__", + "__rshift__", + "__radd__", + "__rsub__", + "__rmul__", + "__rmatmul__", + "__rfloordiv__", + "__rtruediv__", + "__rmod__", + "__rpow__", + "__ror__", + "__rxor__", + "__rand__", + "__rlshift__", + "__rrshift__", + "__get__", + "__set__", + "__delete__", "ArithmeticError", "AssertionError", "AttributeError", @@ -228,8 +286,6 @@ ] # this must match the equivalent function in qstr.c - - def compute_hash(qstr, bytes_hash): hash = 5381 for b in qstr: @@ -305,16 +361,6 @@ def parse_input_headers(infiles): # add the qstr to the list, with order number to retain original order in file order = len(qstrs) - # but put special method names like __add__ at the top of list, so - # that their id's fit into a byte - if ident == "": - # Sort empty qstr above all still - order = -200000 - elif ident == "__dir__": - # Put __dir__ after empty qstr for builtin dir() to work - order = -190000 - elif ident.startswith("__"): - order -= 100000 qstrs[ident] = Qstr(order, ident, qstr) if not qcfgs: @@ -374,11 +420,11 @@ def print_qstr_data(qstrs): q1_values = [q for q in qstrs.values() if q.order >= 0] # go through each qstr in pool 0 and print it out. pool0 has special sort. - for q in sorted(q0_values, key=lambda x: x.order): + for q in sorted(q0_values, key=lambda x: x.qhash): print('QDEF0(MP_QSTR_%s, %d, %d, "%s")' % (q.ident, q.qhash, q.qlen, q.qdata)) # go through each qstr in pool 1 and print it out. pool1 is regularly sorted. - for q in sorted(q1_values, key=lambda x: (x.qhash, x.qlen)): + for q in sorted(q1_values, key=lambda x: x.qhash): print('QDEF1(MP_QSTR_%s, %d, %d, "%s")' % (q.ident, q.qhash, q.qlen, q.qdata)) diff --git a/py/qstr.c b/py/qstr.c index 5b12723ff4184..5cbd6407b18a6 100644 --- a/py/qstr.c +++ b/py/qstr.c @@ -121,7 +121,7 @@ const qstr_pool_t mp_qstr_special_const_pool = { MP_QSTRspecial_const_number_of + 1, // corresponds to number of strings in array just below (qstr_hash_t *)mp_qstr_const_hashes0, (qstr_len_t *)mp_qstr_const_lengths0, - false, // special constant qstrs are not sorted + true, { #ifndef NO_QSTR #define QDEF0(id, hash, len, str) str, @@ -246,7 +246,7 @@ qstr qstr_find_strn(const char *str, size_t str_len) { high = mid; } else { low = mid; - if (cmp == 0) { + if (MP_UNLIKELY(cmp == 0)) { while (low > 0 && pool->hashes[low - 1] == str_hash) { low--; } diff --git a/tools/makemanifest.py b/tools/makemanifest.py index f442eef24c727..c019f9a5058d4 100644 --- a/tools/makemanifest.py +++ b/tools/makemanifest.py @@ -415,7 +415,7 @@ def main(): b'#include "py/emitglue.h"\n' b"extern const qstr_pool_t mp_qstr_const_pool;\n" b"const qstr_pool_t mp_qstr_frozen_const_pool = {\n" - b" (qstr_pool_t*)&mp_qstr_const_pool, MP_QSTRnumber_of, 0, false, 0\n" + b" (qstr_pool_t*)&mp_qstr_const_pool, MP_QSTRnumber_of, 0, 0, NULL, NULL, false, NULL\n" b"};\n" b'const char mp_frozen_names[] = { MP_FROZEN_STR_NAMES "\\0"};\n' b"const mp_raw_code_t *const mp_frozen_mpy_content[] = {NULL};\n" diff --git a/tools/mpy-tool.py b/tools/mpy-tool.py index 015a2b98e6486..d1bfa12718b5c 100755 --- a/tools/mpy-tool.py +++ b/tools/mpy-tool.py @@ -1793,7 +1793,6 @@ def main(): except MPYReadError as er: print(er, file=sys.stderr) sys.exit(1) - base_qstrs = {} if args.hexdump: hexdump_mpy(compiled_modules) pFad - Phonifier reborn

Pfad - The Proxy pFad of © 2024 Garber Painting. All rights reserved.

Note: This service is not intended for secure transactions such as banking, social media, email, or purchasing. Use at your own risk. We assume no liability whatsoever for broken pages.


Alternative Proxies:

Alternative Proxy

pFad Proxy

pFad v3 Proxy

pFad v4 Proxy