py/qstr: Sort qstrs by hash.

amirgon · amirgon · commit 660c81085791 · 2022-02-25T23:45:33.000+02:00
Instead of sorting by string, sort qstrs by (hash, len).
This allows faster binary serach on qstr_find_strn, since it's faster
to compare hashes than strings.
A few strings needed to be moved to special string pool (QDEF0)
because their qstr is assumed to be small (8 bit) on py/scope.c
diff --git a/py/makeqstrdata.py b/py/makeqstrdata.py
@@ -61,7 +61,12 @@
     " ",
     "*",
     "/",
+    "<dictcomp>",
+    "<genexpr>",
+    "<lambda>",
+    "<listcomp>",
     "<module>",
+    "<setcomp>",
     "_",
     "__call__",
     "__class__",
@@ -373,7 +378,7 @@ def print_qstr_data(qstrs):
         print('QDEF0(MP_QSTR_%s, %d, %d, "%s")' % (q.ident, q.qhash, q.qlen, q.qdata))
 
     # go through each qstr in pool 1 and print it out. pool1 is regularly sorted.
-    for q in sorted(q1_values, key=lambda x: x.qstr):
+    for q in sorted(q1_values, key=lambda x: (x.qhash, x.qlen)):
         print('QDEF1(MP_QSTR_%s, %d, %d, "%s")' % (q.ident, q.qhash, q.qlen, q.qdata))
 
 
diff --git a/py/qstr.c b/py/qstr.c
@@ -242,23 +242,25 @@ qstr qstr_find_strn(const char *str, size_t str_len) {
         if (pool->sorted) {
             while (high - low > MP_QSTR_SEARCH_THRESHOLD) {
                 size_t mid = (low + high + 1) / 2;
-                size_t len = pool->lengths[mid];
-                if (len > str_len) {
-                    len = str_len;
-                }
-                int cmp = memcmp(pool->qstrs[mid], str, str_len);
+                int cmp = pool->hashes[mid] - str_hash;
+                if (cmp == 0) cmp = pool->lengths[mid] - str_len;
                 if (cmp < 0) {
                     low = mid;
                 } else if (cmp > 0) {
                     high = mid;
                 } else {
-                    if (pool->lengths[mid] < str_len) {
-                        low = mid;
-                    } else if (pool->lengths[mid] > str_len) {
-                        high = mid;
-                    } else {
-                        return pool->total_prev_len + mid;
-                    }
+                    // avoid a rare (hash,len) collisions
+                    while (MP_UNLIKELY(
+                        pool->lengths[mid] != str_len ||
+                        memcmp(pool->qstrs[mid], str, str_len) != 0)) {
+                        mid++;
+                        if (mid > high || 
+                            pool->hashes[mid] != str_hash ||
+                            pool->lengths[mid] != str_len) {
+                                return 0;
+                            }
+                        }
+                    return pool->total_prev_len + mid;
                 }
             }
         }
diff --git a/tools/mpy-tool.py b/tools/mpy-tool.py
@@ -1273,7 +1273,7 @@ def freeze_mpy(base_qstrs, compiled_modules):
         if q is None or q.qstr_esc in base_qstrs or q.qstr_esc in new:
             continue
         new[q.qstr_esc] = qstrutil.Qstr(len(new), q.qstr_esc, q.str)
-    new = sorted(new.values(), key=lambda x: x.qstr)
+    new = sorted(new.values(), key=lambda x: (x.qhash, x.qlen))
 
     print('#include "py/mpconfig.h"')
     print('#include "py/objint.h"')
