esp32: Experiment with preferring to reserve some ESP-IDF heap.

projectgus · projectgus · commit 78051fbe8c16 · 2023-08-09T15:39:02.000+10:00
gc_get_max_new_split() function extended so the port knows how much memory
Python needs for the current allocation to succeed. It can decide to
reserve some ESP-IDF heap based on the total free heap (or potentially
other considerations).

Signed-off-by: Angus Gratton &lt;angus@redyak.com.au&gt;
diff --git a/ports/esp32/gccollect.c b/ports/esp32/gccollect.c
@@ -83,10 +83,36 @@ void gc_collect(void) {
 
 #if MICROPY_GC_SPLIT_HEAP_AUTO
 
-// The largest new region that is available to become Python heap is the largest
-// free block in the ESP-IDF system heap.
-size_t gc_get_max_new_split(void) {
-    return heap_caps_get_largest_free_block(MALLOC_CAP_DEFAULT);
+// Unless necessary to avoid a Python MemoryError,
+// try to reserve this much free system heap for ESP-IDF
+#define RESERVE_SYSTEM_HEAP (24 * 1024)
+
+size_t gc_get_max_new_split(size_t needed) {
+    multi_heap_info_t info = { 0 };
+    heap_caps_get_info(&info, MALLOC_CAP_DEFAULT);
+
+    // The largest new region that is available to become Python heap is the largest
+    // free block in the ESP-IDF system heap...
+    size_t max = info.largest_free_block;
+
+    if (max <= needed) {
+        return max;
+    }
+
+    // ... unless overall free heap is running low, in which case
+    // prefer to save some RAM for ESP-IDF unless it's needed.
+    if (info.total_free_bytes < max + RESERVE_SYSTEM_HEAP) {
+        if (max > needed + RESERVE_SYSTEM_HEAP) {
+            // Reserve some of the largest free block for the system. New Python
+            // heap area will still be big enough for 'needed'.
+            max -= RESERVE_SYSTEM_HEAP;
+        } else {
+            // Memory is really low, only allow Python exactly what it needs
+            max = needed;
+        }
+    }
+
+    return max;
 }
 
 #endif
diff --git a/py/gc.c b/py/gc.c
@@ -250,7 +250,7 @@ STATIC bool gc_try_add_heap(size_t failed_alloc) {
     // rounding up of partial block sizes.)
     size_t needed = failed_alloc + MAX(2048, failed_alloc * 13 / 512);
 
-    size_t avail = gc_get_max_new_split();
+    size_t avail = gc_get_max_new_split(needed);
 
     DEBUG_printf("gc_try_add_heap failed_alloc " UINT_FMT ", "
         "needed " UINT_FMT ", avail " UINT_FMT " bytes \n",
@@ -1160,7 +1160,7 @@ void gc_dump_info(const mp_print_t *print) {
     mp_printf(print, "GC: total: %u, used: %u, free: %u",
         (uint)info.total, (uint)info.used, (uint)info.free);
     #if MICROPY_GC_SPLIT_HEAP_AUTO
-    mp_printf(print, ", max new split: %u", (uint)gc_get_max_new_split());
+    mp_printf(print, ", max new split: %u", (uint)gc_get_max_new_split(UINT_MAX));
     #endif
     mp_printf(print, "\n No. of 1-blocks: %u, 2-blocks: %u, max blk sz: %u, max free sz: %u\n",
         (uint)info.num_1block, (uint)info.num_2block, (uint)info.max_block, (uint)info.max_free);
diff --git a/py/gc.h b/py/gc.h
@@ -39,7 +39,12 @@ void gc_add(void *start, void *end);
 #if MICROPY_GC_SPLIT_HEAP_AUTO
 // Port must implement this function to return the maximum available block of
 // RAM to allocate a new heap area into using MP_PLAT_ALLOC_HEAP
-size_t gc_get_max_new_split(void);
+//
+// 'needed' is the number of bytes Python needs from the heap for the
+// currently failing allocation to succeed. This can be used to tweak the result
+// and allow Python heap to add more or less RAM. 'needed' will also have value
+// UINT_MAX when called from the mem_info() function, to test the maximum available.
+size_t gc_get_max_new_split(size_t needed);
 #endif // MICROPY_GC_SPLIT_HEAP_AUTO
 #endif // MICROPY_GC_SPLIT_HEAP
 
