esp32: Enable automatic Python heap growth.

projectgus · projectgus · commit 4f67e095b382 · 2023-08-02T17:02:00.000+10:00
Via MICROPY_GC_SPLIT_HEAP_AUTO feature flag added in previous commit.

Tested on ESP32 GENERIC_SPIRAM and GENERIC_S3 configurations, with some
worst-case allocation patterns and the standard test suite.

This work was funded through GitHub Sponsors.

Signed-off-by: Angus Gratton &lt;angus@redyak.com.au&gt;
diff --git a/docs/library/esp32.rst b/docs/library/esp32.rst
@@ -51,13 +51,20 @@ Functions
     buffers and other data. This data is useful to get a sense of how much memory
     is available to ESP-IDF and the networking stack in particular. It may shed
     some light on situations where ESP-IDF operations fail due to allocation failures.
-    The information returned is *not* useful to troubleshoot Python allocation failures,
-    use `micropython.mem_info()` instead.
 
     The capabilities parameter corresponds to ESP-IDF's ``MALLOC_CAP_XXX`` values but the
     two most useful ones are predefined as `esp32.HEAP_DATA` for data heap regions and
     `esp32.HEAP_EXEC` for executable regions as used by the native code emitter.
 
+    Free IDF heap memory in the `esp32.HEAP_DATA` region is available to be
+    automatically added to the MicroPython heap to prevent a MicroPython
+    allocation from failing. However, the information returned here is otherwise
+    *not* useful to troubleshoot Python allocation failures, use
+    `micropython.mem_info()` instead. The "max new split" value in
+    `micropython.mem_info()` output corresponds to the largest free block of
+    ESP-IDF heap that could be automatically added on demand to the MicroPython
+    heap.
+
     The return value is a list of 4-tuples, where each 4-tuple corresponds to one heap
     and contains: the total bytes, the free bytes, the largest free block, and
     the minimum free seen over time.
diff --git a/ports/esp32/gccollect.c b/ports/esp32/gccollect.c
@@ -80,3 +80,15 @@ void gc_collect(void) {
 }
 
 #endif
+
+#if MICROPY_GC_SPLIT_HEAP_AUTO
+
+size_t gc_get_max_new_split(void) {
+    // Need to allow some overhead for heap block metadata.
+    // Am choosing a conservative value as TLSF impl is a bit opaque.
+    static const size_t HEAP_METADATA_OVERHEAD = 64;
+    size_t largest = heap_caps_get_largest_free_block(MALLOC_CAP_DEFAULT);
+    return largest < HEAP_METADATA_OVERHEAD ? 0 : largest - HEAP_METADATA_OVERHEAD;
+}
+
+#endif
diff --git a/ports/esp32/main.c b/ports/esp32/main.c
@@ -100,13 +100,9 @@ void mp_task(void *pvParameter) {
         ESP_LOGE("esp_init", "can't create event loop: 0x%x\n", err);
     }
 
-    // Allocate the uPy heap using malloc and get the largest available region,
-    // limiting to 1/2 total available memory to leave memory for the OS.
-    // When SPIRAM is enabled, this will allocate from SPIRAM.
-    uint32_t caps = MALLOC_CAP_8BIT;
-    size_t heap_total = heap_caps_get_total_size(caps);
-    size_t mp_task_heap_size = MIN(heap_caps_get_largest_free_block(caps), heap_total / 2);
-    void *mp_task_heap = heap_caps_malloc(mp_task_heap_size, caps);
+    /* Heap size starts small, grows on demand */
+    const size_t mp_task_heap_size = 64 * 1024;
+    void *mp_task_heap = MP_PLAT_ALLOC_HEAP(mp_task_heap_size);
 
 soft_reset:
     // initialise the stack pointer for the main thread
diff --git a/ports/esp32/mpconfigport.h b/ports/esp32/mpconfigport.h
@@ -68,6 +68,9 @@
 #define MICROPY_PY_THREAD_GIL               (1)
 #define MICROPY_PY_THREAD_GIL_VM_DIVISOR    (32)
 
+#define MICROPY_GC_SPLIT_HEAP               (1)
+#define MICROPY_GC_SPLIT_HEAP_AUTO          (1)
+
 // extended modules
 #ifndef MICROPY_ESPNOW
 #define MICROPY_ESPNOW                      (1)
