Add ringbuf_queue primiive.

peterhinch · peterhinch · commit 1a6b21969d15 · 2022-11-15T13:43:06.000Z
diff --git a/v3/docs/EVENTS.md b/v3/docs/EVENTS.md
@@ -25,6 +25,7 @@ This document assumes familiarity with `uasyncio`. See [official docs](http://do
   6.2 [EButton](./EVENTS.md#62-ebutton) Debounced pushbutton with double and long press events  
   &nbsp;&nbsp;&nbsp;&nbsp;&nbsp;6.2.1 [The suppress constructor argument](./EVENTS.md#621-the-suppress-constructor-argument)  
   &nbsp;&nbsp;&nbsp;&nbsp;&nbsp;6.2.2 [The sense constructor argument](./EVENTS.md#622-the-sense-constructor-argument)  
+ 7. [Ringbuf queue](./EVENTS.md#7-ringbuf-queue) A MicroPython optimised queue primitive.  
 [Appendix 1 Polling](./EVENTS.md#100-appendix-1-polling)  
 
 # 1. An alternative to callbacks in uasyncio code
@@ -478,6 +479,52 @@ determine whether the button is closed or open.
 
 ###### [Contents](./EVENTS.md#0-contents)
 
+# 7. Ringbuf Queue
+
+The API of the `Queue` aims for CPython compatibility. This is at some cost to
+efficiency. As the name suggests, the `RingbufQueue` class uses a pre-allocated
+circular buffer which may be of any mutable type supporting the buffer protocol
+e.g. `list`, `array` or `bytearray`. 
+
+Attributes of `RingbufQueue`:
+ 1. It is of fixed size, `Queue` can grow to arbitrary size.
+ 2. It uses pre-allocated buffers of various types (`Queue` uses a `list`).
+ 3. It is an asynchronous iterator allowing retrieval with `async for`.
+ 4. It has an "overwrite oldest data" synchronous write mode.
+
+Constructor mandatory arg:
+ * `buf` Buffer for the queue, e.g. list `[0 for _ in range(20)]` or array. A
+ buffer of size `N` can hold a maximum of `N-1` items.
+
+Synchronous methods (immediate return):  
+ * `qsize` No arg. Returns the number of items in the queue.
+ * `empty` No arg. Returns `True` if the queue is empty.
+ * `full` No arg. Returns `True` if the queue is full.
+ * `get_nowait` No arg. Returns an object from the queue. Raises an exception
+ if the queue is empty.
+ * `put_nowait` Arg: the object to put on the queue. Raises an exception if the
+ queue is full. If the calling code ignores the exception the oldest item in
+ the queue will be overwritten. In some applications this can be of use.
+
+Asynchronous methods:  
+ * `put` Arg: the object to put on the queue. If the queue is full, it will
+ block until space is available.
+
+Retrieving items from the queue:
+
+The `RingbufQueue` is an asynchronous iterator. Results are retrieved using
+`async for`:
+```python
+async def handle_queued_data(q):
+    async for obj in q:
+        await asyncio.sleep(0)  # See below
+        # Process obj
+```
+The `sleep` is necessary if you have multiple tasks waiting on the queue,
+otherwise one task hogs all the data.
+
+###### [Contents](./EVENTS.md#0-contents)
+
 # 100 Appendix 1 Polling
 
 The primitives or drivers referenced here do not use polling with the following
diff --git a/v3/docs/TUTORIAL.md b/v3/docs/TUTORIAL.md
@@ -947,8 +947,10 @@ is raised.
 
 ## 3.5 Queue
 
-This is currently an unofficial implementation. Its API is as per CPython
-asyncio.
+This is currently an unofficial implementation. Its API is a subset of that of
+CPython's `asyncio.Queue`. Like `asyncio.Queue` this class is not thread safe.
+A queue class optimised for MicroPython is presented in
+[Ringbuf queue](./EVENTS.md#7-ringbuf-queue).
 
 The `Queue` class provides a means of synchronising producer and consumer
 tasks: the producer puts data items onto the queue with the consumer removing
@@ -1001,14 +1003,15 @@ async def queue_go(delay):
 
 asyncio.run(queue_go(4))
 ```
-In common with CPython's `asyncio.Queue` this class is not thread safe.
 
 ###### [Contents](./TUTORIAL.md#contents)
 
 ## 3.6 ThreadSafeFlag
 
 This requires firmware V1.15 or later.  
-See also [Interfacing uasyncio to interrupts](./INTERRUPTS.md).
+See also [Interfacing uasyncio to interrupts](./INTERRUPTS.md). Because of
+[this issue](https://github.com/micropython/micropython/issues/7965) the
+`ThreadSafeFlag` class does not work under the Unix build.
 
 This official class provides an efficient means of synchronising a task with a
 truly asynchronous event such as a hardware interrupt service routine or code
@@ -1313,7 +1316,9 @@ finally:
 
 ## 3.9 Message
 
-This requires firmware V1.15 or later.
+This requires firmware V1.15 or later. Note that because of
+[this issue](https://github.com/micropython/micropython/issues/7965) the
+`Message` class does not work under the Unix build.
 
 This is an unofficial primitive with no counterpart in CPython asyncio. It uses
 [ThreadSafeFlag](./TUTORIAL.md#36-threadsafeflag) to provide an object similar
diff --git a/v3/primitives/__init__.py b/v3/primitives/__init__.py
@@ -47,6 +47,7 @@ def _handle_exception(loop, context):
     "WaitAny": "events",
     "ESwitch": "events",
     "EButton": "events",
+    "RingbufQueue": "ringbuf_queue",
 }
 
 # Copied from uasyncio.__init__.py
diff --git a/v3/primitives/ringbuf_queue.py b/v3/primitives/ringbuf_queue.py
@@ -0,0 +1,68 @@
+# ringbuf_queue.py Provides RingbufQueue class
+# API differs from CPython
+# Uses pre-allocated ring buffer: can use list or array
+# Asynchronous iterator allowing consumer to use async for
+# put_nowait QueueFull exception can be ignored allowing oldest data to be discarded.
+
+import uasyncio as asyncio
+
+# Exception raised by get_nowait().
+class QueueEmpty(Exception):
+    pass
+
+# Exception raised by put_nowait().
+class QueueFull(Exception):
+    pass
+
+class RingbufQueue:  # MicroPython optimised
+    def __init__(self, buf):
+        self._q = buf
+        self._size = len(buf)
+        self._wi = 0
+        self._ri = 0
+        self._evput = asyncio.Event()  # Triggered by put, tested by get
+        self._evget = asyncio.Event()  # Triggered by get, tested by put
+
+    def full(self):
+        return ((self._wi + 1) % self._size) == self._ri
+
+    def empty(self):
+        return self._ri == self._wi
+
+    def qsize(self):
+        return (self._wi - self._ri) % self._size
+
+    def get_nowait(self):  # Remove and return an item from the queue.
+        # Return an item if one is immediately available, else raise QueueEmpty.
+        if self.empty():
+            raise QueueEmpty()
+        r = self._q[self._ri]
+        self._ri = (self._ri + 1) % self._size
+        return r
+
+    def put_nowait(self, v):
+        self._q[self._wi] = v
+        self._evput.set()  # Schedule any tasks waiting on get
+        self._evput.clear()
+        self._wi = (self._wi + 1) % self._size
+        if self._wi == self._ri:  # Would indicate empty
+            self._ri = (self._ri + 1) % self._size  # Discard a message
+            raise QueueFull  # Caller can ignore if overwrites are OK
+
+    async def put(self, val):  # Usage: await queue.put(item)
+        while self.full():  # Queue full
+            await self._evget.wait()  # May be >1 task waiting on ._evget
+            # Task(s) waiting to get from queue, schedule first Task
+        self.put_nowait(val)
+
+    def __aiter__(self):
+        return self
+
+    async def __anext__(self):
+        while self.empty():  # Empty. May be more than one task waiting on ._evput
+            await self._evput.wait()
+        r = self._q[self._ri]
+        self._ri = (self._ri + 1) % self._size
+        self._evget.set()  # Schedule all tasks waiting on ._evget
+        self._evget.clear()
+        return r
diff --git a/v3/primitives/tests/asyntest.py b/v3/primitives/tests/asyntest.py
@@ -13,11 +13,10 @@
     import uasyncio as asyncio
 except ImportError:
     import asyncio
+import sys
+unix = "linux" in sys.implementation._machine
 
-from primitives.message import Message
-from primitives.barrier import Barrier
-from primitives.semaphore import Semaphore, BoundedSemaphore
-from primitives.condition import Condition
+from primitives import Message, Barrier, Semaphore, BoundedSemaphore, Condition, Queue, RingbufQueue
 
 def print_tests():
     st = '''Available functions:
@@ -30,6 +29,7 @@ def print_tests():
 test(6)  Test BoundedSemaphore.
 test(7)  Test the Condition class.
 test(8)  Test the Queue class.
+test(9)  Test the RingbufQueue class.
 '''
     print('\x1b[32m')
     print(st)
@@ -83,6 +83,9 @@ async def ack_coro(delay):
     print("Time to die...")
 
 def ack_test():
+    if unix:
+        print("Message class is incompatible with Unix build.")
+        return
     printexp('''message was set
 message_wait 1 got message with value 0
 message_wait 2 got message with value 0
@@ -142,6 +145,9 @@ async def run_message_test():
     print('Tasks complete')
 
 def msg_test():
+    if unix:
+        print("Message class is incompatible with Unix build.")
+        return
     printexp('''Test Lock class
 Test Message class
 waiting for message
@@ -389,8 +395,6 @@ def condition_test():
 
 # ************ Queue test ************
 
-from primitives.queue import Queue
-
 async def slow_process():
     await asyncio.sleep(2)
     return 42
@@ -462,7 +466,7 @@ async def queue_go():
         getter(q)
         )
     print('Queue tests complete')
-    print("I've seen starships burn off the shoulder of Orion...")
+    print("I've seen attack ships burn off the shoulder of Orion...")
     print("Time to die...")
 
 def queue_test():
@@ -476,12 +480,86 @@ def queue_test():
 Queue tests complete
 
 
-I've seen starships burn off the shoulder of Orion...
+I've seen attack ships burn off the shoulder of Orion...
 Time to die...
 
 ''', 20)
     asyncio.run(queue_go())
 
+# ************ RingbufQueue test ************
+
+async def qread(q, lst, twr):
+    async for item in q:
+        lst.append(item)
+        await asyncio.sleep_ms(twr)
+
+async def read(q, t, twr=0):
+    lst = []
+    try:
+        await asyncio.wait_for(qread(q, lst, twr), t)
+    except asyncio.TimeoutError:
+        pass
+    return lst
+
+async def put_list(q, lst, twp=0):
+    for item in lst:
+        await q.put(item)
+        await asyncio.sleep_ms(twp)
+
+async def rbq_go():
+    q = RingbufQueue([0 for _ in range(10)])  # 10 elements
+    pl = [n for n in range(15)]
+    print("Read waits on slow write.")
+    asyncio.create_task(put_list(q, pl, 100))
+    rl = await read(q, 2)
+    assert pl == rl
+    print('done')
+    print("Write waits on slow read.")
+    asyncio.create_task(put_list(q, pl))
+    rl = await read(q, 2, 100)
+    assert pl == rl
+    print('done')
+    print("Testing full, empty and qsize methods.")
+    assert q.empty()
+    assert q.qsize() == 0
+    assert not q.full()
+    await put_list(q, (1,2,3))
+    assert not q.empty()
+    assert q.qsize() == 3
+    assert not q.full()
+    print("Done")
+    print("Testing put_nowait and overruns.")
+    nfail = 0
+    for x in range(4, 15):
+        try:
+            q.put_nowait(x)
+        except:
+            nfail += 1
+    assert nfail == 5
+    assert q.full()
+    rl = await read(q, 2)
+    assert rl == [6, 7, 8, 9, 10, 11, 12, 13, 14]
+    print("Tests complete.")
+    print("I've seen attack ships burn off the shoulder of Orion...")
+    print("Time to die...")
+
+def rbq_test():
+    printexp('''Running (runtime = 6s):
+Read waits on slow write.
+done
+Write waits on slow read.
+done
+Testing full, empty and qsize methods.
+Done
+Testing put_nowait and overruns.
+Tests complete.
+I've seen attack ships burn off the shoulder of Orion...
+Time to die...
+
+''', 20)
+    asyncio.run(rbq_go())
+
+# ************ ************
 def test(n):
     try:
         if n == 1:
@@ -500,6 +578,8 @@ def test(n):
             condition_test()  # Test the Condition class.
         elif n == 8:
             queue_test()  # Test the Queue class.
+        elif n == 9:
+            rbq_test()  # Test the RingbufQueue class.
     except KeyboardInterrupt:
         print('Interrupted')
     finally:

Original file line number	Diff line number	Diff line change
`@@ -47,6 +47,7 @@ def _handle_exception(loop, context):`
`47`	`47`	`"WaitAny": "events",`
`48`	`48`	`"ESwitch": "events",`
`49`	`49`	`"EButton": "events",`
	`50`	`+ "RingbufQueue": "ringbuf_queue",`
`50`	`51`	`}`
`51`	`52`
`52`	`53`	`# Copied from uasyncio.__init__.py`