From f8fc78691d4ee401184a62b7974ebdc8f16850ab Mon Sep 17 00:00:00 2001 From: Damien George Date: Sun, 1 Mar 2020 00:06:15 +1100 Subject: [PATCH 01/21] py/mpconfig.h: Enable MICROPY_MODULE_GETATTR by default. To enable lazy loading of submodules (among other things), which is very useful for MicroPython libraries that want to have optional subcomponents. Disabled explicitly on minimal ports. --- ports/bare-arm/mpconfigport.h | 1 + ports/minimal/mpconfigport.h | 1 + py/mpconfig.h | 2 +- 3 files changed, 3 insertions(+), 1 deletion(-) diff --git a/ports/bare-arm/mpconfigport.h b/ports/bare-arm/mpconfigport.h index 273e21451b467..4567676580324 100644 --- a/ports/bare-arm/mpconfigport.h +++ b/ports/bare-arm/mpconfigport.h @@ -43,6 +43,7 @@ #define MICROPY_PY_STRUCT (0) #define MICROPY_PY_SYS (0) #define MICROPY_CPYTHON_COMPAT (0) +#define MICROPY_MODULE_GETATTR (0) #define MICROPY_LONGINT_IMPL (MICROPY_LONGINT_IMPL_NONE) #define MICROPY_FLOAT_IMPL (MICROPY_FLOAT_IMPL_NONE) #define MICROPY_USE_INTERNAL_PRINTF (0) diff --git a/ports/minimal/mpconfigport.h b/ports/minimal/mpconfigport.h index 41e22cd010310..2507d0124d1ca 100644 --- a/ports/minimal/mpconfigport.h +++ b/ports/minimal/mpconfigport.h @@ -55,6 +55,7 @@ #define MICROPY_PY_SYS (0) #define MICROPY_MODULE_FROZEN_MPY (1) #define MICROPY_CPYTHON_COMPAT (0) +#define MICROPY_MODULE_GETATTR (0) #define MICROPY_LONGINT_IMPL (MICROPY_LONGINT_IMPL_NONE) #define MICROPY_FLOAT_IMPL (MICROPY_FLOAT_IMPL_NONE) diff --git a/py/mpconfig.h b/py/mpconfig.h index 371f2ce036539..c5829a3e09568 100644 --- a/py/mpconfig.h +++ b/py/mpconfig.h @@ -729,7 +729,7 @@ typedef double mp_float_t; // Whether to support module-level __getattr__ (see PEP 562) #ifndef MICROPY_MODULE_GETATTR -#define MICROPY_MODULE_GETATTR (0) +#define MICROPY_MODULE_GETATTR (1) #endif // Whether module weak links are supported From ab00f4c44ed55c6796687301636545f80bf99a0e Mon Sep 17 00:00:00 2001 From: Damien George Date: Mon, 2 Mar 2020 15:03:10 +1100 Subject: [PATCH 02/21] qemu-arm: Set default board as mps2-an385 to get more flash for tests. And use Ubuntu bionic for qemu-arm Travic CI job. --- .travis.yml | 1 + ports/qemu-arm/Makefile | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 4164f4fc136c7..ea2b78b12441e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -66,6 +66,7 @@ jobs: # qemu-arm port - stage: test + dist: bionic # needed for more recent version of qemu-system-arm with mps2-an385 target env: NAME="qemu-arm port build and tests" install: - sudo apt-get install gcc-arm-none-eabi diff --git a/ports/qemu-arm/Makefile b/ports/qemu-arm/Makefile index b31284c59b774..f5329fe4d31c5 100644 --- a/ports/qemu-arm/Makefile +++ b/ports/qemu-arm/Makefile @@ -7,7 +7,7 @@ QSTR_DEFS = qstrdefsport.h # include py core make definitions include $(TOP)/py/py.mk -BOARD ?= netduino2 +BOARD ?= mps2-an385 ifeq ($(BOARD),netduino2) CFLAGS += -mthumb -mcpu=cortex-m3 -mfloat-abi=soft From 98ab7643a7e801bd0ad35d7de9635889c22b022a Mon Sep 17 00:00:00 2001 From: Damien George Date: Mon, 2 Mar 2020 15:04:04 +1100 Subject: [PATCH 03/21] travis: Print errors out for OSX job. --- .travis.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.travis.yml b/.travis.yml index ea2b78b12441e..acebdd7b26971 100644 --- a/.travis.yml +++ b/.travis.yml @@ -214,6 +214,8 @@ jobs: - make ${MAKEOPTS} -C ports/unix deplibs - make ${MAKEOPTS} -C ports/unix - make ${MAKEOPTS} -C ports/unix test + after_failure: + - (cd tests && for exp in *.exp; do testbase=$(basename $exp .exp); echo -e "\nFAILURE $testbase"; diff -u $testbase.exp $testbase.out; done) # windows port via mingw - stage: test From c47a3ddf4ac1bdaf74080454e3993b0cb2a97d66 Mon Sep 17 00:00:00 2001 From: Damien George Date: Fri, 13 Mar 2020 14:35:03 +1100 Subject: [PATCH 04/21] py/pairheap: Properly unlink node on pop and delete. This fixes a bug in the pairing-heap implementation when nodes are deleted with mp_pairheap_delete and then reinserted later on. --- py/pairheap.c | 14 +++++++++++--- py/pairheap.h | 9 ++++++--- 2 files changed, 17 insertions(+), 6 deletions(-) diff --git a/py/pairheap.c b/py/pairheap.c index a1eececa0f504..d3a011c4ae7f8 100644 --- a/py/pairheap.c +++ b/py/pairheap.c @@ -72,9 +72,11 @@ mp_pairheap_t *mp_pairheap_pairing(mp_pairheap_lt_t lt, mp_pairheap_t *child) { while (!NEXT_IS_RIGHTMOST_PARENT(child)) { mp_pairheap_t *n1 = child; child = child->next; + n1->next = NULL; if (!NEXT_IS_RIGHTMOST_PARENT(child)) { mp_pairheap_t *n2 = child; child = child->next; + n2->next = NULL; n1 = mp_pairheap_meld(lt, n1, n2); } heap = mp_pairheap_meld(lt, heap, n1); @@ -87,7 +89,9 @@ mp_pairheap_t *mp_pairheap_pairing(mp_pairheap_lt_t lt, mp_pairheap_t *child) { mp_pairheap_t *mp_pairheap_delete(mp_pairheap_lt_t lt, mp_pairheap_t *heap, mp_pairheap_t *node) { // Simple case of the top being the node to delete if (node == heap) { - return mp_pairheap_pairing(lt, heap->child); + mp_pairheap_t *child = heap->child; + node->child = NULL; + return mp_pairheap_pairing(lt, child); } // Case where node is not in the heap @@ -113,18 +117,22 @@ mp_pairheap_t *mp_pairheap_delete(mp_pairheap_lt_t lt, mp_pairheap_t *heap, mp_p node->next = NULL; return heap; } else if (node == parent->child) { + mp_pairheap_t *child = node->child; next = node->next; + node->child = NULL; node->next = NULL; - node = mp_pairheap_pairing(lt, node->child); + node = mp_pairheap_pairing(lt, child); parent->child = node; } else { mp_pairheap_t *n = parent->child; while (node != n->next) { n = n->next; } + mp_pairheap_t *child = node->child; next = node->next; + node->child = NULL; node->next = NULL; - node = mp_pairheap_pairing(lt, node->child); + node = mp_pairheap_pairing(lt, child); if (node == NULL) { node = n; } else { diff --git a/py/pairheap.h b/py/pairheap.h index 8a9138b178764..16ae788097092 100644 --- a/py/pairheap.h +++ b/py/pairheap.h @@ -35,6 +35,7 @@ // Algorithmica 1:111-129, 1986. // https://www.cs.cmu.edu/~sleator/papers/pairing-heaps.pdf +#include #include "py/obj.h" // This struct forms the nodes of the heap and is intended to be extended, by @@ -77,14 +78,16 @@ static inline mp_pairheap_t *mp_pairheap_peek(mp_pairheap_lt_t lt, mp_pairheap_t // Push new node onto existing heap. Returns the new heap. static inline mp_pairheap_t *mp_pairheap_push(mp_pairheap_lt_t lt, mp_pairheap_t *heap, mp_pairheap_t *node) { - node->child = NULL; - node->next = NULL; + assert(node->child == NULL && node->next == NULL); return mp_pairheap_meld(lt, node, heap); // node is first to be stable } // Pop the top off the heap, which must not be empty. Returns the new heap. static inline mp_pairheap_t *mp_pairheap_pop(mp_pairheap_lt_t lt, mp_pairheap_t *heap) { - return mp_pairheap_pairing(lt, heap->child); + assert(heap->next == NULL); + mp_pairheap_t *child = heap->child; + heap->child = NULL; + return mp_pairheap_pairing(lt, child); } #endif // MICROPY_INCLUDED_PY_PAIRHEAP_H From 6c7e78de7223c60f4b762e8b4d33f754d65921d8 Mon Sep 17 00:00:00 2001 From: Damien George Date: Mon, 16 Mar 2020 15:36:43 +1100 Subject: [PATCH 05/21] py/pairheap: Add helper function to initialise a new node. --- py/pairheap.h | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/py/pairheap.h b/py/pairheap.h index 16ae788097092..68b8b0f758d06 100644 --- a/py/pairheap.h +++ b/py/pairheap.h @@ -64,6 +64,13 @@ static inline mp_pairheap_t *mp_pairheap_new(mp_pairheap_lt_t lt) { return NULL; } +// Initialise a single pairing-heap node so it is ready to push on to a heap. +static inline void mp_pairheap_init_node(mp_pairheap_lt_t lt, mp_pairheap_t *node) { + (void)lt; + node->child = NULL; + node->next = NULL; +} + // Test if the heap is empty. static inline bool mp_pairheap_is_empty(mp_pairheap_lt_t lt, mp_pairheap_t *heap) { (void)lt; From f9741d18f6ad8c1aeff93109bda30ff99a687766 Mon Sep 17 00:00:00 2001 From: Damien George Date: Fri, 13 Mar 2020 15:06:10 +1100 Subject: [PATCH 06/21] unix/coverage: Init all pairheap test nodes before using them. --- ports/unix/coverage.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/ports/unix/coverage.c b/ports/unix/coverage.c index 59fcf55245d72..7169c7793bd93 100644 --- a/ports/unix/coverage.c +++ b/ports/unix/coverage.c @@ -149,6 +149,9 @@ STATIC int pairheap_lt(mp_pairheap_t *a, mp_pairheap_t *b) { // ops array contain operations: x>=0 means push(x), x<0 means delete(-x) STATIC void pairheap_test(size_t nops, int *ops) { mp_pairheap_t node[8]; + for (size_t i = 0; i < MP_ARRAY_SIZE(node); ++i) { + mp_pairheap_init_node(pairheap_lt, &node[i]); + } mp_pairheap_t *heap = mp_pairheap_new(pairheap_lt); printf("create:"); for (size_t i = 0; i < nops; ++i) { From f05ae416ff61c9381e467e1bf2558ed005b6cc3a Mon Sep 17 00:00:00 2001 From: Damien George Date: Mon, 16 Mar 2020 15:54:35 +1100 Subject: [PATCH 07/21] stm32/softtimer: Initialise pairing-heap node before pushing to heap. --- ports/stm32/softtimer.c | 1 + 1 file changed, 1 insertion(+) diff --git a/ports/stm32/softtimer.c b/ports/stm32/softtimer.c index ae87e1f76fbef..d0a186c7d0489 100644 --- a/ports/stm32/softtimer.c +++ b/ports/stm32/softtimer.c @@ -81,6 +81,7 @@ void soft_timer_handler(void) { } void soft_timer_insert(soft_timer_entry_t *entry) { + mp_pairheap_init_node(soft_timer_lt, &entry->pairheap); uint32_t irq_state = raise_irq_pri(IRQ_PRI_PENDSV); MP_STATE_PORT(soft_timer_heap) = (soft_timer_entry_t *)mp_pairheap_push(soft_timer_lt, &MP_STATE_PORT(soft_timer_heap)->pairheap, &entry->pairheap); if (entry == MP_STATE_PORT(soft_timer_heap)) { From 63b99443820f53afbdab5201044629d2bfecd73b Mon Sep 17 00:00:00 2001 From: Damien George Date: Wed, 13 Nov 2019 21:07:58 +1100 Subject: [PATCH 08/21] extmod/uasyncio: Add new implementation of uasyncio module. This commit adds a completely new implementation of the uasyncio module. The aim of this version (compared to the original one in micropython-lib) is to be more compatible with CPython's asyncio module, so that one can more easily write code that runs under both MicroPython and CPython (and reuse CPython asyncio libraries, follow CPython asyncio tutorials, etc). Async code is not easy to write and any knowledge users already have from CPython asyncio should transfer to uasyncio without effort, and vice versa. The implementation here attempts to provide good compatibility with CPython's asyncio while still being "micro" enough to run where MicroPython runs. This follows the general philosophy of MicroPython itself, to make it feel like Python. The main change is to use a Task object for each coroutine. This allows more flexibility to queue tasks in various places, eg the main run loop, tasks waiting on events, locks or other tasks. It no longer requires pre-allocating a fixed queue size for the main run loop. A pairing heap is used to queue Tasks. It's currently implemented in pure Python, separated into components with lazy importing for optional components. In the future parts of this implementation can be moved to C to improve speed and reduce memory usage. But the aim is to maintain a pure-Python version as a reference version. --- extmod/uasyncio/__init__.py | 26 ++++ extmod/uasyncio/core.py | 231 ++++++++++++++++++++++++++++++++++++ extmod/uasyncio/event.py | 31 +++++ extmod/uasyncio/funcs.py | 50 ++++++++ extmod/uasyncio/lock.py | 53 +++++++++ extmod/uasyncio/stream.py | 141 ++++++++++++++++++++++ extmod/uasyncio/task.py | 168 ++++++++++++++++++++++++++ 7 files changed, 700 insertions(+) create mode 100644 extmod/uasyncio/__init__.py create mode 100644 extmod/uasyncio/core.py create mode 100644 extmod/uasyncio/event.py create mode 100644 extmod/uasyncio/funcs.py create mode 100644 extmod/uasyncio/lock.py create mode 100644 extmod/uasyncio/stream.py create mode 100644 extmod/uasyncio/task.py diff --git a/extmod/uasyncio/__init__.py b/extmod/uasyncio/__init__.py new file mode 100644 index 0000000000000..6bff13883ae70 --- /dev/null +++ b/extmod/uasyncio/__init__.py @@ -0,0 +1,26 @@ +# MicroPython uasyncio module +# MIT license; Copyright (c) 2019 Damien P. George + +from .core import * + +__version__ = (3, 0, 0) + +_attrs = { + "wait_for": "funcs", + "gather": "funcs", + "Event": "event", + "Lock": "lock", + "open_connection": "stream", + "start_server": "stream", +} + +# Lazy loader, effectively does: +# global attr +# from .mod import attr +def __getattr__(attr): + mod = _attrs.get(attr, None) + if mod is None: + raise AttributeError(attr) + value = getattr(__import__(mod, None, None, True, 1), attr) + globals()[attr] = value + return value diff --git a/extmod/uasyncio/core.py b/extmod/uasyncio/core.py new file mode 100644 index 0000000000000..049e2537f8df6 --- /dev/null +++ b/extmod/uasyncio/core.py @@ -0,0 +1,231 @@ +# MicroPython uasyncio module +# MIT license; Copyright (c) 2019 Damien P. George + +from time import ticks_ms as ticks, ticks_diff, ticks_add +import sys, select + +# Import TaskQueue and Task +from .task import TaskQueue, Task + + +################################################################################ +# Exceptions + + +class CancelledError(BaseException): + pass + + +class TimeoutError(Exception): + pass + + +################################################################################ +# Sleep functions + +# "Yield" once, then raise StopIteration +class SingletonGenerator: + def __init__(self): + self.state = None + self.exc = StopIteration() + + def __iter__(self): + return self + + def __next__(self): + if self.state is not None: + _task_queue.push_sorted(cur_task, self.state) + self.state = None + return None + else: + self.exc.__traceback__ = None + raise self.exc + + +# Pause task execution for the given time (integer in milliseconds, uPy extension) +# Use a SingletonGenerator to do it without allocating on the heap +def sleep_ms(t, sgen=SingletonGenerator()): + assert sgen.state is None + sgen.state = ticks_add(ticks(), t) + return sgen + + +# Pause task execution for the given time (in seconds) +def sleep(t): + return sleep_ms(int(t * 1000)) + + +################################################################################ +# Queue and poller for stream IO + + +class IOQueue: + def __init__(self): + self.poller = select.poll() + self.map = {} # maps id(stream) to [task_waiting_read, task_waiting_write, stream] + + def _enqueue(self, s, idx): + if id(s) not in self.map: + entry = [None, None, s] + entry[idx] = cur_task + self.map[id(s)] = entry + self.poller.register(s, select.POLLIN if idx == 0 else select.POLLOUT) + else: + sm = self.map[id(s)] + assert sm[idx] is None + assert sm[1 - idx] is not None + sm[idx] = cur_task + self.poller.modify(s, select.POLLIN | select.POLLOUT) + # Link task to this IOQueue so it can be removed if needed + cur_task.data = self + + def _dequeue(self, s): + del self.map[id(s)] + self.poller.unregister(s) + + def queue_read(self, s): + self._enqueue(s, 0) + + def queue_write(self, s): + self._enqueue(s, 1) + + def remove(self, task): + while True: + del_s = None + for k in self.map: # Iterate without allocating on the heap + q0, q1, s = self.map[k] + if q0 is task or q1 is task: + del_s = s + break + if del_s is not None: + self._dequeue(s) + else: + break + + def wait_io_event(self, dt): + for s, ev in self.poller.ipoll(dt): + sm = self.map[id(s)] + # print('poll', s, sm, ev) + if ev & ~select.POLLOUT and sm[0] is not None: + # POLLIN or error + _task_queue.push_head(sm[0]) + sm[0] = None + if ev & ~select.POLLIN and sm[1] is not None: + # POLLOUT or error + _task_queue.push_head(sm[1]) + sm[1] = None + if sm[0] is None and sm[1] is None: + self._dequeue(s) + elif sm[0] is None: + self.poller.modify(s, select.POLLOUT) + else: + self.poller.modify(s, select.POLLIN) + + +################################################################################ +# Main run loop + +# TaskQueue of Task instances +_task_queue = TaskQueue() + +# Task queue and poller for stream IO +_io_queue = IOQueue() + + +# Ensure the awaitable is a task +def _promote_to_task(aw): + return aw if isinstance(aw, Task) else create_task(aw) + + +# Create and schedule a new task from a coroutine +def create_task(coro): + if not hasattr(coro, "send"): + raise TypeError("coroutine expected") + t = Task(coro, globals()) + _task_queue.push_head(t) + return t + + +# Keep scheduling tasks until there are none left to schedule +def run_until_complete(main_task=None): + global cur_task + excs_all = (CancelledError, Exception) # To prevent heap allocation in loop + excs_stop = (CancelledError, StopIteration) # To prevent heap allocation in loop + while True: + # Wait until the head of _task_queue is ready to run + dt = 1 + while dt > 0: + dt = -1 + t = _task_queue.peek() + if t: + # A task waiting on _task_queue; "ph_key" is time to schedule task at + dt = max(0, ticks_diff(t.ph_key, ticks())) + elif not _io_queue.map: + # No tasks can be woken so finished running + return + # print('(poll {})'.format(dt), len(_io_queue.map)) + _io_queue.wait_io_event(dt) + + # Get next task to run and continue it + t = _task_queue.pop_head() + cur_task = t + try: + # Continue running the coroutine, it's responsible for rescheduling itself + exc = t.data + if not exc: + t.coro.send(None) + else: + t.data = None + t.coro.throw(exc) + except excs_all as er: + # Check the task is not on any event queue + assert t.data is None + # This task is done, check if it's the main task and then loop should stop + if t is main_task: + if isinstance(er, StopIteration): + return er.value + raise er + # Save return value of coro to pass up to caller + t.data = er + # Schedule any other tasks waiting on the completion of this task + waiting = False + if hasattr(t, "waiting"): + while t.waiting.peek(): + _task_queue.push_head(t.waiting.pop_head()) + waiting = True + t.waiting = None # Free waiting queue head + # Print out exception for detached tasks + if not waiting and not isinstance(er, excs_stop): + print("task raised exception:", t.coro) + sys.print_exception(er) + # Indicate task is done + t.coro = None + + +# Create a new task from a coroutine and run it until it finishes +def run(coro): + return run_until_complete(create_task(coro)) + + +################################################################################ +# Event loop wrapper + + +class Loop: + def create_task(self, coro): + return create_task(coro) + + def run_forever(self): + run_until_complete() + # TODO should keep running until .stop() is called, even if there're no tasks left + + def run_until_complete(self, aw): + return run_until_complete(_promote_to_task(aw)) + + def close(self): + pass + + +# The runq_len and waitq_len arguments are for legacy uasyncio compatibility +def get_event_loop(runq_len=0, waitq_len=0): + return Loop() diff --git a/extmod/uasyncio/event.py b/extmod/uasyncio/event.py new file mode 100644 index 0000000000000..31cb00e055deb --- /dev/null +++ b/extmod/uasyncio/event.py @@ -0,0 +1,31 @@ +# MicroPython uasyncio module +# MIT license; Copyright (c) 2019-2020 Damien P. George + +from . import core + +# Event class for primitive events that can be waited on, set, and cleared +class Event: + def __init__(self): + self.state = False # False=unset; True=set + self.waiting = core.TaskQueue() # Queue of Tasks waiting on completion of this event + + def is_set(self): + return self.state + + def set(self): + # Event becomes set, schedule any tasks waiting on it + while self.waiting.peek(): + core._task_queue.push_head(self.waiting.pop_head()) + self.state = True + + def clear(self): + self.state = False + + async def wait(self): + if not self.state: + # Event not set, put the calling task on the event's waiting queue + self.waiting.push_head(core.cur_task) + # Set calling task's data to the event's queue so it can be removed if needed + core.cur_task.data = self.waiting + yield + return True diff --git a/extmod/uasyncio/funcs.py b/extmod/uasyncio/funcs.py new file mode 100644 index 0000000000000..7a4bddf25695a --- /dev/null +++ b/extmod/uasyncio/funcs.py @@ -0,0 +1,50 @@ +# MicroPython uasyncio module +# MIT license; Copyright (c) 2019-2020 Damien P. George + +from . import core + + +async def wait_for(aw, timeout): + aw = core._promote_to_task(aw) + if timeout is None: + return await aw + + def cancel(aw, timeout): + await core.sleep(timeout) + aw.cancel() + + cancel_task = core.create_task(cancel(aw, timeout)) + try: + ret = await aw + except core.CancelledError: + # Ignore CancelledError from aw, it's probably due to timeout + pass + finally: + # Cancel the "cancel" task if it's still active (optimisation instead of cancel_task.cancel()) + if cancel_task.coro is not None: + core._task_queue.remove(cancel_task) + if cancel_task.coro is None: + # Cancel task ran to completion, ie there was a timeout + raise core.TimeoutError + return ret + + +async def gather(*aws, return_exceptions=False): + ts = [core._promote_to_task(aw) for aw in aws] + for i in range(len(ts)): + try: + # TODO handle cancel of gather itself + # if ts[i].coro: + # iter(ts[i]).waiting.push_head(cur_task) + # try: + # yield + # except CancelledError as er: + # # cancel all waiting tasks + # raise er + ts[i] = await ts[i] + except Exception as er: + if return_exceptions: + ts[i] = er + else: + raise er + return ts diff --git a/extmod/uasyncio/lock.py b/extmod/uasyncio/lock.py new file mode 100644 index 0000000000000..18a55cb483c9a --- /dev/null +++ b/extmod/uasyncio/lock.py @@ -0,0 +1,53 @@ +# MicroPython uasyncio module +# MIT license; Copyright (c) 2019-2020 Damien P. George + +from . import core + +# Lock class for primitive mutex capability +class Lock: + def __init__(self): + # The state can take the following values: + # - 0: unlocked + # - 1: locked + # - : unlocked but this task has been scheduled to acquire the lock next + self.state = 0 + # Queue of Tasks waiting to acquire this Lock + self.waiting = core.TaskQueue() + + def locked(self): + return self.state == 1 + + def release(self): + if self.state != 1: + raise RuntimeError + if self.waiting.peek(): + # Task(s) waiting on lock, schedule next Task + self.state = self.waiting.pop_head() + core._task_queue.push_head(self.state) + else: + # No Task waiting so unlock + self.state = 0 + + async def acquire(self): + if self.state != 0: + # Lock unavailable, put the calling Task on the waiting queue + self.waiting.push_head(core.cur_task) + # Set calling task's data to the lock's queue so it can be removed if needed + core.cur_task.data = self.waiting + try: + yield + except core.CancelledError as er: + if self.state == core.cur_task: + # Cancelled while pending on resume, schedule next waiting Task + self.state = 1 + self.release() + raise er + # Lock available, set it as locked + self.state = 1 + return True + + async def __aenter__(self): + return await self.acquire() + + async def __aexit__(self, exc_type, exc, tb): + return self.release() diff --git a/extmod/uasyncio/stream.py b/extmod/uasyncio/stream.py new file mode 100644 index 0000000000000..7803ac4bfa6bb --- /dev/null +++ b/extmod/uasyncio/stream.py @@ -0,0 +1,141 @@ +# MicroPython uasyncio module +# MIT license; Copyright (c) 2019-2020 Damien P. George + +from . import core + + +class Stream: + def __init__(self, s, e={}): + self.s = s + self.e = e + self.out_buf = b"" + + def get_extra_info(self, v): + return self.e[v] + + async def __aenter__(self): + return self + + async def __aexit__(self, exc_type, exc, tb): + await self.close() + + def close(self): + pass + + async def wait_closed(self): + # TODO yield? + self.s.close() + + async def read(self, n): + yield core._io_queue.queue_read(self.s) + return self.s.read(n) + + async def readline(self): + l = b"" + while True: + yield core._io_queue.queue_read(self.s) + l2 = self.s.readline() # may do multiple reads but won't block + l += l2 + if not l2 or l[-1] == 10: # \n (check l in case l2 is str) + return l + + def write(self, buf): + self.out_buf += buf + + async def drain(self): + mv = memoryview(self.out_buf) + off = 0 + while off < len(mv): + yield core._io_queue.queue_write(self.s) + ret = self.s.write(mv[off:]) + if ret is not None: + off += ret + self.out_buf = b"" + + +# Create a TCP stream connection to a remote host +async def open_connection(host, port): + from uerrno import EINPROGRESS + import usocket as socket + + ai = socket.getaddrinfo(host, port)[0] # TODO this is blocking! + s = socket.socket() + s.setblocking(False) + ss = Stream(s) + try: + s.connect(ai[-1]) + except OSError as er: + if er.args[0] != EINPROGRESS: + raise er + yield core._io_queue.queue_write(s) + return ss, ss + + +# Class representing a TCP stream server, can be closed and used in "async with" +class Server: + async def __aenter__(self): + return self + + async def __aexit__(self, exc_type, exc, tb): + self.close() + await self.wait_closed() + + def close(self): + self.task.cancel() + + async def wait_closed(self): + await self.task + + async def _serve(self, cb, host, port, backlog): + import usocket as socket + + ai = socket.getaddrinfo(host, port)[0] # TODO this is blocking! + s = socket.socket() + s.setblocking(False) + s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + s.bind(ai[-1]) + s.listen(backlog) + self.task = core.cur_task + # Accept incoming connections + while True: + try: + yield core._io_queue.queue_read(s) + except core.CancelledError: + # Shutdown server + s.close() + return + try: + s2, addr = s.accept() + except: + # Ignore a failed accept + continue + s2.setblocking(False) + s2s = Stream(s2, {"peername": addr}) + core.create_task(cb(s2s, s2s)) + + +# Helper function to start a TCP stream server, running as a new task +# TODO could use an accept-callback on socket read activity instead of creating a task +async def start_server(cb, host, port, backlog=5): + s = Server() + core.create_task(s._serve(cb, host, port, backlog)) + return s + + +################################################################################ +# Legacy uasyncio compatibility + + +async def stream_awrite(self, buf, off=0, sz=-1): + if off != 0 or sz != -1: + buf = memoryview(buf) + if sz == -1: + sz = len(buf) + buf = buf[off : off + sz] + self.write(buf) + await self.drain() + + +Stream.aclose = Stream.wait_closed +Stream.awrite = stream_awrite +Stream.awritestr = stream_awrite # TODO explicitly convert to bytes? diff --git a/extmod/uasyncio/task.py b/extmod/uasyncio/task.py new file mode 100644 index 0000000000000..1d5bcc5cf7705 --- /dev/null +++ b/extmod/uasyncio/task.py @@ -0,0 +1,168 @@ +# MicroPython uasyncio module +# MIT license; Copyright (c) 2019-2020 Damien P. George + +# This file contains the core TaskQueue based on a pairing heap, and the core Task class. +# They can optionally be replaced by C implementations. + +from . import core + + +# pairing-heap meld of 2 heaps; O(1) +def ph_meld(h1, h2): + if h1 is None: + return h2 + if h2 is None: + return h1 + lt = core.ticks_diff(h1.ph_key, h2.ph_key) < 0 + if lt: + if h1.ph_child is None: + h1.ph_child = h2 + else: + h1.ph_child_last.ph_next = h2 + h1.ph_child_last = h2 + h2.ph_next = None + h2.ph_rightmost_parent = h1 + return h1 + else: + h1.ph_next = h2.ph_child + h2.ph_child = h1 + if h1.ph_next is None: + h2.ph_child_last = h1 + h1.ph_rightmost_parent = h2 + return h2 + + +# pairing-heap pairing operation; amortised O(log N) +def ph_pairing(child): + heap = None + while child is not None: + n1 = child + child = child.ph_next + n1.ph_next = None + if child is not None: + n2 = child + child = child.ph_next + n2.ph_next = None + n1 = ph_meld(n1, n2) + heap = ph_meld(heap, n1) + return heap + + +# pairing-heap delete of a node; stable, amortised O(log N) +def ph_delete(heap, node): + if node is heap: + child = heap.ph_child + node.ph_child = None + return ph_pairing(child) + # Find parent of node + parent = node + while parent.ph_next is not None: + parent = parent.ph_next + parent = parent.ph_rightmost_parent + # Replace node with pairing of its children + if node is parent.ph_child and node.ph_child is None: + parent.ph_child = node.ph_next + node.ph_next = None + return heap + elif node is parent.ph_child: + child = node.ph_child + next = node.ph_next + node.ph_child = None + node.ph_next = None + node = ph_pairing(child) + parent.ph_child = node + else: + n = parent.ph_child + while node is not n.ph_next: + n = n.ph_next + child = node.ph_child + next = node.ph_next + node.ph_child = None + node.ph_next = None + node = ph_pairing(child) + if node is None: + node = n + else: + n.ph_next = node + node.ph_next = next + if next is None: + node.ph_rightmost_parent = parent + parent.ph_child_last = node + return heap + + +# TaskQueue class based on the above pairing-heap functions. +class TaskQueue: + def __init__(self): + self.heap = None + + def peek(self): + return self.heap + + def push_sorted(self, v, key): + v.data = None + v.ph_key = key + v.ph_child = None + v.ph_next = None + self.heap = ph_meld(v, self.heap) + + def push_head(self, v): + self.push_sorted(v, core.ticks()) + + def pop_head(self): + v = self.heap + self.heap = ph_pairing(self.heap.ph_child) + return v + + def remove(self, v): + self.heap = ph_delete(self.heap, v) + + +# Task class representing a coroutine, can be waited on and cancelled. +class Task: + def __init__(self, coro, globals=None): + self.coro = coro # Coroutine of this Task + self.data = None # General data for queue it is waiting on + self.ph_key = 0 # Pairing heap + self.ph_child = None # Paring heap + self.ph_child_last = None # Paring heap + self.ph_next = None # Paring heap + self.ph_rightmost_parent = None # Paring heap + + def __iter__(self): + if not hasattr(self, "waiting"): + # Lazily allocated head of linked list of Tasks waiting on completion of this task. + self.waiting = TaskQueue() + return self + + def __next__(self): + if not self.coro: + # Task finished, raise return value to caller so it can continue. + raise self.data + else: + # Put calling task on waiting queue. + self.waiting.push_head(core.cur_task) + # Set calling task's data to this task that it waits on, to double-link it. + core.cur_task.data = self + + def cancel(self): + # Check if task is already finished. + if self.coro is None: + return False + # Can't cancel self (not supported yet). + if self is core.cur_task: + raise RuntimeError("cannot cancel self") + # If Task waits on another task then forward the cancel to the one it's waiting on. + while isinstance(self.data, Task): + self = self.data + # Reschedule Task as a cancelled task. + if hasattr(self.data, "remove"): + # Not on the main running queue, remove the task from the queue it's on. + self.data.remove(self) + core._task_queue.push_head(self) + elif core.ticks_diff(self.ph_key, core.ticks()) > 0: + # On the main running queue but scheduled in the future, so bring it forward to now. + core._task_queue.remove(self) + core._task_queue.push_head(self) + self.data = core.CancelledError + return True From c4935f30490d0446e16a51dbf7a6397b771cf804 Mon Sep 17 00:00:00 2001 From: Damien George Date: Wed, 13 Nov 2019 21:08:22 +1100 Subject: [PATCH 09/21] tests/extmod: Add uasyncio tests. All .exp files are included because they require CPython 3.8 which may not always be available. --- tests/extmod/uasyncio_await_return.py | 26 ++++++ tests/extmod/uasyncio_await_return.py.exp | 2 + tests/extmod/uasyncio_basic.py | 43 ++++++++++ tests/extmod/uasyncio_basic.py.exp | 5 ++ tests/extmod/uasyncio_basic2.py | 24 ++++++ tests/extmod/uasyncio_basic2.py.exp | 4 + tests/extmod/uasyncio_cancel_fair.py | 37 +++++++++ tests/extmod/uasyncio_cancel_fair.py.exp | 24 ++++++ tests/extmod/uasyncio_cancel_fair2.py | 37 +++++++++ tests/extmod/uasyncio_cancel_fair2.py.exp | 8 ++ tests/extmod/uasyncio_cancel_self.py | 31 +++++++ tests/extmod/uasyncio_cancel_self.py.exp | 2 + tests/extmod/uasyncio_cancel_task.py | 85 ++++++++++++++++++++ tests/extmod/uasyncio_cancel_task.py.exp | 31 +++++++ tests/extmod/uasyncio_event.py | 98 +++++++++++++++++++++++ tests/extmod/uasyncio_event.py.exp | 33 ++++++++ tests/extmod/uasyncio_event_fair.py | 40 +++++++++ tests/extmod/uasyncio_event_fair.py.exp | 16 ++++ tests/extmod/uasyncio_exception.py | 60 ++++++++++++++ tests/extmod/uasyncio_exception.py.exp | 7 ++ tests/extmod/uasyncio_fair.py | 32 ++++++++ tests/extmod/uasyncio_fair.py.exp | 12 +++ tests/extmod/uasyncio_gather.py | 49 ++++++++++++ tests/extmod/uasyncio_gather.py.exp | 10 +++ tests/extmod/uasyncio_get_event_loop.py | 20 +++++ tests/extmod/uasyncio_heaplock.py | 46 +++++++++++ tests/extmod/uasyncio_heaplock.py.exp | 11 +++ tests/extmod/uasyncio_lock.py | 97 ++++++++++++++++++++++ tests/extmod/uasyncio_lock.py.exp | 41 ++++++++++ tests/extmod/uasyncio_lock_cancel.py | 55 +++++++++++++ tests/extmod/uasyncio_lock_cancel.py.exp | 11 +++ tests/extmod/uasyncio_wait_for.py | 62 ++++++++++++++ tests/extmod/uasyncio_wait_for.py.exp | 15 ++++ tests/extmod/uasyncio_wait_task.py | 77 ++++++++++++++++++ tests/extmod/uasyncio_wait_task.py.exp | 10 +++ 35 files changed, 1161 insertions(+) create mode 100644 tests/extmod/uasyncio_await_return.py create mode 100644 tests/extmod/uasyncio_await_return.py.exp create mode 100644 tests/extmod/uasyncio_basic.py create mode 100644 tests/extmod/uasyncio_basic.py.exp create mode 100644 tests/extmod/uasyncio_basic2.py create mode 100644 tests/extmod/uasyncio_basic2.py.exp create mode 100644 tests/extmod/uasyncio_cancel_fair.py create mode 100644 tests/extmod/uasyncio_cancel_fair.py.exp create mode 100644 tests/extmod/uasyncio_cancel_fair2.py create mode 100644 tests/extmod/uasyncio_cancel_fair2.py.exp create mode 100644 tests/extmod/uasyncio_cancel_self.py create mode 100644 tests/extmod/uasyncio_cancel_self.py.exp create mode 100644 tests/extmod/uasyncio_cancel_task.py create mode 100644 tests/extmod/uasyncio_cancel_task.py.exp create mode 100644 tests/extmod/uasyncio_event.py create mode 100644 tests/extmod/uasyncio_event.py.exp create mode 100644 tests/extmod/uasyncio_event_fair.py create mode 100644 tests/extmod/uasyncio_event_fair.py.exp create mode 100644 tests/extmod/uasyncio_exception.py create mode 100644 tests/extmod/uasyncio_exception.py.exp create mode 100644 tests/extmod/uasyncio_fair.py create mode 100644 tests/extmod/uasyncio_fair.py.exp create mode 100644 tests/extmod/uasyncio_gather.py create mode 100644 tests/extmod/uasyncio_gather.py.exp create mode 100644 tests/extmod/uasyncio_get_event_loop.py create mode 100644 tests/extmod/uasyncio_heaplock.py create mode 100644 tests/extmod/uasyncio_heaplock.py.exp create mode 100644 tests/extmod/uasyncio_lock.py create mode 100644 tests/extmod/uasyncio_lock.py.exp create mode 100644 tests/extmod/uasyncio_lock_cancel.py create mode 100644 tests/extmod/uasyncio_lock_cancel.py.exp create mode 100644 tests/extmod/uasyncio_wait_for.py create mode 100644 tests/extmod/uasyncio_wait_for.py.exp create mode 100644 tests/extmod/uasyncio_wait_task.py create mode 100644 tests/extmod/uasyncio_wait_task.py.exp diff --git a/tests/extmod/uasyncio_await_return.py b/tests/extmod/uasyncio_await_return.py new file mode 100644 index 0000000000000..d375c9ea97ab3 --- /dev/null +++ b/tests/extmod/uasyncio_await_return.py @@ -0,0 +1,26 @@ +# Test that tasks return their value correctly to the caller + +try: + import uasyncio as asyncio +except ImportError: + try: + import asyncio + except ImportError: + print("SKIP") + raise SystemExit + + +async def foo(): + return 42 + + +async def main(): + # Call function directly via an await + print(await foo()) + + # Create a task and await on it + task = asyncio.create_task(foo()) + print(await task) + + +asyncio.run(main()) diff --git a/tests/extmod/uasyncio_await_return.py.exp b/tests/extmod/uasyncio_await_return.py.exp new file mode 100644 index 0000000000000..daaac9e303029 --- /dev/null +++ b/tests/extmod/uasyncio_await_return.py.exp @@ -0,0 +1,2 @@ +42 +42 diff --git a/tests/extmod/uasyncio_basic.py b/tests/extmod/uasyncio_basic.py new file mode 100644 index 0000000000000..f6685fa674676 --- /dev/null +++ b/tests/extmod/uasyncio_basic.py @@ -0,0 +1,43 @@ +try: + import uasyncio as asyncio +except ImportError: + try: + import asyncio + except ImportError: + print("SKIP") + raise SystemExit + + +try: + import utime + + ticks = utime.ticks_ms + ticks_diff = utime.ticks_diff +except: + import time + + ticks = lambda: int(time.time() * 1000) + ticks_diff = lambda t1, t0: t1 - t0 + + +async def delay_print(t, s): + await asyncio.sleep(t) + print(s) + + +async def main(): + print("start") + + await asyncio.sleep(0.001) + print("after sleep") + + t0 = ticks() + await delay_print(0.02, "short") + t1 = ticks() + await delay_print(0.04, "long") + t2 = ticks() + + print("took {} {}".format(round(ticks_diff(t1, t0), -1), round(ticks_diff(t2, t1), -1))) + + +asyncio.run(main()) diff --git a/tests/extmod/uasyncio_basic.py.exp b/tests/extmod/uasyncio_basic.py.exp new file mode 100644 index 0000000000000..447d05d5e0984 --- /dev/null +++ b/tests/extmod/uasyncio_basic.py.exp @@ -0,0 +1,5 @@ +start +after sleep +short +long +took 20 40 diff --git a/tests/extmod/uasyncio_basic2.py b/tests/extmod/uasyncio_basic2.py new file mode 100644 index 0000000000000..a2167e48ee1a7 --- /dev/null +++ b/tests/extmod/uasyncio_basic2.py @@ -0,0 +1,24 @@ +try: + import uasyncio as asyncio +except ImportError: + try: + import asyncio + except ImportError: + print("SKIP") + raise SystemExit + + +async def forever(): + print("forever start") + await asyncio.sleep(10) + + +async def main(): + print("main start") + asyncio.create_task(forever()) + await asyncio.sleep(0.001) + print("main done") + return 42 + + +print(asyncio.run(main())) diff --git a/tests/extmod/uasyncio_basic2.py.exp b/tests/extmod/uasyncio_basic2.py.exp new file mode 100644 index 0000000000000..3ca3521728d44 --- /dev/null +++ b/tests/extmod/uasyncio_basic2.py.exp @@ -0,0 +1,4 @@ +main start +forever start +main done +42 diff --git a/tests/extmod/uasyncio_cancel_fair.py b/tests/extmod/uasyncio_cancel_fair.py new file mode 100644 index 0000000000000..9a7b35c1618af --- /dev/null +++ b/tests/extmod/uasyncio_cancel_fair.py @@ -0,0 +1,37 @@ +# Test fairness of cancelling a task +# That tasks which continuously cancel each other don't take over the scheduler + +try: + import uasyncio as asyncio +except ImportError: + try: + import asyncio + except ImportError: + print("SKIP") + raise SystemExit + + +async def task(id, other): + for i in range(3): + try: + print("start", id) + await asyncio.sleep(0) + print("done", id) + except asyncio.CancelledError as er: + print("cancelled", id) + if other is not None: + print(id, "cancels", other) + tasks[other].cancel() + + +async def main(): + global tasks + tasks = [ + asyncio.create_task(task(0, 1)), + asyncio.create_task(task(1, 0)), + asyncio.create_task(task(2, None)), + ] + await tasks[2] + + +asyncio.run(main()) diff --git a/tests/extmod/uasyncio_cancel_fair.py.exp b/tests/extmod/uasyncio_cancel_fair.py.exp new file mode 100644 index 0000000000000..8f5da08e4ced4 --- /dev/null +++ b/tests/extmod/uasyncio_cancel_fair.py.exp @@ -0,0 +1,24 @@ +start 0 +start 1 +start 2 +done 0 +0 cancels 1 +start 0 +cancelled 1 +1 cancels 0 +start 1 +done 2 +start 2 +cancelled 0 +0 cancels 1 +start 0 +cancelled 1 +1 cancels 0 +start 1 +done 2 +start 2 +cancelled 0 +0 cancels 1 +cancelled 1 +1 cancels 0 +done 2 diff --git a/tests/extmod/uasyncio_cancel_fair2.py b/tests/extmod/uasyncio_cancel_fair2.py new file mode 100644 index 0000000000000..46e40f71b15ca --- /dev/null +++ b/tests/extmod/uasyncio_cancel_fair2.py @@ -0,0 +1,37 @@ +# Test fairness of cancelling a task +# That tasks which keeps being cancelled by multiple other tasks gets a chance to run + +try: + import uasyncio as asyncio +except ImportError: + try: + import asyncio + except ImportError: + print("SKIP") + raise SystemExit + + +async def task_a(): + try: + while True: + print("sleep a") + await asyncio.sleep(0) + except asyncio.CancelledError: + print("cancelled a") + + +async def task_b(id, other): + while other.cancel(): + print("sleep b", id) + await asyncio.sleep(0) + print("done b", id) + + +async def main(): + t = asyncio.create_task(task_a()) + for i in range(3): + asyncio.create_task(task_b(i, t)) + await t + + +asyncio.run(main()) diff --git a/tests/extmod/uasyncio_cancel_fair2.py.exp b/tests/extmod/uasyncio_cancel_fair2.py.exp new file mode 100644 index 0000000000000..e9dd649b453f9 --- /dev/null +++ b/tests/extmod/uasyncio_cancel_fair2.py.exp @@ -0,0 +1,8 @@ +sleep a +sleep b 0 +sleep b 1 +sleep b 2 +cancelled a +done b 0 +done b 1 +done b 2 diff --git a/tests/extmod/uasyncio_cancel_self.py b/tests/extmod/uasyncio_cancel_self.py new file mode 100644 index 0000000000000..660ae66389366 --- /dev/null +++ b/tests/extmod/uasyncio_cancel_self.py @@ -0,0 +1,31 @@ +# Test a task cancelling itself (currently unsupported) + +try: + import uasyncio as asyncio +except ImportError: + try: + import asyncio + except ImportError: + print("SKIP") + raise SystemExit + + +async def task(): + print("task start") + global_task.cancel() + + +async def main(): + global global_task + global_task = asyncio.create_task(task()) + try: + await global_task + except asyncio.CancelledError: + print("main cancel") + print("main done") + + +try: + asyncio.run(main()) +except RuntimeError as er: + print(er) diff --git a/tests/extmod/uasyncio_cancel_self.py.exp b/tests/extmod/uasyncio_cancel_self.py.exp new file mode 100644 index 0000000000000..36fcb0a3f4f54 --- /dev/null +++ b/tests/extmod/uasyncio_cancel_self.py.exp @@ -0,0 +1,2 @@ +task start +cannot cancel self diff --git a/tests/extmod/uasyncio_cancel_task.py b/tests/extmod/uasyncio_cancel_task.py new file mode 100644 index 0000000000000..ec60d85545740 --- /dev/null +++ b/tests/extmod/uasyncio_cancel_task.py @@ -0,0 +1,85 @@ +# Test cancelling a task + +try: + import uasyncio as asyncio +except ImportError: + try: + import asyncio + except ImportError: + print("SKIP") + raise SystemExit + + +async def task(s, allow_cancel): + try: + print("task start") + await asyncio.sleep(s) + print("task done") + except asyncio.CancelledError as er: + print("task cancel") + if allow_cancel: + raise er + + +async def task2(allow_cancel): + print("task 2") + try: + await asyncio.create_task(task(0.05, allow_cancel)) + except asyncio.CancelledError as er: + print("task 2 cancel") + raise er + print("task 2 done") + + +async def main(): + # Cancel task immediately + t = asyncio.create_task(task(2, True)) + print(t.cancel()) + + # Cancel task after it has started + t = asyncio.create_task(task(2, True)) + await asyncio.sleep(0.01) + print(t.cancel()) + print("main sleep") + await asyncio.sleep(0.01) + + # Cancel task multiple times after it has started + t = asyncio.create_task(task(2, True)) + await asyncio.sleep(0.01) + for _ in range(4): + print(t.cancel()) + print("main sleep") + await asyncio.sleep(0.01) + + # Await on a cancelled task + print("main wait") + try: + await t + except asyncio.CancelledError: + print("main got CancelledError") + + # Cancel task after it has finished + t = asyncio.create_task(task(0.01, False)) + await asyncio.sleep(0.05) + print(t.cancel()) + + # Nested: task2 waits on task, task2 is cancelled (should cancel task then task2) + print("----") + t = asyncio.create_task(task2(True)) + await asyncio.sleep(0.01) + print("main cancel") + t.cancel() + print("main sleep") + await asyncio.sleep(0.1) + + # Nested: task2 waits on task, task2 is cancelled but task doesn't allow it (task2 should continue) + print("----") + t = asyncio.create_task(task2(False)) + await asyncio.sleep(0.01) + print("main cancel") + t.cancel() + print("main sleep") + await asyncio.sleep(0.1) + + +asyncio.run(main()) diff --git a/tests/extmod/uasyncio_cancel_task.py.exp b/tests/extmod/uasyncio_cancel_task.py.exp new file mode 100644 index 0000000000000..031b94023fd03 --- /dev/null +++ b/tests/extmod/uasyncio_cancel_task.py.exp @@ -0,0 +1,31 @@ +True +task start +True +main sleep +task cancel +task start +True +True +True +True +main sleep +task cancel +main wait +main got CancelledError +task start +task done +False +---- +task 2 +task start +main cancel +main sleep +task cancel +task 2 cancel +---- +task 2 +task start +main cancel +main sleep +task cancel +task 2 done diff --git a/tests/extmod/uasyncio_event.py b/tests/extmod/uasyncio_event.py new file mode 100644 index 0000000000000..fb8eb9ffa40a7 --- /dev/null +++ b/tests/extmod/uasyncio_event.py @@ -0,0 +1,98 @@ +# Test Event class + +try: + import uasyncio as asyncio +except ImportError: + try: + import asyncio + except ImportError: + print("SKIP") + raise SystemExit + + +async def task(id, ev): + print("start", id) + print(await ev.wait()) + print("end", id) + + +async def task_delay_set(t, ev): + await asyncio.sleep(t) + print("set event") + ev.set() + + +async def main(): + ev = asyncio.Event() + + # Set and clear without anything waiting, and test is_set() + print(ev.is_set()) + ev.set() + print(ev.is_set()) + ev.clear() + print(ev.is_set()) + + # Create 2 tasks waiting on the event + print("----") + asyncio.create_task(task(1, ev)) + asyncio.create_task(task(2, ev)) + print("yield") + await asyncio.sleep(0) + print("set event") + ev.set() + print("yield") + await asyncio.sleep(0) + + # Create a task waiting on the already-set event + print("----") + asyncio.create_task(task(3, ev)) + print("yield") + await asyncio.sleep(0) + + # Clear event, start a task, then set event again + print("----") + print("clear event") + ev.clear() + asyncio.create_task(task(4, ev)) + await asyncio.sleep(0) + print("set event") + ev.set() + await asyncio.sleep(0) + + # Cancel a task waiting on an event (set event then cancel task) + print("----") + ev = asyncio.Event() + t = asyncio.create_task(task(5, ev)) + await asyncio.sleep(0) + ev.set() + t.cancel() + await asyncio.sleep(0.1) + + # Cancel a task waiting on an event (cancel task then set event) + print("----") + ev = asyncio.Event() + t = asyncio.create_task(task(6, ev)) + await asyncio.sleep(0) + t.cancel() + ev.set() + await asyncio.sleep(0.1) + + # Wait for an event that does get set in time + print("----") + ev.clear() + asyncio.create_task(task_delay_set(0.01, ev)) + await asyncio.wait_for(ev.wait(), 0.1) + await asyncio.sleep(0) + + # Wait for an event that doesn't get set in time + print("----") + ev.clear() + asyncio.create_task(task_delay_set(0.1, ev)) + try: + await asyncio.wait_for(ev.wait(), 0.01) + except asyncio.TimeoutError: + print("TimeoutError") + await ev.wait() + + +asyncio.run(main()) diff --git a/tests/extmod/uasyncio_event.py.exp b/tests/extmod/uasyncio_event.py.exp new file mode 100644 index 0000000000000..3188f291e584d --- /dev/null +++ b/tests/extmod/uasyncio_event.py.exp @@ -0,0 +1,33 @@ +False +True +False +---- +yield +start 1 +start 2 +set event +yield +True +end 1 +True +end 2 +---- +yield +start 3 +True +end 3 +---- +clear event +start 4 +set event +True +end 4 +---- +start 5 +---- +start 6 +---- +set event +---- +TimeoutError +set event diff --git a/tests/extmod/uasyncio_event_fair.py b/tests/extmod/uasyncio_event_fair.py new file mode 100644 index 0000000000000..37eca5faef10b --- /dev/null +++ b/tests/extmod/uasyncio_event_fair.py @@ -0,0 +1,40 @@ +# Test fairness of Event.set() +# That tasks which continuously wait on events don't take over the scheduler + +try: + import uasyncio as asyncio +except ImportError: + try: + import asyncio + except ImportError: + print("SKIP") + raise SystemExit + + +async def task1(id): + for i in range(4): + print("sleep", id) + await asyncio.sleep(0) + + +async def task2(id, ev): + for i in range(4): + ev.set() + ev.clear() + print("wait", id) + await ev.wait() + + +async def main(): + ev = asyncio.Event() + tasks = [ + asyncio.create_task(task1(0)), + asyncio.create_task(task2(2, ev)), + asyncio.create_task(task1(1)), + asyncio.create_task(task2(3, ev)), + ] + await tasks[1] + ev.set() + + +asyncio.run(main()) diff --git a/tests/extmod/uasyncio_event_fair.py.exp b/tests/extmod/uasyncio_event_fair.py.exp new file mode 100644 index 0000000000000..fe2e3d6e47a63 --- /dev/null +++ b/tests/extmod/uasyncio_event_fair.py.exp @@ -0,0 +1,16 @@ +sleep 0 +wait 2 +sleep 1 +wait 3 +sleep 0 +sleep 1 +wait 2 +sleep 0 +sleep 1 +wait 3 +sleep 0 +sleep 1 +wait 2 +wait 3 +wait 2 +wait 3 diff --git a/tests/extmod/uasyncio_exception.py b/tests/extmod/uasyncio_exception.py new file mode 100644 index 0000000000000..aae55d63207a2 --- /dev/null +++ b/tests/extmod/uasyncio_exception.py @@ -0,0 +1,60 @@ +# Test general exception handling + +try: + import uasyncio as asyncio +except ImportError: + try: + import asyncio + except ImportError: + print("SKIP") + raise SystemExit + +# main task raising an exception +async def main(): + print("main start") + raise ValueError(1) + print("main done") + + +try: + asyncio.run(main()) +except ValueError as er: + print("ValueError", er.args[0]) + +# sub-task raising an exception +async def task(): + print("task start") + raise ValueError(2) + print("task done") + + +async def main(): + print("main start") + t = asyncio.create_task(task()) + await t + print("main done") + + +try: + asyncio.run(main()) +except ValueError as er: + print("ValueError", er.args[0]) + +# main task raising an exception with sub-task not yet scheduled +# TODO not currently working, task is never scheduled +async def task(): + # print('task run') uncomment this line when it works + pass + + +async def main(): + print("main start") + asyncio.create_task(task()) + raise ValueError(3) + print("main done") + + +try: + asyncio.run(main()) +except ValueError as er: + print("ValueError", er.args[0]) diff --git a/tests/extmod/uasyncio_exception.py.exp b/tests/extmod/uasyncio_exception.py.exp new file mode 100644 index 0000000000000..b2ee860170ece --- /dev/null +++ b/tests/extmod/uasyncio_exception.py.exp @@ -0,0 +1,7 @@ +main start +ValueError 1 +main start +task start +ValueError 2 +main start +ValueError 3 diff --git a/tests/extmod/uasyncio_fair.py b/tests/extmod/uasyncio_fair.py new file mode 100644 index 0000000000000..9b04454bc1f50 --- /dev/null +++ b/tests/extmod/uasyncio_fair.py @@ -0,0 +1,32 @@ +# Test fairness of scheduler + +try: + import uasyncio as asyncio +except ImportError: + try: + import asyncio + except ImportError: + print("SKIP") + raise SystemExit + + +async def task(id, t): + print("task start", id) + while True: + if t > 0: + print("task work", id) + await asyncio.sleep(t) + + +async def main(): + t1 = asyncio.create_task(task(1, -0.01)) + t2 = asyncio.create_task(task(2, 0.1)) + t3 = asyncio.create_task(task(3, 0.2)) + await asyncio.sleep(0.5) + t1.cancel() + t2.cancel() + t3.cancel() + print("finish") + + +asyncio.run(main()) diff --git a/tests/extmod/uasyncio_fair.py.exp b/tests/extmod/uasyncio_fair.py.exp new file mode 100644 index 0000000000000..4428943f46287 --- /dev/null +++ b/tests/extmod/uasyncio_fair.py.exp @@ -0,0 +1,12 @@ +task start 1 +task start 2 +task work 2 +task start 3 +task work 3 +task work 2 +task work 3 +task work 2 +task work 2 +task work 3 +task work 2 +finish diff --git a/tests/extmod/uasyncio_gather.py b/tests/extmod/uasyncio_gather.py new file mode 100644 index 0000000000000..2697a6278b08c --- /dev/null +++ b/tests/extmod/uasyncio_gather.py @@ -0,0 +1,49 @@ +# test uasyncio.gather() function + +try: + import uasyncio as asyncio +except ImportError: + try: + import asyncio + except ImportError: + print("SKIP") + raise SystemExit + + +async def factorial(name, number): + f = 1 + for i in range(2, number + 1): + print("Task {}: Compute factorial({})...".format(name, i)) + await asyncio.sleep(0.01) + f *= i + print("Task {}: factorial({}) = {}".format(name, number, f)) + return f + + +async def task(id): + print("start", id) + await asyncio.sleep(0.2) + print("end", id) + + +async def gather_task(): + print("gather_task") + await asyncio.gather(task(1), task(2)) + print("gather_task2") + + +async def main(): + # Simple gather with return values + print(await asyncio.gather(factorial("A", 2), factorial("B", 3), factorial("C", 4),)) + + # Cancel a multi gather + # TODO doesn't work, Task should not forward cancellation from gather to sub-task + # but rather CancelledError should cancel the gather directly, which will then cancel + # all sub-tasks explicitly + # t = asyncio.create_task(gather_task()) + # await asyncio.sleep(0.1) + # t.cancel() + # await asyncio.sleep(0.01) + + +asyncio.run(main()) diff --git a/tests/extmod/uasyncio_gather.py.exp b/tests/extmod/uasyncio_gather.py.exp new file mode 100644 index 0000000000000..a37578d7eb3c8 --- /dev/null +++ b/tests/extmod/uasyncio_gather.py.exp @@ -0,0 +1,10 @@ +Task A: Compute factorial(2)... +Task B: Compute factorial(2)... +Task C: Compute factorial(2)... +Task A: factorial(2) = 2 +Task B: Compute factorial(3)... +Task C: Compute factorial(3)... +Task B: factorial(3) = 6 +Task C: Compute factorial(4)... +Task C: factorial(4) = 24 +[2, 6, 24] diff --git a/tests/extmod/uasyncio_get_event_loop.py b/tests/extmod/uasyncio_get_event_loop.py new file mode 100644 index 0000000000000..8ccbd6814e96c --- /dev/null +++ b/tests/extmod/uasyncio_get_event_loop.py @@ -0,0 +1,20 @@ +# Test get_event_loop() + +try: + import uasyncio as asyncio +except ImportError: + try: + import asyncio + except ImportError: + print("SKIP") + raise SystemExit + + +async def main(): + print("start") + await asyncio.sleep(0.01) + print("end") + + +loop = asyncio.get_event_loop() +loop.run_until_complete(main()) diff --git a/tests/extmod/uasyncio_heaplock.py b/tests/extmod/uasyncio_heaplock.py new file mode 100644 index 0000000000000..771d3f0d97ab7 --- /dev/null +++ b/tests/extmod/uasyncio_heaplock.py @@ -0,0 +1,46 @@ +# test that basic scheduling of tasks, and uasyncio.sleep_ms, does not use the heap + +import micropython + +# strict stackless builds can't call functions without allocating a frame on the heap +try: + f = lambda: 0 + micropython.heap_lock() + f() + micropython.heap_unlock() +except RuntimeError: + print("SKIP") + raise SystemExit + +try: + import uasyncio as asyncio +except ImportError: + try: + import asyncio + except ImportError: + print("SKIP") + raise SystemExit + + +async def task(id, n, t): + for i in range(n): + print(id, i) + await asyncio.sleep_ms(t) + + +async def main(): + t1 = asyncio.create_task(task(1, 4, 10)) + t2 = asyncio.create_task(task(2, 4, 25)) + + micropython.heap_lock() + + print("start") + await asyncio.sleep_ms(1) + print("sleep") + await asyncio.sleep_ms(100) + print("finish") + + micropython.heap_unlock() + + +asyncio.run(main()) diff --git a/tests/extmod/uasyncio_heaplock.py.exp b/tests/extmod/uasyncio_heaplock.py.exp new file mode 100644 index 0000000000000..a967cc319712f --- /dev/null +++ b/tests/extmod/uasyncio_heaplock.py.exp @@ -0,0 +1,11 @@ +start +1 0 +2 0 +sleep +1 1 +1 2 +2 1 +1 3 +2 2 +2 3 +finish diff --git a/tests/extmod/uasyncio_lock.py b/tests/extmod/uasyncio_lock.py new file mode 100644 index 0000000000000..096a8c82be148 --- /dev/null +++ b/tests/extmod/uasyncio_lock.py @@ -0,0 +1,97 @@ +# Test Lock class + +try: + import uasyncio as asyncio +except ImportError: + try: + import asyncio + except ImportError: + print("SKIP") + raise SystemExit + + +async def task_loop(id, lock): + print("task start", id) + for i in range(3): + async with lock: + print("task have", id, i) + print("task end", id) + + +async def task_sleep(lock): + async with lock: + print("task have", lock.locked()) + await asyncio.sleep(0.2) + print("task release", lock.locked()) + await lock.acquire() + print("task have again") + lock.release() + + +async def task_cancel(id, lock, to_cancel=None): + try: + async with lock: + print("task got", id) + await asyncio.sleep(0.1) + print("task release", id) + if to_cancel: + to_cancel[0].cancel() + except asyncio.CancelledError: + print("task cancel", id) + + +async def main(): + lock = asyncio.Lock() + + # Basic acquire/release + print(lock.locked()) + await lock.acquire() + print(lock.locked()) + await asyncio.sleep(0) + lock.release() + print(lock.locked()) + await asyncio.sleep(0) + + # Use with "async with" + async with lock: + print("have lock") + + # 3 tasks wanting the lock + print("----") + asyncio.create_task(task_loop(1, lock)) + asyncio.create_task(task_loop(2, lock)) + t3 = asyncio.create_task(task_loop(3, lock)) + await lock.acquire() + await asyncio.sleep(0) + lock.release() + await t3 + + # 2 sleeping tasks both wanting the lock + print("----") + asyncio.create_task(task_sleep(lock)) + await asyncio.sleep(0.1) + await task_sleep(lock) + + # 3 tasks, the first cancelling the second, the third should still run + print("----") + ts = [None] + asyncio.create_task(task_cancel(0, lock, ts)) + ts[0] = asyncio.create_task(task_cancel(1, lock)) + asyncio.create_task(task_cancel(2, lock)) + await asyncio.sleep(0.3) + print(lock.locked()) + + # 3 tasks, the second and third being cancelled while waiting on the lock + print("----") + t0 = asyncio.create_task(task_cancel(0, lock)) + t1 = asyncio.create_task(task_cancel(1, lock)) + t2 = asyncio.create_task(task_cancel(2, lock)) + await asyncio.sleep(0.05) + t1.cancel() + await asyncio.sleep(0.1) + t2.cancel() + await asyncio.sleep(0.1) + print(lock.locked()) + + +asyncio.run(main()) diff --git a/tests/extmod/uasyncio_lock.py.exp b/tests/extmod/uasyncio_lock.py.exp new file mode 100644 index 0000000000000..a37dfcbd2e519 --- /dev/null +++ b/tests/extmod/uasyncio_lock.py.exp @@ -0,0 +1,41 @@ +False +True +False +have lock +---- +task start 1 +task start 2 +task start 3 +task have 1 0 +task have 2 0 +task have 3 0 +task have 1 1 +task have 2 1 +task have 3 1 +task have 1 2 +task end 1 +task have 2 2 +task end 2 +task have 3 2 +task end 3 +---- +task have True +task release False +task have True +task release False +task have again +task have again +---- +task got 0 +task release 0 +task cancel 1 +task got 2 +task release 2 +False +---- +task got 0 +task cancel 1 +task release 0 +task got 2 +task cancel 2 +False diff --git a/tests/extmod/uasyncio_lock_cancel.py b/tests/extmod/uasyncio_lock_cancel.py new file mode 100644 index 0000000000000..85b8df8483065 --- /dev/null +++ b/tests/extmod/uasyncio_lock_cancel.py @@ -0,0 +1,55 @@ +# Test that locks work when cancelling multiple waiters on the lock + +try: + import uasyncio as asyncio +except ImportError: + try: + import asyncio + except ImportError: + print("SKIP") + raise SystemExit + + +async def task(i, lock, lock_flag): + print("task", i, "start") + try: + await lock.acquire() + except asyncio.CancelledError: + print("task", i, "cancel") + return + print("task", i, "lock_flag", lock_flag[0]) + lock_flag[0] = True + await asyncio.sleep(0) + lock.release() + lock_flag[0] = False + print("task", i, "done") + + +async def main(): + # Create a lock and acquire it so the tasks below must wait + lock = asyncio.Lock() + await lock.acquire() + lock_flag = [True] + + # Create 4 tasks and let them all run + t0 = asyncio.create_task(task(0, lock, lock_flag)) + t1 = asyncio.create_task(task(1, lock, lock_flag)) + t2 = asyncio.create_task(task(2, lock, lock_flag)) + t3 = asyncio.create_task(task(3, lock, lock_flag)) + await asyncio.sleep(0) + + # Cancel 2 of the tasks (which are waiting on the lock) and release the lock + t1.cancel() + t2.cancel() + lock.release() + lock_flag[0] = False + + # Let the tasks run to completion + for _ in range(4): + await asyncio.sleep(0) + + # The locke should be unlocked + print(lock.locked()) + + +asyncio.run(main()) diff --git a/tests/extmod/uasyncio_lock_cancel.py.exp b/tests/extmod/uasyncio_lock_cancel.py.exp new file mode 100644 index 0000000000000..49ae253887436 --- /dev/null +++ b/tests/extmod/uasyncio_lock_cancel.py.exp @@ -0,0 +1,11 @@ +task 0 start +task 1 start +task 2 start +task 3 start +task 1 cancel +task 2 cancel +task 0 lock_flag False +task 0 done +task 3 lock_flag False +task 3 done +False diff --git a/tests/extmod/uasyncio_wait_for.py b/tests/extmod/uasyncio_wait_for.py new file mode 100644 index 0000000000000..92fd174b846db --- /dev/null +++ b/tests/extmod/uasyncio_wait_for.py @@ -0,0 +1,62 @@ +# Test asyncio.wait_for + +try: + import uasyncio as asyncio +except ImportError: + try: + import asyncio + except ImportError: + print("SKIP") + raise SystemExit + + +async def task(id, t): + print("task start", id) + await asyncio.sleep(t) + print("task end", id) + return id * 2 + + +async def task_catch(): + print("task_catch start") + try: + await asyncio.sleep(0.2) + except asyncio.CancelledError: + print("ignore cancel") + print("task_catch done") + + +async def task_raise(): + print("task start") + raise ValueError + + +async def main(): + # When task finished before the timeout + print(await asyncio.wait_for(task(1, 0.01), 10)) + + # When timeout passes and task is cancelled + try: + print(await asyncio.wait_for(task(2, 10), 0.01)) + except asyncio.TimeoutError: + print("timeout") + + # When timeout passes and task is cancelled, but task ignores the cancellation request + try: + print(await asyncio.wait_for(task_catch(), 0.1)) + except asyncio.TimeoutError: + print("TimeoutError") + + # When task raises an exception + try: + print(await asyncio.wait_for(task_raise(), 1)) + except ValueError: + print("ValueError") + + # Timeout of None means wait forever + print(await asyncio.wait_for(task(3, 0.1), None)) + + print("finish") + + +asyncio.run(main()) diff --git a/tests/extmod/uasyncio_wait_for.py.exp b/tests/extmod/uasyncio_wait_for.py.exp new file mode 100644 index 0000000000000..41a5f2481e908 --- /dev/null +++ b/tests/extmod/uasyncio_wait_for.py.exp @@ -0,0 +1,15 @@ +task start 1 +task end 1 +2 +task start 2 +timeout +task_catch start +ignore cancel +task_catch done +TimeoutError +task start +ValueError +task start 3 +task end 3 +6 +finish diff --git a/tests/extmod/uasyncio_wait_task.py b/tests/extmod/uasyncio_wait_task.py new file mode 100644 index 0000000000000..3c79320c9f5c4 --- /dev/null +++ b/tests/extmod/uasyncio_wait_task.py @@ -0,0 +1,77 @@ +# Test waiting on a task + +try: + import uasyncio as asyncio +except ImportError: + try: + import asyncio + except ImportError: + print("SKIP") + raise SystemExit + + +try: + import utime + + ticks = utime.ticks_ms + ticks_diff = utime.ticks_diff +except: + import time + + ticks = lambda: int(time.time() * 1000) + ticks_diff = lambda t1, t0: t1 - t0 + + +async def task(t): + print("task", t) + + +async def delay_print(t, s): + await asyncio.sleep(t) + print(s) + + +async def task_raise(): + print("task_raise") + raise ValueError + + +async def main(): + print("start") + + # Wait on a task + t = asyncio.create_task(task(1)) + await t + + # Wait on a task that's already done + t = asyncio.create_task(task(2)) + await asyncio.sleep(0.001) + await t + + # Wait again on same task + await t + + print("----") + + # Create 2 tasks + ts1 = asyncio.create_task(delay_print(0.04, "hello")) + ts2 = asyncio.create_task(delay_print(0.08, "world")) + + # Time how long the tasks take to finish, they should execute in parallel + print("start") + t0 = ticks() + await ts1 + t1 = ticks() + await ts2 + t2 = ticks() + print("took {} {}".format(round(ticks_diff(t1, t0), -1), round(ticks_diff(t2, t1), -1))) + + # Wait on a task that raises an exception + t = asyncio.create_task(task_raise()) + try: + await t + except ValueError: + print("ValueError") + + +asyncio.run(main()) diff --git a/tests/extmod/uasyncio_wait_task.py.exp b/tests/extmod/uasyncio_wait_task.py.exp new file mode 100644 index 0000000000000..ee4e70fb4eb1e --- /dev/null +++ b/tests/extmod/uasyncio_wait_task.py.exp @@ -0,0 +1,10 @@ +start +task 1 +task 2 +---- +start +hello +world +took 40 40 +task_raise +ValueError From 5d09a40df91ec8872651f3b1f7118800a86592e1 Mon Sep 17 00:00:00 2001 From: Damien George Date: Thu, 23 Jan 2020 17:57:46 +1100 Subject: [PATCH 10/21] tests/run-tests: Skip uasyncio if no async, and skip one test on native. --- tests/run-tests | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/run-tests b/tests/run-tests index 28f0d76f60d12..df55a4e9ec065 100755 --- a/tests/run-tests +++ b/tests/run-tests @@ -413,6 +413,7 @@ def run_tests(pyb, tests, args, base_path="."): skip_tests.add('basics/scope_implicit.py') # requires checking for unbound local skip_tests.add('basics/try_finally_return2.py') # requires raise_varargs skip_tests.add('basics/unboundlocal.py') # requires checking for unbound local + skip_tests.add('extmod/uasyncio_lock.py') # requires async with skip_tests.add('misc/features.py') # requires raise_varargs skip_tests.add('misc/print_exception.py') # because native doesn't have proper traceback info skip_tests.add('misc/sys_exc_info.py') # sys.exc_info() is not supported for native @@ -441,7 +442,7 @@ def run_tests(pyb, tests, args, base_path="."): is_bytearray = test_name.startswith("bytearray") or test_name.endswith("_bytearray") is_set_type = test_name.startswith("set_") or test_name.startswith("frozenset") is_slice = test_name.find("slice") != -1 or test_name in misc_slice_tests - is_async = test_name.startswith("async_") + is_async = test_name.startswith(("async_", "uasyncio_")) is_const = test_name.startswith("const") is_io_module = test_name.startswith("io_") From 3667effff1d82f4e16b2843ae292ce7b86926b6b Mon Sep 17 00:00:00 2001 From: Damien George Date: Tue, 3 Mar 2020 11:44:42 +1100 Subject: [PATCH 11/21] travis: Exclude some uasyncio tests on OSX. --- .travis.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index acebdd7b26971..fe432b0aa331c 100644 --- a/.travis.yml +++ b/.travis.yml @@ -213,7 +213,8 @@ jobs: - make ${MAKEOPTS} -C ports/unix submodules - make ${MAKEOPTS} -C ports/unix deplibs - make ${MAKEOPTS} -C ports/unix - - make ${MAKEOPTS} -C ports/unix test + # OSX has poor time resolution and the following tests do not have the correct output + - (cd tests && ./run-tests --exclude 'uasyncio_(basic|heaplock|lock|wait_task)') after_failure: - (cd tests && for exp in *.exp; do testbase=$(basename $exp .exp); echo -e "\nFAILURE $testbase"; diff -u $testbase.exp $testbase.out; done) From 18fa65e47402d85c53361798046e306d2cf4bac1 Mon Sep 17 00:00:00 2001 From: Damien George Date: Tue, 10 Mar 2020 02:58:47 +1100 Subject: [PATCH 12/21] tests: Make default MICROPYPATH include extmod to find uasyncio. --- tests/run-multitests.py | 3 +++ tests/run-tests | 4 ++-- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/tests/run-multitests.py b/tests/run-multitests.py index edf2238a5b0e7..4e5bfc3b056c8 100755 --- a/tests/run-multitests.py +++ b/tests/run-multitests.py @@ -382,6 +382,9 @@ def main(): cmd_parser.add_argument("files", nargs="+", help="input test files") cmd_args = cmd_parser.parse_args() + # clear search path to make sure tests use only builtin modules and those in extmod + os.environ['MICROPYPATH'] = os.pathsep + '../extmod' + test_files = prepare_test_file_list(cmd_args.files) max_instances = max(t[1] for t in test_files) diff --git a/tests/run-tests b/tests/run-tests index df55a4e9ec065..e34c2531f067f 100755 --- a/tests/run-tests +++ b/tests/run-tests @@ -628,8 +628,8 @@ the last matching regex is used: tests = args.files if not args.keep_path: - # clear search path to make sure tests use only builtin modules - os.environ['MICROPYPATH'] = '' + # clear search path to make sure tests use only builtin modules and those in extmod + os.environ['MICROPYPATH'] = os.pathsep + '../extmod' # Even if we run completely different tests in a different directory, # we need to access feature_check's from the same directory as the From 38904b89376bf628f7d70174204b5330618d49c0 Mon Sep 17 00:00:00 2001 From: Damien George Date: Tue, 10 Mar 2020 02:59:14 +1100 Subject: [PATCH 13/21] tests/multi_net: Add uasyncio test for TCP server and client. Includes a test where the (non uasyncio) client does a RST on the connection, as a simple TCP server/client test where both sides are using uasyncio, and a test for TCP stream close then write. --- tests/multi_net/uasyncio_tcp_client_rst.py | 56 +++++++++++++++ .../multi_net/uasyncio_tcp_client_rst.py.exp | 5 ++ tests/multi_net/uasyncio_tcp_close_write.py | 69 +++++++++++++++++++ .../multi_net/uasyncio_tcp_close_write.py.exp | 10 +++ tests/multi_net/uasyncio_tcp_server_client.py | 58 ++++++++++++++++ .../uasyncio_tcp_server_client.py.exp | 8 +++ 6 files changed, 206 insertions(+) create mode 100644 tests/multi_net/uasyncio_tcp_client_rst.py create mode 100644 tests/multi_net/uasyncio_tcp_client_rst.py.exp create mode 100644 tests/multi_net/uasyncio_tcp_close_write.py create mode 100644 tests/multi_net/uasyncio_tcp_close_write.py.exp create mode 100644 tests/multi_net/uasyncio_tcp_server_client.py create mode 100644 tests/multi_net/uasyncio_tcp_server_client.py.exp diff --git a/tests/multi_net/uasyncio_tcp_client_rst.py b/tests/multi_net/uasyncio_tcp_client_rst.py new file mode 100644 index 0000000000000..a3a05490c776b --- /dev/null +++ b/tests/multi_net/uasyncio_tcp_client_rst.py @@ -0,0 +1,56 @@ +# Test TCP server with client issuing TCP RST part way through read + +try: + import uasyncio as asyncio +except ImportError: + try: + import asyncio + except ImportError: + print("SKIP") + raise SystemExit + +import struct, time, socket + +PORT = 8000 + + +async def handle_connection(reader, writer): + data = await reader.read(10) # should succeed + print(data) + await asyncio.sleep(0.2) # wait for client to drop connection + try: + data = await reader.read(100) + print(data) + writer.close() + await writer.wait_closed() + except OSError as er: + print("OSError", er.args[0]) + ev.set() + + +async def main(): + global ev + ev = asyncio.Event() + server = await asyncio.start_server(handle_connection, "0.0.0.0", PORT) + multitest.next() + async with server: + await asyncio.wait_for(ev.wait(), 10) + + +def instance0(): + multitest.globals(IP=multitest.get_network_ip()) + asyncio.run(main()) + + +def instance1(): + if not hasattr(socket, "SO_LINGER"): + multitest.skip() + multitest.next() + s = socket.socket() + s.connect(socket.getaddrinfo(IP, PORT)[0][-1]) + lgr_onoff = 1 + lgr_linger = 0 + s.setsockopt(socket.SOL_SOCKET, socket.SO_LINGER, struct.pack("ii", lgr_onoff, lgr_linger)) + s.send(b"GET / HTTP/1.0\r\n\r\n") + time.sleep(0.1) + s.close() # This issues a TCP RST since we've set the linger option diff --git a/tests/multi_net/uasyncio_tcp_client_rst.py.exp b/tests/multi_net/uasyncio_tcp_client_rst.py.exp new file mode 100644 index 0000000000000..920d1bb8d7296 --- /dev/null +++ b/tests/multi_net/uasyncio_tcp_client_rst.py.exp @@ -0,0 +1,5 @@ +--- instance0 --- +b'GET / HTTP' +OSError 104 +--- instance1 --- + diff --git a/tests/multi_net/uasyncio_tcp_close_write.py b/tests/multi_net/uasyncio_tcp_close_write.py new file mode 100644 index 0000000000000..5698ed8b1699d --- /dev/null +++ b/tests/multi_net/uasyncio_tcp_close_write.py @@ -0,0 +1,69 @@ +# Test uasyncio TCP stream closing then writing + +try: + import uasyncio as asyncio +except ImportError: + try: + import asyncio + except ImportError: + print("SKIP") + raise SystemExit + +PORT = 8000 + + +async def handle_connection(reader, writer): + # Write data to ensure connection + writer.write(b"x") + await writer.drain() + + # Read, should return nothing + print("read:", await reader.read(100)) + + # Close connection + print("close") + writer.close() + await writer.wait_closed() + + print("done") + ev.set() + + +async def tcp_server(): + global ev + ev = asyncio.Event() + server = await asyncio.start_server(handle_connection, "0.0.0.0", PORT) + print("server running") + multitest.next() + async with server: + await asyncio.wait_for(ev.wait(), 5) + + +async def tcp_client(): + reader, writer = await asyncio.open_connection(IP, PORT) + + # Read data to ensure connection + print("read:", await reader.read(1)) + + # Close connection + print("close") + writer.close() + await writer.wait_closed() + + # Try writing data to the closed connection + print("write") + try: + writer.write(b"x") + await writer.drain() + except OSError: + print("OSError") + + +def instance0(): + multitest.globals(IP=multitest.get_network_ip()) + asyncio.run(tcp_server()) + + +def instance1(): + multitest.next() + asyncio.run(tcp_client()) diff --git a/tests/multi_net/uasyncio_tcp_close_write.py.exp b/tests/multi_net/uasyncio_tcp_close_write.py.exp new file mode 100644 index 0000000000000..6c0f8d7eab24e --- /dev/null +++ b/tests/multi_net/uasyncio_tcp_close_write.py.exp @@ -0,0 +1,10 @@ +--- instance0 --- +server running +read: b'' +close +done +--- instance1 --- +read: b'x' +close +write +OSError diff --git a/tests/multi_net/uasyncio_tcp_server_client.py b/tests/multi_net/uasyncio_tcp_server_client.py new file mode 100644 index 0000000000000..6a8cb58de53da --- /dev/null +++ b/tests/multi_net/uasyncio_tcp_server_client.py @@ -0,0 +1,58 @@ +# Test uasyncio TCP server and client using start_server() and open_connection() + +try: + import uasyncio as asyncio +except ImportError: + try: + import asyncio + except ImportError: + print("SKIP") + raise SystemExit + +PORT = 8000 + + +async def handle_connection(reader, writer): + # Test that peername exists (but don't check its value, it changes) + writer.get_extra_info("peername") + + data = await reader.read(100) + print("echo:", data) + writer.write(data) + await writer.drain() + + print("close") + writer.close() + await writer.wait_closed() + + print("done") + ev.set() + + +async def tcp_server(): + global ev + ev = asyncio.Event() + server = await asyncio.start_server(handle_connection, "0.0.0.0", PORT) + print("server running") + multitest.next() + async with server: + await asyncio.wait_for(ev.wait(), 10) + + +async def tcp_client(message): + reader, writer = await asyncio.open_connection(IP, PORT) + print("write:", message) + writer.write(message) + await writer.drain() + data = await reader.read(100) + print("read:", data) + + +def instance0(): + multitest.globals(IP=multitest.get_network_ip()) + asyncio.run(tcp_server()) + + +def instance1(): + multitest.next() + asyncio.run(tcp_client(b"client data")) diff --git a/tests/multi_net/uasyncio_tcp_server_client.py.exp b/tests/multi_net/uasyncio_tcp_server_client.py.exp new file mode 100644 index 0000000000000..6dc6a9bbc7e52 --- /dev/null +++ b/tests/multi_net/uasyncio_tcp_server_client.py.exp @@ -0,0 +1,8 @@ +--- instance0 --- +server running +echo: b'client data' +close +done +--- instance1 --- +write: b'client data' +read: b'client data' From 081d06766223b326b6d7eeceae817b7a3a3f57b0 Mon Sep 17 00:00:00 2001 From: Damien George Date: Thu, 12 Mar 2020 11:57:15 +1100 Subject: [PATCH 14/21] tests/net_inet: Add uasyncio internet tests. --- tests/net_inet/uasyncio_cancel_stream.py | 35 +++++++++++++++++++ tests/net_inet/uasyncio_cancel_stream.py.exp | 5 +++ tests/net_inet/uasyncio_open_connection.py | 30 ++++++++++++++++ .../net_inet/uasyncio_open_connection.py.exp | 5 +++ tests/net_inet/uasyncio_tcp_read_headers.py | 34 ++++++++++++++++++ .../net_inet/uasyncio_tcp_read_headers.py.exp | 10 ++++++ 6 files changed, 119 insertions(+) create mode 100644 tests/net_inet/uasyncio_cancel_stream.py create mode 100644 tests/net_inet/uasyncio_cancel_stream.py.exp create mode 100644 tests/net_inet/uasyncio_open_connection.py create mode 100644 tests/net_inet/uasyncio_open_connection.py.exp create mode 100644 tests/net_inet/uasyncio_tcp_read_headers.py create mode 100644 tests/net_inet/uasyncio_tcp_read_headers.py.exp diff --git a/tests/net_inet/uasyncio_cancel_stream.py b/tests/net_inet/uasyncio_cancel_stream.py new file mode 100644 index 0000000000000..6b6b845b0f7a8 --- /dev/null +++ b/tests/net_inet/uasyncio_cancel_stream.py @@ -0,0 +1,35 @@ +# Test cancelling a task waiting on stream IO + +try: + import uasyncio as asyncio +except ImportError: + try: + import asyncio + except ImportError: + print("SKIP") + raise SystemExit + + +async def get(reader): + print("start") + try: + await reader.read(10) + print("fail") + except asyncio.CancelledError: + print("cancelled") + + +async def main(url): + reader, writer = await asyncio.open_connection(url, 80) + task = asyncio.create_task(get(reader)) + await asyncio.sleep(0) + print("cancelling") + task.cancel() + print("waiting") + await task + print("done") + writer.close() + await writer.wait_closed() + + +asyncio.run(main("micropython.org")) diff --git a/tests/net_inet/uasyncio_cancel_stream.py.exp b/tests/net_inet/uasyncio_cancel_stream.py.exp new file mode 100644 index 0000000000000..e3fcfa7b3f588 --- /dev/null +++ b/tests/net_inet/uasyncio_cancel_stream.py.exp @@ -0,0 +1,5 @@ +start +cancelling +waiting +cancelled +done diff --git a/tests/net_inet/uasyncio_open_connection.py b/tests/net_inet/uasyncio_open_connection.py new file mode 100644 index 0000000000000..68285a2437d64 --- /dev/null +++ b/tests/net_inet/uasyncio_open_connection.py @@ -0,0 +1,30 @@ +# Test simple HTTP request with uasyncio.open_connection() + +try: + import uasyncio as asyncio +except ImportError: + try: + import asyncio + except ImportError: + print("SKIP") + raise SystemExit + + +async def http_get(url): + reader, writer = await asyncio.open_connection(url, 80) + + print("write GET") + writer.write(b"GET / HTTP/1.0\r\n\r\n") + await writer.drain() + + print("read response") + data = await reader.read(100) + print("read:", data.split(b"\r\n")[0]) + + print("close") + writer.close() + await writer.wait_closed() + print("done") + + +asyncio.run(http_get("micropython.org")) diff --git a/tests/net_inet/uasyncio_open_connection.py.exp b/tests/net_inet/uasyncio_open_connection.py.exp new file mode 100644 index 0000000000000..c8dea365b5d53 --- /dev/null +++ b/tests/net_inet/uasyncio_open_connection.py.exp @@ -0,0 +1,5 @@ +write GET +read response +read: b'HTTP/1.1 200 OK' +close +done diff --git a/tests/net_inet/uasyncio_tcp_read_headers.py b/tests/net_inet/uasyncio_tcp_read_headers.py new file mode 100644 index 0000000000000..8e4375a4f379f --- /dev/null +++ b/tests/net_inet/uasyncio_tcp_read_headers.py @@ -0,0 +1,34 @@ +# Test uasyncio.open_connection() and stream readline() + +try: + import uasyncio as asyncio +except ImportError: + try: + import asyncio + except ImportError: + print("SKIP") + raise SystemExit + + +async def http_get_headers(url): + reader, writer = await asyncio.open_connection(url, 80) + + print("write GET") + writer.write(b"GET / HTTP/1.0\r\n\r\n") + await writer.drain() + + while True: + line = await reader.readline() + line = line.strip() + if not line: + break + if line.find(b"Date") == -1 and line.find(b"Modified") == -1 and line.find(b"Server") == -1: + print(line) + + print("close") + writer.close() + await writer.wait_closed() + print("done") + + +asyncio.run(http_get_headers("micropython.org")) diff --git a/tests/net_inet/uasyncio_tcp_read_headers.py.exp b/tests/net_inet/uasyncio_tcp_read_headers.py.exp new file mode 100644 index 0000000000000..c200238dc6b83 --- /dev/null +++ b/tests/net_inet/uasyncio_tcp_read_headers.py.exp @@ -0,0 +1,10 @@ +write GET +b'HTTP/1.1 200 OK' +b'Content-Type: text/html' +b'Content-Length: 54' +b'Connection: close' +b'Vary: Accept-Encoding' +b'ETag: "54306c85-36"' +b'Accept-Ranges: bytes' +close +done From bc009fdd62f913e36443f8267ffb6133f537fff3 Mon Sep 17 00:00:00 2001 From: Damien George Date: Thu, 12 Mar 2020 16:46:20 +1100 Subject: [PATCH 15/21] extmod/uasyncio: Add optional implementation of core uasyncio in C. Implements Task and TaskQueue classes in C, using a pairing-heap data structure. Using this reduces RAM use of each Task, and improves overall performance of the uasyncio scheduler. --- extmod/moduasyncio.c | 294 ++++++++++++++++++++++++++++++++++++++++ extmod/uasyncio/core.py | 7 +- py/builtin.h | 1 + py/mpconfig.h | 4 + py/objmodule.c | 3 + py/py.mk | 1 + 6 files changed, 308 insertions(+), 2 deletions(-) create mode 100644 extmod/moduasyncio.c diff --git a/extmod/moduasyncio.c b/extmod/moduasyncio.c new file mode 100644 index 0000000000000..a1aecc5d47a37 --- /dev/null +++ b/extmod/moduasyncio.c @@ -0,0 +1,294 @@ +/* + * This file is part of the MicroPython project, http://micropython.org/ + * + * The MIT License (MIT) + * + * Copyright (c) 2020 Damien P. George + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ + +#include "py/runtime.h" +#include "py/smallint.h" +#include "py/pairheap.h" +#include "py/mphal.h" + +#if MICROPY_PY_UASYNCIO + +typedef struct _mp_obj_task_t { + mp_pairheap_t pairheap; + mp_obj_t coro; + mp_obj_t data; + mp_obj_t waiting; + + mp_obj_t ph_key; +} mp_obj_task_t; + +typedef struct _mp_obj_task_queue_t { + mp_obj_base_t base; + mp_obj_task_t *heap; +} mp_obj_task_queue_t; + +STATIC const mp_obj_type_t task_queue_type; +STATIC const mp_obj_type_t task_type; + +STATIC mp_obj_t task_queue_make_new(const mp_obj_type_t *type, size_t n_args, size_t n_kw, const mp_obj_t *args); + +/******************************************************************************/ +// Ticks for task ordering in pairing heap + +STATIC mp_obj_t ticks(void) { + return MP_OBJ_NEW_SMALL_INT(mp_hal_ticks_ms() & (MICROPY_PY_UTIME_TICKS_PERIOD - 1)); +} + +STATIC mp_int_t ticks_diff(mp_obj_t t1_in, mp_obj_t t0_in) { + mp_uint_t t0 = MP_OBJ_SMALL_INT_VALUE(t0_in); + mp_uint_t t1 = MP_OBJ_SMALL_INT_VALUE(t1_in); + mp_int_t diff = ((t1 - t0 + MICROPY_PY_UTIME_TICKS_PERIOD / 2) & (MICROPY_PY_UTIME_TICKS_PERIOD - 1)) + - MICROPY_PY_UTIME_TICKS_PERIOD / 2; + return diff; +} + +STATIC int task_lt(mp_pairheap_t *n1, mp_pairheap_t *n2) { + mp_obj_task_t *t1 = (mp_obj_task_t *)n1; + mp_obj_task_t *t2 = (mp_obj_task_t *)n2; + return MP_OBJ_SMALL_INT_VALUE(ticks_diff(t1->ph_key, t2->ph_key)) < 0; +} + +/******************************************************************************/ +// TaskQueue class + +STATIC mp_obj_t task_queue_make_new(const mp_obj_type_t *type, size_t n_args, size_t n_kw, const mp_obj_t *args) { + (void)args; + mp_arg_check_num(n_args, n_kw, 0, 0, false); + mp_obj_task_queue_t *self = m_new_obj(mp_obj_task_queue_t); + self->base.type = type; + self->heap = (mp_obj_task_t *)mp_pairheap_new(task_lt); + return MP_OBJ_FROM_PTR(self); +} + +STATIC mp_obj_t task_queue_peek(mp_obj_t self_in) { + mp_obj_task_queue_t *self = MP_OBJ_TO_PTR(self_in); + if (self->heap == NULL) { + return mp_const_none; + } else { + return MP_OBJ_FROM_PTR(self->heap); + } +} +STATIC MP_DEFINE_CONST_FUN_OBJ_1(task_queue_peek_obj, task_queue_peek); + +STATIC mp_obj_t task_queue_push_sorted(size_t n_args, const mp_obj_t *args) { + mp_obj_task_queue_t *self = MP_OBJ_TO_PTR(args[0]); + mp_obj_task_t *task = MP_OBJ_TO_PTR(args[1]); + task->data = mp_const_none; + if (n_args == 2) { + task->ph_key = ticks(); + } else { + assert(mp_obj_is_small_int(args[2])); + task->ph_key = args[2]; + } + self->heap = (mp_obj_task_t *)mp_pairheap_push(task_lt, &self->heap->pairheap, &task->pairheap); + return mp_const_none; +} +STATIC MP_DEFINE_CONST_FUN_OBJ_VAR_BETWEEN(task_queue_push_sorted_obj, 2, 3, task_queue_push_sorted); + +STATIC mp_obj_t task_queue_pop_head(mp_obj_t self_in) { + mp_obj_task_queue_t *self = MP_OBJ_TO_PTR(self_in); + mp_obj_task_t *head = (mp_obj_task_t *)mp_pairheap_peek(task_lt, &self->heap->pairheap); + if (head == NULL) { + mp_raise_msg(&mp_type_IndexError, "empty heap"); + } + self->heap = (mp_obj_task_t *)mp_pairheap_pop(task_lt, &self->heap->pairheap); + return MP_OBJ_FROM_PTR(head); +} +STATIC MP_DEFINE_CONST_FUN_OBJ_1(task_queue_pop_head_obj, task_queue_pop_head); + +STATIC mp_obj_t task_queue_remove(mp_obj_t self_in, mp_obj_t task_in) { + mp_obj_task_queue_t *self = MP_OBJ_TO_PTR(self_in); + mp_obj_task_t *task = MP_OBJ_TO_PTR(task_in); + self->heap = (mp_obj_task_t *)mp_pairheap_delete(task_lt, &self->heap->pairheap, &task->pairheap); + return mp_const_none; +} +STATIC MP_DEFINE_CONST_FUN_OBJ_2(task_queue_remove_obj, task_queue_remove); + +STATIC const mp_rom_map_elem_t task_queue_locals_dict_table[] = { + { MP_ROM_QSTR(MP_QSTR_peek), MP_ROM_PTR(&task_queue_peek_obj) }, + { MP_ROM_QSTR(MP_QSTR_push_sorted), MP_ROM_PTR(&task_queue_push_sorted_obj) }, + { MP_ROM_QSTR(MP_QSTR_push_head), MP_ROM_PTR(&task_queue_push_sorted_obj) }, + { MP_ROM_QSTR(MP_QSTR_pop_head), MP_ROM_PTR(&task_queue_pop_head_obj) }, + { MP_ROM_QSTR(MP_QSTR_remove), MP_ROM_PTR(&task_queue_remove_obj) }, +}; +STATIC MP_DEFINE_CONST_DICT(task_queue_locals_dict, task_queue_locals_dict_table); + +STATIC const mp_obj_type_t task_queue_type = { + { &mp_type_type }, + .name = MP_QSTR_TaskQueue, + .make_new = task_queue_make_new, + .locals_dict = (mp_obj_dict_t *)&task_queue_locals_dict, +}; + +/******************************************************************************/ +// Task class + +// This is the core uasyncio context with cur_task, _task_queue and CancelledError. +STATIC mp_obj_t uasyncio_context = MP_OBJ_NULL; + +STATIC mp_obj_t task_make_new(const mp_obj_type_t *type, size_t n_args, size_t n_kw, const mp_obj_t *args) { + mp_arg_check_num(n_args, n_kw, 1, 2, false); + mp_obj_task_t *self = m_new_obj(mp_obj_task_t); + self->pairheap.base.type = type; + mp_pairheap_init_node(task_lt, &self->pairheap); + self->coro = args[0]; + self->data = mp_const_none; + self->waiting = mp_const_none; + self->ph_key = MP_OBJ_NEW_SMALL_INT(0); + if (n_args == 2) { + uasyncio_context = args[1]; + } + return MP_OBJ_FROM_PTR(self); +} + +STATIC mp_obj_t task_cancel(mp_obj_t self_in) { + mp_obj_task_t *self = MP_OBJ_TO_PTR(self_in); + // Check if task is already finished. + if (self->coro == mp_const_none) { + return mp_const_false; + } + // Can't cancel self (not supported yet). + mp_obj_t cur_task = mp_obj_dict_get(uasyncio_context, MP_OBJ_NEW_QSTR(MP_QSTR_cur_task)); + if (self_in == cur_task) { + mp_raise_msg(&mp_type_RuntimeError, "cannot cancel self"); + } + // If Task waits on another task then forward the cancel to the one it's waiting on. + while (mp_obj_is_subclass_fast(MP_OBJ_FROM_PTR(mp_obj_get_type(self->data)), MP_OBJ_FROM_PTR(&task_type))) { + self = MP_OBJ_TO_PTR(self->data); + } + + mp_obj_t _task_queue = mp_obj_dict_get(uasyncio_context, MP_OBJ_NEW_QSTR(MP_QSTR__task_queue)); + + // Reschedule Task as a cancelled task. + mp_obj_t dest[3]; + mp_load_method_maybe(self->data, MP_QSTR_remove, dest); + if (dest[0] != MP_OBJ_NULL) { + // Not on the main running queue, remove the task from the queue it's on. + dest[2] = MP_OBJ_FROM_PTR(self); + mp_call_method_n_kw(1, 0, dest); + // _task_queue.push_head(self) + dest[0] = _task_queue; + dest[1] = MP_OBJ_FROM_PTR(self); + task_queue_push_sorted(2, dest); + } else if (ticks_diff(self->ph_key, ticks()) > 0) { + // On the main running queue but scheduled in the future, so bring it forward to now. + // _task_queue.remove(self) + task_queue_remove(_task_queue, MP_OBJ_FROM_PTR(self)); + // _task_queue.push_head(self) + dest[0] = _task_queue; + dest[1] = MP_OBJ_FROM_PTR(self); + task_queue_push_sorted(2, dest); + } + + self->data = mp_obj_dict_get(uasyncio_context, MP_OBJ_NEW_QSTR(MP_QSTR_CancelledError)); + + return mp_const_true; +} +STATIC MP_DEFINE_CONST_FUN_OBJ_1(task_cancel_obj, task_cancel); + +STATIC void task_attr(mp_obj_t self_in, qstr attr, mp_obj_t *dest) { + mp_obj_task_t *self = MP_OBJ_TO_PTR(self_in); + if (dest[0] == MP_OBJ_NULL) { + // Load + if (attr == MP_QSTR_coro) { + dest[0] = self->coro; + } else if (attr == MP_QSTR_data) { + dest[0] = self->data; + } else if (attr == MP_QSTR_waiting) { + if (self->waiting != mp_const_none) { + dest[0] = self->waiting; + } + } else if (attr == MP_QSTR_cancel) { + dest[0] = MP_OBJ_FROM_PTR(&task_cancel_obj); + dest[1] = self_in; + } else if (attr == MP_QSTR_ph_key) { + dest[0] = self->ph_key; + } + } else if (dest[1] != MP_OBJ_NULL) { + // Store + if (attr == MP_QSTR_coro) { + self->coro = dest[1]; + dest[0] = MP_OBJ_NULL; + } else if (attr == MP_QSTR_data) { + self->data = dest[1]; + dest[0] = MP_OBJ_NULL; + } else if (attr == MP_QSTR_waiting) { + self->waiting = dest[1]; + dest[0] = MP_OBJ_NULL; + } + } +} + +STATIC mp_obj_t task_getiter(mp_obj_t self_in, mp_obj_iter_buf_t *iter_buf) { + (void)iter_buf; + mp_obj_task_t *self = MP_OBJ_TO_PTR(self_in); + if (self->waiting == mp_const_none) { + self->waiting = task_queue_make_new(&task_queue_type, 0, 0, NULL); + } + return self_in; +} + +STATIC mp_obj_t task_iternext(mp_obj_t self_in) { + mp_obj_task_t *self = MP_OBJ_TO_PTR(self_in); + if (self->coro == mp_const_none) { + // Task finished, raise return value to caller so it can continue. + nlr_raise(self->data); + } else { + // Put calling task on waiting queue. + mp_obj_t cur_task = mp_obj_dict_get(uasyncio_context, MP_OBJ_NEW_QSTR(MP_QSTR_cur_task)); + mp_obj_t args[2] = { self->waiting, cur_task }; + task_queue_push_sorted(2, args); + // Set calling task's data to this task that it waits on, to double-link it. + ((mp_obj_task_t *)MP_OBJ_TO_PTR(cur_task))->data = self_in; + } + return mp_const_none; +} + +STATIC const mp_obj_type_t task_type = { + { &mp_type_type }, + .name = MP_QSTR_Task, + .make_new = task_make_new, + .attr = task_attr, + .getiter = task_getiter, + .iternext = task_iternext, +}; + +/******************************************************************************/ +// C-level uasyncio module + +STATIC const mp_rom_map_elem_t mp_module_uasyncio_globals_table[] = { + { MP_ROM_QSTR(MP_QSTR___name__), MP_ROM_QSTR(MP_QSTR__uasyncio) }, + { MP_ROM_QSTR(MP_QSTR_TaskQueue), MP_ROM_PTR(&task_queue_type) }, + { MP_ROM_QSTR(MP_QSTR_Task), MP_ROM_PTR(&task_type) }, +}; +STATIC MP_DEFINE_CONST_DICT(mp_module_uasyncio_globals, mp_module_uasyncio_globals_table); + +const mp_obj_module_t mp_module_uasyncio = { + .base = { &mp_type_module }, + .globals = (mp_obj_dict_t *)&mp_module_uasyncio_globals, +}; + +#endif // MICROPY_PY_UASYNCIO diff --git a/extmod/uasyncio/core.py b/extmod/uasyncio/core.py index 049e2537f8df6..5d7191d7fda9f 100644 --- a/extmod/uasyncio/core.py +++ b/extmod/uasyncio/core.py @@ -4,8 +4,11 @@ from time import ticks_ms as ticks, ticks_diff, ticks_add import sys, select -# Import TaskQueue and Task -from .task import TaskQueue, Task +# Import TaskQueue and Task, preferring built-in C code over Python code +try: + from _uasyncio import TaskQueue, Task +except: + from .task import TaskQueue, Task ################################################################################ diff --git a/py/builtin.h b/py/builtin.h index 2dbe8a782092c..1e4769cd69a35 100644 --- a/py/builtin.h +++ b/py/builtin.h @@ -103,6 +103,7 @@ extern const mp_obj_module_t mp_module_thread; extern const mp_obj_dict_t mp_module_builtins_globals; // extmod modules +extern const mp_obj_module_t mp_module_uasyncio; extern const mp_obj_module_t mp_module_uerrno; extern const mp_obj_module_t mp_module_uctypes; extern const mp_obj_module_t mp_module_uzlib; diff --git a/py/mpconfig.h b/py/mpconfig.h index c5829a3e09568..d96687b8132cf 100644 --- a/py/mpconfig.h +++ b/py/mpconfig.h @@ -1291,6 +1291,10 @@ typedef double mp_float_t; // Extended modules +#ifndef MICROPY_PY_UASYNCIO +#define MICROPY_PY_UASYNCIO (0) +#endif + #ifndef MICROPY_PY_UCTYPES #define MICROPY_PY_UCTYPES (0) #endif diff --git a/py/objmodule.c b/py/objmodule.c index 79047009f6d2d..060e1bc1e6c2d 100644 --- a/py/objmodule.c +++ b/py/objmodule.c @@ -170,6 +170,9 @@ STATIC const mp_rom_map_elem_t mp_builtin_module_table[] = { // extmod modules + #if MICROPY_PY_UASYNCIO + { MP_ROM_QSTR(MP_QSTR__uasyncio), MP_ROM_PTR(&mp_module_uasyncio) }, + #endif #if MICROPY_PY_UERRNO { MP_ROM_QSTR(MP_QSTR_uerrno), MP_ROM_PTR(&mp_module_uerrno) }, #endif diff --git a/py/py.mk b/py/py.mk index 1a56dcce8f78a..8c90beff71b64 100644 --- a/py/py.mk +++ b/py/py.mk @@ -168,6 +168,7 @@ PY_CORE_O_BASENAME = $(addprefix py/,\ ) PY_EXTMOD_O_BASENAME = \ + extmod/moduasyncio.o \ extmod/moductypes.o \ extmod/modujson.o \ extmod/modure.o \ From 91dd3948e8143eee785e39a5c165a640ab8ffa2f Mon Sep 17 00:00:00 2001 From: Damien George Date: Thu, 12 Mar 2020 23:01:02 +1100 Subject: [PATCH 16/21] unix: Enable uasyncio C helper module on coverage build. --- ports/unix/variants/coverage/mpconfigvariant.h | 1 + 1 file changed, 1 insertion(+) diff --git a/ports/unix/variants/coverage/mpconfigvariant.h b/ports/unix/variants/coverage/mpconfigvariant.h index 12f37a6cbcf6a..802c2fe5f7203 100644 --- a/ports/unix/variants/coverage/mpconfigvariant.h +++ b/ports/unix/variants/coverage/mpconfigvariant.h @@ -50,6 +50,7 @@ #define MICROPY_PY_URANDOM_EXTRA_FUNCS (1) #define MICROPY_PY_IO_BUFFEREDWRITER (1) #define MICROPY_PY_IO_RESOURCE_STREAM (1) +#define MICROPY_PY_UASYNCIO (1) #define MICROPY_PY_URE_DEBUG (1) #define MICROPY_PY_URE_MATCH_GROUPS (1) #define MICROPY_PY_URE_MATCH_SPAN_START_END (1) From c99322f8d8ec6c288dd734e5f5fb344670215692 Mon Sep 17 00:00:00 2001 From: Damien George Date: Sat, 21 Mar 2020 00:59:25 +1100 Subject: [PATCH 17/21] docs/library: Add initial docs for uasyncio module. --- docs/library/index.rst | 1 + docs/library/uasyncio.rst | 263 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 264 insertions(+) create mode 100644 docs/library/uasyncio.rst diff --git a/docs/library/index.rst b/docs/library/index.rst index 34b937d412216..7b1636ce61b33 100644 --- a/docs/library/index.rst +++ b/docs/library/index.rst @@ -79,6 +79,7 @@ it will fallback to loading the built-in ``ujson`` module. math.rst sys.rst uarray.rst + uasyncio.rst ubinascii.rst ucollections.rst uerrno.rst diff --git a/docs/library/uasyncio.rst b/docs/library/uasyncio.rst new file mode 100644 index 0000000000000..ebe0c34ebc885 --- /dev/null +++ b/docs/library/uasyncio.rst @@ -0,0 +1,263 @@ +:mod:`uasyncio` --- asynchronous I/O scheduler +============================================== + +.. module:: uasyncio + :synopsis: asynchronous I/O scheduler for writing concurrent code + +|see_cpython_module| +`asyncio `_ + +Example:: + + import uasyncio + + async def blink(led, period_ms): + while True: + led.on() + await uasyncio.sleep_ms(5) + led.off() + await uasyncio.sleep_ms(period_ms) + + async def main(led1, led2): + uasyncio.create_task(blink(led1, 700)) + uasyncio.create_task(blink(led2, 400)) + await uasyncio.sleep_ms(10_000) + + # Running on a pyboard + from pyb import LED + uasyncio.run(main(LED(1), LED(2))) + + # Running on a generic board + from machine import Pin + uasyncio.run(main(Pin(1), Pin(2))) + +Core functions +-------------- + +.. function:: create_task(coro) + + Create a new task from the given coroutine and schedule it to run. + + Returns the corresponding `Task` object. + +.. function:: run(coro) + + Create a new task from the given coroutine and run it until it completes. + + Returns the value returned by *coro*. + +.. function:: sleep(t) + + Sleep for *t* seconds (can be a float). + + This is a coroutine. + +.. function:: sleep_ms(t) + + Sleep for *t* milliseconds. + + This is a coroutine, and a MicroPython extension. + +Additional functions +-------------------- + +.. function:: wait_for(awaitable, timeout) + + Wait for the *awaitable* to complete, but cancel it if it takes longer + that *timeout* seconds. If *awaitable* is not a task then a task will be + created from it. + + Returns the return value of *awaitable*. + + This is a coroutine. + +.. function:: gather(\*awaitables, return_exceptions=False) + + Run all *awaitables* concurrently. Any *awaitables* that are not tasks are + promoted to tasks. + + Returns a list of return values of all *awaitables*. + + This is a coroutine. + +class Task +---------- + +.. class:: Task() + + This object wraps a coroutine into a running task. Tasks can be waited on + using ``await task``, which will wait for the task to complete and return + the return value of the task. + + Tasks should not be created directly, rather use `create_task` to create them. + +.. method:: Task.cancel() + + Cancel the task by injecting a ``CancelledError`` into it. The task may + or may not ignore this exception. + +class Event +----------- + +.. class:: Event() + + Create a new event which can be used to synchronise tasks. Events start + in the cleared state. + +.. method:: Event.is_set() + + Returns ``True`` if the event is set, ``False`` otherwise. + +.. method:: Event.set() + + Set the event. Any tasks waiting on the event will be scheduled to run. + +.. method:: Event.clear() + + Clear the event. + +.. method:: Event.wait() + + Wait for the event to be set. If the event is already set then it returns + immediately. + + This is a coroutine. + +class Lock +---------- + +.. class:: Lock() + + Create a new lock which can be used to coordinate tasks. Locks start in + the unlocked state. + + In addition to the methods below, locks can be used in an ``async with`` statement. + +.. method:: Lock.locked() + + Returns ``True`` if the lock is locked, otherwise ``False``. + +.. method:: Lock.acquire() + + Wait for the lock to be in the unlocked state and then lock it in an atomic + way. Only one task can acquire the lock at any one time. + + This is a coroutine. + +.. method:: Lock.release() + + Release the lock. If any tasks are waiting on the lock then the next one in the + queue is scheduled to run and the lock remains locked. Otherwise, no tasks are + waiting an the lock becomes unlocked. + +TCP stream connections +---------------------- + +.. function:: open_connection(host, port) + + Open a TCP connection to the given *host* and *port*. The *host* address will be + resolved using `socket.getaddrinfo`, which is currently a blocking call. + + Returns a pair of streams: a reader and a writer stream. + Will raise a socket-specific ``OSError`` if the host could not be resolved or if + the connection could not be made. + + This is a coroutine. + +.. function:: start_server(callback, host, port, backlog=5) + + Start a TCP server on the given *host* and *port*. The *callback* will be + called with incoming, accepted connections, and be passed 2 arguments: reader + and writer streams for the connection. + + Returns a `Server` object. + + This is a coroutine. + +.. class:: Stream() + + This represents a TCP stream connection. To minimise code this class implements + both a reader and a writer. + +.. method:: Stream.get_extra_info(v) + + Get extra information about the stream, given by *v*. The valid values for *v* are: + ``peername``. + +.. method:: Stream.close() + + Close the stream. + +.. method:: Stream.wait_closed() + + Wait for the stream to close. + + This is a coroutine. + +.. method:: Stream.read(n) + + Read up to *n* bytes and return them. + + This is a coroutine. + +.. method:: Stream.readline() + + Read a line and return it. + + This is a coroutine. + +.. method:: Stream.write(buf) + + Accumulated *buf* to the output buffer. The data is only flushed when + `Stream.drain` is called. It is recommended to call `Stream.drain` immediately + after calling this function. + +.. method:: Stream.drain() + + Drain (write) all buffered output data out to the stream. + + This is a coroutine. + +.. class:: Server() + + This represents the server class returned from `start_server`. It can be used + in an ``async with`` statement to close the server upon exit. + +.. method:: Server.close() + + Close the server. + +.. method:: Server.wait_closed() + + Wait for the server to close. + + This is a coroutine. + +Event Loop +---------- + +.. function:: get_event_loop() + + Return the event loop used to schedule and run tasks. See `Loop`. + +.. class:: Loop() + + This represents the object which schedules and runs tasks. It cannot be + created, use `get_event_loop` instead. + +.. method:: Loop.create_task(coro) + + Create a task from the given *coro* and return the new `Task` object. + +.. method:: Loop.run_forever() + + Run the event loop forever. + +.. method:: Loop.run_until_complete(awaitable) + + Run the given *awaitable* until it completes. If *awaitable* is not a task + then it will be promoted to one. + +.. method:: Loop.close() + + Close the event loop. From 3b68f3617565f8f4d05ac23d7be7f094e628bcbf Mon Sep 17 00:00:00 2001 From: Damien George Date: Sat, 21 Mar 2020 22:25:45 +1100 Subject: [PATCH 18/21] extmod/uasyncio: Add manifest.py for freezing uasyncio Py files. --- extmod/uasyncio/manifest.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100644 extmod/uasyncio/manifest.py diff --git a/extmod/uasyncio/manifest.py b/extmod/uasyncio/manifest.py new file mode 100644 index 0000000000000..f5fa27bfcaa35 --- /dev/null +++ b/extmod/uasyncio/manifest.py @@ -0,0 +1,13 @@ +# This list of frozen files doesn't include task.py because that's provided by the C module. +freeze( + "..", + ( + "uasyncio/__init__.py", + "uasyncio/core.py", + "uasyncio/event.py", + "uasyncio/funcs.py", + "uasyncio/lock.py", + "uasyncio/stream.py", + ), + opt=3, +) From 35e2dd0979f80df8cb5fbb22a9766ee13d4ef437 Mon Sep 17 00:00:00 2001 From: Damien George Date: Sat, 21 Mar 2020 22:26:02 +1100 Subject: [PATCH 19/21] stm32: Enable and freeze uasyncio. --- ports/stm32/boards/manifest.py | 1 + ports/stm32/mpconfigport.h | 1 + 2 files changed, 2 insertions(+) diff --git a/ports/stm32/boards/manifest.py b/ports/stm32/boards/manifest.py index 3390773236c70..81b85834101de 100644 --- a/ports/stm32/boards/manifest.py +++ b/ports/stm32/boards/manifest.py @@ -1,3 +1,4 @@ +include("$(MPY_DIR)/extmod/uasyncio/manifest.py") freeze("$(MPY_DIR)/drivers/dht", "dht.py") freeze("$(MPY_DIR)/drivers/display", ("lcd160cr.py", "lcd160cr_test.py")) freeze("$(MPY_DIR)/drivers/onewire", "onewire.py") diff --git a/ports/stm32/mpconfigport.h b/ports/stm32/mpconfigport.h index 95442b6476b5b..a7baa34956052 100644 --- a/ports/stm32/mpconfigport.h +++ b/ports/stm32/mpconfigport.h @@ -134,6 +134,7 @@ #endif // extended modules +#define MICROPY_PY_UASYNCIO (1) #define MICROPY_PY_UCTYPES (1) #define MICROPY_PY_UZLIB (1) #define MICROPY_PY_UJSON (1) From 1d4d688b3b251120f5827a3605ec232d977eaa0f Mon Sep 17 00:00:00 2001 From: Damien George Date: Sun, 22 Mar 2020 23:16:37 +1100 Subject: [PATCH 20/21] esp8266: Enable and freeze uasyncio. Only included in GENERIC build. --- ports/esp8266/boards/GENERIC/manifest.py | 2 ++ ports/esp8266/boards/GENERIC/mpconfigboard.h | 1 + ports/esp8266/boards/GENERIC/mpconfigboard.mk | 2 ++ ports/esp8266/boards/manifest_release.py | 4 ---- 4 files changed, 5 insertions(+), 4 deletions(-) create mode 100644 ports/esp8266/boards/GENERIC/manifest.py diff --git a/ports/esp8266/boards/GENERIC/manifest.py b/ports/esp8266/boards/GENERIC/manifest.py new file mode 100644 index 0000000000000..4e65b256f92c8 --- /dev/null +++ b/ports/esp8266/boards/GENERIC/manifest.py @@ -0,0 +1,2 @@ +include("$(PORT_DIR)/boards/manifest.py") +include("$(MPY_DIR)/extmod/uasyncio/manifest.py") diff --git a/ports/esp8266/boards/GENERIC/mpconfigboard.h b/ports/esp8266/boards/GENERIC/mpconfigboard.h index 8f0505d074710..d33943df80019 100644 --- a/ports/esp8266/boards/GENERIC/mpconfigboard.h +++ b/ports/esp8266/boards/GENERIC/mpconfigboard.h @@ -15,6 +15,7 @@ #define MICROPY_PY_ALL_SPECIAL_METHODS (1) #define MICROPY_PY_IO_FILEIO (1) #define MICROPY_PY_SYS_STDIO_BUFFER (1) +#define MICROPY_PY_UASYNCIO (1) #define MICROPY_PY_URE_SUB (1) #define MICROPY_PY_UCRYPTOLIB (1) #define MICROPY_PY_FRAMEBUF (1) diff --git a/ports/esp8266/boards/GENERIC/mpconfigboard.mk b/ports/esp8266/boards/GENERIC/mpconfigboard.mk index 86593ff60e9e2..820f073d98605 100644 --- a/ports/esp8266/boards/GENERIC/mpconfigboard.mk +++ b/ports/esp8266/boards/GENERIC/mpconfigboard.mk @@ -1 +1,3 @@ MICROPY_VFS_FAT = 1 + +FROZEN_MANIFEST ?= $(BOARD_DIR)/manifest.py diff --git a/ports/esp8266/boards/manifest_release.py b/ports/esp8266/boards/manifest_release.py index ab86fa3ef9b80..5a3194ae9bd45 100644 --- a/ports/esp8266/boards/manifest_release.py +++ b/ports/esp8266/boards/manifest_release.py @@ -6,10 +6,6 @@ # file utilities freeze("$(MPY_LIB_DIR)/upysh", "upysh.py") -# uasyncio -freeze("$(MPY_LIB_DIR)/uasyncio", "uasyncio/__init__.py") -freeze("$(MPY_LIB_DIR)/uasyncio.core", "uasyncio/core.py") - # requests freeze("$(MPY_LIB_DIR)/urequests", "urequests.py") freeze("$(MPY_LIB_DIR)/urllib.urequest", "urllib/urequest.py") From ad004db662b383a9bf310601e13e896c99e11e30 Mon Sep 17 00:00:00 2001 From: Damien George Date: Sun, 22 Mar 2020 23:17:17 +1100 Subject: [PATCH 21/21] esp32: Enable and freeze uasyncio. --- ports/esp32/boards/manifest.py | 1 + ports/esp32/mpconfigport.h | 1 + 2 files changed, 2 insertions(+) diff --git a/ports/esp32/boards/manifest.py b/ports/esp32/boards/manifest.py index bbd907a0e5409..b463c131faa0d 100644 --- a/ports/esp32/boards/manifest.py +++ b/ports/esp32/boards/manifest.py @@ -3,4 +3,5 @@ freeze("$(MPY_DIR)/ports/esp8266/modules", "ntptime.py") freeze("$(MPY_DIR)/drivers/dht", "dht.py") freeze("$(MPY_DIR)/drivers/onewire") +include("$(MPY_DIR)/extmod/uasyncio/manifest.py") include("$(MPY_DIR)/extmod/webrepl/manifest.py") diff --git a/ports/esp32/mpconfigport.h b/ports/esp32/mpconfigport.h index 3b6fb37b4e00c..32968eed2d1b2 100644 --- a/ports/esp32/mpconfigport.h +++ b/ports/esp32/mpconfigport.h @@ -125,6 +125,7 @@ #define MICROPY_PY_THREAD_GIL_VM_DIVISOR (32) // extended modules +#define MICROPY_PY_UASYNCIO (1) #define MICROPY_PY_UCTYPES (1) #define MICROPY_PY_UZLIB (1) #define MICROPY_PY_UJSON (1) pFad - Phonifier reborn

Pfad - The Proxy pFad of © 2024 Garber Painting. All rights reserved.

Note: This service is not intended for secure transactions such as banking, social media, email, or purchasing. Use at your own risk. We assume no liability whatsoever for broken pages.


Alternative Proxies:

Alternative Proxy

pFad Proxy

pFad v3 Proxy

pFad v4 Proxy