Asyncio ThreadSafeQueue and _thread: Program stops after some time without error message #17628
-
I'm working with a 433 MHz OOK receiver connected to an Esp32-WROOM runing MicroPython v1.24.1. I'd like to have the Receiver class running permanently on one core and sending the data via ThreadSafeQueue to the other core for processing and graphic functions. This must be caused by threading / asyncio use since the programm works fine for days without asyncio and using a second thread. Any help would be greatly appreciated. Programfrom machine import Pin
import time
import micropython
import _thread
import array
import asyncio
import gc
# from https://github.com/peterhinch/micropython-async/blob/master/v3/threadsafe/threadsafe_queue.py
from ThreadSafeQueue import ThreadSafeQueue
micropython.alloc_emergency_exception_buf(100)
# container for return data
class Messenger:
def __init__(self, **kwargs):
for key, value in kwargs.items():
setattr(self, key, value)
class RFReceiver:
MAX_PULSES = 500
MIN_HITS = 1
def __init__(self, pin_nr, TSQ, debug_messages = True, debug_function = None):
self.pin_nr = pin_nr # save for debugging
self.TSQ = TSQ
self.debug_messages = debug_messages
self.debug_function = debug_function
self._print(f'Starting receiver on Pin {self.pin_nr}.')
self.pin = Pin(self.pin_nr, Pin.IN)
self.lock = _thread.allocate_lock()
self.pulse_array = array.array('I', [0] * self.MAX_PULSES)
self.pulse_index = 0
self.last_falling_time = 0
self.data_buffer = {}
self.previous_message = 0
self.pin.irq(trigger=Pin.IRQ_RISING | Pin.IRQ_FALLING, handler=self.rx_interrupt) # ISR
self.task = asyncio.create_task(self.loop()) # Task runs forever
def _print(self, message):
message = str(message)
if self.debug_function is not None:
self.debug_function(message)
if self.debug_messages:
timestamp = time.localtime()
timestamp_formatted = '{:02d}:{:02d}:{:02d}'.format(timestamp[3],
timestamp[4],
timestamp[5])
print(f'[{self.__class__.__name__} {timestamp_formatted}] {message}')
def rx_interrupt(self, pin):
current_time = time.ticks_us()
if pin.value() == 1:
if self.last_falling_time != 0:
low_duration = time.ticks_diff(current_time, self.last_falling_time)
if self.pulse_index < self.MAX_PULSES:
if 1850 < low_duration < 2050 or 3750 < low_duration < 4050 or 8500 < low_duration < 9200:
with self.lock:
self.pulse_array[self.pulse_index] = low_duration # write to preallocated array
self.pulse_index += 1
else:
self.last_falling_time = current_time
# data is read periodically and analyzed for valid transmissions
async def loop(self):
while True:
gc.collect()
await asyncio.sleep(2)
pulse_copy = None
with self.lock: # read from preallocated array
if self.pulse_index > 40:
pulse_copy = self.pulse_array[:self.pulse_index]
self.pulse_index = 0
if pulse_copy is not None:
decoded = self.decode_signal_to_binary(pulse_copy)
if decoded:
data = self.decode_binary_data(decoded)
if data:
if self.validate_data(data):
self._print(f'{data.source} ({data.channel}): {data.temp} °C ({data.hits})')
result = [data.source, data.temp] # put data as list in Queue
await self.TSQ.put(result)
def validate_data(self, data):
return True
def decode_signal_to_binary(self, pulse_durations):
if not pulse_durations:
return None
pulse_list = list(pulse_durations)
frames = len(pulse_list)
if frames < 33: # not enough frames
return None
if max(pulse_list) < 8500: # no gap
return None
# Parts of the signal decoding function are based on Peter Hinch's
# micropython_remote RX class provided under MIT license.
gap = round(max(pulse_list) * 0.9)
while pulse_list and pulse_list[0] < gap:
pulse_list.pop(0)
if pulse_list:
pulse_list.pop(0)
res = []
while pulse_list:
lst = []
try:
while pulse_list and pulse_list[0] < gap:
lst.append(pulse_list.pop(0))
if pulse_list:
lst.append(pulse_list.pop(0))
except IndexError:
break
res.append(lst)
res = [r for r in res if len(r) in (33, 38)]
n_messages = len(res)
if n_messages < self.MIN_HITS:
return None
m = [round(sum(x) / n_messages) for x in zip(*res)]
gap_mean = m.pop(-1)
pulse_mean, lower_mean, upper_mean = self.find_clusters(m)
binary_string = ''.join(['0' if i < pulse_mean else '1' for i in m])
result = Messenger(length=len(m), hits=n_messages, pulse=pulse_mean,
high=upper_mean, low=lower_mean, gap=gap_mean,
message=binary_string)
return result
@staticmethod
def find_clusters(data):
overall_mean = sum(data) / len(data)
lower_group = [x for x in data if x < overall_mean]
upper_group = [x for x in data if x >= overall_mean]
lower_mean = sum(lower_group) / len(lower_group) if lower_group else 0
upper_mean = sum(upper_group) / len(upper_group) if upper_group else 0
pulse_mean = (lower_mean + upper_mean) / 2
return int(pulse_mean), int(lower_mean), int(upper_mean)
@staticmethod
def decode_binary_data(msg):
binary_string = msg.message
def convert(b, l): return b - (1 << l) if b & (1 << (l - 1)) else b
signals = {'TP': [37, 16, 9], 'NN': [32, 12, 8]}
for key, (total_len, sync_len, end_len) in signals.items():
if len(binary_string) == total_len:
sync_bits = binary_string[:sync_len]
data_bits = binary_string[sync_len:-end_len]
if len(data_bits) != 12:
return None
signal_source = key
channel = int(sync_bits[-2:], 2) + 1 if key == 'TP' else -1
data_value = convert(int(data_bits, 2), 12)
# Use round to suppress strange results like 9.111111111
temp = round(data_value * 0.1, 1)
msg.temp = temp
msg.source = signal_source
msg.channel = channel
return msg
return None
# from asyncio tutorial
def set_global_exception():
def handle_exception(loop, context):
import sys
sys.print_exception(context["exception"])
sys.exit()
loop = asyncio.get_event_loop()
loop.set_exception_handler(handle_exception)
# core0 Thread -> start async coroutine to start Receiver class
def core0_thread(TSQ):
asyncio.run(core0_async_starter(TSQ))
# start receiver on core0 from async coro
async def core0_async_starter(TSQ):
print('Thread started.')
receiver = RFReceiver(22, TSQ)
while True:
await asyncio.sleep(1)
# core1: print content of TSQ
async def process_data(TSQ):
while True:
async for obj in TSQ:
print(f'Q: {obj}')
await asyncio.sleep(0)
async def main():
gc.threshold(-1)
set_global_exception()
TSQ = ThreadSafeQueue(20) # ThreadSafeQueue
# receive data from OOK receiver and put in TSQ
_thread.start_new_thread(core0_thread, (TSQ,))
# display data from TSQ
asyncio.run(process_data(TSQ))
asyncio.run(main())
Exemplary output: Thread started.
[RFReceiver 16:35:13] Starting receiver on Pin 22.
[RFReceiver 16:35:17] NN (-1): 24.2 °C (4)
Q: ['NN', 24.2]
[RFReceiver 16:35:19] NN (-1): 24.2 °C (1)
Q: ['NN', 24.2]
[RFReceiver 16:35:41] TP (1): 16.6 °C (5)
Q: ['TP', 16.6]
[RFReceiver 16:35:49] NN (-1): 24.2 °C (6)
Q: ['NN', 24.2]
[RFReceiver 16:36:19] NN (-1): 24.2 °C (6)
Q: ['NN', 24.2]
[RFReceiver 16:36:31] TP (1): 16.6 °C (4)
Q: ['TP', 16.6] No more data is displayed after some random time. |
Beta Was this translation helpful? Give feedback.
Replies: 4 comments 4 replies
-
Do you have this: micropython.alloc_emergency_exception_buf(100) in your script? In any case, ESP32 user threads run on CORE1 and can have more than two threads running. CORE0 is used by Wi-Fi, BLE, etc. The RP2 can run only two threads, one in each core. |
Beta Was this translation helpful? Give feedback.
-
@de-dh You have asynchronous routines running in two threads. This is invalid. Asyncio is based on a task queue which is not thread safe. I think you are misunderstanding the relationship between threads and cores. On ESP32 MicroPython runs on a single processor core. Threading is implemented by pre-emptive context switching on that one core, with FreeRTOS running on the other core. As @shariltumin says, RP2 works differently: there is no underlying OS and threads map directly onto cores. This is unique to RP2. My approach to your problem would be to abandon threading and to rely on asyncio for concurrency. |
Beta Was this translation helpful? Give feedback.
-
Thank you both for your answers. If I use asyncio for concurrency, does it still make sense to use a lock from thread library for interfacing the ISR and poll the data in a loop? lock = _thread.allocate_lock()
def rx_interrupt(pin):
global last_rising_time, last_falling_time, low_pulse_index
current_time = time.ticks_us()
if pin.value() == 1:
if last_falling_time != 0:
low_duration = time.ticks_diff(current_time, last_falling_time)
if low_pulse_index < MAX_PULSES:
with lock:
if 1000 < low_duration < 10000:
low_pulse_durations[low_pulse_index] = low_duration
low_pulse_index += 1
else:
last_falling_time = current_time async def poll_data():
while True:
await asyncio.sleep(1)
if low_pulse_index > 30:
with lock:
low_pulse_copy = low_pulse_durations[:low_pulse_index]
low_pulse_index = 0
# process data |
Beta Was this translation helpful? Give feedback.
-
The simplest way to interface between an ISR and asynchronous code is to use a Things get slightly more involved if there is a possibility that another message might be received while the main code is processing the first. Then you need a You should never have a system where an ISR can wait on a You may need to think about where the message decoding takes place: in the ISR or in the main code. This depends on timing constraints but it may be that the best way is for successive ISR calls to assemble the message, firing the |
Beta Was this translation helpful? Give feedback.
@de-dh You have asynchronous routines running in two threads. This is invalid. Asyncio is based on a task queue which is not thread safe.
I think you are misunderstanding the relationship between threads and cores. On ESP32 MicroPython runs on a single processor core. Threading is implemented by pre-emptive context switching on that one core, with FreeRTOS running on the other core. As @shariltumin says, RP2 works differently: there is no underlying OS and threads map directly onto cores. This is unique to RP2.
My approach to your problem would be to abandon threading and to rely on asyncio for concurrency.