From 71884389e16339a70035aa6cabd776c6c8f292b7 Mon Sep 17 00:00:00 2001 From: Hinrich Mahler Date: Sun, 25 Oct 2020 19:35:01 +0100 Subject: [PATCH 01/15] Get started on refactoring MQ --- telegram/bot.py | 400 +++++++++++++++++++++++++++++++++-- telegram/ext/defaults.py | 46 +++- telegram/ext/messagequeue.py | 100 +++++---- telegram/utils/promise.py | 17 +- tests/test_defaults.py | 4 + tests/test_messagequeue.py | 5 +- tests/test_official.py | 1 + 7 files changed, 494 insertions(+), 79 deletions(-) diff --git a/telegram/bot.py b/telegram/bot.py index 03e39889b51..5e91a3c9162 100644 --- a/telegram/bot.py +++ b/telegram/bot.py @@ -92,7 +92,7 @@ ) if TYPE_CHECKING: - from telegram.ext import Defaults + from telegram.ext import Defaults, MessageQueue RT = TypeVar('RT') @@ -101,10 +101,10 @@ def info(func: Callable[..., RT]) -> Callable[..., RT]: @functools.wraps(func) def decorator(self: 'Bot', *args: Any, **kwargs: Any) -> RT: if not self.bot: - self.get_me() + self.get_me(delay_queue=None) if self._commands is None: - self.get_my_commands() + self.get_my_commands(delay_queue=None) result = func(self, *args, **kwargs) return result @@ -125,6 +125,50 @@ def decorator(self: 'Bot', *args: Any, **kwargs: Any) -> RT: return decorate(func, decorator) +def mq(func: Callable[..., RT], *args: Any, **kwargs: Any) -> Callable[..., RT]: + def decorator(self: Union[Callable, Bot], *args: Any, **kwargs: Any) -> RT: + if callable(self): + self = cast('Bot', args[0]) + + if not self.message_queue or not self.message_queue.running: + return func(*args, **kwargs) + + delay_queue = kwargs.pop('delay_queue', None) + if not delay_queue: + return func(*args, **kwargs) + + if delay_queue == self.message_queue.DEFAULT_QUEUE: + # For default queue, check if we're in a group setting or not + arg_spec = inspect.getfullargspec(func) + chat_id: Union[str, int] = '' + if 'chat_id' in kwargs: + chat_id = kwargs['chat_id'] + elif 'chat_id' in arg_spec.args: + idx = arg_spec.args.index('chat_id') + chat_id = args[idx] + + if not chat_id: + is_group = False + elif isinstance(chat_id, str) and chat_id.startswith('@'): + is_group = True + else: + try: + is_group = int(chat_id) < 0 + except ValueError: + is_group = False + + queue = self.message_queue.GROUP_QUEUE if is_group else delay_queue + return self.message_queue.process( # type: ignore[return-value] + func, queue, self, *args, **kwargs + ) + + return self.message_queue.process( # type: ignore[return-value] + func, delay_queue, self, *args, **kwargs + ) + + return decorate(func, decorator) + + class Bot(TelegramObject): """This object represents a Telegram Bot. @@ -138,12 +182,19 @@ class Bot(TelegramObject): private_key_password (:obj:`bytes`, optional): Password for above private key. defaults (:class:`telegram.ext.Defaults`, optional): An object containing default values to be used if not set explicitly in the bot methods. + message_queue (:class:`telegram.ext.MessageQueue`, optional): A message queue to pass + requests through in order to avoid flood limits. Note: - Most bot methods have the argument ``api_kwargs`` which allows to pass arbitrary keywords - to the Telegram API. This can be used to access new features of the API before they were - incorporated into PTB. However, this is not guaranteed to work, i.e. it will fail for - passing files. + * Most bot methods have the argument ``api_kwargs`` which allows to pass arbitrary keywords + to the Telegram API. This can be used to access new features of the API before they were + incorporated into PTB. However, this is not guaranteed to work, i.e. it will fail for + passing files. + * Most bot methods have the argument ``delay_queue`` which allows you to pass the request + to the specified delay queue of the :attr:`message_queue`. This will have an effect only, + if the :class:`telegram.ext.MessageQueue` is set and running. When passing a request + through the :attr:`message_queue`, the bot method will return a + :class:`telegram.utils.Promise` instead of the documented return value. """ @@ -172,6 +223,9 @@ def __new__(cls, *args: Any, **kwargs: Any) -> 'Bot': for kwarg_name in needs_default if (getattr(defaults, kwarg_name) is not DEFAULT_NONE) } + # ... do some special casing for delay_queue because that may depend on the method + if 'delay_queue' in default_kwargs: + default_kwargs['delay_queue'] = defaults.delay_queue_per_method[method_name] # ... apply the defaults using a partial if default_kwargs: setattr(instance, method_name, functools.partial(method, **default_kwargs)) @@ -187,11 +241,12 @@ def __init__( private_key: bytes = None, private_key_password: bytes = None, defaults: 'Defaults' = None, + message_queue: 'MessageQueue' = None, ): self.token = self._validate_token(token) - # Gather default self.defaults = defaults + self.message_queue = message_queue if base_url is None: base_url = 'https://api.telegram.org/bot' @@ -354,7 +409,10 @@ def name(self) -> str: return '@{}'.format(self.username) @log - def get_me(self, timeout: int = None, api_kwargs: JSONDict = None) -> Optional[User]: + @mq + def get_me( + self, timeout: int = None, api_kwargs: JSONDict = None, delay_queue: str = None + ) -> Optional[User]: """A simple method for testing your bot's auth token. Requires no parameters. Args: @@ -363,6 +421,8 @@ def get_me(self, timeout: int = None, api_kwargs: JSONDict = None) -> Optional[U the connection pool). api_kwargs (:obj:`dict`, optional): Arbitrary keyword arguments to be passed to the Telegram API. + delay_queue (:obj:`str`, optional): The name of the delay queue to pass this request + through. Defaults to :obj:`None`. Returns: :class:`telegram.User`: A :class:`telegram.User` instance representing that bot if the @@ -379,6 +439,7 @@ def get_me(self, timeout: int = None, api_kwargs: JSONDict = None) -> Optional[U return self.bot @log + @mq def send_message( self, chat_id: Union[int, str], @@ -390,6 +451,7 @@ def send_message( reply_markup: ReplyMarkup = None, timeout: float = None, api_kwargs: JSONDict = None, + delay_queue: str = None, ) -> Optional[Message]: """Use this method to send text messages. @@ -415,6 +477,8 @@ def send_message( the connection pool). api_kwargs (:obj:`dict`, optional): Arbitrary keyword arguments to be passed to the Telegram API. + delay_queue (:obj:`str`, optional): The name of the delay queue to pass this request + through. Defaults to :obj:`None`. Returns: :class:`telegram.Message`: On success, the sent message is returned. @@ -441,12 +505,14 @@ def send_message( ) @log + @mq def delete_message( self, chat_id: Union[str, int], message_id: Union[str, int], timeout: float = None, api_kwargs: JSONDict = None, + delay_queue: str = None, ) -> bool: """ Use this method to delete a message, including service messages, with the following @@ -471,6 +537,8 @@ def delete_message( the connection pool). api_kwargs (:obj:`dict`, optional): Arbitrary keyword arguments to be passed to the Telegram API. + delay_queue (:obj:`str`, optional): The name of the delay queue to pass this request + through. Defaults to :obj:`None`. Returns: :obj:`bool`: On success, :obj:`True` is returned. @@ -486,6 +554,7 @@ def delete_message( return result # type: ignore[return-value] @log + @mq def forward_message( self, chat_id: Union[int, str], @@ -494,6 +563,7 @@ def forward_message( disable_notification: bool = False, timeout: float = None, api_kwargs: JSONDict = None, + delay_queue: str = None, ) -> Optional[Message]: """Use this method to forward messages of any kind. @@ -510,6 +580,8 @@ def forward_message( the connection pool). api_kwargs (:obj:`dict`, optional): Arbitrary keyword arguments to be passed to the Telegram API. + delay_queue (:obj:`str`, optional): The name of the delay queue to pass this request + through. Defaults to :obj:`None`. Returns: :class:`telegram.Message`: On success, the sent Message is returned. @@ -536,6 +608,7 @@ def forward_message( ) @log + @mq def send_photo( self, chat_id: int, @@ -547,6 +620,7 @@ def send_photo( timeout: float = 20, parse_mode: str = None, api_kwargs: JSONDict = None, + delay_queue: str = None, ) -> Optional[Message]: """Use this method to send photos. @@ -577,6 +651,8 @@ def send_photo( timeout (:obj:`int` | :obj:`float`, optional): Send file timeout (default: 20 seconds). api_kwargs (:obj:`dict`, optional): Arbitrary keyword arguments to be passed to the Telegram API. + delay_queue (:obj:`str`, optional): The name of the delay queue to pass this request + through. Defaults to :obj:`None`. Returns: :class:`telegram.Message`: On success, the sent Message is returned. @@ -609,6 +685,7 @@ def send_photo( ) @log + @mq def send_audio( self, chat_id: Union[int, str], @@ -624,6 +701,7 @@ def send_audio( parse_mode: str = None, thumb: FileLike = None, api_kwargs: JSONDict = None, + delay_queue: str = None, ) -> Optional[Message]: """ Use this method to send audio files, if you want Telegram clients to display them in the @@ -669,6 +747,8 @@ def send_audio( timeout (:obj:`int` | :obj:`float`, optional): Send file timeout (default: 20 seconds). api_kwargs (:obj:`dict`, optional): Arbitrary keyword arguments to be passed to the Telegram API. + delay_queue (:obj:`str`, optional): The name of the delay queue to pass this request + through. Defaults to :obj:`None`. Returns: :class:`telegram.Message`: On success, the sent Message is returned. @@ -712,6 +792,7 @@ def send_audio( ) @log + @mq def send_document( self, chat_id: Union[int, str], @@ -725,6 +806,7 @@ def send_document( parse_mode: str = None, thumb: FileLike = None, api_kwargs: JSONDict = None, + delay_queue: str = None, ) -> Optional[Message]: """ Use this method to send general files. @@ -766,6 +848,8 @@ def send_document( timeout (:obj:`int` | :obj:`float`, optional): Send file timeout (default: 20 seconds). api_kwargs (:obj:`dict`, optional): Arbitrary keyword arguments to be passed to the Telegram API. + delay_queue (:obj:`str`, optional): The name of the delay queue to pass this request + through. Defaults to :obj:`None`. Returns: :class:`telegram.Message`: On success, the sent Message is returned. @@ -803,6 +887,7 @@ def send_document( ) @log + @mq def send_sticker( self, chat_id: Union[int, str], @@ -812,6 +897,7 @@ def send_sticker( reply_markup: ReplyMarkup = None, timeout: float = 20, api_kwargs: JSONDict = None, + delay_queue: str = None, ) -> Optional[Message]: """ Use this method to send static .WEBP or animated .TGS stickers. @@ -838,6 +924,8 @@ def send_sticker( timeout (:obj:`int` | :obj:`float`, optional): Send file timeout (default: 20 seconds). api_kwargs (:obj:`dict`, optional): Arbitrary keyword arguments to be passed to the Telegram API. + delay_queue (:obj:`str`, optional): The name of the delay queue to pass this request + through. Defaults to :obj:`None`. Returns: :class:`telegram.Message`: On success, the sent Message is returned. @@ -865,6 +953,7 @@ def send_sticker( ) @log + @mq def send_video( self, chat_id: Union[int, str], @@ -881,6 +970,7 @@ def send_video( supports_streaming: bool = None, thumb: FileLike = None, api_kwargs: JSONDict = None, + delay_queue: str = None, ) -> Optional[Message]: """ Use this method to send video files, Telegram clients support mp4 videos @@ -929,6 +1019,8 @@ def send_video( timeout (:obj:`int` | :obj:`float`, optional): Send file timeout (default: 20 seconds). api_kwargs (:obj:`dict`, optional): Arbitrary keyword arguments to be passed to the Telegram API. + delay_queue (:obj:`str`, optional): The name of the delay queue to pass this request + through. Defaults to :obj:`None`. Returns: :class:`telegram.Message`: On success, the sent Message is returned. @@ -974,6 +1066,7 @@ def send_video( ) @log + @mq def send_video_note( self, chat_id: Union[int, str], @@ -986,6 +1079,7 @@ def send_video_note( timeout: float = 20, thumb: FileLike = None, api_kwargs: JSONDict = None, + delay_queue: str = None, ) -> Optional[Message]: """ As of v.4.0, Telegram clients support rounded square mp4 videos of up to 1 minute long. @@ -1024,6 +1118,8 @@ def send_video_note( timeout (:obj:`int` | :obj:`float`, optional): Send file timeout (default: 20 seconds). api_kwargs (:obj:`dict`, optional): Arbitrary keyword arguments to be passed to the Telegram API. + delay_queue (:obj:`str`, optional): The name of the delay queue to pass this request + through. Defaults to :obj:`None`. Returns: :class:`telegram.Message`: On success, the sent Message is returned. @@ -1061,6 +1157,7 @@ def send_video_note( ) @log + @mq def send_animation( self, chat_id: Union[int, str], @@ -1076,6 +1173,7 @@ def send_animation( reply_markup: ReplyMarkup = None, timeout: float = 20, api_kwargs: JSONDict = None, + delay_queue: str = None, ) -> Optional[Message]: """ Use this method to send animation files (GIF or H.264/MPEG-4 AVC video without sound). @@ -1118,6 +1216,8 @@ def send_animation( timeout (:obj:`int` | :obj:`float`, optional): Send file timeout (default: 20 seconds). api_kwargs (:obj:`dict`, optional): Arbitrary keyword arguments to be passed to the Telegram API. + delay_queue (:obj:`str`, optional): The name of the delay queue to pass this request + through. Defaults to :obj:`None`. Returns: :class:`telegram.Message`: On success, the sent Message is returned. @@ -1161,6 +1261,7 @@ def send_animation( ) @log + @mq def send_voice( self, chat_id: Union[int, str], @@ -1173,6 +1274,7 @@ def send_voice( timeout: float = 20, parse_mode: str = None, api_kwargs: JSONDict = None, + delay_queue: str = None, ) -> Optional[Message]: """ Use this method to send audio files, if you want Telegram clients to display the file @@ -1208,6 +1310,8 @@ def send_voice( timeout (:obj:`int` | :obj:`float`, optional): Send file timeout (default: 20 seconds). api_kwargs (:obj:`dict`, optional): Arbitrary keyword arguments to be passed to the Telegram API. + delay_queue (:obj:`str`, optional): The name of the delay queue to pass this request + through. Defaults to :obj:`None`. Returns: :class:`telegram.Message`: On success, the sent Message is returned. @@ -1242,6 +1346,7 @@ def send_voice( ) @log + @mq def send_media_group( self, chat_id: Union[int, str], @@ -1250,6 +1355,7 @@ def send_media_group( reply_to_message_id: Union[int, str] = None, timeout: float = 20, api_kwargs: JSONDict = None, + delay_queue: str = None, ) -> List[Optional[Message]]: """Use this method to send a group of photos or videos as an album. @@ -1265,6 +1371,8 @@ def send_media_group( timeout (:obj:`int` | :obj:`float`, optional): Send file timeout (default: 20 seconds). api_kwargs (:obj:`dict`, optional): Arbitrary keyword arguments to be passed to the Telegram API. + delay_queue (:obj:`str`, optional): The name of the delay queue to pass this request + through. Defaults to :obj:`None`. Returns: List[:class:`telegram.Message`]: An array of the sent Messages. @@ -1295,6 +1403,7 @@ def send_media_group( return [Message.de_json(res, self) for res in result] # type: ignore @log + @mq def send_location( self, chat_id: Union[int, str], @@ -1307,6 +1416,7 @@ def send_location( location: Location = None, live_period: int = None, api_kwargs: JSONDict = None, + delay_queue: str = None, ) -> Optional[Message]: """Use this method to send point on the map. @@ -1333,6 +1443,8 @@ def send_location( the connection pool). api_kwargs (:obj:`dict`, optional): Arbitrary keyword arguments to be passed to the Telegram API. + delay_queue (:obj:`str`, optional): The name of the delay queue to pass this request + through. Defaults to :obj:`None`. Returns: :class:`telegram.Message`: On success, the sent Message is returned. @@ -1371,6 +1483,7 @@ def send_location( ) @log + @mq def edit_message_live_location( self, chat_id: Union[str, int] = None, @@ -1382,6 +1495,7 @@ def edit_message_live_location( reply_markup: ReplyMarkup = None, timeout: float = None, api_kwargs: JSONDict = None, + delay_queue: str = None, ) -> Union[Optional[Message], bool]: """Use this method to edit live location messages sent by the bot or via the bot (for inline bots). A location can be edited until its :attr:`live_period` expires or @@ -1408,6 +1522,8 @@ def edit_message_live_location( the connection pool). api_kwargs (:obj:`dict`, optional): Arbitrary keyword arguments to be passed to the Telegram API. + delay_queue (:obj:`str`, optional): The name of the delay queue to pass this request + through. Defaults to :obj:`None`. Returns: :class:`telegram.Message`: On success, if edited message is sent by the bot, the @@ -1444,6 +1560,7 @@ def edit_message_live_location( ) @log + @mq def stop_message_live_location( self, chat_id: Union[str, int] = None, @@ -1452,6 +1569,7 @@ def stop_message_live_location( reply_markup: ReplyMarkup = None, timeout: float = None, api_kwargs: JSONDict = None, + delay_queue: str = None, ) -> Union[Optional[Message], bool]: """Use this method to stop updating a live location message sent by the bot or via the bot (for inline bots) before live_period expires. @@ -1471,6 +1589,8 @@ def stop_message_live_location( the connection pool). api_kwargs (:obj:`dict`, optional): Arbitrary keyword arguments to be passed to the Telegram API. + delay_queue (:obj:`str`, optional): The name of the delay queue to pass this request + through. Defaults to :obj:`None`. Returns: :class:`telegram.Message`: On success, if edited message is sent by the bot, the @@ -1494,6 +1614,7 @@ def stop_message_live_location( ) @log + @mq def send_venue( self, chat_id: Union[int, str], @@ -1509,6 +1630,7 @@ def send_venue( venue: Venue = None, foursquare_type: str = None, api_kwargs: JSONDict = None, + delay_queue: str = None, ) -> Optional[Message]: """Use this method to send information about a venue. @@ -1541,6 +1663,8 @@ def send_venue( the connection pool). api_kwargs (:obj:`dict`, optional): Arbitrary keyword arguments to be passed to the Telegram API. + delay_queue (:obj:`str`, optional): The name of the delay queue to pass this request + through. Defaults to :obj:`None`. Returns: :class:`telegram.Message`: On success, the sent Message is returned. @@ -1587,6 +1711,7 @@ def send_venue( ) @log + @mq def send_contact( self, chat_id: Union[int, str], @@ -1600,6 +1725,7 @@ def send_contact( contact: Contact = None, vcard: str = None, api_kwargs: JSONDict = None, + delay_queue: str = None, ) -> Optional[Message]: """Use this method to send phone contacts. @@ -1628,6 +1754,8 @@ def send_contact( the connection pool). api_kwargs (:obj:`dict`, optional): Arbitrary keyword arguments to be passed to the Telegram API. + delay_queue (:obj:`str`, optional): The name of the delay queue to pass this request + through. Defaults to :obj:`None`. Returns: :class:`telegram.Message`: On success, the sent Message is returned. @@ -1669,6 +1797,7 @@ def send_contact( ) @log + @mq def send_game( self, chat_id: Union[int, str], @@ -1678,6 +1807,7 @@ def send_game( reply_markup: ReplyMarkup = None, timeout: float = None, api_kwargs: JSONDict = None, + delay_queue: str = None, ) -> Optional[Message]: """Use this method to send a game. @@ -1698,6 +1828,8 @@ def send_game( the connection pool). api_kwargs (:obj:`dict`, optional): Arbitrary keyword arguments to be passed to the Telegram API. + delay_queue (:obj:`str`, optional): The name of the delay queue to pass this request + through. Defaults to :obj:`None`. Returns: :class:`telegram.Message`: On success, the sent Message is returned. @@ -1719,12 +1851,14 @@ def send_game( ) @log + @mq def send_chat_action( self, chat_id: Union[str, int], action: str, timeout: float = None, api_kwargs: JSONDict = None, + delay_queue: str = None, ) -> bool: """ Use this method when you need to tell the user that something is happening on the bot's @@ -1743,6 +1877,8 @@ def send_chat_action( the connection pool). api_kwargs (:obj:`dict`, optional): Arbitrary keyword arguments to be passed to the Telegram API. + delay_queue (:obj:`str`, optional): The name of the delay queue to pass this request + through. Defaults to :obj:`None`. Returns: :obj:`bool`: On success, :obj:`True` is returned. @@ -1758,6 +1894,7 @@ def send_chat_action( return result # type: ignore[return-value] @log + @mq def answer_inline_query( self, inline_query_id: str, @@ -1770,6 +1907,7 @@ def answer_inline_query( timeout: float = None, current_offset: str = None, api_kwargs: JSONDict = None, + delay_queue: str = None, ) -> bool: """ Use this method to send answers to an inline query. No more than 50 results per query are @@ -1811,6 +1949,8 @@ def answer_inline_query( the connection pool). api_kwargs (:obj:`dict`, optional): Arbitrary keyword arguments to be passed to the Telegram API. + delay_queue (:obj:`str`, optional): The name of the delay queue to pass this request + through. Defaults to :obj:`None`. Example: An inline bot that sends YouTube videos can ask the user to connect the bot to their @@ -1913,6 +2053,7 @@ def _set_defaults(res): ) @log + @mq def get_user_profile_photos( self, user_id: Union[str, int], @@ -1920,6 +2061,7 @@ def get_user_profile_photos( limit: int = 100, timeout: float = None, api_kwargs: JSONDict = None, + delay_queue: str = None, ) -> Optional[UserProfilePhotos]: """Use this method to get a list of profile pictures for a user. @@ -1934,6 +2076,8 @@ def get_user_profile_photos( the connection pool). api_kwargs (:obj:`dict`, optional): Arbitrary keyword arguments to be passed to the Telegram API. + delay_queue (:obj:`str`, optional): The name of the delay queue to pass this request + through. Defaults to :obj:`None`. Returns: :class:`telegram.UserProfilePhotos` @@ -1954,6 +2098,7 @@ def get_user_profile_photos( return UserProfilePhotos.de_json(result, self) # type: ignore @log + @mq def get_file( self, file_id: Union[ @@ -1961,6 +2106,7 @@ def get_file( ], timeout: float = None, api_kwargs: JSONDict = None, + delay_queue: str = None, ) -> File: """ Use this method to get basic info about a file and prepare it for downloading. For the @@ -1987,6 +2133,8 @@ def get_file( the connection pool). api_kwargs (:obj:`dict`, optional): Arbitrary keyword arguments to be passed to the Telegram API. + delay_queue (:obj:`str`, optional): The name of the delay queue to pass this request + through. Defaults to :obj:`None`. Returns: :class:`telegram.File` @@ -2012,6 +2160,7 @@ def get_file( return File.de_json(result, self) # type: ignore @log + @mq def kick_chat_member( self, chat_id: Union[str, int], @@ -2019,6 +2168,7 @@ def kick_chat_member( timeout: float = None, until_date: Union[int, datetime] = None, api_kwargs: JSONDict = None, + delay_queue: str = None, ) -> bool: """ Use this method to kick a user from a group or a supergroup or a channel. In the case of @@ -2040,6 +2190,8 @@ def kick_chat_member( bot will be used. api_kwargs (:obj:`dict`, optional): Arbitrary keyword arguments to be passed to the Telegram API. + delay_queue (:obj:`str`, optional): The name of the delay queue to pass this request + through. Defaults to :obj:`None`. Returns: :obj:`bool` On success, :obj:`True` is returned. @@ -2062,12 +2214,14 @@ def kick_chat_member( return result # type: ignore[return-value] @log + @mq def unban_chat_member( self, chat_id: Union[str, int], user_id: Union[str, int], timeout: float = None, api_kwargs: JSONDict = None, + delay_queue: str = None, ) -> bool: """Use this method to unban a previously kicked user in a supergroup or channel. @@ -2083,6 +2237,8 @@ def unban_chat_member( the connection pool). api_kwargs (:obj:`dict`, optional): Arbitrary keyword arguments to be passed to the Telegram API. + delay_queue (:obj:`str`, optional): The name of the delay queue to pass this request + through. Defaults to :obj:`None`. Returns: :obj:`bool` On success, :obj:`True` is returned. @@ -2098,6 +2254,7 @@ def unban_chat_member( return result # type: ignore[return-value] @log + @mq def answer_callback_query( self, callback_query_id: str, @@ -2107,6 +2264,7 @@ def answer_callback_query( cache_time: int = None, timeout: float = None, api_kwargs: JSONDict = None, + delay_queue: str = None, ) -> bool: """ Use this method to send answers to callback queries sent from inline keyboards. The answer @@ -2137,6 +2295,8 @@ def answer_callback_query( the connection pool). api_kwargs (:obj:`dict`, optional): Arbitrary keyword arguments to be passed to the Telegram API. + delay_queue (:obj:`str`, optional): The name of the delay queue to pass this request + through. Defaults to :obj:`None`. Returns: :obj:`bool` On success, :obj:`True` is returned. @@ -2161,6 +2321,7 @@ def answer_callback_query( return result # type: ignore[return-value] @log + @mq def edit_message_text( self, text: str, @@ -2172,6 +2333,7 @@ def edit_message_text( reply_markup: ReplyMarkup = None, timeout: float = None, api_kwargs: JSONDict = None, + delay_queue: str = None, ) -> Union[Optional[Message], bool]: """ Use this method to edit text and game messages sent by the bot or via the bot (for inline @@ -2198,6 +2360,8 @@ def edit_message_text( the connection pool). api_kwargs (:obj:`dict`, optional): Arbitrary keyword arguments to be passed to the Telegram API. + delay_queue (:obj:`str`, optional): The name of the delay queue to pass this request + through. Defaults to :obj:`None`. Returns: :class:`telegram.Message`: On success, if edited message is sent by the bot, the @@ -2229,6 +2393,7 @@ def edit_message_text( ) @log + @mq def edit_message_caption( self, chat_id: Union[str, int] = None, @@ -2239,6 +2404,7 @@ def edit_message_caption( timeout: float = None, parse_mode: str = None, api_kwargs: JSONDict = None, + delay_queue: str = None, ) -> Union[Message, bool]: """ Use this method to edit captions of messages sent by the bot or via the bot @@ -2264,6 +2430,8 @@ def edit_message_caption( the connection pool). api_kwargs (:obj:`dict`, optional): Arbitrary keyword arguments to be passed to the Telegram API. + delay_queue (:obj:`str`, optional): The name of the delay queue to pass this request + through. Defaults to :obj:`None`. Returns: :class:`telegram.Message`: On success, if edited message is sent by the bot, the @@ -2301,6 +2469,7 @@ def edit_message_caption( ) @log + @mq def edit_message_media( self, chat_id: Union[str, int] = None, @@ -2310,6 +2479,7 @@ def edit_message_media( reply_markup: ReplyMarkup = None, timeout: float = None, api_kwargs: JSONDict = None, + delay_queue: str = None, ) -> Union[Message, bool]: """ Use this method to edit animation, audio, document, photo, or video messages. If a @@ -2334,6 +2504,8 @@ def edit_message_media( the connection pool). api_kwargs (:obj:`dict`, optional): Arbitrary keyword arguments to be passed to the Telegram API. + delay_queue (:obj:`str`, optional): The name of the delay queue to pass this request + through. Defaults to :obj:`None`. Returns: :class:`telegram.Message`: On success, if edited message is sent by the bot, the @@ -2367,6 +2539,7 @@ def edit_message_media( ) @log + @mq def edit_message_reply_markup( self, chat_id: Union[str, int] = None, @@ -2375,6 +2548,7 @@ def edit_message_reply_markup( reply_markup: ReplyMarkup = None, timeout: float = None, api_kwargs: JSONDict = None, + delay_queue: str = None, ) -> Union[Message, bool]: """ Use this method to edit only the reply markup of messages sent by the bot or via the bot @@ -2395,6 +2569,8 @@ def edit_message_reply_markup( the connection pool). api_kwargs (:obj:`dict`, optional): Arbitrary keyword arguments to be passed to the Telegram API. + delay_queue (:obj:`str`, optional): The name of the delay queue to pass this request + through. Defaults to :obj:`None`. Returns: :class:`telegram.Message`: On success, if edited message is sent by the bot, the @@ -2428,6 +2604,7 @@ def edit_message_reply_markup( ) @log + @mq def get_updates( self, offset: int = None, @@ -2436,6 +2613,7 @@ def get_updates( read_latency: float = 2.0, allowed_updates: List[str] = None, api_kwargs: JSONDict = None, + delay_queue: str = None, ) -> List[Update]: """Use this method to receive incoming updates using long polling. @@ -2462,6 +2640,8 @@ def get_updates( updates may be received for a short period of time. api_kwargs (:obj:`dict`, optional): Arbitrary keyword arguments to be passed to the Telegram API. + delay_queue (:obj:`str`, optional): The name of the delay queue to pass this request + through. Defaults to :obj:`None`. Note: 1. This method will not work if an outgoing webhook is set up. @@ -2617,8 +2797,13 @@ def delete_webhook(self, timeout: float = None, api_kwargs: JSONDict = None) -> return result # type: ignore[return-value] @log + @mq def leave_chat( - self, chat_id: Union[str, int], timeout: float = None, api_kwargs: JSONDict = None + self, + chat_id: Union[str, int], + timeout: float = None, + api_kwargs: JSONDict = None, + delay_queue: str = None, ) -> bool: """Use this method for your bot to leave a group, supergroup or channel. @@ -2630,6 +2815,8 @@ def leave_chat( the connection pool). api_kwargs (:obj:`dict`, optional): Arbitrary keyword arguments to be passed to the Telegram API. + delay_queue (:obj:`str`, optional): The name of the delay queue to pass this request + through. Defaults to :obj:`None`. Returns: :obj:`bool` On success, :obj:`True` is returned. @@ -2645,8 +2832,13 @@ def leave_chat( return result # type: ignore[return-value] @log + @mq def get_chat( - self, chat_id: Union[str, int], timeout: float = None, api_kwargs: JSONDict = None + self, + chat_id: Union[str, int], + timeout: float = None, + api_kwargs: JSONDict = None, + delay_queue: str = None, ) -> Chat: """ Use this method to get up to date information about the chat (current name of the user for @@ -2660,6 +2852,8 @@ def get_chat( the connection pool). api_kwargs (:obj:`dict`, optional): Arbitrary keyword arguments to be passed to the Telegram API. + delay_queue (:obj:`str`, optional): The name of the delay queue to pass this request + through. Defaults to :obj:`None`. Returns: :class:`telegram.Chat` @@ -2678,8 +2872,13 @@ def get_chat( return Chat.de_json(result, self) # type: ignore @log + @mq def get_chat_administrators( - self, chat_id: Union[str, int], timeout: float = None, api_kwargs: JSONDict = None + self, + chat_id: Union[str, int], + timeout: float = None, + api_kwargs: JSONDict = None, + delay_queue: str = None, ) -> List[ChatMember]: """ Use this method to get a list of administrators in a chat. @@ -2692,6 +2891,8 @@ def get_chat_administrators( the connection pool). api_kwargs (:obj:`dict`, optional): Arbitrary keyword arguments to be passed to the Telegram API. + delay_queue (:obj:`str`, optional): The name of the delay queue to pass this request + through. Defaults to :obj:`None`. Returns: List[:class:`telegram.ChatMember`]: On success, returns a list of ``ChatMember`` @@ -2710,8 +2911,13 @@ def get_chat_administrators( return [ChatMember.de_json(x, self) for x in result] # type: ignore @log + @mq def get_chat_members_count( - self, chat_id: Union[str, int], timeout: float = None, api_kwargs: JSONDict = None + self, + chat_id: Union[str, int], + timeout: float = None, + api_kwargs: JSONDict = None, + delay_queue: str = None, ) -> int: """Use this method to get the number of members in a chat. @@ -2723,6 +2929,8 @@ def get_chat_members_count( the connection pool). api_kwargs (:obj:`dict`, optional): Arbitrary keyword arguments to be passed to the Telegram API. + delay_queue (:obj:`str`, optional): The name of the delay queue to pass this request + through. Defaults to :obj:`None`. Returns: :obj:`int`: Number of members in the chat. @@ -2738,12 +2946,14 @@ def get_chat_members_count( return result # type: ignore[return-value] @log + @mq def get_chat_member( self, chat_id: Union[str, int], user_id: Union[str, int], timeout: float = None, api_kwargs: JSONDict = None, + delay_queue: str = None, ) -> ChatMember: """Use this method to get information about a member of a chat. @@ -2756,6 +2966,8 @@ def get_chat_member( the connection pool). api_kwargs (:obj:`dict`, optional): Arbitrary keyword arguments to be passed to the Telegram API. + delay_queue (:obj:`str`, optional): The name of the delay queue to pass this request + through. Defaults to :obj:`None`. Returns: :class:`telegram.ChatMember` @@ -2771,12 +2983,14 @@ def get_chat_member( return ChatMember.de_json(result, self) # type: ignore @log + @mq def set_chat_sticker_set( self, chat_id: Union[str, int], sticker_set_name: str, timeout: float = None, api_kwargs: JSONDict = None, + delay_queue: str = None, ) -> bool: """Use this method to set a new group sticker set for a supergroup. The bot must be an administrator in the chat for this to work and must have the appropriate @@ -2793,6 +3007,8 @@ def set_chat_sticker_set( the connection pool). api_kwargs (:obj:`dict`, optional): Arbitrary keyword arguments to be passed to the Telegram API. + delay_queue (:obj:`str`, optional): The name of the delay queue to pass this request + through. Defaults to :obj:`None`. Returns: :obj:`bool`: On success, :obj:`True` is returned. @@ -2804,8 +3020,13 @@ def set_chat_sticker_set( return result # type: ignore[return-value] @log + @mq def delete_chat_sticker_set( - self, chat_id: Union[str, int], timeout: float = None, api_kwargs: JSONDict = None + self, + chat_id: Union[str, int], + timeout: float = None, + api_kwargs: JSONDict = None, + delay_queue: str = None, ) -> bool: """Use this method to delete a group sticker set from a supergroup. The bot must be an administrator in the chat for this to work and must have the appropriate admin rights. @@ -2820,6 +3041,8 @@ def delete_chat_sticker_set( the connection pool). api_kwargs (:obj:`dict`, optional): Arbitrary keyword arguments to be passed to the Telegram API. + delay_queue (:obj:`str`, optional): The name of the delay queue to pass this request + through. Defaults to :obj:`None`. Returns: :obj:`bool`: On success, :obj:`True` is returned. @@ -2830,6 +3053,7 @@ def delete_chat_sticker_set( return result # type: ignore[return-value] + @log def get_webhook_info(self, timeout: float = None, api_kwargs: JSONDict = None) -> WebhookInfo: """Use this method to get current webhook status. Requires no parameters. @@ -2851,6 +3075,7 @@ def get_webhook_info(self, timeout: float = None, api_kwargs: JSONDict = None) - return WebhookInfo.de_json(result, self) # type: ignore @log + @mq def set_game_score( self, user_id: Union[int, str], @@ -2862,6 +3087,7 @@ def set_game_score( disable_edit_message: bool = None, timeout: float = None, api_kwargs: JSONDict = None, + delay_queue: str = None, ) -> Union[Message, bool]: """ Use this method to set the score of the specified user in a game. @@ -2884,6 +3110,8 @@ def set_game_score( the connection pool). api_kwargs (:obj:`dict`, optional): Arbitrary keyword arguments to be passed to the Telegram API. + delay_queue (:obj:`str`, optional): The name of the delay queue to pass this request + through. Defaults to :obj:`None`. Returns: :class:`telegram.Message`: The edited message, or if the message wasn't sent by the bot @@ -2915,6 +3143,7 @@ def set_game_score( ) @log + @mq def get_game_high_scores( self, user_id: Union[int, str], @@ -2923,6 +3152,7 @@ def get_game_high_scores( inline_message_id: Union[str, int] = None, timeout: float = None, api_kwargs: JSONDict = None, + delay_queue: str = None, ) -> List[GameHighScore]: """ Use this method to get data for high score tables. Will return the score of the specified @@ -2941,6 +3171,8 @@ def get_game_high_scores( the connection pool). api_kwargs (:obj:`dict`, optional): Arbitrary keyword arguments to be passed to the Telegram API. + delay_queue (:obj:`str`, optional): The name of the delay queue to pass this request + through. Defaults to :obj:`None`. Returns: List[:class:`telegram.GameHighScore`] @@ -2963,6 +3195,7 @@ def get_game_high_scores( return [GameHighScore.de_json(hs, self) for hs in result] # type: ignore @log + @mq def send_invoice( self, chat_id: Union[int, str], @@ -2990,6 +3223,7 @@ def send_invoice( send_email_to_provider: bool = None, timeout: float = None, api_kwargs: JSONDict = None, + delay_queue: str = None, ) -> Message: """Use this method to send invoices. @@ -3043,6 +3277,8 @@ def send_invoice( the connection pool). api_kwargs (:obj:`dict`, optional): Arbitrary keyword arguments to be passed to the Telegram API. + delay_queue (:obj:`str`, optional): The name of the delay queue to pass this request + through. Defaults to :obj:`None`. Returns: :class:`telegram.Message`: On success, the sent Message is returned. @@ -3100,6 +3336,7 @@ def send_invoice( ) @log + @mq def answer_shipping_query( self, shipping_query_id: str, @@ -3108,6 +3345,7 @@ def answer_shipping_query( error_message: str = None, timeout: float = None, api_kwargs: JSONDict = None, + delay_queue: str = None, ) -> bool: """ If you sent an invoice requesting a shipping address and the parameter is_flexible was @@ -3130,6 +3368,8 @@ def answer_shipping_query( the connection pool). api_kwargs (:obj:`dict`, optional): Arbitrary keyword arguments to be passed to the Telegram API. + delay_queue (:obj:`str`, optional): The name of the delay queue to pass this request + through. Defaults to :obj:`None`. Returns: :obj:`bool`: On success, :obj:`True` is returned. @@ -3165,6 +3405,7 @@ def answer_shipping_query( return result # type: ignore[return-value] @log + @mq def answer_pre_checkout_query( self, pre_checkout_query_id: str, @@ -3172,6 +3413,7 @@ def answer_pre_checkout_query( error_message: str = None, timeout: float = None, api_kwargs: JSONDict = None, + delay_queue: str = None, ) -> bool: """ Once the user has confirmed their payment and shipping details, the Bot API sends the final @@ -3197,6 +3439,8 @@ def answer_pre_checkout_query( the connection pool). api_kwargs (:obj:`dict`, optional): Arbitrary keyword arguments to be passed to the Telegram API. + delay_queue (:obj:`str`, optional): The name of the delay queue to pass this request + through. Defaults to :obj:`None`. Returns: :obj:`bool`: On success, :obj:`True` is returned. @@ -3224,6 +3468,7 @@ def answer_pre_checkout_query( return result # type: ignore[return-value] @log + @mq def restrict_chat_member( self, chat_id: Union[str, int], @@ -3232,6 +3477,7 @@ def restrict_chat_member( until_date: Union[int, datetime] = None, timeout: float = None, api_kwargs: JSONDict = None, + delay_queue: str = None, ) -> bool: """ Use this method to restrict a user in a supergroup. The bot must be an administrator in @@ -3260,6 +3506,8 @@ def restrict_chat_member( the connection pool). api_kwargs (:obj:`dict`, optional): Arbitrary keyword arguments to be passed to the Telegram API. + delay_queue (:obj:`str`, optional): The name of the delay queue to pass this request + through. Defaults to :obj:`None`. Returns: :obj:`bool`: On success, :obj:`True` is returned. @@ -3285,6 +3533,7 @@ def restrict_chat_member( return result # type: ignore[return-value] @log + @mq def promote_chat_member( self, chat_id: Union[str, int], @@ -3299,6 +3548,7 @@ def promote_chat_member( can_promote_members: bool = None, timeout: float = None, api_kwargs: JSONDict = None, + delay_queue: str = None, ) -> bool: """ Use this method to promote or demote a user in a supergroup or a channel. The bot must be @@ -3332,6 +3582,8 @@ def promote_chat_member( the connection pool). api_kwargs (:obj:`dict`, optional): Arbitrary keyword arguments to be passed to the Telegram API. + delay_queue (:obj:`str`, optional): The name of the delay queue to pass this request + through. Defaults to :obj:`None`. Returns: :obj:`bool`: On success, :obj:`True` is returned. @@ -3364,12 +3616,14 @@ def promote_chat_member( return result # type: ignore[return-value] @log + @mq def set_chat_permissions( self, chat_id: Union[str, int], permissions: ChatPermissions, timeout: float = None, api_kwargs: JSONDict = None, + delay_queue: str = None, ) -> bool: """ Use this method to set default chat permissions for all members. The bot must be an @@ -3385,6 +3639,8 @@ def set_chat_permissions( the connection pool). api_kwargs (:obj:`dict`, optional): Arbitrary keyword arguments to be passed to the Telegram API. + delay_queue (:obj:`str`, optional): The name of the delay queue to pass this request + through. Defaults to :obj:`None`. Returns: :obj:`bool`: On success, :obj:`True` is returned. @@ -3400,6 +3656,7 @@ def set_chat_permissions( return result # type: ignore[return-value] @log + @mq def set_chat_administrator_custom_title( self, chat_id: Union[int, str], @@ -3407,6 +3664,7 @@ def set_chat_administrator_custom_title( custom_title: str, timeout: float = None, api_kwargs: JSONDict = None, + delay_queue: str = None, ) -> bool: """ Use this method to set a custom title for administrators promoted by the bot in a @@ -3423,6 +3681,8 @@ def set_chat_administrator_custom_title( the connection pool). api_kwargs (:obj:`dict`, optional): Arbitrary keyword arguments to be passed to the Telegram API. + delay_queue (:obj:`str`, optional): The name of the delay queue to pass this request + through. Defaults to :obj:`None`. Returns: :obj:`bool`: On success, :obj:`True` is returned. @@ -3440,8 +3700,13 @@ def set_chat_administrator_custom_title( return result # type: ignore[return-value] @log + @mq def export_chat_invite_link( - self, chat_id: Union[str, int], timeout: float = None, api_kwargs: JSONDict = None + self, + chat_id: Union[str, int], + timeout: float = None, + api_kwargs: JSONDict = None, + delay_queue: str = None, ) -> str: """ Use this method to generate a new invite link for a chat; any previously generated link @@ -3456,6 +3721,8 @@ def export_chat_invite_link( the connection pool). api_kwargs (:obj:`dict`, optional): Arbitrary keyword arguments to be passed to the Telegram API. + delay_queue (:obj:`str`, optional): The name of the delay queue to pass this request + through. Defaults to :obj:`None`. Returns: :obj:`str`: New invite link on success. @@ -3471,12 +3738,14 @@ def export_chat_invite_link( return result # type: ignore[return-value] @log + @mq def set_chat_photo( self, chat_id: Union[str, int], photo: FileLike, timeout: float = 20, api_kwargs: JSONDict = None, + delay_queue: str = None, ) -> bool: """Use this method to set a new profile photo for the chat. @@ -3492,6 +3761,8 @@ def set_chat_photo( the connection pool). api_kwargs (:obj:`dict`, optional): Arbitrary keyword arguments to be passed to the Telegram API. + delay_queue (:obj:`str`, optional): The name of the delay queue to pass this request + through. Defaults to :obj:`None`. Returns: :obj:`bool`: On success, :obj:`True` is returned. @@ -3511,8 +3782,13 @@ def set_chat_photo( return result # type: ignore[return-value] @log + @mq def delete_chat_photo( - self, chat_id: Union[str, int], timeout: float = None, api_kwargs: JSONDict = None + self, + chat_id: Union[str, int], + timeout: float = None, + api_kwargs: JSONDict = None, + delay_queue: str = None, ) -> bool: """ Use this method to delete a chat photo. Photos can't be changed for private chats. The bot @@ -3527,6 +3803,8 @@ def delete_chat_photo( the connection pool). api_kwargs (:obj:`dict`, optional): Arbitrary keyword arguments to be passed to the Telegram API. + delay_queue (:obj:`str`, optional): The name of the delay queue to pass this request + through. Defaults to :obj:`None`. Returns: :obj:`bool`: On success, :obj:`True` is returned. @@ -3542,12 +3820,14 @@ def delete_chat_photo( return result # type: ignore[return-value] @log + @mq def set_chat_title( self, chat_id: Union[str, int], title: str, timeout: float = None, api_kwargs: JSONDict = None, + delay_queue: str = None, ) -> bool: """ Use this method to change the title of a chat. Titles can't be changed for private chats. @@ -3563,6 +3843,8 @@ def set_chat_title( the connection pool). api_kwargs (:obj:`dict`, optional): Arbitrary keyword arguments to be passed to the Telegram API. + delay_queue (:obj:`str`, optional): The name of the delay queue to pass this request + through. Defaults to :obj:`None`. Returns: :obj:`bool`: On success, :obj:`True` is returned. @@ -3578,12 +3860,14 @@ def set_chat_title( return result # type: ignore[return-value] @log + @mq def set_chat_description( self, chat_id: Union[str, int], description: str, timeout: float = None, api_kwargs: JSONDict = None, + delay_queue: str = None, ) -> bool: """ Use this method to change the description of a group, a supergroup or a channel. The bot @@ -3599,6 +3883,8 @@ def set_chat_description( the connection pool). api_kwargs (:obj:`dict`, optional): Arbitrary keyword arguments to be passed to the Telegram API. + delay_queue (:obj:`str`, optional): The name of the delay queue to pass this request + through. Defaults to :obj:`None`. Returns: :obj:`bool`: On success, :obj:`True` is returned. @@ -3614,6 +3900,7 @@ def set_chat_description( return result # type: ignore[return-value] @log + @mq def pin_chat_message( self, chat_id: Union[str, int], @@ -3621,6 +3908,7 @@ def pin_chat_message( disable_notification: bool = None, timeout: float = None, api_kwargs: JSONDict = None, + delay_queue: str = None, ) -> bool: """ Use this method to pin a message in a group, a supergroup, or a channel. @@ -3640,6 +3928,8 @@ def pin_chat_message( the connection pool). api_kwargs (:obj:`dict`, optional): Arbitrary keyword arguments to be passed to the Telegram API. + delay_queue (:obj:`str`, optional): The name of the delay queue to pass this request + through. Defaults to :obj:`None`. Returns: :obj:`bool`: On success, :obj:`True` is returned. @@ -3658,8 +3948,13 @@ def pin_chat_message( return result # type: ignore[return-value] @log + @mq def unpin_chat_message( - self, chat_id: Union[str, int], timeout: float = None, api_kwargs: JSONDict = None + self, + chat_id: Union[str, int], + timeout: float = None, + api_kwargs: JSONDict = None, + delay_queue: str = None, ) -> bool: """ Use this method to unpin a message in a group, a supergroup, or a channel. @@ -3675,6 +3970,8 @@ def unpin_chat_message( the connection pool). api_kwargs (:obj:`dict`, optional): Arbitrary keyword arguments to be passed to the Telegram API. + delay_queue (:obj:`str`, optional): The name of the delay queue to pass this request + through. Defaults to :obj:`None`. Returns: :obj:`bool`: On success, :obj:`True` is returned. @@ -3690,8 +3987,13 @@ def unpin_chat_message( return result # type: ignore[return-value] @log + @mq def get_sticker_set( - self, name: str, timeout: float = None, api_kwargs: JSONDict = None + self, + name: str, + timeout: float = None, + api_kwargs: JSONDict = None, + delay_queue: str = None, ) -> StickerSet: """Use this method to get a sticker set. @@ -3702,6 +4004,8 @@ def get_sticker_set( creation of the connection pool). api_kwargs (:obj:`dict`, optional): Arbitrary keyword arguments to be passed to the Telegram API. + delay_queue (:obj:`str`, optional): The name of the delay queue to pass this request + through. Defaults to :obj:`None`. Returns: :class:`telegram.StickerSet` @@ -3717,12 +4021,14 @@ def get_sticker_set( return StickerSet.de_json(result, self) # type: ignore @log + @mq def upload_sticker_file( self, user_id: Union[str, int], png_sticker: Union[str, FileLike], timeout: float = 20, api_kwargs: JSONDict = None, + delay_queue: str = None, ) -> File: """ Use this method to upload a .png file with a sticker for later use in @@ -3743,6 +4049,8 @@ def upload_sticker_file( creation of the connection pool). api_kwargs (:obj:`dict`, optional): Arbitrary keyword arguments to be passed to the Telegram API. + delay_queue (:obj:`str`, optional): The name of the delay queue to pass this request + through. Defaults to :obj:`None`. Returns: :class:`telegram.File`: On success, the uploaded File is returned. @@ -3761,6 +4069,7 @@ def upload_sticker_file( return File.de_json(result, self) # type: ignore @log + @mq def create_new_sticker_set( self, user_id: Union[str, int], @@ -3773,6 +4082,7 @@ def create_new_sticker_set( timeout: float = 20, tgs_sticker: Union[str, FileLike] = None, api_kwargs: JSONDict = None, + delay_queue: str = None, ) -> bool: """ Use this method to create new sticker set owned by a user. @@ -3816,6 +4126,8 @@ def create_new_sticker_set( creation of the connection pool). api_kwargs (:obj:`dict`, optional): Arbitrary keyword arguments to be passed to the Telegram API. + delay_queue (:obj:`str`, optional): The name of the delay queue to pass this request + through. Defaults to :obj:`None`. Returns: :obj:`bool`: On success, :obj:`True` is returned. @@ -3848,6 +4160,7 @@ def create_new_sticker_set( return result # type: ignore[return-value] @log + @mq def add_sticker_to_set( self, user_id: Union[str, int], @@ -3858,6 +4171,7 @@ def add_sticker_to_set( timeout: float = 20, tgs_sticker: Union[str, FileLike] = None, api_kwargs: JSONDict = None, + delay_queue: str = None, ) -> bool: """ Use this method to add a new sticker to a set created by the bot. @@ -3895,6 +4209,8 @@ def add_sticker_to_set( creation of the connection pool). api_kwargs (:obj:`dict`, optional): Arbitrary keyword arguments to be passed to the Telegram API. + delay_queue (:obj:`str`, optional): The name of the delay queue to pass this request + through. Defaults to :obj:`None`. Returns: :obj:`bool`: On success, :obj:`True` is returned. @@ -3925,8 +4241,14 @@ def add_sticker_to_set( return result # type: ignore[return-value] @log + @mq def set_sticker_position_in_set( - self, sticker: str, position: int, timeout: float = None, api_kwargs: JSONDict = None + self, + sticker: str, + position: int, + timeout: float = None, + api_kwargs: JSONDict = None, + delay_queue: str = None, ) -> bool: """Use this method to move a sticker in a set created by the bot to a specific position. @@ -3938,6 +4260,8 @@ def set_sticker_position_in_set( creation of the connection pool). api_kwargs (:obj:`dict`, optional): Arbitrary keyword arguments to be passed to the Telegram API. + delay_queue (:obj:`str`, optional): The name of the delay queue to pass this request + through. Defaults to :obj:`None`. Returns: :obj:`bool`: On success, :obj:`True` is returned. @@ -3955,8 +4279,13 @@ def set_sticker_position_in_set( return result # type: ignore[return-value] @log + @mq def delete_sticker_from_set( - self, sticker: str, timeout: float = None, api_kwargs: JSONDict = None + self, + sticker: str, + timeout: float = None, + api_kwargs: JSONDict = None, + delay_queue: str = None, ) -> bool: """Use this method to delete a sticker from a set created by the bot. @@ -3967,6 +4296,8 @@ def delete_sticker_from_set( creation of the connection pool). api_kwargs (:obj:`dict`, optional): Arbitrary keyword arguments to be passed to the Telegram API. + delay_queue (:obj:`str`, optional): The name of the delay queue to pass this request + through. Defaults to :obj:`None`. Returns: :obj:`bool`: On success, :obj:`True` is returned. @@ -3982,6 +4313,7 @@ def delete_sticker_from_set( return result # type: ignore[return-value] @log + @mq def set_sticker_set_thumb( self, name: str, @@ -3989,6 +4321,7 @@ def set_sticker_set_thumb( thumb: FileLike = None, timeout: float = None, api_kwargs: JSONDict = None, + delay_queue: str = None, ) -> bool: """Use this method to set the thumbnail of a sticker set. Animated thumbnails can be set for animated sticker sets only. @@ -4012,6 +4345,8 @@ def set_sticker_set_thumb( creation of the connection pool). api_kwargs (:obj:`dict`, optional): Arbitrary keyword arguments to be passed to the Telegram API. + delay_queue (:obj:`str`, optional): The name of the delay queue to pass this request + through. Defaults to :obj:`None`. Returns: :obj:`bool`: On success, :obj:`True` is returned. @@ -4032,12 +4367,14 @@ def set_sticker_set_thumb( return result # type: ignore[return-value] @log + @mq def set_passport_data_errors( self, user_id: Union[str, int], errors: List[PassportElementError], timeout: float = None, api_kwargs: JSONDict = None, + delay_queue: str = None, ) -> bool: """ Informs a user that some of the Telegram Passport elements they provided contains errors. @@ -4058,6 +4395,8 @@ def set_passport_data_errors( creation of the connection pool). api_kwargs (:obj:`dict`, optional): Arbitrary keyword arguments to be passed to the Telegram API. + delay_queue (:obj:`str`, optional): The name of the delay queue to pass this request + through. Defaults to :obj:`None`. Returns: :obj:`bool`: On success, :obj:`True` is returned. @@ -4073,6 +4412,7 @@ def set_passport_data_errors( return result # type: ignore[return-value] @log + @mq def send_poll( self, chat_id: Union[int, str], @@ -4092,6 +4432,7 @@ def send_poll( open_period: int = None, close_date: Union[int, datetime] = None, api_kwargs: JSONDict = None, + delay_queue: str = None, ) -> Message: """ Use this method to send a native poll. @@ -4137,6 +4478,8 @@ def send_poll( the connection pool). api_kwargs (:obj:`dict`, optional): Arbitrary keyword arguments to be passed to the Telegram API. + delay_queue (:obj:`str`, optional): The name of the delay queue to pass this request + through. Defaults to :obj:`None`. Returns: :class:`telegram.Message`: On success, the sent Message is returned. @@ -4187,6 +4530,7 @@ def send_poll( ) @log + @mq def stop_poll( self, chat_id: Union[int, str], @@ -4194,6 +4538,7 @@ def stop_poll( reply_markup: ReplyMarkup = None, timeout: float = None, api_kwargs: JSONDict = None, + delay_queue: str = None, ) -> Poll: """ Use this method to stop a poll which was sent by the bot. @@ -4209,6 +4554,8 @@ def stop_poll( the connection pool). api_kwargs (:obj:`dict`, optional): Arbitrary keyword arguments to be passed to the Telegram API. + delay_queue (:obj:`str`, optional): The name of the delay queue to pass this request + through. Defaults to :obj:`None`. Returns: :class:`telegram.Poll`: On success, the stopped Poll with the final results is @@ -4233,6 +4580,7 @@ def stop_poll( return Poll.de_json(result, self) # type: ignore @log + @mq def send_dice( self, chat_id: Union[int, str], @@ -4242,6 +4590,7 @@ def send_dice( timeout: float = None, emoji: str = None, api_kwargs: JSONDict = None, + delay_queue: str = None, ) -> Message: """ Use this method to send an animated emoji, which will have a random value. On success, the @@ -4264,6 +4613,8 @@ def send_dice( the connection pool). api_kwargs (:obj:`dict`, optional): Arbitrary keyword arguments to be passed to the Telegram API. + delay_queue (:obj:`str`, optional): The name of the delay queue to pass this request + through. Defaults to :obj:`None`. Returns: :class:`telegram.Message`: On success, the sent Message is returned. @@ -4290,8 +4641,9 @@ def send_dice( ) @log + @mq def get_my_commands( - self, timeout: float = None, api_kwargs: JSONDict = None + self, timeout: float = None, api_kwargs: JSONDict = None, delay_queue: str = None ) -> List[BotCommand]: """ Use this method to get the current list of the bot's commands. @@ -4302,6 +4654,8 @@ def get_my_commands( the connection pool). api_kwargs (:obj:`dict`, optional): Arbitrary keyword arguments to be passed to the Telegram API. + delay_queue (:obj:`str`, optional): The name of the delay queue to pass this request + through. Defaults to :obj:`None`. Returns: List[:class:`telegram.BotCommand]`: On success, the commands set for the bot @@ -4317,11 +4671,13 @@ def get_my_commands( return self._commands @log + @mq def set_my_commands( self, commands: List[Union[BotCommand, Tuple[str, str]]], timeout: float = None, api_kwargs: JSONDict = None, + delay_queue: str = None, ) -> bool: """ Use this method to change the list of the bot's commands. @@ -4335,6 +4691,8 @@ def set_my_commands( the connection pool). api_kwargs (:obj:`dict`, optional): Arbitrary keyword arguments to be passed to the Telegram API. + delay_queue (:obj:`str`, optional): The name of the delay queue to pass this request + through. Defaults to :obj:`None`. Returns: :obj:`True`: On success diff --git a/telegram/ext/defaults.py b/telegram/ext/defaults.py index 6b041db71d6..57a8803d4c3 100644 --- a/telegram/ext/defaults.py +++ b/telegram/ext/defaults.py @@ -17,8 +17,10 @@ # You should have received a copy of the GNU Lesser Public License # along with this program. If not, see [http://www.gnu.org/licenses/]. """This module contains the class Defaults, which allows to pass default values to Updater.""" +from collections import defaultdict + import pytz -from typing import Union, Optional, Any, NoReturn +from typing import Union, Optional, Any, NoReturn, Dict, DefaultDict from telegram.utils.helpers import DEFAULT_NONE, DefaultValue @@ -41,6 +43,12 @@ class Defaults: be ignored. Default: :obj:`True` in group chats and :obj:`False` in private chats. tzinfo (:obj:`tzinfo`): A timezone to be used for all date(time) objects appearing throughout PTB. + delay_queue (:obj:`str`, optional): A :class:`telegram.ext.DelayQueue` the bots + :class:`telegram.ext.MessageQueue` should use. + delay_queue_per_method (Dict[:obj:`str`, :obj:`str`], optional): A dictionary specifying + for each bot method a :class:`telegram.ext.DelayQueue` the bots + :class:`telegram.ext.MessageQueue` should use. Methods not specified here will use + :attr:`delay_queue`. Parameters: parse_mode (:obj:`str`, optional): Send Markdown or HTML, if you want Telegram apps to show @@ -59,6 +67,12 @@ class Defaults: appearing throughout PTB, i.e. if a timezone naive date(time) object is passed somewhere, it will be assumed to be in ``tzinfo``. Must be a timezone provided by the ``pytz`` module. Defaults to UTC. + delay_queue (:obj:`str`, optional): A :class:`telegram.ext.DelayQueue` the bots + :class:`telegram.ext.MessageQueue` should use. Defaults to :obj:`None`. + delay_queue_per_method (Dict[:obj:`str`, :obj:`str`], optional): A dictionary specifying + for each bot method a :class:`telegram.ext.DelayQueue` the bots + :class:`telegram.ext.MessageQueue` should use. Methods not specified here will use + :attr:`delay_queue`. Defaults to :obj:`None`. """ def __init__( @@ -71,6 +85,8 @@ def __init__( timeout: Union[float, DefaultValue] = DEFAULT_NONE, quote: bool = None, tzinfo: pytz.BaseTzInfo = pytz.utc, + delay_queue: str = None, + delay_queue_per_method: Dict[str, Optional[str]] = None, ): self._parse_mode = parse_mode self._disable_notification = disable_notification @@ -78,6 +94,10 @@ def __init__( self._timeout = timeout self._quote = quote self._tzinfo = tzinfo + self._delay_queue = delay_queue + self._delay_queue_per_method = defaultdict( + lambda: self.delay_queue, delay_queue_per_method or {} + ) @property def parse_mode(self) -> Optional[str]: @@ -145,6 +165,28 @@ def tzinfo(self, value: Any) -> NoReturn: "not have any effect." ) + @property + def delay_queue(self) -> Optional[str]: + return self._delay_queue + + @delay_queue.setter + def delay_queue(self, value: Any) -> NoReturn: + raise AttributeError( + "You can not assign a new value to defaults after because it would " + "not have any effect." + ) + + @property + def delay_queue_per_method(self) -> DefaultDict[str, Optional[str]]: + return self._delay_queue_per_method + + @delay_queue_per_method.setter + def delay_queue_per_method(self, value: Any) -> NoReturn: + raise AttributeError( + "You can not assign a new value to defaults after because it would " + "not have any effect." + ) + def __hash__(self) -> int: return hash( ( @@ -154,6 +196,8 @@ def __hash__(self) -> int: self._timeout, self._quote, self._tzinfo, + self._delay_queue, + ((key, value) for key, value in self._delay_queue_per_method), ) ) diff --git a/telegram/ext/messagequeue.py b/telegram/ext/messagequeue.py index d268b18fdb6..faad634e041 100644 --- a/telegram/ext/messagequeue.py +++ b/telegram/ext/messagequeue.py @@ -20,14 +20,15 @@ # You should have received a copy of the GNU Lesser Public License # along with this program. If not, see [http://www.gnu.org/licenses/] """A throughput-limiting message processor for Telegram bots.""" -from telegram.utils import promise import functools import time import threading import queue as q -from typing import Callable, Any, TYPE_CHECKING, List, NoReturn +from typing import Callable, Any, TYPE_CHECKING, List, NoReturn, ClassVar, Dict + +from telegram.utils.promise import Promise if TYPE_CHECKING: from telegram import Bot @@ -107,7 +108,7 @@ def run(self) -> None: times: List[float] = [] # used to store each callable processing time while True: - item = self._queue.get() + promise = self._queue.get() if self.__exit_req: return # shutdown thread # delay routine @@ -124,11 +125,9 @@ def run(self) -> None: if len(times) >= self.burst_limit: # if throughput limit was hit time.sleep(times[1] - t_delta) # finally process one - try: - func, args, kwargs = item - func(*args, **kwargs) - except Exception as exc: # re-route any exceptions - self.exc_route(exc) # to prevent thread exit + promise.run() + if promise.exception: + self.exc_route(promise.exception) # re-route any exceptions def stop(self, timeout: float = None) -> None: """Used to gently stop processor and shutdown its thread. @@ -156,7 +155,7 @@ def _default_exception_handler(exc: Exception) -> NoReturn: raise exc - def __call__(self, func: Callable, *args: Any, **kwargs: Any) -> None: + def process(self, func: Callable, args: Any, kwargs: Any) -> Promise: """Used to process callbacks in throughput-limiting thread through queue. Args: @@ -169,7 +168,9 @@ def __call__(self, func: Callable, *args: Any, **kwargs: Any) -> None: if not self.is_alive() or self.__exit_req: raise DelayQueueError('Could not process callback in stopped thread') - self._queue.put((func, args, kwargs)) + promise = Promise(func, args, kwargs) + self._queue.put(promise) + return promise # The most straightforward way to implement this is to use 2 sequential delay @@ -185,6 +186,9 @@ class MessageQueue: Callables are processed through *group* ``DelayQueue``, then through *all* ``DelayQueue`` for group-type messages. For non-group messages, only the *all* ``DelayQueue`` is used. + Attributes: + running (:obj:`bool`): Whether this message queue has started it's delay queues or not. + Args: all_burst_limit (:obj:`int`, optional): Number of maximum *all-type* callbacks to process per time-window defined by :attr:`all_time_limit_ms`. Defaults to 30. @@ -213,61 +217,48 @@ def __init__( exc_route: Callable[[Exception], None] = None, autostart: bool = True, ): - # create according delay queues, use composition - self._all_delayq = DelayQueue( - burst_limit=all_burst_limit, - time_limit_ms=all_time_limit_ms, - exc_route=exc_route, - autostart=autostart, - ) - self._group_delayq = DelayQueue( - burst_limit=group_burst_limit, - time_limit_ms=group_time_limit_ms, - exc_route=exc_route, - autostart=autostart, - ) + self.running = False + self._delay_queues: Dict[str, DelayQueue] = { + self.DEFAULT_QUEUE: DelayQueue( + burst_limit=all_burst_limit, + time_limit_ms=all_time_limit_ms, + exc_route=exc_route, + autostart=autostart, + ), + self.GROUP_QUEUE: DelayQueue( + burst_limit=group_burst_limit, + time_limit_ms=group_time_limit_ms, + exc_route=exc_route, + autostart=autostart, + ), + } def start(self) -> None: """Method is used to manually start the ``MessageQueue`` processing.""" - self._all_delayq.start() - self._group_delayq.start() + for dq in self._delay_queues.values(): + dq.start() def stop(self, timeout: float = None) -> None: - self._group_delayq.stop(timeout=timeout) - self._all_delayq.stop(timeout=timeout) + for dq in self._delay_queues.values(): + dq.stop() stop.__doc__ = DelayQueue.stop.__doc__ or '' # reuse docstring if any - def __call__(self, promise: Callable, is_group_msg: bool = False) -> Callable: + def process(self, func: Callable, delay_queue: str, *args: Any, **kwargs: Any) -> Promise: """ Processes callables in throughput-limiting queues to avoid hitting limits (specified with :attr:`burst_limit` and :attr:`time_limit`. - Args: - promise (:obj:`callable`): Mainly the ``telegram.utils.promise.Promise`` (see Notes for - other callables), that is processed in delay queues. - is_group_msg (:obj:`bool`, optional): Defines whether ``promise`` would be processed in - group*+*all* ``DelayQueue``s (if set to :obj:`True`), or only through *all* - ``DelayQueue`` (if set to :obj:`False`), resulting in needed delays to avoid - hitting specified limits. Defaults to :obj:`False`. - - Note: - Method is designed to accept ``telegram.utils.promise.Promise`` as ``promise`` - argument, but other callables could be used too. For example, lambdas or simple - functions could be used to wrap original func to be called with needed args. In that - case, be sure that either wrapper func does not raise outside exceptions or the proper - :attr:`exc_route` handler is provided. - Returns: - :obj:`callable`: Used as ``promise`` argument. + :class:`telegram.ext.Promise`. """ + return self._delay_queues[delay_queue].process(func, args, kwargs) - if not is_group_msg: # ignore middle group delay - self._all_delayq(promise) - else: # use middle group delay - self._group_delayq(self._all_delayq, promise) - return promise + DEFAULT_QUEUE: ClassVar[str] = 'default_delay_queue' + """:obj:`str`: The default delay queue.""" + GROUP_QUEUE: ClassVar[str] = 'group_delay_queue' + """:obj:`str`: The default delay queue for group requests.""" def queuedmessage(method: Callable) -> Callable: @@ -307,10 +298,15 @@ def wrapped(self: 'Bot', *args: Any, **kwargs: Any) -> Any: queued = kwargs.pop( 'queued', self._is_messages_queued_default # type: ignore[attr-defined] ) - isgroup = kwargs.pop('isgroup', False) + is_group = kwargs.pop('isgroup', False) if queued: - prom = promise.Promise(method, (self,) + args, kwargs) - return self._msg_queue(prom, isgroup) # type: ignore[attr-defined] + if not is_group: + return self._msg_queue.process( # type: ignore[attr-defined] + method, MessageQueue.DEFAULT_QUEUE, self, *args, **kwargs + ) + return self._msg_queue.process( # type: ignore[attr-defined] + method, MessageQueue.GROUP_QUEUE, self, *args, **kwargs + ) return method(self, *args, **kwargs) return wrapped diff --git a/telegram/utils/promise.py b/telegram/utils/promise.py index 02905ef601b..e875836dbfb 100644 --- a/telegram/utils/promise.py +++ b/telegram/utils/promise.py @@ -20,6 +20,8 @@ import logging from threading import Event + +from telegram import InputFile from telegram.utils.types import JSONDict, HandlerArg from typing import Callable, List, Tuple, Optional, Union, TypeVar @@ -60,9 +62,20 @@ def __init__( update: HandlerArg = None, error_handling: bool = True, ): - self.pooled_function = pooled_function - self.args = args + + parsed_args = [] + for arg in args: + if InputFile.is_file(arg): + parsed_args.append(InputFile(arg)) + else: + parsed_args.append(arg) + self.args = tuple(parsed_args) self.kwargs = kwargs + for key, value in self.kwargs.items(): + if InputFile.is_file(value): + self.kwargs[key] = InputFile(value) + + self.pooled_function = pooled_function self.update = update self.error_handling = error_handling self.done = Event() diff --git a/tests/test_defaults.py b/tests/test_defaults.py index 5344f538d38..2d96684c387 100644 --- a/tests/test_defaults.py +++ b/tests/test_defaults.py @@ -39,6 +39,10 @@ def test_data_assignment(self, cdp): defaults.quote = True with pytest.raises(AttributeError): defaults.tzinfo = True + with pytest.raises(AttributeError): + defaults.delay_queue = True + with pytest.raises(AttributeError): + defaults.delay_queue_per_method = True def test_equality(self): a = Defaults(parse_mode='HTML', quote=True) diff --git a/tests/test_messagequeue.py b/tests/test_messagequeue.py index f9ebfc90159..661d69c4e39 100644 --- a/tests/test_messagequeue.py +++ b/tests/test_messagequeue.py @@ -17,7 +17,6 @@ # You should have received a copy of the GNU Lesser Public License # along with this program. If not, see [http://www.gnu.org/licenses/]. -import os from time import sleep, perf_counter import pytest @@ -26,8 +25,8 @@ @pytest.mark.skipif( - os.getenv('GITHUB_ACTIONS', False) and os.name == 'nt', - reason="On windows precise timings are not accurate.", + True, + reason="Didn't adjust the tests yet.", ) class TestDelayQueue: N = 128 diff --git a/tests/test_official.py b/tests/test_official.py index 39f5864e396..25c853d3622 100644 --- a/tests/test_official.py +++ b/tests/test_official.py @@ -36,6 +36,7 @@ 'timeout', 'bot', 'api_kwargs', + 'delay_queue', } From 315090164473dc587a9592fedc50250b9ff5f326 Mon Sep 17 00:00:00 2001 From: Hinrich Mahler Date: Tue, 27 Oct 2020 17:31:28 +0100 Subject: [PATCH 02/15] fix stuff --- telegram/bot.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/telegram/bot.py b/telegram/bot.py index 5e91a3c9162..feef65a8889 100644 --- a/telegram/bot.py +++ b/telegram/bot.py @@ -126,7 +126,7 @@ def decorator(self: 'Bot', *args: Any, **kwargs: Any) -> RT: def mq(func: Callable[..., RT], *args: Any, **kwargs: Any) -> Callable[..., RT]: - def decorator(self: Union[Callable, Bot], *args: Any, **kwargs: Any) -> RT: + def decorator(self: Union[Callable, 'Bot'], *args: Any, **kwargs: Any) -> RT: if callable(self): self = cast('Bot', args[0]) From e81f315d1a50e09b35f438d97c42c92dd5385503 Mon Sep 17 00:00:00 2001 From: Hinrich Mahler Date: Thu, 29 Oct 2020 19:26:28 +0100 Subject: [PATCH 03/15] add/remove_dq, error handling & some more untested changes --- telegram/bot.py | 4 +- telegram/ext/dispatcher.py | 53 +++++++------ telegram/ext/messagequeue.py | 147 +++++++++++++++++++++++++++-------- 3 files changed, 145 insertions(+), 59 deletions(-) diff --git a/telegram/bot.py b/telegram/bot.py index feef65a8889..cf9e537c437 100644 --- a/telegram/bot.py +++ b/telegram/bot.py @@ -158,11 +158,11 @@ def decorator(self: Union[Callable, 'Bot'], *args: Any, **kwargs: Any) -> RT: is_group = False queue = self.message_queue.GROUP_QUEUE if is_group else delay_queue - return self.message_queue.process( # type: ignore[return-value] + return self.message_queue.put( # type: ignore[return-value] func, queue, self, *args, **kwargs ) - return self.message_queue.process( # type: ignore[return-value] + return self.message_queue.put( # type: ignore[return-value] func, delay_queue, self, *args, **kwargs ) diff --git a/telegram/ext/dispatcher.py b/telegram/ext/dispatcher.py index c7658e41133..4dde2e54d67 100644 --- a/telegram/ext/dispatcher.py +++ b/telegram/ext/dispatcher.py @@ -264,35 +264,38 @@ def _pooled(self) -> None: promise.run() - if not promise.exception: - self.update_persistence(update=promise.update) - continue + self.post_process_promise(promise) - if isinstance(promise.exception, DispatcherHandlerStop): - self.logger.warning( - 'DispatcherHandlerStop is not supported with async functions; func: %s', - promise.pooled_function.__name__, - ) - continue + def post_process_promise(self, promise: Promise) -> None: + if not promise.exception: + self.update_persistence(update=promise.update) + return - # Avoid infinite recursion of error handlers. - if promise.pooled_function in self.error_handlers: - self.logger.error('An uncaught error was raised while handling the error.') - continue + if isinstance(promise.exception, DispatcherHandlerStop): + self.logger.warning( + 'DispatcherHandlerStop is not supported with async functions; func: %s', + promise.pooled_function.__name__, + ) + return - # Don't perform error handling for a `Promise` with deactivated error handling. This - # should happen only via the deprecated `@run_async` decorator or `Promises` created - # within error handlers - if not promise.error_handling: - self.logger.error('A promise with deactivated error handling raised an error.') - continue + # Avoid infinite recursion of error handlers. + if promise.pooled_function in self.error_handlers: + self.logger.error('An uncaught error was raised while handling the error.') + return - # If we arrive here, an exception happened in the promise and was neither - # DispatcherHandlerStop nor raised by an error handler. So we can and must handle it - try: - self.dispatch_error(promise.update, promise.exception, promise=promise) - except Exception: - self.logger.exception('An uncaught error was raised while handling the error.') + # Don't perform error handling for a `Promise` with deactivated error handling. This + # should happen only via the deprecated `@run_async` decorator or `Promises` created + # within error handlers + if not promise.error_handling: + self.logger.error('A promise with deactivated error handling raised an error.') + return + + # If we arrive here, an exception happened in the promise and was neither + # DispatcherHandlerStop nor raised by an error handler. So we can and must handle it + try: + self.dispatch_error(promise.update, promise.exception, promise=promise) + except Exception: + self.logger.exception('An uncaught error was raised while handling the error.') def run_async( self, func: Callable[..., Any], *args: Any, update: HandlerArg = None, **kwargs: Any diff --git a/telegram/ext/messagequeue.py b/telegram/ext/messagequeue.py index faad634e041..59a90bdc7e2 100644 --- a/telegram/ext/messagequeue.py +++ b/telegram/ext/messagequeue.py @@ -26,12 +26,13 @@ import threading import queue as q -from typing import Callable, Any, TYPE_CHECKING, List, NoReturn, ClassVar, Dict +from typing import Callable, Any, TYPE_CHECKING, List, NoReturn, ClassVar, Dict, Optional from telegram.utils.promise import Promise if TYPE_CHECKING: from telegram import Bot + from telegram.ext import Dispatcher # We need to count < 1s intervals, so the most accurate timer is needed curtime = time.perf_counter @@ -55,10 +56,15 @@ class DelayQueue(threading.Thread): exc_route (:obj:`callable`): A callable, accepting 1 positional argument; used to route exceptions from processor thread to main thread; name (:obj:`str`): Thread's name. + dispatcher (:class:`telegram.ext.Disptacher`): Optional. The dispatcher to use for error + handling. Args: queue (:obj:`Queue`, optional): Used to pass callbacks to thread. Creates ``Queue`` implicitly if not provided. + parent (:class:`telegram.ext.DelayQueue`, optional): Pass another delay queue to put all + requests through that delay queue after they were processed by this queue. Defaults to + :obj:`None`. burst_limit (:obj:`int`, optional): Number of maximum callbacks to process per time-window defined by :attr:`time_limit_ms`. Defaults to 30. time_limit_ms (:obj:`int`, optional): Defines width of time-window used when each @@ -66,7 +72,8 @@ class DelayQueue(threading.Thread): exc_route (:obj:`callable`, optional): A callable, accepting 1 positional argument; used to route exceptions from processor thread to main thread; is called on `Exception` subclass exceptions. If not provided, exceptions are routed through dummy handler, - which re-raises them. + which re-raises them. If :attr:`dispatcher` is set, error handling will *always* be + done by the dispatcher. autostart (:obj:`bool`, optional): If :obj:`True`, processor is started immediately after object's creation; if :obj:`False`, should be started manually by `start` method. Defaults to :obj:`True`. @@ -75,7 +82,7 @@ class DelayQueue(threading.Thread): """ - _instcnt = 0 # instance counter + INSTANCE_COUNT: ClassVar[int] = 0 # instance counter def __init__( self, @@ -85,20 +92,34 @@ def __init__( exc_route: Callable[[Exception], None] = None, autostart: bool = True, name: str = None, + parent: 'DelayQueue' = None, ): self._queue = queue if queue is not None else q.Queue() self.burst_limit = burst_limit self.time_limit = time_limit_ms / 1000 - self.exc_route = exc_route if exc_route is not None else self._default_exception_handler + self.exc_route = exc_route if exc_route else self._default_exception_handler + self.parent = parent + self.dispatcher: Optional['Dispatcher'] = None + self.__exit_req = False # flag to gently exit thread - self.__class__._instcnt += 1 + self.__class__.INSTANCE_COUNT += 1 + if name is None: - name = '{}-{}'.format(self.__class__.__name__, self.__class__._instcnt) + name = '{}-{}'.format('DelayQueue', self.INSTANCE_COUNT) super().__init__(name=name) - self.daemon = False + if autostart: # immediately start processing super().start() + def set_dispatcher(self, dispatcher: 'Dispatcher') -> None: + """ + Sets the dispatcher to use for error handling. + + Args: + dispatcher (:class:`telegram.ext.Dispatcher`): The dispatcher. + """ + self.dispatcher = dispatcher + def run(self) -> None: """ Do not use the method except for unthreaded testing purposes, the method normally is @@ -125,9 +146,14 @@ def run(self) -> None: if len(times) >= self.burst_limit: # if throughput limit was hit time.sleep(times[1] - t_delta) # finally process one - promise.run() - if promise.exception: - self.exc_route(promise.exception) # re-route any exceptions + if self.parent: + self.parent.put(promise) + else: + promise.run() + if self.dispatcher: + self.dispatcher.post_process_promise(promise) + elif promise.exception: + self.exc_route(promise.exception) # re-route any exceptions def stop(self, timeout: float = None) -> None: """Used to gently stop processor and shutdown its thread. @@ -155,20 +181,28 @@ def _default_exception_handler(exc: Exception) -> NoReturn: raise exc - def process(self, func: Callable, args: Any, kwargs: Any) -> Promise: - """Used to process callbacks in throughput-limiting thread through queue. + def put( + self, func: Callable = None, args: Any = None, kwargs: Any = None, promise: Promise = None + ) -> Promise: + """Used to process callbacks in throughput-limiting thread through queue. You must either + pass a :class:`telegram.utils.Promise` or all of ``func``, ``args`` and ``kwargs``. Args: - func (:obj:`callable`): The actual function (or any callable) that is processed through - queue. - *args (:obj:`list`): Variable-length `func` arguments. - **kwargs (:obj:`dict`): Arbitrary keyword-arguments to `func`. + func (:obj:`callable`, optional): The actual function (or any callable) that is + processed through queue. + args (:obj:`list`, optional): Variable-length `func` arguments. + kwargs (:obj:`dict`, optional): Arbitrary keyword-arguments to `func`. + promise (:class:`telegram.utils.Promise`, optional): A promise. """ + if not bool(promise) ^ all([func, args, kwargs]): + raise ValueError('You must pass either a promise or all all func, args, kwargs.') if not self.is_alive() or self.__exit_req: raise DelayQueueError('Could not process callback in stopped thread') - promise = Promise(func, args, kwargs) + + if not promise: + promise = Promise(func, args, kwargs) # type: ignore[arg-type] self._queue.put(promise) return promise @@ -188,6 +222,8 @@ class MessageQueue: Attributes: running (:obj:`bool`): Whether this message queue has started it's delay queues or not. + dispatcher (:class:`telegram.ext.Disptacher`): Optional. The Dispatcher to use for error + handling. Args: all_burst_limit (:obj:`int`, optional): Number of maximum *all-type* callbacks to process @@ -218,33 +254,71 @@ def __init__( autostart: bool = True, ): self.running = False + self.dispatcher: Optional['Dispatcher'] = None self._delay_queues: Dict[str, DelayQueue] = { self.DEFAULT_QUEUE: DelayQueue( burst_limit=all_burst_limit, time_limit_ms=all_time_limit_ms, exc_route=exc_route, autostart=autostart, - ), - self.GROUP_QUEUE: DelayQueue( - burst_limit=group_burst_limit, - time_limit_ms=group_time_limit_ms, - exc_route=exc_route, - autostart=autostart, - ), + name=self.DEFAULT_QUEUE, + ) } + self._delay_queues[self.GROUP_QUEUE] = DelayQueue( + burst_limit=group_burst_limit, + time_limit_ms=group_time_limit_ms, + exc_route=exc_route, + autostart=autostart, + name=self.GROUP_QUEUE, + parent=self._delay_queues[self.DEFAULT_QUEUE], + ) + + def add_delay_queue(self, delay_queue: DelayQueue) -> None: + """ + Adds a new :class:`telegram.ext.DelayQueue` to this message queue. If the message queue is + already running, also starts the delay queue. Also takes care of setting the + :class:`telegram.ext.Dispatcher`, if :attr:`dispatcher` is set. + + Args: + delay_queue (:class:`telegram.ext.DelayQueue`): The delay queue to add. + """ + self._delay_queues[delay_queue.name] = delay_queue + if self.dispatcher: + delay_queue.set_dispatcher(self.dispatcher) + if self.running: + delay_queue.start() + + def remove_delay_queue(self, name: str, timeout: float = None) -> None: + """ + Removes the :class:`telegram.ext.DelayQueue` with the given name. If the message queue is + still running, also stops the delay queue. + + Args: + name (:obj:`str`): The name of the delay queue to remove. + timeout (:obj:`float`, optional): The timeout to pass to + :meth:`telegram.ext.DelayQueue.stop`. + """ + delay_queue = self._delay_queues.pop(name) + if self.running: + delay_queue.stop(timeout) def start(self) -> None: - """Method is used to manually start the ``MessageQueue`` processing.""" + """Starts the all :class:`telegram.ext.DelayQueue` registered for this message queue.""" for dq in self._delay_queues.values(): dq.start() def stop(self, timeout: float = None) -> None: - for dq in self._delay_queues.values(): - dq.stop() + """ + Stops the all :class:`telegram.ext.DelayQueue` registered for this message queue. - stop.__doc__ = DelayQueue.stop.__doc__ or '' # reuse docstring if any + Args: + timeout (:obj:`float`, optional): The timeout to pass to + :meth:`telegram.ext.DelayQueue.stop`. + """ + for dq in self._delay_queues.values(): + dq.stop(timeout) - def process(self, func: Callable, delay_queue: str, *args: Any, **kwargs: Any) -> Promise: + def put(self, func: Callable, delay_queue: str, *args: Any, **kwargs: Any) -> Promise: """ Processes callables in throughput-limiting queues to avoid hitting limits (specified with :attr:`burst_limit` and :attr:`time_limit`. @@ -253,7 +327,16 @@ def process(self, func: Callable, delay_queue: str, *args: Any, **kwargs: Any) - :class:`telegram.ext.Promise`. """ - return self._delay_queues[delay_queue].process(func, args, kwargs) + return self._delay_queues[delay_queue].put(func, args, kwargs) + + def set_dispatcher(self, dispatcher: 'Dispatcher') -> None: + """ + Sets the dispatcher to use for error handling. + + Args: + dispatcher (:class:`telegram.ext.Dispatcher`): The dispatcher. + """ + self.dispatcher = dispatcher DEFAULT_QUEUE: ClassVar[str] = 'default_delay_queue' """:obj:`str`: The default delay queue.""" @@ -301,10 +384,10 @@ def wrapped(self: 'Bot', *args: Any, **kwargs: Any) -> Any: is_group = kwargs.pop('isgroup', False) if queued: if not is_group: - return self._msg_queue.process( # type: ignore[attr-defined] + return self._msg_queue.put( # type: ignore[attr-defined] method, MessageQueue.DEFAULT_QUEUE, self, *args, **kwargs ) - return self._msg_queue.process( # type: ignore[attr-defined] + return self._msg_queue.put( # type: ignore[attr-defined] method, MessageQueue.GROUP_QUEUE, self, *args, **kwargs ) return method(self, *args, **kwargs) From 05ef91d13b208dc060583b9ebaf4e3c156ab6751 Mon Sep 17 00:00:00 2001 From: Hinrich Mahler Date: Sun, 1 Nov 2020 16:08:26 +0100 Subject: [PATCH 04/15] Integrate MQ with Updater --- telegram/ext/messagequeue.py | 97 +++++++++++++++++++++--------------- telegram/ext/updater.py | 16 +++++- 2 files changed, 73 insertions(+), 40 deletions(-) diff --git a/telegram/ext/messagequeue.py b/telegram/ext/messagequeue.py index 3d99e682ebc..5fd5b4ae1aa 100644 --- a/telegram/ext/messagequeue.py +++ b/telegram/ext/messagequeue.py @@ -22,21 +22,21 @@ """A throughput-limiting message processor for Telegram bots.""" import functools +import logging import queue as q import threading import time +import warnings from typing import Callable, Any, TYPE_CHECKING, List, NoReturn, ClassVar, Dict, Optional +from telegram.utils.deprecate import TelegramDeprecationWarning from telegram.utils.promise import Promise if TYPE_CHECKING: from telegram import Bot from telegram.ext import Dispatcher -# We need to count < 1s intervals, so the most accurate timer is needed -curtime = time.perf_counter - class DelayQueueError(RuntimeError): """Indicates processing errors.""" @@ -51,9 +51,9 @@ class DelayQueue(threading.Thread): burst_limit (:obj:`int`): Number of maximum callbacks to process per time-window. time_limit (:obj:`int`): Defines width of time-window used when each processing limit is calculated. - exc_route (:obj:`callable`): A callable, accepting 1 positional argument; used to route - exceptions from processor thread to main thread; name (:obj:`str`): Thread's name. + error_handler (:obj:`callable`): Optional. A callable, accepting 1 positional argument. + Used to route exceptions from processor thread to main thread. dispatcher (:class:`telegram.ext.Disptacher`): Optional. The dispatcher to use for error handling. @@ -67,11 +67,12 @@ class DelayQueue(threading.Thread): defined by :attr:`time_limit_ms`. Defaults to 30. time_limit_ms (:obj:`int`, optional): Defines width of time-window used when each processing limit is calculated. Defaults to 1000. - exc_route (:obj:`callable`, optional): A callable, accepting 1 positional argument; used to - route exceptions from processor thread to main thread; is called on `Exception` + error_handler (:obj:`callable`, optional): A callable, accepting 1 positional argument. + Used to route exceptions from processor thread to main thread. Is called on `Exception` subclass exceptions. If not provided, exceptions are routed through dummy handler, which re-raises them. If :attr:`dispatcher` is set, error handling will *always* be done by the dispatcher. + exc_route (:obj:`callable`, optional): Deprecated alias of :attr:`error_handler`. autostart (:obj:`bool`, optional): If :obj:`True`, processor is started immediately after object's creation; if :obj:`False`, should be started manually by `start` method. Defaults to :obj:`True`. @@ -91,14 +92,25 @@ def __init__( autostart: bool = True, name: str = None, parent: 'DelayQueue' = None, + error_handler: Callable[[Exception], None] = None, ): + self.logger = logging.getLogger(__name__) self._queue = queue if queue is not None else q.Queue() self.burst_limit = burst_limit self.time_limit = time_limit_ms / 1000 - self.exc_route = exc_route if exc_route else self._default_exception_handler self.parent = parent self.dispatcher: Optional['Dispatcher'] = None + if not (bool(exc_route) ^ bool(error_handler)): # pylint: disable=C0325 + raise RuntimeError('Only one of exc_route or error_handler can be passed.') + if exc_route: + warnings.warn( + 'The exc_route argument is deprecated. Use error_handler instead.', + TelegramDeprecationWarning, + stacklevel=2, + ) + self.exc_route = exc_route or error_handler or self._default_exception_handler + self.__exit_req = False # flag to gently exit thread self.__class__.INSTANCE_COUNT += 1 @@ -119,20 +131,17 @@ def set_dispatcher(self, dispatcher: 'Dispatcher') -> None: self.dispatcher = dispatcher def run(self) -> None: - """ - Do not use the method except for unthreaded testing purposes, the method normally is - automatically called by autostart argument. - - """ - times: List[float] = [] # used to store each callable processing time + while True: promise = self._queue.get() if self.__exit_req: return # shutdown thread + # delay routine now = time.perf_counter() t_delta = now - self.time_limit # calculate early to improve perf. + if times and t_delta > times[-1]: # if last call was before the limit time-window # used to impr. perf. in long-interval calls case @@ -143,11 +152,14 @@ def run(self) -> None: times.append(now) if len(times) >= self.burst_limit: # if throughput limit was hit time.sleep(times[1] - t_delta) + # finally process one if self.parent: + # put through parent, if specified self.parent.put(promise) else: promise.run() + # error handling if self.dispatcher: self.dispatcher.post_process_promise(promise) elif promise.exception: @@ -167,16 +179,12 @@ def stop(self, timeout: float = None) -> None: self.__exit_req = True # gently request self._queue.put(None) # put something to unfreeze if frozen + self.logger.debug('Waiting for DelayQueue %s to shut down.', self.name) super().join(timeout=timeout) + self.logger.debug('DelayQueue %s shut down.', self.name) @staticmethod def _default_exception_handler(exc: Exception) -> NoReturn: - """ - Dummy exception handler which re-raises exception in thread. Could be possibly overwritten - by subclasses. - - """ - raise exc def put( @@ -205,18 +213,11 @@ def put( return promise -# The most straightforward way to implement this is to use 2 sequential delay -# queues, like on classic delay chain schematics in electronics. -# So, message path is: -# msg --> group delay if group msg, else no delay --> normal msg delay --> out -# This way OS threading scheduler cares of timings accuracy. -# (see time.time, time.clock, time.perf_counter, time.sleep @ docs.python.org) class MessageQueue: """ Implements callback processing with proper delays to avoid hitting Telegram's message limits. - Contains two ``DelayQueue``, for group and for all messages, interconnected in delay chain. - Callables are processed through *group* ``DelayQueue``, then through *all* ``DelayQueue`` for - group-type messages. For non-group messages, only the *all* ``DelayQueue`` is used. + By default contains two :class:`telegram.ext.DelayQueue` instances, for general requests and + group requests where the default delay queue is the parent of the group requests one. Attributes: running (:obj:`bool`): Whether this message queue has started it's delay queues or not. @@ -232,13 +233,14 @@ class MessageQueue: process per time-window defined by :attr:`group_time_limit_ms`. Defaults to 20. group_time_limit_ms (:obj:`int`, optional): Defines width of *group-type* time-window used when each processing limit is calculated. Defaults to 60000 ms. - exc_route (:obj:`callable`, optional): A callable, accepting one positional argument; used - to route exceptions from processor threads to main thread; is called on ``Exception`` + error_handler (:obj:`callable`, optional): A callable, accepting 1 positional argument. + Used to route exceptions from processor thread to main thread. Is called on `Exception` subclass exceptions. If not provided, exceptions are routed through dummy handler, - which re-raises them. - autostart (:obj:`bool`, optional): If :obj:`True`, processors are started immediately after - object's creation; if :obj:`False`, should be started manually by :attr:`start` method. - Defaults to :obj:`True`. + which re-raises them. If :attr:`dispatcher` is set, error handling will *always* be + done by the dispatcher. + exc_route (:obj:`callable`, optional): Deprecated alias of :attr:`error_handler`. + autostart (:obj:`bool`, optional): If :obj:`True`, both default delay queues are started + immediately after object's creation. Defaults to :obj:`True`. """ @@ -250,14 +252,26 @@ def __init__( group_time_limit_ms: int = 60000, exc_route: Callable[[Exception], None] = None, autostart: bool = True, + error_handler: Callable[[Exception], None] = None, ): + self.logger = logging.getLogger(__name__) self.running = False self.dispatcher: Optional['Dispatcher'] = None + + if not (bool(exc_route) ^ bool(error_handler)): # pylint: disable=C0325 + raise RuntimeError('Only one of exc_route or error_handler can be passed.') + if exc_route: + warnings.warn( + 'The exc_route argument is deprecated. Use error_handler instead.', + TelegramDeprecationWarning, + stacklevel=2, + ) + self._delay_queues: Dict[str, DelayQueue] = { self.DEFAULT_QUEUE: DelayQueue( burst_limit=all_burst_limit, time_limit_ms=all_time_limit_ms, - exc_route=exc_route, + error_handler=exc_route or error_handler, autostart=autostart, name=self.DEFAULT_QUEUE, ) @@ -265,7 +279,7 @@ def __init__( self._delay_queues[self.GROUP_QUEUE] = DelayQueue( burst_limit=group_burst_limit, time_limit_ms=group_time_limit_ms, - exc_route=exc_route, + error_handler=exc_route or error_handler, autostart=autostart, name=self.GROUP_QUEUE, parent=self._delay_queues[self.DEFAULT_QUEUE], @@ -318,8 +332,13 @@ def stop(self, timeout: float = None) -> None: def put(self, func: Callable, delay_queue: str, *args: Any, **kwargs: Any) -> Promise: """ - Processes callables in throughput-limiting queues to avoid hitting limits (specified with - :attr:`burst_limit` and :attr:`time_limit`. + Processes callables in throughput-limiting queues to avoid hitting limits. + + Args: + func (:obj:`callable`): The callable to process + delay_queue (:obj:`str`): The name of the :class:`telegram.ext.DelayQueue` to use. + *args (:obj:`tuple`, optional): Arguments to ``func``. + **kwargs (:obj:`dict`, optional): Keyword arguments to ``func``. Returns: :class:`telegram.ext.Promise`. diff --git a/telegram/ext/updater.py b/telegram/ext/updater.py index 365caaa904c..a9d0e412f79 100644 --- a/telegram/ext/updater.py +++ b/telegram/ext/updater.py @@ -29,7 +29,7 @@ from telegram import Bot, TelegramError from telegram.error import InvalidToken, RetryAfter, TimedOut, Unauthorized -from telegram.ext import Dispatcher, JobQueue +from telegram.ext import Dispatcher, JobQueue, MessageQueue from telegram.utils.deprecate import TelegramDeprecationWarning from telegram.utils.helpers import get_signal_name from telegram.utils.request import Request @@ -94,6 +94,8 @@ class Updater: used). defaults (:class:`telegram.ext.Defaults`, optional): An object containing default values to be used if not set explicitly in the bot methods. + message_queue (:class:`telegram.ext.MessageQueue`, optional): A message queue to use with + the bot. Will be started automatically and the dispatcher will be set. Note: * You must supply either a :attr:`bot` or a :attr:`token` argument. @@ -122,6 +124,7 @@ def __init__( use_context: bool = True, dispatcher: Dispatcher = None, base_file_url: str = None, + message_queue: MessageQueue = None, ): if defaults and bot: @@ -181,6 +184,7 @@ def __init__( private_key=private_key, private_key_password=private_key_password, defaults=defaults, + message_queue=message_queue, ) self.update_queue: Queue = Queue() self.job_queue = JobQueue() @@ -196,6 +200,10 @@ def __init__( use_context=use_context, ) self.job_queue.set_dispatcher(self.dispatcher) + if self.bot.message_queue: + self.bot.message_queue.set_dispatcher(self.dispatcher) + if not self.bot.message_queue.running: + self.bot.message_queue.start() else: con_pool_size = dispatcher.workers + 4 @@ -634,6 +642,7 @@ def stop(self) -> None: self.running = False self._stop_httpd() + self._stop_message_queue() self._stop_dispatcher() self._join_threads() @@ -657,6 +666,11 @@ def _stop_dispatcher(self) -> None: self.logger.debug('Requesting Dispatcher to stop...') self.dispatcher.stop() + def _stop_message_queue(self) -> None: + if self.bot.message_queue: + self.logger.debug('Requesting MessageQueue to stop...') + self.bot.message_queue.stop() + @no_type_check def _join_threads(self) -> None: for thr in self.__threads: From ed86882869bdc0ae9068b5c8610d6c4e137afd27 Mon Sep 17 00:00:00 2001 From: Hinrich Mahler Date: Mon, 23 Nov 2020 00:37:37 +0100 Subject: [PATCH 05/15] Tests for MQ and DQ --- telegram/ext/__init__.py | 4 +- telegram/ext/messagequeue.py | 18 +-- tests/test_messagequeue.py | 279 +++++++++++++++++++++++++++++++---- 3 files changed, 258 insertions(+), 43 deletions(-) diff --git a/telegram/ext/__init__.py b/telegram/ext/__init__.py index b614e292c74..58a4d238a80 100644 --- a/telegram/ext/__init__.py +++ b/telegram/ext/__init__.py @@ -18,6 +18,7 @@ # along with this program. If not, see [http://www.gnu.org/licenses/]. """Extensions over the Telegram Bot API to facilitate bot making""" +from .messagequeue import MessageQueue, DelayQueue, DelayQueueError from .basepersistence import BasePersistence from .picklepersistence import PicklePersistence from .dictpersistence import DictPersistence @@ -39,8 +40,6 @@ from .conversationhandler import ConversationHandler from .precheckoutqueryhandler import PreCheckoutQueryHandler from .shippingqueryhandler import ShippingQueryHandler -from .messagequeue import MessageQueue -from .messagequeue import DelayQueue from .pollanswerhandler import PollAnswerHandler from .pollhandler import PollHandler from .defaults import Defaults @@ -69,6 +68,7 @@ 'ShippingQueryHandler', 'MessageQueue', 'DelayQueue', + 'DelayQueueError', 'DispatcherHandlerStop', 'run_async', 'CallbackContext', diff --git a/telegram/ext/messagequeue.py b/telegram/ext/messagequeue.py index 5fd5b4ae1aa..34b1f054fb4 100644 --- a/telegram/ext/messagequeue.py +++ b/telegram/ext/messagequeue.py @@ -101,8 +101,8 @@ def __init__( self.parent = parent self.dispatcher: Optional['Dispatcher'] = None - if not (bool(exc_route) ^ bool(error_handler)): # pylint: disable=C0325 - raise RuntimeError('Only one of exc_route or error_handler can be passed.') + if exc_route and error_handler: + raise ValueError('Only one of exc_route or error_handler can be passed.') if exc_route: warnings.warn( 'The exc_route argument is deprecated. Use error_handler instead.', @@ -201,7 +201,7 @@ def put( promise (:class:`telegram.utils.Promise`, optional): A promise. """ - if not bool(promise) ^ all([func, args, kwargs]): + if not bool(promise) ^ all(v is not None for v in [func, args, kwargs]): raise ValueError('You must pass either a promise or all all func, args, kwargs.') if not self.is_alive() or self.__exit_req: @@ -254,12 +254,11 @@ def __init__( autostart: bool = True, error_handler: Callable[[Exception], None] = None, ): - self.logger = logging.getLogger(__name__) - self.running = False + self.running = autostart self.dispatcher: Optional['Dispatcher'] = None - if not (bool(exc_route) ^ bool(error_handler)): # pylint: disable=C0325 - raise RuntimeError('Only one of exc_route or error_handler can be passed.') + if exc_route and error_handler: + raise ValueError('Only one of exc_route or error_handler can be passed.') if exc_route: warnings.warn( 'The exc_route argument is deprecated. Use error_handler instead.', @@ -297,7 +296,7 @@ def add_delay_queue(self, delay_queue: DelayQueue) -> None: self._delay_queues[delay_queue.name] = delay_queue if self.dispatcher: delay_queue.set_dispatcher(self.dispatcher) - if self.running: + if self.running and not delay_queue.is_alive(): delay_queue.start() def remove_delay_queue(self, name: str, timeout: float = None) -> None: @@ -311,13 +310,14 @@ def remove_delay_queue(self, name: str, timeout: float = None) -> None: :meth:`telegram.ext.DelayQueue.stop`. """ delay_queue = self._delay_queues.pop(name) - if self.running: + if self.running and delay_queue.is_alive(): delay_queue.stop(timeout) def start(self) -> None: """Starts the all :class:`telegram.ext.DelayQueue` registered for this message queue.""" for delay_queue in self._delay_queues.values(): delay_queue.start() + self.running = True def stop(self, timeout: float = None) -> None: """ diff --git a/tests/test_messagequeue.py b/tests/test_messagequeue.py index 661d69c4e39..5407b5abf3a 100644 --- a/tests/test_messagequeue.py +++ b/tests/test_messagequeue.py @@ -16,54 +16,269 @@ # # You should have received a copy of the GNU Lesser Public License # along with this program. If not, see [http://www.gnu.org/licenses/]. - from time import sleep, perf_counter import pytest -import telegram.ext.messagequeue as mq +from telegram.ext import MessageQueue, DelayQueue, DelayQueueError -@pytest.mark.skipif( - True, - reason="Didn't adjust the tests yet.", -) class TestDelayQueue: N = 128 burst_limit = 30 time_limit_ms = 1000 margin_ms = 0 - testtimes = [] + test_times = [] + test_flag = None + + @pytest.fixture(autouse=True) + def reset(self): + DelayQueue.INSTANCE_COUNT = 0 + self.test_flag = None def call(self): - self.testtimes.append(perf_counter()) + self.test_times.append(perf_counter()) + + def callback_raises_exception(self): + raise DelayQueueError('TestError') + + def test_auto_start_false(self): + delay_queue = DelayQueue(autostart=False) + assert not delay_queue.is_alive() + + def test_name(self): + delay_queue = DelayQueue(autostart=False) + assert delay_queue.name == 'DelayQueue-1' + delay_queue = DelayQueue(autostart=False) + assert delay_queue.name == 'DelayQueue-2' + delay_queue = DelayQueue(name='test_queue', autostart=False) + assert delay_queue.name == 'test_queue' + + def test_exc_route_deprecation(self, recwarn): + with pytest.raises(ValueError, match='Only one of exc_route or '): + DelayQueue(exc_route=True, error_handler=True, autostart=False) - def test_delayqueue_limits(self): - dsp = mq.DelayQueue( + DelayQueue(exc_route=True, autostart=False) + assert len(recwarn) == 1 + assert str(recwarn[0].message).startswith('The exc_route argument is') + + def test_delay_queue_limits(self): + delay_queue = DelayQueue( burst_limit=self.burst_limit, time_limit_ms=self.time_limit_ms, autostart=True ) - assert dsp.is_alive() is True + assert delay_queue.is_alive() is True + + try: + for _ in range(self.N): + delay_queue.put(self.call, [], {}) + + start_time = perf_counter() + # wait up to 20 sec more than needed + app_end_time = ( + (self.N * self.burst_limit / (1000 * self.time_limit_ms)) + start_time + 20 + ) + while not delay_queue._queue.empty() and perf_counter() < app_end_time: + sleep(1) + assert delay_queue._queue.empty() is True # check loop exit condition + + delay_queue.stop() + assert delay_queue.is_alive() is False + + assert self.test_times or self.N == 0 + passes, fails = [], [] + delta = (self.time_limit_ms - self.margin_ms) / 1000 + for start, stop in enumerate(range(self.burst_limit + 1, len(self.test_times))): + part = self.test_times[start:stop] + if (part[-1] - part[0]) >= delta: + passes.append(part) + else: + fails.append(part) + assert fails == [] + finally: + delay_queue.stop() + + def test_put_errors(self): + delay_queue = DelayQueue(autostart=False) + with pytest.raises(DelayQueueError, match='stopped thread'): + delay_queue.put(promise=True) + + delay_queue.start() + try: + with pytest.raises(ValueError, match='You must pass either'): + delay_queue.put() + with pytest.raises(ValueError, match='You must pass either'): + delay_queue.put(promise=True, args=True, kwargs=True, func=True) + with pytest.raises(ValueError, match='You must pass either'): + delay_queue.put(args=True) + finally: + delay_queue.stop() + + def test_default_error_handler_without_dispatcher(self, monkeypatch): + @staticmethod + def exc_route(exception): + self.test_flag = ( + isinstance(exception, DelayQueueError) and str(exception) == 'TestError' + ) + + monkeypatch.setattr(DelayQueue, '_default_exception_handler', exc_route) + + delay_queue = DelayQueue() + try: + delay_queue.put(self.callback_raises_exception, [], {}) + sleep(1) + assert self.test_flag + finally: + delay_queue.stop() + + def test_custom_error_handler_without_dispatcher(self): + def exc_route(exception): + self.test_flag = ( + isinstance(exception, DelayQueueError) and str(exception) == 'TestError' + ) + + delay_queue = DelayQueue(exc_route=exc_route) + try: + delay_queue.put(self.callback_raises_exception, [], {}) + sleep(1) + assert self.test_flag + finally: + delay_queue.stop() + + def test_custom_error_handler_with_dispatcher(self, cdp): + def error_handler(_, context): + self.test_flag = ( + isinstance(context.error, DelayQueueError) and str(context.error) == 'TestError' + ) + + cdp.add_error_handler(error_handler) + delay_queue = DelayQueue() + delay_queue.set_dispatcher(cdp) + try: + delay_queue.put(self.callback_raises_exception, [], {}) + sleep(1) + assert self.test_flag + finally: + delay_queue.stop() + + def test_parent(self, monkeypatch): + def put(*args, **kwargs): + self.test_flag = True + + parent = DelayQueue(name='parent') + monkeypatch.setattr(parent, 'put', put) + + delay_queue = DelayQueue(parent=parent) + try: + delay_queue.put(self.call, [], {}) + sleep(1) + assert self.test_flag + finally: + parent.stop() + delay_queue.stop() + + +class TestMessageQueue: + test_flag = None + + @pytest.fixture(autouse=True) + def reset(self): + DelayQueue.INSTANCE_COUNT = 0 + self.test_flag = None + + def call(self, arg, kwarg=None): + self.test_flag = arg == 1 and kwarg == 'foo' + + def callback_raises_exception(self): + raise DelayQueueError('TestError') + + def test_auto_start_false(self): + message_queue = MessageQueue(autostart=False) + assert not any(thread.is_alive() for thread in message_queue._delay_queues.values()) + + def test_exc_route_deprecation(self, recwarn): + with pytest.raises(ValueError, match='Only one of exc_route or '): + MessageQueue(exc_route=True, error_handler=True, autostart=False) + + MessageQueue(exc_route=True, autostart=False) + assert len(recwarn) == 1 + assert str(recwarn[0].message).startswith('The exc_route argument is') + + def test_add_delay_queue_autostart_false(self): + message_queue = MessageQueue(autostart=False) + delay_queue = DelayQueue(autostart=False, name='dq') + try: + message_queue.add_delay_queue(delay_queue) + assert 'dq' in message_queue._delay_queues + assert not any(thread.is_alive() for thread in message_queue._delay_queues.values()) + message_queue.start() + assert all(thread.is_alive() for thread in message_queue._delay_queues.values()) + + message_queue.stop() + assert not any(thread.is_alive() for thread in message_queue._delay_queues.values()) + finally: + delay_queue.stop() + message_queue.stop() + + @pytest.mark.parametrize('autostart', [True, False]) + def test_add_delay_queue_autostart_true(self, autostart): + message_queue = MessageQueue() + delay_queue = DelayQueue(name='dq', autostart=autostart) + try: + message_queue.add_delay_queue(delay_queue) + assert 'dq' in message_queue._delay_queues + assert delay_queue.is_alive() + assert all(thread.is_alive() for thread in message_queue._delay_queues.values()) + + message_queue.stop() + assert not any(thread.is_alive() for thread in message_queue._delay_queues.values()) + finally: + delay_queue.stop() + message_queue.stop() + + def test_add_delay_queue_dispatcher(self, dp): + message_queue = MessageQueue(autostart=False) + message_queue.set_dispatcher(dispatcher=dp) + delay_queue = DelayQueue(autostart=False, name='dq') + message_queue.add_delay_queue(delay_queue) + assert delay_queue.dispatcher is dp + + @pytest.mark.parametrize('autostart', [True, False]) + def test_remove_delay_queue(self, autostart): + message_queue = MessageQueue(autostart=autostart) + delay_queue = DelayQueue(name='dq') + try: + message_queue.add_delay_queue(delay_queue) + assert 'dq' in message_queue._delay_queues + assert delay_queue.is_alive() + + message_queue.remove_delay_queue('dq') + assert 'dq' not in message_queue._delay_queues + if autostart: + assert not delay_queue.is_alive() + finally: + delay_queue.stop() + if autostart: + message_queue.stop() + + def test_put(self): + group_flag = None + + message_queue = MessageQueue() + original_put = message_queue._delay_queues[MessageQueue.DEFAULT_QUEUE].put + + def put(*args, **kwargs): + nonlocal group_flag + group_flag = True + return original_put(*args, **kwargs) - for _ in range(self.N): - dsp(self.call) + message_queue._delay_queues[MessageQueue.GROUP_QUEUE].put = put - starttime = perf_counter() - # wait up to 20 sec more than needed - app_endtime = (self.N * self.burst_limit / (1000 * self.time_limit_ms)) + starttime + 20 - while not dsp._queue.empty() and perf_counter() < app_endtime: + try: + message_queue.put(self.call, MessageQueue.GROUP_QUEUE, 1, kwarg='foo') sleep(1) - assert dsp._queue.empty() is True # check loop exit condition - - dsp.stop() - assert dsp.is_alive() is False - - assert self.testtimes or self.N == 0 - passes, fails = [], [] - delta = (self.time_limit_ms - self.margin_ms) / 1000 - for start, stop in enumerate(range(self.burst_limit + 1, len(self.testtimes))): - part = self.testtimes[start:stop] - if (part[-1] - part[0]) >= delta: - passes.append(part) - else: - fails.append(part) - assert fails == [] + assert self.test_flag is True + # make sure that group queue was called, too + assert group_flag is True + finally: + message_queue._delay_queues[MessageQueue.GROUP_QUEUE].put = original_put + message_queue.stop() From ada030a90a6a3dabe37e3031009fcfcbf2a2f3ff Mon Sep 17 00:00:00 2001 From: Hinrich Mahler Date: Mon, 23 Nov 2020 16:31:08 +0100 Subject: [PATCH 06/15] Add deprecation warning to decorator --- telegram/ext/messagequeue.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/telegram/ext/messagequeue.py b/telegram/ext/messagequeue.py index 34b1f054fb4..3af87f41991 100644 --- a/telegram/ext/messagequeue.py +++ b/telegram/ext/messagequeue.py @@ -395,6 +395,13 @@ def queuedmessage(method: Callable) -> Callable: @functools.wraps(method) def wrapped(self: 'Bot', *args: Any, **kwargs: Any) -> Any: + warnings.warn( + 'The @queuedmessage decorator is deprecated. Use the `delay_queue` parameter of' + 'the various bot methods instead.', + TelegramDeprecationWarning, + stacklevel=2, + ) + # pylint: disable=W0212 queued = kwargs.pop( 'queued', self._is_messages_queued_default # type: ignore[attr-defined] From d571d70c1e179ea1695e54310fd7ba9eeb557d82 Mon Sep 17 00:00:00 2001 From: Hinrich Mahler Date: Tue, 24 Nov 2020 20:16:26 +0100 Subject: [PATCH 07/15] tests for MQ-decorator --- telegram/ext/messagequeue.py | 4 +- tests/test_messagequeue.py | 83 +++++++++++++++++++++++++++++++++--- 2 files changed, 78 insertions(+), 9 deletions(-) diff --git a/telegram/ext/messagequeue.py b/telegram/ext/messagequeue.py index 6f721d2aa09..2f89c1cab18 100644 --- a/telegram/ext/messagequeue.py +++ b/telegram/ext/messagequeue.py @@ -115,7 +115,7 @@ def __init__( self.__class__.INSTANCE_COUNT += 1 if name is None: - name = f'{self.__class__.__name__}-{self.__class__._instcnt}' + name = f'{self.__class__.__name__}-{self.__class__.INSTANCE_COUNT}' super().__init__(name=name) if autostart: # immediately start processing @@ -156,7 +156,7 @@ def run(self) -> None: # finally process one if self.parent: # put through parent, if specified - self.parent.put(promise) + self.parent.put(promise=promise) else: promise.run() # error handling diff --git a/tests/test_messagequeue.py b/tests/test_messagequeue.py index 5407b5abf3a..a1fdde8834b 100644 --- a/tests/test_messagequeue.py +++ b/tests/test_messagequeue.py @@ -20,7 +20,9 @@ import pytest +from telegram import Bot from telegram.ext import MessageQueue, DelayQueue, DelayQueueError +from telegram.ext.messagequeue import queuedmessage class TestDelayQueue: @@ -78,7 +80,7 @@ def test_delay_queue_limits(self): (self.N * self.burst_limit / (1000 * self.time_limit_ms)) + start_time + 20 ) while not delay_queue._queue.empty() and perf_counter() < app_end_time: - sleep(1) + sleep(0.5) assert delay_queue._queue.empty() is True # check loop exit condition delay_queue.stop() @@ -125,7 +127,7 @@ def exc_route(exception): delay_queue = DelayQueue() try: delay_queue.put(self.callback_raises_exception, [], {}) - sleep(1) + sleep(0.5) assert self.test_flag finally: delay_queue.stop() @@ -139,7 +141,7 @@ def exc_route(exception): delay_queue = DelayQueue(exc_route=exc_route) try: delay_queue.put(self.callback_raises_exception, [], {}) - sleep(1) + sleep(0.5) assert self.test_flag finally: delay_queue.stop() @@ -155,14 +157,14 @@ def error_handler(_, context): delay_queue.set_dispatcher(cdp) try: delay_queue.put(self.callback_raises_exception, [], {}) - sleep(1) + sleep(0.5) assert self.test_flag finally: delay_queue.stop() def test_parent(self, monkeypatch): def put(*args, **kwargs): - self.test_flag = True + self.test_flag = bool(kwargs.pop('promise', False)) parent = DelayQueue(name='parent') monkeypatch.setattr(parent, 'put', put) @@ -170,7 +172,7 @@ def put(*args, **kwargs): delay_queue = DelayQueue(parent=parent) try: delay_queue.put(self.call, [], {}) - sleep(1) + sleep(0.5) assert self.test_flag finally: parent.stop() @@ -275,10 +277,77 @@ def put(*args, **kwargs): try: message_queue.put(self.call, MessageQueue.GROUP_QUEUE, 1, kwarg='foo') - sleep(1) + sleep(0.5) assert self.test_flag is True # make sure that group queue was called, too assert group_flag is True finally: message_queue._delay_queues[MessageQueue.GROUP_QUEUE].put = original_put message_queue.stop() + + +@pytest.fixture(scope='function') +def mq_bot(bot, monkeypatch): + class MQBot(Bot): + def __init__(self, *args, **kwargs): + self.test = None + self.default_count = 0 + self.group_count = 0 + super().__init__(*args, **kwargs) + # below 2 attributes should be provided for decorator usage + self._is_messages_queued_default = True + self._msg_queue = MessageQueue() + + @queuedmessage + def test_method(self, input, *args, **kwargs): + self.test = input + + bot = MQBot(token=bot.token) + + orig_default_put = bot._msg_queue._delay_queues[MessageQueue.DEFAULT_QUEUE].put + orig_group_put = bot._msg_queue._delay_queues[MessageQueue.GROUP_QUEUE].put + + def step_default_counter(*args, **kwargs): + orig_default_put(*args, **kwargs) + bot.default_count += 1 + + def step_group_counter(*args, **kwargs): + orig_group_put(*args, **kwargs) + bot.group_count += 1 + + monkeypatch.setattr( + bot._msg_queue._delay_queues[MessageQueue.DEFAULT_QUEUE], 'put', step_default_counter + ) + monkeypatch.setattr( + bot._msg_queue._delay_queues[MessageQueue.GROUP_QUEUE], 'put', step_group_counter + ) + yield bot + bot._msg_queue.stop() + + +class TestDecorator: + def test_queued_kwarg(self, mq_bot): + mq_bot.test_method('received', queued=False) + sleep(0.5) + assert mq_bot.default_count == 0 + assert mq_bot.group_count == 0 + assert mq_bot.test == 'received' + + mq_bot.test_method('received1') + sleep(0.5) + assert mq_bot.default_count == 1 + assert mq_bot.group_count == 0 + assert mq_bot.test == 'received1' + + def test_isgroup_kwarg(self, mq_bot): + mq_bot.test_method('received', isgroup=False) + sleep(0.5) + assert mq_bot.default_count == 1 + assert mq_bot.group_count == 0 + assert mq_bot.test == 'received' + + mq_bot.test_method('received1', isgroup=True) + sleep(0.5) + assert mq_bot.default_count == 2 + assert mq_bot.group_count == 1 + assert mq_bot.test == 'received1' From 0140ad7c6e5063e878bfc829e7f491e95538b78d Mon Sep 17 00:00:00 2001 From: Hinrich Mahler Date: Thu, 26 Nov 2020 18:53:21 +0100 Subject: [PATCH 08/15] Get @mq to work & tests --- telegram/bot.py | 40 ++++++++++-------- telegram/ext/messagequeue.py | 1 + tests/test_bot.py | 81 ++++++++++++++++++++++++++++++++++++ 3 files changed, 105 insertions(+), 17 deletions(-) diff --git a/telegram/bot.py b/telegram/bot.py index 41a06cb5602..dd1c79938ba 100644 --- a/telegram/bot.py +++ b/telegram/bot.py @@ -114,12 +114,10 @@ def decorator(self: 'Bot', *args: Any, **kwargs: Any) -> RT: return decorator -def log( - func: Callable[..., RT], *args: Any, **kwargs: Any # pylint: disable=W0613 -) -> Callable[..., RT]: +def log(func: Callable[..., RT]) -> Callable[..., RT]: logger = logging.getLogger(func.__module__) - def decorator(self: 'Bot', *args: Any, **kwargs: Any) -> RT: # pylint: disable=W0613 + def decorator(_: Callable, *args: Any, **kwargs: Any) -> RT: # pylint: disable=W0613 logger.debug('Entering: %s', func.__name__) result = func(*args, **kwargs) logger.debug(result) @@ -129,27 +127,29 @@ def decorator(self: 'Bot', *args: Any, **kwargs: Any) -> RT: # pylint: disable= return decorate(func, decorator) -def mq( - func: Callable[..., RT], *args: Any, **kwargs: Any # pylint: disable=W0613 -) -> Callable[..., RT]: - def decorator(self: Union[Callable, 'Bot'], *args: Any, **kwargs: Any) -> RT: - if callable(self): - self = cast('Bot', args[0]) +def mq(func: Callable[..., RT]) -> Callable[..., RT]: + logger = logging.getLogger(func.__module__) + + def decorator(_: Callable, *args: Any, **kwargs: Any) -> RT: + self = cast('Bot', args[0]) + arg_spec = inspect.getfullargspec(func) + idx = arg_spec.args.index('delay_queue') + delay_queue = args[idx] if not self.message_queue or not self.message_queue.running: + if delay_queue: + logger.warning( + 'Ignoring call to MessageQueue, because it is either not set or not running.' + ) return func(*args, **kwargs) - delay_queue = kwargs.pop('delay_queue', None) if not delay_queue: return func(*args, **kwargs) if delay_queue == self.message_queue.DEFAULT_QUEUE: # For default queue, check if we're in a group setting or not - arg_spec = inspect.getfullargspec(func) chat_id: Union[str, int] = '' - if 'chat_id' in kwargs: - chat_id = kwargs['chat_id'] - elif 'chat_id' in arg_spec.args: + if 'chat_id' in arg_spec.args: idx = arg_spec.args.index('chat_id') chat_id = args[idx] @@ -163,13 +163,19 @@ def decorator(self: Union[Callable, 'Bot'], *args: Any, **kwargs: Any) -> RT: except ValueError: is_group = False + logger.debug( + 'Processing MessageQueue call with chat id %s through the %s queue.', + chat_id, + 'group' if is_group else 'default', + ) + queue = self.message_queue.GROUP_QUEUE if is_group else delay_queue return self.message_queue.put( # type: ignore[return-value] - func, queue, self, *args, **kwargs + func, queue, *args, **kwargs ) return self.message_queue.put( # type: ignore[return-value] - func, delay_queue, self, *args, **kwargs + func, delay_queue, *args, **kwargs ) return decorate(func, decorator) diff --git a/telegram/ext/messagequeue.py b/telegram/ext/messagequeue.py index 2f89c1cab18..98a0bce6a9b 100644 --- a/telegram/ext/messagequeue.py +++ b/telegram/ext/messagequeue.py @@ -329,6 +329,7 @@ def stop(self, timeout: float = None) -> None: """ for delay_queue in self._delay_queues.values(): delay_queue.stop(timeout) + self.running = False def put(self, func: Callable, delay_queue: str, *args: Any, **kwargs: Any) -> Promise: """ diff --git a/tests/test_bot.py b/tests/test_bot.py index 43dfc2b71ff..25a7759f216 100644 --- a/tests/test_bot.py +++ b/tests/test_bot.py @@ -16,6 +16,7 @@ # # You should have received a copy of the GNU Lesser Public License # along with this program. If not, see [http://www.gnu.org/licenses/]. +import logging import time import datetime as dtm from platform import python_implementation @@ -45,7 +46,9 @@ ) from telegram.constants import MAX_INLINE_QUERY_RESULTS from telegram.error import BadRequest, InvalidToken, NetworkError, RetryAfter +from telegram.ext import MessageQueue, DelayQueue from telegram.utils.helpers import from_timestamp, escape_markdown, to_timestamp +from telegram.utils.promise import Promise from tests.conftest import expect_bad_request BASE_TIME = time.time() @@ -93,6 +96,15 @@ def inline_results(): return inline_results_callback() +@pytest.fixture(scope='function') +def mq_bot(bot, monkeypatch): + bot.message_queue = MessageQueue() + bot.message_queue.add_delay_queue(DelayQueue(name='custom_dq')) + yield bot + bot.message_queue.stop() + bot.message_queue = None + + class TestBot: @pytest.mark.parametrize( 'token', @@ -1379,3 +1391,72 @@ def test_set_and_get_my_commands_strings(self, bot): assert bc[0].description == 'descr1' assert bc[1].command == 'cmd2' assert bc[1].description == 'descr2' + + @pytest.mark.parametrize( + 'chat_id,expected', + [ + ('123', 'default'), + (123, 'default'), + ('-123', 'group'), + (-123, 'group'), + ('@supergroup', 'group'), + ('foobar', 'default'), + ], + ) + def test_message_queue_group_chat_detection( + self, mq_bot, monkeypatch, chat_id, expected, caplog + ): + def _post(*args, **kwargs): + pass + + monkeypatch.setattr(mq_bot, '_post', _post) + with caplog.at_level(logging.DEBUG): + result = mq_bot.send_message(chat_id, 'text', delay_queue=MessageQueue.DEFAULT_QUEUE) + assert isinstance(result, Promise) + assert len(caplog.records) == 4 + assert expected in caplog.records[1].getMessage() + + with caplog.at_level(logging.DEBUG): + result = mq_bot.send_message( + text='text', chat_id=chat_id, delay_queue=MessageQueue.DEFAULT_QUEUE + ) + assert isinstance(result, Promise) + assert len(caplog.records) == 8 + assert expected in caplog.records[5].getMessage() + + with caplog.at_level(logging.DEBUG): + assert isinstance(mq_bot.get_me(delay_queue=MessageQueue.DEFAULT_QUEUE), Promise) + assert len(caplog.records) == 12 + assert 'default' in caplog.records[9].getMessage() + + def test_stopped_message_queue(self, mq_bot, caplog): + mq_bot.message_queue.stop() + with caplog.at_level(logging.DEBUG): + result = mq_bot.get_me(delay_queue=MessageQueue.DEFAULT_QUEUE) + assert not isinstance(result, Promise) + assert len(caplog.records) >= 2 + assert 'Ignoring call to MessageQueue' in caplog.records[1].getMessage() + + def test_no_message_queue(self, bot, caplog): + with caplog.at_level(logging.DEBUG): + result = bot.get_me(delay_queue=MessageQueue.DEFAULT_QUEUE) + assert not isinstance(result, Promise) + assert len(caplog.records) >= 2 + assert 'Ignoring call to MessageQueue' in caplog.records[1].getMessage() + + def test_message_queue_custom_delay_queue(self, chat_id, mq_bot, monkeypatch): + test_flag = False + orig_put = mq_bot.message_queue._delay_queues['custom_dq'].put + + def put(*args, **kwargs): + nonlocal test_flag + test_flag = True + return orig_put(*args, **kwargs) + + result = mq_bot.send_message(chat_id, 'general kenobi') + assert not isinstance(result, Promise) + + monkeypatch.setattr(mq_bot.message_queue._delay_queues['custom_dq'], 'put', put) + result = mq_bot.send_message(chat_id, 'hello there', delay_queue='custom_dq') + assert isinstance(result, Promise) + assert test_flag From 6233920eab0914551330e6cef0db720fabb1cbd3 Mon Sep 17 00:00:00 2001 From: Hinrich Mahler Date: Thu, 26 Nov 2020 19:08:33 +0100 Subject: [PATCH 09/15] Test for Updater --- telegram/ext/messagequeue.py | 4 ++-- tests/test_updater.py | 16 ++++++++++++++-- 2 files changed, 16 insertions(+), 4 deletions(-) diff --git a/telegram/ext/messagequeue.py b/telegram/ext/messagequeue.py index 98a0bce6a9b..29cdba4942c 100644 --- a/telegram/ext/messagequeue.py +++ b/telegram/ext/messagequeue.py @@ -315,9 +315,9 @@ def remove_delay_queue(self, name: str, timeout: float = None) -> None: def start(self) -> None: """Starts the all :class:`telegram.ext.DelayQueue` registered for this message queue.""" + self.running = True for delay_queue in self._delay_queues.values(): delay_queue.start() - self.running = True def stop(self, timeout: float = None) -> None: """ @@ -327,9 +327,9 @@ def stop(self, timeout: float = None) -> None: timeout (:obj:`float`, optional): The timeout to pass to :meth:`telegram.ext.DelayQueue.stop`. """ + self.running = False for delay_queue in self._delay_queues.values(): delay_queue.stop(timeout) - self.running = False def put(self, func: Callable, delay_queue: str, *args: Any, **kwargs: Any) -> Promise: """ diff --git a/tests/test_updater.py b/tests/test_updater.py index 745836acd3d..b3cae39a64a 100644 --- a/tests/test_updater.py +++ b/tests/test_updater.py @@ -38,13 +38,13 @@ from telegram import TelegramError, Message, User, Chat, Update, Bot from telegram.error import Unauthorized, InvalidToken, TimedOut, RetryAfter -from telegram.ext import Updater, Dispatcher, DictPersistence, Defaults +from telegram.ext import Updater, Dispatcher, DictPersistence, Defaults, MessageQueue from telegram.utils.deprecate import TelegramDeprecationWarning from telegram.utils.webhookhandler import WebhookServer signalskip = pytest.mark.skipif( sys.platform == 'win32', - reason='Can\'t send signals without stopping ' 'whole process on windows', + reason='Can\'t send signals without stopping whole process on windows', ) @@ -583,3 +583,15 @@ def test_mutual_exclude_use_context_dispatcher(self): def test_defaults_warning(self, bot): with pytest.warns(TelegramDeprecationWarning, match='no effect when a Bot is passed'): Updater(bot=bot, defaults=Defaults()) + + def test_message_queue(self, bot, caplog): + updater = Updater(bot.token, message_queue=MessageQueue()) + updater.running = True + try: + assert updater.bot.message_queue.dispatcher is updater.dispatcher + with caplog.at_level(logging.DEBUG): + updater.stop() + assert caplog.records[1].getMessage() == 'Requesting MessageQueue to stop...' + assert not updater.bot.message_queue.running + finally: + updater.stop() From 478d1ffb4657fb588b67cb966fdb4211199bf6e6 Mon Sep 17 00:00:00 2001 From: Hinrich Mahler Date: Thu, 26 Nov 2020 19:21:01 +0100 Subject: [PATCH 10/15] Use tg.constants where possible --- telegram/ext/messagequeue.py | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/telegram/ext/messagequeue.py b/telegram/ext/messagequeue.py index 29cdba4942c..a9e1065e46a 100644 --- a/telegram/ext/messagequeue.py +++ b/telegram/ext/messagequeue.py @@ -32,6 +32,7 @@ from telegram.utils.deprecate import TelegramDeprecationWarning from telegram.utils.promise import Promise +from telegram import constants if TYPE_CHECKING: from telegram import Bot @@ -64,7 +65,8 @@ class DelayQueue(threading.Thread): requests through that delay queue after they were processed by this queue. Defaults to :obj:`None`. burst_limit (:obj:`int`, optional): Number of maximum callbacks to process per time-window - defined by :attr:`time_limit_ms`. Defaults to 30. + defined by :attr:`time_limit_ms`. Defaults to + :attr:`telegram.constants.MAX_MESSAGES_PER_SECOND`. time_limit_ms (:obj:`int`, optional): Defines width of time-window used when each processing limit is calculated. Defaults to 1000. error_handler (:obj:`callable`, optional): A callable, accepting 1 positional argument. @@ -86,7 +88,7 @@ class DelayQueue(threading.Thread): def __init__( self, queue: q.Queue = None, - burst_limit: int = 30, + burst_limit: int = constants.MAX_MESSAGES_PER_SECOND, time_limit_ms: int = 1000, exc_route: Callable[[Exception], None] = None, autostart: bool = True, @@ -226,11 +228,13 @@ class MessageQueue: Args: all_burst_limit (:obj:`int`, optional): Number of maximum *all-type* callbacks to process - per time-window defined by :attr:`all_time_limit_ms`. Defaults to 30. + per time-window defined by :attr:`all_time_limit_ms`. Defaults to + :attr:`telegram.constants.MAX_MESSAGES_PER_SECOND`. all_time_limit_ms (:obj:`int`, optional): Defines width of *all-type* time-window used when each processing limit is calculated. Defaults to 1000 ms. group_burst_limit (:obj:`int`, optional): Number of maximum *group-type* callbacks to - process per time-window defined by :attr:`group_time_limit_ms`. Defaults to 20. + process per time-window defined by :attr:`group_time_limit_ms`. Defaults to + :attr:`telegram.constants.MAX_MESSAGES_PER_MINUTE_PER_GROUP`. group_time_limit_ms (:obj:`int`, optional): Defines width of *group-type* time-window used when each processing limit is calculated. Defaults to 60000 ms. error_handler (:obj:`callable`, optional): A callable, accepting 1 positional argument. @@ -246,9 +250,9 @@ class MessageQueue: def __init__( self, - all_burst_limit: int = 30, + all_burst_limit: int = constants.MAX_MESSAGES_PER_SECOND, all_time_limit_ms: int = 1000, - group_burst_limit: int = 20, + group_burst_limit: int = constants.MAX_MESSAGES_PER_MINUTE_PER_GROUP, group_time_limit_ms: int = 60000, exc_route: Callable[[Exception], None] = None, autostart: bool = True, From 870a077eaa5758e0c438ba82cdc8deb2ae0d1d41 Mon Sep 17 00:00:00 2001 From: Hinrich Mahler Date: Thu, 26 Nov 2020 19:32:58 +0100 Subject: [PATCH 11/15] Increase coverage --- tests/test_bot.py | 12 ++++++++++++ tests/test_updater.py | 12 ++++++++++++ 2 files changed, 24 insertions(+) diff --git a/tests/test_bot.py b/tests/test_bot.py index 25a7759f216..f96e3c5e989 100644 --- a/tests/test_bot.py +++ b/tests/test_bot.py @@ -1460,3 +1460,15 @@ def put(*args, **kwargs): result = mq_bot.send_message(chat_id, 'hello there', delay_queue='custom_dq') assert isinstance(result, Promise) assert test_flag + + @pytest.mark.timeout(10) + def test_message_queue_context_manager(self, mq_bot, chat_id): + with open('tests/data/telegram.gif', 'rb') as document: + with open('tests/data/telegram.jpg', 'rb') as thumb: + promise = mq_bot.send_document( + chat_id, document, thumb=thumb, delay_queue=MessageQueue.DEFAULT_QUEUE + ) + + message = promise.result() + assert message.document + assert message.document.thumb diff --git a/tests/test_updater.py b/tests/test_updater.py index b3cae39a64a..18b949761cc 100644 --- a/tests/test_updater.py +++ b/tests/test_updater.py @@ -595,3 +595,15 @@ def test_message_queue(self, bot, caplog): assert not updater.bot.message_queue.running finally: updater.stop() + + updater = Updater(bot.token, message_queue=MessageQueue(autostart=False)) + updater.running = True + try: + assert updater.bot.message_queue.running + assert updater.bot.message_queue.dispatcher is updater.dispatcher + with caplog.at_level(logging.DEBUG): + updater.stop() + assert caplog.records[1].getMessage() == 'Requesting MessageQueue to stop...' + assert not updater.bot.message_queue.running + finally: + updater.stop() From 3433f78b30c3abf95267fbcc2b73dde5a557fae5 Mon Sep 17 00:00:00 2001 From: Hinrich Mahler Date: Wed, 23 Dec 2020 12:52:39 +0100 Subject: [PATCH 12/15] Drop context manager support & document MQ.delay_queues --- telegram/bot.py | 4 +- telegram/ext/messagequeue.py | 85 +++++++++++++++++++++++------------- telegram/utils/promise.py | 15 +------ tests/test_bot.py | 28 ++++-------- tests/test_messagequeue.py | 40 ++++++++--------- 5 files changed, 85 insertions(+), 87 deletions(-) diff --git a/telegram/bot.py b/telegram/bot.py index a74439703c9..a10d5cb76a3 100644 --- a/telegram/bot.py +++ b/telegram/bot.py @@ -157,7 +157,7 @@ def decorator(_: Callable, *args: Any, **kwargs: Any) -> RT: if not delay_queue: return func(*args, **kwargs) - if delay_queue == self.message_queue.DEFAULT_QUEUE: + if delay_queue == self.message_queue.DEFAULT_QUEUE_NAME: # For default queue, check if we're in a group setting or not chat_id: Union[str, int] = '' if 'chat_id' in arg_spec.args: @@ -180,7 +180,7 @@ def decorator(_: Callable, *args: Any, **kwargs: Any) -> RT: 'group' if is_group else 'default', ) - queue = self.message_queue.GROUP_QUEUE if is_group else delay_queue + queue = self.message_queue.GROUP_QUEUE_NAME if is_group else delay_queue return self.message_queue.put( # type: ignore[return-value] func, queue, *args, **kwargs ) diff --git a/telegram/ext/messagequeue.py b/telegram/ext/messagequeue.py index a9e1065e46a..b2613f01b9f 100644 --- a/telegram/ext/messagequeue.py +++ b/telegram/ext/messagequeue.py @@ -48,15 +48,10 @@ class DelayQueue(threading.Thread): Processes callbacks from queue with specified throughput limits. Creates a separate thread to process callbacks with delays. - Attributes: - burst_limit (:obj:`int`): Number of maximum callbacks to process per time-window. - time_limit (:obj:`int`): Defines width of time-window used when each processing limit is - calculated. - name (:obj:`str`): Thread's name. - error_handler (:obj:`callable`): Optional. A callable, accepting 1 positional argument. - Used to route exceptions from processor thread to main thread. - dispatcher (:class:`telegram.ext.Disptacher`): Optional. The dispatcher to use for error - handling. + Note: + For most use cases, the :attr:`parent` argument should be set to + :attr:`MessageQueue.DEFAULT_QUEUE_NAME` to ensure that the global flood limits are not + exceeded. Args: queue (:obj:`Queue`, optional): Used to pass callbacks to thread. Creates ``Queue`` @@ -81,6 +76,16 @@ class DelayQueue(threading.Thread): name (:obj:`str`, optional): Thread's name. Defaults to ``'DelayQueue-N'``, where N is sequential number of object created. + Attributes: + burst_limit (:obj:`int`): Number of maximum callbacks to process per time-window. + time_limit (:obj:`int`): Defines width of time-window used when each processing limit is + calculated. + name (:obj:`str`): Thread's name. + error_handler (:obj:`callable`): Optional. A callable, accepting 1 positional argument. + Used to route exceptions from processor thread to main thread. + dispatcher (:class:`telegram.ext.Disptacher`): Optional. The dispatcher to use for error + handling. + """ INSTANCE_COUNT: ClassVar[int] = 0 # instance counter @@ -221,11 +226,6 @@ class MessageQueue: By default contains two :class:`telegram.ext.DelayQueue` instances, for general requests and group requests where the default delay queue is the parent of the group requests one. - Attributes: - running (:obj:`bool`): Whether this message queue has started it's delay queues or not. - dispatcher (:class:`telegram.ext.Disptacher`): Optional. The Dispatcher to use for error - handling. - Args: all_burst_limit (:obj:`int`, optional): Number of maximum *all-type* callbacks to process per time-window defined by :attr:`all_time_limit_ms`. Defaults to @@ -246,6 +246,15 @@ class MessageQueue: autostart (:obj:`bool`, optional): If :obj:`True`, both default delay queues are started immediately after object's creation. Defaults to :obj:`True`. + Attributes: + running (:obj:`bool`): Whether this message queue has started it's delay queues or not. + dispatcher (:class:`telegram.ext.Disptacher`): Optional. The Dispatcher to use for error + handling. + delay_queues (Dict[:obj:`str`, :class:`telegram.ext.DelayQueue`]): A dictionary containing + all registered delay queues, where the keys are the names of the delay queues. By + default includes bot :attr:`default_queue` and :attr:`group_queue` under the keys + :attr:`DEFAULT_QUEUE_NAME` and :attr:`GROUP_QUEUE_NAME`, respectively. + """ def __init__( @@ -270,22 +279,22 @@ def __init__( stacklevel=2, ) - self._delay_queues: Dict[str, DelayQueue] = { - self.DEFAULT_QUEUE: DelayQueue( + self.delay_queues: Dict[str, DelayQueue] = { + self.DEFAULT_QUEUE_NAME: DelayQueue( burst_limit=all_burst_limit, time_limit_ms=all_time_limit_ms, error_handler=exc_route or error_handler, autostart=autostart, - name=self.DEFAULT_QUEUE, + name=self.DEFAULT_QUEUE_NAME, ) } - self._delay_queues[self.GROUP_QUEUE] = DelayQueue( + self.delay_queues[self.GROUP_QUEUE_NAME] = DelayQueue( burst_limit=group_burst_limit, time_limit_ms=group_time_limit_ms, error_handler=exc_route or error_handler, autostart=autostart, - name=self.GROUP_QUEUE, - parent=self._delay_queues[self.DEFAULT_QUEUE], + name=self.GROUP_QUEUE_NAME, + parent=self.delay_queues[self.DEFAULT_QUEUE_NAME], ) def add_delay_queue(self, delay_queue: DelayQueue) -> None: @@ -297,7 +306,7 @@ def add_delay_queue(self, delay_queue: DelayQueue) -> None: Args: delay_queue (:class:`telegram.ext.DelayQueue`): The delay queue to add. """ - self._delay_queues[delay_queue.name] = delay_queue + self.delay_queues[delay_queue.name] = delay_queue if self.dispatcher: delay_queue.set_dispatcher(self.dispatcher) if self.running and not delay_queue.is_alive(): @@ -313,14 +322,14 @@ def remove_delay_queue(self, name: str, timeout: float = None) -> None: timeout (:obj:`float`, optional): The timeout to pass to :meth:`telegram.ext.DelayQueue.stop`. """ - delay_queue = self._delay_queues.pop(name) + delay_queue = self.delay_queues.pop(name) if self.running and delay_queue.is_alive(): delay_queue.stop(timeout) def start(self) -> None: """Starts the all :class:`telegram.ext.DelayQueue` registered for this message queue.""" self.running = True - for delay_queue in self._delay_queues.values(): + for delay_queue in self.delay_queues.values(): delay_queue.start() def stop(self, timeout: float = None) -> None: @@ -332,7 +341,7 @@ def stop(self, timeout: float = None) -> None: :meth:`telegram.ext.DelayQueue.stop`. """ self.running = False - for delay_queue in self._delay_queues.values(): + for delay_queue in self.delay_queues.values(): delay_queue.stop(timeout) def put(self, func: Callable, delay_queue: str, *args: Any, **kwargs: Any) -> Promise: @@ -349,7 +358,7 @@ def put(self, func: Callable, delay_queue: str, *args: Any, **kwargs: Any) -> Pr :class:`telegram.ext.Promise`. """ - return self._delay_queues[delay_queue].put(func, args, kwargs) + return self.delay_queues[delay_queue].put(func, args, kwargs) def set_dispatcher(self, dispatcher: 'Dispatcher') -> None: """ @@ -360,10 +369,24 @@ def set_dispatcher(self, dispatcher: 'Dispatcher') -> None: """ self.dispatcher = dispatcher - DEFAULT_QUEUE: ClassVar[str] = 'default_delay_queue' - """:obj:`str`: The default delay queue.""" - GROUP_QUEUE: ClassVar[str] = 'group_delay_queue' - """:obj:`str`: The default delay queue for group requests.""" + DEFAULT_QUEUE_NAME: ClassVar[str] = 'default_delay_queue' + """:obj:`str`: The default delay queues name.""" + GROUP_QUEUE_NAME: ClassVar[str] = 'group_delay_queue' + """:obj:`str`: The name of the default delay queue for group requests.""" + + @property + def default_queue(self) -> DelayQueue: + """ + Shortcut for ``MessageQueue.delay_queues[MessageQueue.DEFAULT_QUEUE_NAME]``. + """ + return self.delay_queues[self.DEFAULT_QUEUE_NAME] + + @property + def group_queue(self) -> DelayQueue: + """ + Shortcut for ``MessageQueue.delay_queues[MessageQueue.GROUP_QUEUE_NAME]``. + """ + return self.delay_queues[self.GROUP_QUEUE_NAME] def queuedmessage(method: Callable) -> Callable: @@ -415,10 +438,10 @@ def wrapped(self: 'Bot', *args: Any, **kwargs: Any) -> Any: if queued: if not is_group: return self._msg_queue.put( # type: ignore[attr-defined] - method, MessageQueue.DEFAULT_QUEUE, self, *args, **kwargs + method, MessageQueue.DEFAULT_QUEUE_NAME, self, *args, **kwargs ) return self._msg_queue.put( # type: ignore[attr-defined] - method, MessageQueue.GROUP_QUEUE, self, *args, **kwargs + method, MessageQueue.GROUP_QUEUE_NAME, self, *args, **kwargs ) return method(self, *args, **kwargs) diff --git a/telegram/utils/promise.py b/telegram/utils/promise.py index d7333a05234..e989ce1aa5e 100644 --- a/telegram/utils/promise.py +++ b/telegram/utils/promise.py @@ -63,20 +63,9 @@ def __init__( update: Any = None, error_handling: bool = True, ): - - parsed_args = [] - for arg in args: - if InputFile.is_file(arg): - parsed_args.append(InputFile(arg)) - else: - parsed_args.append(arg) - self.args = tuple(parsed_args) - self.kwargs = kwargs - for key, value in self.kwargs.items(): - if InputFile.is_file(value): - self.kwargs[key] = InputFile(value) - self.pooled_function = pooled_function + self.args = args + self.kwargs = kwargs self.update = update self.error_handling = error_handling self.done = Event() diff --git a/tests/test_bot.py b/tests/test_bot.py index d2a47767a05..f6bd5c20f4b 100644 --- a/tests/test_bot.py +++ b/tests/test_bot.py @@ -1836,42 +1836,44 @@ def _post(*args, **kwargs): monkeypatch.setattr(mq_bot, '_post', _post) with caplog.at_level(logging.DEBUG): - result = mq_bot.send_message(chat_id, 'text', delay_queue=MessageQueue.DEFAULT_QUEUE) + result = mq_bot.send_message( + chat_id, 'text', delay_queue=MessageQueue.DEFAULT_QUEUE_NAME + ) assert isinstance(result, Promise) assert len(caplog.records) == 4 assert expected in caplog.records[1].getMessage() with caplog.at_level(logging.DEBUG): result = mq_bot.send_message( - text='text', chat_id=chat_id, delay_queue=MessageQueue.DEFAULT_QUEUE + text='text', chat_id=chat_id, delay_queue=MessageQueue.DEFAULT_QUEUE_NAME ) assert isinstance(result, Promise) assert len(caplog.records) == 8 assert expected in caplog.records[5].getMessage() with caplog.at_level(logging.DEBUG): - assert isinstance(mq_bot.get_me(delay_queue=MessageQueue.DEFAULT_QUEUE), Promise) + assert isinstance(mq_bot.get_me(delay_queue=MessageQueue.DEFAULT_QUEUE_NAME), Promise) assert len(caplog.records) == 12 assert 'default' in caplog.records[9].getMessage() def test_stopped_message_queue(self, mq_bot, caplog): mq_bot.message_queue.stop() with caplog.at_level(logging.DEBUG): - result = mq_bot.get_me(delay_queue=MessageQueue.DEFAULT_QUEUE) + result = mq_bot.get_me(delay_queue=MessageQueue.DEFAULT_QUEUE_NAME) assert not isinstance(result, Promise) assert len(caplog.records) >= 2 assert 'Ignoring call to MessageQueue' in caplog.records[1].getMessage() def test_no_message_queue(self, bot, caplog): with caplog.at_level(logging.DEBUG): - result = bot.get_me(delay_queue=MessageQueue.DEFAULT_QUEUE) + result = bot.get_me(delay_queue=MessageQueue.DEFAULT_QUEUE_NAME) assert not isinstance(result, Promise) assert len(caplog.records) >= 2 assert 'Ignoring call to MessageQueue' in caplog.records[1].getMessage() def test_message_queue_custom_delay_queue(self, chat_id, mq_bot, monkeypatch): test_flag = False - orig_put = mq_bot.message_queue._delay_queues['custom_dq'].put + orig_put = mq_bot.message_queue.delay_queues['custom_dq'].put def put(*args, **kwargs): nonlocal test_flag @@ -1881,19 +1883,7 @@ def put(*args, **kwargs): result = mq_bot.send_message(chat_id, 'general kenobi') assert not isinstance(result, Promise) - monkeypatch.setattr(mq_bot.message_queue._delay_queues['custom_dq'], 'put', put) + monkeypatch.setattr(mq_bot.message_queue.delay_queues['custom_dq'], 'put', put) result = mq_bot.send_message(chat_id, 'hello there', delay_queue='custom_dq') assert isinstance(result, Promise) assert test_flag - - @pytest.mark.timeout(10) - def test_message_queue_context_manager(self, mq_bot, chat_id): - with open('tests/data/telegram.gif', 'rb') as document: - with open('tests/data/telegram.jpg', 'rb') as thumb: - promise = mq_bot.send_document( - chat_id, document, thumb=thumb, delay_queue=MessageQueue.DEFAULT_QUEUE - ) - - message = promise.result() - assert message.document - assert message.document.thumb diff --git a/tests/test_messagequeue.py b/tests/test_messagequeue.py index a1fdde8834b..4152306983a 100644 --- a/tests/test_messagequeue.py +++ b/tests/test_messagequeue.py @@ -195,7 +195,7 @@ def callback_raises_exception(self): def test_auto_start_false(self): message_queue = MessageQueue(autostart=False) - assert not any(thread.is_alive() for thread in message_queue._delay_queues.values()) + assert not any(thread.is_alive() for thread in message_queue.delay_queues.values()) def test_exc_route_deprecation(self, recwarn): with pytest.raises(ValueError, match='Only one of exc_route or '): @@ -210,13 +210,13 @@ def test_add_delay_queue_autostart_false(self): delay_queue = DelayQueue(autostart=False, name='dq') try: message_queue.add_delay_queue(delay_queue) - assert 'dq' in message_queue._delay_queues - assert not any(thread.is_alive() for thread in message_queue._delay_queues.values()) + assert 'dq' in message_queue.delay_queues + assert not any(thread.is_alive() for thread in message_queue.delay_queues.values()) message_queue.start() - assert all(thread.is_alive() for thread in message_queue._delay_queues.values()) + assert all(thread.is_alive() for thread in message_queue.delay_queues.values()) message_queue.stop() - assert not any(thread.is_alive() for thread in message_queue._delay_queues.values()) + assert not any(thread.is_alive() for thread in message_queue.delay_queues.values()) finally: delay_queue.stop() message_queue.stop() @@ -227,12 +227,12 @@ def test_add_delay_queue_autostart_true(self, autostart): delay_queue = DelayQueue(name='dq', autostart=autostart) try: message_queue.add_delay_queue(delay_queue) - assert 'dq' in message_queue._delay_queues + assert 'dq' in message_queue.delay_queues assert delay_queue.is_alive() - assert all(thread.is_alive() for thread in message_queue._delay_queues.values()) + assert all(thread.is_alive() for thread in message_queue.delay_queues.values()) message_queue.stop() - assert not any(thread.is_alive() for thread in message_queue._delay_queues.values()) + assert not any(thread.is_alive() for thread in message_queue.delay_queues.values()) finally: delay_queue.stop() message_queue.stop() @@ -250,11 +250,11 @@ def test_remove_delay_queue(self, autostart): delay_queue = DelayQueue(name='dq') try: message_queue.add_delay_queue(delay_queue) - assert 'dq' in message_queue._delay_queues + assert 'dq' in message_queue.delay_queues assert delay_queue.is_alive() message_queue.remove_delay_queue('dq') - assert 'dq' not in message_queue._delay_queues + assert 'dq' not in message_queue.delay_queues if autostart: assert not delay_queue.is_alive() finally: @@ -266,23 +266,23 @@ def test_put(self): group_flag = None message_queue = MessageQueue() - original_put = message_queue._delay_queues[MessageQueue.DEFAULT_QUEUE].put + original_put = message_queue.default_queue.put def put(*args, **kwargs): nonlocal group_flag group_flag = True return original_put(*args, **kwargs) - message_queue._delay_queues[MessageQueue.GROUP_QUEUE].put = put + message_queue.group_queue.put = put try: - message_queue.put(self.call, MessageQueue.GROUP_QUEUE, 1, kwarg='foo') + message_queue.put(self.call, MessageQueue.GROUP_QUEUE_NAME, 1, kwarg='foo') sleep(0.5) assert self.test_flag is True # make sure that group queue was called, too assert group_flag is True finally: - message_queue._delay_queues[MessageQueue.GROUP_QUEUE].put = original_put + message_queue.group_queue.put = original_put message_queue.stop() @@ -304,8 +304,8 @@ def test_method(self, input, *args, **kwargs): bot = MQBot(token=bot.token) - orig_default_put = bot._msg_queue._delay_queues[MessageQueue.DEFAULT_QUEUE].put - orig_group_put = bot._msg_queue._delay_queues[MessageQueue.GROUP_QUEUE].put + orig_default_put = bot._msg_queue.default_queue.put + orig_group_put = bot._msg_queue.group_queue.put def step_default_counter(*args, **kwargs): orig_default_put(*args, **kwargs) @@ -315,12 +315,8 @@ def step_group_counter(*args, **kwargs): orig_group_put(*args, **kwargs) bot.group_count += 1 - monkeypatch.setattr( - bot._msg_queue._delay_queues[MessageQueue.DEFAULT_QUEUE], 'put', step_default_counter - ) - monkeypatch.setattr( - bot._msg_queue._delay_queues[MessageQueue.GROUP_QUEUE], 'put', step_group_counter - ) + monkeypatch.setattr(bot._msg_queue.default_queue, 'put', step_default_counter) + monkeypatch.setattr(bot._msg_queue.group_queue, 'put', step_group_counter) yield bot bot._msg_queue.stop() From 238cf151f15a479a81967a46b0c096c1bcd5a14e Mon Sep 17 00:00:00 2001 From: Hinrich Mahler Date: Wed, 23 Dec 2020 14:53:59 +0100 Subject: [PATCH 13/15] Update docs --- docs/source/telegram.ext.delayqueue.rst | 1 - docs/source/telegram.ext.messagequeue.rst | 1 - telegram/ext/messagequeue.py | 12 +++++++++++- 3 files changed, 11 insertions(+), 3 deletions(-) diff --git a/docs/source/telegram.ext.delayqueue.rst b/docs/source/telegram.ext.delayqueue.rst index ee79b849478..7426a923c74 100644 --- a/docs/source/telegram.ext.delayqueue.rst +++ b/docs/source/telegram.ext.delayqueue.rst @@ -4,4 +4,3 @@ telegram.ext.DelayQueue .. autoclass:: telegram.ext.DelayQueue :members: :show-inheritance: - :special-members: diff --git a/docs/source/telegram.ext.messagequeue.rst b/docs/source/telegram.ext.messagequeue.rst index 98bcb6e6357..f9ff9721044 100644 --- a/docs/source/telegram.ext.messagequeue.rst +++ b/docs/source/telegram.ext.messagequeue.rst @@ -4,4 +4,3 @@ telegram.ext.MessageQueue .. autoclass:: telegram.ext.MessageQueue :members: :show-inheritance: - :special-members: diff --git a/telegram/ext/messagequeue.py b/telegram/ext/messagequeue.py index b2613f01b9f..06b9f6ca6bf 100644 --- a/telegram/ext/messagequeue.py +++ b/telegram/ext/messagequeue.py @@ -48,9 +48,13 @@ class DelayQueue(threading.Thread): Processes callbacks from queue with specified throughput limits. Creates a separate thread to process callbacks with delays. + .. versionchanged:: 13.2 + DelayQueue was almost completely overhauled in v13.2. Please read the docs carefully, if + you're upgrading from lower versions. + Note: For most use cases, the :attr:`parent` argument should be set to - :attr:`MessageQueue.DEFAULT_QUEUE_NAME` to ensure that the global flood limits are not + :attr:`MessageQueue.default_queue` to ensure that the global flood limits are not exceeded. Args: @@ -226,6 +230,10 @@ class MessageQueue: By default contains two :class:`telegram.ext.DelayQueue` instances, for general requests and group requests where the default delay queue is the parent of the group requests one. + .. versionchanged:: 13.2 + MessageQueue was almost completely overhauled in v13.2. Please read the docs carefully, if + you're upgrading from lower versions. + Args: all_burst_limit (:obj:`int`, optional): Number of maximum *all-type* callbacks to process per time-window defined by :attr:`all_time_limit_ms`. Defaults to @@ -392,6 +400,8 @@ def group_queue(self) -> DelayQueue: def queuedmessage(method: Callable) -> Callable: """A decorator to be used with :attr:`telegram.Bot` send* methods. + .. deprecated:: 13.2 + Note: As it probably wouldn't be a good idea to make this decorator a property, it has been coded as decorator function, so it implies that first positional argument to wrapped MUST be From e7d4c2396bfd9c92f531fae05ee05425f95baff7 Mon Sep 17 00:00:00 2001 From: Hinrich Mahler Date: Wed, 23 Dec 2020 15:07:41 +0100 Subject: [PATCH 14/15] Add test for defaults --- tests/test_bot.py | 54 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 54 insertions(+) diff --git a/tests/test_bot.py b/tests/test_bot.py index f6bd5c20f4b..be2c647d22e 100644 --- a/tests/test_bot.py +++ b/tests/test_bot.py @@ -44,6 +44,7 @@ Dice, MessageEntity, ParseMode, + Message, ) from telegram.constants import MAX_INLINE_QUERY_RESULTS from telegram.error import BadRequest, InvalidToken, NetworkError, RetryAfter @@ -1887,3 +1888,56 @@ def put(*args, **kwargs): result = mq_bot.send_message(chat_id, 'hello there', delay_queue='custom_dq') assert isinstance(result, Promise) assert test_flag + + @pytest.mark.parametrize( + 'default_bot', + [ + { + 'delay_queue': MessageQueue.DEFAULT_QUEUE_NAME, + 'delay_queue_per_method': { + 'send_dice': MessageQueue.GROUP_QUEUE_NAME, + 'send_poll': None, + }, + } + ], + indirect=True, + ) + def test_message_queue_with_defaults(self, chat_id, default_bot, monkeypatch): + default_bot.message_queue = MessageQueue() + + default_counter = 0 + group_counter = 0 + orig_default_put = default_bot.message_queue.default_queue.put + orig_group_put = default_bot.message_queue.default_queue.put + + def default_put(*args, **kwargs): + nonlocal default_counter + default_counter += 1 + return orig_default_put(*args, **kwargs) + + def group_put(*args, **kwargs): + nonlocal group_counter + group_counter += 1 + return orig_group_put(*args, **kwargs) + + try: + monkeypatch.setattr(default_bot.message_queue.default_queue, 'put', default_put) + monkeypatch.setattr(default_bot.message_queue.group_queue, 'put', group_put) + + result = default_bot.send_message(chat_id, 'general kenobi') + assert isinstance(result, Promise) + assert default_counter == 1 + assert group_counter == 0 + + result = default_bot.send_poll(chat_id, 'question', options=['1', '2']) + assert isinstance(result, Message) + assert default_counter == 1 + assert group_counter == 0 + + result = default_bot.send_dice(chat_id) + assert isinstance(result, Promise) + assert default_counter == 1 + assert group_counter == 1 + finally: + default_bot.message_queue.stop() + default_bot.message_queue = None From 216ed4e6130e1ad148d79dd1c3a8ea9e12a10cc7 Mon Sep 17 00:00:00 2001 From: Hinrich Mahler Date: Wed, 23 Dec 2020 17:10:28 +0100 Subject: [PATCH 15/15] Add priority functionality --- telegram/ext/messagequeue.py | 50 ++++++++++++++++++++++++++++-------- tests/test_messagequeue.py | 44 +++++++++++++++++++++++++++++++ 2 files changed, 84 insertions(+), 10 deletions(-) diff --git a/telegram/ext/messagequeue.py b/telegram/ext/messagequeue.py index 06b9f6ca6bf..3a91ac34d87 100644 --- a/telegram/ext/messagequeue.py +++ b/telegram/ext/messagequeue.py @@ -23,10 +23,10 @@ import functools import logging -import queue as q import threading import time import warnings +from queue import PriorityQueue from typing import Callable, Any, TYPE_CHECKING, List, NoReturn, ClassVar, Dict, Optional @@ -43,6 +43,23 @@ class DelayQueueError(RuntimeError): """Indicates processing errors.""" +@functools.total_ordering +class PriorityWrapper: + def __init__(self, priority: int, promise: Promise): + self.priority = priority + self.promise = promise + + def __lt__(self, other: object) -> bool: + if not isinstance(other, PriorityWrapper): + raise NotImplementedError + return self.priority < other.priority + + def __eq__(self, other: object) -> bool: + if not isinstance(other, PriorityWrapper): + raise NotImplementedError + return self.priority == other.priority + + class DelayQueue(threading.Thread): """ Processes callbacks from queue with specified throughput limits. Creates a separate thread to @@ -58,8 +75,6 @@ class DelayQueue(threading.Thread): exceeded. Args: - queue (:obj:`Queue`, optional): Used to pass callbacks to thread. Creates ``Queue`` - implicitly if not provided. parent (:class:`telegram.ext.DelayQueue`, optional): Pass another delay queue to put all requests through that delay queue after they were processed by this queue. Defaults to :obj:`None`. @@ -79,6 +94,10 @@ class DelayQueue(threading.Thread): Defaults to :obj:`True`. name (:obj:`str`, optional): Thread's name. Defaults to ``'DelayQueue-N'``, where N is sequential number of object created. + priority (:obj:`int`, optional): Priority of the delay queue. Higher priority callbacks are + processed before lower priority callbacks, even if scheduled later. Higher number means + lower priority (i.e. ``priority = 0`` is processed *before* ``priority = 1``). Only + relevant, if the delay queue has a :attr:`parent``. Defaults to ``0``. Attributes: burst_limit (:obj:`int`): Number of maximum callbacks to process per time-window. @@ -96,7 +115,6 @@ class DelayQueue(threading.Thread): def __init__( self, - queue: q.Queue = None, burst_limit: int = constants.MAX_MESSAGES_PER_SECOND, time_limit_ms: int = 1000, exc_route: Callable[[Exception], None] = None, @@ -104,13 +122,15 @@ def __init__( name: str = None, parent: 'DelayQueue' = None, error_handler: Callable[[Exception], None] = None, + priority: int = 0, ): self.logger = logging.getLogger(__name__) - self._queue = queue if queue is not None else q.Queue() self.burst_limit = burst_limit self.time_limit = time_limit_ms / 1000 self.parent = parent self.dispatcher: Optional['Dispatcher'] = None + self.priority = priority + self._queue: 'PriorityQueue[PriorityWrapper]' = PriorityQueue() if exc_route and error_handler: raise ValueError('Only one of exc_route or error_handler can be passed.') @@ -145,7 +165,7 @@ def run(self) -> None: times: List[float] = [] # used to store each callable processing time while True: - promise = self._queue.get() + promise = self._queue.get().promise if self.__exit_req: return # shutdown thread @@ -167,7 +187,7 @@ def run(self) -> None: # finally process one if self.parent: # put through parent, if specified - self.parent.put(promise=promise) + self.parent.put(promise=promise, priority=self.priority) else: promise.run() # error handling @@ -189,7 +209,11 @@ def stop(self, timeout: float = None) -> None: """ self.__exit_req = True # gently request - self._queue.put(None) # put something to unfreeze if frozen + self._queue.put( + PriorityWrapper( + 0, Promise(PriorityQueue, [], {}) # put something to unfreeze if frozen + ) + ) self.logger.debug('Waiting for DelayQueue %s to shut down.', self.name) super().join(timeout=timeout) self.logger.debug('DelayQueue %s shut down.', self.name) @@ -199,7 +223,12 @@ def _default_exception_handler(exc: Exception) -> NoReturn: raise exc def put( - self, func: Callable = None, args: Any = None, kwargs: Any = None, promise: Promise = None + self, + func: Callable = None, + args: Any = None, + kwargs: Any = None, + promise: Promise = None, + priority: int = 0, ) -> Promise: """Used to process callbacks in throughput-limiting thread through queue. You must either pass a :class:`telegram.utils.Promise` or all of ``func``, ``args`` and ``kwargs``. @@ -210,6 +239,7 @@ def put( args (:obj:`list`, optional): Variable-length `func` arguments. kwargs (:obj:`dict`, optional): Arbitrary keyword-arguments to `func`. promise (:class:`telegram.utils.Promise`, optional): A promise. + priority (:obj:`int`, optional): Priority of the callback. Defaults to ``0``. """ if not bool(promise) ^ all(v is not None for v in [func, args, kwargs]): @@ -220,7 +250,7 @@ def put( if not promise: promise = Promise(func, args, kwargs) # type: ignore[arg-type] - self._queue.put(promise) + self._queue.put(PriorityWrapper(priority, promise)) return promise diff --git a/tests/test_messagequeue.py b/tests/test_messagequeue.py index 4152306983a..942f6555924 100644 --- a/tests/test_messagequeue.py +++ b/tests/test_messagequeue.py @@ -99,6 +99,50 @@ def test_delay_queue_limits(self): finally: delay_queue.stop() + def test_with_priority(self): + parent_queue = DelayQueue() + high_priority_queue = DelayQueue(parent=parent_queue, priority=0) + low_priority_queue = DelayQueue(parent=parent_queue, priority=1) + high_priority_count = 0 + low_priority_count = 0 + event_list = [] + + def low_priority_callback(): + nonlocal low_priority_count + nonlocal event_list + event_list.append((low_priority_count, 'low')) + low_priority_count += 1 + + def high_priority_callback(): + nonlocal high_priority_count + nonlocal event_list + event_list.append((high_priority_count, 'high')) + high_priority_count += 1 + + # enqueue low priority first + for _ in range(3): + low_priority_queue.put(low_priority_callback, args=[], kwargs={}) + + # enqueue high priority second + for _ in range(3): + high_priority_queue.put(high_priority_callback, args=[], kwargs={}) + + try: + sleep(1) + # high priority events should be handled first + assert event_list == [ + (0, 'high'), + (1, 'high'), + (2, 'high'), + (0, 'low'), + (1, 'low'), + (2, 'low'), + ] + finally: + parent_queue.stop() + low_priority_queue.stop() + high_priority_queue.stop() + def test_put_errors(self): delay_queue = DelayQueue(autostart=False) with pytest.raises(DelayQueueError, match='stopped thread'): pFad - Phonifier reborn

Pfad - The Proxy pFad of © 2024 Garber Painting. All rights reserved.

Note: This service is not intended for secure transactions such as banking, social media, email, or purchasing. Use at your own risk. We assume no liability whatsoever for broken pages.


Alternative Proxies:

Alternative Proxy

pFad Proxy

pFad v3 Proxy

pFad v4 Proxy