From 9e47f02fbd99dc94a4d2ea4db9294f4068aa5de7 Mon Sep 17 00:00:00 2001 From: Harshil <37377066+harshil21@users.noreply.github.com> Date: Sat, 22 Jan 2022 02:16:47 +0530 Subject: [PATCH 1/6] simplify persistence --- telegram/ext/_basepersistence.py | 31 +++---- telegram/ext/_dictpersistence.py | 43 ++++----- telegram/ext/_picklepersistence.py | 54 +++++------ tests/test_persistence.py | 141 ++++++++++++----------------- 4 files changed, 116 insertions(+), 153 deletions(-) diff --git a/telegram/ext/_basepersistence.py b/telegram/ext/_basepersistence.py index 81a16a66371..47f2f7b40c3 100644 --- a/telegram/ext/_basepersistence.py +++ b/telegram/ext/_basepersistence.py @@ -19,7 +19,7 @@ """This module contains the BasePersistence class.""" from abc import ABC, abstractmethod from copy import copy -from typing import Dict, Optional, Tuple, cast, ClassVar, Generic, DefaultDict, NamedTuple +from typing import Dict, Optional, Tuple, cast, ClassVar, Generic, DefaultDict, NamedTuple, Union from telegram import Bot from telegram.ext import ExtBot @@ -132,10 +132,10 @@ def __new__( update_bot_data = instance.update_bot_data update_callback_data = instance.update_callback_data - def get_user_data_insert_bot() -> DefaultDict[int, UD]: + def get_user_data_insert_bot() -> Union[DefaultDict[int, UD], Dict[int, UD]]: return instance.insert_bot(get_user_data()) - def get_chat_data_insert_bot() -> DefaultDict[int, CD]: + def get_chat_data_insert_bot() -> Union[DefaultDict[int, CD], Dict[int, CD]]: return instance.insert_bot(get_chat_data()) def get_bot_data_insert_bot() -> BD: @@ -171,10 +171,7 @@ def update_callback_data_replace_bot(data: CDCData) -> None: setattr(instance, 'update_callback_data', update_callback_data_replace_bot) return instance - def __init__( - self, - store_data: PersistenceInput = None, - ): + def __init__(self, store_data: PersistenceInput = None): self.store_data = store_data or PersistenceInput() self.bot: Bot = None # type: ignore[assignment] @@ -398,34 +395,34 @@ def _insert_bot(self, obj: object, memo: Dict[int, object]) -> object: return obj @abstractmethod - def get_user_data(self) -> DefaultDict[int, UD]: + def get_user_data(self) -> Union[DefaultDict[int, UD], Dict[int, UD]]: """Will be called by :class:`telegram.ext.Dispatcher` upon creation with a persistence object. It should return the ``user_data`` if stored, or an empty - :obj:`defaultdict`. In the latter case, the :obj:`defaultdict` should produce values + :obj:`defaultdict` or :obj:`dict`. In the latter case, the dictionary should produce values corresponding to one of the following: * :obj:`dict` * The type from :attr:`telegram.ext.ContextTypes.user_data` - if :class:`telegram.ext.ContextTypes` are used. + if :class:`telegram.ext.ContextTypes` is used. Returns: - DefaultDict[:obj:`int`, :obj:`dict` | :attr:`telegram.ext.ContextTypes.user_data`]: + Dict[:obj:`int`, :obj:`dict` | :attr:`telegram.ext.ContextTypes.user_data`]: The restored user data. """ @abstractmethod - def get_chat_data(self) -> DefaultDict[int, CD]: + def get_chat_data(self) -> Union[DefaultDict[int, CD], Dict[int, CD]]: """Will be called by :class:`telegram.ext.Dispatcher` upon creation with a persistence object. It should return the ``chat_data`` if stored, or an empty - :obj:`defaultdict`. In the latter case, the :obj:`defaultdict` should produce values + :obj:`defaultdict` or :obj:`dict`. In the latter case, the dictionary should produce values corresponding to one of the following: * :obj:`dict` * The type from :attr:`telegram.ext.ContextTypes.chat_data` - if :class:`telegram.ext.ContextTypes` are used. + if :class:`telegram.ext.ContextTypes` is used. Returns: - DefaultDict[:obj:`int`, :obj:`dict` | :attr:`telegram.ext.ContextTypes.chat_data`]: + Dict[:obj:`int`, :obj:`dict` | :attr:`telegram.ext.ContextTypes.chat_data`]: The restored chat data. """ @@ -433,7 +430,7 @@ def get_chat_data(self) -> DefaultDict[int, CD]: def get_bot_data(self) -> BD: """Will be called by :class:`telegram.ext.Dispatcher` upon creation with a persistence object. It should return the ``bot_data`` if stored, or an empty - :obj:`defaultdict`. In the latter case, the :obj:`defaultdict` should produce values + :obj:`dict`. In the latter case, the :obj:`dict` should produce values corresponding to one of the following: * :obj:`dict` @@ -441,7 +438,7 @@ def get_bot_data(self) -> BD: if :class:`telegram.ext.ContextTypes` are used. Returns: - DefaultDict[:obj:`int`, :obj:`dict` | :attr:`telegram.ext.ContextTypes.bot_data`]: + Dict[:obj:`int`, :obj:`dict` | :attr:`telegram.ext.ContextTypes.bot_data`]: The restored bot data. """ diff --git a/telegram/ext/_dictpersistence.py b/telegram/ext/_dictpersistence.py index df3f9aa71f5..48110618600 100644 --- a/telegram/ext/_dictpersistence.py +++ b/telegram/ext/_dictpersistence.py @@ -18,8 +18,7 @@ # along with this program. If not, see [http://www.gnu.org/licenses/]. """This module contains the DictPersistence class.""" -from typing import DefaultDict, Dict, Optional, Tuple, cast -from collections import defaultdict +from typing import Dict, Optional, Tuple, cast from telegram.ext import BasePersistence, PersistenceInput from telegram._utils.types import JSONDict @@ -166,7 +165,7 @@ def __init__( ) from exc @property - def user_data(self) -> Optional[DefaultDict[int, Dict]]: + def user_data(self) -> Optional[Dict[int, Dict]]: """:obj:`dict`: The user_data as a dict.""" return self._user_data @@ -178,7 +177,7 @@ def user_data_json(self) -> str: return json.dumps(self.user_data) @property - def chat_data(self) -> Optional[DefaultDict[int, Dict]]: + def chat_data(self) -> Optional[Dict[int, Dict]]: """:obj:`dict`: The chat_data as a dict.""" return self._chat_data @@ -204,7 +203,7 @@ def bot_data_json(self) -> str: @property def callback_data(self) -> Optional[CDCData]: """Tuple[List[Tuple[:obj:`str`, :obj:`float`, Dict[:obj:`str`, :obj:`Any`]]], \ - Dict[:obj:`str`, :obj:`str`]]: The meta data on the stored callback data. + Dict[:obj:`str`, :obj:`str`]]: The metadata on the stored callback data. .. versionadded:: 13.6 """ @@ -212,7 +211,7 @@ def callback_data(self) -> Optional[CDCData]: @property def callback_data_json(self) -> str: - """:obj:`str`: The meta data on the stored callback data as a JSON-string. + """:obj:`str`: The metadata on the stored callback data as a JSON-string. .. versionadded:: 13.6 """ @@ -232,26 +231,24 @@ def conversations_json(self) -> str: return self._conversations_json return self._encode_conversations_to_json(self.conversations) # type: ignore[arg-type] - def get_user_data(self) -> DefaultDict[int, Dict[object, object]]: - """Returns the user_data created from the ``user_data_json`` or an empty - :obj:`defaultdict`. + def get_user_data(self) -> Dict[int, Dict[object, object]]: + """Returns the user_data created from the ``user_data_json`` or an empty :obj:`dict`. Returns: - :obj:`defaultdict`: The restored user data. + :obj:`dict`: The restored user data. """ if self.user_data is None: - self._user_data = defaultdict(dict) + self._user_data = {} return self.user_data # type: ignore[return-value] - def get_chat_data(self) -> DefaultDict[int, Dict[object, object]]: - """Returns the chat_data created from the ``chat_data_json`` or an empty - :obj:`defaultdict`. + def get_chat_data(self) -> Dict[int, Dict[object, object]]: + """Returns the chat_data created from the ``chat_data_json`` or an empty :obj:`dict`. Returns: - :obj:`defaultdict`: The restored chat data. + :obj:`dict`: The restored chat data. """ if self.chat_data is None: - self._chat_data = defaultdict(dict) + self._chat_data = {} return self.chat_data # type: ignore[return-value] def get_bot_data(self) -> Dict[object, object]: @@ -271,7 +268,7 @@ def get_callback_data(self) -> Optional[CDCData]: Returns: Tuple[List[Tuple[:obj:`str`, :obj:`float`, Dict[:obj:`str`, :obj:`Any`]]], \ - Dict[:obj:`str`, :obj:`str`]]: The restored meta data or :obj:`None`, \ + Dict[:obj:`str`, :obj:`str`]]: The restored metadata or :obj:`None`, \ if no data was stored. """ if self.callback_data is None: @@ -315,8 +312,8 @@ def update_user_data(self, user_id: int, data: Dict) -> None: data (:obj:`dict`): The :attr:`telegram.ext.Dispatcher.user_data` ``[user_id]``. """ if self._user_data is None: - self._user_data = defaultdict(dict) - if self._user_data.get(user_id) == data: + self._user_data = {} + if self._user_data.get(user_id, None) == data: return self._user_data[user_id] = data self._user_data_json = None @@ -329,8 +326,8 @@ def update_chat_data(self, chat_id: int, data: Dict) -> None: data (:obj:`dict`): The :attr:`telegram.ext.Dispatcher.chat_data` ``[chat_id]``. """ if self._chat_data is None: - self._chat_data = defaultdict(dict) - if self._chat_data.get(chat_id) == data: + self._chat_data = {} + if self._chat_data.get(chat_id, None) == data: return self._chat_data[chat_id] = data self._chat_data_json = None @@ -453,7 +450,7 @@ def _decode_conversations_from_json(json_string: str) -> Dict[str, Dict[Tuple, o return conversations @staticmethod - def _decode_user_chat_data_from_json(data: str) -> DefaultDict[int, Dict[object, object]]: + def _decode_user_chat_data_from_json(data: str) -> Dict[int, Dict[object, object]]: """Helper method to decode chat or user data (that uses ints as keys) from a JSON-string. @@ -463,7 +460,7 @@ def _decode_user_chat_data_from_json(data: str) -> DefaultDict[int, Dict[object, Returns: :obj:`dict`: The user/chat_data defaultdict after decoding """ - tmp: DefaultDict[int, Dict[object, object]] = defaultdict(dict) + tmp: Dict[int, Dict[object, object]] = {} decoded_data = json.loads(data) for user, user_data in decoded_data.items(): user = int(user) diff --git a/telegram/ext/_picklepersistence.py b/telegram/ext/_picklepersistence.py index 3fcbb995424..0bb2a4544e4 100644 --- a/telegram/ext/_picklepersistence.py +++ b/telegram/ext/_picklepersistence.py @@ -18,7 +18,6 @@ # along with this program. If not, see [http://www.gnu.org/licenses/]. """This module contains the PicklePersistence class.""" import pickle -from collections import defaultdict from pathlib import Path from typing import ( Any, @@ -27,7 +26,6 @@ Tuple, overload, cast, - DefaultDict, ) from telegram._utils.types import FilePathInput @@ -138,8 +136,8 @@ def __init__( self.filepath = Path(filepath) self.single_file = single_file self.on_flush = on_flush - self.user_data: Optional[DefaultDict[int, UD]] = None - self.chat_data: Optional[DefaultDict[int, CD]] = None + self.user_data: Optional[Dict[int, UD]] = None + self.chat_data: Optional[Dict[int, CD]] = None self.bot_data: Optional[BD] = None self.callback_data: Optional[CDCData] = None self.conversations: Optional[Dict[str, Dict[Tuple, object]]] = None @@ -149,16 +147,16 @@ def _load_singlefile(self) -> None: try: with self.filepath.open("rb") as file: data = pickle.load(file) - self.user_data = defaultdict(self.context_types.user_data, data['user_data']) - self.chat_data = defaultdict(self.context_types.chat_data, data['chat_data']) + self.user_data = data['user_data'] + self.chat_data = data['chat_data'] # For backwards compatibility with files not containing bot data self.bot_data = data.get('bot_data', self.context_types.bot_data()) self.callback_data = data.get('callback_data', {}) self.conversations = data['conversations'] except OSError: self.conversations = {} - self.user_data = defaultdict(self.context_types.user_data) - self.chat_data = defaultdict(self.context_types.chat_data) + self.user_data = {} + self.chat_data = {} self.bot_data = self.context_types.bot_data() self.callback_data = None except pickle.UnpicklingError as exc: @@ -195,41 +193,35 @@ def _dump_file(filepath: Path, data: object) -> None: with filepath.open("wb") as file: pickle.dump(data, file) - def get_user_data(self) -> DefaultDict[int, UD]: - """Returns the user_data from the pickle file if it exists or an empty :obj:`defaultdict`. + def get_user_data(self) -> Dict[int, UD]: + """Returns the user_data from the pickle file if it exists or an empty :obj:`dict`. Returns: - DefaultDict[:obj:`int`, :obj:`dict` | :attr:`telegram.ext.ContextTypes.user_data`]: - The restored user data. + Dict[:obj:`int`, :obj:`dict`]: The restored user data. """ if self.user_data: pass elif not self.single_file: data = self._load_file(Path(f"{self.filepath}_user_data")) if not data: - data = defaultdict(self.context_types.user_data) - else: - data = defaultdict(self.context_types.user_data, data) + data = {} self.user_data = data else: self._load_singlefile() return self.user_data # type: ignore[return-value] - def get_chat_data(self) -> DefaultDict[int, CD]: - """Returns the chat_data from the pickle file if it exists or an empty :obj:`defaultdict`. + def get_chat_data(self) -> Dict[int, CD]: + """Returns the chat_data from the pickle file if it exists or an empty :obj:`dict`. Returns: - DefaultDict[:obj:`int`, :obj:`dict` | :attr:`telegram.ext.ContextTypes.chat_data`]: - The restored chat data. + Dict[:obj:`int`, :obj:`dict`]: The restored chat data. """ if self.chat_data: pass elif not self.single_file: data = self._load_file(Path(f"{self.filepath}_chat_data")) if not data: - data = defaultdict(self.context_types.chat_data) - else: - data = defaultdict(self.context_types.chat_data, data) + data = {} self.chat_data = data else: self._load_singlefile() @@ -260,8 +252,8 @@ def get_callback_data(self) -> Optional[CDCData]: Returns: Optional[Tuple[List[Tuple[:obj:`str`, :obj:`float`, \ - Dict[:obj:`str`, :obj:`Any`]]], Dict[:obj:`str`, :obj:`str`]]: - The restored meta data or :obj:`None`, if no data was stored. + Dict[:obj:`str`, :obj:`Any`]]], Dict[:obj:`str`, :obj:`str`]]]: + The restored metadata or :obj:`None`, if no data was stored. """ if self.callback_data: pass @@ -323,12 +315,11 @@ def update_user_data(self, user_id: int, data: UD) -> None: Args: user_id (:obj:`int`): The user the data might have been changed for. - data (:obj:`dict` | :attr:`telegram.ext.ContextTypes.user_data`): The - :attr:`telegram.ext.Dispatcher.user_data` ``[user_id]``. + data (:obj:`dict`): The :attr:`telegram.ext.Dispatcher.user_data` ``[user_id]``. """ if self.user_data is None: - self.user_data = defaultdict(self.context_types.user_data) - if self.user_data.get(user_id) == data: + self.user_data = {} + if self.user_data.get(user_id, None) == data: # type: ignore[arg-type] return self.user_data[user_id] = data if not self.on_flush: @@ -342,12 +333,11 @@ def update_chat_data(self, chat_id: int, data: CD) -> None: Args: chat_id (:obj:`int`): The chat the data might have been changed for. - data (:obj:`dict` | :attr:`telegram.ext.ContextTypes.chat_data`): The - :attr:`telegram.ext.Dispatcher.chat_data` ``[chat_id]``. + data (:obj:`dict`): The :attr:`telegram.ext.Dispatcher.chat_data` ``[chat_id]``. """ if self.chat_data is None: - self.chat_data = defaultdict(self.context_types.chat_data) - if self.chat_data.get(chat_id) == data: + self.chat_data = {} + if self.chat_data.get(chat_id, None) == data: # type: ignore[arg-type] return self.chat_data[chat_id] = data if not self.on_flush: diff --git a/tests/test_persistence.py b/tests/test_persistence.py index 55ab9f1f15e..fc7f32fcf97 100644 --- a/tests/test_persistence.py +++ b/tests/test_persistence.py @@ -16,26 +16,24 @@ # # You should have received a copy of the GNU Lesser Public License # along with this program. If not, see [http://www.gnu.org/licenses/]. +import logging +import os +import pickle import gzip import signal import uuid +from collections.abc import Container +from collections import defaultdict from pathlib import Path +from time import sleep from threading import Lock -from telegram.ext import PersistenceInput, UpdaterBuilder, CallbackDataCache +import pytest try: import ujson as json except ImportError: import json -import logging -import os -import pickle -from collections import defaultdict -from collections.abc import Container -from time import sleep - -import pytest from telegram import Update, Message, User, Chat, MessageEntity, Bot from telegram.ext import ( @@ -49,6 +47,9 @@ TypeHandler, JobQueue, ContextTypes, + PersistenceInput, + UpdaterBuilder, + CallbackDataCache, ) from telegram.ext._callbackdatacache import _KeyboardData @@ -134,8 +135,8 @@ class BotPersistence(BasePersistence): def __init__(self): super().__init__() self.bot_data = None - self.chat_data = defaultdict(dict) - self.user_data = defaultdict(dict) + self.chat_data = {} + self.user_data = {} self.callback_data = None def get_bot_data(self): @@ -196,16 +197,12 @@ def bot_data(): @pytest.fixture(scope="function") def chat_data(): - return defaultdict( - dict, {-12345: {'test1': 'test2', 'test3': {'test4': 'test5'}}, -67890: {3: 'test4'}} - ) + return {-12345: {'test1': 'test2', 'test3': {'test4': 'test5'}}, -67890: {3: 'test4'}} @pytest.fixture(scope="function") def user_data(): - return defaultdict( - dict, {12345: {'test1': 'test2', 'test3': {'test4': 'test5'}}, 67890: {3: 'test4'}} - ) + return {12345: {'test1': 'test2', 'test3': {'test4': 'test5'}}, 67890: {3: 'test4'}} @pytest.fixture(scope="function") @@ -1040,10 +1037,6 @@ def update(bot): return Update(0, message=message) -class CustomMapping(defaultdict): - pass - - class TestPicklePersistence: def test_slot_behaviour(self, mro_slots, pickle_persistence): inst = pickle_persistence @@ -1059,16 +1052,16 @@ def test_pickle_behaviour_with_slots(self, pickle_persistence): assert retrieved == bot_data def test_no_files_present_multi_file(self, pickle_persistence): - assert pickle_persistence.get_user_data() == defaultdict(dict) - assert pickle_persistence.get_chat_data() == defaultdict(dict) + assert pickle_persistence.get_user_data() == {} + assert pickle_persistence.get_chat_data() == {} assert pickle_persistence.get_bot_data() == {} assert pickle_persistence.get_callback_data() is None assert pickle_persistence.get_conversations('noname') == {} def test_no_files_present_single_file(self, pickle_persistence): pickle_persistence.single_file = True - assert pickle_persistence.get_user_data() == defaultdict(dict) - assert pickle_persistence.get_chat_data() == defaultdict(dict) + assert pickle_persistence.get_user_data() == {} + assert pickle_persistence.get_chat_data() == {} assert pickle_persistence.get_bot_data() == {} assert pickle_persistence.get_callback_data() is None assert pickle_persistence.get_conversations('noname') == {} @@ -1125,16 +1118,14 @@ def test_with_invalid_single_file(self, pickle_persistence, invalid_pickle_files def test_with_good_multi_file(self, pickle_persistence, good_pickle_files): user_data = pickle_persistence.get_user_data() - assert isinstance(user_data, defaultdict) + assert isinstance(user_data, dict) assert user_data[12345]['test1'] == 'test2' assert user_data[67890][3] == 'test4' - assert user_data[54321] == {} chat_data = pickle_persistence.get_chat_data() - assert isinstance(chat_data, defaultdict) + assert isinstance(chat_data, dict) assert chat_data[-12345]['test1'] == 'test2' assert chat_data[-67890][3] == 'test4' - assert chat_data[-54321] == {} bot_data = pickle_persistence.get_bot_data() assert isinstance(bot_data, dict) @@ -1163,16 +1154,14 @@ def test_with_good_multi_file(self, pickle_persistence, good_pickle_files): def test_with_good_single_file(self, pickle_persistence, good_pickle_files): pickle_persistence.single_file = True user_data = pickle_persistence.get_user_data() - assert isinstance(user_data, defaultdict) + assert isinstance(user_data, dict) assert user_data[12345]['test1'] == 'test2' assert user_data[67890][3] == 'test4' - assert user_data[54321] == {} chat_data = pickle_persistence.get_chat_data() - assert isinstance(chat_data, defaultdict) + assert isinstance(chat_data, dict) assert chat_data[-12345]['test1'] == 'test2' assert chat_data[-67890][3] == 'test4' - assert chat_data[-54321] == {} bot_data = pickle_persistence.get_bot_data() assert isinstance(bot_data, dict) @@ -1200,16 +1189,14 @@ def test_with_good_single_file(self, pickle_persistence, good_pickle_files): def test_with_multi_file_wo_bot_data(self, pickle_persistence, pickle_files_wo_bot_data): user_data = pickle_persistence.get_user_data() - assert isinstance(user_data, defaultdict) + assert isinstance(user_data, dict) assert user_data[12345]['test1'] == 'test2' assert user_data[67890][3] == 'test4' - assert user_data[54321] == {} chat_data = pickle_persistence.get_chat_data() - assert isinstance(chat_data, defaultdict) + assert isinstance(chat_data, dict) assert chat_data[-12345]['test1'] == 'test2' assert chat_data[-67890][3] == 'test4' - assert chat_data[-54321] == {} bot_data = pickle_persistence.get_bot_data() assert isinstance(bot_data, dict) @@ -1237,16 +1224,14 @@ def test_with_multi_file_wo_callback_data( self, pickle_persistence, pickle_files_wo_callback_data ): user_data = pickle_persistence.get_user_data() - assert isinstance(user_data, defaultdict) + assert isinstance(user_data, dict) assert user_data[12345]['test1'] == 'test2' assert user_data[67890][3] == 'test4' - assert user_data[54321] == {} chat_data = pickle_persistence.get_chat_data() - assert isinstance(chat_data, defaultdict) + assert isinstance(chat_data, dict) assert chat_data[-12345]['test1'] == 'test2' assert chat_data[-67890][3] == 'test4' - assert chat_data[-54321] == {} bot_data = pickle_persistence.get_bot_data() assert isinstance(bot_data, dict) @@ -1273,16 +1258,14 @@ def test_with_multi_file_wo_callback_data( def test_with_single_file_wo_bot_data(self, pickle_persistence, pickle_files_wo_bot_data): pickle_persistence.single_file = True user_data = pickle_persistence.get_user_data() - assert isinstance(user_data, defaultdict) + assert isinstance(user_data, dict) assert user_data[12345]['test1'] == 'test2' assert user_data[67890][3] == 'test4' - assert user_data[54321] == {} chat_data = pickle_persistence.get_chat_data() - assert isinstance(chat_data, defaultdict) + assert isinstance(chat_data, dict) assert chat_data[-12345]['test1'] == 'test2' assert chat_data[-67890][3] == 'test4' - assert chat_data[-54321] == {} bot_data = pickle_persistence.get_bot_data() assert isinstance(bot_data, dict) @@ -1310,16 +1293,14 @@ def test_with_single_file_wo_callback_data( self, pickle_persistence, pickle_files_wo_callback_data ): user_data = pickle_persistence.get_user_data() - assert isinstance(user_data, defaultdict) + assert isinstance(user_data, dict) assert user_data[12345]['test1'] == 'test2' assert user_data[67890][3] == 'test4' - assert user_data[54321] == {} chat_data = pickle_persistence.get_chat_data() - assert isinstance(chat_data, defaultdict) + assert isinstance(chat_data, dict) assert chat_data[-12345]['test1'] == 'test2' assert chat_data[-67890][3] == 'test4' - assert chat_data[-54321] == {} bot_data = pickle_persistence.get_bot_data() assert isinstance(bot_data, dict) @@ -1353,7 +1334,7 @@ def test_updating_multi_file(self, pickle_persistence, good_pickle_files): pickle_persistence.update_user_data(12345, user_data[12345]) assert pickle_persistence.user_data == user_data with Path('pickletest_user_data').open('rb') as f: - user_data_test = defaultdict(dict, pickle.load(f)) + user_data_test = dict(pickle.load(f)) assert user_data_test == user_data pickle_persistence.drop_user_data(67890) assert 67890 not in pickle_persistence.get_user_data() @@ -1367,7 +1348,7 @@ def test_updating_multi_file(self, pickle_persistence, good_pickle_files): pickle_persistence.update_chat_data(-12345, chat_data[-12345]) assert pickle_persistence.chat_data == chat_data with Path('pickletest_chat_data').open('rb') as f: - chat_data_test = defaultdict(dict, pickle.load(f)) + chat_data_test = dict(pickle.load(f)) assert chat_data_test == chat_data pickle_persistence.drop_chat_data(-67890) assert -67890 not in pickle_persistence.get_chat_data() @@ -1403,7 +1384,7 @@ def test_updating_multi_file(self, pickle_persistence, good_pickle_files): assert pickle_persistence.conversations['name1'] == conversation1 assert pickle_persistence.get_conversations('name1') == conversation1 with Path('pickletest_conversations').open('rb') as f: - conversations_test = defaultdict(dict, pickle.load(f)) + conversations_test = dict(pickle.load(f)) assert conversations_test['name1'] == conversation1 pickle_persistence.conversations = None @@ -1423,7 +1404,7 @@ def test_updating_single_file(self, pickle_persistence, good_pickle_files): pickle_persistence.update_user_data(12345, user_data[12345]) assert pickle_persistence.user_data == user_data with Path('pickletest').open('rb') as f: - user_data_test = defaultdict(dict, pickle.load(f)['user_data']) + user_data_test = dict(pickle.load(f))['user_data'] assert user_data_test == user_data pickle_persistence.drop_user_data(67890) assert 67890 not in pickle_persistence.get_user_data() @@ -1437,7 +1418,7 @@ def test_updating_single_file(self, pickle_persistence, good_pickle_files): pickle_persistence.update_chat_data(-12345, chat_data[-12345]) assert pickle_persistence.chat_data == chat_data with Path('pickletest').open('rb') as f: - chat_data_test = defaultdict(dict, pickle.load(f)['chat_data']) + chat_data_test = dict(pickle.load(f))['chat_data'] assert chat_data_test == chat_data pickle_persistence.drop_chat_data(-67890) assert -67890 not in pickle_persistence.get_chat_data() @@ -1473,7 +1454,7 @@ def test_updating_single_file(self, pickle_persistence, good_pickle_files): assert pickle_persistence.conversations['name1'] == conversation1 assert pickle_persistence.get_conversations('name1') == conversation1 with Path('pickletest').open('rb') as f: - conversations_test = defaultdict(dict, pickle.load(f)['conversations']) + conversations_test = dict(pickle.load(f))['conversations'] assert conversations_test['name1'] == conversation1 pickle_persistence.conversations = None @@ -1502,6 +1483,7 @@ def test_save_on_flush_multi_files(self, pickle_persistence, good_pickle_files): pickle_persistence.on_flush = True user_data = pickle_persistence.get_user_data() + user_data[54321] = {} user_data[54321]['test9'] = 'test 10' assert not pickle_persistence.user_data == user_data @@ -1512,10 +1494,11 @@ def test_save_on_flush_multi_files(self, pickle_persistence, good_pickle_files): assert pickle_persistence.user_data == user_data with Path('pickletest_user_data').open('rb') as f: - user_data_test = defaultdict(dict, pickle.load(f)) + user_data_test = dict(pickle.load(f)) assert not user_data_test == user_data chat_data = pickle_persistence.get_chat_data() + chat_data[54321] = {} chat_data[54321]['test9'] = 'test 10' assert not pickle_persistence.chat_data == chat_data @@ -1526,7 +1509,7 @@ def test_save_on_flush_multi_files(self, pickle_persistence, good_pickle_files): assert pickle_persistence.user_data == user_data with Path('pickletest_chat_data').open('rb') as f: - chat_data_test = defaultdict(dict, pickle.load(f)) + chat_data_test = dict(pickle.load(f)) assert not chat_data_test == chat_data bot_data = pickle_persistence.get_bot_data() @@ -1559,16 +1542,16 @@ def test_save_on_flush_multi_files(self, pickle_persistence, good_pickle_files): assert pickle_persistence.conversations['name1'] == conversation1 with Path('pickletest_conversations').open('rb') as f: - conversations_test = defaultdict(dict, pickle.load(f)) + conversations_test = dict(pickle.load(f)) assert not conversations_test['name1'] == conversation1 pickle_persistence.flush() with Path('pickletest_user_data').open('rb') as f: - user_data_test = defaultdict(dict, pickle.load(f)) + user_data_test = dict(pickle.load(f)) assert user_data_test == user_data with Path('pickletest_chat_data').open('rb') as f: - chat_data_test = defaultdict(dict, pickle.load(f)) + chat_data_test = dict(pickle.load(f)) assert chat_data_test == chat_data with Path('pickletest_bot_data').open('rb') as f: @@ -1576,7 +1559,7 @@ def test_save_on_flush_multi_files(self, pickle_persistence, good_pickle_files): assert bot_data_test == bot_data with Path('pickletest_conversations').open('rb') as f: - conversations_test = defaultdict(dict, pickle.load(f)) + conversations_test = dict(pickle.load(f)) assert conversations_test['name1'] == conversation1 def test_save_on_flush_single_files(self, pickle_persistence, good_pickle_files): @@ -1587,21 +1570,23 @@ def test_save_on_flush_single_files(self, pickle_persistence, good_pickle_files) pickle_persistence.single_file = True user_data = pickle_persistence.get_user_data() + user_data[54321] = {} user_data[54321]['test9'] = 'test 10' assert not pickle_persistence.user_data == user_data pickle_persistence.update_user_data(54321, user_data[54321]) assert pickle_persistence.user_data == user_data with Path('pickletest').open('rb') as f: - user_data_test = defaultdict(dict, pickle.load(f)['user_data']) + user_data_test = dict(pickle.load(f))['user_data'] assert not user_data_test == user_data chat_data = pickle_persistence.get_chat_data() + chat_data[54321] = {} chat_data[54321]['test9'] = 'test 10' assert not pickle_persistence.chat_data == chat_data pickle_persistence.update_chat_data(54321, chat_data[54321]) assert pickle_persistence.chat_data == chat_data with Path('pickletest').open('rb') as f: - chat_data_test = defaultdict(dict, pickle.load(f)['chat_data']) + chat_data_test = dict(pickle.load(f))['chat_data'] assert not chat_data_test == chat_data bot_data = pickle_persistence.get_bot_data() @@ -1628,16 +1613,16 @@ def test_save_on_flush_single_files(self, pickle_persistence, good_pickle_files) pickle_persistence.update_conversation('name1', (123, 123), 5) assert pickle_persistence.conversations['name1'] == conversation1 with Path('pickletest').open('rb') as f: - conversations_test = defaultdict(dict, pickle.load(f)['conversations']) + conversations_test = dict(pickle.load(f))['conversations'] assert not conversations_test['name1'] == conversation1 pickle_persistence.flush() with Path('pickletest').open('rb') as f: - user_data_test = defaultdict(dict, pickle.load(f)['user_data']) + user_data_test = dict(pickle.load(f))['user_data'] assert user_data_test == user_data with Path('pickletest').open('rb') as f: - chat_data_test = defaultdict(dict, pickle.load(f)['chat_data']) + chat_data_test = dict(pickle.load(f))['chat_data'] assert chat_data_test == chat_data with Path('pickletest').open('rb') as f: @@ -1645,7 +1630,7 @@ def test_save_on_flush_single_files(self, pickle_persistence, good_pickle_files) assert bot_data_test == bot_data with Path('pickletest').open('rb') as f: - conversations_test = defaultdict(dict, pickle.load(f)['conversations']) + conversations_test = dict(pickle.load(f))['conversations'] assert conversations_test['name1'] == conversation1 def test_with_handler(self, bot, update, bot_data, pickle_persistence, good_pickle_files): @@ -1925,10 +1910,6 @@ def test_with_context_types(self, ud, cd, bd, singlefile): cc = ContextTypes(user_data=ud, chat_data=cd, bot_data=bd) persistence = PicklePersistence('pickletest', single_file=singlefile, context_types=cc) - assert isinstance(persistence.get_user_data()[1], ud) - assert persistence.get_user_data()[1] == 0 - assert isinstance(persistence.get_chat_data()[1], cd) - assert persistence.get_chat_data()[1] == 0 assert isinstance(persistence.get_bot_data(), bd) assert persistence.get_bot_data() == 0 @@ -1936,8 +1917,8 @@ def test_with_context_types(self, ud, cd, bd, singlefile): persistence.chat_data = None persistence.drop_user_data(123) persistence.drop_chat_data(123) - assert isinstance(persistence.get_user_data(), defaultdict) - assert isinstance(persistence.get_chat_data(), defaultdict) + assert isinstance(persistence.get_user_data(), dict) + assert isinstance(persistence.get_chat_data(), dict) persistence.user_data = None persistence.chat_data = None persistence.update_user_data(1, ud(1)) @@ -1993,8 +1974,8 @@ def test_slot_behaviour(self, mro_slots, recwarn): def test_no_json_given(self): dict_persistence = DictPersistence() - assert dict_persistence.get_user_data() == defaultdict(dict) - assert dict_persistence.get_chat_data() == defaultdict(dict) + assert dict_persistence.get_user_data() == {} + assert dict_persistence.get_chat_data() == {} assert dict_persistence.get_bot_data() == {} assert dict_persistence.get_callback_data() is None assert dict_persistence.get_conversations('noname') == {} @@ -2055,16 +2036,14 @@ def test_good_json_input( callback_data_json=callback_data_json, ) user_data = dict_persistence.get_user_data() - assert isinstance(user_data, defaultdict) + assert isinstance(user_data, dict) assert user_data[12345]['test1'] == 'test2' assert user_data[67890][3] == 'test4' - assert user_data[54321] == {} chat_data = dict_persistence.get_chat_data() - assert isinstance(chat_data, defaultdict) + assert isinstance(chat_data, dict) assert chat_data[-12345]['test1'] == 'test2' assert chat_data[-67890][3] == 'test4' - assert chat_data[-54321] == {} bot_data = dict_persistence.get_bot_data() assert isinstance(bot_data, dict) @@ -2169,7 +2148,7 @@ def test_updating( assert 67890 not in dict_persistence.user_data dict_persistence._user_data = None dict_persistence.drop_user_data(123) - assert isinstance(dict_persistence.get_user_data(), defaultdict) + assert isinstance(dict_persistence.get_user_data(), dict) chat_data = dict_persistence.get_chat_data() chat_data[-12345]['test3']['test4'] = 'test6' @@ -2186,7 +2165,7 @@ def test_updating( assert -67890 not in dict_persistence.chat_data dict_persistence._chat_data = None dict_persistence.drop_chat_data(123) - assert isinstance(dict_persistence.get_chat_data(), defaultdict) + assert isinstance(dict_persistence.get_chat_data(), dict) bot_data = dict_persistence.get_bot_data() bot_data['test3']['test4'] = 'test6' From e2558eb6fccf586a5f0368176263f1e43436199c Mon Sep 17 00:00:00 2001 From: Harshil <37377066+harshil21@users.noreply.github.com> Date: Sat, 22 Jan 2022 02:47:24 +0530 Subject: [PATCH 2/6] some rearrangment in dispatcher --- telegram/ext/_dispatcher.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/telegram/ext/_dispatcher.py b/telegram/ext/_dispatcher.py index f635cf32cf9..859384d1a6f 100644 --- a/telegram/ext/_dispatcher.py +++ b/telegram/ext/_dispatcher.py @@ -321,7 +321,7 @@ def get_instance(cls) -> 'Dispatcher': def _pooled(self) -> None: thr_name = current_thread().name - while 1: + while True: promise = self.__async_queue.get() # If unpacking fails, the thread pool is being closed from Updater._join_async_threads @@ -417,7 +417,7 @@ def start(self, ready: Event = None) -> None: if ready is not None: ready.set() - while 1: + while True: try: # Pop update from update queue. update = self.update_queue.get(True, 1) @@ -527,9 +527,9 @@ def process_update(self, update: object) -> None: self.logger.debug('Error handler stopped further handlers.') break - # Update persistence, if handled - handled_only_async = all(sync_modes) + # Update persistence, if an update was handled if handled: + handled_only_async = all(sync_modes) # Respect default settings if ( all(mode is DEFAULT_FALSE for mode in sync_modes) @@ -739,7 +739,7 @@ def update_persistence(self, update: object = None) -> None: def __update_persistence(self, update: object = None) -> None: if self.persistence: - # We use list() here in order to decouple chat_ids from self._chat_data, as dict view + # We use list() here in order to decouple chat_ids from self.chat_data, as dict view # objects will change, when the dict does and we want to loop over chat_ids chat_ids = list(self.chat_data.keys()) user_ids = list(self.user_data.keys()) @@ -856,10 +856,10 @@ def dispatch_error( :obj:`bool`: :obj:`True` if one of the error handlers raised :class:`telegram.ext.DispatcherHandlerStop`. :obj:`False`, otherwise. """ - async_args = None if not promise else promise.args - async_kwargs = None if not promise else promise.kwargs - if self.error_handlers: + async_args = None if not promise else promise.args + async_kwargs = None if not promise else promise.kwargs + for ( callback, run_async, From b62576073c76c5677c46f9d81d0558f26c8cf119 Mon Sep 17 00:00:00 2001 From: Harshil <37377066+harshil21@users.noreply.github.com> Date: Sat, 22 Jan 2022 22:09:37 +0530 Subject: [PATCH 3/6] keep self.chat_data --- telegram/ext/_dispatcher.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/telegram/ext/_dispatcher.py b/telegram/ext/_dispatcher.py index 859384d1a6f..9d9d2e9a5aa 100644 --- a/telegram/ext/_dispatcher.py +++ b/telegram/ext/_dispatcher.py @@ -321,7 +321,7 @@ def get_instance(cls) -> 'Dispatcher': def _pooled(self) -> None: thr_name = current_thread().name - while True: + while 1: promise = self.__async_queue.get() # If unpacking fails, the thread pool is being closed from Updater._join_async_threads @@ -417,7 +417,7 @@ def start(self, ready: Event = None) -> None: if ready is not None: ready.set() - while True: + while 1: try: # Pop update from update queue. update = self.update_queue.get(True, 1) @@ -527,9 +527,9 @@ def process_update(self, update: object) -> None: self.logger.debug('Error handler stopped further handlers.') break - # Update persistence, if an update was handled + # Update persistence, if handled + handled_only_async = all(sync_modes) if handled: - handled_only_async = all(sync_modes) # Respect default settings if ( all(mode is DEFAULT_FALSE for mode in sync_modes) @@ -856,10 +856,10 @@ def dispatch_error( :obj:`bool`: :obj:`True` if one of the error handlers raised :class:`telegram.ext.DispatcherHandlerStop`. :obj:`False`, otherwise. """ - if self.error_handlers: - async_args = None if not promise else promise.args - async_kwargs = None if not promise else promise.kwargs + async_args = None if not promise else promise.args + async_kwargs = None if not promise else promise.kwargs + if self.error_handlers: for ( callback, run_async, From 128dd0ca300fc1ccfb7c9a574c1da9cdef1b58c0 Mon Sep 17 00:00:00 2001 From: Harshil <37377066+harshil21@users.noreply.github.com> Date: Sat, 22 Jan 2022 22:28:07 +0530 Subject: [PATCH 4/6] review --- telegram/ext/_basepersistence.py | 4 ++-- telegram/ext/_picklepersistence.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/telegram/ext/_basepersistence.py b/telegram/ext/_basepersistence.py index 47f2f7b40c3..a6d588cf0f2 100644 --- a/telegram/ext/_basepersistence.py +++ b/telegram/ext/_basepersistence.py @@ -132,10 +132,10 @@ def __new__( update_bot_data = instance.update_bot_data update_callback_data = instance.update_callback_data - def get_user_data_insert_bot() -> Union[DefaultDict[int, UD], Dict[int, UD]]: + def get_user_data_insert_bot() -> Dict[int, UD]: return instance.insert_bot(get_user_data()) - def get_chat_data_insert_bot() -> Union[DefaultDict[int, CD], Dict[int, CD]]: + def get_chat_data_insert_bot() -> Dict[int, CD]: return instance.insert_bot(get_chat_data()) def get_bot_data_insert_bot() -> BD: diff --git a/telegram/ext/_picklepersistence.py b/telegram/ext/_picklepersistence.py index 0bb2a4544e4..bf2d6b70dc2 100644 --- a/telegram/ext/_picklepersistence.py +++ b/telegram/ext/_picklepersistence.py @@ -319,7 +319,7 @@ def update_user_data(self, user_id: int, data: UD) -> None: """ if self.user_data is None: self.user_data = {} - if self.user_data.get(user_id, None) == data: # type: ignore[arg-type] + if self.user_data.get(user_id) == data: return self.user_data[user_id] = data if not self.on_flush: @@ -337,7 +337,7 @@ def update_chat_data(self, chat_id: int, data: CD) -> None: """ if self.chat_data is None: self.chat_data = {} - if self.chat_data.get(chat_id, None) == data: # type: ignore[arg-type] + if self.chat_data.get(chat_id) == data: return self.chat_data[chat_id] = data if not self.on_flush: From 3b87696dca86207dd51ab28e76cdbdb04d467329 Mon Sep 17 00:00:00 2001 From: Harshil <37377066+harshil21@users.noreply.github.com> Date: Sat, 22 Jan 2022 22:44:36 +0530 Subject: [PATCH 5/6] versionchanged tags --- telegram/ext/_basepersistence.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/telegram/ext/_basepersistence.py b/telegram/ext/_basepersistence.py index a6d588cf0f2..2236b608b28 100644 --- a/telegram/ext/_basepersistence.py +++ b/telegram/ext/_basepersistence.py @@ -19,7 +19,7 @@ """This module contains the BasePersistence class.""" from abc import ABC, abstractmethod from copy import copy -from typing import Dict, Optional, Tuple, cast, ClassVar, Generic, DefaultDict, NamedTuple, Union +from typing import Dict, Optional, Tuple, cast, ClassVar, Generic, NamedTuple from telegram import Bot from telegram.ext import ExtBot @@ -395,7 +395,7 @@ def _insert_bot(self, obj: object, memo: Dict[int, object]) -> object: return obj @abstractmethod - def get_user_data(self) -> Union[DefaultDict[int, UD], Dict[int, UD]]: + def get_user_data(self) -> Dict[int, UD]: """Will be called by :class:`telegram.ext.Dispatcher` upon creation with a persistence object. It should return the ``user_data`` if stored, or an empty :obj:`defaultdict` or :obj:`dict`. In the latter case, the dictionary should produce values @@ -405,13 +405,16 @@ def get_user_data(self) -> Union[DefaultDict[int, UD], Dict[int, UD]]: * The type from :attr:`telegram.ext.ContextTypes.user_data` if :class:`telegram.ext.ContextTypes` is used. + .. versionchanged:: 14.0 + This method may now return a :obj:`dict` instead of a :obj:`collections.defaultdict` + Returns: Dict[:obj:`int`, :obj:`dict` | :attr:`telegram.ext.ContextTypes.user_data`]: The restored user data. """ @abstractmethod - def get_chat_data(self) -> Union[DefaultDict[int, CD], Dict[int, CD]]: + def get_chat_data(self) -> Dict[int, CD]: """Will be called by :class:`telegram.ext.Dispatcher` upon creation with a persistence object. It should return the ``chat_data`` if stored, or an empty :obj:`defaultdict` or :obj:`dict`. In the latter case, the dictionary should produce values @@ -421,6 +424,9 @@ def get_chat_data(self) -> Union[DefaultDict[int, CD], Dict[int, CD]]: * The type from :attr:`telegram.ext.ContextTypes.chat_data` if :class:`telegram.ext.ContextTypes` is used. + .. versionchanged:: 14.0 + This method may now return a :obj:`dict` instead of a :obj:`collections.defaultdict` + Returns: Dict[:obj:`int`, :obj:`dict` | :attr:`telegram.ext.ContextTypes.chat_data`]: The restored chat data. From 2978f7e401483ac4e9f9194b083a58445512e3cd Mon Sep 17 00:00:00 2001 From: Harshil <37377066+harshil21@users.noreply.github.com> Date: Sun, 23 Jan 2022 03:15:35 +0530 Subject: [PATCH 6/6] review 2 --- telegram/ext/_basepersistence.py | 4 ++-- telegram/ext/_dictpersistence.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/telegram/ext/_basepersistence.py b/telegram/ext/_basepersistence.py index 2236b608b28..598d84dd264 100644 --- a/telegram/ext/_basepersistence.py +++ b/telegram/ext/_basepersistence.py @@ -398,7 +398,7 @@ def _insert_bot(self, obj: object, memo: Dict[int, object]) -> object: def get_user_data(self) -> Dict[int, UD]: """Will be called by :class:`telegram.ext.Dispatcher` upon creation with a persistence object. It should return the ``user_data`` if stored, or an empty - :obj:`defaultdict` or :obj:`dict`. In the latter case, the dictionary should produce values + :obj:`dict`. In the latter case, the dictionary should produce values corresponding to one of the following: * :obj:`dict` @@ -417,7 +417,7 @@ def get_user_data(self) -> Dict[int, UD]: def get_chat_data(self) -> Dict[int, CD]: """Will be called by :class:`telegram.ext.Dispatcher` upon creation with a persistence object. It should return the ``chat_data`` if stored, or an empty - :obj:`defaultdict` or :obj:`dict`. In the latter case, the dictionary should produce values + :obj:`dict`. In the latter case, the dictionary should produce values corresponding to one of the following: * :obj:`dict` diff --git a/telegram/ext/_dictpersistence.py b/telegram/ext/_dictpersistence.py index 48110618600..6ea7bb17c53 100644 --- a/telegram/ext/_dictpersistence.py +++ b/telegram/ext/_dictpersistence.py @@ -313,7 +313,7 @@ def update_user_data(self, user_id: int, data: Dict) -> None: """ if self._user_data is None: self._user_data = {} - if self._user_data.get(user_id, None) == data: + if self._user_data.get(user_id) == data: return self._user_data[user_id] = data self._user_data_json = None @@ -327,7 +327,7 @@ def update_chat_data(self, chat_id: int, data: Dict) -> None: """ if self._chat_data is None: self._chat_data = {} - if self._chat_data.get(chat_id, None) == data: + if self._chat_data.get(chat_id) == data: return self._chat_data[chat_id] = data self._chat_data_json = None
Note: This service is not intended for secure transactions such as banking, social media, email, or purchasing. Use at your own risk. We assume no liability whatsoever for broken pages.
Alternative Proxies: