python-telegram-bot · Bibo-Joshi · Feb 11, 2023 · Dec 8, 2022 · Dec 9, 2022 · Dec 9, 2022
diff --git a/.github/CONTRIBUTING.rst b/.github/CONTRIBUTING.rst
@@ -84,35 +84,13 @@ Here's how to make a one-off code change.
 
    -  In addition, PTB uses some formatting/styling and linting tools in the pre-commit setup. Some of those tools also have command line tools that can help to run these tools outside of the pre-commit step. If you'd like to leverage that, please have a look at the `pre-commit config file`_ for an overview of which tools (and which versions of them) are used. For example, we use `Black`_ for code formatting. Plugins for Black exist for some `popular editors`_. You can use those instead of manually formatting everything.
 
-   - Please ensure that the code you write is well-tested.
+   - Please ensure that the code you write is well-tested and that all automated tests still pass. We
+     have dedicated an `testing page`_ to help you with that.
 
-        - In addition to that, we provide the `dev` marker for pytest. If you write one or multiple tests and want to run only those, you can decorate them via `@pytest.mark.dev` and then run it with minimal overhead with `pytest ./path/to/test_file.py -m dev`.
-
-   - Don’t break backward compatibility.
+   - Don't break backward compatibility.
 
    - Add yourself to the AUTHORS.rst_ file in an alphabetical fashion.
 
-   - Before making a commit ensure that all automated tests still pass:
-
-     .. code-block:: bash
-
-        $ pytest -v
-
-     Since the tests can take a while to run, you can speed things up by running them in parallel
-     using `pytest-xdist`_ (note that this may effect the result of the test in some rare cases):
-
-     .. code-block:: bash
-
-        $ pytest -v -n auto --dist=loadfile
-
-     To run ``test_official`` (particularly useful if you made API changes), run
-
-     .. code-block:: bash
-
-        $ export TEST_OFFICIAL=true
-
-     prior to running the tests.
-
    - If you want run style & type checks before committing run
 
      .. code-block:: bash
@@ -287,4 +265,4 @@ break the API classes. For example:
 .. _`RTD build`: https://docs.python-telegram-bot.org/en/doc-fixes
 .. _`CSI`: https://standards.mousepawmedia.com/en/stable/csi.html
 .. _`section`: #documenting
-.. _`pytest-xdist`: https://github.com/pytest-dev/pytest-xdist
+.. _`testing page`: https://github.com/python-telegram-bot/python-telegram-bot/blob/master/tests/README.rst
diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
@@ -44,42 +44,23 @@ jobs:
         # The first & second one are achieved by mocking the corresponding import
         # See test_helpers.py & test_no_passport.py for details
         run: |
-          # Test without passport
-          pytest -v --cov -k test_no_passport.py
+          # We test without optional dependencies first. This includes:
+          # - without pytz
+          # - without jobqueue
+          # - without ratelimiter
+          # - without webhooks
+          # - without arbitrary callback data
+          # - without socks suppport
+          TO_TEST="test_no_passport.py or test_datetime.py or test_defaults.py or test_jobqueue.py or test_applicationbuilder.py or test_ratelimiter.py or test_updater.py or test_callbackdatacache.py or test_request.py"
+          pytest -v --cov -k "${TO_TEST}"
           status=$?
 
-          # test without pytz
-          pytest -v --cov --cov-append -k test_datetime.py
-          status=$(( $? > status ? $? : status))
-          pytest -v --cov --cov-append -k test_defaults.py
-          status=$(( $? > status ? $? : status))
-
-          # test without pytz & jobqueue
-          pytest -v --cov --cov-append -k test_jobqueue.py
-          pytest -v --cov --cov-append -k test_applicationbuilder.py
-          status=$(( $? > status ? $? : status))
-
-          # Test without ratelimiter
-          pytest -v --cov --cov-append -k test_ratelimiter.py
-          status=$(( $? > status ? $? : status))
-
-          # Test without webhooks
-          pytest -v --cov --cov-append -k test_updater.py
-          status=$(( $? > status ? $? : status))
-
-          # Test without callback-data
-          pytest -v --cov --cov-append -k test_callbackdatacache.py
-          status=$(( $? > status ? $? : status))
-
-          # Test without socks
-          pytest -v --cov --cov-append -k test_request.py
-          status=$(( $? > status ? $? : status))
-
           # Test the rest
           export TEST_WITH_OPT_DEPS='true'
           pip install -r requirements-opts.txt
           # `-n auto --dist loadfile` uses pytest-xdist to run each test file on a different CPU
-          # worker
+          # worker. Increasing number of workers has little effect on test duration, but it seems
+          # to increase flakyness, specially on python 3.7 with --dist=loadgroup.
           pytest -v --cov --cov-append -n auto --dist loadfile
           status=$(( $? > status ? $? : status))
           exit ${status}

diff --git a/.github/workflows/type_completeness.yml b/.github/workflows/type_completeness.yml
@@ -37,7 +37,7 @@ jobs:
       - name: Compare Completeness
         uses: jannekem/run-python-script-action@v1
         with:
-          script: | 
+          script: |
             import json
             import os
             from pathlib import Path

diff --git a/docs/source/index.rst b/docs/source/index.rst
@@ -32,7 +32,6 @@
    GitHub Repository <https://github.com/python-telegram-bot/python-telegram-bot/>
    Telegram Channel <https://t.me/pythontelegrambotchannel/>
    Telegram User Group <https://t.me/pythontelegrambotgroup/>
-   contributing
    coc
-
-
+   contributing
+   testing
diff --git a/docs/source/testing.rst b/docs/source/testing.rst
@@ -0,0 +1 @@
+.. include:: ../../tests/README.rst
diff --git a/requirements-dev.txt b/requirements-dev.txt
@@ -1,10 +1,10 @@
-pre-commit
+pre-commit  # needed for pre-commit hooks in the git commit command
 
+# For the test suite
 pytest==7.2.1
-pytest-asyncio==0.20.3
-pytest-timeout==2.1.0 # used to timeout tests
+pytest-asyncio==0.20.3  # needed because pytest doesn't come with native support for coroutines as tests
 pytest-xdist==3.1.0  # xdist runs tests in parallel
-
 flaky  # Used for flaky tests (flaky decorator)
 beautifulsoup4  # used in test_official for parsing tg docs
+
 wheel  # required for building the wheels for releases
diff --git a/setup.cfg b/setup.cfg
@@ -36,7 +36,10 @@ filterwarnings =
 ;    Unfortunately due to https://github.com/pytest-dev/pytest/issues/8343 we can't have this here
 ;    and instead do a trick directly in tests/conftest.py
 ;    ignore::telegram.utils.deprecate.TelegramDeprecationWarning
-markers = dev: If you want to test a specific test, use this
+markers =
+    dev: If you want to test a specific test, use this
+    no_req
+    req
 asyncio_mode = auto
 
 [coverage:run]

diff --git a/tests/README.rst b/tests/README.rst
@@ -0,0 +1,101 @@
+==============
+Testing in PTB
+==============
+
+PTB uses `pytest`_ for testing. To run the tests, you need to
+have pytest installed along with a few other dependencies. You can find the list of dependencies
+in the ``requirements-dev.txt`` file in the root of the repository.
+
+Running tests
+=============
+
+To run the entire test suite, you can use the following command:
+
+.. code-block:: bash
+
+    $ pytest
+
+This will run all the tests, including the ones which make a request to the Telegram servers, which
+may take a long time (total > 13 mins). To run only the tests that don't require a connection, you
+can run the following command:
+
+.. code-block:: bash
+
+    $ pytest -m no_req
+
+Or alternatively, you can run the following command to run only the tests that require a connection:
+
+.. code-block:: bash
+
+    $ pytest -m req
+
+To further speed up the tests, you can run them in parallel using the ``-n`` flag (requires `pytest-xdist`_). But beware that
+this will use multiple CPU cores on your machine. The ``--dist`` flag is used to specify how the
+tests will be distributed across the cores. The ``loadgroup`` option is used to distribute the tests
+such that tests marked with ``@pytest.mark.xdist_group("name")`` are run on the same core — important if you want avoid race conditions in some tests:
+
+.. code-block:: bash
+
+    $ pytest -n auto --dist=loadgroup
+
+This will result in a significant speedup, but may cause some tests to fail. If you want to run
+the failed tests in isolation, you can use the ``--lf`` flag:
+
+.. code-block:: bash
+
+    $ pytest --lf
+
+
+Writing tests
+=============
+
+PTB has a separate test file for every file in the ``telegram.*`` namespace. Further, the tests for
+the ``telegram`` module are split into two classes, based on whether the test methods in them make a
+request or not. When writing tests, make sure to split them into these two classes, and make sure
+to name the test class as: ``TestXXXWithoutRequest`` for tests that don't make a request, and ``TestXXXWithRequest`` for tests that do.
+
+Writing tests is a creative process; allowing you to design your test however you'd like, but there
+are a few conventions that you should follow:
+
+- Each new test class needs a ``test_slot_behaviour``, ``test_to_dict``, ``test_de_json`` and
+  ``test_equality`` (in most cases).
+
+- Make use of pytest's fixtures and parametrize wherever possible. Having knowledge of pytest's
+  tooling can help you as well. You can look at the existing tests for examples and inspiration.
+
+If you have made some API changes, you may want to run ``test_official`` to validate that the changes are
+complete and correct. To run it, export an environment variable first:
+
+.. code-block:: bash
+
+    $ export TEST_OFFICIAL=true
+
+and then run ``pytest tests/test_official.py``.
+
+We also have another marker, ``@pytest.mark.dev``, which you can add to tests that you want to run selectively.
+Use as follows:
+
+.. code-block:: bash
+
+    $ pytest -m dev
+
+
+Bots used in tests
+==================
+
+If you run the tests locally, the test setup will use one of the two public bots available. Which
+bot of the two gets chosen for the test session is random. Whereas when the tests on the
+Github Actions CI are run, the test setup allocates a different, but same bot for every combination of Python version and
+OS.
+
+Thus, number of bots used for testing locally is 2 (called as fallback bots), and on the CI,
+its [3.7, 3.8, 3.9, 3.10, 3.11] x [ubuntu-latest, macos-latest, windows-latest] = 15. Bringing the
+total number of bots used for testing to 17.
+
+
+That's it! If you have any questions, feel free to ask them in the `PTB dev
+group`_.
+
+.. _pytest: https://docs.pytest.org/en/stable/
+.. _pytest-xdist: https://pypi.org/project/pytest-xdist/
+.. _PTB dev group: https://t.me/pythontelegrambotgroup
diff --git a/tests/bots.py b/tests/bots.py
@@ -45,26 +45,27 @@
     BOTS = json.loads(base64.b64decode(BOTS).decode("utf-8"))
     JOB_INDEX = int(JOB_INDEX)
 
-FALLBACKS = json.loads(base64.b64decode(FALLBACKS).decode("utf-8"))
+FALLBACKS = json.loads(base64.b64decode(FALLBACKS).decode("utf-8"))  # type: list[dict[str, str]]
 
 
-def get(name, fallback):
-    # If we have TOKEN, PAYMENT_PROVIDER_TOKEN, CHAT_ID, SUPER_GROUP_ID,
-    # CHANNEL_ID, BOT_NAME, or BOT_USERNAME in the environment, then use that
-    val = os.getenv(name.upper())
-    if val:
-        return val
+class BotInfoProvider:
+    def __init__(self):
+        self._cached = {}
 
+    def get_info(self):
+        if self._cached:
+            return self._cached
+        self._cached = {k: get(k, v) for k, v in random.choice(FALLBACKS).items()}
+        return self._cached
+
+
+def get(key, fallback):
     # If we're running as a github action then fetch bots from the repo secrets
     if GITHUB_ACTION is not None and BOTS is not None and JOB_INDEX is not None:
         try:
-            return BOTS[JOB_INDEX][name]
-        except (KeyError, IndexError):
+            return BOTS[JOB_INDEX][key]
+        except (IndexError, KeyError):
             pass
 
     # Otherwise go with the fallback
     return fallback
-
-
-def get_bot():
-    return {k: get(k, v) for k, v in random.choice(FALLBACKS).items()}