From aecdcfb11de8d82bf7cb5bb5d0b327bfa058da5b Mon Sep 17 00:00:00 2001 From: "d.kovalenko" Date: Mon, 31 Mar 2025 23:12:58 +0300 Subject: [PATCH 01/10] [conftest] Advanced processing of logging This patch does the following things: - it processes the calls of logging.error as test errors - it prints the number of errors/warnings for each test --- tests/conftest.py | 212 +++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 209 insertions(+), 3 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index 9e8ea368..8183549c 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -8,6 +8,7 @@ import pathlib import math import datetime +import typing import _pytest.outcomes import _pytest.unittest @@ -212,6 +213,12 @@ def helper__build_test_id(item: pytest.Function) -> str: return testID + +# ///////////////////////////////////////////////////////////////////////////// + +g_error_msg_count_key = pytest.StashKey[int]() +g_warning_msg_count_key = pytest.StashKey[int]() + # ///////////////////////////////////////////////////////////////////////////// @@ -285,6 +292,16 @@ def helper__makereport__call( assert type(call) == pytest.CallInfo # noqa: E721 assert type(outcome) == pluggy.Result # noqa: E721 + # -------- + item_error_msg_count = item.stash.get(g_error_msg_count_key, 0) + assert type(item_error_msg_count) == int # noqa: E721 + assert item_error_msg_count >= 0 + + item_warning_msg_count = item.stash.get(g_warning_msg_count_key, 0) + assert type(item_warning_msg_count) == int # noqa: E721 + assert item_warning_msg_count >= 0 + + # -------- rep = outcome.get_result() assert rep is not None assert type(rep) == pytest.TestReport # noqa: E721 @@ -336,6 +353,7 @@ def helper__makereport__call( reasonMsgTempl = "XFAIL REASON: {0}" logging.error(call.excinfo.value) + item_error_msg_count += 1 assert type(reasonText) == str # noqa: E721 @@ -350,7 +368,13 @@ def helper__makereport__call( TEST_PROCESS_STATS.incrementFailedTestCount(testID) - logging.error(call.excinfo.value) + if type(call.excinfo.value) == SIGNAL_EXCEPTION: # noqa: E721 + assert item_error_msg_count > 0 + pass + else: + logging.error(call.excinfo.value) + item_error_msg_count += 1 + exitStatus = "FAILED" elif rep.outcome == "passed": assert call.excinfo is None @@ -380,9 +404,11 @@ def helper__makereport__call( # -------- logging.info("*") - logging.info("* DURATION : {0}".format(timedelta_to_human_text(testDurration))) + logging.info("* DURATION : {0}".format(timedelta_to_human_text(testDurration))) logging.info("*") - logging.info("* EXIT STATUS : {0}".format(exitStatus)) + logging.info("* EXIT STATUS : {0}".format(exitStatus)) + logging.info("* ERROR COUNT : {0}".format(item_error_msg_count)) + logging.info("* WARNING COUNT: {0}".format(item_warning_msg_count)) logging.info("*") logging.info("* STOP TEST {0}".format(testID)) logging.info("*") @@ -437,6 +463,186 @@ def pytest_runtest_makereport(item: pytest.Function, call: pytest.CallInfo): # ///////////////////////////////////////////////////////////////////////////// +class LogErrorWrapper2: + _old_method: any + _counter: typing.Optional[int] + + # -------------------------------------------------------------------- + def __init__(self): + self._old_method = None + self._counter = None + + # -------------------------------------------------------------------- + def __enter__(self): + assert self._old_method is None + assert self._counter is None + + self._old_method = logging.error + self._counter = 0 + + logging.error = self + return self + + # -------------------------------------------------------------------- + def __exit__(self, exc_type, exc_val, exc_tb): + assert self._old_method is not None + assert self._counter is not None + + assert logging.error is self + + logging.error = self._old_method + + self._old_method = None + self._counter = None + return False + + # -------------------------------------------------------------------- + def __call__(self, *args, **kwargs): + assert self._old_method is not None + assert self._counter is not None + + assert type(self._counter) == int # noqa: E721 + assert self._counter >= 0 + + r = self._old_method(*args, **kwargs) + + self._counter += 1 + assert self._counter > 0 + + return r + + +# ///////////////////////////////////////////////////////////////////////////// + + +class LogWarningWrapper2: + _old_method: any + _counter: typing.Optional[int] + + # -------------------------------------------------------------------- + def __init__(self): + self._old_method = None + self._counter = None + + # -------------------------------------------------------------------- + def __enter__(self): + assert self._old_method is None + assert self._counter is None + + self._old_method = logging.warning + self._counter = 0 + + logging.warning = self + return self + + # -------------------------------------------------------------------- + def __exit__(self, exc_type, exc_val, exc_tb): + assert self._old_method is not None + assert self._counter is not None + + assert logging.warning is self + + logging.warning = self._old_method + + self._old_method = None + self._counter = None + return False + + # -------------------------------------------------------------------- + def __call__(self, *args, **kwargs): + assert self._old_method is not None + assert self._counter is not None + + assert type(self._counter) == int # noqa: E721 + assert self._counter >= 0 + + r = self._old_method(*args, **kwargs) + + self._counter += 1 + assert self._counter > 0 + + return r + + +# ///////////////////////////////////////////////////////////////////////////// + + +class SIGNAL_EXCEPTION(Exception): + def __init__(self): + pass + + +# ///////////////////////////////////////////////////////////////////////////// + + +@pytest.hookimpl(hookwrapper=True) +def pytest_pyfunc_call(pyfuncitem: pytest.Function): + assert pyfuncitem is not None + assert isinstance(pyfuncitem, pytest.Function) + + debug__log_error_method = logging.error + assert debug__log_error_method is not None + + debug__log_warning_method = logging.warning + assert debug__log_warning_method is not None + + pyfuncitem.stash[g_error_msg_count_key] = 0 + pyfuncitem.stash[g_warning_msg_count_key] = 0 + + try: + with LogErrorWrapper2() as logErrorWrapper, LogWarningWrapper2() as logWarningWrapper: + assert type(logErrorWrapper) == LogErrorWrapper2 # noqa: E721 + assert logErrorWrapper._old_method is not None + assert type(logErrorWrapper._counter) == int # noqa: E721 + assert logErrorWrapper._counter == 0 + assert logging.error is logErrorWrapper + + assert type(logWarningWrapper) == LogWarningWrapper2 # noqa: E721 + assert logWarningWrapper._old_method is not None + assert type(logWarningWrapper._counter) == int # noqa: E721 + assert logWarningWrapper._counter == 0 + assert logging.warning is logWarningWrapper + + r: pluggy.Result = yield + + assert r is not None + assert type(r) == pluggy.Result # noqa: E721 + + assert logErrorWrapper._old_method is not None + assert type(logErrorWrapper._counter) == int # noqa: E721 + assert logErrorWrapper._counter >= 0 + assert logging.error is logErrorWrapper + + assert logWarningWrapper._old_method is not None + assert type(logWarningWrapper._counter) == int # noqa: E721 + assert logWarningWrapper._counter >= 0 + assert logging.warning is logWarningWrapper + + assert g_error_msg_count_key in pyfuncitem.stash + assert g_warning_msg_count_key in pyfuncitem.stash + + assert pyfuncitem.stash[g_error_msg_count_key] == 0 + assert pyfuncitem.stash[g_warning_msg_count_key] == 0 + + pyfuncitem.stash[g_error_msg_count_key] = logErrorWrapper._counter + pyfuncitem.stash[g_warning_msg_count_key] = logWarningWrapper._counter + + if r.exception is not None: + pass + elif logErrorWrapper._counter == 0: + pass + else: + assert logErrorWrapper._counter > 0 + r.force_exception(SIGNAL_EXCEPTION()) + finally: + assert logging.error is debug__log_error_method + assert logging.warning is debug__log_warning_method + pass + + +# ///////////////////////////////////////////////////////////////////////////// + + def helper__calc_W(n: int) -> int: assert n > 0 From 88bb6ae1592f1209f64745d499f2643f10e591f0 Mon Sep 17 00:00:00 2001 From: "d.kovalenko" Date: Tue, 1 Apr 2025 12:00:29 +0300 Subject: [PATCH 02/10] [conftest] XFail does not respect SIGNAL_EXCEPTION --- tests/conftest.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index 8183549c..87ce204e 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -352,8 +352,11 @@ def helper__makereport__call( reasonText = rep.wasxfail reasonMsgTempl = "XFAIL REASON: {0}" - logging.error(call.excinfo.value) - item_error_msg_count += 1 + if type(call.excinfo.value) == SIGNAL_EXCEPTION: # noqa: E721 + pass + else: + logging.error(call.excinfo.value) + item_error_msg_count += 1 assert type(reasonText) == str # noqa: E721 From 24b20c837426ea020be77dc8586f9624925310e0 Mon Sep 17 00:00:00 2001 From: "d.kovalenko" Date: Tue, 1 Apr 2025 12:16:38 +0300 Subject: [PATCH 03/10] [summary] An error count of each test is printed --- tests/conftest.py | 92 ++++++++++++++++++++++++++++++++++++----------- 1 file changed, 72 insertions(+), 20 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index 87ce204e..45404d97 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -109,8 +109,8 @@ class TEST_PROCESS_STATS: cUnexpectedTests: int = 0 cAchtungTests: int = 0 - FailedTests = list[str]() - XFailedTests = list[str]() + FailedTests = list[str, int]() + XFailedTests = list[str, int]() NotXFailedTests = list[str]() AchtungTests = list[str]() @@ -132,19 +132,23 @@ def incrementPassedTestCount() -> None: __class__.cPassedTests += 1 # -------------------------------------------------------------------- - def incrementFailedTestCount(testID: str) -> None: + def incrementFailedTestCount(testID: str, errCount: int) -> None: assert type(testID) == str # noqa: E721 + assert type(errCount) == int # noqa: E721 + assert errCount >= 0 assert type(__class__.FailedTests) == list # noqa: E721 - __class__.FailedTests.append(testID) # raise? + __class__.FailedTests.append((testID, errCount)) # raise? __class__.cFailedTests += 1 # -------------------------------------------------------------------- - def incrementXFailedTestCount(testID: str) -> None: + def incrementXFailedTestCount(testID: str, errCount: int) -> None: assert type(testID) == str # noqa: E721 + assert type(errCount) == int # noqa: E721 + assert errCount >= 0 assert type(__class__.XFailedTests) == list # noqa: E721 - __class__.XFailedTests.append(testID) # raise? + __class__.XFailedTests.append((testID, errCount)) # raise? __class__.cXFailedTests += 1 # -------------------------------------------------------------------- @@ -329,26 +333,25 @@ def helper__makereport__call( if type(call.excinfo.value) == _pytest.outcomes.Skipped: # noqa: E721 assert not hasattr(rep, "wasxfail") - TEST_PROCESS_STATS.incrementSkippedTestCount() - exitStatus = "SKIPPED" reasonText = str(call.excinfo.value) reasonMsgTempl = "SKIP REASON: {0}" - elif type(call.excinfo.value) == _pytest.outcomes.XFailed: # noqa: E721 - TEST_PROCESS_STATS.incrementXFailedTestCount(testID) + TEST_PROCESS_STATS.incrementSkippedTestCount() + elif type(call.excinfo.value) == _pytest.outcomes.XFailed: # noqa: E721 exitStatus = "XFAILED" reasonText = str(call.excinfo.value) reasonMsgTempl = "XFAIL REASON: {0}" + + TEST_PROCESS_STATS.incrementXFailedTestCount(testID, item_error_msg_count) + else: exitStatus = "XFAILED" assert hasattr(rep, "wasxfail") assert rep.wasxfail is not None assert type(rep.wasxfail) == str # noqa: E721 - TEST_PROCESS_STATS.incrementXFailedTestCount(testID) - reasonText = rep.wasxfail reasonMsgTempl = "XFAIL REASON: {0}" @@ -358,6 +361,8 @@ def helper__makereport__call( logging.error(call.excinfo.value) item_error_msg_count += 1 + TEST_PROCESS_STATS.incrementXFailedTestCount(testID, item_error_msg_count) + assert type(reasonText) == str # noqa: E721 if reasonText != "": @@ -369,8 +374,6 @@ def helper__makereport__call( assert call.excinfo is not None assert call.excinfo.value is not None - TEST_PROCESS_STATS.incrementFailedTestCount(testID) - if type(call.excinfo.value) == SIGNAL_EXCEPTION: # noqa: E721 assert item_error_msg_count > 0 pass @@ -378,6 +381,8 @@ def helper__makereport__call( logging.error(call.excinfo.value) item_error_msg_count += 1 + TEST_PROCESS_STATS.incrementFailedTestCount(testID, item_error_msg_count) + exitStatus = "FAILED" elif rep.outcome == "passed": assert call.excinfo is None @@ -676,11 +681,42 @@ def helper__print_test_list(tests: list[str]) -> None: nTest = 0 - while nTest < len(tests): - testID = tests[nTest] - assert type(testID) == str # noqa: E721 + for t in tests: + assert type(t) == str # noqa: E721 + assert t != "" nTest += 1 - logging.info(templateLine.format(nTest, testID)) + logging.info(templateLine.format(nTest, t)) + + +# ------------------------------------------------------------------------ +def helper__print_test_list2(tests: list[str, int]) -> None: + assert type(tests) == list # noqa: E721 + + assert helper__calc_W(9) == 1 + assert helper__calc_W(10) == 2 + assert helper__calc_W(11) == 2 + assert helper__calc_W(99) == 2 + assert helper__calc_W(100) == 3 + assert helper__calc_W(101) == 3 + assert helper__calc_W(999) == 3 + assert helper__calc_W(1000) == 4 + assert helper__calc_W(1001) == 4 + + W = helper__calc_W(len(tests)) + + templateLine = "{0:0" + str(W) + "d}. {1} ({2})" + + nTest = 0 + + for t in tests: + assert type(t) == tuple # noqa: E721 + assert len(t) == 2 + assert type(t[0]) == str # noqa: E721 + assert type(t[1]) == int # noqa: E721 + assert t[0] != "" + assert t[1] >= 0 + nTest += 1 + logging.info(templateLine.format(nTest, t[0], t[1])) # ///////////////////////////////////////////////////////////////////////////// @@ -714,6 +750,22 @@ def LOCAL__print_test_list(header: str, test_count: int, test_list: list[str]): helper__print_test_list(test_list) logging.info("") + def LOCAL__print_test_list2( + header: str, test_count: int, test_list: list[str, int] + ): + assert type(header) == str # noqa: E721 + assert type(test_count) == int # noqa: E721 + assert type(test_list) == list # noqa: E721 + assert header != "" + assert test_count >= 0 + assert len(test_list) == test_count + + LOCAL__print_line1_with_header(header) + logging.info("") + if len(test_list) > 0: + helper__print_test_list2(test_list) + logging.info("") + # fmt: off LOCAL__print_test_list( "ACHTUNG TESTS", @@ -721,13 +773,13 @@ def LOCAL__print_test_list(header: str, test_count: int, test_list: list[str]): TEST_PROCESS_STATS.AchtungTests, ) - LOCAL__print_test_list( + LOCAL__print_test_list2( "FAILED TESTS", TEST_PROCESS_STATS.cFailedTests, TEST_PROCESS_STATS.FailedTests ) - LOCAL__print_test_list( + LOCAL__print_test_list2( "XFAILED TESTS", TEST_PROCESS_STATS.cXFailedTests, TEST_PROCESS_STATS.XFailedTests, From 4c595cc5ba0048be2dc31838d8b32b9d07162a39 Mon Sep 17 00:00:00 2001 From: "d.kovalenko" Date: Tue, 1 Apr 2025 13:36:09 +0300 Subject: [PATCH 04/10] conftest is updated [debug checks] --- tests/conftest.py | 54 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 54 insertions(+) diff --git a/tests/conftest.py b/tests/conftest.py index 45404d97..851e02a3 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -116,65 +116,119 @@ class TEST_PROCESS_STATS: # -------------------------------------------------------------------- def incrementTotalTestCount() -> None: + assert type(__class__.cTotalTests) == int # noqa: E721 + assert __class__.cTotalTests >= 0 + __class__.cTotalTests += 1 + assert __class__.cTotalTests > 0 + # -------------------------------------------------------------------- def incrementNotExecutedTestCount() -> None: + assert type(__class__.cNotExecutedTests) == int # noqa: E721 + assert __class__.cNotExecutedTests >= 0 + __class__.cNotExecutedTests += 1 + assert __class__.cNotExecutedTests > 0 + # -------------------------------------------------------------------- def incrementExecutedTestCount() -> int: + assert type(__class__.cExecutedTests) == int # noqa: E721 + assert __class__.cExecutedTests >= 0 + __class__.cExecutedTests += 1 + + assert __class__.cExecutedTests > 0 return __class__.cExecutedTests # -------------------------------------------------------------------- def incrementPassedTestCount() -> None: + assert type(__class__.cPassedTests) == int # noqa: E721 + assert __class__.cPassedTests >= 0 + __class__.cPassedTests += 1 + assert __class__.cPassedTests > 0 + # -------------------------------------------------------------------- def incrementFailedTestCount(testID: str, errCount: int) -> None: assert type(testID) == str # noqa: E721 assert type(errCount) == int # noqa: E721 assert errCount >= 0 assert type(__class__.FailedTests) == list # noqa: E721 + assert type(__class__.cFailedTests) == int # noqa: E721 + assert __class__.cFailedTests >= 0 __class__.FailedTests.append((testID, errCount)) # raise? __class__.cFailedTests += 1 + assert len(__class__.FailedTests) > 0 + assert __class__.cFailedTests > 0 + assert len(__class__.FailedTests) == __class__.cFailedTests + # -------------------------------------------------------------------- def incrementXFailedTestCount(testID: str, errCount: int) -> None: assert type(testID) == str # noqa: E721 assert type(errCount) == int # noqa: E721 assert errCount >= 0 assert type(__class__.XFailedTests) == list # noqa: E721 + assert type(__class__.cXFailedTests) == int # noqa: E721 + assert __class__.cXFailedTests >= 0 __class__.XFailedTests.append((testID, errCount)) # raise? __class__.cXFailedTests += 1 + assert len(__class__.XFailedTests) > 0 + assert __class__.cXFailedTests > 0 + assert len(__class__.XFailedTests) == __class__.cXFailedTests + # -------------------------------------------------------------------- def incrementSkippedTestCount() -> None: + assert type(__class__.cSkippedTests) == int # noqa: E721 + assert __class__.cSkippedTests >= 0 + __class__.cSkippedTests += 1 + assert __class__.cSkippedTests > 0 + # -------------------------------------------------------------------- def incrementNotXFailedTests(testID: str) -> None: assert type(testID) == str # noqa: E721 assert type(__class__.NotXFailedTests) == list # noqa: E721 + assert type(__class__.cNotXFailedTests) == int # noqa: E721 + assert __class__.cNotXFailedTests >= 0 __class__.NotXFailedTests.append(testID) # raise? __class__.cNotXFailedTests += 1 + assert len(__class__.NotXFailedTests) > 0 + assert __class__.cNotXFailedTests > 0 + assert len(__class__.NotXFailedTests) == __class__.cNotXFailedTests + # -------------------------------------------------------------------- def incrementUnexpectedTests() -> None: + assert type(__class__.cUnexpectedTests) == int # noqa: E721 + assert __class__.cUnexpectedTests >= 0 + __class__.cUnexpectedTests += 1 + assert __class__.cUnexpectedTests > 0 + # -------------------------------------------------------------------- def incrementAchtungTestCount(testID: str) -> None: assert type(testID) == str # noqa: E721 assert type(__class__.AchtungTests) == list # noqa: E721 + assert type(__class__.cAchtungTests) == int # noqa: E721 + assert __class__.cAchtungTests >= 0 __class__.AchtungTests.append(testID) # raise? __class__.cAchtungTests += 1 + assert len(__class__.AchtungTests) > 0 + assert __class__.cAchtungTests > 0 + assert len(__class__.AchtungTests) == __class__.cAchtungTests + # ///////////////////////////////////////////////////////////////////////////// From 4e100caaf1052e82a5a043956b9e2bc5ab89fb65 Mon Sep 17 00:00:00 2001 From: "d.kovalenko" Date: Tue, 1 Apr 2025 13:38:56 +0300 Subject: [PATCH 05/10] conftest is updated [formatting] --- tests/conftest.py | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index 851e02a3..9809ae4d 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -849,17 +849,17 @@ def LOCAL__print_test_list2( LOCAL__print_line1_with_header("SUMMARY STATISTICS") logging.info("") logging.info("[TESTS]") - logging.info(" TOTAL : {0}".format(TEST_PROCESS_STATS.cTotalTests)) - logging.info(" EXECUTED : {0}".format(TEST_PROCESS_STATS.cExecutedTests)) - logging.info(" NOT EXECUTED: {0}".format(TEST_PROCESS_STATS.cNotExecutedTests)) - logging.info(" ACHTUNG : {0}".format(TEST_PROCESS_STATS.cAchtungTests)) + logging.info(" TOTAL : {0}".format(TEST_PROCESS_STATS.cTotalTests)) + logging.info(" EXECUTED : {0}".format(TEST_PROCESS_STATS.cExecutedTests)) + logging.info(" NOT EXECUTED : {0}".format(TEST_PROCESS_STATS.cNotExecutedTests)) + logging.info(" ACHTUNG : {0}".format(TEST_PROCESS_STATS.cAchtungTests)) logging.info("") - logging.info(" PASSED : {0}".format(TEST_PROCESS_STATS.cPassedTests)) - logging.info(" FAILED : {0}".format(TEST_PROCESS_STATS.cFailedTests)) - logging.info(" XFAILED : {0}".format(TEST_PROCESS_STATS.cXFailedTests)) - logging.info(" NOT XFAILED : {0}".format(TEST_PROCESS_STATS.cNotXFailedTests)) - logging.info(" SKIPPED : {0}".format(TEST_PROCESS_STATS.cSkippedTests)) - logging.info(" UNEXPECTED : {0}".format(TEST_PROCESS_STATS.cUnexpectedTests)) + logging.info(" PASSED : {0}".format(TEST_PROCESS_STATS.cPassedTests)) + logging.info(" FAILED : {0}".format(TEST_PROCESS_STATS.cFailedTests)) + logging.info(" XFAILED : {0}".format(TEST_PROCESS_STATS.cXFailedTests)) + logging.info(" NOT XFAILED : {0}".format(TEST_PROCESS_STATS.cNotXFailedTests)) + logging.info(" SKIPPED : {0}".format(TEST_PROCESS_STATS.cSkippedTests)) + logging.info(" UNEXPECTED : {0}".format(TEST_PROCESS_STATS.cUnexpectedTests)) logging.info("") From 202e526f6d392afe1bc51d404dfc1b6f1e61f643 Mon Sep 17 00:00:00 2001 From: "d.kovalenko" Date: Tue, 1 Apr 2025 13:41:35 +0300 Subject: [PATCH 06/10] False XFail message has 'info' status [not warning] Let's do not use error/warning for internal messages. --- tests/conftest.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index 9809ae4d..88b2f7a7 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -446,12 +446,12 @@ def helper__makereport__call( TEST_PROCESS_STATS.incrementNotXFailedTests(testID) - warnMsg = "Test is marked as xfail" + warnMsg = "NOTE: Test is marked as xfail" if rep.wasxfail != "": warnMsg += " [" + rep.wasxfail + "]" - logging.warning(warnMsg) + logging.info(warnMsg) exitStatus = "NOT XFAILED" else: assert not hasattr(rep, "wasxfail") From b8e383d6e958bb4383400a9d5e7123f5c30cd5ca Mon Sep 17 00:00:00 2001 From: "d.kovalenko" Date: Tue, 1 Apr 2025 13:44:31 +0300 Subject: [PATCH 07/10] [summary] List of tests with warnings is printed. --- tests/conftest.py | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/tests/conftest.py b/tests/conftest.py index 88b2f7a7..36a8fedb 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -106,12 +106,14 @@ class TEST_PROCESS_STATS: cXFailedTests: int = 0 cSkippedTests: int = 0 cNotXFailedTests: int = 0 + cWarningTests: int = 0 cUnexpectedTests: int = 0 cAchtungTests: int = 0 FailedTests = list[str, int]() XFailedTests = list[str, int]() NotXFailedTests = list[str]() + WarningTests = list[str, int]() AchtungTests = list[str]() # -------------------------------------------------------------------- @@ -206,6 +208,23 @@ def incrementNotXFailedTests(testID: str) -> None: assert __class__.cNotXFailedTests > 0 assert len(__class__.NotXFailedTests) == __class__.cNotXFailedTests + # -------------------------------------------------------------------- + def incrementWarningTestCount(testID: str, warningCount: int) -> None: + assert type(testID) == str # noqa: E721 + assert type(warningCount) == int # noqa: E721 + assert testID != "" + assert warningCount > 0 + assert type(__class__.WarningTests) == list # noqa: E721 + assert type(__class__.cWarningTests) == int # noqa: E721 + assert __class__.cWarningTests >= 0 + + __class__.WarningTests.append((testID, warningCount)) # raise? + __class__.cWarningTests += 1 + + assert len(__class__.WarningTests) > 0 + assert __class__.cWarningTests > 0 + assert len(__class__.WarningTests) == __class__.cWarningTests + # -------------------------------------------------------------------- def incrementUnexpectedTests() -> None: assert type(__class__.cUnexpectedTests) == int # noqa: E721 @@ -464,6 +483,10 @@ def helper__makereport__call( # [2025-03-28] It may create a useless problem in new environment. # assert False + # -------- + if item_warning_msg_count > 0: + TEST_PROCESS_STATS.incrementWarningTestCount(testID, item_warning_msg_count) + # -------- logging.info("*") logging.info("* DURATION : {0}".format(timedelta_to_human_text(testDurration))) @@ -844,6 +867,12 @@ def LOCAL__print_test_list2( TEST_PROCESS_STATS.cNotXFailedTests, TEST_PROCESS_STATS.NotXFailedTests, ) + + LOCAL__print_test_list2( + "WARNING TESTS", + TEST_PROCESS_STATS.cWarningTests, + TEST_PROCESS_STATS.WarningTests, + ) # fmt: on LOCAL__print_line1_with_header("SUMMARY STATISTICS") @@ -859,6 +888,7 @@ def LOCAL__print_test_list2( logging.info(" XFAILED : {0}".format(TEST_PROCESS_STATS.cXFailedTests)) logging.info(" NOT XFAILED : {0}".format(TEST_PROCESS_STATS.cNotXFailedTests)) logging.info(" SKIPPED : {0}".format(TEST_PROCESS_STATS.cSkippedTests)) + logging.info(" WITH WARNINGS: {0}".format(TEST_PROCESS_STATS.cWarningTests)) logging.info(" UNEXPECTED : {0}".format(TEST_PROCESS_STATS.cUnexpectedTests)) logging.info("") From f56fc94a85325066cf08e8eaf04be824855887d5 Mon Sep 17 00:00:00 2001 From: "d.kovalenko" Date: Tue, 1 Apr 2025 14:36:03 +0300 Subject: [PATCH 08/10] TEST_PROCESS_STATS::incrementFailedTestCount is updated [debug check] --- tests/conftest.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/conftest.py b/tests/conftest.py index 36a8fedb..93782723 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -157,7 +157,7 @@ def incrementPassedTestCount() -> None: def incrementFailedTestCount(testID: str, errCount: int) -> None: assert type(testID) == str # noqa: E721 assert type(errCount) == int # noqa: E721 - assert errCount >= 0 + assert errCount > 0 assert type(__class__.FailedTests) == list # noqa: E721 assert type(__class__.cFailedTests) == int # noqa: E721 assert __class__.cFailedTests >= 0 From 98d032fd048156839d4f568116727d0cf3654670 Mon Sep 17 00:00:00 2001 From: "d.kovalenko" Date: Tue, 1 Apr 2025 14:37:45 +0300 Subject: [PATCH 09/10] [summary] total errors/warnings/duration are printed --- tests/conftest.py | 47 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 47 insertions(+) diff --git a/tests/conftest.py b/tests/conftest.py index 93782723..ff3b3cb4 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -116,6 +116,11 @@ class TEST_PROCESS_STATS: WarningTests = list[str, int]() AchtungTests = list[str]() + cTotalDuration: datetime.timedelta = datetime.timedelta() + + cTotalErrors: int = 0 + cTotalWarnings: int = 0 + # -------------------------------------------------------------------- def incrementTotalTestCount() -> None: assert type(__class__.cTotalTests) == int # noqa: E721 @@ -169,6 +174,14 @@ def incrementFailedTestCount(testID: str, errCount: int) -> None: assert __class__.cFailedTests > 0 assert len(__class__.FailedTests) == __class__.cFailedTests + # -------- + assert type(__class__.cTotalErrors) == int # noqa: E721 + assert __class__.cTotalErrors >= 0 + + __class__.cTotalErrors += errCount + + assert __class__.cTotalErrors > 0 + # -------------------------------------------------------------------- def incrementXFailedTestCount(testID: str, errCount: int) -> None: assert type(testID) == str # noqa: E721 @@ -225,6 +238,14 @@ def incrementWarningTestCount(testID: str, warningCount: int) -> None: assert __class__.cWarningTests > 0 assert len(__class__.WarningTests) == __class__.cWarningTests + # -------- + assert type(__class__.cTotalWarnings) == int # noqa: E721 + assert __class__.cTotalWarnings >= 0 + + __class__.cTotalWarnings += warningCount + + assert __class__.cTotalWarnings > 0 + # -------------------------------------------------------------------- def incrementUnexpectedTests() -> None: assert type(__class__.cUnexpectedTests) == int # noqa: E721 @@ -454,6 +475,7 @@ def helper__makereport__call( logging.error(call.excinfo.value) item_error_msg_count += 1 + assert item_error_msg_count > 0 TEST_PROCESS_STATS.incrementFailedTestCount(testID, item_error_msg_count) exitStatus = "FAILED" @@ -487,6 +509,14 @@ def helper__makereport__call( if item_warning_msg_count > 0: TEST_PROCESS_STATS.incrementWarningTestCount(testID, item_warning_msg_count) + # -------- + assert type(TEST_PROCESS_STATS.cTotalDuration) == datetime.timedelta # noqa: E721 + assert type(testDurration) == datetime.timedelta # noqa: E721 + + TEST_PROCESS_STATS.cTotalDuration += testDurration + + assert testDurration <= TEST_PROCESS_STATS.cTotalDuration + # -------- logging.info("*") logging.info("* DURATION : {0}".format(timedelta_to_human_text(testDurration))) @@ -892,6 +922,23 @@ def LOCAL__print_test_list2( logging.info(" UNEXPECTED : {0}".format(TEST_PROCESS_STATS.cUnexpectedTests)) logging.info("") + assert type(TEST_PROCESS_STATS.cTotalDuration) == datetime.timedelta # noqa: E721 + + LOCAL__print_line1_with_header("TIME") + logging.info("") + logging.info( + " TOTAL DURATION: {0}".format( + timedelta_to_human_text(TEST_PROCESS_STATS.cTotalDuration) + ) + ) + logging.info("") + + LOCAL__print_line1_with_header("TOTAL INFORMATION") + logging.info("") + logging.info(" TOTAL ERROR COUNT : {0}".format(TEST_PROCESS_STATS.cTotalErrors)) + logging.info(" TOTAL WARNING COUNT: {0}".format(TEST_PROCESS_STATS.cTotalWarnings)) + logging.info("") + # ///////////////////////////////////////////////////////////////////////////// From 282213f945a720a2f47660c0c3cbe173f370638b Mon Sep 17 00:00:00 2001 From: "d.kovalenko" Date: Tue, 1 Apr 2025 14:44:34 +0300 Subject: [PATCH 10/10] test_conftest.py--devel is added It is an internal code for testing a conftest.py. --- tests/test_conftest.py--devel | 80 +++++++++++++++++++++++++++++++++++ 1 file changed, 80 insertions(+) create mode 100644 tests/test_conftest.py--devel diff --git a/tests/test_conftest.py--devel b/tests/test_conftest.py--devel new file mode 100644 index 00000000..67c1dafe --- /dev/null +++ b/tests/test_conftest.py--devel @@ -0,0 +1,80 @@ +import pytest +import logging + + +class TestConfest: + def test_failed(self): + raise Exception("TEST EXCEPTION!") + + def test_ok(self): + pass + + @pytest.mark.skip() + def test_mark_skip__no_reason(self): + pass + + @pytest.mark.xfail() + def test_mark_xfail__no_reason(self): + raise Exception("XFAIL EXCEPTION") + + @pytest.mark.xfail() + def test_mark_xfail__no_reason___no_error(self): + pass + + @pytest.mark.skip(reason="reason") + def test_mark_skip__with_reason(self): + pass + + @pytest.mark.xfail(reason="reason") + def test_mark_xfail__with_reason(self): + raise Exception("XFAIL EXCEPTION") + + @pytest.mark.xfail(reason="reason") + def test_mark_xfail__with_reason___no_error(self): + pass + + def test_exc_skip__no_reason(self): + pytest.skip() + + def test_exc_xfail__no_reason(self): + pytest.xfail() + + def test_exc_skip__with_reason(self): + pytest.skip(reason="SKIP REASON") + + def test_exc_xfail__with_reason(self): + pytest.xfail(reason="XFAIL EXCEPTION") + + def test_log_error(self): + logging.error("IT IS A LOG ERROR!") + + def test_log_error_and_exc(self): + logging.error("IT IS A LOG ERROR!") + + raise Exception("TEST EXCEPTION!") + + def test_log_error_and_warning(self): + logging.error("IT IS A LOG ERROR!") + logging.warning("IT IS A LOG WARNING!") + logging.error("IT IS THE SECOND LOG ERROR!") + logging.warning("IT IS THE SECOND LOG WARNING!") + + @pytest.mark.xfail() + def test_log_error_and_xfail_mark_without_reason(self): + logging.error("IT IS A LOG ERROR!") + + @pytest.mark.xfail(reason="It is a reason message") + def test_log_error_and_xfail_mark_with_reason(self): + logging.error("IT IS A LOG ERROR!") + + @pytest.mark.xfail() + def test_two_log_error_and_xfail_mark_without_reason(self): + logging.error("IT IS THE FIRST LOG ERROR!") + logging.info("----------") + logging.error("IT IS THE SECOND LOG ERROR!") + + @pytest.mark.xfail(reason="It is a reason message") + def test_two_log_error_and_xfail_mark_with_reason(self): + logging.error("IT IS THE FIRST LOG ERROR!") + logging.info("----------") + logging.error("IT IS THE SECOND LOG ERROR!") pFad - Phonifier reborn

Pfad - The Proxy pFad of © 2024 Garber Painting. All rights reserved.

Note: This service is not intended for secure transactions such as banking, social media, email, or purchasing. Use at your own risk. We assume no liability whatsoever for broken pages.


Alternative Proxies:

Alternative Proxy

pFad Proxy

pFad v3 Proxy

pFad v4 Proxy