From 09dab3139a3cdc88afc1f3a1e39c07a4aaeedbc4 Mon Sep 17 00:00:00 2001 From: "d.kovalenko" Date: Thu, 3 Apr 2025 15:10:54 +0300 Subject: [PATCH 01/31] PostgresNode refactoring [PostgresNodePortManager and RO-properties] PostgresNode uses PostgresNodePortManager to reserve/release port number New PostgresNode RO-properties are added: - name - host - port - ssh_key --- testgres/node.py | 163 ++++++++++++++++++++++++++++++++++++++++------- 1 file changed, 141 insertions(+), 22 deletions(-) diff --git a/testgres/node.py b/testgres/node.py index 1f8fca6e..ddf7e48d 100644 --- a/testgres/node.py +++ b/testgres/node.py @@ -10,6 +10,7 @@ from queue import Queue import time +import typing try: from collections.abc import Iterable @@ -131,12 +132,47 @@ def __repr__(self): repr(self.process)) +class PostgresNodePortManager: + def reserve_port(self) -> int: + raise NotImplementedError("PostManager::reserve_port is not implemented.") + + def release_port(self, number: int) -> None: + assert type(number) == int # noqa: E721 + raise NotImplementedError("PostManager::release_port is not implemented.") + + +class PostgresNodePortManager__Global(PostgresNodePortManager): + def __init__(self): + pass + + def reserve_port(self) -> int: + return utils.reserve_port() + + def release_port(self, number: int) -> None: + assert type(number) == int # noqa: E721 + return utils.release_port(number) + + class PostgresNode(object): # a max number of node start attempts _C_MAX_START_ATEMPTS = 5 - def __init__(self, name=None, base_dir=None, port=None, conn_params: ConnectionParams = ConnectionParams(), - bin_dir=None, prefix=None, os_ops=None): + _name: typing.Optional[str] + _host: typing.Optional[str] + _port: typing.Optional[int] + _should_free_port: bool + _os_ops: OsOperations + _port_manager: PostgresNodePortManager + + def __init__(self, + name=None, + base_dir=None, + port: typing.Optional[int] = None, + conn_params: ConnectionParams = ConnectionParams(), + bin_dir=None, + prefix=None, + os_ops: typing.Optional[OsOperations] = None, + port_manager: typing.Optional[PostgresNodePortManager] = None): """ PostgresNode constructor. @@ -145,21 +181,26 @@ def __init__(self, name=None, base_dir=None, port=None, conn_params: ConnectionP port: port to accept connections. base_dir: path to node's data directory. bin_dir: path to node's binary directory. + os_ops: None or correct OS operation object. + port_manager: None or correct port manager object. """ + assert port is None or type(port) == int # noqa: E721 + assert os_ops is None or isinstance(os_ops, OsOperations) + assert port_manager is None or isinstance(port_manager, PostgresNodePortManager) # private if os_ops is None: - os_ops = __class__._get_os_ops(conn_params) + self._os_ops = __class__._get_os_ops(conn_params) else: assert conn_params is None + assert isinstance(os_ops, OsOperations) + self._os_ops = os_ops pass - assert os_ops is not None - assert isinstance(os_ops, OsOperations) - self._os_ops = os_ops + assert self._os_ops is not None + assert isinstance(self._os_ops, OsOperations) - self._pg_version = PgVer(get_pg_version2(os_ops, bin_dir)) - self._should_free_port = port is None + self._pg_version = PgVer(get_pg_version2(self._os_ops, bin_dir)) self._base_dir = base_dir self._bin_dir = bin_dir self._prefix = prefix @@ -167,12 +208,30 @@ def __init__(self, name=None, base_dir=None, port=None, conn_params: ConnectionP self._master = None # basic - self.name = name or generate_app_name() + self._name = name or generate_app_name() + self._host = self._os_ops.host - self.host = os_ops.host - self.port = port or utils.reserve_port() + if port is not None: + assert type(port) == int # noqa: E721 + assert port_manager is None + self._port = port + self._should_free_port = False + self._port_manager = None + else: + if port_manager is not None: + assert isinstance(port_manager, PostgresNodePortManager) + self._port_manager = port_manager + else: + self._port_manager = __class__._get_port_manager(self._os_ops) + + assert self._port_manager is not None + assert isinstance(self._port_manager, PostgresNodePortManager) + + self._port = self._port_manager.reserve_port() # raises + assert type(self._port) == int # noqa: E721 + self._should_free_port = True - self.ssh_key = os_ops.ssh_key + assert type(self._port) == int # noqa: E721 # defaults for __exit__() self.cleanup_on_good_exit = testgres_config.node_cleanup_on_good_exit @@ -207,7 +266,11 @@ def __exit__(self, type, value, traceback): def __repr__(self): return "{}(name='{}', port={}, base_dir='{}')".format( - self.__class__.__name__, self.name, self.port, self.base_dir) + self.__class__.__name__, + self.name, + str(self._port) if self._port is not None else "None", + self.base_dir + ) @staticmethod def _get_os_ops(conn_params: ConnectionParams) -> OsOperations: @@ -221,12 +284,28 @@ def _get_os_ops(conn_params: ConnectionParams) -> OsOperations: return LocalOperations(conn_params) + @staticmethod + def _get_port_manager(os_ops: OsOperations) -> PostgresNodePortManager: + assert os_ops is not None + assert isinstance(os_ops, OsOperations) + + # [2025-04-03] It is our old, incorrected behaviour + return PostgresNodePortManager__Global() + def clone_with_new_name_and_base_dir(self, name: str, base_dir: str): assert name is None or type(name) == str # noqa: E721 assert base_dir is None or type(base_dir) == str # noqa: E721 assert __class__ == PostgresNode + if self._port_manager is None: + raise InvalidOperationException("PostgresNode without PortManager can't be cloned.") + + assert self._port_manager is not None + assert isinstance(self._port_manager, PostgresNodePortManager) + assert self._os_ops is not None + assert isinstance(self._os_ops, OsOperations) + node = PostgresNode( name=name, base_dir=base_dir, @@ -243,6 +322,34 @@ def os_ops(self) -> OsOperations: assert isinstance(self._os_ops, OsOperations) return self._os_ops + @property + def name(self) -> str: + if self._name is None: + raise InvalidOperationException("PostgresNode name is not defined.") + assert type(self._name) == str # noqa: E721 + return self._name + + @property + def host(self) -> str: + if self._host is None: + raise InvalidOperationException("PostgresNode host is not defined.") + assert type(self._host) == str # noqa: E721 + return self._host + + @property + def port(self) -> int: + if self._port is None: + raise InvalidOperationException("PostgresNode port is not defined.") + + assert type(self._port) == int # noqa: E721 + return self._port + + @property + def ssh_key(self) -> typing.Optional[str]: + assert self._os_ops is not None + assert isinstance(self._os_ops, OsOperations) + return self._os_ops.ssh_key + @property def pid(self): """ @@ -993,6 +1100,11 @@ def start(self, params=[], wait=True): if self.is_started: return self + if self._port is None: + raise InvalidOperationException("Can't start PostgresNode. Port is node defined.") + + assert type(self._port) == int # noqa: E721 + _params = [self._get_bin_path("pg_ctl"), "-D", self.data_dir, "-l", self.pg_log_file, @@ -1023,6 +1135,8 @@ def LOCAL__raise_cannot_start_node__std(from_exception): LOCAL__raise_cannot_start_node__std(e) else: assert self._should_free_port + assert self._port_manager is not None + assert isinstance(self._port_manager, PostgresNodePortManager) assert __class__._C_MAX_START_ATEMPTS > 1 log_files0 = self._collect_log_files() @@ -1048,20 +1162,20 @@ def LOCAL__raise_cannot_start_node__std(from_exception): log_files0 = log_files1 logging.warning( - "Detected a conflict with using the port {0}. Trying another port after a {1}-second sleep...".format(self.port, timeout) + "Detected a conflict with using the port {0}. Trying another port after a {1}-second sleep...".format(self._port, timeout) ) time.sleep(timeout) timeout = min(2 * timeout, 5) - cur_port = self.port - new_port = utils.reserve_port() # can raise + cur_port = self._port + new_port = self._port_manager.reserve_port() # can raise try: options = {'port': new_port} self.set_auto_conf(options) except: # noqa: E722 - utils.release_port(new_port) + self._port_manager.release_port(new_port) raise - self.port = new_port - utils.release_port(cur_port) + self._port = new_port + self._port_manager.release_port(cur_port) continue break self._maybe_start_logger() @@ -1226,10 +1340,15 @@ def free_port(self): """ if self._should_free_port: - port = self.port + assert type(self._port) == int # noqa: E721 + + assert self._port_manager is not None + assert isinstance(self._port_manager, PostgresNodePortManager) + + port = self._port self._should_free_port = False - self.port = None - utils.release_port(port) + self._port = None + self._port_manager.release_port(port) def cleanup(self, max_attempts=3, full=False): """ From ef095d39a8a0b14bcc32368faf43eeb708b924a9 Mon Sep 17 00:00:00 2001 From: "d.kovalenko" Date: Fri, 4 Apr 2025 16:17:03 +0300 Subject: [PATCH 02/31] [FIX] clone_with_new_name_and_base_dir did not respect port_manager --- testgres/node.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/testgres/node.py b/testgres/node.py index ddf7e48d..32ebca6e 100644 --- a/testgres/node.py +++ b/testgres/node.py @@ -312,7 +312,8 @@ def clone_with_new_name_and_base_dir(self, name: str, base_dir: str): conn_params=None, bin_dir=self._bin_dir, prefix=self._prefix, - os_ops=self._os_ops) + os_ops=self._os_ops, + port_manager=self._port_manager) return node From 0f842fdbd7eaf2944a88047cfeb78e1a543f2923 Mon Sep 17 00:00:00 2001 From: "d.kovalenko" Date: Fri, 4 Apr 2025 16:21:49 +0300 Subject: [PATCH 03/31] PostgresNodePortManager__ThisHost is defined (was: PostgresNodePortManager__Global) It is a singleton. --- testgres/node.py | 23 +++++++++++++++++++++-- 1 file changed, 21 insertions(+), 2 deletions(-) diff --git a/testgres/node.py b/testgres/node.py index 32ebca6e..1802bff5 100644 --- a/testgres/node.py +++ b/testgres/node.py @@ -1,4 +1,6 @@ # coding: utf-8 +from __future__ import annotations + import logging import os import random @@ -133,6 +135,9 @@ def __repr__(self): class PostgresNodePortManager: + def __init__(self): + super().__init__() + def reserve_port(self) -> int: raise NotImplementedError("PostManager::reserve_port is not implemented.") @@ -141,10 +146,24 @@ def release_port(self, number: int) -> None: raise NotImplementedError("PostManager::release_port is not implemented.") -class PostgresNodePortManager__Global(PostgresNodePortManager): +class PostgresNodePortManager__ThisHost(PostgresNodePortManager): + sm_single_instance: PostgresNodePortManager = None + sm_single_instance_guard = threading.Lock() + def __init__(self): pass + def __new__(cls) -> PostgresNodePortManager: + assert __class__ == PostgresNodePortManager__ThisHost + assert __class__.sm_single_instance_guard is not None + + if __class__.sm_single_instance is None: + with __class__.sm_single_instance_guard: + __class__.sm_single_instance = super().__new__(cls) + assert __class__.sm_single_instance + assert type(__class__.sm_single_instance) == __class__ # noqa: E721 + return __class__.sm_single_instance + def reserve_port(self) -> int: return utils.reserve_port() @@ -290,7 +309,7 @@ def _get_port_manager(os_ops: OsOperations) -> PostgresNodePortManager: assert isinstance(os_ops, OsOperations) # [2025-04-03] It is our old, incorrected behaviour - return PostgresNodePortManager__Global() + return PostgresNodePortManager__ThisHost() def clone_with_new_name_and_base_dir(self, name: str, base_dir: str): assert name is None or type(name) == str # noqa: E721 From e1e609e6dd5fe705c6c63c9bc6f3fb045992c268 Mon Sep 17 00:00:00 2001 From: "d.kovalenko" Date: Fri, 4 Apr 2025 16:25:23 +0300 Subject: [PATCH 04/31] PostgresNodePortManager__Generic is added --- testgres/node.py | 55 ++++++++++++++++++++++++- testgres/operations/local_ops.py | 11 +++++ testgres/operations/os_ops.py | 4 ++ testgres/operations/remote_ops.py | 38 ++++++++++++++++++ tests/test_testgres_remote.py | 67 ------------------------------- 5 files changed, 106 insertions(+), 69 deletions(-) diff --git a/testgres/node.py b/testgres/node.py index 1802bff5..7e112751 100644 --- a/testgres/node.py +++ b/testgres/node.py @@ -100,6 +100,8 @@ options_string, \ clean_on_error +from .helpers.port_manager import PortForException + from .backup import NodeBackup from .operations.os_ops import ConnectionParams @@ -172,6 +174,52 @@ def release_port(self, number: int) -> None: return utils.release_port(number) +class PostgresNodePortManager__Generic(PostgresNodePortManager): + _os_ops: OsOperations + _allocated_ports_guard: object + _allocated_ports: set[int] + + def __init__(self, os_ops: OsOperations): + assert os_ops is not None + assert isinstance(os_ops, OsOperations) + self._os_ops = os_ops + self._allocated_ports_guard = threading.Lock() + self._allocated_ports = set[int]() + + def reserve_port(self) -> int: + ports = set(range(1024, 65535)) + assert type(ports) == set # noqa: E721 + + assert self._allocated_ports_guard is not None + assert type(self._allocated_ports) == set # noqa: E721 + + with self._allocated_ports_guard: + ports.difference_update(self._allocated_ports) + + sampled_ports = random.sample(tuple(ports), min(len(ports), 100)) + + for port in sampled_ports: + assert not (port in self._allocated_ports) + + if not self._os_ops.is_port_free(port): + continue + + self._allocated_ports.add(port) + return port + + raise PortForException("Can't select a port") + + def release_port(self, number: int) -> None: + assert type(number) == int # noqa: E721 + + assert self._allocated_ports_guard is not None + assert type(self._allocated_ports) == set # noqa: E721 + + with self._allocated_ports_guard: + assert number in self._allocated_ports + self._allocated_ports.discard(number) + + class PostgresNode(object): # a max number of node start attempts _C_MAX_START_ATEMPTS = 5 @@ -308,8 +356,11 @@ def _get_port_manager(os_ops: OsOperations) -> PostgresNodePortManager: assert os_ops is not None assert isinstance(os_ops, OsOperations) - # [2025-04-03] It is our old, incorrected behaviour - return PostgresNodePortManager__ThisHost() + if isinstance(os_ops, LocalOperations): + return PostgresNodePortManager__ThisHost() + + # TODO: Throw exception "Please define a port manager." + return PostgresNodePortManager__Generic(os_ops) def clone_with_new_name_and_base_dir(self, name: str, base_dir: str): assert name is None or type(name) == str # noqa: E721 diff --git a/testgres/operations/local_ops.py b/testgres/operations/local_ops.py index 35e94210..39c81405 100644 --- a/testgres/operations/local_ops.py +++ b/testgres/operations/local_ops.py @@ -6,6 +6,7 @@ import subprocess import tempfile import time +import socket import psutil @@ -436,6 +437,16 @@ def get_process_children(self, pid): assert type(pid) == int # noqa: E721 return psutil.Process(pid).children() + def is_port_free(self, number: int) -> bool: + assert type(number) == int # noqa: E721 + + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: + try: + s.bind(("", number)) + return True + except OSError: + return False + # Database control def db_connect(self, dbname, user, password=None, host="localhost", port=5432): conn = pglib.connect( diff --git a/testgres/operations/os_ops.py b/testgres/operations/os_ops.py index 3c606871..489a7cb2 100644 --- a/testgres/operations/os_ops.py +++ b/testgres/operations/os_ops.py @@ -127,6 +127,10 @@ def get_pid(self): def get_process_children(self, pid): raise NotImplementedError() + def is_port_free(self, number: int): + assert type(number) == int # noqa: E721 + raise NotImplementedError() + # Database control def db_connect(self, dbname, user, password=None, host="localhost", port=5432): raise NotImplementedError() diff --git a/testgres/operations/remote_ops.py b/testgres/operations/remote_ops.py index e1ad6dac..0547a262 100644 --- a/testgres/operations/remote_ops.py +++ b/testgres/operations/remote_ops.py @@ -629,6 +629,44 @@ def get_process_children(self, pid): raise ExecUtilException(f"Error in getting process children. Error: {result.stderr}") + def is_port_free(self, number: int) -> bool: + assert type(number) == int # noqa: E721 + + cmd = ["nc", "-w", "5", "-z", "-v", "localhost", str(number)] + + exit_status, output, error = self.exec_command(cmd=cmd, encoding=get_default_encoding(), ignore_errors=True, verbose=True) + + assert type(output) == str # noqa: E721 + assert type(error) == str # noqa: E721 + + if exit_status == 0: + return __class__.helper__is_port_free__process_0(output) + + if exit_status == 1: + return __class__.helper__is_port_free__process_1(error) + + errMsg = "nc returns an unknown result code: {0}".format(exit_status) + + RaiseError.CommandExecutionError( + cmd=cmd, + exit_code=exit_status, + msg_arg=errMsg, + error=error, + out=output + ) + + @staticmethod + def helper__is_port_free__process_0(output: str) -> bool: + assert type(output) == str # noqa: E721 + # TODO: check output message + return False + + @staticmethod + def helper__is_port_free__process_1(error: str) -> bool: + assert type(error) == str # noqa: E721 + # TODO: check error message + return True + # Database control def db_connect(self, dbname, user, password=None, host="localhost", port=5432): conn = pglib.connect( diff --git a/tests/test_testgres_remote.py b/tests/test_testgres_remote.py index 2142e5ba..a2aaa18e 100755 --- a/tests/test_testgres_remote.py +++ b/tests/test_testgres_remote.py @@ -4,7 +4,6 @@ import subprocess import pytest -import psutil import logging from .helpers.os_ops_descrs import OsOpsDescrs @@ -27,8 +26,6 @@ get_pg_config # NOTE: those are ugly imports -from ..testgres import bound_ports -from ..testgres.node import ProcessProxy def util_exists(util): @@ -259,70 +256,6 @@ def test_unix_sockets(self): assert (res_exec == [(1,)]) assert (res_psql == b'1\n') - def test_ports_management(self): - assert bound_ports is not None - assert type(bound_ports) == set # noqa: E721 - - if len(bound_ports) != 0: - logging.warning("bound_ports is not empty: {0}".format(bound_ports)) - - stage0__bound_ports = bound_ports.copy() - - with __class__.helper__get_node() as node: - assert bound_ports is not None - assert type(bound_ports) == set # noqa: E721 - - assert node.port is not None - assert type(node.port) == int # noqa: E721 - - logging.info("node port is {0}".format(node.port)) - - assert node.port in bound_ports - assert node.port not in stage0__bound_ports - - assert stage0__bound_ports <= bound_ports - assert len(stage0__bound_ports) + 1 == len(bound_ports) - - stage1__bound_ports = stage0__bound_ports.copy() - stage1__bound_ports.add(node.port) - - assert stage1__bound_ports == bound_ports - - # check that port has been freed successfully - assert bound_ports is not None - assert type(bound_ports) == set # noqa: E721 - assert bound_ports == stage0__bound_ports - - # TODO: Why does not this test work with remote host? - def test_child_process_dies(self): - nAttempt = 0 - - while True: - if nAttempt == 5: - raise Exception("Max attempt number is exceed.") - - nAttempt += 1 - - logging.info("Attempt #{0}".format(nAttempt)) - - # test for FileNotFound exception during child_processes() function - with subprocess.Popen(["sleep", "60"]) as process: - r = process.poll() - - if r is not None: - logging.warning("process.pool() returns an unexpected result: {0}.".format(r)) - continue - - assert r is None - # collect list of processes currently running - children = psutil.Process(os.getpid()).children() - # kill a process, so received children dictionary becomes invalid - process.kill() - process.wait() - # try to handle children list -- missing processes will have ptype "ProcessType.Unknown" - [ProcessProxy(p) for p in children] - break - @staticmethod def helper__get_node(name=None): assert isinstance(__class__.sm_os_ops, OsOperations) From 285e5b7dd1b35fbfd6d94829a9e3d282bebfef73 Mon Sep 17 00:00:00 2001 From: "d.kovalenko" Date: Fri, 4 Apr 2025 16:30:31 +0300 Subject: [PATCH 05/31] PostgresNodePortManager is added in public API --- testgres/__init__.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/testgres/__init__.py b/testgres/__init__.py index 665548d6..f8df8c24 100644 --- a/testgres/__init__.py +++ b/testgres/__init__.py @@ -34,6 +34,7 @@ DumpFormat from .node import PostgresNode, NodeApp +from .node import PostgresNodePortManager from .utils import \ reserve_port, \ @@ -64,6 +65,7 @@ "TestgresException", "ExecUtilException", "QueryException", "TimeoutException", "CatchUpException", "StartNodeException", "InitNodeException", "BackupException", "InvalidOperationException", "XLogMethod", "IsolationLevel", "NodeStatus", "ProcessType", "DumpFormat", "PostgresNode", "NodeApp", + "PostgresNodePortManager", "reserve_port", "release_port", "bound_ports", "get_bin_path", "get_pg_config", "get_pg_version", "First", "Any", "PortManager", "OsOperations", "LocalOperations", "RemoteOperations", "ConnectionParams" From a97486675d5189f6c96adb368fd043cf646513a0 Mon Sep 17 00:00:00 2001 From: "d.kovalenko" Date: Fri, 4 Apr 2025 16:32:02 +0300 Subject: [PATCH 06/31] Test structures were refactored (local, local2, remote) --- tests/helpers/global_data.py | 78 +++++++++ tests/helpers/os_ops_descrs.py | 32 ---- tests/test_os_ops_common.py | 6 +- tests/test_os_ops_local.py | 4 +- tests/test_os_ops_remote.py | 4 +- tests/test_testgres_common.py | 301 +++++++++++++++++---------------- tests/test_testgres_remote.py | 29 ++-- 7 files changed, 262 insertions(+), 192 deletions(-) create mode 100644 tests/helpers/global_data.py delete mode 100644 tests/helpers/os_ops_descrs.py diff --git a/tests/helpers/global_data.py b/tests/helpers/global_data.py new file mode 100644 index 00000000..ea7b2385 --- /dev/null +++ b/tests/helpers/global_data.py @@ -0,0 +1,78 @@ +from ...testgres.operations.os_ops import OsOperations +from ...testgres.operations.os_ops import ConnectionParams +from ...testgres.operations.local_ops import LocalOperations +from ...testgres.operations.remote_ops import RemoteOperations + +from ...testgres.node import PostgresNodePortManager +from ...testgres.node import PostgresNodePortManager__ThisHost +from ...testgres.node import PostgresNodePortManager__Generic + +import os + + +class OsOpsDescr: + sign: str + os_ops: OsOperations + + def __init__(self, sign: str, os_ops: OsOperations): + assert type(sign) == str # noqa: E721 + assert isinstance(os_ops, OsOperations) + self.sign = sign + self.os_ops = os_ops + + +class OsOpsDescrs: + sm_remote_conn_params = ConnectionParams( + host=os.getenv('RDBMS_TESTPOOL1_HOST') or '127.0.0.1', + username=os.getenv('USER'), + ssh_key=os.getenv('RDBMS_TESTPOOL_SSHKEY')) + + sm_remote_os_ops = RemoteOperations(sm_remote_conn_params) + + sm_remote_os_ops_descr = OsOpsDescr("remote_ops", sm_remote_os_ops) + + sm_local_os_ops = LocalOperations() + + sm_local_os_ops_descr = OsOpsDescr("local_ops", sm_local_os_ops) + + +class PortManagers: + sm_remote_port_manager = PostgresNodePortManager__Generic(OsOpsDescrs.sm_remote_os_ops) + + sm_local_port_manager = PostgresNodePortManager__ThisHost() + + sm_local2_port_manager = PostgresNodePortManager__Generic(OsOpsDescrs.sm_local_os_ops) + + +class PostgresNodeService: + sign: str + os_ops: OsOperations + port_manager: PostgresNodePortManager + + def __init__(self, sign: str, os_ops: OsOperations, port_manager: PostgresNodePortManager): + assert type(sign) == str # noqa: E721 + assert isinstance(os_ops, OsOperations) + assert isinstance(port_manager, PostgresNodePortManager) + self.sign = sign + self.os_ops = os_ops + self.port_manager = port_manager + + +class PostgresNodeServices: + sm_remote = PostgresNodeService( + "remote", + OsOpsDescrs.sm_remote_os_ops, + PortManagers.sm_remote_port_manager + ) + + sm_local = PostgresNodeService( + "local", + OsOpsDescrs.sm_local_os_ops, + PortManagers.sm_local_port_manager + ) + + sm_local2 = PostgresNodeService( + "local2", + OsOpsDescrs.sm_local_os_ops, + PortManagers.sm_local2_port_manager + ) diff --git a/tests/helpers/os_ops_descrs.py b/tests/helpers/os_ops_descrs.py deleted file mode 100644 index 02297adb..00000000 --- a/tests/helpers/os_ops_descrs.py +++ /dev/null @@ -1,32 +0,0 @@ -from ...testgres.operations.os_ops import OsOperations -from ...testgres.operations.os_ops import ConnectionParams -from ...testgres.operations.local_ops import LocalOperations -from ...testgres.operations.remote_ops import RemoteOperations - -import os - - -class OsOpsDescr: - os_ops: OsOperations - sign: str - - def __init__(self, os_ops: OsOperations, sign: str): - assert isinstance(os_ops, OsOperations) - assert type(sign) == str # noqa: E721 - self.os_ops = os_ops - self.sign = sign - - -class OsOpsDescrs: - sm_remote_conn_params = ConnectionParams( - host=os.getenv('RDBMS_TESTPOOL1_HOST') or '127.0.0.1', - username=os.getenv('USER'), - ssh_key=os.getenv('RDBMS_TESTPOOL_SSHKEY')) - - sm_remote_os_ops = RemoteOperations(sm_remote_conn_params) - - sm_remote_os_ops_descr = OsOpsDescr(sm_remote_os_ops, "remote_ops") - - sm_local_os_ops = LocalOperations() - - sm_local_os_ops_descr = OsOpsDescr(sm_local_os_ops, "local_ops") diff --git a/tests/test_os_ops_common.py b/tests/test_os_ops_common.py index c3944c3b..1bcc054c 100644 --- a/tests/test_os_ops_common.py +++ b/tests/test_os_ops_common.py @@ -1,7 +1,7 @@ # coding: utf-8 -from .helpers.os_ops_descrs import OsOpsDescr -from .helpers.os_ops_descrs import OsOpsDescrs -from .helpers.os_ops_descrs import OsOperations +from .helpers.global_data import OsOpsDescr +from .helpers.global_data import OsOpsDescrs +from .helpers.global_data import OsOperations from .helpers.run_conditions import RunConditions import os diff --git a/tests/test_os_ops_local.py b/tests/test_os_ops_local.py index 2e3c30b7..f60c3fc9 100644 --- a/tests/test_os_ops_local.py +++ b/tests/test_os_ops_local.py @@ -1,6 +1,6 @@ # coding: utf-8 -from .helpers.os_ops_descrs import OsOpsDescrs -from .helpers.os_ops_descrs import OsOperations +from .helpers.global_data import OsOpsDescrs +from .helpers.global_data import OsOperations import os diff --git a/tests/test_os_ops_remote.py b/tests/test_os_ops_remote.py index 58b09242..338e49f3 100755 --- a/tests/test_os_ops_remote.py +++ b/tests/test_os_ops_remote.py @@ -1,7 +1,7 @@ # coding: utf-8 -from .helpers.os_ops_descrs import OsOpsDescrs -from .helpers.os_ops_descrs import OsOperations +from .helpers.global_data import OsOpsDescrs +from .helpers.global_data import OsOperations from ..testgres import ExecUtilException diff --git a/tests/test_testgres_common.py b/tests/test_testgres_common.py index 4e23c4af..5f88acd0 100644 --- a/tests/test_testgres_common.py +++ b/tests/test_testgres_common.py @@ -1,6 +1,7 @@ -from .helpers.os_ops_descrs import OsOpsDescr -from .helpers.os_ops_descrs import OsOpsDescrs -from .helpers.os_ops_descrs import OsOperations +from .helpers.global_data import PostgresNodeService +from .helpers.global_data import PostgresNodeServices +from .helpers.global_data import OsOperations +from .helpers.global_data import PostgresNodePortManager from ..testgres.node import PgVer from ..testgres.node import PostgresNode @@ -54,22 +55,25 @@ def removing(os_ops: OsOperations, f): class TestTestgresCommon: - sm_os_ops_descrs: list[OsOpsDescr] = [ - OsOpsDescrs.sm_local_os_ops_descr, - OsOpsDescrs.sm_remote_os_ops_descr + sm_node_svcs: list[PostgresNodeService] = [ + PostgresNodeServices.sm_local, + PostgresNodeServices.sm_local2, + PostgresNodeServices.sm_remote, ] @pytest.fixture( - params=[descr.os_ops for descr in sm_os_ops_descrs], - ids=[descr.sign for descr in sm_os_ops_descrs] + params=sm_node_svcs, + ids=[descr.sign for descr in sm_node_svcs] ) - def os_ops(self, request: pytest.FixtureRequest) -> OsOperations: + def node_svc(self, request: pytest.FixtureRequest) -> PostgresNodeService: assert isinstance(request, pytest.FixtureRequest) - assert isinstance(request.param, OsOperations) + assert isinstance(request.param, PostgresNodeService) + assert isinstance(request.param.os_ops, OsOperations) + assert isinstance(request.param.port_manager, PostgresNodePortManager) return request.param - def test_version_management(self, os_ops: OsOperations): - assert isinstance(os_ops, OsOperations) + def test_version_management(self, node_svc: PostgresNodeService): + assert isinstance(node_svc, PostgresNodeService) a = PgVer('10.0') b = PgVer('10') @@ -93,42 +97,42 @@ def test_version_management(self, os_ops: OsOperations): assert (g == k) assert (g > h) - version = get_pg_version2(os_ops) + version = get_pg_version2(node_svc.os_ops) - with __class__.helper__get_node(os_ops) as node: + with __class__.helper__get_node(node_svc) as node: assert (isinstance(version, six.string_types)) assert (isinstance(node.version, PgVer)) assert (node.version == PgVer(version)) - def test_double_init(self, os_ops: OsOperations): - assert isinstance(os_ops, OsOperations) + def test_double_init(self, node_svc: PostgresNodeService): + assert isinstance(node_svc, PostgresNodeService) - with __class__.helper__get_node(os_ops).init() as node: + with __class__.helper__get_node(node_svc).init() as node: # can't initialize node more than once with pytest.raises(expected_exception=InitNodeException): node.init() - def test_init_after_cleanup(self, os_ops: OsOperations): - assert isinstance(os_ops, OsOperations) + def test_init_after_cleanup(self, node_svc: PostgresNodeService): + assert isinstance(node_svc, PostgresNodeService) - with __class__.helper__get_node(os_ops) as node: + with __class__.helper__get_node(node_svc) as node: node.init().start().execute('select 1') node.cleanup() node.init().start().execute('select 1') - def test_init_unique_system_id(self, os_ops: OsOperations): - assert isinstance(os_ops, OsOperations) + def test_init_unique_system_id(self, node_svc: PostgresNodeService): + assert isinstance(node_svc, PostgresNodeService) # this function exists in PostgreSQL 9.6+ - current_version = get_pg_version2(os_ops) + current_version = get_pg_version2(node_svc.os_ops) - __class__.helper__skip_test_if_util_not_exist(os_ops, "pg_resetwal") + __class__.helper__skip_test_if_util_not_exist(node_svc.os_ops, "pg_resetwal") __class__.helper__skip_test_if_pg_version_is_not_ge(current_version, '9.6') query = 'select system_identifier from pg_control_system()' with scoped_config(cache_initdb=False): - with __class__.helper__get_node(os_ops).init().start() as node0: + with __class__.helper__get_node(node_svc).init().start() as node0: id0 = node0.execute(query)[0] with scoped_config(cache_initdb=True, @@ -137,8 +141,8 @@ def test_init_unique_system_id(self, os_ops: OsOperations): assert (config.cached_initdb_unique) # spawn two nodes; ids must be different - with __class__.helper__get_node(os_ops).init().start() as node1, \ - __class__.helper__get_node(os_ops).init().start() as node2: + with __class__.helper__get_node(node_svc).init().start() as node1, \ + __class__.helper__get_node(node_svc).init().start() as node2: id1 = node1.execute(query)[0] id2 = node2.execute(query)[0] @@ -146,44 +150,44 @@ def test_init_unique_system_id(self, os_ops: OsOperations): assert (id1 > id0) assert (id2 > id1) - def test_node_exit(self, os_ops: OsOperations): - assert isinstance(os_ops, OsOperations) + def test_node_exit(self, node_svc: PostgresNodeService): + assert isinstance(node_svc, PostgresNodeService) with pytest.raises(expected_exception=QueryException): - with __class__.helper__get_node(os_ops).init() as node: + with __class__.helper__get_node(node_svc).init() as node: base_dir = node.base_dir node.safe_psql('select 1') # we should save the DB for "debugging" - assert (os_ops.path_exists(base_dir)) - os_ops.rmdirs(base_dir, ignore_errors=True) + assert (node_svc.os_ops.path_exists(base_dir)) + node_svc.os_ops.rmdirs(base_dir, ignore_errors=True) - with __class__.helper__get_node(os_ops).init() as node: + with __class__.helper__get_node(node_svc).init() as node: base_dir = node.base_dir # should have been removed by default - assert not (os_ops.path_exists(base_dir)) + assert not (node_svc.os_ops.path_exists(base_dir)) - def test_double_start(self, os_ops: OsOperations): - assert isinstance(os_ops, OsOperations) + def test_double_start(self, node_svc: PostgresNodeService): + assert isinstance(node_svc, PostgresNodeService) - with __class__.helper__get_node(os_ops).init().start() as node: + with __class__.helper__get_node(node_svc).init().start() as node: # can't start node more than once node.start() assert (node.is_started) - def test_uninitialized_start(self, os_ops: OsOperations): - assert isinstance(os_ops, OsOperations) + def test_uninitialized_start(self, node_svc: PostgresNodeService): + assert isinstance(node_svc, PostgresNodeService) - with __class__.helper__get_node(os_ops) as node: + with __class__.helper__get_node(node_svc) as node: # node is not initialized yet with pytest.raises(expected_exception=StartNodeException): node.start() - def test_restart(self, os_ops: OsOperations): - assert isinstance(os_ops, OsOperations) + def test_restart(self, node_svc: PostgresNodeService): + assert isinstance(node_svc, PostgresNodeService) - with __class__.helper__get_node(os_ops) as node: + with __class__.helper__get_node(node_svc) as node: node.init().start() # restart, ok @@ -198,10 +202,10 @@ def test_restart(self, os_ops: OsOperations): node.append_conf('pg_hba.conf', 'DUMMY') node.restart() - def test_reload(self, os_ops: OsOperations): - assert isinstance(os_ops, OsOperations) + def test_reload(self, node_svc: PostgresNodeService): + assert isinstance(node_svc, PostgresNodeService) - with __class__.helper__get_node(os_ops) as node: + with __class__.helper__get_node(node_svc) as node: node.init().start() # change client_min_messages and save old value @@ -216,24 +220,24 @@ def test_reload(self, os_ops: OsOperations): assert ('debug1' == cmm_new[0][0].lower()) assert (cmm_old != cmm_new) - def test_pg_ctl(self, os_ops: OsOperations): - assert isinstance(os_ops, OsOperations) + def test_pg_ctl(self, node_svc: PostgresNodeService): + assert isinstance(node_svc, PostgresNodeService) - with __class__.helper__get_node(os_ops) as node: + with __class__.helper__get_node(node_svc) as node: node.init().start() status = node.pg_ctl(['status']) assert ('PID' in status) - def test_status(self, os_ops: OsOperations): - assert isinstance(os_ops, OsOperations) + def test_status(self, node_svc: PostgresNodeService): + assert isinstance(node_svc, PostgresNodeService) assert (NodeStatus.Running) assert not (NodeStatus.Stopped) assert not (NodeStatus.Uninitialized) # check statuses after each operation - with __class__.helper__get_node(os_ops) as node: + with __class__.helper__get_node(node_svc) as node: assert (node.pid == 0) assert (node.status() == NodeStatus.Uninitialized) @@ -257,8 +261,8 @@ def test_status(self, os_ops: OsOperations): assert (node.pid == 0) assert (node.status() == NodeStatus.Uninitialized) - def test_child_pids(self, os_ops: OsOperations): - assert isinstance(os_ops, OsOperations) + def test_child_pids(self, node_svc: PostgresNodeService): + assert isinstance(node_svc, PostgresNodeService) master_processes = [ ProcessType.AutovacuumLauncher, @@ -269,7 +273,7 @@ def test_child_pids(self, os_ops: OsOperations): ProcessType.WalWriter, ] - postgresVersion = get_pg_version2(os_ops) + postgresVersion = get_pg_version2(node_svc.os_ops) if __class__.helper__pg_version_ge(postgresVersion, '10'): master_processes.append(ProcessType.LogicalReplicationLauncher) @@ -338,7 +342,7 @@ def LOCAL__check_auxiliary_pids__multiple_attempts( absenceList )) - with __class__.helper__get_node(os_ops).init().start() as master: + with __class__.helper__get_node(node_svc).init().start() as master: # master node doesn't have a source walsender! with pytest.raises(expected_exception=testgres_TestgresException): @@ -380,10 +384,10 @@ def test_exceptions(self): str(ExecUtilException('msg', 'cmd', 1, 'out')) str(QueryException('msg', 'query')) - def test_auto_name(self, os_ops: OsOperations): - assert isinstance(os_ops, OsOperations) + def test_auto_name(self, node_svc: PostgresNodeService): + assert isinstance(node_svc, PostgresNodeService) - with __class__.helper__get_node(os_ops).init(allow_streaming=True).start() as m: + with __class__.helper__get_node(node_svc).init(allow_streaming=True).start() as m: with m.replicate().start() as r: # check that nodes are running assert (m.status()) @@ -417,9 +421,9 @@ def test_file_tail(self): lines = file_tail(f, 1) assert (lines[0] == s3) - def test_isolation_levels(self, os_ops: OsOperations): - assert isinstance(os_ops, OsOperations) - with __class__.helper__get_node(os_ops).init().start() as node: + def test_isolation_levels(self, node_svc: PostgresNodeService): + assert isinstance(node_svc, PostgresNodeService) + with __class__.helper__get_node(node_svc).init().start() as node: with node.connect() as con: # string levels con.begin('Read Uncommitted').commit() @@ -437,17 +441,17 @@ def test_isolation_levels(self, os_ops: OsOperations): with pytest.raises(expected_exception=QueryException): con.begin('Garbage').commit() - def test_users(self, os_ops: OsOperations): - assert isinstance(os_ops, OsOperations) - with __class__.helper__get_node(os_ops).init().start() as node: + def test_users(self, node_svc: PostgresNodeService): + assert isinstance(node_svc, PostgresNodeService) + with __class__.helper__get_node(node_svc).init().start() as node: node.psql('create role test_user login') value = node.safe_psql('select 1', username='test_user') value = __class__.helper__rm_carriage_returns(value) assert (value == b'1\n') - def test_poll_query_until(self, os_ops: OsOperations): - assert isinstance(os_ops, OsOperations) - with __class__.helper__get_node(os_ops) as node: + def test_poll_query_until(self, node_svc: PostgresNodeService): + assert isinstance(node_svc, PostgresNodeService) + with __class__.helper__get_node(node_svc) as node: node.init().start() get_time = 'select extract(epoch from now())' @@ -507,8 +511,8 @@ def test_poll_query_until(self, os_ops: OsOperations): # check 1 arg, ok node.poll_query_until('select true') - def test_logging(self, os_ops: OsOperations): - assert isinstance(os_ops, OsOperations) + def test_logging(self, node_svc: PostgresNodeService): + assert isinstance(node_svc, PostgresNodeService) C_MAX_ATTEMPTS = 50 # This name is used for testgres logging, too. C_NODE_NAME = "testgres_tests." + __class__.__name__ + "test_logging-master-" + uuid.uuid4().hex @@ -529,7 +533,7 @@ def test_logging(self, os_ops: OsOperations): logger.addHandler(handler) with scoped_config(use_python_logging=True): - with __class__.helper__get_node(os_ops, name=C_NODE_NAME) as master: + with __class__.helper__get_node(node_svc, name=C_NODE_NAME) as master: logging.info("Master node is initilizing") master.init() @@ -599,9 +603,9 @@ def LOCAL__test_lines(): # GO HOME! return - def test_psql(self, os_ops: OsOperations): - assert isinstance(os_ops, OsOperations) - with __class__.helper__get_node(os_ops).init().start() as node: + def test_psql(self, node_svc: PostgresNodeService): + assert isinstance(node_svc, PostgresNodeService) + with __class__.helper__get_node(node_svc).init().start() as node: # check returned values (1 arg) res = node.psql('select 1') @@ -636,17 +640,20 @@ def test_psql(self, os_ops: OsOperations): # check psql's default args, fails with pytest.raises(expected_exception=QueryException): - node.psql() + r = node.psql() # raises! + logging.error("node.psql returns [{}]".format(r)) node.stop() # check psql on stopped node, fails with pytest.raises(expected_exception=QueryException): - node.safe_psql('select 1') + # [2025-04-03] This call does not raise exception! I do not know why. + r = node.safe_psql('select 1') # raises! + logging.error("node.safe_psql returns [{}]".format(r)) - def test_safe_psql__expect_error(self, os_ops: OsOperations): - assert isinstance(os_ops, OsOperations) - with __class__.helper__get_node(os_ops).init().start() as node: + def test_safe_psql__expect_error(self, node_svc: PostgresNodeService): + assert isinstance(node_svc, PostgresNodeService) + with __class__.helper__get_node(node_svc).init().start() as node: err = node.safe_psql('select_or_not_select 1', expect_error=True) assert (type(err) == str) # noqa: E721 assert ('select_or_not_select' in err) @@ -663,9 +670,9 @@ def test_safe_psql__expect_error(self, os_ops: OsOperations): res = node.safe_psql("select 1;", expect_error=False) assert (__class__.helper__rm_carriage_returns(res) == b'1\n') - def test_transactions(self, os_ops: OsOperations): - assert isinstance(os_ops, OsOperations) - with __class__.helper__get_node(os_ops).init().start() as node: + def test_transactions(self, node_svc: PostgresNodeService): + assert isinstance(node_svc, PostgresNodeService) + with __class__.helper__get_node(node_svc).init().start() as node: with node.connect() as con: con.begin() @@ -688,9 +695,9 @@ def test_transactions(self, os_ops: OsOperations): con.execute('drop table test') con.commit() - def test_control_data(self, os_ops: OsOperations): - assert isinstance(os_ops, OsOperations) - with __class__.helper__get_node(os_ops) as node: + def test_control_data(self, node_svc: PostgresNodeService): + assert isinstance(node_svc, PostgresNodeService) + with __class__.helper__get_node(node_svc) as node: # node is not initialized yet with pytest.raises(expected_exception=ExecUtilException): @@ -703,9 +710,9 @@ def test_control_data(self, os_ops: OsOperations): assert data is not None assert (any('pg_control' in s for s in data.keys())) - def test_backup_simple(self, os_ops: OsOperations): - assert isinstance(os_ops, OsOperations) - with __class__.helper__get_node(os_ops) as master: + def test_backup_simple(self, node_svc: PostgresNodeService): + assert isinstance(node_svc, PostgresNodeService) + with __class__.helper__get_node(node_svc) as master: # enable streaming for backups master.init(allow_streaming=True) @@ -725,9 +732,9 @@ def test_backup_simple(self, os_ops: OsOperations): res = slave.execute('select * from test order by i asc') assert (res == [(1, ), (2, ), (3, ), (4, )]) - def test_backup_multiple(self, os_ops: OsOperations): - assert isinstance(os_ops, OsOperations) - with __class__.helper__get_node(os_ops) as node: + def test_backup_multiple(self, node_svc: PostgresNodeService): + assert isinstance(node_svc, PostgresNodeService) + with __class__.helper__get_node(node_svc) as node: node.init(allow_streaming=True).start() with node.backup(xlog_method='fetch') as backup1, \ @@ -739,9 +746,9 @@ def test_backup_multiple(self, os_ops: OsOperations): backup.spawn_primary('node2', destroy=False) as node2: assert (node1.base_dir != node2.base_dir) - def test_backup_exhaust(self, os_ops: OsOperations): - assert isinstance(os_ops, OsOperations) - with __class__.helper__get_node(os_ops) as node: + def test_backup_exhaust(self, node_svc: PostgresNodeService): + assert isinstance(node_svc, PostgresNodeService) + with __class__.helper__get_node(node_svc) as node: node.init(allow_streaming=True).start() with node.backup(xlog_method='fetch') as backup: @@ -753,9 +760,9 @@ def test_backup_exhaust(self, os_ops: OsOperations): with pytest.raises(expected_exception=BackupException): backup.spawn_primary() - def test_backup_wrong_xlog_method(self, os_ops: OsOperations): - assert isinstance(os_ops, OsOperations) - with __class__.helper__get_node(os_ops) as node: + def test_backup_wrong_xlog_method(self, node_svc: PostgresNodeService): + assert isinstance(node_svc, PostgresNodeService) + with __class__.helper__get_node(node_svc) as node: node.init(allow_streaming=True).start() with pytest.raises( @@ -764,11 +771,11 @@ def test_backup_wrong_xlog_method(self, os_ops: OsOperations): ): node.backup(xlog_method='wrong') - def test_pg_ctl_wait_option(self, os_ops: OsOperations): - assert isinstance(os_ops, OsOperations) + def test_pg_ctl_wait_option(self, node_svc: PostgresNodeService): + assert isinstance(node_svc, PostgresNodeService) C_MAX_ATTEMPTS = 50 - node = __class__.helper__get_node(os_ops) + node = __class__.helper__get_node(node_svc) assert node.status() == NodeStatus.Uninitialized node.init() assert node.status() == NodeStatus.Stopped @@ -835,9 +842,9 @@ def test_pg_ctl_wait_option(self, os_ops: OsOperations): logging.info("OK. Node is stopped.") node.cleanup() - def test_replicate(self, os_ops: OsOperations): - assert isinstance(os_ops, OsOperations) - with __class__.helper__get_node(os_ops) as node: + def test_replicate(self, node_svc: PostgresNodeService): + assert isinstance(node_svc, PostgresNodeService) + with __class__.helper__get_node(node_svc) as node: node.init(allow_streaming=True).start() with node.replicate().start() as replica: @@ -851,14 +858,14 @@ def test_replicate(self, os_ops: OsOperations): res = node.execute('select * from test') assert (res == []) - def test_synchronous_replication(self, os_ops: OsOperations): - assert isinstance(os_ops, OsOperations) + def test_synchronous_replication(self, node_svc: PostgresNodeService): + assert isinstance(node_svc, PostgresNodeService) - current_version = get_pg_version2(os_ops) + current_version = get_pg_version2(node_svc.os_ops) __class__.helper__skip_test_if_pg_version_is_not_ge(current_version, "9.6") - with __class__.helper__get_node(os_ops) as master: + with __class__.helper__get_node(node_svc) as master: old_version = not __class__.helper__pg_version_ge(current_version, '9.6') master.init(allow_streaming=True).start() @@ -897,14 +904,14 @@ def test_synchronous_replication(self, os_ops: OsOperations): res = standby1.safe_psql('select count(*) from abc') assert (__class__.helper__rm_carriage_returns(res) == b'1000000\n') - def test_logical_replication(self, os_ops: OsOperations): - assert isinstance(os_ops, OsOperations) + def test_logical_replication(self, node_svc: PostgresNodeService): + assert isinstance(node_svc, PostgresNodeService) - current_version = get_pg_version2(os_ops) + current_version = get_pg_version2(node_svc.os_ops) __class__.helper__skip_test_if_pg_version_is_not_ge(current_version, "10") - with __class__.helper__get_node(os_ops) as node1, __class__.helper__get_node(os_ops) as node2: + with __class__.helper__get_node(node_svc) as node1, __class__.helper__get_node(node_svc) as node2: node1.init(allow_logical=True) node1.start() node2.init().start() @@ -971,15 +978,15 @@ def test_logical_replication(self, os_ops: OsOperations): res = node2.execute('select * from test2') assert (res == [('a', ), ('b', )]) - def test_logical_catchup(self, os_ops: OsOperations): - assert isinstance(os_ops, OsOperations) + def test_logical_catchup(self, node_svc: PostgresNodeService): + assert isinstance(node_svc, PostgresNodeService) """ Runs catchup for 100 times to be sure that it is consistent """ - current_version = get_pg_version2(os_ops) + current_version = get_pg_version2(node_svc.os_ops) __class__.helper__skip_test_if_pg_version_is_not_ge(current_version, "10") - with __class__.helper__get_node(os_ops) as node1, __class__.helper__get_node(os_ops) as node2: + with __class__.helper__get_node(node_svc) as node1, __class__.helper__get_node(node_svc) as node2: node1.init(allow_logical=True) node1.start() node2.init().start() @@ -999,20 +1006,20 @@ def test_logical_catchup(self, os_ops: OsOperations): assert (res == [(i, i, )]) node1.execute('delete from test') - def test_logical_replication_fail(self, os_ops: OsOperations): - assert isinstance(os_ops, OsOperations) + def test_logical_replication_fail(self, node_svc: PostgresNodeService): + assert isinstance(node_svc, PostgresNodeService) - current_version = get_pg_version2(os_ops) + current_version = get_pg_version2(node_svc.os_ops) __class__.helper__skip_test_if_pg_version_is_ge(current_version, "10") - with __class__.helper__get_node(os_ops) as node: + with __class__.helper__get_node(node_svc) as node: with pytest.raises(expected_exception=InitNodeException): node.init(allow_logical=True) - def test_replication_slots(self, os_ops: OsOperations): - assert isinstance(os_ops, OsOperations) - with __class__.helper__get_node(os_ops) as node: + def test_replication_slots(self, node_svc: PostgresNodeService): + assert isinstance(node_svc, PostgresNodeService) + with __class__.helper__get_node(node_svc) as node: node.init(allow_streaming=True).start() with node.replicate(slot='slot1').start() as replica: @@ -1022,18 +1029,18 @@ def test_replication_slots(self, os_ops: OsOperations): with pytest.raises(expected_exception=testgres_TestgresException): node.replicate(slot='slot1') - def test_incorrect_catchup(self, os_ops: OsOperations): - assert isinstance(os_ops, OsOperations) - with __class__.helper__get_node(os_ops) as node: + def test_incorrect_catchup(self, node_svc: PostgresNodeService): + assert isinstance(node_svc, PostgresNodeService) + with __class__.helper__get_node(node_svc) as node: node.init(allow_streaming=True).start() # node has no master, can't catch up with pytest.raises(expected_exception=testgres_TestgresException): node.catchup() - def test_promotion(self, os_ops: OsOperations): - assert isinstance(os_ops, OsOperations) - with __class__.helper__get_node(os_ops) as master: + def test_promotion(self, node_svc: PostgresNodeService): + assert isinstance(node_svc, PostgresNodeService) + with __class__.helper__get_node(node_svc) as master: master.init().start() master.safe_psql('create table abc(id serial)') @@ -1046,17 +1053,17 @@ def test_promotion(self, os_ops: OsOperations): res = replica.safe_psql('select * from abc') assert (__class__.helper__rm_carriage_returns(res) == b'1\n') - def test_dump(self, os_ops: OsOperations): - assert isinstance(os_ops, OsOperations) + def test_dump(self, node_svc: PostgresNodeService): + assert isinstance(node_svc, PostgresNodeService) query_create = 'create table test as select generate_series(1, 2) as val' query_select = 'select * from test order by val asc' - with __class__.helper__get_node(os_ops).init().start() as node1: + with __class__.helper__get_node(node_svc).init().start() as node1: node1.execute(query_create) for format in ['plain', 'custom', 'directory', 'tar']: - with removing(os_ops, node1.dump(format=format)) as dump: - with __class__.helper__get_node(os_ops).init().start() as node3: + with removing(node_svc.os_ops, node1.dump(format=format)) as dump: + with __class__.helper__get_node(node_svc).init().start() as node3: if format == 'directory': assert (os.path.isdir(dump)) else: @@ -1066,14 +1073,16 @@ def test_dump(self, os_ops: OsOperations): res = node3.execute(query_select) assert (res == [(1, ), (2, )]) - def test_get_pg_config2(self, os_ops: OsOperations): + def test_get_pg_config2(self, node_svc: PostgresNodeService): + assert isinstance(node_svc, PostgresNodeService) + # check same instances - a = get_pg_config2(os_ops, None) - b = get_pg_config2(os_ops, None) + a = get_pg_config2(node_svc.os_ops, None) + b = get_pg_config2(node_svc.os_ops, None) assert (id(a) == id(b)) # save right before config change - c1 = get_pg_config2(os_ops, None) + c1 = get_pg_config2(node_svc.os_ops, None) # modify setting for this scope with scoped_config(cache_pg_config=False) as config: @@ -1081,20 +1090,26 @@ def test_get_pg_config2(self, os_ops: OsOperations): assert not (config.cache_pg_config) # save right after config change - c2 = get_pg_config2(os_ops, None) + c2 = get_pg_config2(node_svc.os_ops, None) # check different instances after config change assert (id(c1) != id(c2)) # check different instances - a = get_pg_config2(os_ops, None) - b = get_pg_config2(os_ops, None) + a = get_pg_config2(node_svc.os_ops, None) + b = get_pg_config2(node_svc.os_ops, None) assert (id(a) != id(b)) @staticmethod - def helper__get_node(os_ops: OsOperations, name=None): - assert isinstance(os_ops, OsOperations) - return PostgresNode(name, conn_params=None, os_ops=os_ops) + def helper__get_node(node_svc: PostgresNodeService, name=None): + assert isinstance(node_svc, PostgresNodeService) + assert isinstance(node_svc.os_ops, OsOperations) + assert isinstance(node_svc.port_manager, PostgresNodePortManager) + return PostgresNode( + name, + conn_params=None, + os_ops=node_svc.os_ops, + port_manager=node_svc.port_manager) @staticmethod def helper__skip_test_if_pg_version_is_not_ge(ver1: str, ver2: str): diff --git a/tests/test_testgres_remote.py b/tests/test_testgres_remote.py index a2aaa18e..7e777330 100755 --- a/tests/test_testgres_remote.py +++ b/tests/test_testgres_remote.py @@ -6,8 +6,8 @@ import pytest import logging -from .helpers.os_ops_descrs import OsOpsDescrs -from .helpers.os_ops_descrs import OsOperations +from .helpers.global_data import PostgresNodeService +from .helpers.global_data import PostgresNodeServices from .. import testgres @@ -45,17 +45,17 @@ def good_properties(f): class TestTestgresRemote: - sm_os_ops = OsOpsDescrs.sm_remote_os_ops - @pytest.fixture(autouse=True, scope="class") def implicit_fixture(self): + cur_os_ops = PostgresNodeServices.sm_remote.os_ops + assert cur_os_ops is not None + prev_ops = testgres_config.os_ops assert prev_ops is not None - assert __class__.sm_os_ops is not None - testgres_config.set_os_ops(os_ops=__class__.sm_os_ops) - assert testgres_config.os_ops is __class__.sm_os_ops + testgres_config.set_os_ops(os_ops=cur_os_ops) + assert testgres_config.os_ops is cur_os_ops yield - assert testgres_config.os_ops is __class__.sm_os_ops + assert testgres_config.os_ops is cur_os_ops testgres_config.set_os_ops(os_ops=prev_ops) assert testgres_config.os_ops is prev_ops @@ -258,8 +258,17 @@ def test_unix_sockets(self): @staticmethod def helper__get_node(name=None): - assert isinstance(__class__.sm_os_ops, OsOperations) - return testgres.PostgresNode(name, conn_params=None, os_ops=__class__.sm_os_ops) + svc = PostgresNodeServices.sm_remote + + assert isinstance(svc, PostgresNodeService) + assert isinstance(svc.os_ops, testgres.OsOperations) + assert isinstance(svc.port_manager, testgres.PostgresNodePortManager) + + return testgres.PostgresNode( + name, + conn_params=None, + os_ops=svc.os_ops, + port_manager=svc.port_manager) @staticmethod def helper__restore_envvar(name, prev_value): From 110947d88334fdcdbd0c898b7245b7fffaafbad5 Mon Sep 17 00:00:00 2001 From: "d.kovalenko" Date: Fri, 4 Apr 2025 16:42:09 +0300 Subject: [PATCH 07/31] CI files are updated --- Dockerfile--altlinux_10.tmpl | 2 +- Dockerfile--altlinux_11.tmpl | 2 +- run_tests.sh | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Dockerfile--altlinux_10.tmpl b/Dockerfile--altlinux_10.tmpl index a75e35a0..d78b05f5 100644 --- a/Dockerfile--altlinux_10.tmpl +++ b/Dockerfile--altlinux_10.tmpl @@ -115,4 +115,4 @@ ssh-keygen -t rsa -f ~/.ssh/id_rsa -q -N ''; \ cat ~/.ssh/id_rsa.pub >> ~/.ssh/authorized_keys; \ chmod 600 ~/.ssh/authorized_keys; \ ls -la ~/.ssh/; \ -TEST_FILTER=\"TestTestgresLocal or TestOsOpsLocal or local_ops\" bash ./run_tests.sh;" +TEST_FILTER=\"TestTestgresLocal or TestOsOpsLocal or local\" bash ./run_tests.sh;" diff --git a/Dockerfile--altlinux_11.tmpl b/Dockerfile--altlinux_11.tmpl index 5b43da20..5c88585d 100644 --- a/Dockerfile--altlinux_11.tmpl +++ b/Dockerfile--altlinux_11.tmpl @@ -115,4 +115,4 @@ ssh-keygen -t rsa -f ~/.ssh/id_rsa -q -N ''; \ cat ~/.ssh/id_rsa.pub >> ~/.ssh/authorized_keys; \ chmod 600 ~/.ssh/authorized_keys; \ ls -la ~/.ssh/; \ -TEST_FILTER=\"TestTestgresLocal or TestOsOpsLocal or local_ops\" bash ./run_tests.sh;" +TEST_FILTER=\"TestTestgresLocal or TestOsOpsLocal or local\" bash ./run_tests.sh;" diff --git a/run_tests.sh b/run_tests.sh index 8202aff5..65c17dbf 100755 --- a/run_tests.sh +++ b/run_tests.sh @@ -5,7 +5,7 @@ set -eux if [ -z ${TEST_FILTER+x} ]; \ -then export TEST_FILTER="TestTestgresLocal or (TestTestgresCommon and (not remote_ops))"; \ +then export TEST_FILTER="TestTestgresLocal or (TestTestgresCommon and (not remote))"; \ fi # fail early From 4f49dde2709e7ca9c75608bc7f020d95e351a8c5 Mon Sep 17 00:00:00 2001 From: "d.kovalenko" Date: Fri, 4 Apr 2025 18:03:24 +0300 Subject: [PATCH 08/31] TestTestgresCommon.test_pgbench is added - [del] TestTestgresLocal.test_pgbench - [del] TestTestgresRemote.test_pgbench --- tests/test_testgres_common.py | 21 +++++++++++++++++++++ tests/test_testgres_local.py | 21 --------------------- tests/test_testgres_remote.py | 16 ---------------- 3 files changed, 21 insertions(+), 37 deletions(-) diff --git a/tests/test_testgres_common.py b/tests/test_testgres_common.py index 5f88acd0..b65f8870 100644 --- a/tests/test_testgres_common.py +++ b/tests/test_testgres_common.py @@ -38,6 +38,7 @@ import uuid import os import re +import subprocess @contextmanager @@ -1100,6 +1101,26 @@ def test_get_pg_config2(self, node_svc: PostgresNodeService): b = get_pg_config2(node_svc.os_ops, None) assert (id(a) != id(b)) + def test_pgbench(self, node_svc: PostgresNodeService): + assert isinstance(node_svc, PostgresNodeService) + + __class__.helper__skip_test_if_util_not_exist(node_svc.os_ops, "pgbench") + + with __class__.helper__get_node(node_svc).init().start() as node: + # initialize pgbench DB and run benchmarks + node.pgbench_init( + scale=2, + foreign_keys=True, + options=['-q'] + ).pgbench_run(time=2) + + # run TPC-B benchmark + proc = node.pgbench(stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + options=['-T3']) + out = proc.communicate()[0] + assert (b'tps = ' in out) + @staticmethod def helper__get_node(node_svc: PostgresNodeService, name=None): assert isinstance(node_svc, PostgresNodeService) diff --git a/tests/test_testgres_local.py b/tests/test_testgres_local.py index 01f975a0..d326dd9e 100644 --- a/tests/test_testgres_local.py +++ b/tests/test_testgres_local.py @@ -100,27 +100,6 @@ def test_custom_init(self): # there should be no trust entries at all assert not (any('trust' in s for s in lines)) - def test_pgbench(self): - __class__.helper__skip_test_if_util_not_exist("pgbench") - - with get_new_node().init().start() as node: - - # initialize pgbench DB and run benchmarks - node.pgbench_init(scale=2, foreign_keys=True, - options=['-q']).pgbench_run(time=2) - - # run TPC-B benchmark - proc = node.pgbench(stdout=subprocess.PIPE, - stderr=subprocess.STDOUT, - options=['-T3']) - - out, _ = proc.communicate() - out = out.decode('utf-8') - - proc.stdout.close() - - assert ('tps' in out) - def test_pg_config(self): # check same instances a = get_pg_config() diff --git a/tests/test_testgres_remote.py b/tests/test_testgres_remote.py index 7e777330..34257b23 100755 --- a/tests/test_testgres_remote.py +++ b/tests/test_testgres_remote.py @@ -1,7 +1,6 @@ # coding: utf-8 import os import re -import subprocess import pytest import logging @@ -169,21 +168,6 @@ def test_init__unk_LANG_and_LC_CTYPE(self): __class__.helper__restore_envvar("LC_CTYPE", prev_LC_CTYPE) __class__.helper__restore_envvar("LC_COLLATE", prev_LC_COLLATE) - def test_pgbench(self): - __class__.helper__skip_test_if_util_not_exist("pgbench") - - with __class__.helper__get_node().init().start() as node: - # initialize pgbench DB and run benchmarks - node.pgbench_init(scale=2, foreign_keys=True, - options=['-q']).pgbench_run(time=2) - - # run TPC-B benchmark - proc = node.pgbench(stdout=subprocess.PIPE, - stderr=subprocess.STDOUT, - options=['-T3']) - out = proc.communicate()[0] - assert (b'tps = ' in out) - def test_pg_config(self): # check same instances a = get_pg_config() From 4fbf51daa0fcbb8ac33131722af698433177a864 Mon Sep 17 00:00:00 2001 From: "d.kovalenko" Date: Fri, 4 Apr 2025 18:08:17 +0300 Subject: [PATCH 09/31] PostgresNodePortManager is updated [error messages] --- testgres/node.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/testgres/node.py b/testgres/node.py index 7e112751..59cf26f1 100644 --- a/testgres/node.py +++ b/testgres/node.py @@ -141,11 +141,11 @@ def __init__(self): super().__init__() def reserve_port(self) -> int: - raise NotImplementedError("PostManager::reserve_port is not implemented.") + raise NotImplementedError("PostgresNodePortManager::reserve_port is not implemented.") def release_port(self, number: int) -> None: assert type(number) == int # noqa: E721 - raise NotImplementedError("PostManager::release_port is not implemented.") + raise NotImplementedError("PostgresNodePortManager::release_port is not implemented.") class PostgresNodePortManager__ThisHost(PostgresNodePortManager): From 17c73cbd262510920d71ce23a6c3efa3cfbb1ed0 Mon Sep 17 00:00:00 2001 From: "d.kovalenko" Date: Fri, 4 Apr 2025 18:33:11 +0300 Subject: [PATCH 10/31] PostgresNodePortManager(+company) was moved in own file - port_manager.py --- testgres/node.py | 99 ++++------------------------------------ testgres/port_manager.py | 91 ++++++++++++++++++++++++++++++++++++ 2 files changed, 99 insertions(+), 91 deletions(-) create mode 100644 testgres/port_manager.py diff --git a/testgres/node.py b/testgres/node.py index 59cf26f1..c080c3dd 100644 --- a/testgres/node.py +++ b/testgres/node.py @@ -83,14 +83,16 @@ BackupException, \ InvalidOperationException +from .port_manager import PostgresNodePortManager +from .port_manager import PostgresNodePortManager__ThisHost +from .port_manager import PostgresNodePortManager__Generic + from .logger import TestgresLogger from .pubsub import Publication, Subscription from .standby import First -from . import utils - from .utils import \ PgVer, \ eprint, \ @@ -100,8 +102,6 @@ options_string, \ clean_on_error -from .helpers.port_manager import PortForException - from .backup import NodeBackup from .operations.os_ops import ConnectionParams @@ -131,93 +131,10 @@ def __getattr__(self, name): return getattr(self.process, name) def __repr__(self): - return '{}(ptype={}, process={})'.format(self.__class__.__name__, - str(self.ptype), - repr(self.process)) - - -class PostgresNodePortManager: - def __init__(self): - super().__init__() - - def reserve_port(self) -> int: - raise NotImplementedError("PostgresNodePortManager::reserve_port is not implemented.") - - def release_port(self, number: int) -> None: - assert type(number) == int # noqa: E721 - raise NotImplementedError("PostgresNodePortManager::release_port is not implemented.") - - -class PostgresNodePortManager__ThisHost(PostgresNodePortManager): - sm_single_instance: PostgresNodePortManager = None - sm_single_instance_guard = threading.Lock() - - def __init__(self): - pass - - def __new__(cls) -> PostgresNodePortManager: - assert __class__ == PostgresNodePortManager__ThisHost - assert __class__.sm_single_instance_guard is not None - - if __class__.sm_single_instance is None: - with __class__.sm_single_instance_guard: - __class__.sm_single_instance = super().__new__(cls) - assert __class__.sm_single_instance - assert type(__class__.sm_single_instance) == __class__ # noqa: E721 - return __class__.sm_single_instance - - def reserve_port(self) -> int: - return utils.reserve_port() - - def release_port(self, number: int) -> None: - assert type(number) == int # noqa: E721 - return utils.release_port(number) - - -class PostgresNodePortManager__Generic(PostgresNodePortManager): - _os_ops: OsOperations - _allocated_ports_guard: object - _allocated_ports: set[int] - - def __init__(self, os_ops: OsOperations): - assert os_ops is not None - assert isinstance(os_ops, OsOperations) - self._os_ops = os_ops - self._allocated_ports_guard = threading.Lock() - self._allocated_ports = set[int]() - - def reserve_port(self) -> int: - ports = set(range(1024, 65535)) - assert type(ports) == set # noqa: E721 - - assert self._allocated_ports_guard is not None - assert type(self._allocated_ports) == set # noqa: E721 - - with self._allocated_ports_guard: - ports.difference_update(self._allocated_ports) - - sampled_ports = random.sample(tuple(ports), min(len(ports), 100)) - - for port in sampled_ports: - assert not (port in self._allocated_ports) - - if not self._os_ops.is_port_free(port): - continue - - self._allocated_ports.add(port) - return port - - raise PortForException("Can't select a port") - - def release_port(self, number: int) -> None: - assert type(number) == int # noqa: E721 - - assert self._allocated_ports_guard is not None - assert type(self._allocated_ports) == set # noqa: E721 - - with self._allocated_ports_guard: - assert number in self._allocated_ports - self._allocated_ports.discard(number) + return '{}(ptype={}, process={})'.format( + self.__class__.__name__, + str(self.ptype), + repr(self.process)) class PostgresNode(object): diff --git a/testgres/port_manager.py b/testgres/port_manager.py new file mode 100644 index 00000000..32c5db5d --- /dev/null +++ b/testgres/port_manager.py @@ -0,0 +1,91 @@ +from .operations.os_ops import OsOperations + +from .helpers.port_manager import PortForException + +from . import utils + +import threading +import random + +class PostgresNodePortManager: + def __init__(self): + super().__init__() + + def reserve_port(self) -> int: + raise NotImplementedError("PostgresNodePortManager::reserve_port is not implemented.") + + def release_port(self, number: int) -> None: + assert type(number) == int # noqa: E721 + raise NotImplementedError("PostgresNodePortManager::release_port is not implemented.") + + +class PostgresNodePortManager__ThisHost(PostgresNodePortManager): + sm_single_instance: PostgresNodePortManager = None + sm_single_instance_guard = threading.Lock() + + def __init__(self): + pass + + def __new__(cls) -> PostgresNodePortManager: + assert __class__ == PostgresNodePortManager__ThisHost + assert __class__.sm_single_instance_guard is not None + + if __class__.sm_single_instance is None: + with __class__.sm_single_instance_guard: + __class__.sm_single_instance = super().__new__(cls) + assert __class__.sm_single_instance + assert type(__class__.sm_single_instance) == __class__ # noqa: E721 + return __class__.sm_single_instance + + def reserve_port(self) -> int: + return utils.reserve_port() + + def release_port(self, number: int) -> None: + assert type(number) == int # noqa: E721 + return utils.release_port(number) + + +class PostgresNodePortManager__Generic(PostgresNodePortManager): + _os_ops: OsOperations + _allocated_ports_guard: object + _allocated_ports: set[int] + + def __init__(self, os_ops: OsOperations): + assert os_ops is not None + assert isinstance(os_ops, OsOperations) + self._os_ops = os_ops + self._allocated_ports_guard = threading.Lock() + self._allocated_ports = set[int]() + + def reserve_port(self) -> int: + ports = set(range(1024, 65535)) + assert type(ports) == set # noqa: E721 + + assert self._allocated_ports_guard is not None + assert type(self._allocated_ports) == set # noqa: E721 + + with self._allocated_ports_guard: + ports.difference_update(self._allocated_ports) + + sampled_ports = random.sample(tuple(ports), min(len(ports), 100)) + + for port in sampled_ports: + assert not (port in self._allocated_ports) + + if not self._os_ops.is_port_free(port): + continue + + self._allocated_ports.add(port) + return port + + raise PortForException("Can't select a port") + + def release_port(self, number: int) -> None: + assert type(number) == int # noqa: E721 + + assert self._allocated_ports_guard is not None + assert type(self._allocated_ports) == set # noqa: E721 + + with self._allocated_ports_guard: + assert number in self._allocated_ports + self._allocated_ports.discard(number) From b1cee194c7ad9696de67944f10cf8521ff5f1f45 Mon Sep 17 00:00:00 2001 From: "d.kovalenko" Date: Fri, 4 Apr 2025 18:46:22 +0300 Subject: [PATCH 11/31] PortManager was deleted [amen] --- testgres/__init__.py | 4 +--- testgres/exceptions.py | 4 ++++ testgres/helpers/__init__.py | 0 testgres/helpers/port_manager.py | 41 -------------------------------- testgres/port_manager.py | 3 ++- testgres/utils.py | 31 +++++++++++++++++++----- 6 files changed, 32 insertions(+), 51 deletions(-) delete mode 100644 testgres/helpers/__init__.py delete mode 100644 testgres/helpers/port_manager.py diff --git a/testgres/__init__.py b/testgres/__init__.py index f8df8c24..e76518f5 100644 --- a/testgres/__init__.py +++ b/testgres/__init__.py @@ -54,8 +54,6 @@ from .operations.local_ops import LocalOperations from .operations.remote_ops import RemoteOperations -from .helpers.port_manager import PortManager - __all__ = [ "get_new_node", "get_remote_node", @@ -67,6 +65,6 @@ "PostgresNode", "NodeApp", "PostgresNodePortManager", "reserve_port", "release_port", "bound_ports", "get_bin_path", "get_pg_config", "get_pg_version", - "First", "Any", "PortManager", + "First", "Any", "OsOperations", "LocalOperations", "RemoteOperations", "ConnectionParams" ] diff --git a/testgres/exceptions.py b/testgres/exceptions.py index d61d4691..20c1a8cf 100644 --- a/testgres/exceptions.py +++ b/testgres/exceptions.py @@ -7,6 +7,10 @@ class TestgresException(Exception): pass +class PortForException(TestgresException): + pass + + @six.python_2_unicode_compatible class ExecUtilException(TestgresException): def __init__(self, message=None, command=None, exit_code=0, out=None, error=None): diff --git a/testgres/helpers/__init__.py b/testgres/helpers/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/testgres/helpers/port_manager.py b/testgres/helpers/port_manager.py deleted file mode 100644 index cfc5c096..00000000 --- a/testgres/helpers/port_manager.py +++ /dev/null @@ -1,41 +0,0 @@ -import socket -import random -from typing import Set, Iterable, Optional - - -class PortForException(Exception): - pass - - -class PortManager: - def __init__(self, ports_range=(1024, 65535)): - self.ports_range = ports_range - - @staticmethod - def is_port_free(port: int) -> bool: - """Check if a port is free to use.""" - with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: - try: - s.bind(("", port)) - return True - except OSError: - return False - - def find_free_port(self, ports: Optional[Set[int]] = None, exclude_ports: Optional[Iterable[int]] = None) -> int: - """Return a random unused port number.""" - if ports is None: - ports = set(range(1024, 65535)) - - assert type(ports) == set # noqa: E721 - - if exclude_ports is not None: - assert isinstance(exclude_ports, Iterable) - ports.difference_update(exclude_ports) - - sampled_ports = random.sample(tuple(ports), min(len(ports), 100)) - - for port in sampled_ports: - if self.is_port_free(port): - return port - - raise PortForException("Can't select a port") diff --git a/testgres/port_manager.py b/testgres/port_manager.py index 32c5db5d..e27c64a2 100644 --- a/testgres/port_manager.py +++ b/testgres/port_manager.py @@ -1,12 +1,13 @@ from .operations.os_ops import OsOperations -from .helpers.port_manager import PortForException +from .exceptions import PortForException from . import utils import threading import random + class PostgresNodePortManager: def __init__(self): super().__init__() diff --git a/testgres/utils.py b/testgres/utils.py index 92383571..cb0a6f19 100644 --- a/testgres/utils.py +++ b/testgres/utils.py @@ -6,6 +6,8 @@ import os import sys +import socket +import random from contextlib import contextmanager from packaging.version import Version, InvalidVersion @@ -13,7 +15,7 @@ from six import iteritems -from .helpers.port_manager import PortManager +from .exceptions import PortForException from .exceptions import ExecUtilException from .config import testgres_config as tconf from .operations.os_ops import OsOperations @@ -41,11 +43,28 @@ def internal__reserve_port(): """ Generate a new port and add it to 'bound_ports'. """ - port_mng = PortManager() - port = port_mng.find_free_port(exclude_ports=bound_ports) - bound_ports.add(port) - - return port + def LOCAL__is_port_free(port: int) -> bool: + """Check if a port is free to use.""" + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: + try: + s.bind(("", port)) + return True + except OSError: + return False + + ports = set(range(1024, 65535)) + assert type(ports) == set # noqa: E721 + assert type(bound_ports) == set # noqa: E721 + ports.difference_update(bound_ports) + + sampled_ports = random.sample(tuple(ports), min(len(ports), 100)) + + for port in sampled_ports: + if LOCAL__is_port_free(port): + bound_ports.add(port) + return port + + raise PortForException("Can't select a port") def internal__release_port(port): From c5ad9078e98495ae904f23cba8b86a6c6e9ddceb Mon Sep 17 00:00:00 2001 From: "d.kovalenko" Date: Fri, 4 Apr 2025 18:58:17 +0300 Subject: [PATCH 12/31] PostgresNodePortManager was renamed with PortManager --- testgres/__init__.py | 4 ++-- testgres/node.py | 28 ++++++++++++++-------------- testgres/port_manager.py | 16 ++++++++-------- testgres/utils.py | 3 +++ tests/helpers/global_data.py | 18 +++++++++--------- tests/test_testgres_common.py | 6 +++--- tests/test_testgres_remote.py | 2 +- 7 files changed, 40 insertions(+), 37 deletions(-) diff --git a/testgres/__init__.py b/testgres/__init__.py index e76518f5..339ae62e 100644 --- a/testgres/__init__.py +++ b/testgres/__init__.py @@ -34,7 +34,7 @@ DumpFormat from .node import PostgresNode, NodeApp -from .node import PostgresNodePortManager +from .node import PortManager from .utils import \ reserve_port, \ @@ -63,7 +63,7 @@ "TestgresException", "ExecUtilException", "QueryException", "TimeoutException", "CatchUpException", "StartNodeException", "InitNodeException", "BackupException", "InvalidOperationException", "XLogMethod", "IsolationLevel", "NodeStatus", "ProcessType", "DumpFormat", "PostgresNode", "NodeApp", - "PostgresNodePortManager", + "PortManager", "reserve_port", "release_port", "bound_ports", "get_bin_path", "get_pg_config", "get_pg_version", "First", "Any", "OsOperations", "LocalOperations", "RemoteOperations", "ConnectionParams" diff --git a/testgres/node.py b/testgres/node.py index c080c3dd..99ec2032 100644 --- a/testgres/node.py +++ b/testgres/node.py @@ -83,9 +83,9 @@ BackupException, \ InvalidOperationException -from .port_manager import PostgresNodePortManager -from .port_manager import PostgresNodePortManager__ThisHost -from .port_manager import PostgresNodePortManager__Generic +from .port_manager import PortManager +from .port_manager import PortManager__ThisHost +from .port_manager import PortManager__Generic from .logger import TestgresLogger @@ -146,7 +146,7 @@ class PostgresNode(object): _port: typing.Optional[int] _should_free_port: bool _os_ops: OsOperations - _port_manager: PostgresNodePortManager + _port_manager: PortManager def __init__(self, name=None, @@ -156,7 +156,7 @@ def __init__(self, bin_dir=None, prefix=None, os_ops: typing.Optional[OsOperations] = None, - port_manager: typing.Optional[PostgresNodePortManager] = None): + port_manager: typing.Optional[PortManager] = None): """ PostgresNode constructor. @@ -170,7 +170,7 @@ def __init__(self, """ assert port is None or type(port) == int # noqa: E721 assert os_ops is None or isinstance(os_ops, OsOperations) - assert port_manager is None or isinstance(port_manager, PostgresNodePortManager) + assert port_manager is None or isinstance(port_manager, PortManager) # private if os_ops is None: @@ -203,13 +203,13 @@ def __init__(self, self._port_manager = None else: if port_manager is not None: - assert isinstance(port_manager, PostgresNodePortManager) + assert isinstance(port_manager, PortManager) self._port_manager = port_manager else: self._port_manager = __class__._get_port_manager(self._os_ops) assert self._port_manager is not None - assert isinstance(self._port_manager, PostgresNodePortManager) + assert isinstance(self._port_manager, PortManager) self._port = self._port_manager.reserve_port() # raises assert type(self._port) == int # noqa: E721 @@ -269,15 +269,15 @@ def _get_os_ops(conn_params: ConnectionParams) -> OsOperations: return LocalOperations(conn_params) @staticmethod - def _get_port_manager(os_ops: OsOperations) -> PostgresNodePortManager: + def _get_port_manager(os_ops: OsOperations) -> PortManager: assert os_ops is not None assert isinstance(os_ops, OsOperations) if isinstance(os_ops, LocalOperations): - return PostgresNodePortManager__ThisHost() + return PortManager__ThisHost() # TODO: Throw exception "Please define a port manager." - return PostgresNodePortManager__Generic(os_ops) + return PortManager__Generic(os_ops) def clone_with_new_name_and_base_dir(self, name: str, base_dir: str): assert name is None or type(name) == str # noqa: E721 @@ -289,7 +289,7 @@ def clone_with_new_name_and_base_dir(self, name: str, base_dir: str): raise InvalidOperationException("PostgresNode without PortManager can't be cloned.") assert self._port_manager is not None - assert isinstance(self._port_manager, PostgresNodePortManager) + assert isinstance(self._port_manager, PortManager) assert self._os_ops is not None assert isinstance(self._os_ops, OsOperations) @@ -1124,7 +1124,7 @@ def LOCAL__raise_cannot_start_node__std(from_exception): else: assert self._should_free_port assert self._port_manager is not None - assert isinstance(self._port_manager, PostgresNodePortManager) + assert isinstance(self._port_manager, PortManager) assert __class__._C_MAX_START_ATEMPTS > 1 log_files0 = self._collect_log_files() @@ -1331,7 +1331,7 @@ def free_port(self): assert type(self._port) == int # noqa: E721 assert self._port_manager is not None - assert isinstance(self._port_manager, PostgresNodePortManager) + assert isinstance(self._port_manager, PortManager) port = self._port self._should_free_port = False diff --git a/testgres/port_manager.py b/testgres/port_manager.py index e27c64a2..e4c2180c 100644 --- a/testgres/port_manager.py +++ b/testgres/port_manager.py @@ -8,27 +8,27 @@ import random -class PostgresNodePortManager: +class PortManager: def __init__(self): super().__init__() def reserve_port(self) -> int: - raise NotImplementedError("PostgresNodePortManager::reserve_port is not implemented.") + raise NotImplementedError("PortManager::reserve_port is not implemented.") def release_port(self, number: int) -> None: assert type(number) == int # noqa: E721 - raise NotImplementedError("PostgresNodePortManager::release_port is not implemented.") + raise NotImplementedError("PortManager::release_port is not implemented.") -class PostgresNodePortManager__ThisHost(PostgresNodePortManager): - sm_single_instance: PostgresNodePortManager = None +class PortManager__ThisHost(PortManager): + sm_single_instance: PortManager = None sm_single_instance_guard = threading.Lock() def __init__(self): pass - def __new__(cls) -> PostgresNodePortManager: - assert __class__ == PostgresNodePortManager__ThisHost + def __new__(cls) -> PortManager: + assert __class__ == PortManager__ThisHost assert __class__.sm_single_instance_guard is not None if __class__.sm_single_instance is None: @@ -46,7 +46,7 @@ def release_port(self, number: int) -> None: return utils.release_port(number) -class PostgresNodePortManager__Generic(PostgresNodePortManager): +class PortManager__Generic(PortManager): _os_ops: OsOperations _allocated_ports_guard: object _allocated_ports: set[int] diff --git a/testgres/utils.py b/testgres/utils.py index cb0a6f19..10ae81b6 100644 --- a/testgres/utils.py +++ b/testgres/utils.py @@ -72,6 +72,9 @@ def internal__release_port(port): Free port provided by reserve_port(). """ + assert type(port) == int # noqa: E721 + assert port in bound_ports + bound_ports.discard(port) diff --git a/tests/helpers/global_data.py b/tests/helpers/global_data.py index ea7b2385..c21d7dd8 100644 --- a/tests/helpers/global_data.py +++ b/tests/helpers/global_data.py @@ -3,9 +3,9 @@ from ...testgres.operations.local_ops import LocalOperations from ...testgres.operations.remote_ops import RemoteOperations -from ...testgres.node import PostgresNodePortManager -from ...testgres.node import PostgresNodePortManager__ThisHost -from ...testgres.node import PostgresNodePortManager__Generic +from ...testgres.node import PortManager +from ...testgres.node import PortManager__ThisHost +from ...testgres.node import PortManager__Generic import os @@ -37,22 +37,22 @@ class OsOpsDescrs: class PortManagers: - sm_remote_port_manager = PostgresNodePortManager__Generic(OsOpsDescrs.sm_remote_os_ops) + sm_remote_port_manager = PortManager__Generic(OsOpsDescrs.sm_remote_os_ops) - sm_local_port_manager = PostgresNodePortManager__ThisHost() + sm_local_port_manager = PortManager__ThisHost() - sm_local2_port_manager = PostgresNodePortManager__Generic(OsOpsDescrs.sm_local_os_ops) + sm_local2_port_manager = PortManager__Generic(OsOpsDescrs.sm_local_os_ops) class PostgresNodeService: sign: str os_ops: OsOperations - port_manager: PostgresNodePortManager + port_manager: PortManager - def __init__(self, sign: str, os_ops: OsOperations, port_manager: PostgresNodePortManager): + def __init__(self, sign: str, os_ops: OsOperations, port_manager: PortManager): assert type(sign) == str # noqa: E721 assert isinstance(os_ops, OsOperations) - assert isinstance(port_manager, PostgresNodePortManager) + assert isinstance(port_manager, PortManager) self.sign = sign self.os_ops = os_ops self.port_manager = port_manager diff --git a/tests/test_testgres_common.py b/tests/test_testgres_common.py index b65f8870..f2d9c074 100644 --- a/tests/test_testgres_common.py +++ b/tests/test_testgres_common.py @@ -1,7 +1,7 @@ from .helpers.global_data import PostgresNodeService from .helpers.global_data import PostgresNodeServices from .helpers.global_data import OsOperations -from .helpers.global_data import PostgresNodePortManager +from .helpers.global_data import PortManager from ..testgres.node import PgVer from ..testgres.node import PostgresNode @@ -70,7 +70,7 @@ def node_svc(self, request: pytest.FixtureRequest) -> PostgresNodeService: assert isinstance(request, pytest.FixtureRequest) assert isinstance(request.param, PostgresNodeService) assert isinstance(request.param.os_ops, OsOperations) - assert isinstance(request.param.port_manager, PostgresNodePortManager) + assert isinstance(request.param.port_manager, PortManager) return request.param def test_version_management(self, node_svc: PostgresNodeService): @@ -1125,7 +1125,7 @@ def test_pgbench(self, node_svc: PostgresNodeService): def helper__get_node(node_svc: PostgresNodeService, name=None): assert isinstance(node_svc, PostgresNodeService) assert isinstance(node_svc.os_ops, OsOperations) - assert isinstance(node_svc.port_manager, PostgresNodePortManager) + assert isinstance(node_svc.port_manager, PortManager) return PostgresNode( name, conn_params=None, diff --git a/tests/test_testgres_remote.py b/tests/test_testgres_remote.py index 34257b23..2f92679e 100755 --- a/tests/test_testgres_remote.py +++ b/tests/test_testgres_remote.py @@ -246,7 +246,7 @@ def helper__get_node(name=None): assert isinstance(svc, PostgresNodeService) assert isinstance(svc.os_ops, testgres.OsOperations) - assert isinstance(svc.port_manager, testgres.PostgresNodePortManager) + assert isinstance(svc.port_manager, testgres.PortManager) return testgres.PostgresNode( name, From 09670578fad1a2f25cad96b0cba7f4a5c07c3230 Mon Sep 17 00:00:00 2001 From: "d.kovalenko" Date: Fri, 4 Apr 2025 19:22:56 +0300 Subject: [PATCH 13/31] TestTestgresCommon.test_unix_sockets is added --- tests/test_testgres_common.py | 20 ++++++++++++++++++++ tests/test_testgres_local.py | 12 ------------ tests/test_testgres_remote.py | 16 ---------------- 3 files changed, 20 insertions(+), 28 deletions(-) diff --git a/tests/test_testgres_common.py b/tests/test_testgres_common.py index f2d9c074..27ebf23c 100644 --- a/tests/test_testgres_common.py +++ b/tests/test_testgres_common.py @@ -1121,6 +1121,26 @@ def test_pgbench(self, node_svc: PostgresNodeService): out = proc.communicate()[0] assert (b'tps = ' in out) + def test_unix_sockets(self, node_svc: PostgresNodeService): + assert isinstance(node_svc, PostgresNodeService) + + with __class__.helper__get_node(node_svc) as node: + node.init(unix_sockets=False, allow_streaming=True) + node.start() + + res_exec = node.execute('select 1') + assert (res_exec == [(1,)]) + res_psql = node.safe_psql('select 1') + assert (res_psql == b'1\n') + + with node.replicate() as r: + assert type(r) == PostgresNode # noqa: E721 + r.start() + res_exec = r.execute('select 1') + assert (res_exec == [(1,)]) + res_psql = r.safe_psql('select 1') + assert (res_psql == b'1\n') + @staticmethod def helper__get_node(node_svc: PostgresNodeService, name=None): assert isinstance(node_svc, PostgresNodeService) diff --git a/tests/test_testgres_local.py b/tests/test_testgres_local.py index d326dd9e..45bade42 100644 --- a/tests/test_testgres_local.py +++ b/tests/test_testgres_local.py @@ -156,18 +156,6 @@ def test_config_stack(self): assert (TestgresConfig.cached_initdb_dir == d0) - def test_unix_sockets(self): - with get_new_node() as node: - node.init(unix_sockets=False, allow_streaming=True) - node.start() - - node.execute('select 1') - node.safe_psql('select 1') - - with node.replicate().start() as r: - r.execute('select 1') - r.safe_psql('select 1') - def test_ports_management(self): assert bound_ports is not None assert type(bound_ports) == set # noqa: E721 diff --git a/tests/test_testgres_remote.py b/tests/test_testgres_remote.py index 2f92679e..ef4bd0c8 100755 --- a/tests/test_testgres_remote.py +++ b/tests/test_testgres_remote.py @@ -224,22 +224,6 @@ def test_config_stack(self): assert (TestgresConfig.cached_initdb_dir == d0) - def test_unix_sockets(self): - with __class__.helper__get_node() as node: - node.init(unix_sockets=False, allow_streaming=True) - node.start() - - res_exec = node.execute('select 1') - res_psql = node.safe_psql('select 1') - assert (res_exec == [(1,)]) - assert (res_psql == b'1\n') - - with node.replicate().start() as r: - res_exec = r.execute('select 1') - res_psql = r.safe_psql('select 1') - assert (res_exec == [(1,)]) - assert (res_psql == b'1\n') - @staticmethod def helper__get_node(name=None): svc = PostgresNodeServices.sm_remote From 1d450b2a71f15c9a7a4a471c7213f6a326b3e02b Mon Sep 17 00:00:00 2001 From: "d.kovalenko" Date: Fri, 4 Apr 2025 19:39:56 +0300 Subject: [PATCH 14/31] TestTestgresCommon.test_the_same_port is added --- tests/test_testgres_common.py | 34 ++++++++++++++++++++++++++++++++-- tests/test_testgres_local.py | 24 ------------------------ 2 files changed, 32 insertions(+), 26 deletions(-) diff --git a/tests/test_testgres_common.py b/tests/test_testgres_common.py index 27ebf23c..1f0c070d 100644 --- a/tests/test_testgres_common.py +++ b/tests/test_testgres_common.py @@ -1141,16 +1141,46 @@ def test_unix_sockets(self, node_svc: PostgresNodeService): res_psql = r.safe_psql('select 1') assert (res_psql == b'1\n') + def test_the_same_port(self, node_svc: PostgresNodeService): + assert isinstance(node_svc, PostgresNodeService) + + with __class__.helper__get_node(node_svc) as node: + node.init().start() + assert (node._should_free_port) + assert (type(node.port) == int) # noqa: E721 + node_port_copy = node.port + r = node.safe_psql("SELECT 1;") + assert (__class__.helper__rm_carriage_returns(r) == b'1\n') + + with __class__.helper__get_node(node_svc, port=node.port) as node2: + assert (type(node2.port) == int) # noqa: E721 + assert (node2.port == node.port) + assert not (node2._should_free_port) + + with pytest.raises( + expected_exception=StartNodeException, + match=re.escape("Cannot start node") + ): + node2.init().start() + + # node is still working + assert (node.port == node_port_copy) + assert (node._should_free_port) + r = node.safe_psql("SELECT 3;") + assert (__class__.helper__rm_carriage_returns(r) == b'3\n') + @staticmethod - def helper__get_node(node_svc: PostgresNodeService, name=None): + def helper__get_node(node_svc: PostgresNodeService, name=None, port=None): assert isinstance(node_svc, PostgresNodeService) assert isinstance(node_svc.os_ops, OsOperations) assert isinstance(node_svc.port_manager, PortManager) return PostgresNode( name, + port=port, conn_params=None, os_ops=node_svc.os_ops, - port_manager=node_svc.port_manager) + port_manager=node_svc.port_manager if port is None else None + ) @staticmethod def helper__skip_test_if_pg_version_is_not_ge(ver1: str, ver2: str): diff --git a/tests/test_testgres_local.py b/tests/test_testgres_local.py index 45bade42..bef80d0f 100644 --- a/tests/test_testgres_local.py +++ b/tests/test_testgres_local.py @@ -244,30 +244,6 @@ def test_parse_pg_version(self): # Macos assert parse_pg_version("postgres (PostgreSQL) 14.9 (Homebrew)") == "14.9" - def test_the_same_port(self): - with get_new_node() as node: - node.init().start() - assert (node._should_free_port) - assert (type(node.port) == int) # noqa: E721 - node_port_copy = node.port - assert (rm_carriage_returns(node.safe_psql("SELECT 1;")) == b'1\n') - - with get_new_node(port=node.port) as node2: - assert (type(node2.port) == int) # noqa: E721 - assert (node2.port == node.port) - assert not (node2._should_free_port) - - with pytest.raises( - expected_exception=StartNodeException, - match=re.escape("Cannot start node") - ): - node2.init().start() - - # node is still working - assert (node.port == node_port_copy) - assert (node._should_free_port) - assert (rm_carriage_returns(node.safe_psql("SELECT 3;")) == b'3\n') - class tagPortManagerProxy: sm_prev_testgres_reserve_port = None sm_prev_testgres_release_port = None From 4a38b35dda82360a59e6c73f7614998810bcf248 Mon Sep 17 00:00:00 2001 From: "d.kovalenko" Date: Fri, 4 Apr 2025 20:39:12 +0300 Subject: [PATCH 15/31] [TestTestgresCommon] New tests are added - test_port_rereserve_during_node_start - test_port_conflict --- tests/test_testgres_common.py | 167 +++++++++++++++++++++++++++++++++- 1 file changed, 165 insertions(+), 2 deletions(-) diff --git a/tests/test_testgres_common.py b/tests/test_testgres_common.py index 1f0c070d..85368824 100644 --- a/tests/test_testgres_common.py +++ b/tests/test_testgres_common.py @@ -13,6 +13,8 @@ from ..testgres import NodeStatus from ..testgres import IsolationLevel +import testgres + # New name prevents to collect test-functions in TestgresException and fixes # the problem with pytest warning. from ..testgres import TestgresException as testgres_TestgresException @@ -39,6 +41,7 @@ import os import re import subprocess +import typing @contextmanager @@ -1169,17 +1172,177 @@ def test_the_same_port(self, node_svc: PostgresNodeService): r = node.safe_psql("SELECT 3;") assert (__class__.helper__rm_carriage_returns(r) == b'3\n') + class tagPortManagerProxy(PortManager): + m_PrevPortManager: PortManager + + m_DummyPortNumber: int + m_DummyPortMaxUsage: int + + m_DummyPortCurrentUsage: int + m_DummyPortTotalUsage: int + + def __init__(self, prevPortManager: PortManager, dummyPortNumber: int, dummyPortMaxUsage: int): + assert isinstance(prevPortManager, PortManager) + assert type(dummyPortNumber) == int # noqa: E721 + assert type(dummyPortMaxUsage) == int # noqa: E721 + assert dummyPortNumber >= 0 + assert dummyPortMaxUsage >= 0 + + super().__init__() + + self.m_PrevPortManager = prevPortManager + + self.m_DummyPortNumber = dummyPortNumber + self.m_DummyPortMaxUsage = dummyPortMaxUsage + + self.m_DummyPortCurrentUsage = 0 + self.m_DummyPortTotalUsage = 0 + + def __enter__(self): + return self + + def __exit__(self, type, value, traceback): + assert self.m_DummyPortCurrentUsage == 0 + + assert self.m_PrevPortManager is not None + + def reserve_port(self) -> int: + assert type(self.m_DummyPortMaxUsage) == int # noqa: E721 + assert type(self.m_DummyPortTotalUsage) == int # noqa: E721 + assert type(self.m_DummyPortCurrentUsage) == int # noqa: E721 + assert self.m_DummyPortTotalUsage >= 0 + assert self.m_DummyPortCurrentUsage >= 0 + + assert self.m_DummyPortTotalUsage <= self.m_DummyPortMaxUsage + assert self.m_DummyPortCurrentUsage <= self.m_DummyPortTotalUsage + + assert self.m_PrevPortManager is not None + assert isinstance(self.m_PrevPortManager, PortManager) + + if self.m_DummyPortTotalUsage == self.m_DummyPortMaxUsage: + return self.m_PrevPortManager.reserve_port() + + self.m_DummyPortTotalUsage += 1 + self.m_DummyPortCurrentUsage += 1 + return self.m_DummyPortNumber + + def release_port(self, dummyPortNumber: int): + assert type(dummyPortNumber) == int # noqa: E721 + + assert type(self.m_DummyPortMaxUsage) == int # noqa: E721 + assert type(self.m_DummyPortTotalUsage) == int # noqa: E721 + assert type(self.m_DummyPortCurrentUsage) == int # noqa: E721 + assert self.m_DummyPortTotalUsage >= 0 + assert self.m_DummyPortCurrentUsage >= 0 + + assert self.m_DummyPortTotalUsage <= self.m_DummyPortMaxUsage + assert self.m_DummyPortCurrentUsage <= self.m_DummyPortTotalUsage + + assert self.m_PrevPortManager is not None + assert isinstance(self.m_PrevPortManager, PortManager) + + if self.m_DummyPortCurrentUsage > 0 and dummyPortNumber == self.m_DummyPortNumber: + assert self.m_DummyPortTotalUsage > 0 + self.m_DummyPortCurrentUsage -= 1 + return + + return self.m_PrevPortManager.release_port(dummyPortNumber) + + def test_port_rereserve_during_node_start(self, node_svc: PostgresNodeService): + assert type(node_svc) == PostgresNodeService # noqa: E721 + assert testgres.PostgresNode._C_MAX_START_ATEMPTS == 5 + + C_COUNT_OF_BAD_PORT_USAGE = 3 + + with __class__.helper__get_node(node_svc) as node1: + node1.init().start() + assert node1._should_free_port + assert type(node1.port) == int # noqa: E721 + node1_port_copy = node1.port + assert __class__.helper__rm_carriage_returns(node1.safe_psql("SELECT 1;")) == b'1\n' + + with __class__.tagPortManagerProxy(node_svc.port_manager, node1.port, C_COUNT_OF_BAD_PORT_USAGE) as proxy: + assert proxy.m_DummyPortNumber == node1.port + with __class__.helper__get_node(node_svc, port_manager=proxy) as node2: + assert node2._should_free_port + assert node2.port == node1.port + + node2.init().start() + + assert node2.port != node1.port + assert node2._should_free_port + assert proxy.m_DummyPortCurrentUsage == 0 + assert proxy.m_DummyPortTotalUsage == C_COUNT_OF_BAD_PORT_USAGE + assert node2.is_started + r = node2.safe_psql("SELECT 2;") + assert __class__.helper__rm_carriage_returns(r) == b'2\n' + + # node1 is still working + assert node1.port == node1_port_copy + assert node1._should_free_port + r = node1.safe_psql("SELECT 3;") + assert __class__.helper__rm_carriage_returns(r) == b'3\n' + + def test_port_conflict(self, node_svc: PostgresNodeService): + assert type(node_svc) == PostgresNodeService # noqa: E721 + assert testgres.PostgresNode._C_MAX_START_ATEMPTS > 1 + + C_COUNT_OF_BAD_PORT_USAGE = testgres.PostgresNode._C_MAX_START_ATEMPTS + + with __class__.helper__get_node(node_svc) as node1: + node1.init().start() + assert node1._should_free_port + assert type(node1.port) == int # noqa: E721 + node1_port_copy = node1.port + assert __class__.helper__rm_carriage_returns(node1.safe_psql("SELECT 1;")) == b'1\n' + + with __class__.tagPortManagerProxy(node_svc.port_manager, node1.port, C_COUNT_OF_BAD_PORT_USAGE) as proxy: + assert proxy.m_DummyPortNumber == node1.port + with __class__.helper__get_node(node_svc, port_manager=proxy) as node2: + assert node2._should_free_port + assert node2.port == node1.port + + with pytest.raises( + expected_exception=StartNodeException, + match=re.escape("Cannot start node after multiple attempts.") + ): + node2.init().start() + + assert node2.port == node1.port + assert node2._should_free_port + assert proxy.m_DummyPortCurrentUsage == 1 + assert proxy.m_DummyPortTotalUsage == C_COUNT_OF_BAD_PORT_USAGE + assert not node2.is_started + + # node2 must release our dummyPort (node1.port) + assert (proxy.m_DummyPortCurrentUsage == 0) + + # node1 is still working + assert node1.port == node1_port_copy + assert node1._should_free_port + r = node1.safe_psql("SELECT 3;") + assert __class__.helper__rm_carriage_returns(r) == b'3\n' + @staticmethod - def helper__get_node(node_svc: PostgresNodeService, name=None, port=None): + def helper__get_node( + node_svc: PostgresNodeService, + name: typing.Optional[str] = None, + port: typing.Optional[int] = None, + port_manager: typing.Optional[PortManager] = None + ) -> PostgresNode: assert isinstance(node_svc, PostgresNodeService) assert isinstance(node_svc.os_ops, OsOperations) assert isinstance(node_svc.port_manager, PortManager) + + if port_manager is None: + port_manager = node_svc.port_manager + return PostgresNode( name, port=port, conn_params=None, os_ops=node_svc.os_ops, - port_manager=node_svc.port_manager if port is None else None + port_manager=port_manager if port is None else None ) @staticmethod From 322fb237d3ce6d43c950ee5698113fbe36806944 Mon Sep 17 00:00:00 2001 From: "d.kovalenko" Date: Fri, 4 Apr 2025 22:43:36 +0300 Subject: [PATCH 16/31] RemoteOperations::is_port_free is updated --- testgres/operations/remote_ops.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/testgres/operations/remote_ops.py b/testgres/operations/remote_ops.py index 0547a262..f0a172b8 100644 --- a/testgres/operations/remote_ops.py +++ b/testgres/operations/remote_ops.py @@ -640,7 +640,7 @@ def is_port_free(self, number: int) -> bool: assert type(error) == str # noqa: E721 if exit_status == 0: - return __class__.helper__is_port_free__process_0(output) + return __class__.helper__is_port_free__process_0(error) if exit_status == 1: return __class__.helper__is_port_free__process_1(error) @@ -656,15 +656,15 @@ def is_port_free(self, number: int) -> bool: ) @staticmethod - def helper__is_port_free__process_0(output: str) -> bool: - assert type(output) == str # noqa: E721 - # TODO: check output message + def helper__is_port_free__process_0(error: str) -> bool: + assert type(error) == str # noqa: E721 + # TODO: check error message? return False @staticmethod def helper__is_port_free__process_1(error: str) -> bool: assert type(error) == str # noqa: E721 - # TODO: check error message + # TODO: check error message? return True # Database control From 94da63ee0f0cf6cb30393f735650336f73331f5e Mon Sep 17 00:00:00 2001 From: "d.kovalenko" Date: Fri, 4 Apr 2025 22:44:25 +0300 Subject: [PATCH 17/31] Tests for OsOps::is_port_free are added --- tests/test_os_ops_common.py | 99 +++++++++++++++++++++++++++++++++++++ 1 file changed, 99 insertions(+) diff --git a/tests/test_os_ops_common.py b/tests/test_os_ops_common.py index 1bcc054c..dfea848b 100644 --- a/tests/test_os_ops_common.py +++ b/tests/test_os_ops_common.py @@ -10,6 +10,8 @@ import re import tempfile import logging +import socket +import threading from ..testgres import InvalidOperationException from ..testgres import ExecUtilException @@ -648,3 +650,100 @@ def test_touch(self, os_ops: OsOperations): assert os_ops.isfile(filename) os_ops.remove_file(filename) + + def test_is_port_free__true(self, os_ops: OsOperations): + assert isinstance(os_ops, OsOperations) + + C_LIMIT = 10 + + ports = set(range(1024, 65535)) + assert type(ports) == set # noqa: E721 + + ok_count = 0 + no_count = 0 + + for port in ports: + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: + try: + s.bind(("", port)) + except OSError: + continue + + r = os_ops.is_port_free(port) + + if r: + ok_count += 1 + logging.info("OK. Port {} is free.".format(port)) + else: + no_count += 1 + logging.warning("NO. Port {} is not free.".format(port)) + + if ok_count == C_LIMIT: + return + + if no_count == C_LIMIT: + raise RuntimeError("To many false positive test attempts.") + + if ok_count == 0: + raise RuntimeError("No one free port was found.") + + def test_is_port_free__false(self, os_ops: OsOperations): + assert isinstance(os_ops, OsOperations) + + C_LIMIT = 10 + + ports = set(range(1024, 65535)) + assert type(ports) == set # noqa: E721 + + def LOCAL_server(s: socket.socket): + assert s is not None + assert type(s) == socket.socket # noqa: E721 + + try: + while True: + r = s.accept() + + if r is None: + break + except Exception as e: + assert e is not None + pass + + ok_count = 0 + no_count = 0 + + for port in ports: + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: + try: + s.bind(("", port)) + except OSError: + continue + + th = threading.Thread(target=LOCAL_server, args=[s]) + + s.listen(10) + + assert type(th) == threading.Thread # noqa: E721 + th.start() + + try: + r = os_ops.is_port_free(port) + finally: + s.shutdown(2) + th.join() + + if not r: + ok_count += 1 + logging.info("OK. Port {} is not free.".format(port)) + else: + no_count += 1 + logging.warning("NO. Port {} does not accept connection.".format(port)) + + if ok_count == C_LIMIT: + return + + if no_count == C_LIMIT: + raise RuntimeError("To many false positive test attempts.") + + if ok_count == 0: + raise RuntimeError("No one free port was found.") From 88f9b73da7f1cd5a91ecf88dba451b68ae9edc70 Mon Sep 17 00:00:00 2001 From: "d.kovalenko" Date: Fri, 4 Apr 2025 23:14:27 +0300 Subject: [PATCH 18/31] TestTestgresCommon is corrected [python problems] --- tests/test_testgres_common.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/tests/test_testgres_common.py b/tests/test_testgres_common.py index 85368824..2fd678ef 100644 --- a/tests/test_testgres_common.py +++ b/tests/test_testgres_common.py @@ -13,8 +13,6 @@ from ..testgres import NodeStatus from ..testgres import IsolationLevel -import testgres - # New name prevents to collect test-functions in TestgresException and fixes # the problem with pytest warning. from ..testgres import TestgresException as testgres_TestgresException @@ -1250,7 +1248,7 @@ def release_port(self, dummyPortNumber: int): def test_port_rereserve_during_node_start(self, node_svc: PostgresNodeService): assert type(node_svc) == PostgresNodeService # noqa: E721 - assert testgres.PostgresNode._C_MAX_START_ATEMPTS == 5 + assert PostgresNode._C_MAX_START_ATEMPTS == 5 C_COUNT_OF_BAD_PORT_USAGE = 3 @@ -1285,9 +1283,9 @@ def test_port_rereserve_during_node_start(self, node_svc: PostgresNodeService): def test_port_conflict(self, node_svc: PostgresNodeService): assert type(node_svc) == PostgresNodeService # noqa: E721 - assert testgres.PostgresNode._C_MAX_START_ATEMPTS > 1 + assert PostgresNode._C_MAX_START_ATEMPTS > 1 - C_COUNT_OF_BAD_PORT_USAGE = testgres.PostgresNode._C_MAX_START_ATEMPTS + C_COUNT_OF_BAD_PORT_USAGE = PostgresNode._C_MAX_START_ATEMPTS with __class__.helper__get_node(node_svc) as node1: node1.init().start() From d9558cedf01d3dbeac9d31f075edfa613e501e2e Mon Sep 17 00:00:00 2001 From: "d.kovalenko" Date: Sat, 5 Apr 2025 00:17:03 +0300 Subject: [PATCH 19/31] The call of RaiseError.CommandExecutionError is fixed [message, not msg_arg] - RemoteOperations::message - RemoteOperations::path_exists - RemoteOperations::is_port_free --- testgres/operations/remote_ops.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/testgres/operations/remote_ops.py b/testgres/operations/remote_ops.py index f0a172b8..6a678745 100644 --- a/testgres/operations/remote_ops.py +++ b/testgres/operations/remote_ops.py @@ -193,7 +193,7 @@ def is_executable(self, file): RaiseError.CommandExecutionError( cmd=command, exit_code=exit_status, - msg_arg=errMsg, + message=errMsg, error=error, out=output ) @@ -305,7 +305,7 @@ def path_exists(self, path): RaiseError.CommandExecutionError( cmd=command, exit_code=exit_status, - msg_arg=errMsg, + message=errMsg, error=error, out=output ) @@ -650,7 +650,7 @@ def is_port_free(self, number: int) -> bool: RaiseError.CommandExecutionError( cmd=cmd, exit_code=exit_status, - msg_arg=errMsg, + message=errMsg, error=error, out=output ) From 0da4c21075cfa5dbb4b011e81b516362fd2fb77b Mon Sep 17 00:00:00 2001 From: "d.kovalenko" Date: Sat, 5 Apr 2025 00:35:18 +0300 Subject: [PATCH 20/31] [CI] ubuntu 24.04 does not have nc Let's install it (netcat-traditional) explicitly. --- Dockerfile--ubuntu_24_04.tmpl | 1 + 1 file changed, 1 insertion(+) diff --git a/Dockerfile--ubuntu_24_04.tmpl b/Dockerfile--ubuntu_24_04.tmpl index 3bdc6640..7a559776 100644 --- a/Dockerfile--ubuntu_24_04.tmpl +++ b/Dockerfile--ubuntu_24_04.tmpl @@ -10,6 +10,7 @@ RUN apt install -y sudo curl ca-certificates RUN apt update RUN apt install -y openssh-server RUN apt install -y time +RUN apt install -y netcat-traditional RUN apt update RUN apt install -y postgresql-common From 0058508b12cb4354193b684cef26b3a054fcb66e Mon Sep 17 00:00:00 2001 From: "d.kovalenko" Date: Sat, 5 Apr 2025 09:43:06 +0300 Subject: [PATCH 21/31] RemoteOperations is update [private method names] --- testgres/operations/remote_ops.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/testgres/operations/remote_ops.py b/testgres/operations/remote_ops.py index 6a678745..21caa560 100644 --- a/testgres/operations/remote_ops.py +++ b/testgres/operations/remote_ops.py @@ -640,10 +640,10 @@ def is_port_free(self, number: int) -> bool: assert type(error) == str # noqa: E721 if exit_status == 0: - return __class__.helper__is_port_free__process_0(error) + return __class__._is_port_free__process_0(error) if exit_status == 1: - return __class__.helper__is_port_free__process_1(error) + return __class__._is_port_free__process_1(error) errMsg = "nc returns an unknown result code: {0}".format(exit_status) @@ -656,13 +656,13 @@ def is_port_free(self, number: int) -> bool: ) @staticmethod - def helper__is_port_free__process_0(error: str) -> bool: + def _is_port_free__process_0(error: str) -> bool: assert type(error) == str # noqa: E721 # TODO: check error message? return False @staticmethod - def helper__is_port_free__process_1(error: str) -> bool: + def _is_port_free__process_1(error: str) -> bool: assert type(error) == str # noqa: E721 # TODO: check error message? return True From 8f3a56603ccbe931301bb96bf91718bf744793c9 Mon Sep 17 00:00:00 2001 From: "d.kovalenko" Date: Sat, 5 Apr 2025 09:44:48 +0300 Subject: [PATCH 22/31] test_is_port_free__true is updated A number of attempts is increased to 128. The previous value (10) is not enough and test could fail. --- tests/test_os_ops_common.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_os_ops_common.py b/tests/test_os_ops_common.py index dfea848b..7d183775 100644 --- a/tests/test_os_ops_common.py +++ b/tests/test_os_ops_common.py @@ -654,7 +654,7 @@ def test_touch(self, os_ops: OsOperations): def test_is_port_free__true(self, os_ops: OsOperations): assert isinstance(os_ops, OsOperations) - C_LIMIT = 10 + C_LIMIT = 128 ports = set(range(1024, 65535)) assert type(ports) == set # noqa: E721 From d8ebdb76baba0ec36c7de9456c5e375faae54b50 Mon Sep 17 00:00:00 2001 From: "d.kovalenko" Date: Sat, 5 Apr 2025 09:53:01 +0300 Subject: [PATCH 23/31] RemoteOperations::is_port_free is updated (comments) --- testgres/operations/remote_ops.py | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/testgres/operations/remote_ops.py b/testgres/operations/remote_ops.py index 21caa560..ee747e52 100644 --- a/testgres/operations/remote_ops.py +++ b/testgres/operations/remote_ops.py @@ -658,13 +658,23 @@ def is_port_free(self, number: int) -> bool: @staticmethod def _is_port_free__process_0(error: str) -> bool: assert type(error) == str # noqa: E721 - # TODO: check error message? + # + # Example of error text: + # "Connection to localhost (127.0.0.1) 1024 port [tcp/*] succeeded!\n" + # + # May be here is needed to check error message? + # return False @staticmethod def _is_port_free__process_1(error: str) -> bool: assert type(error) == str # noqa: E721 - # TODO: check error message? + # + # Example of error text: + # "nc: connect to localhost (127.0.0.1) port 1024 (tcp) failed: Connection refused\n" + # + # May be here is needed to check error message? + # return True # Database control From 30e472c13271a7d5a851a9455b44bc735138645b Mon Sep 17 00:00:00 2001 From: "d.kovalenko" Date: Sat, 5 Apr 2025 12:12:57 +0300 Subject: [PATCH 24/31] setup.py is updated [testgres.helpers was deleted] --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 3f2474dd..b47a1d8a 100755 --- a/setup.py +++ b/setup.py @@ -29,7 +29,7 @@ setup( version='1.10.5', name='testgres', - packages=['testgres', 'testgres.operations', 'testgres.helpers'], + packages=['testgres', 'testgres.operations'], description='Testing utility for PostgreSQL and its extensions', url='https://github.com/postgrespro/testgres', long_description=readme, From c94bbb58e6a19a2ad6ba94897eacb4b40179bd46 Mon Sep 17 00:00:00 2001 From: "d.kovalenko" Date: Sat, 5 Apr 2025 20:14:13 +0300 Subject: [PATCH 25/31] Comment in node.py is updated --- testgres/node.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/testgres/node.py b/testgres/node.py index 99ec2032..ab516a6a 100644 --- a/testgres/node.py +++ b/testgres/node.py @@ -276,7 +276,7 @@ def _get_port_manager(os_ops: OsOperations) -> PortManager: if isinstance(os_ops, LocalOperations): return PortManager__ThisHost() - # TODO: Throw exception "Please define a port manager." + # TODO: Throw the exception "Please define a port manager." ? return PortManager__Generic(os_ops) def clone_with_new_name_and_base_dir(self, name: str, base_dir: str): From 04f88c7a5ebc6743615b9e4656a04570665f5184 Mon Sep 17 00:00:00 2001 From: "d.kovalenko" Date: Sat, 5 Apr 2025 20:15:09 +0300 Subject: [PATCH 26/31] PostgresNode::_node was deleted [use self._os_ops.host] --- testgres/node.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/testgres/node.py b/testgres/node.py index ab516a6a..15bf3246 100644 --- a/testgres/node.py +++ b/testgres/node.py @@ -142,7 +142,6 @@ class PostgresNode(object): _C_MAX_START_ATEMPTS = 5 _name: typing.Optional[str] - _host: typing.Optional[str] _port: typing.Optional[int] _should_free_port: bool _os_ops: OsOperations @@ -193,7 +192,8 @@ def __init__(self, # basic self._name = name or generate_app_name() - self._host = self._os_ops.host + + assert hasattr(os_ops, "host") if port is not None: assert type(port) == int # noqa: E721 @@ -319,10 +319,9 @@ def name(self) -> str: @property def host(self) -> str: - if self._host is None: - raise InvalidOperationException("PostgresNode host is not defined.") - assert type(self._host) == str # noqa: E721 - return self._host + assert self._os_ops is not None + assert isinstance(self._os_ops, OsOperations) + return self._os_ops.host @property def port(self) -> int: From 0a3442afab05d0d81e18c7e0a200e4001b2b07e2 Mon Sep 17 00:00:00 2001 From: "d.kovalenko" Date: Sat, 5 Apr 2025 20:17:44 +0300 Subject: [PATCH 27/31] PostgresNode::start is corrected [error message] --- testgres/node.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/testgres/node.py b/testgres/node.py index 15bf3246..9902554c 100644 --- a/testgres/node.py +++ b/testgres/node.py @@ -1088,7 +1088,7 @@ def start(self, params=[], wait=True): return self if self._port is None: - raise InvalidOperationException("Can't start PostgresNode. Port is node defined.") + raise InvalidOperationException("Can't start PostgresNode. Port is not defined.") assert type(self._port) == int # noqa: E721 From 13e71d85b39d2f76d944c30f612bde45ddc15b23 Mon Sep 17 00:00:00 2001 From: "d.kovalenko" Date: Sat, 5 Apr 2025 20:35:22 +0300 Subject: [PATCH 28/31] [FIX] PostgresNode.__init__ must not test "os_ops.host" attribute. - [del] assert hasattr(os_ops, "host") During this test we get another exception: <[AttributeError("'PostgresNode' object has no attribute '_port'") raised in repr()] PostgresNode object at 0x782b67d79dc0> --- testgres/node.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/testgres/node.py b/testgres/node.py index 9902554c..7c1ee136 100644 --- a/testgres/node.py +++ b/testgres/node.py @@ -193,8 +193,6 @@ def __init__(self, # basic self._name = name or generate_app_name() - assert hasattr(os_ops, "host") - if port is not None: assert type(port) == int # noqa: E721 assert port_manager is None From 9e14f4a95f5914d7e3c1003cf57d69135b4d44b4 Mon Sep 17 00:00:00 2001 From: "d.kovalenko" Date: Sun, 6 Apr 2025 10:16:56 +0300 Subject: [PATCH 29/31] PostgresNode.free_port always set a port to None Tests are added: - test_try_to_get_port_after_free_manual_port - test_try_to_start_node_after_free_manual_port --- testgres/node.py | 7 +++-- tests/test_testgres_common.py | 59 +++++++++++++++++++++++++++++++++++ 2 files changed, 64 insertions(+), 2 deletions(-) diff --git a/testgres/node.py b/testgres/node.py index 7c1ee136..7457bf7e 100644 --- a/testgres/node.py +++ b/testgres/node.py @@ -1321,10 +1321,13 @@ def pg_ctl(self, params): def free_port(self): """ Reclaim port owned by this node. - NOTE: does not free auto selected ports. + NOTE: this method does not release manually defined port but reset it. """ + assert type(self._should_free_port) == bool - if self._should_free_port: + if not self._should_free_port: + self._port = None + else: assert type(self._port) == int # noqa: E721 assert self._port_manager is not None diff --git a/tests/test_testgres_common.py b/tests/test_testgres_common.py index 2fd678ef..b52883de 100644 --- a/tests/test_testgres_common.py +++ b/tests/test_testgres_common.py @@ -1321,6 +1321,65 @@ def test_port_conflict(self, node_svc: PostgresNodeService): r = node1.safe_psql("SELECT 3;") assert __class__.helper__rm_carriage_returns(r) == b'3\n' + def test_try_to_get_port_after_free_manual_port(self, node_svc: PostgresNodeService): + assert type(node_svc) == PostgresNodeService # noqa: E721 + + assert node_svc.port_manager is not None + assert isinstance(node_svc.port_manager, PortManager) + + with __class__.helper__get_node(node_svc) as node1: + assert node1 is not None + assert type(node1) == PostgresNode + assert node1.port is not None + assert type(node1.port) == int + with __class__.helper__get_node(node_svc, port=node1.port, port_manager=None) as node2: + assert node2 is not None + assert type(node1) == PostgresNode + assert node2 is not node1 + assert node2.port is not None + assert type(node2.port) == int + assert node2.port == node1.port + + logging.info("Release node2 port") + node2.free_port() + + logging.info("try to get node2.port...") + with pytest.raises( + InvalidOperationException, + match="^" + re.escape("PostgresNode port is not defined.") + "$" + ): + p = node2.port + assert p is None + + def test_try_to_start_node_after_free_manual_port(self, node_svc: PostgresNodeService): + assert type(node_svc) == PostgresNodeService # noqa: E721 + + assert node_svc.port_manager is not None + assert isinstance(node_svc.port_manager, PortManager) + + with __class__.helper__get_node(node_svc) as node1: + assert node1 is not None + assert type(node1) == PostgresNode + assert node1.port is not None + assert type(node1.port) == int + with __class__.helper__get_node(node_svc, port=node1.port, port_manager=None) as node2: + assert node2 is not None + assert type(node1) == PostgresNode + assert node2 is not node1 + assert node2.port is not None + assert type(node2.port) == int + assert node2.port == node1.port + + logging.info("Release node2 port") + node2.free_port() + + logging.info("node2 is trying to start...") + with pytest.raises( + InvalidOperationException, + match="^" + re.escape("Can't start PostgresNode. Port is not defined.") + "$" + ): + node2.start() + @staticmethod def helper__get_node( node_svc: PostgresNodeService, From 739ef617cce66cb36ff5c4db8015ee4684f7d3e7 Mon Sep 17 00:00:00 2001 From: "d.kovalenko" Date: Sun, 6 Apr 2025 10:51:38 +0300 Subject: [PATCH 30/31] [FIX] flake8 (noqa: E721) --- testgres/node.py | 2 +- tests/test_testgres_common.py | 16 ++++++++-------- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/testgres/node.py b/testgres/node.py index 7457bf7e..5039fc43 100644 --- a/testgres/node.py +++ b/testgres/node.py @@ -1323,7 +1323,7 @@ def free_port(self): Reclaim port owned by this node. NOTE: this method does not release manually defined port but reset it. """ - assert type(self._should_free_port) == bool + assert type(self._should_free_port) == bool # noqa: E721 if not self._should_free_port: self._port = None diff --git a/tests/test_testgres_common.py b/tests/test_testgres_common.py index b52883de..b286a1c6 100644 --- a/tests/test_testgres_common.py +++ b/tests/test_testgres_common.py @@ -1329,15 +1329,15 @@ def test_try_to_get_port_after_free_manual_port(self, node_svc: PostgresNodeServ with __class__.helper__get_node(node_svc) as node1: assert node1 is not None - assert type(node1) == PostgresNode + assert type(node1) == PostgresNode # noqa: E721 assert node1.port is not None - assert type(node1.port) == int + assert type(node1.port) == int # noqa: E721 with __class__.helper__get_node(node_svc, port=node1.port, port_manager=None) as node2: assert node2 is not None - assert type(node1) == PostgresNode + assert type(node1) == PostgresNode # noqa: E721 assert node2 is not node1 assert node2.port is not None - assert type(node2.port) == int + assert type(node2.port) == int # noqa: E721 assert node2.port == node1.port logging.info("Release node2 port") @@ -1359,15 +1359,15 @@ def test_try_to_start_node_after_free_manual_port(self, node_svc: PostgresNodeSe with __class__.helper__get_node(node_svc) as node1: assert node1 is not None - assert type(node1) == PostgresNode + assert type(node1) == PostgresNode # noqa: E721 assert node1.port is not None - assert type(node1.port) == int + assert type(node1.port) == int # noqa: E721 with __class__.helper__get_node(node_svc, port=node1.port, port_manager=None) as node2: assert node2 is not None - assert type(node1) == PostgresNode + assert type(node1) == PostgresNode # noqa: E721 assert node2 is not node1 assert node2.port is not None - assert type(node2.port) == int + assert type(node2.port) == int # noqa: E721 assert node2.port == node1.port logging.info("Release node2 port") From 696cc1ef8a4ba6e438057f773a9833e0fea0eb08 Mon Sep 17 00:00:00 2001 From: "d.kovalenko" Date: Sun, 6 Apr 2025 10:55:24 +0300 Subject: [PATCH 31/31] PortManager__Generic is refactored --- testgres/port_manager.py | 52 ++++++++++++++++++++++++---------------- 1 file changed, 31 insertions(+), 21 deletions(-) diff --git a/testgres/port_manager.py b/testgres/port_manager.py index e4c2180c..164661e7 100644 --- a/testgres/port_manager.py +++ b/testgres/port_manager.py @@ -48,45 +48,55 @@ def release_port(self, number: int) -> None: class PortManager__Generic(PortManager): _os_ops: OsOperations - _allocated_ports_guard: object - _allocated_ports: set[int] + _guard: object + # TODO: is there better to use bitmap fot _available_ports? + _available_ports: set[int] + _reserved_ports: set[int] def __init__(self, os_ops: OsOperations): assert os_ops is not None assert isinstance(os_ops, OsOperations) self._os_ops = os_ops - self._allocated_ports_guard = threading.Lock() - self._allocated_ports = set[int]() + self._guard = threading.Lock() + self._available_ports = set[int](range(1024, 65535)) + self._reserved_ports = set[int]() def reserve_port(self) -> int: - ports = set(range(1024, 65535)) - assert type(ports) == set # noqa: E721 + assert self._guard is not None + assert type(self._available_ports) == set # noqa: E721t + assert type(self._reserved_ports) == set # noqa: E721 - assert self._allocated_ports_guard is not None - assert type(self._allocated_ports) == set # noqa: E721 - - with self._allocated_ports_guard: - ports.difference_update(self._allocated_ports) - - sampled_ports = random.sample(tuple(ports), min(len(ports), 100)) + with self._guard: + t = tuple(self._available_ports) + assert len(t) == len(self._available_ports) + sampled_ports = random.sample(t, min(len(t), 100)) + t = None for port in sampled_ports: - assert not (port in self._allocated_ports) + assert not (port in self._reserved_ports) + assert port in self._available_ports if not self._os_ops.is_port_free(port): continue - self._allocated_ports.add(port) + self._reserved_ports.add(port) + self._available_ports.discard(port) + assert port in self._reserved_ports + assert not (port in self._available_ports) return port - raise PortForException("Can't select a port") + raise PortForException("Can't select a port.") def release_port(self, number: int) -> None: assert type(number) == int # noqa: E721 - assert self._allocated_ports_guard is not None - assert type(self._allocated_ports) == set # noqa: E721 + assert self._guard is not None + assert type(self._reserved_ports) == set # noqa: E721 - with self._allocated_ports_guard: - assert number in self._allocated_ports - self._allocated_ports.discard(number) + with self._guard: + assert number in self._reserved_ports + assert not (number in self._available_ports) + self._available_ports.add(number) + self._reserved_ports.discard(number) + assert not (number in self._reserved_ports) + assert number in self._available_ports pFad - Phonifier reborn

Pfad - The Proxy pFad of © 2024 Garber Painting. All rights reserved.

Note: This service is not intended for secure transactions such as banking, social media, email, or purchasing. Use at your own risk. We assume no liability whatsoever for broken pages.


Alternative Proxies:

Alternative Proxy

pFad Proxy

pFad v3 Proxy

pFad v4 Proxy