Content-Length: 62962 | pFad | http://github.com/postgrespro/testgres/pull/278.patch

thub.com From 17fb8c334aca3372801858d01dfaf4a3b0c9729d Mon Sep 17 00:00:00 2001 From: "d.kovalenko" Date: Thu, 26 Jun 2025 23:09:16 +0300 Subject: [PATCH 01/18] NodeApp was moved to own py-file (refactoring) --- testgres/__init__.py | 8 ++- testgres/node.py | 163 ----------------------------------------- testgres/node_app.py | 168 +++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 173 insertions(+), 166 deletions(-) create mode 100644 testgres/node_app.py diff --git a/testgres/__init__.py b/testgres/__init__.py index 339ae62e..555784bf 100644 --- a/testgres/__init__.py +++ b/testgres/__init__.py @@ -33,8 +33,9 @@ ProcessType, \ DumpFormat -from .node import PostgresNode, NodeApp +from .node import PostgresNode from .node import PortManager +from .node_app import NodeApp from .utils import \ reserve_port, \ @@ -62,8 +63,9 @@ "NodeConnection", "DatabaseError", "InternalError", "ProgrammingError", "OperationalError", "TestgresException", "ExecUtilException", "QueryException", "TimeoutException", "CatchUpException", "StartNodeException", "InitNodeException", "BackupException", "InvalidOperationException", "XLogMethod", "IsolationLevel", "NodeStatus", "ProcessType", "DumpFormat", - "PostgresNode", "NodeApp", - "PortManager", + NodeApp.__name__, + PostgresNode.__name__, + PortManager.__name__, "reserve_port", "release_port", "bound_ports", "get_bin_path", "get_pg_config", "get_pg_version", "First", "Any", "OsOperations", "LocalOperations", "RemoteOperations", "ConnectionParams" diff --git a/testgres/node.py b/testgres/node.py index 208846a1..eba0d896 100644 --- a/testgres/node.py +++ b/testgres/node.py @@ -7,8 +7,6 @@ import signal import subprocess import threading -import tempfile -import platform from queue import Queue import time @@ -2352,164 +2350,3 @@ def delect_port_conflict(log_reader: PostgresNodeLogReader) -> bool: return True return False - - -class NodeApp: - - def __init__(self, test_path=None, nodes_to_cleanup=None, os_ops=None): - assert os_ops is None or isinstance(os_ops, OsOperations) - - if os_ops is None: - os_ops = LocalOperations.get_single_instance() - - assert isinstance(os_ops, OsOperations) - - if test_path: - if os.path.isabs(test_path): - self.test_path = test_path - else: - self.test_path = os_ops.build_path(os_ops.cwd(), test_path) - else: - self.test_path = os_ops.cwd() - self.nodes_to_cleanup = nodes_to_cleanup if nodes_to_cleanup else [] - self.os_ops = os_ops - - def make_empty( - self, - base_dir=None, - port=None, - bin_dir=None): - real_base_dir = self.os_ops.build_path(self.test_path, base_dir) - self.os_ops.rmdirs(real_base_dir, ignore_errors=True) - self.os_ops.makedirs(real_base_dir) - - node = PostgresNode(base_dir=real_base_dir, port=port, bin_dir=bin_dir) - self.nodes_to_cleanup.append(node) - - return node - - def make_simple( - self, - base_dir=None, - port=None, - set_replication=False, - ptrack_enable=False, - initdb_params=[], - pg_options={}, - checksum=True, - bin_dir=None): - assert type(pg_options) == dict # noqa: E721 - - if checksum and '--data-checksums' not in initdb_params: - initdb_params.append('--data-checksums') - node = self.make_empty(base_dir, port, bin_dir=bin_dir) - node.init( - initdb_params=initdb_params, allow_streaming=set_replication) - - # set major version - pg_version_file = self.os_ops.read(self.os_ops.build_path(node.data_dir, 'PG_VERSION')) - node.major_version_str = str(pg_version_file.rstrip()) - node.major_version = float(node.major_version_str) - - # Set default parameters - options = { - 'max_connections': 100, - 'shared_buffers': '10MB', - 'fsync': 'off', - 'wal_level': 'logical', - 'hot_standby': 'off', - 'log_line_prefix': '%t [%p]: [%l-1] ', - 'log_statement': 'none', - 'log_duration': 'on', - 'log_min_duration_statement': 0, - 'log_connections': 'on', - 'log_disconnections': 'on', - 'restart_after_crash': 'off', - 'autovacuum': 'off', - # unix_socket_directories will be defined later - } - - # Allow replication in pg_hba.conf - if set_replication: - options['max_wal_senders'] = 10 - - if ptrack_enable: - options['ptrack.map_size'] = '1' - options['shared_preload_libraries'] = 'ptrack' - - if node.major_version >= 13: - options['wal_keep_size'] = '200MB' - else: - options['wal_keep_segments'] = '12' - - # Apply given parameters - for option_name, option_value in iteritems(pg_options): - options[option_name] = option_value - - # Define delayed propertyes - if not ("unix_socket_directories" in options.keys()): - options["unix_socket_directories"] = __class__._gettempdir_for_socket() - - # Set config values - node.set_auto_conf(options) - - # kludge for testgres - # https://github.com/postgrespro/testgres/issues/54 - # for PG >= 13 remove 'wal_keep_segments' parameter - if node.major_version >= 13: - node.set_auto_conf({}, 'postgresql.conf', ['wal_keep_segments']) - - return node - - @staticmethod - def _gettempdir_for_socket(): - platform_system_name = platform.system().lower() - - if platform_system_name == "windows": - return __class__._gettempdir() - - # - # [2025-02-17] Hot fix. - # - # Let's use hard coded path as Postgres likes. - # - # pg_config_manual.h: - # - # #ifndef WIN32 - # #define DEFAULT_PGSOCKET_DIR "/tmp" - # #else - # #define DEFAULT_PGSOCKET_DIR "" - # #endif - # - # On the altlinux-10 tempfile.gettempdir() may return - # the path to "private" temp directiry - "/temp/.private//" - # - # But Postgres want to find a socket file in "/tmp" (see above). - # - - return "/tmp" - - @staticmethod - def _gettempdir(): - v = tempfile.gettempdir() - - # - # Paranoid checks - # - if type(v) != str: # noqa: E721 - __class__._raise_bugcheck("tempfile.gettempdir returned a value with type {0}.".format(type(v).__name__)) - - if v == "": - __class__._raise_bugcheck("tempfile.gettempdir returned an empty string.") - - if not os.path.exists(v): - __class__._raise_bugcheck("tempfile.gettempdir returned a not exist path [{0}].".format(v)) - - # OK - return v - - @staticmethod - def _raise_bugcheck(msg): - assert type(msg) == str # noqa: E721 - assert msg != "" - raise Exception("[BUG CHECK] " + msg) diff --git a/testgres/node_app.py b/testgres/node_app.py new file mode 100644 index 00000000..5c9848ee --- /dev/null +++ b/testgres/node_app.py @@ -0,0 +1,168 @@ +from .node import OsOperations +from .node import LocalOperations +from .node import PostgresNode + +import os +import platform +import tempfile + + +class NodeApp: + + def __init__(self, test_path=None, nodes_to_cleanup=None, os_ops=None): + assert os_ops is None or isinstance(os_ops, OsOperations) + + if os_ops is None: + os_ops = LocalOperations.get_single_instance() + + assert isinstance(os_ops, OsOperations) + + if test_path: + if os.path.isabs(test_path): + self.test_path = test_path + else: + self.test_path = os_ops.build_path(os_ops.cwd(), test_path) + else: + self.test_path = os_ops.cwd() + self.nodes_to_cleanup = nodes_to_cleanup if nodes_to_cleanup else [] + self.os_ops = os_ops + + def make_empty( + self, + base_dir=None, + port=None, + bin_dir=None): + real_base_dir = self.os_ops.build_path(self.test_path, base_dir) + self.os_ops.rmdirs(real_base_dir, ignore_errors=True) + self.os_ops.makedirs(real_base_dir) + + node = PostgresNode(base_dir=real_base_dir, port=port, bin_dir=bin_dir) + self.nodes_to_cleanup.append(node) + + return node + + def make_simple( + self, + base_dir=None, + port=None, + set_replication=False, + ptrack_enable=False, + initdb_params=[], + pg_options={}, + checksum=True, + bin_dir=None): + assert type(pg_options) == dict # noqa: E721 + + if checksum and '--data-checksums' not in initdb_params: + initdb_params.append('--data-checksums') + node = self.make_empty(base_dir, port, bin_dir=bin_dir) + node.init( + initdb_params=initdb_params, allow_streaming=set_replication) + + # set major version + pg_version_file = self.os_ops.read(self.os_ops.build_path(node.data_dir, 'PG_VERSION')) + node.major_version_str = str(pg_version_file.rstrip()) + node.major_version = float(node.major_version_str) + + # Set default parameters + options = { + 'max_connections': 100, + 'shared_buffers': '10MB', + 'fsync': 'off', + 'wal_level': 'logical', + 'hot_standby': 'off', + 'log_line_prefix': '%t [%p]: [%l-1] ', + 'log_statement': 'none', + 'log_duration': 'on', + 'log_min_duration_statement': 0, + 'log_connections': 'on', + 'log_disconnections': 'on', + 'restart_after_crash': 'off', + 'autovacuum': 'off', + # unix_socket_directories will be defined later + } + + # Allow replication in pg_hba.conf + if set_replication: + options['max_wal_senders'] = 10 + + if ptrack_enable: + options['ptrack.map_size'] = '1' + options['shared_preload_libraries'] = 'ptrack' + + if node.major_version >= 13: + options['wal_keep_size'] = '200MB' + else: + options['wal_keep_segments'] = '12' + + # Apply given parameters + for option_name, option_value in pg_options.items(): + options[option_name] = option_value + + # Define delayed propertyes + if not ("unix_socket_directories" in options.keys()): + options["unix_socket_directories"] = __class__._gettempdir_for_socket() + + # Set config values + node.set_auto_conf(options) + + # kludge for testgres + # https://github.com/postgrespro/testgres/issues/54 + # for PG >= 13 remove 'wal_keep_segments' parameter + if node.major_version >= 13: + node.set_auto_conf({}, 'postgresql.conf', ['wal_keep_segments']) + + return node + + @staticmethod + def _gettempdir_for_socket(): + platform_system_name = platform.system().lower() + + if platform_system_name == "windows": + return __class__._gettempdir() + + # + # [2025-02-17] Hot fix. + # + # Let's use hard coded path as Postgres likes. + # + # pg_config_manual.h: + # + # #ifndef WIN32 + # #define DEFAULT_PGSOCKET_DIR "/tmp" + # #else + # #define DEFAULT_PGSOCKET_DIR "" + # #endif + # + # On the altlinux-10 tempfile.gettempdir() may return + # the path to "private" temp directiry - "/temp/.private//" + # + # But Postgres want to find a socket file in "/tmp" (see above). + # + + return "/tmp" + + @staticmethod + def _gettempdir(): + v = tempfile.gettempdir() + + # + # Paranoid checks + # + if type(v) != str: # noqa: E721 + __class__._raise_bugcheck("tempfile.gettempdir returned a value with type {0}.".format(type(v).__name__)) + + if v == "": + __class__._raise_bugcheck("tempfile.gettempdir returned an empty string.") + + if not os.path.exists(v): + __class__._raise_bugcheck("tempfile.gettempdir returned a not exist path [{0}].".format(v)) + + # OK + return v + + @staticmethod + def _raise_bugcheck(msg): + assert type(msg) == str # noqa: E721 + assert msg != "" + raise Exception("[BUG CHECK] " + msg) From 8eaa2892f5266c645b8e9e0e7b4c18959c0bf5e4 Mon Sep 17 00:00:00 2001 From: "d.kovalenko" Date: Thu, 26 Jun 2025 23:46:53 +0300 Subject: [PATCH 02/18] NodeApp is refactored totally --- testgres/node_app.py | 103 +++++++++++++++++++++++++++++++------------ 1 file changed, 75 insertions(+), 28 deletions(-) diff --git a/testgres/node_app.py b/testgres/node_app.py index 5c9848ee..4abecffb 100644 --- a/testgres/node_app.py +++ b/testgres/node_app.py @@ -5,36 +5,67 @@ import os import platform import tempfile +import typing -class NodeApp: +T_DICT_STR_STR = typing.Dict[str, str] +T_LIST_STR = typing.List[str] + - def __init__(self, test_path=None, nodes_to_cleanup=None, os_ops=None): +class NodeApp: + _os_ops: OsOperations + _test_path: str + + def __init__( + self, + test_path: typing.Optional[str] = None, + nodes_to_cleanup: typing.Optional[list] = None, + os_ops: typing.Optional[OsOperations] = None + ): + assert test_path is None or type(test_path) == str # noqa: E721 assert os_ops is None or isinstance(os_ops, OsOperations) if os_ops is None: os_ops = LocalOperations.get_single_instance() assert isinstance(os_ops, OsOperations) + self._os_ops = os_ops - if test_path: - if os.path.isabs(test_path): - self.test_path = test_path - else: - self.test_path = os_ops.build_path(os_ops.cwd(), test_path) + if test_path is None: + self._test_path = os_ops.cwd() + elif os.path.isabs(test_path): + self._test_path = test_path else: - self.test_path = os_ops.cwd() + self._test_path = os_ops.build_path(os_ops.cwd(), test_path) + self.nodes_to_cleanup = nodes_to_cleanup if nodes_to_cleanup else [] - self.os_ops = os_ops + + @property + def os_ops(self) -> OsOperations: + assert isinstance(self._os_ops, OsOperations) + return self._os_ops + + @property + def test_path(self) -> str: + assert isinstance(self._test_path, OsOperations) + return self._test_path def make_empty( self, - base_dir=None, - port=None, - bin_dir=None): - real_base_dir = self.os_ops.build_path(self.test_path, base_dir) - self.os_ops.rmdirs(real_base_dir, ignore_errors=True) - self.os_ops.makedirs(real_base_dir) + base_dir: typing.Optional[str] = None, + port: typing.Optional[int] = None, + bin_dir: typing.Optional[str] = None + ) -> PostgresNode: + assert base_dir is None or type(base_dir) == str # noqa: E721 + assert port is None or type(port) == int # noqa: E721 + assert bin_dir is None or type(bin_dir) == str # noqa: E721 + + assert isinstance(self._os_ops, OsOperations) + assert type(self._test_path) == str # noqa: E721 + + real_base_dir = self._os_ops.build_path(self._test_path, base_dir) + self._os_ops.rmdirs(real_base_dir, ignore_errors=True) + self._os_ops.makedirs(real_base_dir) node = PostgresNode(base_dir=real_base_dir, port=port, bin_dir=bin_dir) self.nodes_to_cleanup.append(node) @@ -43,24 +74,40 @@ def make_empty( def make_simple( self, - base_dir=None, - port=None, - set_replication=False, - ptrack_enable=False, - initdb_params=[], - pg_options={}, - checksum=True, - bin_dir=None): + base_dir: typing.Optional[str] = None, + port: typing.Optional[int] = None, + set_replication: bool = False, + ptrack_enable: bool = False, + initdb_params: T_LIST_STR = [], + pg_options: T_DICT_STR_STR = {}, + checksum: bool = True, + bin_dir: typing.Optional[str] = None + ) -> PostgresNode: + assert base_dir is None or type(base_dir) == str # noqa: E721 + assert port is None or type(port) == int # noqa: E721 + assert type(set_replication) == bool # noqa: E721 + assert type(ptrack_enable) == bool # noqa: E721 + assert type(initdb_params) == list # noqa: E721 assert type(pg_options) == dict # noqa: E721 + assert type(checksum) == bool # noqa: E721 + assert bin_dir is None or type(bin_dir) == str # noqa: E721 if checksum and '--data-checksums' not in initdb_params: initdb_params.append('--data-checksums') - node = self.make_empty(base_dir, port, bin_dir=bin_dir) + + node = self.make_empty( + base_dir, + port, + bin_dir=bin_dir + ) + node.init( - initdb_params=initdb_params, allow_streaming=set_replication) + initdb_params=initdb_params, + allow_streaming=set_replication + ) # set major version - pg_version_file = self.os_ops.read(self.os_ops.build_path(node.data_dir, 'PG_VERSION')) + pg_version_file = self._os_ops.read(self._os_ops.build_path(node.data_dir, 'PG_VERSION')) node.major_version_str = str(pg_version_file.rstrip()) node.major_version = float(node.major_version_str) @@ -115,7 +162,7 @@ def make_simple( return node @staticmethod - def _gettempdir_for_socket(): + def _gettempdir_for_socket() -> str: platform_system_name = platform.system().lower() if platform_system_name == "windows": @@ -143,7 +190,7 @@ def _gettempdir_for_socket(): return "/tmp" @staticmethod - def _gettempdir(): + def _gettempdir() -> str: v = tempfile.gettempdir() # From 2246447cffbf46cf502dd7deb01d2a9c9cf248de Mon Sep 17 00:00:00 2001 From: "d.kovalenko" Date: Fri, 27 Jun 2025 00:20:43 +0300 Subject: [PATCH 03/18] NodeApp::test_path is corrected --- testgres/node_app.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/testgres/node_app.py b/testgres/node_app.py index 4abecffb..4376164d 100644 --- a/testgres/node_app.py +++ b/testgres/node_app.py @@ -47,7 +47,7 @@ def os_ops(self) -> OsOperations: @property def test_path(self) -> str: - assert isinstance(self._test_path, OsOperations) + assert type(self._test_path) == str # noqa: E721 return self._test_path def make_empty( From 3fe3e94dc287f3d79717d60e77e395f2825646f7 Mon Sep 17 00:00:00 2001 From: "d.kovalenko" Date: Fri, 27 Jun 2025 11:49:45 +0300 Subject: [PATCH 04/18] Default value of 'pg_options' (NodeApp::make_simple) is None --- testgres/node_app.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/testgres/node_app.py b/testgres/node_app.py index 4376164d..f5f171ed 100644 --- a/testgres/node_app.py +++ b/testgres/node_app.py @@ -79,7 +79,7 @@ def make_simple( set_replication: bool = False, ptrack_enable: bool = False, initdb_params: T_LIST_STR = [], - pg_options: T_DICT_STR_STR = {}, + pg_options: typing.Optional[T_DICT_STR_STR] = None, checksum: bool = True, bin_dir: typing.Optional[str] = None ) -> PostgresNode: @@ -88,7 +88,7 @@ def make_simple( assert type(set_replication) == bool # noqa: E721 assert type(ptrack_enable) == bool # noqa: E721 assert type(initdb_params) == list # noqa: E721 - assert type(pg_options) == dict # noqa: E721 + assert pg_options is None or type(pg_options) == dict # noqa: E721 assert type(checksum) == bool # noqa: E721 assert bin_dir is None or type(bin_dir) == str # noqa: E721 @@ -143,8 +143,10 @@ def make_simple( options['wal_keep_segments'] = '12' # Apply given parameters - for option_name, option_value in pg_options.items(): - options[option_name] = option_value + if pg_options is not None: + assert type(pg_options) == dict + for option_name, option_value in pg_options.items(): + options[option_name] = option_value # Define delayed propertyes if not ("unix_socket_directories" in options.keys()): From 1062a973d763651cc7a560782b0d766be72e2d81 Mon Sep 17 00:00:00 2001 From: "d.kovalenko" Date: Fri, 27 Jun 2025 20:37:04 +0300 Subject: [PATCH 05/18] OsOperations::create_clone is added --- testgres/operations/local_ops.py | 27 +++++++++++++++++++++++- testgres/operations/os_ops.py | 11 ++++++---- testgres/operations/remote_ops.py | 35 ++++++++++++++++++++++++++++++- tests/test_os_ops_common.py | 7 +++++++ 4 files changed, 74 insertions(+), 6 deletions(-) diff --git a/testgres/operations/local_ops.py b/testgres/operations/local_ops.py index 103ee4d1..99d8e322 100644 --- a/testgres/operations/local_ops.py +++ b/testgres/operations/local_ops.py @@ -1,3 +1,5 @@ +from __future__ import annotations + import getpass import logging import os @@ -11,6 +13,7 @@ import psutil import typing import threading +import copy from ..exceptions import ExecUtilException from ..exceptions import InvalidOperationException @@ -29,13 +32,26 @@ class LocalOperations(OsOperations): + sm_dummy_conn_params = ConnectionParams() sm_single_instance: OsOperations = None sm_single_instance_guard = threading.Lock() + # TODO: make it read-only + conn_params: ConnectionParams + host: str + ssh_key: typing.Optional[str] + remote: bool + username: str + def __init__(self, conn_params=None): + super().__init__() + + if conn_params is __class__.sm_dummy_conn_params: + return + if conn_params is None: conn_params = ConnectionParams() - super(LocalOperations, self).__init__(conn_params.username) + self.conn_params = conn_params self.host = conn_params.host self.ssh_key = None @@ -58,6 +74,15 @@ def get_single_instance() -> OsOperations: assert type(__class__.sm_single_instance) == __class__ # noqa: E721 return __class__.sm_single_instance + def create_clone(self) -> LocalOperations: + clone = __class__(__class__.sm_dummy_conn_params) + clone.conn_params = copy.copy(self.conn_params) + clone.host = self.host + clone.ssh_key = self.ssh_key + clone.remote = self.remote + clone.username = self.username + return clone + @staticmethod def _process_output(encoding, temp_file_path): """Process the output of a command from a temporary file.""" diff --git a/testgres/operations/os_ops.py b/testgres/operations/os_ops.py index 7cebc8b0..46422269 100644 --- a/testgres/operations/os_ops.py +++ b/testgres/operations/os_ops.py @@ -1,4 +1,5 @@ -import getpass +from __future__ import annotations + import locale @@ -17,9 +18,11 @@ def get_default_encoding(): class OsOperations: - def __init__(self, username=None): - self.ssh_key = None - self.username = username or getpass.getuser() + def __init__(self): + pass + + def create_clone(self) -> OsOperations: + raise NotImplementedError() # Command execution def exec_command(self, cmd, **kwargs): diff --git a/testgres/operations/remote_ops.py b/testgres/operations/remote_ops.py index 8b973a1d..15d78b1a 100644 --- a/testgres/operations/remote_ops.py +++ b/testgres/operations/remote_ops.py @@ -1,3 +1,5 @@ +from __future__ import annotations + import getpass import os import posixpath @@ -7,6 +9,7 @@ import io import logging import typing +import copy from ..exceptions import ExecUtilException from ..exceptions import InvalidOperationException @@ -42,11 +45,29 @@ def cmdline(self): class RemoteOperations(OsOperations): + sm_dummy_conn_params = ConnectionParams() + + conn_params: ConnectionParams + host: str + port: int + ssh_key: str + ssh_args: list + remote: bool + username: str + ssh_dest: str + def __init__(self, conn_params: ConnectionParams): if not platform.system().lower() == "linux": raise EnvironmentError("Remote operations are supported only on Linux!") - super().__init__(conn_params.username) + if conn_params is None: + raise ValueError("Argument 'conn_params' is None.") + + super().__init__() + + if conn_params is __class__.sm_dummy_conn_params: + return + self.conn_params = conn_params self.host = conn_params.host self.port = conn_params.port @@ -63,6 +84,18 @@ def __init__(self, conn_params: ConnectionParams): def __enter__(self): return self + def create_clone(self) -> RemoteOperations: + clone = __class__(__class__.sm_dummy_conn_params) + clone.conn_params = copy.copy(self.conn_params) + clone.host = self.host + clone.port = self.port + clone.ssh_key = self.ssh_key + clone.ssh_args = copy.copy(self.ssh_args) + clone.remote = self.remote + clone.username = self.username + clone.ssh_dest = self.ssh_dest + return clone + def exec_command( self, cmd, wait_exit=False, verbose=False, expect_error=False, encoding=None, shell=True, text=False, input=None, stdin=None, stdout=None, diff --git a/tests/test_os_ops_common.py b/tests/test_os_ops_common.py index 5b135a4a..d3c85753 100644 --- a/tests/test_os_ops_common.py +++ b/tests/test_os_ops_common.py @@ -37,6 +37,13 @@ def os_ops(self, request: pytest.FixtureRequest) -> OsOperations: assert isinstance(request.param, OsOperations) return request.param + def test_create_clone(self, os_ops: OsOperations): + assert isinstance(os_ops, OsOperations) + clone = os_ops.create_clone() + assert clone is not None + assert clone is not os_ops + assert type(clone) == type(os_ops) # noqa: E721 + def test_exec_command_success(self, os_ops: OsOperations): """ Test exec_command for successful command execution. From 1512fe71b31e00a0406572d1e9fbb1f4ba600ba7 Mon Sep 17 00:00:00 2001 From: "d.kovalenko" Date: Fri, 27 Jun 2025 20:42:50 +0300 Subject: [PATCH 06/18] [#277] NodeApp::make_empty does not allow None/empty base_dir --- testgres/node_app.py | 14 ++++-- tests/test_testgres_common.py | 85 +++++++++++++++++++++++++++++++++++ 2 files changed, 95 insertions(+), 4 deletions(-) diff --git a/testgres/node_app.py b/testgres/node_app.py index f5f171ed..8b996f4c 100644 --- a/testgres/node_app.py +++ b/testgres/node_app.py @@ -52,17 +52,23 @@ def test_path(self) -> str: def make_empty( self, - base_dir: typing.Optional[str] = None, + base_dir: str, port: typing.Optional[int] = None, bin_dir: typing.Optional[str] = None ) -> PostgresNode: - assert base_dir is None or type(base_dir) == str # noqa: E721 + assert type(base_dir) == str # noqa: E721 assert port is None or type(port) == int # noqa: E721 assert bin_dir is None or type(bin_dir) == str # noqa: E721 assert isinstance(self._os_ops, OsOperations) assert type(self._test_path) == str # noqa: E721 + if base_dir is None: + raise ValueError("Argument 'base_dir' is not defined.") + + if base_dir == "": + raise ValueError("Argument 'base_dir' is empty.") + real_base_dir = self._os_ops.build_path(self._test_path, base_dir) self._os_ops.rmdirs(real_base_dir, ignore_errors=True) self._os_ops.makedirs(real_base_dir) @@ -74,7 +80,7 @@ def make_empty( def make_simple( self, - base_dir: typing.Optional[str] = None, + base_dir: str, port: typing.Optional[int] = None, set_replication: bool = False, ptrack_enable: bool = False, @@ -83,7 +89,7 @@ def make_simple( checksum: bool = True, bin_dir: typing.Optional[str] = None ) -> PostgresNode: - assert base_dir is None or type(base_dir) == str # noqa: E721 + assert type(base_dir) == str # noqa: E721 assert port is None or type(port) == int # noqa: E721 assert type(set_replication) == bool # noqa: E721 assert type(ptrack_enable) == bool # noqa: E721 diff --git a/tests/test_testgres_common.py b/tests/test_testgres_common.py index 3f4a4712..cad02003 100644 --- a/tests/test_testgres_common.py +++ b/tests/test_testgres_common.py @@ -1,3 +1,5 @@ +from __future__ import annotations + from .helpers.global_data import PostgresNodeService from .helpers.global_data import PostgresNodeServices from .helpers.global_data import OsOperations @@ -13,6 +15,7 @@ from testgres import ProcessType from testgres import NodeStatus from testgres import IsolationLevel +from testgres import NodeApp # New name prevents to collect test-functions in TestgresException and fixes # the problem with pytest warning. @@ -1584,6 +1587,88 @@ def test_node__no_port_manager(self, node_svc: PostgresNodeService): finally: node_svc.port_manager.release_port(port) + class tag_rmdirs_protector: + _os_ops: OsOperations + _cwd: str + _old_rmdirs: any + _cwd: str + + def __init__(self, os_ops: OsOperations): + self._os_ops = os_ops + self._cwd = os.path.abspath(os_ops.cwd()) + self._old_rmdirs = os_ops.rmdirs + + def __enter__(self): + assert self._os_ops.rmdirs == self._old_rmdirs + self._os_ops.rmdirs = self.proxy__rmdirs + return self + + def __exit__(self, exc_type, exc_val, exc_tb): + assert self._os_ops.rmdirs == self.proxy__rmdirs + self._os_ops.rmdirs = self._old_rmdirs + return False + + def proxy__rmdirs(self, path, ignore_errors=True): + raise Exception("Call of rmdirs is not expected!") + + def test_node_app__make_empty__base_dir_is_None(self, node_svc: PostgresNodeService): + assert type(node_svc) == PostgresNodeService # noqa: E721 + + assert isinstance(node_svc.os_ops, OsOperations) + assert node_svc.port_manager is not None + assert isinstance(node_svc.port_manager, PortManager) + + tmp_dir = node_svc.os_ops.mkdtemp() + assert tmp_dir is not None + assert type(tmp_dir) == str # noqa: E721 + logging.info("temp directory is [{}]".format(tmp_dir)) + + # ----------- + os_ops = node_svc.os_ops.create_clone() + assert os_ops is not node_svc.os_ops + + # ----------- + with __class__.tag_rmdirs_protector(os_ops): + node_app = NodeApp(test_path=tmp_dir, os_ops=os_ops) + + with pytest.raises(expected_exception=ValueError) as x: + node_app.make_empty(base_dir=None) + + assert str(x.value) == "Argument 'base_dir' is not defined." + + # ----------- + logging.info("temp directory [{}] is deleting".format(tmp_dir)) + node_svc.os_ops.rmdir(tmp_dir) + + def test_node_app__make_empty__base_dir_is_Empty(self, node_svc: PostgresNodeService): + assert type(node_svc) == PostgresNodeService # noqa: E721 + + assert isinstance(node_svc.os_ops, OsOperations) + assert node_svc.port_manager is not None + assert isinstance(node_svc.port_manager, PortManager) + + tmp_dir = node_svc.os_ops.mkdtemp() + assert tmp_dir is not None + assert type(tmp_dir) == str # noqa: E721 + logging.info("temp directory is [{}]".format(tmp_dir)) + + # ----------- + os_ops = node_svc.os_ops.create_clone() + assert os_ops is not node_svc.os_ops + + # ----------- + with __class__.tag_rmdirs_protector(os_ops): + node_app = NodeApp(test_path=tmp_dir, os_ops=os_ops) + + with pytest.raises(expected_exception=ValueError) as x: + node_app.make_empty(base_dir="") + + assert str(x.value) == "Argument 'base_dir' is empty." + + # ----------- + logging.info("temp directory [{}] is deleting".format(tmp_dir)) + node_svc.os_ops.rmdir(tmp_dir) + @staticmethod def helper__get_node( node_svc: PostgresNodeService, From b5b0d2b7de0e4b32813efdc527404e10309e4d09 Mon Sep 17 00:00:00 2001 From: "d.kovalenko" Date: Fri, 27 Jun 2025 20:46:52 +0300 Subject: [PATCH 07/18] [#267] NodeApp.make_simple does not change 'initdb_params' - method works with copy of initdb_params - Default value of initdb_params is None --- testgres/node_app.py | 73 +++++++++++++++++++++++++++++++---- tests/test_testgres_common.py | 48 +++++++++++++++++++++++ 2 files changed, 114 insertions(+), 7 deletions(-) diff --git a/testgres/node_app.py b/testgres/node_app.py index 8b996f4c..86d1e7ea 100644 --- a/testgres/node_app.py +++ b/testgres/node_app.py @@ -84,7 +84,7 @@ def make_simple( port: typing.Optional[int] = None, set_replication: bool = False, ptrack_enable: bool = False, - initdb_params: T_LIST_STR = [], + initdb_params: typing.Optional[T_LIST_STR] = None, pg_options: typing.Optional[T_DICT_STR_STR] = None, checksum: bool = True, bin_dir: typing.Optional[str] = None @@ -93,22 +93,31 @@ def make_simple( assert port is None or type(port) == int # noqa: E721 assert type(set_replication) == bool # noqa: E721 assert type(ptrack_enable) == bool # noqa: E721 - assert type(initdb_params) == list # noqa: E721 + assert initdb_params is None or type(initdb_params) == list # noqa: E721 assert pg_options is None or type(pg_options) == dict # noqa: E721 assert type(checksum) == bool # noqa: E721 assert bin_dir is None or type(bin_dir) == str # noqa: E721 - if checksum and '--data-checksums' not in initdb_params: - initdb_params.append('--data-checksums') - node = self.make_empty( base_dir, port, bin_dir=bin_dir ) + final_initdb_params = initdb_params + + if checksum: + final_initdb_params = __class__._paramlist_append_is_not_exist( + initdb_params, + final_initdb_params, + '--data-checksums' + ) + assert final_initdb_params is not initdb_params + assert final_initdb_params is not None + assert '--data-checksums' in final_initdb_params + node.init( - initdb_params=initdb_params, + initdb_params=final_initdb_params, allow_streaming=set_replication ) @@ -150,7 +159,7 @@ def make_simple( # Apply given parameters if pg_options is not None: - assert type(pg_options) == dict + assert type(pg_options) == dict # noqa: E721 for option_name, option_value in pg_options.items(): options[option_name] = option_value @@ -169,6 +178,56 @@ def make_simple( return node + @staticmethod + def _paramlist_has_param( + params: typing.Optional[T_LIST_STR], + param: str + ) -> bool: + assert type(param) == str # noqa: E721 + + if params is None: + return False + + assert type(params) == list # noqa: E721 + + if param in params: + return True + + return False + + @staticmethod + def _paramlist_append( + user_params: typing.Optional[T_LIST_STR], + updated_params: typing.Optional[T_LIST_STR], + param: str, + ) -> T_LIST_STR: + assert user_params is None or type(user_params) == list # noqa: E721 + assert updated_params is None or type(updated_params) == list # noqa: E721 + assert type(param) == str # noqa: E721 + + if updated_params is None: + if user_params is None: + return [param] + + return [*user_params, param] + + assert updated_params is not None + if updated_params is user_params: + return [*user_params, param] + + updated_params.append(param) + return updated_params + + @staticmethod + def _paramlist_append_is_not_exist( + user_params: typing.Optional[T_LIST_STR], + updated_params: typing.Optional[T_LIST_STR], + param: str, + ) -> typing.Optional[T_LIST_STR]: + if __class__._paramlist_has_param(updated_params, param): + return updated_params + return __class__._paramlist_append(user_params, updated_params, param) + @staticmethod def _gettempdir_for_socket() -> str: platform_system_name = platform.system().lower() diff --git a/tests/test_testgres_common.py b/tests/test_testgres_common.py index cad02003..d40207e5 100644 --- a/tests/test_testgres_common.py +++ b/tests/test_testgres_common.py @@ -1669,6 +1669,54 @@ def test_node_app__make_empty__base_dir_is_Empty(self, node_svc: PostgresNodeSer logging.info("temp directory [{}] is deleting".format(tmp_dir)) node_svc.os_ops.rmdir(tmp_dir) + def test_node_app__make_simple__checksum(self, node_svc: PostgresNodeService): + assert type(node_svc) == PostgresNodeService # noqa: E721 + + assert isinstance(node_svc.os_ops, OsOperations) + assert node_svc.port_manager is not None + assert isinstance(node_svc.port_manager, PortManager) + + tmp_dir = node_svc.os_ops.mkdtemp() + assert tmp_dir is not None + assert type(tmp_dir) == str # noqa: E721 + + logging.info("temp directory is [{}]".format(tmp_dir)) + node_app = NodeApp(test_path=tmp_dir, os_ops=node_svc.os_ops) + + C_NODE = "node" + + # ----------- + def LOCAL__test(checksum: bool, initdb_params: typing.Optional[list]): + initdb_params0 = initdb_params + initdb_params0_copy = initdb_params0.copy() if initdb_params0 is not None else None + + with node_app.make_simple(C_NODE, checksum=checksum, initdb_params=initdb_params): + assert initdb_params is initdb_params0 + if initdb_params0 is not None: + assert initdb_params0 == initdb_params0_copy + + assert initdb_params is initdb_params0 + if initdb_params0 is not None: + assert initdb_params0 == initdb_params0_copy + + # ----------- + LOCAL__test(checksum=False, initdb_params=None) + LOCAL__test(checksum=True, initdb_params=None) + + # ----------- + params = [] + LOCAL__test(checksum=False, initdb_params=params) + LOCAL__test(checksum=True, initdb_params=params) + + # ----------- + params = ["--no-sync"] + LOCAL__test(checksum=False, initdb_params=params) + LOCAL__test(checksum=True, initdb_params=params) + + # ----------- + logging.info("temp directory [{}] is deleting".format(tmp_dir)) + node_svc.os_ops.rmdir(tmp_dir) + @staticmethod def helper__get_node( node_svc: PostgresNodeService, From dffde2cabbc85b8a3c5530a7cc13b416fb71e23c Mon Sep 17 00:00:00 2001 From: "d.kovalenko" Date: Fri, 27 Jun 2025 21:15:02 +0300 Subject: [PATCH 08/18] test_node_app__make_empty__base_dir_is_None is corrected --- tests/test_testgres_common.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/tests/test_testgres_common.py b/tests/test_testgres_common.py index d40207e5..0c230b66 100644 --- a/tests/test_testgres_common.py +++ b/tests/test_testgres_common.py @@ -1631,10 +1631,14 @@ def test_node_app__make_empty__base_dir_is_None(self, node_svc: PostgresNodeServ with __class__.tag_rmdirs_protector(os_ops): node_app = NodeApp(test_path=tmp_dir, os_ops=os_ops) - with pytest.raises(expected_exception=ValueError) as x: + with pytest.raises(expected_exception=BaseException) as x: node_app.make_empty(base_dir=None) - assert str(x.value) == "Argument 'base_dir' is not defined." + if type(x.value) == AssertionError: # noqa: E721 + pass + else: + assert type(x.value) == ValueError # noqa: E721 + assert str(x.value) == "Argument 'base_dir' is not defined." # ----------- logging.info("temp directory [{}] is deleting".format(tmp_dir)) From d5c367203460023b6ccda1990f7eec7e6ed0f654 Mon Sep 17 00:00:00 2001 From: "d.kovalenko" Date: Fri, 27 Jun 2025 21:17:48 +0300 Subject: [PATCH 09/18] NodeApp::make_simple is corrected (bad assert) when checksum == True and initdb_params contains "--data-checksums", "assert final_initdb_params is not initdb_params" raised. --- testgres/node_app.py | 1 - tests/test_testgres_common.py | 5 +++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/testgres/node_app.py b/testgres/node_app.py index 86d1e7ea..7d6abd3a 100644 --- a/testgres/node_app.py +++ b/testgres/node_app.py @@ -112,7 +112,6 @@ def make_simple( final_initdb_params, '--data-checksums' ) - assert final_initdb_params is not initdb_params assert final_initdb_params is not None assert '--data-checksums' in final_initdb_params diff --git a/tests/test_testgres_common.py b/tests/test_testgres_common.py index 0c230b66..e8b5fee8 100644 --- a/tests/test_testgres_common.py +++ b/tests/test_testgres_common.py @@ -1717,6 +1717,11 @@ def LOCAL__test(checksum: bool, initdb_params: typing.Optional[list]): LOCAL__test(checksum=False, initdb_params=params) LOCAL__test(checksum=True, initdb_params=params) + # ----------- + params = ["--data-checksums"] + LOCAL__test(checksum=False, initdb_params=params) + LOCAL__test(checksum=True, initdb_params=params) + # ----------- logging.info("temp directory [{}] is deleting".format(tmp_dir)) node_svc.os_ops.rmdir(tmp_dir) From aae53eb2ac34559fcdda1e5a8e82f532fe6e8a65 Mon Sep 17 00:00:00 2001 From: "d.kovalenko" Date: Fri, 27 Jun 2025 22:31:17 +0300 Subject: [PATCH 10/18] test_node_app__make_empty__base_dir_is_xxx is updated --- tests/test_testgres_common.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/test_testgres_common.py b/tests/test_testgres_common.py index e8b5fee8..8939db3a 100644 --- a/tests/test_testgres_common.py +++ b/tests/test_testgres_common.py @@ -1630,6 +1630,7 @@ def test_node_app__make_empty__base_dir_is_None(self, node_svc: PostgresNodeServ # ----------- with __class__.tag_rmdirs_protector(os_ops): node_app = NodeApp(test_path=tmp_dir, os_ops=os_ops) + assert node_app.os_ops is os_ops with pytest.raises(expected_exception=BaseException) as x: node_app.make_empty(base_dir=None) @@ -1663,6 +1664,7 @@ def test_node_app__make_empty__base_dir_is_Empty(self, node_svc: PostgresNodeSer # ----------- with __class__.tag_rmdirs_protector(os_ops): node_app = NodeApp(test_path=tmp_dir, os_ops=os_ops) + assert node_app.os_ops is os_ops with pytest.raises(expected_exception=ValueError) as x: node_app.make_empty(base_dir="") From 488d5566f80ff053f8f3275b4ef894498707e956 Mon Sep 17 00:00:00 2001 From: "d.kovalenko" Date: Fri, 27 Jun 2025 22:45:42 +0300 Subject: [PATCH 11/18] [#265] NodeApp.make_simple links a new node with own os_ops and port_manager objects --- testgres/node_app.py | 28 +++++++++++++++++++++++--- tests/test_testgres_common.py | 38 +++++++++++++++++++++++++++++++++++ 2 files changed, 63 insertions(+), 3 deletions(-) diff --git a/testgres/node_app.py b/testgres/node_app.py index 7d6abd3a..13977033 100644 --- a/testgres/node_app.py +++ b/testgres/node_app.py @@ -1,6 +1,7 @@ from .node import OsOperations from .node import LocalOperations from .node import PostgresNode +from .node import PortManager import os import platform @@ -13,23 +14,27 @@ class NodeApp: - _os_ops: OsOperations _test_path: str + _os_ops: OsOperations + _port_manager: PortManager def __init__( self, test_path: typing.Optional[str] = None, nodes_to_cleanup: typing.Optional[list] = None, - os_ops: typing.Optional[OsOperations] = None + os_ops: typing.Optional[OsOperations] = None, + port_manager: typing.Optional[PortManager] = None, ): assert test_path is None or type(test_path) == str # noqa: E721 assert os_ops is None or isinstance(os_ops, OsOperations) + assert port_manager is None or isinstance(port_manager, PortManager) if os_ops is None: os_ops = LocalOperations.get_single_instance() assert isinstance(os_ops, OsOperations) self._os_ops = os_ops + self._port_manager = port_manager if test_path is None: self._test_path = os_ops.cwd() @@ -45,6 +50,11 @@ def os_ops(self) -> OsOperations: assert isinstance(self._os_ops, OsOperations) return self._os_ops + @property + def port_manager(self) -> PortManager: + assert self._port_manager is None or isinstance(self._port_manager, PortManager) + return self._port_manager + @property def test_path(self) -> str: assert type(self._test_path) == str # noqa: E721 @@ -73,7 +83,19 @@ def make_empty( self._os_ops.rmdirs(real_base_dir, ignore_errors=True) self._os_ops.makedirs(real_base_dir) - node = PostgresNode(base_dir=real_base_dir, port=port, bin_dir=bin_dir) + port_manager: PortManager = None + + if port is None: + port_manager = self._port_manager + + node = PostgresNode( + base_dir=real_base_dir, + port=port, + bin_dir=bin_dir, + os_ops=self._os_ops, + port_manager=port_manager + ) + self.nodes_to_cleanup.append(node) return node diff --git a/tests/test_testgres_common.py b/tests/test_testgres_common.py index 8939db3a..9ecd3543 100644 --- a/tests/test_testgres_common.py +++ b/tests/test_testgres_common.py @@ -1675,6 +1675,44 @@ def test_node_app__make_empty__base_dir_is_Empty(self, node_svc: PostgresNodeSer logging.info("temp directory [{}] is deleting".format(tmp_dir)) node_svc.os_ops.rmdir(tmp_dir) + def test_node_app__make_empty(self, node_svc: PostgresNodeService): + assert type(node_svc) == PostgresNodeService # noqa: E721 + + assert isinstance(node_svc.os_ops, OsOperations) + assert node_svc.port_manager is not None + assert isinstance(node_svc.port_manager, PortManager) + + tmp_dir = node_svc.os_ops.mkdtemp() + assert tmp_dir is not None + assert type(tmp_dir) == str # noqa: E721 + logging.info("temp directory is [{}]".format(tmp_dir)) + + # ----------- + node_app = NodeApp( + test_path=tmp_dir, + os_ops=node_svc.os_ops, + port_manager=node_svc.port_manager + ) + + assert node_app.os_ops is node_svc.os_ops + assert node_app.port_manager is node_svc.port_manager + + try: + node = node_app.make_simple("node") + assert node.os_ops is node_svc.os_ops + assert node.port_manager is node_svc.port_manager + + node.slow_start() + except: # noqa: E722 + node.stop() + raise + + node.cleanup(release_resources=True) + + # ----------- + logging.info("temp directory [{}] is deleting".format(tmp_dir)) + node_svc.os_ops.rmdir(tmp_dir) + def test_node_app__make_simple__checksum(self, node_svc: PostgresNodeService): assert type(node_svc) == PostgresNodeService # noqa: E721 From 688b89f2763cfb13f484662f5076ebcbd4e9c774 Mon Sep 17 00:00:00 2001 From: "d.kovalenko" Date: Fri, 27 Jun 2025 22:51:40 +0300 Subject: [PATCH 12/18] test_node_app__make_empty is corrected --- tests/test_testgres_common.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/test_testgres_common.py b/tests/test_testgres_common.py index 9ecd3543..8dd55e22 100644 --- a/tests/test_testgres_common.py +++ b/tests/test_testgres_common.py @@ -1703,9 +1703,8 @@ def test_node_app__make_empty(self, node_svc: PostgresNodeService): assert node.port_manager is node_svc.port_manager node.slow_start() - except: # noqa: E722 + finally: node.stop() - raise node.cleanup(release_resources=True) From 4f85ce53a9ff81b5d519b664974d2b0b9637d144 Mon Sep 17 00:00:00 2001 From: "d.kovalenko" Date: Fri, 27 Jun 2025 23:13:27 +0300 Subject: [PATCH 13/18] NodeApp::nodes_to_cleanup is RO-property --- testgres/node_app.py | 20 +++++++++++++++----- tests/test_testgres_common.py | 8 ++++++++ 2 files changed, 23 insertions(+), 5 deletions(-) diff --git a/testgres/node_app.py b/testgres/node_app.py index 13977033..4555ca35 100644 --- a/testgres/node_app.py +++ b/testgres/node_app.py @@ -17,6 +17,7 @@ class NodeApp: _test_path: str _os_ops: OsOperations _port_manager: PortManager + _nodes_to_cleanup: typing.List[PostgresNode] def __init__( self, @@ -43,7 +44,15 @@ def __init__( else: self._test_path = os_ops.build_path(os_ops.cwd(), test_path) - self.nodes_to_cleanup = nodes_to_cleanup if nodes_to_cleanup else [] + if nodes_to_cleanup is None: + self._nodes_to_cleanup = [] + else: + self._nodes_to_cleanup = nodes_to_cleanup + + @property + def test_path(self) -> str: + assert type(self._test_path) == str # noqa: E721 + return self._test_path @property def os_ops(self) -> OsOperations: @@ -56,9 +65,9 @@ def port_manager(self) -> PortManager: return self._port_manager @property - def test_path(self) -> str: - assert type(self._test_path) == str # noqa: E721 - return self._test_path + def nodes_to_cleanup(self) -> typing.List[PostgresNode]: + assert type(self._nodes_to_cleanup) == list # noqa: E721 + return self._nodes_to_cleanup def make_empty( self, @@ -96,7 +105,8 @@ def make_empty( port_manager=port_manager ) - self.nodes_to_cleanup.append(node) + assert type(self._nodes_to_cleanup) == list + self._nodes_to_cleanup.append(node) return node diff --git a/tests/test_testgres_common.py b/tests/test_testgres_common.py index 8dd55e22..d8f90e5d 100644 --- a/tests/test_testgres_common.py +++ b/tests/test_testgres_common.py @@ -1696,12 +1696,20 @@ def test_node_app__make_empty(self, node_svc: PostgresNodeService): assert node_app.os_ops is node_svc.os_ops assert node_app.port_manager is node_svc.port_manager + assert type(node_app.nodes_to_cleanup) == list + assert len(node_app.nodes_to_cleanup) == 0 try: node = node_app.make_simple("node") + assert node is not None + assert isinstance(node, PostgresNode) assert node.os_ops is node_svc.os_ops assert node.port_manager is node_svc.port_manager + assert type(node_app.nodes_to_cleanup) == list + assert len(node_app.nodes_to_cleanup) == 1 + assert node_app.nodes_to_cleanup[0] is node + node.slow_start() finally: node.stop() From f79e9a77bc35ac1cd85f55e59e8909784763d3b9 Mon Sep 17 00:00:00 2001 From: "d.kovalenko" Date: Fri, 27 Jun 2025 23:17:55 +0300 Subject: [PATCH 14/18] flake8 --- testgres/node_app.py | 2 +- tests/test_testgres_common.py | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/testgres/node_app.py b/testgres/node_app.py index 4555ca35..8d9108da 100644 --- a/testgres/node_app.py +++ b/testgres/node_app.py @@ -105,7 +105,7 @@ def make_empty( port_manager=port_manager ) - assert type(self._nodes_to_cleanup) == list + assert type(self._nodes_to_cleanup) == list # noqa: E721 self._nodes_to_cleanup.append(node) return node diff --git a/tests/test_testgres_common.py b/tests/test_testgres_common.py index d8f90e5d..071e89ef 100644 --- a/tests/test_testgres_common.py +++ b/tests/test_testgres_common.py @@ -1696,17 +1696,17 @@ def test_node_app__make_empty(self, node_svc: PostgresNodeService): assert node_app.os_ops is node_svc.os_ops assert node_app.port_manager is node_svc.port_manager - assert type(node_app.nodes_to_cleanup) == list + assert type(node_app.nodes_to_cleanup) == list # noqa: E721 assert len(node_app.nodes_to_cleanup) == 0 try: node = node_app.make_simple("node") assert node is not None - assert isinstance(node, PostgresNode) + assert isinstance(node, PostgresNode) assert node.os_ops is node_svc.os_ops assert node.port_manager is node_svc.port_manager - assert type(node_app.nodes_to_cleanup) == list + assert type(node_app.nodes_to_cleanup) == list # noqa: E721 assert len(node_app.nodes_to_cleanup) == 1 assert node_app.nodes_to_cleanup[0] is node From 5817f638d898fab5dae4e2a124277ae81163c479 Mon Sep 17 00:00:00 2001 From: "d.kovalenko" Date: Sat, 28 Jun 2025 11:19:13 +0300 Subject: [PATCH 15/18] test_node_app__make_empty is fixed --- tests/test_testgres_common.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/test_testgres_common.py b/tests/test_testgres_common.py index 071e89ef..9cbf2435 100644 --- a/tests/test_testgres_common.py +++ b/tests/test_testgres_common.py @@ -1699,6 +1699,7 @@ def test_node_app__make_empty(self, node_svc: PostgresNodeService): assert type(node_app.nodes_to_cleanup) == list # noqa: E721 assert len(node_app.nodes_to_cleanup) == 0 + node: PostgresNode = None try: node = node_app.make_simple("node") assert node is not None @@ -1712,7 +1713,9 @@ def test_node_app__make_empty(self, node_svc: PostgresNodeService): node.slow_start() finally: - node.stop() + if node is not None: + node.stop() + node.release_resources() node.cleanup(release_resources=True) From 0547b0b92db8efb7e5dba97479a21a99d187ed19 Mon Sep 17 00:00:00 2001 From: "d.kovalenko" Date: Sat, 28 Jun 2025 11:21:14 +0300 Subject: [PATCH 16/18] PostgresNode::release_resources(self) is added --- testgres/node.py | 43 ++++++++++++++++++++++++++----------------- 1 file changed, 26 insertions(+), 17 deletions(-) diff --git a/testgres/node.py b/testgres/node.py index eba0d896..60d9e305 100644 --- a/testgres/node.py +++ b/testgres/node.py @@ -690,9 +690,6 @@ def _try_shutdown(self, max_attempts, with_force=False): ps_output, ps_command) - def _release_resources(self): - self.free_port() - @staticmethod def _throw_bugcheck__unexpected_result_of_ps(result, cmd): assert type(result) == str # noqa: E721 @@ -1324,25 +1321,18 @@ def pg_ctl(self, params): return execute_utility2(self.os_ops, _params, self.utils_log_file) + def release_resources(self): + """ + Release resorces owned by this node. + """ + return self._release_resources() + def free_port(self): """ Reclaim port owned by this node. NOTE: this method does not release manually defined port but reset it. """ - assert type(self._should_free_port) == bool # noqa: E721 - - if not self._should_free_port: - self._port = None - else: - assert type(self._port) == int # noqa: E721 - - assert self._port_manager is not None - assert isinstance(self._port_manager, PortManager) - - port = self._port - self._should_free_port = False - self._port = None - self._port_manager.release_port(port) + return self._free_port() def cleanup(self, max_attempts=3, full=False, release_resources=False): """ @@ -2156,6 +2146,25 @@ def upgrade_from(self, old_node, options=None, expect_error=False): return self.os_ops.exec_command(upgrade_command, expect_error=expect_error) + def _release_resources(self): + self._free_port() + + def _free_port(self): + assert type(self._should_free_port) == bool # noqa: E721 + + if not self._should_free_port: + self._port = None + else: + assert type(self._port) == int # noqa: E721 + + assert self._port_manager is not None + assert isinstance(self._port_manager, PortManager) + + port = self._port + self._should_free_port = False + self._port = None + self._port_manager.release_port(port) + def _get_bin_path(self, filename): assert self._os_ops is not None assert isinstance(self._os_ops, OsOperations) From 1b12ca396f27cbda281017de7d62c45dff14ca51 Mon Sep 17 00:00:00 2001 From: "d.kovalenko" Date: Sat, 28 Jun 2025 11:26:02 +0300 Subject: [PATCH 17/18] [FIX] NodeApp::make_empty processes a failure of self._nodes_to_cleanup.append(node) --- testgres/node_app.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/testgres/node_app.py b/testgres/node_app.py index 8d9108da..6e7b7c4f 100644 --- a/testgres/node_app.py +++ b/testgres/node_app.py @@ -105,8 +105,12 @@ def make_empty( port_manager=port_manager ) - assert type(self._nodes_to_cleanup) == list # noqa: E721 - self._nodes_to_cleanup.append(node) + try: + assert type(self._nodes_to_cleanup) == list # noqa: E721 + self._nodes_to_cleanup.append(node) + except: # noqa: E722 + node.cleanup(release_resources=True) + raise return node From 1af3dfa245f2146666ea09815f13700da3642935 Mon Sep 17 00:00:00 2001 From: "d.kovalenko" Date: Sat, 28 Jun 2025 11:26:38 +0300 Subject: [PATCH 18/18] New test test_node_app__make_empty_with_explicit_port is added --- tests/test_testgres_common.py | 56 +++++++++++++++++++++++++++++++++++ 1 file changed, 56 insertions(+) diff --git a/tests/test_testgres_common.py b/tests/test_testgres_common.py index 9cbf2435..a7ddbb27 100644 --- a/tests/test_testgres_common.py +++ b/tests/test_testgres_common.py @@ -1776,6 +1776,62 @@ def LOCAL__test(checksum: bool, initdb_params: typing.Optional[list]): logging.info("temp directory [{}] is deleting".format(tmp_dir)) node_svc.os_ops.rmdir(tmp_dir) + def test_node_app__make_empty_with_explicit_port(self, node_svc: PostgresNodeService): + assert type(node_svc) == PostgresNodeService # noqa: E721 + + assert isinstance(node_svc.os_ops, OsOperations) + assert node_svc.port_manager is not None + assert isinstance(node_svc.port_manager, PortManager) + + tmp_dir = node_svc.os_ops.mkdtemp() + assert tmp_dir is not None + assert type(tmp_dir) == str # noqa: E721 + logging.info("temp directory is [{}]".format(tmp_dir)) + + # ----------- + node_app = NodeApp( + test_path=tmp_dir, + os_ops=node_svc.os_ops, + port_manager=node_svc.port_manager + ) + + assert node_app.os_ops is node_svc.os_ops + assert node_app.port_manager is node_svc.port_manager + assert type(node_app.nodes_to_cleanup) == list # noqa: E721 + assert len(node_app.nodes_to_cleanup) == 0 + + port = node_app.port_manager.reserve_port() + assert type(port) == int # noqa: E721 + + node: PostgresNode = None + try: + node = node_app.make_simple("node", port=port) + assert node is not None + assert isinstance(node, PostgresNode) + assert node.os_ops is node_svc.os_ops + assert node.port_manager is None # <--------- + assert node.port == port + assert node._should_free_port == False # noqa: E712 + + assert type(node_app.nodes_to_cleanup) == list # noqa: E721 + assert len(node_app.nodes_to_cleanup) == 1 + assert node_app.nodes_to_cleanup[0] is node + + node.slow_start() + finally: + if node is not None: + node.stop() + node.free_port() + + assert node._port is None + assert not node._should_free_port + + node.cleanup(release_resources=True) + + # ----------- + logging.info("temp directory [{}] is deleting".format(tmp_dir)) + node_svc.os_ops.rmdir(tmp_dir) + @staticmethod def helper__get_node( node_svc: PostgresNodeService,

pFad - (p)hone/(F)rame/(a)nonymizer/(d)eclutterfier! Saves Data!