From 17fb8c334aca3372801858d01dfaf4a3b0c9729d Mon Sep 17 00:00:00 2001 From: "d.kovalenko" Date: Thu, 26 Jun 2025 23:09:16 +0300 Subject: [PATCH 01/18] NodeApp was moved to own py-file (refactoring) --- testgres/__init__.py | 8 ++- testgres/node.py | 163 ----------------------------------------- testgres/node_app.py | 168 +++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 173 insertions(+), 166 deletions(-) create mode 100644 testgres/node_app.py diff --git a/testgres/__init__.py b/testgres/__init__.py index 339ae62e..555784bf 100644 --- a/testgres/__init__.py +++ b/testgres/__init__.py @@ -33,8 +33,9 @@ ProcessType, \ DumpFormat -from .node import PostgresNode, NodeApp +from .node import PostgresNode from .node import PortManager +from .node_app import NodeApp from .utils import \ reserve_port, \ @@ -62,8 +63,9 @@ "NodeConnection", "DatabaseError", "InternalError", "ProgrammingError", "OperationalError", "TestgresException", "ExecUtilException", "QueryException", "TimeoutException", "CatchUpException", "StartNodeException", "InitNodeException", "BackupException", "InvalidOperationException", "XLogMethod", "IsolationLevel", "NodeStatus", "ProcessType", "DumpFormat", - "PostgresNode", "NodeApp", - "PortManager", + NodeApp.__name__, + PostgresNode.__name__, + PortManager.__name__, "reserve_port", "release_port", "bound_ports", "get_bin_path", "get_pg_config", "get_pg_version", "First", "Any", "OsOperations", "LocalOperations", "RemoteOperations", "ConnectionParams" diff --git a/testgres/node.py b/testgres/node.py index 208846a1..eba0d896 100644 --- a/testgres/node.py +++ b/testgres/node.py @@ -7,8 +7,6 @@ import signal import subprocess import threading -import tempfile -import platform from queue import Queue import time @@ -2352,164 +2350,3 @@ def delect_port_conflict(log_reader: PostgresNodeLogReader) -> bool: return True return False - - -class NodeApp: - - def __init__(self, test_path=None, nodes_to_cleanup=None, os_ops=None): - assert os_ops is None or isinstance(os_ops, OsOperations) - - if os_ops is None: - os_ops = LocalOperations.get_single_instance() - - assert isinstance(os_ops, OsOperations) - - if test_path: - if os.path.isabs(test_path): - self.test_path = test_path - else: - self.test_path = os_ops.build_path(os_ops.cwd(), test_path) - else: - self.test_path = os_ops.cwd() - self.nodes_to_cleanup = nodes_to_cleanup if nodes_to_cleanup else [] - self.os_ops = os_ops - - def make_empty( - self, - base_dir=None, - port=None, - bin_dir=None): - real_base_dir = self.os_ops.build_path(self.test_path, base_dir) - self.os_ops.rmdirs(real_base_dir, ignore_errors=True) - self.os_ops.makedirs(real_base_dir) - - node = PostgresNode(base_dir=real_base_dir, port=port, bin_dir=bin_dir) - self.nodes_to_cleanup.append(node) - - return node - - def make_simple( - self, - base_dir=None, - port=None, - set_replication=False, - ptrack_enable=False, - initdb_params=[], - pg_options={}, - checksum=True, - bin_dir=None): - assert type(pg_options) == dict # noqa: E721 - - if checksum and '--data-checksums' not in initdb_params: - initdb_params.append('--data-checksums') - node = self.make_empty(base_dir, port, bin_dir=bin_dir) - node.init( - initdb_params=initdb_params, allow_streaming=set_replication) - - # set major version - pg_version_file = self.os_ops.read(self.os_ops.build_path(node.data_dir, 'PG_VERSION')) - node.major_version_str = str(pg_version_file.rstrip()) - node.major_version = float(node.major_version_str) - - # Set default parameters - options = { - 'max_connections': 100, - 'shared_buffers': '10MB', - 'fsync': 'off', - 'wal_level': 'logical', - 'hot_standby': 'off', - 'log_line_prefix': '%t [%p]: [%l-1] ', - 'log_statement': 'none', - 'log_duration': 'on', - 'log_min_duration_statement': 0, - 'log_connections': 'on', - 'log_disconnections': 'on', - 'restart_after_crash': 'off', - 'autovacuum': 'off', - # unix_socket_directories will be defined later - } - - # Allow replication in pg_hba.conf - if set_replication: - options['max_wal_senders'] = 10 - - if ptrack_enable: - options['ptrack.map_size'] = '1' - options['shared_preload_libraries'] = 'ptrack' - - if node.major_version >= 13: - options['wal_keep_size'] = '200MB' - else: - options['wal_keep_segments'] = '12' - - # Apply given parameters - for option_name, option_value in iteritems(pg_options): - options[option_name] = option_value - - # Define delayed propertyes - if not ("unix_socket_directories" in options.keys()): - options["unix_socket_directories"] = __class__._gettempdir_for_socket() - - # Set config values - node.set_auto_conf(options) - - # kludge for testgres - # https://github.com/postgrespro/testgres/issues/54 - # for PG >= 13 remove 'wal_keep_segments' parameter - if node.major_version >= 13: - node.set_auto_conf({}, 'postgresql.conf', ['wal_keep_segments']) - - return node - - @staticmethod - def _gettempdir_for_socket(): - platform_system_name = platform.system().lower() - - if platform_system_name == "windows": - return __class__._gettempdir() - - # - # [2025-02-17] Hot fix. - # - # Let's use hard coded path as Postgres likes. - # - # pg_config_manual.h: - # - # #ifndef WIN32 - # #define DEFAULT_PGSOCKET_DIR "/tmp" - # #else - # #define DEFAULT_PGSOCKET_DIR "" - # #endif - # - # On the altlinux-10 tempfile.gettempdir() may return - # the path to "private" temp directiry - "/temp/.private//" - # - # But Postgres want to find a socket file in "/tmp" (see above). - # - - return "/tmp" - - @staticmethod - def _gettempdir(): - v = tempfile.gettempdir() - - # - # Paranoid checks - # - if type(v) != str: # noqa: E721 - __class__._raise_bugcheck("tempfile.gettempdir returned a value with type {0}.".format(type(v).__name__)) - - if v == "": - __class__._raise_bugcheck("tempfile.gettempdir returned an empty string.") - - if not os.path.exists(v): - __class__._raise_bugcheck("tempfile.gettempdir returned a not exist path [{0}].".format(v)) - - # OK - return v - - @staticmethod - def _raise_bugcheck(msg): - assert type(msg) == str # noqa: E721 - assert msg != "" - raise Exception("[BUG CHECK] " + msg) diff --git a/testgres/node_app.py b/testgres/node_app.py new file mode 100644 index 00000000..5c9848ee --- /dev/null +++ b/testgres/node_app.py @@ -0,0 +1,168 @@ +from .node import OsOperations +from .node import LocalOperations +from .node import PostgresNode + +import os +import platform +import tempfile + + +class NodeApp: + + def __init__(self, test_path=None, nodes_to_cleanup=None, os_ops=None): + assert os_ops is None or isinstance(os_ops, OsOperations) + + if os_ops is None: + os_ops = LocalOperations.get_single_instance() + + assert isinstance(os_ops, OsOperations) + + if test_path: + if os.path.isabs(test_path): + self.test_path = test_path + else: + self.test_path = os_ops.build_path(os_ops.cwd(), test_path) + else: + self.test_path = os_ops.cwd() + self.nodes_to_cleanup = nodes_to_cleanup if nodes_to_cleanup else [] + self.os_ops = os_ops + + def make_empty( + self, + base_dir=None, + port=None, + bin_dir=None): + real_base_dir = self.os_ops.build_path(self.test_path, base_dir) + self.os_ops.rmdirs(real_base_dir, ignore_errors=True) + self.os_ops.makedirs(real_base_dir) + + node = PostgresNode(base_dir=real_base_dir, port=port, bin_dir=bin_dir) + self.nodes_to_cleanup.append(node) + + return node + + def make_simple( + self, + base_dir=None, + port=None, + set_replication=False, + ptrack_enable=False, + initdb_params=[], + pg_options={}, + checksum=True, + bin_dir=None): + assert type(pg_options) == dict # noqa: E721 + + if checksum and '--data-checksums' not in initdb_params: + initdb_params.append('--data-checksums') + node = self.make_empty(base_dir, port, bin_dir=bin_dir) + node.init( + initdb_params=initdb_params, allow_streaming=set_replication) + + # set major version + pg_version_file = self.os_ops.read(self.os_ops.build_path(node.data_dir, 'PG_VERSION')) + node.major_version_str = str(pg_version_file.rstrip()) + node.major_version = float(node.major_version_str) + + # Set default parameters + options = { + 'max_connections': 100, + 'shared_buffers': '10MB', + 'fsync': 'off', + 'wal_level': 'logical', + 'hot_standby': 'off', + 'log_line_prefix': '%t [%p]: [%l-1] ', + 'log_statement': 'none', + 'log_duration': 'on', + 'log_min_duration_statement': 0, + 'log_connections': 'on', + 'log_disconnections': 'on', + 'restart_after_crash': 'off', + 'autovacuum': 'off', + # unix_socket_directories will be defined later + } + + # Allow replication in pg_hba.conf + if set_replication: + options['max_wal_senders'] = 10 + + if ptrack_enable: + options['ptrack.map_size'] = '1' + options['shared_preload_libraries'] = 'ptrack' + + if node.major_version >= 13: + options['wal_keep_size'] = '200MB' + else: + options['wal_keep_segments'] = '12' + + # Apply given parameters + for option_name, option_value in pg_options.items(): + options[option_name] = option_value + + # Define delayed propertyes + if not ("unix_socket_directories" in options.keys()): + options["unix_socket_directories"] = __class__._gettempdir_for_socket() + + # Set config values + node.set_auto_conf(options) + + # kludge for testgres + # https://github.com/postgrespro/testgres/issues/54 + # for PG >= 13 remove 'wal_keep_segments' parameter + if node.major_version >= 13: + node.set_auto_conf({}, 'postgresql.conf', ['wal_keep_segments']) + + return node + + @staticmethod + def _gettempdir_for_socket(): + platform_system_name = platform.system().lower() + + if platform_system_name == "windows": + return __class__._gettempdir() + + # + # [2025-02-17] Hot fix. + # + # Let's use hard coded path as Postgres likes. + # + # pg_config_manual.h: + # + # #ifndef WIN32 + # #define DEFAULT_PGSOCKET_DIR "/tmp" + # #else + # #define DEFAULT_PGSOCKET_DIR "" + # #endif + # + # On the altlinux-10 tempfile.gettempdir() may return + # the path to "private" temp directiry - "/temp/.private//" + # + # But Postgres want to find a socket file in "/tmp" (see above). + # + + return "/tmp" + + @staticmethod + def _gettempdir(): + v = tempfile.gettempdir() + + # + # Paranoid checks + # + if type(v) != str: # noqa: E721 + __class__._raise_bugcheck("tempfile.gettempdir returned a value with type {0}.".format(type(v).__name__)) + + if v == "": + __class__._raise_bugcheck("tempfile.gettempdir returned an empty string.") + + if not os.path.exists(v): + __class__._raise_bugcheck("tempfile.gettempdir returned a not exist path [{0}].".format(v)) + + # OK + return v + + @staticmethod + def _raise_bugcheck(msg): + assert type(msg) == str # noqa: E721 + assert msg != "" + raise Exception("[BUG CHECK] " + msg) From 8eaa2892f5266c645b8e9e0e7b4c18959c0bf5e4 Mon Sep 17 00:00:00 2001 From: "d.kovalenko" Date: Thu, 26 Jun 2025 23:46:53 +0300 Subject: [PATCH 02/18] NodeApp is refactored totally --- testgres/node_app.py | 103 +++++++++++++++++++++++++++++++------------ 1 file changed, 75 insertions(+), 28 deletions(-) diff --git a/testgres/node_app.py b/testgres/node_app.py index 5c9848ee..4abecffb 100644 --- a/testgres/node_app.py +++ b/testgres/node_app.py @@ -5,36 +5,67 @@ import os import platform import tempfile +import typing -class NodeApp: +T_DICT_STR_STR = typing.Dict[str, str] +T_LIST_STR = typing.List[str] + - def __init__(self, test_path=None, nodes_to_cleanup=None, os_ops=None): +class NodeApp: + _os_ops: OsOperations + _test_path: str + + def __init__( + self, + test_path: typing.Optional[str] = None, + nodes_to_cleanup: typing.Optional[list] = None, + os_ops: typing.Optional[OsOperations] = None + ): + assert test_path is None or type(test_path) == str # noqa: E721 assert os_ops is None or isinstance(os_ops, OsOperations) if os_ops is None: os_ops = LocalOperations.get_single_instance() assert isinstance(os_ops, OsOperations) + self._os_ops = os_ops - if test_path: - if os.path.isabs(test_path): - self.test_path = test_path - else: - self.test_path = os_ops.build_path(os_ops.cwd(), test_path) + if test_path is None: + self._test_path = os_ops.cwd() + elif os.path.isabs(test_path): + self._test_path = test_path else: - self.test_path = os_ops.cwd() + self._test_path = os_ops.build_path(os_ops.cwd(), test_path) + self.nodes_to_cleanup = nodes_to_cleanup if nodes_to_cleanup else [] - self.os_ops = os_ops + + @property + def os_ops(self) -> OsOperations: + assert isinstance(self._os_ops, OsOperations) + return self._os_ops + + @property + def test_path(self) -> str: + assert isinstance(self._test_path, OsOperations) + return self._test_path def make_empty( self, - base_dir=None, - port=None, - bin_dir=None): - real_base_dir = self.os_ops.build_path(self.test_path, base_dir) - self.os_ops.rmdirs(real_base_dir, ignore_errors=True) - self.os_ops.makedirs(real_base_dir) + base_dir: typing.Optional[str] = None, + port: typing.Optional[int] = None, + bin_dir: typing.Optional[str] = None + ) -> PostgresNode: + assert base_dir is None or type(base_dir) == str # noqa: E721 + assert port is None or type(port) == int # noqa: E721 + assert bin_dir is None or type(bin_dir) == str # noqa: E721 + + assert isinstance(self._os_ops, OsOperations) + assert type(self._test_path) == str # noqa: E721 + + real_base_dir = self._os_ops.build_path(self._test_path, base_dir) + self._os_ops.rmdirs(real_base_dir, ignore_errors=True) + self._os_ops.makedirs(real_base_dir) node = PostgresNode(base_dir=real_base_dir, port=port, bin_dir=bin_dir) self.nodes_to_cleanup.append(node) @@ -43,24 +74,40 @@ def make_empty( def make_simple( self, - base_dir=None, - port=None, - set_replication=False, - ptrack_enable=False, - initdb_params=[], - pg_options={}, - checksum=True, - bin_dir=None): + base_dir: typing.Optional[str] = None, + port: typing.Optional[int] = None, + set_replication: bool = False, + ptrack_enable: bool = False, + initdb_params: T_LIST_STR = [], + pg_options: T_DICT_STR_STR = {}, + checksum: bool = True, + bin_dir: typing.Optional[str] = None + ) -> PostgresNode: + assert base_dir is None or type(base_dir) == str # noqa: E721 + assert port is None or type(port) == int # noqa: E721 + assert type(set_replication) == bool # noqa: E721 + assert type(ptrack_enable) == bool # noqa: E721 + assert type(initdb_params) == list # noqa: E721 assert type(pg_options) == dict # noqa: E721 + assert type(checksum) == bool # noqa: E721 + assert bin_dir is None or type(bin_dir) == str # noqa: E721 if checksum and '--data-checksums' not in initdb_params: initdb_params.append('--data-checksums') - node = self.make_empty(base_dir, port, bin_dir=bin_dir) + + node = self.make_empty( + base_dir, + port, + bin_dir=bin_dir + ) + node.init( - initdb_params=initdb_params, allow_streaming=set_replication) + initdb_params=initdb_params, + allow_streaming=set_replication + ) # set major version - pg_version_file = self.os_ops.read(self.os_ops.build_path(node.data_dir, 'PG_VERSION')) + pg_version_file = self._os_ops.read(self._os_ops.build_path(node.data_dir, 'PG_VERSION')) node.major_version_str = str(pg_version_file.rstrip()) node.major_version = float(node.major_version_str) @@ -115,7 +162,7 @@ def make_simple( return node @staticmethod - def _gettempdir_for_socket(): + def _gettempdir_for_socket() -> str: platform_system_name = platform.system().lower() if platform_system_name == "windows": @@ -143,7 +190,7 @@ def _gettempdir_for_socket(): return "/tmp" @staticmethod - def _gettempdir(): + def _gettempdir() -> str: v = tempfile.gettempdir() # From 2246447cffbf46cf502dd7deb01d2a9c9cf248de Mon Sep 17 00:00:00 2001 From: "d.kovalenko" Date: Fri, 27 Jun 2025 00:20:43 +0300 Subject: [PATCH 03/18] NodeApp::test_path is corrected --- testgres/node_app.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/testgres/node_app.py b/testgres/node_app.py index 4abecffb..4376164d 100644 --- a/testgres/node_app.py +++ b/testgres/node_app.py @@ -47,7 +47,7 @@ def os_ops(self) -> OsOperations: @property def test_path(self) -> str: - assert isinstance(self._test_path, OsOperations) + assert type(self._test_path) == str # noqa: E721 return self._test_path def make_empty( From 3fe3e94dc287f3d79717d60e77e395f2825646f7 Mon Sep 17 00:00:00 2001 From: "d.kovalenko" Date: Fri, 27 Jun 2025 11:49:45 +0300 Subject: [PATCH 04/18] Default value of 'pg_options' (NodeApp::make_simple) is None --- testgres/node_app.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/testgres/node_app.py b/testgres/node_app.py index 4376164d..f5f171ed 100644 --- a/testgres/node_app.py +++ b/testgres/node_app.py @@ -79,7 +79,7 @@ def make_simple( set_replication: bool = False, ptrack_enable: bool = False, initdb_params: T_LIST_STR = [], - pg_options: T_DICT_STR_STR = {}, + pg_options: typing.Optional[T_DICT_STR_STR] = None, checksum: bool = True, bin_dir: typing.Optional[str] = None ) -> PostgresNode: @@ -88,7 +88,7 @@ def make_simple( assert type(set_replication) == bool # noqa: E721 assert type(ptrack_enable) == bool # noqa: E721 assert type(initdb_params) == list # noqa: E721 - assert type(pg_options) == dict # noqa: E721 + assert pg_options is None or type(pg_options) == dict # noqa: E721 assert type(checksum) == bool # noqa: E721 assert bin_dir is None or type(bin_dir) == str # noqa: E721 @@ -143,8 +143,10 @@ def make_simple( options['wal_keep_segments'] = '12' # Apply given parameters - for option_name, option_value in pg_options.items(): - options[option_name] = option_value + if pg_options is not None: + assert type(pg_options) == dict + for option_name, option_value in pg_options.items(): + options[option_name] = option_value # Define delayed propertyes if not ("unix_socket_directories" in options.keys()): From 1062a973d763651cc7a560782b0d766be72e2d81 Mon Sep 17 00:00:00 2001 From: "d.kovalenko" Date: Fri, 27 Jun 2025 20:37:04 +0300 Subject: [PATCH 05/18] OsOperations::create_clone is added --- testgres/operations/local_ops.py | 27 +++++++++++++++++++++++- testgres/operations/os_ops.py | 11 ++++++---- testgres/operations/remote_ops.py | 35 ++++++++++++++++++++++++++++++- tests/test_os_ops_common.py | 7 +++++++ 4 files changed, 74 insertions(+), 6 deletions(-) diff --git a/testgres/operations/local_ops.py b/testgres/operations/local_ops.py index 103ee4d1..99d8e322 100644 --- a/testgres/operations/local_ops.py +++ b/testgres/operations/local_ops.py @@ -1,3 +1,5 @@ +from __future__ import annotations + import getpass import logging import os @@ -11,6 +13,7 @@ import psutil import typing import threading +import copy from ..exceptions import ExecUtilException from ..exceptions import InvalidOperationException @@ -29,13 +32,26 @@ class LocalOperations(OsOperations): + sm_dummy_conn_params = ConnectionParams() sm_single_instance: OsOperations = None sm_single_instance_guard = threading.Lock() + # TODO: make it read-only + conn_params: ConnectionParams + host: str + ssh_key: typing.Optional[str] + remote: bool + username: str + def __init__(self, conn_params=None): + super().__init__() + + if conn_params is __class__.sm_dummy_conn_params: + return + if conn_params is None: conn_params = ConnectionParams() - super(LocalOperations, self).__init__(conn_params.username) + self.conn_params = conn_params self.host = conn_params.host self.ssh_key = None @@ -58,6 +74,15 @@ def get_single_instance() -> OsOperations: assert type(__class__.sm_single_instance) == __class__ # noqa: E721 return __class__.sm_single_instance + def create_clone(self) -> LocalOperations: + clone = __class__(__class__.sm_dummy_conn_params) + clone.conn_params = copy.copy(self.conn_params) + clone.host = self.host + clone.ssh_key = self.ssh_key + clone.remote = self.remote + clone.username = self.username + return clone + @staticmethod def _process_output(encoding, temp_file_path): """Process the output of a command from a temporary file.""" diff --git a/testgres/operations/os_ops.py b/testgres/operations/os_ops.py index 7cebc8b0..46422269 100644 --- a/testgres/operations/os_ops.py +++ b/testgres/operations/os_ops.py @@ -1,4 +1,5 @@ -import getpass +from __future__ import annotations + import locale @@ -17,9 +18,11 @@ def get_default_encoding(): class OsOperations: - def __init__(self, username=None): - self.ssh_key = None - self.username = username or getpass.getuser() + def __init__(self): + pass + + def create_clone(self) -> OsOperations: + raise NotImplementedError() # Command execution def exec_command(self, cmd, **kwargs): diff --git a/testgres/operations/remote_ops.py b/testgres/operations/remote_ops.py index 8b973a1d..15d78b1a 100644 --- a/testgres/operations/remote_ops.py +++ b/testgres/operations/remote_ops.py @@ -1,3 +1,5 @@ +from __future__ import annotations + import getpass import os import posixpath @@ -7,6 +9,7 @@ import io import logging import typing +import copy from ..exceptions import ExecUtilException from ..exceptions import InvalidOperationException @@ -42,11 +45,29 @@ def cmdline(self): class RemoteOperations(OsOperations): + sm_dummy_conn_params = ConnectionParams() + + conn_params: ConnectionParams + host: str + port: int + ssh_key: str + ssh_args: list + remote: bool + username: str + ssh_dest: str + def __init__(self, conn_params: ConnectionParams): if not platform.system().lower() == "linux": raise EnvironmentError("Remote operations are supported only on Linux!") - super().__init__(conn_params.username) + if conn_params is None: + raise ValueError("Argument 'conn_params' is None.") + + super().__init__() + + if conn_params is __class__.sm_dummy_conn_params: + return + self.conn_params = conn_params self.host = conn_params.host self.port = conn_params.port @@ -63,6 +84,18 @@ def __init__(self, conn_params: ConnectionParams): def __enter__(self): return self + def create_clone(self) -> RemoteOperations: + clone = __class__(__class__.sm_dummy_conn_params) + clone.conn_params = copy.copy(self.conn_params) + clone.host = self.host + clone.port = self.port + clone.ssh_key = self.ssh_key + clone.ssh_args = copy.copy(self.ssh_args) + clone.remote = self.remote + clone.username = self.username + clone.ssh_dest = self.ssh_dest + return clone + def exec_command( self, cmd, wait_exit=False, verbose=False, expect_error=False, encoding=None, shell=True, text=False, input=None, stdin=None, stdout=None, diff --git a/tests/test_os_ops_common.py b/tests/test_os_ops_common.py index 5b135a4a..d3c85753 100644 --- a/tests/test_os_ops_common.py +++ b/tests/test_os_ops_common.py @@ -37,6 +37,13 @@ def os_ops(self, request: pytest.FixtureRequest) -> OsOperations: assert isinstance(request.param, OsOperations) return request.param + def test_create_clone(self, os_ops: OsOperations): + assert isinstance(os_ops, OsOperations) + clone = os_ops.create_clone() + assert clone is not None + assert clone is not os_ops + assert type(clone) == type(os_ops) # noqa: E721 + def test_exec_command_success(self, os_ops: OsOperations): """ Test exec_command for successful command execution. From 1512fe71b31e00a0406572d1e9fbb1f4ba600ba7 Mon Sep 17 00:00:00 2001 From: "d.kovalenko" Date: Fri, 27 Jun 2025 20:42:50 +0300 Subject: [PATCH 06/18] [#277] NodeApp::make_empty does not allow None/empty base_dir --- testgres/node_app.py | 14 ++++-- tests/test_testgres_common.py | 85 +++++++++++++++++++++++++++++++++++ 2 files changed, 95 insertions(+), 4 deletions(-) diff --git a/testgres/node_app.py b/testgres/node_app.py index f5f171ed..8b996f4c 100644 --- a/testgres/node_app.py +++ b/testgres/node_app.py @@ -52,17 +52,23 @@ def test_path(self) -> str: def make_empty( self, - base_dir: typing.Optional[str] = None, + base_dir: str, port: typing.Optional[int] = None, bin_dir: typing.Optional[str] = None ) -> PostgresNode: - assert base_dir is None or type(base_dir) == str # noqa: E721 + assert type(base_dir) == str # noqa: E721 assert port is None or type(port) == int # noqa: E721 assert bin_dir is None or type(bin_dir) == str # noqa: E721 assert isinstance(self._os_ops, OsOperations) assert type(self._test_path) == str # noqa: E721 + if base_dir is None: + raise ValueError("Argument 'base_dir' is not defined.") + + if base_dir == "": + raise ValueError("Argument 'base_dir' is empty.") + real_base_dir = self._os_ops.build_path(self._test_path, base_dir) self._os_ops.rmdirs(real_base_dir, ignore_errors=True) self._os_ops.makedirs(real_base_dir) @@ -74,7 +80,7 @@ def make_empty( def make_simple( self, - base_dir: typing.Optional[str] = None, + base_dir: str, port: typing.Optional[int] = None, set_replication: bool = False, ptrack_enable: bool = False, @@ -83,7 +89,7 @@ def make_simple( checksum: bool = True, bin_dir: typing.Optional[str] = None ) -> PostgresNode: - assert base_dir is None or type(base_dir) == str # noqa: E721 + assert type(base_dir) == str # noqa: E721 assert port is None or type(port) == int # noqa: E721 assert type(set_replication) == bool # noqa: E721 assert type(ptrack_enable) == bool # noqa: E721 diff --git a/tests/test_testgres_common.py b/tests/test_testgres_common.py index 3f4a4712..cad02003 100644 --- a/tests/test_testgres_common.py +++ b/tests/test_testgres_common.py @@ -1,3 +1,5 @@ +from __future__ import annotations + from .helpers.global_data import PostgresNodeService from .helpers.global_data import PostgresNodeServices from .helpers.global_data import OsOperations @@ -13,6 +15,7 @@ from testgres import ProcessType from testgres import NodeStatus from testgres import IsolationLevel +from testgres import NodeApp # New name prevents to collect test-functions in TestgresException and fixes # the problem with pytest warning. @@ -1584,6 +1587,88 @@ def test_node__no_port_manager(self, node_svc: PostgresNodeService): finally: node_svc.port_manager.release_port(port) + class tag_rmdirs_protector: + _os_ops: OsOperations + _cwd: str + _old_rmdirs: any + _cwd: str + + def __init__(self, os_ops: OsOperations): + self._os_ops = os_ops + self._cwd = os.path.abspath(os_ops.cwd()) + self._old_rmdirs = os_ops.rmdirs + + def __enter__(self): + assert self._os_ops.rmdirs == self._old_rmdirs + self._os_ops.rmdirs = self.proxy__rmdirs + return self + + def __exit__(self, exc_type, exc_val, exc_tb): + assert self._os_ops.rmdirs == self.proxy__rmdirs + self._os_ops.rmdirs = self._old_rmdirs + return False + + def proxy__rmdirs(self, path, ignore_errors=True): + raise Exception("Call of rmdirs is not expected!") + + def test_node_app__make_empty__base_dir_is_None(self, node_svc: PostgresNodeService): + assert type(node_svc) == PostgresNodeService # noqa: E721 + + assert isinstance(node_svc.os_ops, OsOperations) + assert node_svc.port_manager is not None + assert isinstance(node_svc.port_manager, PortManager) + + tmp_dir = node_svc.os_ops.mkdtemp() + assert tmp_dir is not None + assert type(tmp_dir) == str # noqa: E721 + logging.info("temp directory is [{}]".format(tmp_dir)) + + # ----------- + os_ops = node_svc.os_ops.create_clone() + assert os_ops is not node_svc.os_ops + + # ----------- + with __class__.tag_rmdirs_protector(os_ops): + node_app = NodeApp(test_path=tmp_dir, os_ops=os_ops) + + with pytest.raises(expected_exception=ValueError) as x: + node_app.make_empty(base_dir=None) + + assert str(x.value) == "Argument 'base_dir' is not defined." + + # ----------- + logging.info("temp directory [{}] is deleting".format(tmp_dir)) + node_svc.os_ops.rmdir(tmp_dir) + + def test_node_app__make_empty__base_dir_is_Empty(self, node_svc: PostgresNodeService): + assert type(node_svc) == PostgresNodeService # noqa: E721 + + assert isinstance(node_svc.os_ops, OsOperations) + assert node_svc.port_manager is not None + assert isinstance(node_svc.port_manager, PortManager) + + tmp_dir = node_svc.os_ops.mkdtemp() + assert tmp_dir is not None + assert type(tmp_dir) == str # noqa: E721 + logging.info("temp directory is [{}]".format(tmp_dir)) + + # ----------- + os_ops = node_svc.os_ops.create_clone() + assert os_ops is not node_svc.os_ops + + # ----------- + with __class__.tag_rmdirs_protector(os_ops): + node_app = NodeApp(test_path=tmp_dir, os_ops=os_ops) + + with pytest.raises(expected_exception=ValueError) as x: + node_app.make_empty(base_dir="") + + assert str(x.value) == "Argument 'base_dir' is empty." + + # ----------- + logging.info("temp directory [{}] is deleting".format(tmp_dir)) + node_svc.os_ops.rmdir(tmp_dir) + @staticmethod def helper__get_node( node_svc: PostgresNodeService, From b5b0d2b7de0e4b32813efdc527404e10309e4d09 Mon Sep 17 00:00:00 2001 From: "d.kovalenko" Date: Fri, 27 Jun 2025 20:46:52 +0300 Subject: [PATCH 07/18] [#267] NodeApp.make_simple does not change 'initdb_params' - method works with copy of initdb_params - Default value of initdb_params is None --- testgres/node_app.py | 73 +++++++++++++++++++++++++++++++---- tests/test_testgres_common.py | 48 +++++++++++++++++++++++ 2 files changed, 114 insertions(+), 7 deletions(-) diff --git a/testgres/node_app.py b/testgres/node_app.py index 8b996f4c..86d1e7ea 100644 --- a/testgres/node_app.py +++ b/testgres/node_app.py @@ -84,7 +84,7 @@ def make_simple( port: typing.Optional[int] = None, set_replication: bool = False, ptrack_enable: bool = False, - initdb_params: T_LIST_STR = [], + initdb_params: typing.Optional[T_LIST_STR] = None, pg_options: typing.Optional[T_DICT_STR_STR] = None, checksum: bool = True, bin_dir: typing.Optional[str] = None @@ -93,22 +93,31 @@ def make_simple( assert port is None or type(port) == int # noqa: E721 assert type(set_replication) == bool # noqa: E721 assert type(ptrack_enable) == bool # noqa: E721 - assert type(initdb_params) == list # noqa: E721 + assert initdb_params is None or type(initdb_params) == list # noqa: E721 assert pg_options is None or type(pg_options) == dict # noqa: E721 assert type(checksum) == bool # noqa: E721 assert bin_dir is None or type(bin_dir) == str # noqa: E721 - if checksum and '--data-checksums' not in initdb_params: - initdb_params.append('--data-checksums') - node = self.make_empty( base_dir, port, bin_dir=bin_dir ) + final_initdb_params = initdb_params + + if checksum: + final_initdb_params = __class__._paramlist_append_is_not_exist( + initdb_params, + final_initdb_params, + '--data-checksums' + ) + assert final_initdb_params is not initdb_params + assert final_initdb_params is not None + assert '--data-checksums' in final_initdb_params + node.init( - initdb_params=initdb_params, + initdb_params=final_initdb_params, allow_streaming=set_replication ) @@ -150,7 +159,7 @@ def make_simple( # Apply given parameters if pg_options is not None: - assert type(pg_options) == dict + assert type(pg_options) == dict # noqa: E721 for option_name, option_value in pg_options.items(): options[option_name] = option_value @@ -169,6 +178,56 @@ def make_simple( return node + @staticmethod + def _paramlist_has_param( + params: typing.Optional[T_LIST_STR], + param: str + ) -> bool: + assert type(param) == str # noqa: E721 + + if params is None: + return False + + assert type(params) == list # noqa: E721 + + if param in params: + return True + + return False + + @staticmethod + def _paramlist_append( + user_params: typing.Optional[T_LIST_STR], + updated_params: typing.Optional[T_LIST_STR], + param: str, + ) -> T_LIST_STR: + assert user_params is None or type(user_params) == list # noqa: E721 + assert updated_params is None or type(updated_params) == list # noqa: E721 + assert type(param) == str # noqa: E721 + + if updated_params is None: + if user_params is None: + return [param] + + return [*user_params, param] + + assert updated_params is not None + if updated_params is user_params: + return [*user_params, param] + + updated_params.append(param) + return updated_params + + @staticmethod + def _paramlist_append_is_not_exist( + user_params: typing.Optional[T_LIST_STR], + updated_params: typing.Optional[T_LIST_STR], + param: str, + ) -> typing.Optional[T_LIST_STR]: + if __class__._paramlist_has_param(updated_params, param): + return updated_params + return __class__._paramlist_append(user_params, updated_params, param) + @staticmethod def _gettempdir_for_socket() -> str: platform_system_name = platform.system().lower() diff --git a/tests/test_testgres_common.py b/tests/test_testgres_common.py index cad02003..d40207e5 100644 --- a/tests/test_testgres_common.py +++ b/tests/test_testgres_common.py @@ -1669,6 +1669,54 @@ def test_node_app__make_empty__base_dir_is_Empty(self, node_svc: PostgresNodeSer logging.info("temp directory [{}] is deleting".format(tmp_dir)) node_svc.os_ops.rmdir(tmp_dir) + def test_node_app__make_simple__checksum(self, node_svc: PostgresNodeService): + assert type(node_svc) == PostgresNodeService # noqa: E721 + + assert isinstance(node_svc.os_ops, OsOperations) + assert node_svc.port_manager is not None + assert isinstance(node_svc.port_manager, PortManager) + + tmp_dir = node_svc.os_ops.mkdtemp() + assert tmp_dir is not None + assert type(tmp_dir) == str # noqa: E721 + + logging.info("temp directory is [{}]".format(tmp_dir)) + node_app = NodeApp(test_path=tmp_dir, os_ops=node_svc.os_ops) + + C_NODE = "node" + + # ----------- + def LOCAL__test(checksum: bool, initdb_params: typing.Optional[list]): + initdb_params0 = initdb_params + initdb_params0_copy = initdb_params0.copy() if initdb_params0 is not None else None + + with node_app.make_simple(C_NODE, checksum=checksum, initdb_params=initdb_params): + assert initdb_params is initdb_params0 + if initdb_params0 is not None: + assert initdb_params0 == initdb_params0_copy + + assert initdb_params is initdb_params0 + if initdb_params0 is not None: + assert initdb_params0 == initdb_params0_copy + + # ----------- + LOCAL__test(checksum=False, initdb_params=None) + LOCAL__test(checksum=True, initdb_params=None) + + # ----------- + params = [] + LOCAL__test(checksum=False, initdb_params=params) + LOCAL__test(checksum=True, initdb_params=params) + + # ----------- + params = ["--no-sync"] + LOCAL__test(checksum=False, initdb_params=params) + LOCAL__test(checksum=True, initdb_params=params) + + # ----------- + logging.info("temp directory [{}] is deleting".format(tmp_dir)) + node_svc.os_ops.rmdir(tmp_dir) + @staticmethod def helper__get_node( node_svc: PostgresNodeService, From dffde2cabbc85b8a3c5530a7cc13b416fb71e23c Mon Sep 17 00:00:00 2001 From: "d.kovalenko" Date: Fri, 27 Jun 2025 21:15:02 +0300 Subject: [PATCH 08/18] test_node_app__make_empty__base_dir_is_None is corrected --- tests/test_testgres_common.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/tests/test_testgres_common.py b/tests/test_testgres_common.py index d40207e5..0c230b66 100644 --- a/tests/test_testgres_common.py +++ b/tests/test_testgres_common.py @@ -1631,10 +1631,14 @@ def test_node_app__make_empty__base_dir_is_None(self, node_svc: PostgresNodeServ with __class__.tag_rmdirs_protector(os_ops): node_app = NodeApp(test_path=tmp_dir, os_ops=os_ops) - with pytest.raises(expected_exception=ValueError) as x: + with pytest.raises(expected_exception=BaseException) as x: node_app.make_empty(base_dir=None) - assert str(x.value) == "Argument 'base_dir' is not defined." + if type(x.value) == AssertionError: # noqa: E721 + pass + else: + assert type(x.value) == ValueError # noqa: E721 + assert str(x.value) == "Argument 'base_dir' is not defined." # ----------- logging.info("temp directory [{}] is deleting".format(tmp_dir)) From d5c367203460023b6ccda1990f7eec7e6ed0f654 Mon Sep 17 00:00:00 2001 From: "d.kovalenko" Date: Fri, 27 Jun 2025 21:17:48 +0300 Subject: [PATCH 09/18] NodeApp::make_simple is corrected (bad assert) when checksum == True and initdb_params contains "--data-checksums", "assert final_initdb_params is not initdb_params" raised. --- testgres/node_app.py | 1 - tests/test_testgres_common.py | 5 +++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/testgres/node_app.py b/testgres/node_app.py index 86d1e7ea..7d6abd3a 100644 --- a/testgres/node_app.py +++ b/testgres/node_app.py @@ -112,7 +112,6 @@ def make_simple( final_initdb_params, '--data-checksums' ) - assert final_initdb_params is not initdb_params assert final_initdb_params is not None assert '--data-checksums' in final_initdb_params diff --git a/tests/test_testgres_common.py b/tests/test_testgres_common.py index 0c230b66..e8b5fee8 100644 --- a/tests/test_testgres_common.py +++ b/tests/test_testgres_common.py @@ -1717,6 +1717,11 @@ def LOCAL__test(checksum: bool, initdb_params: typing.Optional[list]): LOCAL__test(checksum=False, initdb_params=params) LOCAL__test(checksum=True, initdb_params=params) + # ----------- + params = ["--data-checksums"] + LOCAL__test(checksum=False, initdb_params=params) + LOCAL__test(checksum=True, initdb_params=params) + # ----------- logging.info("temp directory [{}] is deleting".format(tmp_dir)) node_svc.os_ops.rmdir(tmp_dir) From aae53eb2ac34559fcdda1e5a8e82f532fe6e8a65 Mon Sep 17 00:00:00 2001 From: "d.kovalenko" Date: Fri, 27 Jun 2025 22:31:17 +0300 Subject: [PATCH 10/18] test_node_app__make_empty__base_dir_is_xxx is updated --- tests/test_testgres_common.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/test_testgres_common.py b/tests/test_testgres_common.py index e8b5fee8..8939db3a 100644 --- a/tests/test_testgres_common.py +++ b/tests/test_testgres_common.py @@ -1630,6 +1630,7 @@ def test_node_app__make_empty__base_dir_is_None(self, node_svc: PostgresNodeServ # ----------- with __class__.tag_rmdirs_protector(os_ops): node_app = NodeApp(test_path=tmp_dir, os_ops=os_ops) + assert node_app.os_ops is os_ops with pytest.raises(expected_exception=BaseException) as x: node_app.make_empty(base_dir=None) @@ -1663,6 +1664,7 @@ def test_node_app__make_empty__base_dir_is_Empty(self, node_svc: PostgresNodeSer # ----------- with __class__.tag_rmdirs_protector(os_ops): node_app = NodeApp(test_path=tmp_dir, os_ops=os_ops) + assert node_app.os_ops is os_ops with pytest.raises(expected_exception=ValueError) as x: node_app.make_empty(base_dir="") From 488d5566f80ff053f8f3275b4ef894498707e956 Mon Sep 17 00:00:00 2001 From: "d.kovalenko" Date: Fri, 27 Jun 2025 22:45:42 +0300 Subject: [PATCH 11/18] [#265] NodeApp.make_simple links a new node with own os_ops and port_manager objects --- testgres/node_app.py | 28 +++++++++++++++++++++++--- tests/test_testgres_common.py | 38 +++++++++++++++++++++++++++++++++++ 2 files changed, 63 insertions(+), 3 deletions(-) diff --git a/testgres/node_app.py b/testgres/node_app.py index 7d6abd3a..13977033 100644 --- a/testgres/node_app.py +++ b/testgres/node_app.py @@ -1,6 +1,7 @@ from .node import OsOperations from .node import LocalOperations from .node import PostgresNode +from .node import PortManager import os import platform @@ -13,23 +14,27 @@ class NodeApp: - _os_ops: OsOperations _test_path: str + _os_ops: OsOperations + _port_manager: PortManager def __init__( self, test_path: typing.Optional[str] = None, nodes_to_cleanup: typing.Optional[list] = None, - os_ops: typing.Optional[OsOperations] = None + os_ops: typing.Optional[OsOperations] = None, + port_manager: typing.Optional[PortManager] = None, ): assert test_path is None or type(test_path) == str # noqa: E721 assert os_ops is None or isinstance(os_ops, OsOperations) + assert port_manager is None or isinstance(port_manager, PortManager) if os_ops is None: os_ops = LocalOperations.get_single_instance() assert isinstance(os_ops, OsOperations) self._os_ops = os_ops + self._port_manager = port_manager if test_path is None: self._test_path = os_ops.cwd() @@ -45,6 +50,11 @@ def os_ops(self) -> OsOperations: assert isinstance(self._os_ops, OsOperations) return self._os_ops + @property + def port_manager(self) -> PortManager: + assert self._port_manager is None or isinstance(self._port_manager, PortManager) + return self._port_manager + @property def test_path(self) -> str: assert type(self._test_path) == str # noqa: E721 @@ -73,7 +83,19 @@ def make_empty( self._os_ops.rmdirs(real_base_dir, ignore_errors=True) self._os_ops.makedirs(real_base_dir) - node = PostgresNode(base_dir=real_base_dir, port=port, bin_dir=bin_dir) + port_manager: PortManager = None + + if port is None: + port_manager = self._port_manager + + node = PostgresNode( + base_dir=real_base_dir, + port=port, + bin_dir=bin_dir, + os_ops=self._os_ops, + port_manager=port_manager + ) + self.nodes_to_cleanup.append(node) return node diff --git a/tests/test_testgres_common.py b/tests/test_testgres_common.py index 8939db3a..9ecd3543 100644 --- a/tests/test_testgres_common.py +++ b/tests/test_testgres_common.py @@ -1675,6 +1675,44 @@ def test_node_app__make_empty__base_dir_is_Empty(self, node_svc: PostgresNodeSer logging.info("temp directory [{}] is deleting".format(tmp_dir)) node_svc.os_ops.rmdir(tmp_dir) + def test_node_app__make_empty(self, node_svc: PostgresNodeService): + assert type(node_svc) == PostgresNodeService # noqa: E721 + + assert isinstance(node_svc.os_ops, OsOperations) + assert node_svc.port_manager is not None + assert isinstance(node_svc.port_manager, PortManager) + + tmp_dir = node_svc.os_ops.mkdtemp() + assert tmp_dir is not None + assert type(tmp_dir) == str # noqa: E721 + logging.info("temp directory is [{}]".format(tmp_dir)) + + # ----------- + node_app = NodeApp( + test_path=tmp_dir, + os_ops=node_svc.os_ops, + port_manager=node_svc.port_manager + ) + + assert node_app.os_ops is node_svc.os_ops + assert node_app.port_manager is node_svc.port_manager + + try: + node = node_app.make_simple("node") + assert node.os_ops is node_svc.os_ops + assert node.port_manager is node_svc.port_manager + + node.slow_start() + except: # noqa: E722 + node.stop() + raise + + node.cleanup(release_resources=True) + + # ----------- + logging.info("temp directory [{}] is deleting".format(tmp_dir)) + node_svc.os_ops.rmdir(tmp_dir) + def test_node_app__make_simple__checksum(self, node_svc: PostgresNodeService): assert type(node_svc) == PostgresNodeService # noqa: E721 From 688b89f2763cfb13f484662f5076ebcbd4e9c774 Mon Sep 17 00:00:00 2001 From: "d.kovalenko" Date: Fri, 27 Jun 2025 22:51:40 +0300 Subject: [PATCH 12/18] test_node_app__make_empty is corrected --- tests/test_testgres_common.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/test_testgres_common.py b/tests/test_testgres_common.py index 9ecd3543..8dd55e22 100644 --- a/tests/test_testgres_common.py +++ b/tests/test_testgres_common.py @@ -1703,9 +1703,8 @@ def test_node_app__make_empty(self, node_svc: PostgresNodeService): assert node.port_manager is node_svc.port_manager node.slow_start() - except: # noqa: E722 + finally: node.stop() - raise node.cleanup(release_resources=True) From 4f85ce53a9ff81b5d519b664974d2b0b9637d144 Mon Sep 17 00:00:00 2001 From: "d.kovalenko" Date: Fri, 27 Jun 2025 23:13:27 +0300 Subject: [PATCH 13/18] NodeApp::nodes_to_cleanup is RO-property --- testgres/node_app.py | 20 +++++++++++++++----- tests/test_testgres_common.py | 8 ++++++++ 2 files changed, 23 insertions(+), 5 deletions(-) diff --git a/testgres/node_app.py b/testgres/node_app.py index 13977033..4555ca35 100644 --- a/testgres/node_app.py +++ b/testgres/node_app.py @@ -17,6 +17,7 @@ class NodeApp: _test_path: str _os_ops: OsOperations _port_manager: PortManager + _nodes_to_cleanup: typing.List[PostgresNode] def __init__( self, @@ -43,7 +44,15 @@ def __init__( else: self._test_path = os_ops.build_path(os_ops.cwd(), test_path) - self.nodes_to_cleanup = nodes_to_cleanup if nodes_to_cleanup else [] + if nodes_to_cleanup is None: + self._nodes_to_cleanup = [] + else: + self._nodes_to_cleanup = nodes_to_cleanup + + @property + def test_path(self) -> str: + assert type(self._test_path) == str # noqa: E721 + return self._test_path @property def os_ops(self) -> OsOperations: @@ -56,9 +65,9 @@ def port_manager(self) -> PortManager: return self._port_manager @property - def test_path(self) -> str: - assert type(self._test_path) == str # noqa: E721 - return self._test_path + def nodes_to_cleanup(self) -> typing.List[PostgresNode]: + assert type(self._nodes_to_cleanup) == list # noqa: E721 + return self._nodes_to_cleanup def make_empty( self, @@ -96,7 +105,8 @@ def make_empty( port_manager=port_manager ) - self.nodes_to_cleanup.append(node) + assert type(self._nodes_to_cleanup) == list + self._nodes_to_cleanup.append(node) return node diff --git a/tests/test_testgres_common.py b/tests/test_testgres_common.py index 8dd55e22..d8f90e5d 100644 --- a/tests/test_testgres_common.py +++ b/tests/test_testgres_common.py @@ -1696,12 +1696,20 @@ def test_node_app__make_empty(self, node_svc: PostgresNodeService): assert node_app.os_ops is node_svc.os_ops assert node_app.port_manager is node_svc.port_manager + assert type(node_app.nodes_to_cleanup) == list + assert len(node_app.nodes_to_cleanup) == 0 try: node = node_app.make_simple("node") + assert node is not None + assert isinstance(node, PostgresNode) assert node.os_ops is node_svc.os_ops assert node.port_manager is node_svc.port_manager + assert type(node_app.nodes_to_cleanup) == list + assert len(node_app.nodes_to_cleanup) == 1 + assert node_app.nodes_to_cleanup[0] is node + node.slow_start() finally: node.stop() From f79e9a77bc35ac1cd85f55e59e8909784763d3b9 Mon Sep 17 00:00:00 2001 From: "d.kovalenko" Date: Fri, 27 Jun 2025 23:17:55 +0300 Subject: [PATCH 14/18] flake8 --- testgres/node_app.py | 2 +- tests/test_testgres_common.py | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/testgres/node_app.py b/testgres/node_app.py index 4555ca35..8d9108da 100644 --- a/testgres/node_app.py +++ b/testgres/node_app.py @@ -105,7 +105,7 @@ def make_empty( port_manager=port_manager ) - assert type(self._nodes_to_cleanup) == list + assert type(self._nodes_to_cleanup) == list # noqa: E721 self._nodes_to_cleanup.append(node) return node diff --git a/tests/test_testgres_common.py b/tests/test_testgres_common.py index d8f90e5d..071e89ef 100644 --- a/tests/test_testgres_common.py +++ b/tests/test_testgres_common.py @@ -1696,17 +1696,17 @@ def test_node_app__make_empty(self, node_svc: PostgresNodeService): assert node_app.os_ops is node_svc.os_ops assert node_app.port_manager is node_svc.port_manager - assert type(node_app.nodes_to_cleanup) == list + assert type(node_app.nodes_to_cleanup) == list # noqa: E721 assert len(node_app.nodes_to_cleanup) == 0 try: node = node_app.make_simple("node") assert node is not None - assert isinstance(node, PostgresNode) + assert isinstance(node, PostgresNode) assert node.os_ops is node_svc.os_ops assert node.port_manager is node_svc.port_manager - assert type(node_app.nodes_to_cleanup) == list + assert type(node_app.nodes_to_cleanup) == list # noqa: E721 assert len(node_app.nodes_to_cleanup) == 1 assert node_app.nodes_to_cleanup[0] is node From 5817f638d898fab5dae4e2a124277ae81163c479 Mon Sep 17 00:00:00 2001 From: "d.kovalenko" Date: Sat, 28 Jun 2025 11:19:13 +0300 Subject: [PATCH 15/18] test_node_app__make_empty is fixed --- tests/test_testgres_common.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/test_testgres_common.py b/tests/test_testgres_common.py index 071e89ef..9cbf2435 100644 --- a/tests/test_testgres_common.py +++ b/tests/test_testgres_common.py @@ -1699,6 +1699,7 @@ def test_node_app__make_empty(self, node_svc: PostgresNodeService): assert type(node_app.nodes_to_cleanup) == list # noqa: E721 assert len(node_app.nodes_to_cleanup) == 0 + node: PostgresNode = None try: node = node_app.make_simple("node") assert node is not None @@ -1712,7 +1713,9 @@ def test_node_app__make_empty(self, node_svc: PostgresNodeService): node.slow_start() finally: - node.stop() + if node is not None: + node.stop() + node.release_resources() node.cleanup(release_resources=True) From 0547b0b92db8efb7e5dba97479a21a99d187ed19 Mon Sep 17 00:00:00 2001 From: "d.kovalenko" Date: Sat, 28 Jun 2025 11:21:14 +0300 Subject: [PATCH 16/18] PostgresNode::release_resources(self) is added --- testgres/node.py | 43 ++++++++++++++++++++++++++----------------- 1 file changed, 26 insertions(+), 17 deletions(-) diff --git a/testgres/node.py b/testgres/node.py index eba0d896..60d9e305 100644 --- a/testgres/node.py +++ b/testgres/node.py @@ -690,9 +690,6 @@ def _try_shutdown(self, max_attempts, with_force=False): ps_output, ps_command) - def _release_resources(self): - self.free_port() - @staticmethod def _throw_bugcheck__unexpected_result_of_ps(result, cmd): assert type(result) == str # noqa: E721 @@ -1324,25 +1321,18 @@ def pg_ctl(self, params): return execute_utility2(self.os_ops, _params, self.utils_log_file) + def release_resources(self): + """ + Release resorces owned by this node. + """ + return self._release_resources() + def free_port(self): """ Reclaim port owned by this node. NOTE: this method does not release manually defined port but reset it. """ - assert type(self._should_free_port) == bool # noqa: E721 - - if not self._should_free_port: - self._port = None - else: - assert type(self._port) == int # noqa: E721 - - assert self._port_manager is not None - assert isinstance(self._port_manager, PortManager) - - port = self._port - self._should_free_port = False - self._port = None - self._port_manager.release_port(port) + return self._free_port() def cleanup(self, max_attempts=3, full=False, release_resources=False): """ @@ -2156,6 +2146,25 @@ def upgrade_from(self, old_node, options=None, expect_error=False): return self.os_ops.exec_command(upgrade_command, expect_error=expect_error) + def _release_resources(self): + self._free_port() + + def _free_port(self): + assert type(self._should_free_port) == bool # noqa: E721 + + if not self._should_free_port: + self._port = None + else: + assert type(self._port) == int # noqa: E721 + + assert self._port_manager is not None + assert isinstance(self._port_manager, PortManager) + + port = self._port + self._should_free_port = False + self._port = None + self._port_manager.release_port(port) + def _get_bin_path(self, filename): assert self._os_ops is not None assert isinstance(self._os_ops, OsOperations) From 1b12ca396f27cbda281017de7d62c45dff14ca51 Mon Sep 17 00:00:00 2001 From: "d.kovalenko" Date: Sat, 28 Jun 2025 11:26:02 +0300 Subject: [PATCH 17/18] [FIX] NodeApp::make_empty processes a failure of self._nodes_to_cleanup.append(node) --- testgres/node_app.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/testgres/node_app.py b/testgres/node_app.py index 8d9108da..6e7b7c4f 100644 --- a/testgres/node_app.py +++ b/testgres/node_app.py @@ -105,8 +105,12 @@ def make_empty( port_manager=port_manager ) - assert type(self._nodes_to_cleanup) == list # noqa: E721 - self._nodes_to_cleanup.append(node) + try: + assert type(self._nodes_to_cleanup) == list # noqa: E721 + self._nodes_to_cleanup.append(node) + except: # noqa: E722 + node.cleanup(release_resources=True) + raise return node From 1af3dfa245f2146666ea09815f13700da3642935 Mon Sep 17 00:00:00 2001 From: "d.kovalenko" Date: Sat, 28 Jun 2025 11:26:38 +0300 Subject: [PATCH 18/18] New test test_node_app__make_empty_with_explicit_port is added --- tests/test_testgres_common.py | 56 +++++++++++++++++++++++++++++++++++ 1 file changed, 56 insertions(+) diff --git a/tests/test_testgres_common.py b/tests/test_testgres_common.py index 9cbf2435..a7ddbb27 100644 --- a/tests/test_testgres_common.py +++ b/tests/test_testgres_common.py @@ -1776,6 +1776,62 @@ def LOCAL__test(checksum: bool, initdb_params: typing.Optional[list]): logging.info("temp directory [{}] is deleting".format(tmp_dir)) node_svc.os_ops.rmdir(tmp_dir) + def test_node_app__make_empty_with_explicit_port(self, node_svc: PostgresNodeService): + assert type(node_svc) == PostgresNodeService # noqa: E721 + + assert isinstance(node_svc.os_ops, OsOperations) + assert node_svc.port_manager is not None + assert isinstance(node_svc.port_manager, PortManager) + + tmp_dir = node_svc.os_ops.mkdtemp() + assert tmp_dir is not None + assert type(tmp_dir) == str # noqa: E721 + logging.info("temp directory is [{}]".format(tmp_dir)) + + # ----------- + node_app = NodeApp( + test_path=tmp_dir, + os_ops=node_svc.os_ops, + port_manager=node_svc.port_manager + ) + + assert node_app.os_ops is node_svc.os_ops + assert node_app.port_manager is node_svc.port_manager + assert type(node_app.nodes_to_cleanup) == list # noqa: E721 + assert len(node_app.nodes_to_cleanup) == 0 + + port = node_app.port_manager.reserve_port() + assert type(port) == int # noqa: E721 + + node: PostgresNode = None + try: + node = node_app.make_simple("node", port=port) + assert node is not None + assert isinstance(node, PostgresNode) + assert node.os_ops is node_svc.os_ops + assert node.port_manager is None # <--------- + assert node.port == port + assert node._should_free_port == False # noqa: E712 + + assert type(node_app.nodes_to_cleanup) == list # noqa: E721 + assert len(node_app.nodes_to_cleanup) == 1 + assert node_app.nodes_to_cleanup[0] is node + + node.slow_start() + finally: + if node is not None: + node.stop() + node.free_port() + + assert node._port is None + assert not node._should_free_port + + node.cleanup(release_resources=True) + + # ----------- + logging.info("temp directory [{}] is deleting".format(tmp_dir)) + node_svc.os_ops.rmdir(tmp_dir) + @staticmethod def helper__get_node( node_svc: PostgresNodeService, pFad - Phonifier reborn

Pfad - The Proxy pFad of © 2024 Garber Painting. All rights reserved.

Note: This service is not intended for secure transactions such as banking, social media, email, or purchasing. Use at your own risk. We assume no liability whatsoever for broken pages.


Alternative Proxies:

Alternative Proxy

pFad Proxy

pFad v3 Proxy

pFad v4 Proxy