diff --git a/.travis.yml b/.travis.yml index 55b7afa9..b762b0d0 100644 --- a/.travis.yml +++ b/.travis.yml @@ -19,6 +19,7 @@ notifications: on_failure: always env: + - TEST_PLATFORM=std2-all PYTHON_VERSION=3.7 PG_VERSION=17 - TEST_PLATFORM=std2-all PYTHON_VERSION=3.8.0 PG_VERSION=17 - TEST_PLATFORM=std2-all PYTHON_VERSION=3.8 PG_VERSION=17 - TEST_PLATFORM=std2-all PYTHON_VERSION=3.9 PG_VERSION=17 diff --git a/Dockerfile--std-all.tmpl b/Dockerfile--std-all.tmpl index c41c5a06..d19f52a6 100644 --- a/Dockerfile--std-all.tmpl +++ b/Dockerfile--std-all.tmpl @@ -4,11 +4,6 @@ ARG PYTHON_VERSION # --------------------------------------------- base1 FROM postgres:${PG_VERSION}-alpine as base1 -# --------------------------------------------- base2_with_python-2 -FROM base1 as base2_with_python-2 -RUN apk add --no-cache curl python2 python2-dev build-base musl-dev linux-headers py-virtualenv py-pip -ENV PYTHON_VERSION=2 - # --------------------------------------------- base2_with_python-3 FROM base1 as base2_with_python-3 RUN apk add --no-cache curl python3 python3-dev build-base musl-dev linux-headers py-virtualenv diff --git a/Dockerfile--std.tmpl b/Dockerfile--std.tmpl index 91886ede..67aa30b4 100644 --- a/Dockerfile--std.tmpl +++ b/Dockerfile--std.tmpl @@ -4,11 +4,6 @@ ARG PYTHON_VERSION # --------------------------------------------- base1 FROM postgres:${PG_VERSION}-alpine as base1 -# --------------------------------------------- base2_with_python-2 -FROM base1 as base2_with_python-2 -RUN apk add --no-cache curl python2 python2-dev build-base musl-dev linux-headers py-virtualenv py-pip -ENV PYTHON_VERSION=2 - # --------------------------------------------- base2_with_python-3 FROM base1 as base2_with_python-3 RUN apk add --no-cache curl python3 python3-dev build-base musl-dev linux-headers py-virtualenv diff --git a/Dockerfile--std2-all.tmpl b/Dockerfile--std2-all.tmpl index 10d8280c..6a3d817d 100644 --- a/Dockerfile--std2-all.tmpl +++ b/Dockerfile--std2-all.tmpl @@ -20,6 +20,10 @@ RUN apk add openssl openssl-dev RUN apk add sqlite-dev RUN apk add bzip2-dev +# --------------------------------------------- base3_with_python-3.7 +FROM base2_with_python-3 as base3_with_python-3.7 +ENV PYTHON_VERSION=3.7 + # --------------------------------------------- base3_with_python-3.8.0 FROM base2_with_python-3 as base3_with_python-3.8.0 ENV PYTHON_VERSION=3.8.0 diff --git a/README.md b/README.md index a3b854f8..af8172f3 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,7 @@ # testgres -PostgreSQL testing utility. Both Python 2.7 and 3.3+ are supported. +PostgreSQL testing utility. Python 3.7.17+ is supported. ## Installation diff --git a/__init__.py b/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/setup.py b/setup.py index 2c44b18f..0b209181 100755 --- a/setup.py +++ b/setup.py @@ -29,7 +29,7 @@ setup( version='1.11.0', name='testgres', - packages=['testgres', 'testgres.operations'], + packages=['testgres', 'testgres.operations', 'testgres.impl'], description='Testing utility for PostgreSQL and its extensions', url='https://github.com/postgrespro/testgres', long_description=readme, diff --git a/testgres/__init__.py b/testgres/__init__.py index 339ae62e..555784bf 100644 --- a/testgres/__init__.py +++ b/testgres/__init__.py @@ -33,8 +33,9 @@ ProcessType, \ DumpFormat -from .node import PostgresNode, NodeApp +from .node import PostgresNode from .node import PortManager +from .node_app import NodeApp from .utils import \ reserve_port, \ @@ -62,8 +63,9 @@ "NodeConnection", "DatabaseError", "InternalError", "ProgrammingError", "OperationalError", "TestgresException", "ExecUtilException", "QueryException", "TimeoutException", "CatchUpException", "StartNodeException", "InitNodeException", "BackupException", "InvalidOperationException", "XLogMethod", "IsolationLevel", "NodeStatus", "ProcessType", "DumpFormat", - "PostgresNode", "NodeApp", - "PortManager", + NodeApp.__name__, + PostgresNode.__name__, + PortManager.__name__, "reserve_port", "release_port", "bound_ports", "get_bin_path", "get_pg_config", "get_pg_version", "First", "Any", "OsOperations", "LocalOperations", "RemoteOperations", "ConnectionParams" diff --git a/testgres/backup.py b/testgres/backup.py index 388697b7..1d8454c3 100644 --- a/testgres/backup.py +++ b/testgres/backup.py @@ -1,7 +1,5 @@ # coding: utf-8 -import os - from six import raise_from from .enums import XLogMethod @@ -29,7 +27,9 @@ class NodeBackup(object): """ @property def log_file(self): - return os.path.join(self.base_dir, BACKUP_LOG_FILE) + assert self.os_ops is not None + assert isinstance(self.os_ops, OsOperations) + return self.os_ops.build_path(self.base_dir, BACKUP_LOG_FILE) def __init__(self, node, @@ -75,7 +75,7 @@ def __init__(self, # private self._available = True - data_dir = os.path.join(self.base_dir, DATA_DIR) + data_dir = self.os_ops.build_path(self.base_dir, DATA_DIR) _params = [ get_bin_path2(self.os_ops, "pg_basebackup"), @@ -112,10 +112,13 @@ def _prepare_dir(self, destroy): available = not destroy if available: + assert self.os_ops is not None + assert isinstance(self.os_ops, OsOperations) + dest_base_dir = self.os_ops.mkdtemp(prefix=TMP_NODE) - data1 = os.path.join(self.base_dir, DATA_DIR) - data2 = os.path.join(dest_base_dir, DATA_DIR) + data1 = self.os_ops.build_path(self.base_dir, DATA_DIR) + data2 = self.os_ops.build_path(dest_base_dir, DATA_DIR) try: # Copy backup to new data dir @@ -160,10 +163,6 @@ def spawn_primary(self, name=None, destroy=True): assert type(node) == self.original_node.__class__ # noqa: E721 with clean_on_error(node) as node: - - # New nodes should always remove dir tree - node._should_rm_dirs = True - # Set a new port node.append_conf(filename=PG_CONF_FILE, line='\n') node.append_conf(filename=PG_CONF_FILE, port=node.port) @@ -184,14 +183,19 @@ def spawn_replica(self, name=None, destroy=True, slot=None): """ # Build a new PostgresNode - with clean_on_error(self.spawn_primary(name=name, - destroy=destroy)) as node: + node = self.spawn_primary(name=name, destroy=destroy) + assert node is not None + try: # Assign it a master and a recovery file (private magic) node._assign_master(self.original_node) node._create_recovery_conf(username=self.username, slot=slot) + except: # noqa: E722 + # TODO: Pass 'final=True' ? + node.cleanup(release_resources=True) + raise - return node + return node def cleanup(self): """ diff --git a/testgres/cache.py b/testgres/cache.py index 3ac63326..e323c5d1 100644 --- a/testgres/cache.py +++ b/testgres/cache.py @@ -1,7 +1,5 @@ # coding: utf-8 -import os - from six import raise_from from .config import testgres_config @@ -22,12 +20,16 @@ from .operations.os_ops import OsOperations -def cached_initdb(data_dir, logfile=None, params=None, os_ops: OsOperations = LocalOperations(), bin_path=None, cached=True): +def cached_initdb(data_dir, logfile=None, params=None, os_ops: OsOperations = None, bin_path=None, cached=True): """ Perform initdb or use cached node files. """ - assert os_ops is not None + assert os_ops is None or isinstance(os_ops, OsOperations) + + if os_ops is None: + os_ops = LocalOperations.get_single_instance() + assert isinstance(os_ops, OsOperations) def make_utility_path(name): @@ -35,7 +37,7 @@ def make_utility_path(name): assert type(name) == str # noqa: E721 if bin_path: - return os.path.join(bin_path, name) + return os_ops.build_path(bin_path, name) return get_bin_path2(os_ops, name) @@ -68,7 +70,7 @@ def call_initdb(initdb_dir, log=logfile): # XXX: write new unique system id to control file # Some users might rely upon unique system ids, but # our initdb caching mechanism breaks this contract. - pg_control = os.path.join(data_dir, XLOG_CONTROL_FILE) + pg_control = os_ops.build_path(data_dir, XLOG_CONTROL_FILE) system_id = generate_system_id() cur_pg_control = os_ops.read(pg_control, binary=True) new_pg_control = system_id + cur_pg_control[len(system_id):] diff --git a/testgres/config.py b/testgres/config.py index 67d467d3..55d52426 100644 --- a/testgres/config.py +++ b/testgres/config.py @@ -50,8 +50,9 @@ class GlobalConfig(object): _cached_initdb_dir = None """ underlying class attribute for cached_initdb_dir property """ - os_ops = LocalOperations() + os_ops = LocalOperations.get_single_instance() """ OsOperation object that allows work on remote host """ + @property def cached_initdb_dir(self): """ path to a temp directory for cached initdb. """ diff --git a/testgres/impl/port_manager__generic.py b/testgres/impl/port_manager__generic.py new file mode 100755 index 00000000..567ff265 --- /dev/null +++ b/testgres/impl/port_manager__generic.py @@ -0,0 +1,97 @@ +from ..operations.os_ops import OsOperations + +from ..port_manager import PortManager +from ..exceptions import PortForException + +import threading +import random +import typing +import logging + + +class PortManager__Generic(PortManager): + _C_MIN_PORT_NUMBER = 1024 + _C_MAX_PORT_NUMBER = 65535 + + _os_ops: OsOperations + _guard: object + # TODO: is there better to use bitmap fot _available_ports? + _available_ports: typing.Set[int] + _reserved_ports: typing.Set[int] + + def __init__(self, os_ops: OsOperations): + assert __class__._C_MIN_PORT_NUMBER <= __class__._C_MAX_PORT_NUMBER + + assert os_ops is not None + assert isinstance(os_ops, OsOperations) + self._os_ops = os_ops + self._guard = threading.Lock() + + self._available_ports = set( + range(__class__._C_MIN_PORT_NUMBER, __class__._C_MAX_PORT_NUMBER + 1) + ) + assert len(self._available_ports) == ( + (__class__._C_MAX_PORT_NUMBER - __class__._C_MIN_PORT_NUMBER) + 1 + ) + self._reserved_ports = set() + return + + def reserve_port(self) -> int: + assert self._guard is not None + assert type(self._available_ports) == set # noqa: E721t + assert type(self._reserved_ports) == set # noqa: E721 + + with self._guard: + t = tuple(self._available_ports) + assert len(t) == len(self._available_ports) + sampled_ports = random.sample(t, min(len(t), 100)) + t = None + + for port in sampled_ports: + assert type(port) == int # noqa: E721 + assert not (port in self._reserved_ports) + assert port in self._available_ports + + assert port >= __class__._C_MIN_PORT_NUMBER + assert port <= __class__._C_MAX_PORT_NUMBER + + if not self._os_ops.is_port_free(port): + continue + + self._reserved_ports.add(port) + self._available_ports.discard(port) + assert port in self._reserved_ports + assert not (port in self._available_ports) + __class__.helper__send_debug_msg("Port {} is reserved.", port) + return port + + raise PortForException("Can't select a port.") + + def release_port(self, number: int) -> None: + assert type(number) == int # noqa: E721 + assert number >= __class__._C_MIN_PORT_NUMBER + assert number <= __class__._C_MAX_PORT_NUMBER + + assert self._guard is not None + assert type(self._reserved_ports) == set # noqa: E721 + + with self._guard: + assert number in self._reserved_ports + assert not (number in self._available_ports) + self._available_ports.add(number) + self._reserved_ports.discard(number) + assert not (number in self._reserved_ports) + assert number in self._available_ports + __class__.helper__send_debug_msg("Port {} is released.", number) + return + + @staticmethod + def helper__send_debug_msg(msg_template: str, *args) -> None: + assert msg_template is not None + assert args is not None + assert type(msg_template) == str # noqa: E721 + assert type(args) == tuple # noqa: E721 + assert msg_template != "" + s = "[port manager] " + s += msg_template.format(*args) + logging.debug(s) diff --git a/testgres/impl/port_manager__this_host.py b/testgres/impl/port_manager__this_host.py new file mode 100755 index 00000000..0d56f356 --- /dev/null +++ b/testgres/impl/port_manager__this_host.py @@ -0,0 +1,33 @@ +from ..port_manager import PortManager + +from .. import utils + +import threading + + +class PortManager__ThisHost(PortManager): + sm_single_instance: PortManager = None + sm_single_instance_guard = threading.Lock() + + @staticmethod + def get_single_instance() -> PortManager: + assert __class__ == PortManager__ThisHost + assert __class__.sm_single_instance_guard is not None + + if __class__.sm_single_instance is not None: + assert type(__class__.sm_single_instance) == __class__ # noqa: E721 + return __class__.sm_single_instance + + with __class__.sm_single_instance_guard: + if __class__.sm_single_instance is None: + __class__.sm_single_instance = __class__() + assert __class__.sm_single_instance is not None + assert type(__class__.sm_single_instance) == __class__ # noqa: E721 + return __class__.sm_single_instance + + def reserve_port(self) -> int: + return utils.reserve_port() + + def release_port(self, number: int) -> None: + assert type(number) == int # noqa: E721 + return utils.release_port(number) diff --git a/testgres/node.py b/testgres/node.py index 3a294044..60d9e305 100644 --- a/testgres/node.py +++ b/testgres/node.py @@ -7,8 +7,6 @@ import signal import subprocess import threading -import tempfile -import platform from queue import Queue import time @@ -84,8 +82,8 @@ InvalidOperationException from .port_manager import PortManager -from .port_manager import PortManager__ThisHost -from .port_manager import PortManager__Generic +from .impl.port_manager__this_host import PortManager__ThisHost +from .impl.port_manager__generic import PortManager__Generic from .logger import TestgresLogger @@ -93,6 +91,8 @@ from .standby import First +from . import utils + from .utils import \ PgVer, \ eprint, \ @@ -107,7 +107,6 @@ from .operations.os_ops import ConnectionParams from .operations.os_ops import OsOperations from .operations.local_ops import LocalOperations -from .operations.remote_ops import RemoteOperations InternalError = pglib.InternalError ProgrammingError = pglib.ProgrammingError @@ -145,13 +144,13 @@ class PostgresNode(object): _port: typing.Optional[int] _should_free_port: bool _os_ops: OsOperations - _port_manager: PortManager + _port_manager: typing.Optional[PortManager] def __init__(self, name=None, base_dir=None, port: typing.Optional[int] = None, - conn_params: ConnectionParams = ConnectionParams(), + conn_params: ConnectionParams = None, bin_dir=None, prefix=None, os_ops: typing.Optional[OsOperations] = None, @@ -171,11 +170,15 @@ def __init__(self, assert os_ops is None or isinstance(os_ops, OsOperations) assert port_manager is None or isinstance(port_manager, PortManager) + if conn_params is not None: + assert type(conn_params) == ConnectionParams # noqa: E721 + + raise InvalidOperationException("conn_params is deprecated, please use os_ops parameter instead.") + # private if os_ops is None: - self._os_ops = __class__._get_os_ops(conn_params) + self._os_ops = __class__._get_os_ops() else: - assert conn_params is None assert isinstance(os_ops, OsOperations) self._os_ops = os_ops pass @@ -200,11 +203,14 @@ def __init__(self, self._should_free_port = False self._port_manager = None else: - if port_manager is not None: + if port_manager is None: + self._port_manager = __class__._get_port_manager(self._os_ops) + elif os_ops is None: + raise InvalidOperationException("When port_manager is not None you have to define os_ops, too.") + else: assert isinstance(port_manager, PortManager) + assert self._os_ops is os_ops self._port_manager = port_manager - else: - self._port_manager = __class__._get_port_manager(self._os_ops) assert self._port_manager is not None assert isinstance(self._port_manager, PortManager) @@ -231,8 +237,6 @@ def __enter__(self): return self def __exit__(self, type, value, traceback): - self.free_port() - # NOTE: Ctrl+C does not count! got_exception = type is not None and type != KeyboardInterrupt @@ -246,6 +250,8 @@ def __exit__(self, type, value, traceback): else: self._try_shutdown(attempts) + self._release_resources() + def __repr__(self): return "{}(name='{}', port={}, base_dir='{}')".format( self.__class__.__name__, @@ -255,24 +261,22 @@ def __repr__(self): ) @staticmethod - def _get_os_ops(conn_params: ConnectionParams) -> OsOperations: + def _get_os_ops() -> OsOperations: if testgres_config.os_ops: return testgres_config.os_ops - assert type(conn_params) == ConnectionParams # noqa: E721 - - if conn_params.ssh_key: - return RemoteOperations(conn_params) - - return LocalOperations(conn_params) + return LocalOperations.get_single_instance() @staticmethod def _get_port_manager(os_ops: OsOperations) -> PortManager: assert os_ops is not None assert isinstance(os_ops, OsOperations) - if isinstance(os_ops, LocalOperations): - return PortManager__ThisHost() + if os_ops is LocalOperations.get_single_instance(): + assert utils._old_port_manager is not None + assert type(utils._old_port_manager) == PortManager__Generic # noqa: E721 + assert utils._old_port_manager._os_ops is os_ops + return PortManager__ThisHost.get_single_instance() # TODO: Throw the exception "Please define a port manager." ? return PortManager__Generic(os_ops) @@ -294,7 +298,6 @@ def clone_with_new_name_and_base_dir(self, name: str, base_dir: str): node = PostgresNode( name=name, base_dir=base_dir, - conn_params=None, bin_dir=self._bin_dir, prefix=self._prefix, os_ops=self._os_ops, @@ -308,6 +311,11 @@ def os_ops(self) -> OsOperations: assert isinstance(self._os_ops, OsOperations) return self._os_ops + @property + def port_manager(self) -> typing.Optional[PortManager]: + assert self._port_manager is None or isinstance(self._port_manager, PortManager) + return self._port_manager + @property def name(self) -> str: if self._name is None: @@ -563,7 +571,11 @@ def bin_dir(self): @property def logs_dir(self): - path = os.path.join(self.base_dir, LOGS_DIR) + assert self._os_ops is not None + assert isinstance(self._os_ops, OsOperations) + + path = self._os_ops.build_path(self.base_dir, LOGS_DIR) + assert type(path) == str # noqa: E721 # NOTE: it's safe to create a new dir if not self.os_ops.path_exists(path): @@ -573,16 +585,31 @@ def logs_dir(self): @property def data_dir(self): + assert self._os_ops is not None + assert isinstance(self._os_ops, OsOperations) + # NOTE: we can't run initdb without user's args - return os.path.join(self.base_dir, DATA_DIR) + path = self._os_ops.build_path(self.base_dir, DATA_DIR) + assert type(path) == str # noqa: E721 + return path @property def utils_log_file(self): - return os.path.join(self.logs_dir, UTILS_LOG_FILE) + assert self._os_ops is not None + assert isinstance(self._os_ops, OsOperations) + + path = self._os_ops.build_path(self.logs_dir, UTILS_LOG_FILE) + assert type(path) == str # noqa: E721 + return path @property def pg_log_file(self): - return os.path.join(self.logs_dir, PG_LOG_FILE) + assert self._os_ops is not None + assert isinstance(self._os_ops, OsOperations) + + path = self._os_ops.build_path(self.logs_dir, PG_LOG_FILE) + assert type(path) == str # noqa: E721 + return path @property def version(self): @@ -706,7 +733,11 @@ def _create_recovery_conf(self, username, slot=None): ).format(options_string(**conninfo)) # yapf: disable # Since 12 recovery.conf had disappeared if self.version >= PgVer('12'): - signal_name = os.path.join(self.data_dir, "standby.signal") + assert self._os_ops is not None + assert isinstance(self._os_ops, OsOperations) + + signal_name = self._os_ops.build_path(self.data_dir, "standby.signal") + assert type(signal_name) == str # noqa: E721 self.os_ops.touch(signal_name) else: line += "standby_mode=on\n" @@ -755,11 +786,14 @@ def _collect_special_files(self): result = [] # list of important files + last N lines + assert self._os_ops is not None + assert isinstance(self._os_ops, OsOperations) + files = [ - (os.path.join(self.data_dir, PG_CONF_FILE), 0), - (os.path.join(self.data_dir, PG_AUTO_CONF_FILE), 0), - (os.path.join(self.data_dir, RECOVERY_CONF_FILE), 0), - (os.path.join(self.data_dir, HBA_CONF_FILE), 0), + (self._os_ops.build_path(self.data_dir, PG_CONF_FILE), 0), + (self._os_ops.build_path(self.data_dir, PG_AUTO_CONF_FILE), 0), + (self._os_ops.build_path(self.data_dir, RECOVERY_CONF_FILE), 0), + (self._os_ops.build_path(self.data_dir, HBA_CONF_FILE), 0), (self.pg_log_file, testgres_config.error_log_lines) ] # yapf: disable @@ -776,28 +810,6 @@ def _collect_special_files(self): return result - def _collect_log_files(self): - # dictionary of log files + size in bytes - - files = [ - self.pg_log_file - ] # yapf: disable - - result = {} - - for f in files: - # skip missing files - if not self.os_ops.path_exists(f): - continue - - file_size = self.os_ops.get_file_size(f) - assert type(file_size) == int # noqa: E721 - assert file_size >= 0 - - result[f] = file_size - - return result - def init(self, initdb_params=None, cached=True, **kwargs): """ Perform initdb for this node. @@ -813,10 +825,13 @@ def init(self, initdb_params=None, cached=True, **kwargs): """ # initialize this PostgreSQL node + assert self._os_ops is not None + assert isinstance(self._os_ops, OsOperations) + cached_initdb( data_dir=self.data_dir, logfile=self.utils_log_file, - os_ops=self.os_ops, + os_ops=self._os_ops, params=initdb_params, bin_path=self.bin_dir, cached=False) @@ -846,8 +861,11 @@ def default_conf(self, This instance of :class:`.PostgresNode`. """ - postgres_conf = os.path.join(self.data_dir, PG_CONF_FILE) - hba_conf = os.path.join(self.data_dir, HBA_CONF_FILE) + assert self._os_ops is not None + assert isinstance(self._os_ops, OsOperations) + + postgres_conf = self._os_ops.build_path(self.data_dir, PG_CONF_FILE) + hba_conf = self._os_ops.build_path(self.data_dir, HBA_CONF_FILE) # filter lines in hba file # get rid of comments and blank lines @@ -962,7 +980,7 @@ def append_conf(self, line='', filename=PG_CONF_FILE, **kwargs): # format a new config line lines.append('{} = {}'.format(option, value)) - config_name = os.path.join(self.data_dir, filename) + config_name = self._os_ops.build_path(self.data_dir, filename) conf_text = '' for line in lines: conf_text += text_type(line) + '\n' @@ -1051,22 +1069,6 @@ def slow_start(self, replica=False, dbname='template1', username=None, max_attem OperationalError}, max_attempts=max_attempts) - def _detect_port_conflict(self, log_files0, log_files1): - assert type(log_files0) == dict # noqa: E721 - assert type(log_files1) == dict # noqa: E721 - - for file in log_files1.keys(): - read_pos = 0 - - if file in log_files0.keys(): - read_pos = log_files0[file] # the previous size - - file_content = self.os_ops.read_binary(file, read_pos) - file_content_s = file_content.decode() - if 'Is another postmaster already running on port' in file_content_s: - return True - return False - def start(self, params=[], wait=True, exec_env=None): """ Starts the PostgreSQL node using pg_ctl if node has not been started. @@ -1126,8 +1128,7 @@ def LOCAL__raise_cannot_start_node__std(from_exception): assert isinstance(self._port_manager, PortManager) assert __class__._C_MAX_START_ATEMPTS > 1 - log_files0 = self._collect_log_files() - assert type(log_files0) == dict # noqa: E721 + log_reader = PostgresNodeLogReader(self, from_beginnig=False) nAttempt = 0 timeout = 1 @@ -1143,11 +1144,11 @@ def LOCAL__raise_cannot_start_node__std(from_exception): if nAttempt == __class__._C_MAX_START_ATEMPTS: LOCAL__raise_cannot_start_node(e, "Cannot start node after multiple attempts.") - log_files1 = self._collect_log_files() - if not self._detect_port_conflict(log_files0, log_files1): + is_it_port_conflict = PostgresNodeUtils.delect_port_conflict(log_reader) + + if not is_it_port_conflict: LOCAL__raise_cannot_start_node__std(e) - log_files0 = log_files1 logging.warning( "Detected a conflict with using the port {0}. Trying another port after a {1}-second sleep...".format(self._port, timeout) ) @@ -1320,27 +1321,20 @@ def pg_ctl(self, params): return execute_utility2(self.os_ops, _params, self.utils_log_file) + def release_resources(self): + """ + Release resorces owned by this node. + """ + return self._release_resources() + def free_port(self): """ Reclaim port owned by this node. NOTE: this method does not release manually defined port but reset it. """ - assert type(self._should_free_port) == bool # noqa: E721 - - if not self._should_free_port: - self._port = None - else: - assert type(self._port) == int # noqa: E721 - - assert self._port_manager is not None - assert isinstance(self._port_manager, PortManager) + return self._free_port() - port = self._port - self._should_free_port = False - self._port = None - self._port_manager.release_port(port) - - def cleanup(self, max_attempts=3, full=False): + def cleanup(self, max_attempts=3, full=False, release_resources=False): """ Stop node if needed and remove its data/logs directory. NOTE: take a look at TestgresConfig.node_cleanup_full. @@ -1363,6 +1357,9 @@ def cleanup(self, max_attempts=3, full=False): self.os_ops.rmdirs(rm_dir, ignore_errors=False) + if release_resources: + self._release_resources() + return self @method_decorator(positional_args_hack(['dbname', 'query'])) @@ -1372,6 +1369,8 @@ def psql(self, dbname=None, username=None, input=None, + host: typing.Optional[str] = None, + port: typing.Optional[int] = None, **variables): """ Execute a query using psql. @@ -1382,6 +1381,8 @@ def psql(self, dbname: database name to connect to. username: database user name. input: raw input to be passed. + host: an explicit host of server. + port: an explicit port of server. **variables: vars to be set before execution. Returns: @@ -1393,6 +1394,10 @@ def psql(self, >>> psql(query='select 3', ON_ERROR_STOP=1) """ + assert host is None or type(host) == str # noqa: E721 + assert port is None or type(port) == int # noqa: E721 + assert type(variables) == dict # noqa: E721 + return self._psql( ignore_errors=True, query=query, @@ -1400,6 +1405,8 @@ def psql(self, dbname=dbname, username=username, input=input, + host=host, + port=port, **variables ) @@ -1411,7 +1418,11 @@ def _psql( dbname=None, username=None, input=None, + host: typing.Optional[str] = None, + port: typing.Optional[int] = None, **variables): + assert host is None or type(host) == str # noqa: E721 + assert port is None or type(port) == int # noqa: E721 assert type(variables) == dict # noqa: E721 # @@ -1424,10 +1435,21 @@ def _psql( else: raise Exception("Input data must be None or bytes.") + if host is None: + host = self.host + + if port is None: + port = self.port + + assert host is not None + assert port is not None + assert type(host) == str # noqa: E721 + assert type(port) == int # noqa: E721 + psql_params = [ self._get_bin_path("psql"), - "-p", str(self.port), - "-h", self.host, + "-p", str(port), + "-h", host, "-U", username or self.os_ops.username, "-d", dbname or default_dbname(), "-X", # no .psqlrc @@ -2035,8 +2057,11 @@ def set_auto_conf(self, options, config='postgresql.auto.conf', rm_options={}): rm_options (set, optional): A set containing the names of the options to remove. Defaults to an empty set. """ + assert self._os_ops is not None + assert isinstance(self._os_ops, OsOperations) + # parse postgresql.auto.conf - path = os.path.join(self.data_dir, config) + path = self.os_ops.build_path(self.data_dir, config) lines = self.os_ops.readlines(path) current_options = {} @@ -2121,9 +2146,31 @@ def upgrade_from(self, old_node, options=None, expect_error=False): return self.os_ops.exec_command(upgrade_command, expect_error=expect_error) + def _release_resources(self): + self._free_port() + + def _free_port(self): + assert type(self._should_free_port) == bool # noqa: E721 + + if not self._should_free_port: + self._port = None + else: + assert type(self._port) == int # noqa: E721 + + assert self._port_manager is not None + assert isinstance(self._port_manager, PortManager) + + port = self._port + self._should_free_port = False + self._port = None + self._port_manager.release_port(port) + def _get_bin_path(self, filename): + assert self._os_ops is not None + assert isinstance(self._os_ops, OsOperations) + if self.bin_dir: - bin_path = os.path.join(self.bin_dir, filename) + bin_path = self._os_ops.build_path(self.bin_dir, filename) else: bin_path = get_bin_path2(self.os_ops, filename) return bin_path @@ -2153,156 +2200,162 @@ def _escape_config_value(value): return result -class NodeApp: +class PostgresNodeLogReader: + class LogInfo: + position: int - def __init__(self, test_path=None, nodes_to_cleanup=None, os_ops=LocalOperations()): - if test_path: - if os.path.isabs(test_path): - self.test_path = test_path - else: - self.test_path = os.path.join(os_ops.cwd(), test_path) - else: - self.test_path = os_ops.cwd() - self.nodes_to_cleanup = nodes_to_cleanup if nodes_to_cleanup else [] - self.os_ops = os_ops + def __init__(self, position: int): + self.position = position - def make_empty( + # -------------------------------------------------------------------- + class LogDataBlock: + _file_name: str + _position: int + _data: str + + def __init__( self, - base_dir=None, - port=None, - bin_dir=None): - real_base_dir = os.path.join(self.test_path, base_dir) - self.os_ops.rmdirs(real_base_dir, ignore_errors=True) - self.os_ops.makedirs(real_base_dir) + file_name: str, + position: int, + data: str + ): + assert type(file_name) == str # noqa: E721 + assert type(position) == int # noqa: E721 + assert type(data) == str # noqa: E721 + assert file_name != "" + assert position >= 0 + self._file_name = file_name + self._position = position + self._data = data + + @property + def file_name(self) -> str: + assert type(self._file_name) == str # noqa: E721 + assert self._file_name != "" + return self._file_name + + @property + def position(self) -> int: + assert type(self._position) == int # noqa: E721 + assert self._position >= 0 + return self._position + + @property + def data(self) -> str: + assert type(self._data) == str # noqa: E721 + return self._data + + # -------------------------------------------------------------------- + _node: PostgresNode + _logs: typing.Dict[str, LogInfo] + + # -------------------------------------------------------------------- + def __init__(self, node: PostgresNode, from_beginnig: bool): + assert node is not None + assert isinstance(node, PostgresNode) + assert type(from_beginnig) == bool # noqa: E721 + + self._node = node + + if from_beginnig: + self._logs = dict() + else: + self._logs = self._collect_logs() - node = PostgresNode(base_dir=real_base_dir, port=port, bin_dir=bin_dir) - node.should_rm_dirs = True - self.nodes_to_cleanup.append(node) + assert type(self._logs) == dict # noqa: E721 + return - return node + def read(self) -> typing.List[LogDataBlock]: + assert self._node is not None + assert isinstance(self._node, PostgresNode) - def make_simple( - self, - base_dir=None, - port=None, - set_replication=False, - ptrack_enable=False, - initdb_params=[], - pg_options={}, - checksum=True, - bin_dir=None): - assert type(pg_options) == dict # noqa: E721 - - if checksum and '--data-checksums' not in initdb_params: - initdb_params.append('--data-checksums') - node = self.make_empty(base_dir, port, bin_dir=bin_dir) - node.init( - initdb_params=initdb_params, allow_streaming=set_replication) - - # set major version - pg_version_file = self.os_ops.read(os.path.join(node.data_dir, 'PG_VERSION')) - node.major_version_str = str(pg_version_file.rstrip()) - node.major_version = float(node.major_version_str) - - # Set default parameters - options = { - 'max_connections': 100, - 'shared_buffers': '10MB', - 'fsync': 'off', - 'wal_level': 'logical', - 'hot_standby': 'off', - 'log_line_prefix': '%t [%p]: [%l-1] ', - 'log_statement': 'none', - 'log_duration': 'on', - 'log_min_duration_statement': 0, - 'log_connections': 'on', - 'log_disconnections': 'on', - 'restart_after_crash': 'off', - 'autovacuum': 'off', - # unix_socket_directories will be defined later - } - - # Allow replication in pg_hba.conf - if set_replication: - options['max_wal_senders'] = 10 - - if ptrack_enable: - options['ptrack.map_size'] = '1' - options['shared_preload_libraries'] = 'ptrack' - - if node.major_version >= 13: - options['wal_keep_size'] = '200MB' - else: - options['wal_keep_segments'] = '12' + cur_logs: typing.Dict[__class__.LogInfo] = self._collect_logs() + assert cur_logs is not None + assert type(cur_logs) == dict # noqa: E721 - # Apply given parameters - for option_name, option_value in iteritems(pg_options): - options[option_name] = option_value + assert type(self._logs) == dict # noqa: E721 - # Define delayed propertyes - if not ("unix_socket_directories" in options.keys()): - options["unix_socket_directories"] = __class__._gettempdir_for_socket() + result = list() - # Set config values - node.set_auto_conf(options) + for file_name, cur_log_info in cur_logs.items(): + assert type(file_name) == str # noqa: E721 + assert type(cur_log_info) == __class__.LogInfo # noqa: E721 - # kludge for testgres - # https://github.com/postgrespro/testgres/issues/54 - # for PG >= 13 remove 'wal_keep_segments' parameter - if node.major_version >= 13: - node.set_auto_conf({}, 'postgresql.conf', ['wal_keep_segments']) + read_pos = 0 - return node + if file_name in self._logs.keys(): + prev_log_info = self._logs[file_name] + assert type(prev_log_info) == __class__.LogInfo # noqa: E721 + read_pos = prev_log_info.position # the previous size - @staticmethod - def _gettempdir_for_socket(): - platform_system_name = platform.system().lower() + file_content_b = self._node.os_ops.read_binary(file_name, read_pos) + assert type(file_content_b) == bytes # noqa: E721 - if platform_system_name == "windows": - return __class__._gettempdir() + # + # A POTENTIAL PROBLEM: file_content_b may contain an incompleted UTF-8 symbol. + # + file_content_s = file_content_b.decode() + assert type(file_content_s) == str # noqa: E721 - # - # [2025-02-17] Hot fix. - # - # Let's use hard coded path as Postgres likes. - # - # pg_config_manual.h: - # - # #ifndef WIN32 - # #define DEFAULT_PGSOCKET_DIR "/tmp" - # #else - # #define DEFAULT_PGSOCKET_DIR "" - # #endif - # - # On the altlinux-10 tempfile.gettempdir() may return - # the path to "private" temp directiry - "/temp/.private//" - # - # But Postgres want to find a socket file in "/tmp" (see above). - # + next_read_pos = read_pos + len(file_content_b) - return "/tmp" + # It is a research/paranoja check. + # When we will process partial UTF-8 symbol, it must be adjusted. + assert cur_log_info.position <= next_read_pos - @staticmethod - def _gettempdir(): - v = tempfile.gettempdir() + cur_log_info.position = next_read_pos - # - # Paranoid checks - # - if type(v) != str: # noqa: E721 - __class__._raise_bugcheck("tempfile.gettempdir returned a value with type {0}.".format(type(v).__name__)) + block = __class__.LogDataBlock( + file_name, + read_pos, + file_content_s + ) - if v == "": - __class__._raise_bugcheck("tempfile.gettempdir returned an empty string.") + result.append(block) - if not os.path.exists(v): - __class__._raise_bugcheck("tempfile.gettempdir returned a not exist path [{0}].".format(v)) + # A new check point + self._logs = cur_logs - # OK - return v + return result + + def _collect_logs(self) -> typing.Dict[LogInfo]: + assert self._node is not None + assert isinstance(self._node, PostgresNode) + + files = [ + self._node.pg_log_file + ] # yapf: disable + result = dict() + + for f in files: + assert type(f) == str # noqa: E721 + + # skip missing files + if not self._node.os_ops.path_exists(f): + continue + + file_size = self._node.os_ops.get_file_size(f) + assert type(file_size) == int # noqa: E721 + assert file_size >= 0 + + result[f] = __class__.LogInfo(file_size) + + return result + + +class PostgresNodeUtils: @staticmethod - def _raise_bugcheck(msg): - assert type(msg) == str # noqa: E721 - assert msg != "" - raise Exception("[BUG CHECK] " + msg) + def delect_port_conflict(log_reader: PostgresNodeLogReader) -> bool: + assert type(log_reader) == PostgresNodeLogReader # noqa: E721 + + blocks = log_reader.read() + assert type(blocks) == list # noqa: E721 + + for block in blocks: + assert type(block) == PostgresNodeLogReader.LogDataBlock # noqa: E721 + + if 'Is another postmaster already running on port' in block.data: + return True + + return False diff --git a/testgres/node_app.py b/testgres/node_app.py new file mode 100644 index 00000000..6e7b7c4f --- /dev/null +++ b/testgres/node_app.py @@ -0,0 +1,317 @@ +from .node import OsOperations +from .node import LocalOperations +from .node import PostgresNode +from .node import PortManager + +import os +import platform +import tempfile +import typing + + +T_DICT_STR_STR = typing.Dict[str, str] +T_LIST_STR = typing.List[str] + + +class NodeApp: + _test_path: str + _os_ops: OsOperations + _port_manager: PortManager + _nodes_to_cleanup: typing.List[PostgresNode] + + def __init__( + self, + test_path: typing.Optional[str] = None, + nodes_to_cleanup: typing.Optional[list] = None, + os_ops: typing.Optional[OsOperations] = None, + port_manager: typing.Optional[PortManager] = None, + ): + assert test_path is None or type(test_path) == str # noqa: E721 + assert os_ops is None or isinstance(os_ops, OsOperations) + assert port_manager is None or isinstance(port_manager, PortManager) + + if os_ops is None: + os_ops = LocalOperations.get_single_instance() + + assert isinstance(os_ops, OsOperations) + self._os_ops = os_ops + self._port_manager = port_manager + + if test_path is None: + self._test_path = os_ops.cwd() + elif os.path.isabs(test_path): + self._test_path = test_path + else: + self._test_path = os_ops.build_path(os_ops.cwd(), test_path) + + if nodes_to_cleanup is None: + self._nodes_to_cleanup = [] + else: + self._nodes_to_cleanup = nodes_to_cleanup + + @property + def test_path(self) -> str: + assert type(self._test_path) == str # noqa: E721 + return self._test_path + + @property + def os_ops(self) -> OsOperations: + assert isinstance(self._os_ops, OsOperations) + return self._os_ops + + @property + def port_manager(self) -> PortManager: + assert self._port_manager is None or isinstance(self._port_manager, PortManager) + return self._port_manager + + @property + def nodes_to_cleanup(self) -> typing.List[PostgresNode]: + assert type(self._nodes_to_cleanup) == list # noqa: E721 + return self._nodes_to_cleanup + + def make_empty( + self, + base_dir: str, + port: typing.Optional[int] = None, + bin_dir: typing.Optional[str] = None + ) -> PostgresNode: + assert type(base_dir) == str # noqa: E721 + assert port is None or type(port) == int # noqa: E721 + assert bin_dir is None or type(bin_dir) == str # noqa: E721 + + assert isinstance(self._os_ops, OsOperations) + assert type(self._test_path) == str # noqa: E721 + + if base_dir is None: + raise ValueError("Argument 'base_dir' is not defined.") + + if base_dir == "": + raise ValueError("Argument 'base_dir' is empty.") + + real_base_dir = self._os_ops.build_path(self._test_path, base_dir) + self._os_ops.rmdirs(real_base_dir, ignore_errors=True) + self._os_ops.makedirs(real_base_dir) + + port_manager: PortManager = None + + if port is None: + port_manager = self._port_manager + + node = PostgresNode( + base_dir=real_base_dir, + port=port, + bin_dir=bin_dir, + os_ops=self._os_ops, + port_manager=port_manager + ) + + try: + assert type(self._nodes_to_cleanup) == list # noqa: E721 + self._nodes_to_cleanup.append(node) + except: # noqa: E722 + node.cleanup(release_resources=True) + raise + + return node + + def make_simple( + self, + base_dir: str, + port: typing.Optional[int] = None, + set_replication: bool = False, + ptrack_enable: bool = False, + initdb_params: typing.Optional[T_LIST_STR] = None, + pg_options: typing.Optional[T_DICT_STR_STR] = None, + checksum: bool = True, + bin_dir: typing.Optional[str] = None + ) -> PostgresNode: + assert type(base_dir) == str # noqa: E721 + assert port is None or type(port) == int # noqa: E721 + assert type(set_replication) == bool # noqa: E721 + assert type(ptrack_enable) == bool # noqa: E721 + assert initdb_params is None or type(initdb_params) == list # noqa: E721 + assert pg_options is None or type(pg_options) == dict # noqa: E721 + assert type(checksum) == bool # noqa: E721 + assert bin_dir is None or type(bin_dir) == str # noqa: E721 + + node = self.make_empty( + base_dir, + port, + bin_dir=bin_dir + ) + + final_initdb_params = initdb_params + + if checksum: + final_initdb_params = __class__._paramlist_append_is_not_exist( + initdb_params, + final_initdb_params, + '--data-checksums' + ) + assert final_initdb_params is not None + assert '--data-checksums' in final_initdb_params + + node.init( + initdb_params=final_initdb_params, + allow_streaming=set_replication + ) + + # set major version + pg_version_file = self._os_ops.read(self._os_ops.build_path(node.data_dir, 'PG_VERSION')) + node.major_version_str = str(pg_version_file.rstrip()) + node.major_version = float(node.major_version_str) + + # Set default parameters + options = { + 'max_connections': 100, + 'shared_buffers': '10MB', + 'fsync': 'off', + 'wal_level': 'logical', + 'hot_standby': 'off', + 'log_line_prefix': '%t [%p]: [%l-1] ', + 'log_statement': 'none', + 'log_duration': 'on', + 'log_min_duration_statement': 0, + 'log_connections': 'on', + 'log_disconnections': 'on', + 'restart_after_crash': 'off', + 'autovacuum': 'off', + # unix_socket_directories will be defined later + } + + # Allow replication in pg_hba.conf + if set_replication: + options['max_wal_senders'] = 10 + + if ptrack_enable: + options['ptrack.map_size'] = '1' + options['shared_preload_libraries'] = 'ptrack' + + if node.major_version >= 13: + options['wal_keep_size'] = '200MB' + else: + options['wal_keep_segments'] = '12' + + # Apply given parameters + if pg_options is not None: + assert type(pg_options) == dict # noqa: E721 + for option_name, option_value in pg_options.items(): + options[option_name] = option_value + + # Define delayed propertyes + if not ("unix_socket_directories" in options.keys()): + options["unix_socket_directories"] = __class__._gettempdir_for_socket() + + # Set config values + node.set_auto_conf(options) + + # kludge for testgres + # https://github.com/postgrespro/testgres/issues/54 + # for PG >= 13 remove 'wal_keep_segments' parameter + if node.major_version >= 13: + node.set_auto_conf({}, 'postgresql.conf', ['wal_keep_segments']) + + return node + + @staticmethod + def _paramlist_has_param( + params: typing.Optional[T_LIST_STR], + param: str + ) -> bool: + assert type(param) == str # noqa: E721 + + if params is None: + return False + + assert type(params) == list # noqa: E721 + + if param in params: + return True + + return False + + @staticmethod + def _paramlist_append( + user_params: typing.Optional[T_LIST_STR], + updated_params: typing.Optional[T_LIST_STR], + param: str, + ) -> T_LIST_STR: + assert user_params is None or type(user_params) == list # noqa: E721 + assert updated_params is None or type(updated_params) == list # noqa: E721 + assert type(param) == str # noqa: E721 + + if updated_params is None: + if user_params is None: + return [param] + + return [*user_params, param] + + assert updated_params is not None + if updated_params is user_params: + return [*user_params, param] + + updated_params.append(param) + return updated_params + + @staticmethod + def _paramlist_append_is_not_exist( + user_params: typing.Optional[T_LIST_STR], + updated_params: typing.Optional[T_LIST_STR], + param: str, + ) -> typing.Optional[T_LIST_STR]: + if __class__._paramlist_has_param(updated_params, param): + return updated_params + return __class__._paramlist_append(user_params, updated_params, param) + + @staticmethod + def _gettempdir_for_socket() -> str: + platform_system_name = platform.system().lower() + + if platform_system_name == "windows": + return __class__._gettempdir() + + # + # [2025-02-17] Hot fix. + # + # Let's use hard coded path as Postgres likes. + # + # pg_config_manual.h: + # + # #ifndef WIN32 + # #define DEFAULT_PGSOCKET_DIR "/tmp" + # #else + # #define DEFAULT_PGSOCKET_DIR "" + # #endif + # + # On the altlinux-10 tempfile.gettempdir() may return + # the path to "private" temp directiry - "/temp/.private//" + # + # But Postgres want to find a socket file in "/tmp" (see above). + # + + return "/tmp" + + @staticmethod + def _gettempdir() -> str: + v = tempfile.gettempdir() + + # + # Paranoid checks + # + if type(v) != str: # noqa: E721 + __class__._raise_bugcheck("tempfile.gettempdir returned a value with type {0}.".format(type(v).__name__)) + + if v == "": + __class__._raise_bugcheck("tempfile.gettempdir returned an empty string.") + + if not os.path.exists(v): + __class__._raise_bugcheck("tempfile.gettempdir returned a not exist path [{0}].".format(v)) + + # OK + return v + + @staticmethod + def _raise_bugcheck(msg): + assert type(msg) == str # noqa: E721 + assert msg != "" + raise Exception("[BUG CHECK] " + msg) diff --git a/testgres/operations/local_ops.py b/testgres/operations/local_ops.py index 74323bb8..99d8e322 100644 --- a/testgres/operations/local_ops.py +++ b/testgres/operations/local_ops.py @@ -1,3 +1,5 @@ +from __future__ import annotations + import getpass import logging import os @@ -10,6 +12,8 @@ import psutil import typing +import threading +import copy from ..exceptions import ExecUtilException from ..exceptions import InvalidOperationException @@ -28,16 +32,57 @@ class LocalOperations(OsOperations): + sm_dummy_conn_params = ConnectionParams() + sm_single_instance: OsOperations = None + sm_single_instance_guard = threading.Lock() + + # TODO: make it read-only + conn_params: ConnectionParams + host: str + ssh_key: typing.Optional[str] + remote: bool + username: str + def __init__(self, conn_params=None): + super().__init__() + + if conn_params is __class__.sm_dummy_conn_params: + return + if conn_params is None: conn_params = ConnectionParams() - super(LocalOperations, self).__init__(conn_params.username) + self.conn_params = conn_params self.host = conn_params.host self.ssh_key = None self.remote = False self.username = conn_params.username or getpass.getuser() + @staticmethod + def get_single_instance() -> OsOperations: + assert __class__ == LocalOperations + assert __class__.sm_single_instance_guard is not None + + if __class__.sm_single_instance is not None: + assert type(__class__.sm_single_instance) == __class__ # noqa: E721 + return __class__.sm_single_instance + + with __class__.sm_single_instance_guard: + if __class__.sm_single_instance is None: + __class__.sm_single_instance = __class__() + assert __class__.sm_single_instance is not None + assert type(__class__.sm_single_instance) == __class__ # noqa: E721 + return __class__.sm_single_instance + + def create_clone(self) -> LocalOperations: + clone = __class__(__class__.sm_dummy_conn_params) + clone.conn_params = copy.copy(self.conn_params) + clone.host = self.host + clone.ssh_key = self.ssh_key + clone.remote = self.remote + clone.username = self.username + return clone + @staticmethod def _process_output(encoding, temp_file_path): """Process the output of a command from a temporary file.""" @@ -47,8 +92,13 @@ def _process_output(encoding, temp_file_path): output = output.decode(encoding) return output, None # In Windows stderr writing in stdout - def _run_command__nt(self, cmd, shell, input, stdin, stdout, stderr, get_process, timeout, encoding, exec_env=None): + def _run_command__nt( + self, cmd, shell, input, stdin, stdout, stderr, get_process, timeout, encoding, + exec_env: typing.Optional[dict], + cwd: typing.Optional[str], + ): assert exec_env is None or type(exec_env) == dict # noqa: E721 + assert cwd is None or type(cwd) == str # noqa: E721 # TODO: why don't we use the data from input? @@ -84,6 +134,7 @@ def _run_command__nt(self, cmd, shell, input, stdin, stdout, stderr, get_process stdin=stdin or subprocess.PIPE if input is not None else None, stdout=stdout, stderr=stderr, + cwd=cwd, **extParams, ) if get_process: @@ -96,8 +147,13 @@ def _run_command__nt(self, cmd, shell, input, stdin, stdout, stderr, get_process output, error = self._process_output(encoding, temp_file_path) return process, output, error - def _run_command__generic(self, cmd, shell, input, stdin, stdout, stderr, get_process, timeout, encoding, exec_env=None): + def _run_command__generic( + self, cmd, shell, input, stdin, stdout, stderr, get_process, timeout, encoding, + exec_env: typing.Optional[dict], + cwd: typing.Optional[str], + ): assert exec_env is None or type(exec_env) == dict # noqa: E721 + assert cwd is None or type(cwd) == str # noqa: E721 input_prepared = None if not get_process: @@ -134,6 +190,7 @@ def _run_command__generic(self, cmd, shell, input, stdin, stdout, stderr, get_pr stdin=stdin or subprocess.PIPE if input is not None else None, stdout=stdout or subprocess.PIPE, stderr=stderr or subprocess.PIPE, + cwd=cwd, **extParams ) assert not (process is None) @@ -153,26 +210,44 @@ def _run_command__generic(self, cmd, shell, input, stdin, stdout, stderr, get_pr error = error.decode(encoding) return process, output, error - def _run_command(self, cmd, shell, input, stdin, stdout, stderr, get_process, timeout, encoding, exec_env=None): + def _run_command( + self, cmd, shell, input, stdin, stdout, stderr, get_process, timeout, encoding, + exec_env: typing.Optional[dict], + cwd: typing.Optional[str], + ): """Execute a command and return the process and its output.""" + + assert exec_env is None or type(exec_env) == dict # noqa: E721 + assert cwd is None or type(cwd) == str # noqa: E721 + if os.name == 'nt' and stdout is None: # Windows method = __class__._run_command__nt else: # Other OS method = __class__._run_command__generic - return method(self, cmd, shell, input, stdin, stdout, stderr, get_process, timeout, encoding, exec_env=exec_env) + return method(self, cmd, shell, input, stdin, stdout, stderr, get_process, timeout, encoding, exec_env, cwd) - def exec_command(self, cmd, wait_exit=False, verbose=False, expect_error=False, encoding=None, shell=False, - text=False, input=None, stdin=None, stdout=None, stderr=None, get_process=False, timeout=None, - ignore_errors=False, exec_env=None): + def exec_command( + self, cmd, wait_exit=False, verbose=False, expect_error=False, encoding=None, shell=False, + text=False, input=None, stdin=None, stdout=None, stderr=None, get_process=False, timeout=None, + ignore_errors=False, + exec_env: typing.Optional[dict] = None, + cwd: typing.Optional[str] = None + ): """ Execute a command in a subprocess and handle the output based on the provided parameters. """ assert type(expect_error) == bool # noqa: E721 assert type(ignore_errors) == bool # noqa: E721 assert exec_env is None or type(exec_env) == dict # noqa: E721 + assert cwd is None or type(cwd) == str # noqa: E721 + + process, output, error = self._run_command( + cmd, shell, input, stdin, stdout, stderr, get_process, timeout, encoding, + exec_env, + cwd + ) - process, output, error = self._run_command(cmd, shell, input, stdin, stdout, stderr, get_process, timeout, encoding, exec_env=exec_env) if get_process: return process @@ -199,6 +274,13 @@ def exec_command(self, cmd, wait_exit=False, verbose=False, expect_error=False, return output + def build_path(self, a: str, *parts: str) -> str: + assert a is not None + assert parts is not None + assert type(a) == str # noqa: E721 + assert type(parts) == tuple # noqa: E721 + return os.path.join(a, *parts) + # Environment setup def environ(self, var_name): return os.environ.get(var_name) @@ -230,6 +312,10 @@ def makedirs(self, path, remove_existing=False): except FileExistsError: pass + def makedir(self, path: str): + assert type(path) == str # noqa: E721 + os.mkdir(path) + # [2025-02-03] Old name of parameter attempts is "retries". def rmdirs(self, path, ignore_errors=True, attempts=3, delay=1): """ @@ -273,6 +359,10 @@ def rmdirs(self, path, ignore_errors=True, attempts=3, delay=1): # OK! return True + def rmdir(self, path: str): + assert type(path) == str # noqa: E721 + os.rmdir(path) + def listdir(self, path): return os.listdir(path) @@ -500,3 +590,10 @@ def is_port_free(self, number: int) -> bool: return True except OSError: return False + + def get_tempdir(self) -> str: + r = tempfile.gettempdir() + assert r is not None + assert type(r) == str # noqa: E721 + assert os.path.exists(r) + return r diff --git a/testgres/operations/os_ops.py b/testgres/operations/os_ops.py index d25e76bc..46422269 100644 --- a/testgres/operations/os_ops.py +++ b/testgres/operations/os_ops.py @@ -1,4 +1,5 @@ -import getpass +from __future__ import annotations + import locale @@ -17,14 +18,23 @@ def get_default_encoding(): class OsOperations: - def __init__(self, username=None): - self.ssh_key = None - self.username = username or getpass.getuser() + def __init__(self): + pass + + def create_clone(self) -> OsOperations: + raise NotImplementedError() # Command execution def exec_command(self, cmd, **kwargs): raise NotImplementedError() + def build_path(self, a: str, *parts: str) -> str: + assert a is not None + assert parts is not None + assert type(a) == str # noqa: E721 + assert type(parts) == tuple # noqa: E721 + raise NotImplementedError() + # Environment setup def environ(self, var_name): raise NotImplementedError() @@ -53,9 +63,17 @@ def get_name(self): def makedirs(self, path, remove_existing=False): raise NotImplementedError() + def makedir(self, path: str): + assert type(path) == str # noqa: E721 + raise NotImplementedError() + def rmdirs(self, path, ignore_errors=True): raise NotImplementedError() + def rmdir(self, path: str): + assert type(path) == str # noqa: E721 + raise NotImplementedError() + def listdir(self, path): raise NotImplementedError() @@ -122,3 +140,6 @@ def get_process_children(self, pid): def is_port_free(self, number: int): assert type(number) == int # noqa: E721 raise NotImplementedError() + + def get_tempdir(self) -> str: + raise NotImplementedError() diff --git a/testgres/operations/remote_ops.py b/testgres/operations/remote_ops.py index e722a2cb..15d78b1a 100644 --- a/testgres/operations/remote_ops.py +++ b/testgres/operations/remote_ops.py @@ -1,11 +1,15 @@ +from __future__ import annotations + import getpass import os +import posixpath import platform import subprocess import tempfile import io import logging import typing +import copy from ..exceptions import ExecUtilException from ..exceptions import InvalidOperationException @@ -41,11 +45,29 @@ def cmdline(self): class RemoteOperations(OsOperations): + sm_dummy_conn_params = ConnectionParams() + + conn_params: ConnectionParams + host: str + port: int + ssh_key: str + ssh_args: list + remote: bool + username: str + ssh_dest: str + def __init__(self, conn_params: ConnectionParams): if not platform.system().lower() == "linux": raise EnvironmentError("Remote operations are supported only on Linux!") - super().__init__(conn_params.username) + if conn_params is None: + raise ValueError("Argument 'conn_params' is None.") + + super().__init__() + + if conn_params is __class__.sm_dummy_conn_params: + return + self.conn_params = conn_params self.host = conn_params.host self.port = conn_params.port @@ -62,10 +84,25 @@ def __init__(self, conn_params: ConnectionParams): def __enter__(self): return self - def exec_command(self, cmd, wait_exit=False, verbose=False, expect_error=False, - encoding=None, shell=True, text=False, input=None, stdin=None, stdout=None, - stderr=None, get_process=None, timeout=None, ignore_errors=False, - exec_env=None): + def create_clone(self) -> RemoteOperations: + clone = __class__(__class__.sm_dummy_conn_params) + clone.conn_params = copy.copy(self.conn_params) + clone.host = self.host + clone.port = self.port + clone.ssh_key = self.ssh_key + clone.ssh_args = copy.copy(self.ssh_args) + clone.remote = self.remote + clone.username = self.username + clone.ssh_dest = self.ssh_dest + return clone + + def exec_command( + self, cmd, wait_exit=False, verbose=False, expect_error=False, + encoding=None, shell=True, text=False, input=None, stdin=None, stdout=None, + stderr=None, get_process=None, timeout=None, ignore_errors=False, + exec_env: typing.Optional[dict] = None, + cwd: typing.Optional[str] = None + ): """ Execute a command in the SSH session. Args: @@ -74,6 +111,7 @@ def exec_command(self, cmd, wait_exit=False, verbose=False, expect_error=False, assert type(expect_error) == bool # noqa: E721 assert type(ignore_errors) == bool # noqa: E721 assert exec_env is None or type(exec_env) == dict # noqa: E721 + assert cwd is None or type(cwd) == str # noqa: E721 input_prepared = None if not get_process: @@ -81,21 +119,21 @@ def exec_command(self, cmd, wait_exit=False, verbose=False, expect_error=False, assert input_prepared is None or (type(input_prepared) == bytes) # noqa: E721 - if type(cmd) == str: # noqa: E721 - cmd_s = cmd - elif type(cmd) == list: # noqa: E721 - cmd_s = subprocess.list2cmdline(cmd) - else: - raise ValueError("Invalid 'cmd' argument type - {0}".format(type(cmd).__name__)) + cmds = [] - assert type(cmd_s) == str # noqa: E721 + if cwd is not None: + assert type(cwd) == str # noqa: E721 + cmds.append(__class__._build_cmdline(["cd", cwd])) - cmd_items = __class__._make_exec_env_list(exec_env=exec_env) - cmd_items.append(cmd_s) + cmds.append(__class__._build_cmdline(cmd, exec_env)) - env_cmd_s = ';'.join(cmd_items) + assert len(cmds) >= 1 - ssh_cmd = ['ssh', self.ssh_dest] + self.ssh_args + [env_cmd_s] + cmdline = ";".join(cmds) + assert type(cmdline) == str # noqa: E721 + assert cmdline != "" + + ssh_cmd = ['ssh', self.ssh_dest] + self.ssh_args + [cmdline] process = subprocess.Popen(ssh_cmd, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE) assert not (process is None) @@ -138,6 +176,13 @@ def exec_command(self, cmd, wait_exit=False, verbose=False, expect_error=False, return output + def build_path(self, a: str, *parts: str) -> str: + assert a is not None + assert parts is not None + assert type(a) == str # noqa: E721 + assert type(parts) == tuple # noqa: E721 + return __class__._build_path(a, *parts) + # Environment setup def environ(self, var_name: str) -> str: """ @@ -159,7 +204,7 @@ def find_executable(self, executable): search_paths = search_paths.split(self.pathsep) for path in search_paths: - remote_file = os.path.join(path, executable) + remote_file = __class__._build_path(path, executable) if self.isfile(remote_file): return remote_file @@ -225,6 +270,11 @@ def makedirs(self, path, remove_existing=False): raise Exception("Couldn't create dir {} because of error {}".format(path, error)) return result + def makedir(self, path: str): + assert type(path) == str # noqa: E721 + cmd = ["mkdir", path] + self.exec_command(cmd) + def rmdirs(self, path, ignore_errors=True): """ Remove a directory in the remote server. @@ -265,6 +315,11 @@ def rmdirs(self, path, ignore_errors=True): return False return True + def rmdir(self, path: str): + assert type(path) == str # noqa: E721 + cmd = ["rmdir", path] + self.exec_command(cmd) + def listdir(self, path): """ List all files and directories in a directory. @@ -373,7 +428,7 @@ def mkstemp(self, prefix=None): def copytree(self, src, dst): if not os.path.isabs(dst): - dst = os.path.join('~', dst) + dst = __class__._build_path('~', dst) if self.isdir(dst): raise FileExistsError("Directory {} already exists.".format(dst)) return self.exec_command("cp -r {} {}".format(src, dst)) @@ -649,6 +704,34 @@ def is_port_free(self, number: int) -> bool: out=output ) + def get_tempdir(self) -> str: + command = ["mktemp", "-u", "-d"] + + exec_exitcode, exec_output, exec_error = self.exec_command( + command, + verbose=True, + encoding=get_default_encoding(), + ignore_errors=True + ) + + assert type(exec_exitcode) == int # noqa: E721 + assert type(exec_output) == str # noqa: E721 + assert type(exec_error) == str # noqa: E721 + + if exec_exitcode != 0: + RaiseError.CommandExecutionError( + cmd=command, + exit_code=exec_exitcode, + message="Could not detect a temporary directory.", + error=exec_error, + out=exec_output) + + temp_subdir = exec_output.strip() + assert type(temp_subdir) == str # noqa: E721 + temp_dir = os.path.dirname(temp_subdir) + assert type(temp_dir) == str # noqa: E721 + return temp_dir + @staticmethod def _is_port_free__process_0(error: str) -> bool: assert type(error) == str # noqa: E721 @@ -672,7 +755,31 @@ def _is_port_free__process_1(error: str) -> bool: return True @staticmethod - def _make_exec_env_list(exec_env: typing.Dict) -> typing.List[str]: + def _build_cmdline(cmd, exec_env: typing.Dict = None) -> str: + cmd_items = __class__._create_exec_env_list(exec_env) + + assert type(cmd_items) == list # noqa: E721 + + cmd_items.append(__class__._ensure_cmdline(cmd)) + + cmdline = ';'.join(cmd_items) + assert type(cmdline) == str # noqa: E721 + return cmdline + + @staticmethod + def _ensure_cmdline(cmd) -> typing.List[str]: + if type(cmd) == str: # noqa: E721 + cmd_s = cmd + elif type(cmd) == list: # noqa: E721 + cmd_s = subprocess.list2cmdline(cmd) + else: + raise ValueError("Invalid 'cmd' argument type - {0}".format(type(cmd).__name__)) + + assert type(cmd_s) == str # noqa: E721 + return cmd_s + + @staticmethod + def _create_exec_env_list(exec_env: typing.Dict) -> typing.List[str]: env: typing.Dict[str, str] = dict() # ---------------------------------- SYSTEM ENV @@ -734,6 +841,14 @@ def _quote_envvar(value: str) -> str: result += "\"" return result + @staticmethod + def _build_path(a: str, *parts: str) -> str: + assert a is not None + assert parts is not None + assert type(a) == str # noqa: E721 + assert type(parts) == tuple # noqa: E721 + return posixpath.join(a, *parts) + def normalize_error(error): if isinstance(error, bytes): diff --git a/testgres/plugins/pg_probackup2/pg_probackup2/app.py b/testgres/plugins/pg_probackup2/pg_probackup2/app.py index 5166e9b8..2b87b48f 100644 --- a/testgres/plugins/pg_probackup2/pg_probackup2/app.py +++ b/testgres/plugins/pg_probackup2/pg_probackup2/app.py @@ -60,6 +60,7 @@ def __init__(self, test_class: unittest.TestCase, self.archive_compress = init_params.archive_compress self.test_class.output = None self.execution_time = None + self.valgrind_sup_path = init_params.valgrind_sup_path def form_daemon_process(self, cmdline, env): def stream_output(stream: subprocess.PIPE) -> None: @@ -88,6 +89,7 @@ def stream_output(stream: subprocess.PIPE) -> None: return self.process.pid + # ---- Start run function ---- # def run(self, command, gdb=False, old_binary=False, return_id=True, env=None, skip_log_directory=False, expect_error=False, use_backup_dir=True, daemonize=False): """ @@ -98,26 +100,46 @@ def run(self, command, gdb=False, old_binary=False, return_id=True, env=None, gdb: when True it returns GDBObj(), when tuple('suspend', port) it runs probackup in suspended gdb mode with attachable gdb port, for local debugging """ + command = self._add_backup_dir_to_cmd(command, use_backup_dir) + # Old bin or regular one + binary_path = self._get_binary_path(old_binary) + + if not env: + env = self.test_env + # Add additional options if needed + command, strcommand = self._add_options(command, skip_log_directory) + + self.test_class.cmd = f"{binary_path} {strcommand}" + if self.verbose: + print(self.test_class.cmd) + + cmdline = self._form_cmdline(binary_path, command) + + if gdb is True: + # general test flow for using GDBObj + return GDBobj(cmdline, self.test_class) + + return self._execute_command(cmdline, env, command, gdb, expect_error, return_id, daemonize) + + def _add_backup_dir_to_cmd(self, command: list, use_backup_dir: TestBackupDir): if isinstance(use_backup_dir, TestBackupDir): - command = [command[0], *use_backup_dir.pb_args, *command[1:]] + return [command[0], *use_backup_dir.pb_args, *command[1:]] elif use_backup_dir: - command = [command[0], *self.backup_dir.pb_args, *command[1:]] + return [command[0], *self.backup_dir.pb_args, *command[1:]] else: - command = [command[0], *self.backup_dir.pb_args[2:], *command[1:]] - - if not self.probackup_old_path and old_binary: - logging.error('PGPROBACKUPBIN_OLD is not set') - exit(1) + return [command[0], *self.backup_dir.pb_args[2:], *command[1:]] + def _get_binary_path(self, old_binary): if old_binary: - binary_path = self.probackup_old_path - else: - binary_path = self.probackup_path - - if not env: - env = self.test_env + if not self.probackup_old_path: + logging.error('PGPROBACKUPBIN_OLD is not set') + exit(1) + return self.probackup_old_path + return self.probackup_path + def _add_options(self, command: list, skip_log_directory: bool): strcommand = ' '.join(str(p) for p in command) + if '--log-level-file' in strcommand and \ '--log-directory' not in strcommand and \ not skip_log_directory: @@ -125,26 +147,46 @@ def run(self, command, gdb=False, old_binary=False, return_id=True, env=None, strcommand += ' ' + command[-1] if 'pglz' in strcommand and \ - ' -j' not in strcommand and '--thread' not in strcommand: + ' -j' not in strcommand and \ + '--thread' not in strcommand: command += ['-j', '1'] strcommand += ' -j 1' - self.test_class.cmd = binary_path + ' ' + strcommand - if self.verbose: - print(self.test_class.cmd) + return command, strcommand + def _form_cmdline(self, binary_path, command): cmdline = [binary_path, *command] - if gdb is True: - # general test flow for using GDBObj - return GDBobj(cmdline, self.test_class) + if self.valgrind_sup_path and command[0] != "--version": + os.makedirs(self.pb_log_path, exist_ok=True) + if self.valgrind_sup_path and not os.path.isfile(self.valgrind_sup_path): + raise FileNotFoundError(f"PG_PROBACKUP_VALGRIND_SUP should contain path to valgrind suppression file, " + f"but found: {self.valgrind_sup_path}") + valgrind_cmd = [ + "valgrind", + "--gen-suppressions=all", + "--leak-check=full", + "--show-reachable=yes", + "--error-limit=no", + "--show-leak-kinds=all", + "--errors-for-leak-kinds=all", + "--error-exitcode=0", + f"--log-file={os.path.join(self.pb_log_path, f'valgrind-{command[0]}-%p.log')}", + f"--suppressions={self.valgrind_sup_path}", + "--" + ] + cmdline = valgrind_cmd + cmdline + + return cmdline + + def _execute_command(self, cmdline, env, command, gdb, expect_error, return_id, daemonize): try: - if type(gdb) is tuple and gdb[0] == 'suspend': - # special test flow for manually debug probackup + if isinstance(gdb, tuple) and gdb[0] == 'suspend': gdb_port = gdb[1] cmdline = ['gdbserver'] + ['localhost:' + str(gdb_port)] + cmdline logging.warning("pg_probackup gdb suspended, waiting gdb connection on localhost:{0}".format(gdb_port)) + # Execute command start_time = time.time() if daemonize: return self.form_daemon_process(cmdline, env) @@ -174,6 +216,7 @@ def run(self, command, gdb=False, old_binary=False, return_id=True, env=None, return self.test_class.output else: raise ProbackupException(self.test_class.output, self.test_class.cmd) + # ---- End run function ---- # def get_backup_id(self): if init_params.major_version > 2: diff --git a/testgres/plugins/pg_probackup2/pg_probackup2/gdb.py b/testgres/plugins/pg_probackup2/pg_probackup2/gdb.py index 2424c04d..b7ca549e 100644 --- a/testgres/plugins/pg_probackup2/pg_probackup2/gdb.py +++ b/testgres/plugins/pg_probackup2/pg_probackup2/gdb.py @@ -1,6 +1,5 @@ import functools import os -import re import subprocess import sys import unittest @@ -57,13 +56,6 @@ def __init__(self, cmd, env, attach=False): else: self.cmd = self.base_cmd + ['--args'] + cmd - # Get version - gdb_version_number = re.search( - br"^GNU gdb [^\d]*(\d+)\.(\d)", - gdb_version) - self.major_version = int(gdb_version_number.group(1)) - self.minor_version = int(gdb_version_number.group(2)) - if self.verbose: print([' '.join(map(str, self.cmd))]) diff --git a/testgres/plugins/pg_probackup2/pg_probackup2/init_helpers.py b/testgres/plugins/pg_probackup2/pg_probackup2/init_helpers.py index c4570a39..9c62dcf1 100644 --- a/testgres/plugins/pg_probackup2/pg_probackup2/init_helpers.py +++ b/testgres/plugins/pg_probackup2/pg_probackup2/init_helpers.py @@ -214,6 +214,8 @@ def __init__(self): else: raise Exception('Can\'t process pg_probackup version \"{}\": the major version is expected to be a number'.format(self.probackup_version)) + self.valgrind_sup_path = test_env.get('PG_PROBACKUP_VALGRIND_SUP', None) + def test_env(self): return self._test_env.copy() diff --git a/testgres/plugins/pg_probackup2/pg_probackup2/tests/test_basic.py b/testgres/plugins/pg_probackup2/pg_probackup2/tests/test_basic.py index ba788623..2540ddb0 100644 --- a/testgres/plugins/pg_probackup2/pg_probackup2/tests/test_basic.py +++ b/testgres/plugins/pg_probackup2/pg_probackup2/tests/test_basic.py @@ -4,14 +4,14 @@ import shutil import pytest -from ...... import testgres +import testgres from ...pg_probackup2.app import ProbackupApp from ...pg_probackup2.init_helpers import Init, init_params from ..storage.fs_backup import FSTestBackupDir class ProbackupTest: - pg_node: testgres.PostgresNode + pg_node: testgres.NodeApp @staticmethod def probackup_is_available() -> bool: @@ -75,21 +75,30 @@ def helper__build_backup_dir(self, backup='backup'): @pytest.mark.skipif(not ProbackupTest.probackup_is_available(), reason="Check that PGPROBACKUPBIN is defined and is valid.") class TestBasic(ProbackupTest): def test_full_backup(self): + assert self.pg_node is not None + assert type(self.pg_node) == testgres.NodeApp # noqa: E721 + assert self.pb is not None + assert type(self.pb) == ProbackupApp # noqa: E721 + # Setting up a simple test node node = self.pg_node.make_simple('node', pg_options={"fsync": "off", "synchronous_commit": "off"}) - # Initialize and configure Probackup - self.pb.init() - self.pb.add_instance('node', node) - self.pb.set_archiving('node', node) + assert node is not None + assert type(node) == testgres.PostgresNode # noqa: E721 + + with node: + # Initialize and configure Probackup + self.pb.init() + self.pb.add_instance('node', node) + self.pb.set_archiving('node', node) - # Start the node and initialize pgbench - node.slow_start() - node.pgbench_init(scale=100, no_vacuum=True) + # Start the node and initialize pgbench + node.slow_start() + node.pgbench_init(scale=100, no_vacuum=True) - # Perform backup and validation - backup_id = self.pb.backup_node('node', node) - out = self.pb.validate('node', backup_id) + # Perform backup and validation + backup_id = self.pb.backup_node('node', node) + out = self.pb.validate('node', backup_id) - # Check if the backup is valid - assert f"INFO: Backup {backup_id} is valid" in out + # Check if the backup is valid + assert f"INFO: Backup {backup_id} is valid" in out diff --git a/testgres/plugins/pg_probackup2/setup.py b/testgres/plugins/pg_probackup2/setup.py index 7a3212e4..b9b0067e 100644 --- a/testgres/plugins/pg_probackup2/setup.py +++ b/testgres/plugins/pg_probackup2/setup.py @@ -4,7 +4,7 @@ from distutils.core import setup setup( - version='0.1.0', + version='0.1.1', name='testgres_pg_probackup2', packages=['pg_probackup2', 'pg_probackup2.storage'], description='Plugin for testgres that manages pg_probackup2', diff --git a/testgres/port_manager.py b/testgres/port_manager.py index e2530470..1ae696c8 100644 --- a/testgres/port_manager.py +++ b/testgres/port_manager.py @@ -1,14 +1,3 @@ -from .operations.os_ops import OsOperations - -from .exceptions import PortForException - -from . import utils - -import threading -import random -import typing - - class PortManager: def __init__(self): super().__init__() @@ -19,85 +8,3 @@ def reserve_port(self) -> int: def release_port(self, number: int) -> None: assert type(number) == int # noqa: E721 raise NotImplementedError("PortManager::release_port is not implemented.") - - -class PortManager__ThisHost(PortManager): - sm_single_instance: PortManager = None - sm_single_instance_guard = threading.Lock() - - def __init__(self): - pass - - def __new__(cls) -> PortManager: - assert __class__ == PortManager__ThisHost - assert __class__.sm_single_instance_guard is not None - - if __class__.sm_single_instance is None: - with __class__.sm_single_instance_guard: - __class__.sm_single_instance = super().__new__(cls) - assert __class__.sm_single_instance - assert type(__class__.sm_single_instance) == __class__ # noqa: E721 - return __class__.sm_single_instance - - def reserve_port(self) -> int: - return utils.reserve_port() - - def release_port(self, number: int) -> None: - assert type(number) == int # noqa: E721 - return utils.release_port(number) - - -class PortManager__Generic(PortManager): - _os_ops: OsOperations - _guard: object - # TODO: is there better to use bitmap fot _available_ports? - _available_ports: typing.Set[int] - _reserved_ports: typing.Set[int] - - def __init__(self, os_ops: OsOperations): - assert os_ops is not None - assert isinstance(os_ops, OsOperations) - self._os_ops = os_ops - self._guard = threading.Lock() - self._available_ports: typing.Set[int] = set(range(1024, 65535)) - self._reserved_ports: typing.Set[int] = set() - - def reserve_port(self) -> int: - assert self._guard is not None - assert type(self._available_ports) == set # noqa: E721t - assert type(self._reserved_ports) == set # noqa: E721 - - with self._guard: - t = tuple(self._available_ports) - assert len(t) == len(self._available_ports) - sampled_ports = random.sample(t, min(len(t), 100)) - t = None - - for port in sampled_ports: - assert not (port in self._reserved_ports) - assert port in self._available_ports - - if not self._os_ops.is_port_free(port): - continue - - self._reserved_ports.add(port) - self._available_ports.discard(port) - assert port in self._reserved_ports - assert not (port in self._available_ports) - return port - - raise PortForException("Can't select a port.") - - def release_port(self, number: int) -> None: - assert type(number) == int # noqa: E721 - - assert self._guard is not None - assert type(self._reserved_ports) == set # noqa: E721 - - with self._guard: - assert number in self._reserved_ports - assert not (number in self._available_ports) - self._available_ports.add(number) - self._reserved_ports.discard(number) - assert not (number in self._reserved_ports) - assert number in self._available_ports diff --git a/testgres/utils.py b/testgres/utils.py index 2ff6f2a0..7ad4e536 100644 --- a/testgres/utils.py +++ b/testgres/utils.py @@ -6,8 +6,6 @@ import os import sys -import socket -import random from contextlib import contextmanager from packaging.version import Version, InvalidVersion @@ -15,18 +13,25 @@ from six import iteritems -from .exceptions import PortForException from .exceptions import ExecUtilException from .config import testgres_config as tconf from .operations.os_ops import OsOperations from .operations.remote_ops import RemoteOperations +from .operations.local_ops import LocalOperations from .operations.helpers import Helpers as OsHelpers +from .impl.port_manager__generic import PortManager__Generic + # rows returned by PG_CONFIG _pg_config_data = {} +# +# The old, global "port manager" always worked with LOCAL system +# +_old_port_manager = PortManager__Generic(LocalOperations.get_single_instance()) + # ports used by nodes -bound_ports = set() +bound_ports = _old_port_manager._reserved_ports # re-export version type @@ -43,28 +48,7 @@ def internal__reserve_port(): """ Generate a new port and add it to 'bound_ports'. """ - def LOCAL__is_port_free(port: int) -> bool: - """Check if a port is free to use.""" - with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: - try: - s.bind(("", port)) - return True - except OSError: - return False - - ports = set(range(1024, 65535)) - assert type(ports) == set # noqa: E721 - assert type(bound_ports) == set # noqa: E721 - ports.difference_update(bound_ports) - - sampled_ports = random.sample(tuple(ports), min(len(ports), 100)) - - for port in sampled_ports: - if LOCAL__is_port_free(port): - bound_ports.add(port) - return port - - raise PortForException("Can't select a port") + return _old_port_manager.reserve_port() def internal__release_port(port): @@ -73,9 +57,7 @@ def internal__release_port(port): """ assert type(port) == int # noqa: E721 - assert port in bound_ports - - bound_ports.discard(port) + return _old_port_manager.release_port(port) reserve_port = internal__reserve_port @@ -159,17 +141,17 @@ def get_bin_path2(os_ops: OsOperations, filename): if pg_config: bindir = get_pg_config(pg_config, os_ops)["BINDIR"] - return os.path.join(bindir, filename) + return os_ops.build_path(bindir, filename) # try PG_BIN pg_bin = os_ops.environ("PG_BIN") if pg_bin: - return os.path.join(pg_bin, filename) + return os_ops.build_path(pg_bin, filename) pg_config_path = os_ops.find_executable('pg_config') if pg_config_path: bindir = get_pg_config(pg_config_path)["BINDIR"] - return os.path.join(bindir, filename) + return os_ops.build_path(bindir, filename) return filename @@ -231,7 +213,7 @@ def cache_pg_config_data(cmd): # try PG_BIN pg_bin = os.environ.get("PG_BIN") if pg_bin: - cmd = os.path.join(pg_bin, "pg_config") + cmd = os_ops.build_path(pg_bin, "pg_config") return cache_pg_config_data(cmd) # try plain name @@ -245,8 +227,17 @@ def get_pg_version2(os_ops: OsOperations, bin_dir=None): assert os_ops is not None assert isinstance(os_ops, OsOperations) + C_POSTGRES_BINARY = "postgres" + # Get raw version (e.g., postgres (PostgreSQL) 9.5.7) - postgres_path = os.path.join(bin_dir, 'postgres') if bin_dir else get_bin_path2(os_ops, 'postgres') + if bin_dir is None: + postgres_path = get_bin_path2(os_ops, C_POSTGRES_BINARY) + else: + # [2025-06-25] OK ? + assert type(bin_dir) == str # noqa: E721 + assert bin_dir != "" + postgres_path = os_ops.build_path(bin_dir, 'postgres') + cmd = [postgres_path, '--version'] raw_ver = os_ops.exec_command(cmd, encoding='utf-8') diff --git a/tests/conftest.py b/tests/conftest.py index 6f2f9e41..a1a00757 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -9,26 +9,68 @@ import math import datetime import typing +import enum import _pytest.outcomes import _pytest.unittest import _pytest.logging +from packaging.version import Version + # ///////////////////////////////////////////////////////////////////////////// C_ROOT_DIR__RELATIVE = ".." # ///////////////////////////////////////////////////////////////////////////// -# TestConfigPropNames +T_TUPLE__str_int = typing.Tuple[str, int] -class TestConfigPropNames: - TEST_CFG__LOG_DIR = "TEST_CFG__LOG_DIR" +# ///////////////////////////////////////////////////////////////////////////// +# T_PLUGGY_RESULT +if Version(pluggy.__version__) <= Version("1.2"): + T_PLUGGY_RESULT = pluggy._result._Result +else: + T_PLUGGY_RESULT = pluggy.Result # ///////////////////////////////////////////////////////////////////////////// -T_TUPLE__str_int = typing.Tuple[str, int] +g_error_msg_count_key = pytest.StashKey[int]() +g_warning_msg_count_key = pytest.StashKey[int]() +g_critical_msg_count_key = pytest.StashKey[int]() + +# ///////////////////////////////////////////////////////////////////////////// +# T_TEST_PROCESS_KIND + + +class T_TEST_PROCESS_KIND(enum.Enum): + Master = 1 + Worker = 2 + + +# ///////////////////////////////////////////////////////////////////////////// +# T_TEST_PROCESS_MODE + + +class T_TEST_PROCESS_MODE(enum.Enum): + Collect = 1 + ExecTests = 2 + + +# ///////////////////////////////////////////////////////////////////////////// + +g_test_process_kind: typing.Optional[T_TEST_PROCESS_KIND] = None +g_test_process_mode: typing.Optional[T_TEST_PROCESS_MODE] = None + +g_worker_log_is_created: typing.Optional[bool] = None + +# ///////////////////////////////////////////////////////////////////////////// +# TestConfigPropNames + + +class TestConfigPropNames: + TEST_CFG__LOG_DIR = "TEST_CFG__LOG_DIR" + # ///////////////////////////////////////////////////////////////////////////// # TestStartupData__Helper @@ -50,13 +92,24 @@ def CalcRootDir() -> str: r = os.path.abspath(r) return r + # -------------------------------------------------------------------- + def CalcRootLogDir() -> str: + if TestConfigPropNames.TEST_CFG__LOG_DIR in os.environ: + resultPath = os.environ[TestConfigPropNames.TEST_CFG__LOG_DIR] + else: + rootDir = __class__.CalcRootDir() + resultPath = os.path.join(rootDir, "logs") + + assert type(resultPath) == str # noqa: E721 + return resultPath + # -------------------------------------------------------------------- def CalcCurrentTestWorkerSignature() -> str: currentPID = os.getpid() - assert type(currentPID) + assert type(currentPID) == int # noqa: E721 startTS = __class__.sm_StartTS - assert type(startTS) + assert type(startTS) == datetime.datetime # noqa: E721 result = "pytest-{0:04d}{1:02d}{2:02d}_{3:02d}{4:02d}{5:02d}".format( startTS.year, @@ -86,11 +139,18 @@ class TestStartupData: TestStartupData__Helper.CalcCurrentTestWorkerSignature() ) + sm_RootLogDir: str = TestStartupData__Helper.CalcRootLogDir() + # -------------------------------------------------------------------- def GetRootDir() -> str: assert type(__class__.sm_RootDir) == str # noqa: E721 return __class__.sm_RootDir + # -------------------------------------------------------------------- + def GetRootLogDir() -> str: + assert type(__class__.sm_RootLogDir) == str # noqa: E721 + return __class__.sm_RootLogDir + # -------------------------------------------------------------------- def GetCurrentTestWorkerSignature() -> str: assert type(__class__.sm_CurrentTestWorkerSignature) == str # noqa: E721 @@ -318,14 +378,9 @@ def helper__build_test_id(item: pytest.Function) -> str: # ///////////////////////////////////////////////////////////////////////////// -g_error_msg_count_key = pytest.StashKey[int]() -g_warning_msg_count_key = pytest.StashKey[int]() - -# ///////////////////////////////////////////////////////////////////////////// - def helper__makereport__setup( - item: pytest.Function, call: pytest.CallInfo, outcome: pluggy.Result + item: pytest.Function, call: pytest.CallInfo, outcome: T_PLUGGY_RESULT ): assert item is not None assert call is not None @@ -333,7 +388,7 @@ def helper__makereport__setup( # it may be pytest.Function or _pytest.unittest.TestCaseFunction assert isinstance(item, pytest.Function) assert type(call) == pytest.CallInfo # noqa: E721 - assert type(outcome) == pluggy.Result # noqa: E721 + assert type(outcome) == T_PLUGGY_RESULT # noqa: E721 C_LINE1 = "******************************************************" @@ -382,9 +437,19 @@ def helper__makereport__setup( return +# ------------------------------------------------------------------------ +class ExitStatusNames: + FAILED = "FAILED" + PASSED = "PASSED" + XFAILED = "XFAILED" + NOT_XFAILED = "NOT XFAILED" + SKIPPED = "SKIPPED" + UNEXPECTED = "UNEXPECTED" + + # ------------------------------------------------------------------------ def helper__makereport__call( - item: pytest.Function, call: pytest.CallInfo, outcome: pluggy.Result + item: pytest.Function, call: pytest.CallInfo, outcome: T_PLUGGY_RESULT ): assert item is not None assert call is not None @@ -392,13 +457,20 @@ def helper__makereport__call( # it may be pytest.Function or _pytest.unittest.TestCaseFunction assert isinstance(item, pytest.Function) assert type(call) == pytest.CallInfo # noqa: E721 - assert type(outcome) == pluggy.Result # noqa: E721 + assert type(outcome) == T_PLUGGY_RESULT # noqa: E721 # -------- - item_error_msg_count = item.stash.get(g_error_msg_count_key, 0) - assert type(item_error_msg_count) == int # noqa: E721 - assert item_error_msg_count >= 0 + item_error_msg_count1 = item.stash.get(g_error_msg_count_key, 0) + assert type(item_error_msg_count1) == int # noqa: E721 + assert item_error_msg_count1 >= 0 + + item_error_msg_count2 = item.stash.get(g_critical_msg_count_key, 0) + assert type(item_error_msg_count2) == int # noqa: E721 + assert item_error_msg_count2 >= 0 + + item_error_msg_count = item_error_msg_count1 + item_error_msg_count2 + # -------- item_warning_msg_count = item.stash.get(g_warning_msg_count_key, 0) assert type(item_warning_msg_count) == int # noqa: E721 assert item_warning_msg_count >= 0 @@ -424,6 +496,7 @@ def helper__makereport__call( # -------- exitStatus = None + exitStatusInfo = None if rep.outcome == "skipped": assert call.excinfo is not None # research assert call.excinfo.value is not None # research @@ -431,21 +504,21 @@ def helper__makereport__call( if type(call.excinfo.value) == _pytest.outcomes.Skipped: # noqa: E721 assert not hasattr(rep, "wasxfail") - exitStatus = "SKIPPED" + exitStatus = ExitStatusNames.SKIPPED reasonText = str(call.excinfo.value) reasonMsgTempl = "SKIP REASON: {0}" TEST_PROCESS_STATS.incrementSkippedTestCount() - elif type(call.excinfo.value) == _pytest.outcomes.XFailed: # noqa: E721 - exitStatus = "XFAILED" + elif type(call.excinfo.value) == _pytest.outcomes.XFailed: # noqa: E721 E501 + exitStatus = ExitStatusNames.XFAILED reasonText = str(call.excinfo.value) reasonMsgTempl = "XFAIL REASON: {0}" TEST_PROCESS_STATS.incrementXFailedTestCount(testID, item_error_msg_count) else: - exitStatus = "XFAILED" + exitStatus = ExitStatusNames.XFAILED assert hasattr(rep, "wasxfail") assert rep.wasxfail is not None assert type(rep.wasxfail) == str # noqa: E721 @@ -482,7 +555,7 @@ def helper__makereport__call( assert item_error_msg_count > 0 TEST_PROCESS_STATS.incrementFailedTestCount(testID, item_error_msg_count) - exitStatus = "FAILED" + exitStatus = ExitStatusNames.FAILED elif rep.outcome == "passed": assert call.excinfo is None @@ -497,15 +570,16 @@ def helper__makereport__call( warnMsg += " [" + rep.wasxfail + "]" logging.info(warnMsg) - exitStatus = "NOT XFAILED" + exitStatus = ExitStatusNames.NOT_XFAILED else: assert not hasattr(rep, "wasxfail") TEST_PROCESS_STATS.incrementPassedTestCount() - exitStatus = "PASSED" + exitStatus = ExitStatusNames.PASSED else: TEST_PROCESS_STATS.incrementUnexpectedTests() - exitStatus = "UNEXPECTED [{0}]".format(rep.outcome) + exitStatus = ExitStatusNames.UNEXPECTED + exitStatusInfo = rep.outcome # [2025-03-28] It may create a useless problem in new environment. # assert False @@ -513,6 +587,14 @@ def helper__makereport__call( if item_warning_msg_count > 0: TEST_PROCESS_STATS.incrementWarningTestCount(testID, item_warning_msg_count) + # -------- + assert exitStatus is not None + assert type(exitStatus) == str # noqa: E721 + + if exitStatus == ExitStatusNames.FAILED: + assert item_error_msg_count > 0 + pass + # -------- assert type(TEST_PROCESS_STATS.cTotalDuration) == datetime.timedelta # noqa: E721 assert type(testDurration) == datetime.timedelta # noqa: E721 @@ -521,11 +603,17 @@ def helper__makereport__call( assert testDurration <= TEST_PROCESS_STATS.cTotalDuration + # -------- + exitStatusLineData = exitStatus + + if exitStatusInfo is not None: + exitStatusLineData += " [{}]".format(exitStatusInfo) + # -------- logging.info("*") logging.info("* DURATION : {0}".format(timedelta_to_human_text(testDurration))) logging.info("*") - logging.info("* EXIT STATUS : {0}".format(exitStatus)) + logging.info("* EXIT STATUS : {0}".format(exitStatusLineData)) logging.info("* ERROR COUNT : {0}".format(item_error_msg_count)) logging.info("* WARNING COUNT: {0}".format(item_warning_msg_count)) logging.info("*") @@ -552,9 +640,9 @@ def pytest_runtest_makereport(item: pytest.Function, call: pytest.CallInfo): assert isinstance(item, pytest.Function) assert type(call) == pytest.CallInfo # noqa: E721 - outcome: pluggy.Result = yield + outcome = yield assert outcome is not None - assert type(outcome) == pluggy.Result # noqa: E721 + assert type(outcome) == T_PLUGGY_RESULT # noqa: E721 assert type(call.when) == str # noqa: E721 @@ -582,103 +670,87 @@ def pytest_runtest_makereport(item: pytest.Function, call: pytest.CallInfo): # ///////////////////////////////////////////////////////////////////////////// -class LogErrorWrapper2: +class LogWrapper2: _old_method: any - _counter: typing.Optional[int] + _err_counter: typing.Optional[int] + _warn_counter: typing.Optional[int] + + _critical_counter: typing.Optional[int] # -------------------------------------------------------------------- def __init__(self): self._old_method = None - self._counter = None + self._err_counter = None + self._warn_counter = None + + self._critical_counter = None # -------------------------------------------------------------------- def __enter__(self): assert self._old_method is None - assert self._counter is None - - self._old_method = logging.error - self._counter = 0 - - logging.error = self - return self + assert self._err_counter is None + assert self._warn_counter is None - # -------------------------------------------------------------------- - def __exit__(self, exc_type, exc_val, exc_tb): - assert self._old_method is not None - assert self._counter is not None + assert self._critical_counter is None - assert logging.error is self + assert logging.root is not None + assert isinstance(logging.root, logging.RootLogger) - logging.error = self._old_method - - self._old_method = None - self._counter = None - return False - - # -------------------------------------------------------------------- - def __call__(self, *args, **kwargs): - assert self._old_method is not None - assert self._counter is not None - - assert type(self._counter) == int # noqa: E721 - assert self._counter >= 0 - - r = self._old_method(*args, **kwargs) - - self._counter += 1 - assert self._counter > 0 - - return r + self._old_method = logging.root.handle + self._err_counter = 0 + self._warn_counter = 0 + self._critical_counter = 0 -# ///////////////////////////////////////////////////////////////////////////// - - -class LogWarningWrapper2: - _old_method: any - _counter: typing.Optional[int] - - # -------------------------------------------------------------------- - def __init__(self): - self._old_method = None - self._counter = None - - # -------------------------------------------------------------------- - def __enter__(self): - assert self._old_method is None - assert self._counter is None - - self._old_method = logging.warning - self._counter = 0 - - logging.warning = self + logging.root.handle = self return self # -------------------------------------------------------------------- def __exit__(self, exc_type, exc_val, exc_tb): assert self._old_method is not None - assert self._counter is not None + assert self._err_counter is not None + assert self._warn_counter is not None + + assert logging.root is not None + assert isinstance(logging.root, logging.RootLogger) - assert logging.warning is self + assert logging.root.handle is self - logging.warning = self._old_method + logging.root.handle = self._old_method self._old_method = None - self._counter = None + self._err_counter = None + self._warn_counter = None + self._critical_counter = None return False # -------------------------------------------------------------------- - def __call__(self, *args, **kwargs): + def __call__(self, record: logging.LogRecord): + assert record is not None + assert isinstance(record, logging.LogRecord) assert self._old_method is not None - assert self._counter is not None - - assert type(self._counter) == int # noqa: E721 - assert self._counter >= 0 - - r = self._old_method(*args, **kwargs) - - self._counter += 1 - assert self._counter > 0 + assert self._err_counter is not None + assert self._warn_counter is not None + assert self._critical_counter is not None + + assert type(self._err_counter) == int # noqa: E721 + assert self._err_counter >= 0 + assert type(self._warn_counter) == int # noqa: E721 + assert self._warn_counter >= 0 + assert type(self._critical_counter) == int # noqa: E721 + assert self._critical_counter >= 0 + + r = self._old_method(record) + + if record.levelno == logging.ERROR: + self._err_counter += 1 + assert self._err_counter > 0 + elif record.levelno == logging.WARNING: + self._warn_counter += 1 + assert self._warn_counter > 0 + elif record.levelno == logging.CRITICAL: + self._critical_counter += 1 + assert self._critical_counter > 0 return r @@ -699,6 +771,13 @@ def pytest_pyfunc_call(pyfuncitem: pytest.Function): assert pyfuncitem is not None assert isinstance(pyfuncitem, pytest.Function) + assert logging.root is not None + assert isinstance(logging.root, logging.RootLogger) + assert logging.root.handle is not None + + debug__log_handle_method = logging.root.handle + assert debug__log_handle_method is not None + debug__log_error_method = logging.error assert debug__log_error_method is not None @@ -707,55 +786,56 @@ def pytest_pyfunc_call(pyfuncitem: pytest.Function): pyfuncitem.stash[g_error_msg_count_key] = 0 pyfuncitem.stash[g_warning_msg_count_key] = 0 + pyfuncitem.stash[g_critical_msg_count_key] = 0 try: - with LogErrorWrapper2() as logErrorWrapper, LogWarningWrapper2() as logWarningWrapper: - assert type(logErrorWrapper) == LogErrorWrapper2 # noqa: E721 - assert logErrorWrapper._old_method is not None - assert type(logErrorWrapper._counter) == int # noqa: E721 - assert logErrorWrapper._counter == 0 - assert logging.error is logErrorWrapper - - assert type(logWarningWrapper) == LogWarningWrapper2 # noqa: E721 - assert logWarningWrapper._old_method is not None - assert type(logWarningWrapper._counter) == int # noqa: E721 - assert logWarningWrapper._counter == 0 - assert logging.warning is logWarningWrapper - - r: pluggy.Result = yield + with LogWrapper2() as logWrapper: + assert type(logWrapper) == LogWrapper2 # noqa: E721 + assert logWrapper._old_method is not None + assert type(logWrapper._err_counter) == int # noqa: E721 + assert logWrapper._err_counter == 0 + assert type(logWrapper._warn_counter) == int # noqa: E721 + assert logWrapper._warn_counter == 0 + assert type(logWrapper._critical_counter) == int # noqa: E721 + assert logWrapper._critical_counter == 0 + assert logging.root.handle is logWrapper + + r = yield assert r is not None - assert type(r) == pluggy.Result # noqa: E721 + assert type(r) == T_PLUGGY_RESULT # noqa: E721 - assert logErrorWrapper._old_method is not None - assert type(logErrorWrapper._counter) == int # noqa: E721 - assert logErrorWrapper._counter >= 0 - assert logging.error is logErrorWrapper - - assert logWarningWrapper._old_method is not None - assert type(logWarningWrapper._counter) == int # noqa: E721 - assert logWarningWrapper._counter >= 0 - assert logging.warning is logWarningWrapper + assert logWrapper._old_method is not None + assert type(logWrapper._err_counter) == int # noqa: E721 + assert logWrapper._err_counter >= 0 + assert type(logWrapper._warn_counter) == int # noqa: E721 + assert logWrapper._warn_counter >= 0 + assert type(logWrapper._critical_counter) == int # noqa: E721 + assert logWrapper._critical_counter >= 0 + assert logging.root.handle is logWrapper assert g_error_msg_count_key in pyfuncitem.stash assert g_warning_msg_count_key in pyfuncitem.stash + assert g_critical_msg_count_key in pyfuncitem.stash assert pyfuncitem.stash[g_error_msg_count_key] == 0 assert pyfuncitem.stash[g_warning_msg_count_key] == 0 + assert pyfuncitem.stash[g_critical_msg_count_key] == 0 - pyfuncitem.stash[g_error_msg_count_key] = logErrorWrapper._counter - pyfuncitem.stash[g_warning_msg_count_key] = logWarningWrapper._counter + pyfuncitem.stash[g_error_msg_count_key] = logWrapper._err_counter + pyfuncitem.stash[g_warning_msg_count_key] = logWrapper._warn_counter + pyfuncitem.stash[g_critical_msg_count_key] = logWrapper._critical_counter if r.exception is not None: pass - elif logErrorWrapper._counter == 0: - pass - else: - assert logErrorWrapper._counter > 0 + elif logWrapper._err_counter > 0: + r.force_exception(SIGNAL_EXCEPTION()) + elif logWrapper._critical_counter > 0: r.force_exception(SIGNAL_EXCEPTION()) finally: assert logging.error is debug__log_error_method assert logging.warning is debug__log_warning_method + assert logging.root.handle == debug__log_handle_method pass @@ -831,13 +911,37 @@ def helper__print_test_list2(tests: typing.List[T_TUPLE__str_int]) -> None: # ///////////////////////////////////////////////////////////////////////////// +# SUMMARY BUILDER + +@pytest.hookimpl(trylast=True) +def pytest_sessionfinish(): + # + # NOTE: It should execute after logging.pytest_sessionfinish + # + + global g_test_process_kind # noqa: F824 + global g_test_process_mode # noqa: F824 + global g_worker_log_is_created # noqa: F824 + + assert g_test_process_kind is not None + assert type(g_test_process_kind) == T_TEST_PROCESS_KIND # noqa: E721 + + if g_test_process_kind == T_TEST_PROCESS_KIND.Master: + return + + assert g_test_process_kind == T_TEST_PROCESS_KIND.Worker + + assert g_test_process_mode is not None + assert type(g_test_process_mode) == T_TEST_PROCESS_MODE # noqa: E721 + + if g_test_process_mode == T_TEST_PROCESS_MODE.Collect: + return -@pytest.fixture(autouse=True, scope="session") -def run_after_tests(request: pytest.FixtureRequest): - assert isinstance(request, pytest.FixtureRequest) + assert g_test_process_mode == T_TEST_PROCESS_MODE.ExecTests - yield + assert type(g_worker_log_is_created) == bool # noqa: E721 + assert g_worker_log_is_created C_LINE1 = "---------------------------" @@ -847,7 +951,9 @@ def LOCAL__print_line1_with_header(header: str): assert header != "" logging.info(C_LINE1 + " [" + header + "]") - def LOCAL__print_test_list(header: str, test_count: int, test_list: typing.List[str]): + def LOCAL__print_test_list( + header: str, test_count: int, test_list: typing.List[str] + ): assert type(header) == str # noqa: E721 assert type(test_count) == int # noqa: E721 assert type(test_list) == list # noqa: E721 @@ -947,20 +1053,41 @@ def LOCAL__print_test_list2( # ///////////////////////////////////////////////////////////////////////////// +def helper__detect_test_process_kind(config: pytest.Config) -> T_TEST_PROCESS_KIND: + assert isinstance(config, pytest.Config) + + # + # xdist' master process registers DSession plugin. + # + p = config.pluginmanager.get_plugin("dsession") + + if p is not None: + return T_TEST_PROCESS_KIND.Master + + return T_TEST_PROCESS_KIND.Worker + + +# ------------------------------------------------------------------------ +def helper__detect_test_process_mode(config: pytest.Config) -> T_TEST_PROCESS_MODE: + assert isinstance(config, pytest.Config) + + if config.getvalue("collectonly"): + return T_TEST_PROCESS_MODE.Collect + + return T_TEST_PROCESS_MODE.ExecTests + + +# ------------------------------------------------------------------------ @pytest.hookimpl(trylast=True) -def pytest_configure(config: pytest.Config) -> None: +def helper__pytest_configure__logging(config: pytest.Config) -> None: assert isinstance(config, pytest.Config) log_name = TestStartupData.GetCurrentTestWorkerSignature() log_name += ".log" - if TestConfigPropNames.TEST_CFG__LOG_DIR in os.environ: - log_path_v = os.environ[TestConfigPropNames.TEST_CFG__LOG_DIR] - log_path = pathlib.Path(log_path_v) - else: - log_path = config.rootpath.joinpath("logs") + log_dir = TestStartupData.GetRootLogDir() - log_path.mkdir(exist_ok=True) + pathlib.Path(log_dir).mkdir(exist_ok=True) logging_plugin: _pytest.logging.LoggingPlugin = config.pluginmanager.get_plugin( "logging-plugin" @@ -969,7 +1096,46 @@ def pytest_configure(config: pytest.Config) -> None: assert logging_plugin is not None assert isinstance(logging_plugin, _pytest.logging.LoggingPlugin) - logging_plugin.set_log_path(str(log_path / log_name)) + log_file_path = os.path.join(log_dir, log_name) + assert log_file_path is not None + assert type(log_file_path) == str # noqa: E721 + + logging_plugin.set_log_path(log_file_path) + return + + +# ------------------------------------------------------------------------ +@pytest.hookimpl(trylast=True) +def pytest_configure(config: pytest.Config) -> None: + assert isinstance(config, pytest.Config) + + global g_test_process_kind + global g_test_process_mode + global g_worker_log_is_created + + assert g_test_process_kind is None + assert g_test_process_mode is None + assert g_worker_log_is_created is None + + g_test_process_mode = helper__detect_test_process_mode(config) + g_test_process_kind = helper__detect_test_process_kind(config) + + assert type(g_test_process_kind) == T_TEST_PROCESS_KIND # noqa: E721 + assert type(g_test_process_mode) == T_TEST_PROCESS_MODE # noqa: E721 + + if g_test_process_kind == T_TEST_PROCESS_KIND.Master: + pass + else: + assert g_test_process_kind == T_TEST_PROCESS_KIND.Worker + + if g_test_process_mode == T_TEST_PROCESS_MODE.Collect: + g_worker_log_is_created = False + else: + assert g_test_process_mode == T_TEST_PROCESS_MODE.ExecTests + helper__pytest_configure__logging(config) + g_worker_log_is_created = True + + return # ///////////////////////////////////////////////////////////////////////////// diff --git a/tests/helpers/global_data.py b/tests/helpers/global_data.py index c21d7dd8..f3df41a3 100644 --- a/tests/helpers/global_data.py +++ b/tests/helpers/global_data.py @@ -1,11 +1,11 @@ -from ...testgres.operations.os_ops import OsOperations -from ...testgres.operations.os_ops import ConnectionParams -from ...testgres.operations.local_ops import LocalOperations -from ...testgres.operations.remote_ops import RemoteOperations +from testgres.operations.os_ops import OsOperations +from testgres.operations.os_ops import ConnectionParams +from testgres.operations.local_ops import LocalOperations +from testgres.operations.remote_ops import RemoteOperations -from ...testgres.node import PortManager -from ...testgres.node import PortManager__ThisHost -from ...testgres.node import PortManager__Generic +from testgres.node import PortManager +from testgres.node import PortManager__ThisHost +from testgres.node import PortManager__Generic import os @@ -31,7 +31,7 @@ class OsOpsDescrs: sm_remote_os_ops_descr = OsOpsDescr("remote_ops", sm_remote_os_ops) - sm_local_os_ops = LocalOperations() + sm_local_os_ops = LocalOperations.get_single_instance() sm_local_os_ops_descr = OsOpsDescr("local_ops", sm_local_os_ops) @@ -39,7 +39,7 @@ class OsOpsDescrs: class PortManagers: sm_remote_port_manager = PortManager__Generic(OsOpsDescrs.sm_remote_os_ops) - sm_local_port_manager = PortManager__ThisHost() + sm_local_port_manager = PortManager__ThisHost.get_single_instance() sm_local2_port_manager = PortManager__Generic(OsOpsDescrs.sm_local_os_ops) diff --git a/tests/test_config.py b/tests/test_config.py index 05702e9a..a80a11f1 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -1,9 +1,9 @@ -from ..testgres import TestgresConfig -from ..testgres import configure_testgres -from ..testgres import scoped_config -from ..testgres import pop_config +from testgres import TestgresConfig +from testgres import configure_testgres +from testgres import scoped_config +from testgres import pop_config -from .. import testgres +import testgres import pytest diff --git a/tests/test_os_ops_common.py b/tests/test_os_ops_common.py index 17c3151c..d3c85753 100644 --- a/tests/test_os_ops_common.py +++ b/tests/test_os_ops_common.py @@ -13,9 +13,13 @@ import socket import threading import typing +import uuid -from ..testgres import InvalidOperationException -from ..testgres import ExecUtilException +from testgres import InvalidOperationException +from testgres import ExecUtilException + +from concurrent.futures import ThreadPoolExecutor +from concurrent.futures import Future as ThreadFuture class TestOsOpsCommon: @@ -33,6 +37,13 @@ def os_ops(self, request: pytest.FixtureRequest) -> OsOperations: assert isinstance(request.param, OsOperations) return request.param + def test_create_clone(self, os_ops: OsOperations): + assert isinstance(os_ops, OsOperations) + clone = os_ops.create_clone() + assert clone is not None + assert clone is not os_ops + assert type(clone) == type(os_ops) # noqa: E721 + def test_exec_command_success(self, os_ops: OsOperations): """ Test exec_command for successful command execution. @@ -114,6 +125,23 @@ def test_exec_command_with_exec_env(self, os_ops: OsOperations): assert type(response) == bytes # noqa: E721 assert response == b'\n' + def test_exec_command_with_cwd(self, os_ops: OsOperations): + assert isinstance(os_ops, OsOperations) + + RunConditions.skip_if_windows() + + cmd = ["pwd"] + + response = os_ops.exec_command(cmd, cwd="/tmp") + assert response is not None + assert type(response) == bytes # noqa: E721 + assert response == b'/tmp\n' + + response = os_ops.exec_command(cmd) + assert response is not None + assert type(response) == bytes # noqa: E721 + assert response != b'/tmp\n' + def test_exec_command__test_unset(self, os_ops: OsOperations): assert isinstance(os_ops, OsOperations) @@ -812,3 +840,300 @@ def LOCAL_server(s: socket.socket): if ok_count == 0: raise RuntimeError("No one free port was found.") + + def test_get_tmpdir(self, os_ops: OsOperations): + assert isinstance(os_ops, OsOperations) + + dir = os_ops.get_tempdir() + assert type(dir) == str # noqa: E721 + assert os_ops.path_exists(dir) + assert os.path.exists(dir) + + file_path = os.path.join(dir, "testgres--" + uuid.uuid4().hex + ".tmp") + + os_ops.write(file_path, "1234", binary=False) + + assert os_ops.path_exists(file_path) + assert os.path.exists(file_path) + + d = os_ops.read(file_path, binary=False) + + assert d == "1234" + + os_ops.remove_file(file_path) + + assert not os_ops.path_exists(file_path) + assert not os.path.exists(file_path) + + def test_get_tmpdir__compare_with_py_info(self, os_ops: OsOperations): + assert isinstance(os_ops, OsOperations) + + actual_dir = os_ops.get_tempdir() + assert actual_dir is not None + assert type(actual_dir) == str # noqa: E721 + expected_dir = str(tempfile.tempdir) + assert actual_dir == expected_dir + + class tagData_OS_OPS__NUMS: + os_ops_descr: OsOpsDescr + nums: int + + def __init__(self, os_ops_descr: OsOpsDescr, nums: int): + assert isinstance(os_ops_descr, OsOpsDescr) + assert type(nums) == int # noqa: E721 + + self.os_ops_descr = os_ops_descr + self.nums = nums + + sm_test_exclusive_creation__mt__data = [ + tagData_OS_OPS__NUMS(OsOpsDescrs.sm_local_os_ops_descr, 100000), + tagData_OS_OPS__NUMS(OsOpsDescrs.sm_remote_os_ops_descr, 120), + ] + + @pytest.fixture( + params=sm_test_exclusive_creation__mt__data, + ids=[x.os_ops_descr.sign for x in sm_test_exclusive_creation__mt__data] + ) + def data001(self, request: pytest.FixtureRequest) -> tagData_OS_OPS__NUMS: + assert isinstance(request, pytest.FixtureRequest) + return request.param + + def test_mkdir__mt(self, data001: tagData_OS_OPS__NUMS): + assert type(data001) == __class__.tagData_OS_OPS__NUMS # noqa: E721 + + N_WORKERS = 4 + N_NUMBERS = data001.nums + assert type(N_NUMBERS) == int # noqa: E721 + + os_ops = data001.os_ops_descr.os_ops + assert isinstance(os_ops, OsOperations) + + lock_dir_prefix = "test_mkdir_mt--" + uuid.uuid4().hex + + lock_dir = os_ops.mkdtemp(prefix=lock_dir_prefix) + + logging.info("A lock file [{}] is creating ...".format(lock_dir)) + + assert os.path.exists(lock_dir) + + def MAKE_PATH(lock_dir: str, num: int) -> str: + assert type(lock_dir) == str # noqa: E721 + assert type(num) == int # noqa: E721 + return os.path.join(lock_dir, str(num) + ".lock") + + def LOCAL_WORKER(os_ops: OsOperations, + workerID: int, + lock_dir: str, + cNumbers: int, + reservedNumbers: typing.Set[int]) -> None: + assert isinstance(os_ops, OsOperations) + assert type(workerID) == int # noqa: E721 + assert type(lock_dir) == str # noqa: E721 + assert type(cNumbers) == int # noqa: E721 + assert type(reservedNumbers) == set # noqa: E721 + assert cNumbers > 0 + assert len(reservedNumbers) == 0 + + assert os.path.exists(lock_dir) + + def LOG_INFO(template: str, *args: list) -> None: + assert type(template) == str # noqa: E721 + assert type(args) == tuple # noqa: E721 + + msg = template.format(*args) + assert type(msg) == str # noqa: E721 + + logging.info("[Worker #{}] {}".format(workerID, msg)) + return + + LOG_INFO("HELLO! I am here!") + + for num in range(cNumbers): + assert not (num in reservedNumbers) + + file_path = MAKE_PATH(lock_dir, num) + + try: + os_ops.makedir(file_path) + except Exception as e: + LOG_INFO( + "Can't reserve {}. Error ({}): {}", + num, + type(e).__name__, + str(e) + ) + continue + + LOG_INFO("Number {} is reserved!", num) + assert os_ops.path_exists(file_path) + reservedNumbers.add(num) + continue + + n_total = cNumbers + n_ok = len(reservedNumbers) + assert n_ok <= n_total + + LOG_INFO("Finish! OK: {}. FAILED: {}.", n_ok, n_total - n_ok) + return + + # ----------------------- + logging.info("Worker are creating ...") + + threadPool = ThreadPoolExecutor( + max_workers=N_WORKERS, + thread_name_prefix="ex_creator" + ) + + class tadWorkerData: + future: ThreadFuture + reservedNumbers: typing.Set[int] + + workerDatas: typing.List[tadWorkerData] = list() + + nErrors = 0 + + try: + for n in range(N_WORKERS): + logging.info("worker #{} is creating ...".format(n)) + + workerDatas.append(tadWorkerData()) + + workerDatas[n].reservedNumbers = set() + + workerDatas[n].future = threadPool.submit( + LOCAL_WORKER, + os_ops, + n, + lock_dir, + N_NUMBERS, + workerDatas[n].reservedNumbers + ) + + assert workerDatas[n].future is not None + + logging.info("OK. All the workers were created!") + except Exception as e: + nErrors += 1 + logging.error("A problem is detected ({}): {}".format(type(e).__name__, str(e))) + + logging.info("Will wait for stop of all the workers...") + + nWorkers = 0 + + assert type(workerDatas) == list # noqa: E721 + + for i in range(len(workerDatas)): + worker = workerDatas[i].future + + if worker is None: + continue + + nWorkers += 1 + + assert isinstance(worker, ThreadFuture) + + try: + logging.info("Wait for worker #{}".format(i)) + worker.result() + except Exception as e: + nErrors += 1 + logging.error("Worker #{} finished with error ({}): {}".format( + i, + type(e).__name__, + str(e), + )) + continue + + assert nWorkers == N_WORKERS + + if nErrors != 0: + raise RuntimeError("Some problems were detected. Please examine the log messages.") + + logging.info("OK. Let's check worker results!") + + reservedNumbers: typing.Dict[int, int] = dict() + + for i in range(N_WORKERS): + logging.info("Worker #{} is checked ...".format(i)) + + workerNumbers = workerDatas[i].reservedNumbers + assert type(workerNumbers) == set # noqa: E721 + + for n in workerNumbers: + if n < 0 or n >= N_NUMBERS: + nErrors += 1 + logging.error("Unexpected number {}".format(n)) + continue + + if n in reservedNumbers.keys(): + nErrors += 1 + logging.error("Number {} was already reserved by worker #{}".format( + n, + reservedNumbers[n] + )) + else: + reservedNumbers[n] = i + + file_path = MAKE_PATH(lock_dir, n) + if not os_ops.path_exists(file_path): + nErrors += 1 + logging.error("File {} is not found!".format(file_path)) + continue + + continue + + logging.info("OK. Let's check reservedNumbers!") + + for n in range(N_NUMBERS): + if not (n in reservedNumbers.keys()): + nErrors += 1 + logging.error("Number {} is not reserved!".format(n)) + continue + + file_path = MAKE_PATH(lock_dir, n) + if not os_ops.path_exists(file_path): + nErrors += 1 + logging.error("File {} is not found!".format(file_path)) + continue + + # OK! + continue + + logging.info("Verification is finished! Total error count is {}.".format(nErrors)) + + if nErrors == 0: + logging.info("Root lock-directory [{}] will be deleted.".format( + lock_dir + )) + + for n in range(N_NUMBERS): + file_path = MAKE_PATH(lock_dir, n) + try: + os_ops.rmdir(file_path) + except Exception as e: + nErrors += 1 + logging.error("Cannot delete directory [{}]. Error ({}): {}".format( + file_path, + type(e).__name__, + str(e) + )) + continue + + if os_ops.path_exists(file_path): + nErrors += 1 + logging.error("Directory {} is not deleted!".format(file_path)) + continue + + if nErrors == 0: + try: + os_ops.rmdir(lock_dir) + except Exception as e: + nErrors += 1 + logging.error("Cannot delete directory [{}]. Error ({}): {}".format( + lock_dir, + type(e).__name__, + str(e) + )) + + logging.info("Test is finished! Total error count is {}.".format(nErrors)) + return diff --git a/tests/test_os_ops_remote.py b/tests/test_os_ops_remote.py index 338e49f3..65830218 100755 --- a/tests/test_os_ops_remote.py +++ b/tests/test_os_ops_remote.py @@ -3,7 +3,7 @@ from .helpers.global_data import OsOpsDescrs from .helpers.global_data import OsOperations -from ..testgres import ExecUtilException +from testgres import ExecUtilException import os import pytest diff --git a/tests/test_testgres_common.py b/tests/test_testgres_common.py index e1252de2..a7ddbb27 100644 --- a/tests/test_testgres_common.py +++ b/tests/test_testgres_common.py @@ -1,31 +1,36 @@ +from __future__ import annotations + from .helpers.global_data import PostgresNodeService from .helpers.global_data import PostgresNodeServices from .helpers.global_data import OsOperations from .helpers.global_data import PortManager -from ..testgres.node import PgVer -from ..testgres.node import PostgresNode -from ..testgres.utils import get_pg_version2 -from ..testgres.utils import file_tail -from ..testgres.utils import get_bin_path2 -from ..testgres import ProcessType -from ..testgres import NodeStatus -from ..testgres import IsolationLevel +from testgres.node import PgVer +from testgres.node import PostgresNode +from testgres.node import PostgresNodeLogReader +from testgres.node import PostgresNodeUtils +from testgres.utils import get_pg_version2 +from testgres.utils import file_tail +from testgres.utils import get_bin_path2 +from testgres import ProcessType +from testgres import NodeStatus +from testgres import IsolationLevel +from testgres import NodeApp # New name prevents to collect test-functions in TestgresException and fixes # the problem with pytest warning. -from ..testgres import TestgresException as testgres_TestgresException - -from ..testgres import InitNodeException -from ..testgres import StartNodeException -from ..testgres import QueryException -from ..testgres import ExecUtilException -from ..testgres import TimeoutException -from ..testgres import InvalidOperationException -from ..testgres import BackupException -from ..testgres import ProgrammingError -from ..testgres import scoped_config -from ..testgres import First, Any +from testgres import TestgresException as testgres_TestgresException + +from testgres import InitNodeException +from testgres import StartNodeException +from testgres import QueryException +from testgres import ExecUtilException +from testgres import TimeoutException +from testgres import InvalidOperationException +from testgres import BackupException +from testgres import ProgrammingError +from testgres import scoped_config +from testgres import First, Any from contextlib import contextmanager @@ -620,13 +625,12 @@ def LOCAL__test_lines(): assert (master._logger.is_alive()) finally: # It is a hack code to logging cleanup - logging._acquireLock() - assert logging.Logger.manager is not None - assert C_NODE_NAME in logging.Logger.manager.loggerDict.keys() - logging.Logger.manager.loggerDict.pop(C_NODE_NAME, None) - assert not (C_NODE_NAME in logging.Logger.manager.loggerDict.keys()) - assert not (handler in logging._handlers.values()) - logging._releaseLock() + with logging._lock: + assert logging.Logger.manager is not None + assert C_NODE_NAME in logging.Logger.manager.loggerDict.keys() + logging.Logger.manager.loggerDict.pop(C_NODE_NAME, None) + assert not (C_NODE_NAME in logging.Logger.manager.loggerDict.keys()) + assert not (handler in logging._handlers.values()) # GO HOME! return @@ -678,6 +682,89 @@ def test_psql(self, node_svc: PostgresNodeService): r = node.safe_psql('select 1') # raises! logging.error("node.safe_psql returns [{}]".format(r)) + def test_psql__another_port(self, node_svc: PostgresNodeService): + assert isinstance(node_svc, PostgresNodeService) + with __class__.helper__get_node(node_svc).init() as node1: + with __class__.helper__get_node(node_svc).init() as node2: + node1.start() + node2.start() + assert node1.port != node2.port + assert node1.host == node2.host + + node1.stop() + + logging.info("test table in node2 is creating ...") + node2.safe_psql( + dbname="postgres", + query="create table test (id integer);" + ) + + logging.info("try to find test table through node1.psql ...") + res = node1.psql( + dbname="postgres", + query="select count(*) from pg_class where relname='test'", + host=node2.host, + port=node2.port, + ) + assert (__class__.helper__rm_carriage_returns(res) == (0, b'1\n', b'')) + + def test_psql__another_bad_host(self, node_svc: PostgresNodeService): + assert isinstance(node_svc, PostgresNodeService) + with __class__.helper__get_node(node_svc).init() as node: + logging.info("try to execute node1.psql ...") + res = node.psql( + dbname="postgres", + query="select count(*) from pg_class where relname='test'", + host="DUMMY_HOST_NAME", + port=node.port, + ) + + res2 = __class__.helper__rm_carriage_returns(res) + + assert res2[0] != 0 + assert b"DUMMY_HOST_NAME" in res[2] + + def test_safe_psql__another_port(self, node_svc: PostgresNodeService): + assert isinstance(node_svc, PostgresNodeService) + with __class__.helper__get_node(node_svc).init() as node1: + with __class__.helper__get_node(node_svc).init() as node2: + node1.start() + node2.start() + assert node1.port != node2.port + assert node1.host == node2.host + + node1.stop() + + logging.info("test table in node2 is creating ...") + node2.safe_psql( + dbname="postgres", + query="create table test (id integer);" + ) + + logging.info("try to find test table through node1.psql ...") + res = node1.safe_psql( + dbname="postgres", + query="select count(*) from pg_class where relname='test'", + host=node2.host, + port=node2.port, + ) + assert (__class__.helper__rm_carriage_returns(res) == b'1\n') + + def test_safe_psql__another_bad_host(self, node_svc: PostgresNodeService): + assert isinstance(node_svc, PostgresNodeService) + with __class__.helper__get_node(node_svc).init() as node: + logging.info("try to execute node1.psql ...") + + with pytest.raises(expected_exception=Exception) as x: + node.safe_psql( + dbname="postgres", + query="select count(*) from pg_class where relname='test'", + host="DUMMY_HOST_NAME", + port=node.port, + ) + + assert "DUMMY_HOST_NAME" in str(x.value) + def test_safe_psql__expect_error(self, node_svc: PostgresNodeService): assert isinstance(node_svc, PostgresNodeService) with __class__.helper__get_node(node_svc).init().start() as node: @@ -800,15 +887,55 @@ def test_backup_wrong_xlog_method(self, node_svc: PostgresNodeService): def test_pg_ctl_wait_option(self, node_svc: PostgresNodeService): assert isinstance(node_svc, PostgresNodeService) - C_MAX_ATTEMPTS = 50 - node = __class__.helper__get_node(node_svc) + C_MAX_ATTEMPT = 5 + + nAttempt = 0 + + while True: + if nAttempt == C_MAX_ATTEMPT: + raise Exception("PostgresSQL did not start.") + + nAttempt += 1 + logging.info("------------------------ NODE #{}".format( + nAttempt + )) + + with __class__.helper__get_node(node_svc, port=12345) as node: + if self.impl__test_pg_ctl_wait_option(node_svc, node): + break + continue + + logging.info("OK. Test is passed. Number of attempts is {}".format( + nAttempt + )) + return + + def impl__test_pg_ctl_wait_option( + self, + node_svc: PostgresNodeService, + node: PostgresNode + ) -> None: + assert isinstance(node_svc, PostgresNodeService) + assert isinstance(node, PostgresNode) assert node.status() == NodeStatus.Uninitialized + + C_MAX_ATTEMPTS = 50 + node.init() assert node.status() == NodeStatus.Stopped + + node_log_reader = PostgresNodeLogReader(node, from_beginnig=True) + node.start(wait=False) nAttempt = 0 while True: + if PostgresNodeUtils.delect_port_conflict(node_log_reader): + logging.info("Node port {} conflicted with another PostgreSQL instance.".format( + node.port + )) + return False + if nAttempt == C_MAX_ATTEMPTS: # # [2025-03-11] @@ -867,7 +994,7 @@ def test_pg_ctl_wait_option(self, node_svc: PostgresNodeService): raise Exception("Unexpected node status: {0}.".format(s1)) logging.info("OK. Node is stopped.") - node.cleanup() + return True def test_replicate(self, node_svc: PostgresNodeService): assert isinstance(node_svc, PostgresNodeService) @@ -1378,6 +1505,333 @@ def test_try_to_start_node_after_free_manual_port(self, node_svc: PostgresNodeSe ): node2.start() + def test_node__os_ops(self, node_svc: PostgresNodeService): + assert type(node_svc) == PostgresNodeService # noqa: E721 + + assert node_svc.os_ops is not None + assert isinstance(node_svc.os_ops, OsOperations) + + with PostgresNode(name="node", os_ops=node_svc.os_ops, port_manager=node_svc.port_manager) as node: + # retest + assert node_svc.os_ops is not None + assert isinstance(node_svc.os_ops, OsOperations) + + assert node.os_ops is node_svc.os_ops + # one more time + assert node.os_ops is node_svc.os_ops + + def test_node__port_manager(self, node_svc: PostgresNodeService): + assert type(node_svc) == PostgresNodeService # noqa: E721 + + assert node_svc.port_manager is not None + assert isinstance(node_svc.port_manager, PortManager) + + with PostgresNode(name="node", os_ops=node_svc.os_ops, port_manager=node_svc.port_manager) as node: + # retest + assert node_svc.port_manager is not None + assert isinstance(node_svc.port_manager, PortManager) + + assert node.port_manager is node_svc.port_manager + # one more time + assert node.port_manager is node_svc.port_manager + + def test_node__port_manager_and_explicit_port(self, node_svc: PostgresNodeService): + assert type(node_svc) == PostgresNodeService # noqa: E721 + + assert isinstance(node_svc.os_ops, OsOperations) + assert node_svc.port_manager is not None + assert isinstance(node_svc.port_manager, PortManager) + + port = node_svc.port_manager.reserve_port() + assert type(port) == int # noqa: E721 + + try: + with PostgresNode(name="node", port=port, os_ops=node_svc.os_ops) as node: + # retest + assert isinstance(node_svc.os_ops, OsOperations) + assert node_svc.port_manager is not None + assert isinstance(node_svc.port_manager, PortManager) + + assert node.port_manager is None + assert node.os_ops is node_svc.os_ops + + # one more time + assert node.port_manager is None + assert node.os_ops is node_svc.os_ops + finally: + node_svc.port_manager.release_port(port) + + def test_node__no_port_manager(self, node_svc: PostgresNodeService): + assert type(node_svc) == PostgresNodeService # noqa: E721 + + assert isinstance(node_svc.os_ops, OsOperations) + assert node_svc.port_manager is not None + assert isinstance(node_svc.port_manager, PortManager) + + port = node_svc.port_manager.reserve_port() + assert type(port) == int # noqa: E721 + + try: + with PostgresNode(name="node", port=port, os_ops=node_svc.os_ops, port_manager=None) as node: + # retest + assert isinstance(node_svc.os_ops, OsOperations) + assert node_svc.port_manager is not None + assert isinstance(node_svc.port_manager, PortManager) + + assert node.port_manager is None + assert node.os_ops is node_svc.os_ops + + # one more time + assert node.port_manager is None + assert node.os_ops is node_svc.os_ops + finally: + node_svc.port_manager.release_port(port) + + class tag_rmdirs_protector: + _os_ops: OsOperations + _cwd: str + _old_rmdirs: any + _cwd: str + + def __init__(self, os_ops: OsOperations): + self._os_ops = os_ops + self._cwd = os.path.abspath(os_ops.cwd()) + self._old_rmdirs = os_ops.rmdirs + + def __enter__(self): + assert self._os_ops.rmdirs == self._old_rmdirs + self._os_ops.rmdirs = self.proxy__rmdirs + return self + + def __exit__(self, exc_type, exc_val, exc_tb): + assert self._os_ops.rmdirs == self.proxy__rmdirs + self._os_ops.rmdirs = self._old_rmdirs + return False + + def proxy__rmdirs(self, path, ignore_errors=True): + raise Exception("Call of rmdirs is not expected!") + + def test_node_app__make_empty__base_dir_is_None(self, node_svc: PostgresNodeService): + assert type(node_svc) == PostgresNodeService # noqa: E721 + + assert isinstance(node_svc.os_ops, OsOperations) + assert node_svc.port_manager is not None + assert isinstance(node_svc.port_manager, PortManager) + + tmp_dir = node_svc.os_ops.mkdtemp() + assert tmp_dir is not None + assert type(tmp_dir) == str # noqa: E721 + logging.info("temp directory is [{}]".format(tmp_dir)) + + # ----------- + os_ops = node_svc.os_ops.create_clone() + assert os_ops is not node_svc.os_ops + + # ----------- + with __class__.tag_rmdirs_protector(os_ops): + node_app = NodeApp(test_path=tmp_dir, os_ops=os_ops) + assert node_app.os_ops is os_ops + + with pytest.raises(expected_exception=BaseException) as x: + node_app.make_empty(base_dir=None) + + if type(x.value) == AssertionError: # noqa: E721 + pass + else: + assert type(x.value) == ValueError # noqa: E721 + assert str(x.value) == "Argument 'base_dir' is not defined." + + # ----------- + logging.info("temp directory [{}] is deleting".format(tmp_dir)) + node_svc.os_ops.rmdir(tmp_dir) + + def test_node_app__make_empty__base_dir_is_Empty(self, node_svc: PostgresNodeService): + assert type(node_svc) == PostgresNodeService # noqa: E721 + + assert isinstance(node_svc.os_ops, OsOperations) + assert node_svc.port_manager is not None + assert isinstance(node_svc.port_manager, PortManager) + + tmp_dir = node_svc.os_ops.mkdtemp() + assert tmp_dir is not None + assert type(tmp_dir) == str # noqa: E721 + logging.info("temp directory is [{}]".format(tmp_dir)) + + # ----------- + os_ops = node_svc.os_ops.create_clone() + assert os_ops is not node_svc.os_ops + + # ----------- + with __class__.tag_rmdirs_protector(os_ops): + node_app = NodeApp(test_path=tmp_dir, os_ops=os_ops) + assert node_app.os_ops is os_ops + + with pytest.raises(expected_exception=ValueError) as x: + node_app.make_empty(base_dir="") + + assert str(x.value) == "Argument 'base_dir' is empty." + + # ----------- + logging.info("temp directory [{}] is deleting".format(tmp_dir)) + node_svc.os_ops.rmdir(tmp_dir) + + def test_node_app__make_empty(self, node_svc: PostgresNodeService): + assert type(node_svc) == PostgresNodeService # noqa: E721 + + assert isinstance(node_svc.os_ops, OsOperations) + assert node_svc.port_manager is not None + assert isinstance(node_svc.port_manager, PortManager) + + tmp_dir = node_svc.os_ops.mkdtemp() + assert tmp_dir is not None + assert type(tmp_dir) == str # noqa: E721 + logging.info("temp directory is [{}]".format(tmp_dir)) + + # ----------- + node_app = NodeApp( + test_path=tmp_dir, + os_ops=node_svc.os_ops, + port_manager=node_svc.port_manager + ) + + assert node_app.os_ops is node_svc.os_ops + assert node_app.port_manager is node_svc.port_manager + assert type(node_app.nodes_to_cleanup) == list # noqa: E721 + assert len(node_app.nodes_to_cleanup) == 0 + + node: PostgresNode = None + try: + node = node_app.make_simple("node") + assert node is not None + assert isinstance(node, PostgresNode) + assert node.os_ops is node_svc.os_ops + assert node.port_manager is node_svc.port_manager + + assert type(node_app.nodes_to_cleanup) == list # noqa: E721 + assert len(node_app.nodes_to_cleanup) == 1 + assert node_app.nodes_to_cleanup[0] is node + + node.slow_start() + finally: + if node is not None: + node.stop() + node.release_resources() + + node.cleanup(release_resources=True) + + # ----------- + logging.info("temp directory [{}] is deleting".format(tmp_dir)) + node_svc.os_ops.rmdir(tmp_dir) + + def test_node_app__make_simple__checksum(self, node_svc: PostgresNodeService): + assert type(node_svc) == PostgresNodeService # noqa: E721 + + assert isinstance(node_svc.os_ops, OsOperations) + assert node_svc.port_manager is not None + assert isinstance(node_svc.port_manager, PortManager) + + tmp_dir = node_svc.os_ops.mkdtemp() + assert tmp_dir is not None + assert type(tmp_dir) == str # noqa: E721 + + logging.info("temp directory is [{}]".format(tmp_dir)) + node_app = NodeApp(test_path=tmp_dir, os_ops=node_svc.os_ops) + + C_NODE = "node" + + # ----------- + def LOCAL__test(checksum: bool, initdb_params: typing.Optional[list]): + initdb_params0 = initdb_params + initdb_params0_copy = initdb_params0.copy() if initdb_params0 is not None else None + + with node_app.make_simple(C_NODE, checksum=checksum, initdb_params=initdb_params): + assert initdb_params is initdb_params0 + if initdb_params0 is not None: + assert initdb_params0 == initdb_params0_copy + + assert initdb_params is initdb_params0 + if initdb_params0 is not None: + assert initdb_params0 == initdb_params0_copy + + # ----------- + LOCAL__test(checksum=False, initdb_params=None) + LOCAL__test(checksum=True, initdb_params=None) + + # ----------- + params = [] + LOCAL__test(checksum=False, initdb_params=params) + LOCAL__test(checksum=True, initdb_params=params) + + # ----------- + params = ["--no-sync"] + LOCAL__test(checksum=False, initdb_params=params) + LOCAL__test(checksum=True, initdb_params=params) + + # ----------- + params = ["--data-checksums"] + LOCAL__test(checksum=False, initdb_params=params) + LOCAL__test(checksum=True, initdb_params=params) + + # ----------- + logging.info("temp directory [{}] is deleting".format(tmp_dir)) + node_svc.os_ops.rmdir(tmp_dir) + + def test_node_app__make_empty_with_explicit_port(self, node_svc: PostgresNodeService): + assert type(node_svc) == PostgresNodeService # noqa: E721 + + assert isinstance(node_svc.os_ops, OsOperations) + assert node_svc.port_manager is not None + assert isinstance(node_svc.port_manager, PortManager) + + tmp_dir = node_svc.os_ops.mkdtemp() + assert tmp_dir is not None + assert type(tmp_dir) == str # noqa: E721 + logging.info("temp directory is [{}]".format(tmp_dir)) + + # ----------- + node_app = NodeApp( + test_path=tmp_dir, + os_ops=node_svc.os_ops, + port_manager=node_svc.port_manager + ) + + assert node_app.os_ops is node_svc.os_ops + assert node_app.port_manager is node_svc.port_manager + assert type(node_app.nodes_to_cleanup) == list # noqa: E721 + assert len(node_app.nodes_to_cleanup) == 0 + + port = node_app.port_manager.reserve_port() + assert type(port) == int # noqa: E721 + + node: PostgresNode = None + try: + node = node_app.make_simple("node", port=port) + assert node is not None + assert isinstance(node, PostgresNode) + assert node.os_ops is node_svc.os_ops + assert node.port_manager is None # <--------- + assert node.port == port + assert node._should_free_port == False # noqa: E712 + + assert type(node_app.nodes_to_cleanup) == list # noqa: E721 + assert len(node_app.nodes_to_cleanup) == 1 + assert node_app.nodes_to_cleanup[0] is node + + node.slow_start() + finally: + if node is not None: + node.stop() + node.free_port() + + assert node._port is None + assert not node._should_free_port + + node.cleanup(release_resources=True) + + # ----------- + logging.info("temp directory [{}] is deleting".format(tmp_dir)) + node_svc.os_ops.rmdir(tmp_dir) + @staticmethod def helper__get_node( node_svc: PostgresNodeService, @@ -1395,7 +1849,6 @@ def helper__get_node( return PostgresNode( name, port=port, - conn_params=None, os_ops=node_svc.os_ops, port_manager=port_manager if port is None else None ) diff --git a/tests/test_testgres_local.py b/tests/test_testgres_local.py index 9dbd455b..63e5f37e 100644 --- a/tests/test_testgres_local.py +++ b/tests/test_testgres_local.py @@ -7,21 +7,21 @@ import platform import logging -from .. import testgres +import testgres -from ..testgres import StartNodeException -from ..testgres import ExecUtilException -from ..testgres import NodeApp -from ..testgres import scoped_config -from ..testgres import get_new_node -from ..testgres import get_bin_path -from ..testgres import get_pg_config -from ..testgres import get_pg_version +from testgres import StartNodeException +from testgres import ExecUtilException +from testgres import NodeApp +from testgres import scoped_config +from testgres import get_new_node +from testgres import get_bin_path +from testgres import get_pg_config +from testgres import get_pg_version # NOTE: those are ugly imports -from ..testgres.utils import bound_ports -from ..testgres.utils import PgVer -from ..testgres.node import ProcessProxy +from testgres.utils import bound_ports +from testgres.utils import PgVer +from testgres.node import ProcessProxy def pg_version_ge(version): @@ -158,15 +158,15 @@ def test_child_process_dies(self): def test_upgrade_node(self): old_bin_dir = os.path.dirname(get_bin_path("pg_config")) new_bin_dir = os.path.dirname(get_bin_path("pg_config")) - node_old = get_new_node(prefix='node_old', bin_dir=old_bin_dir) - node_old.init() - node_old.start() - node_old.stop() - node_new = get_new_node(prefix='node_new', bin_dir=new_bin_dir) - node_new.init(cached=False) - res = node_new.upgrade_from(old_node=node_old) - node_new.start() - assert (b'Upgrade Complete' in res) + with get_new_node(prefix='node_old', bin_dir=old_bin_dir) as node_old: + node_old.init() + node_old.start() + node_old.stop() + with get_new_node(prefix='node_new', bin_dir=new_bin_dir) as node_new: + node_new.init(cached=False) + res = node_new.upgrade_from(old_node=node_old) + node_new.start() + assert (b'Upgrade Complete' in res) class tagPortManagerProxy: sm_prev_testgres_reserve_port = None @@ -341,10 +341,10 @@ def test_simple_with_bin_dir(self): bin_dir = node.bin_dir app = NodeApp() - correct_bin_dir = app.make_simple(base_dir=node.base_dir, bin_dir=bin_dir) - correct_bin_dir.slow_start() - correct_bin_dir.safe_psql("SELECT 1;") - correct_bin_dir.stop() + with app.make_simple(base_dir=node.base_dir, bin_dir=bin_dir) as correct_bin_dir: + correct_bin_dir.slow_start() + correct_bin_dir.safe_psql("SELECT 1;") + correct_bin_dir.stop() while True: try: diff --git a/tests/test_testgres_remote.py b/tests/test_testgres_remote.py index e38099b7..6a8d068b 100755 --- a/tests/test_testgres_remote.py +++ b/tests/test_testgres_remote.py @@ -7,16 +7,16 @@ from .helpers.global_data import PostgresNodeService from .helpers.global_data import PostgresNodeServices -from .. import testgres +import testgres -from ..testgres.exceptions import InitNodeException -from ..testgres.exceptions import ExecUtilException +from testgres.exceptions import InitNodeException +from testgres.exceptions import ExecUtilException -from ..testgres.config import scoped_config -from ..testgres.config import testgres_config +from testgres.config import scoped_config +from testgres.config import testgres_config -from ..testgres import get_bin_path -from ..testgres import get_pg_config +from testgres import get_bin_path +from testgres import get_pg_config # NOTE: those are ugly imports @@ -173,7 +173,6 @@ def helper__get_node(name=None): return testgres.PostgresNode( name, - conn_params=None, os_ops=svc.os_ops, port_manager=svc.port_manager) diff --git a/tests/test_utils.py b/tests/test_utils.py index c05bd2fe..39e9dda0 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -2,9 +2,9 @@ from .helpers.global_data import OsOpsDescrs from .helpers.global_data import OsOperations -from ..testgres.utils import parse_pg_version -from ..testgres.utils import get_pg_config2 -from ..testgres import scoped_config +from testgres.utils import parse_pg_version +from testgres.utils import get_pg_config2 +from testgres import scoped_config import pytest import typing pFad - Phonifier reborn

Pfad - The Proxy pFad of © 2024 Garber Painting. All rights reserved.

Note: This service is not intended for secure transactions such as banking, social media, email, or purchasing. Use at your own risk. We assume no liability whatsoever for broken pages.


Alternative Proxies:

Alternative Proxy

pFad Proxy

pFad v3 Proxy

pFad v4 Proxy