diff --git a/testgres/config.py b/testgres/config.py index b6c43926..67d467d3 100644 --- a/testgres/config.py +++ b/testgres/config.py @@ -2,6 +2,8 @@ import atexit import copy +import logging +import os import tempfile from contextlib import contextmanager @@ -10,6 +12,10 @@ from .operations.os_ops import OsOperations from .operations.local_ops import LocalOperations +log_level = os.getenv('LOGGING_LEVEL', 'WARNING').upper() +log_format = os.getenv('LOGGING_FORMAT', '%(asctime)s - %(levelname)s - %(message)s') +logging.basicConfig(level=log_level, format=log_format) + class GlobalConfig(object): """ diff --git a/testgres/connection.py b/testgres/connection.py index 49b74844..ccedd135 100644 --- a/testgres/connection.py +++ b/testgres/connection.py @@ -1,4 +1,5 @@ # coding: utf-8 +import logging # we support both pg8000 and psycopg2 try: @@ -110,7 +111,7 @@ def execute(self, query, *args): except ProgrammingError: return None except Exception as e: - print("Error executing query: {}\n {}".format(repr(e), query)) + logging.error("Error executing query: {}\n {}".format(repr(e), query)) return None def close(self): diff --git a/testgres/node.py b/testgres/node.py index e5e8fd5f..b9bf9896 100644 --- a/testgres/node.py +++ b/testgres/node.py @@ -1,5 +1,5 @@ # coding: utf-8 - +import logging import os import random import signal @@ -63,7 +63,6 @@ from .defaults import \ default_dbname, \ - default_username, \ generate_app_name from .exceptions import \ @@ -127,7 +126,7 @@ def __repr__(self): class PostgresNode(object): - def __init__(self, name=None, port=None, base_dir=None, conn_params: ConnectionParams = ConnectionParams(), bin_dir=None, prefix=None): + def __init__(self, name=None, base_dir=None, port=None, conn_params: ConnectionParams = ConnectionParams(), bin_dir=None, prefix=None): """ PostgresNode constructor. @@ -156,9 +155,9 @@ def __init__(self, name=None, port=None, base_dir=None, conn_params: ConnectionP else: self.os_ops = LocalOperations(conn_params) + self.host = self.os_ops.host self.port = port or reserve_port() - self.host = self.os_ops.host self.ssh_key = self.os_ops.ssh_key # defaults for __exit__() @@ -331,22 +330,42 @@ def version(self): """ return self._pg_version - def _try_shutdown(self, max_attempts): + def _try_shutdown(self, max_attempts, with_force=False): attempts = 0 + node_pid = self.pid - # try stopping server N times - while attempts < max_attempts: - try: - self.stop() - break # OK - except ExecUtilException: - pass # one more time - except Exception: - # TODO: probably should kill stray instance - eprint('cannot stop node {}'.format(self.name)) - break - - attempts += 1 + if node_pid > 0: + # try stopping server N times + while attempts < max_attempts: + try: + self.stop() + break # OK + except ExecUtilException: + pass # one more time + except Exception: + eprint('cannot stop node {}'.format(self.name)) + break + + attempts += 1 + + # If force stopping is enabled and PID is valid + if with_force and node_pid != 0: + # If we couldn't stop the node + p_status_output = self.os_ops.exec_command(cmd=f'ps -o pid= -p {node_pid}', shell=True, ignore_errors=True).decode('utf-8') + if self.status() != NodeStatus.Stopped and p_status_output and str(node_pid) in p_status_output: + try: + eprint(f'Force stopping node {self.name} with PID {node_pid}') + self.os_ops.kill(node_pid, signal.SIGKILL, expect_error=False) + except Exception: + # The node has already stopped + pass + + # Check that node stopped - print only column pid without headers + p_status_output = self.os_ops.exec_command(f'ps -o pid= -p {node_pid}', shell=True, ignore_errors=True).decode('utf-8') + if p_status_output and str(node_pid) in p_status_output: + eprint(f'Failed to stop node {self.name}.') + else: + eprint(f'Node {self.name} has been stopped successfully.') def _assign_master(self, master): """NOTE: this is a private method!""" @@ -529,7 +548,9 @@ def get_auth_method(t): u"host\treplication\tall\t127.0.0.1/32\t{}\n".format(auth_host), u"host\treplication\tall\t::1/128\t\t{}\n".format(auth_host), u"host\treplication\tall\t{}/24\t\t{}\n".format(subnet_base, auth_host), - u"host\tall\tall\t{}/24\t\t{}\n".format(subnet_base, auth_host) + u"host\tall\tall\t{}/24\t\t{}\n".format(subnet_base, auth_host), + u"host\tall\tall\tall\t{}\n".format(auth_host), + u"host\treplication\tall\tall\t{}\n".format(auth_host) ] # yapf: disable # write missing lines @@ -683,8 +704,6 @@ def slow_start(self, replica=False, dbname='template1', username=None, max_attem If False, waits for the instance to be in primary mode. Default is False. max_attempts: """ - if not username: - username = default_username() self.start() if replica: @@ -694,7 +713,7 @@ def slow_start(self, replica=False, dbname='template1', username=None, max_attem # Call poll_query_until until the expected value is returned self.poll_query_until(query=query, dbname=dbname, - username=username, + username=username or self.os_ops.username, suppress={InternalError, QueryException, ProgrammingError, @@ -737,11 +756,10 @@ def start(self, params=[], wait=True): if any(len(file) > 1 and 'Is another postmaster already ' 'running on port' in file[1].decode() for file in files): - print("Detected an issue with connecting to port {0}. " - "Trying another port after a 5-second sleep...".format(self.port)) + logging.warning("Detected an issue with connecting to port {0}. " + "Trying another port after a 5-second sleep...".format(self.port)) self.port = reserve_port() - options = {} - options['port'] = str(self.port) + options = {'port': str(self.port)} self.set_auto_conf(options) startup_retries -= 1 time.sleep(5) @@ -915,13 +933,14 @@ def free_port(self): self._should_free_port = False release_port(self.port) - def cleanup(self, max_attempts=3): + def cleanup(self, max_attempts=3, full=False): """ Stop node if needed and remove its data/logs directory. NOTE: take a look at TestgresConfig.node_cleanup_full. Args: max_attempts: how many times should we try to stop()? + full: clean full base dir Returns: This instance of :class:`.PostgresNode`. @@ -930,12 +949,12 @@ def cleanup(self, max_attempts=3): self._try_shutdown(max_attempts) # choose directory to be removed - if testgres_config.node_cleanup_full: + if testgres_config.node_cleanup_full or full: rm_dir = self.base_dir # everything else: rm_dir = self.data_dir # just data, save logs - self.os_ops.rmdirs(rm_dir, ignore_errors=True) + self.os_ops.rmdirs(rm_dir, ignore_errors=False) return self @@ -967,15 +986,13 @@ def psql(self, >>> psql(query='select 3', ON_ERROR_STOP=1) """ - # Set default arguments dbname = dbname or default_dbname() - username = username or default_username() psql_params = [ self._get_bin_path("psql"), "-p", str(self.port), "-h", self.host, - "-U", username, + "-U", username or self.os_ops.username, "-X", # no .psqlrc "-A", # unaligned output "-t", # print rows only @@ -1087,9 +1104,6 @@ def tmpfile(): fname = self.os_ops.mkstemp(prefix=TMP_DUMP) return fname - # Set default arguments - dbname = dbname or default_dbname() - username = username or default_username() filename = filename or tmpfile() _params = [ @@ -1097,8 +1111,8 @@ def tmpfile(): "-p", str(self.port), "-h", self.host, "-f", filename, - "-U", username, - "-d", dbname, + "-U", username or self.os_ops.username, + "-d", dbname or default_dbname(), "-F", format.value ] # yapf: disable @@ -1118,7 +1132,7 @@ def restore(self, filename, dbname=None, username=None): # Set default arguments dbname = dbname or default_dbname() - username = username or default_username() + username = username or self.os_ops.username _params = [ self._get_bin_path("pg_restore"), @@ -1171,7 +1185,6 @@ def poll_query_until(self, assert sleep_time > 0 attempts = 0 while max_attempts == 0 or attempts < max_attempts: - print(f"Pooling {attempts}") try: res = self.execute(dbname=dbname, query=query, @@ -1195,6 +1208,7 @@ def poll_query_until(self, return # done except tuple(suppress or []): + logging.info(f"Trying execute, attempt {attempts + 1}.\nQuery: {query}") pass # we're suppressing them time.sleep(sleep_time) @@ -1388,15 +1402,13 @@ def pgbench(self, if options is None: options = [] - # Set default arguments dbname = dbname or default_dbname() - username = username or default_username() _params = [ self._get_bin_path("pgbench"), "-p", str(self.port), "-h", self.host, - "-U", username, + "-U", username or self.os_ops.username ] + options # yapf: disable # should be the last one @@ -1463,15 +1475,13 @@ def pgbench_run(self, dbname=None, username=None, options=[], **kwargs): >>> pgbench_run(time=10) """ - # Set default arguments dbname = dbname or default_dbname() - username = username or default_username() _params = [ self._get_bin_path("pgbench"), "-p", str(self.port), "-h", self.host, - "-U", username, + "-U", username or self.os_ops.username ] + options # yapf: disable for key, value in iteritems(kwargs): @@ -1639,7 +1649,7 @@ def set_auto_conf(self, options, config='postgresql.auto.conf', rm_options={}): self.os_ops.write(path, auto_conf, truncate=True) - def upgrade_from(self, old_node, options=None): + def upgrade_from(self, old_node, options=None, expect_error=False): """ Upgrade this node from an old node using pg_upgrade. @@ -1667,11 +1677,11 @@ def upgrade_from(self, old_node, options=None): "--old-datadir", old_node.data_dir, "--new-datadir", self.data_dir, "--old-port", str(old_node.port), - "--new-port", str(self.port), + "--new-port", str(self.port) ] upgrade_command += options - return self.os_ops.exec_command(upgrade_command) + return self.os_ops.exec_command(upgrade_command, expect_error=expect_error) def _get_bin_path(self, filename): if self.bin_dir: @@ -1683,19 +1693,26 @@ def _get_bin_path(self, filename): class NodeApp: - def __init__(self, test_path, nodes_to_cleanup, os_ops=LocalOperations()): - self.test_path = test_path - self.nodes_to_cleanup = nodes_to_cleanup + def __init__(self, test_path=None, nodes_to_cleanup=None, os_ops=LocalOperations()): + if test_path: + if os.path.isabs(test_path): + self.test_path = test_path + else: + self.test_path = os.path.join(os_ops.cwd(), test_path) + else: + self.test_path = os_ops.cwd() + self.nodes_to_cleanup = nodes_to_cleanup if nodes_to_cleanup else [] self.os_ops = os_ops def make_empty( self, - base_dir=None): + base_dir=None, + port=None): real_base_dir = os.path.join(self.test_path, base_dir) self.os_ops.rmdirs(real_base_dir, ignore_errors=True) self.os_ops.makedirs(real_base_dir) - node = PostgresNode(base_dir=real_base_dir) + node = PostgresNode(base_dir=real_base_dir, port=port) node.should_rm_dirs = True self.nodes_to_cleanup.append(node) @@ -1704,6 +1721,7 @@ def make_empty( def make_simple( self, base_dir=None, + port=None, set_replication=False, ptrack_enable=False, initdb_params=[], @@ -1711,7 +1729,7 @@ def make_simple( checksum=True): if checksum and '--data-checksums' not in initdb_params: initdb_params.append('--data-checksums') - node = self.make_empty(base_dir) + node = self.make_empty(base_dir, port) node.init( initdb_params=initdb_params, allow_streaming=set_replication) diff --git a/testgres/operations/local_ops.py b/testgres/operations/local_ops.py index ef360d3b..a0a9926d 100644 --- a/testgres/operations/local_ops.py +++ b/testgres/operations/local_ops.py @@ -1,9 +1,11 @@ import getpass +import logging import os import shutil import stat import subprocess import tempfile +import time import psutil @@ -19,13 +21,18 @@ CMD_TIMEOUT_SEC = 60 error_markers = [b'error', b'Permission denied', b'fatal'] +err_out_markers = [b'Failure'] -def has_errors(output): +def has_errors(output=None, error=None): if output: if isinstance(output, str): output = output.encode(get_default_encoding()) - return any(marker in output for marker in error_markers) + return any(marker in output for marker in err_out_markers) + if error: + if isinstance(error, str): + error = error.encode(get_default_encoding()) + return any(marker in error for marker in error_markers) return False @@ -38,7 +45,7 @@ def __init__(self, conn_params=None): self.host = conn_params.host self.ssh_key = None self.remote = False - self.username = conn_params.username or self.get_user() + self.username = conn_params.username or getpass.getuser() @staticmethod def _raise_exec_exception(message, command, exit_code, output): @@ -100,15 +107,16 @@ def _run_command(self, cmd, shell, input, stdin, stdout, stderr, get_process, ti raise ExecUtilException("Command timed out after {} seconds.".format(timeout)) def exec_command(self, cmd, wait_exit=False, verbose=False, expect_error=False, encoding=None, shell=False, - text=False, input=None, stdin=None, stdout=None, stderr=None, get_process=False, timeout=None): + text=False, input=None, stdin=None, stdout=None, stderr=None, get_process=False, timeout=None, + ignore_errors=False): """ Execute a command in a subprocess and handle the output based on the provided parameters. """ process, output, error = self._run_command(cmd, shell, input, stdin, stdout, stderr, get_process, timeout, encoding) if get_process: return process - if process.returncode != 0 or (has_errors(error) and not expect_error): - self._raise_exec_exception('Utility exited with non-zero code. Error `{}`', cmd, process.returncode, error) + if not ignore_errors and ((process.returncode != 0 or has_errors(output=output, error=error)) and not expect_error): + self._raise_exec_exception('Utility exited with non-zero code. Error `{}`', cmd, process.returncode, error or output) if verbose: return process.returncode, output, error @@ -130,10 +138,6 @@ def set_env(self, var_name, var_val): # Check if the directory is already in PATH os.environ[var_name] = var_val - # Get environment variables - def get_user(self): - return self.username or getpass.getuser() - def get_name(self): return os.name @@ -146,8 +150,27 @@ def makedirs(self, path, remove_existing=False): except FileExistsError: pass - def rmdirs(self, path, ignore_errors=True): - return rmtree(path, ignore_errors=ignore_errors) + def rmdirs(self, path, ignore_errors=True, retries=3, delay=1): + """ + Removes a directory and its contents, retrying on failure. + + :param path: Path to the directory. + :param ignore_errors: If True, ignore errors. + :param retries: Number of attempts to remove the directory. + :param delay: Delay between attempts in seconds. + """ + for attempt in range(retries): + try: + rmtree(path, ignore_errors=ignore_errors) + if not os.path.exists(path): + return True + except FileNotFoundError: + return True + except Exception as e: + logging.error(f"Error: Failed to remove directory {path} on attempt {attempt + 1}: {e}") + time.sleep(delay) + logging.error(f"Error: Failed to remove directory {path} after {retries} attempts.") + return False def listdir(self, path): return os.listdir(path) @@ -271,10 +294,10 @@ def remove_file(self, filename): return os.remove(filename) # Processes control - def kill(self, pid, signal): + def kill(self, pid, signal, expect_error=False): # Kill the process cmd = "kill -{} {}".format(signal, pid) - return self.exec_command(cmd) + return self.exec_command(cmd, expect_error=expect_error) def get_pid(self): # Get current process id diff --git a/testgres/operations/os_ops.py b/testgres/operations/os_ops.py index dd6613cf..34242040 100644 --- a/testgres/operations/os_ops.py +++ b/testgres/operations/os_ops.py @@ -1,4 +1,6 @@ +import getpass import locale +import sys try: import psycopg2 as pglib # noqa: F401 @@ -10,20 +12,23 @@ class ConnectionParams: - def __init__(self, host='127.0.0.1', ssh_key=None, username=None): + def __init__(self, host='127.0.0.1', port=None, ssh_key=None, username=None): self.host = host + self.port = port self.ssh_key = ssh_key self.username = username def get_default_encoding(): - return locale.getdefaultlocale()[1] or 'UTF-8' + if not hasattr(locale, 'getencoding'): + locale.getencoding = locale.getpreferredencoding + return locale.getencoding() or 'UTF-8' class OsOperations: def __init__(self, username=None): self.ssh_key = None - self.username = username + self.username = username or getpass.getuser() # Command execution def exec_command(self, cmd, **kwargs): @@ -33,6 +38,13 @@ def exec_command(self, cmd, **kwargs): def environ(self, var_name): raise NotImplementedError() + def cwd(self): + if sys.platform == 'linux': + cmd = 'pwd' + elif sys.platform == 'win32': + cmd = 'cd' + return self.exec_command(cmd).decode().rstrip() + def find_executable(self, executable): raise NotImplementedError() @@ -44,9 +56,8 @@ def set_env(self, var_name, var_val): # Check if the directory is already in PATH raise NotImplementedError() - # Get environment variables def get_user(self): - raise NotImplementedError() + return self.username def get_name(self): raise NotImplementedError() diff --git a/testgres/operations/remote_ops.py b/testgres/operations/remote_ops.py index 01251e1c..20095051 100644 --- a/testgres/operations/remote_ops.py +++ b/testgres/operations/remote_ops.py @@ -1,8 +1,8 @@ -import logging +import getpass import os +import platform import subprocess import tempfile -import platform # we support both pg8000 and psycopg2 try: @@ -44,50 +44,23 @@ def __init__(self, conn_params: ConnectionParams): super().__init__(conn_params.username) self.conn_params = conn_params self.host = conn_params.host + self.port = conn_params.port self.ssh_key = conn_params.ssh_key + self.ssh_args = [] if self.ssh_key: - self.ssh_cmd = ["-i", self.ssh_key] - else: - self.ssh_cmd = [] + self.ssh_args += ["-i", self.ssh_key] + if self.port: + self.ssh_args += ["-p", self.port] self.remote = True - self.username = conn_params.username or self.get_user() - self.add_known_host(self.host) - self.tunnel_process = None + self.username = conn_params.username or getpass.getuser() + self.ssh_dest = f"{self.username}@{self.host}" if conn_params.username else self.host def __enter__(self): return self - def __exit__(self, exc_type, exc_val, exc_tb): - self.close_ssh_tunnel() - - def establish_ssh_tunnel(self, local_port, remote_port): - """ - Establish an SSH tunnel from a local port to a remote PostgreSQL port. - """ - ssh_cmd = ['-N', '-L', f"{local_port}:localhost:{remote_port}"] - self.tunnel_process = self.exec_command(ssh_cmd, get_process=True, timeout=300) - - def close_ssh_tunnel(self): - if hasattr(self, 'tunnel_process'): - self.tunnel_process.terminate() - self.tunnel_process.wait() - del self.tunnel_process - else: - print("No active tunnel to close.") - - def add_known_host(self, host): - known_hosts_path = os.path.expanduser("~/.ssh/known_hosts") - cmd = 'ssh-keyscan -H %s >> %s' % (host, known_hosts_path) - - try: - subprocess.check_call(cmd, shell=True) - logging.info("Successfully added %s to known_hosts." % host) - except subprocess.CalledProcessError as e: - raise Exception("Failed to add %s to known_hosts. Error: %s" % (host, str(e))) - def exec_command(self, cmd, wait_exit=False, verbose=False, expect_error=False, encoding=None, shell=True, text=False, input=None, stdin=None, stdout=None, - stderr=None, get_process=None, timeout=None): + stderr=None, get_process=None, timeout=None, ignore_errors=False): """ Execute a command in the SSH session. Args: @@ -95,9 +68,9 @@ def exec_command(self, cmd, wait_exit=False, verbose=False, expect_error=False, """ ssh_cmd = [] if isinstance(cmd, str): - ssh_cmd = ['ssh', f"{self.username}@{self.host}"] + self.ssh_cmd + [cmd] + ssh_cmd = ['ssh', self.ssh_dest] + self.ssh_args + [cmd] elif isinstance(cmd, list): - ssh_cmd = ['ssh', f"{self.username}@{self.host}"] + self.ssh_cmd + cmd + ssh_cmd = ['ssh', self.ssh_dest] + self.ssh_args + cmd process = subprocess.Popen(ssh_cmd, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE) if get_process: return process @@ -120,10 +93,12 @@ def exec_command(self, cmd, wait_exit=False, verbose=False, expect_error=False, if not error: error_found = 0 else: + error = normalize_error(error) error_found = exit_status != 0 or any( - marker in error for marker in [b'error', b'Permission denied', b'fatal', b'No such file or directory']) + marker in error for marker in ['error', 'Permission denied', 'fatal', 'No such file or directory'] + ) - if error_found: + if not ignore_errors and error_found: if isinstance(error, bytes): message = b"Utility exited with non-zero code. Error: " + error else: @@ -172,10 +147,6 @@ def set_env(self, var_name: str, var_val: str): """ return self.exec_command("export {}={}".format(var_name, var_val)) - # Get environment variables - def get_user(self): - return self.exec_command("echo $USER", encoding=get_default_encoding()).strip() - def get_name(self): cmd = 'python3 -c "import os; print(os.name)"' return self.exec_command(cmd, encoding=get_default_encoding()).strip() @@ -246,9 +217,9 @@ def mkdtemp(self, prefix=None): - prefix (str): The prefix of the temporary directory name. """ if prefix: - command = ["ssh"] + self.ssh_cmd + [f"{self.username}@{self.host}", f"mktemp -d {prefix}XXXXX"] + command = ["ssh"] + self.ssh_args + [self.ssh_dest, f"mktemp -d {prefix}XXXXX"] else: - command = ["ssh"] + self.ssh_cmd + [f"{self.username}@{self.host}", "mktemp -d"] + command = ["ssh"] + self.ssh_args + [self.ssh_dest, "mktemp -d"] result = subprocess.run(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) @@ -291,8 +262,11 @@ def write(self, filename, data, truncate=False, binary=False, read_and_write=Fal mode = "r+b" if binary else "r+" with tempfile.NamedTemporaryFile(mode=mode, delete=False) as tmp_file: + # For scp the port is specified by a "-P" option + scp_args = ['-P' if x == '-p' else x for x in self.ssh_args] + if not truncate: - scp_cmd = ['scp'] + self.ssh_cmd + [f"{self.username}@{self.host}:{filename}", tmp_file.name] + scp_cmd = ['scp'] + scp_args + [f"{self.ssh_dest}:{filename}", tmp_file.name] subprocess.run(scp_cmd, check=False) # The file might not exist yet tmp_file.seek(0, os.SEEK_END) @@ -308,11 +282,11 @@ def write(self, filename, data, truncate=False, binary=False, read_and_write=Fal tmp_file.write(data) tmp_file.flush() - scp_cmd = ['scp'] + self.ssh_cmd + [tmp_file.name, f"{self.username}@{self.host}:{filename}"] + scp_cmd = ['scp'] + scp_args + [tmp_file.name, f"{self.ssh_dest}:{filename}"] subprocess.run(scp_cmd, check=True) remote_directory = os.path.dirname(filename) - mkdir_cmd = ['ssh'] + self.ssh_cmd + [f"{self.username}@{self.host}", f"mkdir -p {remote_directory}"] + mkdir_cmd = ['ssh'] + self.ssh_args + [self.ssh_dest, f"mkdir -p {remote_directory}"] subprocess.run(mkdir_cmd, check=True) os.remove(tmp_file.name) @@ -377,7 +351,7 @@ def get_pid(self): return int(self.exec_command("echo $$", encoding=get_default_encoding())) def get_process_children(self, pid): - command = ["ssh"] + self.ssh_cmd + [f"{self.username}@{self.host}", f"pgrep -P {pid}"] + command = ["ssh"] + self.ssh_args + [self.ssh_dest, f"pgrep -P {pid}"] result = subprocess.run(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) @@ -389,18 +363,17 @@ def get_process_children(self, pid): # Database control def db_connect(self, dbname, user, password=None, host="localhost", port=5432): - """ - Established SSH tunnel and Connects to a PostgreSQL - """ - self.establish_ssh_tunnel(local_port=port, remote_port=5432) - try: - conn = pglib.connect( - host=host, - port=port, - database=dbname, - user=user, - password=password, - ) - return conn - except Exception as e: - raise Exception(f"Could not connect to the database. Error: {e}") + conn = pglib.connect( + host=host, + port=port, + database=dbname, + user=user, + password=password, + ) + return conn + + +def normalize_error(error): + if isinstance(error, bytes): + return error.decode() + return error diff --git a/testgres/plugins/pg_probackup2/pg_probackup2/app.py b/testgres/plugins/pg_probackup2/pg_probackup2/app.py index 1a4ca9e7..e656b66d 100644 --- a/testgres/plugins/pg_probackup2/pg_probackup2/app.py +++ b/testgres/plugins/pg_probackup2/pg_probackup2/app.py @@ -1,6 +1,7 @@ import contextlib import importlib import json +import logging import os import re import subprocess @@ -43,14 +44,14 @@ def __str__(self): class ProbackupApp: def __init__(self, test_class: unittest.TestCase, - pg_node, pb_log_path, test_env, auto_compress_alg, backup_dir): + pg_node, pb_log_path, test_env, auto_compress_alg, backup_dir, probackup_path=None): self.test_class = test_class self.pg_node = pg_node self.pb_log_path = pb_log_path self.test_env = test_env self.auto_compress_alg = auto_compress_alg self.backup_dir = backup_dir - self.probackup_path = init_params.probackup_path + self.probackup_path = probackup_path or init_params.probackup_path self.probackup_old_path = init_params.probackup_old_path self.remote = init_params.remote self.verbose = init_params.verbose @@ -74,7 +75,7 @@ def run(self, command, gdb=False, old_binary=False, return_id=True, env=None, command = [command[0], *self.backup_dir.pb_args, *command[1:]] if not self.probackup_old_path and old_binary: - print('PGPROBACKUPBIN_OLD is not set') + logging.error('PGPROBACKUPBIN_OLD is not set') exit(1) if old_binary: @@ -107,12 +108,11 @@ def run(self, command, gdb=False, old_binary=False, return_id=True, env=None, return GDBobj(cmdline, self.test_class) try: - result = None if type(gdb) is tuple and gdb[0] == 'suspend': # special test flow for manually debug probackup gdb_port = gdb[1] cmdline = ['gdbserver'] + ['localhost:' + str(gdb_port)] + cmdline - print("pg_probackup gdb suspended, waiting gdb connection on localhost:{0}".format(gdb_port)) + logging.warning("pg_probackup gdb suspended, waiting gdb connection on localhost:{0}".format(gdb_port)) start_time = time.time() self.test_class.output = subprocess.check_output( @@ -233,7 +233,7 @@ def backup_node( if options is None: options = [] if not node and not data_dir: - print('You must provide ether node or data_dir for backup') + logging.error('You must provide ether node or data_dir for backup') exit(1) if not datname: @@ -388,6 +388,7 @@ def catchup_node( backup_mode, source_pgdata, destination_node, options=None, remote_host='localhost', + remote_port=None, expect_error=False, gdb=False ): @@ -401,7 +402,9 @@ def catchup_node( '--destination-pgdata={0}'.format(destination_node.data_dir) ] if self.remote: - cmd_list += ['--remote-proto=ssh', '--remote-host=%s' % remote_host] + cmd_list += ['--remote-proto=ssh', f'--remote-host={remote_host}'] + if remote_port: + cmd_list.append(f'--remote-port={remote_port}') if self.verbose: cmd_list += [ '--log-level-file=VERBOSE', @@ -499,7 +502,7 @@ def show( if i == '': backup_record_split.remove(i) if len(header_split) != len(backup_record_split): - print(warning.format( + logging.error(warning.format( header=header, body=body, header_split=header_split, body_split=backup_record_split) @@ -578,7 +581,7 @@ def show_archive( else: show_splitted = self.run(cmd_list + options, old_binary=old_binary, expect_error=expect_error).splitlines() - print(show_splitted) + logging.error(show_splitted) exit(1) def validate( @@ -702,9 +705,13 @@ def set_archiving( if overwrite: archive_command += ' --overwrite' - archive_command += ' --log-level-console=VERBOSE' - archive_command += ' -j 5' - archive_command += ' --batch-size 10' + if init_params.major_version > 2: + archive_command += ' --log-level-console=trace' + else: + archive_command += ' --log-level-console=VERBOSE' + archive_command += ' -j 5' + archive_command += ' --batch-size 10' + archive_command += ' --no-sync' if archive_timeout: @@ -766,7 +773,7 @@ def load_backup_class(fs_type): if fs_type: implementation = fs_type - print("Using ", implementation) + logging.info("Using ", implementation) module_name, class_name = implementation.rsplit(sep='.', maxsplit=1) module = importlib.import_module(module_name) diff --git a/testgres/plugins/pg_probackup2/pg_probackup2/init_helpers.py b/testgres/plugins/pg_probackup2/pg_probackup2/init_helpers.py index 73731a6e..2d19e980 100644 --- a/testgres/plugins/pg_probackup2/pg_probackup2/init_helpers.py +++ b/testgres/plugins/pg_probackup2/pg_probackup2/init_helpers.py @@ -1,3 +1,4 @@ +import logging from functools import reduce import getpass import os @@ -31,7 +32,7 @@ cached_initdb_dir=False, node_cleanup_full=delete_logs) except Exception as e: - print("Can't configure testgres: {0}".format(e)) + logging.warning("Can't configure testgres: {0}".format(e)) class Init(object): @@ -104,7 +105,7 @@ def __init__(self): if os.path.isfile(probackup_path_tmp): if not os.access(probackup_path_tmp, os.X_OK): - print('{0} is not an executable file'.format( + logging.warning('{0} is not an executable file'.format( probackup_path_tmp)) else: self.probackup_path = probackup_path_tmp @@ -114,13 +115,13 @@ def __init__(self): if os.path.isfile(probackup_path_tmp): if not os.access(probackup_path_tmp, os.X_OK): - print('{0} is not an executable file'.format( + logging.warning('{0} is not an executable file'.format( probackup_path_tmp)) else: self.probackup_path = probackup_path_tmp if not self.probackup_path: - print('pg_probackup binary is not found') + logging.error('pg_probackup binary is not found') exit(1) if os.name == 'posix': @@ -207,7 +208,7 @@ def __init__(self): if self.probackup_version.split('.')[0].isdigit(): self.major_version = int(self.probackup_version.split('.')[0]) else: - print('Can\'t process pg_probackup version \"{}\": the major version is expected to be a number'.format(self.probackup_version)) + logging.error('Can\'t process pg_probackup version \"{}\": the major version is expected to be a number'.format(self.probackup_version)) sys.exit(1) def test_env(self): diff --git a/testgres/plugins/pg_probackup2/pg_probackup2/tests/basic_test.py b/testgres/plugins/pg_probackup2/pg_probackup2/tests/basic_test.py index f5a82d38..b63531ec 100644 --- a/testgres/plugins/pg_probackup2/pg_probackup2/tests/basic_test.py +++ b/testgres/plugins/pg_probackup2/pg_probackup2/tests/basic_test.py @@ -1,3 +1,4 @@ +import logging import os import shutil import unittest @@ -14,7 +15,7 @@ def get_module_and_function_name(test_id): module_name = test_id.split('.')[-2] fname = test_id.split('.')[-1] except IndexError: - print(f"Couldn't get module name and function name from test_id: `{test_id}`") + logging.warning(f"Couldn't get module name and function name from test_id: `{test_id}`") module_name, fname = test_id.split('(')[1].split('.')[1], test_id.split('(')[0] return module_name, fname diff --git a/testgres/utils.py b/testgres/utils.py index 745a2555..a4ee7877 100644 --- a/testgres/utils.py +++ b/testgres/utils.py @@ -228,7 +228,6 @@ def eprint(*args, **kwargs): """ Print stuff to stderr. """ - print(*args, file=sys.stderr, **kwargs) pFad - Phonifier reborn

Pfad - The Proxy pFad of © 2024 Garber Painting. All rights reserved.

Note: This service is not intended for secure transactions such as banking, social media, email, or purchasing. Use at your own risk. We assume no liability whatsoever for broken pages.


Alternative Proxies:

Alternative Proxy

pFad Proxy

pFad v3 Proxy

pFad v4 Proxy