Content-Length: 23547 | pFad | http://github.com/postgrespro/testgres/pull/127.patch

thub.com From e8f4da9f63ac2681d7a10996a3eea625378aea6c Mon Sep 17 00:00:00 2001 From: vshepard Date: Wed, 10 Apr 2024 15:36:06 +0200 Subject: [PATCH 01/10] Add port to connection parameters --- testgres/operations/os_ops.py | 3 ++- testgres/operations/remote_ops.py | 3 +++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/testgres/operations/os_ops.py b/testgres/operations/os_ops.py index dd6613cf..236a08c6 100644 --- a/testgres/operations/os_ops.py +++ b/testgres/operations/os_ops.py @@ -10,8 +10,9 @@ class ConnectionParams: - def __init__(self, host='127.0.0.1', ssh_key=None, username=None): + def __init__(self, host='127.0.0.1', port=None, ssh_key=None, username=None): self.host = host + self.port = port self.ssh_key = ssh_key self.username = username diff --git a/testgres/operations/remote_ops.py b/testgres/operations/remote_ops.py index f182768b..34b76500 100644 --- a/testgres/operations/remote_ops.py +++ b/testgres/operations/remote_ops.py @@ -47,10 +47,13 @@ def __init__(self, conn_params: ConnectionParams): self.conn_params = conn_params self.host = conn_params.host self.ssh_key = conn_params.ssh_key + self.port = conn_params.port if self.ssh_key: self.ssh_cmd = ["-i", self.ssh_key] else: self.ssh_cmd = [] + if self.port: + self.ssh_cmd = ["-p", self.port] self.remote = True self.username = conn_params.username or self.get_user() self.add_known_host(self.host) From 80ba614ffc50b42949ed28b097b6b79a3d840f31 Mon Sep 17 00:00:00 2001 From: vshepard Date: Thu, 11 Apr 2024 09:13:23 +0200 Subject: [PATCH 02/10] Add probackup_path to init ProbackupApp --- testgres/plugins/pg_probackup2/pg_probackup2/app.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/testgres/plugins/pg_probackup2/pg_probackup2/app.py b/testgres/plugins/pg_probackup2/pg_probackup2/app.py index 1ab71109..b0e6b960 100644 --- a/testgres/plugins/pg_probackup2/pg_probackup2/app.py +++ b/testgres/plugins/pg_probackup2/pg_probackup2/app.py @@ -43,14 +43,14 @@ def __str__(self): class ProbackupApp: def __init__(self, test_class: unittest.TestCase, - pg_node, pb_log_path, test_env, auto_compress_alg, backup_dir): + pg_node, pb_log_path, test_env, auto_compress_alg, backup_dir, probackup_path=None): self.test_class = test_class self.pg_node = pg_node self.pb_log_path = pb_log_path self.test_env = test_env self.auto_compress_alg = auto_compress_alg self.backup_dir = backup_dir - self.probackup_path = init_params.probackup_path + self.probackup_path = probackup_path or init_params.probackup_path self.probackup_old_path = init_params.probackup_old_path self.remote = init_params.remote self.verbose = init_params.verbose From dc775c20df12e5ad4383bbe551cf2964d72c2bd6 Mon Sep 17 00:00:00 2001 From: vshepard Date: Fri, 12 Apr 2024 11:50:44 +0200 Subject: [PATCH 03/10] Add StrictHostKeyChecking=no for ssh connect --- testgres/operations/remote_ops.py | 18 +++--------------- 1 file changed, 3 insertions(+), 15 deletions(-) diff --git a/testgres/operations/remote_ops.py b/testgres/operations/remote_ops.py index 34b76500..eafdaf2a 100644 --- a/testgres/operations/remote_ops.py +++ b/testgres/operations/remote_ops.py @@ -48,15 +48,13 @@ def __init__(self, conn_params: ConnectionParams): self.host = conn_params.host self.ssh_key = conn_params.ssh_key self.port = conn_params.port + self.ssh_cmd = ["-o StrictHostKeyChecking=no"] if self.ssh_key: - self.ssh_cmd = ["-i", self.ssh_key] - else: - self.ssh_cmd = [] + self.ssh_cmd += ["-i", self.ssh_key] if self.port: - self.ssh_cmd = ["-p", self.port] + self.ssh_cmd += ["-p", self.port] self.remote = True self.username = conn_params.username or self.get_user() - self.add_known_host(self.host) self.tunnel_process = None def __enter__(self): @@ -80,16 +78,6 @@ def close_ssh_tunnel(self): else: print("No active tunnel to close.") - def add_known_host(self, host): - known_hosts_path = os.path.expanduser("~/.ssh/known_hosts") - cmd = 'ssh-keyscan -H %s >> %s' % (host, known_hosts_path) - - try: - subprocess.check_call(cmd, shell=True) - logging.info("Successfully added %s to known_hosts." % host) - except subprocess.CalledProcessError as e: - raise Exception("Failed to add %s to known_hosts. Error: %s" % (host, str(e))) - def exec_command(self, cmd, wait_exit=False, verbose=False, expect_error=False, encoding=None, shell=True, text=False, input=None, stdin=None, stdout=None, stderr=None, get_process=None, timeout=None): From da223402fa95402ce89fcf9bce8ac5df899c7951 Mon Sep 17 00:00:00 2001 From: vshepard Date: Sun, 14 Apr 2024 00:16:49 +0300 Subject: [PATCH 04/10] Add StrictHostKeyChecking=no for ssh connect --- testgres/operations/remote_ops.py | 1 + testgres/utils.py | 2 ++ 2 files changed, 3 insertions(+) diff --git a/testgres/operations/remote_ops.py b/testgres/operations/remote_ops.py index eafdaf2a..c0b7011e 100644 --- a/testgres/operations/remote_ops.py +++ b/testgres/operations/remote_ops.py @@ -16,6 +16,7 @@ raise ImportError("You must have psycopg2 or pg8000 modules installed") from ..exceptions import ExecUtilException +from ..utils import reserve_port from .os_ops import OsOperations, ConnectionParams, get_default_encoding error_markers = [b'error', b'Permission denied', b'fatal', b'No such file or directory'] diff --git a/testgres/utils.py b/testgres/utils.py index 745a2555..633a9b2f 100644 --- a/testgres/utils.py +++ b/testgres/utils.py @@ -74,6 +74,8 @@ def execute_utility(args, logfile=None, verbose=False): # write new log entry if possible if logfile: + if not tconf.os_ops.path_exists(logfile): + tconf.os_ops.touch(logfile) try: tconf.os_ops.write(filename=logfile, data=args, truncate=True) if out: From d08325bec626616d9cdab9ccf92f81f6cd82eb0a Mon Sep 17 00:00:00 2001 From: vshepard Date: Sun, 14 Apr 2024 01:06:05 +0300 Subject: [PATCH 05/10] Add port to scp command --- testgres/operations/remote_ops.py | 9 ++++++--- testgres/utils.py | 2 -- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/testgres/operations/remote_ops.py b/testgres/operations/remote_ops.py index c0b7011e..f5f0a1a5 100644 --- a/testgres/operations/remote_ops.py +++ b/testgres/operations/remote_ops.py @@ -285,8 +285,10 @@ def write(self, filename, data, truncate=False, binary=False, read_and_write=Fal mode = "r+b" if binary else "r+" with tempfile.NamedTemporaryFile(mode=mode, delete=False) as tmp_file: + scp_ssh_cmd = ['-P' if x == '-p' else x for x in self.ssh_cmd] + if not truncate: - scp_cmd = ['scp'] + self.ssh_cmd + [f"{self.username}@{self.host}:{filename}", tmp_file.name] + scp_cmd = ['scp'] + scp_ssh_cmd + [f"{self.username}@{self.host}:{filename}", tmp_file.name] subprocess.run(scp_cmd, check=False) # The file might not exist yet tmp_file.seek(0, os.SEEK_END) @@ -302,11 +304,12 @@ def write(self, filename, data, truncate=False, binary=False, read_and_write=Fal tmp_file.write(data) tmp_file.flush() - scp_cmd = ['scp'] + self.ssh_cmd + [tmp_file.name, f"{self.username}@{self.host}:{filename}"] + # Because in scp we set up port using -P option + scp_cmd = ['scp'] + scp_ssh_cmd + [tmp_file.name, f"{self.username}@{self.host}:{filename}"] subprocess.run(scp_cmd, check=True) remote_directory = os.path.dirname(filename) - mkdir_cmd = ['ssh'] + self.ssh_cmd + [f"{self.username}@{self.host}", f"mkdir -p {remote_directory}"] + mkdir_cmd = ['ssh'] + scp_ssh_cmd + [f"{self.username}@{self.host}", f"mkdir -p {remote_directory}"] subprocess.run(mkdir_cmd, check=True) os.remove(tmp_file.name) diff --git a/testgres/utils.py b/testgres/utils.py index 633a9b2f..745a2555 100644 --- a/testgres/utils.py +++ b/testgres/utils.py @@ -74,8 +74,6 @@ def execute_utility(args, logfile=None, verbose=False): # write new log entry if possible if logfile: - if not tconf.os_ops.path_exists(logfile): - tconf.os_ops.touch(logfile) try: tconf.os_ops.write(filename=logfile, data=args, truncate=True) if out: From f786b8739d09aebccb2270671841461711d46b40 Mon Sep 17 00:00:00 2001 From: vshepard Date: Thu, 25 Apr 2024 09:12:46 +0200 Subject: [PATCH 06/10] PBCKP-781 change defaukt vakue PG_PROBACKUP_S3_HTTPS on ON --- testgres/operations/remote_ops.py | 22 ++++++++++++++-------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/testgres/operations/remote_ops.py b/testgres/operations/remote_ops.py index f5f0a1a5..005d6ac4 100644 --- a/testgres/operations/remote_ops.py +++ b/testgres/operations/remote_ops.py @@ -50,10 +50,10 @@ def __init__(self, conn_params: ConnectionParams): self.ssh_key = conn_params.ssh_key self.port = conn_params.port self.ssh_cmd = ["-o StrictHostKeyChecking=no"] - if self.ssh_key: - self.ssh_cmd += ["-i", self.ssh_key] if self.port: self.ssh_cmd += ["-p", self.port] + if self.ssh_key: + self.ssh_cmd += ["-i", self.ssh_key] self.remote = True self.username = conn_params.username or self.get_user() self.tunnel_process = None @@ -285,6 +285,7 @@ def write(self, filename, data, truncate=False, binary=False, read_and_write=Fal mode = "r+b" if binary else "r+" with tempfile.NamedTemporaryFile(mode=mode, delete=False) as tmp_file: + # Because in scp we set up port using -P option instead -p scp_ssh_cmd = ['-P' if x == '-p' else x for x in self.ssh_cmd] if not truncate: @@ -304,12 +305,11 @@ def write(self, filename, data, truncate=False, binary=False, read_and_write=Fal tmp_file.write(data) tmp_file.flush() - # Because in scp we set up port using -P option scp_cmd = ['scp'] + scp_ssh_cmd + [tmp_file.name, f"{self.username}@{self.host}:{filename}"] subprocess.run(scp_cmd, check=True) - remote_directory = os.path.dirname(filename) - mkdir_cmd = ['ssh'] + scp_ssh_cmd + [f"{self.username}@{self.host}", f"mkdir -p {remote_directory}"] + + mkdir_cmd = ['ssh'] + self.ssh_cmd + [f"{self.username}@{self.host}", f'mkdir -p {remote_directory}'] subprocess.run(mkdir_cmd, check=True) os.remove(tmp_file.name) @@ -387,9 +387,10 @@ def get_process_children(self, pid): # Database control def db_connect(self, dbname, user, password=None, host="localhost", port=5432): """ - Established SSH tunnel and Connects to a PostgreSQL + Establish SSH tunnel and connect to a PostgreSQL database. """ - self.establish_ssh_tunnel(local_port=reserve_port(), remote_port=5432) + self.establish_ssh_tunnel(local_port=port, remote_port=self.conn_params.port) + try: conn = pglib.connect( host=host, @@ -398,6 +399,11 @@ def db_connect(self, dbname, user, password=None, host="localhost", port=5432): user=user, password=password, ) + print("Database connection established successfully.") return conn except Exception as e: - raise Exception(f"Could not connect to the database. Error: {e}") + print(f"Error connecting to the database: {str(e)}") + if self.tunnel_process: + self.tunnel_process.terminate() + print("SSH tunnel closed due to connection failure.") + raise From 48594f67a4fecb6dc2db4620ef143ad3d2348271 Mon Sep 17 00:00:00 2001 From: vshepard Date: Fri, 24 May 2024 22:42:47 +0200 Subject: [PATCH 07/10] Fix ssh command in remote_ops.py --- testgres/node.py | 3 +- testgres/operations/remote_ops.py | 47 ++++++++++++++++++++++--------- 2 files changed, 36 insertions(+), 14 deletions(-) diff --git a/testgres/node.py b/testgres/node.py index d1784cb9..66d211dc 100644 --- a/testgres/node.py +++ b/testgres/node.py @@ -529,7 +529,8 @@ def get_auth_method(t): u"host\treplication\tall\t127.0.0.1/32\t{}\n".format(auth_host), u"host\treplication\tall\t::1/128\t\t{}\n".format(auth_host), u"host\treplication\tall\t{}/24\t\t{}\n".format(subnet_base, auth_host), - u"host\tall\tall\t{}/24\t\t{}\n".format(subnet_base, auth_host) + u"host\tall\tall\t{}/24\t\t{}\n".format(subnet_base, auth_host), + u"host\tall\tall\tall\t{}\n".format(auth_host) ] # yapf: disable # write missing lines diff --git a/testgres/operations/remote_ops.py b/testgres/operations/remote_ops.py index 005d6ac4..cd9ce201 100644 --- a/testgres/operations/remote_ops.py +++ b/testgres/operations/remote_ops.py @@ -1,8 +1,9 @@ -import logging import os +import socket import subprocess import tempfile import platform +import time from ..utils import reserve_port @@ -50,10 +51,10 @@ def __init__(self, conn_params: ConnectionParams): self.ssh_key = conn_params.ssh_key self.port = conn_params.port self.ssh_cmd = ["-o StrictHostKeyChecking=no"] - if self.port: - self.ssh_cmd += ["-p", self.port] if self.ssh_key: self.ssh_cmd += ["-i", self.ssh_key] + if self.port: + self.ssh_cmd += ["-p", self.port] self.remote = True self.username = conn_params.username or self.get_user() self.tunnel_process = None @@ -64,17 +65,36 @@ def __enter__(self): def __exit__(self, exc_type, exc_val, exc_tb): self.close_ssh_tunnel() + @staticmethod + def is_port_open(host, port): + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock: + sock.settimeout(1) # Таймаут для попытки соединения + try: + sock.connect((host, port)) + return True + except socket.error: + return False + def establish_ssh_tunnel(self, local_port, remote_port): """ Establish an SSH tunnel from a local port to a remote PostgreSQL port. """ ssh_cmd = ['-N', '-L', f"{local_port}:localhost:{remote_port}"] self.tunnel_process = self.exec_command(ssh_cmd, get_process=True, timeout=300) + timeout = 10 + start_time = time.time() + while time.time() - start_time < timeout: + if self.is_port_open('localhost', local_port): + print("SSH tunnel established.") + return + time.sleep(0.5) + raise Exception("Failed to establish SSH tunnel within the timeout period.") def close_ssh_tunnel(self): - if hasattr(self, 'tunnel_process'): + if self.tunnel_process: self.tunnel_process.terminate() self.tunnel_process.wait() + print("SSH tunnel closed.") del self.tunnel_process else: print("No active tunnel to close.") @@ -240,9 +260,9 @@ def mkdtemp(self, prefix=None): - prefix (str): The prefix of the temporary directory name. """ if prefix: - command = ["ssh"] + self.ssh_cmd + [f"{self.username}@{self.host}", f"mktemp -d {prefix}XXXXX"] + command = ["ssh" + f"{self.username}@{self.host}"] + self.ssh_cmd + [f"mktemp -d {prefix}XXXXX"] else: - command = ["ssh"] + self.ssh_cmd + [f"{self.username}@{self.host}", "mktemp -d"] + command = ["ssh", f"{self.username}@{self.host}"] + self.ssh_cmd + ["mktemp -d"] result = subprocess.run(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) @@ -285,7 +305,7 @@ def write(self, filename, data, truncate=False, binary=False, read_and_write=Fal mode = "r+b" if binary else "r+" with tempfile.NamedTemporaryFile(mode=mode, delete=False) as tmp_file: - # Because in scp we set up port using -P option instead -p + # Because in scp we set up port using -P option scp_ssh_cmd = ['-P' if x == '-p' else x for x in self.ssh_cmd] if not truncate: @@ -307,9 +327,9 @@ def write(self, filename, data, truncate=False, binary=False, read_and_write=Fal tmp_file.flush() scp_cmd = ['scp'] + scp_ssh_cmd + [tmp_file.name, f"{self.username}@{self.host}:{filename}"] subprocess.run(scp_cmd, check=True) - remote_directory = os.path.dirname(filename) - mkdir_cmd = ['ssh'] + self.ssh_cmd + [f"{self.username}@{self.host}", f'mkdir -p {remote_directory}'] + remote_directory = os.path.dirname(filename) + mkdir_cmd = ['ssh', f"{self.username}@{self.host}"] + self.ssh_cmd + [f"mkdir -p {remote_directory}"] subprocess.run(mkdir_cmd, check=True) os.remove(tmp_file.name) @@ -374,7 +394,7 @@ def get_pid(self): return int(self.exec_command("echo $$", encoding=get_default_encoding())) def get_process_children(self, pid): - command = ["ssh"] + self.ssh_cmd + [f"{self.username}@{self.host}", f"pgrep -P {pid}"] + command = ["ssh", f"{self.username}@{self.host}"] + self.ssh_cmd + [f"pgrep -P {pid}"] result = subprocess.run(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) @@ -389,15 +409,16 @@ def db_connect(self, dbname, user, password=None, host="localhost", port=5432): """ Establish SSH tunnel and connect to a PostgreSQL database. """ - self.establish_ssh_tunnel(local_port=port, remote_port=self.conn_params.port) - + local_port = reserve_port() + self.establish_ssh_tunnel(local_port=local_port, remote_port=port) try: conn = pglib.connect( host=host, - port=port, + port=local_port, database=dbname, user=user, password=password, + timeout=10 ) print("Database connection established successfully.") return conn From fffb23c9b00eac3519230cb8cb9310e130fd7e35 Mon Sep 17 00:00:00 2001 From: vshepard Date: Sun, 2 Jun 2024 23:30:49 +0200 Subject: [PATCH 08/10] Add tunnel_port to remote_ops.py --- testgres/operations/remote_ops.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/testgres/operations/remote_ops.py b/testgres/operations/remote_ops.py index cd9ce201..958dc6c2 100644 --- a/testgres/operations/remote_ops.py +++ b/testgres/operations/remote_ops.py @@ -58,6 +58,7 @@ def __init__(self, conn_params: ConnectionParams): self.remote = True self.username = conn_params.username or self.get_user() self.tunnel_process = None + self.tunnel_port = None def __enter__(self): return self @@ -410,6 +411,7 @@ def db_connect(self, dbname, user, password=None, host="localhost", port=5432): Establish SSH tunnel and connect to a PostgreSQL database. """ local_port = reserve_port() + self.tunnel_port = local_port self.establish_ssh_tunnel(local_port=local_port, remote_port=port) try: conn = pglib.connect( From 1eb9a92fd7fee551fc49939c2048b4a1de571b9f Mon Sep 17 00:00:00 2001 From: vshepard Date: Tue, 4 Jun 2024 20:42:40 +0200 Subject: [PATCH 09/10] Update establish_ssh_tunnel --- setup.py | 2 +- testgres/operations/remote_ops.py | 11 +++++++---- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/setup.py b/setup.py index 412e8823..4f8837c0 100755 --- a/setup.py +++ b/setup.py @@ -27,7 +27,7 @@ readme = f.read() setup( - version='1.10.1', + version='1.10.2', name='testgres', packages=['testgres', 'testgres.operations', 'testgres.helpers'], description='Testing utility for PostgreSQL and its extensions', diff --git a/testgres/operations/remote_ops.py b/testgres/operations/remote_ops.py index 958dc6c2..12f85403 100644 --- a/testgres/operations/remote_ops.py +++ b/testgres/operations/remote_ops.py @@ -76,11 +76,14 @@ def is_port_open(host, port): except socket.error: return False - def establish_ssh_tunnel(self, local_port, remote_port): + def establish_ssh_tunnel(self, local_port, remote_port, host): """ Establish an SSH tunnel from a local port to a remote PostgreSQL port. """ - ssh_cmd = ['-N', '-L', f"{local_port}:localhost:{remote_port}"] + if host != 'localhost': + ssh_cmd = ['-N', '-L', f"localhost:{local_port}:{host}:{remote_port}"] + else: + ssh_cmd = ['-N', '-L', f"{local_port}:{host}:{remote_port}"] self.tunnel_process = self.exec_command(ssh_cmd, get_process=True, timeout=300) timeout = 10 start_time = time.time() @@ -412,10 +415,10 @@ def db_connect(self, dbname, user, password=None, host="localhost", port=5432): """ local_port = reserve_port() self.tunnel_port = local_port - self.establish_ssh_tunnel(local_port=local_port, remote_port=port) + self.establish_ssh_tunnel(local_port=local_port, remote_port=port, host=host) try: conn = pglib.connect( - host=host, + host='localhost', port=local_port, database=dbname, user=user, From 0611d1098fcc97ae1592df9610811a0952f33a61 Mon Sep 17 00:00:00 2001 From: vshepard Date: Fri, 7 Jun 2024 01:01:29 +0200 Subject: [PATCH 10/10] Change pg_hba.conf default params --- testgres/node.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/testgres/node.py b/testgres/node.py index 66d211dc..e94a8e47 100644 --- a/testgres/node.py +++ b/testgres/node.py @@ -530,7 +530,8 @@ def get_auth_method(t): u"host\treplication\tall\t::1/128\t\t{}\n".format(auth_host), u"host\treplication\tall\t{}/24\t\t{}\n".format(subnet_base, auth_host), u"host\tall\tall\t{}/24\t\t{}\n".format(subnet_base, auth_host), - u"host\tall\tall\tall\t{}\n".format(auth_host) + u"host\tall\tall\tall\t{}\n".format(auth_host), + u"host\treplication\tall\tall\t{}\n".format(auth_host) ] # yapf: disable # write missing lines

pFad - (p)hone/(F)rame/(a)nonymizer/(d)eclutterfier! Saves Data!