From e8f4da9f63ac2681d7a10996a3eea625378aea6c Mon Sep 17 00:00:00 2001 From: vshepard Date: Wed, 10 Apr 2024 15:36:06 +0200 Subject: [PATCH 01/10] Add port to connection parameters --- testgres/operations/os_ops.py | 3 ++- testgres/operations/remote_ops.py | 3 +++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/testgres/operations/os_ops.py b/testgres/operations/os_ops.py index dd6613cf..236a08c6 100644 --- a/testgres/operations/os_ops.py +++ b/testgres/operations/os_ops.py @@ -10,8 +10,9 @@ class ConnectionParams: - def __init__(self, host='127.0.0.1', ssh_key=None, username=None): + def __init__(self, host='127.0.0.1', port=None, ssh_key=None, username=None): self.host = host + self.port = port self.ssh_key = ssh_key self.username = username diff --git a/testgres/operations/remote_ops.py b/testgres/operations/remote_ops.py index f182768b..34b76500 100644 --- a/testgres/operations/remote_ops.py +++ b/testgres/operations/remote_ops.py @@ -47,10 +47,13 @@ def __init__(self, conn_params: ConnectionParams): self.conn_params = conn_params self.host = conn_params.host self.ssh_key = conn_params.ssh_key + self.port = conn_params.port if self.ssh_key: self.ssh_cmd = ["-i", self.ssh_key] else: self.ssh_cmd = [] + if self.port: + self.ssh_cmd = ["-p", self.port] self.remote = True self.username = conn_params.username or self.get_user() self.add_known_host(self.host) From 80ba614ffc50b42949ed28b097b6b79a3d840f31 Mon Sep 17 00:00:00 2001 From: vshepard Date: Thu, 11 Apr 2024 09:13:23 +0200 Subject: [PATCH 02/10] Add probackup_path to init ProbackupApp --- testgres/plugins/pg_probackup2/pg_probackup2/app.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/testgres/plugins/pg_probackup2/pg_probackup2/app.py b/testgres/plugins/pg_probackup2/pg_probackup2/app.py index 1ab71109..b0e6b960 100644 --- a/testgres/plugins/pg_probackup2/pg_probackup2/app.py +++ b/testgres/plugins/pg_probackup2/pg_probackup2/app.py @@ -43,14 +43,14 @@ def __str__(self): class ProbackupApp: def __init__(self, test_class: unittest.TestCase, - pg_node, pb_log_path, test_env, auto_compress_alg, backup_dir): + pg_node, pb_log_path, test_env, auto_compress_alg, backup_dir, probackup_path=None): self.test_class = test_class self.pg_node = pg_node self.pb_log_path = pb_log_path self.test_env = test_env self.auto_compress_alg = auto_compress_alg self.backup_dir = backup_dir - self.probackup_path = init_params.probackup_path + self.probackup_path = probackup_path or init_params.probackup_path self.probackup_old_path = init_params.probackup_old_path self.remote = init_params.remote self.verbose = init_params.verbose From dc775c20df12e5ad4383bbe551cf2964d72c2bd6 Mon Sep 17 00:00:00 2001 From: vshepard Date: Fri, 12 Apr 2024 11:50:44 +0200 Subject: [PATCH 03/10] Add StrictHostKeyChecking=no for ssh connect --- testgres/operations/remote_ops.py | 18 +++--------------- 1 file changed, 3 insertions(+), 15 deletions(-) diff --git a/testgres/operations/remote_ops.py b/testgres/operations/remote_ops.py index 34b76500..eafdaf2a 100644 --- a/testgres/operations/remote_ops.py +++ b/testgres/operations/remote_ops.py @@ -48,15 +48,13 @@ def __init__(self, conn_params: ConnectionParams): self.host = conn_params.host self.ssh_key = conn_params.ssh_key self.port = conn_params.port + self.ssh_cmd = ["-o StrictHostKeyChecking=no"] if self.ssh_key: - self.ssh_cmd = ["-i", self.ssh_key] - else: - self.ssh_cmd = [] + self.ssh_cmd += ["-i", self.ssh_key] if self.port: - self.ssh_cmd = ["-p", self.port] + self.ssh_cmd += ["-p", self.port] self.remote = True self.username = conn_params.username or self.get_user() - self.add_known_host(self.host) self.tunnel_process = None def __enter__(self): @@ -80,16 +78,6 @@ def close_ssh_tunnel(self): else: print("No active tunnel to close.") - def add_known_host(self, host): - known_hosts_path = os.path.expanduser("~/.ssh/known_hosts") - cmd = 'ssh-keyscan -H %s >> %s' % (host, known_hosts_path) - - try: - subprocess.check_call(cmd, shell=True) - logging.info("Successfully added %s to known_hosts." % host) - except subprocess.CalledProcessError as e: - raise Exception("Failed to add %s to known_hosts. Error: %s" % (host, str(e))) - def exec_command(self, cmd, wait_exit=False, verbose=False, expect_error=False, encoding=None, shell=True, text=False, input=None, stdin=None, stdout=None, stderr=None, get_process=None, timeout=None): From da223402fa95402ce89fcf9bce8ac5df899c7951 Mon Sep 17 00:00:00 2001 From: vshepard Date: Sun, 14 Apr 2024 00:16:49 +0300 Subject: [PATCH 04/10] Add StrictHostKeyChecking=no for ssh connect --- testgres/operations/remote_ops.py | 1 + testgres/utils.py | 2 ++ 2 files changed, 3 insertions(+) diff --git a/testgres/operations/remote_ops.py b/testgres/operations/remote_ops.py index eafdaf2a..c0b7011e 100644 --- a/testgres/operations/remote_ops.py +++ b/testgres/operations/remote_ops.py @@ -16,6 +16,7 @@ raise ImportError("You must have psycopg2 or pg8000 modules installed") from ..exceptions import ExecUtilException +from ..utils import reserve_port from .os_ops import OsOperations, ConnectionParams, get_default_encoding error_markers = [b'error', b'Permission denied', b'fatal', b'No such file or directory'] diff --git a/testgres/utils.py b/testgres/utils.py index 745a2555..633a9b2f 100644 --- a/testgres/utils.py +++ b/testgres/utils.py @@ -74,6 +74,8 @@ def execute_utility(args, logfile=None, verbose=False): # write new log entry if possible if logfile: + if not tconf.os_ops.path_exists(logfile): + tconf.os_ops.touch(logfile) try: tconf.os_ops.write(filename=logfile, data=args, truncate=True) if out: From d08325bec626616d9cdab9ccf92f81f6cd82eb0a Mon Sep 17 00:00:00 2001 From: vshepard Date: Sun, 14 Apr 2024 01:06:05 +0300 Subject: [PATCH 05/10] Add port to scp command --- testgres/operations/remote_ops.py | 9 ++++++--- testgres/utils.py | 2 -- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/testgres/operations/remote_ops.py b/testgres/operations/remote_ops.py index c0b7011e..f5f0a1a5 100644 --- a/testgres/operations/remote_ops.py +++ b/testgres/operations/remote_ops.py @@ -285,8 +285,10 @@ def write(self, filename, data, truncate=False, binary=False, read_and_write=Fal mode = "r+b" if binary else "r+" with tempfile.NamedTemporaryFile(mode=mode, delete=False) as tmp_file: + scp_ssh_cmd = ['-P' if x == '-p' else x for x in self.ssh_cmd] + if not truncate: - scp_cmd = ['scp'] + self.ssh_cmd + [f"{self.username}@{self.host}:{filename}", tmp_file.name] + scp_cmd = ['scp'] + scp_ssh_cmd + [f"{self.username}@{self.host}:{filename}", tmp_file.name] subprocess.run(scp_cmd, check=False) # The file might not exist yet tmp_file.seek(0, os.SEEK_END) @@ -302,11 +304,12 @@ def write(self, filename, data, truncate=False, binary=False, read_and_write=Fal tmp_file.write(data) tmp_file.flush() - scp_cmd = ['scp'] + self.ssh_cmd + [tmp_file.name, f"{self.username}@{self.host}:{filename}"] + # Because in scp we set up port using -P option + scp_cmd = ['scp'] + scp_ssh_cmd + [tmp_file.name, f"{self.username}@{self.host}:{filename}"] subprocess.run(scp_cmd, check=True) remote_directory = os.path.dirname(filename) - mkdir_cmd = ['ssh'] + self.ssh_cmd + [f"{self.username}@{self.host}", f"mkdir -p {remote_directory}"] + mkdir_cmd = ['ssh'] + scp_ssh_cmd + [f"{self.username}@{self.host}", f"mkdir -p {remote_directory}"] subprocess.run(mkdir_cmd, check=True) os.remove(tmp_file.name) diff --git a/testgres/utils.py b/testgres/utils.py index 633a9b2f..745a2555 100644 --- a/testgres/utils.py +++ b/testgres/utils.py @@ -74,8 +74,6 @@ def execute_utility(args, logfile=None, verbose=False): # write new log entry if possible if logfile: - if not tconf.os_ops.path_exists(logfile): - tconf.os_ops.touch(logfile) try: tconf.os_ops.write(filename=logfile, data=args, truncate=True) if out: From f786b8739d09aebccb2270671841461711d46b40 Mon Sep 17 00:00:00 2001 From: vshepard Date: Thu, 25 Apr 2024 09:12:46 +0200 Subject: [PATCH 06/10] PBCKP-781 change defaukt vakue PG_PROBACKUP_S3_HTTPS on ON --- testgres/operations/remote_ops.py | 22 ++++++++++++++-------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/testgres/operations/remote_ops.py b/testgres/operations/remote_ops.py index f5f0a1a5..005d6ac4 100644 --- a/testgres/operations/remote_ops.py +++ b/testgres/operations/remote_ops.py @@ -50,10 +50,10 @@ def __init__(self, conn_params: ConnectionParams): self.ssh_key = conn_params.ssh_key self.port = conn_params.port self.ssh_cmd = ["-o StrictHostKeyChecking=no"] - if self.ssh_key: - self.ssh_cmd += ["-i", self.ssh_key] if self.port: self.ssh_cmd += ["-p", self.port] + if self.ssh_key: + self.ssh_cmd += ["-i", self.ssh_key] self.remote = True self.username = conn_params.username or self.get_user() self.tunnel_process = None @@ -285,6 +285,7 @@ def write(self, filename, data, truncate=False, binary=False, read_and_write=Fal mode = "r+b" if binary else "r+" with tempfile.NamedTemporaryFile(mode=mode, delete=False) as tmp_file: + # Because in scp we set up port using -P option instead -p scp_ssh_cmd = ['-P' if x == '-p' else x for x in self.ssh_cmd] if not truncate: @@ -304,12 +305,11 @@ def write(self, filename, data, truncate=False, binary=False, read_and_write=Fal tmp_file.write(data) tmp_file.flush() - # Because in scp we set up port using -P option scp_cmd = ['scp'] + scp_ssh_cmd + [tmp_file.name, f"{self.username}@{self.host}:{filename}"] subprocess.run(scp_cmd, check=True) - remote_directory = os.path.dirname(filename) - mkdir_cmd = ['ssh'] + scp_ssh_cmd + [f"{self.username}@{self.host}", f"mkdir -p {remote_directory}"] + + mkdir_cmd = ['ssh'] + self.ssh_cmd + [f"{self.username}@{self.host}", f'mkdir -p {remote_directory}'] subprocess.run(mkdir_cmd, check=True) os.remove(tmp_file.name) @@ -387,9 +387,10 @@ def get_process_children(self, pid): # Database control def db_connect(self, dbname, user, password=None, host="localhost", port=5432): """ - Established SSH tunnel and Connects to a PostgreSQL + Establish SSH tunnel and connect to a PostgreSQL database. """ - self.establish_ssh_tunnel(local_port=reserve_port(), remote_port=5432) + self.establish_ssh_tunnel(local_port=port, remote_port=self.conn_params.port) + try: conn = pglib.connect( host=host, @@ -398,6 +399,11 @@ def db_connect(self, dbname, user, password=None, host="localhost", port=5432): user=user, password=password, ) + print("Database connection established successfully.") return conn except Exception as e: - raise Exception(f"Could not connect to the database. Error: {e}") + print(f"Error connecting to the database: {str(e)}") + if self.tunnel_process: + self.tunnel_process.terminate() + print("SSH tunnel closed due to connection failure.") + raise From 48594f67a4fecb6dc2db4620ef143ad3d2348271 Mon Sep 17 00:00:00 2001 From: vshepard Date: Fri, 24 May 2024 22:42:47 +0200 Subject: [PATCH 07/10] Fix ssh command in remote_ops.py --- testgres/node.py | 3 +- testgres/operations/remote_ops.py | 47 ++++++++++++++++++++++--------- 2 files changed, 36 insertions(+), 14 deletions(-) diff --git a/testgres/node.py b/testgres/node.py index d1784cb9..66d211dc 100644 --- a/testgres/node.py +++ b/testgres/node.py @@ -529,7 +529,8 @@ def get_auth_method(t): u"host\treplication\tall\t127.0.0.1/32\t{}\n".format(auth_host), u"host\treplication\tall\t::1/128\t\t{}\n".format(auth_host), u"host\treplication\tall\t{}/24\t\t{}\n".format(subnet_base, auth_host), - u"host\tall\tall\t{}/24\t\t{}\n".format(subnet_base, auth_host) + u"host\tall\tall\t{}/24\t\t{}\n".format(subnet_base, auth_host), + u"host\tall\tall\tall\t{}\n".format(auth_host) ] # yapf: disable # write missing lines diff --git a/testgres/operations/remote_ops.py b/testgres/operations/remote_ops.py index 005d6ac4..cd9ce201 100644 --- a/testgres/operations/remote_ops.py +++ b/testgres/operations/remote_ops.py @@ -1,8 +1,9 @@ -import logging import os +import socket import subprocess import tempfile import platform +import time from ..utils import reserve_port @@ -50,10 +51,10 @@ def __init__(self, conn_params: ConnectionParams): self.ssh_key = conn_params.ssh_key self.port = conn_params.port self.ssh_cmd = ["-o StrictHostKeyChecking=no"] - if self.port: - self.ssh_cmd += ["-p", self.port] if self.ssh_key: self.ssh_cmd += ["-i", self.ssh_key] + if self.port: + self.ssh_cmd += ["-p", self.port] self.remote = True self.username = conn_params.username or self.get_user() self.tunnel_process = None @@ -64,17 +65,36 @@ def __enter__(self): def __exit__(self, exc_type, exc_val, exc_tb): self.close_ssh_tunnel() + @staticmethod + def is_port_open(host, port): + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock: + sock.settimeout(1) # Таймаут для попытки соединения + try: + sock.connect((host, port)) + return True + except socket.error: + return False + def establish_ssh_tunnel(self, local_port, remote_port): """ Establish an SSH tunnel from a local port to a remote PostgreSQL port. """ ssh_cmd = ['-N', '-L', f"{local_port}:localhost:{remote_port}"] self.tunnel_process = self.exec_command(ssh_cmd, get_process=True, timeout=300) + timeout = 10 + start_time = time.time() + while time.time() - start_time < timeout: + if self.is_port_open('localhost', local_port): + print("SSH tunnel established.") + return + time.sleep(0.5) + raise Exception("Failed to establish SSH tunnel within the timeout period.") def close_ssh_tunnel(self): - if hasattr(self, 'tunnel_process'): + if self.tunnel_process: self.tunnel_process.terminate() self.tunnel_process.wait() + print("SSH tunnel closed.") del self.tunnel_process else: print("No active tunnel to close.") @@ -240,9 +260,9 @@ def mkdtemp(self, prefix=None): - prefix (str): The prefix of the temporary directory name. """ if prefix: - command = ["ssh"] + self.ssh_cmd + [f"{self.username}@{self.host}", f"mktemp -d {prefix}XXXXX"] + command = ["ssh" + f"{self.username}@{self.host}"] + self.ssh_cmd + [f"mktemp -d {prefix}XXXXX"] else: - command = ["ssh"] + self.ssh_cmd + [f"{self.username}@{self.host}", "mktemp -d"] + command = ["ssh", f"{self.username}@{self.host}"] + self.ssh_cmd + ["mktemp -d"] result = subprocess.run(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) @@ -285,7 +305,7 @@ def write(self, filename, data, truncate=False, binary=False, read_and_write=Fal mode = "r+b" if binary else "r+" with tempfile.NamedTemporaryFile(mode=mode, delete=False) as tmp_file: - # Because in scp we set up port using -P option instead -p + # Because in scp we set up port using -P option scp_ssh_cmd = ['-P' if x == '-p' else x for x in self.ssh_cmd] if not truncate: @@ -307,9 +327,9 @@ def write(self, filename, data, truncate=False, binary=False, read_and_write=Fal tmp_file.flush() scp_cmd = ['scp'] + scp_ssh_cmd + [tmp_file.name, f"{self.username}@{self.host}:{filename}"] subprocess.run(scp_cmd, check=True) - remote_directory = os.path.dirname(filename) - mkdir_cmd = ['ssh'] + self.ssh_cmd + [f"{self.username}@{self.host}", f'mkdir -p {remote_directory}'] + remote_directory = os.path.dirname(filename) + mkdir_cmd = ['ssh', f"{self.username}@{self.host}"] + self.ssh_cmd + [f"mkdir -p {remote_directory}"] subprocess.run(mkdir_cmd, check=True) os.remove(tmp_file.name) @@ -374,7 +394,7 @@ def get_pid(self): return int(self.exec_command("echo $$", encoding=get_default_encoding())) def get_process_children(self, pid): - command = ["ssh"] + self.ssh_cmd + [f"{self.username}@{self.host}", f"pgrep -P {pid}"] + command = ["ssh", f"{self.username}@{self.host}"] + self.ssh_cmd + [f"pgrep -P {pid}"] result = subprocess.run(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) @@ -389,15 +409,16 @@ def db_connect(self, dbname, user, password=None, host="localhost", port=5432): """ Establish SSH tunnel and connect to a PostgreSQL database. """ - self.establish_ssh_tunnel(local_port=port, remote_port=self.conn_params.port) - + local_port = reserve_port() + self.establish_ssh_tunnel(local_port=local_port, remote_port=port) try: conn = pglib.connect( host=host, - port=port, + port=local_port, database=dbname, user=user, password=password, + timeout=10 ) print("Database connection established successfully.") return conn From fffb23c9b00eac3519230cb8cb9310e130fd7e35 Mon Sep 17 00:00:00 2001 From: vshepard Date: Sun, 2 Jun 2024 23:30:49 +0200 Subject: [PATCH 08/10] Add tunnel_port to remote_ops.py --- testgres/operations/remote_ops.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/testgres/operations/remote_ops.py b/testgres/operations/remote_ops.py index cd9ce201..958dc6c2 100644 --- a/testgres/operations/remote_ops.py +++ b/testgres/operations/remote_ops.py @@ -58,6 +58,7 @@ def __init__(self, conn_params: ConnectionParams): self.remote = True self.username = conn_params.username or self.get_user() self.tunnel_process = None + self.tunnel_port = None def __enter__(self): return self @@ -410,6 +411,7 @@ def db_connect(self, dbname, user, password=None, host="localhost", port=5432): Establish SSH tunnel and connect to a PostgreSQL database. """ local_port = reserve_port() + self.tunnel_port = local_port self.establish_ssh_tunnel(local_port=local_port, remote_port=port) try: conn = pglib.connect( From 1eb9a92fd7fee551fc49939c2048b4a1de571b9f Mon Sep 17 00:00:00 2001 From: vshepard Date: Tue, 4 Jun 2024 20:42:40 +0200 Subject: [PATCH 09/10] Update establish_ssh_tunnel --- setup.py | 2 +- testgres/operations/remote_ops.py | 11 +++++++---- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/setup.py b/setup.py index 412e8823..4f8837c0 100755 --- a/setup.py +++ b/setup.py @@ -27,7 +27,7 @@ readme = f.read() setup( - version='1.10.1', + version='1.10.2', name='testgres', packages=['testgres', 'testgres.operations', 'testgres.helpers'], description='Testing utility for PostgreSQL and its extensions', diff --git a/testgres/operations/remote_ops.py b/testgres/operations/remote_ops.py index 958dc6c2..12f85403 100644 --- a/testgres/operations/remote_ops.py +++ b/testgres/operations/remote_ops.py @@ -76,11 +76,14 @@ def is_port_open(host, port): except socket.error: return False - def establish_ssh_tunnel(self, local_port, remote_port): + def establish_ssh_tunnel(self, local_port, remote_port, host): """ Establish an SSH tunnel from a local port to a remote PostgreSQL port. """ - ssh_cmd = ['-N', '-L', f"{local_port}:localhost:{remote_port}"] + if host != 'localhost': + ssh_cmd = ['-N', '-L', f"localhost:{local_port}:{host}:{remote_port}"] + else: + ssh_cmd = ['-N', '-L', f"{local_port}:{host}:{remote_port}"] self.tunnel_process = self.exec_command(ssh_cmd, get_process=True, timeout=300) timeout = 10 start_time = time.time() @@ -412,10 +415,10 @@ def db_connect(self, dbname, user, password=None, host="localhost", port=5432): """ local_port = reserve_port() self.tunnel_port = local_port - self.establish_ssh_tunnel(local_port=local_port, remote_port=port) + self.establish_ssh_tunnel(local_port=local_port, remote_port=port, host=host) try: conn = pglib.connect( - host=host, + host='localhost', port=local_port, database=dbname, user=user, From 0611d1098fcc97ae1592df9610811a0952f33a61 Mon Sep 17 00:00:00 2001 From: vshepard Date: Fri, 7 Jun 2024 01:01:29 +0200 Subject: [PATCH 10/10] Change pg_hba.conf default params --- testgres/node.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/testgres/node.py b/testgres/node.py index 66d211dc..e94a8e47 100644 --- a/testgres/node.py +++ b/testgres/node.py @@ -530,7 +530,8 @@ def get_auth_method(t): u"host\treplication\tall\t::1/128\t\t{}\n".format(auth_host), u"host\treplication\tall\t{}/24\t\t{}\n".format(subnet_base, auth_host), u"host\tall\tall\t{}/24\t\t{}\n".format(subnet_base, auth_host), - u"host\tall\tall\tall\t{}\n".format(auth_host) + u"host\tall\tall\tall\t{}\n".format(auth_host), + u"host\treplication\tall\tall\t{}\n".format(auth_host) ] # yapf: disable # write missing lines pFad - Phonifier reborn

Pfad - The Proxy pFad of © 2024 Garber Painting. All rights reserved.

Note: This service is not intended for secure transactions such as banking, social media, email, or purchasing. Use at your own risk. We assume no liability whatsoever for broken pages.


Alternative Proxies:

Alternative Proxy

pFad Proxy

pFad v3 Proxy

pFad v4 Proxy