From 4c70b80b84da5d0c950b645b28aee85593374239 Mon Sep 17 00:00:00 2001 From: egarbuz <165897130+egarbuz@users.noreply.github.com> Date: Thu, 4 Apr 2024 12:02:39 +0300 Subject: [PATCH 01/17] Adding port and database name into params list for add-instance command (#118) --- testgres/plugins/pg_probackup2/pg_probackup2/app.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/testgres/plugins/pg_probackup2/pg_probackup2/app.py b/testgres/plugins/pg_probackup2/pg_probackup2/app.py index 94dcd997..1ab71109 100644 --- a/testgres/plugins/pg_probackup2/pg_probackup2/app.py +++ b/testgres/plugins/pg_probackup2/pg_probackup2/app.py @@ -149,13 +149,19 @@ def init(self, options=None, old_binary=False, skip_log_directory=False, expect_ use_backup_dir=use_backup_dir ) - def add_instance(self, instance, node, old_binary=False, options=None, expect_error=False): + def add_instance(self, instance, node, old_binary=False, options=None, expect_error=False, datname=False): if options is None: options = [] + + if not datname: + datname = 'postgres' + cmd = [ 'add-instance', '--instance={0}'.format(instance), - '-D', node.data_dir + '-D', node.data_dir, + '--pgport', '%i' % node.port, + '--pgdatabase', datname ] # don`t forget to kill old_binary after remote ssh release From 3dc640fb7d6df7e5bbdd1970376773f3d9b7bd09 Mon Sep 17 00:00:00 2001 From: "e.garbuz" Date: Thu, 4 Apr 2024 14:11:09 +0300 Subject: [PATCH 02/17] Value pg_probackup major_version added to the class Init --- testgres/plugins/pg_probackup2/pg_probackup2/init_helpers.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/testgres/plugins/pg_probackup2/pg_probackup2/init_helpers.py b/testgres/plugins/pg_probackup2/pg_probackup2/init_helpers.py index e3dd9e4f..b6bf90f4 100644 --- a/testgres/plugins/pg_probackup2/pg_probackup2/init_helpers.py +++ b/testgres/plugins/pg_probackup2/pg_probackup2/init_helpers.py @@ -204,6 +204,8 @@ def __init__(self): os.environ["PGAPPNAME"] = "pg_probackup" self.delete_logs = delete_logs + self.major_version = int(self.probackup_version.split('.')[0]) + def test_env(self): return self._test_env.copy() From fba3bd36a6f86c6be42f87f3fb10e873b02dbec3 Mon Sep 17 00:00:00 2001 From: Sofia Kopikova Date: Thu, 4 Apr 2024 19:16:02 +0300 Subject: [PATCH 03/17] Fix error 'Is another postmaster already running on port XXX' Sometimes when we abnormally shutdown node its port stays busy. So I added retry attempts to start() function in case we encounter such error. Test for this case was added (test_the_same_port). --- testgres/node.py | 32 ++++++++++++++++++++++++-------- tests/test_simple.py | 8 ++++++++ 2 files changed, 32 insertions(+), 8 deletions(-) diff --git a/testgres/node.py b/testgres/node.py index 0f1dcf98..c9b34d7b 100644 --- a/testgres/node.py +++ b/testgres/node.py @@ -726,14 +726,30 @@ def start(self, params=[], wait=True): "start" ] + params # yapf: disable - try: - exit_status, out, error = execute_utility(_params, self.utils_log_file, verbose=True) - if error and 'does not exist' in error: - raise Exception - except Exception as e: - msg = 'Cannot start node' - files = self._collect_special_files() - raise_from(StartNodeException(msg, files), e) + startup_retries = 5 + while True: + try: + exit_status, out, error = execute_utility(_params, self.utils_log_file, verbose=True) + if error and 'does not exist' in error: + raise Exception + except Exception as e: + files = self._collect_special_files() + if any(len(file) > 1 and 'Is another postmaster already ' + 'running on port' in file[1].decode() for + file in files): + print("Detected an issue with connecting to port {0}. " + "Trying another port after a 5-second sleep...".format(self.port)) + self.port = reserve_port() + options = {} + options['port'] = str(self.port) + self.set_auto_conf(options) + startup_retries -= 1 + time.sleep(5) + continue + + msg = 'Cannot start node' + raise_from(StartNodeException(msg, files), e) + break self._maybe_start_logger() self.is_started = True return self diff --git a/tests/test_simple.py b/tests/test_simple.py index 8cb0d94e..ba23c3ed 100644 --- a/tests/test_simple.py +++ b/tests/test_simple.py @@ -1033,6 +1033,14 @@ def test_parse_pg_version(self): # Macos assert parse_pg_version("postgres (PostgreSQL) 14.9 (Homebrew)") == "14.9" + def test_the_same_port(self): + with get_new_node() as node: + node.init().start() + + with get_new_node() as node2: + node2.port = node.port + node2.init().start() + if __name__ == '__main__': if os.environ.get('ALT_CONFIG'): From 2a0c37b9e139cd0a2fc40eedc4edff1fa96dbf66 Mon Sep 17 00:00:00 2001 From: "e.garbuz" Date: Mon, 8 Apr 2024 17:57:55 +0300 Subject: [PATCH 04/17] Add check on digit for probackup_version --- testgres/plugins/pg_probackup2/pg_probackup2/init_helpers.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/testgres/plugins/pg_probackup2/pg_probackup2/init_helpers.py b/testgres/plugins/pg_probackup2/pg_probackup2/init_helpers.py index b6bf90f4..109a411e 100644 --- a/testgres/plugins/pg_probackup2/pg_probackup2/init_helpers.py +++ b/testgres/plugins/pg_probackup2/pg_probackup2/init_helpers.py @@ -204,7 +204,9 @@ def __init__(self): os.environ["PGAPPNAME"] = "pg_probackup" self.delete_logs = delete_logs - self.major_version = int(self.probackup_version.split('.')[0]) + self.major_version = 0 + if self.probackup_version.split('.')[0].isdigit: + self.major_version = int(self.probackup_version.split('.')[0]) def test_env(self): return self._test_env.copy() From 5db4ae393e4b4f69b5721b27e9c80fc3dcd5c044 Mon Sep 17 00:00:00 2001 From: "e.garbuz" Date: Tue, 9 Apr 2024 13:01:07 +0300 Subject: [PATCH 05/17] Break when pg_probackup version is incorrect --- testgres/plugins/pg_probackup2/pg_probackup2/init_helpers.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/testgres/plugins/pg_probackup2/pg_probackup2/init_helpers.py b/testgres/plugins/pg_probackup2/pg_probackup2/init_helpers.py index 109a411e..318877af 100644 --- a/testgres/plugins/pg_probackup2/pg_probackup2/init_helpers.py +++ b/testgres/plugins/pg_probackup2/pg_probackup2/init_helpers.py @@ -204,9 +204,11 @@ def __init__(self): os.environ["PGAPPNAME"] = "pg_probackup" self.delete_logs = delete_logs - self.major_version = 0 if self.probackup_version.split('.')[0].isdigit: self.major_version = int(self.probackup_version.split('.')[0]) + else: + print('Pg_probackup version is not correct!') + sys.exit(1) def test_env(self): return self._test_env.copy() From 3a90ecd41293aa08c2db662ce73ecad0f34b8636 Mon Sep 17 00:00:00 2001 From: "e.garbuz" Date: Tue, 9 Apr 2024 13:36:42 +0300 Subject: [PATCH 06/17] Adding print incorrect version --- testgres/plugins/pg_probackup2/pg_probackup2/init_helpers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/testgres/plugins/pg_probackup2/pg_probackup2/init_helpers.py b/testgres/plugins/pg_probackup2/pg_probackup2/init_helpers.py index 318877af..f8c9f3b3 100644 --- a/testgres/plugins/pg_probackup2/pg_probackup2/init_helpers.py +++ b/testgres/plugins/pg_probackup2/pg_probackup2/init_helpers.py @@ -207,7 +207,7 @@ def __init__(self): if self.probackup_version.split('.')[0].isdigit: self.major_version = int(self.probackup_version.split('.')[0]) else: - print('Pg_probackup version is not correct!') + print('Pg_probackup version \"{}\" is not correct!'.format(self.probackup_version)) sys.exit(1) def test_env(self): From ea05a4afc54a9ed65769921b9221afec378a6df1 Mon Sep 17 00:00:00 2001 From: "e.garbuz" Date: Wed, 10 Apr 2024 09:45:17 +0300 Subject: [PATCH 07/17] Correction of remarks --- testgres/plugins/pg_probackup2/pg_probackup2/init_helpers.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/testgres/plugins/pg_probackup2/pg_probackup2/init_helpers.py b/testgres/plugins/pg_probackup2/pg_probackup2/init_helpers.py index f8c9f3b3..9231ac04 100644 --- a/testgres/plugins/pg_probackup2/pg_probackup2/init_helpers.py +++ b/testgres/plugins/pg_probackup2/pg_probackup2/init_helpers.py @@ -204,10 +204,10 @@ def __init__(self): os.environ["PGAPPNAME"] = "pg_probackup" self.delete_logs = delete_logs - if self.probackup_version.split('.')[0].isdigit: + if self.probackup_version.split('.')[0].isdigit(): self.major_version = int(self.probackup_version.split('.')[0]) else: - print('Pg_probackup version \"{}\" is not correct!'.format(self.probackup_version)) + print('Pg_probackup version \"{}\" is not correct! Expected that the major pg_probackup version should be a number.'.format(self.probackup_version)) sys.exit(1) def test_env(self): From 5b8a8369cbc0517c4aecacfa35b2955d8e9a4a0d Mon Sep 17 00:00:00 2001 From: "e.garbuz" Date: Wed, 10 Apr 2024 14:23:31 +0300 Subject: [PATCH 08/17] Changed error message --- testgres/plugins/pg_probackup2/pg_probackup2/init_helpers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/testgres/plugins/pg_probackup2/pg_probackup2/init_helpers.py b/testgres/plugins/pg_probackup2/pg_probackup2/init_helpers.py index 9231ac04..73731a6e 100644 --- a/testgres/plugins/pg_probackup2/pg_probackup2/init_helpers.py +++ b/testgres/plugins/pg_probackup2/pg_probackup2/init_helpers.py @@ -207,7 +207,7 @@ def __init__(self): if self.probackup_version.split('.')[0].isdigit(): self.major_version = int(self.probackup_version.split('.')[0]) else: - print('Pg_probackup version \"{}\" is not correct! Expected that the major pg_probackup version should be a number.'.format(self.probackup_version)) + print('Can\'t process pg_probackup version \"{}\": the major version is expected to be a number'.format(self.probackup_version)) sys.exit(1) def test_env(self): From eb0476209523b4809a4a7e20e4f2cd9968dd5cb9 Mon Sep 17 00:00:00 2001 From: Victoria Shepard <5807469+demonolock@users.noreply.github.com> Date: Tue, 7 May 2024 15:53:49 +0300 Subject: [PATCH 09/17] Add options to pg_basebackup (#121) --- testgres/backup.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/testgres/backup.py b/testgres/backup.py index a89e214d..cecb0f7b 100644 --- a/testgres/backup.py +++ b/testgres/backup.py @@ -33,7 +33,8 @@ def __init__(self, node, base_dir=None, username=None, - xlog_method=XLogMethod.fetch): + xlog_method=XLogMethod.fetch, + options=None): """ Create a new backup. @@ -43,6 +44,8 @@ def __init__(self, username: database user name. xlog_method: none | fetch | stream (see docs) """ + if not options: + options = [] self.os_ops = node.os_ops if not node.status(): raise BackupException('Node must be running') @@ -77,6 +80,7 @@ def __init__(self, "-D", data_dir, "-X", xlog_method.value ] # yapf: disable + _params += options execute_utility(_params, self.log_file) def __enter__(self): From 0cf30a7dc0078645052a7fa9afddb5f6999df5d8 Mon Sep 17 00:00:00 2001 From: Victoria Shepard <5807469+demonolock@users.noreply.github.com> Date: Tue, 7 May 2024 20:07:32 +0300 Subject: [PATCH 10/17] Fix port bind (#117) * Fix port bind * Up testgres version - 1.10.1 --- setup.py | 2 +- testgres/operations/remote_ops.py | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index b006c8bf..412e8823 100755 --- a/setup.py +++ b/setup.py @@ -27,7 +27,7 @@ readme = f.read() setup( - version='1.10.0', + version='1.10.1', name='testgres', packages=['testgres', 'testgres.operations', 'testgres.helpers'], description='Testing utility for PostgreSQL and its extensions', diff --git a/testgres/operations/remote_ops.py b/testgres/operations/remote_ops.py index 01251e1c..f182768b 100644 --- a/testgres/operations/remote_ops.py +++ b/testgres/operations/remote_ops.py @@ -4,6 +4,8 @@ import tempfile import platform +from ..utils import reserve_port + # we support both pg8000 and psycopg2 try: import psycopg2 as pglib @@ -392,7 +394,7 @@ def db_connect(self, dbname, user, password=None, host="localhost", port=5432): """ Established SSH tunnel and Connects to a PostgreSQL """ - self.establish_ssh_tunnel(local_port=port, remote_port=5432) + self.establish_ssh_tunnel(local_port=reserve_port(), remote_port=5432) try: conn = pglib.connect( host=host, From 8a25cb375138442c149f93051579a04704bbe38a Mon Sep 17 00:00:00 2001 From: Victoria Shepard <5807469+demonolock@users.noreply.github.com> Date: Tue, 21 May 2024 01:57:29 +0300 Subject: [PATCH 11/17] Add pgbench_with_wait function (#122) --- testgres/node.py | 27 ++++++++++++++++++++++++++- 1 file changed, 26 insertions(+), 1 deletion(-) diff --git a/testgres/node.py b/testgres/node.py index c9b34d7b..d1784cb9 100644 --- a/testgres/node.py +++ b/testgres/node.py @@ -1371,7 +1371,7 @@ def pgbench(self, username=None, stdout=None, stderr=None, - options=[]): + options=None): """ Spawn a pgbench process. @@ -1385,6 +1385,8 @@ def pgbench(self, Returns: Process created by subprocess.Popen. """ + if options is None: + options = [] # Set default arguments dbname = dbname or default_dbname() @@ -1404,6 +1406,29 @@ def pgbench(self, return proc + def pgbench_with_wait(self, + dbname=None, + username=None, + stdout=None, + stderr=None, + options=None): + """ + Do pgbench command and wait. + + Args: + dbname: database name to connect to. + username: database user name. + stdout: stdout file to be used by Popen. + stderr: stderr file to be used by Popen. + options: additional options for pgbench (list). + """ + if options is None: + options = [] + + with self.pgbench(dbname, username, stdout, stderr, options) as pgbench: + pgbench.wait() + return + def pgbench_init(self, **kwargs): """ Small wrapper for pgbench_run(). From e375302a114cd4df3ceed54d6526f250c44c08e7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fabr=C3=ADzio=20de=20Royes=20Mello?= Date: Fri, 24 May 2024 17:36:14 -0300 Subject: [PATCH 12/17] Add options to pg_upgrade (#125) --- testgres/node.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/testgres/node.py b/testgres/node.py index d1784cb9..e5e8fd5f 100644 --- a/testgres/node.py +++ b/testgres/node.py @@ -1639,7 +1639,7 @@ def set_auto_conf(self, options, config='postgresql.auto.conf', rm_options={}): self.os_ops.write(path, auto_conf, truncate=True) - def upgrade_from(self, old_node): + def upgrade_from(self, old_node, options=None): """ Upgrade this node from an old node using pg_upgrade. @@ -1652,6 +1652,9 @@ def upgrade_from(self, old_node): if not os.path.exists(self.data_dir): self.init() + if not options: + options = [] + pg_upgrade_binary = self._get_bin_path("pg_upgrade") if not os.path.exists(pg_upgrade_binary): @@ -1666,6 +1669,7 @@ def upgrade_from(self, old_node): "--old-port", str(old_node.port), "--new-port", str(self.port), ] + upgrade_command += options return self.os_ops.exec_command(upgrade_command) From 63db2752501f9ac58a05accc686aec3cd2825c20 Mon Sep 17 00:00:00 2001 From: Vyacheslav Makarov <50846161+MakSl@users.noreply.github.com> Date: Tue, 28 May 2024 18:07:38 +0300 Subject: [PATCH 13/17] Added command execution time measurement (#123) This will be useful to keep track of possible performance degradation when code changes. --- testgres/plugins/pg_probackup2/pg_probackup2/app.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/testgres/plugins/pg_probackup2/pg_probackup2/app.py b/testgres/plugins/pg_probackup2/pg_probackup2/app.py index 1ab71109..3d2e0101 100644 --- a/testgres/plugins/pg_probackup2/pg_probackup2/app.py +++ b/testgres/plugins/pg_probackup2/pg_probackup2/app.py @@ -56,6 +56,7 @@ def __init__(self, test_class: unittest.TestCase, self.verbose = init_params.verbose self.archive_compress = init_params.archive_compress self.test_class.output = None + self.execution_time = None def run(self, command, gdb=False, old_binary=False, return_id=True, env=None, skip_log_directory=False, expect_error=False, use_backup_dir=True): @@ -113,11 +114,15 @@ def run(self, command, gdb=False, old_binary=False, return_id=True, env=None, cmdline = ['gdbserver'] + ['localhost:' + str(gdb_port)] + cmdline print("pg_probackup gdb suspended, waiting gdb connection on localhost:{0}".format(gdb_port)) + start_time = time.time() self.test_class.output = subprocess.check_output( cmdline, stderr=subprocess.STDOUT, env=env ).decode('utf-8', errors='replace') + end_time = time.time() + self.execution_time = end_time - start_time + if command[0] == 'backup' and return_id: # return backup ID for line in self.test_class.output.splitlines(): From c1cfc26b4b74c36fd6cccda2ee00fdce60dabc92 Mon Sep 17 00:00:00 2001 From: Victoria Shepard <5807469+demonolock@users.noreply.github.com> Date: Thu, 30 May 2024 11:28:33 +0200 Subject: [PATCH 14/17] Add parsing backup_id in run app.py (#126) --- .../plugins/pg_probackup2/pg_probackup2/app.py | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/testgres/plugins/pg_probackup2/pg_probackup2/app.py b/testgres/plugins/pg_probackup2/pg_probackup2/app.py index 3d2e0101..1a4ca9e7 100644 --- a/testgres/plugins/pg_probackup2/pg_probackup2/app.py +++ b/testgres/plugins/pg_probackup2/pg_probackup2/app.py @@ -124,10 +124,7 @@ def run(self, command, gdb=False, old_binary=False, return_id=True, env=None, self.execution_time = end_time - start_time if command[0] == 'backup' and return_id: - # return backup ID - for line in self.test_class.output.splitlines(): - if 'INFO: Backup' and 'completed' in line: - result = line.split()[2] + result = self.get_backup_id() else: result = self.test_class.output if expect_error is True: @@ -144,6 +141,19 @@ def run(self, command, gdb=False, old_binary=False, return_id=True, env=None, else: raise ProbackupException(self.test_class.output, self.test_class.cmd) + def get_backup_id(self): + if init_params.major_version > 2: + pattern = re.compile(r"Backup (.*) completed successfully.") + for line in self.test_class.output.splitlines(): + match = pattern.search(line) + if match: + return match.group(1) + else: + for line in self.test_class.output.splitlines(): + if 'INFO: Backup' and 'completed' in line: + return line.split()[2] + return None + def init(self, options=None, old_binary=False, skip_log_directory=False, expect_error=False, use_backup_dir=True): if options is None: options = [] From b4268f8bdaaf67e0f7677c0f801210a0085bb594 Mon Sep 17 00:00:00 2001 From: Victoria Shepard <5807469+demonolock@users.noreply.github.com> Date: Wed, 5 Jun 2024 09:34:35 +0200 Subject: [PATCH 15/17] Revert "Fix port bind (#117)" (#128) This reverts commit 0cf30a7dc0078645052a7fa9afddb5f6999df5d8. --- setup.py | 2 +- testgres/operations/remote_ops.py | 4 +--- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/setup.py b/setup.py index 412e8823..b006c8bf 100755 --- a/setup.py +++ b/setup.py @@ -27,7 +27,7 @@ readme = f.read() setup( - version='1.10.1', + version='1.10.0', name='testgres', packages=['testgres', 'testgres.operations', 'testgres.helpers'], description='Testing utility for PostgreSQL and its extensions', diff --git a/testgres/operations/remote_ops.py b/testgres/operations/remote_ops.py index f182768b..01251e1c 100644 --- a/testgres/operations/remote_ops.py +++ b/testgres/operations/remote_ops.py @@ -4,8 +4,6 @@ import tempfile import platform -from ..utils import reserve_port - # we support both pg8000 and psycopg2 try: import psycopg2 as pglib @@ -394,7 +392,7 @@ def db_connect(self, dbname, user, password=None, host="localhost", port=5432): """ Established SSH tunnel and Connects to a PostgreSQL """ - self.establish_ssh_tunnel(local_port=reserve_port(), remote_port=5432) + self.establish_ssh_tunnel(local_port=port, remote_port=5432) try: conn = pglib.connect( host=host, From 0728583c9b6a9df660cfd209220d8031893d136c Mon Sep 17 00:00:00 2001 From: asavchkov Date: Mon, 24 Jun 2024 17:01:31 +0700 Subject: [PATCH 16/17] Up version --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index b006c8bf..412e8823 100755 --- a/setup.py +++ b/setup.py @@ -27,7 +27,7 @@ readme = f.read() setup( - version='1.10.0', + version='1.10.1', name='testgres', packages=['testgres', 'testgres.operations', 'testgres.helpers'], description='Testing utility for PostgreSQL and its extensions', From 529b4df245ff73b9db757854174f29bb208a0c4c Mon Sep 17 00:00:00 2001 From: asavchkov Date: Mon, 24 Jun 2024 17:16:50 +0700 Subject: [PATCH 17/17] Up pg_probackup2 version --- testgres/plugins/pg_probackup2/setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/testgres/plugins/pg_probackup2/setup.py b/testgres/plugins/pg_probackup2/setup.py index 381d7ae2..2ff5a503 100644 --- a/testgres/plugins/pg_probackup2/setup.py +++ b/testgres/plugins/pg_probackup2/setup.py @@ -4,7 +4,7 @@ from distutils.core import setup setup( - version='0.0.2', + version='0.0.3', name='testgres_pg_probackup2', packages=['pg_probackup2', 'pg_probackup2.storage'], description='Plugin for testgres that manages pg_probackup2', pFad - Phonifier reborn

Pfad - The Proxy pFad of © 2024 Garber Painting. All rights reserved.

Note: This service is not intended for secure transactions such as banking, social media, email, or purchasing. Use at your own risk. We assume no liability whatsoever for broken pages.


Alternative Proxies:

Alternative Proxy

pFad Proxy

pFad v3 Proxy

pFad v4 Proxy