From a07f10658c0be5c2647e4cc299881b8b1c08b538 Mon Sep 17 00:00:00 2001 From: vshepard Date: Sun, 30 Jun 2024 22:07:05 +0200 Subject: [PATCH 1/3] Add expect_error to pg_upgrade --- testgres/node.py | 5 +++-- testgres/operations/local_ops.py | 13 +++++++++---- 2 files changed, 12 insertions(+), 6 deletions(-) diff --git a/testgres/node.py b/testgres/node.py index 13d13294..36c1b3c2 100644 --- a/testgres/node.py +++ b/testgres/node.py @@ -1627,7 +1627,7 @@ def set_auto_conf(self, options, config='postgresql.auto.conf', rm_options={}): self.os_ops.write(path, auto_conf, truncate=True) - def upgrade_from(self, old_node, options=None): + def upgrade_from(self, old_node, options=None, expect_error=False): """ Upgrade this node from an old node using pg_upgrade. @@ -1656,10 +1656,11 @@ def upgrade_from(self, old_node, options=None): "--new-datadir", self.data_dir, "--old-port", str(old_node.port), "--new-port", str(self.port), + "--copy" ] upgrade_command += options - return self.os_ops.exec_command(upgrade_command) + return self.os_ops.exec_command(upgrade_command, expect_error=expect_error) def _get_bin_path(self, filename): if self.bin_dir: diff --git a/testgres/operations/local_ops.py b/testgres/operations/local_ops.py index 313d7060..756f5449 100644 --- a/testgres/operations/local_ops.py +++ b/testgres/operations/local_ops.py @@ -19,13 +19,18 @@ CMD_TIMEOUT_SEC = 60 error_markers = [b'error', b'Permission denied', b'fatal'] +err_out_markers = [b'Failure'] -def has_errors(output): +def has_errors(output=None, error=None): if output: if isinstance(output, str): output = output.encode(get_default_encoding()) - return any(marker in output for marker in error_markers) + return any(marker in output for marker in err_out_markers) + if error: + if isinstance(error, str): + error = error.encode(get_default_encoding()) + return any(marker in error for marker in error_markers) return False @@ -107,8 +112,8 @@ def exec_command(self, cmd, wait_exit=False, verbose=False, expect_error=False, process, output, error = self._run_command(cmd, shell, input, stdin, stdout, stderr, get_process, timeout, encoding) if get_process: return process - if process.returncode != 0 or (has_errors(error) and not expect_error): - self._raise_exec_exception('Utility exited with non-zero code. Error `{}`', cmd, process.returncode, error) + if (process.returncode != 0 or has_errors(output=output, error=error)) and not expect_error: + self._raise_exec_exception('Utility exited with non-zero code. Error `{}`', cmd, process.returncode, error or output) if verbose: return process.returncode, output, error From a61f794c0f6e246908679d805035bde6418b76ad Mon Sep 17 00:00:00 2001 From: vshepard Date: Mon, 1 Jul 2024 00:27:47 +0200 Subject: [PATCH 2/3] Fix node cleanup - rmdirs --- testgres/node.py | 2 +- testgres/operations/local_ops.py | 22 ++++++++++++++++++++-- 2 files changed, 21 insertions(+), 3 deletions(-) diff --git a/testgres/node.py b/testgres/node.py index 36c1b3c2..70c0d363 100644 --- a/testgres/node.py +++ b/testgres/node.py @@ -932,7 +932,7 @@ def cleanup(self, max_attempts=3): else: rm_dir = self.data_dir # just data, save logs - self.os_ops.rmdirs(rm_dir, ignore_errors=True) + self.os_ops.rmdirs(rm_dir, ignore_errors=False) return self diff --git a/testgres/operations/local_ops.py b/testgres/operations/local_ops.py index 756f5449..47a9685c 100644 --- a/testgres/operations/local_ops.py +++ b/testgres/operations/local_ops.py @@ -4,6 +4,7 @@ import stat import subprocess import tempfile +import time import psutil @@ -147,8 +148,25 @@ def makedirs(self, path, remove_existing=False): except FileExistsError: pass - def rmdirs(self, path, ignore_errors=True): - return rmtree(path, ignore_errors=ignore_errors) + def rmdirs(self, path, ignore_errors=True, retries=3, delay=1): + """ + Removes a directory and its contents, retrying on failure. + + :param path: Path to the directory. + :param ignore_errors: If True, ignore errors. + :param retries: Number of attempts to remove the directory. + :param delay: Delay between attempts in seconds. + """ + for attempt in range(retries): + try: + rmtree(path, ignore_errors=ignore_errors) + if not os.path.exists(path): + return True + except Exception as e: + print(f"Error: Failed to remove directory {path} on attempt {attempt + 1}: {e}") + time.sleep(delay) + print(f"Error: Failed to remove directory {path} after {retries} attempts.") + return False def listdir(self, path): return os.listdir(path) From e5184a4166228d1db842e20c42bd1320be9f9648 Mon Sep 17 00:00:00 2001 From: vshepard Date: Mon, 1 Jul 2024 11:50:36 +0200 Subject: [PATCH 3/3] Add cleanup parameter - clean full dir --- testgres/node.py | 8 ++++---- testgres/operations/local_ops.py | 2 ++ 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/testgres/node.py b/testgres/node.py index 70c0d363..ccf48e42 100644 --- a/testgres/node.py +++ b/testgres/node.py @@ -912,13 +912,14 @@ def free_port(self): self._should_free_port = False release_port(self.port) - def cleanup(self, max_attempts=3): + def cleanup(self, max_attempts=3, full=False): """ Stop node if needed and remove its data/logs directory. NOTE: take a look at TestgresConfig.node_cleanup_full. Args: max_attempts: how many times should we try to stop()? + full: clean full base dir Returns: This instance of :class:`.PostgresNode`. @@ -927,7 +928,7 @@ def cleanup(self, max_attempts=3): self._try_shutdown(max_attempts) # choose directory to be removed - if testgres_config.node_cleanup_full: + if testgres_config.node_cleanup_full or full: rm_dir = self.base_dir # everything else: rm_dir = self.data_dir # just data, save logs @@ -1655,8 +1656,7 @@ def upgrade_from(self, old_node, options=None, expect_error=False): "--old-datadir", old_node.data_dir, "--new-datadir", self.data_dir, "--old-port", str(old_node.port), - "--new-port", str(self.port), - "--copy" + "--new-port", str(self.port) ] upgrade_command += options diff --git a/testgres/operations/local_ops.py b/testgres/operations/local_ops.py index 47a9685c..b05a11e2 100644 --- a/testgres/operations/local_ops.py +++ b/testgres/operations/local_ops.py @@ -162,6 +162,8 @@ def rmdirs(self, path, ignore_errors=True, retries=3, delay=1): rmtree(path, ignore_errors=ignore_errors) if not os.path.exists(path): return True + except FileNotFoundError: + return True except Exception as e: print(f"Error: Failed to remove directory {path} on attempt {attempt + 1}: {e}") time.sleep(delay) pFad - Phonifier reborn

Pfad - The Proxy pFad of © 2024 Garber Painting. All rights reserved.

Note: This service is not intended for secure transactions such as banking, social media, email, or purchasing. Use at your own risk. We assume no liability whatsoever for broken pages.


Alternative Proxies:

Alternative Proxy

pFad Proxy

pFad v3 Proxy

pFad v4 Proxy