Content-Length: 13869 | pFad | http://github.com/postgrespro/testgres/pull/137.patch

thub.com From 9f69b307804e3b295fc1ddbdba4f3fc3caa05ab4 Mon Sep 17 00:00:00 2001 From: vshepard Date: Wed, 3 Jul 2024 23:42:31 +0200 Subject: [PATCH 1/2] Change print on logging --- testgres/config.py | 6 ++++++ testgres/connection.py | 3 ++- testgres/node.py | 11 +++++------ testgres/operations/local_ops.py | 5 +++-- .../plugins/pg_probackup2/pg_probackup2/app.py | 14 +++++++------- .../pg_probackup2/pg_probackup2/init_helpers.py | 11 ++++++----- .../pg_probackup2/tests/basic_test.py | 3 ++- testgres/utils.py | 4 ++-- 8 files changed, 33 insertions(+), 24 deletions(-) diff --git a/testgres/config.py b/testgres/config.py index b6c43926..63719f1d 100644 --- a/testgres/config.py +++ b/testgres/config.py @@ -2,6 +2,8 @@ import atexit import copy +import logging +import os import tempfile from contextlib import contextmanager @@ -10,6 +12,10 @@ from .operations.os_ops import OsOperations from .operations.local_ops import LocalOperations +log_level = os.getenv('LOGGING_LEVEL', 'WARNING').upper() +log_format = os.getenv('LOGGING_FORMAT', '%(asctime)s - %(levelname)s - %(message)s').upper() +logging.basicConfig(level=log_level, format=log_format) + class GlobalConfig(object): """ diff --git a/testgres/connection.py b/testgres/connection.py index 49b74844..ccedd135 100644 --- a/testgres/connection.py +++ b/testgres/connection.py @@ -1,4 +1,5 @@ # coding: utf-8 +import logging # we support both pg8000 and psycopg2 try: @@ -110,7 +111,7 @@ def execute(self, query, *args): except ProgrammingError: return None except Exception as e: - print("Error executing query: {}\n {}".format(repr(e), query)) + logging.error("Error executing query: {}\n {}".format(repr(e), query)) return None def close(self): diff --git a/testgres/node.py b/testgres/node.py index 2ad6ce54..a7bd28da 100644 --- a/testgres/node.py +++ b/testgres/node.py @@ -1,5 +1,5 @@ # coding: utf-8 - +import logging import os import random import signal @@ -736,11 +736,10 @@ def start(self, params=[], wait=True): if any(len(file) > 1 and 'Is another postmaster already ' 'running on port' in file[1].decode() for file in files): - print("Detected an issue with connecting to port {0}. " - "Trying another port after a 5-second sleep...".format(self.port)) + logging.warning("Detected an issue with connecting to port {0}. " + "Trying another port after a 5-second sleep...".format(self.port)) self.port = reserve_port() - options = {} - options['port'] = str(self.port) + options = {'port': str(self.port)} self.set_auto_conf(options) startup_retries -= 1 time.sleep(5) @@ -1166,7 +1165,7 @@ def poll_query_until(self, assert sleep_time > 0 attempts = 0 while max_attempts == 0 or attempts < max_attempts: - print(f"Pooling {attempts}") + logging.info(f"Pooling {attempts}") try: res = self.execute(dbname=dbname, query=query, diff --git a/testgres/operations/local_ops.py b/testgres/operations/local_ops.py index b05a11e2..b518a6cb 100644 --- a/testgres/operations/local_ops.py +++ b/testgres/operations/local_ops.py @@ -1,4 +1,5 @@ import getpass +import logging import os import shutil import stat @@ -165,9 +166,9 @@ def rmdirs(self, path, ignore_errors=True, retries=3, delay=1): except FileNotFoundError: return True except Exception as e: - print(f"Error: Failed to remove directory {path} on attempt {attempt + 1}: {e}") + logging.error(f"Error: Failed to remove directory {path} on attempt {attempt + 1}: {e}") time.sleep(delay) - print(f"Error: Failed to remove directory {path} after {retries} attempts.") + logging.error(f"Error: Failed to remove directory {path} after {retries} attempts.") return False def listdir(self, path): diff --git a/testgres/plugins/pg_probackup2/pg_probackup2/app.py b/testgres/plugins/pg_probackup2/pg_probackup2/app.py index ffad24d3..620dc563 100644 --- a/testgres/plugins/pg_probackup2/pg_probackup2/app.py +++ b/testgres/plugins/pg_probackup2/pg_probackup2/app.py @@ -1,6 +1,7 @@ import contextlib import importlib import json +import logging import os import re import subprocess @@ -74,7 +75,7 @@ def run(self, command, gdb=False, old_binary=False, return_id=True, env=None, command = [command[0], *self.backup_dir.pb_args, *command[1:]] if not self.probackup_old_path and old_binary: - print('PGPROBACKUPBIN_OLD is not set') + logging.error('PGPROBACKUPBIN_OLD is not set') exit(1) if old_binary: @@ -107,12 +108,11 @@ def run(self, command, gdb=False, old_binary=False, return_id=True, env=None, return GDBobj(cmdline, self.test_class) try: - result = None if type(gdb) is tuple and gdb[0] == 'suspend': # special test flow for manually debug probackup gdb_port = gdb[1] cmdline = ['gdbserver'] + ['localhost:' + str(gdb_port)] + cmdline - print("pg_probackup gdb suspended, waiting gdb connection on localhost:{0}".format(gdb_port)) + logging.warning("pg_probackup gdb suspended, waiting gdb connection on localhost:{0}".format(gdb_port)) start_time = time.time() self.test_class.output = subprocess.check_output( @@ -233,7 +233,7 @@ def backup_node( if options is None: options = [] if not node and not data_dir: - print('You must provide ether node or data_dir for backup') + logging.error('You must provide ether node or data_dir for backup') exit(1) if not datname: @@ -502,7 +502,7 @@ def show( if i == '': backup_record_split.remove(i) if len(header_split) != len(backup_record_split): - print(warning.format( + logging.error(warning.format( header=header, body=body, header_split=header_split, body_split=backup_record_split) @@ -581,7 +581,7 @@ def show_archive( else: show_splitted = self.run(cmd_list + options, old_binary=old_binary, expect_error=expect_error).splitlines() - print(show_splitted) + logging.error(show_splitted) exit(1) def validate( @@ -769,7 +769,7 @@ def load_backup_class(fs_type): if fs_type: implementation = fs_type - print("Using ", implementation) + logging.info("Using ", implementation) module_name, class_name = implementation.rsplit(sep='.', maxsplit=1) module = importlib.import_module(module_name) diff --git a/testgres/plugins/pg_probackup2/pg_probackup2/init_helpers.py b/testgres/plugins/pg_probackup2/pg_probackup2/init_helpers.py index 73731a6e..2d19e980 100644 --- a/testgres/plugins/pg_probackup2/pg_probackup2/init_helpers.py +++ b/testgres/plugins/pg_probackup2/pg_probackup2/init_helpers.py @@ -1,3 +1,4 @@ +import logging from functools import reduce import getpass import os @@ -31,7 +32,7 @@ cached_initdb_dir=False, node_cleanup_full=delete_logs) except Exception as e: - print("Can't configure testgres: {0}".format(e)) + logging.warning("Can't configure testgres: {0}".format(e)) class Init(object): @@ -104,7 +105,7 @@ def __init__(self): if os.path.isfile(probackup_path_tmp): if not os.access(probackup_path_tmp, os.X_OK): - print('{0} is not an executable file'.format( + logging.warning('{0} is not an executable file'.format( probackup_path_tmp)) else: self.probackup_path = probackup_path_tmp @@ -114,13 +115,13 @@ def __init__(self): if os.path.isfile(probackup_path_tmp): if not os.access(probackup_path_tmp, os.X_OK): - print('{0} is not an executable file'.format( + logging.warning('{0} is not an executable file'.format( probackup_path_tmp)) else: self.probackup_path = probackup_path_tmp if not self.probackup_path: - print('pg_probackup binary is not found') + logging.error('pg_probackup binary is not found') exit(1) if os.name == 'posix': @@ -207,7 +208,7 @@ def __init__(self): if self.probackup_version.split('.')[0].isdigit(): self.major_version = int(self.probackup_version.split('.')[0]) else: - print('Can\'t process pg_probackup version \"{}\": the major version is expected to be a number'.format(self.probackup_version)) + logging.error('Can\'t process pg_probackup version \"{}\": the major version is expected to be a number'.format(self.probackup_version)) sys.exit(1) def test_env(self): diff --git a/testgres/plugins/pg_probackup2/pg_probackup2/tests/basic_test.py b/testgres/plugins/pg_probackup2/pg_probackup2/tests/basic_test.py index f5a82d38..b63531ec 100644 --- a/testgres/plugins/pg_probackup2/pg_probackup2/tests/basic_test.py +++ b/testgres/plugins/pg_probackup2/pg_probackup2/tests/basic_test.py @@ -1,3 +1,4 @@ +import logging import os import shutil import unittest @@ -14,7 +15,7 @@ def get_module_and_function_name(test_id): module_name = test_id.split('.')[-2] fname = test_id.split('.')[-1] except IndexError: - print(f"Couldn't get module name and function name from test_id: `{test_id}`") + logging.warning(f"Couldn't get module name and function name from test_id: `{test_id}`") module_name, fname = test_id.split('(')[1].split('.')[1], test_id.split('(')[0] return module_name, fname diff --git a/testgres/utils.py b/testgres/utils.py index 745a2555..18a59191 100644 --- a/testgres/utils.py +++ b/testgres/utils.py @@ -3,6 +3,7 @@ from __future__ import division from __future__ import print_function +import logging import os import sys @@ -228,8 +229,7 @@ def eprint(*args, **kwargs): """ Print stuff to stderr. """ - - print(*args, file=sys.stderr, **kwargs) + logging.error(*args, **kwargs) def options_string(separator=u" ", **kwargs): From 0784f4961c335042d4bfdd3ea1626ae3b692a705 Mon Sep 17 00:00:00 2001 From: vshepard Date: Thu, 4 Jul 2024 01:44:08 +0200 Subject: [PATCH 2/2] Normalize error --- testgres/node.py | 2 +- testgres/operations/remote_ops.py | 10 +++++++++- testgres/utils.py | 3 +-- 3 files changed, 11 insertions(+), 4 deletions(-) diff --git a/testgres/node.py b/testgres/node.py index a7bd28da..2ea49529 100644 --- a/testgres/node.py +++ b/testgres/node.py @@ -1165,7 +1165,6 @@ def poll_query_until(self, assert sleep_time > 0 attempts = 0 while max_attempts == 0 or attempts < max_attempts: - logging.info(f"Pooling {attempts}") try: res = self.execute(dbname=dbname, query=query, @@ -1189,6 +1188,7 @@ def poll_query_until(self, return # done except tuple(suppress or []): + logging.info(f"Trying execute, attempt {attempts + 1}.\nQuery: {query}") pass # we're suppressing them time.sleep(sleep_time) diff --git a/testgres/operations/remote_ops.py b/testgres/operations/remote_ops.py index fa031075..f85490ef 100644 --- a/testgres/operations/remote_ops.py +++ b/testgres/operations/remote_ops.py @@ -93,8 +93,10 @@ def exec_command(self, cmd, wait_exit=False, verbose=False, expect_error=False, if not error: error_found = 0 else: + error = normalize_error(error) error_found = exit_status != 0 or any( - marker in error for marker in [b'error', b'Permission denied', b'fatal', b'No such file or directory']) + marker in error for marker in ['error', 'Permission denied', 'fatal', 'No such file or directory'] + ) if error_found: if isinstance(error, bytes): @@ -369,3 +371,9 @@ def db_connect(self, dbname, user, password=None, host="localhost", port=5432): password=password, ) return conn + + +def normalize_error(error): + if isinstance(error, bytes): + return error.decode() + return error diff --git a/testgres/utils.py b/testgres/utils.py index 18a59191..a4ee7877 100644 --- a/testgres/utils.py +++ b/testgres/utils.py @@ -3,7 +3,6 @@ from __future__ import division from __future__ import print_function -import logging import os import sys @@ -229,7 +228,7 @@ def eprint(*args, **kwargs): """ Print stuff to stderr. """ - logging.error(*args, **kwargs) + print(*args, file=sys.stderr, **kwargs) def options_string(separator=u" ", **kwargs):

pFad - (p)hone/(F)rame/(a)nonymizer/(d)eclutterfier! Saves Data!