Content-Length: 15439 | pFad | http://github.com/postgrespro/testgres/pull/187.patch

thub.com From f5fe166eec93c2b727b4ee0145eecdf77d9b38d6 Mon Sep 17 00:00:00 2001 From: "d.kovalenko" Date: Thu, 20 Feb 2025 14:23:34 +0300 Subject: [PATCH 1/6] RemoteOperations::exec_command updated - Exact enumeration of supported 'cmd' types - Refactoring --- testgres/operations/remote_ops.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/testgres/operations/remote_ops.py b/testgres/operations/remote_ops.py index f690e063..a24fce50 100644 --- a/testgres/operations/remote_ops.py +++ b/testgres/operations/remote_ops.py @@ -78,14 +78,17 @@ def exec_command(self, cmd, wait_exit=False, verbose=False, expect_error=False, assert input_prepared is None or (type(input_prepared) == bytes) # noqa: E721 - ssh_cmd = [] - if isinstance(cmd, str): - ssh_cmd = ['ssh', self.ssh_dest] + self.ssh_args + [cmd] - elif isinstance(cmd, list): - ssh_cmd = ['ssh', self.ssh_dest] + self.ssh_args + [subprocess.list2cmdline(cmd)] + if type(cmd) == str: # noqa: E721 + cmd_s = cmd + elif type(cmd) == list: # noqa: E721 + cmd_s = subprocess.list2cmdline(cmd) else: raise ValueError("Invalid 'cmd' argument type - {0}".format(type(cmd).__name__)) + assert type(cmd_s) == str # noqa: E721 + + ssh_cmd = ['ssh', self.ssh_dest] + self.ssh_args + [cmd_s] + process = subprocess.Popen(ssh_cmd, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE) assert not (process is None) if get_process: From 2b1db89c0375c1ec3a130e31fa46ab8b7339141e Mon Sep 17 00:00:00 2001 From: "d.kovalenko" Date: Thu, 20 Feb 2025 20:24:09 +0300 Subject: [PATCH 2/6] RemoteOperations::exec_command explicitly transfers LANG, LANGUAGE and LC_* envvars to the server side It should help resolve a problem with replacing a LANG variable by ssh-server. History. On our internal tests we got a problem on the Debian 11 and PostgresPro STD-13. One test returned the error from initdb: initdb: error: collations with different collate and ctype values ("en_US.UTF-8" and "C.UTF-8" accordingly) are not supported by ICU - TestRunner set variable LANG="C" - Python set variable LC_CTYPE="C.UTF-8" - Test call inidb through command "ssh test@localhost inidb -D ...." - SSH-server replaces LANG with value "en_US.UTF-8" (from etc/default/locale) - initdb calculate collate through this value of LANG variable and get en_US.UTF-8 So we have that: - ctype is C.UTF-8 - collate is en_US.UTF-8 ICU on the Debuan-11 (uconv v2.1 ICU 67.1) does not suppot this combination and inidb rturns the error. This patch generates a new command line for ssh: ssh test@localhost "LANG=\"...\";LC_xxx=\"...\";" It resolves this problem with initdb and should help resolve other problems with execution of command through SSH. Amen. --- testgres/operations/remote_ops.py | 43 ++++++++++++++++++++++++++++++- 1 file changed, 42 insertions(+), 1 deletion(-) diff --git a/testgres/operations/remote_ops.py b/testgres/operations/remote_ops.py index a24fce50..b4d65e95 100644 --- a/testgres/operations/remote_ops.py +++ b/testgres/operations/remote_ops.py @@ -87,7 +87,12 @@ def exec_command(self, cmd, wait_exit=False, verbose=False, expect_error=False, assert type(cmd_s) == str # noqa: E721 - ssh_cmd = ['ssh', self.ssh_dest] + self.ssh_args + [cmd_s] + cmd_items = __class__._make_exec_env_list() + cmd_items.append(cmd_s) + + env_cmd_s = ';'.join(cmd_items) + + ssh_cmd = ['ssh', self.ssh_dest] + self.ssh_args + [env_cmd_s] process = subprocess.Popen(ssh_cmd, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE) assert not (process is None) @@ -510,6 +515,42 @@ def db_connect(self, dbname, user, password=None, host="localhost", port=5432): ) return conn + def _make_exec_env_list() -> list[str]: + result = list[str]() + for envvar in os.environ.items(): + if not __class__._does_put_envvar_into_exec_cmd(envvar[0]): + continue + qvalue = __class__._quote_envvar(envvar[1]) + assert type(qvalue) == str # noqa: E721 + result.append(envvar[0] + "=" + qvalue) + continue + + return result + + sm_envs_for_exec_cmd = ["LANG", "LANGUAGE"] + + def _does_put_envvar_into_exec_cmd(name: str) -> bool: + assert type(name) == str # noqa: E721 + name = name.upper() + if name.startswith("LC_"): + return True + if name in __class__.sm_envs_for_exec_cmd: + return True + return False + + def _quote_envvar(value) -> str: + assert type(value) == str # noqa: E721 + result = "\"" + for ch in value: + if ch == "\"": + result += "\\\"" + elif ch == "\\": + result += "\\\\" + else: + result += ch + result += "\"" + return result + def normalize_error(error): if isinstance(error, bytes): From 4eb833040f2f72e93e07a866b52c123e5003cc5d Mon Sep 17 00:00:00 2001 From: "d.kovalenko" Date: Fri, 21 Feb 2025 15:15:50 +0300 Subject: [PATCH 3/6] New tests in TestgresRemoteTests are added MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit New tests: - test_init__LANG_С - test_init__unk_LANG_and_LC_CTYPE --- tests/test_simple_remote.py | 56 +++++++++++++++++++++++++++++++++++++ 1 file changed, 56 insertions(+) diff --git a/tests/test_simple_remote.py b/tests/test_simple_remote.py index c8dd2964..dd110962 100755 --- a/tests/test_simple_remote.py +++ b/tests/test_simple_remote.py @@ -119,6 +119,56 @@ def test_custom_init(self): # there should be no trust entries at all self.assertFalse(any('trust' in s for s in lines)) + def test_init__LANG_С(self): + # PBCKP-1744 + prev_LANG = os.environ.get("LANG") + + try: + os.environ["LANG"] = "C" + + with get_remote_node(conn_params=conn_params) as node: + node.init().start() + finally: + __class__.helper__restore_envvar("LANG", prev_LANG) + + def test_init__unk_LANG_and_LC_CTYPE(self): + # PBCKP-1744 + prev_LANG = os.environ.get("LANG") + prev_LANGUAGE = os.environ.get("LANGUAGE") + prev_LC_CTYPE = os.environ.get("LC_CTYPE") + prev_LC_COLLATE = os.environ.get("LC_COLLATE") + + try: + os.environ["LANG"] = "UNKNOWN_LANG" + os.environ.pop("LANGUAGE", None) + os.environ["LC_CTYPE"] = "UNKNOWN_CTYPE" + os.environ.pop("LC_COLLATE", None) + + assert os.environ.get("LANG") == "UNKNOWN_LANG" + assert not ("LANGUAGE" in os.environ.keys()) + assert os.environ.get("LC_CTYPE") == "UNKNOWN_CTYPE" + assert not ("LC_COLLATE" in os.environ.keys()) + + while True: + try: + with get_remote_node(conn_params=conn_params): + pass + except testgres.exceptions.ExecUtilException as e: + # warning: setlocale: LC_CTYPE: cannot change locale (UNKNOWN_CTYPE): No such file or directory + # postgres (PostgreSQL) 14.12 + errMsg = str(e) + assert "LC_CTYPE" in errMsg + assert "UNKNOWN_CTYPE" in errMsg + assert "warning: setlocale: LC_CTYPE: cannot change locale (UNKNOWN_CTYPE): No such file or directory" in errMsg + assert "postgres" in errMsg + break + raise Exception("We expected an error!") + finally: + __class__.helper__restore_envvar("LANG", prev_LANG) + __class__.helper__restore_envvar("LANGUAGE", prev_LANGUAGE) + __class__.helper__restore_envvar("LC_CTYPE", prev_LC_CTYPE) + __class__.helper__restore_envvar("LC_COLLATE", prev_LC_COLLATE) + def test_double_init(self): with get_remote_node(conn_params=conn_params).init() as node: # can't initialize node more than once @@ -994,6 +1044,12 @@ def test_child_process_dies(self): # try to handle children list -- missing processes will have ptype "ProcessType.Unknown" [ProcessProxy(p) for p in children] + def helper__restore_envvar(name, prev_value): + if prev_value is None: + os.environ.pop(name, None) + else: + os.environ[name] = prev_value + if __name__ == '__main__': if os_ops.environ('ALT_CONFIG'): From f4dc0623954582a7d5c75b028138ad08d297c6f0 Mon Sep 17 00:00:00 2001 From: "d.kovalenko" Date: Fri, 21 Feb 2025 17:59:13 +0300 Subject: [PATCH 4/6] TestgresRemoteTests.test_init__unk_LANG_and_LC_CTYPE is updated Let's test bad data with '\' and '"' symbols. --- tests/test_simple_remote.py | 71 ++++++++++++++++++++++++------------- 1 file changed, 47 insertions(+), 24 deletions(-) diff --git a/tests/test_simple_remote.py b/tests/test_simple_remote.py index dd110962..2b581ac9 100755 --- a/tests/test_simple_remote.py +++ b/tests/test_simple_remote.py @@ -139,30 +139,53 @@ def test_init__unk_LANG_and_LC_CTYPE(self): prev_LC_COLLATE = os.environ.get("LC_COLLATE") try: - os.environ["LANG"] = "UNKNOWN_LANG" - os.environ.pop("LANGUAGE", None) - os.environ["LC_CTYPE"] = "UNKNOWN_CTYPE" - os.environ.pop("LC_COLLATE", None) - - assert os.environ.get("LANG") == "UNKNOWN_LANG" - assert not ("LANGUAGE" in os.environ.keys()) - assert os.environ.get("LC_CTYPE") == "UNKNOWN_CTYPE" - assert not ("LC_COLLATE" in os.environ.keys()) - - while True: - try: - with get_remote_node(conn_params=conn_params): - pass - except testgres.exceptions.ExecUtilException as e: - # warning: setlocale: LC_CTYPE: cannot change locale (UNKNOWN_CTYPE): No such file or directory - # postgres (PostgreSQL) 14.12 - errMsg = str(e) - assert "LC_CTYPE" in errMsg - assert "UNKNOWN_CTYPE" in errMsg - assert "warning: setlocale: LC_CTYPE: cannot change locale (UNKNOWN_CTYPE): No such file or directory" in errMsg - assert "postgres" in errMsg - break - raise Exception("We expected an error!") + # TODO: Pass unkData through test parameter. + unkDatas = [ + ("UNKNOWN_LANG", "UNKNOWN_CTYPE"), + ("\"UNKNOWN_LANG\"", "\"UNKNOWN_CTYPE\""), + ("\\UNKNOWN_LANG\\", "\\UNKNOWN_CTYPE\\"), + ("\"UNKNOWN_LANG", "UNKNOWN_CTYPE\""), + ("\\UNKNOWN_LANG", "UNKNOWN_CTYPE\\"), + ("\\", "\\"), + ("\"", "\""), + ] + + for unkData in unkDatas: + logging.info("----------------------") + logging.info("Unk LANG is [{0}]".format(unkData[0])) + logging.info("Unk LC_CTYPE is [{0}]".format(unkData[1])) + + os.environ["LANG"] = unkData[0] + os.environ.pop("LANGUAGE", None) + os.environ["LC_CTYPE"] = unkData[1] + os.environ.pop("LC_COLLATE", None) + + assert os.environ.get("LANG") == unkData[0] + assert not ("LANGUAGE" in os.environ.keys()) + assert os.environ.get("LC_CTYPE") == unkData[1] + assert not ("LC_COLLATE" in os.environ.keys()) + + while True: + try: + with get_remote_node(conn_params=conn_params): + pass + except testgres.exceptions.ExecUtilException as e: + # + # Example of an error message: + # + # warning: setlocale: LC_CTYPE: cannot change locale (UNKNOWN_CTYPE): No such file or directory + # postgres (PostgreSQL) 14.12 + # + errMsg = str(e) + + logging.info("Error message is: {0}".format(errMsg)) + + assert "LC_CTYPE" in errMsg + assert unkData[1] in errMsg + assert "warning: setlocale: LC_CTYPE: cannot change locale (" + unkData[1] + "): No such file or directory" in errMsg + assert "postgres" in errMsg + break + raise Exception("We expected an error!") finally: __class__.helper__restore_envvar("LANG", prev_LANG) __class__.helper__restore_envvar("LANGUAGE", prev_LANGUAGE) From ee78bcd2fc7002a1c3b5936425b3709c49d136aa Mon Sep 17 00:00:00 2001 From: "d.kovalenko" Date: Fri, 21 Feb 2025 19:09:43 +0300 Subject: [PATCH 5/6] Static methods are marked with @staticmethod [thanks to Victoria Shepard] The following methods of RemoteOperations were corrected: - _make_exec_env_list - _does_put_envvar_into_exec_cmd - _quote_envvar --- testgres/operations/remote_ops.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/testgres/operations/remote_ops.py b/testgres/operations/remote_ops.py index b4d65e95..f368740b 100644 --- a/testgres/operations/remote_ops.py +++ b/testgres/operations/remote_ops.py @@ -515,6 +515,7 @@ def db_connect(self, dbname, user, password=None, host="localhost", port=5432): ) return conn + @staticmethod def _make_exec_env_list() -> list[str]: result = list[str]() for envvar in os.environ.items(): @@ -529,6 +530,7 @@ def _make_exec_env_list() -> list[str]: sm_envs_for_exec_cmd = ["LANG", "LANGUAGE"] + @staticmethod def _does_put_envvar_into_exec_cmd(name: str) -> bool: assert type(name) == str # noqa: E721 name = name.upper() @@ -538,6 +540,7 @@ def _does_put_envvar_into_exec_cmd(name: str) -> bool: return True return False + @staticmethod def _quote_envvar(value) -> str: assert type(value) == str # noqa: E721 result = "\"" From 5564938c8af955f2f6bde67c57ca003d759c94ce Mon Sep 17 00:00:00 2001 From: "d.kovalenko" Date: Fri, 21 Feb 2025 19:13:28 +0300 Subject: [PATCH 6/6] TestRemoteOperations::_quote_envvar is updated (typification) --- testgres/operations/remote_ops.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/testgres/operations/remote_ops.py b/testgres/operations/remote_ops.py index f368740b..af4c59f9 100644 --- a/testgres/operations/remote_ops.py +++ b/testgres/operations/remote_ops.py @@ -541,7 +541,7 @@ def _does_put_envvar_into_exec_cmd(name: str) -> bool: return False @staticmethod - def _quote_envvar(value) -> str: + def _quote_envvar(value: str) -> str: assert type(value) == str # noqa: E721 result = "\"" for ch in value:

pFad - (p)hone/(F)rame/(a)nonymizer/(d)eclutterfier! Saves Data!