From f37c299e2f8adde503842c3dc2e95a4444917745 Mon Sep 17 00:00:00 2001 From: "d.kovalenko" Date: Sun, 23 Feb 2025 21:43:42 +0300 Subject: [PATCH 01/36] Using pytest [pytest.raises] --- tests/test_simple.py | 78 ++++++++++++++++++++----------------- tests/test_simple_remote.py | 64 ++++++++++++++++-------------- 2 files changed, 76 insertions(+), 66 deletions(-) diff --git a/tests/test_simple.py b/tests/test_simple.py index a751f0a3..9d5388fe 100644 --- a/tests/test_simple.py +++ b/tests/test_simple.py @@ -9,6 +9,7 @@ import time import six import unittest +import pytest import psutil import logging.config @@ -134,7 +135,7 @@ def test_custom_init(self): def test_double_init(self): with get_new_node().init() as node: # can't initialize node more than once - with self.assertRaises(InitNodeException): + with pytest.raises(expected_exception=InitNodeException): node.init() def test_init_after_cleanup(self): @@ -172,7 +173,7 @@ def test_init_unique_system_id(self): def test_node_exit(self): base_dir = None - with self.assertRaises(QueryException): + with pytest.raises(expected_exception=QueryException): with get_new_node().init() as node: base_dir = node.base_dir node.safe_psql('select 1') @@ -196,7 +197,7 @@ def test_double_start(self): def test_uninitialized_start(self): with get_new_node() as node: # node is not initialized yet - with self.assertRaises(StartNodeException): + with pytest.raises(expected_exception=StartNodeException): node.start() def test_restart(self): @@ -211,7 +212,7 @@ def test_restart(self): self.assertEqual(res, [(2, )]) # restart, fail - with self.assertRaises(StartNodeException): + with pytest.raises(expected_exception=StartNodeException): node.append_conf('pg_hba.conf', 'DUMMY') node.restart() @@ -303,13 +304,13 @@ def test_psql(self): self.assertEqual(rm_carriage_returns(_sum), b'6\n') # check psql's default args, fails - with self.assertRaises(QueryException): + with pytest.raises(expected_exception=QueryException): node.psql() node.stop() # check psql on stopped node, fails - with self.assertRaises(QueryException): + with pytest.raises(expected_exception=QueryException): node.safe_psql('select 1') def test_safe_psql__expect_error(self): @@ -320,11 +321,12 @@ def test_safe_psql__expect_error(self): self.assertIn('ERROR: syntax error at or near "select_or_not_select"', err) # --------- - with self.assertRaises(InvalidOperationException) as ctx: + with pytest.raises( + expected_exception=InvalidOperationException, + match="^" + re.escape("Exception was expected, but query finished successfully: `select 1;`.") + "$" + ): node.safe_psql("select 1;", expect_error=True) - self.assertEqual(str(ctx.exception), "Exception was expected, but query finished successfully: `select 1;`.") - # --------- res = node.safe_psql("select 1;", expect_error=False) self.assertEqual(rm_carriage_returns(res), b'1\n') @@ -357,7 +359,7 @@ def test_control_data(self): with get_new_node() as node: # node is not initialized yet - with self.assertRaises(ExecUtilException): + with pytest.raises(expected_exception=ExecUtilException): node.get_control_data() node.init() @@ -374,7 +376,7 @@ def test_backup_simple(self): master.init(allow_streaming=True) # node must be running - with self.assertRaises(BackupException): + with pytest.raises(expected_exception=BackupException): master.backup() # it's time to start node @@ -411,15 +413,17 @@ def test_backup_exhaust(self): pass # now let's try to create one more node - with self.assertRaises(BackupException): + with pytest.raises(expected_exception=BackupException): backup.spawn_primary() def test_backup_wrong_xlog_method(self): with get_new_node() as node: node.init(allow_streaming=True).start() - with self.assertRaises(BackupException, - msg='Invalid xlog_method "wrong"'): + with pytest.raises( + expected_exception=BackupException, + match="^" + re.escape('Invalid xlog_method "wrong"') + "$" + ): node.backup(xlog_method='wrong') def test_pg_ctl_wait_option(self): @@ -551,7 +555,7 @@ def test_logical_replication(self): self.assertListEqual(res, [(1, 1), (2, 2), (3, 3), (4, 4)]) # explicitly add table - with self.assertRaises(ValueError): + with pytest.raises(expected_exception=ValueError): pub.add_tables([]) # fail pub.add_tables(['test2']) node1.safe_psql('insert into test2 values (\'c\')') @@ -588,7 +592,7 @@ def test_logical_catchup(self): @unittest.skipIf(pg_version_ge('10'), 'requires <10') def test_logical_replication_fail(self): with get_new_node() as node: - with self.assertRaises(InitNodeException): + with pytest.raises(expected_exception=InitNodeException): node.init(allow_logical=True) def test_replication_slots(self): @@ -599,7 +603,7 @@ def test_replication_slots(self): replica.execute('select 1') # cannot create new slot with the same name - with self.assertRaises(TestgresException): + with pytest.raises(expected_exception=TestgresException): node.replicate(slot='slot1') def test_incorrect_catchup(self): @@ -607,7 +611,7 @@ def test_incorrect_catchup(self): node.init(allow_streaming=True).start() # node has no master, can't catch up - with self.assertRaises(TestgresException): + with pytest.raises(expected_exception=TestgresException): node.catchup() def test_promotion(self): @@ -664,12 +668,12 @@ def test_poll_query_until(self): self.assertTrue(end_time - start_time >= 5) # check 0 columns - with self.assertRaises(QueryException): + with pytest.raises(expected_exception=QueryException): node.poll_query_until( query='select from pg_catalog.pg_class limit 1') # check None, fail - with self.assertRaises(QueryException): + with pytest.raises(expected_exception=QueryException): node.poll_query_until(query='create table abc (val int)') # check None, ok @@ -682,7 +686,7 @@ def test_poll_query_until(self): expected=None) # check arbitrary expected value, fail - with self.assertRaises(TimeoutException): + with pytest.raises(expected_exception=TimeoutException): node.poll_query_until(query='select 3', expected=1, max_attempts=3, @@ -692,17 +696,17 @@ def test_poll_query_until(self): node.poll_query_until(query='select 2', expected=2) # check timeout - with self.assertRaises(TimeoutException): + with pytest.raises(expected_exception=TimeoutException): node.poll_query_until(query='select 1 > 2', max_attempts=3, sleep_time=0.01) # check ProgrammingError, fail - with self.assertRaises(testgres.ProgrammingError): + with pytest.raises(expected_exception=testgres.ProgrammingError): node.poll_query_until(query='dummy1') # check ProgrammingError, ok - with self.assertRaises(TimeoutException): + with pytest.raises(expected_exception=(TimeoutException)): node.poll_query_until(query='dummy2', max_attempts=3, sleep_time=0.01, @@ -809,11 +813,11 @@ def test_pg_config(self): def test_config_stack(self): # no such option - with self.assertRaises(TypeError): + with pytest.raises(expected_exception=TypeError): configure_testgres(dummy=True) # we have only 1 config in stack - with self.assertRaises(IndexError): + with pytest.raises(expected_exception=IndexError): pop_config() d0 = TestgresConfig.cached_initdb_dir @@ -827,7 +831,7 @@ def test_config_stack(self): stack_size = len(testgres.config.config_stack) # try to break a stack - with self.assertRaises(TypeError): + with pytest.raises(expected_exception=TypeError): with scoped_config(dummy=True): pass @@ -903,7 +907,7 @@ def test_isolation_levels(self): con.begin(IsolationLevel.Serializable).commit() # check wrong level - with self.assertRaises(QueryException): + with pytest.raises(expected_exception=QueryException): con.begin('Garbage').commit() def test_ports_management(self): @@ -980,7 +984,7 @@ def test_child_pids(self): with get_new_node().init().start() as master: # master node doesn't have a source walsender! - with self.assertRaises(TestgresException): + with pytest.raises(expected_exception=TestgresException): master.source_walsender with master.connect() as con: @@ -1008,7 +1012,7 @@ def test_child_pids(self): replica.stop() # there should be no walsender after we've stopped replica - with self.assertRaises(TestgresException): + with pytest.raises(expected_exception=TestgresException): replica.source_walsender def test_child_process_dies(self): @@ -1061,11 +1065,12 @@ def test_the_same_port(self): self.assertEqual(node2.port, node.port) self.assertFalse(node2._should_free_port) - with self.assertRaises(StartNodeException) as ctx: + with pytest.raises( + expected_exception=StartNodeException, + match=re.escape("Cannot start node") + ): node2.init().start() - self.assertIn("Cannot start node", str(ctx.exception)) - # node is still working self.assertEqual(node.port, node_port_copy) self.assertTrue(node._should_free_port) @@ -1218,11 +1223,12 @@ def test_port_conflict(self): self.assertTrue(node2._should_free_port) self.assertEqual(node2.port, node1.port) - with self.assertRaises(StartNodeException) as ctx: + with pytest.raises( + expected_exception=StartNodeException, + match=re.escape("Cannot start node after multiple attempts") + ): node2.init().start() - self.assertIn("Cannot start node after multiple attempts", str(ctx.exception)) - self.assertEqual(node2.port, node1.port) self.assertTrue(node2._should_free_port) self.assertEqual(__class__.tagPortManagerProxy.sm_DummyPortCurrentUsage, 1) diff --git a/tests/test_simple_remote.py b/tests/test_simple_remote.py index 2b581ac9..c548de03 100755 --- a/tests/test_simple_remote.py +++ b/tests/test_simple_remote.py @@ -10,6 +10,7 @@ import time import six import unittest +import pytest import psutil import logging.config @@ -195,7 +196,7 @@ def test_init__unk_LANG_and_LC_CTYPE(self): def test_double_init(self): with get_remote_node(conn_params=conn_params).init() as node: # can't initialize node more than once - with self.assertRaises(InitNodeException): + with pytest.raises(expected_exception=InitNodeException): node.init() def test_init_after_cleanup(self): @@ -230,7 +231,7 @@ def test_init_unique_system_id(self): self.assertGreater(id2, id1) def test_node_exit(self): - with self.assertRaises(QueryException): + with pytest.raises(expected_exception=QueryException): with get_remote_node(conn_params=conn_params).init() as node: base_dir = node.base_dir node.safe_psql('select 1') @@ -254,7 +255,7 @@ def test_double_start(self): def test_uninitialized_start(self): with get_remote_node(conn_params=conn_params) as node: # node is not initialized yet - with self.assertRaises(StartNodeException): + with pytest.raises(expected_exception=StartNodeException): node.start() def test_restart(self): @@ -269,7 +270,7 @@ def test_restart(self): self.assertEqual(res, [(2,)]) # restart, fail - with self.assertRaises(StartNodeException): + with pytest.raises(expected_exception=StartNodeException): node.append_conf('pg_hba.conf', 'DUMMY') node.restart() @@ -360,13 +361,13 @@ def test_psql(self): self.assertEqual(_sum, b'6\n') # check psql's default args, fails - with self.assertRaises(QueryException): + with pytest.raises(expected_exception=QueryException): node.psql() node.stop() # check psql on stopped node, fails - with self.assertRaises(QueryException): + with pytest.raises(expected_exception=QueryException): node.safe_psql('select 1') def test_safe_psql__expect_error(self): @@ -377,11 +378,12 @@ def test_safe_psql__expect_error(self): self.assertIn('ERROR: syntax error at or near "select_or_not_select"', err) # --------- - with self.assertRaises(InvalidOperationException) as ctx: + with pytest.raises( + expected_exception=InvalidOperationException, + match="^" + re.escape("Exception was expected, but query finished successfully: `select 1;`.") + "$" + ): node.safe_psql("select 1;", expect_error=True) - self.assertEqual(str(ctx.exception), "Exception was expected, but query finished successfully: `select 1;`.") - # --------- res = node.safe_psql("select 1;", expect_error=False) self.assertEqual(res, b'1\n') @@ -412,7 +414,7 @@ def test_transactions(self): def test_control_data(self): with get_remote_node(conn_params=conn_params) as node: # node is not initialized yet - with self.assertRaises(ExecUtilException): + with pytest.raises(expected_exception=ExecUtilException): node.get_control_data() node.init() @@ -428,7 +430,7 @@ def test_backup_simple(self): master.init(allow_streaming=True) # node must be running - with self.assertRaises(BackupException): + with pytest.raises(expected_exception=BackupException): master.backup() # it's time to start node @@ -465,15 +467,17 @@ def test_backup_exhaust(self): pass # now let's try to create one more node - with self.assertRaises(BackupException): + with pytest.raises(expected_exception=BackupException): backup.spawn_primary() def test_backup_wrong_xlog_method(self): with get_remote_node(conn_params=conn_params) as node: node.init(allow_streaming=True).start() - with self.assertRaises(BackupException, - msg='Invalid xlog_method "wrong"'): + with pytest.raises( + expected_exception=BackupException, + match="^" + re.escape('Invalid xlog_method "wrong"') + "$" + ): node.backup(xlog_method='wrong') def test_pg_ctl_wait_option(self): @@ -605,7 +609,7 @@ def test_logical_replication(self): self.assertListEqual(res, [(1, 1), (2, 2), (3, 3), (4, 4)]) # explicitly add table - with self.assertRaises(ValueError): + with pytest.raises(expected_exception=ValueError): pub.add_tables([]) # fail pub.add_tables(['test2']) node1.safe_psql('insert into test2 values (\'c\')') @@ -642,7 +646,7 @@ def test_logical_catchup(self): @unittest.skipIf(pg_version_ge('10'), 'requires <10') def test_logical_replication_fail(self): with get_remote_node(conn_params=conn_params) as node: - with self.assertRaises(InitNodeException): + with pytest.raises(expected_exception=InitNodeException): node.init(allow_logical=True) def test_replication_slots(self): @@ -653,7 +657,7 @@ def test_replication_slots(self): replica.execute('select 1') # cannot create new slot with the same name - with self.assertRaises(TestgresException): + with pytest.raises(expected_exception=TestgresException): node.replicate(slot='slot1') def test_incorrect_catchup(self): @@ -661,7 +665,7 @@ def test_incorrect_catchup(self): node.init(allow_streaming=True).start() # node has no master, can't catch up - with self.assertRaises(TestgresException): + with pytest.raises(expected_exception=TestgresException): node.catchup() def test_promotion(self): @@ -717,12 +721,12 @@ def test_poll_query_until(self): self.assertTrue(end_time - start_time >= 5) # check 0 columns - with self.assertRaises(QueryException): + with pytest.raises(expected_exception=QueryException): node.poll_query_until( query='select from pg_catalog.pg_class limit 1') # check None, fail - with self.assertRaises(QueryException): + with pytest.raises(expected_exception=QueryException): node.poll_query_until(query='create table abc (val int)') # check None, ok @@ -735,7 +739,7 @@ def test_poll_query_until(self): expected=None) # check arbitrary expected value, fail - with self.assertRaises(TimeoutException): + with pytest.raises(expected_exception=TimeoutException): node.poll_query_until(query='select 3', expected=1, max_attempts=3, @@ -745,17 +749,17 @@ def test_poll_query_until(self): node.poll_query_until(query='select 2', expected=2) # check timeout - with self.assertRaises(TimeoutException): + with pytest.raises(expected_exception=TimeoutException): node.poll_query_until(query='select 1 > 2', max_attempts=3, sleep_time=0.01) # check ProgrammingError, fail - with self.assertRaises(testgres.ProgrammingError): + with pytest.raises(expected_exception=testgres.ProgrammingError): node.poll_query_until(query='dummy1') # check ProgrammingError, ok - with self.assertRaises(TimeoutException): + with pytest.raises(expected_exception=TimeoutException): node.poll_query_until(query='dummy2', max_attempts=3, sleep_time=0.01, @@ -857,11 +861,11 @@ def test_pg_config(self): def test_config_stack(self): # no such option - with self.assertRaises(TypeError): + with pytest.raises(expected_exception=TypeError): configure_testgres(dummy=True) # we have only 1 config in stack - with self.assertRaises(IndexError): + with pytest.raises(expected_exception=IndexError): pop_config() d0 = TestgresConfig.cached_initdb_dir @@ -875,7 +879,7 @@ def test_config_stack(self): stack_size = len(testgres.config.config_stack) # try to break a stack - with self.assertRaises(TypeError): + with pytest.raises(expected_exception=TypeError): with scoped_config(dummy=True): pass @@ -955,7 +959,7 @@ def test_isolation_levels(self): con.begin(IsolationLevel.Serializable).commit() # check wrong level - with self.assertRaises(QueryException): + with pytest.raises(expected_exception=QueryException): con.begin('Garbage').commit() def test_ports_management(self): @@ -1024,7 +1028,7 @@ def test_child_pids(self): with get_remote_node(conn_params=conn_params).init().start() as master: # master node doesn't have a source walsender! - with self.assertRaises(TestgresException): + with pytest.raises(expected_exception=TestgresException): master.source_walsender with master.connect() as con: @@ -1052,7 +1056,7 @@ def test_child_pids(self): replica.stop() # there should be no walsender after we've stopped replica - with self.assertRaises(TestgresException): + with pytest.raises(expected_exception=TestgresException): replica.source_walsender def test_child_process_dies(self): From 53e468e28b2851b42c90c5886bf89f6528b62c9b Mon Sep 17 00:00:00 2001 From: "d.kovalenko" Date: Sun, 23 Feb 2025 23:13:45 +0300 Subject: [PATCH 02/36] Using pytest [pytest.skip] --- tests/test_simple.py | 52 ++++++++++++++++++++++++++++++++----- tests/test_simple_remote.py | 45 +++++++++++++++++++++++++++----- 2 files changed, 83 insertions(+), 14 deletions(-) diff --git a/tests/test_simple.py b/tests/test_simple.py index 9d5388fe..936aed18 100644 --- a/tests/test_simple.py +++ b/tests/test_simple.py @@ -11,6 +11,7 @@ import unittest import pytest import psutil +import platform import logging.config @@ -144,10 +145,13 @@ def test_init_after_cleanup(self): node.cleanup() node.init().start().execute('select 1') - @unittest.skipUnless(util_exists('pg_resetwal.exe' if os.name == 'nt' else 'pg_resetwal'), 'pgbench might be missing') - @unittest.skipUnless(pg_version_ge('9.6'), 'requires 9.6+') + # @unittest.skipUnless(util_exists('pg_resetwal.exe' if os.name == 'nt' else 'pg_resetwal'), 'pgbench might be missing') + # @unittest.skipUnless(pg_version_ge('9.6'), 'requires 9.6+') def test_init_unique_system_id(self): # this function exists in PostgreSQL 9.6+ + __class__.helper__skip_test_if_util_not_exist("pg_resetwal") + __class__.helper__skip_test_if_pg_version_is_not_ge("9.6") + query = 'select system_identifier from pg_control_system()' with scoped_config(cache_initdb=False): @@ -453,8 +457,10 @@ def test_replicate(self): res = node.execute('select * from test') self.assertListEqual(res, []) - @unittest.skipUnless(pg_version_ge('9.6'), 'requires 9.6+') + # @unittest.skipUnless(pg_version_ge('9.6'), 'requires 9.6+') def test_synchronous_replication(self): + __class__.helper__skip_test_if_pg_version_is_not_ge("9.6") + with get_new_node() as master: old_version = not pg_version_ge('9.6') @@ -494,8 +500,10 @@ def test_synchronous_replication(self): res = standby1.safe_psql('select count(*) from abc') self.assertEqual(rm_carriage_returns(res), b'1000000\n') - @unittest.skipUnless(pg_version_ge('10'), 'requires 10+') + # @unittest.skipUnless(pg_version_ge('10'), 'requires 10+') def test_logical_replication(self): + __class__.helper__skip_test_if_pg_version_is_not_ge("10") + with get_new_node() as node1, get_new_node() as node2: node1.init(allow_logical=True) node1.start() @@ -563,9 +571,11 @@ def test_logical_replication(self): res = node2.execute('select * from test2') self.assertListEqual(res, [('a', ), ('b', )]) - @unittest.skipUnless(pg_version_ge('10'), 'requires 10+') + # @unittest.skipUnless(pg_version_ge('10'), 'requires 10+') def test_logical_catchup(self): """ Runs catchup for 100 times to be sure that it is consistent """ + __class__.helper__skip_test_if_pg_version_is_not_ge("10") + with get_new_node() as node1, get_new_node() as node2: node1.init(allow_logical=True) node1.start() @@ -589,8 +599,10 @@ def test_logical_catchup(self): )]) node1.execute('delete from test') - @unittest.skipIf(pg_version_ge('10'), 'requires <10') + # @unittest.skipIf(pg_version_ge('10'), 'requires <10') def test_logical_replication_fail(self): + __class__.helper__skip_test_if_pg_version_is_ge("10") + with get_new_node() as node: with pytest.raises(expected_exception=InitNodeException): node.init(allow_logical=True) @@ -766,8 +778,10 @@ def test_logging(self): master.restart() self.assertTrue(master._logger.is_alive()) - @unittest.skipUnless(util_exists('pgbench.exe' if os.name == 'nt' else 'pgbench'), 'pgbench might be missing') + # @unittest.skipUnless(util_exists('pgbench.exe' if os.name == 'nt' else 'pgbench'), 'pgbench might be missing') def test_pgbench(self): + __class__.helper__skip_test_if_util_not_exist("pgbench") + with get_new_node().init().start() as node: # initialize pgbench DB and run benchmarks @@ -1312,6 +1326,30 @@ def test_set_auto_conf(self): x[0] + " stored wrong" ) + @staticmethod + def helper__skip_test_if_util_not_exist(name: str): + assert type(name) == str # noqa: E721 + + if platform.system().lower() == "windows": + name2 = name + ".exe" + else: + name2 = name + + if not util_exists(name2): + pytest.skip('might be missing') + + @staticmethod + def helper__skip_test_if_pg_version_is_not_ge(version: str): + assert type(version) == str # noqa: E721 + if not pg_version_ge(version): + pytest.skip('requires {0}+'.format(version)) + + @staticmethod + def helper__skip_test_if_pg_version_is_ge(version: str): + assert type(version) == str # noqa: E721 + if pg_version_ge(version): + pytest.skip('requires <{0}'.format(version)) + if __name__ == '__main__': if os.environ.get('ALT_CONFIG'): diff --git a/tests/test_simple_remote.py b/tests/test_simple_remote.py index c548de03..28291acb 100755 --- a/tests/test_simple_remote.py +++ b/tests/test_simple_remote.py @@ -205,10 +205,13 @@ def test_init_after_cleanup(self): node.cleanup() node.init().start().execute('select 1') - @unittest.skipUnless(util_exists('pg_resetwal'), 'might be missing') - @unittest.skipUnless(pg_version_ge('9.6'), 'requires 9.6+') + # @unittest.skipUnless(util_exists('pg_resetwal'), 'might be missing') + # @unittest.skipUnless(pg_version_ge('9.6'), 'requires 9.6+') def test_init_unique_system_id(self): # this function exists in PostgreSQL 9.6+ + __class__.helper__skip_test_if_util_not_exist("pg_resetwal") + __class__.helper__skip_test_if_pg_version_is_not_ge('9.6') + query = 'select system_identifier from pg_control_system()' with scoped_config(cache_initdb=False): @@ -507,8 +510,10 @@ def test_replicate(self): res = node.execute('select * from test') self.assertListEqual(res, []) - @unittest.skipUnless(pg_version_ge('9.6'), 'requires 9.6+') + # @unittest.skipUnless(pg_version_ge('9.6'), 'requires 9.6+') def test_synchronous_replication(self): + __class__.helper__skip_test_if_pg_version_is_not_ge("9.6") + with get_remote_node(conn_params=conn_params) as master: old_version = not pg_version_ge('9.6') @@ -548,8 +553,10 @@ def test_synchronous_replication(self): res = standby1.safe_psql('select count(*) from abc') self.assertEqual(res, b'1000000\n') - @unittest.skipUnless(pg_version_ge('10'), 'requires 10+') + # @unittest.skipUnless(pg_version_ge('10'), 'requires 10+') def test_logical_replication(self): + __class__.helper__skip_test_if_pg_version_is_not_ge("10") + with get_remote_node(conn_params=conn_params) as node1, get_remote_node(conn_params=conn_params) as node2: node1.init(allow_logical=True) node1.start() @@ -617,9 +624,11 @@ def test_logical_replication(self): res = node2.execute('select * from test2') self.assertListEqual(res, [('a',), ('b',)]) - @unittest.skipUnless(pg_version_ge('10'), 'requires 10+') + # @unittest.skipUnless(pg_version_ge('10'), 'requires 10+') def test_logical_catchup(self): """ Runs catchup for 100 times to be sure that it is consistent """ + __class__.helper__skip_test_if_pg_version_is_not_ge("10") + with get_remote_node(conn_params=conn_params) as node1, get_remote_node(conn_params=conn_params) as node2: node1.init(allow_logical=True) node1.start() @@ -643,8 +652,10 @@ def test_logical_catchup(self): )]) node1.execute('delete from test') - @unittest.skipIf(pg_version_ge('10'), 'requires <10') + # @unittest.skipIf(pg_version_ge('10'), 'requires <10') def test_logical_replication_fail(self): + __class__.helper__skip_test_if_pg_version_is_ge("10") + with get_remote_node(conn_params=conn_params) as node: with pytest.raises(expected_exception=InitNodeException): node.init(allow_logical=True) @@ -820,8 +831,10 @@ def test_logging(self): master.restart() self.assertTrue(master._logger.is_alive()) - @unittest.skipUnless(util_exists('pgbench'), 'might be missing') + # @unittest.skipUnless(util_exists('pgbench'), 'might be missing') def test_pgbench(self): + __class__.helper__skip_test_if_util_not_exist("pgbench") + with get_remote_node(conn_params=conn_params).init().start() as node: # initialize pgbench DB and run benchmarks node.pgbench_init(scale=2, foreign_keys=True, @@ -1077,6 +1090,24 @@ def helper__restore_envvar(name, prev_value): else: os.environ[name] = prev_value + @staticmethod + def helper__skip_test_if_util_not_exist(name: str): + assert type(name) == str # noqa: E721 + if not util_exists(name): + pytest.skip('might be missing') + + @staticmethod + def helper__skip_test_if_pg_version_is_not_ge(version: str): + assert type(version) == str # noqa: E721 + if not pg_version_ge(version): + pytest.skip('requires {0}+'.format(version)) + + @staticmethod + def helper__skip_test_if_pg_version_is_ge(version: str): + assert type(version) == str # noqa: E721 + if pg_version_ge(version): + pytest.skip('requires <{0}'.format(version)) + if __name__ == '__main__': if os_ops.environ('ALT_CONFIG'): From 12d702ad7e653cfe29ff62790125c12c4b151feb Mon Sep 17 00:00:00 2001 From: "d.kovalenko" Date: Sun, 23 Feb 2025 23:29:44 +0300 Subject: [PATCH 03/36] Using pytest [assertIsNotNone] --- tests/test_simple.py | 4 ++-- tests/test_simple_remote.py | 5 +++-- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/tests/test_simple.py b/tests/test_simple.py index 936aed18..23a0f581 100644 --- a/tests/test_simple.py +++ b/tests/test_simple.py @@ -111,7 +111,7 @@ class TestgresTests(unittest.TestCase): def test_node_repr(self): with get_new_node() as node: pattern = r"PostgresNode\(name='.+', port=.+, base_dir='.+'\)" - self.assertIsNotNone(re.match(pattern, str(node))) + assert re.match(pattern, str(node)) is not None def test_custom_init(self): with get_new_node() as node: @@ -370,7 +370,7 @@ def test_control_data(self): data = node.get_control_data() # check returned dict - self.assertIsNotNone(data) + assert data is not None self.assertTrue(any('pg_control' in s for s in data.keys())) def test_backup_simple(self): diff --git a/tests/test_simple_remote.py b/tests/test_simple_remote.py index 28291acb..daad68a2 100755 --- a/tests/test_simple_remote.py +++ b/tests/test_simple_remote.py @@ -99,7 +99,7 @@ class TestgresRemoteTests(unittest.TestCase): def test_node_repr(self): with get_remote_node(conn_params=conn_params) as node: pattern = r"PostgresNode\(name='.+', port=.+, base_dir='.+'\)" - self.assertIsNotNone(re.match(pattern, str(node))) + assert re.match(pattern, str(node)) is not None def test_custom_init(self): with get_remote_node(conn_params=conn_params) as node: @@ -424,7 +424,7 @@ def test_control_data(self): data = node.get_control_data() # check returned dict - self.assertIsNotNone(data) + assert data is not None self.assertTrue(any('pg_control' in s for s in data.keys())) def test_backup_simple(self): @@ -1084,6 +1084,7 @@ def test_child_process_dies(self): # try to handle children list -- missing processes will have ptype "ProcessType.Unknown" [ProcessProxy(p) for p in children] + @staticmethod def helper__restore_envvar(name, prev_value): if prev_value is None: os.environ.pop(name, None) From 1cb664e5cd855cc63e0c6990dc689dc7fece8f6f Mon Sep 17 00:00:00 2001 From: "d.kovalenko" Date: Sun, 23 Feb 2025 23:35:58 +0300 Subject: [PATCH 04/36] Using pytest [assertFalse] --- tests/test_simple.py | 14 +++++++------- tests/test_simple_remote.py | 10 +++++----- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/tests/test_simple.py b/tests/test_simple.py index 23a0f581..293aee6c 100644 --- a/tests/test_simple.py +++ b/tests/test_simple.py @@ -131,7 +131,7 @@ def test_custom_init(self): self.assertGreaterEqual(len(lines), 6) # there should be no trust entries at all - self.assertFalse(any('trust' in s for s in lines)) + assert not (any('trust' in s for s in lines)) def test_double_init(self): with get_new_node().init() as node: @@ -190,7 +190,7 @@ def test_node_exit(self): base_dir = node.base_dir # should have been removed by default - self.assertFalse(os.path.exists(base_dir)) + assert not (os.path.exists(base_dir)) def test_double_start(self): with get_new_node().init().start() as node: @@ -245,8 +245,8 @@ def test_pg_ctl(self): def test_status(self): self.assertTrue(NodeStatus.Running) - self.assertFalse(NodeStatus.Stopped) - self.assertFalse(NodeStatus.Uninitialized) + assert not (NodeStatus.Stopped) + assert not (NodeStatus.Uninitialized) # check statuses after each operation with get_new_node() as node: @@ -812,7 +812,7 @@ def test_pg_config(self): # modify setting for this scope with scoped_config(cache_pg_config=False) as config: # sanity check for value - self.assertFalse(config.cache_pg_config) + assert not (config.cache_pg_config) # save right after config change c2 = get_pg_config() @@ -1077,7 +1077,7 @@ def test_the_same_port(self): with get_new_node(port=node.port) as node2: self.assertEqual(type(node2.port), int) self.assertEqual(node2.port, node.port) - self.assertFalse(node2._should_free_port) + assert not (node2._should_free_port) with pytest.raises( expected_exception=StartNodeException, @@ -1247,7 +1247,7 @@ def test_port_conflict(self): self.assertTrue(node2._should_free_port) self.assertEqual(__class__.tagPortManagerProxy.sm_DummyPortCurrentUsage, 1) self.assertEqual(__class__.tagPortManagerProxy.sm_DummyPortTotalUsage, C_COUNT_OF_BAD_PORT_USAGE) - self.assertFalse(node2.is_started) + assert not (node2.is_started) # node2 must release our dummyPort (node1.port) self.assertEqual(__class__.tagPortManagerProxy.sm_DummyPortCurrentUsage, 0) diff --git a/tests/test_simple_remote.py b/tests/test_simple_remote.py index daad68a2..102f0eb5 100755 --- a/tests/test_simple_remote.py +++ b/tests/test_simple_remote.py @@ -118,7 +118,7 @@ def test_custom_init(self): self.assertGreaterEqual(len(lines), 6) # there should be no trust entries at all - self.assertFalse(any('trust' in s for s in lines)) + assert not (any('trust' in s for s in lines)) def test_init__LANG_ะก(self): # PBCKP-1744 @@ -247,7 +247,7 @@ def test_node_exit(self): base_dir = node.base_dir # should have been removed by default - self.assertFalse(os_ops.path_exists(base_dir)) + assert not (os_ops.path_exists(base_dir)) def test_double_start(self): with get_remote_node(conn_params=conn_params).init().start() as node: @@ -302,8 +302,8 @@ def test_pg_ctl(self): def test_status(self): self.assertTrue(NodeStatus.Running) - self.assertFalse(NodeStatus.Stopped) - self.assertFalse(NodeStatus.Uninitialized) + assert not (NodeStatus.Stopped) + assert not (NodeStatus.Uninitialized) # check statuses after each operation with get_remote_node(conn_params=conn_params) as node: @@ -859,7 +859,7 @@ def test_pg_config(self): # modify setting for this scope with scoped_config(cache_pg_config=False) as config: # sanity check for value - self.assertFalse(config.cache_pg_config) + assert not (config.cache_pg_config) # save right after config change c2 = get_pg_config() From e2e7e8b6d7d26bfa08a7e612a3ab4dc307b7ba11 Mon Sep 17 00:00:00 2001 From: "d.kovalenko" Date: Sun, 23 Feb 2025 23:38:10 +0300 Subject: [PATCH 05/36] Using pytest [assertTrue] --- tests/test_simple.py | 84 ++++++++++++++++++------------------- tests/test_simple_remote.py | 52 +++++++++++------------ 2 files changed, 68 insertions(+), 68 deletions(-) diff --git a/tests/test_simple.py b/tests/test_simple.py index 293aee6c..3489173b 100644 --- a/tests/test_simple.py +++ b/tests/test_simple.py @@ -160,8 +160,8 @@ def test_init_unique_system_id(self): with scoped_config(cache_initdb=True, cached_initdb_unique=True) as config: - self.assertTrue(config.cache_initdb) - self.assertTrue(config.cached_initdb_unique) + assert (config.cache_initdb) + assert (config.cached_initdb_unique) # spawn two nodes; ids must be different with get_new_node().init().start() as node1, \ @@ -183,7 +183,7 @@ def test_node_exit(self): node.safe_psql('select 1') # we should save the DB for "debugging" - self.assertTrue(os.path.exists(base_dir)) + assert (os.path.exists(base_dir)) rmtree(base_dir, ignore_errors=True) with get_new_node().init() as node: @@ -196,7 +196,7 @@ def test_double_start(self): with get_new_node().init().start() as node: # can't start node more than once node.start() - self.assertTrue(node.is_started) + assert (node.is_started) def test_uninitialized_start(self): with get_new_node() as node: @@ -241,10 +241,10 @@ def test_pg_ctl(self): node.init().start() status = node.pg_ctl(['status']) - self.assertTrue('PID' in status) + assert ('PID' in status) def test_status(self): - self.assertTrue(NodeStatus.Running) + assert (NodeStatus.Running) assert not (NodeStatus.Stopped) assert not (NodeStatus.Uninitialized) @@ -320,7 +320,7 @@ def test_psql(self): def test_safe_psql__expect_error(self): with get_new_node().init().start() as node: err = node.safe_psql('select_or_not_select 1', expect_error=True) - self.assertTrue(type(err) == str) # noqa: E721 + assert (type(err) == str) # noqa: E721 self.assertIn('select_or_not_select', err) self.assertIn('ERROR: syntax error at or near "select_or_not_select"', err) @@ -371,7 +371,7 @@ def test_control_data(self): # check returned dict assert data is not None - self.assertTrue(any('pg_control' in s for s in data.keys())) + assert (any('pg_control' in s for s in data.keys())) def test_backup_simple(self): with get_new_node() as master: @@ -651,9 +651,9 @@ def test_dump(self): with removing(node1.dump(format=format)) as dump: with get_new_node().init().start() as node3: if format == 'directory': - self.assertTrue(os.path.isdir(dump)) + assert (os.path.isdir(dump)) else: - self.assertTrue(os.path.isfile(dump)) + assert (os.path.isfile(dump)) # restore dump node3.restore(filename=dump) res = node3.execute(query_select) @@ -677,7 +677,7 @@ def test_poll_query_until(self): node.poll_query_until(query=check_time.format(start_time)) end_time = node.execute(get_time)[0][0] - self.assertTrue(end_time - start_time >= 5) + assert (end_time - start_time >= 5) # check 0 columns with pytest.raises(expected_exception=QueryException): @@ -770,13 +770,13 @@ def test_logging(self): # check that master's port is found with open(logfile.name, 'r') as log: lines = log.readlines() - self.assertTrue(any(node_name in s for s in lines)) + assert (any(node_name in s for s in lines)) # test logger after stop/start/restart master.stop() master.start() master.restart() - self.assertTrue(master._logger.is_alive()) + assert (master._logger.is_alive()) # @unittest.skipUnless(util_exists('pgbench.exe' if os.name == 'nt' else 'pgbench'), 'pgbench might be missing') def test_pgbench(self): @@ -798,7 +798,7 @@ def test_pgbench(self): proc.stdout.close() - self.assertTrue('tps' in out) + assert ('tps' in out) def test_pg_config(self): # check same instances @@ -872,13 +872,13 @@ def test_auto_name(self): with get_new_node().init(allow_streaming=True).start() as m: with m.replicate().start() as r: # check that nodes are running - self.assertTrue(m.status()) - self.assertTrue(r.status()) + assert (m.status()) + assert (r.status()) # check their names self.assertNotEqual(m.name, r.name) - self.assertTrue('testgres' in m.name) - self.assertTrue('testgres' in r.name) + assert ('testgres' in m.name) + assert ('testgres' in r.name) def test_file_tail(self): from testgres.utils import file_tail @@ -957,21 +957,21 @@ def test_version_management(self): g = PgVer('15.3.1bihabeta1') k = PgVer('15.3.1') - self.assertTrue(a == b) - self.assertTrue(b > c) - self.assertTrue(a > c) - self.assertTrue(d > e) - self.assertTrue(e > f) - self.assertTrue(d > f) - self.assertTrue(h > f) - self.assertTrue(h == i) - self.assertTrue(g == k) - self.assertTrue(g > h) + assert (a == b) + assert (b > c) + assert (a > c) + assert (d > e) + assert (e > f) + assert (d > f) + assert (h > f) + assert (h == i) + assert (g == k) + assert (g > h) version = get_pg_version() with get_new_node() as node: - self.assertTrue(isinstance(version, six.string_types)) - self.assertTrue(isinstance(node.version, PgVer)) + assert (isinstance(version, six.string_types)) + assert (isinstance(node.version, PgVer)) self.assertEqual(node.version, PgVer(version)) def test_child_pids(self): @@ -1054,7 +1054,7 @@ def test_upgrade_node(self): node_new.init(cached=False) res = node_new.upgrade_from(old_node=node_old) node_new.start() - self.assertTrue(b'Upgrade Complete' in res) + assert (b'Upgrade Complete' in res) def test_parse_pg_version(self): # Linux Mint @@ -1069,7 +1069,7 @@ def test_parse_pg_version(self): def test_the_same_port(self): with get_new_node() as node: node.init().start() - self.assertTrue(node._should_free_port) + assert (node._should_free_port) self.assertEqual(type(node.port), int) node_port_copy = node.port self.assertEqual(rm_carriage_returns(node.safe_psql("SELECT 1;")), b'1\n') @@ -1087,7 +1087,7 @@ def test_the_same_port(self): # node is still working self.assertEqual(node.port, node_port_copy) - self.assertTrue(node._should_free_port) + assert (node._should_free_port) self.assertEqual(rm_carriage_returns(node.safe_psql("SELECT 3;")), b'3\n') class tagPortManagerProxy: @@ -1193,7 +1193,7 @@ def test_port_rereserve_during_node_start(self): with get_new_node() as node1: node1.init().start() - self.assertTrue(node1._should_free_port) + assert (node1._should_free_port) self.assertEqual(type(node1.port), int) # noqa: E721 node1_port_copy = node1.port self.assertEqual(rm_carriage_returns(node1.safe_psql("SELECT 1;")), b'1\n') @@ -1201,22 +1201,22 @@ def test_port_rereserve_during_node_start(self): with __class__.tagPortManagerProxy(node1.port, C_COUNT_OF_BAD_PORT_USAGE): assert __class__.tagPortManagerProxy.sm_DummyPortNumber == node1.port with get_new_node() as node2: - self.assertTrue(node2._should_free_port) + assert (node2._should_free_port) self.assertEqual(node2.port, node1.port) node2.init().start() self.assertNotEqual(node2.port, node1.port) - self.assertTrue(node2._should_free_port) + assert (node2._should_free_port) self.assertEqual(__class__.tagPortManagerProxy.sm_DummyPortCurrentUsage, 0) self.assertEqual(__class__.tagPortManagerProxy.sm_DummyPortTotalUsage, C_COUNT_OF_BAD_PORT_USAGE) - self.assertTrue(node2.is_started) + assert (node2.is_started) self.assertEqual(rm_carriage_returns(node2.safe_psql("SELECT 2;")), b'2\n') # node1 is still working self.assertEqual(node1.port, node1_port_copy) - self.assertTrue(node1._should_free_port) + assert (node1._should_free_port) self.assertEqual(rm_carriage_returns(node1.safe_psql("SELECT 3;")), b'3\n') def test_port_conflict(self): @@ -1226,7 +1226,7 @@ def test_port_conflict(self): with get_new_node() as node1: node1.init().start() - self.assertTrue(node1._should_free_port) + assert (node1._should_free_port) self.assertEqual(type(node1.port), int) # noqa: E721 node1_port_copy = node1.port self.assertEqual(rm_carriage_returns(node1.safe_psql("SELECT 1;")), b'1\n') @@ -1234,7 +1234,7 @@ def test_port_conflict(self): with __class__.tagPortManagerProxy(node1.port, C_COUNT_OF_BAD_PORT_USAGE): assert __class__.tagPortManagerProxy.sm_DummyPortNumber == node1.port with get_new_node() as node2: - self.assertTrue(node2._should_free_port) + assert (node2._should_free_port) self.assertEqual(node2.port, node1.port) with pytest.raises( @@ -1244,7 +1244,7 @@ def test_port_conflict(self): node2.init().start() self.assertEqual(node2.port, node1.port) - self.assertTrue(node2._should_free_port) + assert (node2._should_free_port) self.assertEqual(__class__.tagPortManagerProxy.sm_DummyPortCurrentUsage, 1) self.assertEqual(__class__.tagPortManagerProxy.sm_DummyPortTotalUsage, C_COUNT_OF_BAD_PORT_USAGE) assert not (node2.is_started) @@ -1254,7 +1254,7 @@ def test_port_conflict(self): # node1 is still working self.assertEqual(node1.port, node1_port_copy) - self.assertTrue(node1._should_free_port) + assert (node1._should_free_port) self.assertEqual(rm_carriage_returns(node1.safe_psql("SELECT 3;")), b'3\n') def test_simple_with_bin_dir(self): diff --git a/tests/test_simple_remote.py b/tests/test_simple_remote.py index 102f0eb5..f615e3aa 100755 --- a/tests/test_simple_remote.py +++ b/tests/test_simple_remote.py @@ -220,8 +220,8 @@ def test_init_unique_system_id(self): with scoped_config(cache_initdb=True, cached_initdb_unique=True) as config: - self.assertTrue(config.cache_initdb) - self.assertTrue(config.cached_initdb_unique) + assert (config.cache_initdb) + assert (config.cached_initdb_unique) # spawn two nodes; ids must be different with get_remote_node(conn_params=conn_params).init().start() as node1, \ @@ -240,7 +240,7 @@ def test_node_exit(self): node.safe_psql('select 1') # we should save the DB for "debugging" - self.assertTrue(os_ops.path_exists(base_dir)) + assert (os_ops.path_exists(base_dir)) os_ops.rmdirs(base_dir, ignore_errors=True) with get_remote_node(conn_params=conn_params).init() as node: @@ -253,7 +253,7 @@ def test_double_start(self): with get_remote_node(conn_params=conn_params).init().start() as node: # can't start node more than once node.start() - self.assertTrue(node.is_started) + assert (node.is_started) def test_uninitialized_start(self): with get_remote_node(conn_params=conn_params) as node: @@ -298,10 +298,10 @@ def test_pg_ctl(self): node.init().start() status = node.pg_ctl(['status']) - self.assertTrue('PID' in status) + assert ('PID' in status) def test_status(self): - self.assertTrue(NodeStatus.Running) + assert (NodeStatus.Running) assert not (NodeStatus.Stopped) assert not (NodeStatus.Uninitialized) @@ -376,7 +376,7 @@ def test_psql(self): def test_safe_psql__expect_error(self): with get_remote_node(conn_params=conn_params).init().start() as node: err = node.safe_psql('select_or_not_select 1', expect_error=True) - self.assertTrue(type(err) == str) # noqa: E721 + assert (type(err) == str) # noqa: E721 self.assertIn('select_or_not_select', err) self.assertIn('ERROR: syntax error at or near "select_or_not_select"', err) @@ -425,7 +425,7 @@ def test_control_data(self): # check returned dict assert data is not None - self.assertTrue(any('pg_control' in s for s in data.keys())) + assert (any('pg_control' in s for s in data.keys())) def test_backup_simple(self): with get_remote_node(conn_params=conn_params) as master: @@ -704,9 +704,9 @@ def test_dump(self): with removing(node1.dump(format=format)) as dump: with get_remote_node(conn_params=conn_params).init().start() as node3: if format == 'directory': - self.assertTrue(os_ops.isdir(dump)) + assert (os_ops.isdir(dump)) else: - self.assertTrue(os_ops.isfile(dump)) + assert (os_ops.isfile(dump)) # restore dump node3.restore(filename=dump) res = node3.execute(query_select) @@ -729,7 +729,7 @@ def test_poll_query_until(self): node.poll_query_until(query=check_time.format(start_time)) end_time = node.execute(get_time)[0][0] - self.assertTrue(end_time - start_time >= 5) + assert (end_time - start_time >= 5) # check 0 columns with pytest.raises(expected_exception=QueryException): @@ -823,13 +823,13 @@ def test_logging(self): # check that master's port is found with open(logfile.name, 'r') as log: lines = log.readlines() - self.assertTrue(any(node_name in s for s in lines)) + assert (any(node_name in s for s in lines)) # test logger after stop/start/restart master.stop() master.start() master.restart() - self.assertTrue(master._logger.is_alive()) + assert (master._logger.is_alive()) # @unittest.skipUnless(util_exists('pgbench'), 'might be missing') def test_pgbench(self): @@ -845,7 +845,7 @@ def test_pgbench(self): stderr=subprocess.STDOUT, options=['-T3']) out = proc.communicate()[0] - self.assertTrue(b'tps = ' in out) + assert (b'tps = ' in out) def test_pg_config(self): # check same instances @@ -923,13 +923,13 @@ def test_auto_name(self): with get_remote_node(conn_params=conn_params).init(allow_streaming=True).start() as m: with m.replicate().start() as r: # check that nodes are running - self.assertTrue(m.status()) - self.assertTrue(r.status()) + assert (m.status()) + assert (r.status()) # check their names self.assertNotEqual(m.name, r.name) - self.assertTrue('testgres' in m.name) - self.assertTrue('testgres' in r.name) + assert ('testgres' in m.name) + assert ('testgres' in r.name) def test_file_tail(self): from testgres.utils import file_tail @@ -1004,17 +1004,17 @@ def test_version_management(self): e = PgVer('15rc1') f = PgVer('15beta4') - self.assertTrue(a == b) - self.assertTrue(b > c) - self.assertTrue(a > c) - self.assertTrue(d > e) - self.assertTrue(e > f) - self.assertTrue(d > f) + assert (a == b) + assert (b > c) + assert (a > c) + assert (d > e) + assert (e > f) + assert (d > f) version = get_pg_version() with get_remote_node(conn_params=conn_params) as node: - self.assertTrue(isinstance(version, six.string_types)) - self.assertTrue(isinstance(node.version, PgVer)) + assert (isinstance(version, six.string_types)) + assert (isinstance(node.version, PgVer)) self.assertEqual(node.version, PgVer(version)) def test_child_pids(self): From 89784a82e601b06031628cb6b490c43a52cc5f00 Mon Sep 17 00:00:00 2001 From: "d.kovalenko" Date: Sun, 23 Feb 2025 23:57:23 +0300 Subject: [PATCH 06/36] Using pytest [assertEqual] --- tests/test_simple.py | 136 ++++++++++++++++++------------------ tests/test_simple_remote.py | 98 +++++++++++++------------- 2 files changed, 117 insertions(+), 117 deletions(-) diff --git a/tests/test_simple.py b/tests/test_simple.py index 3489173b..c3fcd495 100644 --- a/tests/test_simple.py +++ b/tests/test_simple.py @@ -210,10 +210,10 @@ def test_restart(self): # restart, ok res = node.execute('select 1') - self.assertEqual(res, [(1, )]) + assert (res == [(1, )]) node.restart() res = node.execute('select 2') - self.assertEqual(res, [(2, )]) + assert (res == [(2, )]) # restart, fail with pytest.raises(expected_exception=StartNodeException): @@ -233,7 +233,7 @@ def test_reload(self): # check new value cmm_new = node.execute('show client_min_messages') - self.assertEqual('debug1', cmm_new[0][0].lower()) + assert ('debug1' == cmm_new[0][0].lower()) self.assertNotEqual(cmm_old, cmm_new) def test_pg_ctl(self): @@ -250,62 +250,62 @@ def test_status(self): # check statuses after each operation with get_new_node() as node: - self.assertEqual(node.pid, 0) - self.assertEqual(node.status(), NodeStatus.Uninitialized) + assert (node.pid == 0) + assert (node.status() == NodeStatus.Uninitialized) node.init() - self.assertEqual(node.pid, 0) - self.assertEqual(node.status(), NodeStatus.Stopped) + assert (node.pid == 0) + assert (node.status() == NodeStatus.Stopped) node.start() self.assertNotEqual(node.pid, 0) - self.assertEqual(node.status(), NodeStatus.Running) + assert (node.status() == NodeStatus.Running) node.stop() - self.assertEqual(node.pid, 0) - self.assertEqual(node.status(), NodeStatus.Stopped) + assert (node.pid == 0) + assert (node.status() == NodeStatus.Stopped) node.cleanup() - self.assertEqual(node.pid, 0) - self.assertEqual(node.status(), NodeStatus.Uninitialized) + assert (node.pid == 0) + assert (node.status() == NodeStatus.Uninitialized) def test_psql(self): with get_new_node().init().start() as node: # check returned values (1 arg) res = node.psql('select 1') - self.assertEqual(rm_carriage_returns(res), (0, b'1\n', b'')) + assert (rm_carriage_returns(res) == (0, b'1\n', b'')) # check returned values (2 args) res = node.psql('postgres', 'select 2') - self.assertEqual(rm_carriage_returns(res), (0, b'2\n', b'')) + assert (rm_carriage_returns(res) == (0, b'2\n', b'')) # check returned values (named) res = node.psql(query='select 3', dbname='postgres') - self.assertEqual(rm_carriage_returns(res), (0, b'3\n', b'')) + assert (rm_carriage_returns(res) == (0, b'3\n', b'')) # check returned values (1 arg) res = node.safe_psql('select 4') - self.assertEqual(rm_carriage_returns(res), b'4\n') + assert (rm_carriage_returns(res) == b'4\n') # check returned values (2 args) res = node.safe_psql('postgres', 'select 5') - self.assertEqual(rm_carriage_returns(res), b'5\n') + assert (rm_carriage_returns(res) == b'5\n') # check returned values (named) res = node.safe_psql(query='select 6', dbname='postgres') - self.assertEqual(rm_carriage_returns(res), b'6\n') + assert (rm_carriage_returns(res) == b'6\n') # check feeding input node.safe_psql('create table horns (w int)') node.safe_psql('copy horns from stdin (format csv)', input=b"1\n2\n3\n\\.\n") _sum = node.safe_psql('select sum(w) from horns') - self.assertEqual(rm_carriage_returns(_sum), b'6\n') + assert (rm_carriage_returns(_sum) == b'6\n') # check psql's default args, fails with pytest.raises(expected_exception=QueryException): @@ -333,7 +333,7 @@ def test_safe_psql__expect_error(self): # --------- res = node.safe_psql("select 1;", expect_error=False) - self.assertEqual(rm_carriage_returns(res), b'1\n') + assert (rm_carriage_returns(res) == b'1\n') def test_transactions(self): with get_new_node().init().start() as node: @@ -475,11 +475,11 @@ def test_synchronous_replication(self): standby2.start() # check formatting - self.assertEqual( - '1 ("{}", "{}")'.format(standby1.name, standby2.name), + assert ( + '1 ("{}", "{}")'.format(standby1.name, standby2.name) == str(First(1, (standby1, standby2)))) # yapf: disable - self.assertEqual( - 'ANY 1 ("{}", "{}")'.format(standby1.name, standby2.name), + assert ( + 'ANY 1 ("{}", "{}")'.format(standby1.name, standby2.name) == str(Any(1, (standby1, standby2)))) # yapf: disable # set synchronous_standby_names @@ -498,7 +498,7 @@ def test_synchronous_replication(self): master.safe_psql( 'insert into abc select generate_series(1, 1000000)') res = standby1.safe_psql('select count(*) from abc') - self.assertEqual(rm_carriage_returns(res), b'1000000\n') + assert (rm_carriage_returns(res) == b'1000000\n') # @unittest.skipUnless(pg_version_ge('10'), 'requires 10+') def test_logical_replication(self): @@ -638,7 +638,7 @@ def test_promotion(self): # make standby becomes writable master replica.safe_psql('insert into abc values (1)') res = replica.safe_psql('select * from abc') - self.assertEqual(rm_carriage_returns(res), b'1\n') + assert (rm_carriage_returns(res) == b'1\n') def test_dump(self): query_create = 'create table test as select generate_series(1, 2) as val' @@ -664,7 +664,7 @@ def test_users(self): node.psql('create role test_user login') value = node.safe_psql('select 1', username='test_user') value = rm_carriage_returns(value) - self.assertEqual(value, b'1\n') + assert (value == b'1\n') def test_poll_query_until(self): with get_new_node() as node: @@ -804,7 +804,7 @@ def test_pg_config(self): # check same instances a = get_pg_config() b = get_pg_config() - self.assertEqual(id(a), id(b)) + assert (id(a) == id(b)) # save right before config change c1 = get_pg_config() @@ -839,7 +839,7 @@ def test_config_stack(self): d2 = 'dummy_def' with scoped_config(cached_initdb_dir=d1) as c1: - self.assertEqual(c1.cached_initdb_dir, d1) + assert (c1.cached_initdb_dir == d1) with scoped_config(cached_initdb_dir=d2) as c2: stack_size = len(testgres.config.config_stack) @@ -849,12 +849,12 @@ def test_config_stack(self): with scoped_config(dummy=True): pass - self.assertEqual(c2.cached_initdb_dir, d2) - self.assertEqual(len(testgres.config.config_stack), stack_size) + assert (c2.cached_initdb_dir == d2) + assert (len(testgres.config.config_stack) == stack_size) - self.assertEqual(c1.cached_initdb_dir, d1) + assert (c1.cached_initdb_dir == d1) - self.assertEqual(TestgresConfig.cached_initdb_dir, d0) + assert (TestgresConfig.cached_initdb_dir == d0) def test_unix_sockets(self): with get_new_node() as node: @@ -897,13 +897,13 @@ def test_file_tail(self): f.seek(0) lines = file_tail(f, 3) - self.assertEqual(lines[0], s1) - self.assertEqual(lines[1], s2) - self.assertEqual(lines[2], s3) + assert (lines[0] == s1) + assert (lines[1] == s2) + assert (lines[2] == s3) f.seek(0) lines = file_tail(f, 1) - self.assertEqual(lines[0], s3) + assert (lines[0] == s3) def test_isolation_levels(self): with get_new_node().init().start() as node: @@ -926,19 +926,19 @@ def test_isolation_levels(self): def test_ports_management(self): # check that no ports have been bound yet - self.assertEqual(len(bound_ports), 0) + assert (len(bound_ports) == 0) with get_new_node() as node: # check that we've just bound a port - self.assertEqual(len(bound_ports), 1) + assert (len(bound_ports) == 1) # check that bound_ports contains our port port_1 = list(bound_ports)[0] port_2 = node.port - self.assertEqual(port_1, port_2) + assert (port_1 == port_2) # check that port has been freed successfully - self.assertEqual(len(bound_ports), 0) + assert (len(bound_ports) == 0) def test_exceptions(self): str(StartNodeException('msg', [('file', 'lines')])) @@ -972,7 +972,7 @@ def test_version_management(self): with get_new_node() as node: assert (isinstance(version, six.string_types)) assert (isinstance(node.version, PgVer)) - self.assertEqual(node.version, PgVer(version)) + assert (node.version == PgVer(version)) def test_child_pids(self): master_processes = [ @@ -1018,10 +1018,10 @@ def test_child_pids(self): self.assertIn(ptype, replica_pids) # there should be exactly 1 source walsender for replica - self.assertEqual(len(master_pids[ProcessType.WalSender]), 1) + assert (len(master_pids[ProcessType.WalSender]) == 1) pid1 = master_pids[ProcessType.WalSender][0] pid2 = replica.source_walsender.pid - self.assertEqual(pid1, pid2) + assert (pid1 == pid2) replica.stop() @@ -1034,7 +1034,7 @@ def test_child_process_dies(self): cmd = ["timeout", "60"] if os.name == 'nt' else ["sleep", "60"] with subprocess.Popen(cmd, shell=True) as process: # shell=True might be needed on Windows - self.assertEqual(process.poll(), None) + assert (process.poll() == None) # collect list of processes currently running children = psutil.Process(os.getpid()).children() # kill a process, so received children dictionary becomes invalid @@ -1070,13 +1070,13 @@ def test_the_same_port(self): with get_new_node() as node: node.init().start() assert (node._should_free_port) - self.assertEqual(type(node.port), int) + assert (type(node.port) == int) node_port_copy = node.port - self.assertEqual(rm_carriage_returns(node.safe_psql("SELECT 1;")), b'1\n') + assert (rm_carriage_returns(node.safe_psql("SELECT 1;")) == b'1\n') with get_new_node(port=node.port) as node2: - self.assertEqual(type(node2.port), int) - self.assertEqual(node2.port, node.port) + assert (type(node2.port) == int) + assert (node2.port == node.port) assert not (node2._should_free_port) with pytest.raises( @@ -1086,9 +1086,9 @@ def test_the_same_port(self): node2.init().start() # node is still working - self.assertEqual(node.port, node_port_copy) + assert (node.port == node_port_copy) assert (node._should_free_port) - self.assertEqual(rm_carriage_returns(node.safe_psql("SELECT 3;")), b'3\n') + assert (rm_carriage_returns(node.safe_psql("SELECT 3;")) == b'3\n') class tagPortManagerProxy: sm_prev_testgres_reserve_port = None @@ -1194,30 +1194,30 @@ def test_port_rereserve_during_node_start(self): with get_new_node() as node1: node1.init().start() assert (node1._should_free_port) - self.assertEqual(type(node1.port), int) # noqa: E721 + assert (type(node1.port) == int) # noqa: E721 node1_port_copy = node1.port - self.assertEqual(rm_carriage_returns(node1.safe_psql("SELECT 1;")), b'1\n') + assert (rm_carriage_returns(node1.safe_psql("SELECT 1;")) == b'1\n') with __class__.tagPortManagerProxy(node1.port, C_COUNT_OF_BAD_PORT_USAGE): assert __class__.tagPortManagerProxy.sm_DummyPortNumber == node1.port with get_new_node() as node2: assert (node2._should_free_port) - self.assertEqual(node2.port, node1.port) + assert (node2.port == node1.port) node2.init().start() self.assertNotEqual(node2.port, node1.port) assert (node2._should_free_port) - self.assertEqual(__class__.tagPortManagerProxy.sm_DummyPortCurrentUsage, 0) - self.assertEqual(__class__.tagPortManagerProxy.sm_DummyPortTotalUsage, C_COUNT_OF_BAD_PORT_USAGE) + assert (__class__.tagPortManagerProxy.sm_DummyPortCurrentUsage == 0) + assert (__class__.tagPortManagerProxy.sm_DummyPortTotalUsage == C_COUNT_OF_BAD_PORT_USAGE) assert (node2.is_started) - self.assertEqual(rm_carriage_returns(node2.safe_psql("SELECT 2;")), b'2\n') + assert (rm_carriage_returns(node2.safe_psql("SELECT 2;")) == b'2\n') # node1 is still working - self.assertEqual(node1.port, node1_port_copy) + assert (node1.port == node1_port_copy) assert (node1._should_free_port) - self.assertEqual(rm_carriage_returns(node1.safe_psql("SELECT 3;")), b'3\n') + assert (rm_carriage_returns(node1.safe_psql("SELECT 3;")) == b'3\n') def test_port_conflict(self): assert testgres.PostgresNode._C_MAX_START_ATEMPTS > 1 @@ -1227,15 +1227,15 @@ def test_port_conflict(self): with get_new_node() as node1: node1.init().start() assert (node1._should_free_port) - self.assertEqual(type(node1.port), int) # noqa: E721 + assert (type(node1.port) == int) # noqa: E721 node1_port_copy = node1.port - self.assertEqual(rm_carriage_returns(node1.safe_psql("SELECT 1;")), b'1\n') + assert (rm_carriage_returns(node1.safe_psql("SELECT 1;")) == b'1\n') with __class__.tagPortManagerProxy(node1.port, C_COUNT_OF_BAD_PORT_USAGE): assert __class__.tagPortManagerProxy.sm_DummyPortNumber == node1.port with get_new_node() as node2: assert (node2._should_free_port) - self.assertEqual(node2.port, node1.port) + assert (node2.port == node1.port) with pytest.raises( expected_exception=StartNodeException, @@ -1243,19 +1243,19 @@ def test_port_conflict(self): ): node2.init().start() - self.assertEqual(node2.port, node1.port) + assert (node2.port == node1.port) assert (node2._should_free_port) - self.assertEqual(__class__.tagPortManagerProxy.sm_DummyPortCurrentUsage, 1) - self.assertEqual(__class__.tagPortManagerProxy.sm_DummyPortTotalUsage, C_COUNT_OF_BAD_PORT_USAGE) + assert (__class__.tagPortManagerProxy.sm_DummyPortCurrentUsage == 1) + assert (__class__.tagPortManagerProxy.sm_DummyPortTotalUsage == C_COUNT_OF_BAD_PORT_USAGE) assert not (node2.is_started) # node2 must release our dummyPort (node1.port) - self.assertEqual(__class__.tagPortManagerProxy.sm_DummyPortCurrentUsage, 0) + assert (__class__.tagPortManagerProxy.sm_DummyPortCurrentUsage == 0) # node1 is still working - self.assertEqual(node1.port, node1_port_copy) + assert (node1.port == node1_port_copy) assert (node1._should_free_port) - self.assertEqual(rm_carriage_returns(node1.safe_psql("SELECT 3;")), b'3\n') + assert (rm_carriage_returns(node1.safe_psql("SELECT 3;")) == b'3\n') def test_simple_with_bin_dir(self): with get_new_node() as node: diff --git a/tests/test_simple_remote.py b/tests/test_simple_remote.py index f615e3aa..50fa26be 100755 --- a/tests/test_simple_remote.py +++ b/tests/test_simple_remote.py @@ -267,10 +267,10 @@ def test_restart(self): # restart, ok res = node.execute('select 1') - self.assertEqual(res, [(1,)]) + assert (res == [(1,)]) node.restart() res = node.execute('select 2') - self.assertEqual(res, [(2,)]) + assert (res == [(2,)]) # restart, fail with pytest.raises(expected_exception=StartNodeException): @@ -290,7 +290,7 @@ def test_reload(self): # check new value cmm_new = node.execute('show client_min_messages') - self.assertEqual('debug1', cmm_new[0][0].lower()) + assert ('debug1' == cmm_new[0][0].lower()) self.assertNotEqual(cmm_old, cmm_new) def test_pg_ctl(self): @@ -307,61 +307,61 @@ def test_status(self): # check statuses after each operation with get_remote_node(conn_params=conn_params) as node: - self.assertEqual(node.pid, 0) - self.assertEqual(node.status(), NodeStatus.Uninitialized) + assert (node.pid == 0) + assert (node.status() == NodeStatus.Uninitialized) node.init() - self.assertEqual(node.pid, 0) - self.assertEqual(node.status(), NodeStatus.Stopped) + assert (node.pid == 0) + assert (node.status() == NodeStatus.Stopped) node.start() self.assertNotEqual(node.pid, 0) - self.assertEqual(node.status(), NodeStatus.Running) + assert (node.status() == NodeStatus.Running) node.stop() - self.assertEqual(node.pid, 0) - self.assertEqual(node.status(), NodeStatus.Stopped) + assert (node.pid == 0) + assert (node.status() == NodeStatus.Stopped) node.cleanup() - self.assertEqual(node.pid, 0) - self.assertEqual(node.status(), NodeStatus.Uninitialized) + assert (node.pid == 0) + assert (node.status() == NodeStatus.Uninitialized) def test_psql(self): with get_remote_node(conn_params=conn_params).init().start() as node: # check returned values (1 arg) res = node.psql('select 1') - self.assertEqual(res, (0, b'1\n', b'')) + assert (res == (0, b'1\n', b'')) # check returned values (2 args) res = node.psql('postgres', 'select 2') - self.assertEqual(res, (0, b'2\n', b'')) + assert (res == (0, b'2\n', b'')) # check returned values (named) res = node.psql(query='select 3', dbname='postgres') - self.assertEqual(res, (0, b'3\n', b'')) + assert (res == (0, b'3\n', b'')) # check returned values (1 arg) res = node.safe_psql('select 4') - self.assertEqual(res, b'4\n') + assert (res == b'4\n') # check returned values (2 args) res = node.safe_psql('postgres', 'select 5') - self.assertEqual(res, b'5\n') + assert (res == b'5\n') # check returned values (named) res = node.safe_psql(query='select 6', dbname='postgres') - self.assertEqual(res, b'6\n') + assert (res == b'6\n') # check feeding input node.safe_psql('create table horns (w int)') node.safe_psql('copy horns from stdin (format csv)', input=b"1\n2\n3\n\\.\n") _sum = node.safe_psql('select sum(w) from horns') - self.assertEqual(_sum, b'6\n') + assert (_sum == b'6\n') # check psql's default args, fails with pytest.raises(expected_exception=QueryException): @@ -389,7 +389,7 @@ def test_safe_psql__expect_error(self): # --------- res = node.safe_psql("select 1;", expect_error=False) - self.assertEqual(res, b'1\n') + assert (res == b'1\n') def test_transactions(self): with get_remote_node(conn_params=conn_params).init().start() as node: @@ -528,11 +528,11 @@ def test_synchronous_replication(self): standby2.start() # check formatting - self.assertEqual( - '1 ("{}", "{}")'.format(standby1.name, standby2.name), + assert ( + '1 ("{}", "{}")'.format(standby1.name, standby2.name) == str(First(1, (standby1, standby2)))) # yapf: disable - self.assertEqual( - 'ANY 1 ("{}", "{}")'.format(standby1.name, standby2.name), + assert ( + 'ANY 1 ("{}", "{}")'.format(standby1.name, standby2.name) == str(Any(1, (standby1, standby2)))) # yapf: disable # set synchronous_standby_names @@ -551,7 +551,7 @@ def test_synchronous_replication(self): master.safe_psql( 'insert into abc select generate_series(1, 1000000)') res = standby1.safe_psql('select count(*) from abc') - self.assertEqual(res, b'1000000\n') + assert (res == b'1000000\n') # @unittest.skipUnless(pg_version_ge('10'), 'requires 10+') def test_logical_replication(self): @@ -691,7 +691,7 @@ def test_promotion(self): # make standby becomes writable master replica.safe_psql('insert into abc values (1)') res = replica.safe_psql('select * from abc') - self.assertEqual(res, b'1\n') + assert (res == b'1\n') def test_dump(self): query_create = 'create table test as select generate_series(1, 2) as val' @@ -716,7 +716,7 @@ def test_users(self): with get_remote_node(conn_params=conn_params).init().start() as node: node.psql('create role test_user login') value = node.safe_psql('select 1', username='test_user') - self.assertEqual(b'1\n', value) + assert (b'1\n' == value) def test_poll_query_until(self): with get_remote_node(conn_params=conn_params) as node: @@ -851,7 +851,7 @@ def test_pg_config(self): # check same instances a = get_pg_config() b = get_pg_config() - self.assertEqual(id(a), id(b)) + assert (id(a) == id(b)) # save right before config change c1 = get_pg_config() @@ -886,7 +886,7 @@ def test_config_stack(self): d2 = 'dummy_def' with scoped_config(cached_initdb_dir=d1) as c1: - self.assertEqual(c1.cached_initdb_dir, d1) + assert (c1.cached_initdb_dir == d1) with scoped_config(cached_initdb_dir=d2) as c2: stack_size = len(testgres.config.config_stack) @@ -896,12 +896,12 @@ def test_config_stack(self): with scoped_config(dummy=True): pass - self.assertEqual(c2.cached_initdb_dir, d2) - self.assertEqual(len(testgres.config.config_stack), stack_size) + assert (c2.cached_initdb_dir == d2) + assert (len(testgres.config.config_stack) == stack_size) - self.assertEqual(c1.cached_initdb_dir, d1) + assert (c1.cached_initdb_dir == d1) - self.assertEqual(TestgresConfig.cached_initdb_dir, d0) + assert (TestgresConfig.cached_initdb_dir == d0) def test_unix_sockets(self): with get_remote_node(conn_params=conn_params) as node: @@ -910,14 +910,14 @@ def test_unix_sockets(self): res_exec = node.execute('select 1') res_psql = node.safe_psql('select 1') - self.assertEqual(res_exec, [(1,)]) - self.assertEqual(res_psql, b'1\n') + assert (res_exec == [(1,)]) + assert (res_psql == b'1\n') with node.replicate().start() as r: res_exec = r.execute('select 1') res_psql = r.safe_psql('select 1') - self.assertEqual(res_exec, [(1,)]) - self.assertEqual(res_psql, b'1\n') + assert (res_exec == [(1,)]) + assert (res_psql == b'1\n') def test_auto_name(self): with get_remote_node(conn_params=conn_params).init(allow_streaming=True).start() as m: @@ -948,13 +948,13 @@ def test_file_tail(self): f.seek(0) lines = file_tail(f, 3) - self.assertEqual(lines[0], s1) - self.assertEqual(lines[1], s2) - self.assertEqual(lines[2], s3) + assert (lines[0] == s1) + assert (lines[1] == s2) + assert (lines[2] == s3) f.seek(0) lines = file_tail(f, 1) - self.assertEqual(lines[0], s3) + assert (lines[0] == s3) def test_isolation_levels(self): with get_remote_node(conn_params=conn_params).init().start() as node: @@ -977,19 +977,19 @@ def test_isolation_levels(self): def test_ports_management(self): # check that no ports have been bound yet - self.assertEqual(len(bound_ports), 0) + assert (len(bound_ports) == 0) with get_remote_node(conn_params=conn_params) as node: # check that we've just bound a port - self.assertEqual(len(bound_ports), 1) + assert (len(bound_ports) == 1) # check that bound_ports contains our port port_1 = list(bound_ports)[0] port_2 = node.port - self.assertEqual(port_1, port_2) + assert (port_1 == port_2) # check that port has been freed successfully - self.assertEqual(len(bound_ports), 0) + assert (len(bound_ports) == 0) def test_exceptions(self): str(StartNodeException('msg', [('file', 'lines')])) @@ -1015,7 +1015,7 @@ def test_version_management(self): with get_remote_node(conn_params=conn_params) as node: assert (isinstance(version, six.string_types)) assert (isinstance(node.version, PgVer)) - self.assertEqual(node.version, PgVer(version)) + assert (node.version == PgVer(version)) def test_child_pids(self): master_processes = [ @@ -1061,10 +1061,10 @@ def test_child_pids(self): self.assertIn(ptype, replica_pids) # there should be exactly 1 source walsender for replica - self.assertEqual(len(master_pids[ProcessType.WalSender]), 1) + assert (len(master_pids[ProcessType.WalSender]) == 1) pid1 = master_pids[ProcessType.WalSender][0] pid2 = replica.source_walsender.pid - self.assertEqual(pid1, pid2) + assert (pid1 == pid2) replica.stop() @@ -1075,7 +1075,7 @@ def test_child_pids(self): def test_child_process_dies(self): # test for FileNotFound exception during child_processes() function with subprocess.Popen(["sleep", "60"]) as process: - self.assertEqual(process.poll(), None) + assert (process.poll() == None) # collect list of processes currently running children = psutil.Process(os.getpid()).children() # kill a process, so received children dictionary becomes invalid From 2252d60fd25cdf9fedd1d1a99fa8adc4a2f1100c Mon Sep 17 00:00:00 2001 From: "d.kovalenko" Date: Mon, 24 Feb 2025 00:04:19 +0300 Subject: [PATCH 07/36] Using pytest [assertNotEqual] --- tests/test_simple.py | 16 ++++++++-------- tests/test_simple_remote.py | 14 +++++++------- 2 files changed, 15 insertions(+), 15 deletions(-) diff --git a/tests/test_simple.py b/tests/test_simple.py index c3fcd495..70b053c7 100644 --- a/tests/test_simple.py +++ b/tests/test_simple.py @@ -234,7 +234,7 @@ def test_reload(self): # check new value cmm_new = node.execute('show client_min_messages') assert ('debug1' == cmm_new[0][0].lower()) - self.assertNotEqual(cmm_old, cmm_new) + assert (cmm_old != cmm_new) def test_pg_ctl(self): with get_new_node() as node: @@ -260,7 +260,7 @@ def test_status(self): node.start() - self.assertNotEqual(node.pid, 0) + assert (node.pid != 0) assert (node.status() == NodeStatus.Running) node.stop() @@ -400,12 +400,12 @@ def test_backup_multiple(self): with node.backup(xlog_method='fetch') as backup1, \ node.backup(xlog_method='fetch') as backup2: - self.assertNotEqual(backup1.base_dir, backup2.base_dir) + assert (backup1.base_dir != backup2.base_dir) with node.backup(xlog_method='fetch') as backup: with backup.spawn_primary('node1', destroy=False) as node1, \ backup.spawn_primary('node2', destroy=False) as node2: - self.assertNotEqual(node1.base_dir, node2.base_dir) + assert (node1.base_dir != node2.base_dir) def test_backup_exhaust(self): with get_new_node() as node: @@ -818,12 +818,12 @@ def test_pg_config(self): c2 = get_pg_config() # check different instances after config change - self.assertNotEqual(id(c1), id(c2)) + assert (id(c1) != id(c2)) # check different instances a = get_pg_config() b = get_pg_config() - self.assertNotEqual(id(a), id(b)) + assert (id(a) != id(b)) def test_config_stack(self): # no such option @@ -876,7 +876,7 @@ def test_auto_name(self): assert (r.status()) # check their names - self.assertNotEqual(m.name, r.name) + assert (m.name != r.name) assert ('testgres' in m.name) assert ('testgres' in r.name) @@ -1206,7 +1206,7 @@ def test_port_rereserve_during_node_start(self): node2.init().start() - self.assertNotEqual(node2.port, node1.port) + assert (node2.port != node1.port) assert (node2._should_free_port) assert (__class__.tagPortManagerProxy.sm_DummyPortCurrentUsage == 0) assert (__class__.tagPortManagerProxy.sm_DummyPortTotalUsage == C_COUNT_OF_BAD_PORT_USAGE) diff --git a/tests/test_simple_remote.py b/tests/test_simple_remote.py index 50fa26be..0a72162b 100755 --- a/tests/test_simple_remote.py +++ b/tests/test_simple_remote.py @@ -291,7 +291,7 @@ def test_reload(self): # check new value cmm_new = node.execute('show client_min_messages') assert ('debug1' == cmm_new[0][0].lower()) - self.assertNotEqual(cmm_old, cmm_new) + assert (cmm_old != cmm_new) def test_pg_ctl(self): with get_remote_node(conn_params=conn_params) as node: @@ -317,7 +317,7 @@ def test_status(self): node.start() - self.assertNotEqual(node.pid, 0) + assert (node.pid != 0) assert (node.status() == NodeStatus.Running) node.stop() @@ -453,12 +453,12 @@ def test_backup_multiple(self): with node.backup(xlog_method='fetch') as backup1, \ node.backup(xlog_method='fetch') as backup2: - self.assertNotEqual(backup1.base_dir, backup2.base_dir) + assert (backup1.base_dir != backup2.base_dir) with node.backup(xlog_method='fetch') as backup: with backup.spawn_primary('node1', destroy=False) as node1, \ backup.spawn_primary('node2', destroy=False) as node2: - self.assertNotEqual(node1.base_dir, node2.base_dir) + assert (node1.base_dir != node2.base_dir) def test_backup_exhaust(self): with get_remote_node(conn_params=conn_params) as node: @@ -865,12 +865,12 @@ def test_pg_config(self): c2 = get_pg_config() # check different instances after config change - self.assertNotEqual(id(c1), id(c2)) + assert (id(c1) != id(c2)) # check different instances a = get_pg_config() b = get_pg_config() - self.assertNotEqual(id(a), id(b)) + assert (id(a) != id(b)) def test_config_stack(self): # no such option @@ -927,7 +927,7 @@ def test_auto_name(self): assert (r.status()) # check their names - self.assertNotEqual(m.name, r.name) + assert (m.name != r.name) assert ('testgres' in m.name) assert ('testgres' in r.name) From e8aad3d46cb39a77b284af0bb099a93983a26d0b Mon Sep 17 00:00:00 2001 From: "d.kovalenko" Date: Mon, 24 Feb 2025 00:06:05 +0300 Subject: [PATCH 08/36] Using pytest [assertGreaterEqual] --- tests/test_simple.py | 2 +- tests/test_simple_remote.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_simple.py b/tests/test_simple.py index 70b053c7..448fbf01 100644 --- a/tests/test_simple.py +++ b/tests/test_simple.py @@ -128,7 +128,7 @@ def test_custom_init(self): lines = conf.readlines() # check number of lines - self.assertGreaterEqual(len(lines), 6) + assert (len(lines) >= 6) # there should be no trust entries at all assert not (any('trust' in s for s in lines)) diff --git a/tests/test_simple_remote.py b/tests/test_simple_remote.py index 0a72162b..8f36a33b 100755 --- a/tests/test_simple_remote.py +++ b/tests/test_simple_remote.py @@ -115,7 +115,7 @@ def test_custom_init(self): lines = os_ops.readlines(hba_file) # check number of lines - self.assertGreaterEqual(len(lines), 6) + assert (len(lines) >= 6) # there should be no trust entries at all assert not (any('trust' in s for s in lines)) From eaaa276eef770e5612bffcb255ac2d888b608eca Mon Sep 17 00:00:00 2001 From: "d.kovalenko" Date: Mon, 24 Feb 2025 00:13:34 +0300 Subject: [PATCH 09/36] Using pytest [assertGreater] --- tests/test_simple.py | 6 +++--- tests/test_simple_remote.py | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/tests/test_simple.py b/tests/test_simple.py index 448fbf01..5db5b20e 100644 --- a/tests/test_simple.py +++ b/tests/test_simple.py @@ -171,8 +171,8 @@ def test_init_unique_system_id(self): id2 = node2.execute(query)[0] # ids must increase - self.assertGreater(id1, id0) - self.assertGreater(id2, id1) + assert (id1 > id0) + assert (id2 > id1) def test_node_exit(self): base_dir = None @@ -1002,7 +1002,7 @@ def test_child_pids(self): master.source_walsender with master.connect() as con: - self.assertGreater(con.pid, 0) + assert (con.pid > 0) with master.replicate().start() as replica: diff --git a/tests/test_simple_remote.py b/tests/test_simple_remote.py index 8f36a33b..79e5c133 100755 --- a/tests/test_simple_remote.py +++ b/tests/test_simple_remote.py @@ -230,8 +230,8 @@ def test_init_unique_system_id(self): id2 = node2.execute(query)[0] # ids must increase - self.assertGreater(id1, id0) - self.assertGreater(id2, id1) + assert (id1 > id0) + assert (id2 > id1) def test_node_exit(self): with pytest.raises(expected_exception=QueryException): @@ -1045,7 +1045,7 @@ def test_child_pids(self): master.source_walsender with master.connect() as con: - self.assertGreater(con.pid, 0) + assert (con.pid > 0) with master.replicate().start() as replica: From 4afbe00257832ac87c0ba114a279a187a8858851 Mon Sep 17 00:00:00 2001 From: "d.kovalenko" Date: Mon, 24 Feb 2025 00:16:26 +0300 Subject: [PATCH 10/36] Using pytest [assertIn] --- tests/test_simple.py | 15 +++++---------- tests/test_simple_remote.py | 8 ++++---- 2 files changed, 9 insertions(+), 14 deletions(-) diff --git a/tests/test_simple.py b/tests/test_simple.py index 5db5b20e..ea7494b9 100644 --- a/tests/test_simple.py +++ b/tests/test_simple.py @@ -321,8 +321,8 @@ def test_safe_psql__expect_error(self): with get_new_node().init().start() as node: err = node.safe_psql('select_or_not_select 1', expect_error=True) assert (type(err) == str) # noqa: E721 - self.assertIn('select_or_not_select', err) - self.assertIn('ERROR: syntax error at or near "select_or_not_select"', err) + assert ('select_or_not_select' in err) + assert ('ERROR: syntax error at or near "select_or_not_select"' in err) # --------- with pytest.raises( @@ -1011,11 +1011,11 @@ def test_child_pids(self): master_pids = master.auxiliary_pids for ptype in master_processes: - self.assertIn(ptype, master_pids) + assert (ptype in master_pids) replica_pids = replica.auxiliary_pids for ptype in repl_processes: - self.assertIn(ptype, replica_pids) + assert (ptype in replica_pids) # there should be exactly 1 source walsender for replica assert (len(master_pids[ProcessType.WalSender]) == 1) @@ -1320,12 +1320,7 @@ def test_set_auto_conf(self): content = f.read() for x in testData: - self.assertIn( - x[0] + " = " + x[2], - content, - x[0] + " stored wrong" - ) - + assert x[0] + " = " + x[2] in content @staticmethod def helper__skip_test_if_util_not_exist(name: str): assert type(name) == str # noqa: E721 diff --git a/tests/test_simple_remote.py b/tests/test_simple_remote.py index 79e5c133..5bcb7755 100755 --- a/tests/test_simple_remote.py +++ b/tests/test_simple_remote.py @@ -377,8 +377,8 @@ def test_safe_psql__expect_error(self): with get_remote_node(conn_params=conn_params).init().start() as node: err = node.safe_psql('select_or_not_select 1', expect_error=True) assert (type(err) == str) # noqa: E721 - self.assertIn('select_or_not_select', err) - self.assertIn('ERROR: syntax error at or near "select_or_not_select"', err) + assert ('select_or_not_select' in err) + assert ('ERROR: syntax error at or near "select_or_not_select"' in err) # --------- with pytest.raises( @@ -1054,11 +1054,11 @@ def test_child_pids(self): master_pids = master.auxiliary_pids for ptype in master_processes: - self.assertIn(ptype, master_pids) + assert (ptype in master_pids) replica_pids = replica.auxiliary_pids for ptype in repl_processes: - self.assertIn(ptype, replica_pids) + assert (ptype in replica_pids) # there should be exactly 1 source walsender for replica assert (len(master_pids[ProcessType.WalSender]) == 1) From a8e57ae01cba107a3ef46599c4d901d809be7a93 Mon Sep 17 00:00:00 2001 From: "d.kovalenko" Date: Mon, 24 Feb 2025 00:21:01 +0300 Subject: [PATCH 11/36] Using pytest [assertListEqual] --- tests/test_simple.py | 27 ++++++++++++--------------- tests/test_simple_remote.py | 27 ++++++++++++--------------- 2 files changed, 24 insertions(+), 30 deletions(-) diff --git a/tests/test_simple.py b/tests/test_simple.py index ea7494b9..fc5a5300 100644 --- a/tests/test_simple.py +++ b/tests/test_simple.py @@ -347,12 +347,12 @@ def test_transactions(self): con.begin() con.execute('insert into test values (2)') res = con.execute('select * from test order by val asc') - self.assertListEqual(res, [(1, ), (2, )]) + assert (res == [(1, ), (2, )]) con.rollback() con.begin() res = con.execute('select * from test') - self.assertListEqual(res, [(1, )]) + assert (res == [(1, )]) con.rollback() con.begin() @@ -392,7 +392,7 @@ def test_backup_simple(self): with master.backup(xlog_method='stream') as backup: with backup.spawn_primary().start() as slave: res = slave.execute('select * from test order by i asc') - self.assertListEqual(res, [(1, ), (2, ), (3, ), (4, )]) + assert (res == [(1, ), (2, ), (3, ), (4, )]) def test_backup_multiple(self): with get_new_node() as node: @@ -448,14 +448,14 @@ def test_replicate(self): with node.replicate().start() as replica: res = replica.execute('select 1') - self.assertListEqual(res, [(1, )]) + assert (res == [(1, )]) node.execute('create table test (val int)', commit=True) replica.catchup() res = node.execute('select * from test') - self.assertListEqual(res, []) + assert (res == []) # @unittest.skipUnless(pg_version_ge('9.6'), 'requires 9.6+') def test_synchronous_replication(self): @@ -522,7 +522,7 @@ def test_logical_replication(self): # wait until changes apply on subscriber and check them sub.catchup() res = node2.execute('select * from test') - self.assertListEqual(res, [(1, 1), (2, 2)]) + assert (res == [(1, 1), (2, 2)]) # disable and put some new data sub.disable() @@ -532,7 +532,7 @@ def test_logical_replication(self): sub.enable() sub.catchup() res = node2.execute('select * from test') - self.assertListEqual(res, [(1, 1), (2, 2), (3, 3)]) + assert (res == [(1, 1), (2, 2), (3, 3)]) # Add new tables. Since we added "all tables" to publication # (default behaviour of publish() method) we don't need @@ -546,7 +546,7 @@ def test_logical_replication(self): node1.safe_psql('insert into test2 values (\'a\'), (\'b\')') sub.catchup() res = node2.execute('select * from test2') - self.assertListEqual(res, [('a', ), ('b', )]) + assert (res == [('a', ), ('b', )]) # drop subscription sub.drop() @@ -560,7 +560,7 @@ def test_logical_replication(self): node1.safe_psql('insert into test values (4, 4)') sub.catchup() res = node2.execute('select * from test') - self.assertListEqual(res, [(1, 1), (2, 2), (3, 3), (4, 4)]) + assert (res == [(1, 1), (2, 2), (3, 3), (4, 4)]) # explicitly add table with pytest.raises(expected_exception=ValueError): @@ -569,7 +569,7 @@ def test_logical_replication(self): node1.safe_psql('insert into test2 values (\'c\')') sub.catchup() res = node2.execute('select * from test2') - self.assertListEqual(res, [('a', ), ('b', )]) + assert (res == [('a', ), ('b', )]) # @unittest.skipUnless(pg_version_ge('10'), 'requires 10+') def test_logical_catchup(self): @@ -593,10 +593,7 @@ def test_logical_catchup(self): node1.execute('insert into test values ({0}, {0})'.format(i)) sub.catchup() res = node2.execute('select * from test') - self.assertListEqual(res, [( - i, - i, - )]) + assert (res == [(i,i,)]) node1.execute('delete from test') # @unittest.skipIf(pg_version_ge('10'), 'requires <10') @@ -657,7 +654,7 @@ def test_dump(self): # restore dump node3.restore(filename=dump) res = node3.execute(query_select) - self.assertListEqual(res, [(1, ), (2, )]) + assert (res == [(1, ), (2, )]) def test_users(self): with get_new_node().init().start() as node: diff --git a/tests/test_simple_remote.py b/tests/test_simple_remote.py index 5bcb7755..b833077b 100755 --- a/tests/test_simple_remote.py +++ b/tests/test_simple_remote.py @@ -402,12 +402,12 @@ def test_transactions(self): con.begin() con.execute('insert into test values (2)') res = con.execute('select * from test order by val asc') - self.assertListEqual(res, [(1,), (2,)]) + assert (res == [(1,), (2,)]) con.rollback() con.begin() res = con.execute('select * from test') - self.assertListEqual(res, [(1,)]) + assert (res == [(1,)]) con.rollback() con.begin() @@ -445,7 +445,7 @@ def test_backup_simple(self): with master.backup(xlog_method='stream') as backup: with backup.spawn_primary().start() as slave: res = slave.execute('select * from test order by i asc') - self.assertListEqual(res, [(1,), (2,), (3,), (4,)]) + assert (res == [(1,), (2,), (3,), (4,)]) def test_backup_multiple(self): with get_remote_node(conn_params=conn_params) as node: @@ -501,14 +501,14 @@ def test_replicate(self): with node.replicate().start() as replica: res = replica.execute('select 1') - self.assertListEqual(res, [(1,)]) + assert (res == [(1,)]) node.execute('create table test (val int)', commit=True) replica.catchup() res = node.execute('select * from test') - self.assertListEqual(res, []) + assert (res == []) # @unittest.skipUnless(pg_version_ge('9.6'), 'requires 9.6+') def test_synchronous_replication(self): @@ -575,7 +575,7 @@ def test_logical_replication(self): # wait until changes apply on subscriber and check them sub.catchup() res = node2.execute('select * from test') - self.assertListEqual(res, [(1, 1), (2, 2)]) + assert (res == [(1, 1), (2, 2)]) # disable and put some new data sub.disable() @@ -585,7 +585,7 @@ def test_logical_replication(self): sub.enable() sub.catchup() res = node2.execute('select * from test') - self.assertListEqual(res, [(1, 1), (2, 2), (3, 3)]) + assert (res == [(1, 1), (2, 2), (3, 3)]) # Add new tables. Since we added "all tables" to publication # (default behaviour of publish() method) we don't need @@ -599,7 +599,7 @@ def test_logical_replication(self): node1.safe_psql('insert into test2 values (\'a\'), (\'b\')') sub.catchup() res = node2.execute('select * from test2') - self.assertListEqual(res, [('a',), ('b',)]) + assert (res == [('a',), ('b',)]) # drop subscription sub.drop() @@ -613,7 +613,7 @@ def test_logical_replication(self): node1.safe_psql('insert into test values (4, 4)') sub.catchup() res = node2.execute('select * from test') - self.assertListEqual(res, [(1, 1), (2, 2), (3, 3), (4, 4)]) + assert (res == [(1, 1), (2, 2), (3, 3), (4, 4)]) # explicitly add table with pytest.raises(expected_exception=ValueError): @@ -622,7 +622,7 @@ def test_logical_replication(self): node1.safe_psql('insert into test2 values (\'c\')') sub.catchup() res = node2.execute('select * from test2') - self.assertListEqual(res, [('a',), ('b',)]) + assert (res == [('a',), ('b',)]) # @unittest.skipUnless(pg_version_ge('10'), 'requires 10+') def test_logical_catchup(self): @@ -646,10 +646,7 @@ def test_logical_catchup(self): node1.execute('insert into test values ({0}, {0})'.format(i)) sub.catchup() res = node2.execute('select * from test') - self.assertListEqual(res, [( - i, - i, - )]) + assert (res == [(i,i,)]) node1.execute('delete from test') # @unittest.skipIf(pg_version_ge('10'), 'requires <10') @@ -710,7 +707,7 @@ def test_dump(self): # restore dump node3.restore(filename=dump) res = node3.execute(query_select) - self.assertListEqual(res, [(1,), (2,)]) + assert (res == [(1,), (2,)]) def test_users(self): with get_remote_node(conn_params=conn_params).init().start() as node: From df199b92d7d9534dc4ddf9fcdb491d1a559ab44c Mon Sep 17 00:00:00 2001 From: "d.kovalenko" Date: Mon, 24 Feb 2025 00:25:39 +0300 Subject: [PATCH 12/36] unittest is not used --- tests/test_simple.py | 23 +---------------------- tests/test_simple_remote.py | 23 +---------------------- 2 files changed, 2 insertions(+), 44 deletions(-) diff --git a/tests/test_simple.py b/tests/test_simple.py index fc5a5300..36f67537 100644 --- a/tests/test_simple.py +++ b/tests/test_simple.py @@ -8,7 +8,6 @@ import testgres import time import six -import unittest import pytest import psutil import platform @@ -107,7 +106,7 @@ def removing(f): rmtree(f, ignore_errors=True) -class TestgresTests(unittest.TestCase): +class TestgresTests: def test_node_repr(self): with get_new_node() as node: pattern = r"PostgresNode\(name='.+', port=.+, base_dir='.+'\)" @@ -1342,23 +1341,3 @@ def helper__skip_test_if_pg_version_is_ge(version: str): if pg_version_ge(version): pytest.skip('requires <{0}'.format(version)) - -if __name__ == '__main__': - if os.environ.get('ALT_CONFIG'): - suite = unittest.TestSuite() - - # Small subset of tests for alternative configs (PG_BIN or PG_CONFIG) - suite.addTest(TestgresTests('test_pg_config')) - suite.addTest(TestgresTests('test_pg_ctl')) - suite.addTest(TestgresTests('test_psql')) - suite.addTest(TestgresTests('test_replicate')) - - print('Running tests for alternative config:') - for t in suite: - print(t) - print() - - runner = unittest.TextTestRunner() - runner.run(suite) - else: - unittest.main() diff --git a/tests/test_simple_remote.py b/tests/test_simple_remote.py index b833077b..9bd7216c 100755 --- a/tests/test_simple_remote.py +++ b/tests/test_simple_remote.py @@ -9,7 +9,6 @@ import testgres import time import six -import unittest import pytest import psutil @@ -95,7 +94,7 @@ def removing(f): os_ops.rmdirs(f, ignore_errors=True) -class TestgresRemoteTests(unittest.TestCase): +class TestgresRemoteTests: def test_node_repr(self): with get_remote_node(conn_params=conn_params) as node: pattern = r"PostgresNode\(name='.+', port=.+, base_dir='.+'\)" @@ -1106,23 +1105,3 @@ def helper__skip_test_if_pg_version_is_ge(version: str): if pg_version_ge(version): pytest.skip('requires <{0}'.format(version)) - -if __name__ == '__main__': - if os_ops.environ('ALT_CONFIG'): - suite = unittest.TestSuite() - - # Small subset of tests for alternative configs (PG_BIN or PG_CONFIG) - suite.addTest(TestgresRemoteTests('test_pg_config')) - suite.addTest(TestgresRemoteTests('test_pg_ctl')) - suite.addTest(TestgresRemoteTests('test_psql')) - suite.addTest(TestgresRemoteTests('test_replicate')) - - print('Running tests for alternative config:') - for t in suite: - print(t) - print() - - runner = unittest.TextTestRunner() - runner.run(suite) - else: - unittest.main() From a82cf3dbd159a4a52446673a2f687dbc48324128 Mon Sep 17 00:00:00 2001 From: "d.kovalenko" Date: Mon, 24 Feb 2025 00:33:55 +0300 Subject: [PATCH 13/36] Code style (flake8) --- tests/test_simple.py | 18 +++++++++--------- tests/test_simple_remote.py | 13 ++++++------- 2 files changed, 15 insertions(+), 16 deletions(-) diff --git a/tests/test_simple.py b/tests/test_simple.py index 36f67537..278a63a3 100644 --- a/tests/test_simple.py +++ b/tests/test_simple.py @@ -475,11 +475,11 @@ def test_synchronous_replication(self): # check formatting assert ( - '1 ("{}", "{}")'.format(standby1.name, standby2.name) == - str(First(1, (standby1, standby2)))) # yapf: disable + '1 ("{}", "{}")'.format(standby1.name, standby2.name) == str(First(1, (standby1, standby2))) + ) # yapf: disable assert ( - 'ANY 1 ("{}", "{}")'.format(standby1.name, standby2.name) == - str(Any(1, (standby1, standby2)))) # yapf: disable + 'ANY 1 ("{}", "{}")'.format(standby1.name, standby2.name) == str(Any(1, (standby1, standby2))) + ) # yapf: disable # set synchronous_standby_names master.set_synchronous_standbys(First(2, [standby1, standby2])) @@ -592,7 +592,7 @@ def test_logical_catchup(self): node1.execute('insert into test values ({0}, {0})'.format(i)) sub.catchup() res = node2.execute('select * from test') - assert (res == [(i,i,)]) + assert (res == [(i, i, )]) node1.execute('delete from test') # @unittest.skipIf(pg_version_ge('10'), 'requires <10') @@ -1030,7 +1030,7 @@ def test_child_process_dies(self): cmd = ["timeout", "60"] if os.name == 'nt' else ["sleep", "60"] with subprocess.Popen(cmd, shell=True) as process: # shell=True might be needed on Windows - assert (process.poll() == None) + assert (process.poll() is None) # collect list of processes currently running children = psutil.Process(os.getpid()).children() # kill a process, so received children dictionary becomes invalid @@ -1066,12 +1066,12 @@ def test_the_same_port(self): with get_new_node() as node: node.init().start() assert (node._should_free_port) - assert (type(node.port) == int) + assert (type(node.port) == int) # noqa: E721 node_port_copy = node.port assert (rm_carriage_returns(node.safe_psql("SELECT 1;")) == b'1\n') with get_new_node(port=node.port) as node2: - assert (type(node2.port) == int) + assert (type(node2.port) == int) # noqa: E721 assert (node2.port == node.port) assert not (node2._should_free_port) @@ -1317,6 +1317,7 @@ def test_set_auto_conf(self): for x in testData: assert x[0] + " = " + x[2] in content + @staticmethod def helper__skip_test_if_util_not_exist(name: str): assert type(name) == str # noqa: E721 @@ -1340,4 +1341,3 @@ def helper__skip_test_if_pg_version_is_ge(version: str): assert type(version) == str # noqa: E721 if pg_version_ge(version): pytest.skip('requires <{0}'.format(version)) - diff --git a/tests/test_simple_remote.py b/tests/test_simple_remote.py index 9bd7216c..e9be7ac4 100755 --- a/tests/test_simple_remote.py +++ b/tests/test_simple_remote.py @@ -528,11 +528,11 @@ def test_synchronous_replication(self): # check formatting assert ( - '1 ("{}", "{}")'.format(standby1.name, standby2.name) == - str(First(1, (standby1, standby2)))) # yapf: disable + '1 ("{}", "{}")'.format(standby1.name, standby2.name) == str(First(1, (standby1, standby2))) + ) # yapf: disable assert ( - 'ANY 1 ("{}", "{}")'.format(standby1.name, standby2.name) == - str(Any(1, (standby1, standby2)))) # yapf: disable + 'ANY 1 ("{}", "{}")'.format(standby1.name, standby2.name) == str(Any(1, (standby1, standby2))) + ) # yapf: disable # set synchronous_standby_names master.set_synchronous_standbys(First(2, [standby1, standby2])) @@ -645,7 +645,7 @@ def test_logical_catchup(self): node1.execute('insert into test values ({0}, {0})'.format(i)) sub.catchup() res = node2.execute('select * from test') - assert (res == [(i,i,)]) + assert (res == [(i, i, )]) node1.execute('delete from test') # @unittest.skipIf(pg_version_ge('10'), 'requires <10') @@ -1071,7 +1071,7 @@ def test_child_pids(self): def test_child_process_dies(self): # test for FileNotFound exception during child_processes() function with subprocess.Popen(["sleep", "60"]) as process: - assert (process.poll() == None) + assert (process.poll() is None) # collect list of processes currently running children = psutil.Process(os.getpid()).children() # kill a process, so received children dictionary becomes invalid @@ -1104,4 +1104,3 @@ def helper__skip_test_if_pg_version_is_ge(version: str): assert type(version) == str # noqa: E721 if pg_version_ge(version): pytest.skip('requires <{0}'.format(version)) - From 27c4f408c92565a9752faee193f8eaf3a2839f80 Mon Sep 17 00:00:00 2001 From: "d.kovalenko" Date: Mon, 24 Feb 2025 06:45:47 +0300 Subject: [PATCH 14/36] Execution signature is removed --- tests/test_simple.py | 3 --- tests/test_simple_remote.py | 3 --- 2 files changed, 6 deletions(-) diff --git a/tests/test_simple.py b/tests/test_simple.py index 278a63a3..55659e86 100644 --- a/tests/test_simple.py +++ b/tests/test_simple.py @@ -1,6 +1,3 @@ -#!/usr/bin/env python -# coding: utf-8 - import os import re import subprocess diff --git a/tests/test_simple_remote.py b/tests/test_simple_remote.py index e9be7ac4..cfd92ac6 100755 --- a/tests/test_simple_remote.py +++ b/tests/test_simple_remote.py @@ -1,6 +1,3 @@ -#!/usr/bin/env python -# coding: utf-8 - import os import re import subprocess From ae41d166df57a7122cfbeef8d09b27c8aff21b5e Mon Sep 17 00:00:00 2001 From: "d.kovalenko" Date: Mon, 24 Feb 2025 07:03:00 +0300 Subject: [PATCH 15/36] run_tests.sh installs pytest --- run_tests.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/run_tests.sh b/run_tests.sh index 73c459be..a3549242 100755 --- a/run_tests.sh +++ b/run_tests.sh @@ -22,7 +22,7 @@ export VIRTUAL_ENV_DISABLE_PROMPT=1 source $VENV_PATH/bin/activate # install utilities -$PIP install coverage flake8 psutil Sphinx +$PIP install coverage flake8 psutil Sphinx pytest # install testgres' dependencies export PYTHONPATH=$(pwd) From a14b3d787d3ce56244399053f43f0baebfa89230 Mon Sep 17 00:00:00 2001 From: "d.kovalenko" Date: Mon, 24 Feb 2025 10:21:30 +0300 Subject: [PATCH 16/36] run_tests.sh is updated run tests through pytest explicitly --- run_tests.sh | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/run_tests.sh b/run_tests.sh index a3549242..5e7def86 100755 --- a/run_tests.sh +++ b/run_tests.sh @@ -22,7 +22,7 @@ export VIRTUAL_ENV_DISABLE_PROMPT=1 source $VENV_PATH/bin/activate # install utilities -$PIP install coverage flake8 psutil Sphinx pytest +$PIP install coverage flake8 psutil Sphinx pytest pytest-xdist # install testgres' dependencies export PYTHONPATH=$(pwd) @@ -38,21 +38,19 @@ rm -f $COVERAGE_FILE # run tests (PATH) -time coverage run -a tests/test_simple.py +time coverage run -a -m pytest -l -v -n 2 -k "TestgresTests" # run tests (PG_BIN) time \ PG_BIN=$(dirname $(which pg_config)) \ - ALT_CONFIG=1 \ - coverage run -a tests/test_simple.py + coverage run -a -m pytest -l -v -n 2 -k "TestgresTests" # run tests (PG_CONFIG) time \ PG_CONFIG=$(which pg_config) \ - ALT_CONFIG=1 \ - coverage run -a tests/test_simple.py + coverage run -a -m pytest -l -v -n 2 -k "TestgresTests" # show coverage From e6f4448bd992cbe28f67cccfae40c008bc5a3596 Mon Sep 17 00:00:00 2001 From: "d.kovalenko" Date: Mon, 24 Feb 2025 11:51:28 +0300 Subject: [PATCH 17/36] Total refactoring of tests - TestgresRemoteTests does not use global variables and code - Explicit work with ..testgres folder --- run_tests.sh | 4 +- tests/test_local.py | 6 +- tests/test_remote.py | 8 +- tests/test_simple.py | 22 ++--- tests/test_simple_remote.py | 175 ++++++++++++++++++++---------------- 5 files changed, 116 insertions(+), 99 deletions(-) diff --git a/run_tests.sh b/run_tests.sh index 5e7def86..6995b7e4 100755 --- a/run_tests.sh +++ b/run_tests.sh @@ -22,11 +22,11 @@ export VIRTUAL_ENV_DISABLE_PROMPT=1 source $VENV_PATH/bin/activate # install utilities -$PIP install coverage flake8 psutil Sphinx pytest pytest-xdist +$PIP install coverage flake8 psutil Sphinx pytest pytest-xdist psycopg2 six psutil # install testgres' dependencies export PYTHONPATH=$(pwd) -$PIP install . +# $PIP install . # test code quality flake8 . diff --git a/tests/test_local.py b/tests/test_local.py index 4051bfb5..346336bd 100644 --- a/tests/test_local.py +++ b/tests/test_local.py @@ -4,9 +4,9 @@ import re import tempfile -from testgres import ExecUtilException -from testgres import InvalidOperationException -from testgres import LocalOperations +from ..testgres import ExecUtilException +from ..testgres import InvalidOperationException +from ..testgres import LocalOperations from .helpers.run_conditions import RunConditions diff --git a/tests/test_remote.py b/tests/test_remote.py index 4330b92f..21bc9e8f 100755 --- a/tests/test_remote.py +++ b/tests/test_remote.py @@ -4,10 +4,10 @@ import re import tempfile -from testgres import ExecUtilException -from testgres import InvalidOperationException -from testgres import RemoteOperations -from testgres import ConnectionParams +from ..testgres import ExecUtilException +from ..testgres import InvalidOperationException +from ..testgres import RemoteOperations +from ..testgres import ConnectionParams class TestRemoteOperations: diff --git a/tests/test_simple.py b/tests/test_simple.py index 55659e86..30440a72 100644 --- a/tests/test_simple.py +++ b/tests/test_simple.py @@ -2,7 +2,6 @@ import re import subprocess import tempfile -import testgres import time import six import pytest @@ -14,7 +13,9 @@ from contextlib import contextmanager from shutil import rmtree -from testgres import \ +from .. import testgres + +from ..testgres import \ InitNodeException, \ StartNodeException, \ ExecUtilException, \ @@ -25,31 +26,32 @@ InvalidOperationException, \ NodeApp -from testgres import \ +from ..testgres import \ TestgresConfig, \ configure_testgres, \ scoped_config, \ pop_config -from testgres import \ +from ..testgres import \ NodeStatus, \ ProcessType, \ IsolationLevel, \ get_new_node -from testgres import \ +from ..testgres import \ get_bin_path, \ get_pg_config, \ get_pg_version -from testgres import \ +from ..testgres import \ First, \ Any # NOTE: those are ugly imports -from testgres import bound_ports -from testgres.utils import PgVer, parse_pg_version -from testgres.node import ProcessProxy +from ..testgres import bound_ports +from ..testgres.utils import PgVer, parse_pg_version +from ..testgres.utils import file_tail +from ..testgres.node import ProcessProxy def pg_version_ge(version): @@ -874,8 +876,6 @@ def test_auto_name(self): assert ('testgres' in r.name) def test_file_tail(self): - from testgres.utils import file_tail - s1 = "the quick brown fox jumped over that lazy dog\n" s2 = "abc\n" s3 = "def\n" diff --git a/tests/test_simple_remote.py b/tests/test_simple_remote.py index cfd92ac6..403b3c49 100755 --- a/tests/test_simple_remote.py +++ b/tests/test_simple_remote.py @@ -3,7 +3,6 @@ import subprocess import tempfile -import testgres import time import six import pytest @@ -13,7 +12,9 @@ from contextlib import contextmanager -from testgres.exceptions import \ +from .. import testgres + +from ..testgres.exceptions import \ InitNodeException, \ StartNodeException, \ ExecUtilException, \ @@ -23,39 +24,33 @@ TestgresException, \ InvalidOperationException -from testgres.config import \ +from ..testgres.config import \ TestgresConfig, \ configure_testgres, \ scoped_config, \ pop_config, testgres_config -from testgres import \ +from ..testgres import \ NodeStatus, \ ProcessType, \ IsolationLevel, \ get_remote_node, \ RemoteOperations -from testgres import \ +from ..testgres import \ get_bin_path, \ get_pg_config, \ get_pg_version -from testgres import \ +from ..testgres import \ First, \ Any # NOTE: those are ugly imports -from testgres import bound_ports -from testgres.utils import PgVer -from testgres.node import ProcessProxy, ConnectionParams - -conn_params = ConnectionParams(host=os.getenv('RDBMS_TESTPOOL1_HOST') or '127.0.0.1', - username=os.getenv('USER'), - ssh_key=os.getenv('RDBMS_TESTPOOL_SSHKEY')) -os_ops = RemoteOperations(conn_params) -testgres_config.set_os_ops(os_ops=os_ops) - +from ..testgres import bound_ports +from ..testgres.utils import PgVer +from ..testgres.utils import file_tail +from ..testgres.node import ProcessProxy, ConnectionParams def pg_version_ge(version): cur_ver = PgVer(get_pg_version()) @@ -65,16 +60,16 @@ def pg_version_ge(version): def util_exists(util): def good_properties(f): - return (os_ops.path_exists(f) and # noqa: W504 - os_ops.isfile(f) and # noqa: W504 - os_ops.is_executable(f)) # yapf: disable + return (testgres_config.os_ops.path_exists(f) and # noqa: W504 + testgres_config.os_ops.isfile(f) and # noqa: W504 + testgres_config.os_ops.is_executable(f)) # yapf: disable # try to resolve it if good_properties(get_bin_path(util)): return True # check if util is in PATH - for path in os_ops.environ("PATH").split(os_ops.pathsep): + for path in testgres_config.os_ops.environ("PATH").split(testgres_config.os_ops.pathsep): if good_properties(os.path.join(path, util)): return True @@ -84,31 +79,50 @@ def removing(f): try: yield f finally: - if os_ops.isfile(f): - os_ops.remove_file(f) + if testgres_config.os_ops.isfile(f): + testgres_config.os_ops.remove_file(f) - elif os_ops.isdir(f): - os_ops.rmdirs(f, ignore_errors=True) + elif testgres_config.os_ops.isdir(f): + testgres_config.os_ops.rmdirs(f, ignore_errors=True) class TestgresRemoteTests: + sm_conn_params = ConnectionParams( + host=os.getenv('RDBMS_TESTPOOL1_HOST') or '127.0.0.1', + username=os.getenv('USER'), + ssh_key=os.getenv('RDBMS_TESTPOOL_SSHKEY')) + + sm_os_ops = RemoteOperations(sm_conn_params) + + @pytest.fixture(autouse=True, scope="class") + def implicit_fixture(self): + prev_ops = testgres_config.os_ops + assert prev_ops is not None + assert __class__.sm_os_ops is not None + testgres_config.set_os_ops(os_ops=__class__.sm_os_ops) + assert testgres_config.os_ops is __class__.sm_os_ops + yield + assert testgres_config.os_ops is __class__.sm_os_ops + testgres_config.set_os_ops(os_ops=prev_ops) + assert testgres_config.os_ops is prev_ops + def test_node_repr(self): - with get_remote_node(conn_params=conn_params) as node: + with __class__.helper__get_node() as node: pattern = r"PostgresNode\(name='.+', port=.+, base_dir='.+'\)" assert re.match(pattern, str(node)) is not None def test_custom_init(self): - with get_remote_node(conn_params=conn_params) as node: + with __class__.helper__get_node() as node: # enable page checksums node.init(initdb_params=['-k']).start() - with get_remote_node(conn_params=conn_params) as node: + with __class__.helper__get_node() as node: node.init( allow_streaming=True, initdb_params=['--auth-local=reject', '--auth-host=reject']) hba_file = os.path.join(node.data_dir, 'pg_hba.conf') - lines = os_ops.readlines(hba_file) + lines = node.os_ops.readlines(hba_file) # check number of lines assert (len(lines) >= 6) @@ -123,7 +137,7 @@ def test_init__LANG_ะก(self): try: os.environ["LANG"] = "C" - with get_remote_node(conn_params=conn_params) as node: + with __class__.helper__get_node() as node: node.init().start() finally: __class__.helper__restore_envvar("LANG", prev_LANG) @@ -164,9 +178,9 @@ def test_init__unk_LANG_and_LC_CTYPE(self): while True: try: - with get_remote_node(conn_params=conn_params): + with __class__.helper__get_node(): pass - except testgres.exceptions.ExecUtilException as e: + except ExecUtilException as e: # # Example of an error message: # @@ -190,13 +204,13 @@ def test_init__unk_LANG_and_LC_CTYPE(self): __class__.helper__restore_envvar("LC_COLLATE", prev_LC_COLLATE) def test_double_init(self): - with get_remote_node(conn_params=conn_params).init() as node: + with __class__.helper__get_node().init() as node: # can't initialize node more than once with pytest.raises(expected_exception=InitNodeException): node.init() def test_init_after_cleanup(self): - with get_remote_node(conn_params=conn_params) as node: + with __class__.helper__get_node() as node: node.init().start().execute('select 1') node.cleanup() node.init().start().execute('select 1') @@ -211,7 +225,7 @@ def test_init_unique_system_id(self): query = 'select system_identifier from pg_control_system()' with scoped_config(cache_initdb=False): - with get_remote_node(conn_params=conn_params).init().start() as node0: + with __class__.helper__get_node().init().start() as node0: id0 = node0.execute(query)[0] with scoped_config(cache_initdb=True, @@ -220,8 +234,8 @@ def test_init_unique_system_id(self): assert (config.cached_initdb_unique) # spawn two nodes; ids must be different - with get_remote_node(conn_params=conn_params).init().start() as node1, \ - get_remote_node(conn_params=conn_params).init().start() as node2: + with __class__.helper__get_node().init().start() as node1, \ + __class__.helper__get_node().init().start() as node2: id1 = node1.execute(query)[0] id2 = node2.execute(query)[0] @@ -231,34 +245,34 @@ def test_init_unique_system_id(self): def test_node_exit(self): with pytest.raises(expected_exception=QueryException): - with get_remote_node(conn_params=conn_params).init() as node: + with __class__.helper__get_node().init() as node: base_dir = node.base_dir node.safe_psql('select 1') # we should save the DB for "debugging" - assert (os_ops.path_exists(base_dir)) - os_ops.rmdirs(base_dir, ignore_errors=True) + assert (__class__.sm_os_ops.path_exists(base_dir)) + __class__.sm_os_ops.rmdirs(base_dir, ignore_errors=True) - with get_remote_node(conn_params=conn_params).init() as node: + with __class__.helper__get_node().init() as node: base_dir = node.base_dir # should have been removed by default - assert not (os_ops.path_exists(base_dir)) + assert not (__class__.sm_os_ops.path_exists(base_dir)) def test_double_start(self): - with get_remote_node(conn_params=conn_params).init().start() as node: + with __class__.helper__get_node().init().start() as node: # can't start node more than once node.start() assert (node.is_started) def test_uninitialized_start(self): - with get_remote_node(conn_params=conn_params) as node: + with __class__.helper__get_node() as node: # node is not initialized yet with pytest.raises(expected_exception=StartNodeException): node.start() def test_restart(self): - with get_remote_node(conn_params=conn_params) as node: + with __class__.helper__get_node() as node: node.init().start() # restart, ok @@ -274,7 +288,7 @@ def test_restart(self): node.restart() def test_reload(self): - with get_remote_node(conn_params=conn_params) as node: + with __class__.helper__get_node() as node: node.init().start() # change client_min_messages and save old value @@ -290,7 +304,7 @@ def test_reload(self): assert (cmm_old != cmm_new) def test_pg_ctl(self): - with get_remote_node(conn_params=conn_params) as node: + with __class__.helper__get_node() as node: node.init().start() status = node.pg_ctl(['status']) @@ -302,7 +316,7 @@ def test_status(self): assert not (NodeStatus.Uninitialized) # check statuses after each operation - with get_remote_node(conn_params=conn_params) as node: + with __class__.helper__get_node() as node: assert (node.pid == 0) assert (node.status() == NodeStatus.Uninitialized) @@ -327,7 +341,7 @@ def test_status(self): assert (node.status() == NodeStatus.Uninitialized) def test_psql(self): - with get_remote_node(conn_params=conn_params).init().start() as node: + with __class__.helper__get_node().init().start() as node: # check returned values (1 arg) res = node.psql('select 1') assert (res == (0, b'1\n', b'')) @@ -370,7 +384,7 @@ def test_psql(self): node.safe_psql('select 1') def test_safe_psql__expect_error(self): - with get_remote_node(conn_params=conn_params).init().start() as node: + with __class__.helper__get_node().init().start() as node: err = node.safe_psql('select_or_not_select 1', expect_error=True) assert (type(err) == str) # noqa: E721 assert ('select_or_not_select' in err) @@ -388,7 +402,7 @@ def test_safe_psql__expect_error(self): assert (res == b'1\n') def test_transactions(self): - with get_remote_node(conn_params=conn_params).init().start() as node: + with __class__.helper__get_node().init().start() as node: with node.connect() as con: con.begin() con.execute('create table test(val int)') @@ -411,7 +425,7 @@ def test_transactions(self): con.commit() def test_control_data(self): - with get_remote_node(conn_params=conn_params) as node: + with __class__.helper__get_node() as node: # node is not initialized yet with pytest.raises(expected_exception=ExecUtilException): node.get_control_data() @@ -424,7 +438,7 @@ def test_control_data(self): assert (any('pg_control' in s for s in data.keys())) def test_backup_simple(self): - with get_remote_node(conn_params=conn_params) as master: + with __class__.helper__get_node() as master: # enable streaming for backups master.init(allow_streaming=True) @@ -444,7 +458,7 @@ def test_backup_simple(self): assert (res == [(1,), (2,), (3,), (4,)]) def test_backup_multiple(self): - with get_remote_node(conn_params=conn_params) as node: + with __class__.helper__get_node() as node: node.init(allow_streaming=True).start() with node.backup(xlog_method='fetch') as backup1, \ @@ -457,7 +471,7 @@ def test_backup_multiple(self): assert (node1.base_dir != node2.base_dir) def test_backup_exhaust(self): - with get_remote_node(conn_params=conn_params) as node: + with __class__.helper__get_node() as node: node.init(allow_streaming=True).start() with node.backup(xlog_method='fetch') as backup: @@ -470,7 +484,7 @@ def test_backup_exhaust(self): backup.spawn_primary() def test_backup_wrong_xlog_method(self): - with get_remote_node(conn_params=conn_params) as node: + with __class__.helper__get_node() as node: node.init(allow_streaming=True).start() with pytest.raises( @@ -480,7 +494,7 @@ def test_backup_wrong_xlog_method(self): node.backup(xlog_method='wrong') def test_pg_ctl_wait_option(self): - with get_remote_node(conn_params=conn_params) as node: + with __class__.helper__get_node() as node: node.init().start(wait=False) while True: try: @@ -492,7 +506,7 @@ def test_pg_ctl_wait_option(self): pass def test_replicate(self): - with get_remote_node(conn_params=conn_params) as node: + with __class__.helper__get_node() as node: node.init(allow_streaming=True).start() with node.replicate().start() as replica: @@ -510,7 +524,7 @@ def test_replicate(self): def test_synchronous_replication(self): __class__.helper__skip_test_if_pg_version_is_not_ge("9.6") - with get_remote_node(conn_params=conn_params) as master: + with __class__.helper__get_node() as master: old_version = not pg_version_ge('9.6') master.init(allow_streaming=True).start() @@ -553,7 +567,7 @@ def test_synchronous_replication(self): def test_logical_replication(self): __class__.helper__skip_test_if_pg_version_is_not_ge("10") - with get_remote_node(conn_params=conn_params) as node1, get_remote_node(conn_params=conn_params) as node2: + with __class__.helper__get_node() as node1, __class__.helper__get_node() as node2: node1.init(allow_logical=True) node1.start() node2.init().start() @@ -625,7 +639,7 @@ def test_logical_catchup(self): """ Runs catchup for 100 times to be sure that it is consistent """ __class__.helper__skip_test_if_pg_version_is_not_ge("10") - with get_remote_node(conn_params=conn_params) as node1, get_remote_node(conn_params=conn_params) as node2: + with __class__.helper__get_node() as node1, __class__.helper__get_node() as node2: node1.init(allow_logical=True) node1.start() node2.init().start() @@ -649,12 +663,12 @@ def test_logical_catchup(self): def test_logical_replication_fail(self): __class__.helper__skip_test_if_pg_version_is_ge("10") - with get_remote_node(conn_params=conn_params) as node: + with __class__.helper__get_node() as node: with pytest.raises(expected_exception=InitNodeException): node.init(allow_logical=True) def test_replication_slots(self): - with get_remote_node(conn_params=conn_params) as node: + with __class__.helper__get_node() as node: node.init(allow_streaming=True).start() with node.replicate(slot='slot1').start() as replica: @@ -665,7 +679,7 @@ def test_replication_slots(self): node.replicate(slot='slot1') def test_incorrect_catchup(self): - with get_remote_node(conn_params=conn_params) as node: + with __class__.helper__get_node() as node: node.init(allow_streaming=True).start() # node has no master, can't catch up @@ -673,7 +687,7 @@ def test_incorrect_catchup(self): node.catchup() def test_promotion(self): - with get_remote_node(conn_params=conn_params) as master: + with __class__.helper__get_node() as master: master.init().start() master.safe_psql('create table abc(id serial)') @@ -690,29 +704,29 @@ def test_dump(self): query_create = 'create table test as select generate_series(1, 2) as val' query_select = 'select * from test order by val asc' - with get_remote_node(conn_params=conn_params).init().start() as node1: + with __class__.helper__get_node().init().start() as node1: node1.execute(query_create) for format in ['plain', 'custom', 'directory', 'tar']: with removing(node1.dump(format=format)) as dump: - with get_remote_node(conn_params=conn_params).init().start() as node3: + with __class__.helper__get_node().init().start() as node3: if format == 'directory': - assert (os_ops.isdir(dump)) + assert (node1.os_ops.isdir(dump)) else: - assert (os_ops.isfile(dump)) + assert (node1.os_ops.isfile(dump)) # restore dump node3.restore(filename=dump) res = node3.execute(query_select) assert (res == [(1,), (2,)]) def test_users(self): - with get_remote_node(conn_params=conn_params).init().start() as node: + with __class__.helper__get_node().init().start() as node: node.psql('create role test_user login') value = node.safe_psql('select 1', username='test_user') assert (b'1\n' == value) def test_poll_query_until(self): - with get_remote_node(conn_params=conn_params) as node: + with __class__.helper__get_node() as node: node.init().start() get_time = 'select extract(epoch from now())' @@ -828,7 +842,7 @@ def test_logging(self): def test_pgbench(self): __class__.helper__skip_test_if_util_not_exist("pgbench") - with get_remote_node(conn_params=conn_params).init().start() as node: + with __class__.helper__get_node().init().start() as node: # initialize pgbench DB and run benchmarks node.pgbench_init(scale=2, foreign_keys=True, options=['-q']).pgbench_run(time=2) @@ -897,7 +911,7 @@ def test_config_stack(self): assert (TestgresConfig.cached_initdb_dir == d0) def test_unix_sockets(self): - with get_remote_node(conn_params=conn_params) as node: + with __class__.helper__get_node() as node: node.init(unix_sockets=False, allow_streaming=True) node.start() @@ -913,7 +927,7 @@ def test_unix_sockets(self): assert (res_psql == b'1\n') def test_auto_name(self): - with get_remote_node(conn_params=conn_params).init(allow_streaming=True).start() as m: + with __class__.helper__get_node().init(allow_streaming=True).start() as m: with m.replicate().start() as r: # check that nodes are running assert (m.status()) @@ -925,8 +939,6 @@ def test_auto_name(self): assert ('testgres' in r.name) def test_file_tail(self): - from testgres.utils import file_tail - s1 = "the quick brown fox jumped over that lazy dog\n" s2 = "abc\n" s3 = "def\n" @@ -950,7 +962,7 @@ def test_file_tail(self): assert (lines[0] == s3) def test_isolation_levels(self): - with get_remote_node(conn_params=conn_params).init().start() as node: + with __class__.helper__get_node().init().start() as node: with node.connect() as con: # string levels con.begin('Read Uncommitted').commit() @@ -972,7 +984,7 @@ def test_ports_management(self): # check that no ports have been bound yet assert (len(bound_ports) == 0) - with get_remote_node(conn_params=conn_params) as node: + with __class__.helper__get_node() as node: # check that we've just bound a port assert (len(bound_ports) == 1) @@ -1005,7 +1017,7 @@ def test_version_management(self): assert (d > f) version = get_pg_version() - with get_remote_node(conn_params=conn_params) as node: + with __class__.helper__get_node() as node: assert (isinstance(version, six.string_types)) assert (isinstance(node.version, PgVer)) assert (node.version == PgVer(version)) @@ -1031,7 +1043,7 @@ def test_child_pids(self): ProcessType.WalReceiver, ] - with get_remote_node(conn_params=conn_params).init().start() as master: + with __class__.helper__get_node().init().start() as master: # master node doesn't have a source walsender! with pytest.raises(expected_exception=TestgresException): @@ -1077,6 +1089,11 @@ def test_child_process_dies(self): # try to handle children list -- missing processes will have ptype "ProcessType.Unknown" [ProcessProxy(p) for p in children] + @staticmethod + def helper__get_node(): + assert __class__.sm_conn_params is not None + return get_remote_node(conn_params=__class__.sm_conn_params) + @staticmethod def helper__restore_envvar(name, prev_value): if prev_value is None: From 842bc086ba77140bce470b3e5de82f991121a5c2 Mon Sep 17 00:00:00 2001 From: "d.kovalenko" Date: Mon, 24 Feb 2025 11:53:26 +0300 Subject: [PATCH 18/36] Code style (flake8) --- tests/test_simple_remote.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/test_simple_remote.py b/tests/test_simple_remote.py index 403b3c49..e0bc265e 100755 --- a/tests/test_simple_remote.py +++ b/tests/test_simple_remote.py @@ -52,6 +52,7 @@ from ..testgres.utils import file_tail from ..testgres.node import ProcessProxy, ConnectionParams + def pg_version_ge(version): cur_ver = PgVer(get_pg_version()) min_ver = PgVer(version) @@ -93,7 +94,7 @@ class TestgresRemoteTests: ssh_key=os.getenv('RDBMS_TESTPOOL_SSHKEY')) sm_os_ops = RemoteOperations(sm_conn_params) - + @pytest.fixture(autouse=True, scope="class") def implicit_fixture(self): prev_ops = testgres_config.os_ops From 969e49d49ec4d942d5a1a37acda78d5e3cc299ed Mon Sep 17 00:00:00 2001 From: "d.kovalenko" Date: Mon, 24 Feb 2025 16:26:06 +0300 Subject: [PATCH 19/36] Root __init__.py is added It is required for tests. --- __init__.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 __init__.py diff --git a/__init__.py b/__init__.py new file mode 100644 index 00000000..e69de29b From 6e2820df688c70a57ceaf670b39bebb075ec4ed8 Mon Sep 17 00:00:00 2001 From: "d.kovalenko" Date: Mon, 24 Feb 2025 16:34:41 +0300 Subject: [PATCH 20/36] Code style (flake8) --- testgres/cache.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/testgres/cache.py b/testgres/cache.py index 61d44868..3ac63326 100644 --- a/testgres/cache.py +++ b/testgres/cache.py @@ -32,11 +32,11 @@ def cached_initdb(data_dir, logfile=None, params=None, os_ops: OsOperations = Lo def make_utility_path(name): assert name is not None - assert type(name) == str + assert type(name) == str # noqa: E721 if bin_path: return os.path.join(bin_path, name) - + return get_bin_path2(os_ops, name) def call_initdb(initdb_dir, log=logfile): From e4e890905ebfe7d34bc1037adfe25897c18b5a47 Mon Sep 17 00:00:00 2001 From: "d.kovalenko" Date: Mon, 24 Feb 2025 17:43:56 +0300 Subject: [PATCH 21/36] pytest.ini is added --- pytest.ini | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 pytest.ini diff --git a/pytest.ini b/pytest.ini new file mode 100644 index 00000000..61d4f9bc --- /dev/null +++ b/pytest.ini @@ -0,0 +1,9 @@ +[pytest] +testpaths = ./tests +addopts = --strict-markers +markers = +#log_file = logs/pytest.log +log_file_level = NOTSET +log_file_format = %(levelname)8s [%(asctime)s] %(message)s +log_file_date_format=%Y-%m-%d %H:%M:%S + From 66860d0a76ca924e5aa8821542db636bdaa5f540 Mon Sep 17 00:00:00 2001 From: "d.kovalenko" Date: Mon, 24 Feb 2025 18:51:26 +0300 Subject: [PATCH 22/36] TestgresTests::test_ports_management is corrected Let's send warning about a garbage in the container "bound_ports" and continue working. --- tests/test_simple.py | 36 +++++++++++++++++++++++++++--------- 1 file changed, 27 insertions(+), 9 deletions(-) diff --git a/tests/test_simple.py b/tests/test_simple.py index 30440a72..15a26e1b 100644 --- a/tests/test_simple.py +++ b/tests/test_simple.py @@ -918,20 +918,38 @@ def test_isolation_levels(self): con.begin('Garbage').commit() def test_ports_management(self): - # check that no ports have been bound yet - assert (len(bound_ports) == 0) + assert bound_ports is not None + assert type(bound_ports) == set # noqa: E721 + + if len(bound_ports) != 0: + logging.warning("bound_ports is not empty: {0}".format(bound_ports)) + + stage0__bound_ports = bound_ports.copy() with get_new_node() as node: - # check that we've just bound a port - assert (len(bound_ports) == 1) + assert bound_ports is not None + assert type(bound_ports) == set # noqa: E721 + + assert node.port is not None + assert type(node.port) == int # noqa: E721 + + logging.info("node port is {0}".format(node.port)) + + assert node.port in bound_ports + assert node.port not in stage0__bound_ports + + assert stage0__bound_ports <= bound_ports + assert len(stage0__bound_ports) + 1 == len(bound_ports) + + stage1__bound_ports = stage0__bound_ports.copy() + stage1__bound_ports.add(node.port) - # check that bound_ports contains our port - port_1 = list(bound_ports)[0] - port_2 = node.port - assert (port_1 == port_2) + assert stage1__bound_ports == bound_ports # check that port has been freed successfully - assert (len(bound_ports) == 0) + assert bound_ports is not None + assert type(bound_ports) == set # noqa: E721 + assert bound_ports == stage0__bound_ports def test_exceptions(self): str(StartNodeException('msg', [('file', 'lines')])) From aaaa677e20ac97af667a381c29246b021f3077a4 Mon Sep 17 00:00:00 2001 From: "d.kovalenko" Date: Mon, 24 Feb 2025 18:53:20 +0300 Subject: [PATCH 23/36] coding: utf-8 --- tests/helpers/run_conditions.py | 1 + tests/test_local.py | 1 + tests/test_remote.py | 1 + tests/test_simple.py | 1 + tests/test_simple_remote.py | 1 + 5 files changed, 5 insertions(+) diff --git a/tests/helpers/run_conditions.py b/tests/helpers/run_conditions.py index 8d57f753..11357c30 100644 --- a/tests/helpers/run_conditions.py +++ b/tests/helpers/run_conditions.py @@ -1,3 +1,4 @@ +# coding: utf-8 import pytest import platform diff --git a/tests/test_local.py b/tests/test_local.py index 346336bd..60a96c18 100644 --- a/tests/test_local.py +++ b/tests/test_local.py @@ -1,3 +1,4 @@ +# coding: utf-8 import os import pytest diff --git a/tests/test_remote.py b/tests/test_remote.py index 21bc9e8f..8b167e9f 100755 --- a/tests/test_remote.py +++ b/tests/test_remote.py @@ -1,3 +1,4 @@ +# coding: utf-8 import os import pytest diff --git a/tests/test_simple.py b/tests/test_simple.py index 15a26e1b..b9da8ad5 100644 --- a/tests/test_simple.py +++ b/tests/test_simple.py @@ -1,3 +1,4 @@ +# coding: utf-8 import os import re import subprocess diff --git a/tests/test_simple_remote.py b/tests/test_simple_remote.py index 6d681190..99b024f4 100755 --- a/tests/test_simple_remote.py +++ b/tests/test_simple_remote.py @@ -1,3 +1,4 @@ +# coding: utf-8 import os import re import subprocess From 60bb4c52355db7b62812e2b96a0a41d3eda1a67f Mon Sep 17 00:00:00 2001 From: "d.kovalenko" Date: Mon, 24 Feb 2025 20:01:47 +0300 Subject: [PATCH 24/36] Cleanup --- tests/test_simple.py | 7 ------- tests/test_simple_remote.py | 7 ------- 2 files changed, 14 deletions(-) diff --git a/tests/test_simple.py b/tests/test_simple.py index b9da8ad5..4cb63e4f 100644 --- a/tests/test_simple.py +++ b/tests/test_simple.py @@ -144,8 +144,6 @@ def test_init_after_cleanup(self): node.cleanup() node.init().start().execute('select 1') - # @unittest.skipUnless(util_exists('pg_resetwal.exe' if os.name == 'nt' else 'pg_resetwal'), 'pgbench might be missing') - # @unittest.skipUnless(pg_version_ge('9.6'), 'requires 9.6+') def test_init_unique_system_id(self): # this function exists in PostgreSQL 9.6+ __class__.helper__skip_test_if_util_not_exist("pg_resetwal") @@ -456,7 +454,6 @@ def test_replicate(self): res = node.execute('select * from test') assert (res == []) - # @unittest.skipUnless(pg_version_ge('9.6'), 'requires 9.6+') def test_synchronous_replication(self): __class__.helper__skip_test_if_pg_version_is_not_ge("9.6") @@ -499,7 +496,6 @@ def test_synchronous_replication(self): res = standby1.safe_psql('select count(*) from abc') assert (rm_carriage_returns(res) == b'1000000\n') - # @unittest.skipUnless(pg_version_ge('10'), 'requires 10+') def test_logical_replication(self): __class__.helper__skip_test_if_pg_version_is_not_ge("10") @@ -570,7 +566,6 @@ def test_logical_replication(self): res = node2.execute('select * from test2') assert (res == [('a', ), ('b', )]) - # @unittest.skipUnless(pg_version_ge('10'), 'requires 10+') def test_logical_catchup(self): """ Runs catchup for 100 times to be sure that it is consistent """ __class__.helper__skip_test_if_pg_version_is_not_ge("10") @@ -595,7 +590,6 @@ def test_logical_catchup(self): assert (res == [(i, i, )]) node1.execute('delete from test') - # @unittest.skipIf(pg_version_ge('10'), 'requires <10') def test_logical_replication_fail(self): __class__.helper__skip_test_if_pg_version_is_ge("10") @@ -774,7 +768,6 @@ def test_logging(self): master.restart() assert (master._logger.is_alive()) - # @unittest.skipUnless(util_exists('pgbench.exe' if os.name == 'nt' else 'pgbench'), 'pgbench might be missing') def test_pgbench(self): __class__.helper__skip_test_if_util_not_exist("pgbench") diff --git a/tests/test_simple_remote.py b/tests/test_simple_remote.py index 99b024f4..27852428 100755 --- a/tests/test_simple_remote.py +++ b/tests/test_simple_remote.py @@ -217,8 +217,6 @@ def test_init_after_cleanup(self): node.cleanup() node.init().start().execute('select 1') - # @unittest.skipUnless(util_exists('pg_resetwal'), 'might be missing') - # @unittest.skipUnless(pg_version_ge('9.6'), 'requires 9.6+') def test_init_unique_system_id(self): # this function exists in PostgreSQL 9.6+ __class__.helper__skip_test_if_util_not_exist("pg_resetwal") @@ -522,7 +520,6 @@ def test_replicate(self): res = node.execute('select * from test') assert (res == []) - # @unittest.skipUnless(pg_version_ge('9.6'), 'requires 9.6+') def test_synchronous_replication(self): __class__.helper__skip_test_if_pg_version_is_not_ge("9.6") @@ -565,7 +562,6 @@ def test_synchronous_replication(self): res = standby1.safe_psql('select count(*) from abc') assert (res == b'1000000\n') - # @unittest.skipUnless(pg_version_ge('10'), 'requires 10+') def test_logical_replication(self): __class__.helper__skip_test_if_pg_version_is_not_ge("10") @@ -636,7 +632,6 @@ def test_logical_replication(self): res = node2.execute('select * from test2') assert (res == [('a',), ('b',)]) - # @unittest.skipUnless(pg_version_ge('10'), 'requires 10+') def test_logical_catchup(self): """ Runs catchup for 100 times to be sure that it is consistent """ __class__.helper__skip_test_if_pg_version_is_not_ge("10") @@ -661,7 +656,6 @@ def test_logical_catchup(self): assert (res == [(i, i, )]) node1.execute('delete from test') - # @unittest.skipIf(pg_version_ge('10'), 'requires <10') def test_logical_replication_fail(self): __class__.helper__skip_test_if_pg_version_is_ge("10") @@ -840,7 +834,6 @@ def test_logging(self): master.restart() assert (master._logger.is_alive()) - # @unittest.skipUnless(util_exists('pgbench'), 'might be missing') def test_pgbench(self): __class__.helper__skip_test_if_util_not_exist("pgbench") From 2d2532c77e8d7521552c0f3511c119e90d55573e Mon Sep 17 00:00:00 2001 From: "d.kovalenko" Date: Mon, 24 Feb 2025 20:04:22 +0300 Subject: [PATCH 25/36] CI runs all the tests of testgres. --- run_tests.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/run_tests.sh b/run_tests.sh index 6995b7e4..010bfe91 100755 --- a/run_tests.sh +++ b/run_tests.sh @@ -38,19 +38,19 @@ rm -f $COVERAGE_FILE # run tests (PATH) -time coverage run -a -m pytest -l -v -n 2 -k "TestgresTests" +time coverage run -a -m pytest -l -v -n 4 # run tests (PG_BIN) time \ PG_BIN=$(dirname $(which pg_config)) \ - coverage run -a -m pytest -l -v -n 2 -k "TestgresTests" + coverage run -a -m pytest -l -v -n 4 # run tests (PG_CONFIG) time \ PG_CONFIG=$(which pg_config) \ - coverage run -a -m pytest -l -v -n 2 -k "TestgresTests" + coverage run -a -m pytest -l -v -n 4 # show coverage From 537a9acb9dfb26d82251d2d68796a55989be8317 Mon Sep 17 00:00:00 2001 From: vshepard Date: Mon, 24 Feb 2025 09:07:50 +0100 Subject: [PATCH 26/36] Add install ssh (cherry picked from commit fec1e7ac9d6e2bfb43a01f0e370336ba5ed8e971) --- Dockerfile.tmpl | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/Dockerfile.tmpl b/Dockerfile.tmpl index dc5878b6..6749483c 100644 --- a/Dockerfile.tmpl +++ b/Dockerfile.tmpl @@ -9,6 +9,15 @@ RUN if [ "${PYTHON_VERSION}" = "3" ] ; then \ apk add --no-cache curl python3 python3-dev build-base musl-dev \ linux-headers py-virtualenv; \ fi + +# Install OpenSSH +RUN apk add --no-cache openssh + +# Configure SSH +RUN echo "postgres:postgres" | chpasswd && \ + mkdir -p /var/run/sshd && \ + sed -i 's/#PermitRootLogin prohibit-password/PermitRootLogin yes/' /etc/ssh/sshd_config + ENV LANG=C.UTF-8 RUN mkdir -p /pg @@ -19,5 +28,8 @@ ADD . /pg/testgres WORKDIR /pg/testgres RUN chown -R postgres:postgres /pg +# Expose SSH port +EXPOSE 22 + USER postgres -ENTRYPOINT PYTHON_VERSION=${PYTHON_VERSION} /run.sh +ENTRYPOINT PYTHON_VERSION=${PYTHON_VERSION} /run.sh & /usr/sbin/sshd -D From 1f998133a08e149f8b678e867e3615e5cebc8a2d Mon Sep 17 00:00:00 2001 From: "d.kovalenko" Date: Tue, 25 Feb 2025 09:46:03 +0300 Subject: [PATCH 27/36] Revert "Add install ssh" This reverts commit 537a9acb9dfb26d82251d2d68796a55989be8317. --- Dockerfile.tmpl | 14 +------------- 1 file changed, 1 insertion(+), 13 deletions(-) diff --git a/Dockerfile.tmpl b/Dockerfile.tmpl index 6749483c..dc5878b6 100644 --- a/Dockerfile.tmpl +++ b/Dockerfile.tmpl @@ -9,15 +9,6 @@ RUN if [ "${PYTHON_VERSION}" = "3" ] ; then \ apk add --no-cache curl python3 python3-dev build-base musl-dev \ linux-headers py-virtualenv; \ fi - -# Install OpenSSH -RUN apk add --no-cache openssh - -# Configure SSH -RUN echo "postgres:postgres" | chpasswd && \ - mkdir -p /var/run/sshd && \ - sed -i 's/#PermitRootLogin prohibit-password/PermitRootLogin yes/' /etc/ssh/sshd_config - ENV LANG=C.UTF-8 RUN mkdir -p /pg @@ -28,8 +19,5 @@ ADD . /pg/testgres WORKDIR /pg/testgres RUN chown -R postgres:postgres /pg -# Expose SSH port -EXPOSE 22 - USER postgres -ENTRYPOINT PYTHON_VERSION=${PYTHON_VERSION} /run.sh & /usr/sbin/sshd -D +ENTRYPOINT PYTHON_VERSION=${PYTHON_VERSION} /run.sh From 41455d676af49431dfb4d04066294889bff49540 Mon Sep 17 00:00:00 2001 From: "d.kovalenko" Date: Tue, 25 Feb 2025 09:46:03 +0300 Subject: [PATCH 28/36] Revert "CI runs all the tests of testgres." This reverts commit 2d2532c77e8d7521552c0f3511c119e90d55573e. --- run_tests.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/run_tests.sh b/run_tests.sh index 010bfe91..6995b7e4 100755 --- a/run_tests.sh +++ b/run_tests.sh @@ -38,19 +38,19 @@ rm -f $COVERAGE_FILE # run tests (PATH) -time coverage run -a -m pytest -l -v -n 4 +time coverage run -a -m pytest -l -v -n 2 -k "TestgresTests" # run tests (PG_BIN) time \ PG_BIN=$(dirname $(which pg_config)) \ - coverage run -a -m pytest -l -v -n 4 + coverage run -a -m pytest -l -v -n 2 -k "TestgresTests" # run tests (PG_CONFIG) time \ PG_CONFIG=$(which pg_config) \ - coverage run -a -m pytest -l -v -n 4 + coverage run -a -m pytest -l -v -n 2 -k "TestgresTests" # show coverage From 47a51a82b2572e29a4794f392ff482c6464ee692 Mon Sep 17 00:00:00 2001 From: "d.kovalenko" Date: Tue, 25 Feb 2025 12:32:35 +0300 Subject: [PATCH 29/36] Test of probackup plugin is restored It works now (was runned with a fresh probackup2 and vanilla 18devel). --- testgres/plugins/__init__.py | 8 ++++---- .../pg_probackup2/pg_probackup2/tests/__init__.py | 0 .../pg_probackup2/tests/basic_test.py | 14 ++++++++------ 3 files changed, 12 insertions(+), 10 deletions(-) create mode 100644 testgres/plugins/pg_probackup2/pg_probackup2/tests/__init__.py diff --git a/testgres/plugins/__init__.py b/testgres/plugins/__init__.py index 8c19a23b..824eadc6 100644 --- a/testgres/plugins/__init__.py +++ b/testgres/plugins/__init__.py @@ -1,7 +1,7 @@ -from pg_probackup2.gdb import GDBobj -from pg_probackup2.app import ProbackupApp, ProbackupException -from pg_probackup2.init_helpers import init_params -from pg_probackup2.storage.fs_backup import FSTestBackupDir +from .pg_probackup2.pg_probackup2.gdb import GDBobj +from .pg_probackup2.pg_probackup2.app import ProbackupApp, ProbackupException +from .pg_probackup2.pg_probackup2.init_helpers import init_params +from .pg_probackup2.pg_probackup2.storage.fs_backup import FSTestBackupDir __all__ = [ "ProbackupApp", "ProbackupException", "init_params", "FSTestBackupDir", "GDBobj" diff --git a/testgres/plugins/pg_probackup2/pg_probackup2/tests/__init__.py b/testgres/plugins/pg_probackup2/pg_probackup2/tests/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/testgres/plugins/pg_probackup2/pg_probackup2/tests/basic_test.py b/testgres/plugins/pg_probackup2/pg_probackup2/tests/basic_test.py index b63531ec..ececd6e7 100644 --- a/testgres/plugins/pg_probackup2/pg_probackup2/tests/basic_test.py +++ b/testgres/plugins/pg_probackup2/pg_probackup2/tests/basic_test.py @@ -2,10 +2,10 @@ import os import shutil import unittest -import testgres -from pg_probackup2.app import ProbackupApp -from pg_probackup2.init_helpers import Init, init_params -from pg_probackup2.app import build_backup_dir +from ...... import testgres +from ...pg_probackup2.app import ProbackupApp +from ...pg_probackup2.init_helpers import Init, init_params +from ..storage.fs_backup import FSTestBackupDir class TestUtils: @@ -37,11 +37,11 @@ def setup_test_environment(self): def setup_test_paths(self): self.rel_path = os.path.join(self.module_name, self.fname) self.test_path = os.path.join(init_params.tmp_path, self.rel_path) - os.makedirs(self.test_path) + os.makedirs(self.test_path, exist_ok=True) self.pb_log_path = os.path.join(self.test_path, "pb_log") def setup_backup_dir(self): - self.backup_dir = build_backup_dir(self, 'backup') + self.backup_dir = self.build_backup_dir('backup') self.backup_dir.cleanup() def setup_probackup(self): @@ -53,6 +53,8 @@ def tearDown(self): if os.path.exists(self.test_path): shutil.rmtree(self.test_path) + def build_backup_dir(self, backup='backup'): + return FSTestBackupDir(rel_path=self.rel_path, backup=backup) class BasicTest(ProbackupTest): def test_full_backup(self): From f7f163d2186ff314bd54f9ab67fa68ea0bab2721 Mon Sep 17 00:00:00 2001 From: "d.kovalenko" Date: Tue, 25 Feb 2025 13:34:54 +0300 Subject: [PATCH 30/36] The test suite of a probackup plugin is based on pytest --- .../tests/{basic_test.py => test_basic.py} | 64 +++++++++---------- 1 file changed, 32 insertions(+), 32 deletions(-) rename testgres/plugins/pg_probackup2/pg_probackup2/tests/{basic_test.py => test_basic.py} (57%) diff --git a/testgres/plugins/pg_probackup2/pg_probackup2/tests/basic_test.py b/testgres/plugins/pg_probackup2/pg_probackup2/tests/test_basic.py similarity index 57% rename from testgres/plugins/pg_probackup2/pg_probackup2/tests/basic_test.py rename to testgres/plugins/pg_probackup2/pg_probackup2/tests/test_basic.py index ececd6e7..32b29498 100644 --- a/testgres/plugins/pg_probackup2/pg_probackup2/tests/basic_test.py +++ b/testgres/plugins/pg_probackup2/pg_probackup2/tests/test_basic.py @@ -1,62 +1,66 @@ -import logging +from __future__ import annotations + import os import shutil -import unittest +import pytest + from ...... import testgres from ...pg_probackup2.app import ProbackupApp from ...pg_probackup2.init_helpers import Init, init_params from ..storage.fs_backup import FSTestBackupDir -class TestUtils: - @staticmethod - def get_module_and_function_name(test_id): - try: - module_name = test_id.split('.')[-2] - fname = test_id.split('.')[-1] - except IndexError: - logging.warning(f"Couldn't get module name and function name from test_id: `{test_id}`") - module_name, fname = test_id.split('(')[1].split('.')[1], test_id.split('(')[0] - return module_name, fname +class ProbackupTest: + pg_node: testgres.PostgresNode + + @pytest.fixture(autouse=True, scope="function") + def implicit_fixture(self, request: pytest.FixtureRequest): + assert isinstance(request, pytest.FixtureRequest) + self.helper__setUp(request) + yield + self.helper__tearDown() + + def helper__setUp(self, request: pytest.FixtureRequest): + assert isinstance(request, pytest.FixtureRequest) + self.helper__setup_test_environment(request) + self.helper__setup_test_paths() + self.helper__setup_backup_dir() + self.helper__setup_probackup() -class ProbackupTest(unittest.TestCase): - def setUp(self): - self.setup_test_environment() - self.setup_test_paths() - self.setup_backup_dir() - self.setup_probackup() + def helper__setup_test_environment(self, request: pytest.FixtureRequest): + assert isinstance(request, pytest.FixtureRequest) - def setup_test_environment(self): self.output = None self.cmd = None self.nodes_to_cleanup = [] - self.module_name, self.fname = TestUtils.get_module_and_function_name(self.id()) + self.module_name, self.fname = request.node.cls.__name__, request.node.name self.test_env = Init().test_env() - def setup_test_paths(self): + def helper__setup_test_paths(self): self.rel_path = os.path.join(self.module_name, self.fname) self.test_path = os.path.join(init_params.tmp_path, self.rel_path) os.makedirs(self.test_path, exist_ok=True) self.pb_log_path = os.path.join(self.test_path, "pb_log") - def setup_backup_dir(self): - self.backup_dir = self.build_backup_dir('backup') + def helper__setup_backup_dir(self): + self.backup_dir = self.helper__build_backup_dir('backup') self.backup_dir.cleanup() - def setup_probackup(self): + def helper__setup_probackup(self): self.pg_node = testgres.NodeApp(self.test_path, self.nodes_to_cleanup) self.pb = ProbackupApp(self, self.pg_node, self.pb_log_path, self.test_env, auto_compress_alg='zlib', backup_dir=self.backup_dir) - def tearDown(self): + def helper__tearDown(self): if os.path.exists(self.test_path): shutil.rmtree(self.test_path) - def build_backup_dir(self, backup='backup'): + def helper__build_backup_dir(self, backup='backup'): return FSTestBackupDir(rel_path=self.rel_path, backup=backup) -class BasicTest(ProbackupTest): + +class TestBasic(ProbackupTest): def test_full_backup(self): # Setting up a simple test node node = self.pg_node.make_simple('node', pg_options={"fsync": "off", "synchronous_commit": "off"}) @@ -75,8 +79,4 @@ def test_full_backup(self): out = self.pb.validate('node', backup_id) # Check if the backup is valid - self.assertIn(f"INFO: Backup {backup_id} is valid", out) - - -if __name__ == "__main__": - unittest.main() + assert f"INFO: Backup {backup_id} is valid" in out From ed7b1874cf9f5f654eb859fa07cad0fa11cff178 Mon Sep 17 00:00:00 2001 From: "d.kovalenko" Date: Tue, 25 Feb 2025 14:20:45 +0300 Subject: [PATCH 31/36] Probackup plugin is updated Probackup plugin tests - They are skipped if PGPROBACKUPBIN is not defined Global variable init_params is None when PGPROBACKUPBIN is not defined or version is not processed --- .../pg_probackup2/init_helpers.py | 146 ++++++++++-------- .../pg_probackup2/tests/test_basic.py | 13 ++ 2 files changed, 91 insertions(+), 68 deletions(-) diff --git a/testgres/plugins/pg_probackup2/pg_probackup2/init_helpers.py b/testgres/plugins/pg_probackup2/pg_probackup2/init_helpers.py index 078fdbab..08f3a067 100644 --- a/testgres/plugins/pg_probackup2/pg_probackup2/init_helpers.py +++ b/testgres/plugins/pg_probackup2/pg_probackup2/init_helpers.py @@ -1,3 +1,5 @@ +from __future__ import annotations + import logging from functools import reduce import getpass @@ -37,21 +39,27 @@ class Init(object): def __init__(self): + pass + + @staticmethod + def create() -> Init: + SELF = __class__() + if '-v' in sys.argv or '--verbose' in sys.argv: - self.verbose = True + SELF.verbose = True else: - self.verbose = False - - self._pg_config = testgres.get_pg_config() - self.is_enterprise = self._pg_config.get('PGPRO_EDITION', None) == 'enterprise' - self.is_shardman = self._pg_config.get('PGPRO_EDITION', None) == 'shardman' - self.is_pgpro = 'PGPRO_EDITION' in self._pg_config - self.is_nls_enabled = 'enable-nls' in self._pg_config['CONFIGURE'] - self.is_lz4_enabled = '-llz4' in self._pg_config['LIBS'] - version = self._pg_config['VERSION'].rstrip('develalphabetapre') + SELF.verbose = False + + SELF._pg_config = testgres.get_pg_config() + SELF.is_enterprise = SELF._pg_config.get('PGPRO_EDITION', None) == 'enterprise' + SELF.is_shardman = SELF._pg_config.get('PGPRO_EDITION', None) == 'shardman' + SELF.is_pgpro = 'PGPRO_EDITION' in SELF._pg_config + SELF.is_nls_enabled = 'enable-nls' in SELF._pg_config['CONFIGURE'] + SELF.is_lz4_enabled = '-llz4' in SELF._pg_config['LIBS'] + version = SELF._pg_config['VERSION'].rstrip('develalphabetapre') parts = [*version.split(' ')[1].split('.'), '0', '0'][:3] parts[0] = re.match(r'\d+', parts[0]).group() - self.pg_config_version = reduce(lambda v, x: v * 100 + int(x), parts, 0) + SELF.pg_config_version = reduce(lambda v, x: v * 100 + int(x), parts, 0) os.environ['LANGUAGE'] = 'en' # set default locale language to en. All messages will use this locale test_env = os.environ.copy() @@ -75,31 +83,31 @@ def __init__(self): test_env['LC_MESSAGES'] = 'C' test_env['LC_TIME'] = 'C' - self._test_env = test_env + SELF._test_env = test_env # Get the directory from which the script was executed - self.source_path = os.getcwd() + SELF.source_path = os.getcwd() tmp_path = test_env.get('PGPROBACKUP_TMP_DIR') if tmp_path and os.path.isabs(tmp_path): - self.tmp_path = tmp_path + SELF.tmp_path = tmp_path else: - self.tmp_path = os.path.abspath( - os.path.join(self.source_path, tmp_path or os.path.join('tests', 'tmp_dirs')) + SELF.tmp_path = os.path.abspath( + os.path.join(SELF.source_path, tmp_path or os.path.join('tests', 'tmp_dirs')) ) - os.makedirs(self.tmp_path, exist_ok=True) + os.makedirs(SELF.tmp_path, exist_ok=True) - self.username = getpass.getuser() + SELF.username = getpass.getuser() - self.probackup_path = None + SELF.probackup_path = None if 'PGPROBACKUPBIN' in test_env: if shutil.which(test_env["PGPROBACKUPBIN"]): - self.probackup_path = test_env["PGPROBACKUPBIN"] + SELF.probackup_path = test_env["PGPROBACKUPBIN"] else: - if self.verbose: + if SELF.verbose: print('PGPROBACKUPBIN is not an executable file') - if not self.probackup_path: + if not SELF.probackup_path: probackup_path_tmp = os.path.join( testgres.get_pg_config()['BINDIR'], 'pg_probackup') @@ -108,116 +116,118 @@ def __init__(self): logging.warning('{0} is not an executable file'.format( probackup_path_tmp)) else: - self.probackup_path = probackup_path_tmp + SELF.probackup_path = probackup_path_tmp - if not self.probackup_path: - probackup_path_tmp = self.source_path + if not SELF.probackup_path: + probackup_path_tmp = SELF.source_path if os.path.isfile(probackup_path_tmp): if not os.access(probackup_path_tmp, os.X_OK): logging.warning('{0} is not an executable file'.format( probackup_path_tmp)) else: - self.probackup_path = probackup_path_tmp + SELF.probackup_path = probackup_path_tmp - if not self.probackup_path: + if not SELF.probackup_path: logging.error('pg_probackup binary is not found') - exit(1) + return None if os.name == 'posix': - self.EXTERNAL_DIRECTORY_DELIMITER = ':' + SELF.EXTERNAL_DIRECTORY_DELIMITER = ':' os.environ['PATH'] = os.path.dirname( - self.probackup_path) + ':' + os.environ['PATH'] + SELF.probackup_path) + ':' + os.environ['PATH'] elif os.name == 'nt': - self.EXTERNAL_DIRECTORY_DELIMITER = ';' + SELF.EXTERNAL_DIRECTORY_DELIMITER = ';' os.environ['PATH'] = os.path.dirname( - self.probackup_path) + ';' + os.environ['PATH'] + SELF.probackup_path) + ';' + os.environ['PATH'] - self.probackup_old_path = None + SELF.probackup_old_path = None if 'PGPROBACKUPBIN_OLD' in test_env: if (os.path.isfile(test_env['PGPROBACKUPBIN_OLD']) and os.access(test_env['PGPROBACKUPBIN_OLD'], os.X_OK)): - self.probackup_old_path = test_env['PGPROBACKUPBIN_OLD'] + SELF.probackup_old_path = test_env['PGPROBACKUPBIN_OLD'] else: - if self.verbose: + if SELF.verbose: print('PGPROBACKUPBIN_OLD is not an executable file') - self.probackup_version = None - self.old_probackup_version = None + SELF.probackup_version = None + SELF.old_probackup_version = None probackup_version_output = subprocess.check_output( - [self.probackup_path, "--version"], + [SELF.probackup_path, "--version"], stderr=subprocess.STDOUT, ).decode('utf-8') match = re.search(r"\d+\.\d+\.\d+", probackup_version_output) - self.probackup_version = match.group(0) if match else None + SELF.probackup_version = match.group(0) if match else None match = re.search(r"\(compressions: ([^)]*)\)", probackup_version_output) compressions = match.group(1) if match else None if compressions: - self.probackup_compressions = {s.strip() for s in compressions.split(',')} + SELF.probackup_compressions = {s.strip() for s in compressions.split(',')} else: - self.probackup_compressions = [] + SELF.probackup_compressions = [] - if self.probackup_old_path: + if SELF.probackup_old_path: old_probackup_version_output = subprocess.check_output( - [self.probackup_old_path, "--version"], + [SELF.probackup_old_path, "--version"], stderr=subprocess.STDOUT, ).decode('utf-8') match = re.search(r"\d+\.\d+\.\d+", old_probackup_version_output) - self.old_probackup_version = match.group(0) if match else None + SELF.old_probackup_version = match.group(0) if match else None - self.remote = test_env.get('PGPROBACKUP_SSH_REMOTE', None) == 'ON' - self.ptrack = test_env.get('PG_PROBACKUP_PTRACK', None) == 'ON' and self.pg_config_version >= 110000 - self.wal_tree_enabled = test_env.get('PG_PROBACKUP_WAL_TREE_ENABLED', None) == 'ON' + SELF.remote = test_env.get('PGPROBACKUP_SSH_REMOTE', None) == 'ON' + SELF.ptrack = test_env.get('PG_PROBACKUP_PTRACK', None) == 'ON' and SELF.pg_config_version >= 110000 + SELF.wal_tree_enabled = test_env.get('PG_PROBACKUP_WAL_TREE_ENABLED', None) == 'ON' - self.bckp_source = test_env.get('PG_PROBACKUP_SOURCE', 'pro').lower() - if self.bckp_source not in ('base', 'direct', 'pro'): + SELF.bckp_source = test_env.get('PG_PROBACKUP_SOURCE', 'pro').lower() + if SELF.bckp_source not in ('base', 'direct', 'pro'): raise Exception("Wrong PG_PROBACKUP_SOURCE value. Available options: base|direct|pro") - self.paranoia = test_env.get('PG_PROBACKUP_PARANOIA', None) == 'ON' + SELF.paranoia = test_env.get('PG_PROBACKUP_PARANOIA', None) == 'ON' env_compress = test_env.get('ARCHIVE_COMPRESSION', None) if env_compress: env_compress = env_compress.lower() if env_compress in ('on', 'zlib'): - self.compress_suffix = '.gz' - self.archive_compress = 'zlib' + SELF.compress_suffix = '.gz' + SELF.archive_compress = 'zlib' elif env_compress == 'lz4': if not HAVE_LZ4: raise LZ4_error - if 'lz4' not in self.probackup_compressions: + if 'lz4' not in SELF.probackup_compressions: raise Exception("pg_probackup is not compiled with lz4 support") - self.compress_suffix = '.lz4' - self.archive_compress = 'lz4' + SELF.compress_suffix = '.lz4' + SELF.archive_compress = 'lz4' elif env_compress == 'zstd': if not HAVE_ZSTD: raise ZSTD_error - if 'zstd' not in self.probackup_compressions: + if 'zstd' not in SELF.probackup_compressions: raise Exception("pg_probackup is not compiled with zstd support") - self.compress_suffix = '.zst' - self.archive_compress = 'zstd' + SELF.compress_suffix = '.zst' + SELF.archive_compress = 'zstd' else: - self.compress_suffix = '' - self.archive_compress = False + SELF.compress_suffix = '' + SELF.archive_compress = False cfs_compress = test_env.get('PG_PROBACKUP_CFS_COMPRESS', None) if cfs_compress: - self.cfs_compress = cfs_compress.lower() + SELF.cfs_compress = cfs_compress.lower() else: - self.cfs_compress = self.archive_compress + SELF.cfs_compress = SELF.archive_compress os.environ["PGAPPNAME"] = "pg_probackup" - self.delete_logs = delete_logs + SELF.delete_logs = delete_logs - if self.probackup_version.split('.')[0].isdigit(): - self.major_version = int(self.probackup_version.split('.')[0]) + if SELF.probackup_version.split('.')[0].isdigit(): + SELF.major_version = int(SELF.probackup_version.split('.')[0]) else: - logging.error('Can\'t process pg_probackup version \"{}\": the major version is expected to be a number'.format(self.probackup_version)) - sys.exit(1) + logging.error('Can\'t process pg_probackup version \"{}\": the major version is expected to be a number'.format(SELF.probackup_version)) + return None + + return SELF def test_env(self): return self._test_env.copy() -init_params = Init() +init_params = Init.create() diff --git a/testgres/plugins/pg_probackup2/pg_probackup2/tests/test_basic.py b/testgres/plugins/pg_probackup2/pg_probackup2/tests/test_basic.py index 32b29498..ba788623 100644 --- a/testgres/plugins/pg_probackup2/pg_probackup2/tests/test_basic.py +++ b/testgres/plugins/pg_probackup2/pg_probackup2/tests/test_basic.py @@ -13,6 +13,18 @@ class ProbackupTest: pg_node: testgres.PostgresNode + @staticmethod + def probackup_is_available() -> bool: + p = os.environ.get("PGPROBACKUPBIN") + + if p is None: + return False + + if not os.path.exists(p): + return False + + return True + @pytest.fixture(autouse=True, scope="function") def implicit_fixture(self, request: pytest.FixtureRequest): assert isinstance(request, pytest.FixtureRequest) @@ -60,6 +72,7 @@ def helper__build_backup_dir(self, backup='backup'): return FSTestBackupDir(rel_path=self.rel_path, backup=backup) +@pytest.mark.skipif(not ProbackupTest.probackup_is_available(), reason="Check that PGPROBACKUPBIN is defined and is valid.") class TestBasic(ProbackupTest): def test_full_backup(self): # Setting up a simple test node From 8af4a49d3eb2eb94e776db4271619263dce5e87b Mon Sep 17 00:00:00 2001 From: "d.kovalenko" Date: Tue, 25 Feb 2025 14:24:34 +0300 Subject: [PATCH 32/36] CI test use 4 cores --- run_tests.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/run_tests.sh b/run_tests.sh index 6995b7e4..e9d58b54 100755 --- a/run_tests.sh +++ b/run_tests.sh @@ -38,19 +38,19 @@ rm -f $COVERAGE_FILE # run tests (PATH) -time coverage run -a -m pytest -l -v -n 2 -k "TestgresTests" +time coverage run -a -m pytest -l -v -n 4 -k "TestgresTests" # run tests (PG_BIN) time \ PG_BIN=$(dirname $(which pg_config)) \ - coverage run -a -m pytest -l -v -n 2 -k "TestgresTests" + coverage run -a -m pytest -l -v -n 4 -k "TestgresTests" # run tests (PG_CONFIG) time \ PG_CONFIG=$(which pg_config) \ - coverage run -a -m pytest -l -v -n 2 -k "TestgresTests" + coverage run -a -m pytest -l -v -n 4 -k "TestgresTests" # show coverage From d35ca43acdc07bad1540002a50b263283947af86 Mon Sep 17 00:00:00 2001 From: "d.kovalenko" Date: Tue, 25 Feb 2025 15:09:54 +0300 Subject: [PATCH 33/36] testgres.plugins.probackup2.Init was restored [thanks to Yuri Sokolov] --- .../pg_probackup2/init_helpers.py | 151 +++++++++--------- 1 file changed, 72 insertions(+), 79 deletions(-) diff --git a/testgres/plugins/pg_probackup2/pg_probackup2/init_helpers.py b/testgres/plugins/pg_probackup2/pg_probackup2/init_helpers.py index 08f3a067..c4570a39 100644 --- a/testgres/plugins/pg_probackup2/pg_probackup2/init_helpers.py +++ b/testgres/plugins/pg_probackup2/pg_probackup2/init_helpers.py @@ -1,5 +1,3 @@ -from __future__ import annotations - import logging from functools import reduce import getpass @@ -39,27 +37,21 @@ class Init(object): def __init__(self): - pass - - @staticmethod - def create() -> Init: - SELF = __class__() - if '-v' in sys.argv or '--verbose' in sys.argv: - SELF.verbose = True + self.verbose = True else: - SELF.verbose = False - - SELF._pg_config = testgres.get_pg_config() - SELF.is_enterprise = SELF._pg_config.get('PGPRO_EDITION', None) == 'enterprise' - SELF.is_shardman = SELF._pg_config.get('PGPRO_EDITION', None) == 'shardman' - SELF.is_pgpro = 'PGPRO_EDITION' in SELF._pg_config - SELF.is_nls_enabled = 'enable-nls' in SELF._pg_config['CONFIGURE'] - SELF.is_lz4_enabled = '-llz4' in SELF._pg_config['LIBS'] - version = SELF._pg_config['VERSION'].rstrip('develalphabetapre') + self.verbose = False + + self._pg_config = testgres.get_pg_config() + self.is_enterprise = self._pg_config.get('PGPRO_EDITION', None) == 'enterprise' + self.is_shardman = self._pg_config.get('PGPRO_EDITION', None) == 'shardman' + self.is_pgpro = 'PGPRO_EDITION' in self._pg_config + self.is_nls_enabled = 'enable-nls' in self._pg_config['CONFIGURE'] + self.is_lz4_enabled = '-llz4' in self._pg_config['LIBS'] + version = self._pg_config['VERSION'].rstrip('develalphabetapre') parts = [*version.split(' ')[1].split('.'), '0', '0'][:3] parts[0] = re.match(r'\d+', parts[0]).group() - SELF.pg_config_version = reduce(lambda v, x: v * 100 + int(x), parts, 0) + self.pg_config_version = reduce(lambda v, x: v * 100 + int(x), parts, 0) os.environ['LANGUAGE'] = 'en' # set default locale language to en. All messages will use this locale test_env = os.environ.copy() @@ -83,31 +75,31 @@ def create() -> Init: test_env['LC_MESSAGES'] = 'C' test_env['LC_TIME'] = 'C' - SELF._test_env = test_env + self._test_env = test_env # Get the directory from which the script was executed - SELF.source_path = os.getcwd() + self.source_path = os.getcwd() tmp_path = test_env.get('PGPROBACKUP_TMP_DIR') if tmp_path and os.path.isabs(tmp_path): - SELF.tmp_path = tmp_path + self.tmp_path = tmp_path else: - SELF.tmp_path = os.path.abspath( - os.path.join(SELF.source_path, tmp_path or os.path.join('tests', 'tmp_dirs')) + self.tmp_path = os.path.abspath( + os.path.join(self.source_path, tmp_path or os.path.join('tests', 'tmp_dirs')) ) - os.makedirs(SELF.tmp_path, exist_ok=True) + os.makedirs(self.tmp_path, exist_ok=True) - SELF.username = getpass.getuser() + self.username = getpass.getuser() - SELF.probackup_path = None + self.probackup_path = None if 'PGPROBACKUPBIN' in test_env: if shutil.which(test_env["PGPROBACKUPBIN"]): - SELF.probackup_path = test_env["PGPROBACKUPBIN"] + self.probackup_path = test_env["PGPROBACKUPBIN"] else: - if SELF.verbose: + if self.verbose: print('PGPROBACKUPBIN is not an executable file') - if not SELF.probackup_path: + if not self.probackup_path: probackup_path_tmp = os.path.join( testgres.get_pg_config()['BINDIR'], 'pg_probackup') @@ -116,118 +108,119 @@ def create() -> Init: logging.warning('{0} is not an executable file'.format( probackup_path_tmp)) else: - SELF.probackup_path = probackup_path_tmp + self.probackup_path = probackup_path_tmp - if not SELF.probackup_path: - probackup_path_tmp = SELF.source_path + if not self.probackup_path: + probackup_path_tmp = self.source_path if os.path.isfile(probackup_path_tmp): if not os.access(probackup_path_tmp, os.X_OK): logging.warning('{0} is not an executable file'.format( probackup_path_tmp)) else: - SELF.probackup_path = probackup_path_tmp + self.probackup_path = probackup_path_tmp - if not SELF.probackup_path: - logging.error('pg_probackup binary is not found') - return None + if not self.probackup_path: + raise Exception('pg_probackup binary is not found') if os.name == 'posix': - SELF.EXTERNAL_DIRECTORY_DELIMITER = ':' + self.EXTERNAL_DIRECTORY_DELIMITER = ':' os.environ['PATH'] = os.path.dirname( - SELF.probackup_path) + ':' + os.environ['PATH'] + self.probackup_path) + ':' + os.environ['PATH'] elif os.name == 'nt': - SELF.EXTERNAL_DIRECTORY_DELIMITER = ';' + self.EXTERNAL_DIRECTORY_DELIMITER = ';' os.environ['PATH'] = os.path.dirname( - SELF.probackup_path) + ';' + os.environ['PATH'] + self.probackup_path) + ';' + os.environ['PATH'] - SELF.probackup_old_path = None + self.probackup_old_path = None if 'PGPROBACKUPBIN_OLD' in test_env: if (os.path.isfile(test_env['PGPROBACKUPBIN_OLD']) and os.access(test_env['PGPROBACKUPBIN_OLD'], os.X_OK)): - SELF.probackup_old_path = test_env['PGPROBACKUPBIN_OLD'] + self.probackup_old_path = test_env['PGPROBACKUPBIN_OLD'] else: - if SELF.verbose: + if self.verbose: print('PGPROBACKUPBIN_OLD is not an executable file') - SELF.probackup_version = None - SELF.old_probackup_version = None + self.probackup_version = None + self.old_probackup_version = None probackup_version_output = subprocess.check_output( - [SELF.probackup_path, "--version"], + [self.probackup_path, "--version"], stderr=subprocess.STDOUT, ).decode('utf-8') match = re.search(r"\d+\.\d+\.\d+", probackup_version_output) - SELF.probackup_version = match.group(0) if match else None + self.probackup_version = match.group(0) if match else None match = re.search(r"\(compressions: ([^)]*)\)", probackup_version_output) compressions = match.group(1) if match else None if compressions: - SELF.probackup_compressions = {s.strip() for s in compressions.split(',')} + self.probackup_compressions = {s.strip() for s in compressions.split(',')} else: - SELF.probackup_compressions = [] + self.probackup_compressions = [] - if SELF.probackup_old_path: + if self.probackup_old_path: old_probackup_version_output = subprocess.check_output( - [SELF.probackup_old_path, "--version"], + [self.probackup_old_path, "--version"], stderr=subprocess.STDOUT, ).decode('utf-8') match = re.search(r"\d+\.\d+\.\d+", old_probackup_version_output) - SELF.old_probackup_version = match.group(0) if match else None + self.old_probackup_version = match.group(0) if match else None - SELF.remote = test_env.get('PGPROBACKUP_SSH_REMOTE', None) == 'ON' - SELF.ptrack = test_env.get('PG_PROBACKUP_PTRACK', None) == 'ON' and SELF.pg_config_version >= 110000 - SELF.wal_tree_enabled = test_env.get('PG_PROBACKUP_WAL_TREE_ENABLED', None) == 'ON' + self.remote = test_env.get('PGPROBACKUP_SSH_REMOTE', None) == 'ON' + self.ptrack = test_env.get('PG_PROBACKUP_PTRACK', None) == 'ON' and self.pg_config_version >= 110000 + self.wal_tree_enabled = test_env.get('PG_PROBACKUP_WAL_TREE_ENABLED', None) == 'ON' - SELF.bckp_source = test_env.get('PG_PROBACKUP_SOURCE', 'pro').lower() - if SELF.bckp_source not in ('base', 'direct', 'pro'): + self.bckp_source = test_env.get('PG_PROBACKUP_SOURCE', 'pro').lower() + if self.bckp_source not in ('base', 'direct', 'pro'): raise Exception("Wrong PG_PROBACKUP_SOURCE value. Available options: base|direct|pro") - SELF.paranoia = test_env.get('PG_PROBACKUP_PARANOIA', None) == 'ON' + self.paranoia = test_env.get('PG_PROBACKUP_PARANOIA', None) == 'ON' env_compress = test_env.get('ARCHIVE_COMPRESSION', None) if env_compress: env_compress = env_compress.lower() if env_compress in ('on', 'zlib'): - SELF.compress_suffix = '.gz' - SELF.archive_compress = 'zlib' + self.compress_suffix = '.gz' + self.archive_compress = 'zlib' elif env_compress == 'lz4': if not HAVE_LZ4: raise LZ4_error - if 'lz4' not in SELF.probackup_compressions: + if 'lz4' not in self.probackup_compressions: raise Exception("pg_probackup is not compiled with lz4 support") - SELF.compress_suffix = '.lz4' - SELF.archive_compress = 'lz4' + self.compress_suffix = '.lz4' + self.archive_compress = 'lz4' elif env_compress == 'zstd': if not HAVE_ZSTD: raise ZSTD_error - if 'zstd' not in SELF.probackup_compressions: + if 'zstd' not in self.probackup_compressions: raise Exception("pg_probackup is not compiled with zstd support") - SELF.compress_suffix = '.zst' - SELF.archive_compress = 'zstd' + self.compress_suffix = '.zst' + self.archive_compress = 'zstd' else: - SELF.compress_suffix = '' - SELF.archive_compress = False + self.compress_suffix = '' + self.archive_compress = False cfs_compress = test_env.get('PG_PROBACKUP_CFS_COMPRESS', None) if cfs_compress: - SELF.cfs_compress = cfs_compress.lower() + self.cfs_compress = cfs_compress.lower() else: - SELF.cfs_compress = SELF.archive_compress + self.cfs_compress = self.archive_compress os.environ["PGAPPNAME"] = "pg_probackup" - SELF.delete_logs = delete_logs + self.delete_logs = delete_logs - if SELF.probackup_version.split('.')[0].isdigit(): - SELF.major_version = int(SELF.probackup_version.split('.')[0]) + if self.probackup_version.split('.')[0].isdigit(): + self.major_version = int(self.probackup_version.split('.')[0]) else: - logging.error('Can\'t process pg_probackup version \"{}\": the major version is expected to be a number'.format(SELF.probackup_version)) - return None - - return SELF + raise Exception('Can\'t process pg_probackup version \"{}\": the major version is expected to be a number'.format(self.probackup_version)) def test_env(self): return self._test_env.copy() -init_params = Init.create() +try: + init_params = Init() +except Exception as e: + logging.error(str(e)) + logging.warning("testgres.plugins.probackup2.init_params is set to None.") + init_params = None From 181126bbbddb108f0c17a201f9c78e070c26f8f9 Mon Sep 17 00:00:00 2001 From: "d.kovalenko" Date: Tue, 25 Feb 2025 16:09:36 +0300 Subject: [PATCH 34/36] pytest.ini is updated [testpaths] Enumeration of all the known folders with tests. --- pytest.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pytest.ini b/pytest.ini index 61d4f9bc..c94eabc2 100644 --- a/pytest.ini +++ b/pytest.ini @@ -1,5 +1,5 @@ [pytest] -testpaths = ./tests +testpaths = ["./tests", "./testgres/plugins/pg_probackup2/pg_probackup2/tests"] addopts = --strict-markers markers = #log_file = logs/pytest.log From d08f659dffd4af3dc412868f50fe6b75b1dd1ff9 Mon Sep 17 00:00:00 2001 From: "d.kovalenko" Date: Tue, 25 Feb 2025 17:17:42 +0300 Subject: [PATCH 35/36] test_child_pids (local, remote) is updated Multiple attempts and logging are added. --- tests/test_simple.py | 70 +++++++++++++++++++++++++++++++++---- tests/test_simple_remote.py | 70 +++++++++++++++++++++++++++++++++---- 2 files changed, 128 insertions(+), 12 deletions(-) diff --git a/tests/test_simple.py b/tests/test_simple.py index 4cb63e4f..c470b33d 100644 --- a/tests/test_simple.py +++ b/tests/test_simple.py @@ -1000,6 +1000,62 @@ def test_child_pids(self): ProcessType.WalReceiver, ] + def LOCAL__test_auxiliary_pids( + node: testgres.PostgresNode, + expectedTypes: list[ProcessType] + ) -> list[ProcessType]: + # returns list of the absence processes + assert node is not None + assert type(node) == testgres.PostgresNode # noqa: E721 + assert expectedTypes is not None + assert type(expectedTypes) == list # noqa: E721 + + pids = node.auxiliary_pids + assert pids is not None # noqa: E721 + assert type(pids) == dict # noqa: E721 + + result = list[ProcessType]() + for ptype in expectedTypes: + if not (ptype in pids): + result.append(ptype) + return result + + def LOCAL__check_auxiliary_pids__multiple_attempts( + node: testgres.PostgresNode, + expectedTypes: list[ProcessType]): + assert node is not None + assert type(node) == testgres.PostgresNode # noqa: E721 + assert expectedTypes is not None + assert type(expectedTypes) == list # noqa: E721 + + nAttempt = 0 + + while nAttempt < 5: + nAttempt += 1 + + logging.info("Test pids of [{0}] node. Attempt #{1}.".format( + node.name, + nAttempt + )) + + if nAttempt > 1: + time.sleep(1) + + absenceList = LOCAL__test_auxiliary_pids(node, expectedTypes) + assert absenceList is not None + assert type(absenceList) == list # noqa: E721 + if len(absenceList) == 0: + logging.info("Bingo!") + return + + logging.info("These processes are not found: {0}.".format(absenceList)) + continue + + raise Exception("Node {0} does not have the following processes: {1}.".format( + node.name, + absenceList + )) + with get_new_node().init().start() as master: # master node doesn't have a source walsender! @@ -1014,13 +1070,15 @@ def test_child_pids(self): # test __str__ method str(master.child_processes[0]) - master_pids = master.auxiliary_pids - for ptype in master_processes: - assert (ptype in master_pids) + LOCAL__check_auxiliary_pids__multiple_attempts( + master, + master_processes) - replica_pids = replica.auxiliary_pids - for ptype in repl_processes: - assert (ptype in replica_pids) + LOCAL__check_auxiliary_pids__multiple_attempts( + replica, + repl_processes) + + master_pids = master.auxiliary_pids # there should be exactly 1 source walsender for replica assert (len(master_pids[ProcessType.WalSender]) == 1) diff --git a/tests/test_simple_remote.py b/tests/test_simple_remote.py index 27852428..67c98bf8 100755 --- a/tests/test_simple_remote.py +++ b/tests/test_simple_remote.py @@ -1038,6 +1038,62 @@ def test_child_pids(self): ProcessType.WalReceiver, ] + def LOCAL__test_auxiliary_pids( + node: testgres.PostgresNode, + expectedTypes: list[ProcessType] + ) -> list[ProcessType]: + # returns list of the absence processes + assert node is not None + assert type(node) == testgres.PostgresNode # noqa: E721 + assert expectedTypes is not None + assert type(expectedTypes) == list # noqa: E721 + + pids = node.auxiliary_pids + assert pids is not None # noqa: E721 + assert type(pids) == dict # noqa: E721 + + result = list[ProcessType]() + for ptype in expectedTypes: + if not (ptype in pids): + result.append(ptype) + return result + + def LOCAL__check_auxiliary_pids__multiple_attempts( + node: testgres.PostgresNode, + expectedTypes: list[ProcessType]): + assert node is not None + assert type(node) == testgres.PostgresNode # noqa: E721 + assert expectedTypes is not None + assert type(expectedTypes) == list # noqa: E721 + + nAttempt = 0 + + while nAttempt < 5: + nAttempt += 1 + + logging.info("Test pids of [{0}] node. Attempt #{1}.".format( + node.name, + nAttempt + )) + + if nAttempt > 1: + time.sleep(1) + + absenceList = LOCAL__test_auxiliary_pids(node, expectedTypes) + assert absenceList is not None + assert type(absenceList) == list # noqa: E721 + if len(absenceList) == 0: + logging.info("Bingo!") + return + + logging.info("These processes are not found: {0}.".format(absenceList)) + continue + + raise Exception("Node {0} does not have the following processes: {1}.".format( + node.name, + absenceList + )) + with __class__.helper__get_node().init().start() as master: # master node doesn't have a source walsender! @@ -1052,13 +1108,15 @@ def test_child_pids(self): # test __str__ method str(master.child_processes[0]) - master_pids = master.auxiliary_pids - for ptype in master_processes: - assert (ptype in master_pids) + LOCAL__check_auxiliary_pids__multiple_attempts( + master, + master_processes) - replica_pids = replica.auxiliary_pids - for ptype in repl_processes: - assert (ptype in replica_pids) + LOCAL__check_auxiliary_pids__multiple_attempts( + replica, + repl_processes) + + master_pids = master.auxiliary_pids # there should be exactly 1 source walsender for replica assert (len(master_pids[ProcessType.WalSender]) == 1) From e80314648d001356d40520bf3bd3345a5c9868d4 Mon Sep 17 00:00:00 2001 From: "d.kovalenko" Date: Tue, 25 Feb 2025 18:02:49 +0300 Subject: [PATCH 36/36] test_child_process_dies is updated Multiple attempts are added. --- tests/test_simple.py | 35 +++++++++++++++++++++++++--------- tests/test_simple_remote.py | 38 +++++++++++++++++++++++++++---------- 2 files changed, 54 insertions(+), 19 deletions(-) diff --git a/tests/test_simple.py b/tests/test_simple.py index c470b33d..6c433cd4 100644 --- a/tests/test_simple.py +++ b/tests/test_simple.py @@ -1096,15 +1096,32 @@ def test_child_process_dies(self): # test for FileNotFound exception during child_processes() function cmd = ["timeout", "60"] if os.name == 'nt' else ["sleep", "60"] - with subprocess.Popen(cmd, shell=True) as process: # shell=True might be needed on Windows - assert (process.poll() is None) - # collect list of processes currently running - children = psutil.Process(os.getpid()).children() - # kill a process, so received children dictionary becomes invalid - process.kill() - process.wait() - # try to handle children list -- missing processes will have ptype "ProcessType.Unknown" - [ProcessProxy(p) for p in children] + nAttempt = 0 + + while True: + if nAttempt == 5: + raise Exception("Max attempt number is exceed.") + + nAttempt += 1 + + logging.info("Attempt #{0}".format(nAttempt)) + + with subprocess.Popen(cmd, shell=True) as process: # shell=True might be needed on Windows + r = process.poll() + + if r is not None: + logging.warning("process.pool() returns an unexpected result: {0}.".format(r)) + continue + + assert r is None + # collect list of processes currently running + children = psutil.Process(os.getpid()).children() + # kill a process, so received children dictionary becomes invalid + process.kill() + process.wait() + # try to handle children list -- missing processes will have ptype "ProcessType.Unknown" + [ProcessProxy(p) for p in children] + break def test_upgrade_node(self): old_bin_dir = os.path.dirname(get_bin_path("pg_config")) diff --git a/tests/test_simple_remote.py b/tests/test_simple_remote.py index 67c98bf8..e7cc5e5c 100755 --- a/tests/test_simple_remote.py +++ b/tests/test_simple_remote.py @@ -1130,17 +1130,35 @@ def LOCAL__check_auxiliary_pids__multiple_attempts( with pytest.raises(expected_exception=TestgresException): replica.source_walsender + # TODO: Why does not this test work with remote host? def test_child_process_dies(self): - # test for FileNotFound exception during child_processes() function - with subprocess.Popen(["sleep", "60"]) as process: - assert (process.poll() is None) - # collect list of processes currently running - children = psutil.Process(os.getpid()).children() - # kill a process, so received children dictionary becomes invalid - process.kill() - process.wait() - # try to handle children list -- missing processes will have ptype "ProcessType.Unknown" - [ProcessProxy(p) for p in children] + nAttempt = 0 + + while True: + if nAttempt == 5: + raise Exception("Max attempt number is exceed.") + + nAttempt += 1 + + logging.info("Attempt #{0}".format(nAttempt)) + + # test for FileNotFound exception during child_processes() function + with subprocess.Popen(["sleep", "60"]) as process: + r = process.poll() + + if r is not None: + logging.warning("process.pool() returns an unexpected result: {0}.".format(r)) + continue + + assert r is None + # collect list of processes currently running + children = psutil.Process(os.getpid()).children() + # kill a process, so received children dictionary becomes invalid + process.kill() + process.wait() + # try to handle children list -- missing processes will have ptype "ProcessType.Unknown" + [ProcessProxy(p) for p in children] + break @staticmethod def helper__get_node(): pFad - Phonifier reborn

Pfad - The Proxy pFad of © 2024 Garber Painting. All rights reserved.

Note: This service is not intended for secure transactions such as banking, social media, email, or purchasing. Use at your own risk. We assume no liability whatsoever for broken pages.


Alternative Proxies:

Alternative Proxy

pFad Proxy

pFad v3 Proxy

pFad v4 Proxy