Skip to content

Commit 4227a98

Browse files
zilderzilder
committed
travis-ci python tests
1 parent 4a97264 commit 4227a98

File tree

2 files changed

+176
-137
lines changed

2 files changed

+176
-137
lines changed

‎tests/partitioning_test.py

Lines changed: 154 additions & 135 deletions
Original file line numberDiff line numberDiff line change
@@ -40,101 +40,115 @@ def catchup_replica(self, master, replica):
4040
'FROM pg_stat_replication WHERE application_name = \'%s\''
4141
% replica.name)
4242

43+
def printlog(self, logfile):
44+
with open(logfile, 'r') as log:
45+
for line in log.readlines():
46+
print line
47+
4348
def test_concurrent(self):
4449
"""Tests concurrent partitioning"""
4550
node = get_new_node('test')
46-
node.init()
47-
node.append_conf('postgresql.conf', 'shared_preload_libraries=\'pg_pathman\'\n')
48-
node.start()
49-
self.init_test_data(node)
50-
51-
node.psql('postgres', 'select partition_table_concurrently(\'abc\')')
52-
53-
while True:
54-
# update some rows to check for deadlocks
55-
# import ipdb; ipdb.set_trace()
56-
node.safe_psql('postgres',
57-
'''
58-
update abc set t = 'test'
59-
where id in (select (random() * 300000)::int from generate_series(1, 3000))
60-
''')
61-
62-
count = node.execute('postgres', 'select count(*) from pathman_concurrent_part_tasks')
63-
64-
# if there is no active workers then it means work is done
65-
if count[0][0] == 0:
66-
break
67-
time.sleep(1)
68-
69-
data = node.execute('postgres', 'select count(*) from only abc')
70-
self.assertEqual(data[0][0], 0)
71-
data = node.execute('postgres', 'select count(*) from abc')
72-
self.assertEqual(data[0][0], 300000)
73-
74-
node.stop()
51+
try:
52+
node.init()
53+
node.append_conf('postgresql.conf', 'shared_preload_libraries=\'pg_pathman\'\n')
54+
node.start()
55+
self.init_test_data(node)
56+
57+
node.psql('postgres', 'select partition_table_concurrently(\'abc\')')
58+
59+
while True:
60+
# update some rows to check for deadlocks
61+
# import ipdb; ipdb.set_trace()
62+
node.safe_psql('postgres',
63+
'''
64+
update abc set t = 'test'
65+
where id in (select (random() * 300000)::int from generate_series(1, 3000))
66+
''')
67+
68+
count = node.execute('postgres', 'select count(*) from pathman_concurrent_part_tasks')
69+
70+
# if there is no active workers then it means work is done
71+
if count[0][0] == 0:
72+
break
73+
time.sleep(1)
74+
75+
data = node.execute('postgres', 'select count(*) from only abc')
76+
self.assertEqual(data[0][0], 0)
77+
data = node.execute('postgres', 'select count(*) from abc')
78+
self.assertEqual(data[0][0], 300000)
79+
80+
node.stop()
81+
except Exception, e:
82+
self.printlog(node.logs_dir + '/postgresql.log')
83+
raise e
7584

7685
def test_replication(self):
7786
"""Tests how pg_pathman works with replication"""
7887
node = get_new_node('master')
7988
replica = get_new_node('repl')
8089

81-
# initialize master server
82-
node.init(allows_streaming=True)
83-
node.append_conf('postgresql.conf', 'shared_preload_libraries=\'pg_pathman\'\n')
84-
node.start()
85-
node.backup('my_backup')
86-
87-
# initialize replica from backup
88-
replica.init_from_backup(node, 'my_backup', has_streaming=True)
89-
replica.start()
90-
91-
# initialize pg_pathman extension and some test data
92-
self.init_test_data(node)
93-
94-
# wait until replica catches up
95-
self.catchup_replica(node, replica)
96-
97-
# check that results are equal
98-
self.assertEqual(
99-
node.psql('postgres', 'explain (costs off) select * from abc'),
100-
replica.psql('postgres', 'explain (costs off) select * from abc')
101-
)
102-
103-
# enable parent and see if it is enabled in replica
104-
node.psql('postgres', 'select enable_parent(\'abc\'')
105-
106-
self.catchup_replica(node, replica)
107-
self.assertEqual(
108-
node.psql('postgres', 'explain (costs off) select * from abc'),
109-
replica.psql('postgres', 'explain (costs off) select * from abc')
110-
)
111-
self.assertEqual(
112-
node.psql('postgres', 'select * from abc'),
113-
replica.psql('postgres', 'select * from abc')
114-
)
115-
self.assertEqual(
116-
node.execute('postgres', 'select count(*) from abc')[0][0],
117-
300000
118-
)
119-
120-
# check that direct UPDATE in pathman_config_params invalidates
121-
# cache
122-
node.psql(
123-
'postgres',
124-
'update pathman_config_params set enable_parent = false')
125-
self.catchup_replica(node, replica)
126-
self.assertEqual(
127-
node.psql('postgres', 'explain (costs off) select * from abc'),
128-
replica.psql('postgres', 'explain (costs off) select * from abc')
129-
)
130-
self.assertEqual(
131-
node.psql('postgres', 'select * from abc'),
132-
replica.psql('postgres', 'select * from abc')
133-
)
134-
self.assertEqual(
135-
node.execute('postgres', 'select count(*) from abc')[0][0],
136-
0
137-
)
90+
try:
91+
# initialize master server
92+
node.init(allows_streaming=True)
93+
node.append_conf('postgresql.conf', 'shared_preload_libraries=\'pg_pathman\'\n')
94+
node.start()
95+
node.backup('my_backup')
96+
97+
# initialize replica from backup
98+
replica.init_from_backup(node, 'my_backup', has_streaming=True)
99+
replica.start()
100+
101+
# initialize pg_pathman extension and some test data
102+
self.init_test_data(node)
103+
104+
# wait until replica catches up
105+
self.catchup_replica(node, replica)
106+
107+
# check that results are equal
108+
self.assertEqual(
109+
node.psql('postgres', 'explain (costs off) select * from abc'),
110+
replica.psql('postgres', 'explain (costs off) select * from abc')
111+
)
112+
113+
# enable parent and see if it is enabled in replica
114+
node.psql('postgres', 'select enable_parent(\'abc\'')
115+
116+
self.catchup_replica(node, replica)
117+
self.assertEqual(
118+
node.psql('postgres', 'explain (costs off) select * from abc'),
119+
replica.psql('postgres', 'explain (costs off) select * from abc')
120+
)
121+
self.assertEqual(
122+
node.psql('postgres', 'select * from abc'),
123+
replica.psql('postgres', 'select * from abc')
124+
)
125+
self.assertEqual(
126+
node.execute('postgres', 'select count(*) from abc')[0][0],
127+
300000
128+
)
129+
130+
# check that direct UPDATE in pathman_config_params invalidates
131+
# cache
132+
node.psql(
133+
'postgres',
134+
'update pathman_config_params set enable_parent = false')
135+
self.catchup_replica(node, replica)
136+
self.assertEqual(
137+
node.psql('postgres', 'explain (costs off) select * from abc'),
138+
replica.psql('postgres', 'explain (costs off) select * from abc')
139+
)
140+
self.assertEqual(
141+
node.psql('postgres', 'select * from abc'),
142+
replica.psql('postgres', 'select * from abc')
143+
)
144+
self.assertEqual(
145+
node.execute('postgres', 'select count(*) from abc')[0][0],
146+
0
147+
)
148+
except Exception, e:
149+
self.printlog(node.logs_dir + '/postgresql.log')
150+
self.printlog(replica.logs_dir + '/postgresql.log')
151+
raise e
138152

139153
def test_locks(self):
140154
"""Test that a session trying to create new partitions waits for other
@@ -170,61 +184,66 @@ def add_partition(node, flag, query):
170184

171185
# Initialize master server
172186
node = get_new_node('master')
173-
node.init()
174-
node.append_conf('postgresql.conf', 'shared_preload_libraries=\'pg_pathman\'\n')
175-
node.start()
176-
node.safe_psql(
177-
'postgres',
178-
'create extension pg_pathman; '
179-
+ 'create table abc(id serial, t text); '
180-
+ 'insert into abc select generate_series(1, 100000); '
181-
+ 'select create_range_partitions(\'abc\', \'id\', 1, 50000);'
182-
)
183-
184-
# Start transaction that will create partition
185-
con = node.connect()
186-
con.begin()
187-
con.execute('select append_range_partition(\'abc\')')
188-
189-
# Start threads that suppose to add new partitions and wait some time
190-
query = [
191-
'select prepend_range_partition(\'abc\')',
192-
'select append_range_partition(\'abc\')',
193-
'select add_range_partition(\'abc\', 500000, 550000)',
194-
]
195-
threads = []
196-
for i in range(3):
197-
thread = \
198-
threading.Thread(target=add_partition, args=(node, flags[i], query[i]))
199-
threads.append(thread)
200-
thread.start()
201-
time.sleep(3)
202-
203-
# This threads should wait until current transaction finished
204-
with lock:
187+
188+
try:
189+
node.init()
190+
node.append_conf('postgresql.conf', 'shared_preload_libraries=\'pg_pathman\'\n')
191+
node.start()
192+
node.safe_psql(
193+
'postgres',
194+
'create extension pg_pathman; '
195+
+ 'create table abc(id serial, t text); '
196+
+ 'insert into abc select generate_series(1, 100000); '
197+
+ 'select create_range_partitions(\'abc\', \'id\', 1, 50000);'
198+
)
199+
200+
# Start transaction that will create partition
201+
con = node.connect()
202+
con.begin()
203+
con.execute('select append_range_partition(\'abc\')')
204+
205+
# Start threads that suppose to add new partitions and wait some time
206+
query = [
207+
'select prepend_range_partition(\'abc\')',
208+
'select append_range_partition(\'abc\')',
209+
'select add_range_partition(\'abc\', 500000, 550000)',
210+
]
211+
threads = []
205212
for i in range(3):
206-
self.assertEqual(flags[i].get(), False)
213+
thread = \
214+
threading.Thread(target=add_partition, args=(node, flags[i], query[i]))
215+
threads.append(thread)
216+
thread.start()
217+
time.sleep(3)
207218

208-
# Commit transaction. Since then other sessions can create partitions
209-
con.commit()
219+
# This threads should wait until current transaction finished
220+
with lock:
221+
for i in range(3):
222+
self.assertEqual(flags[i].get(), False)
210223

211-
# Now wait until each thread finishes
212-
for i in range(3):
213-
threads[i].join()
224+
# Commit transaction. Since then other sessions can create partitions
225+
con.commit()
214226

215-
# Check flags, it should be true which means that threads are finished
216-
with lock:
227+
# Now wait until each thread finishes
217228
for i in range(3):
218-
self.assertEqual(flags[i].get(), True)
229+
threads[i].join()
219230

220-
# Check that all partitions are created
221-
self.assertEqual(
222-
node.safe_psql(
223-
'postgres',
224-
'select count(*) from pg_inherits where inhparent=\'abc\'::regclass'
225-
),
226-
'6\n'
227-
)
231+
# Check flags, it should be true which means that threads are finished
232+
with lock:
233+
for i in range(3):
234+
self.assertEqual(flags[i].get(), True)
235+
236+
# Check that all partitions are created
237+
self.assertEqual(
238+
node.safe_psql(
239+
'postgres',
240+
'select count(*) from pg_inherits where inhparent=\'abc\'::regclass'
241+
),
242+
'6\n'
243+
)
244+
except Exception, e:
245+
self.printlog(node.logs_dir + '/postgresql.log')
246+
raise e
228247

229248
if __name__ == "__main__":
230249
unittest.main()

‎travis/pg-travis-test.sh

Lines changed: 22 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,8 @@ sudo apt-get update
66

77

88
# required packages
9-
packages="postgresql-$PGVERSION postgresql-server-dev-$PGVERSION postgresql-common"
9+
apt_packages="postgresql-$PGVERSION postgresql-server-dev-$PGVERSION postgresql-common python-pip python-dev build-essential"
10+
pip_packages="testgres"
1011

1112
# exit code
1213
status=0
@@ -25,7 +26,7 @@ echo 'exit 0' | sudo tee /etc/init.d/postgresql
2526
sudo chmod a+x /etc/init.d/postgresql
2627

2728
# install required packages
28-
sudo apt-get -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" -y install -qq $packages
29+
sudo apt-get -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" -y install -qq $apt_packages
2930

3031
# create cluster 'test'
3132
sudo pg_createcluster --start $PGVERSION test -p 55435 -- -A trust
@@ -73,4 +74,23 @@ PGPORT=55435 make installcheck USE_PGXS=1 PGUSER=postgres PG_CONFIG=$config_path
7374

7475
# show diff if it exists
7576
if test -f regression.diffs; then cat regression.diffs; fi
77+
78+
set +u
79+
80+
# create a virtual environment and activate it
81+
virtualenv /tmp/envs/pg_pathman
82+
source /tmp/envs/pg_pathman/bin/activate
83+
84+
# install pip packages
85+
pip install $pip_packages
86+
87+
# set permission to write postgres locks
88+
sudo chmod a+w /var/run/postgresql/
89+
90+
# run python tests
91+
cd tests
92+
PG_CONFIG=$config_path python -m unittest partitioning_test || status=$?
93+
94+
set -u
95+
7696
exit $status

0 commit comments

Comments
 (0)
pFad - Phonifier reborn

Pfad - The Proxy pFad of © 2024 Garber Painting. All rights reserved.

Note: This service is not intended for secure transactions such as banking, social media, email, or purchasing. Use at your own risk. We assume no liability whatsoever for broken pages.


Alternative Proxies:

Alternative Proxy

pFad Proxy

pFad v3 Proxy

pFad v4 Proxy