Skip to content

Commit cf3656d

Browse files
kelvichkelvich
committed
raftable tests
1 parent e7b9388 commit cf3656d

File tree

12 files changed

+488
-17
lines changed

12 files changed

+488
-17
lines changed

‎contrib/mmts/tests2/lib/bank_client.py

Lines changed: 22 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -9,8 +9,8 @@ class ClientCollection(object):
99
def __init__(self, connstrs):
1010
self._clients = []
1111

12-
for cs in connstrs:
13-
b = BankClient(cs)
12+
for i, cs in enumerate(connstrs):
13+
b = BankClient(cs, i)
1414
self._clients.append(b)
1515

1616
self._clients[0].initialize()
@@ -27,17 +27,21 @@ def start(self):
2727
client.start()
2828

2929
def stop(self):
30+
print('collection stop called', self._clients)
3031
for client in self._clients:
32+
print('stop coll')
3133
client.stop()
3234

3335

3436
class BankClient(object):
3537

36-
def __init__(self, connstr):
38+
def __init__(self, connstr, node_id):
3739
self.connstr = connstr
40+
self.node_id = node_id
3841
self.run = Value('b', True)
3942
self._history = EventHistory()
4043
self.accounts = 10000
44+
self.show_errors = True
4145

4246
def initialize(self):
4347
conn = psycopg2.connect(self.connstr)
@@ -59,6 +63,10 @@ def initialize(self):
5963
def history(self):
6064
return self._history
6165

66+
def print_error(self, arg, comment=''):
67+
if self.show_errors:
68+
print('Node', self.node_id, 'got error', arg, comment)
69+
6270
def check_total(self):
6371
conn, cur = self.connect()
6472

@@ -72,16 +80,13 @@ def check_total(self):
7280
print("Isolation error, total = %d" % (res[0],))
7381
raise BaseException
7482
except psycopg2.InterfaceError:
75-
print("Got error: ", sys.exc_info())
76-
print("Reconnecting")
7783
conn, cur = self.connect(reconnect=True)
7884
except:
79-
print("Got error: ", sys.exc_info())
85+
self.print_error(sys.exc_info(),'3')
8086
self.history.register_finish(event_id, 'rollback')
8187
else:
8288
self.history.register_finish(event_id, 'commit')
8389

84-
8590
cur.close()
8691
conn.close()
8792

@@ -110,14 +115,10 @@ def transfer_money(self):
110115
(amount, to_uid))
111116

112117
conn.commit()
113-
except psycopg2.DatabaseError:
114-
print("Got error: ", sys.exc_info())
115-
print("Reconnecting")
116-
117-
self.history.register_finish(event_id, 'rollback')
118+
except psycopg2.InterfaceError:
118119
conn, cur = self.connect(reconnect=True)
119120
except:
120-
print("Got error: ", sys.exc_info())
121+
self.print_error(sys.exc_info(),'1')
121122
self.history.register_finish(event_id, 'rollback')
122123
else:
123124
self.history.register_finish(event_id, 'commit')
@@ -127,16 +128,17 @@ def transfer_money(self):
127128

128129
def connect(self, reconnect=False):
129130

130-
while self.run.value:
131+
while True:
131132
try:
132133
conn = psycopg2.connect(self.connstr)
133134
cur = conn.cursor()
135+
return conn, cur
134136
except:
135-
print("Got error: ", sys.exc_info())
137+
self.print_error(sys.exc_info(),'2')
136138
if not reconnect:
137139
raise
138-
else:
139-
return conn, cur
140+
if not self.run.value:
141+
raise
140142

141143
# def watchdog(self):
142144
# while self.run.value:
@@ -156,7 +158,10 @@ def start(self):
156158
return
157159

158160
def stop(self):
161+
print('Stopping!');
159162
self.run.value = False
163+
self.total_process.join()
164+
self.transfer_process.join()
160165
return
161166

162167
def cleanup(self):

‎contrib/mmts/tests2/test_recovery.py

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@ def setUp(self):
1515
self.clients.start()
1616

1717
def tearDown(self):
18+
print('tearDown')
1819
self.clients.stop()
1920
self.clients[0].cleanup()
2021
subprocess.check_call(['blockade','join'])

‎contrib/raftable/tests/.gitignore

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
.blockade
2+
.vagrant
3+
*.swp
4+
*.pyc

‎contrib/raftable/tests/Dockerfile

Lines changed: 60 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,60 @@
1+
# vim:set ft=dockerfile:
2+
FROM debian:jessie
3+
4+
# explicitly set user/group IDs
5+
RUN groupadd -r postgres --gid=999 && useradd -r -g postgres --uid=999 postgres
6+
7+
# make the "en_US.UTF-8" locale so postgres will be utf-8 enabled by default
8+
RUN apt-get update && apt-get install -y locales && rm -rf /var/lib/apt/lists/* \
9+
&& localedef -i en_US -c -f UTF-8 -A /usr/share/locale/locale.alias en_US.UTF-8
10+
ENV LANG en_US.utf8
11+
12+
# use git to fetch sources
13+
RUN apt-get update \
14+
&& apt-get install -y git \
15+
&& rm -rf /var/lib/apt/lists/*
16+
17+
# postgres build deps
18+
RUN apt-get update && apt-get install -y \
19+
make \
20+
gcc \
21+
libreadline-dev \
22+
bison \
23+
flex \
24+
zlib1g-dev \
25+
&& rm -rf /var/lib/apt/lists/*
26+
27+
RUN mkdir /pg
28+
RUN chown postgres:postgres /pg
29+
30+
USER postgres
31+
WORKDIR /pg
32+
ENV CFLAGS -O0
33+
RUN git clone https://github.com/postgrespro/postgres_cluster.git --depth 1
34+
WORKDIR /pg/postgres_cluster
35+
RUN ./configure --enable-cassert --enable-debug --prefix=/usr/local/
36+
RUN make -j 4
37+
38+
USER root
39+
RUN make install
40+
RUN cd /pg/postgres_cluster/contrib/pg_tsdtm && make install
41+
RUN cd /pg/postgres_cluster/contrib/raftable && make install
42+
RUN cd /pg/postgres_cluster/contrib/mmts && make install
43+
RUN cd /pg/postgres_cluster/contrib/postgres_fdw && make install
44+
RUN mkdir -p /var/lib/postgresql/data && chown -R postgres /var/lib/postgresql/data
45+
RUN mkdir -p /run/postgresql && chown -R postgres /run/postgresql
46+
47+
USER postgres
48+
ENV PATH /usr/local/bin:$PATH
49+
ENV PGDATA /var/lib/postgresql/data
50+
VOLUME /var/lib/postgresql/data
51+
52+
COPY docker-entrypoint.sh /
53+
54+
ENTRYPOINT ["/docker-entrypoint.sh"]
55+
56+
EXPOSE 5432
57+
CMD ["postgres"]
58+
59+
60+

‎contrib/raftable/tests/Vagrantfile

Lines changed: 44 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,44 @@
1+
script = <<SCRIPT
2+
#!/bin/bash -e
3+
4+
if [ ! -f /etc/default/docker ]; then
5+
echo "/etc/default/docker not found -- is docker installed?" >&2
6+
exit 1
7+
fi
8+
9+
apt-get -y install python-pip python-virtualenv
10+
apt-get -y install postgresql-server-dev-all python-dev postgresql-client
11+
12+
cd /vagrant
13+
14+
export PIP_DOWNLOAD_CACHE=/vagrant/.pip_download_cache
15+
16+
pip install -r requirements.txt
17+
18+
SCRIPT
19+
20+
Vagrant.configure(2) do |config|
21+
22+
config.vm.box = "ubuntu/trusty64"
23+
24+
if Vagrant.has_plugin?("vagrant-cachier")
25+
config.cache.scope = :box
26+
end
27+
28+
config.vm.provider :virtualbox do |vb, override|
29+
vb.memory = 2048
30+
end
31+
32+
# there are obviously vagrant versions with a
33+
# broken 'docker' provisioner that can be fixed
34+
# by invoking an 'apt-get update' *before* docker itself
35+
config.vm.provision "shell", inline: "apt-get update"
36+
37+
# fetch the ubuntu:latest image that is required for
38+
# the test suite
39+
config.vm.provision "docker", images: ["ubuntu:trusty"]
40+
41+
# kick off the tests automatically
42+
config.vm.provision "shell", inline: script
43+
end
44+

‎contrib/raftable/tests/blockade.yml

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,32 @@
1+
version: '2'
2+
3+
containers:
4+
5+
node1:
6+
container_name: node1
7+
image: pgmmts
8+
environment:
9+
POSTGRES_USER: 'postgres'
10+
NODE_ID: 1
11+
ports:
12+
5432: 5432
13+
14+
node2:
15+
container_name: node2
16+
image: pgmmts
17+
environment:
18+
POSTGRES_USER: 'postgres'
19+
NODE_ID: 2
20+
ports:
21+
5433: 5432
22+
23+
node3:
24+
container_name: node3
25+
image: pgmmts
26+
environment:
27+
POSTGRES_USER: 'postgres'
28+
NODE_ID: 3
29+
ports:
30+
5434: 5432
31+
32+

‎contrib/raftable/tests/docker-entrypoint.sh

Lines changed: 88 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,88 @@
1+
#!/bin/bash
2+
set -e
3+
4+
# wtf is that?
5+
if [ "${1:0:1}" = '-' ]; then
6+
set -- postgres "$@"
7+
fi
8+
9+
if [ "$1" = 'postgres' ]; then
10+
mkdir -p "$PGDATA"
11+
chmod 700 "$PGDATA"
12+
chown -R postgres "$PGDATA"
13+
14+
# look specifically for PG_VERSION, as it is expected in the DB dir
15+
if [ ! -s "$PGDATA/PG_VERSION" ]; then
16+
initdb
17+
18+
if [ "$POSTGRES_PASSWORD" ]; then
19+
pass="PASSWORD '$POSTGRES_PASSWORD'"
20+
authMethod=md5
21+
else
22+
pass=
23+
authMethod=trust
24+
fi
25+
26+
{ echo; echo "host all all 0.0.0.0/0 $authMethod"; } >> "$PGDATA/pg_hba.conf"
27+
{ echo; echo "host replication all 0.0.0.0/0 $authMethod"; } >> "$PGDATA/pg_hba.conf"
28+
29+
############################################################################
30+
31+
# internal start of server in order to allow set-up using psql-client
32+
# does not listen on TCP/IP and waits until start finishes
33+
pg_ctl -D "$PGDATA" \
34+
-o "-c listen_addresses=''" \
35+
-w start
36+
37+
: ${POSTGRES_USER:=postgres}
38+
: ${POSTGRES_DB:=$POSTGRES_USER}
39+
export POSTGRES_USER POSTGRES_DB
40+
41+
psql=( psql -v ON_ERROR_STOP=1 )
42+
43+
if [ "$POSTGRES_DB" != 'postgres' ]; then
44+
"${psql[@]}" --username postgres <<-EOSQL
45+
CREATE DATABASE "$POSTGRES_DB" ;
46+
EOSQL
47+
echo
48+
fi
49+
50+
if [ "$POSTGRES_USER" = 'postgres' ]; then
51+
op='ALTER'
52+
else
53+
op='CREATE'
54+
fi
55+
"${psql[@]}" --username postgres <<-EOSQL
56+
$op USER "$POSTGRES_USER" WITH SUPERUSER $pass ;
57+
EOSQL
58+
echo
59+
60+
############################################################################
61+
62+
RAFT_PEERS='1:172.18.0.2:6666, 2:172.18.0.4:6666, 3:172.18.0.3:6666'
63+
64+
cat <<-EOF >> $PGDATA/postgresql.conf
65+
listen_addresses='*'
66+
max_prepared_transactions = 100
67+
synchronous_commit = off
68+
wal_level = logical
69+
max_worker_processes = 15
70+
max_replication_slots = 10
71+
max_wal_senders = 10
72+
shared_preload_libraries = 'raftable'
73+
raftable.id = $NODE_ID
74+
raftable.peers = '$RAFT_PEERS'
75+
EOF
76+
77+
tail -n 20 $PGDATA/postgresql.conf
78+
79+
pg_ctl -D "$PGDATA" -m fast -w stop
80+
81+
echo
82+
echo 'PostgreSQL init process complete; ready for start up.'
83+
echo
84+
fi
85+
fi
86+
87+
exec "$@"
88+

‎contrib/raftable/tests/lib/__init__.py

Whitespace-only changes.

0 commit comments

Comments
 (0)
pFad - Phonifier reborn

Pfad - The Proxy pFad of © 2024 Garber Painting. All rights reserved.

Note: This service is not intended for secure transactions such as banking, social media, email, or purchasing. Use at your own risk. We assume no liability whatsoever for broken pages.


Alternative Proxies:

Alternative Proxy

pFad Proxy

pFad v3 Proxy

pFad v4 Proxy