Skip to content

Commit f73f993

Browse files
knizhnikknizhnik
committed
Use synchronous commit
2 parents 7dc0fe6 + 59befec commit f73f993

File tree

9 files changed

+110
-86
lines changed

9 files changed

+110
-86
lines changed

‎.dockerignore

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
*/*/.git
2+
.git
3+
.vscode

‎Dockerfile

Lines changed: 52 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,52 @@
1+
# vim:set ft=dockerfile:
2+
FROM debian:jessie
3+
4+
# explicitly set user/group IDs
5+
RUN groupadd -r postgres --gid=999 && useradd -r -g postgres --uid=999 postgres
6+
7+
# make the "en_US.UTF-8" locale so postgres will be utf-8 enabled by default
8+
RUN apt-get update && apt-get install -y locales && rm -rf /var/lib/apt/lists/* \
9+
&& localedef -i en_US -c -f UTF-8 -A /usr/share/locale/locale.alias en_US.UTF-8
10+
ENV LANG en_US.utf8
11+
12+
# postgres build deps
13+
RUN apt-get update && apt-get install -y \
14+
git \
15+
make \
16+
gcc \
17+
gdb \
18+
libreadline-dev \
19+
bison \
20+
flex \
21+
zlib1g-dev \
22+
sudo \
23+
&& rm -rf /var/lib/apt/lists/*
24+
25+
RUN mkdir /pg && chown postgres:postgres /pg
26+
# We need that to allow editing of /proc/sys/kernel/core_pattern
27+
# from docker-entrypoint.sh
28+
RUN echo "postgres ALL=(ALL:ALL) NOPASSWD:ALL" >> /etc/sudoers
29+
30+
COPY ./ /pg/src
31+
RUN chown -R postgres:postgres /pg/src
32+
33+
USER postgres
34+
ENV CFLAGS -O0
35+
WORKDIR /pg
36+
37+
RUN cd /pg/src && \
38+
ls -la && \
39+
whoami && \
40+
./configure --enable-cassert --enable-debug --prefix=/pg/install && \
41+
make -j 4 install
42+
43+
ENV PATH /pg/install/bin:$PATH
44+
ENV PGDATA /pg/data
45+
46+
RUN cd /pg/src/contrib/raftable && make clean && make install
47+
RUN cd /pg/src/contrib/mmts && make clean && make install
48+
49+
ENTRYPOINT ["/pg/src/contrib/mmts/tests2/docker-entrypoint.sh"]
50+
51+
EXPOSE 5432
52+
CMD ["postgres"]

‎GNUmakefile.in

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,20 @@ include $(top_builddir)/src/Makefile.global
1010

1111
$(call recurse,all install,src config)
1212

13+
xcheck:
14+
#pip install -r tests2/requirements.txt
15+
# docker build -t pgmmts .
16+
cd contrib/mmts/tests2 && blockade destroy || true
17+
cd contrib/mmts/tests2 && docker rm node1 || true
18+
cd contrib/mmts/tests2 && docker rm node2 || true
19+
cd contrib/mmts/tests2 && docker rm node3 || true
20+
cd contrib/mmts/tests2 && docker network rm tests2_net || true
21+
cd contrib/mmts/tests2 && docker network rm tests2_net || true
22+
cd contrib/mmts/tests2 && blockade up
23+
sleep 20 # wait for mmts init
24+
cd contrib/mmts/tests2 && python test_recovery.py || true
25+
#cd contrib/mmts/tests2 && blockade destroy
26+
1327
all:
1428
+@echo "All of PostgreSQL successfully made. Ready to install."
1529

‎contrib/mmts/Dockerfile

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -31,7 +31,7 @@ USER postgres
3131
ENV CFLAGS -O0
3232
WORKDIR /pg
3333

34-
ENV REBUILD 2
34+
ENV REBUILD 5
3535

3636
RUN cd /pg && \
3737
git clone https://github.com/postgrespro/postgres_cluster.git --depth 1 && \

‎contrib/mmts/tests2/docker-entrypoint.sh

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -68,13 +68,15 @@ if [ "$1" = 'postgres' ]; then
6868
listen_addresses='*'
6969
max_prepared_transactions = 100
7070
synchronous_commit = on
71+
fsync = off
7172
wal_level = logical
7273
max_worker_processes = 30
7374
max_replication_slots = 10
7475
max_wal_senders = 10
7576
shared_preload_libraries = 'raftable,multimaster'
76-
default_transaction_isolation = 'repeatable read'
77+
default_transaction_isolation = 'repeatable read'
7778
log_checkpoints = on
79+
checkpoint_timeout = 30
7880
log_autovacuum_min_duration = 0
7981
8082
raftable.id = $NODE_ID

‎contrib/mmts/tests2/lib/bank_client.py

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -33,11 +33,14 @@ def stop(self):
3333
for client in self._clients:
3434
client.stop()
3535

36-
def print_agg(self):
36+
def aggregate(self, echo=True):
3737
aggs = []
3838
for client in self._clients:
3939
aggs.append(client.history.aggregate())
4040

41+
if not echo:
42+
return aggs
43+
4144
columns = ['running', 'running_latency', 'max_latency', 'finish']
4245

4346
print("\t\t", end="")
@@ -61,10 +64,7 @@ def print_agg(self):
6164

6265
print("")
6366

64-
def set_acc_to_tx(self, max_acc):
65-
for client in self._clients:
66-
client.set_acc_to_tx(max_acc)
67-
67+
return aggs
6868

6969
class BankClient(object):
7070

@@ -141,7 +141,7 @@ def exec_tx(self, name, tx_block):
141141
tx_block(conn, cur)
142142
self.history.register_finish(event_id, 'Commit')
143143
except psycopg2.Error as e:
144-
print("=== node%d: %s" % (self.node_id, e.pgerror))
144+
# print("=== node%d: %s" % (self.node_id, e.pgerror))
145145
self.history.register_finish(event_id, e.pgerror)
146146
#time.sleep(0.2)
147147

‎contrib/mmts/tests2/lib/event_history.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -85,7 +85,7 @@ def aggregate(self):
8585
self.events = []
8686

8787
for value in self.running_events.itervalues():
88-
print(value)
88+
# print(value)
8989

9090
if value['name'] not in agg:
9191
agg[value['name']] = copy.deepcopy(self.agg_template)

‎contrib/mmts/tests2/test_recovery.py

Lines changed: 26 additions & 75 deletions
Original file line numberDiff line numberDiff line change
@@ -6,105 +6,56 @@
66
class RecoveryTest(unittest.TestCase):
77
@classmethod
88
def setUpClass(self):
9-
#subprocess.check_call(['blockade','up'])
109
self.clients = ClientCollection([
1110
"dbname=postgres host=127.0.0.1 user=postgres",
1211
"dbname=postgres host=127.0.0.1 user=postgres port=5433",
1312
"dbname=postgres host=127.0.0.1 user=postgres port=5434"
1413
])
14+
self.clients.start()
15+
time.sleep(5)
1516

1617
@classmethod
1718
def tearDownClass(self):
1819
print('tearDown')
19-
#subprocess.check_call(['blockade','join'])
20-
21-
# in case of error
2220
self.clients.stop()
23-
#self.clients[0].cleanup()
2421

25-
def test_0_0_0(self):
26-
print('### normalOpsTest ###')
22+
# def test_normal_operations(self):
23+
# print('### normalOpsTest ###')
24+
25+
# for i in range(3):
26+
# time.sleep(3)
27+
# aggs = self.clients.aggregate()
28+
# for agg in aggs:
29+
# # there were some commits
30+
# self.assertTrue( agg['transfer'] > 0 )
31+
32+
def test_node_prtition(self):
33+
print('### nodePartitionTest ###')
2734

28-
time.sleep(10)
2935
subprocess.check_call(['blockade','partition','node3'])
3036
print('### blockade node3 ###')
3137

32-
self.clients.set_acc_to_tx(10000)
33-
self.clients.start()
38+
# clear tx history
39+
self.clients.aggregate(echo=False)
3440

35-
for i in range(10):
41+
for i in range(3):
3642
time.sleep(3)
37-
self.clients.print_agg()
38-
print("\n")
43+
aggs = self.clients.aggregate()
44+
self.assertTrue( aggs[0]['transfer']['finish']['Commit'] > 0 )
45+
self.assertTrue( aggs[1]['transfer']['finish']['Commit'] > 0 )
46+
self.assertTrue( 'Commit' not in aggs[2]['transfer'] )
3947

4048
subprocess.check_call(['blockade','join'])
4149
print('### deblockade node3 ###')
4250

51+
# clear tx history
52+
self.clients.aggregate(echo=False)
53+
4354
for i in range(1000):
4455
time.sleep(3)
45-
t = datetime.datetime.now()
46-
self.clients.print_agg()
47-
print("took %f seconds" % ( (datetime.datetime.now()-t).total_seconds(),))
48-
print("\n")
49-
50-
self.clients.stop()
51-
52-
56+
aggs = self.clients.aggregate()
57+
print(i, aggs)
5358

54-
# def test_0_normal_operation(self):
55-
# print('### normalOpsTest ###')
56-
#
57-
# self.clients.set_acc_to_tx(10000)
58-
# self.clients.start()
59-
#
60-
# for i in range(5):
61-
# time.sleep(3)
62-
# for client in self.clients:
63-
# agg = client.history.aggregate()
64-
# print(agg)
65-
# self.assertTrue(agg['transfer']['finish']['Commit'] > 0)
66-
# print("\n")
67-
#
68-
# self.clients.stop()
69-
#
70-
# def test_1_distributed_deadlock(self):
71-
# print('### DDD test ###')
72-
#
73-
# self.clients.set_acc_to_tx(10)
74-
# self.clients.start()
75-
#
76-
# for i in range(5):
77-
# time.sleep(3)
78-
# for client in self.clients:
79-
# agg = client.history.aggregate()
80-
# print(agg)
81-
# self.assertTrue(agg['transfer']['finish']['Commit'] > 0)
82-
# print("\n")
83-
#
84-
# self.clients.stop()
85-
#
86-
# def test_2_node_disconnect(self):
87-
# print('### disconnectTest ###')
88-
#
89-
# self.clients.set_acc_to_tx(10000)
90-
# self.clients.start()
91-
#
92-
# subprocess.check_call(['blockade','partition','node3'])
93-
# print('Node3 disconnected')
94-
#
95-
# # give cluster some time to discover problem
96-
# time.sleep(3)
97-
#
98-
# for i in range(5):
99-
# time.sleep(3)
100-
# for client in self.clients:
101-
# agg = client.history.aggregate()
102-
# print(agg)
103-
# self.assertTrue(agg['transfer']['finish']['Commit'] > 0)
104-
# print("\n")
105-
#
106-
# subprocess.check_call(['blockade','join'])
107-
# self.clients.stop()
10859

10960
if __name__ == '__main__':
11061
unittest.main()

‎src/backend/access/transam/twophase.c

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1218,7 +1218,6 @@ ReadTwoPhaseFile(TransactionId xid, bool give_warnings)
12181218
if (crc_offset != MAXALIGN(crc_offset))
12191219
{
12201220
CloseTransientFile(fd);
1221-
fprintf(stderr, "wrong crc offset in two-phase file \"%s\"\n", path);
12221221
return NULL;
12231222
}
12241223

@@ -1245,7 +1244,6 @@ ReadTwoPhaseFile(TransactionId xid, bool give_warnings)
12451244
if (hdr->magic != TWOPHASE_MAGIC || hdr->total_len != stat.st_size)
12461245
{
12471246
pfree(buf);
1248-
fprintf(stderr, "muggle two-phase file \"%s\": no magic\n", path);
12491247
return NULL;
12501248
}
12511249

@@ -1622,6 +1620,10 @@ RecreateTwoPhaseFile(TransactionId xid, void *content, int len)
16221620
pg_crc32c statefile_crc;
16231621
int fd;
16241622

1623+
/* Crutch to fix crc and len check on 2pc file reading */
1624+
Assert( ((TwoPhaseFileHeader *) content)->total_len - sizeof(pg_crc32c) <= len);
1625+
len = ((TwoPhaseFileHeader *) content)->total_len - sizeof(pg_crc32c);
1626+
16251627
/* Recompute CRC */
16261628
INIT_CRC32C(statefile_crc);
16271629
COMP_CRC32C(statefile_crc, content, len);

0 commit comments

Comments
 (0)
pFad - Phonifier reborn

Pfad - The Proxy pFad of © 2024 Garber Painting. All rights reserved.

Note: This service is not intended for secure transactions such as banking, social media, email, or purchasing. Use at your own risk. We assume no liability whatsoever for broken pages.


Alternative Proxies:

Alternative Proxy

pFad Proxy

pFad v3 Proxy

pFad v4 Proxy