Skip to content

Commit a09fc09

Browse files
kelvichkelvich
committed
pg_shard test suite
1 parent 316e377 commit a09fc09

File tree

3 files changed

+365
-0
lines changed

3 files changed

+365
-0
lines changed

‎contrib/pg_xtm/tests/pg_shard_transfers.go

Lines changed: 118 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,118 @@
1+
package main
2+
3+
import (
4+
"fmt"
5+
"sync"
6+
// "github.com/jackc/pgx"
7+
"pgx"
8+
)
9+
10+
const (
11+
TRANSFER_CONNECTIONS = 1
12+
INIT_AMOUNT = 10000
13+
N_ITERATIONS = 5000
14+
N_ACCOUNTS = TRANSFER_CONNECTIONS
15+
)
16+
17+
var cfg = pgx.ConnConfig{
18+
Host: "127.0.0.1",
19+
Port: 5433,
20+
// Database: "postgres",
21+
}
22+
23+
var running = false
24+
25+
func transfer(id int, wg *sync.WaitGroup) {
26+
var err error
27+
var conn *pgx.Conn
28+
29+
conn, err = pgx.Connect(cfg)
30+
checkErr(err)
31+
defer conn.Close()
32+
33+
for i:=0; i < N_ITERATIONS; i++ {
34+
exec(conn, "begin")
35+
exec(conn, "update t_10000 set v = v + 1 where u=3")
36+
exec(conn, "update t_10000 set v = v - 1 where u=4")
37+
exec(conn, "commit")
38+
}
39+
40+
wg.Done()
41+
}
42+
43+
func inspect(wg *sync.WaitGroup) {
44+
var sum int64
45+
var prevSum int64 = 0
46+
47+
conn, err := pgx.Connect(cfg)
48+
checkErr(err)
49+
50+
for running {
51+
sum = execQuery(conn, "select sum(v) from t_10000")
52+
if sum != prevSum {
53+
fmt.Println("Total = ", sum);
54+
prevSum = sum
55+
}
56+
}
57+
58+
conn.Close()
59+
wg.Done()
60+
}
61+
62+
func main() {
63+
// var transferWg sync.WaitGroup
64+
// var inspectWg sync.WaitGroup
65+
var err error
66+
var conn *pgx.Conn
67+
var s int64
68+
69+
conn, err = pgx.Connect(cfg)
70+
checkErr(err)
71+
defer conn.Close()
72+
73+
// err = conn.QueryRow("select sum(v) from t_10000").Scan(&s)
74+
// checkErr(err)
75+
76+
s = execQuery(conn, "select sum(v) from t_10000")
77+
fmt.Println(s)
78+
79+
80+
// transferWg.Add(TRANSFER_CONNECTIONS)
81+
// for i:=0; i<TRANSFER_CONNECTIONS; i++ {
82+
// go transfer(i, &transferWg)
83+
// }
84+
85+
// running = true
86+
// inspectWg.Add(1)
87+
// go inspect(&inspectWg)
88+
89+
// transferWg.Wait()
90+
91+
// running = false
92+
// inspectWg.Wait()
93+
94+
fmt.Printf("done\n")
95+
}
96+
97+
func exec(conn *pgx.Conn, stmt string) {
98+
var err error
99+
_, err = conn.Exec(stmt)
100+
checkErr(err)
101+
}
102+
103+
func execQuery(conn *pgx.Conn, stmt string) int64 {
104+
var err error
105+
var result int64
106+
// result, err = conn.SimpleQuery(stmt)
107+
// err = conn.QueryRow(stmt).Scan(&result)
108+
err = conn.SimpleQuery(stmt).Scan(&result)
109+
checkErr(err)
110+
return result
111+
}
112+
113+
func checkErr(err error) {
114+
if err != nil {
115+
panic(err)
116+
}
117+
}
118+

‎install_pg_shard.sh

Lines changed: 117 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,117 @@
1+
#!/bin/sh
2+
3+
PG_SHARD_DIR=~/code/pg_shard_master
4+
PG_DIR=~/code/postgresql
5+
PG_XTM_DIR=$PG_DIR/contrib/pg_xtm
6+
7+
8+
########################################################################
9+
# Stop old stuff
10+
########################################################################
11+
./install/bin/pg_ctl -D ./install/data1 stop
12+
./install/bin/pg_ctl -D ./install/data2 stop
13+
./install/bin/pg_ctl -D ./install/data3 stop
14+
killall dtmd
15+
rm -rf install
16+
17+
18+
########################################################################
19+
# Build and run dtm and postgres
20+
########################################################################
21+
make install # assuming configured with --prefix=./install
22+
23+
cd $PG_SHARD_DIR
24+
make clean
25+
PATH=~/code/postgresql/install/bin/:$PATH make
26+
PATH=~/code/postgresql/install/bin/:$PATH make install
27+
28+
cd $PG_DIR
29+
./install/bin/initdb -D ./install/data1
30+
./install/bin/initdb -D ./install/data2
31+
./install/bin/initdb -D ./install/data3
32+
33+
34+
sed -i '' 's/#port =.*/port = 5433/' ./install/data2/postgresql.conf
35+
sed -i '' 's/#port =.*/port = 5434/' ./install/data3/postgresql.conf
36+
37+
sed -i '' "s/shared_preload_libraries =.*/shared_preload_libraries = 'pg_shard'/" ./install/data1/postgresql.conf
38+
sed -i '' "s/shared_preload_libraries =.*/shared_preload_libraries = ''/" ./install/data2/postgresql.conf
39+
sed -i '' "s/shared_preload_libraries =.*/shared_preload_libraries = ''/" ./install/data3/postgresql.conf
40+
41+
sed -i '' 's/#fsync =.*/fsync = off/' ./install/data1/postgresql.conf
42+
sed -i '' 's/#fsync =.*/fsync = off/' ./install/data2/postgresql.conf
43+
sed -i '' 's/#fsync =.*/fsync = off/' ./install/data3/postgresql.conf
44+
45+
46+
# echo 'pg_shard.use_dtm_transactions=1' >> ./install/data1/postgresql.conf
47+
# echo 'pg_shard.use_dtm_transactions=1' >> ./install/data2/postgresql.conf
48+
# echo 'pg_shard.use_dtm_transactions=1' >> ./install/data3/postgresql.conf
49+
50+
51+
52+
./install/bin/pg_ctl -D ./install/data1 -l ./install/data1/log start
53+
./install/bin/pg_ctl -D ./install/data2 -l ./install/data2/log start
54+
./install/bin/pg_ctl -D ./install/data3 -l ./install/data3/log start
55+
56+
57+
########################################################################
58+
# Configure pg_shard
59+
########################################################################
60+
61+
echo "127.0.0.1 5433" > ./install/data1/pg_worker_list.conf
62+
echo "127.0.0.1 5434" >> ./install/data1/pg_worker_list.conf
63+
64+
echo "127.0.0.1 5433" > ./install/data2/pg_worker_list.conf
65+
echo "127.0.0.1 5434" >> ./install/data2/pg_worker_list.conf
66+
67+
echo "127.0.0.1 5433" > ./install/data3/pg_worker_list.conf
68+
echo "127.0.0.1 5434" >> ./install/data3/pg_worker_list.conf
69+
70+
71+
./install/bin/createdb `whoami`
72+
./install/bin/createdb `whoami` -p5433
73+
./install/bin/createdb `whoami` -p5434
74+
75+
76+
# ./install/bin/psql -p 5433 << SQL
77+
# CREATE EXTENSION pg_dtm;
78+
# SQL
79+
80+
# ./install/bin/psql -p 5434 << SQL
81+
# CREATE EXTENSION pg_dtm;
82+
# SQL
83+
84+
./install/bin/psql << SQL
85+
86+
CREATE EXTENSION pg_shard;
87+
CREATE TABLE t(u int, v int);
88+
SELECT master_create_distributed_table(table_name := 't', partition_column := 'u');
89+
SELECT master_create_worker_shards(table_name := 't', shard_count := 8, replication_factor := 1);
90+
insert into t values(1,10000);
91+
insert into t values(2,10000);
92+
insert into t values(3,10000);
93+
insert into t values(4,10000);
94+
insert into t values(5,10000);
95+
insert into t values(6,10000);
96+
insert into t values(7,10000);
97+
insert into t values(8,10000);
98+
99+
SQL
100+
101+
102+
103+
104+
# insert into t (select generate_series(0,10), random()::integer);
105+
106+
# cd contrib/pg_xtm/dtmd
107+
# make clean
108+
# make
109+
# rm -rf /tmp/clog/*
110+
# ./bin/dtmd
111+
112+
113+
114+
115+
116+
117+

‎install_pg_shard_xtm.sh

Lines changed: 130 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,130 @@
1+
#!/bin/sh
2+
3+
PG_SHARD_DIR=~/code/pg_shard
4+
PG_DIR=~/code/postgresql
5+
PG_XTM_DIR=$PG_DIR/contrib/pg_xtm
6+
7+
8+
########################################################################
9+
# Stop old stuff
10+
########################################################################
11+
./install/bin/pg_ctl -D ./install/data1 stop
12+
./install/bin/pg_ctl -D ./install/data2 stop
13+
./install/bin/pg_ctl -D ./install/data3 stop
14+
killall dtmd
15+
rm -rf install
16+
17+
18+
########################################################################
19+
# Build and run dtm and postgres
20+
########################################################################
21+
make install # assuming configured with --prefix=./install
22+
23+
cd $PG_SHARD_DIR
24+
make clean
25+
PATH=~/code/postgresql/install/bin/:$PATH make
26+
PATH=~/code/postgresql/install/bin/:$PATH make install
27+
28+
cd $PG_XTM_DIR
29+
make clean
30+
make
31+
make install
32+
33+
cd dtmd
34+
make clean
35+
make
36+
rm -rf /tmp/clog/*
37+
./bin/dtmd &
38+
39+
cd $PG_DIR
40+
41+
./install/bin/initdb -D ./install/data1
42+
./install/bin/initdb -D ./install/data2
43+
./install/bin/initdb -D ./install/data3
44+
45+
46+
sed -i '' 's/#port =.*/port = 5433/' ./install/data2/postgresql.conf
47+
sed -i '' 's/#port =.*/port = 5434/' ./install/data3/postgresql.conf
48+
49+
sed -i '' "s/shared_preload_libraries =.*/shared_preload_libraries = 'pg_dtm,pg_shard'/" ./install/data1/postgresql.conf
50+
sed -i '' "s/shared_preload_libraries =.*/shared_preload_libraries = 'pg_dtm,pg_shard'/" ./install/data2/postgresql.conf
51+
sed -i '' "s/shared_preload_libraries =.*/shared_preload_libraries = 'pg_dtm,pg_shard'/" ./install/data3/postgresql.conf
52+
53+
sed -i '' 's/#fsync =.*/fsync = off/' ./install/data1/postgresql.conf
54+
sed -i '' 's/#fsync =.*/fsync = off/' ./install/data2/postgresql.conf
55+
sed -i '' 's/#fsync =.*/fsync = off/' ./install/data3/postgresql.conf
56+
57+
58+
echo 'pg_shard.use_dtm_transactions=1' >> ./install/data1/postgresql.conf
59+
echo 'pg_shard.use_dtm_transactions=1' >> ./install/data2/postgresql.conf
60+
echo 'pg_shard.use_dtm_transactions=1' >> ./install/data3/postgresql.conf
61+
62+
63+
64+
./install/bin/pg_ctl -D ./install/data1 -l ./install/data1/log start
65+
./install/bin/pg_ctl -D ./install/data2 -l ./install/data2/log start
66+
./install/bin/pg_ctl -D ./install/data3 -l ./install/data3/log start
67+
68+
69+
########################################################################
70+
# Configure pg_shard
71+
########################################################################
72+
73+
echo "127.0.0.1 5433" > ./install/data1/pg_worker_list.conf
74+
echo "127.0.0.1 5434" >> ./install/data1/pg_worker_list.conf
75+
76+
echo "127.0.0.1 5433" > ./install/data2/pg_worker_list.conf
77+
echo "127.0.0.1 5434" >> ./install/data2/pg_worker_list.conf
78+
79+
echo "127.0.0.1 5433" > ./install/data3/pg_worker_list.conf
80+
echo "127.0.0.1 5434" >> ./install/data3/pg_worker_list.conf
81+
82+
83+
./install/bin/createdb `whoami`
84+
./install/bin/createdb `whoami` -p5433
85+
./install/bin/createdb `whoami` -p5434
86+
87+
88+
./install/bin/psql -p 5433 << SQL
89+
CREATE EXTENSION pg_dtm;
90+
SQL
91+
92+
./install/bin/psql -p 5434 << SQL
93+
CREATE EXTENSION pg_dtm;
94+
SQL
95+
96+
./install/bin/psql << SQL
97+
98+
CREATE EXTENSION pg_dtm;
99+
CREATE EXTENSION pg_shard;
100+
CREATE TABLE t(u int, v int);
101+
SELECT master_create_distributed_table(table_name := 't', partition_column := 'u');
102+
SELECT master_create_worker_shards(table_name := 't', shard_count := 8, replication_factor := 1);
103+
insert into t values(1,10000);
104+
insert into t values(2,10000);
105+
insert into t values(3,10000);
106+
insert into t values(4,10000);
107+
insert into t values(5,10000);
108+
insert into t values(6,10000);
109+
insert into t values(7,10000);
110+
insert into t values(8,10000);
111+
112+
SQL
113+
114+
115+
116+
117+
# insert into t (select generate_series(0,10), random()::integer);
118+
119+
# cd contrib/pg_xtm/dtmd
120+
# make clean
121+
# make
122+
# rm -rf /tmp/clog/*
123+
# ./bin/dtmd
124+
125+
126+
127+
128+
129+
130+

0 commit comments

Comments
 (0)
pFad - Phonifier reborn

Pfad - The Proxy pFad of © 2024 Garber Painting. All rights reserved.

Note: This service is not intended for secure transactions such as banking, social media, email, or purchasing. Use at your own risk. We assume no liability whatsoever for broken pages.


Alternative Proxies:

Alternative Proxy

pFad Proxy

pFad v3 Proxy

pFad v4 Proxy