Merge branch 'xtm' of https://gitlab.postgrespro.ru/pgpro-dev/postgrespro into xtm

knizhnik · knizhnik · commit f7cf59c07b11 · 2015-10-14T18:16:14.000+03:00
diff --git a/contrib/pg_dtm/dtmd/include/limits.h b/contrib/pg_dtm/dtmd/include/limits.h
@@ -6,6 +6,6 @@
 
 #define BUFFER_SIZE (64 * 1024)
 #define LISTEN_QUEUE_SIZE 100
-#define MAX_STREAMS 128
+#define MAX_STREAMS 1024
 
 #endif
diff --git a/contrib/pg_dtm/dtmd/include/server.h b/contrib/pg_dtm/dtmd/include/server.h
@@ -2,6 +2,7 @@
 #define SERVER_H
 
 #include <stdbool.h>
+#include "int.h"
 
 /*
  * You should not want to know what is inside those structures.
@@ -95,6 +96,6 @@ bool client_message_finish(client_t client);
  *
  * Returns 'true' on success, 'false' otherwise.
  */
-bool client_message_shortcut(client_t client, long long arg);
+bool client_message_shortcut(client_t client, xid_t arg);
 
 #endif
diff --git a/contrib/pg_dtm/dtmd/src/clog.c b/contrib/pg_dtm/dtmd/src/clog.c
@@ -133,7 +133,7 @@ int clog_read(clog_t clog, xid_t xid) {
 bool clog_write(clog_t clog, xid_t xid, int status) {
 	clogfile_t *file = clog_xid_to_file(clog, xid);
 	if (!file) {
-		debug("xid %016llx out of range, creating the file\n", xid);
+		debug("xid %u out of range, creating the file\n", xid);
 		clogfile_t newfile;
 		if (!clogfile_open_by_id(&newfile, clog->datadir, XID_TO_FILEID(xid), true)) {
 			shout(
diff --git a/contrib/pg_dtm/dtmd/src/server.c b/contrib/pg_dtm/dtmd/src/server.c
@@ -325,7 +325,7 @@ bool client_message_finish(client_t client) {
 	return stream_message_finish(client->stream);
 }
 
-bool client_message_shortcut(client_t client, long long arg) {
+bool client_message_shortcut(client_t client, xid_t arg) {
 	if (!stream_message_start(client->stream, client->chan)) {
 		return false;
 	}
diff --git a/contrib/pg_dtm/libdtm.c b/contrib/pg_dtm/libdtm.c
@@ -39,8 +39,9 @@ static DTMConn DtmConnect(char *host, int port)
 	DTMConn dtm;
 	int sd;
 
-	if (strcmp(host, "localhost") == 0)
+	if (host == NULL)
 	{
+		// use a UNIX socket
 		struct sockaddr sock;
 		int len = offsetof(struct sockaddr, sa_data) + snprintf(sock.sa_data, sizeof(sock.sa_data), "%s/p%u", dtm_unix_sock_dir, port);
 		sock.sa_family = AF_UNIX;
@@ -62,6 +63,7 @@ static DTMConn DtmConnect(char *host, int port)
 	}
 	else
 	{
+		// use an IP socket
 		struct addrinfo *addrs = NULL;
 		struct addrinfo hint;
 		char portstr[6];
@@ -227,7 +229,9 @@ void DtmGlobalConfig(char *host, int port, char* sock_dir) {
 		free(dtmhost);
 		dtmhost = NULL;
 	}
-	dtmhost = strdup(host);
+	if (host) {
+		dtmhost = strdup(host);
+	}
 	dtmport = port;
 	dtm_unix_sock_dir = sock_dir;
 }
@@ -237,14 +241,14 @@ static DTMConn GetConnection()
 	static DTMConn dtm = NULL;
 	if (dtm == NULL)
 	{
-		if (dtmhost) {
-			dtm = DtmConnect(dtmhost, dtmport);
-			if (dtm == NULL)
-			{
-				elog(ERROR, "Failed to connect to DTMD %s:%d", dtmhost, dtmport);
+		dtm = DtmConnect(dtmhost, dtmport);
+		if (dtm == NULL)
+		{
+			if (dtmhost) {
+				elog(ERROR, "Failed to connect to DTMD at tcp %s:%d", dtmhost, dtmport);
+			} else {
+				elog(ERROR, "Failed to connect to DTMD at unix %d", dtmport);
 			}
-		} else {
-			/* elog(ERROR, "DTMD address not specified"); */
 		}
 	}
 	return dtm;
diff --git a/contrib/pg_dtm/pg_dtm.c b/contrib/pg_dtm/pg_dtm.c
@@ -654,9 +654,6 @@ static void DtmInitialize()
 		dtm->nReservedXids = 0;
         dtm->minXid = InvalidTransactionId;
         RegisterXactCallback(DtmXactCallback, NULL);
-        if (DtmBufferSize != 0) { 
-            RegisterBackgroundWorker(&DtmWorker);
-        }
 	}
 	LWLockRelease(AddinShmemInitLock);
 
@@ -793,7 +790,13 @@ _PG_init(void)
 		NULL
 	);
 
-	DtmGlobalConfig(DtmHost, DtmPort, Unix_socket_directories);
+
+	if (DtmBufferSize != 0) { 
+		DtmGlobalConfig(NULL, DtmPort, Unix_socket_directories);
+		RegisterBackgroundWorker(&DtmWorker);
+	} else {
+		DtmGlobalConfig(DtmHost, DtmPort, Unix_socket_directories);
+	}
 
 	/*
 	 * Install hooks.
@@ -899,9 +902,6 @@ void DtmBackgroundWorker(Datum arg)
     params.file = unix_sock_path;
     params.buffer_size = DtmBufferSize;
     
-	DtmGlobalConfig("localhost", DtmPort, Unix_socket_directories);
-
     ShubInitialize(&shub, &params);
-    
     ShubLoop(&shub);
 }
diff --git a/contrib/pg_dtm/tests/daemons.go b/contrib/pg_dtm/tests/daemons.go
@@ -91,6 +91,7 @@ func postgres(bin string, datadir string, port int, nodeid int, wg *sync.WaitGro
 		bin,
 		"-D", datadir,
 		"-p", strconv.Itoa(port),
+		"-c", "dtm.buffer_size=65536",
 		"-c", "dtm.host=127.0.0.1",
 		"-c", "dtm.port=" + strconv.Itoa(5431),
 		"-c", "autovacuum=off",
diff --git a/contrib/pg_dtm/tests/deploy_layouts/cluster.yml b/contrib/pg_dtm/tests/deploy_layouts/cluster.yml
@@ -1,5 +1,5 @@
 ---
-- hosts: master-workers
+- hosts: dtm
   roles:
   - role: postgrespro
     deploy_dtm: true
@@ -10,5 +10,5 @@
     pg_port: 15432
     deploy_postgres: true
     pg_dtm_enable: true
-    pg_dtm_host: "{{groups['master-workers'][0]}}"
+    pg_dtm_host: "{{ groups['dtm'][0] }}"
 
diff --git a/contrib/pg_dtm/tests/deploy_layouts/cluster_nodtm.yml b/contrib/pg_dtm/tests/deploy_layouts/cluster_nodtm.yml
diff --git a/contrib/pg_dtm/tests/deploy_layouts/cluster_pg_shard.yml b/contrib/pg_dtm/tests/deploy_layouts/cluster_pg_shard.yml
@@ -0,0 +1,15 @@
+---
+- hosts: master-workers
+  roles:
+  - role: postgrespro
+    deploy_dtm: true
+
+- hosts: master-workers:workers
+  roles:
+  - role: postgrespro
+    pg_port: 15432
+    deploy_postgres: true
+    deploy_pg_shard: true
+    pg_dtm_enable: true
+    pg_dtm_host: "{{ groups['master-workers'][0] }}"
+
diff --git a/contrib/pg_dtm/tests/deploy_layouts/cluster_pg_shard_nodtm.yml b/contrib/pg_dtm/tests/deploy_layouts/cluster_pg_shard_nodtm.yml
diff --git a/contrib/pg_dtm/tests/farms/localhost b/contrib/pg_dtm/tests/farms/localhost
@@ -0,0 +1,2 @@
+[master-workers]
+localhost
diff --git a/contrib/pg_dtm/tests/farms/mephi b/contrib/pg_dtm/tests/farms/mephi
@@ -1,15 +1,15 @@
-[client]
+[clients]
 br.theor.mephi.ru
+blade3 ansible_ssh_host=85.143.113.31 ansible_ssh_port=4203
 
 [workers]
-blade3 ansible_ssh_host=85.143.113.31 ansible_ssh_port=4203
 blade4 ansible_ssh_host=85.143.113.31 ansible_ssh_port=4204
 blade5 ansible_ssh_host=85.143.113.31 ansible_ssh_port=4205
 blade6 ansible_ssh_host=85.143.113.31 ansible_ssh_port=4206
 blade7 ansible_ssh_host=85.143.113.31 ansible_ssh_port=4207
 blade8 ansible_ssh_host=85.143.113.31 ansible_ssh_port=4208
 
-[master-workers]
+[dtm]
 blade8 ansible_ssh_host=85.143.113.31 ansible_ssh_port=4208
 
 
diff --git a/contrib/pg_dtm/tests/farms/sai b/contrib/pg_dtm/tests/farms/sai
@@ -1,9 +1,12 @@
+[clients]
+158.250.29.4 ansible_ssh_user=s.kelvich
+158.250.29.5 ansible_ssh_user=s.kelvich
+
 [workers]
-astro4 ansible_ssh_host=158.250.29.4 ansible_ssh_user=s.kelvich
-astro5 ansible_ssh_host=158.250.29.5 ansible_ssh_user=s.kelvich
-astro6 ansible_ssh_host=158.250.29.6 ansible_ssh_user=s.kelvich
-astro8 ansible_ssh_host=158.250.29.8 ansible_ssh_user=s.kelvich
-astro9 ansible_ssh_host=158.250.29.9 ansible_ssh_user=s.kelvich
+158.250.29.6 ansible_ssh_user=s.kelvich
+158.250.29.8 ansible_ssh_user=s.kelvich
+158.250.29.9 ansible_ssh_user=s.kelvich
+158.250.29.10 ansible_ssh_user=s.kelvich
 
-[master-workers]
-astro10 ansible_ssh_host=158.250.29.10 ansible_ssh_user=s.kelvich
+[dtm]
+158.250.29.10 ansible_ssh_user=s.kelvich
diff --git a/contrib/pg_dtm/tests/transfers-remote.sh b/contrib/pg_dtm/tests/transfers-remote.sh
@@ -1,10 +1,15 @@
 #!/bin/sh
 
+FARM="sai"
+WORKERS=3
+TRANS_SRV="s.kelvich@158.250.29.10"
+
+
 if [ ! -f ./transfers.linux ]; then
-    GOOS=linux GOARCH=amd64 go build -o ./transfers ./transfers.go
+    GOOS=linux GOARCH=amd64 go build -o ./transfers.linux ./transfers.go
 fi
 
-ansible master-workers -i farms/mephi -m copy -a "src=transfers dest=~/transfers mode=a+x"
+ansible master-workers -i farms/$FARM -m copy -a "src=transfers.linux dest=~/transfers mode=a+x"
 
 
 
@@ -18,3 +23,17 @@ ssh -p4207 br.theor.mephi.ru "./transfers -d 'host=blade8 dbname=postgres user=s
 
 
 
+ssh s.kelvich@158.250.29.10 "./transfers -d 'host=158.250.29.9 port=15432 dbname=postgres' -d 'host=158.250.29.8 port=15432 dbname=postgres'  -d 'host=158.250.29.4 port=15432 dbname=postgres'  -d 'host=158.250.29.5 port=15432 dbname=postgres'  -d 'host=158.250.29.6 port=15432 dbname=postgres'  -g -w 64"
+
+
+ssh s.kelvich@158.250.29.10 "./transfers -d 'host=158.250.29.9 user=s.kelvich port=15432 dbname=postgres'  -g -w 128"
+
+ssh s.kelvich@158.250.29.10 "./transfers -d 'host=158.250.29.9 user=s.kelvich port=15432 dbname=postgres' -d 'host=158.250.29.8 user=s.kelvich port=15432 dbname=postgres'  -g -w 128"
+
+ssh s.kelvich@158.250.29.10 "./transfers -d 'host=158.250.29.9 user=s.kelvich port=15432 dbname=postgres' -d 'host=158.250.29.8 user=s.kelvich port=15432 dbname=postgres' -d 'host=158.250.29.6 user=s.kelvich port=15432 dbname=postgres' -d 'host=158.250.29.5 user=s.kelvich port=15432 dbname=postgres' -d 'host=158.250.29.4 user=s.kelvich port=15432 dbname=postgres'    -g -w 128"
+
+
+
+
+./transfers -d 'host=127.0.0.1 port=5432 dbname=postgres' -d 'host=127.0.0.1 port=5433 dbname=postgres' -g -w 128
+
diff --git a/contrib/pg_dtm/tests/transfers.go b/contrib/pg_dtm/tests/transfers.go
@@ -32,6 +32,9 @@ var cfg struct {
 
     Verbose bool
     UseDtm bool
+    InitOnly bool
+    SkipInit bool
+
     Isolation string // "repeatable read" or "read committed"
 
     Accounts struct {
@@ -108,6 +111,8 @@ func init() {
     flag.BoolVar(&cfg.Writers.AllowLocal, "l", false, "Allow local updates")
     flag.BoolVar(&cfg.Writers.PrivateRows, "p", false, "Private rows (avoid waits/aborts caused by concurrent updates of the same rows)")
     flag.BoolVar(&cfg.Writers.UseCursors, "c", false, "Use cursors for updates")
+    flag.BoolVar(&cfg.InitOnly, "f", false, "Only feed databses with data")
+    flag.BoolVar(&cfg.SkipInit, "s", false, "Skip init phase")
     flag.Parse()
 
     if len(cfg.ConnStrs) == 0 {
@@ -141,8 +146,15 @@ func init() {
 
 func main() {
     start := time.Now()
-    prepare(cfg.ConnStrs)
-    fmt.Printf("database prepared in %0.2f seconds\n", time.Since(start).Seconds())
+
+    if (!cfg.SkipInit){
+        prepare(cfg.ConnStrs)
+        fmt.Printf("database prepared in %0.2f seconds\n", time.Since(start).Seconds())
+    }
+
+    if (cfg.InitOnly) {
+        return
+    }
 
     var writerWg sync.WaitGroup
     var readerWg sync.WaitGroup
diff --git a/contrib/pg_dtm/tests/transfers.yml b/contrib/pg_dtm/tests/transfers.yml
@@ -0,0 +1,23 @@
+---
+
+- hosts: masters
+  gather_facts: no
+  tasks:
+  - name: copy transfers binary
+    copy: src=transfers.linux dest=~/transfers mode=a+x
+
+- hosts: 158.250.29.4
+  gather_facts: no
+  tasks:
+  - name: setup the databases
+    shell: "./transfers -d 'host=158.250.29.10 user=s.kelvich port=15432 dbname=postgres' -d 'host=158.250.29.9 user=s.kelvich port=15432 dbname=postgres' -d 'host=158.250.29.8 user=s.kelvich port=15432 dbname=postgres' -d 'host=158.250.29.5 user=s.kelvich port=15432 dbname=postgres' -f -g -w 200 -r 0"
+    register: transfers_result
+  - debug: var=transfers_result
+
+- hosts: masters
+  gather_facts: no
+  tasks:
+  - name: run transfers
+    shell: "./transfers -d 'host=158.250.29.10 user=s.kelvich port=15432 dbname=postgres' -d 'host=158.250.29.9 user=s.kelvich port=15432 dbname=postgres' -d 'host=158.250.29.8 user=s.kelvich port=15432 dbname=postgres' -d 'host=158.250.29.5 user=s.kelvich port=15432 dbname=postgres' -s -g -w 200 -r 0"
+    register: transfers_result
+  - debug: var=transfers_result

Original file line number	Diff line number	Diff line change
`@@ -325,7 +325,7 @@ bool client_message_finish(client_t client) {`
`325`	`325`	`return stream_message_finish(client->stream);`
`326`	`326`	`}`
`327`	`327`
`328`		`-bool client_message_shortcut(client_t client, long long arg) {`
	`328`	`+bool client_message_shortcut(client_t client, xid_t arg) {`
`329`	`329`	`if (!stream_message_start(client->stream, client->chan)) {`
`330`	`330`	`return false;`
`331`	`331`	`}`
Original file line number	Diff line number	Diff line change
`@@ -39,8 +39,9 @@ static DTMConn DtmConnect(char *host, int port)`
`39`	`39`	`DTMConn dtm;`
`40`	`40`	`int sd;`
`41`	`41`
`42`		`- if (strcmp(host, "localhost") == 0)`
	`42`	`+ if (host == NULL)`
`43`	`43`	`{`
	`44`	`+ // use a UNIX socket`
`44`	`45`	`struct sockaddr sock;`
`45`	`46`	`int len = offsetof(struct sockaddr, sa_data) + snprintf(sock.sa_data, sizeof(sock.sa_data), "%s/p%u", dtm_unix_sock_dir, port);`
`46`	`47`	`sock.sa_family = AF_UNIX;`
`@@ -62,6 +63,7 @@ static DTMConn DtmConnect(char *host, int port)`
`62`	`63`	`}`
`63`	`64`	`else`
`64`	`65`	`{`
	`66`	`+ // use an IP socket`
`65`	`67`	`struct addrinfo *addrs = NULL;`
`66`	`68`	`struct addrinfo hint;`
`67`	`69`	`char portstr[6];`
`@@ -227,7 +229,9 @@ void DtmGlobalConfig(char host, int port, char sock_dir) {`
`227`	`229`	`free(dtmhost);`
`228`	`230`	`dtmhost = NULL;`
`229`	`231`	`}`
`230`		`- dtmhost = strdup(host);`
	`232`	`+ if (host) {`
	`233`	`+ dtmhost = strdup(host);`
	`234`	`+ }`
`231`	`235`	`dtmport = port;`
`232`	`236`	`dtm_unix_sock_dir = sock_dir;`
`233`	`237`	`}`
`@@ -237,14 +241,14 @@ static DTMConn GetConnection()`
`237`	`241`	`static DTMConn dtm = NULL;`
`238`	`242`	`if (dtm == NULL)`
`239`	`243`	`{`
`240`		`- if (dtmhost) {`
`241`		`- dtm = DtmConnect(dtmhost, dtmport);`
`242`		`- if (dtm == NULL)`
`243`		`- {`
`244`		`- elog(ERROR, "Failed to connect to DTMD %s:%d", dtmhost, dtmport);`
	`244`	`+ dtm = DtmConnect(dtmhost, dtmport);`
	`245`	`+ if (dtm == NULL)`
	`246`	`+ {`
	`247`	`+ if (dtmhost) {`
	`248`	`+ elog(ERROR, "Failed to connect to DTMD at tcp %s:%d", dtmhost, dtmport);`
	`249`	`+ } else {`
	`250`	`+ elog(ERROR, "Failed to connect to DTMD at unix %d", dtmport);`
`245`	`251`	`}`
`246`		`- } else {`
`247`		`- /* elog(ERROR, "DTMD address not specified"); */`
`248`	`252`	`}`
`249`	`253`	`}`
`250`	`254`	`return dtm;`