Perform select with timeout in arbiter

knizhnik · knizhnik · commit 9cbf9b503f11 · 2016-03-04T23:27:45.000+03:00
diff --git a/contrib/mmts/arbiter.c b/contrib/mmts/arbiter.c
@@ -297,8 +297,7 @@ static int MtmConnectSocket(char const* host, int port, int max_attempts)
 				
 			/* Some node considered that I am dead, so switch to recovery mode */
 			if (BIT_CHECK(msg.disabledNodeMask, MtmNodeId-1)) { 
-				elog(WARNING, "Node is switched to recovery mode");
-				ds->status = MTM_RECOVERY;
+				MtmClusterSwitchMode(MTM_RECOVERY);
 			}
 			/* Combine disable masks from all node. Is it actually correct or we should better check availability of nodes ourselves? */
 			ds->disabledNodeMask |= msg.disabledNodeMask;
@@ -345,8 +344,7 @@ static void MtmOpenConnections()
 		elog(WARNING, "Node is out of quorum: only %d nodes from %d are accssible", ds->nNodes, MtmNodes);
 		ds->status = MTM_OFFLINE;
 	} else if (ds->status == MTM_INITIALIZATION) { 
-		elog(WARNING, "Switch to CONNECTED mode");
-		ds->status = MTM_CONNECTED;
+		MtmClusterSwitchMode(MTM_CONNECTED);
 	}
 }
 
@@ -403,9 +401,9 @@ static void MtmAcceptOneConnection()
 			} else { 
 				elog(NOTICE, "Arbiter established connection with node %d", msg.node); 
 				BIT_CLEAR(ds->connectivityMask, msg.node-1);
-				MtmOnNodeConnect(msg.node);
 				MtmRegisterSocket(fd, msg.node-1);
 				sockets[msg.node-1] = fd;
+				MtmOnNodeConnect(msg.node);
 			}
 		}
 	}
@@ -586,7 +584,7 @@ static void MtmTransReceiver(Datum arg)
 
 	while (true) {
 #if USE_EPOLL
-        n = epoll_wait(epollfd, events, nNodes, -1);
+        n = epoll_wait(epollfd, events, nNodes, MtmKeepaliveTimeout/1000);
 		if (n < 0) { 
 			elog(ERROR, "Arbiter failed to poll sockets: %d", errno);
 		}
@@ -600,9 +598,12 @@ static void MtmTransReceiver(Datum arg)
 #else
         fd_set events;
 		do { 
+			struct timeval tv;
 			events = inset;
-			rc = select(max_fd+1, &events, NULL, NULL, NULL);
-		} while (rc < 0 && MtmRecovery());
+			tv.tv_sec = MtmKeepAliveTimeout/USEC;
+			tv.tv_usec = MtmKeepAliveTimeout%USEC;
+			n = select(max_fd+1, &events, NULL, NULL, &tv);
+		} while (n < 0 && MtmRecovery());
 		
 		if (rc < 0) { 
 			elog(ERROR, "Arbiter failed to select sockets: %d", errno);
@@ -742,6 +743,10 @@ static void MtmTransReceiver(Datum arg)
 				}
 			}
 		}
+		if (n == 0 && ds->disabledNodeMask != 0) { 
+			/* If timeout is expired and there are didabled nodes, then recheck cluster's state */
+			MtmUpdateClusterStatus();
+		}
 	}
 }
 
diff --git a/contrib/mmts/multimaster.c b/contrib/mmts/multimaster.c
@@ -77,7 +77,6 @@ typedef enum
 
 #define MTM_SHMEM_SIZE (64*1024*1024)
 #define MTM_HASH_SIZE  100003
-#define USEC 1000000
 #define MIN_WAIT_TIMEOUT 1000
 #define MAX_WAIT_TIMEOUT 100000
 #define STATUS_POLL_DELAY USEC
@@ -132,6 +131,15 @@ static TransactionManager MtmTM = {
 	MtmGetName 
 };
 
+static char const* const MtmNodeStatusMnem[] = 
+{ 
+	"Intialization", 
+	"Offline", 
+	"Connected",
+	"Online",
+	"Recovery"
+};
+
 bool  MtmDoReplication;
 char* MtmDatabaseName;
 
@@ -509,8 +517,9 @@ MtmBeginTransaction(MtmCurrentTrans* x)
         x->isReplicated = false;
         x->isDistributed = MtmIsUserTransaction();
 		if (x->isDistributed && dtm->status != MTM_ONLINE) { 
-			MtmUnlock();
-			elog(ERROR, "Multimaster node is offline");
+			/* reject all user's transactions at offline cluster */
+			MtmUnlock();			
+			elog(ERROR, "Multimaster node is not online");
 		}
 		x->containsDML = false;
 		x->isPrepared = false;
@@ -594,6 +603,9 @@ static void MtmPrecommitTransaction(MtmCurrentTrans* x)
 			
 	if (dtm->disabledNodeMask != 0) { 
 		MtmUpdateClusterStatus();
+		if (dtm->status != MTM_ONLINE) { 
+			elog(ERROR, "Abort current transaction because this cluster node is not online");			
+		}
 	}
 
 	MtmLock(LW_EXCLUSIVE);
@@ -1084,21 +1096,13 @@ _PG_fini(void)
  */
 
 
-void MtmSwitchToNormalMode()
-{
-	dtm->status = MTM_ONLINE;
-	elog(WARNING, "Switch to normal mode");
-	/* ??? Something else to do here? */
-}
-
-void MtmSwitchToRecoveryMode()
+void MtmClusterSwitchMode(MtmNodeStatus mode)
 {
-	dtm->status = MTM_RECOVERY;
+	dtm->status = mode;
+	elog(WARNING, "Switch to %s mode", MtmNodeStatusMnem[mode]);
 	/* ??? Something else to do here? */
-	elog(ERROR, "Switch to normal mode");
 }
 
-
 void MtmJoinTransaction(GlobalTransactionId* gtid, csn_t globalSnapshot)
 {
 	csn_t localSnapshot;
@@ -1117,7 +1121,7 @@ void MtmJoinTransaction(GlobalTransactionId* gtid, csn_t globalSnapshot)
 		Assert(dtm->status == MTM_RECOVERY);
 	} else if (dtm->status == MTM_RECOVERY) { 
 		/* When recovery is completed we get normal transaction ID and switch to normal mode */
-		MtmSwitchToNormalMode();
+		MtmClusterSwitchMode(MTM_ONLINE);
 	}
 	dtmTx.gtid = *gtid;
 	dtmTx.xid = GetCurrentTransactionId();
@@ -1670,7 +1674,10 @@ MtmBuildConnectivityMatrix(nodemask_t* matrix)
 	}
 }	
 
-
+/**
+ * Build connectivity graph, find clique in it and extend disabledNodeMask by nodes not included in clique.
+ * This function returns false if current node is excluded from cluster, true otherwise
+ */
 void MtmUpdateClusterStatus(void)
 {
 	nodemask_t mask, clique;
@@ -1683,17 +1690,29 @@ void MtmUpdateClusterStatus(void)
 	clique = MtmFindMaxClique(matrix, MtmNodes, &clique_size);
 	if (clique_size >= MtmNodes/2+1) { /* have quorum */
 		MtmLock(LW_EXCLUSIVE);
-		mask = ~clique & (((nodemask_t)1 << MtmNodes)-1) & ~dtm->disabledNodeMask;
+		mask = ~clique & (((nodemask_t)1 << MtmNodes)-1) & ~dtm->disabledNodeMask; /* new disabled nodes mask */
 		for (i = 0; mask != 0; i++, mask >>= 1) {
 			if (mask & 1) { 
 				dtm->nNodes -= 1;
 				BIT_SET(dtm->disabledNodeMask, i);
 			}
 		}
+		mask = clique & dtm->disabledNodeMask; /* new enabled nodes mask */		
+		for (i = 0; mask != 0; i++, mask >>= 1) {
+			if (mask & 1) { 
+				dtm->nNodes += 1;
+				BIT_CLEAR(dtm->disabledNodeMask, i);
+			}
+		}
 		MtmUnlock();
 		if (BIT_CHECK(dtm->disabledNodeMask, MtmNodeId-1)) { 
-            /* I was excluded from cluster:( */
-			MtmSwitchToRecoveryMode();
+			if (dtm->status == MTM_ONLINE) {
+				/* I was excluded from cluster:( */
+				MtmClusterSwitchMode(MTM_OFFLINE);
+			}
+		} else if (dtm->status == MTM_OFFLINE) {
+			/* Should we somehow restart logical receivers? */ 
+			MtmClusterSwitchMode(MTM_RECOVERY);
 		}
 	} else { 
 		elog(WARNING, "Clique %lx has no quorum", clique);
diff --git a/contrib/mmts/multimaster.h b/contrib/mmts/multimaster.h
@@ -20,6 +20,8 @@
 #define MULTIMASTER_MIN_PROTO_VERSION   1
 #define MULTIMASTER_MAX_PROTO_VERSION   1
 
+#define USEC 1000000
+
 #define Natts_mtm_ddl_log 2
 #define Anum_mtm_ddl_log_issued		1
 #define Anum_mtm_ddl_log_query		2
@@ -124,6 +126,7 @@ extern char* MtmDatabaseName;
 extern int   MtmConnectAttempts;
 extern int   MtmConnectTimeout;
 extern int   MtmReconnectAttempts;
+extern int   MtmKeepaliveTimeout;
 
 extern void  MtmArbiterInitialize(void);
 extern int   MtmStartReceivers(char* nodes, int nodeId);
@@ -148,6 +151,5 @@ extern timestamp_t MtmGetCurrentTime(void);
 extern void  MtmSleep(timestamp_t interval);
 extern bool  MtmIsRecoveredNode(int nodeId);
 extern void  MtmUpdateClusterStatus(void);
-extern void MtmSwitchToNormalMode(void);
-extern void MtmSwitchToRecoveryMode(void);
+extern void  MtmClusterSwitchMode(MtmNodeStatus mode);
 #endif

Original file line number	Diff line number	Diff line change
`@@ -297,8 +297,7 @@ static int MtmConnectSocket(char const* host, int port, int max_attempts)`
`297`	`297`
`298`	`298`	`/* Some node considered that I am dead, so switch to recovery mode */`
`299`	`299`	`if (BIT_CHECK(msg.disabledNodeMask, MtmNodeId-1)) {`
`300`		`- elog(WARNING, "Node is switched to recovery mode");`
`301`		`- ds->status = MTM_RECOVERY;`
	`300`	`+ MtmClusterSwitchMode(MTM_RECOVERY);`
`302`	`301`	`}`
`303`	`302`	`/* Combine disable masks from all node. Is it actually correct or we should better check availability of nodes ourselves? */`
`304`	`303`	`ds->disabledNodeMask \|= msg.disabledNodeMask;`
`@@ -345,8 +344,7 @@ static void MtmOpenConnections()`
`345`	`344`	`elog(WARNING, "Node is out of quorum: only %d nodes from %d are accssible", ds->nNodes, MtmNodes);`
`346`	`345`	`ds->status = MTM_OFFLINE;`
`347`	`346`	`} else if (ds->status == MTM_INITIALIZATION) {`
`348`		`- elog(WARNING, "Switch to CONNECTED mode");`
`349`		`- ds->status = MTM_CONNECTED;`
	`347`	`+ MtmClusterSwitchMode(MTM_CONNECTED);`
`350`	`348`	`}`
`351`	`349`	`}`
`352`	`350`
`@@ -403,9 +401,9 @@ static void MtmAcceptOneConnection()`
`403`	`401`	`} else {`
`404`	`402`	`elog(NOTICE, "Arbiter established connection with node %d", msg.node);`
`405`	`403`	`BIT_CLEAR(ds->connectivityMask, msg.node-1);`
`406`		`- MtmOnNodeConnect(msg.node);`
`407`	`404`	`MtmRegisterSocket(fd, msg.node-1);`
`408`	`405`	`sockets[msg.node-1] = fd;`
	`406`	`+ MtmOnNodeConnect(msg.node);`
`409`	`407`	`}`
`410`	`408`	`}`
`411`	`409`	`}`
`@@ -586,7 +584,7 @@ static void MtmTransReceiver(Datum arg)`
`586`	`584`
`587`	`585`	`while (true) {`
`588`	`586`	`#if USE_EPOLL`
`589`		`- n = epoll_wait(epollfd, events, nNodes, -1);`
	`587`	`+ n = epoll_wait(epollfd, events, nNodes, MtmKeepaliveTimeout/1000);`
`590`	`588`	`if (n < 0) {`
`591`	`589`	`elog(ERROR, "Arbiter failed to poll sockets: %d", errno);`
`592`	`590`	`}`
`@@ -600,9 +598,12 @@ static void MtmTransReceiver(Datum arg)`
`600`	`598`	`#else`
`601`	`599`	`fd_set events;`
`602`	`600`	`do {`
	`601`	`+ struct timeval tv;`
`603`	`602`	`events = inset;`
`604`		`- rc = select(max_fd+1, &events, NULL, NULL, NULL);`
`605`		`- } while (rc < 0 && MtmRecovery());`
	`603`	`+ tv.tv_sec = MtmKeepAliveTimeout/USEC;`
	`604`	`+ tv.tv_usec = MtmKeepAliveTimeout%USEC;`
	`605`	`+ n = select(max_fd+1, &events, NULL, NULL, &tv);`
	`606`	`+ } while (n < 0 && MtmRecovery());`
`606`	`607`
`607`	`608`	`if (rc < 0) {`
`608`	`609`	`elog(ERROR, "Arbiter failed to select sockets: %d", errno);`
`@@ -742,6 +743,10 @@ static void MtmTransReceiver(Datum arg)`
`742`	`743`	`}`
`743`	`744`	`}`
`744`	`745`	`}`
	`746`	`+ if (n == 0 && ds->disabledNodeMask != 0) {`
	`747`	`+ /* If timeout is expired and there are didabled nodes, then recheck cluster's state */`
	`748`	`+ MtmUpdateClusterStatus();`
	`749`	`+ }`
`745`	`750`	`}`
`746`	`751`	`}`
`747`	`752`