Avoid writes in critical section

knizhnik · kelvich · commit fa309d441fac · 2017-11-13T00:17:06.000+03:00
diff --git a/arbiter.c b/arbiter.c
@@ -74,9 +74,9 @@
 
 #include "multimaster.h"
 
-#define MAX_ROUTES      16
-#define BUFFER_SIZE     1024
-#define HANDSHAKE_MAGIC 0xCAFEDEED
+#define MAX_ROUTES       16
+#define INIT_BUFFER_SIZE 1024
+#define HANDSHAKE_MAGIC  0xCAFEDEED
 
 typedef struct
 {
@@ -98,7 +98,8 @@ typedef struct
 typedef struct 
 {
 	int used;
-	MtmArbiterMessage data[BUFFER_SIZE];
+	int size;
+	MtmArbiterMessage* data;
 } MtmBuffer;
 
 static int*      sockets;
@@ -450,11 +451,14 @@ static int MtmConnectSocket(char const* host, int port, int max_attempts)
 	}
 	
 	/* Some node considered that I am dead, so switch to recovery mode */
+	MtmLock(LW_EXCLUSIVE);
 	if (BIT_CHECK(resp.disabledNodeMask, MtmNodeId-1)) { 
 		elog(WARNING, "Node %d thinks that I was dead", resp.node);
 		BIT_SET(Mtm->disabledNodeMask, MtmNodeId-1);
 		MtmSwitchClusterMode(MTM_RECOVERY);
 	}
+	MtmUnlock();
+
 	return sd;
 }
 
@@ -491,7 +495,9 @@ static bool MtmSendToNode(int node, void const* buf, int size)
 	while (true) {
 		if (sockets[node] >= 0 && BIT_CHECK(Mtm->reconnectMask, node)) {
 			elog(WARNING, "Arbiter is forced to reconnect to node %d", node+1); 
+			MtmLock(LW_EXCLUSIVE);		
 			BIT_CLEAR(Mtm->reconnectMask, node);
+			MtmUnlock();
 			close(sockets[node]);
 			sockets[node] = -1;
 		}
@@ -506,7 +512,9 @@ static bool MtmSendToNode(int node, void const* buf, int size)
 				MtmOnNodeDisconnect(node+1);
 				return false;
 			}
+			MtmLock(LW_EXCLUSIVE);		
 			BIT_CLEAR(Mtm->reconnectMask, node);
+			MtmUnlock();
 			MTM_LOG3("Arbiter restablished connection with node %d", node+1);
 		} else { 
 			return true;
@@ -552,7 +560,6 @@ static void MtmAcceptOneConnection()
 				close(fd);
 			} else { 
 				MTM_LOG1("Arbiter established connection with node %d", req.hdr.node); 
-				BIT_CLEAR(Mtm->connectivityMask, req.hdr.node-1);
 				MtmRegisterSocket(fd, req.hdr.node-1);
 				sockets[req.hdr.node-1] = fd;
 				MtmOnNodeConnect(req.hdr.node);
@@ -598,12 +605,9 @@ static void MtmAcceptIncomingConnections()
 static void MtmAppendBuffer(MtmBuffer* txBuffer, TransactionId xid, int node, MtmTransState* ts)
 {
 	MtmBuffer* buf = &txBuffer[node];
-	if (buf->used == BUFFER_SIZE) { 
-		if (!MtmSendToNode(node, buf->data, buf->used*sizeof(MtmArbiterMessage))) {			
-			buf->used = 0;
-			return;
-		}
-		buf->used = 0;
+	if (buf->used == buf->size) {
+		buf->size = buf->size ? buf->size*2 : INIT_BUFFER_SIZE;
+		buf->data = repalloc(buf->data, buf->size * sizeof(MtmArbiterMessage));
 	}
 	buf->data[buf->used].dxid = xid;
 
@@ -640,7 +644,7 @@ static void MtmTransSender(Datum arg)
 	int nNodes = MtmMaxNodes;
 	int i;
 
-	MtmBuffer* txBuffer = (MtmBuffer*)palloc(sizeof(MtmBuffer)*nNodes);
+	MtmBuffer* txBuffer = (MtmBuffer*)palloc0(sizeof(MtmBuffer)*nNodes);
 
 	InitializeTimeouts();
 
@@ -655,10 +659,6 @@ static void MtmTransSender(Datum arg)
 
 	MtmOpenConnections();
 
-	for (i = 0; i < nNodes; i++) { 
-		txBuffer[i].used = 0;
-	}
-
 	while (!stop) {
 		MtmTransState* ts;		
 		PGSemaphoreLock(&Mtm->votingSemaphore);
@@ -723,7 +723,7 @@ static void MtmTransReceiver(Datum arg)
 	int nNodes = MtmMaxNodes;
 	int nResponses;
 	int i, j, n, rc;
-	MtmBuffer* rxBuffer = (MtmBuffer*)palloc(sizeof(MtmBuffer)*nNodes);
+	MtmBuffer* rxBuffer = (MtmBuffer*)palloc0(sizeof(MtmBuffer)*nNodes);
 	timestamp_t lastHeartbeatCheck = MtmGetSystemTime();
 	timestamp_t now;
 
@@ -744,7 +744,8 @@ static void MtmTransReceiver(Datum arg)
 	MtmAcceptIncomingConnections();
 
 	for (i = 0; i < nNodes; i++) { 
-		rxBuffer[i].used = 0;
+		rxBuffer[i].size = INIT_BUFFER_SIZE;
+		rxBuffer[i].data = palloc(INIT_BUFFER_SIZE*sizeof(MtmArbiterMessage));
 	}
 
 	while (!stop) {
@@ -788,7 +789,7 @@ static void MtmTransReceiver(Datum arg)
 					continue;
 				}  
 				
-				rc = MtmReadFromNode(i, (char*)rxBuffer[i].data + rxBuffer[i].used, BUFFER_SIZE-rxBuffer[i].used);
+				rc = MtmReadFromNode(i, (char*)rxBuffer[i].data + rxBuffer[i].used, rxBuffer[i].size-rxBuffer[i].used);
 				if (rc <= 0) { 
 					continue;
 				}
diff --git a/multimaster.c b/multimaster.c
@@ -1432,8 +1432,11 @@ void MtmOnNodeDisconnect(int nodeId)
 		/* Avoid false detection of node failure and prevent node status blinking */
 		return;
 	}
+	MtmLock(LW_EXCLUSIVE);
 	BIT_SET(Mtm->connectivityMask, nodeId-1);
 	BIT_SET(Mtm->reconnectMask, nodeId-1);
+	MtmUnlock();
+
 	RaftableSet(psprintf("node-mask-%d", MtmNodeId), &Mtm->connectivityMask, sizeof Mtm->connectivityMask, false);
 
 	MtmSleep(MSEC_TO_USEC(MtmRaftPollDelay));
@@ -1460,7 +1463,10 @@ void MtmOnNodeDisconnect(int nodeId)
 
 void MtmOnNodeConnect(int nodeId)
 {
+	MtmLock(LW_EXCLUSIVE);	
 	BIT_CLEAR(Mtm->connectivityMask, nodeId-1);
+	MtmUnlock();
+
 	MTM_LOG1("Reconnect node %d", nodeId);
 	RaftableSet(psprintf("node-mask-%d", MtmNodeId), &Mtm->connectivityMask, sizeof Mtm->connectivityMask, false);
 }
@@ -2194,6 +2200,8 @@ void MtmRecoverNode(int nodeId)
 	
 void MtmDropNode(int nodeId, bool dropSlot)
 {
+	MtmLock(LW_EXCLUSIVE);
+
 	if (!BIT_CHECK(Mtm->disabledNodeMask, nodeId-1))
 	{
 		if (nodeId <= 0 || nodeId > Mtm->nLiveNodes) 
@@ -2211,6 +2219,8 @@ void MtmDropNode(int nodeId, bool dropSlot)
 			ReplicationSlotDrop(psprintf(MULTIMASTER_SLOT_PATTERN, nodeId));
 		}		
 	}
+
+	MtmUnlock();
 }
 static void
 MtmOnProcExit(int code, Datum arg)
@@ -2401,7 +2411,9 @@ mtm_add_node(PG_FUNCTION_ARGS)
 	} 
 	else 
 	{ 
-		int nodeId = Mtm->nAllNodes;
+		int nodeId;
+		MtmLock(LW_EXCLUSIVE);	
+		nodeId = Mtm->nAllNodes;
 		elog(NOTICE, "Add node %d: '%s'", nodeId+1, connStr);
 		MtmUpdateNodeConnectionInfo(&Mtm->nodes[nodeId].con, connStr);
 		Mtm->nodes[nodeId].transDelay = 0;
@@ -2411,6 +2423,8 @@ mtm_add_node(PG_FUNCTION_ARGS)
 
 		BIT_SET(Mtm->disabledNodeMask, nodeId);
 		Mtm->nAllNodes += 1;
+		MtmUnlock();
+
 		MtmStartReceiver(nodeId+1, true);
 	}
     PG_RETURN_VOID();

Original file line number	Diff line number	Diff line change
`@@ -1432,8 +1432,11 @@ void MtmOnNodeDisconnect(int nodeId)`
`1432`	`1432`	`/* Avoid false detection of node failure and prevent node status blinking */`
`1433`	`1433`	`return;`
`1434`	`1434`	`}`
	`1435`	`+ MtmLock(LW_EXCLUSIVE);`
`1435`	`1436`	`BIT_SET(Mtm->connectivityMask, nodeId-1);`
`1436`	`1437`	`BIT_SET(Mtm->reconnectMask, nodeId-1);`
	`1438`	`+ MtmUnlock();`
	`1439`	`+`
`1437`	`1440`	`RaftableSet(psprintf("node-mask-%d", MtmNodeId), &Mtm->connectivityMask, sizeof Mtm->connectivityMask, false);`
`1438`	`1441`
`1439`	`1442`	`MtmSleep(MSEC_TO_USEC(MtmRaftPollDelay));`
`@@ -1460,7 +1463,10 @@ void MtmOnNodeDisconnect(int nodeId)`
`1460`	`1463`
`1461`	`1464`	`void MtmOnNodeConnect(int nodeId)`
`1462`	`1465`	`{`
	`1466`	`+ MtmLock(LW_EXCLUSIVE);`
`1463`	`1467`	`BIT_CLEAR(Mtm->connectivityMask, nodeId-1);`
	`1468`	`+ MtmUnlock();`
	`1469`	`+`
`1464`	`1470`	`MTM_LOG1("Reconnect node %d", nodeId);`
`1465`	`1471`	`RaftableSet(psprintf("node-mask-%d", MtmNodeId), &Mtm->connectivityMask, sizeof Mtm->connectivityMask, false);`
`1466`	`1472`	`}`
`@@ -2194,6 +2200,8 @@ void MtmRecoverNode(int nodeId)`
`2194`	`2200`
`2195`	`2201`	`void MtmDropNode(int nodeId, bool dropSlot)`
`2196`	`2202`	`{`
	`2203`	`+ MtmLock(LW_EXCLUSIVE);`
	`2204`	`+`
`2197`	`2205`	`if (!BIT_CHECK(Mtm->disabledNodeMask, nodeId-1))`
`2198`	`2206`	`{`
`2199`	`2207`	`if (nodeId <= 0 \|\| nodeId > Mtm->nLiveNodes)`
`@@ -2211,6 +2219,8 @@ void MtmDropNode(int nodeId, bool dropSlot)`
`2211`	`2219`	`ReplicationSlotDrop(psprintf(MULTIMASTER_SLOT_PATTERN, nodeId));`
`2212`	`2220`	`}`
`2213`	`2221`	`}`
	`2222`	`+`
	`2223`	`+ MtmUnlock();`
`2214`	`2224`	`}`
`2215`	`2225`	`static void`
`2216`	`2226`	`MtmOnProcExit(int code, Datum arg)`
`@@ -2401,7 +2411,9 @@ mtm_add_node(PG_FUNCTION_ARGS)`
`2401`	`2411`	`}`
`2402`	`2412`	`else`
`2403`	`2413`	`{`
`2404`		`- int nodeId = Mtm->nAllNodes;`
	`2414`	`+ int nodeId;`
	`2415`	`+ MtmLock(LW_EXCLUSIVE);`
	`2416`	`+ nodeId = Mtm->nAllNodes;`
`2405`	`2417`	`elog(NOTICE, "Add node %d: '%s'", nodeId+1, connStr);`
`2406`	`2418`	`MtmUpdateNodeConnectionInfo(&Mtm->nodes[nodeId].con, connStr);`
`2407`	`2419`	`Mtm->nodes[nodeId].transDelay = 0;`
`@@ -2411,6 +2423,8 @@ mtm_add_node(PG_FUNCTION_ARGS)`
`2411`	`2423`
`2412`	`2424`	`BIT_SET(Mtm->disabledNodeMask, nodeId);`
`2413`	`2425`	`Mtm->nAllNodes += 1;`
	`2426`	`+ MtmUnlock();`
	`2427`	`+`
`2414`	`2428`	`MtmStartReceiver(nodeId+1, true);`
`2415`	`2429`	`}`
`2416`	`2430`	`PG_RETURN_VOID();`