Detect node disconnection in case of pglogical_receiver failures

knizhnik · knizhnik · commit 466f3c1c40e1 · 2016-05-05T17:38:09.000+03:00
diff --git a/contrib/mmts/multimaster.c b/contrib/mmts/multimaster.c
@@ -1290,7 +1290,7 @@ void MtmCheckQuorum(void)
 		}
 	} else {
 		if (Mtm->status == MTM_IN_MINORITY) { 
-			MTM_LOG1("Node is in majority: dissbled mask %lx", (long) Mtm->disabledNodeMask);
+			MTM_LOG1("Node is in majority: disabled mask %lx", (long) Mtm->disabledNodeMask);
 			MtmSwitchClusterMode(MTM_ONLINE);
 		}
 	}
diff --git a/contrib/mmts/pglogical_receiver.c b/contrib/mmts/pglogical_receiver.c
@@ -194,7 +194,7 @@ static char const* const MtmReplicationModeName[] =
 {
 	"recovered", /* SLOT_CREATE_NEW: recovery of node is completed so drop old slot and restart replication from the current position in WAL */
 	"recovery",  /* SLOT_OPEN_EXISTED: perform recorvery of the node by applying all data from theslot from specified point */
-	"normal"     /* SLOT_OPEN_ALWAYS: normal mode: use existeed slot or create new one and start receiving data from it from the specified position */
+	"normal"     /* SLOT_OPEN_ALWAYS: normal mode: use existed slot or create new one and start receiving data from it from the specified position */
 };
 
 static void
@@ -248,6 +248,7 @@ pglogical_receiver_main(Datum main_arg)
 		PQfinish(conn);
 		ereport(WARNING, (errmsg("%s: Could not establish connection to remote server",
 								 worker_proc)));
+		/* Do not make decision about node status here because at startup peer node may just no yet started */
 		/* MtmOnNodeDisconnect(nodeId); */
 		proc_exit(1);
 	}
@@ -271,6 +272,7 @@ pglogical_receiver_main(Datum main_arg)
 				PQclear(res);
 				ereport(ERROR, (errmsg("%s: Could not create logical slot",
 									   worker_proc)));
+				MtmOnNodeDisconnect(nodeId);
 				proc_exit(1);
 			}
 		}
@@ -312,6 +314,7 @@ pglogical_receiver_main(Datum main_arg)
 		PQclear(res);
 		ereport(WARNING, (errmsg("%s: Could not start logical replication",
 								 worker_proc)));
+		MtmOnNodeDisconnect(nodeId);
 		proc_exit(1);
 	}
 	PQclear(res);
@@ -402,6 +405,7 @@ pglogical_receiver_main(Datum main_arg)
 				{
 					ereport(LOG, (errmsg("%s: streaming header too small: %d",
 										 worker_proc, rc)));
+					MtmOnNodeDisconnect(nodeId);
 					proc_exit(1);
 				}
 				replyRequested = copybuf[pos];
@@ -421,15 +425,18 @@ pglogical_receiver_main(Datum main_arg)
 					int64 now = feGetCurrentTimestamp();
 
 					/* Leave is feedback is not sent properly */
-					if (!sendFeedback(conn, now, nodeId))
+					if (!sendFeedback(conn, now, nodeId)) {
+						MtmOnNodeDisconnect(nodeId);
 						proc_exit(1);
+					}
 				}
 				continue;
 			}
 			else if (copybuf[0] != 'w')
 			{
 				ereport(LOG, (errmsg("%s: Incorrect streaming header",
 									 worker_proc)));
+				MtmOnNodeDisconnect(nodeId);
 				proc_exit(1);
 			}
 
@@ -538,6 +545,7 @@ pglogical_receiver_main(Datum main_arg)
 			{
 				ereport(LOG, (errmsg("%s: Incorrect status received... Leaving.",
 									 worker_proc)));
+				MtmOnNodeDisconnect(nodeId);
 				proc_exit(1);
 			}
 
@@ -546,6 +554,7 @@ pglogical_receiver_main(Datum main_arg)
 			{
 				ereport(LOG, (errmsg("%s: Data remaining on the socket... Leaving.",
 									 worker_proc)));
+				MtmOnNodeDisconnect(nodeId);
 				proc_exit(1);
 			}
 			continue;
@@ -564,6 +573,7 @@ pglogical_receiver_main(Datum main_arg)
 		{
 			ereport(LOG, (errmsg("%s: Failure while receiving changes...",
 								 worker_proc)));
+			MtmOnNodeDisconnect(nodeId);
 			proc_exit(1);
 		}
 	}

Original file line number	Diff line number	Diff line change
`@@ -1290,7 +1290,7 @@ void MtmCheckQuorum(void)`
`1290`	`1290`	`}`
`1291`	`1291`	`} else {`
`1292`	`1292`	`if (Mtm->status == MTM_IN_MINORITY) {`
`1293`		`- MTM_LOG1("Node is in majority: dissbled mask %lx", (long) Mtm->disabledNodeMask);`
	`1293`	`+ MTM_LOG1("Node is in majority: disabled mask %lx", (long) Mtm->disabledNodeMask);`
`1294`	`1294`	`MtmSwitchClusterMode(MTM_ONLINE);`
`1295`	`1295`	`}`
`1296`	`1296`	`}`
Original file line number	Diff line number	Diff line change
`@@ -194,7 +194,7 @@ static char const* const MtmReplicationModeName[] =`
`194`	`194`	`{`
`195`	`195`	`"recovered", /* SLOT_CREATE_NEW: recovery of node is completed so drop old slot and restart replication from the current position in WAL */`
`196`	`196`	`"recovery", /* SLOT_OPEN_EXISTED: perform recorvery of the node by applying all data from theslot from specified point */`
`197`		`- "normal" /* SLOT_OPEN_ALWAYS: normal mode: use existeed slot or create new one and start receiving data from it from the specified position */`
	`197`	`+ "normal" /* SLOT_OPEN_ALWAYS: normal mode: use existed slot or create new one and start receiving data from it from the specified position */`
`198`	`198`	`};`
`199`	`199`
`200`	`200`	`static void`
`@@ -248,6 +248,7 @@ pglogical_receiver_main(Datum main_arg)`
`248`	`248`	`PQfinish(conn);`
`249`	`249`	`ereport(WARNING, (errmsg("%s: Could not establish connection to remote server",`
`250`	`250`	`worker_proc)));`
	`251`	`+ /* Do not make decision about node status here because at startup peer node may just no yet started */`
`251`	`252`	`/* MtmOnNodeDisconnect(nodeId); */`
`252`	`253`	`proc_exit(1);`
`253`	`254`	`}`
`@@ -271,6 +272,7 @@ pglogical_receiver_main(Datum main_arg)`
`271`	`272`	`PQclear(res);`
`272`	`273`	`ereport(ERROR, (errmsg("%s: Could not create logical slot",`
`273`	`274`	`worker_proc)));`
	`275`	`+ MtmOnNodeDisconnect(nodeId);`
`274`	`276`	`proc_exit(1);`
`275`	`277`	`}`
`276`	`278`	`}`
`@@ -312,6 +314,7 @@ pglogical_receiver_main(Datum main_arg)`
`312`	`314`	`PQclear(res);`
`313`	`315`	`ereport(WARNING, (errmsg("%s: Could not start logical replication",`
`314`	`316`	`worker_proc)));`
	`317`	`+ MtmOnNodeDisconnect(nodeId);`
`315`	`318`	`proc_exit(1);`
`316`	`319`	`}`
`317`	`320`	`PQclear(res);`
`@@ -402,6 +405,7 @@ pglogical_receiver_main(Datum main_arg)`
`402`	`405`	`{`
`403`	`406`	`ereport(LOG, (errmsg("%s: streaming header too small: %d",`
`404`	`407`	`worker_proc, rc)));`
	`408`	`+ MtmOnNodeDisconnect(nodeId);`
`405`	`409`	`proc_exit(1);`
`406`	`410`	`}`
`407`	`411`	`replyRequested = copybuf[pos];`
`@@ -421,15 +425,18 @@ pglogical_receiver_main(Datum main_arg)`
`421`	`425`	`int64 now = feGetCurrentTimestamp();`
`422`	`426`
`423`	`427`	`/* Leave is feedback is not sent properly */`
`424`		`- if (!sendFeedback(conn, now, nodeId))`
	`428`	`+ if (!sendFeedback(conn, now, nodeId)) {`
	`429`	`+ MtmOnNodeDisconnect(nodeId);`
`425`	`430`	`proc_exit(1);`
	`431`	`+ }`
`426`	`432`	`}`
`427`	`433`	`continue;`
`428`	`434`	`}`
`429`	`435`	`else if (copybuf[0] != 'w')`
`430`	`436`	`{`
`431`	`437`	`ereport(LOG, (errmsg("%s: Incorrect streaming header",`
`432`	`438`	`worker_proc)));`
	`439`	`+ MtmOnNodeDisconnect(nodeId);`
`433`	`440`	`proc_exit(1);`
`434`	`441`	`}`
`435`	`442`
`@@ -538,6 +545,7 @@ pglogical_receiver_main(Datum main_arg)`
`538`	`545`	`{`
`539`	`546`	`ereport(LOG, (errmsg("%s: Incorrect status received... Leaving.",`
`540`	`547`	`worker_proc)));`
	`548`	`+ MtmOnNodeDisconnect(nodeId);`
`541`	`549`	`proc_exit(1);`
`542`	`550`	`}`
`543`	`551`
`@@ -546,6 +554,7 @@ pglogical_receiver_main(Datum main_arg)`
`546`	`554`	`{`
`547`	`555`	`ereport(LOG, (errmsg("%s: Data remaining on the socket... Leaving.",`
`548`	`556`	`worker_proc)));`
	`557`	`+ MtmOnNodeDisconnect(nodeId);`
`549`	`558`	`proc_exit(1);`
`550`	`559`	`}`
`551`	`560`	`continue;`
`@@ -564,6 +573,7 @@ pglogical_receiver_main(Datum main_arg)`
`564`	`573`	`{`
`565`	`574`	`ereport(LOG, (errmsg("%s: Failure while receiving changes...",`
`566`	`575`	`worker_proc)));`
	`576`	`+ MtmOnNodeDisconnect(nodeId);`
`567`	`577`	`proc_exit(1);`
`568`	`578`	`}`
`569`	`579`	`}`