Distributed deadlock detection

knizhnik · knizhnik · commit 69f6a0bff98d · 2015-11-11T17:42:45.000+03:00
diff --git a/contrib/pg_dtm/dtmd/src/ddd.c b/contrib/pg_dtm/dtmd/src/ddd.c
@@ -13,7 +13,6 @@ void initGraph(Graph* graph)
     graph->freeEdges = NULL;
     graph->freeVertexes = NULL;
     graph->marker = 0;
-    graph->min_deadlock_duration = 3;
 }
 
 static inline Edge* newEdge(Graph* graph)
@@ -54,7 +53,6 @@ static inline Vertex* newVertex(Graph* graph)
     } else { 
         graph->freeVertexes = v->next;
     }
-    v->deadlock_duration = 0;
     return v;
 }
 
@@ -148,9 +146,8 @@ bool detectDeadLock(Graph* graph, xid_t root)
     for (v = graph->hashtable[root % MAX_TRANSACTIONS]; v != NULL; v = v->next) { 
         if (v->xid == root) { 
             if (recursiveTraverseGraph(v, v, ++graph->marker)) { 
-                return ++v->deadlock_duration >= graph->min_deadlock_duration;
+                return true;
             }
-            v->deadlock_duration = 0;
             break;
         }
     }
diff --git a/contrib/pg_dtm/dtmd/src/main.c b/contrib/pg_dtm/dtmd/src/main.c
@@ -599,7 +599,7 @@ static void onmessage(client_t client, size_t len, char *data) {
 
 static void usage(char *prog) {
 	printf(
-		"Usage: %s [-d DATADIR] [-k] [-a HOST] [-p PORT] [-l LOGFILE] [-m MIN_DEADLOCK_DURATION]\n"
+		"Usage: %s [-d DATADIR] [-k] [-a HOST] [-p PORT] [-l LOGFILE]\n"
 		"   arbiter will try to kill the other one running at\n"
 		"   the same DATADIR.\n"
 		"   -l : Run as a daemon and write output to LOGFILE.\n"
@@ -716,9 +716,6 @@ int main(int argc, char **argv) {
 			case 'k':
 				assassin = true;
 				break;
-			case 'm':
-                graph.min_deadlock_duration = atoi(optarg);
-                break;
 			default:
 				usage(argv[0]);
 				return EXIT_FAILURE;
diff --git a/contrib/pg_dtm/pg_dtm.c b/contrib/pg_dtm/pg_dtm.c
@@ -71,7 +71,7 @@ void _PG_fini(void);
 static Snapshot DtmGetSnapshot(Snapshot snapshot);
 static void DtmMergeWithGlobalSnapshot(Snapshot snapshot);
 static XidStatus DtmGetTransactionStatus(TransactionId xid, XLogRecPtr *lsn);
-static void DtmSetTransactionStatus(TransactionId xid, int nsubxids, TransactionId *subxids, XidStatus status, XLogRecPtr lsn);
+static bool DtmSetTransactionStatus(TransactionId xid, int nsubxids, TransactionId *subxids, XidStatus status, XLogRecPtr lsn);
 static void DtmUpdateRecentXmin(Snapshot snapshot);
 static void DtmInitialize(void);
 static void DtmXactCallback(XactEvent event, void *arg);
@@ -627,8 +627,9 @@ static XidStatus DtmGetTransactionStatus(TransactionId xid, XLogRecPtr *lsn)
 	return status;
 }
 
-static void DtmSetTransactionStatus(TransactionId xid, int nsubxids, TransactionId *subxids, XidStatus status, XLogRecPtr lsn)
+static bool DtmSetTransactionStatus(TransactionId xid, int nsubxids, TransactionId *subxids, XidStatus status, XLogRecPtr lsn)
 {
+    bool acknowledged = true;
 	XTM_INFO("%d: DtmSetTransactionStatus %u = %u\n", getpid(), xid, status);
 	if (!RecoveryInProgress())
 	{
@@ -640,7 +641,7 @@ static void DtmSetTransactionStatus(TransactionId xid, int nsubxids, Transaction
 				PgTransactionIdSetTreeStatus(xid, nsubxids, subxids, status, lsn);
 				DtmGlobalSetTransStatus(xid, status, false);
 				XTM_INFO("Abort transaction %d\n", xid);
-				return;
+				return true;
 			}
 			else
 			{
@@ -649,8 +650,13 @@ static void DtmSetTransactionStatus(TransactionId xid, int nsubxids, Transaction
 				LWLockAcquire(dtm->hashLock, LW_EXCLUSIVE);
 				hash_search(xid_in_doubt, &DtmNextXid, HASH_ENTER, NULL);
 				LWLockRelease(dtm->hashLock);
-				DtmGlobalSetTransStatus(xid, status, true);
-				XTM_INFO("Commit transaction %d\n", xid);
+				if (!DtmGlobalSetTransStatus(xid, status, true)) { 
+                    acknowledged = false;
+                    XTM_INFO("Commit of transaction %d in rejected by DTM\n", xid);
+                    status = TRANSACTION_STATUS_ABORTED;
+                } else { 
+                    XTM_INFO("Commit transaction %d\n", xid);
+                }
 			}
 		}
 		else
@@ -661,11 +667,13 @@ static void DtmSetTransactionStatus(TransactionId xid, int nsubxids, Transaction
 	else
 	{
 		XidStatus gs;
-		gs = DtmGlobalGetTransStatus(xid, false);
-		if (gs != TRANSACTION_STATUS_UNKNOWN)
+		gs = DtmGlobalGetTransStatus(xid, false);        
+		if (gs != TRANSACTION_STATUS_UNKNOWN) { 
+            acknowledged = status == gs;
 			status = gs;
+        }
 	}
-	PgTransactionIdSetTreeStatus(xid, nsubxids, subxids, status, lsn);
+	return PgTransactionIdSetTreeStatus(xid, nsubxids, subxids, status, lsn) && acknowledged;
 }
 
 static uint32 dtm_xid_hash_fn(const void *key, Size keysize)
@@ -991,27 +999,35 @@ static void DtmSerializeLock(PROCLOCK* proclock, void* arg)
 {
     ByteBuffer* buf = (ByteBuffer*)arg;
     LOCK* lock = proclock->tag.myLock;
-    PGPROC* proc = proclock->tag.myProc;
+    PGPROC* proc = proclock->tag.myProc; 
+
     if (lock != NULL) {
-        if (proc->waitLock == lock) { 
+        PGXACT* srcPgXact = &ProcGlobal->allPgXact[proc->pgprocno];
+        
+        if (TransactionIdIsValid(srcPgXact->xid) && proc->waitLock == lock) { 
             LockMethod lockMethodTable = GetLocksMethodTable(lock);
             int numLockModes = lockMethodTable->numLockModes;
             int conflictMask = lockMethodTable->conflictTab[proc->waitLockMode];
             SHM_QUEUE *procLocks = &(lock->procLocks);
             int lm;
             
-            ByteBufferAppendInt32(buf, proc->lxid); /* waiting transaction */
+            ByteBufferAppendInt32(buf, srcPgXact->xid); /* waiting transaction */
             proclock = (PROCLOCK *) SHMQueueNext(procLocks, procLocks,
                                                  offsetof(PROCLOCK, lockLink));
             while (proclock)
             {
                 if (proc != proclock->tag.myProc) { 
-                    for (lm = 1; lm <= numLockModes; lm++)
-                    {
-                        if ((proclock->holdMask & LOCKBIT_ON(lm)) && (conflictMask & LOCKBIT_ON(lm)))
+                    PGXACT* dstPgXact = &ProcGlobal->allPgXact[proclock->tag.myProc->pgprocno];
+                    if (TransactionIdIsValid(dstPgXact->xid)) { 
+                        Assert(srcPgXact->xid != dstPgXact->xid);
+                        for (lm = 1; lm <= numLockModes; lm++)
                         {
-                            ByteBufferAppendInt32(buf, proclock->tag.myProc->lxid); /* transaction holding lock */
-                            break;
+                            if ((proclock->holdMask & LOCKBIT_ON(lm)) && (conflictMask & LOCKBIT_ON(lm)))
+                            {
+                                XTM_INFO("%d: %u(%u) waits for %u(%u)\n", getpid(), srcPgXact->xid, proc->pid, dstPgXact->xid, proclock->tag.myProc->pid);
+                                ByteBufferAppendInt32(buf, dstPgXact->xid); /* transaction holding lock */
+                                break;
+                            }
                         }
                     }
                 }
@@ -1025,12 +1041,19 @@ static void DtmSerializeLock(PROCLOCK* proclock, void* arg)
 
 bool DtmDetectGlobalDeadLock(PGPROC* proc)
 {
-    bool hasDeadlock;
+    bool hasDeadlock = false;
     ByteBuffer buf;
-    ByteBufferAlloc(&buf);
-    EnumerateLocks(DtmSerializeLock, &buf);
-    hasDeadlock = DtmGlobalDetectDeadLock(PostPortNumber, proc->lxid, buf.data, buf.used);
-    ByteBufferFree(&buf);
-    elog(NOTICE, "Deadlock detected for transaction %u", proc->lxid);
+    PGXACT* pgxact = &ProcGlobal->allPgXact[proc->pgprocno];
+
+    if (TransactionIdIsValid(pgxact->xid)) { 
+        ByteBufferAlloc(&buf);
+        XTM_INFO("%d: wait graph begin\n", getpid());
+        EnumerateLocks(DtmSerializeLock, &buf);
+        XTM_INFO("%d: wait graph end\n", getpid());
+        hasDeadlock = DtmGlobalDetectDeadLock(PostPortNumber, pgxact->xid, buf.data, buf.used);
+        ByteBufferFree(&buf);
+        XTM_INFO("%d: deadlock detected for %u\n", getpid(), pgxact->xid);
+        elog(WARNING, "Deadlock detected for transaction %u", pgxact->xid);
+    }
     return hasDeadlock;
 }
diff --git a/contrib/pg_dtm/tests/dtmbench.cpp b/contrib/pg_dtm/tests/dtmbench.cpp
@@ -161,8 +161,10 @@ void* writer(void* arg)
     for (int i = 0; i < cfg.nIterations; i++)
     { 
         int srcCon, dstCon;
-        int srcAcc = (random() % ((cfg.nAccounts-cfg.nWriters)/cfg.nWriters))*cfg.nWriters + t.id;
-        int dstAcc = (random() % ((cfg.nAccounts-cfg.nWriters)/cfg.nWriters))*cfg.nWriters + t.id;
+        //int srcAcc = (random() % ((cfg.nAccounts-cfg.nWriters)/cfg.nWriters))*cfg.nWriters + t.id;
+        //int dstAcc = (random() % ((cfg.nAccounts-cfg.nWriters)/cfg.nWriters))*cfg.nWriters + t.id;
+        int srcAcc = random() % cfg.nAccounts;
+        int dstAcc = random() % cfg.nAccounts;
 
         do { 
             srcCon = random() % cfg.connections.size();
@@ -183,7 +185,7 @@ void* writer(void* arg)
             exec(dstTx, "update t set v = v + 1 where u=%d", dstAcc);
         } catch (pqxx_exception const& x) { 
             exec(srcTx, "rollback");
-            exec(srcTx, "rollback");
+            exec(dstTx, "rollback");
             t.aborts += 1;
             i -= 1;
             continue;
diff --git a/src/backend/access/transam/xact.c b/src/backend/access/transam/xact.c
@@ -1312,6 +1312,7 @@ RecordTransactionCommit(void)
 		MyPgXact->delayChkpt = false;
 		END_CRIT_SECTION();
         if (!committed) {
+            CurrentTransactionState->state = TRANS_ABORT;
             elog(ERROR, "Transaction commit rejected by XTM");
         }
 	}
@@ -5405,7 +5406,7 @@ xact_redo_commit(xl_xact_parsed_commit *parsed,
 		StandbyReleaseLockTree(xid, 0, NULL);
 	}
     if (!committed) {
-        elog(NOTICE, "XTM rejected recovert of tran saction %u", xid);
+        elog(WARNING, "XTM rejected recovery of transaction %u", xid);
     }
 	if (parsed->xinfo & XACT_XINFO_HAS_ORIGIN)
 	{

Original file line number	Diff line number	Diff line change
`@@ -13,7 +13,6 @@ void initGraph(Graph* graph)`
`13`	`13`	`graph->freeEdges = NULL;`
`14`	`14`	`graph->freeVertexes = NULL;`
`15`	`15`	`graph->marker = 0;`
`16`		`- graph->min_deadlock_duration = 3;`
`17`	`16`	`}`
`18`	`17`
`19`	`18`	`static inline Edge* newEdge(Graph* graph)`
`@@ -54,7 +53,6 @@ static inline Vertex* newVertex(Graph* graph)`
`54`	`53`	`} else {`
`55`	`54`	`graph->freeVertexes = v->next;`
`56`	`55`	`}`
`57`		`- v->deadlock_duration = 0;`
`58`	`56`	`return v;`
`59`	`57`	`}`
`60`	`58`
`@@ -148,9 +146,8 @@ bool detectDeadLock(Graph* graph, xid_t root)`
`148`	`146`	`for (v = graph->hashtable[root % MAX_TRANSACTIONS]; v != NULL; v = v->next) {`
`149`	`147`	`if (v->xid == root) {`
`150`	`148`	`if (recursiveTraverseGraph(v, v, ++graph->marker)) {`
`151`		`- return ++v->deadlock_duration >= graph->min_deadlock_duration;`
	`149`	`+ return true;`
`152`	`150`	`}`
`153`		`- v->deadlock_duration = 0;`
`154`	`151`	`break;`
`155`	`152`	`}`
`156`	`153`	`}`
Original file line number	Diff line number	Diff line change
`@@ -1312,6 +1312,7 @@ RecordTransactionCommit(void)`
`1312`	`1312`	`MyPgXact->delayChkpt = false;`
`1313`	`1313`	`END_CRIT_SECTION();`
`1314`	`1314`	`if (!committed) {`
	`1315`	`+ CurrentTransactionState->state = TRANS_ABORT;`
`1315`	`1316`	`elog(ERROR, "Transaction commit rejected by XTM");`
`1316`	`1317`	`}`
`1317`	`1318`	`}`
`@@ -5405,7 +5406,7 @@ xact_redo_commit(xl_xact_parsed_commit *parsed,`
`5405`	`5406`	`StandbyReleaseLockTree(xid, 0, NULL);`
`5406`	`5407`	`}`
`5407`	`5408`	`if (!committed) {`
`5408`		`- elog(NOTICE, "XTM rejected recovert of tran saction %u", xid);`
	`5409`	`+ elog(WARNING, "XTM rejected recovery of transaction %u", xid);`
`5409`	`5410`	`}`
`5410`	`5411`	`if (parsed->xinfo & XACT_XINFO_HAS_ORIGIN)`
`5411`	`5412`	`{`