Merge branch 'PGPROEE9_6_CFS_385' into PGPROEE9_6

vbwagner · vbwagner · commit 0ae6e6d5d042 · 2017-02-17T23:05:54.000+03:00
diff --git a/src/backend/storage/file/cfs.c b/src/backend/storage/file/cfs.c
@@ -55,6 +55,7 @@
 #include "utils/resowner_private.h"
 #include "postmaster/bgworker.h"
 
+
 /*
  * GUC variable that defines compression level.
  * 0 - no compression, 1 - max speed,
@@ -417,15 +418,21 @@ int cfs_msync(FileMap* map)
 FileMap* cfs_mmap(int md)
 {
 	FileMap* map;
+	if (ftruncate(md, sizeof(FileMap)) != 0) 
+	{
+		return (FileMap*)MAP_FAILED;
+	}
+	
 #ifdef WIN32
-	HANDLE mh = CreateFileMapping(_get_osfhandle(md), NULL, PAGE_READWRITE, 
+	{
+		HANDLE mh = CreateFileMapping(_get_osfhandle(md), NULL, PAGE_READWRITE, 
 								  0, (DWORD)sizeof(FileMap), NULL);
-	if (mh == NULL)
-		return (FileMap*)MAP_FAILED;
-
-	map = (FileMap*)MapViewOfFile(mh, FILE_MAP_ALL_ACCESS, 0, 0, 0); 
-	CloseHandle(mh);
+		if (mh == NULL)
+			return (FileMap*)MAP_FAILED;
 
+		map = (FileMap*)MapViewOfFile(mh, FILE_MAP_ALL_ACCESS, 0, 0, 0); 
+		CloseHandle(mh);
+	}
 	if (map == NULL)
 		return (FileMap*)MAP_FAILED;
 
@@ -533,17 +540,44 @@ static bool cfs_write_file(int fd, void const* data, uint32 size)
 void cfs_lock_file(FileMap* map, char const* file_path)
 {
 	long delay = CFS_LOCK_MIN_TIMEOUT;
+	int n_attempts = 0;
 
 	while (true)
 	{
 		uint64 count = pg_atomic_fetch_add_u32(&map->lock, 1);
+		bool revokeLock = false;
 
 		if (count < CFS_GC_LOCK)
 			break;
-
-		if (InRecovery)
+		
+		if (InRecovery) 
+		{
+			revokeLock = true;
+		} 
+		else 
 		{
-			/* Uhhh... looks like last GC was interrupted.
+			if (!pg_atomic_unlocked_test_flag(&cfs_state->gc_started))
+			{
+				if (++n_attempts > MAX_LOCK_ATTEMPTS) 
+				{
+					/* So there is GC lock, but no active GC process during MAX_LOCK_ATTEMPTS.
+					 * Most likely it means that GC is crashed (may be together with other postgres processes or even OS)
+					 * without releasing lock. And for some reasons recovery was not performed and this page left locked.
+					 * We should revoke the the lock to allow access to this segment.
+					 */
+					revokeLock = true;
+				}
+			}
+			else
+			{
+				n_attempts = 0; /* Reset counter of attempts because GC is in progress */
+			}
+		}
+		if (revokeLock 
+            /* use gc_started flag to prevent race condition with other backends and GC */
+			&& pg_atomic_test_set_flag(&cfs_state->gc_started)) 
+		{
+			/* Ugggh... looks like last GC was interrupted.
 			 * Try to recover the file.
 			 */
 			char* map_bck_path = psprintf("%s.cfm.bck", file_path);
@@ -565,18 +599,20 @@ void cfs_lock_file(FileMap* map, char const* file_path)
 			else
 			{
 				/* Presence of backup file means that we still have
-				 * unchanged data and map files. Just remove backup files,
-				 * grab lock and continue processing
+				 * unchanged data and map files. Just remove backup files and 
+				 * revoke GC lock.
 				 */
 				unlink(file_bck_path);
 				unlink(map_bck_path);
 			}
 
+			pg_atomic_clear_flag(&cfs_state->gc_started);
+			count = pg_atomic_fetch_sub_u32(&map->lock, CFS_GC_LOCK); /* revoke GC lock */
+			Assert((int)count > 0);
 			pfree(file_bck_path);
 			pfree(map_bck_path);
 			break;
-		}
-
+		} 
 		pg_atomic_fetch_sub_u32(&map->lock, 1);
 		pg_usleep(delay);
 		if (delay < CFS_LOCK_MAX_TIMEOUT)
diff --git a/src/backend/storage/file/copydir.c b/src/backend/storage/file/copydir.c
@@ -319,6 +319,7 @@ copy_zip_file(char *fromfile, bool from_compressed,
 	Assert(sep != NULL);
 
 	if ((sscanf(sep+1, "%d.%d%n", &relno, &segno, &n) != 2
+		&& sscanf(sep+1, "%d_init%n", &relno, &n) != 1
 		&& sscanf(sep+1, "%d%n", &relno, &n) != 1)
 		|| sep[n+1] != '\0'
 		|| relno < FirstNormalObjectId)
@@ -328,7 +329,7 @@ copy_zip_file(char *fromfile, bool from_compressed,
 	}
 
 	if (to_compressed)
-		fprintf(stderr, "Compress file %s, relno=%d, sep[n+1]=%s\n",
+		elog(DEBUG2, "Compress file %s, relno=%d, sep[n+1]=%s\n",
 				tofile, relno, &sep[n+1]);
 
 	/* Open the files */
diff --git a/src/backend/storage/file/fd.c b/src/backend/storage/file/fd.c
@@ -1322,9 +1322,6 @@ PathNameOpenFile(FileName fileName, int fileFlags, int fileMode)
 			goto io_error;
 		}
 
-		if (ftruncate(vfdP->md, sizeof(FileMap)) != 0)
-			elog(LOG, "OPEN MAP ftruncate FAILED: %d", errno);
-
 		vfdP->map = cfs_mmap(vfdP->md);
 		if (vfdP->map == MAP_FAILED)
 		{
diff --git a/src/backend/storage/file/reinit.c b/src/backend/storage/file/reinit.c
@@ -25,9 +25,9 @@
 #include "utils/memutils.h"
 
 static void ResetUnloggedRelationsInTablespaceDir(const char *tsdirname,
-									  int op);
+												  int op, bool compressed);
 static void ResetUnloggedRelationsInDbspaceDir(const char *dbspacedirname,
-								   int op);
+											   int op, bool compressed);
 static bool parse_filename_for_nontemp_relation(const char *name,
 									int *oidchars, ForkNumber *fork);
 
@@ -71,7 +71,7 @@ ResetUnloggedRelations(int op)
 	/*
 	 * First process unlogged files in pg_default ($PGDATA/base)
 	 */
-	ResetUnloggedRelationsInTablespaceDir("base", op);
+	ResetUnloggedRelationsInTablespaceDir("base", op, false);
 
 	/*
 	 * Cycle through directories for all non-default tablespaces.
@@ -80,13 +80,25 @@ ResetUnloggedRelations(int op)
 
 	while ((spc_de = ReadDir(spc_dir, "pg_tblspc")) != NULL)
 	{
+		FILE* compressionFile;
+
 		if (strcmp(spc_de->d_name, ".") == 0 ||
 			strcmp(spc_de->d_name, "..") == 0)
 			continue;
 
+		snprintf(temp_path, sizeof(temp_path), "pg_tblspc/%s/%s/pg_compression",
+				 spc_de->d_name, TABLESPACE_VERSION_DIRECTORY);
+		
+		compressionFile = fopen(temp_path, "r");
+		if (compressionFile) 
+		{ 
+			fclose(compressionFile);
+		}
+		
 		snprintf(temp_path, sizeof(temp_path), "pg_tblspc/%s/%s",
 				 spc_de->d_name, TABLESPACE_VERSION_DIRECTORY);
-		ResetUnloggedRelationsInTablespaceDir(temp_path, op);
+
+		ResetUnloggedRelationsInTablespaceDir(temp_path, op, compressionFile != NULL);
 	}
 
 	FreeDir(spc_dir);
@@ -100,7 +112,7 @@ ResetUnloggedRelations(int op)
 
 /* Process one tablespace directory for ResetUnloggedRelations */
 static void
-ResetUnloggedRelationsInTablespaceDir(const char *tsdirname, int op)
+ResetUnloggedRelationsInTablespaceDir(const char *tsdirname, int op, bool compressed)
 {
 	DIR		   *ts_dir;
 	struct dirent *de;
@@ -133,15 +145,15 @@ ResetUnloggedRelationsInTablespaceDir(const char *tsdirname, int op)
 
 		snprintf(dbspace_path, sizeof(dbspace_path), "%s/%s",
 				 tsdirname, de->d_name);
-		ResetUnloggedRelationsInDbspaceDir(dbspace_path, op);
+		ResetUnloggedRelationsInDbspaceDir(dbspace_path, op, compressed);
 	}
 
 	FreeDir(ts_dir);
 }
 
 /* Process one per-dbspace directory for ResetUnloggedRelations */
 static void
-ResetUnloggedRelationsInDbspaceDir(const char *dbspacedirname, int op)
+ResetUnloggedRelationsInDbspaceDir(const char *dbspacedirname, int op, bool compressed)
 {
 	DIR		   *dbspace_dir;
 	struct dirent *de;
@@ -332,8 +344,13 @@ ResetUnloggedRelationsInDbspaceDir(const char *dbspacedirname, int op)
 					 strlen(forkNames[INIT_FORKNUM]));
 
 			/* OK, we're ready to perform the actual copy. */
-			elog(DEBUG2, "copying %s to %s", srcpath, dstpath);
-			copy_file(srcpath, dstpath);
+			if (compressed) { 
+				elog(DEBUG2, "copying %s to %s with compression", srcpath, dstpath);
+				copy_zip_file(srcpath, false, dstpath, true);
+			} else { 
+				elog(DEBUG2, "copying %s to %s", srcpath, dstpath);
+				copy_file(srcpath, dstpath);
+			}
 		}
 
 		FreeDir(dbspace_dir);
diff --git a/src/include/storage/cfs.h b/src/include/storage/cfs.h
@@ -15,6 +15,15 @@
 #define CFS_DISABLE_TIMEOUT  1000   /* milliseconds */
 #define CFS_ESTIMATE_PROBES  10
 
+/* 
+ * Maximal number of attempts backend should perform to lock file segment.
+ * If during all this attempts gc_started flag is not set (it means that no GC is active at this moment),
+ * then it means that for some reasons GC process was crashed (may be together with other postgres processes or even OS)
+ * without releasing lock. And for some reasons recovery was not performed and this page left locked. Цe should revoke this lock
+ * to allow access to this database segment.
+ */
+#define MAX_LOCK_ATTEMPTS 100
+
 /* Maximal size of buffer for compressing (size) bytes where (size) is equal to PostgreSQL page size.
  * Some compression algorithms requires to provide buffer large enough for worst case and immediately returns error is buffer is not enough.
  * Accurate calculation of required buffer size is not needed here and doubling buffer size works for all used compression algorithms. */

Original file line number	Diff line number	Diff line change
`@@ -1322,9 +1322,6 @@ PathNameOpenFile(FileName fileName, int fileFlags, int fileMode)`
`1322`	`1322`	`goto io_error;`
`1323`	`1323`	`}`
`1324`	`1324`
`1325`		`- if (ftruncate(vfdP->md, sizeof(FileMap)) != 0)`
`1326`		`- elog(LOG, "OPEN MAP ftruncate FAILED: %d", errno);`
`1327`		`-`
`1328`	`1325`	`vfdP->map = cfs_mmap(vfdP->md);`
`1329`	`1326`	`if (vfdP->map == MAP_FAILED)`
`1330`	`1327`	`{`