python scripts

zilder · zilder · commit f5ba66089f1c · 2016-08-19T20:51:20.000+03:00
diff --git a/.gitignore b/.gitignore
@@ -5,4 +5,5 @@ regression.diffs
 regression.out
 *.o
 *.so
+*.pyc
 pg_pathman--*.sql
diff --git a/init.sql b/init.sql
@@ -140,95 +140,95 @@ CREATE TYPE @extschema@.PathmanRange (
 	output = pathman_range_out
 );
 
-/*
- * Copy rows to partitions
- */
--- CREATE OR REPLACE FUNCTION @extschema@.partition_data(
---     p_relation regclass
---     , p_min ANYELEMENT DEFAULT NULL::text
---     , p_max ANYELEMENT DEFAULT NULL::text
---     , p_limit INT DEFAULT NULL
---     , OUT p_total BIGINT)
--- AS
--- $$
--- DECLARE
---     v_attr         TEXT;
---     v_limit_clause TEXT := '';
---     v_where_clause TEXT := '';
--- BEGIN
---     SELECT attname INTO v_attr
---     FROM @extschema@.pathman_config WHERE partrel = p_relation;
-
---     PERFORM @extschema@.debug_capture();
-
---     p_total := 0;
-
---     /* Format LIMIT clause if needed */
---     IF NOT p_limit IS NULL THEN
---         v_limit_clause := format('LIMIT %s', p_limit);
---     END IF;
-
---     /* Format WHERE clause if needed */
---     IF NOT p_min IS NULL THEN
---         v_where_clause := format('%1$s >= $1', v_attr);
---     END IF;
-
---     IF NOT p_max IS NULL THEN
---         IF NOT p_min IS NULL THEN
---             v_where_clause := v_where_clause || ' AND ';
---         END IF;
---         v_where_clause := v_where_clause || format('%1$s < $2', v_attr);
---     END IF;
-
---     IF v_where_clause != '' THEN
---         v_where_clause := 'WHERE ' || v_where_clause;
---     END IF;
-
---     /* Lock rows and copy data */
---     RAISE NOTICE 'Copying data to partitions...';
---     EXECUTE format('
---         WITH data AS (
---             DELETE FROM ONLY %1$s WHERE ctid IN (
---                 SELECT ctid FROM ONLY %1$s %2$s %3$s FOR UPDATE NOWAIT
---             ) RETURNING *)
---         INSERT INTO %1$s SELECT * FROM data'
---         , p_relation, v_where_clause, v_limit_clause)
---     USING p_min, p_max;
-
---     GET DIAGNOSTICS p_total = ROW_COUNT;
---     RETURN;
--- END
--- $$
--- LANGUAGE plpgsql;
-
 /*
  * Copy rows to partitions
  */
 CREATE OR REPLACE FUNCTION @extschema@.partition_data(
-	parent_relid	REGCLASS,
-	OUT p_total		BIGINT)
+    p_relation regclass
+    , p_min ANYELEMENT DEFAULT NULL::text
+    , p_max ANYELEMENT DEFAULT NULL::text
+    , p_limit INT DEFAULT NULL
+    , OUT p_total BIGINT)
 AS
 $$
 DECLARE
-	relname		TEXT;
-	rec			RECORD;
-	cnt			BIGINT := 0;
-
+    v_attr         TEXT;
+    v_limit_clause TEXT := '';
+    v_where_clause TEXT := '';
 BEGIN
-	p_total := 0;
-
-	/* Create partitions and copy rest of the data */
-	EXECUTE format('WITH part_data AS (DELETE FROM ONLY %1$s RETURNING *)
-					INSERT INTO %1$s SELECT * FROM part_data',
-				   @extschema@.get_schema_qualified_name(parent_relid));
-
-	/* Get number of inserted rows */
-	GET DIAGNOSTICS p_total = ROW_COUNT;
-	RETURN;
+    SELECT attname INTO v_attr
+    FROM @extschema@.pathman_config WHERE partrel = p_relation;
+
+    PERFORM @extschema@.debug_capture();
+
+    p_total := 0;
+
+    /* Format LIMIT clause if needed */
+    IF NOT p_limit IS NULL THEN
+        v_limit_clause := format('LIMIT %s', p_limit);
+    END IF;
+
+    /* Format WHERE clause if needed */
+    IF NOT p_min IS NULL THEN
+        v_where_clause := format('%1$s >= $1', v_attr);
+    END IF;
+
+    IF NOT p_max IS NULL THEN
+        IF NOT p_min IS NULL THEN
+            v_where_clause := v_where_clause || ' AND ';
+        END IF;
+        v_where_clause := v_where_clause || format('%1$s < $2', v_attr);
+    END IF;
+
+    IF v_where_clause != '' THEN
+        v_where_clause := 'WHERE ' || v_where_clause;
+    END IF;
+
+    /* Lock rows and copy data */
+    RAISE NOTICE 'Copying data to partitions...';
+    EXECUTE format('
+        WITH data AS (
+            DELETE FROM ONLY %1$s WHERE ctid IN (
+                SELECT ctid FROM ONLY %1$s %2$s %3$s FOR UPDATE NOWAIT
+            ) RETURNING *)
+        INSERT INTO %1$s SELECT * FROM data'
+        , p_relation, v_where_clause, v_limit_clause)
+    USING p_min, p_max;
+
+    GET DIAGNOSTICS p_total = ROW_COUNT;
+    RETURN;
 END
 $$
 LANGUAGE plpgsql;
 
+/*
+ * Copy rows to partitions
+ */
+-- CREATE OR REPLACE FUNCTION @extschema@.partition_data(
+-- 	parent_relid	REGCLASS,
+-- 	OUT p_total		BIGINT)
+-- AS
+-- $$
+-- DECLARE
+-- 	relname		TEXT;
+-- 	rec			RECORD;
+-- 	cnt			BIGINT := 0;
+
+-- BEGIN
+-- 	p_total := 0;
+
+-- 	/* Create partitions and copy rest of the data */
+-- 	EXECUTE format('WITH part_data AS (DELETE FROM ONLY %1$s RETURNING *)
+-- 					INSERT INTO %1$s SELECT * FROM part_data',
+-- 				   @extschema@.get_schema_qualified_name(parent_relid));
+
+-- 	/* Get number of inserted rows */
+-- 	GET DIAGNOSTICS p_total = ROW_COUNT;
+-- 	RETURN;
+-- END
+-- $$
+-- LANGUAGE plpgsql;
+
 /*
  * Disable pathman partitioning for specified relation
  */
diff --git a/src/pathman.h b/src/pathman.h
@@ -28,9 +28,9 @@
 
 
 /* Check PostgreSQL version (9.5.4 contains an important fix for BGW) */
-#if PG_VERSION_NUM < 90504
-	#error "Cannot build pg_pathman with PostgreSQL version lower than 9.5.4"
-#endif
+// #if PG_VERSION_NUM < 90504
+// 	#error "Cannot build pg_pathman with PostgreSQL version lower than 9.5.4"
+// #endif
 
 /* Get CString representation of Datum (simple wrapper) */
 #ifdef USE_ASSERT_CHECKING
diff --git a/tests/concurrent_partitioning_test.py b/tests/concurrent_partitioning_test.py
@@ -0,0 +1,64 @@
+#coding: utf-8
+"""
+ concurrent_partitioning_test.py
+        Tests concurrent partitioning worker with simultaneous update queries
+
+ Copyright (c) 2015-2016, Postgres Professional
+"""
+
+import unittest
+from testgres import get_new_node, clean_all, stop_all
+from subprocess import Popen, PIPE
+import subprocess
+import time
+
+
+class ConcurrentTest(unittest.TestCase):
+
+	def setUp(self):
+		pass
+
+	def tearDown(self):
+		stop_all()
+		# clean_all()
+
+	def test_concurrent(self):
+		setup_cmd = [
+			'create extension pg_pathman',
+			'create table abc(id serial, t text)',
+			'insert into abc select generate_series(1, 300000)',
+			'select create_hash_partitions(\'abc\', \'id\', 3, p_partition_data := false)',
+		]
+
+		node = get_new_node('test')
+		node.init()
+		node.append_conf('postgresql.conf', 'shared_preload_libraries=\'pg_pathman\'\n')
+		node.start()
+
+		for cmd in setup_cmd:
+			node.safe_psql('postgres', cmd)
+
+		node.psql('postgres', 'select partition_data_worker(\'abc\')')
+
+		while True:
+			# update some rows to check for deadlocks
+			node.safe_psql('postgres', 
+				'''update abc set t = 'test'
+				where id in (select (random() * 300000)::int from generate_series(1, 3000))''')
+
+			count = node.execute('postgres', 'select count(*) from pathman_active_workers')
+
+			# if there is no active workers then it means work is done
+			if count[0][0] == 0:
+				break
+			time.sleep(1)
+
+		data = node.execute('postgres', 'select count(*) from only abc')
+		self.assertEqual(data[0][0], 0)
+		data = node.execute('postgres', 'select count(*) from abc')
+		self.assertEqual(data[0][0], 300000)
+
+		node.stop()
+
+if __name__ == "__main__":
+    unittest.main()
