dtmbench and perf.yml changes

kelvich · kelvich · commit b616354bde44 · 2015-11-12T19:45:30.000+03:00
diff --git a/tests/deploy_layouts/cluster.yml b/tests/deploy_layouts/cluster.yml
@@ -30,9 +30,11 @@
   - name: enable dtm extension on datanodes
     lineinfile:
       dest: "{{pg_datadir}}/postgresql.conf"
-      regexp: "^shared_preload_libraries"
-      line: "shared_preload_libraries = 'pg_dtm'"
+      line: "{{item}}"
       state: present
+    with_items:
+    - "shared_preload_libraries = 'pg_dtm'"
+    - "dtm.host = '{{ groups['nodes'][0] }}'"
 
   - name: restart postgrespro
     command: "{{pg_dst}}/bin/pg_ctl restart -w -D {{pg_datadir}} -l {{pg_datadir}}/pg.log"
diff --git a/tests/dtmbench.cpp b/tests/dtmbench.cpp
@@ -62,15 +62,19 @@ struct config
     int nWriters;
     int nIterations;
     int nAccounts;
+    int startId;
+    int diapason;
     char const* isolationLevel;
     vector<string> connections;
 
     config() {
         nReaders = 1;
         nWriters = 10;
         nIterations = 1000;
-        nAccounts = 1000;      
+        nAccounts = 100000;
         isolationLevel = "read committed";
+        startId = 1;
+        diapason = 100000;
     }
 };
 
@@ -154,23 +158,24 @@ void* reader(void* arg)
 void* writer(void* arg)
 {
     thread& t = *(thread*)arg;
-    vector< unique_ptr<connection> > conns(cfg.connections.size());
-    for (size_t i = 0; i < conns.size(); i++) {
-        conns[i] = new connection(cfg.connections[i]);
-    }
+    connection *srcCon, *dstCon;
+
+    srcCon = new connection(cfg.connections[t.id % cfg.connections.size()]);
+    dstCon = new connection(cfg.connections[(t.id + 1) % cfg.connections.size()]);
+
     for (int i = 0; i < cfg.nIterations; i++)
     { 
-        int srcCon, dstCon;
-        int srcAcc = (random() % ((cfg.nAccounts-cfg.nWriters)/cfg.nWriters))*cfg.nWriters + t.id;
-        int dstAcc = (random() % ((cfg.nAccounts-cfg.nWriters)/cfg.nWriters))*cfg.nWriters + t.id;
-
-        do { 
-            srcCon = random() % cfg.connections.size();
-            dstCon = random() % cfg.connections.size();
-        } while (srcCon == dstCon);
-        
-        nontransaction srcTx(*conns[srcCon]);
-        nontransaction dstTx(*conns[dstCon]);
+        int srcAcc = cfg.startId + random() % cfg.diapason;
+        int dstAcc = cfg.startId + random() % cfg.diapason;
+
+        if (srcAcc > dstAcc) {
+            int tmpAcc = dstAcc;
+            dstAcc = srcAcc;
+            srcAcc = tmpAcc;
+        }
+
+        nontransaction srcTx(*srcCon);
+        nontransaction dstTx(*dstCon);
         
         xid_t xid = execQuery(srcTx, "select dtm_begin_transaction()");
         exec(dstTx, "select dtm_join_transaction(%u)", xid);
@@ -188,13 +193,14 @@ void* writer(void* arg)
             i -= 1;
             continue;
         }
+
         pipeline srcPipe(srcTx);
         pipeline dstPipe(dstTx);
         srcPipe.insert("commit transaction");
         dstPipe.insert("commit transaction");
         srcPipe.complete();
         dstPipe.complete();
-        
+
         t.proceeded += 1;
     }
     return NULL;
@@ -217,6 +223,12 @@ void initializeDatabase()
 int main (int argc, char* argv[])
 {
     bool initialize = false;
+
+    if (argc == 1){
+        printf("Use -h to show usage options\n");
+        return 1;
+    }
+
     for (int i = 1; i < argc; i++) { 
         if (argv[i][0] == '-') { 
             switch (argv[i][1]) { 
@@ -232,11 +244,16 @@ int main (int argc, char* argv[])
             case 'n':
                 cfg.nIterations = atoi(argv[++i]);
                 continue;
-            case 'c':
-                cfg.connections.push_back(string(argv[++i]));
+            case 's':
+                cfg.startId = atoi(argv[++i]);
+                continue;
+            case 'd':
+                cfg.diapason = atoi(argv[++i]);
                 continue;
             case 'l':
                 cfg.isolationLevel = argv[++i];
+            case 'C':
+                cfg.connections.push_back(string(argv[++i]));
                 continue;
             case 'i':
                 initialize = true;
@@ -246,15 +263,26 @@ int main (int argc, char* argv[])
         printf("Options:\n"
                "\t-r N\tnumber of readers (1)\n"
                "\t-w N\tnumber of writers (10)\n"
-               "\t-a N\tnumber of accounts (1000)\n"
+               "\t-a N\tnumber of accounts (100000)\n"
+               "\t-s N\tperform updates starting from this id (1)\n"
+               "\t-d N\tperform updates in this diapason (100000)\n"
                "\t-n N\tnumber of iterations (1000)\n"
                "\t-l STR\tisolation level (read committed)\n"
                "\t-c STR\tdatabase connection string\n"
+               "\t-C STR\tdatabase connection string\n"
                "\t-i\tinitialize datanase\n");
         return 1;
     }
+
+    if (cfg.startId + cfg.diapason - 1 > cfg.nAccounts) {
+        printf("startId + diapason should be less that nAccounts. Exiting.\n");
+        return 1;
+    }
+
     if (initialize) { 
         initializeDatabase();
+        printf("%d account inserted\n", cfg.nAccounts);
+        return 0;
     }
 
     time_t start = getCurrentTime();
@@ -287,6 +315,19 @@ int main (int argc, char* argv[])
     }
  
     time_t elapsed = getCurrentTime() - start;
-    printf("TPS(updates)=%f, TPS(selects)=%f, aborts=%ld\n", (double)(nWrites*USEC)/elapsed, (double)(nReads*USEC)/elapsed, nAborts);
+
+    printf(
+        "{\"update_tps\":%f, \"read_tps\":%f,"
+        " \"readers\":%d, \"writers\":%d,"
+        " \"accounts\":%d, \"iterations\":%d, \"hosts\":%d}\n",
+        (double)(nWrites*USEC)/elapsed,
+        (double)(nReads*USEC)/elapsed,
+        cfg.nReaders,
+        cfg.nWriters,
+        cfg.nAccounts,
+        cfg.nIterations,
+        cfg.connections.size()
+        );
+
     return 0;
 }
diff --git a/tests/perf.yml b/tests/perf.yml
@@ -6,7 +6,7 @@
 
   - name: generate connstrings
     set_fact:
-      connstr: "-d 'host={{item}} user=cluster port=15432 dbname=postgres' "
+      connstr: "-C 'host={{item}} user={{ansible_ssh_user}} port=15432 dbname=postgres' "
     with_items:
       groups['nodes'] | reverse | batch(nnodes | d(2) | int) | first
     register: connstrs
@@ -15,22 +15,44 @@
     set_fact:
       connections: "{{ connstrs.results | map(attribute='ansible_facts.connstr') | join }}"
 
-  - name: copy transfers binary
-    copy: src=transfers.linux dest=~/transfers mode=a+x
+  - name: copy transfers source
+    copy: src=./{{item}} dest=~/{{item}} mode=0755
+    with_items:
+      - "dtmbench.cpp"
+
+  - name: compile dtmbench
+    shell: "g++ -g -Wall -O2 -o dtmbench dtmbench.cpp -lpqxx -lpq -pthread"
+
+  - name: compile dtmbench
+    shell: "mv dtmbench ~/pg_cluster/install/bin/dtmbench"
 
 - hosts: clients[0]
   gather_facts: no
   tasks:
-  - name: fill the databases
-    shell: "./transfers {{connections}} -f -g"
-    register: transfers_result
-  - debug: "var=transfers_result"
+  - name: init database
+    shell: "~/pg_cluster/install/bin/dtmbench {{connections}} -a 2000000 -i"
+    register: init_result
+  - debug: var=init_result
 
 - hosts: clients
   gather_facts: no
   tasks:
+
+  - local_action: shell echo "Bench started at `date`" >> perf.results
+
   - name: run transfers
-    shell: "./transfers {{connections}} -s {{runkeys | d('-g -w 60 -m -o -u 10000 -r 0')}} -k {{hostvars[inventory_hostname]['offset']}} "
+    shell: >
+      ~/pg_cluster/install/bin/dtmbench {{connections}}
+      -w {{ (nconns | d(100)| int)*(nnodes | d(2) | int)/(2*( groups['clients'] | count))}}
+      -s {{offset}} -d 100000 -r 1 -n 20000 -a 500000 |
+      tee -a perf.results |
+      sed "s/^/`hostname`:/"
     register: transfers_result
+    environment:
+      LD_LIBRARY_PATH: "/home/{{ansible_ssh_user}}/pg_cluster/install/lib"
+
   - debug: var=transfers_result
 
+  - local_action: 'shell echo "{{transfers_result.stdout }}" >> perf.results'
+
+  - local_action: shell echo "Bench finished at `date`" >> perf.results
