[taler-grid5k] 159/189: 'cluster' (db) scripts added

[gnunet]

This is an automated email from the git hooks/post-receive script.

marco-boss pushed a commit to branch master
in repository grid5k.

commit aa59701ad4d4ff8ca1e56125d22eeb20f6afff49
Author: Boss Marco <bossm8@bfh.ch>
AuthorDate: Tue Apr 19 22:44:37 2022 +0200

    'cluster' (db) scripts added
---
 additional/grafana/load-statistics.json | 437 +++++++++++++++++++++-----------
 experiment/scripts/benchmark.sh         |   7 +-
 experiment/scripts/database.sh          |   7 +-
 experiment/scripts/db-cluster.sh        |  56 ++++
 experiment/scripts/exch-cluster.sh      |  58 +++++
 experiment/scripts/exchange.sh          |  17 +-
 experiment/scripts/proxy-cluster.sh     |   7 +
 7 files changed, 429 insertions(+), 160 deletions(-)

diff --git a/additional/grafana/load-statistics.json 
b/additional/grafana/load-statistics.json
index 59bddda..69f0244 100644
--- a/additional/grafana/load-statistics.json
+++ b/additional/grafana/load-statistics.json
@@ -23,7 +23,7 @@
       "type": "grafana",
       "id": "grafana",
       "name": "Grafana",
-      "version": "8.4.3"
+      "version": "8.4.6"
     },
     {
       "type": "datasource",
@@ -335,13 +335,12 @@
         "x": 0,
         "y": 10
       },
-      "id": 21,
+      "id": 15,
       "panels": [],
-      "title": "Wallet",
+      "title": "Database",
       "type": "row"
     },
     {
-      "description": "Number of requests versus average wallet node CPU usage",
       "fieldConfig": {
         "defaults": {
           "color": {
@@ -423,7 +422,7 @@
         "x": 0,
         "y": 11
       },
-      "id": 19,
+      "id": 3,
       "maxDataPoints": 500,
       "options": {
         "legend": {
@@ -444,9 +443,7 @@
           },
           "exemplar": true,
           "expr": "sum(rate(nginx_http_requests_total{} [$__rate_interval]))",
-          "instant": false,
           "interval": "",
-          "intervalFactor": 1,
           "legendFormat": "Requests",
           "refId": "A"
         },
@@ -456,31 +453,18 @@
             "uid": "${DS_PROMETHEUS}"
           },
           "exemplar": true,
-          "expr": "100 - (avg(rate(node_cpu_seconds_total{mode=\"idle\", 
component=\"wallet\"}[$__rate_interval]))*100)",
+          "expr": "100 - (avg by (instance) 
(rate(node_cpu_seconds_total{mode=\"idle\", 
component=\"database\"}[$__rate_interval]))*100)",
           "hide": false,
           "interval": "",
-          "intervalFactor": 1,
           "legendFormat": "CPU Busy",
           "refId": "B"
         }
       ],
-      "title": " [Wallet] Requests vs Average CPU",
+      "title": "[Database] Requests vs CPU",
       "type": "timeseries"
     },
     {
-      "collapsed": false,
-      "gridPos": {
-        "h": 1,
-        "w": 24,
-        "x": 0,
-        "y": 20
-      },
-      "id": 15,
-      "panels": [],
-      "title": "Database",
-      "type": "row"
-    },
-    {
+      "description": "",
       "fieldConfig": {
         "defaults": {
           "color": {
@@ -529,7 +513,7 @@
           {
             "matcher": {
               "id": "byRegexp",
-              "options": "CPU Busy"
+              "options": "Network (.*)"
             },
             "properties": [
               {
@@ -538,7 +522,7 @@
               },
               {
                 "id": "unit",
-                "value": "percent"
+                "value": "bps"
               }
             ]
           },
@@ -559,10 +543,10 @@
       "gridPos": {
         "h": 9,
         "w": 12,
-        "x": 0,
-        "y": 21
+        "x": 12,
+        "y": 11
       },
-      "id": 3,
+      "id": 8,
       "maxDataPoints": 500,
       "options": {
         "legend": {
@@ -577,30 +561,24 @@
       },
       "targets": [
         {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "${DS_PROMETHEUS}"
-          },
           "exemplar": true,
           "expr": "sum(rate(nginx_http_requests_total{} [$__rate_interval]))",
           "interval": "",
+          "intervalFactor": 1,
           "legendFormat": "Requests",
           "refId": "A"
         },
         {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "${DS_PROMETHEUS}"
-          },
           "exemplar": true,
-          "expr": "100 - (avg by (instance) 
(rate(node_cpu_seconds_total{mode=\"idle\", 
component=\"database\"}[$__rate_interval]))*100)",
+          "expr": 
"sum(rate(node_network_receive_bytes_total{component=\"database\"}[$__rate_interval])
 + rate(node_network_transmit_bytes_total{component=\"database\"} 
[$__rate_interval]))*8",
           "hide": false,
           "interval": "",
-          "legendFormat": "CPU Busy",
+          "intervalFactor": 1,
+          "legendFormat": "Network Bandwidth",
           "refId": "B"
         }
       ],
-      "title": "[Database] Requests vs CPU",
+      "title": "[Database] Requests vs Network Traffic",
       "type": "timeseries"
     },
     {
@@ -652,8 +630,8 @@
         "overrides": [
           {
             "matcher": {
-              "id": "byRegexp",
-              "options": "Network (.*)"
+              "id": "byName",
+              "options": "Database Size"
             },
             "properties": [
               {
@@ -662,10 +640,119 @@
               },
               {
                 "id": "unit",
-                "value": "bps"
+                "value": "bytes"
               }
             ]
           },
+          {
+            "matcher": {
+              "id": "byName",
+              "options": "Total Requests"
+            },
+            "properties": [
+              {
+                "id": "unit",
+                "value": "short"
+              }
+            ]
+          }
+        ]
+      },
+      "gridPos": {
+        "h": 9,
+        "w": 12,
+        "x": 0,
+        "y": 20
+      },
+      "id": 5,
+      "maxDataPoints": 500,
+      "options": {
+        "legend": {
+          "calcs": [],
+          "displayMode": "list",
+          "placement": "bottom"
+        },
+        "tooltip": {
+          "mode": "single",
+          "sort": "none"
+        }
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${DS_PROMETHEUS}"
+          },
+          "exemplar": true,
+          "expr": "sum(rate(nginx_http_requests_total{} [$__rate_interval]))",
+          "interval": "",
+          "intervalFactor": 1,
+          "legendFormat": "Requests",
+          "refId": "A"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${DS_PROMETHEUS}"
+          },
+          "exemplar": true,
+          "expr": "sum(pg_database_size_bytes{instance=\"127.0.0.1:9187\"})",
+          "hide": false,
+          "interval": "",
+          "legendFormat": "Database Size",
+          "refId": "C"
+        }
+      ],
+      "title": "[Database] Total Requests vs Disk Space used",
+      "type": "timeseries"
+    },
+    {
+      "description": "",
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "drawStyle": "line",
+            "fillOpacity": 0,
+            "gradientMode": "none",
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            },
+            "lineInterpolation": "linear",
+            "lineWidth": 2,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "auto",
+            "spanNulls": false,
+            "stacking": {
+              "group": "A",
+              "mode": "none"
+            },
+            "thresholdsStyle": {
+              "mode": "off"
+            }
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              }
+            ]
+          }
+        },
+        "overrides": [
           {
             "matcher": {
               "id": "byName",
@@ -677,6 +764,22 @@
                 "value": "Requests / Second"
               }
             ]
+          },
+          {
+            "matcher": {
+              "id": "byRegexp",
+              "options": "I/O Utilization(.*)"
+            },
+            "properties": [
+              {
+                "id": "custom.axisPlacement",
+                "value": "right"
+              },
+              {
+                "id": "unit",
+                "value": "percentunit"
+              }
+            ]
           }
         ]
       },
@@ -684,9 +787,9 @@
         "h": 9,
         "w": 12,
         "x": 12,
-        "y": 21
+        "y": 20
       },
-      "id": 8,
+      "id": 9,
       "maxDataPoints": 500,
       "options": {
         "legend": {
@@ -701,6 +804,10 @@
       },
       "targets": [
         {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${DS_PROMETHEUS}"
+          },
           "exemplar": true,
           "expr": "sum(rate(nginx_http_requests_total{} [$__rate_interval]))",
           "interval": "",
@@ -709,16 +816,19 @@
           "refId": "A"
         },
         {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${DS_PROMETHEUS}"
+          },
           "exemplar": true,
-          "expr": 
"sum(rate(node_network_receive_bytes_total{component=\"database\"}[$__rate_interval])
 + rate(node_network_transmit_bytes_total{component=\"database\"} 
[$__rate_interval]))*8",
+          "expr": "sum by (device) 
(rate(node_disk_io_time_seconds_total{component=\"database\"} 
[$__rate_interval]))",
           "hide": false,
           "interval": "",
-          "intervalFactor": 1,
-          "legendFormat": "Network Bandwidth",
-          "refId": "B"
+          "legendFormat": "I/O Utilization ({{device}})",
+          "refId": "C"
         }
       ],
-      "title": "[Database] Requests vs Network Traffic",
+      "title": "[Database] Requests vs I/O Utilization",
       "type": "timeseries"
     },
     {
@@ -771,7 +881,7 @@
           {
             "matcher": {
               "id": "byName",
-              "options": "Database Size"
+              "options": "Cache Hit Rate"
             },
             "properties": [
               {
@@ -780,19 +890,19 @@
               },
               {
                 "id": "unit",
-                "value": "bytes"
+                "value": "percentunit"
               }
             ]
           },
           {
             "matcher": {
               "id": "byName",
-              "options": "Total Requests"
+              "options": "Requests"
             },
             "properties": [
               {
-                "id": "unit",
-                "value": "short"
+                "id": "custom.axisLabel",
+                "value": "Requests / Second"
               }
             ]
           }
@@ -802,9 +912,9 @@
         "h": 9,
         "w": 12,
         "x": 0,
-        "y": 30
+        "y": 29
       },
-      "id": 5,
+      "id": 10,
       "maxDataPoints": 500,
       "options": {
         "legend": {
@@ -820,7 +930,7 @@
       "targets": [
         {
           "exemplar": true,
-          "expr": "nginx_http_requests_total{}",
+          "expr": "sum(rate(nginx_http_requests_total{} [$__rate_interval]))",
           "interval": "",
           "intervalFactor": 1,
           "legendFormat": "Requests",
@@ -828,17 +938,21 @@
         },
         {
           "exemplar": true,
-          "expr": "sum(pg_database_size_bytes{instance=\"127.0.0.1:9187\"})",
+          "expr": "pg_stat_database_blks_hit{instance=\"127.0.0.1:9187\", 
server=\"db.perf.taler:5432\", datname!~\"postgres|template.*|\"} / 
(pg_stat_database_blks_read{instance=\"127.0.0.1:9187\", 
server=\"db.perf.taler:5432\", datname!~\"postgres|template.*|\"} + 
pg_stat_database_blks_hit{instance=\"127.0.0.1:9187\", 
server=\"db.perf.taler:5432\", datname!~\"postgres|template.*|\"})",
           "hide": false,
           "interval": "",
-          "legendFormat": "Database Size",
+          "legendFormat": "Cache Hit Rate",
           "refId": "C"
         }
       ],
-      "title": "Total Requests vs Disk Space used",
+      "title": "[Database] Requests vs Cache Hit Rate",
       "type": "timeseries"
     },
     {
+      "datasource": {
+        "type": "datasource",
+        "uid": "-- Mixed --"
+      },
       "description": "",
       "fieldConfig": {
         "defaults": {
@@ -888,28 +1002,42 @@
           {
             "matcher": {
               "id": "byName",
-              "options": "Requests"
+              "options": "Slow Queries"
             },
             "properties": [
+              {
+                "id": "custom.axisPlacement",
+                "value": "right"
+              },
               {
                 "id": "custom.axisLabel",
-                "value": "Requests / Second"
+                "value": "Slow Queries / Second"
+              },
+              {
+                "id": "unit",
+                "value": "none"
+              },
+              {
+                "id": "custom.lineStyle",
+                "value": {
+                  "fill": "solid"
+                }
+              },
+              {
+                "id": "custom.fillOpacity",
+                "value": 15
               }
             ]
           },
           {
             "matcher": {
-              "id": "byRegexp",
-              "options": "I/O Utilization(.*)"
+              "id": "byName",
+              "options": "Requests"
             },
             "properties": [
               {
-                "id": "custom.axisPlacement",
-                "value": "right"
-              },
-              {
-                "id": "unit",
-                "value": "percentunit"
+                "id": "custom.axisLabel",
+                "value": "Requests / Second"
               }
             ]
           }
@@ -919,9 +1047,9 @@
         "h": 9,
         "w": 12,
         "x": 12,
-        "y": 30
+        "y": 29
       },
-      "id": 9,
+      "id": 11,
       "maxDataPoints": 500,
       "options": {
         "legend": {
@@ -949,22 +1077,20 @@
         },
         {
           "datasource": {
-            "type": "prometheus",
-            "uid": "${DS_PROMETHEUS}"
+            "type": "loki",
+            "uid": "${DS_LOKI}"
           },
-          "exemplar": true,
-          "expr": "sum by (device) 
(rate(node_disk_io_time_seconds_total{component=\"database\"} 
[$__rate_interval]))",
+          "expr": "rate({app=\"taler-database\", type=\"slow-query\"} [1m])",
           "hide": false,
-          "interval": "",
-          "legendFormat": "I/O Utilization ({{device}})",
+          "legendFormat": "Slow Queries",
           "refId": "C"
         }
       ],
-      "title": "Requests vs I/O Utilization",
+      "title": "[Database] Requests vs Slow Queries",
       "type": "timeseries"
     },
     {
-      "description": "",
+      "description": "Serialization Errors per Second and Request-Type",
       "fieldConfig": {
         "defaults": {
           "color": {
@@ -976,7 +1102,7 @@
             "barAlignment": 0,
             "drawStyle": "line",
             "fillOpacity": 0,
-            "gradientMode": "none",
+            "gradientMode": "opacity",
             "hideFrom": {
               "legend": false,
               "tooltip": false,
@@ -1007,13 +1133,14 @@
                 "value": null
               }
             ]
-          }
+          },
+          "unit": "none"
         },
         "overrides": [
           {
             "matcher": {
               "id": "byName",
-              "options": "Cache Hit Rate"
+              "options": "Serialization Errors"
             },
             "properties": [
               {
@@ -1021,8 +1148,11 @@
                 "value": "right"
               },
               {
-                "id": "unit",
-                "value": "percentunit"
+                "id": "custom.axisLabel",
+                "value": "Serialization Errors / Second"
+              },
+              {
+                "id": "unit"
               }
             ]
           },
@@ -1043,11 +1173,10 @@
       "gridPos": {
         "h": 9,
         "w": 12,
-        "x": 0,
-        "y": 39
+        "x": 6,
+        "y": 38
       },
-      "id": 10,
-      "maxDataPoints": 500,
+      "id": 18,
       "options": {
         "legend": {
           "calcs": [],
@@ -1059,33 +1188,44 @@
           "sort": "none"
         }
       },
+      "pluginVersion": "8.2.5",
       "targets": [
         {
           "exemplar": true,
           "expr": "sum(rate(nginx_http_requests_total{} [$__rate_interval]))",
+          "hide": false,
           "interval": "",
-          "intervalFactor": 1,
           "legendFormat": "Requests",
-          "refId": "A"
+          "refId": "B"
         },
         {
           "exemplar": true,
-          "expr": "pg_stat_database_blks_hit{instance=\"127.0.0.1:9187\", 
server=\"db.perf.taler:5432\", datname!~\"postgres|template.*|\"} / 
(pg_stat_database_blks_read{instance=\"127.0.0.1:9187\", 
server=\"db.perf.taler:5432\", datname!~\"postgres|template.*|\"} + 
pg_stat_database_blks_hit{instance=\"127.0.0.1:9187\", 
server=\"db.perf.taler:5432\", datname!~\"postgres|template.*|\"})",
-          "hide": false,
+          "expr": "sum (rate(taler_exchange_serialization_failures{}[1m]))",
+          "instant": false,
           "interval": "",
-          "legendFormat": "Cache Hit Rate",
-          "refId": "C"
+          "legendFormat": "Serialization Errors",
+          "refId": "A"
         }
       ],
-      "title": "Requests vs Cache Hit Rate",
+      "title": "[Database] Requests vs Serialization Errors",
+      "transformations": [],
       "type": "timeseries"
     },
     {
-      "datasource": {
-        "type": "datasource",
-        "uid": "-- Mixed --"
+      "collapsed": false,
+      "gridPos": {
+        "h": 1,
+        "w": 24,
+        "x": 0,
+        "y": 47
       },
-      "description": "",
+      "id": 21,
+      "panels": [],
+      "title": "Others",
+      "type": "row"
+    },
+    {
+      "description": "Number of requests versus average wallet node CPU usage",
       "fieldConfig": {
         "defaults": {
           "color": {
@@ -1133,31 +1273,17 @@
         "overrides": [
           {
             "matcher": {
-              "id": "byName",
-              "options": "Slow Queries"
+              "id": "byRegexp",
+              "options": "CPU Busy"
             },
             "properties": [
               {
                 "id": "custom.axisPlacement",
                 "value": "right"
               },
-              {
-                "id": "custom.axisLabel",
-                "value": "Slow Queries / Second"
-              },
               {
                 "id": "unit",
-                "value": "none"
-              },
-              {
-                "id": "custom.lineStyle",
-                "value": {
-                  "fill": "solid"
-                }
-              },
-              {
-                "id": "custom.fillOpacity",
-                "value": 15
+                "value": "percent"
               }
             ]
           },
@@ -1178,10 +1304,10 @@
       "gridPos": {
         "h": 9,
         "w": 12,
-        "x": 12,
-        "y": 39
+        "x": 0,
+        "y": 48
       },
-      "id": 11,
+      "id": 19,
       "maxDataPoints": 500,
       "options": {
         "legend": {
@@ -1202,6 +1328,7 @@
           },
           "exemplar": true,
           "expr": "sum(rate(nginx_http_requests_total{} [$__rate_interval]))",
+          "instant": false,
           "interval": "",
           "intervalFactor": 1,
           "legendFormat": "Requests",
@@ -1209,20 +1336,23 @@
         },
         {
           "datasource": {
-            "type": "loki",
-            "uid": "${DS_LOKI}"
+            "type": "prometheus",
+            "uid": "${DS_PROMETHEUS}"
           },
-          "expr": "rate({app=\"taler-database\", type=\"slow-query\"} [1m])",
+          "exemplar": true,
+          "expr": "100 - (avg(rate(node_cpu_seconds_total{mode=\"idle\", 
component=\"wallet\"}[$__rate_interval]))*100)",
           "hide": false,
-          "legendFormat": "Slow Queries",
-          "refId": "C"
+          "interval": "",
+          "intervalFactor": 1,
+          "legendFormat": "CPU Busy",
+          "refId": "B"
         }
       ],
-      "title": "Requests vs Slow Queries",
+      "title": " [Wallet] Requests vs Average CPU",
       "type": "timeseries"
     },
     {
-      "description": "Serialization Errors per Second and Request-Type",
+      "description": "Number of requests versus average exchange proxy node 
CPU usage",
       "fieldConfig": {
         "defaults": {
           "color": {
@@ -1234,7 +1364,7 @@
             "barAlignment": 0,
             "drawStyle": "line",
             "fillOpacity": 0,
-            "gradientMode": "opacity",
+            "gradientMode": "none",
             "hideFrom": {
               "legend": false,
               "tooltip": false,
@@ -1265,14 +1395,13 @@
                 "value": null
               }
             ]
-          },
-          "unit": "none"
+          }
         },
         "overrides": [
           {
             "matcher": {
-              "id": "byName",
-              "options": "Serialization Errors"
+              "id": "byRegexp",
+              "options": "CPU Busy"
             },
             "properties": [
               {
@@ -1280,11 +1409,8 @@
                 "value": "right"
               },
               {
-                "id": "custom.axisLabel",
-                "value": "Serialization Errors / Second"
-              },
-              {
-                "id": "unit"
+                "id": "unit",
+                "value": "percent"
               }
             ]
           },
@@ -1305,10 +1431,11 @@
       "gridPos": {
         "h": 9,
         "w": 12,
-        "x": 0,
+        "x": 12,
         "y": 48
       },
-      "id": 18,
+      "id": 22,
+      "maxDataPoints": 500,
       "options": {
         "legend": {
           "calcs": [],
@@ -1320,27 +1447,35 @@
           "sort": "none"
         }
       },
-      "pluginVersion": "8.2.5",
       "targets": [
         {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${DS_PROMETHEUS}"
+          },
           "exemplar": true,
           "expr": "sum(rate(nginx_http_requests_total{} [$__rate_interval]))",
-          "hide": false,
+          "instant": false,
           "interval": "",
+          "intervalFactor": 1,
           "legendFormat": "Requests",
-          "refId": "B"
+          "refId": "A"
         },
         {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${DS_PROMETHEUS}"
+          },
           "exemplar": true,
-          "expr": "sum (rate(taler_exchange_serialization_failures{}[1m]))",
-          "instant": false,
+          "expr": "100 - (avg(rate(node_cpu_seconds_total{mode=\"idle\", 
component=\"proxy\"}[$__rate_interval]))*100)",
+          "hide": false,
           "interval": "",
-          "legendFormat": "Serialization Errors",
-          "refId": "A"
+          "intervalFactor": 1,
+          "legendFormat": "CPU Busy",
+          "refId": "B"
         }
       ],
-      "title": "Requests vs Serialization Errors",
-      "transformations": [],
+      "title": " [Exch Proxy] Requests vs Average CPU",
       "type": "timeseries"
     },
     {
@@ -1431,8 +1566,8 @@
       "gridPos": {
         "h": 9,
         "w": 12,
-        "x": 12,
-        "y": 48
+        "x": 6,
+        "y": 57
       },
       "id": 16,
       "options": {
@@ -1493,6 +1628,6 @@
   "timezone": "",
   "title": "Load Statistics",
   "uid": "rkyhDAt7z",
-  "version": 63,
+  "version": 70,
   "weekStart": ""
 }
\ No newline at end of file
diff --git a/experiment/scripts/benchmark.sh b/experiment/scripts/benchmark.sh
index e37f15c..b02dae8 100755
--- a/experiment/scripts/benchmark.sh
+++ b/experiment/scripts/benchmark.sh
@@ -25,10 +25,13 @@ function start_wallet_bench() {
     LOG_LEVEL=INFO
   fi
 
-  if (($1 % 2)); then
+  R=$(shuf -i 1-3 -n 1)
+  if [[ $(($R % 3)) == 0 ]]; then
     DOMAIN=eproxy-1.perf.taler
-  else
+  elif [[ $(($R % 3)) == 1 ]]; then
     DOMAIN=eproxy-2.perf.taler
+  else
+    DOMAIN=eproxy-3.perf.taler
   fi
 
   NUM_DEPOSITS=$((RANDOM %21))
diff --git a/experiment/scripts/database.sh b/experiment/scripts/database.sh
index 2c26367..1766fd6 100755
--- a/experiment/scripts/database.sh
+++ b/experiment/scripts/database.sh
@@ -133,8 +133,11 @@ function setup_config() {
 
   # use 25% of the available memory 
   # (https://www.postgresql.org/docs/13/runtime-config-resource.html)
-  shared_buffers=${SHARED_MEM}kB
-  effective_cache_size=${CACHE_SIZE}kB
+ # shared_buffers=${SHARED_MEM}kB
+ # effective_cache_size=${CACHE_SIZE}kB
+  shared_buffers=24551953kB
+  effective_cache_size=75491895kB
+
 
   huge_pages=on
   
diff --git a/experiment/scripts/db-cluster.sh b/experiment/scripts/db-cluster.sh
new file mode 100755
index 0000000..51226b5
--- /dev/null
+++ b/experiment/scripts/db-cluster.sh
@@ -0,0 +1,56 @@
+#!/bin/bash
+
+PORT=$((5432 + ${1}))
+
+pg_createcluster 13 main${1} -p $PORT
+
+mv /var/lib/postgresql/13/main${1} /tmp/postgresql/13/
+
+if [ -d /mnt/disk ]; then
+  mv /tmp/postgresql/13/main${1}/pg_wal /mnt/disk/pg_wal${1}
+  ln -sf /mnt/disk/pg_wal${1} /tmp/postgresql/13/main${1}/pg_wal
+  chown postgres:postgres /tmp/postgresql/13/main${1}/pg_wal
+fi
+
+cp /etc/postgresql/13/main/exchange.conf /etc/postgresql/13/main/pg_hba.conf 
/etc/postgresql/13/main${1}
+sed -i "s|/tmp/postgresql/13/main|/tmp/postgresql/13/main${1}|g" 
/etc/postgresql/13/main${1}/exchange.conf
+echo "include = 'exchange.conf'" >> /etc/postgresql/13/main${1}/postgresql.conf
+
+systemctl restart postgresql@13-main${1}.service
+
+echo "
+[exchangedb-postgres]
+CONFIG=postgres://:${PORT}/${DB_NAME}
+" > /etc/taler/secrets/exchange-db.secret.conf
+
+su postgres << EOF
+psql postgres -p ${PORT} -tAc "DROP DATABASE IF EXISTS \"${DB_NAME}\";"
+psql postgres -p ${PORT} -tAc "SELECT 1 FROM pg_roles WHERE 
rolname='taler-exchange-httpd'" | \
+  grep -q 1 || \
+  createuser -p ${PORT} taler-exchange-httpd
+psql -p ${PORT} -tAc "SELECT 1 FROM pg_database WHERE datname='${DB_NAME}'" | \
+  grep -q 1 || \
+  createdb -p ${PORT} -O taler-exchange-httpd "${DB_NAME}"
+psql -p ${PORT} -tAc "CREATE EXTENSION IF NOT EXISTS pg_stat_statements"
+EOF
+
+su postgres << EOF
+psql postgres -p ${PORT} -tAc "SELECT 1 FROM pg_roles WHERE 
rolname='${DB_USER}'" | \
+  grep -q 1 || \
+  psql -p ${PORT} << END
+    CREATE USER "${DB_USER}" with encrypted password '${DB_PASSWORD}';
+END
+EOF
+
+sudo -u taler-exchange-httpd taler-exchange-dbinit -r || true
+sudo -u taler-exchange-httpd taler-exchange-dbinit -s || true
+sudo -u taler-exchange-httpd taler-exchange-dbinit -P ${NUM_PARTITIONS}
+
+su taler-exchange-httpd -s /bin/bash << EOF
+psql -p ${PORT} -d "${DB_NAME}"
+GRANT SELECT,INSERT,UPDATE ON ALL TABLES IN SCHEMA public TO "${DB_USER}";
+GRANT USAGE ON ALL SEQUENCES IN SCHEMA public TO "${DB_USER}";
+EOF
+
+ssh -o StrictHostKeyChecking=no monitor.${DNS_ZONE} \
+       "sed -i \"s/DATA_SOURCE_NAME.*'$//\" 
/etc/default/prometheus-postgres-exporter && sed -i 
\"s|DATA_SOURCE_NAME.*|&,postgresql://postgres@db.${DNS_ZONE}:${PORT}'|\" 
/etc/default/prometheus-postgres-exporter && systemctl restart 
prometheus-postgres-exporter" 
diff --git a/experiment/scripts/exch-cluster.sh 
b/experiment/scripts/exch-cluster.sh
new file mode 100755
index 0000000..e941c2d
--- /dev/null
+++ b/experiment/scripts/exch-cluster.sh
@@ -0,0 +1,58 @@
+#!/bin/bash
+
+source /root/scripts/helpers.sh
+
+DB_PORT=$((5432 + ${1}))
+
+systemctl stop taler-exchange-*
+
+sed -i -e "s/exchange-secmod-rsa/exchange-secmod-rsa-${1}/g" \
+       -e "s/exchange-secmod-cs/exchange-secmod-cs-${1}/g" \
+       -e "s/exchange-secmod-eddsa/exchange-secmod-eddsa-${1}/g" \
+       /etc/taler/conf.d/exchange-secmod.conf
+
+echo "
+[exchangedb-postgres]
+CONFIG=postgresql://${DB_USER}:${DB_PASSWORD}@db.${DNS_ZONE}:${DB_PORT}/${DB_NAME}
+" > /etc/taler/secrets/exchange-db.secret.conf
+
+rm -rf /var/lib/taler/exchange-offline/*
+
+MASTER_KEY=$(sudo -u taler-exchange-offline taler-exchange-offline setup)
+
+sed -i -e "s/MASTER_PUBLIC_KEY.*/MASTER_PUBLIC_KEY = ${MASTER_KEY}/g" \
+       /etc/taler/conf.d/exchange-business.conf
+
+# Setup the shared key directory when we use a secondary node
+if [[ ${NUM_EXCHANGES} != "1" ]]; then
+  rm -rf /home/${G5K_USER}/taler/exchange-secmod-{cs,rsa,eddsa}-${1} || true
+  mkdir -p /home/${G5K_USER}/taler/exchange-secmod-{cs,rsa,eddsa}-${1}
+fi
+
+systemctl restart taler-exchange-httpd@10000.service
+
+wait_for_keys "${NODE_NAME}.${DNS_ZONE}:10000/management"
+
+sleep 5
+
+taler-exchange-offline download > sig-req.json
+taler-exchange-offline sign < sig-req.json > sig-res.json
+taler-exchange-offline enable-account 
"payto://x-taler-bank/bank.${DNS_ZONE}/Exchange" > acct-res.json
+taler-exchange-offline wire-fee $(date +%Y) x-taler-bank KUDOS:0 KUDOS:0 
KUDOS:0 > fee-res.json
+taler-exchange-offline upload < sig-res.json
+taler-exchange-offline upload < acct-res.json
+taler-exchange-offline upload < fee-res.json
+
+source ~/scripts/exchange-wirewatch.sh init-start
+source ~/scripts/exchange-aggregator.sh init-start
+source ~/scripts/exchange-closer.sh init-start
+source ~/scripts/exchange-transfer.sh init-start
+
+let "START=1+10000"
+let "END=$START+$((${NUM_EXCHANGE_PROCESSES}-1))-1"
+
+for PORT in $(seq $START $END); do
+  systemctl restart taler-exchange-httpd@"${PORT}".socket \
+                    taler-exchange-httpd@"${PORT}".service
+  sleep 0.05
+done
diff --git a/experiment/scripts/exchange.sh b/experiment/scripts/exchange.sh
index 6db2c23..12a5f8d 100755
--- a/experiment/scripts/exchange.sh
+++ b/experiment/scripts/exchange.sh
@@ -3,13 +3,20 @@ INFO_MSG="
 Setup the Exchange node
 Start taler-exchange-httpd 
 
-Each exchange-http daemon, will get its own port starting from 10001, 
-unless the first, this one will get port 80
+Each exchange-http daemon, will get its own port starting from 10000
+Detects it there are seperate nodes for aggregator and co., if not
+the processes are started too.
 "
 OPT_MSG="
-init:
-  Initialize the applications and start them
-  uses NUM_EXCHANGE_PROCESSES
+init-primary:
+  Initialize the 'master exchange' and start NUM_EXCHANGE_PROCESSES
+  exchange processes
+  Sets up key material on the NFS to use for the secondary exchanges
+
+init-secondary:
+  Initialize the 'slave exchange' and start the processes
+  Waits until the master is done and then starts
+  NUM_EXCHANGE_PROCESSES exchange processes.
 
 start NUM:
   Start another NUM exchange-http daemons
diff --git a/experiment/scripts/proxy-cluster.sh 
b/experiment/scripts/proxy-cluster.sh
new file mode 100755
index 0000000..85f5ab2
--- /dev/null
+++ b/experiment/scripts/proxy-cluster.sh
@@ -0,0 +1,7 @@
+#!/bin/bash
+
+for i in $@; do
+  sed -i "/exchange-${i}.${DNS_ZONE}/d" /etc/nginx/sites-enabled/exchange
+done
+
+systemctl reload nginx

-- 
To stop receiving notification emails like this one, please contact
gnunet@gnunet.org.