[Date Prev][Date Next][Thread Prev][Thread Next][Date Index][Thread Index]
[taler-grid5k] 137/189: fix some issues in recover
|
From: |
gnunet |
|
Subject: |
[taler-grid5k] 137/189: fix some issues in recover |
|
Date: |
Thu, 28 Apr 2022 10:48:27 +0200 |
This is an automated email from the git hooks/post-receive script.
marco-boss pushed a commit to branch master
in repository grid5k.
commit 9f5db12780ab0f442a56e4cc2afb32baa8a9ccfa
Author: Boss Marco <bossm8@bfh.ch>
AuthorDate: Fri Apr 8 21:16:48 2022 +0200
fix some issues in recover
---
additional/grafana/load-statistics.json | 1221 ++++++++++++++++++++++---------
additional/recover/docker-compose.yaml | 2 +-
configs/etc/monitor/loki.yaml | 2 +-
experiment/scripts/data-backup.sh | 4 +-
4 files changed, 886 insertions(+), 343 deletions(-)
diff --git a/additional/grafana/load-statistics.json
b/additional/grafana/load-statistics.json
index b2f361c..59bddda 100644
--- a/additional/grafana/load-statistics.json
+++ b/additional/grafana/load-statistics.json
@@ -19,12 +19,6 @@
],
"__elements": [],
"__requires": [
- {
- "type": "panel",
- "id": "bargauge",
- "name": "Bar gauge",
- "version": ""
- },
{
"type": "grafana",
"id": "grafana",
@@ -43,18 +37,6 @@
"name": "Prometheus",
"version": "1.0.0"
},
- {
- "type": "panel",
- "id": "stat",
- "name": "Stat",
- "version": ""
- },
- {
- "type": "panel",
- "id": "table",
- "name": "Table",
- "version": ""
- },
{
"type": "panel",
"id": "timeseries",
@@ -96,20 +78,19 @@
"x": 0,
"y": 0
},
- "id": 18,
+ "id": 13,
"panels": [],
- "title": "Cryptographic Operations",
+ "title": "Exchange",
"type": "row"
},
{
- "description": "Number of signatures created with the configured cipher
of the exchange (rsa or cs).",
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
- "axisLabel": "Signatures per Second",
+ "axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "line",
@@ -121,7 +102,7 @@
"viz": false
},
"lineInterpolation": "linear",
- "lineWidth": 1,
+ "lineWidth": 2,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
@@ -145,18 +126,47 @@
"value": null
}
]
- },
- "unit": "none"
+ }
},
- "overrides": []
+ "overrides": [
+ {
+ "matcher": {
+ "id": "byRegexp",
+ "options": "CPU Busy"
+ },
+ "properties": [
+ {
+ "id": "custom.axisPlacement",
+ "value": "right"
+ },
+ {
+ "id": "unit",
+ "value": "percent"
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "Requests"
+ },
+ "properties": [
+ {
+ "id": "custom.axisLabel",
+ "value": "Requests / Second"
+ }
+ ]
+ }
+ ]
},
"gridPos": {
- "h": 8,
+ "h": 9,
"w": 12,
"x": 0,
"y": 1
},
- "id": 22,
+ "id": 2,
+ "maxDataPoints": 500,
"options": {
"legend": {
"calcs": [],
@@ -175,24 +185,39 @@
"uid": "${DS_PROMETHEUS}"
},
"exemplar": true,
- "expr": "sum by(type) (rate(taler_exchange_num_signatures{}[1m]))",
+ "expr": "sum(rate(nginx_http_requests_total{} [$__rate_interval]))",
+ "instant": false,
"interval": "",
- "legendFormat": "{{ type }}",
+ "intervalFactor": 1,
+ "legendFormat": "Requests",
"refId": "A"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "exemplar": true,
+ "expr": "100 - (avg by (instance)
(rate(node_cpu_seconds_total{mode=\"idle\",
component=\"exchange\"}[$__rate_interval]))*100)",
+ "hide": false,
+ "interval": "",
+ "intervalFactor": 1,
+ "legendFormat": "CPU Busy",
+ "refId": "B"
}
],
- "title": "Signatures",
+ "title": " [Exchange] Requests vs CPU",
"type": "timeseries"
},
{
- "description": "Number of signature verifications with the configured
cipher of the exchange (rsa or cs).",
+ "description": "",
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
- "axisLabel": "Verifications per Second",
+ "axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "line",
@@ -204,7 +229,7 @@
"viz": false
},
"lineInterpolation": "linear",
- "lineWidth": 1,
+ "lineWidth": 2,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
@@ -228,18 +253,47 @@
"value": null
}
]
- },
- "unit": "none"
+ }
},
- "overrides": []
+ "overrides": [
+ {
+ "matcher": {
+ "id": "byRegexp",
+ "options": "Network (.*)"
+ },
+ "properties": [
+ {
+ "id": "custom.axisPlacement",
+ "value": "right"
+ },
+ {
+ "id": "unit",
+ "value": "bps"
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "Requests"
+ },
+ "properties": [
+ {
+ "id": "custom.axisLabel",
+ "value": "Requests / Second"
+ }
+ ]
+ }
+ ]
},
"gridPos": {
- "h": 8,
+ "h": 9,
"w": 12,
"x": 12,
"y": 1
},
- "id": 20,
+ "id": 7,
+ "maxDataPoints": 500,
"options": {
"legend": {
"calcs": [],
@@ -253,29 +307,48 @@
},
"targets": [
{
- "datasource": {
- "type": "prometheus",
- "uid": "${DS_PROMETHEUS}"
- },
"exemplar": true,
- "expr": "sum by (type)
(rate(taler_exchange_num_signature_verifications{}[1m]))",
+ "expr": "sum(rate(nginx_http_requests_total{} [$__rate_interval]))",
"interval": "",
- "legendFormat": "{{ type }}",
+ "intervalFactor": 1,
+ "legendFormat": "Requests",
"refId": "A"
+ },
+ {
+ "exemplar": true,
+ "expr":
"sum(rate(node_network_receive_bytes_total{component=\"exchange\"}[$__rate_interval])
+ rate(node_network_transmit_bytes_total{component=\"exchange\"}
[$__rate_interval]))*8",
+ "hide": false,
+ "interval": "",
+ "intervalFactor": 1,
+ "legendFormat": "Network Bandwidth",
+ "refId": "B"
}
],
- "title": "Signature Verifications",
+ "title": "[Exchange] Requests vs Network Traffic",
"type": "timeseries"
},
{
- "description": "Number of key exchanges",
+ "collapsed": false,
+ "gridPos": {
+ "h": 1,
+ "w": 24,
+ "x": 0,
+ "y": 10
+ },
+ "id": 21,
+ "panels": [],
+ "title": "Wallet",
+ "type": "row"
+ },
+ {
+ "description": "Number of requests versus average wallet node CPU usage",
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
- "axisLabel": "Signatures per Second",
+ "axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "line",
@@ -287,7 +360,7 @@
"viz": false
},
"lineInterpolation": "linear",
- "lineWidth": 1,
+ "lineWidth": 2,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
@@ -311,18 +384,47 @@
"value": null
}
]
- },
- "unit": "none"
+ }
},
- "overrides": []
+ "overrides": [
+ {
+ "matcher": {
+ "id": "byRegexp",
+ "options": "CPU Busy"
+ },
+ "properties": [
+ {
+ "id": "custom.axisPlacement",
+ "value": "right"
+ },
+ {
+ "id": "unit",
+ "value": "percent"
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "Requests"
+ },
+ "properties": [
+ {
+ "id": "custom.axisLabel",
+ "value": "Requests / Second"
+ }
+ ]
+ }
+ ]
},
"gridPos": {
- "h": 8,
+ "h": 9,
"w": 12,
"x": 0,
- "y": 9
+ "y": 11
},
- "id": 23,
+ "id": 19,
+ "maxDataPoints": 500,
"options": {
"legend": {
"calcs": [],
@@ -341,13 +443,28 @@
"uid": "${DS_PROMETHEUS}"
},
"exemplar": true,
- "expr": "sum by (type) (rate(taler_exchange_num_keyexchanges{}
[2m]))",
+ "expr": "sum(rate(nginx_http_requests_total{} [$__rate_interval]))",
+ "instant": false,
"interval": "",
- "legendFormat": "{{ type }}",
+ "intervalFactor": 1,
+ "legendFormat": "Requests",
"refId": "A"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "exemplar": true,
+ "expr": "100 - (avg(rate(node_cpu_seconds_total{mode=\"idle\",
component=\"wallet\"}[$__rate_interval]))*100)",
+ "hide": false,
+ "interval": "",
+ "intervalFactor": 1,
+ "legendFormat": "CPU Busy",
+ "refId": "B"
}
],
- "title": "Key Exchanges",
+ "title": " [Wallet] Requests vs Average CPU",
"type": "timeseries"
},
{
@@ -356,38 +473,33 @@
"h": 1,
"w": 24,
"x": 0,
- "y": 17
+ "y": 20
},
- "id": 16,
+ "id": 15,
"panels": [],
"title": "Database",
"type": "row"
},
{
- "datasource": {
- "type": "datasource",
- "uid": "-- Mixed --"
- },
- "description": "",
"fieldConfig": {
"defaults": {
"color": {
- "mode": "thresholds"
+ "mode": "palette-classic"
},
"custom": {
- "axisLabel": "Slow Queries / Second",
+ "axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "line",
- "fillOpacity": 19,
- "gradientMode": "hue",
+ "fillOpacity": 0,
+ "gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"lineInterpolation": "linear",
- "lineWidth": 1,
+ "lineWidth": 2,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
@@ -409,32 +521,53 @@
{
"color": "green",
"value": null
- },
+ }
+ ]
+ }
+ },
+ "overrides": [
+ {
+ "matcher": {
+ "id": "byRegexp",
+ "options": "CPU Busy"
+ },
+ "properties": [
{
- "color": "#EAB839",
- "value": 5
+ "id": "custom.axisPlacement",
+ "value": "right"
},
{
- "color": "red",
- "value": 10
+ "id": "unit",
+ "value": "percent"
}
]
},
- "unit": "SQ/s"
- },
- "overrides": []
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "Requests"
+ },
+ "properties": [
+ {
+ "id": "custom.axisLabel",
+ "value": "Requests / Second"
+ }
+ ]
+ }
+ ]
},
"gridPos": {
"h": 9,
"w": 12,
"x": 0,
- "y": 18
+ "y": 21
},
- "id": 4,
+ "id": 3,
+ "maxDataPoints": 500,
"options": {
"legend": {
"calcs": [],
- "displayMode": "hidden",
+ "displayMode": "list",
"placement": "bottom"
},
"tooltip": {
@@ -445,26 +578,40 @@
"targets": [
{
"datasource": {
- "type": "loki",
- "uid": "${DS_LOKI}"
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
},
- "expr": "rate({app=\"taler-database\", type=\"slow-query\"} [1m])",
- "hide": false,
+ "exemplar": true,
+ "expr": "sum(rate(nginx_http_requests_total{} [$__rate_interval]))",
+ "interval": "",
+ "legendFormat": "Requests",
"refId": "A"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "exemplar": true,
+ "expr": "100 - (avg by (instance)
(rate(node_cpu_seconds_total{mode=\"idle\",
component=\"database\"}[$__rate_interval]))*100)",
+ "hide": false,
+ "interval": "",
+ "legendFormat": "CPU Busy",
+ "refId": "B"
}
],
- "title": "Slow Queries",
+ "title": "[Database] Requests vs CPU",
"type": "timeseries"
},
{
- "description": "Serialization Errors per Second and Request-Type",
+ "description": "",
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
- "axisLabel": "Serialization Errors / Second",
+ "axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "line",
@@ -476,7 +623,7 @@
"viz": false
},
"lineInterpolation": "linear",
- "lineWidth": 1,
+ "lineWidth": 2,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
@@ -500,18 +647,47 @@
"value": null
}
]
- },
- "unit": "none"
+ }
},
- "overrides": []
+ "overrides": [
+ {
+ "matcher": {
+ "id": "byRegexp",
+ "options": "Network (.*)"
+ },
+ "properties": [
+ {
+ "id": "custom.axisPlacement",
+ "value": "right"
+ },
+ {
+ "id": "unit",
+ "value": "bps"
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "Requests"
+ },
+ "properties": [
+ {
+ "id": "custom.axisLabel",
+ "value": "Requests / Second"
+ }
+ ]
+ }
+ ]
},
"gridPos": {
"h": 9,
"w": 12,
"x": 12,
- "y": 18
+ "y": 21
},
- "id": 2,
+ "id": 8,
+ "maxDataPoints": 500,
"options": {
"legend": {
"calcs": [],
@@ -523,94 +699,181 @@
"sort": "none"
}
},
- "pluginVersion": "8.2.5",
"targets": [
{
- "datasource": {
- "type": "prometheus",
- "uid": "${DS_PROMETHEUS}"
- },
"exemplar": true,
- "expr": "sum by (type)
(rate(taler_exchange_serialization_failures{}[1m]))",
- "instant": false,
+ "expr": "sum(rate(nginx_http_requests_total{} [$__rate_interval]))",
"interval": "",
- "legendFormat": "{{type}}",
+ "intervalFactor": 1,
+ "legendFormat": "Requests",
"refId": "A"
+ },
+ {
+ "exemplar": true,
+ "expr":
"sum(rate(node_network_receive_bytes_total{component=\"database\"}[$__rate_interval])
+ rate(node_network_transmit_bytes_total{component=\"database\"}
[$__rate_interval]))*8",
+ "hide": false,
+ "interval": "",
+ "intervalFactor": 1,
+ "legendFormat": "Network Bandwidth",
+ "refId": "B"
}
],
- "title": "Serialization Errors",
- "transformations": [],
+ "title": "[Database] Requests vs Network Traffic",
"type": "timeseries"
},
{
- "description": "Number of serialization errors by request endpoint
(type) since the last exchange restarts",
+ "description": "",
"fieldConfig": {
"defaults": {
"color": {
- "mode": "thresholds"
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "lineInterpolation": "linear",
+ "lineWidth": 2,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
- "color": "text",
+ "color": "green",
"value": null
- },
- {
- "color": "red",
- "value": 100
}
]
}
},
- "overrides": []
- },
- "gridPos": {
- "h": 8,
- "w": 10,
- "x": 0,
- "y": 27
- },
- "id": 6,
- "options": {
- "displayMode": "basic",
- "orientation": "horizontal",
- "reduceOptions": {
- "calcs": [
- "lastNotNull"
- ],
- "fields": "",
- "values": false
- },
- "showUnfilled": true,
- "text": {}
- },
- "pluginVersion": "8.4.3",
- "targets": [
- {
- "exemplar": true,
- "expr": "sort_desc (sum by(type)
(taler_exchange_serialization_failures{}))",
- "instant": true,
+ "overrides": [
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "Database Size"
+ },
+ "properties": [
+ {
+ "id": "custom.axisPlacement",
+ "value": "right"
+ },
+ {
+ "id": "unit",
+ "value": "bytes"
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "Total Requests"
+ },
+ "properties": [
+ {
+ "id": "unit",
+ "value": "short"
+ }
+ ]
+ }
+ ]
+ },
+ "gridPos": {
+ "h": 9,
+ "w": 12,
+ "x": 0,
+ "y": 30
+ },
+ "id": 5,
+ "maxDataPoints": 500,
+ "options": {
+ "legend": {
+ "calcs": [],
+ "displayMode": "list",
+ "placement": "bottom"
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "targets": [
+ {
+ "exemplar": true,
+ "expr": "nginx_http_requests_total{}",
"interval": "",
- "legendFormat": "{{type}}",
+ "intervalFactor": 1,
+ "legendFormat": "Requests",
"refId": "A"
+ },
+ {
+ "exemplar": true,
+ "expr": "sum(pg_database_size_bytes{instance=\"127.0.0.1:9187\"})",
+ "hide": false,
+ "interval": "",
+ "legendFormat": "Database Size",
+ "refId": "C"
}
],
- "title": "Serialization Errors by Request Type",
- "transformations": [],
- "type": "bargauge"
+ "title": "Total Requests vs Disk Space used",
+ "type": "timeseries"
},
{
- "description": "Total number of serialization errors that happened since
last exchange restarts",
+ "description": "",
"fieldConfig": {
"defaults": {
"color": {
- "fixedColor": "text",
- "mode": "fixed"
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "lineInterpolation": "linear",
+ "lineWidth": 2,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
},
"mappings": [],
- "noValue": "0",
"thresholds": {
"mode": "absolute",
"steps": [
@@ -621,127 +884,242 @@
]
}
},
- "overrides": []
+ "overrides": [
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "Requests"
+ },
+ "properties": [
+ {
+ "id": "custom.axisLabel",
+ "value": "Requests / Second"
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byRegexp",
+ "options": "I/O Utilization(.*)"
+ },
+ "properties": [
+ {
+ "id": "custom.axisPlacement",
+ "value": "right"
+ },
+ {
+ "id": "unit",
+ "value": "percentunit"
+ }
+ ]
+ }
+ ]
},
"gridPos": {
- "h": 4,
- "w": 4,
- "x": 10,
- "y": 27
+ "h": 9,
+ "w": 12,
+ "x": 12,
+ "y": 30
},
- "id": 8,
+ "id": 9,
+ "maxDataPoints": 500,
"options": {
- "colorMode": "value",
- "graphMode": "none",
- "justifyMode": "auto",
- "orientation": "auto",
- "reduceOptions": {
- "calcs": [
- "lastNotNull"
- ],
- "fields": "",
- "values": false
- },
- "text": {},
- "textMode": "auto"
- },
- "pluginVersion": "8.4.3",
+ "legend": {
+ "calcs": [],
+ "displayMode": "list",
+ "placement": "bottom"
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
"targets": [
{
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
"exemplar": true,
- "expr": "sum(taler_exchange_serialization_failures{})",
+ "expr": "sum(rate(nginx_http_requests_total{} [$__rate_interval]))",
"interval": "",
- "legendFormat": "",
+ "intervalFactor": 1,
+ "legendFormat": "Requests",
"refId": "A"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "exemplar": true,
+ "expr": "sum by (device)
(rate(node_disk_io_time_seconds_total{component=\"database\"}
[$__rate_interval]))",
+ "hide": false,
+ "interval": "",
+ "legendFormat": "I/O Utilization ({{device}})",
+ "refId": "C"
}
],
- "title": "Total Serialization Errors",
- "type": "stat"
+ "title": "Requests vs I/O Utilization",
+ "type": "timeseries"
},
{
- "description": "Number of serialization errors per request since the
last exchange restarts",
+ "description": "",
"fieldConfig": {
"defaults": {
"color": {
- "mode": "thresholds"
- },
- "mappings": [
- {
- "options": {
- "match": "nan",
- "result": {
- "index": 0,
- "text": "-"
- }
- },
- "type": "special"
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "lineInterpolation": "linear",
+ "lineWidth": 2,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
}
- ],
+ },
+ "mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
- "color": "text",
+ "color": "green",
"value": null
+ }
+ ]
+ }
+ },
+ "overrides": [
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "Cache Hit Rate"
+ },
+ "properties": [
+ {
+ "id": "custom.axisPlacement",
+ "value": "right"
},
{
- "color": "red",
- "value": 2
+ "id": "unit",
+ "value": "percentunit"
}
]
},
- "unit": "percentunit"
- },
- "overrides": []
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "Requests"
+ },
+ "properties": [
+ {
+ "id": "custom.axisLabel",
+ "value": "Requests / Second"
+ }
+ ]
+ }
+ ]
},
"gridPos": {
- "h": 8,
- "w": 10,
- "x": 14,
- "y": 27
+ "h": 9,
+ "w": 12,
+ "x": 0,
+ "y": 39
},
- "id": 12,
+ "id": 10,
+ "maxDataPoints": 500,
"options": {
- "displayMode": "basic",
- "orientation": "horizontal",
- "reduceOptions": {
- "calcs": [
- "lastNotNull"
- ],
- "fields": "",
- "values": false
- },
- "showUnfilled": true,
- "text": {}
- },
- "pluginVersion": "8.4.3",
+ "legend": {
+ "calcs": [],
+ "displayMode": "list",
+ "placement": "bottom"
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
"targets": [
{
"exemplar": true,
- "expr": "sort_desc(sum by(type)
(rate(taler_exchange_serialization_failures{}[10m]))\n/\nsum by(type)
(rate(taler_exchange_received_requests{}[10m])))",
- "instant": true,
+ "expr": "sum(rate(nginx_http_requests_total{} [$__rate_interval]))",
"interval": "",
- "legendFormat": "{{type}}",
+ "intervalFactor": 1,
+ "legendFormat": "Requests",
"refId": "A"
+ },
+ {
+ "exemplar": true,
+ "expr": "pg_stat_database_blks_hit{instance=\"127.0.0.1:9187\",
server=\"db.perf.taler:5432\", datname!~\"postgres|template.*|\"} /
(pg_stat_database_blks_read{instance=\"127.0.0.1:9187\",
server=\"db.perf.taler:5432\", datname!~\"postgres|template.*|\"} +
pg_stat_database_blks_hit{instance=\"127.0.0.1:9187\",
server=\"db.perf.taler:5432\", datname!~\"postgres|template.*|\"})",
+ "hide": false,
+ "interval": "",
+ "legendFormat": "Cache Hit Rate",
+ "refId": "C"
}
],
- "title": "Serialization Errors / Request",
- "type": "bargauge"
+ "title": "Requests vs Cache Hit Rate",
+ "type": "timeseries"
},
{
"datasource": {
"type": "datasource",
"uid": "-- Mixed --"
},
- "description": "Number of queries which took longer than the configured
duration",
+ "description": "",
"fieldConfig": {
"defaults": {
"color": {
- "fixedColor": "text",
- "mode": "fixed"
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "lineInterpolation": "linear",
+ "lineWidth": 2,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
},
"mappings": [],
- "noValue": "0",
"thresholds": {
"mode": "absolute",
"steps": [
@@ -752,61 +1130,131 @@
]
}
},
- "overrides": []
+ "overrides": [
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "Slow Queries"
+ },
+ "properties": [
+ {
+ "id": "custom.axisPlacement",
+ "value": "right"
+ },
+ {
+ "id": "custom.axisLabel",
+ "value": "Slow Queries / Second"
+ },
+ {
+ "id": "unit",
+ "value": "none"
+ },
+ {
+ "id": "custom.lineStyle",
+ "value": {
+ "fill": "solid"
+ }
+ },
+ {
+ "id": "custom.fillOpacity",
+ "value": 15
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "Requests"
+ },
+ "properties": [
+ {
+ "id": "custom.axisLabel",
+ "value": "Requests / Second"
+ }
+ ]
+ }
+ ]
},
"gridPos": {
- "h": 4,
- "w": 4,
- "x": 10,
- "y": 31
+ "h": 9,
+ "w": 12,
+ "x": 12,
+ "y": 39
},
- "id": 10,
+ "id": 11,
+ "maxDataPoints": 500,
"options": {
- "colorMode": "value",
- "graphMode": "none",
- "justifyMode": "auto",
- "orientation": "auto",
- "reduceOptions": {
- "calcs": [
- "lastNotNull"
- ],
- "fields": "",
- "values": false
- },
- "text": {},
- "textMode": "auto"
- },
- "pluginVersion": "8.4.3",
+ "legend": {
+ "calcs": [],
+ "displayMode": "list",
+ "placement": "bottom"
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
"targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "exemplar": true,
+ "expr": "sum(rate(nginx_http_requests_total{} [$__rate_interval]))",
+ "interval": "",
+ "intervalFactor": 1,
+ "legendFormat": "Requests",
+ "refId": "A"
+ },
{
"datasource": {
"type": "loki",
"uid": "${DS_LOKI}"
},
- "expr": "sum(count_over_time({app=\"taler-database\",
type=\"slow-query\"} [$__range]))",
- "instant": true,
- "range": false,
- "refId": "A"
+ "expr": "rate({app=\"taler-database\", type=\"slow-query\"} [1m])",
+ "hide": false,
+ "legendFormat": "Slow Queries",
+ "refId": "C"
}
],
- "title": "Total Slow Queries",
- "type": "stat"
+ "title": "Requests vs Slow Queries",
+ "type": "timeseries"
},
{
- "datasource": {
- "type": "loki",
- "uid": "${DS_LOKI}"
- },
- "description": "",
+ "description": "Serialization Errors per Second and Request-Type",
"fieldConfig": {
"defaults": {
"color": {
- "mode": "thresholds"
+ "mode": "palette-classic"
},
"custom": {
- "align": "auto",
- "displayMode": "json-view",
- "filterable": false
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "gradientMode": "opacity",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "lineInterpolation": "linear",
+ "lineWidth": 2,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
},
"mappings": [],
"thresholds": {
@@ -817,127 +1265,222 @@
"value": null
}
]
- }
+ },
+ "unit": "none"
},
"overrides": [
{
"matcher": {
"id": "byName",
- "options": "Count"
+ "options": "Serialization Errors"
},
"properties": [
{
- "id": "custom.width",
- "value": 100
+ "id": "custom.axisPlacement",
+ "value": "right"
+ },
+ {
+ "id": "custom.axisLabel",
+ "value": "Serialization Errors / Second"
+ },
+ {
+ "id": "unit"
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "Requests"
+ },
+ "properties": [
+ {
+ "id": "custom.axisLabel",
+ "value": "Requests / Second"
}
]
}
]
},
"gridPos": {
- "h": 7,
- "w": 24,
+ "h": 9,
+ "w": 12,
"x": 0,
- "y": 35
+ "y": 48
},
- "id": 14,
+ "id": 18,
"options": {
- "footer": {
- "fields": "",
- "reducer": [
- "sum"
- ],
- "show": false
- },
- "frameIndex": 1,
- "showHeader": true,
- "sortBy": []
- },
- "pluginVersion": "8.4.3",
+ "legend": {
+ "calcs": [],
+ "displayMode": "list",
+ "placement": "bottom"
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "8.2.5",
"targets": [
{
- "datasource": {
- "type": "loki",
- "uid": "${DS_LOKI}"
- },
- "expr": "sum by (query) (count_over_time({app=\"taler-database\",
type=\"slow-query\"} | regexp \"(?P<query>.*)\" [$__range]))",
- "instant": true,
- "range": false,
+ "exemplar": true,
+ "expr": "sum(rate(nginx_http_requests_total{} [$__rate_interval]))",
+ "hide": false,
+ "interval": "",
+ "legendFormat": "Requests",
+ "refId": "B"
+ },
+ {
+ "exemplar": true,
+ "expr": "sum (rate(taler_exchange_serialization_failures{}[1m]))",
+ "instant": false,
+ "interval": "",
+ "legendFormat": "Serialization Errors",
"refId": "A"
}
],
- "title": "Slow Query Occurences",
- "transformations": [
- {
- "id": "groupBy",
- "options": {
- "fields": {
- "Query": {
- "aggregations": [
- "count"
- ],
- "operation": "groupby"
+ "title": "Requests vs Serialization Errors",
+ "transformations": [],
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "datasource",
+ "uid": "-- Mixed --"
+ },
+ "description": "",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "lineInterpolation": "linear",
+ "lineWidth": 2,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
+ }
+ ]
+ },
+ "unit": "s"
+ },
+ "overrides": [
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "Slow Queries"
+ },
+ "properties": [
+ {
+ "id": "custom.axisPlacement",
+ "value": "right"
},
- "Value": {
- "aggregations": [
- "lastNotNull"
- ],
- "operation": "aggregate"
+ {
+ "id": "custom.axisLabel",
+ "value": "Slow Queries / Second"
},
- "Value #A": {
- "aggregations": [
- "lastNotNull"
- ],
- "operation": "aggregate"
+ {
+ "id": "unit",
+ "value": "none"
},
- "app": {
- "aggregations": [
- "count"
- ],
- "operation": "aggregate"
+ {
+ "id": "custom.drawStyle",
+ "value": "line"
+ },
+ {
+ "id": "custom.lineStyle"
},
- "query": {
- "aggregations": [],
- "operation": "groupby"
+ {
+ "id": "custom.fillOpacity",
+ "value": 15
}
- }
+ ]
}
+ ]
+ },
+ "gridPos": {
+ "h": 9,
+ "w": 12,
+ "x": 12,
+ "y": 48
+ },
+ "id": 16,
+ "options": {
+ "legend": {
+ "calcs": [],
+ "displayMode": "list",
+ "placement": "bottom"
},
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "targets": [
{
- "id": "organize",
- "options": {
- "excludeByName": {},
- "indexByName": {},
- "renameByName": {
- "Value #A (lastNotNull)": "Count",
- "Value (lastNotNull)": "Count",
- "app (count)": "Count",
- "query": "Query"
- }
- }
+ "datasource": {
+ "type": "loki",
+ "uid": "${DS_LOKI}"
+ },
+ "expr": "rate({app=\"taler-database\", type=\"slow-query\"} [1m])",
+ "hide": false,
+ "legendFormat": "Slow Queries",
+ "refId": "A",
+ "resolution": 1
},
{
- "id": "sortBy",
- "options": {
- "fields": {},
- "sort": [
- {
- "desc": true,
- "field": "Count"
- }
- ]
- }
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "exemplar": true,
+ "expr": "avg
((rate(taler_requests_response_time_hist_sum{endpoint!~\"keys|terms\",
status=\"200\"} [2m]) ) /
(rate(taler_requests_response_time_hist_count{endpoint!~\"keys|terms\",
status=\"200\"} [2m])))",
+ "hide": false,
+ "interval": "",
+ "legendFormat": "Average Response Time",
+ "refId": "B"
}
],
- "type": "table"
+ "title": "Proxy Response Time vs Slow Queries",
+ "type": "timeseries"
}
],
"refresh": "10s",
"schemaVersion": 35,
"style": "dark",
"tags": [
- "performance",
- "taler"
+ "taler",
+ "performance"
],
"templating": {
"list": []
@@ -948,8 +1491,8 @@
},
"timepicker": {},
"timezone": "",
- "title": "Exchange",
- "uid": "NYO0EIE7k",
- "version": 32,
+ "title": "Load Statistics",
+ "uid": "rkyhDAt7z",
+ "version": 63,
"weekStart": ""
}
\ No newline at end of file
diff --git a/additional/recover/docker-compose.yaml
b/additional/recover/docker-compose.yaml
index 018f84c..e117b2b 100644
--- a/additional/recover/docker-compose.yaml
+++ b/additional/recover/docker-compose.yaml
@@ -49,9 +49,9 @@ services:
container_name: grafana-renderer
loki:
+ hostname: loki
image: grafana/loki:latest
restart: 'no'
- hostname: loki
container_name: loki
user: "${U_ID}:${G_ID}"
ports:
diff --git a/configs/etc/monitor/loki.yaml b/configs/etc/monitor/loki.yaml
index e00581b..461ea19 100644
--- a/configs/etc/monitor/loki.yaml
+++ b/configs/etc/monitor/loki.yaml
@@ -10,7 +10,7 @@ querier:
ingester:
wal:
enabled: true
- dir: /tmp/wal
+ dir: /tmp/loki/wal
lifecycler:
address: 127.0.0.1
ring:
diff --git a/experiment/scripts/data-backup.sh
b/experiment/scripts/data-backup.sh
index c250e2e..f6d9411 100755
--- a/experiment/scripts/data-backup.sh
+++ b/experiment/scripts/data-backup.sh
@@ -14,7 +14,7 @@ LOKI_DATA_DIR=/tmp/loki
SNAPSHOT_COPY_PATH=${LOG_DIR}/tmp-data-snapshot
SNAPSHOT_PERS_FILE=${LOG_DIR}/data-snapshot.tar.gz
-mkdir "${SNAPSHOT_COPY_PATH}"
+mkdir "${SNAPSHOT_COPY_PATH}" || true
END_TIME=$(date +%s)
@@ -36,5 +36,5 @@ SNAPSHOT_TIME=${END_TIME}
" >> "${SNAPSHOT_COPY_PATH}/times.env"
rm -rf "${SNAPSHOT_PERS_FILE}"
-tar -czvf "${SNAPSHOT_PERS_FILE}" -C "${SNAPSHOT_COPY_PATH}"
"${SNAPSHOT_COPY_PATH}"
+tar -C "${SNAPSHOT_COPY_PATH}" -czvf "${SNAPSHOT_PERS_FILE}" .
rm -rf "${SNAPSHOT_COPY_PATH}"
--
To stop receiving notification emails like this one, please contact
gnunet@gnunet.org.
- [taler-grid5k] 129/189: slow query time, (continued)
- [taler-grid5k] 129/189: slow query time, gnunet, 2022/04/28
- [taler-grid5k] 152/189: deduplicate promtail, gnunet, 2022/04/28
- [taler-grid5k] 169/189: centos add node-exporter, gnunet, 2022/04/28
- [taler-grid5k] 185/189: update taler-perf, gnunet, 2022/04/28
- [taler-grid5k] 187/189: rename to be clearer, gnunet, 2022/04/28
- [taler-grid5k] 173/189: createusers not required for db only, gnunet, 2022/04/28
- [taler-grid5k] 121/189: fix gitignore, gnunet, 2022/04/28
- [taler-grid5k] 164/189: update, gnunet, 2022/04/28
- [taler-grid5k] 171/189: fix pg repo in centos, gnunet, 2022/04/28
- [taler-grid5k] 134/189: try source, gnunet, 2022/04/28
- [taler-grid5k] 137/189: fix some issues in recover,
gnunet <=
- [taler-grid5k] 150/189: various imrovements / fixes, gnunet, 2022/04/28
- [taler-grid5k] 170/189: add node-exporter service files as not present in centos, gnunet, 2022/04/28
- [taler-grid5k] 139/189: fix taler-perf script, gnunet, 2022/04/28
- [taler-grid5k] 180/189: add centos db initialization script, gnunet, 2022/04/28
- [taler-grid5k] 136/189: update dashboards, gnunet, 2022/04/28
- [taler-grid5k] 163/189: update centos, gnunet, 2022/04/28
- [taler-grid5k] 165/189: remove --no-install-recommends, gnunet, 2022/04/28
- [taler-grid5k] 174/189: add mitigations=off in grid5000 base.yaml as it gets overriden, gnunet, 2022/04/28
- [taler-grid5k] 183/189: fix promtail, gnunet, 2022/04/28
- [taler-grid5k] 184/189: add documentation, gnunet, 2022/04/28