[taler-grid5k] 27/189: include shards in monitoring

[gnunet]

This is an automated email from the git hooks/post-receive script.

marco-boss pushed a commit to branch master
in repository grid5k.

commit e34f0d3adbb4c3fe29d0f9b8df18907f5dffebf5
Author: Boss Marco <bossm8@bfh.ch>
AuthorDate: Wed Feb 23 18:58:56 2022 +0100

    include shards in monitoring
---
 configs/etc/default/prometheus-postgres-exporter |  3 +-
 configs/etc/monitor/node-exporters.yaml.tpl      |  8 ++---
 experiment/env                                   |  4 ++-
 experiment/scripts/database.sh                   | 38 +++++++++++-------------
 experiment/scripts/helpers.sh                    | 10 +++++++
 experiment/scripts/monitor.sh                    | 21 +++++++++++++
 6 files changed, 58 insertions(+), 26 deletions(-)

diff --git a/configs/etc/default/prometheus-postgres-exporter 
b/configs/etc/default/prometheus-postgres-exporter
index fe82513..b8ea6e3 100644
--- a/configs/etc/default/prometheus-postgres-exporter
+++ b/configs/etc/default/prometheus-postgres-exporter
@@ -4,7 +4,8 @@
 
 # DATA_SOURCE_NAME='postgresql://login:password@hostname:port/'
 # DATA_SOURCE_NAME='user=prometheus host=/run/postgresql dbname=postgres'
-DATA_SOURCE_NAME='<EXCHANGE_DB_URL_HERE>,<MERCHANT_DB_URL_HERE>'
+# IMPORTANT - place no comma between merchant and shards, will be handled by 
the scripts
+DATA_SOURCE_NAME='<EXCHANGE_DB_URL_HERE>,<MERCHANT_DB_URL_HERE><SHARD_DB_URLS_HERE>'
 
 # Set the command-line arguments to pass to the server.
 ARGS='--extend.query-path=/etc/monitor/postgres-exporter.yaml'
diff --git a/configs/etc/monitor/node-exporters.yaml.tpl 
b/configs/etc/monitor/node-exporters.yaml.tpl
index 762ef7c..a159c39 100644
--- a/configs/etc/monitor/node-exporters.yaml.tpl
+++ b/configs/etc/monitor/node-exporters.yaml.tpl
@@ -32,7 +32,7 @@
         component: 'merchant'
       targets:
       - '${MERCHANT_DOMAIN}:9100'
-#    - labels:
-#        component: 'shard'
-#      targets:
-#      # <SHARDS_HERE>
+    - labels:
+        component: 'shard'
+      targets:
+      # <SHARDS_HERE>
diff --git a/experiment/env b/experiment/env
index 68c2d62..944e7d5 100644
--- a/experiment/env
+++ b/experiment/env
@@ -22,7 +22,8 @@ G5K_COMMIT_SHA=master
 
 # Domain names for the hosts inside the grid.
 # The most certainly do not need to be adjusted.
-# If you do anyway, make sure the wallets contain a wildcard (*)
+# If you do anyway, make sure the wallets and shards contain a wildcard (*)
+# and that the helpers.sh functions get_<node>_hosts are edited accordingly 
(cut)
 # Otherwise the experiment will not work
     BANK_DOMAIN=bank.${DNS_ZONE}
 DATABASE_DOMAIN=db.${DNS_ZONE}
@@ -32,6 +33,7 @@ MERCHANT_DOMAIN=merchant.${DNS_ZONE}
  MONITOR_DOMAIN=monitor.${DNS_ZONE}
    PROXY_DOMAIN=proxy.${DNS_ZONE}
   WALLET_DOMAIN=wallet.*.${DNS_ZONE}
+   SHARD_DOMAIN=shard.*.${DNS_ZONE}
 
 # Arguments to pass to all taler-exchange-* binaries at startup.
 # E.g. -L INFO
diff --git a/experiment/scripts/database.sh b/experiment/scripts/database.sh
index 1c6e7c2..4113935 100755
--- a/experiment/scripts/database.sh
+++ b/experiment/scripts/database.sh
@@ -14,17 +14,16 @@ source ~/scripts/helpers.sh
 cd /tmp 
 
 function setup_disks() {
-  if [[ "$(hostname)" =~ "dahu" ]]; then
-    echo 'start=2048, type=83' | sfdisk /dev/disk1 || true
-    yes | mkfs.ext4 -m 0 /dev/disk1 || true
-    mkdir /mnt/disk || true
-    mount /dev/disk1 /mnt/disk || true
-    if [ ! -L /var/lib/postgresql/${POSTGRES_VERSION}/main/pg_wal ]; then
-      rm -rf /mnt/disk/pg_wal || true
-      mv /var/lib/postgresql/${POSTGRES_VERSION}/main/pg_wal/ /mnt/disk
-      ln -s /mnt/disk/pg_wal 
/var/lib/postgresql/${POSTGRES_VERSION}/main/pg_wal
-      chown -R postgres:postgres 
/var/lib/postgresql/${POSTGRES_VERSION}/main/pg_wal
-    fi
+  test -c /dev/disk1 || ( echo "No second disk for WAL available"; return )
+  echo 'start=2048, type=83' | sfdisk /dev/disk1 || true
+  yes | mkfs.ext4 -m 0 /dev/disk1 || true
+  mkdir /mnt/disk || true
+  mount /dev/disk1 /mnt/disk || true
+  if [ ! -L /var/lib/postgresql/${POSTGRES_VERSION}/main/pg_wal ]; then
+    rm -rf /mnt/disk/pg_wal || true
+    mv /var/lib/postgresql/${POSTGRES_VERSION}/main/pg_wal/ /mnt/disk
+    ln -s /mnt/disk/pg_wal /var/lib/postgresql/${POSTGRES_VERSION}/main/pg_wal
+    chown -R postgres:postgres 
/var/lib/postgresql/${POSTGRES_VERSION}/main/pg_wal
   fi
 }
 
@@ -69,11 +68,11 @@ function setup_config() {
   log_error_verbosity=terse
   syslog_ident='taler-database'
   log_min_duration_statement=300
-  shared_preload_libraries='pg_stat_statements' #,auto_explain'
-  # auto_explain.log_min_duration='300ms'
-  # auto_explain.log_analyze=true
-  # auto_explain.log_buffers=true
-  # auto_explain.log_wal=true
+  shared_preload_libraries='pg_stat_statements,auto_explain'
+  auto_explain.log_min_duration='300ms'
+  auto_explain.log_analyze=true
+  auto_explain.log_buffers=true
+  auto_explain.log_wal=true
   
   # use 25% of the available memory 
   # (https://www.postgresql.org/docs/13/runtime-config-resource.html)
@@ -202,14 +201,13 @@ psql -d "${DB_NAME}" -tAc "SELECT prepare_sharding();"
 EOF
 
   let "i=1"
-  for SHOST in ${SHARD_HOSTS//|/ }; do
-  SHARD_NAME=$(echo ${SHOST} | cut -d "." -f 1)
+  for SHARD in $(get_shard_hosts); do
     su postgres << EOF
-psql -d "${DB_NAME}" -tAc "SELECT create_shard_server('${SHOST}',
+psql -d "${DB_NAME}" -tAc "SELECT 
create_shard_server('${SHARD_DOMAIN//\*/${SHARD}}',
                                                       5432,
                                                       '${DB_USER}',
                                                       '${DB_PASSWORD}',
-                                                      '${SHARD_NAME}',
+                                                      '${SHARD}',
                                                       ${NUM_SHARDS},
                                                       ${i},
                                                       '${DB_NAME}');"
diff --git a/experiment/scripts/helpers.sh b/experiment/scripts/helpers.sh
index 413043e..a7238c3 100755
--- a/experiment/scripts/helpers.sh
+++ b/experiment/scripts/helpers.sh
@@ -147,6 +147,16 @@ function get_wallet_hosts() {
   echo ${WALLETS[@]}
 }
 
+# Get all Grid5000 hosts which host shards
+# Returns only the Grid5000 node - e.g. graoully-1
+function get_shard_hosts() {
+  IFS=$'\n' read -r -d '' -a SHARDS < <(\
+    dig -t AXFR "${DNS_ZONE}" "@${DNS_HOSTS}" \
+    | grep ${SHARD_DOMAIN} | awk '{print $2}' | cut -d '.' -f 2 \
+  )
+  echo ${SHARDS[@]}
+}
+
 # Get all exchanges which are registered in the nginx proxy config
 # Returns the whole domain - e.g. exchange.perf.taler:80
 function get_exchanges() {
diff --git a/experiment/scripts/monitor.sh b/experiment/scripts/monitor.sh
index 292b4ff..24f24d2 100755
--- a/experiment/scripts/monitor.sh
+++ b/experiment/scripts/monitor.sh
@@ -66,6 +66,13 @@ function configure_prometheus_and_exporters() {
   sed -i -e 
"s\<EXCHANGE_DB_URL_HERE>\postgresql://postgres@${DATABASE_DOMAIN}:5432\g" \
          -e 
"s\<MERCHANT_DB_URL_HERE>\postgresql://postgres@${MERCHANT_DOMAIN}:5432\g" \
              /etc/default/prometheus-postgres-exporter
+
+  CFG=","
+  for SHARD in $(get_shard_hosts); do
+    CFG="${CFG}postgresql://postgres@${SHARD_DOMAIN//\*/${SHARD}}:5432"
+  fi
+  sed -i "s|<SHARD_DB_URLS_HERE>|${CFG}|g" \
+          /etc/default/prometheus-postgres-exporter
   
   sed -i "s\<PROXY_URL_HERE>\http://${PROXY_DOMAIN}/stub_status\g"; \
           /etc/default/prometheus-nginx-exporter
@@ -108,6 +115,18 @@ function add_exchanges_to_prometheus() {
   done
 }
 
+# Add shard servers to be monitored
+# Requires no argument, since shard servers are retrieved from
+function add_shards_to_prometheus() {
+  for SHARD in $(get_shard_hosts); do
+    if ! grep -q "${SHARD}" /etc/monitor/prometheus.yaml;
+    then
+      sed -i "/<SHARDS_HERE/a \ \ \ \ \ \ - '${SHARD_DOMAIN//\*/${SHARD}}'" \
+              /etc/monitor/prometheus.yaml
+    fi
+  done
+}
+
 # Remove N exchanges which have been stopped
 # from the config so they will not be scraped anymore
 # $1: N - number of exchanges to remove
@@ -151,6 +170,8 @@ function init_monitor() {
     add_exchanges_to_prometheus
   fi
 
+  add_shards_to_prometheus
+
   systemctl restart prometheus-nginx-exporter \
                     prometheus-postgres-exporter \
                     prometheus

-- 
To stop receiving notification emails like this one, please contact
gnunet@gnunet.org.