[taler-grid5k] 157/189: fix persist script, delete manually

[gnunet]

This is an automated email from the git hooks/post-receive script.

marco-boss pushed a commit to branch master
in repository grid5k.

commit ec096cd5e4e252d5592fde03f59ac8457762e9b5
Author: Boss Marco <bossm8@bfh.ch>
AuthorDate: Sat Apr 16 18:47:16 2022 +0200

    fix persist script, delete manually
---
 additional/persist.sh | 172 ++++++++++++++++++++++++++++----------------------
 experiment/README.md  |  28 ++++----
 2 files changed, 108 insertions(+), 92 deletions(-)

diff --git a/additional/persist.sh b/additional/persist.sh
index dc5aab6..e1b77c4 100755
--- a/additional/persist.sh
+++ b/additional/persist.sh
@@ -16,82 +16,104 @@ function work_in_progress() {
   sleep 1
 }
 
-echo "
-Make sure that:
-
-- Your ssh key to access the grid is added to the ssh-agent with 'ssh-add 
<KEY>'
-- The config to access the grid is setup as described in:
-  
https://www.grid5000.fr/w/SSH#Using_SSH_ProxyCommand_feature_to_ease_the_access_to_hosts_inside_Grid.275000
-"
-
-if [ -z ${1} ]; then
-  echo "Missing experiment name"
-  exit 1
-fi
-if [ -z ${2} ]; then
-  echo "No plots will be included"
-else
-  echo "The file '${2}' will be deleted when this script finishes, do you want 
to contiue? [y/n]" 
-  while true; do
-    read -n 1 -s yn
-    case $yn in
-      [Yy]*) break 2;;
-      [Nn]*) exit ;;
-      *) echo "Please enter 'y' or 'n'" ;;
-    esac
+function backup() {
+  echo "
+  Make sure that:
+  
+  - Your ssh key to access the grid is added to the ssh-agent with 'ssh-add 
<KEY>'
+  - The config to access the grid is setup as described in:
+    
https://www.grid5000.fr/w/SSH#Using_SSH_ProxyCommand_feature_to_ease_the_access_to_hosts_inside_Grid.275000
+  "
+  
+  if [ -z ${1} ]; then
+    echo "Missing experiment name"
+    exit 1
+  fi
+  if [ -z ${2} ]; then
+    echo "No plots will be included"
+  else
+    echo "The file '${2}' will be deleted when this script finishes, do you 
want to contiue? [y/n]" 
+    while true; do
+      read -n 1 -s yn
+      case $yn in
+        [Yy]*) break 2;;
+        [Nn]*) exit ;;
+        *) echo "Please enter 'y' or 'n'" ;;
+      esac
+    done
+  fi
+  
+  if [[ $(ssh-add -L) =~ "The agent has no" ]]; then
+    echo "No ssh key added, aborting"
+    exit 1
+  fi
+  
+  # Slugify the name
+  NAME=$(\
+    echo "$1" | \
+    sed -r s/[~\^]+//g | \
+    sed -r s/[^a-zA-Z0-9]+/-/g | \
+    sed -r s/^-+\|-+$//g | \
+    tr A-Z a-z \
+  )
+  BACKUP_NAME="${NAME}-$(date +%d-%m-%y).tar"
+  BACKUP_DIR="archives"
+  
+  G5K_BACKUP="g5k-$(date +%s).tar.gz"
+  
+  echo "Creating archive, this can take some time"
+  ssh g5k 'rm -rf grenoble/g5k.tar* && find . -type l -maxdepth 1 -exec tar 
--ignore-failed-read -rf grenoble/g5k.tar -C {} exp-logs exp-data espec-times 
2>&1 \; | grep -vi "no such file"' &
+  
+  while kill -0 $! > /dev/null 2>&1; do
+    work_in_progress
   done
-fi
-
-if [[ $(ssh-add -L) =~ "The agent has no" ]]; then
-  echo "No ssh key added, aborting"
-  exit 1
-fi
-
-# Slugify the name
-NAME=$(\
-  echo "$1" | \
-  sed -r s/[~\^]+//g | \
-  sed -r s/[^a-zA-Z0-9]+/-/g | \
-  sed -r s/^-+\|-+$//g | \
-  tr A-Z a-z \
-)
-BACKUP_NAME="${NAME}-$(date +%d-%m-%y).tar"
-BACKUP_DIR="archives"
-
-G5K_BACKUP="g5k-$(date +%s).tar.gz"
-
-echo "Creating archive, this can take some time"
-ssh g5k 'rm -rf grenoble/g5k.tar* && find . -type l -exec tar 
--ignore-failed-read -rf grenoble/g5k.tar -C {} exp-logs exp-data espec-times 
2>&1 \; | grep -vi "no such file"' &
-
-while kill -0 $! > /dev/null 2>&1; do
-  work_in_progress
-done
-
-echo "Compressing archive, this may take some time"
-ssh g5k 'gzip grenoble/g5k.tar' &
-
-while kill -0 $! > /dev/null 2>&1; do
-  work_in_progress
-done
-
-echo "Copying archive to local machine"
-scp g5k:~/grenoble/g5k.tar.gz ${G5K_BACKUP}
-
-echo "Cleaning up g5k storage"
-ssh g5k 'rm grenoble/g5k.tar.gz && $(find . -type l -exec rm -rf {}/exp-logs/ 
{}/exp-data {}/espec-times \;)' &
-
-while kill -0 $! > /dev/null 2>&1; do
-  work_in_progress
-done
-
-test -d ${BACKUP_DIR} || mkdir ${BACKUP_DIR}
-
-echo "Creating Backup Tar ${BACKUP_DIR}/${BACKUP_NAME}"
-tar -cvf ${BACKUP_DIR}/${BACKUP_NAME} ${G5K_BACKUP} ${2} &
+  
+  echo "Compressing archive, this may take some time"
+  ssh g5k 'gzip grenoble/g5k.tar' &
+  
+  while kill -0 $! > /dev/null 2>&1; do
+    work_in_progress
+  done
+  
+  echo "Copying archive to local machine"
+  scp g5k:~/grenoble/g5k.tar.gz ${G5K_BACKUP}
+  
+  test -d ${BACKUP_DIR} || mkdir ${BACKUP_DIR}
+  
+  echo "Creating Backup Tar ${BACKUP_DIR}/${BACKUP_NAME}"
+  tar -cvf ${BACKUP_DIR}/${BACKUP_NAME} ${G5K_BACKUP} ${2} &
+  
+  while kill -0 $! > /dev/null 2>&1; do
+    work_in_progress
+  done
+  
+  rm ${G5K_BACKUP} ${2}
+}
 
-while kill -0 $! > /dev/null 2>&1; do
-  work_in_progress
-done
+function delete() {
+  echo "Cleaning up g5k storage"
+  ssh g5k 'rm grenoble/g5k.tar.gz && $(find . -type l -exec rm -rf 
{}/exp-logs/ {}/exp-data {}/espec-times \;)' &
+  
+  while kill -0 $! > /dev/null 2>&1; do
+    work_in_progress
+  done
+}
 
-rm ${G5K_BACKUP} ${2}
+case $1 in
+  -b|--backup)
+    backup $2 $3
+    ;;
+  -d|--delete)
+    delete
+    ;;
+  *)
+    echo "Usage: ./persist.sh <OPTIONS>"
+    echo ""
+    echo "OPTIONS:"
+    echo " -b | --backup <experiment-name> <addtitional-plot-archive>"
+    echo "    backup exp-data, exp-logs and espec-times from g5k NFS"
+    echo " -d | --delete"
+    echo "    delete exp-data, exp-logs and espec-times from g5k NFS"
+    ;;
+esac
 
diff --git a/experiment/README.md b/experiment/README.md
index 0517538..271aa7b 100644
--- a/experiment/README.md
+++ b/experiment/README.md
@@ -3,9 +3,8 @@
 ## Files
 
 * experiment-specitication.yml: [ESpec](https://jfed.ilabt.imec.be/espec/) for 
jFed 
-* taler.rspec: Complete set of nodes to run an experiment
-* infra.rspec: Contains all nodes for the Taler infrastructure only
-* wallets.rspec: Contains an additional set of nodes for wallets
+* taler.rspec: Complete set of nodes to run an experiment (others contain more 
wallets
+  or shards for example). Find more in ../aditional/rpsecs
 * env: template file to add enviroment variables needed for the experiment
 * scripts: Bash scripts which will be run in the experiment
   
@@ -14,7 +13,7 @@
 To successfully run an experiment the following steps must be made:
 
 **NOTE** An external Grafana instance with Taler Performance Dashboards is 
needed
-         Dashboards can be found in `additional` (TBD)
+         Dashboards can be found in `additional/grafana`
          Install instructions can be found on 
[grafana.com](https://grafana.com/docs/grafana/latest/installation/)
          Once installed, two datasources must be added - Prometheus and Loki, 
they will be updated from the experiment
 
@@ -25,22 +24,18 @@ To successfully run an experiment the following steps must 
be made:
 * Specify the experiment name and time
 * Wait until taler.rspec is allocated successfully and nodes are ready
 * Click (Re)Run ESpec for the job (use the type Directory and select this 
directory (experiment))
-* If any error ocurr just press (Re)Run Espec again because sometimes there 
are still unindentidfied errors
+* If any error ocurrs just press (Re)Run Espec again because sometimes there 
are still unindentidfied errors
 
-### Add addtional Wallet Hosts
+### Start wallets
 
-TODO - DOES NOT WORK YET - DNS host needs to be known ..
-
-* Load wallets.rspec into jFed and click Run
-* Click (Re)Run ESpec for the wallet job (type directory)
-* Run `talet-perf update prometheus` on any deployed node to make prometheus 
aware of the freshly added wallets
+* Run `talet-perf start wallets N` where N is any number
 
 **NOTE** On `taler-perf`, when not using a terminal opened from jFed make sure 
to forward the ssh-agent
          to make the script work. E.g. `ssh -A graoully-3.nancy.g5k` 
 
 ## Rebuild Taler Binaries
 
-To quickly test fixes of new commits in gnunet,exchange,merchant and wallet, 
there is a script `scripts/instal.sh`
+To quickly test fixes of new commits in gnunet,exchange,merchant and wallet, 
there is a script `scripts/install.sh`
 which can be run inside a running experiment rather than rebuilding the whole 
image.
 
 To do so copy the following snippet into the `Multi Command` window in jFed:
@@ -73,9 +68,8 @@ be stopped otherwise.
 To persists the dashboards as png plots, there is a script in `../additional` 
which creates png
 plots based on a configuration. Please refer to the README located in the 
specified directory.
 
-### Application Logs
+### Experiment Persistance
 
-The logs of the taler applications will be moved periodically to 
`/home/<G5K_USER>/taler-logs`. 
-They can then be retrieved using scp to the respective site they where running 
on
-(e.g. nancy/taler-logs for grvingt and others).
-Make sure to back them up before restarting an experiment, since they will be 
deleted in a new one.
+The script `../additional/persist.sh` can be used to backup and clean the data 
in the grid5k NFS.
+This archive created can then be passed to `../additional/recover.sh`, which 
will run a local Grafana setup
+in which the experiment can be inspected again.

-- 
To stop receiving notification emails like this one, please contact
gnunet@gnunet.org.