workbench: use different wait/sleep times when waiting for the pool to stop

Author: fmaste

nix/workbench/backend/nomad.sh

@@ -1921,7 +1921,9 @@ backend_nomad() {
     ;;
 
     wait-pools-stopped )
-      local usage="USAGE: wb backend $op RUN-DIR"
+      local usage="USAGE: wb backend $op SLEEP-SECONDS RUN-DIR"
+      # This parameters is added by the nomad backend being used.
+      local sleep_seconds=${1:?$usage}; shift
       local dir=${1:?$usage}; shift
       local generator_task=$(envjqr 'generator_task_name')
 
@@ -1976,7 +1978,10 @@ backend_nomad() {
           local elapsed="$(($(date +%s) - start_time))"
           echo -ne "\b\b\b\b\b\b"
           printf "%6d" "${elapsed}"
-          sleep 1
+          # This time is different between local and cloud backends to avoid
+          # unnecesary Nomad specific traffic and at the same time be less
+          # sensitive to network failures.
+          sleep "${sleep_seconds}"
         done # While
         if ! test -f "${dir}"/flag/cluster-stopping
         then

nix/workbench/backend/nomad/cloud.sh

@@ -52,6 +52,15 @@ backend_nomadcloud() {
       deploy-genesis-nomadcloud             "$@"
     ;;
 
+    wait-pools-stopped )
+      # It passes the sleep time (in seconds) required argument.
+      # This time is different between local and cloud backends to avoid
+      # unnecesary Nomad specific traffic (~99% happens waiting for node-0, the
+      # first one it waits to stop inside a loop) and at the same time be less
+      # sensitive to network failures.
+      backend_nomad wait-pools-stopped   60 "$@"
+    ;;
+
     fetch-logs )
       # Only if running on "perf" exclusive nodes we use SSH, if not
       # `nomad exec`, because we need to have an exclusive port open for us.
@@ -157,10 +166,6 @@ backend_nomadcloud() {
       backend_nomad wait-node-stopped       "$@"
     ;;
 
-    wait-pools-stopped )
-      backend_nomad wait-pools-stopped      "$@"
-    ;;
-
     stop-all )
       backend_nomad stop-all                "$@"
     ;;

nix/workbench/backend/nomad/exec.sh

@@ -52,6 +52,15 @@ backend_nomadexec() {
       deploy-genesis-nomadexec              "$@"
     ;;
 
+    wait-pools-stopped )
+      # It passes the sleep time (in seconds) required argument.
+      # This time is different between local and cloud backends to avoid
+      # unnecesary Nomad specific traffic (~99% happens waiting for node-0, the
+      # first one it waits to stop inside a loop) and at the same time be less
+      # sensitive to network failures.
+      backend_nomad wait-pools-stopped    1 "$@"
+    ;;
+
     # All or clean up everything!
     # Called after `scenario.sh` without an exit trap!
     stop-cluster )
@@ -113,10 +122,6 @@ backend_nomadexec() {
       backend_nomad wait-node-stopped       "$@"
     ;;
 
-    wait-pools-stopped )
-      backend_nomad wait-pools-stopped      "$@"
-    ;;
-
     stop-all )
       backend_nomad stop-all                "$@"
     ;;

nix/workbench/backend/nomad/podman.sh

@@ -52,6 +52,15 @@ backend_nomadpodman() {
       deploy-genesis-nomadpodman            "$@"
     ;;
 
+    wait-pools-stopped )
+      # It passes the sleep time (in seconds) required argument.
+      # This time is different between local and cloud backends to avoid
+      # unnecesary Nomad specific traffic (~99% happens waiting for node-0, the
+      # first one it waits to stop inside a loop) and at the same time be less
+      # sensitive to network failures.
+      backend_nomad wait-pools-stopped    1 "$@"
+    ;;
+
     # All or clean up everything!
     # Called after `scenario.sh` without an exit trap!
     stop-cluster )
@@ -113,10 +122,6 @@ backend_nomadpodman() {
       backend_nomad wait-node-stopped       "$@"
     ;;
 
-    wait-pools-stopped )
-      backend_nomad wait-pools-stopped      "$@"
-    ;;
-
     stop-all )
       backend_nomad stop-all                "$@"
     ;;