modules/drumknotty: wrap services in infinite loops

hosts/bicep/services/mysql/backup.nix

@@ -41,38 +41,24 @@ in
     path = with pkgs; [
       cfg.package
       coreutils
-      diffutils
       zstd
     ];
 
-    script = ''
+    script = let
+      rotations = 2;
+    in ''
       set -euo pipefail
 
-      dump() {
-        local name="$1" out tmp
-        out="$STATE_DIRECTORY/$name.sql.zst"
-        tmp="$out.tmp"
-        shift
-        "$@" | zstd -9 --rsyncable -f -o "$tmp"
-        if cmp -s "$tmp" "$out" 2>/dev/null; then
-          rm -f "$tmp"
-        else
-          mv -f "$tmp" "$out"
-        fi
-      }
+      OUT_FILE="$STATE_DIRECTORY/mysql-dump-$(date --iso-8601).sql.zst"
 
-      declare -A keep
-      while IFS= read -r db; do
-        [ -n "$db" ] || continue
-        dump "$db" mysqldump --skip-dump-date --databases "$db"
-        keep["$db.sql.zst"]=1
-      done < <(mysql -N -e 'SHOW DATABASES' | grep -vE '^(information_schema|performance_schema)$')
+      mysqldump --all-databases | zstd --compress -9 --rsyncable -o "$OUT_FILE"
 
-      # drop dumps of databases that no longer exist
-      for f in "$STATE_DIRECTORY"/*.sql.zst; do
-        [ -e "$f" ] || continue
-        base="$(basename "$f")"
-        [ -n "''${keep[$base]:-}" ] || rm -f "$f"
+      # NOTE: this needs to be a hardlink for rrsync to allow sending it
+      rm "$STATE_DIRECTORY/mysql-dump-latest.sql.zst" ||:
+      ln -T "$OUT_FILE" "$STATE_DIRECTORY/mysql-dump-latest.sql.zst"
+
+      while [ "$(find "$STATE_DIRECTORY" -type f -printf '.' | wc -c)" -gt '${toString (rotations + 1)}' ]; do
+        rm "$(find "$STATE_DIRECTORY" -type f -printf '%T+ %p\n' | sort | head -n 1 | cut -d' ' -f2)"
       done
     '';
 

hosts/bicep/services/postgresql/backup.nix

@@ -41,42 +41,25 @@ in
 
     path = with pkgs; [
       coreutils
-      diffutils
       zstd
       cfg.package
     ];
 
-    script = ''
+    script = let
+      rotations = 2;
+    in ''
       set -euo pipefail
 
-      dump() {
-        local name="$1" out tmp
-        out="$STATE_DIRECTORY/$name.sql.zst"
-        tmp="$out.tmp"
-        shift
-        "$@" | zstd -9 --rsyncable -f -o "$tmp"
-        if cmp -s "$tmp" "$out" 2>/dev/null; then
-          rm -f "$tmp"
-        else
-          mv -f "$tmp" "$out"
-        fi
-      }
+      OUT_FILE="$STATE_DIRECTORY/postgresql-dump-$(date --iso-8601).sql.zst"
 
-      declare -A keep
-      dump globals pg_dumpall -U postgres --globals-only --restrict-key=backup
-      keep[globals.sql.zst]=1
+      pg_dumpall -U postgres | zstd --compress -9 --rsyncable -o "$OUT_FILE"
 
-      while IFS= read -r db; do
-        [ -n "$db" ] || continue
-        dump "$db" pg_dump -U postgres -C -d "$db" --restrict-key=backup
-        keep["$db.sql.zst"]=1
-      done < <(psql -U postgres -tAc "SELECT datname FROM pg_database WHERE datallowconn ORDER BY datname")
+      # NOTE: this needs to be a hardlink for rrsync to allow sending it
+      rm "$STATE_DIRECTORY/postgresql-dump-latest.sql.zst" ||:
+      ln -T "$OUT_FILE" "$STATE_DIRECTORY/postgresql-dump-latest.sql.zst"
 
-      # drop dumps of databases that no longer exist
-      for f in "$STATE_DIRECTORY"/*.sql.zst; do
-        [ -e "$f" ] || continue
-        base="$(basename "$f")"
-        [ -n "''${keep[$base]:-}" ] || rm -f "$f"
+      while [ "$(find "$STATE_DIRECTORY" -type f -printf '.' | wc -c)" -gt '${toString (rotations + 1)}' ]; do
+        rm "$(find "$STATE_DIRECTORY" -type f -printf '%T+ %p\n' | sort | head -n 1 | cut -d' ' -f2)"
       done
     '';
 

modules/drumknotty/default.nix

@@ -138,6 +138,15 @@ in
 
         ExecStart =
           let
+            mkRespawningCommand = name: command: pkgs.writeShellScript "drumknotty-${name}-watchdog" ''
+              while true; do
+                ${command}
+                echo
+                echo "${name} exited, restarting in 3s..." >&2
+                sleep 3
+              done
+            '';
+
             screenrc = let
               convertToFile = lines: lib.pipe lines [
                 lib.concatLists
@@ -157,8 +166,7 @@ in
                   config = "/etc/dibbler/dibbler.toml";
                 };
               in lib.optionals cfg.dibbler.enable [
-                "screen -t dibbler ${lib.getExe cfg.dibbler.package} ${dibblerArgs} loop"
-
+                "screen -t dibbler ${mkRespawningCommand "dibbler" "${lib.getExe cfg.dibbler.package} ${dibblerArgs} loop"}"
               ])
 
               (let
@@ -166,7 +174,7 @@ in
                   config = "/etc/worblehat/config.toml";
                 };
               in lib.optionals cfg.worblehat.enable [
-                "screen -t worblehat ${lib.getExe cfg.worblehat.package} ${worblehatArgs} cli"
+                "screen -t worblehat ${mkRespawningCommand "worblehat" "${lib.getExe cfg.worblehat.package} ${worblehatArgs} cli"}"
               ])
 
               [ "select 0" ]