bicep/mysql: add timed jobs for analyze and optimize

base/default.nix

@@ -30,9 +30,9 @@
     ./services/journald-upload.nix
     ./services/logrotate.nix
     ./services/nginx.nix
+    ./services/nullmailer.nix
     ./services/openssh.nix
     ./services/polkit.nix
-    ./services/postfix.nix
     ./services/prometheus-flake-input-exporter.nix
     ./services/prometheus-node-exporter.nix
     ./services/prometheus-systemd-exporter.nix

base/services/irqbalance.nix

@@ -1,4 +1,22 @@
-{ ... }:
+{ config, pkgs, lib, ... }:
+let
+  cfg = config.services.irqbalance;
+in
 {
   services.irqbalance.enable = true;
+
+  # irqbalance only has meaningful work to do on multi-socket machines, so
+  # skip starting it pointlessly everywhere else.
+  systemd.services.irqbalance.serviceConfig.ExecCondition = let
+    isMultiSocket = pkgs.writeShellApplication {
+      name = "irqbalance-is-multi-socket";
+      runtimeInputs = [ pkgs.coreutils ];
+      text = ''
+        sockets=$(cat /sys/devices/system/cpu/cpu*/topology/physical_package_id | sort -u | wc -l)
+        [ "$sockets" -gt 1 ]
+      '';
+    };
+  in lib.mkIf cfg.enable [
+    (lib.getExe isMultiSocket)
+  ];
 }

base/services/nullmailer.nix

@@ -0,0 +1,15 @@
+{ config, lib, ... }:
+{
+  services.nullmailer = {
+    enable = true;
+
+    config = {
+      adminaddr = "root@pvv.ntnu.no";
+      defaultdomain = "pvv.ntnu.no";
+      defaulthost = "pvv.ntnu.no";
+
+      me = lib.mkDefault config.networking.fqdn;
+      remotes = lib.mkDefault "smtp.pvv.ntnu.no smtp port=465 tls";
+    };
+  };
+}

base/services/postfix.nix

@@ -1,22 +0,0 @@
-{ config, pkgs, lib, ... }:
-let
-  cfg = config.services.postfix;
-in
-{
-  services.postfix = {
-    enable = true;
-
-    settings.main = {
-      myhostname = "${config.networking.hostName}.pvv.ntnu.no";
-      mydomain = "pvv.ntnu.no";
-
-      # Nothing should be delivered to this machine
-      mydestination = [ ];
-
-      relayhost = [ "smtp.pvv.ntnu.no:465" ];
-
-      smtp_tls_wrappermode = "yes";
-      smtp_tls_security_level = "encrypt";
-    };
-  };
-}

flake.nix

@@ -268,6 +268,7 @@
           modules = [
             inputs.nix-gitea-themes.nixosModules.default
             inputs.disko.nixosModules.disko
+            self.nixosModules.robots-txt
           ];
         };
 

hosts/bicep/services/mysql/default.nix

@@ -4,7 +4,10 @@ let
   dataDir = "/data/mysql";
 in
 {
-  imports = [ ./backup.nix ];
+  imports = [
+    ./backup.nix
+    ./timed-jobs.nix
+  ];
 
   sops.secrets."mysql/password" = {
     owner = "mysql";
@@ -54,6 +57,18 @@ in
     mode = "0700";
   };
 
+  fileSystems.${dataDir} = lib.mkIf cfg.enable {
+    device = dataDir;
+    fsType = "none";
+    options = [
+      "bind"
+      "noatime"
+      "noauto"
+      "x-systemd.requires=systemd-tmpfiles-setup.service"
+      "x-systemd.requires=systemd-tmpfiles-resetup.service"
+    ];
+  };
+
   systemd.services.mysql = lib.mkIf cfg.enable {
     after = [
       "systemd-tmpfiles-setup.service"
@@ -61,6 +76,7 @@ in
     ];
 
     serviceConfig = {
+      RequiresMountsFor = [ dataDir ];
       BindPaths = [ "${dataDir}:${cfg.dataDir}" ];
 
       LogsDirectory = "mysql";

hosts/bicep/services/mysql/timed-jobs.nix

@@ -0,0 +1,48 @@
+{ config, lib, pkgs, ... }:
+let
+  cfg = config.services.mysql;
+in
+{
+  config = lib.mkIf cfg.enable {
+    systemd.services = {
+      mysql-analyze = {
+        requires = [ "mysql.service" ];
+        after = [ "mysql.service" ];
+        description = "Refresh MariaDB optimizer statistics for all databases";
+        startAt = "Mon 05:00:00";
+        serviceConfig = {
+          Type = "oneshot";
+          User = cfg.user;
+          Group = cfg.group;
+
+          Nice = 19;
+          IOSchedulingClass = "best-effort";
+          IOSchedulingPriority = 7;
+
+          ExecStart = "${lib.getExe' cfg.package "mariadb-check"} --all-databases --analyze";
+        };
+      };
+
+      mysql-optimize = {
+        requires = [ "mysql.service" ];
+        after = [ "mysql.service" ];
+        description = "Check, repair and optimize all MariaDB databases";
+        startAt = "*-*-01 04:00:00";
+        serviceConfig = {
+          Type = "oneshot";
+          User = cfg.user;
+          Group = cfg.group;
+
+          Nice = 19;
+          IOSchedulingClass = "best-effort";
+          IOSchedulingPriority = 7;
+
+          ExecStart = [
+            "${lib.getExe' cfg.package "mariadb-check"} --all-databases --auto-repair"
+            "${lib.getExe' cfg.package "mariadb-check"} --all-databases --optimize"
+          ];
+        };
+      };
+    };
+  };
+}

hosts/bicep/services/postgresql/default.nix

@@ -14,6 +14,7 @@ in
     extensions = ps: with ps; [ pg_repack ];
     enableTCPIP = true;
 
+    # NOTE: md5 accepts both md5 and scram-sha-256
     authentication = ''
       host all all ${values.ipv4-space} md5
       host all all ${values.ipv6-space} md5
@@ -76,6 +77,15 @@ in
       maintenance_io_concurrency = 100;
       wal_recycle = true;
 
+      # -------------------------------- #
+
+      # Authentication
+      password_encryption = "scram-sha-256";
+
+      # Logging
+      log_connections = "authorization";
+      log_disconnections = true;
+
       # SSL
       ssl = true;
       ssl_cert_file = "/run/credentials/postgresql.service/cert";
@@ -89,6 +99,18 @@ in
     mode = "0700";
   };
 
+  fileSystems."/data/postgresql" = lib.mkIf cfg.enable {
+    device = "/data/postgresql";
+    fsType = "none";
+    options = [
+      "bind"
+      "noatime"
+      "noauto"
+      "x-systemd.requires=systemd-tmpfiles-setup.service"
+      "x-systemd.requires=systemd-tmpfiles-resetup.service"
+    ];
+  };
+
   systemd.services.postgresql-setup = lib.mkIf cfg.enable {
     after = [
       "systemd-tmpfiles-setup.service"
@@ -100,6 +122,7 @@ in
         "key:/etc/certs/postgres.key"
       ];
 
+      RequiresMountsFor = [ "/data/postgresql" ];
       BindPaths = [ "/data/postgresql:/var/lib/postgresql" ];
     };
   };
@@ -115,6 +138,7 @@ in
         "key:/etc/certs/postgres.key"
       ];
 
+      RequiresMountsFor = [ "/data/postgresql" ];
       BindPaths = [ "/data/postgresql:/var/lib/postgresql" ];
     };
   };

hosts/ildkule/services/monitoring/gatus/default.nix

@@ -5,6 +5,8 @@ in
 {
   imports = [
     ./minecraft-checker.nix
+    ./postgres-checker.nix
+    ./mariadb-checker.nix
   ];
 
   services.gatus = {
@@ -133,6 +135,18 @@ in
             "[BODY].ok == true"
           ];
         })
+        (mkService "PostgreSQL" "http://localhost:1338" // {
+          conditions = [
+            "[STATUS] == 200"
+            "[BODY].ok == true"
+          ];
+        })
+        (mkService "MariaDB" "http://localhost:1339" // {
+          conditions = [
+            "[STATUS] == 200"
+            "[BODY].ok == true"
+          ];
+        })
         (mkService "Email (SMTP)" "starttls://mail.pvv.ntnu.no:587")
         (mkService "Email (POP3)" "tls://mail.pvv.ntnu.no:995")
         (mkService "Email (IMAP)" "tls://mail.pvv.ntnu.no:993")

hosts/ildkule/services/monitoring/gatus/mariadb-checker.nix

@@ -0,0 +1,79 @@
+{ config, pkgs, ... }:
+{
+  sops.secrets."keys/gatus/mariadb" = {
+    restartUnits = [ "phh-gatus-mariadb-checker.service" ];
+  };
+
+  sops.templates."gatus-mariadb-checker.env" = {
+    restartUnits = [ "phh-gatus-mariadb-checker.service" ];
+    content = ''
+      MYSQL_HOST=mysql.pvv.ntnu.no
+      MYSQL_PORT=3306
+      MYSQL_DATABASE=mysql
+      MYSQL_USER=gatus_healthcheck
+      MYSQL_PASSWORD=${config.sops.placeholder."keys/gatus/mariadb"}
+    '';
+  };
+
+  services.python-http-handlers."gatus-mariadb-checker" = {
+    listenStreams = [ "127.0.0.1:1339" ];
+    libraries = with pkgs.python3Packages; [
+      pymysql
+    ];
+    serviceConfig = {
+      EnvironmentFile = config.sops.templates."gatus-mariadb-checker.env".path;
+    };
+    handler = ''
+      import os
+      import json
+      import pymysql
+
+      class Handler(BaseHTTPRequestHandler):
+          def do_GET(self):
+              try:
+                  conn = pymysql.connect(
+                      host=os.environ["MYSQL_HOST"],
+                      port=int(os.environ.get("MYSQL_PORT", "3306")),
+                      user=os.environ["MYSQL_USER"],
+                      password=os.environ["MYSQL_PASSWORD"],
+                      database=os.environ.get("MYSQL_DATABASE", "mysql"),
+                      connect_timeout=5,
+                  )
+
+                  try:
+                      with conn.cursor() as cur:
+                          cur.execute("SELECT VERSION();")
+                          (version,) = cur.fetchone()
+
+                          cur.execute("SHOW STATUS LIKE 'Threads_connected';")
+                          (_, connections) = cur.fetchone()
+
+                          cur.execute("SHOW DATABASES;")
+                          databases = cur.fetchall()
+                  finally:
+                      conn.close()
+
+                  body = {
+                      "ok": True,
+                      "version": version,
+                      "connections": int(connections),
+                      "databases": len(databases),
+                  }
+                  data = json.dumps(body).encode()
+
+                  self.send_response(200)
+                  self.send_header("Content-Type", "application/json")
+                  self.send_header("Content-Length", str(len(data)))
+                  self.end_headers()
+                  self.wfile.write(data)
+
+              except Exception as e:
+                  data = json.dumps({"ok": False, "error": str(e)}).encode()
+                  self.send_response(500)
+                  self.send_header("Content-Type", "application/json")
+                  self.send_header("Content-Length", str(len(data)))
+                  self.end_headers()
+                  self.wfile.write(data)
+    '';
+  };
+}

hosts/ildkule/services/monitoring/gatus/postgres-checker.nix

@@ -0,0 +1,79 @@
+{ config, pkgs, ... }:
+{
+  sops.secrets."keys/gatus/postgres" = {
+    restartUnits = [ "phh-gatus-postgres-checker.service" ];
+  };
+
+  sops.templates."gatus-postgres-checker.env" = {
+    restartUnits = [ "phh-gatus-postgres-checker.service" ];
+    content = ''
+      PGHOST=postgres.pvv.ntnu.no
+      PGPORT=5432
+      PGDATABASE=postgres
+      PGUSER=gatus_healthcheck
+      PGPASSWORD=${config.sops.placeholder."keys/gatus/postgres"}
+    '';
+  };
+
+  services.python-http-handlers."gatus-postgres-checker" = {
+    listenStreams = [ "127.0.0.1:1338" ];
+    libraries = with pkgs.python3Packages; [
+      psycopg2
+    ];
+    serviceConfig = {
+      EnvironmentFile = config.sops.templates."gatus-postgres-checker.env".path;
+    };
+    handler = ''
+      import os
+      import json
+      import psycopg2
+
+      class Handler(BaseHTTPRequestHandler):
+          def do_GET(self):
+              try:
+                  conn = psycopg2.connect(
+                      host=os.environ["PGHOST"],
+                      port=os.environ.get("PGPORT", "5432"),
+                      dbname=os.environ.get("PGDATABASE", "postgres"),
+                      user=os.environ["PGUSER"],
+                      password=os.environ["PGPASSWORD"],
+                      connect_timeout=5,
+                  )
+
+                  try:
+                      with conn.cursor() as cur:
+                          cur.execute("SELECT version();")
+                          (version,) = cur.fetchone()
+
+                          cur.execute("SELECT count(*) FROM pg_stat_activity;")
+                          (connections,) = cur.fetchone()
+
+                          cur.execute("SELECT count(*) FROM pg_database WHERE NOT datistemplate;")
+                          (databases,) = cur.fetchone()
+                  finally:
+                      conn.close()
+
+                  body = {
+                      "ok": True,
+                      "version": version,
+                      "connections": connections,
+                      "databases": databases,
+                  }
+                  data = json.dumps(body).encode()
+
+                  self.send_response(200)
+                  self.send_header("Content-Type", "application/json")
+                  self.send_header("Content-Length", str(len(data)))
+                  self.end_headers()
+                  self.wfile.write(data)
+
+              except Exception as e:
+                  data = json.dumps({"ok": False, "error": str(e)}).encode()
+                  self.send_response(500)
+                  self.send_header("Content-Type", "application/json")
+                  self.send_header("Content-Length", str(len(data)))
+                  self.end_headers()
+                  self.wfile.write(data)
+    '';
+  };
+}

hosts/kommode/services/gitea/default.nix

@@ -196,6 +196,108 @@ in {
     };
   };
 
+  environment.robots-txt."gitea" = {
+    virtualHost = domain;
+    rules = [
+      {
+        pre_comment = ''
+          Gitea internals
+
+          See these for more information:
+          - https://gitea.com/robots.txt
+          - https://codeberg.org/robots.txt
+        '';
+        User-agent = "*";
+        Disallow = [
+          "/api/*"
+          "/avatars"
+          "/*/*/src/commit/*"
+          "/*/*/commit/*"
+          "/*/*/*/refs/*"
+          "/*/*/*/star"
+          "/*/*/*/watch"
+          "/*/*/labels"
+          "/*/*/activity/*"
+          "/vendor/*"
+          "/swagger.*.json"
+          "/repo/create"
+          "/repo/migrate"
+          "/org/create"
+          "/*/*/fork"
+          "/*/*/watchers"
+          "/*/*/stargazers"
+          "/*/*/forks"
+          "*/.git/"
+          "/*.git"
+          "/*.atom"
+          "/*.rss"
+        ];
+        Crawl-delay = "2";
+      }
+      {
+        pre_comment = "Language Spam";
+        Disallow = "/*?lang=";
+      }
+      {
+        pre_comment = ''
+          AI bots
+
+          Sourced from:
+          - https://www.vg.no/robots.txt
+          - https://codeberg.org/robots.txt
+        '';
+        User-agent = [
+          "AI2Bot"
+          "Ai2Bot-Dolma"
+          "Amazonbot"
+          "Applebot-Extended"
+          "Bytespider"
+          "CCBot"
+          "ChatGPT-User"
+          "Claude-Web"
+          "ClaudeBot"
+          "Crawlspace"
+          "Diffbot"
+          "FacebookBot"
+          "FriendlyCrawler"
+          "GPTBot"
+          "Google-Extended"
+          "ICC-Crawler"
+          "ImagesiftBot"
+          "Kangaroo Bot"
+          "Meta-ExternalAgent"
+          "OAI-SearchBot"
+          "Omgili"
+          "Omgilibot"
+          "PanguBot"
+          "PerplexityBot"
+          "PetalBot"
+          "Scrapy"
+          "SemrushBot-OCOB"
+          "Sidetrade indexer bot"
+          "Timpibot"
+          "VelenPublicWebCrawler"
+          "Webzio-Extended"
+          "YouBot"
+          "anthropic-ai"
+          "cohere-ai"
+          "cohere-training-data-crawler"
+          "facebookexternalhit"
+          "iaskspider/2.0"
+          "img2dataset"
+          "meta-externalagent"
+          "omgili"
+          "omgilibot"
+        ];
+        Disallow = "/";
+        Crawl-delay = "2";
+      }
+      {
+        Sitemap = "https://${domain}/sitemap.xml";
+      }
+    ];
+  };
+
   networking.firewall.allowedTCPPorts = [ sshPort ];
 
   services.rsync-pull-targets = {

hosts/temmie/services/userweb/mail.nix

@@ -1,14 +1,13 @@
 { config, lib, pkgs, ... }:
 {
-  services.postfix.enable = lib.mkForce false;
+  assertions = [{
+    assertion = config.services.nullmailer.enable;
+    message = ''
+      Expected nullmailer to be enabled for temmie userweb.
 
-  services.nullmailer = {
+      If you change the default sendmail implementation, please make sure that temmie userweb works correctly!
-    enable = true;
+    '';
-    config = {
+  }];
-      me = config.networking.fqdn;
-      remotes = "mail.pvv.ntnu.no smtp --port=25";
-    };
-  };
 
   services.bro = {
     enable = true;

modules/python-http-handlers.nix

@@ -36,9 +36,6 @@ in
           description = ''
             For each item in this list, a `ListenStream`
             option in the `[Socket]` section will be created.
-
-            Only a single `ListenStream` is currently supported by the handler script; if
-            you need more than one, you'll have to adjust {option}`handler` accordingly.
           '';
         };
 
@@ -101,10 +98,8 @@ in
             "E303" # too many blank lines
             "E305" # expected 2 blank lines after end of function or class
             "E306" # expected 1 blank line before a nested definition
-            "E402" # module level import not at top of file
             "E501" # max line length
             "E704" # multiple statements on one line (def)
-            "F811" # redefined while unused
           ];
           description = ''
             A list of flake8 rules to ignore while linting the python code.
@@ -127,11 +122,6 @@ in
                     self.send_header("Content-Length", str(len(data)))
                     self.end_headers()
                     self.wfile.write(data)
-
-            def on_reload():
-                # This function is called when the service receives SIGHUP (e.g. via `systemctl reload`).
-                # You can use it to clear caches or re-read state. Completely optional
-                pass
           '';
           description = ''
             Python code including the HTTP handler for the server.
@@ -155,7 +145,6 @@ in
           inherit (v) listenStreams;
           socketConfig = {
             Accept = false;
-            FileDescriptorName = v.name;
           } // v.socketConfig;
         };
       }))
@@ -168,126 +157,30 @@ in
         inherit (v) name;
         value = {
           serviceConfig = {
-            Type = "notify-reload";
+            Type = "simple";
-            NotifyAccess = "main";
             DynamicUser = true;
-            TimeoutStopSec = "35s";
 
             ExecStart = let
               package = pkgs.writers.writePython3Bin "${v.name}-bin" {
                 inherit (v) libraries flakeIgnore;
               } ''
-                import os
-                import signal
                 import socket
-                import socketserver
-                import threading
-                import time
                 from http.server import HTTPServer, BaseHTTPRequestHandler
 
-                SOCKET_NAME = "${v.name}"
-                SHUTDOWN_TIMEOUT = 30
-
-                def sd_notify(message: str):
-                    addr = os.environ.get("NOTIFY_SOCKET")
-                    if not addr:
-                        return
-                    if addr[0] == "@":
-                        addr = "\0" + addr[1:]
-                    with socket.socket(socket.AF_UNIX, socket.SOCK_DGRAM | socket.SOCK_CLOEXEC) as sock:
-                        sock.connect(addr)
-                        sock.sendall(message.encode())
-
-                def sd_listen_fd(name: str) -> int:
-                    if os.environ.get("LISTEN_PID") != str(os.getpid()):
-                        raise RuntimeError("No sockets were passed to this service by systemd")
-
-                    try:
-                        count = int(os.environ.get("LISTEN_FDS", "0"))
-                    except ValueError:
-                        count = 0
-
-                    raw_names = os.environ.get("LISTEN_FDNAMES")
-                    names = raw_names.split(":") if raw_names else []
-
-                    for i in range(count):
-                        if i < len(names) and names[i] == name:
-                            return 3 + i
-
-                    raise RuntimeError(
-                        "No systemd socket named %r was passed to this service; check the "
-                        "FileDescriptorName= of the corresponding .socket unit" % name
-                    )
-
-                class Server(socketserver.ThreadingMixIn, HTTPServer):
-                    daemon_threads = True
-
-                    def server_bind(): pass
-                    def server_activate(): pass
-
-                    def __init__(self, *args, **kwargs):
-                        super().__init__(*args, **kwargs)
-                        self._request_threads = []
-                        self._request_threads_lock = threading.Lock()
-
-                    def process_request(self, request, client_address):
-                        thread = threading.Thread(
-                            target=self.process_request_thread,
-                            args=(request, client_address),
-                        )
-                        thread.daemon = self.daemon_threads
-                        with self._request_threads_lock:
-                            self._request_threads.append(thread)
-                        thread.start()
-
-                    def join_request_threads(self, timeout):
-                        deadline = time.monotonic() + timeout
-                        with self._request_threads_lock:
-                            threads = list(self._request_threads)
-                        for thread in threads:
-                            thread.join(max(deadline - time.monotonic(), 0))
-
-                def handle_reload(signum, frame):
-                    monotonic_usec = time.clock_gettime_ns(time.CLOCK_MONOTONIC) // 1000
-                    sd_notify("RELOADING=1\nMONOTONIC_USEC=%d" % monotonic_usec)
-
-                    on_reload = globals().get("on_reload")
-                    if callable(on_reload):
-                        on_reload()
-
-                    sd_notify("READY=1")
-
-                shutdown_requested = threading.Event()
-
-                def handle_sigterm(signum, frame):
-                    sd_notify("STOPPING=1")
-                    shutdown_requested.set()
-
                 ${v.handler}
 
-                assert "Handler" in globals(), "You must define a class Handler(BaseHTTPRequestHandler) in the handler code"
+                class NoBindHTTPServer(HTTPServer):
+                    def server_bind(): pass
+                    def server_activate(): pass
 
                 def main():
-                    signal.signal(signal.SIGHUP, handle_reload)
+                    httpd = NoBindHTTPServer(
-                    signal.signal(signal.SIGTERM, handle_sigterm)
+                        ("", 0),
-
+                        Handler,
-                    fd = sd_listen_fd(SOCKET_NAME)
+                        bind_and_activate=False,
-
+                    )
-                    httpd = Server(("", 0), Handler, bind_and_activate=False)
+                    httpd.socket = socket.fromfd(3, socket.AF_INET, socket.SOCK_STREAM)
-                    httpd.socket = socket.socket(fileno=fd)
+                    httpd.serve_forever()
-
-                    server_thread = threading.Thread(target=httpd.serve_forever, name="http-server", daemon=True)
-                    server_thread.start()
-
-                    sd_notify("READY=1")
-                    shutdown_requested.wait()
-
-                    deadline = time.monotonic() + SHUTDOWN_TIMEOUT
-
-                    httpd.shutdown()
-                    server_thread.join(max(deadline - time.monotonic(), 0))
-                    httpd.join_request_threads(max(deadline - time.monotonic(), 0))
-                    httpd.server_close()
 
                 if __name__ == '__main__':
                     main()

secrets/ildkule/ildkule.yaml

@@ -1,6 +1,9 @@
 config:
     mysqld_exporter_password: ENC[AES256_GCM,data:I9K+QMqaN3FOOVKzeOR9Q6UERStXX0P8WEHyN1jzzbM=,iv:UxvIdlfAyJvNuxPkU4+guKPa0fiD0vVLzHOTYktcmso=,tag:ltnIqEwESYx9HBu8UN0ZLw==,type:str]
 keys:
+    gatus:
+        mariadb: ENC[AES256_GCM,data:U7CGtjHacJlKcReEw3JO86okCVgLAkP92iX1bHGNw8qeImpvk5mNw77Ox7tbsChvozrJWa/Npu2pTBsNnVYYYA==,iv:hWa+KPiBc3AnH1tHCGJa8LhPXZ1o8sxufpGe7nUjV0E=,tag:mso5HEfmWAOGw350408X7g==,type:str]
+        postgres: ENC[AES256_GCM,data:7fI0JeZDK1MX5poXWaMo67PXbjdjvM+xPKioHSHmGdS8iNDGe5Ym6LlS/hOnPvvDuNFyHjKKizqDst0227U2gg==,iv:D3bz/xyFrnQsUCiMEFq8JOjSudyiw+idEMWQ1e6Uk3k=,tag:8XqadKVHK9NMWehK7V5J0Q==,type:str]
     grafana:
         secret_key: ENC[AES256_GCM,data:+WoAJbDBEgKs0RoHT+7oEELAVQ+/2Xt+5RTMSXg23moCqVRx+Gzll9P5Drw=,iv:AkRn/Y20iEe5i1T+84wAgLCTFtAox2G3giyawAkltAw=,tag:BZbt5Wb5lYLIJBm/pfP4GQ==,type:str]
         admin_password: ENC[AES256_GCM,data:ttKwfC4WuXeL/6x4,iv:x1X+e3z08CR992GzC62YnFIN7SGrE81/nDNrgcgVzx0=,tag:YajUoy61kYbpeGeC7yNrXQ==,type:str]
@@ -9,8 +12,7 @@ keys:
         postgres_exporter_knakelibrak_env: ENC[AES256_GCM,data:xjC7DGXrW2GIJq8XioIZb+jSe/Hzcz0tv9cUHmX/n1nhI+D64lYt+EKnq1+RX/vJzU4sTaKjveKBh88Qqnv6RQm+MZC//dIxcvnnAdl50qnHZyBCaFFEzSNI8I8vGyArMk8Ja72clBq3kMpUz/pLBP0qDrjblKDoWkU=,iv:ZW98hJy8A5t4Oxtu17R3tM7gou183VLbgBsHA8LFuJY=,tag:VMOvQz3X/XDylV1YFg2Jsg==,type:str]
 sops:
     age:
-        - recipient: age102e6y8gah0ntr6fxqnkpepc8ar29p6ls7ks9ka7v8w87q8scm9yqmc2u8d
+        - enc: |
-          enc: |
             -----BEGIN AGE ENCRYPTED FILE-----
             YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSAyTWRSM3IwMmxtTmZVcCsw
             OUhlakxHZzgrSEhEdUZFTXE1anNjQ2wvdkZFCnB6S1l3TXQ3ZGFYWmtYM1cwMFZT
@@ -18,8 +20,8 @@ sops:
             YlltQ3FBU3RBYUx4TnNPRk1SUWNqZG8KAJjc09x553ncaWduGLsnIHdroaOmMasP
             /fq0GzW6UNfmE2rQ6qrQti21B37/sN0WMLCSPLUPG45kBgx20GG4hQ==
             -----END AGE ENCRYPTED FILE-----
-        - recipient: age1ug30gg4y7ftuya0wdv7q0vh4egn00wlv2th7mt7cgc2ze46wmvyq9lq6ge
+          recipient: age102e6y8gah0ntr6fxqnkpepc8ar29p6ls7ks9ka7v8w87q8scm9yqmc2u8d
-          enc: |
+        - enc: |
             -----BEGIN AGE ENCRYPTED FILE-----
             YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBTOWFvcFFzc1lmNVdmV3lX
             cVVLNHowcGdENzI5UUJLZTNSMHhjNlI2c0FnCmdwOS9oL1kwTnhwbXRodWxxWVE3
@@ -27,8 +29,8 @@ sops:
             eDVwd2dKMG9FRW1OY1pyUkhLeWw3b0EKtJpsQ/Ss39ZLiRNqUhn8sdB3hpQy7Syv
             ererqhMkqmDugGEHPk6KpZuj7DVSK1di7JgA2qZOUPzI7UpxjaC0Kg==
             -----END AGE ENCRYPTED FILE-----
-        - recipient: age1mrnldl334l2nszuta6ywvewng0fswv2dz9l5g4qcwe3nj4yxf92qjskdx6
+          recipient: age1ug30gg4y7ftuya0wdv7q0vh4egn00wlv2th7mt7cgc2ze46wmvyq9lq6ge
-          enc: |
+        - enc: |
             -----BEGIN AGE ENCRYPTED FILE-----
             YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBOQU1iVXRkQmo4b3F1Vngz
             S0pkNkVFR1FUb1djdmI0eHh3V3BBTDJTSTJZClA0S0Z0cTdFRmRaOEZQdHQzdGZ3
@@ -36,8 +38,8 @@ sops:
             REdjRFZyY2pNdEd6cmgvQisyVDhLUEkKRItJ0CGbzlEB5RNAyem4feMVhTfcLef3
             QIqltZ2l4LLexnkECi3FCJZHxrbUa+/RF6p1DsueUw7LLUnOcphB9A==
             -----END AGE ENCRYPTED FILE-----
-        - recipient: age1hmpdk4h69wxpwqk9tkud39f66hprhehxtzhgw97r6dvr7v0mx5jscsuhkn
+          recipient: age1mrnldl334l2nszuta6ywvewng0fswv2dz9l5g4qcwe3nj4yxf92qjskdx6
-          enc: |
+        - enc: |
             -----BEGIN AGE ENCRYPTED FILE-----
             YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSAvZTBpaWM0c1hmODBaK0Iw
             bmp1NzRacklXMHU0K2J2d0g5ajBiNlRNWGhRCm1DOWI0cm5BdTdlNmFzM2JVekNk
@@ -45,8 +47,8 @@ sops:
             U0ZCT2toZ1ZMZ3E0bXRhSTQvNGFWNVkKhxfQDIDe2LQW7OMBJv0J267AW1wI32df
             ZQxd657TEqzm7i19azrCS0jyRbfj2MYzEJAtTGiGZaNC9uKDFzBhKw==
             -----END AGE ENCRYPTED FILE-----
-        - recipient: age1wrssr4z4g6vl3fd3qme5cewchmmhm0j2xe6wf2meu4r6ycn37anse98mfs
+          recipient: age1hmpdk4h69wxpwqk9tkud39f66hprhehxtzhgw97r6dvr7v0mx5jscsuhkn
-          enc: |
+        - enc: |
             -----BEGIN AGE ENCRYPTED FILE-----
             YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBmRlJROHRKb21YUnlicmc1
             MlptQllEcXFhajNKS1krMDdUMWk5QWo4eVJBCndGSlhXS1Vaa2RSTllIcmF1ZVpl
@@ -54,8 +56,8 @@ sops:
             TE41aFdjU0h0ekQ2Zjg4Z3VQVjFWcnMK6zjSalqeYjyc4NH6nOeghlhYJydrz4pM
             N5ZcXjRbrIVFdhbYnvQGKvGKZm0kK6vjzBjdT7BM6ctr8cq/qrz1xQ==
             -----END AGE ENCRYPTED FILE-----
-        - recipient: age1zhxul786an743u0fascv4wtc5xduu7qfy803lfs539yzhgmlq5ds2lznt5
+          recipient: age1wrssr4z4g6vl3fd3qme5cewchmmhm0j2xe6wf2meu4r6ycn37anse98mfs
-          enc: |
+        - enc: |
             -----BEGIN AGE ENCRYPTED FILE-----
             YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBIRXpBa2tYc2xEeldub0VK
             bkFwOFdlUGRZM0FVT0tyUW1RWnl0TjRUTUh3CnZlMC92MU1hRW1yZU1NSUdoUEZh
@@ -63,8 +65,8 @@ sops:
             aXFQWlNVQ2laVm1ETStRemNZRXc3TUEKlPYSU3gp67dsPfbEJkru4ieMvspC7+pu
             rfp315HLyj1FGhrA8f2qOxE/PYI2rn0yKm80KffWBV7ylX/uonm4Fg==
             -----END AGE ENCRYPTED FILE-----
-        - recipient: age1sqs7urnzsdy64efmd0zukzv3gs5pnjksuxd7nqmdwdy5l0nqnunq6hyune
+          recipient: age1zhxul786an743u0fascv4wtc5xduu7qfy803lfs539yzhgmlq5ds2lznt5
-          enc: |
+        - enc: |
             -----BEGIN AGE ENCRYPTED FILE-----
             YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBsaTEzOFBEeG9LVThSVmQ5
             VXBoaFpueFRCbFJ1akE4RWc1aE1HUmVGcXdZCjFnbU0wd2drazNsTmNBMHNuOFhO
@@ -72,8 +74,9 @@ sops:
             dC9meDZlc3d3aUJEVjc4REF0Y1BLcGcK79LbJzc5KVgEgyJR11crGuX8YcVoJBbT
             Fin7Zoon06L7qx0Zw5u27wV7RKMnYT7hOMiWs6660ZTLcYJ5M1aEZQ==
             -----END AGE ENCRYPTED FILE-----
-    lastmodified: "2025-03-16T20:08:18Z"
+          recipient: age1sqs7urnzsdy64efmd0zukzv3gs5pnjksuxd7nqmdwdy5l0nqnunq6hyune
-    mac: ENC[AES256_GCM,data:C2tpWppc13jKJq5d4nmAKQOaNWHm27TKwxAxm1fi2lejN1lqUaoz5bHfTBA7MfaWvuP5uZnfbtG32eeu48mnlWpo58XRUFFecAhb9JUpW9s5IR3/nbzLNkGU7H5C0oWPrxI4thd+bAVduIgBjjFyGj1pe6J9db3c0yUWRwNlwGU=,iv:YpoQ4psiFYOWLGipxv1QvRvr034XFsyn2Bhyy39HmOo=,tag:ByiCWygFC/VokVTbdLoLgg==,type:str]
+    lastmodified: "2026-07-21T09:40:54Z"
+    mac: ENC[AES256_GCM,data:FLpmsW9I21qNMFs6Tn/1AzFo+RcmCwtVdAaaGIrGlUi2l3gzb//ZK6ruI/VYx1R5DoX1OhD3EIg6aLMSOT6jiHf+8GwfhdijpayG1nrgeZW3SuFbjVlCYNMPH3DOJJB82U3T+ozSbpnZEtdUwmNWcZJt5F2hOEnoaNa+jLEKDao=,iv:qyQuyWuu+w2ynRvhIx0QVtw+tieO3dbQsUxEB5Brzp0=,tag:qYAj4h6foWm16YqIdZ6xbQ==,type:str]
     pgp:
         - created_at: "2026-05-20T17:35:58Z"
           enc: |-
@@ -96,4 +99,4 @@ sops:
             -----END PGP MESSAGE-----
           fp: F7D37890228A907440E1FD4846B9228E814A2AAC
     unencrypted_suffix: _unencrypted
-    version: 3.9.4
+    version: 3.13.2