modules/python-http-handlers: better daemon handling

This commit extends the python http handler template with the following
things:

- Integration with systemd notify
- Named systemd sockets, to avoid depending on expected FD number
- Graceful and customizable handling of SIGHUP reloads
- Graceful shutdown on SIGTERM with timeout
- Multithreaded HTTP server, one thread per request
- Better error message on undefined handler

base/default.nix

@@ -30,9 +30,9 @@
     ./services/journald-upload.nix
     ./services/logrotate.nix
     ./services/nginx.nix
-    ./services/nullmailer.nix
     ./services/openssh.nix
     ./services/polkit.nix
+    ./services/postfix.nix
     ./services/prometheus-flake-input-exporter.nix
     ./services/prometheus-node-exporter.nix
     ./services/prometheus-systemd-exporter.nix

base/services/irqbalance.nix

@@ -1,22 +1,4 @@
-{ config, pkgs, lib, ... }:
+{ ... }:
-let
-  cfg = config.services.irqbalance;
-in
 {
   services.irqbalance.enable = true;
-
-  # irqbalance only has meaningful work to do on multi-socket machines, so
-  # skip starting it pointlessly everywhere else.
-  systemd.services.irqbalance.serviceConfig.ExecCondition = let
-    isMultiSocket = pkgs.writeShellApplication {
-      name = "irqbalance-is-multi-socket";
-      runtimeInputs = [ pkgs.coreutils ];
-      text = ''
-        sockets=$(cat /sys/devices/system/cpu/cpu*/topology/physical_package_id | sort -u | wc -l)
-        [ "$sockets" -gt 1 ]
-      '';
-    };
-  in lib.mkIf cfg.enable [
-    (lib.getExe isMultiSocket)
-  ];
 }

base/services/nullmailer.nix

@@ -1,15 +0,0 @@
-{ config, lib, ... }:
-{
-  services.nullmailer = {
-    enable = true;
-
-    config = {
-      adminaddr = "root@pvv.ntnu.no";
-      defaultdomain = "pvv.ntnu.no";
-      defaulthost = "pvv.ntnu.no";
-
-      me = lib.mkDefault config.networking.fqdn;
-      remotes = lib.mkDefault "smtp.pvv.ntnu.no smtp port=465 tls";
-    };
-  };
-}

base/services/postfix.nix

@@ -0,0 +1,22 @@
+{ config, pkgs, lib, ... }:
+let
+  cfg = config.services.postfix;
+in
+{
+  services.postfix = {
+    enable = true;
+
+    settings.main = {
+      myhostname = "${config.networking.hostName}.pvv.ntnu.no";
+      mydomain = "pvv.ntnu.no";
+
+      # Nothing should be delivered to this machine
+      mydestination = [ ];
+
+      relayhost = [ "smtp.pvv.ntnu.no:465" ];
+
+      smtp_tls_wrappermode = "yes";
+      smtp_tls_security_level = "encrypt";
+    };
+  };
+}

flake.nix

@@ -268,7 +268,6 @@
           modules = [
             inputs.nix-gitea-themes.nixosModules.default
             inputs.disko.nixosModules.disko
-            self.nixosModules.robots-txt
           ];
         };
 

hosts/bicep/services/mysql/default.nix

@@ -4,10 +4,7 @@ let
   dataDir = "/data/mysql";
 in
 {
-  imports = [
+  imports = [ ./backup.nix ];
-    ./backup.nix
-    ./timed-jobs.nix
-  ];
 
   sops.secrets."mysql/password" = {
     owner = "mysql";
@@ -57,18 +54,6 @@ in
     mode = "0700";
   };
 
-  fileSystems.${dataDir} = lib.mkIf cfg.enable {
-    device = dataDir;
-    fsType = "none";
-    options = [
-      "bind"
-      "noatime"
-      "noauto"
-      "x-systemd.requires=systemd-tmpfiles-setup.service"
-      "x-systemd.requires=systemd-tmpfiles-resetup.service"
-    ];
-  };
-
   systemd.services.mysql = lib.mkIf cfg.enable {
     after = [
       "systemd-tmpfiles-setup.service"
@@ -76,7 +61,6 @@ in
     ];
 
     serviceConfig = {
-      RequiresMountsFor = [ dataDir ];
       BindPaths = [ "${dataDir}:${cfg.dataDir}" ];
 
       LogsDirectory = "mysql";

hosts/bicep/services/mysql/timed-jobs.nix

@@ -1,48 +0,0 @@
-{ config, lib, pkgs, ... }:
-let
-  cfg = config.services.mysql;
-in
-{
-  config = lib.mkIf cfg.enable {
-    systemd.services = {
-      mysql-analyze = {
-        requires = [ "mysql.service" ];
-        after = [ "mysql.service" ];
-        description = "Refresh MariaDB optimizer statistics for all databases";
-        startAt = "Mon 05:00:00";
-        serviceConfig = {
-          Type = "oneshot";
-          User = cfg.user;
-          Group = cfg.group;
-
-          Nice = 19;
-          IOSchedulingClass = "best-effort";
-          IOSchedulingPriority = 7;
-
-          ExecStart = "${lib.getExe' cfg.package "mariadb-check"} --all-databases --analyze";
-        };
-      };
-
-      mysql-optimize = {
-        requires = [ "mysql.service" ];
-        after = [ "mysql.service" ];
-        description = "Check, repair and optimize all MariaDB databases";
-        startAt = "*-*-01 04:00:00";
-        serviceConfig = {
-          Type = "oneshot";
-          User = cfg.user;
-          Group = cfg.group;
-
-          Nice = 19;
-          IOSchedulingClass = "best-effort";
-          IOSchedulingPriority = 7;
-
-          ExecStart = [
-            "${lib.getExe' cfg.package "mariadb-check"} --all-databases --auto-repair"
-            "${lib.getExe' cfg.package "mariadb-check"} --all-databases --optimize"
-          ];
-        };
-      };
-    };
-  };
-}

hosts/bicep/services/postgresql/default.nix

@@ -14,7 +14,6 @@ in
     extensions = ps: with ps; [ pg_repack ];
     enableTCPIP = true;
 
-    # NOTE: md5 accepts both md5 and scram-sha-256
     authentication = ''
       host all all ${values.ipv4-space} md5
       host all all ${values.ipv6-space} md5
@@ -77,15 +76,6 @@ in
       maintenance_io_concurrency = 100;
       wal_recycle = true;
 
-      # -------------------------------- #
-
-      # Authentication
-      password_encryption = "scram-sha-256";
-
-      # Logging
-      log_connections = "authorization";
-      log_disconnections = true;
-
       # SSL
       ssl = true;
       ssl_cert_file = "/run/credentials/postgresql.service/cert";
@@ -99,18 +89,6 @@ in
     mode = "0700";
   };
 
-  fileSystems."/data/postgresql" = lib.mkIf cfg.enable {
-    device = "/data/postgresql";
-    fsType = "none";
-    options = [
-      "bind"
-      "noatime"
-      "noauto"
-      "x-systemd.requires=systemd-tmpfiles-setup.service"
-      "x-systemd.requires=systemd-tmpfiles-resetup.service"
-    ];
-  };
-
   systemd.services.postgresql-setup = lib.mkIf cfg.enable {
     after = [
       "systemd-tmpfiles-setup.service"
@@ -122,7 +100,6 @@ in
         "key:/etc/certs/postgres.key"
       ];
 
-      RequiresMountsFor = [ "/data/postgresql" ];
       BindPaths = [ "/data/postgresql:/var/lib/postgresql" ];
     };
   };
@@ -138,7 +115,6 @@ in
         "key:/etc/certs/postgres.key"
       ];
 
-      RequiresMountsFor = [ "/data/postgresql" ];
       BindPaths = [ "/data/postgresql:/var/lib/postgresql" ];
     };
   };

hosts/ildkule/services/monitoring/gatus/default.nix

@@ -5,8 +5,6 @@ in
 {
   imports = [
     ./minecraft-checker.nix
-    ./postgres-checker.nix
-    ./mariadb-checker.nix
   ];
 
   services.gatus = {
@@ -135,18 +133,6 @@ in
             "[BODY].ok == true"
           ];
         })
-        (mkService "PostgreSQL" "http://localhost:1338" // {
-          conditions = [
-            "[STATUS] == 200"
-            "[BODY].ok == true"
-          ];
-        })
-        (mkService "MariaDB" "http://localhost:1339" // {
-          conditions = [
-            "[STATUS] == 200"
-            "[BODY].ok == true"
-          ];
-        })
         (mkService "Email (SMTP)" "starttls://mail.pvv.ntnu.no:587")
         (mkService "Email (POP3)" "tls://mail.pvv.ntnu.no:995")
         (mkService "Email (IMAP)" "tls://mail.pvv.ntnu.no:993")

hosts/ildkule/services/monitoring/gatus/mariadb-checker.nix

@@ -1,79 +0,0 @@
-{ config, pkgs, ... }:
-{
-  sops.secrets."keys/gatus/mariadb" = {
-    restartUnits = [ "phh-gatus-mariadb-checker.service" ];
-  };
-
-  sops.templates."gatus-mariadb-checker.env" = {
-    restartUnits = [ "phh-gatus-mariadb-checker.service" ];
-    content = ''
-      MYSQL_HOST=mysql.pvv.ntnu.no
-      MYSQL_PORT=3306
-      MYSQL_DATABASE=mysql
-      MYSQL_USER=gatus_healthcheck
-      MYSQL_PASSWORD=${config.sops.placeholder."keys/gatus/mariadb"}
-    '';
-  };
-
-  services.python-http-handlers."gatus-mariadb-checker" = {
-    listenStreams = [ "127.0.0.1:1339" ];
-    libraries = with pkgs.python3Packages; [
-      pymysql
-    ];
-    serviceConfig = {
-      EnvironmentFile = config.sops.templates."gatus-mariadb-checker.env".path;
-    };
-    handler = ''
-      import os
-      import json
-      import pymysql
-
-      class Handler(BaseHTTPRequestHandler):
-          def do_GET(self):
-              try:
-                  conn = pymysql.connect(
-                      host=os.environ["MYSQL_HOST"],
-                      port=int(os.environ.get("MYSQL_PORT", "3306")),
-                      user=os.environ["MYSQL_USER"],
-                      password=os.environ["MYSQL_PASSWORD"],
-                      database=os.environ.get("MYSQL_DATABASE", "mysql"),
-                      connect_timeout=5,
-                  )
-
-                  try:
-                      with conn.cursor() as cur:
-                          cur.execute("SELECT VERSION();")
-                          (version,) = cur.fetchone()
-
-                          cur.execute("SHOW STATUS LIKE 'Threads_connected';")
-                          (_, connections) = cur.fetchone()
-
-                          cur.execute("SHOW DATABASES;")
-                          databases = cur.fetchall()
-                  finally:
-                      conn.close()
-
-                  body = {
-                      "ok": True,
-                      "version": version,
-                      "connections": int(connections),
-                      "databases": len(databases),
-                  }
-                  data = json.dumps(body).encode()
-
-                  self.send_response(200)
-                  self.send_header("Content-Type", "application/json")
-                  self.send_header("Content-Length", str(len(data)))
-                  self.end_headers()
-                  self.wfile.write(data)
-
-              except Exception as e:
-                  data = json.dumps({"ok": False, "error": str(e)}).encode()
-                  self.send_response(500)
-                  self.send_header("Content-Type", "application/json")
-                  self.send_header("Content-Length", str(len(data)))
-                  self.end_headers()
-                  self.wfile.write(data)
-    '';
-  };
-}

hosts/ildkule/services/monitoring/gatus/postgres-checker.nix

@@ -1,79 +0,0 @@
-{ config, pkgs, ... }:
-{
-  sops.secrets."keys/gatus/postgres" = {
-    restartUnits = [ "phh-gatus-postgres-checker.service" ];
-  };
-
-  sops.templates."gatus-postgres-checker.env" = {
-    restartUnits = [ "phh-gatus-postgres-checker.service" ];
-    content = ''
-      PGHOST=postgres.pvv.ntnu.no
-      PGPORT=5432
-      PGDATABASE=postgres
-      PGUSER=gatus_healthcheck
-      PGPASSWORD=${config.sops.placeholder."keys/gatus/postgres"}
-    '';
-  };
-
-  services.python-http-handlers."gatus-postgres-checker" = {
-    listenStreams = [ "127.0.0.1:1338" ];
-    libraries = with pkgs.python3Packages; [
-      psycopg2
-    ];
-    serviceConfig = {
-      EnvironmentFile = config.sops.templates."gatus-postgres-checker.env".path;
-    };
-    handler = ''
-      import os
-      import json
-      import psycopg2
-
-      class Handler(BaseHTTPRequestHandler):
-          def do_GET(self):
-              try:
-                  conn = psycopg2.connect(
-                      host=os.environ["PGHOST"],
-                      port=os.environ.get("PGPORT", "5432"),
-                      dbname=os.environ.get("PGDATABASE", "postgres"),
-                      user=os.environ["PGUSER"],
-                      password=os.environ["PGPASSWORD"],
-                      connect_timeout=5,
-                  )
-
-                  try:
-                      with conn.cursor() as cur:
-                          cur.execute("SELECT version();")
-                          (version,) = cur.fetchone()
-
-                          cur.execute("SELECT count(*) FROM pg_stat_activity;")
-                          (connections,) = cur.fetchone()
-
-                          cur.execute("SELECT count(*) FROM pg_database WHERE NOT datistemplate;")
-                          (databases,) = cur.fetchone()
-                  finally:
-                      conn.close()
-
-                  body = {
-                      "ok": True,
-                      "version": version,
-                      "connections": connections,
-                      "databases": databases,
-                  }
-                  data = json.dumps(body).encode()
-
-                  self.send_response(200)
-                  self.send_header("Content-Type", "application/json")
-                  self.send_header("Content-Length", str(len(data)))
-                  self.end_headers()
-                  self.wfile.write(data)
-
-              except Exception as e:
-                  data = json.dumps({"ok": False, "error": str(e)}).encode()
-                  self.send_response(500)
-                  self.send_header("Content-Type", "application/json")
-                  self.send_header("Content-Length", str(len(data)))
-                  self.end_headers()
-                  self.wfile.write(data)
-    '';
-  };
-}

hosts/kommode/services/gitea/default.nix

@@ -196,108 +196,6 @@ in {
     };
   };
 
-  environment.robots-txt."gitea" = {
-    virtualHost = domain;
-    rules = [
-      {
-        pre_comment = ''
-          Gitea internals
-
-          See these for more information:
-          - https://gitea.com/robots.txt
-          - https://codeberg.org/robots.txt
-        '';
-        User-agent = "*";
-        Disallow = [
-          "/api/*"
-          "/avatars"
-          "/*/*/src/commit/*"
-          "/*/*/commit/*"
-          "/*/*/*/refs/*"
-          "/*/*/*/star"
-          "/*/*/*/watch"
-          "/*/*/labels"
-          "/*/*/activity/*"
-          "/vendor/*"
-          "/swagger.*.json"
-          "/repo/create"
-          "/repo/migrate"
-          "/org/create"
-          "/*/*/fork"
-          "/*/*/watchers"
-          "/*/*/stargazers"
-          "/*/*/forks"
-          "*/.git/"
-          "/*.git"
-          "/*.atom"
-          "/*.rss"
-        ];
-        Crawl-delay = "2";
-      }
-      {
-        pre_comment = "Language Spam";
-        Disallow = "/*?lang=";
-      }
-      {
-        pre_comment = ''
-          AI bots
-
-          Sourced from:
-          - https://www.vg.no/robots.txt
-          - https://codeberg.org/robots.txt
-        '';
-        User-agent = [
-          "AI2Bot"
-          "Ai2Bot-Dolma"
-          "Amazonbot"
-          "Applebot-Extended"
-          "Bytespider"
-          "CCBot"
-          "ChatGPT-User"
-          "Claude-Web"
-          "ClaudeBot"
-          "Crawlspace"
-          "Diffbot"
-          "FacebookBot"
-          "FriendlyCrawler"
-          "GPTBot"
-          "Google-Extended"
-          "ICC-Crawler"
-          "ImagesiftBot"
-          "Kangaroo Bot"
-          "Meta-ExternalAgent"
-          "OAI-SearchBot"
-          "Omgili"
-          "Omgilibot"
-          "PanguBot"
-          "PerplexityBot"
-          "PetalBot"
-          "Scrapy"
-          "SemrushBot-OCOB"
-          "Sidetrade indexer bot"
-          "Timpibot"
-          "VelenPublicWebCrawler"
-          "Webzio-Extended"
-          "YouBot"
-          "anthropic-ai"
-          "cohere-ai"
-          "cohere-training-data-crawler"
-          "facebookexternalhit"
-          "iaskspider/2.0"
-          "img2dataset"
-          "meta-externalagent"
-          "omgili"
-          "omgilibot"
-        ];
-        Disallow = "/";
-        Crawl-delay = "2";
-      }
-      {
-        Sitemap = "https://${domain}/sitemap.xml";
-      }
-    ];
-  };
-
   networking.firewall.allowedTCPPorts = [ sshPort ];
 
   services.rsync-pull-targets = {

hosts/temmie/services/userweb/mail.nix

@@ -1,13 +1,14 @@
 { config, lib, pkgs, ... }:
 {
-  assertions = [{
+  services.postfix.enable = lib.mkForce false;
-    assertion = config.services.nullmailer.enable;
-    message = ''
-      Expected nullmailer to be enabled for temmie userweb.
 
-      If you change the default sendmail implementation, please make sure that temmie userweb works correctly!
+  services.nullmailer = {
-    '';
+    enable = true;
-  }];
+    config = {
+      me = config.networking.fqdn;
+      remotes = "mail.pvv.ntnu.no smtp --port=25";
+    };
+  };
 
   services.bro = {
     enable = true;

modules/python-http-handlers.nix

@@ -36,6 +36,9 @@ in
           description = ''
             For each item in this list, a `ListenStream`
             option in the `[Socket]` section will be created.
+
+            Only a single `ListenStream` is currently supported by the handler script; if
+            you need more than one, you'll have to adjust {option}`handler` accordingly.
           '';
         };
 
@@ -98,8 +101,10 @@ in
             "E303" # too many blank lines
             "E305" # expected 2 blank lines after end of function or class
             "E306" # expected 1 blank line before a nested definition
+            "E402" # module level import not at top of file
             "E501" # max line length
             "E704" # multiple statements on one line (def)
+            "F811" # redefined while unused
           ];
           description = ''
             A list of flake8 rules to ignore while linting the python code.
@@ -122,6 +127,11 @@ in
                     self.send_header("Content-Length", str(len(data)))
                     self.end_headers()
                     self.wfile.write(data)
+
+            def on_reload():
+                # This function is called when the service receives SIGHUP (e.g. via `systemctl reload`).
+                # You can use it to clear caches or re-read state. Completely optional
+                pass
           '';
           description = ''
             Python code including the HTTP handler for the server.
@@ -145,6 +155,7 @@ in
           inherit (v) listenStreams;
           socketConfig = {
             Accept = false;
+            FileDescriptorName = v.name;
           } // v.socketConfig;
         };
       }))
@@ -157,30 +168,126 @@ in
         inherit (v) name;
         value = {
           serviceConfig = {
-            Type = "simple";
+            Type = "notify-reload";
+            NotifyAccess = "main";
             DynamicUser = true;
+            TimeoutStopSec = "35s";
 
             ExecStart = let
               package = pkgs.writers.writePython3Bin "${v.name}-bin" {
                 inherit (v) libraries flakeIgnore;
               } ''
+                import os
+                import signal
                 import socket
+                import socketserver
+                import threading
+                import time
                 from http.server import HTTPServer, BaseHTTPRequestHandler
 
-                ${v.handler}
+                SOCKET_NAME = "${v.name}"
+                SHUTDOWN_TIMEOUT = 30
+
+                def sd_notify(message: str):
+                    addr = os.environ.get("NOTIFY_SOCKET")
+                    if not addr:
+                        return
+                    if addr[0] == "@":
+                        addr = "\0" + addr[1:]
+                    with socket.socket(socket.AF_UNIX, socket.SOCK_DGRAM | socket.SOCK_CLOEXEC) as sock:
+                        sock.connect(addr)
+                        sock.sendall(message.encode())
+
+                def sd_listen_fd(name: str) -> int:
+                    if os.environ.get("LISTEN_PID") != str(os.getpid()):
+                        raise RuntimeError("No sockets were passed to this service by systemd")
+
+                    try:
+                        count = int(os.environ.get("LISTEN_FDS", "0"))
+                    except ValueError:
+                        count = 0
+
+                    raw_names = os.environ.get("LISTEN_FDNAMES")
+                    names = raw_names.split(":") if raw_names else []
+
+                    for i in range(count):
+                        if i < len(names) and names[i] == name:
+                            return 3 + i
+
+                    raise RuntimeError(
+                        "No systemd socket named %r was passed to this service; check the "
+                        "FileDescriptorName= of the corresponding .socket unit" % name
+                    )
+
+                class Server(socketserver.ThreadingMixIn, HTTPServer):
+                    daemon_threads = True
 
-                class NoBindHTTPServer(HTTPServer):
                     def server_bind(): pass
                     def server_activate(): pass
 
+                    def __init__(self, *args, **kwargs):
+                        super().__init__(*args, **kwargs)
+                        self._request_threads = []
+                        self._request_threads_lock = threading.Lock()
+
+                    def process_request(self, request, client_address):
+                        thread = threading.Thread(
+                            target=self.process_request_thread,
+                            args=(request, client_address),
+                        )
+                        thread.daemon = self.daemon_threads
+                        with self._request_threads_lock:
+                            self._request_threads.append(thread)
+                        thread.start()
+
+                    def join_request_threads(self, timeout):
+                        deadline = time.monotonic() + timeout
+                        with self._request_threads_lock:
+                            threads = list(self._request_threads)
+                        for thread in threads:
+                            thread.join(max(deadline - time.monotonic(), 0))
+
+                def handle_reload(signum, frame):
+                    monotonic_usec = time.clock_gettime_ns(time.CLOCK_MONOTONIC) // 1000
+                    sd_notify("RELOADING=1\nMONOTONIC_USEC=%d" % monotonic_usec)
+
+                    on_reload = globals().get("on_reload")
+                    if callable(on_reload):
+                        on_reload()
+
+                    sd_notify("READY=1")
+
+                shutdown_requested = threading.Event()
+
+                def handle_sigterm(signum, frame):
+                    sd_notify("STOPPING=1")
+                    shutdown_requested.set()
+
+                ${v.handler}
+
+                assert "Handler" in globals(), "You must define a class Handler(BaseHTTPRequestHandler) in the handler code"
+
                 def main():
-                    httpd = NoBindHTTPServer(
+                    signal.signal(signal.SIGHUP, handle_reload)
-                        ("", 0),
+                    signal.signal(signal.SIGTERM, handle_sigterm)
-                        Handler,
+
-                        bind_and_activate=False,
+                    fd = sd_listen_fd(SOCKET_NAME)
-                    )
+
-                    httpd.socket = socket.fromfd(3, socket.AF_INET, socket.SOCK_STREAM)
+                    httpd = Server(("", 0), Handler, bind_and_activate=False)
-                    httpd.serve_forever()
+                    httpd.socket = socket.socket(fileno=fd)
+
+                    server_thread = threading.Thread(target=httpd.serve_forever, name="http-server", daemon=True)
+                    server_thread.start()
+
+                    sd_notify("READY=1")
+                    shutdown_requested.wait()
+
+                    deadline = time.monotonic() + SHUTDOWN_TIMEOUT
+
+                    httpd.shutdown()
+                    server_thread.join(max(deadline - time.monotonic(), 0))
+                    httpd.join_request_threads(max(deadline - time.monotonic(), 0))
+                    httpd.server_close()
 
                 if __name__ == '__main__':
                     main()

secrets/ildkule/ildkule.yaml

@@ -1,9 +1,6 @@
 config:
     mysqld_exporter_password: ENC[AES256_GCM,data:I9K+QMqaN3FOOVKzeOR9Q6UERStXX0P8WEHyN1jzzbM=,iv:UxvIdlfAyJvNuxPkU4+guKPa0fiD0vVLzHOTYktcmso=,tag:ltnIqEwESYx9HBu8UN0ZLw==,type:str]
 keys:
-    gatus:
-        mariadb: ENC[AES256_GCM,data:U7CGtjHacJlKcReEw3JO86okCVgLAkP92iX1bHGNw8qeImpvk5mNw77Ox7tbsChvozrJWa/Npu2pTBsNnVYYYA==,iv:hWa+KPiBc3AnH1tHCGJa8LhPXZ1o8sxufpGe7nUjV0E=,tag:mso5HEfmWAOGw350408X7g==,type:str]
-        postgres: ENC[AES256_GCM,data:7fI0JeZDK1MX5poXWaMo67PXbjdjvM+xPKioHSHmGdS8iNDGe5Ym6LlS/hOnPvvDuNFyHjKKizqDst0227U2gg==,iv:D3bz/xyFrnQsUCiMEFq8JOjSudyiw+idEMWQ1e6Uk3k=,tag:8XqadKVHK9NMWehK7V5J0Q==,type:str]
     grafana:
         secret_key: ENC[AES256_GCM,data:+WoAJbDBEgKs0RoHT+7oEELAVQ+/2Xt+5RTMSXg23moCqVRx+Gzll9P5Drw=,iv:AkRn/Y20iEe5i1T+84wAgLCTFtAox2G3giyawAkltAw=,tag:BZbt5Wb5lYLIJBm/pfP4GQ==,type:str]
         admin_password: ENC[AES256_GCM,data:ttKwfC4WuXeL/6x4,iv:x1X+e3z08CR992GzC62YnFIN7SGrE81/nDNrgcgVzx0=,tag:YajUoy61kYbpeGeC7yNrXQ==,type:str]
@@ -12,7 +9,8 @@ keys:
         postgres_exporter_knakelibrak_env: ENC[AES256_GCM,data:xjC7DGXrW2GIJq8XioIZb+jSe/Hzcz0tv9cUHmX/n1nhI+D64lYt+EKnq1+RX/vJzU4sTaKjveKBh88Qqnv6RQm+MZC//dIxcvnnAdl50qnHZyBCaFFEzSNI8I8vGyArMk8Ja72clBq3kMpUz/pLBP0qDrjblKDoWkU=,iv:ZW98hJy8A5t4Oxtu17R3tM7gou183VLbgBsHA8LFuJY=,tag:VMOvQz3X/XDylV1YFg2Jsg==,type:str]
 sops:
     age:
-        - enc: |
+        - recipient: age102e6y8gah0ntr6fxqnkpepc8ar29p6ls7ks9ka7v8w87q8scm9yqmc2u8d
+          enc: |
             -----BEGIN AGE ENCRYPTED FILE-----
             YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSAyTWRSM3IwMmxtTmZVcCsw
             OUhlakxHZzgrSEhEdUZFTXE1anNjQ2wvdkZFCnB6S1l3TXQ3ZGFYWmtYM1cwMFZT
@@ -20,8 +18,8 @@ sops:
             YlltQ3FBU3RBYUx4TnNPRk1SUWNqZG8KAJjc09x553ncaWduGLsnIHdroaOmMasP
             /fq0GzW6UNfmE2rQ6qrQti21B37/sN0WMLCSPLUPG45kBgx20GG4hQ==
             -----END AGE ENCRYPTED FILE-----
-          recipient: age102e6y8gah0ntr6fxqnkpepc8ar29p6ls7ks9ka7v8w87q8scm9yqmc2u8d
+        - recipient: age1ug30gg4y7ftuya0wdv7q0vh4egn00wlv2th7mt7cgc2ze46wmvyq9lq6ge
-        - enc: |
+          enc: |
             -----BEGIN AGE ENCRYPTED FILE-----
             YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBTOWFvcFFzc1lmNVdmV3lX
             cVVLNHowcGdENzI5UUJLZTNSMHhjNlI2c0FnCmdwOS9oL1kwTnhwbXRodWxxWVE3
@@ -29,8 +27,8 @@ sops:
             eDVwd2dKMG9FRW1OY1pyUkhLeWw3b0EKtJpsQ/Ss39ZLiRNqUhn8sdB3hpQy7Syv
             ererqhMkqmDugGEHPk6KpZuj7DVSK1di7JgA2qZOUPzI7UpxjaC0Kg==
             -----END AGE ENCRYPTED FILE-----
-          recipient: age1ug30gg4y7ftuya0wdv7q0vh4egn00wlv2th7mt7cgc2ze46wmvyq9lq6ge
+        - recipient: age1mrnldl334l2nszuta6ywvewng0fswv2dz9l5g4qcwe3nj4yxf92qjskdx6
-        - enc: |
+          enc: |
             -----BEGIN AGE ENCRYPTED FILE-----
             YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBOQU1iVXRkQmo4b3F1Vngz
             S0pkNkVFR1FUb1djdmI0eHh3V3BBTDJTSTJZClA0S0Z0cTdFRmRaOEZQdHQzdGZ3
@@ -38,8 +36,8 @@ sops:
             REdjRFZyY2pNdEd6cmgvQisyVDhLUEkKRItJ0CGbzlEB5RNAyem4feMVhTfcLef3
             QIqltZ2l4LLexnkECi3FCJZHxrbUa+/RF6p1DsueUw7LLUnOcphB9A==
             -----END AGE ENCRYPTED FILE-----
-          recipient: age1mrnldl334l2nszuta6ywvewng0fswv2dz9l5g4qcwe3nj4yxf92qjskdx6
+        - recipient: age1hmpdk4h69wxpwqk9tkud39f66hprhehxtzhgw97r6dvr7v0mx5jscsuhkn
-        - enc: |
+          enc: |
             -----BEGIN AGE ENCRYPTED FILE-----
             YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSAvZTBpaWM0c1hmODBaK0Iw
             bmp1NzRacklXMHU0K2J2d0g5ajBiNlRNWGhRCm1DOWI0cm5BdTdlNmFzM2JVekNk
@@ -47,8 +45,8 @@ sops:
             U0ZCT2toZ1ZMZ3E0bXRhSTQvNGFWNVkKhxfQDIDe2LQW7OMBJv0J267AW1wI32df
             ZQxd657TEqzm7i19azrCS0jyRbfj2MYzEJAtTGiGZaNC9uKDFzBhKw==
             -----END AGE ENCRYPTED FILE-----
-          recipient: age1hmpdk4h69wxpwqk9tkud39f66hprhehxtzhgw97r6dvr7v0mx5jscsuhkn
+        - recipient: age1wrssr4z4g6vl3fd3qme5cewchmmhm0j2xe6wf2meu4r6ycn37anse98mfs
-        - enc: |
+          enc: |
             -----BEGIN AGE ENCRYPTED FILE-----
             YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBmRlJROHRKb21YUnlicmc1
             MlptQllEcXFhajNKS1krMDdUMWk5QWo4eVJBCndGSlhXS1Vaa2RSTllIcmF1ZVpl
@@ -56,8 +54,8 @@ sops:
             TE41aFdjU0h0ekQ2Zjg4Z3VQVjFWcnMK6zjSalqeYjyc4NH6nOeghlhYJydrz4pM
             N5ZcXjRbrIVFdhbYnvQGKvGKZm0kK6vjzBjdT7BM6ctr8cq/qrz1xQ==
             -----END AGE ENCRYPTED FILE-----
-          recipient: age1wrssr4z4g6vl3fd3qme5cewchmmhm0j2xe6wf2meu4r6ycn37anse98mfs
+        - recipient: age1zhxul786an743u0fascv4wtc5xduu7qfy803lfs539yzhgmlq5ds2lznt5
-        - enc: |
+          enc: |
             -----BEGIN AGE ENCRYPTED FILE-----
             YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBIRXpBa2tYc2xEeldub0VK
             bkFwOFdlUGRZM0FVT0tyUW1RWnl0TjRUTUh3CnZlMC92MU1hRW1yZU1NSUdoUEZh
@@ -65,8 +63,8 @@ sops:
             aXFQWlNVQ2laVm1ETStRemNZRXc3TUEKlPYSU3gp67dsPfbEJkru4ieMvspC7+pu
             rfp315HLyj1FGhrA8f2qOxE/PYI2rn0yKm80KffWBV7ylX/uonm4Fg==
             -----END AGE ENCRYPTED FILE-----
-          recipient: age1zhxul786an743u0fascv4wtc5xduu7qfy803lfs539yzhgmlq5ds2lznt5
+        - recipient: age1sqs7urnzsdy64efmd0zukzv3gs5pnjksuxd7nqmdwdy5l0nqnunq6hyune
-        - enc: |
+          enc: |
             -----BEGIN AGE ENCRYPTED FILE-----
             YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBsaTEzOFBEeG9LVThSVmQ5
             VXBoaFpueFRCbFJ1akE4RWc1aE1HUmVGcXdZCjFnbU0wd2drazNsTmNBMHNuOFhO
@@ -74,9 +72,8 @@ sops:
             dC9meDZlc3d3aUJEVjc4REF0Y1BLcGcK79LbJzc5KVgEgyJR11crGuX8YcVoJBbT
             Fin7Zoon06L7qx0Zw5u27wV7RKMnYT7hOMiWs6660ZTLcYJ5M1aEZQ==
             -----END AGE ENCRYPTED FILE-----
-          recipient: age1sqs7urnzsdy64efmd0zukzv3gs5pnjksuxd7nqmdwdy5l0nqnunq6hyune
+    lastmodified: "2025-03-16T20:08:18Z"
-    lastmodified: "2026-07-21T09:40:54Z"
+    mac: ENC[AES256_GCM,data:C2tpWppc13jKJq5d4nmAKQOaNWHm27TKwxAxm1fi2lejN1lqUaoz5bHfTBA7MfaWvuP5uZnfbtG32eeu48mnlWpo58XRUFFecAhb9JUpW9s5IR3/nbzLNkGU7H5C0oWPrxI4thd+bAVduIgBjjFyGj1pe6J9db3c0yUWRwNlwGU=,iv:YpoQ4psiFYOWLGipxv1QvRvr034XFsyn2Bhyy39HmOo=,tag:ByiCWygFC/VokVTbdLoLgg==,type:str]
-    mac: ENC[AES256_GCM,data:FLpmsW9I21qNMFs6Tn/1AzFo+RcmCwtVdAaaGIrGlUi2l3gzb//ZK6ruI/VYx1R5DoX1OhD3EIg6aLMSOT6jiHf+8GwfhdijpayG1nrgeZW3SuFbjVlCYNMPH3DOJJB82U3T+ozSbpnZEtdUwmNWcZJt5F2hOEnoaNa+jLEKDao=,iv:qyQuyWuu+w2ynRvhIx0QVtw+tieO3dbQsUxEB5Brzp0=,tag:qYAj4h6foWm16YqIdZ6xbQ==,type:str]
     pgp:
         - created_at: "2026-05-20T17:35:58Z"
           enc: |-
@@ -99,4 +96,4 @@ sops:
             -----END PGP MESSAGE-----
           fp: F7D37890228A907440E1FD4846B9228E814A2AAC
     unencrypted_suffix: _unencrypted
-    version: 3.13.2
+    version: 3.9.4