bekkalokk/gitea: fix ci runner network issues

This PR is made while moving Ildkule from PVE on joshua, to Openstack on stack.it.ntnu.no.

- The main monitoring dashboard is moved from https://ildkule.pvv.ntnu.no to https://grafana.pvv.ntnu.no.
- A new service is added: uptime-kuma on https://uptime.pvv.ntnu.no.
- The (hardware) configuration for ildkule is updated to fit the new virtualization environment, boot loader, network interfaces, etc.
- Metrics exporters on other hosts should be updated to allow connections from the new host

As this is the first proper server running on openstack, and therefore outside our main IP range, we might discover challenges in our network structure. For example, the database servers usually only allow connections from this range, so Ildkule can no longer access it. This should be explored, documented and/or fixed as we move more services.

Reviewed-on: #36
Co-authored-by: Felix Albrigtsen <felix@albrigtsen.it>
Co-committed-by: Felix Albrigtsen <felix@albrigtsen.it>

.sops.yaml

@@ -7,7 +7,7 @@ keys:
 
   # Hosts
   - &host_jokum age1gp8ye4g2mmw3may5xg0zsy7mm04glfz3788mmdx9cvcsdxs9hg0s0cc9kt
-  - &host_ildkule age1hn45n46ypyrvypv0mwfnpt9ddrlmw34dwlpf33n8v67jexr3lucq6ahc9x
+  - &host_ildkule age1x28hmzvuv6f2n66c0jtqcca3h9rput8d7j5uek6jcpx8n9egd52sqpejq0
   - &host_bekkalokk age12nj59tguy9wg882updc2vjdusx5srnxmjyfaqve4zx6jnnsaw3qsyjq6zd
   - &host_bicep age1sl43gc9cw939z5tgha2lpwf0xxxgcnlw7w4xem4sqgmt2pt264vq0dmwx2
 

base.nix

@@ -101,19 +101,19 @@
     recommendedOptimisation = true;
     recommendedGzipSettings = true;
 
-    appendConfig = lib.mkIf (!config.services.matrix-synapse-next.enable or false) ''
+    appendConfig = ''
       pcre_jit on;
       worker_processes auto;
       worker_rlimit_nofile 100000;
     '';
-    eventsConfig = lib.mkIf (!config.services.matrix-synapse-next.enable or false) ''
+    eventsConfig = ''
       worker_connections 2048;
       use epoll;
       multi_accept on;
     '';
   };
 
-  systemd.services.nginx.serviceConfig = lib.mkIf (!config.services.matrix-synapse-next.enable or false) {
+  systemd.services.nginx.serviceConfig = lib.mkIf config.services.nginx.enable {
     LimitNOFILE = 65536;
   };
 

flake.lock

@@ -7,11 +7,11 @@
         ]
       },
       "locked": {
-        "lastModified": 1712356478,
-        "narHash": "sha256-kTcEtrQIRnexu5lAbLsmUcfR2CrmsACF1s3ZFw1NEVA=",
+        "lastModified": 1712798444,
+        "narHash": "sha256-aAksVB7zMfBQTz0q2Lw3o78HM3Bg2FRziX2D6qnh+sk=",
         "owner": "nix-community",
         "repo": "disko",
-        "rev": "0a17298c0d96190ef3be729d594ba202b9c53beb",
+        "rev": "a297cb1cb0337ee10a7a0f9517954501d8f6f74d",
         "type": "github"
       },
       "original": {
@@ -27,11 +27,11 @@
         ]
       },
       "locked": {
-        "lastModified": 1696346665,
-        "narHash": "sha256-J6Tf6a/zhFZ8SereluHLrvgPsIVm2CGHHA8wrbhZB3Y=",
+        "lastModified": 1712875951,
+        "narHash": "sha256-4kcRd2Q2XM4r+U2zp+LADjrzazKpWvs0WrMKPktEEkc=",
         "owner": "Programvareverkstedet",
         "repo": "grzegorz",
-        "rev": "9b9c3ac7d408ac7c6d67544b201e6b169afacb03",
+        "rev": "9eaba26b1671e8810cb135997c867ac3550e685a",
         "type": "github"
       },
       "original": {
@@ -102,11 +102,11 @@
     },
     "nixpkgs": {
       "locked": {
-        "lastModified": 1712386448,
-        "narHash": "sha256-kacQwZ5WnJv4HH5s8tlZTClyiwVP8XAaywI5I7QqLIY=",
+        "lastModified": 1712848736,
+        "narHash": "sha256-CzZwhqyLlebljv1zFS2KWVH/3byHND0LfaO1jKsGuVo=",
         "owner": "NixOS",
         "repo": "nixpkgs",
-        "rev": "7644b4bc09c6329bcd82561a076fd7add697d092",
+        "rev": "1d6a23f11e44d0fb64b3237569b87658a9eb5643",
         "type": "github"
       },
       "original": {
@@ -117,11 +117,11 @@
     },
     "nixpkgs-stable": {
       "locked": {
-        "lastModified": 1711819797,
-        "narHash": "sha256-tNeB6emxj74Y6ctwmsjtMlzUMn458sBmwnD35U5KIM4=",
+        "lastModified": 1712437997,
+        "narHash": "sha256-g0whLLwRvgO2FsyhY8fNk+TWenS3jg5UdlWL4uqgFeo=",
         "owner": "NixOS",
         "repo": "nixpkgs",
-        "rev": "2b4e3ca0091049c6fbb4908c66b05b77eaef9f0c",
+        "rev": "e38d7cb66ea4f7a0eb6681920615dfcc30fc2920",
         "type": "github"
       },
       "original": {
@@ -133,11 +133,11 @@
     },
     "nixpkgs-unstable": {
       "locked": {
-        "lastModified": 1712381113,
-        "narHash": "sha256-YL8miM11o/jMqOwt5DsdyhPgh/JgCl1kOIzvX7ukniY=",
+        "lastModified": 1712837137,
+        "narHash": "sha256-9joaU/GD35J9Utb0ipelQbOcvsw5eoYTmSarLV3MbNk=",
         "owner": "NixOS",
         "repo": "nixpkgs",
-        "rev": "6cc8dbb00974248cdd1b7ebd05cbc7c0799ce974",
+        "rev": "681d4a87b26b1dcaae7ffe6cf88c9912c575415f",
         "type": "github"
       },
       "original": {
@@ -208,11 +208,11 @@
         "nixpkgs-stable": "nixpkgs-stable"
       },
       "locked": {
-        "lastModified": 1711855048,
-        "narHash": "sha256-HxegAPnQJSC4cbEbF4Iq3YTlFHZKLiNTk8147EbLdGg=",
+        "lastModified": 1712617241,
+        "narHash": "sha256-a4hbls4vlLRMciv62YrYT/Xs/3Cubce8WFHPUDWwzf8=",
         "owner": "Mic92",
         "repo": "sops-nix",
-        "rev": "99b1e37f9fc0960d064a7862eb7adfb92e64fa10",
+        "rev": "538c114cfdf1f0458f507087b1dcf018ce1c0c4c",
         "type": "github"
       },
       "original": {

flake.nix

@@ -64,7 +64,11 @@
 
           pkgs = import nixpkgs {
             inherit system;
-            overlays = [ ] ++ config.overlays or [ ];
+            overlays = [
+              (import ./overlays/nginx-test.nix
+                (builtins.attrNames self.nixosConfigurations.${name}.config.security.acme.certs)
+              )
+            ] ++ config.overlays or [ ];
           };
         }
         (removeAttrs config [ "modules" "overlays" ])
@@ -138,7 +142,7 @@
 
         simplesamlphp = pkgs.callPackage ./packages/simplesamlphp { };
 
-        mediawiki-extensions = pkgs.callPackage ./packages/mediawiki-extensions { };
+        # mediawiki-extensions = pkgs.callPackage ./packages/mediawiki-extensions { };
       } // nixlib.genAttrs allMachines
         (machine: self.nixosConfigurations.${machine}.config.system.build.toplevel);
     };

hosts/bekkalokk/configuration.nix

@@ -6,8 +6,6 @@
     ../../base.nix
     ../../misc/metrics-exporters.nix
 
-    #./services/keycloak.nix
-
     ./services/website
     ./services/nginx.nix
     ./services/gitea/default.nix

hosts/bekkalokk/services/gitea/ci.nix

@@ -27,5 +27,8 @@ lib.mkMerge [
   (mkRunner "alpha")
   (mkRunner "beta")
   (mkRunner "epsilon")
-  { virtualisation.podman.enable = true; }
+  {
+    virtualisation.podman.enable = true;
+    networking.firewall.interfaces."podman+".allowedUDPPorts = [ 53 5353 ];
+  }
 ]

hosts/bekkalokk/services/keycloak.nix

@@ -1,24 +0,0 @@
-{ pkgs, config, values, ... }:
-{
-  sops.secrets."keys/postgres/keycloak" = {
-    owner = "keycloak";
-    group = "keycloak";
-    restartUnits = [ "keycloak.service" ];
-  };
-
-  services.keycloak = {
-    enable = true;
-
-    settings = {
-      hostname = "auth.pvv.ntnu.no";
-      # hostname-strict-backchannel = true;
-    };
-
-    database = {
-      host = values.hosts.bicep.ipv4;
-      createLocally = false;
-      passwordFile = config.sops.secrets."keys/postgres/keycloak".path;
-      caCert = "${pkgs.cacert}/etc/ssl/certs/ca-bundle.crt";
-    };
-  };
-}

hosts/bekkalokk/services/website/fetch-gallery.nix

@@ -34,8 +34,8 @@ in {
       # Delete files and directories that exists in the gallery that don't exist in the tarball
       filesToRemove=$(uniq -u <(sort <(find . -not -path "./.thumbnails*") <(tar -tf ${transferDir}/gallery.tar.gz | sed 's|/$||')))
       while IFS= read fname; do
-        rm -f $fname ||:
-        rm -f .thumbnails/$fname.png ||:
+        rm -f "$fname" ||:
+        rm -f ".thumbnails/$fname.png" ||:
       done <<< "$filesToRemove"
 
       find . -type d -empty -delete
@@ -44,11 +44,17 @@ in {
       images=$(find . -type f -not -path "./.thumbnails*")
 
       while IFS= read fname; do
-        [ -f ".thumbnails/$fname.png" ] && continue ||:
+        # Skip this file if an up-to-date thumbnail already exists
+        if [ -f ".thumbnails/$fname.png" ] && \
+           [ "$(date -R -r "$fname")" == "$(date -R -r ".thumbnails/$fname.png")" ]
+        then
+          continue
+        fi
 
         echo "Creating thumbnail for $fname"
         mkdir -p $(dirname ".thumbnails/$fname")
-        convert -define jpeg:size=200x200 "$fname" -thumbnail 500 -auto-orient ".thumbnails/$fname.png" ||:
+        convert -define jpeg:size=200x200 "$fname" -thumbnail 300 -auto-orient ".thumbnails/$fname.png" ||:
+	touch -m -d "$(date -R -r "$fname")" ".thumbnails/$fname.png"
       done <<< "$images"
     '';
 

hosts/bicep/services/matrix/smtp-authenticator/smtp_auth_provider.py

@@ -7,6 +7,9 @@ from synapse import module_api
 
 import re
 
+import logging
+logger = logging.getLogger(__name__)
+
 class SMTPAuthProvider:
     def __init__(self, config: dict, api: module_api):
         self.api = api
@@ -43,8 +46,13 @@ class SMTPAuthProvider:
 
         if result == True:
             userid = self.api.get_qualified_user_id(username)
-            if not self.api.check_user_exists(userid):
-                self.api.register_user(username)
+
+            userid = await self.api.check_user_exists(userid)
+            if not userid:
+                logger.info(f"user did not exist, registering {username}")
+                userid = await self.api.register_user(username)
+                logger.info(f"registered userid: {userid}")
             return (userid, None)
         else:
+            logger.info("returning None")
             return None

hosts/bicep/services/matrix/synapse.nix

@@ -134,80 +134,6 @@ in {
         "129.241.0.0/16"
         "2001:700:300::/44"
       ];
-
-      saml2_config = {
-        sp_config.metadata.remote = [
-          { url = "https://idp.pvv.ntnu.no/simplesaml/saml2/idp/metadata.php"; }
-        ];
-
-        description = [ "Matrix Synapse SP" "en" ];
-        name = [ "Matrix Synapse SP" "en" ];
-
-        ui_info = {
-          display_name = [
-            {
-              lang = "en";
-              text = "PVV Matrix login";
-            }
-          ];
-          description = [
-            {
-              lang = "en";
-              text = "Matrix is a modern free and open federated chat protocol";
-            }
-          ];
-          #information_url = [
-          #  {
-          #    lang = "en";
-          #    text = "";
-          #  };
-          #];
-          #privacy_statement_url = [
-          #  {
-          #    lang = "en";
-          #    text = "";
-          #  };
-          #];
-          keywords = [
-            {
-              lang = "en";
-              text = [ "Matrix" "Element" ];
-            }
-          ];
-          #logo = [
-          #  {
-          #    lang = "en";
-          #    text = "";
-          #    width = "";
-          #    height = "";
-          #  }
-          #];
-        };
-
-        organization = {
-          name = "Programvareverkstedet";
-          display_name = [ "Programvareverkstedet" "en" ];
-          url = "https://www.pvv.ntnu.no";
-        };
-        contact_person = [
-          { given_name = "Drift";
-            sur_name = "King";
-            email_adress = [ "drift@pvv.ntnu.no" ];
-            contact_type = "technical";
-          }
-        ];
-
-        user_mapping_provider = {
-          config = {
-            mxid_source_attribute =  "uid"; # What is this supposed to be?
-            mxid_mapping = "hexencode";
-          };
-        };
-
-        #attribute_requirements = [
-        #  {attribute = "userGroup"; value = "medlem";} # Do we have this?
-        #];
-      };
     };
   };
 
@@ -244,6 +170,8 @@ in {
         extraConfig = ''
           allow ${values.hosts.ildkule.ipv4};
           allow ${values.hosts.ildkule.ipv6};
+          allow ${values.hosts.ildkule.ipv4_global};
+          allow ${values.hosts.ildkule.ipv6_global};
           deny all;
         '';
       }))
@@ -255,6 +183,8 @@ in {
       extraConfig = ''
         allow ${values.hosts.ildkule.ipv4};
         allow ${values.hosts.ildkule.ipv6};
+        allow ${values.hosts.ildkule.ipv4_global};
+        allow ${values.hosts.ildkule.ipv6_global};
         deny all;
       '';
     };

hosts/bicep/services/nginx/default.nix

@@ -11,15 +11,5 @@
       "127.0.0.2"
       "[::1]"
     ];
-
-    appendConfig = ''
-      worker_processes 8;
-      worker_rlimit_nofile 8192;
-    '';
-
-    eventsConfig = ''
-      multi_accept on;
-      worker_connections 4096;
-    '';
   };
 }

hosts/ildkule/configuration.nix

@@ -6,8 +6,8 @@
       ../../base.nix
       ../../misc/metrics-exporters.nix
 
+      ./services/monitoring
       ./services/nginx
-      ./services/metrics
     ];
 
   sops.defaultSopsFile = ../../secrets/ildkule/ildkule.yaml;
@@ -15,28 +15,21 @@
   sops.age.keyFile = "/var/lib/sops-nix/key.txt";
   sops.age.generateKey = true;
 
-  boot.loader.systemd-boot.enable = true;
-  boot.loader.efi.canTouchEfiVariables = true;
+  boot.loader.grub.device = "/dev/vda";
+  boot.tmp.cleanOnBoot = true;
+  zramSwap.enable = true;
 
   networking.hostName = "ildkule"; # Define your hostname.
-
-  systemd.network.networks."30-ens18" = values.defaultNetworkConfig // {
-    matchConfig.Name = "ens18";
-    address = with values.hosts.ildkule; [ (ipv4 + "/25") (ipv6 + "/64") ];
+  systemd.network.networks."30-all" = values.defaultNetworkConfig // {
+    matchConfig.Name = "en*";
+    DHCP = "yes";
+    gateway = [ ];
   };
 
   # List packages installed in system profile
   environment.systemPackages = with pkgs; [
   ];
 
-  # List services that you want to enable:
-
-  # This value determines the NixOS release from which the default
-  # settings for stateful data, like file locations and database versions
-  # on your system were taken. It‘s perfectly fine and recommended to leave
-  # this value at the release version of the first install of this system.
-  # Before changing this value read the documentation for this option
-  # (e.g. man configuration.nix or on https://nixos.org/nixos/options.html).
-  system.stateVersion = "21.11"; # Did you read the comment?
+  system.stateVersion = "23.11"; # Did you read the comment?
 
 }

hosts/ildkule/hardware-configuration.nix

@@ -1,37 +1,9 @@
-# Do not modify this file!  It was generated by ‘nixos-generate-config’
-# and may be overwritten by future invocations.  Please make changes
-# to /etc/nixos/configuration.nix instead.
-{ config, lib, pkgs, modulesPath, ... }:
-
+{ modulesPath, lib, ... }:
 {
-  imports =
-    [ (modulesPath + "/profiles/qemu-guest.nix")
-    ];
+  imports = [ (modulesPath + "/profiles/qemu-guest.nix") ];
+  boot.initrd.availableKernelModules = [ "ata_piix" "uhci_hcd" "xen_blkfront" "vmw_pvscsi" ];
+  boot.initrd.kernelModules = [ "nvme" ];
+  fileSystems."/" = { device = "/dev/vda1"; fsType = "ext4"; };
 
-  boot.initrd.availableKernelModules = [ "ata_piix" "uhci_hcd" "virtio_pci" "sr_mod" "virtio_blk" ];
-  boot.initrd.kernelModules = [ ];
-  boot.kernelModules = [ ];
-  boot.extraModulePackages = [ ];
-
-  fileSystems."/" =
-    { device = "/dev/disk/by-uuid/afe70fe4-681a-4675-8cbd-e5d08cdcf5b5";
-      fsType = "ext4";
-    };
-
-  fileSystems."/boot" =
-    { device = "/dev/disk/by-uuid/B71A-E5CD";
-      fsType = "vfat";
-    };
-
-  swapDevices = [ ];
-
-  # Enables DHCP on each ethernet and wireless interface. In case of scripted networking
-  # (the default) this is the recommended approach. When using systemd-networkd it's
-  # still possible to use this option, but it's recommended to use it in conjunction
-  # with explicit per-interface declarations with `networking.interfaces.<interface>.useDHCP`.
   networking.useDHCP = lib.mkDefault true;
-  # networking.interfaces.ens18.useDHCP = lib.mkDefault true;
-
-  nixpkgs.hostPlatform = lib.mkDefault "x86_64-linux";
-  hardware.cpu.intel.updateMicrocode = lib.mkDefault config.hardware.enableRedistributableFirmware;
 }

hosts/ildkule/services/monitoring/default.nix

@@ -2,8 +2,9 @@
 
 {
   imports = [
-    ./prometheus
     ./grafana.nix
     ./loki.nix
+    ./prometheus
+    ./uptime-kuma.nix
   ];
 }

hosts/ildkule/services/monitoring/grafana.nix

@@ -7,7 +7,6 @@ in {
   in {
     "keys/grafana/secret_key" = { inherit owner group; };
     "keys/grafana/admin_password" = { inherit owner group; };
-    "keys/postgres/grafana" = { inherit owner group; };
   };
 
   services.grafana = {
@@ -18,7 +17,7 @@ in {
       secretFile = path: "$__file{${path}}";
     in {
       server = {
-        domain = "ildkule.pvv.ntnu.no";
+        domain = "grafana.pvv.ntnu.no";
         http_port = 2342;
         http_addr = "127.0.0.1";
       };
@@ -27,13 +26,6 @@ in {
         secret_key = secretFile config.sops.secrets."keys/grafana/secret_key".path;
         admin_password = secretFile config.sops.secrets."keys/grafana/admin_password".path;
       };
-
-      database = {
-        type = "postgres";
-        user = "grafana";
-        host = "${values.hosts.bicep.ipv4}:5432";
-        password = secretFile config.sops.secrets."keys/postgres/grafana".path;
-      };
     };
 
     provision = {

hosts/ildkule/services/monitoring/uptime-kuma.nix

@@ -0,0 +1,20 @@
+{ config, pkgs, lib, ... }:
+let
+  cfg = config.services.uptime-kuma;
+  domain = "status.pvv.ntnu.no";
+in {
+  services.uptime-kuma = {
+    enable = true;
+    settings = {
+      PORT = "5059";
+      HOST = "127.0.1.2";
+    };
+  };
+
+  services.nginx.virtualHosts.${domain} = {
+    enableACME = true;
+    forceSSL = true;
+    kTLS = true;
+    locations."/".proxyPass = "http://${cfg.settings.HOST}:${cfg.settings.PORT}";
+  };
+}

hosts/ildkule/services/nginx/default.nix

@@ -3,13 +3,5 @@
   services.nginx = {
     enable = true;
     enableReload = true;
-    defaultListenAddresses = [
-      values.hosts.ildkule.ipv4
-      "[${values.hosts.ildkule.ipv6}]"
-
-      "127.0.0.1"
-      "127.0.0.2"
-      "[::1]"
-    ];
   };
 }

misc/metrics-exporters.nix

@@ -14,6 +14,8 @@
       "::1"
       values.hosts.ildkule.ipv4
       values.hosts.ildkule.ipv6
+      values.hosts.ildkule.ipv4_global
+      values.hosts.ildkule.ipv6_global
     ];
   };
 

overlays/nginx-test.nix

@@ -0,0 +1,28 @@
+acme-certs: final: prev:
+  let
+    lib = final.lib;
+    crt = "${final.path}/nixos/tests/common/acme/server/acme.test.cert.pem";
+    key = "${final.path}/nixos/tests/common/acme/server/acme.test.key.pem";
+  in {
+  writers = prev.writers // {
+    writeNginxConfig = name: text: final.runCommandLocal name {
+      nginxConfig = prev.writers.writeNginxConfig name text;
+      nativeBuildInputs = [ final.bubblewrap ];
+    } ''
+      ln -s "$nginxConfig" "$out"
+      set +o pipefail
+      bwrap \
+        --ro-bind "${crt}" "/etc/certs/nginx.crt" \
+        --ro-bind "${key}" "/etc/certs/nginx.key" \
+        --ro-bind "/nix" "/nix" \
+        --ro-bind "/etc/hosts" "/etc/hosts" \
+        --dir "/run/nginx" \
+        --dir "/tmp" \
+        --dir "/var/log/nginx" \
+        ${lib.concatMapStrings (name: "--ro-bind \"${crt}\" \"/var/lib/acme/${name}/fullchain.pem\" \\") acme-certs}
+        ${lib.concatMapStrings (name: "--ro-bind \"${key}\" \"/var/lib/acme/${name}/key.pem\" \\") acme-certs}
+        ${lib.concatMapStrings (name: "--ro-bind \"${crt}\" \"/var/lib/acme/${name}/chain.pem\" \\") acme-certs}
+        ${lib.getExe' final.nginx "nginx"} -t -c "$out" |& grep "syntax is ok"
+    '';
+  };
+}

secrets/bekkalokk/bekkalokk.yaml

@@ -15,8 +15,6 @@ mediawiki:
         postgres_password: ENC[AES256_GCM,data:FzykBVtJbA+Bey1GE5VqnSuv2GeobH1j,iv:wayQH3+y0FYFkr3JjmulI53SADk0Ikur/2mUS5kFrTk=,tag:d+nQ/se2bDA5aaQfBicnPQ==,type:str]
         cookie_salt: ENC[AES256_GCM,data:BioRPAvL4F9ORBJDFdqHot81RhVpAOf32v1ah3pvOLq8E88bxGyKFQZxAwpIL3UkWQIsWMnEerm5MEMYL1C2OQ==,iv:yMVqiPTQ8hO1IVAax6PIkD0V9YTOEunwDTtnGcmy6Kc=,tag:Z4+bZF4olLlkx7YpXeQiUw==,type:str]
         admin_password: ENC[AES256_GCM,data:4eUXvcO7NLOWke9XShfKzj+x3FvqPONa,iv:3iZ+BTBTZ7yMJ0HT14cEMebKZattWUcYEevRsl/6WOk=,tag:CU0iDhPP2ndztdX5U5A4cw==,type:str]
-keycloak:
-    database: ENC[AES256_GCM,data:76+AZnNR5EiturTP7BdOCKE90bFFkfGlRtviSP5NHxPbb3RfFPJEMlwtzA==,iv:nS7VTossHdlrHjPeethhX+Ysp9ukrb5JD7kjG28OFpY=,tag:OMpiEv9nQA7v6lWJfNxEEw==,type:str]
 idp:
     cookie_salt: ENC[AES256_GCM,data:cyV6HDCPHKQIa8T1+rFBFh6EuHtG5B508lg6uFYENK7qVpYuiTUIokdVQhY8SRLs2mECx/ampgnUHxCRB/Cc/A==,iv:QRrRUhzRQrLkmg38rrYtCEfF8U4/7ZHZUDSEq++BlbI=,tag:fLqFSLd+CKqJvmCh1fx8vg==,type:str]
     admin_password: ENC[AES256_GCM,data:Vf33Oenk6x6BIij1uW8RQDjTPcKhUVYA,iv:RNeyCNpTAYdBPrZwE3Y6CCjoAML/3XUvjfJCrr06IEU=,tag:zVOrx1oXnEyr/VwFCFaCDQ==,type:str]
@@ -62,8 +60,8 @@ sops:
             akVjeTNTeGorZjJQOVlMeCtPRUVYL3MK+VMvGxrbzGz4Q3sdaDDWjal+OiK+JYKX
             GHiMXVHQJZu/RrlxMjHKN6V3iaqxZpuvLAEJ2Lzy5EOHPtuiiRyeHQ==
             -----END AGE ENCRYPTED FILE-----
-    lastmodified: "2024-04-08T20:36:13Z"
-    mac: ENC[AES256_GCM,data:IObBR2H3cPIvBNWSo7A5xwyKeg2HbFkfxcU1U5BpRx2gvNb9/h7lextQ6IWPHOS/LZRXY+lZdhX6zLf6aLQjxTATZcrcF2BIu6YM5wppXOjVxhy3dkItU5TGPuxBdZEZ1bEpFu3B1Ooc1UdWvEWqZz2Mcn2akoX53Mj2vU8WGO8=,iv:fHvPHO33y6y3OSbVkojw2+XnpCNHO1AnCm2RnuwxPVA=,tag:m44YZd4Q4DEHCoDCoayqsg==,type:str]
+    lastmodified: "2024-04-14T21:58:31Z"
+    mac: ENC[AES256_GCM,data:+o7YvaaKTjN/uZT5mv3z9FgIbXwG4NPJePWwRmtkBINn9X+vrCmYOXqWhKw7qfInn4Ftcg0FA7cYFZe5Pv8MNp+f8v1yoiLrVX12cxmEYtqTXJz7pNeD2st1YjGJKihNi2/fyCCf4YBCGN+8Ze//HeVf7/tfWNB+ysyC9g9Tze4=,iv:C6XBCVXn8GuNeaWGdJRnUIh1us0i8fSoxu9Sx7Feb58=,tag:W0RLPPv7eP5kCNrhMG3z7A==,type:str]
     pgp:
         - created_at: "2023-05-21T00:28:40Z"
           enc: |

secrets/ildkule/ildkule.yaml

@@ -15,7 +15,6 @@ keys:
         secret_key: ENC[AES256_GCM,data:+WoAJbDBEgKs0RoHT+7oEELAVQ+/2Xt+5RTMSXg23moCqVRx+Gzll9P5Drw=,iv:AkRn/Y20iEe5i1T+84wAgLCTFtAox2G3giyawAkltAw=,tag:BZbt5Wb5lYLIJBm/pfP4GQ==,type:str]
         admin_password: ENC[AES256_GCM,data:ttKwfC4WuXeL/6x4,iv:x1X+e3z08CR992GzC62YnFIN7SGrE81/nDNrgcgVzx0=,tag:YajUoy61kYbpeGeC7yNrXQ==,type:str]
     postgres:
-        grafana: ENC[AES256_GCM,data:D6qkg98WZYzKYegSNBb31v8o+KHisGmJ+ab5Ut7EMtsJz36kUup5RS4EbtM=,iv:rfE1uH1QycKMTpSq2p1ntQ2BIvptAh2J3l/QcQhiuLo=,tag:QxmGFcekjFRPf6orN86IxQ==,type:str]
         postgres_exporter_env: ENC[AES256_GCM,data:8MEoikoA6tFNm9qZbk0DFWANd7nRs5QSqrsGLoLKPIc1xykJaXTlyP5v8ywVGR8j7bfPs4p6QfpUIWK8CCnfQ1QhsFPXUMksl8p+K+xuMakYZr9OoWigGqvOHpFb9blfBN1FBdRrk38REXWAMUn74KSRI9v+0i5lpC4=,iv:anpjWVUadKfSAm9XbkeAKu+jAk+LxcpVYQ+gUe5szYw=,tag:4tzb/8B/e1uVoqTsQGlcKA==,type:str]
         postgres_exporter_knakelibrak_env: ENC[AES256_GCM,data:xjC7DGXrW2GIJq8XioIZb+jSe/Hzcz0tv9cUHmX/n1nhI+D64lYt+EKnq1+RX/vJzU4sTaKjveKBh88Qqnv6RQm+MZC//dIxcvnnAdl50qnHZyBCaFFEzSNI8I8vGyArMk8Ja72clBq3kMpUz/pLBP0qDrjblKDoWkU=,iv:ZW98hJy8A5t4Oxtu17R3tM7gou183VLbgBsHA8LFuJY=,tag:VMOvQz3X/XDylV1YFg2Jsg==,type:str]
 sops:
@@ -24,55 +23,55 @@ sops:
     azure_kv: []
     hc_vault: []
     age:
-        - recipient: age1mrnldl334l2nszuta6ywvewng0fswv2dz9l5g4qcwe3nj4yxf92qjskdx6
+        - recipient: age1x28hmzvuv6f2n66c0jtqcca3h9rput8d7j5uek6jcpx8n9egd52sqpejq0
           enc: |
             -----BEGIN AGE ENCRYPTED FILE-----
-            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBrN3lJM2xWTUZ3UkRBaENI
-            VmJiWDlQbHd0VUNYdllPdURyQmUvL3lKMzJzCkZlRFVxbmNLOVNqUFg1akJQQlBP
-            VmdOMUdjZ1M4U2lLVEpGaGI5NjNTR2MKLS0tIDRlQUtucEZhZmRYbmpadVdKK01v
-            cWxCQlBRR1VaZTBDQnkzNGE0WGttWm8KK5s/coWNsdCP5lKQ8LMK7/3ku179+Lg1
-            4ujTVn4LhvXy6JvgGTWS/UbMmJjJebVxkulzf5St3YMMs2mcIYjOtA==
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBURkY4WTZhQzJoREpxV1Vr
+            aUExZ1dxNkIyMkJtUXpqOWtTT1J0MGpmMkY4ClR4Wm1FTmhKN2pIMENRdERrWVY2
+            SUlHblpEc3VackMrbFpHUUJwM2ltZHcKLS0tIEovMEtiOWc1L2tzZDh3ekZKbStr
+            NEFkcW03ZTRJODNxTlVuUnFlcFFUUncKEZzOeUtRsZiuugTLzG2xU4eJ3XtVuop7
+            hhlDBL/YoFn/CO3HjqFdCVv33QoPu7KKMeV52pbVEnv93mvdEeFxVA==
             -----END AGE ENCRYPTED FILE-----
         - recipient: age17tagmpwqjk3mdy45rfesrfey6h863x8wfq38wh33tkrlrywxducs0k6tpq
           enc: |
             -----BEGIN AGE ENCRYPTED FILE-----
-            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBSa25taGsxdlhrUS96cXBi
-            cUo3WDVmdEhKN256THJhS2tHSitDRkVraDJNCmhGZzlFUDFkN0JKNkFWUlVLVzcz
-            MjFhcDdmcmpxdTA3V3JRREFNVmNUbEEKLS0tIFNSU2xNZzN2Y1ZzR2hFM0dOK0Zy
-            Tmk4bXd0ZHhPemxDSDREb3IvSjFza1EKsjtC6J3kYGRe8oLAoUZmg1BUmpkMyC98
-            uYq+IQmfJt48R/MKDei00j1w3zIK5+E5GU4o8+jILzwfpzYUUZWwiA==
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBSY3cxSGFvdDdWcFVLRTRy
+            Zng2VnhjZlFkc1RQN0NqUjJGeW02WlFaMlFZCjVZc2x2UXNXS1I2WDBxeHdjNUdr
+            WnZGc0l5NlArekUwUGU3Qkdub25EVm8KLS0tIDB2bGo3ZURtZ0pSZjFzcGpOdW5D
+            aTI3aTBUS0d1MzFmMTVMbUlFYTR4VlUKzOvNCAzan1GTXjoRxeySkUYIYtI4Mpvu
+            MC0Q8e350SyoOsrF7fUvw+Ru68fDMLW27H6Ly36xP7D3eo/h4eZVXw==
             -----END AGE ENCRYPTED FILE-----
-        - recipient: age1hn45n46ypyrvypv0mwfnpt9ddrlmw34dwlpf33n8v67jexr3lucq6ahc9x
+        - recipient: age1mrnldl334l2nszuta6ywvewng0fswv2dz9l5g4qcwe3nj4yxf92qjskdx6
           enc: |
             -----BEGIN AGE ENCRYPTED FILE-----
-            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBPb09qTTc4cjRMcjIzRmxu
-            RzZWTDBNTGdvaEc2VFJPakYvakRMK1RnS1FnCktHRVkwZGlUUXl4UTBRcGxMQzdn
-            QVBCYVdlWEw5NW9tNytJTGIzRlpwa0UKLS0tIGdDdUtFMUgyT0phMXBxZE41Y1h4
-            a2hQVVprakt5NURpNXdQUjREczJKWTgKn60yrLqco9brlqigAolO8rEkww9z3y3u
-            KmefLVZCGfoko+fnKLVE9UKFS/tAowqgPS1qE76u1Mmkk6yqZoG9rg==
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBFbDBYMDNQcWkySC8vN05t
+            U3hLMjlYVUE3Zms3U2R4R0VnMUtFcmVQclZvClY4aWZEYWZPdkltMElkUWxQeUtP
+            TEF0a0txbVQ4d3lrelp3cG9TbG5OSkEKLS0tIHR1V3JIVEwwUjM3RVdES2pQUmhP
+            T1MwME1tbGQ2NysrOEVNYVZRT1R0YmcKFpfe9GfH7s779CNQswRm/W7zwYO6wK11
+            z6IGPxtBlUGdshYiHA1BEz7fMVg3ZolL2D98cTNMM24U89Gssiw9qw==
             -----END AGE ENCRYPTED FILE-----
-    lastmodified: "2023-07-08T12:46:19Z"
-    mac: ENC[AES256_GCM,data:bQWG/GgSIv5LdhGTsyx3ENOAywtYVKjzK6nxOnUEZvD+RSi6jxj9Wze7qOhXvgjKWCz/cZj5oSuMQNRyoI8p8xJdxf0+UNdX8uPT05HiKuF7CBcXzprjKri/H6yFp87epOM9fMdn7ZUACn/iT1IZBo+5OuMtDnqVUm/GEmMcsog=,iv:ll/vEeiXsD3crbbxEFsJlxGbm9dZDUPC4GeO95RZZX4=,tag:TGCA721vK9EY3xlY6zIeIg==,type:str]
+    lastmodified: "2024-04-20T23:41:59Z"
+    mac: ENC[AES256_GCM,data:38Ask+adT2FshF8DYEfCWeVWt4KiaJsTXhF7Ib3xxdfQ6vAixM2OXTaK/qqUvN6gQok9TFF+HMJBJ+jezV00nVcKUYn04FaU2/D2zdam44eEEYEEovmfAZ6vbC+CiDv4d/DCc3hnYtDZCEgUTfP4gsZ9rLZFAOwaOFWRJxcDi6Y=,iv:BzuWdTjn6LhscNeouHjM7IYKxTahA8PzzlHSCYZ618s=,tag:BWtPbNwzdOJb788eOO5ZNA==,type:str]
     pgp:
-        - created_at: "2023-01-21T19:52:08Z"
-          enc: |
+        - created_at: "2024-04-20T23:15:17Z"
+          enc: |-
             -----BEGIN PGP MESSAGE-----
 
-            hQIMA0av/duuklWYAQ//Sw5EHNbC9iPXcHSULYVmSMOQCAH7GSGvaaFvey/KffPD
-            5gbFr00vIi1JfjYXmYfn3KKpUfs/mMMo5NzYU2Ou5fWcPsqFLXOwubebuf61X6p7
-            7YfLYQMnjgBzkpb972AJl2tWUlcBcOz89tIw3oMi8R5vvXjRjEdDY8Yp+Z2Apj9V
-            YJCoSIe6RLBlubMs4I6VIOaTaKIM1DWthg95dozlShXYsEgFTYaJ6FbN9RuZOZPa
-            KzFs2DXtbylXXJtiCArQCHnOgA1Jnp80VvMYLO1ldteQhqGdmnxnqwjETx/uqy4l
-            QE31LcRf2JFKi0BBJdQfEqBGW9LD4Mjfwi6tWbHq4Mn29u8IT6z5HJIB99JRAV/9
-            RfBPzF7UVLq2baWxDwG/M6TvZlVJPdAyhJ5QqhkVdrWir7D1D108u+cgtJWw+vlS
-            cP3hT73LWCo2bXUrHXxFnrWdDQQSDpew/x2cTHUNvqdqLZgMJWdZgh+mXOQLjzHP
-            xGkjt0ae5/CEnUIse/Qt3SyoKN3rGVKJgoQ4D0AeBFU5z7NEOx7Ebl9t6IgVnJIB
-            sDJXg+7jJ8A0V1xGan6BP8dFi7m0aAJH0xi8RB9jC1ZRVNxUjFow3Szh0JQ7u2P5
-            4jZ3FT4tWzPzLQsgJUd/H41QyKSd3ke4VMf97mEKULJ7prtXdyxQfRDcE93UgVXS
-            XAF0u7pIl+O2RlJtki+UvuwVDszPBRSmGmfiQa4vsYfXahO4fmBjhdl2hdLtz82F
-            dh+dPu+RSD9OKwIhUwsDLtWWlI/4BvIB1yXbQxP2MyjZm3uVf1CtgUHyjWw8
-            =rri5
+            hQIMA0av/duuklWYAQ/+LSTWjii2dblTAkuqHan3uuuRRpt1ppmHEgHYkQZD+RzE
+            g+ljNaM/BPqci7Kr1NHFDw+cU2MYm/40Tz63l1cvfE3NEoVefsmoA5voNI3G/bx/
+            LTAe2aacPwO/TNoLtrCgRkzNyKXluUkM9OoIvkvB5DEGjYbe82+gI5Zi+NbW9N/p
+            5ilr9Cc1jvIivjZMGGPLRgkAc/twOOuyrZlsFd9kddAL9YFO7wpd/dko886y1jE5
+            jz9n9F4SKYOcgLPqZuG1iZ8qaA2zGT2bP2caai/QJAmL90stQCiRWtQgB8KeWugm
+            nRFBm5BLamtoqjXXwzdtXGKbFAhvL5/h+kPxnJDjylfFVbgCpoWJ/fxdE5xxxZtq
+            zCcGCQQsaa85eWkBByhu7TdwyAW7bJCm8z6kfFPGqhNDkS8ifxnEWm6ulgYVokiL
+            WVBvuQCd1s8KSExs6zNWGcGlqgvcbovHXyVlmLeqZfBA7i/vYqksZtBT47rG7nCS
+            YGfHy69yVrMdj4KrLuMXNfjtS92hkQqWmCyl5X5zOSJXqEL2dorMzSZn89gK4nL4
+            V4zOKkKtsj2MqynYn/XAoUf3AfYs2wtRhJiU+r/q+rx9Hx31H8mnUuUerT58yQCY
+            mAkjIhTzvZcWIalQo7xnZhos4p1IYaA7MAuGC6HxuWVaOsyiFkRaKwB9svWyZ/DS
+            XAFID3fQ1xfNyYsW8nvXQmvZubnhE+dAQPaiAFP9ujY4RVXWBFOrV6NAs7y/LID/
+            89lpfWN87JWSJWUk6DCD3AQ+1GiBCFy7uswUJkG4zou1RQBSl7X88ziVDILU
+            =tXkN
             -----END PGP MESSAGE-----
           fp: F7D37890228A907440E1FD4846B9228E814A2AAC
     unencrypted_suffix: _unencrypted
-    version: 3.7.3
+    version: 3.8.1

values.nix

@@ -30,8 +30,11 @@ in rec {
       ipv6 = pvv-ipv6 168;
     };
     ildkule = {
-      ipv4 = pvv-ipv4 187;
-      ipv6 = pvv-ipv6 "1:187";
+      ipv4 = "10.212.25.209";
+      ipv6 = "2001:700:300:6025:f816:3eff:feee:812d";
+
+      ipv4_global = "129.241.153.213";
+      ipv6_global = "2001:700:300:6026:f816:3eff:fe58:f1e8";
     };
     bicep = {
       ipv4 = pvv-ipv4 209;