Upgrade ildkule (!36)

This PR is made while moving Ildkule from PVE on joshua, to Openstack on stack.it.ntnu.no.

- The main monitoring dashboard is moved from https://ildkule.pvv.ntnu.no to https://grafana.pvv.ntnu.no.
- A new service is added: uptime-kuma on https://uptime.pvv.ntnu.no.
- The (hardware) configuration for ildkule is updated to fit the new virtualization environment, boot loader, network interfaces, etc.
- Metrics exporters on other hosts should be updated to allow connections from the new host

As this is the first proper server running on openstack, and therefore outside our main IP range, we might discover challenges in our network structure. For example, the database servers usually only allow connections from this range, so Ildkule can no longer access it. This should be explored, documented and/or fixed as we move more services.

Reviewed-on: #36
Co-authored-by: Felix Albrigtsen <felix@albrigtsen.it>
Co-committed-by: Felix Albrigtsen <felix@albrigtsen.it>

.sops.yaml

@@ -7,7 +7,7 @@ keys:
 
   # Hosts
   - &host_jokum age1gp8ye4g2mmw3may5xg0zsy7mm04glfz3788mmdx9cvcsdxs9hg0s0cc9kt
-  - &host_ildkule age1hn45n46ypyrvypv0mwfnpt9ddrlmw34dwlpf33n8v67jexr3lucq6ahc9x
+  - &host_ildkule age1x28hmzvuv6f2n66c0jtqcca3h9rput8d7j5uek6jcpx8n9egd52sqpejq0
   - &host_bekkalokk age12nj59tguy9wg882updc2vjdusx5srnxmjyfaqve4zx6jnnsaw3qsyjq6zd
   - &host_bicep age1sl43gc9cw939z5tgha2lpwf0xxxgcnlw7w4xem4sqgmt2pt264vq0dmwx2
 

hosts/ildkule/configuration.nix

@@ -6,8 +6,8 @@
       ../../base.nix
       ../../misc/metrics-exporters.nix
 
+      ./services/monitoring
       ./services/nginx
-      ./services/metrics
     ];
 
   sops.defaultSopsFile = ../../secrets/ildkule/ildkule.yaml;
@@ -15,28 +15,21 @@
   sops.age.keyFile = "/var/lib/sops-nix/key.txt";
   sops.age.generateKey = true;
 
-  boot.loader.systemd-boot.enable = true;
+  boot.loader.grub.device = "/dev/vda";
-  boot.loader.efi.canTouchEfiVariables = true;
+  boot.tmp.cleanOnBoot = true;
+  zramSwap.enable = true;
 
   networking.hostName = "ildkule"; # Define your hostname.
-
+  systemd.network.networks."30-all" = values.defaultNetworkConfig // {
-  systemd.network.networks."30-ens18" = values.defaultNetworkConfig // {
+    matchConfig.Name = "en*";
-    matchConfig.Name = "ens18";
+    DHCP = "yes";
-    address = with values.hosts.ildkule; [ (ipv4 + "/25") (ipv6 + "/64") ];
+    gateway = [ ];
   };
 
   # List packages installed in system profile
   environment.systemPackages = with pkgs; [
   ];
 
-  # List services that you want to enable:
+  system.stateVersion = "23.11"; # Did you read the comment?
-
-  # This value determines the NixOS release from which the default
-  # settings for stateful data, like file locations and database versions
-  # on your system were taken. It‘s perfectly fine and recommended to leave
-  # this value at the release version of the first install of this system.
-  # Before changing this value read the documentation for this option
-  # (e.g. man configuration.nix or on https://nixos.org/nixos/options.html).
-  system.stateVersion = "21.11"; # Did you read the comment?
 
 }

hosts/ildkule/hardware-configuration.nix

@@ -1,37 +1,9 @@
-# Do not modify this file!  It was generated by ‘nixos-generate-config’
+{ modulesPath, lib, ... }:
-# and may be overwritten by future invocations.  Please make changes
-# to /etc/nixos/configuration.nix instead.
-{ config, lib, pkgs, modulesPath, ... }:
-
 {
-  imports =
+  imports = [ (modulesPath + "/profiles/qemu-guest.nix") ];
-    [ (modulesPath + "/profiles/qemu-guest.nix")
+  boot.initrd.availableKernelModules = [ "ata_piix" "uhci_hcd" "xen_blkfront" "vmw_pvscsi" ];
-    ];
+  boot.initrd.kernelModules = [ "nvme" ];
+  fileSystems."/" = { device = "/dev/vda1"; fsType = "ext4"; };
 
-  boot.initrd.availableKernelModules = [ "ata_piix" "uhci_hcd" "virtio_pci" "sr_mod" "virtio_blk" ];
-  boot.initrd.kernelModules = [ ];
-  boot.kernelModules = [ ];
-  boot.extraModulePackages = [ ];
-
-  fileSystems."/" =
-    { device = "/dev/disk/by-uuid/afe70fe4-681a-4675-8cbd-e5d08cdcf5b5";
-      fsType = "ext4";
-    };
-
-  fileSystems."/boot" =
-    { device = "/dev/disk/by-uuid/B71A-E5CD";
-      fsType = "vfat";
-    };
-
-  swapDevices = [ ];
-
-  # Enables DHCP on each ethernet and wireless interface. In case of scripted networking
-  # (the default) this is the recommended approach. When using systemd-networkd it's
-  # still possible to use this option, but it's recommended to use it in conjunction
-  # with explicit per-interface declarations with `networking.interfaces.<interface>.useDHCP`.
   networking.useDHCP = lib.mkDefault true;
-  # networking.interfaces.ens18.useDHCP = lib.mkDefault true;
-
-  nixpkgs.hostPlatform = lib.mkDefault "x86_64-linux";
-  hardware.cpu.intel.updateMicrocode = lib.mkDefault config.hardware.enableRedistributableFirmware;
 }

hosts/ildkule/services/monitoring/default.nix

@@ -2,8 +2,9 @@
 
 {
   imports = [
-    ./prometheus
     ./grafana.nix
     ./loki.nix
+    ./prometheus
+    ./uptime-kuma.nix
   ];
 }

hosts/ildkule/services/monitoring/grafana.nix

@@ -7,7 +7,6 @@ in {
   in {
     "keys/grafana/secret_key" = { inherit owner group; };
     "keys/grafana/admin_password" = { inherit owner group; };
-    "keys/postgres/grafana" = { inherit owner group; };
   };
 
   services.grafana = {
@@ -18,7 +17,7 @@ in {
       secretFile = path: "$__file{${path}}";
     in {
       server = {
-        domain = "ildkule.pvv.ntnu.no";
+        domain = "grafana.pvv.ntnu.no";
         http_port = 2342;
         http_addr = "127.0.0.1";
       };
@@ -27,13 +26,6 @@ in {
         secret_key = secretFile config.sops.secrets."keys/grafana/secret_key".path;
         admin_password = secretFile config.sops.secrets."keys/grafana/admin_password".path;
       };
-
-      database = {
-        type = "postgres";
-        user = "grafana";
-        host = "${values.hosts.bicep.ipv4}:5432";
-        password = secretFile config.sops.secrets."keys/postgres/grafana".path;
-      };
     };
 
     provision = {

hosts/ildkule/services/monitoring/uptime-kuma.nix

@@ -0,0 +1,20 @@
+{ config, pkgs, lib, ... }:
+let
+  cfg = config.services.uptime-kuma;
+  domain = "status.pvv.ntnu.no";
+in {
+  services.uptime-kuma = {
+    enable = true;
+    settings = {
+      PORT = "5059";
+      HOST = "127.0.1.2";
+    };
+  };
+
+  services.nginx.virtualHosts.${domain} = {
+    enableACME = true;
+    forceSSL = true;
+    kTLS = true;
+    locations."/".proxyPass = "http://${cfg.settings.HOST}:${cfg.settings.PORT}";
+  };
+}

hosts/ildkule/services/nginx/default.nix

@@ -3,13 +3,5 @@
   services.nginx = {
     enable = true;
     enableReload = true;
-    defaultListenAddresses = [
-      values.hosts.ildkule.ipv4
-      "[${values.hosts.ildkule.ipv6}]"
-
-      "127.0.0.1"
-      "127.0.0.2"
-      "[::1]"
-    ];
   };
 }

misc/metrics-exporters.nix

@@ -14,6 +14,8 @@
       "::1"
       values.hosts.ildkule.ipv4
       values.hosts.ildkule.ipv6
+      values.hosts.ildkule.ipv4_global
+      values.hosts.ildkule.ipv6_global
     ];
   };
 

secrets/ildkule/ildkule.yaml

@@ -15,7 +15,6 @@ keys:
         secret_key: ENC[AES256_GCM,data:+WoAJbDBEgKs0RoHT+7oEELAVQ+/2Xt+5RTMSXg23moCqVRx+Gzll9P5Drw=,iv:AkRn/Y20iEe5i1T+84wAgLCTFtAox2G3giyawAkltAw=,tag:BZbt5Wb5lYLIJBm/pfP4GQ==,type:str]
         admin_password: ENC[AES256_GCM,data:ttKwfC4WuXeL/6x4,iv:x1X+e3z08CR992GzC62YnFIN7SGrE81/nDNrgcgVzx0=,tag:YajUoy61kYbpeGeC7yNrXQ==,type:str]
     postgres:
-        grafana: ENC[AES256_GCM,data:D6qkg98WZYzKYegSNBb31v8o+KHisGmJ+ab5Ut7EMtsJz36kUup5RS4EbtM=,iv:rfE1uH1QycKMTpSq2p1ntQ2BIvptAh2J3l/QcQhiuLo=,tag:QxmGFcekjFRPf6orN86IxQ==,type:str]
         postgres_exporter_env: ENC[AES256_GCM,data:8MEoikoA6tFNm9qZbk0DFWANd7nRs5QSqrsGLoLKPIc1xykJaXTlyP5v8ywVGR8j7bfPs4p6QfpUIWK8CCnfQ1QhsFPXUMksl8p+K+xuMakYZr9OoWigGqvOHpFb9blfBN1FBdRrk38REXWAMUn74KSRI9v+0i5lpC4=,iv:anpjWVUadKfSAm9XbkeAKu+jAk+LxcpVYQ+gUe5szYw=,tag:4tzb/8B/e1uVoqTsQGlcKA==,type:str]
         postgres_exporter_knakelibrak_env: ENC[AES256_GCM,data:xjC7DGXrW2GIJq8XioIZb+jSe/Hzcz0tv9cUHmX/n1nhI+D64lYt+EKnq1+RX/vJzU4sTaKjveKBh88Qqnv6RQm+MZC//dIxcvnnAdl50qnHZyBCaFFEzSNI8I8vGyArMk8Ja72clBq3kMpUz/pLBP0qDrjblKDoWkU=,iv:ZW98hJy8A5t4Oxtu17R3tM7gou183VLbgBsHA8LFuJY=,tag:VMOvQz3X/XDylV1YFg2Jsg==,type:str]
 sops:
@@ -24,55 +23,55 @@ sops:
     azure_kv: []
     hc_vault: []
     age:
-        - recipient: age1mrnldl334l2nszuta6ywvewng0fswv2dz9l5g4qcwe3nj4yxf92qjskdx6
+        - recipient: age1x28hmzvuv6f2n66c0jtqcca3h9rput8d7j5uek6jcpx8n9egd52sqpejq0
           enc: |
             -----BEGIN AGE ENCRYPTED FILE-----
-            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBrN3lJM2xWTUZ3UkRBaENI
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBURkY4WTZhQzJoREpxV1Vr
-            VmJiWDlQbHd0VUNYdllPdURyQmUvL3lKMzJzCkZlRFVxbmNLOVNqUFg1akJQQlBP
+            aUExZ1dxNkIyMkJtUXpqOWtTT1J0MGpmMkY4ClR4Wm1FTmhKN2pIMENRdERrWVY2
-            VmdOMUdjZ1M4U2lLVEpGaGI5NjNTR2MKLS0tIDRlQUtucEZhZmRYbmpadVdKK01v
+            SUlHblpEc3VackMrbFpHUUJwM2ltZHcKLS0tIEovMEtiOWc1L2tzZDh3ekZKbStr
-            cWxCQlBRR1VaZTBDQnkzNGE0WGttWm8KK5s/coWNsdCP5lKQ8LMK7/3ku179+Lg1
+            NEFkcW03ZTRJODNxTlVuUnFlcFFUUncKEZzOeUtRsZiuugTLzG2xU4eJ3XtVuop7
-            4ujTVn4LhvXy6JvgGTWS/UbMmJjJebVxkulzf5St3YMMs2mcIYjOtA==
+            hhlDBL/YoFn/CO3HjqFdCVv33QoPu7KKMeV52pbVEnv93mvdEeFxVA==
             -----END AGE ENCRYPTED FILE-----
         - recipient: age17tagmpwqjk3mdy45rfesrfey6h863x8wfq38wh33tkrlrywxducs0k6tpq
           enc: |
             -----BEGIN AGE ENCRYPTED FILE-----
-            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBSa25taGsxdlhrUS96cXBi
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBSY3cxSGFvdDdWcFVLRTRy
-            cUo3WDVmdEhKN256THJhS2tHSitDRkVraDJNCmhGZzlFUDFkN0JKNkFWUlVLVzcz
+            Zng2VnhjZlFkc1RQN0NqUjJGeW02WlFaMlFZCjVZc2x2UXNXS1I2WDBxeHdjNUdr
-            MjFhcDdmcmpxdTA3V3JRREFNVmNUbEEKLS0tIFNSU2xNZzN2Y1ZzR2hFM0dOK0Zy
+            WnZGc0l5NlArekUwUGU3Qkdub25EVm8KLS0tIDB2bGo3ZURtZ0pSZjFzcGpOdW5D
-            Tmk4bXd0ZHhPemxDSDREb3IvSjFza1EKsjtC6J3kYGRe8oLAoUZmg1BUmpkMyC98
+            aTI3aTBUS0d1MzFmMTVMbUlFYTR4VlUKzOvNCAzan1GTXjoRxeySkUYIYtI4Mpvu
-            uYq+IQmfJt48R/MKDei00j1w3zIK5+E5GU4o8+jILzwfpzYUUZWwiA==
+            MC0Q8e350SyoOsrF7fUvw+Ru68fDMLW27H6Ly36xP7D3eo/h4eZVXw==
             -----END AGE ENCRYPTED FILE-----
-        - recipient: age1hn45n46ypyrvypv0mwfnpt9ddrlmw34dwlpf33n8v67jexr3lucq6ahc9x
+        - recipient: age1mrnldl334l2nszuta6ywvewng0fswv2dz9l5g4qcwe3nj4yxf92qjskdx6
           enc: |
             -----BEGIN AGE ENCRYPTED FILE-----
-            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBPb09qTTc4cjRMcjIzRmxu
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBFbDBYMDNQcWkySC8vN05t
-            RzZWTDBNTGdvaEc2VFJPakYvakRMK1RnS1FnCktHRVkwZGlUUXl4UTBRcGxMQzdn
+            U3hLMjlYVUE3Zms3U2R4R0VnMUtFcmVQclZvClY4aWZEYWZPdkltMElkUWxQeUtP
-            QVBCYVdlWEw5NW9tNytJTGIzRlpwa0UKLS0tIGdDdUtFMUgyT0phMXBxZE41Y1h4
+            TEF0a0txbVQ4d3lrelp3cG9TbG5OSkEKLS0tIHR1V3JIVEwwUjM3RVdES2pQUmhP
-            a2hQVVprakt5NURpNXdQUjREczJKWTgKn60yrLqco9brlqigAolO8rEkww9z3y3u
+            T1MwME1tbGQ2NysrOEVNYVZRT1R0YmcKFpfe9GfH7s779CNQswRm/W7zwYO6wK11
-            KmefLVZCGfoko+fnKLVE9UKFS/tAowqgPS1qE76u1Mmkk6yqZoG9rg==
+            z6IGPxtBlUGdshYiHA1BEz7fMVg3ZolL2D98cTNMM24U89Gssiw9qw==
             -----END AGE ENCRYPTED FILE-----
-    lastmodified: "2023-07-08T12:46:19Z"
+    lastmodified: "2024-04-20T23:41:59Z"
-    mac: ENC[AES256_GCM,data:bQWG/GgSIv5LdhGTsyx3ENOAywtYVKjzK6nxOnUEZvD+RSi6jxj9Wze7qOhXvgjKWCz/cZj5oSuMQNRyoI8p8xJdxf0+UNdX8uPT05HiKuF7CBcXzprjKri/H6yFp87epOM9fMdn7ZUACn/iT1IZBo+5OuMtDnqVUm/GEmMcsog=,iv:ll/vEeiXsD3crbbxEFsJlxGbm9dZDUPC4GeO95RZZX4=,tag:TGCA721vK9EY3xlY6zIeIg==,type:str]
+    mac: ENC[AES256_GCM,data:38Ask+adT2FshF8DYEfCWeVWt4KiaJsTXhF7Ib3xxdfQ6vAixM2OXTaK/qqUvN6gQok9TFF+HMJBJ+jezV00nVcKUYn04FaU2/D2zdam44eEEYEEovmfAZ6vbC+CiDv4d/DCc3hnYtDZCEgUTfP4gsZ9rLZFAOwaOFWRJxcDi6Y=,iv:BzuWdTjn6LhscNeouHjM7IYKxTahA8PzzlHSCYZ618s=,tag:BWtPbNwzdOJb788eOO5ZNA==,type:str]
     pgp:
-        - created_at: "2023-01-21T19:52:08Z"
+        - created_at: "2024-04-20T23:15:17Z"
-          enc: |
+          enc: |-
             -----BEGIN PGP MESSAGE-----
 
-            hQIMA0av/duuklWYAQ//Sw5EHNbC9iPXcHSULYVmSMOQCAH7GSGvaaFvey/KffPD
+            hQIMA0av/duuklWYAQ/+LSTWjii2dblTAkuqHan3uuuRRpt1ppmHEgHYkQZD+RzE
-            5gbFr00vIi1JfjYXmYfn3KKpUfs/mMMo5NzYU2Ou5fWcPsqFLXOwubebuf61X6p7
+            g+ljNaM/BPqci7Kr1NHFDw+cU2MYm/40Tz63l1cvfE3NEoVefsmoA5voNI3G/bx/
-            7YfLYQMnjgBzkpb972AJl2tWUlcBcOz89tIw3oMi8R5vvXjRjEdDY8Yp+Z2Apj9V
+            LTAe2aacPwO/TNoLtrCgRkzNyKXluUkM9OoIvkvB5DEGjYbe82+gI5Zi+NbW9N/p
-            YJCoSIe6RLBlubMs4I6VIOaTaKIM1DWthg95dozlShXYsEgFTYaJ6FbN9RuZOZPa
+            5ilr9Cc1jvIivjZMGGPLRgkAc/twOOuyrZlsFd9kddAL9YFO7wpd/dko886y1jE5
-            KzFs2DXtbylXXJtiCArQCHnOgA1Jnp80VvMYLO1ldteQhqGdmnxnqwjETx/uqy4l
+            jz9n9F4SKYOcgLPqZuG1iZ8qaA2zGT2bP2caai/QJAmL90stQCiRWtQgB8KeWugm
-            QE31LcRf2JFKi0BBJdQfEqBGW9LD4Mjfwi6tWbHq4Mn29u8IT6z5HJIB99JRAV/9
+            nRFBm5BLamtoqjXXwzdtXGKbFAhvL5/h+kPxnJDjylfFVbgCpoWJ/fxdE5xxxZtq
-            RfBPzF7UVLq2baWxDwG/M6TvZlVJPdAyhJ5QqhkVdrWir7D1D108u+cgtJWw+vlS
+            zCcGCQQsaa85eWkBByhu7TdwyAW7bJCm8z6kfFPGqhNDkS8ifxnEWm6ulgYVokiL
-            cP3hT73LWCo2bXUrHXxFnrWdDQQSDpew/x2cTHUNvqdqLZgMJWdZgh+mXOQLjzHP
+            WVBvuQCd1s8KSExs6zNWGcGlqgvcbovHXyVlmLeqZfBA7i/vYqksZtBT47rG7nCS
-            xGkjt0ae5/CEnUIse/Qt3SyoKN3rGVKJgoQ4D0AeBFU5z7NEOx7Ebl9t6IgVnJIB
+            YGfHy69yVrMdj4KrLuMXNfjtS92hkQqWmCyl5X5zOSJXqEL2dorMzSZn89gK4nL4
-            sDJXg+7jJ8A0V1xGan6BP8dFi7m0aAJH0xi8RB9jC1ZRVNxUjFow3Szh0JQ7u2P5
+            V4zOKkKtsj2MqynYn/XAoUf3AfYs2wtRhJiU+r/q+rx9Hx31H8mnUuUerT58yQCY
-            4jZ3FT4tWzPzLQsgJUd/H41QyKSd3ke4VMf97mEKULJ7prtXdyxQfRDcE93UgVXS
+            mAkjIhTzvZcWIalQo7xnZhos4p1IYaA7MAuGC6HxuWVaOsyiFkRaKwB9svWyZ/DS
-            XAF0u7pIl+O2RlJtki+UvuwVDszPBRSmGmfiQa4vsYfXahO4fmBjhdl2hdLtz82F
+            XAFID3fQ1xfNyYsW8nvXQmvZubnhE+dAQPaiAFP9ujY4RVXWBFOrV6NAs7y/LID/
-            dh+dPu+RSD9OKwIhUwsDLtWWlI/4BvIB1yXbQxP2MyjZm3uVf1CtgUHyjWw8
+            89lpfWN87JWSJWUk6DCD3AQ+1GiBCFy7uswUJkG4zou1RQBSl7X88ziVDILU
-            =rri5
+            =tXkN
             -----END PGP MESSAGE-----
           fp: F7D37890228A907440E1FD4846B9228E814A2AAC
     unencrypted_suffix: _unencrypted
-    version: 3.7.3
+    version: 3.8.1

values.nix

@@ -30,8 +30,11 @@ in rec {
       ipv6 = pvv-ipv6 168;
     };
     ildkule = {
-      ipv4 = pvv-ipv4 187;
+      ipv4 = "10.212.25.209";
-      ipv6 = pvv-ipv6 "1:187";
+      ipv6 = "2001:700:300:6025:f816:3eff:feee:812d";
+
+      ipv4_global = "129.241.153.213";
+      ipv6_global = "2001:700:300:6026:f816:3eff:fe58:f1e8";
     };
     bicep = {
       ipv4 = pvv-ipv4 209;