ildkule/loki: restrict incoming connections to pvv + ntnu

Co-authored-by: Øystein Kristoffer Tveit <oysteikt@pvv.ntnu.no>

base/default.nix

@@ -14,7 +14,6 @@
 
     ./mitigations.nix
 
-    ./flake-input-exporter.nix
     ./hardening.nix
     ./networking.nix
     ./nix.nix
@@ -34,9 +33,11 @@
     ./services/openssh.nix
     ./services/polkit.nix
     ./services/postfix.nix
+    ./services/prometheus-flake-input-exporter.nix
     ./services/prometheus-node-exporter.nix
     ./services/prometheus-systemd-exporter.nix
     ./services/roowho2.nix
+    ./services/rsyslogd.nix
     ./services/scrutiny-collector.nix
     ./services/smartd.nix
     ./services/thermald.nix

base/flake-input-exporter.nix

@@ -1,55 +0,0 @@
-{
-  config,
-  inputs,
-  lib,
-  pkgs,
-  values,
-  ...
-}:
-let
-  data = lib.flip lib.mapAttrs inputs (
-    name: input: {
-      inherit (input)
-        lastModified
-        ;
-    }
-  );
-  folder = pkgs.writeTextDir "share/flake-inputs" (
-    lib.concatMapStringsSep "\n" (
-      { name, value }: ''nixos_last_modified_input{flake="${name}"} ${toString value.lastModified}''
-    ) (lib.attrsToList data)
-  );
-  port = 9102;
-in
-{
-  services.nginx.virtualHosts."${config.networking.fqdn}-nixos-metrics" = {
-    serverName = config.networking.fqdn;
-    serverAliases = [
-      "${config.networking.hostName}.pvv.org"
-    ];
-    locations."/metrics" = {
-      root = "${folder}/share";
-      tryFiles = "/flake-inputs =404";
-      extraConfig = ''
-        default_type text/plain;
-      '';
-    };
-    listen = [
-      {
-        inherit port;
-        addr = "0.0.0.0";
-      }
-    ];
-    extraConfig = ''
-      allow ${values.hosts.ildkule.ipv4}/32;
-      allow ${values.hosts.ildkule.ipv6}/128;
-      allow 127.0.0.1/32;
-      allow ::1/128;
-      allow ${values.ipv4-space};
-      allow ${values.ipv6-space};
-      deny all;
-    '';
-  };
-
-  networking.firewall.allowedTCPPorts = [ port ];
-}

base/services/fluentbit.nix

@@ -62,8 +62,10 @@ in
           name = "loki";
           match = "*";
 
-          host = "ildkule.pvv.ntnu.no";
-          port = 3100;
+          host = "loki.pvv.ntnu.no";
+          port = 443;
+          tls = "on";
+          "tls.verify" = "on";
           uri = "/loki/api/v1/push";
           compress = "gzip";
 

base/services/nginx.nix

@@ -1,18 +1,5 @@
 { config, lib, ... }:
 {
-  # nginx return 444 for all nonexistent virtualhosts
-
-  systemd.services.nginx.after = [ "generate-snakeoil-certs.service" ];
-
-  environment.snakeoil-certs = lib.mkIf config.services.nginx.enable {
-    "/etc/certs/nginx" = {
-      owner = "nginx";
-      group = "nginx";
-    };
-  };
-
-  networking.firewall.allowedTCPPorts = lib.mkIf config.services.nginx.enable [ 80 443 ];
-
   services.nginx = {
     recommendedTlsSettings = true;
     recommendedProxySettings = true;
@@ -60,17 +47,8 @@
           ];
         }
       ];
-      sslCertificate = "/etc/certs/nginx.crt";
-      sslCertificateKey = "/etc/certs/nginx.key";
-      addSSL = true;
-      extraConfig = "return 444;";
-    };
-
-    ${config.networking.fqdn} = {
-      sslCertificate = lib.mkDefault "/etc/certs/nginx.crt";
-      sslCertificateKey = lib.mkDefault "/etc/certs/nginx.key";
-      addSSL = lib.mkDefault true;
-      extraConfig = lib.mkDefault "return 444;";
     };
   };
+
+  networking.firewall.allowedTCPPorts = lib.mkIf config.services.nginx.enable [ 80 443 ];
 }

base/services/prometheus-flake-input-exporter.nix

@@ -0,0 +1,47 @@
+{
+  config,
+  inputs,
+  lib,
+  pkgs,
+  values,
+  ...
+}:
+let
+  data = lib.flip lib.mapAttrs inputs (
+    name: input: {
+      inherit (input)
+        lastModified
+        ;
+    }
+  );
+  folder = pkgs.writeTextDir "share/flake-inputs" (
+    lib.concatMapStringsSep "\n" (
+      { name, value }: ''nixos_last_modified_input{flake="${name}"} ${toString value.lastModified}''
+    ) (lib.attrsToList data)
+  );
+in
+{
+  services.nginx = {
+    enable = lib.mkDefault true;
+
+    virtualHosts.${config.networking.fqdn} = lib.mkIf config.services.nginx.enable {
+      forceSSL = true;
+      enableACME = true;
+      kTLS = true;
+
+      locations."/prometheus-nixos-flake-input-exporter/metrics" = {
+        root = "${folder}/share";
+        tryFiles = "/flake-inputs =404";
+        extraConfig = ''
+          default_type text/plain;
+
+          allow 127.0.0.1;
+          allow ::1;
+          allow ${values.hosts.ildkule.ipv4};
+          allow ${values.hosts.ildkule.ipv6};
+          deny all;
+        '';
+      };
+    };
+  };
+}

base/services/prometheus-node-exporter.nix

@@ -5,19 +5,30 @@ in
 {
   services.prometheus.exporters.node = {
     enable = lib.mkDefault true;
+    listenAddress = "127.0.0.1";
     port = 9100;
     enabledCollectors = [ "systemd" ];
   };
 
-  systemd.services.prometheus-node-exporter.serviceConfig = lib.mkIf cfg.enable {
-    IPAddressDeny = "any";
-    IPAddressAllow = [
-      "127.0.0.1"
-      "::1"
-      values.hosts.ildkule.ipv4
-      values.hosts.ildkule.ipv6
-    ];
-  };
+  services.nginx = {
+    enable = lib.mkDefault true;
 
-  networking.firewall.allowedTCPPorts = lib.mkIf cfg.enable [ cfg.port ];
+    virtualHosts.${config.networking.fqdn} = lib.mkIf config.services.nginx.enable {
+      forceSSL = true;
+      enableACME = true;
+      kTLS = true;
+
+      locations."/prometheus-node-exporter/metrics" = {
+        proxyPass = "http://localhost:${toString cfg.port}/metrics";
+
+        extraConfig = ''
+          allow 127.0.0.1;
+          allow ::1;
+          allow ${values.hosts.ildkule.ipv4};
+          allow ${values.hosts.ildkule.ipv6};
+          deny all;
+        '';
+      };
+    };
+  };
 }

base/services/prometheus-systemd-exporter.nix

@@ -5,6 +5,7 @@ in
 {
   services.prometheus.exporters.systemd = {
     enable = lib.mkDefault true;
+    listenAddress = "127.0.0.1";
     port = 9101;
     extraFlags = [
       "--systemd.collector.enable-restart-count"
@@ -12,15 +13,25 @@ in
     ];
   };
 
-  systemd.services.prometheus-systemd-exporter.serviceConfig = {
-    IPAddressDeny = "any";
-    IPAddressAllow = [
-      "127.0.0.1"
-      "::1"
-      values.hosts.ildkule.ipv4
-      values.hosts.ildkule.ipv6
-    ];
-  };
+  services.nginx = {
+    enable = lib.mkDefault true;
 
-  networking.firewall.allowedTCPPorts = lib.mkIf cfg.enable [ cfg.port ];
+    virtualHosts.${config.networking.fqdn} = lib.mkIf config.services.nginx.enable {
+      forceSSL = true;
+      enableACME = true;
+      kTLS = true;
+
+      locations."/prometheus-systemd-exporter/metrics" = {
+        proxyPass = "http://localhost:${toString cfg.port}/metrics";
+
+        extraConfig = ''
+          allow 127.0.0.1;
+          allow ::1;
+          allow ${values.hosts.ildkule.ipv4};
+          allow ${values.hosts.ildkule.ipv6};
+          deny all;
+        '';
+      };
+    };
+  };
 }

base/services/rsyslogd.nix

@@ -0,0 +1,13 @@
+{ ... }:
+{
+  services.rsyslogd = {
+    enable = true;
+    defaultConfig = ''
+      *.* @loghost.pvv.ntnu.no
+    '';
+  };
+
+  services.journald.extraConfig = ''
+    ForwardToSyslog=yes
+  '';
+}

hosts/ildkule/services/monitoring/loki.nix

@@ -1,4 +1,4 @@
-{ config, pkgs, ... }:
+{ config, pkgs, values, ... }:
 
 let
   cfg = config.services.loki;
@@ -9,8 +9,8 @@ in {
     configuration = {
       auth_enabled = false;
       server = {
-        http_listen_port = 3100;
-        http_listen_address = "0.0.0.0";
+        http_listen_port = 31832;
+        http_listen_address = "127.0.0.1";
         grpc_listen_port = 9096;
       };
 
@@ -81,5 +81,25 @@ in {
     };
   };
 
-  networking.firewall.allowedTCPPorts = [ cfg.configuration.server.http_listen_port ];
+  services.nginx.virtualHosts."loki.pvv.ntnu.no" = {
+    forceSSL = true;
+    enableACME = true;
+    kTLS = true;
+
+    locations = {
+      "/".return = "403";
+      "/loki/api/v1/push" = {
+        proxyPass = "http://${cfg.configuration.server.http_listen_address}:${toString cfg.configuration.server.http_listen_port}/loki/api/v1/push";
+        extraConfig = ''
+          allow 127.0.0.1;
+          allow ::1;
+          allow ${values.ipv4-space};
+          allow ${values.ipv6-space};
+          allow ${values.ntnu.ipv4-space};
+          allow ${values.ntnu.ipv6-space};
+          deny all;
+        '';
+      };
+    };
+  };
 }

hosts/ildkule/services/monitoring/prometheus/default.nix

@@ -8,7 +8,6 @@ in {
     ./matrix-synapse.nix
     ./mysqld.nix
     ./postgres.nix
-    ./dibbler.nix
   ];
 
   services.prometheus = {

hosts/ildkule/services/monitoring/prometheus/dibbler.nix

@@ -1,96 +0,0 @@
-{
-  config,
-  lib,
-  pkgs,
-  ...
-}:
-let
-  cfg = config.services.prometheus.exporters.sql;
-  configFile =
-    if cfg.configFile != null then
-      cfg.configFile
-    else
-      let
-        nameInline = lib.mapAttrsToList (k: v: v // { name = k; });
-        renameStartupSql = j: removeAttrs (j // { startup_sql = j.startupSql; }) [ "startupSql" ];
-        configuration = {
-          jobs = map renameStartupSql (
-            nameInline (lib.mapAttrs (k: v: (v // { queries = nameInline v.queries; })) cfg.configuration.jobs)
-          );
-        };
-      in
-      builtins.toFile "config.yaml" (builtins.toJSON configuration);
-in
-{
-  sops.secrets."config/postgresql_dibbler_password" = { };
-
-  services.prometheus.scrapeConfigs = [
-    {
-      job_name = "sql_exporter";
-      scrape_interval = "1m";
-      scheme = "http";
-
-      static_configs = [
-        {
-          targets = [ "localhost:9237" ];
-        }
-      ];
-    }
-  ];
-
-  services.prometheus.exporters.sql = {
-    enable = true;
-    configuration = {
-      jobs.dibbler = {
-        interval = "1m";
-
-        queries."daily_purchase_sum" = {
-          help = "Sum of purchases for the current day.";
-          labels = [ "thing" ];
-          values = [ "sum" ];
-          query = "SELECT SUM(price) FROM purchases GROUP BY DATE(time) ORDER BY DATE(time) DESC LIMIT 1";
-        };
-
-        queries."total_purchase_sum" = {
-          help = "Sum of all purchases.";
-          labels = [ "thing" ];
-          values = [ "sum" ];
-          query = "SELECT SUM(price) FROM purchases";
-        };
-
-        queries."total_stock_value" = {
-          help = "The value of all stock in dibbler.";
-          labels = [ "thing" ];
-          values = [ "sum" ];
-          query = "SELECT SUM(price * stock) FROM products";
-        };
-
-        queries."user_credit_sum" = {
-          help = "The sum of all user credit.";
-          labels = [ "thing" ];
-          values = [ "sum" ];
-          query = "SELECT SUM(credit) FROM users";
-        };
-      };
-    };
-  };
-
-  systemd.services."prometheus-sql-exporter".serviceConfig = {
-    RuntimeDirectory = "prometheus-sql-exporter";
-    LoadCredential = "postgresql_dibbler_password:${
-      config.sops.secrets."config/postgresql_dibbler_password".path
-    }";
-    ExecStartPre = ''
-      |${lib.getExe pkgs.jq} \
-      --null-input \
-      --compact-output \
-      --slurpfile config '${configFile}' \
-      --rawfile pw '%d/postgresql_dibbler_password' \
-      --from-file '${pkgs.writeText "prometheus-sql-exec-start-jq-filter" ''
-        ("postgres://pvv_vv:\($pw | gsub("\n"; ""))@postgres.pvv.ntnu.no") as $pg_uri
-        | $config[0]
-        | .jobs[0].connections[0] = $pg_uri
-      ''}' > /run/prometheus-sql-exporter/config.yaml
-    '';
-  };
-}

hosts/ildkule/services/monitoring/prometheus/machines.nix

@@ -6,32 +6,63 @@
     targets = map (port: "${name}.pvv.ntnu.no:${toString port}") ports;
   };
 
+  nixosMachines = [
+    "ildkule"
+    "bekkalokk"
+    "bicep"
+    "brzeczyszczykiewicz"
+    "georg"
+    "gluttony"
+    "kommode"
+    "lupine-1"
+    "lupine-2"
+    "lupine-3"
+    "lupine-4"
+    "lupine-5"
+    # TODO: export prometheus stats via apache on temmie
+    # "temmie"
+    "wenche"
+  ];
+
   defaultNodeExporterPort = 9100;
-  defaultSystemdExporterPort = 9101;
-  defaultNixosExporterPort = 9102;
 in {
-  services.prometheus.scrapeConfigs = [{
-    job_name = "base_info";
-    static_configs = [
-      (mkHostScrapeConfig "ildkule" [ cfg.exporters.node.port cfg.exporters.systemd.port defaultNixosExporterPort ])
-
-      (mkHostScrapeConfig "bekkalokk" [ defaultNodeExporterPort defaultSystemdExporterPort defaultNixosExporterPort ])
-      (mkHostScrapeConfig "bicep" [ defaultNodeExporterPort defaultSystemdExporterPort defaultNixosExporterPort ])
-      (mkHostScrapeConfig "brzeczyszczykiewicz" [ defaultNodeExporterPort defaultSystemdExporterPort defaultNixosExporterPort ])
-      (mkHostScrapeConfig "georg" [ defaultNodeExporterPort defaultSystemdExporterPort defaultNixosExporterPort ])
-      (mkHostScrapeConfig "gluttony" [ defaultNodeExporterPort defaultSystemdExporterPort defaultNixosExporterPort ])
-      (mkHostScrapeConfig "kommode" [ defaultNodeExporterPort defaultSystemdExporterPort defaultNixosExporterPort ])
-      (mkHostScrapeConfig "lupine-1" [ defaultNodeExporterPort defaultSystemdExporterPort defaultNixosExporterPort ])
-      (mkHostScrapeConfig "lupine-2" [ defaultNodeExporterPort defaultSystemdExporterPort defaultNixosExporterPort ])
-      (mkHostScrapeConfig "lupine-3" [ defaultNodeExporterPort defaultSystemdExporterPort defaultNixosExporterPort ])
-      (mkHostScrapeConfig "lupine-4" [ defaultNodeExporterPort defaultSystemdExporterPort defaultNixosExporterPort ])
-      (mkHostScrapeConfig "lupine-5" [ defaultNodeExporterPort defaultSystemdExporterPort defaultNixosExporterPort ])
-      (mkHostScrapeConfig "temmie" [ defaultNodeExporterPort defaultSystemdExporterPort defaultNixosExporterPort ])
-      (mkHostScrapeConfig "wenche" [ defaultNodeExporterPort defaultSystemdExporterPort defaultNixosExporterPort ])
-
-      (mkHostScrapeConfig "hildring" [ defaultNodeExporterPort ])
-      (mkHostScrapeConfig "isvegg" [ defaultNodeExporterPort ])
-      (mkHostScrapeConfig "microbel" [ defaultNodeExporterPort ])
-    ];
-  }];
+  services.prometheus.scrapeConfigs = [
+    {
+      job_name = "nixos-node";
+      scheme = "https";
+      metrics_path = "/prometheus-node-exporter/metrics";
+      static_configs = map (name: {
+        labels.hostname = name;
+        targets = [ "${name}.pvv.ntnu.no:443" ];
+      }) nixosMachines;
+    }
+    {
+      job_name = "nixos-systemd";
+      scheme = "https";
+      metrics_path = "/prometheus-systemd-exporter/metrics";
+      static_configs = map (name: {
+        labels.hostname = name;
+        targets = [ "${name}.pvv.ntnu.no:443" ];
+      }) nixosMachines;
+    }
+    {
+      job_name = "nixos-flake-input";
+      scheme = "https";
+      metrics_path = "/prometheus-nixos-flake-input-exporter/metrics";
+      static_configs = map (name: {
+        labels.hostname = name;
+        targets = [ "${name}.pvv.ntnu.no:443" ];
+      }) nixosMachines;
+    }
+    {
+      job_name = "non-nixos-node";
+      scheme = "http";
+      metrics_path = "/metrics";
+      static_configs = [
+        (mkHostScrapeConfig "hildring" [ defaultNodeExporterPort ])
+        (mkHostScrapeConfig "isvegg" [ defaultNodeExporterPort ])
+        (mkHostScrapeConfig "microbel" [ defaultNodeExporterPort ])
+      ];
+    }
+  ];
 }

hosts/temmie/services/userweb/default.nix

@@ -212,6 +212,19 @@ in
     extraConfig = ''
       TraceEnable on
       LogLevel warn rewrite:trace3
+
+      <FilesMatch ".+\.ph(p[3457]?|t|tml)$">
+          SetHandler application/x-httpd-php
+      </FilesMatch>
+      <FilesMatch ".+\.phps$">
+          SetHandler application/x-httpd-php-source
+          Require all denied
+      </FilesMatch>
+      <FilesMatch "\.pl$">
+          SetHandler modperl
+          PerlResponseHandler ModPerl::Registry
+          Options +ExecCGI
+      </FilesMatch>
     '';
 
     virtualHosts."temmie.pvv.ntnu.no" = {
@@ -332,7 +345,6 @@ in
         ":${lib.getExe pkgs.gnused} -i '$ a\\\\wwwrun:x:54:54:Apache httpd user:/var/empty:/run/current-system/sw/bin/bash' /run/httpd/pamunix-out/passwd"
         ":${lib.getExe pkgs.gnused} -i '$ a\\\\root:x:0:' /run/httpd/pamunix-out/group"
         ":${lib.getExe pkgs.gnused} -i '$ a\\\\wwwrun:x:54:' /run/httpd/pamunix-out/group"
-        "${lib.getExe' pkgs.coreutils "cat"} /run/httpd/pamunix-out/passwd"
         "+${lib.getExe' pkgs.coreutils "chown"} root:root /run/httpd/pamunix-out/passwd /run/httpd/pamunix-out/group"
         "+${lib.getExe' pkgs.coreutils "chmod"} 0644 /run/httpd/pamunix-out/passwd /run/httpd/pamunix-out/group"
         "+${lib.getExe pkgs.mount} --bind /run/httpd/pamunix-out/passwd /etc/passwd"
@@ -447,7 +459,13 @@ in
           "/share"
         ];
       });
-      BindPaths = map (l: "/run/pvv-home-mounts/${l}:/home/pvv/${l}") homeLetters;
+      BindPaths = lib.mapCartesianProduct ({ directoryFn, letter }: "/run/pvv-home-mounts/${letter}:${directoryFn letter}${letter}") {
+        directoryFn = [
+          (_: "/home/pvv/")
+          (l: "/amd/homepvv${l}/")
+        ];
+        letter = homeLetters;
+      };
     };
   };
 

secrets/ildkule/ildkule.yaml

@@ -1,6 +1,5 @@
 config:
     mysqld_exporter_password: ENC[AES256_GCM,data:I9K+QMqaN3FOOVKzeOR9Q6UERStXX0P8WEHyN1jzzbM=,iv:UxvIdlfAyJvNuxPkU4+guKPa0fiD0vVLzHOTYktcmso=,tag:ltnIqEwESYx9HBu8UN0ZLw==,type:str]
-    postgresql_dibbler_password: ENC[AES256_GCM,data:wP4CVz9qRE3CJrblWWYqOIkcH3LM5H81,iv:j8zr1TRNlPPqIppYlWhDoKlL7m2Ph2wQlU6bvFj8R9A=,tag:Cfp8zbYkoLqI7DTDpOBlJw==,type:str]
 keys:
     grafana:
         secret_key: ENC[AES256_GCM,data:+WoAJbDBEgKs0RoHT+7oEELAVQ+/2Xt+5RTMSXg23moCqVRx+Gzll9P5Drw=,iv:AkRn/Y20iEe5i1T+84wAgLCTFtAox2G3giyawAkltAw=,tag:BZbt5Wb5lYLIJBm/pfP4GQ==,type:str]
@@ -73,8 +72,8 @@ sops:
             dC9meDZlc3d3aUJEVjc4REF0Y1BLcGcK79LbJzc5KVgEgyJR11crGuX8YcVoJBbT
             Fin7Zoon06L7qx0Zw5u27wV7RKMnYT7hOMiWs6660ZTLcYJ5M1aEZQ==
             -----END AGE ENCRYPTED FILE-----
-    lastmodified: "2026-06-07T12:51:04Z"
-    mac: ENC[AES256_GCM,data:otGwzc3Bme1PGRU4zWRRf3kmAf5EDjT8sPkDK/zDrO0ve+d3x99Qls4DBZV8G6FsFtUXbeKVo70I5VpqUgaIef+nyMl6zzWkW1wpbKIBYjh5fkaP0xxhLnw+shrB088PAXT0wkP7hJBLO0ZJgTrpprJKhmeqVj2HEXBSJ1wSjk8=,iv:cjqtdbWBVLUfpQdykb3vKDKa/VC/kGBybwYtG/eStqc=,tag:remOj4bX/H/LlAPgcXHAdA==,type:str]
+    lastmodified: "2025-03-16T20:08:18Z"
+    mac: ENC[AES256_GCM,data:C2tpWppc13jKJq5d4nmAKQOaNWHm27TKwxAxm1fi2lejN1lqUaoz5bHfTBA7MfaWvuP5uZnfbtG32eeu48mnlWpo58XRUFFecAhb9JUpW9s5IR3/nbzLNkGU7H5C0oWPrxI4thd+bAVduIgBjjFyGj1pe6J9db3c0yUWRwNlwGU=,iv:YpoQ4psiFYOWLGipxv1QvRvr034XFsyn2Bhyy39HmOo=,tag:ByiCWygFC/VokVTbdLoLgg==,type:str]
     pgp:
         - created_at: "2026-05-20T17:35:58Z"
           enc: |-
@@ -97,4 +96,4 @@ sops:
             -----END PGP MESSAGE-----
           fp: F7D37890228A907440E1FD4846B9228E814A2AAC
     unencrypted_suffix: _unencrypted
-    version: 3.12.2
+    version: 3.9.4

users/vegardbm.nix

@@ -1,4 +1,4 @@
-{ pkgs, ... }:
+{ pkgs, config, ... }:
 {
   users.users.vegardbm = {
     isNormalUser = true;
@@ -8,12 +8,14 @@
       "drift"
       "nix-builder-users"
     ];
+    shell = if config.programs.zsh.enable then pkgs.zsh else pkgs.bash;
     packages = with pkgs; [
       btop
       eza
     ];
     openssh.authorizedKeys.keys = [
       "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIDVA3HqEx3je6L1AC+bP8sTxu3ZTKvTCR0npCyOVAYK5 vbm@arch-xeon"
+      "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAICrYATHNvBNAHr9G+VwZIaAQPe02iRgAjqtZkW4x/dje vbm@talos"
     ];
   };
 }