temmie/userweb: run log processors as separate systemd units

This lets us divide up some of the logic making httpd itself less
brittle, and also reduces the amount of privileges for httpd.

base/default.nix

@@ -95,6 +95,10 @@
     AllowHibernation = lib.mkDefault false;
   };
 
+  systemd.slices."system-monitoring" = {
+    description = "Monitoring related services";
+  };
+
   # users.mutableUsers = lib.mkDefault false;
 
   users.groups."drift".name = "drift";

base/services/fluentbit.nix

@@ -88,6 +88,7 @@ in
 
   systemd.services.fluent-bit = lib.mkIf cfg.enable {
     serviceConfig = {
+      Slice = "system-monitoring.slice";
       StateDirectory = "fluent-bit";
 
       # NOTE: This hardening might be way too strong for general purpose use, don't upstream this.

base/services/journald-upload.nix

@@ -14,6 +14,7 @@ in
   };
 
   systemd.services."systemd-journal-upload".serviceConfig = lib.mkIf cfg.enable {
+    Slice = "system-monitoring.slice";
     IPAddressDeny = "any";
     IPAddressAllow = [
       values.hosts.ildkule.ipv4

base/services/prometheus-node-exporter.nix

@@ -10,7 +10,7 @@ in
     enabledCollectors = [ "systemd" ];
   };
 
-  services.nginx = {
+  services.nginx = lib.mkIf cfg.enable {
     enable = lib.mkDefault true;
 
     virtualHosts.${config.networking.fqdn} = lib.mkIf config.services.nginx.enable {
@@ -31,4 +31,8 @@ in
       };
     };
   };
+
+  systemd.services = lib.mkIf cfg.enable {
+    "prometheus-node-exporter".serviceConfig.Slice = "system-monitoring.slice";
+  };
 }

base/services/prometheus-systemd-exporter.nix

@@ -13,7 +13,7 @@ in
     ];
   };
 
-  services.nginx = {
+  services.nginx = lib.mkIf cfg.enable {
     enable = lib.mkDefault true;
 
     virtualHosts.${config.networking.fqdn} = lib.mkIf config.services.nginx.enable {
@@ -34,4 +34,8 @@ in
       };
     };
   };
+
+  systemd.services = lib.mkIf cfg.enable {
+    "prometheus-systemd-exporter".serviceConfig.Slice = "system-monitoring.slice";
+  };
 }

base/services/rsyslogd.nix

@@ -1,13 +1,20 @@
-{ ... }:
+{ config, lib, ... }:
+let
+  cfg = config.services.rsyslogd;
+in
 {
   services.rsyslogd = {
-    enable = true;
+    enable = lib.mkDefault true;
     defaultConfig = ''
       *.* @loghost.pvv.ntnu.no
     '';
   };
 
-  services.journald.extraConfig = ''
+  services.journald.extraConfig = lib.mkIf cfg.enable ''
     ForwardToSyslog=yes
   '';
+
+  systemd.services = lib.mkIf cfg.enable {
+    "syslog".serviceConfig.Slice = "system-monitoring.slice";
+  };
 }

base/services/uptimed.nix

@@ -23,7 +23,7 @@ in
       };
     };
 
-    systemd.services.uptimed = lib.mkIf (cfg.enable) {
+    systemd.services.uptimed = lib.mkIf cfg.enable {
       serviceConfig = let
         uptimed = pkgs.uptimed.overrideAttrs (prev: {
           postPatch = ''
@@ -35,6 +35,8 @@ in
         });
 
       in {
+        Slice = "system-monitoring.slice";
+
         Type = "notify";
 
         ExecStart = lib.mkForce "${uptimed}/sbin/uptimed -f";

hosts/temmie/services/userweb/default.nix

@@ -14,8 +14,6 @@ let
     upload_max_filesize = "40M";
   });
 
-  apache-log-processor = pkgs.callPackage ./apache-log-processor { };
-
   # https://nixos.org/manual/nixpkgs/stable/#ssec-php-user-guide-installing-with-extensions
   phpEnv = pkgs.php.buildEnv {
     extensions = { all, ... }: with all; [
@@ -161,13 +159,9 @@ in
 {
   imports = [
     ./mail.nix
+    ./log-processor.nix
   ];
 
-  sops.secrets = {
-    "httpd/passwd-ssh-key" = { };
-    "httpd/ssh-known-hosts" = { };
-  };
-
   services.httpd = {
     enable = true;
     adminAddr = "drift@pvv.ntnu.no";
@@ -237,8 +231,8 @@ in
 
       extraConfig = ''
         CustomLog "${cfg.logDir}/access.log" combined
-        CustomLog "|${lib.getExe apache-log-processor} access" combined
-        ErrorLog "|${lib.getExe apache-log-processor} error"
+        CustomLog "/run/httpd-log-processor-access.fifo" combined
+        ErrorLog "/run/httpd-log-processor-error.fifo"
         ScriptLog "${cfg.logDir}/cgi.log"
 
         UserDir ${lib.concatMapStringsSep " " (l: "/home/pvv/${l}/*/web-docs") homeLetters}
@@ -298,9 +292,26 @@ in
   users.users."wwwrun".uid = lib.mkForce 33;
   users.groups."wwwrun".gid = lib.mkForce 33;
 
+  systemd.targets.userweb = {
+    description = "PVV HTTPD UserWeb";
+  };
+
+  systemd.slices.system-userweb = {
+    description = "PVV HTTPD UserWeb";
+  };
+
   systemd.services.httpd = {
-    after = [ "pvv-homedirs.target" ];
-    requires = [ "pvv-homedirs.target" ];
+    after = [
+      "pvv-homedirs.target"
+      "httpd-log-processor@access.socket"
+      "httpd-log-processor@error.socket"
+    ];
+    requires = [
+      "pvv-homedirs.target"
+      "httpd-log-processor@access.socket"
+      "httpd-log-processor@error.socket"
+    ];
+    requiredBy = [ "userweb.target" ];
 
     environment = {
       PATH = lib.mkForce "/usr/bin";
@@ -308,64 +319,18 @@ in
 
     serviceConfig = {
       Type = lib.mkForce "notify";
-
-      ExecStartPre = let
-        rsyncCommand = ''${lib.getExe pkgs.rsync} -e "${pkgs.openssh}/bin/ssh -o UserKnownHostsFile=%d/ssh-known-hosts -i %d/sshkey" -avz'';
-      in lib.mkForce [
-        "${lib.getExe (pkgs.writeShellApplication {
-          name = "http-exec-start-pre-remove-old-semaphores";
-          text = ''
-            # Get rid of old semaphores.  These tend to accumulate across
-            # server restarts, eventually preventing it from restarting
-            # successfully.
-            for i in $(${pkgs.util-linux}/bin/ipcs -s | grep ' ${cfg.user} ' | cut -f2 -d ' '); do
-                ${pkgs.util-linux}/bin/ipcrm -s "$i"
-            done
-          '';
-        })}"
-
-        "${rsyncCommand} pvv@smtp.pvv.ntnu.no:/etc/passwd /run/httpd/pamunix-in/"
-        "${rsyncCommand} pvv@smtp.pvv.ntnu.no:/etc/group /run/httpd/pamunix-in/"
-
-        (let
-          args = lib.cli.toCommandLineShellGNU { } {
-            passwd-file = "/run/httpd/pamunix-in/passwd";
-            group-file = "/run/httpd/pamunix-in/group";
-            output-dir = "/run/httpd/pamunix-out";
-            shadow-file = pkgs.emptyFile;
-
-            output-passwd = true;
-
-            ignore-user-file = toString ./ignore_user_file.txt;
-            ignore-group-file = toString ./ignore_group_file.txt;
-          };
-        in ''${lib.getExe pkgs.passwd2systemd-users} ${args}'')
-        "${lib.getExe' pkgs.coreutils "shred"} -u /run/httpd/pamunix-in/passwd /run/httpd/pamunix-in/group"
-        ":${lib.getExe pkgs.gnused} -i '$ a\\\\root:x:0:0:System administrator:/root:/run/current-system/sw/bin/bash' /run/httpd/pamunix-out/passwd"
-        ":${lib.getExe pkgs.gnused} -i '$ a\\\\wwwrun:x:54:54:Apache httpd user:/var/empty:/run/current-system/sw/bin/bash' /run/httpd/pamunix-out/passwd"
-        ":${lib.getExe pkgs.gnused} -i '$ a\\\\root:x:0:' /run/httpd/pamunix-out/group"
-        ":${lib.getExe pkgs.gnused} -i '$ a\\\\wwwrun:x:54:' /run/httpd/pamunix-out/group"
-        "+${lib.getExe' pkgs.coreutils "chown"} root:root /run/httpd/pamunix-out/passwd /run/httpd/pamunix-out/group"
-        "+${lib.getExe' pkgs.coreutils "chmod"} 0644 /run/httpd/pamunix-out/passwd /run/httpd/pamunix-out/group"
-        "+${lib.getExe pkgs.mount} --bind /run/httpd/pamunix-out/passwd /etc/passwd"
-        "+${lib.getExe pkgs.mount} --bind /run/httpd/pamunix-out/group  /etc/group"
-      ];
       ExecStart = lib.mkForce "${cfg.package}/bin/httpd -D FOREGROUND -f /etc/httpd/httpd.conf -k start";
       ExecReload = lib.mkForce "${cfg.package}/bin/httpd -f /etc/httpd/httpd.conf -k graceful";
       ExecStop = lib.mkForce "";
       KillMode = "mixed";
-
-      LoadCredential=[
-        "sshkey:${config.sops.secrets."httpd/passwd-ssh-key".path}"
-        "ssh-known-hosts:${config.sops.secrets."httpd/ssh-known-hosts".path}"
-      ];
+      Slice = "system-userweb.slice";
 
       ConfigurationDirectory = [ "httpd" ];
       LogsDirectory = [ "httpd" ];
       LogsDirectoryMode = "0700";
 
-      AmbientCapabilities = [ "CAP_NET_BIND_SERVICE" "CAP_SETUID" "CAP_SETGID" ] ++ lib.optionals debug [ "CAP_SYS_PTRACE" ];
-      CapabilityBoundingSet = [ "CAP_NET_BIND_SERVICE" "CAP_SETUID" "CAP_SETGID" ] ++ lib.optionals debug [ "CAP_SYS_PTRACE" ];
+      AmbientCapabilities = [ "CAP_NET_BIND_SERVICE" ] ++ lib.optionals debug [ "CAP_SYS_PTRACE" ];
+      CapabilityBoundingSet = [ "CAP_NET_BIND_SERVICE" ] ++ lib.optionals debug [ "CAP_SYS_PTRACE" ];
       LockPersonality = !debug;
       PrivateDevices = true;
       PrivateTmp = true;
@@ -393,48 +358,19 @@ in
         "tcp:443"
       ];
       SystemCallArchitectures = "native";
-      SystemCallFilter = lib.mkIf (!debug) [
-         "@system-service"
-         "@setuid"
-      ];
+      SystemCallFilter = lib.mkIf (!debug) [ "@system-service" ];
       UMask = "0077";
 
       RuntimeDirectoryMode = "0750";
-      RuntimeDirectory = [
-        "httpd/root-mnt"
-        "httpd/pamunix-in"
-        "httpd/pamunix-out"
-      ];
+      RuntimeDirectory = [ "httpd/root-mnt" ];
       RootDirectory = "/run/httpd/root-mnt";
       MountAPIVFS = true;
       BindReadOnlyPaths = [
         builtins.storeDir
         "/etc"
         "/dev/null"
-        "/etc/resolv.conf"
         "/var/lib/acme"
-
-        "-/run/httpd/pamunix-out/passwd:/etc/passwd"
-        "-/run/httpd/pamunix-out/group:/etc/group"
-
-        "${pkgs.writeText "userweb-fake-nsswitch.conf" ''
-          passwd:    files
-          group:     files
-          shadow:    files
-          sudoers:   files
-
-          hosts:     mymachines resolve [!UNAVAIL=return] files myhostname dns
-          networks:  files
-
-          ethers:    files
-          services:  files
-          protocols: files
-          rpc:       files
-
-          subuid:    files
-          subgid:    files
-        ''}:/etc/nsswitch.conf"
-
+        "/var/run/nscd"
         "${fhsEnv}/bin:/bin"
         "${fhsEnv}/sbin:/sbin"
         "${fhsEnv}/lib:/lib"
@@ -459,13 +395,16 @@ in
           "/share"
         ];
       });
-      BindPaths = lib.mapCartesianProduct ({ directoryFn, letter }: "/run/pvv-home-mounts/${letter}:${directoryFn letter}${letter}") {
+      BindPaths = (lib.mapCartesianProduct ({ directoryFn, letter }: "/run/pvv-home-mounts/${letter}:${directoryFn letter}${letter}") {
         directoryFn = [
           (_: "/home/pvv/")
           (l: "/amd/homepvv${l}/")
         ];
         letter = homeLetters;
-      };
+      }) ++ [
+        "/run/httpd-log-processor-access.fifo"
+        "/run/httpd-log-processor-error.fifo"
+      ];
     };
   };
 

hosts/temmie/services/userweb/log-processor.nix

@@ -0,0 +1,167 @@
+{ config, lib, pkgs, values, ... }:
+let
+  homeLetters = [ "a" "b" "c" "d" "h" "i" "j" "k" "l" "m" "z" ];
+
+  apache-log-processor = pkgs.callPackage ./apache-log-processor { };
+in
+{
+  sops.secrets = {
+    "httpd/passwd-ssh-key" = { };
+    "httpd/ssh-known-hosts" = { };
+  };
+
+  systemd.targets.sockets.wants = [
+    "httpd-log-processor@access.socket"
+    "httpd-log-processor@error.socket"
+  ];
+
+  systemd.sockets."httpd-log-processor@" = lib.mkIf config.services.httpd.enable {
+    requiredBy = [ "userweb.target" ];
+    socketConfig = {
+      ListenFIFO = "/run/httpd-log-processor-%i.fifo";
+      RemoveOnStop = true;
+
+      SocketUser = "wwwrun";
+      SocketGroup = "wwwrun";
+      SocketMode = "0600";
+    };
+  };
+
+  systemd.services."httpd-log-processor@" = lib.mkIf config.services.httpd.enable {
+    requiredBy = [ "userweb.target" ];
+    serviceConfig = {
+      User = "wwwrun";
+      Group = "wwwrun";
+      Slice = "system-userweb.slice";
+      Restart = "on-failure";
+
+      StandardInput = "socket";
+      StandardOutput = "journal";
+      StandardError = "journal";
+
+      LoadCredential = [
+        "sshkey:${config.sops.secrets."httpd/passwd-ssh-key".path}"
+        "ssh-known-hosts:${config.sops.secrets."httpd/ssh-known-hosts".path}"
+      ];
+      ExecStartPre = let
+        rsyncArgs = lib.cli.toCommandLineShellGNU { } {
+          archive = true;
+          verbose = true;
+          compress = true;
+          rsh = "${lib.getExe' pkgs.openssh "ssh"} -o BatchMode=yes -o UserKnownHostsFile=%d/ssh-known-hosts -i %d/sshkey";
+        };
+        inputDir = "/run/httpd-log-processor-%i/pamunix-in";
+        outputDir = "/run/httpd-log-processor-%i/pamunix-out";
+      in lib.mkForce [
+        "${lib.getExe pkgs.rsync} ${rsyncArgs} pvv@smtp.pvv.ntnu.no:/etc/passwd ${inputDir}/"
+        "${lib.getExe pkgs.rsync} ${rsyncArgs} pvv@smtp.pvv.ntnu.no:/etc/group ${inputDir}/"
+
+        (let
+          args = lib.cli.toCommandLineShellGNU { } {
+            passwd-file = "${inputDir}/passwd";
+            group-file = "${inputDir}/group";
+            output-dir = outputDir;
+            shadow-file = pkgs.emptyFile;
+
+            output-passwd = true;
+
+            ignore-user-file = toString ./ignore_user_file.txt;
+            ignore-group-file = toString ./ignore_group_file.txt;
+          };
+        in ''${lib.getExe pkgs.passwd2systemd-users} ${args}'')
+        "${lib.getExe' pkgs.coreutils "shred"} -u ${inputDir}/passwd ${inputDir}/group"
+        ":${lib.getExe pkgs.gnused} -i '$ a\\\\root:x:0:0:System administrator:/root:/run/current-system/sw/bin/bash' ${outputDir}/passwd"
+        ":${lib.getExe pkgs.gnused} -i '$ a\\\\wwwrun:x:54:54:Apache httpd user:/var/empty:/run/current-system/sw/bin/bash' ${outputDir}/passwd"
+        ":${lib.getExe pkgs.gnused} -i '$ a\\\\root:x:0:' ${outputDir}/group"
+        ":${lib.getExe pkgs.gnused} -i '$ a\\\\wwwrun:x:54:' ${outputDir}/group"
+        "+${lib.getExe' pkgs.coreutils "chown"} root:root ${outputDir}/passwd ${outputDir}/group"
+        "+${lib.getExe' pkgs.coreutils "chmod"} 0644 ${outputDir}/passwd ${outputDir}/group"
+        "+${lib.getExe pkgs.mount} --bind ${outputDir}/passwd /etc/passwd"
+        "+${lib.getExe pkgs.mount} --bind ${outputDir}/group  /etc/group"
+      ];
+
+      ExecStart = "${lib.getExe apache-log-processor} %i";
+
+      AmbientCapabilities = [ "CAP_SETUID" "CAP_SETGID" ];
+      CapabilityBoundingSet = [ "CAP_SETUID" "CAP_SETGID" ];
+      DeviceAllow = [ "" ];
+      LockPersonality = true;
+      MemoryDenyWriteExecute = true;
+      PrivateDevices = true;
+      PrivateIPC = true;
+      PrivateTmp = true;
+      # PrivateUsers = true;
+      ProcSubset = "pid";
+      ProtectClock = true;
+      ProtectControlGroups = true;
+      ProtectHome = "tmpfs";
+      ProtectHostname = true;
+      ProtectKernelLogs = true;
+      ProtectKernelModules = true;
+      ProtectProc = "invisible";
+      ProtectSystem = "strict";
+      ProtectKernelTunables = true;
+      RemoveIPC = true;
+      RestrictAddressFamilies = [
+        "AF_INET"
+        "AF_INET6"
+      ];
+      RestrictNamespaces = true;
+      RestrictRealtime = true;
+      RestrictSUIDSGID = true;
+      SocketBindDeny = "any";
+      SystemCallArchitectures = "native";
+      SystemCallFilter = [
+         "@system-service"
+         "@setuid"
+      ];
+      UMask = "0077";
+
+      IPAddressAllow = [
+        "127.0.0.53" # systemd-resolved
+        values.hosts.microbel.ipv4
+        values.hosts.microbel.ipv6
+      ];
+      IPAddressDeny = "any";
+
+      RootDirectory = "/run/httpd-log-processor-%i/root-mnt";
+      MountAPIVFS = true;
+
+      RuntimeDirectoryMode = "0750";
+      RuntimeDirectory = [
+        "httpd-log-processor-%i/root-mnt"
+        "httpd-log-processor-%i/pamunix-in"
+        "httpd-log-processor-%i/pamunix-out"
+      ];
+      BindReadOnlyPaths = [
+        builtins.storeDir
+        "/etc"
+        "/etc/resolv.conf"
+
+        "-/run/httpd-log-processor-%i/pamunix-out/passwd:/etc/passwd"
+        "-/run/httpd-log-processor-%i/pamunix-out/group:/etc/group"
+
+        "${pkgs.writeText "userweb-fake-nsswitch.conf" ''
+          passwd:    files
+          group:     files
+          shadow:    files
+          sudoers:   files
+
+          hosts:     mymachines resolve [!UNAVAIL=return] files myhostname dns
+          networks:  files
+
+          ethers:    files
+          services:  files
+          protocols: files
+          rpc:       files
+
+          subuid:    files
+          subgid:    files
+        ''}:/etc/nsswitch.conf"
+      ];
+      BindPaths = map (l: "/run/pvv-home-mounts/${l}:/home/pvv/${l}") homeLetters ++ [
+        "/var/log/httpd"
+      ];
+    };
+  };
+}