Compare commits

..

4 Commits

Author SHA1 Message Date
oysteikt 384534bf86 treewide: add missing systemd ordering directives
Eval nix flake / evals (push) Successful in 6m28s
Eval nix flake / evals (pull_request) Successful in 7m5s
In particular, we were missing `sops-install-secrets.service` in a ton
of places, and `network-online.target` for some homebrewn services.

This has caused failure of some services on machine startup.
2026-07-22 17:56:06 +09:00
oysteikt 9a837d210d bicep/postgres: log connections/disconnections
Build topology graph / evals (push) Successful in 3m23s
Eval nix flake / evals (push) Successful in 8m59s
2026-07-21 16:39:25 +09:00
oysteikt d3c9fabbc2 base/irqbalance: only start on multi-socket machines 2026-07-21 16:38:38 +09:00
oysteikt 950b163869 bicep/postgresql: use scram-sha-256 as default password algorithm 2026-07-21 16:32:49 +09:00
30 changed files with 241 additions and 176 deletions
+20 -2
View File
@@ -1,4 +1,22 @@
{ ... }:
{ config, pkgs, lib, ... }:
let
cfg = config.services.irqbalance;
in
{
services.irqbalance.enable = true;
}
# irqbalance only has meaningful work to do on multi-socket machines, so
# skip starting it pointlessly everywhere else.
systemd.services.irqbalance.serviceConfig.ExecCondition = let
isMultiSocket = pkgs.writeShellApplication {
name = "irqbalance-is-multi-socket";
runtimeInputs = [ pkgs.coreutils ];
text = ''
sockets=$(cat /sys/devices/system/cpu/cpu*/topology/physical_package_id | sort -u | wc -l)
[ "$sockets" -gt 1 ]
'';
};
in lib.mkIf cfg.enable [
(lib.getExe isMultiSocket)
];
}
@@ -177,6 +177,11 @@ in
};
};
systemd.services."phpfpm-idp" = {
after = [ "sops-install-secrets.service" ];
requires = [ "sops-install-secrets.service" ];
};
services.nginx.virtualHosts."idp.pvv.ntnu.no" = {
forceSSL = true;
enableACME = true;
@@ -287,6 +287,7 @@ in {
systemd.services.mediawiki-init = lib.mkIf cfg.enable {
after = [ "sops-install-secrets.service" ];
requires = [ "sops-install-secrets.service" ];
serviceConfig = {
UMask = lib.mkForce "0007";
};
@@ -294,6 +295,7 @@ in {
systemd.services.phpfpm-mediawiki = lib.mkIf cfg.enable {
after = [ "sops-install-secrets.service" ];
requires = [ "sops-install-secrets.service" ];
serviceConfig = {
UMask = lib.mkForce "0007";
};
+5
View File
@@ -61,6 +61,11 @@ in {
};
};
systemd.services.vaultwarden = {
after = [ "sops-install-secrets.service" ];
requires = [ "sops-install-secrets.service" ];
};
services.nginx.virtualHosts."${domain}" = {
forceSSL = true;
enableACME = true;
@@ -49,6 +49,11 @@ in
'';
};
systemd.services."phpfpm-roundcube" = {
after = [ "sops-install-secrets.service" ];
requires = [ "sops-install-secrets.service" ];
};
# TODO: move this back to `webmail.pvv.ntnu.no/roundcube` subpath
services.nginx.virtualHosts.${domain} = {
@@ -116,6 +116,11 @@ in {
];
};
systemd.services."phpfpm-pvv-nettsiden" = {
after = [ "sops-install-secrets.service" ];
requires = [ "sops-install-secrets.service" ];
};
services.nginx.virtualHosts."pvv.ntnu.no" = {
globalRedirect = cfg.domainName;
redirectCode = 307;
+5
View File
@@ -35,4 +35,9 @@ in {
onCalendar = "*-*-* 09:00:00";
};
};
systemd.services.pvv-calendar-bot = {
after = [ "sops-install-secrets.service" ];
requires = [ "sops-install-secrets.service" ];
};
}
@@ -20,6 +20,8 @@ in
lfs = false;
};
# TODO: add a module setting to specify additional systemd dependencies
# and then mark "sops-install-secrets.service" as a dependency here
instances = let
defaultGithubConfig = {
settings.token_file = config.sops.secrets."gickup/github-token".path;
+5
View File
@@ -55,6 +55,11 @@
# It needs this to be allowed to access the files with the acme group
systemd.services.coturn.serviceConfig.PrivateUsers = lib.mkForce false;
systemd.services.coturn = {
requires = [ "sops-install-secrets.service" ];
after = [ "sops-install-secrets.service" ];
};
systemd.services."acme-${config.services.coturn.realm}".serviceConfig = {
AmbientCapabilities = [ "CAP_NET_BIND_SERVICE" ];
};
+2
View File
@@ -39,6 +39,8 @@ in
};
systemd.services.mx-puppet-discord = {
after = [ "sops-install-secrets.service" ];
requires = [ "sops-install-secrets.service" ];
serviceConfig.SupplementaryGroups = [
config.users.groups.keys-matrix-registrations.name
];
@@ -183,6 +183,7 @@ in
};
services.matrix-hookshot.serviceDependencies = [
"sops-install-secrets.service"
"matrix-synapse.target"
"nginx.service"
];
+6 -1
View File
@@ -43,7 +43,12 @@ in
keyFile = config.sops.templates."matrix-livekit-keyfile".path;
};
systemd.services.lk-jwt-service.environment.LIVEKIT_FULL_ACCESS_HOMESERVERS = lib.mkIf cfg.enable (builtins.concatStringsSep "," [ "pvv.ntnu.no" "dodsorf.as" ]);
systemd.services.lk-jwt-service = lib.mkIf cfg.enable {
after = [ "sops-install-secrets.service" ];
requires = [ "sops-install-secrets.service" ];
environment.LIVEKIT_FULL_ACCESS_HOMESERVERS = builtins.concatStringsSep "," [ "pvv.ntnu.no" "dodsorf.as" ];
};
services.nginx.virtualHosts.${matrixDomain} = lib.mkIf cfg.enable {
locations."^~ /livekit/jwt/" = {
+51 -46
View File
@@ -55,52 +55,57 @@
pantalaimon.username = "bot_admin";
};
systemd.services.mjolnir.serviceConfig = {
DynamicUser = true;
RuntimeDirectory = [ "mjolnir/root-mnt" ];
RootDirectory = "/run/mjolnir/root-mnt";
BindReadOnlyPaths = [
config.sops.secrets."matrix/mjolnir/access_token".path
builtins.storeDir
"/etc"
"/run/nscd"
"/var/run/nscd"
];
systemd.services.mjolnir = {
requires = [ "sops-install-secrets.service" ];
after = [ "sops-install-secrets.service" ];
AmbientCapabilities = "";
CapabilityBoundingSet = "";
LockPersonality = true;
MemoryDenyWriteExecute = false; # node needs this
NoNewPrivileges = true;
PrivateDevices = true;
PrivateMounts = true;
PrivateTmp = true;
PrivateUsers = true;
ProcSubset = "pid";
ProtectClock = true;
ProtectControlGroups = true;
ProtectHome = true;
ProtectHostname = true;
ProtectKernelLogs = true;
ProtectKernelModules = true;
ProtectKernelTunables = true;
ProtectProc = "invisible";
ProtectSystem = "strict";
RemoveIPC = true;
RestrictAddressFamilies = [
"AF_INET"
"AF_INET6"
"AF_UNIX"
];
RestrictNamespaces = true;
RestrictRealtime = true;
RestrictSUIDSGID = true;
SystemCallArchitectures = "native";
SystemCallFilter = [
"@system-service"
"~@privileged"
"~@resources"
];
UMask = "0077";
serviceConfig = {
DynamicUser = true;
RuntimeDirectory = [ "mjolnir/root-mnt" ];
RootDirectory = "/run/mjolnir/root-mnt";
BindReadOnlyPaths = [
config.sops.secrets."matrix/mjolnir/access_token".path
builtins.storeDir
"/etc"
"/run/nscd"
"/var/run/nscd"
];
AmbientCapabilities = "";
CapabilityBoundingSet = "";
LockPersonality = true;
MemoryDenyWriteExecute = false; # node needs this
NoNewPrivileges = true;
PrivateDevices = true;
PrivateMounts = true;
PrivateTmp = true;
PrivateUsers = true;
ProcSubset = "pid";
ProtectClock = true;
ProtectControlGroups = true;
ProtectHome = true;
ProtectHostname = true;
ProtectKernelLogs = true;
ProtectKernelModules = true;
ProtectKernelTunables = true;
ProtectProc = "invisible";
ProtectSystem = "strict";
RemoveIPC = true;
RestrictAddressFamilies = [
"AF_INET"
"AF_INET6"
"AF_UNIX"
];
RestrictNamespaces = true;
RestrictRealtime = true;
RestrictSUIDSGID = true;
SystemCallArchitectures = "native";
SystemCallFilter = [
"@system-service"
"~@privileged"
"~@resources"
];
UMask = "0077";
};
};
}
@@ -56,7 +56,15 @@ in
enableSynapseIntegration = false;
};
systemd.services."matrix-ooye-pre-start" = {
after = [ "sops-install-secrets.service" ];
requires = [ "sops-install-secrets.service" ];
};
systemd.services."matrix-ooye" = {
after = [ "sops-install-secrets.service" ];
requires = [ "sops-install-secrets.service" ];
serviceConfig = {
RuntimeDirectory = [ "matrix-ooye/root-mnt" ];
RootDirectory = "/run/matrix-ooye/root-mnt";
+6 -1
View File
@@ -44,6 +44,11 @@ in {
};
};
systemd.targets."matrix-synapse" = {
after = [ "sops-install-secrets.service" ];
requires = [ "sops-install-secrets.service" ];
};
services.matrix-synapse-next = {
enable = true;
@@ -74,7 +79,7 @@ in {
name = "psycopg2";
args = {
host = "/var/run/postgresql";
database = "synapse";
dbname = "synapse";
user = "matrix-synapse";
cp_min = 1;
cp_max = 5;
@@ -23,6 +23,15 @@ in
};
systemd.services.minecraft-heatmap-ingest-logs = lib.mkIf cfg.enable {
after = [
"sops-install-secrets.service"
"network-online.target"
];
requires = [
"sops-install-secrets.service"
"network-online.target"
];
serviceConfig = {
LoadCredential = [
"sshkey:${config.sops.secrets."minecraft-heatmap/ssh-key/private".path}"
+4
View File
@@ -56,9 +56,13 @@ in
systemd.services.mysql = lib.mkIf cfg.enable {
after = [
"sops-install-secrets.service"
"systemd-tmpfiles-setup.service"
"systemd-tmpfiles-resetup.service"
];
requires = [
"sops-install-secrets.service"
];
serviceConfig = {
BindPaths = [ "${dataDir}:${cfg.dataDir}" ];
@@ -14,6 +14,7 @@ in
extensions = ps: with ps; [ pg_repack ];
enableTCPIP = true;
# NOTE: md5 accepts both md5 and scram-sha-256
authentication = ''
host all all ${values.ipv4-space} md5
host all all ${values.ipv6-space} md5
@@ -76,6 +77,15 @@ in
maintenance_io_concurrency = 100;
wal_recycle = true;
# -------------------------------- #
# Authentication
password_encryption = "scram-sha-256";
# Logging
log_connections = "authorization";
log_disconnections = true;
# SSL
ssl = true;
ssl_cert_file = "/run/credentials/postgresql.service/cert";
+9
View File
@@ -82,6 +82,15 @@ in {
};
systemd.services."render-bluemap-maps" = {
after = [
"sops-install-secrets.service"
"network-online.target"
];
requires = [
"sops-install-secrets.service"
"network-online.target"
];
serviceConfig = {
StateDirectory = [ "bluemap/world" ];
ExecStartPre = let
@@ -79,6 +79,11 @@ in {
};
};
systemd.services.grafana = {
after = [ "sops-install-secrets.service" ];
requires = [ "sops-install-secrets.service" ];
};
services.nginx.virtualHosts.${cfg.settings.server.domain} = {
enableACME = true;
forceSSL = true;
@@ -37,9 +37,15 @@ in {
};
};
systemd.services.prometheus-postgres-exporter-knakelibrak.serviceConfig = let
localCfg = config.services.prometheus.exporters.postgres;
in lib.recursiveUpdate config.systemd.services.prometheus-postgres-exporter.serviceConfig {
systemd.services.prometheus-postgres-exporter-knakelibrak = {
after = [ "sops-install-secrets.service" ];
requires = [ "sops-install-secrets.service" ];
# TODO: is it really necessary to use recursiveUpdate on the entire attrset here?
# Why don't we just update the relevant attrs directly?
serviceConfig = let
localCfg = config.services.prometheus.exporters.postgres;
in lib.recursiveUpdate config.systemd.services.prometheus-postgres-exporter.serviceConfig {
EnvironmentFile = config.sops.secrets."keys/postgres/postgres_exporter_knakelibrak_env".path;
ExecStart = ''
${pkgs.prometheus-postgres-exporter}/bin/postgres_exporter \
@@ -48,4 +54,5 @@ in {
${lib.concatStringsSep " \\\n " localCfg.extraFlags}
'';
};
};
}
+11 -2
View File
@@ -154,8 +154,14 @@ in {
environment.systemPackages = [ cfg.package ];
systemd.services.gitea = lib.mkIf cfg.enable {
wants = [ "redis-gitea.service" ];
after = [ "redis-gitea.service" ];
after = [
"sops-install-secrets.service"
"redis-gitea.service"
];
requires = [
"sops-install-secrets.service"
"redis-gitea.service"
];
serviceConfig = {
CPUSchedulingPolicy = "batch";
@@ -216,6 +222,9 @@ in {
};
systemd.services.gitea-dump = {
after = [ "sops-install-secrets.service" ];
requires = [ "sops-install-secrets.service" ];
serviceConfig.ExecStart = let
args = lib.cli.toCommandLineShellGNU { } {
type = cfg.dump.type;
+4
View File
@@ -32,7 +32,11 @@ in
systemd.services.gitea-ensure-gnupg-homedir = {
description = "Import gpg key for gitea";
before = [ "gitea.service" ];
after = [ "sops-install-secrets.service" ];
requires = [ "sops-install-secrets.service" ];
environment = { inherit GNUPGHOME; };
serviceConfig = {
Type = "oneshot";
@@ -12,6 +12,16 @@ in
systemd.services.gitea-import-users = lib.mkIf cfg.enable {
enable = true;
environment.PASSWD_FILE_PATH = "/run/gitea-import-users/passwd";
after = [
"sops-install-secrets.service"
"gitea.service"
"network-online.target"
];
requires = [
"sops-install-secrets.service"
"gitea.service"
"network-online.target"
];
serviceConfig = {
ExecStartPre = ''${pkgs.rsync}/bin/rsync -e "${pkgs.openssh}/bin/ssh -o UserKnownHostsFile=$CREDENTIALS_DIRECTORY/ssh-known-hosts -i $CREDENTIALS_DIRECTORY/sshkey" -a pvv@smtp.pvv.ntnu.no:/etc/passwd /run/gitea-import-users/passwd'';
ExecStart = pkgs.writers.writePython3 "gitea-import-users" {
@@ -48,7 +48,16 @@ in
# %d - secrets directory
systemd.services."gitea-web-secret-provider@" = {
description = "Ensure all repos in %i has an SSH key to push web content";
requires = [ "gitea.service" "network.target" ];
after = [
"sops-install-secrets.service"
"gitea.service"
"network-online.target"
];
requires = [
"sops-install-secrets.service"
"gitea.service"
"network-online.target"
];
serviceConfig = {
Slice = "system-giteaweb.slice";
Type = "oneshot";
+5
View File
@@ -94,6 +94,11 @@ in
};
};
systemd.services."openvpn-ov-firewall" = {
after = [ "sops-install-secrets.service" ];
requires = [ "sops-install-secrets.service" ];
};
networking.firewall = {
allowedTCPPorts = [ 1194 ];
allowedUDPPorts = [ 1194 ];
+5
View File
@@ -53,5 +53,10 @@
};
};
systemd.services.drumknotty-screen-session = {
after = [ "sops-install-secrets.service" ];
requires = [ "sops-install-secrets.service" ];
};
services.roowho2.settings.rwhod.ignoreUsers = [ "drumknotty" ];
}
@@ -29,9 +29,15 @@ in
systemd.services."httpd-passwd-sync" = {
requiredBy = [ "userweb.target" ];
after = [
"network-online.target"
"sops-install-secrets.service"
"systemd-tmpfiles-setup.service"
"systemd-tmpfiles-resetup.service"
];
requires = [
"network-online.target"
"sops-install-secrets.service"
];
serviceConfig = {
Type = "oneshot";
Slice = "system-userweb.slice";
+4 -2
View File
@@ -159,7 +159,6 @@ in
description = "Out of Your Element - a Discord bridge for Matrix.";
wants = [
"network-online.target"
"matrix-synapse.service"
"conduit.service"
"dendrite.service"
@@ -168,7 +167,10 @@ in
"matrix-ooye-pre-start.service"
"network-online.target"
];
requires = [ "matrix-ooye-pre-start.service" ];
requires = [
"network-online.target"
"matrix-ooye-pre-start.service"
];
wantedBy = [ "multi-user.target" ];
startLimitIntervalSec = 5;
+11 -118
View File
@@ -36,9 +36,6 @@ in
description = ''
For each item in this list, a `ListenStream`
option in the `[Socket]` section will be created.
Only a single `ListenStream` is currently supported by the handler script; if
you need more than one, you'll have to adjust {option}`handler` accordingly.
'';
};
@@ -101,10 +98,8 @@ in
"E303" # too many blank lines
"E305" # expected 2 blank lines after end of function or class
"E306" # expected 1 blank line before a nested definition
"E402" # module level import not at top of file
"E501" # max line length
"E704" # multiple statements on one line (def)
"F811" # redefined while unused
];
description = ''
A list of flake8 rules to ignore while linting the python code.
@@ -127,11 +122,6 @@ in
self.send_header("Content-Length", str(len(data)))
self.end_headers()
self.wfile.write(data)
def on_reload():
# This function is called when the service receives SIGHUP (e.g. via `systemctl reload`).
# You can use it to clear caches or re-read state. Completely optional
pass
'';
description = ''
Python code including the HTTP handler for the server.
@@ -155,7 +145,6 @@ in
inherit (v) listenStreams;
socketConfig = {
Accept = false;
FileDescriptorName = v.name;
} // v.socketConfig;
};
}))
@@ -168,126 +157,30 @@ in
inherit (v) name;
value = {
serviceConfig = {
Type = "notify-reload";
NotifyAccess = "main";
Type = "simple";
DynamicUser = true;
TimeoutStopSec = "35s";
ExecStart = let
package = pkgs.writers.writePython3Bin "${v.name}-bin" {
inherit (v) libraries flakeIgnore;
} ''
import os
import signal
import socket
import socketserver
import threading
import time
from http.server import HTTPServer, BaseHTTPRequestHandler
SOCKET_NAME = "${v.name}"
SHUTDOWN_TIMEOUT = 30
def sd_notify(message: str):
addr = os.environ.get("NOTIFY_SOCKET")
if not addr:
return
if addr[0] == "@":
addr = "\0" + addr[1:]
with socket.socket(socket.AF_UNIX, socket.SOCK_DGRAM | socket.SOCK_CLOEXEC) as sock:
sock.connect(addr)
sock.sendall(message.encode())
def sd_listen_fd(name: str) -> int:
if os.environ.get("LISTEN_PID") != str(os.getpid()):
raise RuntimeError("No sockets were passed to this service by systemd")
try:
count = int(os.environ.get("LISTEN_FDS", "0"))
except ValueError:
count = 0
raw_names = os.environ.get("LISTEN_FDNAMES")
names = raw_names.split(":") if raw_names else []
for i in range(count):
if i < len(names) and names[i] == name:
return 3 + i
raise RuntimeError(
"No systemd socket named %r was passed to this service; check the "
"FileDescriptorName= of the corresponding .socket unit" % name
)
class Server(socketserver.ThreadingMixIn, HTTPServer):
daemon_threads = True
def server_bind(): pass
def server_activate(): pass
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
self._request_threads = []
self._request_threads_lock = threading.Lock()
def process_request(self, request, client_address):
thread = threading.Thread(
target=self.process_request_thread,
args=(request, client_address),
)
thread.daemon = self.daemon_threads
with self._request_threads_lock:
self._request_threads.append(thread)
thread.start()
def join_request_threads(self, timeout):
deadline = time.monotonic() + timeout
with self._request_threads_lock:
threads = list(self._request_threads)
for thread in threads:
thread.join(max(deadline - time.monotonic(), 0))
def handle_reload(signum, frame):
monotonic_usec = time.clock_gettime_ns(time.CLOCK_MONOTONIC) // 1000
sd_notify("RELOADING=1\nMONOTONIC_USEC=%d" % monotonic_usec)
on_reload = globals().get("on_reload")
if callable(on_reload):
on_reload()
sd_notify("READY=1")
shutdown_requested = threading.Event()
def handle_sigterm(signum, frame):
sd_notify("STOPPING=1")
shutdown_requested.set()
${v.handler}
assert "Handler" in globals(), "You must define a class Handler(BaseHTTPRequestHandler) in the handler code"
class NoBindHTTPServer(HTTPServer):
def server_bind(): pass
def server_activate(): pass
def main():
signal.signal(signal.SIGHUP, handle_reload)
signal.signal(signal.SIGTERM, handle_sigterm)
fd = sd_listen_fd(SOCKET_NAME)
httpd = Server(("", 0), Handler, bind_and_activate=False)
httpd.socket = socket.socket(fileno=fd)
server_thread = threading.Thread(target=httpd.serve_forever, name="http-server", daemon=True)
server_thread.start()
sd_notify("READY=1")
shutdown_requested.wait()
deadline = time.monotonic() + SHUTDOWN_TIMEOUT
httpd.shutdown()
server_thread.join(max(deadline - time.monotonic(), 0))
httpd.join_request_threads(max(deadline - time.monotonic(), 0))
httpd.server_close()
httpd = NoBindHTTPServer(
("", 0),
Handler,
bind_and_activate=False,
)
httpd.socket = socket.fromfd(3, socket.AF_INET, socket.SOCK_STREAM)
httpd.serve_forever()
if __name__ == '__main__':
main()