bekkalokk/gitea: add robots.txt

base/default.nix

@@ -10,6 +10,8 @@
 
     ./services/acme.nix
     ./services/auto-upgrade.nix
+    ./services/dbus.nix
+    ./services/fwupd.nix
     ./services/irqbalance.nix
     ./services/logrotate.nix
     ./services/nginx.nix
@@ -17,9 +19,12 @@
     ./services/postfix.nix
     ./services/smartd.nix
     ./services/thermald.nix
+    ./services/userborn.nix
+    ./services/userdbd.nix
   ];
 
   boot.tmp.cleanOnBoot = lib.mkDefault true;
+  boot.kernelPackages = lib.mkDefault pkgs.linuxPackages_latest;
 
   time.timeZone = "Europe/Oslo";
 
@@ -47,6 +52,8 @@
 
   programs.zsh.enable = true;
 
+  security.lockKernelModules = true;
+  security.protectKernelImage = true;
   security.sudo.execWheelOnly = true;
   security.sudo.extraConfig = ''
     Defaults lecture = never

base/nix.nix

@@ -5,10 +5,10 @@
       automatic = true;
       options = "--delete-older-than 2d";
     };
+    optimise.automatic = true;
 
     settings = {
       allow-dirty = true;
-      auto-optimise-store = true;
       builders-use-substitutes = true;
       experimental-features = [ "nix-command" "flakes" ];
       log-lines = 50;

base/services/dbus.nix

@@ -0,0 +1,7 @@
+{ ... }:
+{
+  services.dbus = {
+    enable = true;
+    implementation = "broker";
+  };
+}

base/services/fwupd.nix

@@ -0,0 +1,4 @@
+{ ... }:
+{
+  services.fwupd.enable = true;
+}

base/services/userborn.nix

@@ -0,0 +1,4 @@
+{ ... }:
+{
+  services.userborn.enable = true;
+}

base/services/userdbd.nix

@@ -0,0 +1,4 @@
+{ ... }:
+{
+  services.userdbd.enable = true;
+}

flake.lock

@@ -7,11 +7,11 @@
         ]
       },
       "locked": {
-        "lastModified": 1740485968,
+        "lastModified": 1741786315,
-        "narHash": "sha256-WK+PZHbfDjLyveXAxpnrfagiFgZWaTJglewBWniTn2Y=",
+        "narHash": "sha256-VT65AE2syHVj6v/DGB496bqBnu1PXrrzwlw07/Zpllc=",
         "owner": "nix-community",
         "repo": "disko",
-        "rev": "19c1140419c4f1cdf88ad4c1cfb6605597628940",
+        "rev": "0d8c6ad4a43906d14abd5c60e0ffe7b587b213de",
         "type": "github"
       },
       "original": {
@@ -88,16 +88,16 @@
         ]
       },
       "locked": {
-        "lastModified": 1727410897,
+        "lastModified": 1735857245,
-        "narHash": "sha256-tWsyxvf421ieWUJYgjV7m1eTdr2ZkO3vId7vmtvfFpQ=",
+        "narHash": "sha256-AKLLPrgXTxgzll3DqVUMa4QlPlRN3QceutgFBmEf8Nk=",
         "owner": "dali99",
         "repo": "nixos-matrix-modules",
-        "rev": "ff787d410cba17882cd7b6e2e22cc88d4064193c",
+        "rev": "da9dc0479ffe22362793c87dc089035facf6ec4d",
         "type": "github"
       },
       "original": {
         "owner": "dali99",
-        "ref": "v0.6.1",
+        "ref": "0.7.0",
         "repo": "nixos-matrix-modules",
         "type": "github"
       }
@@ -110,11 +110,11 @@
         "rev": "1b4087bd3322a2e2ba84271c8fcc013e6b641a58",
         "revCount": 2,
         "type": "git",
-        "url": "https://git.pvv.ntnu.no/Drift/minecraft-data.git"
+        "url": "https://git.pvv.ntnu.no/Projects/minecraft-kartverket.git"
       },
       "original": {
         "type": "git",
-        "url": "https://git.pvv.ntnu.no/Drift/minecraft-data.git"
+        "url": "https://git.pvv.ntnu.no/Projects/minecraft-kartverket.git"
       }
     },
     "nix-gitea-themes": {
@@ -139,11 +139,11 @@
     },
     "nixpkgs": {
       "locked": {
-        "lastModified": 1740782485,
+        "lastModified": 1741969460,
-        "narHash": "sha256-GkDJDqHYlPKZFdyxzZHtljxNRsosKB1GCrblqlvLFgo=",
+        "narHash": "sha256-SCNxTTBfMJV7XuTcLUfdAd6cgCGsazzi+DoPrceQrZ0=",
         "owner": "NixOS",
         "repo": "nixpkgs",
-        "rev": "dd5c2540983641bbaabdfc665931592d4c9989e8",
+        "rev": "68612419aa6c9fd5b178b81e6fabbdf46d300ea4",
         "type": "github"
       },
       "original": {
@@ -155,11 +155,11 @@
     },
     "nixpkgs-unstable": {
       "locked": {
-        "lastModified": 1740848276,
+        "lastModified": 1741960758,
-        "narHash": "sha256-bYeI3FEs824X+MJYksKboNlmglehzplqzn+XvcojWMc=",
+        "narHash": "sha256-pSGMbfkxF7TSeco54W+B1q+g22YCVp1qXHgtrdgtyR4=",
         "owner": "NixOS",
         "repo": "nixpkgs",
-        "rev": "e9b0ff70ddc61c42548501b0fafb86bb49cca858",
+        "rev": "845dc1e9cbc2e48640b8968af58b4a19db67aa8f",
         "type": "github"
       },
       "original": {
@@ -196,11 +196,11 @@
         ]
       },
       "locked": {
-        "lastModified": 1737151758,
+        "lastModified": 1741738148,
-        "narHash": "sha256-yZBsefIarFUEhFRj+rCGMp9Zvag3MCafqV/JfGVRVwc=",
+        "narHash": "sha256-cJo6nbcJEOjkazkZ194NDnlsZe0W0wpxeUh2/886uC8=",
-        "ref": "refs/heads/master",
+        "ref": "refs/heads/main",
-        "rev": "a4ebe6ded0c8c124561a41cb329ff30891914b5e",
+        "rev": "c1802e7cf27c7cf8b4890354c982a4eef5b11593",
-        "revCount": 475,
+        "revCount": 486,
         "type": "git",
         "url": "https://git.pvv.ntnu.no/Projects/nettsiden.git"
       },
@@ -253,11 +253,11 @@
         ]
       },
       "locked": {
-        "lastModified": 1739262228,
+        "lastModified": 1741861888,
-        "narHash": "sha256-7JAGezJ0Dn5qIyA2+T4Dt/xQgAbhCglh6lzCekTVMeU=",
+        "narHash": "sha256-ynOgXAyToeE1UdLNfrUn/hL7MN0OpIS2BtNdLjpjPf0=",
         "owner": "Mic92",
         "repo": "sops-nix",
-        "rev": "07af005bb7d60c7f118d9d9f5530485da5d1e975",
+        "rev": "d016ce0365b87d848a57c12ffcfdc71da7a2b55f",
         "type": "github"
       },
       "original": {

flake.nix

@@ -17,7 +17,7 @@
     pvv-calendar-bot.url = "git+https://git.pvv.ntnu.no/Projects/calendar-bot.git";
     pvv-calendar-bot.inputs.nixpkgs.follows = "nixpkgs";
 
-    matrix-next.url = "github:dali99/nixos-matrix-modules/v0.6.1";
+    matrix-next.url = "github:dali99/nixos-matrix-modules/0.7.0";
     matrix-next.inputs.nixpkgs.follows = "nixpkgs";
 
     nix-gitea-themes.url = "git+https://git.pvv.ntnu.no/oysteikt/nix-gitea-themes.git";
@@ -30,7 +30,7 @@
     grzegorz-clients.url = "git+https://git.pvv.ntnu.no/Grzegorz/grzegorz-clients.git";
     grzegorz-clients.inputs.nixpkgs.follows = "nixpkgs";
 
-    minecraft-data.url = "git+https://git.pvv.ntnu.no/Drift/minecraft-data.git";
+    minecraft-data.url = "git+https://git.pvv.ntnu.no/Projects/minecraft-kartverket.git";
   };
 
   outputs = { self, nixpkgs, nixpkgs-unstable, sops-nix, disko, ... }@inputs:
@@ -105,6 +105,7 @@
         modules = [
           inputs.nix-gitea-themes.nixosModules.default
           inputs.pvv-nettsiden.nixosModules.default
+          self.nixosModules.robots-txt
         ];
       };
       bob = stableNixosConfig "bob" {
@@ -150,6 +151,7 @@
     nixosModules = {
       snakeoil-certs = ./modules/snakeoil-certs.nix;
       snappymail = ./modules/snappymail.nix;
+      robots-txt = ./modules/robots-txt.nix;
     };
 
     devShells = forAllSystems (system: {

hosts/bekkalokk/services/gitea/customization.nix

@@ -0,0 +1,52 @@
+{ config, pkgs, lib, fp, ... }:
+let
+  cfg = config.services.gitea;
+in
+{
+  services.gitea-themes.monokai = pkgs.gitea-theme-monokai;
+
+  systemd.services.gitea-customization = lib.mkIf cfg.enable {
+    description = "Install extra customization in gitea's CUSTOM_DIR";
+    wantedBy = [ "gitea.service" ];
+    requiredBy = [ "gitea.service" ];
+
+    serviceConfig =  {
+      Type = "oneshot";
+      User = cfg.user;
+      Group = cfg.group;
+    };
+
+    script = let
+      logo-svg = fp /assets/logo_blue_regular.svg;
+      logo-png = fp /assets/logo_blue_regular.png;
+      extraLinks = pkgs.writeText "gitea-extra-links.tmpl" ''
+        <a class="item" href="https://www.pvv.ntnu.no/">PVV</a>
+        <a class="item" href="https://wiki.pvv.ntnu.no/">Wiki</a>
+        <a class="item" href="https://git.pvv.ntnu.no/Drift/-/projects/4">Tokyo Drift Issues</a>
+      '';
+
+      project-labels = (pkgs.formats.yaml { }).generate "gitea-project-labels.yaml" {
+        labels = lib.importJSON ./labels/projects.json;
+      };
+
+      customTemplates = pkgs.runCommandLocal "gitea-templates" {
+        nativeBuildInputs = with pkgs; [
+          coreutils
+          gnused
+        ];
+      } ''
+        # Bigger icons
+        install -Dm444 "${cfg.package.src}/templates/repo/icon.tmpl" "$out/repo/icon.tmpl"
+        sed -i -e 's/24/48/g' "$out/repo/icon.tmpl"
+      '';
+    in ''
+      install -Dm444 ${logo-svg} ${cfg.customDir}/public/assets/img/logo.svg
+      install -Dm444 ${logo-png} ${cfg.customDir}/public/assets/img/logo.png
+      install -Dm444 ${./loading.apng} ${cfg.customDir}/public/assets/img/loading.png
+      install -Dm444 ${extraLinks} ${cfg.customDir}/templates/custom/extra_links.tmpl
+      install -Dm444 ${project-labels} ${cfg.customDir}/options/label/project-labels.yaml
+
+      "${lib.getExe pkgs.rsync}" -a "${customTemplates}/" ${cfg.customDir}/templates/
+    '';
+  };
+}

hosts/bekkalokk/services/gitea/default.nix

@@ -1,10 +1,11 @@
-{ config, values, fp, pkgs, lib, ... }:
+{ config, values, lib, unstablePkgs, ... }:
 let
   cfg = config.services.gitea;
   domain = "git.pvv.ntnu.no";
   sshPort  = 2222;
 in {
   imports = [
+    ./customization.nix
     ./gpg.nix
     ./import-users
     ./web-secret-provider
@@ -25,6 +26,8 @@ in {
     enable = true;
     appName = "PVV Git";
 
+    package = unstablePkgs.gitea;
+
     database = {
       type = "postgres";
       host = "postgres.pvv.ntnu.no";
@@ -130,6 +133,11 @@ in {
       };
       "ui.meta".DESCRIPTION = "Bokstavelig talt programvareverkstedet";
     };
+
+    dump = {
+      enable = true;
+      type = "tar.gz";
+    };
   };
 
   environment.systemPackages = [ cfg.package ];
@@ -155,42 +163,108 @@ in {
     };
   };
 
-  networking.firewall.allowedTCPPorts = [ sshPort ];
+  environment.robots-txt."gitea" = {
+    virtualHost = domain;
+    rules = [
+      {
+        pre_comment = ''
+          Gitea internals
 
-  # Extra customization
+          See these for more information:
+          - https://gitea.com/robots.txt
+          - https://codeberg.org/robots.txt
+        '';
+        User-agent = "*";
+        Disallow = [
+          "/api/*"
+          "/avatars"
+          "/*/*/src/commit/*"
+          "/*/*/commit/*"
+          "/*/*/*/refs/*"
+          "/*/*/*/star"
+          "/*/*/*/watch"
+          "/*/*/labels"
+          "/*/*/activity/*"
+          "/vendor/*"
+          "/swagger.*.json"
+          "/repo/create"
+          "/repo/migrate"
+          "/org/create"
+          "/*/*/fork"
+          "/*/*/watchers"
+          "/*/*/stargazers"
+          "/*/*/forks"
+          "*/.git/"
+          "/*.git"
+          "/*.atom"
+          "/*.rss"
+        ];
+      }
+      {
+        pre_comment = "Language Spam";
+        Disallow = "/*?lang=";
+      }
+      {
+        pre_comment = ''
+          AI bots
 
-  services.gitea-themes.monokai = pkgs.gitea-theme-monokai;
+          Sourced from:
-
+          - https://www.vg.no/robots.txt
-  systemd.services.install-gitea-customization = {
+          - https://codeberg.org/robots.txt
-    description = "Install extra customization in gitea's CUSTOM_DIR";
+        '';
-    wantedBy = [ "gitea.service" ];
+        User-agent = [
-    requiredBy = [ "gitea.service" ];
+          "AI2Bot"
-
+          "Ai2Bot-Dolma"
-    serviceConfig =  {
+          "Amazonbot"
-      Type = "oneshot";
+          "Applebot-Extended"
-      User = cfg.user;
+          "Bytespider"
-      Group = cfg.group;
+          "CCBot"
-    };
+          "ChatGPT-User"
-
+          "Claude-Web"
-    script = let
+          "ClaudeBot"
-      logo-svg = fp /assets/logo_blue_regular.svg;
+          "Crawlspace"
-      logo-png = fp /assets/logo_blue_regular.png;
+          "Diffbot"
-      extraLinks = pkgs.writeText "gitea-extra-links.tmpl" ''
+          "FacebookBot"
-        <a class="item" href="https://www.pvv.ntnu.no/">PVV</a>
+          "FriendlyCrawler"
-        <a class="item" href="https://wiki.pvv.ntnu.no/">Wiki</a>
+          "GPTBot"
-        <a class="item" href="https://git.pvv.ntnu.no/Drift/-/projects/4">Tokyo Drift Issues</a>
+          "Google-Extended"
-        <a class="item" href="https://wiki.pvv.ntnu.no/wiki/Tjenester/Kodelager">Howto</a>
+          "ICC-Crawler"
-      '';
+          "ImagesiftBot"
-
+          "Kangaroo Bot"
-      project-labels = (pkgs.formats.yaml { }).generate "gitea-project-labels.yaml" {
+          "Meta-ExternalAgent"
-        labels = lib.importJSON ./labels/projects.json;
+          "OAI-SearchBot"
-      };
+          "Omgili"
-    in ''
+          "Omgilibot"
-      install -Dm444 ${logo-svg} ${cfg.customDir}/public/assets/img/logo.svg
+          "PanguBot"
-      install -Dm444 ${logo-png} ${cfg.customDir}/public/assets/img/logo.png
+          "PerplexityBot"
-      install -Dm444 ${./loading.apng} ${cfg.customDir}/public/assets/img/loading.png
+          "PetalBot"
-      install -Dm444 ${extraLinks} ${cfg.customDir}/templates/custom/extra_links.tmpl
+          "Scrapy"
-      install -Dm444 ${project-labels} ${cfg.customDir}/options/label/project-labels.yaml
+          "SemrushBot-OCOB"
-    '';
+          "Sidetrade indexer bot"
+          "Timpibot"
+          "VelenPublicWebCrawler"
+          "Webzio-Extended"
+          "YouBot"
+          "anthropic-ai"
+          "cohere-ai"
+          "cohere-training-data-crawler"
+          "facebookexternalhit"
+          "iaskspider/2.0"
+          "img2dataset"
+          "meta-externalagent"
+          "omgili"
+          "omgilibot"
+        ];
+        Disallow = "/";
+      }
+      {
+        Crawl-delay = "2";
+      }
+      {
+        Sitemap = "https://${domain}/sitemap.xml";
+      }
+    ];
   };
+
+  networking.firewall.allowedTCPPorts = [ sshPort ];
 }

misc/builder.nix

@@ -2,4 +2,10 @@
 
 {
   nix.settings.trusted-users = [ "@nix-builder-users" ];
+  nix.daemonCPUSchedPolicy = "batch";
+
+  boot.binfmt.emulatedSystems = [
+    "aarch64-linux"
+    "armv7l-linux"
+  ];
 }

modules/robots-txt.nix

@@ -0,0 +1,116 @@
+{ config, pkgs, lib, ... }:
+let
+  cfg = config.environment.robots-txt;
+
+  robots-txt-format = {
+    type = let
+      coercedStrToNonEmptyListOfStr = lib.types.coercedTo lib.types.str lib.singleton (lib.types.nonEmptyListOf lib.types.str);
+    in lib.types.listOf (lib.types.submodule {
+      freeformType = lib.types.attrsOf coercedStrToNonEmptyListOfStr;
+      options = {
+        pre_comment = lib.mkOption {
+          description = "Comment to add before the rule";
+          type = lib.types.lines;
+          default = "";
+        };
+        post_comment = lib.mkOption {
+          description = "Comment to add after the rule";
+          type = lib.types.lines;
+          default = "";
+        };
+      };
+    });
+
+    generate = name: value: let
+      makeComment = comment: lib.pipe comment [
+        (lib.splitString "\n")
+        (lib.map (line: if line == "" then "#" else "# ${line}"))
+        (lib.concatStringsSep "\n")
+      ];
+
+      ruleToString = rule: let
+        user_agent = rule.User-agent or [];
+        pre_comment = rule.pre_comment;
+        post_comment = rule.post_comment;
+        rest = builtins.removeAttrs rule [ "User-agent" "pre_comment" "post_comment" ];
+      in lib.concatStringsSep "\n" (lib.filter (x: x != null) [
+        (if (pre_comment != "") then makeComment pre_comment else null)
+        (let
+          user-agents = lib.concatMapStringsSep "\n" (value: "User-agent: ${value}") user_agent;
+        in
+          if user_agent == [] then null else user-agents
+        )
+        (lib.pipe rest [
+          (lib.mapAttrsToList (ruleName: map (value: "${ruleName}: ${value}")))
+          lib.concatLists
+          (lib.concatStringsSep "\n")
+        ])
+        (if (post_comment != "") then makeComment post_comment else null)
+      ]);
+
+      content = lib.concatMapStringsSep "\n\n" ruleToString value;
+    in pkgs.writeText name content;
+  };
+in
+{
+  options.environment.robots-txt = lib.mkOption {
+    default = { };
+    description = ''
+      Different instances of robots.txt to use with web services.
+    '';
+    type = lib.types.attrsOf (lib.types.submodule ({ name, ... }: {
+      options = {
+        enable = lib.mkEnableOption "this instance of robots.txt" // {
+          default = true;
+        };
+
+        path = lib.mkOption {
+          description = "The resulting path of the dir containing the robots.txt file";
+          type = lib.types.path;
+          readOnly = true;
+          default = "/etc/robots-txt/${name}";
+        };
+
+        rules = lib.mkOption {
+          description = "Rules to include in robots.txt";
+          default = [ ];
+          example = [
+            { User-agent = "Googlebot"; Disallow = "/no-googlebot"; }
+            { User-agent = "Bingbot"; Disallow = [ "/no-bingbot" "/no-bingbot2" ]; }
+          ];
+          type = robots-txt-format.type;
+        };
+
+        virtualHost = lib.mkOption {
+          description = "An nginx virtual host to add the robots.txt to";
+          type = lib.types.nullOr lib.types.str;
+          default = null;
+        };
+      };
+    }));
+  };
+
+  config = {
+    environment.etc = lib.mapAttrs' (name: value: {
+      name = "robots-txt/${name}/robots.txt";
+      value.source = robots-txt-format.generate name value.rules;
+    }) cfg;
+
+    services.nginx.virtualHosts = lib.pipe cfg [
+      (lib.filterAttrs (_: value: value.virtualHost != null))
+      (lib.mapAttrs' (name: value: {
+        name = value.virtualHost;
+        value = {
+          locations = {
+            "= /robots.txt" = {
+              extraConfig = ''
+                add_header Content-Type text/plain;
+              '';
+              root = cfg.${name}.path;
+            };
+          };
+        };
+      }))
+    ];
+  };
+}

shell.nix

@@ -11,14 +11,14 @@ pkgs.mkShellNoCC {
     editorconfig-checker
   ];
 
-  shellHook = ''
+  env = {
-    export OS_AUTH_URL=https://api.stack.it.ntnu.no:5000
+    OS_AUTH_URL = "https://api.stack.it.ntnu.no:5000";
-    export OS_PROJECT_ID=b78432a088954cdc850976db13cfd61c
+    OS_PROJECT_ID = "b78432a088954cdc850976db13cfd61c";
-    export OS_PROJECT_NAME="STUDORG_Programvareverkstedet"
+    OS_PROJECT_NAME = "STUDORG_Programvareverkstedet";
-    export OS_USER_DOMAIN_NAME="NTNU"
+    OS_USER_DOMAIN_NAME = "NTNU";
-    export OS_PROJECT_DOMAIN_ID="d3f99bcdaf974685ad0c74c2e5d259db"
+    OS_PROJECT_DOMAIN_ID = "d3f99bcdaf974685ad0c74c2e5d259db";
-    export OS_REGION_NAME="NTNU-IT"
+    OS_REGION_NAME = "NTNU-IT";
-    export OS_INTERFACE=public
+    OS_INTERFACE = "public";
-    export OS_IDENTITY_API_VERSION=3
+    OS_IDENTITY_API_VERSION = "3";
-  '';
+  };
 }