bicep: add git-mirroring service

flake.nix

@@ -91,6 +91,7 @@
         modules = [
           inputs.matrix-next.nixosModules.default
           inputs.pvv-calendar-bot.nixosModules.default
+          self.nixosModules.gickup
         ];
         overlays = [
           inputs.pvv-calendar-bot.overlays.x86_64-linux.default
@@ -164,6 +165,7 @@
       snakeoil-certs = ./modules/snakeoil-certs.nix;
       snappymail = ./modules/snappymail.nix;
       robots-txt = ./modules/robots-txt.nix;
+      gickup = ./modules/gickup;
     };
 
     devShells = forAllSystems (system: {

hosts/bicep/configuration.nix

@@ -8,6 +8,7 @@
     ./services/nginx
 
     ./services/calendar-bot.nix
+    ./services/git-mirrors
     ./services/mysql.nix
     ./services/postgres.nix
 

hosts/bicep/services/git-mirrors/default.nix

@@ -0,0 +1,92 @@
+{ config, pkgs, lib, fp, ... }:
+let
+  cfg = config.services.gickup;
+in
+{
+  sops.secrets."gickup/github-token" = {
+    owner = "gickup";
+  };
+
+  services.gickup = {
+    enable = true;
+
+    dataDir = "/data/gickup";
+
+    destinationSettings = {
+      structured = true;
+      zip = false;
+      keep = 10;
+      bare = true;
+      lfs = true;
+    };
+
+    instances = let
+      defaultGithubConfig = {
+        settings.token_file = config.sops.secrets."gickup/github-token".path;
+      };
+      defaultGitlabConfig = {
+        # settings.token_file = ...
+      };
+    in {
+      "github:Git-Mediawiki/Git-Mediawiki" = defaultGithubConfig;
+      "github:NixOS/nixpkgs" = defaultGithubConfig;
+      "github:go-gitea/gitea" = defaultGithubConfig;
+      "github:heimdal/heimdal" = defaultGithubConfig;
+      "github:saltstack/salt" = defaultGithubConfig;
+      "github:typst/typst" = defaultGithubConfig;
+      "github:unmojang/FjordLauncher" = defaultGithubConfig;
+      "github:unmojang/drasl" = defaultGithubConfig;
+      "github:yushijinhun/authlib-injector" = defaultGithubConfig;
+
+      "gitlab:mx-puppet/discord/better-discord.js" = defaultGitlabConfig;
+      "gitlab:mx-puppet/discord/discord-markdown" = defaultGitlabConfig;
+      "gitlab:mx-puppet/discord/matrix-discord-parser" = defaultGitlabConfig;
+      "gitlab:mx-puppet/discord/mx-puppet-discord" = defaultGitlabConfig;
+      "gitlab:mx-puppet/mx-puppet-bridge" = defaultGitlabConfig;
+
+      "any:glibc" = {
+        settings.url = "https://sourceware.org/git/glibc.git";
+      };
+    };
+  };
+
+  services.cgit = let
+    domain = "bicep.pvv.ntnu.no";
+  in {
+    ${domain} = {
+      enable = true;
+      package = pkgs.callPackage (fp /packages/cgit.nix) { };
+      group = "gickup";
+      scanPath = "${cfg.dataDir}/linktree";
+      settings = {
+        enable-commit-graph = true;
+        enable-follow-links = true;
+        enable-http-clone = true;
+        enable-remote-branches = true;
+        clone-url = "https://${domain}/$CGIT_REPO_URL";
+        remove-suffix = true;
+        root-title = "PVVSPPP";
+        root-desc = "PVV Speiler Praktisk og Prominent Programvare";
+        snapshots = "all";
+        logo = "/PVV-logo.png";
+      };
+    };
+  };
+
+  services.nginx.virtualHosts."bicep.pvv.ntnu.no" = {
+    forceSSL = true;
+    enableACME = true;
+
+    locations."= /PVV-logo.png".alias = let
+      small-pvv-logo = pkgs.runCommandLocal "pvv-logo-96x96" {
+        nativeBuildInputs = [ pkgs.imagemagick ];
+      } ''
+        magick '${fp /assets/logo_blue_regular.svg}' -resize 96x96 PNG:"$out"
+      '';
+    in toString small-pvv-logo;
+  };
+
+  systemd.services."fcgiwrap-cgit-bicep.pvv.ntnu.no" = {
+    serviceConfig.BindReadOnlyPaths = [ cfg.dataDir ];
+  };
+}

modules/gickup/default.nix

@@ -0,0 +1,312 @@
+{ config, pkgs, lib, utils, ... }:
+let
+  cfg = config.services.gickup;
+  format = pkgs.formats.yaml { };
+in
+{
+  imports = [
+    ./set-description.nix
+    ./hardlink-files.nix
+    ./import-from-toml.nix
+    ./update-linktree.nix
+  ];
+
+  options.services.gickup = {
+    enable = lib.mkEnableOption "gickup, a git repository mirroring service";
+
+    package = lib.mkPackageOption pkgs "gickup" { };
+    gitPackage = lib.mkPackageOption pkgs "git" { };
+    gitLfsPackage = lib.mkPackageOption pkgs "git-lfs" { };
+
+    dataDir = lib.mkOption {
+      type = lib.types.path;
+      description = "The directory to mirror repositories to.";
+      default = "/var/lib/gickup";
+      example = "/data/gickup";
+    };
+
+    destinationSettings = lib.mkOption {
+      description = ''
+        Settings for destination local, see gickup configuration file
+
+        Note that `path` will be set automatically to `/var/lib/gickup`
+      '';
+      type = lib.types.submodule {
+        freeformType = format.type;
+      };
+      default = { };
+      example = {
+        structured = true;
+        zip = false;
+        keep = 10;
+        bare = true;
+        lfs = true;
+      };
+    };
+
+    instances = lib.mkOption {
+      type = lib.types.attrsOf (lib.types.submodule (submoduleInputs@{ name, ... }: let
+        submoduleName = name;
+
+        nameParts = rec {
+          repoType = builtins.head (lib.splitString ":" submoduleName);
+
+          owner = if repoType == "any"
+                  then null
+                  else lib.pipe submoduleName [
+                    (lib.removePrefix "${repoType}:")
+                    (lib.splitString "/")
+                    builtins.head
+                    lib.toLower
+                  ];
+
+          repo = if repoType == "any"
+                 then null
+                 else lib.pipe submoduleName [
+                    (lib.removePrefix "${repoType}:")
+                    (lib.splitString "/")
+                    lib.last
+                    lib.toLower
+                  ];
+
+          slug = if repoType == "any"
+                 then builtins.replaceStrings [ ":" "/" ] [ "-" "-" ] submoduleName
+                 else "${repoType}-${owner}-${repo}";
+        };
+      in {
+        options = {
+          interval = lib.mkOption {
+            type = lib.types.str;
+            default = "daily";
+            example = "weekly";
+            description = ''
+              Specification (in the format described by {manpage}`systemd.time(7)`) of the time
+              interval at which to run the service.
+            '';
+          };
+
+          type = lib.mkOption {
+            type = lib.types.enum [
+              "github"
+              "gitlab"
+              "gitea"
+              "gogs"
+              "bitbucket"
+              "onedev"
+              "sourcehut"
+              "any"
+            ];
+            example = "github";
+            default = nameParts.repoType;
+            description = ''
+              The type of the repository to mirror.
+            '';
+          };
+
+          owner = lib.mkOption {
+            type = with lib.types; nullOr str;
+            example = "go-gitea";
+            default = nameParts.owner;
+            description = ''
+              The owner of the repository to mirror (if applicable)
+            '';
+          };
+
+          repo = lib.mkOption {
+            type = with lib.types; nullOr str;
+            example = "gitea";
+            default = nameParts.repo;
+            description = ''
+              The name of the repository to mirror (if applicable)
+            '';
+          };
+
+          slug = lib.mkOption {
+            type = lib.types.str;
+            default = nameParts.slug;
+            example = "github-go-gitea-gitea";
+            description = ''
+              The slug of the repository to mirror.
+            '';
+          };
+
+          description = lib.mkOption {
+            type = with lib.types; nullOr str;
+            example = "A project which does this and that";
+            description = ''
+              A description of the project. This isn't used directly by gickup for anything,
+              but can be useful if gickup is used together with cgit or similar.
+            '';
+          };
+
+          settings = lib.mkOption {
+            description = "Instance specific settings, see gickup configuration file";
+            type = lib.types.submodule {
+              freeformType = format.type;
+            };
+            default = { };
+            example = {
+              username = "gickup";
+              password = "hunter2";
+              wiki = true;
+              issues = true;
+            };
+          };
+        };
+      }));
+    };
+  };
+
+  config = lib.mkIf cfg.enable {
+    users.users.gickup = {
+      isSystemUser = true;
+      group = "gickup";
+      home = "/var/lib/gickup";
+    };
+
+    users.groups.gickup = { };
+
+    services.gickup.destinationSettings.path = "/var/lib/gickup/raw";
+
+    systemd.tmpfiles.settings."10-gickup" = lib.mkIf (cfg.dataDir != "/var/lib/gickup") {
+      ${cfg.dataDir}.d = {
+        user = "gickup";
+        group = "gickup";
+        mode = "0755";
+      };
+    };
+
+    systemd.slices."system-gickup" = {
+      description = "Gickup git repository mirroring service";
+      after = [ "network.target" ];
+    };
+
+    systemd.targets.gickup = {
+      description = "Gickup git repository mirroring service";
+      wants = map ({ slug, ... }: "gickup@${slug}.service") (lib.attrValues cfg.instances);
+    };
+
+    systemd.timers = {
+      "gickup@" = {
+        description = "Gickup git repository mirroring service for %i";
+
+        timerConfig = {
+          OnCalendar = "daily";
+          RandomizedDelaySec = "1h";
+          Persistent = true;
+          AccuracySec = "1s";
+        };
+      };
+    }
+    //
+    # Overrides for mirrors which are not "daily"
+    (lib.pipe cfg.instances [
+      builtins.attrValues
+      (builtins.filter (instance: instance.interval != "daily"))
+      (map ({ slug, interval, ... }: {
+        name = "gickup@${slug}";
+        value = {
+          overrideStrategy = "asDropin";
+          timerConfig.OnCalendar = interval;
+        };
+      }))
+      builtins.listToAttrs
+    ]);
+
+    systemd.targets.timers.wants = map ({ slug, ... }: "gickup@${slug}.timer") (lib.attrValues cfg.instances);
+
+    systemd.services = {
+      "gickup@" = let
+        configDir = lib.pipe cfg.instances [
+          (lib.mapAttrsToList (name: instance: {
+            name = "${instance.slug}.yml";
+            path = format.generate "gickup-configuration-${name}.yml" {
+              destination.local = [ cfg.destinationSettings ];
+              source.${instance.type} = [
+                (
+                  (lib.optionalAttrs (instance.type != "any") {
+                    user = instance.owner;
+                    includeorgs = [ instance.owner ];
+                    include = [ instance.repo ];
+                  })
+                  //
+                  instance.settings
+                )
+              ];
+            };
+          }))
+          (pkgs.linkFarm "gickup-configuration-files")
+        ];
+      in {
+        description = "Gickup git repository mirroring service for %i";
+        after = [ "network.target" ];
+
+        path = [
+          cfg.gitPackage
+          cfg.gitLfsPackage
+        ];
+
+        restartIfChanged = false;
+
+        serviceConfig = {
+          Type = "oneshot";
+          ExecStart = "'${pkgs.gickup}/bin/gickup' '${configDir}/%i.yml'";
+          ExecStartPost = "";
+
+          User = "gickup";
+          Group = "gickup";
+
+          BindPaths = lib.optionals (cfg.dataDir != "/var/lib/gickup") [
+            "${cfg.dataDir}:/var/lib/gickup"
+          ];
+
+          Slice = "system-gickup.slice";
+
+          SyslogIdentifier = "gickup-%i";
+          StateDirectory = "gickup";
+          # WorkingDirectory = "gickup";
+          # RuntimeDirectory = "gickup";
+          # RuntimeDirectoryMode = "0700";
+
+          # https://discourse.nixos.org/t/how-to-prevent-custom-systemd-service-from-restarting-on-nixos-rebuild-switch/43431
+          RemainAfterExit = true;
+
+          # Hardening options
+          AmbientCapabilities = [];
+          LockPersonality = true;
+          NoNewPrivileges = true;
+          PrivateDevices = true;
+          PrivateMounts = true;
+          PrivateTmp = true;
+          PrivateUsers = true;
+          ProcSubset = "pid";
+          ProtectClock = true;
+          ProtectControlGroups = true;
+          ProtectHome = true;
+          ProtectHostname = true;
+          ProtectKernelLogs = true;
+          ProtectKernelModules = true;
+          ProtectKernelTunables = true;
+          # ProtectProc = "invisible";
+          # ProtectSystem = "strict";
+          RemoveIPC = true;
+          RestrictAddressFamilies = [
+            "AF_INET"
+            "AF_INET6"
+          ];
+          RestrictNamespaces = true;
+          RestrictRealtime = true;
+          RestrictSUIDSGID = true;
+          SystemCallArchitectures = "native";
+          # SystemCallFilter = [
+          #   "@system-service"
+          #   "~@resources"
+          #   "~@privileged"
+          # ];
+          UMask = "0002";
+          CapabilityBoundingSet = [];
+        };
+      };
+    };
+  };
+}

modules/gickup/hardlink-files.nix

@@ -0,0 +1,42 @@
+{ config, lib, pkgs, ... }:
+let
+  cfg = config.services.gickup;
+in
+{
+  config = lib.mkIf cfg.enable {
+    # TODO: add a service that will look at the backed up files and hardlink
+    #       the ones that have a matching hash together to save space. This can
+    #       either run routinely (i.e. trigger by systemd-timer), or be activated
+    #       whenever a gickup@<slug>.service finishes. The latter is probably better.
+
+    # systemd.services."gickup-hardlink" = {
+    #   serviceConfig = {
+    #     Type = "oneshot";
+    #     ExecStart = let
+    #       script = pkgs.writeShellApplication {
+    #         name = "gickup-hardlink-files.sh";
+    #         runtimeInputs = [ pkgs.coreutils pkgs.jdupes ];
+    #         text = ''
+
+    #         '';
+    #       };
+    #     in lib.getExe script;
+
+    #     User = "gickup";
+    #     Group = "gickup";
+
+    #     BindPaths = lib.optionals (cfg.dataDir != "/var/lib/gickup") [
+    #       "${cfg.dataDir}:/var/lib/gickup"
+    #     ];
+
+    #     Slice = "system-gickup.slice";
+
+    #     StateDirectory = "gickup";
+
+    #     # Hardening options
+    #     # TODO:
+    #     PrivateNetwork = true;
+    #   };
+    # };
+  };
+}

modules/gickup/import-from-toml.nix

@@ -0,0 +1,11 @@
+{ config, lib, pkgs, ... }:
+
+let
+  cfg = config.services.gickup;
+in
+{
+  config = lib.mkIf cfg.enable {
+    # TODO: import cfg.instances from a toml file to make it easier for non-nix users
+    #       to add repositories to mirror
+  };
+}

modules/gickup/set-description.nix

@@ -0,0 +1,9 @@
+{ config, lib, pkgs, ... }:
+let
+  cfg = config.services.gickup;
+in
+{
+  config = lib.mkIf cfg.enable {
+    # TODO: create .git/description files for each repo where cfg.instances.<instance>.description is set
+  };
+}

modules/gickup/update-linktree.nix

@@ -0,0 +1,76 @@
+{ config, lib, pkgs, ... }:
+let
+  cfg = config.services.gickup;
+in
+{
+  config = lib.mkIf cfg.enable {
+    # TODO: run upon completion of cloning a repository
+
+    # TODO: update symlink for one repo at a time (e.g. gickup-linktree@<instance>.service)
+    systemd.services."gickup-linktree" = {
+      serviceConfig = {
+        Type = "oneshot";
+        ExecStart = let
+          script = pkgs.writeShellApplication {
+            name = "gickup-update-symlink-tree.sh";
+            runtimeInputs = [
+              pkgs.coreutils
+              pkgs.findutils
+            ];
+            text = ''
+              shopt -s nullglob
+
+              for repository in ./*/*/*; do
+                REPOSITORY_RELATIVE_DIRS=''${repository#"./"}
+
+                echo "Checking $REPOSITORY_RELATIVE_DIRS"
+
+                declare -a REVISIONS
+                readarray -t REVISIONS < <(find "$repository" -mindepth 1 -maxdepth 1 -printf "%f\n" | sort --numeric-sort --reverse)
+
+                if [[ "''${#REVISIONS[@]}" == 0 ]]; then
+                  echo "Found no revisions for $repository, continuing"
+                  continue
+                fi
+
+                LAST_REVISION="''${REVISIONS[0]}"
+                SYMLINK_PATH="../linktree/''${REPOSITORY_RELATIVE_DIRS}"
+
+                mkdir -p "$(dirname "$SYMLINK_PATH")"
+
+                EXPECTED_SYMLINK_TARGET=$(realpath "''${repository}/''${LAST_REVISION}")
+                EXISTING_SYMLINK_TARGET=$(realpath "$SYMLINK_PATH" || echo "<none>")
+
+                if [[ "$EXISTING_SYMLINK_TARGET" != "$EXPECTED_SYMLINK_TARGET" ]]; then
+                  echo "Updating symlink for $REPOSITORY_RELATIVE_DIRS"
+                  rm "$SYMLINK_PATH" ||:
+                  ln -rs "$EXPECTED_SYMLINK_TARGET" "$SYMLINK_PATH"
+                else
+                  echo "Symlink already up to date, continuing..."
+                fi
+
+                echo "---"
+              done
+            '';
+          };
+        in lib.getExe script;
+
+        User = "gickup";
+        Group = "gickup";
+
+        BindPaths = lib.optionals (cfg.dataDir != "/var/lib/gickup") [
+          "${cfg.dataDir}:/var/lib/gickup"
+        ];
+
+        Slice = "system-gickup.slice";
+
+        StateDirectory = "gickup";
+        WorkingDirectory = "/var/lib/gickup/raw";
+
+        # Hardening options
+        # TODO:
+        PrivateNetwork = true;
+      };
+    };
+  };
+}

packages/cgit.nix

@@ -0,0 +1,21 @@
+{ cgit, fetchurl, ... }:
+let
+  pname = cgit.pname;
+  commit = "09d24d7cd0b7e85633f2f43808b12871bb209d69";
+in
+cgit.overrideAttrs (_: {
+  version = "1.2.3-unstable-2024.07.16";
+
+  src = fetchurl {
+    url = "https://git.zx2c4.com/cgit/snapshot/${pname}-${commit}.tar.xz";
+    hash = "sha256-gfgjAXnWRqVCP+4cmYOVdB/3OFOLJl2WBOc3bFVDsjw=";
+  };
+
+  # cgit is tightly coupled with git and needs a git source tree to build.
+  # IMPORTANT: Remember to check which git version cgit needs on every version
+  # bump (look for "GIT_VER" in the top-level Makefile).
+  gitSrc = fetchurl {
+    url = "mirror://kernel/software/scm/git/git-2.46.0.tar.xz";
+    hash = "sha256-fxI0YqKLfKPr4mB0hfcWhVTCsQ38FVx+xGMAZmrCf5U=";
+  };
+})

secrets/bicep/bicep.yaml

@@ -3,6 +3,8 @@ calendar-bot:
     mysql_password: ENC[AES256_GCM,data:Gqag8yOgPH3ntoT5TmaqJWv1j+si2qIyz5Ryfw5E2A==,iv:kQDcxnPfwJQcFovI4f87UDt18F8ah3z5xeY86KmdCyY=,tag:A1sCSNXJziAmtUWohqwJgg==,type:str]
 mysql:
     password: ENC[AES256_GCM,data:KqEe0TVdeMIzPKsmFg9x0X9xWijnOk306ycyXTm2Tpqo/O0F,iv:Y+hlQ8n1ZIP9ncXBzd2kCSs/DWVTWhiEluFVwZFKRCA=,tag:xlaUk0Wftk62LpYE5pKNQw==,type:str]
+gickup:
+    github-token: ENC[AES256_GCM,data:H/yBDLIvEXunmaUha3c2vUWKLRIbl9QrC0t13AQDRCTnrvhabeiUFLNxZ/F+4B6sZ2aPSgZoB69WwnHvh1wLdiFp1qLWKW/jQPvzZOxE4n+jXrnSOutUWktbPzVj,iv:KFW4jRru93JIl9doVFtcNkJDWp89NlzWjPDflHxcL/U=,tag:YtgyRxkoZO9MkuP3DJh7zA==,type:str]
 sops:
     kms: []
     gcp_kms: []
@@ -63,8 +65,8 @@ sops:
             cTh5bnJ3WW90aXRCSUp6NHFYeU1tZ0kK4afdtJwGNu6wLRI0fuu+mBVeqVeB0rgX
             0q5hwyzjiRnHnyjF38CmcGgydSfDRmF6P+WIMbCwXC6LwfRhAmBGPg==
             -----END AGE ENCRYPTED FILE-----
-    lastmodified: "2024-08-15T21:18:33Z"
+    lastmodified: "2025-05-07T21:34:48Z"
-    mac: ENC[AES256_GCM,data:uR5HgeDAYqoqB9kk1V6p0T30+v6WpQJi4+qIeCDRnoUPnQKUVR10hvBhICck+E+Uh8p+tGhM6Uf3YrAJAV0ZCUiNJjtwDJQQLUDT53vdOAXN4xADCQqNuhgVwVMaruoTheEiwOswRuhFeEwy0gBj3Ze2pu47lueHYclmEzumLeQ=,iv:t0UyXN2YaR2m7M/pV2wTLJG5wVfqTIUs7wSQMmyeTVw=,tag:O7dIffzrDAXz3kGx5uazhw==,type:str]
+    mac: ENC[AES256_GCM,data:n6GHD+nQmZL17WvUZiMCBLRHbtpoKU6U8o/Oraj0VSRi/pQ74QWGVEcIX87kFjBvR2C+UPd3KwXzjQHhjUfHpz9EjIGi6tXLTTo8K3ptd2wCL8MW418TVO4KV+BFmHGT4kwlbdoqaJ2SA7HcfXNaC68e/2CTXhtkLpIwGXtYWJA=,iv:iC5QX/JMwno4mBljPdorNmcQSD2wy/wOYvGrUoC2yzg=,tag:GuFNQ6+d6o9DYC6Do/IEqQ==,type:str]
     pgp:
         - created_at: "2024-08-04T00:03:40Z"
           enc: |-
@@ -87,4 +89,4 @@ sops:
             -----END PGP MESSAGE-----
           fp: F7D37890228A907440E1FD4846B9228E814A2AAC
     unencrypted_suffix: _unencrypted
-    version: 3.9.0
+    version: 3.9.4