Add exim monitoring to prometheus

.sops.yaml

@@ -3,6 +3,7 @@ keys:
   - &user_felixalb age1mrnldl334l2nszuta6ywvewng0fswv2dz9l5g4qcwe3nj4yxf92qjskdx6
   - &user_oysteikt F7D37890228A907440E1FD4846B9228E814A2AAC
   - &host_jokum age1n4vc3dhv8puqz6ntwrkkpdfj0q002hexqee48wzahll8cmce2ezssrq608
+  - &host_ildkule age1hn45n46ypyrvypv0mwfnpt9ddrlmw34dwlpf33n8v67jexr3lucq6ahc9x
 creation_rules:
   # Global secrets
   - path_regex: secrets/[^/]+\.yaml$
@@ -27,5 +28,6 @@ creation_rules:
     - age:
       - *user_felixalb
       - *user_danio
+      - *host_ildkule
       pgp:
-      - *user_oysteikt
+      - *user_oysteikt

README.MD

@@ -16,7 +16,7 @@ Det er sikkert lurt å lage en PR først om du ikke er vandt til nix enda.
 Innen 24h skal alle systemene hente ned den nye konfigurasjonen og deploye den.
 
 Du kan tvinge en maskin til å oppdatere seg før dette ved å kjøre:
-`nixos-rebuild switch --flake "git+https://git.pvv.ntnu.no/Drift/pvv-nixos-config.git"`
+`nixos-rebuild switch --update-input nixpkgs --update-input unstable --no-write-lock-file --flake git+https://git.pvv.ntnu.no/Drift/pvv-nixos-config.git --upgrade`
 
 som root på maskinen.
 

base.nix

@@ -36,24 +36,16 @@
 
   nix.settings.experimental-features = [ "nix-command" "flakes" ];
 
-  # This makes commandline tools like nix run nixpkgs#hello
+  /* This makes commandline tools like
-  # and nix-shell -p hello use the same channel the system was built with
+  ** nix run nixpkgs#hello
+  ** and nix-shell -p hello
+  ** use the same channel the system
+  ** was built with
+  */
   nix.registry = {
     nixpkgs.flake = inputs.nixpkgs;
   };
-  nix.nixPath = [
+  nix.nixPath = [ "nixpkgs=${inputs.nixpkgs}" ];
-    "nixpkgs=${inputs.nixpkgs}"
-    "nixpkgs-overlays=${./overlays-compat}/"
-  ];
-
-  # Allows access to nixpkgs-unstable via pkgs.unstable
-  nixpkgs.overlays = let
-    unstable-overlay = final: prev: {
-      unstable = inputs.unstable.legacyPackages.${prev.system};
-    };
-  in [
-    unstable-overlay
-  ];
 
   environment.systemPackages = with pkgs; [
     file

flake.lock

@@ -2,11 +2,11 @@
   "nodes": {
     "matrix-next": {
       "locked": {
-        "lastModified": 1671480255,
+        "lastModified": 1671009204,
-        "narHash": "sha256-06G6xYTFPVuvmN/k2QDeBk9XIp4LDxEKWRL3aLAFFNo=",
+        "narHash": "sha256-gqA9po/KmHyh44XYqv/LfFJ1+MGufhaaD6DhDqBeaF8=",
         "owner": "dali99",
         "repo": "nixos-matrix-modules",
-        "rev": "f42306e0a2df064f1beb6dfcc1776ea33e7ae9df",
+        "rev": "43dbc17526576cb8e0980cef51c48b6598f97550",
         "type": "github"
       },
       "original": {
@@ -18,11 +18,11 @@
     },
     "nixpkgs": {
       "locked": {
-        "lastModified": 1671445885,
+        "lastModified": 1670946965,
-        "narHash": "sha256-oDCTgQiqr3y62NrU+viFCYXKln24wgUdYaf4ynXRPgI=",
+        "narHash": "sha256-PDJfKgK/aSV3ISnD1TbKpLPW85LO/AQI73yQjbwribA=",
         "owner": "NixOS",
         "repo": "nixpkgs",
-        "rev": "d3cf188b498a7dc936de67254ba2fdafcf5a1368",
+        "rev": "265caf30fa0a5148395b62777389b57eb0a537fd",
         "type": "github"
       },
       "original": {
@@ -34,11 +34,11 @@
     },
     "nixpkgs-stable": {
       "locked": {
-        "lastModified": 1671459584,
+        "lastModified": 1670146390,
-        "narHash": "sha256-6wRK7xmeHfClJ0ICOkax1avLZVGTDqBodQlkl/opccY=",
+        "narHash": "sha256-XrEoDpuloRHHbUkbPnhF2bQ0uwHllXq3NHxtuVe/QK4=",
         "owner": "NixOS",
         "repo": "nixpkgs",
-        "rev": "87b58217c9a05edcf7630b9be32570f889217aef",
+        "rev": "86370507cb20c905800527539fc049a2bf09c667",
         "type": "github"
       },
       "original": {
@@ -64,11 +64,11 @@
         "nixpkgs-stable": "nixpkgs-stable"
       },
       "locked": {
-        "lastModified": 1671472949,
+        "lastModified": 1670149631,
-        "narHash": "sha256-9iHSGpljCX+RypahQssBXPwkru9onfKfceCTeVrMpH4=",
+        "narHash": "sha256-rwmtlxx45PvOeZNP51wql/cWjY3rqzIR3Oj2Y+V7jM0=",
         "owner": "Mic92",
         "repo": "sops-nix",
-        "rev": "32840f16ffa0856cdf9503a8658f2dd42bf70342",
+        "rev": "da98a111623101c64474a14983d83dad8f09f93d",
         "type": "github"
       },
       "original": {
@@ -79,11 +79,11 @@
     },
     "unstable": {
       "locked": {
-        "lastModified": 1671442489,
+        "lastModified": 1670918062,
-        "narHash": "sha256-pSCuSrG+XxWCs5IZ90eKIxDIZy4rM22YSFMRZ/fiixc=",
+        "narHash": "sha256-iOhkyBYUU9Jfkk0lvI4ahpjyrTsLXj9uyJWwmjKg+gg=",
         "owner": "NixOS",
         "repo": "nixpkgs",
-        "rev": "ff07b107adeda2164b29f8feb4a86ed012854dfb",
+        "rev": "84575b0bd882be979516f4fecfe4d7c8de8f6a92",
         "type": "github"
       },
       "original": {

flake.nix

@@ -22,9 +22,8 @@
     nixosConfigurations = {
       jokum = nixpkgs.lib.nixosSystem {
         system = "x86_64-linux";
-        specialArgs = { inherit inputs; };
+        specialArgs = { inherit unstable inputs; };
         modules = [
-          ./base.nix
           ./hosts/jokum/configuration.nix
           sops-nix.nixosModules.sops
 
@@ -33,9 +32,8 @@
       };
       ildkule = nixpkgs.lib.nixosSystem {
         system = "x86_64-linux";
-        specialArgs = { inherit inputs; };
+        specialArgs = { inherit unstable inputs; };
         modules = [
-          ./base.nix
           ./hosts/ildkule/configuration.nix
           sops-nix.nixosModules.sops
         ];

hosts/ildkule/configuration.nix

@@ -3,9 +3,11 @@
   imports = [
       # Include the results of the hardware scan.
       ./hardware-configuration.nix
-
       ../../base.nix
-      # Users can just import any configuration they want even for non-user things. Improve the users/default.nix to just load some specific attributes if this isn't wanted
+      ../../misc/metrics-exporters.nix
+
+      ./services/nginx
+      ./services/metrics
     ];
 
   sops.defaultSopsFile = ../../secrets/ildkule/ildkule.yaml;

hosts/ildkule/hardware-configuration.nix

@@ -0,0 +1,37 @@
+# Do not modify this file!  It was generated by ‘nixos-generate-config’
+# and may be overwritten by future invocations.  Please make changes
+# to /etc/nixos/configuration.nix instead.
+{ config, lib, pkgs, modulesPath, ... }:
+
+{
+  imports =
+    [ (modulesPath + "/profiles/qemu-guest.nix")
+    ];
+
+  boot.initrd.availableKernelModules = [ "ata_piix" "uhci_hcd" "virtio_pci" "sr_mod" "virtio_blk" ];
+  boot.initrd.kernelModules = [ ];
+  boot.kernelModules = [ ];
+  boot.extraModulePackages = [ ];
+
+  fileSystems."/" =
+    { device = "/dev/disk/by-uuid/afe70fe4-681a-4675-8cbd-e5d08cdcf5b5";
+      fsType = "ext4";
+    };
+
+  fileSystems."/boot" =
+    { device = "/dev/disk/by-uuid/B71A-E5CD";
+      fsType = "vfat";
+    };
+
+  swapDevices = [ ];
+
+  # Enables DHCP on each ethernet and wireless interface. In case of scripted networking
+  # (the default) this is the recommended approach. When using systemd-networkd it's
+  # still possible to use this option, but it's recommended to use it in conjunction
+  # with explicit per-interface declarations with `networking.interfaces.<interface>.useDHCP`.
+  networking.useDHCP = lib.mkDefault true;
+  # networking.interfaces.ens18.useDHCP = lib.mkDefault true;
+
+  nixpkgs.hostPlatform = lib.mkDefault "x86_64-linux";
+  hardware.cpu.intel.updateMicrocode = lib.mkDefault config.hardware.enableRedistributableFirmware;
+}

hosts/ildkule/services/metrics/default.nix

@@ -0,0 +1,9 @@
+{ config, pkgs, ... }:
+
+{
+  imports = [
+    ./prometheus.nix
+    ./grafana.nix
+    ./loki.nix
+  ];
+}

hosts/ildkule/services/metrics/grafana.nix

@@ -0,0 +1,60 @@
+{ config, pkgs, ... }:
+
+let
+  cfg = config.services.grafana;
+in {
+  services.grafana = {
+    enable = true;
+    settings.server = {
+      domain = "ildkule.pvv.ntnu.no";
+      http_port = 2342;
+      http_addr = "127.0.0.1";
+    };
+    provision = {
+      enable = true;
+      datasources.settings.datasources = [
+        {
+          name = "Ildkule Prometheus";
+          type = "prometheus";
+          url = ("http://${config.services.prometheus.listenAddress}:${toString config.services.prometheus.port}");
+         isDefault = true;
+        }
+        {
+          name = "Ildkule loki";
+          type = "loki";
+          url = ("http://${config.services.loki.configuration.server.http_listen_address}:${toString config.services.loki.configuration.server.http_listen_port}");
+        }
+      ];
+      dashboards.settings.providers = [
+        {
+          name = "Node Exporter Full";
+          type = "file";
+          url = "https://grafana.com/api/dashboards/1860/revisions/29/download";
+          options.path = dashboards/node-exporter-full.json;
+        }
+        {
+          name = "Matrix Synapse";
+          type = "file";
+          url = "https://raw.githubusercontent.com/matrix-org/synapse/develop/contrib/grafana/synapse.json";
+          options.path = dashboards/synapse.json;
+        }
+      ];
+
+    };
+  };
+
+  services.nginx.virtualHosts.${cfg.settings.server.domain} = {
+    enableACME = true;
+    forceSSL = true;
+    locations = {
+      "/" = {
+        proxyPass = "http://127.0.0.1:${toString cfg.settings.server.http_port}";
+        proxyWebsockets = true;
+        extraConfig = ''
+          proxy_buffers 8 1024k;
+          proxy_buffer_size 1024k;
+        '';
+      };
+    };
+  };
+}

hosts/ildkule/services/metrics/loki.nix

@@ -0,0 +1,86 @@
+{ config, pkgs, ... }:
+
+let
+  cfg = config.services.loki;
+in {
+  services.loki = {
+    enable = true;
+    configuration = {
+      auth_enabled = false;
+      server = {
+        http_listen_port = 3100;
+        http_listen_address = "0.0.0.0";
+        grpc_listen_port = 9096;
+      };
+
+      ingester = {
+        wal = {
+          enabled = true;
+          dir = "/var/lib/loki/wal";
+        };
+        lifecycler = {
+          address = "127.0.0.1";
+          ring = {
+            kvstore = {
+              store = "inmemory";
+            };
+            replication_factor = 1;
+          };
+          final_sleep = "0s";
+        };
+        chunk_idle_period = "1h";
+      };
+
+      schema_config = {
+        configs = [
+          {
+            from = "2022-12-01";
+            store = "boltdb-shipper";
+            object_store = "filesystem";
+            schema = "v11";
+            index = {
+              prefix = "index_";
+              period = "24h";
+            };
+          }
+        ];
+      };
+
+      storage_config = {
+        boltdb_shipper = {
+          active_index_directory = "/var/lib/loki/boltdb-shipper-index";
+          cache_location = "/var/lib/loki/boltdb-shipper-cache";
+          shared_store = "filesystem";
+          cache_ttl = "24h";
+        };
+        filesystem = {
+          directory = "/var/lib/loki/chunks";
+        };
+      };
+
+      limits_config = {
+        enforce_metric_name = false;
+        reject_old_samples = true;
+        reject_old_samples_max_age = "72h";
+      };
+
+      compactor = {
+        working_directory = "/var/lib/loki/compactor";
+        shared_store = "filesystem";
+      };
+
+      # ruler = {
+      #   storage = {
+      #     type = "local";
+      #     local = {
+      #       directory = "/var/lib/loki/rules";
+      #     };
+      #   };
+      #   rule_path = "/etc/loki/rules";
+      #   alertmanager_url = "http://localhost:9093";
+      # };
+    };
+  };
+
+  networking.firewall.allowedTCPPorts = [ cfg.configuration.server.http_listen_port ];
+}

hosts/ildkule/services/metrics/prometheus.nix

@@ -0,0 +1,76 @@
+{ config, pkgs, ... }:
+
+let
+  cfg = config.services.prometheus;
+in {
+  services.prometheus = {
+    enable = true;
+    listenAddress = "127.0.0.1";
+    port = 9001;
+
+    scrapeConfigs = [
+      {
+        job_name = "node";
+        static_configs = [
+          {
+            targets = [
+              "ildkule.pvv.ntnu.no:${toString cfg.exporters.node.port}"
+              "microbel.pvv.ntnu.no:9100"
+              "isvegg.pvv.ntnu.no:9100"
+              "knakelibrak.pvv.ntnu.no:9100"
+            ];
+          }
+        ];
+      }
+      {
+        job_name = "exim";
+        scrape_interval = "60s";
+        static_configs = [
+          {
+            targets = [
+              "microbel.pvv.ntnu.no:9636"
+            ];
+          }
+        ];
+      }
+      {
+        job_name = "synapse";
+        scrape_interval = "15s";
+        scheme = "https";
+        http_sd_configs = [
+          {
+            url = "https://matrix.pvv.ntnu.no/metrics/config.json";
+          }
+        ];
+        relabel_configs = [
+          {
+            source_labels = [ "__address__" ];
+            regex = "[^/]+(/.*)";
+            target_label = "__metrics_path__";
+          }
+          {
+            source_labels = [ "__address__" ];
+            regex = "([^/]+)/.*";
+            target_label = "instance";
+          }
+          {
+            source_labels = [ "__address__" ];
+            regex = "[^/]+\\/+[^/]+/(.*)/\\d+$";
+            target_label = "job";
+          }
+          {
+            source_labels = [ "__address__" ];
+            regex = "[^/]+\\/+[^/]+/.*/(\\d+)$";
+            target_label = "index";
+          }
+          {
+            source_labels = [ "__address__" ];
+            regex = "([^/]+)/.*";
+            target_label = "__address__";
+          }
+        ];
+      }
+    ];
+    ruleFiles = [ rules/synapse-v2.rules ];
+  };
+}

hosts/ildkule/services/metrics/rules/synapse-v2.rules

@@ -0,0 +1,74 @@
+groups:
+- name: synapse
+  rules:
+
+  ###
+  ### Prometheus Console Only
+  ### The following rules are only needed if you use the Prometheus Console
+  ### in contrib/prometheus/consoles/synapse.html
+  ###
+  - record: 'synapse_federation_client_sent'
+    labels:
+      type: "EDU"
+    expr: 'synapse_federation_client_sent_edus_total + 0'
+  - record: 'synapse_federation_client_sent'
+    labels:
+      type: "PDU"
+    expr: 'synapse_federation_client_sent_pdu_destinations_count_total + 0'
+  - record: 'synapse_federation_client_sent'
+    labels:
+      type: "Query"
+    expr: 'sum(synapse_federation_client_sent_queries) by (job)'
+
+  - record: 'synapse_federation_server_received'
+    labels:
+      type: "EDU"
+    expr: 'synapse_federation_server_received_edus_total + 0'
+  - record: 'synapse_federation_server_received'
+    labels:
+      type: "PDU"
+    expr: 'synapse_federation_server_received_pdus_total + 0'
+  - record: 'synapse_federation_server_received'
+    labels:
+      type: "Query"
+    expr: 'sum(synapse_federation_server_received_queries) by (job)'
+
+  - record: 'synapse_federation_transaction_queue_pending'
+    labels:
+      type: "EDU"
+    expr: 'synapse_federation_transaction_queue_pending_edus + 0'
+  - record: 'synapse_federation_transaction_queue_pending'
+    labels:
+      type: "PDU"
+    expr: 'synapse_federation_transaction_queue_pending_pdus + 0'
+  ###
+  ### End of 'Prometheus Console Only' rules block
+  ###
+
+
+  ###
+  ### Grafana Only
+  ### The following rules are only needed if you use the Grafana dashboard
+  ### in contrib/grafana/synapse.json
+  ###
+  - record: synapse_storage_events_persisted_by_source_type
+    expr: sum without(type, origin_type, origin_entity) (synapse_storage_events_persisted_events_sep_total{origin_type="remote"})
+    labels:
+      type: remote
+  - record: synapse_storage_events_persisted_by_source_type
+    expr: sum without(type, origin_type, origin_entity) (synapse_storage_events_persisted_events_sep_total{origin_entity="*client*",origin_type="local"})
+    labels:
+      type: local
+  - record: synapse_storage_events_persisted_by_source_type
+    expr: sum without(type, origin_type, origin_entity) (synapse_storage_events_persisted_events_sep_total{origin_entity!="*client*",origin_type="local"})
+    labels:
+      type: bridges
+
+  - record: synapse_storage_events_persisted_by_event_type
+    expr: sum without(origin_entity, origin_type) (synapse_storage_events_persisted_events_sep_total)
+
+  - record: synapse_storage_events_persisted_by_origin
+    expr: sum without(type) (synapse_storage_events_persisted_events_sep_total)
+  ###
+  ### End of 'Grafana Only' rules block
+  ###

hosts/jokum/configuration.nix

@@ -4,6 +4,7 @@
       # Include the results of the hardware scan.
       ./hardware-configuration.nix
 
+      ../../base.nix
       # Users can just import any configuration they want even for non-user things. Improve the users/default.nix to just load some specific attributes if this isn't wanted
 
       ../../misc/rust-motd.nix

misc/metrics-exporters.nix

@@ -0,0 +1,43 @@
+{ config, pkgs, ... }:
+
+{
+  services.prometheus.exporters.node = {
+    enable = true;
+    port = 9100;
+    enabledCollectors = [ "systemd" ];
+  };
+
+  services.promtail = {
+    enable = true;
+    configuration = {
+      server = {
+        http_listen_port = 28183;
+        grpc_listen_port = 0;
+      };
+      clients = [
+        {
+          url = "http://ildkule.pvv.ntnu.no:3100/loki/api/v1/push";
+        }
+      ];
+      scrape_configs = [
+        {
+          job_name = "systemd-journal";
+          journal = {
+            max_age = "12h";
+            labels = {
+              job = "systemd-journal";
+              host = config.networking.hostName;
+            };
+          };
+          relabel_configs = [
+            {
+              source_labels = [ "__journal__systemd_unit" ];
+              target_label = "unit";
+            }
+          ];
+        }
+      ];
+    };
+  };
+
+}

overlays-compat/overlays.nix

@@ -1,8 +0,0 @@
-self: super:
-with super.lib;
-let
-  # Load the system config and get the `nixpkgs.overlays` option
-  overlays = (import <nixpkgs/nixos> { }).config.nixpkgs.overlays;
-in
-  # Apply all overlays to the input of the current "main" overlay
-  foldl' (flip extends) (_: super) overlays self

users/felixalb.nix

@@ -4,5 +4,8 @@
     isNormalUser = true;
     extraGroups = [ "wheel" ]; # Enable ‘sudo’ for the user.
     shell = pkgs.zsh;
+    openssh.authorizedKeys.keys = [
+      "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIDKzPICGew7uN0cmvRmbwkwTCodTBUgEhkoftQnZuO4Q felixalbrigtsen@gmail.com"
+    ];
   };
 }