Merge branch 'prometheus-stack' of Drift/pvv-nixos-config into main

.sops.yaml

@@ -3,6 +3,7 @@ keys:
   - &user_felixalb age1mrnldl334l2nszuta6ywvewng0fswv2dz9l5g4qcwe3nj4yxf92qjskdx6
   - &user_oysteikt F7D37890228A907440E1FD4846B9228E814A2AAC
   - &host_jokum age1n4vc3dhv8puqz6ntwrkkpdfj0q002hexqee48wzahll8cmce2ezssrq608
+  - &host_ildkule age1hn45n46ypyrvypv0mwfnpt9ddrlmw34dwlpf33n8v67jexr3lucq6ahc9x
 creation_rules:
   # Global secrets
   - path_regex: secrets/[^/]+\.yaml$
@@ -27,5 +28,6 @@ creation_rules:
     - age:
       - *user_felixalb
       - *user_danio
+      - *host_ildkule
       pgp:
       - *user_oysteikt

hosts/ildkule/configuration.nix

@@ -3,9 +3,11 @@
   imports = [
       # Include the results of the hardware scan.
       ./hardware-configuration.nix
-
       ../../base.nix
-      # Users can just import any configuration they want even for non-user things. Improve the users/default.nix to just load some specific attributes if this isn't wanted
+      ../../misc/metrics-exporters.nix
+
+      ./services/nginx
+      ./services/metrics
     ];
 
   sops.defaultSopsFile = ../../secrets/ildkule/ildkule.yaml;

hosts/ildkule/hardware-configuration.nix

@@ -0,0 +1,37 @@
+# Do not modify this file!  It was generated by ‘nixos-generate-config’
+# and may be overwritten by future invocations.  Please make changes
+# to /etc/nixos/configuration.nix instead.
+{ config, lib, pkgs, modulesPath, ... }:
+
+{
+  imports =
+    [ (modulesPath + "/profiles/qemu-guest.nix")
+    ];
+
+  boot.initrd.availableKernelModules = [ "ata_piix" "uhci_hcd" "virtio_pci" "sr_mod" "virtio_blk" ];
+  boot.initrd.kernelModules = [ ];
+  boot.kernelModules = [ ];
+  boot.extraModulePackages = [ ];
+
+  fileSystems."/" =
+    { device = "/dev/disk/by-uuid/afe70fe4-681a-4675-8cbd-e5d08cdcf5b5";
+      fsType = "ext4";
+    };
+
+  fileSystems."/boot" =
+    { device = "/dev/disk/by-uuid/B71A-E5CD";
+      fsType = "vfat";
+    };
+
+  swapDevices = [ ];
+
+  # Enables DHCP on each ethernet and wireless interface. In case of scripted networking
+  # (the default) this is the recommended approach. When using systemd-networkd it's
+  # still possible to use this option, but it's recommended to use it in conjunction
+  # with explicit per-interface declarations with `networking.interfaces.<interface>.useDHCP`.
+  networking.useDHCP = lib.mkDefault true;
+  # networking.interfaces.ens18.useDHCP = lib.mkDefault true;
+
+  nixpkgs.hostPlatform = lib.mkDefault "x86_64-linux";
+  hardware.cpu.intel.updateMicrocode = lib.mkDefault config.hardware.enableRedistributableFirmware;
+}

hosts/ildkule/services/metrics/default.nix

@@ -0,0 +1,9 @@
+{ config, pkgs, ... }:
+
+{
+  imports = [
+    ./prometheus.nix
+    ./grafana.nix
+    ./loki.nix
+  ];
+}

hosts/ildkule/services/metrics/grafana.nix

@@ -0,0 +1,60 @@
+{ config, pkgs, ... }:
+
+let
+  cfg = config.services.grafana;
+in {
+  services.grafana = {
+    enable = true;
+    settings.server = {
+      domain = "ildkule.pvv.ntnu.no";
+      http_port = 2342;
+      http_addr = "127.0.0.1";
+    };
+    provision = {
+      enable = true;
+      datasources.settings.datasources = [
+        {
+          name = "Ildkule Prometheus";
+          type = "prometheus";
+          url = ("http://${config.services.prometheus.listenAddress}:${toString config.services.prometheus.port}");
+         isDefault = true;
+        }
+        {
+          name = "Ildkule loki";
+          type = "loki";
+          url = ("http://${config.services.loki.configuration.server.http_listen_address}:${toString config.services.loki.configuration.server.http_listen_port}");
+        }
+      ];
+      dashboards.settings.providers = [
+        {
+          name = "Node Exporter Full";
+          type = "file";
+          url = "https://grafana.com/api/dashboards/1860/revisions/29/download";
+          options.path = dashboards/node-exporter-full.json;
+        }
+        {
+          name = "Matrix Synapse";
+          type = "file";
+          url = "https://raw.githubusercontent.com/matrix-org/synapse/develop/contrib/grafana/synapse.json";
+          options.path = dashboards/synapse.json;
+        }
+      ];
+
+    };
+  };
+
+  services.nginx.virtualHosts.${cfg.settings.server.domain} = {
+    enableACME = true;
+    forceSSL = true;
+    locations = {
+      "/" = {
+        proxyPass = "http://127.0.0.1:${toString cfg.settings.server.http_port}";
+        proxyWebsockets = true;
+        extraConfig = ''
+          proxy_buffers 8 1024k;
+          proxy_buffer_size 1024k;
+        '';
+      };
+    };
+  };
+}

hosts/ildkule/services/metrics/loki.nix

@@ -0,0 +1,86 @@
+{ config, pkgs, ... }:
+
+let
+  cfg = config.services.loki;
+in {
+  services.loki = {
+    enable = true;
+    configuration = {
+      auth_enabled = false;
+      server = {
+        http_listen_port = 3100;
+        http_listen_address = "0.0.0.0";
+        grpc_listen_port = 9096;
+      };
+
+      ingester = {
+        wal = {
+          enabled = true;
+          dir = "/var/lib/loki/wal";
+        };
+        lifecycler = {
+          address = "127.0.0.1";
+          ring = {
+            kvstore = {
+              store = "inmemory";
+            };
+            replication_factor = 1;
+          };
+          final_sleep = "0s";
+        };
+        chunk_idle_period = "1h";
+      };
+
+      schema_config = {
+        configs = [
+          {
+            from = "2022-12-01";
+            store = "boltdb-shipper";
+            object_store = "filesystem";
+            schema = "v11";
+            index = {
+              prefix = "index_";
+              period = "24h";
+            };
+          }
+        ];
+      };
+
+      storage_config = {
+        boltdb_shipper = {
+          active_index_directory = "/var/lib/loki/boltdb-shipper-index";
+          cache_location = "/var/lib/loki/boltdb-shipper-cache";
+          shared_store = "filesystem";
+          cache_ttl = "24h";
+        };
+        filesystem = {
+          directory = "/var/lib/loki/chunks";
+        };
+      };
+
+      limits_config = {
+        enforce_metric_name = false;
+        reject_old_samples = true;
+        reject_old_samples_max_age = "72h";
+      };
+
+      compactor = {
+        working_directory = "/var/lib/loki/compactor";
+        shared_store = "filesystem";
+      };
+
+      # ruler = {
+      #   storage = {
+      #     type = "local";
+      #     local = {
+      #       directory = "/var/lib/loki/rules";
+      #     };
+      #   };
+      #   rule_path = "/etc/loki/rules";
+      #   alertmanager_url = "http://localhost:9093";
+      # };
+    };
+  };
+
+  networking.firewall.allowedTCPPorts = [ cfg.configuration.server.http_listen_port ];
+}

hosts/ildkule/services/metrics/prometheus.nix

@@ -0,0 +1,65 @@
+{ config, pkgs, ... }:
+
+let
+  cfg = config.services.prometheus;
+in {
+  services.prometheus = {
+    enable = true;
+    listenAddress = "127.0.0.1";
+    port = 9001;
+
+    scrapeConfigs = [
+      {
+        job_name = "node";
+        static_configs = [
+          {
+            targets = [
+              "ildkule.pvv.ntnu.no:${toString cfg.exporters.node.port}"
+              "microbel.pvv.ntnu.no:9100"
+              "isvegg.pvv.ntnu.no:9100"
+              "knakelibrak.pvv.ntnu.no:9100"
+            ];
+          }
+        ];
+      }
+      {
+        job_name = "synapse";
+        scrape_interval = "15s";
+        scheme = "https";
+        http_sd_configs = [
+          {
+            url = "https://matrix.pvv.ntnu.no/metrics/config.json";
+          }
+        ];
+        relabel_configs = [
+          {
+            source_labels = [ "__address__" ];
+            regex = "[^/]+(/.*)";
+            target_label = "__metrics_path__";
+          }
+          {
+            source_labels = [ "__address__" ];
+            regex = "([^/]+)/.*";
+            target_label = "instance";
+          }
+          {
+            source_labels = [ "__address__" ];
+            regex = "[^/]+\\/+[^/]+/(.*)/\\d+$";
+            target_label = "job";
+          }
+          {
+            source_labels = [ "__address__" ];
+            regex = "[^/]+\\/+[^/]+/.*/(\\d+)$";
+            target_label = "index";
+          }
+          {
+            source_labels = [ "__address__" ];
+            regex = "([^/]+)/.*";
+            target_label = "__address__";
+          }
+        ];
+      }
+    ];
+    ruleFiles = [ rules/synapse-v2.rules ];
+  };
+}

hosts/ildkule/services/metrics/rules/synapse-v2.rules

@@ -0,0 +1,74 @@
+groups:
+- name: synapse
+  rules:
+
+  ###
+  ### Prometheus Console Only
+  ### The following rules are only needed if you use the Prometheus Console
+  ### in contrib/prometheus/consoles/synapse.html
+  ###
+  - record: 'synapse_federation_client_sent'
+    labels:
+      type: "EDU"
+    expr: 'synapse_federation_client_sent_edus_total + 0'
+  - record: 'synapse_federation_client_sent'
+    labels:
+      type: "PDU"
+    expr: 'synapse_federation_client_sent_pdu_destinations_count_total + 0'
+  - record: 'synapse_federation_client_sent'
+    labels:
+      type: "Query"
+    expr: 'sum(synapse_federation_client_sent_queries) by (job)'
+
+  - record: 'synapse_federation_server_received'
+    labels:
+      type: "EDU"
+    expr: 'synapse_federation_server_received_edus_total + 0'
+  - record: 'synapse_federation_server_received'
+    labels:
+      type: "PDU"
+    expr: 'synapse_federation_server_received_pdus_total + 0'
+  - record: 'synapse_federation_server_received'
+    labels:
+      type: "Query"
+    expr: 'sum(synapse_federation_server_received_queries) by (job)'
+
+  - record: 'synapse_federation_transaction_queue_pending'
+    labels:
+      type: "EDU"
+    expr: 'synapse_federation_transaction_queue_pending_edus + 0'
+  - record: 'synapse_federation_transaction_queue_pending'
+    labels:
+      type: "PDU"
+    expr: 'synapse_federation_transaction_queue_pending_pdus + 0'
+  ###
+  ### End of 'Prometheus Console Only' rules block
+  ###
+
+
+  ###
+  ### Grafana Only
+  ### The following rules are only needed if you use the Grafana dashboard
+  ### in contrib/grafana/synapse.json
+  ###
+  - record: synapse_storage_events_persisted_by_source_type
+    expr: sum without(type, origin_type, origin_entity) (synapse_storage_events_persisted_events_sep_total{origin_type="remote"})
+    labels:
+      type: remote
+  - record: synapse_storage_events_persisted_by_source_type
+    expr: sum without(type, origin_type, origin_entity) (synapse_storage_events_persisted_events_sep_total{origin_entity="*client*",origin_type="local"})
+    labels:
+      type: local
+  - record: synapse_storage_events_persisted_by_source_type
+    expr: sum without(type, origin_type, origin_entity) (synapse_storage_events_persisted_events_sep_total{origin_entity!="*client*",origin_type="local"})
+    labels:
+      type: bridges
+
+  - record: synapse_storage_events_persisted_by_event_type
+    expr: sum without(origin_entity, origin_type) (synapse_storage_events_persisted_events_sep_total)
+
+  - record: synapse_storage_events_persisted_by_origin
+    expr: sum without(type) (synapse_storage_events_persisted_events_sep_total)
+  ###
+  ### End of 'Grafana Only' rules block
+  ###

misc/metrics-exporters.nix

@@ -0,0 +1,43 @@
+{ config, pkgs, ... }:
+
+{
+  services.prometheus.exporters.node = {
+    enable = true;
+    port = 9100;
+    enabledCollectors = [ "systemd" ];
+  };
+
+  services.promtail = {
+    enable = true;
+    configuration = {
+      server = {
+        http_listen_port = 28183;
+        grpc_listen_port = 0;
+      };
+      clients = [
+        {
+          url = "http://ildkule.pvv.ntnu.no:3100/loki/api/v1/push";
+        }
+      ];
+      scrape_configs = [
+        {
+          job_name = "systemd-journal";
+          journal = {
+            max_age = "12h";
+            labels = {
+              job = "systemd-journal";
+              host = config.networking.hostName;
+            };
+          };
+          relabel_configs = [
+            {
+              source_labels = [ "__journal__systemd_unit" ];
+              target_label = "unit";
+            }
+          ];
+        }
+      ];
+    };
+  };
+
+}

users/felixalb.nix

@@ -4,5 +4,8 @@
     isNormalUser = true;
     extraGroups = [ "wheel" ]; # Enable ‘sudo’ for the user.
     shell = pkgs.zsh;
+    openssh.authorizedKeys.keys = [
+      "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIDKzPICGew7uN0cmvRmbwkwTCodTBUgEhkoftQnZuO4Q felixalbrigtsen@gmail.com"
+    ];
   };
 }