Add a new metrics stack

hosts/ildkule/configuration.nix

@@ -5,7 +5,10 @@
       ./hardware-configuration.nix
 
       ../../base.nix
+      ../../misc/rust-motd.nix
+      ../../misc/metrics-exporters.nix
       # Users can just import any configuration they want even for non-user things. Improve the users/default.nix to just load some specific attributes if this isn't wanted
+      ./services/metrics
     ];
 
   sops.defaultSopsFile = ../../secrets/ildkule/ildkule.yaml;

hosts/ildkule/services/metrics/default.nix

@@ -0,0 +1,9 @@
+{ config, pkgs, ... }:
+
+{
+  imports = [
+    ./prometheus.nix
+    ./grafana.nix
+    ./loki.nix
+  ];
+}

hosts/ildkule/services/metrics/grafana.nix

@@ -0,0 +1,29 @@
+{ config, pkgs, ... }:
+
+{
+  services.grafana = {
+    enable = true;
+    settings.server = {
+      domain = "ildkule.pvv.ntnu.no";
+      http_port = 2342;
+      http_addr = "127.0.0.1";
+    };
+  };
+
+  services.nginx.virtualHosts.${config.services.grafana.domain} = {
+    locations = {
+      "/" = {
+        proxyPass = "http://${config.services.grafana.addr}:${toString config.services.grafana.port}";
+        proxyWebsockets = true;
+        extraConfig = ''
+          proxy_buffers 8 1024k;
+          proxy_buffer_size 1024k;
+          proxy_set_header Host $host;
+          proxy_set_header X-Real-IP $remote_addr;
+          proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+          proxy_set_header X-Forwarded-Proto $scheme;
+        '';
+      };
+    };
+  };
+}

hosts/ildkule/services/metrics/loki.nix

@@ -0,0 +1,79 @@
+{ config, pkgs, ... }:
+
+{
+  services.loki = {
+    enable = true;
+    configuration = {
+      auth_enabled = false;
+      server = {
+        http_listen_port = 3100;
+        grpc_listen_port = 9096;
+      };
+
+      ingester = {
+        wal = {
+          enabled = true;
+          dir = "/var/lib/loki/wal";
+        };
+        lifecycler = {
+          ring = {
+            kvstore = {
+              store = "inmemory";
+            };
+            replication_factor = 1;
+          };
+          final_sleep = "0s";
+        };
+        chunk_idle_period = "10m";
+      };
+
+      schema_config = {
+        configs = [
+          {
+            from = "2022-01-01";
+            store = "boltdb";
+            object_store = "filesystem";
+            schema = "v11";
+            index = {
+              prefix = "index_";
+              period = "48h";
+            };
+          }
+        ];
+      };
+
+      storage_config = {
+        boltdb_shipper = {
+          active_index_directory = "/var/lib/loki/index";
+          shared_store = "filesystem";
+          cache_ttl = "24h";
+        };
+        filesystem = {
+          directory = "/var/lib/loki/chunks";
+        };
+      };
+
+      limits_config = {
+        enforce_metric_name = false;
+        reject_old_samples = true;
+        reject_old_samples_max_age = "72h";
+      };
+
+      compactor = {
+        working_directory = "/var/lib/loki/compactor";
+        shared_store = "filesystem";
+      };
+
+      # ruler = {
+      #   storage = {
+      #     type = "local";
+      #     local = {
+      #       directory = "/var/lib/loki/rules";
+      #     };
+      #   };
+      #   rule_path = "/etc/loki/rules";
+      #   alertmanager_url = "http://localhost:9093";
+      # };
+    };
+  };
+}

hosts/ildkule/services/metrics/prometheus.nix

@@ -0,0 +1,23 @@
+{ config, pkgs, ... }:
+
+{
+  services.prometheus = {
+    enable = true;
+    port = 9001;
+
+    scrapeConfigs = [
+      {
+        job_name = "node";
+        static_configs = [
+          {
+            targets = [
+              "ildkule.pvv.ntnu.no:${toString config.services.prometheus.exporters.node.port}"
+              "microbel.pvv.ntnu.no:9100"
+              "knakelibrak.pvv.ntnu.no:9100"
+            ];
+          }
+        ];
+      }
+    ];
+  };
+}

misc/metrics-exporters.nix

@@ -0,0 +1,46 @@
+{ config, pkgs, ... }:
+
+{
+  services.prometheus.exporters.node = {
+    enable = true;
+    port = 9100;
+    enabledCollectors = [ "systemd" ];
+  };
+
+  services.promtail = {
+    enable = true;
+    configuration = {
+      server = {
+        http_listen_port = 28183;
+        grpc_listen_port = 0;
+      };
+      positions = {
+        filename = "/var/lib/promtail/positions.yaml";
+      };
+      clients = [
+        {
+          url = "http://ildkule.pvv.ntnu.no:3100/loki/api/v1/push";
+        }
+      ];
+      scrape_configs = [
+        {
+          job_name = "systemd-journal";
+          journal = {
+            max_age = "12h";
+            labels = {
+              job = "systemd-journal";
+              host = config.networking.hostName;
+            };
+          };
+          relabel_configs = [
+            {
+              source_labels = [ "__journal__systemd_unit" ];
+              target_label = "unit";
+            }
+          ];
+        }
+      ];
+    };
+  };
+
+}