ildkule: restructure prometheus config

hosts/ildkule/services/metrics/prometheus/default.nix

@@ -0,0 +1,14 @@
+{ config, ... }: {
+  imports = [
+    ./node.nix
+    ./matrix-synapse.nix
+  ];
+  
+  services.prometheus = {
+    enable = true;
+    listenAddress = "127.0.0.1";
+    port = 9001;
+
+    ruleFiles = [ rules/synapse-v2.rules ];
+  };
+}

hosts/ildkule/services/metrics/prometheus/matrix-synapse.nix

@@ -0,0 +1,40 @@
+{ ... }:
+{
+  services.prometheus.scrapeConfigs = [{
+    job_name = "synapse";
+    scrape_interval = "15s";
+    scheme = "https";
+
+    http_sd_configs = [{
+      url = "https://matrix.pvv.ntnu.no/metrics/config.json";
+    }];
+
+    relabel_configs = [
+      {
+        source_labels = [ "__address__" ];
+        regex = "[^/]+(/.*)";
+        target_label = "__metrics_path__";
+      }
+      {
+        source_labels = [ "__address__" ];
+        regex = "([^/]+)/.*";
+        target_label = "instance";
+      }
+      {
+        source_labels = [ "__address__" ];
+        regex = "[^/]+\\/+[^/]+/(.*)/\\d+$";
+        target_label = "job";
+      }
+      {
+        source_labels = [ "__address__" ];
+        regex = "[^/]+\\/+[^/]+/.*/(\\d+)$";
+        target_label = "index";
+      }
+      {
+        source_labels = [ "__address__" ];
+        regex = "([^/]+)/.*";
+        target_label = "__address__";
+      }
+    ];
+  }];
+}

hosts/ildkule/services/metrics/prometheus/node.nix

@@ -0,0 +1,20 @@
+{ config, ... }: let
+  cfg = config.services.prometheus;
+in {
+  services.prometheus.scrapeConfigs = [{
+    job_name = "node";
+    static_configs = [
+      {
+        targets = [
+          "ildkule.pvv.ntnu.no:${toString cfg.exporters.node.port}"
+          "microbel.pvv.ntnu.no:9100"
+          "isvegg.pvv.ntnu.no:9100"
+          "knakelibrak.pvv.ntnu.no:9100"
+          "hildring.pvv.ntnu.no:9100"
+          "bicep.pvv.ntnu.no:9100"
+          "jokum.pvv.ntnu.no:9100"
+        ];
+      }
+    ];
+  }];
+}

hosts/ildkule/services/metrics/prometheus/rules/synapse-v2.rules

@@ -0,0 +1,74 @@
+groups:
+- name: synapse
+  rules:
+
+  ###
+  ### Prometheus Console Only
+  ### The following rules are only needed if you use the Prometheus Console
+  ### in contrib/prometheus/consoles/synapse.html
+  ###
+  - record: 'synapse_federation_client_sent'
+    labels:
+      type: "EDU"
+    expr: 'synapse_federation_client_sent_edus_total + 0'
+  - record: 'synapse_federation_client_sent'
+    labels:
+      type: "PDU"
+    expr: 'synapse_federation_client_sent_pdu_destinations_count_total + 0'
+  - record: 'synapse_federation_client_sent'
+    labels:
+      type: "Query"
+    expr: 'sum(synapse_federation_client_sent_queries) by (job)'
+
+  - record: 'synapse_federation_server_received'
+    labels:
+      type: "EDU"
+    expr: 'synapse_federation_server_received_edus_total + 0'
+  - record: 'synapse_federation_server_received'
+    labels:
+      type: "PDU"
+    expr: 'synapse_federation_server_received_pdus_total + 0'
+  - record: 'synapse_federation_server_received'
+    labels:
+      type: "Query"
+    expr: 'sum(synapse_federation_server_received_queries) by (job)'
+
+  - record: 'synapse_federation_transaction_queue_pending'
+    labels:
+      type: "EDU"
+    expr: 'synapse_federation_transaction_queue_pending_edus + 0'
+  - record: 'synapse_federation_transaction_queue_pending'
+    labels:
+      type: "PDU"
+    expr: 'synapse_federation_transaction_queue_pending_pdus + 0'
+  ###
+  ### End of 'Prometheus Console Only' rules block
+  ###
+
+
+  ###
+  ### Grafana Only
+  ### The following rules are only needed if you use the Grafana dashboard
+  ### in contrib/grafana/synapse.json
+  ###
+  - record: synapse_storage_events_persisted_by_source_type
+    expr: sum without(type, origin_type, origin_entity) (synapse_storage_events_persisted_events_sep_total{origin_type="remote"})
+    labels:
+      type: remote
+  - record: synapse_storage_events_persisted_by_source_type
+    expr: sum without(type, origin_type, origin_entity) (synapse_storage_events_persisted_events_sep_total{origin_entity="*client*",origin_type="local"})
+    labels:
+      type: local
+  - record: synapse_storage_events_persisted_by_source_type
+    expr: sum without(type, origin_type, origin_entity) (synapse_storage_events_persisted_events_sep_total{origin_entity!="*client*",origin_type="local"})
+    labels:
+      type: bridges
+
+  - record: synapse_storage_events_persisted_by_event_type
+    expr: sum without(origin_entity, origin_type) (synapse_storage_events_persisted_events_sep_total)
+
+  - record: synapse_storage_events_persisted_by_origin
+    expr: sum without(type) (synapse_storage_events_persisted_events_sep_total)
+  ###
+  ### End of 'Grafana Only' rules block
+  ###