From 2236863f09d733de2793c747c9504519a5e04b84 Mon Sep 17 00:00:00 2001 From: Felix Albrigtsen Date: Mon, 19 Dec 2022 22:56:42 +0100 Subject: [PATCH] Add a new metrics stack --- hosts/ildkule/configuration.nix | 3 + hosts/ildkule/services/metrics/default.nix | 9 +++ hosts/ildkule/services/metrics/grafana.nix | 29 +++++++ hosts/ildkule/services/metrics/loki.nix | 79 +++++++++++++++++++ hosts/ildkule/services/metrics/prometheus.nix | 23 ++++++ misc/metrics-exporters.nix | 46 +++++++++++ 6 files changed, 189 insertions(+) create mode 100644 hosts/ildkule/services/metrics/default.nix create mode 100644 hosts/ildkule/services/metrics/grafana.nix create mode 100644 hosts/ildkule/services/metrics/loki.nix create mode 100644 hosts/ildkule/services/metrics/prometheus.nix create mode 100644 misc/metrics-exporters.nix diff --git a/hosts/ildkule/configuration.nix b/hosts/ildkule/configuration.nix index a4259c98..0febbae2 100644 --- a/hosts/ildkule/configuration.nix +++ b/hosts/ildkule/configuration.nix @@ -5,7 +5,10 @@ ./hardware-configuration.nix ../../base.nix + ../../misc/rust-motd.nix + ../../misc/metrics-exporters.nix # Users can just import any configuration they want even for non-user things. Improve the users/default.nix to just load some specific attributes if this isn't wanted + ./services/metrics ]; sops.defaultSopsFile = ../../secrets/ildkule/ildkule.yaml; diff --git a/hosts/ildkule/services/metrics/default.nix b/hosts/ildkule/services/metrics/default.nix new file mode 100644 index 00000000..3493e568 --- /dev/null +++ b/hosts/ildkule/services/metrics/default.nix @@ -0,0 +1,9 @@ +{ config, pkgs, ... }: + +{ + imports = [ + ./prometheus.nix + ./grafana.nix + ./loki.nix + ]; +} diff --git a/hosts/ildkule/services/metrics/grafana.nix b/hosts/ildkule/services/metrics/grafana.nix new file mode 100644 index 00000000..5adacc34 --- /dev/null +++ b/hosts/ildkule/services/metrics/grafana.nix @@ -0,0 +1,29 @@ +{ config, pkgs, ... }: + +{ + services.grafana = { + enable = true; + settings.server = { + domain = "ildkule.pvv.ntnu.no"; + http_port = 2342; + http_addr = "127.0.0.1"; + }; + }; + + services.nginx.virtualHosts.${config.services.grafana.domain} = { + locations = { + "/" = { + proxyPass = "http://${config.services.grafana.addr}:${toString config.services.grafana.port}"; + proxyWebsockets = true; + extraConfig = '' + proxy_buffers 8 1024k; + proxy_buffer_size 1024k; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + ''; + }; + }; + }; +} diff --git a/hosts/ildkule/services/metrics/loki.nix b/hosts/ildkule/services/metrics/loki.nix new file mode 100644 index 00000000..603b409e --- /dev/null +++ b/hosts/ildkule/services/metrics/loki.nix @@ -0,0 +1,79 @@ +{ config, pkgs, ... }: + +{ + services.loki = { + enable = true; + configuration = { + auth_enabled = false; + server = { + http_listen_port = 3100; + grpc_listen_port = 9096; + }; + + ingester = { + wal = { + enabled = true; + dir = "/var/lib/loki/wal"; + }; + lifecycler = { + ring = { + kvstore = { + store = "inmemory"; + }; + replication_factor = 1; + }; + final_sleep = "0s"; + }; + chunk_idle_period = "10m"; + }; + + schema_config = { + configs = [ + { + from = "2022-01-01"; + store = "boltdb"; + object_store = "filesystem"; + schema = "v11"; + index = { + prefix = "index_"; + period = "48h"; + }; + } + ]; + }; + + storage_config = { + boltdb_shipper = { + active_index_directory = "/var/lib/loki/index"; + shared_store = "filesystem"; + cache_ttl = "24h"; + }; + filesystem = { + directory = "/var/lib/loki/chunks"; + }; + }; + + limits_config = { + enforce_metric_name = false; + reject_old_samples = true; + reject_old_samples_max_age = "72h"; + }; + + compactor = { + working_directory = "/var/lib/loki/compactor"; + shared_store = "filesystem"; + }; + + # ruler = { + # storage = { + # type = "local"; + # local = { + # directory = "/var/lib/loki/rules"; + # }; + # }; + # rule_path = "/etc/loki/rules"; + # alertmanager_url = "http://localhost:9093"; + # }; + }; + }; +} diff --git a/hosts/ildkule/services/metrics/prometheus.nix b/hosts/ildkule/services/metrics/prometheus.nix new file mode 100644 index 00000000..375a3f36 --- /dev/null +++ b/hosts/ildkule/services/metrics/prometheus.nix @@ -0,0 +1,23 @@ +{ config, pkgs, ... }: + +{ + services.prometheus = { + enable = true; + port = 9001; + + scrapeConfigs = [ + { + job_name = "node"; + static_configs = [ + { + targets = [ + "ildkule.pvv.ntnu.no:${toString config.services.prometheus.exporters.node.port}" + "microbel.pvv.ntnu.no:9100" + "knakelibrak.pvv.ntnu.no:9100" + ]; + } + ]; + } + ]; + }; +} diff --git a/misc/metrics-exporters.nix b/misc/metrics-exporters.nix new file mode 100644 index 00000000..e3c4faec --- /dev/null +++ b/misc/metrics-exporters.nix @@ -0,0 +1,46 @@ +{ config, pkgs, ... }: + +{ + services.prometheus.exporters.node = { + enable = true; + port = 9100; + enabledCollectors = [ "systemd" ]; + }; + + services.promtail = { + enable = true; + configuration = { + server = { + http_listen_port = 28183; + grpc_listen_port = 0; + }; + positions = { + filename = "/var/lib/promtail/positions.yaml"; + }; + clients = [ + { + url = "http://ildkule.pvv.ntnu.no:3100/loki/api/v1/push"; + } + ]; + scrape_configs = [ + { + job_name = "systemd-journal"; + journal = { + max_age = "12h"; + labels = { + job = "systemd-journal"; + host = config.networking.hostName; + }; + }; + relabel_configs = [ + { + source_labels = [ "__journal__systemd_unit" ]; + target_label = "unit"; + } + ]; + } + ]; + }; + }; + +}