1
0
Fork 0

Add a new metrics stack

This commit is contained in:
Felix Albrigtsen 2022-12-19 22:56:42 +01:00
parent 50133047e6
commit 2236863f09
6 changed files with 189 additions and 0 deletions

View File

@ -5,7 +5,10 @@
./hardware-configuration.nix
../../base.nix
../../misc/rust-motd.nix
../../misc/metrics-exporters.nix
# Users can just import any configuration they want even for non-user things. Improve the users/default.nix to just load some specific attributes if this isn't wanted
./services/metrics
];
sops.defaultSopsFile = ../../secrets/ildkule/ildkule.yaml;

View File

@ -0,0 +1,9 @@
{ config, pkgs, ... }:
{
imports = [
./prometheus.nix
./grafana.nix
./loki.nix
];
}

View File

@ -0,0 +1,29 @@
{ config, pkgs, ... }:
{
services.grafana = {
enable = true;
settings.server = {
domain = "ildkule.pvv.ntnu.no";
http_port = 2342;
http_addr = "127.0.0.1";
};
};
services.nginx.virtualHosts.${config.services.grafana.domain} = {
locations = {
"/" = {
proxyPass = "http://${config.services.grafana.addr}:${toString config.services.grafana.port}";
proxyWebsockets = true;
extraConfig = ''
proxy_buffers 8 1024k;
proxy_buffer_size 1024k;
proxy_set_header Host $host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Proto $scheme;
'';
};
};
};
}

View File

@ -0,0 +1,79 @@
{ config, pkgs, ... }:
{
services.loki = {
enable = true;
configuration = {
auth_enabled = false;
server = {
http_listen_port = 3100;
grpc_listen_port = 9096;
};
ingester = {
wal = {
enabled = true;
dir = "/var/lib/loki/wal";
};
lifecycler = {
ring = {
kvstore = {
store = "inmemory";
};
replication_factor = 1;
};
final_sleep = "0s";
};
chunk_idle_period = "10m";
};
schema_config = {
configs = [
{
from = "2022-01-01";
store = "boltdb";
object_store = "filesystem";
schema = "v11";
index = {
prefix = "index_";
period = "48h";
};
}
];
};
storage_config = {
boltdb_shipper = {
active_index_directory = "/var/lib/loki/index";
shared_store = "filesystem";
cache_ttl = "24h";
};
filesystem = {
directory = "/var/lib/loki/chunks";
};
};
limits_config = {
enforce_metric_name = false;
reject_old_samples = true;
reject_old_samples_max_age = "72h";
};
compactor = {
working_directory = "/var/lib/loki/compactor";
shared_store = "filesystem";
};
# ruler = {
# storage = {
# type = "local";
# local = {
# directory = "/var/lib/loki/rules";
# };
# };
# rule_path = "/etc/loki/rules";
# alertmanager_url = "http://localhost:9093";
# };
};
};
}

View File

@ -0,0 +1,23 @@
{ config, pkgs, ... }:
{
services.prometheus = {
enable = true;
port = 9001;
scrapeConfigs = [
{
job_name = "node";
static_configs = [
{
targets = [
"ildkule.pvv.ntnu.no:${toString config.services.prometheus.exporters.node.port}"
"microbel.pvv.ntnu.no:9100"
"knakelibrak.pvv.ntnu.no:9100"
];
}
];
}
];
};
}

View File

@ -0,0 +1,46 @@
{ config, pkgs, ... }:
{
services.prometheus.exporters.node = {
enable = true;
port = 9100;
enabledCollectors = [ "systemd" ];
};
services.promtail = {
enable = true;
configuration = {
server = {
http_listen_port = 28183;
grpc_listen_port = 0;
};
positions = {
filename = "/var/lib/promtail/positions.yaml";
};
clients = [
{
url = "http://ildkule.pvv.ntnu.no:3100/loki/api/v1/push";
}
];
scrape_configs = [
{
job_name = "systemd-journal";
journal = {
max_age = "12h";
labels = {
job = "systemd-journal";
host = config.networking.hostName;
};
};
relabel_configs = [
{
source_labels = [ "__journal__systemd_unit" ];
target_label = "unit";
}
];
}
];
};
};
}