Merge branch 'prometheus-stack' of Drift/pvv-nixos-config into main

This commit is contained in:
Felix Albrigtsen 2023-01-14 22:49:09 +01:00 committed by Gogs
commit 43c76502a7
12 changed files with 36336 additions and 3 deletions

View File

@ -3,6 +3,7 @@ keys:
- &user_felixalb age1mrnldl334l2nszuta6ywvewng0fswv2dz9l5g4qcwe3nj4yxf92qjskdx6 - &user_felixalb age1mrnldl334l2nszuta6ywvewng0fswv2dz9l5g4qcwe3nj4yxf92qjskdx6
- &user_oysteikt F7D37890228A907440E1FD4846B9228E814A2AAC - &user_oysteikt F7D37890228A907440E1FD4846B9228E814A2AAC
- &host_jokum age1n4vc3dhv8puqz6ntwrkkpdfj0q002hexqee48wzahll8cmce2ezssrq608 - &host_jokum age1n4vc3dhv8puqz6ntwrkkpdfj0q002hexqee48wzahll8cmce2ezssrq608
- &host_ildkule age1hn45n46ypyrvypv0mwfnpt9ddrlmw34dwlpf33n8v67jexr3lucq6ahc9x
creation_rules: creation_rules:
# Global secrets # Global secrets
- path_regex: secrets/[^/]+\.yaml$ - path_regex: secrets/[^/]+\.yaml$
@ -27,5 +28,6 @@ creation_rules:
- age: - age:
- *user_felixalb - *user_felixalb
- *user_danio - *user_danio
- *host_ildkule
pgp: pgp:
- *user_oysteikt - *user_oysteikt

View File

@ -3,9 +3,11 @@
imports = [ imports = [
# Include the results of the hardware scan. # Include the results of the hardware scan.
./hardware-configuration.nix ./hardware-configuration.nix
../../base.nix ../../base.nix
# Users can just import any configuration they want even for non-user things. Improve the users/default.nix to just load some specific attributes if this isn't wanted ../../misc/metrics-exporters.nix
./services/nginx
./services/metrics
]; ];
sops.defaultSopsFile = ../../secrets/ildkule/ildkule.yaml; sops.defaultSopsFile = ../../secrets/ildkule/ildkule.yaml;

View File

@ -0,0 +1,37 @@
# Do not modify this file! It was generated by nixos-generate-config
# and may be overwritten by future invocations. Please make changes
# to /etc/nixos/configuration.nix instead.
{ config, lib, pkgs, modulesPath, ... }:
{
imports =
[ (modulesPath + "/profiles/qemu-guest.nix")
];
boot.initrd.availableKernelModules = [ "ata_piix" "uhci_hcd" "virtio_pci" "sr_mod" "virtio_blk" ];
boot.initrd.kernelModules = [ ];
boot.kernelModules = [ ];
boot.extraModulePackages = [ ];
fileSystems."/" =
{ device = "/dev/disk/by-uuid/afe70fe4-681a-4675-8cbd-e5d08cdcf5b5";
fsType = "ext4";
};
fileSystems."/boot" =
{ device = "/dev/disk/by-uuid/B71A-E5CD";
fsType = "vfat";
};
swapDevices = [ ];
# Enables DHCP on each ethernet and wireless interface. In case of scripted networking
# (the default) this is the recommended approach. When using systemd-networkd it's
# still possible to use this option, but it's recommended to use it in conjunction
# with explicit per-interface declarations with `networking.interfaces.<interface>.useDHCP`.
networking.useDHCP = lib.mkDefault true;
# networking.interfaces.ens18.useDHCP = lib.mkDefault true;
nixpkgs.hostPlatform = lib.mkDefault "x86_64-linux";
hardware.cpu.intel.updateMicrocode = lib.mkDefault config.hardware.enableRedistributableFirmware;
}

File diff suppressed because it is too large Load Diff

File diff suppressed because it is too large Load Diff

View File

@ -0,0 +1,9 @@
{ config, pkgs, ... }:
{
imports = [
./prometheus.nix
./grafana.nix
./loki.nix
];
}

View File

@ -0,0 +1,60 @@
{ config, pkgs, ... }:
let
cfg = config.services.grafana;
in {
services.grafana = {
enable = true;
settings.server = {
domain = "ildkule.pvv.ntnu.no";
http_port = 2342;
http_addr = "127.0.0.1";
};
provision = {
enable = true;
datasources.settings.datasources = [
{
name = "Ildkule Prometheus";
type = "prometheus";
url = ("http://${config.services.prometheus.listenAddress}:${toString config.services.prometheus.port}");
isDefault = true;
}
{
name = "Ildkule loki";
type = "loki";
url = ("http://${config.services.loki.configuration.server.http_listen_address}:${toString config.services.loki.configuration.server.http_listen_port}");
}
];
dashboards.settings.providers = [
{
name = "Node Exporter Full";
type = "file";
url = "https://grafana.com/api/dashboards/1860/revisions/29/download";
options.path = dashboards/node-exporter-full.json;
}
{
name = "Matrix Synapse";
type = "file";
url = "https://raw.githubusercontent.com/matrix-org/synapse/develop/contrib/grafana/synapse.json";
options.path = dashboards/synapse.json;
}
];
};
};
services.nginx.virtualHosts.${cfg.settings.server.domain} = {
enableACME = true;
forceSSL = true;
locations = {
"/" = {
proxyPass = "http://127.0.0.1:${toString cfg.settings.server.http_port}";
proxyWebsockets = true;
extraConfig = ''
proxy_buffers 8 1024k;
proxy_buffer_size 1024k;
'';
};
};
};
}

View File

@ -0,0 +1,86 @@
{ config, pkgs, ... }:
let
cfg = config.services.loki;
in {
services.loki = {
enable = true;
configuration = {
auth_enabled = false;
server = {
http_listen_port = 3100;
http_listen_address = "0.0.0.0";
grpc_listen_port = 9096;
};
ingester = {
wal = {
enabled = true;
dir = "/var/lib/loki/wal";
};
lifecycler = {
address = "127.0.0.1";
ring = {
kvstore = {
store = "inmemory";
};
replication_factor = 1;
};
final_sleep = "0s";
};
chunk_idle_period = "1h";
};
schema_config = {
configs = [
{
from = "2022-12-01";
store = "boltdb-shipper";
object_store = "filesystem";
schema = "v11";
index = {
prefix = "index_";
period = "24h";
};
}
];
};
storage_config = {
boltdb_shipper = {
active_index_directory = "/var/lib/loki/boltdb-shipper-index";
cache_location = "/var/lib/loki/boltdb-shipper-cache";
shared_store = "filesystem";
cache_ttl = "24h";
};
filesystem = {
directory = "/var/lib/loki/chunks";
};
};
limits_config = {
enforce_metric_name = false;
reject_old_samples = true;
reject_old_samples_max_age = "72h";
};
compactor = {
working_directory = "/var/lib/loki/compactor";
shared_store = "filesystem";
};
# ruler = {
# storage = {
# type = "local";
# local = {
# directory = "/var/lib/loki/rules";
# };
# };
# rule_path = "/etc/loki/rules";
# alertmanager_url = "http://localhost:9093";
# };
};
};
networking.firewall.allowedTCPPorts = [ cfg.configuration.server.http_listen_port ];
}

View File

@ -0,0 +1,65 @@
{ config, pkgs, ... }:
let
cfg = config.services.prometheus;
in {
services.prometheus = {
enable = true;
listenAddress = "127.0.0.1";
port = 9001;
scrapeConfigs = [
{
job_name = "node";
static_configs = [
{
targets = [
"ildkule.pvv.ntnu.no:${toString cfg.exporters.node.port}"
"microbel.pvv.ntnu.no:9100"
"isvegg.pvv.ntnu.no:9100"
"knakelibrak.pvv.ntnu.no:9100"
];
}
];
}
{
job_name = "synapse";
scrape_interval = "15s";
scheme = "https";
http_sd_configs = [
{
url = "https://matrix.pvv.ntnu.no/metrics/config.json";
}
];
relabel_configs = [
{
source_labels = [ "__address__" ];
regex = "[^/]+(/.*)";
target_label = "__metrics_path__";
}
{
source_labels = [ "__address__" ];
regex = "([^/]+)/.*";
target_label = "instance";
}
{
source_labels = [ "__address__" ];
regex = "[^/]+\\/+[^/]+/(.*)/\\d+$";
target_label = "job";
}
{
source_labels = [ "__address__" ];
regex = "[^/]+\\/+[^/]+/.*/(\\d+)$";
target_label = "index";
}
{
source_labels = [ "__address__" ];
regex = "([^/]+)/.*";
target_label = "__address__";
}
];
}
];
ruleFiles = [ rules/synapse-v2.rules ];
};
}

View File

@ -0,0 +1,74 @@
groups:
- name: synapse
rules:
###
### Prometheus Console Only
### The following rules are only needed if you use the Prometheus Console
### in contrib/prometheus/consoles/synapse.html
###
- record: 'synapse_federation_client_sent'
labels:
type: "EDU"
expr: 'synapse_federation_client_sent_edus_total + 0'
- record: 'synapse_federation_client_sent'
labels:
type: "PDU"
expr: 'synapse_federation_client_sent_pdu_destinations_count_total + 0'
- record: 'synapse_federation_client_sent'
labels:
type: "Query"
expr: 'sum(synapse_federation_client_sent_queries) by (job)'
- record: 'synapse_federation_server_received'
labels:
type: "EDU"
expr: 'synapse_federation_server_received_edus_total + 0'
- record: 'synapse_federation_server_received'
labels:
type: "PDU"
expr: 'synapse_federation_server_received_pdus_total + 0'
- record: 'synapse_federation_server_received'
labels:
type: "Query"
expr: 'sum(synapse_federation_server_received_queries) by (job)'
- record: 'synapse_federation_transaction_queue_pending'
labels:
type: "EDU"
expr: 'synapse_federation_transaction_queue_pending_edus + 0'
- record: 'synapse_federation_transaction_queue_pending'
labels:
type: "PDU"
expr: 'synapse_federation_transaction_queue_pending_pdus + 0'
###
### End of 'Prometheus Console Only' rules block
###
###
### Grafana Only
### The following rules are only needed if you use the Grafana dashboard
### in contrib/grafana/synapse.json
###
- record: synapse_storage_events_persisted_by_source_type
expr: sum without(type, origin_type, origin_entity) (synapse_storage_events_persisted_events_sep_total{origin_type="remote"})
labels:
type: remote
- record: synapse_storage_events_persisted_by_source_type
expr: sum without(type, origin_type, origin_entity) (synapse_storage_events_persisted_events_sep_total{origin_entity="*client*",origin_type="local"})
labels:
type: local
- record: synapse_storage_events_persisted_by_source_type
expr: sum without(type, origin_type, origin_entity) (synapse_storage_events_persisted_events_sep_total{origin_entity!="*client*",origin_type="local"})
labels:
type: bridges
- record: synapse_storage_events_persisted_by_event_type
expr: sum without(origin_entity, origin_type) (synapse_storage_events_persisted_events_sep_total)
- record: synapse_storage_events_persisted_by_origin
expr: sum without(type) (synapse_storage_events_persisted_events_sep_total)
###
### End of 'Grafana Only' rules block
###

View File

@ -0,0 +1,43 @@
{ config, pkgs, ... }:
{
services.prometheus.exporters.node = {
enable = true;
port = 9100;
enabledCollectors = [ "systemd" ];
};
services.promtail = {
enable = true;
configuration = {
server = {
http_listen_port = 28183;
grpc_listen_port = 0;
};
clients = [
{
url = "http://ildkule.pvv.ntnu.no:3100/loki/api/v1/push";
}
];
scrape_configs = [
{
job_name = "systemd-journal";
journal = {
max_age = "12h";
labels = {
job = "systemd-journal";
host = config.networking.hostName;
};
};
relabel_configs = [
{
source_labels = [ "__journal__systemd_unit" ];
target_label = "unit";
}
];
}
];
};
};
}

View File

@ -4,5 +4,8 @@
isNormalUser = true; isNormalUser = true;
extraGroups = [ "wheel" ]; # Enable sudo for the user. extraGroups = [ "wheel" ]; # Enable sudo for the user.
shell = pkgs.zsh; shell = pkgs.zsh;
openssh.authorizedKeys.keys = [
"ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIDKzPICGew7uN0cmvRmbwkwTCodTBUgEhkoftQnZuO4Q felixalbrigtsen@gmail.com"
];
}; };
} }