Drift/pvv-nixos-config
17
6
Fork 1
Code Issues Pull Requests 6 Actions Wiki Activity

export and scrape dibbler prometheus stats #143

Open
vegardbm wants to merge 1 commits from dibbler-prometheus into main
pull from: dibbler-prometheus
merge into: Drift:main
Conversation 6 Commits 1 Files Changed 4
+101 -3
4 changed files with 101 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files
hosts/ildkule/services/monitoring/prometheus/default.nix
+1
View File
@@ -8,6 +8,7 @@ in {
./matrix-synapse.nix
./mysqld.nix
./postgres.nix
./dibbler.nix
];
services.prometheus = {
hosts/ildkule/services/monitoring/prometheus/dibbler.nix
+96
View File
@@ -0,0 +1,96 @@
{
config,
lib,
pkgs,
...
}:
let
cfg = config.services.prometheus.exporters.sql;
configFile =
if cfg.configFile != null then
cfg.configFile
else
let
nameInline = lib.mapAttrsToList (k: v: v // { name = k; });
renameStartupSql = j: removeAttrs (j // { startup_sql = j.startupSql; }) [ "startupSql" ];
configuration = {
jobs = map renameStartupSql (
nameInline (lib.mapAttrs (k: v: (v // { queries = nameInline v.queries; })) cfg.configuration.jobs)
vegardbm marked this conversation as resolved Outdated
oysteikt commented 2026-06-07 10:55:51 +02:00
Outdated
Review
Copy Link
Copy Source

bro...

bro...
vegardbm commented 2026-06-07 10:57:42 +02:00
Outdated
Review
Copy Link
Copy Source

lol, lmao even

lol, lmao even
);
};
in
builtins.toFile "config.yaml" (builtins.toJSON configuration);
in
{
sops.secrets."config/postgresql_dibbler_password" = { };
services.prometheus.scrapeConfigs = [
{
job_name = "sql_exporter";
scrape_interval = "1m";
scheme = "http";
static_configs = [
{
targets = [ "localhost:9237" ];
}
];
}
];
services.prometheus.exporters.sql = {
enable = true;
configuration = {
jobs.dibbler = {
interval = "1m";
queries."daily_purchase_sum" = {
help = "Sum of purchases for the current day.";
labels = [ "thing" ];
values = [ "sum" ];
query = "SELECT SUM(price) FROM purchases GROUP BY DATE(time) ORDER BY DATE(time) DESC LIMIT 1";
vegardbm commented 2026-06-07 18:03:52 +02:00
Outdated
Review
Copy Link
Copy Source

This is likely inefficient.

This is likely inefficient.
oysteikt commented 2026-06-08 02:34:02 +02:00
Outdated
Review
Copy Link
Copy Source

If we put a database index on time it's probably gonna be less inefficient. How fast does it run with current data?

If we put a database index on `time` it's probably gonna be less inefficient. How fast does it run with current data?
vegardbm commented 2026-06-08 11:13:19 +02:00
Outdated
Review
Copy Link
Copy Source
pvv_vv=*# EXPLAIN ANALYZE SELECT SUM(price) FROM purchases GROUP BY DATE(time) ORDER BY DATE(time) DESC LIMIT 1;
                                                             QUERY PLAN                                                             
------------------------------------------------------------------------------------------------------------------------------------
 Limit  (cost=2359.35..2359.35 rows=1 width=12) (actual time=30.052..30.054 rows=1.00 loops=1)
   Buffers: shared hit=367
   ->  Sort  (cost=2359.35..2501.88 rows=57010 width=12) (actual time=30.051..30.052 rows=1.00 loops=1)
         Sort Key: (date("time")) DESC
         Sort Method: top-N heapsort  Memory: 25kB
         Buffers: shared hit=367
         ->  HashAggregate  (cost=1361.68..2074.30 rows=57010 width=12) (actual time=28.429..29.387 rows=5194.00 loops=1)
               Group Key: date("time")
               Batches: 1  Memory Usage: 1305kB
               Buffers: shared hit=364
               ->  Seq Scan on purchases  (cost=0.00..1076.62 rows=57010 width=8) (actual time=0.034..14.387 rows=57038.00 loops=1)
                     Buffers: shared hit=364
 Planning:
   Buffers: shared hit=51
 Planning Time: 0.349 ms
 Execution Time: 31.113 ms
(16 rows)
```text pvv_vv=*# EXPLAIN ANALYZE SELECT SUM(price) FROM purchases GROUP BY DATE(time) ORDER BY DATE(time) DESC LIMIT 1; QUERY PLAN ------------------------------------------------------------------------------------------------------------------------------------ Limit (cost=2359.35..2359.35 rows=1 width=12) (actual time=30.052..30.054 rows=1.00 loops=1) Buffers: shared hit=367 -> Sort (cost=2359.35..2501.88 rows=57010 width=12) (actual time=30.051..30.052 rows=1.00 loops=1) Sort Key: (date("time")) DESC Sort Method: top-N heapsort Memory: 25kB Buffers: shared hit=367 -> HashAggregate (cost=1361.68..2074.30 rows=57010 width=12) (actual time=28.429..29.387 rows=5194.00 loops=1) Group Key: date("time") Batches: 1 Memory Usage: 1305kB Buffers: shared hit=364 -> Seq Scan on purchases (cost=0.00..1076.62 rows=57010 width=8) (actual time=0.034..14.387 rows=57038.00 loops=1) Buffers: shared hit=364 Planning: Buffers: shared hit=51 Planning Time: 0.349 ms Execution Time: 31.113 ms (16 rows) ```
vegardbm commented 2026-06-08 11:16:28 +02:00
Outdated
Review
Copy Link
Copy Source

There is another one which is also pretty bad compared to the others:

pvv_vv=*# EXPLAIN ANALYZE SELECT SUM(price) FROM purchases;
                                                      QUERY PLAN                                                      
----------------------------------------------------------------------------------------------------------------------
 Aggregate  (cost=1076.62..1076.63 rows=1 width=8) (actual time=16.758..16.759 rows=1.00 loops=1)
   Buffers: shared hit=364
   ->  Seq Scan on purchases  (cost=0.00..934.10 rows=57010 width=4) (actual time=0.030..8.101 rows=57038.00 loops=1)
         Buffers: shared hit=364
 Planning Time: 0.042 ms
 Execution Time: 16.784 ms
(6 rows)
There is another one which is also pretty bad compared to the others: ``` pvv_vv=*# EXPLAIN ANALYZE SELECT SUM(price) FROM purchases; QUERY PLAN ---------------------------------------------------------------------------------------------------------------------- Aggregate (cost=1076.62..1076.63 rows=1 width=8) (actual time=16.758..16.759 rows=1.00 loops=1) Buffers: shared hit=364 -> Seq Scan on purchases (cost=0.00..934.10 rows=57010 width=4) (actual time=0.030..8.101 rows=57038.00 loops=1) Buffers: shared hit=364 Planning Time: 0.042 ms Execution Time: 16.784 ms (6 rows) ```
oysteikt commented 2026-06-08 14:01:41 +02:00
Outdated
Review
Copy Link
Copy Source

Yeah, but that one is kinda inherent. There's no way to sum all purchases without actually looping over all the purchases and summing them. You can speed either by creating a index on the expression (effectively moving the calculation to being done everytime someone makes a purchase) or by doing more fancy things to only sum news rows onto a previously cached sum.

I wouldn't really worry though, it's still just 16 milliseconds

Yeah, but that one is kinda inherent. There's no way to sum all purchases without actually looping over all the purchases and summing them. You can speed either by creating a index on the expression (effectively moving the calculation to being done everytime someone makes a purchase) or by doing more fancy things to only sum news rows onto a previously cached sum. I wouldn't really worry though, it's still just 16 milliseconds
oysteikt commented 2026-06-08 14:44:25 +02:00
Outdated
Review
Copy Link
Copy Source

Here's the relevant statements ftr:

CREATE INDEX "purchases_by_date" ON "purchases"(DATE("date"));
CREATE INDEX "purchases_price_sum" ON "purchases"(SUM("price"));
Here's the relevant statements ftr: ```sql CREATE INDEX "purchases_by_date" ON "purchases"(DATE("date")); CREATE INDEX "purchases_price_sum" ON "purchases"(SUM("price")); ```
};
queries."total_purchase_sum" = {
help = "Sum of all purchases.";
labels = [ "thing" ];
values = [ "sum" ];
query = "SELECT SUM(price) FROM purchases";
};
queries."total_stock_value" = {
help = "The value of all stock in dibbler.";
labels = [ "thing" ];
values = [ "sum" ];
query = "SELECT SUM(price * stock) FROM products";
};
queries."user_credit_sum" = {
help = "The sum of all user credit.";
labels = [ "thing" ];
values = [ "sum" ];
query = "SELECT SUM(credit) FROM users";
};
};
};
};
systemd.services."prometheus-sql-exporter".serviceConfig = {
RuntimeDirectory = "prometheus-sql-exporter";
LoadCredential = "postgresql_dibbler_password:${
config.sops.secrets."config/postgresql_dibbler_password".path
}";
ExecStartPre = ''
vegardbm marked this conversation as resolved Outdated
oysteikt commented 2026-06-08 03:23:11 +02:00
Outdated
Review
Copy Link
Copy Source

This one is kinda hard to read.

Modern systemd allows you to prepend a | to get rid of the /bin/sh invocation. jq also has --slurpfile and --rawfile to get rid of most of the file trickery. Also, newlines are allowed with a \ just like in ExecStart (alternatively you can construct the args with lib.cli.toCommandLineShellGNU)

ExecStartPre = ''
  |${lib.getExe pkgs.jq} \
    --null-input \
    --compact-output \
    --slurpfile config '${configFile}' \
    --rawfile pw '%d/postgresql_dibbler_password' \
    --from-file '${pkgs.writeText "prometheus-sql-exec-start-jq-filter" ''
      ("postgres://pvv_vv:\($pw | gsub("\n"; ""))@postgres.pvv.ntnu.no") as $pg_uri
      | $config[0]
      | .jobs[0].connections[0] = $pg_uri
    ''}' > /run/prometheus-sql-exporter/config.yaml
'';
This one is kinda hard to read. Modern systemd allows you to prepend a `|` to get rid of the `/bin/sh` invocation. `jq` also has `--slurpfile` and `--rawfile` to get rid of most of the file trickery. Also, newlines are allowed with a `\` just like in `ExecStart` (alternatively you can construct the args with `lib.cli.toCommandLineShellGNU`) ``` ExecStartPre = '' |${lib.getExe pkgs.jq} \ --null-input \ --compact-output \ --slurpfile config '${configFile}' \ --rawfile pw '%d/postgresql_dibbler_password' \ --from-file '${pkgs.writeText "prometheus-sql-exec-start-jq-filter" '' ("postgres://pvv_vv:\($pw | gsub("\n"; ""))@postgres.pvv.ntnu.no") as $pg_uri | $config[0] | .jobs[0].connections[0] = $pg_uri ''}' > /run/prometheus-sql-exporter/config.yaml ''; ```
vegardbm commented 2026-06-08 10:56:00 +02:00
Outdated
Review
Copy Link
Copy Source

What you provided works. Modified to use |, --slurpfile and --rawfile.

What you provided works. Modified to use |, --slurpfile and --rawfile.
|${lib.getExe pkgs.jq} \
--null-input \
--compact-output \
--slurpfile config '${configFile}' \
--rawfile pw '%d/postgresql_dibbler_password' \
--from-file '${pkgs.writeText "prometheus-sql-exec-start-jq-filter" ''
("postgres://pvv_vv:\($pw | gsub("\n"; ""))@postgres.pvv.ntnu.no") as $pg_uri
| $config[0]
| .jobs[0].connections[0] = $pg_uri
''}' > /run/prometheus-sql-exporter/config.yaml
'';
};
}
md
View File
secrets/ildkule/ildkule.yaml
+4 -3
View File
@@ -1,5 +1,6 @@
config:
mysqld_exporter_password: ENC[AES256_GCM,data:I9K+QMqaN3FOOVKzeOR9Q6UERStXX0P8WEHyN1jzzbM=,iv:UxvIdlfAyJvNuxPkU4+guKPa0fiD0vVLzHOTYktcmso=,tag:ltnIqEwESYx9HBu8UN0ZLw==,type:str]
postgresql_dibbler_password: ENC[AES256_GCM,data:wP4CVz9qRE3CJrblWWYqOIkcH3LM5H81,iv:j8zr1TRNlPPqIppYlWhDoKlL7m2Ph2wQlU6bvFj8R9A=,tag:Cfp8zbYkoLqI7DTDpOBlJw==,type:str]
vegardbm commented 2026-06-07 18:10:38 +02:00
Outdated
Review
Copy Link
Copy Source

This duplicates the dibbler postgresql database password.

This duplicates the dibbler postgresql database password.
oysteikt commented 2026-06-08 02:30:24 +02:00
Outdated
Review
Copy Link
Copy Source

Probably fine for now, we don't have very good order in our sops files. At some point we should do some spring cleaning and create shared files for shared passwords

Probably fine for now, we don't have very good order in our sops files. At some point we should do some spring cleaning and create shared files for shared passwords
keys:
grafana:
secret_key: ENC[AES256_GCM,data:+WoAJbDBEgKs0RoHT+7oEELAVQ+/2Xt+5RTMSXg23moCqVRx+Gzll9P5Drw=,iv:AkRn/Y20iEe5i1T+84wAgLCTFtAox2G3giyawAkltAw=,tag:BZbt5Wb5lYLIJBm/pfP4GQ==,type:str]
@@ -72,8 +73,8 @@ sops:
dC9meDZlc3d3aUJEVjc4REF0Y1BLcGcK79LbJzc5KVgEgyJR11crGuX8YcVoJBbT
Fin7Zoon06L7qx0Zw5u27wV7RKMnYT7hOMiWs6660ZTLcYJ5M1aEZQ==
-----END AGE ENCRYPTED FILE-----
lastmodified: "2025-03-16T20:08:18Z"
mac: ENC[AES256_GCM,data:C2tpWppc13jKJq5d4nmAKQOaNWHm27TKwxAxm1fi2lejN1lqUaoz5bHfTBA7MfaWvuP5uZnfbtG32eeu48mnlWpo58XRUFFecAhb9JUpW9s5IR3/nbzLNkGU7H5C0oWPrxI4thd+bAVduIgBjjFyGj1pe6J9db3c0yUWRwNlwGU=,iv:YpoQ4psiFYOWLGipxv1QvRvr034XFsyn2Bhyy39HmOo=,tag:ByiCWygFC/VokVTbdLoLgg==,type:str]
lastmodified: "2026-06-07T12:51:04Z"
mac: ENC[AES256_GCM,data:otGwzc3Bme1PGRU4zWRRf3kmAf5EDjT8sPkDK/zDrO0ve+d3x99Qls4DBZV8G6FsFtUXbeKVo70I5VpqUgaIef+nyMl6zzWkW1wpbKIBYjh5fkaP0xxhLnw+shrB088PAXT0wkP7hJBLO0ZJgTrpprJKhmeqVj2HEXBSJ1wSjk8=,iv:cjqtdbWBVLUfpQdykb3vKDKa/VC/kGBybwYtG/eStqc=,tag:remOj4bX/H/LlAPgcXHAdA==,type:str]
pgp:
- created_at: "2026-05-20T17:35:58Z"
enc: |-
@@ -96,4 +97,4 @@ sops:
-----END PGP MESSAGE-----
fp: F7D37890228A907440E1FD4846B9228E814A2AAC
unencrypted_suffix: _unencrypted
version: 3.9.4
version: 3.12.2