From 5184c6993a4a9b70f096f63830f0af43be1fcac6 Mon Sep 17 00:00:00 2001
From: Peder Bergebakken Sundt <pbsds@hotmail.com>
Date: Thu, 27 Feb 2025 16:16:46 +0100
Subject: [PATCH] I'm a silly goose

---
 .gitignore |  3 +++
 README.md  |  9 ++++---
 do.sh      | 78 +++++++++++++++++++++++++++++++++++++++++++++++-------
 3 files changed, 76 insertions(+), 14 deletions(-)
 create mode 100644 .gitignore

diff --git a/.gitignore b/.gitignore
new file mode 100644
index 0000000..b7ca877
--- /dev/null
+++ b/.gitignore
@@ -0,0 +1,3 @@
+.direnv/
+data/
+html/
diff --git a/README.md b/README.md
index cefb3fe..e02fcdf 100644
--- a/README.md
+++ b/README.md
@@ -1,6 +1,7 @@
-annoying show-stoppers:
 
-https://github.com/numtide/nar-serve/issues/4
+* greps for manpages with nix-locate
+* fetches the nar archives non-recursively
+* extracts manpages
+* converts to html
 
-
-narinfos and nar archives are not that hard to parse though, i could just implement it myself
+This involved implementing a multiprocessing limit and lru file cache using bash...
diff --git a/do.sh b/do.sh
index bf228ea..14b730c 100644
--- a/do.sh
+++ b/do.sh
@@ -1,24 +1,81 @@
 #!/usr/bin/env nix-shell
-#!nix-shell -i bash -p nar-serve wget nix-locate man xz gzip bzip2
+#!nix-shell -i bash -p lix wget nix-locate man xz gzip bzip2
 set -euo pipefail
 
 N_WORKERS=4
+LRU_MAX_SIZE=50
 
 # TODO: allow to pick which nix-index to fetch and use
 
+
 # https://github.com/numtide/nar-serve
-HTTP_ADDR=localhost:8383 nar-serve &
-NAR_SERVE_PID=$!
-trap "set -x; kill $NAR_SERVE_PID" EXIT SIGHUP SIGINT SIGQUIT
-let N_WORKERS+=1
+#HTTP_ADDR=localhost:8383 nar-serve &
+#NAR_SERVE_PID=$!
+#trap "set -x; kill $NAR_SERVE_PID" EXIT SIGHUP SIGINT SIGQUIT
+#let N_WORKERS+=1
+
+nix_substituters=($(nix config show substituters))
+nar-fetch() {
+    local storepath="$1"
+    local outpath="$2"
+
+    # /nix/store/xxx-foobar/* -> xxx
+    local storehash="$(cut -c12-43 <<<"$storepath")"
+
+    if [[ -s "$outpath" ]]; then
+        return 0
+
+    elif [[ -e "$storepath" ]]; then
+        cp -v "$storepath" "$outpath"
+
+    elif [[ -e .direnv/nar-cache/"$storehash".nar ]]; then
+        mkdir -p "$(dirname "$outpath")"
+        nix nar cat .direnv/nar-cache/"$storehash.nar" "/$(cut <<<"$storepath" -d/ -f5-)" >"$outpath"
+        return 0
+
+    else
+        mkdir -p .direnv/nar-cache
+
+        local prev=
+        for substituter in "${nix_substituters[@]}"; do
+            [[ "$substituter" != "$prev" ]] || continue
+            prev="$substituter"
+
+            local location="$((set -x; curl -L "$substituter/$storehash.narinfo") | grep '^URL: ' | cut -d' ' -f2-)" || true
+            [[ -n "$location" ]] || continue
+
+            # lru the cache
+            find .direnv/nar-cache/ -maxdepth 1 -printf "%T+ %p\n" | sort | cut -d' ' -f2- | tail "-n+$LRU_MAX_SIZE" | xargs -d$'\n' rm -v
+
+            if [[ "$location" =~ .*\.nar\.xz$ ]]; then
+                (set -x; curl -L "$substituter/$location") | xz --decompress --stdout > .direnv/nar-cache/"$storehash.nar"
+            elif [[ "$location" =~ .*\.nar$ ]]; then
+                (set -x; curl -L "$substituter/$location") > .direnv/nar-cache/"$storehash.nar"
+            else
+                return 1
+            fi
+
+            mkdir -p "$(dirname "$outpath")"
+            nix nar cat .direnv/nar-cache/"$storehash.nar" "/$(cut <<<"$storepath" -d/ -f5-)" >"$outpath"
+            return 0
+        done
+
+    fi
+}
+
 
 while read attrpath size type storepath _; do
+    if ! [[ "$storepath" =~ /nix/store/.* ]]; then
+        continue
+    fi
     outpath="data/$attrpath/$(cut -d/ -f7- <<<"$storepath")"
-    test -s "$outpath" || (
-        mkdir -vp "$(dirname "$outpath")"
-        set -x
-        wget http://localhost:8383"$storepath" -O "$outpath" || true
-    )
+
+    #test -s "$outpath" || (
+    #    mkdir -vp "$(dirname "$outpath")"
+    #    set -x
+    #    wget http://localhost:8383"$storepath" -O "$outpath" || true
+    #)
+    nar-fetch "$storepath" "$outpath" || continue
 
     if [[ ! -s "$outpath" ]]; then
         continue
@@ -49,6 +106,7 @@ while read attrpath size type storepath _; do
     ) &
 
     while [[ $(jobs -p | wc -l) -ge $N_WORKERS ]]; do
+        >&2 echo "Waiting..."
         wait -n < <(jobs -p) || true
     done