From 8b1d7a5ab585d433c5b9dd827e7f150df3aacb68 Mon Sep 17 00:00:00 2001 From: Peder Bergebakken Sundt Date: Sat, 3 May 2025 15:03:33 +0200 Subject: [PATCH] lkjdsalkjdsalkj --- justfile | 185 +++++++++++++++++++++++++++++++++++++++++++------------ 1 file changed, 145 insertions(+), 40 deletions(-) diff --git a/justfile b/justfile index dd02214..faca3fd 100644 --- a/justfile +++ b/justfile @@ -6,6 +6,7 @@ set positional-arguments := true export invokedir := invocation_directory() export epoch := `date +%s` export GUM_FILTER_HEIGHT := "15" +export FZF_DEFAULT_OPTS := "--height 15 --cycle --bind 'ctrl-a:toggle-all' " + env('FZF_DEFAULT_OPTS', "") export GIT := `command -v git-wait >/dev/null && echo "git-wait" || echo "git"` # will trigger a direnv reload when we enter $SHELL export DIRENV_WATCHES := "" @@ -146,41 +147,85 @@ __list_packages_attrpath_filtered +attrpaths: # +# _JUST_NIX_INSTANTIATE_ARGS="--system darwin" # TODO: remove, use for unfree or cross intead [no-cd] instantiate-packages +$attrpaths="": #!/usr/bin/env -S bash -euo pipefail - # slower, but resitant to bad attrpaths test -n "${attrpaths:-}" || set -- $(just _some_packages) - if [[ -d "${_INSTANTIATE_CACHE:-}" ]]; then - mkdir -p "$_INSTANTIATE_CACHE"/{roots,paths} + attrpaths=("$@") + + readarray -td $'\n' systems < <( just _some_systems ) + for system in "${systems[@]}"; do + just _instantiate_packages "$system" "${attrpaths[@]}" + done + +[no-cd] +instantiate-packages-extra-suffixes +$attrpaths="": + #!/usr/bin/env -S bash -euo pipefail + test -n "${attrpaths:-}" || set -- $(just _some_packages) + attrpaths=("$@") + [[ "${#attrpaths[@]}" -ne 0 ]] + suffixes=( + "%s" + "%s.src" + "%s.patches" + "%s.passthru.tests" + "%s.cargoDeps" + ) + readarray -td $'\n' selected_suffixes < <( + # gum choose --no-limit "${suffixes[@]}" + printf "%s\n" "${suffixes[@]}" | fzf --layout=reverse --multi + ) + [[ "${#selected_suffixes[@]}" -ne 0 ]] + readarray -td $'\n' selected_attrpaths < <( + for suffix in "${selected_suffixes[@]}"; do + printf "$suffix\n" "${attrpaths[@]}" + done + ) + just instantiate-packages "${selected_attrpaths[@]}" + +[no-cd] +_instantiate_packages $system +$attrpaths: + #!/usr/bin/env -S bash -euo pipefail + # slower than normal nix-instantiate, but handles bad attrpaths, TODO: parallel eval? + # supports envvar _JUST_NIX_INSTANTIATE_CACHE + # supports envvar _JUST_NIX_INSTANTIATE_ARGS + declare -a attrpaths=("${@:2}") + if [[ -d "${_JUST_NIX_INSTANTIATE_CACHE:-}" ]]; then + mkdir -p "$_JUST_NIX_INSTANTIATE_CACHE"/{roots,paths} worker() { local attrpath="$1" - if [[ -s "$_INSTANTIATE_CACHE/paths/$attrpath" ]]; then - cat "$_INSTANTIATE_CACHE/paths/$attrpath" + if [[ -s "$_JUST_NIX_INSTANTIATE_CACHE/paths/$attrpath.drv" ]]; then + cat "$_JUST_NIX_INSTANTIATE_CACHE/paths/$attrpath.drv" else NIXPKGS_ALLOW_UNFREE=1 nix-instantiate . -A "$attrpath" \ + --system "$system" \ ${_JUST_NIX_INSTANTIATE_ARGS:-} \ - --add-root "$_INSTANTIATE_CACHE/roots/$attrpath" \ - | tee "$_INSTANTIATE_CACHE/paths/$attrpath" + --add-root "$_JUST_NIX_INSTANTIATE_CACHE/roots/$attrpath.drv" \ + | tee "$_JUST_NIX_INSTANTIATE_CACHE/paths/$attrpath.drv" fi } - printf "%s\n" "$@" | grep . | \ + printf "%s\n" "${attrpaths[@]}" | grep . | \ xe -j0 -s "$(declare -f worker); worker \"\$@\"" else - printf "%s\n" "$@" | grep . | \ - xe -j0 -s 'NIXPKGS_ALLOW_UNFREE=1 nix-instantiate . -A "$1" '"${_JUST_NIX_INSTANTIATE_ARGS:-}"'' + printf "%s\n" "${attrpaths[@]}" | grep . | \ + xe -j0 -s 'NIXPKGS_ALLOW_UNFREE=1 nix-instantiate . -A "$1" --system "$system" '"${_JUST_NIX_INSTANTIATE_ARGS:-}"'' fi # _JUST_NIX_BUILD_ARGS='--check' # _JUST_NIX_BUILD_ARGS='-j0 --builders "@/etc/nix/machines; ssh://pederbs@heid.idi.ntnu.no x86_64-linux - 24 5 big-parallel"' # _JUST_NIX_INSTANTIATE_ARGS='--system darwin' -@build-packages +$attrpaths="": - test -n "${attrpaths:-}" || set -- $(just _some_packages); \ +@build-packages *$attrpaths: + [[ "$#" -gt 0 ]] || set -- $(just _some_packages); \ cd "$invokedir"; just _build_packages results "$@" [no-cd] _build_packages $outdir +$attrpaths: + just __build_packages "$(just _a_system)" "$@" + +[no-cd] +__build_packages $system $outdir +$attrpaths: #!/usr/bin/env -S bash -euo pipefail -x # hint: supports _JUST_NIX_BUILD_ARGS # hint: supports _JUST_NIX_INSTANTIATE_ARGS @@ -190,20 +235,32 @@ _build_packages $outdir +$attrpaths: fd . "$outdir" --type l -X rm -v fd \\\.log$ "$outdir" --type f -X rm -v } - mkdir -p "$outdir" - export _INSTANTIATE_CACHE=$(mktemp -d) + declare -a attrpaths=("${@:3}") + mkdir -p "$outdir/.gcroot" + # export _JUST_NIX_INSTANTIATE_CACHE=$(mktemp -d) export NOTFOUNDDIR="$(mktemp -d)"; touch "$NOTFOUNDDIR"/no-eval - shift - just instantiate-packages "$@" | tee >(command cat >&2) \ - | eval xargs nom-build --keep-going --no-out-link ${_JUST_NIX_BUILD_ARGS:-} ||: + just _instantiate_packages "$system" "${attrpaths[@]}" | tee >(command cat >&2) \ + | while IFS=! read path output; do printf "%s\n" "$(realpath "$path")${output:+!"$output"}"; done \ + | xargs -d'\n' nom-build --keep-going --no-out-link ${_JUST_NIX_BUILD_ARGS:-} ||: + # | eval xargs nom-build --keep-going --no-out-link ${_JUST_NIX_BUILD_ARGS:-} ||: worker() { set -x local attrpath="$1" local dst="$outdir/$attrpath" - local drvpath=$( just instantiate-packages "$attrpath" ) - # NIXPKGS_ALLOW_UNFREE=1 nix-build . -A "$attrpath" ${_JUST_NIX_INSTANTIATE_ARGS:-} -j0 --option builders "" --option substitute false -o "$dst" >&/dev/null + local gcroot="$outdir/.gcroot/$attrpath" # workaround since .drv!dev will always append "-dev" to --out-path + local drvpath=$( just _instantiate_packages "$system" "$attrpath" ) + # NIXPKGS_ALLOW_UNFREE=1 nix-build . -A "$attrpath" --system "$system" ${_JUST_NIX_INSTANTIATE_ARGS:-} -j0 --option builders "" --option substitute false -o "$dst" >&/dev/null if [[ -n "$drvpath" ]]; then - NIXPKGS_ALLOW_UNFREE=1 nix-build "$drvpath" -j0 --option builders "" --option substitute false -o "$dst" >&/dev/null + # NIXPKGS_ALLOW_UNFREE=1 nix-build "$drvpath" -j0 --option builders "" --option substitute false --out-link "$dst" >&/dev/null + # NIXPKGS_ALLOW_UNFREE=1 nix-build "$drvpath" -j0 --option builders "" --option substitute false --out-link "$gcroot" >&/dev/null + nix-build "$drvpath" -j0 --option builders "" --option substitute false --out-link "$gcroot" >&/dev/null + if [[ "$drvpath" =~ '!' ]]; then + # :( + gcroot="$gcroot-$(cut -d! -f2 <<<"$drvpath")" + fi + if [[ -e "$gcroot" ]]; then + ln -s "$(readlink "$gcroot")" $dst + fi fi # local outpaths=$( nix-store -q --outputs "$drvpath" ) # doesn't gcroot, some outputs may not be pulled from builders if [[ -L "$dst" ]]; then @@ -213,18 +270,18 @@ _build_packages $outdir +$attrpaths: ln -s "$NOTFOUNDDIR"/no-eval "$dst" else ln -s /build-failure-"$attrpath" "$dst" - nix log ".#$attrpath" ${_JUST_NIX_INSTANTIATE_ARGS:-} > "$dst".log 2>/dev/null ||: + nix log ".#$attrpath" --system "$system" ${_JUST_NIX_INSTANTIATE_ARGS:-} > "$dst".log 2>/dev/null ||: fi fi } - printf "%s\n" "$@" | xe -j0 -s "$(declare -f worker); worker \"\$@\"" - # echo -e "#!/usr/bin/env bash\ntest -f flake.nix || cd ..; env _JUST_NIX_INSTANTIATE_ARGS=\"\$*\" just _build_packages \"$outdir\"" $attrpaths > "$outdir"/_rerun.sh + printf "%s\n" "${attrpaths[@]}" | xe -j0 -s "$(declare -f worker); worker \"\$@\"" + # echo -e "#!/usr/bin/env bash\ntest -f flake.nix || cd ..; env _JUST_NIX_INSTANTIATE_ARGS=\"\$*\" just __build_packages "$system" \"$outdir\"" $attrpaths > "$outdir"/_rerun.sh cat <<-EOF >"$outdir"/_rerun.sh #!/usr/bin/env bash while [[ ! -f flake.nix ]]; do cd .. done - _JUST_NIX_INSTANTIATE_ARGS="\$*" just _build_packages "$outdir" $attrpaths + _JUST_NIX_INSTANTIATE_ARGS="\$*" just __build_packages "$system" "$outdir" $attrpaths EOF chmod +x "$outdir"/_rerun.sh fd . "$outdir" -l @@ -232,11 +289,15 @@ _build_packages $outdir +$attrpaths: # TODO: use in 'bump' [no-cd] build-package-sources +packages: + just _build_package_sources "$(just _a_system)" "$@" + +[no-cd] +_build_package_sources $system +packages: #!/usr/bin/env -S bash -euo pipefail -x [[ -e .git && -f flake.nix ]] || { printf >&2 "%s\n" "ERROR: not in repo root!"; false; } - declare packages=("$@") + declare packages=("${@:2}") # TODO: allow selecting different names than result-src and results-src-writeable - just _build_packages results-src "${packages[@]/%/.src}" + just __build_packages "$system" results-src "${packages[@]/%/.src}" [[ -d results-src-writeable ]] && rm -rf results-src-writeable ||: mkdir -p results-src-writeable for result in results-src/* ; do @@ -311,7 +372,7 @@ fixup-commit: test -n "$($GIT diff --name-only --staged)" || { $GIT ls-files --modified --others --exclude-standard | grep -v ^results- | - fzf --layout=reverse --multi --height 15 | + fzf --layout=reverse --multi | xe $GIT add --patch } test -n "$($GIT diff --name-only --staged)" || { @@ -356,10 +417,11 @@ bump-new-worktree *packages: #!/usr/bin/env -S bash -euo pipefail -x declare -a packages=("$@") [[ -n "${packages[*]}" ]] || packages=($(just _some_packages)) + export JUST_SYSTEM="$(just _a_system)" worktree=$(just _new_worktree bump "$(printf "%s\n" "${packages[0]}" | rev | cut -d. -f1 | rev)") [[ -n "worktree" ]] cd "$worktree" - just bump-here "${packages[@]}" + bump-here "${packages[@]}" NIXPKGS_ALLOW_UNFREE=1 "$SHELL" [no-cd] @@ -368,7 +430,8 @@ bump-here *packages: [[ -e .git && flake.nix ]] declare -a packages=("$@") [[ -n "${packages[*]}" ]] || packages=($(just _some_packages)) - printf "%s.src\n" "${packages[@]}" | xargs just _build_packages results-src-old ||: # TODO: unpack? + system="$(just _a_system)" + printf "%s.src\n" "${packages[@]}" | xargs just __build_packages "$system" results-src-old ||: # TODO: unpack? config=(env NIXPKGS_ALLOW_UNFREE=1 NIXPKGS_ALLOW_BROKEN=1 @@ -388,10 +451,10 @@ bump-here *packages: fi done set +e - printf "%s.src\n" "${packages[@]}" | xargs just _build_packages results-src-new # TODO: unpack? + printf "%s.src\n" "${packages[@]}" | xargs just __build_packages "$system" results-src-new # TODO: unpack? just _build_packages results "${packages[@]}" # TODO: filter non-existing tests: - printf "%s.tests\n" "${packages[@]}" | env _JUST_NIX_BUILD_ARGS="-j1" xargs just _build_packages results-tests + printf "%s.tests\n" "${packages[@]}" | env _JUST_NIX_BUILD_ARGS="-j1" xargs just __build_packages "$system" results-tests printf "HEAD^%s " $(seq 0 $(($#-1))) | xe $GIT show fd . -l results results-tests --max-depth 1 # test -L result && fd . result --type x --color=always | tee update-executables.txt @@ -404,6 +467,7 @@ fix *packages: declare packages=("$@") [[ "${#packages[@]}" -gt 0 ]] || packages=($(just _some_packages)) [[ "${#packages[@]}" -gt 0 ]] || { printf >&2 "%s\n" "ERROR: no packages chosen..."; false; } + export JUST_SYSTEM="$(just _a_system)" cd "$(just _new_worktree fix "$(echo "${packages[0]}" | rev | cut -d. -f1 | rev)")" just _fix "${packages[@]}" ||: $SHELL @@ -413,13 +477,14 @@ _fix +packages: #!/usr/bin/env -S bash -euo pipefail [[ -e .git && -f flake.nix ]] || { printf >&2 "%s\n" "ERROR: not in repo root!"; false; } declare packages=("$@") + export JUST_SYSTEM="$(just _a_system)" just build-package-sources "${packages[@]}" NIXPKGS_ALLOW_UNFREE=1 just build-packages "${packages[@]}" ||: if command -v bat >/dev/null; then bat build.nix --style header-filename else cat build.nix fi - echo "./results/_rerun.sh --system $(nix config show system)" + echo "./results/_rerun.sh --system $JUST_SYSTEM" pr $number=`just _a_pr`: #!/usr/bin/env -S bash -euo pipefail @@ -429,12 +494,19 @@ pr $number=`just _a_pr`: elif test -d prs/pr-"$number"-*; then cd prs/pr-"$number"-* else + # TODO: support --onto + upstream_branch=$( just _a_upstream_release_branch "Rebase?" "" ) (cd master; $GIT worktree prune - $GIT worktree add ../prs/pr-"$number" + (set -x; $GIT worktree add ../prs/pr-"$number") ) cd prs/pr-"$number" - gh pr checkout "$number" + (set -x; gh pr checkout "$number") + if [[ -n "$upstream_branch" ]]; then + remote="$(tr <<<"upstream_branch" -d/ -f1)" + branch="$(tr <<<"upstream_branch" -d/ -f2)" + (set -x; $GIT pull --rebase "$remote" "$branch" || $GIT rebase --abort) ||: + fi fi $SHELL @@ -710,7 +782,7 @@ _packages_json: git -C master/ pull upstream master 1>&2 time nix-env > packages.json \ --extra-experimental-features no-url-literals \ - --option system x86_64-linux \ + --option system "$(just _this_system)" \ -f ./master \ -qaP \ --json \ @@ -730,20 +802,53 @@ _packages_json: @_some_commits: # TODO: check if in a git repo #cd "$invokedir"; $GIT log --oneline -n800 | gum filter --placeholder "Pick commit..." --height 10 --no-limit | cut -d' ' -f1 - cd "$invokedir"; $GIT log --oneline -n800 | fzf --layout=reverse --multi --height 15 | cut -d' ' -f1 + cd "$invokedir"; $GIT log --oneline -n800 | fzf --layout=reverse --multi | cut -d' ' -f1 @_a_commit_title: cd "$invokedir"; rev="$(just _a_commit)" || exit $?; [[ -n "$rev" ]] && git log --format="%s" -n1 "$rev" | grep . @_a_package: _packages_json - #gum spin --show-output just list-packages | cut -f1 | fzf --sync --layout=reverse --height 15 | grep . + #gum spin --show-output just list-packages | cut -f1 | fzf --sync --layout=reverse | grep . just list-packages >/dev/null - just list-packages | cut -f1 | fzf --sync --layout=reverse --height 15 | grep . + just list-packages | cut -f1 | fzf --sync --layout=reverse | grep . @_some_packages: _packages_json - #gum spin --show-output just list-packages | cut -f1 | fzf --sync --layout=reverse --height 15 --multi | grep . + #gum spin --show-output just list-packages | cut -f1 | fzf --sync --layout=reverse --multi | grep . just list-packages >/dev/null - just list-packages | cut -f1 | fzf --sync --layout=reverse --height 15 --multi | grep . + just list-packages | cut -f1 | fzf --sync --layout=reverse --multi | grep . + +@_this_system: + nix config show system + +_a_system: + #!/usr/bin/env -S bash -euo pipefail + if [[ -n "${JUST_SYSTEM:-}" ]]; then + printf "%s\n" "$JUST_SYSTEM" + exit 0 + fi + # nix eval --file master/default.nix lib.systems.flakeExposed --json | jq .[] -r | xargs -d'\n' gum choose --selected "$(just _this_system)" + systems="$(nix eval --file master/default.nix lib.systems.flakeExposed --json)" + system="$(just _this_system)" + # system="aarch64-darwin" + jq <<<"$systems" .[] -r | fzf --sync --layout=reverse $( + jq <<<"$systems" --arg system "$system" -r \ + 'map(. == $system) | index(true) | if . == null then "" else @sh "--bind load:pos(\(. + 1))" end' + ) + +_some_systems: + #!/usr/bin/env -S bash -euo pipefail + if [[ -n "${JUST_SYSTEM:-}" ]]; then + printf "%s\n" "$JUST_SYSTEM" + exit 0 + fi + # nix eval --file master/default.nix lib.systems.flakeExposed --json | jq .[] -r | xargs -d'\n' gum choose --no-limit --selected "$(just _this_system)" + systems="$(nix eval --file master/default.nix lib.systems.flakeExposed --json)" + system="$(just _this_system)" + # system="aarch64-darwin" + jq <<<"$systems" .[] -r | fzf --sync --layout=reverse --multi $( + jq <<<"$systems" --arg system "$system" -r \ + 'map(. == $system) | index(true) | if . == null then "" else @sh "--bind load:pos(\(. + 1))" end' + ) @_a_pr: cd master; gh pr list --limit 1000 --json 'number,title' --state open | jq '.[]|"\(.number) - \(.title)"' -r | gum filter --placeholder "Pick PR..." --height 10 | cut -d' ' -f1