pederbs/just-nixpkgs
0
2
Fork 0
Code Issues Pull Requests Releases Activity

jlasljkasdklj

Browse Source
This commit is contained in:
Peder Bergebakken Sundt
2025-05-04 03:37:53 +02:00
parent b92cca2534
commit 6be2b8fe2e
2 changed files with 133 additions and 36 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
.envrc
View File

@@ -53,3 +53,6 @@ export NIX_PATH_ORIG="${NIX_PATH:-}"
# pkgs/top-level/impure.nix go brrr
export NIXPKGS_CONFIG="$(realpath config.nix)"
export NIX_PATH="$NIX_PATH${NIX_PATH:+:}nixpkgs-overlays=$(realpath overlays.nix)"
# TODO: any better way to avoid this leaking?
unset JUST_SYSTEM

166
justfile
View File

@@ -30,6 +30,8 @@ export NIX_PATH := `echo "${NIX_PATH_ORIG:-"${NIX_PATH:-}"}"`
cd "$invokedir"; just "$(gum filter $(just --summary --unsorted | tr ' ' '\n' | grep '^build-'))" "$@"
@bump *args:
cd "$invokedir"; just "$(gum filter $(just --summary --unsorted | tr ' ' '\n' | grep '^bump-'))" "$@"
@get *args:
cd "$invokedir"; just "$(gum filter $(just --summary --unsorted | tr ' ' '\n' | grep '^get-'))" "$@"
# === info helpers ===
@@ -216,6 +218,7 @@ _instantiate_packages $system +$attrpaths:
# _JUST_NIX_BUILD_ARGS='--check'
# _JUST_NIX_BUILD_ARGS='-j0 --builders "@/etc/nix/machines; ssh://pederbs@heid.idi.ntnu.no x86_64-linux - 24 5 big-parallel"'
# _JUST_NIX_INSTANTIATE_ARGS='--system darwin'
# TODO: is ^ now broken?
@build-packages *$attrpaths:
[[ "$#" -gt 0 ]] || set -- $(just _some_packages); \
cd "$invokedir"; just _build_packages results "$@"
@@ -226,34 +229,36 @@ _build_packages $outdir +$attrpaths:
[no-cd]
__build_packages $system $outdir +$attrpaths:
#!/usr/bin/env -S bash -euo pipefail -x
#!/usr/bin/env -S bash -euo pipefail
# hint: supports _JUST_NIX_BUILD_ARGS
# hint: supports _JUST_NIX_INSTANTIATE_ARGS
test -z "${_JUST_NIX_BUILD_ARGS:-}" || echo >&2 "_JUST_NIX_BUILD_ARGS = $_JUST_NIX_BUILD_ARGS"
test -z "${_JUST_NIX_INSTANTIATE_ARGS:-}" || echo >&2 "_JUST_NIX_INSTANTIATE_ARGS = $_JUST_NIX_INSTANTIATE_ARGS"
test -d "$outdir" && {
rm -rf "$outdir"/.gcroot
fd . "$outdir" --type l -X rm -v
fd \\\.log$ "$outdir" --type f -X rm -v
}
declare -a attrpaths=("${@:3}")
mkdir -p "$outdir/.gcroot"
# export _JUST_NIX_INSTANTIATE_CACHE=$(mktemp -d)
export _JUST_NIX_INSTANTIATE_CACHE=$(mktemp -d)
export NOTFOUNDDIR="$(mktemp -d)"; touch "$NOTFOUNDDIR"/no-eval
# set -x
just _instantiate_packages "$system" "${attrpaths[@]}" | tee >(command cat >&2) \
| while IFS=! read path output; do printf "%s\n" "$(realpath "$path")${output:+!"$output"}"; done \
| xargs -d'\n' nom-build --keep-going --no-out-link ${_JUST_NIX_BUILD_ARGS:-} ||:
# | eval xargs nom-build --keep-going --no-out-link ${_JUST_NIX_BUILD_ARGS:-} ||:
worker() {
set -x
local attrpath="$1"
local dst="$outdir/$attrpath"
local gcroot="$outdir/.gcroot/$attrpath" # workaround since .drv!dev will always append "-dev" to --out-path
local drvpath=$( just _instantiate_packages "$system" "$attrpath" )
set -x
# NIXPKGS_ALLOW_UNFREE=1 nix-build . -A "$attrpath" --system "$system" ${_JUST_NIX_INSTANTIATE_ARGS:-} -j0 --option builders "" --option substitute false -o "$dst" >&/dev/null
if [[ -n "$drvpath" ]]; then
# NIXPKGS_ALLOW_UNFREE=1 nix-build "$drvpath" -j0 --option builders "" --option substitute false --out-link "$dst" >&/dev/null
# NIXPKGS_ALLOW_UNFREE=1 nix-build "$drvpath" -j0 --option builders "" --option substitute false --out-link "$gcroot" >&/dev/null
nix-build "$drvpath" -j0 --option builders "" --option substitute false --out-link "$gcroot" >&/dev/null
nix-build "$drvpath" -j0 --option builders "" --option substitute false --out-link "$gcroot" >&/dev/null ||:
if [[ "$drvpath" =~ '!' ]]; then
# :(
gcroot="$gcroot-$(cut -d! -f2 <<<"$drvpath")"
@@ -274,22 +279,25 @@ __build_packages $system $outdir +$attrpaths:
fi
fi
}
printf "%s\n" "${attrpaths[@]}" | xe -j0 -s "$(declare -f worker); worker \"\$@\""
printf "%s\n" "${attrpaths[@]}" | xe -j0 -s "set -euo pipefail; $(declare -f worker); worker \"\$@\"" ||:
# echo -e "#!/usr/bin/env bash\ntest -f flake.nix || cd ..; env _JUST_NIX_INSTANTIATE_ARGS=\"\$*\" just __build_packages "$system" \"$outdir\"" $attrpaths > "$outdir"/_rerun.sh
cat <<-EOF >"$outdir"/_rerun.sh
#!/usr/bin/env bash
while [[ ! -f flake.nix ]]; do
cd ..
done
_JUST_NIX_INSTANTIATE_ARGS="\$*" just __build_packages "$system" "$outdir" $attrpaths
_JUST_NIX_INSTANTIATE_ARGS="\$*" just __build_packages "\${1:-"$system"}" $(printf " %q" "$outdir" "${attrpaths[@]}")
EOF
chmod +x "$outdir"/_rerun.sh
fd . "$outdir" -l
# TODO: use in 'bump'
[no-cd]
build-package-sources +packages:
just _build_package_sources "$(just _a_system)" "$@"
build-package-sources *packages:
#!/usr/bin/env -S bash -euo pipefail
packages=("$@")
[[ "${#packages[@]}" -gt 0 ]] || readarray -td $'\n' packages < <( just _some_packages )
just _build_package_sources "$(just _a_system)" "${packages[@]}"
[no-cd]
_build_package_sources $system +packages:
@@ -418,6 +426,7 @@ bump-new-worktree *packages:
declare -a packages=("$@")
[[ -n "${packages[*]}" ]] || packages=($(just _some_packages))
export JUST_SYSTEM="$(just _a_system)"
just get-versions "${packages[0]}"
worktree=$(just _new_worktree bump "$(printf "%s\n" "${packages[0]}" | rev | cut -d. -f1 | rev)")
[[ -n "worktree" ]]
cd "$worktree"
@@ -432,13 +441,14 @@ bump-here *packages:
[[ -n "${packages[*]}" ]] || packages=($(just _some_packages))
system="$(just _a_system)"
printf "%s.src\n" "${packages[@]}" | xargs just __build_packages "$system" results-src-old ||: # TODO: unpack?
config=(env
config=(
env
NIXPKGS_ALLOW_UNFREE=1
NIXPKGS_ALLOW_BROKEN=1
NIXPKGS_ALLOW_INSECURE=1
)
for package in "${packages[@]}"; do
if nix eval -f . "$package".passthru.updateScript >&/dev/null; then
if nix eval --log-format raw --file default.nix "$package".passthru.updateScript >&/dev/null; then
HEAD=$($GIT rev-parse HEAD)
"${config[@]}" nix-update "$package" --use-update-script --update-script-args "--argstr skip-prompt true" --commit
if [[ $($GIT rev-parse HEAD) = "$HEAD" ]]; then
@@ -452,7 +462,7 @@ bump-here *packages:
done
set +e
printf "%s.src\n" "${packages[@]}" | xargs just __build_packages "$system" results-src-new # TODO: unpack?
just _build_packages results "${packages[@]}"
just __build_packages "$system" results "${packages[@]}"
# TODO: filter non-existing tests:
printf "%s.tests\n" "${packages[@]}" | env _JUST_NIX_BUILD_ARGS="-j1" xargs just __build_packages "$system" results-tests
printf "HEAD^%s " $(seq 0 $(($#-1))) | xe $GIT show
@@ -468,8 +478,13 @@ fix *packages:
[[ "${#packages[@]}" -gt 0 ]] || packages=($(just _some_packages))
[[ "${#packages[@]}" -gt 0 ]] || { printf >&2 "%s\n" "ERROR: no packages chosen..."; false; }
export JUST_SYSTEM="$(just _a_system)"
just open-package-urls "${packages[0]}"
cd "$(just _new_worktree fix "$(echo "${packages[0]}" | rev | cut -d. -f1 | rev)")"
just _fix "${packages[@]}" ||:
if gum confirm "Try to bump?" --default=no; then
rm -rf results-src
just bump-here "${packages[@]}"
fi
$SHELL
[no-cd]
@@ -542,6 +557,7 @@ pr $number=`just _a_pr`:
test -s "$FNAME" && bat --language markdown --style plain --pager never "$FNAME" && gum confirm && gh pr {{ mode }} "$NUMBER" --body-file "$FNAME"; \
rm -v "$FNAME"
# TODO: git nomad?
remote-branch $remote_ref=`just _a_remote_branch origin | cut -d/ -f2-`:
#!/usr/bin/env -S bash -euo pipefail
branch_dir=$(cut <<<"$remote_ref" -d- -f1)
@@ -586,6 +602,7 @@ _mk_worktree $name $dir $base="upstream/master" $onto="":
$GIT branch "$name" "$branch" --no-track
$GIT worktree add ../"$dir" "$name"
cd ../"$dir"
# TODO: use this to determine base instead of using rebase-onto: baserev="$(git merge-base "$base" "$onto")"
$GIT pull --rebase "$remote" "$branch"
[[ -z "$onto" ]] || just rebase-onto HEAD "$onto"
@@ -657,6 +674,7 @@ fix-periodic-merge-conflict $from=`just _a_upstream_release_branch "From?"` $to=
# === setup ===
# setup or update master and staging++
setup:
#!/usr/bin/env -S bash -euo pipefail
if [[ ! -e master/.git ]]; then
@@ -675,32 +693,32 @@ setup:
fi
cd master
if ! $GIT remote get-url upstream >&/dev/null; then
$GIT remote add upstream "https://github.com/NixOS/nixpkgs.git"
(set -x; $GIT remote add upstream "https://github.com/NixOS/nixpkgs.git")
fi
add_upstream() {
# TODO: check the mtime on the HEAD file instead?
if [[ $(( "$epoch" - "$($GIT log --oneline --format="%ct" HEAD -n1)" )) -gt $(( 60*60*24 )) ]]; then
(set -x; $GIT pull --rebase --autostash upstream master >&2)
fi
add_upstream() (
local branch="$1"
if [[ ! -d ../upstream/"$branch" ]]; then
set -x
$GIT fetch upstream "$branch"
$GIT worktree add ../upstream/"$branch" -B "$branch" upstream/"$branch"
else
cd ../upstream/"$branch"
if [[ $(( "$epoch" - "$($GIT log --oneline --format="%ct" HEAD -n1)" )) -gt $(( 60*60*24 )) ]]; then
set -x
$GIT pull --rebase --autostash upstream "$branch" >&2
fi
fi
}
add_upstream staging
add_upstream staging-next
add_upstream staging-24.11
add_upstream staging-next-24.11
# TODO: test
#upstreams-fetch:
# cd master; $GIT fetch upstream master
# cd master; $GIT fetch upstream staging
# cd master; $GIT fetch upstream staging-next
# cd master; $GIT fetch upstream release-24.05
# cd master; $GIT fetch upstream staging-24.05
# cd master; $GIT fetch upstream release-23.11
# cd master; $GIT fetch upstream staging-23.11
# cd master; $GIT fetch upstream python-updates
# === helpers ===
# TODO stat the file and redo if too old, or tie it to the master checkout commit
)
declare -a branches=()
readarray -td $'\n' branches < <( just _upstream_release_branches "" "" | grep -vxF master )
# TODo: clean old release branches
for branch in "${branches[@]}"; do
add_upstream "$branch"
done
clean:
#!/usr/bin/env -S bash -euo pipefail
@@ -770,6 +788,75 @@ _mk_clean_list:
printf "%s\n" "( cd master && git worktree prune )"
printf "%s\n" "rmdir */ --ignore-fail-on-non-empty"
# === get/is/test ===
get-versions $attrpath=`just _a_package` *paths: setup
#!/usr/bin/env -S bash -euo pipefail
declare -a paths=("${@:2}")
if [[ "${#paths[@]}" -eq 0 ]]; then
readarray -td $'\n' paths < <( just _upstream_release_branches "upstream/" "" )
paths=(master "${paths[@]}")
fi
just _get_version "$attrpath" "${paths[@]}" | column --table --separator ":" --output-separator " : " | bat -l conf --style=plain --paging=never
_get_version $attrpath +paths:
#!/usr/bin/env -S bash -euo pipefail
declare -a paths=("${@:2}")
for path in "${paths[@]}"; do
if [[ -d "$path" ]]; then
sleep 0.1 # HACK, sort it properly instead
echo "$attrpath.version @ $path: $(
nix eval --log-format raw --file "$path"/default.nix "$attrpath" \
--apply 'let lib = import ./master/lib; in lib.getVersion' --raw
)" &
fi
done
wait
[no-cd]
test-merge $rev=`cd "$invokedir"; just _a_commit` *branches:
#!/usr/bin/env -S bash -euo pipefail
declare -a branches=("${@:2}")
[[ "${#branches[@]}" -gt 0 ]] || {
just setup # update
# readarray -td $'\n' branches < <( just _upstream_release_branches "upstream/" "" )
readarray -td $'\n' branches < <( prefix=upstream/ just _some_upstream_release_branches "Select targets" )
}
[[ "${#branches[@]}" -gt 0 ]] || ! echo >&2 "ERROR: no target branches chosen" || false
retval=0
for branch in "${branches[@]}"; do
echo >&2 "Checking $branch..."
if ! $GIT merge-tree "$branch" "$rev" >/dev/null; then
echo >&2 "MERGE-CONFLICT with $branch"
retval=1
[[ -t 1 ]] || break # short circuit if only retcode matters
fi
done
if [[ "$retval" -eq 0]]; then
echo >&2 "No conflicts!"
fi
exit "$retval"
open-package-urls $package=`just _a_package`:
#!/usr/bin/env -S bash -euo pipefail
fpkg="$(jq -rn --arg x "$package" '$x | @uri')"
bpkg="$(jq -rn --arg x "$(rev <<<"$package" | cut -d. -f1 | rev)" '$x | @uri')"
# homepage="$(nix eval --log-format raw --file master/default.nix "$package".meta.homepage --raw ||:)"
srcpage="$(nix eval --log-format raw --file master/default.nix "$package".src.meta.homepage --raw ||:)"
position="$({ nix eval --log-format raw --file master/default.nix "$package".meta.position --raw ||: ; } | rev | cut -d: -f2- | rev | sd -F "$PWD/master/" "")"
declare -a urls=()
urls+=("https://hydra.nixos.org/job/nixos/trunk-combined/nixpkgs.${fpkg}.x86_64-linux")
urls+=("https://github.com/NixOS/nixpkgs/pulls?q=is%3Apr+in%3Atitle+${bpkg}")
urls+=("https://github.com/NixOS/nixpkgs/issues?q=is%3Aissue+in%3Atitle+${bpkg}")
[[ -z "$position" ]] || urls+=(https://github.com/NixOS/nixpkgs/blame/master/"$position")
[[ -z "$position" ]] || urls+=(https://github.com/NixOS/nixpkgs/commits/master/"$position")
# [[ -z "$homepage" ]] || urls+=("$homepage")
[[ -z "$srcpage" ]] || urls+=("$srcpage")
urls+=("https://search.nixos.org/packages?channel=unstable&query=${fpkg}")
printf "%s\n" "${urls[@]}" | fzf --reverse --multi | xe xdg-open
# === internal helpers ===
_packages_json:
#!/usr/bin/env -S bash -euo pipefail
if [[ ! -s packages.json ||
@@ -792,6 +879,9 @@ _packages_json:
--arg config '{ allowAliases = false; }'
fi
@_this_commit:
cd "$invokedir"; $GIT log -n1 --pretty='format:%H'
@_a_commit *extra_revs:
# TODO: check if in a nixpkgs git repo
{ \
@@ -823,11 +913,11 @@ _packages_json:
_a_system:
#!/usr/bin/env -S bash -euo pipefail
if [[ -n "${JUST_SYSTEM:-}" ]]; then
printf "%s\n" "$JUST_SYSTEM"
printf "%s\n" "$JUST_SYSTEM" | head -n1
exit 0
fi
# nix eval --file master/default.nix lib.systems.flakeExposed --json | jq .[] -r | xargs -d'\n' gum choose --selected "$(just _this_system)"
systems="$(nix eval --file master/default.nix lib.systems.flakeExposed --json)"
systems="$(nix eval --log-format raw --file master/default.nix lib.systems.flakeExposed --json)"
system="$(just _this_system)"
# system="aarch64-darwin"
jq <<<"$systems" .[] -r | fzf --sync --layout=reverse $(
@@ -841,8 +931,8 @@ _some_systems:
printf "%s\n" "$JUST_SYSTEM"
exit 0
fi
# nix eval --file master/default.nix lib.systems.flakeExposed --json | jq .[] -r | xargs -d'\n' gum choose --no-limit --selected "$(just _this_system)"
systems="$(nix eval --file master/default.nix lib.systems.flakeExposed --json)"
# nix eval --log-format raw --file master/default.nix lib.systems.flakeExposed --json | jq .[] -r | xargs -d'\n' gum choose --no-limit --selected "$(just _this_system)"
systems="$(nix eval --log-format raw --file master/default.nix lib.systems.flakeExposed --json)"
system="$(just _this_system)"
# system="aarch64-darwin"
jq <<<"$systems" .[] -r | fzf --sync --layout=reverse --multi $(
@@ -864,7 +954,11 @@ _some_systems:
staging-next-24.11
@_a_upstream_release_branch $header *extra_branches:
shift; gum choose --header="$header" "$@" $(just _upstream_release_branches "${prefix:-upstream/}")
shift; gum filter --header="$header" "$@" $(just _upstream_release_branches "${prefix:-upstream/}" "${suffix:-}")
# --ordered
@_some_upstream_release_branches $header *extra_branches:
shift; gum filter --header="$header" "$@" $(just _upstream_release_branches "${prefix:-upstream/}" "${suffix:-}") --no-limit
# --ordered
@_sanitize_pr_url $number: