diff --git a/.gitignore b/.gitignore index 72e8ffc..22f4fa3 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,4 @@ * +!/justfile +!/.gitignore +!/clean.sh diff --git a/justfile b/justfile index f179304..0d51e2a 100644 --- a/justfile +++ b/justfile @@ -9,57 +9,64 @@ export GUM_FILTER_HEIGHT := "15" export GIT := `command -v git-wait >/dev/null && echo "git-wait" || echo "git"` # required programs: git nix nom gum fzf tee nix-update -#export NIXPKGS_ALLOW_UNFREE := "1" -#export NIXPKGS_ALLOW_INSECURE := "1" -#export NIXPKGS_ALLOW_BROKEN := "1" +[no-cd] @_default: - cd "$invokedir"; just "$(gum filter $(just --summary --unsorted))" + just "$(gum filter $(just --summary --unsorted))" # === info helpers === -@list-packages +$cut_args="-c1-": _packages_json - cachefile="${XDG_RUNTIME_DIR:-"${TMPDIR:-/tmp}"}"/packages.json.tsv; \ - if test -s "$cachefile" && test "$(stat -c %Y packages.json )" -le "$(stat -c %Y "$cachefile" )"; then \ - cat "$cachefile" | cut "$@" ; \ - else \ - just _list_packages | tee "$cachefile" | cut "$@"; \ +list-packages +$cut_args="-c1-": _packages_json + #!/usr/bin/env -S bash -euo pipefail + cachefile=/tmp/nixpkgs-packages.json.tsv + if [[ -s "$cachefile" && "$cachefile" -nt packages.json ]]; then + cut <"$cachefile" "$@" + else + jq "$cachefile" fi -@_list_packages: _packages_json - #jq &2 echo "ripgrep args missing!"; false; } - just list-packages | grep --fixed-strings "$( \ - cd "$invokedir"; set -x; rg "$@" pkgs/ -tnix -l | sed 's/\(.*\)/\t\1\t/' | sort | head -n 2500 \ - | grep -v pkgs/development/haskell-modules/hackage-packages.nix \ - | grep -v pkgs/top-level/ \ - )" + +@_list_packages_fname_filtered: + xargs -d $'\n' -n 2000 just __list_packages_fname_filtered +@__list_packages_fname_filtered +fnames: + just list-packages | grep --fixed-strings "$(printf "\t%s\t\n" "$@")" \ + | grep -v $'\t'pkgs/top-level/ \ + | grep -v $'\t'pkgs/applications/editors/vim/plugins/ \ + | grep -v $'\t'pkgs/applications/editors/vim/plugins/ \ + | grep -v $'\t'pkgs/applications/editors/vscode/extensions/default.nix \ + | grep -v $'\t'pkgs/build-support/cc-wrapper/default.nix \ + | grep -v $'\t'pkgs/build-support/trivial-builders/default.nix \ + | grep -v $'\t'pkgs/desktops/gnome/extensions/buildGnomeExtension.nix \ + | grep -v $'\t'pkgs/development/compilers/chicken/5/default.nix \ + | grep -v $'\t'pkgs/development/haskell-modules/hackage-packages.nix \ + | grep -v $'\t'pkgs/development/lisp-modules/imported.nix \ + | grep -v $'\t'pkgs/development/lua-modules/generated-packages.nix \ + | grep -v $'\t'pkgs/development/python-modules/mypy-boto3/default.nix \ + | grep -v $'\t'pkgs/development/python-modules/types-aiobotocore-packages/default.nix \ + | grep -v $'\t'pkgs/os-specific/linux/nvidia-x11/generic.nix \ + | grep -v $'\t'pkgs/servers/home-assistant/default.nix \ + | grep -v $'\t'pkgs/tools/package-management/akku/default.nix \ + | grep -v $'\t'pkgs/tools/typesetting/tex/texlive/build-texlive-package.nix + +@list-ripgrepped-packages +$ripgrep_args: + [[ -e .git && -e flake.nix ]] || { echo >&2 "ERROR; Not in repo root"; false; } + [[ -n "$ripgrep_args" ]] || { echo >&2 "ERROR; No ripgrep args"; false; } + cd "$invokedir"; rg "$@" pkgs/ -tnix -l | just _list_packages_fname_filtered @list-dirty-packages: - # TODO: switch to 'git diff HEAD --numstat'? - test -n "$(cd "$invokedir"; $GIT ls-files --modified)" || ( >&2 echo "No unstaged dirty files found!"; false ) - just list-packages | grep --fixed-strings "$( \ - cd "$invokedir"; $GIT ls-files --modified | sed 's/\(.*\)/\t\1\t/' \ - | grep -v pkgs/development/haskell-modules/hackage-packages.nix \ - | grep -v pkgs/top-level/ \ - )" + test -n "$($GIT diff HEAD --name-only)" || { echo >&2 "No unstaged dirty files found!"; false; } + cd "$invokedir"; $GIT diff HEAD --name-only | just _list_packages_fname_filtered -@list-touched-packages-since $rev=`cd "$invokedir"; just _a_commit`: _packages_json - test -n "$rev" || ( >&2 echo "you must pick a revision to compare against!"; false ) - just list-packages | grep --fixed-strings "$( \ - cd "$invokedir"; $GIT diff "$rev" --numstat | cut -f3 \ - | grep -v pkgs/development/haskell-modules/hackage-packages.nix \ - | grep -v pkgs/top-level/ \ - )" +@list-touched-packages-since $rev=`cd "$invokedir"; just _a_commit`: + test -n "$rev" || { echo >&2 "you must pick a revision to compare against!"; false; } + cd "$invokedir"; $GIT diff "$rev" --name-only | just _list_packages_fname_filtered @list-unmerged-packages: - -cd "$invokedir"; $GIT fetch upstream + cd "$invokedir"; $GIT fetch upstream ||: cd "$invokedir"; $GIT log HEAD --not --remotes --oneline | cut -d' ' -f2- | cut -d: -f1 | grep -vE '(^treewide|[, ])' | sort -u # === build helpers === @@ -77,14 +84,6 @@ export GIT := `command -v git-wait >/dev/null && echo "git-wait" || echo "git"` [[ -n "$extra_args" ]] && export _JUST_NIX_INSTANTIATE_ARGS="$(shift; shift; printf " %q" "$@")"; \ cd "$invokedir"; just list-dirty-packages | cut -f1 | xargs printf "$attr_prefix"'%s'"$attr_suffix " | xargs just instantiate-packages -@build-unmerged-packages $attr_prefix="" $attr_suffix="" +$extra_args="": - [[ -n "$extra_args" ]] && export _JUST_NIX_INSTANTIATE_ARGS="$(shift; shift; printf " %q" "$@")"; \ - cd "$invokedir"; just list-unmerged-packages | xargs printf "$attr_prefix"'%s'"$attr_suffix " | xargs just build-packages - -@instantiate-unmerged-packages $attr_prefix="" $attr_suffix="" +$extra_args="": - [[ -n "$extra_args" ]] && export _JUST_NIX_INSTANTIATE_ARGS="$(shift; shift; printf " %q" "$@")"; \ - cd "$invokedir"; just list-unmerged-packages | xargs printf "$attr_prefix"'%s'"$attr_suffix " | xargs just instantiate-packages - @build-touched-packages-since $attr_prefix="" $attr_suffix="" +$extra_args="": [[ -n "$extra_args" ]] && export _JUST_NIX_INSTANTIATE_ARGS="$(shift; shift; printf " %q" "$@")"; \ cd "$invokedir"; just list-touched-packages-since | cut -f1 | xargs printf "$attr_prefix"'%s'"$attr_suffix " | xargs just build-packages @@ -93,61 +92,97 @@ export GIT := `command -v git-wait >/dev/null && echo "git-wait" || echo "git"` [[ -n "$extra_args" ]] && export _JUST_NIX_INSTANTIATE_ARGS="$(shift; shift; printf " %q" "$@")"; \ cd "$invokedir"; just list-touched-packages-since | cut -f1 | xargs printf "$attr_prefix"'%s'"$attr_suffix " | xargs just instantiate-packages +@build-unmerged-packages $attr_prefix="" $attr_suffix="" +$extra_args="": + [[ -n "$extra_args" ]] && export _JUST_NIX_INSTANTIATE_ARGS="$(shift; shift; printf " %q" "$@")"; \ + cd "$invokedir"; just list-unmerged-packages | xargs printf "$attr_prefix"'%s'"$attr_suffix " | xargs just build-packages + +@instantiate-unmerged-packages $attr_prefix="" $attr_suffix="" +$extra_args="": + [[ -n "$extra_args" ]] && export _JUST_NIX_INSTANTIATE_ARGS="$(shift; shift; printf " %q" "$@")"; \ + cd "$invokedir"; just list-unmerged-packages | xargs printf "$attr_prefix"'%s'"$attr_suffix " | xargs just instantiate-packages + # -@instantiate-packages +$attrpaths="": +[no-cd] +instantiate-packages +$attrpaths="": + #!/usr/bin/env -S bash -euo pipefail # slower, but resitant to bad attrpaths - -test -n "${attrpaths:-}" || set -- $(just _some_packages); \ - cd "$invokedir"; \ - if [[ -v _INSTANTIATE_CACHE && -d "${_INSTANTIATE_CACHE:-}" ]]; then \ - mkdir -p "$_INSTANTIATE_CACHE"/root; \ - printf -- "%s\n" "$@" | grep . | NIXPKGS_ALLOW_UNFREE=1 xe -j0 -s 'C="$_INSTANTIATE_CACHE/$1"; test -s "$C" && cat "$C" || ( nix-instantiate . -A "$1" '"${_JUST_NIX_INSTANTIATE_ARGS:-}"' --add-root "$_INSTANTIATE_CACHE/roots/$1" | tee "$C")'; \ - else \ - printf -- "%s\n" "$@" | grep . | NIXPKGS_ALLOW_UNFREE=1 xe -j0 -s 'nix-instantiate . -A "$1" '"${_JUST_NIX_INSTANTIATE_ARGS:-}"''; \ + test -n "${attrpaths:-}" || set -- $(just _some_packages) + if [[ -d "${_INSTANTIATE_CACHE:-}" ]]; then + mkdir -p "$_INSTANTIATE_CACHE"/{roots,paths} + worker() { + local attrpath="$1" + if [[ -s "$_INSTANTIATE_CACHE/paths/$attrpath" ]]; then + cat "$_INSTANTIATE_CACHE/paths/$attrpath" + else + NIXPKGS_ALLOW_UNFREE=1 nix-instantiate . -A "$attrpath" \ + ${_JUST_NIX_INSTANTIATE_ARGS:-} \ + --add-root "$_INSTANTIATE_CACHE/roots/$attrpath" \ + | tee "$_INSTANTIATE_CACHE/paths/$attrpath" + fi + } + printf "%s\n" "$@" | grep . | \ + xe -j0 -s "$(declare -f worker); worker \"\$@\"" + else + printf "%s\n" "$@" | grep . | \ + xe -j0 -s 'NIXPKGS_ALLOW_UNFREE=1 nix-instantiate . -A "$1" '"${_JUST_NIX_INSTANTIATE_ARGS:-}"'' fi - # TODO: filter the "warning: you did not specify '--add-root'; the result might be removed by the garbage collector" + # _JUST_NIX_BUILD_ARGS='--check' # _JUST_NIX_BUILD_ARGS='-j0 --builders "@/etc/nix/machines; ssh://pederbs@heid.idi.ntnu.no x86_64-linux - 24 5 big-parallel"' # _JUST_NIX_INSTANTIATE_ARGS='--system darwin' -@build-packages +$attrpaths: +@build-packages +$attrpaths="": test -n "${attrpaths:-}" || set -- $(just _some_packages); \ cd "$invokedir"; just _build_packages results "$@" -@_build_packages $outdir +$attrpaths: +[no-cd] +_build_packages $outdir +$attrpaths: + #!/usr/bin/env -S bash -euo pipefail # hint: supports _JUST_NIX_BUILD_ARGS # hint: supports _JUST_NIX_INSTANTIATE_ARGS - test -z "${_JUST_NIX_BUILD_ARGS:-}" || >&2 echo "_JUST_NIX_BUILD_ARGS = $_JUST_NIX_BUILD_ARGS" - test -z "${_JUST_NIX_INSTANTIATE_ARGS:-}" || >&2 echo "_JUST_NIX_INSTANTIATE_ARGS = $_JUST_NIX_INSTANTIATE_ARGS" - # -test -d "$invokedir"/"$outdir" && rm -v "$invokedir"/"$outdir"/* - -test -d "$invokedir"/"$outdir" && { \ - fd . "$invokedir"/"$outdir" --type l -X rm -v; \ - fd \\\.log$ "$invokedir"/"$outdir" --type f -X rm -v; \ + test -z "${_JUST_NIX_BUILD_ARGS:-}" || echo >&2 "_JUST_NIX_BUILD_ARGS = $_JUST_NIX_BUILD_ARGS" + test -z "${_JUST_NIX_INSTANTIATE_ARGS:-}" || echo >&2 "_JUST_NIX_INSTANTIATE_ARGS = $_JUST_NIX_INSTANTIATE_ARGS" + test -d "$outdir" && { + fd . "$outdir" --type l -X rm -v + fd \\\.log$ "$outdir" --type f -X rm -v } - mkdir -p "$invokedir"/"$outdir" - cd "$invokedir"; \ - export _INSTANTIATE_CACHE=$(mktemp -d); \ - export NOTFOUNDDIR="$(mktemp -d)"; touch "$NOTFOUNDDIR"/not-found; \ - shift; \ - just instantiate-packages "$@" | tee >(cat>&2) | eval xargs nom-build --keep-going --no-out-link ${_JUST_NIX_BUILD_ARGS:-}; \ - for attrpath in "$@"; do ( \ - NIXPKGS_ALLOW_UNFREE=1 nix-build . -A "$attrpath" ${_JUST_NIX_INSTANTIATE_ARGS:-} -j0 --option builders "" --option substitute false -o "$outdir"/"$attrpath" 2>/dev/null >/dev/null; \ - if test -L "$outdir"/"$attrpath"; then \ - nix log $(readlink "$invokedir"/"$outdir"/"$attrpath") > "$outdir"/"$attrpath".log 2>/dev/null; \ - else \ - if [[ -z "$(just instantiate-packages "$attrpath")" ]]; then \ - ln -s "$NOTFOUNDDIR"/not-found "$outdir"/"$attrpath"; \ - else \ - ln -s /build-failure-"$attrpath" "$outdir"/"$attrpath"; \ - nix log ".#$attrpath" ${_JUST_NIX_INSTANTIATE_ARGS:-} > "$outdir"/"$attrpath".log 2>/dev/null || true; \ - fi \ - fi \ - ) & \ - while test "$(jobs -p | wc -l)" -ge 8; do wait < <(jobs -p); done; \ - done; wait - echo -e "#!/usr/bin/env bash\ntest -f flake.nix || cd ..; env _JUST_NIX_INSTANTIATE_ARGS=\"\$*\" just _build_packages \"$outdir\"" $attrpaths > "$invokedir"/"$outdir"/_rerun.sh - chmod +x "$invokedir"/"$outdir"/_rerun.sh - cd "$invokedir"; fd . "$outdir" -l + mkdir -p "$outdir" + export _INSTANTIATE_CACHE=$(mktemp -d) + export NOTFOUNDDIR="$(mktemp -d)"; touch "$NOTFOUNDDIR"/no-eval + shift + just instantiate-packages "$@" | tee >(command cat >&2) \ + | eval xargs nom-build --keep-going --no-out-link ${_JUST_NIX_BUILD_ARGS:-} + worker() { + local attrpath="$1" + local dst="$outdir/$attrpath" + local drvpath=$( just instantiate-packages "$attrpath" ) + # NIXPKGS_ALLOW_UNFREE=1 nix-build . -A "$attrpath" ${_JUST_NIX_INSTANTIATE_ARGS:-} -j0 --option builders "" --option substitute false -o "$dst" >&/dev/null + if [[ -n "$drvpath" ]]; then + NIXPKGS_ALLOW_UNFREE=1 nix-build "$drvpath" -j0 --option builders "" --option substitute false -o "$dst" >&/dev/null + fi + # local outpaths=$( nix-store -q --outputs "$drvpath" ) # doesn't gcroot, some outputs may not be pulled from builders + if [[ -L "$dst" ]]; then + nix log $(readlink "$dst") > "$dst".log 2>/dev/null + else + if [[ -z "$drvpath" ]]; then + ln -s "$NOTFOUNDDIR"/no-eval "$dst" + else + ln -s /build-failure-"$attrpath" "$dst" + nix log ".#$attrpath" ${_JUST_NIX_INSTANTIATE_ARGS:-} > "$dst".log 2>/dev/null || true + fi + fi + } + printf "%s\n" "$@" | xe -j0 -s "$(declare -f worker); worker \"\$@\"" + # echo -e "#!/usr/bin/env bash\ntest -f flake.nix || cd ..; env _JUST_NIX_INSTANTIATE_ARGS=\"\$*\" just _build_packages \"$outdir\"" $attrpaths > "$outdir"/_rerun.sh + cat <<-EOF >"$outdir"/_rerun.sh + #!/usr/bin/env bash + while [[ ! -f flake.nix ]]; do + cd .. + done + _JUST_NIX_INSTANTIATE_ARGS="\$*" just _build_packages "$outdir" $attrpaths + EOF + chmod +x "$outdir"/_rerun.sh + fd . "$outdir" -l @build-pr-packages $pr=`just _a_pr` $merge=`gum choose --header="which refs/pull//xxx ?" merge head` $attrpaths=`just _some_packages` $args=`gum choose --no-limit -- -j0 "--system "{aarch64,x86_64}-{darwin,linux} --rebuild`: -printf "github:nixos/nixpkgs/refs/pull/"$(just _sanitize_pr_url "$pr")"/$merge#%s" $attrpaths | xargs echo + nom build $args @@ -157,110 +192,148 @@ export GIT := `command -v git-wait >/dev/null && echo "git-wait" || echo "git"` # === commit helpers === #@commit-with-pkgname: -# cd "$invokedir"; test -n "$(git diff HEAD --name-only --staged)" || { >&2 echo "No files are staged!"; false; } +# cd "$invokedir"; test -n "$(git diff HEAD --name-only --staged)" || { echo >&2 "No files are staged!"; false; } # cd "$invokedir"; $GIT -c commit.template=<(echo "$(just list-packages | grep --fixed-strings "$(git diff HEAD --name-only --staged)" | cut -f1 | head -n1): ") commit -@commit-dirty-packages +$message=`gum input --placeholder="commit message, (attrpath: this message)"`: - test -n "$message" - (cd "$invokedir"; just list-dirty-packages) | \ - while read attrpath position row; do \ - test -f "$invokedir/$position" || continue; \ - ( cd "$invokedir"; $GIT diff -s --exit-code "$position" || ( set -x; $GIT add "$position" ; $GIT commit -m "$attrpath: $*" ) ); \ +[no-cd] +commit-dirty-packages +$message=`gum input --placeholder="commit message, (attrpath: this message)"`: + #!/usr/bin/env -S bash -euo pipefail + [[ -n "$message" ]] + just list-dirty-packages | + while read attrpath position row; do + [[ -f "$position" ]] || continue + $GIT diff -s --exit-code "$position" || ( + set -x + $GIT add "$position" + $GIT commit -m "$attrpath: $*" + ) done +[no-cd] @fixup-commit: + #!/usr/bin/env -S bash -euo pipefail # TODO: list commits first - cd "$invokedir"; test -n "$($GIT diff --name-only --staged)" || { \ - $GIT ls-files --modified --others --exclude-standard | grep -v ^results- | fzf --layout=reverse --multi --height 15 | xe $GIT add --patch; \ + test -n "$($GIT diff --name-only --staged)" || { + $GIT ls-files --modified --others --exclude-standard | + grep -v ^results- | + fzf --layout=reverse --multi --height 15 | + xe $GIT add --patch } - cd "$invokedir"; test -n "$($GIT diff --name-only --staged)" || { >&2 echo Nothing is staged!; false; } - cd "$invokedir"; just fixup-commit-staged + test -n "$($GIT diff --name-only --staged)" || { + echo >&2 Nothing is staged! + false + } + just fixup-commit-staged -@fixup-commit-staged commit=`cd "$invokedir"; just _a_commit`: +fixup-commit-staged commit=`cd "$invokedir"; just _a_commit`: + #!/usr/bin/env -S bash -euo pipefail test -n "{{ commit }}" # just exit if no selection was made - cd "$invokedir"; test -n "$($GIT diff --name-only --staged)" || { >&2 echo Nothing is staged!; false; } - cd "$invokedir"; $GIT diff --staged + test -n "$($GIT diff --name-only --staged)" || { echo >&2 Nothing is staged!; false; } + $GIT diff --staged echo; gum format "# ========= into ========="; echo - cd "$invokedir"; $GIT log {{ commit }} -n1 - @gum confirm - cd "$invokedir"; $GIT commit --fixup={{ commit }} - cd "$invokedir"; GIT_SEQUENCE_EDITOR=true $GIT rebase -i --autostash --autosquash "$($GIT log -n1 --pretty=%P {{ commit }})" + $GIT log {{ commit }} -n1 + gum confirm + $GIT commit --fixup={{ commit }} + GIT_SEQUENCE_EDITOR=true $GIT rebase -i --autostash --autosquash "$($GIT log -n1 --pretty=%P {{ commit }})" rebase-commits commit=`cd "$invokedir"; just _a_commit`: cd "$invokedir"; $GIT rebase -i --autostash --autosquash "$($GIT log -n1 --pretty=%P {{ commit }})" # === speed worktrees === -bump +$packages=`just _some_packages`: - cd "$(just _new_worktree bump "$(printf "%s\n" $packages | head -n1 | rev | cut -d. -f1 | rev)")"; just _bump $packages; NIXPKGS_ALLOW_UNFREE=1 $SHELL +@bump +$packages=`just _some_packages`: + cd "$(just _new_worktree bump "$(printf "%s\n" $packages | head -n1 | rev | cut -d. -f1 | rev)")"; \ + just _bump $packages; \ + NIXPKGS_ALLOW_UNFREE=1 "$SHELL" _bump +$packages: - test -e "$invokedir"/.git - -cd "$invokedir"; printf "%s.src\n" "$@" | xargs just _build_packages results-src-old # TODO: unpack? - cd "$invokedir"; for package in "$@"; do \ - if nix eval -f . "$package".passthru.updateScript >&/dev/null; then \ - HEAD=$($GIT rev-parse HEAD); \ - NIXPKGS_ALLOW_UNFREE=1 NIXPKGS_ALLOW_BROKEN=1 NIXPKGS_ALLOW_INSECURE=1 nix-update "$package" --use-update-script --update-script-args "--argstr skip-prompt true" --commit; \ - if [[ $($GIT rev-parse HEAD) = "$HEAD" ]]; then \ - echo "no commit found, trying without the updateScript..."; \ - $GIT restore .; \ - NIXPKGS_ALLOW_UNFREE=1 NIXPKGS_ALLOW_BROKEN=1 NIXPKGS_ALLOW_INSECURE=1 nix-update "$package" --commit || true; \ - fi \ - else \ - NIXPKGS_ALLOW_UNFREE=1 NIXPKGS_ALLOW_BROKEN=1 NIXPKGS_ALLOW_INSECURE=1 nix-update "$package" --commit || true; \ - fi \ + #!/usr/bin/env -S bash -euo pipefail + test -e .git + printf "%s.src\n" "$@" | xargs just _build_packages results-src-old ||; # TODO: unpack? + config=( + NIXPKGS_ALLOW_UNFREE=1 + NIXPKGS_ALLOW_BROKEN=1 + NIXPKGS_ALLOW_INSECURE=1 + ) + for package in "$@"; do + if nix eval -f . "$package".passthru.updateScript >&/dev/null; then + HEAD=$($GIT rev-parse HEAD) + "${config[@]}" nix-update "$package" --use-update-script --update-script-args "--argstr skip-prompt true" --commit + if [[ $($GIT rev-parse HEAD) = "$HEAD" ]]; then + echo "no commit found, trying without the updateScript..." + $GIT restore . + "${config[@]}" nix-update "$package" --commit || true + fi + else + "${config[@]}" nix-update "$package" --commit || true + fi done - -cd "$invokedir"; printf "%s.src\n" "$@" | xargs just _build_packages results-src-new # TODO: unpack? - -cd "$invokedir"; just _build_packages results "$@" - @# TODO: filter non-existing tests: - -cd "$invokedir"; printf "%s.tests\n" "$@" | env _JUST_NIX_BUILD_ARGS="-j1" xargs just _build_packages results-tests - -cd "$invokedir"; printf "HEAD^%s " $(seq 0 $(($#-1))) | xe $GIT show - -cd "$invokedir"; fd . -l results results-tests --max-depth 1 - # -cd "$invokedir"; test -L result && fd . result --type x --color=always | tee update-executables.txt - # -cd "$invokedir"; $GIT show --name-only --pretty="" | xe statix fix - cd "$invokedir"; $GIT diff + set +e + printf "%s.src\n" "$@" | xargs just _build_packages results-src-new # TODO: unpack? + just _build_packages results "$@" + # TODO: filter non-existing tests: + printf "%s.tests\n" "$@" | env _JUST_NIX_BUILD_ARGS="-j1" xargs just _build_packages results-tests + printf "HEAD^%s " $(seq 0 $(($#-1))) | xe $GIT show + fd . -l results results-tests --max-depth 1 + # test -L result && fd . result --type x --color=always | tee update-executables.txt + # $GIT show --name-only --pretty="" | xe statix fix + $GIT diff printf "delta results-src-{old,new}/%s/.\n" "$@" #fix package=`just _a_package`: # ./mk-fix.sh "{{package}}" @fix $package=`just _a_package`: - cd "$(just _new_worktree fix "$(echo "$package" | rev | cut -d. -f1 | rev)")"; just _fix "$package" && $SHELL + cd "$(just _new_worktree fix "$(echo "$package" | rev | cut -d. -f1 | rev)")"; \ + just _fix "$package" && $SHELL -@_fix $package: - test -e "$invokedir"/.git - cd "$invokedir"; nix-build . -A "$package".src --out-link result-src - -cd "$invokedir"; test -d $(readlink result-src) && { mkdir -p result-src-writeable; rsync -rxL result-src/ result-src-writeable/ ; chmod -R +w result-src-writeable ;} - -cd "$invokedir"; test -f $(readlink result-src) && { \ - mkdir result-src-writeable; \ - atool $(readlink result-src) --extract-to=result-src-writeable; \ - if test $(ls result-src-writeable | wc -l ) -eq 1 && test -d result-src-writeable/* ; then \ - mv result-src-writeable{,-old}; \ - mv result-src-writeable-old/* result-src-writeable; \ - rmdir result-src-writeable-old; \ - fi \ - } - -test -d "$invokedir"/result-src-writeable; cd "$invokedir"/result-src-writeable; $GIT init ; $GIT add . ; $GIT commit -m "Base" >/dev/null - { \ - echo '#!/usr/bin/env -S nom build -f'; \ - echo 'let pkgs = import ./. {}; in pkgs.{{ package }}.overrideAttrs { src = ./result-src-writeable; }'; \ - } > "$invokedir"/build.nix - chmod +x "$invokedir"/build.nix - cd "$invokedir"; NIXPKGS_ALLOW_UNFREE=1 just build-packages "$package" || true +[no-cd] +_fix $package: + #!/usr/bin/env -S bash -euo pipefail + test -e .git + nix-build . -A "$package".src --out-link result-src + test -d $(readlink result-src) && { + mkdir -p result-src-writeable + rsync -rxL result-src/ result-src-writeable/ + chmod -R +w result-src-writeable + } ||: + test -f $(readlink result-src) && { + mkdir result-src-writeable + atool $(readlink result-src) --extract-to=result-src-writeable + if test $(ls result-src-writeable | wc -l ) -eq 1 && test -d result-src-writeable/* ; then + mv result-src-writeable{,-old} + mv result-src-writeable-old/* result-src-writeable + rmdir result-src-writeable-old + fi + } ||: + test -d result-src-writeable && ( + cd result-src-writeable + $GIT init + $GIT add . + $GIT commit -m "Base" >/dev/null + ) + cat <<-"EOF" > build.nix + #!/usr/bin/env -S nom build -f + let pkgs = import ./. {}; in pkgs.{{ package }}.overrideAttrs { src = ./result-src-writeable; } + EOF + chmod +x build.nix + NIXPKGS_ALLOW_UNFREE=1 just build-packages "$package" ||: echo "# $package" echo ' ./build.nix' echo ' ./results/_rerun.sh' -@pr $number=`just _a_pr`: - if test -d prs/pr-"$(just _sanitize_pr_url "$number")"-* ; then \ - cd prs/pr-"$(just _sanitize_pr_url "$number")"-* && $SHELL ; \ - else \ - just _pr "$(just _sanitize_pr_url "$number")" ; \ +pr $number=`just _a_pr`: + #!/usr/bin/env -S bash -euo pipefail + number=$(just _sanitize_pr_url "$number") + if test -d prs/pr-"$number"-* ; then + cd prs/pr-"$number"-* && $SHELL + else + cd "$(just _new_worktree prs pr-{{ number }} "upstream/master" "")" + gh pr checkout {{ number }} + $GIT branch -D prs-$(basename "$PWD") + $SHELL fi -_pr $number: - cd "$(just _new_worktree prs pr-{{ number }} "upstream/master" "")" ; gh pr checkout {{ number }} && $GIT branch -D prs-$(basename "$PWD") && $SHELL - @pr-comment-diff: cd "$invokedir"; just _pr_post_diff comment @@ -295,9 +368,14 @@ _pr $number: # TODO: clean -@remote-branch $remote_ref=`just _a_remote_branch origin | cut -d/ -f2-`: - cd master; $GIT worktree add --track ../"$(cut <<<"$remote_ref" -d- -f1)"/"$(cut <<<"$remote_ref" -d- -f2-)" -b "$remote_ref" origin/$remote_ref - cd "$(cut <<<"$remote_ref" -d- -f1)"/"$(cut <<<"$remote_ref" -d- -f2-)"; $SHELL +remote-branch $remote_ref=`just _a_remote_branch origin | cut -d/ -f2-`: + #!/usr/bin/env -S bash -euo pipefail + branch_dir=$(cut <<<"$remote_ref" -d- -f1) + branch_base=$(cut <<<"$remote_ref" -d- -f2-) + cd master + $GIT worktree add --track ../"$branch_dir/$branch_base" -b "$remote_ref" origin/"$remote_ref" + cd ../"$branch_dir/$branch_base" + $SHELL # === worktrees === # TODO: onto release-xx.yy doesn't work @@ -306,48 +384,87 @@ _pr $number: @new-worktree *args: cd "$(just _new_worktree "$@")"; $SHELL -@_new_worktree $type=`gum choose feat fix init bump doc migrate` $name=`gum input --placeholder="Branch name?"` $base=`just _a_upstream_release_branch "Base?"` $onto=`just _a_upstream_release_branch "Onto?" ""`: - test -n "$type" - test -n "$name" - just _mk_worktree "$type-$(printf "%s" "$name" | tr '[:space:]/' -- | tr -s -)-{{ epoch }}" "$type/$(printf "%s" "$name" | tr '[:space:]/' -- | tr -s -)-{{ epoch }}" "$base" "$onto" 1>&2 - echo "$type/$(printf "%s" "$name" | tr '[:space:]/' -- | tr -s -)-{{ epoch }}" +@_a_wt_type: + gum choose feat fix init bump doc migrate +@_a_wt_name: + gum input --placeholder="Branch name?" +_new_worktree $type=`just _a_wt_type` $name=`just a_wt_name` $base=`just _a_upstream_release_branch "Base?"` $onto=`just _a_upstream_release_branch "Onto?" ""`: + #!/usr/bin/env -S bash -euo pipefail + [[ -n "$type" ]] + [[ -n "$name" ]] + name="$(printf "%s" "$name" | tr '[:space:]/' -- | tr -s -)" + just _mk_worktree 1>&2 \ + "$type-$name-{{ epoch }}" \ + "$type/$name-{{ epoch }}" \ + "$base" \ + "$onto" + echo "$type/$name-{{ epoch }}" -@_mk_worktree $name $dir $base="upstream/master" $onto="": - test -n "$name" - test -n "$dir" - cd master; $GIT worktree prune - cd master; $GIT fetch "$(cut -d/ -f1 <<<"$base")" "$(cut -d/ -f2 <<<"$base")" - cd master; $GIT branch "$name" "$(cut -d/ -f2 <<<"$base")" --no-track - cd master; $GIT worktree add ../"$dir" "$name" - cd "$dir"; $GIT pull "$(cut -d/ -f1 <<<"$base")" "$(cut -d/ -f2 <<<"$base")" - test -z "$onto" || { cd "$dir"; just rebase-onto HEAD "$onto"; } +_mk_worktree $name $dir $base="upstream/master" $onto="": + #!/usr/bin/env -S bash -euo pipefail -x + [[ -n "$name" ]] + [[ -n "$dir" ]] + remote="$(cut -d/ -f1 <<<"$base")" + branch="$(cut -d/ -f2- <<<"$base")" + cd master + $GIT worktree prune + $GIT fetch "$remote" "$branch" + $GIT branch "$name" "$branch" --no-track + $GIT worktree add ../"$dir" "$name" + cd ../"$dir" + $GIT pull "$remote" "$branch" + test -z "$onto" || just rebase-onto HEAD "$onto" -@pop-commits-to-new-worktree $commits=`cd "$invokedir"; just _some_commits`: - test -n "$commits" || { >&2 echo "no commits were selected!"; false;} +pop-commits-to-new-worktree $commits=`cd "$invokedir"; just _some_commits`: + #!/usr/bin/env -S bash -euo pipefail + test -n "$commits" || { echo >&2 "no commits were selected!"; false;} # we make a nasty assumption here: the last commit passed in is also the topologically oldest one in the git history # we also assume the revs in the interactive rebase are just as short or longer as those returned by 'just _some_commits' - new_worktree="$(just _new_worktree "$(gum choose feat fix init bump doc migrate)" "$(gum input --placeholder="Branch name?" --value="$(cd "$invokedir"; git show --oneline --no-patch "$(head -n1 <<<"$commits")" | cut -d' ' -f2- | cut -d: -f1 | rev | cut -d. -f1 | rev)")" )" && \ - (cd "$new_worktree"; tac <<<"$commits" | xargs git cherry-pick) \ - && { \ - (cd "$invokedir"; EDITOR="sed -i $(xargs <<<"$commits" printf " -e 's/^pick %s/#/g'")" git rebase "$(tail -n1 <<<"$commits")"^^ -i) \ - cd "$new_worktree" && git log -n "$(wc -l <<<"$commits" )" && $SHELL; \ - true; \ - } \ - || { \ - >&2 echo "The commits have not been popped."; \ - cd "$new_worktree" && git status; \ - >&2 echo "The commits have not been popped."; \ - $SHELL; \ - } + type="$(just _a_wt_type)" + name="$( + cd "$invokedir"; + git show --oneline --no-patch "$(head -n1 <<<"$commits")" | + cut -d' ' -f2- | cut -d: -f1 | rev | cut -d. -f1 | rev + )" + cd "$( + just _new_worktree "$type" "$(gum input --placeholder="Branch name?" --value="$name")" + )" + if (tac <<<"$commits" | xargs git cherry-pick); then + ( + cd "$invokedir" + EDITOR="sed -i $(xargs <<<"$commits" printf " -e 's/^pick %s/#/g'")" \ + git rebase "$(tail -n1 <<<"$commits")"^^ -i + ) + git log -n "$(wc -l <<<"$commits" )" + $SHELL + else + echo >&2 "The commits have not been popped." + git status + echo >&2 "The commits have not been popped." + $SHELL + fi +[no-cd] @_rebase_onto $onto: - test -n "$onto" - cd "$invokedir"; $GIT fetch upstream "$onto" ; $GIT rebase --onto upstream/"$onto"...HEAD upstream/master --autostash + [[ -n "$onto" ]] + $GIT fetch upstream "$onto" + $GIT rebase --onto upstream/"$onto"...HEAD upstream/master --autostash +[no-cd] @rebase-onto $base=`just _a_upstream_release_branch "Base?" HEAD` $onto=`just _a_upstream_release_branch "Onto?"`: - cd "$invokedir"; test "$base" = HEAD || { remote="$(cut -d/ -f1 <<<"$base")"; branch="$(cut -d/ -f2- <<<"$base")"; set -x; $GIT fetch "${remote:-upstream}" "$branch"; } - cd "$invokedir"; test "$onto" = HEAD || { remote="$(cut -d/ -f1 <<<"$onto")"; branch="$(cut -d/ -f2- <<<"$onto")"; set -x; $GIT fetch "${remote:-upstream}" "$branch"; } - #cd "$invokedir"; set -x; $GIT rebase --onto upstream/"$onto"...upstream/"$base" upstream/"$base" --autostash + #!/usr/bin/env -S bash -euo pipefail + if [[ "$base" != HEAD ]] ; then + remote="$(cut -d/ -f1 <<<"$base")" + branch="$(cut -d/ -f2- <<<"$base")" + (set -x; $GIT fetch "${remote:-upstream}" "$branch") + fi + if [[ "$onto" != HEAD ]] ; then + remote="$(cut -d/ -f1 <<<"$onto")" + branch="$(cut -d/ -f2- <<<"$onto")" + (set -x; $GIT fetch "${remote:-upstream}" "$branch") + fi + # $GIT rebase --onto upstream/"$onto"...HEAD upstream/master --autostash + $GIT rebase --onto "$onto"..."$base" $base --autostash # fix periodic merge conflicts @@ -366,13 +483,35 @@ fix-periodic-merge-conflict $from=`just _a_upstream_release_branch "From?"` $to= # === setup === setup: - test -e master/.git || $GIT clone "git@github.com:$(gum input --header "$GIT clone git@github.com:< username >/nixpkgs.git" --value="$(whoami)")/nixpkgs.git" master - test -e master/.git - cd master; $GIT remote get-url upstream >/dev/null || $GIT remote add upstream "https://github.com/NixOS/nixpkgs.git" - test -d upstream/staging || ( cd master && $GIT worktree add ../upstream/staging -B staging upstream/staging ) - test -d upstream/staging-next || ( cd master && $GIT worktree add ../upstream/staging-next -B staging-next upstream/staging-next ) - test -d upstream/staging-24.11 || ( cd master && $GIT worktree add ../upstream/staging-24.11 -B staging-24.11 upstream/staging-24.11 ) - test -d upstream/staging-next-24.11 || ( cd master && $GIT worktree add ../upstream/staging-next-24.11 -B staging-next-24.11 upstream/staging-next-24.11 ) + #!/usr/bin/env -S bash -euo pipefail + if [[ ! -e master/.git ]]; then + gh_user=$(gum input \ + --header "git clone git@github.com:< username >/nixpkgs.git" \ + --value="$( + if command -v gh >/dev/null && gh auth status >&/dev/null; then + gh api user --jq .login + else + whoami + fi + )" + ) + $GIT clone "git@github.com:${gh_user}/nixpkgs.git" master + [[ -e master/.git ]] + fi + cd master + if ! $GIT remote get-url upstream >&/dev/null; then + $GIT remote add upstream "https://github.com/NixOS/nixpkgs.git" + fi + add_upstream() { + local branch="$1" + if [[ ! -d ../upstream/"$branch" ]]; then + $GIT worktree add ../upstream/"$branch" -B "$branch" upstream/"$branch" + fi + } + add_upstream staging + add_upstream staging-next + add_upstream staging-24.11 + add_upstream staging-next-24.11 # TODO: test #upstreams-fetch: @@ -387,23 +526,34 @@ setup: # === helpers === # TODO stat the file and redo if too old, or tie it to the master checkout commit -# TODO: dump it in /tmp? -@_packages_json: - ( \ - test -s packages.json \ - && test $(( "$epoch" - "$(stat -c %Y packages.json )" )) -lt $(( 60*60*24*7 )) \ - ) || { set -x; \ - (cd master; git diff --exit-code --quiet) || { \ - >&2 echo "ERROR: $(realpath master) is dirty!"; \ - false; \ - }; \ - (cd master; git pull upstream master); \ - time nix-env --extra-experimental-features no-url-literals --option system x86_64-linux -f ./master -qaP --json --meta --show-trace --no-allow-import-from-derivation --arg config '{ allowAliases = false; }' > packages.json; \ - } +_packages_json: + #!/usr/bin/env -S bash -euo pipefail + if [[ ! -s packages.json || + $(( "$epoch" - "$(stat -c %Y packages.json )" )) -gt $(( 60*60*24*7 )) + ]]; then + set -x + if ! $GIT -C master/ diff HEAD --exit-code --quiet; then + echo >&2 "ERROR: $(realpath master) is dirty!" + fi + git -C master/ pull upstream master + time nix-env > packages.json \ + --extra-experimental-features no-url-literals \ + --option system x86_64-linux \ + -f ./master \ + -qaP \ + --json \ + --meta \ + --show-trace \ + --no-allow-import-from-derivation \ + --arg config '{ allowAliases = false; }' + fi -@_a_commit: - # TODO: check if in a git repo - cd "$invokedir"; $GIT log --oneline -n800 | gum filter --placeholder "Pick commit..." --height 10 | cut -d' ' -f1 +@_a_commit *extra_revs: + # TODO: check if in a nixpkgs git repo + { \ + test $# -eq 0 || printf "%s\n" "$@"; \ + cd "$invokedir"; $GIT log --oneline -n800; \ + } | gum filter --placeholder "Pick commit..." --height 10 | cut -d' ' -f1 @_some_commits: # TODO: check if in a git repo