Files
just-nixpkgs/overlays.nix
T
2025-06-12 15:45:26 +02:00

631 lines
22 KiB
Nix
Executable File

#!/usr/bin/env -S nix-instantiate --parse
let
lib = import ./master/lib;
forBuildHost = p: p.__spliced.buildHost or p;
# merge = sets: lib.foldl' lib.recursiveUpdate {} sets;
appendIfMissing = list: val: if builtins.elem val list then list else list ++ [ val ];
mkDerivationOverlay =
let
# from pkgs/stdenv/adapters.nix
# via https://github.com/caffineehacker/nix/blob/35601d216ccaf4c2705294281403e6797be082d0/machines/framework/configuration.nix#L42
# simplified to require addAttrsToDerivation
withOldMkDerivation =
stdenvSuperArgs: k: stdenvSelf:
let
mkDerivationFromStdenv-super = stdenvSuperArgs.mkDerivationFromStdenv;
mkDerivationSuper = mkDerivationFromStdenv-super stdenvSelf;
in
k stdenvSelf mkDerivationSuper;
# from pkgs/stdenv/adapters.nix
extendMkDerivationArgs =
old: f:
withOldMkDerivation old (
_: mkDerivationSuper: args:
(mkDerivationSuper args).overrideAttrs f
);
# TODO: need we override any other stdenvs?
mkStdenvOverlay = f: final: prev: {
stdenv = (prev.addAttrsToDerivation { } prev.stdenv).override (
f final prev prev.stdenv.mkDerivation
);
# stdenvNoCC = (prev.addAttrsToDerivation { } prev.stdenvNoCC).override (f final prev prev.stdenvNoCC.mkDerivation);
# stdenvLLVM = (prev.addAttrsToDerivation { } prev.stdenvLLVM).override (f final prev prev.stdenvLLVM.mkDerivation);
};
in
f:
mkStdenvOverlay (
final: prev: prevMkDerivation: stdenv: {
# mkDerivation = lib.extendMkDerivation {
# constructDrv = prevStdenv.mkDerivation;
# extendDrvArgs = f final prev stdenv;
# };
mkDerivationFromStdenv = extendMkDerivationArgs stdenv (
finalAttrs: prevAttrs:
lib.optionalAttrs (!(lib.strings.hasInfix "bootstrap" stdenv.name)) (
let
stdenv' = stdenv // {
# mkDerivation = stdenv.mkDerivationFromStdenv stdenv;
mkDerivation = prevMkDerivation; # HACK
};
in
f final prev stdenv' finalAttrs prevAttrs
)
);
}
);
# none of these overlays should cause a mass-rebuild, they should only add optional functionality
# TODO: reduce eval time addition, any faster ways to override mkDerivation?
# TODO: mkDerivation.passthru.withOil: add <package>.withOil which run build with osh, https://oils.pub/osh.html#2-you-get-precise-error-messages
# TODO: mkDerivation.passthru.withGDB: add <package>.withGDB which runs the full build with gdb
# TODO: mkDerivation.passthru.withDebInfo: like overlay.withDebug but with cmakeBuildType = "RelWithDebInfo" mode
# usage: nix-build . -A <package>.withDebug
overlays.withDebug = mkDerivationOverlay (
final: prev: prevStdenv: finalAttrs: prevAttrs: {
passthru = {
withDebug = prevStdenv.mkDerivation (
prevAttrs
// {
separateDebugInfo = false;
dontStrip = true;
# TODO: more debug flags for more builders
cmakeBuildType = "Debug";
}
);
} // prevAttrs.passthru or { };
}
);
# usage: nix-build . -A <package>.withoutChecks
overlays.withoutChecks = mkDerivationOverlay (
final: prev: prevStdenv: finalAttrs: prevAttrs: {
passthru = {
withoutChecks = prevStdenv.mkDerivation (
prevAttrs
// {
doCheck = false;
doInstallCheck = false;
# stub phases and inputs to reduce closure, in case the builder doesn't do this correctly
checkInputs = [];
nativeCheckInputs = [];
preCheck = ":";
postCheck = ":";
checkPhase = ":";
preInstallCheck = ":";
postInstallCheck = ":";
installCheckPhase = ":";
}
);
} // prevAttrs.passthru or { };
}
);
# usage: nix-build . -A <package>.srcOnly
overlays.srcOnly = mkDerivationOverlay (
final: prev: prevStdenv: finalAttrs: prevAttrs: {
passthru = {
srcOnly = prev.srcOnly finalAttrs.finalPackage;
} // prevAttrs.passthru or { };
}
);
# usage: nix-build . -A <package>.src.invalidated
overlays.withInvalidateFetcherByDrvHash = mkDerivationOverlay (
final: prev: prevStdenv: finalAttrs: prevAttrs: {
passthru = {
# invalidated = prevStdenv.mkDerivation prevAttrs;
invalidated = prev.invalidateFetcherByDrvHash prevStdenv.mkDerivation prevAttrs;
} // prevAttrs.passthru or { };
}
);
# https://github.com/NixOS/nixpkgs/pull/392938
# usage: nix-build . -A <package>.src.unpacked
overlays.withUnpacked = final: prev: {
fetchurl = (lib.mirrorFunctionArgs prev.fetchurl) (
args:
let
fetched = prev.fetchurl args;
in
fetched.overrideAttrs (old: {
passthru = {
unpacked =
if old.outputHashMode == "recursive" then
fetched
else
prev.srcOnly {
name = "${fetched.name}-unpacked";
src = fetched;
stdenv = prev.stdenvNoCC;
inherit (old) preferLocalBuild;
};
} // old.passthru or { };
})
);
fetchgit = (lib.mirrorFunctionArgs prev.fetchgit) (
args:
(prev.fetchgit args).overrideAttrs (old: {
passthru = {
unpacked = prev.fetchgit args;
} // old.passthru or { };
})
);
fetchhg = (lib.mirrorFunctionArgs prev.fetchhg) (
args:
(prev.fetchhg args).overrideAttrs (old: {
passthru = {
unpacked = prev.fetchhg args;
} // old.passthru or { };
})
);
fetchsvn = (lib.mirrorFunctionArgs prev.fetchsvn) (
args:
(prev.fetchsvn args).overrideAttrs (old: {
passthru = {
unpacked = prev.fetchsvn args;
} // old.passthru or { };
})
);
};
# usage: nix-build . -A <package>.tests.mainProgram
overlays.withTestMainProgram = mkDerivationOverlay (
final: prev: prevStdenv: finalAttrs: prevAttrs: {
passthru = prevAttrs.passthru or { } // {
tests =
lib.optionalAttrs ((prevAttrs.meta or { }) ? mainProgram) {
mainProgram = final.runCommandLocal "${finalAttrs.finalPackage.name}-check-mainProgram" { } ''
fname=${lib.getBin finalAttrs.finalPackage}/bin/${lib.escapeShellArg finalAttrs.finalPackage.meta.mainProgram}
if [[ -x "$fname" ]]; then
echo "Found $fname"
touch $out
elif [[ -f "$fname" ]]; then
echo >&2 "ERROR: '$fname' is not executable!"
false
elif [[ -d "$fname" ]]; then
echo >&2 "ERROR: '$fname' is a directory!"
false
else
echo >&2 "ERROR: '$fname' not found!"
(set -x; find ${lib.getBin finalAttrs.finalPackage}/bin -type f)
false
fi
'';
}
// (prevAttrs.passthru.tests or { });
};
}
);
# usage: nix-build . -A <package>.shellcheck
overlays.withShellCheck = mkDerivationOverlay (
final: prev: prevStdenv: finalAttrs: prevAttrs:
let
snippets = lib.pipe finalAttrs [
(lib.filterAttrs (
name: val:
lib.hasPrefix "pre" name
|| lib.hasPrefix "post" name
|| lib.hasSuffix "Phase" name
|| builtins.elem name (finalAttrs.passAsFile or [ ])
))
(lib.filterAttrs (name: val: lib.isString val && val != ""))
];
files = [
# "${finalAttrs.finalPackage.stdenv}/setup" # noisy
# finalAttrs.finalPackage.builder # usually a binary
];
in
{
passthru = {
shellcheck =
prev.runCommandNoCCLocal "${finalAttrs.pname}-shellcheck"
{
nativeBuildInputs = [ prev.shellcheck ];
}
''
declare -A snippets=(${
lib.concatLines (
lib.mapAttrsToList (name: val: "[${lib.escapeShellArg name}]=${lib.escapeShellArg val}") snippets
)
})
declare -a files=(${lib.escapeShellArgs files})
# set -x
mkdir -p $out/snippets
for snippetName in "''${!snippets[@]}"; do
snippet="''${snippets[$snippetName]}"
[[ -n "$snippet" ]] || continue
printf "snippet %q:\n" "$snippetName"
{
shellcheck --format tty --color=always --shell=bash - <<<"''$snippet" ||:
} | tee "$out/snippets/$snippetName"
done
for file in "''${files[@]}"; do
[[ -r "$file" ]] || continue
printf "file %q:\n" "$file"
mkdir -p "$out/files/$(dirname "$(realpath "$file")")"
{
shellcheck --format tty --color=always --shell=bash "$file" ||:
} | tee "$out/files/$(realpath "$file")"
done
'';
} // (prevAttrs.passthru or { });
}
);
# usage: nix-build ./dev.nix -A <package>.tests.checkLdd
overlays.withTestCheckLdd = mkDerivationOverlay (
final: prev: prevStdenv: finalAttrs: prevAttrs: {
passthru = prevAttrs.passthru or { } // {
tests = prevAttrs.tests or { } // {
checkLdd =
prev.runCommandLocal "${finalAttrs.finalPackage.name}-ldd"
{
nativeBuildInputs = [
prev.glibc.bin
prev.fd
];
}
''
worker() (
local output
set -u +e
output="$(ldd "$1" 2>&1)"
retcode="$?"
if [[ "$retcode" -ne 0 && "$output" != $'$\tnot a dynamic executable' ]]; then
echo "error: ldd $1 (retcode=$retcode)"
cat <<<"$output"
touch any_error
elif grep <<<"$output" -q " not found"; then
# TODO: show executable permission errors too
echo "error: ldd $1 (log detection)"
grep <<<"$output" " not found"
touch any_error
fi
)
fd_args=(
.
${
lib.pipe finalAttrs.finalPackage.outputs [
(map (output: finalAttrs.finalPackage.${output}))
lib.escapeShellArgs
]
}
--type f
--hidden
--no-ignore
--follow # symlinks
-x bash -c "$(declare -f worker); worker \"\$@\"" --
)
fd "''${fd_args[@]}"
if [[ ! -e any_error ]]; then
touch $out
fi
'';
};
};
}
);
# TODO: this way of overriding does not work
# Not a treewide enable, just for a single derivation
# usage: nix-build . -A <package>.withCuda
# usage: nix-build . -A <package>.withRocm
overlays.withCudaOrRocm = final: prev: {
lib = prev.lib.extend (
finalLib: prevLib: {
# makeOverridable :: (AttrSet -> a) -> AttrSet -> a
makeOverridable =
f: args:
let
fArgs = lib.functionArgs f;
# Creates a functor with the same arguments as f
mirrorArgs = lib.mirrorFunctionArgs f;
fWithCuda = mirrorArgs (
origArgs:
f' (
origArgs
// lib.optionalAttrs (fArgs ? cudaSupport) {
cudaSupport = true;
}
// lib.optionalAttrs (fArgs ? config) {
config = fArgs.config // {
cudaSupport = true;
};
}
)
);
fWithRocm = mirrorArgs (
origArgs:
f' (
origArgs
// lib.optionalAttrs (fArgs ? rocmSupport) {
rocmSupport = true;
}
// lib.optionalAttrs (fArgs ? config) {
config = fArgs.config // {
rocmSupport = true;
};
}
)
);
f' = mirrorArgs (
origArgs:
let
result = f origArgs;
in
if lib.isAttrs result then
if result ? overrideAttrs then
result.overrideAttrs (old: {
passthru = {
withCuda = fWithCuda origArgs;
withRocm = fWithRocm origArgs;
} // old.passthru or { };
})
else
{
withCuda = fWithCuda origArgs;
withRocm = fWithRocm origArgs;
}
// result
else
result
);
in
prevLib.makeOverridable f' args;
}
);
};
# usage: nix-build . -A <package>.withCcache
overlays.withCcache =
let
ccacheConfig = ''
export CCACHE_DIR=/var/cache/ccache
export CCACHE_UMASK=007
export CCACHE_COMPRESS=1
# may break reproducibility or cause cache poisoning
# https://wiki.nixos.org/wiki/CCache#Sloppiness
# https://github.com/NixOS/nixpkgs/issues/109033
export CCACHE_SLOPPINESS=random_seed
if [[ ! -d "$CCACHE_DIR" ]]; then
echo >&2 "ERROR: \$CCACHE_DIR=$CCACHE_DIR not found!"
exit 1
elif [[ ! -w "$CCACHE_DIR" ]]; then
echo >&2 "ERROR: \$CCACHE_DIR=$CCACHE_DIR not writeable!"
exit 1
fi
'';
in
final: prev:
{
ccacheWrapper = prev.ccacheWrapper.override {
extraConfig = ccacheConfig;
};
# TODO: swap to this scope if in cudaPackages withCcache
# cudaPackages' = final.cudaPackages.overrideScope (
# cudaFinal: cudaPrev: {
# cuda_nvcc = cudaPrev.cuda_nvcc.overrideAttrs (oldAttrs: {
# nativeBuildInputs = oldAttrs.nativeBuildInputs or [ ] ++ [ final.makeWrapper ];
# postFixup =
# oldAttrs.postFixup or ""
# + ''
# mv "$out/bin/nvcc" "$out/bin/.nvcc-no-ccache"
# args=(
# "${prev.lib.getExe (forBuildHost final.ccache)}"
# "$out/bin/nvcc"
# --run ${prev.lib.escapeShellArg ccacheConfig}
# --add-flags "$out/bin/.nvcc-no-ccache"
# )
# makeWrapper "''${args[@]}"
# '';
# });
# }
# );
# pythonPackagesExtensions = prev.pythonPackagesExtensions ++ [
# (pythonFinal: pythonPrev: {
# })
# ];
}
// (mkDerivationOverlay (final: prev: prevStdenv: finalAttrs: prevAttrs: {
passthru = {
# withCcache = prevStdenv.mkDerivation (
withCcache = prev.ccacheStdenv.mkDerivation (
{
requiredSystemFeatures = prevAttrs.requiredSystemFeatures or [ ] ++ [
"ccache"
#"sccache"
##"sccache-redis"
];
#RUSTC_WRAPPER = lib.getExe final.sccache;
#SCCACHE_DIR = "/var/cache/sccache";
#SCCACHE_CACHE_ZSTD_LEVEL = 19;
## SCCACHE_REDIS_ENDPOINT = "redis+unix://run/redis-nix-sccache/redis.sock";
## SCCACHE_IDLE_TIMEOUT = 0;
## SCCACHE_LOG = "debug";
## SCCACHE_NO_DAEMON = true;
}
// prevAttrs
);
} // prevAttrs.passthru or { };
}) final prev);
# very hacky, not guaranteed to work, but may save a lot of .whl rebuilds
# usage: nix-build . -A python3Packages.<package>.twostage
# usage: nix-build . -A python3Packages.<package>.twostage.first
overlays.withTwostagePythonBuildPassthru = final: prev: {
pythonPackagesExtensions = prev.pythonPackagesExtensions ++ [
(
pythonFinal: pythonPrev:
let
mkTwostage =
buildPythonPackage:
(
args:
buildPythonPackage (
args
// {
passthru = {
twostage =
let
# TODO: assert "dist" in finalPackage.outputs?
first = buildPythonPackage (
args
// {
name = "${args.name or "${args.pname}-${args.version}"}-wheels";
outputs = [ "out" ];
withDistOutput = true;
dontCheckRuntimeDeps = true;
pythonRelaxDeps = null;
pythonRemoveDeps = null;
pypaBuildFlags = appendIfMissing (args.pypaBuildFlags or [ ]) "--skip-dependency-check";
installPhase = ''
# runHook preInstall
mkdir $out # TODO: store all created build artifacts in $out?
pythonOutputDistPhase
exit 0
'';
dependencies = [ ]; # no runtime deps needed
# buildInputs = [ ]; # not semantically removable from .whl build
doCheck = false;
checkInputs = [ ];
nativeCheckInputs = [ ];
doInstallCheck = false;
nativeInstallCheckInputs = [ ];
pythonImportsCheck = null;
checkPhase = ":";
preCheck = ":";
postCheck = ":";
fixupPhase = ":";
preFixup = ":";
postFixup = ":";
installCheckPhase = null;
preInstallCheck = null;
postInstallCheck = null;
}
);
second = buildPythonPackage (
args
// {
buildPhase = ''
[[ ! -d dist ]] || ! echo >&2 "ERROR: dist/ found at start of buildPhase"
cp -r ${first.dist} dist
chmod -R +w dist/
runHook postBuild
'';
passthru = {
inherit first second;
} // (args.passthru or { });
}
);
in
second;
} // (args.passthru or { });
}
)
);
in
{
buildPythonPackage = mkTwostage pythonPrev.buildPythonPackage;
buildPythonApplication = mkTwostage pythonPrev.buildPythonApplication;
}
)
];
};
# usage: nativeBuildInputs = [ pytestCheckHookWithGdb ];
overlays.withPytestCheckHooks = final: prev: {
pythonPackagesExtensions = prev.pythonPackagesExtensions ++ [
(
pythonFinal: pythonPrev:
let
# TODO: export NIX_DEBUG_INFO_DIRS or "set debug-file-directory ${symlinkJoin ...}"
gdb-batch = [
"set style enabled on"
# "layout asm"
# export NIX_DEBUG_INFO_DIRS=/nix/store/xd69daaly33m7zid6g31glwmml7lk93f-openssl-1.0.2p-debug/lib/debug
# "set debug-file-directory /nix/store/xd69daaly33m7zid6g31glwmml7lk93f-openssl-1.0.2p-debug/lib/debug"
"run"
# "thread apply all backtrace"
# "backtrace"
"backtrace full"
"quit"
];
gdb-args = lib.pipe gdb-batch [
(lib.map (x: "-ex ${lib.escapeShellArg x}"))
(lib.concatStringsSep " ")
];
mkPytestCheckHook =
prefix:
pythonFinal.pytestCheckHook.override (old: {
makePythonHook =
args:
old.makePythonHook (
args
// {
substitutions = args.substitutions // {
pythonCheckInterpreter = "${prefix} ${args.substitutions.pythonCheckInterpreter}";
};
}
);
});
in
{
pytestCheckHookWithGdb = mkPytestCheckHook "${lib.getExe final.gdb} -batch ${gdb-args} --return-child-result --args";
pytestCheckHookWithGdbTimeout = mkPytestCheckHook "timeout 300 ${lib.getExe final.gdb} -batch ${gdb-args} --return-child-result --args";
pytestCheckHookWithValgrind = mkPytestCheckHook "${lib.getExe final.valgrind} --leak-check=full --show-leak-kinds=all --";
# = mkPytestCheckHook "${lib.getExe final.valgrind} --";
}
)
];
};
overlays.toplevelCudaRocmPkgs =
final: prev:
let
mkPkgs =
{
config ? { },
# overlays ? [ ],
}:
import prev.path {
# if we don't specify overlays then impure.nix will do its thing
# overlays = prev.overlays + overlays;
config = prev.config // config;
};
in
{
pkgsCuda = mkPkgs { config.cudaSupport = true; };
pkgsRocm = mkPkgs { config.rocmSupport = true; };
};
in
[
overlays.withDebug
overlays.withoutChecks
overlays.srcOnly
overlays.withInvalidateFetcherByDrvHash
overlays.withUnpacked
overlays.withTestMainProgram
overlays.withShellCheck
overlays.withTestCheckLdd
# overlays.withCudaOrRocm # WIP
overlays.withCcache # WIP
overlays.withTwostagePythonBuildPassthru
overlays.withPytestCheckHooks
overlays.toplevelCudaRocmPkgs
]