453 lines
16 KiB
Nix
Executable File
453 lines
16 KiB
Nix
Executable File
#!/usr/bin/env -S nix-instantiate --parse
|
|
let
|
|
lib = import ./master/lib;
|
|
# merge = sets: lib.foldl' lib.recursiveUpdate {} sets;
|
|
appendIfMissing = list: val: if builtins.elem val list then list else list ++ [ val ];
|
|
|
|
mkDerivationOverlay =
|
|
let
|
|
# from pkgs/stdenv/adapters.nix
|
|
# via https://github.com/caffineehacker/nix/blob/35601d216ccaf4c2705294281403e6797be082d0/machines/framework/configuration.nix#L42
|
|
# simplified to require addAttrsToDerivation
|
|
withOldMkDerivation =
|
|
stdenvSuperArgs: k: stdenvSelf:
|
|
let
|
|
mkDerivationFromStdenv-super = stdenvSuperArgs.mkDerivationFromStdenv;
|
|
mkDerivationSuper = mkDerivationFromStdenv-super stdenvSelf;
|
|
in
|
|
k stdenvSelf mkDerivationSuper;
|
|
|
|
# from pkgs/stdenv/adapters.nix
|
|
extendMkDerivationArgs =
|
|
old: f:
|
|
withOldMkDerivation old (
|
|
_: mkDerivationSuper: args:
|
|
(mkDerivationSuper args).overrideAttrs f
|
|
);
|
|
|
|
# TODO: need we override any other stdenvs?
|
|
mkStdenvOverlay = f: final: prev: {
|
|
stdenv = (prev.addAttrsToDerivation { } prev.stdenv).override (
|
|
f final prev prev.stdenv.mkDerivation
|
|
);
|
|
# stdenvNoCC = (prev.addAttrsToDerivation { } prev.stdenvNoCC).override (f final prev prev.stdenvNoCC.mkDerivation);
|
|
# stdenvLLVM = (prev.addAttrsToDerivation { } prev.stdenvLLVM).override (f final prev prev.stdenvLLVM.mkDerivation);
|
|
};
|
|
|
|
in
|
|
f:
|
|
mkStdenvOverlay (
|
|
final: prev: prevMkDerivation: stdenv: {
|
|
# mkDerivation = lib.extendMkDerivation {
|
|
# constructDrv = prevStdenv.mkDerivation;
|
|
# extendDrvArgs = f final prev stdenv;
|
|
# };
|
|
mkDerivationFromStdenv = extendMkDerivationArgs stdenv (
|
|
finalAttrs: prevAttrs: # TODO: drop finalAttrs?
|
|
lib.optionalAttrs (!(lib.strings.hasInfix "bootstrap" stdenv.name)) (
|
|
let
|
|
stdenv' = stdenv // {
|
|
# mkDerivation = stdenv.mkDerivationFromStdenv stdenv;
|
|
mkDerivation = prevMkDerivation; # HACK
|
|
};
|
|
in
|
|
f final prev stdenv' finalAttrs prevAttrs
|
|
)
|
|
);
|
|
}
|
|
);
|
|
|
|
# none of these overlays should cause a mass-rebuild, they should only add optional functionality
|
|
|
|
# TODO: cccache and sccache
|
|
# TODO: mkDerivation.passthru.withOil: add <package>.withOil which run build with osh, https://oils.pub/osh.html#2-you-get-precise-error-messages
|
|
# TODO: mkDerivation.passthru.withGDB: add <package>.withGDB which runs the full build with gdb
|
|
# TODO: mkDerivation.passthru.withDebInfo: like overlay.withDebug but with cmakeBuildType = "RelWithDebInfo" mode
|
|
|
|
# usage: nix-build ./dev.nix -A <package>.withDebug
|
|
overlays.withDebug = mkDerivationOverlay (
|
|
final: prev: prevStdenv: finalAttrs: prevAttrs: {
|
|
passthru = {
|
|
withDebug = prevStdenv.mkDerivation (
|
|
prevAttrs
|
|
// {
|
|
separateDebugInfo = false;
|
|
dontStrip = true;
|
|
# TODO: more debug flags for more builders
|
|
cmakeBuildType = "Debug";
|
|
}
|
|
);
|
|
} // prevAttrs.passthru or { };
|
|
}
|
|
);
|
|
|
|
# usage: nix-build ./dev.nix -A <package>.srcOnly
|
|
overlays.srcOnly = mkDerivationOverlay (
|
|
final: prev: prevStdenv: finalAttrs: prevAttrs: {
|
|
passthru = {
|
|
srcOnly = prev.srcOnly finalAttrs.finalPackage;
|
|
} // prevAttrs.passthru or { };
|
|
}
|
|
);
|
|
|
|
# usage: nix-build ./dev.nix -A <package>.src.invalidated
|
|
overlays.withInvalidateFetcherByDrvHash = mkDerivationOverlay (
|
|
final: prev: prevStdenv: finalAttrs: prevAttrs: {
|
|
passthru = {
|
|
# invalidated = prevStdenv.mkDerivation prevAttrs;
|
|
invalidated = prev.invalidateFetcherByDrvHash prevStdenv.mkDerivation prevAttrs;
|
|
} // prevAttrs.passthru or { };
|
|
}
|
|
);
|
|
|
|
# https://github.com/NixOS/nixpkgs/pull/392938
|
|
# usage: nix-build ./dev.nix -A <package>.src.unpacked
|
|
overlays.withUnpacked = final: prev: {
|
|
fetchurl = (lib.mirrorFunctionArgs prev.fetchurl) (
|
|
args:
|
|
let
|
|
fetched = prev.fetchurl args;
|
|
in
|
|
fetched.overrideAttrs (old: {
|
|
passthru = {
|
|
unpacked =
|
|
if old.outputHashMode == "recursive" then
|
|
fetched
|
|
else
|
|
prev.srcOnly {
|
|
name = "${fetched.name}-unpacked";
|
|
src = fetched;
|
|
stdenv = prev.stdenvNoCC;
|
|
inherit (old) preferLocalBuild;
|
|
};
|
|
} // old.passthru or { };
|
|
})
|
|
);
|
|
fetchgit = (lib.mirrorFunctionArgs prev.fetchgit) (
|
|
args:
|
|
(prev.fetchgit args).overrideAttrs (old: {
|
|
passthru = {
|
|
unpacked = prev.fetchgit args;
|
|
} // old.passthru or { };
|
|
})
|
|
);
|
|
fetchhg = (lib.mirrorFunctionArgs prev.fetchhg) (
|
|
args:
|
|
(prev.fetchhg args).overrideAttrs (old: {
|
|
passthru = {
|
|
unpacked = prev.fetchhg args;
|
|
} // old.passthru or { };
|
|
})
|
|
);
|
|
fetchsvn = (lib.mirrorFunctionArgs prev.fetchsvn) (
|
|
args:
|
|
(prev.fetchsvn args).overrideAttrs (old: {
|
|
passthru = {
|
|
unpacked = prev.fetchsvn args;
|
|
} // old.passthru or { };
|
|
})
|
|
);
|
|
};
|
|
|
|
# usage: nix-build ./dev.nix -A <package>.shellcheck
|
|
overlays.withShellCheck = mkDerivationOverlay (
|
|
final: prev: prevStdenv: finalAttrs: prevAttrs:
|
|
let
|
|
snippets = lib.pipe finalAttrs [
|
|
(lib.filterAttrs (
|
|
name: val:
|
|
lib.hasPrefix "pre" name
|
|
|| lib.hasPrefix "post" name
|
|
|| lib.hasSuffix "Phase" name
|
|
|| builtins.elem name (finalAttrs.passAsFile or [ ])
|
|
))
|
|
(lib.filterAttrs (name: val: lib.isString val && val != ""))
|
|
];
|
|
files = [
|
|
# "${finalAttrs.finalPackage.stdenv}/setup" # noisy
|
|
# finalAttrs.finalPackage.builder # usually a binary
|
|
];
|
|
in
|
|
{
|
|
passthru = {
|
|
shellcheck =
|
|
prev.runCommandNoCCLocal "${finalAttrs.pname}-shellcheck"
|
|
{
|
|
nativeBuildInputs = [ prev.shellcheck ];
|
|
}
|
|
''
|
|
declare -A snippets=(${
|
|
lib.concatLines (
|
|
lib.mapAttrsToList (name: val: "[${lib.escapeShellArg name}]=${lib.escapeShellArg val}") snippets
|
|
)
|
|
})
|
|
declare -a files=(${lib.escapeShellArgs files})
|
|
# set -x
|
|
mkdir -p $out/snippets
|
|
for snippetName in "''${!snippets[@]}"; do
|
|
snippet="''${snippets[$snippetName]}"
|
|
[[ -n "$snippet" ]] || continue
|
|
printf "snippet %q:\n" "$snippetName"
|
|
{
|
|
shellcheck --format tty --color=always --shell=bash - <<<"''$snippet" ||:
|
|
} | tee "$out/snippets/$snippetName"
|
|
done
|
|
for file in "''${files[@]}"; do
|
|
[[ -r "$file" ]] || continue
|
|
printf "file %q:\n" "$file"
|
|
mkdir -p "$out/files/$(dirname "$(realpath "$file")")"
|
|
{
|
|
shellcheck --format tty --color=always --shell=bash "$file" ||:
|
|
} | tee "$out/files/$(realpath "$file")"
|
|
done
|
|
'';
|
|
} // (prevAttrs.passthru or { });
|
|
}
|
|
);
|
|
|
|
# TODO: this way of overriding does not work
|
|
# usage: nix-build ./dev.nix -A <package>.withCuda
|
|
# usage: nix-build ./dev.nix -A <package>.withRocm
|
|
overlays.withCudaOrRocm = final: prev: {
|
|
lib = prev.lib.extend (
|
|
finalLib: prevLib: {
|
|
# makeOverridable :: (AttrSet -> a) -> AttrSet -> a
|
|
makeOverridable =
|
|
f: args:
|
|
let
|
|
fArgs = lib.functionArgs f;
|
|
# Creates a functor with the same arguments as f
|
|
mirrorArgs = lib.mirrorFunctionArgs f;
|
|
fWithCuda = mirrorArgs (
|
|
origArgs:
|
|
f' (
|
|
origArgs
|
|
// lib.optionalAttrs (fArgs ? cudaSupport) {
|
|
cudaSupport = true;
|
|
}
|
|
// lib.optionalAttrs (fArgs ? config) {
|
|
config = fArgs.config // {
|
|
cudaSupport = true;
|
|
};
|
|
}
|
|
)
|
|
);
|
|
fWithRocm = mirrorArgs (
|
|
origArgs:
|
|
f' (
|
|
origArgs
|
|
// lib.optionalAttrs (fArgs ? rocmSupport) {
|
|
rocmSupport = true;
|
|
}
|
|
// lib.optionalAttrs (fArgs ? config) {
|
|
config = fArgs.config // {
|
|
rocmSupport = true;
|
|
};
|
|
}
|
|
)
|
|
);
|
|
f' = mirrorArgs (
|
|
origArgs:
|
|
let
|
|
result = f origArgs;
|
|
in
|
|
if lib.isAttrs result then
|
|
if result ? overrideAttrs then
|
|
result.overrideAttrs (old: {
|
|
passthru = {
|
|
withCuda = fWithCuda origArgs;
|
|
withRocm = fWithRocm origArgs;
|
|
} // old.passthru or { };
|
|
})
|
|
else
|
|
{
|
|
withCuda = fWithCuda origArgs;
|
|
withRocm = fWithRocm origArgs;
|
|
}
|
|
// result
|
|
else
|
|
result
|
|
);
|
|
in
|
|
prevLib.makeOverridable f' args;
|
|
}
|
|
);
|
|
};
|
|
|
|
# very hacky, not guaranteed to work, but may save a lot of .whl rebuilds
|
|
# usage: nix-build ./dev.nix -A python3Packages.<package>.twostage
|
|
# usage: nix-build ./dev.nix -A python3Packages.<package>.twostage.first
|
|
overlays.withTwostagePythonBuildPassthru = final: prev: {
|
|
pythonPackagesExtensions = prev.pythonPackagesExtensions ++ [
|
|
(
|
|
pythonFinal: pythonPrev:
|
|
let
|
|
mkTwostage =
|
|
buildPythonPackage:
|
|
(
|
|
args:
|
|
buildPythonPackage (
|
|
args
|
|
// {
|
|
passthru = {
|
|
twostage =
|
|
let
|
|
# TODO: assert "dist" in finalPackage.outputs?
|
|
first = buildPythonPackage (
|
|
args
|
|
// {
|
|
name = "${args.name or "${args.pname}-${args.version}"}-wheels";
|
|
outputs = [ "out" ];
|
|
withDistOutput = true;
|
|
dontCheckRuntimeDeps = true;
|
|
pythonRelaxDeps = null;
|
|
pythonRemoveDeps = null;
|
|
pypaBuildFlags = appendIfMissing (args.pypaBuildFlags or [ ]) "--skip-dependency-check";
|
|
installPhase = ''
|
|
# runHook preInstall
|
|
mkdir $out # TODO: store all created build artifacts in $out?
|
|
pythonOutputDistPhase
|
|
exit 0
|
|
'';
|
|
dependencies = [ ]; # no runtime deps needed
|
|
# buildInputs = [ ]; # not semantically removable from .whl build
|
|
doCheck = false;
|
|
checkInputs = [ ];
|
|
nativeCheckInputs = [ ];
|
|
doInstallCheck = false;
|
|
nativeInstallCheckInputs = [ ];
|
|
pythonImportsCheck = null;
|
|
checkPhase = ":";
|
|
preCheck = ":";
|
|
postCheck = ":";
|
|
fixupPhase = ":";
|
|
preFixup = ":";
|
|
postFixup = ":";
|
|
installCheckPhase = null;
|
|
preInstallCheck = null;
|
|
postInstallCheck = null;
|
|
}
|
|
);
|
|
second = buildPythonPackage (
|
|
args
|
|
// {
|
|
buildPhase = ''
|
|
[[ ! -d dist ]] || ! echo >&2 "ERROR: dist/ found at start of buildPhase"
|
|
cp -r ${first.dist} dist
|
|
chmod -R +w dist/
|
|
runHook postBuild
|
|
'';
|
|
passthru = {
|
|
inherit first second;
|
|
} // (args.passthru or { });
|
|
}
|
|
);
|
|
in
|
|
second;
|
|
} // (args.passthru or { });
|
|
}
|
|
)
|
|
);
|
|
in
|
|
{
|
|
buildPythonPackage = mkTwostage pythonPrev.buildPythonPackage;
|
|
buildPythonApplication = mkTwostage pythonPrev.buildPythonApplication;
|
|
}
|
|
)
|
|
];
|
|
};
|
|
|
|
# usage: nativeBuildInputs = [ pytestCheckHookWithGdb ];
|
|
overlays.withPytestCheckHooks = final: prev: {
|
|
pythonPackagesExtensions = prev.pythonPackagesExtensions ++ [
|
|
(pythonFinal: pythonPrev: {
|
|
pytestCheckHookWithGdb = pythonFinal.pytestCheckHook.override (old: {
|
|
makePythonHook =
|
|
args:
|
|
old.makePythonHook (
|
|
args
|
|
// {
|
|
substitutions =
|
|
let
|
|
# TODO: export NIX_DEBUG_INFO_DIRS or "set debug-file-directory ${symlinkJoin ...}"
|
|
gdb-args =
|
|
lib.pipe
|
|
[
|
|
"set style enabled on"
|
|
# "layout asm"
|
|
# export NIX_DEBUG_INFO_DIRS=/nix/store/xd69daaly33m7zid6g31glwmml7lk93f-openssl-1.0.2p-debug/lib/debug
|
|
# "set debug-file-directory /nix/store/xd69daaly33m7zid6g31glwmml7lk93f-openssl-1.0.2p-debug/lib/debug"
|
|
"run"
|
|
# "thread apply all backtrace"
|
|
# "backtrace"
|
|
"backtrace full"
|
|
"quit"
|
|
]
|
|
[
|
|
(lib.map (x: "-ex ${lib.escapeShellArg x}"))
|
|
(lib.concatStringsSep " ")
|
|
];
|
|
in
|
|
args.substitutions
|
|
// {
|
|
pythonCheckInterpreter = "${lib.getExe final.gdb} -batch ${gdb-args} --return-child-result --args ${args.substitutions.pythonCheckInterpreter}";
|
|
};
|
|
}
|
|
);
|
|
});
|
|
pytestCheckHookWithValgrind = pythonFinal.pytestCheckHook.override (old: {
|
|
makePythonHook =
|
|
args:
|
|
old.makePythonHook (
|
|
args
|
|
// {
|
|
substitutions = args.substitutions // {
|
|
pythonCheckInterpreter = "${lib.getExe final.valgrind} --leak-check=full --show-leak-kinds=all -- ${args.substitutions.pythonCheckInterpreter}";
|
|
# pythonCheckInterpreter = "${lib.getExe final.valgrind} -- ${args.substitutions.pythonCheckInterpreter}";
|
|
};
|
|
}
|
|
);
|
|
});
|
|
})
|
|
];
|
|
};
|
|
|
|
overlays.toplevelCudaRocmPkgs =
|
|
final: prev:
|
|
let
|
|
mkPkgs =
|
|
{
|
|
config ? { },
|
|
# overlays ? [ ],
|
|
}:
|
|
import prev.path {
|
|
# if we don't specify overlays then impure.nix will do its thing
|
|
# overlays = prev.overlays + overlays;
|
|
config = prev.config // config;
|
|
};
|
|
in
|
|
{
|
|
pkgsCuda = mkPkgs { config.cudaSupport = true; };
|
|
pkgsRocm = mkPkgs { config.rocmSupport = true; };
|
|
# pkgsDarwin?
|
|
# pkgsLinux?
|
|
# pkgsFreeBSD?
|
|
# pkgsNetBSD?
|
|
# pkgsOpenBSD?
|
|
# pkgsWindows?
|
|
};
|
|
|
|
in
|
|
[
|
|
overlays.withShellCheck
|
|
overlays.withDebug
|
|
overlays.srcOnly
|
|
overlays.withInvalidateFetcherByDrvHash
|
|
overlays.withUnpacked
|
|
overlays.withTwostagePythonBuildPassthru
|
|
overlays.withPytestCheckHooks
|
|
overlays.toplevelCudaRocmPkgs
|
|
# WIP:
|
|
# overlays.withCudaOrRocm
|
|
]
|