#!/usr/bin/env -S nix-instantiate --parse { ... }@cliArgs: let lib = import ./lib; # merge = sets: lib.foldl' lib.recursiveUpdate {} sets; apply = val: f: f val; appendIfMissing = list: val: if builtins.elem val list then list else list ++ [ val ]; mkPkgs = args: import ./. ( apply (lib.recursiveUpdate cliArgs args) (prev: rec { # system = ; # crossSystem = ; config.allowUnfree = true; # config.allowInsecure = true; # config.allowBroken = true; config.checkMeta = true; overlays = prev.overlays or [ ] ++ [ overlay.withShellCheck overlay.withDebug overlay.withTwostagePythonBuildPassthru # overlay.withPytestCheckHooks ]; }) ); mkDerivationOverlay = let # from pkgs/stdenv/adapters.nix # via https://github.com/caffineehacker/nix/blob/35601d216ccaf4c2705294281403e6797be082d0/machines/framework/configuration.nix#L42 # simplified to require addAttrsToDerivation withOldMkDerivation = stdenvSuperArgs: k: stdenvSelf: let mkDerivationFromStdenv-super = stdenvSuperArgs.mkDerivationFromStdenv; mkDerivationSuper = mkDerivationFromStdenv-super stdenvSelf; in k stdenvSelf mkDerivationSuper; # from pkgs/stdenv/adapters.nix extendMkDerivationArgs = old: f: withOldMkDerivation old ( _: mkDerivationSuper: args: (mkDerivationSuper args).overrideAttrs f ); # TODO: need we override any other stdenvs? mkStdenvOverlay = f: final: prev: { stdenv = (prev.addAttrsToDerivation { } prev.stdenv).override (f final prev prev.stdenv.mkDerivation); # stdenvNoCC = (prev.addAttrsToDerivation { } prev.stdenvNoCC).override (f final prev prev.stdenvNoCC.mkDerivation); # stdenvLLVM = (prev.addAttrsToDerivation { } prev.stdenvLLVM).override (f final prev prev.stdenvLLVM.mkDerivation); }; in f: mkStdenvOverlay ( final: prev: prevMkDerivation: stdenv: { # mkDerivation = lib.extendMkDerivation { # constructDrv = prevStdenv.mkDerivation; # extendDrvArgs = f final prev stdenv; # }; mkDerivationFromStdenv = extendMkDerivationArgs stdenv ( finalAttrs: prevAttrs: # TODO: drop finalAttrs? lib.optionalAttrs (!(lib.strings.hasInfix "bootstrap" stdenv.name)) ( let stdenv' = stdenv // { # mkDerivation = stdenv.mkDerivationFromStdenv stdenv; mkDerivation = prevMkDerivation; }; in f final prev stdenv' finalAttrs prevAttrs ) ); } ); # none of these overlays should cause a mass-rebuild, they should only add optional functionality # TODO: fetchurl.passthru.unpacked # TODO: mkDerivation.passthru.withOil: add .withOil which run build with osh, https://oils.pub/osh.html#2-you-get-precise-error-messages # TODO: mkDerivation.passthru.withGDB: add .withGDB which runs the full build with gdb # TODO: mkDerivation.passthru.withDebInfo: like overlay.withDebug but with cmakeBuildType = "RelWithDebInfo" mode # usage: nix-build ./dev.nix -A .withDebug overlay.withDebug = mkDerivationOverlay ( final: prev: prevStdenv: finalAttrs: prevAttrs: { passthru = { # withDebug = prevStdenv; withDebug = prevStdenv.mkDerivation ( prevAttrs // { separateDebugInfo = false; dontStrip = true; # TODO: more debug flags for more builders cmakeBuildType = "Debug"; } ); } // prevAttrs.passthru or { }; } ); # usage: nix-build ./dev.nix -A .shellcheck overlay.withShellCheck = mkDerivationOverlay ( final: prev: prevStdenv: finalAttrs: prevAttrs: let snippets = lib.pipe finalAttrs [ (lib.filterAttrs ( name: val: lib.hasPrefix "pre" name || lib.hasPrefix "post" name || lib.hasSuffix "Phase" name || builtins.elem name (finalAttrs.passAsFile or [ ]) )) (lib.filterAttrs (name: val: lib.isString val && val != "")) ]; files = [ # "${finalAttrs.finalPackage.stdenv}/setup" # noisy # finalAttrs.finalPackage.builder # usually a binary ]; in { passthru = { shellcheck = prev.runCommandNoCCLocal "${finalAttrs.pname}-shellcheck" { nativeBuildInputs = [ prev.shellcheck ]; } '' declare -A snippets=(${ lib.concatLines ( lib.mapAttrsToList (name: val: "[${lib.escapeShellArg name}]=${lib.escapeShellArg val}") snippets ) }) declare -a files=(${lib.escapeShellArgs files}) # set -x mkdir -p $out/snippets for snippetName in "''${!snippets[@]}"; do snippet="''${snippets[$snippetName]}" [[ -n "$snippet" ]] || continue printf "snippet %q:\n" "$snippetName" { shellcheck --format tty --color=always --shell=bash - <<<"''$snippet" ||: } | tee "$out/snippets/$snippetName" done for file in "''${files[@]}"; do [[ -r "$file" ]] || continue printf "file %q:\n" "$file" mkdir -p "$out/files/$(dirname "$(realpath "$file")")" { shellcheck --format tty --color=always --shell=bash "$file" ||: } | tee "$out/files/$(realpath "$file")" done ''; } // (prevAttrs.passthru or { }); } ); # very hacky, not guaranteed to work, but may save a lot of .whl rebuilds # usage: nix-build ./dev.nix -A python3Packages..twostage overlay.withTwostagePythonBuildPassthru = final: prev: { pythonPackagesExtensions = prev.pythonPackagesExtensions ++ [ ( pythonFinal: pythonPrev: let mkTwostage = builder: ( args: builder ( args // { passthru = { twostage = let # TODO: assert "dist" in finalPackage.outputs? first = builder ( args // { name = "${args.name or "${args.pname}-${args.version}"}-wheels"; outputs = [ "out" ]; withDistOutput = true; dontCheckRuntimeDeps = true; pythonRelaxDeps = null; pythonRemoveDeps = null; pypaBuildFlags = appendIfMissing (args.pypaBuildFlags or [ ]) "--skip-dependency-check"; installPhase = '' # runHook preInstall mkdir $out # TODO: store all created build artifacts in $out? pythonOutputDistPhase exit 0 ''; dependencies = [ ]; # no runtime deps needed # buildInputs = [ ]; # not semantically removable from .whl build doCheck = false; checkInputs = [ ]; nativeCheckInputs = [ ]; doInstallCheck = false; nativeInstallCheckInputs = [ ]; pythonImportsCheck = null; checkPhase = ":"; preCheck = ":"; postCheck = ":"; fixupPhase = ":"; preFixup = ":"; postFixup = ":"; installCheckPhase = null; preInstallCheck = null; postInstallCheck = null; } ); second = builder ( args // { buildPhase = '' [[ ! -d dist ]] || ! echo >&2 "ERROR: dist/ found at start of buildPhase" cp -r ${first.dist} dist chmod -R +w dist/ runHook postBuild ''; passthru = { inherit first second; } // (args.passthru or { }); } ); in second; } // (args.passthru or { }); } ) ); in { buildPythonPackage = mkTwostage pythonPrev.buildPythonPackage; buildPythonApplication = mkTwostage pythonPrev.buildPythonApplication; } ) ]; }; # usage: nativeBuildInputs = [ pytestCheckHookWithGdb ]; overlay.withPytestCheckHooks = final: prev: { pythonPackagesExtensions = prev.pythonPackagesExtensions ++ [ (pythonFinal: pythonPrev: { pytestCheckHookWithGdb = pythonFinal.pytestCheckHook.override (old: { makePythonHook = args: old.makePythonHook ( args // { substitutions = let # TODO: export NIX_DEBUG_INFO_DIRS or "set debug-file-directory ${symlinkJoin ...}" gdb-args = lib.pipe [ "set style enabled on" # "layout asm" # export NIX_DEBUG_INFO_DIRS=/nix/store/xd69daaly33m7zid6g31glwmml7lk93f-openssl-1.0.2p-debug/lib/debug # "set debug-file-directory /nix/store/xd69daaly33m7zid6g31glwmml7lk93f-openssl-1.0.2p-debug/lib/debug" "run" # "thread apply all backtrace" # "backtrace" "backtrace full" "quit" ] [ (lib.map (x: "-ex ${lib.escapeShellArg x}")) (lib.concatStringsSep " ") ]; in args.substitutions // { pythonCheckInterpreter = "${lib.getExe final.gdb} -batch ${gdb-args} --return-child-result --args ${args.substitutions.pythonCheckInterpreter}"; }; } ); }); pytestCheckHookWithValgrind = pythonFinal.pytestCheckHook.override (old: { makePythonHook = args: old.makePythonHook ( args // { substitutions = args.substitutions // { pythonCheckInterpreter = "${lib.getExe final.valgrind} --leak-check=full --show-leak-kinds=all -- ${args.substitutions.pythonCheckInterpreter}"; # pythonCheckInterpreter = "${lib.getExe final.valgrind} -- ${args.substitutions.pythonCheckInterpreter}"; }; } ); }); }) ]; }; in (mkPkgs { }) // { pkgsCuda = mkPkgs { config.enableCuda = true; }; pkgsRocm = mkPkgs { config.enableRocm = true; }; # pkgsDarwin # pkgsLinux # pkgsFreeBSD # pkgsNetBSD # pkgsOpenBSD # pkgsWindows }