From a583d1a73f9c2be441bea113a1fc8a7f27c4f048 Mon Sep 17 00:00:00 2001 From: Peder Bergebakken Sundt Date: Thu, 24 Apr 2025 16:59:51 +0200 Subject: [PATCH] add dev --- .gitignore | 1 + dev.nix | 312 +++++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 313 insertions(+) create mode 100755 dev.nix diff --git a/.gitignore b/.gitignore index 2527486..249b310 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,5 @@ * !/justfile +!/dev.nix !/.gitignore !/.envrc diff --git a/dev.nix b/dev.nix new file mode 100755 index 0000000..6d2f871 --- /dev/null +++ b/dev.nix @@ -0,0 +1,312 @@ +#!/usr/bin/env -S nix-instantiate --parse +{ ... }@cliArgs: +let + lib = import ./lib; + # merge = sets: lib.foldl' lib.recursiveUpdate {} sets; + apply = val: f: f val; + appendIfMissing = list: val: if builtins.elem val list then list else list ++ [ val ]; + + mkPkgs = + args: + import ./. ( + apply (lib.recursiveUpdate cliArgs args) (prev: rec { + # system = ; + # crossSystem = ; + config.allowUnfree = true; + # config.allowInsecure = true; + # config.allowBroken = true; + config.checkMeta = true; + overlays = prev.overlays or [ ] ++ [ + overlay.withShellCheck + overlay.withDebug + overlay.withTwostagePythonBuildPassthru + # overlay.withPytestCheckHooks + ]; + }) + ); + + mkDerivationOverlay = + let + # from pkgs/stdenv/adapters.nix + # via https://github.com/caffineehacker/nix/blob/35601d216ccaf4c2705294281403e6797be082d0/machines/framework/configuration.nix#L42 + # simplified to require addAttrsToDerivation + withOldMkDerivation = + stdenvSuperArgs: k: stdenvSelf: + let + mkDerivationFromStdenv-super = stdenvSuperArgs.mkDerivationFromStdenv; + mkDerivationSuper = mkDerivationFromStdenv-super stdenvSelf; + in + k stdenvSelf mkDerivationSuper; + + # from pkgs/stdenv/adapters.nix + extendMkDerivationArgs = + old: f: + withOldMkDerivation old ( + _: mkDerivationSuper: args: + (mkDerivationSuper args).overrideAttrs f + ); + + # TODO: need we override any other stdenvs? + mkStdenvOverlay = f: final: prev: { + stdenv = (prev.addAttrsToDerivation { } prev.stdenv).override (f final prev prev.stdenv.mkDerivation); + # stdenvNoCC = (prev.addAttrsToDerivation { } prev.stdenvNoCC).override (f final prev prev.stdenvNoCC.mkDerivation); + # stdenvLLVM = (prev.addAttrsToDerivation { } prev.stdenvLLVM).override (f final prev prev.stdenvLLVM.mkDerivation); + }; + + in + f: + mkStdenvOverlay ( + final: prev: prevMkDerivation: stdenv: { + # mkDerivation = lib.extendMkDerivation { + # constructDrv = prevStdenv.mkDerivation; + # extendDrvArgs = f final prev stdenv; + # }; + mkDerivationFromStdenv = extendMkDerivationArgs stdenv ( + finalAttrs: prevAttrs: # TODO: drop finalAttrs? + lib.optionalAttrs (!(lib.strings.hasInfix "bootstrap" stdenv.name)) ( + let + stdenv' = stdenv // { + # mkDerivation = stdenv.mkDerivationFromStdenv stdenv; + mkDerivation = prevMkDerivation; + }; + in + f final prev stdenv' finalAttrs prevAttrs + ) + ); + } + ); + + # none of these overlays should cause a mass-rebuild, they should only add optional functionality + + # TODO: fetchurl.passthru.unpacked + # TODO: mkDerivation.passthru.withOil: add .withOil which run build with osh, https://oils.pub/osh.html#2-you-get-precise-error-messages + # TODO: mkDerivation.passthru.withGDB: add .withGDB which runs the full build with gdb + # TODO: mkDerivation.passthru.withDebInfo: like overlay.withDebug but with cmakeBuildType = "RelWithDebInfo" mode + + # usage: nix-build ./dev.nix -A .withDebug + overlay.withDebug = mkDerivationOverlay ( + final: prev: prevStdenv: finalAttrs: prevAttrs: { + passthru = { + # withDebug = prevStdenv; + withDebug = prevStdenv.mkDerivation ( + prevAttrs + // { + separateDebugInfo = false; + dontStrip = true; + # TODO: more debug flags for more builders + cmakeBuildType = "Debug"; + } + ); + } // prevAttrs.passthru or { }; + } + ); + + # usage: nix-build ./dev.nix -A .shellcheck + overlay.withShellCheck = mkDerivationOverlay ( + final: prev: prevStdenv: finalAttrs: prevAttrs: + let + snippets = lib.pipe finalAttrs [ + (lib.filterAttrs ( + name: val: + lib.hasPrefix "pre" name + || lib.hasPrefix "post" name + || lib.hasSuffix "Phase" name + || builtins.elem name (finalAttrs.passAsFile or [ ]) + )) + (lib.filterAttrs (name: val: lib.isString val && val != "")) + ]; + files = [ + # "${finalAttrs.finalPackage.stdenv}/setup" # noisy + # finalAttrs.finalPackage.builder # usually a binary + ]; + in + { + passthru = { + shellcheck = + prev.runCommandNoCCLocal "${finalAttrs.pname}-shellcheck" + { + nativeBuildInputs = [ prev.shellcheck ]; + } + '' + declare -A snippets=(${ + lib.concatLines ( + lib.mapAttrsToList (name: val: "[${lib.escapeShellArg name}]=${lib.escapeShellArg val}") snippets + ) + }) + declare -a files=(${lib.escapeShellArgs files}) + # set -x + mkdir -p $out/snippets + for snippetName in "''${!snippets[@]}"; do + snippet="''${snippets[$snippetName]}" + [[ -n "$snippet" ]] || continue + printf "snippet %q:\n" "$snippetName" + { + shellcheck --format tty --color=always --shell=bash - <<<"''$snippet" ||: + } | tee "$out/snippets/$snippetName" + done + for file in "''${files[@]}"; do + [[ -r "$file" ]] || continue + printf "file %q:\n" "$file" + mkdir -p "$out/files/$(dirname "$(realpath "$file")")" + { + shellcheck --format tty --color=always --shell=bash "$file" ||: + } | tee "$out/files/$(realpath "$file")" + done + ''; + } // (prevAttrs.passthru or { }); + } + ); + + # very hacky, not guaranteed to work, but may save a lot of .whl rebuilds + # usage: nix-build ./dev.nix -A python3Packages..twostage + overlay.withTwostagePythonBuildPassthru = final: prev: { + pythonPackagesExtensions = prev.pythonPackagesExtensions ++ [ + ( + pythonFinal: pythonPrev: + let + mkTwostage = + builder: + ( + args: + builder ( + args + // { + passthru = { + twostage = + let + # TODO: assert "dist" in finalPackage.outputs? + first = builder ( + args + // { + name = "${args.name or "${args.pname}-${args.version}"}-wheels"; + outputs = [ "out" ]; + withDistOutput = true; + dontCheckRuntimeDeps = true; + pythonRelaxDeps = null; + pythonRemoveDeps = null; + pypaBuildFlags = appendIfMissing (args.pypaBuildFlags or [ ]) "--skip-dependency-check"; + installPhase = '' + # runHook preInstall + mkdir $out # TODO: store all created build artifacts in $out? + pythonOutputDistPhase + exit 0 + ''; + dependencies = [ ]; # no runtime deps needed + # buildInputs = [ ]; # not semantically removable from .whl build + doCheck = false; + checkInputs = [ ]; + nativeCheckInputs = [ ]; + doInstallCheck = false; + nativeInstallCheckInputs = [ ]; + pythonImportsCheck = null; + checkPhase = ":"; + preCheck = ":"; + postCheck = ":"; + fixupPhase = ":"; + preFixup = ":"; + postFixup = ":"; + installCheckPhase = null; + preInstallCheck = null; + postInstallCheck = null; + } + ); + second = builder ( + args + // { + buildPhase = '' + [[ ! -d dist ]] || ! echo >&2 "ERROR: dist/ found at start of buildPhase" + cp -r ${first.dist} dist + chmod -R +w dist/ + runHook postBuild + ''; + passthru = { + inherit first second; + } // (args.passthru or { }); + } + ); + in + second; + } // (args.passthru or { }); + } + ) + ); + in + { + buildPythonPackage = mkTwostage pythonPrev.buildPythonPackage; + buildPythonApplication = mkTwostage pythonPrev.buildPythonApplication; + } + ) + ]; + }; + + # usage: nativeBuildInputs = [ pytestCheckHookWithGdb ]; + overlay.withPytestCheckHooks = final: prev: { + pythonPackagesExtensions = prev.pythonPackagesExtensions ++ [ + (pythonFinal: pythonPrev: { + pytestCheckHookWithGdb = pythonFinal.pytestCheckHook.override (old: { + makePythonHook = + args: + old.makePythonHook ( + args + // { + substitutions = + let + # TODO: export NIX_DEBUG_INFO_DIRS or "set debug-file-directory ${symlinkJoin ...}" + gdb-args = + lib.pipe + [ + "set style enabled on" + # "layout asm" + # export NIX_DEBUG_INFO_DIRS=/nix/store/xd69daaly33m7zid6g31glwmml7lk93f-openssl-1.0.2p-debug/lib/debug + # "set debug-file-directory /nix/store/xd69daaly33m7zid6g31glwmml7lk93f-openssl-1.0.2p-debug/lib/debug" + "run" + # "thread apply all backtrace" + # "backtrace" + "backtrace full" + "quit" + ] + [ + (lib.map (x: "-ex ${lib.escapeShellArg x}")) + (lib.concatStringsSep " ") + ]; + in + args.substitutions + // { + pythonCheckInterpreter = "${lib.getExe final.gdb} -batch ${gdb-args} --return-child-result --args ${args.substitutions.pythonCheckInterpreter}"; + }; + } + ); + }); + pytestCheckHookWithValgrind = pythonFinal.pytestCheckHook.override (old: { + makePythonHook = + args: + old.makePythonHook ( + args + // { + substitutions = args.substitutions // { + pythonCheckInterpreter = "${lib.getExe final.valgrind} --leak-check=full --show-leak-kinds=all -- ${args.substitutions.pythonCheckInterpreter}"; + # pythonCheckInterpreter = "${lib.getExe final.valgrind} -- ${args.substitutions.pythonCheckInterpreter}"; + }; + } + ); + }); + }) + ]; + }; + +in +(mkPkgs { }) +// { + pkgsCuda = mkPkgs { + config.enableCuda = true; + }; + pkgsRocm = mkPkgs { + config.enableRocm = true; + }; + # pkgsDarwin + # pkgsLinux + # pkgsFreeBSD + # pkgsNetBSD + # pkgsOpenBSD + # pkgsWindows +}