From bd9352035f3a8d8d1d98ff078b1e3eaa7024e92d Mon Sep 17 00:00:00 2001 From: Peder Bergebakken Sundt Date: Wed, 22 Nov 2023 22:18:02 +0100 Subject: [PATCH] madness --- .gitignore | 2 + .remoteenv | 3 + .remoteignore.toml | 11 +++ flake.lock | 220 ++++++++++++++++++++++++++++++++++++++++++++- flake.nix | 135 ++++++++++++++++++++++++++-- 5 files changed, 362 insertions(+), 9 deletions(-) create mode 100644 .remoteenv create mode 100644 .remoteignore.toml diff --git a/.gitignore b/.gitignore index 8228ec6..db3c5e1 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,4 @@ .remote.toml .direnv/ +result +result-* diff --git a/.remoteenv b/.remoteenv new file mode 100644 index 0000000..4dde7cc --- /dev/null +++ b/.remoteenv @@ -0,0 +1,3 @@ +#!/usr/bin/env bash +export APPTAINER_BIND="/usr,/lib,/lib64"; +export SINGULARITY_BIND="/usr,/lib,/lib64"; diff --git a/.remoteignore.toml b/.remoteignore.toml new file mode 100644 index 0000000..f5c5730 --- /dev/null +++ b/.remoteignore.toml @@ -0,0 +1,11 @@ +[push] +exclude = [] +include = [] + +[pull] +exclude = [] +include = [] + +[both] +exclude = [ ".remote.toml", "result", "result-*", ".git", ".direnv" ] +include = [] diff --git a/flake.lock b/flake.lock index 7f4e5e1..6cc3326 100644 --- a/flake.lock +++ b/flake.lock @@ -1,5 +1,110 @@ { "nodes": { + "flake-parts": { + "inputs": { + "nixpkgs-lib": [ + "nixpkgs-unfree", + "nixpkgs" + ] + }, + "locked": { + "lastModified": 1696343447, + "narHash": "sha256-B2xAZKLkkeRFG5XcHHSXXcP7To9Xzr59KXeZiRf4vdQ=", + "owner": "hercules-ci", + "repo": "flake-parts", + "rev": "c9afaba3dfa4085dbd2ccb38dfade5141e33d9d4", + "type": "github" + }, + "original": { + "owner": "hercules-ci", + "repo": "flake-parts", + "type": "github" + } + }, + "flake-parts_2": { + "inputs": { + "nixpkgs-lib": [ + "nixpkgs-unfree", + "hercules-ci-effects", + "hercules-ci-agent", + "nixpkgs" + ] + }, + "locked": { + "lastModified": 1688466019, + "narHash": "sha256-VeM2akYrBYMsb4W/MmBo1zmaMfgbL4cH3Pu8PGyIwJ0=", + "owner": "hercules-ci", + "repo": "flake-parts", + "rev": "8e8d955c22df93dbe24f19ea04f47a74adbdc5ec", + "type": "github" + }, + "original": { + "owner": "hercules-ci", + "repo": "flake-parts", + "type": "github" + } + }, + "haskell-flake": { + "locked": { + "lastModified": 1684780604, + "narHash": "sha256-2uMZsewmRn7rRtAnnQNw1lj0uZBMh4m6Cs/7dV5YF08=", + "owner": "srid", + "repo": "haskell-flake", + "rev": "74210fa80a49f1b6f67223debdbf1494596ff9f2", + "type": "github" + }, + "original": { + "owner": "srid", + "ref": "0.3.0", + "repo": "haskell-flake", + "type": "github" + } + }, + "hercules-ci-agent": { + "inputs": { + "flake-parts": "flake-parts_2", + "haskell-flake": "haskell-flake", + "nixpkgs": "nixpkgs_2" + }, + "locked": { + "lastModified": 1688568579, + "narHash": "sha256-ON0M56wtY/TIIGPkXDlJboAmuYwc73Hi8X9iJGtxOhM=", + "owner": "hercules-ci", + "repo": "hercules-ci-agent", + "rev": "367dd8cd649b57009a6502e878005a1e54ad78c5", + "type": "github" + }, + "original": { + "id": "hercules-ci-agent", + "type": "indirect" + } + }, + "hercules-ci-effects": { + "inputs": { + "flake-parts": [ + "nixpkgs-unfree", + "flake-parts" + ], + "hercules-ci-agent": "hercules-ci-agent", + "nixpkgs": [ + "nixpkgs-unfree", + "nixpkgs" + ] + }, + "locked": { + "lastModified": 1695684520, + "narHash": "sha256-yORqGB0i1OtEf9MOCCT2BIbOd8txPZn216CM+ylMmhY=", + "owner": "hercules-ci", + "repo": "hercules-ci-effects", + "rev": "91fae5824f5f1199f61693c6590b4a89abaed9d7", + "type": "github" + }, + "original": { + "owner": "hercules-ci", + "repo": "hercules-ci-effects", + "type": "github" + } + }, "nixpkgs": { "locked": { "lastModified": 1696419054, @@ -10,13 +115,122 @@ "type": "github" }, "original": { - "id": "nixpkgs", - "type": "indirect" + "owner": "NixOS", + "ref": "nixpkgs-unstable", + "repo": "nixpkgs", + "type": "github" + } + }, + "nixpkgs-master": { + "locked": { + "lastModified": 1696470233, + "narHash": "sha256-pL/8IjifB2uoWLXdIZK4GuQrKcI6sIDZwC+q39JYUxI=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "c748544b2f3bc3c7716c91ccedb2b2fef300e789", + "type": "github" + }, + "original": { + "owner": "NixOS", + "ref": "master", + "repo": "nixpkgs", + "type": "github" + } + }, + "nixpkgs-nixos-unstable": { + "locked": { + "lastModified": 1696193975, + "narHash": "sha256-mnQjUcYgp9Guu3RNVAB2Srr1TqKcPpRXmJf4LJk6KRY=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "fdd898f8f79e8d2f99ed2ab6b3751811ef683242", + "type": "github" + }, + "original": { + "owner": "NixOS", + "ref": "nixos-unstable", + "repo": "nixpkgs", + "type": "github" + } + }, + "nixpkgs-nixpkgs-unstable": { + "locked": { + "lastModified": 1696419054, + "narHash": "sha256-EdR+dIKCfqL3voZUDYwcvgRDOektQB9KbhBVcE0/3Mo=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "7131f3c223a2d799568e4b278380cd9dac2b8579", + "type": "github" + }, + "original": { + "owner": "NixOS", + "ref": "nixpkgs-unstable", + "repo": "nixpkgs", + "type": "github" + } + }, + "nixpkgs-release": { + "locked": { + "lastModified": 1696374741, + "narHash": "sha256-gt8B3G0ryizT9HSB4cCO8QoxdbsHnrQH+/BdKxOwqF0=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "8a4c17493e5c39769f79117937c79e1c88de6729", + "type": "github" + }, + "original": { + "owner": "NixOS", + "ref": "nixos-23.05", + "repo": "nixpkgs", + "type": "github" + } + }, + "nixpkgs-unfree": { + "inputs": { + "flake-parts": "flake-parts", + "hercules-ci-effects": "hercules-ci-effects", + "nixpkgs": [ + "nixpkgs" + ], + "nixpkgs-master": "nixpkgs-master", + "nixpkgs-nixos-unstable": "nixpkgs-nixos-unstable", + "nixpkgs-nixpkgs-unstable": "nixpkgs-nixpkgs-unstable", + "nixpkgs-release": "nixpkgs-release" + }, + "locked": { + "lastModified": 1696473081, + "narHash": "sha256-XqCeprTkcI86AbZDO24XykeqtahLCwoy/eJSag0h4Ro=", + "owner": "SomeoneSerge", + "repo": "nixpkgs-unfree", + "rev": "a10b2b7f708d54dbc8c25c8d5af2a98feddd3bd0", + "type": "github" + }, + "original": { + "owner": "SomeoneSerge", + "repo": "nixpkgs-unfree", + "type": "github" + } + }, + "nixpkgs_2": { + "locked": { + "lastModified": 1688322751, + "narHash": "sha256-eW62dC5f33oKZL7VWlomttbUnOTHrAbte9yNUNW8rbk=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "0fbe93c5a7cac99f90b60bdf5f149383daaa615f", + "type": "github" + }, + "original": { + "owner": "NixOS", + "ref": "nixos-unstable", + "repo": "nixpkgs", + "type": "github" } }, "root": { "inputs": { - "nixpkgs": "nixpkgs" + "nixpkgs": "nixpkgs", + "nixpkgs-unfree": "nixpkgs-unfree" } } }, diff --git a/flake.nix b/flake.nix index bd21746..fb8f635 100644 --- a/flake.nix +++ b/flake.nix @@ -1,22 +1,144 @@ { + inputs.nixpkgs.url = github:NixOS/nixpkgs/nixpkgs-unstable; + inputs.nixpkgs-unfree.url = github:SomeoneSerge/nixpkgs-unfree; + inputs.nixpkgs-unfree.inputs.nixpkgs.follows = "nixpkgs"; + #inputs.nix2container.url = "github:nlewo/nix2container"; + #inputs.nix2container.inputs.nixpkgs.follows = "nixpkgs"; + + nixConfig.extra-substituters = [ + "https://cuda-maintainers.cachix.org" + ]; + nixConfig.extra-trusted-public-keys = [ + "cuda-maintainers.cachix.org-1:0dq3bujKpuEPMCX6U4WylrUDZ9JyUG0VpVZa7CNfq5E=" + ]; + + # TODO: https://www.canva.dev/blog/engineering/supporting-gpu-accelerated-machine-learning-with-kubernetes-and-nix/ + outputs = { self, nixpkgs, - ... - } @ inputs: + nixpkgs-unfree, + ... } @ inputs: let - forSystems = systems: f: nixpkgs.lib.genAttrs systems (system: f rec { + #flake = inputs: system: nixpkgs.lib.mapAttrs (name: flake: { + # nixos = flake.nixosModules + # or null; + # pkgs = flake.packages.${system} + # or flake.legacyPackages.${system} + # or null; + # lib = flake.lib.${system} + # or flake.lib + # or null; + #}) inputs; + forSystems = systems: f: nixpkgs.lib.genAttrs systems (system: f { inherit system; - pkgs = nixpkgs.legacyPackages.${system}; - lib = nixpkgs.legacyPackages.${system}.lib; + pkgs = nixpkgs-unfree.legacyPackages.${system}; + #pkgs = nixpkgs.legacyPackages.${system}; + #pkgs = nixpkgs { config.allowUnfree = true; config.cudaSupport = true; }; + lib = nixpkgs-unfree.legacyPackages.${system}.lib; + # flakes = flake inputs system; }); forAllSystems = forSystems [ "x86_64-linux" "aarch64-linux" ]; in { + packages = forAllSystems ({ system, pkgs, lib, ...}: rec { + + # to try this, inside the default devShell do: + # apptainer shell $(nix build .#apptainer --print-out-paths --no-link) + apptainer = let + # https://github.com/NixOS/nixpkgs/issues/177908#issuecomment-1495625986 + mk-singularity = name: { + contents, + runscript ? "#!/bin/sh\nexec ${pkgs.hello}/bin/hello", + startscript ? "#!/bin/sh\nexec ${pkgs.hello}/bin/hello", + env ? {}, + shellHook ? "", + }: + pkgs.runCommand "${name}.sqfs" { + outputs = [ "out" "tree" ]; + nativeBuildInputs = [ pkgs.squashfsTools ]; + env.shellHookData = shellHook; + env.closureInfo = pkgs.closureInfo { + rootPaths = contents ++ [ pkgs.bashInteractive ]; + }; + env.environVars = pkgs.writeText "env" (lib.pipe env [ + (lib.mapAttrsToList (key: val: "${key}=${lib.escapeShellArg val}")) + (lib.concatStringsSep "\n") + ]); + } '' + set -o pipefail + set -x + mkdir -p $tree/{bin,etc/ssl/certs,dev,proc,sys,usr/bin,.singularity.d/{actions,env,libs}} + cd $tree + cp -na --parents $(cat $closureInfo/store-paths) . + touch etc/{passwd,group} + #ln -s /bin usr/ + #ln -s ${pkgs.bashInteractive}/bin/bash bin/sh + cp -a ${pkgs.pkgsStatic.bashInteractive}/bin/bash bin/sh + cp -a ${pkgs.pkgsStatic.nix}/bin/* bin/ + cp -a ${pkgs.pkgsStatic.nix}/etc/profile.d/nix.sh .singularity.d/env/ + cp -a ${pkgs.cacert}/etc/ssl/certs/ca-bundle.crt etc/ssl/certs/ca-certificates.crt + for p in ${lib.concatStringsSep " " contents}; do + ln -sn $p/bin/* bin/ || true + done + echo "${runscript}" >.singularity.d/runscript + echo "${startscript}" >.singularity.d/startscript + chmod +x .singularity.d/{runscript,startscript} + cat "$environVars" >.singularity.d/env/99-nix-env.sh + echo "$shellHookData" >.singularity.d/env/99-nix-shell-hook.sh + mksquashfs $tree $out -no-hardlinks -all-root + ''; + in mk-singularity "testing123" { + env.PS1 = "\\033[01;32m\\u@\\h\\033[33m(nix) \\033[01;34m\\W\\033[01;32m\\$\\033[00m "; + #shellHook = '' + # export PS1="\033[33m(nix)\033[00m $PS1" + #''; + contents = with pkgs; [ + #pkgsStatic.nix + ]; + }; + #} '' + # mkdir unpack + # tar xzvf ${docker-img}/image.tgz -C unpack + # # Singularity can't handle .gz + # tar -C unpack/ -cvf layer.tar . + # # TODO: Allow for module of user defined nightly, opposed to using src + # singularity build $out Singularity.nightly + # ''; + #}; + + # https://nixos.org/manual/nixpkgs/stable/#ssec-pkgs-dockerTools-buildImage + hpc-oci = pkgs.dockerTools.buildLayeredImage { + name = "hpc-oci"; + #config.Cmd = [ "${pkgs.mysql}/bin/mysqld" ]; + config.Cmd = [ "/bin/bash" ]; + config.WorkingDir = "/data"; + config.Volumes."/data" = { }; + #copyToRoot = pkgs.buildEnv { + # name = "image-root"; + # pathsToLink = [ "/bin" ]; + # paths = with pkgs; [ + # redis + # ]; + #}; + }; + #hpc-oci2 = flakes.nix2container.pkgs.nix2container.buildImage { + # name = "hello"; + # config.entrypoint = ["${pkgs.hello}/bin/hello" ]; + #}; + }); devShells = forAllSystems ({ pkgs, ...}: { - default = pkgs.mkShell { + + + default = pkgs.mkShellNoCC { + #env.APPTAINER_BINDPATH = ".direnv/nix:/nix"; + #env.SINGULARITY_BINDPATH = ".direnv/nix:/nix"; + env.APPTAINER_BINDPATH = "/usr,/lib,/lib64,.direnv/nix:/nix"; + env.SINGULARITY_BINDPATH = "/usr,/lib,/lib64,.direnv/nix:/nix"; + #env.APPTAINER_BINDPATH = "/usr,/lib,/lib64"; + #env.SINGULARITY_BINDPATH = "/usr,/lib,/lib64"; packages = with pkgs; [ remote-exec (python3.withPackages (ps: with ps; [ @@ -24,6 +146,7 @@ ])) ]; }; + }); }; }