diff --git a/flake.nix b/flake.nix index a83932a..3436e1d 100644 --- a/flake.nix +++ b/flake.nix @@ -11,17 +11,20 @@ nixpkgs, ... } @ inputs: let - systems = [ - "x86_64-linux" - "aarch64-linux" - ]; - forAllSystems = f: nixpkgs.lib.genAttrs systems (system: f { + forSystems = systems: f: nixpkgs.lib.genAttrs systems (system: f { inherit system; pkgs = nixpkgs.legacyPackages.${system}; lib = nixpkgs.legacyPackages.${system}.lib; }); + forAllSystems = forSystems [ + "x86_64-linux" + "aarch64-linux" + ]; in { inherit inputs; + packages = forAllSystems ({pkgs, ...}: { + + }); devShells = forAllSystems ({pkgs, ...}: { default = pkgs.mkShell { packages = with pkgs; [ @@ -35,7 +38,8 @@ httpx rich typer - dataset + #dataset + diskcache python-lsp-server ])) diff --git a/main.py b/main.py index b85ecbf..8c65288 100755 --- a/main.py +++ b/main.py @@ -3,8 +3,9 @@ from concurrent.futures import ThreadPoolExecutor, as_completed from enum import Enum from pathlib import Path from typing import Literal, TypedDict -from functool import lru_cache -import dataset +from functools import lru_cache +import diskcache +#import dataset import httpx import json import os @@ -18,9 +19,7 @@ import typer HERE = Path(__file__).parent.resolve() -#CACHE = dataset.connect(f"sqlite:///{HERE}") -CACHE = dataset.connect("sqlite:///ppm-cache.db") -CACHE.create_table("packages", primary_id="name") +persistent_cache = diskcache.FanoutCache(Path(__file__).parent / ".cache") def run(cmd: list[str] | str, **kw) -> subprocess.CompletedProcess: if isinstance(cmd, str): @@ -67,8 +66,12 @@ class Package(TypedDict): class PackageDetailed(Package): versions : dict[str, PackageMeta] +@persistent_cache.memoize() def api_get(path, kw={}, check=True, **params) -> httpx.Response: - resp = httpx.get(f"{API}/{path}", params=params, timeout=30, **kw) + url = f"{API}/{path}" + print(f"GET {url!r}...", file=sys.stderr) + resp = httpx.get(url, params=params, timeout=30, **kw) + print(f"GET {url!r}, {resp.is_success = }", file=sys.stderr) if check: resp.raise_for_status() return resp @@ -122,7 +125,7 @@ def mk_derivation( package : Package, version : None | str = None, format : bool = True, - dir : Path = HERE / "pkgs", + workdir : Path = HERE / "pkgs", call_package : bool = True, ) -> str: #url = get_tarball(package.name, package["metadata"]["version"], url_only=True) @@ -163,8 +166,8 @@ def mk_derivation( is_yarn = True else: - (dir / "lockfiles").mkdir(parents=True, exist_ok=True) - lock_path = dir / "lockfiles" / f"{name}.json" + (workdir / "lockfiles").mkdir(parents=True, exist_ok=True) + lock_path = workdir / "lockfiles" / f"{name}-{version}.json" # TODO: somehow sandbox this if not lock_path.is_file(): @@ -174,7 +177,7 @@ def mk_derivation( run(["npm", "install", "--package-lock-only"], cwd=tmp, stdout=None) shutil.move(f"{tmp}/package-lock.json", lock_path) - extra.append(f'postPatch = "ln -s ${{./lockfiles/{name}.json}} ./package-lock.json";') + extra.append(f'postPatch = "ln -s ${{./lockfiles/{name}-{version}.json}} ./package-lock.json";') #assert not is_yarn if is_yarn: @@ -196,7 +199,7 @@ def mk_derivation( {' '.join(extra)} nativeBuildInputs = [ python3 ]; # node-gyp npmFlags = [ "--legacy-peer-deps" ]; - ELECTRON_SKIP_BINARY_DOWNLOAD = "1"; # + ELECTRON_SKIP_BINARY_DOWNLOAD = "1"; NODE_OPTIONS = "--no-experimental-fetch"; # https://github.com/parcel-bundler/parcel/issues/8005 meta = {{ homepage = "https://web.pulsar-edit.dev/packages/{name}"; @@ -264,21 +267,24 @@ def search( @app.command() def crawl(pages: int = 10, j: int = 1): - # TODO: have it populate a cache - # TODO: make the getters use the cache - raise NotImplementedError with ThreadPoolExecutor(max_workers=j or None) as e: - futures = [e.submit(search_packages, page=page) for page in range(pages)] - #futures += [e.submit(search_themes, page=page) for page in range(pages)] + futures = [] + for page in range(pages): + @futures.append + @e.submit + def future(page=page): + return search_packages(page=page) + #return search_themes(page=page) + for future in as_completed(futures): for package in future.result(): - print(package.name) + print(package["name"], file=sys.stderr) print(json.dumps(package)) @app.command() def drv(name: str, version: str | None = None, dir: Path = HERE / "pkgs"): package = get_package(name) - expr = mk_derivation(package, dir=dir) + expr = mk_derivation(package, workdir=dir) print(expr) try: out_path = run([