whoohaa

flake.nix

@@ -11,17 +11,20 @@
     nixpkgs,
     ... } @ inputs:
   let
-    systems = [
-      "x86_64-linux"
-      "aarch64-linux"
-    ];
-    forAllSystems = f: nixpkgs.lib.genAttrs systems (system: f {
+    forSystems = systems: f: nixpkgs.lib.genAttrs systems (system: f {
       inherit system;
       pkgs = nixpkgs.legacyPackages.${system};
       lib = nixpkgs.legacyPackages.${system}.lib;
     });
+    forAllSystems = forSystems [
+      "x86_64-linux"
+      "aarch64-linux"
+    ];
   in {
     inherit inputs;
+    packages = forAllSystems ({pkgs, ...}: {
+
+    });
     devShells = forAllSystems ({pkgs, ...}: {
       default = pkgs.mkShell {
         packages = with pkgs; [
@@ -35,7 +38,8 @@
             httpx
             rich
             typer
-            dataset
+            #dataset
+            diskcache
             python-lsp-server
           ]))
 

main.py

@@ -3,8 +3,9 @@ from concurrent.futures import ThreadPoolExecutor, as_completed
 from enum import Enum
 from pathlib import Path
 from typing import Literal, TypedDict
-from functool import lru_cache
-import dataset
+from functools import lru_cache
+import diskcache
+#import dataset
 import httpx
 import json
 import os
@@ -18,9 +19,7 @@ import typer
 
 HERE = Path(__file__).parent.resolve()
 
-#CACHE = dataset.connect(f"sqlite:///{HERE}")
-CACHE = dataset.connect("sqlite:///ppm-cache.db")
-CACHE.create_table("packages", primary_id="name")
+persistent_cache = diskcache.FanoutCache(Path(__file__).parent / ".cache")
 
 def run(cmd: list[str] | str, **kw) -> subprocess.CompletedProcess:
     if isinstance(cmd, str):
@@ -67,8 +66,12 @@ class Package(TypedDict):
 class PackageDetailed(Package):
     versions : dict[str, PackageMeta]
 
+@persistent_cache.memoize()
 def api_get(path, kw={}, check=True, **params) -> httpx.Response:
-    resp = httpx.get(f"{API}/{path}", params=params, timeout=30, **kw)
+    url = f"{API}/{path}"
+    print(f"GET {url!r}...", file=sys.stderr)
+    resp = httpx.get(url, params=params, timeout=30, **kw)
+    print(f"GET {url!r}, {resp.is_success = }", file=sys.stderr)
     if check: resp.raise_for_status()
     return resp
 
@@ -122,7 +125,7 @@ def mk_derivation(
         package      : Package,
         version      : None | str = None,
         format       : bool       = True,
-        dir          : Path       = HERE / "pkgs",
+        workdir      : Path       = HERE / "pkgs",
         call_package : bool       = True,
     ) -> str:
     #url = get_tarball(package.name, package["metadata"]["version"], url_only=True)
@@ -163,8 +166,8 @@ def mk_derivation(
         is_yarn = True
 
     else:
-        (dir / "lockfiles").mkdir(parents=True, exist_ok=True)
-        lock_path = dir / "lockfiles" / f"{name}.json"
+        (workdir / "lockfiles").mkdir(parents=True, exist_ok=True)
+        lock_path = workdir / "lockfiles" / f"{name}-{version}.json"
 
         # TODO: somehow sandbox this
         if not lock_path.is_file():
@@ -174,7 +177,7 @@ def mk_derivation(
                 run(["npm", "install", "--package-lock-only"], cwd=tmp, stdout=None)
                 shutil.move(f"{tmp}/package-lock.json", lock_path)
 
-        extra.append(f'postPatch = "ln -s ${{./lockfiles/{name}.json}} ./package-lock.json";')
+        extra.append(f'postPatch = "ln -s ${{./lockfiles/{name}-{version}.json}} ./package-lock.json";')
 
     #assert not is_yarn
     if is_yarn:
@@ -196,7 +199,7 @@ def mk_derivation(
             {' '.join(extra)}
             nativeBuildInputs = [ python3 ]; # node-gyp
             npmFlags = [ "--legacy-peer-deps" ];
-            ELECTRON_SKIP_BINARY_DOWNLOAD = "1"; #
+            ELECTRON_SKIP_BINARY_DOWNLOAD = "1";
             NODE_OPTIONS = "--no-experimental-fetch"; # https://github.com/parcel-bundler/parcel/issues/8005
             meta = {{
                 homepage = "https://web.pulsar-edit.dev/packages/{name}";
@@ -264,21 +267,24 @@ def search(
 
 @app.command()
 def crawl(pages: int = 10, j: int = 1):
-    # TODO: have it populate a cache
-    # TODO: make the getters use the cache
-    raise NotImplementedError
     with ThreadPoolExecutor(max_workers=j or None) as e:
-        futures  = [e.submit(search_packages, page=page) for page in range(pages)]
-        #futures += [e.submit(search_themes,   page=page) for page in range(pages)]
+        futures  = []
+        for page in range(pages):
+            @futures.append
+            @e.submit
+            def future(page=page):
+                return search_packages(page=page)
+                #return search_themes(page=page)
+
         for future in as_completed(futures):
             for package in future.result():
-                print(package.name)
+                print(package["name"], file=sys.stderr)
                 print(json.dumps(package))
 
 @app.command()
 def drv(name: str, version: str | None = None, dir: Path = HERE / "pkgs"):
     package = get_package(name)
-    expr = mk_derivation(package, dir=dir)
+    expr = mk_derivation(package, workdir=dir)
     print(expr)
     try:
         out_path = run([