README.md

@@ -1,9 +1,5 @@
-# jadb
+# ja_db
 
 [![built with nix](https://builtwithnix.org/badge.svg)](https://builtwithnix.org)
 
-An SQLite database containing open source japanese language translation data combined from several sources
 
-- **JMDict:** https://edrdg.org/jmdict/j_jmdict.html
-- **RADKFILE/KRADFILE:** https://www.edrdg.org/krad/kradinf.html
-- **KANJIDIC2:** https://www.edrdg.org/kanjidic/kanjd2index_legacy.html

bin/ja_db.dart

@@ -12,7 +12,7 @@ Future<void> main(List<String> arguments) async {
   final db = await createDatabaseFactoryFfi(ffiInit: () {
     open.overrideForAll(() => DynamicLibrary.open(arguments[0]));
   })
-    .openDatabase(Directory.current.uri.resolve('jadb.sqlite').path);
+    .openDatabase(Directory.current.uri.resolve('main.db').path);
   await addDataFromJMdict(db);
   await addDataFromRADKFILE(db);
   await addDataFromKANJIDIC(db);

flake.lock

@@ -1,24 +1,9 @@
 {
   "nodes": {
-    "flake-utils": {
-      "locked": {
-        "lastModified": 1649676176,
-        "narHash": "sha256-OWKJratjt2RW151VUlJPRALb7OU2S5s+f0vLj4o1bHM=",
-        "owner": "numtide",
-        "repo": "flake-utils",
-        "rev": "a4b154ebbdc88c8498a5c7b01589addc9e9cb678",
-        "type": "github"
-      },
-      "original": {
-        "owner": "numtide",
-        "repo": "flake-utils",
-        "type": "github"
-      }
-    },
-    "jmdict-src": {
+    "JMdictSrc": {
       "flake": false,
       "locked": {
-        "narHash": "sha256-997+CJDljSggj2Kh9+1Xd+yvLJUkleQ7b5iuYIEwswc=",
+        "narHash": "sha256-7hGfayIXi8mH80025s7Pgqk7jwOvZlQMHsxHv0sq5ek=",
         "type": "file",
         "url": "http://ftp.edrdg.org/pub/Nihongo/JMdict.gz"
       },
@@ -27,10 +12,10 @@
         "url": "http://ftp.edrdg.org/pub/Nihongo/JMdict.gz"
       }
     },
-    "jmdict-with-examples-src": {
+    "JMdictWithExamplesSrc": {
       "flake": false,
       "locked": {
-        "narHash": "sha256-6GU10LW5XO76ptXC+DNBRusuFI1E1i8Sda63sTXCsFU=",
+        "narHash": "sha256-OtxCfNaIbRVBuztTiNim7xjxGDFrV9MCPpymLLYBMnk=",
         "type": "file",
         "url": "http://ftp.edrdg.org/pub/Nihongo/JMdict_e_examp.gz"
       },
@@ -39,7 +24,7 @@
         "url": "http://ftp.edrdg.org/pub/Nihongo/JMdict_e_examp.gz"
       }
     },
-    "kanjidic2-src": {
+    "KANJIDIC2Src": {
       "flake": false,
       "locked": {
         "narHash": "sha256-BaVQaAEisPC60ohz8Gyr5r0Fe4Qrjupj6VKQqMvq9Eo=",
@@ -51,9 +36,38 @@
         "url": "http://nihongo.monash.edu/kanjidic2/kanjidic2.xml.gz"
       }
     },
+    "RADKFILESrc": {
+      "flake": false,
+      "locked": {
+        "narHash": "sha256-rO2z5GPt3g6osZOlpyWysmIbRV2Gw4AR4XvngVTHNpk=",
+        "type": "file",
+        "url": "http://ftp.usf.edu/pub/ftp.monash.edu.au/pub/nihongo/radkfile.gz"
+      },
+      "original": {
+        "type": "file",
+        "url": "http://ftp.usf.edu/pub/ftp.monash.edu.au/pub/nihongo/radkfile.gz"
+      }
+    },
+    "flake-utils": {
+      "locked": {
+        "lastModified": 1659877975,
+        "narHash": "sha256-zllb8aq3YO3h8B/U0/J1WBgAL8EX5yWf5pMj3G0NAmc=",
+        "owner": "numtide",
+        "repo": "flake-utils",
+        "rev": "c0e246b9b83f637f4681389ecabcb2681b4f3af0",
+        "type": "github"
+      },
+      "original": {
+        "owner": "numtide",
+        "repo": "flake-utils",
+        "type": "github"
+      }
+    },
     "nix-dart": {
       "inputs": {
-        "flake-utils": "flake-utils",
+        "flake-utils": [
+          "flake-utils"
+        ],
         "nixpkgs": [
           "nixpkgs"
         ],
@@ -75,16 +89,16 @@
     },
     "nixpkgs": {
       "locked": {
-        "lastModified": 1731386116,
-        "narHash": "sha256-lKA770aUmjPHdTaJWnP3yQ9OI1TigenUqVC3wweqZuI=",
+        "lastModified": 1661187878,
+        "narHash": "sha256-/wCqoQB1BsaVi4nb8Iz0PreeBNMTim0p78NLtyWejFE=",
         "owner": "NixOS",
         "repo": "nixpkgs",
-        "rev": "689fed12a013f56d4c4d3f612489634267d86529",
+        "rev": "52527082ea267fe486f0648582d57c85486b2031",
         "type": "github"
       },
       "original": {
         "id": "nixpkgs",
-        "ref": "nixos-24.05",
+        "ref": "nixos-22.05",
         "type": "indirect"
       }
     },
@@ -104,26 +118,15 @@
         "type": "github"
       }
     },
-    "radkfile-src": {
-      "flake": false,
-      "locked": {
-        "narHash": "sha256-rO2z5GPt3g6osZOlpyWysmIbRV2Gw4AR4XvngVTHNpk=",
-        "type": "file",
-        "url": "http://ftp.usf.edu/pub/ftp.monash.edu.au/pub/nihongo/radkfile.gz"
-      },
-      "original": {
-        "type": "file",
-        "url": "http://ftp.usf.edu/pub/ftp.monash.edu.au/pub/nihongo/radkfile.gz"
-      }
-    },
     "root": {
       "inputs": {
-        "jmdict-src": "jmdict-src",
-        "jmdict-with-examples-src": "jmdict-with-examples-src",
-        "kanjidic2-src": "kanjidic2-src",
+        "JMdictSrc": "JMdictSrc",
+        "JMdictWithExamplesSrc": "JMdictWithExamplesSrc",
+        "KANJIDIC2Src": "KANJIDIC2Src",
+        "RADKFILESrc": "RADKFILESrc",
+        "flake-utils": "flake-utils",
         "nix-dart": "nix-dart",
-        "nixpkgs": "nixpkgs",
-        "radkfile-src": "radkfile-src"
+        "nixpkgs": "nixpkgs"
       }
     }
   },

flake.nix

@@ -2,29 +2,37 @@
   description = "A SQLite database containing open source japanese language translation data";
 
   inputs = {
-    nixpkgs.url = "nixpkgs/nixos-24.05";
+    nixpkgs.url = "nixpkgs/nixos-22.05";
 
-    nix-dart = {
-      url = "github:tadfisher/nix-dart";
+    flake-utils = {
+      url = "github:numtide/flake-utils";
       inputs.nixpkgs.follows = "nixpkgs";
     };
 
-    jmdict-src = {
+    nix-dart = {
+      url = "github:tadfisher/nix-dart";
+      inputs = {
+        nixpkgs.follows = "nixpkgs";
+        flake-utils.follows = "flake-utils";
+      };
+    };
+
+    JMdictSrc = {
       url = "http://ftp.edrdg.org/pub/Nihongo/JMdict.gz";
       flake = false;
     };
 
-    jmdict-with-examples-src = {
+    JMdictWithExamplesSrc = {
       url = "http://ftp.edrdg.org/pub/Nihongo/JMdict_e_examp.gz";
       flake = false;
     };
 
-    radkfile-src = {
+    RADKFILESrc = {
       url = "http://ftp.usf.edu/pub/ftp.monash.edu.au/pub/nihongo/radkfile.gz";
       flake = false;
     };
 
-    kanjidic2-src = {
+    KANJIDIC2Src = {
       url = "http://nihongo.monash.edu/kanjidic2/kanjidic2.xml.gz";
       flake = false;
     };
@@ -34,96 +42,216 @@
   outputs = {
     self,
     nixpkgs,
+    flake-utils,
     nix-dart,
-    jmdict-src,
-    jmdict-with-examples-src,
-    radkfile-src,
-    kanjidic2-src
+    JMdictSrc,
+    JMdictWithExamplesSrc,
+    RADKFILESrc,
+    KANJIDIC2Src
   }: let
-    inherit (nixpkgs) lib;
-    systems = [
-      "x86_64-linux"
-      "aarch64-linux"
-      "x86_64-darwin"
-      "aarch64-darwin"
-      "armv7l-linux"
-    ];
-
-    forAllSystems = f: lib.genAttrs systems (system: let
+    system = "x86_64-linux";
     pkgs = import nixpkgs {
       inherit system;
       overlays = [
+        # (final: prev: { dart = nix-dart.packages.${system}.dart; })
         nix-dart.overlay
       ];
     };
-    in f system pkgs);
-  in {
-    apps = forAllSystems (system: pkgs: {
-      default = let
-        inherit (self.packages.${system}) docs;
-      in {
-        type = "app";
-        program = "${pkgs.writeShellScript "host-docs" ''
-          ${pkgs.python3} -m http.server -d ${docs}
-        ''}";
-      };
-    });
+    inherit (pkgs) lib;
 
-    devShells = forAllSystems (system: pkgs: {
-      default = pkgs.mkShell {
+  in {
+    devShell.${system} = pkgs.mkShell {
       buildInputs = with pkgs; [
         nix-dart.packages.${system}.pub2nix-lock
         dart
         gnumake
-          sqlite-interactive
+        sqlite
         sqlite-web
         sqlint
         sqlfluff
       ];
     };
-    });
 
-    # defaultPackage.${system} = self.packages.${system}.database;
+    defaultPackage.${system} = self.packages.${system}.database;
+
+    packages.${system} = let
+      inherit (pkgs.stdenv) mkDerivation;
+      dbName = "main.db";
 
-    packages = let
       edrdgMetadata = {
-        license = [{
+        license = {
           shortName = "EDRDG";
           fullName = "Electronic Dictionary Research and Development Group General Dictionary Licence";
           url = "http://www.csse.monash.edu.au/~jwb/edrdg/licence.html";
-        }];
-        maintainers = [ lib.maintainers.h7x4 ];
+        };
+        maintainers = [ "h7x4 <h7x4@nani.wtf>" ];
         platforms = lib.platforms.all;
       };
+    in {
+      JMdict = mkDerivation {
+        name = "JMdict";
 
-      src = lib.cleanSource ./.;
-    in forAllSystems (system: pkgs: {
-      default = self.packages.${system}.database;
+        srcs = [
+          JMdictSrc
+          JMdictWithExamplesSrc
+        ];
+        dontUnpack = true;
 
-      jmdict = pkgs.callPackage ./nix/jmdict.nix {
-          inherit jmdict-src jmdict-with-examples-src edrdgMetadata;
+        nativeBuildInputs = with pkgs; [ xmlformat ];
+        buildPhase = ''
+          gzip -dkc ${JMdictSrc} > JMdict.xml
+          gzip -dkc ${JMdictWithExamplesSrc} > JMdict_with_examples.xml
+          xmlformat -i JMdict.xml
+          xmlformat -i JMdict_with_examples.xml
+        '';
+
+        installPhase = ''
+          mkdir $out
+          cp JMdict.xml $out
+          cp JMdict_with_examples.xml $out
+        '';
+
+        meta = edrdgMetadata // {
+          description = "A Japanese-Multilingual Dictionary providing lexical data for japanese words";
+          homepage = "https://www.edrdg.org/jmdict/j_jmdict.html";
+        };
       };
 
-      radkfile =  pkgs.callPackage ./nix/radkfile.nix {
-        inherit radkfile-src edrdgMetadata;
+      RADKFILE = mkDerivation {
+        name = "RADKFILE";
+
+        src = RADKFILESrc;
+        dontUnpack = true;
+
+        buildPhase = ''
+          gzip -dkc $src > radkfile
+          iconv -f EUC-JP -t UTF-8 -o radkfile_utf8 radkfile
+        '';
+
+        installPhase = ''
+          mkdir $out
+          cp radkfile_utf8 $out
+        '';
+
+        meta = edrdgMetadata // {
+          description = "A file providing searchable decompositions of kanji characters";
+          homepage = "https://www.edrdg.org/krad/kradinf.html";
+        };
       };
 
-      kanjidic2 = pkgs.callPackage ./nix/kanjidic2.nix {
-        inherit kanjidic2-src edrdgMetadata;
+      KANJIDIC2 = mkDerivation {
+        name = "KANJIDIC2";
+
+        src = KANJIDIC2Src;
+        dontUnpack = true;
+
+        nativeBuildInputs = with pkgs; [ xmlformat ];
+        buildPhase = ''
+          gzip -dkc ${KANJIDIC2Src} > kanjidic2.xml
+        '';
+          # xmlformat -i kanjidic2.xml
+
+        installPhase = ''
+          mkdir $out
+          cp kanjidic2.xml $out
+        '';
+
+        meta = edrdgMetadata // {
+          # description = "A Japanese-Multilingual Dictionary providing lexical data for japanese words";
+          # homepage = "https://www.edrdg.org/jmdict/j_jmdict.html";
+        };
       };
 
-      database-tool = pkgs.callPackage ./nix/database_tool.nix {
-        inherit nix-dart src;
+      database_generator = let
+        buildDartPackage = nix-dart.builders.${system}.buildDartPackage.override {
+          dart = nix-dart.packages.${system}.dart-dev;
         };
+      in (buildDartPackage {
+        pname = "database_generator";
+        version = "1.0";
 
-      database = pkgs.callPackage ./nix/database.nix {
-        inherit (self.packages.${system}) database-tool jmdict radkfile kanjidic2;
-        inherit src;
-      };
+        src = builtins.filterSource (path: type: baseNameOf path != ".dart_tool") ./.;
 
-      docs = pkgs.callPackage ./nix/docs.nix {
-        inherit (self.packages.${system}) database;
-      };
+        preBuild = ''
+          dart --version
+        '';
+
+        specFile = ./pubspec.yaml;
+        lockFile = ./pub2nix.lock;
+      }).overrideAttrs(old: {
+        buildInputs = [nix-dart.packages.${system}.dart-dev];
+        buildPhase = builtins.replaceStrings ["pub"] ["dart pub"] old.buildPhase;
       });
+
+      database = mkDerivation {
+        name = "database";
+        src = builtins.filterSource (path: type: baseNameOf path != dbName) ./.;
+
+        nativeBuildInputs = with pkgs; [
+          sqlite
+          self.packages.${system}.database_generator
+        ];
+        buildPhase = ''
+          mkdir -p data
+          ln -s ${self.packages.${system}.JMdict}/* data
+          ln -s ${self.packages.${system}.RADKFILE}/* data
+          ln -s ${self.packages.${system}.KANJIDIC2}/* data
+
+          for migration in migrations/*.sql; do
+            sqlite3 ${dbName} < $migration
+          done
+
+          ja_db ${pkgs.sqlite.out}/lib/libsqlite3.so
+        '';
+
+        installPhase = ''
+          mkdir -p $out
+          cp ${dbName} $out/${dbName}
+        '';
       };
+
+      docs = mkDerivation {
+        name = "docs";
+        src = self.packages.${system}.database;
+        nativeBuildInputs = with pkgs; [
+          sqlite
+          schemaspy
+          sqlite-jdbc
+        ];
+
+        buildPhase = let
+          properties = pkgs.writeText "sqlite.properties" ''
+            description=SQLite
+            driver=org.sqlite.JDBC
+            driverPath=${pkgs.sqlite-jdbc}/share/java/sqlite-jdbc-3.25.2.jar
+            connectionSpec=jdbc:sqlite:<db>
+          '';
+
+          args = pkgs.writeText "schemaspy.properties" ''
+            schemaspy.cat="%"
+            schemaspy.t=sqlite
+            schemaspy.sso=true
+            schemaspy.db=${dbName}
+            schemaspy.o=docs
+            schemaspy.s=schema.sql
+          '';
+
+        in ''
+          sqlite3 main.db ".schema" > schema.sql
+          cp ${args} ./schemaspy.properties
+          ls
+          schemaspy -t ${properties}
+        '';
+
+        installPhase = ''
+          cp -r docs $out
+        '';
+      };
+    };
+
+    hydraJobs = {
+      inherit (self.packages.${system}) database docs;
+    };
+  };
+
 }

nix/database.nix

@@ -1,43 +0,0 @@
-{
-  lib,
-  stdenvNoCC,
-  src,
-  database-tool,
-  jmdict,
-  radkfile,
-  kanjidic2,
-  sqlite,
-}:
-stdenvNoCC.mkDerivation {
-  name = "jadb";
-  inherit src;
-
-  nativeBuildInputs = [
-    database-tool
-    sqlite
-  ];
-  buildPhase = ''
-    runHook preBuild
-
-    mkdir -p data
-    ln -s ${jmdict}/* data
-    ln -s ${radkfile}/* data
-    ln -s ${kanjidic2}/* data
-
-    for migration in migrations/*.sql; do
-      sqlite3 jadb.sqlite < $migration
-    done
-
-    ${lib.getExe database-tool} ${sqlite.out}/lib/libsqlite3.so
-
-    runHook postBuild
-  '';
-
-  installPhase = ''
-    runHook preInstall
-
-    install -Dm644 -t $out jadb.sqlite
-
-    runHook postInstall
-  '';
-}

nix/database_tool.nix

@@ -1,27 +0,0 @@
-{
-  nix-dart,
-  system,
-  src,
-}:
-let
-  buildDartPackage = nix-dart.builders.${system}.buildDartPackage.override {
-    dart = nix-dart.packages.${system}.dart-dev;
-  };
-in (buildDartPackage {
-  pname = "database_tool";
-  version = "1.0";
-
-  inherit src;
-
-  preBuild = ''
-    dart --version
-  '';
-
-  specFile = ../pubspec.yaml;
-  lockFile = ../pub2nix.lock;
-
-  meta.mainProgram = "ja_db";
-}).overrideAttrs(old: {
-  buildInputs = [nix-dart.packages.${system}.dart-dev];
-  buildPhase = builtins.replaceStrings ["pub"] ["dart pub"] old.buildPhase;
-})

nix/docs.nix

@@ -1,53 +0,0 @@
-{
-  stdenvNoCC,
-  database,
-  sqlite,
-  schemaspy,
-  sqlite-jdbc,
-  writeText,
-
-}:
-stdenvNoCC.mkDerivation {
-  name = "docs";
-  src = database;
-  nativeBuildInputs = [
-    sqlite
-    schemaspy
-    sqlite-jdbc
-  ];
-
-  buildPhase = let
-    properties = writeText "sqlite.properties" ''
-      description=SQLite
-      driver=org.sqlite.JDBC
-      driverPath=${sqlite-jdbc}/share/java/sqlite-jdbc-3.25.2.jar
-      connectionSpec=jdbc:sqlite:<db>
-    '';
-
-    args = writeText "schemaspy.properties" ''
-      schemaspy.cat="%"
-      schemaspy.t=sqlite
-      schemaspy.sso=true
-      schemaspy.db=jadb.sqlite
-      schemaspy.o=docs
-      schemaspy.s=schema.sql
-    '';
-
-  in ''
-    runHook preBuild
-
-    sqlite3 jadb.sqlite ".schema" > schema.sql
-    cp ${args} ./schemaspy.properties
-    schemaspy -t ${properties}
-
-    runHook postBuild
-  '';
-
-  installPhase = ''
-    runHook preBuild
-
-    cp -r docs $out
-
-    runHook postBuild
-  '';
-}

nix/jmdict.nix

@@ -1,45 +0,0 @@
-{
-  stdenvNoCC,
-  jmdict-src,
-  jmdict-with-examples-src,
-  xmlformat,
-  gzip,
-  edrdgMetadata,
-}:
-stdenvNoCC.mkDerivation {
-  name = "jmdict";
-
-  srcs = [
-    jmdict-src
-    jmdict-with-examples-src
-  ];
-  dontUnpack = true;
-
-  nativeBuildInputs = [
-    gzip
-    xmlformat
-  ];
-  buildPhase = ''
-    runHook preBuild
-
-    gzip -dkc ${jmdict-src} > JMdict.xml
-    gzip -dkc ${jmdict-with-examples-src} > JMdict_with_examples.xml
-    xmlformat -i JMdict.xml
-    xmlformat -i JMdict_with_examples.xml
-
-    runHook postBuild
-  '';
-
-  installPhase = ''
-    runHook preInstall
-
-    install -Dt $out JMdict.xml JMdict_with_examples.xml
-
-    runHook postInstall
-  '';
-
-  meta = edrdgMetadata // {
-    description = "A Japanese-Multilingual Dictionary providing lexical data for japanese words";
-    homepage = "https://www.edrdg.org/jmdict/j_jmdict.html";
-  };
-}

nix/kanjidic2.nix

@@ -1,39 +0,0 @@
-{
-  stdenvNoCC,
-  kanjidic2-src,
-  xmlformat,
-  gzip,
-  edrdgMetadata,
-}:
-stdenvNoCC.mkDerivation {
-  name = "kanjidic2";
-
-  src = kanjidic2-src;
-  dontUnpack = true;
-
-  nativeBuildInputs = [
-    gzip
-    xmlformat
-  ];
-  buildPhase = ''
-    runHook preBuild
-
-    gzip -dkc ${kanjidic2-src} > kanjidic2.xml
-    # xmlformat -i kanjidic2.xml
-
-    runHook postBuild
-  '';
-
-  installPhase = ''
-    runHook preInstall
-
-    install -Dt $out kanjidic2.xml
-
-    runHook postInstall
-  '';
-
-  meta = edrdgMetadata // {
-    # description = "A Japanese-Multilingual Dictionary providing lexical data for japanese words";
-    # homepage = "https://www.edrdg.org/jmdict/j_jmdict.html";
-  };
-}

nix/radkfile.nix

@@ -1,39 +0,0 @@
-{
-  stdenv,
-  radkfile-src,
-  gzip,
-  iconv,
-  edrdgMetadata,
-}:
-stdenv.mkDerivation {
-  name = "radkfile";
-
-  src = radkfile-src;
-  dontUnpack = true;
-
-  nativeBuildInputs = [
-    gzip
-    iconv
-  ];
-  buildPhase = ''
-    runHook preBuild
-
-    gzip -dkc $src > radkfile
-    iconv -f EUC-JP -t UTF-8 -o radkfile_utf8 radkfile
-
-    runHook postBuild
-  '';
-
-  installPhase = ''
-    runHook preInstall
-
-    install -Dt $out radkfile_utf8
-
-    runHook postInstall
-  '';
-
-  meta = edrdgMetadata // {
-    description = "A file providing searchable decompositions of kanji characters";
-    homepage = "https://www.edrdg.org/krad/kradinf.html";
-  };
-}