WIP

flake.nix

@@ -43,7 +43,12 @@
       "armv7l-linux"
     ];
 
-    forAllSystems = f: lib.genAttrs systems (system: f system nixpkgs.legacyPackages.${system});
+    forAllSystems = f: lib.genAttrs systems (system: let
+      pkgs = import nixpkgs {
+        inherit system;
+        overlays = [ self.overlays.sqlite-icu-ext ];
+      };
+    in f system pkgs);
   in {
     apps = forAllSystems (system: pkgs: {
       default = {
@@ -77,16 +82,12 @@
 
     devShells = forAllSystems (system: pkgs: {
       default = pkgs.mkShell {
-        buildInputs = with pkgs; [
+        packages = with pkgs; [
           dart
           gnumake
           lcov
           sqldiff
-          sqlite-analyzer
+          sqlite-interactive-icu-ext
-          sqlite-interactive
-          sqlite-web
-          # sqlint
-          sqlfluff
         ];
         env = {
           LIBSQLITE_PATH = "${pkgs.sqlite.out}/lib/libsqlite3.so";
@@ -94,8 +95,34 @@
           LD_LIBRARY_PATH = lib.makeLibraryPath [ pkgs.sqlite ];
         };
       };
+
+      sqlite-debugging = pkgs.mkShell {
+        packages = with pkgs; [
+          sqlite-interactive-icu-ext
+          sqlite-analyzer
+          sqlite-web
+          sqlint
+          sqlfluff
+        ];
+      };
     });
 
+    overlays.sqlite-icu-ext = final: prev: let
+      overrideArgs = prev': {
+        configureFlags = prev'.configureFlags ++ [
+          "--with-icu-config=${lib.getExe' prev.icu.dev "icu-config"}"
+          "--enable-icu-collations"
+        ];
+
+        buildInputs = prev'.buildInputs ++ [
+          prev.icu
+        ];
+      };
+    in {
+      sqlite-icu-ext = prev.sqlite.overrideAttrs overrideArgs;
+      sqlite-interactive-icu-ext = prev.sqlite-interactive.overrideAttrs overrideArgs;
+    };
+
     packages = let
       edrdgMetadata = {
         license = [{
@@ -129,6 +156,8 @@
         ln -s ${src} $out
       '';
 
+      inherit (pkgs) sqlite-icu-ext sqlite-interactive-icu-ext;
+
       jmdict = pkgs.callPackage ./nix/jmdict.nix {
         inherit jmdict-src jmdict-with-examples-src edrdgMetadata;
       };
@@ -143,17 +172,20 @@
 
       database-tool = pkgs.callPackage ./nix/database_tool.nix {
         inherit src;
+        sqlite = pkgs.sqlite-icu-ext;
       };
 
       database = pkgs.callPackage ./nix/database.nix {
         inherit (self.packages.${system}) database-tool jmdict radkfile kanjidic2;
         inherit src;
+        sqlite = pkgs.sqlite-icu-ext;
       };
 
       database-wal = pkgs.callPackage ./nix/database.nix {
         inherit (self.packages.${system}) database-tool jmdict radkfile kanjidic2;
         inherit src;
         wal = true;
+        sqlite = pkgs.sqlite-icu-ext;
       };
 
       docs = pkgs.callPackage ./nix/docs.nix {

lib/_data_ingestion/jmdict/seed_data.dart

@@ -10,8 +10,9 @@ import 'package:sqflite_common/sqlite_api.dart';
 class ResolvedXref {
   Entry entry;
   bool ambiguous;
+  int? senseOrderNum;
 
-  ResolvedXref(this.entry, this.ambiguous);
+  ResolvedXref(this.entry, this.ambiguous, senseOrderNum);
 }
 
 /// Resolves an xref (pair of kanji, optionally reading, and optionally sense number) to an a specific
@@ -74,9 +75,10 @@ ResolvedXref resolveXref(
       'kanjiRef: ${xref.kanjiRef}, readingRef: ${xref.readingRef}, '
       'senseOrderNum: ${xref.senseOrderNum}',
     );
-    return ResolvedXref(candidateEntries.first, true);
+
+    return ResolvedXref(candidateEntries.first, true, xref.senseOrderNum);
   } else {
-    return ResolvedXref(candidateEntries.first, false);
+    return ResolvedXref(candidateEntries.first, false, xref.senseOrderNum);
   }
 }
 
@@ -161,14 +163,14 @@ Future<void> seedJMDictData(List<Entry> entries, Database db) async {
         b.insert(JMdictTableNames.senseRestrictedToKanji, {
           'entryId': e.entryId,
           'senseId': s.senseId,
-          'kanji': rk,
+          'kanjiOrderNum': e.kanji.indexWhere((k) => k.reading == rk) + 1,
         });
       }
       for (final rr in s.restrictedToReading) {
         b.insert(JMdictTableNames.senseRestrictedToReading, {
           'entryId': e.entryId,
           'senseId': s.senseId,
-          'reading': rr,
+          'readingOrderNum': e.readings.indexWhere((r) => r.reading == rr) + 1,
         });
       }
       for (final ls in s.languageSource) {
@@ -229,9 +231,7 @@ Future<void> seedJMDictData(List<Entry> entries, Database db) async {
         b.insert(JMdictTableNames.senseSeeAlso, {
           'senseId': s.senseId,
           'xrefEntryId': resolvedEntry.entry.entryId,
-          'seeAlsoKanji': xref.kanjiRef,
+          'xrefSenseOrderNum': resolvedEntry.senseOrderNum,
-          'seeAlsoReading': xref.readingRef,
-          'seeAlsoSense': xref.senseOrderNum,
           'ambiguous': resolvedEntry.ambiguous,
         });
       }
@@ -256,9 +256,6 @@ Future<void> seedJMDictData(List<Entry> entries, Database db) async {
         b.insert(JMdictTableNames.senseAntonyms, {
           'senseId': s.senseId,
           'xrefEntryId': resolvedEntry.entry.entryId,
-          'antonymKanji': ant.kanjiRef,
-          'antonymReading': ant.readingRef,
-          'antonymSense': ant.senseOrderNum,
           'ambiguous': resolvedEntry.ambiguous,
         });
       }

lib/_data_ingestion/open_local_db.dart

@@ -27,6 +27,7 @@ Future<Database> openLocalDb({
           await db.execute('PRAGMA journal_mode = WAL');
         }
         await db.execute('PRAGMA foreign_keys = ON');
+        await db.execute("SELECT icu_load_collation('ja_JP', 'japanese')");
       },
       readOnly: !readWrite,
     ),

migrations/0001_JMDict.sql

@@ -1,3 +1,5 @@
+SELECT icu_load_collation('ja_JP', 'japanese');
+
 CREATE TABLE "JMdict_Version" (
   "version" VARCHAR(10) PRIMARY KEY NOT NULL,
   "date" DATE NOT NULL,
@@ -55,13 +57,13 @@ CREATE TABLE "JMdict_KanjiElement" (
   "elementId" INTEGER PRIMARY KEY,
   "entryId" INTEGER NOT NULL REFERENCES "JMdict_Entry"("entryId"),
   "orderNum" INTEGER NOT NULL,
-  "reading" TEXT NOT NULL,
+  "reading" TEXT NOT NULL COLLATE japanese,
   "news" INTEGER CHECK ("news" BETWEEN 1 AND 2),
   "ichi" INTEGER CHECK ("ichi" BETWEEN 1 AND 2),
   "spec" INTEGER CHECK ("spec" BETWEEN 1 AND 2),
   "gai" INTEGER CHECK ("gai" BETWEEN 1 AND 2),
   "nf" INTEGER CHECK ("nf" BETWEEN 1 AND 48),
-  UNIQUE("entryId", "reading"),
+  -- UNIQUE("entryId", "reading"),
   UNIQUE("entryId", "orderNum")
 ) WITHOUT ROWID;
 
@@ -80,14 +82,14 @@ CREATE TABLE "JMdict_ReadingElement" (
   "elementId" INTEGER PRIMARY KEY,
   "entryId" INTEGER NOT NULL REFERENCES "JMdict_Entry"("entryId"),
   "orderNum" INTEGER NOT NULL,
-  "reading" TEXT NOT NULL,
+  "reading" TEXT NOT NULL COLLATE japanese,
   "readingDoesNotMatchKanji" BOOLEAN NOT NULL DEFAULT FALSE,
   "news" INTEGER CHECK ("news" BETWEEN 1 AND 2),
   "ichi" INTEGER CHECK ("ichi" BETWEEN 1 AND 2),
   "spec" INTEGER CHECK ("spec" BETWEEN 1 AND 2),
   "gai" INTEGER CHECK ("gai" BETWEEN 1 AND 2),
   "nf" INTEGER CHECK ("nf" BETWEEN 1 AND 48),
-  UNIQUE("entryId", "reading"),
+  -- UNIQUE("entryId", "reading"),
   UNIQUE("entryId", "orderNum")
 ) WITHOUT ROWID;
 
@@ -120,17 +122,17 @@ CREATE INDEX "JMdict_Sense_byEntryId_byOrderNum" ON "JMdict_Sense"("entryId", "o
 CREATE TABLE "JMdict_SenseRestrictedToKanji" (
   "entryId" INTEGER NOT NULL,
   "senseId" INTEGER NOT NULL REFERENCES "JMdict_Sense"("senseId"),
-  "kanji" TEXT NOT NULL,
+  "kanjiOrderNum" INTEGER NOT NULL CHECK ("kanjiOrderNum" > 0),
-  FOREIGN KEY ("entryId", "kanji") REFERENCES "JMdict_KanjiElement"("entryId", "reading"),
+  FOREIGN KEY ("entryId", "kanjiOrderNum") REFERENCES "JMdict_KanjiElement"("entryId", "orderNum"),
-  PRIMARY KEY ("entryId", "senseId", "kanji")
+  PRIMARY KEY ("entryId", "senseId", "kanjiOrderNum")
 ) WITHOUT ROWID;
 
 CREATE TABLE "JMdict_SenseRestrictedToReading" (
   "entryId" INTEGER NOT NULL,
   "senseId" INTEGER NOT NULL REFERENCES "JMdict_Sense"("senseId"),
-  "reading" TEXT NOT NULL,
+  "readingOrderNum" INTEGER NOT NULL CHECK ("readingOrderNum" > 0),
-  FOREIGN KEY ("entryId", "reading") REFERENCES "JMdict_ReadingElement"("entryId", "reading"),
+  FOREIGN KEY ("entryId", "readingOrderNum") REFERENCES "JMdict_ReadingElement"("entryId", "orderNum"),
-  PRIMARY KEY ("entryId", "senseId", "reading")
+  PRIMARY KEY ("entryId", "senseId", "readingOrderNum")
 ) WITHOUT ROWID;
 
 -- In order to add xrefs, you will need to have added the entry to xref to.
@@ -145,32 +147,23 @@ CREATE TABLE "JMdict_SenseRestrictedToReading" (
 
 CREATE TABLE "JMdict_SenseSeeAlso" (
   "senseId" INTEGER NOT NULL REFERENCES "JMdict_Sense"("senseId"),
-  "xrefEntryId" INTEGER NOT NULL,
+  "xrefEntryId" INTEGER NOT NULL REFERENCES "JMdict_Entry"("entryId"),
-  "seeAlsoReading" TEXT,
+  -- Sometimes the cross reference is to a specific sense
-  "seeAlsoKanji" TEXT,
+  "xrefSenseOrderNum" INTEGER,
-  "seeAlsoSense" INTEGER,
   -- For some entries, the cross reference is ambiguous. This means that while the ingestion
   -- has determined some xrefEntryId, it is not guaranteed to be the correct one.
   "ambiguous" BOOLEAN NOT NULL DEFAULT FALSE,
-  FOREIGN KEY ("xrefEntryId", "seeAlsoKanji") REFERENCES "JMdict_KanjiElement"("entryId", "reading"),
+  FOREIGN KEY ("xrefEntryId", "xrefSenseOrderNum") REFERENCES "JMdict_Sense"("entryId", "orderNum"),
-  FOREIGN KEY ("xrefEntryId", "seeAlsoReading") REFERENCES "JMdict_ReadingElement"("entryId", "reading"),
+  UNIQUE("senseId", "xrefEntryId", "xrefSenseOrderNum")
-  FOREIGN KEY ("xrefEntryId", "seeAlsoSense") REFERENCES "JMdict_Sense"("entryId", "orderNum"),
-  UNIQUE("senseId", "xrefEntryId", "seeAlsoReading", "seeAlsoKanji", "seeAlsoSense")
 );
 
 CREATE TABLE "JMdict_SenseAntonym" (
   "senseId" INTEGER NOT NULL REFERENCES "JMdict_Sense"("senseId"),
-  "xrefEntryId" INTEGER NOT NULL,
+  "xrefEntryId" INTEGER NOT NULL REFERENCES "JMdict_Entry"("entryId"),
-  "antonymReading" TEXT,
-  "antonymKanji" TEXT,
-  "antonymSense" INTEGER,
   -- For some entries, the cross reference is ambiguous. This means that while the ingestion
   -- has determined some xrefEntryId, it is not guaranteed to be the correct one.
   "ambiguous" BOOLEAN NOT NULL DEFAULT FALSE,
-  FOREIGN KEY ("xrefEntryId", "antonymKanji") REFERENCES "JMdict_KanjiElement"("entryId", "reading"),
+  UNIQUE("senseId", "xrefEntryId")
-  FOREIGN KEY ("xrefEntryId", "antonymReading") REFERENCES "JMdict_ReadingElement"("entryId", "reading"),
-  FOREIGN KEY ("xrefEntryId", "antonymSense") REFERENCES "JMdict_Sense"("entryId", "orderNum"),
-  UNIQUE("senseId", "xrefEntryId", "antonymReading", "antonymKanji", "antonymSense")
 );
 
 -- These cross references are going to be mostly accessed from a sense

nix/database_tool.nix

@@ -1,6 +1,7 @@
 {
   src,
   buildDartApplication,
+  sqlite,
 }:
 buildDartApplication {
   pname = "jadb-database-tool";
@@ -9,6 +10,9 @@ buildDartApplication {
 
   dartEntryPoints."bin/jadb" = "bin/jadb.dart";
 
+  # NOTE: here we are overriding the implicitly added runtimeDependency from the package fixup in pub2nix.
+  runtimeDependencies = [ sqlite ];
+
   # NOTE: the default dart hooks are using `dart compile`, which is not able to call the
   #       new dart build hooks required to use package:sqlite3 >= 3.0.0. So we override
   #       these phases to use `dart build` instead.