WIP

.gitea/workflows/build-and-test.yml

@@ -16,15 +16,14 @@ jobs:
       uses: https://github.com/cachix/install-nix-action@v31
       with:
         extra_nix_config: |
-          experimental-features = nix-command flakes
           show-trace = true
           max-jobs = auto
           trusted-users = root
           experimental-features = nix-command flakes
           build-users-group =
 
-    - name: Update database inputs
-      run: nix flake update datasources
+    # - name: Update database inputs
+    #   run: nix flake update datasources
 
     - name: Build database
       run: nix build .#database -L

benchmark/benchmark.dart

@@ -1,5 +1,7 @@
-import './search/word_search.dart';
+import './search/english_word_search.dart';
+import './search/japanese_word_search.dart';
 
 Future<void> main() async {
-  await WordSearchBenchmark.main();
+  await EnglishWordSearchBenchmark.main();
+  await JapaneseWordSearchBenchmark.main();
 }

benchmark/search/english_word_search.dart

@@ -4,23 +4,23 @@ import 'package:sqflite_common/sqlite_api.dart';
 
 import '../../test/search/setup_database_connection.dart';
 
-class WordSearchBenchmark extends AsyncBenchmarkBase {
+class EnglishWordSearchBenchmark extends AsyncBenchmarkBase {
   Database? connection;
 
   static final List<String> searchTerms = [
     'kana',
     'kanji',
-    'kawaii',
+    'cute',
     'sushi',
     'ramen',
   ];
 
-  WordSearchBenchmark() : super('WordSearchBenchmark');
+  EnglishWordSearchBenchmark() : super('EnglishWordSearchBenchmark');
 
   static Future<void> main() async {
-    print('Running WordSearchBenchmark...');
-    await WordSearchBenchmark().report();
-    print('Finished WordSearchBenchmark');
+    print('Running EnglishWordSearchBenchmark...');
+    await EnglishWordSearchBenchmark().report();
+    print('Finished EnglishWordSearchBenchmark');
   }
 
   @override
@@ -31,7 +31,11 @@ class WordSearchBenchmark extends AsyncBenchmarkBase {
   @override
   Future<void> run() async {
     for (final term in searchTerms) {
-      await connection!.jadbSearchWord(term);
+      final result = await connection!.jadbSearchWord(term);
+      assert(
+        result?.isNotEmpty ?? false,
+        'Expected search results for term "$term"',
+      );
     }
   }
 

benchmark/search/japanese_word_search.dart

@@ -0,0 +1,49 @@
+import 'package:benchmark_harness/benchmark_harness.dart';
+import 'package:jadb/search.dart';
+import 'package:sqflite_common/sqlite_api.dart';
+
+import '../../test/search/setup_database_connection.dart';
+
+class JapaneseWordSearchBenchmark extends AsyncBenchmarkBase {
+  Database? connection;
+
+  static final List<String> searchTerms = [
+    '仮名',
+    '漢字',
+    'かわいい',
+    'すし',
+    'ラメン',
+  ];
+
+  JapaneseWordSearchBenchmark() : super('JapaneseWordSearchBenchmark');
+
+  static Future<void> main() async {
+    print('Running JapaneseWordSearchBenchmark...');
+    await JapaneseWordSearchBenchmark().report();
+    print('Finished JapaneseWordSearchBenchmark');
+  }
+
+  @override
+  Future<void> setup() async {
+    connection = await setupDatabaseConnection();
+  }
+
+  @override
+  Future<void> run() async {
+    for (final term in searchTerms) {
+      final result = await connection!.jadbSearchWord(term);
+      assert(
+        result?.isNotEmpty ?? false,
+        'Expected search results for term "$term"',
+      );
+    }
+  }
+
+  @override
+  Future<void> teardown() async {
+    await connection?.close();
+  }
+
+  // @override
+  // Future<void> exercise() => run();
+}

docs/database.md

@@ -25,4 +25,5 @@ The `JMdict_EntryScore` table is used to store the score of each entry, which is
 
 The table is automatically generated from other tables via triggers, and should be considered as a materialized view.
 
-There is a score row for every single entry in both `JMdict_KanjiElement` and `JMdict_ReadingElement`, split by the `type` field.
+<s>There is a score row for every single entry in both `JMdict_KanjiElement` and `JMdict_ReadingElement`, split by the `type` field.</s>
+This is no longer true, we now only store the rows for which the score is not `0`. The `type` field is now also virtual, since the `elementId` fields for both kanji and readings are unique to each other.

flake.lock

@@ -7,11 +7,11 @@
         ]
       },
       "locked": {
-        "lastModified": 1775550160,
-        "narHash": "sha256-bgvKrMGUPaDY4EZv+82z1ccYoxwaergdVw/3PZhc2Fc=",
+        "lastModified": 1776081209,
+        "narHash": "sha256-zR1115tcOPnYLk6NznSf7YslyaJLc/MGayEHShitx18=",
         "ref": "refs/heads/main",
-        "rev": "f46229af3678124c5ea7c8dff3292747d0274f69",
-        "revCount": 8,
+        "rev": "7fe3552bb16e1d315c0b27b243e5eb53cd9e86fc",
+        "revCount": 13,
         "type": "git",
         "url": "https://git.pvv.ntnu.no/Mugiten/datasources.git"
       },
@@ -22,11 +22,11 @@
     },
     "nixpkgs": {
       "locked": {
-        "lastModified": 1775423009,
-        "narHash": "sha256-vPKLpjhIVWdDrfiUM8atW6YkIggCEKdSAlJPzzhkQlw=",
+        "lastModified": 1777954456,
+        "narHash": "sha256-hGdgeU2Nk87RAuZyYjyDjFL6LK7dAZN5RE9+hrDTkDU=",
         "owner": "NixOS",
         "repo": "nixpkgs",
-        "rev": "68d8aa3d661f0e6bd5862291b5bb263b2a6595c9",
+        "rev": "549bd84d6279f9852cae6225e372cc67fb91a4c1",
         "type": "github"
       },
       "original": {

flake.nix

@@ -116,6 +116,8 @@
         ln -s ${src} $out
       '';
 
+      fts5-icu-tokenizer = pkgs.callPackage ./nix/fts5-icu-tokenizer/package.nix { };
+
       inherit (datasources.packages.${system}) jmdict radkfile kanjidic2;
 
       database-tool = pkgs.callPackage ./nix/database_tool.nix {

lib/_data_ingestion/jmdict/objects.dart

@@ -106,9 +106,8 @@ class Glossary extends SQLWritable {
 
   @override
   Map<String, Object?> get sqlValue => {
-    'language': language,
+    // 'language': language,
     'phrase': phrase,
-    'type': type,
   };
 }
 

lib/_data_ingestion/jmdict/seed_data.dart

@@ -1,4 +1,5 @@
 import 'dart:collection';
+import 'dart:io';
 
 import 'package:collection/collection.dart';
 import 'package:jadb/_data_ingestion/jmdict/objects.dart';
@@ -26,15 +27,44 @@ ResolvedXref resolveXref(
   SplayTreeMap<String, Set<Entry>> entriesByReading,
   XRefParts xref,
 ) {
-  List<Entry> candidateEntries = switch ((xref.kanjiRef, xref.readingRef)) {
-    (null, null) => throw Exception(
-      'Xref $xref has no kanji or reading reference',
-    ),
-    (final String k, null) => entriesByKanji[k]!.toList(),
-    (null, final String r) => entriesByReading[r]!.toList(),
-    (final String k, final String r) =>
-      entriesByKanji[k]!.intersection(entriesByReading[r]!).toList(),
-  };
+  late List<Entry> candidateEntries;
+  switch ((xref.kanjiRef, xref.readingRef)) {
+    case (null, null):
+      throw Exception('Xref $xref has no kanji or reading reference');
+
+    case (final String k, null):
+      if (!entriesByKanji.containsKey(k)) {
+        throw Exception(
+          'Xref $xref has kanji reference "$k" but no entries found with that kanji',
+        );
+      }
+      candidateEntries = entriesByKanji[k]!.toList();
+      break;
+
+    case (null, final String r):
+      if (!entriesByReading.containsKey(r)) {
+        throw Exception(
+          'Xref $xref has reading reference "$r" but no entries found with that reading',
+        );
+      }
+      candidateEntries = entriesByReading[r]!.toList();
+      break;
+
+    case (final String k, final String r):
+      if (!entriesByKanji.containsKey(k)) {
+        throw Exception(
+          'Xref $xref has kanji reference "$k" but no entries found with that kanji',
+        );
+      }
+      if (!entriesByReading.containsKey(r)) {
+        throw Exception(
+          'Xref $xref has reading reference "$r" but no entries found with that reading',
+        );
+      }
+      candidateEntries = entriesByKanji[k]!
+          .intersection(entriesByReading[r]!)
+          .toList();
+  }
 
   // Filter out entries that don't have the number of senses specified in the xref
   if (xref.senseOrderNum != null) {
@@ -84,6 +114,21 @@ Future<void> seedJMDictData(List<Entry> entries, Database db) async {
   print('  [JMdict] Batch 1 - Kanji and readings');
   Batch b = db.batch();
 
+  if (Platform.environment['JMDICT_VERSION'] != null &&
+      Platform.environment['JMDICT_DATE'] != null &&
+      Platform.environment['JMDICT_HASH'] != null) {
+    b.insert(JMdictTableNames.version, {
+      'version': Platform.environment['JMDICT_VERSION']!,
+      'date': Platform.environment['JMDICT_DATE']!,
+      'hash': Platform.environment['JMDICT_HASH']!,
+    });
+  } else {
+    print(
+      'WARNING: JMDICT version information not found in environment variables. '
+      'This may cause issues with future updates.',
+    );
+  }
+
   for (final e in entries) {
     b.insert(JMdictTableNames.entry, e.sqlValue);
 
@@ -172,6 +217,14 @@ Future<void> seedJMDictData(List<Entry> entries, Database db) async {
           JMdictTableNames.senseGlossary,
           g.sqlValue..addAll({'senseId': s.senseId}),
         );
+
+        if (g.type != null) {
+          b.insert(JMdictTableNames.senseGlossaryType, {
+            'senseId': s.senseId,
+            'phrase': g.phrase,
+            'type': g.type!,
+          });
+        }
       }
     }
   }
@@ -219,8 +272,6 @@ Future<void> seedJMDictData(List<Entry> entries, Database db) async {
         b.insert(JMdictTableNames.senseSeeAlso, {
           'senseId': s.senseId,
           'xrefEntryId': resolvedEntry.entry.entryId,
-          'seeAlsoKanji': xref.kanjiRef,
-          'seeAlsoReading': xref.readingRef,
           'seeAlsoSense': xref.senseOrderNum != null
               ? xref.senseOrderNum! - 1
               : null,
@@ -248,8 +299,6 @@ Future<void> seedJMDictData(List<Entry> entries, Database db) async {
         b.insert(JMdictTableNames.senseAntonyms, {
           'senseId': s.senseId,
           'xrefEntryId': resolvedEntry.entry.entryId,
-          'antonymKanji': ant.kanjiRef,
-          'antonymReading': ant.readingRef,
           'antonymSense': ant.senseOrderNum != null
               ? ant.senseOrderNum! - 1
               : null,

lib/_data_ingestion/kanjidic/seed_data.dart

@@ -1,3 +1,5 @@
+import 'dart:io';
+
 import 'package:jadb/table_names/kanjidic.dart';
 import 'package:sqflite_common/sqlite_api.dart';
 
@@ -5,6 +7,22 @@ import 'objects.dart';
 
 Future<void> seedKANJIDICData(List<Character> characters, Database db) async {
   final b = db.batch();
+
+  if (Platform.environment['KANJIDIC_VERSION'] != null &&
+      Platform.environment['KANJIDIC_DATE'] != null &&
+      Platform.environment['KANJIDIC_HASH'] != null) {
+    b.insert(KANJIDICTableNames.version, {
+      'version': Platform.environment['KANJIDIC_VERSION']!,
+      'date': Platform.environment['KANJIDIC_DATE']!,
+      'hash': Platform.environment['KANJIDIC_HASH']!,
+    });
+  } else {
+    print(
+      'WARNING: KANJIDIC version information not found in environment variables. '
+      'This may cause issues with future updates.',
+    );
+  }
+
   for (final c in characters) {
     // if (c.dictionaryReferences.any((e) =>
     //     c.dictionaryReferences
@@ -30,10 +48,7 @@ Future<void> seedKANJIDICData(List<Character> characters, Database db) async {
     }
 
     if (c.jlpt != null) {
-      b.insert(KANJIDICTableNames.jlpt, {
-        'kanji': c.literal,
-        'jlpt': c.jlpt!,
-      });
+      b.insert(KANJIDICTableNames.jlpt, {'kanji': c.literal, 'jlpt': c.jlpt!});
     }
 
     for (final n in c.radicalName) {

lib/_data_ingestion/radkfile/seed_data.dart

@@ -1,9 +1,26 @@
+import 'dart:io';
+
 import 'package:jadb/table_names/radkfile.dart';
 import 'package:sqflite_common/sqlite_api.dart';
 
 Future<void> seedRADKFILEData(Iterable<String> blocks, Database db) async {
   final b = db.batch();
 
+  if (Platform.environment['RADKFILE_VERSION'] != null &&
+      Platform.environment['RADKFILE_DATE'] != null &&
+      Platform.environment['RADKFILE_HASH'] != null) {
+    b.insert(RADKFILETableNames.version, {
+      'version': Platform.environment['RADKFILE_VERSION']!,
+      'date': Platform.environment['RADKFILE_DATE']!,
+      'hash': Platform.environment['RADKFILE_HASH']!,
+    });
+  } else {
+    print(
+      'WARNING: RADKFILE version information not found in environment variables. '
+      'This may cause issues with future updates.',
+    );
+  }
+
   for (final block in blocks) {
     final String radical = block[1];
     final List<String> kanjiList =

lib/_data_ingestion/tanos-jlpt/seed_data.dart

@@ -1,3 +1,5 @@
+import 'dart:io';
+
 import 'package:jadb/table_names/tanos_jlpt.dart';
 import 'package:sqflite_common/sqlite_api.dart';
 
@@ -7,6 +9,21 @@ Future<void> seedTanosJLPTData(
 ) async {
   final Batch b = db.batch();
 
+  if (Platform.environment['TANOS_JLPT_VERSION'] != null &&
+      Platform.environment['TANOS_JLPT_DATE'] != null &&
+      Platform.environment['TANOS_JLPT_HASH'] != null) {
+    b.insert(TanosJLPTTableNames.version, {
+      'version': Platform.environment['TANOS_JLPT_VERSION']!,
+      'date': Platform.environment['TANOS_JLPT_DATE']!,
+      'hash': Platform.environment['TANOS_JLPT_HASH']!,
+    });
+  } else {
+    print(
+      'WARNING: Tanos JLPT version information not found in environment variables. '
+      'This may cause issues with future updates.',
+    );
+  }
+
   for (final jlptLevel in resolvedEntries.entries) {
     final level = jlptLevel.key;
     final entryIds = jlptLevel.value;

lib/search.dart

@@ -18,12 +18,12 @@ extension JaDBConnection on DatabaseExecutor {
       searchKanjiWithDbConnection(this, kanji);
 
   /// Search for a kanji in the database.
-  Future<Map<String, KanjiSearchResult>> jadbGetManyKanji(Set<String> kanji) =>
+  Future<Map<String, KanjiSearchResult>> jadbGetManyKanji(Iterable<String> kanji) =>
       searchManyKanjiWithDbConnection(this, kanji);
 
   /// Filter a list of characters, and return the ones that are listed in the kanji dictionary.
   Future<List<String>> filterKanji(
-    List<String> kanji, {
+    Iterable<String> kanji, {
     bool deduplicate = false,
   }) => filterKanjiWithDbConnection(this, kanji, deduplicate);
 

lib/search/filter_kanji.dart

@@ -6,7 +6,7 @@ import 'package:sqflite_common/sqflite.dart';
 /// If [deduplicate] is true, the returned list will deduplicate the input kanji list before returning the filtered results.
 Future<List<String>> filterKanjiWithDbConnection(
   DatabaseExecutor connection,
-  List<String> kanji,
+  Iterable<String> kanji,
   bool deduplicate,
 ) async {
   final Set<String> filteredKanji = await connection
@@ -14,7 +14,7 @@ Future<List<String>> filterKanjiWithDbConnection(
       SELECT "literal"
       FROM "${KANJIDICTableNames.character}"
       WHERE "literal" IN (${kanji.map((_) => '?').join(',')})
-    ''', kanji)
+    ''', kanji.toList())
       .then((value) => value.map((e) => e['literal'] as String).toSet());
 
   if (deduplicate) {

lib/search/kanji_search.dart

@@ -274,7 +274,7 @@ Future<KanjiSearchResult?> searchKanjiWithDbConnection(
 /// Searches for multiple kanji at once, returning a map of kanji to their search results.
 Future<Map<String, KanjiSearchResult>> searchManyKanjiWithDbConnection(
   DatabaseExecutor connection,
-  Set<String> kanji,
+  Iterable<String> kanji,
 ) async {
   if (kanji.isEmpty) {
     return {};

lib/search/word_search/entry_id_query.dart

@@ -68,25 +68,25 @@ String _filterFTSSensitiveCharacters(String word) {
               "$tableName"."entryId",
               100
               + (("${tableName}FTS"."reading" = ?) * 10000)
-              + "JMdict_EntryScore"."score"
+              + (("$tableName"."orderNum" = 0) * 20)
+              + COALESCE("JMdict_EntryScore"."score", 0)
               AS "score"
             FROM "${tableName}FTS"
             JOIN "$tableName" USING ("elementId")
-            JOIN "JMdict_EntryScore" USING ("elementId")
+            LEFT JOIN "JMdict_EntryScore" USING ("elementId")
             WHERE "${tableName}FTS"."reading" MATCH ? || '*'
-              AND "JMdict_EntryScore"."elementId" ${tableName == JMdictTableNames.kanjiElement ? '<' : '>='} 1000000000
           ),
           non_fts_results AS (
             SELECT DISTINCT
               "$tableName"."entryId",
               50
-              + "JMdict_EntryScore"."score"
+              + (("$tableName"."orderNum" = 0) * 20)
+              + COALESCE("JMdict_EntryScore"."score", 0)
               AS "score"
             FROM "$tableName"
-            JOIN "JMdict_EntryScore" USING ("elementId")
+            LEFT JOIN "JMdict_EntryScore" USING ("elementId")
             WHERE "reading" LIKE '%' || ? || '%'
               AND "$tableName"."entryId" NOT IN (SELECT "entryId" FROM "fts_results")
-              AND "JMdict_EntryScore"."elementId" ${tableName == JMdictTableNames.kanjiElement ? '<' : '>='} 1000000000
           )
 
         SELECT ${countOnly ? 'COUNT(DISTINCT "entryId") AS count' : '"entryId", MAX("score") AS "score"'}
@@ -198,16 +198,16 @@ Future<List<ScoredEntryId>> _queryEnglish(
     '''
     SELECT
       "${JMdictTableNames.sense}"."entryId",
-      MAX("JMdict_EntryScore"."score")
+      COALESCE(MAX("JMdict_EntryScore"."score"), 0)
       + (("${JMdictTableNames.senseGlossary}"."phrase" = ?1 AND "${JMdictTableNames.sense}"."orderNum" = 0) * 50)
       + (("${JMdictTableNames.senseGlossary}"."phrase" = ?1 AND "${JMdictTableNames.sense}"."orderNum" = 1) * 30)
-      + (("${JMdictTableNames.senseGlossary}"."phrase" = ?1) * 20)
+      + (("${JMdictTableNames.senseGlossary}"."phrase" = ?1 AND "${JMdictTableNames.sense}"."orderNum" > 1) * 20)
       as "score"
     FROM "${JMdictTableNames.senseGlossary}"
     JOIN "${JMdictTableNames.sense}" USING ("senseId")
-    JOIN "JMdict_EntryScore" USING ("entryId")
+    LEFT JOIN "JMdict_EntryScore" USING ("entryId")
     WHERE "${JMdictTableNames.senseGlossary}"."phrase" LIKE ?2
-    GROUP BY "JMdict_EntryScore"."entryId"
+    GROUP BY "${JMdictTableNames.sense}"."entryId"
     ORDER BY
       "score" DESC,
       "${JMdictTableNames.sense}"."entryId" ASC
@@ -215,7 +215,7 @@ Future<List<ScoredEntryId>> _queryEnglish(
     ${offset != null ? 'OFFSET ?4' : ''}
   '''
         .trim(),
-    [word, '%${word.replaceAll('%', '')}%', if (pageSize != null) pageSize, if (offset != null) offset],
+    [word, '%${word.replaceAll('%', '')}%', ?pageSize, ?offset],
   );
 
   return result

lib/table_names/jmdict.dart

@@ -11,6 +11,7 @@ abstract class JMdictTableNames {
   static const String senseDialect = 'JMdict_SenseDialect';
   static const String senseField = 'JMdict_SenseField';
   static const String senseGlossary = 'JMdict_SenseGlossary';
+  static const String senseGlossaryType = 'JMdict_SenseGlossaryType';
   static const String senseInfo = 'JMdict_SenseInfo';
   static const String senseMisc = 'JMdict_SenseMisc';
   static const String sensePOS = 'JMdict_SensePOS';
@@ -33,6 +34,7 @@ abstract class JMdictTableNames {
     senseDialect,
     senseField,
     senseGlossary,
+    senseGlossaryType,
     senseInfo,
     senseMisc,
     sensePOS,

migrations/0001_JMDict.sql

@@ -139,8 +139,6 @@ CREATE TABLE "JMdict_SenseRestrictedToReading" (
 CREATE TABLE "JMdict_SenseSeeAlso" (
   "senseId" INTEGER NOT NULL REFERENCES "JMdict_Sense"("senseId"),
   "xrefEntryId" INTEGER NOT NULL,
-  "seeAlsoReading" TEXT,
-  "seeAlsoKanji" TEXT,
   "seeAlsoSense" INTEGER,
   -- For some entries, the cross reference is ambiguous. This means that while the ingestion
   -- has determined some xrefEntryId, it is not guaranteed to be the correct one.
@@ -153,18 +151,14 @@ CREATE TABLE "JMdict_SenseSeeAlso" (
     END
   ) VIRTUAL,
 
-  FOREIGN KEY ("xrefEntryId", "seeAlsoKanji") REFERENCES "JMdict_KanjiElement"("entryId", "reading"),
-  FOREIGN KEY ("xrefEntryId", "seeAlsoReading") REFERENCES "JMdict_ReadingElement"("entryId", "reading"),
   FOREIGN KEY ("seeAlsoSenseKey") REFERENCES "JMdict_Sense"("senseId"),
 
-  UNIQUE("senseId", "xrefEntryId", "seeAlsoReading", "seeAlsoKanji", "seeAlsoSense")
+  PRIMARY KEY ("senseId", "xrefEntryId", "seeAlsoSense")
 );
 
 CREATE TABLE "JMdict_SenseAntonym" (
   "senseId" INTEGER NOT NULL REFERENCES "JMdict_Sense"("senseId"),
   "xrefEntryId" INTEGER NOT NULL,
-  "antonymReading" TEXT,
-  "antonymKanji" TEXT,
   "antonymSense" INTEGER,
   -- For some entries, the cross reference is ambiguous. This means that while the ingestion
   -- has determined some xrefEntryId, it is not guaranteed to be the correct one.
@@ -177,11 +171,9 @@ CREATE TABLE "JMdict_SenseAntonym" (
     END
   ) VIRTUAL,
 
-  FOREIGN KEY ("xrefEntryId", "antonymKanji") REFERENCES "JMdict_KanjiElement"("entryId", "reading"),
-  FOREIGN KEY ("xrefEntryId", "antonymReading") REFERENCES "JMdict_ReadingElement"("entryId", "reading"),
   FOREIGN KEY ("antonymSenseKey") REFERENCES "JMdict_Sense"("senseId"),
 
-  UNIQUE("senseId", "xrefEntryId", "antonymReading", "antonymKanji", "antonymSense")
+  PRIMARY KEY ("senseId", "xrefEntryId", "antonymSense")
 );
 
 -- These cross references are going to be mostly accessed from a sense
@@ -232,13 +224,21 @@ CREATE TABLE "JMdict_SenseDialect" (
 CREATE TABLE "JMdict_SenseGlossary" (
   "senseId" INTEGER NOT NULL REFERENCES "JMdict_Sense"("senseId"),
   "phrase" TEXT NOT NULL,
-  "language" CHAR(3) NOT NULL DEFAULT "eng",
-  "type" TEXT,
-  PRIMARY KEY ("senseId", "language", "phrase")
+  -- "language" CHAR(3) NOT NULL DEFAULT "eng",
+  -- PRIMARY KEY ("senseId", "language", "phrase")
+  PRIMARY KEY ("senseId", "phrase")
 ) WITHOUT ROWID;
 
 CREATE INDEX "JMdict_SenseGlossary_byPhrase" ON JMdict_SenseGlossary("phrase");
 
+CREATE TABLE "JMdict_SenseGlossaryType" (
+  "senseId" INTEGER NOT NULL REFERENCES "JMdict_Sense"("senseId"),
+  "phrase" TEXT NOT NULL,
+  "type" TEXT NOT NULL,
+  PRIMARY KEY ("senseId", "phrase", "type"),
+  FOREIGN KEY ("senseId", "phrase") REFERENCES "JMdict_SenseGlossary"("senseId", "phrase")
+) WITHOUT ROWID;
+
 CREATE TABLE "JMdict_SenseInfo" (
   "senseId" INTEGER NOT NULL REFERENCES "JMdict_Sense"("senseId"),
   "info" TEXT NOT NULL,

migrations/0005_JMDict_search_index_tables.sql

@@ -1,6 +1,6 @@
 CREATE TABLE "JMdict_EntryScore" (
   "elementId" INTEGER PRIMARY KEY,
-  "score" INTEGER NOT NULL DEFAULT 0,
+  "score" INTEGER NOT NULL,
   "common" BOOLEAN NOT NULL DEFAULT FALSE,
 
   "entryId" INTEGER NOT NULL GENERATED ALWAYS AS (("elementId" / 100) % 10000000) STORED,
@@ -15,8 +15,7 @@ CREATE TABLE "JMdict_EntryScore" (
 CREATE INDEX "JMdict_EntryScore_byElementId_byScore" ON "JMdict_EntryScore"("elementId", "score");
 CREATE INDEX "JMdict_EntryScore_byScore" ON "JMdict_EntryScore"("score");
 
-CREATE INDEX "JMdict_EntryScore_byElementId_byCommon" ON "JMdict_EntryScore"("elementId", "common");
-CREATE INDEX "JMdict_EntryScore_byCommon" ON "JMdict_EntryScore"("common");
+CREATE INDEX "JMdict_EntryScore_byCommon" ON "JMdict_EntryScore"("common") WHERE "common" = TRUE;
 
 -- NOTE: these views are deduplicated in order not to perform an unnecessary
 --       UNION on every trigger
@@ -46,7 +45,7 @@ SELECT
   + (("spec" IS 2) * 5)
   + (("gai" IS 1) * 10)
   + (("gai" IS 2) * 5)
-  + (("orderNum" IS 0) * 20)
+  -- + (("orderNum" IS 0) * 20)
   - (substr(COALESCE("JMdict_JLPTTag"."jlptLevel", 'N0'), 2) * -5)
   AS "score"
 FROM "JMdict_ReadingElement"
@@ -77,7 +76,7 @@ SELECT
   + (("spec" IS 2) * 5)
   + (("gai" IS 1) * 10)
   + (("gai" IS 2) * 5)
-  + (("orderNum" IS 0) * 20)
+  -- + (("orderNum" IS 0) * 20)
   - (substr(COALESCE("JMdict_JLPTTag"."jlptLevel", 'N0'), 2) * -5)
   AS "score"
 FROM "JMdict_KanjiElement"
@@ -103,7 +102,8 @@ BEGIN
   )
   SELECT "elementId", "score", "common"
   FROM "JMdict_EntryScoreView_Reading"
-  WHERE "elementId" = NEW."elementId";
+  WHERE "elementId" = NEW."elementId"
+    AND "score" > 0;
 END;
 
 CREATE TRIGGER "JMdict_EntryScore_Update_JMdict_ReadingElement"
@@ -116,6 +116,10 @@ BEGIN
     "common" = "JMdict_EntryScoreView_Reading"."common"
   FROM "JMdict_EntryScoreView_Reading"
   WHERE "elementId" = NEW."elementId";
+
+  DELETE FROM "JMdict_EntryScore"
+  WHERE "elementId" = NEW."elementId"
+    AND "score" <= 0;
 END;
 
 CREATE TRIGGER "JMdict_EntryScore_Delete_JMdict_ReadingElement"
@@ -137,7 +141,8 @@ BEGIN
   )
   SELECT "elementId", "score", "common"
   FROM "JMdict_EntryScoreView_Kanji"
-  WHERE "elementId" = NEW."elementId";
+  WHERE "elementId" = NEW."elementId"
+    AND "score" > 0;
 END;
 
 CREATE TRIGGER "JMdict_EntryScore_Update_JMdict_KanjiElement"
@@ -150,6 +155,10 @@ BEGIN
     "common" = "JMdict_EntryScoreView_Kanji"."common"
   FROM "JMdict_EntryScoreView_Kanji"
   WHERE "elementId" = NEW."elementId";
+
+  DELETE FROM "JMdict_EntryScore"
+  WHERE "elementId" = NEW."elementId"
+    AND "score" <= 0;
 END;
 
 CREATE TRIGGER "JMdict_EntryScore_Delete_JMdict_KanjiElement"
@@ -199,4 +208,8 @@ BEGIN
   WHERE "JMdict_EntryScoreView"."entryId" = OLD."entryId"
     AND "JMdict_EntryScore"."entryId" = OLD."entryId"
     AND "JMdict_EntryScoreView"."elementId" = "JMdict_EntryScore"."elementId";
+
+  DELETE FROM "JMdict_EntryScore"
+  WHERE "elementId" = OLD."elementId"
+    AND "score" <= 0;
 END;

migrations/0006_RADKFILE.sql

@@ -18,4 +18,3 @@ CREATE TABLE "RADKFILE" (
 ) WITHOUT ROWID;
 
 CREATE INDEX "RADK" ON "RADKFILE"("radical");
-CREATE INDEX "KRAD" ON "RADKFILE"("kanji");

migrations/0008_XREF__KANJIDIC_Radical___RADKFILE.sql

@@ -4,7 +4,6 @@ CREATE TABLE "XREF__KANJIDIC_Radical__RADKFILE"(
   PRIMARY KEY ("radicalId", "radicalSymbol")
 ) WITHOUT ROWID;
 
-CREATE INDEX "XREF__KANJIDIC_Radical__RADKFILE__byRadicalId" ON "XREF__KANJIDIC_Radical__RADKFILE"("radicalId");
 CREATE INDEX "XREF__KANJIDIC_Radical__RADKFILE__byRadicalSymbol" ON "XREF__KANJIDIC_Radical__RADKFILE"("radicalSymbol");
 
 /* Source: https://ctext.org/kangxi-zidian */

migrations/0009_XREF__JMdict_KanjiElement__KANJIDIC_Character.sql

@@ -1,10 +1,10 @@
-CREATE TABLE "XREF__JMdict_KanjiElement__KANJIDIC_Character"(
-  "entryId" INTEGER NOT NULL,
-  "reading" TEXT NOT NULL,
-  "kanji" CHAR(1) NOT NULL REFERENCES "KANJIDIC_Character"("literal"),
-  PRIMARY KEY ("entryId", "reading", "kanji"),
-  FOREIGN KEY ("entryId", "reading") REFERENCES "JMdict_KanjiElement"("entryId", "reading")
-) WITHOUT ROWID;
+-- CREATE TABLE "XREF__JMdict_KanjiElement__KANJIDIC_Character"(
+--   "entryId" INTEGER NOT NULL,
+--   "reading" TEXT NOT NULL,
+--   "kanji" CHAR(1) NOT NULL REFERENCES "KANJIDIC_Character"("literal"),
+--   PRIMARY KEY ("entryId", "reading", "kanji"),
+--   FOREIGN KEY ("entryId", "reading") REFERENCES "JMdict_KanjiElement"("entryId", "reading")
+-- ) WITHOUT ROWID;
 
-CREATE INDEX "XREF__JMdict_KanjiElement__KANJIDIC_Character__byEntryId_byReading" ON "XREF__JMdict_KanjiElement__KANJIDIC_Character"("entryId", "reading");
-CREATE INDEX "XREF__JMdict_KanjiElement__KANJIDIC_Character__byKanji" ON "XREF__JMdict_KanjiElement__KANJIDIC_Character"("kanji");
+-- CREATE INDEX "XREF__JMdict_KanjiElement__KANJIDIC_Character__byEntryId_byReading" ON "XREF__JMdict_KanjiElement__KANJIDIC_Character"("entryId", "reading");
+-- CREATE INDEX "XREF__JMdict_KanjiElement__KANJIDIC_Character__byKanji" ON "XREF__JMdict_KanjiElement__KANJIDIC_Character"("kanji");

migrations/0010_Views.sql

@@ -75,13 +75,7 @@ SELECT DISTINCT "radical" FROM "RADKFILE";
 CREATE VIEW "JMdict_CombinedEntryScore"
 AS
 SELECT
-  CASE
-    WHEN "JMdict_EntryScore"."type" = 'k'
-      THEN (SELECT entryId FROM "JMdict_KanjiElement" WHERE "elementId" = "JMdict_EntryScore"."elementId")
-    WHEN "JMdict_EntryScore"."type" = 'r'
-      THEN (SELECT entryId FROM "JMdict_ReadingElement" WHERE "elementId" = "JMdict_EntryScore"."elementId")
-  END AS "entryId",
   MAX("JMdict_EntryScore"."score") AS "score",
   MAX("JMdict_EntryScore"."common") AS "common"
 FROM "JMdict_EntryScore"
-GROUP BY "entryId";
+GROUP BY "JMdict_EntryScore"."entryId";

nix/database.nix

@@ -19,13 +19,31 @@ stdenvNoCC.mkDerivation {
     sqlite
   ];
 
+  env = {
+    JMDICT_VERSION = jmdict.version;
+    JMDICT_DATE = jmdict.date;
+    JMDICT_HASH = jmdict.hash;
+
+    KANJIDIC_VERSION = kanjidic2.version;
+    KANJIDIC_DATE = kanjidic2.date;
+    KANJIDIC_HASH = kanjidic2.hash;
+
+    RADKFILE_VERSION = radkfile.version;
+    RADKFILE_DATE = radkfile.date;
+    RADKFILE_HASH = radkfile.hash;
+
+    TANOS_JLPT_VERSION = tanos-jlpt.version;
+    TANOS_JLPT_DATE = tanos-jlpt.date;
+    TANOS_JLPT_HASH = tanos-jlpt.hash;
+  };
+
   buildPhase = ''
     runHook preBuild
 
     mkdir -p data
     ln -s '${jmdict}'/* data/
-    ln -s '${radkfile}'/* data/
     ln -s '${kanjidic2}'/* data/
+    ln -s '${radkfile}'/* data/
     ln -s '${tanos-jlpt}' data/tanos-jlpt
 
     for migration in migrations/*.sql; do

nix/fts5-icu-tokenizer/0001-provide-sqlite-externally.patch

@@ -0,0 +1,20 @@
+diff --git i/CMakeLists.txt w/CMakeLists.txt
+index 9d99543..11ce4a4 100644
+--- i/CMakeLists.txt
++++ w/CMakeLists.txt
+@@ -131,6 +131,15 @@ if(NOT SQLite3_FOUND)
+   endif()
+ endif()
+ 
++if(SQLite3_FOUND AND NOT TARGET SQLite3::SQLite3)
++  add_library(SQLite3::SQLite3 UNKNOWN IMPORTED)
++
++  set_target_properties(SQLite3::SQLite3 PROPERTIES
++    IMPORTED_LOCATION "${SQLite3_LIBRARIES}"
++    INTERFACE_INCLUDE_DIRECTORIES "${SQLite3_INCLUDE_DIRS}"
++  )
++endif()
++
+ # --- Configure the Library ---
+ 
+ # Select source files based on API version

nix/fts5-icu-tokenizer/package.nix

@@ -0,0 +1,40 @@
+{
+  lib,
+  stdenv,
+  fetchFromSourcehut,
+  cmake,
+  pkg-config,
+  icu,
+  sqlite,
+}:
+
+stdenv.mkDerivation (finalAttrs: {
+  pname = "fts5-icu-tokenizer";
+  version = "5.5";
+  src = fetchFromSourcehut {
+    vc = "hg";
+    owner = "~cwt";
+    repo = "fts5-icu-tokenizer";
+    rev = "v${finalAttrs.version}";
+    hash = "sha256-7Klsu9d1sY+W0buo6kwYdCyDA/u2dBTgu6WuttomTBo=";
+  };
+
+  patches = [
+    ./0001-provide-sqlite-externally.patch
+  ];
+
+  nativeBuildInputs = [
+    cmake
+    pkg-config
+  ];
+
+  buildInputs = [
+    icu
+    sqlite
+  ];
+
+  cmakeFlags = [
+    (lib.cmakeFeature "LOCALE" "ja")
+    (lib.cmakeFeature "API_VERSION" "v2")
+  ];
+})

test/search/radical_search_test.dart

@@ -0,0 +1,79 @@
+import 'package:collection/collection.dart';
+import 'package:jadb/const_data/radicals.dart';
+import 'package:jadb/search.dart';
+import 'package:jadb/table_names/radkfile.dart';
+import 'package:test/test.dart';
+
+import 'setup_database_connection.dart';
+
+void main() {
+  test(
+    'All constant radicals should exist in the database',
+    () async {
+      final connection = await setupDatabaseConnection();
+      final allRadicalsInDb = await connection.query(
+        RADKFILETableNames.radkfile,
+        columns: ['radical'],
+        distinct: true,
+      );
+
+      final radicalsInDb = allRadicalsInDb
+          .map((e) => e['radical'] as String)
+          .toSet();
+
+      final missingRadicals = radicals.values.flattenedToSet.difference(
+        radicalsInDb,
+      );
+
+      expect(
+        missingRadicals,
+        isEmpty,
+        reason: 'Missing radicals in database: $missingRadicals',
+      );
+    },
+    skip: 'Test is valid, code is broken, fix me',
+  );
+
+  test(
+    'All radicals in database should be in the constant radical list',
+    () async {
+      final connection = await setupDatabaseConnection();
+      final allRadicalsInDb = await connection.query(
+        RADKFILETableNames.radkfile,
+        columns: ['radical'],
+        distinct: true,
+      );
+
+      final radicalsInDb = allRadicalsInDb
+          .map((e) => e['radical'] as String)
+          .toSet();
+
+      final extraRadicals = radicalsInDb.difference(
+        radicals.values.flattenedToSet,
+      );
+
+      expect(
+        extraRadicals,
+        isEmpty,
+        reason:
+            'Extra radicals in database missing in the constant list: $extraRadicals',
+      );
+    },
+    skip: 'Test is valid, code is broken, fix me',
+  );
+
+  group(
+    'All radicals should return results',
+    () {
+      for (final radical in radicals.values.flattened) {
+        test(' - $radical', () async {
+          final connection = await setupDatabaseConnection();
+          final result = await connection.jadbSearchKanjiByRadicals([radical]);
+          expect(result, isNotEmpty);
+        });
+      }
+    },
+    skip:
+        'These will be automatically fixed once the other radical tests are passing',
+  );
+}

test/util/romaji_transliteration_test.dart

@@ -8,7 +8,7 @@ void main() {
       expect(result, 'かたまり');
     });
 
-    test('Basic test with diacritics', () {
+    test('Basic test with dakuten', () {
       final result = transliterateLatinToHiragana('gadamari');
       expect(result, 'がだまり');
     });
@@ -54,7 +54,7 @@ void main() {
 
     test('Basic test', expectSpans('katamari', ['か', 'た', 'ま', 'り']));
     test(
-      'Basic test with diacritics',
+      'Basic test with dakuten',
       expectSpans('gadamari', ['が', 'だ', 'ま', 'り']),
     );
     test('wi and we', expectSpans('wiwe', ['うぃ', 'うぇ']));
@@ -72,7 +72,7 @@ void main() {
       expect(result, 'katamari');
     });
 
-    test('Basic test with diacritics', () {
+    test('Basic test with dakuten', () {
       final result = transliterateHiraganaToLatin('がだまり');
       expect(result, 'gadamari');
     });
@@ -91,6 +91,21 @@ void main() {
       final result = transliterateHiraganaToLatin('かっぱ');
       expect(result, 'kappa');
     });
+
+    test('Iteration mark', () {
+      final result = transliterateHiraganaToLatin('さゝき');
+      expect(result, 'sasaki');
+    }, skip: 'Not yet implemented');
+
+    test('Iteration mark with dakuten', () {
+      final result = transliterateHiraganaToLatin('あひゞき');
+      expect(result, 'ahibiki');
+    }, skip: 'Not yet implemented');
+
+    test('Yori', () {
+      final result = transliterateHiraganaToLatin('ゟ');
+      expect(result, 'yori');
+    }, skip: 'Not yet implemented');
   });
 
   group('Hiragana -> Romaji Spans', () {
@@ -110,7 +125,7 @@ void main() {
 
     test('Basic test', expectSpans('かたまり', ['ka', 'ta', 'ma', 'ri']));
     test(
-      'Basic test with diacritics',
+      'Basic test with dakuten',
       expectSpans('がだまり', ['ga', 'da', 'ma', 'ri']),
     );
     test('wi and we', expectSpans('うぃうぇ', ['whi', 'whe']));
@@ -118,5 +133,17 @@ void main() {
 
     // TODO: fix the implementation
     // test('Double consonant', expectSpans('かっぱ', ['ka', 'ppa']));
+
+    test(
+      'Iteration mark',
+      expectSpans('さゝき', ['sa', 'sa', 'ki']),
+      skip: 'Not yet implemented',
+    );
+    test(
+      'Iteration mark with dakuten',
+      expectSpans('あひゞき', ['a', 'hi', 'bi', 'ki']),
+      skip: 'Not yet implemented',
+    );
+    test('Yori', expectSpans('ゟ', ['yori']), skip: 'Not yet implemented');
   });
 }