WIP

.gitea/workflows/build-and-test.yml

@@ -16,14 +16,15 @@ jobs:
       uses: https://github.com/cachix/install-nix-action@v31
       with:
         extra_nix_config: |
+          experimental-features = nix-command flakes
           show-trace = true
           max-jobs = auto
           trusted-users = root
           experimental-features = nix-command flakes
           build-users-group =
 
-    # - name: Update database inputs
-    #   run: nix flake update datasources
+    - name: Update database inputs
+      run: nix flake update datasources
 
     - name: Build database
       run: nix build .#database -L

flake.lock

@@ -22,11 +22,11 @@
     },
     "nixpkgs": {
       "locked": {
-        "lastModified": 1777954456,
-        "narHash": "sha256-hGdgeU2Nk87RAuZyYjyDjFL6LK7dAZN5RE9+hrDTkDU=",
+        "lastModified": 1775423009,
+        "narHash": "sha256-vPKLpjhIVWdDrfiUM8atW6YkIggCEKdSAlJPzzhkQlw=",
         "owner": "NixOS",
         "repo": "nixpkgs",
-        "rev": "549bd84d6279f9852cae6225e372cc67fb91a4c1",
+        "rev": "68d8aa3d661f0e6bd5862291b5bb263b2a6595c9",
         "type": "github"
       },
       "original": {

lib/_data_ingestion/jmdict/seed_data.dart

@@ -27,44 +27,15 @@ ResolvedXref resolveXref(
   SplayTreeMap<String, Set<Entry>> entriesByReading,
   XRefParts xref,
 ) {
-  late List<Entry> candidateEntries;
-  switch ((xref.kanjiRef, xref.readingRef)) {
-    case (null, null):
-      throw Exception('Xref $xref has no kanji or reading reference');
-
-    case (final String k, null):
-      if (!entriesByKanji.containsKey(k)) {
-        throw Exception(
-          'Xref $xref has kanji reference "$k" but no entries found with that kanji',
-        );
-      }
-      candidateEntries = entriesByKanji[k]!.toList();
-      break;
-
-    case (null, final String r):
-      if (!entriesByReading.containsKey(r)) {
-        throw Exception(
-          'Xref $xref has reading reference "$r" but no entries found with that reading',
-        );
-      }
-      candidateEntries = entriesByReading[r]!.toList();
-      break;
-
-    case (final String k, final String r):
-      if (!entriesByKanji.containsKey(k)) {
-        throw Exception(
-          'Xref $xref has kanji reference "$k" but no entries found with that kanji',
-        );
-      }
-      if (!entriesByReading.containsKey(r)) {
-        throw Exception(
-          'Xref $xref has reading reference "$r" but no entries found with that reading',
-        );
-      }
-      candidateEntries = entriesByKanji[k]!
-          .intersection(entriesByReading[r]!)
-          .toList();
-  }
+  List<Entry> candidateEntries = switch ((xref.kanjiRef, xref.readingRef)) {
+    (null, null) => throw Exception(
+      'Xref $xref has no kanji or reading reference',
+    ),
+    (final String k, null) => entriesByKanji[k]!.toList(),
+    (null, final String r) => entriesByReading[r]!.toList(),
+    (final String k, final String r) =>
+      entriesByKanji[k]!.intersection(entriesByReading[r]!).toList(),
+  };
 
   // Filter out entries that don't have the number of senses specified in the xref
   if (xref.senseOrderNum != null) {

lib/search.dart

@@ -18,12 +18,12 @@ extension JaDBConnection on DatabaseExecutor {
       searchKanjiWithDbConnection(this, kanji);
 
   /// Search for a kanji in the database.
-  Future<Map<String, KanjiSearchResult>> jadbGetManyKanji(Iterable<String> kanji) =>
+  Future<Map<String, KanjiSearchResult>> jadbGetManyKanji(Set<String> kanji) =>
       searchManyKanjiWithDbConnection(this, kanji);
 
   /// Filter a list of characters, and return the ones that are listed in the kanji dictionary.
   Future<List<String>> filterKanji(
-    Iterable<String> kanji, {
+    List<String> kanji, {
     bool deduplicate = false,
   }) => filterKanjiWithDbConnection(this, kanji, deduplicate);
 

lib/search/filter_kanji.dart

@@ -6,7 +6,7 @@ import 'package:sqflite_common/sqflite.dart';
 /// If [deduplicate] is true, the returned list will deduplicate the input kanji list before returning the filtered results.
 Future<List<String>> filterKanjiWithDbConnection(
   DatabaseExecutor connection,
-  Iterable<String> kanji,
+  List<String> kanji,
   bool deduplicate,
 ) async {
   final Set<String> filteredKanji = await connection
@@ -14,7 +14,7 @@ Future<List<String>> filterKanjiWithDbConnection(
       SELECT "literal"
       FROM "${KANJIDICTableNames.character}"
       WHERE "literal" IN (${kanji.map((_) => '?').join(',')})
-    ''', kanji.toList())
+    ''', kanji)
       .then((value) => value.map((e) => e['literal'] as String).toSet());
 
   if (deduplicate) {

lib/search/kanji_search.dart

@@ -274,7 +274,7 @@ Future<KanjiSearchResult?> searchKanjiWithDbConnection(
 /// Searches for multiple kanji at once, returning a map of kanji to their search results.
 Future<Map<String, KanjiSearchResult>> searchManyKanjiWithDbConnection(
   DatabaseExecutor connection,
-  Iterable<String> kanji,
+  Set<String> kanji,
 ) async {
   if (kanji.isEmpty) {
     return {};

migrations/0001_JMDict.sql

@@ -229,7 +229,7 @@ CREATE TABLE "JMdict_SenseGlossary" (
   PRIMARY KEY ("senseId", "phrase")
 ) WITHOUT ROWID;
 
-CREATE INDEX "JMdict_SenseGlossary_byPhrase" ON JMdict_SenseGlossary("phrase");
+-- CREATE INDEX "JMdict_SenseGlossary_byPhrase" ON JMdict_SenseGlossary("phrase");
 
 CREATE TABLE "JMdict_SenseGlossaryType" (
   "senseId" INTEGER NOT NULL REFERENCES "JMdict_Sense"("senseId"),

migrations/0005_JMDict_search_index_tables.sql

@@ -1,6 +1,6 @@
 CREATE TABLE "JMdict_EntryScore" (
   "elementId" INTEGER PRIMARY KEY,
-  "score" INTEGER NOT NULL,
+  "score" INTEGER NOT NULL DEFAULT 0,
   "common" BOOLEAN NOT NULL DEFAULT FALSE,
 
   "entryId" INTEGER NOT NULL GENERATED ALWAYS AS (("elementId" / 100) % 10000000) STORED,
@@ -15,7 +15,8 @@ CREATE TABLE "JMdict_EntryScore" (
 CREATE INDEX "JMdict_EntryScore_byElementId_byScore" ON "JMdict_EntryScore"("elementId", "score");
 CREATE INDEX "JMdict_EntryScore_byScore" ON "JMdict_EntryScore"("score");
 
-CREATE INDEX "JMdict_EntryScore_byCommon" ON "JMdict_EntryScore"("common") WHERE "common" = TRUE;
+CREATE INDEX "JMdict_EntryScore_byElementId_byCommon" ON "JMdict_EntryScore"("elementId", "common");
+CREATE INDEX "JMdict_EntryScore_byCommon" ON "JMdict_EntryScore"("common");
 
 -- NOTE: these views are deduplicated in order not to perform an unnecessary
 --       UNION on every trigger

test/util/romaji_transliteration_test.dart

@@ -8,7 +8,7 @@ void main() {
       expect(result, 'かたまり');
     });
 
-    test('Basic test with dakuten', () {
+    test('Basic test with diacritics', () {
       final result = transliterateLatinToHiragana('gadamari');
       expect(result, 'がだまり');
     });
@@ -54,7 +54,7 @@ void main() {
 
     test('Basic test', expectSpans('katamari', ['か', 'た', 'ま', 'り']));
     test(
-      'Basic test with dakuten',
+      'Basic test with diacritics',
       expectSpans('gadamari', ['が', 'だ', 'ま', 'り']),
     );
     test('wi and we', expectSpans('wiwe', ['うぃ', 'うぇ']));
@@ -72,7 +72,7 @@ void main() {
       expect(result, 'katamari');
     });
 
-    test('Basic test with dakuten', () {
+    test('Basic test with diacritics', () {
       final result = transliterateHiraganaToLatin('がだまり');
       expect(result, 'gadamari');
     });
@@ -91,21 +91,6 @@ void main() {
       final result = transliterateHiraganaToLatin('かっぱ');
       expect(result, 'kappa');
     });
-
-    test('Iteration mark', () {
-      final result = transliterateHiraganaToLatin('さゝき');
-      expect(result, 'sasaki');
-    }, skip: 'Not yet implemented');
-
-    test('Iteration mark with dakuten', () {
-      final result = transliterateHiraganaToLatin('あひゞき');
-      expect(result, 'ahibiki');
-    }, skip: 'Not yet implemented');
-
-    test('Yori', () {
-      final result = transliterateHiraganaToLatin('ゟ');
-      expect(result, 'yori');
-    }, skip: 'Not yet implemented');
   });
 
   group('Hiragana -> Romaji Spans', () {
@@ -125,7 +110,7 @@ void main() {
 
     test('Basic test', expectSpans('かたまり', ['ka', 'ta', 'ma', 'ri']));
     test(
-      'Basic test with dakuten',
+      'Basic test with diacritics',
       expectSpans('がだまり', ['ga', 'da', 'ma', 'ri']),
     );
     test('wi and we', expectSpans('うぃうぇ', ['whi', 'whe']));
@@ -133,17 +118,5 @@ void main() {
 
     // TODO: fix the implementation
     // test('Double consonant', expectSpans('かっぱ', ['ka', 'ppa']));
-
-    test(
-      'Iteration mark',
-      expectSpans('さゝき', ['sa', 'sa', 'ki']),
-      skip: 'Not yet implemented',
-    );
-    test(
-      'Iteration mark with dakuten',
-      expectSpans('あひゞき', ['a', 'hi', 'bi', 'ki']),
-      skip: 'Not yet implemented',
-    );
-    test('Yori', expectSpans('ゟ', ['yori']), skip: 'Not yet implemented');
   });
 }