WIP

flake.lock

@@ -7,11 +7,11 @@
         ]
       },
       "locked": {
-        "lastModified": 1780302182,
-        "narHash": "sha256-IfC+dpdjjlkzrWlm+p851T43GsR04wMAPqGn63jisJ4=",
+        "lastModified": 1776081209,
+        "narHash": "sha256-zR1115tcOPnYLk6NznSf7YslyaJLc/MGayEHShitx18=",
         "ref": "refs/heads/main",
-        "rev": "c116674dd1e0b879660e6237e54904aa825d4511",
-        "revCount": 29,
+        "rev": "7fe3552bb16e1d315c0b27b243e5eb53cd9e86fc",
+        "revCount": 13,
         "type": "git",
         "url": "https://git.pvv.ntnu.no/Mugiten/datasources.git"
       },
@@ -22,11 +22,11 @@
     },
     "nixpkgs": {
       "locked": {
-        "lastModified": 1779560665,
-        "narHash": "sha256-tpyBcxPpcQb8ukyNF7DoCwfSY3VPsxHoYwj00Cayv5o=",
+        "lastModified": 1777954456,
+        "narHash": "sha256-hGdgeU2Nk87RAuZyYjyDjFL6LK7dAZN5RE9+hrDTkDU=",
         "owner": "NixOS",
         "repo": "nixpkgs",
-        "rev": "64c08a7ca051951c8eae34e3e3cb1e202fe36786",
+        "rev": "549bd84d6279f9852cae6225e372cc67fb91a4c1",
         "type": "github"
       },
       "original": {

flake.nix

@@ -116,6 +116,8 @@
         ln -s ${src} $out
       '';
 
+      fts5-icu-tokenizer = pkgs.callPackage ./nix/fts5-icu-tokenizer/package.nix { };
+
       inherit (datasources.packages.${system}) jmdict radkfile kanjidic2;
 
       database-tool = pkgs.callPackage ./nix/database_tool.nix {

lib/_data_ingestion/jmdict/objects.dart

@@ -45,7 +45,9 @@ class KanjiElement extends Element {
   });
 
   @override
-  Map<String, Object?> get sqlValue => {...super.sqlValue};
+  Map<String, Object?> get sqlValue => {
+    ...super.sqlValue,
+  };
 }
 
 class ReadingElement extends Element {
@@ -127,19 +129,6 @@ class XRefParts {
     'readingRef': readingRef,
     'senseOrderNum': senseOrderNum,
   };
-
-  @override
-  bool operator ==(Object other) {
-    if (identical(this, other)) return true;
-
-    return other is XRefParts &&
-        other.kanjiRef == kanjiRef &&
-        other.readingRef == readingRef &&
-        other.senseOrderNum == senseOrderNum;
-  }
-
-  @override
-  int get hashCode => Object.hash(kanjiRef, readingRef, senseOrderNum);
 }
 
 class XRef {
@@ -179,7 +168,9 @@ class Sense extends SQLWritable {
   });
 
   @override
-  Map<String, Object?> get sqlValue => {'senseId': senseId};
+  Map<String, Object?> get sqlValue => {
+    'senseId': senseId,
+  };
 
   bool get isEmpty =>
       antonyms.isEmpty &&

lib/_data_ingestion/jmdict/seed_data.dart

@@ -15,12 +15,6 @@ class ResolvedXref {
   const ResolvedXref(this.entry, this.ambiguous);
 }
 
-// A constant map of xref parts to jmdict id for unresolvable xrefs.
-final xrefExceptions = {
-  // NOTE: see https://www.edrdg.org/jmwsgi/entr.py?svc=jmdict&g=2870981.1~2369718 for details
-  XRefParts(kanjiRef: 'プレストレスト', readingRef: 'コンクリート'): 2472380,
-};
-
 /// Resolves an xref (pair of kanji, optionally reading, and optionally sense number) to an a specific
 /// JMdict entry, if possible.
 ///
@@ -34,27 +28,6 @@ ResolvedXref resolveXref(
   XRefParts xref,
 ) {
   late List<Entry> candidateEntries;
-
-  if (xrefExceptions.containsKey(xref)) {
-    final exceptionEntryId = xrefExceptions[xref]!;
-    // NOTE: this is slow, but we have few exceptions. Let's wait for JMdict XML-NG to be released so we can delete this :)
-    final exceptionEntry =
-        entriesByKanji.values
-            .expand((set) => set)
-            .firstWhereOrNull((entry) => entry.entryId == exceptionEntryId) ??
-        entriesByReading.values
-            .expand((set) => set)
-            .firstWhereOrNull((entry) => entry.entryId == exceptionEntryId);
-
-    if (exceptionEntry != null) {
-      return ResolvedXref(exceptionEntry, false);
-    } else {
-      throw Exception(
-        'Xref $xref matches an exception entry ID $exceptionEntryId, but that entry was not found among the candidates.',
-      );
-    }
-  }
-
   switch ((xref.kanjiRef, xref.readingRef)) {
     case (null, null):
       throw Exception('Xref $xref has no kanji or reading reference');

lib/search.dart

@@ -4,7 +4,6 @@ import 'package:jadb/models/word_search/word_search_result.dart';
 import 'package:jadb/search/filter_kanji.dart';
 import 'package:jadb/search/kanji_search.dart';
 import 'package:jadb/search/radical_search.dart';
-import 'package:jadb/search/versions.dart';
 import 'package:jadb/search/word_search/word_search.dart';
 import 'package:sqflite_common/sqlite_api.dart';
 
@@ -19,9 +18,8 @@ extension JaDBConnection on DatabaseExecutor {
       searchKanjiWithDbConnection(this, kanji);
 
   /// Search for a kanji in the database.
-  Future<Map<String, KanjiSearchResult>> jadbGetManyKanji(
-    Iterable<String> kanji,
-  ) => searchManyKanjiWithDbConnection(this, kanji);
+  Future<Map<String, KanjiSearchResult>> jadbGetManyKanji(Iterable<String> kanji) =>
+      searchManyKanjiWithDbConnection(this, kanji);
 
   /// Filter a list of characters, and return the ones that are listed in the kanji dictionary.
   Future<List<String>> filterKanji(
@@ -70,8 +68,4 @@ extension JaDBConnection on DatabaseExecutor {
   /// of the radicals, and return those.
   Future<List<String>> jadbSearchKanjiByRadicals(List<String> radicals) =>
       searchKanjiByRadicalsWithDbConnection(this, radicals);
-
-  /// Retrieve the version information for all datasources in the database.
-  Future<DatasourceVersions> jadbGetDatasourceVersions() =>
-      getDatasourceVersions(this);
 }

lib/search/versions.dart

@@ -1,84 +0,0 @@
-import 'package:jadb/table_names/jmdict.dart';
-import 'package:jadb/table_names/kanjidic.dart';
-import 'package:jadb/table_names/radkfile.dart';
-import 'package:jadb/table_names/tanos_jlpt.dart';
-import 'package:sqflite_common/sqlite_api.dart';
-
-class DatasourceVersions {
-  final String jmdictVersion;
-  final DateTime jmdictDate;
-  final String jmdictHash;
-
-  final String kanjidic2Version;
-  final DateTime kanjidic2Date;
-  final String kanjidic2Hash;
-
-  final String radkfileVersion;
-  final DateTime radkfileDate;
-  final String radkfileHash;
-
-  final String tanosJlptVersion;
-  final DateTime tanosJlptDate;
-  final String tanosJlptHash;
-
-  const DatasourceVersions({
-    required this.jmdictVersion,
-    required this.jmdictDate,
-    required this.jmdictHash,
-    required this.kanjidic2Version,
-    required this.kanjidic2Date,
-    required this.kanjidic2Hash,
-    required this.radkfileVersion,
-    required this.radkfileDate,
-    required this.radkfileHash,
-    required this.tanosJlptVersion,
-    required this.tanosJlptDate,
-    required this.tanosJlptHash,
-  });
-}
-
-DateTime _parseDateTime(String dateString) {
-  try {
-    return DateTime.parse(dateString);
-  } catch (e) {
-    if (RegExp(r'^\d{4}-\d{2}$').hasMatch(dateString)) {
-      return DateTime.parse('$dateString-01');
-    } else if (RegExp(r'^\d{4}$').hasMatch(dateString)) {
-      return DateTime.parse('$dateString-01-01');
-    } else {
-      throw FormatException('Invalid date format: $dateString');
-    }
-  }
-}
-
-Future<DatasourceVersions> getDatasourceVersions(
-  final DatabaseExecutor connection,
-) async {
-  final jmdictVersion = await connection
-      .query(JMdictTableNames.version)
-      .then((rows) => rows.first);
-  final kanjidic2Version = await connection
-      .query(KANJIDICTableNames.version)
-      .then((rows) => rows.first);
-  final radkfileVersion = await connection
-      .query(RADKFILETableNames.version)
-      .then((rows) => rows.first);
-  final tanosJlptVersion = await connection
-      .query(TanosJLPTTableNames.version)
-      .then((rows) => rows.first);
-
-  return DatasourceVersions(
-    jmdictVersion: jmdictVersion['version'] as String,
-    jmdictDate: _parseDateTime(jmdictVersion['date'].toString()),
-    jmdictHash: jmdictVersion['hash'] as String,
-    kanjidic2Version: kanjidic2Version['version'] as String,
-    kanjidic2Date: _parseDateTime(kanjidic2Version['date'].toString()),
-    kanjidic2Hash: kanjidic2Version['hash'] as String,
-    radkfileVersion: radkfileVersion['version'] as String,
-    radkfileDate: _parseDateTime(radkfileVersion['date'].toString()),
-    radkfileHash: radkfileVersion['hash'] as String,
-    tanosJlptVersion: tanosJlptVersion['version'] as String,
-    tanosJlptDate: _parseDateTime(tanosJlptVersion['date'].toString()),
-    tanosJlptHash: tanosJlptVersion['hash'] as String,
-  );
-}

lib/version.dart

@@ -1 +0,0 @@
-const int jadbSchemaVersion = 1;

nix/fts5-icu-tokenizer/0001-provide-sqlite-externally.patch

@@ -0,0 +1,20 @@
+diff --git i/CMakeLists.txt w/CMakeLists.txt
+index 9d99543..11ce4a4 100644
+--- i/CMakeLists.txt
++++ w/CMakeLists.txt
+@@ -131,6 +131,15 @@ if(NOT SQLite3_FOUND)
+   endif()
+ endif()
+ 
++if(SQLite3_FOUND AND NOT TARGET SQLite3::SQLite3)
++  add_library(SQLite3::SQLite3 UNKNOWN IMPORTED)
++
++  set_target_properties(SQLite3::SQLite3 PROPERTIES
++    IMPORTED_LOCATION "${SQLite3_LIBRARIES}"
++    INTERFACE_INCLUDE_DIRECTORIES "${SQLite3_INCLUDE_DIRS}"
++  )
++endif()
++
+ # --- Configure the Library ---
+ 
+ # Select source files based on API version

nix/fts5-icu-tokenizer/package.nix

@@ -0,0 +1,40 @@
+{
+  lib,
+  stdenv,
+  fetchFromSourcehut,
+  cmake,
+  pkg-config,
+  icu,
+  sqlite,
+}:
+
+stdenv.mkDerivation (finalAttrs: {
+  pname = "fts5-icu-tokenizer";
+  version = "5.5";
+  src = fetchFromSourcehut {
+    vc = "hg";
+    owner = "~cwt";
+    repo = "fts5-icu-tokenizer";
+    rev = "v${finalAttrs.version}";
+    hash = "sha256-7Klsu9d1sY+W0buo6kwYdCyDA/u2dBTgu6WuttomTBo=";
+  };
+
+  patches = [
+    ./0001-provide-sqlite-externally.patch
+  ];
+
+  nativeBuildInputs = [
+    cmake
+    pkg-config
+  ];
+
+  buildInputs = [
+    icu
+    sqlite
+  ];
+
+  cmakeFlags = [
+    (lib.cmakeFeature "LOCALE" "ja")
+    (lib.cmakeFeature "API_VERSION" "v2")
+  ];
+})

pubspec.lock

@@ -5,18 +5,18 @@ packages:
     dependency: transitive
     description:
       name: _fe_analyzer_shared
-      sha256: cd6add6f846f35fb79f3c315296703c1a24f3cfd7f4739d91a74961c1c7e9f1b
+      sha256: "8d718c5c58904f9937290fd5dbf2d6a0e02456867706bfb6cd7b81d394e738d5"
       url: "https://pub.dev"
     source: hosted
-    version: "100.0.0"
+    version: "98.0.0"
   analyzer:
     dependency: transitive
     description:
       name: analyzer
-      sha256: "6ba98576948803398b69e3a444df24eacdbe12ed699c7014e120ea38552debbf"
+      sha256: "6141ad5d092d1e1d13929c0504658bbeccc1703505830d7c26e859908f5efc88"
       url: "https://pub.dev"
     source: hosted
-    version: "13.0.0"
+    version: "12.0.0"
   args:
     dependency: "direct main"
     description:
@@ -61,10 +61,10 @@ packages:
     dependency: transitive
     description:
       name: code_assets
-      sha256: bf394f466ba9205f1812a0433b392d6af280f155f56651eda7c18cc32ed493b8
+      sha256: "83ccdaa064c980b5596c35dd64a8d3ecc68620174ab9b90b6343b753aa721687"
       url: "https://pub.dev"
     source: hosted
-    version: "1.2.1"
+    version: "1.0.0"
   collection:
     dependency: "direct main"
     description:
@@ -149,10 +149,10 @@ packages:
     dependency: transitive
     description:
       name: hooks
-      sha256: "62ae9bb76d02526c7c2110a19b6e6ad788fe28d35e553e35efb02a41a46ab43a"
+      sha256: e79ed1e8e1929bc6ecb6ec85f0cb519c887aa5b423705ded0d0f2d9226def388
       url: "https://pub.dev"
     source: hosted
-    version: "2.0.1"
+    version: "1.0.2"
   http_multi_server:
     dependency: transitive
     description:
@@ -197,10 +197,10 @@ packages:
     dependency: transitive
     description:
       name: matcher
-      sha256: "31bd099b47c10cd1aeb55146a2d46ce0277630ecef3f7dae54ad7873f36696cd"
+      sha256: dc0b7dc7651697ea4ff3e69ef44b0407ea32c487a39fff6a4004fa585e901861
       url: "https://pub.dev"
     source: hosted
-    version: "0.12.20"
+    version: "0.12.19"
   meta:
     dependency: transitive
     description:
@@ -221,10 +221,10 @@ packages:
     dependency: transitive
     description:
       name: native_toolchain_c
-      sha256: f59351d28f49520cd3a74eb1f41c5f19ae15e53c65a3231d14af672e46510a96
+      sha256: "6ba77bb18063eebe9de401f5e6437e95e1438af0a87a3a39084fbd37c90df572"
       url: "https://pub.dev"
     source: hosted
-    version: "0.19.1"
+    version: "0.17.6"
   node_preamble:
     dependency: transitive
     description:
@@ -273,14 +273,6 @@ packages:
       url: "https://pub.dev"
     source: hosted
     version: "2.2.0"
-  record_use:
-    dependency: transitive
-    description:
-      name: record_use
-      sha256: "2551bd8eecfe95d14ae75f6021ad0248be5c27f138c2ec12fcb52b500b3ba1ed"
-      url: "https://pub.dev"
-    source: hosted
-    version: "0.6.0"
   shelf:
     dependency: transitive
     description:
@@ -341,26 +333,26 @@ packages:
     dependency: "direct main"
     description:
       name: sqflite_common
-      sha256: "1581ffbf7a0e333b380d6a30737d78516b826cb35beb7fb0bf8a3ea0c678b465"
+      sha256: "6ef422a4525ecc601db6c0a2233ff448c731307906e92cabc9ba292afaae16a6"
       url: "https://pub.dev"
     source: hosted
-    version: "2.5.8"
+    version: "2.5.6"
   sqflite_common_ffi:
     dependency: "direct main"
     description:
       name: sqflite_common_ffi
-      sha256: cd0c7f7de39a08f2d54ef144d9058c46eca8461879aaa648025643455c1e5a20
+      sha256: c59fcdc143839a77581f7a7c4de018e53682408903a0a0800b95ef2dc4033eff
       url: "https://pub.dev"
     source: hosted
-    version: "2.4.0+3"
+    version: "2.4.0+2"
   sqlite3:
     dependency: "direct main"
     description:
       name: sqlite3
-      sha256: "9488c7d2cdb1091c91cacf7e207cff81b28bff8e366f042bad3afe7d34afe189"
+      sha256: caa693ad15a587a2b4fde093b728131a1827903872171089dedb16f7665d3a91
       url: "https://pub.dev"
     source: hosted
-    version: "3.3.2"
+    version: "3.2.0"
   stack_trace:
     dependency: transitive
     description:
@@ -389,10 +381,10 @@ packages:
     dependency: transitive
     description:
       name: synchronized
-      sha256: "63896c27e81b28f8cb4e69ead0d3e8f03f1d1e5fc531a3e579cabed6a2c7c9e5"
+      sha256: c254ade258ec8282947a0acbbc90b9575b4f19673533ee46f2f6e9b3aeefd7c0
       url: "https://pub.dev"
     source: hosted
-    version: "3.4.0+1"
+    version: "3.4.0"
   term_glyph:
     dependency: transitive
     description:
@@ -405,26 +397,26 @@ packages:
     dependency: "direct dev"
     description:
       name: test
-      sha256: ca578dc12bb8b2f40b67b7d3bd2fac4f31c01a6ff7130a14e2597b919934507f
+      sha256: "8d9ceddbab833f180fbefed08afa76d7c03513dfdba87ffcec2718b02bbcbf20"
       url: "https://pub.dev"
     source: hosted
-    version: "1.31.1"
+    version: "1.31.0"
   test_api:
     dependency: transitive
     description:
       name: test_api
-      sha256: "2a122cbe059f8b610d3a5415f42e255b6c17b1f21eee1d960f31080237fb4f11"
+      sha256: "949a932224383300f01be9221c39180316445ecb8e7547f70a41a35bf421fb9e"
       url: "https://pub.dev"
     source: hosted
-    version: "0.7.12"
+    version: "0.7.11"
   test_core:
     dependency: transitive
     description:
       name: test_core
-      sha256: d2e98ec12998368dc59ddd47ab709f2cd55acd6b66dc7db764455a44082f4bc5
+      sha256: "1991d4cfe85d5043241acac92962c3977c8d2f2add1ee73130c7b286417d1d34"
       url: "https://pub.dev"
     source: hosted
-    version: "0.6.18"
+    version: "0.6.17"
   typed_data:
     dependency: transitive
     description:
@@ -437,10 +429,10 @@ packages:
     dependency: transitive
     description:
       name: vm_service
-      sha256: "0016aef94fc66495ac78af5859181e3f3bf2026bd8eecc72b9565601e19ab360"
+      sha256: "45caa6c5917fa127b5dbcfbd1fa60b14e583afdc08bfc96dda38886ca252eb60"
       url: "https://pub.dev"
     source: hosted
-    version: "15.2.0"
+    version: "15.0.2"
   watcher:
     dependency: transitive
     description:
@@ -485,10 +477,10 @@ packages:
     dependency: "direct main"
     description:
       name: xml
-      sha256: "67f0aff7be013d107995e9b75bf4e7f2c3ef2dfdb2c8e68024bba0a7fd5756a4"
+      sha256: "971043b3a0d3da28727e40ed3e0b5d18b742fa5a68665cca88e74b7876d5e025"
       url: "https://pub.dev"
     source: hosted
-    version: "7.0.1"
+    version: "6.6.1"
   yaml:
     dependency: transitive
     description:
@@ -498,4 +490,4 @@ packages:
     source: hosted
     version: "3.1.3"
 sdks:
-  dart: ">=3.11.0 <4.0.0"
+  dart: ">=3.10.1 <4.0.0"

pubspec.yaml

@@ -15,7 +15,7 @@ dependencies:
   sqflite_common: ^2.5.0
   sqflite_common_ffi: ^2.3.0
   sqlite3: ^3.1.6
-  xml: '>=6.0.0 < 8.0.0'
+  xml: ^6.5.0
 
 dev_dependencies:
   benchmark_harness: ^2.4.0

test/search/datasource_versions_test.dart

@@ -1,12 +0,0 @@
-import 'package:jadb/search.dart';
-import 'package:test/test.dart';
-
-import 'setup_database_connection.dart';
-
-void main() {
-  test('Retrieve datasource versions', () async {
-    final connection = await setupDatabaseConnection();
-    final result = await connection.jadbGetDatasourceVersions();
-    expect(result, isNotNull);
-  });
-}