Generate matchspans for word search results

.gitea/workflows/build.yml

@@ -0,0 +1,38 @@
+name: "Build database"
+on:
+  pull_request:
+  push:
+jobs:
+  evals:
+    runs-on: debian-latest
+    steps:
+    - uses: actions/checkout@v6
+
+    - name: Install sudo
+      run: apt-get update && apt-get -y install sudo
+
+    - name: Install nix
+      uses: https://github.com/cachix/install-nix-action@v31
+
+    - name: Configure nix
+      run: echo -e "show-trace = true\nmax-jobs = auto\ntrusted-users = root\nexperimental-features = nix-command flakes\nbuild-users-group =" > /etc/nix/nix.conf
+
+    - name: Update database inputs
+      run: |
+        nix flake update jmdict-src
+        nix flake update jmdict-with-examples-src
+        nix flake update radkfile-src
+        nix flake update kanjidic2-src
+
+    - name: Build database
+      run: nix build .#database -L
+
+    - name: Upload database as artifact
+      uses: actions/upload-artifact@v3
+      with:
+        name: jadb-${{ gitea.sha }}.zip
+        path: result/jadb.sqlite
+        if-no-files-found: error
+        retention-days: 15
+        # Already compressed
+        compression: 0

.gitea/workflows/test.yml

@@ -0,0 +1,31 @@
+name: "Run tests"
+on:
+  pull_request:
+  push:
+jobs:
+  evals:
+    runs-on: debian-latest
+    steps:
+    - uses: actions/checkout@v6
+
+    - name: Install sudo
+      run: apt-get update && apt-get -y install sudo
+
+    - name: Install nix
+      uses: https://github.com/cachix/install-nix-action@v31
+
+    - name: Configure nix
+      run: echo -e "show-trace = true\nmax-jobs = auto\ntrusted-users = root\nexperimental-features = nix-command flakes\nbuild-users-group =" > /etc/nix/nix.conf
+
+    - name: Update database inputs
+      run: |
+        nix flake update jmdict-src
+        nix flake update jmdict-with-examples-src
+        nix flake update radkfile-src
+        nix flake update kanjidic2-src
+
+    - name: Build database
+      run: nix build .#database -L
+
+    - name: Run tests
+      run: nix develop .# --command dart test

flake.lock

@@ -3,7 +3,7 @@
     "jmdict-src": {
       "flake": false,
       "locked": {
-        "narHash": "sha256-5Y4ySJadyNF/Ckjv9rEjIpLnoN0YpbN+cvOawqiuo5Y=",
+        "narHash": "sha256-1if5Z1ynrCd05ySrvD6ZA1PfKBayhBFzUOe5vplwYXM=",
         "type": "file",
         "url": "http://ftp.edrdg.org/pub/Nihongo/JMdict_e.gz"
       },
@@ -15,7 +15,7 @@
     "jmdict-with-examples-src": {
       "flake": false,
       "locked": {
-        "narHash": "sha256-/lOum1C/0zuq9W+g/TajsOgkTeai8vW4ubUdfX8ahX0=",
+        "narHash": "sha256-3Eb8iVSZFvuf4yH/53tDdN6Znt+tvvra6kd7GIv4LYE=",
         "type": "file",
         "url": "http://ftp.edrdg.org/pub/Nihongo/JMdict_e_examp.gz"
       },
@@ -27,7 +27,7 @@
     "kanjidic2-src": {
       "flake": false,
       "locked": {
-        "narHash": "sha256-2RCsAsosBjMAgTzmd8YLa5qP9HIVy6wP4DoMNy1LCKM=",
+        "narHash": "sha256-mg2cP3rX1wm+dTAQCNHthVcKUH5PZRhGbHv1AP2EwJQ=",
         "type": "file",
         "url": "https://www.edrdg.org/kanjidic/kanjidic2.xml.gz"
       },
@@ -38,11 +38,11 @@
     },
     "nixpkgs": {
       "locked": {
-        "lastModified": 1752480373,
-        "narHash": "sha256-JHQbm+OcGp32wAsXTE/FLYGNpb+4GLi5oTvCxwSoBOA=",
+        "lastModified": 1771369470,
+        "narHash": "sha256-0NBlEBKkN3lufyvFegY4TYv5mCNHbi5OmBDrzihbBMQ=",
         "owner": "NixOS",
         "repo": "nixpkgs",
-        "rev": "62e0f05ede1da0d54515d4ea8ce9c733f12d9f08",
+        "rev": "0182a361324364ae3f436a63005877674cf45efb",
         "type": "github"
       },
       "original": {
@@ -54,13 +54,13 @@
     "radkfile-src": {
       "flake": false,
       "locked": {
-        "narHash": "sha256-rO2z5GPt3g6osZOlpyWysmIbRV2Gw4AR4XvngVTHNpk=",
+        "narHash": "sha256-DHpMUE2Umje8PbzXUCS6pHZeXQ5+WTxbjSkGU3erDHQ=",
         "type": "file",
-        "url": "http://ftp.usf.edu/pub/ftp.monash.edu.au/pub/nihongo/radkfile.gz"
+        "url": "http://ftp.edrdg.org/pub/Nihongo/radkfile.gz"
       },
       "original": {
         "type": "file",
-        "url": "http://ftp.usf.edu/pub/ftp.monash.edu.au/pub/nihongo/radkfile.gz"
+        "url": "http://ftp.edrdg.org/pub/Nihongo/radkfile.gz"
       }
     },
     "root": {

flake.nix

@@ -16,7 +16,7 @@
     };
 
     radkfile-src = {
-      url = "http://ftp.usf.edu/pub/ftp.monash.edu.au/pub/nihongo/radkfile.gz";
+      url = "http://ftp.edrdg.org/pub/Nihongo/radkfile.gz";
       flake = false;
     };
 
@@ -83,7 +83,7 @@
           sqlite-interactive
           sqlite-analyzer
           sqlite-web
-          sqlint
+          # sqlint
           sqlfluff
         ];
         env = {
@@ -109,10 +109,14 @@
       in !(lib.any (b: b) [
           (!(lib.cleanSourceFilter path type))
           (baseName == ".github" && type == "directory")
+          (baseName == ".gitea" && type == "directory")
+
           (baseName == "nix" && type == "directory")
           (baseName == ".envrc" && type == "regular")
           (baseName == "flake.lock" && type == "regular")
           (baseName == "flake.nix" && type == "regular")
+
+          (baseName == ".sqlfluff" && type == "regular")
         ])) ./.;
 
     in forAllSystems (system: pkgs: {
@@ -123,7 +127,7 @@
       '';
 
       jmdict = pkgs.callPackage ./nix/jmdict.nix {
-          inherit jmdict-src jmdict-with-examples-src edrdgMetadata;
+        inherit jmdict-src jmdict-with-examples-src edrdgMetadata;
       };
 
       radkfile =  pkgs.callPackage ./nix/radkfile.nix {

lib/_data_ingestion/open_local_db.dart

@@ -1,9 +1,7 @@
-import 'dart:ffi';
 import 'dart:io';
 
 import 'package:jadb/search.dart';
 import 'package:sqflite_common_ffi/sqflite_ffi.dart';
-import 'package:sqlite3/open.dart';
 
 Future<Database> openLocalDb({
   String? libsqlitePath,
@@ -12,32 +10,17 @@ Future<Database> openLocalDb({
   bool verifyTablesExist = true,
   bool walMode = false,
 }) async {
-  libsqlitePath ??= Platform.environment['LIBSQLITE_PATH'];
   jadbPath ??= Platform.environment['JADB_PATH'];
   jadbPath ??= Directory.current.uri.resolve('jadb.sqlite').path;
 
-  libsqlitePath = (libsqlitePath == null)
-      ? null
-      : File(libsqlitePath).resolveSymbolicLinksSync();
   jadbPath = File(jadbPath).resolveSymbolicLinksSync();
 
-  if (libsqlitePath == null) {
-    throw Exception('LIBSQLITE_PATH is not set');
-  }
-
-  if (!File(libsqlitePath).existsSync()) {
-    throw Exception('LIBSQLITE_PATH does not exist: $libsqlitePath');
-  }
-
   if (!File(jadbPath).existsSync()) {
     throw Exception('JADB_PATH does not exist: $jadbPath');
   }
 
   final db =
-      await createDatabaseFactoryFfi(
-        ffiInit: () =>
-            open.overrideForAll(() => DynamicLibrary.open(libsqlitePath!)),
-      ).openDatabase(
+      await createDatabaseFactoryFfi().openDatabase(
         jadbPath,
         options: OpenDatabaseOptions(
           onConfigure: (db) async {

lib/_data_ingestion/tanos-jlpt/csv_parser.dart

@@ -3,12 +3,20 @@ import 'dart:io';
 
 import 'package:csv/csv.dart';
 import 'package:jadb/_data_ingestion/tanos-jlpt/objects.dart';
+import 'package:xml/xml_events.dart';
 
 Future<List<JLPTRankedWord>> parseJLPTRankedWords(
   Map<String, File> files,
 ) async {
   final List<JLPTRankedWord> result = [];
 
+  final codec = CsvCodec(
+    fieldDelimiter: ',',
+    lineDelimiter: '\n',
+    quoteMode: QuoteMode.strings,
+    escapeCharacter: '\\',
+  );
+
   for (final entry in files.entries) {
     final jlptLevel = entry.key;
     final file = entry.value;
@@ -17,42 +25,42 @@ Future<List<JLPTRankedWord>> parseJLPTRankedWords(
       throw Exception('File $jlptLevel does not exist');
     }
 
-    final rows = await file
+    final words = await file
         .openRead()
         .transform(utf8.decoder)
-        .transform(CsvToListConverter())
+        .transform(codec.decoder)
+        .flatten()
+        .map((row) {
+          if (row.length != 3) {
+            throw Exception('Invalid line in $jlptLevel: $row');
+          }
+          return row;
+        })
+        .map((row) => row.map((e) => e as String).toList())
+        .map((row) {
+          final kanji = row[0].isEmpty
+              ? null
+              : row[0]
+                    .replaceFirst(RegExp('^お・'), '')
+                    .replaceAll(RegExp(r'（.*）'), '');
+
+          final readings = row[1]
+              .split(RegExp('[・/、(:?s+)]'))
+              .map((e) => e.trim())
+              .toList();
+
+          final meanings = row[2].split(',').expand(cleanMeaning).toList();
+
+          return JLPTRankedWord(
+            readings: readings,
+            kanji: kanji,
+            jlptLevel: jlptLevel,
+            meanings: meanings,
+          );
+        })
         .toList();
 
-    for (final row in rows) {
-      if (row.length != 3) {
-        throw Exception('Invalid line in $jlptLevel: $row');
-      }
-
-      final kanji = (row[0] as String).isEmpty
-          ? null
-          : (row[0] as String)
-                .replaceFirst(RegExp('^お・'), '')
-                .replaceAll(RegExp(r'（.*）'), '');
-
-      final readings = (row[1] as String)
-          .split(RegExp('[・/、(:?s+)]'))
-          .map((e) => e.trim())
-          .toList();
-
-      final meanings = (row[2] as String)
-          .split(',')
-          .expand(cleanMeaning)
-          .toList();
-
-      result.add(
-        JLPTRankedWord(
-          readings: readings,
-          kanji: kanji,
-          jlptLevel: jlptLevel,
-          meanings: meanings,
-        ),
-      );
-    }
+    result.addAll(words);
   }
 
   return result;

lib/models/word_search/word_search_match_span.dart

@@ -0,0 +1,62 @@
+enum WordSearchMatchSpanType { kanji, kana, sense }
+
+/// A span of a word search result that corresponds to a match for a kanji, kana, or sense.
+class WordSearchMatchSpan {
+  /// Which subtype of the word search result this span corresponds to - either a kanji, a kana, or a sense.
+  final WordSearchMatchSpanType spanType;
+
+  /// The index of the kanji/kana/sense in the word search result that this span corresponds to.
+  final int index;
+
+  /// When matching a 'sense', this is the index of the English definition in that sense that this span corresponds to. Otherwise, this is always 0.
+  final int subIndex;
+
+  /// The start of the span (inclusive)
+  final int start;
+
+  /// The end of the span (inclusive)
+  final int end;
+
+  WordSearchMatchSpan({
+    required this.spanType,
+    required this.index,
+    required this.start,
+    required this.end,
+    this.subIndex = 0,
+  });
+
+  @override
+  String toString() {
+    return 'WordSearchMatchSpan(spanType: $spanType, index: $index, start: $start, end: $end)';
+  }
+
+  Map<String, Object?> toJson() => {
+    'spanType': spanType.toString().split('.').last,
+    'index': index,
+    'start': start,
+    'end': end,
+  };
+
+  factory WordSearchMatchSpan.fromJson(Map<String, dynamic> json) =>
+      WordSearchMatchSpan(
+        spanType: WordSearchMatchSpanType.values.firstWhere(
+          (e) => e.toString().split('.').last == json['spanType'],
+        ),
+        index: json['index'] as int,
+        start: json['start'] as int,
+        end: json['end'] as int,
+      );
+
+  @override
+  int get hashCode => Object.hash(spanType, index, start, end);
+
+  @override
+  bool operator ==(Object other) {
+    if (identical(this, other)) return true;
+    return other is WordSearchMatchSpan &&
+        other.spanType == spanType &&
+        other.index == index &&
+        other.start == start &&
+        other.end == end;
+  }
+}

lib/models/word_search/word_search_result.dart

@@ -1,9 +1,11 @@
 import 'package:jadb/models/common/jlpt_level.dart';
 import 'package:jadb/models/jmdict/jmdict_kanji_info.dart';
 import 'package:jadb/models/jmdict/jmdict_reading_info.dart';
+import 'package:jadb/models/word_search/word_search_match_span.dart';
 import 'package:jadb/models/word_search/word_search_ruby.dart';
 import 'package:jadb/models/word_search/word_search_sense.dart';
 import 'package:jadb/models/word_search/word_search_sources.dart';
+import 'package:jadb/search/word_search/word_search.dart';
 
 /// A class representing a single dictionary entry from a word search.
 class WordSearchResult {
@@ -34,7 +36,15 @@ class WordSearchResult {
   /// A class listing the sources used to make up the data for this word search result.
   final WordSearchSources sources;
 
-  const WordSearchResult({
+  /// A list of spans, specifying which part of this word result matched the search keyword.
+  ///
+  /// Note that this is considered ephemeral data - it does not originate from the dictionary,
+  /// and unlike the rest of the class it varies based on external information (the searchword).
+  /// It will *NOT* be exported to JSON, but can be reinferred by invoking [inferMatchSpans] with
+  /// the original searchword.
+  List<WordSearchMatchSpan>? matchSpans;
+
+  WordSearchResult({
     required this.score,
     required this.entryId,
     required this.isCommon,
@@ -44,6 +54,7 @@ class WordSearchResult {
     required this.senses,
     required this.jlptLevel,
     required this.sources,
+    this.matchSpans,
   });
 
   Map<String, dynamic> toJson() => {
@@ -81,6 +92,77 @@ class WordSearchResult {
         sources: WordSearchSources.fromJson(json['sources']),
       );
 
+  factory WordSearchResult.empty() => WordSearchResult(
+    score: 0,
+    entryId: 0,
+    isCommon: false,
+    japanese: [],
+    kanjiInfo: {},
+    readingInfo: {},
+    senses: [],
+    jlptLevel: JlptLevel.none,
+    sources: WordSearchSources.empty(),
+  );
+
+  /// Infers which part(s) of this word search result matched the search keyword, and populates [matchSpans] accordingly.
+  void inferMatchSpans(
+    String searchword, {
+    SearchMode searchMode = SearchMode.Auto,
+  }) {
+    // TODO: handle wildcards like '?' and '*' when that becomes supported in the search.
+    // TODO: If the searchMode is provided, we can use that to narrow down which part of the word search results to look at.
+
+    final regex = RegExp(RegExp.escape(searchword));
+    final matchSpans = <WordSearchMatchSpan>[];
+
+    for (final (i, japanese) in japanese.indexed) {
+      final baseMatches = regex.allMatches(japanese.base);
+      matchSpans.addAll(
+        baseMatches.map(
+          (match) => WordSearchMatchSpan(
+            spanType: WordSearchMatchSpanType.kanji,
+            index: i,
+            start: match.start,
+            end: match.end,
+          ),
+        ),
+      );
+
+      if (japanese.furigana != null) {
+        final furiganaMatches = regex.allMatches(japanese.furigana!);
+        matchSpans.addAll(
+          furiganaMatches.map(
+            (match) => WordSearchMatchSpan(
+              spanType: WordSearchMatchSpanType.kana,
+              index: i,
+              start: match.start,
+              end: match.end,
+            ),
+          ),
+        );
+      }
+    }
+
+    for (final (i, sense) in senses.indexed) {
+      for (final (k, definition) in sense.englishDefinitions.indexed) {
+        final definitionMatches = regex.allMatches(definition);
+        matchSpans.addAll(
+          definitionMatches.map(
+            (match) => WordSearchMatchSpan(
+              spanType: WordSearchMatchSpanType.sense,
+              index: i,
+              subIndex: k,
+              start: match.start,
+              end: match.end,
+            ),
+          ),
+        );
+      }
+    }
+
+    this.matchSpans = matchSpans;
+  }
+
   String _formatJapaneseWord(WordSearchRuby word) =>
       word.furigana == null ? word.base : '${word.base} (${word.furigana})';
 

lib/models/word_search/word_search_sources.dart

@@ -9,6 +9,8 @@ class WordSearchSources {
 
   const WordSearchSources({this.jmdict = true, this.jmnedict = false});
 
+  factory WordSearchSources.empty() => const WordSearchSources();
+
   Map<String, Object?> get sqlValue => {'jmdict': jmdict, 'jmnedict': jmnedict};
 
   Map<String, dynamic> toJson() => {'jmdict': jmdict, 'jmnedict': jmnedict};

lib/search/filter_kanji.dart

@@ -1,6 +1,9 @@
 import 'package:jadb/table_names/kanjidic.dart';
 import 'package:sqflite_common/sqflite.dart';
 
+/// Filters a list of kanji characters, returning only those that exist in the database.
+///
+/// If [deduplicate] is true, the returned list will deduplicate the input kanji list before returning the filtered results.
 Future<List<String>> filterKanjiWithDbConnection(
   DatabaseExecutor connection,
   List<String> kanji,

lib/search/kanji_search.dart

@@ -3,9 +3,9 @@ import 'package:jadb/models/kanji_search/kanji_search_radical.dart';
 import 'package:jadb/models/kanji_search/kanji_search_result.dart';
 import 'package:jadb/table_names/kanjidic.dart';
 import 'package:jadb/table_names/radkfile.dart';
-import 'package:jadb/util/romaji_transliteration.dart';
 import 'package:sqflite_common/sqflite.dart';
 
+/// Searches for a kanji character and returns its details, or null if the kanji is not found in the database.
 Future<KanjiSearchResult?> searchKanjiWithDbConnection(
   DatabaseExecutor connection,
   String kanji,
@@ -196,10 +196,7 @@ Future<KanjiSearchResult?> searchKanjiWithDbConnection(
     meanings: meanings.map((item) => item['meaning'] as String).toList(),
     kunyomi: kunyomis.map((item) => item['yomi'] as String).toList(),
     parts: parts.map((item) => item['radical'] as String).toList(),
-    onyomi: onyomis
-        .map((item) => item['yomi'] as String)
-        .map(transliterateHiraganaToKatakana)
-        .toList(),
+    onyomi: onyomis.map((item) => item['yomi'] as String).toList(),
     radical: radical,
     codepoints: {
       for (final codepoint in codepoints)
@@ -217,6 +214,7 @@ Future<KanjiSearchResult?> searchKanjiWithDbConnection(
 
 // TODO: Use fewer queries with `IN` clauses to reduce the number of queries
 
+/// Searches for multiple kanji at once, returning a map of kanji to their search results.
 Future<Map<String, KanjiSearchResult>> searchManyKanjiWithDbConnection(
   DatabaseExecutor connection,
   Set<String> kanji,

lib/search/radical_search.dart

@@ -3,10 +3,16 @@ import 'package:sqflite_common/sqlite_api.dart';
 
 // TODO: validate that the list of radicals all are valid radicals
 
+/// Returns a list of radicals that are part of any kanji that contains all of the input radicals.
+///
+/// This can be used to limit the choices of additional radicals provided to a user,
+/// so that any choice they make will still yield at least one kanji.
 Future<List<String>> searchRemainingRadicalsWithDbConnection(
   DatabaseExecutor connection,
   List<String> radicals,
 ) async {
+  final distinctRadicals = radicals.toSet();
+
   final queryResult = await connection.rawQuery(
     '''
       SELECT DISTINCT "radical"
@@ -14,12 +20,12 @@ Future<List<String>> searchRemainingRadicalsWithDbConnection(
       WHERE "kanji" IN (
         SELECT "kanji"
         FROM "${RADKFILETableNames.radkfile}"
-        WHERE "radical" IN (${List.filled(radicals.length, '?').join(',')})
+        WHERE "radical" IN (${List.filled(distinctRadicals.length, '?').join(',')})
         GROUP BY "kanji"
         HAVING COUNT(DISTINCT "radical") = ?
       )
     ''',
-    [...radicals, radicals.length],
+    [...distinctRadicals, distinctRadicals.length],
   );
 
   final remainingRadicals = queryResult
@@ -29,19 +35,22 @@ Future<List<String>> searchRemainingRadicalsWithDbConnection(
   return remainingRadicals;
 }
 
+/// Returns a list of kanji that contain all of the input radicals.
 Future<List<String>> searchKanjiByRadicalsWithDbConnection(
   DatabaseExecutor connection,
   List<String> radicals,
 ) async {
+  final distinctRadicals = radicals.toSet();
+
   final queryResult = await connection.rawQuery(
     '''
       SELECT "kanji"
       FROM "${RADKFILETableNames.radkfile}"
-      WHERE "radical" IN (${List.filled(radicals.length, '?').join(',')})
+      WHERE "radical" IN (${List.filled(distinctRadicals.length, '?').join(',')})
       GROUP BY "kanji"
       HAVING COUNT(DISTINCT "radical") = ?
     ''',
-    [...radicals, radicals.length],
+    [...distinctRadicals, distinctRadicals.length],
   );
 
   final kanji = queryResult.map((row) => row['kanji'] as String).toList();

lib/search/word_search/word_search.dart

@@ -15,6 +15,7 @@ import 'package:sqflite_common/sqlite_api.dart';
 
 enum SearchMode { Auto, English, Kanji, MixedKanji, Kana, MixedKana }
 
+/// Searches for an input string, returning a list of results with their details. Returns null if the input string is empty.
 Future<List<WordSearchResult>?> searchWordWithDbConnection(
   DatabaseExecutor connection,
   String word, {
@@ -51,9 +52,14 @@ Future<List<WordSearchResult>?> searchWordWithDbConnection(
     linearWordQueryData: linearWordQueryData,
   );
 
+  for (final resultEntry in result) {
+    resultEntry.inferMatchSpans(word, searchMode: searchMode);
+  }
+
   return result;
 }
 
+/// Searches for an input string, returning the amount of results that the search would yield without pagination.
 Future<int?> searchWordCountWithDbConnection(
   DatabaseExecutor connection,
   String word, {
@@ -72,6 +78,7 @@ Future<int?> searchWordCountWithDbConnection(
   return entryIdCount;
 }
 
+/// Fetches a single word by its entry ID, returning null if not found.
 Future<WordSearchResult?> getWordByIdWithDbConnection(
   DatabaseExecutor connection,
   int id,
@@ -107,6 +114,7 @@ Future<WordSearchResult?> getWordByIdWithDbConnection(
   return result.firstOrNull;
 }
 
+/// Fetches multiple words by their entry IDs, returning a map from entry ID to result.
 Future<Map<int, WordSearchResult>> getWordsByIdsWithDbConnection(
   DatabaseExecutor connection,
   Set<int> ids,

nix/database_tool.nix

@@ -7,6 +7,29 @@ buildDartApplication {
   version = "1.0.0";
   inherit src;
 
+  dartEntryPoints."bin/jadb" = "bin/jadb.dart";
+
+  # NOTE: the default dart hooks are using `dart compile`, which is not able to call the
+  #       new dart build hooks required to use package:sqlite3 >= 3.0.0. So we override
+  #       these phases to use `dart build` instead.
+  buildPhase = ''
+    runHook preBuild
+
+    mkdir -p "$out/bin"
+    dart build cli --target "bin/jadb.dart"
+
+    runHook postBuild
+  '';
+
+  installPhase = ''
+    runHook preInstall
+
+    mkdir -p "$out"
+    mv build/cli/*/bundle/* "$out/"
+
+    runHook postInstall
+  '';
+
   autoPubspecLock = ../pubspec.lock;
 
   meta.mainProgram = "jadb";

pubspec.lock

@@ -5,18 +5,18 @@ packages:
     dependency: transitive
     description:
       name: _fe_analyzer_shared
-      sha256: da0d9209ca76bde579f2da330aeb9df62b6319c834fa7baae052021b0462401f
+      sha256: "3b19a47f6ea7c2632760777c78174f47f6aec1e05f0cd611380d4593b8af1dbc"
       url: "https://pub.dev"
     source: hosted
-    version: "85.0.0"
+    version: "96.0.0"
   analyzer:
     dependency: transitive
     description:
       name: analyzer
-      sha256: b1ade5707ab7a90dfd519eaac78a7184341d19adb6096c68d499b59c7c6cf880
+      sha256: "0c516bc4ad36a1a75759e54d5047cb9d15cded4459df01aa35a0b5ec7db2c2a0"
       url: "https://pub.dev"
     source: hosted
-    version: "7.7.0"
+    version: "10.2.0"
   args:
     dependency: "direct main"
     description:
@@ -49,6 +49,14 @@ packages:
       url: "https://pub.dev"
     source: hosted
     version: "0.2.0"
+  code_assets:
+    dependency: transitive
+    description:
+      name: code_assets
+      sha256: "83ccdaa064c980b5596c35dd64a8d3ecc68620174ab9b90b6343b753aa721687"
+      url: "https://pub.dev"
+    source: hosted
+    version: "1.0.0"
   collection:
     dependency: "direct main"
     description:
@@ -77,34 +85,34 @@ packages:
     dependency: transitive
     description:
       name: crypto
-      sha256: "1e445881f28f22d6140f181e07737b22f1e099a5e1ff94b0af2f9e4a463f4855"
+      sha256: c8ea0233063ba03258fbcf2ca4d6dadfefe14f02fab57702265467a19f27fadf
       url: "https://pub.dev"
     source: hosted
-    version: "3.0.6"
+    version: "3.0.7"
   csv:
     dependency: "direct main"
     description:
       name: csv
-      sha256: c6aa2679b2a18cb57652920f674488d89712efaf4d3fdf2e537215b35fc19d6c
+      sha256: bef2950f7a753eb82f894a2eabc3072e73cf21c17096296a5a992797e50b1d0d
       url: "https://pub.dev"
     source: hosted
-    version: "6.0.0"
+    version: "7.1.0"
   equatable:
     dependency: "direct main"
     description:
       name: equatable
-      sha256: "567c64b3cb4cf82397aac55f4f0cbd3ca20d77c6c03bedbc4ceaddc08904aef7"
+      sha256: "3e0141505477fd8ad55d6eb4e7776d3fe8430be8e497ccb1521370c3f21a3e2b"
       url: "https://pub.dev"
     source: hosted
-    version: "2.0.7"
+    version: "2.0.8"
   ffi:
     dependency: transitive
     description:
       name: ffi
-      sha256: "289279317b4b16eb2bb7e271abccd4bf84ec9bdcbe999e278a94b804f5630418"
+      sha256: "6d7fd89431262d8f3125e81b50d3847a091d846eafcd4fdb88dd06f36d705a45"
       url: "https://pub.dev"
     source: hosted
-    version: "2.1.4"
+    version: "2.2.0"
   file:
     dependency: transitive
     description:
@@ -129,6 +137,14 @@ packages:
       url: "https://pub.dev"
     source: hosted
     version: "2.1.3"
+  hooks:
+    dependency: transitive
+    description:
+      name: hooks
+      sha256: "7a08a0d684cb3b8fb604b78455d5d352f502b68079f7b80b831c62220ab0a4f6"
+      url: "https://pub.dev"
+    source: hosted
+    version: "1.0.1"
   http_multi_server:
     dependency: transitive
     description:
@@ -153,22 +169,14 @@ packages:
       url: "https://pub.dev"
     source: hosted
     version: "1.0.5"
-  js:
-    dependency: transitive
-    description:
-      name: js
-      sha256: "53385261521cc4a0c4658fd0ad07a7d14591cf8fc33abbceae306ddb974888dc"
-      url: "https://pub.dev"
-    source: hosted
-    version: "0.7.2"
   lints:
     dependency: "direct dev"
     description:
       name: lints
-      sha256: a5e2b223cb7c9c8efdc663ef484fdd95bb243bff242ef5b13e26883547fce9a0
+      sha256: "12f842a479589fea194fe5c5a3095abc7be0c1f2ddfa9a0e76aed1dbd26a87df"
       url: "https://pub.dev"
     source: hosted
-    version: "6.0.0"
+    version: "6.1.0"
   logging:
     dependency: transitive
     description:
@@ -181,18 +189,18 @@ packages:
     dependency: transitive
     description:
       name: matcher
-      sha256: dc58c723c3c24bf8d3e2d3ad3f2f9d7bd9cf43ec6feaa64181775e60190153f2
+      sha256: "12956d0ad8390bbcc63ca2e1469c0619946ccb52809807067a7020d57e647aa6"
       url: "https://pub.dev"
     source: hosted
-    version: "0.12.17"
+    version: "0.12.18"
   meta:
     dependency: transitive
     description:
       name: meta
-      sha256: "23f08335362185a5ea2ad3a4e597f1375e78bce8a040df5c600c8d3552ef2394"
+      sha256: "9f29b9bcc8ee287b1a31e0d01be0eae99a930dbffdaecf04b3f3d82a969f296f"
       url: "https://pub.dev"
     source: hosted
-    version: "1.17.0"
+    version: "1.18.1"
   mime:
     dependency: transitive
     description:
@@ -201,6 +209,14 @@ packages:
       url: "https://pub.dev"
     source: hosted
     version: "2.0.0"
+  native_toolchain_c:
+    dependency: transitive
+    description:
+      name: native_toolchain_c
+      sha256: "89e83885ba09da5fdf2cdacc8002a712ca238c28b7f717910b34bcd27b0d03ac"
+      url: "https://pub.dev"
+    source: hosted
+    version: "0.17.4"
   node_preamble:
     dependency: transitive
     description:
@@ -229,18 +245,18 @@ packages:
     dependency: transitive
     description:
       name: petitparser
-      sha256: "9436fe11f82d7cc1642a8671e5aa4149ffa9ae9116e6cf6dd665fc0653e3825c"
+      sha256: "91bd59303e9f769f108f8df05e371341b15d59e995e6806aefab827b58336675"
       url: "https://pub.dev"
     source: hosted
-    version: "7.0.0"
+    version: "7.0.2"
   pool:
     dependency: transitive
     description:
       name: pool
-      sha256: "20fe868b6314b322ea036ba325e6fc0711a22948856475e2c2b6306e8ab39c2a"
+      sha256: "978783255c543aa3586a1b3c21f6e9d720eb315376a915872c61ef8b5c20177d"
       url: "https://pub.dev"
     source: hosted
-    version: "1.5.1"
+    version: "1.5.2"
   pub_semver:
     dependency: transitive
     description:
@@ -301,34 +317,34 @@ packages:
     dependency: transitive
     description:
       name: source_span
-      sha256: "254ee5351d6cb365c859e20ee823c3bb479bf4a293c22d17a9f1bf144ce86f7c"
+      sha256: "56a02f1f4cd1a2d96303c0144c93bd6d909eea6bee6bf5a0e0b685edbd4c47ab"
       url: "https://pub.dev"
     source: hosted
-    version: "1.10.1"
+    version: "1.10.2"
   sqflite_common:
     dependency: "direct main"
     description:
       name: sqflite_common
-      sha256: "84731e8bfd8303a3389903e01fb2141b6e59b5973cacbb0929021df08dddbe8b"
+      sha256: "6ef422a4525ecc601db6c0a2233ff448c731307906e92cabc9ba292afaae16a6"
       url: "https://pub.dev"
     source: hosted
-    version: "2.5.5"
+    version: "2.5.6"
   sqflite_common_ffi:
     dependency: "direct main"
     description:
       name: sqflite_common_ffi
-      sha256: "9faa2fedc5385ef238ce772589f7718c24cdddd27419b609bb9c6f703ea27988"
+      sha256: c59fcdc143839a77581f7a7c4de018e53682408903a0a0800b95ef2dc4033eff
       url: "https://pub.dev"
     source: hosted
-    version: "2.3.6"
+    version: "2.4.0+2"
   sqlite3:
     dependency: "direct main"
     description:
       name: sqlite3
-      sha256: "608b56d594e4c8498c972c8f1507209f9fd74939971b948ddbbfbfd1c9cb3c15"
+      sha256: b7cf6b37667f6a921281797d2499ffc60fb878b161058d422064f0ddc78f6aa6
       url: "https://pub.dev"
     source: hosted
-    version: "2.7.7"
+    version: "3.1.6"
   stack_trace:
     dependency: transitive
     description:
@@ -373,26 +389,26 @@ packages:
     dependency: "direct dev"
     description:
       name: test
-      sha256: "65e29d831719be0591f7b3b1a32a3cda258ec98c58c7b25f7b84241bc31215bb"
+      sha256: "54c516bbb7cee2754d327ad4fca637f78abfc3cbcc5ace83b3eda117e42cd71a"
       url: "https://pub.dev"
     source: hosted
-    version: "1.26.2"
+    version: "1.29.0"
   test_api:
     dependency: transitive
     description:
       name: test_api
-      sha256: "522f00f556e73044315fa4585ec3270f1808a4b186c936e612cab0b565ff1e00"
+      sha256: "93167629bfc610f71560ab9312acdda4959de4df6fac7492c89ff0d3886f6636"
       url: "https://pub.dev"
     source: hosted
-    version: "0.7.6"
+    version: "0.7.9"
   test_core:
     dependency: transitive
     description:
       name: test_core
-      sha256: "80bf5a02b60af04b09e14f6fe68b921aad119493e26e490deaca5993fef1b05a"
+      sha256: "394f07d21f0f2255ec9e3989f21e54d3c7dc0e6e9dbce160e5a9c1a6be0e2943"
       url: "https://pub.dev"
     source: hosted
-    version: "0.6.11"
+    version: "0.6.15"
   typed_data:
     dependency: transitive
     description:
@@ -413,10 +429,10 @@ packages:
     dependency: transitive
     description:
       name: watcher
-      sha256: "0b7fd4a0bbc4b92641dbf20adfd7e3fd1398fe17102d94b674234563e110088a"
+      sha256: "1398c9f081a753f9226febe8900fce8f7d0a67163334e1c94a2438339d79d635"
       url: "https://pub.dev"
     source: hosted
-    version: "1.1.2"
+    version: "1.2.1"
   web:
     dependency: transitive
     description:
@@ -453,10 +469,10 @@ packages:
     dependency: "direct main"
     description:
       name: xml
-      sha256: "3202a47961c1a0af6097c9f8c1b492d705248ba309e6f7a72410422c05046851"
+      sha256: "971043b3a0d3da28727e40ed3e0b5d18b742fa5a68665cca88e74b7876d5e025"
       url: "https://pub.dev"
     source: hosted
-    version: "6.6.0"
+    version: "6.6.1"
   yaml:
     dependency: transitive
     description:
@@ -466,4 +482,4 @@ packages:
     source: hosted
     version: "3.1.3"
 sdks:
-  dart: ">=3.8.0 <4.0.0"
+  dart: ">=3.10.1 <4.0.0"

pubspec.yaml

@@ -4,17 +4,17 @@ version: 1.0.0
 homepage: https://git.pvv.ntnu.no/oysteikt/jadb
 
 environment:
-  sdk: '^3.8.0'
+  sdk: '^3.9.0'
 
 dependencies:
   args: ^2.7.0
   collection: ^1.19.0
-  csv: ^6.0.0
+  csv: ^7.1.0
   equatable: ^2.0.0
   path: ^1.9.1
   sqflite_common: ^2.5.0
   sqflite_common_ffi: ^2.3.0
-  sqlite3: ^2.7.7
+  sqlite3: ^3.1.6
   xml: ^6.5.0
 
 dev_dependencies:
@@ -24,6 +24,11 @@ dev_dependencies:
 executables:
   jadb: jadb
 
+hooks:
+  user_defines:
+    sqlite3:
+      source: system
+
 topics:
   - database
   - dictionary

test/models/create_empty_db_test.dart

@@ -4,20 +4,11 @@ import 'dart:io';
 import 'package:jadb/models/create_empty_db.dart';
 import 'package:jadb/search.dart';
 import 'package:sqflite_common_ffi/sqflite_ffi.dart';
-import 'package:sqlite3/open.dart';
+// import 'package:sqlite3/open.dart';
 import 'package:test/test.dart';
 
 Future<DatabaseExecutor> setup_inmemory_database() async {
-  final libsqlitePath = Platform.environment['LIBSQLITE_PATH'];
-
-  if (libsqlitePath == null) {
-    throw Exception('LIBSQLITE_PATH is not set');
-  }
-
-  final dbConnection = await createDatabaseFactoryFfi(
-    ffiInit: () =>
-        open.overrideForAll(() => DynamicLibrary.open(libsqlitePath)),
-  ).openDatabase(':memory:');
+  final dbConnection = await createDatabaseFactoryFfi().openDatabase(':memory:');
 
   return dbConnection;
 }

test/search/search_match_inference_test.dart

@@ -0,0 +1,194 @@
+import 'package:jadb/models/common/jlpt_level.dart';
+import 'package:jadb/models/word_search/word_search_match_span.dart';
+import 'package:jadb/models/word_search/word_search_result.dart';
+import 'package:jadb/models/word_search/word_search_ruby.dart';
+import 'package:jadb/models/word_search/word_search_sense.dart';
+import 'package:jadb/models/word_search/word_search_sources.dart';
+import 'package:test/test.dart';
+
+void main() {
+  test('Infer match whole word', () {
+    final wordSearchResult = WordSearchResult(
+      entryId: 0,
+      score: 0,
+      isCommon: false,
+      jlptLevel: JlptLevel.none,
+      kanjiInfo: {},
+      readingInfo: {},
+      japanese: [WordSearchRuby(base: '仮名')],
+      senses: [],
+      sources: WordSearchSources.empty(),
+    );
+
+    wordSearchResult.inferMatchSpans('仮名');
+
+    expect(wordSearchResult.matchSpans, [
+      WordSearchMatchSpan(
+        spanType: WordSearchMatchSpanType.kanji,
+        start: 0,
+        end: 2,
+        index: 0,
+      ),
+    ]);
+  });
+
+  test('Infer match part of word', () {
+    final wordSearchResult = WordSearchResult(
+      entryId: 0,
+      score: 0,
+      isCommon: false,
+      jlptLevel: JlptLevel.none,
+      kanjiInfo: {},
+      readingInfo: {},
+      japanese: [WordSearchRuby(base: '仮名')],
+      senses: [],
+      sources: WordSearchSources.empty(),
+    );
+
+    wordSearchResult.inferMatchSpans('仮');
+
+    expect(wordSearchResult.matchSpans, [
+      WordSearchMatchSpan(
+        spanType: WordSearchMatchSpanType.kanji,
+        start: 0,
+        end: 1,
+        index: 0,
+      ),
+    ]);
+  });
+
+  test('Infer match in middle of word', () {
+    final wordSearchResult = WordSearchResult(
+      entryId: 0,
+      score: 0,
+      isCommon: false,
+      jlptLevel: JlptLevel.none,
+      kanjiInfo: {},
+      readingInfo: {},
+      japanese: [WordSearchRuby(base: 'ありがとう')],
+      senses: [],
+      sources: WordSearchSources.empty(),
+    );
+
+    wordSearchResult.inferMatchSpans('りがと');
+
+    expect(wordSearchResult.matchSpans, [
+      WordSearchMatchSpan(
+        spanType: WordSearchMatchSpanType.kanji,
+        start: 1,
+        end: 4,
+        index: 0,
+      ),
+    ]);
+  });
+
+  test('Infer match in furigana', () {
+    final wordSearchResult = WordSearchResult(
+      entryId: 0,
+      score: 0,
+      isCommon: false,
+      jlptLevel: JlptLevel.none,
+      kanjiInfo: {},
+      readingInfo: {},
+      japanese: [WordSearchRuby(base: '仮名', furigana: 'かな')],
+      senses: [],
+      sources: WordSearchSources.empty(),
+    );
+
+    wordSearchResult.inferMatchSpans('かな');
+
+    expect(wordSearchResult.matchSpans, [
+      WordSearchMatchSpan(
+        spanType: WordSearchMatchSpanType.kana,
+        start: 0,
+        end: 2,
+        index: 0,
+      ),
+    ]);
+  });
+
+  test('Infer match in sense', () {
+    final wordSearchResult = WordSearchResult(
+      entryId: 0,
+      score: 0,
+      isCommon: false,
+      jlptLevel: JlptLevel.none,
+      kanjiInfo: {},
+      readingInfo: {},
+      japanese: [WordSearchRuby(base: '仮名')],
+      senses: [
+        WordSearchSense(
+          antonyms: [],
+          dialects: [],
+          englishDefinitions: ['kana'],
+          fields: [],
+          info: [],
+          languageSource: [],
+          misc: [],
+          partsOfSpeech: [],
+          restrictedToKanji: [],
+          restrictedToReading: [],
+          seeAlso: [],
+        ),
+      ],
+      sources: WordSearchSources.empty(),
+    );
+
+    wordSearchResult.inferMatchSpans('kana');
+
+    expect(wordSearchResult.matchSpans, [
+      WordSearchMatchSpan(
+        spanType: WordSearchMatchSpanType.sense,
+        start: 0,
+        end: 4,
+        index: 0,
+      ),
+    ]);
+  });
+
+  test('Infer multiple matches', () {
+    final wordSearchResult = WordSearchResult(
+      entryId: 0,
+      score: 0,
+      isCommon: false,
+      jlptLevel: JlptLevel.none,
+      kanjiInfo: {},
+      readingInfo: {},
+      japanese: [WordSearchRuby(base: '仮名', furigana: 'かな')],
+      senses: [
+        WordSearchSense(
+          antonyms: [],
+          dialects: [],
+          englishDefinitions: ['kana', 'the kana'],
+          fields: [],
+          info: [],
+          languageSource: [],
+          misc: [],
+          partsOfSpeech: [],
+          restrictedToKanji: [],
+          restrictedToReading: [],
+          seeAlso: [],
+        ),
+      ],
+      sources: WordSearchSources.empty(),
+    );
+
+    wordSearchResult.inferMatchSpans('kana');
+
+    expect(wordSearchResult.matchSpans, [
+      WordSearchMatchSpan(
+        spanType: WordSearchMatchSpanType.sense,
+        start: 0,
+        end: 4,
+        index: 0,
+      ),
+      WordSearchMatchSpan(
+        spanType: WordSearchMatchSpanType.sense,
+        start: 4,
+        end: 8,
+        index: 0,
+        subIndex: 1,
+      ),
+    ]);
+  });
+}