WIP

This was earlier used to compensate for a double nesting bug. This has
been fixed in the latest version of the xml package.

.gitea/workflows/build-and-test.yml

@@ -0,0 +1,74 @@
+name: "Build and test"
+on:
+  workflow_dispatch:
+  pull_request:
+  push:
+jobs:
+  build:
+    runs-on: debian-latest
+    steps:
+    - uses: actions/checkout@v6
+
+    - name: Install sudo
+      run: apt-get update && apt-get -y install sudo
+
+    - name: Install nix
+      uses: https://github.com/cachix/install-nix-action@v31
+      with:
+        extra_nix_config: |
+          experimental-features = nix-command flakes
+          show-trace = true
+          max-jobs = auto
+          trusted-users = root
+          experimental-features = nix-command flakes
+          build-users-group =
+
+    - name: Update database inputs
+      run: |
+        nix flake update jmdict-src
+        nix flake update jmdict-with-examples-src
+        nix flake update radkfile-src
+        nix flake update kanjidic2-src
+
+    - name: Build database
+      run: nix build .#database -L
+
+    - name: Upload database as artifact
+      uses: actions/upload-artifact@v3
+      with:
+        name: jadb-${{ gitea.sha }}.zip
+        path: result/jadb.sqlite
+        if-no-files-found: error
+        retention-days: 15
+        # Already compressed
+        compression: 0
+
+    - name: Print database statistics
+      run: nix develop .# --command sqlite3_analyzer result/jadb.sqlite
+
+    # TODO: Defer failure of tests until after the coverage report is generated and uploaded.
+    - name: Run tests
+      run: nix develop .# --command dart run test --concurrency=1 --coverage-path=coverage/lcov.info
+
+    - name: Generate coverage report
+      run: |
+          GENHTML_ARGS=(
+            --current-date="$(date)"
+            --dark-mode
+            --output-directory coverage/report
+          )
+
+          nix develop .# --command genhtml "${GENHTML_ARGS[@]}" coverage/lcov.info
+
+    - name: Upload coverage report
+      uses: https://git.pvv.ntnu.no/Projects/rsync-action@v2
+      with:
+        source: ./coverage
+        target: jadb/${{ gitea.ref_name }}/
+        username: oysteikt
+        ssh-key: ${{ secrets.OYSTEIKT_GITEA_WEBDOCS_SSH_KEY }}
+        host: microbel.pvv.ntnu.no
+        known-hosts: "microbel.pvv.ntnu.no ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBEq0yasKP0mH6PI6ypmuzPzMnbHELo9k+YB5yW534aKudKZS65YsHJKQ9vapOtmegrn5MQbCCgrshf+/XwZcjbM="
+
+    - name: Run benchmarks
+      run: nix develop .# --command dart run benchmark_harness:bench --flavor jit

.gitea/workflows/build.yml

@@ -1,38 +0,0 @@
-name: "Build database"
-on:
-  pull_request:
-  push:
-jobs:
-  evals:
-    runs-on: debian-latest
-    steps:
-    - uses: actions/checkout@v6
-
-    - name: Install sudo
-      run: apt-get update && apt-get -y install sudo
-
-    - name: Install nix
-      uses: https://github.com/cachix/install-nix-action@v31
-
-    - name: Configure nix
-      run: echo -e "show-trace = true\nmax-jobs = auto\ntrusted-users = root\nexperimental-features = nix-command flakes\nbuild-users-group =" > /etc/nix/nix.conf
-
-    - name: Update database inputs
-      run: |
-        nix flake update jmdict-src
-        nix flake update jmdict-with-examples-src
-        nix flake update radkfile-src
-        nix flake update kanjidic2-src
-
-    - name: Build database
-      run: nix build .#database -L
-
-    - name: Upload database as artifact
-      uses: actions/upload-artifact@v3
-      with:
-        name: jadb-${{ gitea.sha }}.zip
-        path: result/jadb.sqlite
-        if-no-files-found: error
-        retention-days: 15
-        # Already compressed
-        compression: 0

.gitea/workflows/test.yml

@@ -1,31 +0,0 @@
-name: "Run tests"
-on:
-  pull_request:
-  push:
-jobs:
-  evals:
-    runs-on: debian-latest
-    steps:
-    - uses: actions/checkout@v6
-
-    - name: Install sudo
-      run: apt-get update && apt-get -y install sudo
-
-    - name: Install nix
-      uses: https://github.com/cachix/install-nix-action@v31
-
-    - name: Configure nix
-      run: echo -e "show-trace = true\nmax-jobs = auto\ntrusted-users = root\nexperimental-features = nix-command flakes\nbuild-users-group =" > /etc/nix/nix.conf
-
-    - name: Update database inputs
-      run: |
-        nix flake update jmdict-src
-        nix flake update jmdict-with-examples-src
-        nix flake update radkfile-src
-        nix flake update kanjidic2-src
-
-    - name: Build database
-      run: nix build .#database -L
-
-    - name: Run tests
-      run: nix develop .# --command dart test

.gitignore

@@ -8,6 +8,7 @@
 # Conventional directory for build output.
 /doc/
 /build/
+/coverage/
 main.db
 
 # Nix

README.md

@@ -1,7 +1,9 @@
-# jadb
-
 [![built with nix](https://builtwithnix.org/badge.svg)](https://builtwithnix.org)
 
+[Latest coverage report](https://www.pvv.ntnu.no/~oysteikt/gitea/jadb/main/coverage/report/)
+
+# jadb
+
 An SQLite database containing open source japanese dictionary data combined from several sources
 
 Note that while the license for the code is MIT, the data has various licenses.
@@ -16,3 +18,4 @@ Note that while the license for the code is MIT, the data has various licenses.
 | **Tanos JLPT levels:** | https://www.tanos.co.uk/jlpt/ |
 | **Kangxi Radicals:**   | https://ctext.org/kangxi-zidian |
 
+See [docs/overview.md](./docs/overview.md) for notes and implementation details.

benchmark/benchmark.dart

@@ -0,0 +1,5 @@
+import './search/word_search.dart';
+
+Future<void> main() async {
+  await WordSearchBenchmark.main();
+}

benchmark/search/word_search.dart

@@ -0,0 +1,45 @@
+import 'package:benchmark_harness/benchmark_harness.dart';
+import 'package:jadb/search.dart';
+import 'package:sqflite_common/sqlite_api.dart';
+
+import '../../test/search/setup_database_connection.dart';
+
+class WordSearchBenchmark extends AsyncBenchmarkBase {
+  Database? connection;
+
+  static final List<String> searchTerms = [
+    'kana',
+    'kanji',
+    'kawaii',
+    'sushi',
+    'ramen',
+  ];
+
+  WordSearchBenchmark() : super('WordSearchBenchmark');
+
+  static Future<void> main() async {
+    print('Running WordSearchBenchmark...');
+    await WordSearchBenchmark().report();
+    print('Finished WordSearchBenchmark');
+  }
+
+  @override
+  Future<void> setup() async {
+    connection = await setupDatabaseConnection();
+  }
+
+  @override
+  Future<void> run() async {
+    for (final term in searchTerms) {
+      await connection!.jadbSearchWord(term);
+    }
+  }
+
+  @override
+  Future<void> teardown() async {
+    await connection?.close();
+  }
+
+  // @override
+  // Future<void> exercise() => run();
+}

docs/lemmatizer.md

@@ -0,0 +1,13 @@
+# Lemmatizer
+
+The lemmatizer is still quite experimental, but will play a more important role in the project in the future.
+
+It is a manual implementation of a [Finite State Transducer](https://en.wikipedia.org/wiki/Morphological_dictionary#Finite_State_Transducers) for morphological parsing. The FST is used to recursively remove affixes from a word until it (hopefully) deconjugates into its dictionary form. This iterative deconjugation tree will then be combined with queries into the dictionary data to determine if the deconjugation leads to a real known word.
+
+Each separate rule is a separate static object declared in `lib/util/lemmatizer/rules`.
+
+There is a cli subcommand for testing the tool interactively, you can run
+
+```bash
+dart run jadb lemmatize -w '食べさせられない'
+```

docs/overview.md

@@ -0,0 +1,27 @@
+# Overview
+
+This is the documentation for `jadb`. Since I'm currently the only one working on it, the documentation is more or less just notes to myself, to ensure I remember how and why I implemented certain features in a certain way a few months down the road. This is not a comprehensive and formal documentation for downstream use, neither for developers nor end-users.
+
+- [Word Search](./word-search.md)
+- [Lemmatizer](./lemmatizer.md)
+
+## Project structure
+
+- `lib/_data_ingestion` contains all the code for reading data sources, transforming them and compiling them into an SQLite database. This is for the most part isolated from the rest of the codebase, and should not be depended on by any code used for querying the database.
+- `lib/cli` contains code for cli tooling (e.g. argument parsing, subcommand handling, etc.)
+- `lib/const_data` contains database data that is small enough to warrant being hardcoded as dart constants.
+- `lib/models` contains all the code for representing the database schema as Dart classes, and for converting between those classes and the actual database.
+- `lib/search` contains all the code for searching the database.
+- `lib/util/lemmatizer` contains the code for lemmatization, which will be used by the search code in the future.
+- `migrations` contains raw SQL files for creating the database schema.
+
+## SQLite naming conventions
+
+> [!WARNING]
+> All of these conventions are actually not enforced yet, it will be fixed at some point.
+
+- Indices are prefixed with `IDX__`
+- Crossref tables are prefixed with `XREF__`
+- Trigger names are prefixed with `TRG__`
+- Views are prefixed with `VW__`
+- All data sources should have a `<datasource>_Version` table, which contains a single row with the version of the data source used to generate the database.

docs/word-search.md

@@ -0,0 +1,21 @@
+# Word search
+
+The word search procedure is currently split into 3 parts:
+
+1. **Entry ID query**:
+
+Use a complex query with various scoring factors to try to get list of
+database ids pointing at dictionary entries, sorted by how likely we think this
+word is the word that the caller is looking for. The output here is a `List<int>`
+
+2. **Data Query**:
+
+Takes the entry id list from the last search, and performs all queries needed to retrieve
+all the dictionary data for those IDs. The result is a struct with a bunch of flattened lists
+with data for all the dictionary entries. These lists are sorted by the order that the ids
+were provided.
+
+3. **Regrouping**:
+
+Takes the flattened data, and regroups the items into structs with a more "hierarchical" structure.
+All data tagged with the same ID will end up in the same struct. Returns a list of these structs.

flake.lock

@@ -3,7 +3,7 @@
     "jmdict-src": {
       "flake": false,
       "locked": {
-        "narHash": "sha256-lh46uougUzBrRhhwa7cOb32j5Jt9/RjBUhlVjwVzsII=",
+        "narHash": "sha256-eOc3a/AYNRFF3w6lWhyf0Sh92xeXS7+9Qvn0tvvH6Ys=",
         "type": "file",
         "url": "http://ftp.edrdg.org/pub/Nihongo/JMdict_e.gz"
       },
@@ -15,7 +15,7 @@
     "jmdict-with-examples-src": {
       "flake": false,
       "locked": {
-        "narHash": "sha256-5oS2xDyetbuSM6ax3LUjYA3N60x+D3Hg41HEXGFMqLQ=",
+        "narHash": "sha256-nx+WMkscWvA/XImKM7NESYVmICwSgXWOO1KPXasHY94=",
         "type": "file",
         "url": "http://ftp.edrdg.org/pub/Nihongo/JMdict_e_examp.gz"
       },
@@ -27,7 +27,7 @@
     "kanjidic2-src": {
       "flake": false,
       "locked": {
-        "narHash": "sha256-orSeQqSxhn9TtX3anYtbiMEm7nFkuomGnIKoVIUR2CM=",
+        "narHash": "sha256-2T/cAS/kZmVMURStgHVhz524+J9+v5onKs8eEYf2fY0=",
         "type": "file",
         "url": "https://www.edrdg.org/kanjidic/kanjidic2.xml.gz"
       },
@@ -38,11 +38,11 @@
     },
     "nixpkgs": {
       "locked": {
-        "lastModified": 1771848320,
-        "narHash": "sha256-0MAd+0mun3K/Ns8JATeHT1sX28faLII5hVLq0L3BdZU=",
+        "lastModified": 1774386573,
+        "narHash": "sha256-4hAV26quOxdC6iyG7kYaZcM3VOskcPUrdCQd/nx8obc=",
         "owner": "NixOS",
         "repo": "nixpkgs",
-        "rev": "2fc6539b481e1d2569f25f8799236694180c0993",
+        "rev": "46db2e09e1d3f113a13c0d7b81e2f221c63b8ce9",
         "type": "github"
       },
       "original": {

flake.nix

@@ -43,7 +43,12 @@
       "armv7l-linux"
     ];
 
-    forAllSystems = f: lib.genAttrs systems (system: f system nixpkgs.legacyPackages.${system});
+    forAllSystems = f: lib.genAttrs systems (system: let
+      pkgs = import nixpkgs {
+        inherit system;
+        overlays = [ self.overlays.sqlite-icu-ext ];
+      };
+    in f system pkgs);
   in {
     apps = forAllSystems (system: pkgs: {
       default = {
@@ -77,22 +82,47 @@
 
     devShells = forAllSystems (system: pkgs: {
       default = pkgs.mkShell {
-        buildInputs = with pkgs; [
+        packages = with pkgs; [
           dart
           gnumake
-          sqlite-interactive
-          sqlite-analyzer
-          sqlite-web
-          # sqlint
-          sqlfluff
+          lcov
+          sqldiff
+          sqlite-interactive-icu-ext
         ];
         env = {
           LIBSQLITE_PATH = "${pkgs.sqlite.out}/lib/libsqlite3.so";
           JADB_PATH = "result/jadb.sqlite";
+          LD_LIBRARY_PATH = lib.makeLibraryPath [ pkgs.sqlite ];
         };
       };
+
+      sqlite-debugging = pkgs.mkShell {
+        packages = with pkgs; [
+          sqlite-interactive-icu-ext
+          sqlite-analyzer
+          sqlite-web
+          sqlint
+          sqlfluff
+        ];
+      };
     });
 
+    overlays.sqlite-icu-ext = final: prev: let
+      overrideArgs = prev': {
+        configureFlags = prev'.configureFlags ++ [
+          "--with-icu-config=${lib.getExe' prev.icu.dev "icu-config"}"
+          "--enable-icu-collations"
+        ];
+
+        buildInputs = prev'.buildInputs ++ [
+          prev.icu
+        ];
+      };
+    in {
+      sqlite-icu-ext = prev.sqlite.overrideAttrs overrideArgs;
+      sqlite-interactive-icu-ext = prev.sqlite-interactive.overrideAttrs overrideArgs;
+    };
+
     packages = let
       edrdgMetadata = {
         license = [{
@@ -126,6 +156,8 @@
         ln -s ${src} $out
       '';
 
+      inherit (pkgs) sqlite-icu-ext sqlite-interactive-icu-ext;
+
       jmdict = pkgs.callPackage ./nix/jmdict.nix {
         inherit jmdict-src jmdict-with-examples-src edrdgMetadata;
       };
@@ -140,17 +172,20 @@
 
       database-tool = pkgs.callPackage ./nix/database_tool.nix {
         inherit src;
+        sqlite = pkgs.sqlite-icu-ext;
       };
 
       database = pkgs.callPackage ./nix/database.nix {
         inherit (self.packages.${system}) database-tool jmdict radkfile kanjidic2;
         inherit src;
+        sqlite = pkgs.sqlite-icu-ext;
       };
 
       database-wal = pkgs.callPackage ./nix/database.nix {
         inherit (self.packages.${system}) database-tool jmdict radkfile kanjidic2;
         inherit src;
         wal = true;
+        sqlite = pkgs.sqlite-icu-ext;
       };
 
       docs = pkgs.callPackage ./nix/docs.nix {

lib/_data_ingestion/jmdict/seed_data.dart

@@ -5,13 +5,23 @@ import 'package:jadb/_data_ingestion/jmdict/objects.dart';
 import 'package:jadb/table_names/jmdict.dart';
 import 'package:sqflite_common/sqlite_api.dart';
 
+/// A wrapper for the result of resolving an xref, which includes the resolved entry and a flag
+/// indicating whether the xref was ambiguous (i.e. could refer to multiple entries).
 class ResolvedXref {
   Entry entry;
   bool ambiguous;
+  int? senseOrderNum;
 
-  ResolvedXref(this.entry, this.ambiguous);
+  ResolvedXref(this.entry, this.ambiguous, senseOrderNum);
 }
 
+/// Resolves an xref (pair of kanji, optionally reading, and optionally sense number) to an a specific
+/// JMdict entry, if possible.
+///
+/// If the xref is ambiguous (i.e. it could refer to multiple entries), the
+/// first entry is returned, and the returned value is marked as ambiguous.
+///
+/// If the xref cannot be resolved to any entry at all, an exception is thrown.
 ResolvedXref resolveXref(
   SplayTreeMap<String, Set<Entry>> entriesByKanji,
   SplayTreeMap<String, Set<Entry>> entriesByReading,
@@ -65,9 +75,10 @@ ResolvedXref resolveXref(
       'kanjiRef: ${xref.kanjiRef}, readingRef: ${xref.readingRef}, '
       'senseOrderNum: ${xref.senseOrderNum}',
     );
-    return ResolvedXref(candidateEntries.first, true);
+
+    return ResolvedXref(candidateEntries.first, true, xref.senseOrderNum);
   } else {
-    return ResolvedXref(candidateEntries.first, false);
+    return ResolvedXref(candidateEntries.first, false, xref.senseOrderNum);
   }
 }
 
@@ -152,14 +163,14 @@ Future<void> seedJMDictData(List<Entry> entries, Database db) async {
         b.insert(JMdictTableNames.senseRestrictedToKanji, {
           'entryId': e.entryId,
           'senseId': s.senseId,
-          'kanji': rk,
+          'kanjiOrderNum': e.kanji.indexWhere((k) => k.reading == rk) + 1,
         });
       }
       for (final rr in s.restrictedToReading) {
         b.insert(JMdictTableNames.senseRestrictedToReading, {
           'entryId': e.entryId,
           'senseId': s.senseId,
-          'reading': rr,
+          'readingOrderNum': e.readings.indexWhere((r) => r.reading == rr) + 1,
         });
       }
       for (final ls in s.languageSource) {
@@ -181,24 +192,17 @@ Future<void> seedJMDictData(List<Entry> entries, Database db) async {
 
   print('  [JMdict] Building xref trees');
   final SplayTreeMap<String, Set<Entry>> entriesByKanji = SplayTreeMap();
+  final SplayTreeMap<String, Set<Entry>> entriesByReading = SplayTreeMap();
 
   for (final entry in entries) {
     for (final kanji in entry.kanji) {
-      if (entriesByKanji.containsKey(kanji.reading)) {
-        entriesByKanji.update(kanji.reading, (list) => list..add(entry));
-      } else {
-        entriesByKanji.putIfAbsent(kanji.reading, () => {entry});
-      }
+      entriesByKanji.putIfAbsent(kanji.reading, () => {});
+      entriesByKanji.update(kanji.reading, (set) => set..add(entry));
     }
-  }
-  final SplayTreeMap<String, Set<Entry>> entriesByReading = SplayTreeMap();
-  for (final entry in entries) {
+
     for (final reading in entry.readings) {
-      if (entriesByReading.containsKey(reading.reading)) {
-        entriesByReading.update(reading.reading, (list) => list..add(entry));
-      } else {
-        entriesByReading.putIfAbsent(reading.reading, () => {entry});
-      }
+      entriesByReading.putIfAbsent(reading.reading, () => {});
+      entriesByReading.update(reading.reading, (set) => set..add(entry));
     }
   }
 
@@ -207,6 +211,7 @@ Future<void> seedJMDictData(List<Entry> entries, Database db) async {
 
   for (final e in entries) {
     for (final s in e.senses) {
+      final seenSeeAlsoXrefs = <int>{};
       for (final xref in s.seeAlso) {
         final resolvedEntry = resolveXref(
           entriesByKanji,
@@ -214,16 +219,24 @@ Future<void> seedJMDictData(List<Entry> entries, Database db) async {
           xref,
         );
 
+        if (seenSeeAlsoXrefs.contains(resolvedEntry.entry.entryId)) {
+          print(
+            'WARNING: Skipping duplicate seeAlso xref from sense ${s.senseId} to entry ${resolvedEntry.entry.entryId}\n'
+            '  (kanjiRef: ${xref.kanjiRef}, readingRef: ${xref.readingRef}, senseOrderNum: ${xref.senseOrderNum})',
+          );
+          continue;
+        }
+        seenSeeAlsoXrefs.add(resolvedEntry.entry.entryId);
+
         b.insert(JMdictTableNames.senseSeeAlso, {
           'senseId': s.senseId,
           'xrefEntryId': resolvedEntry.entry.entryId,
-          'seeAlsoKanji': xref.kanjiRef,
-          'seeAlsoReading': xref.readingRef,
-          'seeAlsoSense': xref.senseOrderNum,
+          'xrefSenseOrderNum': resolvedEntry.senseOrderNum,
           'ambiguous': resolvedEntry.ambiguous,
         });
       }
 
+      final seenAntonymXrefs = <int>{};
       for (final ant in s.antonyms) {
         final resolvedEntry = resolveXref(
           entriesByKanji,
@@ -231,12 +244,18 @@ Future<void> seedJMDictData(List<Entry> entries, Database db) async {
           ant,
         );
 
+        if (seenAntonymXrefs.contains(resolvedEntry.entry.entryId)) {
+          print(
+            'WARNING: Skipping duplicate antonym xref from sense ${s.senseId} to entry ${resolvedEntry.entry.entryId}\n'
+            '  (kanjiRef: ${ant.kanjiRef}, readingRef: ${ant.readingRef}, senseOrderNum: ${ant.senseOrderNum})',
+          );
+          continue;
+        }
+        seenAntonymXrefs.add(resolvedEntry.entry.entryId);
+
         b.insert(JMdictTableNames.senseAntonyms, {
           'senseId': s.senseId,
           'xrefEntryId': resolvedEntry.entry.entryId,
-          'antonymKanji': ant.kanjiRef,
-          'antonymReading': ant.readingRef,
-          'antonymSense': ant.senseOrderNum,
           'ambiguous': resolvedEntry.ambiguous,
         });
       }

lib/_data_ingestion/jmdict/xml_parser.dart

@@ -10,14 +10,15 @@ List<int?> getPriorityValues(XmlElement e, String prefix) {
     final txt = pri.innerText;
     if (txt.startsWith('news')) {
       news = int.parse(txt.substring(4));
-    } else if (txt.startsWith('ichi'))
+    } else if (txt.startsWith('ichi')) {
       ichi = int.parse(txt.substring(4));
-    else if (txt.startsWith('spec'))
+    } else if (txt.startsWith('spec')) {
       spec = int.parse(txt.substring(4));
-    else if (txt.startsWith('gai'))
+    } else if (txt.startsWith('gai')) {
       gai = int.parse(txt.substring(3));
-    else if (txt.startsWith('nf'))
+    } else if (txt.startsWith('nf')) {
       nf = int.parse(txt.substring(2));
+    }
   }
   return [news, ichi, spec, gai, nf];
 }
@@ -79,16 +80,16 @@ List<Entry> parseJMDictData(XmlElement root) {
     final List<ReadingElement> readingEls = [];
     final List<Sense> senses = [];
 
-    for (final (kanjiNum, k_ele) in entry.findElements('k_ele').indexed) {
-      final kePri = getPriorityValues(k_ele, 'ke');
+    for (final (kanjiNum, kEle) in entry.findElements('k_ele').indexed) {
+      final kePri = getPriorityValues(kEle, 'ke');
       kanjiEls.add(
         KanjiElement(
           orderNum: kanjiNum + 1,
-          info: k_ele
+          info: kEle
               .findElements('ke_inf')
               .map((e) => e.innerText.substring(1, e.innerText.length - 1))
               .toList(),
-          reading: k_ele.findElements('keb').first.innerText,
+          reading: kEle.findElements('keb').first.innerText,
           news: kePri[0],
           ichi: kePri[1],
           spec: kePri[2],
@@ -98,24 +99,24 @@ List<Entry> parseJMDictData(XmlElement root) {
       );
     }
 
-    for (final (orderNum, r_ele) in entry.findElements('r_ele').indexed) {
-      final rePri = getPriorityValues(r_ele, 're');
-      final readingDoesNotMatchKanji = r_ele
+    for (final (orderNum, rEle) in entry.findElements('r_ele').indexed) {
+      final rePri = getPriorityValues(rEle, 're');
+      final readingDoesNotMatchKanji = rEle
           .findElements('re_nokanji')
           .isNotEmpty;
       readingEls.add(
         ReadingElement(
           orderNum: orderNum + 1,
           readingDoesNotMatchKanji: readingDoesNotMatchKanji,
-          info: r_ele
+          info: rEle
               .findElements('re_inf')
               .map((e) => e.innerText.substring(1, e.innerText.length - 1))
               .toList(),
-          restrictions: r_ele
+          restrictions: rEle
               .findElements('re_restr')
               .map((e) => e.innerText)
               .toList(),
-          reading: r_ele.findElements('reb').first.innerText,
+          reading: rEle.findElements('reb').first.innerText,
           news: rePri[0],
           ichi: rePri[1],
           spec: rePri[2],

lib/_data_ingestion/open_local_db.dart

@@ -19,19 +19,19 @@ Future<Database> openLocalDb({
     throw Exception('JADB_PATH does not exist: $jadbPath');
   }
 
-  final db =
-      await createDatabaseFactoryFfi().openDatabase(
-        jadbPath,
-        options: OpenDatabaseOptions(
-          onConfigure: (db) async {
-            if (walMode) {
-              await db.execute('PRAGMA journal_mode = WAL');
-            }
-            await db.execute('PRAGMA foreign_keys = ON');
-          },
-          readOnly: !readWrite,
-        ),
-      );
+  final db = await createDatabaseFactoryFfi().openDatabase(
+    jadbPath,
+    options: OpenDatabaseOptions(
+      onConfigure: (db) async {
+        if (walMode) {
+          await db.execute('PRAGMA journal_mode = WAL');
+        }
+        await db.execute('PRAGMA foreign_keys = ON');
+        await db.execute("SELECT icu_load_collation('ja_JP', 'japanese')");
+      },
+      readOnly: !readWrite,
+    ),
+  );
 
   if (verifyTablesExist) {
     await db.jadbVerifyTables();

lib/_data_ingestion/tanos-jlpt/csv_parser.dart

@@ -10,11 +10,12 @@ Future<List<JLPTRankedWord>> parseJLPTRankedWords(
 ) async {
   final List<JLPTRankedWord> result = [];
 
-  final codec = CsvCodec(
+  final codec = Csv(
     fieldDelimiter: ',',
     lineDelimiter: '\n',
     quoteMode: QuoteMode.strings,
     escapeCharacter: '\\',
+    parseHeaders: false,
   );
 
   for (final entry in files.entries) {
@@ -29,7 +30,6 @@ Future<List<JLPTRankedWord>> parseJLPTRankedWords(
         .openRead()
         .transform(utf8.decoder)
         .transform(codec.decoder)
-        .flatten()
         .map((row) {
           if (row.length != 3) {
             throw Exception('Invalid line in $jlptLevel: $row');

lib/_data_ingestion/tanos-jlpt/overrides.dart

@@ -1,4 +1,4 @@
-const Map<(String?, String), int?> TANOS_JLPT_OVERRIDES = {
+const Map<(String?, String), int?> tanosJLPTOverrides = {
   // N5:
   (null, 'あなた'): 1223615,
   (null, 'あの'): 1000430,

lib/_data_ingestion/tanos-jlpt/resolve.dart

@@ -76,7 +76,7 @@ Future<int?> findEntry(
   if ((entryIds.isEmpty || entryIds.length > 1) && useOverrides) {
     print('No entry found, trying to fetch from overrides');
     final overrideEntries = word.readings
-        .map((reading) => TANOS_JLPT_OVERRIDES[(word.kanji, reading)])
+        .map((reading) => tanosJLPTOverrides[(word.kanji, reading)])
         .whereType<int>()
         .toSet();
 
@@ -86,7 +86,7 @@ Future<int?> findEntry(
       );
     } else if (overrideEntries.isEmpty &&
         !word.readings.any(
-          (reading) => TANOS_JLPT_OVERRIDES.containsKey((word.kanji, reading)),
+          (reading) => tanosJLPTOverrides.containsKey((word.kanji, reading)),
         )) {
       throw Exception(
         'No override entry found for ${word.toString()}: $entryIds',

lib/const_data/kanji_grades.dart

@@ -1,6 +1,5 @@
 /// Jouyou kanji sorted primarily by grades and secondarily by strokes.
-const Map<int, Map<int, List<String>>>
-JOUYOU_KANJI_BY_GRADE_AND_STROKE_COUNT = {
+const Map<int, Map<int, List<String>>> jouyouKanjiByGradeAndStrokeCount = {
   1: {
     1: ['一'],
     2: ['力', '八', '入', '二', '人', '十', '七', '九'],
@@ -1861,8 +1860,8 @@ JOUYOU_KANJI_BY_GRADE_AND_STROKE_COUNT = {
   },
 };
 
-final Map<int, List<String>> JOUYOU_KANJI_BY_GRADES =
-    JOUYOU_KANJI_BY_GRADE_AND_STROKE_COUNT.entries
+final Map<int, List<String>> jouyouKanjiByGrades =
+    jouyouKanjiByGradeAndStrokeCount.entries
         .expand((entry) => entry.value.entries)
         .map((entry) => MapEntry(entry.key, entry.value))
         .fold<Map<int, List<String>>>(

lib/const_data/radicals.dart

@@ -1,4 +1,4 @@
-const Map<int, List<String>> RADICALS = {
+const Map<int, List<String>> radicals = {
   1: ['一', '｜', '丶', 'ノ', '乙', '亅'],
   2: [
     '二',

lib/models/word_search/word_search_result.dart

@@ -1,11 +1,13 @@
 import 'package:jadb/models/common/jlpt_level.dart';
 import 'package:jadb/models/jmdict/jmdict_kanji_info.dart';
+import 'package:jadb/models/jmdict/jmdict_misc.dart';
 import 'package:jadb/models/jmdict/jmdict_reading_info.dart';
 import 'package:jadb/models/word_search/word_search_match_span.dart';
 import 'package:jadb/models/word_search/word_search_ruby.dart';
 import 'package:jadb/models/word_search/word_search_sense.dart';
 import 'package:jadb/models/word_search/word_search_sources.dart';
 import 'package:jadb/search/word_search/word_search.dart';
+import 'package:jadb/util/romaji_transliteration.dart';
 
 /// A class representing a single dictionary entry from a word search.
 class WordSearchResult {
@@ -44,6 +46,42 @@ class WordSearchResult {
   /// the original searchword.
   List<WordSearchMatchSpan>? matchSpans;
 
+  /// Whether the first item in [japanese] contains kanji that likely is rare.
+  bool get hasUnusualKanji =>
+      (japanese.first.furigana != null &&
+          kanjiInfo[japanese.first.base] == JMdictKanjiInfo.rK) ||
+      senses.where((sense) => sense.misc.contains(JMdictMisc.onlyKana)).length >
+          (senses.length / 2);
+
+  /// All contents of [japanese], transliterated to romaji
+  List<String> get romaji => japanese
+      .map((word) => transliterateKanaToLatin(word.furigana ?? word.base))
+      .toList();
+
+  /// All contents of [japanase], where the furigana has either been transliterated to romaji, or
+  /// contains the furigana transliteration of [WordSearchRuby.base].
+  List<WordSearchRuby> get romajiRubys => japanese
+      .map(
+        (word) => WordSearchRuby(
+          base: word.base,
+          furigana: word.furigana != null
+              ? transliterateKanaToLatin(word.furigana!)
+              : transliterateKanaToLatin(word.base),
+        ),
+      )
+      .toList();
+
+  /// The same list of spans as [matchSpans], but the positions have been adjusted for romaji conversion
+  ///
+  /// This is mostly useful in conjunction with [romajiRubys].
+  List<WordSearchMatchSpan>? get romajiMatchSpans {
+    if (matchSpans == null) {
+      return null;
+    }
+
+    throw UnimplementedError('Not yet implemented');
+  }
+
   WordSearchResult({
     required this.score,
     required this.entryId,
@@ -107,7 +145,7 @@ class WordSearchResult {
   /// Infers which part(s) of this word search result matched the search keyword, and populates [matchSpans] accordingly.
   void inferMatchSpans(
     String searchword, {
-    SearchMode searchMode = SearchMode.Auto,
+    SearchMode searchMode = SearchMode.auto,
   }) {
     // TODO: handle wildcards like '?' and '*' when that becomes supported in the search.
     // TODO: If the searchMode is provided, we can use that to narrow down which part of the word search results to look at.
@@ -163,7 +201,7 @@ class WordSearchResult {
     this.matchSpans = matchSpans;
   }
 
-  String _formatJapaneseWord(WordSearchRuby word) =>
+  static String _formatJapaneseWord(WordSearchRuby word) =>
       word.furigana == null ? word.base : '${word.base} (${word.furigana})';
 
   @override

lib/search.dart

@@ -30,7 +30,7 @@ extension JaDBConnection on DatabaseExecutor {
   /// Search for a word in the database.
   Future<List<WordSearchResult>?> jadbSearchWord(
     String word, {
-    SearchMode searchMode = SearchMode.Auto,
+    SearchMode searchMode = SearchMode.auto,
     int page = 0,
     int? pageSize,
   }) => searchWordWithDbConnection(
@@ -54,7 +54,7 @@ extension JaDBConnection on DatabaseExecutor {
   /// Search for a word in the database, and return the count of results.
   Future<int?> jadbSearchWordCount(
     String word, {
-    SearchMode searchMode = SearchMode.Auto,
+    SearchMode searchMode = SearchMode.auto,
   }) => searchWordCountWithDbConnection(this, word, searchMode: searchMode);
 
   /// Given a list of radicals, search which kanji contains all

lib/search/filter_kanji.dart

@@ -18,7 +18,15 @@ Future<List<String>> filterKanjiWithDbConnection(
       .then((value) => value.map((e) => e['literal'] as String).toSet());
 
   if (deduplicate) {
-    return filteredKanji.toList();
+    final List<String> result = [];
+    final Set<String> seen = {};
+    for (final k in kanji) {
+      if (filteredKanji.contains(k) && !seen.contains(k)) {
+        result.add(k);
+        seen.add(k);
+      }
+    }
+    return result;
   } else {
     return kanji.where((k) => filteredKanji.contains(k)).toList();
   }

lib/search/kanji_search.dart

@@ -5,140 +5,186 @@ import 'package:jadb/table_names/kanjidic.dart';
 import 'package:jadb/table_names/radkfile.dart';
 import 'package:sqflite_common/sqflite.dart';
 
+Future<List<Map<String, Object?>>> _charactersQuery(
+  DatabaseExecutor connection,
+  String kanji,
+) => connection.query(
+  KANJIDICTableNames.character,
+  where: 'literal = ?',
+  whereArgs: [kanji],
+);
+
+Future<List<Map<String, Object?>>> _codepointsQuery(
+  DatabaseExecutor connection,
+  String kanji,
+) => connection.query(
+  KANJIDICTableNames.codepoint,
+  where: 'kanji = ?',
+  whereArgs: [kanji],
+);
+
+Future<List<Map<String, Object?>>> _kunyomisQuery(
+  DatabaseExecutor connection,
+  String kanji,
+) => connection.query(
+  KANJIDICTableNames.kunyomi,
+  where: 'kanji = ?',
+  whereArgs: [kanji],
+  orderBy: 'orderNum',
+);
+
+Future<List<Map<String, Object?>>> _onyomisQuery(
+  DatabaseExecutor connection,
+  String kanji,
+) => connection.query(
+  KANJIDICTableNames.onyomi,
+  where: 'kanji = ?',
+  whereArgs: [kanji],
+  orderBy: 'orderNum',
+);
+
+Future<List<Map<String, Object?>>> _meaningsQuery(
+  DatabaseExecutor connection,
+  String kanji,
+) => connection.query(
+  KANJIDICTableNames.meaning,
+  where: 'kanji = ? AND language = ?',
+  whereArgs: [kanji, 'eng'],
+  orderBy: 'orderNum',
+);
+
+Future<List<Map<String, Object?>>> _nanorisQuery(
+  DatabaseExecutor connection,
+  String kanji,
+) => connection.query(
+  KANJIDICTableNames.nanori,
+  where: 'kanji = ?',
+  whereArgs: [kanji],
+);
+
+Future<List<Map<String, Object?>>> _dictionaryReferencesQuery(
+  DatabaseExecutor connection,
+  String kanji,
+) => connection.query(
+  KANJIDICTableNames.dictionaryReference,
+  where: 'kanji = ?',
+  whereArgs: [kanji],
+);
+
+Future<List<Map<String, Object?>>> _queryCodesQuery(
+  DatabaseExecutor connection,
+  String kanji,
+) => connection.query(
+  KANJIDICTableNames.queryCode,
+  where: 'kanji = ?',
+  whereArgs: [kanji],
+);
+
+Future<List<Map<String, Object?>>> _radicalsQuery(
+  DatabaseExecutor connection,
+  String kanji,
+) => connection.rawQuery(
+  '''
+    SELECT DISTINCT
+      "XREF__KANJIDIC_Radical__RADKFILE"."radicalSymbol" AS "symbol",
+      "names"
+    FROM "${KANJIDICTableNames.radical}"
+    JOIN "XREF__KANJIDIC_Radical__RADKFILE" USING ("radicalId")
+    LEFT JOIN (
+      SELECT "radicalId", group_concat("name") AS "names"
+      FROM "${KANJIDICTableNames.radicalName}"
+      GROUP BY "radicalId"
+    ) USING ("radicalId")
+    WHERE "${KANJIDICTableNames.radical}"."kanji" = ?
+  ''',
+  [kanji],
+);
+
+Future<List<Map<String, Object?>>> _partsQuery(
+  DatabaseExecutor connection,
+  String kanji,
+) => connection.query(
+  RADKFILETableNames.radkfile,
+  where: 'kanji = ?',
+  whereArgs: [kanji],
+);
+
+Future<List<Map<String, Object?>>> _readingsQuery(
+  DatabaseExecutor connection,
+  String kanji,
+) => connection.query(
+  KANJIDICTableNames.reading,
+  where: 'kanji = ?',
+  whereArgs: [kanji],
+);
+
+Future<List<Map<String, Object?>>> _strokeMiscountsQuery(
+  DatabaseExecutor connection,
+  String kanji,
+) => connection.query(
+  KANJIDICTableNames.strokeMiscount,
+  where: 'kanji = ?',
+  whereArgs: [kanji],
+);
+
+// Future<List<Map<String, Object?>>> _variantsQuery(
+//   DatabaseExecutor connection,
+//   String kanji,
+// ) => connection.query(
+//   KANJIDICTableNames.variant,
+//   where: 'kanji = ?',
+//   whereArgs: [kanji],
+// );
+
 /// Searches for a kanji character and returns its details, or null if the kanji is not found in the database.
 Future<KanjiSearchResult?> searchKanjiWithDbConnection(
   DatabaseExecutor connection,
   String kanji,
 ) async {
   late final List<Map<String, Object?>> characters;
-  final charactersQuery = connection.query(
-    KANJIDICTableNames.character,
-    where: 'literal = ?',
-    whereArgs: [kanji],
-  );
-
   late final List<Map<String, Object?>> codepoints;
-  final codepointsQuery = connection.query(
-    KANJIDICTableNames.codepoint,
-    where: 'kanji = ?',
-    whereArgs: [kanji],
-  );
-
   late final List<Map<String, Object?>> kunyomis;
-  final kunyomisQuery = connection.query(
-    KANJIDICTableNames.kunyomi,
-    where: 'kanji = ?',
-    whereArgs: [kanji],
-    orderBy: 'orderNum',
-  );
-
   late final List<Map<String, Object?>> onyomis;
-  final onyomisQuery = connection.query(
-    KANJIDICTableNames.onyomi,
-    where: 'kanji = ?',
-    whereArgs: [kanji],
-    orderBy: 'orderNum',
-  );
-
   late final List<Map<String, Object?>> meanings;
-  final meaningsQuery = connection.query(
-    KANJIDICTableNames.meaning,
-    where: 'kanji = ? AND language = ?',
-    whereArgs: [kanji, 'eng'],
-    orderBy: 'orderNum',
-  );
-
   late final List<Map<String, Object?>> nanoris;
-  final nanorisQuery = connection.query(
-    KANJIDICTableNames.nanori,
-    where: 'kanji = ?',
-    whereArgs: [kanji],
-  );
-
   late final List<Map<String, Object?>> dictionaryReferences;
-  final dictionaryReferencesQuery = connection.query(
-    KANJIDICTableNames.dictionaryReference,
-    where: 'kanji = ?',
-    whereArgs: [kanji],
-  );
-
   late final List<Map<String, Object?>> queryCodes;
-  final queryCodesQuery = connection.query(
-    KANJIDICTableNames.queryCode,
-    where: 'kanji = ?',
-    whereArgs: [kanji],
-  );
-
   late final List<Map<String, Object?>> radicals;
-  final radicalsQuery = connection.rawQuery(
-    '''
-      SELECT DISTINCT
-        "XREF__KANJIDIC_Radical__RADKFILE"."radicalSymbol" AS "symbol",
-        "names"
-      FROM "${KANJIDICTableNames.radical}"
-      JOIN "XREF__KANJIDIC_Radical__RADKFILE" USING ("radicalId")
-      LEFT JOIN (
-        SELECT "radicalId", group_concat("name") AS "names"
-        FROM "${KANJIDICTableNames.radicalName}"
-        GROUP BY "radicalId"
-      ) USING ("radicalId")
-      WHERE "${KANJIDICTableNames.radical}"."kanji" = ?
-    ''',
-    [kanji],
-  );
-
   late final List<Map<String, Object?>> parts;
-  final partsQuery = connection.query(
-    RADKFILETableNames.radkfile,
-    where: 'kanji = ?',
-    whereArgs: [kanji],
-  );
-
   late final List<Map<String, Object?>> readings;
-  final readingsQuery = connection.query(
-    KANJIDICTableNames.reading,
-    where: 'kanji = ?',
-    whereArgs: [kanji],
-  );
-
   late final List<Map<String, Object?>> strokeMiscounts;
-  final strokeMiscountsQuery = connection.query(
-    KANJIDICTableNames.strokeMiscount,
-    where: 'kanji = ?',
-    whereArgs: [kanji],
-  );
-
   // TODO: add variant data to result
   // late final List<Map<String, Object?>> variants;
-  // final variants_query = connection.query(
-  //   KANJIDICTableNames.variant,
-  //   where: "kanji = ?",
-  //   whereArgs: [kanji],
-  // );
 
   // TODO: Search for kunyomi and onyomi usage of the characters
-  //       from JMDict. We'll need to fuzzy aquery JMDict_KanjiElement for mathces,
+  //       from JMDict. We'll need to fuzzy aquery JMDict_KanjiElement for matches,
   //       filter JMdict_ReadingElement for kunyomi/onyomi, and then sort the main entry
   //       by JLPT, news frequency, etc.
 
-  await charactersQuery.then((value) => characters = value);
+  await _charactersQuery(connection, kanji).then((value) => characters = value);
 
   if (characters.isEmpty) {
     return null;
   }
 
   await Future.wait({
-    codepointsQuery.then((value) => codepoints = value),
-    kunyomisQuery.then((value) => kunyomis = value),
-    onyomisQuery.then((value) => onyomis = value),
-    meaningsQuery.then((value) => meanings = value),
-    nanorisQuery.then((value) => nanoris = value),
-    dictionaryReferencesQuery.then((value) => dictionaryReferences = value),
-    queryCodesQuery.then((value) => queryCodes = value),
-    radicalsQuery.then((value) => radicals = value),
-    partsQuery.then((value) => parts = value),
-    readingsQuery.then((value) => readings = value),
-    strokeMiscountsQuery.then((value) => strokeMiscounts = value),
+    _codepointsQuery(connection, kanji).then((value) => codepoints = value),
+    _kunyomisQuery(connection, kanji).then((value) => kunyomis = value),
+    _onyomisQuery(connection, kanji).then((value) => onyomis = value),
+    _meaningsQuery(connection, kanji).then((value) => meanings = value),
+    _nanorisQuery(connection, kanji).then((value) => nanoris = value),
+    _dictionaryReferencesQuery(
+      connection,
+      kanji,
+    ).then((value) => dictionaryReferences = value),
+    _queryCodesQuery(connection, kanji).then((value) => queryCodes = value),
+    _radicalsQuery(connection, kanji).then((value) => radicals = value),
+    _partsQuery(connection, kanji).then((value) => parts = value),
+    _readingsQuery(connection, kanji).then((value) => readings = value),
+    _strokeMiscountsQuery(
+      connection,
+      kanji,
+    ).then((value) => strokeMiscounts = value),
     // variants_query.then((value) => variants = value),
   });
 

lib/search/word_search/data_query.dart

@@ -53,274 +53,363 @@ class LinearWordQueryData {
   });
 }
 
+Future<List<Map<String, Object?>>> _sensesQuery(
+  DatabaseExecutor connection,
+  List<int> entryIds,
+) => connection.query(
+  JMdictTableNames.sense,
+  where: 'entryId IN (${List.filled(entryIds.length, '?').join(',')})',
+  whereArgs: entryIds,
+);
+
+Future<List<Map<String, Object?>>> _readingelementsQuery(
+  DatabaseExecutor connection,
+  List<int> entryIds,
+) => connection.query(
+  JMdictTableNames.readingElement,
+  where: 'entryId IN (${List.filled(entryIds.length, '?').join(',')})',
+  whereArgs: entryIds,
+  orderBy: 'orderNum',
+);
+
+Future<List<Map<String, Object?>>> _kanjielementsQuery(
+  DatabaseExecutor connection,
+  List<int> entryIds,
+) => connection.query(
+  JMdictTableNames.kanjiElement,
+  where: 'entryId IN (${List.filled(entryIds.length, '?').join(',')})',
+  whereArgs: entryIds,
+  orderBy: 'orderNum',
+);
+
+Future<List<Map<String, Object?>>> _jlpttagsQuery(
+  DatabaseExecutor connection,
+  List<int> entryIds,
+) => connection.query(
+  TanosJLPTTableNames.jlptTag,
+  where: 'entryId IN (${List.filled(entryIds.length, '?').join(',')})',
+  whereArgs: entryIds,
+);
+
+Future<List<Map<String, Object?>>> _commonentriesQuery(
+  DatabaseExecutor connection,
+  List<int> entryIds,
+) => connection.query(
+  'JMdict_EntryCommon',
+  where: 'entryId IN (${List.filled(entryIds.length, '?').join(',')})',
+  whereArgs: entryIds,
+);
+
+// Sense queries
+
+Future<List<Map<String, Object?>>> _senseantonymsQuery(
+  DatabaseExecutor connection,
+  List<int> senseIds,
+) => connection.rawQuery(
+  """
+      SELECT
+        "${JMdictTableNames.senseAntonyms}".senseId,
+        "${JMdictTableNames.senseAntonyms}".ambiguous,
+        "${JMdictTableNames.senseAntonyms}".xrefEntryId,
+        "JMdict_BaseAndFurigana"."base",
+        "JMdict_BaseAndFurigana"."furigana"
+      FROM "${JMdictTableNames.senseAntonyms}"
+      JOIN "JMdict_BaseAndFurigana"
+        ON "${JMdictTableNames.senseAntonyms}"."xrefEntryId" = "JMdict_BaseAndFurigana"."entryId"
+      WHERE
+        "senseId" IN (${List.filled(senseIds.length, '?').join(',')})
+        AND "JMdict_BaseAndFurigana"."isFirst"
+      ORDER BY
+        "${JMdictTableNames.senseAntonyms}"."senseId",
+        "${JMdictTableNames.senseAntonyms}"."xrefEntryId"
+  """,
+  [...senseIds],
+);
+
+Future<List<Map<String, Object?>>> _senseseealsosQuery(
+  DatabaseExecutor connection,
+  List<int> senseIds,
+) => connection.rawQuery(
+  """
+      SELECT
+        "${JMdictTableNames.senseSeeAlso}"."senseId",
+        "${JMdictTableNames.senseSeeAlso}"."ambiguous",
+        "${JMdictTableNames.senseSeeAlso}"."xrefEntryId",
+        "JMdict_BaseAndFurigana"."base",
+        "JMdict_BaseAndFurigana"."furigana"
+      FROM "${JMdictTableNames.senseSeeAlso}"
+      JOIN "JMdict_BaseAndFurigana"
+        ON "${JMdictTableNames.senseSeeAlso}"."xrefEntryId" = "JMdict_BaseAndFurigana"."entryId"
+      WHERE
+        "senseId" IN (${List.filled(senseIds.length, '?').join(',')})
+        AND "JMdict_BaseAndFurigana"."isFirst"
+      ORDER BY
+        "${JMdictTableNames.senseSeeAlso}"."senseId",
+        "${JMdictTableNames.senseSeeAlso}"."xrefEntryId"
+  """,
+  [...senseIds],
+);
+
+Future<List<Map<String, Object?>>> _sensedialectsQuery(
+  DatabaseExecutor connection,
+  List<int> senseIds,
+) => connection.query(
+  JMdictTableNames.senseDialect,
+  where: 'senseId IN (${List.filled(senseIds.length, '?').join(',')})',
+  whereArgs: senseIds,
+);
+
+Future<List<Map<String, Object?>>> _sensefieldsQuery(
+  DatabaseExecutor connection,
+  List<int> senseIds,
+) => connection.query(
+  JMdictTableNames.senseField,
+  where: 'senseId IN (${List.filled(senseIds.length, '?').join(',')})',
+  whereArgs: senseIds,
+);
+
+Future<List<Map<String, Object?>>> _senseglossariesQuery(
+  DatabaseExecutor connection,
+  List<int> senseIds,
+) => connection.query(
+  JMdictTableNames.senseGlossary,
+  where: 'senseId IN (${List.filled(senseIds.length, '?').join(',')})',
+  whereArgs: senseIds,
+);
+
+Future<List<Map<String, Object?>>> _senseinfosQuery(
+  DatabaseExecutor connection,
+  List<int> senseIds,
+) => connection.query(
+  JMdictTableNames.senseInfo,
+  where: 'senseId IN (${List.filled(senseIds.length, '?').join(',')})',
+  whereArgs: senseIds,
+);
+
+Future<List<Map<String, Object?>>> _senselanguagesourcesQuery(
+  DatabaseExecutor connection,
+  List<int> senseIds,
+) => connection.query(
+  JMdictTableNames.senseLanguageSource,
+  where: 'senseId IN (${List.filled(senseIds.length, '?').join(',')})',
+  whereArgs: senseIds,
+);
+
+Future<List<Map<String, Object?>>> _sensemiscsQuery(
+  DatabaseExecutor connection,
+  List<int> senseIds,
+) => connection.query(
+  JMdictTableNames.senseMisc,
+  where: 'senseId IN (${List.filled(senseIds.length, '?').join(',')})',
+  whereArgs: senseIds,
+);
+
+Future<List<Map<String, Object?>>> _sensepossQuery(
+  DatabaseExecutor connection,
+  List<int> senseIds,
+) => connection.query(
+  JMdictTableNames.sensePOS,
+  where: 'senseId IN (${List.filled(senseIds.length, '?').join(',')})',
+  whereArgs: senseIds,
+);
+
+Future<List<Map<String, Object?>>> _senserestrictedtokanjisQuery(
+  DatabaseExecutor connection,
+  List<int> senseIds,
+) => connection.query(
+  JMdictTableNames.senseRestrictedToKanji,
+  where: 'senseId IN (${List.filled(senseIds.length, '?').join(',')})',
+  whereArgs: senseIds,
+);
+
+Future<List<Map<String, Object?>>> _senserestrictedtoreadingsQuery(
+  DatabaseExecutor connection,
+  List<int> senseIds,
+) => connection.query(
+  JMdictTableNames.senseRestrictedToReading,
+  where: 'senseId IN (${List.filled(senseIds.length, '?').join(',')})',
+  whereArgs: senseIds,
+);
+
+Future<List<Map<String, Object?>>> _examplesentencesQuery(
+  DatabaseExecutor connection,
+  List<int> senseIds,
+) => connection.query(
+  'JMdict_ExampleSentence',
+  where: 'senseId IN (${List.filled(senseIds.length, '?').join(',')})',
+  whereArgs: senseIds,
+);
+
+// Reading/kanji elements queries
+
+Future<List<Map<String, Object?>>> _readingelementinfosQuery(
+  DatabaseExecutor connection,
+  List<int> readingIds,
+) => connection.query(
+  JMdictTableNames.readingInfo,
+  where: '(elementId) IN (${List.filled(readingIds.length, '?').join(',')})',
+  whereArgs: readingIds,
+);
+
+Future<List<Map<String, Object?>>> _readingelementrestrictionsQuery(
+  DatabaseExecutor connection,
+  List<int> readingIds,
+) => connection.query(
+  JMdictTableNames.readingRestriction,
+  where: '(elementId) IN (${List.filled(readingIds.length, '?').join(',')})',
+  whereArgs: readingIds,
+);
+
+Future<List<Map<String, Object?>>> _kanjielementinfosQuery(
+  DatabaseExecutor connection,
+  List<int> kanjiIds,
+) => connection.query(
+  JMdictTableNames.kanjiInfo,
+  where: '(elementId) IN (${List.filled(kanjiIds.length, '?').join(',')})',
+  whereArgs: kanjiIds,
+);
+
+// Xref queries
+
+Future<LinearWordQueryData?> _senseantonymdataQuery(
+  DatabaseExecutor connection,
+  List<int> entryIds,
+) => fetchLinearWordQueryData(connection, entryIds, fetchXrefData: false);
+
+Future<LinearWordQueryData?> _senseseealsodataQuery(
+  DatabaseExecutor connection,
+  List<int> entryIds,
+) => fetchLinearWordQueryData(connection, entryIds, fetchXrefData: false);
+
+// Full query
+
 Future<LinearWordQueryData> fetchLinearWordQueryData(
   DatabaseExecutor connection,
   List<int> entryIds, {
   bool fetchXrefData = true,
 }) async {
   late final List<Map<String, Object?>> senses;
-  final Future<List<Map<String, Object?>>> sensesQuery = connection.query(
-    JMdictTableNames.sense,
-    where: 'entryId IN (${List.filled(entryIds.length, '?').join(',')})',
-    whereArgs: entryIds,
-  );
-
   late final List<Map<String, Object?>> readingElements;
-  final Future<List<Map<String, Object?>>> readingelementsQuery = connection
-      .query(
-        JMdictTableNames.readingElement,
-        where: 'entryId IN (${List.filled(entryIds.length, '?').join(',')})',
-        whereArgs: entryIds,
-        orderBy: 'orderNum',
-      );
-
   late final List<Map<String, Object?>> kanjiElements;
-  final Future<List<Map<String, Object?>>> kanjielementsQuery = connection
-      .query(
-        JMdictTableNames.kanjiElement,
-        where: 'entryId IN (${List.filled(entryIds.length, '?').join(',')})',
-        whereArgs: entryIds,
-        orderBy: 'orderNum',
-      );
-
   late final List<Map<String, Object?>> jlptTags;
-  final Future<List<Map<String, Object?>>> jlpttagsQuery = connection.query(
-    TanosJLPTTableNames.jlptTag,
-    where: 'entryId IN (${List.filled(entryIds.length, '?').join(',')})',
-    whereArgs: entryIds,
-  );
-
   late final List<Map<String, Object?>> commonEntries;
-  final Future<List<Map<String, Object?>>> commonentriesQuery = connection
-      .query(
-        'JMdict_EntryCommon',
-        where: 'entryId IN (${List.filled(entryIds.length, '?').join(',')})',
-        whereArgs: entryIds,
-      );
 
   await Future.wait([
-    sensesQuery.then((value) => senses = value),
-    readingelementsQuery.then((value) => readingElements = value),
-    kanjielementsQuery.then((value) => kanjiElements = value),
-    jlpttagsQuery.then((value) => jlptTags = value),
-    commonentriesQuery.then((value) => commonEntries = value),
+    _sensesQuery(connection, entryIds).then((value) => senses = value),
+    _readingelementsQuery(
+      connection,
+      entryIds,
+    ).then((value) => readingElements = value),
+    _kanjielementsQuery(
+      connection,
+      entryIds,
+    ).then((value) => kanjiElements = value),
+    _jlpttagsQuery(connection, entryIds).then((value) => jlptTags = value),
+    _commonentriesQuery(
+      connection,
+      entryIds,
+    ).then((value) => commonEntries = value),
   ]);
 
-  // Sense queries
-
   final senseIds = senses.map((sense) => sense['senseId'] as int).toList();
 
   late final List<Map<String, Object?>> senseAntonyms;
-  final Future<List<Map<String, Object?>>> senseantonymsQuery = connection
-      .rawQuery(
-        """
-        SELECT
-          "${JMdictTableNames.senseAntonyms}".senseId,
-          "${JMdictTableNames.senseAntonyms}".ambiguous,
-          "${JMdictTableNames.senseAntonyms}".xrefEntryId,
-          "JMdict_BaseAndFurigana"."base",
-          "JMdict_BaseAndFurigana"."furigana"
-        FROM "${JMdictTableNames.senseAntonyms}"
-        JOIN "JMdict_BaseAndFurigana"
-          ON "${JMdictTableNames.senseAntonyms}"."xrefEntryId" = "JMdict_BaseAndFurigana"."entryId"
-        WHERE
-          "senseId" IN (${List.filled(senseIds.length, '?').join(',')})
-          AND "JMdict_BaseAndFurigana"."isFirst"
-        ORDER BY
-          "${JMdictTableNames.senseAntonyms}"."senseId",
-          "${JMdictTableNames.senseAntonyms}"."xrefEntryId"
-    """,
-        [...senseIds],
-      );
-
   late final List<Map<String, Object?>> senseDialects;
-  final Future<List<Map<String, Object?>>> sensedialectsQuery = connection
-      .query(
-        JMdictTableNames.senseDialect,
-        where: 'senseId IN (${List.filled(senseIds.length, '?').join(',')})',
-        whereArgs: senseIds,
-      );
-
   late final List<Map<String, Object?>> senseFields;
-  final Future<List<Map<String, Object?>>> sensefieldsQuery = connection.query(
-    JMdictTableNames.senseField,
-    where: 'senseId IN (${List.filled(senseIds.length, '?').join(',')})',
-    whereArgs: senseIds,
-  );
-
   late final List<Map<String, Object?>> senseGlossaries;
-  final Future<List<Map<String, Object?>>> senseglossariesQuery = connection
-      .query(
-        JMdictTableNames.senseGlossary,
-        where: 'senseId IN (${List.filled(senseIds.length, '?').join(',')})',
-        whereArgs: senseIds,
-      );
-
   late final List<Map<String, Object?>> senseInfos;
-  final Future<List<Map<String, Object?>>> senseinfosQuery = connection.query(
-    JMdictTableNames.senseInfo,
-    where: 'senseId IN (${List.filled(senseIds.length, '?').join(',')})',
-    whereArgs: senseIds,
-  );
-
   late final List<Map<String, Object?>> senseLanguageSources;
-  final Future<List<Map<String, Object?>>> senselanguagesourcesQuery =
-      connection.query(
-        JMdictTableNames.senseLanguageSource,
-        where: 'senseId IN (${List.filled(senseIds.length, '?').join(',')})',
-        whereArgs: senseIds,
-      );
-
   late final List<Map<String, Object?>> senseMiscs;
-  final Future<List<Map<String, Object?>>> sensemiscsQuery = connection.query(
-    JMdictTableNames.senseMisc,
-    where: 'senseId IN (${List.filled(senseIds.length, '?').join(',')})',
-    whereArgs: senseIds,
-  );
-
   late final List<Map<String, Object?>> sensePOSs;
-  final Future<List<Map<String, Object?>>> sensepossQuery = connection.query(
-    JMdictTableNames.sensePOS,
-    where: 'senseId IN (${List.filled(senseIds.length, '?').join(',')})',
-    whereArgs: senseIds,
-  );
-
   late final List<Map<String, Object?>> senseRestrictedToKanjis;
-  final Future<List<Map<String, Object?>>> senserestrictedtokanjisQuery =
-      connection.query(
-        JMdictTableNames.senseRestrictedToKanji,
-        where: 'senseId IN (${List.filled(senseIds.length, '?').join(',')})',
-        whereArgs: senseIds,
-      );
-
   late final List<Map<String, Object?>> senseRestrictedToReadings;
-  final Future<List<Map<String, Object?>>> senserestrictedtoreadingsQuery =
-      connection.query(
-        JMdictTableNames.senseRestrictedToReading,
-        where: 'senseId IN (${List.filled(senseIds.length, '?').join(',')})',
-        whereArgs: senseIds,
-      );
-
   late final List<Map<String, Object?>> senseSeeAlsos;
-  final Future<List<Map<String, Object?>>> senseseealsosQuery = connection
-      .rawQuery(
-        """
-        SELECT
-          "${JMdictTableNames.senseSeeAlso}"."senseId",
-          "${JMdictTableNames.senseSeeAlso}"."ambiguous",
-          "${JMdictTableNames.senseSeeAlso}"."xrefEntryId",
-          "JMdict_BaseAndFurigana"."base",
-          "JMdict_BaseAndFurigana"."furigana"
-        FROM "${JMdictTableNames.senseSeeAlso}"
-        JOIN "JMdict_BaseAndFurigana"
-          ON "${JMdictTableNames.senseSeeAlso}"."xrefEntryId" = "JMdict_BaseAndFurigana"."entryId"
-        WHERE
-          "senseId" IN (${List.filled(senseIds.length, '?').join(',')})
-          AND "JMdict_BaseAndFurigana"."isFirst"
-        ORDER BY
-          "${JMdictTableNames.senseSeeAlso}"."senseId",
-          "${JMdictTableNames.senseSeeAlso}"."xrefEntryId"
-    """,
-        [...senseIds],
-      );
-
   late final List<Map<String, Object?>> exampleSentences;
-  final Future<List<Map<String, Object?>>> examplesentencesQuery = connection
-      .query(
-        'JMdict_ExampleSentence',
-        where: 'senseId IN (${List.filled(senseIds.length, '?').join(',')})',
-        whereArgs: senseIds,
-      );
-
-  // Reading queries
 
   final readingIds = readingElements
       .map((element) => element['elementId'] as int)
       .toList();
 
-  late final List<Map<String, Object?>> readingElementInfos;
-  final Future<List<Map<String, Object?>>> readingelementinfosQuery =
-      connection.query(
-        JMdictTableNames.readingInfo,
-        where:
-            '(elementId) IN (${List.filled(readingIds.length, '?').join(',')})',
-        whereArgs: readingIds,
-      );
-
-  late final List<Map<String, Object?>> readingElementRestrictions;
-  final Future<List<Map<String, Object?>>> readingelementrestrictionsQuery =
-      connection.query(
-        JMdictTableNames.readingRestriction,
-        where:
-            '(elementId) IN (${List.filled(readingIds.length, '?').join(',')})',
-        whereArgs: readingIds,
-      );
-
-  // Kanji queries
-
   final kanjiIds = kanjiElements
       .map((element) => element['elementId'] as int)
       .toList();
 
+  late final List<Map<String, Object?>> readingElementInfos;
+  late final List<Map<String, Object?>> readingElementRestrictions;
+
   late final List<Map<String, Object?>> kanjiElementInfos;
-  final Future<List<Map<String, Object?>>> kanjielementinfosQuery = connection
-      .query(
-        JMdictTableNames.kanjiInfo,
-        where:
-            '(elementId) IN (${List.filled(kanjiIds.length, '?').join(',')})',
-        whereArgs: kanjiIds,
-      );
 
   // Xref data queries
   await Future.wait([
-    senseantonymsQuery.then((value) => senseAntonyms = value),
-    senseseealsosQuery.then((value) => senseSeeAlsos = value),
+    _senseantonymsQuery(
+      connection,
+      senseIds,
+    ).then((value) => senseAntonyms = value),
+    _senseseealsosQuery(
+      connection,
+      senseIds,
+    ).then((value) => senseSeeAlsos = value),
   ]);
 
-  late final LinearWordQueryData? senseAntonymData;
-  final Future<LinearWordQueryData?> senseantonymdataQuery =
-      fetchXrefData
-      ? fetchLinearWordQueryData(
-          connection,
-          senseAntonyms
-              .map((antonym) => antonym['xrefEntryId'] as int)
-              .toList(),
-          fetchXrefData: false,
-        )
-      : Future.value(null);
-
-  late final LinearWordQueryData? senseSeeAlsoData;
-  final Future<LinearWordQueryData?> senseseealsodataQuery =
-      fetchXrefData
-      ? fetchLinearWordQueryData(
-          connection,
-          senseSeeAlsos.map((seeAlso) => seeAlso['xrefEntryId'] as int).toList(),
-          fetchXrefData: false,
-        )
-      : Future.value(null);
+  LinearWordQueryData? senseAntonymData;
+  LinearWordQueryData? senseSeeAlsoData;
 
   await Future.wait([
-    sensedialectsQuery.then((value) => senseDialects = value),
-    sensefieldsQuery.then((value) => senseFields = value),
-    senseglossariesQuery.then((value) => senseGlossaries = value),
-    senseinfosQuery.then((value) => senseInfos = value),
-    senselanguagesourcesQuery.then((value) => senseLanguageSources = value),
-    sensemiscsQuery.then((value) => senseMiscs = value),
-    sensepossQuery.then((value) => sensePOSs = value),
-    senserestrictedtokanjisQuery.then(
-      (value) => senseRestrictedToKanjis = value,
-    ),
-    senserestrictedtoreadingsQuery.then(
-      (value) => senseRestrictedToReadings = value,
-    ),
-    examplesentencesQuery.then((value) => exampleSentences = value),
-    readingelementinfosQuery.then((value) => readingElementInfos = value),
-    readingelementrestrictionsQuery.then(
-      (value) => readingElementRestrictions = value,
-    ),
-    kanjielementinfosQuery.then((value) => kanjiElementInfos = value),
-    senseantonymdataQuery.then((value) => senseAntonymData = value),
-    senseseealsodataQuery.then((value) => senseSeeAlsoData = value),
+    _sensedialectsQuery(
+      connection,
+      senseIds,
+    ).then((value) => senseDialects = value),
+    _sensefieldsQuery(
+      connection,
+      senseIds,
+    ).then((value) => senseFields = value),
+    _senseglossariesQuery(
+      connection,
+      senseIds,
+    ).then((value) => senseGlossaries = value),
+    _senseinfosQuery(connection, senseIds).then((value) => senseInfos = value),
+    _senselanguagesourcesQuery(
+      connection,
+      senseIds,
+    ).then((value) => senseLanguageSources = value),
+    _sensemiscsQuery(connection, senseIds).then((value) => senseMiscs = value),
+    _sensepossQuery(connection, senseIds).then((value) => sensePOSs = value),
+    _senserestrictedtokanjisQuery(
+      connection,
+      senseIds,
+    ).then((value) => senseRestrictedToKanjis = value),
+    _senserestrictedtoreadingsQuery(
+      connection,
+      senseIds,
+    ).then((value) => senseRestrictedToReadings = value),
+    _examplesentencesQuery(
+      connection,
+      senseIds,
+    ).then((value) => exampleSentences = value),
+    _readingelementinfosQuery(
+      connection,
+      readingIds,
+    ).then((value) => readingElementInfos = value),
+    _readingelementrestrictionsQuery(
+      connection,
+      readingIds,
+    ).then((value) => readingElementRestrictions = value),
+    _kanjielementinfosQuery(
+      connection,
+      kanjiIds,
+    ).then((value) => kanjiElementInfos = value),
+
+    if (fetchXrefData)
+      _senseantonymdataQuery(
+        connection,
+        senseAntonyms.map((antonym) => antonym['xrefEntryId'] as int).toList(),
+      ).then((value) => senseAntonymData = value),
+
+    if (fetchXrefData)
+      _senseseealsodataQuery(
+        connection,
+        senseSeeAlsos.map((seeAlso) => seeAlso['xrefEntryId'] as int).toList(),
+      ).then((value) => senseSeeAlsoData = value),
   ]);
 
   return LinearWordQueryData(

lib/search/word_search/entry_id_query.dart

@@ -15,15 +15,15 @@ SearchMode _determineSearchMode(String word) {
   final bool containsAscii = RegExp(r'[A-Za-z]').hasMatch(word);
 
   if (containsKanji && containsAscii) {
-    return SearchMode.MixedKanji;
+    return SearchMode.mixedKanji;
   } else if (containsKanji) {
-    return SearchMode.Kanji;
+    return SearchMode.kanji;
   } else if (containsAscii) {
-    return SearchMode.English;
+    return SearchMode.english;
   } else if (word.contains(hiraganaRegex) || word.contains(katakanaRegex)) {
-    return SearchMode.Kana;
+    return SearchMode.kana;
   } else {
-    return SearchMode.MixedKana;
+    return SearchMode.mixedKana;
   }
 }
 
@@ -199,23 +199,23 @@ Future<List<ScoredEntryId>> _queryEnglish(
     SELECT
       "${JMdictTableNames.sense}"."entryId",
       MAX("JMdict_EntryScore"."score")
-      + (("${JMdictTableNames.senseGlossary}"."phrase" = ? AND "${JMdictTableNames.sense}"."orderNum" = 1) * 50)
-      + (("${JMdictTableNames.senseGlossary}"."phrase" = ? AND "${JMdictTableNames.sense}"."orderNum" = 2) * 30)
-      + (("${JMdictTableNames.senseGlossary}"."phrase" = ?) * 20)
+      + (("${JMdictTableNames.senseGlossary}"."phrase" = ?1 AND "${JMdictTableNames.sense}"."orderNum" = 1) * 50)
+      + (("${JMdictTableNames.senseGlossary}"."phrase" = ?1 AND "${JMdictTableNames.sense}"."orderNum" = 2) * 30)
+      + (("${JMdictTableNames.senseGlossary}"."phrase" = ?1) * 20)
       as "score"
     FROM "${JMdictTableNames.senseGlossary}"
     JOIN "${JMdictTableNames.sense}" USING ("senseId")
     JOIN "JMdict_EntryScore" USING ("entryId")
-    WHERE "${JMdictTableNames.senseGlossary}"."phrase" LIKE ?
+    WHERE "${JMdictTableNames.senseGlossary}"."phrase" LIKE ?2
     GROUP BY "JMdict_EntryScore"."entryId"
     ORDER BY
       "score" DESC,
       "${JMdictTableNames.sense}"."entryId" ASC
-    LIMIT ?
-    OFFSET ?
+    ${pageSize != null ? 'LIMIT ?3' : ''}
+    ${offset != null ? 'OFFSET ?4' : ''}
   '''
         .trim(),
-    [word, word, word, '%${word.replaceAll('%', '')}%', pageSize, offset],
+    [word, '%${word.replaceAll('%', '')}%', if (pageSize != null) pageSize, if (offset != null) offset],
   );
 
   return result
@@ -246,7 +246,7 @@ Future<List<ScoredEntryId>> fetchEntryIds(
   int? pageSize,
   int? offset,
 ) async {
-  if (searchMode == SearchMode.Auto) {
+  if (searchMode == SearchMode.auto) {
     searchMode = _determineSearchMode(word);
   }
 
@@ -254,20 +254,20 @@ Future<List<ScoredEntryId>> fetchEntryIds(
 
   late final List<ScoredEntryId> entryIds;
   switch (searchMode) {
-    case SearchMode.Kanji:
+    case SearchMode.kanji:
       entryIds = await _queryKanji(connection, word, pageSize, offset);
       break;
 
-    case SearchMode.Kana:
+    case SearchMode.kana:
       entryIds = await _queryKana(connection, word, pageSize, offset);
       break;
 
-    case SearchMode.English:
+    case SearchMode.english:
       entryIds = await _queryEnglish(connection, word, pageSize, offset);
       break;
 
-    case SearchMode.MixedKana:
-    case SearchMode.MixedKanji:
+    case SearchMode.mixedKana:
+    case SearchMode.mixedKanji:
     default:
       throw UnimplementedError('Search mode $searchMode is not implemented');
   }
@@ -280,7 +280,7 @@ Future<int?> fetchEntryIdCount(
   String word,
   SearchMode searchMode,
 ) async {
-  if (searchMode == SearchMode.Auto) {
+  if (searchMode == SearchMode.auto) {
     searchMode = _determineSearchMode(word);
   }
 
@@ -289,20 +289,20 @@ Future<int?> fetchEntryIdCount(
   late final int? entryIdCount;
 
   switch (searchMode) {
-    case SearchMode.Kanji:
+    case SearchMode.kanji:
       entryIdCount = await _queryKanjiCount(connection, word);
       break;
 
-    case SearchMode.Kana:
+    case SearchMode.kana:
       entryIdCount = await _queryKanaCount(connection, word);
       break;
 
-    case SearchMode.English:
+    case SearchMode.english:
       entryIdCount = await _queryEnglishCount(connection, word);
       break;
 
-    case SearchMode.MixedKana:
-    case SearchMode.MixedKanji:
+    case SearchMode.mixedKana:
+    case SearchMode.mixedKanji:
     default:
       throw UnimplementedError('Search mode $searchMode is not implemented');
   }

lib/search/word_search/regrouping.dart

@@ -54,7 +54,7 @@ List<WordSearchResult> regroupWordSearchResults({
         .where((element) => element['entryId'] == scoredEntryId.entryId)
         .toList();
 
-    final GroupedWordResult entryReadingElementsGrouped = _regroup_words(
+    final GroupedWordResult entryReadingElementsGrouped = _regroupWords(
       entryId: scoredEntryId.entryId,
       readingElements: entryReadingElements,
       kanjiElements: entryKanjiElements,
@@ -64,7 +64,7 @@ List<WordSearchResult> regroupWordSearchResults({
       kanjiElementInfos: linearWordQueryData.kanjiElementInfos,
     );
 
-    final List<WordSearchSense> entrySensesGrouped = _regroup_senses(
+    final List<WordSearchSense> entrySensesGrouped = _regroupSenses(
       senses: entrySenses,
       senseAntonyms: linearWordQueryData.senseAntonyms,
       senseDialects: linearWordQueryData.senseDialects,
@@ -112,7 +112,7 @@ class GroupedWordResult {
   });
 }
 
-GroupedWordResult _regroup_words({
+GroupedWordResult _regroupWords({
   required int entryId,
   required List<Map<String, Object?>> kanjiElements,
   required List<Map<String, Object?>> kanjiElementInfos,
@@ -195,7 +195,7 @@ GroupedWordResult _regroup_words({
   );
 }
 
-List<WordSearchSense> _regroup_senses({
+List<WordSearchSense> _regroupSenses({
   required List<Map<String, Object?>> senses,
   required List<Map<String, Object?>> senseAntonyms,
   required List<Map<String, Object?>> senseDialects,

lib/search/word_search/word_search.dart

@@ -13,13 +13,31 @@ import 'package:jadb/search/word_search/regrouping.dart';
 import 'package:jadb/table_names/jmdict.dart';
 import 'package:sqflite_common/sqlite_api.dart';
 
-enum SearchMode { Auto, English, Kanji, MixedKanji, Kana, MixedKana }
+enum SearchMode {
+  /// Try to autodetect what is being searched for
+  auto,
+
+  /// Search for english words
+  english,
+
+  /// Search for the kanji reading of a word
+  kanji,
+
+  /// Search for the kanji reading of a word, mixed in with kana/romaji
+  mixedKanji,
+
+  /// Search for the kana reading of a word
+  kana,
+
+  /// Search for the kana reading of a word, mixed in with romaji
+  mixedKana,
+}
 
 /// Searches for an input string, returning a list of results with their details. Returns null if the input string is empty.
 Future<List<WordSearchResult>?> searchWordWithDbConnection(
   DatabaseExecutor connection,
   String word, {
-  SearchMode searchMode = SearchMode.Auto,
+  SearchMode searchMode = SearchMode.auto,
   int page = 0,
   int? pageSize,
 }) async {
@@ -63,7 +81,7 @@ Future<List<WordSearchResult>?> searchWordWithDbConnection(
 Future<int?> searchWordCountWithDbConnection(
   DatabaseExecutor connection,
   String word, {
-  SearchMode searchMode = SearchMode.Auto,
+  SearchMode searchMode = SearchMode.auto,
 }) async {
   if (word.isEmpty) {
     return null;

lib/table_names/jmdict.dart

@@ -1,4 +1,5 @@
 abstract class JMdictTableNames {
+  static const String version = 'JMdict_Version';
   static const String entry = 'JMdict_Entry';
   static const String kanjiElement = 'JMdict_KanjiElement';
   static const String kanjiInfo = 'JMdict_KanjiElementInfo';
@@ -20,6 +21,7 @@ abstract class JMdictTableNames {
   static const String senseSeeAlso = 'JMdict_SenseSeeAlso';
 
   static Set<String> get allTables => {
+    version,
     entry,
     kanjiElement,
     kanjiInfo,

lib/table_names/kanjidic.dart

@@ -1,4 +1,5 @@
 abstract class KANJIDICTableNames {
+  static const String version = 'KANJIDIC_Version';
   static const String character = 'KANJIDIC_Character';
   static const String radicalName = 'KANJIDIC_RadicalName';
   static const String codepoint = 'KANJIDIC_Codepoint';
@@ -17,6 +18,7 @@ abstract class KANJIDICTableNames {
   static const String nanori = 'KANJIDIC_Nanori';
 
   static Set<String> get allTables => {
+    version,
     character,
     radicalName,
     codepoint,

lib/table_names/radkfile.dart

@@ -1,5 +1,6 @@
 abstract class RADKFILETableNames {
+  static const String version = 'RADKFILE_Version';
   static const String radkfile = 'RADKFILE';
 
-  static Set<String> get allTables => {radkfile};
+  static Set<String> get allTables => {version, radkfile};
 }

lib/table_names/tanos_jlpt.dart

@@ -1,5 +1,6 @@
 abstract class TanosJLPTTableNames {
+  static const String version = 'JMdict_JLPT_Version';
   static const String jlptTag = 'JMdict_JLPTTag';
 
-  static Set<String> get allTables => {jlptTag};
+  static Set<String> get allTables => {version, jlptTag};
 }

lib/util/lemmatizer/lemmatizer.dart

@@ -1,3 +1,4 @@
+import 'package:collection/collection.dart';
 import 'package:jadb/util/lemmatizer/rules.dart';
 
 enum WordClass {
@@ -10,6 +11,8 @@ enum WordClass {
   adverb,
   particle,
   input,
+
+  // TODO: add toString and fromString so it can be parsed by the cli
 }
 
 enum LemmatizationRuleType { prefix, suffix }
@@ -18,7 +21,7 @@ class LemmatizationRule {
   final String name;
   final AllomorphPattern pattern;
   final WordClass wordClass;
-  final List<WordClass>? validChildClasses;
+  final Set<WordClass>? validChildClasses;
   final bool terminal;
 
   const LemmatizationRule({
@@ -38,9 +41,9 @@ class LemmatizationRule {
     required String pattern,
     required String? replacement,
     required WordClass wordClass,
-    validChildClasses,
-    terminal = false,
-    lookAheadBehind = const [''],
+    Set<WordClass>? validChildClasses,
+    bool terminal = false,
+    List<Pattern> lookAheadBehind = const [''],
     LemmatizationRuleType type = LemmatizationRuleType.suffix,
   }) : this(
          name: name,
@@ -55,6 +58,27 @@ class LemmatizationRule {
          terminal: terminal,
          wordClass: wordClass,
        );
+
+  @override
+  int get hashCode => Object.hash(
+    name,
+    pattern,
+    wordClass,
+    validChildClasses,
+    terminal,
+    SetEquality().hash(validChildClasses),
+  );
+
+  @override
+  bool operator ==(Object other) {
+    if (identical(this, other)) return true;
+    return other is LemmatizationRule &&
+        other.name == name &&
+        other.pattern == pattern &&
+        other.wordClass == wordClass &&
+        other.terminal == terminal &&
+        SetEquality().equals(validChildClasses, other.validChildClasses);
+  }
 }
 
 /// Represents a set of patterns for matching allomorphs in a word.
@@ -71,6 +95,7 @@ class AllomorphPattern {
     this.lookAheadBehind = const [''],
   });
 
+  /// Convert the [patterns] into regexes
   List<(String, Pattern)> get allPatternCombinations {
     final combinations = <(String, Pattern)>[];
     for (final l in lookAheadBehind) {
@@ -94,6 +119,7 @@ class AllomorphPattern {
     return combinations;
   }
 
+  /// Check whether an input string matches any of the [patterns]
   bool matches(String word) {
     for (final (_, p) in allPatternCombinations) {
       if (p is String) {
@@ -111,6 +137,9 @@ class AllomorphPattern {
     return false;
   }
 
+  /// Apply the replacement for this pattern.
+  ///
+  /// If none of the [patterns] apply, this function returns `null`.
   List<String>? apply(String word) {
     for (final (affix, p) in allPatternCombinations) {
       switch ((type, p is RegExp)) {
@@ -157,6 +186,22 @@ class AllomorphPattern {
     }
     return null;
   }
+
+  @override
+  int get hashCode => Object.hash(
+    type,
+    ListEquality().hash(lookAheadBehind),
+    MapEquality().hash(patterns),
+  );
+
+  @override
+  bool operator ==(Object other) {
+    if (identical(this, other)) return true;
+    return other is AllomorphPattern &&
+        other.type == type &&
+        ListEquality().equals(other.lookAheadBehind, lookAheadBehind) &&
+        MapEquality().equals(other.patterns, patterns);
+  }
 }
 
 class Lemmatized {
@@ -203,9 +248,10 @@ List<Lemmatized> _lemmatize(LemmatizationRule parentRule, String word) {
 
   final filteredLemmatizationRules = parentRule.validChildClasses == null
       ? lemmatizationRules
-      : lemmatizationRules.where(
-          (r) => parentRule.validChildClasses!.contains(r.wordClass),
-        );
+      : [
+          for (final wordClass in parentRule.validChildClasses!)
+            ...lemmatizationRulesByWordClass[wordClass]!,
+        ];
 
   for (final rule in filteredLemmatizationRules) {
     if (rule.matches(word)) {

lib/util/lemmatizer/rules.dart

@@ -1,10 +1,17 @@
 import 'package:jadb/util/lemmatizer/lemmatizer.dart';
-import 'package:jadb/util/lemmatizer/rules/godan-verbs.dart';
-import 'package:jadb/util/lemmatizer/rules/i-adjectives.dart';
-import 'package:jadb/util/lemmatizer/rules/ichidan-verbs.dart';
+import 'package:jadb/util/lemmatizer/rules/godan_verbs.dart';
+import 'package:jadb/util/lemmatizer/rules/i_adjectives.dart';
+import 'package:jadb/util/lemmatizer/rules/ichidan_verbs.dart';
 
-List<LemmatizationRule> lemmatizationRules = [
+final List<LemmatizationRule> lemmatizationRules = List.unmodifiable([
   ...ichidanVerbLemmatizationRules,
   ...godanVerbLemmatizationRules,
   ...iAdjectiveLemmatizationRules,
-];
+]);
+
+final Map<WordClass, List<LemmatizationRule>> lemmatizationRulesByWordClass =
+    Map.unmodifiable({
+      WordClass.ichidanVerb: ichidanVerbLemmatizationRules,
+      WordClass.iAdjective: iAdjectiveLemmatizationRules,
+      WordClass.godanVerb: godanVerbLemmatizationRules,
+    });

lib/util/lemmatizer/rules/godan-verbs.dart

@@ -1,457 +0,0 @@
-import 'package:jadb/util/lemmatizer/lemmatizer.dart';
-
-List<LemmatizationRule> godanVerbLemmatizationRules = [
-  LemmatizationRule(
-    name: 'Godan verb - base form',
-    terminal: true,
-    pattern: AllomorphPattern(
-      patterns: {
-        'う': ['う'],
-        'く': ['く'],
-        'ぐ': ['ぐ'],
-        'す': ['す'],
-        'つ': ['つ'],
-        'ぬ': ['ぬ'],
-        'ぶ': ['ぶ'],
-        'む': ['む'],
-        'る': ['る'],
-      },
-      type: LemmatizationRuleType.suffix,
-    ),
-    validChildClasses: [WordClass.godanVerb],
-    wordClass: WordClass.godanVerb,
-  ),
-  LemmatizationRule(
-    name: 'Godan verb - negative form',
-    pattern: AllomorphPattern(
-      patterns: {
-        'わない': ['う'],
-        'かない': ['く'],
-        'がない': ['ぐ'],
-        'さない': ['す'],
-        'たない': ['つ'],
-        'なない': ['ぬ'],
-        'ばない': ['ぶ'],
-        'まない': ['む'],
-        'らない': ['る'],
-      },
-      type: LemmatizationRuleType.suffix,
-    ),
-    validChildClasses: [WordClass.godanVerb],
-    wordClass: WordClass.godanVerb,
-  ),
-  LemmatizationRule(
-    name: 'Godan verb - past form',
-    pattern: AllomorphPattern(
-      patterns: {
-        'した': ['す'],
-        'った': ['る', 'つ', 'う'],
-        'んだ': ['む', 'ぬ', 'ぶ'],
-        'いだ': ['ぐ'],
-        'いた': ['く'],
-      },
-      type: LemmatizationRuleType.suffix,
-    ),
-    validChildClasses: [WordClass.godanVerb],
-    wordClass: WordClass.godanVerb,
-  ),
-  LemmatizationRule(
-    name: 'Godan verb - te-form',
-    pattern: AllomorphPattern(
-      patterns: {
-        'いて': ['く', 'ぐ'],
-        'して': ['す'],
-        'って': ['る', 'つ', 'う'],
-        'んで': ['む', 'ぬ', 'ぶ'],
-      },
-      type: LemmatizationRuleType.suffix,
-    ),
-    validChildClasses: [WordClass.godanVerb],
-    wordClass: WordClass.godanVerb,
-  ),
-  LemmatizationRule(
-    name: 'Godan verb - te-form with いる',
-    pattern: AllomorphPattern(
-      patterns: {
-        'いている': ['く', 'ぐ'],
-        'している': ['す'],
-        'っている': ['る', 'つ', 'う'],
-        'んでいる': ['む', 'ぬ', 'ぶ'],
-      },
-      type: LemmatizationRuleType.suffix,
-    ),
-    validChildClasses: [WordClass.godanVerb],
-    wordClass: WordClass.godanVerb,
-  ),
-  LemmatizationRule(
-    name: 'Godan verb - te-form with いた',
-    pattern: AllomorphPattern(
-      patterns: {
-        'いていた': ['く', 'ぐ'],
-        'していた': ['す'],
-        'っていた': ['る', 'つ', 'う'],
-        'んでいた': ['む', 'ぬ', 'ぶ'],
-      },
-      type: LemmatizationRuleType.suffix,
-    ),
-    validChildClasses: [WordClass.godanVerb],
-    wordClass: WordClass.godanVerb,
-  ),
-  LemmatizationRule(
-    name: 'Godan verb - conditional form',
-    pattern: AllomorphPattern(
-      patterns: {
-        'けば': ['く'],
-        'げば': ['ぐ'],
-        'せば': ['す'],
-        'てば': ['つ', 'る', 'う'],
-        'ねば': ['ぬ'],
-        'べば': ['ぶ'],
-        'めば': ['む'],
-      },
-      type: LemmatizationRuleType.suffix,
-    ),
-    validChildClasses: [WordClass.godanVerb],
-    wordClass: WordClass.godanVerb,
-  ),
-  LemmatizationRule(
-    name: 'Godan verb - volitional form',
-    pattern: AllomorphPattern(
-      patterns: {
-        'おう': ['う'],
-        'こう': ['く'],
-        'ごう': ['ぐ'],
-        'そう': ['す'],
-        'とう': ['つ', 'る', 'う'],
-        'のう': ['ぬ'],
-        'ぼう': ['ぶ'],
-        'もう': ['む'],
-      },
-      type: LemmatizationRuleType.suffix,
-    ),
-    validChildClasses: [WordClass.godanVerb],
-    wordClass: WordClass.godanVerb,
-  ),
-  LemmatizationRule(
-    name: 'Godan verb - potential form',
-    pattern: AllomorphPattern(
-      patterns: {
-        'ける': ['く'],
-        'げる': ['ぐ'],
-        'せる': ['す'],
-        'てる': ['つ', 'る', 'う'],
-        'ねる': ['ぬ'],
-        'べる': ['ぶ'],
-        'める': ['む'],
-      },
-      type: LemmatizationRuleType.suffix,
-    ),
-    validChildClasses: [WordClass.godanVerb],
-    wordClass: WordClass.godanVerb,
-  ),
-  LemmatizationRule(
-    name: 'Godan verb - passive form',
-    pattern: AllomorphPattern(
-      patterns: {
-        'かれる': ['く'],
-        'がれる': ['ぐ'],
-        'される': ['す'],
-        'たれる': ['つ', 'る', 'う'],
-        'なれる': ['ぬ'],
-        'ばれる': ['ぶ'],
-        'まれる': ['む'],
-      },
-      type: LemmatizationRuleType.suffix,
-    ),
-    validChildClasses: [WordClass.godanVerb],
-    wordClass: WordClass.godanVerb,
-  ),
-  LemmatizationRule(
-    name: 'Godan verb - causative form',
-    pattern: AllomorphPattern(
-      patterns: {
-        'かせる': ['く'],
-        'がせる': ['ぐ'],
-        'させる': ['す'],
-        'たせる': ['つ', 'る', 'う'],
-        'なせる': ['ぬ'],
-        'ばせる': ['ぶ'],
-        'ませる': ['む'],
-      },
-      type: LemmatizationRuleType.suffix,
-    ),
-    validChildClasses: [WordClass.godanVerb],
-    wordClass: WordClass.godanVerb,
-  ),
-  LemmatizationRule(
-    name: 'Godan verb - causative-passive form',
-    pattern: AllomorphPattern(
-      patterns: {
-        'かされる': ['く'],
-        'がされる': ['ぐ'],
-        'される': ['す'],
-        'たされる': ['つ', 'る', 'う'],
-        'なされる': ['ぬ'],
-        'ばされる': ['ぶ'],
-        'まされる': ['む'],
-      },
-      type: LemmatizationRuleType.suffix,
-    ),
-    validChildClasses: [WordClass.godanVerb],
-    wordClass: WordClass.godanVerb,
-  ),
-  LemmatizationRule(
-    name: 'Godan verb - imperative form',
-    pattern: AllomorphPattern(
-      patterns: {
-        'え': ['う'],
-        'け': ['く'],
-        'げ': ['ぐ'],
-        'せ': ['す'],
-        'て': ['つ', 'る', 'う'],
-        'ね': ['ぬ'],
-        'べ': ['ぶ'],
-        'め': ['む'],
-      },
-      type: LemmatizationRuleType.suffix,
-    ),
-    validChildClasses: [WordClass.godanVerb],
-    wordClass: WordClass.godanVerb,
-  ),
-  LemmatizationRule(
-    name: 'Godan verb - negative past form',
-    pattern: AllomorphPattern(
-      patterns: {
-        'わなかった': ['う'],
-        'かなかった': ['く'],
-        'がなかった': ['ぐ'],
-        'さなかった': ['す'],
-        'たなかった': ['つ'],
-        'ななかった': ['ぬ'],
-        'ばなかった': ['ぶ'],
-        'まなかった': ['む'],
-        'らなかった': ['る'],
-      },
-      type: LemmatizationRuleType.suffix,
-    ),
-    validChildClasses: [WordClass.godanVerb],
-    wordClass: WordClass.godanVerb,
-  ),
-  LemmatizationRule(
-    name: 'Godan verb - negative te-form',
-    pattern: AllomorphPattern(
-      patterns: {
-        'わなくて': ['う'],
-        'かなくて': ['く'],
-        'がなくて': ['ぐ'],
-        'さなくて': ['す'],
-        'たなくて': ['つ'],
-        'ななくて': ['ぬ'],
-        'ばなくて': ['ぶ'],
-        'まなくて': ['む'],
-        'らなくて': ['る'],
-      },
-      type: LemmatizationRuleType.suffix,
-    ),
-    validChildClasses: [WordClass.godanVerb],
-    wordClass: WordClass.godanVerb,
-  ),
-  LemmatizationRule(
-    name: 'Godan verb - negative conditional form',
-    pattern: AllomorphPattern(
-      patterns: {
-        'わなければ': ['う'],
-        'かなければ': ['く'],
-        'がなければ': ['ぐ'],
-        'さなければ': ['す'],
-        'たなければ': ['つ'],
-        'ななければ': ['ぬ'],
-        'ばなければ': ['ぶ'],
-        'まなければ': ['む'],
-        'らなければ': ['る'],
-      },
-      type: LemmatizationRuleType.suffix,
-    ),
-    validChildClasses: [WordClass.godanVerb],
-    wordClass: WordClass.godanVerb,
-  ),
-  LemmatizationRule(
-    name: 'Godan verb - negative volitional form',
-    pattern: AllomorphPattern(
-      patterns: {
-        'うまい': ['う'],
-        'くまい': ['く'],
-        'ぐまい': ['ぐ'],
-        'すまい': ['す'],
-        'つまい': ['つ', 'る', 'う'],
-        'ぬまい': ['ぬ'],
-        'ぶまい': ['ぶ'],
-        'むまい': ['む'],
-      },
-      type: LemmatizationRuleType.suffix,
-    ),
-    validChildClasses: [WordClass.godanVerb],
-    wordClass: WordClass.godanVerb,
-  ),
-  LemmatizationRule(
-    name: 'Godan verb - negative potential form',
-    pattern: AllomorphPattern(
-      patterns: {
-        'けない': ['く'],
-        'げない': ['ぐ'],
-        'せない': ['す'],
-        'てない': ['つ', 'る', 'う'],
-        'ねない': ['ぬ'],
-        'べない': ['ぶ'],
-        'めない': ['む'],
-      },
-      type: LemmatizationRuleType.suffix,
-    ),
-    validChildClasses: [WordClass.godanVerb],
-    wordClass: WordClass.godanVerb,
-  ),
-  LemmatizationRule(
-    name: 'Godan verb - negative passive form',
-    pattern: AllomorphPattern(
-      patterns: {
-        'かれない': ['く'],
-        'がれない': ['ぐ'],
-        'されない': ['す'],
-        'たれない': ['つ', 'る', 'う'],
-        'なれない': ['ぬ'],
-        'ばれない': ['ぶ'],
-        'まれない': ['む'],
-      },
-      type: LemmatizationRuleType.suffix,
-    ),
-    validChildClasses: [WordClass.godanVerb],
-    wordClass: WordClass.godanVerb,
-  ),
-  LemmatizationRule(
-    name: 'Godan verb - negative causative form',
-    pattern: AllomorphPattern(
-      patterns: {
-        'かせない': ['く'],
-        'がせない': ['ぐ'],
-        'させない': ['す'],
-        'たせない': ['つ', 'る', 'う'],
-        'なせない': ['ぬ'],
-        'ばせない': ['ぶ'],
-        'ませない': ['む'],
-      },
-      type: LemmatizationRuleType.suffix,
-    ),
-    validChildClasses: [WordClass.godanVerb],
-    wordClass: WordClass.godanVerb,
-  ),
-  LemmatizationRule(
-    name: 'Godan verb - negative causative-passive form',
-    pattern: AllomorphPattern(
-      patterns: {
-        'かされない': ['く'],
-        'がされない': ['ぐ'],
-        'されない': ['す'],
-        'たされない': ['つ', 'る', 'う'],
-        'なされない': ['ぬ'],
-        'ばされない': ['ぶ'],
-        'まされない': ['む'],
-      },
-      type: LemmatizationRuleType.suffix,
-    ),
-    validChildClasses: [WordClass.godanVerb],
-    wordClass: WordClass.godanVerb,
-  ),
-  LemmatizationRule(
-    name: 'Godan verb - negative imperative form',
-    pattern: AllomorphPattern(
-      patterns: {
-        'うな': ['う'],
-        'くな': ['く'],
-        'ぐな': ['ぐ'],
-        'すな': ['す'],
-        'つな': ['つ'],
-        'ぬな': ['ぬ'],
-        'ぶな': ['ぶ'],
-        'むな': ['む'],
-        'るな': ['る'],
-      },
-      type: LemmatizationRuleType.suffix,
-    ),
-    validChildClasses: [WordClass.godanVerb],
-    wordClass: WordClass.godanVerb,
-  ),
-  LemmatizationRule(
-    name: 'Godan verb - desire form',
-    pattern: AllomorphPattern(
-      patterns: {
-        'きたい': ['く'],
-        'ぎたい': ['ぐ'],
-        'したい': ['す'],
-        'ちたい': ['つ'],
-        'にたい': ['ぬ'],
-        'びたい': ['ぶ'],
-        'みたい': ['む'],
-        'りたい': ['る'],
-      },
-      type: LemmatizationRuleType.suffix,
-    ),
-    validChildClasses: [WordClass.godanVerb],
-    wordClass: WordClass.godanVerb,
-  ),
-  LemmatizationRule(
-    name: 'Godan verb - negative desire form',
-    pattern: AllomorphPattern(
-      patterns: {
-        'いたくない': ['う'],
-        'きたくない': ['く'],
-        'ぎたくない': ['ぐ'],
-        'したくない': ['す'],
-        'ちたくない': ['つ'],
-        'にたくない': ['ぬ'],
-        'びたくない': ['ぶ'],
-        'みたくない': ['む'],
-        'りたくない': ['る'],
-      },
-      type: LemmatizationRuleType.suffix,
-    ),
-    validChildClasses: [WordClass.godanVerb],
-    wordClass: WordClass.godanVerb,
-  ),
-  LemmatizationRule(
-    name: 'Godan verb - past desire form',
-    pattern: AllomorphPattern(
-      patterns: {
-        'きたかった': ['く'],
-        'ぎたかった': ['ぐ'],
-        'したかった': ['す'],
-        'ちたかった': ['つ'],
-        'にたかった': ['ぬ'],
-        'びたかった': ['ぶ'],
-        'みたかった': ['む'],
-        'りたかった': ['る'],
-      },
-      type: LemmatizationRuleType.suffix,
-    ),
-    validChildClasses: [WordClass.godanVerb],
-    wordClass: WordClass.godanVerb,
-  ),
-  LemmatizationRule(
-    name: 'Godan verb - negative past desire form',
-    pattern: AllomorphPattern(
-      patterns: {
-        'いたくなかった': ['う'],
-        'きたくなかった': ['く'],
-        'ぎたくなかった': ['ぐ'],
-        'したくなかった': ['す'],
-        'ちたくなかった': ['つ'],
-        'にたくなかった': ['ぬ'],
-        'びたくなかった': ['ぶ'],
-        'みたくなかった': ['む'],
-        'りたくなかった': ['る'],
-      },
-      type: LemmatizationRuleType.suffix,
-    ),
-    validChildClasses: [WordClass.godanVerb],
-    wordClass: WordClass.godanVerb,
-  ),
-];

lib/util/lemmatizer/rules/godan_verbs.dart

@@ -0,0 +1,509 @@
+import 'package:jadb/util/lemmatizer/lemmatizer.dart';
+
+final LemmatizationRule godanVerbBase = LemmatizationRule(
+  name: 'Godan verb - base form',
+  terminal: true,
+  pattern: AllomorphPattern(
+    patterns: {
+      'う': ['う'],
+      'く': ['く'],
+      'ぐ': ['ぐ'],
+      'す': ['す'],
+      'つ': ['つ'],
+      'ぬ': ['ぬ'],
+      'ぶ': ['ぶ'],
+      'む': ['む'],
+      'る': ['る'],
+    },
+    type: LemmatizationRuleType.suffix,
+  ),
+  validChildClasses: {WordClass.godanVerb},
+  wordClass: WordClass.godanVerb,
+);
+
+final LemmatizationRule godanVerbNegative = LemmatizationRule(
+  name: 'Godan verb - negative form',
+  pattern: AllomorphPattern(
+    patterns: {
+      'わない': ['う'],
+      'かない': ['く'],
+      'がない': ['ぐ'],
+      'さない': ['す'],
+      'たない': ['つ'],
+      'なない': ['ぬ'],
+      'ばない': ['ぶ'],
+      'まない': ['む'],
+      'らない': ['る'],
+    },
+    type: LemmatizationRuleType.suffix,
+  ),
+  validChildClasses: {WordClass.godanVerb},
+  wordClass: WordClass.godanVerb,
+);
+
+final LemmatizationRule godanVerbPast = LemmatizationRule(
+  name: 'Godan verb - past form',
+  pattern: AllomorphPattern(
+    patterns: {
+      'した': ['す'],
+      'った': ['る', 'つ', 'う'],
+      'んだ': ['む', 'ぬ', 'ぶ'],
+      'いだ': ['ぐ'],
+      'いた': ['く'],
+    },
+    type: LemmatizationRuleType.suffix,
+  ),
+  validChildClasses: {WordClass.godanVerb},
+  wordClass: WordClass.godanVerb,
+);
+
+final LemmatizationRule godanVerbTe = LemmatizationRule(
+  name: 'Godan verb - te-form',
+  pattern: AllomorphPattern(
+    patterns: {
+      'いて': ['く', 'ぐ'],
+      'して': ['す'],
+      'って': ['る', 'つ', 'う'],
+      'んで': ['む', 'ぬ', 'ぶ'],
+    },
+    type: LemmatizationRuleType.suffix,
+  ),
+  validChildClasses: {WordClass.godanVerb},
+  wordClass: WordClass.godanVerb,
+);
+
+final LemmatizationRule godanVerbTeiru = LemmatizationRule(
+  name: 'Godan verb - te-form with いる',
+  pattern: AllomorphPattern(
+    patterns: {
+      'いている': ['く', 'ぐ'],
+      'している': ['す'],
+      'っている': ['る', 'つ', 'う'],
+      'んでいる': ['む', 'ぬ', 'ぶ'],
+    },
+    type: LemmatizationRuleType.suffix,
+  ),
+  validChildClasses: {WordClass.godanVerb},
+  wordClass: WordClass.godanVerb,
+);
+
+final LemmatizationRule godanVerbTeita = LemmatizationRule(
+  name: 'Godan verb - te-form with いた',
+  pattern: AllomorphPattern(
+    patterns: {
+      'いていた': ['く', 'ぐ'],
+      'していた': ['す'],
+      'っていた': ['る', 'つ', 'う'],
+      'んでいた': ['む', 'ぬ', 'ぶ'],
+    },
+    type: LemmatizationRuleType.suffix,
+  ),
+  validChildClasses: {WordClass.godanVerb},
+  wordClass: WordClass.godanVerb,
+);
+
+final LemmatizationRule godanVerbConditional = LemmatizationRule(
+  name: 'Godan verb - conditional form',
+  pattern: AllomorphPattern(
+    patterns: {
+      'けば': ['く'],
+      'げば': ['ぐ'],
+      'せば': ['す'],
+      'てば': ['つ', 'る', 'う'],
+      'ねば': ['ぬ'],
+      'べば': ['ぶ'],
+      'めば': ['む'],
+    },
+    type: LemmatizationRuleType.suffix,
+  ),
+  validChildClasses: {WordClass.godanVerb},
+  wordClass: WordClass.godanVerb,
+);
+
+final LemmatizationRule godanVerbVolitional = LemmatizationRule(
+  name: 'Godan verb - volitional form',
+  pattern: AllomorphPattern(
+    patterns: {
+      'おう': ['う'],
+      'こう': ['く'],
+      'ごう': ['ぐ'],
+      'そう': ['す'],
+      'とう': ['つ', 'る', 'う'],
+      'のう': ['ぬ'],
+      'ぼう': ['ぶ'],
+      'もう': ['む'],
+    },
+    type: LemmatizationRuleType.suffix,
+  ),
+  validChildClasses: {WordClass.godanVerb},
+  wordClass: WordClass.godanVerb,
+);
+
+final LemmatizationRule godanVerbPotential = LemmatizationRule(
+  name: 'Godan verb - potential form',
+  pattern: AllomorphPattern(
+    patterns: {
+      'ける': ['く'],
+      'げる': ['ぐ'],
+      'せる': ['す'],
+      'てる': ['つ', 'る', 'う'],
+      'ねる': ['ぬ'],
+      'べる': ['ぶ'],
+      'める': ['む'],
+    },
+    type: LemmatizationRuleType.suffix,
+  ),
+  validChildClasses: {WordClass.godanVerb},
+  wordClass: WordClass.godanVerb,
+);
+
+final LemmatizationRule godanVerbPassive = LemmatizationRule(
+  name: 'Godan verb - passive form',
+  pattern: AllomorphPattern(
+    patterns: {
+      'かれる': ['く'],
+      'がれる': ['ぐ'],
+      'される': ['す'],
+      'たれる': ['つ', 'る', 'う'],
+      'なれる': ['ぬ'],
+      'ばれる': ['ぶ'],
+      'まれる': ['む'],
+    },
+    type: LemmatizationRuleType.suffix,
+  ),
+  validChildClasses: {WordClass.godanVerb},
+  wordClass: WordClass.godanVerb,
+);
+
+final LemmatizationRule godanVerbCausative = LemmatizationRule(
+  name: 'Godan verb - causative form',
+  pattern: AllomorphPattern(
+    patterns: {
+      'かせる': ['く'],
+      'がせる': ['ぐ'],
+      'させる': ['す'],
+      'たせる': ['つ', 'る', 'う'],
+      'なせる': ['ぬ'],
+      'ばせる': ['ぶ'],
+      'ませる': ['む'],
+    },
+    type: LemmatizationRuleType.suffix,
+  ),
+  validChildClasses: {WordClass.godanVerb},
+  wordClass: WordClass.godanVerb,
+);
+
+final LemmatizationRule godanVerbCausativePassive = LemmatizationRule(
+  name: 'Godan verb - causative-passive form',
+  pattern: AllomorphPattern(
+    patterns: {
+      'かされる': ['く'],
+      'がされる': ['ぐ'],
+      'される': ['す'],
+      'たされる': ['つ', 'る', 'う'],
+      'なされる': ['ぬ'],
+      'ばされる': ['ぶ'],
+      'まされる': ['む'],
+    },
+    type: LemmatizationRuleType.suffix,
+  ),
+  validChildClasses: {WordClass.godanVerb},
+  wordClass: WordClass.godanVerb,
+);
+
+final LemmatizationRule godanVerbImperative = LemmatizationRule(
+  name: 'Godan verb - imperative form',
+  pattern: AllomorphPattern(
+    patterns: {
+      'え': ['う'],
+      'け': ['く'],
+      'げ': ['ぐ'],
+      'せ': ['す'],
+      'て': ['つ', 'る', 'う'],
+      'ね': ['ぬ'],
+      'べ': ['ぶ'],
+      'め': ['む'],
+    },
+    type: LemmatizationRuleType.suffix,
+  ),
+  validChildClasses: {WordClass.godanVerb},
+  wordClass: WordClass.godanVerb,
+);
+
+final LemmatizationRule godanVerbNegativePast = LemmatizationRule(
+  name: 'Godan verb - negative past form',
+  pattern: AllomorphPattern(
+    patterns: {
+      'わなかった': ['う'],
+      'かなかった': ['く'],
+      'がなかった': ['ぐ'],
+      'さなかった': ['す'],
+      'たなかった': ['つ'],
+      'ななかった': ['ぬ'],
+      'ばなかった': ['ぶ'],
+      'まなかった': ['む'],
+      'らなかった': ['る'],
+    },
+    type: LemmatizationRuleType.suffix,
+  ),
+  validChildClasses: {WordClass.godanVerb},
+  wordClass: WordClass.godanVerb,
+);
+
+final LemmatizationRule godanVerbNegativeTe = LemmatizationRule(
+  name: 'Godan verb - negative te-form',
+  pattern: AllomorphPattern(
+    patterns: {
+      'わなくて': ['う'],
+      'かなくて': ['く'],
+      'がなくて': ['ぐ'],
+      'さなくて': ['す'],
+      'たなくて': ['つ'],
+      'ななくて': ['ぬ'],
+      'ばなくて': ['ぶ'],
+      'まなくて': ['む'],
+      'らなくて': ['る'],
+    },
+    type: LemmatizationRuleType.suffix,
+  ),
+  validChildClasses: {WordClass.godanVerb},
+  wordClass: WordClass.godanVerb,
+);
+
+final LemmatizationRule godanVerbNegativeConditional = LemmatizationRule(
+  name: 'Godan verb - negative conditional form',
+  pattern: AllomorphPattern(
+    patterns: {
+      'わなければ': ['う'],
+      'かなければ': ['く'],
+      'がなければ': ['ぐ'],
+      'さなければ': ['す'],
+      'たなければ': ['つ'],
+      'ななければ': ['ぬ'],
+      'ばなければ': ['ぶ'],
+      'まなければ': ['む'],
+      'らなければ': ['る'],
+    },
+    type: LemmatizationRuleType.suffix,
+  ),
+  validChildClasses: {WordClass.godanVerb},
+  wordClass: WordClass.godanVerb,
+);
+
+final LemmatizationRule godanVerbNegativeVolitional = LemmatizationRule(
+  name: 'Godan verb - negative volitional form',
+  pattern: AllomorphPattern(
+    patterns: {
+      'うまい': ['う'],
+      'くまい': ['く'],
+      'ぐまい': ['ぐ'],
+      'すまい': ['す'],
+      'つまい': ['つ', 'る', 'う'],
+      'ぬまい': ['ぬ'],
+      'ぶまい': ['ぶ'],
+      'むまい': ['む'],
+    },
+    type: LemmatizationRuleType.suffix,
+  ),
+  validChildClasses: {WordClass.godanVerb},
+  wordClass: WordClass.godanVerb,
+);
+
+final LemmatizationRule godanVerbNegativePotential = LemmatizationRule(
+  name: 'Godan verb - negative potential form',
+  pattern: AllomorphPattern(
+    patterns: {
+      'けない': ['く'],
+      'げない': ['ぐ'],
+      'せない': ['す'],
+      'てない': ['つ', 'る', 'う'],
+      'ねない': ['ぬ'],
+      'べない': ['ぶ'],
+      'めない': ['む'],
+    },
+    type: LemmatizationRuleType.suffix,
+  ),
+  validChildClasses: {WordClass.godanVerb},
+  wordClass: WordClass.godanVerb,
+);
+
+final LemmatizationRule godanVerbNegativePassive = LemmatizationRule(
+  name: 'Godan verb - negative passive form',
+  pattern: AllomorphPattern(
+    patterns: {
+      'かれない': ['く'],
+      'がれない': ['ぐ'],
+      'されない': ['す'],
+      'たれない': ['つ', 'る', 'う'],
+      'なれない': ['ぬ'],
+      'ばれない': ['ぶ'],
+      'まれない': ['む'],
+    },
+    type: LemmatizationRuleType.suffix,
+  ),
+  validChildClasses: {WordClass.godanVerb},
+  wordClass: WordClass.godanVerb,
+);
+
+final LemmatizationRule godanVerbNegativeCausative = LemmatizationRule(
+  name: 'Godan verb - negative causative form',
+  pattern: AllomorphPattern(
+    patterns: {
+      'かせない': ['く'],
+      'がせない': ['ぐ'],
+      'させない': ['す'],
+      'たせない': ['つ', 'る', 'う'],
+      'なせない': ['ぬ'],
+      'ばせない': ['ぶ'],
+      'ませない': ['む'],
+    },
+    type: LemmatizationRuleType.suffix,
+  ),
+  validChildClasses: {WordClass.godanVerb},
+  wordClass: WordClass.godanVerb,
+);
+
+final LemmatizationRule godanVerbNegativeCausativePassive = LemmatizationRule(
+  name: 'Godan verb - negative causative-passive form',
+  pattern: AllomorphPattern(
+    patterns: {
+      'かされない': ['く'],
+      'がされない': ['ぐ'],
+      'されない': ['す'],
+      'たされない': ['つ', 'る', 'う'],
+      'なされない': ['ぬ'],
+      'ばされない': ['ぶ'],
+      'まされない': ['む'],
+    },
+    type: LemmatizationRuleType.suffix,
+  ),
+  validChildClasses: {WordClass.godanVerb},
+  wordClass: WordClass.godanVerb,
+);
+
+final LemmatizationRule godanVerbNegativeImperative = LemmatizationRule(
+  name: 'Godan verb - negative imperative form',
+  pattern: AllomorphPattern(
+    patterns: {
+      'うな': ['う'],
+      'くな': ['く'],
+      'ぐな': ['ぐ'],
+      'すな': ['す'],
+      'つな': ['つ'],
+      'ぬな': ['ぬ'],
+      'ぶな': ['ぶ'],
+      'むな': ['む'],
+      'るな': ['る'],
+    },
+    type: LemmatizationRuleType.suffix,
+  ),
+  validChildClasses: {WordClass.godanVerb},
+  wordClass: WordClass.godanVerb,
+);
+
+final LemmatizationRule godanVerbDesire = LemmatizationRule(
+  name: 'Godan verb - desire form',
+  pattern: AllomorphPattern(
+    patterns: {
+      'きたい': ['く'],
+      'ぎたい': ['ぐ'],
+      'したい': ['す'],
+      'ちたい': ['つ'],
+      'にたい': ['ぬ'],
+      'びたい': ['ぶ'],
+      'みたい': ['む'],
+      'りたい': ['る'],
+    },
+    type: LemmatizationRuleType.suffix,
+  ),
+  validChildClasses: {WordClass.godanVerb},
+  wordClass: WordClass.godanVerb,
+);
+
+final LemmatizationRule godanVerbNegativeDesire = LemmatizationRule(
+  name: 'Godan verb - negative desire form',
+  pattern: AllomorphPattern(
+    patterns: {
+      'いたくない': ['う'],
+      'きたくない': ['く'],
+      'ぎたくない': ['ぐ'],
+      'したくない': ['す'],
+      'ちたくない': ['つ'],
+      'にたくない': ['ぬ'],
+      'びたくない': ['ぶ'],
+      'みたくない': ['む'],
+      'りたくない': ['る'],
+    },
+    type: LemmatizationRuleType.suffix,
+  ),
+  validChildClasses: {WordClass.godanVerb},
+  wordClass: WordClass.godanVerb,
+);
+
+final LemmatizationRule godanVerbPastDesire = LemmatizationRule(
+  name: 'Godan verb - past desire form',
+  pattern: AllomorphPattern(
+    patterns: {
+      'きたかった': ['く'],
+      'ぎたかった': ['ぐ'],
+      'したかった': ['す'],
+      'ちたかった': ['つ'],
+      'にたかった': ['ぬ'],
+      'びたかった': ['ぶ'],
+      'みたかった': ['む'],
+      'りたかった': ['る'],
+    },
+    type: LemmatizationRuleType.suffix,
+  ),
+  validChildClasses: {WordClass.godanVerb},
+  wordClass: WordClass.godanVerb,
+);
+
+final LemmatizationRule godanVerbNegativePastDesire = LemmatizationRule(
+  name: 'Godan verb - negative past desire form',
+  pattern: AllomorphPattern(
+    patterns: {
+      'いたくなかった': ['う'],
+      'きたくなかった': ['く'],
+      'ぎたくなかった': ['ぐ'],
+      'したくなかった': ['す'],
+      'ちたくなかった': ['つ'],
+      'にたくなかった': ['ぬ'],
+      'びたくなかった': ['ぶ'],
+      'みたくなかった': ['む'],
+      'りたくなかった': ['る'],
+    },
+    type: LemmatizationRuleType.suffix,
+  ),
+  validChildClasses: {WordClass.godanVerb},
+  wordClass: WordClass.godanVerb,
+);
+
+final List<LemmatizationRule> godanVerbLemmatizationRules = List.unmodifiable([
+  godanVerbBase,
+  godanVerbNegative,
+  godanVerbPast,
+  godanVerbTe,
+  godanVerbTeiru,
+  godanVerbTeita,
+  godanVerbConditional,
+  godanVerbVolitional,
+  godanVerbPotential,
+  godanVerbPassive,
+  godanVerbCausative,
+  godanVerbCausativePassive,
+  godanVerbImperative,
+  godanVerbNegativePast,
+  godanVerbNegativeTe,
+  godanVerbNegativeConditional,
+  godanVerbNegativeVolitional,
+  godanVerbNegativePotential,
+  godanVerbNegativePassive,
+  godanVerbNegativeCausative,
+  godanVerbNegativeCausativePassive,
+  godanVerbNegativeImperative,
+  godanVerbDesire,
+  godanVerbNegativeDesire,
+  godanVerbPastDesire,
+  godanVerbNegativePastDesire,
+]);

lib/util/lemmatizer/rules/i-adjectives.dart

@@ -1,61 +0,0 @@
-import 'package:jadb/util/lemmatizer/lemmatizer.dart';
-
-List<LemmatizationRule> iAdjectiveLemmatizationRules = [
-  LemmatizationRule.simple(
-    name: 'I adjective - base form',
-    terminal: true,
-    pattern: 'い',
-    replacement: 'い',
-    validChildClasses: [WordClass.iAdjective],
-    wordClass: WordClass.iAdjective,
-  ),
-  LemmatizationRule.simple(
-    name: 'I adjective - negative form',
-    pattern: 'くない',
-    replacement: 'い',
-    validChildClasses: [WordClass.iAdjective],
-    wordClass: WordClass.iAdjective,
-  ),
-  LemmatizationRule.simple(
-    name: 'I adjective - past form',
-    pattern: 'かった',
-    replacement: 'い',
-    validChildClasses: [WordClass.iAdjective],
-    wordClass: WordClass.iAdjective,
-  ),
-  LemmatizationRule.simple(
-    name: 'I adjective - negative past form',
-    pattern: 'くなかった',
-    replacement: 'い',
-    validChildClasses: [WordClass.iAdjective],
-    wordClass: WordClass.iAdjective,
-  ),
-  LemmatizationRule.simple(
-    name: 'I adjective - te-form',
-    pattern: 'くて',
-    replacement: 'い',
-    validChildClasses: [WordClass.iAdjective],
-    wordClass: WordClass.iAdjective,
-  ),
-  LemmatizationRule.simple(
-    name: 'I adjective - conditional form',
-    pattern: 'ければ',
-    replacement: 'い',
-    validChildClasses: [WordClass.iAdjective],
-    wordClass: WordClass.iAdjective,
-  ),
-  LemmatizationRule.simple(
-    name: 'I adjective - volitional form',
-    pattern: 'かろう',
-    replacement: 'い',
-    validChildClasses: [WordClass.iAdjective],
-    wordClass: WordClass.iAdjective,
-  ),
-  LemmatizationRule.simple(
-    name: 'I adjective - continuative form',
-    pattern: 'く',
-    replacement: 'い',
-    validChildClasses: [WordClass.iAdjective],
-    wordClass: WordClass.iAdjective,
-  ),
-];

lib/util/lemmatizer/rules/i_adjectives.dart

@@ -0,0 +1,77 @@
+import 'package:jadb/util/lemmatizer/lemmatizer.dart';
+
+final LemmatizationRule iAdjectiveBase = LemmatizationRule.simple(
+  name: 'I adjective - base form',
+  terminal: true,
+  pattern: 'い',
+  replacement: 'い',
+  validChildClasses: {WordClass.iAdjective},
+  wordClass: WordClass.iAdjective,
+);
+
+final LemmatizationRule iAdjectiveNegative = LemmatizationRule.simple(
+  name: 'I adjective - negative form',
+  pattern: 'くない',
+  replacement: 'い',
+  validChildClasses: {WordClass.iAdjective},
+  wordClass: WordClass.iAdjective,
+);
+
+final LemmatizationRule iAdjectivePast = LemmatizationRule.simple(
+  name: 'I adjective - past form',
+  pattern: 'かった',
+  replacement: 'い',
+  validChildClasses: {WordClass.iAdjective},
+  wordClass: WordClass.iAdjective,
+);
+
+final LemmatizationRule iAdjectiveNegativePast = LemmatizationRule.simple(
+  name: 'I adjective - negative past form',
+  pattern: 'くなかった',
+  replacement: 'い',
+  validChildClasses: {WordClass.iAdjective},
+  wordClass: WordClass.iAdjective,
+);
+
+final LemmatizationRule iAdjectiveTe = LemmatizationRule.simple(
+  name: 'I adjective - te-form',
+  pattern: 'くて',
+  replacement: 'い',
+  validChildClasses: {WordClass.iAdjective},
+  wordClass: WordClass.iAdjective,
+);
+
+final LemmatizationRule iAdjectiveConditional = LemmatizationRule.simple(
+  name: 'I adjective - conditional form',
+  pattern: 'ければ',
+  replacement: 'い',
+  validChildClasses: {WordClass.iAdjective},
+  wordClass: WordClass.iAdjective,
+);
+
+final LemmatizationRule iAdjectiveVolitional = LemmatizationRule.simple(
+  name: 'I adjective - volitional form',
+  pattern: 'かろう',
+  replacement: 'い',
+  validChildClasses: {WordClass.iAdjective},
+  wordClass: WordClass.iAdjective,
+);
+
+final LemmatizationRule iAdjectiveContinuative = LemmatizationRule.simple(
+  name: 'I adjective - continuative form',
+  pattern: 'く',
+  replacement: 'い',
+  validChildClasses: {WordClass.iAdjective},
+  wordClass: WordClass.iAdjective,
+);
+
+final List<LemmatizationRule> iAdjectiveLemmatizationRules = List.unmodifiable([
+  iAdjectiveBase,
+  iAdjectiveNegative,
+  iAdjectivePast,
+  iAdjectiveNegativePast,
+  iAdjectiveTe,
+  iAdjectiveConditional,
+  iAdjectiveVolitional,
+  iAdjectiveContinuative,
+]);

lib/util/lemmatizer/rules/ichidan-verbs.dart

@@ -1,241 +0,0 @@
-import 'package:jadb/util/lemmatizer/lemmatizer.dart';
-import 'package:jadb/util/text_filtering.dart';
-
-List<Pattern> lookBehinds = [
-  kanjiRegex,
-  'き',
-  'ぎ',
-  'し',
-  'じ',
-  'ち',
-  'ぢ',
-  'に',
-  'ひ',
-  'び',
-  'び',
-  'み',
-  'り',
-  'け',
-  'げ',
-  'せ',
-  'ぜ',
-  'て',
-  'で',
-  'ね',
-  'へ',
-  'べ',
-  'め',
-  'れ',
-];
-
-List<LemmatizationRule> ichidanVerbLemmatizationRules = [
-  LemmatizationRule.simple(
-    name: 'Ichidan verb - base form',
-    terminal: true,
-    pattern: 'る',
-    replacement: 'る',
-    lookAheadBehind: lookBehinds,
-    validChildClasses: [WordClass.ichidanVerb],
-    wordClass: WordClass.ichidanVerb,
-  ),
-  LemmatizationRule.simple(
-    name: 'Ichidan verb - negative form',
-    pattern: 'ない',
-    replacement: 'る',
-    lookAheadBehind: lookBehinds,
-    validChildClasses: [WordClass.ichidanVerb],
-    wordClass: WordClass.ichidanVerb,
-  ),
-  LemmatizationRule.simple(
-    name: 'Ichidan verb - past form',
-    pattern: 'た',
-    replacement: 'る',
-    lookAheadBehind: lookBehinds,
-    validChildClasses: [WordClass.ichidanVerb],
-    wordClass: WordClass.ichidanVerb,
-  ),
-  LemmatizationRule.simple(
-    name: 'Ichidan verb - te-form',
-    pattern: 'て',
-    replacement: 'る',
-    lookAheadBehind: lookBehinds,
-    validChildClasses: [WordClass.ichidanVerb],
-    wordClass: WordClass.ichidanVerb,
-  ),
-  LemmatizationRule.simple(
-    name: 'Ichidan verb - te-form with いる',
-    pattern: 'ている',
-    replacement: 'る',
-    lookAheadBehind: lookBehinds,
-    validChildClasses: [WordClass.ichidanVerb],
-    wordClass: WordClass.ichidanVerb,
-  ),
-  LemmatizationRule.simple(
-    name: 'Ichidan verb - te-form with いた',
-    pattern: 'ていた',
-    replacement: 'る',
-    lookAheadBehind: lookBehinds,
-    validChildClasses: [WordClass.ichidanVerb],
-    wordClass: WordClass.ichidanVerb,
-  ),
-  LemmatizationRule.simple(
-    name: 'Ichidan verb - conditional form',
-    pattern: 'れば',
-    replacement: 'る',
-    lookAheadBehind: lookBehinds,
-    validChildClasses: [WordClass.ichidanVerb],
-    wordClass: WordClass.ichidanVerb,
-  ),
-  LemmatizationRule.simple(
-    name: 'Ichidan verb - volitional form',
-    pattern: 'よう',
-    replacement: 'る',
-    lookAheadBehind: lookBehinds,
-    validChildClasses: [WordClass.ichidanVerb],
-    wordClass: WordClass.ichidanVerb,
-  ),
-  LemmatizationRule.simple(
-    name: 'Ichidan verb - potential form',
-    pattern: 'られる',
-    replacement: 'る',
-    lookAheadBehind: lookBehinds,
-    validChildClasses: [WordClass.ichidanVerb],
-    wordClass: WordClass.ichidanVerb,
-  ),
-  LemmatizationRule.simple(
-    name: 'Ichidan verb - passive form',
-    pattern: 'られる',
-    replacement: 'る',
-    lookAheadBehind: lookBehinds,
-    validChildClasses: [WordClass.ichidanVerb],
-    wordClass: WordClass.ichidanVerb,
-  ),
-  LemmatizationRule.simple(
-    name: 'Ichidan verb - causative form',
-    pattern: 'させる',
-    replacement: 'る',
-    lookAheadBehind: lookBehinds,
-    validChildClasses: [WordClass.ichidanVerb],
-    wordClass: WordClass.ichidanVerb,
-  ),
-  LemmatizationRule.simple(
-    name: 'Ichidan verb - causative passive form',
-    pattern: 'させられる',
-    replacement: 'る',
-    lookAheadBehind: lookBehinds,
-    validChildClasses: [WordClass.ichidanVerb],
-    wordClass: WordClass.ichidanVerb,
-  ),
-  LemmatizationRule.simple(
-    name: 'Ichidan verb - imperative form',
-    pattern: 'れ',
-    replacement: 'る',
-    lookAheadBehind: lookBehinds,
-    validChildClasses: [WordClass.ichidanVerb],
-    wordClass: WordClass.ichidanVerb,
-  ),
-  LemmatizationRule.simple(
-    name: 'Ichidan verb - negative past form',
-    pattern: 'なかった',
-    replacement: 'る',
-    lookAheadBehind: lookBehinds,
-    validChildClasses: [WordClass.ichidanVerb],
-    wordClass: WordClass.ichidanVerb,
-  ),
-  LemmatizationRule.simple(
-    name: 'Ichidan verb - negative te-form',
-    pattern: 'なくて',
-    replacement: 'る',
-    lookAheadBehind: lookBehinds,
-    validChildClasses: [WordClass.ichidanVerb],
-    wordClass: WordClass.ichidanVerb,
-  ),
-  LemmatizationRule.simple(
-    name: 'Ichidan verb - negative conditional form',
-    pattern: 'なければ',
-    replacement: 'る',
-    lookAheadBehind: lookBehinds,
-    validChildClasses: [WordClass.ichidanVerb],
-    wordClass: WordClass.ichidanVerb,
-  ),
-  LemmatizationRule.simple(
-    name: 'Ichidan verb - negative volitional form',
-    pattern: 'なかろう',
-    replacement: 'る',
-    lookAheadBehind: lookBehinds,
-    validChildClasses: [WordClass.ichidanVerb],
-    wordClass: WordClass.ichidanVerb,
-  ),
-  LemmatizationRule.simple(
-    name: 'Ichidan verb - negative potential form',
-    pattern: 'られない',
-    replacement: 'る',
-    lookAheadBehind: lookBehinds,
-    validChildClasses: [WordClass.ichidanVerb],
-    wordClass: WordClass.ichidanVerb,
-  ),
-  LemmatizationRule.simple(
-    name: 'Ichidan verb - negative passive form',
-    pattern: 'られない',
-    replacement: 'る',
-    lookAheadBehind: lookBehinds,
-    validChildClasses: [WordClass.ichidanVerb],
-    wordClass: WordClass.ichidanVerb,
-  ),
-  LemmatizationRule.simple(
-    name: 'Ichidan verb - negative causative form',
-    pattern: 'させない',
-    replacement: 'る',
-    lookAheadBehind: lookBehinds,
-    validChildClasses: [WordClass.ichidanVerb],
-    wordClass: WordClass.ichidanVerb,
-  ),
-  LemmatizationRule.simple(
-    name: 'Ichidan verb - negative causative passive form',
-    pattern: 'させられない',
-    replacement: 'る',
-    lookAheadBehind: lookBehinds,
-    validChildClasses: [WordClass.ichidanVerb],
-    wordClass: WordClass.ichidanVerb,
-  ),
-  LemmatizationRule.simple(
-    name: 'Ichidan verb - negative imperative form',
-    pattern: 'るな',
-    replacement: 'る',
-    lookAheadBehind: lookBehinds,
-    validChildClasses: [WordClass.ichidanVerb],
-    wordClass: WordClass.ichidanVerb,
-  ),
-  LemmatizationRule.simple(
-    name: 'Ichidan verb - desire form',
-    pattern: 'たい',
-    replacement: 'る',
-    lookAheadBehind: lookBehinds,
-    validChildClasses: [WordClass.ichidanVerb],
-    wordClass: WordClass.ichidanVerb,
-  ),
-  LemmatizationRule.simple(
-    name: 'Ichidan verb - negative desire form',
-    pattern: 'たくない',
-    replacement: 'る',
-    lookAheadBehind: lookBehinds,
-    validChildClasses: [WordClass.ichidanVerb],
-    wordClass: WordClass.ichidanVerb,
-  ),
-  LemmatizationRule.simple(
-    name: 'Ichidan verb - past desire form',
-    pattern: 'たかった',
-    replacement: 'る',
-    lookAheadBehind: lookBehinds,
-    validChildClasses: [WordClass.ichidanVerb],
-    wordClass: WordClass.ichidanVerb,
-  ),
-  LemmatizationRule.simple(
-    name: 'Ichidan verb - negative past desire form',
-    pattern: 'たくなかった',
-    replacement: 'る',
-    lookAheadBehind: lookBehinds,
-    validChildClasses: [WordClass.ichidanVerb],
-    wordClass: WordClass.ichidanVerb,
-  ),
-];

lib/util/lemmatizer/rules/ichidan_verbs.dart

@@ -0,0 +1,331 @@
+import 'package:jadb/util/lemmatizer/lemmatizer.dart';
+import 'package:jadb/util/text_filtering.dart';
+
+final List<Pattern> _lookBehinds = [
+  kanjiRegex,
+  'き',
+  'ぎ',
+  'し',
+  'じ',
+  'ち',
+  'ぢ',
+  'に',
+  'ひ',
+  'び',
+  'び',
+  'み',
+  'り',
+  'け',
+  'げ',
+  'せ',
+  'ぜ',
+  'て',
+  'で',
+  'ね',
+  'へ',
+  'べ',
+  'め',
+  'れ',
+];
+
+final LemmatizationRule ichidanVerbBase = LemmatizationRule.simple(
+  name: 'Ichidan verb - base form',
+  terminal: true,
+  pattern: 'る',
+  replacement: 'る',
+  lookAheadBehind: _lookBehinds,
+  validChildClasses: {WordClass.ichidanVerb},
+  wordClass: WordClass.ichidanVerb,
+);
+
+final LemmatizationRule ichidanVerbNegative = LemmatizationRule.simple(
+  name: 'Ichidan verb - negative form',
+  pattern: 'ない',
+  replacement: 'る',
+  lookAheadBehind: _lookBehinds,
+  validChildClasses: {WordClass.ichidanVerb},
+  wordClass: WordClass.ichidanVerb,
+);
+final LemmatizationRule ichidanVerbPast = LemmatizationRule.simple(
+  name: 'Ichidan verb - past form',
+  pattern: 'た',
+  replacement: 'る',
+  lookAheadBehind: _lookBehinds,
+  validChildClasses: {WordClass.ichidanVerb},
+  wordClass: WordClass.ichidanVerb,
+);
+
+final LemmatizationRule ichidanVerbTe = LemmatizationRule.simple(
+  name: 'Ichidan verb - te-form',
+  pattern: 'て',
+  replacement: 'る',
+  lookAheadBehind: _lookBehinds,
+  validChildClasses: {WordClass.ichidanVerb},
+  wordClass: WordClass.ichidanVerb,
+);
+
+final LemmatizationRule ichidanVerbTeiru = LemmatizationRule.simple(
+  name: 'Ichidan verb - te-form with いる',
+  pattern: 'ている',
+  replacement: 'る',
+  lookAheadBehind: _lookBehinds,
+  validChildClasses: {WordClass.ichidanVerb},
+  wordClass: WordClass.ichidanVerb,
+);
+
+final LemmatizationRule ichidanVerbTeita = LemmatizationRule.simple(
+  name: 'Ichidan verb - te-form with いた',
+  pattern: 'ていた',
+  replacement: 'る',
+  lookAheadBehind: _lookBehinds,
+  validChildClasses: {WordClass.ichidanVerb},
+  wordClass: WordClass.ichidanVerb,
+);
+
+final LemmatizationRule ichidanVerbConditional = LemmatizationRule.simple(
+  name: 'Ichidan verb - conditional form',
+  pattern: 'れば',
+  replacement: 'る',
+  lookAheadBehind: _lookBehinds,
+  validChildClasses: {WordClass.ichidanVerb},
+  wordClass: WordClass.ichidanVerb,
+);
+
+final LemmatizationRule ichidanVerbVolitional = LemmatizationRule.simple(
+  name: 'Ichidan verb - volitional form',
+  pattern: 'よう',
+  replacement: 'る',
+  lookAheadBehind: _lookBehinds,
+  validChildClasses: {WordClass.ichidanVerb},
+  wordClass: WordClass.ichidanVerb,
+);
+
+final LemmatizationRule ichidanVerbPotential = LemmatizationRule.simple(
+  name: 'Ichidan verb - potential form',
+  pattern: 'られる',
+  replacement: 'る',
+  lookAheadBehind: _lookBehinds,
+  validChildClasses: {WordClass.ichidanVerb},
+  wordClass: WordClass.ichidanVerb,
+);
+
+final LemmatizationRule ichidanVerbPassive = LemmatizationRule.simple(
+  name: 'Ichidan verb - passive form',
+  pattern: 'られる',
+  replacement: 'る',
+  lookAheadBehind: _lookBehinds,
+  validChildClasses: {WordClass.ichidanVerb},
+  wordClass: WordClass.ichidanVerb,
+);
+
+final LemmatizationRule ichidanVerbCausative = LemmatizationRule.simple(
+  name: 'Ichidan verb - causative form',
+  pattern: 'させる',
+  replacement: 'る',
+  lookAheadBehind: _lookBehinds,
+  validChildClasses: {WordClass.ichidanVerb},
+  wordClass: WordClass.ichidanVerb,
+);
+
+final LemmatizationRule ichidanVerbCausativePassive = LemmatizationRule.simple(
+  name: 'Ichidan verb - causative passive form',
+  pattern: 'させられる',
+  replacement: 'る',
+  lookAheadBehind: _lookBehinds,
+  validChildClasses: {WordClass.ichidanVerb},
+  wordClass: WordClass.ichidanVerb,
+);
+
+final LemmatizationRule ichidanVerbImperative = LemmatizationRule.simple(
+  name: 'Ichidan verb - imperative form',
+  pattern: 'れ',
+  replacement: 'る',
+  lookAheadBehind: _lookBehinds,
+  validChildClasses: {WordClass.ichidanVerb},
+  wordClass: WordClass.ichidanVerb,
+);
+
+final LemmatizationRule ichidanVerbNegativePast = LemmatizationRule.simple(
+  name: 'Ichidan verb - negative past form',
+  pattern: 'なかった',
+  replacement: 'る',
+  lookAheadBehind: _lookBehinds,
+  validChildClasses: {WordClass.ichidanVerb},
+  wordClass: WordClass.ichidanVerb,
+);
+
+final LemmatizationRule ichidanVerbNegativeTe = LemmatizationRule.simple(
+  name: 'Ichidan verb - negative te-form',
+  pattern: 'なくて',
+  replacement: 'る',
+  lookAheadBehind: _lookBehinds,
+  validChildClasses: {WordClass.ichidanVerb},
+  wordClass: WordClass.ichidanVerb,
+);
+
+final LemmatizationRule ichidanVerbNegativeConditional =
+    LemmatizationRule.simple(
+      name: 'Ichidan verb - negative conditional form',
+      pattern: 'なければ',
+      replacement: 'る',
+      lookAheadBehind: _lookBehinds,
+      validChildClasses: {WordClass.ichidanVerb},
+      wordClass: WordClass.ichidanVerb,
+    );
+
+final LemmatizationRule ichidanVerbNegativeConditionalVariant1 =
+    LemmatizationRule.simple(
+      name: 'Ichidan verb - negative conditional form (informal variant)',
+      pattern: 'なきゃ',
+      replacement: 'る',
+      lookAheadBehind: _lookBehinds,
+      validChildClasses: {WordClass.ichidanVerb},
+      wordClass: WordClass.ichidanVerb,
+    );
+
+final LemmatizationRule ichidanVerbNegativeConditionalVariant2 =
+    LemmatizationRule.simple(
+      name: 'Ichidan verb - negative conditional form (informal variant)',
+      pattern: 'なくちゃ',
+      replacement: 'る',
+      lookAheadBehind: _lookBehinds,
+      validChildClasses: {WordClass.ichidanVerb},
+      wordClass: WordClass.ichidanVerb,
+    );
+
+final LemmatizationRule ichidanVerbNegativeConditionalVariant3 =
+    LemmatizationRule.simple(
+      name: 'Ichidan verb - negative conditional form (informal variant)',
+      pattern: 'ないと',
+      replacement: 'る',
+      lookAheadBehind: _lookBehinds,
+      validChildClasses: {WordClass.ichidanVerb},
+      wordClass: WordClass.ichidanVerb,
+    );
+
+final LemmatizationRule ichidanVerbNegativeVolitional =
+    LemmatizationRule.simple(
+      name: 'Ichidan verb - negative volitional form',
+      pattern: 'なかろう',
+      replacement: 'る',
+      lookAheadBehind: _lookBehinds,
+      validChildClasses: {WordClass.ichidanVerb},
+      wordClass: WordClass.ichidanVerb,
+    );
+
+final LemmatizationRule ichidanVerbNegativePotential = LemmatizationRule.simple(
+  name: 'Ichidan verb - negative potential form',
+  pattern: 'られない',
+  replacement: 'る',
+  lookAheadBehind: _lookBehinds,
+  validChildClasses: {WordClass.ichidanVerb},
+  wordClass: WordClass.ichidanVerb,
+);
+
+final LemmatizationRule ichidanVerbNegativePassive = LemmatizationRule.simple(
+  name: 'Ichidan verb - negative passive form',
+  pattern: 'られない',
+  replacement: 'る',
+  lookAheadBehind: _lookBehinds,
+  validChildClasses: {WordClass.ichidanVerb},
+  wordClass: WordClass.ichidanVerb,
+);
+
+final LemmatizationRule ichidanVerbNegativeCausative = LemmatizationRule.simple(
+  name: 'Ichidan verb - negative causative form',
+  pattern: 'させない',
+  replacement: 'る',
+  lookAheadBehind: _lookBehinds,
+  validChildClasses: {WordClass.ichidanVerb},
+  wordClass: WordClass.ichidanVerb,
+);
+
+final LemmatizationRule ichidanVerbNegativeCausativePassive =
+    LemmatizationRule.simple(
+      name: 'Ichidan verb - negative causative passive form',
+      pattern: 'させられない',
+      replacement: 'る',
+      lookAheadBehind: _lookBehinds,
+      validChildClasses: {WordClass.ichidanVerb},
+      wordClass: WordClass.ichidanVerb,
+    );
+
+final LemmatizationRule ichidanVerbNegativeImperative =
+    LemmatizationRule.simple(
+      name: 'Ichidan verb - negative imperative form',
+      pattern: 'るな',
+      replacement: 'る',
+      lookAheadBehind: _lookBehinds,
+      validChildClasses: {WordClass.ichidanVerb},
+      wordClass: WordClass.ichidanVerb,
+    );
+
+final LemmatizationRule ichidanVerbDesire = LemmatizationRule.simple(
+  name: 'Ichidan verb - desire form',
+  pattern: 'たい',
+  replacement: 'る',
+  lookAheadBehind: _lookBehinds,
+  validChildClasses: {WordClass.ichidanVerb},
+  wordClass: WordClass.ichidanVerb,
+);
+
+final LemmatizationRule ichidanVerbNegativeDesire = LemmatizationRule.simple(
+  name: 'Ichidan verb - negative desire form',
+  pattern: 'たくない',
+  replacement: 'る',
+  lookAheadBehind: _lookBehinds,
+  validChildClasses: {WordClass.ichidanVerb},
+  wordClass: WordClass.ichidanVerb,
+);
+
+final LemmatizationRule ichidanVerbPastDesire = LemmatizationRule.simple(
+  name: 'Ichidan verb - past desire form',
+  pattern: 'たかった',
+  replacement: 'る',
+  lookAheadBehind: _lookBehinds,
+  validChildClasses: {WordClass.ichidanVerb},
+  wordClass: WordClass.ichidanVerb,
+);
+
+final LemmatizationRule ichidanVerbNegativePastDesire =
+    LemmatizationRule.simple(
+      name: 'Ichidan verb - negative past desire form',
+      pattern: 'たくなかった',
+      replacement: 'る',
+      lookAheadBehind: _lookBehinds,
+      validChildClasses: {WordClass.ichidanVerb},
+      wordClass: WordClass.ichidanVerb,
+    );
+
+final List<LemmatizationRule> ichidanVerbLemmatizationRules =
+    List.unmodifiable([
+      ichidanVerbBase,
+      ichidanVerbNegative,
+      ichidanVerbPast,
+      ichidanVerbTe,
+      ichidanVerbTeiru,
+      ichidanVerbTeita,
+      ichidanVerbConditional,
+      ichidanVerbVolitional,
+      ichidanVerbPotential,
+      ichidanVerbPassive,
+      ichidanVerbCausative,
+      ichidanVerbCausativePassive,
+      ichidanVerbImperative,
+      ichidanVerbNegativePast,
+      ichidanVerbNegativeTe,
+      ichidanVerbNegativeConditional,
+      ichidanVerbNegativeConditionalVariant1,
+      ichidanVerbNegativeConditionalVariant2,
+      ichidanVerbNegativeConditionalVariant3,
+      ichidanVerbNegativeVolitional,
+      ichidanVerbNegativePotential,
+      ichidanVerbNegativePassive,
+      ichidanVerbNegativeCausative,
+      ichidanVerbNegativeCausativePassive,
+      ichidanVerbNegativeImperative,
+      ichidanVerbDesire,
+      ichidanVerbNegativeDesire,
+      ichidanVerbPastDesire,
+      ichidanVerbNegativePastDesire,
+    ]);

lib/util/romaji_transliteration.dart

@@ -1,9 +1,9 @@
 // Source: https://github.com/Kimtaro/ve/blob/master/lib/providers/japanese_transliterators.rb
 
-const hiragana_syllabic_n = 'ん';
-const hiragana_small_tsu = 'っ';
+const hiraganaSyllabicN = 'ん';
+const hiraganaSmallTsu = 'っ';
 
-const Map<String, String> hiragana_to_latin = {
+const Map<String, String> hiraganaToLatin = {
   'あ': 'a',
   'い': 'i',
   'う': 'u',
@@ -209,7 +209,7 @@ const Map<String, String> hiragana_to_latin = {
   'ゟ': 'yori',
 };
 
-const Map<String, String> latin_to_hiragana = {
+const Map<String, String> latinToHiragana = {
   'a': 'あ',
   'i': 'い',
   'u': 'う',
@@ -481,12 +481,13 @@ const Map<String, String> latin_to_hiragana = {
   '#~': '〜',
 };
 
-bool _smallTsu(String forConversion) => forConversion == hiragana_small_tsu;
+bool _smallTsu(String forConversion) => forConversion == hiraganaSmallTsu;
 bool _nFollowedByYuYeYo(String forConversion, String kana) =>
-    forConversion == hiragana_syllabic_n &&
+    forConversion == hiraganaSyllabicN &&
     kana.length > 1 &&
     'やゆよ'.contains(kana.substring(1, 2));
 
+/// Transliterates a string of hiragana characters to Latin script (romaji).
 String transliterateHiraganaToLatin(String hiragana) {
   String kana = hiragana;
   String romaji = '';
@@ -505,7 +506,7 @@ String transliterateHiraganaToLatin(String hiragana) {
       } else if (_nFollowedByYuYeYo(forConversion, kana)) {
         mora = "n'";
       }
-      mora ??= hiragana_to_latin[forConversion];
+      mora ??= hiraganaToLatin[forConversion];
 
       if (mora != null) {
         if (geminate) {
@@ -524,15 +525,61 @@ String transliterateHiraganaToLatin(String hiragana) {
   return romaji;
 }
 
+/// Returns a list of pairs of indices into the input and output strings,
+/// indicating which characters in the input string correspond to which characters in the output string.
+List<(int, int)> transliterateHiraganaToLatinSpan(String hiragana) {
+  String kana = hiragana;
+  String romaji = '';
+  final List<(int, int)> spans = [];
+  bool geminate = false;
+  int kanaIndex = 0;
+
+  while (kana.isNotEmpty) {
+    final lengths = [if (kana.length > 1) 2, 1];
+    for (final length in lengths) {
+      final String forConversion = kana.substring(0, length);
+      String? mora;
+
+      if (_smallTsu(forConversion)) {
+        geminate = true;
+        kana = kana.replaceRange(0, length, '');
+        break;
+      } else if (_nFollowedByYuYeYo(forConversion, kana)) {
+        mora = "n'";
+      }
+      mora ??= hiraganaToLatin[forConversion];
+
+      if (mora != null) {
+        if (geminate) {
+          geminate = false;
+          romaji += mora.substring(0, 1);
+        }
+        spans.add((kanaIndex, romaji.length));
+        romaji += mora;
+        kana = kana.replaceRange(0, length, '');
+        kanaIndex += length;
+        break;
+      } else if (length == 1) {
+        spans.add((kanaIndex, romaji.length));
+        romaji += forConversion;
+        kana = kana.replaceRange(0, length, '');
+        kanaIndex += length;
+      }
+    }
+  }
+  return spans;
+}
+
 bool _doubleNFollowedByAIUEO(String forConversion) =>
     RegExp(r'^nn[aiueo]$').hasMatch(forConversion);
 bool _hasTableMatch(String forConversion) =>
-    latin_to_hiragana[forConversion] != null;
+    latinToHiragana[forConversion] != null;
 bool _hasDoubleConsonant(String forConversion, int length) =>
     forConversion == 'tch' ||
     (length == 2 &&
         RegExp(r'^([kgsztdnbpmyrlwchf])\1$').hasMatch(forConversion));
 
+/// Transliterates a string of Latin script (romaji) to hiragana characters.
 String transliterateLatinToHiragana(String latin) {
   String romaji = latin
       .toLowerCase()
@@ -549,12 +596,12 @@ String transliterateLatinToHiragana(String latin) {
       final String forConversion = romaji.substring(0, length);
 
       if (_doubleNFollowedByAIUEO(forConversion)) {
-        mora = hiragana_syllabic_n;
+        mora = hiraganaSyllabicN;
         forRemoval = 1;
       } else if (_hasTableMatch(forConversion)) {
-        mora = latin_to_hiragana[forConversion];
+        mora = latinToHiragana[forConversion];
       } else if (_hasDoubleConsonant(forConversion, length)) {
-        mora = hiragana_small_tsu;
+        mora = hiraganaSmallTsu;
         forRemoval = 1;
       }
 
@@ -572,6 +619,53 @@ String transliterateLatinToHiragana(String latin) {
   return kana;
 }
 
+/// Returns a list of pairs of indices into the input and output strings,
+/// indicating which characters in the input string correspond to which characters in the output string.
+List<(int, int)> transliterateLatinToHiraganaSpan(String latin) {
+  String romaji = latin
+      .toLowerCase()
+      .replaceAll('mb', 'nb')
+      .replaceAll('mp', 'np');
+  String kana = '';
+  final List<(int, int)> spans = [];
+  int latinIndex = 0;
+
+  while (romaji.isNotEmpty) {
+    final lengths = [if (romaji.length > 2) 3, if (romaji.length > 1) 2, 1];
+
+    for (final length in lengths) {
+      String? mora;
+      int forRemoval = length;
+      final String forConversion = romaji.substring(0, length);
+
+      if (_doubleNFollowedByAIUEO(forConversion)) {
+        mora = hiraganaSyllabicN;
+        forRemoval = 1;
+      } else if (_hasTableMatch(forConversion)) {
+        mora = latinToHiragana[forConversion];
+      } else if (_hasDoubleConsonant(forConversion, length)) {
+        mora = hiraganaSmallTsu;
+        forRemoval = 1;
+      }
+
+      if (mora != null) {
+        spans.add((latinIndex, kana.length));
+        kana += mora;
+        romaji = romaji.replaceRange(0, forRemoval, '');
+        latinIndex += forRemoval;
+        break;
+      } else if (length == 1) {
+        spans.add((latinIndex, kana.length));
+        kana += forConversion;
+        romaji = romaji.replaceRange(0, 1, '');
+        latinIndex += 1;
+      }
+    }
+  }
+
+  return spans;
+}
+
 String _transposeCodepointsInRange(
   String text,
   int distance,
@@ -583,15 +677,19 @@ String _transposeCodepointsInRange(
   ),
 );
 
+/// Transliterates a string of kana characters (hiragana or katakana) to Latin script (romaji).
 String transliterateKanaToLatin(String kana) =>
     transliterateHiraganaToLatin(transliterateKatakanaToHiragana(kana));
 
+/// Transliterates a string of Latin script (romaji) to katakana characters.
 String transliterateLatinToKatakana(String latin) =>
     transliterateHiraganaToKatakana(transliterateLatinToHiragana(latin));
 
+/// Transliterates a string of katakana characters to hiragana characters.
 String transliterateKatakanaToHiragana(String katakana) =>
     _transposeCodepointsInRange(katakana, -96, 12449, 12534);
 
+/// Transliterates a string of hiragana characters to katakana characters.
 String transliterateHiraganaToKatakana(String hiragana) =>
     _transposeCodepointsInRange(hiragana, 96, 12353, 12438);
 

migrations/0001_JMDict.sql

@@ -1,3 +1,18 @@
+SELECT icu_load_collation('ja_JP', 'japanese');
+
+CREATE TABLE "JMdict_Version" (
+  "version" VARCHAR(10) PRIMARY KEY NOT NULL,
+  "date" DATE NOT NULL,
+  "hash" VARCHAR(64) NOT NULL
+) WITHOUT ROWID;
+
+CREATE TRIGGER "JMdict_Version_SingleRow"
+BEFORE INSERT ON "JMdict_Version"
+WHEN (SELECT COUNT(*) FROM "JMdict_Version") >= 1
+BEGIN
+    SELECT RAISE(FAIL, 'Only one row allowed in JMdict_Version');
+END;
+
 CREATE TABLE "JMdict_InfoDialect" (
   "id" VARCHAR(4) PRIMARY KEY NOT NULL,
   "description" TEXT NOT NULL
@@ -42,13 +57,13 @@ CREATE TABLE "JMdict_KanjiElement" (
   "elementId" INTEGER PRIMARY KEY,
   "entryId" INTEGER NOT NULL REFERENCES "JMdict_Entry"("entryId"),
   "orderNum" INTEGER NOT NULL,
-  "reading" TEXT NOT NULL,
+  "reading" TEXT NOT NULL COLLATE japanese,
   "news" INTEGER CHECK ("news" BETWEEN 1 AND 2),
   "ichi" INTEGER CHECK ("ichi" BETWEEN 1 AND 2),
   "spec" INTEGER CHECK ("spec" BETWEEN 1 AND 2),
   "gai" INTEGER CHECK ("gai" BETWEEN 1 AND 2),
   "nf" INTEGER CHECK ("nf" BETWEEN 1 AND 48),
-  UNIQUE("entryId", "reading"),
+  -- UNIQUE("entryId", "reading"),
   UNIQUE("entryId", "orderNum")
 ) WITHOUT ROWID;
 
@@ -67,14 +82,14 @@ CREATE TABLE "JMdict_ReadingElement" (
   "elementId" INTEGER PRIMARY KEY,
   "entryId" INTEGER NOT NULL REFERENCES "JMdict_Entry"("entryId"),
   "orderNum" INTEGER NOT NULL,
-  "reading" TEXT NOT NULL,
+  "reading" TEXT NOT NULL COLLATE japanese,
   "readingDoesNotMatchKanji" BOOLEAN NOT NULL DEFAULT FALSE,
   "news" INTEGER CHECK ("news" BETWEEN 1 AND 2),
   "ichi" INTEGER CHECK ("ichi" BETWEEN 1 AND 2),
   "spec" INTEGER CHECK ("spec" BETWEEN 1 AND 2),
   "gai" INTEGER CHECK ("gai" BETWEEN 1 AND 2),
   "nf" INTEGER CHECK ("nf" BETWEEN 1 AND 48),
-  UNIQUE("entryId", "reading"),
+  -- UNIQUE("entryId", "reading"),
   UNIQUE("entryId", "orderNum")
 ) WITHOUT ROWID;
 
@@ -107,17 +122,17 @@ CREATE INDEX "JMdict_Sense_byEntryId_byOrderNum" ON "JMdict_Sense"("entryId", "o
 CREATE TABLE "JMdict_SenseRestrictedToKanji" (
   "entryId" INTEGER NOT NULL,
   "senseId" INTEGER NOT NULL REFERENCES "JMdict_Sense"("senseId"),
-  "kanji" TEXT NOT NULL,
-  FOREIGN KEY ("entryId", "kanji") REFERENCES "JMdict_KanjiElement"("entryId", "reading"),
-  PRIMARY KEY ("entryId", "senseId", "kanji")
+  "kanjiOrderNum" INTEGER NOT NULL CHECK ("kanjiOrderNum" > 0),
+  FOREIGN KEY ("entryId", "kanjiOrderNum") REFERENCES "JMdict_KanjiElement"("entryId", "orderNum"),
+  PRIMARY KEY ("entryId", "senseId", "kanjiOrderNum")
 ) WITHOUT ROWID;
 
 CREATE TABLE "JMdict_SenseRestrictedToReading" (
   "entryId" INTEGER NOT NULL,
   "senseId" INTEGER NOT NULL REFERENCES "JMdict_Sense"("senseId"),
-  "reading" TEXT NOT NULL,
-  FOREIGN KEY ("entryId", "reading") REFERENCES "JMdict_ReadingElement"("entryId", "reading"),
-  PRIMARY KEY ("entryId", "senseId", "reading")
+  "readingOrderNum" INTEGER NOT NULL CHECK ("readingOrderNum" > 0),
+  FOREIGN KEY ("entryId", "readingOrderNum") REFERENCES "JMdict_ReadingElement"("entryId", "orderNum"),
+  PRIMARY KEY ("entryId", "senseId", "readingOrderNum")
 ) WITHOUT ROWID;
 
 -- In order to add xrefs, you will need to have added the entry to xref to.
@@ -132,32 +147,23 @@ CREATE TABLE "JMdict_SenseRestrictedToReading" (
 
 CREATE TABLE "JMdict_SenseSeeAlso" (
   "senseId" INTEGER NOT NULL REFERENCES "JMdict_Sense"("senseId"),
-  "xrefEntryId" INTEGER NOT NULL,
-  "seeAlsoReading" TEXT,
-  "seeAlsoKanji" TEXT,
-  "seeAlsoSense" INTEGER,
+  "xrefEntryId" INTEGER NOT NULL REFERENCES "JMdict_Entry"("entryId"),
+  -- Sometimes the cross reference is to a specific sense
+  "xrefSenseOrderNum" INTEGER,
   -- For some entries, the cross reference is ambiguous. This means that while the ingestion
   -- has determined some xrefEntryId, it is not guaranteed to be the correct one.
   "ambiguous" BOOLEAN NOT NULL DEFAULT FALSE,
-  FOREIGN KEY ("xrefEntryId", "seeAlsoKanji") REFERENCES "JMdict_KanjiElement"("entryId", "reading"),
-  FOREIGN KEY ("xrefEntryId", "seeAlsoReading") REFERENCES "JMdict_ReadingElement"("entryId", "reading"),
-  FOREIGN KEY ("xrefEntryId", "seeAlsoSense") REFERENCES "JMdict_Sense"("entryId", "orderNum"),
-  UNIQUE("senseId", "xrefEntryId", "seeAlsoReading", "seeAlsoKanji", "seeAlsoSense")
+  FOREIGN KEY ("xrefEntryId", "xrefSenseOrderNum") REFERENCES "JMdict_Sense"("entryId", "orderNum"),
+  UNIQUE("senseId", "xrefEntryId", "xrefSenseOrderNum")
 );
 
 CREATE TABLE "JMdict_SenseAntonym" (
   "senseId" INTEGER NOT NULL REFERENCES "JMdict_Sense"("senseId"),
-  "xrefEntryId" INTEGER NOT NULL,
-  "antonymReading" TEXT,
-  "antonymKanji" TEXT,
-  "antonymSense" INTEGER,
+  "xrefEntryId" INTEGER NOT NULL REFERENCES "JMdict_Entry"("entryId"),
   -- For some entries, the cross reference is ambiguous. This means that while the ingestion
   -- has determined some xrefEntryId, it is not guaranteed to be the correct one.
   "ambiguous" BOOLEAN NOT NULL DEFAULT FALSE,
-  FOREIGN KEY ("xrefEntryId", "antonymKanji") REFERENCES "JMdict_KanjiElement"("entryId", "reading"),
-  FOREIGN KEY ("xrefEntryId", "antonymReading") REFERENCES "JMdict_ReadingElement"("entryId", "reading"),
-  FOREIGN KEY ("xrefEntryId", "antonymSense") REFERENCES "JMdict_Sense"("entryId", "orderNum"),
-  UNIQUE("senseId", "xrefEntryId", "antonymReading", "antonymKanji", "antonymSense")
+  UNIQUE("senseId", "xrefEntryId")
 );
 
 -- These cross references are going to be mostly accessed from a sense

migrations/0004_JMDict_tanos_jlpt_tags.sql

@@ -1,3 +1,16 @@
+CREATE TABLE "JMdict_JLPT_Version" (
+  "version" VARCHAR(10) PRIMARY KEY NOT NULL,
+  "date" DATE NOT NULL,
+  "hash" VARCHAR(64) NOT NULL
+) WITHOUT ROWID;
+
+CREATE TRIGGER "JMdict_JLPT_Version_SingleRow"
+BEFORE INSERT ON "JMdict_JLPT_Version"
+WHEN (SELECT COUNT(*) FROM "JMdict_JLPT_Version") >= 1
+BEGIN
+    SELECT RAISE(FAIL, 'Only one row allowed in JMdict_JLPT_Version');
+END;
+
 CREATE TABLE "JMdict_JLPTTag" (
   "entryId" INTEGER NOT NULL,
   "jlptLevel" CHAR(2) NOT NULL CHECK ("jlptLevel" in ('N5', 'N4', 'N3', 'N2', 'N1')),

migrations/0006_RADKFILE.sql

@@ -1,3 +1,16 @@
+CREATE TABLE "RADKFILE_Version" (
+  "version" VARCHAR(10) PRIMARY KEY NOT NULL,
+  "date" DATE NOT NULL,
+  "hash" VARCHAR(64) NOT NULL
+) WITHOUT ROWID;
+
+CREATE TRIGGER "RADKFILE_Version_SingleRow"
+BEFORE INSERT ON "RADKFILE_Version"
+WHEN (SELECT COUNT(*) FROM "RADKFILE_Version") >= 1
+BEGIN
+    SELECT RAISE(FAIL, 'Only one row allowed in RADKFILE_Version');
+END;
+
 CREATE TABLE "RADKFILE" (
   "kanji" CHAR(1) NOT NULL,
   "radical" CHAR(1) NOT NULL,

migrations/0007_KANJIDIC2.sql

@@ -1,3 +1,16 @@
+CREATE TABLE "KANJIDIC_Version" (
+  "version" VARCHAR(10) PRIMARY KEY NOT NULL,
+  "date" DATE NOT NULL,
+  "hash" VARCHAR(64) NOT NULL
+) WITHOUT ROWID;
+
+CREATE TRIGGER "KANJIDIC_Version_SingleRow"
+BEFORE INSERT ON "KANJIDIC_Version"
+WHEN (SELECT COUNT(*) FROM "KANJIDIC_Version") >= 1
+BEGIN
+    SELECT RAISE(FAIL, 'Only one row allowed in KANJIDIC_Version');
+END;
+
 CREATE TABLE "KANJIDIC_Character" (
   "literal" CHAR(1) NOT NULL PRIMARY KEY,
   "grade" INTEGER CHECK ("grade" BETWEEN 1 AND 10),

nix/database_tool.nix

@@ -1,6 +1,7 @@
 {
   src,
   buildDartApplication,
+  sqlite,
 }:
 buildDartApplication {
   pname = "jadb-database-tool";
@@ -9,6 +10,9 @@ buildDartApplication {
 
   dartEntryPoints."bin/jadb" = "bin/jadb.dart";
 
+  # NOTE: here we are overriding the implicitly added runtimeDependency from the package fixup in pub2nix.
+  runtimeDependencies = [ sqlite ];
+
   # NOTE: the default dart hooks are using `dart compile`, which is not able to call the
   #       new dart build hooks required to use package:sqlite3 >= 3.0.0. So we override
   #       these phases to use `dart build` instead.

pubspec.lock

@@ -5,18 +5,18 @@ packages:
     dependency: transitive
     description:
       name: _fe_analyzer_shared
-      sha256: "3b19a47f6ea7c2632760777c78174f47f6aec1e05f0cd611380d4593b8af1dbc"
+      sha256: "8d718c5c58904f9937290fd5dbf2d6a0e02456867706bfb6cd7b81d394e738d5"
       url: "https://pub.dev"
     source: hosted
-    version: "96.0.0"
+    version: "98.0.0"
   analyzer:
     dependency: transitive
     description:
       name: analyzer
-      sha256: "0c516bc4ad36a1a75759e54d5047cb9d15cded4459df01aa35a0b5ec7db2c2a0"
+      sha256: "6141ad5d092d1e1d13929c0504658bbeccc1703505830d7c26e859908f5efc88"
       url: "https://pub.dev"
     source: hosted
-    version: "10.2.0"
+    version: "12.0.0"
   args:
     dependency: "direct main"
     description:
@@ -29,10 +29,18 @@ packages:
     dependency: transitive
     description:
       name: async
-      sha256: "758e6d74e971c3e5aceb4110bfd6698efc7f501675bcfe0c775459a8140750eb"
+      sha256: e2eb0491ba5ddb6177742d2da23904574082139b07c1e33b8503b9f46f3e1a37
       url: "https://pub.dev"
     source: hosted
-    version: "2.13.0"
+    version: "2.13.1"
+  benchmark_harness:
+    dependency: "direct dev"
+    description:
+      name: benchmark_harness
+      sha256: a2d3c4c83cac0126bf38e41eaf7bd9ed4f6635f1ee1a0cbc6f79fa9736c62cbd
+      url: "https://pub.dev"
+    source: hosted
+    version: "2.4.0"
   boolean_selector:
     dependency: transitive
     description:
@@ -93,10 +101,10 @@ packages:
     dependency: "direct main"
     description:
       name: csv
-      sha256: bef2950f7a753eb82f894a2eabc3072e73cf21c17096296a5a992797e50b1d0d
+      sha256: "2e0a52fb729f2faacd19c9c0c954ff450bba37aa8ab999410309e2342e7013a2"
       url: "https://pub.dev"
     source: hosted
-    version: "7.1.0"
+    version: "8.0.0"
   equatable:
     dependency: "direct main"
     description:
@@ -141,10 +149,10 @@ packages:
     dependency: transitive
     description:
       name: hooks
-      sha256: "7a08a0d684cb3b8fb604b78455d5d352f502b68079f7b80b831c62220ab0a4f6"
+      sha256: e79ed1e8e1929bc6ecb6ec85f0cb519c887aa5b423705ded0d0f2d9226def388
       url: "https://pub.dev"
     source: hosted
-    version: "1.0.1"
+    version: "1.0.2"
   http_multi_server:
     dependency: transitive
     description:
@@ -189,18 +197,18 @@ packages:
     dependency: transitive
     description:
       name: matcher
-      sha256: "12956d0ad8390bbcc63ca2e1469c0619946ccb52809807067a7020d57e647aa6"
+      sha256: dc0b7dc7651697ea4ff3e69ef44b0407ea32c487a39fff6a4004fa585e901861
       url: "https://pub.dev"
     source: hosted
-    version: "0.12.18"
+    version: "0.12.19"
   meta:
     dependency: transitive
     description:
       name: meta
-      sha256: "9f29b9bcc8ee287b1a31e0d01be0eae99a930dbffdaecf04b3f3d82a969f296f"
+      sha256: df0c643f44ad098eb37988027a8e2b2b5a031fd3977f06bbfd3a76637e8df739
       url: "https://pub.dev"
     source: hosted
-    version: "1.18.1"
+    version: "1.18.2"
   mime:
     dependency: transitive
     description:
@@ -213,10 +221,10 @@ packages:
     dependency: transitive
     description:
       name: native_toolchain_c
-      sha256: "89e83885ba09da5fdf2cdacc8002a712ca238c28b7f717910b34bcd27b0d03ac"
+      sha256: "6ba77bb18063eebe9de401f5e6437e95e1438af0a87a3a39084fbd37c90df572"
       url: "https://pub.dev"
     source: hosted
-    version: "0.17.4"
+    version: "0.17.6"
   node_preamble:
     dependency: transitive
     description:
@@ -341,10 +349,10 @@ packages:
     dependency: "direct main"
     description:
       name: sqlite3
-      sha256: b7cf6b37667f6a921281797d2499ffc60fb878b161058d422064f0ddc78f6aa6
+      sha256: caa693ad15a587a2b4fde093b728131a1827903872171089dedb16f7665d3a91
       url: "https://pub.dev"
     source: hosted
-    version: "3.1.6"
+    version: "3.2.0"
   stack_trace:
     dependency: transitive
     description:
@@ -389,26 +397,26 @@ packages:
     dependency: "direct dev"
     description:
       name: test
-      sha256: "54c516bbb7cee2754d327ad4fca637f78abfc3cbcc5ace83b3eda117e42cd71a"
+      sha256: "8d9ceddbab833f180fbefed08afa76d7c03513dfdba87ffcec2718b02bbcbf20"
       url: "https://pub.dev"
     source: hosted
-    version: "1.29.0"
+    version: "1.31.0"
   test_api:
     dependency: transitive
     description:
       name: test_api
-      sha256: "93167629bfc610f71560ab9312acdda4959de4df6fac7492c89ff0d3886f6636"
+      sha256: "949a932224383300f01be9221c39180316445ecb8e7547f70a41a35bf421fb9e"
       url: "https://pub.dev"
     source: hosted
-    version: "0.7.9"
+    version: "0.7.11"
   test_core:
     dependency: transitive
     description:
       name: test_core
-      sha256: "394f07d21f0f2255ec9e3989f21e54d3c7dc0e6e9dbce160e5a9c1a6be0e2943"
+      sha256: "1991d4cfe85d5043241acac92962c3977c8d2f2add1ee73130c7b286417d1d34"
       url: "https://pub.dev"
     source: hosted
-    version: "0.6.15"
+    version: "0.6.17"
   typed_data:
     dependency: transitive
     description:

pubspec.yaml

@@ -9,7 +9,7 @@ environment:
 dependencies:
   args: ^2.7.0
   collection: ^1.19.0
-  csv: ^7.1.0
+  csv: ^8.0.0
   equatable: ^2.0.0
   path: ^1.9.1
   sqflite_common: ^2.5.0
@@ -18,6 +18,7 @@ dependencies:
   xml: ^6.5.0
 
 dev_dependencies:
+  benchmark_harness: ^2.4.0
   lints: ^6.0.0
   test: ^1.25.15
 

test/const_data/kanji_grades.dart

@@ -0,0 +1,21 @@
+import 'package:collection/collection.dart';
+import 'package:jadb/const_data/kanji_grades.dart';
+import 'package:test/test.dart';
+
+void main() {
+  test('All constant kanji in jouyouKanjiByGrades are 2136 in total', () {
+    expect(jouyouKanjiByGrades.values.flattenedToSet.length, 2136);
+  });
+
+  // test('All constant kanji in jouyouKanjiByGrades are present in KANJIDIC2', () {
+
+  // });
+
+  // test('All constant kanji in jouyouKanjiByGrades have matching grade as in KANJIDIC2', () {
+
+  // });
+
+  // test('All constant kanji in jouyouKanjiByGradesAndStrokeCount have matching stroke count as in KANJIDIC2', () {
+
+  // });
+}

test/const_data/radicals_test.dart

@@ -0,0 +1,17 @@
+import 'package:collection/collection.dart';
+import 'package:jadb/const_data/radicals.dart';
+import 'package:test/test.dart';
+
+void main() {
+  test('All constant radicals are 253 in total', () {
+    expect(radicals.values.flattenedToSet.length, 253);
+  });
+
+  // test('All constant radicals have at least 1 associated kanji in KANJIDIC2', () {
+
+  // });
+
+  // test('All constant radicals match the stroke order listed in KANJIDIC2', () {
+
+  // });
+}

test/jouyou_kanji_test.dart

@@ -1,9 +0,0 @@
-import 'package:collection/collection.dart';
-import 'package:jadb/const_data/kanji_grades.dart';
-import 'package:test/test.dart';
-
-void main() {
-  test('Assert 2136 kanji in jouyou set', () {
-    expect(JOUYOU_KANJI_BY_GRADES.values.flattenedToSet.length, 2136);
-  });
-}

test/models/create_empty_db_test.dart

@@ -1,21 +1,20 @@
-import 'dart:ffi';
-import 'dart:io';
-
 import 'package:jadb/models/create_empty_db.dart';
 import 'package:jadb/search.dart';
 import 'package:sqflite_common_ffi/sqflite_ffi.dart';
 // import 'package:sqlite3/open.dart';
 import 'package:test/test.dart';
 
-Future<DatabaseExecutor> setup_inmemory_database() async {
-  final dbConnection = await createDatabaseFactoryFfi().openDatabase(':memory:');
+Future<DatabaseExecutor> setupInMemoryDatabase() async {
+  final dbConnection = await createDatabaseFactoryFfi().openDatabase(
+    ':memory:',
+  );
 
   return dbConnection;
 }
 
 void main() {
   test('Create empty db', () async {
-    final db = await setup_inmemory_database();
+    final db = await setupInMemoryDatabase();
 
     await createEmptyDb(db);
 

test/search/filter_kanji_test.dart

@@ -5,7 +5,7 @@ import 'setup_database_connection.dart';
 
 void main() {
   test('Filter kanji', () async {
-    final connection = await setup_database_connection();
+    final connection = await setupDatabaseConnection();
 
     final result = await connection.filterKanji([
       'a',
@@ -26,4 +26,27 @@ void main() {
 
     expect(result.join(), '漢字地字');
   });
+
+  test('Filter kanji - deduplicate', () async {
+    final connection = await setupDatabaseConnection();
+
+    final result = await connection.filterKanji([
+      'a',
+      'b',
+      'c',
+      '漢',
+      '字',
+      '地',
+      '字',
+      'か',
+      'な',
+      '.',
+      '!',
+      '@',
+      ';',
+      '々',
+    ], deduplicate: true);
+
+    expect(result.join(), '漢字地');
+  });
 }

test/search/kanji_search_test.dart

@@ -6,16 +6,16 @@ import 'setup_database_connection.dart';
 
 void main() {
   test('Search a kanji', () async {
-    final connection = await setup_database_connection();
+    final connection = await setupDatabaseConnection();
 
     final result = await connection.jadbSearchKanji('漢');
     expect(result, isNotNull);
   });
 
   group('Search all jouyou kanji', () {
-    JOUYOU_KANJI_BY_GRADES.forEach((grade, characters) {
+    jouyouKanjiByGrades.forEach((grade, characters) {
       test('Search all kanji in grade $grade', () async {
-        final connection = await setup_database_connection();
+        final connection = await setupDatabaseConnection();
 
         for (final character in characters) {
           final result = await connection.jadbSearchKanji(character);

test/search/setup_database_connection.dart

@@ -3,7 +3,7 @@ import 'dart:io';
 import 'package:jadb/_data_ingestion/open_local_db.dart';
 import 'package:sqflite_common/sqlite_api.dart';
 
-Future<Database> setup_database_connection() async {
+Future<Database> setupDatabaseConnection() async {
   final libSqlitePath = Platform.environment['LIBSQLITE_PATH'];
   final jadbPath = Platform.environment['JADB_PATH'];
 

test/search/word_search_test.dart

@@ -5,43 +5,43 @@ import 'setup_database_connection.dart';
 
 void main() {
   test('Search a word  - english - auto', () async {
-    final connection = await setup_database_connection();
+    final connection = await setupDatabaseConnection();
     final result = await connection.jadbSearchWord('kana');
     expect(result, isNotNull);
   });
 
   test('Get word search count - english - auto', () async {
-    final connection = await setup_database_connection();
+    final connection = await setupDatabaseConnection();
     final result = await connection.jadbSearchWordCount('kana');
     expect(result, isNotNull);
   });
 
   test('Search a word  - japanese kana - auto', () async {
-    final connection = await setup_database_connection();
+    final connection = await setupDatabaseConnection();
     final result = await connection.jadbSearchWord('かな');
     expect(result, isNotNull);
   });
 
   test('Get word search count - japanese kana - auto', () async {
-    final connection = await setup_database_connection();
+    final connection = await setupDatabaseConnection();
     final result = await connection.jadbSearchWordCount('かな');
     expect(result, isNotNull);
   });
 
   test('Search a word  - japanese kanji - auto', () async {
-    final connection = await setup_database_connection();
+    final connection = await setupDatabaseConnection();
     final result = await connection.jadbSearchWord('仮名');
     expect(result, isNotNull);
   });
 
   test('Get word search count - japanese kanji - auto', () async {
-    final connection = await setup_database_connection();
+    final connection = await setupDatabaseConnection();
     final result = await connection.jadbSearchWordCount('仮名');
     expect(result, isNotNull);
   });
 
   test('Get a word by id', () async {
-    final connection = await setup_database_connection();
+    final connection = await setupDatabaseConnection();
     final result = await connection.jadbGetWordById(1577090);
     expect(result, isNotNull);
   });
@@ -49,7 +49,7 @@ void main() {
   test(
     'Serialize all words',
     () async {
-      final connection = await setup_database_connection();
+      final connection = await setupDatabaseConnection();
 
       // Test serializing all words
       for (final letter in 'aiueoksthnmyrw'.split('')) {

test/util/lemmatizer/lemmatizer_test.dart

@@ -0,0 +1,51 @@
+import 'package:jadb/util/lemmatizer/lemmatizer.dart';
+import 'package:jadb/util/lemmatizer/rules/godan_verbs.dart';
+import 'package:jadb/util/lemmatizer/rules/ichidan_verbs.dart';
+import 'package:test/test.dart';
+
+const List<String> ichidanVerbs = [
+  '食べる',
+  '食べた',
+  '食べさせられた',
+  '食べたい',
+  '食べたくない',
+  '食べたくなかった',
+];
+const List<String> godanVerbs = [
+  '泳ぐ',
+  '泳いだ',
+  '泳げる',
+  // '泳げれた',
+];
+
+bool findRuleRecursively(Lemmatized result, LemmatizationRule expectedRule) {
+  if (result.rule == expectedRule) {
+    return true;
+  }
+
+  for (final c in result.children) {
+    if (findRuleRecursively(c, expectedRule)) {
+      return true;
+    }
+  }
+
+  return false;
+}
+
+void main() {
+  group('Lemmatize Ichidan Verbs', () {
+    for (final v in ichidanVerbs) {
+      test('Lemmatize Ichidan Verb $v', () {
+        expect(findRuleRecursively(lemmatize(v), ichidanVerbBase), true);
+      });
+    }
+  });
+
+  group('Lemmatize Godan Verbs', () {
+    for (final v in godanVerbs) {
+      test('Lemmatize Godan Verb $v', () {
+        expect(findRuleRecursively(lemmatize(v), godanVerbBase), true);
+      });
+    }
+  });
+}

test/util/lemmatizer/rules/godan_verbs_test.dart

@@ -0,0 +1,14 @@
+import 'package:jadb/util/lemmatizer/rules/godan_verbs.dart';
+import 'package:test/test.dart';
+
+void main() {
+  test('Test Godan Verb Base Rule', () {
+    expect(godanVerbBase.matches('泳ぐ'), true);
+    expect(godanVerbBase.apply('泳ぐ'), ['泳ぐ']);
+  });
+
+  test('Test Godan Verb Negative Rule', () {
+    expect(godanVerbNegative.matches('泳がない'), true);
+    expect(godanVerbNegative.apply('泳がない'), ['泳ぐ']);
+  });
+}

test/util/lemmatizer/rules/i_adjectives_test.dart

@@ -0,0 +1,15 @@
+import 'package:jadb/util/lemmatizer/rules/i_adjectives.dart';
+import 'package:test/test.dart';
+
+void main() {
+  test('Test i-adjective Base Rule', () {
+    expect(iAdjectiveBase.matches('怪しい'), true);
+    expect(iAdjectiveBase.apply('怪しい'), ['怪しい']);
+  });
+
+
+  test('Test i-adjective Negative Rule', () {
+    expect(iAdjectiveNegative.matches('怪しくない'), true);
+    expect(iAdjectiveNegative.apply('怪しくない'), ['怪しい']);
+  });
+}

test/util/lemmatizer/rules/ichidan_verbs_test.dart

@@ -0,0 +1,14 @@
+import 'package:jadb/util/lemmatizer/rules/ichidan_verbs.dart';
+import 'package:test/test.dart';
+
+void main() {
+  test('Test Ichidan Verb Base Rule', () {
+    expect(ichidanVerbBase.matches('食べる'), true);
+    expect(ichidanVerbBase.apply('食べる'), ['食べる']);
+  });
+
+  test('Test Ichidan Verb Negative Rule', () {
+    expect(ichidanVerbNegative.matches('食べない'), true);
+    expect(ichidanVerbNegative.apply('食べない'), ['食べる']);
+  });
+}

test/util/lemmatizer/rules_test.dart

@@ -0,0 +1,15 @@
+import 'package:jadb/util/lemmatizer/lemmatizer.dart';
+import 'package:jadb/util/lemmatizer/rules.dart';
+import 'package:test/test.dart';
+
+void main() {
+  test('Assert lemmatizerRulesByWordClass is correct', () {
+    for (final entry in lemmatizationRulesByWordClass.entries) {
+      final WordClass wordClass = entry.key;
+      final List<LemmatizationRule> rules = entry.value;
+      for (final LemmatizationRule rule in rules) {
+        expect(wordClass, rule.wordClass);
+      }
+    }
+  });
+}

test/util/romaji_transliteration_test.dart

@@ -37,6 +37,35 @@ void main() {
     });
   });
 
+  group('Romaji -> Hiragana Spans', () {
+    void Function() expectSpans(String input, List<String> expected) => () {
+      final result = transliterateLatinToHiraganaSpan(input);
+      final trans = transliterateLatinToHiragana(input);
+      for (int i = 0; i < result.length; i++) {
+        expect(
+          trans.substring(
+            result[i].$2,
+            i == result.length - 1 ? trans.length : result[i + 1].$2,
+          ),
+          expected[i],
+        );
+      }
+    };
+
+    test('Basic test', expectSpans('katamari', ['か', 'た', 'ま', 'り']));
+    test(
+      'Basic test with diacritics',
+      expectSpans('gadamari', ['が', 'だ', 'ま', 'り']),
+    );
+    test('wi and we', expectSpans('wiwe', ['うぃ', 'うぇ']));
+    test('nb = mb', expectSpans('kanpai', ['か', 'ん', 'ぱ', 'い']));
+    test('nb = mb', expectSpans('kampai', ['か', 'ん', 'ぱ', 'い']));
+    test('Double n', expectSpans('konnichiha', ['こ', 'ん', 'に', 'ち', 'は']));
+
+    // TODO: fix the implementation
+    // test('Double consonant', expectSpans('kappa', ['か', 'っぱ']));
+  });
+
   group('Hiragana -> Romaji', () {
     test('Basic test', () {
       final result = transliterateHiraganaToLatin('かたまり');
@@ -63,4 +92,31 @@ void main() {
       expect(result, 'kappa');
     });
   });
+
+  group('Hiragana -> Romaji Spans', () {
+    void Function() expectSpans(String input, List<String> expected) => () {
+      final result = transliterateHiraganaToLatinSpan(input);
+      final trans = transliterateHiraganaToLatin(input);
+      for (int i = 0; i < result.length; i++) {
+        expect(
+          trans.substring(
+            result[i].$2,
+            i == result.length - 1 ? trans.length : result[i + 1].$2,
+          ),
+          expected[i],
+        );
+      }
+    };
+
+    test('Basic test', expectSpans('かたまり', ['ka', 'ta', 'ma', 'ri']));
+    test(
+      'Basic test with diacritics',
+      expectSpans('がだまり', ['ga', 'da', 'ma', 'ri']),
+    );
+    test('wi and we', expectSpans('うぃうぇ', ['whi', 'whe']));
+    test('Double n', expectSpans('こんにちは', ['ko', 'n', 'ni', 'chi', 'ha']));
+
+    // TODO: fix the implementation
+    // test('Double consonant', expectSpans('かっぱ', ['ka', 'ppa']));
+  });
 }