WIP

This was earlier used to compensate for a double nesting bug. This has
been fixed in the latest version of the xml package.

.gitea/workflows/build-and-test.yml

@@ -0,0 +1,74 @@
+name: "Build and test"
+on:
+  workflow_dispatch:
+  pull_request:
+  push:
+jobs:
+  build:
+    runs-on: debian-latest
+    steps:
+    - uses: actions/checkout@v6
+
+    - name: Install sudo
+      run: apt-get update && apt-get -y install sudo
+
+    - name: Install nix
+      uses: https://github.com/cachix/install-nix-action@v31
+      with:
+        extra_nix_config: |
+          experimental-features = nix-command flakes
+          show-trace = true
+          max-jobs = auto
+          trusted-users = root
+          experimental-features = nix-command flakes
+          build-users-group =
+
+    - name: Update database inputs
+      run: |
+        nix flake update jmdict-src
+        nix flake update jmdict-with-examples-src
+        nix flake update radkfile-src
+        nix flake update kanjidic2-src
+
+    - name: Build database
+      run: nix build .#database -L
+
+    - name: Upload database as artifact
+      uses: actions/upload-artifact@v3
+      with:
+        name: jadb-${{ gitea.sha }}.zip
+        path: result/jadb.sqlite
+        if-no-files-found: error
+        retention-days: 15
+        # Already compressed
+        compression: 0
+
+    - name: Print database statistics
+      run: nix develop .# --command sqlite3_analyzer result/jadb.sqlite
+
+    # TODO: Defer failure of tests until after the coverage report is generated and uploaded.
+    - name: Run tests
+      run: nix develop .# --command dart run test --concurrency=1 --coverage-path=coverage/lcov.info
+
+    - name: Generate coverage report
+      run: |
+          GENHTML_ARGS=(
+            --current-date="$(date)"
+            --dark-mode
+            --output-directory coverage/report
+          )
+
+          nix develop .# --command genhtml "${GENHTML_ARGS[@]}" coverage/lcov.info
+
+    - name: Upload coverage report
+      uses: https://git.pvv.ntnu.no/Projects/rsync-action@v2
+      with:
+        source: ./coverage
+        target: jadb/${{ gitea.ref_name }}/
+        username: oysteikt
+        ssh-key: ${{ secrets.OYSTEIKT_GITEA_WEBDOCS_SSH_KEY }}
+        host: microbel.pvv.ntnu.no
+        known-hosts: "microbel.pvv.ntnu.no ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBEq0yasKP0mH6PI6ypmuzPzMnbHELo9k+YB5yW534aKudKZS65YsHJKQ9vapOtmegrn5MQbCCgrshf+/XwZcjbM="
+
+    - name: Run benchmarks
+      run: nix develop .# --command dart run benchmark_harness:bench --flavor jit

.gitignore

@@ -6,7 +6,9 @@
 .packages
 
 # Conventional directory for build output.
+/doc/
 /build/
+/coverage/
 main.db
 
 # Nix

README.md

@@ -1,7 +1,9 @@
-# jadb
-
 [![built with nix](https://builtwithnix.org/badge.svg)](https://builtwithnix.org)
 
+[Latest coverage report](https://www.pvv.ntnu.no/~oysteikt/gitea/jadb/main/coverage/report/)
+
+# jadb
+
 An SQLite database containing open source japanese dictionary data combined from several sources
 
 Note that while the license for the code is MIT, the data has various licenses.
@@ -16,3 +18,4 @@ Note that while the license for the code is MIT, the data has various licenses.
 | **Tanos JLPT levels:** | https://www.tanos.co.uk/jlpt/ |
 | **Kangxi Radicals:**   | https://ctext.org/kangxi-zidian |
 
+See [docs/overview.md](./docs/overview.md) for notes and implementation details.

analysis_options.yaml

@@ -0,0 +1,41 @@
+# This file configures the analyzer, which statically analyzes Dart code to
+# check for errors, warnings, and lints.
+#
+# The issues identified by the analyzer are surfaced in the UI of Dart-enabled
+# IDEs (https://dart.dev/tools#ides-and-editors). The analyzer can also be
+# invoked from the command line by running `flutter analyze`.
+
+# The following line activates a set of recommended lints for Flutter apps,
+# packages, and plugins designed to encourage good coding practices.
+include:
+  - package:lints/recommended.yaml
+
+linter:
+  # The lint rules applied to this project can be customized in the
+  # section below to disable rules from the `package:flutter_lints/flutter.yaml`
+  # included above or to enable additional rules. A list of all available lints
+  # and their documentation is published at https://dart.dev/lints.
+  #
+  # Instead of disabling a lint rule for the entire project in the
+  # section below, it can also be suppressed for a single line of code
+  # or a specific dart file by using the `// ignore: name_of_lint` and
+  # `// ignore_for_file: name_of_lint` syntax on the line or in the file
+  # producing the lint.
+  rules:
+    always_declare_return_types: true
+    annotate_redeclares: true
+    avoid_print: false
+    avoid_setters_without_getters: true
+    avoid_slow_async_io: true
+    directives_ordering: true
+    eol_at_end_of_file: true
+    prefer_const_declarations: true
+    prefer_contains: true
+    prefer_final_fields: true
+    prefer_final_locals: true
+    prefer_single_quotes: true
+    use_key_in_widget_constructors: true
+    use_null_aware_elements: true
+
+# Additional information about this file can be found at
+# https://dart.dev/guides/language/analysis-options

benchmark/benchmark.dart

@@ -0,0 +1,5 @@
+import './search/word_search.dart';
+
+Future<void> main() async {
+  await WordSearchBenchmark.main();
+}

benchmark/search/word_search.dart

@@ -0,0 +1,45 @@
+import 'package:benchmark_harness/benchmark_harness.dart';
+import 'package:jadb/search.dart';
+import 'package:sqflite_common/sqlite_api.dart';
+
+import '../../test/search/setup_database_connection.dart';
+
+class WordSearchBenchmark extends AsyncBenchmarkBase {
+  Database? connection;
+
+  static final List<String> searchTerms = [
+    'kana',
+    'kanji',
+    'kawaii',
+    'sushi',
+    'ramen',
+  ];
+
+  WordSearchBenchmark() : super('WordSearchBenchmark');
+
+  static Future<void> main() async {
+    print('Running WordSearchBenchmark...');
+    await WordSearchBenchmark().report();
+    print('Finished WordSearchBenchmark');
+  }
+
+  @override
+  Future<void> setup() async {
+    connection = await setupDatabaseConnection();
+  }
+
+  @override
+  Future<void> run() async {
+    for (final term in searchTerms) {
+      await connection!.jadbSearchWord(term);
+    }
+  }
+
+  @override
+  Future<void> teardown() async {
+    await connection?.close();
+  }
+
+  // @override
+  // Future<void> exercise() => run();
+}

bin/jadb.dart

@@ -2,18 +2,20 @@ import 'package:args/command_runner.dart';
 
 import 'package:jadb/cli/commands/create_db.dart';
 import 'package:jadb/cli/commands/create_tanos_jlpt_mappings.dart';
+import 'package:jadb/cli/commands/lemmatize.dart';
 import 'package:jadb/cli/commands/query_kanji.dart';
 import 'package:jadb/cli/commands/query_word.dart';
 
 Future<void> main(List<String> args) async {
   final runner = CommandRunner(
     'jadb',
-    "CLI tool to help creating and testing the jadb database",
+    'CLI tool to help creating and testing the jadb database',
   );
 
   runner.addCommand(CreateDb());
   runner.addCommand(QueryKanji());
   runner.addCommand(QueryWord());
+  runner.addCommand(Lemmatize());
   runner.addCommand(CreateTanosJlptMappings());
 
   runner.run(args);

docs/lemmatizer.md

@@ -0,0 +1,13 @@
+# Lemmatizer
+
+The lemmatizer is still quite experimental, but will play a more important role in the project in the future.
+
+It is a manual implementation of a [Finite State Transducer](https://en.wikipedia.org/wiki/Morphological_dictionary#Finite_State_Transducers) for morphological parsing. The FST is used to recursively remove affixes from a word until it (hopefully) deconjugates into its dictionary form. This iterative deconjugation tree will then be combined with queries into the dictionary data to determine if the deconjugation leads to a real known word.
+
+Each separate rule is a separate static object declared in `lib/util/lemmatizer/rules`.
+
+There is a cli subcommand for testing the tool interactively, you can run
+
+```bash
+dart run jadb lemmatize -w '食べさせられない'
+```

docs/overview.md

@@ -0,0 +1,27 @@
+# Overview
+
+This is the documentation for `jadb`. Since I'm currently the only one working on it, the documentation is more or less just notes to myself, to ensure I remember how and why I implemented certain features in a certain way a few months down the road. This is not a comprehensive and formal documentation for downstream use, neither for developers nor end-users.
+
+- [Word Search](./word-search.md)
+- [Lemmatizer](./lemmatizer.md)
+
+## Project structure
+
+- `lib/_data_ingestion` contains all the code for reading data sources, transforming them and compiling them into an SQLite database. This is for the most part isolated from the rest of the codebase, and should not be depended on by any code used for querying the database.
+- `lib/cli` contains code for cli tooling (e.g. argument parsing, subcommand handling, etc.)
+- `lib/const_data` contains database data that is small enough to warrant being hardcoded as dart constants.
+- `lib/models` contains all the code for representing the database schema as Dart classes, and for converting between those classes and the actual database.
+- `lib/search` contains all the code for searching the database.
+- `lib/util/lemmatizer` contains the code for lemmatization, which will be used by the search code in the future.
+- `migrations` contains raw SQL files for creating the database schema.
+
+## SQLite naming conventions
+
+> [!WARNING]
+> All of these conventions are actually not enforced yet, it will be fixed at some point.
+
+- Indices are prefixed with `IDX__`
+- Crossref tables are prefixed with `XREF__`
+- Trigger names are prefixed with `TRG__`
+- Views are prefixed with `VW__`
+- All data sources should have a `<datasource>_Version` table, which contains a single row with the version of the data source used to generate the database.

docs/word-search.md

@@ -0,0 +1,21 @@
+# Word search
+
+The word search procedure is currently split into 3 parts:
+
+1. **Entry ID query**:
+
+Use a complex query with various scoring factors to try to get list of
+database ids pointing at dictionary entries, sorted by how likely we think this
+word is the word that the caller is looking for. The output here is a `List<int>`
+
+2. **Data Query**:
+
+Takes the entry id list from the last search, and performs all queries needed to retrieve
+all the dictionary data for those IDs. The result is a struct with a bunch of flattened lists
+with data for all the dictionary entries. These lists are sorted by the order that the ids
+were provided.
+
+3. **Regrouping**:
+
+Takes the flattened data, and regroups the items into structs with a more "hierarchical" structure.
+All data tagged with the same ID will end up in the same struct. Returns a list of these structs.

flake.lock

@@ -3,7 +3,7 @@
     "jmdict-src": {
       "flake": false,
       "locked": {
-        "narHash": "sha256-84P7r/fFlBnawy6yChrD9WMHmOWcEGWUmoK70N4rdGQ=",
+        "narHash": "sha256-eOc3a/AYNRFF3w6lWhyf0Sh92xeXS7+9Qvn0tvvH6Ys=",
         "type": "file",
         "url": "http://ftp.edrdg.org/pub/Nihongo/JMdict_e.gz"
       },
@@ -15,7 +15,7 @@
     "jmdict-with-examples-src": {
       "flake": false,
       "locked": {
-        "narHash": "sha256-PM0sv7VcsCya2Ek02CI7hVwB3Jawn6bICSI+dsJK0yo=",
+        "narHash": "sha256-nx+WMkscWvA/XImKM7NESYVmICwSgXWOO1KPXasHY94=",
         "type": "file",
         "url": "http://ftp.edrdg.org/pub/Nihongo/JMdict_e_examp.gz"
       },
@@ -27,7 +27,7 @@
     "kanjidic2-src": {
       "flake": false,
       "locked": {
-        "narHash": "sha256-Lc0wUPpuDKuMDv2t87//w3z20RX8SMJI2iIRtUJ8fn0=",
+        "narHash": "sha256-2T/cAS/kZmVMURStgHVhz524+J9+v5onKs8eEYf2fY0=",
         "type": "file",
         "url": "https://www.edrdg.org/kanjidic/kanjidic2.xml.gz"
       },
@@ -38,11 +38,11 @@
     },
     "nixpkgs": {
       "locked": {
-        "lastModified": 1746904237,
-        "narHash": "sha256-3e+AVBczosP5dCLQmMoMEogM57gmZ2qrVSrmq9aResQ=",
+        "lastModified": 1774386573,
+        "narHash": "sha256-4hAV26quOxdC6iyG7kYaZcM3VOskcPUrdCQd/nx8obc=",
         "owner": "NixOS",
         "repo": "nixpkgs",
-        "rev": "d89fc19e405cb2d55ce7cc114356846a0ee5e956",
+        "rev": "46db2e09e1d3f113a13c0d7b81e2f221c63b8ce9",
         "type": "github"
       },
       "original": {
@@ -54,13 +54,13 @@
     "radkfile-src": {
       "flake": false,
       "locked": {
-        "narHash": "sha256-rO2z5GPt3g6osZOlpyWysmIbRV2Gw4AR4XvngVTHNpk=",
+        "narHash": "sha256-DHpMUE2Umje8PbzXUCS6pHZeXQ5+WTxbjSkGU3erDHQ=",
         "type": "file",
-        "url": "http://ftp.usf.edu/pub/ftp.monash.edu.au/pub/nihongo/radkfile.gz"
+        "url": "http://ftp.edrdg.org/pub/Nihongo/radkfile.gz"
       },
       "original": {
         "type": "file",
-        "url": "http://ftp.usf.edu/pub/ftp.monash.edu.au/pub/nihongo/radkfile.gz"
+        "url": "http://ftp.edrdg.org/pub/Nihongo/radkfile.gz"
       }
     },
     "root": {

flake.nix

@@ -16,7 +16,7 @@
     };
 
     radkfile-src = {
-      url = "http://ftp.usf.edu/pub/ftp.monash.edu.au/pub/nihongo/radkfile.gz";
+      url = "http://ftp.edrdg.org/pub/Nihongo/radkfile.gz";
       flake = false;
     };
 
@@ -43,7 +43,12 @@
       "armv7l-linux"
     ];
 
-    forAllSystems = f: lib.genAttrs systems (system: f system nixpkgs.legacyPackages.${system});
+    forAllSystems = f: lib.genAttrs systems (system: let
+      pkgs = import nixpkgs {
+        inherit system;
+        overlays = [ self.overlays.sqlite-icu-ext ];
+      };
+    in f system pkgs);
   in {
     apps = forAllSystems (system: pkgs: {
       default = {
@@ -77,21 +82,47 @@
 
     devShells = forAllSystems (system: pkgs: {
       default = pkgs.mkShell {
-        buildInputs = with pkgs; [
+        packages = with pkgs; [
           dart
           gnumake
-          sqlite-interactive
-          sqlite-web
-          sqlint
-          sqlfluff
+          lcov
+          sqldiff
+          sqlite-interactive-icu-ext
         ];
         env = {
           LIBSQLITE_PATH = "${pkgs.sqlite.out}/lib/libsqlite3.so";
           JADB_PATH = "result/jadb.sqlite";
+          LD_LIBRARY_PATH = lib.makeLibraryPath [ pkgs.sqlite ];
         };
       };
+
+      sqlite-debugging = pkgs.mkShell {
+        packages = with pkgs; [
+          sqlite-interactive-icu-ext
+          sqlite-analyzer
+          sqlite-web
+          sqlint
+          sqlfluff
+        ];
+      };
     });
 
+    overlays.sqlite-icu-ext = final: prev: let
+      overrideArgs = prev': {
+        configureFlags = prev'.configureFlags ++ [
+          "--with-icu-config=${lib.getExe' prev.icu.dev "icu-config"}"
+          "--enable-icu-collations"
+        ];
+
+        buildInputs = prev'.buildInputs ++ [
+          prev.icu
+        ];
+      };
+    in {
+      sqlite-icu-ext = prev.sqlite.overrideAttrs overrideArgs;
+      sqlite-interactive-icu-ext = prev.sqlite-interactive.overrideAttrs overrideArgs;
+    };
+
     packages = let
       edrdgMetadata = {
         license = [{
@@ -103,12 +134,32 @@
         platforms = lib.platforms.all;
       };
 
-      src = lib.cleanSource ./.;
+      src = builtins.filterSource (path: type: let
+        baseName = baseNameOf (toString path);
+      in !(lib.any (b: b) [
+          (!(lib.cleanSourceFilter path type))
+          (baseName == ".github" && type == "directory")
+          (baseName == ".gitea" && type == "directory")
+
+          (baseName == "nix" && type == "directory")
+          (baseName == ".envrc" && type == "regular")
+          (baseName == "flake.lock" && type == "regular")
+          (baseName == "flake.nix" && type == "regular")
+
+          (baseName == ".sqlfluff" && type == "regular")
+        ])) ./.;
+
     in forAllSystems (system: pkgs: {
       default = self.packages.${system}.database;
 
+      filteredSource = pkgs.runCommandLocal "filtered-source" { } ''
+        ln -s ${src} $out
+      '';
+
+      inherit (pkgs) sqlite-icu-ext sqlite-interactive-icu-ext;
+
       jmdict = pkgs.callPackage ./nix/jmdict.nix {
-          inherit jmdict-src jmdict-with-examples-src edrdgMetadata;
+        inherit jmdict-src jmdict-with-examples-src edrdgMetadata;
       };
 
       radkfile =  pkgs.callPackage ./nix/radkfile.nix {
@@ -121,11 +172,20 @@
 
       database-tool = pkgs.callPackage ./nix/database_tool.nix {
         inherit src;
+        sqlite = pkgs.sqlite-icu-ext;
       };
 
       database = pkgs.callPackage ./nix/database.nix {
         inherit (self.packages.${system}) database-tool jmdict radkfile kanjidic2;
         inherit src;
+        sqlite = pkgs.sqlite-icu-ext;
+      };
+
+      database-wal = pkgs.callPackage ./nix/database.nix {
+        inherit (self.packages.${system}) database-tool jmdict radkfile kanjidic2;
+        inherit src;
+        wal = true;
+        sqlite = pkgs.sqlite-icu-ext;
       };
 
       docs = pkgs.callPackage ./nix/docs.nix {

lib/_data_ingestion/jmdict/objects.dart

@@ -16,14 +16,15 @@ abstract class Element extends SQLWritable {
     this.nf,
   });
 
+  @override
   Map<String, Object?> get sqlValue => {
-        'reading': reading,
-        'news': news,
-        'ichi': ichi,
-        'spec': spec,
-        'gai': gai,
-        'nf': nf,
-      };
+    'reading': reading,
+    'news': news,
+    'ichi': ichi,
+    'spec': spec,
+    'gai': gai,
+    'nf': nf,
+  };
 }
 
 class KanjiElement extends Element {
@@ -33,26 +34,19 @@ class KanjiElement extends Element {
   KanjiElement({
     this.info = const [],
     required this.orderNum,
-    required String reading,
-    int? news,
-    int? ichi,
-    int? spec,
-    int? gai,
-    int? nf,
-  }) : super(
-          reading: reading,
-          news: news,
-          ichi: ichi,
-          spec: spec,
-          gai: gai,
-          nf: nf,
-        );
+    required super.reading,
+    super.news,
+    super.ichi,
+    super.spec,
+    super.gai,
+    super.nf,
+  });
 
   @override
   Map<String, Object?> get sqlValue => {
-        ...super.sqlValue,
-        'orderNum': orderNum,
-      };
+    ...super.sqlValue,
+    'orderNum': orderNum,
+  };
 }
 
 class ReadingElement extends Element {
@@ -66,27 +60,20 @@ class ReadingElement extends Element {
     required this.readingDoesNotMatchKanji,
     this.info = const [],
     this.restrictions = const [],
-    required String reading,
-    int? news,
-    int? ichi,
-    int? spec,
-    int? gai,
-    int? nf,
-  }) : super(
-          reading: reading,
-          news: news,
-          ichi: ichi,
-          spec: spec,
-          gai: gai,
-          nf: nf,
-        );
+    required super.reading,
+    super.news,
+    super.ichi,
+    super.spec,
+    super.gai,
+    super.nf,
+  });
 
   @override
   Map<String, Object?> get sqlValue => {
-        ...super.sqlValue,
-        'orderNum': orderNum,
-        'readingDoesNotMatchKanji': readingDoesNotMatchKanji,
-      };
+    ...super.sqlValue,
+    'orderNum': orderNum,
+    'readingDoesNotMatchKanji': readingDoesNotMatchKanji,
+  };
 }
 
 class LanguageSource extends SQLWritable {
@@ -97,18 +84,18 @@ class LanguageSource extends SQLWritable {
 
   const LanguageSource({
     required this.language,
-    this.phrase,
+    required this.phrase,
     this.fullyDescribesSense = true,
     this.constructedFromSmallerWords = false,
   });
 
   @override
   Map<String, Object?> get sqlValue => {
-        'language': language,
-        'phrase': phrase,
-        'fullyDescribesSense': fullyDescribesSense,
-        'constructedFromSmallerWords': constructedFromSmallerWords,
-      };
+    'language': language,
+    'phrase': phrase,
+    'fullyDescribesSense': fullyDescribesSense,
+    'constructedFromSmallerWords': constructedFromSmallerWords,
+  };
 }
 
 class Glossary extends SQLWritable {
@@ -116,52 +103,45 @@ class Glossary extends SQLWritable {
   final String phrase;
   final String? type;
 
-  const Glossary({
-    required this.language,
-    required this.phrase,
-    this.type,
-  });
+  const Glossary({required this.language, required this.phrase, this.type});
 
+  @override
   Map<String, Object?> get sqlValue => {
-        'language': language,
-        'phrase': phrase,
-        'type': type,
-      };
+    'language': language,
+    'phrase': phrase,
+    'type': type,
+  };
 }
 
-final kanaRegex =
-    RegExp(r'^[\p{Script=Katakana}\p{Script=Hiragana}ー]+$', unicode: true);
+final kanaRegex = RegExp(
+  r'^[\p{Script=Katakana}\p{Script=Hiragana}ー]+$',
+  unicode: true,
+);
 
 class XRefParts {
   final String? kanjiRef;
   final String? readingRef;
   final int? senseOrderNum;
 
-  const XRefParts({
-    this.kanjiRef,
-    this.readingRef,
-    this.senseOrderNum,
-  }) : assert(kanjiRef != null || readingRef != null);
+  const XRefParts({this.kanjiRef, this.readingRef, this.senseOrderNum})
+    : assert(kanjiRef != null || readingRef != null);
 
   Map<String, Object?> toJson() => {
-        'kanjiRef': kanjiRef,
-        'readingRef': readingRef,
-        'senseOrderNum': senseOrderNum,
-      };
+    'kanjiRef': kanjiRef,
+    'readingRef': readingRef,
+    'senseOrderNum': senseOrderNum,
+  };
 }
 
 class XRef {
   final String entryId;
   final String reading;
 
-  const XRef({
-    required this.entryId,
-    required this.reading,
-  });
+  const XRef({required this.entryId, required this.reading});
 }
 
 class Sense extends SQLWritable {
-  final int id;
+  final int senseId;
   final int orderNum;
   final List<XRefParts> antonyms;
   final List<String> dialects;
@@ -176,7 +156,7 @@ class Sense extends SQLWritable {
   final List<XRefParts> seeAlso;
 
   const Sense({
-    required this.id,
+    required this.senseId,
     required this.orderNum,
     this.antonyms = const [],
     this.dialects = const [],
@@ -193,11 +173,12 @@ class Sense extends SQLWritable {
 
   @override
   Map<String, Object?> get sqlValue => {
-        'id': id,
-        'orderNum': orderNum,
-      };
+    'senseId': senseId,
+    'orderNum': orderNum,
+  };
 
-  bool get isEmpty => antonyms.isEmpty &&
+  bool get isEmpty =>
+      antonyms.isEmpty &&
       dialects.isEmpty &&
       fields.isEmpty &&
       info.isEmpty &&
@@ -211,17 +192,18 @@ class Sense extends SQLWritable {
 }
 
 class Entry extends SQLWritable {
-  final int id;
+  final int entryId;
   final List<KanjiElement> kanji;
   final List<ReadingElement> readings;
   final List<Sense> senses;
 
   const Entry({
-    required this.id,
+    required this.entryId,
     required this.kanji,
     required this.readings,
     required this.senses,
   });
 
-  Map<String, Object?> get sqlValue => {'id': id};
+  @override
+  Map<String, Object?> get sqlValue => {'entryId': entryId};
 }

lib/_data_ingestion/jmdict/seed_data.dart

@@ -2,35 +2,46 @@ import 'dart:collection';
 
 import 'package:collection/collection.dart';
 import 'package:jadb/_data_ingestion/jmdict/objects.dart';
-import 'package:jadb/_data_ingestion/jmdict/table_names.dart';
-import 'package:jadb/util/romaji_transliteration.dart';
+import 'package:jadb/table_names/jmdict.dart';
 import 'package:sqflite_common/sqlite_api.dart';
 
+/// A wrapper for the result of resolving an xref, which includes the resolved entry and a flag
+/// indicating whether the xref was ambiguous (i.e. could refer to multiple entries).
 class ResolvedXref {
   Entry entry;
   bool ambiguous;
+  int? senseOrderNum;
 
-  ResolvedXref(this.entry, this.ambiguous);
+  ResolvedXref(this.entry, this.ambiguous, senseOrderNum);
 }
 
+/// Resolves an xref (pair of kanji, optionally reading, and optionally sense number) to an a specific
+/// JMdict entry, if possible.
+///
+/// If the xref is ambiguous (i.e. it could refer to multiple entries), the
+/// first entry is returned, and the returned value is marked as ambiguous.
+///
+/// If the xref cannot be resolved to any entry at all, an exception is thrown.
 ResolvedXref resolveXref(
   SplayTreeMap<String, Set<Entry>> entriesByKanji,
   SplayTreeMap<String, Set<Entry>> entriesByReading,
   XRefParts xref,
 ) {
   List<Entry> candidateEntries = switch ((xref.kanjiRef, xref.readingRef)) {
-    (null, null) =>
-      throw Exception('Xref $xref has no kanji or reading reference'),
-    (String k, null) => entriesByKanji[k]!.toList(),
-    (null, String r) => entriesByReading[r]!.toList(),
-    (String k, String r) =>
+    (null, null) => throw Exception(
+      'Xref $xref has no kanji or reading reference',
+    ),
+    (final String k, null) => entriesByKanji[k]!.toList(),
+    (null, final String r) => entriesByReading[r]!.toList(),
+    (final String k, final String r) =>
       entriesByKanji[k]!.intersection(entriesByReading[r]!).toList(),
   };
 
   // Filter out entries that don't have the number of senses specified in the xref
   if (xref.senseOrderNum != null) {
-    candidateEntries
-        .retainWhere((entry) => entry.senses.length >= xref.senseOrderNum!);
+    candidateEntries.retainWhere(
+      (entry) => entry.senses.length >= xref.senseOrderNum!,
+    );
   }
 
   // If the xref has a reading ref but no kanji ref, and there are multiple
@@ -39,8 +50,9 @@ ResolvedXref resolveXref(
   if (xref.kanjiRef == null &&
       xref.readingRef != null &&
       candidateEntries.length > 1) {
-    final candidatesWithEmptyKanji =
-        candidateEntries.where((entry) => entry.kanji.length == 0).toList();
+    final candidatesWithEmptyKanji = candidateEntries
+        .where((entry) => entry.kanji.isEmpty)
+        .toList();
 
     if (candidatesWithEmptyKanji.isNotEmpty) {
       candidateEntries = candidatesWithEmptyKanji;
@@ -51,7 +63,7 @@ ResolvedXref resolveXref(
   // entry in case there are multiple candidates left.
   candidateEntries.sortBy<num>((entry) => entry.senses.length);
 
-  if (candidateEntries.length == 0) {
+  if (candidateEntries.isEmpty) {
     throw Exception(
       'SKIPPING: Xref $xref has ${candidateEntries.length} entries, '
       'kanjiRef: ${xref.kanjiRef}, readingRef: ${xref.readingRef}, '
@@ -63,84 +75,54 @@ ResolvedXref resolveXref(
       'kanjiRef: ${xref.kanjiRef}, readingRef: ${xref.readingRef}, '
       'senseOrderNum: ${xref.senseOrderNum}',
     );
-    return ResolvedXref(candidateEntries.first, true);
+
+    return ResolvedXref(candidateEntries.first, true, xref.senseOrderNum);
   } else {
-    return ResolvedXref(candidateEntries.first, false);
+    return ResolvedXref(candidateEntries.first, false, xref.senseOrderNum);
   }
 }
 
 Future<void> seedJMDictData(List<Entry> entries, Database db) async {
   print('  [JMdict] Batch 1 - Kanji and readings');
   Batch b = db.batch();
+
+  int elementId = 0;
   for (final e in entries) {
     b.insert(JMdictTableNames.entry, e.sqlValue);
+
     for (final k in e.kanji) {
-      b.insert(JMdictTableNames.kanjiElement, k.sqlValue..addAll({'entryId': e.id}));
-      // b.insert(
-      //   JMdictTableNames.entryByKana,
-      //   {'entryId': e.id, 'kana': transliterateKatakanaToHiragana(k.reading)},
-      //   // Some entries have the same reading twice with difference in katakana and hiragana
-      //   conflictAlgorithm: ConflictAlgorithm.ignore,
-      // );
+      elementId++;
+      b.insert(
+        JMdictTableNames.kanjiElement,
+        k.sqlValue..addAll({'entryId': e.entryId, 'elementId': elementId}),
+      );
+
       for (final i in k.info) {
-        b.insert(
-          JMdictTableNames.kanjiInfo,
-          {
-            'entryId': e.id,
-            'reading': k.reading,
-            'info': i,
-          },
-        );
+        b.insert(JMdictTableNames.kanjiInfo, {
+          'elementId': elementId,
+          'info': i,
+        });
       }
     }
+
     for (final r in e.readings) {
+      elementId++;
       b.insert(
         JMdictTableNames.readingElement,
-        r.sqlValue..addAll({'entryId': e.id}),
+        r.sqlValue..addAll({'entryId': e.entryId, 'elementId': elementId}),
       );
 
-      b.insert(
-        JMdictTableNames.entryByKana,
-        {
-          'entryId': e.id,
-          'kana': transliterateKanaToLatin(r.reading),
-        },
-        // Some entries have the same reading twice with difference in katakana and hiragana
-        conflictAlgorithm: ConflictAlgorithm.ignore,
-      );
       for (final i in r.info) {
-        b.insert(
-          JMdictTableNames.readingInfo,
-          {
-            'entryId': e.id,
-            'reading': r.reading,
-            'info': i,
-          },
-        );
+        b.insert(JMdictTableNames.readingInfo, {
+          'elementId': elementId,
+          'info': i,
+        });
       }
       for (final res in r.restrictions) {
-        b.insert(
-          JMdictTableNames.readingRestriction,
-          {
-            'entryId': e.id,
-            'reading': r.reading,
-            'restriction': res,
-          },
-        );
-      }
-    }
-
-    for (final s in e.senses) {
-      for (final g in s.glossary) {
-        b.insert(
-          JMdictTableNames.entryByEnglish,
-          {
-            'entryId': e.id,
-            'english': g.phrase,
-          },
-          // Some entries have the same reading twice with difference in katakana and hiragana
-          conflictAlgorithm: ConflictAlgorithm.ignore,
-        );
+        b.insert(JMdictTableNames.readingRestriction, {
+          'elementId': elementId,
+          'restriction': res,
+        });
       }
     }
   }
@@ -149,52 +131,58 @@ Future<void> seedJMDictData(List<Entry> entries, Database db) async {
 
   print('  [JMdict] Batch 2 - Senses');
   b = db.batch();
+
   for (final e in entries) {
     for (final s in e.senses) {
-      b.insert(JMdictTableNames.sense, s.sqlValue..addAll({'entryId': e.id}));
+      b.insert(
+        JMdictTableNames.sense,
+        s.sqlValue..addAll({'entryId': e.entryId}),
+      );
       for (final d in s.dialects) {
-        b.insert(JMdictTableNames.senseDialect, {'senseId': s.id, 'dialect': d});
+        b.insert(JMdictTableNames.senseDialect, {
+          'senseId': s.senseId,
+          'dialect': d,
+        });
       }
       for (final f in s.fields) {
-        b.insert(JMdictTableNames.senseField, {'senseId': s.id, 'field': f});
+        b.insert(JMdictTableNames.senseField, {
+          'senseId': s.senseId,
+          'field': f,
+        });
       }
       for (final i in s.info) {
-        b.insert(JMdictTableNames.senseInfo, {'senseId': s.id, 'info': i});
+        b.insert(JMdictTableNames.senseInfo, {'senseId': s.senseId, 'info': i});
       }
       for (final m in s.misc) {
-        b.insert(JMdictTableNames.senseMisc, {'senseId': s.id, 'misc': m});
+        b.insert(JMdictTableNames.senseMisc, {'senseId': s.senseId, 'misc': m});
       }
       for (final p in s.pos) {
-        b.insert(JMdictTableNames.sensePOS, {'senseId': s.id, 'pos': p});
-      }
-      for (final l in s.languageSource) {
-        b.insert(
-          JMdictTableNames.senseLanguageSource,
-          l.sqlValue..addAll({'senseId': s.id}),
-        );
+        b.insert(JMdictTableNames.sensePOS, {'senseId': s.senseId, 'pos': p});
       }
       for (final rk in s.restrictedToKanji) {
-        b.insert(
-          JMdictTableNames.senseRestrictedToKanji,
-          {'entryId': e.id, 'senseId': s.id, 'kanji': rk},
-        );
+        b.insert(JMdictTableNames.senseRestrictedToKanji, {
+          'entryId': e.entryId,
+          'senseId': s.senseId,
+          'kanjiOrderNum': e.kanji.indexWhere((k) => k.reading == rk) + 1,
+        });
       }
       for (final rr in s.restrictedToReading) {
-        b.insert(
-          JMdictTableNames.senseRestrictedToReading,
-          {'entryId': e.id, 'senseId': s.id, 'reading': rr},
-        );
+        b.insert(JMdictTableNames.senseRestrictedToReading, {
+          'entryId': e.entryId,
+          'senseId': s.senseId,
+          'readingOrderNum': e.readings.indexWhere((r) => r.reading == rr) + 1,
+        });
       }
       for (final ls in s.languageSource) {
         b.insert(
           JMdictTableNames.senseLanguageSource,
-          ls.sqlValue..addAll({'senseId': s.id}),
+          ls.sqlValue..addAll({'senseId': s.senseId}),
         );
       }
       for (final g in s.glossary) {
         b.insert(
           JMdictTableNames.senseGlossary,
-          g.sqlValue..addAll({'senseId': s.id}),
+          g.sqlValue..addAll({'senseId': s.senseId}),
         );
       }
     }
@@ -203,24 +191,18 @@ Future<void> seedJMDictData(List<Entry> entries, Database db) async {
   await b.commit(noResult: true);
 
   print('  [JMdict] Building xref trees');
-  SplayTreeMap<String, Set<Entry>> entriesByKanji = SplayTreeMap();
+  final SplayTreeMap<String, Set<Entry>> entriesByKanji = SplayTreeMap();
+  final SplayTreeMap<String, Set<Entry>> entriesByReading = SplayTreeMap();
+
   for (final entry in entries) {
     for (final kanji in entry.kanji) {
-      if (entriesByKanji.containsKey(kanji.reading)) {
-        entriesByKanji.update(kanji.reading, (list) => list..add(entry));
-      } else {
-        entriesByKanji.putIfAbsent(kanji.reading, () => {entry});
-      }
+      entriesByKanji.putIfAbsent(kanji.reading, () => {});
+      entriesByKanji.update(kanji.reading, (set) => set..add(entry));
     }
-  }
-  SplayTreeMap<String, Set<Entry>> entriesByReading = SplayTreeMap();
-  for (final entry in entries) {
+
     for (final reading in entry.readings) {
-      if (entriesByReading.containsKey(reading.reading)) {
-        entriesByReading.update(reading.reading, (list) => list..add(entry));
-      } else {
-        entriesByReading.putIfAbsent(reading.reading, () => {entry});
-      }
+      entriesByReading.putIfAbsent(reading.reading, () => {});
+      entriesByReading.update(reading.reading, (set) => set..add(entry));
     }
   }
 
@@ -229,6 +211,7 @@ Future<void> seedJMDictData(List<Entry> entries, Database db) async {
 
   for (final e in entries) {
     for (final s in e.senses) {
+      final seenSeeAlsoXrefs = <int>{};
       for (final xref in s.seeAlso) {
         final resolvedEntry = resolveXref(
           entriesByKanji,
@@ -236,19 +219,24 @@ Future<void> seedJMDictData(List<Entry> entries, Database db) async {
           xref,
         );
 
-        b.insert(
-          JMdictTableNames.senseSeeAlso,
-          {
-            'senseId': s.id,
-            'xrefEntryId': resolvedEntry.entry.id,
-            'seeAlsoKanji': xref.kanjiRef,
-            'seeAlsoReading': xref.readingRef,
-            'seeAlsoSense': xref.senseOrderNum,
-            'ambiguous': resolvedEntry.ambiguous,
-          },
-        );
+        if (seenSeeAlsoXrefs.contains(resolvedEntry.entry.entryId)) {
+          print(
+            'WARNING: Skipping duplicate seeAlso xref from sense ${s.senseId} to entry ${resolvedEntry.entry.entryId}\n'
+            '  (kanjiRef: ${xref.kanjiRef}, readingRef: ${xref.readingRef}, senseOrderNum: ${xref.senseOrderNum})',
+          );
+          continue;
+        }
+        seenSeeAlsoXrefs.add(resolvedEntry.entry.entryId);
+
+        b.insert(JMdictTableNames.senseSeeAlso, {
+          'senseId': s.senseId,
+          'xrefEntryId': resolvedEntry.entry.entryId,
+          'xrefSenseOrderNum': resolvedEntry.senseOrderNum,
+          'ambiguous': resolvedEntry.ambiguous,
+        });
       }
 
+      final seenAntonymXrefs = <int>{};
       for (final ant in s.antonyms) {
         final resolvedEntry = resolveXref(
           entriesByKanji,
@@ -256,12 +244,18 @@ Future<void> seedJMDictData(List<Entry> entries, Database db) async {
           ant,
         );
 
+        if (seenAntonymXrefs.contains(resolvedEntry.entry.entryId)) {
+          print(
+            'WARNING: Skipping duplicate antonym xref from sense ${s.senseId} to entry ${resolvedEntry.entry.entryId}\n'
+            '  (kanjiRef: ${ant.kanjiRef}, readingRef: ${ant.readingRef}, senseOrderNum: ${ant.senseOrderNum})',
+          );
+          continue;
+        }
+        seenAntonymXrefs.add(resolvedEntry.entry.entryId);
+
         b.insert(JMdictTableNames.senseAntonyms, {
-          'senseId': s.id,
-          'xrefEntryId': resolvedEntry.entry.id,
-          'antonymKanji': ant.kanjiRef,
-          'antonymReading': ant.readingRef,
-          'antonymSense': ant.senseOrderNum,
+          'senseId': s.senseId,
+          'xrefEntryId': resolvedEntry.entry.entryId,
           'ambiguous': resolvedEntry.ambiguous,
         });
       }

lib/_data_ingestion/jmdict/xml_parser.dart

@@ -8,15 +8,17 @@ List<int?> getPriorityValues(XmlElement e, String prefix) {
   int? news, ichi, spec, gai, nf;
   for (final pri in e.findElements('${prefix}_pri')) {
     final txt = pri.innerText;
-    if (txt.startsWith('news'))
+    if (txt.startsWith('news')) {
       news = int.parse(txt.substring(4));
-    else if (txt.startsWith('ichi'))
+    } else if (txt.startsWith('ichi')) {
       ichi = int.parse(txt.substring(4));
-    else if (txt.startsWith('spec'))
+    } else if (txt.startsWith('spec')) {
       spec = int.parse(txt.substring(4));
-    else if (txt.startsWith('gai'))
+    } else if (txt.startsWith('gai')) {
       gai = int.parse(txt.substring(3));
-    else if (txt.startsWith('nf')) nf = int.parse(txt.substring(2));
+    } else if (txt.startsWith('nf')) {
+      nf = int.parse(txt.substring(2));
+    }
   }
   return [news, ichi, spec, gai, nf];
 }
@@ -46,10 +48,7 @@ XRefParts parseXrefParts(String s) {
           );
         }
       } else {
-        result = XRefParts(
-          kanjiRef: parts[0],
-          readingRef: parts[1],
-        );
+        result = XRefParts(kanjiRef: parts[0], readingRef: parts[1]);
       }
       break;
 
@@ -81,45 +80,48 @@ List<Entry> parseJMDictData(XmlElement root) {
     final List<ReadingElement> readingEls = [];
     final List<Sense> senses = [];
 
-    for (final (kanjiNum, k_ele) in entry.findElements('k_ele').indexed) {
-      final ke_pri = getPriorityValues(k_ele, 'ke');
+    for (final (kanjiNum, kEle) in entry.findElements('k_ele').indexed) {
+      final kePri = getPriorityValues(kEle, 'ke');
       kanjiEls.add(
         KanjiElement(
           orderNum: kanjiNum + 1,
-          info: k_ele
+          info: kEle
               .findElements('ke_inf')
               .map((e) => e.innerText.substring(1, e.innerText.length - 1))
               .toList(),
-          reading: k_ele.findElements('keb').first.innerText,
-          news: ke_pri[0],
-          ichi: ke_pri[1],
-          spec: ke_pri[2],
-          gai: ke_pri[3],
-          nf: ke_pri[4],
+          reading: kEle.findElements('keb').first.innerText,
+          news: kePri[0],
+          ichi: kePri[1],
+          spec: kePri[2],
+          gai: kePri[3],
+          nf: kePri[4],
         ),
       );
     }
 
-    for (final (orderNum, r_ele) in entry.findElements('r_ele').indexed) {
-      final re_pri = getPriorityValues(r_ele, 're');
-      final readingDoesNotMatchKanji =
-          r_ele.findElements('re_nokanji').isNotEmpty;
+    for (final (orderNum, rEle) in entry.findElements('r_ele').indexed) {
+      final rePri = getPriorityValues(rEle, 're');
+      final readingDoesNotMatchKanji = rEle
+          .findElements('re_nokanji')
+          .isNotEmpty;
       readingEls.add(
         ReadingElement(
           orderNum: orderNum + 1,
           readingDoesNotMatchKanji: readingDoesNotMatchKanji,
-          info: r_ele
+          info: rEle
               .findElements('re_inf')
               .map((e) => e.innerText.substring(1, e.innerText.length - 1))
               .toList(),
-          restrictions:
-              r_ele.findElements('re_restr').map((e) => e.innerText).toList(),
-          reading: r_ele.findElements('reb').first.innerText,
-          news: re_pri[0],
-          ichi: re_pri[1],
-          spec: re_pri[2],
-          gai: re_pri[3],
-          nf: re_pri[4],
+          restrictions: rEle
+              .findElements('re_restr')
+              .map((e) => e.innerText)
+              .toList(),
+          reading: rEle.findElements('reb').first.innerText,
+          news: rePri[0],
+          ichi: rePri[1],
+          spec: rePri[2],
+          gai: rePri[3],
+          nf: rePri[4],
         ),
       );
     }
@@ -127,12 +129,16 @@ List<Entry> parseJMDictData(XmlElement root) {
     for (final (orderNum, sense) in entry.findElements('sense').indexed) {
       senseId++;
       final result = Sense(
-        id: senseId,
+        senseId: senseId,
         orderNum: orderNum + 1,
-        restrictedToKanji:
-            sense.findElements('stagk').map((e) => e.innerText).toList(),
-        restrictedToReading:
-            sense.findElements('stagr').map((e) => e.innerText).toList(),
+        restrictedToKanji: sense
+            .findElements('stagk')
+            .map((e) => e.innerText)
+            .toList(),
+        restrictedToReading: sense
+            .findElements('stagr')
+            .map((e) => e.innerText)
+            .toList(),
         pos: sense
             .findElements('pos')
             .map((e) => e.innerText.substring(1, e.innerText.length - 1))
@@ -151,6 +157,7 @@ List<Entry> parseJMDictData(XmlElement root) {
             .map(
               (e) => LanguageSource(
                 language: e.getAttribute('xml:lang') ?? 'eng',
+                phrase: e.innerText.isNotEmpty ? e.innerText : null,
                 fullyDescribesSense: e.getAttribute('ls_type') == 'part',
                 constructedFromSmallerWords: e.getAttribute('ls_wasei') == 'y',
               ),
@@ -189,7 +196,7 @@ List<Entry> parseJMDictData(XmlElement root) {
 
     entries.add(
       Entry(
-        id: entryId,
+        entryId: entryId,
         kanji: kanjiEls,
         readings: readingEls,
         senses: senses,

lib/_data_ingestion/kanjidic/objects.dart

@@ -13,45 +13,33 @@ class CodePoint extends SQLWritable {
 
   @override
   Map<String, Object?> get sqlValue => {
-        'kanji': kanji,
-        'type': type,
-        'codepoint': codepoint,
-      };
+    'kanji': kanji,
+    'type': type,
+    'codepoint': codepoint,
+  };
 }
 
 class Radical extends SQLWritable {
   final String kanji;
-  final String type;
-  final String radical;
+  final int radicalId;
 
-  const Radical({
-    required this.kanji,
-    required this.type,
-    required this.radical,
-  });
+  const Radical({required this.kanji, required this.radicalId});
 
   @override
-  Map<String, Object?> get sqlValue => {
-        'kanji': kanji,
-        'type': type,
-        'radical': radical,
-      };
+  Map<String, Object?> get sqlValue => {'kanji': kanji, 'radicalId': radicalId};
 }
 
 class StrokeMiscount extends SQLWritable {
   final String kanji;
   final int strokeCount;
 
-  const StrokeMiscount({
-    required this.kanji,
-    required this.strokeCount,
-  });
+  const StrokeMiscount({required this.kanji, required this.strokeCount});
 
   @override
   Map<String, Object?> get sqlValue => {
-        'kanji': kanji,
-        'strokeCount': strokeCount,
-      };
+    'kanji': kanji,
+    'strokeCount': strokeCount,
+  };
 }
 
 class Variant extends SQLWritable {
@@ -67,10 +55,10 @@ class Variant extends SQLWritable {
 
   @override
   Map<String, Object?> get sqlValue => {
-        'kanji': kanji,
-        'type': type,
-        'variant': variant,
-      };
+    'kanji': kanji,
+    'type': type,
+    'variant': variant,
+  };
 }
 
 class DictionaryReference extends SQLWritable {
@@ -86,10 +74,10 @@ class DictionaryReference extends SQLWritable {
 
   @override
   Map<String, Object?> get sqlValue => {
-        'kanji': kanji,
-        'type': type,
-        'ref': ref,
-      };
+    'kanji': kanji,
+    'type': type,
+    'ref': ref,
+  };
 }
 
 class DictionaryReferenceMoro extends SQLWritable {
@@ -107,11 +95,11 @@ class DictionaryReferenceMoro extends SQLWritable {
 
   @override
   Map<String, Object?> get sqlValue => {
-        'kanji': kanji,
-        'ref': ref,
-        'volume': volume,
-        'page': page,
-      };
+    'kanji': kanji,
+    'ref': ref,
+    'volume': volume,
+    'page': page,
+  };
 }
 
 class QueryCode extends SQLWritable {
@@ -129,11 +117,11 @@ class QueryCode extends SQLWritable {
 
   @override
   Map<String, Object?> get sqlValue => {
-        'kanji': kanji,
-        'code': code,
-        'type': type,
-        'skipMisclassification': skipMisclassification,
-      };
+    'kanji': kanji,
+    'code': code,
+    'type': type,
+    'skipMisclassification': skipMisclassification,
+  };
 }
 
 class Reading extends SQLWritable {
@@ -149,10 +137,10 @@ class Reading extends SQLWritable {
 
   @override
   Map<String, Object?> get sqlValue => {
-        'kanji': kanji,
-        'type': type,
-        'reading': reading,
-      };
+    'kanji': kanji,
+    'type': type,
+    'reading': reading,
+  };
 }
 
 class Kunyomi extends SQLWritable {
@@ -168,10 +156,10 @@ class Kunyomi extends SQLWritable {
 
   @override
   Map<String, Object?> get sqlValue => {
-        'kanji': kanji,
-        'yomi': yomi,
-        'isJouyou': isJouyou,
-      };
+    'kanji': kanji,
+    'yomi': yomi,
+    'isJouyou': isJouyou,
+  };
 }
 
 class Onyomi extends SQLWritable {
@@ -189,11 +177,11 @@ class Onyomi extends SQLWritable {
 
   @override
   Map<String, Object?> get sqlValue => {
-        'kanji': kanji,
-        'yomi': yomi,
-        'isJouyou': isJouyou,
-        'type': type,
-      };
+    'kanji': kanji,
+    'yomi': yomi,
+    'isJouyou': isJouyou,
+    'type': type,
+  };
 }
 
 class Meaning extends SQLWritable {
@@ -209,10 +197,10 @@ class Meaning extends SQLWritable {
 
   @override
   Map<String, Object?> get sqlValue => {
-        'kanji': kanji,
-        'language': language,
-        'meaning': meaning,
-      };
+    'kanji': kanji,
+    'language': language,
+    'meaning': meaning,
+  };
 }
 
 class Character extends SQLWritable {
@@ -224,7 +212,7 @@ class Character extends SQLWritable {
 
   final List<String> radicalName;
   final List<CodePoint> codepoints;
-  final List<Radical> radicals;
+  final Radical? radical;
   final List<int> strokeMiscounts;
   final List<Variant> variants;
   final List<DictionaryReference> dictionaryReferences;
@@ -244,7 +232,7 @@ class Character extends SQLWritable {
     this.jlpt,
     this.radicalName = const [],
     this.codepoints = const [],
-    this.radicals = const [],
+    required this.radical,
     this.strokeMiscounts = const [],
     this.variants = const [],
     this.dictionaryReferences = const [],
@@ -257,11 +245,12 @@ class Character extends SQLWritable {
     this.nanori = const [],
   });
 
+  @override
   Map<String, Object?> get sqlValue => {
-        'literal': literal,
-        'grade': grade,
-        'strokeCount': strokeCount,
-        'frequency': frequency,
-        'jlpt': jlpt,
-      };
+    'literal': literal,
+    'grade': grade,
+    'strokeCount': strokeCount,
+    'frequency': frequency,
+    'jlpt': jlpt,
+  };
 }

lib/_data_ingestion/kanjidic/seed_data.dart

@@ -1,4 +1,4 @@
-import 'package:jadb/_data_ingestion/kanjidic/table_names.dart';
+import 'package:jadb/table_names/kanjidic.dart';
 import 'package:sqflite_common/sqlite_api.dart';
 
 import 'objects.dart';
@@ -14,23 +14,27 @@ Future<void> seedKANJIDICData(List<Character> characters, Database db) async {
     //       print(c.dictionaryReferences.map((e) => e.sqlValue).toList());
     //     }
     b.insert(KANJIDICTableNames.character, c.sqlValue);
+
     for (final n in c.radicalName) {
-      b.insert(KANJIDICTableNames.radicalName, {'kanji': c.literal, 'name': n});
+      assert(c.radical != null, 'Radical name without radical');
+      b.insert(
+        KANJIDICTableNames.radicalName,
+        {'radicalId': c.radical!.radicalId, 'name': n},
+        conflictAlgorithm: ConflictAlgorithm.ignore,
+      );
     }
+
     for (final cp in c.codepoints) {
       b.insert(KANJIDICTableNames.codepoint, cp.sqlValue);
     }
-    for (final r in c.radicals) {
-      b.insert(KANJIDICTableNames.radical, r.sqlValue);
+    if (c.radical != null) {
+      b.insert(KANJIDICTableNames.radical, c.radical!.sqlValue);
     }
     for (final sm in c.strokeMiscounts) {
-      b.insert(
-        KANJIDICTableNames.strokeMiscount,
-        {
-          'kanji': c.literal,
-          'strokeCount': sm,
-        },
-      );
+      b.insert(KANJIDICTableNames.strokeMiscount, {
+        'kanji': c.literal,
+        'strokeCount': sm,
+      });
     }
     for (final v in c.variants) {
       b.insert(KANJIDICTableNames.variant, v.sqlValue);
@@ -52,23 +56,26 @@ Future<void> seedKANJIDICData(List<Character> characters, Database db) async {
     for (final r in c.readings) {
       b.insert(KANJIDICTableNames.reading, r.sqlValue);
     }
-    for (final k in c.kunyomi) {
-      b.insert(KANJIDICTableNames.kunyomi, k.sqlValue);
+    for (final (i, y) in c.kunyomi.indexed) {
+      b.insert(
+        KANJIDICTableNames.kunyomi,
+        y.sqlValue..addAll({'orderNum': i + 1}),
+      );
     }
-    for (final o in c.onyomi) {
-      b.insert(KANJIDICTableNames.onyomi, o.sqlValue);
+    for (final (i, y) in c.onyomi.indexed) {
+      b.insert(
+        KANJIDICTableNames.onyomi,
+        y.sqlValue..addAll({'orderNum': i + 1}),
+      );
     }
-    for (final m in c.meanings) {
-      b.insert(KANJIDICTableNames.meaning, m.sqlValue);
+    for (final (i, m) in c.meanings.indexed) {
+      b.insert(
+        KANJIDICTableNames.meaning,
+        m.sqlValue..addAll({'orderNum': i + 1}),
+      );
     }
     for (final n in c.nanori) {
-      b.insert(
-        KANJIDICTableNames.nanori,
-        {
-          'kanji': c.literal,
-          'nanori': n,
-        },
-      );
+      b.insert(KANJIDICTableNames.nanori, {'kanji': c.literal, 'nanori': n});
     }
   }
   await b.commit(noResult: true);

lib/_data_ingestion/kanjidic/xml_parser.dart

@@ -1,4 +1,5 @@
 import 'package:jadb/_data_ingestion/kanjidic/objects.dart';
+import 'package:jadb/util/romaji_transliteration.dart';
 import 'package:xml/xml.dart';
 
 List<Character> parseKANJIDICData(XmlElement root) {
@@ -9,30 +10,33 @@ List<Character> parseKANJIDICData(XmlElement root) {
     final codepoint = c.findElements('codepoint').firstOrNull;
     final radical = c.findElements('radical').firstOrNull;
     final misc = c.findElements('misc').first;
-    final dic_number = c.findElements('dic_number').firstOrNull;
-    final query_code = c.findElements('query_code').first;
-    final reading_meaning = c.findElements('reading_meaning').firstOrNull;
+    final dicNumber = c.findElements('dic_number').firstOrNull;
+    final queryCode = c.findElements('query_code').first;
+    final readingMeaning = c.findElements('reading_meaning').firstOrNull;
 
     // TODO: Group readings and meanings by their rmgroup parent node.
 
     result.add(
       Character(
         literal: kanji,
-        strokeCount:
-            int.parse(misc.findElements('stroke_count').first.innerText),
+        strokeCount: int.parse(
+          misc.findElements('stroke_count').first.innerText,
+        ),
         grade: int.tryParse(
-            misc.findElements('grade').firstOrNull?.innerText ?? ''),
+          misc.findElements('grade').firstOrNull?.innerText ?? '',
+        ),
         frequency: int.tryParse(
-            misc.findElements('freq').firstOrNull?.innerText ?? ''),
+          misc.findElements('freq').firstOrNull?.innerText ?? '',
+        ),
         jlpt: int.tryParse(
           misc.findElements('jlpt').firstOrNull?.innerText ?? '',
         ),
-        radicalName: radical
-                ?.findElements('rad_name')
-                .map((e) => e.innerText)
-                .toList() ??
-            [],
-        codepoints: codepoint
+        radicalName: misc
+            .findElements('rad_name')
+            .map((e) => e.innerText)
+            .toList(),
+        codepoints:
+            codepoint
                 ?.findElements('cp_value')
                 .map(
                   (e) => CodePoint(
@@ -43,17 +47,13 @@ List<Character> parseKANJIDICData(XmlElement root) {
                 )
                 .toList() ??
             [],
-        radicals: radical
-                ?.findElements('rad_value')
-                .map(
-                  (e) => Radical(
-                    kanji: kanji,
-                    type: e.getAttribute('rad_type')!,
-                    radical: e.innerText,
-                  ),
-                )
-                .toList() ??
-            [],
+        radical: radical
+            ?.findElements('rad_value')
+            .where((e) => e.getAttribute('rad_type') == 'classical')
+            .map(
+              (e) => Radical(kanji: kanji, radicalId: int.parse(e.innerText)),
+            )
+            .firstOrNull,
         strokeMiscounts: misc
             .findElements('stroke_count')
             .skip(1)
@@ -69,7 +69,8 @@ List<Character> parseKANJIDICData(XmlElement root) {
               ),
             )
             .toList(),
-        dictionaryReferences: dic_number
+        dictionaryReferences:
+            dicNumber
                 ?.findElements('dic_ref')
                 .where((e) => e.getAttribute('dr_type') != 'moro')
                 .map(
@@ -81,7 +82,8 @@ List<Character> parseKANJIDICData(XmlElement root) {
                 )
                 .toList() ??
             [],
-        dictionaryReferencesMoro: dic_number
+        dictionaryReferencesMoro:
+            dicNumber
                 ?.findElements('dic_ref')
                 .where((e) => e.getAttribute('dr_type') == 'moro')
                 .map(
@@ -94,7 +96,7 @@ List<Character> parseKANJIDICData(XmlElement root) {
                 )
                 .toList() ??
             [],
-        querycodes: query_code
+        querycodes: queryCode
             .findElements('q_code')
             .map(
               (e) => QueryCode(
@@ -105,7 +107,8 @@ List<Character> parseKANJIDICData(XmlElement root) {
               ),
             )
             .toList(),
-        readings: reading_meaning
+        readings:
+            readingMeaning
                 ?.findAllElements('reading')
                 .where(
                   (e) =>
@@ -120,7 +123,8 @@ List<Character> parseKANJIDICData(XmlElement root) {
                 )
                 .toList() ??
             [],
-        kunyomi: reading_meaning
+        kunyomi:
+            readingMeaning
                 ?.findAllElements('reading')
                 .where((e) => e.getAttribute('r_type') == 'ja_kun')
                 .map(
@@ -132,19 +136,22 @@ List<Character> parseKANJIDICData(XmlElement root) {
                 )
                 .toList() ??
             [],
-        onyomi: reading_meaning
+        onyomi:
+            readingMeaning
                 ?.findAllElements('reading')
                 .where((e) => e.getAttribute('r_type') == 'ja_on')
                 .map(
                   (e) => Onyomi(
-                      kanji: kanji,
-                      yomi: e.innerText,
-                      isJouyou: e.getAttribute('r_status') == 'jy',
-                      type: e.getAttribute('on_type')),
+                    kanji: kanji,
+                    yomi: transliterateKatakanaToHiragana(e.innerText),
+                    isJouyou: e.getAttribute('r_status') == 'jy',
+                    type: e.getAttribute('on_type'),
+                  ),
                 )
                 .toList() ??
             [],
-        meanings: reading_meaning
+        meanings:
+            readingMeaning
                 ?.findAllElements('meaning')
                 .map(
                   (e) => Meaning(
@@ -155,7 +162,8 @@ List<Character> parseKANJIDICData(XmlElement root) {
                 )
                 .toList() ??
             [],
-        nanori: reading_meaning
+        nanori:
+            readingMeaning
                 ?.findElements('nanori')
                 .map((e) => e.innerText)
                 .toList() ??

lib/_data_ingestion/open_local_db.dart

@@ -1,90 +1,41 @@
-import 'dart:ffi';
 import 'dart:io';
 
-import 'package:jadb/_data_ingestion/jmdict/table_names.dart';
-import 'package:jadb/_data_ingestion/kanjidic/table_names.dart';
-import 'package:jadb/_data_ingestion/radkfile/table_names.dart';
-import 'package:jadb/_data_ingestion/tanos-jlpt/table_names.dart';
+import 'package:jadb/search.dart';
 import 'package:sqflite_common_ffi/sqflite_ffi.dart';
-import 'package:sqlite3/open.dart';
 
 Future<Database> openLocalDb({
   String? libsqlitePath,
   String? jadbPath,
   bool readWrite = false,
-  bool assertTablesExist = true,
+  bool verifyTablesExist = true,
+  bool walMode = false,
 }) async {
-  libsqlitePath ??= Platform.environment['LIBSQLITE_PATH'];
   jadbPath ??= Platform.environment['JADB_PATH'];
   jadbPath ??= Directory.current.uri.resolve('jadb.sqlite').path;
 
-  libsqlitePath = (libsqlitePath == null)
-      ? null
-      : File(libsqlitePath).resolveSymbolicLinksSync();
   jadbPath = File(jadbPath).resolveSymbolicLinksSync();
 
-  if (libsqlitePath == null) {
-    throw Exception("LIBSQLITE_PATH is not set");
-  }
-
-  if (!File(libsqlitePath).existsSync()) {
-    throw Exception("LIBSQLITE_PATH does not exist: $libsqlitePath");
-  }
-
   if (!File(jadbPath).existsSync()) {
-    throw Exception("JADB_PATH does not exist: $jadbPath");
+    throw Exception('JADB_PATH does not exist: $jadbPath');
   }
 
-  final db = await createDatabaseFactoryFfi(
-    ffiInit: () =>
-        open.overrideForAll(() => DynamicLibrary.open(libsqlitePath!)),
-  ).openDatabase(
+  final db = await createDatabaseFactoryFfi().openDatabase(
     jadbPath,
     options: OpenDatabaseOptions(
-      onOpen: (db) {
-        db.execute("PRAGMA foreign_keys = ON");
+      onConfigure: (db) async {
+        if (walMode) {
+          await db.execute('PRAGMA journal_mode = WAL');
+        }
+        await db.execute('PRAGMA foreign_keys = ON');
+        await db.execute("SELECT icu_load_collation('ja_JP', 'japanese')");
       },
       readOnly: !readWrite,
     ),
   );
 
-  if (assertTablesExist) {
-    await _assertTablesExist(db);
+  if (verifyTablesExist) {
+    await db.jadbVerifyTables();
   }
 
   return db;
 }
-
-Future<void> _assertTablesExist(Database db) async {
-  final Set<String> tables = await db
-      .query(
-    'sqlite_master',
-    columns: ['name'],
-    where: 'type = ?',
-    whereArgs: ['table'],
-  )
-      .then((result) {
-    return result.map((row) => row['name'] as String).toSet();
-  });
-
-  final Set<String> expectedTables = {
-    ...JMdictTableNames.allTables,
-    ...KANJIDICTableNames.allTables,
-    ...RADKFILETableNames.allTables,
-    ...TanosJLPTTableNames.allTables,
-  };
-
-  final missingTables = expectedTables.difference(tables);
-
-  if (missingTables.isNotEmpty) {
-    throw Exception([
-      'Missing tables:',
-      missingTables.map((table) => '  - $table').join('\n'),
-      '',
-      'Found tables:\n',
-      tables.map((table) => '  - $table').join('\n'),
-      '',
-      'Please ensure the database is correctly set up.',
-    ].join('\n'));
-  }
-}

lib/_data_ingestion/radkfile/parser.dart

@@ -3,8 +3,10 @@ import 'dart:io';
 Iterable<String> parseRADKFILEBlocks(File radkfile) {
   final String content = File('data/tmp/radkfile_utf8').readAsStringSync();
 
-  final Iterable<String> blocks =
-      content.replaceAll(RegExp(r'^#.*$'), '').split(r'$').skip(2);
+  final Iterable<String> blocks = content
+      .replaceAll(RegExp(r'^#.*$'), '')
+      .split(r'$')
+      .skip(2);
 
   return blocks;
 }

lib/_data_ingestion/radkfile/seed_data.dart

@@ -1,27 +1,20 @@
-import 'package:jadb/_data_ingestion/radkfile/table_names.dart';
+import 'package:jadb/table_names/radkfile.dart';
 import 'package:sqflite_common/sqlite_api.dart';
 
-Future<void> seedRADKFILEData(
-  Iterable<String> blocks,
-  Database db,
-) async {
+Future<void> seedRADKFILEData(Iterable<String> blocks, Database db) async {
   final b = db.batch();
 
   for (final block in blocks) {
     final String radical = block[1];
-    final List<String> kanjiList = block
-        .replaceFirst(RegExp(r'.*\n'), '')
-        .split('')
-      ..removeWhere((e) => e == '' || e == '\n');
+    final List<String> kanjiList =
+        block.replaceFirst(RegExp(r'.*\n'), '').split('')
+          ..removeWhere((e) => e == '' || e == '\n');
 
     for (final kanji in kanjiList.toSet()) {
-      b.insert(
-        RADKFILETableNames.radkfile,
-        {
-          'radical': radical,
-          'kanji': kanji,
-        },
-      );
+      b.insert(RADKFILETableNames.radkfile, {
+        'radical': radical,
+        'kanji': kanji,
+      });
     }
   }
 

lib/_data_ingestion/radkfile/table_names.dart

@@ -1,7 +0,0 @@
-abstract class RADKFILETableNames {
-  static const String radkfile = 'RADKFILE';
-
-  static Set<String> get allTables => {
-        radkfile,
-      };
-}

lib/_data_ingestion/seed_database.dart

@@ -17,14 +17,17 @@ Future<void> seedData(Database db) async {
   await parseAndSeedDataFromRADKFILE(db);
   await parseAndSeedDataFromKANJIDIC(db);
   await parseAndSeedDataFromTanosJLPT(db);
+
+  print('Performing VACUUM');
+  await db.execute('VACUUM');
 }
 
 Future<void> parseAndSeedDataFromJMdict(Database db) async {
   print('[JMdict] Reading file content...');
-  String rawXML = File('data/tmp/JMdict.xml').readAsStringSync();
+  final String rawXML = File('data/tmp/JMdict.xml').readAsStringSync();
 
   print('[JMdict] Parsing XML tags...');
-  XmlElement root = XmlDocument.parse(rawXML).getElement('JMdict')!;
+  final XmlElement root = XmlDocument.parse(rawXML).getElement('JMdict')!;
 
   print('[JMdict] Parsing XML content...');
   final entries = parseJMDictData(root);
@@ -35,10 +38,10 @@ Future<void> parseAndSeedDataFromJMdict(Database db) async {
 
 Future<void> parseAndSeedDataFromKANJIDIC(Database db) async {
   print('[KANJIDIC2] Reading file...');
-  String rawXML = File('data/tmp/kanjidic2.xml').readAsStringSync();
+  final String rawXML = File('data/tmp/kanjidic2.xml').readAsStringSync();
 
   print('[KANJIDIC2] Parsing XML...');
-  XmlElement root = XmlDocument.parse(rawXML).getElement('kanjidic2')!;
+  final XmlElement root = XmlDocument.parse(rawXML).getElement('kanjidic2')!;
 
   print('[KANJIDIC2] Parsing XML content...');
   final entries = parseKANJIDICData(root);
@@ -49,7 +52,7 @@ Future<void> parseAndSeedDataFromKANJIDIC(Database db) async {
 
 Future<void> parseAndSeedDataFromRADKFILE(Database db) async {
   print('[RADKFILE] Reading file...');
-  File raw = File('data/tmp/RADKFILE');
+  final File raw = File('data/tmp/RADKFILE');
 
   print('[RADKFILE] Parsing content...');
   final blocks = parseRADKFILEBlocks(raw);
@@ -60,7 +63,7 @@ Future<void> parseAndSeedDataFromRADKFILE(Database db) async {
 
 Future<void> parseAndSeedDataFromTanosJLPT(Database db) async {
   print('[TANOS-JLPT] Reading files...');
-  Map<String, File> files = {
+  final Map<String, File> files = {
     'N1': File('data/tanos-jlpt/n1.csv'),
     'N2': File('data/tanos-jlpt/n2.csv'),
     'N3': File('data/tanos-jlpt/n3.csv'),

lib/_data_ingestion/sql_writable.dart

@@ -3,7 +3,7 @@ abstract class SQLWritable {
   const SQLWritable();
 
   /// Returns a map of the object's properties and their values.
-  /// 
+  ///
   /// Note that there might be properties in the object which is meant to be
   /// inserted into a different table. These properties will/should be excluded
   /// from this map.

lib/_data_ingestion/tanos-jlpt/csv_parser.dart

@@ -3,52 +3,64 @@ import 'dart:io';
 
 import 'package:csv/csv.dart';
 import 'package:jadb/_data_ingestion/tanos-jlpt/objects.dart';
+import 'package:xml/xml_events.dart';
 
 Future<List<JLPTRankedWord>> parseJLPTRankedWords(
   Map<String, File> files,
 ) async {
   final List<JLPTRankedWord> result = [];
 
+  final codec = Csv(
+    fieldDelimiter: ',',
+    lineDelimiter: '\n',
+    quoteMode: QuoteMode.strings,
+    escapeCharacter: '\\',
+    parseHeaders: false,
+  );
+
   for (final entry in files.entries) {
     final jlptLevel = entry.key;
     final file = entry.value;
 
     if (!file.existsSync()) {
-      throw Exception("File $jlptLevel does not exist");
+      throw Exception('File $jlptLevel does not exist');
     }
 
-    final rows = await file
+    final words = await file
         .openRead()
         .transform(utf8.decoder)
-        .transform(CsvToListConverter())
+        .transform(codec.decoder)
+        .map((row) {
+          if (row.length != 3) {
+            throw Exception('Invalid line in $jlptLevel: $row');
+          }
+          return row;
+        })
+        .map((row) => row.map((e) => e as String).toList())
+        .map((row) {
+          final kanji = row[0].isEmpty
+              ? null
+              : row[0]
+                    .replaceFirst(RegExp('^お・'), '')
+                    .replaceAll(RegExp(r'（.*）'), '');
+
+          final readings = row[1]
+              .split(RegExp('[・/、(:?s+)]'))
+              .map((e) => e.trim())
+              .toList();
+
+          final meanings = row[2].split(',').expand(cleanMeaning).toList();
+
+          return JLPTRankedWord(
+            readings: readings,
+            kanji: kanji,
+            jlptLevel: jlptLevel,
+            meanings: meanings,
+          );
+        })
         .toList();
 
-    for (final row in rows) {
-      if (row.length != 3) {
-        throw Exception("Invalid line in $jlptLevel: $row");
-      }
-
-      final kanji = (row[0] as String).isEmpty
-          ? null
-          : (row[0] as String)
-              .replaceFirst(RegExp('^お・'), '')
-              .replaceAll(RegExp(r'（.*）'), '');
-
-      final readings = (row[1] as String)
-          .split(RegExp('[・/、(:?\s+)]'))
-          .map((e) => e.trim())
-          .toList();
-
-      final meanings =
-          (row[2] as String).split(',').expand(cleanMeaning).toList();
-
-      result.add(JLPTRankedWord(
-        readings: readings,
-        kanji: kanji,
-        jlptLevel: jlptLevel,
-        meanings: meanings,
-      ));
-    }
+    result.addAll(words);
   }
 
   return result;

lib/_data_ingestion/tanos-jlpt/objects.dart

@@ -13,5 +13,5 @@ class JLPTRankedWord {
 
   @override
   String toString() =>
-      '(${jlptLevel},${kanji},"${readings.join(",")}","${meanings.join(",")})';
+      '($jlptLevel,$kanji,"${readings.join(",")}","${meanings.join(",")})';
 }

lib/_data_ingestion/tanos-jlpt/overrides.dart

@@ -1,4 +1,4 @@
-const Map<(String?, String), int?> TANOS_JLPT_OVERRIDES = {
+const Map<(String?, String), int?> tanosJLPTOverrides = {
   // N5:
   (null, 'あなた'): 1223615,
   (null, 'あの'): 1000430,

lib/_data_ingestion/tanos-jlpt/resolve.dart

@@ -1,47 +1,43 @@
-import 'package:jadb/_data_ingestion/jmdict/table_names.dart';
 import 'package:jadb/_data_ingestion/tanos-jlpt/objects.dart';
 import 'package:jadb/_data_ingestion/tanos-jlpt/overrides.dart';
-import 'package:jadb/util/sqlite_utils.dart';
+import 'package:jadb/table_names/jmdict.dart';
 import 'package:sqflite_common/sqlite_api.dart';
 
-Future<List<int>> _findReadingCandidates(
-  JLPTRankedWord word,
-  Database db,
-) =>
-    db
-        .query(
-          JMdictTableNames.readingElement,
-          columns: ['entryId'],
-          where:
-              'reading IN (${word.readings.map((e) => escapeStringValue(e)).join(',')})',
-        )
-        .then((rows) => rows.map((row) => row['entryId'] as int).toList());
+Future<List<int>> _findReadingCandidates(JLPTRankedWord word, Database db) => db
+    .query(
+      JMdictTableNames.readingElement,
+      columns: ['entryId'],
+      where:
+          '"reading" IN (${List.filled(word.readings.length, '?').join(',')})',
+      whereArgs: [...word.readings],
+    )
+    .then((rows) => rows.map((row) => row['entryId'] as int).toList());
 
-Future<List<int>> _findKanjiCandidates(
-  JLPTRankedWord word,
-  Database db,
-) =>
-    db
-        .query(
-          JMdictTableNames.kanjiElement,
-          columns: ['entryId'],
-          where: 'reading = ?',
-          whereArgs: [word.kanji],
-        )
-        .then((rows) => rows.map((row) => row['entryId'] as int).toList());
+Future<List<int>> _findKanjiCandidates(JLPTRankedWord word, Database db) => db
+    .query(
+      JMdictTableNames.kanjiElement,
+      columns: ['entryId'],
+      where: 'reading = ?',
+      whereArgs: [word.kanji],
+    )
+    .then((rows) => rows.map((row) => row['entryId'] as int).toList());
 
 Future<List<(int, String)>> _findSenseCandidates(
   JLPTRankedWord word,
   Database db,
-) =>
-    db
-        .rawQuery('SELECT entryId, phrase '
-            'FROM ${JMdictTableNames.senseGlossary} '
-            'JOIN ${JMdictTableNames.sense} ON ${JMdictTableNames.senseGlossary}.senseId = ${JMdictTableNames.sense}.id '
-            'WHERE phrase IN (${word.meanings.map((e) => escapeStringValue(e)).join(',')})')
-        .then((rows) => rows
-            .map((row) => (row['entryId'] as int, row['phrase'] as String))
-            .toList());
+) => db
+    .rawQuery(
+      'SELECT entryId, phrase '
+      'FROM "${JMdictTableNames.senseGlossary}" '
+      'JOIN "${JMdictTableNames.sense}" USING (senseId)'
+      'WHERE phrase IN (${List.filled(word.meanings.length, '?').join(',')})',
+      [...word.meanings],
+    )
+    .then(
+      (rows) => rows
+          .map((row) => (row['entryId'] as int, row['phrase'] as String))
+          .toList(),
+    );
 
 Future<int?> findEntry(
   JLPTRankedWord word,
@@ -49,8 +45,10 @@ Future<int?> findEntry(
   bool useOverrides = true,
 }) async {
   final List<int> readingCandidates = await _findReadingCandidates(word, db);
-  final List<(int, String)> senseCandidates =
-      await _findSenseCandidates(word, db);
+  final List<(int, String)> senseCandidates = await _findSenseCandidates(
+    word,
+    db,
+  );
 
   List<int> entryIds;
 
@@ -65,8 +63,10 @@ Future<int?> findEntry(
       print('No entry found, trying to combine with senses');
 
       entryIds = readingCandidates
-          .where((readingId) =>
-              senseCandidates.any((sense) => sense.$1 == readingId))
+          .where(
+            (readingId) =>
+                senseCandidates.any((sense) => sense.$1 == readingId),
+          )
           .toList();
     }
   } else {
@@ -76,18 +76,21 @@ Future<int?> findEntry(
   if ((entryIds.isEmpty || entryIds.length > 1) && useOverrides) {
     print('No entry found, trying to fetch from overrides');
     final overrideEntries = word.readings
-        .map((reading) => TANOS_JLPT_OVERRIDES[(word.kanji, reading)])
+        .map((reading) => tanosJLPTOverrides[(word.kanji, reading)])
         .whereType<int>()
         .toSet();
 
     if (overrideEntries.length > 1) {
       throw Exception(
-          'Multiple override entries found for ${word.toString()}: $entryIds');
-    } else if (overrideEntries.length == 0 &&
-        !word.readings.any((reading) =>
-            TANOS_JLPT_OVERRIDES.containsKey((word.kanji, reading)))) {
+        'Multiple override entries found for ${word.toString()}: $entryIds',
+      );
+    } else if (overrideEntries.isEmpty &&
+        !word.readings.any(
+          (reading) => tanosJLPTOverrides.containsKey((word.kanji, reading)),
+        )) {
       throw Exception(
-          'No override entry found for ${word.toString()}: $entryIds');
+        'No override entry found for ${word.toString()}: $entryIds',
+      );
     }
 
     print('Found override: ${overrideEntries.firstOrNull}');
@@ -97,7 +100,8 @@ Future<int?> findEntry(
 
   if (entryIds.length > 1) {
     throw Exception(
-        'Multiple override entries found for ${word.toString()}: $entryIds');
+      'Multiple override entries found for ${word.toString()}: $entryIds',
+    );
   } else if (entryIds.isEmpty) {
     throw Exception('No entry found for ${word.toString()}');
   }
@@ -123,7 +127,9 @@ Future<Map<String, Set<int>>> resolveAllEntries(
       if (resolved != null) {
         result[word.jlptLevel]!.add(resolved);
       }
-    } catch (e) {}
+    } catch (e) {
+      print('ERROR: $e');
+    }
   }
 
   return result;

lib/_data_ingestion/tanos-jlpt/seed_data.dart

@@ -1,24 +1,21 @@
-import 'package:jadb/_data_ingestion/tanos-jlpt/table_names.dart';
+import 'package:jadb/table_names/tanos_jlpt.dart';
 import 'package:sqflite_common/sqlite_api.dart';
 
 Future<void> seedTanosJLPTData(
   Map<String, Set<int>> resolvedEntries,
   Database db,
 ) async {
-  Batch b = db.batch();
+  final Batch b = db.batch();
 
   for (final jlptLevel in resolvedEntries.entries) {
     final level = jlptLevel.key;
     final entryIds = jlptLevel.value;
 
     for (final entryId in entryIds) {
-      b.insert(
-        TanosJLPTTableNames.jlptTag,
-        {
-          'entryId': entryId,
-          'jlptLevel': level,
-        },
-      );
+      b.insert(TanosJLPTTableNames.jlptTag, {
+        'entryId': entryId,
+        'jlptLevel': level,
+      });
     }
   }
 

lib/_data_ingestion/tanos-jlpt/table_names.dart

@@ -1,5 +0,0 @@
-abstract class TanosJLPTTableNames {
-  static const String jlptTag = 'JMdict_JLPTTag';
-
-  static Set<String> get allTables => {jlptTag};
-}

lib/cli/commands/create_db.dart

@@ -1,19 +1,30 @@
 import 'dart:io';
 
+import 'package:args/command_runner.dart';
 import 'package:jadb/_data_ingestion/open_local_db.dart';
 import 'package:jadb/_data_ingestion/seed_database.dart';
-
-import 'package:args/command_runner.dart';
 import 'package:jadb/cli/args.dart';
 
 class CreateDb extends Command {
-  final name = "create-db";
-  final description = "Create the database";
+  @override
+  final name = 'create-db';
+  @override
+  final description = 'Create the database';
 
   CreateDb() {
     addLibsqliteArg(argParser);
+    argParser.addFlag(
+      'wal',
+      help: '''Whether to use Write-Ahead Logging (WAL) mode.
+
+          This is recommended for better performance, but may not be used with
+          the readonly NixOS store.
+      ''',
+      defaultsTo: false,
+    );
   }
 
+  @override
   Future<void> run() async {
     if (argResults!.option('libsqlite') == null) {
       print(argParser.usage);
@@ -22,15 +33,26 @@ class CreateDb extends Command {
 
     final db = await openLocalDb(
       libsqlitePath: argResults!.option('libsqlite')!,
+      walMode: argResults!.flag('wal'),
       readWrite: true,
     );
 
-    seedData(db).then((_) {
-      print("Database created successfully");
-    }).catchError((error) {
-      print("Error creating database: $error");
-    }).whenComplete(() {
-      db.close();
-    });
+    bool failed = false;
+    await seedData(db)
+        .then((_) {
+          print('Database created successfully');
+        })
+        .catchError((error) {
+          print('Error creating database: $error');
+          failed = true;
+        })
+        .whenComplete(() {
+          db.close();
+        });
+    if (failed) {
+      exit(1);
+    } else {
+      exit(0);
+    }
   }
 }

lib/cli/commands/create_tanos_jlpt_mappings.dart

@@ -1,8 +1,7 @@
 import 'dart:io';
 
-import 'package:jadb/_data_ingestion/open_local_db.dart';
-
 import 'package:args/command_runner.dart';
+import 'package:jadb/_data_ingestion/open_local_db.dart';
 import 'package:jadb/_data_ingestion/tanos-jlpt/csv_parser.dart';
 import 'package:jadb/_data_ingestion/tanos-jlpt/objects.dart';
 import 'package:jadb/_data_ingestion/tanos-jlpt/resolve.dart';
@@ -10,9 +9,11 @@ import 'package:jadb/cli/args.dart';
 import 'package:sqflite_common/sqlite_api.dart';
 
 class CreateTanosJlptMappings extends Command {
-  final name = "create-tanos-jlpt-mappings";
+  @override
+  final name = 'create-tanos-jlpt-mappings';
+  @override
   final description =
-      "Resolve Tanos JLPT data against JMDict. This tool is useful to create overrides for ambiguous references";
+      'Resolve Tanos JLPT data against JMDict. This tool is useful to create overrides for ambiguous references';
 
   CreateTanosJlptMappings() {
     addLibsqliteArg(argParser);
@@ -26,6 +27,7 @@ class CreateTanosJlptMappings extends Command {
     );
   }
 
+  @override
   Future<void> run() async {
     if (argResults!.option('libsqlite') == null ||
         argResults!.option('jadb') == null) {
@@ -40,7 +42,7 @@ class CreateTanosJlptMappings extends Command {
 
     final useOverrides = argResults!.flag('overrides');
 
-    Map<String, File> files = {
+    final Map<String, File> files = {
       'N1': File('data/tanos-jlpt/n1.csv'),
       'N2': File('data/tanos-jlpt/n2.csv'),
       'N3': File('data/tanos-jlpt/n3.csv'),
@@ -59,11 +61,12 @@ Future<void> resolveExisting(
   Database db,
   bool useOverrides,
 ) async {
-  List<JLPTRankedWord> missingWords = [];
+  final List<JLPTRankedWord> missingWords = [];
   for (final (i, word) in rankedWords.indexed) {
     try {
       print(
-          '[${(i + 1).toString().padLeft(4, '0')}/${rankedWords.length}] ${word.toString()}');
+        '[${(i + 1).toString().padLeft(4, '0')}/${rankedWords.length}] ${word.toString()}',
+      );
       await findEntry(word, db, useOverrides: useOverrides);
     } catch (e) {
       print(e);
@@ -78,16 +81,19 @@ Future<void> resolveExisting(
 
   print('Statistics:');
   for (final jlptLevel in ['N5', 'N4', 'N3', 'N2', 'N1']) {
-    final missingWordCount =
-        missingWords.where((e) => e.jlptLevel == jlptLevel).length;
-    final totalWordCount =
-        rankedWords.where((e) => e.jlptLevel == jlptLevel).length;
+    final missingWordCount = missingWords
+        .where((e) => e.jlptLevel == jlptLevel)
+        .length;
+    final totalWordCount = rankedWords
+        .where((e) => e.jlptLevel == jlptLevel)
+        .length;
 
-    final failureRate =
-        ((missingWordCount / totalWordCount) * 100).toStringAsFixed(2);
+    final failureRate = ((missingWordCount / totalWordCount) * 100)
+        .toStringAsFixed(2);
 
     print(
-        '${jlptLevel} failures: [${missingWordCount}/${totalWordCount}] (${failureRate}%)');
+      '$jlptLevel failures: [$missingWordCount/$totalWordCount] ($failureRate%)',
+    );
   }
 
   print('Not able to determine the entry for ${missingWords.length} words');

lib/cli/commands/lemmatize.dart

@@ -0,0 +1,48 @@
+// import 'dart:io';
+
+import 'package:args/command_runner.dart';
+// import 'package:jadb/_data_ingestion/open_local_db.dart';
+import 'package:jadb/cli/args.dart';
+import 'package:jadb/util/lemmatizer/lemmatizer.dart';
+
+class Lemmatize extends Command {
+  @override
+  final name = 'lemmatize';
+  @override
+  final description = 'Lemmatize a word using the Jadb lemmatizer';
+
+  Lemmatize() {
+    addLibsqliteArg(argParser);
+    addJadbArg(argParser);
+    argParser.addOption(
+      'word',
+      abbr: 'w',
+      help: 'The word to search for.',
+      valueHelp: 'WORD',
+    );
+  }
+
+  @override
+  Future<void> run() async {
+    // if (argResults!.option('libsqlite') == null ||
+    //     argResults!.option('jadb') == null) {
+    //   print(argParser.usage);
+    //   exit(64);
+    // }
+
+    // final db = await openLocalDb(
+    //   jadbPath: argResults!.option('jadb')!,
+    //   libsqlitePath: argResults!.option('libsqlite')!,
+    // );
+
+    final String searchWord = argResults!.option('word') ?? '食べたくない';
+
+    final time = Stopwatch()..start();
+    final result = lemmatize(searchWord);
+    time.stop();
+
+    print(result.toString());
+
+    print('Lemmatization took ${time.elapsedMilliseconds}ms');
+  }
+}

lib/cli/commands/query_kanji.dart

@@ -1,22 +1,25 @@
-
 import 'dart:convert';
 import 'dart:io';
 
+import 'package:args/command_runner.dart';
 import 'package:jadb/_data_ingestion/open_local_db.dart';
 import 'package:jadb/cli/args.dart';
 import 'package:jadb/search.dart';
 
-import 'package:args/command_runner.dart';
-
 class QueryKanji extends Command {
-  final name = "query-kanji";
-  final description = "Query the database for kanji data";
+  @override
+  final name = 'query-kanji';
+  @override
+  final description = 'Query the database for kanji data';
+  @override
+  final invocation = 'jadb query-kanji [options] <kanji>';
 
   QueryKanji() {
     addLibsqliteArg(argParser);
     addJadbArg(argParser);
   }
 
+  @override
   Future<void> run() async {
     if (argResults!.option('libsqlite') == null ||
         argResults!.option('jadb') == null) {
@@ -29,12 +32,25 @@ class QueryKanji extends Command {
       libsqlitePath: argResults!.option('libsqlite')!,
     );
 
-    final result = await JaDBConnection(db).searchKanji('漢');
+    if (argResults!.rest.length != 1) {
+      print('You need to provide exactly one kanji character to search for.');
+      print('');
+      printUsage();
+      exit(64);
+    }
+
+    final String kanji = argResults!.rest.first.trim();
+
+    final time = Stopwatch()..start();
+    final result = await JaDBConnection(db).jadbSearchKanji(kanji);
+    time.stop();
 
     if (result == null) {
-      print("No such kanji");
+      print('No such kanji');
     } else {
       print(JsonEncoder.withIndent('  ').convert(result.toJson()));
     }
+
+    print('Query took ${time.elapsedMilliseconds}ms');
   }
 }

lib/cli/commands/query_word.dart

@@ -1,26 +1,38 @@
-
 import 'dart:convert';
 import 'dart:io';
 
+import 'package:args/command_runner.dart';
 import 'package:jadb/_data_ingestion/open_local_db.dart';
 import 'package:jadb/cli/args.dart';
 import 'package:jadb/search.dart';
-
-import 'package:args/command_runner.dart';
+import 'package:sqflite_common/sqflite.dart';
 
 class QueryWord extends Command {
-  final name = "query-word";
-  final description = "Query the database for word data";
+  @override
+  final name = 'query-word';
+  @override
+  final description = 'Query the database for word data';
+  @override
+  final invocation = 'jadb query-word [options] (<word> | <ID>)';
 
   QueryWord() {
     addLibsqliteArg(argParser);
     addJadbArg(argParser);
+
+    argParser.addFlag('json', abbr: 'j', help: 'Output results in JSON format');
+
+    argParser.addOption('page', abbr: 'p', valueHelp: 'NUM', defaultsTo: '0');
+
+    argParser.addOption('pageSize', valueHelp: 'NUM', defaultsTo: '30');
   }
 
+  @override
   Future<void> run() async {
     if (argResults!.option('libsqlite') == null ||
         argResults!.option('jadb') == null) {
-      print(argParser.usage);
+      print('You need to provide both libsqlite and jadb paths.');
+      print('');
+      printUsage();
       exit(64);
     }
 
@@ -29,15 +41,81 @@ class QueryWord extends Command {
       libsqlitePath: argResults!.option('libsqlite')!,
     );
 
-    final result = await JaDBConnection(db).searchWord('kana');
+    if (argResults!.rest.isEmpty) {
+      print('You need to provide a word or ID to search for.');
+      print('');
+      printUsage();
+      exit(64);
+    }
 
-    if (result == null) {
-      print("Invalid search");
-    } else if (result.isEmpty) {
-      print("No matches");
+    final String searchWord = argResults!.rest.join(' ');
+    final int? maybeId = int.tryParse(searchWord);
+
+    if (maybeId != null && maybeId >= 1000000) {
+      await _searchId(db, maybeId, argResults!.flag('json'));
     } else {
-      print(JsonEncoder.withIndent('  ')
-          .convert(result.map((e) => e.toJson()).toList()));
+      await _searchWord(
+        db,
+        searchWord,
+        argResults!.flag('json'),
+        int.parse(argResults!.option('page')!),
+        int.parse(argResults!.option('pageSize')!),
+      );
     }
   }
+
+  Future<void> _searchId(DatabaseExecutor db, int id, bool jsonOutput) async {
+    final time = Stopwatch()..start();
+    final result = await JaDBConnection(db).jadbGetWordById(id);
+    time.stop();
+
+    if (result == null) {
+      print('Invalid ID');
+    } else {
+      if (jsonOutput) {
+        print(JsonEncoder.withIndent('  ').convert(result));
+      } else {
+        print(result.toString());
+      }
+    }
+
+    print('Query took ${time.elapsedMilliseconds}ms');
+  }
+
+  Future<void> _searchWord(
+    DatabaseExecutor db,
+    String searchWord,
+    bool jsonOutput,
+    int page,
+    int pageSize,
+  ) async {
+    final time = Stopwatch()..start();
+    final count = await JaDBConnection(db).jadbSearchWordCount(searchWord);
+    time.stop();
+
+    final time2 = Stopwatch()..start();
+    final result = await JaDBConnection(
+      db,
+    ).jadbSearchWord(searchWord, page: page, pageSize: pageSize);
+    time2.stop();
+
+    if (result == null) {
+      print('Invalid search');
+    } else if (result.isEmpty) {
+      print('No matches');
+    } else {
+      if (jsonOutput) {
+        print(JsonEncoder.withIndent('  ').convert(result));
+      } else {
+        for (final e in result) {
+          print(e.toString());
+          print('');
+        }
+      }
+    }
+
+    print('Total count: $count');
+    print('Count query took ${time.elapsedMilliseconds}ms');
+    print('Query took ${time2.elapsedMilliseconds}ms');
+  }
 }

lib/const_data/radicals.dart

@@ -0,0 +1,217 @@
+const Map<int, List<String>> radicals = {
+  1: ['一', '｜', '丶', 'ノ', '乙', '亅'],
+  2: [
+    '二',
+    '亠',
+    '人',
+    '⺅',
+    '𠆢',
+    '儿',
+    '入',
+    'ハ',
+    '丷',
+    '冂',
+    '冖',
+    '冫',
+    '几',
+    '凵',
+    '刀',
+    '⺉',
+    '力',
+    '勹',
+    '匕',
+    '匚',
+    '十',
+    '卜',
+    '卩',
+    '厂',
+    '厶',
+    '又',
+    'マ',
+    '九',
+    'ユ',
+    '乃',
+    '𠂉',
+  ],
+  3: [
+    '⻌',
+    '口',
+    '囗',
+    '土',
+    '士',
+    '夂',
+    '夕',
+    '大',
+    '女',
+    '子',
+    '宀',
+    '寸',
+    '小',
+    '⺌',
+    '尢',
+    '尸',
+    '屮',
+    '山',
+    '川',
+    '巛',
+    '工',
+    '已',
+    '巾',
+    '干',
+    '幺',
+    '广',
+    '廴',
+    '廾',
+    '弋',
+    '弓',
+    'ヨ',
+    '彑',
+    '彡',
+    '彳',
+    '⺖',
+    '⺘',
+    '⺡',
+    '⺨',
+    '⺾',
+    '⻏',
+    '⻖',
+    '也',
+    '亡',
+    '及',
+    '久',
+  ],
+  4: [
+    '⺹',
+    '心',
+    '戈',
+    '戸',
+    '手',
+    '支',
+    '攵',
+    '文',
+    '斗',
+    '斤',
+    '方',
+    '无',
+    '日',
+    '曰',
+    '月',
+    '木',
+    '欠',
+    '止',
+    '歹',
+    '殳',
+    '比',
+    '毛',
+    '氏',
+    '气',
+    '水',
+    '火',
+    '⺣',
+    '爪',
+    '父',
+    '爻',
+    '爿',
+    '片',
+    '牛',
+    '犬',
+    '⺭',
+    '王',
+    '元',
+    '井',
+    '勿',
+    '尤',
+    '五',
+    '屯',
+    '巴',
+    '毋',
+  ],
+  5: [
+    '玄',
+    '瓦',
+    '甘',
+    '生',
+    '用',
+    '田',
+    '疋',
+    '疒',
+    '癶',
+    '白',
+    '皮',
+    '皿',
+    '目',
+    '矛',
+    '矢',
+    '石',
+    '示',
+    '禸',
+    '禾',
+    '穴',
+    '立',
+    '⻂',
+    '世',
+    '巨',
+    '冊',
+    '母',
+    '⺲',
+    '牙',
+  ],
+  6: [
+    '瓜',
+    '竹',
+    '米',
+    '糸',
+    '缶',
+    '羊',
+    '羽',
+    '而',
+    '耒',
+    '耳',
+    '聿',
+    '肉',
+    '自',
+    '至',
+    '臼',
+    '舌',
+    '舟',
+    '艮',
+    '色',
+    '虍',
+    '虫',
+    '血',
+    '行',
+    '衣',
+    '西',
+  ],
+  7: [
+    '臣',
+    '見',
+    '角',
+    '言',
+    '谷',
+    '豆',
+    '豕',
+    '豸',
+    '貝',
+    '赤',
+    '走',
+    '足',
+    '身',
+    '車',
+    '辛',
+    '辰',
+    '酉',
+    '釆',
+    '里',
+    '舛',
+    '麦',
+  ],
+  8: ['金', '長', '門', '隶', '隹', '雨', '青', '非', '奄', '岡', '免', '斉'],
+  9: ['面', '革', '韭', '音', '頁', '風', '飛', '食', '首', '香', '品'],
+  10: ['馬', '骨', '高', '髟', '鬥', '鬯', '鬲', '鬼', '竜', '韋'],
+  11: ['魚', '鳥', '鹵', '鹿', '麻', '亀', '啇', '黄', '黒'],
+  12: ['黍', '黹', '無', '歯'],
+  13: ['黽', '鼎', '鼓', '鼠'],
+  14: ['鼻', '齊'],
+  17: ['龠'],
+};

lib/models/common/jlpt_level.dart

@@ -1 +1,56 @@
-enum JlptLevel { none, n5, n4, n3, n2, n1 }
+enum JlptLevel implements Comparable<JlptLevel> {
+  none,
+  n1,
+  n2,
+  n3,
+  n4,
+  n5;
+
+  factory JlptLevel.fromString(String? level) {
+    switch (level?.toUpperCase()) {
+      case 'N1':
+        return JlptLevel.n1;
+      case 'N2':
+        return JlptLevel.n2;
+      case 'N3':
+        return JlptLevel.n3;
+      case 'N4':
+        return JlptLevel.n4;
+      case 'N5':
+        return JlptLevel.n5;
+      default:
+        return JlptLevel.none;
+    }
+  }
+
+  String? toNullableString() {
+    switch (this) {
+      case JlptLevel.n1:
+        return 'N1';
+      case JlptLevel.n2:
+        return 'N2';
+      case JlptLevel.n3:
+        return 'N3';
+      case JlptLevel.n4:
+        return 'N4';
+      case JlptLevel.n5:
+        return 'N5';
+      case JlptLevel.none:
+        return null;
+    }
+  }
+
+  int? get asInt =>
+      this == JlptLevel.none ? null : JlptLevel.values.indexOf(this);
+
+  @override
+  String toString() => toNullableString() ?? 'N/A';
+
+  Object? toJson() => toNullableString();
+
+  factory JlptLevel.fromJson(Object? json) =>
+      JlptLevel.fromString(json as String?);
+
+  @override
+  int compareTo(JlptLevel other) => index - other.index;
+}

lib/models/create_empty_db.dart

@@ -0,0 +1,27 @@
+import 'dart:io';
+import 'dart:isolate';
+import 'package:path/path.dart';
+
+import 'package:sqflite_common/sqlite_api.dart';
+
+String migrationDirPath() {
+  final packageUri = Uri.parse('package:jadb/');
+  final packagePath = Isolate.resolvePackageUriSync(packageUri);
+  return packagePath!.resolve('../migrations').toFilePath();
+}
+
+Future<void> createEmptyDb(DatabaseExecutor db) async {
+  final List<String> migrationFiles = [];
+  for (final file in Directory(migrationDirPath()).listSync()) {
+    if (file is File && file.path.endsWith('.sql')) {
+      migrationFiles.add(file.path);
+    }
+  }
+
+  migrationFiles.sort((a, b) => basename(a).compareTo(basename(b)));
+
+  for (final file in migrationFiles) {
+    final sql = await File(file).readAsString();
+    await db.execute(sql);
+  }
+}

lib/models/jmdict/jmdict_dialect.dart

@@ -19,21 +19,14 @@ enum JMdictDialect {
   final String id;
   final String description;
 
-  const JMdictDialect({
-    required this.id,
-    required this.description,
-  });
+  const JMdictDialect({required this.id, required this.description});
 
-  static JMdictDialect fromId(String id) =>
-      JMdictDialect.values.firstWhere(
-        (e) => e.id == id,
-        orElse: () => throw Exception('Unknown id: $id'),
-      );
+  static JMdictDialect fromId(String id) => JMdictDialect.values.firstWhere(
+    (e) => e.id == id,
+    orElse: () => throw Exception('Unknown id: $id'),
+  );
 
-  Map<String, Object?> toJson() => {
-        'id': id,
-        'description': description,
-      };
+  Map<String, Object?> toJson() => {'id': id, 'description': description};
 
   static JMdictDialect fromJson(Map<String, Object?> json) =>
       JMdictDialect.values.firstWhere(

lib/models/jmdict/jmdict_field.dart

@@ -102,21 +102,14 @@ enum JMdictField {
   final String id;
   final String description;
 
-  const JMdictField({
-    required this.id,
-    required this.description,
-  });
+  const JMdictField({required this.id, required this.description});
 
-  static JMdictField fromId(String id) =>
-      JMdictField.values.firstWhere(
-        (e) => e.id == id,
-        orElse: () => throw Exception('Unknown id: $id'),
-      );
+  static JMdictField fromId(String id) => JMdictField.values.firstWhere(
+    (e) => e.id == id,
+    orElse: () => throw Exception('Unknown id: $id'),
+  );
 
-  Map<String, Object?> toJson() => {
-        'id': id,
-        'description': description,
-      };
+  Map<String, Object?> toJson() => {'id': id, 'description': description};
 
   static JMdictField fromJson(Map<String, Object?> json) =>
       JMdictField.values.firstWhere(

lib/models/jmdict/jmdict_kanji_info.dart

@@ -13,21 +13,14 @@ enum JMdictKanjiInfo {
   final String id;
   final String description;
 
-  const JMdictKanjiInfo({
-    required this.id,
-    required this.description,
-  });
+  const JMdictKanjiInfo({required this.id, required this.description});
 
-  static JMdictKanjiInfo fromId(String id) =>
-      JMdictKanjiInfo.values.firstWhere(
-        (e) => e.id == id,
-        orElse: () => throw Exception('Unknown id: $id'),
-      );
+  static JMdictKanjiInfo fromId(String id) => JMdictKanjiInfo.values.firstWhere(
+    (e) => e.id == id,
+    orElse: () => throw Exception('Unknown id: $id'),
+  );
 
-  Map<String, Object?> toJson() => {
-        'id': id,
-        'description': description,
-      };
+  Map<String, Object?> toJson() => {'id': id, 'description': description};
 
   static JMdictKanjiInfo fromJson(Map<String, Object?> json) =>
       JMdictKanjiInfo.values.firstWhere(

lib/models/jmdict/jmdict_misc.dart

@@ -74,21 +74,14 @@ enum JMdictMisc {
   final String id;
   final String description;
 
-  const JMdictMisc({
-    required this.id,
-    required this.description,
-  });
+  const JMdictMisc({required this.id, required this.description});
 
-  static JMdictMisc fromId(String id) =>
-      JMdictMisc.values.firstWhere(
-        (e) => e.id == id,
-        orElse: () => throw Exception('Unknown id: $id'),
-      );
+  static JMdictMisc fromId(String id) => JMdictMisc.values.firstWhere(
+    (e) => e.id == id,
+    orElse: () => throw Exception('Unknown id: $id'),
+  );
 
-  Map<String, Object?> toJson() => {
-        'id': id,
-        'description': description,
-      };
+  Map<String, Object?> toJson() => {'id': id, 'description': description};
 
   static JMdictMisc fromJson(Map<String, Object?> json) =>
       JMdictMisc.values.firstWhere(

lib/models/jmdict/jmdict_pos.dart

@@ -7,14 +7,21 @@ enum JMdictPOS {
   adjIx(id: 'adj-ix', description: 'adjective (keiyoushi) - yoi/ii class'),
   adjKari(id: 'adj-kari', description: '\'kari\' adjective (archaic)'),
   adjKu(id: 'adj-ku', description: '\'ku\' adjective (archaic)'),
-  adjNa(id: 'adj-na', description: 'adjectival nouns or quasi-adjectives (keiyodoshi)'),
+  adjNa(
+    id: 'adj-na',
+    description: 'adjectival nouns or quasi-adjectives (keiyodoshi)',
+  ),
   adjNari(id: 'adj-nari', description: 'archaic/formal form of na-adjective'),
-  adjNo(id: 'adj-no', description: 'nouns which may take the genitive case particle ''no'''),
+  adjNo(
+    id: 'adj-no',
+    description: 'nouns which may take the genitive case particle \'no\'',
+    shortDescription: 'Na-adjective (keiyodoshi)',
+  ),
   adjPn(id: 'adj-pn', description: 'pre-noun adjectival (rentaishi)'),
   adjShiku(id: 'adj-shiku', description: '\'shiku\' adjective (archaic)'),
   adjT(id: 'adj-t', description: '\'taru\' adjective'),
   adv(id: 'adv', description: 'adverb (fukushi)'),
-  advTo(id: 'adv-to', description: 'adverb taking the ''to'' particle'),
+  advTo(id: 'adv-to', description: 'adverb taking the \'to\' particle'),
   aux(id: 'aux', description: 'auxiliary'),
   auxAdj(id: 'aux-adj', description: 'auxiliary adjective'),
   auxV(id: 'aux-v', description: 'auxiliary verb'),
@@ -23,7 +30,11 @@ enum JMdictPOS {
   ctr(id: 'ctr', description: 'counter'),
   exp(id: 'exp', description: 'expressions (phrases, clauses, etc.)'),
   int(id: 'int', description: 'interjection (kandoushi)'),
-  n(id: 'n', description: 'noun (common) (futsuumeishi)'),
+  n(
+    id: 'n',
+    description: 'noun (common) (futsuumeishi)',
+    shortDescription: 'noun',
+  ),
   nAdv(id: 'n-adv', description: 'adverbial noun (fukushitekimeishi)'),
   nPr(id: 'n-pr', description: 'proper noun'),
   nPref(id: 'n-pref', description: 'noun, used as a prefix'),
@@ -38,81 +49,164 @@ enum JMdictPOS {
   vUnspec(id: 'v-unspec', description: 'verb unspecified'),
   v1(id: 'v1', description: 'Ichidan verb'),
   v1S(id: 'v1-s', description: 'Ichidan verb - kureru special class'),
-  v2aS(id: 'v2a-s', description: 'Nidan verb with ''u'' ending (archaic)'),
-  v2bK(id: 'v2b-k', description: 'Nidan verb (upper class) with ''bu'' ending (archaic)'),
-  v2bS(id: 'v2b-s', description: 'Nidan verb (lower class) with ''bu'' ending (archaic)'),
-  v2dK(id: 'v2d-k', description: 'Nidan verb (upper class) with ''dzu'' ending (archaic)'),
-  v2dS(id: 'v2d-s', description: 'Nidan verb (lower class) with ''dzu'' ending (archaic)'),
-  v2gK(id: 'v2g-k', description: 'Nidan verb (upper class) with ''gu'' ending (archaic)'),
-  v2gS(id: 'v2g-s', description: 'Nidan verb (lower class) with ''gu'' ending (archaic)'),
-  v2hK(id: 'v2h-k', description: 'Nidan verb (upper class) with ''hu/fu'' ending (archaic)'),
-  v2hS(id: 'v2h-s', description: 'Nidan verb (lower class) with ''hu/fu'' ending (archaic)'),
-  v2kK(id: 'v2k-k', description: 'Nidan verb (upper class) with ''ku'' ending (archaic)'),
-  v2kS(id: 'v2k-s', description: 'Nidan verb (lower class) with ''ku'' ending (archaic)'),
-  v2mK(id: 'v2m-k', description: 'Nidan verb (upper class) with ''mu'' ending (archaic)'),
-  v2mS(id: 'v2m-s', description: 'Nidan verb (lower class) with ''mu'' ending (archaic)'),
-  v2nS(id: 'v2n-s', description: 'Nidan verb (lower class) with ''nu'' ending (archaic)'),
-  v2rK(id: 'v2r-k', description: 'Nidan verb (upper class) with ''ru'' ending (archaic)'),
-  v2rS(id: 'v2r-s', description: 'Nidan verb (lower class) with ''ru'' ending (archaic)'),
-  v2sS(id: 'v2s-s', description: 'Nidan verb (lower class) with ''su'' ending (archaic)'),
-  v2tK(id: 'v2t-k', description: 'Nidan verb (upper class) with ''tsu'' ending (archaic)'),
-  v2tS(id: 'v2t-s', description: 'Nidan verb (lower class) with ''tsu'' ending (archaic)'),
-  v2wS(id: 'v2w-s', description: 'Nidan verb (lower class) with ''u'' ending and ''we'' conjugation (archaic)'),
-  v2yK(id: 'v2y-k', description: 'Nidan verb (upper class) with ''yu'' ending (archaic)'),
-  v2yS(id: 'v2y-s', description: 'Nidan verb (lower class) with ''yu'' ending (archaic)'),
-  v2zS(id: 'v2z-s', description: 'Nidan verb (lower class) with ''zu'' ending (archaic)'),
-  v4b(id: 'v4b', description: 'Yodan verb with ''bu'' ending (archaic)'),
-  v4g(id: 'v4g', description: 'Yodan verb with ''gu'' ending (archaic)'),
-  v4h(id: 'v4h', description: 'Yodan verb with ''hu/fu'' ending (archaic)'),
-  v4k(id: 'v4k', description: 'Yodan verb with ''ku'' ending (archaic)'),
-  v4m(id: 'v4m', description: 'Yodan verb with ''mu'' ending (archaic)'),
-  v4n(id: 'v4n', description: 'Yodan verb with ''nu'' ending (archaic)'),
-  v4r(id: 'v4r', description: 'Yodan verb with ''ru'' ending (archaic)'),
-  v4s(id: 'v4s', description: 'Yodan verb with ''su'' ending (archaic)'),
-  v4t(id: 'v4t', description: 'Yodan verb with ''tsu'' ending (archaic)'),
+  v2aS(id: 'v2a-s', description: 'Nidan verb with \'u\' ending (archaic)'),
+  v2bK(
+    id: 'v2b-k',
+    description: 'Nidan verb (upper class) with \'bu\' ending (archaic)',
+  ),
+  v2bS(
+    id: 'v2b-s',
+    description: 'Nidan verb (lower class) with \'bu\' ending (archaic)',
+  ),
+  v2dK(
+    id: 'v2d-k',
+    description: 'Nidan verb (upper class) with \'dzu\' ending (archaic)',
+  ),
+  v2dS(
+    id: 'v2d-s',
+    description: 'Nidan verb (lower class) with \'dzu\' ending (archaic)',
+  ),
+  v2gK(
+    id: 'v2g-k',
+    description: 'Nidan verb (upper class) with \'gu\' ending (archaic)',
+  ),
+  v2gS(
+    id: 'v2g-s',
+    description: 'Nidan verb (lower class) with \'gu\' ending (archaic)',
+  ),
+  v2hK(
+    id: 'v2h-k',
+    description: 'Nidan verb (upper class) with \'hu/fu\' ending (archaic)',
+  ),
+  v2hS(
+    id: 'v2h-s',
+    description: 'Nidan verb (lower class) with \'hu/fu\' ending (archaic)',
+  ),
+  v2kK(
+    id: 'v2k-k',
+    description: 'Nidan verb (upper class) with \'ku\' ending (archaic)',
+  ),
+  v2kS(
+    id: 'v2k-s',
+    description: 'Nidan verb (lower class) with \'ku\' ending (archaic)',
+  ),
+  v2mK(
+    id: 'v2m-k',
+    description: 'Nidan verb (upper class) with \'mu\' ending (archaic)',
+  ),
+  v2mS(
+    id: 'v2m-s',
+    description: 'Nidan verb (lower class) with \'mu\' ending (archaic)',
+  ),
+  v2nS(
+    id: 'v2n-s',
+    description: 'Nidan verb (lower class) with \'nu\' ending (archaic)',
+  ),
+  v2rK(
+    id: 'v2r-k',
+    description: 'Nidan verb (upper class) with \'ru\' ending (archaic)',
+  ),
+  v2rS(
+    id: 'v2r-s',
+    description: 'Nidan verb (lower class) with \'ru\' ending (archaic)',
+  ),
+  v2sS(
+    id: 'v2s-s',
+    description: 'Nidan verb (lower class) with \'su\' ending (archaic)',
+  ),
+  v2tK(
+    id: 'v2t-k',
+    description: 'Nidan verb (upper class) with \'tsu\' ending (archaic)',
+  ),
+  v2tS(
+    id: 'v2t-s',
+    description: 'Nidan verb (lower class) with \'tsu\' ending (archaic)',
+  ),
+  v2wS(
+    id: 'v2w-s',
+    description:
+        'Nidan verb (lower class) with \'u\' ending and \'we\' conjugation (archaic)',
+  ),
+  v2yK(
+    id: 'v2y-k',
+    description: 'Nidan verb (upper class) with \'yu\' ending (archaic)',
+  ),
+  v2yS(
+    id: 'v2y-s',
+    description: 'Nidan verb (lower class) with \'yu\' ending (archaic)',
+  ),
+  v2zS(
+    id: 'v2z-s',
+    description: 'Nidan verb (lower class) with \'zu\' ending (archaic)',
+  ),
+  v4b(id: 'v4b', description: 'Yodan verb with \'bu\' ending (archaic)'),
+  v4g(id: 'v4g', description: 'Yodan verb with \'gu\' ending (archaic)'),
+  v4h(id: 'v4h', description: 'Yodan verb with \'hu/fu\' ending (archaic)'),
+  v4k(id: 'v4k', description: 'Yodan verb with \'ku\' ending (archaic)'),
+  v4m(id: 'v4m', description: 'Yodan verb with \'mu\' ending (archaic)'),
+  v4n(id: 'v4n', description: 'Yodan verb with \'nu\' ending (archaic)'),
+  v4r(id: 'v4r', description: 'Yodan verb with \'ru\' ending (archaic)'),
+  v4s(id: 'v4s', description: 'Yodan verb with \'su\' ending (archaic)'),
+  v4t(id: 'v4t', description: 'Yodan verb with \'tsu\' ending (archaic)'),
   v5aru(id: 'v5aru', description: 'Godan verb - -aru special class'),
-  v5b(id: 'v5b', description: 'Godan verb with ''bu'' ending'),
-  v5g(id: 'v5g', description: 'Godan verb with ''gu'' ending'),
-  v5k(id: 'v5k', description: 'Godan verb with ''ku'' ending'),
+  v5b(id: 'v5b', description: 'Godan verb with \'bu\' ending'),
+  v5g(id: 'v5g', description: 'Godan verb with \'gu\' ending'),
+  v5k(id: 'v5k', description: 'Godan verb with \'ku\' ending'),
   v5kS(id: 'v5k-s', description: 'Godan verb - Iku/Yuku special class'),
-  v5m(id: 'v5m', description: 'Godan verb with ''mu'' ending'),
-  v5n(id: 'v5n', description: 'Godan verb with ''nu'' ending'),
-  v5r(id: 'v5r', description: 'Godan verb with ''ru'' ending'),
-  v5rI(id: 'v5r-i', description: 'Godan verb with ''ru'' ending (irregular verb)'),
-  v5s(id: 'v5s', description: 'Godan verb with ''su'' ending'),
-  v5t(id: 'v5t', description: 'Godan verb with ''tsu'' ending'),
-  v5u(id: 'v5u', description: 'Godan verb with ''u'' ending'),
-  v5uS(id: 'v5u-s', description: 'Godan verb with ''u'' ending (special class)'),
-  v5uru(id: 'v5uru', description: 'Godan verb - Uru old class verb (old form of Eru)'),
+  v5m(id: 'v5m', description: 'Godan verb with \'mu\' ending'),
+  v5n(id: 'v5n', description: 'Godan verb with \'nu\' ending'),
+  v5r(id: 'v5r', description: 'Godan verb with \'ru\' ending'),
+  v5rI(
+    id: 'v5r-i',
+    description: 'Godan verb with \'ru\' ending (irregular verb)',
+  ),
+  v5s(id: 'v5s', description: 'Godan verb with \'su\' ending'),
+  v5t(id: 'v5t', description: 'Godan verb with \'tsu\' ending'),
+  v5u(id: 'v5u', description: 'Godan verb with \'u\' ending'),
+  v5uS(
+    id: 'v5u-s',
+    description: 'Godan verb with \'u\' ending (special class)',
+  ),
+  v5uru(
+    id: 'v5uru',
+    description: 'Godan verb - Uru old class verb (old form of Eru)',
+  ),
   vi(id: 'vi', description: 'intransitive verb'),
   vk(id: 'vk', description: 'Kuru verb - special class'),
   vn(id: 'vn', description: 'irregular nu verb'),
   vr(id: 'vr', description: 'irregular ru verb, plain form ends with -ri'),
-  vs(id: 'vs', description: 'noun or participle which takes the aux. verb suru'),
-  vsC(id: 'vs-c', description: 'suru verb - precursor to the modern suru'),
+  vs(
+    id: 'vs',
+    description: 'noun or participle which takes the aux. verb suru',
+    shortDescription: 'suru verb',
+  ),
+  vsC(id: 'vs-c', description: 'su verb - precursor to the modern suru'),
   vsI(id: 'vs-i', description: 'suru verb - included'),
   vsS(id: 'vs-s', description: 'suru verb - special class'),
   vt(id: 'vt', description: 'transitive verb'),
-  vz(id: 'vz', description: 'Ichidan verb - zuru verb (alternative form of -jiru verbs)');
+  vz(
+    id: 'vz',
+    description: 'Ichidan verb - zuru verb (alternative form of -jiru verbs)',
+  );
 
   final String id;
   final String description;
+  final String? _shortDescription;
 
   const JMdictPOS({
     required this.id,
     required this.description,
-  });
+    String? shortDescription,
+  }) : _shortDescription = shortDescription;
 
-  static JMdictPOS fromId(String id) =>
-      JMdictPOS.values.firstWhere(
-        (e) => e.id == id,
-        orElse: () => throw Exception('Unknown id: $id'),
-      );
+  String get shortDescription => _shortDescription ?? description;
 
-  Map<String, Object?> toJson() => {
-        'id': id,
-        'description': description,
-      };
+  static JMdictPOS fromId(String id) => JMdictPOS.values.firstWhere(
+    (e) => e.id == id,
+    orElse: () => throw Exception('Unknown id: $id'),
+  );
+
+  Map<String, Object?> toJson() => {'id': id, 'description': description};
 
   static JMdictPOS fromJson(Map<String, Object?> json) =>
       JMdictPOS.values.firstWhere(

lib/models/jmdict/jmdict_reading_info.dart

@@ -15,10 +15,7 @@ enum JMdictReadingInfo {
   final String id;
   final String description;
 
-  const JMdictReadingInfo({
-    required this.id,
-    required this.description,
-  });
+  const JMdictReadingInfo({required this.id, required this.description});
 
   static JMdictReadingInfo fromId(String id) =>
       JMdictReadingInfo.values.firstWhere(
@@ -26,10 +23,7 @@ enum JMdictReadingInfo {
         orElse: () => throw Exception('Unknown id: $id'),
       );
 
-  Map<String, Object?> toJson() => {
-        'id': id,
-        'description': description,
-      };
+  Map<String, Object?> toJson() => {'id': id, 'description': description};
 
   static JMdictReadingInfo fromJson(Map<String, Object?> json) =>
       JMdictReadingInfo.values.firstWhere(

lib/models/kanji_search/kanji_search_radical.dart

@@ -4,37 +4,43 @@ class KanjiSearchRadical extends Equatable {
   /// The radical symbol.
   final String symbol;
 
+  /// The names of this radical.
+  ///
+  /// Each name might refer to a specific form of the radical.
+  final List<String> names;
+
   /// The radical forms used in this kanji.
+  ///
+  /// (e.g. "亻" for "人", "氵" for "水")
   final List<String> forms;
 
-  /// The meaning of the radical.
-  final String meaning;
+  /// The meanings of the radical.
+  final List<String> meanings;
 
   // ignore: public_member_api_docs
   const KanjiSearchRadical({
     required this.symbol,
-    this.forms = const [],
-    required this.meaning,
+    required this.names,
+    required this.forms,
+    required this.meanings,
   });
 
   @override
-  List<Object> get props => [
-        symbol,
-        forms,
-        meaning,
-      ];
+  List<Object> get props => [symbol, names, forms, meanings];
 
   Map<String, dynamic> toJson() => {
-        'symbol': symbol,
-        'forms': forms,
-        'meaning': meaning,
-      };
+    'symbol': symbol,
+    'names': names,
+    'forms': forms,
+    'meanings': meanings,
+  };
 
   factory KanjiSearchRadical.fromJson(Map<String, dynamic> json) {
     return KanjiSearchRadical(
       symbol: json['symbol'] as String,
+      names: (json['names'] as List).map((e) => e as String).toList(),
       forms: (json['forms'] as List).map((e) => e as String).toList(),
-      meaning: json['meaning'] as String,
+      meanings: (json['meanings'] as List).map((e) => e as String).toList(),
     );
   }
 }

lib/models/kanji_search/kanji_search_result.dart

@@ -7,7 +7,8 @@ class KanjiSearchResult extends Equatable {
   final String kanji;
 
   /// The school level that the kanji is taught in, if applicable.
-  final String? taughtIn;
+  /// Ranges from `1` to `10` (except 7)
+  final int? taughtIn;
 
   /// The lowest JLPT exam that this kanji is likely to appear in, if applicable.
   ///
@@ -38,64 +39,101 @@ class KanjiSearchResult extends Equatable {
   /// Information about this character's radical, if applicable.
   final KanjiSearchRadical? radical;
 
-  // TODO: document more accurately what kind of parts?
-  /// The parts used in this kanji.
+  /// All radicals/kanji parts that make up this kanji.
+  ///
+  /// Note that this list might not always be complete.
   final List<String> parts;
 
+  /// Ids for the kanji's symbol in different encoding systems
+  /// (e.g. JIS213, JIS208, UCS, etc.)
+  final Map<String, String> codepoints;
+
+  /// The kanji's nanori readings.
+  ///
+  /// Nanori readings are special readings of kanji used in names.
+  final List<String> nanori;
+
+  /// How to read this kanji in different languages.
+  final Map<String, List<String>> alternativeLanguageReadings;
+
+  /// Common miscounts of the kanji's strokes.
+  final List<int> strokeMiscounts;
+
+  /// Query codes for looking up this kanji in different indexing systems.
+  final Map<String, List<String>> queryCodes;
+
+  /// References to other dictionaries that contain this kanji.
+  final Map<String, String> dictionaryReferences;
+
   const KanjiSearchResult({
     required this.kanji,
-    this.taughtIn,
-    this.jlptLevel,
-    this.newspaperFrequencyRank,
+    required this.taughtIn,
+    required this.jlptLevel,
+    required this.newspaperFrequencyRank,
     required this.strokeCount,
     required this.meanings,
-    this.kunyomi = const [],
-    this.onyomi = const [],
+    required this.kunyomi,
+    required this.onyomi,
     // this.kunyomiExamples = const [],
     // this.onyomiExamples = const [],
-    this.radical,
-    this.parts = const [],
+    required this.radical,
+    required this.parts,
+    required this.codepoints,
+    required this.nanori,
+    required this.alternativeLanguageReadings,
+    required this.strokeMiscounts,
+    required this.queryCodes,
+    required this.dictionaryReferences,
   });
 
   @override
   // ignore: public_member_api_docs
   List<Object?> get props => [
-        taughtIn,
-        jlptLevel,
-        newspaperFrequencyRank,
-        strokeCount,
-        meanings,
-        kunyomi,
-        onyomi,
-        // kunyomiExamples,
-        // onyomiExamples,
-        radical,
-        parts,
-      ];
+    taughtIn,
+    jlptLevel,
+    newspaperFrequencyRank,
+    strokeCount,
+    meanings,
+    kunyomi,
+    onyomi,
+    // kunyomiExamples,
+    // onyomiExamples,
+    radical,
+    parts,
+    codepoints,
+    kanji,
+    nanori,
+    alternativeLanguageReadings,
+    strokeMiscounts,
+    queryCodes,
+    dictionaryReferences,
+  ];
 
   Map<String, dynamic> toJson() => {
-        'kanji': kanji,
-        'taughtIn': taughtIn,
-        'jlptLevel': jlptLevel,
-        'newspaperFrequencyRank': newspaperFrequencyRank,
-        'strokeCount': strokeCount,
-        'meanings': meanings,
-        'kunyomi': kunyomi,
-        'onyomi': onyomi,
-        // 'onyomiExamples': onyomiExamples,
-        // 'kunyomiExamples': kunyomiExamples,
-        'radical': radical?.toJson(),
-        'parts': parts,
-        // 'strokeOrderDiagramUri': strokeOrderDiagramUri,
-        // 'strokeOrderSvgUri': strokeOrderSvgUri,
-        // 'strokeOrderGifUri': strokeOrderGifUri,
-        // 'uri': uri,
-      };
+    'kanji': kanji,
+    'taughtIn': taughtIn,
+    'jlptLevel': jlptLevel,
+    'newspaperFrequencyRank': newspaperFrequencyRank,
+    'strokeCount': strokeCount,
+    'meanings': meanings,
+    'kunyomi': kunyomi,
+    'onyomi': onyomi,
+    // 'onyomiExamples': onyomiExamples,
+    // 'kunyomiExamples': kunyomiExamples,
+    'radical': radical?.toJson(),
+    'parts': parts,
+    'codepoints': codepoints,
+    'nanori': nanori,
+    'alternativeLanguageReadings': alternativeLanguageReadings,
+    'strokeMiscounts': strokeMiscounts,
+    'queryCodes': queryCodes,
+    'dictionaryReferences': dictionaryReferences,
+  };
 
   factory KanjiSearchResult.fromJson(Map<String, dynamic> json) {
     return KanjiSearchResult(
       kanji: json['kanji'] as String,
-      taughtIn: json['taughtIn'] as String?,
+      taughtIn: json['taughtIn'] as int?,
       jlptLevel: json['jlptLevel'] as String?,
       newspaperFrequencyRank: json['newspaperFrequencyRank'] as int?,
       strokeCount: json['strokeCount'] as int,
@@ -112,6 +150,26 @@ class KanjiSearchResult extends Equatable {
           ? KanjiSearchRadical.fromJson(json['radical'])
           : null,
       parts: (json['parts'] as List).map((e) => e as String).toList(),
+      codepoints: (json['codepoints'] as Map<String, dynamic>).map(
+        (key, value) => MapEntry(key, value as String),
+      ),
+      nanori: (json['nanori'] as List).map((e) => e as String).toList(),
+      alternativeLanguageReadings:
+          (json['alternativeLanguageReadings'] as Map<String, dynamic>).map(
+            (key, value) =>
+                MapEntry(key, (value as List).map((e) => e as String).toList()),
+          ),
+      strokeMiscounts: (json['strokeMiscounts'] as List)
+          .map((e) => e as int)
+          .toList(),
+      queryCodes: (json['queryCodes'] as Map<String, dynamic>).map(
+        (key, value) =>
+            MapEntry(key, (value as List).map((e) => e as String).toList()),
+      ),
+      dictionaryReferences:
+          (json['dictionaryReferences'] as Map<String, dynamic>).map(
+            (key, value) => MapEntry(key, value as String),
+          ),
     );
   }
 }

lib/models/radkfile/radicals_search_result.dart

@@ -1,3 +0,0 @@
-class RadicalsSearchResult {
-  // TODO: implement me
-}

lib/models/verify_tables.dart

@@ -0,0 +1,41 @@
+import 'package:jadb/table_names/jmdict.dart';
+import 'package:jadb/table_names/kanjidic.dart';
+import 'package:jadb/table_names/radkfile.dart';
+import 'package:jadb/table_names/tanos_jlpt.dart';
+import 'package:sqflite_common/sqlite_api.dart';
+
+Future<void> verifyTablesWithDbConnection(DatabaseExecutor db) async {
+  final Set<String> tables = await db
+      .query(
+        'sqlite_master',
+        columns: ['name'],
+        where: 'type = ?',
+        whereArgs: ['table'],
+      )
+      .then((result) {
+        return result.map((row) => row['name'] as String).toSet();
+      });
+
+  final Set<String> expectedTables = {
+    ...JMdictTableNames.allTables,
+    ...KANJIDICTableNames.allTables,
+    ...RADKFILETableNames.allTables,
+    ...TanosJLPTTableNames.allTables,
+  };
+
+  final missingTables = expectedTables.difference(tables);
+
+  if (missingTables.isNotEmpty) {
+    throw Exception(
+      [
+        'Missing tables:',
+        missingTables.map((table) => '  - $table').join('\n'),
+        '',
+        'Found tables:\n',
+        tables.map((table) => '  - $table').join('\n'),
+        '',
+        'Please ensure the database is correctly set up.',
+      ].join('\n'),
+    );
+  }
+}

lib/models/word_search/word_search_match_span.dart

@@ -0,0 +1,62 @@
+enum WordSearchMatchSpanType { kanji, kana, sense }
+
+/// A span of a word search result that corresponds to a match for a kanji, kana, or sense.
+class WordSearchMatchSpan {
+  /// Which subtype of the word search result this span corresponds to - either a kanji, a kana, or a sense.
+  final WordSearchMatchSpanType spanType;
+
+  /// The index of the kanji/kana/sense in the word search result that this span corresponds to.
+  final int index;
+
+  /// When matching a 'sense', this is the index of the English definition in that sense that this span corresponds to. Otherwise, this is always 0.
+  final int subIndex;
+
+  /// The start of the span (inclusive)
+  final int start;
+
+  /// The end of the span (inclusive)
+  final int end;
+
+  WordSearchMatchSpan({
+    required this.spanType,
+    required this.index,
+    required this.start,
+    required this.end,
+    this.subIndex = 0,
+  });
+
+  @override
+  String toString() {
+    return 'WordSearchMatchSpan(spanType: $spanType, index: $index, start: $start, end: $end)';
+  }
+
+  Map<String, Object?> toJson() => {
+    'spanType': spanType.toString().split('.').last,
+    'index': index,
+    'start': start,
+    'end': end,
+  };
+
+  factory WordSearchMatchSpan.fromJson(Map<String, dynamic> json) =>
+      WordSearchMatchSpan(
+        spanType: WordSearchMatchSpanType.values.firstWhere(
+          (e) => e.toString().split('.').last == json['spanType'],
+        ),
+        index: json['index'] as int,
+        start: json['start'] as int,
+        end: json['end'] as int,
+      );
+
+  @override
+  int get hashCode => Object.hash(spanType, index, start, end);
+
+  @override
+  bool operator ==(Object other) {
+    if (identical(this, other)) return true;
+    return other is WordSearchMatchSpan &&
+        other.spanType == spanType &&
+        other.index == index &&
+        other.start == start &&
+        other.end == end;
+  }
+}

lib/models/word_search/word_search_result.dart

@@ -1,14 +1,25 @@
+import 'package:jadb/models/common/jlpt_level.dart';
 import 'package:jadb/models/jmdict/jmdict_kanji_info.dart';
+import 'package:jadb/models/jmdict/jmdict_misc.dart';
 import 'package:jadb/models/jmdict/jmdict_reading_info.dart';
+import 'package:jadb/models/word_search/word_search_match_span.dart';
 import 'package:jadb/models/word_search/word_search_ruby.dart';
 import 'package:jadb/models/word_search/word_search_sense.dart';
 import 'package:jadb/models/word_search/word_search_sources.dart';
+import 'package:jadb/search/word_search/word_search.dart';
+import 'package:jadb/util/romaji_transliteration.dart';
 
 /// A class representing a single dictionary entry from a word search.
 class WordSearchResult {
+  /// The score of the entry, used for sorting results.
+  final int score;
+
   /// The ID of the entry in the database.
   final int entryId;
 
+  /// Whether the word is common or not.
+  final bool isCommon;
+
   /// The variants of the word in Japanese.
   final List<WordSearchRuby> japanese;
 
@@ -21,32 +32,88 @@ class WordSearchResult {
   /// The meanings of the word, including parts of speech and other information.
   final List<WordSearchSense> senses;
 
+  /// The JLPT level of the word.
+  final JlptLevel jlptLevel;
+
   /// A class listing the sources used to make up the data for this word search result.
   final WordSearchSources sources;
 
-  const WordSearchResult({
+  /// A list of spans, specifying which part of this word result matched the search keyword.
+  ///
+  /// Note that this is considered ephemeral data - it does not originate from the dictionary,
+  /// and unlike the rest of the class it varies based on external information (the searchword).
+  /// It will *NOT* be exported to JSON, but can be reinferred by invoking [inferMatchSpans] with
+  /// the original searchword.
+  List<WordSearchMatchSpan>? matchSpans;
+
+  /// Whether the first item in [japanese] contains kanji that likely is rare.
+  bool get hasUnusualKanji =>
+      (japanese.first.furigana != null &&
+          kanjiInfo[japanese.first.base] == JMdictKanjiInfo.rK) ||
+      senses.where((sense) => sense.misc.contains(JMdictMisc.onlyKana)).length >
+          (senses.length / 2);
+
+  /// All contents of [japanese], transliterated to romaji
+  List<String> get romaji => japanese
+      .map((word) => transliterateKanaToLatin(word.furigana ?? word.base))
+      .toList();
+
+  /// All contents of [japanase], where the furigana has either been transliterated to romaji, or
+  /// contains the furigana transliteration of [WordSearchRuby.base].
+  List<WordSearchRuby> get romajiRubys => japanese
+      .map(
+        (word) => WordSearchRuby(
+          base: word.base,
+          furigana: word.furigana != null
+              ? transliterateKanaToLatin(word.furigana!)
+              : transliterateKanaToLatin(word.base),
+        ),
+      )
+      .toList();
+
+  /// The same list of spans as [matchSpans], but the positions have been adjusted for romaji conversion
+  ///
+  /// This is mostly useful in conjunction with [romajiRubys].
+  List<WordSearchMatchSpan>? get romajiMatchSpans {
+    if (matchSpans == null) {
+      return null;
+    }
+
+    throw UnimplementedError('Not yet implemented');
+  }
+
+  WordSearchResult({
+    required this.score,
     required this.entryId,
+    required this.isCommon,
     required this.japanese,
     required this.kanjiInfo,
     required this.readingInfo,
     required this.senses,
+    required this.jlptLevel,
     required this.sources,
+    this.matchSpans,
   });
 
   Map<String, dynamic> toJson() => {
-        'entryId': entryId,
-        'japanese': japanese.map((e) => e.toJson()).toList(),
-        'kanjiInfo':
-            kanjiInfo.map((key, value) => MapEntry(key, value.toJson())),
-        'readingInfo':
-            readingInfo.map((key, value) => MapEntry(key, value.toJson())),
-        'senses': senses.map((e) => e.toJson()).toList(),
-        'sources': sources.toJson(),
-      };
+    '_score': score,
+    'entryId': entryId,
+    'isCommon': isCommon,
+    'japanese': japanese.map((e) => e.toJson()).toList(),
+    'kanjiInfo': kanjiInfo.map((key, value) => MapEntry(key, value.toJson())),
+    'readingInfo': readingInfo.map(
+      (key, value) => MapEntry(key, value.toJson()),
+    ),
+    'senses': senses.map((e) => e.toJson()).toList(),
+    'jlptLevel': jlptLevel.toJson(),
+    'sources': sources.toJson(),
+  };
 
   factory WordSearchResult.fromJson(Map<String, dynamic> json) =>
       WordSearchResult(
+        score: json['_score'] as int,
         entryId: json['entryId'] as int,
+        isCommon: json['isCommon'] as bool,
         japanese: (json['japanese'] as List<dynamic>)
             .map((e) => WordSearchRuby.fromJson(e))
             .toList(),
@@ -59,6 +126,95 @@ class WordSearchResult {
         senses: (json['senses'] as List<dynamic>)
             .map((e) => WordSearchSense.fromJson(e))
             .toList(),
+        jlptLevel: JlptLevel.fromJson(json['jlptLevel'] as Object?),
         sources: WordSearchSources.fromJson(json['sources']),
       );
+
+  factory WordSearchResult.empty() => WordSearchResult(
+    score: 0,
+    entryId: 0,
+    isCommon: false,
+    japanese: [],
+    kanjiInfo: {},
+    readingInfo: {},
+    senses: [],
+    jlptLevel: JlptLevel.none,
+    sources: WordSearchSources.empty(),
+  );
+
+  /// Infers which part(s) of this word search result matched the search keyword, and populates [matchSpans] accordingly.
+  void inferMatchSpans(
+    String searchword, {
+    SearchMode searchMode = SearchMode.auto,
+  }) {
+    // TODO: handle wildcards like '?' and '*' when that becomes supported in the search.
+    // TODO: If the searchMode is provided, we can use that to narrow down which part of the word search results to look at.
+
+    final regex = RegExp(RegExp.escape(searchword));
+    final matchSpans = <WordSearchMatchSpan>[];
+
+    for (final (i, japanese) in japanese.indexed) {
+      final baseMatches = regex.allMatches(japanese.base);
+      matchSpans.addAll(
+        baseMatches.map(
+          (match) => WordSearchMatchSpan(
+            spanType: WordSearchMatchSpanType.kanji,
+            index: i,
+            start: match.start,
+            end: match.end,
+          ),
+        ),
+      );
+
+      if (japanese.furigana != null) {
+        final furiganaMatches = regex.allMatches(japanese.furigana!);
+        matchSpans.addAll(
+          furiganaMatches.map(
+            (match) => WordSearchMatchSpan(
+              spanType: WordSearchMatchSpanType.kana,
+              index: i,
+              start: match.start,
+              end: match.end,
+            ),
+          ),
+        );
+      }
+    }
+
+    for (final (i, sense) in senses.indexed) {
+      for (final (k, definition) in sense.englishDefinitions.indexed) {
+        final definitionMatches = regex.allMatches(definition);
+        matchSpans.addAll(
+          definitionMatches.map(
+            (match) => WordSearchMatchSpan(
+              spanType: WordSearchMatchSpanType.sense,
+              index: i,
+              subIndex: k,
+              start: match.start,
+              end: match.end,
+            ),
+          ),
+        );
+      }
+    }
+
+    this.matchSpans = matchSpans;
+  }
+
+  static String _formatJapaneseWord(WordSearchRuby word) =>
+      word.furigana == null ? word.base : '${word.base} (${word.furigana})';
+
+  @override
+  String toString() {
+    final japaneseWord = _formatJapaneseWord(japanese[0]);
+    final isCommonString = isCommon ? '(C)' : '';
+    final jlptLevelString = '(${jlptLevel.toString()})';
+
+    return '''
+$score | [$entryId] $japaneseWord $isCommonString $jlptLevelString
+      Other forms: ${japanese.skip(1).map(_formatJapaneseWord).join(', ')}
+      Senses: ${senses.map((s) => s.englishDefinitions).join(', ')}
+    '''
+        .trim();
+  }
 }

lib/models/word_search/word_search_ruby.dart

@@ -6,18 +6,12 @@ class WordSearchRuby {
   /// Furigana, if applicable.
   String? furigana;
 
-  WordSearchRuby({
-    required this.base,
-    this.furigana,
-  });
+  WordSearchRuby({required this.base, this.furigana});
 
-  Map<String, dynamic> toJson() => {
-        'base': base,
-        'furigana': furigana,
-      };
+  Map<String, dynamic> toJson() => {'base': base, 'furigana': furigana};
 
   factory WordSearchRuby.fromJson(Map<String, dynamic> json) => WordSearchRuby(
-        base: json['base'] as String,
-        furigana: json['furigana'] as String?,
-      );
+    base: json['base'] as String,
+    furigana: json['furigana'] as String?,
+  );
 }

lib/models/word_search/word_search_sense.dart

@@ -2,6 +2,7 @@ import 'package:jadb/models/jmdict/jmdict_dialect.dart';
 import 'package:jadb/models/jmdict/jmdict_field.dart';
 import 'package:jadb/models/jmdict/jmdict_misc.dart';
 import 'package:jadb/models/jmdict/jmdict_pos.dart';
+import 'package:jadb/models/word_search/word_search_sense_language_source.dart';
 import 'package:jadb/models/word_search/word_search_xref_entry.dart';
 
 class WordSearchSense {
@@ -38,7 +39,7 @@ class WordSearchSense {
   // TODO: there is a lot more info to collect in the languageSource data
 
   /// Information about the the origin of the word, if loaned from another language.
-  final List<String> languageSource;
+  final List<WordSearchSenseLanguageSource> languageSource;
 
   // TODO: add example sentences
 
@@ -70,18 +71,18 @@ class WordSearchSense {
       languageSource.isEmpty;
 
   Map<String, dynamic> toJson() => {
-        'englishDefinitions': englishDefinitions,
-        'partsOfSpeech': partsOfSpeech.map((e) => e.toJson()).toList(),
-        'seeAlso': seeAlso.map((e) => e.toJson()).toList(),
-        'antonyms': antonyms.map((e) => e.toJson()).toList(),
-        'restrictedToReading': restrictedToReading,
-        'restrictedToKanji': restrictedToKanji,
-        'fields': fields.map((e) => e.toJson()).toList(),
-        'dialects': dialects.map((e) => e.toJson()).toList(),
-        'misc': misc.map((e) => e.toJson()).toList(),
-        'info': info,
-        'languageSource': languageSource,
-      };
+    'englishDefinitions': englishDefinitions,
+    'partsOfSpeech': partsOfSpeech.map((e) => e.toJson()).toList(),
+    'seeAlso': seeAlso.map((e) => e.toJson()).toList(),
+    'antonyms': antonyms.map((e) => e.toJson()).toList(),
+    'restrictedToReading': restrictedToReading,
+    'restrictedToKanji': restrictedToKanji,
+    'fields': fields.map((e) => e.toJson()).toList(),
+    'dialects': dialects.map((e) => e.toJson()).toList(),
+    'misc': misc.map((e) => e.toJson()).toList(),
+    'info': info,
+    'languageSource': languageSource,
+  };
 
   factory WordSearchSense.fromJson(Map<String, dynamic> json) =>
       WordSearchSense(
@@ -103,9 +104,12 @@ class WordSearchSense {
         dialects: (json['dialects'] as List)
             .map((e) => JMdictDialect.fromJson(e))
             .toList(),
-        misc:
-            (json['misc'] as List).map((e) => JMdictMisc.fromJson(e)).toList(),
+        misc: (json['misc'] as List)
+            .map((e) => JMdictMisc.fromJson(e))
+            .toList(),
         info: List<String>.from(json['info']),
-        languageSource: List<String>.from(json['languageSource']),
+        languageSource: (json['languageSource'] as List)
+            .map((e) => WordSearchSenseLanguageSource.fromJson(e))
+            .toList(),
       );
 }

lib/models/word_search/word_search_sense_language_source.dart

@@ -0,0 +1,30 @@
+/// A reference to a foreign language where this sense originates from.
+class WordSearchSenseLanguageSource {
+  final String language;
+  final String? phrase;
+  final bool fullyDescribesSense;
+  final bool constructedFromSmallerWords;
+
+  const WordSearchSenseLanguageSource({
+    required this.language,
+    this.phrase,
+    this.fullyDescribesSense = true,
+    this.constructedFromSmallerWords = false,
+  });
+
+  Map<String, Object?> toJson() => {
+    'language': language,
+    'phrase': phrase,
+    'fullyDescribesSense': fullyDescribesSense,
+    'constructedFromSmallerWords': constructedFromSmallerWords,
+  };
+
+  factory WordSearchSenseLanguageSource.fromJson(Map<String, dynamic> json) =>
+      WordSearchSenseLanguageSource(
+        language: json['language'],
+        phrase: json['phrase'],
+        fullyDescribesSense: json['fullyDescribesSense'] ?? true,
+        constructedFromSmallerWords:
+            json['constructedFromSmallerWords'] ?? false,
+      );
+}

lib/models/word_search/word_search_sources.dart

@@ -7,20 +7,13 @@ class WordSearchSources {
   /// Whether JMnedict was used.
   final bool jmnedict;
 
-  const WordSearchSources({
-    this.jmdict = true,
-    this.jmnedict = false,
-  });
+  const WordSearchSources({this.jmdict = true, this.jmnedict = false});
 
-  Map<String, Object?> get sqlValue => {
-        'jmdict': jmdict,
-        'jmnedict': jmnedict,
-      };
+  factory WordSearchSources.empty() => const WordSearchSources();
 
-  Map<String, dynamic> toJson() => {
-        'jmdict': jmdict,
-        'jmnedict': jmnedict,
-      };
+  Map<String, Object?> get sqlValue => {'jmdict': jmdict, 'jmnedict': jmnedict};
+
+  Map<String, dynamic> toJson() => {'jmdict': jmdict, 'jmnedict': jmnedict};
 
   factory WordSearchSources.fromJson(Map<String, dynamic> json) =>
       WordSearchSources(

lib/models/word_search/word_search_xref_entry.dart

@@ -1,25 +1,45 @@
+import 'package:jadb/models/word_search/word_search_result.dart';
+
 /// A cross-reference entry from one word-result to another entry.
 class WordSearchXrefEntry {
   /// The ID of the entry that this entry cross-references to.
   final int entryId;
 
+  /// The base word of the cross-referenced entry.
+  final String baseWord;
+
+  /// The furigana of the cross-referenced entry, if any.
+  final String? furigana;
+
   /// Whether the entryId was ambiguous during the creation of the
   /// database (and hence might be incorrect).
   final bool ambiguous;
 
+  /// The result of the cross-reference, may or may not be included in the query.
+  final WordSearchResult? xrefResult;
+
   const WordSearchXrefEntry({
     required this.entryId,
     required this.ambiguous,
+    required this.baseWord,
+    required this.furigana,
+    required this.xrefResult,
   });
 
   Map<String, dynamic> toJson() => {
-        'entryId': entryId,
-        'ambiguous': ambiguous,
-      };
+    'entryId': entryId,
+    'ambiguous': ambiguous,
+    'baseWord': baseWord,
+    'furigana': furigana,
+    'xrefResult': xrefResult?.toJson(),
+  };
 
   factory WordSearchXrefEntry.fromJson(Map<String, dynamic> json) =>
       WordSearchXrefEntry(
         entryId: json['entryId'] as int,
         ambiguous: json['ambiguous'] as bool,
+        baseWord: json['baseWord'] as String,
+        furigana: json['furigana'] as String?,
+        xrefResult: null,
       );
 }

lib/search.dart

@@ -1,25 +1,71 @@
-import 'package:jadb/models/word_search/word_search_result.dart';
 import 'package:jadb/models/kanji_search/kanji_search_result.dart';
-import 'package:jadb/models/radkfile/radicals_search_result.dart';
-import 'package:jadb/search/word_search.dart';
-
+import 'package:jadb/models/verify_tables.dart';
+import 'package:jadb/models/word_search/word_search_result.dart';
+import 'package:jadb/search/filter_kanji.dart';
 import 'package:jadb/search/kanji_search.dart';
-
+import 'package:jadb/search/radical_search.dart';
+import 'package:jadb/search/word_search/word_search.dart';
 import 'package:sqflite_common/sqlite_api.dart';
 
-class JaDBConnection {
-  final DatabaseExecutor _connection;
+extension JaDBConnection on DatabaseExecutor {
+  /// Ensure that the database contain all JaDB tables.
+  ///
+  /// This will throw an exception if any of the tables are missing.
+  Future<void> jadbVerifyTables() => verifyTablesWithDbConnection(this);
 
-  const JaDBConnection(this._connection);
+  /// Search for a kanji in the database.
+  Future<KanjiSearchResult?> jadbSearchKanji(String kanji) =>
+      searchKanjiWithDbConnection(this, kanji);
 
-  Future<KanjiSearchResult?> searchKanji(String kanji) async =>
-      searchKanjiWithDbConnection(this._connection, kanji);
+  /// Search for a kanji in the database.
+  Future<Map<String, KanjiSearchResult>> jadbGetManyKanji(Set<String> kanji) =>
+      searchManyKanjiWithDbConnection(this, kanji);
 
-  Future<RadicalsSearchResult> searchKanjiByRadicals(
-      List<String> radicals) async {
-    throw UnimplementedError();
-  }
+  /// Filter a list of characters, and return the ones that are listed in the kanji dictionary.
+  Future<List<String>> filterKanji(
+    List<String> kanji, {
+    bool deduplicate = false,
+  }) => filterKanjiWithDbConnection(this, kanji, deduplicate);
 
-  Future<List<WordSearchResult>?> searchWord(String word) async =>
-      searchWordWithDbConnection(this._connection, word);
+  /// Search for a word in the database.
+  Future<List<WordSearchResult>?> jadbSearchWord(
+    String word, {
+    SearchMode searchMode = SearchMode.auto,
+    int page = 0,
+    int? pageSize,
+  }) => searchWordWithDbConnection(
+    this,
+    word,
+    searchMode: searchMode,
+    page: page,
+    pageSize: pageSize,
+  );
+
+  ///
+  Future<WordSearchResult?> jadbGetWordById(int id) =>
+      getWordByIdWithDbConnection(this, id);
+
+  /// Get a list of words by their IDs.
+  ///
+  /// IDs for which no result is found are omitted from the returned value.
+  Future<Map<int, WordSearchResult>> jadbGetManyWordsByIds(Set<int> ids) =>
+      getWordsByIdsWithDbConnection(this, ids);
+
+  /// Search for a word in the database, and return the count of results.
+  Future<int?> jadbSearchWordCount(
+    String word, {
+    SearchMode searchMode = SearchMode.auto,
+  }) => searchWordCountWithDbConnection(this, word, searchMode: searchMode);
+
+  /// Given a list of radicals, search which kanji contains all
+  /// of the radicals, find their other radicals, and return those.
+  /// This is used to figure out which remaining combinations of radicals
+  /// the user can search for without getting zero results.
+  Future<List<String>> jadbSearchRemainingRadicals(List<String> radicals) =>
+      searchRemainingRadicalsWithDbConnection(this, radicals);
+
+  /// Given a list of radicals, search which kanji contains all
+  /// of the radicals, and return those.
+  Future<List<String>> jadbSearchKanjiByRadicals(List<String> radicals) =>
+      searchKanjiByRadicalsWithDbConnection(this, radicals);
 }

lib/search/filter_kanji.dart

@@ -0,0 +1,33 @@
+import 'package:jadb/table_names/kanjidic.dart';
+import 'package:sqflite_common/sqflite.dart';
+
+/// Filters a list of kanji characters, returning only those that exist in the database.
+///
+/// If [deduplicate] is true, the returned list will deduplicate the input kanji list before returning the filtered results.
+Future<List<String>> filterKanjiWithDbConnection(
+  DatabaseExecutor connection,
+  List<String> kanji,
+  bool deduplicate,
+) async {
+  final Set<String> filteredKanji = await connection
+      .rawQuery('''
+      SELECT "literal"
+      FROM "${KANJIDICTableNames.character}"
+      WHERE "literal" IN (${kanji.map((_) => '?').join(',')})
+    ''', kanji)
+      .then((value) => value.map((e) => e['literal'] as String).toSet());
+
+  if (deduplicate) {
+    final List<String> result = [];
+    final Set<String> seen = {};
+    for (final k in kanji) {
+      if (filteredKanji.contains(k) && !seen.contains(k)) {
+        result.add(k);
+        seen.add(k);
+      }
+    }
+    return result;
+  } else {
+    return kanji.where((k) => filteredKanji.contains(k)).toList();
+  }
+}

lib/search/kanji_search.dart

@@ -1,141 +1,229 @@
+import 'package:collection/collection.dart';
+import 'package:jadb/models/kanji_search/kanji_search_radical.dart';
 import 'package:jadb/models/kanji_search/kanji_search_result.dart';
+import 'package:jadb/table_names/kanjidic.dart';
+import 'package:jadb/table_names/radkfile.dart';
 import 'package:sqflite_common/sqflite.dart';
 
+Future<List<Map<String, Object?>>> _charactersQuery(
+  DatabaseExecutor connection,
+  String kanji,
+) => connection.query(
+  KANJIDICTableNames.character,
+  where: 'literal = ?',
+  whereArgs: [kanji],
+);
+
+Future<List<Map<String, Object?>>> _codepointsQuery(
+  DatabaseExecutor connection,
+  String kanji,
+) => connection.query(
+  KANJIDICTableNames.codepoint,
+  where: 'kanji = ?',
+  whereArgs: [kanji],
+);
+
+Future<List<Map<String, Object?>>> _kunyomisQuery(
+  DatabaseExecutor connection,
+  String kanji,
+) => connection.query(
+  KANJIDICTableNames.kunyomi,
+  where: 'kanji = ?',
+  whereArgs: [kanji],
+  orderBy: 'orderNum',
+);
+
+Future<List<Map<String, Object?>>> _onyomisQuery(
+  DatabaseExecutor connection,
+  String kanji,
+) => connection.query(
+  KANJIDICTableNames.onyomi,
+  where: 'kanji = ?',
+  whereArgs: [kanji],
+  orderBy: 'orderNum',
+);
+
+Future<List<Map<String, Object?>>> _meaningsQuery(
+  DatabaseExecutor connection,
+  String kanji,
+) => connection.query(
+  KANJIDICTableNames.meaning,
+  where: 'kanji = ? AND language = ?',
+  whereArgs: [kanji, 'eng'],
+  orderBy: 'orderNum',
+);
+
+Future<List<Map<String, Object?>>> _nanorisQuery(
+  DatabaseExecutor connection,
+  String kanji,
+) => connection.query(
+  KANJIDICTableNames.nanori,
+  where: 'kanji = ?',
+  whereArgs: [kanji],
+);
+
+Future<List<Map<String, Object?>>> _dictionaryReferencesQuery(
+  DatabaseExecutor connection,
+  String kanji,
+) => connection.query(
+  KANJIDICTableNames.dictionaryReference,
+  where: 'kanji = ?',
+  whereArgs: [kanji],
+);
+
+Future<List<Map<String, Object?>>> _queryCodesQuery(
+  DatabaseExecutor connection,
+  String kanji,
+) => connection.query(
+  KANJIDICTableNames.queryCode,
+  where: 'kanji = ?',
+  whereArgs: [kanji],
+);
+
+Future<List<Map<String, Object?>>> _radicalsQuery(
+  DatabaseExecutor connection,
+  String kanji,
+) => connection.rawQuery(
+  '''
+    SELECT DISTINCT
+      "XREF__KANJIDIC_Radical__RADKFILE"."radicalSymbol" AS "symbol",
+      "names"
+    FROM "${KANJIDICTableNames.radical}"
+    JOIN "XREF__KANJIDIC_Radical__RADKFILE" USING ("radicalId")
+    LEFT JOIN (
+      SELECT "radicalId", group_concat("name") AS "names"
+      FROM "${KANJIDICTableNames.radicalName}"
+      GROUP BY "radicalId"
+    ) USING ("radicalId")
+    WHERE "${KANJIDICTableNames.radical}"."kanji" = ?
+  ''',
+  [kanji],
+);
+
+Future<List<Map<String, Object?>>> _partsQuery(
+  DatabaseExecutor connection,
+  String kanji,
+) => connection.query(
+  RADKFILETableNames.radkfile,
+  where: 'kanji = ?',
+  whereArgs: [kanji],
+);
+
+Future<List<Map<String, Object?>>> _readingsQuery(
+  DatabaseExecutor connection,
+  String kanji,
+) => connection.query(
+  KANJIDICTableNames.reading,
+  where: 'kanji = ?',
+  whereArgs: [kanji],
+);
+
+Future<List<Map<String, Object?>>> _strokeMiscountsQuery(
+  DatabaseExecutor connection,
+  String kanji,
+) => connection.query(
+  KANJIDICTableNames.strokeMiscount,
+  where: 'kanji = ?',
+  whereArgs: [kanji],
+);
+
+// Future<List<Map<String, Object?>>> _variantsQuery(
+//   DatabaseExecutor connection,
+//   String kanji,
+// ) => connection.query(
+//   KANJIDICTableNames.variant,
+//   where: 'kanji = ?',
+//   whereArgs: [kanji],
+// );
+
+/// Searches for a kanji character and returns its details, or null if the kanji is not found in the database.
 Future<KanjiSearchResult?> searchKanjiWithDbConnection(
   DatabaseExecutor connection,
   String kanji,
 ) async {
   late final List<Map<String, Object?>> characters;
-  final characters_query = connection.query(
-    "KANJIDIC_Character",
-    where: "KANJIDIC_Character.literal = ?",
-    whereArgs: [kanji],
-  );
-
   late final List<Map<String, Object?>> codepoints;
-  final codepoints_query = connection.query(
-    "KANJIDIC_Codepoint",
-    where: "KANJIDIC_Codepoint.kanji = ?",
-    whereArgs: [kanji],
-  );
-
   late final List<Map<String, Object?>> kunyomis;
-  final kunyomis_query = connection.query(
-    "KANJIDIC_Kunyomi",
-    where: "KANJIDIC_Kunyomi.kanji = ?",
-    whereArgs: [kanji],
-  );
-
   late final List<Map<String, Object?>> onyomis;
-  final onyomis_query = connection.query(
-    "KANJIDIC_Onyomi",
-    where: "KANJIDIC_Onyomi.kanji = ?",
-    whereArgs: [kanji],
-  );
-
   late final List<Map<String, Object?>> meanings;
-  final meanings_query = connection.query(
-    "KANJIDIC_Meaning",
-    where: "KANJIDIC_Meaning.kanji = ? AND KANJIDIC_Meaning.language = ?",
-    whereArgs: [kanji, 'eng'],
-  );
-
   late final List<Map<String, Object?>> nanoris;
-  final nanoris_query = connection.query(
-    "KANJIDIC_Nanori",
-    where: "KANJIDIC_Nanori.kanji = ?",
-    whereArgs: [kanji],
-  );
-
-  late final List<Map<String, Object?>> dictionary_references;
-  final dictionary_references_query = connection.query(
-    "KANJIDIC_DictionaryReference",
-    where: "KANJIDIC_DictionaryReference.kanji = ?",
-    whereArgs: [kanji],
-  );
-
-  late final List<Map<String, Object?>> query_codes;
-  final query_codes_query = connection.query(
-    "KANJIDIC_QueryCode",
-    where: "KANJIDIC_QueryCode.kanji = ?",
-    whereArgs: [kanji],
-  );
-
+  late final List<Map<String, Object?>> dictionaryReferences;
+  late final List<Map<String, Object?>> queryCodes;
   late final List<Map<String, Object?>> radicals;
-  final radicals_query = connection.query(
-    "KANJIDIC_Radical",
-    where: "KANJIDIC_Radical.kanji = ?",
-    whereArgs: [kanji],
-  );
-
-  late final List<Map<String, Object?>> radical_names;
-  final radical_names_query = connection.query(
-    "KANJIDIC_RadicalName",
-    where: "KANJIDIC_RadicalName.kanji = ?",
-    whereArgs: [kanji],
-  );
-
+  late final List<Map<String, Object?>> parts;
   late final List<Map<String, Object?>> readings;
-  final readings_query = connection.query(
-    "KANJIDIC_Reading",
-    where: "KANJIDIC_Reading.kanji = ?",
-    whereArgs: [kanji],
-  );
+  late final List<Map<String, Object?>> strokeMiscounts;
+  // TODO: add variant data to result
+  // late final List<Map<String, Object?>> variants;
 
-  late final List<Map<String, Object?>> stroke_miscounts;
-  final stroke_miscounts_query = connection.query(
-    "KANJIDIC_StrokeMiscount",
-    where: "KANJIDIC_StrokeMiscount.kanji = ?",
-    whereArgs: [kanji],
-  );
+  // TODO: Search for kunyomi and onyomi usage of the characters
+  //       from JMDict. We'll need to fuzzy aquery JMDict_KanjiElement for matches,
+  //       filter JMdict_ReadingElement for kunyomi/onyomi, and then sort the main entry
+  //       by JLPT, news frequency, etc.
 
-  late final List<Map<String, Object?>> variants;
-  final variants_query = connection.query(
-    "KANJIDIC_Variant",
-    where: "KANJIDIC_Variant.kanji = ?",
-    whereArgs: [kanji],
-  );
-
-// TODO: Search for kunyomi and onyomi usage of the characters
-//       from JMDict. We'll need to fuzzy aquery JMDict_KanjiElement for mathces,
-//       filter JMdict_ReadingElement for kunyomi/onyomi, and then sort the main entry
-//       by JLPT, news frequency, etc.
-
-  await characters_query.then((value) => characters = value);
+  await _charactersQuery(connection, kanji).then((value) => characters = value);
 
   if (characters.isEmpty) {
     return null;
   }
 
   await Future.wait({
-    codepoints_query.then((value) => codepoints = value),
-    kunyomis_query.then((value) => kunyomis = value),
-    onyomis_query.then((value) => onyomis = value),
-    meanings_query.then((value) => meanings = value),
-    nanoris_query.then((value) => nanoris = value),
-    dictionary_references_query.then((value) => dictionary_references = value),
-    query_codes_query.then((value) => query_codes = value),
-    radicals_query.then((value) => radicals = value),
-    radical_names_query.then((value) => radical_names = value),
-    readings_query.then((value) => readings = value),
-    stroke_miscounts_query.then((value) => stroke_miscounts = value),
-    variants_query.then((value) => variants = value),
+    _codepointsQuery(connection, kanji).then((value) => codepoints = value),
+    _kunyomisQuery(connection, kanji).then((value) => kunyomis = value),
+    _onyomisQuery(connection, kanji).then((value) => onyomis = value),
+    _meaningsQuery(connection, kanji).then((value) => meanings = value),
+    _nanorisQuery(connection, kanji).then((value) => nanoris = value),
+    _dictionaryReferencesQuery(
+      connection,
+      kanji,
+    ).then((value) => dictionaryReferences = value),
+    _queryCodesQuery(connection, kanji).then((value) => queryCodes = value),
+    _radicalsQuery(connection, kanji).then((value) => radicals = value),
+    _partsQuery(connection, kanji).then((value) => parts = value),
+    _readingsQuery(connection, kanji).then((value) => readings = value),
+    _strokeMiscountsQuery(
+      connection,
+      kanji,
+    ).then((value) => strokeMiscounts = value),
+    // variants_query.then((value) => variants = value),
   });
 
   final entry = characters.first;
 
-  final String? grade = {
-    1: 'grade 1',
-    2: 'grade 2',
-    3: 'grade 3',
-    4: 'grade 4',
-    5: 'grade 5',
-    6: 'grade 6',
-    7: 'grade 7',
-    8: 'grade 8',
-    9: 'grade 9',
-    10: 'grade 10',
-  }[entry['grade'] as int?];
+  assert(radicals.length <= 1, 'There should be at most one radical per kanji');
+  final radical = radicals.isNotEmpty
+      ? KanjiSearchRadical(
+          symbol: radicals.first['symbol'] as String,
+          names: (radicals.first['names'] as String?)?.split(',') ?? [],
+          // TODO: add radical form data
+          forms: [],
+          // TODO: add radical meaning data
+          meanings: [],
+        )
+      : null;
+
+  final alternativeLanguageReadings = readings
+      .groupListsBy((item) => item['type'] as String)
+      .map(
+        (key, value) => MapEntry(
+          key,
+          value.map((item) => item['reading'] as String).toList(),
+        ),
+      );
+
+  // TODO: Add `SKIPMisclassification` to the entries
+  final queryCodes_ = queryCodes
+      .groupListsBy((item) => item['type'] as String)
+      .map(
+        (key, value) =>
+            MapEntry(key, value.map((item) => item['code'] as String).toList()),
+      );
+
+  // TODO: Add `volume` and `page` to the entries
+  final dictionaryReferences_ = {
+    for (final entry in dictionaryReferences)
+      entry['type'] as String: entry['ref'] as String,
+  };
 
   final String? jlptLevel = {
     5: 'N5',
@@ -147,12 +235,48 @@ Future<KanjiSearchResult?> searchKanjiWithDbConnection(
 
   return KanjiSearchResult(
     kanji: entry['literal']! as String,
-    taughtIn: grade,
+    taughtIn: entry['grade'] as int?,
     jlptLevel: jlptLevel,
     newspaperFrequencyRank: entry['frequency'] as int?,
     strokeCount: entry['strokeCount'] as int,
     meanings: meanings.map((item) => item['meaning'] as String).toList(),
     kunyomi: kunyomis.map((item) => item['yomi'] as String).toList(),
+    parts: parts.map((item) => item['radical'] as String).toList(),
     onyomi: onyomis.map((item) => item['yomi'] as String).toList(),
+    radical: radical,
+    codepoints: {
+      for (final codepoint in codepoints)
+        codepoint['type'] as String: codepoint['codepoint'] as String,
+    },
+    nanori: nanoris.map((item) => item['nanori'] as String).toList(),
+    alternativeLanguageReadings: alternativeLanguageReadings,
+    strokeMiscounts: strokeMiscounts
+        .map((item) => item['strokeCount'] as int)
+        .toList(),
+    queryCodes: queryCodes_,
+    dictionaryReferences: dictionaryReferences_,
   );
 }
+
+// TODO: Use fewer queries with `IN` clauses to reduce the number of queries
+
+/// Searches for multiple kanji at once, returning a map of kanji to their search results.
+Future<Map<String, KanjiSearchResult>> searchManyKanjiWithDbConnection(
+  DatabaseExecutor connection,
+  Set<String> kanji,
+) async {
+  if (kanji.isEmpty) {
+    return {};
+  }
+
+  final results = <String, KanjiSearchResult>{};
+
+  for (final k in kanji) {
+    final result = await searchKanjiWithDbConnection(connection, k);
+    if (result != null) {
+      results[k] = result;
+    }
+  }
+
+  return results;
+}

lib/search/radical_search.dart

@@ -0,0 +1,59 @@
+import 'package:jadb/table_names/radkfile.dart';
+import 'package:sqflite_common/sqlite_api.dart';
+
+// TODO: validate that the list of radicals all are valid radicals
+
+/// Returns a list of radicals that are part of any kanji that contains all of the input radicals.
+///
+/// This can be used to limit the choices of additional radicals provided to a user,
+/// so that any choice they make will still yield at least one kanji.
+Future<List<String>> searchRemainingRadicalsWithDbConnection(
+  DatabaseExecutor connection,
+  List<String> radicals,
+) async {
+  final distinctRadicals = radicals.toSet();
+
+  final queryResult = await connection.rawQuery(
+    '''
+      SELECT DISTINCT "radical"
+      FROM "${RADKFILETableNames.radkfile}"
+      WHERE "kanji" IN (
+        SELECT "kanji"
+        FROM "${RADKFILETableNames.radkfile}"
+        WHERE "radical" IN (${List.filled(distinctRadicals.length, '?').join(',')})
+        GROUP BY "kanji"
+        HAVING COUNT(DISTINCT "radical") = ?
+      )
+    ''',
+    [...distinctRadicals, distinctRadicals.length],
+  );
+
+  final remainingRadicals = queryResult
+      .map((row) => row['radical'] as String)
+      .toList();
+
+  return remainingRadicals;
+}
+
+/// Returns a list of kanji that contain all of the input radicals.
+Future<List<String>> searchKanjiByRadicalsWithDbConnection(
+  DatabaseExecutor connection,
+  List<String> radicals,
+) async {
+  final distinctRadicals = radicals.toSet();
+
+  final queryResult = await connection.rawQuery(
+    '''
+      SELECT "kanji"
+      FROM "${RADKFILETableNames.radkfile}"
+      WHERE "radical" IN (${List.filled(distinctRadicals.length, '?').join(',')})
+      GROUP BY "kanji"
+      HAVING COUNT(DISTINCT "radical") = ?
+    ''',
+    [...distinctRadicals, distinctRadicals.length],
+  );
+
+  final kanji = queryResult.map((row) => row['kanji'] as String).toList();
+
+  return kanji;
+}

lib/search/word_search.dart

@@ -1,503 +0,0 @@
-import 'package:collection/collection.dart';
-import 'package:jadb/models/jmdict/jmdict_dialect.dart';
-import 'package:jadb/models/jmdict/jmdict_field.dart';
-import 'package:jadb/models/jmdict/jmdict_kanji_info.dart';
-import 'package:jadb/models/jmdict/jmdict_misc.dart';
-import 'package:jadb/models/jmdict/jmdict_pos.dart';
-import 'package:jadb/models/jmdict/jmdict_reading_info.dart';
-import 'package:jadb/models/word_search/word_search_result.dart';
-import 'package:jadb/models/word_search/word_search_ruby.dart';
-import 'package:jadb/models/word_search/word_search_sense.dart';
-import 'package:jadb/models/word_search/word_search_sources.dart';
-import 'package:jadb/models/word_search/word_search_xref_entry.dart';
-import 'package:jadb/util/sqlite_utils.dart';
-import 'package:sqflite_common/sqlite_api.dart';
-
-// TODO: Support globs
-
-// TODO: Support tags
-
-// TODO: Prefer original kana type when sorting results
-
-// TODO: Support mixing kana and romaji
-
-Future<List<WordSearchResult>?> searchWordWithDbConnection(
-  DatabaseExecutor connection,
-  String word, {
-  bool isKana = true,
-}) async {
-  if (word.isEmpty) {
-    return null;
-  }
-
-  late final List<int> entryIds;
-  if (isKana) {
-    entryIds = (await connection.query(
-      'JMdict_EntryByKana',
-      where: 'kana LIKE ?',
-      whereArgs: ['$word%'],
-    ))
-        .map((row) => row['entryId'] as int)
-        .toList();
-  } else {
-    entryIds = (await connection.query(
-      'JMdict_EntryByEnglish',
-      where: 'english LIKE ?',
-      whereArgs: ['$word%'],
-    ))
-        .map((row) => row['entryId'] as int)
-        .toList();
-  }
-
-  if (entryIds.isEmpty) {
-    return [];
-  }
-
-  late final List<Map<String, Object?>> senses;
-  final Future<List<Map<String, Object?>>> senses_query = connection.query(
-    'JMdict_Sense',
-    where: 'entryId IN (${entryIds.join(',')})',
-  );
-
-  late final List<Map<String, Object?>> readingElements;
-  final Future<List<Map<String, Object?>>> readingElements_query =
-      connection.query(
-    'JMdict_ReadingElement',
-    where: 'entryId IN (${entryIds.join(',')})',
-  );
-
-  late final List<Map<String, Object?>> kanjiElements;
-  final Future<List<Map<String, Object?>>> kanjiElements_query =
-      connection.query(
-    'JMdict_KanjiElement',
-    where: 'entryId IN (${entryIds.join(',')})',
-  );
-
-  await Future.wait([
-    senses_query.then((value) => senses = value),
-    readingElements_query.then((value) => readingElements = value),
-    kanjiElements_query.then((value) => kanjiElements = value),
-  ]);
-
-  // Sense queries
-
-  final senseIds = senses.map((element) => element['id'] as int).toList();
-
-  late final List<Map<String, Object?>> senseAntonyms;
-  final Future<List<Map<String, Object?>>> senseAntonyms_query =
-      connection.query(
-    'JMdict_SenseAntonym',
-    where: 'senseId IN (${senseIds.join(',')})',
-  );
-
-  late final List<Map<String, Object?>> senseDialects;
-  final Future<List<Map<String, Object?>>> senseDialects_query =
-      connection.query(
-    'JMdict_SenseDialect',
-    where: 'senseId IN (${senseIds.join(',')})',
-  );
-
-  late final List<Map<String, Object?>> senseFields;
-  final Future<List<Map<String, Object?>>> senseFields_query = connection.query(
-    'JMdict_SenseField',
-    where: 'senseId IN (${senseIds.join(',')})',
-  );
-
-  late final List<Map<String, Object?>> senseGlossaries;
-  final Future<List<Map<String, Object?>>> senseGlossaries_query =
-      connection.query(
-    'JMdict_SenseGlossary',
-    where: 'senseId IN (${senseIds.join(',')})',
-  );
-
-  late final List<Map<String, Object?>> senseInfos;
-  final Future<List<Map<String, Object?>>> senseInfos_query = connection.query(
-    'JMdict_SenseInfo',
-    where: 'senseId IN (${senseIds.join(',')})',
-  );
-
-  late final List<Map<String, Object?>> senseLanguageSources;
-  final Future<List<Map<String, Object?>>> senseLanguageSources_query =
-      connection.query(
-    'JMdict_SenseLanguageSource',
-    where: 'senseId IN (${senseIds.join(',')})',
-  );
-
-  late final List<Map<String, Object?>> senseMiscs;
-  final Future<List<Map<String, Object?>>> senseMiscs_query = connection.query(
-    'JMdict_SenseMisc',
-    where: 'senseId IN (${senseIds.join(',')})',
-  );
-
-  late final List<Map<String, Object?>> sensePOSs;
-  final Future<List<Map<String, Object?>>> sensePOSs_query = connection.query(
-    'JMdict_SensePOS',
-    where: 'senseId IN (${senseIds.join(',')})',
-  );
-
-  late final List<Map<String, Object?>> senseRestrictedToKanjis;
-  final Future<List<Map<String, Object?>>> senseRestrictedToKanjis_query =
-      connection.query(
-    'JMdict_SenseRestrictedToKanji',
-    where: 'senseId IN (${senseIds.join(',')})',
-  );
-
-  late final List<Map<String, Object?>> senseRestrictedToReadings;
-  final Future<List<Map<String, Object?>>> senseRestrictedToReadings_query =
-      connection.query(
-    'JMdict_SenseRestrictedToReading',
-    where: 'senseId IN (${senseIds.join(',')})',
-  );
-
-  late final List<Map<String, Object?>> senseSeeAlsos;
-  final Future<List<Map<String, Object?>>> senseSeeAlsos_query =
-      connection.query(
-    'JMdict_SenseSeeAlso',
-    where: 'senseId IN (${senseIds.join(',')})',
-  );
-
-  late final List<Map<String, Object?>> exampleSentences;
-  final Future<List<Map<String, Object?>>> exampleSentences_query =
-      connection.query(
-    'JMdict_ExampleSentence',
-    where: 'senseId IN (${senseIds.join(',')})',
-  );
-
-  // Reading queries
-
-  final readingIds = readingElements
-      .map((element) => (
-            element['entryId'] as int,
-            escapeStringValue(element['reading'] as String)
-          ))
-      .toList();
-
-  late final List<Map<String, Object?>> readingElementInfos;
-  final Future<List<Map<String, Object?>>> readingElementInfos_query =
-      connection.query(
-    'JMdict_ReadingElementInfo',
-    where: '(entryId, reading) IN (${readingIds.join(',')})',
-  );
-
-  late final List<Map<String, Object?>> readingElementRestrictions;
-  final Future<List<Map<String, Object?>>> readingElementRestrictions_query =
-      connection.query(
-    'JMdict_ReadingElementRestriction',
-    where: '(entryId, reading) IN (${readingIds.join(',')})',
-  );
-
-  // Kanji queries
-
-  final kanjiIds = kanjiElements
-      .map((element) => (
-            element['entryId'] as int,
-            escapeStringValue(element['reading'] as String)
-          ))
-      .toList();
-
-  late final List<Map<String, Object?>> kanjiElementInfos;
-  final Future<List<Map<String, Object?>>> kanjiElementInfos_query =
-      connection.query(
-    'JMdict_KanjiElementInfo',
-    where: '(entryId, reading) IN (${kanjiIds.join(',')})',
-  );
-
-  await Future.wait([
-    senseAntonyms_query.then((value) => senseAntonyms = value),
-    senseDialects_query.then((value) => senseDialects = value),
-    senseFields_query.then((value) => senseFields = value),
-    senseGlossaries_query.then((value) => senseGlossaries = value),
-    senseInfos_query.then((value) => senseInfos = value),
-    senseLanguageSources_query.then((value) => senseLanguageSources = value),
-    senseMiscs_query.then((value) => senseMiscs = value),
-    sensePOSs_query.then((value) => sensePOSs = value),
-    senseRestrictedToKanjis_query
-        .then((value) => senseRestrictedToKanjis = value),
-    senseRestrictedToReadings_query
-        .then((value) => senseRestrictedToReadings = value),
-    senseSeeAlsos_query.then((value) => senseSeeAlsos = value),
-    exampleSentences_query.then((value) => exampleSentences = value),
-    readingElementInfos_query.then((value) => readingElementInfos = value),
-    readingElementRestrictions_query
-        .then((value) => readingElementRestrictions = value),
-    kanjiElementInfos_query.then((value) => kanjiElementInfos = value),
-  ]);
-
-  return _regroupWordSearchResults(
-    entryIds: entryIds,
-    readingElements: readingElements,
-    kanjiElements: kanjiElements,
-    senses: senses,
-    senseAntonyms: senseAntonyms,
-    senseDialects: senseDialects,
-    senseFields: senseFields,
-    senseGlossaries: senseGlossaries,
-    senseInfos: senseInfos,
-    senseLanguageSources: senseLanguageSources,
-    senseMiscs: senseMiscs,
-    sensePOSs: sensePOSs,
-    senseRestrictedToKanjis: senseRestrictedToKanjis,
-    senseRestrictedToReadings: senseRestrictedToReadings,
-    senseSeeAlsos: senseSeeAlsos,
-    exampleSentences: exampleSentences,
-    readingElementInfos: readingElementInfos,
-    readingElementRestrictions: readingElementRestrictions,
-    kanjiElementInfos: kanjiElementInfos,
-  );
-}
-
-List<WordSearchResult> _regroupWordSearchResults({
-  required List<int> entryIds,
-  required List<Map<String, Object?>> readingElements,
-  required List<Map<String, Object?>> kanjiElements,
-  required List<Map<String, Object?>> senses,
-  required List<Map<String, Object?>> senseAntonyms,
-  required List<Map<String, Object?>> senseDialects,
-  required List<Map<String, Object?>> senseFields,
-  required List<Map<String, Object?>> senseGlossaries,
-  required List<Map<String, Object?>> senseInfos,
-  required List<Map<String, Object?>> senseLanguageSources,
-  required List<Map<String, Object?>> senseMiscs,
-  required List<Map<String, Object?>> sensePOSs,
-  required List<Map<String, Object?>> senseRestrictedToKanjis,
-  required List<Map<String, Object?>> senseRestrictedToReadings,
-  required List<Map<String, Object?>> senseSeeAlsos,
-  required List<Map<String, Object?>> exampleSentences,
-  required List<Map<String, Object?>> readingElementInfos,
-  required List<Map<String, Object?>> readingElementRestrictions,
-  required List<Map<String, Object?>> kanjiElementInfos,
-}) {
-  final List<WordSearchResult> results = [];
-
-  for (final entryId in entryIds) {
-    final List<Map<String, Object?>> entryReadingElements = readingElements
-        .where((element) => element['entryId'] == entryId)
-        .toList();
-
-    final List<Map<String, Object?>> entryKanjiElements = kanjiElements
-        .where((element) => element['entryId'] == entryId)
-        .toList();
-
-    final List<Map<String, Object?>> entrySenses =
-        senses.where((element) => element['entryId'] == entryId).toList();
-
-    final GroupedWordResult entryReadingElementsGrouped = _regroup_words(
-      entryId: entryId,
-      readingElements: entryReadingElements,
-      kanjiElements: entryKanjiElements,
-      readingElementInfos: readingElementInfos,
-      readingElementRestrictions: readingElementRestrictions,
-      kanjiElementInfos: kanjiElementInfos,
-    );
-
-    final List<WordSearchSense> entrySensesGrouped = _regroup_senses(
-      senses: entrySenses,
-      senseAntonyms: senseAntonyms,
-      senseDialects: senseDialects,
-      senseFields: senseFields,
-      senseGlossaries: senseGlossaries,
-      senseInfos: senseInfos,
-      senseLanguageSources: senseLanguageSources,
-      senseMiscs: senseMiscs,
-      sensePOSs: sensePOSs,
-      senseRestrictedToKanjis: senseRestrictedToKanjis,
-      senseRestrictedToReadings: senseRestrictedToReadings,
-      senseSeeAlsos: senseSeeAlsos,
-      exampleSentences: exampleSentences,
-    );
-
-    results.add(
-      WordSearchResult(
-        entryId: entryId,
-        japanese: entryReadingElementsGrouped.rubys,
-        kanjiInfo: entryReadingElementsGrouped.kanjiInfos,
-        readingInfo: entryReadingElementsGrouped.readingInfos,
-        senses: entrySensesGrouped,
-        sources: const WordSearchSources(
-          jmdict: true,
-          jmnedict: false,
-        ),
-      ),
-    );
-  }
-
-  return results;
-}
-
-class GroupedWordResult {
-  final List<WordSearchRuby> rubys;
-  final Map<String, JMdictReadingInfo> readingInfos;
-  final Map<String, JMdictKanjiInfo> kanjiInfos;
-
-  const GroupedWordResult({
-    required this.rubys,
-    required this.readingInfos,
-    required this.kanjiInfos,
-  });
-}
-
-GroupedWordResult _regroup_words({
-  required int entryId,
-  required List<Map<String, Object?>> kanjiElements,
-  required List<Map<String, Object?>> kanjiElementInfos,
-  required List<Map<String, Object?>> readingElements,
-  required List<Map<String, Object?>> readingElementInfos,
-  required List<Map<String, Object?>> readingElementRestrictions,
-}) {
-  final List<WordSearchRuby> result = [];
-
-  final kanjiElements_ =
-      kanjiElements.where((element) => element['entryId'] == entryId).toList();
-
-  final readingElements_ = readingElements
-      .where((element) => element['entryId'] == entryId)
-      .toList();
-
-  final readingElementRestrictions_ = readingElementRestrictions
-      .where((element) => element['entryId'] == entryId)
-      .toList();
-
-  for (final readingElement in readingElements_) {
-    for (final kanjiElement in kanjiElements_) {
-      final kanji = kanjiElement['reading'] as String;
-      final reading = readingElement['reading'] as String;
-
-      final doesNotMatchKanji = readingElement['doesNotMatchKanji'] == 1;
-      if (doesNotMatchKanji) {
-        continue;
-      }
-
-      final restrictions = readingElementRestrictions_
-          .where((element) => element['reading'] == reading)
-          .toList();
-
-      if (restrictions.isNotEmpty &&
-          !restrictions.any((element) => element['restriction'] == kanji)) {
-        continue;
-      }
-
-      final ruby = WordSearchRuby(
-        base: kanji,
-        furigana: reading,
-      );
-      result.add(ruby);
-    }
-  }
-
-  for (final readingElement
-      in readingElements_.where((e) => e['doesNotMatchKanji'] == 1)) {
-    final reading = readingElement['reading'] as String;
-    final ruby = WordSearchRuby(
-      base: reading,
-    );
-    result.add(ruby);
-  }
-
-  return GroupedWordResult(
-    rubys: result,
-    readingInfos: Map.fromEntries(
-      readingElementInfos.map((e) => MapEntry(
-            e['reading'] as String,
-            JMdictReadingInfo.fromId(e['info'] as String),
-          )),
-    ),
-    kanjiInfos: Map.fromEntries(
-      kanjiElementInfos.map((e) => MapEntry(
-            e['reading'] as String,
-            JMdictKanjiInfo.fromId(e['info'] as String),
-          )),
-    ),
-  );
-}
-
-List<WordSearchSense> _regroup_senses({
-  required List<Map<String, Object?>> senses,
-  required List<Map<String, Object?>> senseAntonyms,
-  required List<Map<String, Object?>> senseDialects,
-  required List<Map<String, Object?>> senseFields,
-  required List<Map<String, Object?>> senseGlossaries,
-  required List<Map<String, Object?>> senseInfos,
-  required List<Map<String, Object?>> senseLanguageSources,
-  required List<Map<String, Object?>> senseMiscs,
-  required List<Map<String, Object?>> sensePOSs,
-  required List<Map<String, Object?>> senseRestrictedToKanjis,
-  required List<Map<String, Object?>> senseRestrictedToReadings,
-  required List<Map<String, Object?>> senseSeeAlsos,
-  required List<Map<String, Object?>> exampleSentences,
-}) {
-  final groupedSenseAntonyms =
-      senseAntonyms.groupListsBy((element) => element['senseId'] as int);
-  final groupedSenseDialects =
-      senseDialects.groupListsBy((element) => element['senseId'] as int);
-  final groupedSenseFields =
-      senseFields.groupListsBy((element) => element['senseId'] as int);
-  final groupedSenseGlossaries =
-      senseGlossaries.groupListsBy((element) => element['senseId'] as int);
-  final groupedSenseInfos =
-      senseInfos.groupListsBy((element) => element['senseId'] as int);
-  final groupedSenseLanguageSources =
-      senseLanguageSources.groupListsBy((element) => element['senseId'] as int);
-  final groupedSenseMiscs =
-      senseMiscs.groupListsBy((element) => element['senseId'] as int);
-  final groupedSensePOSs =
-      sensePOSs.groupListsBy((element) => element['senseId'] as int);
-  final groupedSenseRestrictedToKanjis = senseRestrictedToKanjis
-      .groupListsBy((element) => element['senseId'] as int);
-  final groupedSenseRestrictedToReadings = senseRestrictedToReadings
-      .groupListsBy((element) => element['senseId'] as int);
-  final groupedSenseSeeAlsos =
-      senseSeeAlsos.groupListsBy((element) => element['senseId'] as int);
-
-  final List<WordSearchSense> result = [];
-  for (final sense in senses) {
-    final int senseId = sense['id'] as int;
-
-    final antonyms = groupedSenseAntonyms[senseId] ?? [];
-    final dialects = groupedSenseDialects[senseId] ?? [];
-    final fields = groupedSenseFields[senseId] ?? [];
-    final glossaries = groupedSenseGlossaries[senseId] ?? [];
-    final infos = groupedSenseInfos[senseId] ?? [];
-    final languageSources = groupedSenseLanguageSources[senseId] ?? [];
-    final miscs = groupedSenseMiscs[senseId] ?? [];
-    final pos = groupedSensePOSs[senseId] ?? [];
-    final restrictedToKanjis = groupedSenseRestrictedToKanjis[senseId] ?? [];
-    final restrictedToReadings =
-        groupedSenseRestrictedToReadings[senseId] ?? [];
-    final seeAlsos = groupedSenseSeeAlsos[senseId] ?? [];
-
-    final resultSense = WordSearchSense(
-      englishDefinitions: glossaries.map((e) => e['phrase'] as String).toList(),
-      partsOfSpeech:
-          pos.map((e) => JMdictPOS.fromId(e['pos'] as String)).toList(),
-      seeAlso: seeAlsos
-          .map((e) => WordSearchXrefEntry(
-                entryId: e['xrefEntryId'] as int,
-                ambiguous: e['ambiguous'] == 1,
-              ))
-          .toList(),
-      antonyms: antonyms
-          .map((e) => WordSearchXrefEntry(
-                entryId: e['xrefEntryId'] as int,
-                ambiguous: e['ambiguous'] == 1,
-              ))
-          .toList(),
-      restrictedToReading:
-          restrictedToReadings.map((e) => e['reading'] as String).toList(),
-      restrictedToKanji:
-          restrictedToKanjis.map((e) => e['kanji'] as String).toList(),
-      fields:
-          fields.map((e) => JMdictField.fromId(e['field'] as String)).toList(),
-      dialects: dialects
-          .map((e) => JMdictDialect.fromId(e['dialect'] as String))
-          .toList(),
-      misc: miscs.map((e) => JMdictMisc.fromId(e['misc'] as String)).toList(),
-      info: infos.map((e) => e['info'] as String).toList(),
-      languageSource:
-          languageSources.map((e) => e['language'] as String).toList(),
-    );
-
-    result.add(resultSense);
-  }
-
-  return result;
-}

lib/search/word_search/data_query.dart

@@ -0,0 +1,439 @@
+import 'package:jadb/table_names/jmdict.dart';
+import 'package:jadb/table_names/tanos_jlpt.dart';
+import 'package:sqflite_common/sqflite.dart';
+
+class LinearWordQueryData {
+  final List<Map<String, Object?>> senses;
+  final List<Map<String, Object?>> readingElements;
+  final List<Map<String, Object?>> kanjiElements;
+  final List<Map<String, Object?>> jlptTags;
+  final List<Map<String, Object?>> commonEntries;
+  final List<Map<String, Object?>> senseAntonyms;
+  final List<Map<String, Object?>> senseDialects;
+  final List<Map<String, Object?>> senseFields;
+  final List<Map<String, Object?>> senseGlossaries;
+  final List<Map<String, Object?>> senseInfos;
+  final List<Map<String, Object?>> senseLanguageSources;
+  final List<Map<String, Object?>> senseMiscs;
+  final List<Map<String, Object?>> sensePOSs;
+  final List<Map<String, Object?>> senseRestrictedToKanjis;
+  final List<Map<String, Object?>> senseRestrictedToReadings;
+  final List<Map<String, Object?>> senseSeeAlsos;
+  final List<Map<String, Object?>> exampleSentences;
+  final List<Map<String, Object?>> readingElementInfos;
+  final List<Map<String, Object?>> readingElementRestrictions;
+  final List<Map<String, Object?>> kanjiElementInfos;
+
+  final LinearWordQueryData? senseAntonymData;
+  final LinearWordQueryData? senseSeeAlsoData;
+
+  const LinearWordQueryData({
+    required this.senses,
+    required this.readingElements,
+    required this.kanjiElements,
+    required this.jlptTags,
+    required this.commonEntries,
+    required this.senseAntonyms,
+    required this.senseDialects,
+    required this.senseFields,
+    required this.senseGlossaries,
+    required this.senseInfos,
+    required this.senseLanguageSources,
+    required this.senseMiscs,
+    required this.sensePOSs,
+    required this.senseRestrictedToKanjis,
+    required this.senseRestrictedToReadings,
+    required this.senseSeeAlsos,
+    required this.exampleSentences,
+    required this.readingElementInfos,
+    required this.readingElementRestrictions,
+    required this.kanjiElementInfos,
+    required this.senseAntonymData,
+    required this.senseSeeAlsoData,
+  });
+}
+
+Future<List<Map<String, Object?>>> _sensesQuery(
+  DatabaseExecutor connection,
+  List<int> entryIds,
+) => connection.query(
+  JMdictTableNames.sense,
+  where: 'entryId IN (${List.filled(entryIds.length, '?').join(',')})',
+  whereArgs: entryIds,
+);
+
+Future<List<Map<String, Object?>>> _readingelementsQuery(
+  DatabaseExecutor connection,
+  List<int> entryIds,
+) => connection.query(
+  JMdictTableNames.readingElement,
+  where: 'entryId IN (${List.filled(entryIds.length, '?').join(',')})',
+  whereArgs: entryIds,
+  orderBy: 'orderNum',
+);
+
+Future<List<Map<String, Object?>>> _kanjielementsQuery(
+  DatabaseExecutor connection,
+  List<int> entryIds,
+) => connection.query(
+  JMdictTableNames.kanjiElement,
+  where: 'entryId IN (${List.filled(entryIds.length, '?').join(',')})',
+  whereArgs: entryIds,
+  orderBy: 'orderNum',
+);
+
+Future<List<Map<String, Object?>>> _jlpttagsQuery(
+  DatabaseExecutor connection,
+  List<int> entryIds,
+) => connection.query(
+  TanosJLPTTableNames.jlptTag,
+  where: 'entryId IN (${List.filled(entryIds.length, '?').join(',')})',
+  whereArgs: entryIds,
+);
+
+Future<List<Map<String, Object?>>> _commonentriesQuery(
+  DatabaseExecutor connection,
+  List<int> entryIds,
+) => connection.query(
+  'JMdict_EntryCommon',
+  where: 'entryId IN (${List.filled(entryIds.length, '?').join(',')})',
+  whereArgs: entryIds,
+);
+
+// Sense queries
+
+Future<List<Map<String, Object?>>> _senseantonymsQuery(
+  DatabaseExecutor connection,
+  List<int> senseIds,
+) => connection.rawQuery(
+  """
+      SELECT
+        "${JMdictTableNames.senseAntonyms}".senseId,
+        "${JMdictTableNames.senseAntonyms}".ambiguous,
+        "${JMdictTableNames.senseAntonyms}".xrefEntryId,
+        "JMdict_BaseAndFurigana"."base",
+        "JMdict_BaseAndFurigana"."furigana"
+      FROM "${JMdictTableNames.senseAntonyms}"
+      JOIN "JMdict_BaseAndFurigana"
+        ON "${JMdictTableNames.senseAntonyms}"."xrefEntryId" = "JMdict_BaseAndFurigana"."entryId"
+      WHERE
+        "senseId" IN (${List.filled(senseIds.length, '?').join(',')})
+        AND "JMdict_BaseAndFurigana"."isFirst"
+      ORDER BY
+        "${JMdictTableNames.senseAntonyms}"."senseId",
+        "${JMdictTableNames.senseAntonyms}"."xrefEntryId"
+  """,
+  [...senseIds],
+);
+
+Future<List<Map<String, Object?>>> _senseseealsosQuery(
+  DatabaseExecutor connection,
+  List<int> senseIds,
+) => connection.rawQuery(
+  """
+      SELECT
+        "${JMdictTableNames.senseSeeAlso}"."senseId",
+        "${JMdictTableNames.senseSeeAlso}"."ambiguous",
+        "${JMdictTableNames.senseSeeAlso}"."xrefEntryId",
+        "JMdict_BaseAndFurigana"."base",
+        "JMdict_BaseAndFurigana"."furigana"
+      FROM "${JMdictTableNames.senseSeeAlso}"
+      JOIN "JMdict_BaseAndFurigana"
+        ON "${JMdictTableNames.senseSeeAlso}"."xrefEntryId" = "JMdict_BaseAndFurigana"."entryId"
+      WHERE
+        "senseId" IN (${List.filled(senseIds.length, '?').join(',')})
+        AND "JMdict_BaseAndFurigana"."isFirst"
+      ORDER BY
+        "${JMdictTableNames.senseSeeAlso}"."senseId",
+        "${JMdictTableNames.senseSeeAlso}"."xrefEntryId"
+  """,
+  [...senseIds],
+);
+
+Future<List<Map<String, Object?>>> _sensedialectsQuery(
+  DatabaseExecutor connection,
+  List<int> senseIds,
+) => connection.query(
+  JMdictTableNames.senseDialect,
+  where: 'senseId IN (${List.filled(senseIds.length, '?').join(',')})',
+  whereArgs: senseIds,
+);
+
+Future<List<Map<String, Object?>>> _sensefieldsQuery(
+  DatabaseExecutor connection,
+  List<int> senseIds,
+) => connection.query(
+  JMdictTableNames.senseField,
+  where: 'senseId IN (${List.filled(senseIds.length, '?').join(',')})',
+  whereArgs: senseIds,
+);
+
+Future<List<Map<String, Object?>>> _senseglossariesQuery(
+  DatabaseExecutor connection,
+  List<int> senseIds,
+) => connection.query(
+  JMdictTableNames.senseGlossary,
+  where: 'senseId IN (${List.filled(senseIds.length, '?').join(',')})',
+  whereArgs: senseIds,
+);
+
+Future<List<Map<String, Object?>>> _senseinfosQuery(
+  DatabaseExecutor connection,
+  List<int> senseIds,
+) => connection.query(
+  JMdictTableNames.senseInfo,
+  where: 'senseId IN (${List.filled(senseIds.length, '?').join(',')})',
+  whereArgs: senseIds,
+);
+
+Future<List<Map<String, Object?>>> _senselanguagesourcesQuery(
+  DatabaseExecutor connection,
+  List<int> senseIds,
+) => connection.query(
+  JMdictTableNames.senseLanguageSource,
+  where: 'senseId IN (${List.filled(senseIds.length, '?').join(',')})',
+  whereArgs: senseIds,
+);
+
+Future<List<Map<String, Object?>>> _sensemiscsQuery(
+  DatabaseExecutor connection,
+  List<int> senseIds,
+) => connection.query(
+  JMdictTableNames.senseMisc,
+  where: 'senseId IN (${List.filled(senseIds.length, '?').join(',')})',
+  whereArgs: senseIds,
+);
+
+Future<List<Map<String, Object?>>> _sensepossQuery(
+  DatabaseExecutor connection,
+  List<int> senseIds,
+) => connection.query(
+  JMdictTableNames.sensePOS,
+  where: 'senseId IN (${List.filled(senseIds.length, '?').join(',')})',
+  whereArgs: senseIds,
+);
+
+Future<List<Map<String, Object?>>> _senserestrictedtokanjisQuery(
+  DatabaseExecutor connection,
+  List<int> senseIds,
+) => connection.query(
+  JMdictTableNames.senseRestrictedToKanji,
+  where: 'senseId IN (${List.filled(senseIds.length, '?').join(',')})',
+  whereArgs: senseIds,
+);
+
+Future<List<Map<String, Object?>>> _senserestrictedtoreadingsQuery(
+  DatabaseExecutor connection,
+  List<int> senseIds,
+) => connection.query(
+  JMdictTableNames.senseRestrictedToReading,
+  where: 'senseId IN (${List.filled(senseIds.length, '?').join(',')})',
+  whereArgs: senseIds,
+);
+
+Future<List<Map<String, Object?>>> _examplesentencesQuery(
+  DatabaseExecutor connection,
+  List<int> senseIds,
+) => connection.query(
+  'JMdict_ExampleSentence',
+  where: 'senseId IN (${List.filled(senseIds.length, '?').join(',')})',
+  whereArgs: senseIds,
+);
+
+// Reading/kanji elements queries
+
+Future<List<Map<String, Object?>>> _readingelementinfosQuery(
+  DatabaseExecutor connection,
+  List<int> readingIds,
+) => connection.query(
+  JMdictTableNames.readingInfo,
+  where: '(elementId) IN (${List.filled(readingIds.length, '?').join(',')})',
+  whereArgs: readingIds,
+);
+
+Future<List<Map<String, Object?>>> _readingelementrestrictionsQuery(
+  DatabaseExecutor connection,
+  List<int> readingIds,
+) => connection.query(
+  JMdictTableNames.readingRestriction,
+  where: '(elementId) IN (${List.filled(readingIds.length, '?').join(',')})',
+  whereArgs: readingIds,
+);
+
+Future<List<Map<String, Object?>>> _kanjielementinfosQuery(
+  DatabaseExecutor connection,
+  List<int> kanjiIds,
+) => connection.query(
+  JMdictTableNames.kanjiInfo,
+  where: '(elementId) IN (${List.filled(kanjiIds.length, '?').join(',')})',
+  whereArgs: kanjiIds,
+);
+
+// Xref queries
+
+Future<LinearWordQueryData?> _senseantonymdataQuery(
+  DatabaseExecutor connection,
+  List<int> entryIds,
+) => fetchLinearWordQueryData(connection, entryIds, fetchXrefData: false);
+
+Future<LinearWordQueryData?> _senseseealsodataQuery(
+  DatabaseExecutor connection,
+  List<int> entryIds,
+) => fetchLinearWordQueryData(connection, entryIds, fetchXrefData: false);
+
+// Full query
+
+Future<LinearWordQueryData> fetchLinearWordQueryData(
+  DatabaseExecutor connection,
+  List<int> entryIds, {
+  bool fetchXrefData = true,
+}) async {
+  late final List<Map<String, Object?>> senses;
+  late final List<Map<String, Object?>> readingElements;
+  late final List<Map<String, Object?>> kanjiElements;
+  late final List<Map<String, Object?>> jlptTags;
+  late final List<Map<String, Object?>> commonEntries;
+
+  await Future.wait([
+    _sensesQuery(connection, entryIds).then((value) => senses = value),
+    _readingelementsQuery(
+      connection,
+      entryIds,
+    ).then((value) => readingElements = value),
+    _kanjielementsQuery(
+      connection,
+      entryIds,
+    ).then((value) => kanjiElements = value),
+    _jlpttagsQuery(connection, entryIds).then((value) => jlptTags = value),
+    _commonentriesQuery(
+      connection,
+      entryIds,
+    ).then((value) => commonEntries = value),
+  ]);
+
+  final senseIds = senses.map((sense) => sense['senseId'] as int).toList();
+
+  late final List<Map<String, Object?>> senseAntonyms;
+  late final List<Map<String, Object?>> senseDialects;
+  late final List<Map<String, Object?>> senseFields;
+  late final List<Map<String, Object?>> senseGlossaries;
+  late final List<Map<String, Object?>> senseInfos;
+  late final List<Map<String, Object?>> senseLanguageSources;
+  late final List<Map<String, Object?>> senseMiscs;
+  late final List<Map<String, Object?>> sensePOSs;
+  late final List<Map<String, Object?>> senseRestrictedToKanjis;
+  late final List<Map<String, Object?>> senseRestrictedToReadings;
+  late final List<Map<String, Object?>> senseSeeAlsos;
+  late final List<Map<String, Object?>> exampleSentences;
+
+  final readingIds = readingElements
+      .map((element) => element['elementId'] as int)
+      .toList();
+
+  final kanjiIds = kanjiElements
+      .map((element) => element['elementId'] as int)
+      .toList();
+
+  late final List<Map<String, Object?>> readingElementInfos;
+  late final List<Map<String, Object?>> readingElementRestrictions;
+
+  late final List<Map<String, Object?>> kanjiElementInfos;
+
+  // Xref data queries
+  await Future.wait([
+    _senseantonymsQuery(
+      connection,
+      senseIds,
+    ).then((value) => senseAntonyms = value),
+    _senseseealsosQuery(
+      connection,
+      senseIds,
+    ).then((value) => senseSeeAlsos = value),
+  ]);
+
+  LinearWordQueryData? senseAntonymData;
+  LinearWordQueryData? senseSeeAlsoData;
+
+  await Future.wait([
+    _sensedialectsQuery(
+      connection,
+      senseIds,
+    ).then((value) => senseDialects = value),
+    _sensefieldsQuery(
+      connection,
+      senseIds,
+    ).then((value) => senseFields = value),
+    _senseglossariesQuery(
+      connection,
+      senseIds,
+    ).then((value) => senseGlossaries = value),
+    _senseinfosQuery(connection, senseIds).then((value) => senseInfos = value),
+    _senselanguagesourcesQuery(
+      connection,
+      senseIds,
+    ).then((value) => senseLanguageSources = value),
+    _sensemiscsQuery(connection, senseIds).then((value) => senseMiscs = value),
+    _sensepossQuery(connection, senseIds).then((value) => sensePOSs = value),
+    _senserestrictedtokanjisQuery(
+      connection,
+      senseIds,
+    ).then((value) => senseRestrictedToKanjis = value),
+    _senserestrictedtoreadingsQuery(
+      connection,
+      senseIds,
+    ).then((value) => senseRestrictedToReadings = value),
+    _examplesentencesQuery(
+      connection,
+      senseIds,
+    ).then((value) => exampleSentences = value),
+    _readingelementinfosQuery(
+      connection,
+      readingIds,
+    ).then((value) => readingElementInfos = value),
+    _readingelementrestrictionsQuery(
+      connection,
+      readingIds,
+    ).then((value) => readingElementRestrictions = value),
+    _kanjielementinfosQuery(
+      connection,
+      kanjiIds,
+    ).then((value) => kanjiElementInfos = value),
+
+    if (fetchXrefData)
+      _senseantonymdataQuery(
+        connection,
+        senseAntonyms.map((antonym) => antonym['xrefEntryId'] as int).toList(),
+      ).then((value) => senseAntonymData = value),
+
+    if (fetchXrefData)
+      _senseseealsodataQuery(
+        connection,
+        senseSeeAlsos.map((seeAlso) => seeAlso['xrefEntryId'] as int).toList(),
+      ).then((value) => senseSeeAlsoData = value),
+  ]);
+
+  return LinearWordQueryData(
+    senses: senses,
+    readingElements: readingElements,
+    kanjiElements: kanjiElements,
+    jlptTags: jlptTags,
+    commonEntries: commonEntries,
+    senseAntonyms: senseAntonyms,
+    senseDialects: senseDialects,
+    senseFields: senseFields,
+    senseGlossaries: senseGlossaries,
+    senseInfos: senseInfos,
+    senseLanguageSources: senseLanguageSources,
+    senseMiscs: senseMiscs,
+    sensePOSs: sensePOSs,
+    senseRestrictedToKanjis: senseRestrictedToKanjis,
+    senseRestrictedToReadings: senseRestrictedToReadings,
+    senseSeeAlsos: senseSeeAlsos,
+    exampleSentences: exampleSentences,
+    readingElementInfos: readingElementInfos,
+    readingElementRestrictions: readingElementRestrictions,
+    kanjiElementInfos: kanjiElementInfos,
+    senseAntonymData: senseAntonymData,
+    senseSeeAlsoData: senseSeeAlsoData,
+  );
+}

lib/search/word_search/entry_id_query.dart

@@ -0,0 +1,311 @@
+import 'package:jadb/search/word_search/word_search.dart';
+import 'package:jadb/table_names/jmdict.dart';
+import 'package:jadb/util/text_filtering.dart';
+import 'package:sqflite_common/sqlite_api.dart';
+
+class ScoredEntryId {
+  final int entryId;
+  final int score;
+
+  const ScoredEntryId(this.entryId, this.score);
+}
+
+SearchMode _determineSearchMode(String word) {
+  final bool containsKanji = kanjiRegex.hasMatch(word);
+  final bool containsAscii = RegExp(r'[A-Za-z]').hasMatch(word);
+
+  if (containsKanji && containsAscii) {
+    return SearchMode.mixedKanji;
+  } else if (containsKanji) {
+    return SearchMode.kanji;
+  } else if (containsAscii) {
+    return SearchMode.english;
+  } else if (word.contains(hiraganaRegex) || word.contains(katakanaRegex)) {
+    return SearchMode.kana;
+  } else {
+    return SearchMode.mixedKana;
+  }
+}
+
+/// FTS reacts to certain characters, so we should filter them out.
+String _filterFTSSensitiveCharacters(String word) {
+  return word
+      .replaceAll('.', '')
+      .replaceAll('-', '')
+      .replaceAll('*', '')
+      .replaceAll('+', '')
+      .replaceAll('(', '')
+      .replaceAll(')', '')
+      .replaceAll('^', '')
+      .replaceAll('"', '');
+}
+
+(String, List<Object?>) _kanjiReadingTemplate(
+  String tableName,
+  String word, {
+  int? pageSize,
+  int? offset,
+  bool countOnly = false,
+}) {
+  assert(
+    tableName == JMdictTableNames.kanjiElement ||
+        tableName == JMdictTableNames.readingElement,
+  );
+  assert(!countOnly || pageSize == null);
+  assert(!countOnly || offset == null);
+  assert(pageSize == null || pageSize > 0);
+  assert(offset == null || offset >= 0);
+  assert(
+    offset == null || pageSize != null,
+    'Offset should only be used with pageSize set',
+  );
+
+  return (
+    '''
+        WITH
+          fts_results AS (
+            SELECT DISTINCT
+              "$tableName"."entryId",
+              100
+              + (("${tableName}FTS"."reading" = ?) * 10000)
+              + "JMdict_EntryScore"."score"
+              AS "score"
+            FROM "${tableName}FTS"
+            JOIN "$tableName" USING ("elementId")
+            JOIN "JMdict_EntryScore" USING ("elementId")
+            WHERE "${tableName}FTS"."reading" MATCH ? || '*'
+              AND "JMdict_EntryScore"."type" = '${tableName == JMdictTableNames.kanjiElement ? 'k' : 'r'}'
+          ),
+          non_fts_results AS (
+            SELECT DISTINCT
+              "$tableName"."entryId",
+              50
+              + "JMdict_EntryScore"."score"
+              AS "score"
+            FROM "$tableName"
+            JOIN "JMdict_EntryScore" USING ("elementId")
+            WHERE "reading" LIKE '%' || ? || '%'
+              AND "$tableName"."entryId" NOT IN (SELECT "entryId" FROM "fts_results")
+              AND "JMdict_EntryScore"."type" = '${tableName == JMdictTableNames.kanjiElement ? 'k' : 'r'}'
+          )
+
+        SELECT ${countOnly ? 'COUNT(DISTINCT "entryId") AS count' : '"entryId", MAX("score") AS "score"'}
+        FROM (
+          SELECT * FROM "fts_results"
+          UNION
+          SELECT * FROM "non_fts_results"
+        )
+        ${!countOnly ? 'GROUP BY "entryId"' : ''}
+        ${!countOnly ? 'ORDER BY "score" DESC, "entryId" ASC' : ''}
+        ${pageSize != null ? 'LIMIT ?' : ''}
+        ${offset != null ? 'OFFSET ?' : ''}
+      '''
+        .trim(),
+    [
+      _filterFTSSensitiveCharacters(word),
+      _filterFTSSensitiveCharacters(word),
+      _filterFTSSensitiveCharacters(word),
+      ?pageSize,
+      ?offset,
+    ],
+  );
+}
+
+Future<List<ScoredEntryId>> _queryKanji(
+  DatabaseExecutor connection,
+  String word,
+  int? pageSize,
+  int? offset,
+) {
+  final (query, args) = _kanjiReadingTemplate(
+    JMdictTableNames.kanjiElement,
+    word,
+    pageSize: pageSize,
+    offset: offset,
+  );
+  return connection
+      .rawQuery(query, args)
+      .then(
+        (result) => result
+            .map(
+              (row) =>
+                  ScoredEntryId(row['entryId'] as int, row['score'] as int),
+            )
+            .toList(),
+      );
+}
+
+Future<int> _queryKanjiCount(DatabaseExecutor connection, String word) {
+  final (query, args) = _kanjiReadingTemplate(
+    JMdictTableNames.kanjiElement,
+    word,
+    countOnly: true,
+  );
+  return connection
+      .rawQuery(query, args)
+      .then((result) => result.firstOrNull?['count'] as int? ?? 0);
+}
+
+Future<List<ScoredEntryId>> _queryKana(
+  DatabaseExecutor connection,
+  String word,
+  int? pageSize,
+  int? offset,
+) {
+  final (query, args) = _kanjiReadingTemplate(
+    JMdictTableNames.readingElement,
+    word,
+    pageSize: pageSize,
+    offset: offset,
+  );
+  return connection
+      .rawQuery(query, args)
+      .then(
+        (result) => result
+            .map(
+              (row) =>
+                  ScoredEntryId(row['entryId'] as int, row['score'] as int),
+            )
+            .toList(),
+      );
+}
+
+Future<int> _queryKanaCount(DatabaseExecutor connection, String word) {
+  final (query, args) = _kanjiReadingTemplate(
+    JMdictTableNames.readingElement,
+    word,
+    countOnly: true,
+  );
+  return connection
+      .rawQuery(query, args)
+      .then((result) => result.firstOrNull?['count'] as int? ?? 0);
+}
+
+Future<List<ScoredEntryId>> _queryEnglish(
+  DatabaseExecutor connection,
+  String word,
+  int? pageSize,
+  int? offset,
+) async {
+  assert(pageSize == null || pageSize > 0);
+  assert(offset == null || offset >= 0);
+  assert(
+    offset == null || pageSize != null,
+    'Offset should only be used with pageSize set',
+  );
+
+  final result = await connection.rawQuery(
+    '''
+    SELECT
+      "${JMdictTableNames.sense}"."entryId",
+      MAX("JMdict_EntryScore"."score")
+      + (("${JMdictTableNames.senseGlossary}"."phrase" = ?1 AND "${JMdictTableNames.sense}"."orderNum" = 1) * 50)
+      + (("${JMdictTableNames.senseGlossary}"."phrase" = ?1 AND "${JMdictTableNames.sense}"."orderNum" = 2) * 30)
+      + (("${JMdictTableNames.senseGlossary}"."phrase" = ?1) * 20)
+      as "score"
+    FROM "${JMdictTableNames.senseGlossary}"
+    JOIN "${JMdictTableNames.sense}" USING ("senseId")
+    JOIN "JMdict_EntryScore" USING ("entryId")
+    WHERE "${JMdictTableNames.senseGlossary}"."phrase" LIKE ?2
+    GROUP BY "JMdict_EntryScore"."entryId"
+    ORDER BY
+      "score" DESC,
+      "${JMdictTableNames.sense}"."entryId" ASC
+    ${pageSize != null ? 'LIMIT ?3' : ''}
+    ${offset != null ? 'OFFSET ?4' : ''}
+  '''
+        .trim(),
+    [word, '%${word.replaceAll('%', '')}%', if (pageSize != null) pageSize, if (offset != null) offset],
+  );
+
+  return result
+      .map((row) => ScoredEntryId(row['entryId'] as int, row['score'] as int))
+      .toList();
+}
+
+Future<int> _queryEnglishCount(DatabaseExecutor connection, String word) async {
+  final result = await connection.rawQuery(
+    '''
+      SELECT
+        COUNT(DISTINCT "${JMdictTableNames.sense}"."entryId") AS "count"
+      FROM "${JMdictTableNames.senseGlossary}"
+      JOIN "${JMdictTableNames.sense}" USING ("senseId")
+      WHERE "${JMdictTableNames.senseGlossary}"."phrase" LIKE ?
+    '''
+        .trim(),
+    ['%$word%'],
+  );
+
+  return result.first['count'] as int;
+}
+
+Future<List<ScoredEntryId>> fetchEntryIds(
+  DatabaseExecutor connection,
+  String word,
+  SearchMode searchMode,
+  int? pageSize,
+  int? offset,
+) async {
+  if (searchMode == SearchMode.auto) {
+    searchMode = _determineSearchMode(word);
+  }
+
+  assert(word.isNotEmpty, 'Word should not be empty when fetching entry IDs');
+
+  late final List<ScoredEntryId> entryIds;
+  switch (searchMode) {
+    case SearchMode.kanji:
+      entryIds = await _queryKanji(connection, word, pageSize, offset);
+      break;
+
+    case SearchMode.kana:
+      entryIds = await _queryKana(connection, word, pageSize, offset);
+      break;
+
+    case SearchMode.english:
+      entryIds = await _queryEnglish(connection, word, pageSize, offset);
+      break;
+
+    case SearchMode.mixedKana:
+    case SearchMode.mixedKanji:
+    default:
+      throw UnimplementedError('Search mode $searchMode is not implemented');
+  }
+
+  return entryIds;
+}
+
+Future<int?> fetchEntryIdCount(
+  DatabaseExecutor connection,
+  String word,
+  SearchMode searchMode,
+) async {
+  if (searchMode == SearchMode.auto) {
+    searchMode = _determineSearchMode(word);
+  }
+
+  assert(word.isNotEmpty, 'Word should not be empty when fetching entry IDs');
+
+  late final int? entryIdCount;
+
+  switch (searchMode) {
+    case SearchMode.kanji:
+      entryIdCount = await _queryKanjiCount(connection, word);
+      break;
+
+    case SearchMode.kana:
+      entryIdCount = await _queryKanaCount(connection, word);
+      break;
+
+    case SearchMode.english:
+      entryIdCount = await _queryEnglishCount(connection, word);
+      break;
+
+    case SearchMode.mixedKana:
+    case SearchMode.mixedKanji:
+    default:
+      throw UnimplementedError('Search mode $searchMode is not implemented');
+  }
+
+  return entryIdCount;
+}

lib/search/word_search/regrouping.dart

@@ -0,0 +1,348 @@
+import 'package:collection/collection.dart';
+import 'package:jadb/models/common/jlpt_level.dart';
+import 'package:jadb/models/jmdict/jmdict_dialect.dart';
+import 'package:jadb/models/jmdict/jmdict_field.dart';
+import 'package:jadb/models/jmdict/jmdict_kanji_info.dart';
+import 'package:jadb/models/jmdict/jmdict_misc.dart';
+import 'package:jadb/models/jmdict/jmdict_pos.dart';
+import 'package:jadb/models/jmdict/jmdict_reading_info.dart';
+import 'package:jadb/models/word_search/word_search_result.dart';
+import 'package:jadb/models/word_search/word_search_ruby.dart';
+import 'package:jadb/models/word_search/word_search_sense.dart';
+import 'package:jadb/models/word_search/word_search_sense_language_source.dart';
+import 'package:jadb/models/word_search/word_search_sources.dart';
+import 'package:jadb/models/word_search/word_search_xref_entry.dart';
+import 'package:jadb/search/word_search/data_query.dart';
+import 'package:jadb/search/word_search/entry_id_query.dart';
+
+List<WordSearchResult> regroupWordSearchResults({
+  required List<ScoredEntryId> entryIds,
+  required LinearWordQueryData linearWordQueryData,
+}) {
+  final List<WordSearchResult> results = [];
+
+  final commonEntryIds = linearWordQueryData.commonEntries
+      .map((entry) => entry['entryId'] as int)
+      .toSet();
+
+  for (final scoredEntryId in entryIds) {
+    final List<Map<String, Object?>> entryReadingElements = linearWordQueryData
+        .readingElements
+        .where((element) => element['entryId'] == scoredEntryId.entryId)
+        .toList();
+
+    final List<Map<String, Object?>> entryKanjiElements = linearWordQueryData
+        .kanjiElements
+        .where((element) => element['entryId'] == scoredEntryId.entryId)
+        .toList();
+
+    final List<Map<String, Object?>> entryJlptTags = linearWordQueryData
+        .jlptTags
+        .where((element) => element['entryId'] == scoredEntryId.entryId)
+        .toList();
+
+    final jlptLevel =
+        entryJlptTags
+            .map((e) => JlptLevel.fromString(e['jlptLevel'] as String?))
+            .sorted((a, b) => b.compareTo(a))
+            .firstOrNull ??
+        JlptLevel.none;
+
+    final isCommon = commonEntryIds.contains(scoredEntryId.entryId);
+
+    final List<Map<String, Object?>> entrySenses = linearWordQueryData.senses
+        .where((element) => element['entryId'] == scoredEntryId.entryId)
+        .toList();
+
+    final GroupedWordResult entryReadingElementsGrouped = _regroupWords(
+      entryId: scoredEntryId.entryId,
+      readingElements: entryReadingElements,
+      kanjiElements: entryKanjiElements,
+      readingElementInfos: linearWordQueryData.readingElementInfos,
+      readingElementRestrictions:
+          linearWordQueryData.readingElementRestrictions,
+      kanjiElementInfos: linearWordQueryData.kanjiElementInfos,
+    );
+
+    final List<WordSearchSense> entrySensesGrouped = _regroupSenses(
+      senses: entrySenses,
+      senseAntonyms: linearWordQueryData.senseAntonyms,
+      senseDialects: linearWordQueryData.senseDialects,
+      senseFields: linearWordQueryData.senseFields,
+      senseGlossaries: linearWordQueryData.senseGlossaries,
+      senseInfos: linearWordQueryData.senseInfos,
+      senseLanguageSources: linearWordQueryData.senseLanguageSources,
+      senseMiscs: linearWordQueryData.senseMiscs,
+      sensePOSs: linearWordQueryData.sensePOSs,
+      senseRestrictedToKanjis: linearWordQueryData.senseRestrictedToKanjis,
+      senseRestrictedToReadings: linearWordQueryData.senseRestrictedToReadings,
+      senseSeeAlsos: linearWordQueryData.senseSeeAlsos,
+      exampleSentences: linearWordQueryData.exampleSentences,
+      senseSeeAlsosXrefData: linearWordQueryData.senseSeeAlsoData,
+      senseAntonymsXrefData: linearWordQueryData.senseAntonymData,
+    );
+
+    results.add(
+      WordSearchResult(
+        score: scoredEntryId.score,
+        entryId: scoredEntryId.entryId,
+        isCommon: isCommon,
+        japanese: entryReadingElementsGrouped.rubys,
+        kanjiInfo: entryReadingElementsGrouped.kanjiInfos,
+        readingInfo: entryReadingElementsGrouped.readingInfos,
+        senses: entrySensesGrouped,
+        jlptLevel: jlptLevel,
+        sources: const WordSearchSources(jmdict: true, jmnedict: false),
+      ),
+    );
+  }
+
+  return results;
+}
+
+class GroupedWordResult {
+  final List<WordSearchRuby> rubys;
+  final Map<String, JMdictReadingInfo> readingInfos;
+  final Map<String, JMdictKanjiInfo> kanjiInfos;
+
+  const GroupedWordResult({
+    required this.rubys,
+    required this.readingInfos,
+    required this.kanjiInfos,
+  });
+}
+
+GroupedWordResult _regroupWords({
+  required int entryId,
+  required List<Map<String, Object?>> kanjiElements,
+  required List<Map<String, Object?>> kanjiElementInfos,
+  required List<Map<String, Object?>> readingElements,
+  required List<Map<String, Object?>> readingElementInfos,
+  required List<Map<String, Object?>> readingElementRestrictions,
+}) {
+  final List<WordSearchRuby> rubys = [];
+
+  final kanjiElements_ = kanjiElements
+      .where((element) => element['entryId'] == entryId)
+      .toList();
+
+  final readingElements_ = readingElements
+      .where((element) => element['entryId'] == entryId)
+      .toList();
+
+  final readingElementRestrictions_ = readingElementRestrictions
+      .where((element) => element['entryId'] == entryId)
+      .toList();
+
+  for (final readingElement in readingElements_) {
+    if (readingElement['doesNotMatchKanji'] == 1 || kanjiElements_.isEmpty) {
+      final ruby = WordSearchRuby(base: readingElement['reading'] as String);
+      rubys.add(ruby);
+
+      continue;
+    }
+
+    for (final kanjiElement in kanjiElements_) {
+      final kanji = kanjiElement['reading'] as String;
+      final reading = readingElement['reading'] as String;
+
+      final restrictions = readingElementRestrictions_
+          .where((element) => element['reading'] == reading)
+          .toList();
+
+      if (restrictions.isNotEmpty &&
+          !restrictions.any((element) => element['restriction'] == kanji)) {
+        continue;
+      }
+
+      final ruby = WordSearchRuby(base: kanji, furigana: reading);
+      rubys.add(ruby);
+    }
+  }
+
+  assert(rubys.isNotEmpty, 'No readings found for entryId: $entryId');
+
+  final Map<int, String> readingElementIdsToReading = {
+    for (final element in readingElements_)
+      element['elementId'] as int: element['reading'] as String,
+  };
+
+  final Map<int, String> kanjiElementIdsToReading = {
+    for (final element in kanjiElements_)
+      element['elementId'] as int: element['reading'] as String,
+  };
+
+  final readingElementInfos_ = readingElementInfos
+      .where((element) => element['entryId'] == entryId)
+      .toList();
+
+  final kanjiElementInfos_ = kanjiElementInfos
+      .where((element) => element['entryId'] == entryId)
+      .toList();
+
+  return GroupedWordResult(
+    rubys: rubys,
+    readingInfos: {
+      for (final rei in readingElementInfos_)
+        readingElementIdsToReading[rei['elementId'] as int]!:
+            JMdictReadingInfo.fromId(rei['info'] as String),
+    },
+    kanjiInfos: {
+      for (final kei in kanjiElementInfos_)
+        kanjiElementIdsToReading[kei['elementId'] as int]!:
+            JMdictKanjiInfo.fromId(kei['info'] as String),
+    },
+  );
+}
+
+List<WordSearchSense> _regroupSenses({
+  required List<Map<String, Object?>> senses,
+  required List<Map<String, Object?>> senseAntonyms,
+  required List<Map<String, Object?>> senseDialects,
+  required List<Map<String, Object?>> senseFields,
+  required List<Map<String, Object?>> senseGlossaries,
+  required List<Map<String, Object?>> senseInfos,
+  required List<Map<String, Object?>> senseLanguageSources,
+  required List<Map<String, Object?>> senseMiscs,
+  required List<Map<String, Object?>> sensePOSs,
+  required List<Map<String, Object?>> senseRestrictedToKanjis,
+  required List<Map<String, Object?>> senseRestrictedToReadings,
+  required List<Map<String, Object?>> senseSeeAlsos,
+  required List<Map<String, Object?>> exampleSentences,
+  required LinearWordQueryData? senseSeeAlsosXrefData,
+  required LinearWordQueryData? senseAntonymsXrefData,
+}) {
+  final groupedSenseAntonyms = senseAntonyms.groupListsBy(
+    (element) => element['senseId'] as int,
+  );
+  final groupedSenseDialects = senseDialects.groupListsBy(
+    (element) => element['senseId'] as int,
+  );
+  final groupedSenseFields = senseFields.groupListsBy(
+    (element) => element['senseId'] as int,
+  );
+  final groupedSenseGlossaries = senseGlossaries.groupListsBy(
+    (element) => element['senseId'] as int,
+  );
+  final groupedSenseInfos = senseInfos.groupListsBy(
+    (element) => element['senseId'] as int,
+  );
+  final groupedSenseLanguageSources = senseLanguageSources.groupListsBy(
+    (element) => element['senseId'] as int,
+  );
+  final groupedSenseMiscs = senseMiscs.groupListsBy(
+    (element) => element['senseId'] as int,
+  );
+  final groupedSensePOSs = sensePOSs.groupListsBy(
+    (element) => element['senseId'] as int,
+  );
+  final groupedSenseRestrictedToKanjis = senseRestrictedToKanjis.groupListsBy(
+    (element) => element['senseId'] as int,
+  );
+  final groupedSenseRestrictedToReadings = senseRestrictedToReadings
+      .groupListsBy((element) => element['senseId'] as int);
+  final groupedSenseSeeAlsos = senseSeeAlsos.groupListsBy(
+    (element) => element['senseId'] as int,
+  );
+
+  final List<WordSearchSense> result = [];
+  for (final sense in senses) {
+    final int senseId = sense['senseId'] as int;
+
+    final antonyms = groupedSenseAntonyms[senseId] ?? [];
+    final dialects = groupedSenseDialects[senseId] ?? [];
+    final fields = groupedSenseFields[senseId] ?? [];
+    final glossaries = groupedSenseGlossaries[senseId] ?? [];
+    final infos = groupedSenseInfos[senseId] ?? [];
+    final languageSources = groupedSenseLanguageSources[senseId] ?? [];
+    final miscs = groupedSenseMiscs[senseId] ?? [];
+    final pos = groupedSensePOSs[senseId] ?? [];
+    final restrictedToKanjis = groupedSenseRestrictedToKanjis[senseId] ?? [];
+    final restrictedToReadings =
+        groupedSenseRestrictedToReadings[senseId] ?? [];
+    final seeAlsos = groupedSenseSeeAlsos[senseId] ?? [];
+
+    final List<WordSearchResult> seeAlsosWordResults =
+        senseSeeAlsosXrefData != null
+        ? regroupWordSearchResults(
+            entryIds: seeAlsos
+                .map((e) => ScoredEntryId(e['xrefEntryId'] as int, 0))
+                .toList(),
+            linearWordQueryData: senseSeeAlsosXrefData,
+          )
+        : [];
+    final List<WordSearchResult> antonymsWordResults =
+        senseAntonymsXrefData != null
+        ? regroupWordSearchResults(
+            entryIds: antonyms
+                .map((e) => ScoredEntryId(e['xrefEntryId'] as int, 0))
+                .toList(),
+            linearWordQueryData: senseAntonymsXrefData,
+          )
+        : [];
+
+    final resultSense = WordSearchSense(
+      englishDefinitions: glossaries.map((e) => e['phrase'] as String).toList(),
+      partsOfSpeech: pos
+          .map((e) => JMdictPOS.fromId(e['pos'] as String))
+          .toList(),
+      seeAlso: seeAlsos.asMap().entries.map<WordSearchXrefEntry>((mapEntry) {
+        final i = mapEntry.key;
+        final e = mapEntry.value;
+
+        return WordSearchXrefEntry(
+          entryId: e['xrefEntryId'] as int,
+          baseWord: e['base'] as String,
+          furigana: e['furigana'] as String?,
+          ambiguous: e['ambiguous'] == 1,
+          xrefResult: seeAlsosWordResults.isNotEmpty
+              ? seeAlsosWordResults[i]
+              : null,
+        );
+      }).toList(),
+      antonyms: antonyms.asMap().entries.map<WordSearchXrefEntry>((mapEntry) {
+        final i = mapEntry.key;
+        final e = mapEntry.value;
+
+        return WordSearchXrefEntry(
+          entryId: e['xrefEntryId'] as int,
+          baseWord: e['base'] as String,
+          furigana: e['furigana'] as String?,
+          ambiguous: e['ambiguous'] == 1,
+          xrefResult: antonymsWordResults.isNotEmpty
+              ? antonymsWordResults[i]
+              : null,
+        );
+      }).toList(),
+      restrictedToReading: restrictedToReadings
+          .map((e) => e['reading'] as String)
+          .toList(),
+      restrictedToKanji: restrictedToKanjis
+          .map((e) => e['kanji'] as String)
+          .toList(),
+      fields: fields
+          .map((e) => JMdictField.fromId(e['field'] as String))
+          .toList(),
+      dialects: dialects
+          .map((e) => JMdictDialect.fromId(e['dialect'] as String))
+          .toList(),
+      misc: miscs.map((e) => JMdictMisc.fromId(e['misc'] as String)).toList(),
+      info: infos.map((e) => e['info'] as String).toList(),
+      languageSource: languageSources
+          .map(
+            (e) => WordSearchSenseLanguageSource(
+              language: e['language'] as String,
+              phrase: e['phrase'] as String?,
+              fullyDescribesSense: e['fullyDescribesSense'] == 1,
+              constructedFromSmallerWords:
+                  e['constructedFromSmallerWords'] == 1,
+            ),
+          )
+          .toList(),
+    );
+
+    result.add(resultSense);
+  }
+
+  return result;
+}

lib/search/word_search/word_search.dart

@@ -0,0 +1,157 @@
+// TODO: Support globs
+
+// TODO: Support tags
+
+// TODO: Prefer original kana type when sorting results
+
+// TODO: Support mixing kana and romaji
+//
+import 'package:jadb/models/word_search/word_search_result.dart';
+import 'package:jadb/search/word_search/data_query.dart';
+import 'package:jadb/search/word_search/entry_id_query.dart';
+import 'package:jadb/search/word_search/regrouping.dart';
+import 'package:jadb/table_names/jmdict.dart';
+import 'package:sqflite_common/sqlite_api.dart';
+
+enum SearchMode {
+  /// Try to autodetect what is being searched for
+  auto,
+
+  /// Search for english words
+  english,
+
+  /// Search for the kanji reading of a word
+  kanji,
+
+  /// Search for the kanji reading of a word, mixed in with kana/romaji
+  mixedKanji,
+
+  /// Search for the kana reading of a word
+  kana,
+
+  /// Search for the kana reading of a word, mixed in with romaji
+  mixedKana,
+}
+
+/// Searches for an input string, returning a list of results with their details. Returns null if the input string is empty.
+Future<List<WordSearchResult>?> searchWordWithDbConnection(
+  DatabaseExecutor connection,
+  String word, {
+  SearchMode searchMode = SearchMode.auto,
+  int page = 0,
+  int? pageSize,
+}) async {
+  if (word.isEmpty) {
+    return null;
+  }
+
+  final int? offset = pageSize != null ? page * pageSize : null;
+  final List<ScoredEntryId> entryIds = await fetchEntryIds(
+    connection,
+    word,
+    searchMode,
+    pageSize,
+    offset,
+  );
+
+  if (entryIds.isEmpty) {
+    // TODO: try conjugation search
+    return [];
+  }
+
+  final LinearWordQueryData linearWordQueryData =
+      await fetchLinearWordQueryData(
+        connection,
+        entryIds.map((e) => e.entryId).toList(),
+      );
+
+  final result = regroupWordSearchResults(
+    entryIds: entryIds,
+    linearWordQueryData: linearWordQueryData,
+  );
+
+  for (final resultEntry in result) {
+    resultEntry.inferMatchSpans(word, searchMode: searchMode);
+  }
+
+  return result;
+}
+
+/// Searches for an input string, returning the amount of results that the search would yield without pagination.
+Future<int?> searchWordCountWithDbConnection(
+  DatabaseExecutor connection,
+  String word, {
+  SearchMode searchMode = SearchMode.auto,
+}) async {
+  if (word.isEmpty) {
+    return null;
+  }
+
+  final int? entryIdCount = await fetchEntryIdCount(
+    connection,
+    word,
+    searchMode,
+  );
+
+  return entryIdCount;
+}
+
+/// Fetches a single word by its entry ID, returning null if not found.
+Future<WordSearchResult?> getWordByIdWithDbConnection(
+  DatabaseExecutor connection,
+  int id,
+) async {
+  if (id <= 0) {
+    return null;
+  }
+
+  final exists = await connection
+      .rawQuery(
+        'SELECT EXISTS(SELECT 1 FROM "${JMdictTableNames.entry}" WHERE "entryId" = ?)',
+        [id],
+      )
+      .then((value) => value.isNotEmpty && value.first.values.first == 1);
+
+  if (!exists) {
+    return null;
+  }
+
+  final LinearWordQueryData linearWordQueryData =
+      await fetchLinearWordQueryData(connection, [id]);
+
+  final result = regroupWordSearchResults(
+    entryIds: [ScoredEntryId(id, 0)],
+    linearWordQueryData: linearWordQueryData,
+  );
+
+  assert(
+    result.length == 1,
+    'Expected exactly one result for entryId $id, but got ${result.length}',
+  );
+
+  return result.firstOrNull;
+}
+
+/// Fetches multiple words by their entry IDs, returning a map from entry ID to result.
+Future<Map<int, WordSearchResult>> getWordsByIdsWithDbConnection(
+  DatabaseExecutor connection,
+  Set<int> ids,
+) async {
+  if (ids.isEmpty) {
+    return {};
+  }
+
+  final LinearWordQueryData linearWordQueryData =
+      await fetchLinearWordQueryData(connection, ids.toList());
+
+  final List<ScoredEntryId> entryIds = ids
+      .map((id) => ScoredEntryId(id, 0)) // Score is not used here
+      .toList();
+
+  final results = regroupWordSearchResults(
+    entryIds: entryIds,
+    linearWordQueryData: linearWordQueryData,
+  );
+
+  return {for (var r in results) r.entryId: r};
+}

lib/table_names/jmdict.dart

@@ -1,7 +1,6 @@
 abstract class JMdictTableNames {
+  static const String version = 'JMdict_Version';
   static const String entry = 'JMdict_Entry';
-  static const String entryByKana = 'JMdict_EntryByKana';
-  static const String entryByEnglish = 'JMdict_EntryByEnglish';
   static const String kanjiElement = 'JMdict_KanjiElement';
   static const String kanjiInfo = 'JMdict_KanjiElementInfo';
   static const String readingElement = 'JMdict_ReadingElement';
@@ -22,25 +21,24 @@ abstract class JMdictTableNames {
   static const String senseSeeAlso = 'JMdict_SenseSeeAlso';
 
   static Set<String> get allTables => {
-        entry,
-        entryByKana,
-        entryByEnglish,
-        kanjiElement,
-        kanjiInfo,
-        readingElement,
-        readingInfo,
-        readingRestriction,
-        sense,
-        senseAntonyms,
-        senseDialect,
-        senseField,
-        senseGlossary,
-        senseInfo,
-        senseMisc,
-        sensePOS,
-        senseLanguageSource,
-        senseRestrictedToKanji,
-        senseRestrictedToReading,
-        senseSeeAlso
-      };
+    version,
+    entry,
+    kanjiElement,
+    kanjiInfo,
+    readingElement,
+    readingInfo,
+    readingRestriction,
+    sense,
+    senseAntonyms,
+    senseDialect,
+    senseField,
+    senseGlossary,
+    senseInfo,
+    senseMisc,
+    sensePOS,
+    senseLanguageSource,
+    senseRestrictedToKanji,
+    senseRestrictedToReading,
+    senseSeeAlso,
+  };
 }

lib/table_names/kanjidic.dart

@@ -1,4 +1,5 @@
 abstract class KANJIDICTableNames {
+  static const String version = 'KANJIDIC_Version';
   static const String character = 'KANJIDIC_Character';
   static const String radicalName = 'KANJIDIC_RadicalName';
   static const String codepoint = 'KANJIDIC_Codepoint';
@@ -17,19 +18,20 @@ abstract class KANJIDICTableNames {
   static const String nanori = 'KANJIDIC_Nanori';
 
   static Set<String> get allTables => {
-        character,
-        radicalName,
-        codepoint,
-        radical,
-        strokeMiscount,
-        variant,
-        dictionaryReference,
-        dictionaryReferenceMoro,
-        queryCode,
-        reading,
-        kunyomi,
-        onyomi,
-        meaning,
-        nanori
-      };
+    version,
+    character,
+    radicalName,
+    codepoint,
+    radical,
+    strokeMiscount,
+    variant,
+    dictionaryReference,
+    dictionaryReferenceMoro,
+    queryCode,
+    reading,
+    kunyomi,
+    onyomi,
+    meaning,
+    nanori,
+  };
 }

lib/table_names/radkfile.dart

@@ -0,0 +1,6 @@
+abstract class RADKFILETableNames {
+  static const String version = 'RADKFILE_Version';
+  static const String radkfile = 'RADKFILE';
+
+  static Set<String> get allTables => {version, radkfile};
+}

lib/table_names/tanos_jlpt.dart

@@ -0,0 +1,6 @@
+abstract class TanosJLPTTableNames {
+  static const String version = 'JMdict_JLPT_Version';
+  static const String jlptTag = 'JMdict_JLPTTag';
+
+  static Set<String> get allTables => {version, jlptTag};
+}

lib/util/datetime_extensions.dart

@@ -276,28 +276,22 @@ extension on DateTime {
   /// See more info here:
   /// - https://en.wikipedia.org/wiki/Nanboku-ch%C5%8D_period
   /// - http://www.kumamotokokufu-h.ed.jp/kumamoto/bungaku/nengoui.html
-  String? japaneseEra({bool nanbokuchouPeriodUsesNorth = true}) {
+  String? japaneseEra() {
     throw UnimplementedError('This function is not implemented yet.');
 
-    if (this.year < 645) {
+    if (year < 645) {
       return null;
     }
 
-    if (this.year < periodsNanbokuchouNorth.keys.first.$1) {
+    if (year < periodsNanbokuchouNorth.keys.first.$1) {
       // TODO: find first where year <= this.year and jump one period back.
     }
   }
 
-  String get japaneseWeekdayPrefix => [
-  	    '月',
-  	    '火',
-  	    '水',
-  	    '木',
-  	    '金',
-  	    '土',
-  	    '日',
-  	  ][weekday - 1];
+  String get japaneseWeekdayPrefix =>
+      ['月', '火', '水', '木', '金', '土', '日'][weekday - 1];
 
   /// Returns the date in Japanese format.
-  String japaneseDate({bool showWeekday = false}) => '$month月$day日' + (showWeekday ? '（$japaneseWeekdayPrefix）' : '');
+  String japaneseDate({bool showWeekday = false}) =>
+      '$month月$day日${showWeekday ? '（$japaneseWeekdayPrefix）' : ''}';
 }

lib/util/lemmatizer/lemmatizer.dart

@@ -0,0 +1,287 @@
+import 'package:collection/collection.dart';
+import 'package:jadb/util/lemmatizer/rules.dart';
+
+enum WordClass {
+  noun,
+  ichidanVerb,
+  godanVerb,
+  irregularVerb,
+  iAdjective,
+  nAdjective,
+  adverb,
+  particle,
+  input,
+
+  // TODO: add toString and fromString so it can be parsed by the cli
+}
+
+enum LemmatizationRuleType { prefix, suffix }
+
+class LemmatizationRule {
+  final String name;
+  final AllomorphPattern pattern;
+  final WordClass wordClass;
+  final Set<WordClass>? validChildClasses;
+  final bool terminal;
+
+  const LemmatizationRule({
+    required this.name,
+    required this.pattern,
+    required this.wordClass,
+    this.validChildClasses,
+    this.terminal = false,
+  });
+
+  bool matches(String word) => pattern.matches(word);
+
+  List<String>? apply(String word) => pattern.apply(word);
+
+  LemmatizationRule.simple({
+    required String name,
+    required String pattern,
+    required String? replacement,
+    required WordClass wordClass,
+    Set<WordClass>? validChildClasses,
+    bool terminal = false,
+    List<Pattern> lookAheadBehind = const [''],
+    LemmatizationRuleType type = LemmatizationRuleType.suffix,
+  }) : this(
+         name: name,
+         pattern: AllomorphPattern(
+           patterns: {
+             pattern: replacement != null ? [replacement] : null,
+           },
+           type: type,
+           lookAheadBehind: lookAheadBehind,
+         ),
+         validChildClasses: validChildClasses,
+         terminal: terminal,
+         wordClass: wordClass,
+       );
+
+  @override
+  int get hashCode => Object.hash(
+    name,
+    pattern,
+    wordClass,
+    validChildClasses,
+    terminal,
+    SetEquality().hash(validChildClasses),
+  );
+
+  @override
+  bool operator ==(Object other) {
+    if (identical(this, other)) return true;
+    return other is LemmatizationRule &&
+        other.name == name &&
+        other.pattern == pattern &&
+        other.wordClass == wordClass &&
+        other.terminal == terminal &&
+        SetEquality().equals(validChildClasses, other.validChildClasses);
+  }
+}
+
+/// Represents a set of patterns for matching allomorphs in a word.
+/// The patterns can be either a prefix or a suffix, and they can include
+/// replacement characters for deconjugating into base forms.
+class AllomorphPattern {
+  final List<Pattern> lookAheadBehind;
+  final Map<String, List<String>?> patterns;
+  final LemmatizationRuleType type;
+
+  const AllomorphPattern({
+    required this.patterns,
+    required this.type,
+    this.lookAheadBehind = const [''],
+  });
+
+  /// Convert the [patterns] into regexes
+  List<(String, Pattern)> get allPatternCombinations {
+    final combinations = <(String, Pattern)>[];
+    for (final l in lookAheadBehind) {
+      for (final p in patterns.keys) {
+        switch ((type, l is RegExp)) {
+          case (LemmatizationRuleType.prefix, true):
+            combinations.add((p, RegExp('^($p)(${(l as RegExp).pattern})')));
+            break;
+          case (LemmatizationRuleType.prefix, false):
+            combinations.add((p, '$p$l'));
+            break;
+          case (LemmatizationRuleType.suffix, true):
+            combinations.add((p, RegExp('(${(l as RegExp).pattern})($p)\$')));
+            break;
+          case (LemmatizationRuleType.suffix, false):
+            combinations.add((p, '$l$p'));
+            break;
+        }
+      }
+    }
+    return combinations;
+  }
+
+  /// Check whether an input string matches any of the [patterns]
+  bool matches(String word) {
+    for (final (_, p) in allPatternCombinations) {
+      if (p is String) {
+        if (type == LemmatizationRuleType.prefix
+            ? word.startsWith(p)
+            : word.endsWith(p)) {
+          return true;
+        }
+      } else if (p is RegExp) {
+        if (p.hasMatch(word)) {
+          return true;
+        }
+      }
+    }
+    return false;
+  }
+
+  /// Apply the replacement for this pattern.
+  ///
+  /// If none of the [patterns] apply, this function returns `null`.
+  List<String>? apply(String word) {
+    for (final (affix, p) in allPatternCombinations) {
+      switch ((type, p is RegExp)) {
+        case (LemmatizationRuleType.prefix, true):
+          final match = (p as RegExp).firstMatch(word);
+          if (match != null) {
+            final prefix = match.group(1)!;
+            assert(prefix == affix);
+            final suffix = word.substring(prefix.length);
+            return patterns[prefix] != null
+                ? patterns[prefix]!.map((s) => s + suffix).toList()
+                : [suffix];
+          }
+          break;
+        case (LemmatizationRuleType.prefix, false):
+          if (word.startsWith(p as String)) {
+            return patterns[affix] != null
+                ? patterns[affix]!
+                      .map((s) => s + word.substring(affix.length))
+                      .toList()
+                : [word.substring(affix.length)];
+          }
+          break;
+        case (LemmatizationRuleType.suffix, true):
+          final match = (p as RegExp).firstMatch(word);
+          if (match != null) {
+            final suffix = match.group(2)!;
+            assert(suffix == affix);
+            final prefix = word.substring(0, word.length - suffix.length);
+            return patterns[suffix] != null
+                ? patterns[suffix]!.map((s) => prefix + s).toList()
+                : [prefix];
+          }
+          break;
+        case (LemmatizationRuleType.suffix, false):
+          if (word.endsWith(p as String)) {
+            final prefix = word.substring(0, word.length - affix.length);
+            return patterns[affix] != null
+                ? patterns[affix]!.map((s) => prefix + s).toList()
+                : [prefix];
+          }
+          break;
+      }
+    }
+    return null;
+  }
+
+  @override
+  int get hashCode => Object.hash(
+    type,
+    ListEquality().hash(lookAheadBehind),
+    MapEquality().hash(patterns),
+  );
+
+  @override
+  bool operator ==(Object other) {
+    if (identical(this, other)) return true;
+    return other is AllomorphPattern &&
+        other.type == type &&
+        ListEquality().equals(other.lookAheadBehind, lookAheadBehind) &&
+        MapEquality().equals(other.patterns, patterns);
+  }
+}
+
+class Lemmatized {
+  final String original;
+  final LemmatizationRule rule;
+  final int variant;
+  final List<Lemmatized> children;
+
+  const Lemmatized({
+    required this.original,
+    required this.rule,
+    this.variant = 0,
+    this.children = const [],
+  });
+
+  String? get applied {
+    final applied = rule.apply(original);
+    if (applied == null || applied.isEmpty) {
+      return null;
+    }
+    return applied[variant];
+  }
+
+  @override
+  String toString() {
+    final childrenString = children
+        .map((c) => '  - ${c.toString().split('\n').join('\n  ')}')
+        .join('\n');
+
+    if (children.isEmpty) {
+      return '$original (${rule.name}) -> ${applied ?? '<null>'}';
+    } else {
+      return '$original (${rule.name}) -> ${applied ?? '<null>'}\n$childrenString';
+    }
+  }
+}
+
+List<Lemmatized> _lemmatize(LemmatizationRule parentRule, String word) {
+  final children = <Lemmatized>[];
+
+  if (parentRule.terminal) {
+    return children;
+  }
+
+  final filteredLemmatizationRules = parentRule.validChildClasses == null
+      ? lemmatizationRules
+      : [
+          for (final wordClass in parentRule.validChildClasses!)
+            ...lemmatizationRulesByWordClass[wordClass]!,
+        ];
+
+  for (final rule in filteredLemmatizationRules) {
+    if (rule.matches(word)) {
+      final applied = rule.apply(word);
+      for (final (i, a) in (applied ?? []).indexed) {
+        final subChildren = _lemmatize(rule, a);
+        children.add(
+          Lemmatized(
+            original: word,
+            rule: rule,
+            variant: i,
+            children: subChildren,
+          ),
+        );
+      }
+    }
+  }
+  return children;
+}
+
+Lemmatized lemmatize(String word) {
+  final inputRule = LemmatizationRule.simple(
+    name: 'Input',
+    pattern: '',
+    replacement: null,
+    wordClass: WordClass.input,
+  );
+  return Lemmatized(
+    original: word,
+    rule: inputRule,
+    children: _lemmatize(inputRule, word),
+  );
+}

lib/util/lemmatizer/rules.dart

@@ -0,0 +1,17 @@
+import 'package:jadb/util/lemmatizer/lemmatizer.dart';
+import 'package:jadb/util/lemmatizer/rules/godan_verbs.dart';
+import 'package:jadb/util/lemmatizer/rules/i_adjectives.dart';
+import 'package:jadb/util/lemmatizer/rules/ichidan_verbs.dart';
+
+final List<LemmatizationRule> lemmatizationRules = List.unmodifiable([
+  ...ichidanVerbLemmatizationRules,
+  ...godanVerbLemmatizationRules,
+  ...iAdjectiveLemmatizationRules,
+]);
+
+final Map<WordClass, List<LemmatizationRule>> lemmatizationRulesByWordClass =
+    Map.unmodifiable({
+      WordClass.ichidanVerb: ichidanVerbLemmatizationRules,
+      WordClass.iAdjective: iAdjectiveLemmatizationRules,
+      WordClass.godanVerb: godanVerbLemmatizationRules,
+    });

lib/util/lemmatizer/rules/godan_verbs.dart

@@ -0,0 +1,509 @@
+import 'package:jadb/util/lemmatizer/lemmatizer.dart';
+
+final LemmatizationRule godanVerbBase = LemmatizationRule(
+  name: 'Godan verb - base form',
+  terminal: true,
+  pattern: AllomorphPattern(
+    patterns: {
+      'う': ['う'],
+      'く': ['く'],
+      'ぐ': ['ぐ'],
+      'す': ['す'],
+      'つ': ['つ'],
+      'ぬ': ['ぬ'],
+      'ぶ': ['ぶ'],
+      'む': ['む'],
+      'る': ['る'],
+    },
+    type: LemmatizationRuleType.suffix,
+  ),
+  validChildClasses: {WordClass.godanVerb},
+  wordClass: WordClass.godanVerb,
+);
+
+final LemmatizationRule godanVerbNegative = LemmatizationRule(
+  name: 'Godan verb - negative form',
+  pattern: AllomorphPattern(
+    patterns: {
+      'わない': ['う'],
+      'かない': ['く'],
+      'がない': ['ぐ'],
+      'さない': ['す'],
+      'たない': ['つ'],
+      'なない': ['ぬ'],
+      'ばない': ['ぶ'],
+      'まない': ['む'],
+      'らない': ['る'],
+    },
+    type: LemmatizationRuleType.suffix,
+  ),
+  validChildClasses: {WordClass.godanVerb},
+  wordClass: WordClass.godanVerb,
+);
+
+final LemmatizationRule godanVerbPast = LemmatizationRule(
+  name: 'Godan verb - past form',
+  pattern: AllomorphPattern(
+    patterns: {
+      'した': ['す'],
+      'った': ['る', 'つ', 'う'],
+      'んだ': ['む', 'ぬ', 'ぶ'],
+      'いだ': ['ぐ'],
+      'いた': ['く'],
+    },
+    type: LemmatizationRuleType.suffix,
+  ),
+  validChildClasses: {WordClass.godanVerb},
+  wordClass: WordClass.godanVerb,
+);
+
+final LemmatizationRule godanVerbTe = LemmatizationRule(
+  name: 'Godan verb - te-form',
+  pattern: AllomorphPattern(
+    patterns: {
+      'いて': ['く', 'ぐ'],
+      'して': ['す'],
+      'って': ['る', 'つ', 'う'],
+      'んで': ['む', 'ぬ', 'ぶ'],
+    },
+    type: LemmatizationRuleType.suffix,
+  ),
+  validChildClasses: {WordClass.godanVerb},
+  wordClass: WordClass.godanVerb,
+);
+
+final LemmatizationRule godanVerbTeiru = LemmatizationRule(
+  name: 'Godan verb - te-form with いる',
+  pattern: AllomorphPattern(
+    patterns: {
+      'いている': ['く', 'ぐ'],
+      'している': ['す'],
+      'っている': ['る', 'つ', 'う'],
+      'んでいる': ['む', 'ぬ', 'ぶ'],
+    },
+    type: LemmatizationRuleType.suffix,
+  ),
+  validChildClasses: {WordClass.godanVerb},
+  wordClass: WordClass.godanVerb,
+);
+
+final LemmatizationRule godanVerbTeita = LemmatizationRule(
+  name: 'Godan verb - te-form with いた',
+  pattern: AllomorphPattern(
+    patterns: {
+      'いていた': ['く', 'ぐ'],
+      'していた': ['す'],
+      'っていた': ['る', 'つ', 'う'],
+      'んでいた': ['む', 'ぬ', 'ぶ'],
+    },
+    type: LemmatizationRuleType.suffix,
+  ),
+  validChildClasses: {WordClass.godanVerb},
+  wordClass: WordClass.godanVerb,
+);
+
+final LemmatizationRule godanVerbConditional = LemmatizationRule(
+  name: 'Godan verb - conditional form',
+  pattern: AllomorphPattern(
+    patterns: {
+      'けば': ['く'],
+      'げば': ['ぐ'],
+      'せば': ['す'],
+      'てば': ['つ', 'る', 'う'],
+      'ねば': ['ぬ'],
+      'べば': ['ぶ'],
+      'めば': ['む'],
+    },
+    type: LemmatizationRuleType.suffix,
+  ),
+  validChildClasses: {WordClass.godanVerb},
+  wordClass: WordClass.godanVerb,
+);
+
+final LemmatizationRule godanVerbVolitional = LemmatizationRule(
+  name: 'Godan verb - volitional form',
+  pattern: AllomorphPattern(
+    patterns: {
+      'おう': ['う'],
+      'こう': ['く'],
+      'ごう': ['ぐ'],
+      'そう': ['す'],
+      'とう': ['つ', 'る', 'う'],
+      'のう': ['ぬ'],
+      'ぼう': ['ぶ'],
+      'もう': ['む'],
+    },
+    type: LemmatizationRuleType.suffix,
+  ),
+  validChildClasses: {WordClass.godanVerb},
+  wordClass: WordClass.godanVerb,
+);
+
+final LemmatizationRule godanVerbPotential = LemmatizationRule(
+  name: 'Godan verb - potential form',
+  pattern: AllomorphPattern(
+    patterns: {
+      'ける': ['く'],
+      'げる': ['ぐ'],
+      'せる': ['す'],
+      'てる': ['つ', 'る', 'う'],
+      'ねる': ['ぬ'],
+      'べる': ['ぶ'],
+      'める': ['む'],
+    },
+    type: LemmatizationRuleType.suffix,
+  ),
+  validChildClasses: {WordClass.godanVerb},
+  wordClass: WordClass.godanVerb,
+);
+
+final LemmatizationRule godanVerbPassive = LemmatizationRule(
+  name: 'Godan verb - passive form',
+  pattern: AllomorphPattern(
+    patterns: {
+      'かれる': ['く'],
+      'がれる': ['ぐ'],
+      'される': ['す'],
+      'たれる': ['つ', 'る', 'う'],
+      'なれる': ['ぬ'],
+      'ばれる': ['ぶ'],
+      'まれる': ['む'],
+    },
+    type: LemmatizationRuleType.suffix,
+  ),
+  validChildClasses: {WordClass.godanVerb},
+  wordClass: WordClass.godanVerb,
+);
+
+final LemmatizationRule godanVerbCausative = LemmatizationRule(
+  name: 'Godan verb - causative form',
+  pattern: AllomorphPattern(
+    patterns: {
+      'かせる': ['く'],
+      'がせる': ['ぐ'],
+      'させる': ['す'],
+      'たせる': ['つ', 'る', 'う'],
+      'なせる': ['ぬ'],
+      'ばせる': ['ぶ'],
+      'ませる': ['む'],
+    },
+    type: LemmatizationRuleType.suffix,
+  ),
+  validChildClasses: {WordClass.godanVerb},
+  wordClass: WordClass.godanVerb,
+);
+
+final LemmatizationRule godanVerbCausativePassive = LemmatizationRule(
+  name: 'Godan verb - causative-passive form',
+  pattern: AllomorphPattern(
+    patterns: {
+      'かされる': ['く'],
+      'がされる': ['ぐ'],
+      'される': ['す'],
+      'たされる': ['つ', 'る', 'う'],
+      'なされる': ['ぬ'],
+      'ばされる': ['ぶ'],
+      'まされる': ['む'],
+    },
+    type: LemmatizationRuleType.suffix,
+  ),
+  validChildClasses: {WordClass.godanVerb},
+  wordClass: WordClass.godanVerb,
+);
+
+final LemmatizationRule godanVerbImperative = LemmatizationRule(
+  name: 'Godan verb - imperative form',
+  pattern: AllomorphPattern(
+    patterns: {
+      'え': ['う'],
+      'け': ['く'],
+      'げ': ['ぐ'],
+      'せ': ['す'],
+      'て': ['つ', 'る', 'う'],
+      'ね': ['ぬ'],
+      'べ': ['ぶ'],
+      'め': ['む'],
+    },
+    type: LemmatizationRuleType.suffix,
+  ),
+  validChildClasses: {WordClass.godanVerb},
+  wordClass: WordClass.godanVerb,
+);
+
+final LemmatizationRule godanVerbNegativePast = LemmatizationRule(
+  name: 'Godan verb - negative past form',
+  pattern: AllomorphPattern(
+    patterns: {
+      'わなかった': ['う'],
+      'かなかった': ['く'],
+      'がなかった': ['ぐ'],
+      'さなかった': ['す'],
+      'たなかった': ['つ'],
+      'ななかった': ['ぬ'],
+      'ばなかった': ['ぶ'],
+      'まなかった': ['む'],
+      'らなかった': ['る'],
+    },
+    type: LemmatizationRuleType.suffix,
+  ),
+  validChildClasses: {WordClass.godanVerb},
+  wordClass: WordClass.godanVerb,
+);
+
+final LemmatizationRule godanVerbNegativeTe = LemmatizationRule(
+  name: 'Godan verb - negative te-form',
+  pattern: AllomorphPattern(
+    patterns: {
+      'わなくて': ['う'],
+      'かなくて': ['く'],
+      'がなくて': ['ぐ'],
+      'さなくて': ['す'],
+      'たなくて': ['つ'],
+      'ななくて': ['ぬ'],
+      'ばなくて': ['ぶ'],
+      'まなくて': ['む'],
+      'らなくて': ['る'],
+    },
+    type: LemmatizationRuleType.suffix,
+  ),
+  validChildClasses: {WordClass.godanVerb},
+  wordClass: WordClass.godanVerb,
+);
+
+final LemmatizationRule godanVerbNegativeConditional = LemmatizationRule(
+  name: 'Godan verb - negative conditional form',
+  pattern: AllomorphPattern(
+    patterns: {
+      'わなければ': ['う'],
+      'かなければ': ['く'],
+      'がなければ': ['ぐ'],
+      'さなければ': ['す'],
+      'たなければ': ['つ'],
+      'ななければ': ['ぬ'],
+      'ばなければ': ['ぶ'],
+      'まなければ': ['む'],
+      'らなければ': ['る'],
+    },
+    type: LemmatizationRuleType.suffix,
+  ),
+  validChildClasses: {WordClass.godanVerb},
+  wordClass: WordClass.godanVerb,
+);
+
+final LemmatizationRule godanVerbNegativeVolitional = LemmatizationRule(
+  name: 'Godan verb - negative volitional form',
+  pattern: AllomorphPattern(
+    patterns: {
+      'うまい': ['う'],
+      'くまい': ['く'],
+      'ぐまい': ['ぐ'],
+      'すまい': ['す'],
+      'つまい': ['つ', 'る', 'う'],
+      'ぬまい': ['ぬ'],
+      'ぶまい': ['ぶ'],
+      'むまい': ['む'],
+    },
+    type: LemmatizationRuleType.suffix,
+  ),
+  validChildClasses: {WordClass.godanVerb},
+  wordClass: WordClass.godanVerb,
+);
+
+final LemmatizationRule godanVerbNegativePotential = LemmatizationRule(
+  name: 'Godan verb - negative potential form',
+  pattern: AllomorphPattern(
+    patterns: {
+      'けない': ['く'],
+      'げない': ['ぐ'],
+      'せない': ['す'],
+      'てない': ['つ', 'る', 'う'],
+      'ねない': ['ぬ'],
+      'べない': ['ぶ'],
+      'めない': ['む'],
+    },
+    type: LemmatizationRuleType.suffix,
+  ),
+  validChildClasses: {WordClass.godanVerb},
+  wordClass: WordClass.godanVerb,
+);
+
+final LemmatizationRule godanVerbNegativePassive = LemmatizationRule(
+  name: 'Godan verb - negative passive form',
+  pattern: AllomorphPattern(
+    patterns: {
+      'かれない': ['く'],
+      'がれない': ['ぐ'],
+      'されない': ['す'],
+      'たれない': ['つ', 'る', 'う'],
+      'なれない': ['ぬ'],
+      'ばれない': ['ぶ'],
+      'まれない': ['む'],
+    },
+    type: LemmatizationRuleType.suffix,
+  ),
+  validChildClasses: {WordClass.godanVerb},
+  wordClass: WordClass.godanVerb,
+);
+
+final LemmatizationRule godanVerbNegativeCausative = LemmatizationRule(
+  name: 'Godan verb - negative causative form',
+  pattern: AllomorphPattern(
+    patterns: {
+      'かせない': ['く'],
+      'がせない': ['ぐ'],
+      'させない': ['す'],
+      'たせない': ['つ', 'る', 'う'],
+      'なせない': ['ぬ'],
+      'ばせない': ['ぶ'],
+      'ませない': ['む'],
+    },
+    type: LemmatizationRuleType.suffix,
+  ),
+  validChildClasses: {WordClass.godanVerb},
+  wordClass: WordClass.godanVerb,
+);
+
+final LemmatizationRule godanVerbNegativeCausativePassive = LemmatizationRule(
+  name: 'Godan verb - negative causative-passive form',
+  pattern: AllomorphPattern(
+    patterns: {
+      'かされない': ['く'],
+      'がされない': ['ぐ'],
+      'されない': ['す'],
+      'たされない': ['つ', 'る', 'う'],
+      'なされない': ['ぬ'],
+      'ばされない': ['ぶ'],
+      'まされない': ['む'],
+    },
+    type: LemmatizationRuleType.suffix,
+  ),
+  validChildClasses: {WordClass.godanVerb},
+  wordClass: WordClass.godanVerb,
+);
+
+final LemmatizationRule godanVerbNegativeImperative = LemmatizationRule(
+  name: 'Godan verb - negative imperative form',
+  pattern: AllomorphPattern(
+    patterns: {
+      'うな': ['う'],
+      'くな': ['く'],
+      'ぐな': ['ぐ'],
+      'すな': ['す'],
+      'つな': ['つ'],
+      'ぬな': ['ぬ'],
+      'ぶな': ['ぶ'],
+      'むな': ['む'],
+      'るな': ['る'],
+    },
+    type: LemmatizationRuleType.suffix,
+  ),
+  validChildClasses: {WordClass.godanVerb},
+  wordClass: WordClass.godanVerb,
+);
+
+final LemmatizationRule godanVerbDesire = LemmatizationRule(
+  name: 'Godan verb - desire form',
+  pattern: AllomorphPattern(
+    patterns: {
+      'きたい': ['く'],
+      'ぎたい': ['ぐ'],
+      'したい': ['す'],
+      'ちたい': ['つ'],
+      'にたい': ['ぬ'],
+      'びたい': ['ぶ'],
+      'みたい': ['む'],
+      'りたい': ['る'],
+    },
+    type: LemmatizationRuleType.suffix,
+  ),
+  validChildClasses: {WordClass.godanVerb},
+  wordClass: WordClass.godanVerb,
+);
+
+final LemmatizationRule godanVerbNegativeDesire = LemmatizationRule(
+  name: 'Godan verb - negative desire form',
+  pattern: AllomorphPattern(
+    patterns: {
+      'いたくない': ['う'],
+      'きたくない': ['く'],
+      'ぎたくない': ['ぐ'],
+      'したくない': ['す'],
+      'ちたくない': ['つ'],
+      'にたくない': ['ぬ'],
+      'びたくない': ['ぶ'],
+      'みたくない': ['む'],
+      'りたくない': ['る'],
+    },
+    type: LemmatizationRuleType.suffix,
+  ),
+  validChildClasses: {WordClass.godanVerb},
+  wordClass: WordClass.godanVerb,
+);
+
+final LemmatizationRule godanVerbPastDesire = LemmatizationRule(
+  name: 'Godan verb - past desire form',
+  pattern: AllomorphPattern(
+    patterns: {
+      'きたかった': ['く'],
+      'ぎたかった': ['ぐ'],
+      'したかった': ['す'],
+      'ちたかった': ['つ'],
+      'にたかった': ['ぬ'],
+      'びたかった': ['ぶ'],
+      'みたかった': ['む'],
+      'りたかった': ['る'],
+    },
+    type: LemmatizationRuleType.suffix,
+  ),
+  validChildClasses: {WordClass.godanVerb},
+  wordClass: WordClass.godanVerb,
+);
+
+final LemmatizationRule godanVerbNegativePastDesire = LemmatizationRule(
+  name: 'Godan verb - negative past desire form',
+  pattern: AllomorphPattern(
+    patterns: {
+      'いたくなかった': ['う'],
+      'きたくなかった': ['く'],
+      'ぎたくなかった': ['ぐ'],
+      'したくなかった': ['す'],
+      'ちたくなかった': ['つ'],
+      'にたくなかった': ['ぬ'],
+      'びたくなかった': ['ぶ'],
+      'みたくなかった': ['む'],
+      'りたくなかった': ['る'],
+    },
+    type: LemmatizationRuleType.suffix,
+  ),
+  validChildClasses: {WordClass.godanVerb},
+  wordClass: WordClass.godanVerb,
+);
+
+final List<LemmatizationRule> godanVerbLemmatizationRules = List.unmodifiable([
+  godanVerbBase,
+  godanVerbNegative,
+  godanVerbPast,
+  godanVerbTe,
+  godanVerbTeiru,
+  godanVerbTeita,
+  godanVerbConditional,
+  godanVerbVolitional,
+  godanVerbPotential,
+  godanVerbPassive,
+  godanVerbCausative,
+  godanVerbCausativePassive,
+  godanVerbImperative,
+  godanVerbNegativePast,
+  godanVerbNegativeTe,
+  godanVerbNegativeConditional,
+  godanVerbNegativeVolitional,
+  godanVerbNegativePotential,
+  godanVerbNegativePassive,
+  godanVerbNegativeCausative,
+  godanVerbNegativeCausativePassive,
+  godanVerbNegativeImperative,
+  godanVerbDesire,
+  godanVerbNegativeDesire,
+  godanVerbPastDesire,
+  godanVerbNegativePastDesire,
+]);

lib/util/lemmatizer/rules/i_adjectives.dart

@@ -0,0 +1,77 @@
+import 'package:jadb/util/lemmatizer/lemmatizer.dart';
+
+final LemmatizationRule iAdjectiveBase = LemmatizationRule.simple(
+  name: 'I adjective - base form',
+  terminal: true,
+  pattern: 'い',
+  replacement: 'い',
+  validChildClasses: {WordClass.iAdjective},
+  wordClass: WordClass.iAdjective,
+);
+
+final LemmatizationRule iAdjectiveNegative = LemmatizationRule.simple(
+  name: 'I adjective - negative form',
+  pattern: 'くない',
+  replacement: 'い',
+  validChildClasses: {WordClass.iAdjective},
+  wordClass: WordClass.iAdjective,
+);
+
+final LemmatizationRule iAdjectivePast = LemmatizationRule.simple(
+  name: 'I adjective - past form',
+  pattern: 'かった',
+  replacement: 'い',
+  validChildClasses: {WordClass.iAdjective},
+  wordClass: WordClass.iAdjective,
+);
+
+final LemmatizationRule iAdjectiveNegativePast = LemmatizationRule.simple(
+  name: 'I adjective - negative past form',
+  pattern: 'くなかった',
+  replacement: 'い',
+  validChildClasses: {WordClass.iAdjective},
+  wordClass: WordClass.iAdjective,
+);
+
+final LemmatizationRule iAdjectiveTe = LemmatizationRule.simple(
+  name: 'I adjective - te-form',
+  pattern: 'くて',
+  replacement: 'い',
+  validChildClasses: {WordClass.iAdjective},
+  wordClass: WordClass.iAdjective,
+);
+
+final LemmatizationRule iAdjectiveConditional = LemmatizationRule.simple(
+  name: 'I adjective - conditional form',
+  pattern: 'ければ',
+  replacement: 'い',
+  validChildClasses: {WordClass.iAdjective},
+  wordClass: WordClass.iAdjective,
+);
+
+final LemmatizationRule iAdjectiveVolitional = LemmatizationRule.simple(
+  name: 'I adjective - volitional form',
+  pattern: 'かろう',
+  replacement: 'い',
+  validChildClasses: {WordClass.iAdjective},
+  wordClass: WordClass.iAdjective,
+);
+
+final LemmatizationRule iAdjectiveContinuative = LemmatizationRule.simple(
+  name: 'I adjective - continuative form',
+  pattern: 'く',
+  replacement: 'い',
+  validChildClasses: {WordClass.iAdjective},
+  wordClass: WordClass.iAdjective,
+);
+
+final List<LemmatizationRule> iAdjectiveLemmatizationRules = List.unmodifiable([
+  iAdjectiveBase,
+  iAdjectiveNegative,
+  iAdjectivePast,
+  iAdjectiveNegativePast,
+  iAdjectiveTe,
+  iAdjectiveConditional,
+  iAdjectiveVolitional,
+  iAdjectiveContinuative,
+]);

lib/util/lemmatizer/rules/ichidan_verbs.dart

@@ -0,0 +1,331 @@
+import 'package:jadb/util/lemmatizer/lemmatizer.dart';
+import 'package:jadb/util/text_filtering.dart';
+
+final List<Pattern> _lookBehinds = [
+  kanjiRegex,
+  'き',
+  'ぎ',
+  'し',
+  'じ',
+  'ち',
+  'ぢ',
+  'に',
+  'ひ',
+  'び',
+  'び',
+  'み',
+  'り',
+  'け',
+  'げ',
+  'せ',
+  'ぜ',
+  'て',
+  'で',
+  'ね',
+  'へ',
+  'べ',
+  'め',
+  'れ',
+];
+
+final LemmatizationRule ichidanVerbBase = LemmatizationRule.simple(
+  name: 'Ichidan verb - base form',
+  terminal: true,
+  pattern: 'る',
+  replacement: 'る',
+  lookAheadBehind: _lookBehinds,
+  validChildClasses: {WordClass.ichidanVerb},
+  wordClass: WordClass.ichidanVerb,
+);
+
+final LemmatizationRule ichidanVerbNegative = LemmatizationRule.simple(
+  name: 'Ichidan verb - negative form',
+  pattern: 'ない',
+  replacement: 'る',
+  lookAheadBehind: _lookBehinds,
+  validChildClasses: {WordClass.ichidanVerb},
+  wordClass: WordClass.ichidanVerb,
+);
+final LemmatizationRule ichidanVerbPast = LemmatizationRule.simple(
+  name: 'Ichidan verb - past form',
+  pattern: 'た',
+  replacement: 'る',
+  lookAheadBehind: _lookBehinds,
+  validChildClasses: {WordClass.ichidanVerb},
+  wordClass: WordClass.ichidanVerb,
+);
+
+final LemmatizationRule ichidanVerbTe = LemmatizationRule.simple(
+  name: 'Ichidan verb - te-form',
+  pattern: 'て',
+  replacement: 'る',
+  lookAheadBehind: _lookBehinds,
+  validChildClasses: {WordClass.ichidanVerb},
+  wordClass: WordClass.ichidanVerb,
+);
+
+final LemmatizationRule ichidanVerbTeiru = LemmatizationRule.simple(
+  name: 'Ichidan verb - te-form with いる',
+  pattern: 'ている',
+  replacement: 'る',
+  lookAheadBehind: _lookBehinds,
+  validChildClasses: {WordClass.ichidanVerb},
+  wordClass: WordClass.ichidanVerb,
+);
+
+final LemmatizationRule ichidanVerbTeita = LemmatizationRule.simple(
+  name: 'Ichidan verb - te-form with いた',
+  pattern: 'ていた',
+  replacement: 'る',
+  lookAheadBehind: _lookBehinds,
+  validChildClasses: {WordClass.ichidanVerb},
+  wordClass: WordClass.ichidanVerb,
+);
+
+final LemmatizationRule ichidanVerbConditional = LemmatizationRule.simple(
+  name: 'Ichidan verb - conditional form',
+  pattern: 'れば',
+  replacement: 'る',
+  lookAheadBehind: _lookBehinds,
+  validChildClasses: {WordClass.ichidanVerb},
+  wordClass: WordClass.ichidanVerb,
+);
+
+final LemmatizationRule ichidanVerbVolitional = LemmatizationRule.simple(
+  name: 'Ichidan verb - volitional form',
+  pattern: 'よう',
+  replacement: 'る',
+  lookAheadBehind: _lookBehinds,
+  validChildClasses: {WordClass.ichidanVerb},
+  wordClass: WordClass.ichidanVerb,
+);
+
+final LemmatizationRule ichidanVerbPotential = LemmatizationRule.simple(
+  name: 'Ichidan verb - potential form',
+  pattern: 'られる',
+  replacement: 'る',
+  lookAheadBehind: _lookBehinds,
+  validChildClasses: {WordClass.ichidanVerb},
+  wordClass: WordClass.ichidanVerb,
+);
+
+final LemmatizationRule ichidanVerbPassive = LemmatizationRule.simple(
+  name: 'Ichidan verb - passive form',
+  pattern: 'られる',
+  replacement: 'る',
+  lookAheadBehind: _lookBehinds,
+  validChildClasses: {WordClass.ichidanVerb},
+  wordClass: WordClass.ichidanVerb,
+);
+
+final LemmatizationRule ichidanVerbCausative = LemmatizationRule.simple(
+  name: 'Ichidan verb - causative form',
+  pattern: 'させる',
+  replacement: 'る',
+  lookAheadBehind: _lookBehinds,
+  validChildClasses: {WordClass.ichidanVerb},
+  wordClass: WordClass.ichidanVerb,
+);
+
+final LemmatizationRule ichidanVerbCausativePassive = LemmatizationRule.simple(
+  name: 'Ichidan verb - causative passive form',
+  pattern: 'させられる',
+  replacement: 'る',
+  lookAheadBehind: _lookBehinds,
+  validChildClasses: {WordClass.ichidanVerb},
+  wordClass: WordClass.ichidanVerb,
+);
+
+final LemmatizationRule ichidanVerbImperative = LemmatizationRule.simple(
+  name: 'Ichidan verb - imperative form',
+  pattern: 'れ',
+  replacement: 'る',
+  lookAheadBehind: _lookBehinds,
+  validChildClasses: {WordClass.ichidanVerb},
+  wordClass: WordClass.ichidanVerb,
+);
+
+final LemmatizationRule ichidanVerbNegativePast = LemmatizationRule.simple(
+  name: 'Ichidan verb - negative past form',
+  pattern: 'なかった',
+  replacement: 'る',
+  lookAheadBehind: _lookBehinds,
+  validChildClasses: {WordClass.ichidanVerb},
+  wordClass: WordClass.ichidanVerb,
+);
+
+final LemmatizationRule ichidanVerbNegativeTe = LemmatizationRule.simple(
+  name: 'Ichidan verb - negative te-form',
+  pattern: 'なくて',
+  replacement: 'る',
+  lookAheadBehind: _lookBehinds,
+  validChildClasses: {WordClass.ichidanVerb},
+  wordClass: WordClass.ichidanVerb,
+);
+
+final LemmatizationRule ichidanVerbNegativeConditional =
+    LemmatizationRule.simple(
+      name: 'Ichidan verb - negative conditional form',
+      pattern: 'なければ',
+      replacement: 'る',
+      lookAheadBehind: _lookBehinds,
+      validChildClasses: {WordClass.ichidanVerb},
+      wordClass: WordClass.ichidanVerb,
+    );
+
+final LemmatizationRule ichidanVerbNegativeConditionalVariant1 =
+    LemmatizationRule.simple(
+      name: 'Ichidan verb - negative conditional form (informal variant)',
+      pattern: 'なきゃ',
+      replacement: 'る',
+      lookAheadBehind: _lookBehinds,
+      validChildClasses: {WordClass.ichidanVerb},
+      wordClass: WordClass.ichidanVerb,
+    );
+
+final LemmatizationRule ichidanVerbNegativeConditionalVariant2 =
+    LemmatizationRule.simple(
+      name: 'Ichidan verb - negative conditional form (informal variant)',
+      pattern: 'なくちゃ',
+      replacement: 'る',
+      lookAheadBehind: _lookBehinds,
+      validChildClasses: {WordClass.ichidanVerb},
+      wordClass: WordClass.ichidanVerb,
+    );
+
+final LemmatizationRule ichidanVerbNegativeConditionalVariant3 =
+    LemmatizationRule.simple(
+      name: 'Ichidan verb - negative conditional form (informal variant)',
+      pattern: 'ないと',
+      replacement: 'る',
+      lookAheadBehind: _lookBehinds,
+      validChildClasses: {WordClass.ichidanVerb},
+      wordClass: WordClass.ichidanVerb,
+    );
+
+final LemmatizationRule ichidanVerbNegativeVolitional =
+    LemmatizationRule.simple(
+      name: 'Ichidan verb - negative volitional form',
+      pattern: 'なかろう',
+      replacement: 'る',
+      lookAheadBehind: _lookBehinds,
+      validChildClasses: {WordClass.ichidanVerb},
+      wordClass: WordClass.ichidanVerb,
+    );
+
+final LemmatizationRule ichidanVerbNegativePotential = LemmatizationRule.simple(
+  name: 'Ichidan verb - negative potential form',
+  pattern: 'られない',
+  replacement: 'る',
+  lookAheadBehind: _lookBehinds,
+  validChildClasses: {WordClass.ichidanVerb},
+  wordClass: WordClass.ichidanVerb,
+);
+
+final LemmatizationRule ichidanVerbNegativePassive = LemmatizationRule.simple(
+  name: 'Ichidan verb - negative passive form',
+  pattern: 'られない',
+  replacement: 'る',
+  lookAheadBehind: _lookBehinds,
+  validChildClasses: {WordClass.ichidanVerb},
+  wordClass: WordClass.ichidanVerb,
+);
+
+final LemmatizationRule ichidanVerbNegativeCausative = LemmatizationRule.simple(
+  name: 'Ichidan verb - negative causative form',
+  pattern: 'させない',
+  replacement: 'る',
+  lookAheadBehind: _lookBehinds,
+  validChildClasses: {WordClass.ichidanVerb},
+  wordClass: WordClass.ichidanVerb,
+);
+
+final LemmatizationRule ichidanVerbNegativeCausativePassive =
+    LemmatizationRule.simple(
+      name: 'Ichidan verb - negative causative passive form',
+      pattern: 'させられない',
+      replacement: 'る',
+      lookAheadBehind: _lookBehinds,
+      validChildClasses: {WordClass.ichidanVerb},
+      wordClass: WordClass.ichidanVerb,
+    );
+
+final LemmatizationRule ichidanVerbNegativeImperative =
+    LemmatizationRule.simple(
+      name: 'Ichidan verb - negative imperative form',
+      pattern: 'るな',
+      replacement: 'る',
+      lookAheadBehind: _lookBehinds,
+      validChildClasses: {WordClass.ichidanVerb},
+      wordClass: WordClass.ichidanVerb,
+    );
+
+final LemmatizationRule ichidanVerbDesire = LemmatizationRule.simple(
+  name: 'Ichidan verb - desire form',
+  pattern: 'たい',
+  replacement: 'る',
+  lookAheadBehind: _lookBehinds,
+  validChildClasses: {WordClass.ichidanVerb},
+  wordClass: WordClass.ichidanVerb,
+);
+
+final LemmatizationRule ichidanVerbNegativeDesire = LemmatizationRule.simple(
+  name: 'Ichidan verb - negative desire form',
+  pattern: 'たくない',
+  replacement: 'る',
+  lookAheadBehind: _lookBehinds,
+  validChildClasses: {WordClass.ichidanVerb},
+  wordClass: WordClass.ichidanVerb,
+);
+
+final LemmatizationRule ichidanVerbPastDesire = LemmatizationRule.simple(
+  name: 'Ichidan verb - past desire form',
+  pattern: 'たかった',
+  replacement: 'る',
+  lookAheadBehind: _lookBehinds,
+  validChildClasses: {WordClass.ichidanVerb},
+  wordClass: WordClass.ichidanVerb,
+);
+
+final LemmatizationRule ichidanVerbNegativePastDesire =
+    LemmatizationRule.simple(
+      name: 'Ichidan verb - negative past desire form',
+      pattern: 'たくなかった',
+      replacement: 'る',
+      lookAheadBehind: _lookBehinds,
+      validChildClasses: {WordClass.ichidanVerb},
+      wordClass: WordClass.ichidanVerb,
+    );
+
+final List<LemmatizationRule> ichidanVerbLemmatizationRules =
+    List.unmodifiable([
+      ichidanVerbBase,
+      ichidanVerbNegative,
+      ichidanVerbPast,
+      ichidanVerbTe,
+      ichidanVerbTeiru,
+      ichidanVerbTeita,
+      ichidanVerbConditional,
+      ichidanVerbVolitional,
+      ichidanVerbPotential,
+      ichidanVerbPassive,
+      ichidanVerbCausative,
+      ichidanVerbCausativePassive,
+      ichidanVerbImperative,
+      ichidanVerbNegativePast,
+      ichidanVerbNegativeTe,
+      ichidanVerbNegativeConditional,
+      ichidanVerbNegativeConditionalVariant1,
+      ichidanVerbNegativeConditionalVariant2,
+      ichidanVerbNegativeConditionalVariant3,
+      ichidanVerbNegativeVolitional,
+      ichidanVerbNegativePotential,
+      ichidanVerbNegativePassive,
+      ichidanVerbNegativeCausative,
+      ichidanVerbNegativeCausativePassive,
+      ichidanVerbNegativeImperative,
+      ichidanVerbDesire,
+      ichidanVerbNegativeDesire,
+      ichidanVerbPastDesire,
+      ichidanVerbNegativePastDesire,
+    ]);

lib/util/romaji_transliteration.dart

@@ -1,9 +1,9 @@
 // Source: https://github.com/Kimtaro/ve/blob/master/lib/providers/japanese_transliterators.rb
 
-const hiragana_syllabic_n = 'ん';
-const hiragana_small_tsu = 'っ';
+const hiraganaSyllabicN = 'ん';
+const hiraganaSmallTsu = 'っ';
 
-const Map<String, String> hiragana_to_latin = {
+const Map<String, String> hiraganaToLatin = {
   'あ': 'a',
   'い': 'i',
   'う': 'u',
@@ -209,7 +209,7 @@ const Map<String, String> hiragana_to_latin = {
   'ゟ': 'yori',
 };
 
-const Map<String, String> latin_to_hiragana = {
+const Map<String, String> latinToHiragana = {
   'a': 'あ',
   'i': 'い',
   'u': 'う',
@@ -481,12 +481,13 @@ const Map<String, String> latin_to_hiragana = {
   '#~': '〜',
 };
 
-bool _smallTsu(String for_conversion) => for_conversion == hiragana_small_tsu;
-bool _nFollowedByYuYeYo(String for_conversion, String kana) =>
-    for_conversion == hiragana_syllabic_n &&
+bool _smallTsu(String forConversion) => forConversion == hiraganaSmallTsu;
+bool _nFollowedByYuYeYo(String forConversion, String kana) =>
+    forConversion == hiraganaSyllabicN &&
     kana.length > 1 &&
     'やゆよ'.contains(kana.substring(1, 2));
 
+/// Transliterates a string of hiragana characters to Latin script (romaji).
 String transliterateHiraganaToLatin(String hiragana) {
   String kana = hiragana;
   String romaji = '';
@@ -495,17 +496,17 @@ String transliterateHiraganaToLatin(String hiragana) {
   while (kana.isNotEmpty) {
     final lengths = [if (kana.length > 1) 2, 1];
     for (final length in lengths) {
-      final String for_conversion = kana.substring(0, length);
+      final String forConversion = kana.substring(0, length);
       String? mora;
 
-      if (_smallTsu(for_conversion)) {
+      if (_smallTsu(forConversion)) {
         geminate = true;
         kana = kana.replaceRange(0, length, '');
         break;
-      } else if (_nFollowedByYuYeYo(for_conversion, kana)) {
+      } else if (_nFollowedByYuYeYo(forConversion, kana)) {
         mora = "n'";
       }
-      mora ??= hiragana_to_latin[for_conversion];
+      mora ??= hiraganaToLatin[forConversion];
 
       if (mora != null) {
         if (geminate) {
@@ -516,7 +517,7 @@ String transliterateHiraganaToLatin(String hiragana) {
         kana = kana.replaceRange(0, length, '');
         break;
       } else if (length == 1) {
-        romaji += for_conversion;
+        romaji += forConversion;
         kana = kana.replaceRange(0, length, '');
       }
     }
@@ -524,48 +525,92 @@ String transliterateHiraganaToLatin(String hiragana) {
   return romaji;
 }
 
-bool _doubleNFollowedByAIUEO(String for_conversion) =>
-    RegExp(r'^nn[aiueo]$').hasMatch(for_conversion);
-bool _hasTableMatch(String for_conversion) =>
-    latin_to_hiragana[for_conversion] != null;
-bool _hasDoubleConsonant(String for_conversion, int length) =>
-    for_conversion == 'tch' ||
-    (length == 2 &&
-        RegExp(r'^([kgsztdnbpmyrlwchf])\1$').hasMatch(for_conversion));
+/// Returns a list of pairs of indices into the input and output strings,
+/// indicating which characters in the input string correspond to which characters in the output string.
+List<(int, int)> transliterateHiraganaToLatinSpan(String hiragana) {
+  String kana = hiragana;
+  String romaji = '';
+  final List<(int, int)> spans = [];
+  bool geminate = false;
+  int kanaIndex = 0;
 
+  while (kana.isNotEmpty) {
+    final lengths = [if (kana.length > 1) 2, 1];
+    for (final length in lengths) {
+      final String forConversion = kana.substring(0, length);
+      String? mora;
+
+      if (_smallTsu(forConversion)) {
+        geminate = true;
+        kana = kana.replaceRange(0, length, '');
+        break;
+      } else if (_nFollowedByYuYeYo(forConversion, kana)) {
+        mora = "n'";
+      }
+      mora ??= hiraganaToLatin[forConversion];
+
+      if (mora != null) {
+        if (geminate) {
+          geminate = false;
+          romaji += mora.substring(0, 1);
+        }
+        spans.add((kanaIndex, romaji.length));
+        romaji += mora;
+        kana = kana.replaceRange(0, length, '');
+        kanaIndex += length;
+        break;
+      } else if (length == 1) {
+        spans.add((kanaIndex, romaji.length));
+        romaji += forConversion;
+        kana = kana.replaceRange(0, length, '');
+        kanaIndex += length;
+      }
+    }
+  }
+  return spans;
+}
+
+bool _doubleNFollowedByAIUEO(String forConversion) =>
+    RegExp(r'^nn[aiueo]$').hasMatch(forConversion);
+bool _hasTableMatch(String forConversion) =>
+    latinToHiragana[forConversion] != null;
+bool _hasDoubleConsonant(String forConversion, int length) =>
+    forConversion == 'tch' ||
+    (length == 2 &&
+        RegExp(r'^([kgsztdnbpmyrlwchf])\1$').hasMatch(forConversion));
+
+/// Transliterates a string of Latin script (romaji) to hiragana characters.
 String transliterateLatinToHiragana(String latin) {
-  String romaji =
-      latin.toLowerCase().replaceAll('mb', 'nb').replaceAll('mp', 'np');
+  String romaji = latin
+      .toLowerCase()
+      .replaceAll('mb', 'nb')
+      .replaceAll('mp', 'np');
   String kana = '';
 
   while (romaji.isNotEmpty) {
-    final lengths = [
-      if (romaji.length > 2) 3,
-      if (romaji.length > 1) 2,
-      1,
-    ];
+    final lengths = [if (romaji.length > 2) 3, if (romaji.length > 1) 2, 1];
 
     for (final length in lengths) {
       String? mora;
-      int for_removal = length;
-      final String for_conversion = romaji.substring(0, length);
+      int forRemoval = length;
+      final String forConversion = romaji.substring(0, length);
 
-      if (_doubleNFollowedByAIUEO(for_conversion)) {
-        mora = hiragana_syllabic_n;
-        for_removal = 1;
-      } else if (_hasTableMatch(for_conversion)) {
-        mora = latin_to_hiragana[for_conversion];
-      } else if (_hasDoubleConsonant(for_conversion, length)) {
-        mora = hiragana_small_tsu;
-        for_removal = 1;
+      if (_doubleNFollowedByAIUEO(forConversion)) {
+        mora = hiraganaSyllabicN;
+        forRemoval = 1;
+      } else if (_hasTableMatch(forConversion)) {
+        mora = latinToHiragana[forConversion];
+      } else if (_hasDoubleConsonant(forConversion, length)) {
+        mora = hiraganaSmallTsu;
+        forRemoval = 1;
       }
 
       if (mora != null) {
         kana += mora;
-        romaji = romaji.replaceRange(0, for_removal, '');
+        romaji = romaji.replaceRange(0, forRemoval, '');
         break;
       } else if (length == 1) {
-        kana += for_conversion;
+        kana += forConversion;
         romaji = romaji.replaceRange(0, 1, '');
       }
     }
@@ -574,37 +619,83 @@ String transliterateLatinToHiragana(String latin) {
   return kana;
 }
 
+/// Returns a list of pairs of indices into the input and output strings,
+/// indicating which characters in the input string correspond to which characters in the output string.
+List<(int, int)> transliterateLatinToHiraganaSpan(String latin) {
+  String romaji = latin
+      .toLowerCase()
+      .replaceAll('mb', 'nb')
+      .replaceAll('mp', 'np');
+  String kana = '';
+  final List<(int, int)> spans = [];
+  int latinIndex = 0;
+
+  while (romaji.isNotEmpty) {
+    final lengths = [if (romaji.length > 2) 3, if (romaji.length > 1) 2, 1];
+
+    for (final length in lengths) {
+      String? mora;
+      int forRemoval = length;
+      final String forConversion = romaji.substring(0, length);
+
+      if (_doubleNFollowedByAIUEO(forConversion)) {
+        mora = hiraganaSyllabicN;
+        forRemoval = 1;
+      } else if (_hasTableMatch(forConversion)) {
+        mora = latinToHiragana[forConversion];
+      } else if (_hasDoubleConsonant(forConversion, length)) {
+        mora = hiraganaSmallTsu;
+        forRemoval = 1;
+      }
+
+      if (mora != null) {
+        spans.add((latinIndex, kana.length));
+        kana += mora;
+        romaji = romaji.replaceRange(0, forRemoval, '');
+        latinIndex += forRemoval;
+        break;
+      } else if (length == 1) {
+        spans.add((latinIndex, kana.length));
+        kana += forConversion;
+        romaji = romaji.replaceRange(0, 1, '');
+        latinIndex += 1;
+      }
+    }
+  }
+
+  return spans;
+}
+
 String _transposeCodepointsInRange(
   String text,
   int distance,
   int rangeStart,
   int rangeEnd,
-) =>
-    String.fromCharCodes(
-      text.codeUnits
-          .map((c) => c + ((rangeStart <= c && c <= rangeEnd) ? distance : 0)),
-    );
+) => String.fromCharCodes(
+  text.codeUnits.map(
+    (c) => c + ((rangeStart <= c && c <= rangeEnd) ? distance : 0),
+  ),
+);
 
+/// Transliterates a string of kana characters (hiragana or katakana) to Latin script (romaji).
 String transliterateKanaToLatin(String kana) =>
     transliterateHiraganaToLatin(transliterateKatakanaToHiragana(kana));
 
+/// Transliterates a string of Latin script (romaji) to katakana characters.
 String transliterateLatinToKatakana(String latin) =>
     transliterateHiraganaToKatakana(transliterateLatinToHiragana(latin));
 
+/// Transliterates a string of katakana characters to hiragana characters.
 String transliterateKatakanaToHiragana(String katakana) =>
     _transposeCodepointsInRange(katakana, -96, 12449, 12534);
 
+/// Transliterates a string of hiragana characters to katakana characters.
 String transliterateHiraganaToKatakana(String hiragana) =>
     _transposeCodepointsInRange(hiragana, 96, 12353, 12438);
 
 String transliterateFullwidthRomajiToHalfwidth(String halfwidth) =>
     _transposeCodepointsInRange(
-      _transposeCodepointsInRange(
-        halfwidth,
-        -65248,
-        65281,
-        65374,
-      ),
+      _transposeCodepointsInRange(halfwidth, -65248, 65281, 65374),
       -12256,
       12288,
       12288,
@@ -612,12 +703,7 @@ String transliterateFullwidthRomajiToHalfwidth(String halfwidth) =>
 
 String transliterateHalfwidthRomajiToFullwidth(String halfwidth) =>
     _transposeCodepointsInRange(
-      _transposeCodepointsInRange(
-        halfwidth,
-        65248,
-        33,
-        126,
-      ),
+      _transposeCodepointsInRange(halfwidth, 65248, 33, 126),
       12256,
       32,
       32,

lib/util/sqlite_utils.dart

@@ -1,3 +1,3 @@
 String escapeStringValue(String value) {
-  return "'" + value.replaceAll("'", "''") + "'";
+  return "'${value.replaceAll("'", "''")}'";
 }

lib/util/text_filtering.dart

@@ -4,7 +4,7 @@
 /// See https://www.regular-expressions.info/unicode.html
 ///
 /// Remember to turn on the unicode flag when making a new RegExp.
-const String rawKanjiRegex = r'\p{Script=Hani}';
+const String rawCJKRegex = r'\p{Script=Hani}';
 
 /// The string version of a regex that will match any katakana.
 /// This includes the ranges (), ()
@@ -22,7 +22,24 @@ const String rawKatakanaRegex = r'\p{Script=Katakana}';
 /// Remember to turn on the unicode flag when making a new RegExp.
 const String rawHiraganaRegex = r'\p{Script=Hiragana}';
 
+/// The string version of a regex that will match any kanji.
+/// This includes the ranges (), ()
+///
+/// See https://www.regular-expressions.info/unicode.html
+///
+/// Remember to turn on the unicode flag when making a new RegExp.
+const String rawKanjiRegex = r'[\u3400-\u4DB5\u4E00-\u9FCB\uF900-\uFA6A]';
 
-final RegExp kanjiRegex = RegExp(rawKanjiRegex, unicode: true);
+final RegExp cjkRegex = RegExp(rawCJKRegex, unicode: true);
 final RegExp katakanaRegex = RegExp(rawKatakanaRegex, unicode: true);
 final RegExp hiraganaRegex = RegExp(rawHiraganaRegex, unicode: true);
+final RegExp kanjiRegex = RegExp(rawKanjiRegex, unicode: true);
+
+List<String> filterKanjiSuggestions(String string) {
+  return kanjiRegex
+      .allMatches(string)
+      .map((match) => match.group(0))
+      .where((element) => element != null)
+      .map((element) => element!)
+      .toList();
+}

migrations/0001_JMDict.sql

@@ -1,3 +1,18 @@
+SELECT icu_load_collation('ja_JP', 'japanese');
+
+CREATE TABLE "JMdict_Version" (
+  "version" VARCHAR(10) PRIMARY KEY NOT NULL,
+  "date" DATE NOT NULL,
+  "hash" VARCHAR(64) NOT NULL
+) WITHOUT ROWID;
+
+CREATE TRIGGER "JMdict_Version_SingleRow"
+BEFORE INSERT ON "JMdict_Version"
+WHEN (SELECT COUNT(*) FROM "JMdict_Version") >= 1
+BEGIN
+    SELECT RAISE(FAIL, 'Only one row allowed in JMdict_Version');
+END;
+
 CREATE TABLE "JMdict_InfoDialect" (
   "id" VARCHAR(4) PRIMARY KEY NOT NULL,
   "description" TEXT NOT NULL
@@ -33,95 +48,92 @@ CREATE TABLE "JMdict_InfoReading" (
 -- not implement a check for it.
 
 CREATE TABLE "JMdict_Entry" (
-  "id" INTEGER PRIMARY KEY
+  "entryId" INTEGER PRIMARY KEY
 );
 
 -- KanjiElement
 
 CREATE TABLE "JMdict_KanjiElement" (
-  "entryId" INTEGER NOT NULL REFERENCES "JMdict_Entry"("id"),
-  "orderNum" INTEGER,
-  "reading" TEXT NOT NULL,
+  "elementId" INTEGER PRIMARY KEY,
+  "entryId" INTEGER NOT NULL REFERENCES "JMdict_Entry"("entryId"),
+  "orderNum" INTEGER NOT NULL,
+  "reading" TEXT NOT NULL COLLATE japanese,
   "news" INTEGER CHECK ("news" BETWEEN 1 AND 2),
   "ichi" INTEGER CHECK ("ichi" BETWEEN 1 AND 2),
   "spec" INTEGER CHECK ("spec" BETWEEN 1 AND 2),
   "gai" INTEGER CHECK ("gai" BETWEEN 1 AND 2),
   "nf" INTEGER CHECK ("nf" BETWEEN 1 AND 48),
-  PRIMARY KEY ("entryId", "reading")
+  -- UNIQUE("entryId", "reading"),
+  UNIQUE("entryId", "orderNum")
 ) WITHOUT ROWID;
 
+CREATE INDEX "JMdict_KanjiElement_byEntryId_byOrderNum" ON "JMdict_KanjiElement"("entryId", "orderNum");
 CREATE INDEX "JMdict_KanjiElement_byReading" ON "JMdict_KanjiElement"("reading");
 
 CREATE TABLE "JMdict_KanjiElementInfo" (
-  "entryId" INTEGER NOT NULL,
-  "reading" TEXT NOT NULL,
+  "elementId" INTEGER NOT NULL REFERENCES "JMdict_KanjiElement"("elementId"),
   "info" TEXT NOT NULL REFERENCES "JMdict_InfoKanji"("id"),
-  FOREIGN KEY ("entryId", "reading")
-    REFERENCES "JMdict_KanjiElement"("entryId", "reading"),
-  PRIMARY KEY ("entryId", "reading", "info")
+  PRIMARY KEY ("elementId", "info")
 ) WITHOUT ROWID;
 
 -- ReadingElement
 
 CREATE TABLE "JMdict_ReadingElement" (
-  "entryId" INTEGER NOT NULL REFERENCES "JMdict_Entry"("id"),
-  "orderNum" INTEGER,
-  "reading" TEXT NOT NULL,
+  "elementId" INTEGER PRIMARY KEY,
+  "entryId" INTEGER NOT NULL REFERENCES "JMdict_Entry"("entryId"),
+  "orderNum" INTEGER NOT NULL,
+  "reading" TEXT NOT NULL COLLATE japanese,
   "readingDoesNotMatchKanji" BOOLEAN NOT NULL DEFAULT FALSE,
   "news" INTEGER CHECK ("news" BETWEEN 1 AND 2),
   "ichi" INTEGER CHECK ("ichi" BETWEEN 1 AND 2),
   "spec" INTEGER CHECK ("spec" BETWEEN 1 AND 2),
   "gai" INTEGER CHECK ("gai" BETWEEN 1 AND 2),
   "nf" INTEGER CHECK ("nf" BETWEEN 1 AND 48),
-  PRIMARY KEY ("entryId", "reading")
+  -- UNIQUE("entryId", "reading"),
+  UNIQUE("entryId", "orderNum")
 ) WITHOUT ROWID;
 
+CREATE INDEX "JMdict_ReadingElement_byEntryId_byOrderNum" ON "JMdict_ReadingElement"("entryId", "orderNum");
 CREATE INDEX "JMdict_ReadingElement_byReading" ON "JMdict_ReadingElement"("reading");
 
 CREATE TABLE "JMdict_ReadingElementRestriction" (
-  "entryId" INTEGER NOT NULL,
-  "reading" TEXT NOT NULL,
+  "elementId" INTEGER NOT NULL REFERENCES "JMdict_ReadingElement"("elementId"),
   "restriction" TEXT NOT NULL,
-  FOREIGN KEY ("entryId", "reading")
-    REFERENCES "JMdict_ReadingElement"("entryId", "reading"),
-  PRIMARY KEY ("entryId", "reading", "restriction")
+  PRIMARY KEY ("elementId", "restriction")
 ) WITHOUT ROWID;
 
 CREATE TABLE "JMdict_ReadingElementInfo" (
-  "entryId" INTEGER NOT NULL,
-  "reading" TEXT NOT NULL,
+  "elementId" INTEGER NOT NULL REFERENCES "JMdict_ReadingElement"("elementId"),
   "info" TEXT NOT NULL REFERENCES "JMdict_InfoReading"("id"),
-  FOREIGN KEY ("entryId", "reading")
-    REFERENCES "JMdict_ReadingElement"("entryId", "reading"),
-  PRIMARY KEY ("entryId", "reading", "info")
+  PRIMARY KEY ("elementId", "info")
 ) WITHOUT ROWID;
 
 -- Sense
 
 CREATE TABLE "JMdict_Sense" (
-  "id" INTEGER PRIMARY KEY AUTOINCREMENT,
-  "entryId" INTEGER REFERENCES "JMdict_Entry"("id"),
-  "orderNum" INTEGER,
+  "senseId" INTEGER PRIMARY KEY,
+  "entryId" INTEGER NOT NULL REFERENCES "JMdict_Entry"("entryId"),
+  "orderNum" INTEGER NOT NULL,
   UNIQUE("entryId", "orderNum")
 );
 
 CREATE INDEX "JMdict_Sense_byEntryId_byOrderNum" ON "JMdict_Sense"("entryId", "orderNum");
 
 CREATE TABLE "JMdict_SenseRestrictedToKanji" (
-  "entryId" INTEGER,
-  "senseId" INTEGER REFERENCES "JMdict_Sense"("id"),
-  "kanji" TEXT,
-  FOREIGN KEY ("entryId", "kanji") REFERENCES "JMdict_KanjiElement"("entryId", "reading"),
-  PRIMARY KEY ("entryId", "senseId", "kanji")
-);
+  "entryId" INTEGER NOT NULL,
+  "senseId" INTEGER NOT NULL REFERENCES "JMdict_Sense"("senseId"),
+  "kanjiOrderNum" INTEGER NOT NULL CHECK ("kanjiOrderNum" > 0),
+  FOREIGN KEY ("entryId", "kanjiOrderNum") REFERENCES "JMdict_KanjiElement"("entryId", "orderNum"),
+  PRIMARY KEY ("entryId", "senseId", "kanjiOrderNum")
+) WITHOUT ROWID;
 
 CREATE TABLE "JMdict_SenseRestrictedToReading" (
-  "entryId" INTEGER,
-  "senseId" INTEGER REFERENCES "JMdict_Sense"("id"),
-  "reading" TEXT,
-  FOREIGN KEY ("entryId", "reading") REFERENCES "JMdict_ReadingElement"("entryId", "reading"),
-  PRIMARY KEY ("entryId", "senseId", "reading")
-);
+  "entryId" INTEGER NOT NULL,
+  "senseId" INTEGER NOT NULL REFERENCES "JMdict_Sense"("senseId"),
+  "readingOrderNum" INTEGER NOT NULL CHECK ("readingOrderNum" > 0),
+  FOREIGN KEY ("entryId", "readingOrderNum") REFERENCES "JMdict_ReadingElement"("entryId", "orderNum"),
+  PRIMARY KEY ("entryId", "senseId", "readingOrderNum")
+) WITHOUT ROWID;
 
 -- In order to add xrefs, you will need to have added the entry to xref to.
 -- These should be added in a second pass of the dictionary file.
@@ -134,37 +146,24 @@ CREATE TABLE "JMdict_SenseRestrictedToReading" (
 -- These two things also concern "SenseAntonym"
 
 CREATE TABLE "JMdict_SenseSeeAlso" (
-  "senseId" INTEGER REFERENCES "JMdict_Sense"("id"),
-  "xrefEntryId" INTEGER,
-  "seeAlsoReading" TEXT,
-  "seeAlsoKanji" TEXT,
-  "seeAlsoSense" INTEGER,
+  "senseId" INTEGER NOT NULL REFERENCES "JMdict_Sense"("senseId"),
+  "xrefEntryId" INTEGER NOT NULL REFERENCES "JMdict_Entry"("entryId"),
+  -- Sometimes the cross reference is to a specific sense
+  "xrefSenseOrderNum" INTEGER,
   -- For some entries, the cross reference is ambiguous. This means that while the ingestion
   -- has determined some xrefEntryId, it is not guaranteed to be the correct one.
-  "ambiguous" BOOLEAN,
-  CHECK ("seeAlsoReading" = NULL <> "seeAlsoKanji" = NULL),
-  -- CHECK("seeAlsoSense" = NULL OR "seeAlsoSense")
-  --  TODO: Check that if seeAlsoSense is present, it refers to a sense connected to xrefEntryId.
-  FOREIGN KEY ("xrefEntryId", "seeAlsoKanji") REFERENCES "JMdict_KanjiElement"("entryId", "reading"),
-  FOREIGN KEY ("xrefEntryId", "seeAlsoReading") REFERENCES "JMdict_ReadingElement"("entryId", "reading"),
-  FOREIGN KEY ("xrefEntryId", "seeAlsoSense") REFERENCES "JMdict_Sense"("entryId", "orderNum"),
-  PRIMARY KEY ("senseId", "xrefEntryId", "seeAlsoReading", "seeAlsoKanji", "seeAlsoSense")
+  "ambiguous" BOOLEAN NOT NULL DEFAULT FALSE,
+  FOREIGN KEY ("xrefEntryId", "xrefSenseOrderNum") REFERENCES "JMdict_Sense"("entryId", "orderNum"),
+  UNIQUE("senseId", "xrefEntryId", "xrefSenseOrderNum")
 );
 
 CREATE TABLE "JMdict_SenseAntonym" (
-  "senseId" INTEGER REFERENCES "JMdict_Sense"("id"),
-  "xrefEntryId" INTEGER,
-  "antonymReading" TEXT,
-  "antonymKanji" TEXT,
-  "antonymSense" INTEGER,
+  "senseId" INTEGER NOT NULL REFERENCES "JMdict_Sense"("senseId"),
+  "xrefEntryId" INTEGER NOT NULL REFERENCES "JMdict_Entry"("entryId"),
   -- For some entries, the cross reference is ambiguous. This means that while the ingestion
   -- has determined some xrefEntryId, it is not guaranteed to be the correct one.
-  "ambiguous" BOOLEAN,
-  CHECK ("antonymReading" = NULL <> "antonymKanji" = NULL),
-  FOREIGN KEY ("xrefEntryId", "antonymKanji") REFERENCES "JMdict_KanjiElement"("entryId", "reading"),
-  FOREIGN KEY ("xrefEntryId", "antonymReading") REFERENCES "JMdict_ReadingElement"("entryId", "reading"),
-  FOREIGN KEY ("xrefEntryId", "antonymSense") REFERENCES "JMdict_Sense"("entryId", "orderNum"),
-  PRIMARY KEY ("senseId", "xrefEntryId", "antonymReading", "antonymKanji", "antonymSense")
+  "ambiguous" BOOLEAN NOT NULL DEFAULT FALSE,
+  UNIQUE("senseId", "xrefEntryId")
 );
 
 -- These cross references are going to be mostly accessed from a sense
@@ -173,7 +172,7 @@ CREATE INDEX "JMdict_SenseSeeAlso_bySenseId" ON "JMdict_SenseSeeAlso"("senseId")
 CREATE INDEX "JMdict_SenseAntonym_bySenseId" ON "JMdict_SenseAntonym"("senseId");
 
 CREATE TABLE "JMdict_SensePOS" (
-  "senseId" INTEGER NOT NULL REFERENCES "JMdict_Sense"("id"),
+  "senseId" INTEGER NOT NULL REFERENCES "JMdict_Sense"("senseId"),
   "pos" TEXT NOT NULL REFERENCES "JMdict_InfoPOS"("id"),
   PRIMARY KEY ("senseId", "pos")
 ) WITHOUT ROWID;
@@ -181,28 +180,28 @@ CREATE TABLE "JMdict_SensePOS" (
 CREATE TABLE "JMdict_SenseField" (
   "senseId" INTEGER NOT NULL,
   "field" TEXT NOT NULL,
-  FOREIGN KEY ("senseId") REFERENCES "JMdict_Sense"("id"),
+  FOREIGN KEY ("senseId") REFERENCES "JMdict_Sense"("senseId"),
   FOREIGN KEY ("field") REFERENCES "JMdict_InfoField"("id"),
   PRIMARY KEY ("senseId", "field")
 ) WITHOUT ROWID;
 
 CREATE TABLE "JMdict_SenseMisc" (
-  "senseId" INTEGER NOT NULL REFERENCES "JMdict_Sense"("id"),
+  "senseId" INTEGER NOT NULL REFERENCES "JMdict_Sense"("senseId"),
   "misc" TEXT NOT NULL REFERENCES "JMdict_InfoMisc"("id"),
   PRIMARY KEY ("senseId", "misc")
 ) WITHOUT ROWID;
 
 CREATE TABLE "JMdict_SenseLanguageSource" (
-  "senseId" INTEGER NOT NULL REFERENCES "JMdict_Sense"("id"),
+  "senseId" INTEGER NOT NULL REFERENCES "JMdict_Sense"("senseId"),
   "language" CHAR(3) NOT NULL DEFAULT "eng",
   "phrase" TEXT,
   "fullyDescribesSense" BOOLEAN NOT NULL DEFAULT TRUE,
   "constructedFromSmallerWords" BOOLEAN NOT NULL DEFAULT FALSE,
-  PRIMARY KEY ("senseId", "language", "phrase")
+  UNIQUE("senseId", "language", "phrase")
 );
 
 CREATE TABLE "JMdict_SenseDialect" (
-  "senseId" INTEGER NOT NULL REFERENCES "JMdict_Sense"("id"),
+  "senseId" INTEGER NOT NULL REFERENCES "JMdict_Sense"("senseId"),
   "dialect" TEXT NOT NULL REFERENCES "JMdict_InfoDialect"("id"),
   PRIMARY KEY ("senseId", "dialect")
 ) WITHOUT ROWID;
@@ -213,7 +212,7 @@ CREATE TABLE "JMdict_SenseDialect" (
 -- will be omitted.
 
 CREATE TABLE "JMdict_SenseGlossary" (
-  "senseId" INTEGER NOT NULL REFERENCES "JMdict_Sense"("id"),
+  "senseId" INTEGER NOT NULL REFERENCES "JMdict_Sense"("senseId"),
   "phrase" TEXT NOT NULL,
   "language" CHAR(3) NOT NULL DEFAULT "eng",
   "type" TEXT,
@@ -223,7 +222,7 @@ CREATE TABLE "JMdict_SenseGlossary" (
 CREATE INDEX "JMdict_SenseGlossary_byPhrase" ON JMdict_SenseGlossary("phrase");
 
 CREATE TABLE "JMdict_SenseInfo" (
-  "senseId" INTEGER NOT NULL REFERENCES "JMdict_Sense"("id"),
+  "senseId" INTEGER NOT NULL REFERENCES "JMdict_Sense"("senseId"),
   "info" TEXT NOT NULL,
   PRIMARY KEY ("senseId", "info")
 ) WITHOUT ROWID;
@@ -232,8 +231,7 @@ CREATE TABLE "JMdict_SenseInfo" (
 -- the Tanaka Corpus, so I will leave the type out for now.
 
 CREATE TABLE "JMdict_ExampleSentence" (
-  "id" INTEGER PRIMARY KEY,
-  "senseId" INTEGER REFERENCES "JMdict_Sense"("id"),
+  "senseId" INTEGER NOT NULL REFERENCES "JMdict_Sense"("senseId"),
   "word" TEXT NOT NULL,
   "source" TEXT NOT NULL,
   "sourceLanguage" CHAR(3) NOT NULL DEFAULT "eng",

migrations/0003_JMDict_FTS5_Ttables.sql

@@ -0,0 +1,55 @@
+CREATE VIRTUAL TABLE "JMdict_KanjiElementFTS" USING FTS5("elementId" UNINDEXED, "reading");
+
+CREATE TRIGGER "JMdict_KanjiElement_InsertFTS"
+AFTER INSERT ON "JMdict_KanjiElement"
+BEGIN
+  INSERT INTO "JMdict_KanjiElementFTS"("elementId", "reading")
+  VALUES (NEW."elementId", NEW."reading");
+END;
+
+CREATE TRIGGER "JMdict_KanjiElement_UpdateFTS"
+AFTER UPDATE OF "elementId", "reading"
+ON "JMdict_KanjiElement"
+BEGIN
+  UPDATE "JMdict_KanjiElementFTS"
+  SET
+    "elementId" = NEW."elementId",
+    "reading" = NEW."reading"
+  WHERE "elementId" = OLD."elementId";
+END;
+
+CREATE TRIGGER "JMdict_KanjiElement_DeleteFTS"
+AFTER DELETE ON "JMdict_KanjiElement"
+BEGIN
+  DELETE FROM "JMdict_KanjiElementFTS"
+  WHERE "elementId" = OLD."elementId";
+END;
+
+
+
+CREATE VIRTUAL TABLE "JMdict_ReadingElementFTS" USING FTS5("elementId" UNINDEXED, "reading");
+
+CREATE TRIGGER "JMdict_ReadingElement_InsertFTS"
+AFTER INSERT ON "JMdict_ReadingElement"
+BEGIN
+  INSERT INTO "JMdict_ReadingElementFTS"("elementId", "reading")
+  VALUES (NEW."elementId", NEW."reading");
+END;
+
+CREATE TRIGGER "JMdict_ReadingElement_UpdateFTS"
+AFTER UPDATE OF "elementId", "reading"
+ON "JMdict_ReadingElement"
+BEGIN
+  UPDATE "JMdict_ReadingElementFTS"
+  SET
+    "elementId" = NEW."elementId",
+    "reading" = NEW."reading"
+  WHERE "elementId" = OLD."elementId";
+END;
+
+CREATE TRIGGER "JMdict_ReadingElement_DeleteFTS"
+AFTER DELETE ON "JMdict_ReadingElement"
+BEGIN
+  DELETE FROM "JMdict_ReadingElementFTS"
+  WHERE "elementId" = OLD."elementId";
+END;

migrations/0003_RADKFILE.sql

@@ -1,11 +0,0 @@
-CREATE TABLE "RADKFILE" (
-  "kanji" CHAR(1) NOT NULL,
-  "radical" CHAR(1) NOT NULL,
-  PRIMARY KEY ("kanji", "radical")
-) WITHOUT ROWID;
-
-CREATE INDEX "RADK" ON "RADKFILE"("radical");
-CREATE INDEX "KRAD" ON "RADKFILE"("kanji");
-
-CREATE VIEW "RADKFILE_Radicals" AS
-SELECT DISTINCT "radical" FROM "RADKFILE";

migrations/0004_JMDict_tanos_jlpt_tags.sql

@@ -0,0 +1,23 @@
+CREATE TABLE "JMdict_JLPT_Version" (
+  "version" VARCHAR(10) PRIMARY KEY NOT NULL,
+  "date" DATE NOT NULL,
+  "hash" VARCHAR(64) NOT NULL
+) WITHOUT ROWID;
+
+CREATE TRIGGER "JMdict_JLPT_Version_SingleRow"
+BEFORE INSERT ON "JMdict_JLPT_Version"
+WHEN (SELECT COUNT(*) FROM "JMdict_JLPT_Version") >= 1
+BEGIN
+    SELECT RAISE(FAIL, 'Only one row allowed in JMdict_JLPT_Version');
+END;
+
+CREATE TABLE "JMdict_JLPTTag" (
+  "entryId" INTEGER NOT NULL,
+  "jlptLevel" CHAR(2) NOT NULL CHECK ("jlptLevel" in ('N5', 'N4', 'N3', 'N2', 'N1')),
+  FOREIGN KEY ("entryId")
+    REFERENCES "JMdict_Entry"("entryId"),
+  PRIMARY KEY ("entryId", "jlptLevel")
+) WITHOUT ROWID;
+
+CREATE INDEX "JMdict_JLPTTag_byEntryId" ON "JMdict_JLPTTag"("entryId");
+CREATE INDEX "JMdict_JLPTTag_byJlptLevel" ON "JMdict_JLPTTag"("jlptLevel");

migrations/0005_JMDict_search_index_tables.sql

@@ -0,0 +1,207 @@
+CREATE TABLE "JMdict_EntryScore" (
+  "type" CHAR(1) NOT NULL CHECK ("type" IN ('r', 'k')),
+  "entryId" INTEGER NOT NULL REFERENCES "JMdict_Entry"("entryId"),
+  "elementId" INTEGER NOT NULL,
+  "score" INTEGER NOT NULL DEFAULT 0,
+  "common" BOOLEAN NOT NULL DEFAULT FALSE,
+  PRIMARY KEY ("type", "elementId")
+) WITHOUT ROWID;
+
+CREATE INDEX "JMdict_EntryScore_byElementId_byScore" ON "JMdict_EntryScore"("elementId", "score");
+CREATE INDEX "JMdict_EntryScore_byScore" ON "JMdict_EntryScore"("score");
+CREATE INDEX "JMdict_EntryScore_byCommon" ON "JMdict_EntryScore"("common");
+
+CREATE INDEX "JMdict_EntryScore_byType_byElementId_byScore" ON "JMdict_EntryScore"("type", "elementId", "score");
+CREATE INDEX "JMdict_EntryScore_byType_byScore" ON "JMdict_EntryScore"("type", "score");
+CREATE INDEX "JMdict_EntryScore_byType_byCommon" ON "JMdict_EntryScore"("type", "common");
+
+-- NOTE: these views are deduplicated in order not to perform an unnecessary
+--       UNION on every trigger
+
+CREATE VIEW "JMdict_EntryScoreView_Reading" AS
+SELECT
+  'r' AS "type",
+  "JMdict_ReadingElement"."entryId",
+  "JMdict_ReadingElement"."elementId",
+  (
+    "news" IS 1
+    OR "ichi" IS 1
+    OR "spec" IS 1
+    OR "gai" IS 1
+  )
+  AS "common",
+  ((
+    "news" IS 1
+    OR "ichi" IS 1
+    OR "spec" IS 1
+    OR "gai" IS 1
+  ) * 50)
+  + (("news" IS 1) * 10)
+  + (("news" IS 2) * 5)
+  + (("ichi" IS 1) * 10)
+  + (("ichi" IS 2) * 5)
+  + (("spec" IS 1) * 10)
+  + (("spec" IS 2) * 5)
+  + (("gai" IS 1) * 10)
+  + (("gai" IS 2) * 5)
+  + (("orderNum" IS 1) * 20)
+  - (substr(COALESCE("JMdict_JLPTTag"."jlptLevel", 'N0'), 2) * -5)
+  AS "score"
+FROM "JMdict_ReadingElement"
+LEFT JOIN "JMdict_JLPTTag" USING ("entryId");
+
+CREATE VIEW "JMdict_EntryScoreView_Kanji" AS
+SELECT
+  'k' AS "type",
+  "JMdict_KanjiElement"."entryId",
+  "JMdict_KanjiElement"."elementId",
+  (
+    "news" IS 1
+    OR "ichi" IS 1
+    OR "spec" IS 1
+    OR "gai" IS 1
+  )
+  AS "common",
+  ((
+    "news" IS 1
+    OR "ichi" IS 1
+    OR "spec" IS 1
+    OR "gai" IS 1
+  ) * 50)
+  + (("news" IS 1) * 10)
+  + (("news" IS 2) * 5)
+  + (("ichi" IS 1) * 10)
+  + (("ichi" IS 2) * 5)
+  + (("spec" IS 1) * 10)
+  + (("spec" IS 2) * 5)
+  + (("gai" IS 1) * 10)
+  + (("gai" IS 2) * 5)
+  + (("orderNum" IS 1) * 20)
+  - (substr(COALESCE("JMdict_JLPTTag"."jlptLevel", 'N0'), 2) * -5)
+  AS "score"
+FROM "JMdict_KanjiElement"
+LEFT JOIN "JMdict_JLPTTag" USING ("entryId");
+
+CREATE VIEW "JMdict_EntryScoreView" AS
+SELECT *
+FROM "JMdict_EntryScoreView_Kanji"
+UNION ALL
+SELECT *
+FROM "JMdict_EntryScoreView_Reading";
+
+
+--- JMdict_ReadingElement triggers
+
+CREATE TRIGGER "JMdict_EntryScore_Insert_JMdict_ReadingElement"
+AFTER INSERT ON "JMdict_ReadingElement"
+BEGIN
+  INSERT INTO "JMdict_EntryScore" (
+    "type",
+    "entryId",
+    "elementId",
+    "score",
+    "common"
+  )
+  SELECT "type", "entryId", "elementId", "score", "common"
+  FROM "JMdict_EntryScoreView_Reading"
+  WHERE "elementId" = NEW."elementId";
+END;
+
+CREATE TRIGGER "JMdict_EntryScore_Update_JMdict_ReadingElement"
+AFTER UPDATE OF "news", "ichi", "spec", "gai", "nf", "orderNum"
+ON "JMdict_ReadingElement"
+BEGIN
+  UPDATE "JMdict_EntryScore"
+  SET
+    "score" = "JMdict_EntryScoreView_Reading"."score",
+    "common" = "JMdict_EntryScoreView_Reading"."common"
+  FROM "JMdict_EntryScoreView_Reading"
+  WHERE "elementId" = NEW."elementId";
+END;
+
+CREATE TRIGGER "JMdict_EntryScore_Delete_JMdict_ReadingElement"
+AFTER DELETE ON "JMdict_ReadingElement"
+BEGIN
+  DELETE FROM "JMdict_EntryScore"
+  WHERE "type" = 'r'
+    AND "elementId" = OLD."elementId";
+END;
+
+--- JMdict_KanjiElement triggers
+
+CREATE TRIGGER "JMdict_EntryScore_Insert_JMdict_KanjiElement"
+AFTER INSERT ON "JMdict_KanjiElement"
+BEGIN
+  INSERT INTO "JMdict_EntryScore" (
+    "type",
+    "entryId",
+    "elementId",
+    "score",
+    "common"
+  )
+  SELECT "type", "entryId", "elementId", "score", "common"
+  FROM "JMdict_EntryScoreView_Kanji"
+  WHERE "elementId" = NEW."elementId";
+END;
+
+CREATE TRIGGER "JMdict_EntryScore_Update_JMdict_KanjiElement"
+AFTER UPDATE OF "news", "ichi", "spec", "gai", "nf", "orderNum"
+ON "JMdict_KanjiElement"
+BEGIN
+  UPDATE "JMdict_EntryScore"
+  SET
+    "score" = "JMdict_EntryScoreView_Kanji"."score",
+    "common" = "JMdict_EntryScoreView_Kanji"."common"
+  FROM "JMdict_EntryScoreView_Kanji"
+  WHERE "elementId" = NEW."elementId";
+END;
+
+CREATE TRIGGER "JMdict_EntryScore_Delete_JMdict_KanjiElement"
+AFTER DELETE ON "JMdict_KanjiElement"
+BEGIN
+  DELETE FROM "JMdict_EntryScore"
+  WHERE "type" = 'k'
+    AND "elementId" = OLD."elementId";
+END;
+
+--- JMdict_JLPTTag triggers
+
+CREATE TRIGGER "JMdict_EntryScore_Insert_JMdict_JLPTTag"
+AFTER INSERT ON "JMdict_JLPTTag"
+BEGIN
+  UPDATE "JMdict_EntryScore"
+  SET
+    "score" = "JMdict_EntryScoreView"."score",
+    "common" = "JMdict_EntryScoreView"."common"
+  FROM "JMdict_EntryScoreView"
+  WHERE "JMdict_EntryScoreView"."entryId" = NEW."entryId"
+    AND "JMdict_EntryScore"."entryId" = NEW."entryId"
+    AND "JMdict_EntryScoreView"."elementId" = "JMdict_EntryScore"."elementId";
+END;
+
+CREATE TRIGGER "JMdict_EntryScore_Update_JMdict_JLPTTag"
+AFTER UPDATE OF "jlptLevel"
+ON "JMdict_JLPTTag"
+BEGIN
+  UPDATE "JMdict_EntryScore"
+  SET
+    "score" = "JMdict_EntryScoreView"."score",
+    "common" = "JMdict_EntryScoreView"."common"
+  FROM "JMdict_EntryScoreView"
+  WHERE "JMdict_EntryScoreView"."entryId" = NEW."entryId"
+    AND "JMdict_EntryScore"."entryId" = NEW."entryId"
+    AND "JMdict_EntryScoreView"."elementId" = "JMdict_EntryScore"."elementId";
+END;
+
+CREATE TRIGGER "JMdict_EntryScore_Delete_JMdict_JLPTTag"
+AFTER DELETE ON "JMdict_JLPTTag"
+BEGIN
+  UPDATE "JMdict_EntryScore"
+  SET
+    "score" = "JMdict_EntryScoreView"."score",
+    "common" = "JMdict_EntryScoreView"."common"
+  FROM "JMdict_EntryScoreView"
+  WHERE "JMdict_EntryScoreView"."entryId" = OLD."entryId"
+    AND "JMdict_EntryScore"."entryId" = OLD."entryId"
+    AND "JMdict_EntryScoreView"."elementId" = "JMdict_EntryScore"."elementId";
+END;

migrations/0006_RADKFILE.sql

@@ -0,0 +1,21 @@
+CREATE TABLE "RADKFILE_Version" (
+  "version" VARCHAR(10) PRIMARY KEY NOT NULL,
+  "date" DATE NOT NULL,
+  "hash" VARCHAR(64) NOT NULL
+) WITHOUT ROWID;
+
+CREATE TRIGGER "RADKFILE_Version_SingleRow"
+BEFORE INSERT ON "RADKFILE_Version"
+WHEN (SELECT COUNT(*) FROM "RADKFILE_Version") >= 1
+BEGIN
+    SELECT RAISE(FAIL, 'Only one row allowed in RADKFILE_Version');
+END;
+
+CREATE TABLE "RADKFILE" (
+  "kanji" CHAR(1) NOT NULL,
+  "radical" CHAR(1) NOT NULL,
+  PRIMARY KEY ("kanji", "radical")
+) WITHOUT ROWID;
+
+CREATE INDEX "RADK" ON "RADKFILE"("radical");
+CREATE INDEX "KRAD" ON "RADKFILE"("kanji");

migrations/0007_JMdict_Entry_lookup_tables.sql

@@ -1,22 +0,0 @@
--- These tables are for optimizing searches.
-
--- In order to include results from both, the software should
--- first check if the searchword is convertible to kana, and then
--- potentially get results from both by doing a union between two
--- selects.
-
-CREATE TABLE "JMdict_EntryByKana" (
-  "kana" TEXT NOT NULL,
-  "entryId" INTEGER NOT NULL REFERENCES "JMdict_Entry"("id"),
-  PRIMARY KEY ("kana", "entryId")
-) WITHOUT ROWID;
-
-CREATE INDEX "JMdict_EntryByKana_byKana" ON "JMdict_EntryByKana"("kana");
-
-CREATE TABLE "JMdict_EntryByEnglish" (
-  "english" TEXT NOT NULL,
-  "entryId" INTEGER NOT NULL REFERENCES "JMdict_Entry"("id"),
-  PRIMARY KEY ("english", "entryId")
-) WITHOUT ROWID;
-
-CREATE INDEX "JMdict_EntryByEnglish_byEnglish" ON "JMdict_EntryByEnglish"("english");

migrations/0007_KANJIDIC2.sql

@@ -1,3 +1,16 @@
+CREATE TABLE "KANJIDIC_Version" (
+  "version" VARCHAR(10) PRIMARY KEY NOT NULL,
+  "date" DATE NOT NULL,
+  "hash" VARCHAR(64) NOT NULL
+) WITHOUT ROWID;
+
+CREATE TRIGGER "KANJIDIC_Version_SingleRow"
+BEFORE INSERT ON "KANJIDIC_Version"
+WHEN (SELECT COUNT(*) FROM "KANJIDIC_Version") >= 1
+BEGIN
+    SELECT RAISE(FAIL, 'Only one row allowed in KANJIDIC_Version');
+END;
+
 CREATE TABLE "KANJIDIC_Character" (
   "literal" CHAR(1) NOT NULL PRIMARY KEY,
   "grade" INTEGER CHECK ("grade" BETWEEN 1 AND 10),
@@ -6,12 +19,6 @@ CREATE TABLE "KANJIDIC_Character" (
   "jlpt" INTEGER
 ) WITHOUT ROWID;
 
-CREATE TABLE "KANJIDIC_RadicalName" (
-  "kanji" CHAR(1) NOT NULL REFERENCES "KANJIDIC_Character"("literal"),
-  "name" TEXT NOT NULL,
-  PRIMARY KEY("kanji", "name")
-) WITHOUT ROWID;
-
 CREATE TABLE "KANJIDIC_Codepoint" (
   "kanji" CHAR(1) NOT NULL REFERENCES "KANJIDIC_Character"("literal"),
   "type" VARCHAR(6) NOT NULL CHECK ("type" IN ('jis208', 'jis212', 'jis213', 'ucs')),
@@ -22,12 +29,25 @@ CREATE TABLE "KANJIDIC_Codepoint" (
 CREATE INDEX "KANJIDIC_Codepoint_byCharacter" ON "KANJIDIC_Codepoint"("kanji");
 
 CREATE TABLE "KANJIDIC_Radical" (
-  "kanji" CHAR(1) NOT NULL REFERENCES "KANJIDIC_Character"("literal"),
-  "type" VARCHAR(9) NOT NULL CHECK ("type" IN ('classical', 'nelson_c')),
-  "radical" INTEGER NOT NULL CHECK ("radical" BETWEEN 1 AND IIF("type" = 'classical', 214, 213)),
-  PRIMARY KEY("kanji", "type")
+  "kanji" CHAR(1) NOT NULL PRIMARY KEY REFERENCES "KANJIDIC_Character"("literal"),
+  "radicalId" INTEGER NOT NULL CHECK ("radicalId" BETWEEN 1 AND 214)
 ) WITHOUT ROWID;
 
+CREATE INDEX "KANJIDIC_Radical_byRadicalId" ON "KANJIDIC_Radical"("radicalId");
+
+CREATE TABLE "KANJIDIC_RadicalNelsonCId" (
+  "radicalId" INTEGER NOT NULL PRIMARY KEY CHECK ("radicalId" BETWEEN 1 AND 214),
+  "nelsonId" INTEGER UNIQUE NOT NULL CHECK ("nelsonId" BETWEEN 1 AND 213)
+);
+
+CREATE TABLE "KANJIDIC_RadicalName" (
+  "radicalId" INTEGER NOT NULL CHECK ("radicalId" BETWEEN 1 AND 214),
+  "name" TEXT NOT NULL,
+  PRIMARY KEY("radicalId", "name")
+) WITHOUT ROWID;
+
+CREATE INDEX "KANJIDIC_RadicalName_byRadicalId" ON "KANJIDIC_RadicalName"("radicalId");
+
 CREATE TABLE "KANJIDIC_StrokeMiscount" (
   "kanji" CHAR(1) NOT NULL REFERENCES "KANJIDIC_Character"("literal"),
   "strokeCount" INTEGER NOT NULL,
@@ -106,6 +126,7 @@ CREATE TABLE "KANJIDIC_QueryCode" (
   "code" VARCHAR(7) NOT NULL,
   "type" VARCHAR(11) NOT NULL CHECK ("type" IN ('skip', 'sh_desc', 'four_corner', 'deroo', 'misclass')),
   "SKIPMisclassification" VARCHAR(15),
+  CHECK ("SKIPMisclassification" IS NULL OR "type" = 'skip'),
   PRIMARY KEY ("kanji", "type", "code")
 ) WITHOUT ROWID;
 
@@ -120,30 +141,39 @@ CREATE INDEX "KANJIDIC_Reading_byReading" ON "KANJIDIC_Reading"("reading");
 
 CREATE TABLE "KANJIDIC_Kunyomi" (
   "kanji" CHAR(1) NOT NULL REFERENCES "KANJIDIC_Character"("literal"),
+  "orderNum" INTEGER NOT NULL,
   "yomi" TEXT NOT NULL,
   "isJouyou" BOOLEAN,
+  UNIQUE("kanji", "orderNum"),
   PRIMARY KEY ("kanji", "yomi")
 ) WITHOUT ROWID;
 
+CREATE INDEX "KANJIDIC_Kunyomi_byKanji_byOrderNum" ON "KANJIDIC_Kunyomi"("kanji", "orderNum");
 CREATE INDEX "KANJIDIC_Kunyomi_byYomi" ON "KANJIDIC_Kunyomi"("yomi");
 
 CREATE TABLE "KANJIDIC_Onyomi" (
   "kanji" CHAR(1) NOT NULL REFERENCES "KANJIDIC_Character"("literal"),
+  "orderNum" INTEGER NOT NULL,
   "yomi" TEXT NOT NULL,
   "type" VARCHAR(7) CHECK ("type" IN ('kan', 'go', 'tou', 'kan''you')),
   "isJouyou" BOOLEAN,
+  UNIQUE("kanji", "orderNum"),
   PRIMARY KEY ("kanji", "yomi")
 ) WITHOUT ROWID;
 
+CREATE INDEX "KANJIDIC_Onyomi_byKanji_byOrderNum" ON "KANJIDIC_Onyomi"("kanji", "orderNum");
 CREATE INDEX "KANJIDIC_Onyomi_byYomi" ON "KANJIDIC_Onyomi"("yomi");
 
 CREATE TABLE "KANJIDIC_Meaning" (
   "kanji" CHAR(1) NOT NULL REFERENCES "KANJIDIC_Character"("literal"),
+  "orderNum" INTEGER NOT NULL,
   "language" CHAR(3) NOT NULL DEFAULT "eng",
   "meaning" TEXT NOT NULL,
+  UNIQUE("kanji", "orderNum"),
   PRIMARY KEY ("kanji", "language", "meaning")
 ) WITHOUT ROWID;
 
+CREATE INDEX "KANJIDIC_Meaning_byKanji_byOrderNum" ON "KANJIDIC_Meaning"("kanji", "orderNum");
 CREATE INDEX "KANJIDIC_Meaning_byMeaning" ON "KANJIDIC_Meaning"("meaning");
 
 CREATE TABLE "KANJIDIC_Nanori" (

migrations/0008_TANOS_JLPT_TAGS.sql

@@ -1,10 +0,0 @@
-CREATE TABLE "JMdict_JLPTTag" (
-  "entryId" INTEGER NOT NULL,
-  "jlptLevel" CHAR(2) NOT NULL CHECK ("jlptLevel" in ('N5', 'N4', 'N3', 'N2', 'N1')),
-  FOREIGN KEY ("entryId")
-    REFERENCES "JMdict_Entry"("id"),
-  PRIMARY KEY ("entryId", "jlptLevel")
-) WITHOUT ROWID;
-
-CREATE INDEX "JMdict_JLPTTag_byEntryId" ON "JMdict_JLPTTag"("entryId");
-CREATE INDEX "JMdict_JLPTTag_byJlptLevel" ON "JMdict_JLPTTag"("jlptLevel");

migrations/0008_XREF__KANJIDIC_Radical___RADKFILE.sql

@@ -1,13 +1,11 @@
 CREATE TABLE "XREF__KANJIDIC_Radical__RADKFILE"(
-  "radicalId" INTEGER NOT NULL,
-  "radicalSymbol" CHAR(1) NOT NULL REFERENCES "RADKFILE"("radical"),
-  "radicalType" VARCHAR(9) NOT NULL CHECK ("radicalType" IN ('classical', 'nelson_c')) DEFAULT 'classical',
-  PRIMARY KEY ("radicalId", "radicalSymbol", "radicalType"),
-  FOREIGN KEY ("radicalId", "radicalType") REFERENCES "KANJIDIC_Radical"("radical", "type")
+  "radicalId" INTEGER NOT NULL CHECK ("radicalId" BETWEEN 1 AND 214),
+  "radicalSymbol" CHAR(1) UNIQUE NOT NULL REFERENCES "RADKFILE"("radical"),
+  PRIMARY KEY ("radicalId", "radicalSymbol")
 ) WITHOUT ROWID;
 
 CREATE INDEX "XREF__KANJIDIC_Radical__RADKFILE__byRadicalId" ON "XREF__KANJIDIC_Radical__RADKFILE"("radicalId");
-CREATE INDEX "XREF__KANJIDIC_Radical__RADKFILE__byRadicalSymbol_byRadicalType" ON "XREF__KANJIDIC_Radical__RADKFILE"("radicalSymbol", "radicalType");
+CREATE INDEX "XREF__KANJIDIC_Radical__RADKFILE__byRadicalSymbol" ON "XREF__KANJIDIC_Radical__RADKFILE"("radicalSymbol");
 
 /* Source: https://ctext.org/kangxi-zidian */
 INSERT INTO "XREF__KANJIDIC_Radical__RADKFILE"("radicalId", "radicalSymbol") VALUES

migrations/0010_Views.sql

@@ -0,0 +1,89 @@
+CREATE VIEW "JMdict_EntryByKana"("kana", "entryId")
+AS
+SELECT
+  "JMdict_ReadingElement"."reading" AS "kana",
+  "JMdict_ReadingElement"."entryId" AS "entryId"
+FROM "JMdict_ReadingElement";
+
+
+CREATE VIEW "JMdict_EntryByEnglish"("english", "entryId")
+AS
+SELECT
+  "JMdict_SenseGlossary"."phrase" AS "english",
+  "JMdict_Sense"."senseId" AS "entryId"
+FROM "JMdict_SenseGlossary" JOIN "JMdict_Sense" USING("senseId");
+
+
+CREATE VIEW "JMdict_BaseAndFurigana"("entryId", "base", "furigana", "isFirst", "kanjiOrderNum", "readingOrderNum")
+AS
+SELECT
+  "JMdict_Entry"."entryId" AS "entryId",
+  CASE WHEN (
+    "JMdict_KanjiElement"."reading" IS NOT NULL
+    AND NOT "JMdict_ReadingElement"."readingDoesNotMatchKanji"
+  )
+    THEN "JMdict_KanjiElement"."reading"
+    ELSE "JMdict_ReadingElement"."reading"
+  END AS "base",
+  CASE WHEN (
+    "JMdict_KanjiElement"."reading" IS NOT NULL
+    AND NOT "JMdict_ReadingElement"."readingDoesNotMatchKanji"
+  )
+    THEN "JMdict_ReadingElement"."reading"
+    ELSE NULL
+  END AS "furigana",
+  COALESCE("JMdict_KanjiElement"."orderNum", 1)
+      + "JMdict_ReadingElement"."orderNum"
+      = 2
+  AS "isFirst",
+  "JMdict_KanjiElement"."orderNum" AS "kanjiOrderNum",
+  "JMdict_ReadingElement"."orderNum" AS "readingOrderNum"
+FROM "JMdict_Entry"
+LEFT JOIN "JMdict_KanjiElement" USING("entryId")
+LEFT JOIN "JMdict_ReadingElement" USING("entryId");
+
+CREATE VIEW "JMdict_EntryCommon"("entryId")
+AS
+SELECT DISTINCT "entryId"
+FROM "JMdict_EntryScore"
+WHERE "JMdict_EntryScore"."common" = 1;
+
+-- TODO: Make it possible to match words that contain the
+--       kanji as an infix
+
+CREATE VIEW "KANJIDIC_ExampleEntries"("kanji", "entryId")
+AS
+SELECT
+  "JMdict_KanjiElement"."entryId",
+  "KANJIDIC_Character"."literal" AS "kanji",
+  "JMdict_KanjiElement"."reading"
+FROM
+  "KANJIDIC_Character"
+JOIN "JMdict_KanjiElementFTS"
+  ON "JMdict_KanjiElementFTS"."reading" MATCH "KANJIDIC_Character"."literal" || '*'
+JOIN "JMdict_KanjiElement"
+  ON "JMdict_KanjiElementFTS"."entryId" = "JMdict_KanjiElement"."entryId"
+    AND "JMdict_KanjiElementFTS"."reading" LIKE '%' || "JMdict_KanjiElement"."reading"
+JOIN "JMdict_EntryScore"
+  ON "JMdict_EntryScore"."type" = 'k'
+    AND "JMdict_KanjiElement"."entryId" = "JMdict_EntryScore"."entryId"
+    AND "JMdict_KanjiElement"."reading" = "JMdict_EntryScore"."reading"
+WHERE "JMdict_EntryScore"."common" = 1;
+
+
+CREATE VIEW "RADKFILE_Radicals" AS
+SELECT DISTINCT "radical" FROM "RADKFILE";
+
+CREATE VIEW "JMdict_CombinedEntryScore"
+AS
+SELECT
+  CASE
+    WHEN "JMdict_EntryScore"."type" = 'k'
+      THEN (SELECT entryId FROM "JMdict_KanjiElement" WHERE "elementId" = "JMdict_EntryScore"."elementId")
+    WHEN "JMdict_EntryScore"."type" = 'r'
+      THEN (SELECT entryId FROM "JMdict_ReadingElement" WHERE "elementId" = "JMdict_EntryScore"."elementId")
+  END AS "entryId",
+  MAX("JMdict_EntryScore"."score") AS "score",
+  MAX("JMdict_EntryScore"."common") AS "common"
+FROM "JMdict_EntryScore"
+GROUP BY "entryId";

nix/database.nix

@@ -7,6 +7,7 @@
   radkfile,
   kanjidic2,
   sqlite,
+  wal ? false,
 }:
 stdenvNoCC.mkDerivation {
   name = "jadb";
@@ -16,7 +17,7 @@ stdenvNoCC.mkDerivation {
     database-tool
     sqlite
   ];
-  
+
   buildPhase = ''
     runHook preBuild
 
@@ -29,7 +30,9 @@ stdenvNoCC.mkDerivation {
       sqlite3 jadb.sqlite < "$migration"
     done
 
-    "${lib.getExe database-tool}" create-db --libsqlite "${sqlite.out}/lib/libsqlite3.so"
+    "${lib.getExe database-tool}" create-db \
+      ${lib.optionalString wal "--wal"}  \
+      --libsqlite "${sqlite.out}/lib/libsqlite3.so"
 
     runHook postBuild
   '';

nix/database_tool.nix

@@ -1,12 +1,39 @@
 {
   src,
   buildDartApplication,
+  sqlite,
 }:
 buildDartApplication {
   pname = "jadb-database-tool";
   version = "1.0.0";
   inherit src;
 
+  dartEntryPoints."bin/jadb" = "bin/jadb.dart";
+
+  # NOTE: here we are overriding the implicitly added runtimeDependency from the package fixup in pub2nix.
+  runtimeDependencies = [ sqlite ];
+
+  # NOTE: the default dart hooks are using `dart compile`, which is not able to call the
+  #       new dart build hooks required to use package:sqlite3 >= 3.0.0. So we override
+  #       these phases to use `dart build` instead.
+  buildPhase = ''
+    runHook preBuild
+
+    mkdir -p "$out/bin"
+    dart build cli --target "bin/jadb.dart"
+
+    runHook postBuild
+  '';
+
+  installPhase = ''
+    runHook preInstall
+
+    mkdir -p "$out"
+    mv build/cli/*/bundle/* "$out/"
+
+    runHook postInstall
+  '';
+
   autoPubspecLock = ../pubspec.lock;
 
   meta.mainProgram = "jadb";

nix/docs.nix

@@ -10,6 +10,7 @@
 stdenvNoCC.mkDerivation {
   name = "docs";
   src = database;
+
   nativeBuildInputs = [
     sqlite
     schemaspy

nix/jmdict.nix

@@ -9,16 +9,17 @@
 stdenvNoCC.mkDerivation {
   name = "jmdict";
 
+  dontUnpack = true;
   srcs = [
     jmdict-src
     jmdict-with-examples-src
   ];
-  dontUnpack = true;
 
   nativeBuildInputs = [
     gzip
     xmlformat
   ];
+
   buildPhase = ''
     runHook preBuild
 

nix/kanjidic2.nix

@@ -15,6 +15,7 @@ stdenvNoCC.mkDerivation {
     gzip
     xmlformat
   ];
+
   buildPhase = ''
     runHook preBuild
 

nix/radkfile.nix

@@ -15,6 +15,7 @@ stdenv.mkDerivation {
     gzip
     iconv
   ];
+
   buildPhase = ''
     runHook preBuild
 

pubspec.lock

@@ -5,18 +5,18 @@ packages:
     dependency: transitive
     description:
       name: _fe_analyzer_shared
-      sha256: e55636ed79578b9abca5fecf9437947798f5ef7456308b5cb85720b793eac92f
+      sha256: "8d718c5c58904f9937290fd5dbf2d6a0e02456867706bfb6cd7b81d394e738d5"
       url: "https://pub.dev"
     source: hosted
-    version: "82.0.0"
+    version: "98.0.0"
   analyzer:
     dependency: transitive
     description:
       name: analyzer
-      sha256: "13c1e6c6fd460522ea840abec3f677cc226f5fec7872c04ad7b425517ccf54f7"
+      sha256: "6141ad5d092d1e1d13929c0504658bbeccc1703505830d7c26e859908f5efc88"
       url: "https://pub.dev"
     source: hosted
-    version: "7.4.4"
+    version: "12.0.0"
   args:
     dependency: "direct main"
     description:
@@ -29,10 +29,18 @@ packages:
     dependency: transitive
     description:
       name: async
-      sha256: "758e6d74e971c3e5aceb4110bfd6698efc7f501675bcfe0c775459a8140750eb"
+      sha256: e2eb0491ba5ddb6177742d2da23904574082139b07c1e33b8503b9f46f3e1a37
       url: "https://pub.dev"
     source: hosted
-    version: "2.13.0"
+    version: "2.13.1"
+  benchmark_harness:
+    dependency: "direct dev"
+    description:
+      name: benchmark_harness
+      sha256: a2d3c4c83cac0126bf38e41eaf7bd9ed4f6635f1ee1a0cbc6f79fa9736c62cbd
+      url: "https://pub.dev"
+    source: hosted
+    version: "2.4.0"
   boolean_selector:
     dependency: transitive
     description:
@@ -49,6 +57,14 @@ packages:
       url: "https://pub.dev"
     source: hosted
     version: "0.2.0"
+  code_assets:
+    dependency: transitive
+    description:
+      name: code_assets
+      sha256: "83ccdaa064c980b5596c35dd64a8d3ecc68620174ab9b90b6343b753aa721687"
+      url: "https://pub.dev"
+    source: hosted
+    version: "1.0.0"
   collection:
     dependency: "direct main"
     description:
@@ -69,42 +85,42 @@ packages:
     dependency: transitive
     description:
       name: coverage
-      sha256: "9086475ef2da7102a0c0a4e37e1e30707e7fb7b6d28c209f559a9c5f8ce42016"
+      sha256: "5da775aa218eaf2151c721b16c01c7676fbfdd99cebba2bf64e8b807a28ff94d"
       url: "https://pub.dev"
     source: hosted
-    version: "1.12.0"
+    version: "1.15.0"
   crypto:
     dependency: transitive
     description:
       name: crypto
-      sha256: "1e445881f28f22d6140f181e07737b22f1e099a5e1ff94b0af2f9e4a463f4855"
+      sha256: c8ea0233063ba03258fbcf2ca4d6dadfefe14f02fab57702265467a19f27fadf
       url: "https://pub.dev"
     source: hosted
-    version: "3.0.6"
+    version: "3.0.7"
   csv:
     dependency: "direct main"
     description:
       name: csv
-      sha256: c6aa2679b2a18cb57652920f674488d89712efaf4d3fdf2e537215b35fc19d6c
+      sha256: "2e0a52fb729f2faacd19c9c0c954ff450bba37aa8ab999410309e2342e7013a2"
       url: "https://pub.dev"
     source: hosted
-    version: "6.0.0"
+    version: "8.0.0"
   equatable:
     dependency: "direct main"
     description:
       name: equatable
-      sha256: "567c64b3cb4cf82397aac55f4f0cbd3ca20d77c6c03bedbc4ceaddc08904aef7"
+      sha256: "3e0141505477fd8ad55d6eb4e7776d3fe8430be8e497ccb1521370c3f21a3e2b"
       url: "https://pub.dev"
     source: hosted
-    version: "2.0.7"
+    version: "2.0.8"
   ffi:
     dependency: transitive
     description:
       name: ffi
-      sha256: "289279317b4b16eb2bb7e271abccd4bf84ec9bdcbe999e278a94b804f5630418"
+      sha256: "6d7fd89431262d8f3125e81b50d3847a091d846eafcd4fdb88dd06f36d705a45"
       url: "https://pub.dev"
     source: hosted
-    version: "2.1.4"
+    version: "2.2.0"
   file:
     dependency: transitive
     description:
@@ -129,6 +145,14 @@ packages:
       url: "https://pub.dev"
     source: hosted
     version: "2.1.3"
+  hooks:
+    dependency: transitive
+    description:
+      name: hooks
+      sha256: e79ed1e8e1929bc6ecb6ec85f0cb519c887aa5b423705ded0d0f2d9226def388
+      url: "https://pub.dev"
+    source: hosted
+    version: "1.0.2"
   http_multi_server:
     dependency: transitive
     description:
@@ -153,22 +177,14 @@ packages:
       url: "https://pub.dev"
     source: hosted
     version: "1.0.5"
-  js:
-    dependency: transitive
-    description:
-      name: js
-      sha256: "53385261521cc4a0c4658fd0ad07a7d14591cf8fc33abbceae306ddb974888dc"
-      url: "https://pub.dev"
-    source: hosted
-    version: "0.7.2"
   lints:
     dependency: "direct dev"
     description:
       name: lints
-      sha256: c35bb79562d980e9a453fc715854e1ed39e24e7d0297a880ef54e17f9874a9d7
+      sha256: "12f842a479589fea194fe5c5a3095abc7be0c1f2ddfa9a0e76aed1dbd26a87df"
       url: "https://pub.dev"
     source: hosted
-    version: "5.1.1"
+    version: "6.1.0"
   logging:
     dependency: transitive
     description:
@@ -181,18 +197,18 @@ packages:
     dependency: transitive
     description:
       name: matcher
-      sha256: dc58c723c3c24bf8d3e2d3ad3f2f9d7bd9cf43ec6feaa64181775e60190153f2
+      sha256: dc0b7dc7651697ea4ff3e69ef44b0407ea32c487a39fff6a4004fa585e901861
       url: "https://pub.dev"
     source: hosted
-    version: "0.12.17"
+    version: "0.12.19"
   meta:
     dependency: transitive
     description:
       name: meta
-      sha256: e3641ec5d63ebf0d9b41bd43201a66e3fc79a65db5f61fc181f04cd27aab950c
+      sha256: df0c643f44ad098eb37988027a8e2b2b5a031fd3977f06bbfd3a76637e8df739
       url: "https://pub.dev"
     source: hosted
-    version: "1.16.0"
+    version: "1.18.2"
   mime:
     dependency: transitive
     description:
@@ -201,6 +217,14 @@ packages:
       url: "https://pub.dev"
     source: hosted
     version: "2.0.0"
+  native_toolchain_c:
+    dependency: transitive
+    description:
+      name: native_toolchain_c
+      sha256: "6ba77bb18063eebe9de401f5e6437e95e1438af0a87a3a39084fbd37c90df572"
+      url: "https://pub.dev"
+    source: hosted
+    version: "0.17.6"
   node_preamble:
     dependency: transitive
     description:
@@ -218,7 +242,7 @@ packages:
     source: hosted
     version: "2.2.0"
   path:
-    dependency: transitive
+    dependency: "direct main"
     description:
       name: path
       sha256: "75cca69d1490965be98c73ceaea117e8a04dd21217b37b292c9ddbec0d955bc5"
@@ -229,18 +253,18 @@ packages:
     dependency: transitive
     description:
       name: petitparser
-      sha256: "07c8f0b1913bcde1ff0d26e57ace2f3012ccbf2b204e070290dad3bb22797646"
+      sha256: "91bd59303e9f769f108f8df05e371341b15d59e995e6806aefab827b58336675"
       url: "https://pub.dev"
     source: hosted
-    version: "6.1.0"
+    version: "7.0.2"
   pool:
     dependency: transitive
     description:
       name: pool
-      sha256: "20fe868b6314b322ea036ba325e6fc0711a22948856475e2c2b6306e8ab39c2a"
+      sha256: "978783255c543aa3586a1b3c21f6e9d720eb315376a915872c61ef8b5c20177d"
       url: "https://pub.dev"
     source: hosted
-    version: "1.5.1"
+    version: "1.5.2"
   pub_semver:
     dependency: transitive
     description:
@@ -301,34 +325,34 @@ packages:
     dependency: transitive
     description:
       name: source_span
-      sha256: "254ee5351d6cb365c859e20ee823c3bb479bf4a293c22d17a9f1bf144ce86f7c"
+      sha256: "56a02f1f4cd1a2d96303c0144c93bd6d909eea6bee6bf5a0e0b685edbd4c47ab"
       url: "https://pub.dev"
     source: hosted
-    version: "1.10.1"
+    version: "1.10.2"
   sqflite_common:
     dependency: "direct main"
     description:
       name: sqflite_common
-      sha256: "84731e8bfd8303a3389903e01fb2141b6e59b5973cacbb0929021df08dddbe8b"
+      sha256: "6ef422a4525ecc601db6c0a2233ff448c731307906e92cabc9ba292afaae16a6"
       url: "https://pub.dev"
     source: hosted
-    version: "2.5.5"
+    version: "2.5.6"
   sqflite_common_ffi:
     dependency: "direct main"
     description:
       name: sqflite_common_ffi
-      sha256: "1f3ef3888d3bfbb47785cc1dda0dc7dd7ebd8c1955d32a9e8e9dae1e38d1c4c1"
+      sha256: c59fcdc143839a77581f7a7c4de018e53682408903a0a0800b95ef2dc4033eff
       url: "https://pub.dev"
     source: hosted
-    version: "2.3.5"
+    version: "2.4.0+2"
   sqlite3:
-    dependency: transitive
+    dependency: "direct main"
     description:
       name: sqlite3
-      sha256: "310af39c40dd0bb2058538333c9d9840a2725ae0b9f77e4fd09ad6696aa8f66e"
+      sha256: caa693ad15a587a2b4fde093b728131a1827903872171089dedb16f7665d3a91
       url: "https://pub.dev"
     source: hosted
-    version: "2.7.5"
+    version: "3.2.0"
   stack_trace:
     dependency: transitive
     description:
@@ -357,10 +381,10 @@ packages:
     dependency: transitive
     description:
       name: synchronized
-      sha256: "0669c70faae6270521ee4f05bffd2919892d42d1276e6c495be80174b6bc0ef6"
+      sha256: c254ade258ec8282947a0acbbc90b9575b4f19673533ee46f2f6e9b3aeefd7c0
       url: "https://pub.dev"
     source: hosted
-    version: "3.3.1"
+    version: "3.4.0"
   term_glyph:
     dependency: transitive
     description:
@@ -373,26 +397,26 @@ packages:
     dependency: "direct dev"
     description:
       name: test
-      sha256: "301b213cd241ca982e9ba50266bd3f5bd1ea33f1455554c5abb85d1be0e2d87e"
+      sha256: "8d9ceddbab833f180fbefed08afa76d7c03513dfdba87ffcec2718b02bbcbf20"
       url: "https://pub.dev"
     source: hosted
-    version: "1.25.15"
+    version: "1.31.0"
   test_api:
     dependency: transitive
     description:
       name: test_api
-      sha256: fb31f383e2ee25fbbfe06b40fe21e1e458d14080e3c67e7ba0acfde4df4e0bbd
+      sha256: "949a932224383300f01be9221c39180316445ecb8e7547f70a41a35bf421fb9e"
       url: "https://pub.dev"
     source: hosted
-    version: "0.7.4"
+    version: "0.7.11"
   test_core:
     dependency: transitive
     description:
       name: test_core
-      sha256: "84d17c3486c8dfdbe5e12a50c8ae176d15e2a771b96909a9442b40173649ccaa"
+      sha256: "1991d4cfe85d5043241acac92962c3977c8d2f2add1ee73130c7b286417d1d34"
       url: "https://pub.dev"
     source: hosted
-    version: "0.6.8"
+    version: "0.6.17"
   typed_data:
     dependency: transitive
     description:
@@ -405,18 +429,18 @@ packages:
     dependency: transitive
     description:
       name: vm_service
-      sha256: ddfa8d30d89985b96407efce8acbdd124701f96741f2d981ca860662f1c0dc02
+      sha256: "45caa6c5917fa127b5dbcfbd1fa60b14e583afdc08bfc96dda38886ca252eb60"
       url: "https://pub.dev"
     source: hosted
-    version: "15.0.0"
+    version: "15.0.2"
   watcher:
     dependency: transitive
     description:
       name: watcher
-      sha256: "69da27e49efa56a15f8afe8f4438c4ec02eff0a117df1b22ea4aad194fe1c104"
+      sha256: "1398c9f081a753f9226febe8900fce8f7d0a67163334e1c94a2438339d79d635"
       url: "https://pub.dev"
     source: hosted
-    version: "1.1.1"
+    version: "1.2.1"
   web:
     dependency: transitive
     description:
@@ -429,10 +453,10 @@ packages:
     dependency: transitive
     description:
       name: web_socket
-      sha256: bfe6f435f6ec49cb6c01da1e275ae4228719e59a6b067048c51e72d9d63bcc4b
+      sha256: "34d64019aa8e36bf9842ac014bb5d2f5586ca73df5e4d9bf5c936975cae6982c"
       url: "https://pub.dev"
     source: hosted
-    version: "1.0.0"
+    version: "1.0.1"
   web_socket_channel:
     dependency: transitive
     description:
@@ -453,10 +477,10 @@ packages:
     dependency: "direct main"
     description:
       name: xml
-      sha256: b015a8ad1c488f66851d762d3090a21c600e479dc75e68328c52774040cf9226
+      sha256: "971043b3a0d3da28727e40ed3e0b5d18b742fa5a68665cca88e74b7876d5e025"
       url: "https://pub.dev"
     source: hosted
-    version: "6.5.0"
+    version: "6.6.1"
   yaml:
     dependency: transitive
     description:
@@ -466,4 +490,4 @@ packages:
     source: hosted
     version: "3.1.3"
 sdks:
-  dart: ">=3.7.0 <4.0.0"
+  dart: ">=3.10.1 <4.0.0"

pubspec.yaml

@@ -4,24 +4,32 @@ version: 1.0.0
 homepage: https://git.pvv.ntnu.no/oysteikt/jadb
 
 environment:
-  sdk: '>=3.0.0 <4.0.0'
+  sdk: '^3.9.0'
 
 dependencies:
   args: ^2.7.0
-  collection: ^1.19.1
-  csv: ^6.0.0
-  equatable: ^2.0.7
-  sqflite_common: ^2.5.5
-  sqflite_common_ffi: ^2.3.5
+  collection: ^1.19.0
+  csv: ^8.0.0
+  equatable: ^2.0.0
+  path: ^1.9.1
+  sqflite_common: ^2.5.0
+  sqflite_common_ffi: ^2.3.0
+  sqlite3: ^3.1.6
   xml: ^6.5.0
 
 dev_dependencies:
-  lints: ^5.0.0
+  benchmark_harness: ^2.4.0
+  lints: ^6.0.0
   test: ^1.25.15
 
 executables:
   jadb: jadb
 
+hooks:
+  user_defines:
+    sqlite3:
+      source: system
+
 topics:
   - database
   - dictionary

test/const_data/kanji_grades.dart

@@ -0,0 +1,21 @@
+import 'package:collection/collection.dart';
+import 'package:jadb/const_data/kanji_grades.dart';
+import 'package:test/test.dart';
+
+void main() {
+  test('All constant kanji in jouyouKanjiByGrades are 2136 in total', () {
+    expect(jouyouKanjiByGrades.values.flattenedToSet.length, 2136);
+  });
+
+  // test('All constant kanji in jouyouKanjiByGrades are present in KANJIDIC2', () {
+
+  // });
+
+  // test('All constant kanji in jouyouKanjiByGrades have matching grade as in KANJIDIC2', () {
+
+  // });
+
+  // test('All constant kanji in jouyouKanjiByGradesAndStrokeCount have matching stroke count as in KANJIDIC2', () {
+
+  // });
+}