WIP: add kanjivg data

2026-04-09 16:13:02 +09:00
30 changed files with 258 additions and 336 deletions
--- a/benchmark/benchmark.dart
+++ b/benchmark/benchmark.dart
@@ -1,7 +1,5 @@
-import './search/english_word_search.dart';
-import './search/japanese_word_search.dart';
+import './search/word_search.dart';

 Future<void> main() async {
-  await EnglishWordSearchBenchmark.main();
-  await JapaneseWordSearchBenchmark.main();
+  await WordSearchBenchmark.main();
 }
--- a/benchmark/search/japanese_word_search.dart
+++ b/benchmark/search/japanese_word_search.dart
@@ -1,49 +0,0 @@
-import 'package:benchmark_harness/benchmark_harness.dart';
-import 'package:jadb/search.dart';
-import 'package:sqflite_common/sqlite_api.dart';
-
-import '../../test/search/setup_database_connection.dart';
-
-class JapaneseWordSearchBenchmark extends AsyncBenchmarkBase {
-  Database? connection;
-
-  static final List<String> searchTerms = [
-    '仮名',
-    '漢字',
-    'かわいい',
-    'すし',
-    'ラメン',
-  ];
-
-  JapaneseWordSearchBenchmark() : super('JapaneseWordSearchBenchmark');
-
-  static Future<void> main() async {
-    print('Running JapaneseWordSearchBenchmark...');
-    await JapaneseWordSearchBenchmark().report();
-    print('Finished JapaneseWordSearchBenchmark');
-  }
-
-  @override
-  Future<void> setup() async {
-    connection = await setupDatabaseConnection();
-  }
-
-  @override
-  Future<void> run() async {
-    for (final term in searchTerms) {
-      final result = await connection!.jadbSearchWord(term);
-      assert(
-        result?.isNotEmpty ?? false,
-        'Expected search results for term "$term"',
-      );
-    }
-  }
-
-  @override
-  Future<void> teardown() async {
-    await connection?.close();
-  }
-
-  // @override
-  // Future<void> exercise() => run();
-}
--- a/benchmark/search/english_word_search.dart
+++ b/benchmark/search/english_word_search.dart
@@ -4,23 +4,23 @@ import 'package:sqflite_common/sqlite_api.dart';

 import '../../test/search/setup_database_connection.dart';

-class EnglishWordSearchBenchmark extends AsyncBenchmarkBase {
+class WordSearchBenchmark extends AsyncBenchmarkBase {
  Database? connection;

  static final List<String> searchTerms = [
    'kana',
    'kanji',
-    'cute',
+    'kawaii',
    'sushi',
    'ramen',
  ];

-  EnglishWordSearchBenchmark() : super('EnglishWordSearchBenchmark');
+  WordSearchBenchmark() : super('WordSearchBenchmark');

  static Future<void> main() async {
-    print('Running EnglishWordSearchBenchmark...');
-    await EnglishWordSearchBenchmark().report();
-    print('Finished EnglishWordSearchBenchmark');
+    print('Running WordSearchBenchmark...');
+    await WordSearchBenchmark().report();
+    print('Finished WordSearchBenchmark');
  }

  @override
@@ -31,11 +31,7 @@ class EnglishWordSearchBenchmark extends AsyncBenchmarkBase {
  @override
  Future<void> run() async {
    for (final term in searchTerms) {
-      final result = await connection!.jadbSearchWord(term);
-      assert(
-        result?.isNotEmpty ?? false,
-        'Expected search results for term "$term"',
-      );
+      await connection!.jadbSearchWord(term);
    }
  }

--- a/docs/database.md
+++ b/docs/database.md
@@ -25,5 +25,4 @@ The `JMdict_EntryScore` table is used to store the score of each entry, which is

 The table is automatically generated from other tables via triggers, and should be considered as a materialized view.

-<s>There is a score row for every single entry in both `JMdict_KanjiElement` and `JMdict_ReadingElement`, split by the `type` field.</s>
-This is no longer true, we now only store the rows for which the score is not `0`. The `type` field is now also virtual, since the `elementId` fields for both kanji and readings are unique to each other.
+There is a score row for every single entry in both `JMdict_KanjiElement` and `JMdict_ReadingElement`, split by the `type` field.
--- a/flake.lock
+++ b/flake.lock
@@ -7,11 +7,11 @@
        ]
      },
      "locked": {
-        "lastModified": 1776081209,
-        "narHash": "sha256-zR1115tcOPnYLk6NznSf7YslyaJLc/MGayEHShitx18=",
+        "lastModified": 1775550160,
+        "narHash": "sha256-bgvKrMGUPaDY4EZv+82z1ccYoxwaergdVw/3PZhc2Fc=",
        "ref": "refs/heads/main",
-        "rev": "7fe3552bb16e1d315c0b27b243e5eb53cd9e86fc",
-        "revCount": 13,
+        "rev": "f46229af3678124c5ea7c8dff3292747d0274f69",
+        "revCount": 8,
        "type": "git",
        "url": "https://git.pvv.ntnu.no/Mugiten/datasources.git"
      },
@@ -20,6 +20,22 @@
        "url": "https://git.pvv.ntnu.no/Mugiten/datasources.git"
      }
    },
+    "kanjivg-src": {
+      "flake": false,
+      "locked": {
+        "lastModified": 1775218066,
+        "narHash": "sha256-iYv9xakgoGt/JwwdKDUCpSAF36hBtKlX9oN7xiLowjs=",
+        "ref": "refs/heads/master",
+        "rev": "544d319f79348c092d567b662f27f33dacfa60cd",
+        "revCount": 2215,
+        "type": "git",
+        "url": "https://git.pvv.ntnu.no/mugiten/kanjivg.git"
+      },
+      "original": {
+        "type": "git",
+        "url": "https://git.pvv.ntnu.no/mugiten/kanjivg.git"
+      }
+    },
    "nixpkgs": {
      "locked": {
        "lastModified": 1775423009,
@@ -38,6 +54,7 @@
    "root": {
      "inputs": {
        "datasources": "datasources",
+        "kanjivg-src": "kanjivg-src",
        "nixpkgs": "nixpkgs"
      }
    }
--- a/flake.nix
+++ b/flake.nix
@@ -8,12 +8,18 @@
      url = "git+https://git.pvv.ntnu.no/Mugiten/datasources.git";
      inputs.nixpkgs.follows = "nixpkgs";
    };
+
+    kanjivg-src = {
+      url = "git+https://git.pvv.ntnu.no/mugiten/kanjivg.git";
+      flake = false;
+    };
  };

  outputs = {
    self,
    nixpkgs,
    datasources,
+    kanjivg-src,
  }: let
    inherit (nixpkgs) lib;
    systems = [
--- a/lib/_data_ingestion/jmdict/objects.dart
+++ b/lib/_data_ingestion/jmdict/objects.dart
@@ -106,8 +106,9 @@ class Glossary extends SQLWritable {

  @override
  Map<String, Object?> get sqlValue => {
-    // 'language': language,
+    'language': language,
    'phrase': phrase,
+    'type': type,
  };
 }

--- a/lib/_data_ingestion/jmdict/seed_data.dart
+++ b/lib/_data_ingestion/jmdict/seed_data.dart
@@ -1,5 +1,4 @@
 import 'dart:collection';
-import 'dart:io';

 import 'package:collection/collection.dart';
 import 'package:jadb/_data_ingestion/jmdict/objects.dart';
@@ -85,21 +84,6 @@ Future<void> seedJMDictData(List<Entry> entries, Database db) async {
  print('  [JMdict] Batch 1 - Kanji and readings');
  Batch b = db.batch();

-  if (Platform.environment['JMDICT_VERSION'] != null &&
-      Platform.environment['JMDICT_DATE'] != null &&
-      Platform.environment['JMDICT_HASH'] != null) {
-    b.insert(JMdictTableNames.version, {
-      'version': Platform.environment['JMDICT_VERSION']!,
-      'date': Platform.environment['JMDICT_DATE']!,
-      'hash': Platform.environment['JMDICT_HASH']!,
-    });
-  } else {
-    print(
-      'WARNING: JMDICT version information not found in environment variables. '
-      'This may cause issues with future updates.',
-    );
-  }
-
  for (final e in entries) {
    b.insert(JMdictTableNames.entry, e.sqlValue);

@@ -188,14 +172,6 @@ Future<void> seedJMDictData(List<Entry> entries, Database db) async {
          JMdictTableNames.senseGlossary,
          g.sqlValue..addAll({'senseId': s.senseId}),
        );
-
-        if (g.type != null) {
-          b.insert(JMdictTableNames.senseGlossaryType, {
-            'senseId': s.senseId,
-            'phrase': g.phrase,
-            'type': g.type!,
-          });
-        }
      }
    }
  }
@@ -243,6 +219,8 @@ Future<void> seedJMDictData(List<Entry> entries, Database db) async {
        b.insert(JMdictTableNames.senseSeeAlso, {
          'senseId': s.senseId,
          'xrefEntryId': resolvedEntry.entry.entryId,
+          'seeAlsoKanji': xref.kanjiRef,
+          'seeAlsoReading': xref.readingRef,
          'seeAlsoSense': xref.senseOrderNum != null
              ? xref.senseOrderNum! - 1
              : null,
@@ -270,6 +248,8 @@ Future<void> seedJMDictData(List<Entry> entries, Database db) async {
        b.insert(JMdictTableNames.senseAntonyms, {
          'senseId': s.senseId,
          'xrefEntryId': resolvedEntry.entry.entryId,
+          'antonymKanji': ant.kanjiRef,
+          'antonymReading': ant.readingRef,
          'antonymSense': ant.senseOrderNum != null
              ? ant.senseOrderNum! - 1
              : null,
--- a/lib/_data_ingestion/kanjidic/seed_data.dart
+++ b/lib/_data_ingestion/kanjidic/seed_data.dart
@@ -1,5 +1,3 @@
-import 'dart:io';
-
 import 'package:jadb/table_names/kanjidic.dart';
 import 'package:sqflite_common/sqlite_api.dart';

@@ -7,22 +5,6 @@ import 'objects.dart';

 Future<void> seedKANJIDICData(List<Character> characters, Database db) async {
  final b = db.batch();
-
-  if (Platform.environment['KANJIDIC_VERSION'] != null &&
-      Platform.environment['KANJIDIC_DATE'] != null &&
-      Platform.environment['KANJIDIC_HASH'] != null) {
-    b.insert(KANJIDICTableNames.version, {
-      'version': Platform.environment['KANJIDIC_VERSION']!,
-      'date': Platform.environment['KANJIDIC_DATE']!,
-      'hash': Platform.environment['KANJIDIC_HASH']!,
-    });
-  } else {
-    print(
-      'WARNING: KANJIDIC version information not found in environment variables. '
-      'This may cause issues with future updates.',
-    );
-  }
-
  for (final c in characters) {
    // if (c.dictionaryReferences.any((e) =>
    //     c.dictionaryReferences
@@ -48,7 +30,10 @@ Future<void> seedKANJIDICData(List<Character> characters, Database db) async {
    }

    if (c.jlpt != null) {
-      b.insert(KANJIDICTableNames.jlpt, {'kanji': c.literal, 'jlpt': c.jlpt!});
+      b.insert(KANJIDICTableNames.jlpt, {
+        'kanji': c.literal,
+        'jlpt': c.jlpt!,
+      });
    }

    for (final n in c.radicalName) {
--- a/lib/_data_ingestion/kanjivg/objects.dart
+++ b/lib/_data_ingestion/kanjivg/objects.dart
@@ -0,0 +1,92 @@
+import 'package:jadb/_data_ingestion/sql_writable.dart';
+
+/// Enum set in the kvg:position attribute, used by `<g>` elements in the KanjiVG SVG files.
+enum KanjiPathGroupPosition {
+  bottom,
+  kamae,
+  kamaec,
+  left,
+  middle,
+  nyo,
+  nyoc,
+  right,
+  tare,
+  tarec,
+  top,
+}
+
+/// Contents of a \<g> element in the KanjiVG SVG files.
+class KanjiPathGroupTreeNode extends SQLWritable {
+  final String id;
+  final List<KanjiPathGroupTreeNode> children;
+  final String? element;
+  final String? original;
+  final KanjiPathGroupPosition? position;
+  final String? radical;
+  final int? part;
+
+  KanjiPathGroupTreeNode({
+    required this.id,
+    this.children = const [],
+    this.element,
+    this.original,
+    this.position,
+    this.radical,
+    this.part,
+  });
+
+  @override
+  Map<String, Object?> get sqlValue => {
+    'id': id,
+    'element': element,
+    'original': original,
+    'position': position?.name,
+    'radical': radical,
+    'part': part,
+  };
+}
+
+/// Contents of a `<text>` element in the StrokeNumber's group in the KanjiVG SVG files
+class KanjiStrokeNumber extends SQLWritable {
+  final int num;
+  final double x;
+  final double y;
+
+  KanjiStrokeNumber(this.num, this.x, this.y);
+
+  @override
+  Map<String, Object?> get sqlValue => {'num': num, 'x': x, 'y': y};
+}
+
+/// Contents of a `<path>` element in the KanjiVG SVG files
+class KanjiVGPath extends SQLWritable {
+  final String id;
+  final String type;
+  final String svgPath;
+
+  KanjiVGPath({required this.id, required this.type, required this.svgPath});
+
+  @override
+  Map<String, Object?> get sqlValue => {
+    'id': id,
+    'type': type,
+    'svgPath': svgPath,
+  };
+}
+
+class KanjiVGItem extends SQLWritable {
+  final String character;
+  final List<KanjiVGPath> paths;
+  final List<KanjiStrokeNumber> strokeNumbers;
+  final List<KanjiPathGroupTreeNode> pathGroups;
+
+  KanjiVGItem({
+    required this.character,
+    required this.paths,
+    required this.strokeNumbers,
+    required this.pathGroups,
+  });
+
+  @override
+  Map<String, Object?> get sqlValue => {'character': character};
+}
--- a/lib/_data_ingestion/kanjivg/seed_data.dart
+++ b/lib/_data_ingestion/kanjivg/seed_data.dart
@@ -0,0 +1,7 @@
+import 'package:sqflite_common/sqflite.dart';
+
+Future<void> seedKanjiVGData(Iterable<String> xmlContents, Database db) async {
+  final b = db.batch();
+
+  await b.commit(noResult: true);
+}
--- a/lib/_data_ingestion/radkfile/seed_data.dart
+++ b/lib/_data_ingestion/radkfile/seed_data.dart
@@ -1,26 +1,9 @@
-import 'dart:io';
-
 import 'package:jadb/table_names/radkfile.dart';
 import 'package:sqflite_common/sqlite_api.dart';

 Future<void> seedRADKFILEData(Iterable<String> blocks, Database db) async {
  final b = db.batch();

-  if (Platform.environment['RADKFILE_VERSION'] != null &&
-      Platform.environment['RADKFILE_DATE'] != null &&
-      Platform.environment['RADKFILE_HASH'] != null) {
-    b.insert(RADKFILETableNames.version, {
-      'version': Platform.environment['RADKFILE_VERSION']!,
-      'date': Platform.environment['RADKFILE_DATE']!,
-      'hash': Platform.environment['RADKFILE_HASH']!,
-    });
-  } else {
-    print(
-      'WARNING: RADKFILE version information not found in environment variables. '
-      'This may cause issues with future updates.',
-    );
-  }
-
  for (final block in blocks) {
    final String radical = block[1];
    final List<String> kanjiList =
--- a/lib/_data_ingestion/tanos-jlpt/seed_data.dart
+++ b/lib/_data_ingestion/tanos-jlpt/seed_data.dart
@@ -1,5 +1,3 @@
-import 'dart:io';
-
 import 'package:jadb/table_names/tanos_jlpt.dart';
 import 'package:sqflite_common/sqlite_api.dart';

@@ -9,21 +7,6 @@ Future<void> seedTanosJLPTData(
 ) async {
  final Batch b = db.batch();

-  if (Platform.environment['TANOS_JLPT_VERSION'] != null &&
-      Platform.environment['TANOS_JLPT_DATE'] != null &&
-      Platform.environment['TANOS_JLPT_HASH'] != null) {
-    b.insert(TanosJLPTTableNames.version, {
-      'version': Platform.environment['TANOS_JLPT_VERSION']!,
-      'date': Platform.environment['TANOS_JLPT_DATE']!,
-      'hash': Platform.environment['TANOS_JLPT_HASH']!,
-    });
-  } else {
-    print(
-      'WARNING: Tanos JLPT version information not found in environment variables. '
-      'This may cause issues with future updates.',
-    );
-  }
-
  for (final jlptLevel in resolvedEntries.entries) {
    final level = jlptLevel.key;
    final entryIds = jlptLevel.value;
--- a/lib/models/verify_tables.dart
+++ b/lib/models/verify_tables.dart
@@ -1,5 +1,6 @@
 import 'package:jadb/table_names/jmdict.dart';
 import 'package:jadb/table_names/kanjidic.dart';
+import 'package:jadb/table_names/kanjivg.dart';
 import 'package:jadb/table_names/radkfile.dart';
 import 'package:jadb/table_names/tanos_jlpt.dart';
 import 'package:sqflite_common/sqlite_api.dart';
@@ -21,6 +22,7 @@ Future<void> verifyTablesWithDbConnection(DatabaseExecutor db) async {
    ...KANJIDICTableNames.allTables,
    ...RADKFILETableNames.allTables,
    ...TanosJLPTTableNames.allTables,
+    ...KanjiVGTableNames.allTables,
  };

  final missingTables = expectedTables.difference(tables);
--- a/lib/search.dart
+++ b/lib/search.dart
@@ -18,12 +18,12 @@ extension JaDBConnection on DatabaseExecutor {
      searchKanjiWithDbConnection(this, kanji);

  /// Search for a kanji in the database.
-  Future<Map<String, KanjiSearchResult>> jadbGetManyKanji(Iterable<String> kanji) =>
+  Future<Map<String, KanjiSearchResult>> jadbGetManyKanji(Set<String> kanji) =>
      searchManyKanjiWithDbConnection(this, kanji);

  /// Filter a list of characters, and return the ones that are listed in the kanji dictionary.
  Future<List<String>> filterKanji(
-    Iterable<String> kanji, {
+    List<String> kanji, {
    bool deduplicate = false,
  }) => filterKanjiWithDbConnection(this, kanji, deduplicate);

--- a/lib/search/filter_kanji.dart
+++ b/lib/search/filter_kanji.dart
@@ -6,7 +6,7 @@ import 'package:sqflite_common/sqflite.dart';
 /// If [deduplicate] is true, the returned list will deduplicate the input kanji list before returning the filtered results.
 Future<List<String>> filterKanjiWithDbConnection(
  DatabaseExecutor connection,
-  Iterable<String> kanji,
+  List<String> kanji,
  bool deduplicate,
 ) async {
  final Set<String> filteredKanji = await connection
@@ -14,7 +14,7 @@ Future<List<String>> filterKanjiWithDbConnection(
      SELECT "literal"
      FROM "${KANJIDICTableNames.character}"
      WHERE "literal" IN (${kanji.map((_) => '?').join(',')})
-    ''', kanji.toList())
+    ''', kanji)
      .then((value) => value.map((e) => e['literal'] as String).toSet());

  if (deduplicate) {
--- a/lib/search/kanji_search.dart
+++ b/lib/search/kanji_search.dart
@@ -274,7 +274,7 @@ Future<KanjiSearchResult?> searchKanjiWithDbConnection(
 /// Searches for multiple kanji at once, returning a map of kanji to their search results.
 Future<Map<String, KanjiSearchResult>> searchManyKanjiWithDbConnection(
  DatabaseExecutor connection,
-  Iterable<String> kanji,
+  Set<String> kanji,
 ) async {
  if (kanji.isEmpty) {
    return {};
--- a/lib/search/word_search/entry_id_query.dart
+++ b/lib/search/word_search/entry_id_query.dart
@@ -68,25 +68,25 @@ String _filterFTSSensitiveCharacters(String word) {
              "$tableName"."entryId",
              100
              + (("${tableName}FTS"."reading" = ?) * 10000)
-              + (("$tableName"."orderNum" = 0) * 20)
-              + COALESCE("JMdict_EntryScore"."score", 0)
+              + "JMdict_EntryScore"."score"
              AS "score"
            FROM "${tableName}FTS"
            JOIN "$tableName" USING ("elementId")
-            LEFT JOIN "JMdict_EntryScore" USING ("elementId")
+            JOIN "JMdict_EntryScore" USING ("elementId")
            WHERE "${tableName}FTS"."reading" MATCH ? || '*'
+              AND "JMdict_EntryScore"."elementId" ${tableName == JMdictTableNames.kanjiElement ? '<' : '>='} 1000000000
          ),
          non_fts_results AS (
            SELECT DISTINCT
              "$tableName"."entryId",
              50
-              + (("$tableName"."orderNum" = 0) * 20)
-              + COALESCE("JMdict_EntryScore"."score", 0)
+              + "JMdict_EntryScore"."score"
              AS "score"
            FROM "$tableName"
-            LEFT JOIN "JMdict_EntryScore" USING ("elementId")
+            JOIN "JMdict_EntryScore" USING ("elementId")
            WHERE "reading" LIKE '%' || ? || '%'
              AND "$tableName"."entryId" NOT IN (SELECT "entryId" FROM "fts_results")
+              AND "JMdict_EntryScore"."elementId" ${tableName == JMdictTableNames.kanjiElement ? '<' : '>='} 1000000000
          )

        SELECT ${countOnly ? 'COUNT(DISTINCT "entryId") AS count' : '"entryId", MAX("score") AS "score"'}
@@ -198,16 +198,16 @@ Future<List<ScoredEntryId>> _queryEnglish(
    '''
    SELECT
      "${JMdictTableNames.sense}"."entryId",
-      COALESCE(MAX("JMdict_EntryScore"."score"), 0)
+      MAX("JMdict_EntryScore"."score")
      + (("${JMdictTableNames.senseGlossary}"."phrase" = ?1 AND "${JMdictTableNames.sense}"."orderNum" = 0) * 50)
      + (("${JMdictTableNames.senseGlossary}"."phrase" = ?1 AND "${JMdictTableNames.sense}"."orderNum" = 1) * 30)
-      + (("${JMdictTableNames.senseGlossary}"."phrase" = ?1 AND "${JMdictTableNames.sense}"."orderNum" > 1) * 20)
+      + (("${JMdictTableNames.senseGlossary}"."phrase" = ?1) * 20)
      as "score"
    FROM "${JMdictTableNames.senseGlossary}"
    JOIN "${JMdictTableNames.sense}" USING ("senseId")
-    LEFT JOIN "JMdict_EntryScore" USING ("entryId")
+    JOIN "JMdict_EntryScore" USING ("entryId")
    WHERE "${JMdictTableNames.senseGlossary}"."phrase" LIKE ?2
-    GROUP BY "${JMdictTableNames.sense}"."entryId"
+    GROUP BY "JMdict_EntryScore"."entryId"
    ORDER BY
      "score" DESC,
      "${JMdictTableNames.sense}"."entryId" ASC
@@ -215,7 +215,7 @@ Future<List<ScoredEntryId>> _queryEnglish(
    ${offset != null ? 'OFFSET ?4' : ''}
  '''
        .trim(),
-    [word, '%${word.replaceAll('%', '')}%', ?pageSize, ?offset],
+    [word, '%${word.replaceAll('%', '')}%', if (pageSize != null) pageSize, if (offset != null) offset],
  );

  return result
--- a/lib/table_names/jmdict.dart
+++ b/lib/table_names/jmdict.dart
@@ -11,7 +11,6 @@ abstract class JMdictTableNames {
  static const String senseDialect = 'JMdict_SenseDialect';
  static const String senseField = 'JMdict_SenseField';
  static const String senseGlossary = 'JMdict_SenseGlossary';
-  static const String senseGlossaryType = 'JMdict_SenseGlossaryType';
  static const String senseInfo = 'JMdict_SenseInfo';
  static const String senseMisc = 'JMdict_SenseMisc';
  static const String sensePOS = 'JMdict_SensePOS';
@@ -34,7 +33,6 @@ abstract class JMdictTableNames {
    senseDialect,
    senseField,
    senseGlossary,
-    senseGlossaryType,
    senseInfo,
    senseMisc,
    sensePOS,
--- a/lib/table_names/kanjivg.dart
+++ b/lib/table_names/kanjivg.dart
@@ -0,0 +1,9 @@
+abstract class KanjiVGTableNames {
+  static const String version = 'KanjiVG_Version';
+  static const String entry = 'KanjiVG_Entry';
+  static const String path = 'KanjiVG_Path';
+  static const String strokeNumber = 'KanjiVG_StrokeNumber';
+  static const String pathGroup = 'KanjiVG_PathGroup';
+
+  static Set<String> get allTables => {version, entry, path, strokeNumber, pathGroup};
+}
--- a/migrations/0001_JMDict.sql
+++ b/migrations/0001_JMDict.sql
@@ -139,6 +139,8 @@ CREATE TABLE "JMdict_SenseRestrictedToReading" (
 CREATE TABLE "JMdict_SenseSeeAlso" (
  "senseId" INTEGER NOT NULL REFERENCES "JMdict_Sense"("senseId"),
  "xrefEntryId" INTEGER NOT NULL,
+  "seeAlsoReading" TEXT,
+  "seeAlsoKanji" TEXT,
  "seeAlsoSense" INTEGER,
  -- For some entries, the cross reference is ambiguous. This means that while the ingestion
  -- has determined some xrefEntryId, it is not guaranteed to be the correct one.
@@ -151,14 +153,18 @@ CREATE TABLE "JMdict_SenseSeeAlso" (
    END
  ) VIRTUAL,

+  FOREIGN KEY ("xrefEntryId", "seeAlsoKanji") REFERENCES "JMdict_KanjiElement"("entryId", "reading"),
+  FOREIGN KEY ("xrefEntryId", "seeAlsoReading") REFERENCES "JMdict_ReadingElement"("entryId", "reading"),
  FOREIGN KEY ("seeAlsoSenseKey") REFERENCES "JMdict_Sense"("senseId"),

-  PRIMARY KEY ("senseId", "xrefEntryId", "seeAlsoSense")
+  UNIQUE("senseId", "xrefEntryId", "seeAlsoReading", "seeAlsoKanji", "seeAlsoSense")
 );

 CREATE TABLE "JMdict_SenseAntonym" (
  "senseId" INTEGER NOT NULL REFERENCES "JMdict_Sense"("senseId"),
  "xrefEntryId" INTEGER NOT NULL,
+  "antonymReading" TEXT,
+  "antonymKanji" TEXT,
  "antonymSense" INTEGER,
  -- For some entries, the cross reference is ambiguous. This means that while the ingestion
  -- has determined some xrefEntryId, it is not guaranteed to be the correct one.
@@ -171,9 +177,11 @@ CREATE TABLE "JMdict_SenseAntonym" (
    END
  ) VIRTUAL,

+  FOREIGN KEY ("xrefEntryId", "antonymKanji") REFERENCES "JMdict_KanjiElement"("entryId", "reading"),
+  FOREIGN KEY ("xrefEntryId", "antonymReading") REFERENCES "JMdict_ReadingElement"("entryId", "reading"),
  FOREIGN KEY ("antonymSenseKey") REFERENCES "JMdict_Sense"("senseId"),

-  PRIMARY KEY ("senseId", "xrefEntryId", "antonymSense")
+  UNIQUE("senseId", "xrefEntryId", "antonymReading", "antonymKanji", "antonymSense")
 );

 -- These cross references are going to be mostly accessed from a sense
@@ -224,21 +232,13 @@ CREATE TABLE "JMdict_SenseDialect" (
 CREATE TABLE "JMdict_SenseGlossary" (
  "senseId" INTEGER NOT NULL REFERENCES "JMdict_Sense"("senseId"),
  "phrase" TEXT NOT NULL,
-  -- "language" CHAR(3) NOT NULL DEFAULT "eng",
-  -- PRIMARY KEY ("senseId", "language", "phrase")
-  PRIMARY KEY ("senseId", "phrase")
+  "language" CHAR(3) NOT NULL DEFAULT "eng",
+  "type" TEXT,
+  PRIMARY KEY ("senseId", "language", "phrase")
 ) WITHOUT ROWID;

 CREATE INDEX "JMdict_SenseGlossary_byPhrase" ON JMdict_SenseGlossary("phrase");

-CREATE TABLE "JMdict_SenseGlossaryType" (
-  "senseId" INTEGER NOT NULL REFERENCES "JMdict_Sense"("senseId"),
-  "phrase" TEXT NOT NULL,
-  "type" TEXT NOT NULL,
-  PRIMARY KEY ("senseId", "phrase", "type"),
-  FOREIGN KEY ("senseId", "phrase") REFERENCES "JMdict_SenseGlossary"("senseId", "phrase")
-) WITHOUT ROWID;
-
 CREATE TABLE "JMdict_SenseInfo" (
  "senseId" INTEGER NOT NULL REFERENCES "JMdict_Sense"("senseId"),
  "info" TEXT NOT NULL,
--- a/migrations/0005_JMDict_search_index_tables.sql
+++ b/migrations/0005_JMDict_search_index_tables.sql
@@ -46,7 +46,7 @@ SELECT
  + (("spec" IS 2) * 5)
  + (("gai" IS 1) * 10)
  + (("gai" IS 2) * 5)
-  -- + (("orderNum" IS 0) * 20)
+  + (("orderNum" IS 0) * 20)
  - (substr(COALESCE("JMdict_JLPTTag"."jlptLevel", 'N0'), 2) * -5)
  AS "score"
 FROM "JMdict_ReadingElement"
@@ -77,7 +77,7 @@ SELECT
  + (("spec" IS 2) * 5)
  + (("gai" IS 1) * 10)
  + (("gai" IS 2) * 5)
-  -- + (("orderNum" IS 0) * 20)
+  + (("orderNum" IS 0) * 20)
  - (substr(COALESCE("JMdict_JLPTTag"."jlptLevel", 'N0'), 2) * -5)
  AS "score"
 FROM "JMdict_KanjiElement"
@@ -103,8 +103,7 @@ BEGIN
  )
  SELECT "elementId", "score", "common"
  FROM "JMdict_EntryScoreView_Reading"
-  WHERE "elementId" = NEW."elementId"
-    AND "score" > 0;
+  WHERE "elementId" = NEW."elementId";
 END;

 CREATE TRIGGER "JMdict_EntryScore_Update_JMdict_ReadingElement"
@@ -117,10 +116,6 @@ BEGIN
    "common" = "JMdict_EntryScoreView_Reading"."common"
  FROM "JMdict_EntryScoreView_Reading"
  WHERE "elementId" = NEW."elementId";
-
-  DELETE FROM "JMdict_EntryScore"
-  WHERE "elementId" = NEW."elementId"
-    AND "score" <= 0;
 END;

 CREATE TRIGGER "JMdict_EntryScore_Delete_JMdict_ReadingElement"
@@ -142,8 +137,7 @@ BEGIN
  )
  SELECT "elementId", "score", "common"
  FROM "JMdict_EntryScoreView_Kanji"
-  WHERE "elementId" = NEW."elementId"
-    AND "score" > 0;
+  WHERE "elementId" = NEW."elementId";
 END;

 CREATE TRIGGER "JMdict_EntryScore_Update_JMdict_KanjiElement"
@@ -156,10 +150,6 @@ BEGIN
    "common" = "JMdict_EntryScoreView_Kanji"."common"
  FROM "JMdict_EntryScoreView_Kanji"
  WHERE "elementId" = NEW."elementId";
-
-  DELETE FROM "JMdict_EntryScore"
-  WHERE "elementId" = NEW."elementId"
-    AND "score" <= 0;
 END;

 CREATE TRIGGER "JMdict_EntryScore_Delete_JMdict_KanjiElement"
@@ -209,8 +199,4 @@ BEGIN
  WHERE "JMdict_EntryScoreView"."entryId" = OLD."entryId"
    AND "JMdict_EntryScore"."entryId" = OLD."entryId"
    AND "JMdict_EntryScoreView"."elementId" = "JMdict_EntryScore"."elementId";
-
-  DELETE FROM "JMdict_EntryScore"
-  WHERE "elementId" = OLD."elementId"
-    AND "score" <= 0;
 END;
--- a/migrations/0006_RADKFILE.sql
+++ b/migrations/0006_RADKFILE.sql
@@ -18,3 +18,4 @@ CREATE TABLE "RADKFILE" (
 ) WITHOUT ROWID;

 CREATE INDEX "RADK" ON "RADKFILE"("radical");
+CREATE INDEX "KRAD" ON "RADKFILE"("kanji");
--- a/migrations/0008_XREF__KANJIDIC_Radical___RADKFILE.sql
+++ b/migrations/0008_XREF__KANJIDIC_Radical___RADKFILE.sql
@@ -4,6 +4,7 @@ CREATE TABLE "XREF__KANJIDIC_Radical__RADKFILE"(
  PRIMARY KEY ("radicalId", "radicalSymbol")
 ) WITHOUT ROWID;

+CREATE INDEX "XREF__KANJIDIC_Radical__RADKFILE__byRadicalId" ON "XREF__KANJIDIC_Radical__RADKFILE"("radicalId");
 CREATE INDEX "XREF__KANJIDIC_Radical__RADKFILE__byRadicalSymbol" ON "XREF__KANJIDIC_Radical__RADKFILE"("radicalSymbol");

 /* Source: https://ctext.org/kangxi-zidian */
--- a/migrations/0009_XREF__JMdict_KanjiElement__KANJIDIC_Character.sql
+++ b/migrations/0009_XREF__JMdict_KanjiElement__KANJIDIC_Character.sql
@@ -1,10 +1,10 @@
-- CREATE TABLE "XREF__JMdict_KanjiElement__KANJIDIC_Character"(
--   "entryId" INTEGER NOT NULL,
--   "reading" TEXT NOT NULL,
--   "kanji" CHAR(1) NOT NULL REFERENCES "KANJIDIC_Character"("literal"),
--   PRIMARY KEY ("entryId", "reading", "kanji"),
--   FOREIGN KEY ("entryId", "reading") REFERENCES "JMdict_KanjiElement"("entryId", "reading")
-- ) WITHOUT ROWID;
+CREATE TABLE "XREF__JMdict_KanjiElement__KANJIDIC_Character"(
+  "entryId" INTEGER NOT NULL,
+  "reading" TEXT NOT NULL,
+  "kanji" CHAR(1) NOT NULL REFERENCES "KANJIDIC_Character"("literal"),
+  PRIMARY KEY ("entryId", "reading", "kanji"),
+  FOREIGN KEY ("entryId", "reading") REFERENCES "JMdict_KanjiElement"("entryId", "reading")
+) WITHOUT ROWID;

-- CREATE INDEX "XREF__JMdict_KanjiElement__KANJIDIC_Character__byEntryId_byReading" ON "XREF__JMdict_KanjiElement__KANJIDIC_Character"("entryId", "reading");
-- CREATE INDEX "XREF__JMdict_KanjiElement__KANJIDIC_Character__byKanji" ON "XREF__JMdict_KanjiElement__KANJIDIC_Character"("kanji");
+CREATE INDEX "XREF__JMdict_KanjiElement__KANJIDIC_Character__byEntryId_byReading" ON "XREF__JMdict_KanjiElement__KANJIDIC_Character"("entryId", "reading");
+CREATE INDEX "XREF__JMdict_KanjiElement__KANJIDIC_Character__byKanji" ON "XREF__JMdict_KanjiElement__KANJIDIC_Character"("kanji");
--- a/migrations/0010_Views.sql
+++ b/migrations/0010_Views.sql
@@ -75,7 +75,13 @@ SELECT DISTINCT "radical" FROM "RADKFILE";
 CREATE VIEW "JMdict_CombinedEntryScore"
 AS
 SELECT
+  CASE
+    WHEN "JMdict_EntryScore"."type" = 'k'
+      THEN (SELECT entryId FROM "JMdict_KanjiElement" WHERE "elementId" = "JMdict_EntryScore"."elementId")
+    WHEN "JMdict_EntryScore"."type" = 'r'
+      THEN (SELECT entryId FROM "JMdict_ReadingElement" WHERE "elementId" = "JMdict_EntryScore"."elementId")
+  END AS "entryId",
  MAX("JMdict_EntryScore"."score") AS "score",
  MAX("JMdict_EntryScore"."common") AS "common"
 FROM "JMdict_EntryScore"
-GROUP BY "JMdict_EntryScore"."entryId";
+GROUP BY "entryId";
--- a/migrations/0011_KanjiVG.sql
+++ b/migrations/0011_KanjiVG.sql
@@ -0,0 +1,45 @@
+CREATE TABLE "KanjiVG_Version" (
+  "version" VARCHAR(10) PRIMARY KEY NOT NULL,
+  "date" DATE NOT NULL,
+  "hash" VARCHAR(64) NOT NULL
+) WITHOUT ROWID;
+
+CREATE TRIGGER "KanjiVG_Version_SingleRow"
+BEFORE INSERT ON "KanjiVG_Version"
+WHEN (SELECT COUNT(*) FROM "KanjiVG_Version") >= 1
+BEGIN
+    SELECT RAISE(FAIL, 'Only one row allowed in KanjiVG_Version');
+END;
+
+CREATE TABLE "KanjiVG_Entry" (
+  "character" CHAR(1) PRIMARY KEY NOT NULL
+) WITHOUT ROWID;
+
+CREATE TABLE "KanjiVG_StrokeNumber" (
+  "character" CHAR(1) NOT NULL REFERENCES "KanjiVG_Entry"("character"),
+  "strokeNum" INTEGER NOT NULL,
+  "x" REAL NOT NULL,
+  "y" REAL NOT NULL,
+  PRIMARY KEY ("character", "strokeNum")
+) WITHOUT ROWID;
+
+CREATE TABLE "KanjiVG_Path" (
+  "character" CHAR(1) NOT NULL REFERENCES "KanjiVG_Entry"("character"),
+  "pathId" TEXT NOT NULL,
+  "type" VARCHAR(10) NOT NULL,
+  "svgPath" TEXT NOT NULL,
+  PRIMARY KEY ("character", "pathId")
+) WITHOUT ROWID;
+
+CREATE TABLE "KanjiVG_PathGroup" (
+  "character" CHAR(1) NOT NULL REFERENCES "KanjiVG_Entry"("character"),
+  "groupId" TEXT NOT NULL,
+  "parentGroupId" TEXT REFERENCES "KanjiVG_PathGroup"("groupId"),
+  "element" TEXT,
+  "original" TEXT,
+  "position" VARCHAR(10),
+  "radical" TEXT,
+  "part" INTEGER,
+  PRIMARY KEY ("character", "groupId"),
+  CHECK ("position" IN ('bottom', 'kamae', 'kamaec', 'left', 'middle', 'nyo', 'nyoc', 'right', 'tare', 'tarec', 'top') OR "position" IS NULL)
+) WITHOUT ROWID;
--- a/nix/database.nix
+++ b/nix/database.nix
@@ -19,31 +19,13 @@ stdenvNoCC.mkDerivation {
    sqlite
  ];

-  env = {
-    JMDICT_VERSION = jmdict.version;
-    JMDICT_DATE = jmdict.date;
-    JMDICT_HASH = jmdict.hash;
-
-    KANJIDIC_VERSION = kanjidic2.version;
-    KANJIDIC_DATE = kanjidic2.date;
-    KANJIDIC_HASH = kanjidic2.hash;
-
-    RADKFILE_VERSION = radkfile.version;
-    RADKFILE_DATE = radkfile.date;
-    RADKFILE_HASH = radkfile.hash;
-
-    TANOS_JLPT_VERSION = tanos-jlpt.version;
-    TANOS_JLPT_DATE = tanos-jlpt.date;
-    TANOS_JLPT_HASH = tanos-jlpt.hash;
-  };
-
  buildPhase = ''
    runHook preBuild

    mkdir -p data
    ln -s '${jmdict}'/* data/
-    ln -s '${kanjidic2}'/* data/
    ln -s '${radkfile}'/* data/
+    ln -s '${kanjidic2}'/* data/
    ln -s '${tanos-jlpt}' data/tanos-jlpt

    for migration in migrations/*.sql; do
--- a/test/search/radical_search_test.dart
+++ b/test/search/radical_search_test.dart
@@ -1,79 +0,0 @@
-import 'package:collection/collection.dart';
-import 'package:jadb/const_data/radicals.dart';
-import 'package:jadb/search.dart';
-import 'package:jadb/table_names/radkfile.dart';
-import 'package:test/test.dart';
-
-import 'setup_database_connection.dart';
-
-void main() {
-  test(
-    'All constant radicals should exist in the database',
-    () async {
-      final connection = await setupDatabaseConnection();
-      final allRadicalsInDb = await connection.query(
-        RADKFILETableNames.radkfile,
-        columns: ['radical'],
-        distinct: true,
-      );
-
-      final radicalsInDb = allRadicalsInDb
-          .map((e) => e['radical'] as String)
-          .toSet();
-
-      final missingRadicals = radicals.values.flattenedToSet.difference(
-        radicalsInDb,
-      );
-
-      expect(
-        missingRadicals,
-        isEmpty,
-        reason: 'Missing radicals in database: $missingRadicals',
-      );
-    },
-    skip: 'Test is valid, code is broken, fix me',
-  );
-
-  test(
-    'All radicals in database should be in the constant radical list',
-    () async {
-      final connection = await setupDatabaseConnection();
-      final allRadicalsInDb = await connection.query(
-        RADKFILETableNames.radkfile,
-        columns: ['radical'],
-        distinct: true,
-      );
-
-      final radicalsInDb = allRadicalsInDb
-          .map((e) => e['radical'] as String)
-          .toSet();
-
-      final extraRadicals = radicalsInDb.difference(
-        radicals.values.flattenedToSet,
-      );
-
-      expect(
-        extraRadicals,
-        isEmpty,
-        reason:
-            'Extra radicals in database missing in the constant list: $extraRadicals',
-      );
-    },
-    skip: 'Test is valid, code is broken, fix me',
-  );
-
-  group(
-    'All radicals should return results',
-    () {
-      for (final radical in radicals.values.flattened) {
-        test(' - $radical', () async {
-          final connection = await setupDatabaseConnection();
-          final result = await connection.jadbSearchKanjiByRadicals([radical]);
-          expect(result, isNotEmpty);
-        });
-      }
-    },
-    skip:
-        'These will be automatically fixed once the other radical tests are passing',
-  );
-}
--- a/test/util/romaji_transliteration_test.dart
+++ b/test/util/romaji_transliteration_test.dart
@@ -8,7 +8,7 @@ void main() {
      expect(result, 'かたまり');
    });

-    test('Basic test with dakuten', () {
+    test('Basic test with diacritics', () {
      final result = transliterateLatinToHiragana('gadamari');
      expect(result, 'がだまり');
    });
@@ -54,7 +54,7 @@ void main() {

    test('Basic test', expectSpans('katamari', ['か', 'た', 'ま', 'り']));
    test(
-      'Basic test with dakuten',
+      'Basic test with diacritics',
      expectSpans('gadamari', ['が', 'だ', 'ま', 'り']),
    );
    test('wi and we', expectSpans('wiwe', ['うぃ', 'うぇ']));
@@ -72,7 +72,7 @@ void main() {
      expect(result, 'katamari');
    });

-    test('Basic test with dakuten', () {
+    test('Basic test with diacritics', () {
      final result = transliterateHiraganaToLatin('がだまり');
      expect(result, 'gadamari');
    });
@@ -91,21 +91,6 @@ void main() {
      final result = transliterateHiraganaToLatin('かっぱ');
      expect(result, 'kappa');
    });
-
-    test('Iteration mark', () {
-      final result = transliterateHiraganaToLatin('さゝき');
-      expect(result, 'sasaki');
-    }, skip: 'Not yet implemented');
-
-    test('Iteration mark with dakuten', () {
-      final result = transliterateHiraganaToLatin('あひゞき');
-      expect(result, 'ahibiki');
-    }, skip: 'Not yet implemented');
-
-    test('Yori', () {
-      final result = transliterateHiraganaToLatin('ゟ');
-      expect(result, 'yori');
-    }, skip: 'Not yet implemented');
  });

  group('Hiragana -> Romaji Spans', () {
@@ -125,7 +110,7 @@ void main() {

    test('Basic test', expectSpans('かたまり', ['ka', 'ta', 'ma', 'ri']));
    test(
-      'Basic test with dakuten',
+      'Basic test with diacritics',
      expectSpans('がだまり', ['ga', 'da', 'ma', 'ri']),
    );
    test('wi and we', expectSpans('うぃうぇ', ['whi', 'whe']));
@@ -133,17 +118,5 @@ void main() {

    // TODO: fix the implementation
    // test('Double consonant', expectSpans('かっぱ', ['ka', 'ppa']));
-
-    test(
-      'Iteration mark',
-      expectSpans('さゝき', ['sa', 'sa', 'ki']),
-      skip: 'Not yet implemented',
-    );
-    test(
-      'Iteration mark with dakuten',
-      expectSpans('あひゞき', ['a', 'hi', 'bi', 'ki']),
-      skip: 'Not yet implemented',
-    );
-    test('Yori', expectSpans('ゟ', ['yori']), skip: 'Not yet implemented');
  });
 }