1 Commits

Author SHA1 Message Date
781e650f0b WIP: use ids for \{kanji,reading\}Element tables 2025-06-24 01:01:07 +02:00
79 changed files with 1823 additions and 2633 deletions

View File

@@ -1,38 +0,0 @@
name: "Build database"
on:
pull_request:
push:
jobs:
evals:
runs-on: debian-latest
steps:
- uses: actions/checkout@v6
- name: Install sudo
run: apt-get update && apt-get -y install sudo
- name: Install nix
uses: https://github.com/cachix/install-nix-action@v31
- name: Configure nix
run: echo -e "show-trace = true\nmax-jobs = auto\ntrusted-users = root\nexperimental-features = nix-command flakes\nbuild-users-group =" > /etc/nix/nix.conf
- name: Update database inputs
run: |
nix flake update jmdict-src
nix flake update jmdict-with-examples-src
nix flake update radkfile-src
nix flake update kanjidic2-src
- name: Build database
run: nix build .#database -L
- name: Upload database as artifact
uses: actions/upload-artifact@v3
with:
name: jadb-${{ gitea.sha }}.zip
path: result/jadb.sqlite
if-no-files-found: error
retention-days: 15
# Already compressed
compression: 0

View File

@@ -1,31 +0,0 @@
name: "Run tests"
on:
pull_request:
push:
jobs:
evals:
runs-on: debian-latest
steps:
- uses: actions/checkout@v6
- name: Install sudo
run: apt-get update && apt-get -y install sudo
- name: Install nix
uses: https://github.com/cachix/install-nix-action@v31
- name: Configure nix
run: echo -e "show-trace = true\nmax-jobs = auto\ntrusted-users = root\nexperimental-features = nix-command flakes\nbuild-users-group =" > /etc/nix/nix.conf
- name: Update database inputs
run: |
nix flake update jmdict-src
nix flake update jmdict-with-examples-src
nix flake update radkfile-src
nix flake update kanjidic2-src
- name: Build database
run: nix build .#database -L
- name: Run tests
run: nix develop .# --command dart test

View File

@@ -16,26 +16,3 @@ Note that while the license for the code is MIT, the data has various licenses.
| **Tanos JLPT levels:** | https://www.tanos.co.uk/jlpt/ | | **Tanos JLPT levels:** | https://www.tanos.co.uk/jlpt/ |
| **Kangxi Radicals:** | https://ctext.org/kangxi-zidian | | **Kangxi Radicals:** | https://ctext.org/kangxi-zidian |
## Implementation details
### Word search
The word search procedure is currently split into 3 parts:
1. **Entry ID query**:
Use a complex query with various scoring factors to try to get list of
database ids pointing at dictionary entries, sorted by how likely we think this
word is the word that the caller is looking for. The output here is a `List<int>`
2. **Data Query**:
Takes the entry id list from the last search, and performs all queries needed to retrieve
all the dictionary data for those IDs. The result is a struct with a bunch of flattened lists
with data for all the dictionary entries. These lists are sorted by the order that the ids
were provided.
3. **Regrouping**:
Takes the flattened data, and regroups the items into structs with a more "hierarchical" structure.
All data tagged with the same ID will end up in the same struct. Returns a list of these structs.

View File

@@ -1,41 +0,0 @@
# This file configures the analyzer, which statically analyzes Dart code to
# check for errors, warnings, and lints.
#
# The issues identified by the analyzer are surfaced in the UI of Dart-enabled
# IDEs (https://dart.dev/tools#ides-and-editors). The analyzer can also be
# invoked from the command line by running `flutter analyze`.
# The following line activates a set of recommended lints for Flutter apps,
# packages, and plugins designed to encourage good coding practices.
include:
- package:lints/recommended.yaml
linter:
# The lint rules applied to this project can be customized in the
# section below to disable rules from the `package:flutter_lints/flutter.yaml`
# included above or to enable additional rules. A list of all available lints
# and their documentation is published at https://dart.dev/lints.
#
# Instead of disabling a lint rule for the entire project in the
# section below, it can also be suppressed for a single line of code
# or a specific dart file by using the `// ignore: name_of_lint` and
# `// ignore_for_file: name_of_lint` syntax on the line or in the file
# producing the lint.
rules:
always_declare_return_types: true
annotate_redeclares: true
avoid_print: false
avoid_setters_without_getters: true
avoid_slow_async_io: true
directives_ordering: true
eol_at_end_of_file: true
prefer_const_declarations: true
prefer_contains: true
prefer_final_fields: true
prefer_final_locals: true
prefer_single_quotes: true
use_key_in_widget_constructors: true
use_null_aware_elements: true
# Additional information about this file can be found at
# https://dart.dev/guides/language/analysis-options

View File

@@ -9,7 +9,7 @@ import 'package:jadb/cli/commands/query_word.dart';
Future<void> main(List<String> args) async { Future<void> main(List<String> args) async {
final runner = CommandRunner( final runner = CommandRunner(
'jadb', 'jadb',
'CLI tool to help creating and testing the jadb database', "CLI tool to help creating and testing the jadb database",
); );
runner.addCommand(CreateDb()); runner.addCommand(CreateDb());

18
flake.lock generated
View File

@@ -3,7 +3,7 @@
"jmdict-src": { "jmdict-src": {
"flake": false, "flake": false,
"locked": { "locked": {
"narHash": "sha256-lh46uougUzBrRhhwa7cOb32j5Jt9/RjBUhlVjwVzsII=", "narHash": "sha256-84P7r/fFlBnawy6yChrD9WMHmOWcEGWUmoK70N4rdGQ=",
"type": "file", "type": "file",
"url": "http://ftp.edrdg.org/pub/Nihongo/JMdict_e.gz" "url": "http://ftp.edrdg.org/pub/Nihongo/JMdict_e.gz"
}, },
@@ -15,7 +15,7 @@
"jmdict-with-examples-src": { "jmdict-with-examples-src": {
"flake": false, "flake": false,
"locked": { "locked": {
"narHash": "sha256-5oS2xDyetbuSM6ax3LUjYA3N60x+D3Hg41HEXGFMqLQ=", "narHash": "sha256-PM0sv7VcsCya2Ek02CI7hVwB3Jawn6bICSI+dsJK0yo=",
"type": "file", "type": "file",
"url": "http://ftp.edrdg.org/pub/Nihongo/JMdict_e_examp.gz" "url": "http://ftp.edrdg.org/pub/Nihongo/JMdict_e_examp.gz"
}, },
@@ -27,7 +27,7 @@
"kanjidic2-src": { "kanjidic2-src": {
"flake": false, "flake": false,
"locked": { "locked": {
"narHash": "sha256-orSeQqSxhn9TtX3anYtbiMEm7nFkuomGnIKoVIUR2CM=", "narHash": "sha256-Lc0wUPpuDKuMDv2t87//w3z20RX8SMJI2iIRtUJ8fn0=",
"type": "file", "type": "file",
"url": "https://www.edrdg.org/kanjidic/kanjidic2.xml.gz" "url": "https://www.edrdg.org/kanjidic/kanjidic2.xml.gz"
}, },
@@ -38,11 +38,11 @@
}, },
"nixpkgs": { "nixpkgs": {
"locked": { "locked": {
"lastModified": 1771848320, "lastModified": 1746904237,
"narHash": "sha256-0MAd+0mun3K/Ns8JATeHT1sX28faLII5hVLq0L3BdZU=", "narHash": "sha256-3e+AVBczosP5dCLQmMoMEogM57gmZ2qrVSrmq9aResQ=",
"owner": "NixOS", "owner": "NixOS",
"repo": "nixpkgs", "repo": "nixpkgs",
"rev": "2fc6539b481e1d2569f25f8799236694180c0993", "rev": "d89fc19e405cb2d55ce7cc114356846a0ee5e956",
"type": "github" "type": "github"
}, },
"original": { "original": {
@@ -54,13 +54,13 @@
"radkfile-src": { "radkfile-src": {
"flake": false, "flake": false,
"locked": { "locked": {
"narHash": "sha256-DHpMUE2Umje8PbzXUCS6pHZeXQ5+WTxbjSkGU3erDHQ=", "narHash": "sha256-rO2z5GPt3g6osZOlpyWysmIbRV2Gw4AR4XvngVTHNpk=",
"type": "file", "type": "file",
"url": "http://ftp.edrdg.org/pub/Nihongo/radkfile.gz" "url": "http://ftp.usf.edu/pub/ftp.monash.edu.au/pub/nihongo/radkfile.gz"
}, },
"original": { "original": {
"type": "file", "type": "file",
"url": "http://ftp.edrdg.org/pub/Nihongo/radkfile.gz" "url": "http://ftp.usf.edu/pub/ftp.monash.edu.au/pub/nihongo/radkfile.gz"
} }
}, },
"root": { "root": {

View File

@@ -16,7 +16,7 @@
}; };
radkfile-src = { radkfile-src = {
url = "http://ftp.edrdg.org/pub/Nihongo/radkfile.gz"; url = "http://ftp.usf.edu/pub/ftp.monash.edu.au/pub/nihongo/radkfile.gz";
flake = false; flake = false;
}; };
@@ -83,7 +83,7 @@
sqlite-interactive sqlite-interactive
sqlite-analyzer sqlite-analyzer
sqlite-web sqlite-web
# sqlint sqlint
sqlfluff sqlfluff
]; ];
env = { env = {
@@ -104,30 +104,12 @@
platforms = lib.platforms.all; platforms = lib.platforms.all;
}; };
src = builtins.filterSource (path: type: let src = lib.cleanSource ./.;
baseName = baseNameOf (toString path);
in !(lib.any (b: b) [
(!(lib.cleanSourceFilter path type))
(baseName == ".github" && type == "directory")
(baseName == ".gitea" && type == "directory")
(baseName == "nix" && type == "directory")
(baseName == ".envrc" && type == "regular")
(baseName == "flake.lock" && type == "regular")
(baseName == "flake.nix" && type == "regular")
(baseName == ".sqlfluff" && type == "regular")
])) ./.;
in forAllSystems (system: pkgs: { in forAllSystems (system: pkgs: {
default = self.packages.${system}.database; default = self.packages.${system}.database;
filteredSource = pkgs.runCommandLocal "filtered-source" { } ''
ln -s ${src} $out
'';
jmdict = pkgs.callPackage ./nix/jmdict.nix { jmdict = pkgs.callPackage ./nix/jmdict.nix {
inherit jmdict-src jmdict-with-examples-src edrdgMetadata; inherit jmdict-src jmdict-with-examples-src edrdgMetadata;
}; };
radkfile = pkgs.callPackage ./nix/radkfile.nix { radkfile = pkgs.callPackage ./nix/radkfile.nix {

View File

@@ -16,15 +16,14 @@ abstract class Element extends SQLWritable {
this.nf, this.nf,
}); });
@override
Map<String, Object?> get sqlValue => { Map<String, Object?> get sqlValue => {
'reading': reading, 'reading': reading,
'news': news, 'news': news,
'ichi': ichi, 'ichi': ichi,
'spec': spec, 'spec': spec,
'gai': gai, 'gai': gai,
'nf': nf, 'nf': nf,
}; };
} }
class KanjiElement extends Element { class KanjiElement extends Element {
@@ -34,19 +33,26 @@ class KanjiElement extends Element {
KanjiElement({ KanjiElement({
this.info = const [], this.info = const [],
required this.orderNum, required this.orderNum,
required super.reading, required String reading,
super.news, int? news,
super.ichi, int? ichi,
super.spec, int? spec,
super.gai, int? gai,
super.nf, int? nf,
}); }) : super(
reading: reading,
news: news,
ichi: ichi,
spec: spec,
gai: gai,
nf: nf,
);
@override @override
Map<String, Object?> get sqlValue => { Map<String, Object?> get sqlValue => {
...super.sqlValue, ...super.sqlValue,
'orderNum': orderNum, 'orderNum': orderNum,
}; };
} }
class ReadingElement extends Element { class ReadingElement extends Element {
@@ -60,20 +66,27 @@ class ReadingElement extends Element {
required this.readingDoesNotMatchKanji, required this.readingDoesNotMatchKanji,
this.info = const [], this.info = const [],
this.restrictions = const [], this.restrictions = const [],
required super.reading, required String reading,
super.news, int? news,
super.ichi, int? ichi,
super.spec, int? spec,
super.gai, int? gai,
super.nf, int? nf,
}); }) : super(
reading: reading,
news: news,
ichi: ichi,
spec: spec,
gai: gai,
nf: nf,
);
@override @override
Map<String, Object?> get sqlValue => { Map<String, Object?> get sqlValue => {
...super.sqlValue, ...super.sqlValue,
'orderNum': orderNum, 'orderNum': orderNum,
'readingDoesNotMatchKanji': readingDoesNotMatchKanji, 'readingDoesNotMatchKanji': readingDoesNotMatchKanji,
}; };
} }
class LanguageSource extends SQLWritable { class LanguageSource extends SQLWritable {
@@ -91,11 +104,11 @@ class LanguageSource extends SQLWritable {
@override @override
Map<String, Object?> get sqlValue => { Map<String, Object?> get sqlValue => {
'language': language, 'language': language,
'phrase': phrase, 'phrase': phrase,
'fullyDescribesSense': fullyDescribesSense, 'fullyDescribesSense': fullyDescribesSense,
'constructedFromSmallerWords': constructedFromSmallerWords, 'constructedFromSmallerWords': constructedFromSmallerWords,
}; };
} }
class Glossary extends SQLWritable { class Glossary extends SQLWritable {
@@ -103,41 +116,48 @@ class Glossary extends SQLWritable {
final String phrase; final String phrase;
final String? type; final String? type;
const Glossary({required this.language, required this.phrase, this.type}); const Glossary({
required this.language,
required this.phrase,
this.type,
});
@override
Map<String, Object?> get sqlValue => { Map<String, Object?> get sqlValue => {
'language': language, 'language': language,
'phrase': phrase, 'phrase': phrase,
'type': type, 'type': type,
}; };
} }
final kanaRegex = RegExp( final kanaRegex =
r'^[\p{Script=Katakana}\p{Script=Hiragana}ー]+$', RegExp(r'^[\p{Script=Katakana}\p{Script=Hiragana}ー]+$', unicode: true);
unicode: true,
);
class XRefParts { class XRefParts {
final String? kanjiRef; final String? kanjiRef;
final String? readingRef; final String? readingRef;
final int? senseOrderNum; final int? senseOrderNum;
const XRefParts({this.kanjiRef, this.readingRef, this.senseOrderNum}) const XRefParts({
: assert(kanjiRef != null || readingRef != null); this.kanjiRef,
this.readingRef,
this.senseOrderNum,
}) : assert(kanjiRef != null || readingRef != null);
Map<String, Object?> toJson() => { Map<String, Object?> toJson() => {
'kanjiRef': kanjiRef, 'kanjiRef': kanjiRef,
'readingRef': readingRef, 'readingRef': readingRef,
'senseOrderNum': senseOrderNum, 'senseOrderNum': senseOrderNum,
}; };
} }
class XRef { class XRef {
final String entryId; final String entryId;
final String reading; final String reading;
const XRef({required this.entryId, required this.reading}); const XRef({
required this.entryId,
required this.reading,
});
} }
class Sense extends SQLWritable { class Sense extends SQLWritable {
@@ -173,9 +193,9 @@ class Sense extends SQLWritable {
@override @override
Map<String, Object?> get sqlValue => { Map<String, Object?> get sqlValue => {
'senseId': senseId, 'senseId': senseId,
'orderNum': orderNum, 'orderNum': orderNum,
}; };
bool get isEmpty => bool get isEmpty =>
antonyms.isEmpty && antonyms.isEmpty &&
@@ -204,6 +224,5 @@ class Entry extends SQLWritable {
required this.senses, required this.senses,
}); });
@override
Map<String, Object?> get sqlValue => {'entryId': entryId}; Map<String, Object?> get sqlValue => {'entryId': entryId};
} }

View File

@@ -18,20 +18,18 @@ ResolvedXref resolveXref(
XRefParts xref, XRefParts xref,
) { ) {
List<Entry> candidateEntries = switch ((xref.kanjiRef, xref.readingRef)) { List<Entry> candidateEntries = switch ((xref.kanjiRef, xref.readingRef)) {
(null, null) => throw Exception( (null, null) =>
'Xref $xref has no kanji or reading reference', throw Exception('Xref $xref has no kanji or reading reference'),
), (String k, null) => entriesByKanji[k]!.toList(),
(final String k, null) => entriesByKanji[k]!.toList(), (null, String r) => entriesByReading[r]!.toList(),
(null, final String r) => entriesByReading[r]!.toList(), (String k, String r) =>
(final String k, final String r) =>
entriesByKanji[k]!.intersection(entriesByReading[r]!).toList(), entriesByKanji[k]!.intersection(entriesByReading[r]!).toList(),
}; };
// Filter out entries that don't have the number of senses specified in the xref // Filter out entries that don't have the number of senses specified in the xref
if (xref.senseOrderNum != null) { if (xref.senseOrderNum != null) {
candidateEntries.retainWhere( candidateEntries
(entry) => entry.senses.length >= xref.senseOrderNum!, .retainWhere((entry) => entry.senses.length >= xref.senseOrderNum!);
);
} }
// If the xref has a reading ref but no kanji ref, and there are multiple // If the xref has a reading ref but no kanji ref, and there are multiple
@@ -40,9 +38,8 @@ ResolvedXref resolveXref(
if (xref.kanjiRef == null && if (xref.kanjiRef == null &&
xref.readingRef != null && xref.readingRef != null &&
candidateEntries.length > 1) { candidateEntries.length > 1) {
final candidatesWithEmptyKanji = candidateEntries final candidatesWithEmptyKanji =
.where((entry) => entry.kanji.isEmpty) candidateEntries.where((entry) => entry.kanji.length == 0).toList();
.toList();
if (candidatesWithEmptyKanji.isNotEmpty) { if (candidatesWithEmptyKanji.isNotEmpty) {
candidateEntries = candidatesWithEmptyKanji; candidateEntries = candidatesWithEmptyKanji;
@@ -53,7 +50,7 @@ ResolvedXref resolveXref(
// entry in case there are multiple candidates left. // entry in case there are multiple candidates left.
candidateEntries.sortBy<num>((entry) => entry.senses.length); candidateEntries.sortBy<num>((entry) => entry.senses.length);
if (candidateEntries.isEmpty) { if (candidateEntries.length == 0) {
throw Exception( throw Exception(
'SKIPPING: Xref $xref has ${candidateEntries.length} entries, ' 'SKIPPING: Xref $xref has ${candidateEntries.length} entries, '
'kanjiRef: ${xref.kanjiRef}, readingRef: ${xref.readingRef}, ' 'kanjiRef: ${xref.kanjiRef}, readingRef: ${xref.readingRef}, '
@@ -75,43 +72,51 @@ Future<void> seedJMDictData(List<Entry> entries, Database db) async {
print(' [JMdict] Batch 1 - Kanji and readings'); print(' [JMdict] Batch 1 - Kanji and readings');
Batch b = db.batch(); Batch b = db.batch();
int elementId = 0;
for (final e in entries) { for (final e in entries) {
b.insert(JMdictTableNames.entry, e.sqlValue); b.insert(JMdictTableNames.entry, e.sqlValue);
for (final k in e.kanji) { for (final k in e.kanji) {
elementId++;
b.insert( b.insert(
JMdictTableNames.kanjiElement, JMdictTableNames.kanjiElement,
k.sqlValue..addAll({'entryId': e.entryId, 'elementId': elementId}), k.sqlValue..addAll({'entryId': e.entryId}),
); );
for (final i in k.info) { for (final i in k.info) {
b.insert(JMdictTableNames.kanjiInfo, { b.insert(
'elementId': elementId, JMdictTableNames.kanjiInfo,
'info': i, {
}); 'entryId': e.entryId,
'reading': k.reading,
'info': i,
},
);
} }
} }
for (final r in e.readings) { for (final r in e.readings) {
elementId++;
b.insert( b.insert(
JMdictTableNames.readingElement, JMdictTableNames.readingElement,
r.sqlValue..addAll({'entryId': e.entryId, 'elementId': elementId}), r.sqlValue..addAll({'entryId': e.entryId}),
); );
for (final i in r.info) { for (final i in r.info) {
b.insert(JMdictTableNames.readingInfo, { b.insert(
'elementId': elementId, JMdictTableNames.readingInfo,
'info': i, {
}); 'entryId': e.entryId,
'reading': r.reading,
'info': i,
},
);
} }
for (final res in r.restrictions) { for (final res in r.restrictions) {
b.insert(JMdictTableNames.readingRestriction, { b.insert(
'elementId': elementId, JMdictTableNames.readingRestriction,
'restriction': res, {
}); 'entryId': e.entryId,
'reading': r.reading,
'restriction': res,
},
);
} }
} }
} }
@@ -124,20 +129,16 @@ Future<void> seedJMDictData(List<Entry> entries, Database db) async {
for (final e in entries) { for (final e in entries) {
for (final s in e.senses) { for (final s in e.senses) {
b.insert( b.insert(
JMdictTableNames.sense, JMdictTableNames.sense, s.sqlValue..addAll({'entryId': e.entryId}));
s.sqlValue..addAll({'entryId': e.entryId}),
);
for (final d in s.dialects) { for (final d in s.dialects) {
b.insert(JMdictTableNames.senseDialect, { b.insert(
'senseId': s.senseId, JMdictTableNames.senseDialect,
'dialect': d, {'senseId': s.senseId, 'dialect': d},
}); );
} }
for (final f in s.fields) { for (final f in s.fields) {
b.insert(JMdictTableNames.senseField, { b.insert(
'senseId': s.senseId, JMdictTableNames.senseField, {'senseId': s.senseId, 'field': f});
'field': f,
});
} }
for (final i in s.info) { for (final i in s.info) {
b.insert(JMdictTableNames.senseInfo, {'senseId': s.senseId, 'info': i}); b.insert(JMdictTableNames.senseInfo, {'senseId': s.senseId, 'info': i});
@@ -149,18 +150,16 @@ Future<void> seedJMDictData(List<Entry> entries, Database db) async {
b.insert(JMdictTableNames.sensePOS, {'senseId': s.senseId, 'pos': p}); b.insert(JMdictTableNames.sensePOS, {'senseId': s.senseId, 'pos': p});
} }
for (final rk in s.restrictedToKanji) { for (final rk in s.restrictedToKanji) {
b.insert(JMdictTableNames.senseRestrictedToKanji, { b.insert(
'entryId': e.entryId, JMdictTableNames.senseRestrictedToKanji,
'senseId': s.senseId, {'entryId': e.entryId, 'senseId': s.senseId, 'kanji': rk},
'kanji': rk, );
});
} }
for (final rr in s.restrictedToReading) { for (final rr in s.restrictedToReading) {
b.insert(JMdictTableNames.senseRestrictedToReading, { b.insert(
'entryId': e.entryId, JMdictTableNames.senseRestrictedToReading,
'senseId': s.senseId, {'entryId': e.entryId, 'senseId': s.senseId, 'reading': rr},
'reading': rr, );
});
} }
for (final ls in s.languageSource) { for (final ls in s.languageSource) {
b.insert( b.insert(
@@ -180,7 +179,7 @@ Future<void> seedJMDictData(List<Entry> entries, Database db) async {
await b.commit(noResult: true); await b.commit(noResult: true);
print(' [JMdict] Building xref trees'); print(' [JMdict] Building xref trees');
final SplayTreeMap<String, Set<Entry>> entriesByKanji = SplayTreeMap(); SplayTreeMap<String, Set<Entry>> entriesByKanji = SplayTreeMap();
for (final entry in entries) { for (final entry in entries) {
for (final kanji in entry.kanji) { for (final kanji in entry.kanji) {
@@ -191,7 +190,7 @@ Future<void> seedJMDictData(List<Entry> entries, Database db) async {
} }
} }
} }
final SplayTreeMap<String, Set<Entry>> entriesByReading = SplayTreeMap(); SplayTreeMap<String, Set<Entry>> entriesByReading = SplayTreeMap();
for (final entry in entries) { for (final entry in entries) {
for (final reading in entry.readings) { for (final reading in entry.readings) {
if (entriesByReading.containsKey(reading.reading)) { if (entriesByReading.containsKey(reading.reading)) {
@@ -214,14 +213,17 @@ Future<void> seedJMDictData(List<Entry> entries, Database db) async {
xref, xref,
); );
b.insert(JMdictTableNames.senseSeeAlso, { b.insert(
'senseId': s.senseId, JMdictTableNames.senseSeeAlso,
'xrefEntryId': resolvedEntry.entry.entryId, {
'seeAlsoKanji': xref.kanjiRef, 'senseId': s.senseId,
'seeAlsoReading': xref.readingRef, 'xrefEntryId': resolvedEntry.entry.entryId,
'seeAlsoSense': xref.senseOrderNum, 'seeAlsoKanji': xref.kanjiRef,
'ambiguous': resolvedEntry.ambiguous, 'seeAlsoReading': xref.readingRef,
}); 'seeAlsoSense': xref.senseOrderNum,
'ambiguous': resolvedEntry.ambiguous,
},
);
} }
for (final ant in s.antonyms) { for (final ant in s.antonyms) {

View File

@@ -8,17 +8,15 @@ List<int?> getPriorityValues(XmlElement e, String prefix) {
int? news, ichi, spec, gai, nf; int? news, ichi, spec, gai, nf;
for (final pri in e.findElements('${prefix}_pri')) { for (final pri in e.findElements('${prefix}_pri')) {
final txt = pri.innerText; final txt = pri.innerText;
if (txt.startsWith('news')) { if (txt.startsWith('news'))
news = int.parse(txt.substring(4)); news = int.parse(txt.substring(4));
} else if (txt.startsWith('ichi')) { else if (txt.startsWith('ichi'))
ichi = int.parse(txt.substring(4)); ichi = int.parse(txt.substring(4));
} else if (txt.startsWith('spec')) { else if (txt.startsWith('spec'))
spec = int.parse(txt.substring(4)); spec = int.parse(txt.substring(4));
} else if (txt.startsWith('gai')) { else if (txt.startsWith('gai'))
gai = int.parse(txt.substring(3)); gai = int.parse(txt.substring(3));
} else if (txt.startsWith('nf')) { else if (txt.startsWith('nf')) nf = int.parse(txt.substring(2));
nf = int.parse(txt.substring(2));
}
} }
return [news, ichi, spec, gai, nf]; return [news, ichi, spec, gai, nf];
} }
@@ -48,7 +46,10 @@ XRefParts parseXrefParts(String s) {
); );
} }
} else { } else {
result = XRefParts(kanjiRef: parts[0], readingRef: parts[1]); result = XRefParts(
kanjiRef: parts[0],
readingRef: parts[1],
);
} }
break; break;
@@ -80,48 +81,45 @@ List<Entry> parseJMDictData(XmlElement root) {
final List<ReadingElement> readingEls = []; final List<ReadingElement> readingEls = [];
final List<Sense> senses = []; final List<Sense> senses = [];
for (final (kanjiNum, kEle) in entry.findElements('k_ele').indexed) { for (final (kanjiNum, k_ele) in entry.findElements('k_ele').indexed) {
final kePri = getPriorityValues(kEle, 'ke'); final ke_pri = getPriorityValues(k_ele, 'ke');
kanjiEls.add( kanjiEls.add(
KanjiElement( KanjiElement(
orderNum: kanjiNum + 1, orderNum: kanjiNum + 1,
info: kEle info: k_ele
.findElements('ke_inf') .findElements('ke_inf')
.map((e) => e.innerText.substring(1, e.innerText.length - 1)) .map((e) => e.innerText.substring(1, e.innerText.length - 1))
.toList(), .toList(),
reading: kEle.findElements('keb').first.innerText, reading: k_ele.findElements('keb').first.innerText,
news: kePri[0], news: ke_pri[0],
ichi: kePri[1], ichi: ke_pri[1],
spec: kePri[2], spec: ke_pri[2],
gai: kePri[3], gai: ke_pri[3],
nf: kePri[4], nf: ke_pri[4],
), ),
); );
} }
for (final (orderNum, rEle) in entry.findElements('r_ele').indexed) { for (final (orderNum, r_ele) in entry.findElements('r_ele').indexed) {
final rePri = getPriorityValues(rEle, 're'); final re_pri = getPriorityValues(r_ele, 're');
final readingDoesNotMatchKanji = rEle final readingDoesNotMatchKanji =
.findElements('re_nokanji') r_ele.findElements('re_nokanji').isNotEmpty;
.isNotEmpty;
readingEls.add( readingEls.add(
ReadingElement( ReadingElement(
orderNum: orderNum + 1, orderNum: orderNum + 1,
readingDoesNotMatchKanji: readingDoesNotMatchKanji, readingDoesNotMatchKanji: readingDoesNotMatchKanji,
info: rEle info: r_ele
.findElements('re_inf') .findElements('re_inf')
.map((e) => e.innerText.substring(1, e.innerText.length - 1)) .map((e) => e.innerText.substring(1, e.innerText.length - 1))
.toList(), .toList(),
restrictions: rEle restrictions:
.findElements('re_restr') r_ele.findElements('re_restr').map((e) => e.innerText).toList(),
.map((e) => e.innerText) reading: r_ele.findElements('reb').first.innerText,
.toList(), news: re_pri[0],
reading: rEle.findElements('reb').first.innerText, ichi: re_pri[1],
news: rePri[0], spec: re_pri[2],
ichi: rePri[1], gai: re_pri[3],
spec: rePri[2], nf: re_pri[4],
gai: rePri[3],
nf: rePri[4],
), ),
); );
} }
@@ -131,14 +129,10 @@ List<Entry> parseJMDictData(XmlElement root) {
final result = Sense( final result = Sense(
senseId: senseId, senseId: senseId,
orderNum: orderNum + 1, orderNum: orderNum + 1,
restrictedToKanji: sense restrictedToKanji:
.findElements('stagk') sense.findElements('stagk').map((e) => e.innerText).toList(),
.map((e) => e.innerText) restrictedToReading:
.toList(), sense.findElements('stagr').map((e) => e.innerText).toList(),
restrictedToReading: sense
.findElements('stagr')
.map((e) => e.innerText)
.toList(),
pos: sense pos: sense
.findElements('pos') .findElements('pos')
.map((e) => e.innerText.substring(1, e.innerText.length - 1)) .map((e) => e.innerText.substring(1, e.innerText.length - 1))

View File

@@ -13,33 +13,42 @@ class CodePoint extends SQLWritable {
@override @override
Map<String, Object?> get sqlValue => { Map<String, Object?> get sqlValue => {
'kanji': kanji, 'kanji': kanji,
'type': type, 'type': type,
'codepoint': codepoint, 'codepoint': codepoint,
}; };
} }
class Radical extends SQLWritable { class Radical extends SQLWritable {
final String kanji; final String kanji;
final int radicalId; final int radicalId;
const Radical({required this.kanji, required this.radicalId}); const Radical({
required this.kanji,
required this.radicalId,
});
@override @override
Map<String, Object?> get sqlValue => {'kanji': kanji, 'radicalId': radicalId}; Map<String, Object?> get sqlValue => {
'kanji': kanji,
'radicalId': radicalId,
};
} }
class StrokeMiscount extends SQLWritable { class StrokeMiscount extends SQLWritable {
final String kanji; final String kanji;
final int strokeCount; final int strokeCount;
const StrokeMiscount({required this.kanji, required this.strokeCount}); const StrokeMiscount({
required this.kanji,
required this.strokeCount,
});
@override @override
Map<String, Object?> get sqlValue => { Map<String, Object?> get sqlValue => {
'kanji': kanji, 'kanji': kanji,
'strokeCount': strokeCount, 'strokeCount': strokeCount,
}; };
} }
class Variant extends SQLWritable { class Variant extends SQLWritable {
@@ -55,10 +64,10 @@ class Variant extends SQLWritable {
@override @override
Map<String, Object?> get sqlValue => { Map<String, Object?> get sqlValue => {
'kanji': kanji, 'kanji': kanji,
'type': type, 'type': type,
'variant': variant, 'variant': variant,
}; };
} }
class DictionaryReference extends SQLWritable { class DictionaryReference extends SQLWritable {
@@ -74,10 +83,10 @@ class DictionaryReference extends SQLWritable {
@override @override
Map<String, Object?> get sqlValue => { Map<String, Object?> get sqlValue => {
'kanji': kanji, 'kanji': kanji,
'type': type, 'type': type,
'ref': ref, 'ref': ref,
}; };
} }
class DictionaryReferenceMoro extends SQLWritable { class DictionaryReferenceMoro extends SQLWritable {
@@ -95,11 +104,11 @@ class DictionaryReferenceMoro extends SQLWritable {
@override @override
Map<String, Object?> get sqlValue => { Map<String, Object?> get sqlValue => {
'kanji': kanji, 'kanji': kanji,
'ref': ref, 'ref': ref,
'volume': volume, 'volume': volume,
'page': page, 'page': page,
}; };
} }
class QueryCode extends SQLWritable { class QueryCode extends SQLWritable {
@@ -117,11 +126,11 @@ class QueryCode extends SQLWritable {
@override @override
Map<String, Object?> get sqlValue => { Map<String, Object?> get sqlValue => {
'kanji': kanji, 'kanji': kanji,
'code': code, 'code': code,
'type': type, 'type': type,
'skipMisclassification': skipMisclassification, 'skipMisclassification': skipMisclassification,
}; };
} }
class Reading extends SQLWritable { class Reading extends SQLWritable {
@@ -137,10 +146,10 @@ class Reading extends SQLWritable {
@override @override
Map<String, Object?> get sqlValue => { Map<String, Object?> get sqlValue => {
'kanji': kanji, 'kanji': kanji,
'type': type, 'type': type,
'reading': reading, 'reading': reading,
}; };
} }
class Kunyomi extends SQLWritable { class Kunyomi extends SQLWritable {
@@ -156,10 +165,10 @@ class Kunyomi extends SQLWritable {
@override @override
Map<String, Object?> get sqlValue => { Map<String, Object?> get sqlValue => {
'kanji': kanji, 'kanji': kanji,
'yomi': yomi, 'yomi': yomi,
'isJouyou': isJouyou, 'isJouyou': isJouyou,
}; };
} }
class Onyomi extends SQLWritable { class Onyomi extends SQLWritable {
@@ -177,11 +186,11 @@ class Onyomi extends SQLWritable {
@override @override
Map<String, Object?> get sqlValue => { Map<String, Object?> get sqlValue => {
'kanji': kanji, 'kanji': kanji,
'yomi': yomi, 'yomi': yomi,
'isJouyou': isJouyou, 'isJouyou': isJouyou,
'type': type, 'type': type,
}; };
} }
class Meaning extends SQLWritable { class Meaning extends SQLWritable {
@@ -197,10 +206,10 @@ class Meaning extends SQLWritable {
@override @override
Map<String, Object?> get sqlValue => { Map<String, Object?> get sqlValue => {
'kanji': kanji, 'kanji': kanji,
'language': language, 'language': language,
'meaning': meaning, 'meaning': meaning,
}; };
} }
class Character extends SQLWritable { class Character extends SQLWritable {
@@ -245,12 +254,11 @@ class Character extends SQLWritable {
this.nanori = const [], this.nanori = const [],
}); });
@override
Map<String, Object?> get sqlValue => { Map<String, Object?> get sqlValue => {
'literal': literal, 'literal': literal,
'grade': grade, 'grade': grade,
'strokeCount': strokeCount, 'strokeCount': strokeCount,
'frequency': frequency, 'frequency': frequency,
'jlpt': jlpt, 'jlpt': jlpt,
}; };
} }

View File

@@ -19,7 +19,10 @@ Future<void> seedKANJIDICData(List<Character> characters, Database db) async {
assert(c.radical != null, 'Radical name without radical'); assert(c.radical != null, 'Radical name without radical');
b.insert( b.insert(
KANJIDICTableNames.radicalName, KANJIDICTableNames.radicalName,
{'radicalId': c.radical!.radicalId, 'name': n}, {
'radicalId': c.radical!.radicalId,
'name': n,
},
conflictAlgorithm: ConflictAlgorithm.ignore, conflictAlgorithm: ConflictAlgorithm.ignore,
); );
} }
@@ -31,10 +34,13 @@ Future<void> seedKANJIDICData(List<Character> characters, Database db) async {
b.insert(KANJIDICTableNames.radical, c.radical!.sqlValue); b.insert(KANJIDICTableNames.radical, c.radical!.sqlValue);
} }
for (final sm in c.strokeMiscounts) { for (final sm in c.strokeMiscounts) {
b.insert(KANJIDICTableNames.strokeMiscount, { b.insert(
'kanji': c.literal, KANJIDICTableNames.strokeMiscount,
'strokeCount': sm, {
}); 'kanji': c.literal,
'strokeCount': sm,
},
);
} }
for (final v in c.variants) { for (final v in c.variants) {
b.insert(KANJIDICTableNames.variant, v.sqlValue); b.insert(KANJIDICTableNames.variant, v.sqlValue);
@@ -58,24 +64,24 @@ Future<void> seedKANJIDICData(List<Character> characters, Database db) async {
} }
for (final (i, y) in c.kunyomi.indexed) { for (final (i, y) in c.kunyomi.indexed) {
b.insert( b.insert(
KANJIDICTableNames.kunyomi, KANJIDICTableNames.kunyomi, y.sqlValue..addAll({'orderNum': i + 1}));
y.sqlValue..addAll({'orderNum': i + 1}),
);
} }
for (final (i, y) in c.onyomi.indexed) { for (final (i, y) in c.onyomi.indexed) {
b.insert( b.insert(
KANJIDICTableNames.onyomi, KANJIDICTableNames.onyomi, y.sqlValue..addAll({'orderNum': i + 1}));
y.sqlValue..addAll({'orderNum': i + 1}),
);
} }
for (final (i, m) in c.meanings.indexed) { for (final (i, m) in c.meanings.indexed) {
b.insert( b.insert(
KANJIDICTableNames.meaning, KANJIDICTableNames.meaning, m.sqlValue..addAll({'orderNum': i + 1}));
m.sqlValue..addAll({'orderNum': i + 1}),
);
} }
for (final n in c.nanori) { for (final n in c.nanori) {
b.insert(KANJIDICTableNames.nanori, {'kanji': c.literal, 'nanori': n}); b.insert(
KANJIDICTableNames.nanori,
{
'kanji': c.literal,
'nanori': n,
},
);
} }
} }
await b.commit(noResult: true); await b.commit(noResult: true);

View File

@@ -1,5 +1,4 @@
import 'package:jadb/_data_ingestion/kanjidic/objects.dart'; import 'package:jadb/_data_ingestion/kanjidic/objects.dart';
import 'package:jadb/util/romaji_transliteration.dart';
import 'package:xml/xml.dart'; import 'package:xml/xml.dart';
List<Character> parseKANJIDICData(XmlElement root) { List<Character> parseKANJIDICData(XmlElement root) {
@@ -10,33 +9,27 @@ List<Character> parseKANJIDICData(XmlElement root) {
final codepoint = c.findElements('codepoint').firstOrNull; final codepoint = c.findElements('codepoint').firstOrNull;
final radical = c.findElements('radical').firstOrNull; final radical = c.findElements('radical').firstOrNull;
final misc = c.findElements('misc').first; final misc = c.findElements('misc').first;
final dicNumber = c.findElements('dic_number').firstOrNull; final dic_number = c.findElements('dic_number').firstOrNull;
final queryCode = c.findElements('query_code').first; final query_code = c.findElements('query_code').first;
final readingMeaning = c.findElements('reading_meaning').firstOrNull; final reading_meaning = c.findElements('reading_meaning').firstOrNull;
// TODO: Group readings and meanings by their rmgroup parent node. // TODO: Group readings and meanings by their rmgroup parent node.
result.add( result.add(
Character( Character(
literal: kanji, literal: kanji,
strokeCount: int.parse( strokeCount:
misc.findElements('stroke_count').first.innerText, int.parse(misc.findElements('stroke_count').first.innerText),
),
grade: int.tryParse( grade: int.tryParse(
misc.findElements('grade').firstOrNull?.innerText ?? '', misc.findElements('grade').firstOrNull?.innerText ?? ''),
),
frequency: int.tryParse( frequency: int.tryParse(
misc.findElements('freq').firstOrNull?.innerText ?? '', misc.findElements('freq').firstOrNull?.innerText ?? ''),
),
jlpt: int.tryParse( jlpt: int.tryParse(
misc.findElements('jlpt').firstOrNull?.innerText ?? '', misc.findElements('jlpt').firstOrNull?.innerText ?? '',
), ),
radicalName: misc radicalName:
.findElements('rad_name') misc.findElements('rad_name').map((e) => e.innerText).toList(),
.map((e) => e.innerText) codepoints: codepoint
.toList(),
codepoints:
codepoint
?.findElements('cp_value') ?.findElements('cp_value')
.map( .map(
(e) => CodePoint( (e) => CodePoint(
@@ -51,7 +44,10 @@ List<Character> parseKANJIDICData(XmlElement root) {
?.findElements('rad_value') ?.findElements('rad_value')
.where((e) => e.getAttribute('rad_type') == 'classical') .where((e) => e.getAttribute('rad_type') == 'classical')
.map( .map(
(e) => Radical(kanji: kanji, radicalId: int.parse(e.innerText)), (e) => Radical(
kanji: kanji,
radicalId: int.parse(e.innerText),
),
) )
.firstOrNull, .firstOrNull,
strokeMiscounts: misc strokeMiscounts: misc
@@ -69,8 +65,7 @@ List<Character> parseKANJIDICData(XmlElement root) {
), ),
) )
.toList(), .toList(),
dictionaryReferences: dictionaryReferences: dic_number
dicNumber
?.findElements('dic_ref') ?.findElements('dic_ref')
.where((e) => e.getAttribute('dr_type') != 'moro') .where((e) => e.getAttribute('dr_type') != 'moro')
.map( .map(
@@ -82,8 +77,7 @@ List<Character> parseKANJIDICData(XmlElement root) {
) )
.toList() ?? .toList() ??
[], [],
dictionaryReferencesMoro: dictionaryReferencesMoro: dic_number
dicNumber
?.findElements('dic_ref') ?.findElements('dic_ref')
.where((e) => e.getAttribute('dr_type') == 'moro') .where((e) => e.getAttribute('dr_type') == 'moro')
.map( .map(
@@ -96,7 +90,7 @@ List<Character> parseKANJIDICData(XmlElement root) {
) )
.toList() ?? .toList() ??
[], [],
querycodes: queryCode querycodes: query_code
.findElements('q_code') .findElements('q_code')
.map( .map(
(e) => QueryCode( (e) => QueryCode(
@@ -107,8 +101,7 @@ List<Character> parseKANJIDICData(XmlElement root) {
), ),
) )
.toList(), .toList(),
readings: readings: reading_meaning
readingMeaning
?.findAllElements('reading') ?.findAllElements('reading')
.where( .where(
(e) => (e) =>
@@ -123,8 +116,7 @@ List<Character> parseKANJIDICData(XmlElement root) {
) )
.toList() ?? .toList() ??
[], [],
kunyomi: kunyomi: reading_meaning
readingMeaning
?.findAllElements('reading') ?.findAllElements('reading')
.where((e) => e.getAttribute('r_type') == 'ja_kun') .where((e) => e.getAttribute('r_type') == 'ja_kun')
.map( .map(
@@ -136,22 +128,19 @@ List<Character> parseKANJIDICData(XmlElement root) {
) )
.toList() ?? .toList() ??
[], [],
onyomi: onyomi: reading_meaning
readingMeaning
?.findAllElements('reading') ?.findAllElements('reading')
.where((e) => e.getAttribute('r_type') == 'ja_on') .where((e) => e.getAttribute('r_type') == 'ja_on')
.map( .map(
(e) => Onyomi( (e) => Onyomi(
kanji: kanji, kanji: kanji,
yomi: transliterateKatakanaToHiragana(e.innerText), yomi: e.innerText,
isJouyou: e.getAttribute('r_status') == 'jy', isJouyou: e.getAttribute('r_status') == 'jy',
type: e.getAttribute('on_type'), type: e.getAttribute('on_type')),
),
) )
.toList() ?? .toList() ??
[], [],
meanings: meanings: reading_meaning
readingMeaning
?.findAllElements('meaning') ?.findAllElements('meaning')
.map( .map(
(e) => Meaning( (e) => Meaning(
@@ -162,8 +151,7 @@ List<Character> parseKANJIDICData(XmlElement root) {
) )
.toList() ?? .toList() ??
[], [],
nanori: nanori: reading_meaning
readingMeaning
?.findElements('nanori') ?.findElements('nanori')
.map((e) => e.innerText) .map((e) => e.innerText)
.toList() ?? .toList() ??

View File

@@ -1,7 +1,9 @@
import 'dart:ffi';
import 'dart:io'; import 'dart:io';
import 'package:jadb/search.dart'; import 'package:jadb/search.dart';
import 'package:sqflite_common_ffi/sqflite_ffi.dart'; import 'package:sqflite_common_ffi/sqflite_ffi.dart';
import 'package:sqlite3/open.dart';
Future<Database> openLocalDb({ Future<Database> openLocalDb({
String? libsqlitePath, String? libsqlitePath,
@@ -10,23 +12,38 @@ Future<Database> openLocalDb({
bool verifyTablesExist = true, bool verifyTablesExist = true,
bool walMode = false, bool walMode = false,
}) async { }) async {
libsqlitePath ??= Platform.environment['LIBSQLITE_PATH'];
jadbPath ??= Platform.environment['JADB_PATH']; jadbPath ??= Platform.environment['JADB_PATH'];
jadbPath ??= Directory.current.uri.resolve('jadb.sqlite').path; jadbPath ??= Directory.current.uri.resolve('jadb.sqlite').path;
libsqlitePath = (libsqlitePath == null)
? null
: File(libsqlitePath).resolveSymbolicLinksSync();
jadbPath = File(jadbPath).resolveSymbolicLinksSync(); jadbPath = File(jadbPath).resolveSymbolicLinksSync();
if (!File(jadbPath).existsSync()) { if (libsqlitePath == null) {
throw Exception('JADB_PATH does not exist: $jadbPath'); throw Exception("LIBSQLITE_PATH is not set");
} }
final db = await createDatabaseFactoryFfi().openDatabase( if (!File(libsqlitePath).existsSync()) {
throw Exception("LIBSQLITE_PATH does not exist: $libsqlitePath");
}
if (!File(jadbPath).existsSync()) {
throw Exception("JADB_PATH does not exist: $jadbPath");
}
final db = await createDatabaseFactoryFfi(
ffiInit: () =>
open.overrideForAll(() => DynamicLibrary.open(libsqlitePath!)),
).openDatabase(
jadbPath, jadbPath,
options: OpenDatabaseOptions( options: OpenDatabaseOptions(
onConfigure: (db) async { onConfigure: (db) async {
if (walMode) { if (walMode) {
await db.execute('PRAGMA journal_mode = WAL'); await db.execute("PRAGMA journal_mode = WAL");
} }
await db.execute('PRAGMA foreign_keys = ON'); await db.execute("PRAGMA foreign_keys = ON");
}, },
readOnly: !readWrite, readOnly: !readWrite,
), ),

View File

@@ -3,10 +3,8 @@ import 'dart:io';
Iterable<String> parseRADKFILEBlocks(File radkfile) { Iterable<String> parseRADKFILEBlocks(File radkfile) {
final String content = File('data/tmp/radkfile_utf8').readAsStringSync(); final String content = File('data/tmp/radkfile_utf8').readAsStringSync();
final Iterable<String> blocks = content final Iterable<String> blocks =
.replaceAll(RegExp(r'^#.*$'), '') content.replaceAll(RegExp(r'^#.*$'), '').split(r'$').skip(2);
.split(r'$')
.skip(2);
return blocks; return blocks;
} }

View File

@@ -1,20 +1,27 @@
import 'package:jadb/table_names/radkfile.dart'; import 'package:jadb/table_names/radkfile.dart';
import 'package:sqflite_common/sqlite_api.dart'; import 'package:sqflite_common/sqlite_api.dart';
Future<void> seedRADKFILEData(Iterable<String> blocks, Database db) async { Future<void> seedRADKFILEData(
Iterable<String> blocks,
Database db,
) async {
final b = db.batch(); final b = db.batch();
for (final block in blocks) { for (final block in blocks) {
final String radical = block[1]; final String radical = block[1];
final List<String> kanjiList = final List<String> kanjiList = block
block.replaceFirst(RegExp(r'.*\n'), '').split('') .replaceFirst(RegExp(r'.*\n'), '')
..removeWhere((e) => e == '' || e == '\n'); .split('')
..removeWhere((e) => e == '' || e == '\n');
for (final kanji in kanjiList.toSet()) { for (final kanji in kanjiList.toSet()) {
b.insert(RADKFILETableNames.radkfile, { b.insert(
'radical': radical, RADKFILETableNames.radkfile,
'kanji': kanji, {
}); 'radical': radical,
'kanji': kanji,
},
);
} }
} }

View File

@@ -24,10 +24,10 @@ Future<void> seedData(Database db) async {
Future<void> parseAndSeedDataFromJMdict(Database db) async { Future<void> parseAndSeedDataFromJMdict(Database db) async {
print('[JMdict] Reading file content...'); print('[JMdict] Reading file content...');
final String rawXML = File('data/tmp/JMdict.xml').readAsStringSync(); String rawXML = File('data/tmp/JMdict.xml').readAsStringSync();
print('[JMdict] Parsing XML tags...'); print('[JMdict] Parsing XML tags...');
final XmlElement root = XmlDocument.parse(rawXML).getElement('JMdict')!; XmlElement root = XmlDocument.parse(rawXML).getElement('JMdict')!;
print('[JMdict] Parsing XML content...'); print('[JMdict] Parsing XML content...');
final entries = parseJMDictData(root); final entries = parseJMDictData(root);
@@ -38,10 +38,10 @@ Future<void> parseAndSeedDataFromJMdict(Database db) async {
Future<void> parseAndSeedDataFromKANJIDIC(Database db) async { Future<void> parseAndSeedDataFromKANJIDIC(Database db) async {
print('[KANJIDIC2] Reading file...'); print('[KANJIDIC2] Reading file...');
final String rawXML = File('data/tmp/kanjidic2.xml').readAsStringSync(); String rawXML = File('data/tmp/kanjidic2.xml').readAsStringSync();
print('[KANJIDIC2] Parsing XML...'); print('[KANJIDIC2] Parsing XML...');
final XmlElement root = XmlDocument.parse(rawXML).getElement('kanjidic2')!; XmlElement root = XmlDocument.parse(rawXML).getElement('kanjidic2')!;
print('[KANJIDIC2] Parsing XML content...'); print('[KANJIDIC2] Parsing XML content...');
final entries = parseKANJIDICData(root); final entries = parseKANJIDICData(root);
@@ -52,7 +52,7 @@ Future<void> parseAndSeedDataFromKANJIDIC(Database db) async {
Future<void> parseAndSeedDataFromRADKFILE(Database db) async { Future<void> parseAndSeedDataFromRADKFILE(Database db) async {
print('[RADKFILE] Reading file...'); print('[RADKFILE] Reading file...');
final File raw = File('data/tmp/RADKFILE'); File raw = File('data/tmp/RADKFILE');
print('[RADKFILE] Parsing content...'); print('[RADKFILE] Parsing content...');
final blocks = parseRADKFILEBlocks(raw); final blocks = parseRADKFILEBlocks(raw);
@@ -63,7 +63,7 @@ Future<void> parseAndSeedDataFromRADKFILE(Database db) async {
Future<void> parseAndSeedDataFromTanosJLPT(Database db) async { Future<void> parseAndSeedDataFromTanosJLPT(Database db) async {
print('[TANOS-JLPT] Reading files...'); print('[TANOS-JLPT] Reading files...');
final Map<String, File> files = { Map<String, File> files = {
'N1': File('data/tanos-jlpt/n1.csv'), 'N1': File('data/tanos-jlpt/n1.csv'),
'N2': File('data/tanos-jlpt/n2.csv'), 'N2': File('data/tanos-jlpt/n2.csv'),
'N3': File('data/tanos-jlpt/n3.csv'), 'N3': File('data/tanos-jlpt/n3.csv'),

View File

@@ -3,64 +3,52 @@ import 'dart:io';
import 'package:csv/csv.dart'; import 'package:csv/csv.dart';
import 'package:jadb/_data_ingestion/tanos-jlpt/objects.dart'; import 'package:jadb/_data_ingestion/tanos-jlpt/objects.dart';
import 'package:xml/xml_events.dart';
Future<List<JLPTRankedWord>> parseJLPTRankedWords( Future<List<JLPTRankedWord>> parseJLPTRankedWords(
Map<String, File> files, Map<String, File> files,
) async { ) async {
final List<JLPTRankedWord> result = []; final List<JLPTRankedWord> result = [];
final codec = CsvCodec(
fieldDelimiter: ',',
lineDelimiter: '\n',
quoteMode: QuoteMode.strings,
escapeCharacter: '\\',
);
for (final entry in files.entries) { for (final entry in files.entries) {
final jlptLevel = entry.key; final jlptLevel = entry.key;
final file = entry.value; final file = entry.value;
if (!file.existsSync()) { if (!file.existsSync()) {
throw Exception('File $jlptLevel does not exist'); throw Exception("File $jlptLevel does not exist");
} }
final words = await file final rows = await file
.openRead() .openRead()
.transform(utf8.decoder) .transform(utf8.decoder)
.transform(codec.decoder) .transform(CsvToListConverter())
.flatten()
.map((row) {
if (row.length != 3) {
throw Exception('Invalid line in $jlptLevel: $row');
}
return row;
})
.map((row) => row.map((e) => e as String).toList())
.map((row) {
final kanji = row[0].isEmpty
? null
: row[0]
.replaceFirst(RegExp('^お・'), '')
.replaceAll(RegExp(r'.*'), '');
final readings = row[1]
.split(RegExp('[・/、(:?s+)]'))
.map((e) => e.trim())
.toList();
final meanings = row[2].split(',').expand(cleanMeaning).toList();
return JLPTRankedWord(
readings: readings,
kanji: kanji,
jlptLevel: jlptLevel,
meanings: meanings,
);
})
.toList(); .toList();
result.addAll(words); for (final row in rows) {
if (row.length != 3) {
throw Exception("Invalid line in $jlptLevel: $row");
}
final kanji = (row[0] as String).isEmpty
? null
: (row[0] as String)
.replaceFirst(RegExp('^お・'), '')
.replaceAll(RegExp(r'.*'), '');
final readings = (row[1] as String)
.split(RegExp('[・/、(:?\s+)]'))
.map((e) => e.trim())
.toList();
final meanings =
(row[2] as String).split(',').expand(cleanMeaning).toList();
result.add(JLPTRankedWord(
readings: readings,
kanji: kanji,
jlptLevel: jlptLevel,
meanings: meanings,
));
}
} }
return result; return result;

View File

@@ -13,5 +13,5 @@ class JLPTRankedWord {
@override @override
String toString() => String toString() =>
'($jlptLevel,$kanji,"${readings.join(",")}","${meanings.join(",")})'; '(${jlptLevel},${kanji},"${readings.join(",")}","${meanings.join(",")})';
} }

View File

@@ -1,4 +1,4 @@
const Map<(String?, String), int?> tanosJLPTOverrides = { const Map<(String?, String), int?> TANOS_JLPT_OVERRIDES = {
// N5: // N5:
(null, 'あなた'): 1223615, (null, 'あなた'): 1223615,
(null, 'あの'): 1000430, (null, 'あの'): 1000430,

View File

@@ -1,39 +1,49 @@
import 'package:jadb/table_names/jmdict.dart';
import 'package:jadb/_data_ingestion/tanos-jlpt/objects.dart'; import 'package:jadb/_data_ingestion/tanos-jlpt/objects.dart';
import 'package:jadb/_data_ingestion/tanos-jlpt/overrides.dart'; import 'package:jadb/_data_ingestion/tanos-jlpt/overrides.dart';
import 'package:jadb/table_names/jmdict.dart';
import 'package:sqflite_common/sqlite_api.dart'; import 'package:sqflite_common/sqlite_api.dart';
Future<List<int>> _findReadingCandidates(JLPTRankedWord word, Database db) => db Future<List<int>> _findReadingCandidates(
.query( JLPTRankedWord word,
JMdictTableNames.readingElement, Database db,
columns: ['entryId'], ) =>
where: db
'"reading" IN (${List.filled(word.readings.length, '?').join(',')})', .query(
whereArgs: [...word.readings], JMdictTableNames.readingElement,
) columns: ['entryId'],
.then((rows) => rows.map((row) => row['entryId'] as int).toList()); where:
'"reading" IN (${List.filled(word.readings.length, '?').join(',')})',
whereArgs: [...word.readings],
)
.then((rows) => rows.map((row) => row['entryId'] as int).toList());
Future<List<int>> _findKanjiCandidates(JLPTRankedWord word, Database db) => db Future<List<int>> _findKanjiCandidates(
.query( JLPTRankedWord word,
JMdictTableNames.kanjiElement, Database db,
columns: ['entryId'], ) =>
where: 'reading = ?', db
whereArgs: [word.kanji], .query(
) JMdictTableNames.kanjiElement,
.then((rows) => rows.map((row) => row['entryId'] as int).toList()); columns: ['entryId'],
where: 'reading = ?',
whereArgs: [word.kanji],
)
.then((rows) => rows.map((row) => row['entryId'] as int).toList());
Future<List<(int, String)>> _findSenseCandidates( Future<List<(int, String)>> _findSenseCandidates(
JLPTRankedWord word, JLPTRankedWord word,
Database db, Database db,
) => db ) =>
.rawQuery( db.rawQuery(
'SELECT entryId, phrase ' 'SELECT entryId, phrase '
'FROM "${JMdictTableNames.senseGlossary}" ' 'FROM "${JMdictTableNames.senseGlossary}" '
'JOIN "${JMdictTableNames.sense}" USING (senseId)' 'JOIN "${JMdictTableNames.sense}" USING (senseId)'
'WHERE phrase IN (${List.filled(word.meanings.length, '?').join(',')})', 'WHERE phrase IN (${List.filled(
word.meanings.length,
'?',
).join(',')})',
[...word.meanings], [...word.meanings],
) ).then(
.then(
(rows) => rows (rows) => rows
.map((row) => (row['entryId'] as int, row['phrase'] as String)) .map((row) => (row['entryId'] as int, row['phrase'] as String))
.toList(), .toList(),
@@ -45,10 +55,8 @@ Future<int?> findEntry(
bool useOverrides = true, bool useOverrides = true,
}) async { }) async {
final List<int> readingCandidates = await _findReadingCandidates(word, db); final List<int> readingCandidates = await _findReadingCandidates(word, db);
final List<(int, String)> senseCandidates = await _findSenseCandidates( final List<(int, String)> senseCandidates =
word, await _findSenseCandidates(word, db);
db,
);
List<int> entryIds; List<int> entryIds;
@@ -63,10 +71,8 @@ Future<int?> findEntry(
print('No entry found, trying to combine with senses'); print('No entry found, trying to combine with senses');
entryIds = readingCandidates entryIds = readingCandidates
.where( .where((readingId) =>
(readingId) => senseCandidates.any((sense) => sense.$1 == readingId))
senseCandidates.any((sense) => sense.$1 == readingId),
)
.toList(); .toList();
} }
} else { } else {
@@ -76,21 +82,18 @@ Future<int?> findEntry(
if ((entryIds.isEmpty || entryIds.length > 1) && useOverrides) { if ((entryIds.isEmpty || entryIds.length > 1) && useOverrides) {
print('No entry found, trying to fetch from overrides'); print('No entry found, trying to fetch from overrides');
final overrideEntries = word.readings final overrideEntries = word.readings
.map((reading) => tanosJLPTOverrides[(word.kanji, reading)]) .map((reading) => TANOS_JLPT_OVERRIDES[(word.kanji, reading)])
.whereType<int>() .whereType<int>()
.toSet(); .toSet();
if (overrideEntries.length > 1) { if (overrideEntries.length > 1) {
throw Exception( throw Exception(
'Multiple override entries found for ${word.toString()}: $entryIds', 'Multiple override entries found for ${word.toString()}: $entryIds');
); } else if (overrideEntries.length == 0 &&
} else if (overrideEntries.isEmpty && !word.readings.any((reading) =>
!word.readings.any( TANOS_JLPT_OVERRIDES.containsKey((word.kanji, reading)))) {
(reading) => tanosJLPTOverrides.containsKey((word.kanji, reading)),
)) {
throw Exception( throw Exception(
'No override entry found for ${word.toString()}: $entryIds', 'No override entry found for ${word.toString()}: $entryIds');
);
} }
print('Found override: ${overrideEntries.firstOrNull}'); print('Found override: ${overrideEntries.firstOrNull}');
@@ -100,8 +103,7 @@ Future<int?> findEntry(
if (entryIds.length > 1) { if (entryIds.length > 1) {
throw Exception( throw Exception(
'Multiple override entries found for ${word.toString()}: $entryIds', 'Multiple override entries found for ${word.toString()}: $entryIds');
);
} else if (entryIds.isEmpty) { } else if (entryIds.isEmpty) {
throw Exception('No entry found for ${word.toString()}'); throw Exception('No entry found for ${word.toString()}');
} }

View File

@@ -5,17 +5,20 @@ Future<void> seedTanosJLPTData(
Map<String, Set<int>> resolvedEntries, Map<String, Set<int>> resolvedEntries,
Database db, Database db,
) async { ) async {
final Batch b = db.batch(); Batch b = db.batch();
for (final jlptLevel in resolvedEntries.entries) { for (final jlptLevel in resolvedEntries.entries) {
final level = jlptLevel.key; final level = jlptLevel.key;
final entryIds = jlptLevel.value; final entryIds = jlptLevel.value;
for (final entryId in entryIds) { for (final entryId in entryIds) {
b.insert(TanosJLPTTableNames.jlptTag, { b.insert(
'entryId': entryId, TanosJLPTTableNames.jlptTag,
'jlptLevel': level, {
}); 'entryId': entryId,
'jlptLevel': level,
},
);
} }
} }

View File

@@ -1,15 +1,14 @@
import 'dart:io'; import 'dart:io';
import 'package:args/command_runner.dart';
import 'package:jadb/_data_ingestion/open_local_db.dart'; import 'package:jadb/_data_ingestion/open_local_db.dart';
import 'package:jadb/_data_ingestion/seed_database.dart'; import 'package:jadb/_data_ingestion/seed_database.dart';
import 'package:args/command_runner.dart';
import 'package:jadb/cli/args.dart'; import 'package:jadb/cli/args.dart';
class CreateDb extends Command { class CreateDb extends Command {
@override final name = "create-db";
final name = 'create-db'; final description = "Create the database";
@override
final description = 'Create the database';
CreateDb() { CreateDb() {
addLibsqliteArg(argParser); addLibsqliteArg(argParser);
@@ -24,7 +23,6 @@ class CreateDb extends Command {
); );
} }
@override
Future<void> run() async { Future<void> run() async {
if (argResults!.option('libsqlite') == null) { if (argResults!.option('libsqlite') == null) {
print(argParser.usage); print(argParser.usage);
@@ -37,22 +35,12 @@ class CreateDb extends Command {
readWrite: true, readWrite: true,
); );
bool failed = false; await seedData(db).then((_) {
await seedData(db) print("Database created successfully");
.then((_) { }).catchError((error) {
print('Database created successfully'); print("Error creating database: $error");
}) }).whenComplete(() {
.catchError((error) { db.close();
print('Error creating database: $error'); });
failed = true;
})
.whenComplete(() {
db.close();
});
if (failed) {
exit(1);
} else {
exit(0);
}
} }
} }

View File

@@ -1,7 +1,8 @@
import 'dart:io'; import 'dart:io';
import 'package:args/command_runner.dart';
import 'package:jadb/_data_ingestion/open_local_db.dart'; import 'package:jadb/_data_ingestion/open_local_db.dart';
import 'package:args/command_runner.dart';
import 'package:jadb/_data_ingestion/tanos-jlpt/csv_parser.dart'; import 'package:jadb/_data_ingestion/tanos-jlpt/csv_parser.dart';
import 'package:jadb/_data_ingestion/tanos-jlpt/objects.dart'; import 'package:jadb/_data_ingestion/tanos-jlpt/objects.dart';
import 'package:jadb/_data_ingestion/tanos-jlpt/resolve.dart'; import 'package:jadb/_data_ingestion/tanos-jlpt/resolve.dart';
@@ -9,11 +10,9 @@ import 'package:jadb/cli/args.dart';
import 'package:sqflite_common/sqlite_api.dart'; import 'package:sqflite_common/sqlite_api.dart';
class CreateTanosJlptMappings extends Command { class CreateTanosJlptMappings extends Command {
@override final name = "create-tanos-jlpt-mappings";
final name = 'create-tanos-jlpt-mappings';
@override
final description = final description =
'Resolve Tanos JLPT data against JMDict. This tool is useful to create overrides for ambiguous references'; "Resolve Tanos JLPT data against JMDict. This tool is useful to create overrides for ambiguous references";
CreateTanosJlptMappings() { CreateTanosJlptMappings() {
addLibsqliteArg(argParser); addLibsqliteArg(argParser);
@@ -27,7 +26,6 @@ class CreateTanosJlptMappings extends Command {
); );
} }
@override
Future<void> run() async { Future<void> run() async {
if (argResults!.option('libsqlite') == null || if (argResults!.option('libsqlite') == null ||
argResults!.option('jadb') == null) { argResults!.option('jadb') == null) {
@@ -42,7 +40,7 @@ class CreateTanosJlptMappings extends Command {
final useOverrides = argResults!.flag('overrides'); final useOverrides = argResults!.flag('overrides');
final Map<String, File> files = { Map<String, File> files = {
'N1': File('data/tanos-jlpt/n1.csv'), 'N1': File('data/tanos-jlpt/n1.csv'),
'N2': File('data/tanos-jlpt/n2.csv'), 'N2': File('data/tanos-jlpt/n2.csv'),
'N3': File('data/tanos-jlpt/n3.csv'), 'N3': File('data/tanos-jlpt/n3.csv'),
@@ -61,12 +59,11 @@ Future<void> resolveExisting(
Database db, Database db,
bool useOverrides, bool useOverrides,
) async { ) async {
final List<JLPTRankedWord> missingWords = []; List<JLPTRankedWord> missingWords = [];
for (final (i, word) in rankedWords.indexed) { for (final (i, word) in rankedWords.indexed) {
try { try {
print( print(
'[${(i + 1).toString().padLeft(4, '0')}/${rankedWords.length}] ${word.toString()}', '[${(i + 1).toString().padLeft(4, '0')}/${rankedWords.length}] ${word.toString()}');
);
await findEntry(word, db, useOverrides: useOverrides); await findEntry(word, db, useOverrides: useOverrides);
} catch (e) { } catch (e) {
print(e); print(e);
@@ -81,19 +78,16 @@ Future<void> resolveExisting(
print('Statistics:'); print('Statistics:');
for (final jlptLevel in ['N5', 'N4', 'N3', 'N2', 'N1']) { for (final jlptLevel in ['N5', 'N4', 'N3', 'N2', 'N1']) {
final missingWordCount = missingWords final missingWordCount =
.where((e) => e.jlptLevel == jlptLevel) missingWords.where((e) => e.jlptLevel == jlptLevel).length;
.length; final totalWordCount =
final totalWordCount = rankedWords rankedWords.where((e) => e.jlptLevel == jlptLevel).length;
.where((e) => e.jlptLevel == jlptLevel)
.length;
final failureRate = ((missingWordCount / totalWordCount) * 100) final failureRate =
.toStringAsFixed(2); ((missingWordCount / totalWordCount) * 100).toStringAsFixed(2);
print( print(
'$jlptLevel failures: [$missingWordCount/$totalWordCount] ($failureRate%)', '${jlptLevel} failures: [${missingWordCount}/${totalWordCount}] (${failureRate}%)');
);
} }
print('Not able to determine the entry for ${missingWords.length} words'); print('Not able to determine the entry for ${missingWords.length} words');

View File

@@ -1,15 +1,14 @@
// import 'dart:io'; // import 'dart:io';
import 'package:args/command_runner.dart';
// import 'package:jadb/_data_ingestion/open_local_db.dart'; // import 'package:jadb/_data_ingestion/open_local_db.dart';
import 'package:jadb/cli/args.dart'; import 'package:jadb/cli/args.dart';
import 'package:args/command_runner.dart';
import 'package:jadb/util/lemmatizer/lemmatizer.dart'; import 'package:jadb/util/lemmatizer/lemmatizer.dart';
class Lemmatize extends Command { class Lemmatize extends Command {
@override final name = "lemmatize";
final name = 'lemmatize'; final description = "Lemmatize a word using the Jadb lemmatizer";
@override
final description = 'Lemmatize a word using the Jadb lemmatizer';
Lemmatize() { Lemmatize() {
addLibsqliteArg(argParser); addLibsqliteArg(argParser);
@@ -22,7 +21,6 @@ class Lemmatize extends Command {
); );
} }
@override
Future<void> run() async { Future<void> run() async {
// if (argResults!.option('libsqlite') == null || // if (argResults!.option('libsqlite') == null ||
// argResults!.option('jadb') == null) { // argResults!.option('jadb') == null) {
@@ -43,6 +41,6 @@ class Lemmatize extends Command {
print(result.toString()); print(result.toString());
print('Lemmatization took ${time.elapsedMilliseconds}ms'); print("Lemmatization took ${time.elapsedMilliseconds}ms");
} }
} }

View File

@@ -1,25 +1,27 @@
import 'dart:convert'; import 'dart:convert';
import 'dart:io'; import 'dart:io';
import 'package:args/command_runner.dart';
import 'package:jadb/_data_ingestion/open_local_db.dart'; import 'package:jadb/_data_ingestion/open_local_db.dart';
import 'package:jadb/cli/args.dart'; import 'package:jadb/cli/args.dart';
import 'package:jadb/search.dart'; import 'package:jadb/search.dart';
import 'package:args/command_runner.dart';
class QueryKanji extends Command { class QueryKanji extends Command {
@override final name = "query-kanji";
final name = 'query-kanji'; final description = "Query the database for kanji data";
@override
final description = 'Query the database for kanji data';
@override
final invocation = 'jadb query-kanji [options] <kanji>';
QueryKanji() { QueryKanji() {
addLibsqliteArg(argParser); addLibsqliteArg(argParser);
addJadbArg(argParser); addJadbArg(argParser);
argParser.addOption(
'kanji',
abbr: 'k',
help: 'The kanji to search for.',
valueHelp: 'KANJI',
);
} }
@override
Future<void> run() async { Future<void> run() async {
if (argResults!.option('libsqlite') == null || if (argResults!.option('libsqlite') == null ||
argResults!.option('jadb') == null) { argResults!.option('jadb') == null) {
@@ -32,25 +34,18 @@ class QueryKanji extends Command {
libsqlitePath: argResults!.option('libsqlite')!, libsqlitePath: argResults!.option('libsqlite')!,
); );
if (argResults!.rest.length != 1) {
print('You need to provide exactly one kanji character to search for.');
print('');
printUsage();
exit(64);
}
final String kanji = argResults!.rest.first.trim();
final time = Stopwatch()..start(); final time = Stopwatch()..start();
final result = await JaDBConnection(db).jadbSearchKanji(kanji); final result = await JaDBConnection(db).jadbSearchKanji(
argResults!.option('kanji') ?? '',
);
time.stop(); time.stop();
if (result == null) { if (result == null) {
print('No such kanji'); print("No such kanji");
} else { } else {
print(JsonEncoder.withIndent(' ').convert(result.toJson())); print(JsonEncoder.withIndent(' ').convert(result.toJson()));
} }
print('Query took ${time.elapsedMilliseconds}ms'); print("Query took ${time.elapsedMilliseconds}ms");
} }
} }

View File

@@ -1,38 +1,30 @@
import 'dart:convert';
import 'dart:io'; import 'dart:io';
import 'package:args/command_runner.dart';
import 'package:jadb/_data_ingestion/open_local_db.dart'; import 'package:jadb/_data_ingestion/open_local_db.dart';
import 'package:jadb/cli/args.dart'; import 'package:jadb/cli/args.dart';
import 'package:jadb/search.dart'; import 'package:jadb/search.dart';
import 'package:sqflite_common/sqflite.dart';
import 'package:args/command_runner.dart';
class QueryWord extends Command { class QueryWord extends Command {
@override final name = "query-word";
final name = 'query-word'; final description = "Query the database for word data";
@override
final description = 'Query the database for word data';
@override
final invocation = 'jadb query-word [options] (<word> | <ID>)';
QueryWord() { QueryWord() {
addLibsqliteArg(argParser); addLibsqliteArg(argParser);
addJadbArg(argParser); addJadbArg(argParser);
argParser.addOption(
argParser.addFlag('json', abbr: 'j', help: 'Output results in JSON format'); 'word',
abbr: 'w',
argParser.addOption('page', abbr: 'p', valueHelp: 'NUM', defaultsTo: '0'); help: 'The word to search for.',
valueHelp: 'WORD',
argParser.addOption('pageSize', valueHelp: 'NUM', defaultsTo: '30'); );
} }
@override
Future<void> run() async { Future<void> run() async {
if (argResults!.option('libsqlite') == null || if (argResults!.option('libsqlite') == null ||
argResults!.option('jadb') == null) { argResults!.option('jadb') == null) {
print('You need to provide both libsqlite and jadb paths.'); print(argParser.usage);
print('');
printUsage();
exit(64); exit(64);
} }
@@ -41,81 +33,29 @@ class QueryWord extends Command {
libsqlitePath: argResults!.option('libsqlite')!, libsqlitePath: argResults!.option('libsqlite')!,
); );
if (argResults!.rest.isEmpty) { final String searchWord = argResults!.option('word') ?? 'かな';
print('You need to provide a word or ID to search for.');
print('');
printUsage();
exit(64);
}
final String searchWord = argResults!.rest.join(' ');
final int? maybeId = int.tryParse(searchWord);
if (maybeId != null && maybeId >= 1000000) {
await _searchId(db, maybeId, argResults!.flag('json'));
} else {
await _searchWord(
db,
searchWord,
argResults!.flag('json'),
int.parse(argResults!.option('page')!),
int.parse(argResults!.option('pageSize')!),
);
}
}
Future<void> _searchId(DatabaseExecutor db, int id, bool jsonOutput) async {
final time = Stopwatch()..start();
final result = await JaDBConnection(db).jadbGetWordById(id);
time.stop();
if (result == null) {
print('Invalid ID');
} else {
if (jsonOutput) {
print(JsonEncoder.withIndent(' ').convert(result));
} else {
print(result.toString());
}
}
print('Query took ${time.elapsedMilliseconds}ms');
}
Future<void> _searchWord(
DatabaseExecutor db,
String searchWord,
bool jsonOutput,
int page,
int pageSize,
) async {
final time = Stopwatch()..start(); final time = Stopwatch()..start();
final count = await JaDBConnection(db).jadbSearchWordCount(searchWord); final count = await JaDBConnection(db).jadbSearchWordCount(searchWord);
time.stop(); time.stop();
final time2 = Stopwatch()..start(); final time2 = Stopwatch()..start();
final result = await JaDBConnection( final result = await JaDBConnection(db).jadbSearchWord(searchWord);
db,
).jadbSearchWord(searchWord, page: page, pageSize: pageSize);
time2.stop(); time2.stop();
if (result == null) { if (result == null) {
print('Invalid search'); print("Invalid search");
} else if (result.isEmpty) { } else if (result.isEmpty) {
print('No matches'); print("No matches");
} else { } else {
if (jsonOutput) { for (final e in result) {
print(JsonEncoder.withIndent(' ').convert(result)); print(e.toString());
} else { print("");
for (final e in result) {
print(e.toString());
print('');
}
} }
} }
print('Total count: $count'); print("Total count: ${count}");
print('Count query took ${time.elapsedMilliseconds}ms'); print("Count query took ${time.elapsedMilliseconds}ms");
print('Query took ${time2.elapsedMilliseconds}ms'); print("Query took ${time2.elapsedMilliseconds}ms");
} }
} }

View File

@@ -1,5 +1,6 @@
/// Jouyou kanji sorted primarily by grades and secondarily by strokes. /// Jouyou kanji sorted primarily by grades and secondarily by strokes.
const Map<int, Map<int, List<String>>> jouyouKanjiByGradeAndStrokeCount = { const Map<int, Map<int, List<String>>> JOUYOU_KANJI_BY_GRADE_AND_STROKE_COUNT =
{
1: { 1: {
1: [''], 1: [''],
2: ['', '', '', '', '', '', '', ''], 2: ['', '', '', '', '', '', '', ''],
@@ -11,7 +12,7 @@ const Map<int, Map<int, List<String>>> jouyouKanjiByGradeAndStrokeCount = {
8: ['', '', '', '', '', ''], 8: ['', '', '', '', '', ''],
9: ['', ''], 9: ['', ''],
10: [''], 10: [''],
12: [''], 12: ['']
}, },
2: { 2: {
2: [''], 2: [''],
@@ -34,7 +35,7 @@ const Map<int, Map<int, List<String>>> jouyouKanjiByGradeAndStrokeCount = {
'', '',
'', '',
'', '',
'', ''
], ],
5: ['', '', '', '', '', '', '', '', '', '', '', ''], 5: ['', '', '', '', '', '', '', '', '', '', '', ''],
6: [ 6: [
@@ -57,7 +58,7 @@ const Map<int, Map<int, List<String>>> jouyouKanjiByGradeAndStrokeCount = {
'', '',
'', '',
'', '',
'', ''
], ],
7: [ 7: [
'', '',
@@ -77,7 +78,7 @@ const Map<int, Map<int, List<String>>> jouyouKanjiByGradeAndStrokeCount = {
'', '',
'', '',
'', '',
'', ''
], ],
8: [ 8: [
'', '',
@@ -94,7 +95,7 @@ const Map<int, Map<int, List<String>>> jouyouKanjiByGradeAndStrokeCount = {
'', '',
'', '',
'', '',
'', ''
], ],
9: [ 9: [
'', '',
@@ -114,7 +115,7 @@ const Map<int, Map<int, List<String>>> jouyouKanjiByGradeAndStrokeCount = {
'', '',
'', '',
'', '',
'', ''
], ],
10: ['', '', '', '', '', '', '', '', '', '', '', ''], 10: ['', '', '', '', '', '', '', '', '', '', '', ''],
11: ['', '', '', '', '', '', '', '', '', '', '', '', ''], 11: ['', '', '', '', '', '', '', '', '', '', '', '', ''],
@@ -123,7 +124,7 @@ const Map<int, Map<int, List<String>>> jouyouKanjiByGradeAndStrokeCount = {
14: ['', '', '', '', '', ''], 14: ['', '', '', '', '', ''],
15: [''], 15: [''],
16: ['', ''], 16: ['', ''],
18: ['', ''], 18: ['', '']
}, },
3: { 3: {
2: [''], 2: [''],
@@ -145,7 +146,7 @@ const Map<int, Map<int, List<String>>> jouyouKanjiByGradeAndStrokeCount = {
'', '',
'', '',
'', '',
'', ''
], ],
6: ['', '', '', '', '', '', '', '', '', '', '', '', '', ''], 6: ['', '', '', '', '', '', '', '', '', '', '', '', '', ''],
7: ['', '', '', '', '', '', '', '', '', '', '', '', '', ''], 7: ['', '', '', '', '', '', '', '', '', '', '', '', '', ''],
@@ -177,7 +178,7 @@ const Map<int, Map<int, List<String>>> jouyouKanjiByGradeAndStrokeCount = {
'', '',
'', '',
'', '',
'', ''
], ],
9: [ 9: [
'', '',
@@ -209,7 +210,7 @@ const Map<int, Map<int, List<String>>> jouyouKanjiByGradeAndStrokeCount = {
'', '',
'', '',
'', '',
'', ''
], ],
10: [ 10: [
'', '',
@@ -231,7 +232,7 @@ const Map<int, Map<int, List<String>>> jouyouKanjiByGradeAndStrokeCount = {
'', '',
'', '',
'', '',
'', ''
], ],
11: [ 11: [
'', '',
@@ -252,7 +253,7 @@ const Map<int, Map<int, List<String>>> jouyouKanjiByGradeAndStrokeCount = {
'', '',
'', '',
'', '',
'', ''
], ],
12: [ 12: [
'', '',
@@ -281,13 +282,13 @@ const Map<int, Map<int, List<String>>> jouyouKanjiByGradeAndStrokeCount = {
'', '',
'', '',
'', '',
'', ''
], ],
13: ['', '', '', '', '', '', '', '', '', '', ''], 13: ['', '', '', '', '', '', '', '', '', '', ''],
14: ['', '', '', '', '', ''], 14: ['', '', '', '', '', ''],
15: ['', '調', '', ''], 15: ['', '調', '', ''],
16: ['', '', '', ''], 16: ['', '', '', ''],
18: [''], 18: ['']
}, },
4: { 4: {
4: ['', '', '', '', ''], 4: ['', '', '', '', ''],
@@ -317,7 +318,7 @@ const Map<int, Map<int, List<String>>> jouyouKanjiByGradeAndStrokeCount = {
'', '',
'', '',
'', '',
'', ''
], ],
8: [ 8: [
'', '',
@@ -345,7 +346,7 @@ const Map<int, Map<int, List<String>>> jouyouKanjiByGradeAndStrokeCount = {
'', '',
'', '',
'', '',
'', ''
], ],
9: [ 9: [
'', '',
@@ -366,7 +367,7 @@ const Map<int, Map<int, List<String>>> jouyouKanjiByGradeAndStrokeCount = {
'', '',
'', '',
'', '',
'', ''
], ],
10: [ 10: [
'', '',
@@ -388,7 +389,7 @@ const Map<int, Map<int, List<String>>> jouyouKanjiByGradeAndStrokeCount = {
'', '',
'', '',
'', '',
'', ''
], ],
11: [ 11: [
'', '',
@@ -409,7 +410,7 @@ const Map<int, Map<int, List<String>>> jouyouKanjiByGradeAndStrokeCount = {
'', '',
'', '',
'', '',
'', ''
], ],
12: [ 12: [
'', '',
@@ -433,7 +434,7 @@ const Map<int, Map<int, List<String>>> jouyouKanjiByGradeAndStrokeCount = {
'', '',
'', '',
'', '',
'', ''
], ],
13: ['', '', '', '', '', '', '', '', '', '', ''], 13: ['', '', '', '', '', '', '', '', '', '', ''],
14: ['', '', '', '', '', '', '', '', '', ''], 14: ['', '', '', '', '', '', '', '', '', ''],
@@ -441,7 +442,7 @@ const Map<int, Map<int, List<String>>> jouyouKanjiByGradeAndStrokeCount = {
16: ['', '', ''], 16: ['', '', ''],
18: ['', '', ''], 18: ['', '', ''],
19: ['', ''], 19: ['', ''],
20: ['', ''], 20: ['', '']
}, },
5: { 5: {
3: ['', ''], 3: ['', ''],
@@ -463,7 +464,7 @@ const Map<int, Map<int, List<String>>> jouyouKanjiByGradeAndStrokeCount = {
'', '',
'', '',
'', '',
'', ''
], ],
8: [ 8: [
'', '',
@@ -483,7 +484,7 @@ const Map<int, Map<int, List<String>>> jouyouKanjiByGradeAndStrokeCount = {
'', '',
'', '',
'', '',
'', ''
], ],
9: ['', '', '', '', '', '', '', '', '', '', '', '', ''], 9: ['', '', '', '', '', '', '', '', '', '', '', '', ''],
10: [ 10: [
@@ -504,7 +505,7 @@ const Map<int, Map<int, List<String>>> jouyouKanjiByGradeAndStrokeCount = {
'', '',
'', '',
'', '',
'', ''
], ],
11: [ 11: [
'', '',
@@ -536,7 +537,7 @@ const Map<int, Map<int, List<String>>> jouyouKanjiByGradeAndStrokeCount = {
'', '',
'', '',
'', '',
'', ''
], ],
12: [ 12: [
'貿', '貿',
@@ -560,7 +561,7 @@ const Map<int, Map<int, List<String>>> jouyouKanjiByGradeAndStrokeCount = {
'', '',
'', '',
'', '',
'', ''
], ],
13: ['', '', '', '', '', '', '', '', '', '', '', '', '', ''], 13: ['', '', '', '', '', '', '', '', '', '', '', '', '', ''],
14: [ 14: [
@@ -582,14 +583,14 @@ const Map<int, Map<int, List<String>>> jouyouKanjiByGradeAndStrokeCount = {
'', '',
'', '',
'', '',
'', ''
], ],
15: ['', '', '', '', '', '', '', ''], 15: ['', '', '', '', '', '', '', ''],
16: ['', '', '', '', ''], 16: ['', '', '', '', ''],
17: ['', '', ''], 17: ['', '', ''],
18: ['', '', ''], 18: ['', '', ''],
19: [''], 19: [''],
20: [''], 20: ['']
}, },
6: { 6: {
3: ['', '', '', ''], 3: ['', '', '', ''],
@@ -617,7 +618,7 @@ const Map<int, Map<int, List<String>>> jouyouKanjiByGradeAndStrokeCount = {
'', '',
'', '',
'沿', '沿',
'', ''
], ],
9: [ 9: [
'', '',
@@ -640,7 +641,7 @@ const Map<int, Map<int, List<String>>> jouyouKanjiByGradeAndStrokeCount = {
'', '',
'', '',
'', '',
'', ''
], ],
10: [ 10: [
'', '',
@@ -666,7 +667,7 @@ const Map<int, Map<int, List<String>>> jouyouKanjiByGradeAndStrokeCount = {
'', '',
'', '',
'', '',
'', ''
], ],
11: [ 11: [
'', '',
@@ -688,7 +689,7 @@ const Map<int, Map<int, List<String>>> jouyouKanjiByGradeAndStrokeCount = {
'', '',
'', '',
'', '',
'', ''
], ],
12: [ 12: [
'', '',
@@ -709,7 +710,7 @@ const Map<int, Map<int, List<String>>> jouyouKanjiByGradeAndStrokeCount = {
'', '',
'', '',
'', '',
'', ''
], ],
13: [ 13: [
'', '',
@@ -726,14 +727,14 @@ const Map<int, Map<int, List<String>>> jouyouKanjiByGradeAndStrokeCount = {
'', '',
'', '',
'', '',
'', ''
], ],
14: ['', '', '', '', '', '', '', '', '', '', '', ''], 14: ['', '', '', '', '', '', '', '', '', '', '', ''],
15: ['', '', '', '', '', '', '', '', '', ''], 15: ['', '', '', '', '', '', '', '', '', ''],
16: ['', '', '', '', '', '', '', ''], 16: ['', '', '', '', '', '', '', ''],
17: ['', '', '', ''], 17: ['', '', '', ''],
18: ['', '', ''], 18: ['', '', ''],
19: ['', ''], 19: ['', '']
}, },
7: { 7: {
1: [''], 1: [''],
@@ -759,7 +760,7 @@ const Map<int, Map<int, List<String>>> jouyouKanjiByGradeAndStrokeCount = {
'', '',
'', '',
'', '',
'', ''
], ],
5: [ 5: [
'', '',
@@ -789,7 +790,7 @@ const Map<int, Map<int, List<String>>> jouyouKanjiByGradeAndStrokeCount = {
'', '',
'', '',
'', '',
'', ''
], ],
6: [ 6: [
'', '',
@@ -830,7 +831,7 @@ const Map<int, Map<int, List<String>>> jouyouKanjiByGradeAndStrokeCount = {
'', '',
'', '',
'', '',
'', ''
], ],
7: [ 7: [
'', '',
@@ -895,7 +896,7 @@ const Map<int, Map<int, List<String>>> jouyouKanjiByGradeAndStrokeCount = {
'', '',
'', '',
'', '',
'', ''
], ],
8: [ 8: [
'', '',
@@ -988,7 +989,7 @@ const Map<int, Map<int, List<String>>> jouyouKanjiByGradeAndStrokeCount = {
'', '',
'', '',
'', '',
'', ''
], ],
9: [ 9: [
'', '',
@@ -1080,7 +1081,7 @@ const Map<int, Map<int, List<String>>> jouyouKanjiByGradeAndStrokeCount = {
'', '',
'', '',
'', '',
'', ''
], ],
10: [ 10: [
'', '',
@@ -1205,7 +1206,7 @@ const Map<int, Map<int, List<String>>> jouyouKanjiByGradeAndStrokeCount = {
'', '',
'', '',
'', '',
'', ''
], ],
11: [ 11: [
'', '',
@@ -1322,7 +1323,7 @@ const Map<int, Map<int, List<String>>> jouyouKanjiByGradeAndStrokeCount = {
'', '',
'', '',
'', '',
'', ''
], ],
12: [ 12: [
'', '',
@@ -1434,7 +1435,7 @@ const Map<int, Map<int, List<String>>> jouyouKanjiByGradeAndStrokeCount = {
'', '',
'', '',
'', '',
'', ''
], ],
13: [ 13: [
'', '',
@@ -1551,7 +1552,7 @@ const Map<int, Map<int, List<String>>> jouyouKanjiByGradeAndStrokeCount = {
'', '',
'', '',
'', '',
'', ''
], ],
14: [ 14: [
'', '',
@@ -1616,7 +1617,7 @@ const Map<int, Map<int, List<String>>> jouyouKanjiByGradeAndStrokeCount = {
'', '',
'', '',
'', '',
'', ''
], ],
15: [ 15: [
'', '',
@@ -1705,7 +1706,7 @@ const Map<int, Map<int, List<String>>> jouyouKanjiByGradeAndStrokeCount = {
'', '',
'', '',
'', '',
'', ''
], ],
16: [ 16: [
'', '',
@@ -1763,7 +1764,7 @@ const Map<int, Map<int, List<String>>> jouyouKanjiByGradeAndStrokeCount = {
'', '',
'', '',
'', '',
'', ''
], ],
17: [ 17: [
'', '',
@@ -1800,7 +1801,7 @@ const Map<int, Map<int, List<String>>> jouyouKanjiByGradeAndStrokeCount = {
'', '',
'', '',
'', '',
'', ''
], ],
18: [ 18: [
'', '',
@@ -1829,7 +1830,7 @@ const Map<int, Map<int, List<String>>> jouyouKanjiByGradeAndStrokeCount = {
'', '',
'', '',
'', '',
'', ''
], ],
19: [ 19: [
'', '',
@@ -1850,23 +1851,22 @@ const Map<int, Map<int, List<String>>> jouyouKanjiByGradeAndStrokeCount = {
'', '',
'', '',
'', '',
'', ''
], ],
20: ['', '', '', '', '', '', '', ''], 20: ['', '', '', '', '', '', '', ''],
21: ['', '', '', '', '', ''], 21: ['', '', '', '', '', ''],
22: ['', '', ''], 22: ['', '', ''],
23: [''], 23: [''],
29: [''], 29: ['']
}, },
}; };
final Map<int, List<String>> jouyouKanjiByGrades = final Map<int, List<String>> JOUYOU_KANJI_BY_GRADES =
jouyouKanjiByGradeAndStrokeCount.entries JOUYOU_KANJI_BY_GRADE_AND_STROKE_COUNT.entries
.expand((entry) => entry.value.entries) .expand((entry) => entry.value.entries)
.map((entry) => MapEntry(entry.key, entry.value)) .map((entry) => MapEntry(entry.key, entry.value))
.fold<Map<int, List<String>>>( .fold<Map<int, List<String>>>(
{}, {},
(acc, entry) => acc (acc, entry) => acc
..putIfAbsent(entry.key, () => []) ..putIfAbsent(entry.key, () => [])
..update(entry.key, (value) => value..addAll(entry.value)), ..update(entry.key, (value) => value..addAll(entry.value)));
);

View File

@@ -1,4 +1,4 @@
const Map<int, List<String>> radicals = { const Map<int, List<String>> RADICALS = {
1: ['', '', '', '', '', ''], 1: ['', '', '', '', '', ''],
2: [ 2: [
'', '',
@@ -31,7 +31,7 @@ const Map<int, List<String>> radicals = {
'', '',
'', '',
'', '',
'𠂉', '𠂉'
], ],
3: [ 3: [
'', '',
@@ -78,7 +78,7 @@ const Map<int, List<String>> radicals = {
'', '',
'', '',
'', '',
'', ''
], ],
4: [ 4: [
'', '',
@@ -124,7 +124,7 @@ const Map<int, List<String>> radicals = {
'', '',
'', '',
'', '',
'', ''
], ],
5: [ 5: [
'', '',
@@ -154,7 +154,7 @@ const Map<int, List<String>> radicals = {
'', '',
'', '',
'', '',
'', ''
], ],
6: [ 6: [
'', '',
@@ -181,7 +181,7 @@ const Map<int, List<String>> radicals = {
'', '',
'', '',
'', '',
'西', '西'
], ],
7: [ 7: [
'', '',
@@ -204,7 +204,7 @@ const Map<int, List<String>> radicals = {
'', '',
'', '',
'', '',
'', ''
], ],
8: ['', '', '', '', '', '', '', '', '', '', '', ''], 8: ['', '', '', '', '', '', '', '', '', '', '', ''],
9: ['', '', '', '', '', '', '', '', '', '', ''], 9: ['', '', '', '', '', '', '', '', '', '', ''],

View File

@@ -43,7 +43,6 @@ enum JlptLevel implements Comparable<JlptLevel> {
int? get asInt => int? get asInt =>
this == JlptLevel.none ? null : JlptLevel.values.indexOf(this); this == JlptLevel.none ? null : JlptLevel.values.indexOf(this);
@override
String toString() => toNullableString() ?? 'N/A'; String toString() => toNullableString() ?? 'N/A';
Object? toJson() => toNullableString(); Object? toJson() => toNullableString();

View File

@@ -11,7 +11,7 @@ String migrationDirPath() {
} }
Future<void> createEmptyDb(DatabaseExecutor db) async { Future<void> createEmptyDb(DatabaseExecutor db) async {
final List<String> migrationFiles = []; List<String> migrationFiles = [];
for (final file in Directory(migrationDirPath()).listSync()) { for (final file in Directory(migrationDirPath()).listSync()) {
if (file is File && file.path.endsWith('.sql')) { if (file is File && file.path.endsWith('.sql')) {
migrationFiles.add(file.path); migrationFiles.add(file.path);

View File

@@ -19,14 +19,20 @@ enum JMdictDialect {
final String id; final String id;
final String description; final String description;
const JMdictDialect({required this.id, required this.description}); const JMdictDialect({
required this.id,
required this.description,
});
static JMdictDialect fromId(String id) => JMdictDialect.values.firstWhere( static JMdictDialect fromId(String id) => JMdictDialect.values.firstWhere(
(e) => e.id == id, (e) => e.id == id,
orElse: () => throw Exception('Unknown id: $id'), orElse: () => throw Exception('Unknown id: $id'),
); );
Map<String, Object?> toJson() => {'id': id, 'description': description}; Map<String, Object?> toJson() => {
'id': id,
'description': description,
};
static JMdictDialect fromJson(Map<String, Object?> json) => static JMdictDialect fromJson(Map<String, Object?> json) =>
JMdictDialect.values.firstWhere( JMdictDialect.values.firstWhere(

View File

@@ -102,14 +102,20 @@ enum JMdictField {
final String id; final String id;
final String description; final String description;
const JMdictField({required this.id, required this.description}); const JMdictField({
required this.id,
required this.description,
});
static JMdictField fromId(String id) => JMdictField.values.firstWhere( static JMdictField fromId(String id) => JMdictField.values.firstWhere(
(e) => e.id == id, (e) => e.id == id,
orElse: () => throw Exception('Unknown id: $id'), orElse: () => throw Exception('Unknown id: $id'),
); );
Map<String, Object?> toJson() => {'id': id, 'description': description}; Map<String, Object?> toJson() => {
'id': id,
'description': description,
};
static JMdictField fromJson(Map<String, Object?> json) => static JMdictField fromJson(Map<String, Object?> json) =>
JMdictField.values.firstWhere( JMdictField.values.firstWhere(

View File

@@ -13,14 +13,20 @@ enum JMdictKanjiInfo {
final String id; final String id;
final String description; final String description;
const JMdictKanjiInfo({required this.id, required this.description}); const JMdictKanjiInfo({
required this.id,
required this.description,
});
static JMdictKanjiInfo fromId(String id) => JMdictKanjiInfo.values.firstWhere( static JMdictKanjiInfo fromId(String id) => JMdictKanjiInfo.values.firstWhere(
(e) => e.id == id, (e) => e.id == id,
orElse: () => throw Exception('Unknown id: $id'), orElse: () => throw Exception('Unknown id: $id'),
); );
Map<String, Object?> toJson() => {'id': id, 'description': description}; Map<String, Object?> toJson() => {
'id': id,
'description': description,
};
static JMdictKanjiInfo fromJson(Map<String, Object?> json) => static JMdictKanjiInfo fromJson(Map<String, Object?> json) =>
JMdictKanjiInfo.values.firstWhere( JMdictKanjiInfo.values.firstWhere(

View File

@@ -74,14 +74,20 @@ enum JMdictMisc {
final String id; final String id;
final String description; final String description;
const JMdictMisc({required this.id, required this.description}); const JMdictMisc({
required this.id,
required this.description,
});
static JMdictMisc fromId(String id) => JMdictMisc.values.firstWhere( static JMdictMisc fromId(String id) => JMdictMisc.values.firstWhere(
(e) => e.id == id, (e) => e.id == id,
orElse: () => throw Exception('Unknown id: $id'), orElse: () => throw Exception('Unknown id: $id'),
); );
Map<String, Object?> toJson() => {'id': id, 'description': description}; Map<String, Object?> toJson() => {
'id': id,
'description': description,
};
static JMdictMisc fromJson(Map<String, Object?> json) => static JMdictMisc fromJson(Map<String, Object?> json) =>
JMdictMisc.values.firstWhere( JMdictMisc.values.firstWhere(

View File

@@ -202,11 +202,14 @@ enum JMdictPOS {
String get shortDescription => _shortDescription ?? description; String get shortDescription => _shortDescription ?? description;
static JMdictPOS fromId(String id) => JMdictPOS.values.firstWhere( static JMdictPOS fromId(String id) => JMdictPOS.values.firstWhere(
(e) => e.id == id, (e) => e.id == id,
orElse: () => throw Exception('Unknown id: $id'), orElse: () => throw Exception('Unknown id: $id'),
); );
Map<String, Object?> toJson() => {'id': id, 'description': description}; Map<String, Object?> toJson() => {
'id': id,
'description': description,
};
static JMdictPOS fromJson(Map<String, Object?> json) => static JMdictPOS fromJson(Map<String, Object?> json) =>
JMdictPOS.values.firstWhere( JMdictPOS.values.firstWhere(

View File

@@ -15,7 +15,10 @@ enum JMdictReadingInfo {
final String id; final String id;
final String description; final String description;
const JMdictReadingInfo({required this.id, required this.description}); const JMdictReadingInfo({
required this.id,
required this.description,
});
static JMdictReadingInfo fromId(String id) => static JMdictReadingInfo fromId(String id) =>
JMdictReadingInfo.values.firstWhere( JMdictReadingInfo.values.firstWhere(
@@ -23,7 +26,10 @@ enum JMdictReadingInfo {
orElse: () => throw Exception('Unknown id: $id'), orElse: () => throw Exception('Unknown id: $id'),
); );
Map<String, Object?> toJson() => {'id': id, 'description': description}; Map<String, Object?> toJson() => {
'id': id,
'description': description,
};
static JMdictReadingInfo fromJson(Map<String, Object?> json) => static JMdictReadingInfo fromJson(Map<String, Object?> json) =>
JMdictReadingInfo.values.firstWhere( JMdictReadingInfo.values.firstWhere(

View File

@@ -26,14 +26,19 @@ class KanjiSearchRadical extends Equatable {
}); });
@override @override
List<Object> get props => [symbol, names, forms, meanings]; List<Object> get props => [
symbol,
this.names,
forms,
meanings,
];
Map<String, dynamic> toJson() => { Map<String, dynamic> toJson() => {
'symbol': symbol, 'symbol': symbol,
'names': names, 'names': names,
'forms': forms, 'forms': forms,
'meanings': meanings, 'meanings': meanings,
}; };
factory KanjiSearchRadical.fromJson(Map<String, dynamic> json) { factory KanjiSearchRadical.fromJson(Map<String, dynamic> json) {
return KanjiSearchRadical( return KanjiSearchRadical(

View File

@@ -89,46 +89,46 @@ class KanjiSearchResult extends Equatable {
@override @override
// ignore: public_member_api_docs // ignore: public_member_api_docs
List<Object?> get props => [ List<Object?> get props => [
taughtIn, taughtIn,
jlptLevel, jlptLevel,
newspaperFrequencyRank, newspaperFrequencyRank,
strokeCount, strokeCount,
meanings, meanings,
kunyomi, kunyomi,
onyomi, onyomi,
// kunyomiExamples, // kunyomiExamples,
// onyomiExamples, // onyomiExamples,
radical, radical,
parts, parts,
codepoints, codepoints,
kanji, kanji,
nanori, nanori,
alternativeLanguageReadings, alternativeLanguageReadings,
strokeMiscounts, strokeMiscounts,
queryCodes, queryCodes,
dictionaryReferences, dictionaryReferences,
]; ];
Map<String, dynamic> toJson() => { Map<String, dynamic> toJson() => {
'kanji': kanji, 'kanji': kanji,
'taughtIn': taughtIn, 'taughtIn': taughtIn,
'jlptLevel': jlptLevel, 'jlptLevel': jlptLevel,
'newspaperFrequencyRank': newspaperFrequencyRank, 'newspaperFrequencyRank': newspaperFrequencyRank,
'strokeCount': strokeCount, 'strokeCount': strokeCount,
'meanings': meanings, 'meanings': meanings,
'kunyomi': kunyomi, 'kunyomi': kunyomi,
'onyomi': onyomi, 'onyomi': onyomi,
// 'onyomiExamples': onyomiExamples, // 'onyomiExamples': onyomiExamples,
// 'kunyomiExamples': kunyomiExamples, // 'kunyomiExamples': kunyomiExamples,
'radical': radical?.toJson(), 'radical': radical?.toJson(),
'parts': parts, 'parts': parts,
'codepoints': codepoints, 'codepoints': codepoints,
'nanori': nanori, 'nanori': nanori,
'alternativeLanguageReadings': alternativeLanguageReadings, 'alternativeLanguageReadings': alternativeLanguageReadings,
'strokeMiscounts': strokeMiscounts, 'strokeMiscounts': strokeMiscounts,
'queryCodes': queryCodes, 'queryCodes': queryCodes,
'dictionaryReferences': dictionaryReferences, 'dictionaryReferences': dictionaryReferences,
}; };
factory KanjiSearchResult.fromJson(Map<String, dynamic> json) { factory KanjiSearchResult.fromJson(Map<String, dynamic> json) {
return KanjiSearchResult( return KanjiSearchResult(
@@ -156,20 +156,23 @@ class KanjiSearchResult extends Equatable {
nanori: (json['nanori'] as List).map((e) => e as String).toList(), nanori: (json['nanori'] as List).map((e) => e as String).toList(),
alternativeLanguageReadings: alternativeLanguageReadings:
(json['alternativeLanguageReadings'] as Map<String, dynamic>).map( (json['alternativeLanguageReadings'] as Map<String, dynamic>).map(
(key, value) => (key, value) => MapEntry(
MapEntry(key, (value as List).map((e) => e as String).toList()), key,
), (value as List).map((e) => e as String).toList(),
strokeMiscounts: (json['strokeMiscounts'] as List) ),
.map((e) => e as int) ),
.toList(), strokeMiscounts:
(json['strokeMiscounts'] as List).map((e) => e as int).toList(),
queryCodes: (json['queryCodes'] as Map<String, dynamic>).map( queryCodes: (json['queryCodes'] as Map<String, dynamic>).map(
(key, value) => (key, value) => MapEntry(
MapEntry(key, (value as List).map((e) => e as String).toList()), key,
(value as List).map((e) => e as String).toList(),
),
), ),
dictionaryReferences: dictionaryReferences:
(json['dictionaryReferences'] as Map<String, dynamic>).map( (json['dictionaryReferences'] as Map<String, dynamic>).map(
(key, value) => MapEntry(key, value as String), (key, value) => MapEntry(key, value as String),
), ),
); );
} }
} }

View File

@@ -7,14 +7,14 @@ import 'package:sqflite_common/sqlite_api.dart';
Future<void> verifyTablesWithDbConnection(DatabaseExecutor db) async { Future<void> verifyTablesWithDbConnection(DatabaseExecutor db) async {
final Set<String> tables = await db final Set<String> tables = await db
.query( .query(
'sqlite_master', 'sqlite_master',
columns: ['name'], columns: ['name'],
where: 'type = ?', where: 'type = ?',
whereArgs: ['table'], whereArgs: ['table'],
) )
.then((result) { .then((result) {
return result.map((row) => row['name'] as String).toSet(); return result.map((row) => row['name'] as String).toSet();
}); });
final Set<String> expectedTables = { final Set<String> expectedTables = {
...JMdictTableNames.allTables, ...JMdictTableNames.allTables,
@@ -26,16 +26,14 @@ Future<void> verifyTablesWithDbConnection(DatabaseExecutor db) async {
final missingTables = expectedTables.difference(tables); final missingTables = expectedTables.difference(tables);
if (missingTables.isNotEmpty) { if (missingTables.isNotEmpty) {
throw Exception( throw Exception([
[ 'Missing tables:',
'Missing tables:', missingTables.map((table) => ' - $table').join('\n'),
missingTables.map((table) => ' - $table').join('\n'), '',
'', 'Found tables:\n',
'Found tables:\n', tables.map((table) => ' - $table').join('\n'),
tables.map((table) => ' - $table').join('\n'), '',
'', 'Please ensure the database is correctly set up.',
'Please ensure the database is correctly set up.', ].join('\n'));
].join('\n'),
);
} }
} }

View File

@@ -1,62 +0,0 @@
enum WordSearchMatchSpanType { kanji, kana, sense }
/// A span of a word search result that corresponds to a match for a kanji, kana, or sense.
class WordSearchMatchSpan {
/// Which subtype of the word search result this span corresponds to - either a kanji, a kana, or a sense.
final WordSearchMatchSpanType spanType;
/// The index of the kanji/kana/sense in the word search result that this span corresponds to.
final int index;
/// When matching a 'sense', this is the index of the English definition in that sense that this span corresponds to. Otherwise, this is always 0.
final int subIndex;
/// The start of the span (inclusive)
final int start;
/// The end of the span (inclusive)
final int end;
WordSearchMatchSpan({
required this.spanType,
required this.index,
required this.start,
required this.end,
this.subIndex = 0,
});
@override
String toString() {
return 'WordSearchMatchSpan(spanType: $spanType, index: $index, start: $start, end: $end)';
}
Map<String, Object?> toJson() => {
'spanType': spanType.toString().split('.').last,
'index': index,
'start': start,
'end': end,
};
factory WordSearchMatchSpan.fromJson(Map<String, dynamic> json) =>
WordSearchMatchSpan(
spanType: WordSearchMatchSpanType.values.firstWhere(
(e) => e.toString().split('.').last == json['spanType'],
),
index: json['index'] as int,
start: json['start'] as int,
end: json['end'] as int,
);
@override
int get hashCode => Object.hash(spanType, index, start, end);
@override
bool operator ==(Object other) {
if (identical(this, other)) return true;
return other is WordSearchMatchSpan &&
other.spanType == spanType &&
other.index == index &&
other.start == start &&
other.end == end;
}
}

View File

@@ -1,12 +1,9 @@
import 'package:jadb/models/common/jlpt_level.dart'; import 'package:jadb/models/common/jlpt_level.dart';
import 'package:jadb/models/jmdict/jmdict_kanji_info.dart'; import 'package:jadb/models/jmdict/jmdict_kanji_info.dart';
import 'package:jadb/models/jmdict/jmdict_reading_info.dart'; import 'package:jadb/models/jmdict/jmdict_reading_info.dart';
import 'package:jadb/models/word_search/word_search_match_span.dart';
import 'package:jadb/models/word_search/word_search_ruby.dart'; import 'package:jadb/models/word_search/word_search_ruby.dart';
import 'package:jadb/models/word_search/word_search_sense.dart'; import 'package:jadb/models/word_search/word_search_sense.dart';
import 'package:jadb/models/word_search/word_search_sources.dart'; import 'package:jadb/models/word_search/word_search_sources.dart';
import 'package:jadb/search/word_search/word_search.dart';
import 'package:jadb/util/romaji_transliteration.dart';
/// A class representing a single dictionary entry from a word search. /// A class representing a single dictionary entry from a word search.
class WordSearchResult { class WordSearchResult {
@@ -37,44 +34,7 @@ class WordSearchResult {
/// A class listing the sources used to make up the data for this word search result. /// A class listing the sources used to make up the data for this word search result.
final WordSearchSources sources; final WordSearchSources sources;
/// A list of spans, specifying which part of this word result matched the search keyword. const WordSearchResult({
///
/// Note that this is considered ephemeral data - it does not originate from the dictionary,
/// and unlike the rest of the class it varies based on external information (the searchword).
/// It will *NOT* be exported to JSON, but can be reinferred by invoking [inferMatchSpans] with
/// the original searchword.
List<WordSearchMatchSpan>? matchSpans;
/// All contents of [japanese], transliterated to romaji
List<String> get romaji => japanese
.map((word) => transliterateKanaToLatin(word.furigana ?? word.base))
.toList();
/// All contents of [japanase], where the furigana has either been transliterated to romaji, or
/// contains the furigana transliteration of [WordSearchRuby.base].
List<WordSearchRuby> get romajiRubys => japanese
.map(
(word) => WordSearchRuby(
base: word.base,
furigana: word.furigana != null
? transliterateKanaToLatin(word.furigana!)
: transliterateKanaToLatin(word.base),
),
)
.toList();
/// The same list of spans as [matchSpans], but the positions have been adjusted for romaji conversion
///
/// This is mostly useful in conjunction with [romajiRubys].
List<WordSearchMatchSpan>? get romajiMatchSpans {
if (matchSpans == null) {
return null;
}
throw UnimplementedError('Not yet implemented');
}
WordSearchResult({
required this.score, required this.score,
required this.entryId, required this.entryId,
required this.isCommon, required this.isCommon,
@@ -84,22 +44,21 @@ class WordSearchResult {
required this.senses, required this.senses,
required this.jlptLevel, required this.jlptLevel,
required this.sources, required this.sources,
this.matchSpans,
}); });
Map<String, dynamic> toJson() => { Map<String, dynamic> toJson() => {
'_score': score, '_score': score,
'entryId': entryId, 'entryId': entryId,
'isCommon': isCommon, 'isCommon': isCommon,
'japanese': japanese.map((e) => e.toJson()).toList(), 'japanese': japanese.map((e) => e.toJson()).toList(),
'kanjiInfo': kanjiInfo.map((key, value) => MapEntry(key, value.toJson())), 'kanjiInfo':
'readingInfo': readingInfo.map( kanjiInfo.map((key, value) => MapEntry(key, value.toJson())),
(key, value) => MapEntry(key, value.toJson()), 'readingInfo':
), readingInfo.map((key, value) => MapEntry(key, value.toJson())),
'senses': senses.map((e) => e.toJson()).toList(), 'senses': senses.map((e) => e.toJson()).toList(),
'jlptLevel': jlptLevel.toJson(), 'jlptLevel': jlptLevel.toJson(),
'sources': sources.toJson(), 'sources': sources.toJson(),
}; };
factory WordSearchResult.fromJson(Map<String, dynamic> json) => factory WordSearchResult.fromJson(Map<String, dynamic> json) =>
WordSearchResult( WordSearchResult(
@@ -122,88 +81,17 @@ class WordSearchResult {
sources: WordSearchSources.fromJson(json['sources']), sources: WordSearchSources.fromJson(json['sources']),
); );
factory WordSearchResult.empty() => WordSearchResult( String _formatJapaneseWord(WordSearchRuby word) =>
score: 0, word.furigana == null ? word.base : "${word.base} (${word.furigana})";
entryId: 0,
isCommon: false,
japanese: [],
kanjiInfo: {},
readingInfo: {},
senses: [],
jlptLevel: JlptLevel.none,
sources: WordSearchSources.empty(),
);
/// Infers which part(s) of this word search result matched the search keyword, and populates [matchSpans] accordingly.
void inferMatchSpans(
String searchword, {
SearchMode searchMode = SearchMode.auto,
}) {
// TODO: handle wildcards like '?' and '*' when that becomes supported in the search.
// TODO: If the searchMode is provided, we can use that to narrow down which part of the word search results to look at.
final regex = RegExp(RegExp.escape(searchword));
final matchSpans = <WordSearchMatchSpan>[];
for (final (i, japanese) in japanese.indexed) {
final baseMatches = regex.allMatches(japanese.base);
matchSpans.addAll(
baseMatches.map(
(match) => WordSearchMatchSpan(
spanType: WordSearchMatchSpanType.kanji,
index: i,
start: match.start,
end: match.end,
),
),
);
if (japanese.furigana != null) {
final furiganaMatches = regex.allMatches(japanese.furigana!);
matchSpans.addAll(
furiganaMatches.map(
(match) => WordSearchMatchSpan(
spanType: WordSearchMatchSpanType.kana,
index: i,
start: match.start,
end: match.end,
),
),
);
}
}
for (final (i, sense) in senses.indexed) {
for (final (k, definition) in sense.englishDefinitions.indexed) {
final definitionMatches = regex.allMatches(definition);
matchSpans.addAll(
definitionMatches.map(
(match) => WordSearchMatchSpan(
spanType: WordSearchMatchSpanType.sense,
index: i,
subIndex: k,
start: match.start,
end: match.end,
),
),
);
}
}
this.matchSpans = matchSpans;
}
static String _formatJapaneseWord(WordSearchRuby word) =>
word.furigana == null ? word.base : '${word.base} (${word.furigana})';
@override @override
String toString() { String toString() {
final japaneseWord = _formatJapaneseWord(japanese[0]); final japaneseWord = _formatJapaneseWord(japanese[0]);
final isCommonString = isCommon ? '(C)' : ''; final isCommonString = isCommon ? '(C)' : '';
final jlptLevelString = '(${jlptLevel.toString()})'; final jlptLevelString = "(${jlptLevel.toString()})";
return ''' return '''
$score | [$entryId] $japaneseWord $isCommonString $jlptLevelString ${score} | [$entryId] $japaneseWord $isCommonString $jlptLevelString
Other forms: ${japanese.skip(1).map(_formatJapaneseWord).join(', ')} Other forms: ${japanese.skip(1).map(_formatJapaneseWord).join(', ')}
Senses: ${senses.map((s) => s.englishDefinitions).join(', ')} Senses: ${senses.map((s) => s.englishDefinitions).join(', ')}
''' '''

View File

@@ -6,12 +6,18 @@ class WordSearchRuby {
/// Furigana, if applicable. /// Furigana, if applicable.
String? furigana; String? furigana;
WordSearchRuby({required this.base, this.furigana}); WordSearchRuby({
required this.base,
this.furigana,
});
Map<String, dynamic> toJson() => {'base': base, 'furigana': furigana}; Map<String, dynamic> toJson() => {
'base': base,
'furigana': furigana,
};
factory WordSearchRuby.fromJson(Map<String, dynamic> json) => WordSearchRuby( factory WordSearchRuby.fromJson(Map<String, dynamic> json) => WordSearchRuby(
base: json['base'] as String, base: json['base'] as String,
furigana: json['furigana'] as String?, furigana: json['furigana'] as String?,
); );
} }

View File

@@ -71,18 +71,18 @@ class WordSearchSense {
languageSource.isEmpty; languageSource.isEmpty;
Map<String, dynamic> toJson() => { Map<String, dynamic> toJson() => {
'englishDefinitions': englishDefinitions, 'englishDefinitions': englishDefinitions,
'partsOfSpeech': partsOfSpeech.map((e) => e.toJson()).toList(), 'partsOfSpeech': partsOfSpeech.map((e) => e.toJson()).toList(),
'seeAlso': seeAlso.map((e) => e.toJson()).toList(), 'seeAlso': seeAlso.map((e) => e.toJson()).toList(),
'antonyms': antonyms.map((e) => e.toJson()).toList(), 'antonyms': antonyms.map((e) => e.toJson()).toList(),
'restrictedToReading': restrictedToReading, 'restrictedToReading': restrictedToReading,
'restrictedToKanji': restrictedToKanji, 'restrictedToKanji': restrictedToKanji,
'fields': fields.map((e) => e.toJson()).toList(), 'fields': fields.map((e) => e.toJson()).toList(),
'dialects': dialects.map((e) => e.toJson()).toList(), 'dialects': dialects.map((e) => e.toJson()).toList(),
'misc': misc.map((e) => e.toJson()).toList(), 'misc': misc.map((e) => e.toJson()).toList(),
'info': info, 'info': info,
'languageSource': languageSource, 'languageSource': languageSource,
}; };
factory WordSearchSense.fromJson(Map<String, dynamic> json) => factory WordSearchSense.fromJson(Map<String, dynamic> json) =>
WordSearchSense( WordSearchSense(
@@ -104,9 +104,8 @@ class WordSearchSense {
dialects: (json['dialects'] as List) dialects: (json['dialects'] as List)
.map((e) => JMdictDialect.fromJson(e)) .map((e) => JMdictDialect.fromJson(e))
.toList(), .toList(),
misc: (json['misc'] as List) misc:
.map((e) => JMdictMisc.fromJson(e)) (json['misc'] as List).map((e) => JMdictMisc.fromJson(e)).toList(),
.toList(),
info: List<String>.from(json['info']), info: List<String>.from(json['info']),
languageSource: (json['languageSource'] as List) languageSource: (json['languageSource'] as List)
.map((e) => WordSearchSenseLanguageSource.fromJson(e)) .map((e) => WordSearchSenseLanguageSource.fromJson(e))

View File

@@ -13,11 +13,11 @@ class WordSearchSenseLanguageSource {
}); });
Map<String, Object?> toJson() => { Map<String, Object?> toJson() => {
'language': language, 'language': language,
'phrase': phrase, 'phrase': phrase,
'fullyDescribesSense': fullyDescribesSense, 'fullyDescribesSense': fullyDescribesSense,
'constructedFromSmallerWords': constructedFromSmallerWords, 'constructedFromSmallerWords': constructedFromSmallerWords,
}; };
factory WordSearchSenseLanguageSource.fromJson(Map<String, dynamic> json) => factory WordSearchSenseLanguageSource.fromJson(Map<String, dynamic> json) =>
WordSearchSenseLanguageSource( WordSearchSenseLanguageSource(

View File

@@ -7,13 +7,20 @@ class WordSearchSources {
/// Whether JMnedict was used. /// Whether JMnedict was used.
final bool jmnedict; final bool jmnedict;
const WordSearchSources({this.jmdict = true, this.jmnedict = false}); const WordSearchSources({
this.jmdict = true,
this.jmnedict = false,
});
factory WordSearchSources.empty() => const WordSearchSources(); Map<String, Object?> get sqlValue => {
'jmdict': jmdict,
'jmnedict': jmnedict,
};
Map<String, Object?> get sqlValue => {'jmdict': jmdict, 'jmnedict': jmnedict}; Map<String, dynamic> toJson() => {
'jmdict': jmdict,
Map<String, dynamic> toJson() => {'jmdict': jmdict, 'jmnedict': jmnedict}; 'jmnedict': jmnedict,
};
factory WordSearchSources.fromJson(Map<String, dynamic> json) => factory WordSearchSources.fromJson(Map<String, dynamic> json) =>
WordSearchSources( WordSearchSources(

View File

@@ -1,5 +1,3 @@
import 'package:jadb/models/word_search/word_search_result.dart';
/// A cross-reference entry from one word-result to another entry. /// A cross-reference entry from one word-result to another entry.
class WordSearchXrefEntry { class WordSearchXrefEntry {
/// The ID of the entry that this entry cross-references to. /// The ID of the entry that this entry cross-references to.
@@ -15,24 +13,19 @@ class WordSearchXrefEntry {
/// database (and hence might be incorrect). /// database (and hence might be incorrect).
final bool ambiguous; final bool ambiguous;
/// The result of the cross-reference, may or may not be included in the query.
final WordSearchResult? xrefResult;
const WordSearchXrefEntry({ const WordSearchXrefEntry({
required this.entryId, required this.entryId,
required this.ambiguous, required this.ambiguous,
required this.baseWord, required this.baseWord,
required this.furigana, required this.furigana,
required this.xrefResult,
}); });
Map<String, dynamic> toJson() => { Map<String, dynamic> toJson() => {
'entryId': entryId, 'entryId': entryId,
'ambiguous': ambiguous, 'ambiguous': ambiguous,
'baseWord': baseWord, 'baseWord': baseWord,
'furigana': furigana, 'furigana': furigana,
'xrefResult': xrefResult?.toJson(), };
};
factory WordSearchXrefEntry.fromJson(Map<String, dynamic> json) => factory WordSearchXrefEntry.fromJson(Map<String, dynamic> json) =>
WordSearchXrefEntry( WordSearchXrefEntry(
@@ -40,6 +33,5 @@ class WordSearchXrefEntry {
ambiguous: json['ambiguous'] as bool, ambiguous: json['ambiguous'] as bool,
baseWord: json['baseWord'] as String, baseWord: json['baseWord'] as String,
furigana: json['furigana'] as String?, furigana: json['furigana'] as String?,
xrefResult: null,
); );
} }

View File

@@ -1,10 +1,12 @@
import 'package:jadb/models/kanji_search/kanji_search_result.dart';
import 'package:jadb/models/verify_tables.dart'; import 'package:jadb/models/verify_tables.dart';
import 'package:jadb/models/word_search/word_search_result.dart'; import 'package:jadb/models/word_search/word_search_result.dart';
import 'package:jadb/models/kanji_search/kanji_search_result.dart';
import 'package:jadb/search/filter_kanji.dart'; import 'package:jadb/search/filter_kanji.dart';
import 'package:jadb/search/kanji_search.dart';
import 'package:jadb/search/radical_search.dart'; import 'package:jadb/search/radical_search.dart';
import 'package:jadb/search/word_search/word_search.dart'; import 'package:jadb/search/word_search/word_search.dart';
import 'package:jadb/search/kanji_search.dart';
import 'package:sqflite_common/sqlite_api.dart'; import 'package:sqflite_common/sqlite_api.dart';
extension JaDBConnection on DatabaseExecutor { extension JaDBConnection on DatabaseExecutor {
@@ -17,45 +19,38 @@ extension JaDBConnection on DatabaseExecutor {
Future<KanjiSearchResult?> jadbSearchKanji(String kanji) => Future<KanjiSearchResult?> jadbSearchKanji(String kanji) =>
searchKanjiWithDbConnection(this, kanji); searchKanjiWithDbConnection(this, kanji);
/// Search for a kanji in the database.
Future<Map<String, KanjiSearchResult>> jadbGetManyKanji(Set<String> kanji) =>
searchManyKanjiWithDbConnection(this, kanji);
/// Filter a list of characters, and return the ones that are listed in the kanji dictionary. /// Filter a list of characters, and return the ones that are listed in the kanji dictionary.
Future<List<String>> filterKanji( Future<List<String>> filterKanji(
List<String> kanji, { List<String> kanji, {
bool deduplicate = false, bool deduplicate = false,
}) => filterKanjiWithDbConnection(this, kanji, deduplicate); }) =>
filterKanjiWithDbConnection(this, kanji, deduplicate);
/// Search for a word in the database. /// Search for a word in the database.
Future<List<WordSearchResult>?> jadbSearchWord( Future<List<WordSearchResult>?> jadbSearchWord(
String word, { String word, {
SearchMode searchMode = SearchMode.auto, SearchMode searchMode = SearchMode.Auto,
int page = 0, int page = 0,
int? pageSize, int pageSize = 10,
}) => searchWordWithDbConnection( }) =>
this, searchWordWithDbConnection(
word, this,
searchMode: searchMode, word,
page: page, searchMode,
pageSize: pageSize, page,
); pageSize,
);
/// ///
Future<WordSearchResult?> jadbGetWordById(int id) => Future<WordSearchResult?> jadbGetWordById(int id) =>
getWordByIdWithDbConnection(this, id); getWordByIdWithDbConnection(this, id);
/// Get a list of words by their IDs.
///
/// IDs for which no result is found are omitted from the returned value.
Future<Map<int, WordSearchResult>> jadbGetManyWordsByIds(Set<int> ids) =>
getWordsByIdsWithDbConnection(this, ids);
/// Search for a word in the database, and return the count of results. /// Search for a word in the database, and return the count of results.
Future<int?> jadbSearchWordCount( Future<int?> jadbSearchWordCount(
String word, { String word, {
SearchMode searchMode = SearchMode.auto, SearchMode searchMode = SearchMode.Auto,
}) => searchWordCountWithDbConnection(this, word, searchMode: searchMode); }) =>
searchWordCountWithDbConnection(this, word, searchMode);
/// Given a list of radicals, search which kanji contains all /// Given a list of radicals, search which kanji contains all
/// of the radicals, find their other radicals, and return those. /// of the radicals, find their other radicals, and return those.

View File

@@ -1,21 +1,19 @@
import 'package:jadb/table_names/kanjidic.dart'; import 'package:jadb/table_names/kanjidic.dart';
import 'package:sqflite_common/sqflite.dart'; import 'package:sqflite_common/sqflite.dart';
/// Filters a list of kanji characters, returning only those that exist in the database.
///
/// If [deduplicate] is true, the returned list will deduplicate the input kanji list before returning the filtered results.
Future<List<String>> filterKanjiWithDbConnection( Future<List<String>> filterKanjiWithDbConnection(
DatabaseExecutor connection, DatabaseExecutor connection,
List<String> kanji, List<String> kanji,
bool deduplicate, bool deduplicate,
) async { ) async {
final Set<String> filteredKanji = await connection final Set<String> filteredKanji = await connection.rawQuery(
.rawQuery(''' '''
SELECT "literal" SELECT "literal"
FROM "${KANJIDICTableNames.character}" FROM "${KANJIDICTableNames.character}"
WHERE "literal" IN (${kanji.map((_) => '?').join(',')}) WHERE "literal" IN (${kanji.map((_) => '?').join(',')})
''', kanji) ''',
.then((value) => value.map((e) => e['literal'] as String).toSet()); kanji,
).then((value) => value.map((e) => e['literal'] as String).toSet());
if (deduplicate) { if (deduplicate) {
return filteredKanji.toList(); return filteredKanji.toList();

View File

@@ -1,190 +1,143 @@
import 'package:collection/collection.dart'; import 'package:collection/collection.dart';
import 'package:jadb/models/kanji_search/kanji_search_radical.dart';
import 'package:jadb/models/kanji_search/kanji_search_result.dart';
import 'package:jadb/table_names/kanjidic.dart'; import 'package:jadb/table_names/kanjidic.dart';
import 'package:jadb/table_names/radkfile.dart'; import 'package:jadb/table_names/radkfile.dart';
import 'package:jadb/models/kanji_search/kanji_search_radical.dart';
import 'package:jadb/models/kanji_search/kanji_search_result.dart';
import 'package:sqflite_common/sqflite.dart'; import 'package:sqflite_common/sqflite.dart';
Future<List<Map<String, Object?>>> _charactersQuery(
DatabaseExecutor connection,
String kanji,
) => connection.query(
KANJIDICTableNames.character,
where: 'literal = ?',
whereArgs: [kanji],
);
Future<List<Map<String, Object?>>> _codepointsQuery(
DatabaseExecutor connection,
String kanji,
) => connection.query(
KANJIDICTableNames.codepoint,
where: 'kanji = ?',
whereArgs: [kanji],
);
Future<List<Map<String, Object?>>> _kunyomisQuery(
DatabaseExecutor connection,
String kanji,
) => connection.query(
KANJIDICTableNames.kunyomi,
where: 'kanji = ?',
whereArgs: [kanji],
orderBy: 'orderNum',
);
Future<List<Map<String, Object?>>> _onyomisQuery(
DatabaseExecutor connection,
String kanji,
) => connection.query(
KANJIDICTableNames.onyomi,
where: 'kanji = ?',
whereArgs: [kanji],
orderBy: 'orderNum',
);
Future<List<Map<String, Object?>>> _meaningsQuery(
DatabaseExecutor connection,
String kanji,
) => connection.query(
KANJIDICTableNames.meaning,
where: 'kanji = ? AND language = ?',
whereArgs: [kanji, 'eng'],
orderBy: 'orderNum',
);
Future<List<Map<String, Object?>>> _nanorisQuery(
DatabaseExecutor connection,
String kanji,
) => connection.query(
KANJIDICTableNames.nanori,
where: 'kanji = ?',
whereArgs: [kanji],
);
Future<List<Map<String, Object?>>> _dictionaryReferencesQuery(
DatabaseExecutor connection,
String kanji,
) => connection.query(
KANJIDICTableNames.dictionaryReference,
where: 'kanji = ?',
whereArgs: [kanji],
);
Future<List<Map<String, Object?>>> _queryCodesQuery(
DatabaseExecutor connection,
String kanji,
) => connection.query(
KANJIDICTableNames.queryCode,
where: 'kanji = ?',
whereArgs: [kanji],
);
Future<List<Map<String, Object?>>> _radicalsQuery(
DatabaseExecutor connection,
String kanji,
) => connection.rawQuery(
'''
SELECT DISTINCT
"XREF__KANJIDIC_Radical__RADKFILE"."radicalSymbol" AS "symbol",
"names"
FROM "${KANJIDICTableNames.radical}"
JOIN "XREF__KANJIDIC_Radical__RADKFILE" USING ("radicalId")
LEFT JOIN (
SELECT "radicalId", group_concat("name") AS "names"
FROM "${KANJIDICTableNames.radicalName}"
GROUP BY "radicalId"
) USING ("radicalId")
WHERE "${KANJIDICTableNames.radical}"."kanji" = ?
''',
[kanji],
);
Future<List<Map<String, Object?>>> _partsQuery(
DatabaseExecutor connection,
String kanji,
) => connection.query(
RADKFILETableNames.radkfile,
where: 'kanji = ?',
whereArgs: [kanji],
);
Future<List<Map<String, Object?>>> _readingsQuery(
DatabaseExecutor connection,
String kanji,
) => connection.query(
KANJIDICTableNames.reading,
where: 'kanji = ?',
whereArgs: [kanji],
);
Future<List<Map<String, Object?>>> _strokeMiscountsQuery(
DatabaseExecutor connection,
String kanji,
) => connection.query(
KANJIDICTableNames.strokeMiscount,
where: 'kanji = ?',
whereArgs: [kanji],
);
// Future<List<Map<String, Object?>>> _variantsQuery(
// DatabaseExecutor connection,
// String kanji,
// ) => connection.query(
// KANJIDICTableNames.variant,
// where: 'kanji = ?',
// whereArgs: [kanji],
// );
/// Searches for a kanji character and returns its details, or null if the kanji is not found in the database.
Future<KanjiSearchResult?> searchKanjiWithDbConnection( Future<KanjiSearchResult?> searchKanjiWithDbConnection(
DatabaseExecutor connection, DatabaseExecutor connection,
String kanji, String kanji,
) async { ) async {
late final List<Map<String, Object?>> characters; late final List<Map<String, Object?>> characters;
final characters_query = connection.query(
KANJIDICTableNames.character,
where: "literal = ?",
whereArgs: [kanji],
);
late final List<Map<String, Object?>> codepoints; late final List<Map<String, Object?>> codepoints;
final codepoints_query = connection.query(
KANJIDICTableNames.codepoint,
where: "kanji = ?",
whereArgs: [kanji],
);
late final List<Map<String, Object?>> kunyomis; late final List<Map<String, Object?>> kunyomis;
final kunyomis_query = connection.query(
KANJIDICTableNames.kunyomi,
where: "kanji = ?",
whereArgs: [kanji],
orderBy: "orderNum",
);
late final List<Map<String, Object?>> onyomis; late final List<Map<String, Object?>> onyomis;
final onyomis_query = connection.query(
KANJIDICTableNames.onyomi,
where: "kanji = ?",
whereArgs: [kanji],
orderBy: "orderNum",
);
late final List<Map<String, Object?>> meanings; late final List<Map<String, Object?>> meanings;
final meanings_query = connection.query(
KANJIDICTableNames.meaning,
where: "kanji = ? AND language = ?",
whereArgs: [kanji, 'eng'],
orderBy: "orderNum",
);
late final List<Map<String, Object?>> nanoris; late final List<Map<String, Object?>> nanoris;
late final List<Map<String, Object?>> dictionaryReferences; final nanoris_query = connection.query(
late final List<Map<String, Object?>> queryCodes; KANJIDICTableNames.nanori,
where: "kanji = ?",
whereArgs: [kanji],
);
late final List<Map<String, Object?>> dictionary_references;
final dictionary_references_query = connection.query(
KANJIDICTableNames.dictionaryReference,
where: "kanji = ?",
whereArgs: [kanji],
);
late final List<Map<String, Object?>> query_codes;
final query_codes_query = connection.query(
KANJIDICTableNames.queryCode,
where: "kanji = ?",
whereArgs: [kanji],
);
late final List<Map<String, Object?>> radicals; late final List<Map<String, Object?>> radicals;
final radicals_query = connection.rawQuery(
'''
SELECT DISTINCT
"XREF__KANJIDIC_Radical__RADKFILE"."radicalSymbol" AS "symbol",
"names"
FROM "${KANJIDICTableNames.radical}"
JOIN "XREF__KANJIDIC_Radical__RADKFILE" USING ("radicalId")
LEFT JOIN (
SELECT "radicalId", group_concat("name") AS "names"
FROM "${KANJIDICTableNames.radicalName}"
GROUP BY "radicalId"
) USING ("radicalId")
WHERE "${KANJIDICTableNames.radical}"."kanji" = ?
''',
[kanji],
);
late final List<Map<String, Object?>> parts; late final List<Map<String, Object?>> parts;
final parts_query = connection.query(
RADKFILETableNames.radkfile,
where: "kanji = ?",
whereArgs: [kanji],
);
late final List<Map<String, Object?>> readings; late final List<Map<String, Object?>> readings;
late final List<Map<String, Object?>> strokeMiscounts; final readings_query = connection.query(
KANJIDICTableNames.reading,
where: "kanji = ?",
whereArgs: [kanji],
);
late final List<Map<String, Object?>> stroke_miscounts;
final stroke_miscounts_query = connection.query(
KANJIDICTableNames.strokeMiscount,
where: "kanji = ?",
whereArgs: [kanji],
);
// TODO: add variant data to result // TODO: add variant data to result
// late final List<Map<String, Object?>> variants; // late final List<Map<String, Object?>> variants;
// final variants_query = connection.query(
// KANJIDICTableNames.variant,
// where: "kanji = ?",
// whereArgs: [kanji],
// );
// TODO: Search for kunyomi and onyomi usage of the characters // TODO: Search for kunyomi and onyomi usage of the characters
// from JMDict. We'll need to fuzzy aquery JMDict_KanjiElement for matches, // from JMDict. We'll need to fuzzy aquery JMDict_KanjiElement for mathces,
// filter JMdict_ReadingElement for kunyomi/onyomi, and then sort the main entry // filter JMdict_ReadingElement for kunyomi/onyomi, and then sort the main entry
// by JLPT, news frequency, etc. // by JLPT, news frequency, etc.
await _charactersQuery(connection, kanji).then((value) => characters = value); await characters_query.then((value) => characters = value);
if (characters.isEmpty) { if (characters.isEmpty) {
return null; return null;
} }
await Future.wait({ await Future.wait({
_codepointsQuery(connection, kanji).then((value) => codepoints = value), codepoints_query.then((value) => codepoints = value),
_kunyomisQuery(connection, kanji).then((value) => kunyomis = value), kunyomis_query.then((value) => kunyomis = value),
_onyomisQuery(connection, kanji).then((value) => onyomis = value), onyomis_query.then((value) => onyomis = value),
_meaningsQuery(connection, kanji).then((value) => meanings = value), meanings_query.then((value) => meanings = value),
_nanorisQuery(connection, kanji).then((value) => nanoris = value), nanoris_query.then((value) => nanoris = value),
_dictionaryReferencesQuery( dictionary_references_query.then((value) => dictionary_references = value),
connection, query_codes_query.then((value) => query_codes = value),
kanji, radicals_query.then((value) => radicals = value),
).then((value) => dictionaryReferences = value), parts_query.then((value) => parts = value),
_queryCodesQuery(connection, kanji).then((value) => queryCodes = value), readings_query.then((value) => readings = value),
_radicalsQuery(connection, kanji).then((value) => radicals = value), stroke_miscounts_query.then((value) => stroke_miscounts = value),
_partsQuery(connection, kanji).then((value) => parts = value),
_readingsQuery(connection, kanji).then((value) => readings = value),
_strokeMiscountsQuery(
connection,
kanji,
).then((value) => strokeMiscounts = value),
// variants_query.then((value) => variants = value), // variants_query.then((value) => variants = value),
}); });
@@ -203,7 +156,9 @@ Future<KanjiSearchResult?> searchKanjiWithDbConnection(
: null; : null;
final alternativeLanguageReadings = readings final alternativeLanguageReadings = readings
.groupListsBy((item) => item['type'] as String) .groupListsBy(
(item) => item['type'] as String,
)
.map( .map(
(key, value) => MapEntry( (key, value) => MapEntry(
key, key,
@@ -212,16 +167,20 @@ Future<KanjiSearchResult?> searchKanjiWithDbConnection(
); );
// TODO: Add `SKIPMisclassification` to the entries // TODO: Add `SKIPMisclassification` to the entries
final queryCodes_ = queryCodes final queryCodes = query_codes
.groupListsBy((item) => item['type'] as String) .groupListsBy(
(item) => item['type'] as String,
)
.map( .map(
(key, value) => (key, value) => MapEntry(
MapEntry(key, value.map((item) => item['code'] as String).toList()), key,
value.map((item) => item['code'] as String).toList(),
),
); );
// TODO: Add `volume` and `page` to the entries // TODO: Add `volume` and `page` to the entries
final dictionaryReferences_ = { final dictionaryReferences = {
for (final entry in dictionaryReferences) for (final entry in dictionary_references)
entry['type'] as String: entry['ref'] as String, entry['type'] as String: entry['ref'] as String,
}; };
@@ -250,33 +209,9 @@ Future<KanjiSearchResult?> searchKanjiWithDbConnection(
}, },
nanori: nanoris.map((item) => item['nanori'] as String).toList(), nanori: nanoris.map((item) => item['nanori'] as String).toList(),
alternativeLanguageReadings: alternativeLanguageReadings, alternativeLanguageReadings: alternativeLanguageReadings,
strokeMiscounts: strokeMiscounts strokeMiscounts:
.map((item) => item['strokeCount'] as int) stroke_miscounts.map((item) => item['strokeCount'] as int).toList(),
.toList(), queryCodes: queryCodes,
queryCodes: queryCodes_, dictionaryReferences: dictionaryReferences,
dictionaryReferences: dictionaryReferences_,
); );
} }
// TODO: Use fewer queries with `IN` clauses to reduce the number of queries
/// Searches for multiple kanji at once, returning a map of kanji to their search results.
Future<Map<String, KanjiSearchResult>> searchManyKanjiWithDbConnection(
DatabaseExecutor connection,
Set<String> kanji,
) async {
if (kanji.isEmpty) {
return {};
}
final results = <String, KanjiSearchResult>{};
for (final k in kanji) {
final result = await searchKanjiWithDbConnection(connection, k);
if (result != null) {
results[k] = result;
}
}
return results;
}

View File

@@ -3,16 +3,10 @@ import 'package:sqflite_common/sqlite_api.dart';
// TODO: validate that the list of radicals all are valid radicals // TODO: validate that the list of radicals all are valid radicals
/// Returns a list of radicals that are part of any kanji that contains all of the input radicals.
///
/// This can be used to limit the choices of additional radicals provided to a user,
/// so that any choice they make will still yield at least one kanji.
Future<List<String>> searchRemainingRadicalsWithDbConnection( Future<List<String>> searchRemainingRadicalsWithDbConnection(
DatabaseExecutor connection, DatabaseExecutor connection,
List<String> radicals, List<String> radicals,
) async { ) async {
final distinctRadicals = radicals.toSet();
final queryResult = await connection.rawQuery( final queryResult = await connection.rawQuery(
''' '''
SELECT DISTINCT "radical" SELECT DISTINCT "radical"
@@ -20,37 +14,39 @@ Future<List<String>> searchRemainingRadicalsWithDbConnection(
WHERE "kanji" IN ( WHERE "kanji" IN (
SELECT "kanji" SELECT "kanji"
FROM "${RADKFILETableNames.radkfile}" FROM "${RADKFILETableNames.radkfile}"
WHERE "radical" IN (${List.filled(distinctRadicals.length, '?').join(',')}) WHERE "radical" IN (${List.filled(radicals.length, '?').join(',')})
GROUP BY "kanji" GROUP BY "kanji"
HAVING COUNT(DISTINCT "radical") = ? HAVING COUNT(DISTINCT "radical") = ?
) )
''', ''',
[...distinctRadicals, distinctRadicals.length], [
...radicals,
radicals.length,
],
); );
final remainingRadicals = queryResult final remainingRadicals =
.map((row) => row['radical'] as String) queryResult.map((row) => row['radical'] as String).toList();
.toList();
return remainingRadicals; return remainingRadicals;
} }
/// Returns a list of kanji that contain all of the input radicals.
Future<List<String>> searchKanjiByRadicalsWithDbConnection( Future<List<String>> searchKanjiByRadicalsWithDbConnection(
DatabaseExecutor connection, DatabaseExecutor connection,
List<String> radicals, List<String> radicals,
) async { ) async {
final distinctRadicals = radicals.toSet();
final queryResult = await connection.rawQuery( final queryResult = await connection.rawQuery(
''' '''
SELECT "kanji" SELECT "kanji"
FROM "${RADKFILETableNames.radkfile}" FROM "${RADKFILETableNames.radkfile}"
WHERE "radical" IN (${List.filled(distinctRadicals.length, '?').join(',')}) WHERE "radical" IN (${List.filled(radicals.length, '?').join(',')})
GROUP BY "kanji" GROUP BY "kanji"
HAVING COUNT(DISTINCT "radical") = ? HAVING COUNT(DISTINCT "radical") = ?
''', ''',
[...distinctRadicals, distinctRadicals.length], [
...radicals,
radicals.length,
],
); );
final kanji = queryResult.map((row) => row['kanji'] as String).toList(); final kanji = queryResult.map((row) => row['kanji'] as String).toList();

View File

@@ -1,5 +1,6 @@
import 'package:jadb/table_names/jmdict.dart'; import 'package:jadb/table_names/jmdict.dart';
import 'package:jadb/table_names/tanos_jlpt.dart'; import 'package:jadb/table_names/tanos_jlpt.dart';
import 'package:jadb/util/sqlite_utils.dart';
import 'package:sqflite_common/sqflite.dart'; import 'package:sqflite_common/sqflite.dart';
class LinearWordQueryData { class LinearWordQueryData {
@@ -24,9 +25,6 @@ class LinearWordQueryData {
final List<Map<String, Object?>> readingElementRestrictions; final List<Map<String, Object?>> readingElementRestrictions;
final List<Map<String, Object?>> kanjiElementInfos; final List<Map<String, Object?>> kanjiElementInfos;
final LinearWordQueryData? senseAntonymData;
final LinearWordQueryData? senseSeeAlsoData;
const LinearWordQueryData({ const LinearWordQueryData({
required this.senses, required this.senses,
required this.readingElements, required this.readingElements,
@@ -48,368 +46,245 @@ class LinearWordQueryData {
required this.readingElementInfos, required this.readingElementInfos,
required this.readingElementRestrictions, required this.readingElementRestrictions,
required this.kanjiElementInfos, required this.kanjiElementInfos,
required this.senseAntonymData,
required this.senseSeeAlsoData,
}); });
} }
Future<List<Map<String, Object?>>> _sensesQuery(
DatabaseExecutor connection,
List<int> entryIds,
) => connection.query(
JMdictTableNames.sense,
where: 'entryId IN (${List.filled(entryIds.length, '?').join(',')})',
whereArgs: entryIds,
);
Future<List<Map<String, Object?>>> _readingelementsQuery(
DatabaseExecutor connection,
List<int> entryIds,
) => connection.query(
JMdictTableNames.readingElement,
where: 'entryId IN (${List.filled(entryIds.length, '?').join(',')})',
whereArgs: entryIds,
orderBy: 'orderNum',
);
Future<List<Map<String, Object?>>> _kanjielementsQuery(
DatabaseExecutor connection,
List<int> entryIds,
) => connection.query(
JMdictTableNames.kanjiElement,
where: 'entryId IN (${List.filled(entryIds.length, '?').join(',')})',
whereArgs: entryIds,
orderBy: 'orderNum',
);
Future<List<Map<String, Object?>>> _jlpttagsQuery(
DatabaseExecutor connection,
List<int> entryIds,
) => connection.query(
TanosJLPTTableNames.jlptTag,
where: 'entryId IN (${List.filled(entryIds.length, '?').join(',')})',
whereArgs: entryIds,
);
Future<List<Map<String, Object?>>> _commonentriesQuery(
DatabaseExecutor connection,
List<int> entryIds,
) => connection.query(
'JMdict_EntryCommon',
where: 'entryId IN (${List.filled(entryIds.length, '?').join(',')})',
whereArgs: entryIds,
);
// Sense queries
Future<List<Map<String, Object?>>> _senseantonymsQuery(
DatabaseExecutor connection,
List<int> senseIds,
) => connection.rawQuery(
"""
SELECT
"${JMdictTableNames.senseAntonyms}".senseId,
"${JMdictTableNames.senseAntonyms}".ambiguous,
"${JMdictTableNames.senseAntonyms}".xrefEntryId,
"JMdict_BaseAndFurigana"."base",
"JMdict_BaseAndFurigana"."furigana"
FROM "${JMdictTableNames.senseAntonyms}"
JOIN "JMdict_BaseAndFurigana"
ON "${JMdictTableNames.senseAntonyms}"."xrefEntryId" = "JMdict_BaseAndFurigana"."entryId"
WHERE
"senseId" IN (${List.filled(senseIds.length, '?').join(',')})
AND "JMdict_BaseAndFurigana"."isFirst"
ORDER BY
"${JMdictTableNames.senseAntonyms}"."senseId",
"${JMdictTableNames.senseAntonyms}"."xrefEntryId"
""",
[...senseIds],
);
Future<List<Map<String, Object?>>> senseseealsosQuery(
DatabaseExecutor connection,
List<int> senseIds,
) => connection.rawQuery(
"""
SELECT
"${JMdictTableNames.senseSeeAlso}"."senseId",
"${JMdictTableNames.senseSeeAlso}"."ambiguous",
"${JMdictTableNames.senseSeeAlso}"."xrefEntryId",
"JMdict_BaseAndFurigana"."base",
"JMdict_BaseAndFurigana"."furigana"
FROM "${JMdictTableNames.senseSeeAlso}"
JOIN "JMdict_BaseAndFurigana"
ON "${JMdictTableNames.senseSeeAlso}"."xrefEntryId" = "JMdict_BaseAndFurigana"."entryId"
WHERE
"senseId" IN (${List.filled(senseIds.length, '?').join(',')})
AND "JMdict_BaseAndFurigana"."isFirst"
ORDER BY
"${JMdictTableNames.senseSeeAlso}"."senseId",
"${JMdictTableNames.senseSeeAlso}"."xrefEntryId"
""",
[...senseIds],
);
Future<List<Map<String, Object?>>> _sensedialectsQuery(
DatabaseExecutor connection,
List<int> senseIds,
) => connection.query(
JMdictTableNames.senseDialect,
where: 'senseId IN (${List.filled(senseIds.length, '?').join(',')})',
whereArgs: senseIds,
);
Future<List<Map<String, Object?>>> _sensefieldsQuery(
DatabaseExecutor connection,
List<int> senseIds,
) => connection.query(
JMdictTableNames.senseField,
where: 'senseId IN (${List.filled(senseIds.length, '?').join(',')})',
whereArgs: senseIds,
);
Future<List<Map<String, Object?>>> _senseglossariesQuery(
DatabaseExecutor connection,
List<int> senseIds,
) => connection.query(
JMdictTableNames.senseGlossary,
where: 'senseId IN (${List.filled(senseIds.length, '?').join(',')})',
whereArgs: senseIds,
);
Future<List<Map<String, Object?>>> _senseinfosQuery(
DatabaseExecutor connection,
List<int> senseIds,
) => connection.query(
JMdictTableNames.senseInfo,
where: 'senseId IN (${List.filled(senseIds.length, '?').join(',')})',
whereArgs: senseIds,
);
Future<List<Map<String, Object?>>> _senselanguagesourcesQuery(
DatabaseExecutor connection,
List<int> senseIds,
) => connection.query(
JMdictTableNames.senseLanguageSource,
where: 'senseId IN (${List.filled(senseIds.length, '?').join(',')})',
whereArgs: senseIds,
);
Future<List<Map<String, Object?>>> _sensemiscsQuery(
DatabaseExecutor connection,
List<int> senseIds,
) => connection.query(
JMdictTableNames.senseMisc,
where: 'senseId IN (${List.filled(senseIds.length, '?').join(',')})',
whereArgs: senseIds,
);
Future<List<Map<String, Object?>>> _sensepossQuery(
DatabaseExecutor connection,
List<int> senseIds,
) => connection.query(
JMdictTableNames.sensePOS,
where: 'senseId IN (${List.filled(senseIds.length, '?').join(',')})',
whereArgs: senseIds,
);
Future<List<Map<String, Object?>>> _senserestrictedtokanjisQuery(
DatabaseExecutor connection,
List<int> senseIds,
) => connection.query(
JMdictTableNames.senseRestrictedToKanji,
where: 'senseId IN (${List.filled(senseIds.length, '?').join(',')})',
whereArgs: senseIds,
);
Future<List<Map<String, Object?>>> _senserestrictedtoreadingsQuery(
DatabaseExecutor connection,
List<int> senseIds,
) => connection.query(
JMdictTableNames.senseRestrictedToReading,
where: 'senseId IN (${List.filled(senseIds.length, '?').join(',')})',
whereArgs: senseIds,
);
Future<List<Map<String, Object?>>> _examplesentencesQuery(
DatabaseExecutor connection,
List<int> senseIds,
) => connection.query(
'JMdict_ExampleSentence',
where: 'senseId IN (${List.filled(senseIds.length, '?').join(',')})',
whereArgs: senseIds,
);
// Reading/kanji elements queries
Future<List<Map<String, Object?>>> _readingelementinfosQuery(
DatabaseExecutor connection,
List<int> readingIds,
) => connection.query(
JMdictTableNames.readingInfo,
where: '(elementId) IN (${List.filled(readingIds.length, '?').join(',')})',
whereArgs: readingIds,
);
Future<List<Map<String, Object?>>> _readingelementrestrictionsQuery(
DatabaseExecutor connection,
List<int> readingIds,
) => connection.query(
JMdictTableNames.readingRestriction,
where: '(elementId) IN (${List.filled(readingIds.length, '?').join(',')})',
whereArgs: readingIds,
);
Future<List<Map<String, Object?>>> _kanjielementinfosQuery(
DatabaseExecutor connection,
List<int> kanjiIds,
) => connection.query(
JMdictTableNames.kanjiInfo,
where: '(elementId) IN (${List.filled(kanjiIds.length, '?').join(',')})',
whereArgs: kanjiIds,
);
// Xref queries
Future<LinearWordQueryData?> _senseantonymdataQuery(
DatabaseExecutor connection,
List<int> entryIds,
) => fetchLinearWordQueryData(connection, entryIds, fetchXrefData: false);
Future<LinearWordQueryData?> _senseseealsodataQuery(
DatabaseExecutor connection,
List<int> entryIds,
) => fetchLinearWordQueryData(connection, entryIds, fetchXrefData: false);
// Full query
Future<LinearWordQueryData> fetchLinearWordQueryData( Future<LinearWordQueryData> fetchLinearWordQueryData(
DatabaseExecutor connection, DatabaseExecutor connection,
List<int> entryIds, { List<int> entryIds,
bool fetchXrefData = true, ) async {
}) async {
late final List<Map<String, Object?>> senses; late final List<Map<String, Object?>> senses;
final Future<List<Map<String, Object?>>> senses_query = connection.query(
JMdictTableNames.sense,
where: 'entryId IN (${List.filled(entryIds.length, '?').join(',')})',
whereArgs: entryIds,
);
late final List<Map<String, Object?>> readingElements; late final List<Map<String, Object?>> readingElements;
final Future<List<Map<String, Object?>>> readingElements_query =
connection.query(
JMdictTableNames.readingElement,
where: 'entryId IN (${List.filled(entryIds.length, '?').join(',')})',
whereArgs: entryIds,
orderBy: 'orderNum',
);
late final List<Map<String, Object?>> kanjiElements; late final List<Map<String, Object?>> kanjiElements;
final Future<List<Map<String, Object?>>> kanjiElements_query =
connection.query(
JMdictTableNames.kanjiElement,
where: 'entryId IN (${List.filled(entryIds.length, '?').join(',')})',
whereArgs: entryIds,
orderBy: 'orderNum',
);
late final List<Map<String, Object?>> jlptTags; late final List<Map<String, Object?>> jlptTags;
final Future<List<Map<String, Object?>>> jlptTags_query = connection.query(
TanosJLPTTableNames.jlptTag,
where: 'entryId IN (${List.filled(entryIds.length, '?').join(',')})',
whereArgs: entryIds,
);
late final List<Map<String, Object?>> commonEntries; late final List<Map<String, Object?>> commonEntries;
final Future<List<Map<String, Object?>>> commonEntries_query =
connection.query(
'JMdict_EntryCommon',
where: 'entryId IN (${List.filled(entryIds.length, '?').join(',')})',
whereArgs: entryIds,
);
await Future.wait([ await Future.wait([
_sensesQuery(connection, entryIds).then((value) => senses = value), senses_query.then((value) => senses = value),
_readingelementsQuery( readingElements_query.then((value) => readingElements = value),
connection, kanjiElements_query.then((value) => kanjiElements = value),
entryIds, jlptTags_query.then((value) => jlptTags = value),
).then((value) => readingElements = value), commonEntries_query.then((value) => commonEntries = value),
_kanjielementsQuery(
connection,
entryIds,
).then((value) => kanjiElements = value),
_jlpttagsQuery(connection, entryIds).then((value) => jlptTags = value),
_commonentriesQuery(
connection,
entryIds,
).then((value) => commonEntries = value),
]); ]);
// Sense queries
final senseIds = senses.map((sense) => sense['senseId'] as int).toList(); final senseIds = senses.map((sense) => sense['senseId'] as int).toList();
late final List<Map<String, Object?>> senseAntonyms; late final List<Map<String, Object?>> senseAntonyms;
final Future<List<Map<String, Object?>>> senseAntonyms_query =
connection.rawQuery(
"""
SELECT
"${JMdictTableNames.senseAntonyms}".senseId,
"${JMdictTableNames.senseAntonyms}".ambiguous,
"${JMdictTableNames.senseAntonyms}".xrefEntryId,
"JMdict_BaseAndFurigana"."base",
"JMdict_BaseAndFurigana"."furigana"
FROM "${JMdictTableNames.senseAntonyms}"
JOIN "JMdict_BaseAndFurigana"
ON "${JMdictTableNames.senseAntonyms}"."xrefEntryId" = "JMdict_BaseAndFurigana"."entryId"
WHERE
"senseId" IN (${List.filled(senseIds.length, '?').join(',')})
AND "JMdict_BaseAndFurigana"."isFirst"
ORDER BY
"${JMdictTableNames.senseAntonyms}"."senseId",
"${JMdictTableNames.senseAntonyms}"."xrefEntryId"
""",
[...senseIds],
);
late final List<Map<String, Object?>> senseDialects; late final List<Map<String, Object?>> senseDialects;
final Future<List<Map<String, Object?>>> senseDialects_query =
connection.query(
JMdictTableNames.senseDialect,
where: 'senseId IN (${List.filled(senseIds.length, '?').join(',')})',
whereArgs: senseIds,
);
late final List<Map<String, Object?>> senseFields; late final List<Map<String, Object?>> senseFields;
final Future<List<Map<String, Object?>>> senseFields_query = connection.query(
JMdictTableNames.senseField,
where: 'senseId IN (${List.filled(senseIds.length, '?').join(',')})',
whereArgs: senseIds,
);
late final List<Map<String, Object?>> senseGlossaries; late final List<Map<String, Object?>> senseGlossaries;
final Future<List<Map<String, Object?>>> senseGlossaries_query =
connection.query(
JMdictTableNames.senseGlossary,
where: 'senseId IN (${List.filled(senseIds.length, '?').join(',')})',
whereArgs: senseIds,
);
late final List<Map<String, Object?>> senseInfos; late final List<Map<String, Object?>> senseInfos;
final Future<List<Map<String, Object?>>> senseInfos_query = connection.query(
JMdictTableNames.senseInfo,
where: 'senseId IN (${List.filled(senseIds.length, '?').join(',')})',
whereArgs: senseIds,
);
late final List<Map<String, Object?>> senseLanguageSources; late final List<Map<String, Object?>> senseLanguageSources;
final Future<List<Map<String, Object?>>> senseLanguageSources_query =
connection.query(
JMdictTableNames.senseLanguageSource,
where: 'senseId IN (${List.filled(senseIds.length, '?').join(',')})',
whereArgs: senseIds,
);
late final List<Map<String, Object?>> senseMiscs; late final List<Map<String, Object?>> senseMiscs;
final Future<List<Map<String, Object?>>> senseMiscs_query = connection.query(
JMdictTableNames.senseMisc,
where: 'senseId IN (${List.filled(senseIds.length, '?').join(',')})',
whereArgs: senseIds,
);
late final List<Map<String, Object?>> sensePOSs; late final List<Map<String, Object?>> sensePOSs;
final Future<List<Map<String, Object?>>> sensePOSs_query = connection.query(
JMdictTableNames.sensePOS,
where: 'senseId IN (${List.filled(senseIds.length, '?').join(',')})',
whereArgs: senseIds,
);
late final List<Map<String, Object?>> senseRestrictedToKanjis; late final List<Map<String, Object?>> senseRestrictedToKanjis;
final Future<List<Map<String, Object?>>> senseRestrictedToKanjis_query =
connection.query(
JMdictTableNames.senseRestrictedToKanji,
where: 'senseId IN (${List.filled(senseIds.length, '?').join(',')})',
whereArgs: senseIds,
);
late final List<Map<String, Object?>> senseRestrictedToReadings; late final List<Map<String, Object?>> senseRestrictedToReadings;
final Future<List<Map<String, Object?>>> senseRestrictedToReadings_query =
connection.query(
JMdictTableNames.senseRestrictedToReading,
where: 'senseId IN (${List.filled(senseIds.length, '?').join(',')})',
whereArgs: senseIds,
);
late final List<Map<String, Object?>> senseSeeAlsos; late final List<Map<String, Object?>> senseSeeAlsos;
final Future<List<Map<String, Object?>>> senseSeeAlsos_query =
connection.rawQuery(
"""
SELECT
"${JMdictTableNames.senseSeeAlso}"."senseId",
"${JMdictTableNames.senseSeeAlso}"."ambiguous",
"${JMdictTableNames.senseSeeAlso}"."xrefEntryId",
"JMdict_BaseAndFurigana"."base",
"JMdict_BaseAndFurigana"."furigana"
FROM "${JMdictTableNames.senseSeeAlso}"
JOIN "JMdict_BaseAndFurigana"
ON "${JMdictTableNames.senseSeeAlso}"."xrefEntryId" = "JMdict_BaseAndFurigana"."entryId"
WHERE
"senseId" IN (${List.filled(senseIds.length, '?').join(',')})
AND "JMdict_BaseAndFurigana"."isFirst"
ORDER BY
"${JMdictTableNames.senseSeeAlso}"."senseId",
"${JMdictTableNames.senseSeeAlso}"."xrefEntryId"
""",
[...senseIds],
);
late final List<Map<String, Object?>> exampleSentences; late final List<Map<String, Object?>> exampleSentences;
final Future<List<Map<String, Object?>>> exampleSentences_query =
connection.query(
'JMdict_ExampleSentence',
where: 'senseId IN (${List.filled(senseIds.length, '?').join(',')})',
whereArgs: senseIds,
);
// Reading queries
final readingIds = readingElements final readingIds = readingElements
.map((element) => element['elementId'] as int) .map((element) => (
.toList(); element['entryId'] as int,
escapeStringValue(element['reading'] as String)
final kanjiIds = kanjiElements ))
.map((element) => element['elementId'] as int)
.toList(); .toList();
late final List<Map<String, Object?>> readingElementInfos; late final List<Map<String, Object?>> readingElementInfos;
final Future<List<Map<String, Object?>>> readingElementInfos_query =
connection.query(
JMdictTableNames.readingInfo,
where: '(entryId, reading) IN (${readingIds.join(',')})',
);
late final List<Map<String, Object?>> readingElementRestrictions; late final List<Map<String, Object?>> readingElementRestrictions;
final Future<List<Map<String, Object?>>> readingElementRestrictions_query =
connection.query(
JMdictTableNames.readingRestriction,
where: '(entryId, reading) IN (${readingIds.join(',')})',
);
// Kanji queries
final kanjiIds = kanjiElements
.map((element) => (
element['entryId'] as int,
escapeStringValue(element['reading'] as String)
))
.toList();
late final List<Map<String, Object?>> kanjiElementInfos; late final List<Map<String, Object?>> kanjiElementInfos;
final Future<List<Map<String, Object?>>> kanjiElementInfos_query =
// Xref data queries connection.query(
await Future.wait([ JMdictTableNames.kanjiInfo,
_senseantonymsQuery( where: '(entryId, reading) IN (${kanjiIds.join(',')})',
connection, );
senseIds,
).then((value) => senseAntonyms = value),
senseseealsosQuery(
connection,
senseIds,
).then((value) => senseSeeAlsos = value),
]);
LinearWordQueryData? senseAntonymData;
LinearWordQueryData? senseSeeAlsoData;
await Future.wait([ await Future.wait([
_sensedialectsQuery( senseAntonyms_query.then((value) => senseAntonyms = value),
connection, senseDialects_query.then((value) => senseDialects = value),
senseIds, senseFields_query.then((value) => senseFields = value),
).then((value) => senseDialects = value), senseGlossaries_query.then((value) => senseGlossaries = value),
_sensefieldsQuery( senseInfos_query.then((value) => senseInfos = value),
connection, senseLanguageSources_query.then((value) => senseLanguageSources = value),
senseIds, senseMiscs_query.then((value) => senseMiscs = value),
).then((value) => senseFields = value), sensePOSs_query.then((value) => sensePOSs = value),
_senseglossariesQuery( senseRestrictedToKanjis_query
connection, .then((value) => senseRestrictedToKanjis = value),
senseIds, senseRestrictedToReadings_query
).then((value) => senseGlossaries = value), .then((value) => senseRestrictedToReadings = value),
_senseinfosQuery(connection, senseIds).then((value) => senseInfos = value), senseSeeAlsos_query.then((value) => senseSeeAlsos = value),
_senselanguagesourcesQuery( exampleSentences_query.then((value) => exampleSentences = value),
connection, readingElementInfos_query.then((value) => readingElementInfos = value),
senseIds, readingElementRestrictions_query
).then((value) => senseLanguageSources = value), .then((value) => readingElementRestrictions = value),
_sensemiscsQuery(connection, senseIds).then((value) => senseMiscs = value), kanjiElementInfos_query.then((value) => kanjiElementInfos = value),
_sensepossQuery(connection, senseIds).then((value) => sensePOSs = value),
_senserestrictedtokanjisQuery(
connection,
senseIds,
).then((value) => senseRestrictedToKanjis = value),
_senserestrictedtoreadingsQuery(
connection,
senseIds,
).then((value) => senseRestrictedToReadings = value),
_examplesentencesQuery(
connection,
senseIds,
).then((value) => exampleSentences = value),
_readingelementinfosQuery(
connection,
readingIds,
).then((value) => readingElementInfos = value),
_readingelementrestrictionsQuery(
connection,
readingIds,
).then((value) => readingElementRestrictions = value),
_kanjielementinfosQuery(
connection,
kanjiIds,
).then((value) => kanjiElementInfos = value),
if (fetchXrefData)
_senseantonymdataQuery(
connection,
senseAntonyms.map((antonym) => antonym['xrefEntryId'] as int).toList(),
).then((value) => senseAntonymData = value),
if (fetchXrefData)
_senseseealsodataQuery(
connection,
senseSeeAlsos.map((seeAlso) => seeAlso['xrefEntryId'] as int).toList(),
).then((value) => senseSeeAlsoData = value),
]); ]);
return LinearWordQueryData( return LinearWordQueryData(
@@ -433,7 +308,5 @@ Future<LinearWordQueryData> fetchLinearWordQueryData(
readingElementInfos: readingElementInfos, readingElementInfos: readingElementInfos,
readingElementRestrictions: readingElementRestrictions, readingElementRestrictions: readingElementRestrictions,
kanjiElementInfos: kanjiElementInfos, kanjiElementInfos: kanjiElementInfos,
senseAntonymData: senseAntonymData,
senseSeeAlsoData: senseSeeAlsoData,
); );
} }

View File

@@ -1,5 +1,5 @@
import 'package:jadb/search/word_search/word_search.dart';
import 'package:jadb/table_names/jmdict.dart'; import 'package:jadb/table_names/jmdict.dart';
import 'package:jadb/search/word_search/word_search.dart';
import 'package:jadb/util/text_filtering.dart'; import 'package:jadb/util/text_filtering.dart';
import 'package:sqflite_common/sqlite_api.dart'; import 'package:sqflite_common/sqlite_api.dart';
@@ -15,15 +15,15 @@ SearchMode _determineSearchMode(String word) {
final bool containsAscii = RegExp(r'[A-Za-z]').hasMatch(word); final bool containsAscii = RegExp(r'[A-Za-z]').hasMatch(word);
if (containsKanji && containsAscii) { if (containsKanji && containsAscii) {
return SearchMode.mixedKanji; return SearchMode.MixedKanji;
} else if (containsKanji) { } else if (containsKanji) {
return SearchMode.kanji; return SearchMode.Kanji;
} else if (containsAscii) { } else if (containsAscii) {
return SearchMode.english; return SearchMode.English;
} else if (word.contains(hiraganaRegex) || word.contains(katakanaRegex)) { } else if (word.contains(hiraganaRegex) || word.contains(katakanaRegex)) {
return SearchMode.kana; return SearchMode.Kana;
} else { } else {
return SearchMode.mixedKana; return SearchMode.MixedKana;
} }
} }
@@ -37,105 +37,91 @@ String _filterFTSSensitiveCharacters(String word) {
.replaceAll('(', '') .replaceAll('(', '')
.replaceAll(')', '') .replaceAll(')', '')
.replaceAll('^', '') .replaceAll('^', '')
.replaceAll('"', ''); .replaceAll('\"', '');
} }
(String, List<Object?>) _kanjiReadingTemplate( (String, List<Object?>) _kanjiReadingTemplate(
String tableName, String tableName,
String word, { String word, {
int? pageSize, int pageSize = 10,
int? offset,
bool countOnly = false, bool countOnly = false,
}) { }) =>
assert( (
tableName == JMdictTableNames.kanjiElement || '''
tableName == JMdictTableNames.readingElement,
);
assert(!countOnly || pageSize == null);
assert(!countOnly || offset == null);
assert(pageSize == null || pageSize > 0);
assert(offset == null || offset >= 0);
assert(
offset == null || pageSize != null,
'Offset should only be used with pageSize set',
);
return (
'''
WITH WITH
fts_results AS ( fts_results AS (
SELECT DISTINCT SELECT DISTINCT
"$tableName"."entryId", "${tableName}FTS"."entryId",
100 100
+ (("${tableName}FTS"."reading" = ?) * 10000) + (("${tableName}FTS"."reading" = ?) * 50)
+ "JMdict_EntryScore"."score" + "JMdict_EntryScore"."score"
AS "score" AS "score"
FROM "${tableName}FTS" FROM "${tableName}FTS"
JOIN "$tableName" USING ("elementId") JOIN "${tableName}" USING ("entryId", "reading")
JOIN "JMdict_EntryScore" USING ("elementId") JOIN "JMdict_EntryScore" USING ("entryId", "reading")
WHERE "${tableName}FTS"."reading" MATCH ? || '*' WHERE "${tableName}FTS"."reading" MATCH ? || '*'
AND "JMdict_EntryScore"."type" = '${tableName == JMdictTableNames.kanjiElement ? 'k' : 'r'}' AND "JMdict_EntryScore"."type" = '${tableName == JMdictTableNames.kanjiElement ? 'kanji' : 'reading'}'
ORDER BY
"JMdict_EntryScore"."score" DESC
${!countOnly ? 'LIMIT ?' : ''}
), ),
non_fts_results AS ( non_fts_results AS (
SELECT DISTINCT SELECT DISTINCT
"$tableName"."entryId", "${tableName}"."entryId",
50 50
+ "JMdict_EntryScore"."score" + "JMdict_EntryScore"."score"
AS "score" AS "score"
FROM "$tableName" FROM "${tableName}"
JOIN "JMdict_EntryScore" USING ("elementId") JOIN "JMdict_EntryScore" USING ("entryId", "reading")
WHERE "reading" LIKE '%' || ? || '%' WHERE "reading" LIKE '%' || ? || '%'
AND "$tableName"."entryId" NOT IN (SELECT "entryId" FROM "fts_results") AND "entryId" NOT IN (SELECT "entryId" FROM "fts_results")
AND "JMdict_EntryScore"."type" = '${tableName == JMdictTableNames.kanjiElement ? 'k' : 'r'}' AND "JMdict_EntryScore"."type" = '${tableName == JMdictTableNames.kanjiElement ? 'kanji' : 'reading'}'
ORDER BY
"JMdict_EntryScore"."score" DESC,
"${tableName}"."entryId" ASC
${!countOnly ? 'LIMIT ?' : ''}
) )
SELECT ${countOnly ? 'COUNT(DISTINCT "entryId") AS count' : '"entryId", MAX("score") AS "score"'} ${countOnly ? 'SELECT COUNT("entryId") AS count' : 'SELECT "entryId", "score"'}
FROM ( FROM (
SELECT * FROM "fts_results" SELECT * FROM fts_results
UNION UNION ALL
SELECT * FROM "non_fts_results" SELECT * FROM non_fts_results
) )
${!countOnly ? 'GROUP BY "entryId"' : ''}
${!countOnly ? 'ORDER BY "score" DESC, "entryId" ASC' : ''}
${pageSize != null ? 'LIMIT ?' : ''}
${offset != null ? 'OFFSET ?' : ''}
''' '''
.trim(), .trim(),
[ [
_filterFTSSensitiveCharacters(word), _filterFTSSensitiveCharacters(word),
_filterFTSSensitiveCharacters(word), _filterFTSSensitiveCharacters(word),
_filterFTSSensitiveCharacters(word), if (!countOnly) pageSize,
?pageSize, _filterFTSSensitiveCharacters(word),
?offset, if (!countOnly) pageSize,
], ]
); );
}
Future<List<ScoredEntryId>> _queryKanji( Future<List<ScoredEntryId>> _queryKanji(
DatabaseExecutor connection, DatabaseExecutor connection,
String word, String word,
int? pageSize, int pageSize,
int? offset, int? offset,
) { ) {
final (query, args) = _kanjiReadingTemplate( final (query, args) = _kanjiReadingTemplate(
JMdictTableNames.kanjiElement, JMdictTableNames.kanjiElement,
word, word,
pageSize: pageSize, pageSize: pageSize,
offset: offset,
); );
return connection return connection.rawQuery(query, args).then((result) => result
.rawQuery(query, args) .map((row) => ScoredEntryId(
.then( row['entryId'] as int,
(result) => result row['score'] as int,
.map( ))
(row) => .toList());
ScoredEntryId(row['entryId'] as int, row['score'] as int),
)
.toList(),
);
} }
Future<int> _queryKanjiCount(DatabaseExecutor connection, String word) { Future<int> _queryKanjiCount(
DatabaseExecutor connection,
String word,
) {
final (query, args) = _kanjiReadingTemplate( final (query, args) = _kanjiReadingTemplate(
JMdictTableNames.kanjiElement, JMdictTableNames.kanjiElement,
word, word,
@@ -143,34 +129,32 @@ Future<int> _queryKanjiCount(DatabaseExecutor connection, String word) {
); );
return connection return connection
.rawQuery(query, args) .rawQuery(query, args)
.then((result) => result.firstOrNull?['count'] as int? ?? 0); .then((result) => result.first['count'] as int);
} }
Future<List<ScoredEntryId>> _queryKana( Future<List<ScoredEntryId>> _queryKana(
DatabaseExecutor connection, DatabaseExecutor connection,
String word, String word,
int? pageSize, int pageSize,
int? offset, int? offset,
) { ) {
final (query, args) = _kanjiReadingTemplate( final (query, args) = _kanjiReadingTemplate(
JMdictTableNames.readingElement, JMdictTableNames.readingElement,
word, word,
pageSize: pageSize, pageSize: pageSize,
offset: offset,
); );
return connection return connection.rawQuery(query, args).then((result) => result
.rawQuery(query, args) .map((row) => ScoredEntryId(
.then( row['entryId'] as int,
(result) => result row['score'] as int,
.map( ))
(row) => .toList());
ScoredEntryId(row['entryId'] as int, row['score'] as int),
)
.toList(),
);
} }
Future<int> _queryKanaCount(DatabaseExecutor connection, String word) { Future<int> _queryKanaCount(
DatabaseExecutor connection,
String word,
) {
final (query, args) = _kanjiReadingTemplate( final (query, args) = _kanjiReadingTemplate(
JMdictTableNames.readingElement, JMdictTableNames.readingElement,
word, word,
@@ -178,22 +162,15 @@ Future<int> _queryKanaCount(DatabaseExecutor connection, String word) {
); );
return connection return connection
.rawQuery(query, args) .rawQuery(query, args)
.then((result) => result.firstOrNull?['count'] as int? ?? 0); .then((result) => result.first['count'] as int);
} }
Future<List<ScoredEntryId>> _queryEnglish( Future<List<ScoredEntryId>> _queryEnglish(
DatabaseExecutor connection, DatabaseExecutor connection,
String word, String word,
int? pageSize, int pageSize,
int? offset, int? offset,
) async { ) async {
assert(pageSize == null || pageSize > 0);
assert(offset == null || offset >= 0);
assert(
offset == null || pageSize != null,
'Offset should only be used with pageSize set',
);
final result = await connection.rawQuery( final result = await connection.rawQuery(
''' '''
SELECT SELECT
@@ -215,25 +192,41 @@ Future<List<ScoredEntryId>> _queryEnglish(
OFFSET ? OFFSET ?
''' '''
.trim(), .trim(),
[word, word, word, '%${word.replaceAll('%', '')}%', pageSize, offset], [
word,
word,
word,
'%${word.replaceAll('%', '')}%',
pageSize,
offset,
],
); );
return result return result
.map((row) => ScoredEntryId(row['entryId'] as int, row['score'] as int)) .map((row) => ScoredEntryId(
row['entryId'] as int,
row['score'] as int,
))
.toList(); .toList();
} }
Future<int> _queryEnglishCount(DatabaseExecutor connection, String word) async { Future<int> _queryEnglishCount(
DatabaseExecutor connection,
String word,
) async {
final result = await connection.rawQuery( final result = await connection.rawQuery(
''' '''
SELECT
COUNT(DISTINCT "${JMdictTableNames.sense}"."entryId") AS "count" SELECT
FROM "${JMdictTableNames.senseGlossary}" COUNT(DISTINCT "${JMdictTableNames.sense}"."entryId") AS "count"
JOIN "${JMdictTableNames.sense}" USING ("senseId") FROM "${JMdictTableNames.senseGlossary}"
WHERE "${JMdictTableNames.senseGlossary}"."phrase" LIKE ? JOIN "${JMdictTableNames.sense}" USING ("senseId")
''' WHERE "${JMdictTableNames.senseGlossary}"."phrase" LIKE ?
'''
.trim(), .trim(),
['%$word%'], [
'%$word%',
],
); );
return result.first['count'] as int; return result.first['count'] as int;
@@ -243,34 +236,55 @@ Future<List<ScoredEntryId>> fetchEntryIds(
DatabaseExecutor connection, DatabaseExecutor connection,
String word, String word,
SearchMode searchMode, SearchMode searchMode,
int? pageSize, int pageSize,
int? offset, int? offset,
) async { ) async {
if (searchMode == SearchMode.auto) { if (searchMode == SearchMode.Auto) {
searchMode = _determineSearchMode(word); searchMode = _determineSearchMode(word);
} }
assert(word.isNotEmpty, 'Word should not be empty when fetching entry IDs'); assert(
word.isNotEmpty,
'Word should not be empty when fetching entry IDs',
);
late final List<ScoredEntryId> entryIds; late final List<ScoredEntryId> entryIds;
switch (searchMode) { switch (searchMode) {
case SearchMode.kanji: case SearchMode.Kanji:
entryIds = await _queryKanji(connection, word, pageSize, offset); entryIds = await _queryKanji(
connection,
word,
pageSize,
offset,
);
break; break;
case SearchMode.kana: case SearchMode.Kana:
entryIds = await _queryKana(connection, word, pageSize, offset); entryIds = await _queryKana(
connection,
word,
pageSize,
offset,
);
break; break;
case SearchMode.english: case SearchMode.English:
entryIds = await _queryEnglish(connection, word, pageSize, offset); entryIds = await _queryEnglish(
connection,
word,
pageSize,
offset,
);
break; break;
case SearchMode.mixedKana: case SearchMode.MixedKana:
case SearchMode.mixedKanji: case SearchMode.MixedKanji:
default: default:
throw UnimplementedError('Search mode $searchMode is not implemented'); throw UnimplementedError(
'Search mode $searchMode is not implemented',
);
} }
;
return entryIds; return entryIds;
} }
@@ -280,31 +294,45 @@ Future<int?> fetchEntryIdCount(
String word, String word,
SearchMode searchMode, SearchMode searchMode,
) async { ) async {
if (searchMode == SearchMode.auto) { if (searchMode == SearchMode.Auto) {
searchMode = _determineSearchMode(word); searchMode = _determineSearchMode(word);
} }
assert(word.isNotEmpty, 'Word should not be empty when fetching entry IDs'); assert(
word.isNotEmpty,
'Word should not be empty when fetching entry IDs',
);
late final int? entryIdCount; late final int? entryIdCount;
switch (searchMode) { switch (searchMode) {
case SearchMode.kanji: case SearchMode.Kanji:
entryIdCount = await _queryKanjiCount(connection, word); entryIdCount = await _queryKanjiCount(
connection,
word,
);
break; break;
case SearchMode.kana: case SearchMode.Kana:
entryIdCount = await _queryKanaCount(connection, word); entryIdCount = await _queryKanaCount(
connection,
word,
);
break; break;
case SearchMode.english: case SearchMode.English:
entryIdCount = await _queryEnglishCount(connection, word); entryIdCount = await _queryEnglishCount(
connection,
word,
);
break; break;
case SearchMode.mixedKana: case SearchMode.MixedKana:
case SearchMode.mixedKanji: case SearchMode.MixedKanji:
default: default:
throw UnimplementedError('Search mode $searchMode is not implemented'); throw UnimplementedError(
'Search mode $searchMode is not implemented',
);
} }
return entryIdCount; return entryIdCount;

View File

@@ -12,37 +12,50 @@ import 'package:jadb/models/word_search/word_search_sense.dart';
import 'package:jadb/models/word_search/word_search_sense_language_source.dart'; import 'package:jadb/models/word_search/word_search_sense_language_source.dart';
import 'package:jadb/models/word_search/word_search_sources.dart'; import 'package:jadb/models/word_search/word_search_sources.dart';
import 'package:jadb/models/word_search/word_search_xref_entry.dart'; import 'package:jadb/models/word_search/word_search_xref_entry.dart';
import 'package:jadb/search/word_search/data_query.dart';
import 'package:jadb/search/word_search/entry_id_query.dart'; import 'package:jadb/search/word_search/entry_id_query.dart';
List<WordSearchResult> regroupWordSearchResults({ List<WordSearchResult> regroupWordSearchResults({
required List<ScoredEntryId> entryIds, required List<ScoredEntryId> entryIds,
required LinearWordQueryData linearWordQueryData, required List<Map<String, Object?>> readingElements,
required List<Map<String, Object?>> kanjiElements,
required List<Map<String, Object?>> jlptTags,
required List<Map<String, Object?>> commonEntries,
required List<Map<String, Object?>> senses,
required List<Map<String, Object?>> senseAntonyms,
required List<Map<String, Object?>> senseDialects,
required List<Map<String, Object?>> senseFields,
required List<Map<String, Object?>> senseGlossaries,
required List<Map<String, Object?>> senseInfos,
required List<Map<String, Object?>> senseLanguageSources,
required List<Map<String, Object?>> senseMiscs,
required List<Map<String, Object?>> sensePOSs,
required List<Map<String, Object?>> senseRestrictedToKanjis,
required List<Map<String, Object?>> senseRestrictedToReadings,
required List<Map<String, Object?>> senseSeeAlsos,
required List<Map<String, Object?>> exampleSentences,
required List<Map<String, Object?>> readingElementInfos,
required List<Map<String, Object?>> readingElementRestrictions,
required List<Map<String, Object?>> kanjiElementInfos,
}) { }) {
final List<WordSearchResult> results = []; final List<WordSearchResult> results = [];
final commonEntryIds = linearWordQueryData.commonEntries final commonEntryIds =
.map((entry) => entry['entryId'] as int) commonEntries.map((entry) => entry['entryId'] as int).toSet();
.toSet();
for (final scoredEntryId in entryIds) { for (final scoredEntryId in entryIds) {
final List<Map<String, Object?>> entryReadingElements = linearWordQueryData final List<Map<String, Object?>> entryReadingElements = readingElements
.readingElements
.where((element) => element['entryId'] == scoredEntryId.entryId) .where((element) => element['entryId'] == scoredEntryId.entryId)
.toList(); .toList();
final List<Map<String, Object?>> entryKanjiElements = linearWordQueryData final List<Map<String, Object?>> entryKanjiElements = kanjiElements
.kanjiElements
.where((element) => element['entryId'] == scoredEntryId.entryId) .where((element) => element['entryId'] == scoredEntryId.entryId)
.toList(); .toList();
final List<Map<String, Object?>> entryJlptTags = linearWordQueryData final List<Map<String, Object?>> entryJlptTags = jlptTags
.jlptTags
.where((element) => element['entryId'] == scoredEntryId.entryId) .where((element) => element['entryId'] == scoredEntryId.entryId)
.toList(); .toList();
final jlptLevel = final jlptLevel = entryJlptTags
entryJlptTags
.map((e) => JlptLevel.fromString(e['jlptLevel'] as String?)) .map((e) => JlptLevel.fromString(e['jlptLevel'] as String?))
.sorted((a, b) => b.compareTo(a)) .sorted((a, b) => b.compareTo(a))
.firstOrNull ?? .firstOrNull ??
@@ -50,36 +63,33 @@ List<WordSearchResult> regroupWordSearchResults({
final isCommon = commonEntryIds.contains(scoredEntryId.entryId); final isCommon = commonEntryIds.contains(scoredEntryId.entryId);
final List<Map<String, Object?>> entrySenses = linearWordQueryData.senses final List<Map<String, Object?>> entrySenses = senses
.where((element) => element['entryId'] == scoredEntryId.entryId) .where((element) => element['entryId'] == scoredEntryId.entryId)
.toList(); .toList();
final GroupedWordResult entryReadingElementsGrouped = _regroupWords( final GroupedWordResult entryReadingElementsGrouped = _regroup_words(
entryId: scoredEntryId.entryId, entryId: scoredEntryId.entryId,
readingElements: entryReadingElements, readingElements: entryReadingElements,
kanjiElements: entryKanjiElements, kanjiElements: entryKanjiElements,
readingElementInfos: linearWordQueryData.readingElementInfos, readingElementInfos: readingElementInfos,
readingElementRestrictions: readingElementRestrictions: readingElementRestrictions,
linearWordQueryData.readingElementRestrictions, kanjiElementInfos: kanjiElementInfos,
kanjiElementInfos: linearWordQueryData.kanjiElementInfos,
); );
final List<WordSearchSense> entrySensesGrouped = _regroupSenses( final List<WordSearchSense> entrySensesGrouped = _regroup_senses(
senses: entrySenses, senses: entrySenses,
senseAntonyms: linearWordQueryData.senseAntonyms, senseAntonyms: senseAntonyms,
senseDialects: linearWordQueryData.senseDialects, senseDialects: senseDialects,
senseFields: linearWordQueryData.senseFields, senseFields: senseFields,
senseGlossaries: linearWordQueryData.senseGlossaries, senseGlossaries: senseGlossaries,
senseInfos: linearWordQueryData.senseInfos, senseInfos: senseInfos,
senseLanguageSources: linearWordQueryData.senseLanguageSources, senseLanguageSources: senseLanguageSources,
senseMiscs: linearWordQueryData.senseMiscs, senseMiscs: senseMiscs,
sensePOSs: linearWordQueryData.sensePOSs, sensePOSs: sensePOSs,
senseRestrictedToKanjis: linearWordQueryData.senseRestrictedToKanjis, senseRestrictedToKanjis: senseRestrictedToKanjis,
senseRestrictedToReadings: linearWordQueryData.senseRestrictedToReadings, senseRestrictedToReadings: senseRestrictedToReadings,
senseSeeAlsos: linearWordQueryData.senseSeeAlsos, senseSeeAlsos: senseSeeAlsos,
exampleSentences: linearWordQueryData.exampleSentences, exampleSentences: exampleSentences,
senseSeeAlsosXrefData: linearWordQueryData.senseSeeAlsoData,
senseAntonymsXrefData: linearWordQueryData.senseAntonymData,
); );
results.add( results.add(
@@ -92,7 +102,10 @@ List<WordSearchResult> regroupWordSearchResults({
readingInfo: entryReadingElementsGrouped.readingInfos, readingInfo: entryReadingElementsGrouped.readingInfos,
senses: entrySensesGrouped, senses: entrySensesGrouped,
jlptLevel: jlptLevel, jlptLevel: jlptLevel,
sources: const WordSearchSources(jmdict: true, jmnedict: false), sources: const WordSearchSources(
jmdict: true,
jmnedict: false,
),
), ),
); );
} }
@@ -112,7 +125,7 @@ class GroupedWordResult {
}); });
} }
GroupedWordResult _regroupWords({ GroupedWordResult _regroup_words({
required int entryId, required int entryId,
required List<Map<String, Object?>> kanjiElements, required List<Map<String, Object?>> kanjiElements,
required List<Map<String, Object?>> kanjiElementInfos, required List<Map<String, Object?>> kanjiElementInfos,
@@ -122,9 +135,8 @@ GroupedWordResult _regroupWords({
}) { }) {
final List<WordSearchRuby> rubys = []; final List<WordSearchRuby> rubys = [];
final kanjiElements_ = kanjiElements final kanjiElements_ =
.where((element) => element['entryId'] == entryId) kanjiElements.where((element) => element['entryId'] == entryId).toList();
.toList();
final readingElements_ = readingElements final readingElements_ = readingElements
.where((element) => element['entryId'] == entryId) .where((element) => element['entryId'] == entryId)
@@ -136,7 +148,9 @@ GroupedWordResult _regroupWords({
for (final readingElement in readingElements_) { for (final readingElement in readingElements_) {
if (readingElement['doesNotMatchKanji'] == 1 || kanjiElements_.isEmpty) { if (readingElement['doesNotMatchKanji'] == 1 || kanjiElements_.isEmpty) {
final ruby = WordSearchRuby(base: readingElement['reading'] as String); final ruby = WordSearchRuby(
base: readingElement['reading'] as String,
);
rubys.add(ruby); rubys.add(ruby);
continue; continue;
@@ -155,47 +169,34 @@ GroupedWordResult _regroupWords({
continue; continue;
} }
final ruby = WordSearchRuby(base: kanji, furigana: reading); final ruby = WordSearchRuby(
base: kanji,
furigana: reading,
);
rubys.add(ruby); rubys.add(ruby);
} }
} }
assert(rubys.isNotEmpty, 'No readings found for entryId: $entryId'); assert(
rubys.isNotEmpty,
final Map<int, String> readingElementIdsToReading = { 'No readings found for entryId: $entryId',
for (final element in readingElements_) );
element['elementId'] as int: element['reading'] as String,
};
final Map<int, String> kanjiElementIdsToReading = {
for (final element in kanjiElements_)
element['elementId'] as int: element['reading'] as String,
};
final readingElementInfos_ = readingElementInfos
.where((element) => element['entryId'] == entryId)
.toList();
final kanjiElementInfos_ = kanjiElementInfos
.where((element) => element['entryId'] == entryId)
.toList();
return GroupedWordResult( return GroupedWordResult(
rubys: rubys, rubys: rubys,
readingInfos: { readingInfos: {
for (final rei in readingElementInfos_) for (final rei in readingElementInfos)
readingElementIdsToReading[rei['elementId'] as int]!: rei['reading'] as String:
JMdictReadingInfo.fromId(rei['info'] as String), JMdictReadingInfo.fromId(rei['info'] as String),
}, },
kanjiInfos: { kanjiInfos: {
for (final kei in kanjiElementInfos_) for (final kei in kanjiElementInfos)
kanjiElementIdsToReading[kei['elementId'] as int]!: kei['reading'] as String: JMdictKanjiInfo.fromId(kei['info'] as String),
JMdictKanjiInfo.fromId(kei['info'] as String),
}, },
); );
} }
List<WordSearchSense> _regroupSenses({ List<WordSearchSense> _regroup_senses({
required List<Map<String, Object?>> senses, required List<Map<String, Object?>> senses,
required List<Map<String, Object?>> senseAntonyms, required List<Map<String, Object?>> senseAntonyms,
required List<Map<String, Object?>> senseDialects, required List<Map<String, Object?>> senseDialects,
@@ -209,41 +210,29 @@ List<WordSearchSense> _regroupSenses({
required List<Map<String, Object?>> senseRestrictedToReadings, required List<Map<String, Object?>> senseRestrictedToReadings,
required List<Map<String, Object?>> senseSeeAlsos, required List<Map<String, Object?>> senseSeeAlsos,
required List<Map<String, Object?>> exampleSentences, required List<Map<String, Object?>> exampleSentences,
required LinearWordQueryData? senseSeeAlsosXrefData,
required LinearWordQueryData? senseAntonymsXrefData,
}) { }) {
final groupedSenseAntonyms = senseAntonyms.groupListsBy( final groupedSenseAntonyms =
(element) => element['senseId'] as int, senseAntonyms.groupListsBy((element) => element['senseId'] as int);
); final groupedSenseDialects =
final groupedSenseDialects = senseDialects.groupListsBy( senseDialects.groupListsBy((element) => element['senseId'] as int);
(element) => element['senseId'] as int, final groupedSenseFields =
); senseFields.groupListsBy((element) => element['senseId'] as int);
final groupedSenseFields = senseFields.groupListsBy( final groupedSenseGlossaries =
(element) => element['senseId'] as int, senseGlossaries.groupListsBy((element) => element['senseId'] as int);
); final groupedSenseInfos =
final groupedSenseGlossaries = senseGlossaries.groupListsBy( senseInfos.groupListsBy((element) => element['senseId'] as int);
(element) => element['senseId'] as int, final groupedSenseLanguageSources =
); senseLanguageSources.groupListsBy((element) => element['senseId'] as int);
final groupedSenseInfos = senseInfos.groupListsBy( final groupedSenseMiscs =
(element) => element['senseId'] as int, senseMiscs.groupListsBy((element) => element['senseId'] as int);
); final groupedSensePOSs =
final groupedSenseLanguageSources = senseLanguageSources.groupListsBy( sensePOSs.groupListsBy((element) => element['senseId'] as int);
(element) => element['senseId'] as int, final groupedSenseRestrictedToKanjis = senseRestrictedToKanjis
); .groupListsBy((element) => element['senseId'] as int);
final groupedSenseMiscs = senseMiscs.groupListsBy(
(element) => element['senseId'] as int,
);
final groupedSensePOSs = sensePOSs.groupListsBy(
(element) => element['senseId'] as int,
);
final groupedSenseRestrictedToKanjis = senseRestrictedToKanjis.groupListsBy(
(element) => element['senseId'] as int,
);
final groupedSenseRestrictedToReadings = senseRestrictedToReadings final groupedSenseRestrictedToReadings = senseRestrictedToReadings
.groupListsBy((element) => element['senseId'] as int); .groupListsBy((element) => element['senseId'] as int);
final groupedSenseSeeAlsos = senseSeeAlsos.groupListsBy( final groupedSenseSeeAlsos =
(element) => element['senseId'] as int, senseSeeAlsos.groupListsBy((element) => element['senseId'] as int);
);
final List<WordSearchSense> result = []; final List<WordSearchSense> result = [];
for (final sense in senses) { for (final sense in senses) {
@@ -262,82 +251,45 @@ List<WordSearchSense> _regroupSenses({
groupedSenseRestrictedToReadings[senseId] ?? []; groupedSenseRestrictedToReadings[senseId] ?? [];
final seeAlsos = groupedSenseSeeAlsos[senseId] ?? []; final seeAlsos = groupedSenseSeeAlsos[senseId] ?? [];
final List<WordSearchResult> seeAlsosWordResults =
senseSeeAlsosXrefData != null
? regroupWordSearchResults(
entryIds: seeAlsos
.map((e) => ScoredEntryId(e['xrefEntryId'] as int, 0))
.toList(),
linearWordQueryData: senseSeeAlsosXrefData,
)
: [];
final List<WordSearchResult> antonymsWordResults =
senseAntonymsXrefData != null
? regroupWordSearchResults(
entryIds: antonyms
.map((e) => ScoredEntryId(e['xrefEntryId'] as int, 0))
.toList(),
linearWordQueryData: senseAntonymsXrefData,
)
: [];
final resultSense = WordSearchSense( final resultSense = WordSearchSense(
englishDefinitions: glossaries.map((e) => e['phrase'] as String).toList(), englishDefinitions: glossaries.map((e) => e['phrase'] as String).toList(),
partsOfSpeech: pos partsOfSpeech:
.map((e) => JMdictPOS.fromId(e['pos'] as String)) pos.map((e) => JMdictPOS.fromId(e['pos'] as String)).toList(),
seeAlso: seeAlsos
.map((e) => WordSearchXrefEntry(
entryId: e['xrefEntryId'] as int,
baseWord: e['base'] as String,
furigana: e['furigana'] as String?,
ambiguous: e['ambiguous'] == 1,
))
.toList(), .toList(),
seeAlso: seeAlsos.asMap().entries.map<WordSearchXrefEntry>((mapEntry) { antonyms: antonyms
final i = mapEntry.key; .map((e) => WordSearchXrefEntry(
final e = mapEntry.value; entryId: e['xrefEntryId'] as int,
baseWord: e['base'] as String,
return WordSearchXrefEntry( furigana: e['furigana'] as String?,
entryId: e['xrefEntryId'] as int, ambiguous: e['ambiguous'] == 1,
baseWord: e['base'] as String, ))
furigana: e['furigana'] as String?,
ambiguous: e['ambiguous'] == 1,
xrefResult: seeAlsosWordResults.isNotEmpty
? seeAlsosWordResults[i]
: null,
);
}).toList(),
antonyms: antonyms.asMap().entries.map<WordSearchXrefEntry>((mapEntry) {
final i = mapEntry.key;
final e = mapEntry.value;
return WordSearchXrefEntry(
entryId: e['xrefEntryId'] as int,
baseWord: e['base'] as String,
furigana: e['furigana'] as String?,
ambiguous: e['ambiguous'] == 1,
xrefResult: antonymsWordResults.isNotEmpty
? antonymsWordResults[i]
: null,
);
}).toList(),
restrictedToReading: restrictedToReadings
.map((e) => e['reading'] as String)
.toList(),
restrictedToKanji: restrictedToKanjis
.map((e) => e['kanji'] as String)
.toList(),
fields: fields
.map((e) => JMdictField.fromId(e['field'] as String))
.toList(), .toList(),
restrictedToReading:
restrictedToReadings.map((e) => e['reading'] as String).toList(),
restrictedToKanji:
restrictedToKanjis.map((e) => e['kanji'] as String).toList(),
fields:
fields.map((e) => JMdictField.fromId(e['field'] as String)).toList(),
dialects: dialects dialects: dialects
.map((e) => JMdictDialect.fromId(e['dialect'] as String)) .map((e) => JMdictDialect.fromId(e['dialect'] as String))
.toList(), .toList(),
misc: miscs.map((e) => JMdictMisc.fromId(e['misc'] as String)).toList(), misc: miscs.map((e) => JMdictMisc.fromId(e['misc'] as String)).toList(),
info: infos.map((e) => e['info'] as String).toList(), info: infos.map((e) => e['info'] as String).toList(),
languageSource: languageSources languageSource: languageSources
.map( .map((e) => WordSearchSenseLanguageSource(
(e) => WordSearchSenseLanguageSource( language: e['language'] as String,
language: e['language'] as String, phrase: e['phrase'] as String?,
phrase: e['phrase'] as String?, fullyDescribesSense: e['fullyDescribesSense'] == 1,
fullyDescribesSense: e['fullyDescribesSense'] == 1, constructedFromSmallerWords:
constructedFromSmallerWords: e['constructedFromSmallerWords'] == 1,
e['constructedFromSmallerWords'] == 1, ))
),
)
.toList(), .toList(),
); );

View File

@@ -14,38 +14,26 @@ import 'package:jadb/table_names/jmdict.dart';
import 'package:sqflite_common/sqlite_api.dart'; import 'package:sqflite_common/sqlite_api.dart';
enum SearchMode { enum SearchMode {
/// Try to autodetect what is being searched for Auto,
auto, English,
Kanji,
/// Search for english words MixedKanji,
english, Kana,
MixedKana,
/// Search for the kanji reading of a word
kanji,
/// Search for the kanji reading of a word, mixed in with kana/romaji
mixedKanji,
/// Search for the kana reading of a word
kana,
/// Search for the kana reading of a word, mixed in with romaji
mixedKana,
} }
/// Searches for an input string, returning a list of results with their details. Returns null if the input string is empty.
Future<List<WordSearchResult>?> searchWordWithDbConnection( Future<List<WordSearchResult>?> searchWordWithDbConnection(
DatabaseExecutor connection, DatabaseExecutor connection,
String word, { String word,
SearchMode searchMode = SearchMode.auto, SearchMode searchMode,
int page = 0, int page,
int? pageSize, int pageSize,
}) async { ) async {
if (word.isEmpty) { if (word.isEmpty) {
return null; return null;
} }
final int? offset = pageSize != null ? page * pageSize : null; final offset = page * pageSize;
final List<ScoredEntryId> entryIds = await fetchEntryIds( final List<ScoredEntryId> entryIds = await fetchEntryIds(
connection, connection,
word, word,
@@ -55,34 +43,47 @@ Future<List<WordSearchResult>?> searchWordWithDbConnection(
); );
if (entryIds.isEmpty) { if (entryIds.isEmpty) {
// TODO: try conjugation search
return []; return [];
} }
final LinearWordQueryData linearWordQueryData = final LinearWordQueryData linearWordQueryData =
await fetchLinearWordQueryData( await fetchLinearWordQueryData(
connection, connection,
entryIds.map((e) => e.entryId).toList(), entryIds.map((e) => e.entryId).toList(),
); );
final result = regroupWordSearchResults( final result = regroupWordSearchResults(
entryIds: entryIds, entryIds: entryIds,
linearWordQueryData: linearWordQueryData, readingElements: linearWordQueryData.readingElements,
kanjiElements: linearWordQueryData.kanjiElements,
jlptTags: linearWordQueryData.jlptTags,
commonEntries: linearWordQueryData.commonEntries,
senses: linearWordQueryData.senses,
senseAntonyms: linearWordQueryData.senseAntonyms,
senseDialects: linearWordQueryData.senseDialects,
senseFields: linearWordQueryData.senseFields,
senseGlossaries: linearWordQueryData.senseGlossaries,
senseInfos: linearWordQueryData.senseInfos,
senseLanguageSources: linearWordQueryData.senseLanguageSources,
senseMiscs: linearWordQueryData.senseMiscs,
sensePOSs: linearWordQueryData.sensePOSs,
senseRestrictedToKanjis: linearWordQueryData.senseRestrictedToKanjis,
senseRestrictedToReadings: linearWordQueryData.senseRestrictedToReadings,
senseSeeAlsos: linearWordQueryData.senseSeeAlsos,
exampleSentences: linearWordQueryData.exampleSentences,
readingElementInfos: linearWordQueryData.readingElementInfos,
readingElementRestrictions: linearWordQueryData.readingElementRestrictions,
kanjiElementInfos: linearWordQueryData.kanjiElementInfos,
); );
for (final resultEntry in result) {
resultEntry.inferMatchSpans(word, searchMode: searchMode);
}
return result; return result;
} }
/// Searches for an input string, returning the amount of results that the search would yield without pagination.
Future<int?> searchWordCountWithDbConnection( Future<int?> searchWordCountWithDbConnection(
DatabaseExecutor connection, DatabaseExecutor connection,
String word, { String word,
SearchMode searchMode = SearchMode.auto, SearchMode searchMode,
}) async { ) async {
if (word.isEmpty) { if (word.isEmpty) {
return null; return null;
} }
@@ -96,7 +97,6 @@ Future<int?> searchWordCountWithDbConnection(
return entryIdCount; return entryIdCount;
} }
/// Fetches a single word by its entry ID, returning null if not found.
Future<WordSearchResult?> getWordByIdWithDbConnection( Future<WordSearchResult?> getWordByIdWithDbConnection(
DatabaseExecutor connection, DatabaseExecutor connection,
int id, int id,
@@ -105,23 +105,43 @@ Future<WordSearchResult?> getWordByIdWithDbConnection(
return null; return null;
} }
final exists = await connection final exists = await connection.rawQuery(
.rawQuery( 'SELECT EXISTS(SELECT 1 FROM "${JMdictTableNames.entry}" WHERE "entryId" = ?)',
'SELECT EXISTS(SELECT 1 FROM "${JMdictTableNames.entry}" WHERE "entryId" = ?)', [id],
[id], ).then((value) => value.isNotEmpty && value.first.values.first == 1);
)
.then((value) => value.isNotEmpty && value.first.values.first == 1);
if (!exists) { if (!exists) {
return null; return null;
} }
final LinearWordQueryData linearWordQueryData = final LinearWordQueryData linearWordQueryData =
await fetchLinearWordQueryData(connection, [id]); await fetchLinearWordQueryData(
connection,
[id],
);
final result = regroupWordSearchResults( final result = regroupWordSearchResults(
entryIds: [ScoredEntryId(id, 0)], entryIds: [ScoredEntryId(id, 0)],
linearWordQueryData: linearWordQueryData, readingElements: linearWordQueryData.readingElements,
kanjiElements: linearWordQueryData.kanjiElements,
jlptTags: linearWordQueryData.jlptTags,
commonEntries: linearWordQueryData.commonEntries,
senses: linearWordQueryData.senses,
senseAntonyms: linearWordQueryData.senseAntonyms,
senseDialects: linearWordQueryData.senseDialects,
senseFields: linearWordQueryData.senseFields,
senseGlossaries: linearWordQueryData.senseGlossaries,
senseInfos: linearWordQueryData.senseInfos,
senseLanguageSources: linearWordQueryData.senseLanguageSources,
senseMiscs: linearWordQueryData.senseMiscs,
sensePOSs: linearWordQueryData.sensePOSs,
senseRestrictedToKanjis: linearWordQueryData.senseRestrictedToKanjis,
senseRestrictedToReadings: linearWordQueryData.senseRestrictedToReadings,
senseSeeAlsos: linearWordQueryData.senseSeeAlsos,
exampleSentences: linearWordQueryData.exampleSentences,
readingElementInfos: linearWordQueryData.readingElementInfos,
readingElementRestrictions: linearWordQueryData.readingElementRestrictions,
kanjiElementInfos: linearWordQueryData.kanjiElementInfos,
); );
assert( assert(
@@ -131,27 +151,3 @@ Future<WordSearchResult?> getWordByIdWithDbConnection(
return result.firstOrNull; return result.firstOrNull;
} }
/// Fetches multiple words by their entry IDs, returning a map from entry ID to result.
Future<Map<int, WordSearchResult>> getWordsByIdsWithDbConnection(
DatabaseExecutor connection,
Set<int> ids,
) async {
if (ids.isEmpty) {
return {};
}
final LinearWordQueryData linearWordQueryData =
await fetchLinearWordQueryData(connection, ids.toList());
final List<ScoredEntryId> entryIds = ids
.map((id) => ScoredEntryId(id, 0)) // Score is not used here
.toList();
final results = regroupWordSearchResults(
entryIds: entryIds,
linearWordQueryData: linearWordQueryData,
);
return {for (var r in results) r.entryId: r};
}

View File

@@ -20,23 +20,23 @@ abstract class JMdictTableNames {
static const String senseSeeAlso = 'JMdict_SenseSeeAlso'; static const String senseSeeAlso = 'JMdict_SenseSeeAlso';
static Set<String> get allTables => { static Set<String> get allTables => {
entry, entry,
kanjiElement, kanjiElement,
kanjiInfo, kanjiInfo,
readingElement, readingElement,
readingInfo, readingInfo,
readingRestriction, readingRestriction,
sense, sense,
senseAntonyms, senseAntonyms,
senseDialect, senseDialect,
senseField, senseField,
senseGlossary, senseGlossary,
senseInfo, senseInfo,
senseMisc, senseMisc,
sensePOS, sensePOS,
senseLanguageSource, senseLanguageSource,
senseRestrictedToKanji, senseRestrictedToKanji,
senseRestrictedToReading, senseRestrictedToReading,
senseSeeAlso, senseSeeAlso
}; };
} }

View File

@@ -17,19 +17,19 @@ abstract class KANJIDICTableNames {
static const String nanori = 'KANJIDIC_Nanori'; static const String nanori = 'KANJIDIC_Nanori';
static Set<String> get allTables => { static Set<String> get allTables => {
character, character,
radicalName, radicalName,
codepoint, codepoint,
radical, radical,
strokeMiscount, strokeMiscount,
variant, variant,
dictionaryReference, dictionaryReference,
dictionaryReferenceMoro, dictionaryReferenceMoro,
queryCode, queryCode,
reading, reading,
kunyomi, kunyomi,
onyomi, onyomi,
meaning, meaning,
nanori, nanori
}; };
} }

View File

@@ -1,5 +1,7 @@
abstract class RADKFILETableNames { abstract class RADKFILETableNames {
static const String radkfile = 'RADKFILE'; static const String radkfile = 'RADKFILE';
static Set<String> get allTables => {radkfile}; static Set<String> get allTables => {
radkfile,
};
} }

View File

@@ -276,22 +276,29 @@ extension on DateTime {
/// See more info here: /// See more info here:
/// - https://en.wikipedia.org/wiki/Nanboku-ch%C5%8D_period /// - https://en.wikipedia.org/wiki/Nanboku-ch%C5%8D_period
/// - http://www.kumamotokokufu-h.ed.jp/kumamoto/bungaku/nengoui.html /// - http://www.kumamotokokufu-h.ed.jp/kumamoto/bungaku/nengoui.html
String? japaneseEra() { String? japaneseEra({bool nanbokuchouPeriodUsesNorth = true}) {
throw UnimplementedError('This function is not implemented yet.'); throw UnimplementedError('This function is not implemented yet.');
if (year < 645) { if (this.year < 645) {
return null; return null;
} }
if (year < periodsNanbokuchouNorth.keys.first.$1) { if (this.year < periodsNanbokuchouNorth.keys.first.$1) {
// TODO: find first where year <= this.year and jump one period back. // TODO: find first where year <= this.year and jump one period back.
} }
} }
String get japaneseWeekdayPrefix => String get japaneseWeekdayPrefix => [
['', '', '', '', '', '', ''][weekday - 1]; '',
'',
'',
'',
'',
'',
'',
][weekday - 1];
/// Returns the date in Japanese format. /// Returns the date in Japanese format.
String japaneseDate({bool showWeekday = false}) => String japaneseDate({bool showWeekday = false}) =>
'$month月$day日${showWeekday ? '$japaneseWeekdayPrefix' : ''}'; '$month月$day日' + (showWeekday ? '$japaneseWeekdayPrefix' : '');
} }

View File

@@ -12,7 +12,10 @@ enum WordClass {
input, input,
} }
enum LemmatizationRuleType { prefix, suffix } enum LemmatizationRuleType {
prefix,
suffix,
}
class LemmatizationRule { class LemmatizationRule {
final String name; final String name;
@@ -43,18 +46,18 @@ class LemmatizationRule {
lookAheadBehind = const [''], lookAheadBehind = const [''],
LemmatizationRuleType type = LemmatizationRuleType.suffix, LemmatizationRuleType type = LemmatizationRuleType.suffix,
}) : this( }) : this(
name: name, name: name,
pattern: AllomorphPattern( pattern: AllomorphPattern(
patterns: { patterns: {
pattern: replacement != null ? [replacement] : null, pattern: replacement != null ? [replacement] : null
}, },
type: type, type: type,
lookAheadBehind: lookAheadBehind, lookAheadBehind: lookAheadBehind,
), ),
validChildClasses: validChildClasses, validChildClasses: validChildClasses,
terminal: terminal, terminal: terminal,
wordClass: wordClass, wordClass: wordClass,
); );
} }
/// Represents a set of patterns for matching allomorphs in a word. /// Represents a set of patterns for matching allomorphs in a word.
@@ -129,8 +132,8 @@ class AllomorphPattern {
if (word.startsWith(p as String)) { if (word.startsWith(p as String)) {
return patterns[affix] != null return patterns[affix] != null
? patterns[affix]! ? patterns[affix]!
.map((s) => s + word.substring(affix.length)) .map((s) => s + word.substring(affix.length))
.toList() .toList()
: [word.substring(affix.length)]; : [word.substring(affix.length)];
} }
break; break;
@@ -183,7 +186,7 @@ class Lemmatized {
@override @override
String toString() { String toString() {
final childrenString = children final childrenString = children
.map((c) => ' - ${c.toString().split('\n').join('\n ')}') .map((c) => ' - ' + c.toString().split('\n').join('\n '))
.join('\n'); .join('\n');
if (children.isEmpty) { if (children.isEmpty) {
@@ -236,6 +239,9 @@ Lemmatized lemmatize(String word) {
return Lemmatized( return Lemmatized(
original: word, original: word,
rule: inputRule, rule: inputRule,
children: _lemmatize(inputRule, word), children: _lemmatize(
inputRule,
word,
),
); );
} }

View File

@@ -1,7 +1,7 @@
import 'package:jadb/util/lemmatizer/lemmatizer.dart'; import 'package:jadb/util/lemmatizer/lemmatizer.dart';
import 'package:jadb/util/lemmatizer/rules/godan_verbs.dart'; import 'package:jadb/util/lemmatizer/rules/godan-verbs.dart';
import 'package:jadb/util/lemmatizer/rules/i_adjectives.dart'; import 'package:jadb/util/lemmatizer/rules/i-adjectives.dart';
import 'package:jadb/util/lemmatizer/rules/ichidan_verbs.dart'; import 'package:jadb/util/lemmatizer/rules/ichidan-verbs.dart';
List<LemmatizationRule> lemmatizationRules = [ List<LemmatizationRule> lemmatizationRules = [
...ichidanVerbLemmatizationRules, ...ichidanVerbLemmatizationRules,

View File

@@ -1,9 +1,9 @@
// Source: https://github.com/Kimtaro/ve/blob/master/lib/providers/japanese_transliterators.rb // Source: https://github.com/Kimtaro/ve/blob/master/lib/providers/japanese_transliterators.rb
const hiraganaSyllabicN = ''; const hiragana_syllabic_n = '';
const hiraganaSmallTsu = ''; const hiragana_small_tsu = '';
const Map<String, String> hiraganaToLatin = { const Map<String, String> hiragana_to_latin = {
'': 'a', '': 'a',
'': 'i', '': 'i',
'': 'u', '': 'u',
@@ -209,7 +209,7 @@ const Map<String, String> hiraganaToLatin = {
'': 'yori', '': 'yori',
}; };
const Map<String, String> latinToHiragana = { const Map<String, String> latin_to_hiragana = {
'a': '', 'a': '',
'i': '', 'i': '',
'u': '', 'u': '',
@@ -481,9 +481,9 @@ const Map<String, String> latinToHiragana = {
'#~': '', '#~': '',
}; };
bool _smallTsu(String forConversion) => forConversion == hiraganaSmallTsu; bool _smallTsu(String for_conversion) => for_conversion == hiragana_small_tsu;
bool _nFollowedByYuYeYo(String forConversion, String kana) => bool _nFollowedByYuYeYo(String for_conversion, String kana) =>
forConversion == hiraganaSyllabicN && for_conversion == hiragana_syllabic_n &&
kana.length > 1 && kana.length > 1 &&
'やゆよ'.contains(kana.substring(1, 2)); 'やゆよ'.contains(kana.substring(1, 2));
@@ -495,17 +495,17 @@ String transliterateHiraganaToLatin(String hiragana) {
while (kana.isNotEmpty) { while (kana.isNotEmpty) {
final lengths = [if (kana.length > 1) 2, 1]; final lengths = [if (kana.length > 1) 2, 1];
for (final length in lengths) { for (final length in lengths) {
final String forConversion = kana.substring(0, length); final String for_conversion = kana.substring(0, length);
String? mora; String? mora;
if (_smallTsu(forConversion)) { if (_smallTsu(for_conversion)) {
geminate = true; geminate = true;
kana = kana.replaceRange(0, length, ''); kana = kana.replaceRange(0, length, '');
break; break;
} else if (_nFollowedByYuYeYo(forConversion, kana)) { } else if (_nFollowedByYuYeYo(for_conversion, kana)) {
mora = "n'"; mora = "n'";
} }
mora ??= hiraganaToLatin[forConversion]; mora ??= hiragana_to_latin[for_conversion];
if (mora != null) { if (mora != null) {
if (geminate) { if (geminate) {
@@ -516,7 +516,7 @@ String transliterateHiraganaToLatin(String hiragana) {
kana = kana.replaceRange(0, length, ''); kana = kana.replaceRange(0, length, '');
break; break;
} else if (length == 1) { } else if (length == 1) {
romaji += forConversion; romaji += for_conversion;
kana = kana.replaceRange(0, length, ''); kana = kana.replaceRange(0, length, '');
} }
} }
@@ -524,46 +524,48 @@ String transliterateHiraganaToLatin(String hiragana) {
return romaji; return romaji;
} }
bool _doubleNFollowedByAIUEO(String forConversion) => bool _doubleNFollowedByAIUEO(String for_conversion) =>
RegExp(r'^nn[aiueo]$').hasMatch(forConversion); RegExp(r'^nn[aiueo]$').hasMatch(for_conversion);
bool _hasTableMatch(String forConversion) => bool _hasTableMatch(String for_conversion) =>
latinToHiragana[forConversion] != null; latin_to_hiragana[for_conversion] != null;
bool _hasDoubleConsonant(String forConversion, int length) => bool _hasDoubleConsonant(String for_conversion, int length) =>
forConversion == 'tch' || for_conversion == 'tch' ||
(length == 2 && (length == 2 &&
RegExp(r'^([kgsztdnbpmyrlwchf])\1$').hasMatch(forConversion)); RegExp(r'^([kgsztdnbpmyrlwchf])\1$').hasMatch(for_conversion));
String transliterateLatinToHiragana(String latin) { String transliterateLatinToHiragana(String latin) {
String romaji = latin String romaji =
.toLowerCase() latin.toLowerCase().replaceAll('mb', 'nb').replaceAll('mp', 'np');
.replaceAll('mb', 'nb')
.replaceAll('mp', 'np');
String kana = ''; String kana = '';
while (romaji.isNotEmpty) { while (romaji.isNotEmpty) {
final lengths = [if (romaji.length > 2) 3, if (romaji.length > 1) 2, 1]; final lengths = [
if (romaji.length > 2) 3,
if (romaji.length > 1) 2,
1,
];
for (final length in lengths) { for (final length in lengths) {
String? mora; String? mora;
int forRemoval = length; int for_removal = length;
final String forConversion = romaji.substring(0, length); final String for_conversion = romaji.substring(0, length);
if (_doubleNFollowedByAIUEO(forConversion)) { if (_doubleNFollowedByAIUEO(for_conversion)) {
mora = hiraganaSyllabicN; mora = hiragana_syllabic_n;
forRemoval = 1; for_removal = 1;
} else if (_hasTableMatch(forConversion)) { } else if (_hasTableMatch(for_conversion)) {
mora = latinToHiragana[forConversion]; mora = latin_to_hiragana[for_conversion];
} else if (_hasDoubleConsonant(forConversion, length)) { } else if (_hasDoubleConsonant(for_conversion, length)) {
mora = hiraganaSmallTsu; mora = hiragana_small_tsu;
forRemoval = 1; for_removal = 1;
} }
if (mora != null) { if (mora != null) {
kana += mora; kana += mora;
romaji = romaji.replaceRange(0, forRemoval, ''); romaji = romaji.replaceRange(0, for_removal, '');
break; break;
} else if (length == 1) { } else if (length == 1) {
kana += forConversion; kana += for_conversion;
romaji = romaji.replaceRange(0, 1, ''); romaji = romaji.replaceRange(0, 1, '');
} }
} }
@@ -577,11 +579,11 @@ String _transposeCodepointsInRange(
int distance, int distance,
int rangeStart, int rangeStart,
int rangeEnd, int rangeEnd,
) => String.fromCharCodes( ) =>
text.codeUnits.map( String.fromCharCodes(
(c) => c + ((rangeStart <= c && c <= rangeEnd) ? distance : 0), text.codeUnits
), .map((c) => c + ((rangeStart <= c && c <= rangeEnd) ? distance : 0)),
); );
String transliterateKanaToLatin(String kana) => String transliterateKanaToLatin(String kana) =>
transliterateHiraganaToLatin(transliterateKatakanaToHiragana(kana)); transliterateHiraganaToLatin(transliterateKatakanaToHiragana(kana));
@@ -597,7 +599,12 @@ String transliterateHiraganaToKatakana(String hiragana) =>
String transliterateFullwidthRomajiToHalfwidth(String halfwidth) => String transliterateFullwidthRomajiToHalfwidth(String halfwidth) =>
_transposeCodepointsInRange( _transposeCodepointsInRange(
_transposeCodepointsInRange(halfwidth, -65248, 65281, 65374), _transposeCodepointsInRange(
halfwidth,
-65248,
65281,
65374,
),
-12256, -12256,
12288, 12288,
12288, 12288,
@@ -605,7 +612,12 @@ String transliterateFullwidthRomajiToHalfwidth(String halfwidth) =>
String transliterateHalfwidthRomajiToFullwidth(String halfwidth) => String transliterateHalfwidthRomajiToFullwidth(String halfwidth) =>
_transposeCodepointsInRange( _transposeCodepointsInRange(
_transposeCodepointsInRange(halfwidth, 65248, 33, 126), _transposeCodepointsInRange(
halfwidth,
65248,
33,
126,
),
12256, 12256,
32, 32,
32, 32,

View File

@@ -1,3 +1,3 @@
String escapeStringValue(String value) { String escapeStringValue(String value) {
return "'${value.replaceAll("'", "''")}'"; return "'" + value.replaceAll("'", "''") + "'";
} }

View File

@@ -1,6 +1,5 @@
CREATE TABLE "JMdict_EntryScore" ( CREATE TABLE "JMdict_EntryScore" (
"type" CHAR(1) NOT NULL CHECK ("type" IN ('r', 'k')), "type" TEXT NOT NULL CHECK ("type" IN ('reading', 'kanji')),
"entryId" INTEGER NOT NULL REFERENCES "JMdict_Entry"("entryId"),
"elementId" INTEGER NOT NULL, "elementId" INTEGER NOT NULL,
"score" INTEGER NOT NULL DEFAULT 0, "score" INTEGER NOT NULL DEFAULT 0,
"common" BOOLEAN NOT NULL DEFAULT FALSE, "common" BOOLEAN NOT NULL DEFAULT FALSE,
@@ -20,8 +19,7 @@ CREATE INDEX "JMdict_EntryScore_byType_byCommon" ON "JMdict_EntryScore"("type",
CREATE VIEW "JMdict_EntryScoreView_Reading" AS CREATE VIEW "JMdict_EntryScoreView_Reading" AS
SELECT SELECT
'r' AS "type", 'reading' AS "type",
"JMdict_ReadingElement"."entryId",
"JMdict_ReadingElement"."elementId", "JMdict_ReadingElement"."elementId",
( (
"news" IS 1 "news" IS 1
@@ -52,8 +50,7 @@ LEFT JOIN "JMdict_JLPTTag" USING ("entryId");
CREATE VIEW "JMdict_EntryScoreView_Kanji" AS CREATE VIEW "JMdict_EntryScoreView_Kanji" AS
SELECT SELECT
'k' AS "type", 'kanji' AS "type",
"JMdict_KanjiElement"."entryId",
"JMdict_KanjiElement"."elementId", "JMdict_KanjiElement"."elementId",
( (
"news" IS 1 "news" IS 1
@@ -97,12 +94,11 @@ AFTER INSERT ON "JMdict_ReadingElement"
BEGIN BEGIN
INSERT INTO "JMdict_EntryScore" ( INSERT INTO "JMdict_EntryScore" (
"type", "type",
"entryId",
"elementId", "elementId",
"score", "score",
"common" "common"
) )
SELECT "type", "entryId", "elementId", "score", "common" SELECT "type", "elementId", "score", "common"
FROM "JMdict_EntryScoreView_Reading" FROM "JMdict_EntryScoreView_Reading"
WHERE "elementId" = NEW."elementId"; WHERE "elementId" = NEW."elementId";
END; END;
@@ -123,7 +119,7 @@ CREATE TRIGGER "JMdict_EntryScore_Delete_JMdict_ReadingElement"
AFTER DELETE ON "JMdict_ReadingElement" AFTER DELETE ON "JMdict_ReadingElement"
BEGIN BEGIN
DELETE FROM "JMdict_EntryScore" DELETE FROM "JMdict_EntryScore"
WHERE "type" = 'r' WHERE "type" = 'reading'
AND "elementId" = OLD."elementId"; AND "elementId" = OLD."elementId";
END; END;
@@ -134,12 +130,11 @@ AFTER INSERT ON "JMdict_KanjiElement"
BEGIN BEGIN
INSERT INTO "JMdict_EntryScore" ( INSERT INTO "JMdict_EntryScore" (
"type", "type",
"entryId",
"elementId", "elementId",
"score", "score",
"common" "common"
) )
SELECT "type", "entryId", "elementId", "score", "common" SELECT "type", "elementId", "score", "common"
FROM "JMdict_EntryScoreView_Kanji" FROM "JMdict_EntryScoreView_Kanji"
WHERE "elementId" = NEW."elementId"; WHERE "elementId" = NEW."elementId";
END; END;
@@ -160,7 +155,7 @@ CREATE TRIGGER "JMdict_EntryScore_Delete_JMdict_KanjiElement"
AFTER DELETE ON "JMdict_KanjiElement" AFTER DELETE ON "JMdict_KanjiElement"
BEGIN BEGIN
DELETE FROM "JMdict_EntryScore" DELETE FROM "JMdict_EntryScore"
WHERE "type" = 'k' WHERE "type" = 'kanji'
AND "elementId" = OLD."elementId"; AND "elementId" = OLD."elementId";
END; END;
@@ -174,9 +169,26 @@ BEGIN
"score" = "JMdict_EntryScoreView"."score", "score" = "JMdict_EntryScoreView"."score",
"common" = "JMdict_EntryScoreView"."common" "common" = "JMdict_EntryScoreView"."common"
FROM "JMdict_EntryScoreView" FROM "JMdict_EntryScoreView"
WHERE "JMdict_EntryScoreView"."entryId" = NEW."entryId" WHERE
AND "JMdict_EntryScore"."entryId" = NEW."entryId" (
AND "JMdict_EntryScoreView"."elementId" = "JMdict_EntryScore"."elementId"; (
"JMdict_EntryScoreView"."type" = 'kanji'
AND
"JMdict_EntryScoreView"."elementId" IN (
SELECT "elementId" FROM "JMdict_KanjiElement" WHERE "entryId" = NEW."entryId"
)
)
OR
(
"JMdict_EntryScoreView"."type" = 'reading'
AND
"JMdict_EntryScoreView"."elementId" IN (
SELECT "elementId" FROM "JMdict_ReadingElement" WHERE "entryId" = NEW."entryId"
)
)
)
AND "JMdict_EntryScoreView"."entryId" = "JMdict_EntryScore"."entryId"
AND "JMdict_EntryScoreView"."reading" = "JMdict_EntryScore"."reading";
END; END;
CREATE TRIGGER "JMdict_EntryScore_Update_JMdict_JLPTTag" CREATE TRIGGER "JMdict_EntryScore_Update_JMdict_JLPTTag"
@@ -188,9 +200,26 @@ BEGIN
"score" = "JMdict_EntryScoreView"."score", "score" = "JMdict_EntryScoreView"."score",
"common" = "JMdict_EntryScoreView"."common" "common" = "JMdict_EntryScoreView"."common"
FROM "JMdict_EntryScoreView" FROM "JMdict_EntryScoreView"
WHERE "JMdict_EntryScoreView"."entryId" = NEW."entryId" WHERE
AND "JMdict_EntryScore"."entryId" = NEW."entryId" (
AND "JMdict_EntryScoreView"."elementId" = "JMdict_EntryScore"."elementId"; (
"JMdict_EntryScoreView"."type" = 'kanji'
AND
"JMdict_EntryScoreView"."elementId" IN (
SELECT "elementId" FROM "JMdict_KanjiElement" WHERE "entryId" = NEW."entryId"
)
)
OR
(
"JMdict_EntryScoreView"."type" = 'reading'
AND
"JMdict_EntryScoreView"."elementId" IN (
SELECT "elementId" FROM "JMdict_ReadingElement" WHERE "entryId" = NEW."entryId"
)
)
)
AND "JMdict_EntryScoreView"."entryId" = "JMdict_EntryScore"."entryId"
AND "JMdict_EntryScoreView"."reading" = "JMdict_EntryScore"."reading";
END; END;
CREATE TRIGGER "JMdict_EntryScore_Delete_JMdict_JLPTTag" CREATE TRIGGER "JMdict_EntryScore_Delete_JMdict_JLPTTag"
@@ -201,7 +230,24 @@ BEGIN
"score" = "JMdict_EntryScoreView"."score", "score" = "JMdict_EntryScoreView"."score",
"common" = "JMdict_EntryScoreView"."common" "common" = "JMdict_EntryScoreView"."common"
FROM "JMdict_EntryScoreView" FROM "JMdict_EntryScoreView"
WHERE "JMdict_EntryScoreView"."entryId" = OLD."entryId" WHERE
AND "JMdict_EntryScore"."entryId" = OLD."entryId" (
AND "JMdict_EntryScoreView"."elementId" = "JMdict_EntryScore"."elementId"; (
"JMdict_EntryScoreView"."type" = 'kanji'
AND
"JMdict_EntryScoreView"."elementId" IN (
SELECT "elementId" FROM "JMdict_KanjiElement" WHERE "entryId" = OLD."entryId"
)
)
OR
(
"JMdict_EntryScoreView"."type" = 'reading'
AND
"JMdict_EntryScoreView"."elementId" IN (
SELECT "elementId" FROM "JMdict_ReadingElement" WHERE "entryId" = OLD."entryId"
)
)
)
AND "JMdict_EntryScoreView"."entryId" = "JMdict_EntryScore"."entryId"
AND "JMdict_EntryScoreView"."reading" = "JMdict_EntryScore"."reading";
END; END;

View File

@@ -65,7 +65,7 @@ JOIN "JMdict_KanjiElement"
ON "JMdict_KanjiElementFTS"."entryId" = "JMdict_KanjiElement"."entryId" ON "JMdict_KanjiElementFTS"."entryId" = "JMdict_KanjiElement"."entryId"
AND "JMdict_KanjiElementFTS"."reading" LIKE '%' || "JMdict_KanjiElement"."reading" AND "JMdict_KanjiElementFTS"."reading" LIKE '%' || "JMdict_KanjiElement"."reading"
JOIN "JMdict_EntryScore" JOIN "JMdict_EntryScore"
ON "JMdict_EntryScore"."type" = 'k' ON "JMdict_EntryScore"."type" = 'kanji'
AND "JMdict_KanjiElement"."entryId" = "JMdict_EntryScore"."entryId" AND "JMdict_KanjiElement"."entryId" = "JMdict_EntryScore"."entryId"
AND "JMdict_KanjiElement"."reading" = "JMdict_EntryScore"."reading" AND "JMdict_KanjiElement"."reading" = "JMdict_EntryScore"."reading"
WHERE "JMdict_EntryScore"."common" = 1; WHERE "JMdict_EntryScore"."common" = 1;
@@ -78,9 +78,9 @@ CREATE VIEW "JMdict_CombinedEntryScore"
AS AS
SELECT SELECT
CASE CASE
WHEN "JMdict_EntryScore"."type" = 'k' WHEN "JMdict_EntryScore"."type" = 'kanji'
THEN (SELECT entryId FROM "JMdict_KanjiElement" WHERE "elementId" = "JMdict_EntryScore"."elementId") THEN (SELECT entryId FROM "JMdict_KanjiElement" WHERE "elementId" = "JMdict_EntryScore"."elementId")
WHEN "JMdict_EntryScore"."type" = 'r' WHEN "JMdict_EntryScore"."type" = 'reading'
THEN (SELECT entryId FROM "JMdict_ReadingElement" WHERE "elementId" = "JMdict_EntryScore"."elementId") THEN (SELECT entryId FROM "JMdict_ReadingElement" WHERE "elementId" = "JMdict_EntryScore"."elementId")
END AS "entryId", END AS "entryId",
MAX("JMdict_EntryScore"."score") AS "score", MAX("JMdict_EntryScore"."score") AS "score",

View File

@@ -7,29 +7,6 @@ buildDartApplication {
version = "1.0.0"; version = "1.0.0";
inherit src; inherit src;
dartEntryPoints."bin/jadb" = "bin/jadb.dart";
# NOTE: the default dart hooks are using `dart compile`, which is not able to call the
# new dart build hooks required to use package:sqlite3 >= 3.0.0. So we override
# these phases to use `dart build` instead.
buildPhase = ''
runHook preBuild
mkdir -p "$out/bin"
dart build cli --target "bin/jadb.dart"
runHook postBuild
'';
installPhase = ''
runHook preInstall
mkdir -p "$out"
mv build/cli/*/bundle/* "$out/"
runHook postInstall
'';
autoPubspecLock = ../pubspec.lock; autoPubspecLock = ../pubspec.lock;
meta.mainProgram = "jadb"; meta.mainProgram = "jadb";

View File

@@ -5,18 +5,18 @@ packages:
dependency: transitive dependency: transitive
description: description:
name: _fe_analyzer_shared name: _fe_analyzer_shared
sha256: "3b19a47f6ea7c2632760777c78174f47f6aec1e05f0cd611380d4593b8af1dbc" sha256: e55636ed79578b9abca5fecf9437947798f5ef7456308b5cb85720b793eac92f
url: "https://pub.dev" url: "https://pub.dev"
source: hosted source: hosted
version: "96.0.0" version: "82.0.0"
analyzer: analyzer:
dependency: transitive dependency: transitive
description: description:
name: analyzer name: analyzer
sha256: "0c516bc4ad36a1a75759e54d5047cb9d15cded4459df01aa35a0b5ec7db2c2a0" sha256: "904ae5bb474d32c38fb9482e2d925d5454cda04ddd0e55d2e6826bc72f6ba8c0"
url: "https://pub.dev" url: "https://pub.dev"
source: hosted source: hosted
version: "10.2.0" version: "7.4.5"
args: args:
dependency: "direct main" dependency: "direct main"
description: description:
@@ -49,14 +49,6 @@ packages:
url: "https://pub.dev" url: "https://pub.dev"
source: hosted source: hosted
version: "0.2.0" version: "0.2.0"
code_assets:
dependency: transitive
description:
name: code_assets
sha256: "83ccdaa064c980b5596c35dd64a8d3ecc68620174ab9b90b6343b753aa721687"
url: "https://pub.dev"
source: hosted
version: "1.0.0"
collection: collection:
dependency: "direct main" dependency: "direct main"
description: description:
@@ -77,42 +69,42 @@ packages:
dependency: transitive dependency: transitive
description: description:
name: coverage name: coverage
sha256: "5da775aa218eaf2151c721b16c01c7676fbfdd99cebba2bf64e8b807a28ff94d" sha256: "802bd084fb82e55df091ec8ad1553a7331b61c08251eef19a508b6f3f3a9858d"
url: "https://pub.dev" url: "https://pub.dev"
source: hosted source: hosted
version: "1.15.0" version: "1.13.1"
crypto: crypto:
dependency: transitive dependency: transitive
description: description:
name: crypto name: crypto
sha256: c8ea0233063ba03258fbcf2ca4d6dadfefe14f02fab57702265467a19f27fadf sha256: "1e445881f28f22d6140f181e07737b22f1e099a5e1ff94b0af2f9e4a463f4855"
url: "https://pub.dev" url: "https://pub.dev"
source: hosted source: hosted
version: "3.0.7" version: "3.0.6"
csv: csv:
dependency: "direct main" dependency: "direct main"
description: description:
name: csv name: csv
sha256: bef2950f7a753eb82f894a2eabc3072e73cf21c17096296a5a992797e50b1d0d sha256: c6aa2679b2a18cb57652920f674488d89712efaf4d3fdf2e537215b35fc19d6c
url: "https://pub.dev" url: "https://pub.dev"
source: hosted source: hosted
version: "7.1.0" version: "6.0.0"
equatable: equatable:
dependency: "direct main" dependency: "direct main"
description: description:
name: equatable name: equatable
sha256: "3e0141505477fd8ad55d6eb4e7776d3fe8430be8e497ccb1521370c3f21a3e2b" sha256: "567c64b3cb4cf82397aac55f4f0cbd3ca20d77c6c03bedbc4ceaddc08904aef7"
url: "https://pub.dev" url: "https://pub.dev"
source: hosted source: hosted
version: "2.0.8" version: "2.0.7"
ffi: ffi:
dependency: transitive dependency: transitive
description: description:
name: ffi name: ffi
sha256: "6d7fd89431262d8f3125e81b50d3847a091d846eafcd4fdb88dd06f36d705a45" sha256: "289279317b4b16eb2bb7e271abccd4bf84ec9bdcbe999e278a94b804f5630418"
url: "https://pub.dev" url: "https://pub.dev"
source: hosted source: hosted
version: "2.2.0" version: "2.1.4"
file: file:
dependency: transitive dependency: transitive
description: description:
@@ -137,14 +129,6 @@ packages:
url: "https://pub.dev" url: "https://pub.dev"
source: hosted source: hosted
version: "2.1.3" version: "2.1.3"
hooks:
dependency: transitive
description:
name: hooks
sha256: "7a08a0d684cb3b8fb604b78455d5d352f502b68079f7b80b831c62220ab0a4f6"
url: "https://pub.dev"
source: hosted
version: "1.0.1"
http_multi_server: http_multi_server:
dependency: transitive dependency: transitive
description: description:
@@ -169,14 +153,22 @@ packages:
url: "https://pub.dev" url: "https://pub.dev"
source: hosted source: hosted
version: "1.0.5" version: "1.0.5"
js:
dependency: transitive
description:
name: js
sha256: "53385261521cc4a0c4658fd0ad07a7d14591cf8fc33abbceae306ddb974888dc"
url: "https://pub.dev"
source: hosted
version: "0.7.2"
lints: lints:
dependency: "direct dev" dependency: "direct dev"
description: description:
name: lints name: lints
sha256: "12f842a479589fea194fe5c5a3095abc7be0c1f2ddfa9a0e76aed1dbd26a87df" sha256: c35bb79562d980e9a453fc715854e1ed39e24e7d0297a880ef54e17f9874a9d7
url: "https://pub.dev" url: "https://pub.dev"
source: hosted source: hosted
version: "6.1.0" version: "5.1.1"
logging: logging:
dependency: transitive dependency: transitive
description: description:
@@ -189,18 +181,18 @@ packages:
dependency: transitive dependency: transitive
description: description:
name: matcher name: matcher
sha256: "12956d0ad8390bbcc63ca2e1469c0619946ccb52809807067a7020d57e647aa6" sha256: dc58c723c3c24bf8d3e2d3ad3f2f9d7bd9cf43ec6feaa64181775e60190153f2
url: "https://pub.dev" url: "https://pub.dev"
source: hosted source: hosted
version: "0.12.18" version: "0.12.17"
meta: meta:
dependency: transitive dependency: transitive
description: description:
name: meta name: meta
sha256: "9f29b9bcc8ee287b1a31e0d01be0eae99a930dbffdaecf04b3f3d82a969f296f" sha256: "23f08335362185a5ea2ad3a4e597f1375e78bce8a040df5c600c8d3552ef2394"
url: "https://pub.dev" url: "https://pub.dev"
source: hosted source: hosted
version: "1.18.1" version: "1.17.0"
mime: mime:
dependency: transitive dependency: transitive
description: description:
@@ -209,14 +201,6 @@ packages:
url: "https://pub.dev" url: "https://pub.dev"
source: hosted source: hosted
version: "2.0.0" version: "2.0.0"
native_toolchain_c:
dependency: transitive
description:
name: native_toolchain_c
sha256: "89e83885ba09da5fdf2cdacc8002a712ca238c28b7f717910b34bcd27b0d03ac"
url: "https://pub.dev"
source: hosted
version: "0.17.4"
node_preamble: node_preamble:
dependency: transitive dependency: transitive
description: description:
@@ -234,7 +218,7 @@ packages:
source: hosted source: hosted
version: "2.2.0" version: "2.2.0"
path: path:
dependency: "direct main" dependency: transitive
description: description:
name: path name: path
sha256: "75cca69d1490965be98c73ceaea117e8a04dd21217b37b292c9ddbec0d955bc5" sha256: "75cca69d1490965be98c73ceaea117e8a04dd21217b37b292c9ddbec0d955bc5"
@@ -245,18 +229,18 @@ packages:
dependency: transitive dependency: transitive
description: description:
name: petitparser name: petitparser
sha256: "91bd59303e9f769f108f8df05e371341b15d59e995e6806aefab827b58336675" sha256: "07c8f0b1913bcde1ff0d26e57ace2f3012ccbf2b204e070290dad3bb22797646"
url: "https://pub.dev" url: "https://pub.dev"
source: hosted source: hosted
version: "7.0.2" version: "6.1.0"
pool: pool:
dependency: transitive dependency: transitive
description: description:
name: pool name: pool
sha256: "978783255c543aa3586a1b3c21f6e9d720eb315376a915872c61ef8b5c20177d" sha256: "20fe868b6314b322ea036ba325e6fc0711a22948856475e2c2b6306e8ab39c2a"
url: "https://pub.dev" url: "https://pub.dev"
source: hosted source: hosted
version: "1.5.2" version: "1.5.1"
pub_semver: pub_semver:
dependency: transitive dependency: transitive
description: description:
@@ -317,34 +301,34 @@ packages:
dependency: transitive dependency: transitive
description: description:
name: source_span name: source_span
sha256: "56a02f1f4cd1a2d96303c0144c93bd6d909eea6bee6bf5a0e0b685edbd4c47ab" sha256: "254ee5351d6cb365c859e20ee823c3bb479bf4a293c22d17a9f1bf144ce86f7c"
url: "https://pub.dev" url: "https://pub.dev"
source: hosted source: hosted
version: "1.10.2" version: "1.10.1"
sqflite_common: sqflite_common:
dependency: "direct main" dependency: "direct main"
description: description:
name: sqflite_common name: sqflite_common
sha256: "6ef422a4525ecc601db6c0a2233ff448c731307906e92cabc9ba292afaae16a6" sha256: "84731e8bfd8303a3389903e01fb2141b6e59b5973cacbb0929021df08dddbe8b"
url: "https://pub.dev" url: "https://pub.dev"
source: hosted source: hosted
version: "2.5.6" version: "2.5.5"
sqflite_common_ffi: sqflite_common_ffi:
dependency: "direct main" dependency: "direct main"
description: description:
name: sqflite_common_ffi name: sqflite_common_ffi
sha256: c59fcdc143839a77581f7a7c4de018e53682408903a0a0800b95ef2dc4033eff sha256: "1f3ef3888d3bfbb47785cc1dda0dc7dd7ebd8c1955d32a9e8e9dae1e38d1c4c1"
url: "https://pub.dev" url: "https://pub.dev"
source: hosted source: hosted
version: "2.4.0+2" version: "2.3.5"
sqlite3: sqlite3:
dependency: "direct main" dependency: transitive
description: description:
name: sqlite3 name: sqlite3
sha256: b7cf6b37667f6a921281797d2499ffc60fb878b161058d422064f0ddc78f6aa6 sha256: "310af39c40dd0bb2058538333c9d9840a2725ae0b9f77e4fd09ad6696aa8f66e"
url: "https://pub.dev" url: "https://pub.dev"
source: hosted source: hosted
version: "3.1.6" version: "2.7.5"
stack_trace: stack_trace:
dependency: transitive dependency: transitive
description: description:
@@ -373,10 +357,10 @@ packages:
dependency: transitive dependency: transitive
description: description:
name: synchronized name: synchronized
sha256: c254ade258ec8282947a0acbbc90b9575b4f19673533ee46f2f6e9b3aeefd7c0 sha256: "0669c70faae6270521ee4f05bffd2919892d42d1276e6c495be80174b6bc0ef6"
url: "https://pub.dev" url: "https://pub.dev"
source: hosted source: hosted
version: "3.4.0" version: "3.3.1"
term_glyph: term_glyph:
dependency: transitive dependency: transitive
description: description:
@@ -389,26 +373,26 @@ packages:
dependency: "direct dev" dependency: "direct dev"
description: description:
name: test name: test
sha256: "54c516bbb7cee2754d327ad4fca637f78abfc3cbcc5ace83b3eda117e42cd71a" sha256: "0561f3a2cfd33d10232360f16dfcab9351cfb7ad9b23e6cd6e8c7fb0d62c7ac3"
url: "https://pub.dev" url: "https://pub.dev"
source: hosted source: hosted
version: "1.29.0" version: "1.26.1"
test_api: test_api:
dependency: transitive dependency: transitive
description: description:
name: test_api name: test_api
sha256: "93167629bfc610f71560ab9312acdda4959de4df6fac7492c89ff0d3886f6636" sha256: "522f00f556e73044315fa4585ec3270f1808a4b186c936e612cab0b565ff1e00"
url: "https://pub.dev" url: "https://pub.dev"
source: hosted source: hosted
version: "0.7.9" version: "0.7.6"
test_core: test_core:
dependency: transitive dependency: transitive
description: description:
name: test_core name: test_core
sha256: "394f07d21f0f2255ec9e3989f21e54d3c7dc0e6e9dbce160e5a9c1a6be0e2943" sha256: "8619a9a45be044b71fe2cd6b77b54fd60f1c67904c38d48706e2852a2bda1c60"
url: "https://pub.dev" url: "https://pub.dev"
source: hosted source: hosted
version: "0.6.15" version: "0.6.10"
typed_data: typed_data:
dependency: transitive dependency: transitive
description: description:
@@ -421,18 +405,18 @@ packages:
dependency: transitive dependency: transitive
description: description:
name: vm_service name: vm_service
sha256: "45caa6c5917fa127b5dbcfbd1fa60b14e583afdc08bfc96dda38886ca252eb60" sha256: ddfa8d30d89985b96407efce8acbdd124701f96741f2d981ca860662f1c0dc02
url: "https://pub.dev" url: "https://pub.dev"
source: hosted source: hosted
version: "15.0.2" version: "15.0.0"
watcher: watcher:
dependency: transitive dependency: transitive
description: description:
name: watcher name: watcher
sha256: "1398c9f081a753f9226febe8900fce8f7d0a67163334e1c94a2438339d79d635" sha256: "69da27e49efa56a15f8afe8f4438c4ec02eff0a117df1b22ea4aad194fe1c104"
url: "https://pub.dev" url: "https://pub.dev"
source: hosted source: hosted
version: "1.2.1" version: "1.1.1"
web: web:
dependency: transitive dependency: transitive
description: description:
@@ -469,10 +453,10 @@ packages:
dependency: "direct main" dependency: "direct main"
description: description:
name: xml name: xml
sha256: "971043b3a0d3da28727e40ed3e0b5d18b742fa5a68665cca88e74b7876d5e025" sha256: b015a8ad1c488f66851d762d3090a21c600e479dc75e68328c52774040cf9226
url: "https://pub.dev" url: "https://pub.dev"
source: hosted source: hosted
version: "6.6.1" version: "6.5.0"
yaml: yaml:
dependency: transitive dependency: transitive
description: description:
@@ -482,4 +466,4 @@ packages:
source: hosted source: hosted
version: "3.1.3" version: "3.1.3"
sdks: sdks:
dart: ">=3.10.1 <4.0.0" dart: ">=3.7.0 <4.0.0"

View File

@@ -4,31 +4,24 @@ version: 1.0.0
homepage: https://git.pvv.ntnu.no/oysteikt/jadb homepage: https://git.pvv.ntnu.no/oysteikt/jadb
environment: environment:
sdk: '^3.9.0' sdk: '>=3.2.0 <4.0.0'
dependencies: dependencies:
args: ^2.7.0 args: ^2.7.0
collection: ^1.19.0 collection: ^1.19.0
csv: ^7.1.0 csv: ^6.0.0
equatable: ^2.0.0 equatable: ^2.0.0
path: ^1.9.1
sqflite_common: ^2.5.0 sqflite_common: ^2.5.0
sqflite_common_ffi: ^2.3.0 sqflite_common_ffi: ^2.3.0
sqlite3: ^3.1.6
xml: ^6.5.0 xml: ^6.5.0
dev_dependencies: dev_dependencies:
lints: ^6.0.0 lints: ^5.0.0
test: ^1.25.15 test: ^1.25.15
executables: executables:
jadb: jadb jadb: jadb
hooks:
user_defines:
sqlite3:
source: system
topics: topics:
- database - database
- dictionary - dictionary

View File

@@ -3,7 +3,7 @@ import 'package:jadb/const_data/kanji_grades.dart';
import 'package:test/test.dart'; import 'package:test/test.dart';
void main() { void main() {
test('Assert 2136 kanji in jouyou set', () { test("Assert 2136 kanji in jouyou set", () {
expect(jouyouKanjiByGrades.values.flattenedToSet.length, 2136); expect(JOUYOU_KANJI_BY_GRADES.values.flattenedToSet.length, 2136);
}); });
} }

View File

@@ -1,20 +1,30 @@
import 'dart:ffi';
import 'dart:io';
import 'package:jadb/models/create_empty_db.dart'; import 'package:jadb/models/create_empty_db.dart';
import 'package:jadb/search.dart'; import 'package:jadb/search.dart';
import 'package:sqflite_common_ffi/sqflite_ffi.dart'; import 'package:sqflite_common_ffi/sqflite_ffi.dart';
// import 'package:sqlite3/open.dart';
import 'package:test/test.dart'; import 'package:test/test.dart';
import 'package:sqlite3/open.dart';
Future<DatabaseExecutor> setupInMemoryDatabase() async { Future<DatabaseExecutor> setup_inmemory_database() async {
final dbConnection = await createDatabaseFactoryFfi().openDatabase( final libsqlitePath = Platform.environment['LIBSQLITE_PATH'];
':memory:',
);
return dbConnection; if (libsqlitePath == null) {
throw Exception("LIBSQLITE_PATH is not set");
}
final db_connection = await createDatabaseFactoryFfi(
ffiInit: () =>
open.overrideForAll(() => DynamicLibrary.open(libsqlitePath)),
).openDatabase(':memory:');
return db_connection;
} }
void main() { void main() {
test('Create empty db', () async { test("Create empty db", () async {
final db = await setupInMemoryDatabase(); final db = await setup_inmemory_database();
await createEmptyDb(db); await createEmptyDb(db);

View File

@@ -4,26 +4,29 @@ import 'package:test/test.dart';
import 'setup_database_connection.dart'; import 'setup_database_connection.dart';
void main() { void main() {
test('Filter kanji', () async { test("Filter kanji", () async {
final connection = await setupDatabaseConnection(); final connection = await setup_database_connection();
final result = await connection.filterKanji([ final result = await connection.filterKanji(
'a', [
'b', "a",
'c', "b",
'', "c",
'', "",
'', "",
'', "",
'', "",
'', "",
'.', "",
'!', ".",
'@', "!",
';', "@",
'', ";",
], deduplicate: false); "",
],
deduplicate: false,
);
expect(result.join(), '漢字地字'); expect(result.join(), "漢字地字");
}); });
} }

View File

@@ -5,17 +5,17 @@ import 'package:test/test.dart';
import 'setup_database_connection.dart'; import 'setup_database_connection.dart';
void main() { void main() {
test('Search a kanji', () async { test("Search a kanji", () async {
final connection = await setupDatabaseConnection(); final connection = await setup_database_connection();
final result = await connection.jadbSearchKanji(''); final result = await connection.jadbSearchKanji('');
expect(result, isNotNull); expect(result, isNotNull);
}); });
group('Search all jouyou kanji', () { group("Search all jouyou kanji", () {
jouyouKanjiByGrades.forEach((grade, characters) { JOUYOU_KANJI_BY_GRADES.forEach((grade, characters) {
test('Search all kanji in grade $grade', () async { test("Search all kanji in grade $grade", () async {
final connection = await setupDatabaseConnection(); final connection = await setup_database_connection();
for (final character in characters) { for (final character in characters) {
final result = await connection.jadbSearchKanji(character); final result = await connection.jadbSearchKanji(character);

View File

@@ -1,257 +0,0 @@
import 'package:jadb/models/common/jlpt_level.dart';
import 'package:jadb/models/word_search/word_search_match_span.dart';
import 'package:jadb/models/word_search/word_search_result.dart';
import 'package:jadb/models/word_search/word_search_ruby.dart';
import 'package:jadb/models/word_search/word_search_sense.dart';
import 'package:jadb/models/word_search/word_search_sources.dart';
import 'package:test/test.dart';
void main() {
test('Infer match whole word', () {
final wordSearchResult = WordSearchResult(
entryId: 0,
score: 0,
isCommon: false,
jlptLevel: JlptLevel.none,
kanjiInfo: {},
readingInfo: {},
japanese: [WordSearchRuby(base: '仮名')],
senses: [],
sources: WordSearchSources.empty(),
);
wordSearchResult.inferMatchSpans('仮名');
expect(wordSearchResult.matchSpans, [
WordSearchMatchSpan(
spanType: WordSearchMatchSpanType.kanji,
start: 0,
end: 2,
index: 0,
),
]);
});
test('Infer match part of word', () {
final wordSearchResult = WordSearchResult(
entryId: 0,
score: 0,
isCommon: false,
jlptLevel: JlptLevel.none,
kanjiInfo: {},
readingInfo: {},
japanese: [WordSearchRuby(base: '仮名')],
senses: [],
sources: WordSearchSources.empty(),
);
wordSearchResult.inferMatchSpans('');
expect(wordSearchResult.matchSpans, [
WordSearchMatchSpan(
spanType: WordSearchMatchSpanType.kanji,
start: 0,
end: 1,
index: 0,
),
]);
});
test('Infer match in middle of word', () {
final wordSearchResult = WordSearchResult(
entryId: 0,
score: 0,
isCommon: false,
jlptLevel: JlptLevel.none,
kanjiInfo: {},
readingInfo: {},
japanese: [WordSearchRuby(base: 'ありがとう')],
senses: [],
sources: WordSearchSources.empty(),
);
wordSearchResult.inferMatchSpans('りがと');
expect(wordSearchResult.matchSpans, [
WordSearchMatchSpan(
spanType: WordSearchMatchSpanType.kanji,
start: 1,
end: 4,
index: 0,
),
]);
});
test('Infer match in furigana', () {
final wordSearchResult = WordSearchResult(
entryId: 0,
score: 0,
isCommon: false,
jlptLevel: JlptLevel.none,
kanjiInfo: {},
readingInfo: {},
japanese: [WordSearchRuby(base: '仮名', furigana: 'かな')],
senses: [],
sources: WordSearchSources.empty(),
);
wordSearchResult.inferMatchSpans('かな');
expect(wordSearchResult.matchSpans, [
WordSearchMatchSpan(
spanType: WordSearchMatchSpanType.kana,
start: 0,
end: 2,
index: 0,
),
]);
});
test('Infer match in sense', () {
final wordSearchResult = WordSearchResult(
entryId: 0,
score: 0,
isCommon: false,
jlptLevel: JlptLevel.none,
kanjiInfo: {},
readingInfo: {},
japanese: [WordSearchRuby(base: '仮名')],
senses: [
WordSearchSense(
antonyms: [],
dialects: [],
englishDefinitions: ['kana'],
fields: [],
info: [],
languageSource: [],
misc: [],
partsOfSpeech: [],
restrictedToKanji: [],
restrictedToReading: [],
seeAlso: [],
),
],
sources: WordSearchSources.empty(),
);
wordSearchResult.inferMatchSpans('kana');
expect(wordSearchResult.matchSpans, [
WordSearchMatchSpan(
spanType: WordSearchMatchSpanType.sense,
start: 0,
end: 4,
index: 0,
),
]);
});
test('Infer multiple matches', () {
final wordSearchResult = WordSearchResult(
entryId: 0,
score: 0,
isCommon: false,
jlptLevel: JlptLevel.none,
kanjiInfo: {},
readingInfo: {},
japanese: [WordSearchRuby(base: '仮名', furigana: 'かな')],
senses: [
WordSearchSense(
antonyms: [],
dialects: [],
englishDefinitions: ['kana', 'the kana'],
fields: [],
info: [],
languageSource: [],
misc: [],
partsOfSpeech: [],
restrictedToKanji: [],
restrictedToReading: [],
seeAlso: [],
),
],
sources: WordSearchSources.empty(),
);
wordSearchResult.inferMatchSpans('kana');
expect(wordSearchResult.matchSpans, [
WordSearchMatchSpan(
spanType: WordSearchMatchSpanType.sense,
start: 0,
end: 4,
index: 0,
),
WordSearchMatchSpan(
spanType: WordSearchMatchSpanType.sense,
start: 4,
end: 8,
index: 0,
subIndex: 1,
),
]);
});
test('Infer match with no matches', () {
final wordSearchResult = WordSearchResult(
entryId: 0,
score: 0,
isCommon: false,
jlptLevel: JlptLevel.none,
kanjiInfo: {},
readingInfo: {},
japanese: [WordSearchRuby(base: '仮名', furigana: 'かな')],
senses: [
WordSearchSense(
antonyms: [],
dialects: [],
englishDefinitions: ['kana'],
fields: [],
info: [],
languageSource: [],
misc: [],
partsOfSpeech: [],
restrictedToKanji: [],
restrictedToReading: [],
seeAlso: [],
),
],
sources: WordSearchSources.empty(),
);
wordSearchResult.inferMatchSpans('xyz');
expect(wordSearchResult.matchSpans, isEmpty);
});
test('Infer multiple matches of same substring', () {
final wordSearchResult = WordSearchResult(
entryId: 0,
score: 0,
isCommon: false,
jlptLevel: JlptLevel.none,
kanjiInfo: {},
readingInfo: {},
japanese: [WordSearchRuby(base: 'ああ')],
senses: [],
sources: WordSearchSources.empty(),
);
wordSearchResult.inferMatchSpans('');
expect(wordSearchResult.matchSpans, [
WordSearchMatchSpan(
spanType: WordSearchMatchSpanType.kanji,
start: 0,
end: 1,
index: 0,
),
WordSearchMatchSpan(
spanType: WordSearchMatchSpanType.kanji,
start: 1,
end: 2,
index: 0,
),
]);
});
}

View File

@@ -3,22 +3,22 @@ import 'dart:io';
import 'package:jadb/_data_ingestion/open_local_db.dart'; import 'package:jadb/_data_ingestion/open_local_db.dart';
import 'package:sqflite_common/sqlite_api.dart'; import 'package:sqflite_common/sqlite_api.dart';
Future<Database> setupDatabaseConnection() async { Future<Database> setup_database_connection() async {
final libSqlitePath = Platform.environment['LIBSQLITE_PATH']; final lib_sqlite_path = Platform.environment['LIBSQLITE_PATH'];
final jadbPath = Platform.environment['JADB_PATH']; final jadb_path = Platform.environment['JADB_PATH'];
if (libSqlitePath == null) { if (lib_sqlite_path == null) {
throw Exception('LIBSQLITE_PATH is not set'); throw Exception("LIBSQLITE_PATH is not set");
} }
if (jadbPath == null) { if (jadb_path == null) {
throw Exception('JADB_PATH is not set'); throw Exception("JADB_PATH is not set");
} }
final dbConnection = await openLocalDb( final db_connection = await openLocalDb(
libsqlitePath: libSqlitePath, libsqlitePath: lib_sqlite_path,
jadbPath: jadbPath, jadbPath: jadb_path,
); );
return dbConnection; return db_connection;
} }

View File

@@ -4,59 +4,29 @@ import 'package:test/test.dart';
import 'setup_database_connection.dart'; import 'setup_database_connection.dart';
void main() { void main() {
test('Search a word - english - auto', () async { test("Search a word", () async {
final connection = await setupDatabaseConnection(); final connection = await setup_database_connection();
final result = await connection.jadbSearchWord('kana'); final result = await connection.jadbSearchWord("kana");
expect(result, isNotNull); expect(result, isNotNull);
}); });
test('Get word search count - english - auto', () async { test("Get a word by id", () async {
final connection = await setupDatabaseConnection(); final connection = await setup_database_connection();
final result = await connection.jadbSearchWordCount('kana');
expect(result, isNotNull);
});
test('Search a word - japanese kana - auto', () async {
final connection = await setupDatabaseConnection();
final result = await connection.jadbSearchWord('かな');
expect(result, isNotNull);
});
test('Get word search count - japanese kana - auto', () async {
final connection = await setupDatabaseConnection();
final result = await connection.jadbSearchWordCount('かな');
expect(result, isNotNull);
});
test('Search a word - japanese kanji - auto', () async {
final connection = await setupDatabaseConnection();
final result = await connection.jadbSearchWord('仮名');
expect(result, isNotNull);
});
test('Get word search count - japanese kanji - auto', () async {
final connection = await setupDatabaseConnection();
final result = await connection.jadbSearchWordCount('仮名');
expect(result, isNotNull);
});
test('Get a word by id', () async {
final connection = await setupDatabaseConnection();
final result = await connection.jadbGetWordById(1577090); final result = await connection.jadbGetWordById(1577090);
expect(result, isNotNull); expect(result, isNotNull);
}); });
test( test(
'Serialize all words', "Serialize all words",
() async { () async {
final connection = await setupDatabaseConnection(); final connection = await setup_database_connection();
// Test serializing all words // Test serializing all words
for (final letter in 'aiueoksthnmyrw'.split('')) { for (final letter in "aiueoksthnmyrw".split("")) {
await connection.jadbSearchWord(letter); await connection.jadbSearchWord(letter);
} }
}, },
timeout: Timeout.factor(100), timeout: Timeout.factor(100),
skip: 'Very slow test', skip: "Very slow test",
); );
} }

View File

@@ -2,65 +2,65 @@ import 'package:jadb/util/romaji_transliteration.dart';
import 'package:test/test.dart'; import 'package:test/test.dart';
void main() { void main() {
group('Romaji -> Hiragana', () { group("Romaji -> Hiragana", () {
test('Basic test', () { test("Basic test", () {
final result = transliterateLatinToHiragana('katamari'); final result = transliterateLatinToHiragana("katamari");
expect(result, 'かたまり'); expect(result, "かたまり");
}); });
test('Basic test with diacritics', () { test("Basic test with diacritics", () {
final result = transliterateLatinToHiragana('gadamari'); final result = transliterateLatinToHiragana("gadamari");
expect(result, 'がだまり'); expect(result, "がだまり");
}); });
test('wi and we', () { test("wi and we", () {
final result = transliterateLatinToHiragana('wiwe'); final result = transliterateLatinToHiragana("wiwe");
expect(result, 'うぃうぇ'); expect(result, "うぃうぇ");
}); });
test('nb = mb', () { test("nb = mb", () {
final result = transliterateLatinToHiragana('kanpai'); final result = transliterateLatinToHiragana("kanpai");
expect(result, 'かんぱい'); expect(result, "かんぱい");
final result2 = transliterateLatinToHiragana('kampai'); final result2 = transliterateLatinToHiragana("kampai");
expect(result2, 'かんぱい'); expect(result2, "かんぱい");
}); });
test('Double n', () { test("Double n", () {
final result = transliterateLatinToHiragana('konnichiha'); final result = transliterateLatinToHiragana("konnichiha");
expect(result, 'こんにちは'); expect(result, "こんにちは");
}); });
test('Double consonant', () { test("Double consonant", () {
final result = transliterateLatinToHiragana('kappa'); final result = transliterateLatinToHiragana("kappa");
expect(result, 'かっぱ'); expect(result, "かっぱ");
}); });
}); });
group('Hiragana -> Romaji', () { group("Hiragana -> Romaji", () {
test('Basic test', () { test("Basic test", () {
final result = transliterateHiraganaToLatin('かたまり'); final result = transliterateHiraganaToLatin("かたまり");
expect(result, 'katamari'); expect(result, "katamari");
}); });
test('Basic test with diacritics', () { test("Basic test with diacritics", () {
final result = transliterateHiraganaToLatin('がだまり'); final result = transliterateHiraganaToLatin("がだまり");
expect(result, 'gadamari'); expect(result, "gadamari");
}); });
test('whi and whe', () { test("whi and whe", () {
final result = transliterateHiraganaToLatin('うぃうぇ'); final result = transliterateHiraganaToLatin("うぃうぇ");
expect(result, 'whiwhe'); expect(result, "whiwhe");
}); });
test('Double n', () { test("Double n", () {
final result = transliterateHiraganaToLatin('こんにちは'); final result = transliterateHiraganaToLatin("こんにちは");
expect(result, 'konnichiha'); expect(result, "konnichiha");
}); });
test('Double consonant', () { test("Double consonant", () {
final result = transliterateHiraganaToLatin('かっぱ'); final result = transliterateHiraganaToLatin("かっぱ");
expect(result, 'kappa'); expect(result, "kappa");
}); });
}); });
} }