1 Commits

Author SHA1 Message Date
0ca55f065f WIP: kanjidic: split news ichi gai nf into separate tables 2025-06-25 20:50:12 +02:00
79 changed files with 1821 additions and 2509 deletions

View File

@@ -1,38 +0,0 @@
name: "Build database"
on:
pull_request:
push:
jobs:
evals:
runs-on: debian-latest
steps:
- uses: actions/checkout@v6
- name: Install sudo
run: apt-get update && apt-get -y install sudo
- name: Install nix
uses: https://github.com/cachix/install-nix-action@v31
- name: Configure nix
run: echo -e "show-trace = true\nmax-jobs = auto\ntrusted-users = root\nexperimental-features = nix-command flakes\nbuild-users-group =" > /etc/nix/nix.conf
- name: Update database inputs
run: |
nix flake update jmdict-src
nix flake update jmdict-with-examples-src
nix flake update radkfile-src
nix flake update kanjidic2-src
- name: Build database
run: nix build .#database -L
- name: Upload database as artifact
uses: actions/upload-artifact@v3
with:
name: jadb-${{ gitea.sha }}.zip
path: result/jadb.sqlite
if-no-files-found: error
retention-days: 15
# Already compressed
compression: 0

View File

@@ -1,31 +0,0 @@
name: "Run tests"
on:
pull_request:
push:
jobs:
evals:
runs-on: debian-latest
steps:
- uses: actions/checkout@v6
- name: Install sudo
run: apt-get update && apt-get -y install sudo
- name: Install nix
uses: https://github.com/cachix/install-nix-action@v31
- name: Configure nix
run: echo -e "show-trace = true\nmax-jobs = auto\ntrusted-users = root\nexperimental-features = nix-command flakes\nbuild-users-group =" > /etc/nix/nix.conf
- name: Update database inputs
run: |
nix flake update jmdict-src
nix flake update jmdict-with-examples-src
nix flake update radkfile-src
nix flake update kanjidic2-src
- name: Build database
run: nix build .#database -L
- name: Run tests
run: nix develop .# --command dart test

View File

@@ -16,26 +16,3 @@ Note that while the license for the code is MIT, the data has various licenses.
| **Tanos JLPT levels:** | https://www.tanos.co.uk/jlpt/ |
| **Kangxi Radicals:** | https://ctext.org/kangxi-zidian |
## Implementation details
### Word search
The word search procedure is currently split into 3 parts:
1. **Entry ID query**:
Use a complex query with various scoring factors to try to get list of
database ids pointing at dictionary entries, sorted by how likely we think this
word is the word that the caller is looking for. The output here is a `List<int>`
2. **Data Query**:
Takes the entry id list from the last search, and performs all queries needed to retrieve
all the dictionary data for those IDs. The result is a struct with a bunch of flattened lists
with data for all the dictionary entries. These lists are sorted by the order that the ids
were provided.
3. **Regrouping**:
Takes the flattened data, and regroups the items into structs with a more "hierarchical" structure.
All data tagged with the same ID will end up in the same struct. Returns a list of these structs.

View File

@@ -1,41 +0,0 @@
# This file configures the analyzer, which statically analyzes Dart code to
# check for errors, warnings, and lints.
#
# The issues identified by the analyzer are surfaced in the UI of Dart-enabled
# IDEs (https://dart.dev/tools#ides-and-editors). The analyzer can also be
# invoked from the command line by running `flutter analyze`.
# The following line activates a set of recommended lints for Flutter apps,
# packages, and plugins designed to encourage good coding practices.
include:
- package:lints/recommended.yaml
linter:
# The lint rules applied to this project can be customized in the
# section below to disable rules from the `package:flutter_lints/flutter.yaml`
# included above or to enable additional rules. A list of all available lints
# and their documentation is published at https://dart.dev/lints.
#
# Instead of disabling a lint rule for the entire project in the
# section below, it can also be suppressed for a single line of code
# or a specific dart file by using the `// ignore: name_of_lint` and
# `// ignore_for_file: name_of_lint` syntax on the line or in the file
# producing the lint.
rules:
always_declare_return_types: true
annotate_redeclares: true
avoid_print: false
avoid_setters_without_getters: true
avoid_slow_async_io: true
directives_ordering: true
eol_at_end_of_file: true
prefer_const_declarations: true
prefer_contains: true
prefer_final_fields: true
prefer_final_locals: true
prefer_single_quotes: true
use_key_in_widget_constructors: true
use_null_aware_elements: true
# Additional information about this file can be found at
# https://dart.dev/guides/language/analysis-options

View File

@@ -9,7 +9,7 @@ import 'package:jadb/cli/commands/query_word.dart';
Future<void> main(List<String> args) async {
final runner = CommandRunner(
'jadb',
'CLI tool to help creating and testing the jadb database',
"CLI tool to help creating and testing the jadb database",
);
runner.addCommand(CreateDb());

18
flake.lock generated
View File

@@ -3,7 +3,7 @@
"jmdict-src": {
"flake": false,
"locked": {
"narHash": "sha256-lh46uougUzBrRhhwa7cOb32j5Jt9/RjBUhlVjwVzsII=",
"narHash": "sha256-84P7r/fFlBnawy6yChrD9WMHmOWcEGWUmoK70N4rdGQ=",
"type": "file",
"url": "http://ftp.edrdg.org/pub/Nihongo/JMdict_e.gz"
},
@@ -15,7 +15,7 @@
"jmdict-with-examples-src": {
"flake": false,
"locked": {
"narHash": "sha256-5oS2xDyetbuSM6ax3LUjYA3N60x+D3Hg41HEXGFMqLQ=",
"narHash": "sha256-PM0sv7VcsCya2Ek02CI7hVwB3Jawn6bICSI+dsJK0yo=",
"type": "file",
"url": "http://ftp.edrdg.org/pub/Nihongo/JMdict_e_examp.gz"
},
@@ -27,7 +27,7 @@
"kanjidic2-src": {
"flake": false,
"locked": {
"narHash": "sha256-orSeQqSxhn9TtX3anYtbiMEm7nFkuomGnIKoVIUR2CM=",
"narHash": "sha256-Lc0wUPpuDKuMDv2t87//w3z20RX8SMJI2iIRtUJ8fn0=",
"type": "file",
"url": "https://www.edrdg.org/kanjidic/kanjidic2.xml.gz"
},
@@ -38,11 +38,11 @@
},
"nixpkgs": {
"locked": {
"lastModified": 1771848320,
"narHash": "sha256-0MAd+0mun3K/Ns8JATeHT1sX28faLII5hVLq0L3BdZU=",
"lastModified": 1746904237,
"narHash": "sha256-3e+AVBczosP5dCLQmMoMEogM57gmZ2qrVSrmq9aResQ=",
"owner": "NixOS",
"repo": "nixpkgs",
"rev": "2fc6539b481e1d2569f25f8799236694180c0993",
"rev": "d89fc19e405cb2d55ce7cc114356846a0ee5e956",
"type": "github"
},
"original": {
@@ -54,13 +54,13 @@
"radkfile-src": {
"flake": false,
"locked": {
"narHash": "sha256-DHpMUE2Umje8PbzXUCS6pHZeXQ5+WTxbjSkGU3erDHQ=",
"narHash": "sha256-rO2z5GPt3g6osZOlpyWysmIbRV2Gw4AR4XvngVTHNpk=",
"type": "file",
"url": "http://ftp.edrdg.org/pub/Nihongo/radkfile.gz"
"url": "http://ftp.usf.edu/pub/ftp.monash.edu.au/pub/nihongo/radkfile.gz"
},
"original": {
"type": "file",
"url": "http://ftp.edrdg.org/pub/Nihongo/radkfile.gz"
"url": "http://ftp.usf.edu/pub/ftp.monash.edu.au/pub/nihongo/radkfile.gz"
}
},
"root": {

View File

@@ -16,7 +16,7 @@
};
radkfile-src = {
url = "http://ftp.edrdg.org/pub/Nihongo/radkfile.gz";
url = "http://ftp.usf.edu/pub/ftp.monash.edu.au/pub/nihongo/radkfile.gz";
flake = false;
};
@@ -83,7 +83,7 @@
sqlite-interactive
sqlite-analyzer
sqlite-web
# sqlint
sqlint
sqlfluff
];
env = {
@@ -109,14 +109,10 @@
in !(lib.any (b: b) [
(!(lib.cleanSourceFilter path type))
(baseName == ".github" && type == "directory")
(baseName == ".gitea" && type == "directory")
(baseName == "nix" && type == "directory")
(baseName == ".envrc" && type == "regular")
(baseName == "flake.lock" && type == "regular")
(baseName == "flake.nix" && type == "regular")
(baseName == ".sqlfluff" && type == "regular")
])) ./.;
in forAllSystems (system: pkgs: {
@@ -127,7 +123,7 @@
'';
jmdict = pkgs.callPackage ./nix/jmdict.nix {
inherit jmdict-src jmdict-with-examples-src edrdgMetadata;
inherit jmdict-src jmdict-with-examples-src edrdgMetadata;
};
radkfile = pkgs.callPackage ./nix/radkfile.nix {

View File

@@ -16,15 +16,14 @@ abstract class Element extends SQLWritable {
this.nf,
});
@override
Map<String, Object?> get sqlValue => {
'reading': reading,
'news': news,
'ichi': ichi,
'spec': spec,
'gai': gai,
'nf': nf,
};
'reading': reading,
'news': news,
'ichi': ichi,
'spec': spec,
'gai': gai,
'nf': nf,
};
}
class KanjiElement extends Element {
@@ -34,19 +33,26 @@ class KanjiElement extends Element {
KanjiElement({
this.info = const [],
required this.orderNum,
required super.reading,
super.news,
super.ichi,
super.spec,
super.gai,
super.nf,
});
required String reading,
int? news,
int? ichi,
int? spec,
int? gai,
int? nf,
}) : super(
reading: reading,
news: news,
ichi: ichi,
spec: spec,
gai: gai,
nf: nf,
);
@override
Map<String, Object?> get sqlValue => {
...super.sqlValue,
'orderNum': orderNum,
};
...super.sqlValue,
'orderNum': orderNum,
};
}
class ReadingElement extends Element {
@@ -60,20 +66,27 @@ class ReadingElement extends Element {
required this.readingDoesNotMatchKanji,
this.info = const [],
this.restrictions = const [],
required super.reading,
super.news,
super.ichi,
super.spec,
super.gai,
super.nf,
});
required String reading,
int? news,
int? ichi,
int? spec,
int? gai,
int? nf,
}) : super(
reading: reading,
news: news,
ichi: ichi,
spec: spec,
gai: gai,
nf: nf,
);
@override
Map<String, Object?> get sqlValue => {
...super.sqlValue,
'orderNum': orderNum,
'readingDoesNotMatchKanji': readingDoesNotMatchKanji,
};
...super.sqlValue,
'orderNum': orderNum,
'readingDoesNotMatchKanji': readingDoesNotMatchKanji,
};
}
class LanguageSource extends SQLWritable {
@@ -91,11 +104,11 @@ class LanguageSource extends SQLWritable {
@override
Map<String, Object?> get sqlValue => {
'language': language,
'phrase': phrase,
'fullyDescribesSense': fullyDescribesSense,
'constructedFromSmallerWords': constructedFromSmallerWords,
};
'language': language,
'phrase': phrase,
'fullyDescribesSense': fullyDescribesSense,
'constructedFromSmallerWords': constructedFromSmallerWords,
};
}
class Glossary extends SQLWritable {
@@ -103,41 +116,48 @@ class Glossary extends SQLWritable {
final String phrase;
final String? type;
const Glossary({required this.language, required this.phrase, this.type});
const Glossary({
required this.language,
required this.phrase,
this.type,
});
@override
Map<String, Object?> get sqlValue => {
'language': language,
'phrase': phrase,
'type': type,
};
'language': language,
'phrase': phrase,
'type': type,
};
}
final kanaRegex = RegExp(
r'^[\p{Script=Katakana}\p{Script=Hiragana}ー]+$',
unicode: true,
);
final kanaRegex =
RegExp(r'^[\p{Script=Katakana}\p{Script=Hiragana}ー]+$', unicode: true);
class XRefParts {
final String? kanjiRef;
final String? readingRef;
final int? senseOrderNum;
const XRefParts({this.kanjiRef, this.readingRef, this.senseOrderNum})
: assert(kanjiRef != null || readingRef != null);
const XRefParts({
this.kanjiRef,
this.readingRef,
this.senseOrderNum,
}) : assert(kanjiRef != null || readingRef != null);
Map<String, Object?> toJson() => {
'kanjiRef': kanjiRef,
'readingRef': readingRef,
'senseOrderNum': senseOrderNum,
};
'kanjiRef': kanjiRef,
'readingRef': readingRef,
'senseOrderNum': senseOrderNum,
};
}
class XRef {
final String entryId;
final String reading;
const XRef({required this.entryId, required this.reading});
const XRef({
required this.entryId,
required this.reading,
});
}
class Sense extends SQLWritable {
@@ -173,9 +193,9 @@ class Sense extends SQLWritable {
@override
Map<String, Object?> get sqlValue => {
'senseId': senseId,
'orderNum': orderNum,
};
'senseId': senseId,
'orderNum': orderNum,
};
bool get isEmpty =>
antonyms.isEmpty &&
@@ -204,6 +224,5 @@ class Entry extends SQLWritable {
required this.senses,
});
@override
Map<String, Object?> get sqlValue => {'entryId': entryId};
}

View File

@@ -18,20 +18,18 @@ ResolvedXref resolveXref(
XRefParts xref,
) {
List<Entry> candidateEntries = switch ((xref.kanjiRef, xref.readingRef)) {
(null, null) => throw Exception(
'Xref $xref has no kanji or reading reference',
),
(final String k, null) => entriesByKanji[k]!.toList(),
(null, final String r) => entriesByReading[r]!.toList(),
(final String k, final String r) =>
(null, null) =>
throw Exception('Xref $xref has no kanji or reading reference'),
(String k, null) => entriesByKanji[k]!.toList(),
(null, String r) => entriesByReading[r]!.toList(),
(String k, String r) =>
entriesByKanji[k]!.intersection(entriesByReading[r]!).toList(),
};
// Filter out entries that don't have the number of senses specified in the xref
if (xref.senseOrderNum != null) {
candidateEntries.retainWhere(
(entry) => entry.senses.length >= xref.senseOrderNum!,
);
candidateEntries
.retainWhere((entry) => entry.senses.length >= xref.senseOrderNum!);
}
// If the xref has a reading ref but no kanji ref, and there are multiple
@@ -40,9 +38,8 @@ ResolvedXref resolveXref(
if (xref.kanjiRef == null &&
xref.readingRef != null &&
candidateEntries.length > 1) {
final candidatesWithEmptyKanji = candidateEntries
.where((entry) => entry.kanji.isEmpty)
.toList();
final candidatesWithEmptyKanji =
candidateEntries.where((entry) => entry.kanji.length == 0).toList();
if (candidatesWithEmptyKanji.isNotEmpty) {
candidateEntries = candidatesWithEmptyKanji;
@@ -53,7 +50,7 @@ ResolvedXref resolveXref(
// entry in case there are multiple candidates left.
candidateEntries.sortBy<num>((entry) => entry.senses.length);
if (candidateEntries.isEmpty) {
if (candidateEntries.length == 0) {
throw Exception(
'SKIPPING: Xref $xref has ${candidateEntries.length} entries, '
'kanjiRef: ${xref.kanjiRef}, readingRef: ${xref.readingRef}, '
@@ -83,14 +80,20 @@ Future<void> seedJMDictData(List<Entry> entries, Database db) async {
elementId++;
b.insert(
JMdictTableNames.kanjiElement,
k.sqlValue..addAll({'entryId': e.entryId, 'elementId': elementId}),
k.sqlValue..addAll({
'entryId': e.entryId,
'elementId': elementId,
}),
);
for (final i in k.info) {
b.insert(JMdictTableNames.kanjiInfo, {
'elementId': elementId,
'info': i,
});
b.insert(
JMdictTableNames.kanjiInfo,
{
'elementId': elementId,
'info': i,
},
);
}
}
@@ -98,20 +101,29 @@ Future<void> seedJMDictData(List<Entry> entries, Database db) async {
elementId++;
b.insert(
JMdictTableNames.readingElement,
r.sqlValue..addAll({'entryId': e.entryId, 'elementId': elementId}),
r.sqlValue..addAll({
'entryId': e.entryId,
'elementId': elementId,
}),
);
for (final i in r.info) {
b.insert(JMdictTableNames.readingInfo, {
'elementId': elementId,
'info': i,
});
b.insert(
JMdictTableNames.readingInfo,
{
'elementId': elementId,
'info': i,
},
);
}
for (final res in r.restrictions) {
b.insert(JMdictTableNames.readingRestriction, {
'elementId': elementId,
'restriction': res,
});
b.insert(
JMdictTableNames.readingRestriction,
{
'elementId': elementId,
'restriction': res,
},
);
}
}
}
@@ -124,20 +136,16 @@ Future<void> seedJMDictData(List<Entry> entries, Database db) async {
for (final e in entries) {
for (final s in e.senses) {
b.insert(
JMdictTableNames.sense,
s.sqlValue..addAll({'entryId': e.entryId}),
);
JMdictTableNames.sense, s.sqlValue..addAll({'entryId': e.entryId}));
for (final d in s.dialects) {
b.insert(JMdictTableNames.senseDialect, {
'senseId': s.senseId,
'dialect': d,
});
b.insert(
JMdictTableNames.senseDialect,
{'senseId': s.senseId, 'dialect': d},
);
}
for (final f in s.fields) {
b.insert(JMdictTableNames.senseField, {
'senseId': s.senseId,
'field': f,
});
b.insert(
JMdictTableNames.senseField, {'senseId': s.senseId, 'field': f});
}
for (final i in s.info) {
b.insert(JMdictTableNames.senseInfo, {'senseId': s.senseId, 'info': i});
@@ -149,18 +157,16 @@ Future<void> seedJMDictData(List<Entry> entries, Database db) async {
b.insert(JMdictTableNames.sensePOS, {'senseId': s.senseId, 'pos': p});
}
for (final rk in s.restrictedToKanji) {
b.insert(JMdictTableNames.senseRestrictedToKanji, {
'entryId': e.entryId,
'senseId': s.senseId,
'kanji': rk,
});
b.insert(
JMdictTableNames.senseRestrictedToKanji,
{'entryId': e.entryId, 'senseId': s.senseId, 'kanji': rk},
);
}
for (final rr in s.restrictedToReading) {
b.insert(JMdictTableNames.senseRestrictedToReading, {
'entryId': e.entryId,
'senseId': s.senseId,
'reading': rr,
});
b.insert(
JMdictTableNames.senseRestrictedToReading,
{'entryId': e.entryId, 'senseId': s.senseId, 'reading': rr},
);
}
for (final ls in s.languageSource) {
b.insert(
@@ -180,7 +186,7 @@ Future<void> seedJMDictData(List<Entry> entries, Database db) async {
await b.commit(noResult: true);
print(' [JMdict] Building xref trees');
final SplayTreeMap<String, Set<Entry>> entriesByKanji = SplayTreeMap();
SplayTreeMap<String, Set<Entry>> entriesByKanji = SplayTreeMap();
for (final entry in entries) {
for (final kanji in entry.kanji) {
@@ -191,7 +197,7 @@ Future<void> seedJMDictData(List<Entry> entries, Database db) async {
}
}
}
final SplayTreeMap<String, Set<Entry>> entriesByReading = SplayTreeMap();
SplayTreeMap<String, Set<Entry>> entriesByReading = SplayTreeMap();
for (final entry in entries) {
for (final reading in entry.readings) {
if (entriesByReading.containsKey(reading.reading)) {
@@ -214,14 +220,17 @@ Future<void> seedJMDictData(List<Entry> entries, Database db) async {
xref,
);
b.insert(JMdictTableNames.senseSeeAlso, {
'senseId': s.senseId,
'xrefEntryId': resolvedEntry.entry.entryId,
'seeAlsoKanji': xref.kanjiRef,
'seeAlsoReading': xref.readingRef,
'seeAlsoSense': xref.senseOrderNum,
'ambiguous': resolvedEntry.ambiguous,
});
b.insert(
JMdictTableNames.senseSeeAlso,
{
'senseId': s.senseId,
'xrefEntryId': resolvedEntry.entry.entryId,
'seeAlsoKanji': xref.kanjiRef,
'seeAlsoReading': xref.readingRef,
'seeAlsoSense': xref.senseOrderNum,
'ambiguous': resolvedEntry.ambiguous,
},
);
}
for (final ant in s.antonyms) {

View File

@@ -8,17 +8,15 @@ List<int?> getPriorityValues(XmlElement e, String prefix) {
int? news, ichi, spec, gai, nf;
for (final pri in e.findElements('${prefix}_pri')) {
final txt = pri.innerText;
if (txt.startsWith('news')) {
if (txt.startsWith('news'))
news = int.parse(txt.substring(4));
} else if (txt.startsWith('ichi')) {
else if (txt.startsWith('ichi'))
ichi = int.parse(txt.substring(4));
} else if (txt.startsWith('spec')) {
else if (txt.startsWith('spec'))
spec = int.parse(txt.substring(4));
} else if (txt.startsWith('gai')) {
else if (txt.startsWith('gai'))
gai = int.parse(txt.substring(3));
} else if (txt.startsWith('nf')) {
nf = int.parse(txt.substring(2));
}
else if (txt.startsWith('nf')) nf = int.parse(txt.substring(2));
}
return [news, ichi, spec, gai, nf];
}
@@ -48,7 +46,10 @@ XRefParts parseXrefParts(String s) {
);
}
} else {
result = XRefParts(kanjiRef: parts[0], readingRef: parts[1]);
result = XRefParts(
kanjiRef: parts[0],
readingRef: parts[1],
);
}
break;
@@ -80,48 +81,45 @@ List<Entry> parseJMDictData(XmlElement root) {
final List<ReadingElement> readingEls = [];
final List<Sense> senses = [];
for (final (kanjiNum, kEle) in entry.findElements('k_ele').indexed) {
final kePri = getPriorityValues(kEle, 'ke');
for (final (kanjiNum, k_ele) in entry.findElements('k_ele').indexed) {
final ke_pri = getPriorityValues(k_ele, 'ke');
kanjiEls.add(
KanjiElement(
orderNum: kanjiNum + 1,
info: kEle
info: k_ele
.findElements('ke_inf')
.map((e) => e.innerText.substring(1, e.innerText.length - 1))
.toList(),
reading: kEle.findElements('keb').first.innerText,
news: kePri[0],
ichi: kePri[1],
spec: kePri[2],
gai: kePri[3],
nf: kePri[4],
reading: k_ele.findElements('keb').first.innerText,
news: ke_pri[0],
ichi: ke_pri[1],
spec: ke_pri[2],
gai: ke_pri[3],
nf: ke_pri[4],
),
);
}
for (final (orderNum, rEle) in entry.findElements('r_ele').indexed) {
final rePri = getPriorityValues(rEle, 're');
final readingDoesNotMatchKanji = rEle
.findElements('re_nokanji')
.isNotEmpty;
for (final (orderNum, r_ele) in entry.findElements('r_ele').indexed) {
final re_pri = getPriorityValues(r_ele, 're');
final readingDoesNotMatchKanji =
r_ele.findElements('re_nokanji').isNotEmpty;
readingEls.add(
ReadingElement(
orderNum: orderNum + 1,
readingDoesNotMatchKanji: readingDoesNotMatchKanji,
info: rEle
info: r_ele
.findElements('re_inf')
.map((e) => e.innerText.substring(1, e.innerText.length - 1))
.toList(),
restrictions: rEle
.findElements('re_restr')
.map((e) => e.innerText)
.toList(),
reading: rEle.findElements('reb').first.innerText,
news: rePri[0],
ichi: rePri[1],
spec: rePri[2],
gai: rePri[3],
nf: rePri[4],
restrictions:
r_ele.findElements('re_restr').map((e) => e.innerText).toList(),
reading: r_ele.findElements('reb').first.innerText,
news: re_pri[0],
ichi: re_pri[1],
spec: re_pri[2],
gai: re_pri[3],
nf: re_pri[4],
),
);
}
@@ -131,14 +129,10 @@ List<Entry> parseJMDictData(XmlElement root) {
final result = Sense(
senseId: senseId,
orderNum: orderNum + 1,
restrictedToKanji: sense
.findElements('stagk')
.map((e) => e.innerText)
.toList(),
restrictedToReading: sense
.findElements('stagr')
.map((e) => e.innerText)
.toList(),
restrictedToKanji:
sense.findElements('stagk').map((e) => e.innerText).toList(),
restrictedToReading:
sense.findElements('stagr').map((e) => e.innerText).toList(),
pos: sense
.findElements('pos')
.map((e) => e.innerText.substring(1, e.innerText.length - 1))

View File

@@ -13,33 +13,42 @@ class CodePoint extends SQLWritable {
@override
Map<String, Object?> get sqlValue => {
'kanji': kanji,
'type': type,
'codepoint': codepoint,
};
'kanji': kanji,
'type': type,
'codepoint': codepoint,
};
}
class Radical extends SQLWritable {
final String kanji;
final int radicalId;
const Radical({required this.kanji, required this.radicalId});
const Radical({
required this.kanji,
required this.radicalId,
});
@override
Map<String, Object?> get sqlValue => {'kanji': kanji, 'radicalId': radicalId};
Map<String, Object?> get sqlValue => {
'kanji': kanji,
'radicalId': radicalId,
};
}
class StrokeMiscount extends SQLWritable {
final String kanji;
final int strokeCount;
const StrokeMiscount({required this.kanji, required this.strokeCount});
const StrokeMiscount({
required this.kanji,
required this.strokeCount,
});
@override
Map<String, Object?> get sqlValue => {
'kanji': kanji,
'strokeCount': strokeCount,
};
'kanji': kanji,
'strokeCount': strokeCount,
};
}
class Variant extends SQLWritable {
@@ -55,10 +64,10 @@ class Variant extends SQLWritable {
@override
Map<String, Object?> get sqlValue => {
'kanji': kanji,
'type': type,
'variant': variant,
};
'kanji': kanji,
'type': type,
'variant': variant,
};
}
class DictionaryReference extends SQLWritable {
@@ -74,10 +83,10 @@ class DictionaryReference extends SQLWritable {
@override
Map<String, Object?> get sqlValue => {
'kanji': kanji,
'type': type,
'ref': ref,
};
'kanji': kanji,
'type': type,
'ref': ref,
};
}
class DictionaryReferenceMoro extends SQLWritable {
@@ -95,11 +104,11 @@ class DictionaryReferenceMoro extends SQLWritable {
@override
Map<String, Object?> get sqlValue => {
'kanji': kanji,
'ref': ref,
'volume': volume,
'page': page,
};
'kanji': kanji,
'ref': ref,
'volume': volume,
'page': page,
};
}
class QueryCode extends SQLWritable {
@@ -117,11 +126,11 @@ class QueryCode extends SQLWritable {
@override
Map<String, Object?> get sqlValue => {
'kanji': kanji,
'code': code,
'type': type,
'skipMisclassification': skipMisclassification,
};
'kanji': kanji,
'code': code,
'type': type,
'skipMisclassification': skipMisclassification,
};
}
class Reading extends SQLWritable {
@@ -137,10 +146,10 @@ class Reading extends SQLWritable {
@override
Map<String, Object?> get sqlValue => {
'kanji': kanji,
'type': type,
'reading': reading,
};
'kanji': kanji,
'type': type,
'reading': reading,
};
}
class Kunyomi extends SQLWritable {
@@ -156,10 +165,10 @@ class Kunyomi extends SQLWritable {
@override
Map<String, Object?> get sqlValue => {
'kanji': kanji,
'yomi': yomi,
'isJouyou': isJouyou,
};
'kanji': kanji,
'yomi': yomi,
'isJouyou': isJouyou,
};
}
class Onyomi extends SQLWritable {
@@ -177,11 +186,11 @@ class Onyomi extends SQLWritable {
@override
Map<String, Object?> get sqlValue => {
'kanji': kanji,
'yomi': yomi,
'isJouyou': isJouyou,
'type': type,
};
'kanji': kanji,
'yomi': yomi,
'isJouyou': isJouyou,
'type': type,
};
}
class Meaning extends SQLWritable {
@@ -197,10 +206,10 @@ class Meaning extends SQLWritable {
@override
Map<String, Object?> get sqlValue => {
'kanji': kanji,
'language': language,
'meaning': meaning,
};
'kanji': kanji,
'language': language,
'meaning': meaning,
};
}
class Character extends SQLWritable {
@@ -245,12 +254,11 @@ class Character extends SQLWritable {
this.nanori = const [],
});
@override
Map<String, Object?> get sqlValue => {
'literal': literal,
'grade': grade,
'strokeCount': strokeCount,
'frequency': frequency,
'jlpt': jlpt,
};
'literal': literal,
'grade': grade,
'strokeCount': strokeCount,
'frequency': frequency,
'jlpt': jlpt,
};
}

View File

@@ -19,7 +19,10 @@ Future<void> seedKANJIDICData(List<Character> characters, Database db) async {
assert(c.radical != null, 'Radical name without radical');
b.insert(
KANJIDICTableNames.radicalName,
{'radicalId': c.radical!.radicalId, 'name': n},
{
'radicalId': c.radical!.radicalId,
'name': n,
},
conflictAlgorithm: ConflictAlgorithm.ignore,
);
}
@@ -31,10 +34,13 @@ Future<void> seedKANJIDICData(List<Character> characters, Database db) async {
b.insert(KANJIDICTableNames.radical, c.radical!.sqlValue);
}
for (final sm in c.strokeMiscounts) {
b.insert(KANJIDICTableNames.strokeMiscount, {
'kanji': c.literal,
'strokeCount': sm,
});
b.insert(
KANJIDICTableNames.strokeMiscount,
{
'kanji': c.literal,
'strokeCount': sm,
},
);
}
for (final v in c.variants) {
b.insert(KANJIDICTableNames.variant, v.sqlValue);
@@ -58,24 +64,24 @@ Future<void> seedKANJIDICData(List<Character> characters, Database db) async {
}
for (final (i, y) in c.kunyomi.indexed) {
b.insert(
KANJIDICTableNames.kunyomi,
y.sqlValue..addAll({'orderNum': i + 1}),
);
KANJIDICTableNames.kunyomi, y.sqlValue..addAll({'orderNum': i + 1}));
}
for (final (i, y) in c.onyomi.indexed) {
b.insert(
KANJIDICTableNames.onyomi,
y.sqlValue..addAll({'orderNum': i + 1}),
);
KANJIDICTableNames.onyomi, y.sqlValue..addAll({'orderNum': i + 1}));
}
for (final (i, m) in c.meanings.indexed) {
b.insert(
KANJIDICTableNames.meaning,
m.sqlValue..addAll({'orderNum': i + 1}),
);
KANJIDICTableNames.meaning, m.sqlValue..addAll({'orderNum': i + 1}));
}
for (final n in c.nanori) {
b.insert(KANJIDICTableNames.nanori, {'kanji': c.literal, 'nanori': n});
b.insert(
KANJIDICTableNames.nanori,
{
'kanji': c.literal,
'nanori': n,
},
);
}
}
await b.commit(noResult: true);

View File

@@ -10,33 +10,27 @@ List<Character> parseKANJIDICData(XmlElement root) {
final codepoint = c.findElements('codepoint').firstOrNull;
final radical = c.findElements('radical').firstOrNull;
final misc = c.findElements('misc').first;
final dicNumber = c.findElements('dic_number').firstOrNull;
final queryCode = c.findElements('query_code').first;
final readingMeaning = c.findElements('reading_meaning').firstOrNull;
final dic_number = c.findElements('dic_number').firstOrNull;
final query_code = c.findElements('query_code').first;
final reading_meaning = c.findElements('reading_meaning').firstOrNull;
// TODO: Group readings and meanings by their rmgroup parent node.
result.add(
Character(
literal: kanji,
strokeCount: int.parse(
misc.findElements('stroke_count').first.innerText,
),
strokeCount:
int.parse(misc.findElements('stroke_count').first.innerText),
grade: int.tryParse(
misc.findElements('grade').firstOrNull?.innerText ?? '',
),
misc.findElements('grade').firstOrNull?.innerText ?? ''),
frequency: int.tryParse(
misc.findElements('freq').firstOrNull?.innerText ?? '',
),
misc.findElements('freq').firstOrNull?.innerText ?? ''),
jlpt: int.tryParse(
misc.findElements('jlpt').firstOrNull?.innerText ?? '',
),
radicalName: misc
.findElements('rad_name')
.map((e) => e.innerText)
.toList(),
codepoints:
codepoint
radicalName:
misc.findElements('rad_name').map((e) => e.innerText).toList(),
codepoints: codepoint
?.findElements('cp_value')
.map(
(e) => CodePoint(
@@ -51,7 +45,10 @@ List<Character> parseKANJIDICData(XmlElement root) {
?.findElements('rad_value')
.where((e) => e.getAttribute('rad_type') == 'classical')
.map(
(e) => Radical(kanji: kanji, radicalId: int.parse(e.innerText)),
(e) => Radical(
kanji: kanji,
radicalId: int.parse(e.innerText),
),
)
.firstOrNull,
strokeMiscounts: misc
@@ -69,8 +66,7 @@ List<Character> parseKANJIDICData(XmlElement root) {
),
)
.toList(),
dictionaryReferences:
dicNumber
dictionaryReferences: dic_number
?.findElements('dic_ref')
.where((e) => e.getAttribute('dr_type') != 'moro')
.map(
@@ -82,8 +78,7 @@ List<Character> parseKANJIDICData(XmlElement root) {
)
.toList() ??
[],
dictionaryReferencesMoro:
dicNumber
dictionaryReferencesMoro: dic_number
?.findElements('dic_ref')
.where((e) => e.getAttribute('dr_type') == 'moro')
.map(
@@ -96,7 +91,7 @@ List<Character> parseKANJIDICData(XmlElement root) {
)
.toList() ??
[],
querycodes: queryCode
querycodes: query_code
.findElements('q_code')
.map(
(e) => QueryCode(
@@ -107,8 +102,7 @@ List<Character> parseKANJIDICData(XmlElement root) {
),
)
.toList(),
readings:
readingMeaning
readings: reading_meaning
?.findAllElements('reading')
.where(
(e) =>
@@ -123,8 +117,7 @@ List<Character> parseKANJIDICData(XmlElement root) {
)
.toList() ??
[],
kunyomi:
readingMeaning
kunyomi: reading_meaning
?.findAllElements('reading')
.where((e) => e.getAttribute('r_type') == 'ja_kun')
.map(
@@ -136,22 +129,19 @@ List<Character> parseKANJIDICData(XmlElement root) {
)
.toList() ??
[],
onyomi:
readingMeaning
onyomi: reading_meaning
?.findAllElements('reading')
.where((e) => e.getAttribute('r_type') == 'ja_on')
.map(
(e) => Onyomi(
kanji: kanji,
yomi: transliterateKatakanaToHiragana(e.innerText),
isJouyou: e.getAttribute('r_status') == 'jy',
type: e.getAttribute('on_type'),
),
kanji: kanji,
yomi: transliterateKatakanaToHiragana(e.innerText),
isJouyou: e.getAttribute('r_status') == 'jy',
type: e.getAttribute('on_type')),
)
.toList() ??
[],
meanings:
readingMeaning
meanings: reading_meaning
?.findAllElements('meaning')
.map(
(e) => Meaning(
@@ -162,8 +152,7 @@ List<Character> parseKANJIDICData(XmlElement root) {
)
.toList() ??
[],
nanori:
readingMeaning
nanori: reading_meaning
?.findElements('nanori')
.map((e) => e.innerText)
.toList() ??

View File

@@ -1,7 +1,9 @@
import 'dart:ffi';
import 'dart:io';
import 'package:jadb/search.dart';
import 'package:sqflite_common_ffi/sqflite_ffi.dart';
import 'package:sqlite3/open.dart';
Future<Database> openLocalDb({
String? libsqlitePath,
@@ -10,23 +12,38 @@ Future<Database> openLocalDb({
bool verifyTablesExist = true,
bool walMode = false,
}) async {
libsqlitePath ??= Platform.environment['LIBSQLITE_PATH'];
jadbPath ??= Platform.environment['JADB_PATH'];
jadbPath ??= Directory.current.uri.resolve('jadb.sqlite').path;
libsqlitePath = (libsqlitePath == null)
? null
: File(libsqlitePath).resolveSymbolicLinksSync();
jadbPath = File(jadbPath).resolveSymbolicLinksSync();
if (!File(jadbPath).existsSync()) {
throw Exception('JADB_PATH does not exist: $jadbPath');
if (libsqlitePath == null) {
throw Exception("LIBSQLITE_PATH is not set");
}
final db = await createDatabaseFactoryFfi().openDatabase(
if (!File(libsqlitePath).existsSync()) {
throw Exception("LIBSQLITE_PATH does not exist: $libsqlitePath");
}
if (!File(jadbPath).existsSync()) {
throw Exception("JADB_PATH does not exist: $jadbPath");
}
final db = await createDatabaseFactoryFfi(
ffiInit: () =>
open.overrideForAll(() => DynamicLibrary.open(libsqlitePath!)),
).openDatabase(
jadbPath,
options: OpenDatabaseOptions(
onConfigure: (db) async {
if (walMode) {
await db.execute('PRAGMA journal_mode = WAL');
await db.execute("PRAGMA journal_mode = WAL");
}
await db.execute('PRAGMA foreign_keys = ON');
await db.execute("PRAGMA foreign_keys = ON");
},
readOnly: !readWrite,
),

View File

@@ -3,10 +3,8 @@ import 'dart:io';
Iterable<String> parseRADKFILEBlocks(File radkfile) {
final String content = File('data/tmp/radkfile_utf8').readAsStringSync();
final Iterable<String> blocks = content
.replaceAll(RegExp(r'^#.*$'), '')
.split(r'$')
.skip(2);
final Iterable<String> blocks =
content.replaceAll(RegExp(r'^#.*$'), '').split(r'$').skip(2);
return blocks;
}

View File

@@ -1,20 +1,27 @@
import 'package:jadb/table_names/radkfile.dart';
import 'package:sqflite_common/sqlite_api.dart';
Future<void> seedRADKFILEData(Iterable<String> blocks, Database db) async {
Future<void> seedRADKFILEData(
Iterable<String> blocks,
Database db,
) async {
final b = db.batch();
for (final block in blocks) {
final String radical = block[1];
final List<String> kanjiList =
block.replaceFirst(RegExp(r'.*\n'), '').split('')
..removeWhere((e) => e == '' || e == '\n');
final List<String> kanjiList = block
.replaceFirst(RegExp(r'.*\n'), '')
.split('')
..removeWhere((e) => e == '' || e == '\n');
for (final kanji in kanjiList.toSet()) {
b.insert(RADKFILETableNames.radkfile, {
'radical': radical,
'kanji': kanji,
});
b.insert(
RADKFILETableNames.radkfile,
{
'radical': radical,
'kanji': kanji,
},
);
}
}

View File

@@ -24,10 +24,10 @@ Future<void> seedData(Database db) async {
Future<void> parseAndSeedDataFromJMdict(Database db) async {
print('[JMdict] Reading file content...');
final String rawXML = File('data/tmp/JMdict.xml').readAsStringSync();
String rawXML = File('data/tmp/JMdict.xml').readAsStringSync();
print('[JMdict] Parsing XML tags...');
final XmlElement root = XmlDocument.parse(rawXML).getElement('JMdict')!;
XmlElement root = XmlDocument.parse(rawXML).getElement('JMdict')!;
print('[JMdict] Parsing XML content...');
final entries = parseJMDictData(root);
@@ -38,10 +38,10 @@ Future<void> parseAndSeedDataFromJMdict(Database db) async {
Future<void> parseAndSeedDataFromKANJIDIC(Database db) async {
print('[KANJIDIC2] Reading file...');
final String rawXML = File('data/tmp/kanjidic2.xml').readAsStringSync();
String rawXML = File('data/tmp/kanjidic2.xml').readAsStringSync();
print('[KANJIDIC2] Parsing XML...');
final XmlElement root = XmlDocument.parse(rawXML).getElement('kanjidic2')!;
XmlElement root = XmlDocument.parse(rawXML).getElement('kanjidic2')!;
print('[KANJIDIC2] Parsing XML content...');
final entries = parseKANJIDICData(root);
@@ -52,7 +52,7 @@ Future<void> parseAndSeedDataFromKANJIDIC(Database db) async {
Future<void> parseAndSeedDataFromRADKFILE(Database db) async {
print('[RADKFILE] Reading file...');
final File raw = File('data/tmp/RADKFILE');
File raw = File('data/tmp/RADKFILE');
print('[RADKFILE] Parsing content...');
final blocks = parseRADKFILEBlocks(raw);
@@ -63,7 +63,7 @@ Future<void> parseAndSeedDataFromRADKFILE(Database db) async {
Future<void> parseAndSeedDataFromTanosJLPT(Database db) async {
print('[TANOS-JLPT] Reading files...');
final Map<String, File> files = {
Map<String, File> files = {
'N1': File('data/tanos-jlpt/n1.csv'),
'N2': File('data/tanos-jlpt/n2.csv'),
'N3': File('data/tanos-jlpt/n3.csv'),

View File

@@ -3,64 +3,52 @@ import 'dart:io';
import 'package:csv/csv.dart';
import 'package:jadb/_data_ingestion/tanos-jlpt/objects.dart';
import 'package:xml/xml_events.dart';
Future<List<JLPTRankedWord>> parseJLPTRankedWords(
Map<String, File> files,
) async {
final List<JLPTRankedWord> result = [];
final codec = CsvCodec(
fieldDelimiter: ',',
lineDelimiter: '\n',
quoteMode: QuoteMode.strings,
escapeCharacter: '\\',
);
for (final entry in files.entries) {
final jlptLevel = entry.key;
final file = entry.value;
if (!file.existsSync()) {
throw Exception('File $jlptLevel does not exist');
throw Exception("File $jlptLevel does not exist");
}
final words = await file
final rows = await file
.openRead()
.transform(utf8.decoder)
.transform(codec.decoder)
.flatten()
.map((row) {
if (row.length != 3) {
throw Exception('Invalid line in $jlptLevel: $row');
}
return row;
})
.map((row) => row.map((e) => e as String).toList())
.map((row) {
final kanji = row[0].isEmpty
? null
: row[0]
.replaceFirst(RegExp('^お・'), '')
.replaceAll(RegExp(r'.*'), '');
final readings = row[1]
.split(RegExp('[・/、(:?s+)]'))
.map((e) => e.trim())
.toList();
final meanings = row[2].split(',').expand(cleanMeaning).toList();
return JLPTRankedWord(
readings: readings,
kanji: kanji,
jlptLevel: jlptLevel,
meanings: meanings,
);
})
.transform(CsvToListConverter())
.toList();
result.addAll(words);
for (final row in rows) {
if (row.length != 3) {
throw Exception("Invalid line in $jlptLevel: $row");
}
final kanji = (row[0] as String).isEmpty
? null
: (row[0] as String)
.replaceFirst(RegExp('^お・'), '')
.replaceAll(RegExp(r'.*'), '');
final readings = (row[1] as String)
.split(RegExp('[・/、(:?\s+)]'))
.map((e) => e.trim())
.toList();
final meanings =
(row[2] as String).split(',').expand(cleanMeaning).toList();
result.add(JLPTRankedWord(
readings: readings,
kanji: kanji,
jlptLevel: jlptLevel,
meanings: meanings,
));
}
}
return result;

View File

@@ -13,5 +13,5 @@ class JLPTRankedWord {
@override
String toString() =>
'($jlptLevel,$kanji,"${readings.join(",")}","${meanings.join(",")})';
'(${jlptLevel},${kanji},"${readings.join(",")}","${meanings.join(",")})';
}

View File

@@ -1,4 +1,4 @@
const Map<(String?, String), int?> tanosJLPTOverrides = {
const Map<(String?, String), int?> TANOS_JLPT_OVERRIDES = {
// N5:
(null, 'あなた'): 1223615,
(null, 'あの'): 1000430,

View File

@@ -1,39 +1,49 @@
import 'package:jadb/table_names/jmdict.dart';
import 'package:jadb/_data_ingestion/tanos-jlpt/objects.dart';
import 'package:jadb/_data_ingestion/tanos-jlpt/overrides.dart';
import 'package:jadb/table_names/jmdict.dart';
import 'package:sqflite_common/sqlite_api.dart';
Future<List<int>> _findReadingCandidates(JLPTRankedWord word, Database db) => db
.query(
JMdictTableNames.readingElement,
columns: ['entryId'],
where:
'"reading" IN (${List.filled(word.readings.length, '?').join(',')})',
whereArgs: [...word.readings],
)
.then((rows) => rows.map((row) => row['entryId'] as int).toList());
Future<List<int>> _findReadingCandidates(
JLPTRankedWord word,
Database db,
) =>
db
.query(
JMdictTableNames.readingElement,
columns: ['entryId'],
where:
'"reading" IN (${List.filled(word.readings.length, '?').join(',')})',
whereArgs: [...word.readings],
)
.then((rows) => rows.map((row) => row['entryId'] as int).toList());
Future<List<int>> _findKanjiCandidates(JLPTRankedWord word, Database db) => db
.query(
JMdictTableNames.kanjiElement,
columns: ['entryId'],
where: 'reading = ?',
whereArgs: [word.kanji],
)
.then((rows) => rows.map((row) => row['entryId'] as int).toList());
Future<List<int>> _findKanjiCandidates(
JLPTRankedWord word,
Database db,
) =>
db
.query(
JMdictTableNames.kanjiElement,
columns: ['entryId'],
where: 'reading = ?',
whereArgs: [word.kanji],
)
.then((rows) => rows.map((row) => row['entryId'] as int).toList());
Future<List<(int, String)>> _findSenseCandidates(
JLPTRankedWord word,
Database db,
) => db
.rawQuery(
) =>
db.rawQuery(
'SELECT entryId, phrase '
'FROM "${JMdictTableNames.senseGlossary}" '
'JOIN "${JMdictTableNames.sense}" USING (senseId)'
'WHERE phrase IN (${List.filled(word.meanings.length, '?').join(',')})',
'WHERE phrase IN (${List.filled(
word.meanings.length,
'?',
).join(',')})',
[...word.meanings],
)
.then(
).then(
(rows) => rows
.map((row) => (row['entryId'] as int, row['phrase'] as String))
.toList(),
@@ -45,10 +55,8 @@ Future<int?> findEntry(
bool useOverrides = true,
}) async {
final List<int> readingCandidates = await _findReadingCandidates(word, db);
final List<(int, String)> senseCandidates = await _findSenseCandidates(
word,
db,
);
final List<(int, String)> senseCandidates =
await _findSenseCandidates(word, db);
List<int> entryIds;
@@ -63,10 +71,8 @@ Future<int?> findEntry(
print('No entry found, trying to combine with senses');
entryIds = readingCandidates
.where(
(readingId) =>
senseCandidates.any((sense) => sense.$1 == readingId),
)
.where((readingId) =>
senseCandidates.any((sense) => sense.$1 == readingId))
.toList();
}
} else {
@@ -76,21 +82,18 @@ Future<int?> findEntry(
if ((entryIds.isEmpty || entryIds.length > 1) && useOverrides) {
print('No entry found, trying to fetch from overrides');
final overrideEntries = word.readings
.map((reading) => tanosJLPTOverrides[(word.kanji, reading)])
.map((reading) => TANOS_JLPT_OVERRIDES[(word.kanji, reading)])
.whereType<int>()
.toSet();
if (overrideEntries.length > 1) {
throw Exception(
'Multiple override entries found for ${word.toString()}: $entryIds',
);
} else if (overrideEntries.isEmpty &&
!word.readings.any(
(reading) => tanosJLPTOverrides.containsKey((word.kanji, reading)),
)) {
'Multiple override entries found for ${word.toString()}: $entryIds');
} else if (overrideEntries.length == 0 &&
!word.readings.any((reading) =>
TANOS_JLPT_OVERRIDES.containsKey((word.kanji, reading)))) {
throw Exception(
'No override entry found for ${word.toString()}: $entryIds',
);
'No override entry found for ${word.toString()}: $entryIds');
}
print('Found override: ${overrideEntries.firstOrNull}');
@@ -100,8 +103,7 @@ Future<int?> findEntry(
if (entryIds.length > 1) {
throw Exception(
'Multiple override entries found for ${word.toString()}: $entryIds',
);
'Multiple override entries found for ${word.toString()}: $entryIds');
} else if (entryIds.isEmpty) {
throw Exception('No entry found for ${word.toString()}');
}

View File

@@ -5,17 +5,20 @@ Future<void> seedTanosJLPTData(
Map<String, Set<int>> resolvedEntries,
Database db,
) async {
final Batch b = db.batch();
Batch b = db.batch();
for (final jlptLevel in resolvedEntries.entries) {
final level = jlptLevel.key;
final entryIds = jlptLevel.value;
for (final entryId in entryIds) {
b.insert(TanosJLPTTableNames.jlptTag, {
'entryId': entryId,
'jlptLevel': level,
});
b.insert(
TanosJLPTTableNames.jlptTag,
{
'entryId': entryId,
'jlptLevel': level,
},
);
}
}

View File

@@ -1,15 +1,14 @@
import 'dart:io';
import 'package:args/command_runner.dart';
import 'package:jadb/_data_ingestion/open_local_db.dart';
import 'package:jadb/_data_ingestion/seed_database.dart';
import 'package:args/command_runner.dart';
import 'package:jadb/cli/args.dart';
class CreateDb extends Command {
@override
final name = 'create-db';
@override
final description = 'Create the database';
final name = "create-db";
final description = "Create the database";
CreateDb() {
addLibsqliteArg(argParser);
@@ -24,7 +23,6 @@ class CreateDb extends Command {
);
}
@override
Future<void> run() async {
if (argResults!.option('libsqlite') == null) {
print(argParser.usage);
@@ -38,17 +36,14 @@ class CreateDb extends Command {
);
bool failed = false;
await seedData(db)
.then((_) {
print('Database created successfully');
})
.catchError((error) {
print('Error creating database: $error');
failed = true;
})
.whenComplete(() {
db.close();
});
await seedData(db).then((_) {
print("Database created successfully");
}).catchError((error) {
print("Error creating database: $error");
failed = true;
}).whenComplete(() {
db.close();
});
if (failed) {
exit(1);
} else {

View File

@@ -1,7 +1,8 @@
import 'dart:io';
import 'package:args/command_runner.dart';
import 'package:jadb/_data_ingestion/open_local_db.dart';
import 'package:args/command_runner.dart';
import 'package:jadb/_data_ingestion/tanos-jlpt/csv_parser.dart';
import 'package:jadb/_data_ingestion/tanos-jlpt/objects.dart';
import 'package:jadb/_data_ingestion/tanos-jlpt/resolve.dart';
@@ -9,11 +10,9 @@ import 'package:jadb/cli/args.dart';
import 'package:sqflite_common/sqlite_api.dart';
class CreateTanosJlptMappings extends Command {
@override
final name = 'create-tanos-jlpt-mappings';
@override
final name = "create-tanos-jlpt-mappings";
final description =
'Resolve Tanos JLPT data against JMDict. This tool is useful to create overrides for ambiguous references';
"Resolve Tanos JLPT data against JMDict. This tool is useful to create overrides for ambiguous references";
CreateTanosJlptMappings() {
addLibsqliteArg(argParser);
@@ -27,7 +26,6 @@ class CreateTanosJlptMappings extends Command {
);
}
@override
Future<void> run() async {
if (argResults!.option('libsqlite') == null ||
argResults!.option('jadb') == null) {
@@ -42,7 +40,7 @@ class CreateTanosJlptMappings extends Command {
final useOverrides = argResults!.flag('overrides');
final Map<String, File> files = {
Map<String, File> files = {
'N1': File('data/tanos-jlpt/n1.csv'),
'N2': File('data/tanos-jlpt/n2.csv'),
'N3': File('data/tanos-jlpt/n3.csv'),
@@ -61,12 +59,11 @@ Future<void> resolveExisting(
Database db,
bool useOverrides,
) async {
final List<JLPTRankedWord> missingWords = [];
List<JLPTRankedWord> missingWords = [];
for (final (i, word) in rankedWords.indexed) {
try {
print(
'[${(i + 1).toString().padLeft(4, '0')}/${rankedWords.length}] ${word.toString()}',
);
'[${(i + 1).toString().padLeft(4, '0')}/${rankedWords.length}] ${word.toString()}');
await findEntry(word, db, useOverrides: useOverrides);
} catch (e) {
print(e);
@@ -81,19 +78,16 @@ Future<void> resolveExisting(
print('Statistics:');
for (final jlptLevel in ['N5', 'N4', 'N3', 'N2', 'N1']) {
final missingWordCount = missingWords
.where((e) => e.jlptLevel == jlptLevel)
.length;
final totalWordCount = rankedWords
.where((e) => e.jlptLevel == jlptLevel)
.length;
final missingWordCount =
missingWords.where((e) => e.jlptLevel == jlptLevel).length;
final totalWordCount =
rankedWords.where((e) => e.jlptLevel == jlptLevel).length;
final failureRate = ((missingWordCount / totalWordCount) * 100)
.toStringAsFixed(2);
final failureRate =
((missingWordCount / totalWordCount) * 100).toStringAsFixed(2);
print(
'$jlptLevel failures: [$missingWordCount/$totalWordCount] ($failureRate%)',
);
'${jlptLevel} failures: [${missingWordCount}/${totalWordCount}] (${failureRate}%)');
}
print('Not able to determine the entry for ${missingWords.length} words');

View File

@@ -1,15 +1,14 @@
// import 'dart:io';
import 'package:args/command_runner.dart';
// import 'package:jadb/_data_ingestion/open_local_db.dart';
import 'package:jadb/cli/args.dart';
import 'package:args/command_runner.dart';
import 'package:jadb/util/lemmatizer/lemmatizer.dart';
class Lemmatize extends Command {
@override
final name = 'lemmatize';
@override
final description = 'Lemmatize a word using the Jadb lemmatizer';
final name = "lemmatize";
final description = "Lemmatize a word using the Jadb lemmatizer";
Lemmatize() {
addLibsqliteArg(argParser);
@@ -22,7 +21,6 @@ class Lemmatize extends Command {
);
}
@override
Future<void> run() async {
// if (argResults!.option('libsqlite') == null ||
// argResults!.option('jadb') == null) {
@@ -43,6 +41,6 @@ class Lemmatize extends Command {
print(result.toString());
print('Lemmatization took ${time.elapsedMilliseconds}ms');
print("Lemmatization took ${time.elapsedMilliseconds}ms");
}
}

View File

@@ -1,25 +1,22 @@
import 'dart:convert';
import 'dart:io';
import 'package:args/command_runner.dart';
import 'package:jadb/_data_ingestion/open_local_db.dart';
import 'package:jadb/cli/args.dart';
import 'package:jadb/search.dart';
import 'package:args/command_runner.dart';
class QueryKanji extends Command {
@override
final name = 'query-kanji';
@override
final description = 'Query the database for kanji data';
@override
final invocation = 'jadb query-kanji [options] <kanji>';
final name = "query-kanji";
final description = "Query the database for kanji data";
final invocation = "jadb query-kanji [options] <kanji>";
QueryKanji() {
addLibsqliteArg(argParser);
addJadbArg(argParser);
}
@override
Future<void> run() async {
if (argResults!.option('libsqlite') == null ||
argResults!.option('jadb') == null) {
@@ -46,11 +43,11 @@ class QueryKanji extends Command {
time.stop();
if (result == null) {
print('No such kanji');
print("No such kanji");
} else {
print(JsonEncoder.withIndent(' ').convert(result.toJson()));
}
print('Query took ${time.elapsedMilliseconds}ms');
print("Query took ${time.elapsedMilliseconds}ms");
}
}

View File

@@ -1,36 +1,26 @@
import 'dart:convert';
import 'dart:io';
import 'package:args/command_runner.dart';
import 'package:jadb/_data_ingestion/open_local_db.dart';
import 'package:jadb/cli/args.dart';
import 'package:jadb/search.dart';
import 'package:args/command_runner.dart';
import 'package:sqflite_common/sqflite.dart';
class QueryWord extends Command {
@override
final name = 'query-word';
@override
final description = 'Query the database for word data';
@override
final invocation = 'jadb query-word [options] (<word> | <ID>)';
final name = "query-word";
final description = "Query the database for word data";
final invocation = "jadb query-word [options] (<word> | <ID>)";
QueryWord() {
addLibsqliteArg(argParser);
addJadbArg(argParser);
argParser.addFlag('json', abbr: 'j', help: 'Output results in JSON format');
argParser.addOption('page', abbr: 'p', valueHelp: 'NUM', defaultsTo: '0');
argParser.addOption('pageSize', valueHelp: 'NUM', defaultsTo: '30');
}
@override
Future<void> run() async {
if (argResults!.option('libsqlite') == null ||
argResults!.option('jadb') == null) {
print('You need to provide both libsqlite and jadb paths.');
print("You need to provide both libsqlite and jadb paths.");
print('');
printUsage();
exit(64);
@@ -48,74 +38,52 @@ class QueryWord extends Command {
exit(64);
}
final String searchWord = argResults!.rest.join(' ');
final String searchWord = argResults!.rest.join(" ");
final int? maybeId = int.tryParse(searchWord);
if (maybeId != null && maybeId >= 1000000) {
await _searchId(db, maybeId, argResults!.flag('json'));
await _searchId(db, maybeId);
} else {
await _searchWord(
db,
searchWord,
argResults!.flag('json'),
int.parse(argResults!.option('page')!),
int.parse(argResults!.option('pageSize')!),
);
await _searchWord(db, searchWord);
}
}
Future<void> _searchId(DatabaseExecutor db, int id, bool jsonOutput) async {
Future<void> _searchId(DatabaseExecutor db, int id) async {
final time = Stopwatch()..start();
final result = await JaDBConnection(db).jadbGetWordById(id);
time.stop();
if (result == null) {
print('Invalid ID');
print("Invalid ID");
} else {
if (jsonOutput) {
print(JsonEncoder.withIndent(' ').convert(result));
} else {
print(result.toString());
}
print(result.toString());
}
print('Query took ${time.elapsedMilliseconds}ms');
print("Query took ${time.elapsedMilliseconds}ms");
}
Future<void> _searchWord(
DatabaseExecutor db,
String searchWord,
bool jsonOutput,
int page,
int pageSize,
) async {
Future<void> _searchWord(DatabaseExecutor db, String searchWord) async {
final time = Stopwatch()..start();
final count = await JaDBConnection(db).jadbSearchWordCount(searchWord);
time.stop();
final time2 = Stopwatch()..start();
final result = await JaDBConnection(
db,
).jadbSearchWord(searchWord, page: page, pageSize: pageSize);
final result = await JaDBConnection(db).jadbSearchWord(searchWord);
time2.stop();
if (result == null) {
print('Invalid search');
print("Invalid search");
} else if (result.isEmpty) {
print('No matches');
print("No matches");
} else {
if (jsonOutput) {
print(JsonEncoder.withIndent(' ').convert(result));
} else {
for (final e in result) {
print(e.toString());
print('');
}
for (final e in result) {
print(e.toString());
print("");
}
}
print('Total count: $count');
print('Count query took ${time.elapsedMilliseconds}ms');
print('Query took ${time2.elapsedMilliseconds}ms');
print("Total count: ${count}");
print("Count query took ${time.elapsedMilliseconds}ms");
print("Query took ${time2.elapsedMilliseconds}ms");
}
}

View File

@@ -1,5 +1,6 @@
/// Jouyou kanji sorted primarily by grades and secondarily by strokes.
const Map<int, Map<int, List<String>>> jouyouKanjiByGradeAndStrokeCount = {
const Map<int, Map<int, List<String>>> JOUYOU_KANJI_BY_GRADE_AND_STROKE_COUNT =
{
1: {
1: [''],
2: ['', '', '', '', '', '', '', ''],
@@ -11,7 +12,7 @@ const Map<int, Map<int, List<String>>> jouyouKanjiByGradeAndStrokeCount = {
8: ['', '', '', '', '', ''],
9: ['', ''],
10: [''],
12: [''],
12: ['']
},
2: {
2: [''],
@@ -34,7 +35,7 @@ const Map<int, Map<int, List<String>>> jouyouKanjiByGradeAndStrokeCount = {
'',
'',
'',
'',
''
],
5: ['', '', '', '', '', '', '', '', '', '', '', ''],
6: [
@@ -57,7 +58,7 @@ const Map<int, Map<int, List<String>>> jouyouKanjiByGradeAndStrokeCount = {
'',
'',
'',
'',
''
],
7: [
'',
@@ -77,7 +78,7 @@ const Map<int, Map<int, List<String>>> jouyouKanjiByGradeAndStrokeCount = {
'',
'',
'',
'',
''
],
8: [
'',
@@ -94,7 +95,7 @@ const Map<int, Map<int, List<String>>> jouyouKanjiByGradeAndStrokeCount = {
'',
'',
'',
'',
''
],
9: [
'',
@@ -114,7 +115,7 @@ const Map<int, Map<int, List<String>>> jouyouKanjiByGradeAndStrokeCount = {
'',
'',
'',
'',
''
],
10: ['', '', '', '', '', '', '', '', '', '', '', ''],
11: ['', '', '', '', '', '', '', '', '', '', '', '', ''],
@@ -123,7 +124,7 @@ const Map<int, Map<int, List<String>>> jouyouKanjiByGradeAndStrokeCount = {
14: ['', '', '', '', '', ''],
15: [''],
16: ['', ''],
18: ['', ''],
18: ['', '']
},
3: {
2: [''],
@@ -145,7 +146,7 @@ const Map<int, Map<int, List<String>>> jouyouKanjiByGradeAndStrokeCount = {
'',
'',
'',
'',
''
],
6: ['', '', '', '', '', '', '', '', '', '', '', '', '', ''],
7: ['', '', '', '', '', '', '', '', '', '', '', '', '', ''],
@@ -177,7 +178,7 @@ const Map<int, Map<int, List<String>>> jouyouKanjiByGradeAndStrokeCount = {
'',
'',
'',
'',
''
],
9: [
'',
@@ -209,7 +210,7 @@ const Map<int, Map<int, List<String>>> jouyouKanjiByGradeAndStrokeCount = {
'',
'',
'',
'',
''
],
10: [
'',
@@ -231,7 +232,7 @@ const Map<int, Map<int, List<String>>> jouyouKanjiByGradeAndStrokeCount = {
'',
'',
'',
'',
''
],
11: [
'',
@@ -252,7 +253,7 @@ const Map<int, Map<int, List<String>>> jouyouKanjiByGradeAndStrokeCount = {
'',
'',
'',
'',
''
],
12: [
'',
@@ -281,13 +282,13 @@ const Map<int, Map<int, List<String>>> jouyouKanjiByGradeAndStrokeCount = {
'',
'',
'',
'',
''
],
13: ['', '', '', '', '', '', '', '', '', '', ''],
14: ['', '', '', '', '', ''],
15: ['', '調', '', ''],
16: ['', '', '', ''],
18: [''],
18: ['']
},
4: {
4: ['', '', '', '', ''],
@@ -317,7 +318,7 @@ const Map<int, Map<int, List<String>>> jouyouKanjiByGradeAndStrokeCount = {
'',
'',
'',
'',
''
],
8: [
'',
@@ -345,7 +346,7 @@ const Map<int, Map<int, List<String>>> jouyouKanjiByGradeAndStrokeCount = {
'',
'',
'',
'',
''
],
9: [
'',
@@ -366,7 +367,7 @@ const Map<int, Map<int, List<String>>> jouyouKanjiByGradeAndStrokeCount = {
'',
'',
'',
'',
''
],
10: [
'',
@@ -388,7 +389,7 @@ const Map<int, Map<int, List<String>>> jouyouKanjiByGradeAndStrokeCount = {
'',
'',
'',
'',
''
],
11: [
'',
@@ -409,7 +410,7 @@ const Map<int, Map<int, List<String>>> jouyouKanjiByGradeAndStrokeCount = {
'',
'',
'',
'',
''
],
12: [
'',
@@ -433,7 +434,7 @@ const Map<int, Map<int, List<String>>> jouyouKanjiByGradeAndStrokeCount = {
'',
'',
'',
'',
''
],
13: ['', '', '', '', '', '', '', '', '', '', ''],
14: ['', '', '', '', '', '', '', '', '', ''],
@@ -441,7 +442,7 @@ const Map<int, Map<int, List<String>>> jouyouKanjiByGradeAndStrokeCount = {
16: ['', '', ''],
18: ['', '', ''],
19: ['', ''],
20: ['', ''],
20: ['', '']
},
5: {
3: ['', ''],
@@ -463,7 +464,7 @@ const Map<int, Map<int, List<String>>> jouyouKanjiByGradeAndStrokeCount = {
'',
'',
'',
'',
''
],
8: [
'',
@@ -483,7 +484,7 @@ const Map<int, Map<int, List<String>>> jouyouKanjiByGradeAndStrokeCount = {
'',
'',
'',
'',
''
],
9: ['', '', '', '', '', '', '', '', '', '', '', '', ''],
10: [
@@ -504,7 +505,7 @@ const Map<int, Map<int, List<String>>> jouyouKanjiByGradeAndStrokeCount = {
'',
'',
'',
'',
''
],
11: [
'',
@@ -536,7 +537,7 @@ const Map<int, Map<int, List<String>>> jouyouKanjiByGradeAndStrokeCount = {
'',
'',
'',
'',
''
],
12: [
'貿',
@@ -560,7 +561,7 @@ const Map<int, Map<int, List<String>>> jouyouKanjiByGradeAndStrokeCount = {
'',
'',
'',
'',
''
],
13: ['', '', '', '', '', '', '', '', '', '', '', '', '', ''],
14: [
@@ -582,14 +583,14 @@ const Map<int, Map<int, List<String>>> jouyouKanjiByGradeAndStrokeCount = {
'',
'',
'',
'',
''
],
15: ['', '', '', '', '', '', '', ''],
16: ['', '', '', '', ''],
17: ['', '', ''],
18: ['', '', ''],
19: [''],
20: [''],
20: ['']
},
6: {
3: ['', '', '', ''],
@@ -617,7 +618,7 @@ const Map<int, Map<int, List<String>>> jouyouKanjiByGradeAndStrokeCount = {
'',
'',
'沿',
'',
''
],
9: [
'',
@@ -640,7 +641,7 @@ const Map<int, Map<int, List<String>>> jouyouKanjiByGradeAndStrokeCount = {
'',
'',
'',
'',
''
],
10: [
'',
@@ -666,7 +667,7 @@ const Map<int, Map<int, List<String>>> jouyouKanjiByGradeAndStrokeCount = {
'',
'',
'',
'',
''
],
11: [
'',
@@ -688,7 +689,7 @@ const Map<int, Map<int, List<String>>> jouyouKanjiByGradeAndStrokeCount = {
'',
'',
'',
'',
''
],
12: [
'',
@@ -709,7 +710,7 @@ const Map<int, Map<int, List<String>>> jouyouKanjiByGradeAndStrokeCount = {
'',
'',
'',
'',
''
],
13: [
'',
@@ -726,14 +727,14 @@ const Map<int, Map<int, List<String>>> jouyouKanjiByGradeAndStrokeCount = {
'',
'',
'',
'',
''
],
14: ['', '', '', '', '', '', '', '', '', '', '', ''],
15: ['', '', '', '', '', '', '', '', '', ''],
16: ['', '', '', '', '', '', '', ''],
17: ['', '', '', ''],
18: ['', '', ''],
19: ['', ''],
19: ['', '']
},
7: {
1: [''],
@@ -759,7 +760,7 @@ const Map<int, Map<int, List<String>>> jouyouKanjiByGradeAndStrokeCount = {
'',
'',
'',
'',
''
],
5: [
'',
@@ -789,7 +790,7 @@ const Map<int, Map<int, List<String>>> jouyouKanjiByGradeAndStrokeCount = {
'',
'',
'',
'',
''
],
6: [
'',
@@ -830,7 +831,7 @@ const Map<int, Map<int, List<String>>> jouyouKanjiByGradeAndStrokeCount = {
'',
'',
'',
'',
''
],
7: [
'',
@@ -895,7 +896,7 @@ const Map<int, Map<int, List<String>>> jouyouKanjiByGradeAndStrokeCount = {
'',
'',
'',
'',
''
],
8: [
'',
@@ -988,7 +989,7 @@ const Map<int, Map<int, List<String>>> jouyouKanjiByGradeAndStrokeCount = {
'',
'',
'',
'',
''
],
9: [
'',
@@ -1080,7 +1081,7 @@ const Map<int, Map<int, List<String>>> jouyouKanjiByGradeAndStrokeCount = {
'',
'',
'',
'',
''
],
10: [
'',
@@ -1205,7 +1206,7 @@ const Map<int, Map<int, List<String>>> jouyouKanjiByGradeAndStrokeCount = {
'',
'',
'',
'',
''
],
11: [
'',
@@ -1322,7 +1323,7 @@ const Map<int, Map<int, List<String>>> jouyouKanjiByGradeAndStrokeCount = {
'',
'',
'',
'',
''
],
12: [
'',
@@ -1434,7 +1435,7 @@ const Map<int, Map<int, List<String>>> jouyouKanjiByGradeAndStrokeCount = {
'',
'',
'',
'',
''
],
13: [
'',
@@ -1551,7 +1552,7 @@ const Map<int, Map<int, List<String>>> jouyouKanjiByGradeAndStrokeCount = {
'',
'',
'',
'',
''
],
14: [
'',
@@ -1616,7 +1617,7 @@ const Map<int, Map<int, List<String>>> jouyouKanjiByGradeAndStrokeCount = {
'',
'',
'',
'',
''
],
15: [
'',
@@ -1705,7 +1706,7 @@ const Map<int, Map<int, List<String>>> jouyouKanjiByGradeAndStrokeCount = {
'',
'',
'',
'',
''
],
16: [
'',
@@ -1763,7 +1764,7 @@ const Map<int, Map<int, List<String>>> jouyouKanjiByGradeAndStrokeCount = {
'',
'',
'',
'',
''
],
17: [
'',
@@ -1800,7 +1801,7 @@ const Map<int, Map<int, List<String>>> jouyouKanjiByGradeAndStrokeCount = {
'',
'',
'',
'',
''
],
18: [
'',
@@ -1829,7 +1830,7 @@ const Map<int, Map<int, List<String>>> jouyouKanjiByGradeAndStrokeCount = {
'',
'',
'',
'',
''
],
19: [
'',
@@ -1850,23 +1851,22 @@ const Map<int, Map<int, List<String>>> jouyouKanjiByGradeAndStrokeCount = {
'',
'',
'',
'',
''
],
20: ['', '', '', '', '', '', '', ''],
21: ['', '', '', '', '', ''],
22: ['', '', ''],
23: [''],
29: [''],
29: ['']
},
};
final Map<int, List<String>> jouyouKanjiByGrades =
jouyouKanjiByGradeAndStrokeCount.entries
final Map<int, List<String>> JOUYOU_KANJI_BY_GRADES =
JOUYOU_KANJI_BY_GRADE_AND_STROKE_COUNT.entries
.expand((entry) => entry.value.entries)
.map((entry) => MapEntry(entry.key, entry.value))
.fold<Map<int, List<String>>>(
{},
(acc, entry) => acc
..putIfAbsent(entry.key, () => [])
..update(entry.key, (value) => value..addAll(entry.value)),
);
{},
(acc, entry) => acc
..putIfAbsent(entry.key, () => [])
..update(entry.key, (value) => value..addAll(entry.value)));

View File

@@ -1,4 +1,4 @@
const Map<int, List<String>> radicals = {
const Map<int, List<String>> RADICALS = {
1: ['', '', '', '', '', ''],
2: [
'',
@@ -31,7 +31,7 @@ const Map<int, List<String>> radicals = {
'',
'',
'',
'𠂉',
'𠂉'
],
3: [
'',
@@ -78,7 +78,7 @@ const Map<int, List<String>> radicals = {
'',
'',
'',
'',
''
],
4: [
'',
@@ -124,7 +124,7 @@ const Map<int, List<String>> radicals = {
'',
'',
'',
'',
''
],
5: [
'',
@@ -154,7 +154,7 @@ const Map<int, List<String>> radicals = {
'',
'',
'',
'',
''
],
6: [
'',
@@ -181,7 +181,7 @@ const Map<int, List<String>> radicals = {
'',
'',
'',
'西',
'西'
],
7: [
'',
@@ -204,7 +204,7 @@ const Map<int, List<String>> radicals = {
'',
'',
'',
'',
''
],
8: ['', '', '', '', '', '', '', '', '', '', '', ''],
9: ['', '', '', '', '', '', '', '', '', '', ''],

View File

@@ -43,7 +43,6 @@ enum JlptLevel implements Comparable<JlptLevel> {
int? get asInt =>
this == JlptLevel.none ? null : JlptLevel.values.indexOf(this);
@override
String toString() => toNullableString() ?? 'N/A';
Object? toJson() => toNullableString();

View File

@@ -11,7 +11,7 @@ String migrationDirPath() {
}
Future<void> createEmptyDb(DatabaseExecutor db) async {
final List<String> migrationFiles = [];
List<String> migrationFiles = [];
for (final file in Directory(migrationDirPath()).listSync()) {
if (file is File && file.path.endsWith('.sql')) {
migrationFiles.add(file.path);

View File

@@ -19,14 +19,20 @@ enum JMdictDialect {
final String id;
final String description;
const JMdictDialect({required this.id, required this.description});
const JMdictDialect({
required this.id,
required this.description,
});
static JMdictDialect fromId(String id) => JMdictDialect.values.firstWhere(
(e) => e.id == id,
orElse: () => throw Exception('Unknown id: $id'),
);
(e) => e.id == id,
orElse: () => throw Exception('Unknown id: $id'),
);
Map<String, Object?> toJson() => {'id': id, 'description': description};
Map<String, Object?> toJson() => {
'id': id,
'description': description,
};
static JMdictDialect fromJson(Map<String, Object?> json) =>
JMdictDialect.values.firstWhere(

View File

@@ -102,14 +102,20 @@ enum JMdictField {
final String id;
final String description;
const JMdictField({required this.id, required this.description});
const JMdictField({
required this.id,
required this.description,
});
static JMdictField fromId(String id) => JMdictField.values.firstWhere(
(e) => e.id == id,
orElse: () => throw Exception('Unknown id: $id'),
);
(e) => e.id == id,
orElse: () => throw Exception('Unknown id: $id'),
);
Map<String, Object?> toJson() => {'id': id, 'description': description};
Map<String, Object?> toJson() => {
'id': id,
'description': description,
};
static JMdictField fromJson(Map<String, Object?> json) =>
JMdictField.values.firstWhere(

View File

@@ -13,14 +13,20 @@ enum JMdictKanjiInfo {
final String id;
final String description;
const JMdictKanjiInfo({required this.id, required this.description});
const JMdictKanjiInfo({
required this.id,
required this.description,
});
static JMdictKanjiInfo fromId(String id) => JMdictKanjiInfo.values.firstWhere(
(e) => e.id == id,
orElse: () => throw Exception('Unknown id: $id'),
);
(e) => e.id == id,
orElse: () => throw Exception('Unknown id: $id'),
);
Map<String, Object?> toJson() => {'id': id, 'description': description};
Map<String, Object?> toJson() => {
'id': id,
'description': description,
};
static JMdictKanjiInfo fromJson(Map<String, Object?> json) =>
JMdictKanjiInfo.values.firstWhere(

View File

@@ -74,14 +74,20 @@ enum JMdictMisc {
final String id;
final String description;
const JMdictMisc({required this.id, required this.description});
const JMdictMisc({
required this.id,
required this.description,
});
static JMdictMisc fromId(String id) => JMdictMisc.values.firstWhere(
(e) => e.id == id,
orElse: () => throw Exception('Unknown id: $id'),
);
(e) => e.id == id,
orElse: () => throw Exception('Unknown id: $id'),
);
Map<String, Object?> toJson() => {'id': id, 'description': description};
Map<String, Object?> toJson() => {
'id': id,
'description': description,
};
static JMdictMisc fromJson(Map<String, Object?> json) =>
JMdictMisc.values.firstWhere(

View File

@@ -202,11 +202,14 @@ enum JMdictPOS {
String get shortDescription => _shortDescription ?? description;
static JMdictPOS fromId(String id) => JMdictPOS.values.firstWhere(
(e) => e.id == id,
orElse: () => throw Exception('Unknown id: $id'),
);
(e) => e.id == id,
orElse: () => throw Exception('Unknown id: $id'),
);
Map<String, Object?> toJson() => {'id': id, 'description': description};
Map<String, Object?> toJson() => {
'id': id,
'description': description,
};
static JMdictPOS fromJson(Map<String, Object?> json) =>
JMdictPOS.values.firstWhere(

View File

@@ -15,7 +15,10 @@ enum JMdictReadingInfo {
final String id;
final String description;
const JMdictReadingInfo({required this.id, required this.description});
const JMdictReadingInfo({
required this.id,
required this.description,
});
static JMdictReadingInfo fromId(String id) =>
JMdictReadingInfo.values.firstWhere(
@@ -23,7 +26,10 @@ enum JMdictReadingInfo {
orElse: () => throw Exception('Unknown id: $id'),
);
Map<String, Object?> toJson() => {'id': id, 'description': description};
Map<String, Object?> toJson() => {
'id': id,
'description': description,
};
static JMdictReadingInfo fromJson(Map<String, Object?> json) =>
JMdictReadingInfo.values.firstWhere(

View File

@@ -26,14 +26,19 @@ class KanjiSearchRadical extends Equatable {
});
@override
List<Object> get props => [symbol, names, forms, meanings];
List<Object> get props => [
symbol,
this.names,
forms,
meanings,
];
Map<String, dynamic> toJson() => {
'symbol': symbol,
'names': names,
'forms': forms,
'meanings': meanings,
};
'symbol': symbol,
'names': names,
'forms': forms,
'meanings': meanings,
};
factory KanjiSearchRadical.fromJson(Map<String, dynamic> json) {
return KanjiSearchRadical(

View File

@@ -89,46 +89,46 @@ class KanjiSearchResult extends Equatable {
@override
// ignore: public_member_api_docs
List<Object?> get props => [
taughtIn,
jlptLevel,
newspaperFrequencyRank,
strokeCount,
meanings,
kunyomi,
onyomi,
// kunyomiExamples,
// onyomiExamples,
radical,
parts,
codepoints,
kanji,
nanori,
alternativeLanguageReadings,
strokeMiscounts,
queryCodes,
dictionaryReferences,
];
taughtIn,
jlptLevel,
newspaperFrequencyRank,
strokeCount,
meanings,
kunyomi,
onyomi,
// kunyomiExamples,
// onyomiExamples,
radical,
parts,
codepoints,
kanji,
nanori,
alternativeLanguageReadings,
strokeMiscounts,
queryCodes,
dictionaryReferences,
];
Map<String, dynamic> toJson() => {
'kanji': kanji,
'taughtIn': taughtIn,
'jlptLevel': jlptLevel,
'newspaperFrequencyRank': newspaperFrequencyRank,
'strokeCount': strokeCount,
'meanings': meanings,
'kunyomi': kunyomi,
'onyomi': onyomi,
// 'onyomiExamples': onyomiExamples,
// 'kunyomiExamples': kunyomiExamples,
'radical': radical?.toJson(),
'parts': parts,
'codepoints': codepoints,
'nanori': nanori,
'alternativeLanguageReadings': alternativeLanguageReadings,
'strokeMiscounts': strokeMiscounts,
'queryCodes': queryCodes,
'dictionaryReferences': dictionaryReferences,
};
'kanji': kanji,
'taughtIn': taughtIn,
'jlptLevel': jlptLevel,
'newspaperFrequencyRank': newspaperFrequencyRank,
'strokeCount': strokeCount,
'meanings': meanings,
'kunyomi': kunyomi,
'onyomi': onyomi,
// 'onyomiExamples': onyomiExamples,
// 'kunyomiExamples': kunyomiExamples,
'radical': radical?.toJson(),
'parts': parts,
'codepoints': codepoints,
'nanori': nanori,
'alternativeLanguageReadings': alternativeLanguageReadings,
'strokeMiscounts': strokeMiscounts,
'queryCodes': queryCodes,
'dictionaryReferences': dictionaryReferences,
};
factory KanjiSearchResult.fromJson(Map<String, dynamic> json) {
return KanjiSearchResult(
@@ -156,20 +156,23 @@ class KanjiSearchResult extends Equatable {
nanori: (json['nanori'] as List).map((e) => e as String).toList(),
alternativeLanguageReadings:
(json['alternativeLanguageReadings'] as Map<String, dynamic>).map(
(key, value) =>
MapEntry(key, (value as List).map((e) => e as String).toList()),
),
strokeMiscounts: (json['strokeMiscounts'] as List)
.map((e) => e as int)
.toList(),
(key, value) => MapEntry(
key,
(value as List).map((e) => e as String).toList(),
),
),
strokeMiscounts:
(json['strokeMiscounts'] as List).map((e) => e as int).toList(),
queryCodes: (json['queryCodes'] as Map<String, dynamic>).map(
(key, value) =>
MapEntry(key, (value as List).map((e) => e as String).toList()),
(key, value) => MapEntry(
key,
(value as List).map((e) => e as String).toList(),
),
),
dictionaryReferences:
(json['dictionaryReferences'] as Map<String, dynamic>).map(
(key, value) => MapEntry(key, value as String),
),
(key, value) => MapEntry(key, value as String),
),
);
}
}

View File

@@ -7,14 +7,14 @@ import 'package:sqflite_common/sqlite_api.dart';
Future<void> verifyTablesWithDbConnection(DatabaseExecutor db) async {
final Set<String> tables = await db
.query(
'sqlite_master',
columns: ['name'],
where: 'type = ?',
whereArgs: ['table'],
)
'sqlite_master',
columns: ['name'],
where: 'type = ?',
whereArgs: ['table'],
)
.then((result) {
return result.map((row) => row['name'] as String).toSet();
});
return result.map((row) => row['name'] as String).toSet();
});
final Set<String> expectedTables = {
...JMdictTableNames.allTables,
@@ -26,16 +26,14 @@ Future<void> verifyTablesWithDbConnection(DatabaseExecutor db) async {
final missingTables = expectedTables.difference(tables);
if (missingTables.isNotEmpty) {
throw Exception(
[
'Missing tables:',
missingTables.map((table) => ' - $table').join('\n'),
'',
'Found tables:\n',
tables.map((table) => ' - $table').join('\n'),
'',
'Please ensure the database is correctly set up.',
].join('\n'),
);
throw Exception([
'Missing tables:',
missingTables.map((table) => ' - $table').join('\n'),
'',
'Found tables:\n',
tables.map((table) => ' - $table').join('\n'),
'',
'Please ensure the database is correctly set up.',
].join('\n'));
}
}

View File

@@ -1,62 +0,0 @@
enum WordSearchMatchSpanType { kanji, kana, sense }
/// A span of a word search result that corresponds to a match for a kanji, kana, or sense.
class WordSearchMatchSpan {
/// Which subtype of the word search result this span corresponds to - either a kanji, a kana, or a sense.
final WordSearchMatchSpanType spanType;
/// The index of the kanji/kana/sense in the word search result that this span corresponds to.
final int index;
/// When matching a 'sense', this is the index of the English definition in that sense that this span corresponds to. Otherwise, this is always 0.
final int subIndex;
/// The start of the span (inclusive)
final int start;
/// The end of the span (inclusive)
final int end;
WordSearchMatchSpan({
required this.spanType,
required this.index,
required this.start,
required this.end,
this.subIndex = 0,
});
@override
String toString() {
return 'WordSearchMatchSpan(spanType: $spanType, index: $index, start: $start, end: $end)';
}
Map<String, Object?> toJson() => {
'spanType': spanType.toString().split('.').last,
'index': index,
'start': start,
'end': end,
};
factory WordSearchMatchSpan.fromJson(Map<String, dynamic> json) =>
WordSearchMatchSpan(
spanType: WordSearchMatchSpanType.values.firstWhere(
(e) => e.toString().split('.').last == json['spanType'],
),
index: json['index'] as int,
start: json['start'] as int,
end: json['end'] as int,
);
@override
int get hashCode => Object.hash(spanType, index, start, end);
@override
bool operator ==(Object other) {
if (identical(this, other)) return true;
return other is WordSearchMatchSpan &&
other.spanType == spanType &&
other.index == index &&
other.start == start &&
other.end == end;
}
}

View File

@@ -1,12 +1,9 @@
import 'package:jadb/models/common/jlpt_level.dart';
import 'package:jadb/models/jmdict/jmdict_kanji_info.dart';
import 'package:jadb/models/jmdict/jmdict_reading_info.dart';
import 'package:jadb/models/word_search/word_search_match_span.dart';
import 'package:jadb/models/word_search/word_search_ruby.dart';
import 'package:jadb/models/word_search/word_search_sense.dart';
import 'package:jadb/models/word_search/word_search_sources.dart';
import 'package:jadb/search/word_search/word_search.dart';
import 'package:jadb/util/romaji_transliteration.dart';
/// A class representing a single dictionary entry from a word search.
class WordSearchResult {
@@ -37,44 +34,7 @@ class WordSearchResult {
/// A class listing the sources used to make up the data for this word search result.
final WordSearchSources sources;
/// A list of spans, specifying which part of this word result matched the search keyword.
///
/// Note that this is considered ephemeral data - it does not originate from the dictionary,
/// and unlike the rest of the class it varies based on external information (the searchword).
/// It will *NOT* be exported to JSON, but can be reinferred by invoking [inferMatchSpans] with
/// the original searchword.
List<WordSearchMatchSpan>? matchSpans;
/// All contents of [japanese], transliterated to romaji
List<String> get romaji => japanese
.map((word) => transliterateKanaToLatin(word.furigana ?? word.base))
.toList();
/// All contents of [japanase], where the furigana has either been transliterated to romaji, or
/// contains the furigana transliteration of [WordSearchRuby.base].
List<WordSearchRuby> get romajiRubys => japanese
.map(
(word) => WordSearchRuby(
base: word.base,
furigana: word.furigana != null
? transliterateKanaToLatin(word.furigana!)
: transliterateKanaToLatin(word.base),
),
)
.toList();
/// The same list of spans as [matchSpans], but the positions have been adjusted for romaji conversion
///
/// This is mostly useful in conjunction with [romajiRubys].
List<WordSearchMatchSpan>? get romajiMatchSpans {
if (matchSpans == null) {
return null;
}
throw UnimplementedError('Not yet implemented');
}
WordSearchResult({
const WordSearchResult({
required this.score,
required this.entryId,
required this.isCommon,
@@ -84,22 +44,21 @@ class WordSearchResult {
required this.senses,
required this.jlptLevel,
required this.sources,
this.matchSpans,
});
Map<String, dynamic> toJson() => {
'_score': score,
'entryId': entryId,
'isCommon': isCommon,
'japanese': japanese.map((e) => e.toJson()).toList(),
'kanjiInfo': kanjiInfo.map((key, value) => MapEntry(key, value.toJson())),
'readingInfo': readingInfo.map(
(key, value) => MapEntry(key, value.toJson()),
),
'senses': senses.map((e) => e.toJson()).toList(),
'jlptLevel': jlptLevel.toJson(),
'sources': sources.toJson(),
};
'_score': score,
'entryId': entryId,
'isCommon': isCommon,
'japanese': japanese.map((e) => e.toJson()).toList(),
'kanjiInfo':
kanjiInfo.map((key, value) => MapEntry(key, value.toJson())),
'readingInfo':
readingInfo.map((key, value) => MapEntry(key, value.toJson())),
'senses': senses.map((e) => e.toJson()).toList(),
'jlptLevel': jlptLevel.toJson(),
'sources': sources.toJson(),
};
factory WordSearchResult.fromJson(Map<String, dynamic> json) =>
WordSearchResult(
@@ -122,88 +81,17 @@ class WordSearchResult {
sources: WordSearchSources.fromJson(json['sources']),
);
factory WordSearchResult.empty() => WordSearchResult(
score: 0,
entryId: 0,
isCommon: false,
japanese: [],
kanjiInfo: {},
readingInfo: {},
senses: [],
jlptLevel: JlptLevel.none,
sources: WordSearchSources.empty(),
);
/// Infers which part(s) of this word search result matched the search keyword, and populates [matchSpans] accordingly.
void inferMatchSpans(
String searchword, {
SearchMode searchMode = SearchMode.auto,
}) {
// TODO: handle wildcards like '?' and '*' when that becomes supported in the search.
// TODO: If the searchMode is provided, we can use that to narrow down which part of the word search results to look at.
final regex = RegExp(RegExp.escape(searchword));
final matchSpans = <WordSearchMatchSpan>[];
for (final (i, japanese) in japanese.indexed) {
final baseMatches = regex.allMatches(japanese.base);
matchSpans.addAll(
baseMatches.map(
(match) => WordSearchMatchSpan(
spanType: WordSearchMatchSpanType.kanji,
index: i,
start: match.start,
end: match.end,
),
),
);
if (japanese.furigana != null) {
final furiganaMatches = regex.allMatches(japanese.furigana!);
matchSpans.addAll(
furiganaMatches.map(
(match) => WordSearchMatchSpan(
spanType: WordSearchMatchSpanType.kana,
index: i,
start: match.start,
end: match.end,
),
),
);
}
}
for (final (i, sense) in senses.indexed) {
for (final (k, definition) in sense.englishDefinitions.indexed) {
final definitionMatches = regex.allMatches(definition);
matchSpans.addAll(
definitionMatches.map(
(match) => WordSearchMatchSpan(
spanType: WordSearchMatchSpanType.sense,
index: i,
subIndex: k,
start: match.start,
end: match.end,
),
),
);
}
}
this.matchSpans = matchSpans;
}
static String _formatJapaneseWord(WordSearchRuby word) =>
word.furigana == null ? word.base : '${word.base} (${word.furigana})';
String _formatJapaneseWord(WordSearchRuby word) =>
word.furigana == null ? word.base : "${word.base} (${word.furigana})";
@override
String toString() {
final japaneseWord = _formatJapaneseWord(japanese[0]);
final isCommonString = isCommon ? '(C)' : '';
final jlptLevelString = '(${jlptLevel.toString()})';
final jlptLevelString = "(${jlptLevel.toString()})";
return '''
$score | [$entryId] $japaneseWord $isCommonString $jlptLevelString
${score} | [$entryId] $japaneseWord $isCommonString $jlptLevelString
Other forms: ${japanese.skip(1).map(_formatJapaneseWord).join(', ')}
Senses: ${senses.map((s) => s.englishDefinitions).join(', ')}
'''

View File

@@ -6,12 +6,18 @@ class WordSearchRuby {
/// Furigana, if applicable.
String? furigana;
WordSearchRuby({required this.base, this.furigana});
WordSearchRuby({
required this.base,
this.furigana,
});
Map<String, dynamic> toJson() => {'base': base, 'furigana': furigana};
Map<String, dynamic> toJson() => {
'base': base,
'furigana': furigana,
};
factory WordSearchRuby.fromJson(Map<String, dynamic> json) => WordSearchRuby(
base: json['base'] as String,
furigana: json['furigana'] as String?,
);
base: json['base'] as String,
furigana: json['furigana'] as String?,
);
}

View File

@@ -71,18 +71,18 @@ class WordSearchSense {
languageSource.isEmpty;
Map<String, dynamic> toJson() => {
'englishDefinitions': englishDefinitions,
'partsOfSpeech': partsOfSpeech.map((e) => e.toJson()).toList(),
'seeAlso': seeAlso.map((e) => e.toJson()).toList(),
'antonyms': antonyms.map((e) => e.toJson()).toList(),
'restrictedToReading': restrictedToReading,
'restrictedToKanji': restrictedToKanji,
'fields': fields.map((e) => e.toJson()).toList(),
'dialects': dialects.map((e) => e.toJson()).toList(),
'misc': misc.map((e) => e.toJson()).toList(),
'info': info,
'languageSource': languageSource,
};
'englishDefinitions': englishDefinitions,
'partsOfSpeech': partsOfSpeech.map((e) => e.toJson()).toList(),
'seeAlso': seeAlso.map((e) => e.toJson()).toList(),
'antonyms': antonyms.map((e) => e.toJson()).toList(),
'restrictedToReading': restrictedToReading,
'restrictedToKanji': restrictedToKanji,
'fields': fields.map((e) => e.toJson()).toList(),
'dialects': dialects.map((e) => e.toJson()).toList(),
'misc': misc.map((e) => e.toJson()).toList(),
'info': info,
'languageSource': languageSource,
};
factory WordSearchSense.fromJson(Map<String, dynamic> json) =>
WordSearchSense(
@@ -104,9 +104,8 @@ class WordSearchSense {
dialects: (json['dialects'] as List)
.map((e) => JMdictDialect.fromJson(e))
.toList(),
misc: (json['misc'] as List)
.map((e) => JMdictMisc.fromJson(e))
.toList(),
misc:
(json['misc'] as List).map((e) => JMdictMisc.fromJson(e)).toList(),
info: List<String>.from(json['info']),
languageSource: (json['languageSource'] as List)
.map((e) => WordSearchSenseLanguageSource.fromJson(e))

View File

@@ -13,11 +13,11 @@ class WordSearchSenseLanguageSource {
});
Map<String, Object?> toJson() => {
'language': language,
'phrase': phrase,
'fullyDescribesSense': fullyDescribesSense,
'constructedFromSmallerWords': constructedFromSmallerWords,
};
'language': language,
'phrase': phrase,
'fullyDescribesSense': fullyDescribesSense,
'constructedFromSmallerWords': constructedFromSmallerWords,
};
factory WordSearchSenseLanguageSource.fromJson(Map<String, dynamic> json) =>
WordSearchSenseLanguageSource(

View File

@@ -7,13 +7,20 @@ class WordSearchSources {
/// Whether JMnedict was used.
final bool jmnedict;
const WordSearchSources({this.jmdict = true, this.jmnedict = false});
const WordSearchSources({
this.jmdict = true,
this.jmnedict = false,
});
factory WordSearchSources.empty() => const WordSearchSources();
Map<String, Object?> get sqlValue => {
'jmdict': jmdict,
'jmnedict': jmnedict,
};
Map<String, Object?> get sqlValue => {'jmdict': jmdict, 'jmnedict': jmnedict};
Map<String, dynamic> toJson() => {'jmdict': jmdict, 'jmnedict': jmnedict};
Map<String, dynamic> toJson() => {
'jmdict': jmdict,
'jmnedict': jmnedict,
};
factory WordSearchSources.fromJson(Map<String, dynamic> json) =>
WordSearchSources(

View File

@@ -1,5 +1,3 @@
import 'package:jadb/models/word_search/word_search_result.dart';
/// A cross-reference entry from one word-result to another entry.
class WordSearchXrefEntry {
/// The ID of the entry that this entry cross-references to.
@@ -15,24 +13,19 @@ class WordSearchXrefEntry {
/// database (and hence might be incorrect).
final bool ambiguous;
/// The result of the cross-reference, may or may not be included in the query.
final WordSearchResult? xrefResult;
const WordSearchXrefEntry({
required this.entryId,
required this.ambiguous,
required this.baseWord,
required this.furigana,
required this.xrefResult,
});
Map<String, dynamic> toJson() => {
'entryId': entryId,
'ambiguous': ambiguous,
'baseWord': baseWord,
'furigana': furigana,
'xrefResult': xrefResult?.toJson(),
};
'entryId': entryId,
'ambiguous': ambiguous,
'baseWord': baseWord,
'furigana': furigana,
};
factory WordSearchXrefEntry.fromJson(Map<String, dynamic> json) =>
WordSearchXrefEntry(
@@ -40,6 +33,5 @@ class WordSearchXrefEntry {
ambiguous: json['ambiguous'] as bool,
baseWord: json['baseWord'] as String,
furigana: json['furigana'] as String?,
xrefResult: null,
);
}

View File

@@ -1,10 +1,12 @@
import 'package:jadb/models/kanji_search/kanji_search_result.dart';
import 'package:jadb/models/verify_tables.dart';
import 'package:jadb/models/word_search/word_search_result.dart';
import 'package:jadb/models/kanji_search/kanji_search_result.dart';
import 'package:jadb/search/filter_kanji.dart';
import 'package:jadb/search/kanji_search.dart';
import 'package:jadb/search/radical_search.dart';
import 'package:jadb/search/word_search/word_search.dart';
import 'package:jadb/search/kanji_search.dart';
import 'package:sqflite_common/sqlite_api.dart';
extension JaDBConnection on DatabaseExecutor {
@@ -17,45 +19,38 @@ extension JaDBConnection on DatabaseExecutor {
Future<KanjiSearchResult?> jadbSearchKanji(String kanji) =>
searchKanjiWithDbConnection(this, kanji);
/// Search for a kanji in the database.
Future<Map<String, KanjiSearchResult>> jadbGetManyKanji(Set<String> kanji) =>
searchManyKanjiWithDbConnection(this, kanji);
/// Filter a list of characters, and return the ones that are listed in the kanji dictionary.
Future<List<String>> filterKanji(
List<String> kanji, {
bool deduplicate = false,
}) => filterKanjiWithDbConnection(this, kanji, deduplicate);
}) =>
filterKanjiWithDbConnection(this, kanji, deduplicate);
/// Search for a word in the database.
Future<List<WordSearchResult>?> jadbSearchWord(
String word, {
SearchMode searchMode = SearchMode.auto,
SearchMode searchMode = SearchMode.Auto,
int page = 0,
int? pageSize,
}) => searchWordWithDbConnection(
this,
word,
searchMode: searchMode,
page: page,
pageSize: pageSize,
);
int pageSize = 10,
}) =>
searchWordWithDbConnection(
this,
word,
searchMode,
page,
pageSize,
);
///
Future<WordSearchResult?> jadbGetWordById(int id) =>
getWordByIdWithDbConnection(this, id);
/// Get a list of words by their IDs.
///
/// IDs for which no result is found are omitted from the returned value.
Future<Map<int, WordSearchResult>> jadbGetManyWordsByIds(Set<int> ids) =>
getWordsByIdsWithDbConnection(this, ids);
/// Search for a word in the database, and return the count of results.
Future<int?> jadbSearchWordCount(
String word, {
SearchMode searchMode = SearchMode.auto,
}) => searchWordCountWithDbConnection(this, word, searchMode: searchMode);
SearchMode searchMode = SearchMode.Auto,
}) =>
searchWordCountWithDbConnection(this, word, searchMode);
/// Given a list of radicals, search which kanji contains all
/// of the radicals, find their other radicals, and return those.

View File

@@ -1,21 +1,19 @@
import 'package:jadb/table_names/kanjidic.dart';
import 'package:sqflite_common/sqflite.dart';
/// Filters a list of kanji characters, returning only those that exist in the database.
///
/// If [deduplicate] is true, the returned list will deduplicate the input kanji list before returning the filtered results.
Future<List<String>> filterKanjiWithDbConnection(
DatabaseExecutor connection,
List<String> kanji,
bool deduplicate,
) async {
final Set<String> filteredKanji = await connection
.rawQuery('''
final Set<String> filteredKanji = await connection.rawQuery(
'''
SELECT "literal"
FROM "${KANJIDICTableNames.character}"
WHERE "literal" IN (${kanji.map((_) => '?').join(',')})
''', kanji)
.then((value) => value.map((e) => e['literal'] as String).toSet());
''',
kanji,
).then((value) => value.map((e) => e['literal'] as String).toSet());
if (deduplicate) {
return filteredKanji.toList();

View File

@@ -1,190 +1,144 @@
import 'package:collection/collection.dart';
import 'package:jadb/models/kanji_search/kanji_search_radical.dart';
import 'package:jadb/models/kanji_search/kanji_search_result.dart';
import 'package:jadb/table_names/kanjidic.dart';
import 'package:jadb/table_names/radkfile.dart';
import 'package:jadb/models/kanji_search/kanji_search_radical.dart';
import 'package:jadb/models/kanji_search/kanji_search_result.dart';
import 'package:jadb/util/romaji_transliteration.dart';
import 'package:sqflite_common/sqflite.dart';
Future<List<Map<String, Object?>>> _charactersQuery(
DatabaseExecutor connection,
String kanji,
) => connection.query(
KANJIDICTableNames.character,
where: 'literal = ?',
whereArgs: [kanji],
);
Future<List<Map<String, Object?>>> _codepointsQuery(
DatabaseExecutor connection,
String kanji,
) => connection.query(
KANJIDICTableNames.codepoint,
where: 'kanji = ?',
whereArgs: [kanji],
);
Future<List<Map<String, Object?>>> _kunyomisQuery(
DatabaseExecutor connection,
String kanji,
) => connection.query(
KANJIDICTableNames.kunyomi,
where: 'kanji = ?',
whereArgs: [kanji],
orderBy: 'orderNum',
);
Future<List<Map<String, Object?>>> _onyomisQuery(
DatabaseExecutor connection,
String kanji,
) => connection.query(
KANJIDICTableNames.onyomi,
where: 'kanji = ?',
whereArgs: [kanji],
orderBy: 'orderNum',
);
Future<List<Map<String, Object?>>> _meaningsQuery(
DatabaseExecutor connection,
String kanji,
) => connection.query(
KANJIDICTableNames.meaning,
where: 'kanji = ? AND language = ?',
whereArgs: [kanji, 'eng'],
orderBy: 'orderNum',
);
Future<List<Map<String, Object?>>> _nanorisQuery(
DatabaseExecutor connection,
String kanji,
) => connection.query(
KANJIDICTableNames.nanori,
where: 'kanji = ?',
whereArgs: [kanji],
);
Future<List<Map<String, Object?>>> _dictionaryReferencesQuery(
DatabaseExecutor connection,
String kanji,
) => connection.query(
KANJIDICTableNames.dictionaryReference,
where: 'kanji = ?',
whereArgs: [kanji],
);
Future<List<Map<String, Object?>>> _queryCodesQuery(
DatabaseExecutor connection,
String kanji,
) => connection.query(
KANJIDICTableNames.queryCode,
where: 'kanji = ?',
whereArgs: [kanji],
);
Future<List<Map<String, Object?>>> _radicalsQuery(
DatabaseExecutor connection,
String kanji,
) => connection.rawQuery(
'''
SELECT DISTINCT
"XREF__KANJIDIC_Radical__RADKFILE"."radicalSymbol" AS "symbol",
"names"
FROM "${KANJIDICTableNames.radical}"
JOIN "XREF__KANJIDIC_Radical__RADKFILE" USING ("radicalId")
LEFT JOIN (
SELECT "radicalId", group_concat("name") AS "names"
FROM "${KANJIDICTableNames.radicalName}"
GROUP BY "radicalId"
) USING ("radicalId")
WHERE "${KANJIDICTableNames.radical}"."kanji" = ?
''',
[kanji],
);
Future<List<Map<String, Object?>>> _partsQuery(
DatabaseExecutor connection,
String kanji,
) => connection.query(
RADKFILETableNames.radkfile,
where: 'kanji = ?',
whereArgs: [kanji],
);
Future<List<Map<String, Object?>>> _readingsQuery(
DatabaseExecutor connection,
String kanji,
) => connection.query(
KANJIDICTableNames.reading,
where: 'kanji = ?',
whereArgs: [kanji],
);
Future<List<Map<String, Object?>>> _strokeMiscountsQuery(
DatabaseExecutor connection,
String kanji,
) => connection.query(
KANJIDICTableNames.strokeMiscount,
where: 'kanji = ?',
whereArgs: [kanji],
);
// Future<List<Map<String, Object?>>> _variantsQuery(
// DatabaseExecutor connection,
// String kanji,
// ) => connection.query(
// KANJIDICTableNames.variant,
// where: 'kanji = ?',
// whereArgs: [kanji],
// );
/// Searches for a kanji character and returns its details, or null if the kanji is not found in the database.
Future<KanjiSearchResult?> searchKanjiWithDbConnection(
DatabaseExecutor connection,
String kanji,
) async {
late final List<Map<String, Object?>> characters;
final characters_query = connection.query(
KANJIDICTableNames.character,
where: "literal = ?",
whereArgs: [kanji],
);
late final List<Map<String, Object?>> codepoints;
final codepoints_query = connection.query(
KANJIDICTableNames.codepoint,
where: "kanji = ?",
whereArgs: [kanji],
);
late final List<Map<String, Object?>> kunyomis;
final kunyomis_query = connection.query(
KANJIDICTableNames.kunyomi,
where: "kanji = ?",
whereArgs: [kanji],
orderBy: "orderNum",
);
late final List<Map<String, Object?>> onyomis;
final onyomis_query = connection.query(
KANJIDICTableNames.onyomi,
where: "kanji = ?",
whereArgs: [kanji],
orderBy: "orderNum",
);
late final List<Map<String, Object?>> meanings;
final meanings_query = connection.query(
KANJIDICTableNames.meaning,
where: "kanji = ? AND language = ?",
whereArgs: [kanji, 'eng'],
orderBy: "orderNum",
);
late final List<Map<String, Object?>> nanoris;
late final List<Map<String, Object?>> dictionaryReferences;
late final List<Map<String, Object?>> queryCodes;
final nanoris_query = connection.query(
KANJIDICTableNames.nanori,
where: "kanji = ?",
whereArgs: [kanji],
);
late final List<Map<String, Object?>> dictionary_references;
final dictionary_references_query = connection.query(
KANJIDICTableNames.dictionaryReference,
where: "kanji = ?",
whereArgs: [kanji],
);
late final List<Map<String, Object?>> query_codes;
final query_codes_query = connection.query(
KANJIDICTableNames.queryCode,
where: "kanji = ?",
whereArgs: [kanji],
);
late final List<Map<String, Object?>> radicals;
final radicals_query = connection.rawQuery(
'''
SELECT DISTINCT
"XREF__KANJIDIC_Radical__RADKFILE"."radicalSymbol" AS "symbol",
"names"
FROM "${KANJIDICTableNames.radical}"
JOIN "XREF__KANJIDIC_Radical__RADKFILE" USING ("radicalId")
LEFT JOIN (
SELECT "radicalId", group_concat("name") AS "names"
FROM "${KANJIDICTableNames.radicalName}"
GROUP BY "radicalId"
) USING ("radicalId")
WHERE "${KANJIDICTableNames.radical}"."kanji" = ?
''',
[kanji],
);
late final List<Map<String, Object?>> parts;
final parts_query = connection.query(
RADKFILETableNames.radkfile,
where: "kanji = ?",
whereArgs: [kanji],
);
late final List<Map<String, Object?>> readings;
late final List<Map<String, Object?>> strokeMiscounts;
final readings_query = connection.query(
KANJIDICTableNames.reading,
where: "kanji = ?",
whereArgs: [kanji],
);
late final List<Map<String, Object?>> stroke_miscounts;
final stroke_miscounts_query = connection.query(
KANJIDICTableNames.strokeMiscount,
where: "kanji = ?",
whereArgs: [kanji],
);
// TODO: add variant data to result
// late final List<Map<String, Object?>> variants;
// final variants_query = connection.query(
// KANJIDICTableNames.variant,
// where: "kanji = ?",
// whereArgs: [kanji],
// );
// TODO: Search for kunyomi and onyomi usage of the characters
// from JMDict. We'll need to fuzzy aquery JMDict_KanjiElement for matches,
// filter JMdict_ReadingElement for kunyomi/onyomi, and then sort the main entry
// by JLPT, news frequency, etc.
// TODO: Search for kunyomi and onyomi usage of the characters
// from JMDict. We'll need to fuzzy aquery JMDict_KanjiElement for mathces,
// filter JMdict_ReadingElement for kunyomi/onyomi, and then sort the main entry
// by JLPT, news frequency, etc.
await _charactersQuery(connection, kanji).then((value) => characters = value);
await characters_query.then((value) => characters = value);
if (characters.isEmpty) {
return null;
}
await Future.wait({
_codepointsQuery(connection, kanji).then((value) => codepoints = value),
_kunyomisQuery(connection, kanji).then((value) => kunyomis = value),
_onyomisQuery(connection, kanji).then((value) => onyomis = value),
_meaningsQuery(connection, kanji).then((value) => meanings = value),
_nanorisQuery(connection, kanji).then((value) => nanoris = value),
_dictionaryReferencesQuery(
connection,
kanji,
).then((value) => dictionaryReferences = value),
_queryCodesQuery(connection, kanji).then((value) => queryCodes = value),
_radicalsQuery(connection, kanji).then((value) => radicals = value),
_partsQuery(connection, kanji).then((value) => parts = value),
_readingsQuery(connection, kanji).then((value) => readings = value),
_strokeMiscountsQuery(
connection,
kanji,
).then((value) => strokeMiscounts = value),
codepoints_query.then((value) => codepoints = value),
kunyomis_query.then((value) => kunyomis = value),
onyomis_query.then((value) => onyomis = value),
meanings_query.then((value) => meanings = value),
nanoris_query.then((value) => nanoris = value),
dictionary_references_query.then((value) => dictionary_references = value),
query_codes_query.then((value) => query_codes = value),
radicals_query.then((value) => radicals = value),
parts_query.then((value) => parts = value),
readings_query.then((value) => readings = value),
stroke_miscounts_query.then((value) => stroke_miscounts = value),
// variants_query.then((value) => variants = value),
});
@@ -203,7 +157,9 @@ Future<KanjiSearchResult?> searchKanjiWithDbConnection(
: null;
final alternativeLanguageReadings = readings
.groupListsBy((item) => item['type'] as String)
.groupListsBy(
(item) => item['type'] as String,
)
.map(
(key, value) => MapEntry(
key,
@@ -212,16 +168,20 @@ Future<KanjiSearchResult?> searchKanjiWithDbConnection(
);
// TODO: Add `SKIPMisclassification` to the entries
final queryCodes_ = queryCodes
.groupListsBy((item) => item['type'] as String)
final queryCodes = query_codes
.groupListsBy(
(item) => item['type'] as String,
)
.map(
(key, value) =>
MapEntry(key, value.map((item) => item['code'] as String).toList()),
(key, value) => MapEntry(
key,
value.map((item) => item['code'] as String).toList(),
),
);
// TODO: Add `volume` and `page` to the entries
final dictionaryReferences_ = {
for (final entry in dictionaryReferences)
final dictionaryReferences = {
for (final entry in dictionary_references)
entry['type'] as String: entry['ref'] as String,
};
@@ -242,7 +202,10 @@ Future<KanjiSearchResult?> searchKanjiWithDbConnection(
meanings: meanings.map((item) => item['meaning'] as String).toList(),
kunyomi: kunyomis.map((item) => item['yomi'] as String).toList(),
parts: parts.map((item) => item['radical'] as String).toList(),
onyomi: onyomis.map((item) => item['yomi'] as String).toList(),
onyomi: onyomis
.map((item) => item['yomi'] as String)
.map(transliterateHiraganaToKatakana)
.toList(),
radical: radical,
codepoints: {
for (final codepoint in codepoints)
@@ -250,33 +213,9 @@ Future<KanjiSearchResult?> searchKanjiWithDbConnection(
},
nanori: nanoris.map((item) => item['nanori'] as String).toList(),
alternativeLanguageReadings: alternativeLanguageReadings,
strokeMiscounts: strokeMiscounts
.map((item) => item['strokeCount'] as int)
.toList(),
queryCodes: queryCodes_,
dictionaryReferences: dictionaryReferences_,
strokeMiscounts:
stroke_miscounts.map((item) => item['strokeCount'] as int).toList(),
queryCodes: queryCodes,
dictionaryReferences: dictionaryReferences,
);
}
// TODO: Use fewer queries with `IN` clauses to reduce the number of queries
/// Searches for multiple kanji at once, returning a map of kanji to their search results.
Future<Map<String, KanjiSearchResult>> searchManyKanjiWithDbConnection(
DatabaseExecutor connection,
Set<String> kanji,
) async {
if (kanji.isEmpty) {
return {};
}
final results = <String, KanjiSearchResult>{};
for (final k in kanji) {
final result = await searchKanjiWithDbConnection(connection, k);
if (result != null) {
results[k] = result;
}
}
return results;
}

View File

@@ -3,16 +3,10 @@ import 'package:sqflite_common/sqlite_api.dart';
// TODO: validate that the list of radicals all are valid radicals
/// Returns a list of radicals that are part of any kanji that contains all of the input radicals.
///
/// This can be used to limit the choices of additional radicals provided to a user,
/// so that any choice they make will still yield at least one kanji.
Future<List<String>> searchRemainingRadicalsWithDbConnection(
DatabaseExecutor connection,
List<String> radicals,
) async {
final distinctRadicals = radicals.toSet();
final queryResult = await connection.rawQuery(
'''
SELECT DISTINCT "radical"
@@ -20,37 +14,39 @@ Future<List<String>> searchRemainingRadicalsWithDbConnection(
WHERE "kanji" IN (
SELECT "kanji"
FROM "${RADKFILETableNames.radkfile}"
WHERE "radical" IN (${List.filled(distinctRadicals.length, '?').join(',')})
WHERE "radical" IN (${List.filled(radicals.length, '?').join(',')})
GROUP BY "kanji"
HAVING COUNT(DISTINCT "radical") = ?
)
''',
[...distinctRadicals, distinctRadicals.length],
[
...radicals,
radicals.length,
],
);
final remainingRadicals = queryResult
.map((row) => row['radical'] as String)
.toList();
final remainingRadicals =
queryResult.map((row) => row['radical'] as String).toList();
return remainingRadicals;
}
/// Returns a list of kanji that contain all of the input radicals.
Future<List<String>> searchKanjiByRadicalsWithDbConnection(
DatabaseExecutor connection,
List<String> radicals,
) async {
final distinctRadicals = radicals.toSet();
final queryResult = await connection.rawQuery(
'''
SELECT "kanji"
FROM "${RADKFILETableNames.radkfile}"
WHERE "radical" IN (${List.filled(distinctRadicals.length, '?').join(',')})
WHERE "radical" IN (${List.filled(radicals.length, '?').join(',')})
GROUP BY "kanji"
HAVING COUNT(DISTINCT "radical") = ?
''',
[...distinctRadicals, distinctRadicals.length],
[
...radicals,
radicals.length,
],
);
final kanji = queryResult.map((row) => row['kanji'] as String).toList();

View File

@@ -1,5 +1,6 @@
import 'package:jadb/table_names/jmdict.dart';
import 'package:jadb/table_names/tanos_jlpt.dart';
import 'package:jadb/util/sqlite_utils.dart';
import 'package:sqflite_common/sqflite.dart';
class LinearWordQueryData {
@@ -24,9 +25,6 @@ class LinearWordQueryData {
final List<Map<String, Object?>> readingElementRestrictions;
final List<Map<String, Object?>> kanjiElementInfos;
final LinearWordQueryData? senseAntonymData;
final LinearWordQueryData? senseSeeAlsoData;
const LinearWordQueryData({
required this.senses,
required this.readingElements,
@@ -48,368 +46,242 @@ class LinearWordQueryData {
required this.readingElementInfos,
required this.readingElementRestrictions,
required this.kanjiElementInfos,
required this.senseAntonymData,
required this.senseSeeAlsoData,
});
}
Future<List<Map<String, Object?>>> _sensesQuery(
DatabaseExecutor connection,
List<int> entryIds,
) => connection.query(
JMdictTableNames.sense,
where: 'entryId IN (${List.filled(entryIds.length, '?').join(',')})',
whereArgs: entryIds,
);
Future<List<Map<String, Object?>>> _readingelementsQuery(
DatabaseExecutor connection,
List<int> entryIds,
) => connection.query(
JMdictTableNames.readingElement,
where: 'entryId IN (${List.filled(entryIds.length, '?').join(',')})',
whereArgs: entryIds,
orderBy: 'orderNum',
);
Future<List<Map<String, Object?>>> _kanjielementsQuery(
DatabaseExecutor connection,
List<int> entryIds,
) => connection.query(
JMdictTableNames.kanjiElement,
where: 'entryId IN (${List.filled(entryIds.length, '?').join(',')})',
whereArgs: entryIds,
orderBy: 'orderNum',
);
Future<List<Map<String, Object?>>> _jlpttagsQuery(
DatabaseExecutor connection,
List<int> entryIds,
) => connection.query(
TanosJLPTTableNames.jlptTag,
where: 'entryId IN (${List.filled(entryIds.length, '?').join(',')})',
whereArgs: entryIds,
);
Future<List<Map<String, Object?>>> _commonentriesQuery(
DatabaseExecutor connection,
List<int> entryIds,
) => connection.query(
'JMdict_EntryCommon',
where: 'entryId IN (${List.filled(entryIds.length, '?').join(',')})',
whereArgs: entryIds,
);
// Sense queries
Future<List<Map<String, Object?>>> _senseantonymsQuery(
DatabaseExecutor connection,
List<int> senseIds,
) => connection.rawQuery(
"""
SELECT
"${JMdictTableNames.senseAntonyms}".senseId,
"${JMdictTableNames.senseAntonyms}".ambiguous,
"${JMdictTableNames.senseAntonyms}".xrefEntryId,
"JMdict_BaseAndFurigana"."base",
"JMdict_BaseAndFurigana"."furigana"
FROM "${JMdictTableNames.senseAntonyms}"
JOIN "JMdict_BaseAndFurigana"
ON "${JMdictTableNames.senseAntonyms}"."xrefEntryId" = "JMdict_BaseAndFurigana"."entryId"
WHERE
"senseId" IN (${List.filled(senseIds.length, '?').join(',')})
AND "JMdict_BaseAndFurigana"."isFirst"
ORDER BY
"${JMdictTableNames.senseAntonyms}"."senseId",
"${JMdictTableNames.senseAntonyms}"."xrefEntryId"
""",
[...senseIds],
);
Future<List<Map<String, Object?>>> senseseealsosQuery(
DatabaseExecutor connection,
List<int> senseIds,
) => connection.rawQuery(
"""
SELECT
"${JMdictTableNames.senseSeeAlso}"."senseId",
"${JMdictTableNames.senseSeeAlso}"."ambiguous",
"${JMdictTableNames.senseSeeAlso}"."xrefEntryId",
"JMdict_BaseAndFurigana"."base",
"JMdict_BaseAndFurigana"."furigana"
FROM "${JMdictTableNames.senseSeeAlso}"
JOIN "JMdict_BaseAndFurigana"
ON "${JMdictTableNames.senseSeeAlso}"."xrefEntryId" = "JMdict_BaseAndFurigana"."entryId"
WHERE
"senseId" IN (${List.filled(senseIds.length, '?').join(',')})
AND "JMdict_BaseAndFurigana"."isFirst"
ORDER BY
"${JMdictTableNames.senseSeeAlso}"."senseId",
"${JMdictTableNames.senseSeeAlso}"."xrefEntryId"
""",
[...senseIds],
);
Future<List<Map<String, Object?>>> _sensedialectsQuery(
DatabaseExecutor connection,
List<int> senseIds,
) => connection.query(
JMdictTableNames.senseDialect,
where: 'senseId IN (${List.filled(senseIds.length, '?').join(',')})',
whereArgs: senseIds,
);
Future<List<Map<String, Object?>>> _sensefieldsQuery(
DatabaseExecutor connection,
List<int> senseIds,
) => connection.query(
JMdictTableNames.senseField,
where: 'senseId IN (${List.filled(senseIds.length, '?').join(',')})',
whereArgs: senseIds,
);
Future<List<Map<String, Object?>>> _senseglossariesQuery(
DatabaseExecutor connection,
List<int> senseIds,
) => connection.query(
JMdictTableNames.senseGlossary,
where: 'senseId IN (${List.filled(senseIds.length, '?').join(',')})',
whereArgs: senseIds,
);
Future<List<Map<String, Object?>>> _senseinfosQuery(
DatabaseExecutor connection,
List<int> senseIds,
) => connection.query(
JMdictTableNames.senseInfo,
where: 'senseId IN (${List.filled(senseIds.length, '?').join(',')})',
whereArgs: senseIds,
);
Future<List<Map<String, Object?>>> _senselanguagesourcesQuery(
DatabaseExecutor connection,
List<int> senseIds,
) => connection.query(
JMdictTableNames.senseLanguageSource,
where: 'senseId IN (${List.filled(senseIds.length, '?').join(',')})',
whereArgs: senseIds,
);
Future<List<Map<String, Object?>>> _sensemiscsQuery(
DatabaseExecutor connection,
List<int> senseIds,
) => connection.query(
JMdictTableNames.senseMisc,
where: 'senseId IN (${List.filled(senseIds.length, '?').join(',')})',
whereArgs: senseIds,
);
Future<List<Map<String, Object?>>> _sensepossQuery(
DatabaseExecutor connection,
List<int> senseIds,
) => connection.query(
JMdictTableNames.sensePOS,
where: 'senseId IN (${List.filled(senseIds.length, '?').join(',')})',
whereArgs: senseIds,
);
Future<List<Map<String, Object?>>> _senserestrictedtokanjisQuery(
DatabaseExecutor connection,
List<int> senseIds,
) => connection.query(
JMdictTableNames.senseRestrictedToKanji,
where: 'senseId IN (${List.filled(senseIds.length, '?').join(',')})',
whereArgs: senseIds,
);
Future<List<Map<String, Object?>>> _senserestrictedtoreadingsQuery(
DatabaseExecutor connection,
List<int> senseIds,
) => connection.query(
JMdictTableNames.senseRestrictedToReading,
where: 'senseId IN (${List.filled(senseIds.length, '?').join(',')})',
whereArgs: senseIds,
);
Future<List<Map<String, Object?>>> _examplesentencesQuery(
DatabaseExecutor connection,
List<int> senseIds,
) => connection.query(
'JMdict_ExampleSentence',
where: 'senseId IN (${List.filled(senseIds.length, '?').join(',')})',
whereArgs: senseIds,
);
// Reading/kanji elements queries
Future<List<Map<String, Object?>>> _readingelementinfosQuery(
DatabaseExecutor connection,
List<int> readingIds,
) => connection.query(
JMdictTableNames.readingInfo,
where: '(elementId) IN (${List.filled(readingIds.length, '?').join(',')})',
whereArgs: readingIds,
);
Future<List<Map<String, Object?>>> _readingelementrestrictionsQuery(
DatabaseExecutor connection,
List<int> readingIds,
) => connection.query(
JMdictTableNames.readingRestriction,
where: '(elementId) IN (${List.filled(readingIds.length, '?').join(',')})',
whereArgs: readingIds,
);
Future<List<Map<String, Object?>>> _kanjielementinfosQuery(
DatabaseExecutor connection,
List<int> kanjiIds,
) => connection.query(
JMdictTableNames.kanjiInfo,
where: '(elementId) IN (${List.filled(kanjiIds.length, '?').join(',')})',
whereArgs: kanjiIds,
);
// Xref queries
Future<LinearWordQueryData?> _senseantonymdataQuery(
DatabaseExecutor connection,
List<int> entryIds,
) => fetchLinearWordQueryData(connection, entryIds, fetchXrefData: false);
Future<LinearWordQueryData?> _senseseealsodataQuery(
DatabaseExecutor connection,
List<int> entryIds,
) => fetchLinearWordQueryData(connection, entryIds, fetchXrefData: false);
// Full query
Future<LinearWordQueryData> fetchLinearWordQueryData(
DatabaseExecutor connection,
List<int> entryIds, {
bool fetchXrefData = true,
}) async {
List<int> entryIds,
) async {
late final List<Map<String, Object?>> senses;
final Future<List<Map<String, Object?>>> senses_query = connection.query(
JMdictTableNames.sense,
where: 'entryId IN (${List.filled(entryIds.length, '?').join(',')})',
whereArgs: entryIds,
);
late final List<Map<String, Object?>> readingElements;
final Future<List<Map<String, Object?>>> readingElements_query =
connection.query(
JMdictTableNames.readingElement,
where: 'entryId IN (${List.filled(entryIds.length, '?').join(',')})',
whereArgs: entryIds,
orderBy: 'orderNum',
);
late final List<Map<String, Object?>> kanjiElements;
final Future<List<Map<String, Object?>>> kanjiElements_query =
connection.query(
JMdictTableNames.kanjiElement,
where: 'entryId IN (${List.filled(entryIds.length, '?').join(',')})',
whereArgs: entryIds,
orderBy: 'orderNum',
);
late final List<Map<String, Object?>> jlptTags;
final Future<List<Map<String, Object?>>> jlptTags_query = connection.query(
TanosJLPTTableNames.jlptTag,
where: 'entryId IN (${List.filled(entryIds.length, '?').join(',')})',
whereArgs: entryIds,
);
late final List<Map<String, Object?>> commonEntries;
final Future<List<Map<String, Object?>>> commonEntries_query =
connection.query(
'JMdict_EntryCommon',
where: 'entryId IN (${List.filled(entryIds.length, '?').join(',')})',
whereArgs: entryIds,
);
await Future.wait([
_sensesQuery(connection, entryIds).then((value) => senses = value),
_readingelementsQuery(
connection,
entryIds,
).then((value) => readingElements = value),
_kanjielementsQuery(
connection,
entryIds,
).then((value) => kanjiElements = value),
_jlpttagsQuery(connection, entryIds).then((value) => jlptTags = value),
_commonentriesQuery(
connection,
entryIds,
).then((value) => commonEntries = value),
senses_query.then((value) => senses = value),
readingElements_query.then((value) => readingElements = value),
kanjiElements_query.then((value) => kanjiElements = value),
jlptTags_query.then((value) => jlptTags = value),
commonEntries_query.then((value) => commonEntries = value),
]);
// Sense queries
final senseIds = senses.map((sense) => sense['senseId'] as int).toList();
late final List<Map<String, Object?>> senseAntonyms;
final Future<List<Map<String, Object?>>> senseAntonyms_query =
connection.rawQuery(
"""
SELECT
"${JMdictTableNames.senseAntonyms}".senseId,
"${JMdictTableNames.senseAntonyms}".ambiguous,
"${JMdictTableNames.senseAntonyms}".xrefEntryId,
"JMdict_BaseAndFurigana"."base",
"JMdict_BaseAndFurigana"."furigana"
FROM "${JMdictTableNames.senseAntonyms}"
JOIN "JMdict_BaseAndFurigana"
ON "${JMdictTableNames.senseAntonyms}"."xrefEntryId" = "JMdict_BaseAndFurigana"."entryId"
WHERE
"senseId" IN (${List.filled(senseIds.length, '?').join(',')})
AND "JMdict_BaseAndFurigana"."isFirst"
ORDER BY
"${JMdictTableNames.senseAntonyms}"."senseId",
"${JMdictTableNames.senseAntonyms}"."xrefEntryId"
""",
[...senseIds],
);
late final List<Map<String, Object?>> senseDialects;
final Future<List<Map<String, Object?>>> senseDialects_query =
connection.query(
JMdictTableNames.senseDialect,
where: 'senseId IN (${List.filled(senseIds.length, '?').join(',')})',
whereArgs: senseIds,
);
late final List<Map<String, Object?>> senseFields;
final Future<List<Map<String, Object?>>> senseFields_query = connection.query(
JMdictTableNames.senseField,
where: 'senseId IN (${List.filled(senseIds.length, '?').join(',')})',
whereArgs: senseIds,
);
late final List<Map<String, Object?>> senseGlossaries;
final Future<List<Map<String, Object?>>> senseGlossaries_query =
connection.query(
JMdictTableNames.senseGlossary,
where: 'senseId IN (${List.filled(senseIds.length, '?').join(',')})',
whereArgs: senseIds,
);
late final List<Map<String, Object?>> senseInfos;
final Future<List<Map<String, Object?>>> senseInfos_query = connection.query(
JMdictTableNames.senseInfo,
where: 'senseId IN (${List.filled(senseIds.length, '?').join(',')})',
whereArgs: senseIds,
);
late final List<Map<String, Object?>> senseLanguageSources;
final Future<List<Map<String, Object?>>> senseLanguageSources_query =
connection.query(
JMdictTableNames.senseLanguageSource,
where: 'senseId IN (${List.filled(senseIds.length, '?').join(',')})',
whereArgs: senseIds,
);
late final List<Map<String, Object?>> senseMiscs;
final Future<List<Map<String, Object?>>> senseMiscs_query = connection.query(
JMdictTableNames.senseMisc,
where: 'senseId IN (${List.filled(senseIds.length, '?').join(',')})',
whereArgs: senseIds,
);
late final List<Map<String, Object?>> sensePOSs;
final Future<List<Map<String, Object?>>> sensePOSs_query = connection.query(
JMdictTableNames.sensePOS,
where: 'senseId IN (${List.filled(senseIds.length, '?').join(',')})',
whereArgs: senseIds,
);
late final List<Map<String, Object?>> senseRestrictedToKanjis;
final Future<List<Map<String, Object?>>> senseRestrictedToKanjis_query =
connection.query(
JMdictTableNames.senseRestrictedToKanji,
where: 'senseId IN (${List.filled(senseIds.length, '?').join(',')})',
whereArgs: senseIds,
);
late final List<Map<String, Object?>> senseRestrictedToReadings;
final Future<List<Map<String, Object?>>> senseRestrictedToReadings_query =
connection.query(
JMdictTableNames.senseRestrictedToReading,
where: 'senseId IN (${List.filled(senseIds.length, '?').join(',')})',
whereArgs: senseIds,
);
late final List<Map<String, Object?>> senseSeeAlsos;
final Future<List<Map<String, Object?>>> senseSeeAlsos_query =
connection.rawQuery(
"""
SELECT
"${JMdictTableNames.senseSeeAlso}"."senseId",
"${JMdictTableNames.senseSeeAlso}"."ambiguous",
"${JMdictTableNames.senseSeeAlso}"."xrefEntryId",
"JMdict_BaseAndFurigana"."base",
"JMdict_BaseAndFurigana"."furigana"
FROM "${JMdictTableNames.senseSeeAlso}"
JOIN "JMdict_BaseAndFurigana"
ON "${JMdictTableNames.senseSeeAlso}"."xrefEntryId" = "JMdict_BaseAndFurigana"."entryId"
WHERE
"senseId" IN (${List.filled(senseIds.length, '?').join(',')})
AND "JMdict_BaseAndFurigana"."isFirst"
ORDER BY
"${JMdictTableNames.senseSeeAlso}"."senseId",
"${JMdictTableNames.senseSeeAlso}"."xrefEntryId"
""",
[...senseIds],
);
late final List<Map<String, Object?>> exampleSentences;
final Future<List<Map<String, Object?>>> exampleSentences_query =
connection.query(
'JMdict_ExampleSentence',
where: 'senseId IN (${List.filled(senseIds.length, '?').join(',')})',
whereArgs: senseIds,
);
// Reading queries
final readingIds = readingElements
.map((element) => element['elementId'] as int)
.toList();
late final List<Map<String, Object?>> readingElementInfos;
final Future<List<Map<String, Object?>>> readingElementInfos_query =
connection.query(
JMdictTableNames.readingInfo,
where: '(elementId) IN (${List.filled(readingIds.length, '?').join(',')})',
whereArgs: readingIds,
);
late final List<Map<String, Object?>> readingElementRestrictions;
final Future<List<Map<String, Object?>>> readingElementRestrictions_query =
connection.query(
JMdictTableNames.readingRestriction,
where: '(elementId) IN (${List.filled(readingIds.length, '?').join(',')})',
whereArgs: readingIds,
);
// Kanji queries
final kanjiIds = kanjiElements
.map((element) => element['elementId'] as int)
.toList();
late final List<Map<String, Object?>> readingElementInfos;
late final List<Map<String, Object?>> readingElementRestrictions;
late final List<Map<String, Object?>> kanjiElementInfos;
// Xref data queries
await Future.wait([
_senseantonymsQuery(
connection,
senseIds,
).then((value) => senseAntonyms = value),
senseseealsosQuery(
connection,
senseIds,
).then((value) => senseSeeAlsos = value),
]);
LinearWordQueryData? senseAntonymData;
LinearWordQueryData? senseSeeAlsoData;
final Future<List<Map<String, Object?>>> kanjiElementInfos_query =
connection.query(
JMdictTableNames.kanjiInfo,
where: '(elementId) IN (${List.filled(kanjiIds.length, '?').join(',')})',
whereArgs: kanjiIds,
);
await Future.wait([
_sensedialectsQuery(
connection,
senseIds,
).then((value) => senseDialects = value),
_sensefieldsQuery(
connection,
senseIds,
).then((value) => senseFields = value),
_senseglossariesQuery(
connection,
senseIds,
).then((value) => senseGlossaries = value),
_senseinfosQuery(connection, senseIds).then((value) => senseInfos = value),
_senselanguagesourcesQuery(
connection,
senseIds,
).then((value) => senseLanguageSources = value),
_sensemiscsQuery(connection, senseIds).then((value) => senseMiscs = value),
_sensepossQuery(connection, senseIds).then((value) => sensePOSs = value),
_senserestrictedtokanjisQuery(
connection,
senseIds,
).then((value) => senseRestrictedToKanjis = value),
_senserestrictedtoreadingsQuery(
connection,
senseIds,
).then((value) => senseRestrictedToReadings = value),
_examplesentencesQuery(
connection,
senseIds,
).then((value) => exampleSentences = value),
_readingelementinfosQuery(
connection,
readingIds,
).then((value) => readingElementInfos = value),
_readingelementrestrictionsQuery(
connection,
readingIds,
).then((value) => readingElementRestrictions = value),
_kanjielementinfosQuery(
connection,
kanjiIds,
).then((value) => kanjiElementInfos = value),
if (fetchXrefData)
_senseantonymdataQuery(
connection,
senseAntonyms.map((antonym) => antonym['xrefEntryId'] as int).toList(),
).then((value) => senseAntonymData = value),
if (fetchXrefData)
_senseseealsodataQuery(
connection,
senseSeeAlsos.map((seeAlso) => seeAlso['xrefEntryId'] as int).toList(),
).then((value) => senseSeeAlsoData = value),
senseAntonyms_query.then((value) => senseAntonyms = value),
senseDialects_query.then((value) => senseDialects = value),
senseFields_query.then((value) => senseFields = value),
senseGlossaries_query.then((value) => senseGlossaries = value),
senseInfos_query.then((value) => senseInfos = value),
senseLanguageSources_query.then((value) => senseLanguageSources = value),
senseMiscs_query.then((value) => senseMiscs = value),
sensePOSs_query.then((value) => sensePOSs = value),
senseRestrictedToKanjis_query
.then((value) => senseRestrictedToKanjis = value),
senseRestrictedToReadings_query
.then((value) => senseRestrictedToReadings = value),
senseSeeAlsos_query.then((value) => senseSeeAlsos = value),
exampleSentences_query.then((value) => exampleSentences = value),
readingElementInfos_query.then((value) => readingElementInfos = value),
readingElementRestrictions_query
.then((value) => readingElementRestrictions = value),
kanjiElementInfos_query.then((value) => kanjiElementInfos = value),
]);
return LinearWordQueryData(
@@ -433,7 +305,5 @@ Future<LinearWordQueryData> fetchLinearWordQueryData(
readingElementInfos: readingElementInfos,
readingElementRestrictions: readingElementRestrictions,
kanjiElementInfos: kanjiElementInfos,
senseAntonymData: senseAntonymData,
senseSeeAlsoData: senseSeeAlsoData,
);
}

View File

@@ -1,5 +1,5 @@
import 'package:jadb/search/word_search/word_search.dart';
import 'package:jadb/table_names/jmdict.dart';
import 'package:jadb/search/word_search/word_search.dart';
import 'package:jadb/util/text_filtering.dart';
import 'package:sqflite_common/sqlite_api.dart';
@@ -15,15 +15,15 @@ SearchMode _determineSearchMode(String word) {
final bool containsAscii = RegExp(r'[A-Za-z]').hasMatch(word);
if (containsKanji && containsAscii) {
return SearchMode.mixedKanji;
return SearchMode.MixedKanji;
} else if (containsKanji) {
return SearchMode.kanji;
return SearchMode.Kanji;
} else if (containsAscii) {
return SearchMode.english;
return SearchMode.English;
} else if (word.contains(hiraganaRegex) || word.contains(katakanaRegex)) {
return SearchMode.kana;
return SearchMode.Kana;
} else {
return SearchMode.mixedKana;
return SearchMode.MixedKana;
}
}
@@ -37,105 +37,91 @@ String _filterFTSSensitiveCharacters(String word) {
.replaceAll('(', '')
.replaceAll(')', '')
.replaceAll('^', '')
.replaceAll('"', '');
.replaceAll('\"', '');
}
(String, List<Object?>) _kanjiReadingTemplate(
String tableName,
String word, {
int? pageSize,
int? offset,
int pageSize = 10,
bool countOnly = false,
}) {
assert(
tableName == JMdictTableNames.kanjiElement ||
tableName == JMdictTableNames.readingElement,
);
assert(!countOnly || pageSize == null);
assert(!countOnly || offset == null);
assert(pageSize == null || pageSize > 0);
assert(offset == null || offset >= 0);
assert(
offset == null || pageSize != null,
'Offset should only be used with pageSize set',
);
return (
'''
}) =>
(
'''
WITH
fts_results AS (
SELECT DISTINCT
"$tableName"."entryId",
"${tableName}"."entryId",
100
+ (("${tableName}FTS"."reading" = ?) * 10000)
+ (("${tableName}FTS"."reading" = ?) * 50)
+ "JMdict_EntryScore"."score"
AS "score"
FROM "${tableName}FTS"
JOIN "$tableName" USING ("elementId")
JOIN "${tableName}" USING ("elementId")
JOIN "JMdict_EntryScore" USING ("elementId")
WHERE "${tableName}FTS"."reading" MATCH ? || '*'
AND "JMdict_EntryScore"."type" = '${tableName == JMdictTableNames.kanjiElement ? 'k' : 'r'}'
ORDER BY
"JMdict_EntryScore"."score" DESC
${!countOnly ? 'LIMIT ?' : ''}
),
non_fts_results AS (
SELECT DISTINCT
"$tableName"."entryId",
"${tableName}"."entryId",
50
+ "JMdict_EntryScore"."score"
AS "score"
FROM "$tableName"
FROM "${tableName}"
JOIN "JMdict_EntryScore" USING ("elementId")
WHERE "reading" LIKE '%' || ? || '%'
AND "$tableName"."entryId" NOT IN (SELECT "entryId" FROM "fts_results")
AND "${tableName}"."entryId" NOT IN (SELECT "entryId" FROM "fts_results")
AND "JMdict_EntryScore"."type" = '${tableName == JMdictTableNames.kanjiElement ? 'k' : 'r'}'
ORDER BY
"JMdict_EntryScore"."score" DESC,
"${tableName}"."entryId" ASC
${!countOnly ? 'LIMIT ?' : ''}
)
SELECT ${countOnly ? 'COUNT(DISTINCT "entryId") AS count' : '"entryId", MAX("score") AS "score"'}
${countOnly ? 'SELECT COUNT("entryId") AS count' : 'SELECT "entryId", "score"'}
FROM (
SELECT * FROM "fts_results"
UNION
SELECT * FROM "non_fts_results"
SELECT * FROM fts_results
UNION ALL
SELECT * FROM non_fts_results
)
${!countOnly ? 'GROUP BY "entryId"' : ''}
${!countOnly ? 'ORDER BY "score" DESC, "entryId" ASC' : ''}
${pageSize != null ? 'LIMIT ?' : ''}
${offset != null ? 'OFFSET ?' : ''}
'''
.trim(),
[
_filterFTSSensitiveCharacters(word),
_filterFTSSensitiveCharacters(word),
_filterFTSSensitiveCharacters(word),
?pageSize,
?offset,
],
);
}
.trim(),
[
_filterFTSSensitiveCharacters(word),
_filterFTSSensitiveCharacters(word),
if (!countOnly) pageSize,
_filterFTSSensitiveCharacters(word),
if (!countOnly) pageSize,
]
);
Future<List<ScoredEntryId>> _queryKanji(
DatabaseExecutor connection,
String word,
int? pageSize,
int pageSize,
int? offset,
) {
final (query, args) = _kanjiReadingTemplate(
JMdictTableNames.kanjiElement,
word,
pageSize: pageSize,
offset: offset,
);
return connection
.rawQuery(query, args)
.then(
(result) => result
.map(
(row) =>
ScoredEntryId(row['entryId'] as int, row['score'] as int),
)
.toList(),
);
return connection.rawQuery(query, args).then((result) => result
.map((row) => ScoredEntryId(
row['entryId'] as int,
row['score'] as int,
))
.toList());
}
Future<int> _queryKanjiCount(DatabaseExecutor connection, String word) {
Future<int> _queryKanjiCount(
DatabaseExecutor connection,
String word,
) {
final (query, args) = _kanjiReadingTemplate(
JMdictTableNames.kanjiElement,
word,
@@ -143,34 +129,32 @@ Future<int> _queryKanjiCount(DatabaseExecutor connection, String word) {
);
return connection
.rawQuery(query, args)
.then((result) => result.firstOrNull?['count'] as int? ?? 0);
.then((result) => result.first['count'] as int);
}
Future<List<ScoredEntryId>> _queryKana(
DatabaseExecutor connection,
String word,
int? pageSize,
int pageSize,
int? offset,
) {
final (query, args) = _kanjiReadingTemplate(
JMdictTableNames.readingElement,
word,
pageSize: pageSize,
offset: offset,
);
return connection
.rawQuery(query, args)
.then(
(result) => result
.map(
(row) =>
ScoredEntryId(row['entryId'] as int, row['score'] as int),
)
.toList(),
);
return connection.rawQuery(query, args).then((result) => result
.map((row) => ScoredEntryId(
row['entryId'] as int,
row['score'] as int,
))
.toList());
}
Future<int> _queryKanaCount(DatabaseExecutor connection, String word) {
Future<int> _queryKanaCount(
DatabaseExecutor connection,
String word,
) {
final (query, args) = _kanjiReadingTemplate(
JMdictTableNames.readingElement,
word,
@@ -178,22 +162,15 @@ Future<int> _queryKanaCount(DatabaseExecutor connection, String word) {
);
return connection
.rawQuery(query, args)
.then((result) => result.firstOrNull?['count'] as int? ?? 0);
.then((result) => result.first['count'] as int);
}
Future<List<ScoredEntryId>> _queryEnglish(
DatabaseExecutor connection,
String word,
int? pageSize,
int pageSize,
int? offset,
) async {
assert(pageSize == null || pageSize > 0);
assert(offset == null || offset >= 0);
assert(
offset == null || pageSize != null,
'Offset should only be used with pageSize set',
);
final result = await connection.rawQuery(
'''
SELECT
@@ -215,25 +192,41 @@ Future<List<ScoredEntryId>> _queryEnglish(
OFFSET ?
'''
.trim(),
[word, word, word, '%${word.replaceAll('%', '')}%', pageSize, offset],
[
word,
word,
word,
'%${word.replaceAll('%', '')}%',
pageSize,
offset,
],
);
return result
.map((row) => ScoredEntryId(row['entryId'] as int, row['score'] as int))
.map((row) => ScoredEntryId(
row['entryId'] as int,
row['score'] as int,
))
.toList();
}
Future<int> _queryEnglishCount(DatabaseExecutor connection, String word) async {
Future<int> _queryEnglishCount(
DatabaseExecutor connection,
String word,
) async {
final result = await connection.rawQuery(
'''
SELECT
COUNT(DISTINCT "${JMdictTableNames.sense}"."entryId") AS "count"
FROM "${JMdictTableNames.senseGlossary}"
JOIN "${JMdictTableNames.sense}" USING ("senseId")
WHERE "${JMdictTableNames.senseGlossary}"."phrase" LIKE ?
'''
SELECT
COUNT(DISTINCT "${JMdictTableNames.sense}"."entryId") AS "count"
FROM "${JMdictTableNames.senseGlossary}"
JOIN "${JMdictTableNames.sense}" USING ("senseId")
WHERE "${JMdictTableNames.senseGlossary}"."phrase" LIKE ?
'''
.trim(),
['%$word%'],
[
'%$word%',
],
);
return result.first['count'] as int;
@@ -243,34 +236,55 @@ Future<List<ScoredEntryId>> fetchEntryIds(
DatabaseExecutor connection,
String word,
SearchMode searchMode,
int? pageSize,
int pageSize,
int? offset,
) async {
if (searchMode == SearchMode.auto) {
if (searchMode == SearchMode.Auto) {
searchMode = _determineSearchMode(word);
}
assert(word.isNotEmpty, 'Word should not be empty when fetching entry IDs');
assert(
word.isNotEmpty,
'Word should not be empty when fetching entry IDs',
);
late final List<ScoredEntryId> entryIds;
switch (searchMode) {
case SearchMode.kanji:
entryIds = await _queryKanji(connection, word, pageSize, offset);
case SearchMode.Kanji:
entryIds = await _queryKanji(
connection,
word,
pageSize,
offset,
);
break;
case SearchMode.kana:
entryIds = await _queryKana(connection, word, pageSize, offset);
case SearchMode.Kana:
entryIds = await _queryKana(
connection,
word,
pageSize,
offset,
);
break;
case SearchMode.english:
entryIds = await _queryEnglish(connection, word, pageSize, offset);
case SearchMode.English:
entryIds = await _queryEnglish(
connection,
word,
pageSize,
offset,
);
break;
case SearchMode.mixedKana:
case SearchMode.mixedKanji:
case SearchMode.MixedKana:
case SearchMode.MixedKanji:
default:
throw UnimplementedError('Search mode $searchMode is not implemented');
throw UnimplementedError(
'Search mode $searchMode is not implemented',
);
}
;
return entryIds;
}
@@ -280,31 +294,45 @@ Future<int?> fetchEntryIdCount(
String word,
SearchMode searchMode,
) async {
if (searchMode == SearchMode.auto) {
if (searchMode == SearchMode.Auto) {
searchMode = _determineSearchMode(word);
}
assert(word.isNotEmpty, 'Word should not be empty when fetching entry IDs');
assert(
word.isNotEmpty,
'Word should not be empty when fetching entry IDs',
);
late final int? entryIdCount;
switch (searchMode) {
case SearchMode.kanji:
entryIdCount = await _queryKanjiCount(connection, word);
case SearchMode.Kanji:
entryIdCount = await _queryKanjiCount(
connection,
word,
);
break;
case SearchMode.kana:
entryIdCount = await _queryKanaCount(connection, word);
case SearchMode.Kana:
entryIdCount = await _queryKanaCount(
connection,
word,
);
break;
case SearchMode.english:
entryIdCount = await _queryEnglishCount(connection, word);
case SearchMode.English:
entryIdCount = await _queryEnglishCount(
connection,
word,
);
break;
case SearchMode.mixedKana:
case SearchMode.mixedKanji:
case SearchMode.MixedKana:
case SearchMode.MixedKanji:
default:
throw UnimplementedError('Search mode $searchMode is not implemented');
throw UnimplementedError(
'Search mode $searchMode is not implemented',
);
}
return entryIdCount;

View File

@@ -12,37 +12,50 @@ import 'package:jadb/models/word_search/word_search_sense.dart';
import 'package:jadb/models/word_search/word_search_sense_language_source.dart';
import 'package:jadb/models/word_search/word_search_sources.dart';
import 'package:jadb/models/word_search/word_search_xref_entry.dart';
import 'package:jadb/search/word_search/data_query.dart';
import 'package:jadb/search/word_search/entry_id_query.dart';
List<WordSearchResult> regroupWordSearchResults({
required List<ScoredEntryId> entryIds,
required LinearWordQueryData linearWordQueryData,
required List<Map<String, Object?>> readingElements,
required List<Map<String, Object?>> kanjiElements,
required List<Map<String, Object?>> jlptTags,
required List<Map<String, Object?>> commonEntries,
required List<Map<String, Object?>> senses,
required List<Map<String, Object?>> senseAntonyms,
required List<Map<String, Object?>> senseDialects,
required List<Map<String, Object?>> senseFields,
required List<Map<String, Object?>> senseGlossaries,
required List<Map<String, Object?>> senseInfos,
required List<Map<String, Object?>> senseLanguageSources,
required List<Map<String, Object?>> senseMiscs,
required List<Map<String, Object?>> sensePOSs,
required List<Map<String, Object?>> senseRestrictedToKanjis,
required List<Map<String, Object?>> senseRestrictedToReadings,
required List<Map<String, Object?>> senseSeeAlsos,
required List<Map<String, Object?>> exampleSentences,
required List<Map<String, Object?>> readingElementInfos,
required List<Map<String, Object?>> readingElementRestrictions,
required List<Map<String, Object?>> kanjiElementInfos,
}) {
final List<WordSearchResult> results = [];
final commonEntryIds = linearWordQueryData.commonEntries
.map((entry) => entry['entryId'] as int)
.toSet();
final commonEntryIds =
commonEntries.map((entry) => entry['entryId'] as int).toSet();
for (final scoredEntryId in entryIds) {
final List<Map<String, Object?>> entryReadingElements = linearWordQueryData
.readingElements
final List<Map<String, Object?>> entryReadingElements = readingElements
.where((element) => element['entryId'] == scoredEntryId.entryId)
.toList();
final List<Map<String, Object?>> entryKanjiElements = linearWordQueryData
.kanjiElements
final List<Map<String, Object?>> entryKanjiElements = kanjiElements
.where((element) => element['entryId'] == scoredEntryId.entryId)
.toList();
final List<Map<String, Object?>> entryJlptTags = linearWordQueryData
.jlptTags
final List<Map<String, Object?>> entryJlptTags = jlptTags
.where((element) => element['entryId'] == scoredEntryId.entryId)
.toList();
final jlptLevel =
entryJlptTags
final jlptLevel = entryJlptTags
.map((e) => JlptLevel.fromString(e['jlptLevel'] as String?))
.sorted((a, b) => b.compareTo(a))
.firstOrNull ??
@@ -50,36 +63,33 @@ List<WordSearchResult> regroupWordSearchResults({
final isCommon = commonEntryIds.contains(scoredEntryId.entryId);
final List<Map<String, Object?>> entrySenses = linearWordQueryData.senses
final List<Map<String, Object?>> entrySenses = senses
.where((element) => element['entryId'] == scoredEntryId.entryId)
.toList();
final GroupedWordResult entryReadingElementsGrouped = _regroupWords(
final GroupedWordResult entryReadingElementsGrouped = _regroup_words(
entryId: scoredEntryId.entryId,
readingElements: entryReadingElements,
kanjiElements: entryKanjiElements,
readingElementInfos: linearWordQueryData.readingElementInfos,
readingElementRestrictions:
linearWordQueryData.readingElementRestrictions,
kanjiElementInfos: linearWordQueryData.kanjiElementInfos,
readingElementInfos: readingElementInfos,
readingElementRestrictions: readingElementRestrictions,
kanjiElementInfos: kanjiElementInfos,
);
final List<WordSearchSense> entrySensesGrouped = _regroupSenses(
final List<WordSearchSense> entrySensesGrouped = _regroup_senses(
senses: entrySenses,
senseAntonyms: linearWordQueryData.senseAntonyms,
senseDialects: linearWordQueryData.senseDialects,
senseFields: linearWordQueryData.senseFields,
senseGlossaries: linearWordQueryData.senseGlossaries,
senseInfos: linearWordQueryData.senseInfos,
senseLanguageSources: linearWordQueryData.senseLanguageSources,
senseMiscs: linearWordQueryData.senseMiscs,
sensePOSs: linearWordQueryData.sensePOSs,
senseRestrictedToKanjis: linearWordQueryData.senseRestrictedToKanjis,
senseRestrictedToReadings: linearWordQueryData.senseRestrictedToReadings,
senseSeeAlsos: linearWordQueryData.senseSeeAlsos,
exampleSentences: linearWordQueryData.exampleSentences,
senseSeeAlsosXrefData: linearWordQueryData.senseSeeAlsoData,
senseAntonymsXrefData: linearWordQueryData.senseAntonymData,
senseAntonyms: senseAntonyms,
senseDialects: senseDialects,
senseFields: senseFields,
senseGlossaries: senseGlossaries,
senseInfos: senseInfos,
senseLanguageSources: senseLanguageSources,
senseMiscs: senseMiscs,
sensePOSs: sensePOSs,
senseRestrictedToKanjis: senseRestrictedToKanjis,
senseRestrictedToReadings: senseRestrictedToReadings,
senseSeeAlsos: senseSeeAlsos,
exampleSentences: exampleSentences,
);
results.add(
@@ -92,7 +102,10 @@ List<WordSearchResult> regroupWordSearchResults({
readingInfo: entryReadingElementsGrouped.readingInfos,
senses: entrySensesGrouped,
jlptLevel: jlptLevel,
sources: const WordSearchSources(jmdict: true, jmnedict: false),
sources: const WordSearchSources(
jmdict: true,
jmnedict: false,
),
),
);
}
@@ -112,7 +125,7 @@ class GroupedWordResult {
});
}
GroupedWordResult _regroupWords({
GroupedWordResult _regroup_words({
required int entryId,
required List<Map<String, Object?>> kanjiElements,
required List<Map<String, Object?>> kanjiElementInfos,
@@ -122,9 +135,8 @@ GroupedWordResult _regroupWords({
}) {
final List<WordSearchRuby> rubys = [];
final kanjiElements_ = kanjiElements
.where((element) => element['entryId'] == entryId)
.toList();
final kanjiElements_ =
kanjiElements.where((element) => element['entryId'] == entryId).toList();
final readingElements_ = readingElements
.where((element) => element['entryId'] == entryId)
@@ -136,7 +148,9 @@ GroupedWordResult _regroupWords({
for (final readingElement in readingElements_) {
if (readingElement['doesNotMatchKanji'] == 1 || kanjiElements_.isEmpty) {
final ruby = WordSearchRuby(base: readingElement['reading'] as String);
final ruby = WordSearchRuby(
base: readingElement['reading'] as String,
);
rubys.add(ruby);
continue;
@@ -155,12 +169,18 @@ GroupedWordResult _regroupWords({
continue;
}
final ruby = WordSearchRuby(base: kanji, furigana: reading);
final ruby = WordSearchRuby(
base: kanji,
furigana: reading,
);
rubys.add(ruby);
}
}
assert(rubys.isNotEmpty, 'No readings found for entryId: $entryId');
assert(
rubys.isNotEmpty,
'No readings found for entryId: $entryId',
);
final Map<int, String> readingElementIdsToReading = {
for (final element in readingElements_)
@@ -190,12 +210,12 @@ GroupedWordResult _regroupWords({
kanjiInfos: {
for (final kei in kanjiElementInfos_)
kanjiElementIdsToReading[kei['elementId'] as int]!:
JMdictKanjiInfo.fromId(kei['info'] as String),
JMdictKanjiInfo.fromId(kei['info'] as String),
},
);
}
List<WordSearchSense> _regroupSenses({
List<WordSearchSense> _regroup_senses({
required List<Map<String, Object?>> senses,
required List<Map<String, Object?>> senseAntonyms,
required List<Map<String, Object?>> senseDialects,
@@ -209,41 +229,29 @@ List<WordSearchSense> _regroupSenses({
required List<Map<String, Object?>> senseRestrictedToReadings,
required List<Map<String, Object?>> senseSeeAlsos,
required List<Map<String, Object?>> exampleSentences,
required LinearWordQueryData? senseSeeAlsosXrefData,
required LinearWordQueryData? senseAntonymsXrefData,
}) {
final groupedSenseAntonyms = senseAntonyms.groupListsBy(
(element) => element['senseId'] as int,
);
final groupedSenseDialects = senseDialects.groupListsBy(
(element) => element['senseId'] as int,
);
final groupedSenseFields = senseFields.groupListsBy(
(element) => element['senseId'] as int,
);
final groupedSenseGlossaries = senseGlossaries.groupListsBy(
(element) => element['senseId'] as int,
);
final groupedSenseInfos = senseInfos.groupListsBy(
(element) => element['senseId'] as int,
);
final groupedSenseLanguageSources = senseLanguageSources.groupListsBy(
(element) => element['senseId'] as int,
);
final groupedSenseMiscs = senseMiscs.groupListsBy(
(element) => element['senseId'] as int,
);
final groupedSensePOSs = sensePOSs.groupListsBy(
(element) => element['senseId'] as int,
);
final groupedSenseRestrictedToKanjis = senseRestrictedToKanjis.groupListsBy(
(element) => element['senseId'] as int,
);
final groupedSenseAntonyms =
senseAntonyms.groupListsBy((element) => element['senseId'] as int);
final groupedSenseDialects =
senseDialects.groupListsBy((element) => element['senseId'] as int);
final groupedSenseFields =
senseFields.groupListsBy((element) => element['senseId'] as int);
final groupedSenseGlossaries =
senseGlossaries.groupListsBy((element) => element['senseId'] as int);
final groupedSenseInfos =
senseInfos.groupListsBy((element) => element['senseId'] as int);
final groupedSenseLanguageSources =
senseLanguageSources.groupListsBy((element) => element['senseId'] as int);
final groupedSenseMiscs =
senseMiscs.groupListsBy((element) => element['senseId'] as int);
final groupedSensePOSs =
sensePOSs.groupListsBy((element) => element['senseId'] as int);
final groupedSenseRestrictedToKanjis = senseRestrictedToKanjis
.groupListsBy((element) => element['senseId'] as int);
final groupedSenseRestrictedToReadings = senseRestrictedToReadings
.groupListsBy((element) => element['senseId'] as int);
final groupedSenseSeeAlsos = senseSeeAlsos.groupListsBy(
(element) => element['senseId'] as int,
);
final groupedSenseSeeAlsos =
senseSeeAlsos.groupListsBy((element) => element['senseId'] as int);
final List<WordSearchSense> result = [];
for (final sense in senses) {
@@ -262,82 +270,45 @@ List<WordSearchSense> _regroupSenses({
groupedSenseRestrictedToReadings[senseId] ?? [];
final seeAlsos = groupedSenseSeeAlsos[senseId] ?? [];
final List<WordSearchResult> seeAlsosWordResults =
senseSeeAlsosXrefData != null
? regroupWordSearchResults(
entryIds: seeAlsos
.map((e) => ScoredEntryId(e['xrefEntryId'] as int, 0))
.toList(),
linearWordQueryData: senseSeeAlsosXrefData,
)
: [];
final List<WordSearchResult> antonymsWordResults =
senseAntonymsXrefData != null
? regroupWordSearchResults(
entryIds: antonyms
.map((e) => ScoredEntryId(e['xrefEntryId'] as int, 0))
.toList(),
linearWordQueryData: senseAntonymsXrefData,
)
: [];
final resultSense = WordSearchSense(
englishDefinitions: glossaries.map((e) => e['phrase'] as String).toList(),
partsOfSpeech: pos
.map((e) => JMdictPOS.fromId(e['pos'] as String))
partsOfSpeech:
pos.map((e) => JMdictPOS.fromId(e['pos'] as String)).toList(),
seeAlso: seeAlsos
.map((e) => WordSearchXrefEntry(
entryId: e['xrefEntryId'] as int,
baseWord: e['base'] as String,
furigana: e['furigana'] as String?,
ambiguous: e['ambiguous'] == 1,
))
.toList(),
seeAlso: seeAlsos.asMap().entries.map<WordSearchXrefEntry>((mapEntry) {
final i = mapEntry.key;
final e = mapEntry.value;
return WordSearchXrefEntry(
entryId: e['xrefEntryId'] as int,
baseWord: e['base'] as String,
furigana: e['furigana'] as String?,
ambiguous: e['ambiguous'] == 1,
xrefResult: seeAlsosWordResults.isNotEmpty
? seeAlsosWordResults[i]
: null,
);
}).toList(),
antonyms: antonyms.asMap().entries.map<WordSearchXrefEntry>((mapEntry) {
final i = mapEntry.key;
final e = mapEntry.value;
return WordSearchXrefEntry(
entryId: e['xrefEntryId'] as int,
baseWord: e['base'] as String,
furigana: e['furigana'] as String?,
ambiguous: e['ambiguous'] == 1,
xrefResult: antonymsWordResults.isNotEmpty
? antonymsWordResults[i]
: null,
);
}).toList(),
restrictedToReading: restrictedToReadings
.map((e) => e['reading'] as String)
.toList(),
restrictedToKanji: restrictedToKanjis
.map((e) => e['kanji'] as String)
.toList(),
fields: fields
.map((e) => JMdictField.fromId(e['field'] as String))
antonyms: antonyms
.map((e) => WordSearchXrefEntry(
entryId: e['xrefEntryId'] as int,
baseWord: e['base'] as String,
furigana: e['furigana'] as String?,
ambiguous: e['ambiguous'] == 1,
))
.toList(),
restrictedToReading:
restrictedToReadings.map((e) => e['reading'] as String).toList(),
restrictedToKanji:
restrictedToKanjis.map((e) => e['kanji'] as String).toList(),
fields:
fields.map((e) => JMdictField.fromId(e['field'] as String)).toList(),
dialects: dialects
.map((e) => JMdictDialect.fromId(e['dialect'] as String))
.toList(),
misc: miscs.map((e) => JMdictMisc.fromId(e['misc'] as String)).toList(),
info: infos.map((e) => e['info'] as String).toList(),
languageSource: languageSources
.map(
(e) => WordSearchSenseLanguageSource(
language: e['language'] as String,
phrase: e['phrase'] as String?,
fullyDescribesSense: e['fullyDescribesSense'] == 1,
constructedFromSmallerWords:
e['constructedFromSmallerWords'] == 1,
),
)
.map((e) => WordSearchSenseLanguageSource(
language: e['language'] as String,
phrase: e['phrase'] as String?,
fullyDescribesSense: e['fullyDescribesSense'] == 1,
constructedFromSmallerWords:
e['constructedFromSmallerWords'] == 1,
))
.toList(),
);

View File

@@ -14,38 +14,26 @@ import 'package:jadb/table_names/jmdict.dart';
import 'package:sqflite_common/sqlite_api.dart';
enum SearchMode {
/// Try to autodetect what is being searched for
auto,
/// Search for english words
english,
/// Search for the kanji reading of a word
kanji,
/// Search for the kanji reading of a word, mixed in with kana/romaji
mixedKanji,
/// Search for the kana reading of a word
kana,
/// Search for the kana reading of a word, mixed in with romaji
mixedKana,
Auto,
English,
Kanji,
MixedKanji,
Kana,
MixedKana,
}
/// Searches for an input string, returning a list of results with their details. Returns null if the input string is empty.
Future<List<WordSearchResult>?> searchWordWithDbConnection(
DatabaseExecutor connection,
String word, {
SearchMode searchMode = SearchMode.auto,
int page = 0,
int? pageSize,
}) async {
String word,
SearchMode searchMode,
int page,
int pageSize,
) async {
if (word.isEmpty) {
return null;
}
final int? offset = pageSize != null ? page * pageSize : null;
final offset = page * pageSize;
final List<ScoredEntryId> entryIds = await fetchEntryIds(
connection,
word,
@@ -61,28 +49,42 @@ Future<List<WordSearchResult>?> searchWordWithDbConnection(
final LinearWordQueryData linearWordQueryData =
await fetchLinearWordQueryData(
connection,
entryIds.map((e) => e.entryId).toList(),
);
connection,
entryIds.map((e) => e.entryId).toList(),
);
final result = regroupWordSearchResults(
entryIds: entryIds,
linearWordQueryData: linearWordQueryData,
readingElements: linearWordQueryData.readingElements,
kanjiElements: linearWordQueryData.kanjiElements,
jlptTags: linearWordQueryData.jlptTags,
commonEntries: linearWordQueryData.commonEntries,
senses: linearWordQueryData.senses,
senseAntonyms: linearWordQueryData.senseAntonyms,
senseDialects: linearWordQueryData.senseDialects,
senseFields: linearWordQueryData.senseFields,
senseGlossaries: linearWordQueryData.senseGlossaries,
senseInfos: linearWordQueryData.senseInfos,
senseLanguageSources: linearWordQueryData.senseLanguageSources,
senseMiscs: linearWordQueryData.senseMiscs,
sensePOSs: linearWordQueryData.sensePOSs,
senseRestrictedToKanjis: linearWordQueryData.senseRestrictedToKanjis,
senseRestrictedToReadings: linearWordQueryData.senseRestrictedToReadings,
senseSeeAlsos: linearWordQueryData.senseSeeAlsos,
exampleSentences: linearWordQueryData.exampleSentences,
readingElementInfos: linearWordQueryData.readingElementInfos,
readingElementRestrictions: linearWordQueryData.readingElementRestrictions,
kanjiElementInfos: linearWordQueryData.kanjiElementInfos,
);
for (final resultEntry in result) {
resultEntry.inferMatchSpans(word, searchMode: searchMode);
}
return result;
}
/// Searches for an input string, returning the amount of results that the search would yield without pagination.
Future<int?> searchWordCountWithDbConnection(
DatabaseExecutor connection,
String word, {
SearchMode searchMode = SearchMode.auto,
}) async {
String word,
SearchMode searchMode,
) async {
if (word.isEmpty) {
return null;
}
@@ -96,7 +98,6 @@ Future<int?> searchWordCountWithDbConnection(
return entryIdCount;
}
/// Fetches a single word by its entry ID, returning null if not found.
Future<WordSearchResult?> getWordByIdWithDbConnection(
DatabaseExecutor connection,
int id,
@@ -105,23 +106,43 @@ Future<WordSearchResult?> getWordByIdWithDbConnection(
return null;
}
final exists = await connection
.rawQuery(
'SELECT EXISTS(SELECT 1 FROM "${JMdictTableNames.entry}" WHERE "entryId" = ?)',
[id],
)
.then((value) => value.isNotEmpty && value.first.values.first == 1);
final exists = await connection.rawQuery(
'SELECT EXISTS(SELECT 1 FROM "${JMdictTableNames.entry}" WHERE "entryId" = ?)',
[id],
).then((value) => value.isNotEmpty && value.first.values.first == 1);
if (!exists) {
return null;
}
final LinearWordQueryData linearWordQueryData =
await fetchLinearWordQueryData(connection, [id]);
await fetchLinearWordQueryData(
connection,
[id],
);
final result = regroupWordSearchResults(
entryIds: [ScoredEntryId(id, 0)],
linearWordQueryData: linearWordQueryData,
readingElements: linearWordQueryData.readingElements,
kanjiElements: linearWordQueryData.kanjiElements,
jlptTags: linearWordQueryData.jlptTags,
commonEntries: linearWordQueryData.commonEntries,
senses: linearWordQueryData.senses,
senseAntonyms: linearWordQueryData.senseAntonyms,
senseDialects: linearWordQueryData.senseDialects,
senseFields: linearWordQueryData.senseFields,
senseGlossaries: linearWordQueryData.senseGlossaries,
senseInfos: linearWordQueryData.senseInfos,
senseLanguageSources: linearWordQueryData.senseLanguageSources,
senseMiscs: linearWordQueryData.senseMiscs,
sensePOSs: linearWordQueryData.sensePOSs,
senseRestrictedToKanjis: linearWordQueryData.senseRestrictedToKanjis,
senseRestrictedToReadings: linearWordQueryData.senseRestrictedToReadings,
senseSeeAlsos: linearWordQueryData.senseSeeAlsos,
exampleSentences: linearWordQueryData.exampleSentences,
readingElementInfos: linearWordQueryData.readingElementInfos,
readingElementRestrictions: linearWordQueryData.readingElementRestrictions,
kanjiElementInfos: linearWordQueryData.kanjiElementInfos,
);
assert(
@@ -131,27 +152,3 @@ Future<WordSearchResult?> getWordByIdWithDbConnection(
return result.firstOrNull;
}
/// Fetches multiple words by their entry IDs, returning a map from entry ID to result.
Future<Map<int, WordSearchResult>> getWordsByIdsWithDbConnection(
DatabaseExecutor connection,
Set<int> ids,
) async {
if (ids.isEmpty) {
return {};
}
final LinearWordQueryData linearWordQueryData =
await fetchLinearWordQueryData(connection, ids.toList());
final List<ScoredEntryId> entryIds = ids
.map((id) => ScoredEntryId(id, 0)) // Score is not used here
.toList();
final results = regroupWordSearchResults(
entryIds: entryIds,
linearWordQueryData: linearWordQueryData,
);
return {for (var r in results) r.entryId: r};
}

View File

@@ -20,23 +20,23 @@ abstract class JMdictTableNames {
static const String senseSeeAlso = 'JMdict_SenseSeeAlso';
static Set<String> get allTables => {
entry,
kanjiElement,
kanjiInfo,
readingElement,
readingInfo,
readingRestriction,
sense,
senseAntonyms,
senseDialect,
senseField,
senseGlossary,
senseInfo,
senseMisc,
sensePOS,
senseLanguageSource,
senseRestrictedToKanji,
senseRestrictedToReading,
senseSeeAlso,
};
entry,
kanjiElement,
kanjiInfo,
readingElement,
readingInfo,
readingRestriction,
sense,
senseAntonyms,
senseDialect,
senseField,
senseGlossary,
senseInfo,
senseMisc,
sensePOS,
senseLanguageSource,
senseRestrictedToKanji,
senseRestrictedToReading,
senseSeeAlso
};
}

View File

@@ -17,19 +17,19 @@ abstract class KANJIDICTableNames {
static const String nanori = 'KANJIDIC_Nanori';
static Set<String> get allTables => {
character,
radicalName,
codepoint,
radical,
strokeMiscount,
variant,
dictionaryReference,
dictionaryReferenceMoro,
queryCode,
reading,
kunyomi,
onyomi,
meaning,
nanori,
};
character,
radicalName,
codepoint,
radical,
strokeMiscount,
variant,
dictionaryReference,
dictionaryReferenceMoro,
queryCode,
reading,
kunyomi,
onyomi,
meaning,
nanori
};
}

View File

@@ -1,5 +1,7 @@
abstract class RADKFILETableNames {
static const String radkfile = 'RADKFILE';
static Set<String> get allTables => {radkfile};
static Set<String> get allTables => {
radkfile,
};
}

View File

@@ -276,22 +276,29 @@ extension on DateTime {
/// See more info here:
/// - https://en.wikipedia.org/wiki/Nanboku-ch%C5%8D_period
/// - http://www.kumamotokokufu-h.ed.jp/kumamoto/bungaku/nengoui.html
String? japaneseEra() {
String? japaneseEra({bool nanbokuchouPeriodUsesNorth = true}) {
throw UnimplementedError('This function is not implemented yet.');
if (year < 645) {
if (this.year < 645) {
return null;
}
if (year < periodsNanbokuchouNorth.keys.first.$1) {
if (this.year < periodsNanbokuchouNorth.keys.first.$1) {
// TODO: find first where year <= this.year and jump one period back.
}
}
String get japaneseWeekdayPrefix =>
['', '', '', '', '', '', ''][weekday - 1];
String get japaneseWeekdayPrefix => [
'',
'',
'',
'',
'',
'',
'',
][weekday - 1];
/// Returns the date in Japanese format.
String japaneseDate({bool showWeekday = false}) =>
'$month月$day日${showWeekday ? '$japaneseWeekdayPrefix' : ''}';
'$month月$day日' + (showWeekday ? '$japaneseWeekdayPrefix' : '');
}

View File

@@ -12,7 +12,10 @@ enum WordClass {
input,
}
enum LemmatizationRuleType { prefix, suffix }
enum LemmatizationRuleType {
prefix,
suffix,
}
class LemmatizationRule {
final String name;
@@ -43,18 +46,18 @@ class LemmatizationRule {
lookAheadBehind = const [''],
LemmatizationRuleType type = LemmatizationRuleType.suffix,
}) : this(
name: name,
pattern: AllomorphPattern(
patterns: {
pattern: replacement != null ? [replacement] : null,
},
type: type,
lookAheadBehind: lookAheadBehind,
),
validChildClasses: validChildClasses,
terminal: terminal,
wordClass: wordClass,
);
name: name,
pattern: AllomorphPattern(
patterns: {
pattern: replacement != null ? [replacement] : null
},
type: type,
lookAheadBehind: lookAheadBehind,
),
validChildClasses: validChildClasses,
terminal: terminal,
wordClass: wordClass,
);
}
/// Represents a set of patterns for matching allomorphs in a word.
@@ -129,8 +132,8 @@ class AllomorphPattern {
if (word.startsWith(p as String)) {
return patterns[affix] != null
? patterns[affix]!
.map((s) => s + word.substring(affix.length))
.toList()
.map((s) => s + word.substring(affix.length))
.toList()
: [word.substring(affix.length)];
}
break;
@@ -183,7 +186,7 @@ class Lemmatized {
@override
String toString() {
final childrenString = children
.map((c) => ' - ${c.toString().split('\n').join('\n ')}')
.map((c) => ' - ' + c.toString().split('\n').join('\n '))
.join('\n');
if (children.isEmpty) {
@@ -236,6 +239,9 @@ Lemmatized lemmatize(String word) {
return Lemmatized(
original: word,
rule: inputRule,
children: _lemmatize(inputRule, word),
children: _lemmatize(
inputRule,
word,
),
);
}

View File

@@ -1,7 +1,7 @@
import 'package:jadb/util/lemmatizer/lemmatizer.dart';
import 'package:jadb/util/lemmatizer/rules/godan_verbs.dart';
import 'package:jadb/util/lemmatizer/rules/i_adjectives.dart';
import 'package:jadb/util/lemmatizer/rules/ichidan_verbs.dart';
import 'package:jadb/util/lemmatizer/rules/godan-verbs.dart';
import 'package:jadb/util/lemmatizer/rules/i-adjectives.dart';
import 'package:jadb/util/lemmatizer/rules/ichidan-verbs.dart';
List<LemmatizationRule> lemmatizationRules = [
...ichidanVerbLemmatizationRules,

View File

@@ -1,9 +1,9 @@
// Source: https://github.com/Kimtaro/ve/blob/master/lib/providers/japanese_transliterators.rb
const hiraganaSyllabicN = '';
const hiraganaSmallTsu = '';
const hiragana_syllabic_n = '';
const hiragana_small_tsu = '';
const Map<String, String> hiraganaToLatin = {
const Map<String, String> hiragana_to_latin = {
'': 'a',
'': 'i',
'': 'u',
@@ -209,7 +209,7 @@ const Map<String, String> hiraganaToLatin = {
'': 'yori',
};
const Map<String, String> latinToHiragana = {
const Map<String, String> latin_to_hiragana = {
'a': '',
'i': '',
'u': '',
@@ -481,9 +481,9 @@ const Map<String, String> latinToHiragana = {
'#~': '',
};
bool _smallTsu(String forConversion) => forConversion == hiraganaSmallTsu;
bool _nFollowedByYuYeYo(String forConversion, String kana) =>
forConversion == hiraganaSyllabicN &&
bool _smallTsu(String for_conversion) => for_conversion == hiragana_small_tsu;
bool _nFollowedByYuYeYo(String for_conversion, String kana) =>
for_conversion == hiragana_syllabic_n &&
kana.length > 1 &&
'やゆよ'.contains(kana.substring(1, 2));
@@ -495,17 +495,17 @@ String transliterateHiraganaToLatin(String hiragana) {
while (kana.isNotEmpty) {
final lengths = [if (kana.length > 1) 2, 1];
for (final length in lengths) {
final String forConversion = kana.substring(0, length);
final String for_conversion = kana.substring(0, length);
String? mora;
if (_smallTsu(forConversion)) {
if (_smallTsu(for_conversion)) {
geminate = true;
kana = kana.replaceRange(0, length, '');
break;
} else if (_nFollowedByYuYeYo(forConversion, kana)) {
} else if (_nFollowedByYuYeYo(for_conversion, kana)) {
mora = "n'";
}
mora ??= hiraganaToLatin[forConversion];
mora ??= hiragana_to_latin[for_conversion];
if (mora != null) {
if (geminate) {
@@ -516,7 +516,7 @@ String transliterateHiraganaToLatin(String hiragana) {
kana = kana.replaceRange(0, length, '');
break;
} else if (length == 1) {
romaji += forConversion;
romaji += for_conversion;
kana = kana.replaceRange(0, length, '');
}
}
@@ -524,46 +524,48 @@ String transliterateHiraganaToLatin(String hiragana) {
return romaji;
}
bool _doubleNFollowedByAIUEO(String forConversion) =>
RegExp(r'^nn[aiueo]$').hasMatch(forConversion);
bool _hasTableMatch(String forConversion) =>
latinToHiragana[forConversion] != null;
bool _hasDoubleConsonant(String forConversion, int length) =>
forConversion == 'tch' ||
bool _doubleNFollowedByAIUEO(String for_conversion) =>
RegExp(r'^nn[aiueo]$').hasMatch(for_conversion);
bool _hasTableMatch(String for_conversion) =>
latin_to_hiragana[for_conversion] != null;
bool _hasDoubleConsonant(String for_conversion, int length) =>
for_conversion == 'tch' ||
(length == 2 &&
RegExp(r'^([kgsztdnbpmyrlwchf])\1$').hasMatch(forConversion));
RegExp(r'^([kgsztdnbpmyrlwchf])\1$').hasMatch(for_conversion));
String transliterateLatinToHiragana(String latin) {
String romaji = latin
.toLowerCase()
.replaceAll('mb', 'nb')
.replaceAll('mp', 'np');
String romaji =
latin.toLowerCase().replaceAll('mb', 'nb').replaceAll('mp', 'np');
String kana = '';
while (romaji.isNotEmpty) {
final lengths = [if (romaji.length > 2) 3, if (romaji.length > 1) 2, 1];
final lengths = [
if (romaji.length > 2) 3,
if (romaji.length > 1) 2,
1,
];
for (final length in lengths) {
String? mora;
int forRemoval = length;
final String forConversion = romaji.substring(0, length);
int for_removal = length;
final String for_conversion = romaji.substring(0, length);
if (_doubleNFollowedByAIUEO(forConversion)) {
mora = hiraganaSyllabicN;
forRemoval = 1;
} else if (_hasTableMatch(forConversion)) {
mora = latinToHiragana[forConversion];
} else if (_hasDoubleConsonant(forConversion, length)) {
mora = hiraganaSmallTsu;
forRemoval = 1;
if (_doubleNFollowedByAIUEO(for_conversion)) {
mora = hiragana_syllabic_n;
for_removal = 1;
} else if (_hasTableMatch(for_conversion)) {
mora = latin_to_hiragana[for_conversion];
} else if (_hasDoubleConsonant(for_conversion, length)) {
mora = hiragana_small_tsu;
for_removal = 1;
}
if (mora != null) {
kana += mora;
romaji = romaji.replaceRange(0, forRemoval, '');
romaji = romaji.replaceRange(0, for_removal, '');
break;
} else if (length == 1) {
kana += forConversion;
kana += for_conversion;
romaji = romaji.replaceRange(0, 1, '');
}
}
@@ -577,11 +579,11 @@ String _transposeCodepointsInRange(
int distance,
int rangeStart,
int rangeEnd,
) => String.fromCharCodes(
text.codeUnits.map(
(c) => c + ((rangeStart <= c && c <= rangeEnd) ? distance : 0),
),
);
) =>
String.fromCharCodes(
text.codeUnits
.map((c) => c + ((rangeStart <= c && c <= rangeEnd) ? distance : 0)),
);
String transliterateKanaToLatin(String kana) =>
transliterateHiraganaToLatin(transliterateKatakanaToHiragana(kana));
@@ -597,7 +599,12 @@ String transliterateHiraganaToKatakana(String hiragana) =>
String transliterateFullwidthRomajiToHalfwidth(String halfwidth) =>
_transposeCodepointsInRange(
_transposeCodepointsInRange(halfwidth, -65248, 65281, 65374),
_transposeCodepointsInRange(
halfwidth,
-65248,
65281,
65374,
),
-12256,
12288,
12288,
@@ -605,7 +612,12 @@ String transliterateFullwidthRomajiToHalfwidth(String halfwidth) =>
String transliterateHalfwidthRomajiToFullwidth(String halfwidth) =>
_transposeCodepointsInRange(
_transposeCodepointsInRange(halfwidth, 65248, 33, 126),
_transposeCodepointsInRange(
halfwidth,
65248,
33,
126,
),
12256,
32,
32,

View File

@@ -1,3 +1,3 @@
String escapeStringValue(String value) {
return "'${value.replaceAll("'", "''")}'";
return "'" + value.replaceAll("'", "''") + "'";
}

View File

@@ -43,11 +43,6 @@ CREATE TABLE "JMdict_KanjiElement" (
"entryId" INTEGER NOT NULL REFERENCES "JMdict_Entry"("entryId"),
"orderNum" INTEGER NOT NULL,
"reading" TEXT NOT NULL,
"news" INTEGER CHECK ("news" BETWEEN 1 AND 2),
"ichi" INTEGER CHECK ("ichi" BETWEEN 1 AND 2),
"spec" INTEGER CHECK ("spec" BETWEEN 1 AND 2),
"gai" INTEGER CHECK ("gai" BETWEEN 1 AND 2),
"nf" INTEGER CHECK ("nf" BETWEEN 1 AND 48),
UNIQUE("entryId", "reading"),
UNIQUE("entryId", "orderNum")
) WITHOUT ROWID;
@@ -55,6 +50,31 @@ CREATE TABLE "JMdict_KanjiElement" (
CREATE INDEX "JMdict_KanjiElement_byEntryId_byOrderNum" ON "JMdict_KanjiElement"("entryId", "orderNum");
CREATE INDEX "JMdict_KanjiElement_byReading" ON "JMdict_KanjiElement"("reading");
CREATE TABLE "JMdict_KanjiElementNews" (
"elementId" INTEGER PRIMARY KEY REFERENCES "JMdict_KanjiElement"("elementId"),
"news" INTEGER NOT NULL CHECK ("news" BETWEEN 1 AND 2)
);
CREATE TABLE "JMdict_KanjiElementIchi" (
"elementId" INTEGER PRIMARY KEY REFERENCES "JMdict_KanjiElement"("elementId"),
"ichi" INTEGER NOT NULL CHECK ("ichi" BETWEEN 1 AND 2)
);
CREATE TABLE "JMdict_KanjiElementSpec" (
"elementId" INTEGER PRIMARY KEY REFERENCES "JMdict_KanjiElement"("elementId"),
"spec" INTEGER NOT NULL CHECK ("spec" BETWEEN 1 AND 2)
);
CREATE TABLE "JMdict_KanjiElementGai" (
"elementId" INTEGER PRIMARY KEY REFERENCES "JMdict_KanjiElement"("elementId"),
"gai" INTEGER NOT NULL CHECK ("gai" BETWEEN 1 AND 2)
);
CREATE TABLE "JMdict_KanjiElementNF" (
"elementId" INTEGER PRIMARY KEY REFERENCES "JMdict_KanjiElement"("elementId"),
"nf" INTEGER NOT NULL CHECK ("nf" BETWEEN 1 AND 48)
);
CREATE TABLE "JMdict_KanjiElementInfo" (
"elementId" INTEGER NOT NULL REFERENCES "JMdict_KanjiElement"("elementId"),
"info" TEXT NOT NULL REFERENCES "JMdict_InfoKanji"("id"),
@@ -81,6 +101,36 @@ CREATE TABLE "JMdict_ReadingElement" (
CREATE INDEX "JMdict_ReadingElement_byEntryId_byOrderNum" ON "JMdict_ReadingElement"("entryId", "orderNum");
CREATE INDEX "JMdict_ReadingElement_byReading" ON "JMdict_ReadingElement"("reading");
CREATE TABLE "JMdict_ReadingElementReadingDoesNotMatchKanji" (
"elementId" INTEGER PRIMARY KEY REFERENCES "JMdict_ReadingElement"("elementId"),
"readingDoesNotMatchKanji" BOOLEAN NOT NULL,
);
CREATE TABLE "JMdict_ReadingElementNews" (
"elementId" INTEGER PRIMARY KEY REFERENCES "JMdict_ReadingElement"("elementId"),
"news" INTEGER NOT NULL CHECK ("news" BETWEEN 1 AND 2)
);
CREATE TABLE "JMdict_ReadingElementIchi" (
"elementId" INTEGER PRIMARY KEY REFERENCES "JMdict_ReadingElement"("elementId"),
"ichi" INTEGER NOT NULL CHECK ("ichi" BETWEEN 1 AND 2)
);
CREATE TABLE "JMdict_ReadingElementSpec" (
"elementId" INTEGER PRIMARY KEY REFERENCES "JMdict_ReadingElement"("elementId"),
"spec" INTEGER NOT NULL CHECK ("spec" BETWEEN 1 AND 2)
);
CREATE TABLE "JMdict_ReadingElementGai" (
"elementId" INTEGER PRIMARY KEY REFERENCES "JMdict_ReadingElement"("elementId"),
"gai" INTEGER NOT NULL CHECK ("gai" BETWEEN 1 AND 2)
);
CREATE TABLE "JMdict_ReadingElementNF" (
"elementId" INTEGER PRIMARY KEY REFERENCES "JMdict_ReadingElement"("elementId"),
"nf" INTEGER NOT NULL CHECK ("nf" BETWEEN 1 AND 48)
);
CREATE TABLE "JMdict_ReadingElementRestriction" (
"elementId" INTEGER NOT NULL REFERENCES "JMdict_ReadingElement"("elementId"),
"restriction" TEXT NOT NULL,

View File

@@ -48,6 +48,10 @@ SELECT
- (substr(COALESCE("JMdict_JLPTTag"."jlptLevel", 'N0'), 2) * -5)
AS "score"
FROM "JMdict_ReadingElement"
LEFT JOIN "JMdict_ReadingElementNews" USING ("elementId")
LEFT JOIN "JMdict_ReadingElementIchi" USING ("elementId")
LEFT JOIN "JMdict_ReadingElementSpec" USING ("elementId")
LEFT JOIN "JMdict_ReadingElementGai" USING ("elementId")
LEFT JOIN "JMdict_JLPTTag" USING ("entryId");
CREATE VIEW "JMdict_EntryScoreView_Kanji" AS
@@ -80,6 +84,10 @@ SELECT
- (substr(COALESCE("JMdict_JLPTTag"."jlptLevel", 'N0'), 2) * -5)
AS "score"
FROM "JMdict_KanjiElement"
LEFT JOIN "JMdict_KanjiElementNews" USING ("elementId")
LEFT JOIN "JMdict_KanjiElementIchi" USING ("elementId")
LEFT JOIN "JMdict_KanjiElementSpec" USING ("elementId")
LEFT JOIN "JMdict_KanjiElementGai" USING ("elementId")
LEFT JOIN "JMdict_JLPTTag" USING ("entryId");
CREATE VIEW "JMdict_EntryScoreView" AS

View File

@@ -7,29 +7,6 @@ buildDartApplication {
version = "1.0.0";
inherit src;
dartEntryPoints."bin/jadb" = "bin/jadb.dart";
# NOTE: the default dart hooks are using `dart compile`, which is not able to call the
# new dart build hooks required to use package:sqlite3 >= 3.0.0. So we override
# these phases to use `dart build` instead.
buildPhase = ''
runHook preBuild
mkdir -p "$out/bin"
dart build cli --target "bin/jadb.dart"
runHook postBuild
'';
installPhase = ''
runHook preInstall
mkdir -p "$out"
mv build/cli/*/bundle/* "$out/"
runHook postInstall
'';
autoPubspecLock = ../pubspec.lock;
meta.mainProgram = "jadb";

View File

@@ -5,18 +5,18 @@ packages:
dependency: transitive
description:
name: _fe_analyzer_shared
sha256: "3b19a47f6ea7c2632760777c78174f47f6aec1e05f0cd611380d4593b8af1dbc"
sha256: e55636ed79578b9abca5fecf9437947798f5ef7456308b5cb85720b793eac92f
url: "https://pub.dev"
source: hosted
version: "96.0.0"
version: "82.0.0"
analyzer:
dependency: transitive
description:
name: analyzer
sha256: "0c516bc4ad36a1a75759e54d5047cb9d15cded4459df01aa35a0b5ec7db2c2a0"
sha256: "904ae5bb474d32c38fb9482e2d925d5454cda04ddd0e55d2e6826bc72f6ba8c0"
url: "https://pub.dev"
source: hosted
version: "10.2.0"
version: "7.4.5"
args:
dependency: "direct main"
description:
@@ -49,14 +49,6 @@ packages:
url: "https://pub.dev"
source: hosted
version: "0.2.0"
code_assets:
dependency: transitive
description:
name: code_assets
sha256: "83ccdaa064c980b5596c35dd64a8d3ecc68620174ab9b90b6343b753aa721687"
url: "https://pub.dev"
source: hosted
version: "1.0.0"
collection:
dependency: "direct main"
description:
@@ -77,42 +69,42 @@ packages:
dependency: transitive
description:
name: coverage
sha256: "5da775aa218eaf2151c721b16c01c7676fbfdd99cebba2bf64e8b807a28ff94d"
sha256: "802bd084fb82e55df091ec8ad1553a7331b61c08251eef19a508b6f3f3a9858d"
url: "https://pub.dev"
source: hosted
version: "1.15.0"
version: "1.13.1"
crypto:
dependency: transitive
description:
name: crypto
sha256: c8ea0233063ba03258fbcf2ca4d6dadfefe14f02fab57702265467a19f27fadf
sha256: "1e445881f28f22d6140f181e07737b22f1e099a5e1ff94b0af2f9e4a463f4855"
url: "https://pub.dev"
source: hosted
version: "3.0.7"
version: "3.0.6"
csv:
dependency: "direct main"
description:
name: csv
sha256: bef2950f7a753eb82f894a2eabc3072e73cf21c17096296a5a992797e50b1d0d
sha256: c6aa2679b2a18cb57652920f674488d89712efaf4d3fdf2e537215b35fc19d6c
url: "https://pub.dev"
source: hosted
version: "7.1.0"
version: "6.0.0"
equatable:
dependency: "direct main"
description:
name: equatable
sha256: "3e0141505477fd8ad55d6eb4e7776d3fe8430be8e497ccb1521370c3f21a3e2b"
sha256: "567c64b3cb4cf82397aac55f4f0cbd3ca20d77c6c03bedbc4ceaddc08904aef7"
url: "https://pub.dev"
source: hosted
version: "2.0.8"
version: "2.0.7"
ffi:
dependency: transitive
description:
name: ffi
sha256: "6d7fd89431262d8f3125e81b50d3847a091d846eafcd4fdb88dd06f36d705a45"
sha256: "289279317b4b16eb2bb7e271abccd4bf84ec9bdcbe999e278a94b804f5630418"
url: "https://pub.dev"
source: hosted
version: "2.2.0"
version: "2.1.4"
file:
dependency: transitive
description:
@@ -137,14 +129,6 @@ packages:
url: "https://pub.dev"
source: hosted
version: "2.1.3"
hooks:
dependency: transitive
description:
name: hooks
sha256: "7a08a0d684cb3b8fb604b78455d5d352f502b68079f7b80b831c62220ab0a4f6"
url: "https://pub.dev"
source: hosted
version: "1.0.1"
http_multi_server:
dependency: transitive
description:
@@ -169,14 +153,22 @@ packages:
url: "https://pub.dev"
source: hosted
version: "1.0.5"
js:
dependency: transitive
description:
name: js
sha256: "53385261521cc4a0c4658fd0ad07a7d14591cf8fc33abbceae306ddb974888dc"
url: "https://pub.dev"
source: hosted
version: "0.7.2"
lints:
dependency: "direct dev"
description:
name: lints
sha256: "12f842a479589fea194fe5c5a3095abc7be0c1f2ddfa9a0e76aed1dbd26a87df"
sha256: c35bb79562d980e9a453fc715854e1ed39e24e7d0297a880ef54e17f9874a9d7
url: "https://pub.dev"
source: hosted
version: "6.1.0"
version: "5.1.1"
logging:
dependency: transitive
description:
@@ -189,18 +181,18 @@ packages:
dependency: transitive
description:
name: matcher
sha256: "12956d0ad8390bbcc63ca2e1469c0619946ccb52809807067a7020d57e647aa6"
sha256: dc58c723c3c24bf8d3e2d3ad3f2f9d7bd9cf43ec6feaa64181775e60190153f2
url: "https://pub.dev"
source: hosted
version: "0.12.18"
version: "0.12.17"
meta:
dependency: transitive
description:
name: meta
sha256: "9f29b9bcc8ee287b1a31e0d01be0eae99a930dbffdaecf04b3f3d82a969f296f"
sha256: "23f08335362185a5ea2ad3a4e597f1375e78bce8a040df5c600c8d3552ef2394"
url: "https://pub.dev"
source: hosted
version: "1.18.1"
version: "1.17.0"
mime:
dependency: transitive
description:
@@ -209,14 +201,6 @@ packages:
url: "https://pub.dev"
source: hosted
version: "2.0.0"
native_toolchain_c:
dependency: transitive
description:
name: native_toolchain_c
sha256: "89e83885ba09da5fdf2cdacc8002a712ca238c28b7f717910b34bcd27b0d03ac"
url: "https://pub.dev"
source: hosted
version: "0.17.4"
node_preamble:
dependency: transitive
description:
@@ -234,7 +218,7 @@ packages:
source: hosted
version: "2.2.0"
path:
dependency: "direct main"
dependency: transitive
description:
name: path
sha256: "75cca69d1490965be98c73ceaea117e8a04dd21217b37b292c9ddbec0d955bc5"
@@ -245,18 +229,18 @@ packages:
dependency: transitive
description:
name: petitparser
sha256: "91bd59303e9f769f108f8df05e371341b15d59e995e6806aefab827b58336675"
sha256: "07c8f0b1913bcde1ff0d26e57ace2f3012ccbf2b204e070290dad3bb22797646"
url: "https://pub.dev"
source: hosted
version: "7.0.2"
version: "6.1.0"
pool:
dependency: transitive
description:
name: pool
sha256: "978783255c543aa3586a1b3c21f6e9d720eb315376a915872c61ef8b5c20177d"
sha256: "20fe868b6314b322ea036ba325e6fc0711a22948856475e2c2b6306e8ab39c2a"
url: "https://pub.dev"
source: hosted
version: "1.5.2"
version: "1.5.1"
pub_semver:
dependency: transitive
description:
@@ -317,34 +301,34 @@ packages:
dependency: transitive
description:
name: source_span
sha256: "56a02f1f4cd1a2d96303c0144c93bd6d909eea6bee6bf5a0e0b685edbd4c47ab"
sha256: "254ee5351d6cb365c859e20ee823c3bb479bf4a293c22d17a9f1bf144ce86f7c"
url: "https://pub.dev"
source: hosted
version: "1.10.2"
version: "1.10.1"
sqflite_common:
dependency: "direct main"
description:
name: sqflite_common
sha256: "6ef422a4525ecc601db6c0a2233ff448c731307906e92cabc9ba292afaae16a6"
sha256: "84731e8bfd8303a3389903e01fb2141b6e59b5973cacbb0929021df08dddbe8b"
url: "https://pub.dev"
source: hosted
version: "2.5.6"
version: "2.5.5"
sqflite_common_ffi:
dependency: "direct main"
description:
name: sqflite_common_ffi
sha256: c59fcdc143839a77581f7a7c4de018e53682408903a0a0800b95ef2dc4033eff
sha256: "1f3ef3888d3bfbb47785cc1dda0dc7dd7ebd8c1955d32a9e8e9dae1e38d1c4c1"
url: "https://pub.dev"
source: hosted
version: "2.4.0+2"
version: "2.3.5"
sqlite3:
dependency: "direct main"
dependency: transitive
description:
name: sqlite3
sha256: b7cf6b37667f6a921281797d2499ffc60fb878b161058d422064f0ddc78f6aa6
sha256: "310af39c40dd0bb2058538333c9d9840a2725ae0b9f77e4fd09ad6696aa8f66e"
url: "https://pub.dev"
source: hosted
version: "3.1.6"
version: "2.7.5"
stack_trace:
dependency: transitive
description:
@@ -373,10 +357,10 @@ packages:
dependency: transitive
description:
name: synchronized
sha256: c254ade258ec8282947a0acbbc90b9575b4f19673533ee46f2f6e9b3aeefd7c0
sha256: "0669c70faae6270521ee4f05bffd2919892d42d1276e6c495be80174b6bc0ef6"
url: "https://pub.dev"
source: hosted
version: "3.4.0"
version: "3.3.1"
term_glyph:
dependency: transitive
description:
@@ -389,26 +373,26 @@ packages:
dependency: "direct dev"
description:
name: test
sha256: "54c516bbb7cee2754d327ad4fca637f78abfc3cbcc5ace83b3eda117e42cd71a"
sha256: "0561f3a2cfd33d10232360f16dfcab9351cfb7ad9b23e6cd6e8c7fb0d62c7ac3"
url: "https://pub.dev"
source: hosted
version: "1.29.0"
version: "1.26.1"
test_api:
dependency: transitive
description:
name: test_api
sha256: "93167629bfc610f71560ab9312acdda4959de4df6fac7492c89ff0d3886f6636"
sha256: "522f00f556e73044315fa4585ec3270f1808a4b186c936e612cab0b565ff1e00"
url: "https://pub.dev"
source: hosted
version: "0.7.9"
version: "0.7.6"
test_core:
dependency: transitive
description:
name: test_core
sha256: "394f07d21f0f2255ec9e3989f21e54d3c7dc0e6e9dbce160e5a9c1a6be0e2943"
sha256: "8619a9a45be044b71fe2cd6b77b54fd60f1c67904c38d48706e2852a2bda1c60"
url: "https://pub.dev"
source: hosted
version: "0.6.15"
version: "0.6.10"
typed_data:
dependency: transitive
description:
@@ -421,18 +405,18 @@ packages:
dependency: transitive
description:
name: vm_service
sha256: "45caa6c5917fa127b5dbcfbd1fa60b14e583afdc08bfc96dda38886ca252eb60"
sha256: ddfa8d30d89985b96407efce8acbdd124701f96741f2d981ca860662f1c0dc02
url: "https://pub.dev"
source: hosted
version: "15.0.2"
version: "15.0.0"
watcher:
dependency: transitive
description:
name: watcher
sha256: "1398c9f081a753f9226febe8900fce8f7d0a67163334e1c94a2438339d79d635"
sha256: "69da27e49efa56a15f8afe8f4438c4ec02eff0a117df1b22ea4aad194fe1c104"
url: "https://pub.dev"
source: hosted
version: "1.2.1"
version: "1.1.1"
web:
dependency: transitive
description:
@@ -469,10 +453,10 @@ packages:
dependency: "direct main"
description:
name: xml
sha256: "971043b3a0d3da28727e40ed3e0b5d18b742fa5a68665cca88e74b7876d5e025"
sha256: b015a8ad1c488f66851d762d3090a21c600e479dc75e68328c52774040cf9226
url: "https://pub.dev"
source: hosted
version: "6.6.1"
version: "6.5.0"
yaml:
dependency: transitive
description:
@@ -482,4 +466,4 @@ packages:
source: hosted
version: "3.1.3"
sdks:
dart: ">=3.10.1 <4.0.0"
dart: ">=3.7.0 <4.0.0"

View File

@@ -4,31 +4,24 @@ version: 1.0.0
homepage: https://git.pvv.ntnu.no/oysteikt/jadb
environment:
sdk: '^3.9.0'
sdk: '>=3.2.0 <4.0.0'
dependencies:
args: ^2.7.0
collection: ^1.19.0
csv: ^7.1.0
csv: ^6.0.0
equatable: ^2.0.0
path: ^1.9.1
sqflite_common: ^2.5.0
sqflite_common_ffi: ^2.3.0
sqlite3: ^3.1.6
xml: ^6.5.0
dev_dependencies:
lints: ^6.0.0
lints: ^5.0.0
test: ^1.25.15
executables:
jadb: jadb
hooks:
user_defines:
sqlite3:
source: system
topics:
- database
- dictionary

View File

@@ -3,7 +3,7 @@ import 'package:jadb/const_data/kanji_grades.dart';
import 'package:test/test.dart';
void main() {
test('Assert 2136 kanji in jouyou set', () {
expect(jouyouKanjiByGrades.values.flattenedToSet.length, 2136);
test("Assert 2136 kanji in jouyou set", () {
expect(JOUYOU_KANJI_BY_GRADES.values.flattenedToSet.length, 2136);
});
}

View File

@@ -1,20 +1,30 @@
import 'dart:ffi';
import 'dart:io';
import 'package:jadb/models/create_empty_db.dart';
import 'package:jadb/search.dart';
import 'package:sqflite_common_ffi/sqflite_ffi.dart';
// import 'package:sqlite3/open.dart';
import 'package:test/test.dart';
import 'package:sqlite3/open.dart';
Future<DatabaseExecutor> setupInMemoryDatabase() async {
final dbConnection = await createDatabaseFactoryFfi().openDatabase(
':memory:',
);
Future<DatabaseExecutor> setup_inmemory_database() async {
final libsqlitePath = Platform.environment['LIBSQLITE_PATH'];
return dbConnection;
if (libsqlitePath == null) {
throw Exception("LIBSQLITE_PATH is not set");
}
final db_connection = await createDatabaseFactoryFfi(
ffiInit: () =>
open.overrideForAll(() => DynamicLibrary.open(libsqlitePath)),
).openDatabase(':memory:');
return db_connection;
}
void main() {
test('Create empty db', () async {
final db = await setupInMemoryDatabase();
test("Create empty db", () async {
final db = await setup_inmemory_database();
await createEmptyDb(db);

View File

@@ -4,26 +4,29 @@ import 'package:test/test.dart';
import 'setup_database_connection.dart';
void main() {
test('Filter kanji', () async {
final connection = await setupDatabaseConnection();
test("Filter kanji", () async {
final connection = await setup_database_connection();
final result = await connection.filterKanji([
'a',
'b',
'c',
'',
'',
'',
'',
'',
'',
'.',
'!',
'@',
';',
'',
], deduplicate: false);
final result = await connection.filterKanji(
[
"a",
"b",
"c",
"",
"",
"",
"",
"",
"",
".",
"!",
"@",
";",
"",
],
deduplicate: false,
);
expect(result.join(), '漢字地字');
expect(result.join(), "漢字地字");
});
}

View File

@@ -5,17 +5,17 @@ import 'package:test/test.dart';
import 'setup_database_connection.dart';
void main() {
test('Search a kanji', () async {
final connection = await setupDatabaseConnection();
test("Search a kanji", () async {
final connection = await setup_database_connection();
final result = await connection.jadbSearchKanji('');
expect(result, isNotNull);
});
group('Search all jouyou kanji', () {
jouyouKanjiByGrades.forEach((grade, characters) {
test('Search all kanji in grade $grade', () async {
final connection = await setupDatabaseConnection();
group("Search all jouyou kanji", () {
JOUYOU_KANJI_BY_GRADES.forEach((grade, characters) {
test("Search all kanji in grade $grade", () async {
final connection = await setup_database_connection();
for (final character in characters) {
final result = await connection.jadbSearchKanji(character);

View File

@@ -1,257 +0,0 @@
import 'package:jadb/models/common/jlpt_level.dart';
import 'package:jadb/models/word_search/word_search_match_span.dart';
import 'package:jadb/models/word_search/word_search_result.dart';
import 'package:jadb/models/word_search/word_search_ruby.dart';
import 'package:jadb/models/word_search/word_search_sense.dart';
import 'package:jadb/models/word_search/word_search_sources.dart';
import 'package:test/test.dart';
void main() {
test('Infer match whole word', () {
final wordSearchResult = WordSearchResult(
entryId: 0,
score: 0,
isCommon: false,
jlptLevel: JlptLevel.none,
kanjiInfo: {},
readingInfo: {},
japanese: [WordSearchRuby(base: '仮名')],
senses: [],
sources: WordSearchSources.empty(),
);
wordSearchResult.inferMatchSpans('仮名');
expect(wordSearchResult.matchSpans, [
WordSearchMatchSpan(
spanType: WordSearchMatchSpanType.kanji,
start: 0,
end: 2,
index: 0,
),
]);
});
test('Infer match part of word', () {
final wordSearchResult = WordSearchResult(
entryId: 0,
score: 0,
isCommon: false,
jlptLevel: JlptLevel.none,
kanjiInfo: {},
readingInfo: {},
japanese: [WordSearchRuby(base: '仮名')],
senses: [],
sources: WordSearchSources.empty(),
);
wordSearchResult.inferMatchSpans('');
expect(wordSearchResult.matchSpans, [
WordSearchMatchSpan(
spanType: WordSearchMatchSpanType.kanji,
start: 0,
end: 1,
index: 0,
),
]);
});
test('Infer match in middle of word', () {
final wordSearchResult = WordSearchResult(
entryId: 0,
score: 0,
isCommon: false,
jlptLevel: JlptLevel.none,
kanjiInfo: {},
readingInfo: {},
japanese: [WordSearchRuby(base: 'ありがとう')],
senses: [],
sources: WordSearchSources.empty(),
);
wordSearchResult.inferMatchSpans('りがと');
expect(wordSearchResult.matchSpans, [
WordSearchMatchSpan(
spanType: WordSearchMatchSpanType.kanji,
start: 1,
end: 4,
index: 0,
),
]);
});
test('Infer match in furigana', () {
final wordSearchResult = WordSearchResult(
entryId: 0,
score: 0,
isCommon: false,
jlptLevel: JlptLevel.none,
kanjiInfo: {},
readingInfo: {},
japanese: [WordSearchRuby(base: '仮名', furigana: 'かな')],
senses: [],
sources: WordSearchSources.empty(),
);
wordSearchResult.inferMatchSpans('かな');
expect(wordSearchResult.matchSpans, [
WordSearchMatchSpan(
spanType: WordSearchMatchSpanType.kana,
start: 0,
end: 2,
index: 0,
),
]);
});
test('Infer match in sense', () {
final wordSearchResult = WordSearchResult(
entryId: 0,
score: 0,
isCommon: false,
jlptLevel: JlptLevel.none,
kanjiInfo: {},
readingInfo: {},
japanese: [WordSearchRuby(base: '仮名')],
senses: [
WordSearchSense(
antonyms: [],
dialects: [],
englishDefinitions: ['kana'],
fields: [],
info: [],
languageSource: [],
misc: [],
partsOfSpeech: [],
restrictedToKanji: [],
restrictedToReading: [],
seeAlso: [],
),
],
sources: WordSearchSources.empty(),
);
wordSearchResult.inferMatchSpans('kana');
expect(wordSearchResult.matchSpans, [
WordSearchMatchSpan(
spanType: WordSearchMatchSpanType.sense,
start: 0,
end: 4,
index: 0,
),
]);
});
test('Infer multiple matches', () {
final wordSearchResult = WordSearchResult(
entryId: 0,
score: 0,
isCommon: false,
jlptLevel: JlptLevel.none,
kanjiInfo: {},
readingInfo: {},
japanese: [WordSearchRuby(base: '仮名', furigana: 'かな')],
senses: [
WordSearchSense(
antonyms: [],
dialects: [],
englishDefinitions: ['kana', 'the kana'],
fields: [],
info: [],
languageSource: [],
misc: [],
partsOfSpeech: [],
restrictedToKanji: [],
restrictedToReading: [],
seeAlso: [],
),
],
sources: WordSearchSources.empty(),
);
wordSearchResult.inferMatchSpans('kana');
expect(wordSearchResult.matchSpans, [
WordSearchMatchSpan(
spanType: WordSearchMatchSpanType.sense,
start: 0,
end: 4,
index: 0,
),
WordSearchMatchSpan(
spanType: WordSearchMatchSpanType.sense,
start: 4,
end: 8,
index: 0,
subIndex: 1,
),
]);
});
test('Infer match with no matches', () {
final wordSearchResult = WordSearchResult(
entryId: 0,
score: 0,
isCommon: false,
jlptLevel: JlptLevel.none,
kanjiInfo: {},
readingInfo: {},
japanese: [WordSearchRuby(base: '仮名', furigana: 'かな')],
senses: [
WordSearchSense(
antonyms: [],
dialects: [],
englishDefinitions: ['kana'],
fields: [],
info: [],
languageSource: [],
misc: [],
partsOfSpeech: [],
restrictedToKanji: [],
restrictedToReading: [],
seeAlso: [],
),
],
sources: WordSearchSources.empty(),
);
wordSearchResult.inferMatchSpans('xyz');
expect(wordSearchResult.matchSpans, isEmpty);
});
test('Infer multiple matches of same substring', () {
final wordSearchResult = WordSearchResult(
entryId: 0,
score: 0,
isCommon: false,
jlptLevel: JlptLevel.none,
kanjiInfo: {},
readingInfo: {},
japanese: [WordSearchRuby(base: 'ああ')],
senses: [],
sources: WordSearchSources.empty(),
);
wordSearchResult.inferMatchSpans('');
expect(wordSearchResult.matchSpans, [
WordSearchMatchSpan(
spanType: WordSearchMatchSpanType.kanji,
start: 0,
end: 1,
index: 0,
),
WordSearchMatchSpan(
spanType: WordSearchMatchSpanType.kanji,
start: 1,
end: 2,
index: 0,
),
]);
});
}

View File

@@ -3,22 +3,22 @@ import 'dart:io';
import 'package:jadb/_data_ingestion/open_local_db.dart';
import 'package:sqflite_common/sqlite_api.dart';
Future<Database> setupDatabaseConnection() async {
final libSqlitePath = Platform.environment['LIBSQLITE_PATH'];
final jadbPath = Platform.environment['JADB_PATH'];
Future<Database> setup_database_connection() async {
final lib_sqlite_path = Platform.environment['LIBSQLITE_PATH'];
final jadb_path = Platform.environment['JADB_PATH'];
if (libSqlitePath == null) {
throw Exception('LIBSQLITE_PATH is not set');
if (lib_sqlite_path == null) {
throw Exception("LIBSQLITE_PATH is not set");
}
if (jadbPath == null) {
throw Exception('JADB_PATH is not set');
if (jadb_path == null) {
throw Exception("JADB_PATH is not set");
}
final dbConnection = await openLocalDb(
libsqlitePath: libSqlitePath,
jadbPath: jadbPath,
final db_connection = await openLocalDb(
libsqlitePath: lib_sqlite_path,
jadbPath: jadb_path,
);
return dbConnection;
return db_connection;
}

View File

@@ -4,59 +4,59 @@ import 'package:test/test.dart';
import 'setup_database_connection.dart';
void main() {
test('Search a word - english - auto', () async {
final connection = await setupDatabaseConnection();
final result = await connection.jadbSearchWord('kana');
test("Search a word - english - auto", () async {
final connection = await setup_database_connection();
final result = await connection.jadbSearchWord("kana");
expect(result, isNotNull);
});
test('Get word search count - english - auto', () async {
final connection = await setupDatabaseConnection();
final result = await connection.jadbSearchWordCount('kana');
test("Get word search count - english - auto", () async {
final connection = await setup_database_connection();
final result = await connection.jadbSearchWordCount("kana");
expect(result, isNotNull);
});
test('Search a word - japanese kana - auto', () async {
final connection = await setupDatabaseConnection();
final result = await connection.jadbSearchWord('かな');
test("Search a word - japanese kana - auto", () async {
final connection = await setup_database_connection();
final result = await connection.jadbSearchWord("かな");
expect(result, isNotNull);
});
test('Get word search count - japanese kana - auto', () async {
final connection = await setupDatabaseConnection();
final result = await connection.jadbSearchWordCount('かな');
test("Get word search count - japanese kana - auto", () async {
final connection = await setup_database_connection();
final result = await connection.jadbSearchWordCount("かな");
expect(result, isNotNull);
});
test('Search a word - japanese kanji - auto', () async {
final connection = await setupDatabaseConnection();
final result = await connection.jadbSearchWord('仮名');
test("Search a word - japanese kanji - auto", () async {
final connection = await setup_database_connection();
final result = await connection.jadbSearchWord("仮名");
expect(result, isNotNull);
});
test('Get word search count - japanese kanji - auto', () async {
final connection = await setupDatabaseConnection();
final result = await connection.jadbSearchWordCount('仮名');
test("Get word search count - japanese kanji - auto", () async {
final connection = await setup_database_connection();
final result = await connection.jadbSearchWordCount("仮名");
expect(result, isNotNull);
});
test('Get a word by id', () async {
final connection = await setupDatabaseConnection();
test("Get a word by id", () async {
final connection = await setup_database_connection();
final result = await connection.jadbGetWordById(1577090);
expect(result, isNotNull);
});
test(
'Serialize all words',
"Serialize all words",
() async {
final connection = await setupDatabaseConnection();
final connection = await setup_database_connection();
// Test serializing all words
for (final letter in 'aiueoksthnmyrw'.split('')) {
for (final letter in "aiueoksthnmyrw".split("")) {
await connection.jadbSearchWord(letter);
}
},
timeout: Timeout.factor(100),
skip: 'Very slow test',
skip: "Very slow test",
);
}

View File

@@ -2,65 +2,65 @@ import 'package:jadb/util/romaji_transliteration.dart';
import 'package:test/test.dart';
void main() {
group('Romaji -> Hiragana', () {
test('Basic test', () {
final result = transliterateLatinToHiragana('katamari');
expect(result, 'かたまり');
group("Romaji -> Hiragana", () {
test("Basic test", () {
final result = transliterateLatinToHiragana("katamari");
expect(result, "かたまり");
});
test('Basic test with diacritics', () {
final result = transliterateLatinToHiragana('gadamari');
expect(result, 'がだまり');
test("Basic test with diacritics", () {
final result = transliterateLatinToHiragana("gadamari");
expect(result, "がだまり");
});
test('wi and we', () {
final result = transliterateLatinToHiragana('wiwe');
expect(result, 'うぃうぇ');
test("wi and we", () {
final result = transliterateLatinToHiragana("wiwe");
expect(result, "うぃうぇ");
});
test('nb = mb', () {
final result = transliterateLatinToHiragana('kanpai');
expect(result, 'かんぱい');
test("nb = mb", () {
final result = transliterateLatinToHiragana("kanpai");
expect(result, "かんぱい");
final result2 = transliterateLatinToHiragana('kampai');
expect(result2, 'かんぱい');
final result2 = transliterateLatinToHiragana("kampai");
expect(result2, "かんぱい");
});
test('Double n', () {
final result = transliterateLatinToHiragana('konnichiha');
expect(result, 'こんにちは');
test("Double n", () {
final result = transliterateLatinToHiragana("konnichiha");
expect(result, "こんにちは");
});
test('Double consonant', () {
final result = transliterateLatinToHiragana('kappa');
expect(result, 'かっぱ');
test("Double consonant", () {
final result = transliterateLatinToHiragana("kappa");
expect(result, "かっぱ");
});
});
group('Hiragana -> Romaji', () {
test('Basic test', () {
final result = transliterateHiraganaToLatin('かたまり');
expect(result, 'katamari');
group("Hiragana -> Romaji", () {
test("Basic test", () {
final result = transliterateHiraganaToLatin("かたまり");
expect(result, "katamari");
});
test('Basic test with diacritics', () {
final result = transliterateHiraganaToLatin('がだまり');
expect(result, 'gadamari');
test("Basic test with diacritics", () {
final result = transliterateHiraganaToLatin("がだまり");
expect(result, "gadamari");
});
test('whi and whe', () {
final result = transliterateHiraganaToLatin('うぃうぇ');
expect(result, 'whiwhe');
test("whi and whe", () {
final result = transliterateHiraganaToLatin("うぃうぇ");
expect(result, "whiwhe");
});
test('Double n', () {
final result = transliterateHiraganaToLatin('こんにちは');
expect(result, 'konnichiha');
test("Double n", () {
final result = transliterateHiraganaToLatin("こんにちは");
expect(result, "konnichiha");
});
test('Double consonant', () {
final result = transliterateHiraganaToLatin('かっぱ');
expect(result, 'kappa');
test("Double consonant", () {
final result = transliterateHiraganaToLatin("かっぱ");
expect(result, "kappa");
});
});
}