init commit

2022-06-20 20:06:07 +02:00
commit 5cf0b95d8b
21 changed files with 3043 additions and 0 deletions
@@ -0,0 +1,235 @@
+import '../common.dart';
+import '../objects.dart';
+
+class TableNames {
+  static const String entry = 'JMdict_Entry';
+  static const String entryByKana = 'JMdict_EntryByKana';
+  static const String entryByEnglish = 'JMdict_EntryByEnglish';
+  static const String kanjiElement = 'JMdict_KanjiElement';
+  static const String kanjiInfo = 'JMdict_KanjiElementInfo';
+  static const String readingElement = 'JMdict_ReadingElement';
+  static const String readingInfo = 'JMdict_ReadingElementInfo';
+  static const String readingRestriction = 'JMdict_ReadingElementRestriction';
+  static const String sense = 'JMdict_Sense';
+  static const String senseAntonyms = 'JMdict_SenseAntonym';
+  static const String senseDialect = 'JMdict_SenseDialect';
+  static const String senseField = 'JMdict_SenseField';
+  static const String senseGlossary = 'JMdict_SenseGlossary';
+  static const String senseInfo = 'JMdict_SenseInfo';
+  static const String senseLanguageSource = 'JMdict_SenseLanguageSource';
+  static const String senseMisc = 'JMdict_SenseMisc';
+  static const String sensePOS = 'JMdict_SensePOS';
+  static const String senseRestrictedToKanji = 'JMdict_SenseRestrictedToKanji';
+  static const String senseRestrictedToReading = 'JMdict_SenseRestrictedToReading';
+  static const String senseSeeAlso = 'JMdict_SenseSeeAlso';
+}
+
+abstract class Element extends SQLWritable {
+  final String reading;
+  final int? news;
+  final int? ichi;
+  final int? spec;
+  final int? gai;
+  final int? nf;
+  const Element({
+    required this.reading,
+    this.news,
+    this.ichi,
+    this.spec,
+    this.gai,
+    this.nf,
+  });
+
+  Map<String, Object?> get sqlValue => {
+        'reading': reading,
+        'news': news,
+        'ichi': ichi,
+        'spec': spec,
+        'gai': gai,
+        'nf': nf,
+      };
+}
+
+class KanjiElement extends Element {
+  List<String> info;
+
+  KanjiElement({
+    this.info = const [],
+    required String reading,
+    int? news,
+    int? ichi,
+    int? spec,
+    int? gai,
+    int? nf,
+  }) : super(
+          reading: reading,
+          news: news,
+          ichi: ichi,
+          spec: spec,
+          gai: gai,
+          nf: nf,
+        );
+}
+
+class ReadingElement extends Element {
+  List<String> info;
+  List<String> restrictions;
+
+  ReadingElement({
+    this.info = const [],
+    this.restrictions = const [],
+    required String reading,
+    int? news,
+    int? ichi,
+    int? spec,
+    int? gai,
+    int? nf,
+  }) : super(
+          reading: reading,
+          news: news,
+          ichi: ichi,
+          spec: spec,
+          gai: gai,
+          nf: nf,
+        );
+}
+
+class LanguageSource extends SQLWritable {
+  final String language;
+  final String? phrase;
+  final bool fullyDescribesSense;
+  final bool constructedFromSmallerWords;
+
+  const LanguageSource({
+    required this.language,
+    this.phrase,
+    this.fullyDescribesSense = true,
+    this.constructedFromSmallerWords = false,
+  });
+
+  @override
+  Map<String, Object?> get sqlValue => {
+        'language': language,
+        'phrase': phrase,
+        'fullyDescribesSense': fullyDescribesSense,
+        'constructedFromSmallerWords': constructedFromSmallerWords,
+      };
+}
+
+class Glossary extends SQLWritable {
+  final String language;
+  final String phrase;
+  final String? type;
+
+  const Glossary({
+    required this.language,
+    required this.phrase,
+    this.type,
+  });
+
+  Map<String, Object?> get sqlValue => {
+        'language': language,
+        'phrase': phrase,
+        'type': type,
+      };
+}
+
+final kanaRegex =
+    RegExp(r'^[\p{Script=Katakana}\p{Script=Hiragana}ー]+$', unicode: true);
+
+class XRefParts {
+  final String? kanjiRef;
+  final String? readingRef;
+  final int? senseNum;
+
+  const XRefParts({
+    this.kanjiRef,
+    this.readingRef,
+    this.senseNum,
+  }) : assert(kanjiRef != null || readingRef != null);
+
+  factory XRefParts.fromString(String s) {
+    final parts = s.split('・');
+    if (parts.length == 1) {
+      if (parts[0].contains(kanaRegex)) {
+        return XRefParts(readingRef: parts[0]);
+      }
+      return XRefParts(kanjiRef: parts[0]);
+    } else if (parts.length == 2) {
+      if (int.tryParse(parts[1]) != null) {
+        if (parts[0].contains(kanaRegex)) {
+          return XRefParts(readingRef: parts[0], senseNum: int.parse(parts[1]));
+        }
+        return XRefParts(kanjiRef: parts[0], senseNum: int.parse(parts[1]));
+      }
+      return XRefParts(kanjiRef: parts[0], readingRef: parts[1]);
+    } else if (parts.length == 3) {
+      return XRefParts(
+        kanjiRef: parts[0],
+        readingRef: parts[1],
+        senseNum: int.parse(parts[2]),
+      );
+    }
+
+    return XRefParts();
+  }
+}
+
+class XRef {
+  final String entryId;
+  final String reading;
+
+  const XRef({
+    required this.entryId,
+    required this.reading,
+  });
+}
+
+class Sense extends SQLWritable {
+  final int id;
+  final List<XRefParts> antonyms;
+  final List<String> dialects;
+  final List<String> fields;
+  final List<String> info;
+  final List<LanguageSource> languageSource;
+  final List<Glossary> glossary;
+  final List<String> misc;
+  final List<String> pos;
+  final List<String> restrictedToKanji;
+  final List<String> restrictedToReading;
+  final List<XRefParts> seeAlso;
+
+  const Sense({
+    required this.id,
+    this.antonyms = const [],
+    this.dialects = const [],
+    this.fields = const [],
+    this.info = const [],
+    this.languageSource = const [],
+    this.glossary = const [],
+    this.misc = const [],
+    this.pos = const [],
+    this.restrictedToKanji = const [],
+    this.restrictedToReading = const [],
+    this.seeAlso = const [],
+  });
+
+  @override
+  Map<String, Object?> get sqlValue => {};
+}
+
+class Entry extends SQLWritable {
+  final int id;
+  final List<KanjiElement> kanji;
+  final List<ReadingElement> readings;
+  final List<Sense> senses;
+
+  const Entry({
+    required this.id,
+    required this.kanji,
+    required this.readings,
+    required this.senses,
+  });
+
+  Map<String, Object?> get sqlValue => {'id': id};
+}
@@ -0,0 +1,346 @@
+import 'dart:collection';
+import 'dart:io';
+
+import 'package:sqflite_common/sqlite_api.dart';
+import 'package:xml/xml.dart';
+
+import '../romaji_transliteration.dart';
+import 'objects.dart';
+
+List<int?> getPriNums(XmlElement e, String prefix) {
+  int? news, ichi, spec, gai, nf;
+  for (final pri in e.findElements('${prefix}_pri')) {
+    final txt = pri.innerText;
+    if (txt.startsWith('news'))
+      news = int.parse(txt.substring(4));
+    else if (txt.startsWith('ichi'))
+      ichi = int.parse(txt.substring(4));
+    else if (txt.startsWith('spec'))
+      spec = int.parse(txt.substring(4));
+    else if (txt.startsWith('gai'))
+      gai = int.parse(txt.substring(3));
+    else if (txt.startsWith('nf')) nf = int.parse(txt.substring(2));
+  }
+  return [news, ichi, spec, gai, nf];
+}
+
+List<Entry> transformXML(XmlElement root) {
+  final List<Entry> entries = [];
+
+  int senseId = 0;
+  for (final entry in root.childElements) {
+    final entryId = int.parse(entry.findElements('ent_seq').first.innerText);
+
+    final List<KanjiElement> kanjiEls = [];
+    final List<ReadingElement> readingEls = [];
+    final List<Sense> senses = [];
+
+    for (final k_ele in entry.findAllElements('k_ele')) {
+      final ke_pri = getPriNums(k_ele, 'ke');
+      kanjiEls.add(
+        KanjiElement(
+          info: k_ele.findElements('ke_inf').map((e) => e.innerText).toList(),
+          reading: k_ele.findElements('keb').first.innerText,
+          news: ke_pri[0],
+          ichi: ke_pri[1],
+          spec: ke_pri[2],
+          gai: ke_pri[3],
+          nf: ke_pri[4],
+        ),
+      );
+    }
+
+    for (final r_ele in entry.findAllElements('r_ele')) {
+      final re_pri = getPriNums(r_ele, 're');
+      readingEls.add(
+        ReadingElement(
+          info: r_ele
+              .findElements('re_inf')
+              .map((e) => e.innerText.substring(1, e.innerText.length - 1))
+              .toList(),
+          restrictions:
+              r_ele.findElements('re_restr').map((e) => e.innerText).toList(),
+          reading: r_ele.findElements('reb').first.innerText,
+          news: re_pri[0],
+          ichi: re_pri[1],
+          spec: re_pri[2],
+          gai: re_pri[3],
+          nf: re_pri[4],
+        ),
+      );
+    }
+
+    for (final sense in entry.findAllElements('sense')) {
+      senseId++;
+      senses.add(
+        Sense(
+          id: senseId,
+          restrictedToKanji:
+              sense.findElements('stagk').map((e) => e.innerText).toList(),
+          restrictedToReading:
+              sense.findElements('stagr').map((e) => e.innerText).toList(),
+          pos: sense
+              .findElements('pos')
+              .map((e) => e.innerText.substring(1, e.innerText.length - 1))
+              .toList(),
+          misc: sense
+              .findElements('misc')
+              .map((e) => e.innerText.substring(1, e.innerText.length - 1))
+              .toList(),
+          dialects: sense
+              .findElements('dial')
+              .map((e) => e.innerText.substring(1, e.innerText.length - 1))
+              .toList(),
+          info: sense.findElements('s_inf').map((e) => e.innerText).toList(),
+          languageSource: sense
+              .findElements('lsource')
+              .map(
+                (e) => LanguageSource(
+                  language: e.getAttribute('xml:lang') ?? 'eng',
+                  fullyDescribesSense: e.getAttribute('ls_type') == 'part',
+                  constructedFromSmallerWords:
+                      e.getAttribute('ls_wasei') == 'y',
+                ),
+              )
+              .toList(),
+          glossary: sense
+              .findElements('gloss')
+              .map(
+                (e) => Glossary(
+                  language: e.getAttribute('xml:lang') ?? 'eng',
+                  phrase: e.innerText,
+                  type: e.getAttribute('g_type'),
+                ),
+              )
+              .toList(),
+          antonyms: sense
+              .findElements('ant')
+              .map((e) => XRefParts.fromString(e.innerText))
+              .toList(),
+          seeAlso: sense
+              .findElements('xref')
+              .map((e) => XRefParts.fromString(e.innerText))
+              .toList(),
+        ),
+      );
+    }
+
+    entries.add(
+      Entry(
+        id: entryId,
+        kanji: kanjiEls,
+        readings: readingEls,
+        senses: senses,
+      ),
+    );
+  }
+
+  return entries;
+}
+
+Future<void> insertIntoDB(List<Entry> entries, Database db) async {
+  print('  [JMdict] Batch 1');
+  Batch b = db.batch();
+  for (final e in entries) {
+    b.insert(TableNames.entry, e.sqlValue);
+    for (final k in e.kanji) {
+      b.insert(TableNames.kanjiElement, k.sqlValue..addAll({'entryId': e.id}));
+      // b.insert(
+      //   TableNames.entryByKana,
+      //   {'entryId': e.id, 'kana': transliterateKatakanaToHiragana(k.reading)},
+      //   // Some entries have the same reading twice with difference in katakana and hiragana
+      //   conflictAlgorithm: ConflictAlgorithm.ignore,
+      // );
+      for (final i in k.info) {
+        b.insert(
+          TableNames.kanjiInfo,
+          {'entryId': e.id, 'reading': k.reading, 'info': i},
+        );
+      }
+    }
+    for (final r in e.readings) {
+      b.insert(
+        TableNames.readingElement,
+        r.sqlValue..addAll({'entryId': e.id}),
+      );
+
+      b.insert(
+        TableNames.entryByKana,
+        {'entryId': e.id, 'kana': transliterateKanaToLatin(r.reading)},
+        // Some entries have the same reading twice with difference in katakana and hiragana
+        conflictAlgorithm: ConflictAlgorithm.ignore,
+      );
+      for (final i in r.info) {
+        b.insert(
+          TableNames.readingInfo,
+          {'entryId': e.id, 'reading': r.reading, 'info': i},
+        );
+      }
+      for (final res in r.restrictions) {
+        b.insert(
+          TableNames.readingRestriction,
+          {'entryId': e.id, 'reading': r.reading, 'restriction': res},
+        );
+      }
+    }
+
+    for (final s in e.senses) {
+      for (final g in s.glossary) {
+        if (g.language == "eng")
+          b.insert(
+            TableNames.entryByEnglish,
+            {'entryId': e.id, 'english': g.phrase},
+            // Some entries have the same reading twice with difference in katakana and hiragana
+            conflictAlgorithm: ConflictAlgorithm.ignore,
+          );
+      }
+    }
+  }
+
+  await b.commit();
+
+  print('  [JMdict] Building trees');
+  SplayTreeMap<String, Set<Entry>> entriesByKanji = SplayTreeMap();
+  for (final entry in entries) {
+    for (final kanji in entry.kanji) {
+      if (entriesByKanji.containsKey(kanji.reading)) {
+        entriesByKanji.update(kanji.reading, (list) => list..add(entry));
+      } else {
+        entriesByKanji.putIfAbsent(kanji.reading, () => {entry});
+      }
+    }
+  }
+  SplayTreeMap<String, Set<Entry>> entriesByReading = SplayTreeMap();
+  for (final entry in entries) {
+    for (final reading in entry.readings) {
+      if (entriesByReading.containsKey(reading.reading)) {
+        entriesByReading.update(reading.reading, (list) => list..add(entry));
+      } else {
+        entriesByReading.putIfAbsent(reading.reading, () => {entry});
+      }
+    }
+  }
+
+  print('  [JMdict] Batch 2');
+  b = db.batch();
+
+  for (final e in entries) {
+    for (final s in e.senses) {
+      b.insert(
+          TableNames.sense, s.sqlValue..addAll({'id': s.id, 'entryId': e.id}));
+
+      for (final d in s.dialects) {
+        b.insert(TableNames.senseDialect, {'senseId': s.id, 'dialect': d});
+      }
+      for (final f in s.fields) {
+        b.insert(TableNames.senseField, {'senseId': s.id, 'field': f});
+      }
+      for (final i in s.info) {
+        b.insert(TableNames.senseInfo, {'senseId': s.id, 'info': i});
+      }
+      for (final m in s.misc) {
+        b.insert(TableNames.senseMisc, {'senseId': s.id, 'misc': m});
+      }
+      for (final p in s.pos) {
+        b.insert(TableNames.sensePOS, {'senseId': s.id, 'pos': p});
+      }
+      for (final l in s.languageSource) {
+        b.insert(
+          TableNames.senseLanguageSource,
+          l.sqlValue..addAll({'senseId': s.id}),
+        );
+      }
+      for (final rk in s.restrictedToKanji) {
+        b.insert(
+          TableNames.senseRestrictedToKanji,
+          {'entryId': e.id, 'senseId': s.id, 'kanji': rk},
+        );
+      }
+      for (final rr in s.restrictedToReading) {
+        b.insert(
+          TableNames.senseRestrictedToReading,
+          {'entryId': e.id, 'senseId': s.id, 'reading': rr},
+        );
+      }
+      for (final ls in s.languageSource) {
+        b.insert(
+          TableNames.senseLanguageSource,
+          ls.sqlValue..addAll({'senseId': s.id}),
+        );
+      }
+      for (final g in s.glossary) {
+        if (g.language == 'eng')
+          b.insert(
+            TableNames.senseGlossary,
+            g.sqlValue..addAll({'senseId': s.id}),
+            // There are some duplicate glossary, especially in
+            // the other languages.
+            conflictAlgorithm: ConflictAlgorithm.ignore,
+          );
+      }
+
+      for (final xref in s.seeAlso) {
+        final Set<Entry> entries;
+        if (xref.kanjiRef != null && xref.readingRef != null) {
+          entries = entriesByKanji[xref.kanjiRef]!
+              .difference(entriesByReading[xref.readingRef]!);
+        } else if (xref.kanjiRef != null) {
+          entries = entriesByKanji[xref.kanjiRef]!;
+        } else {
+          entries = entriesByReading[xref.readingRef]!;
+        }
+        for (final ex in entries)
+          if (!(xref.senseNum != null && xref.senseNum! > ex.senses.length)) {
+            b.insert(
+              TableNames.senseSeeAlso,
+              {
+                'senseId': s.id,
+                'xrefEntryId': ex.id,
+                'seeAlsoKanji': xref.kanjiRef,
+                'seeAlsoReading': xref.readingRef,
+                'seeAlsoSense': xref.senseNum,
+              },
+            );
+          }
+      }
+      for (final ant in s.antonyms) {
+        final Set<Entry> entries;
+        if (ant.kanjiRef != null && ant.readingRef != null) {
+          entries = entriesByKanji[ant.kanjiRef]!
+              .difference(entriesByReading[ant.readingRef]!);
+        } else if (ant.kanjiRef != null) {
+          entries = entriesByKanji[ant.kanjiRef]!;
+        } else {
+          entries = entriesByReading[ant.readingRef]!;
+        }
+        for (final ex in entries) {
+          if (!(ant.senseNum != null && ant.senseNum! > ex.senses.length)) {
+            b.insert(TableNames.senseAntonyms, {
+              'senseId': s.id,
+              'xrefEntryId': ex.id,
+              'antonymKanji': ant.kanjiRef,
+              'antonymReading': ant.readingRef,
+              'antonymSense': ant.senseNum,
+            });
+          }
+        }
+      }
+    }
+  }
+
+  await b.commit();
+}
+
+Future<void> addDataFromJMdict(Database db) async {
+  print('[JMdict] Reading file...');
+  String rawXML = File('data/JMdict.xml').readAsStringSync();
+
+  print('[JMdict] Parsing XML...');
+  XmlElement root = XmlDocument.parse(rawXML).getElement('JMdict')!;
+
+  print('[JMdict] Transforming data...');
+  final entries = transformXML(root);
+
+  print('[JMdict] Writing to database...');
+  await insertIntoDB(entries, db);
+}