search/word_search: score matching kana higher than converted kana

.gitea/workflows/build-and-test.yml

@@ -16,19 +16,14 @@ jobs:
       uses: https://github.com/cachix/install-nix-action@v31
       with:
         extra_nix_config: |
-          experimental-features = nix-command flakes
           show-trace = true
           max-jobs = auto
           trusted-users = root
           experimental-features = nix-command flakes
           build-users-group =
 
-    - name: Update database inputs
-      run: |
-        nix flake update jmdict-src
-        nix flake update jmdict-with-examples-src
-        nix flake update radkfile-src
-        nix flake update kanjidic2-src
+    # - name: Update database inputs
+    #   run: nix flake update datasources
 
     - name: Build database
       run: nix build .#database -L
@@ -44,7 +39,7 @@ jobs:
         compression: 0
 
     - name: Print database statistics
-      run: nix develop .# --command sqlite3_analyzer result/jadb.sqlite
+      run: nix develop .#sqlite-debugging --command sqlite3_analyzer result/jadb.sqlite
 
     # TODO: Defer failure of tests until after the coverage report is generated and uploaded.
     - name: Run tests

benchmark/benchmark.dart

@@ -1,5 +1,7 @@
-import './search/word_search.dart';
+import './search/english_word_search.dart';
+import './search/japanese_word_search.dart';
 
 Future<void> main() async {
-  await WordSearchBenchmark.main();
+  await EnglishWordSearchBenchmark.main();
+  await JapaneseWordSearchBenchmark.main();
 }

benchmark/search/english_word_search.dart

@@ -4,23 +4,23 @@ import 'package:sqflite_common/sqlite_api.dart';
 
 import '../../test/search/setup_database_connection.dart';
 
-class WordSearchBenchmark extends AsyncBenchmarkBase {
+class EnglishWordSearchBenchmark extends AsyncBenchmarkBase {
   Database? connection;
 
   static final List<String> searchTerms = [
     'kana',
     'kanji',
-    'kawaii',
+    'cute',
     'sushi',
     'ramen',
   ];
 
-  WordSearchBenchmark() : super('WordSearchBenchmark');
+  EnglishWordSearchBenchmark() : super('EnglishWordSearchBenchmark');
 
   static Future<void> main() async {
-    print('Running WordSearchBenchmark...');
-    await WordSearchBenchmark().report();
-    print('Finished WordSearchBenchmark');
+    print('Running EnglishWordSearchBenchmark...');
+    await EnglishWordSearchBenchmark().report();
+    print('Finished EnglishWordSearchBenchmark');
   }
 
   @override
@@ -31,7 +31,11 @@ class WordSearchBenchmark extends AsyncBenchmarkBase {
   @override
   Future<void> run() async {
     for (final term in searchTerms) {
-      await connection!.jadbSearchWord(term);
+      final result = await connection!.jadbSearchWord(term);
+      assert(
+        result?.isNotEmpty ?? false,
+        'Expected search results for term "$term"',
+      );
     }
   }
 

benchmark/search/japanese_word_search.dart

@@ -0,0 +1,49 @@
+import 'package:benchmark_harness/benchmark_harness.dart';
+import 'package:jadb/search.dart';
+import 'package:sqflite_common/sqlite_api.dart';
+
+import '../../test/search/setup_database_connection.dart';
+
+class JapaneseWordSearchBenchmark extends AsyncBenchmarkBase {
+  Database? connection;
+
+  static final List<String> searchTerms = [
+    '仮名',
+    '漢字',
+    'かわいい',
+    'すし',
+    'ラメン',
+  ];
+
+  JapaneseWordSearchBenchmark() : super('JapaneseWordSearchBenchmark');
+
+  static Future<void> main() async {
+    print('Running JapaneseWordSearchBenchmark...');
+    await JapaneseWordSearchBenchmark().report();
+    print('Finished JapaneseWordSearchBenchmark');
+  }
+
+  @override
+  Future<void> setup() async {
+    connection = await setupDatabaseConnection();
+  }
+
+  @override
+  Future<void> run() async {
+    for (final term in searchTerms) {
+      final result = await connection!.jadbSearchWord(term);
+      assert(
+        result?.isNotEmpty ?? false,
+        'Expected search results for term "$term"',
+      );
+    }
+  }
+
+  @override
+  Future<void> teardown() async {
+    await connection?.close();
+  }
+
+  // @override
+  // Future<void> exercise() => run();
+}

data/tanos-jlpt/n4.csv

@@ -1,582 +0,0 @@
-,あ,Ah
-,ああ,like that
-間,あいだ,a space
-合う,あう,to match
-,あかちゃん,baby
-上る,あがる,to rise
-赤ん坊,あかんぼう,baby
-空く,あく,"to open, to become empty"
-,あげる,to give
-浅い,あさい,"shallow, superficial"
-味,あじ,flavour
-明日,あす・あした,tomorrow
-遊び,あそび,play
-集る,あつまる,to gather
-集める,あつめる,to collect something
-謝る,あやまる,to apologize
-安心,あんしん,relief
-安全,あんぜん,safety
-,あんな,such
-以下,いか,less than
-以外,いがい,with the exception of
-医学,いがく,medical science
-生きる,いきる,to live
-意見,いけん,opinion
-石,いし,stone
-,いじめる,to tease
-以上,いじょう,"more than, this is all"
-急ぐ,いそぐ,to hurry
-致す,いたす,(humble) to do
-一度,いちど,once
-一生懸命,いっしょうけんめい,with utmost effort
-,いっぱい,full
-糸,いと,thread
-以内,いない,within
-田舎,いなか,countryside
-祈る,いのる,to pray
-,いらっしゃる,"(respectful) to be, to come or to go"
-植える,うえる,"to plant, to grow"
-受付,うけつけ,receipt
-受ける,うける,to take a lesson or test
-動く,うごく,to move
-,うち,within
-打つ,うつ,to hit
-美しい,うつくしい,beautiful
-写す,うつす,to copy or photograph
-移る,うつる,to move house or transfer
-腕,うで,arm
-裏,うら,reverse side
-売り場,うりば,place where things are sold
-,うん,(informal) yes
-運転手,うんてんしゅ,driver
-枝,えだ,"branch, twig"
-選ぶ,えらぶ,to choose
-遠慮,えんりょ・する,"to be reserved, to be restrained"
-,おいでになる,(respectful) to be
-お祝い,おいわい,congratulation
-,おかげ,"owing to, thanks to"
-,おかしい,strange or funny
-億,おく,one hundred million
-屋上,おくじょう,rooftop
-贈り物,おくりもの,gift
-送る,おくる,to send
-遅れる,おくれる,to be late
-起す,おこす,to wake
-行う,おこなう,to do
-怒る,おこる,"to get angry, to be angry"
-押し入れ,おしいれ,closet
-お嬢さん,おじょうさん,young lady
-お宅,おたく,(polite) your house
-落る,おちる,to fall or drop
-,おっしゃる,(respectful) to say
-夫,おっと,husband
-,おつり,"change from purchase, balance"
-音,おと,"sound, note"
-落す,おとす,to drop
-踊り,おどり,a dance
-踊る,おどる,to dance
-驚く,おどろく,to be surprised
-お祭り,おまつり,festival
-お見舞い,おみまい,"calling on someone who is ill, enquiry"
-お土産,おみやげ,souvenir
-思い出す,おもいだす,to remember
-思う,おもう,"to think, to feel"
-,おもちゃ,toy
-表,おもて,the front
-親,おや,parents
-泳ぎ方,およぎかた,way of swimming
-下りる,おりる,"to get off, to descend"
-折る,おる,to break or to fold
-お礼,おれい,expression of gratitude
-折れる,おれる,to break or be folded
-終わり,おわり,the end
-海岸,かいがん,coast
-会議,かいぎ,meeting
-会議室,かいぎしつ,meeting room
-会場,かいじょう,assembly hall or meeting place
-会話,かいわ,conversation
-帰り,かえり,return
-変える,かえる,to change
-科学,かがく,science
-鏡,かがみ,mirror
-掛ける,かける,to hang something
-飾る,かざる,to decorate
-火事,かじ,fire
-,ガス,gas
-堅/硬/固い,かたい,hard
-形,かたち,shape
-片付ける,かたづける,to tidy up
-課長,かちょう,section manager
-勝つ,かつ,to win
-家内,かない,housewife
-悲しい,かなしい,sad
-必ず,かならず,"certainly,necessarily"
-お・金持ち,かねもち/おかねもち,rich man
-彼女,かのじょ,"she,girlfriend"
-壁,かべ,wall
-髪,かみ,hair
-噛む,かむ,"to bite,to chew"
-通う,かよう,to commute
-彼,かれ,"he,boyfriend"
-乾く,かわく,to get dry
-代わり,かわり,"substitute,alternate"
-変わる,かわる,to change
-考える,かんがえる,to consider
-関係,かんけい,relationship
-看護師,かんごし, nurse
-簡単,かんたん,simple
-気,き,"spirit,mood"
-機会,きかい,opportunity
-危険,きけん,danger
-聞こえる,きこえる,to be heard
-汽車,きしゃ,steam train
-技術,ぎじゅつ,"art,technology,skill"
-季節,きせつ,season
-規則,きそく,regulations
-,きっと,surely
-絹,きぬ,silk
-厳しい,きびしい,strict
-気分,きぶん,mood
-決る,きまる,to be decided
-君,きみ,(informal) You
-決める,きめる,to decide
-気持ち,きもち,"feeling,mood"
-着物,きもの,kimono
-客,きゃく,"guest,customer"
-急,きゅう,"urgent, steep"
-急行,きゅうこう,"speedy, express"
-教育,きょういく,education
-教会,きょうかい,church
-競争,きょうそう,competition
-興味,きょうみ,an interest
-近所,きんじょ,neighbourhood
-具合,ぐあい,"condition,health"
-空気,くうき,"air,atmosphere"
-空港,くうこう,airport
-草,くさ,grass
-首,くび,neck
-雲,くも,cloud
-比べる,くらべる,to compare
-,くれる,to give
-暮れる,くれる,"to get dark,to come to an end"
-君,くん,suffix for familiar young male
-毛,け,hair or fur
-経済,けいざい,"finance,economy"
-警察,けいさつ,police
-景色,けしき,"scene,landscape"
-消しゴム,けしゴム,eraser
-下宿,げしゅく,lodging
-決して,けっして,never
-,けれど/けれども,however
-原因,げんいん,"cause,source"
-,けんか・する,to quarrel
-研究,けんきゅう,research
-研究室,けんきゅうしつ,"study room,laboratory"
-見物,けんぶつ,sightseeing
-子,こ,child
-,こう,this way
-郊外,こうがい,outskirts
-講義,こうぎ,lecture
-工業,こうぎょう,the manufacturing industry
-高校,こうこう,high school
-高校生,こうこうせい,high school student
-工場,こうじょう/こうば,"factory,plant,mill,workshop"
-校長,こうちょう,headmaster
-交通,こうつう,"traffic,transportation"
-講堂,こうどう,auditorium
-高等学校,こうとうがっこう,high school
-公務員,こうむいん,"civil servant, government worker"
-国際,こくさい,international
-心,こころ,"heart, mind, core"
-御主人,ごしゅじん,(honorable) your husband
-故障,こしょう・する,to break-down
-ご存じ,ごぞんじ,(respect form ) to know
-答,こたえ,response
-,ごちそう,a feast
-小鳥,ことり,small bird
-,このあいだ,"the other day,recently"
-,このごろ,"these days,nowadays"
-細かい,こまかい,"small, fine"
-込む,こむ,to include
-米,こめ,uncooked rice
-,ごらんになる,(respectful) to see
-,これから,after this
-怖い,こわい,frightening
-壊す,こわす,to break
-壊れる,こわれる,to be broken
-今度,こんど,"now,next time"
-今夜,こんや,tonight
-最近,さいきん,"latest,nowadays"
-最後,さいご,"last,end"
-最初,さいしょ,"beginning,first"
-坂,さか,"slope,hill"
-探す,さがす,to look for
-下る,さがる,"to get down,to descend"
-盛ん,さかん,"popularity,prosperous"
-下げる,さげる,"to hang,to lower,to move back"
-差し上げる,さしあげる,(polite) to give
-,さっき,some time ago
-寂しい,さびしい,lonely
-さ来月,さらいげつ,the month after next
-さ来週,さらいしゅう,the week after next
-騒ぐ,さわぐ,"to make noise,to be excited"
-触る,さわる,to touch
-産業,さんぎょう,industry
-残念,ざんねん,disappointment
-市,し,city
-字,じ,character
-試合,しあい,"match,game"
-仕方,しかた,method
-試験,しけん,examination
-事故,じこ,accident
-地震,じしん,earthquake
-時代,じだい,era
-下着,したぎ,underwear
-,しっかり,"firmly,steadily"
-失敗,しっぱい,"failure,mistake"
-辞典,じてん,dictionary
-品物,しなもの,goods
-,しばらく,little while
-島,しま,island
-市民,しみん,citizen
-事務所,じむしょ,office
-社会,しゃかい,"society,public"
-社長,しゃちょう,company president
-自由,じゆう,freedom
-習慣,しゅうかん,"custom,manners"
-住所,じゅうしょ,"an address,a residence"
-柔道,じゅうどう,judo
-十分,じゅうぶん,enough
-趣味,しゅみ,hobby
-紹介,しょうかい,introduction
-小学校,しょうがっこう,elementary school
-小説,しょうせつ,novel
-将来,しょうらい,"future,prospects"
-食料品,しょくりょうひん,groceries
-女性,じょせい,woman
-知らせる,しらせる,to notify
-調べる,しらべる,to investigate
-人口,じんこう,population
-神社,じんじゃ,Shinto shrine
-親切,しんせつ,kindness
-新聞社,しんぶんしゃ,newspaper company
-水泳,すいえい,swimming
-水道,すいどう,water supply
-数学,すうがく,"mathematics,arithmetic"
-過ぎる,すぎる,to exceed
-凄い,すごい,terrific
-進む,すすむ,to make progress
-,すっかり,completely
-,すっと,"straight,all of a sudden"
-捨てる,すてる,to throw away
-砂,すな,sand
-滑る,すべる,"to slide,to slip"
-隅,すみ,"corner,nook"
-済む,すむ,to finish
-,すり,pickpocket
-,すると,then
-生活,せいかつ・する,to live
-生産,せいさん・する,to produce
-政治,せいじ,"politics,government"
-西洋,せいよう,western countries
-世界,せかい,the world
-席,せき,seat
-説明,せつめい,explanation
-背中,せなか,back of the body
-線,せん,line
-戦争,せんそう,war
-先輩,せんぱい,senior
-,そう,really
-育てる,そだてる,"to rear,to bring up"
-卒業,そつぎょう,graduation
-祖父,そふ,grandfather
-祖母,そぼ,grandmother
-,それで,because of that
-,それに,moreover
-,それほど,to that extent
-,そろそろ,"gradually,soon"
-,そんな,that sort of
-,そんなに,"so much,like that"
-退院,たいいん・する,to leave hospital
-大学生,だいがくせい,university student
-大事,だいじ,"important,valuable,serious matter"
-大体,だいたい,generally
-,たいてい,usually
-大分,だいぶ,greatly
-台風,たいふう,typhoon
-倒れる,たおれる,to break down
-,だから,"so,therefore"
-確か,たしか,definite
-足す,たす,to add a number
-訪ねる,たずねる,to visit
-尋ねる,たずねる,to ask
-正しい,ただしい,correct
-畳,たたみ,Japanese straw mat
-立てる,たてる,to stand something up
-建てる,たてる,to build
-例えば,たとえば,for example
-棚,たな,shelves
-楽しみ,たのしみ,joy
-楽む,たのしむ,to enjoy oneself
-,たまに,occasionally
-為,ため,in order to
-足りる,たりる,to be enough
-男性,だんせい,male
-暖房,だんぼう,heating
-血,ち,blood
-,チェック・する,to check
-力,ちから,"strength,power"
-,ちっとも,not at all (used with a negative verb)
-,ちゃん,suffix for familiar  person
-注意,ちゅうい,caution
-中学校,ちゅうがっこう,"junior high school,middle school"
-注射,ちゅうしゃ,injection
-駐車場,ちゅうしゃじょう,parking lot
-地理,ちり,geography
-捕まえる,つかまえる,to seize
-付く,つく,to be attached
-漬ける,つける,"to soak,to pickle"
-都合,つごう,"circumstances,convenience"
-伝える,つたえる,to report
-続く,つづく,to be continued
-続ける,つづける,to continue
-包む,つつむ,to wrap
-妻,つま,my wife
-,つもり,intention
-釣る,つる,to fish
-丁寧,ていねい,polite
-適当,てきとう,suitability
-手伝う,てつだう,to assist
-手袋,てぶくろ,glove
-寺,てら,temple
-点,てん,"point,dot"
-店員,てんいん,shop assistant
-天気予報,てんきよほう,weather forecast
-電灯,でんとう,electric light
-電報,でんぽう,telegram
-展覧会,てんらんかい,exhibition
-都,と,metropolitan
-道具,どうぐ,"tool,means"
-,とうとう,"finally, after all"
-動物園,どうぶつえん,zoo
-遠く,とおく,distant
-通る,とおる,to go through
-特に,とくに,"particularly,especially"
-特別,とくべつ,special
-,とこや,barber
-途中,とちゅう,on the way
-特急,とっきゅう,limited express train (faster than an express train)
-届ける,とどける,"to send, to deliver, to report"
-泊まる,とまる,to lodge at
-止める,とめる,to stop something
-取り替える,とりかえる,to exchange
-泥棒,どろぼう,thief
-,どんどん,more and more
-直す,なおす,"to fix,to repair"
-直る,なおる,"to be fixed,to be repaired"
-治る,なおる,"to be cured,to heal"
-泣く,なく,to weep
-無くなる,なくなる,"to disappear,to get lost"
-亡くなる,なくなる,to die
-投げる,なげる,to throw or cast away
-,なさる,(respectful) to do
-鳴る,なる,to sound
-,なるべく,as much as possible
-,なるほど,now I understand
-慣れる,なれる,to grow accustomed to
-苦い,にがい,bitter
-二階建て,にかいだて,two storied
-逃げる,にげる,to escape
-日記,にっき,journal
-入院,にゅういん・する,"to hospitalise, hospitalisation"
-入学,にゅうがく・する,to enter school or university
-似る,にる,to be similar
-人形,にんぎょう,"doll, figure"
-盗む,ぬすむ,to steal
-塗る,ぬる,"to paint, to colour, to plaster"
-,ぬれる,to get wet
-,ねだん,price
-熱,ねつ,fever
-寝坊,ねぼう,sleeping in late
-眠い,ねむい,sleepy
-眠る,ねむる,to sleep
-残る,のこる,to remain
-乗り換える,のりかえる,to change between buses or trains
-乗り物,のりもの,vehicle
-葉,は,leaf
-場合,ばあい,situation
-倍,ばい,double
-拝見,はいけん・する,(humble) to look at
-歯医者,はいしゃ,dentist
-運ぶ,はこぶ,to transport
-始める,はじめる,to begin
-場所,ばしょ,location
-,はず,it should be so
-恥ずかしい,はずかしい,embarrassed
-発音,はつおん,pronunciation
-,はっきり,clearly
-花見,はなみ,cherry-blossom viewing
-林,はやし,"woods,forester"
-払う,はらう,to pay
-番組,ばんぐみ,television or radio program
-反対,はんたい,opposition
-日,ひ,"day, sun"
-火,ひ,fire
-冷える,ひえる,to grow cold
-光,ひかり,light
-光る,ひかる,"to shine,to glitter"
-引き出し,ひきだし,"drawer,drawing out"
-,ひきだす,to withdraw
-,ひげ,beard
-飛行場,ひこうじょう,airport
-久しぶり,ひさしぶり,after a long time
-美術館,びじゅつかん,art gallery
-非常に,ひじょうに,extremely
-引っ越す,ひっこす,to move house
-必要,ひつよう,necessary
-,ひどい,awful
-開く,ひらく,to open an event
-昼間,ひるま,"daytime,during the day"
-昼休み,ひるやすみ,noon break
-拾う,ひろう,"to pick up,to gather"
-増える,ふえる,to increase
-深い,ふかい,deep
-複雑,ふくざつ,"complexity,complication"
-復習,ふくしゅう,revision
-部長,ぶちょう,head of a section
-普通,ふつう,"usually, or a train that stops at every station"
-,ぶどう,grapes
-太る,ふとる,to become fat
-布団,ふとん,"Japanese bedding, futon"
-舟,ふね,ship
-不便,ふべん,inconvenience
-踏む,ふむ,to step on
-降り出す,ふりだす,to start to rain
-文化,ぶんか,culture
-文学,ぶんがく,literature
-文法,ぶんぽう,grammar
-別,べつ,different
-変,へん,strange
-返事,へんじ,reply
-貿易,ぼうえき,trade
-法律,ほうりつ,law
-僕,ぼく,I (used by males)
-星,ほし,star
-,ほとんど,mostly
-,ほめる,to praise
-翻訳,ほんやく,translation
-参る,まいる,"(humble) to go,to come"
-負ける,まける,to lose
-,または,"or,otherwise"
-間違える,まちがえる,to make a mistake
-間に合う,まにあう,to be in time for
-周り,まわり,surroundings
-回る,まわる,to go around
-漫画,まんが,comic
-真中,まんなか,middle
-見える,みえる,to be in sight
-湖,みずうみ,lake
-味噌,みそ,"miso, soybean paste"
-見つかる,みつかる,to be discovered
-見つける,みつける,to discover
-皆,みな,everybody
-港,みなと,harbour
-向かう,むかう,to face
-迎える,むかえる,to go out to meet
-昔,むかし,"old times, old days, long ago, formerly"
-虫,むし,insect
-息子,むすこ,(humble) son
-娘,むすめ,(humble) daughter
-無理,むり,impossible
-召し上がる,めしあがる,(polite) to eat
-珍しい,めずらしい,rare
-申し上げる,もうしあげる,"(humble) to say,to tell"
-申す,もうす,"(humble) to be called,to say"
-,もうすぐ,soon
-,もし,if
-戻る,もどる,to turn back
-木綿,もめん,cotton
-森,もり,forest
-焼く,やく,"to bake,to grill"
-約束,やくそく,promise
-役に立つ,やくにたつ,to be helpful
-焼ける,やける,"to burn,to be roasted"
-優しい,やさしい,kind
-痩せる,やせる,to become thin
-,やっと,at last
-止む,やむ,to stop
-止める,やめる,to stop
-柔らかい,やわらかい,soft
-湯,ゆ,hot water
-指,ゆび,finger
-指輪,ゆびわ,a ring
-夢,ゆめ,dream
-揺れる,ゆれる,"to shake,to sway"
-用,よう,use
-用意,ようい,preparation
-用事,ようじ,things to do
-汚れる,よごれる,to get dirty
-予習,よしゅう,preparation for a lesson
-予定,よてい,arrangement
-予約,よやく,reservation
-寄る,よる,to visit
-喜ぶ,よろこぶ,to be delighted
-理由,りゆう,reason
-利用,りよう,utilization
-両方,りょうほう,both sides
-旅館,りょかん,Japanese hotel
-留守,るす,absence
-冷房,れいぼう,air conditioning
-歴史,れきし,history
-連絡,れんらく,contact
-沸かす,わかす,"to boil,to heat"
-別れる,わかれる,to separate
-沸く,わく,"to boil, to grow hot,to get excited"
-訳,わけ,"meaning,reason"
-忘れ物,わすれもの,lost article
-笑う,わらう,"to laugh,to smile"
-割合,わりあい,"rate,ratio,percentage"
-割れる,われる,to break
-,アクセサリー,accessory
-,アジア,Asia
-,アナウンサー,announcer
-,アフリカ,Africa
-,アメリカ,America
-,アルコール,alcohol
-,アルバイト,part-time job
-,エスカレーター,escalator
-,オートバイ,motorcycle
-,カーテン,curtain
-,ガス,gas
-,ガソリン,petrol
-,ガソリンスタンド,petrol station
-,ガラス,a glass pane
-,ケーキ,cake
-消しゴム,けしゴム,"eraser, rubber"
-,コンサート,concert
-,コンピューター,computer
-,サラダ,salad
-,サンダル,sandal
-,サンドイッチ,sandwich
-,ジャム,jam
-,スーツ,suit
-,スーツケース,suitcase
-,スクリーン,screen
-,ステーキ,steak
-,ステレオ,stereo
-,ソフト,soft
-,タイプ,"type,style"
-,チェック・する,to check
-,テキスト,"text,text book"
-,テニス,tennis
-,パート,part time
-,パソコン,personal computer
-,ハンドバッグ,handbag　
-,ピアノ,piano
-,ビル,building or bill
-,ファックス,fax
-,プレゼント,present
-,ベル,bell
-,レジ,register
-,レポート/リポート,report
-,ワープロ,word processor

data/tanos-jlpt/n5.csv

@@ -1,669 +0,0 @@
-会う,あう,to meet
-青,あお,blue
-青い,あおい,blue
-赤,あか,red
-赤い,あかい,red
-明い,あかるい,bright
-秋,あき,autumn
-開く,あく,"to open,to become open"
-開ける,あける,to open
-上げる,あげる,to give
-朝,あさ,morning
-朝御飯,あさごはん,breakfast
-,あさって,day after tomorrow
-足,あし,"foot,leg"
-明日,あした,tomorrow
-,あそこ,over there
-遊ぶ,あそぶ,"to play,to make a visit"
-暖かい,あたたかい,warm
-頭,あたま,head
-新しい,あたらしい,new
-,あちら,there
-暑い,あつい,hot
-熱い,あつい,hot to the touch
-厚い,あつい,"kind, deep, thick"
-,あっち,over there
-後,あと,afterwards
-,あなた,you
-兄,あに,(humble) older brother
-姉,あね,(humble) older sister
-,あの,that over there
-,あの,um...
-,アパート,apartment
-,あびる,"to bathe,to shower"
-危ない,あぶない,dangerous
-甘い,あまい,sweet
-,あまり,not very
-雨,あめ,rain
-飴,あめ,candy
-洗う,あらう,to wash
-,ある,"to be,to have (used for inanimate objects)"
-歩く,あるく,to walk
-,あれ,that
-,いい/よい,good
-,いいえ,no
-言う,いう,to say
-家,いえ,house
-,いかが,how
-行く,いく,to go
-,いくつ,"how many?,how old?"
-,いくら,how much?
-池,いけ,pond
-医者,いしゃ,medical doctor
-,いす,chair
-忙しい,いそがしい,"busy,irritated"
-痛い,いたい,painful
-一,いち,one
-一日,いちにち,"(1) one day, (2) first of month"
-,いちばん,"best,first"
-,いつ,when
-五日,いつか,"five days, fifth day"
-一緒,いっしょ,together
-五つ,いつつ,five
-,いつも,always
-犬,いぬ,dog
-今,いま,now
-意味,いみ,meaning
-妹,いもうと,(humble) younger sister
-嫌,いや,unpleasant
-入口,いりぐち,entrance
-居る,いる,"to be, to have (used for people and animals)"
-要る,いる,to need
-入れる,いれる,to put in
-色,いろ,colour
-,いろいろ,various
-上,うえ,on top of
-後ろ,うしろ,behind
-薄い,うすい,"thin,weak"
-歌,うた,song
-歌う,うたう,to sing
-生まれる,うまれる,to be born
-海,うみ,sea
-売る,うる,to sell
-煩い,うるさい,"noisy,annoying"
-上着,うわぎ,jacket
-絵,え,picture
-映画,えいが,movie
-映画館,えいがかん,cinema
-英語,えいご,English language
-,ええ,yes
-駅,えき,station
-,エレベーター,elevator
-鉛筆,えんぴつ,pencil
-,おいしい,delicious
-多い,おおい,many
-大きい,おおきい,big
-大きな,おおきな,big
-大勢,おおぜい,great number of people
-お母さん,おかあさん,(honorable) mother
-お菓子,おかし,"sweets, candy"
-お金,おかね,money
-起きる,おきる,to get up
-置く,おく,to put
-奥さん,おくさん,(honorable) wife
-お酒,おさけ,"alcohol, rice wine"
-お皿,おさら,"plate, dish"
-伯父/叔父,おじいさん,"grandfather,male senior citizen"
-教える,おしえる,"to teach,to tell"
-伯父/叔父,おじさん,"uncle,middle aged gentleman"
-押す,おす,"to push, to stamp something"
-遅い,おそい,"late,slow"
-お茶,おちゃ,green tea
-お手洗い,おてあらい,bathroom
-お父さん,おとうさん,(honorable) father
-弟,おとうと,younger brother
-男,おとこ,man
-男の子,おとこのこ,boy
-一昨日,おととい,day before yesterday
-一昨年,おととし,year before last
-大人,おとな,adult
-,おなか,stomach
-同じ,おなじ,same
-お兄さん,おにいさん,(honorable) older brother
-お姉さん,おねえさん,(honorable) older sister
-,おばあさん,"grandmother,female senior-citizen"
-伯母さん/叔母さん,おばさん,aunt
-お風呂,おふろ,bath
-お弁当,おべんとう,boxed lunch
-覚える,おぼえる,to remember
-,おまわりさん,friendly term for policeman
-重い,おもい,heavy
-,おもしろい,interesting
-泳ぐ,およぐ,to swim
-降りる,おりる,"to get off, to descend"
-終る,おわる,to finish
-音楽,おんがく,music
-女,おんな,woman
-女の子,おんなのこ,girl
-外国,がいこく,foreign country
-外国人,がいこくじん,foreigner
-会社,かいしゃ,company
-階段,かいだん,stairs
-買い物,かいもの,shopping
-買う,かう,to buy
-返す,かえす,to return something
-帰る,かえる,to go back
-,かかる,to take time or money
-,かぎ,key
-書く,かく,to write
-学生,がくせい,student
-,かける,to call by phone
-傘,かさ,umbrella
-貸す,かす,to lend
-風,かぜ,wind
-風邪,かぜ,a cold
-家族,かぞく,family
-方,かた,"person, way of doing"
-学校,がっこう,school
-,カップ,cup
-家庭,かてい,household
-角,かど,a corner
-,かばん,"bag,basket"
-花瓶,かびん,a vase
-紙,かみ,paper
-,カメラ,camera
-火曜日,かようび,Tuesday
-辛い,からい,spicy
-体,からだ,body
-借りる,かりる,to borrow
-軽い,かるい,light
-,カレー,curry
-,カレンダー,calendar
-川/河,かわ,river
-,かわいい,cute
-漢字,かんじ,Chinese character
-木,き,"tree,wood"
-黄色,きいろ,yellow
-黄色い,きいろい,yellow
-消える,きえる,to disappear
-聞く,きく,"to hear,to listen to,to ask"
-北,きた,north
-,ギター,guitar
-汚い,きたない,dirty
-喫茶店,きっさてん,coffee lounge
-切手,きって,postage stamp
-切符,きっぷ,ticket
-昨日,きのう,yesterday
-九,きゅう / く,nine
-牛肉,ぎゅうにく,beef
-牛乳,ぎゅうにゅう,milk
-今日,きょう,today
-教室,きょうしつ,classroom
-兄弟,きょうだい,(humble) siblings
-去年,きょねん,last year
-嫌い,きらい,hate
-切る,きる,to cut
-着る,きる,to put on from the shoulders down
-,きれい,"pretty,clean"
-,キロ/キログラム,kilogram
-,キロ/キロメートル,kilometre
-銀行,ぎんこう,bank
-金曜日,きんようび,Friday
-薬,くすり,medicine
-,ください,please
-果物,くだもの,fruit
-口,くち,"mouth,opening"
-靴,くつ,shoes
-靴下,くつした,socks
-国,くに,country
-曇り,くもり,cloudy weather
-曇る,くもる,"to become cloudy,to become dim"
-暗い,くらい,gloomy
-,クラス,class
-,グラム,gram
-来る,くる,to come
-車,くるま,"car,vehicle"
-黒,くろ,black
-黒い,くろい,black
-警官,けいかん,policeman
-今朝,けさ,this morning
-消す,けす,"to erase,to turn off power"
-結構,けっこう,"splendid,enough"
-結婚,けっこん,marriage
-月曜日,げつようび,Monday
-玄関,げんかん,entry hall
-元気,げんき,"health, vitality"
-五,ご,five
-公園,こうえん,park
-交差点,こうさてん,intersection
-紅茶,こうちゃ,black tea
-交番,こうばん,police box
-声,こえ,voice
-,コート,"coat,tennis court"
-,コーヒー,coffee
-,ここ,here
-午後,ごご,afternoon
-九日,ここのか,"nine days, ninth day"
-九つ,ここのつ,nine
-午前,ごぜん,morning
-答える,こたえる,to answer
-,こちら,this person or way
-,こっち,this person or way
-,コップ,a glass
-今年,ことし,this year
-言葉,ことば,"word,language"
-子供,こども,child
-,この,this
-御飯,ごはん,"cooked rice,meal"
-,コピーする,to copy
-困る,こまる,to be worried
-,これ,this
-今月,こんげつ,this month
-今週,こんしゅう,this week
-,こんな,such
-今晩,こんばん,this evening
-,さあ,well…
-財布,さいふ,wallet
-魚,さかな,fish
-先,さき,"the future,previous"
-咲く,さく,to bloom
-作文,さくぶん,"composition,writing"
-差す,さす,"to stretch out hands,to raise an umbrella"
-雑誌,ざっし,magazine
-砂糖,さとう,sugar
-寒い,さむい,cold
-さ来年,さらいねん,year after next
-三,さん,three
-散歩,さんぽする,to stroll
-四,し / よん,four
-塩,しお,salt
-,しかし,however
-時間,じかん,time
-仕事,しごと,job
-辞書,じしょ,dictionary
-静か,しずか,quiet
-下,した,below
-七,しち / なな,seven
-質問,しつもん,question
-自転車,じてんしゃ,bicycle
-自動車,じどうしゃ,automobile
-死ぬ,しぬ,to die
-字引,じびき,dictionary
-自分,じぶん,oneself
-閉まる,しまる,"to close,to be closed"
-閉める,しめる,to close something
-締める,しめる,to tie
-,じゃ/じゃあ,well then…
-写真,しゃしん,photograph
-,シャツ,shirt
-,シャワー,shower
-十,じゅう  とお,ten
-授業,じゅぎょう,"lesson,class work"
-宿題,しゅくだい,homework
-上手,じょうず,skillful
-丈夫,じょうぶ,"strong,durable"
-,しょうゆ,soy sauce
-食堂,しょくどう,dining hall
-知る,しる,to know
-白,しろ,white
-白い,しろい,white
-新聞,しんぶん,newspaper
-水曜日,すいようび,Wednesday
-吸う,すう,"to smoke,to suck"
-,スカート,skirt
-好き,すき,likeable
-少ない,すくない,a few
-,すぐに,instantly
-少し,すこし,few
-涼しい,すずしい,refreshing
-,ストーブ,heater
-,スプーン,spoon
-,スポーツ,sport
-,ズボン,trousers
-住む,すむ,to live in
-,スリッパ,slippers
-,する,to do
-座る,すわる,to sit
-背,せ,"height,stature"
-生徒,せいと,pupil
-,セーター,"sweater,jumper"
-,せっけん,soap
-背広,せびろ,business suit
-狭い,せまい,narrow
-,ゼロ,zero
-千,せん,thousand
-先月,せんげつ,last month
-先週,せんしゅう,last week
-先生,せんせい,"teacher,doctor"
-洗濯,せんたく,washing
-全部,ぜんぶ,all
-掃除,そうじする,"to clean, to sweep"
-,そうして/そして,and
-,そこ,that place
-,そちら,over there
-,そっち,over there
-外,そと,outside
-,その,that
-,そば,"near,beside"
-空,そら,sky
-,それ,that
-,それから,after that
-,それでは,in that situation
-大学,だいがく,university
-大使館,たいしかん,embassy
-大丈夫,だいじょうぶ,all right
-大好き,だいすき,to be very likeable
-大切,たいせつ,important
-台所,だいどころ,kitchen
-,たいへん,very
-,たいへん,difficult situation
-高い,たかい,"tall, expensive"
-,たくさん,many
-,タクシー,taxi
-出す,だす,to put out
-立つ,たつ,to stand
-,たて,"length,height"
-建物,たてもの,building
-楽しい,たのしい,enjoyable
-頼む,たのむ,to ask
-,たばこ,"tobacco,cigarettes"
-,たぶん,probably
-食べ物,たべもの,food
-食べる,たべる,to eat
-卵,たまご,egg
-誰,だれ,who
-誰,だれか,somebody
-誕生日,たんじょうび,birthday
-,だんだん,gradually
-小さい,ちいさい,little
-小さな,ちいさな,little
-近い,ちかい,near
-違う,ちがう,to differ
-近く,ちかく,near
-地下鉄,ちかてつ,underground train
-地図,ちず,map
-茶色,ちゃいろ,brown
-,ちゃわん,rice bowl
-,ちょうど,exactly
-,ちょっと,somewhat
-一日,ついたち,first of month
-使う,つかう,to use
-疲れる,つかれる,to get tired
-次,つぎ,next
-着く,つく,to arrive at
-机,つくえ,desk
-作る,つくる,to make
-,つける,to turn on
-勤める,つとめる,to work for someone
-,つまらない,boring
-冷たい,つめたい,cold to the touch
-強い,つよい,powerful
-手,て,hand
-,テープ,tape
-,テーブル,table
-,テープレコーダー,tape recorder
-出かける,でかける,to go out
-手紙,てがみ,letter
-,できる,to be able to
-出口,でぐち,exit
-,テスト,test
-,では,with that...
-,デパート,department store
-,でも,but
-出る,でる,"to appear,to leave"
-,テレビ,television
-天気,てんき,weather
-電気,でんき,"electricity,electric light"
-電車,でんしゃ,electric train
-電話,でんわ,telephone
-戸,と,Japanese style door
-,ドア,Western style door
-,トイレ,toilet
-,どう,"how,in what way"
-,どうして,for what reason
-,どうぞ,please
-動物,どうぶつ,animal
-,どうも,thanks
-遠い,とおい,far
-十日,とおか,"ten days,the tenth day"
-時々,ときどき,sometimes
-時計,とけい,"watch,clock"
-,どこ,where
-所,ところ,place
-年,とし,year
-図書館,としょかん,library
-,どちら,which of two
-,どっち,which
-,とても,very
-,どなた,who
-隣,となり,next door to
-,どの,which
-飛ぶ,とぶ,"to fly,to hop"
-止まる,とまる,to come to a halt
-友達,ともだち,friend
-土曜日,どようび,Saturday
-鳥,とり,bird
-とり肉,とりにく,chicken meat
-取る,とる,to take something
-撮る,とる,to take a photo or record a film
-,どれ,which (of three or more)
-,ナイフ,knife
-中,なか,middle
-長い,ながい,long
-鳴く,なく,"animal noise. to chirp, roar or croak etc."
-無くす,なくす,to lose something
-,なぜ,why
-夏,なつ,summer
-夏休み,なつやすみ,summer holiday
-,など,et cetera
-七つ,ななつ,seven
-七日,なのか,"seven days,the seventh day"
-名前,なまえ,name
-習う,ならう,to learn
-並ぶ,ならぶ,"to line up,to stand in a line"
-並べる,ならべる,"to line up,to set up"
-,なる,to become
-何,なん/なに,what
-二,に,two
-賑やか,にぎやか,"bustling,busy"
-肉,にく,meat
-西,にし,west
-日曜日,にちようび,Sunday
-荷物,にもつ,luggage
-,ニュース,news
-庭,にわ,garden
-脱ぐ,ぬぐ,to take off clothes
-温い,ぬるい,luke warm
-,ネクタイ,"tie,necktie"
-猫,ねこ,cat
-寝る,ねる,"to go to bed,to sleep"
-,ノート,"notebook,exercise book"
-登る,のぼる,to climb
-飲み物,のみもの,a drink
-飲む,のむ,to drink
-乗る,のる,"to get on,to ride"
-歯,は,tooth
-,パーティー,party
-,はい,yes
-灰皿,はいざら,ashtray
-入る,はいる,"to enter,to contain"
-葉書,はがき,postcard
-,はく,"to wear,to put on trousers"
-箱,はこ,box
-橋,はし,bridge
-,はし,chopsticks
-始まる,はじまる,to begin
-初め/始め,はじめ,beginning
-初めて,はじめて,for the first time
-走る,はしる,to run
-,バス,bus
-,バター,butter
-二十歳,はたち,"20 years old,20th year"
-働く,はたらく,to work
-八,はち,eight
-二十日,はつか,"twenty days,twentieth"
-花,はな,flower
-鼻,はな,nose
-話,はなし,"talk,story"
-話す,はなす,to speak
-早い,はやい,early
-速い,はやい,quick
-春,はる,spring
-貼る,はる,to stick
-晴れ,はれ,clear weather
-晴れる,はれる,to be sunny
-半,はん,half
-晩,ばん,evening
-,パン,bread
-,ハンカチ,handkerchief
-番号,ばんごう,number
-晩御飯,ばんごはん,evening meal
-半分,はんぶん,half minute
-東,ひがし,east
-引く,ひく,to pull
-弾く,ひく,"to play an instrument with strings, including piano"
-低い,ひくい,"short,low"
-飛行機,ひこうき,aeroplane
-左,ひだり,left hand side
-人,ひと,person
-一つ,ひとつ,one
-一月,ひとつき,one month
-一人,ひとり,one person
-暇,ひま,free time
-百,ひゃく,hundred
-病院,びょういん,hospital
-病気,びょうき,illness
-昼,ひる,"noon, daytime"
-昼御飯,ひるごはん,midday meal
-広い,ひろい,"spacious,wide"
-,フィルム,roll of film
-封筒,ふうとう,envelope
-,プール,swimming pool
-,フォーク,fork
-吹く,ふく,to blow
-服,ふく,clothes
-二つ,ふたつ,two
-豚肉,ぶたにく,pork
-二人,ふたり,two people
-二日,ふつか,"two days, second day of the month"
-太い,ふとい,fat
-冬,ふゆ,winter
-降る,ふる,"to fall, e.g. rain or snow"
-古い,ふるい,old (not used for people)
-,ふろ,bath
-文章,ぶんしょう,"sentence,text"
-,ページ,page
-下手,へた,unskillful
-,ベッド,bed
-,ペット,pet
-部屋,へや,room
-辺,へん,area
-,ペン,pen
-勉強,べんきょうする,to study
-便利,べんり,"useful, convenient"
-帽子,ぼうし,hat
-,ボールペン,ball-point pen
-,ほか,"other, the rest"
-,ポケット,pocket
-欲しい,ほしい,want
-,ポスト,post
-細い,ほそい,thin
-,ボタン,button
-,ホテル,hotel
-本,ほん,book
-本棚,ほんだな,bookshelves
-,ほんとう,truth
-毎朝,まいあさ,every morning
-毎月,まいげつ/まいつき,every month
-毎週,まいしゅう,every week
-毎日,まいにち,every day
-毎年,まいねん/まいとし,every year
-毎晩,まいばん,every night
-前,まえ,before
-曲る,まがる,"to turn,to bend"
-,まずい,unpleasant
-,また,"again,and"
-,まだ,"yet,still"
-町,まち,"town,city"
-待つ,まつ,to wait
-,まっすぐ,"straight ahead,direct"
-,マッチ,match
-窓,まど,window
-丸い/円い,まるい,"round,circular"
-万,まん,ten thousand
-万年筆,まんねんひつ,fountain pen
-磨く,みがく,"to brush teeth, to polish"
-右,みぎ,right side
-短い,みじかい,short
-水,みず,water
-店,みせ,shop
-見せる,みせる,to show
-道,みち,street
-三日,みっか,"three days, third day of the month"
-三つ,みっつ,three
-緑,みどり,green
-皆さん,みなさん,everyone
-南,みなみ,south
-耳,みみ,ear
-見る  観る,みる,"to see, to watch"
-,みんな,everyone
-六日,むいか,"six days, sixth day of the month"
-向こう,むこう,over there
-難しい,むずかしい,difficult
-六つ,むっつ,six
-村,むら,village
-目,め,eye
-,メートル,metre
-眼鏡,めがね,glasses
-,もう,already
-もう一度,もういちど,again
-木曜日,もくようび,Thursday
-持つ,もつ,to hold
-,もっと,more
-物,もの,thing
-門,もん,gate
-問題,もんだい,problem
-八百屋,やおや,greengrocer
-野菜,やさい,vegetable
-易しい,やさしい,"easy, simple"
-安い,やすい,cheap
-休み,やすみ,"rest,holiday"
-休む,やすむ,to rest
-八つ,やっつ,eight
-山,やま,mountain
-,やる,to do
-夕方,ゆうがた,evening
-夕飯,ゆうはん,dinner
-郵便局,ゆうびんきょく,post office
-昨夜,ゆうべ,last night
-有名,ゆうめい,famous
-雪,ゆき,snow
-行く,ゆく,to go
-,ゆっくりと,slowly
-八日,ようか,"eight days, eighth day of the month"
-洋服,ようふく,western-style clothes
-,よく,"often, well"
-横,よこ,"beside,side,width"
-四日,よっか,"four days, fouth day of the month"
-四つ,よっつ,four
-呼ぶ,よぶ,"to call out,to invite"
-読む,よむ,to read
-夜,よる,"evening,night"
-弱い,よわい,weak
-来月,らいげつ,next month
-来週,らいしゅう,next week
-来年,らいねん,next year
-,ラジオ,radio
-,ラジカセ / ラジオカセット,radio cassette player
-,りっぱ,splendid
-留学生,りゅうがくせい,overseas student
-両親,りょうしん,both parents
-料理,りょうり,cuisine
-旅行,りょこう,travel
-零,れい,zero
-冷蔵庫,れいぞうこ,refrigerator
-,レコード,record
-,レストラン,restaurant
-練習,れんしゅうする,to practice
-廊下,ろうか,corridor
-六,ろく,six
-,ワイシャツ,business shirt
-若い,わかい,young
-分かる,わかる,to be understood
-忘れる,わすれる,to forget
-私,わたくし,"(humble) I,myself"
-私,わたし,"I,myself"
-渡す,わたす,to hand over
-渡る,わたる,to go across
-悪い,わるい,bad
-,より、ほう,Used for comparison.

docs/database.md

@@ -0,0 +1,29 @@
+# Database
+
+Here are some choices that have been made when designing the schema
+
+### `JMdict_{Reading,Kanji}Element.elementId` and `JMdict_Sense.senseId`
+
+The `elementId`/`senseId` field acts as a unique identifier for each individual element in these tables.
+It is a packed version of the `(entryId, orderNum)` pair, where the first number is given 7 digits and the second is given 2 digits (max count found so far is `40`).
+Since `entryId` already is a field in the table, it would technically have been fine to store the `orderNum` as a separate field,
+but it is easier to be able to refer to the entries without a composite foreign key in other tables.
+
+(NOTE: `entryId` is now inferred from `elementId` within sqlite using a generated column, so saying it is "stored in a separate field" might be a stretch)
+
+In addition, the reading element id's are added with `1000000000` to make them unique from the kanji element id's. This reduces the amount of space needed for indices in some locations, because you can simply filter out each part with `>` or `<`.
+
+We used to generate the `elementId` separately from `orderNum` as a sequential id, but it lead to all values
+shifting whenever the data was updated, leading to very big diffs. Making it be a unique composite of data coming
+from the source data itself means that the values will be stable across updates.
+
+Due to the way the data is structured, we can use the `elementId` as the ordering number as well.
+
+### `JMdict_EntryScore`
+
+The `JMdict_EntryScore` table is used to store the score of each entry, which is used for sorting search results. The score is calculated based on a number of variables.
+
+The table is automatically generated from other tables via triggers, and should be considered as a materialized view.
+
+<s>There is a score row for every single entry in both `JMdict_KanjiElement` and `JMdict_ReadingElement`, split by the `type` field.</s>
+This is no longer true, we now only store the rows for which the score is not `0`. The `type` field is now also virtual, since the `elementId` fields for both kanji and readings are unique to each other.

docs/overview.md

@@ -3,6 +3,7 @@
 This is the documentation for `jadb`. Since I'm currently the only one working on it, the documentation is more or less just notes to myself, to ensure I remember how and why I implemented certain features in a certain way a few months down the road. This is not a comprehensive and formal documentation for downstream use, neither for developers nor end-users.
 
 - [Word Search](./word-search.md)
+- [Database](./database.md)
 - [Lemmatizer](./lemmatizer.md)
 
 ## Project structure

flake.lock

@@ -1,48 +1,69 @@
 {
   "nodes": {
-    "jmdict-src": {
-      "flake": false,
+    "crane": {
       "locked": {
-        "narHash": "sha256-eOc3a/AYNRFF3w6lWhyf0Sh92xeXS7+9Qvn0tvvH6Ys=",
-        "type": "file",
-        "url": "http://ftp.edrdg.org/pub/Nihongo/JMdict_e.gz"
+        "lastModified": 1780099841,
+        "narHash": "sha256-EVZd2RsbpreRUDSi9rBwPY+ZxoyMaiEBbZxxhljbaS4=",
+        "owner": "ipetkov",
+        "repo": "crane",
+        "rev": "0532eb17955225173906d671fb36306bdeb1e2dc",
+        "type": "github"
       },
       "original": {
-        "type": "file",
-        "url": "http://ftp.edrdg.org/pub/Nihongo/JMdict_e.gz"
+        "owner": "ipetkov",
+        "repo": "crane",
+        "type": "github"
       }
     },
-    "jmdict-with-examples-src": {
-      "flake": false,
+    "datasources": {
+      "inputs": {
+        "nixpkgs": [
+          "nixpkgs"
+        ]
+      },
       "locked": {
-        "narHash": "sha256-nx+WMkscWvA/XImKM7NESYVmICwSgXWOO1KPXasHY94=",
-        "type": "file",
-        "url": "http://ftp.edrdg.org/pub/Nihongo/JMdict_e_examp.gz"
+        "lastModified": 1780302182,
+        "narHash": "sha256-IfC+dpdjjlkzrWlm+p851T43GsR04wMAPqGn63jisJ4=",
+        "ref": "refs/heads/main",
+        "rev": "c116674dd1e0b879660e6237e54904aa825d4511",
+        "revCount": 29,
+        "type": "git",
+        "url": "https://git.pvv.ntnu.no/Mugiten/datasources.git"
       },
       "original": {
-        "type": "file",
-        "url": "http://ftp.edrdg.org/pub/Nihongo/JMdict_e_examp.gz"
+        "type": "git",
+        "url": "https://git.pvv.ntnu.no/Mugiten/datasources.git"
       }
     },
-    "kanjidic2-src": {
-      "flake": false,
+    "nix-sqlite": {
+      "inputs": {
+        "nixpkgs": [
+          "tamerye",
+          "nixpkgs"
+        ]
+      },
       "locked": {
-        "narHash": "sha256-2T/cAS/kZmVMURStgHVhz524+J9+v5onKs8eEYf2fY0=",
-        "type": "file",
-        "url": "https://www.edrdg.org/kanjidic/kanjidic2.xml.gz"
+        "lastModified": 1780224621,
+        "narHash": "sha256-mGAHKHEzh+J83GzYCyuM6xdfBPFYTBjNAmDmbHKRZ5U=",
+        "ref": "main",
+        "rev": "8afa09b948ec5ccbe488c21d14d458060720d313",
+        "revCount": 28,
+        "type": "git",
+        "url": "https://git.pvv.ntnu.no/mugiten/nix-custom-sqlite.git"
       },
       "original": {
-        "type": "file",
-        "url": "https://www.edrdg.org/kanjidic/kanjidic2.xml.gz"
+        "ref": "main",
+        "type": "git",
+        "url": "https://git.pvv.ntnu.no/mugiten/nix-custom-sqlite.git"
       }
     },
     "nixpkgs": {
       "locked": {
-        "lastModified": 1774386573,
-        "narHash": "sha256-4hAV26quOxdC6iyG7kYaZcM3VOskcPUrdCQd/nx8obc=",
+        "lastModified": 1779560665,
+        "narHash": "sha256-tpyBcxPpcQb8ukyNF7DoCwfSY3VPsxHoYwj00Cayv5o=",
         "owner": "NixOS",
         "repo": "nixpkgs",
-        "rev": "46db2e09e1d3f113a13c0d7b81e2f221c63b8ce9",
+        "rev": "64c08a7ca051951c8eae34e3e3cb1e202fe36786",
         "type": "github"
       },
       "original": {
@@ -51,25 +72,56 @@
         "type": "indirect"
       }
     },
-    "radkfile-src": {
-      "flake": false,
-      "locked": {
-        "narHash": "sha256-DHpMUE2Umje8PbzXUCS6pHZeXQ5+WTxbjSkGU3erDHQ=",
-        "type": "file",
-        "url": "http://ftp.edrdg.org/pub/Nihongo/radkfile.gz"
-      },
-      "original": {
-        "type": "file",
-        "url": "http://ftp.edrdg.org/pub/Nihongo/radkfile.gz"
-      }
-    },
     "root": {
       "inputs": {
-        "jmdict-src": "jmdict-src",
-        "jmdict-with-examples-src": "jmdict-with-examples-src",
-        "kanjidic2-src": "kanjidic2-src",
+        "datasources": "datasources",
         "nixpkgs": "nixpkgs",
-        "radkfile-src": "radkfile-src"
+        "tamerye": "tamerye"
+      }
+    },
+    "rust-overlay": {
+      "inputs": {
+        "nixpkgs": [
+          "tamerye",
+          "nixpkgs"
+        ]
+      },
+      "locked": {
+        "lastModified": 1780197589,
+        "narHash": "sha256-FVCr2Ij/jKf59a4LW481eeOF6rJRreOBrVgW/aUBTrw=",
+        "owner": "oxalica",
+        "repo": "rust-overlay",
+        "rev": "21632e942d89bf1cce4e5a63d7e58a215a0cbfcc",
+        "type": "github"
+      },
+      "original": {
+        "owner": "oxalica",
+        "repo": "rust-overlay",
+        "type": "github"
+      }
+    },
+    "tamerye": {
+      "inputs": {
+        "crane": "crane",
+        "nix-sqlite": "nix-sqlite",
+        "nixpkgs": [
+          "nixpkgs"
+        ],
+        "rust-overlay": "rust-overlay"
+      },
+      "locked": {
+        "lastModified": 1780505561,
+        "narHash": "sha256-534k5H8k6GcFUGy7ENsOvuZ2Lf+6WcOf3vsqfrsnNlo=",
+        "ref": "main",
+        "rev": "f9789bb582218218e29a7ac674c5edf0d3609a5d",
+        "revCount": 17,
+        "type": "git",
+        "url": "https://git.pvv.ntnu.no/Mugiten/tamerye.git"
+      },
+      "original": {
+        "ref": "main",
+        "type": "git",
+        "url": "https://git.pvv.ntnu.no/Mugiten/tamerye.git"
       }
     }
   },

flake.nix

@@ -4,35 +4,22 @@
   inputs = {
     nixpkgs.url = "nixpkgs/nixos-unstable";
 
-    jmdict-src = {
-      # url = "http://ftp.edrdg.org/pub/Nihongo/JMdict.gz";
-      url = "http://ftp.edrdg.org/pub/Nihongo/JMdict_e.gz";
-      flake = false;
+    tamerye = {
+      url = "git+https://git.pvv.ntnu.no/Mugiten/tamerye.git?ref=main";
+      inputs.nixpkgs.follows = "nixpkgs";
     };
 
-    jmdict-with-examples-src = {
-      url = "http://ftp.edrdg.org/pub/Nihongo/JMdict_e_examp.gz";
-      flake = false;
-    };
-
-    radkfile-src = {
-      url = "http://ftp.edrdg.org/pub/Nihongo/radkfile.gz";
-      flake = false;
-    };
-
-    kanjidic2-src = {
-      url = "https://www.edrdg.org/kanjidic/kanjidic2.xml.gz";
-      flake = false;
+    datasources = {
+      url = "git+https://git.pvv.ntnu.no/Mugiten/datasources.git";
+      inputs.nixpkgs.follows = "nixpkgs";
     };
   };
 
   outputs = {
     self,
     nixpkgs,
-    jmdict-src,
-    jmdict-with-examples-src,
-    radkfile-src,
-    kanjidic2-src
+    tamerye,
+    datasources,
   }: let
     inherit (nixpkgs) lib;
     systems = [
@@ -46,7 +33,9 @@
     forAllSystems = f: lib.genAttrs systems (system: let
       pkgs = import nixpkgs {
         inherit system;
-        overlays = [ self.overlays.sqlite-icu-ext ];
+        overlays = [
+          tamerye.overlays.default
+        ];
       };
     in f system pkgs);
   in {
@@ -59,7 +48,7 @@
 
             runtimeEnv = {
               JADB_PATH = "${self.packages.${system}.database}/jadb.sqlite";
-              LIBSQLITE_PATH = "${pkgs.sqlite.out}/lib/libsqlite3.so";
+              LIBSQLITE_PATH = "${pkgs.tamerye-sqlite}/lib/libsqlite3.so";
             };
 
             text = ''
@@ -87,31 +76,26 @@
           gnumake
           lcov
           sqldiff
-          sqlite-interactive-icu-ext
+          tamerye-sqlite-cli
         ];
         env = {
-          LIBSQLITE_PATH = "${pkgs.sqlite.out}/lib/libsqlite3.so";
+          LIBSQLITE_PATH = "${pkgs.tamerye-sqlite}/lib/libsqlite3.so";
           JADB_PATH = "result/jadb.sqlite";
-          LD_LIBRARY_PATH = lib.makeLibraryPath [ pkgs.sqlite ];
+          LD_LIBRARY_PATH = lib.makeLibraryPath [ pkgs.tamerye-sqlite ];
         };
       };
 
       sqlite-debugging = pkgs.mkShell {
         packages = with pkgs; [
-          sqlite-interactive-icu-ext
+          tamerye-sqlite-cli
           sqlite-analyzer
           sqlite-web
-          sqlint
+          # sqlint
           sqlfluff
         ];
       };
     });
 
-    overlays.sqlite-icu-ext = final: prev: {
-      sqlite-icu-ext = prev.callPackage ./nix/sqlite.nix { };
-      sqlite-interactive-icu-ext = prev.callPackage ./nix/sqlite.nix { sqlite = prev.sqlite-interactive; };
-    };
-
     packages = let
       edrdgMetadata = {
         license = [{
@@ -145,38 +129,26 @@
         ln -s ${src} $out
       '';
 
-      sqlite = pkgs.callPackage ./nix/sqlite.nix { };
-
-      fts5-icu-tokenizer = pkgs.callPackage ./nix/fts5-icu-tokenizer/package.nix { };
-
-      jmdict = pkgs.callPackage ./nix/jmdict.nix {
-        inherit jmdict-src jmdict-with-examples-src edrdgMetadata;
-      };
-
-      radkfile =  pkgs.callPackage ./nix/radkfile.nix {
-        inherit radkfile-src edrdgMetadata;
-      };
-
-      kanjidic2 = pkgs.callPackage ./nix/kanjidic2.nix {
-        inherit kanjidic2-src edrdgMetadata;
-      };
+      inherit (datasources.packages.${system}) jmdict radkfile kanjidic2;
 
       database-tool = pkgs.callPackage ./nix/database_tool.nix {
+        sqlite = pkgs.tamerye-sqlite;
         inherit src;
-        sqlite = pkgs.sqlite-icu-ext;
       };
 
       database = pkgs.callPackage ./nix/database.nix {
-        inherit (self.packages.${system}) database-tool jmdict radkfile kanjidic2;
+        sqlite = pkgs.tamerye-sqlite-cli;
+        inherit (datasources.packages.${system}) jmdict radkfile kanjidic2 tanos-jlpt;
+        inherit (self.packages.${system}) database-tool;
         inherit src;
-        sqlite = pkgs.sqlite-icu-ext;
       };
 
       database-wal = pkgs.callPackage ./nix/database.nix {
-        inherit (self.packages.${system}) database-tool jmdict radkfile kanjidic2;
+        sqlite = pkgs.tamerye-sqlite-cli;
+        inherit (datasources.packages.${system}) jmdict radkfile kanjidic2 tanos-jlpt;
+        inherit (self.packages.${system}) database-tool;
         inherit src;
         wal = true;
-        sqlite = pkgs.sqlite-icu-ext;
       };
 
       docs = pkgs.callPackage ./nix/docs.nix {

lib/_data_ingestion/jmdict/objects.dart

@@ -1,13 +1,15 @@
 import 'package:jadb/_data_ingestion/sql_writable.dart';
 
 abstract class Element extends SQLWritable {
+  final int elementId;
   final String reading;
   final int? news;
   final int? ichi;
   final int? spec;
   final int? gai;
   final int? nf;
-  const Element({
+  Element({
+    required this.elementId,
     required this.reading,
     this.news,
     this.ichi,
@@ -18,6 +20,7 @@ abstract class Element extends SQLWritable {
 
   @override
   Map<String, Object?> get sqlValue => {
+    'elementId': elementId,
     'reading': reading,
     'news': news,
     'ichi': ichi,
@@ -28,12 +31,11 @@ abstract class Element extends SQLWritable {
 }
 
 class KanjiElement extends Element {
-  int orderNum;
   List<String> info;
 
   KanjiElement({
     this.info = const [],
-    required this.orderNum,
+    required super.elementId,
     required super.reading,
     super.news,
     super.ichi,
@@ -43,23 +45,19 @@ class KanjiElement extends Element {
   });
 
   @override
-  Map<String, Object?> get sqlValue => {
-    ...super.sqlValue,
-    'orderNum': orderNum,
-  };
+  Map<String, Object?> get sqlValue => {...super.sqlValue};
 }
 
 class ReadingElement extends Element {
-  int orderNum;
   bool readingDoesNotMatchKanji;
   List<String> info;
   List<String> restrictions;
 
   ReadingElement({
-    required this.orderNum,
     required this.readingDoesNotMatchKanji,
     this.info = const [],
     this.restrictions = const [],
+    required super.elementId,
     required super.reading,
     super.news,
     super.ichi,
@@ -71,7 +69,6 @@ class ReadingElement extends Element {
   @override
   Map<String, Object?> get sqlValue => {
     ...super.sqlValue,
-    'orderNum': orderNum,
     'readingDoesNotMatchKanji': readingDoesNotMatchKanji,
   };
 }
@@ -107,9 +104,8 @@ class Glossary extends SQLWritable {
 
   @override
   Map<String, Object?> get sqlValue => {
-    'language': language,
+    // 'language': language,
     'phrase': phrase,
-    'type': type,
   };
 }
 
@@ -131,6 +127,19 @@ class XRefParts {
     'readingRef': readingRef,
     'senseOrderNum': senseOrderNum,
   };
+
+  @override
+  bool operator ==(Object other) {
+    if (identical(this, other)) return true;
+
+    return other is XRefParts &&
+        other.kanjiRef == kanjiRef &&
+        other.readingRef == readingRef &&
+        other.senseOrderNum == senseOrderNum;
+  }
+
+  @override
+  int get hashCode => Object.hash(kanjiRef, readingRef, senseOrderNum);
 }
 
 class XRef {
@@ -142,7 +151,6 @@ class XRef {
 
 class Sense extends SQLWritable {
   final int senseId;
-  final int orderNum;
   final List<XRefParts> antonyms;
   final List<String> dialects;
   final List<String> fields;
@@ -157,7 +165,6 @@ class Sense extends SQLWritable {
 
   const Sense({
     required this.senseId,
-    required this.orderNum,
     this.antonyms = const [],
     this.dialects = const [],
     this.fields = const [],
@@ -172,10 +179,7 @@ class Sense extends SQLWritable {
   });
 
   @override
-  Map<String, Object?> get sqlValue => {
-    'senseId': senseId,
-    'orderNum': orderNum,
-  };
+  Map<String, Object?> get sqlValue => {'senseId': senseId};
 
   bool get isEmpty =>
       antonyms.isEmpty &&

lib/_data_ingestion/jmdict/seed_data.dart

@@ -1,4 +1,5 @@
 import 'dart:collection';
+import 'dart:io';
 
 import 'package:collection/collection.dart';
 import 'package:jadb/_data_ingestion/jmdict/objects.dart';
@@ -8,13 +9,18 @@ import 'package:sqflite_common/sqlite_api.dart';
 /// A wrapper for the result of resolving an xref, which includes the resolved entry and a flag
 /// indicating whether the xref was ambiguous (i.e. could refer to multiple entries).
 class ResolvedXref {
-  Entry entry;
-  bool ambiguous;
-  int? senseOrderNum;
+  final Entry entry;
+  final bool ambiguous;
 
-  ResolvedXref(this.entry, this.ambiguous, senseOrderNum);
+  const ResolvedXref(this.entry, this.ambiguous);
 }
 
+// A constant map of xref parts to jmdict id for unresolvable xrefs.
+final xrefExceptions = {
+  // NOTE: see https://www.edrdg.org/jmwsgi/entr.py?svc=jmdict&g=2870981.1~2369718 for details
+  XRefParts(kanjiRef: 'プレストレスト', readingRef: 'コンクリート'): 2472380,
+};
+
 /// Resolves an xref (pair of kanji, optionally reading, and optionally sense number) to an a specific
 /// JMdict entry, if possible.
 ///
@@ -27,15 +33,65 @@ ResolvedXref resolveXref(
   SplayTreeMap<String, Set<Entry>> entriesByReading,
   XRefParts xref,
 ) {
-  List<Entry> candidateEntries = switch ((xref.kanjiRef, xref.readingRef)) {
-    (null, null) => throw Exception(
-      'Xref $xref has no kanji or reading reference',
-    ),
-    (final String k, null) => entriesByKanji[k]!.toList(),
-    (null, final String r) => entriesByReading[r]!.toList(),
-    (final String k, final String r) =>
-      entriesByKanji[k]!.intersection(entriesByReading[r]!).toList(),
-  };
+  late List<Entry> candidateEntries;
+
+  if (xrefExceptions.containsKey(xref)) {
+    final exceptionEntryId = xrefExceptions[xref]!;
+    // NOTE: this is slow, but we have few exceptions. Let's wait for JMdict XML-NG to be released so we can delete this :)
+    final exceptionEntry =
+        entriesByKanji.values
+            .expand((set) => set)
+            .firstWhereOrNull((entry) => entry.entryId == exceptionEntryId) ??
+        entriesByReading.values
+            .expand((set) => set)
+            .firstWhereOrNull((entry) => entry.entryId == exceptionEntryId);
+
+    if (exceptionEntry != null) {
+      return ResolvedXref(exceptionEntry, false);
+    } else {
+      throw Exception(
+        'Xref $xref matches an exception entry ID $exceptionEntryId, but that entry was not found among the candidates.',
+      );
+    }
+  }
+
+  switch ((xref.kanjiRef, xref.readingRef)) {
+    case (null, null):
+      throw Exception('Xref $xref has no kanji or reading reference');
+
+    case (final String k, null):
+      if (!entriesByKanji.containsKey(k)) {
+        throw Exception(
+          'Xref $xref has kanji reference "$k" but no entries found with that kanji',
+        );
+      }
+      candidateEntries = entriesByKanji[k]!.toList();
+      break;
+
+    case (null, final String r):
+      if (!entriesByReading.containsKey(r)) {
+        throw Exception(
+          'Xref $xref has reading reference "$r" but no entries found with that reading',
+        );
+      }
+      candidateEntries = entriesByReading[r]!.toList();
+      break;
+
+    case (final String k, final String r):
+      if (!entriesByKanji.containsKey(k)) {
+        throw Exception(
+          'Xref $xref has kanji reference "$k" but no entries found with that kanji',
+        );
+      }
+      if (!entriesByReading.containsKey(r)) {
+        throw Exception(
+          'Xref $xref has reading reference "$r" but no entries found with that reading',
+        );
+      }
+      candidateEntries = entriesByKanji[k]!
+          .intersection(entriesByReading[r]!)
+          .toList();
+  }
 
   // Filter out entries that don't have the number of senses specified in the xref
   if (xref.senseOrderNum != null) {
@@ -75,10 +131,9 @@ ResolvedXref resolveXref(
       'kanjiRef: ${xref.kanjiRef}, readingRef: ${xref.readingRef}, '
       'senseOrderNum: ${xref.senseOrderNum}',
     );
-
-    return ResolvedXref(candidateEntries.first, true, xref.senseOrderNum);
+    return ResolvedXref(candidateEntries.first, true);
   } else {
-    return ResolvedXref(candidateEntries.first, false, xref.senseOrderNum);
+    return ResolvedXref(candidateEntries.first, false);
   }
 }
 
@@ -86,41 +141,47 @@ Future<void> seedJMDictData(List<Entry> entries, Database db) async {
   print('  [JMdict] Batch 1 - Kanji and readings');
   Batch b = db.batch();
 
-  int elementId = 0;
+  if (Platform.environment['JMDICT_VERSION'] != null &&
+      Platform.environment['JMDICT_DATE'] != null &&
+      Platform.environment['JMDICT_HASH'] != null) {
+    b.insert(JMdictTableNames.version, {
+      'version': Platform.environment['JMDICT_VERSION']!,
+      'date': Platform.environment['JMDICT_DATE']!,
+      'hash': Platform.environment['JMDICT_HASH']!,
+    });
+  } else {
+    print(
+      'WARNING: JMDICT version information not found in environment variables. '
+      'This may cause issues with future updates.',
+    );
+  }
+
   for (final e in entries) {
     b.insert(JMdictTableNames.entry, e.sqlValue);
 
     for (final k in e.kanji) {
-      elementId++;
-      b.insert(
-        JMdictTableNames.kanjiElement,
-        k.sqlValue..addAll({'entryId': e.entryId, 'elementId': elementId}),
-      );
+      b.insert(JMdictTableNames.kanjiElement, k.sqlValue);
 
       for (final i in k.info) {
         b.insert(JMdictTableNames.kanjiInfo, {
-          'elementId': elementId,
+          'elementId': k.elementId,
           'info': i,
         });
       }
     }
 
     for (final r in e.readings) {
-      elementId++;
-      b.insert(
-        JMdictTableNames.readingElement,
-        r.sqlValue..addAll({'entryId': e.entryId, 'elementId': elementId}),
-      );
+      b.insert(JMdictTableNames.readingElement, r.sqlValue);
 
       for (final i in r.info) {
         b.insert(JMdictTableNames.readingInfo, {
-          'elementId': elementId,
+          'elementId': r.elementId,
           'info': i,
         });
       }
       for (final res in r.restrictions) {
         b.insert(JMdictTableNames.readingRestriction, {
-          'elementId': elementId,
+          'elementId': r.elementId,
           'restriction': res,
         });
       }
@@ -134,10 +195,7 @@ Future<void> seedJMDictData(List<Entry> entries, Database db) async {
 
   for (final e in entries) {
     for (final s in e.senses) {
-      b.insert(
-        JMdictTableNames.sense,
-        s.sqlValue..addAll({'entryId': e.entryId}),
-      );
+      b.insert(JMdictTableNames.sense, s.sqlValue);
       for (final d in s.dialects) {
         b.insert(JMdictTableNames.senseDialect, {
           'senseId': s.senseId,
@@ -161,16 +219,18 @@ Future<void> seedJMDictData(List<Entry> entries, Database db) async {
       }
       for (final rk in s.restrictedToKanji) {
         b.insert(JMdictTableNames.senseRestrictedToKanji, {
-          'entryId': e.entryId,
           'senseId': s.senseId,
-          'kanjiOrderNum': e.kanji.indexWhere((k) => k.reading == rk) + 1,
+          'kanjiElementId': e.kanji
+              .firstWhere((k) => k.reading == rk)
+              .elementId,
         });
       }
       for (final rr in s.restrictedToReading) {
         b.insert(JMdictTableNames.senseRestrictedToReading, {
-          'entryId': e.entryId,
           'senseId': s.senseId,
-          'readingOrderNum': e.readings.indexWhere((r) => r.reading == rr) + 1,
+          'readingElementId': e.readings
+              .firstWhere((r) => r.reading == rr)
+              .elementId,
         });
       }
       for (final ls in s.languageSource) {
@@ -184,6 +244,14 @@ Future<void> seedJMDictData(List<Entry> entries, Database db) async {
           JMdictTableNames.senseGlossary,
           g.sqlValue..addAll({'senseId': s.senseId}),
         );
+
+        if (g.type != null) {
+          b.insert(JMdictTableNames.senseGlossaryType, {
+            'senseId': s.senseId,
+            'phrase': g.phrase,
+            'type': g.type!,
+          });
+        }
       }
     }
   }
@@ -231,7 +299,9 @@ Future<void> seedJMDictData(List<Entry> entries, Database db) async {
         b.insert(JMdictTableNames.senseSeeAlso, {
           'senseId': s.senseId,
           'xrefEntryId': resolvedEntry.entry.entryId,
-          'xrefSenseOrderNum': resolvedEntry.senseOrderNum,
+          'seeAlsoSense': xref.senseOrderNum != null
+              ? xref.senseOrderNum! - 1
+              : null,
           'ambiguous': resolvedEntry.ambiguous,
         });
       }
@@ -256,6 +326,9 @@ Future<void> seedJMDictData(List<Entry> entries, Database db) async {
         b.insert(JMdictTableNames.senseAntonyms, {
           'senseId': s.senseId,
           'xrefEntryId': resolvedEntry.entry.entryId,
+          'antonymSense': ant.senseOrderNum != null
+              ? ant.senseOrderNum! - 1
+              : null,
           'ambiguous': resolvedEntry.ambiguous,
         });
       }

lib/_data_ingestion/jmdict/xml_parser.dart

@@ -71,8 +71,6 @@ XRefParts parseXrefParts(String s) {
 List<Entry> parseJMDictData(XmlElement root) {
   final List<Entry> entries = [];
 
-  int senseId = 0;
-
   for (final entry in root.childElements) {
     final entryId = int.parse(entry.findElements('ent_seq').first.innerText);
 
@@ -80,11 +78,18 @@ List<Entry> parseJMDictData(XmlElement root) {
     final List<ReadingElement> readingEls = [];
     final List<Sense> senses = [];
 
-    for (final (kanjiNum, kEle) in entry.findElements('k_ele').indexed) {
+    for (final (orderNum, kEle) in entry.findElements('k_ele').indexed) {
+      assert(
+        orderNum < 100,
+        'Entry $entryId has more than 100 kanji elements, which will break the elementId generation logic.',
+      );
+      final elementId = entryId * 100 + orderNum;
+
       final kePri = getPriorityValues(kEle, 'ke');
+
       kanjiEls.add(
         KanjiElement(
-          orderNum: kanjiNum + 1,
+          elementId: elementId,
           info: kEle
               .findElements('ke_inf')
               .map((e) => e.innerText.substring(1, e.innerText.length - 1))
@@ -100,13 +105,20 @@ List<Entry> parseJMDictData(XmlElement root) {
     }
 
     for (final (orderNum, rEle) in entry.findElements('r_ele').indexed) {
+      assert(
+        orderNum < 100,
+        'Entry $entryId has more than 100 readings, which will break the elementId generation logic.',
+      );
+      final elementId = 1_000_000_000 + entryId * 100 + orderNum;
+
       final rePri = getPriorityValues(rEle, 're');
       final readingDoesNotMatchKanji = rEle
           .findElements('re_nokanji')
           .isNotEmpty;
+
       readingEls.add(
         ReadingElement(
-          orderNum: orderNum + 1,
+          elementId: elementId,
           readingDoesNotMatchKanji: readingDoesNotMatchKanji,
           info: rEle
               .findElements('re_inf')
@@ -127,10 +139,14 @@ List<Entry> parseJMDictData(XmlElement root) {
     }
 
     for (final (orderNum, sense) in entry.findElements('sense').indexed) {
-      senseId++;
+      assert(
+        orderNum < 100,
+        'Entry $entryId has more than 100 senses, which will break the senseId generation logic.',
+      );
+      final senseId = entryId * 100 + orderNum;
+
       final result = Sense(
         senseId: senseId,
-        orderNum: orderNum + 1,
         restrictedToKanji: sense
             .findElements('stagk')
             .map((e) => e.innerText)

lib/_data_ingestion/kanjidic/objects.dart

@@ -248,9 +248,6 @@ class Character extends SQLWritable {
   @override
   Map<String, Object?> get sqlValue => {
     'literal': literal,
-    'grade': grade,
     'strokeCount': strokeCount,
-    'frequency': frequency,
-    'jlpt': jlpt,
   };
 }

lib/_data_ingestion/kanjidic/seed_data.dart

@@ -1,3 +1,5 @@
+import 'dart:io';
+
 import 'package:jadb/table_names/kanjidic.dart';
 import 'package:sqflite_common/sqlite_api.dart';
 
@@ -5,6 +7,22 @@ import 'objects.dart';
 
 Future<void> seedKANJIDICData(List<Character> characters, Database db) async {
   final b = db.batch();
+
+  if (Platform.environment['KANJIDIC_VERSION'] != null &&
+      Platform.environment['KANJIDIC_DATE'] != null &&
+      Platform.environment['KANJIDIC_HASH'] != null) {
+    b.insert(KANJIDICTableNames.version, {
+      'version': Platform.environment['KANJIDIC_VERSION']!,
+      'date': Platform.environment['KANJIDIC_DATE']!,
+      'hash': Platform.environment['KANJIDIC_HASH']!,
+    });
+  } else {
+    print(
+      'WARNING: KANJIDIC version information not found in environment variables. '
+      'This may cause issues with future updates.',
+    );
+  }
+
   for (final c in characters) {
     // if (c.dictionaryReferences.any((e) =>
     //     c.dictionaryReferences
@@ -15,6 +33,24 @@ Future<void> seedKANJIDICData(List<Character> characters, Database db) async {
     //     }
     b.insert(KANJIDICTableNames.character, c.sqlValue);
 
+    if (c.grade != null) {
+      b.insert(KANJIDICTableNames.grade, {
+        'kanji': c.literal,
+        'grade': c.grade!,
+      });
+    }
+
+    if (c.frequency != null) {
+      b.insert(KANJIDICTableNames.frequency, {
+        'kanji': c.literal,
+        'frequency': c.frequency!,
+      });
+    }
+
+    if (c.jlpt != null) {
+      b.insert(KANJIDICTableNames.jlpt, {'kanji': c.literal, 'jlpt': c.jlpt!});
+    }
+
     for (final n in c.radicalName) {
       assert(c.radical != null, 'Radical name without radical');
       b.insert(

lib/_data_ingestion/open_local_db.dart

@@ -27,7 +27,6 @@ Future<Database> openLocalDb({
           await db.execute('PRAGMA journal_mode = WAL');
         }
         await db.execute('PRAGMA foreign_keys = ON');
-        await db.execute("SELECT icu_load_collation('ja_JP', 'japanese')");
       },
       readOnly: !readWrite,
     ),

lib/_data_ingestion/radkfile/parser.dart

@@ -1,7 +1,7 @@
 import 'dart:io';
 
 Iterable<String> parseRADKFILEBlocks(File radkfile) {
-  final String content = File('data/tmp/radkfile_utf8').readAsStringSync();
+  final String content = radkfile.readAsStringSync();
 
   final Iterable<String> blocks = content
       .replaceAll(RegExp(r'^#.*$'), '')

lib/_data_ingestion/radkfile/seed_data.dart

@@ -1,9 +1,26 @@
+import 'dart:io';
+
 import 'package:jadb/table_names/radkfile.dart';
 import 'package:sqflite_common/sqlite_api.dart';
 
 Future<void> seedRADKFILEData(Iterable<String> blocks, Database db) async {
   final b = db.batch();
 
+  if (Platform.environment['RADKFILE_VERSION'] != null &&
+      Platform.environment['RADKFILE_DATE'] != null &&
+      Platform.environment['RADKFILE_HASH'] != null) {
+    b.insert(RADKFILETableNames.version, {
+      'version': Platform.environment['RADKFILE_VERSION']!,
+      'date': Platform.environment['RADKFILE_DATE']!,
+      'hash': Platform.environment['RADKFILE_HASH']!,
+    });
+  } else {
+    print(
+      'WARNING: RADKFILE version information not found in environment variables. '
+      'This may cause issues with future updates.',
+    );
+  }
+
   for (final block in blocks) {
     final String radical = block[1];
     final List<String> kanjiList =

lib/_data_ingestion/seed_database.dart

@@ -24,7 +24,7 @@ Future<void> seedData(Database db) async {
 
 Future<void> parseAndSeedDataFromJMdict(Database db) async {
   print('[JMdict] Reading file content...');
-  final String rawXML = File('data/tmp/JMdict.xml').readAsStringSync();
+  final String rawXML = File('data/JMdict.xml').readAsStringSync();
 
   print('[JMdict] Parsing XML tags...');
   final XmlElement root = XmlDocument.parse(rawXML).getElement('JMdict')!;
@@ -38,7 +38,7 @@ Future<void> parseAndSeedDataFromJMdict(Database db) async {
 
 Future<void> parseAndSeedDataFromKANJIDIC(Database db) async {
   print('[KANJIDIC2] Reading file...');
-  final String rawXML = File('data/tmp/kanjidic2.xml').readAsStringSync();
+  final String rawXML = File('data/kanjidic2.xml').readAsStringSync();
 
   print('[KANJIDIC2] Parsing XML...');
   final XmlElement root = XmlDocument.parse(rawXML).getElement('kanjidic2')!;
@@ -52,7 +52,7 @@ Future<void> parseAndSeedDataFromKANJIDIC(Database db) async {
 
 Future<void> parseAndSeedDataFromRADKFILE(Database db) async {
   print('[RADKFILE] Reading file...');
-  final File raw = File('data/tmp/RADKFILE');
+  final File raw = File('data/RADKFILE');
 
   print('[RADKFILE] Parsing content...');
   final blocks = parseRADKFILEBlocks(raw);

lib/_data_ingestion/tanos-jlpt/csv_parser.dart

@@ -3,7 +3,6 @@ import 'dart:io';
 
 import 'package:csv/csv.dart';
 import 'package:jadb/_data_ingestion/tanos-jlpt/objects.dart';
-import 'package:xml/xml_events.dart';
 
 Future<List<JLPTRankedWord>> parseJLPTRankedWords(
   Map<String, File> files,

lib/_data_ingestion/tanos-jlpt/seed_data.dart

@@ -1,3 +1,5 @@
+import 'dart:io';
+
 import 'package:jadb/table_names/tanos_jlpt.dart';
 import 'package:sqflite_common/sqlite_api.dart';
 
@@ -7,6 +9,21 @@ Future<void> seedTanosJLPTData(
 ) async {
   final Batch b = db.batch();
 
+  if (Platform.environment['TANOS_JLPT_VERSION'] != null &&
+      Platform.environment['TANOS_JLPT_DATE'] != null &&
+      Platform.environment['TANOS_JLPT_HASH'] != null) {
+    b.insert(TanosJLPTTableNames.version, {
+      'version': Platform.environment['TANOS_JLPT_VERSION']!,
+      'date': Platform.environment['TANOS_JLPT_DATE']!,
+      'hash': Platform.environment['TANOS_JLPT_HASH']!,
+    });
+  } else {
+    print(
+      'WARNING: Tanos JLPT version information not found in environment variables. '
+      'This may cause issues with future updates.',
+    );
+  }
+
   for (final jlptLevel in resolvedEntries.entries) {
     final level = jlptLevel.key;
     final entryIds = jlptLevel.value;

lib/const_data/radicals.dart

@@ -1,217 +1,310 @@
-const Map<int, List<String>> radicals = {
-  1: ['一', '｜', '丶', 'ノ', '乙', '亅'],
+import 'package:collection/collection.dart';
+
+class RadkfileRadical {
+  /// The formal variant of the radical
+  ///
+  /// This is the one you want to use for searching.
+  final String formalVariant;
+
+  /// The informal variant of the radical (if it differs from the formal one).
+  ///
+  /// This is the one you should display to the user.
+  final String? informalVariant;
+
+  /// The number of strokes in this radical.
+  final int strokeCount;
+
+  const RadkfileRadical({
+    required this.formalVariant,
+    this.informalVariant,
+    required this.strokeCount,
+  });
+}
+
+const Map<int, List<RadkfileRadical>> radicals = {
+  1: [
+    RadkfileRadical(formalVariant: '一', strokeCount: 1),
+    RadkfileRadical(formalVariant: '｜', strokeCount: 1),
+    RadkfileRadical(formalVariant: '丶', strokeCount: 1),
+    RadkfileRadical(formalVariant: 'ノ', strokeCount: 1),
+    RadkfileRadical(formalVariant: '乙', strokeCount: 1),
+    RadkfileRadical(formalVariant: '亅', strokeCount: 1),
+  ],
   2: [
-    '二',
-    '亠',
-    '人',
-    '⺅',
-    '𠆢',
-    '儿',
-    '入',
-    'ハ',
-    '丷',
-    '冂',
-    '冖',
-    '冫',
-    '几',
-    '凵',
-    '刀',
-    '⺉',
-    '力',
-    '勹',
-    '匕',
-    '匚',
-    '十',
-    '卜',
-    '卩',
-    '厂',
-    '厶',
-    '又',
-    'マ',
-    '九',
-    'ユ',
-    '乃',
-    '𠂉',
+    RadkfileRadical(formalVariant: '二', strokeCount: 2),
+    RadkfileRadical(formalVariant: '亠', strokeCount: 2),
+    RadkfileRadical(formalVariant: '人', strokeCount: 2),
+    RadkfileRadical(formalVariant: '化', informalVariant: '⺅', strokeCount: 2),
+    RadkfileRadical(formalVariant: '个', informalVariant: '𠆢', strokeCount: 2),
+    RadkfileRadical(formalVariant: '儿', strokeCount: 2),
+    RadkfileRadical(formalVariant: '入', strokeCount: 2),
+    RadkfileRadical(formalVariant: 'ハ', strokeCount: 2),
+    RadkfileRadical(formalVariant: '并', strokeCount: 2),
+    RadkfileRadical(formalVariant: '冂', strokeCount: 2),
+    RadkfileRadical(formalVariant: '冖', strokeCount: 2),
+    RadkfileRadical(formalVariant: '冫', strokeCount: 2),
+    RadkfileRadical(formalVariant: '几', strokeCount: 2),
+    RadkfileRadical(formalVariant: '凵', strokeCount: 2),
+    RadkfileRadical(formalVariant: '刀', strokeCount: 2),
+    RadkfileRadical(formalVariant: '刈', strokeCount: 2),
+    RadkfileRadical(formalVariant: '力', strokeCount: 2),
+    RadkfileRadical(formalVariant: '勹', strokeCount: 2),
+    RadkfileRadical(formalVariant: '匕', strokeCount: 2),
+    RadkfileRadical(formalVariant: '匚', strokeCount: 2),
+    RadkfileRadical(formalVariant: '十', strokeCount: 2),
+    RadkfileRadical(formalVariant: '卜', strokeCount: 2),
+    RadkfileRadical(formalVariant: '卩', strokeCount: 2),
+    RadkfileRadical(formalVariant: '厂', strokeCount: 2),
+    RadkfileRadical(formalVariant: '厶', strokeCount: 2),
+    RadkfileRadical(formalVariant: '又', strokeCount: 2),
+    RadkfileRadical(formalVariant: 'マ', strokeCount: 2),
+    RadkfileRadical(formalVariant: '九', strokeCount: 2),
+    RadkfileRadical(formalVariant: 'ユ', strokeCount: 2),
+    RadkfileRadical(formalVariant: '乃', strokeCount: 2),
+    RadkfileRadical(formalVariant: '乞', strokeCount: 2),
   ],
   3: [
-    '⻌',
-    '口',
-    '囗',
-    '土',
-    '士',
-    '夂',
-    '夕',
-    '大',
-    '女',
-    '子',
-    '宀',
-    '寸',
-    '小',
-    '⺌',
-    '尢',
-    '尸',
-    '屮',
-    '山',
-    '川',
-    '巛',
-    '工',
-    '已',
-    '巾',
-    '干',
-    '幺',
-    '广',
-    '廴',
-    '廾',
-    '弋',
-    '弓',
-    'ヨ',
-    '彑',
-    '彡',
-    '彳',
-    '⺖',
-    '⺘',
-    '⺡',
-    '⺨',
-    '⺾',
-    '⻏',
-    '⻖',
-    '也',
-    '亡',
-    '及',
-    '久',
+    RadkfileRadical(formalVariant: '込', informalVariant: '⻌', strokeCount: 3),
+    RadkfileRadical(formalVariant: '口', strokeCount: 3),
+    RadkfileRadical(formalVariant: '囗', strokeCount: 3),
+    RadkfileRadical(formalVariant: '土', strokeCount: 3),
+    RadkfileRadical(formalVariant: '士', strokeCount: 3),
+    RadkfileRadical(formalVariant: '夂', strokeCount: 3),
+    RadkfileRadical(formalVariant: '夕', strokeCount: 3),
+    RadkfileRadical(formalVariant: '大', strokeCount: 3),
+    RadkfileRadical(formalVariant: '女', strokeCount: 3),
+    RadkfileRadical(formalVariant: '子', strokeCount: 3),
+    RadkfileRadical(formalVariant: '宀', strokeCount: 3),
+    RadkfileRadical(formalVariant: '寸', strokeCount: 3),
+    RadkfileRadical(formalVariant: '小', strokeCount: 3),
+    RadkfileRadical(formalVariant: '尚', informalVariant: '⺌', strokeCount: 3),
+    RadkfileRadical(formalVariant: '尢', strokeCount: 3),
+    RadkfileRadical(formalVariant: '尸', strokeCount: 3),
+    RadkfileRadical(formalVariant: '屮', strokeCount: 3),
+    RadkfileRadical(formalVariant: '山', strokeCount: 3),
+    RadkfileRadical(formalVariant: '川', strokeCount: 3),
+    RadkfileRadical(formalVariant: '巛', strokeCount: 3),
+    RadkfileRadical(formalVariant: '工', strokeCount: 3),
+    RadkfileRadical(formalVariant: '已', strokeCount: 3),
+    RadkfileRadical(formalVariant: '巾', strokeCount: 3),
+    RadkfileRadical(formalVariant: '干', strokeCount: 3),
+    RadkfileRadical(formalVariant: '幺', strokeCount: 3),
+    RadkfileRadical(formalVariant: '广', strokeCount: 3),
+    RadkfileRadical(formalVariant: '廴', strokeCount: 3),
+    RadkfileRadical(formalVariant: '廾', strokeCount: 3),
+    RadkfileRadical(formalVariant: '弋', strokeCount: 3),
+    RadkfileRadical(formalVariant: '弓', strokeCount: 3),
+    RadkfileRadical(formalVariant: 'ヨ', strokeCount: 3),
+    RadkfileRadical(formalVariant: '彑', strokeCount: 3),
+    RadkfileRadical(formalVariant: '彡', strokeCount: 3),
+    RadkfileRadical(formalVariant: '彳', strokeCount: 3),
+    RadkfileRadical(formalVariant: '忙', informalVariant: '⺖', strokeCount: 3),
+    RadkfileRadical(formalVariant: '扎', informalVariant: '⺘', strokeCount: 3),
+    RadkfileRadical(formalVariant: '汁', informalVariant: '⺡', strokeCount: 3),
+    RadkfileRadical(formalVariant: '犯', informalVariant: '⺨', strokeCount: 3),
+    RadkfileRadical(formalVariant: '艾', informalVariant: '⺾', strokeCount: 3),
+    RadkfileRadical(formalVariant: '邦', informalVariant: '⻏', strokeCount: 3),
+    RadkfileRadical(formalVariant: '阡', informalVariant: '⻖', strokeCount: 3),
+    RadkfileRadical(formalVariant: '也', strokeCount: 3),
+    RadkfileRadical(formalVariant: '亡', strokeCount: 3),
+    RadkfileRadical(formalVariant: '及', strokeCount: 3),
+    RadkfileRadical(formalVariant: '久', strokeCount: 3),
   ],
   4: [
-    '⺹',
-    '心',
-    '戈',
-    '戸',
-    '手',
-    '支',
-    '攵',
-    '文',
-    '斗',
-    '斤',
-    '方',
-    '无',
-    '日',
-    '曰',
-    '月',
-    '木',
-    '欠',
-    '止',
-    '歹',
-    '殳',
-    '比',
-    '毛',
-    '氏',
-    '气',
-    '水',
-    '火',
-    '⺣',
-    '爪',
-    '父',
-    '爻',
-    '爿',
-    '片',
-    '牛',
-    '犬',
-    '⺭',
-    '王',
-    '元',
-    '井',
-    '勿',
-    '尤',
-    '五',
-    '屯',
-    '巴',
-    '毋',
+    RadkfileRadical(formalVariant: '老', informalVariant: '⺹', strokeCount: 4),
+    RadkfileRadical(formalVariant: '心', strokeCount: 4),
+    RadkfileRadical(formalVariant: '戈', strokeCount: 4),
+    RadkfileRadical(formalVariant: '戸', strokeCount: 4),
+    RadkfileRadical(formalVariant: '手', strokeCount: 4),
+    RadkfileRadical(formalVariant: '支', strokeCount: 4),
+    RadkfileRadical(formalVariant: '攵', strokeCount: 4),
+    RadkfileRadical(formalVariant: '文', strokeCount: 4),
+    RadkfileRadical(formalVariant: '斗', strokeCount: 4),
+    RadkfileRadical(formalVariant: '斤', strokeCount: 4),
+    RadkfileRadical(formalVariant: '方', strokeCount: 4),
+    RadkfileRadical(formalVariant: '无', strokeCount: 4),
+    RadkfileRadical(formalVariant: '日', strokeCount: 4),
+    RadkfileRadical(formalVariant: '曰', strokeCount: 4),
+    RadkfileRadical(formalVariant: '月', strokeCount: 4),
+    RadkfileRadical(formalVariant: '木', strokeCount: 4),
+    RadkfileRadical(formalVariant: '欠', strokeCount: 4),
+    RadkfileRadical(formalVariant: '止', strokeCount: 4),
+    RadkfileRadical(formalVariant: '歹', strokeCount: 4),
+    RadkfileRadical(formalVariant: '殳', strokeCount: 4),
+    RadkfileRadical(formalVariant: '比', strokeCount: 4),
+    RadkfileRadical(formalVariant: '毛', strokeCount: 4),
+    RadkfileRadical(formalVariant: '氏', strokeCount: 4),
+    RadkfileRadical(formalVariant: '气', strokeCount: 4),
+    RadkfileRadical(formalVariant: '水', strokeCount: 4),
+    RadkfileRadical(formalVariant: '火', strokeCount: 4),
+    RadkfileRadical(formalVariant: '杰', strokeCount: 4),
+    RadkfileRadical(formalVariant: '爪', strokeCount: 4),
+    RadkfileRadical(formalVariant: '父', strokeCount: 4),
+    RadkfileRadical(formalVariant: '爻', strokeCount: 4),
+    RadkfileRadical(formalVariant: '爿', strokeCount: 4),
+    RadkfileRadical(formalVariant: '片', strokeCount: 4),
+    RadkfileRadical(formalVariant: '牛', strokeCount: 4),
+    RadkfileRadical(formalVariant: '犬', strokeCount: 4),
+    RadkfileRadical(formalVariant: '礼', strokeCount: 4),
+    RadkfileRadical(formalVariant: '王', strokeCount: 4),
+    RadkfileRadical(formalVariant: '元', strokeCount: 4),
+    RadkfileRadical(formalVariant: '井', strokeCount: 4),
+    RadkfileRadical(formalVariant: '勿', strokeCount: 4),
+    RadkfileRadical(formalVariant: '尤', strokeCount: 4),
+    RadkfileRadical(formalVariant: '五', strokeCount: 4),
+    RadkfileRadical(formalVariant: '屯', strokeCount: 4),
+    RadkfileRadical(formalVariant: '巴', strokeCount: 4),
+    RadkfileRadical(formalVariant: '毋', strokeCount: 4),
   ],
   5: [
-    '玄',
-    '瓦',
-    '甘',
-    '生',
-    '用',
-    '田',
-    '疋',
-    '疒',
-    '癶',
-    '白',
-    '皮',
-    '皿',
-    '目',
-    '矛',
-    '矢',
-    '石',
-    '示',
-    '禸',
-    '禾',
-    '穴',
-    '立',
-    '⻂',
-    '世',
-    '巨',
-    '冊',
-    '母',
-    '⺲',
-    '牙',
+    RadkfileRadical(formalVariant: '玄', strokeCount: 5),
+    RadkfileRadical(formalVariant: '瓦', strokeCount: 5),
+    RadkfileRadical(formalVariant: '甘', strokeCount: 5),
+    RadkfileRadical(formalVariant: '生', strokeCount: 5),
+    RadkfileRadical(formalVariant: '用', strokeCount: 5),
+    RadkfileRadical(formalVariant: '田', strokeCount: 5),
+    RadkfileRadical(formalVariant: '疋', strokeCount: 5),
+    RadkfileRadical(formalVariant: '疔', strokeCount: 5),
+    RadkfileRadical(formalVariant: '癶', strokeCount: 5),
+    RadkfileRadical(formalVariant: '白', strokeCount: 5),
+    RadkfileRadical(formalVariant: '皮', strokeCount: 5),
+    RadkfileRadical(formalVariant: '皿', strokeCount: 5),
+    RadkfileRadical(formalVariant: '目', strokeCount: 5),
+    RadkfileRadical(formalVariant: '矛', strokeCount: 5),
+    RadkfileRadical(formalVariant: '矢', strokeCount: 5),
+    RadkfileRadical(formalVariant: '石', strokeCount: 5),
+    RadkfileRadical(formalVariant: '示', strokeCount: 5),
+    RadkfileRadical(formalVariant: '禹', strokeCount: 5),
+    RadkfileRadical(formalVariant: '禾', strokeCount: 5),
+    RadkfileRadical(formalVariant: '穴', strokeCount: 5),
+    RadkfileRadical(formalVariant: '立', strokeCount: 5),
+    RadkfileRadical(formalVariant: '初', strokeCount: 5),
+    RadkfileRadical(formalVariant: '世', strokeCount: 5),
+    RadkfileRadical(formalVariant: '巨', strokeCount: 5),
+    RadkfileRadical(formalVariant: '冊', strokeCount: 5),
+    RadkfileRadical(formalVariant: '母', strokeCount: 5),
+    RadkfileRadical(formalVariant: '買', strokeCount: 5),
+    RadkfileRadical(formalVariant: '牙', strokeCount: 5),
   ],
   6: [
-    '瓜',
-    '竹',
-    '米',
-    '糸',
-    '缶',
-    '羊',
-    '羽',
-    '而',
-    '耒',
-    '耳',
-    '聿',
-    '肉',
-    '自',
-    '至',
-    '臼',
-    '舌',
-    '舟',
-    '艮',
-    '色',
-    '虍',
-    '虫',
-    '血',
-    '行',
-    '衣',
-    '西',
+    RadkfileRadical(formalVariant: '瓜', strokeCount: 6),
+    RadkfileRadical(formalVariant: '竹', strokeCount: 6),
+    RadkfileRadical(formalVariant: '米', strokeCount: 6),
+    RadkfileRadical(formalVariant: '糸', strokeCount: 6),
+    RadkfileRadical(formalVariant: '缶', strokeCount: 6),
+    RadkfileRadical(formalVariant: '羊', strokeCount: 6),
+    RadkfileRadical(formalVariant: '羽', strokeCount: 6),
+    RadkfileRadical(formalVariant: '而', strokeCount: 6),
+    RadkfileRadical(formalVariant: '耒', strokeCount: 6),
+    RadkfileRadical(formalVariant: '耳', strokeCount: 6),
+    RadkfileRadical(formalVariant: '聿', strokeCount: 6),
+    RadkfileRadical(formalVariant: '肉', strokeCount: 6),
+    RadkfileRadical(formalVariant: '自', strokeCount: 6),
+    RadkfileRadical(formalVariant: '至', strokeCount: 6),
+    RadkfileRadical(formalVariant: '臼', strokeCount: 6),
+    RadkfileRadical(formalVariant: '舌', strokeCount: 6),
+    RadkfileRadical(formalVariant: '舟', strokeCount: 6),
+    RadkfileRadical(formalVariant: '艮', strokeCount: 6),
+    RadkfileRadical(formalVariant: '色', strokeCount: 6),
+    RadkfileRadical(formalVariant: '虍', strokeCount: 6),
+    RadkfileRadical(formalVariant: '虫', strokeCount: 6),
+    RadkfileRadical(formalVariant: '血', strokeCount: 6),
+    RadkfileRadical(formalVariant: '行', strokeCount: 6),
+    RadkfileRadical(formalVariant: '衣', strokeCount: 6),
+    RadkfileRadical(formalVariant: '西', strokeCount: 6),
   ],
   7: [
-    '臣',
-    '見',
-    '角',
-    '言',
-    '谷',
-    '豆',
-    '豕',
-    '豸',
-    '貝',
-    '赤',
-    '走',
-    '足',
-    '身',
-    '車',
-    '辛',
-    '辰',
-    '酉',
-    '釆',
-    '里',
-    '舛',
-    '麦',
+    RadkfileRadical(formalVariant: '臣', strokeCount: 7),
+    RadkfileRadical(formalVariant: '見', strokeCount: 7),
+    RadkfileRadical(formalVariant: '角', strokeCount: 7),
+    RadkfileRadical(formalVariant: '言', strokeCount: 7),
+    RadkfileRadical(formalVariant: '谷', strokeCount: 7),
+    RadkfileRadical(formalVariant: '豆', strokeCount: 7),
+    RadkfileRadical(formalVariant: '豕', strokeCount: 7),
+    RadkfileRadical(formalVariant: '豸', strokeCount: 7),
+    RadkfileRadical(formalVariant: '貝', strokeCount: 7),
+    RadkfileRadical(formalVariant: '赤', strokeCount: 7),
+    RadkfileRadical(formalVariant: '走', strokeCount: 7),
+    RadkfileRadical(formalVariant: '足', strokeCount: 7),
+    RadkfileRadical(formalVariant: '身', strokeCount: 7),
+    RadkfileRadical(formalVariant: '車', strokeCount: 7),
+    RadkfileRadical(formalVariant: '辛', strokeCount: 7),
+    RadkfileRadical(formalVariant: '辰', strokeCount: 7),
+    RadkfileRadical(formalVariant: '酉', strokeCount: 7),
+    RadkfileRadical(formalVariant: '釆', strokeCount: 7),
+    RadkfileRadical(formalVariant: '里', strokeCount: 7),
+    RadkfileRadical(formalVariant: '舛', strokeCount: 7),
+    RadkfileRadical(formalVariant: '麦', strokeCount: 7),
   ],
-  8: ['金', '長', '門', '隶', '隹', '雨', '青', '非', '奄', '岡', '免', '斉'],
-  9: ['面', '革', '韭', '音', '頁', '風', '飛', '食', '首', '香', '品'],
-  10: ['馬', '骨', '高', '髟', '鬥', '鬯', '鬲', '鬼', '竜', '韋'],
-  11: ['魚', '鳥', '鹵', '鹿', '麻', '亀', '啇', '黄', '黒'],
-  12: ['黍', '黹', '無', '歯'],
-  13: ['黽', '鼎', '鼓', '鼠'],
-  14: ['鼻', '齊'],
-  17: ['龠'],
+  8: [
+    RadkfileRadical(formalVariant: '金', strokeCount: 8),
+    RadkfileRadical(formalVariant: '長', strokeCount: 8),
+    RadkfileRadical(formalVariant: '門', strokeCount: 8),
+    RadkfileRadical(formalVariant: '隶', strokeCount: 8),
+    RadkfileRadical(formalVariant: '隹', strokeCount: 8),
+    RadkfileRadical(formalVariant: '雨', strokeCount: 8),
+    RadkfileRadical(formalVariant: '青', strokeCount: 8),
+    RadkfileRadical(formalVariant: '非', strokeCount: 8),
+    RadkfileRadical(formalVariant: '奄', strokeCount: 8),
+    RadkfileRadical(formalVariant: '岡', strokeCount: 8),
+    RadkfileRadical(formalVariant: '免', strokeCount: 8),
+    RadkfileRadical(formalVariant: '斉', strokeCount: 8),
+  ],
+  9: [
+    RadkfileRadical(formalVariant: '面', strokeCount: 9),
+    RadkfileRadical(formalVariant: '革', strokeCount: 9),
+    RadkfileRadical(formalVariant: '韭', strokeCount: 9),
+    RadkfileRadical(formalVariant: '音', strokeCount: 9),
+    RadkfileRadical(formalVariant: '頁', strokeCount: 9),
+    RadkfileRadical(formalVariant: '風', strokeCount: 9),
+    RadkfileRadical(formalVariant: '飛', strokeCount: 9),
+    RadkfileRadical(formalVariant: '食', strokeCount: 9),
+    RadkfileRadical(formalVariant: '首', strokeCount: 9),
+    RadkfileRadical(formalVariant: '香', strokeCount: 9),
+    RadkfileRadical(formalVariant: '品', strokeCount: 9),
+  ],
+  10: [
+    RadkfileRadical(formalVariant: '馬', strokeCount: 10),
+    RadkfileRadical(formalVariant: '骨', strokeCount: 10),
+    RadkfileRadical(formalVariant: '高', strokeCount: 10),
+    RadkfileRadical(formalVariant: '髟', strokeCount: 10),
+    RadkfileRadical(formalVariant: '鬥', strokeCount: 10),
+    RadkfileRadical(formalVariant: '鬯', strokeCount: 10),
+    RadkfileRadical(formalVariant: '鬲', strokeCount: 10),
+    RadkfileRadical(formalVariant: '鬼', strokeCount: 10),
+    RadkfileRadical(formalVariant: '竜', strokeCount: 10),
+    RadkfileRadical(formalVariant: '韋', strokeCount: 10),
+  ],
+  11: [
+    RadkfileRadical(formalVariant: '魚', strokeCount: 11),
+    RadkfileRadical(formalVariant: '鳥', strokeCount: 11),
+    RadkfileRadical(formalVariant: '鹵', strokeCount: 11),
+    RadkfileRadical(formalVariant: '鹿', strokeCount: 11),
+    RadkfileRadical(formalVariant: '麻', strokeCount: 11),
+    RadkfileRadical(formalVariant: '亀', strokeCount: 11),
+    RadkfileRadical(formalVariant: '滴', strokeCount: 11),
+    RadkfileRadical(formalVariant: '黄', strokeCount: 11),
+    RadkfileRadical(formalVariant: '黒', strokeCount: 11),
+  ],
+  12: [
+    RadkfileRadical(formalVariant: '黍', strokeCount: 12),
+    RadkfileRadical(formalVariant: '黹', strokeCount: 12),
+    RadkfileRadical(formalVariant: '無', strokeCount: 12),
+    RadkfileRadical(formalVariant: '歯', strokeCount: 12),
+  ],
+  13: [
+    RadkfileRadical(formalVariant: '黽', strokeCount: 13),
+    RadkfileRadical(formalVariant: '鼎', strokeCount: 13),
+    RadkfileRadical(formalVariant: '鼓', strokeCount: 13),
+    RadkfileRadical(formalVariant: '鼠', strokeCount: 13),
+  ],
+  14: [
+    RadkfileRadical(formalVariant: '鼻', strokeCount: 14),
+    RadkfileRadical(formalVariant: '齊', strokeCount: 14),
+  ],
+  17: [RadkfileRadical(formalVariant: '龠', strokeCount: 17)],
+};
+
+final Map<String, RadkfileRadical> radicalsByFormalVariant = {
+  for (final r in radicals.values.flattened) r.formalVariant: r,
 };

lib/models/word_search/word_search_match_span.dart

@@ -33,6 +33,7 @@ class WordSearchMatchSpan {
   Map<String, Object?> toJson() => {
     'spanType': spanType.toString().split('.').last,
     'index': index,
+    'subIndex': subIndex,
     'start': start,
     'end': end,
   };
@@ -43,12 +44,13 @@ class WordSearchMatchSpan {
           (e) => e.toString().split('.').last == json['spanType'],
         ),
         index: json['index'] as int,
+        subIndex: json['subIndex'] as int? ?? 0,
         start: json['start'] as int,
         end: json['end'] as int,
       );
 
   @override
-  int get hashCode => Object.hash(spanType, index, start, end);
+  int get hashCode => Object.hash(spanType, index, subIndex, start, end);
 
   @override
   bool operator ==(Object other) {
@@ -56,6 +58,7 @@ class WordSearchMatchSpan {
     return other is WordSearchMatchSpan &&
         other.spanType == spanType &&
         other.index == index &&
+        other.subIndex == subIndex &&
         other.start == start &&
         other.end == end;
   }

lib/models/word_search/word_search_sense.dart

@@ -81,7 +81,7 @@ class WordSearchSense {
     'dialects': dialects.map((e) => e.toJson()).toList(),
     'misc': misc.map((e) => e.toJson()).toList(),
     'info': info,
-    'languageSource': languageSource,
+    'languageSource': languageSource.map((e) => e.toJson()).toList(),
   };
 
   factory WordSearchSense.fromJson(Map<String, dynamic> json) =>

lib/models/word_search/word_search_xref_entry.dart

@@ -16,6 +16,8 @@ class WordSearchXrefEntry {
   final bool ambiguous;
 
   /// The result of the cross-reference, may or may not be included in the query.
+  ///
+  /// Be careful not to introduce circular references when using this field.
   final WordSearchResult? xrefResult;
 
   const WordSearchXrefEntry({
@@ -40,6 +42,10 @@ class WordSearchXrefEntry {
         ambiguous: json['ambiguous'] as bool,
         baseWord: json['baseWord'] as String,
         furigana: json['furigana'] as String?,
-        xrefResult: null,
+        xrefResult: json['xrefResult'] != null
+            ? WordSearchResult.fromJson(
+                Map<String, dynamic>.from(json['xrefResult'] as Map),
+              )
+            : null,
       );
 }

lib/search.dart

@@ -1,9 +1,11 @@
+import 'package:jadb/const_data/radicals.dart';
 import 'package:jadb/models/kanji_search/kanji_search_result.dart';
 import 'package:jadb/models/verify_tables.dart';
 import 'package:jadb/models/word_search/word_search_result.dart';
 import 'package:jadb/search/filter_kanji.dart';
 import 'package:jadb/search/kanji_search.dart';
 import 'package:jadb/search/radical_search.dart';
+import 'package:jadb/search/versions.dart';
 import 'package:jadb/search/word_search/word_search.dart';
 import 'package:sqflite_common/sqlite_api.dart';
 
@@ -18,12 +20,13 @@ extension JaDBConnection on DatabaseExecutor {
       searchKanjiWithDbConnection(this, kanji);
 
   /// Search for a kanji in the database.
-  Future<Map<String, KanjiSearchResult>> jadbGetManyKanji(Set<String> kanji) =>
-      searchManyKanjiWithDbConnection(this, kanji);
+  Future<Map<String, KanjiSearchResult>> jadbGetManyKanji(
+    Iterable<String> kanji,
+  ) => searchManyKanjiWithDbConnection(this, kanji);
 
   /// Filter a list of characters, and return the ones that are listed in the kanji dictionary.
   Future<List<String>> filterKanji(
-    List<String> kanji, {
+    Iterable<String> kanji, {
     bool deduplicate = false,
   }) => filterKanjiWithDbConnection(this, kanji, deduplicate);
 
@@ -61,11 +64,17 @@ extension JaDBConnection on DatabaseExecutor {
   /// of the radicals, find their other radicals, and return those.
   /// This is used to figure out which remaining combinations of radicals
   /// the user can search for without getting zero results.
-  Future<List<String>> jadbSearchRemainingRadicals(List<String> radicals) =>
-      searchRemainingRadicalsWithDbConnection(this, radicals);
+  Future<List<RadkfileRadical>> jadbSearchRemainingRadicals(
+    List<RadkfileRadical> radicals,
+  ) => searchRemainingRadicalsWithDbConnection(this, radicals);
 
   /// Given a list of radicals, search which kanji contains all
   /// of the radicals, and return those.
-  Future<List<String>> jadbSearchKanjiByRadicals(List<String> radicals) =>
-      searchKanjiByRadicalsWithDbConnection(this, radicals);
+  Future<List<String>> jadbSearchKanjiByRadicals(
+    List<RadkfileRadical> radicals,
+  ) => searchKanjiByRadicalsWithDbConnection(this, radicals);
+
+  /// Retrieve the version information for all datasources in the database.
+  Future<DatasourceVersions> jadbGetDatasourceVersions() =>
+      getDatasourceVersions(this);
 }

lib/search/filter_kanji.dart

@@ -6,7 +6,7 @@ import 'package:sqflite_common/sqflite.dart';
 /// If [deduplicate] is true, the returned list will deduplicate the input kanji list before returning the filtered results.
 Future<List<String>> filterKanjiWithDbConnection(
   DatabaseExecutor connection,
-  List<String> kanji,
+  Iterable<String> kanji,
   bool deduplicate,
 ) async {
   final Set<String> filteredKanji = await connection
@@ -14,7 +14,7 @@ Future<List<String>> filterKanjiWithDbConnection(
       SELECT "literal"
       FROM "${KANJIDICTableNames.character}"
       WHERE "literal" IN (${kanji.map((_) => '?').join(',')})
-    ''', kanji)
+    ''', kanji.toList())
       .then((value) => value.map((e) => e['literal'] as String).toSet());
 
   if (deduplicate) {

lib/search/kanji_search.dart

@@ -8,10 +8,21 @@ import 'package:sqflite_common/sqflite.dart';
 Future<List<Map<String, Object?>>> _charactersQuery(
   DatabaseExecutor connection,
   String kanji,
-) => connection.query(
-  KANJIDICTableNames.character,
-  where: 'literal = ?',
-  whereArgs: [kanji],
+) => connection.rawQuery(
+  '''
+    SELECT
+      "${KANJIDICTableNames.character}"."literal",
+      "${KANJIDICTableNames.character}"."strokeCount",
+      "${KANJIDICTableNames.grade}"."grade",
+      "${KANJIDICTableNames.jlpt}"."jlpt",
+      "${KANJIDICTableNames.frequency}"."frequency"
+    FROM "${KANJIDICTableNames.character}"
+    LEFT JOIN "${KANJIDICTableNames.grade}" ON "${KANJIDICTableNames.character}"."literal" = "${KANJIDICTableNames.grade}"."kanji"
+    LEFT JOIN "${KANJIDICTableNames.jlpt}" ON "${KANJIDICTableNames.character}"."literal" = "${KANJIDICTableNames.jlpt}"."kanji"
+    LEFT JOIN "${KANJIDICTableNames.frequency}" ON "${KANJIDICTableNames.character}"."literal" = "${KANJIDICTableNames.frequency}"."kanji"
+    WHERE "literal" = ?
+  ''',
+  [kanji],
 );
 
 Future<List<Map<String, Object?>>> _codepointsQuery(
@@ -263,7 +274,7 @@ Future<KanjiSearchResult?> searchKanjiWithDbConnection(
 /// Searches for multiple kanji at once, returning a map of kanji to their search results.
 Future<Map<String, KanjiSearchResult>> searchManyKanjiWithDbConnection(
   DatabaseExecutor connection,
-  Set<String> kanji,
+  Iterable<String> kanji,
 ) async {
   if (kanji.isEmpty) {
     return {};

lib/search/radical_search.dart

@@ -1,3 +1,4 @@
+import 'package:jadb/const_data/radicals.dart';
 import 'package:jadb/table_names/radkfile.dart';
 import 'package:sqflite_common/sqlite_api.dart';
 
@@ -7,9 +8,9 @@ import 'package:sqflite_common/sqlite_api.dart';
 ///
 /// This can be used to limit the choices of additional radicals provided to a user,
 /// so that any choice they make will still yield at least one kanji.
-Future<List<String>> searchRemainingRadicalsWithDbConnection(
+Future<List<RadkfileRadical>> searchRemainingRadicalsWithDbConnection(
   DatabaseExecutor connection,
-  List<String> radicals,
+  List<RadkfileRadical> radicals,
 ) async {
   final distinctRadicals = radicals.toSet();
 
@@ -25,11 +26,14 @@ Future<List<String>> searchRemainingRadicalsWithDbConnection(
         HAVING COUNT(DISTINCT "radical") = ?
       )
     ''',
-    [...distinctRadicals, distinctRadicals.length],
+    [...distinctRadicals.map((r) => r.formalVariant), distinctRadicals.length],
   );
 
   final remainingRadicals = queryResult
       .map((row) => row['radical'] as String)
+      // TODO: maybe we should do some runtime checking here, just to throw a sensible error
+      //       if something goes horribly wrong?
+      .map((r) => radicalsByFormalVariant[r]!)
       .toList();
 
   return remainingRadicals;
@@ -38,7 +42,7 @@ Future<List<String>> searchRemainingRadicalsWithDbConnection(
 /// Returns a list of kanji that contain all of the input radicals.
 Future<List<String>> searchKanjiByRadicalsWithDbConnection(
   DatabaseExecutor connection,
-  List<String> radicals,
+  List<RadkfileRadical> radicals,
 ) async {
   final distinctRadicals = radicals.toSet();
 
@@ -50,7 +54,7 @@ Future<List<String>> searchKanjiByRadicalsWithDbConnection(
       GROUP BY "kanji"
       HAVING COUNT(DISTINCT "radical") = ?
     ''',
-    [...distinctRadicals, distinctRadicals.length],
+    [...distinctRadicals.map((r) => r.formalVariant), distinctRadicals.length],
   );
 
   final kanji = queryResult.map((row) => row['kanji'] as String).toList();

lib/search/versions.dart

@@ -0,0 +1,84 @@
+import 'package:jadb/table_names/jmdict.dart';
+import 'package:jadb/table_names/kanjidic.dart';
+import 'package:jadb/table_names/radkfile.dart';
+import 'package:jadb/table_names/tanos_jlpt.dart';
+import 'package:sqflite_common/sqlite_api.dart';
+
+class DatasourceVersions {
+  final String jmdictVersion;
+  final DateTime jmdictDate;
+  final String jmdictHash;
+
+  final String kanjidic2Version;
+  final DateTime kanjidic2Date;
+  final String kanjidic2Hash;
+
+  final String radkfileVersion;
+  final DateTime radkfileDate;
+  final String radkfileHash;
+
+  final String tanosJlptVersion;
+  final DateTime tanosJlptDate;
+  final String tanosJlptHash;
+
+  const DatasourceVersions({
+    required this.jmdictVersion,
+    required this.jmdictDate,
+    required this.jmdictHash,
+    required this.kanjidic2Version,
+    required this.kanjidic2Date,
+    required this.kanjidic2Hash,
+    required this.radkfileVersion,
+    required this.radkfileDate,
+    required this.radkfileHash,
+    required this.tanosJlptVersion,
+    required this.tanosJlptDate,
+    required this.tanosJlptHash,
+  });
+}
+
+DateTime _parseDateTime(String dateString) {
+  try {
+    return DateTime.parse(dateString);
+  } catch (e) {
+    if (RegExp(r'^\d{4}-\d{2}$').hasMatch(dateString)) {
+      return DateTime.parse('$dateString-01');
+    } else if (RegExp(r'^\d{4}$').hasMatch(dateString)) {
+      return DateTime.parse('$dateString-01-01');
+    } else {
+      throw FormatException('Invalid date format: $dateString');
+    }
+  }
+}
+
+Future<DatasourceVersions> getDatasourceVersions(
+  final DatabaseExecutor connection,
+) async {
+  final jmdictVersion = await connection
+      .query(JMdictTableNames.version)
+      .then((rows) => rows.first);
+  final kanjidic2Version = await connection
+      .query(KANJIDICTableNames.version)
+      .then((rows) => rows.first);
+  final radkfileVersion = await connection
+      .query(RADKFILETableNames.version)
+      .then((rows) => rows.first);
+  final tanosJlptVersion = await connection
+      .query(TanosJLPTTableNames.version)
+      .then((rows) => rows.first);
+
+  return DatasourceVersions(
+    jmdictVersion: jmdictVersion['version'] as String,
+    jmdictDate: _parseDateTime(jmdictVersion['date'].toString()),
+    jmdictHash: jmdictVersion['hash'] as String,
+    kanjidic2Version: kanjidic2Version['version'] as String,
+    kanjidic2Date: _parseDateTime(kanjidic2Version['date'].toString()),
+    kanjidic2Hash: kanjidic2Version['hash'] as String,
+    radkfileVersion: radkfileVersion['version'] as String,
+    radkfileDate: _parseDateTime(radkfileVersion['date'].toString()),
+    radkfileHash: radkfileVersion['hash'] as String,
+    tanosJlptVersion: tanosJlptVersion['version'] as String,
+    tanosJlptDate: _parseDateTime(tanosJlptVersion['date'].toString()),
+    tanosJlptHash: tanosJlptVersion['hash'] as String,
+  );
+}

lib/search/word_search/data_query.dart

@@ -69,7 +69,7 @@ Future<List<Map<String, Object?>>> _readingelementsQuery(
   JMdictTableNames.readingElement,
   where: 'entryId IN (${List.filled(entryIds.length, '?').join(',')})',
   whereArgs: entryIds,
-  orderBy: 'orderNum',
+  orderBy: 'elementId',
 );
 
 Future<List<Map<String, Object?>>> _kanjielementsQuery(
@@ -79,7 +79,7 @@ Future<List<Map<String, Object?>>> _kanjielementsQuery(
   JMdictTableNames.kanjiElement,
   where: 'entryId IN (${List.filled(entryIds.length, '?').join(',')})',
   whereArgs: entryIds,
-  orderBy: 'orderNum',
+  orderBy: 'elementId',
 );
 
 Future<List<Map<String, Object?>>> _jlpttagsQuery(
@@ -216,19 +216,43 @@ Future<List<Map<String, Object?>>> _sensepossQuery(
 Future<List<Map<String, Object?>>> _senserestrictedtokanjisQuery(
   DatabaseExecutor connection,
   List<int> senseIds,
-) => connection.query(
-  JMdictTableNames.senseRestrictedToKanji,
-  where: 'senseId IN (${List.filled(senseIds.length, '?').join(',')})',
-  whereArgs: senseIds,
+) => connection.rawQuery(
+  """
+      SELECT
+        "${JMdictTableNames.senseRestrictedToKanji}".senseId,
+        "${JMdictTableNames.senseRestrictedToKanji}".kanjiElementId,
+        "${JMdictTableNames.kanjiElement}".reading
+      FROM "${JMdictTableNames.senseRestrictedToKanji}"
+      JOIN "${JMdictTableNames.kanjiElement}"
+        ON "${JMdictTableNames.senseRestrictedToKanji}"."kanjiElementId" = "${JMdictTableNames.kanjiElement}"."elementId"
+      WHERE
+        "senseId" IN (${List.filled(senseIds.length, '?').join(',')})
+      ORDER BY
+        "${JMdictTableNames.senseRestrictedToKanji}"."senseId",
+        "${JMdictTableNames.senseRestrictedToKanji}"."kanjiElementId"
+  """,
+  [...senseIds],
 );
 
 Future<List<Map<String, Object?>>> _senserestrictedtoreadingsQuery(
   DatabaseExecutor connection,
   List<int> senseIds,
-) => connection.query(
-  JMdictTableNames.senseRestrictedToReading,
-  where: 'senseId IN (${List.filled(senseIds.length, '?').join(',')})',
-  whereArgs: senseIds,
+) => connection.rawQuery(
+  """
+      SELECT
+        "${JMdictTableNames.senseRestrictedToReading}".senseId,
+        "${JMdictTableNames.senseRestrictedToReading}".readingElementId,
+        "${JMdictTableNames.readingElement}".reading
+      FROM "${JMdictTableNames.senseRestrictedToReading}"
+      JOIN "${JMdictTableNames.readingElement}"
+        ON "${JMdictTableNames.senseRestrictedToReading}"."readingElementId" = "${JMdictTableNames.readingElement}"."elementId"
+      WHERE
+        "senseId" IN (${List.filled(senseIds.length, '?').join(',')})
+      ORDER BY
+        "${JMdictTableNames.senseRestrictedToReading}"."senseId",
+        "${JMdictTableNames.senseRestrictedToReading}"."readingElementId"
+  """,
+  [...senseIds],
 );
 
 Future<List<Map<String, Object?>>> _examplesentencesQuery(

lib/search/word_search/entry_id_query.dart

@@ -67,26 +67,29 @@ String _filterFTSSensitiveCharacters(String word) {
             SELECT DISTINCT
               "$tableName"."entryId",
               100
-              + (("${tableName}FTS"."reading" = ?) * 10000)
-              + "JMdict_EntryScore"."score"
+              + (("$tableName"."reading" = ?1) * 100000)
+              + (("${tableName}FTS"."reading" = normalize_jp(?1)) * 10000)
+              + (("$tableName"."reading" LIKE ?1 || '%') * 20)
+              + (("$tableName"."orderNum" = 0) * 20)
+              + COALESCE("JMdict_EntryScore"."score", 0)
               AS "score"
             FROM "${tableName}FTS"
             JOIN "$tableName" USING ("elementId")
-            JOIN "JMdict_EntryScore" USING ("elementId")
-            WHERE "${tableName}FTS"."reading" MATCH ? || '*'
-              AND "JMdict_EntryScore"."type" = '${tableName == JMdictTableNames.kanjiElement ? 'k' : 'r'}'
+            LEFT JOIN "JMdict_EntryScore" USING ("elementId")
+            WHERE "${tableName}FTS"."reading" MATCH normalize_jp(?1) || '*'
           ),
           non_fts_results AS (
             SELECT DISTINCT
               "$tableName"."entryId",
               50
-              + "JMdict_EntryScore"."score"
+              + (("$tableName"."reading" LIKE '%' || ?1 || '%') * 20)
+              + (("$tableName"."orderNum" = 0) * 20)
+              + COALESCE("JMdict_EntryScore"."score", 0)
               AS "score"
             FROM "$tableName"
-            JOIN "JMdict_EntryScore" USING ("elementId")
-            WHERE "reading" LIKE '%' || ? || '%'
+            LEFT JOIN "JMdict_EntryScore" USING ("elementId")
+            WHERE "$tableName"."reading" LIKE '%' || normalize_jp(?1) || '%'
               AND "$tableName"."entryId" NOT IN (SELECT "entryId" FROM "fts_results")
-              AND "JMdict_EntryScore"."type" = '${tableName == JMdictTableNames.kanjiElement ? 'k' : 'r'}'
           )
 
         SELECT ${countOnly ? 'COUNT(DISTINCT "entryId") AS count' : '"entryId", MAX("score") AS "score"'}
@@ -102,8 +105,6 @@ String _filterFTSSensitiveCharacters(String word) {
       '''
         .trim(),
     [
-      _filterFTSSensitiveCharacters(word),
-      _filterFTSSensitiveCharacters(word),
       _filterFTSSensitiveCharacters(word),
       ?pageSize,
       ?offset,
@@ -198,16 +199,16 @@ Future<List<ScoredEntryId>> _queryEnglish(
     '''
     SELECT
       "${JMdictTableNames.sense}"."entryId",
-      MAX("JMdict_EntryScore"."score")
-      + (("${JMdictTableNames.senseGlossary}"."phrase" = ?1 AND "${JMdictTableNames.sense}"."orderNum" = 1) * 50)
-      + (("${JMdictTableNames.senseGlossary}"."phrase" = ?1 AND "${JMdictTableNames.sense}"."orderNum" = 2) * 30)
-      + (("${JMdictTableNames.senseGlossary}"."phrase" = ?1) * 20)
+      COALESCE(MAX("JMdict_EntryScore"."score"), 0)
+      + (("${JMdictTableNames.senseGlossary}"."phrase" = ?1 AND "${JMdictTableNames.sense}"."orderNum" = 0) * 50)
+      + (("${JMdictTableNames.senseGlossary}"."phrase" = ?1 AND "${JMdictTableNames.sense}"."orderNum" = 1) * 30)
+      + (("${JMdictTableNames.senseGlossary}"."phrase" = ?1 AND "${JMdictTableNames.sense}"."orderNum" > 1) * 20)
       as "score"
     FROM "${JMdictTableNames.senseGlossary}"
     JOIN "${JMdictTableNames.sense}" USING ("senseId")
-    JOIN "JMdict_EntryScore" USING ("entryId")
+    LEFT JOIN "JMdict_EntryScore" USING ("entryId")
     WHERE "${JMdictTableNames.senseGlossary}"."phrase" LIKE ?2
-    GROUP BY "JMdict_EntryScore"."entryId"
+    GROUP BY "${JMdictTableNames.sense}"."entryId"
     ORDER BY
       "score" DESC,
       "${JMdictTableNames.sense}"."entryId" ASC
@@ -215,7 +216,7 @@ Future<List<ScoredEntryId>> _queryEnglish(
     ${offset != null ? 'OFFSET ?4' : ''}
   '''
         .trim(),
-    [word, '%${word.replaceAll('%', '')}%', if (pageSize != null) pageSize, if (offset != null) offset],
+    [word, '%${word.replaceAll('%', '')}%', ?pageSize, ?offset],
   );
 
   return result

lib/search/word_search/regrouping.dart

@@ -318,7 +318,7 @@ List<WordSearchSense> _regroupSenses({
           .map((e) => e['reading'] as String)
           .toList(),
       restrictedToKanji: restrictedToKanjis
-          .map((e) => e['kanji'] as String)
+          .map((e) => e['reading'] as String)
           .toList(),
       fields: fields
           .map((e) => JMdictField.fromId(e['field'] as String))

lib/table_names/jmdict.dart

@@ -11,6 +11,7 @@ abstract class JMdictTableNames {
   static const String senseDialect = 'JMdict_SenseDialect';
   static const String senseField = 'JMdict_SenseField';
   static const String senseGlossary = 'JMdict_SenseGlossary';
+  static const String senseGlossaryType = 'JMdict_SenseGlossaryType';
   static const String senseInfo = 'JMdict_SenseInfo';
   static const String senseMisc = 'JMdict_SenseMisc';
   static const String sensePOS = 'JMdict_SensePOS';
@@ -33,6 +34,7 @@ abstract class JMdictTableNames {
     senseDialect,
     senseField,
     senseGlossary,
+    senseGlossaryType,
     senseInfo,
     senseMisc,
     sensePOS,

lib/table_names/kanjidic.dart

@@ -1,6 +1,9 @@
 abstract class KANJIDICTableNames {
   static const String version = 'KANJIDIC_Version';
   static const String character = 'KANJIDIC_Character';
+  static const String grade = 'KANJIDIC_Grade';
+  static const String frequency = 'KANJIDIC_Frequency';
+  static const String jlpt = 'KANJIDIC_JLPT';
   static const String radicalName = 'KANJIDIC_RadicalName';
   static const String codepoint = 'KANJIDIC_Codepoint';
   static const String radical = 'KANJIDIC_Radical';
@@ -20,6 +23,9 @@ abstract class KANJIDICTableNames {
   static Set<String> get allTables => {
     version,
     character,
+    grade,
+    frequency,
+    jlpt,
     radicalName,
     codepoint,
     radical,

lib/version.dart

@@ -0,0 +1 @@
+const int jadbSchemaVersion = 2;

migrations/0001_JMDict.sql

@@ -1,5 +1,3 @@
-SELECT icu_load_collation('ja_JP', 'japanese');
-
 CREATE TABLE "JMdict_Version" (
   "version" VARCHAR(10) PRIMARY KEY NOT NULL,
   "date" DATE NOT NULL,
@@ -55,20 +53,20 @@ CREATE TABLE "JMdict_Entry" (
 
 CREATE TABLE "JMdict_KanjiElement" (
   "elementId" INTEGER PRIMARY KEY,
-  "entryId" INTEGER NOT NULL REFERENCES "JMdict_Entry"("entryId"),
-  "orderNum" INTEGER NOT NULL,
-  "reading" TEXT NOT NULL COLLATE japanese,
+  "entryId" INTEGER NOT NULL GENERATED ALWAYS AS ("elementId" / 100) STORED,
+  "orderNum" INTEGER NOT NULL GENERATED ALWAYS AS ("elementId" % 100) VIRTUAL,
+  "reading" TEXT NOT NULL,
   "news" INTEGER CHECK ("news" BETWEEN 1 AND 2),
   "ichi" INTEGER CHECK ("ichi" BETWEEN 1 AND 2),
   "spec" INTEGER CHECK ("spec" BETWEEN 1 AND 2),
   "gai" INTEGER CHECK ("gai" BETWEEN 1 AND 2),
   "nf" INTEGER CHECK ("nf" BETWEEN 1 AND 48),
-  -- UNIQUE("entryId", "reading"),
-  UNIQUE("entryId", "orderNum")
+  FOREIGN KEY ("entryId") REFERENCES "JMdict_Entry"("entryId"),
+  UNIQUE("entryId", "reading")
 ) WITHOUT ROWID;
 
-CREATE INDEX "JMdict_KanjiElement_byEntryId_byOrderNum" ON "JMdict_KanjiElement"("entryId", "orderNum");
 CREATE INDEX "JMdict_KanjiElement_byReading" ON "JMdict_KanjiElement"("reading");
+CREATE INDEX "JMdict_KanjiElement_byNormalizedReading" ON "JMdict_KanjiElement"(normalize_jp("reading"));
 
 CREATE TABLE "JMdict_KanjiElementInfo" (
   "elementId" INTEGER NOT NULL REFERENCES "JMdict_KanjiElement"("elementId"),
@@ -80,21 +78,21 @@ CREATE TABLE "JMdict_KanjiElementInfo" (
 
 CREATE TABLE "JMdict_ReadingElement" (
   "elementId" INTEGER PRIMARY KEY,
-  "entryId" INTEGER NOT NULL REFERENCES "JMdict_Entry"("entryId"),
-  "orderNum" INTEGER NOT NULL,
-  "reading" TEXT NOT NULL COLLATE japanese,
+  "entryId" INTEGER NOT NULL GENERATED ALWAYS AS (("elementId" / 100) % 10000000) STORED,
+  "orderNum" INTEGER NOT NULL GENERATED ALWAYS AS ("elementId" % 100) VIRTUAL,
+  "reading" TEXT NOT NULL,
   "readingDoesNotMatchKanji" BOOLEAN NOT NULL DEFAULT FALSE,
   "news" INTEGER CHECK ("news" BETWEEN 1 AND 2),
   "ichi" INTEGER CHECK ("ichi" BETWEEN 1 AND 2),
   "spec" INTEGER CHECK ("spec" BETWEEN 1 AND 2),
   "gai" INTEGER CHECK ("gai" BETWEEN 1 AND 2),
   "nf" INTEGER CHECK ("nf" BETWEEN 1 AND 48),
-  -- UNIQUE("entryId", "reading"),
-  UNIQUE("entryId", "orderNum")
+  FOREIGN KEY ("entryId") REFERENCES "JMdict_Entry"("entryId"),
+  UNIQUE("entryId", "reading")
 ) WITHOUT ROWID;
 
-CREATE INDEX "JMdict_ReadingElement_byEntryId_byOrderNum" ON "JMdict_ReadingElement"("entryId", "orderNum");
 CREATE INDEX "JMdict_ReadingElement_byReading" ON "JMdict_ReadingElement"("reading");
+CREATE INDEX "JMdict_ReadingElement_byNormalizedReading" ON "JMdict_ReadingElement"(normalize_jp("reading"));
 
 CREATE TABLE "JMdict_ReadingElementRestriction" (
   "elementId" INTEGER NOT NULL REFERENCES "JMdict_ReadingElement"("elementId"),
@@ -112,27 +110,22 @@ CREATE TABLE "JMdict_ReadingElementInfo" (
 
 CREATE TABLE "JMdict_Sense" (
   "senseId" INTEGER PRIMARY KEY,
-  "entryId" INTEGER NOT NULL REFERENCES "JMdict_Entry"("entryId"),
-  "orderNum" INTEGER NOT NULL,
+  "entryId" INTEGER NOT NULL GENERATED ALWAYS AS ("senseId" / 100) STORED,
+  "orderNum" INTEGER NOT NULL GENERATED ALWAYS AS ("senseId" % 100) VIRTUAL,
+  FOREIGN KEY ("entryId") REFERENCES "JMdict_Entry"("entryId"),
   UNIQUE("entryId", "orderNum")
 );
 
-CREATE INDEX "JMdict_Sense_byEntryId_byOrderNum" ON "JMdict_Sense"("entryId", "orderNum");
-
 CREATE TABLE "JMdict_SenseRestrictedToKanji" (
-  "entryId" INTEGER NOT NULL,
   "senseId" INTEGER NOT NULL REFERENCES "JMdict_Sense"("senseId"),
-  "kanjiOrderNum" INTEGER NOT NULL CHECK ("kanjiOrderNum" > 0),
-  FOREIGN KEY ("entryId", "kanjiOrderNum") REFERENCES "JMdict_KanjiElement"("entryId", "orderNum"),
-  PRIMARY KEY ("entryId", "senseId", "kanjiOrderNum")
+  "kanjiElementId" INTEGER NOT NULL REFERENCES "JMdict_KanjiElement"("elementId"),
+  PRIMARY KEY ("senseId", "kanjiElementId")
 ) WITHOUT ROWID;
 
 CREATE TABLE "JMdict_SenseRestrictedToReading" (
-  "entryId" INTEGER NOT NULL,
   "senseId" INTEGER NOT NULL REFERENCES "JMdict_Sense"("senseId"),
-  "readingOrderNum" INTEGER NOT NULL CHECK ("readingOrderNum" > 0),
-  FOREIGN KEY ("entryId", "readingOrderNum") REFERENCES "JMdict_ReadingElement"("entryId", "orderNum"),
-  PRIMARY KEY ("entryId", "senseId", "readingOrderNum")
+  "readingElementId" INTEGER NOT NULL REFERENCES "JMdict_ReadingElement"("elementId"),
+  PRIMARY KEY ("senseId", "readingElementId")
 ) WITHOUT ROWID;
 
 -- In order to add xrefs, you will need to have added the entry to xref to.
@@ -147,23 +140,42 @@ CREATE TABLE "JMdict_SenseRestrictedToReading" (
 
 CREATE TABLE "JMdict_SenseSeeAlso" (
   "senseId" INTEGER NOT NULL REFERENCES "JMdict_Sense"("senseId"),
-  "xrefEntryId" INTEGER NOT NULL REFERENCES "JMdict_Entry"("entryId"),
-  -- Sometimes the cross reference is to a specific sense
-  "xrefSenseOrderNum" INTEGER,
+  "xrefEntryId" INTEGER NOT NULL,
+  "seeAlsoSense" INTEGER,
   -- For some entries, the cross reference is ambiguous. This means that while the ingestion
   -- has determined some xrefEntryId, it is not guaranteed to be the correct one.
   "ambiguous" BOOLEAN NOT NULL DEFAULT FALSE,
-  FOREIGN KEY ("xrefEntryId", "xrefSenseOrderNum") REFERENCES "JMdict_Sense"("entryId", "orderNum"),
-  UNIQUE("senseId", "xrefEntryId", "xrefSenseOrderNum")
+
+  "seeAlsoSenseKey" INTEGER GENERATED ALWAYS AS (
+    CASE
+      WHEN "seeAlsoSense" IS NOT NULL THEN ("xrefEntryId" * 100) + "seeAlsoSense"
+      ELSE NULL
+    END
+  ) VIRTUAL,
+
+  FOREIGN KEY ("seeAlsoSenseKey") REFERENCES "JMdict_Sense"("senseId"),
+
+  PRIMARY KEY ("senseId", "xrefEntryId", "seeAlsoSense")
 );
 
 CREATE TABLE "JMdict_SenseAntonym" (
   "senseId" INTEGER NOT NULL REFERENCES "JMdict_Sense"("senseId"),
-  "xrefEntryId" INTEGER NOT NULL REFERENCES "JMdict_Entry"("entryId"),
+  "xrefEntryId" INTEGER NOT NULL,
+  "antonymSense" INTEGER,
   -- For some entries, the cross reference is ambiguous. This means that while the ingestion
   -- has determined some xrefEntryId, it is not guaranteed to be the correct one.
   "ambiguous" BOOLEAN NOT NULL DEFAULT FALSE,
-  UNIQUE("senseId", "xrefEntryId")
+
+  "antonymSenseKey" INTEGER GENERATED ALWAYS AS (
+    CASE
+      WHEN "antonymSense" IS NOT NULL THEN ("xrefEntryId" * 100) + "antonymSense"
+      ELSE NULL
+    END
+  ) VIRTUAL,
+
+  FOREIGN KEY ("antonymSenseKey") REFERENCES "JMdict_Sense"("senseId"),
+
+  PRIMARY KEY ("senseId", "xrefEntryId", "antonymSense")
 );
 
 -- These cross references are going to be mostly accessed from a sense
@@ -214,13 +226,21 @@ CREATE TABLE "JMdict_SenseDialect" (
 CREATE TABLE "JMdict_SenseGlossary" (
   "senseId" INTEGER NOT NULL REFERENCES "JMdict_Sense"("senseId"),
   "phrase" TEXT NOT NULL,
-  "language" CHAR(3) NOT NULL DEFAULT "eng",
-  "type" TEXT,
-  PRIMARY KEY ("senseId", "language", "phrase")
+  -- "language" CHAR(3) NOT NULL DEFAULT "eng",
+  -- PRIMARY KEY ("senseId", "language", "phrase")
+  PRIMARY KEY ("senseId", "phrase")
 ) WITHOUT ROWID;
 
 CREATE INDEX "JMdict_SenseGlossary_byPhrase" ON JMdict_SenseGlossary("phrase");
 
+CREATE TABLE "JMdict_SenseGlossaryType" (
+  "senseId" INTEGER NOT NULL REFERENCES "JMdict_Sense"("senseId"),
+  "phrase" TEXT NOT NULL,
+  "type" TEXT NOT NULL,
+  PRIMARY KEY ("senseId", "phrase", "type"),
+  FOREIGN KEY ("senseId", "phrase") REFERENCES "JMdict_SenseGlossary"("senseId", "phrase")
+) WITHOUT ROWID;
+
 CREATE TABLE "JMdict_SenseInfo" (
   "senseId" INTEGER NOT NULL REFERENCES "JMdict_Sense"("senseId"),
   "info" TEXT NOT NULL,

migrations/0003_JMDict_FTS5_Ttables.sql

@@ -4,7 +4,7 @@ CREATE TRIGGER "JMdict_KanjiElement_InsertFTS"
 AFTER INSERT ON "JMdict_KanjiElement"
 BEGIN
   INSERT INTO "JMdict_KanjiElementFTS"("elementId", "reading")
-  VALUES (NEW."elementId", NEW."reading");
+  VALUES (NEW."elementId", normalize_jp(NEW."reading"));
 END;
 
 CREATE TRIGGER "JMdict_KanjiElement_UpdateFTS"
@@ -14,7 +14,7 @@ BEGIN
   UPDATE "JMdict_KanjiElementFTS"
   SET
     "elementId" = NEW."elementId",
-    "reading" = NEW."reading"
+    "reading" = normalize_jp(NEW."reading")
   WHERE "elementId" = OLD."elementId";
 END;
 
@@ -33,7 +33,7 @@ CREATE TRIGGER "JMdict_ReadingElement_InsertFTS"
 AFTER INSERT ON "JMdict_ReadingElement"
 BEGIN
   INSERT INTO "JMdict_ReadingElementFTS"("elementId", "reading")
-  VALUES (NEW."elementId", NEW."reading");
+  VALUES (NEW."elementId", normalize_jp(NEW."reading"));
 END;
 
 CREATE TRIGGER "JMdict_ReadingElement_UpdateFTS"
@@ -43,7 +43,7 @@ BEGIN
   UPDATE "JMdict_ReadingElementFTS"
   SET
     "elementId" = NEW."elementId",
-    "reading" = NEW."reading"
+    "reading" = normalize_jp(NEW."reading")
   WHERE "elementId" = OLD."elementId";
 END;
 

migrations/0005_JMDict_search_index_tables.sql

@@ -1,26 +1,27 @@
 CREATE TABLE "JMdict_EntryScore" (
-  "type" CHAR(1) NOT NULL CHECK ("type" IN ('r', 'k')),
-  "entryId" INTEGER NOT NULL REFERENCES "JMdict_Entry"("entryId"),
-  "elementId" INTEGER NOT NULL,
-  "score" INTEGER NOT NULL DEFAULT 0,
+  "elementId" INTEGER PRIMARY KEY,
+  "score" INTEGER NOT NULL,
   "common" BOOLEAN NOT NULL DEFAULT FALSE,
-  PRIMARY KEY ("type", "elementId")
+
+  "entryId" INTEGER NOT NULL GENERATED ALWAYS AS (("elementId" / 100) % 10000000) STORED,
+  "type" CHAR(1) NOT NULL GENERATED ALWAYS AS (CASE
+    WHEN "elementId" / 1000000000 = 0 THEN 'k'
+    ELSE 'r'
+  END) VIRTUAL,
+
+  FOREIGN KEY ("entryId") REFERENCES "JMdict_Entry"("entryId")
 ) WITHOUT ROWID;
 
 CREATE INDEX "JMdict_EntryScore_byElementId_byScore" ON "JMdict_EntryScore"("elementId", "score");
 CREATE INDEX "JMdict_EntryScore_byScore" ON "JMdict_EntryScore"("score");
-CREATE INDEX "JMdict_EntryScore_byCommon" ON "JMdict_EntryScore"("common");
 
-CREATE INDEX "JMdict_EntryScore_byType_byElementId_byScore" ON "JMdict_EntryScore"("type", "elementId", "score");
-CREATE INDEX "JMdict_EntryScore_byType_byScore" ON "JMdict_EntryScore"("type", "score");
-CREATE INDEX "JMdict_EntryScore_byType_byCommon" ON "JMdict_EntryScore"("type", "common");
+CREATE INDEX "JMdict_EntryScore_byCommon" ON "JMdict_EntryScore"("common") WHERE "common" = TRUE;
 
 -- NOTE: these views are deduplicated in order not to perform an unnecessary
 --       UNION on every trigger
 
 CREATE VIEW "JMdict_EntryScoreView_Reading" AS
 SELECT
-  'r' AS "type",
   "JMdict_ReadingElement"."entryId",
   "JMdict_ReadingElement"."elementId",
   (
@@ -44,7 +45,7 @@ SELECT
   + (("spec" IS 2) * 5)
   + (("gai" IS 1) * 10)
   + (("gai" IS 2) * 5)
-  + (("orderNum" IS 1) * 20)
+  -- + (("orderNum" IS 0) * 20)
   - (substr(COALESCE("JMdict_JLPTTag"."jlptLevel", 'N0'), 2) * -5)
   AS "score"
 FROM "JMdict_ReadingElement"
@@ -52,7 +53,6 @@ LEFT JOIN "JMdict_JLPTTag" USING ("entryId");
 
 CREATE VIEW "JMdict_EntryScoreView_Kanji" AS
 SELECT
-  'k' AS "type",
   "JMdict_KanjiElement"."entryId",
   "JMdict_KanjiElement"."elementId",
   (
@@ -76,7 +76,7 @@ SELECT
   + (("spec" IS 2) * 5)
   + (("gai" IS 1) * 10)
   + (("gai" IS 2) * 5)
-  + (("orderNum" IS 1) * 20)
+  -- + (("orderNum" IS 0) * 20)
   - (substr(COALESCE("JMdict_JLPTTag"."jlptLevel", 'N0'), 2) * -5)
   AS "score"
 FROM "JMdict_KanjiElement"
@@ -96,19 +96,18 @@ CREATE TRIGGER "JMdict_EntryScore_Insert_JMdict_ReadingElement"
 AFTER INSERT ON "JMdict_ReadingElement"
 BEGIN
   INSERT INTO "JMdict_EntryScore" (
-    "type",
-    "entryId",
     "elementId",
     "score",
     "common"
   )
-  SELECT "type", "entryId", "elementId", "score", "common"
+  SELECT "elementId", "score", "common"
   FROM "JMdict_EntryScoreView_Reading"
-  WHERE "elementId" = NEW."elementId";
+  WHERE "elementId" = NEW."elementId"
+    AND "score" > 0;
 END;
 
 CREATE TRIGGER "JMdict_EntryScore_Update_JMdict_ReadingElement"
-AFTER UPDATE OF "news", "ichi", "spec", "gai", "nf", "orderNum"
+AFTER UPDATE OF "news", "ichi", "spec", "gai", "nf", "elementId"
 ON "JMdict_ReadingElement"
 BEGIN
   UPDATE "JMdict_EntryScore"
@@ -117,14 +116,17 @@ BEGIN
     "common" = "JMdict_EntryScoreView_Reading"."common"
   FROM "JMdict_EntryScoreView_Reading"
   WHERE "elementId" = NEW."elementId";
+
+  DELETE FROM "JMdict_EntryScore"
+  WHERE "elementId" = NEW."elementId"
+    AND "score" <= 0;
 END;
 
 CREATE TRIGGER "JMdict_EntryScore_Delete_JMdict_ReadingElement"
 AFTER DELETE ON "JMdict_ReadingElement"
 BEGIN
   DELETE FROM "JMdict_EntryScore"
-  WHERE "type" = 'r'
-    AND "elementId" = OLD."elementId";
+  WHERE "elementId" = OLD."elementId";
 END;
 
 --- JMdict_KanjiElement triggers
@@ -133,19 +135,18 @@ CREATE TRIGGER "JMdict_EntryScore_Insert_JMdict_KanjiElement"
 AFTER INSERT ON "JMdict_KanjiElement"
 BEGIN
   INSERT INTO "JMdict_EntryScore" (
-    "type",
-    "entryId",
     "elementId",
     "score",
     "common"
   )
-  SELECT "type", "entryId", "elementId", "score", "common"
+  SELECT "elementId", "score", "common"
   FROM "JMdict_EntryScoreView_Kanji"
-  WHERE "elementId" = NEW."elementId";
+  WHERE "elementId" = NEW."elementId"
+    AND "score" > 0;
 END;
 
 CREATE TRIGGER "JMdict_EntryScore_Update_JMdict_KanjiElement"
-AFTER UPDATE OF "news", "ichi", "spec", "gai", "nf", "orderNum"
+AFTER UPDATE OF "news", "ichi", "spec", "gai", "nf", "elementId"
 ON "JMdict_KanjiElement"
 BEGIN
   UPDATE "JMdict_EntryScore"
@@ -154,14 +155,17 @@ BEGIN
     "common" = "JMdict_EntryScoreView_Kanji"."common"
   FROM "JMdict_EntryScoreView_Kanji"
   WHERE "elementId" = NEW."elementId";
+
+  DELETE FROM "JMdict_EntryScore"
+  WHERE "elementId" = NEW."elementId"
+    AND "score" <= 0;
 END;
 
 CREATE TRIGGER "JMdict_EntryScore_Delete_JMdict_KanjiElement"
 AFTER DELETE ON "JMdict_KanjiElement"
 BEGIN
   DELETE FROM "JMdict_EntryScore"
-  WHERE "type" = 'k'
-    AND "elementId" = OLD."elementId";
+  WHERE "elementId" = OLD."elementId";
 END;
 
 --- JMdict_JLPTTag triggers
@@ -204,4 +208,8 @@ BEGIN
   WHERE "JMdict_EntryScoreView"."entryId" = OLD."entryId"
     AND "JMdict_EntryScore"."entryId" = OLD."entryId"
     AND "JMdict_EntryScoreView"."elementId" = "JMdict_EntryScore"."elementId";
+
+  DELETE FROM "JMdict_EntryScore"
+  WHERE "elementId" = OLD."elementId"
+    AND "score" <= 0;
 END;

migrations/0006_RADKFILE.sql

@@ -18,4 +18,3 @@ CREATE TABLE "RADKFILE" (
 ) WITHOUT ROWID;
 
 CREATE INDEX "RADK" ON "RADKFILE"("radical");
-CREATE INDEX "KRAD" ON "RADKFILE"("kanji");

migrations/0007_KANJIDIC2.sql

@@ -19,6 +19,21 @@ CREATE TABLE "KANJIDIC_Character" (
   "jlpt" INTEGER
 ) WITHOUT ROWID;
 
+CREATE TABLE "KANJIDIC_Grade" (
+  "kanji" CHAR(1) NOT NULL PRIMARY KEY REFERENCES "KANJIDIC_Character"("literal"),
+  "grade" INTEGER NOT NULL CHECK ("grade" BETWEEN 1 AND 10)
+) WITHOUT ROWID;
+
+CREATE TABLE "KANJIDIC_Frequency" (
+  "kanji" CHAR(1) NOT NULL PRIMARY KEY REFERENCES "KANJIDIC_Character"("literal"),
+  "frequency" INTEGER NOT NULL
+) WITHOUT ROWID;
+
+CREATE TABLE "KANJIDIC_JLPT" (
+  "kanji" CHAR(1) NOT NULL PRIMARY KEY REFERENCES "KANJIDIC_Character"("literal"),
+  "jlpt" INTEGER NOT NULL CHECK ("jlpt" BETWEEN 1 AND 5)
+) WITHOUT ROWID;
+
 CREATE TABLE "KANJIDIC_Codepoint" (
   "kanji" CHAR(1) NOT NULL REFERENCES "KANJIDIC_Character"("literal"),
   "type" VARCHAR(6) NOT NULL CHECK ("type" IN ('jis208', 'jis212', 'jis213', 'ucs')),

migrations/0008_XREF__KANJIDIC_Radical___RADKFILE.sql

@@ -4,7 +4,6 @@ CREATE TABLE "XREF__KANJIDIC_Radical__RADKFILE"(
   PRIMARY KEY ("radicalId", "radicalSymbol")
 ) WITHOUT ROWID;
 
-CREATE INDEX "XREF__KANJIDIC_Radical__RADKFILE__byRadicalId" ON "XREF__KANJIDIC_Radical__RADKFILE"("radicalId");
 CREATE INDEX "XREF__KANJIDIC_Radical__RADKFILE__byRadicalSymbol" ON "XREF__KANJIDIC_Radical__RADKFILE"("radicalSymbol");
 
 /* Source: https://ctext.org/kangxi-zidian */

migrations/0009_XREF__JMdict_KanjiElement__KANJIDIC_Character.sql

@@ -1,10 +1,10 @@
-CREATE TABLE "XREF__JMdict_KanjiElement__KANJIDIC_Character"(
-  "entryId" INTEGER NOT NULL,
-  "reading" TEXT NOT NULL,
-  "kanji" CHAR(1) NOT NULL REFERENCES "KANJIDIC_Character"("literal"),
-  PRIMARY KEY ("entryId", "reading", "kanji"),
-  FOREIGN KEY ("entryId", "reading") REFERENCES "JMdict_KanjiElement"("entryId", "reading")
-) WITHOUT ROWID;
+-- CREATE TABLE "XREF__JMdict_KanjiElement__KANJIDIC_Character"(
+--   "entryId" INTEGER NOT NULL,
+--   "reading" TEXT NOT NULL,
+--   "kanji" CHAR(1) NOT NULL REFERENCES "KANJIDIC_Character"("literal"),
+--   PRIMARY KEY ("entryId", "reading", "kanji"),
+--   FOREIGN KEY ("entryId", "reading") REFERENCES "JMdict_KanjiElement"("entryId", "reading")
+-- ) WITHOUT ROWID;
 
-CREATE INDEX "XREF__JMdict_KanjiElement__KANJIDIC_Character__byEntryId_byReading" ON "XREF__JMdict_KanjiElement__KANJIDIC_Character"("entryId", "reading");
-CREATE INDEX "XREF__JMdict_KanjiElement__KANJIDIC_Character__byKanji" ON "XREF__JMdict_KanjiElement__KANJIDIC_Character"("kanji");
+-- CREATE INDEX "XREF__JMdict_KanjiElement__KANJIDIC_Character__byEntryId_byReading" ON "XREF__JMdict_KanjiElement__KANJIDIC_Character"("entryId", "reading");
+-- CREATE INDEX "XREF__JMdict_KanjiElement__KANJIDIC_Character__byKanji" ON "XREF__JMdict_KanjiElement__KANJIDIC_Character"("kanji");

migrations/0010_Views.sql

@@ -32,9 +32,9 @@ SELECT
     THEN "JMdict_ReadingElement"."reading"
     ELSE NULL
   END AS "furigana",
-  COALESCE("JMdict_KanjiElement"."orderNum", 1)
+  COALESCE("JMdict_KanjiElement"."orderNum", 0)
       + "JMdict_ReadingElement"."orderNum"
-      = 2
+      = 0
   AS "isFirst",
   "JMdict_KanjiElement"."orderNum" AS "kanjiOrderNum",
   "JMdict_ReadingElement"."orderNum" AS "readingOrderNum"
@@ -65,9 +65,7 @@ JOIN "JMdict_KanjiElement"
   ON "JMdict_KanjiElementFTS"."entryId" = "JMdict_KanjiElement"."entryId"
     AND "JMdict_KanjiElementFTS"."reading" LIKE '%' || "JMdict_KanjiElement"."reading"
 JOIN "JMdict_EntryScore"
-  ON "JMdict_EntryScore"."type" = 'k'
-    AND "JMdict_KanjiElement"."entryId" = "JMdict_EntryScore"."entryId"
-    AND "JMdict_KanjiElement"."reading" = "JMdict_EntryScore"."reading"
+  ON "JMdict_EntryScore"."elementId" = "JMdict_KanjiElement"."elementId"
 WHERE "JMdict_EntryScore"."common" = 1;
 
 
@@ -77,13 +75,7 @@ SELECT DISTINCT "radical" FROM "RADKFILE";
 CREATE VIEW "JMdict_CombinedEntryScore"
 AS
 SELECT
-  CASE
-    WHEN "JMdict_EntryScore"."type" = 'k'
-      THEN (SELECT entryId FROM "JMdict_KanjiElement" WHERE "elementId" = "JMdict_EntryScore"."elementId")
-    WHEN "JMdict_EntryScore"."type" = 'r'
-      THEN (SELECT entryId FROM "JMdict_ReadingElement" WHERE "elementId" = "JMdict_EntryScore"."elementId")
-  END AS "entryId",
   MAX("JMdict_EntryScore"."score") AS "score",
   MAX("JMdict_EntryScore"."common") AS "common"
 FROM "JMdict_EntryScore"
-GROUP BY "entryId";
+GROUP BY "JMdict_EntryScore"."entryId";

nix/database.nix

@@ -6,6 +6,7 @@
   jmdict,
   radkfile,
   kanjidic2,
+  tanos-jlpt,
   sqlite,
   wal ? false,
 }:
@@ -18,13 +19,32 @@ stdenvNoCC.mkDerivation {
     sqlite
   ];
 
+  env = {
+    JMDICT_VERSION = jmdict.version;
+    JMDICT_DATE = jmdict.date;
+    JMDICT_HASH = jmdict.hash;
+
+    KANJIDIC_VERSION = kanjidic2.version;
+    KANJIDIC_DATE = kanjidic2.date;
+    KANJIDIC_HASH = kanjidic2.hash;
+
+    RADKFILE_VERSION = radkfile.version;
+    RADKFILE_DATE = radkfile.date;
+    RADKFILE_HASH = radkfile.hash;
+
+    TANOS_JLPT_VERSION = tanos-jlpt.version;
+    TANOS_JLPT_DATE = tanos-jlpt.date;
+    TANOS_JLPT_HASH = tanos-jlpt.hash;
+  };
+
   buildPhase = ''
     runHook preBuild
 
-    mkdir -p data/tmp
-    ln -s "${jmdict}"/* data/tmp
-    ln -s "${radkfile}"/* data/tmp
-    ln -s "${kanjidic2}"/* data/tmp
+    mkdir -p data
+    ln -s '${jmdict}'/* data/
+    ln -s '${kanjidic2}'/* data/
+    ln -s '${radkfile}'/* data/
+    ln -s '${tanos-jlpt}' data/tanos-jlpt
 
     for migration in migrations/*.sql; do
       sqlite3 jadb.sqlite < "$migration"

nix/database_tool.nix

@@ -2,6 +2,9 @@
   src,
   buildDartApplication,
   sqlite,
+
+  callPackage,
+  path,
 }:
 buildDartApplication {
   pname = "jadb-database-tool";
@@ -10,9 +13,6 @@ buildDartApplication {
 
   dartEntryPoints."bin/jadb" = "bin/jadb.dart";
 
-  # NOTE: here we are overriding the implicitly added runtimeDependency from the package fixup in pub2nix.
-  runtimeDependencies = [ sqlite ];
-
   # NOTE: the default dart hooks are using `dart compile`, which is not able to call the
   #       new dart build hooks required to use package:sqlite3 >= 3.0.0. So we override
   #       these phases to use `dart build` instead.
@@ -36,5 +36,9 @@ buildDartApplication {
 
   autoPubspecLock = ../pubspec.lock;
 
+  customSourceBuilders.sqlite3 = callPackage "${path}/pkgs/development/compilers/dart/package-source-builders/sqlite3/default.nix" {
+    inherit sqlite;
+  };
+
   meta.mainProgram = "jadb";
 }

nix/fts5-icu-tokenizer/0001-provide-sqlite-externally.patch

@@ -1,20 +0,0 @@
-diff --git i/CMakeLists.txt w/CMakeLists.txt
-index 9d99543..11ce4a4 100644
---- i/CMakeLists.txt
-+++ w/CMakeLists.txt
-@@ -131,6 +131,15 @@ if(NOT SQLite3_FOUND)
-   endif()
- endif()
- 
-+if(SQLite3_FOUND AND NOT TARGET SQLite3::SQLite3)
-+  add_library(SQLite3::SQLite3 UNKNOWN IMPORTED)
-+
-+  set_target_properties(SQLite3::SQLite3 PROPERTIES
-+    IMPORTED_LOCATION "${SQLite3_LIBRARIES}"
-+    INTERFACE_INCLUDE_DIRECTORIES "${SQLite3_INCLUDE_DIRS}"
-+  )
-+endif()
-+
- # --- Configure the Library ---
- 
- # Select source files based on API version

nix/fts5-icu-tokenizer/package.nix

@@ -1,40 +0,0 @@
-{
-  lib,
-  stdenv,
-  fetchFromSourcehut,
-  cmake,
-  pkg-config,
-  icu,
-  sqlite,
-}:
-
-stdenv.mkDerivation (finalAttrs: {
-  pname = "fts5-icu-tokenizer";
-  version = "5.5";
-  src = fetchFromSourcehut {
-    vc = "hg";
-    owner = "~cwt";
-    repo = "fts5-icu-tokenizer";
-    rev = "v${finalAttrs.version}";
-    hash = "sha256-7Klsu9d1sY+W0buo6kwYdCyDA/u2dBTgu6WuttomTBo=";
-  };
-
-  patches = [
-    ./0001-provide-sqlite-externally.patch
-  ];
-
-  nativeBuildInputs = [
-    cmake
-    pkg-config
-  ];
-
-  buildInputs = [
-    icu
-    sqlite
-  ];
-
-  cmakeFlags = [
-    (lib.cmakeFeature "LOCALE" "ja")
-    (lib.cmakeFeature "API_VERSION" "v2")
-  ];
-})

nix/jmdict.nix

@@ -1,46 +0,0 @@
-{
-  stdenvNoCC,
-  jmdict-src,
-  jmdict-with-examples-src,
-  xmlformat,
-  gzip,
-  edrdgMetadata,
-}:
-stdenvNoCC.mkDerivation {
-  name = "jmdict";
-
-  dontUnpack = true;
-  srcs = [
-    jmdict-src
-    jmdict-with-examples-src
-  ];
-
-  nativeBuildInputs = [
-    gzip
-    xmlformat
-  ];
-
-  buildPhase = ''
-    runHook preBuild
-
-    gzip -dkc "${jmdict-src}" > JMdict.xml
-    gzip -dkc "${jmdict-with-examples-src}" > JMdict_with_examples.xml
-    xmlformat -i JMdict.xml
-    xmlformat -i JMdict_with_examples.xml
-
-    runHook postBuild
-  '';
-
-  installPhase = ''
-    runHook preInstall
-
-    install -Dt "$out" JMdict.xml JMdict_with_examples.xml
-
-    runHook postInstall
-  '';
-
-  meta = edrdgMetadata // {
-    description = "A Japanese-Multilingual Dictionary providing lexical data for japanese words";
-    homepage = "https://www.edrdg.org/jmdict/j_jmdict.html";
-  };
-}

nix/kanjidic2.nix

@@ -1,40 +0,0 @@
-{
-  stdenvNoCC,
-  kanjidic2-src,
-  xmlformat,
-  gzip,
-  edrdgMetadata,
-}:
-stdenvNoCC.mkDerivation {
-  name = "kanjidic2";
-
-  src = kanjidic2-src;
-  dontUnpack = true;
-
-  nativeBuildInputs = [
-    gzip
-    xmlformat
-  ];
-
-  buildPhase = ''
-    runHook preBuild
-
-    gzip -dkc "${kanjidic2-src}" > kanjidic2.xml
-    xmlformat -i kanjidic2.xml
-
-    runHook postBuild
-  '';
-
-  installPhase = ''
-    runHook preInstall
-
-    install -Dt "$out" kanjidic2.xml
-
-    runHook postInstall
-  '';
-
-  meta = edrdgMetadata // {
-    description = "A consolidated XML-format kanji database";
-    homepage = "https://www.edrdg.org/kanjidic/kanjd2index_legacy.html";
-  };
-}

nix/radkfile.nix

@@ -1,40 +0,0 @@
-{
-  stdenv,
-  radkfile-src,
-  gzip,
-  iconv,
-  edrdgMetadata,
-}:
-stdenv.mkDerivation {
-  name = "radkfile";
-
-  src = radkfile-src;
-  dontUnpack = true;
-
-  nativeBuildInputs = [
-    gzip
-    iconv
-  ];
-
-  buildPhase = ''
-    runHook preBuild
-
-    gzip -dkc "$src" > radkfile
-    iconv -f EUC-JP -t UTF-8 -o radkfile_utf8 radkfile
-
-    runHook postBuild
-  '';
-
-  installPhase = ''
-    runHook preInstall
-
-    install -Dt "$out" radkfile_utf8
-
-    runHook postInstall
-  '';
-
-  meta = edrdgMetadata // {
-    description = "A file providing searchable decompositions of kanji characters";
-    homepage = "https://www.edrdg.org/krad/kradinf.html";
-  };
-}

nix/sqlite.nix

@@ -1,17 +0,0 @@
-{
-  lib,
-  sqlite,
-  libiconv,
-  icu,
-}:
-
-sqlite.overrideAttrs (oldAttrs: {
-  name = "sqlite-icu-ext";
-
-  buildInputs = oldAttrs.buildInputs ++ [ icu ];
-
-  configureFlags = oldAttrs.configureFlags ++ [
-    "--with-icu-config=${lib.getExe' icu.dev "icu-config"}"
-    "--enable-icu-collations"
-  ];
-})

pubspec.lock

@@ -5,18 +5,18 @@ packages:
     dependency: transitive
     description:
       name: _fe_analyzer_shared
-      sha256: "8d718c5c58904f9937290fd5dbf2d6a0e02456867706bfb6cd7b81d394e738d5"
+      sha256: cd6add6f846f35fb79f3c315296703c1a24f3cfd7f4739d91a74961c1c7e9f1b
       url: "https://pub.dev"
     source: hosted
-    version: "98.0.0"
+    version: "100.0.0"
   analyzer:
     dependency: transitive
     description:
       name: analyzer
-      sha256: "6141ad5d092d1e1d13929c0504658bbeccc1703505830d7c26e859908f5efc88"
+      sha256: "6ba98576948803398b69e3a444df24eacdbe12ed699c7014e120ea38552debbf"
       url: "https://pub.dev"
     source: hosted
-    version: "12.0.0"
+    version: "13.0.0"
   args:
     dependency: "direct main"
     description:
@@ -61,10 +61,10 @@ packages:
     dependency: transitive
     description:
       name: code_assets
-      sha256: "83ccdaa064c980b5596c35dd64a8d3ecc68620174ab9b90b6343b753aa721687"
+      sha256: bf394f466ba9205f1812a0433b392d6af280f155f56651eda7c18cc32ed493b8
       url: "https://pub.dev"
     source: hosted
-    version: "1.0.0"
+    version: "1.2.1"
   collection:
     dependency: "direct main"
     description:
@@ -149,10 +149,10 @@ packages:
     dependency: transitive
     description:
       name: hooks
-      sha256: e79ed1e8e1929bc6ecb6ec85f0cb519c887aa5b423705ded0d0f2d9226def388
+      sha256: "62ae9bb76d02526c7c2110a19b6e6ad788fe28d35e553e35efb02a41a46ab43a"
       url: "https://pub.dev"
     source: hosted
-    version: "1.0.2"
+    version: "2.0.1"
   http_multi_server:
     dependency: transitive
     description:
@@ -197,10 +197,10 @@ packages:
     dependency: transitive
     description:
       name: matcher
-      sha256: dc0b7dc7651697ea4ff3e69ef44b0407ea32c487a39fff6a4004fa585e901861
+      sha256: "31bd099b47c10cd1aeb55146a2d46ce0277630ecef3f7dae54ad7873f36696cd"
       url: "https://pub.dev"
     source: hosted
-    version: "0.12.19"
+    version: "0.12.20"
   meta:
     dependency: transitive
     description:
@@ -221,10 +221,10 @@ packages:
     dependency: transitive
     description:
       name: native_toolchain_c
-      sha256: "6ba77bb18063eebe9de401f5e6437e95e1438af0a87a3a39084fbd37c90df572"
+      sha256: f59351d28f49520cd3a74eb1f41c5f19ae15e53c65a3231d14af672e46510a96
       url: "https://pub.dev"
     source: hosted
-    version: "0.17.6"
+    version: "0.19.1"
   node_preamble:
     dependency: transitive
     description:
@@ -273,6 +273,14 @@ packages:
       url: "https://pub.dev"
     source: hosted
     version: "2.2.0"
+  record_use:
+    dependency: transitive
+    description:
+      name: record_use
+      sha256: "2551bd8eecfe95d14ae75f6021ad0248be5c27f138c2ec12fcb52b500b3ba1ed"
+      url: "https://pub.dev"
+    source: hosted
+    version: "0.6.0"
   shelf:
     dependency: transitive
     description:
@@ -333,26 +341,26 @@ packages:
     dependency: "direct main"
     description:
       name: sqflite_common
-      sha256: "6ef422a4525ecc601db6c0a2233ff448c731307906e92cabc9ba292afaae16a6"
+      sha256: "1581ffbf7a0e333b380d6a30737d78516b826cb35beb7fb0bf8a3ea0c678b465"
       url: "https://pub.dev"
     source: hosted
-    version: "2.5.6"
+    version: "2.5.8"
   sqflite_common_ffi:
     dependency: "direct main"
     description:
       name: sqflite_common_ffi
-      sha256: c59fcdc143839a77581f7a7c4de018e53682408903a0a0800b95ef2dc4033eff
+      sha256: cd0c7f7de39a08f2d54ef144d9058c46eca8461879aaa648025643455c1e5a20
       url: "https://pub.dev"
     source: hosted
-    version: "2.4.0+2"
+    version: "2.4.0+3"
   sqlite3:
     dependency: "direct main"
     description:
       name: sqlite3
-      sha256: caa693ad15a587a2b4fde093b728131a1827903872171089dedb16f7665d3a91
+      sha256: "9488c7d2cdb1091c91cacf7e207cff81b28bff8e366f042bad3afe7d34afe189"
       url: "https://pub.dev"
     source: hosted
-    version: "3.2.0"
+    version: "3.3.2"
   stack_trace:
     dependency: transitive
     description:
@@ -381,10 +389,10 @@ packages:
     dependency: transitive
     description:
       name: synchronized
-      sha256: c254ade258ec8282947a0acbbc90b9575b4f19673533ee46f2f6e9b3aeefd7c0
+      sha256: "63896c27e81b28f8cb4e69ead0d3e8f03f1d1e5fc531a3e579cabed6a2c7c9e5"
       url: "https://pub.dev"
     source: hosted
-    version: "3.4.0"
+    version: "3.4.0+1"
   term_glyph:
     dependency: transitive
     description:
@@ -397,26 +405,26 @@ packages:
     dependency: "direct dev"
     description:
       name: test
-      sha256: "8d9ceddbab833f180fbefed08afa76d7c03513dfdba87ffcec2718b02bbcbf20"
+      sha256: ca578dc12bb8b2f40b67b7d3bd2fac4f31c01a6ff7130a14e2597b919934507f
       url: "https://pub.dev"
     source: hosted
-    version: "1.31.0"
+    version: "1.31.1"
   test_api:
     dependency: transitive
     description:
       name: test_api
-      sha256: "949a932224383300f01be9221c39180316445ecb8e7547f70a41a35bf421fb9e"
+      sha256: "2a122cbe059f8b610d3a5415f42e255b6c17b1f21eee1d960f31080237fb4f11"
       url: "https://pub.dev"
     source: hosted
-    version: "0.7.11"
+    version: "0.7.12"
   test_core:
     dependency: transitive
     description:
       name: test_core
-      sha256: "1991d4cfe85d5043241acac92962c3977c8d2f2add1ee73130c7b286417d1d34"
+      sha256: d2e98ec12998368dc59ddd47ab709f2cd55acd6b66dc7db764455a44082f4bc5
       url: "https://pub.dev"
     source: hosted
-    version: "0.6.17"
+    version: "0.6.18"
   typed_data:
     dependency: transitive
     description:
@@ -429,10 +437,10 @@ packages:
     dependency: transitive
     description:
       name: vm_service
-      sha256: "45caa6c5917fa127b5dbcfbd1fa60b14e583afdc08bfc96dda38886ca252eb60"
+      sha256: "0016aef94fc66495ac78af5859181e3f3bf2026bd8eecc72b9565601e19ab360"
       url: "https://pub.dev"
     source: hosted
-    version: "15.0.2"
+    version: "15.2.0"
   watcher:
     dependency: transitive
     description:
@@ -477,10 +485,10 @@ packages:
     dependency: "direct main"
     description:
       name: xml
-      sha256: "971043b3a0d3da28727e40ed3e0b5d18b742fa5a68665cca88e74b7876d5e025"
+      sha256: "67f0aff7be013d107995e9b75bf4e7f2c3ef2dfdb2c8e68024bba0a7fd5756a4"
       url: "https://pub.dev"
     source: hosted
-    version: "6.6.1"
+    version: "7.0.1"
   yaml:
     dependency: transitive
     description:
@@ -490,4 +498,4 @@ packages:
     source: hosted
     version: "3.1.3"
 sdks:
-  dart: ">=3.10.1 <4.0.0"
+  dart: ">=3.11.0 <4.0.0"

pubspec.yaml

@@ -15,7 +15,7 @@ dependencies:
   sqflite_common: ^2.5.0
   sqflite_common_ffi: ^2.3.0
   sqlite3: ^3.1.6
-  xml: ^6.5.0
+  xml: '>=6.0.0 < 8.0.0'
 
 dev_dependencies:
   benchmark_harness: ^2.4.0

test/models/kanji_search_test.dart

@@ -0,0 +1,93 @@
+import 'dart:convert';
+
+import 'package:jadb/models/kanji_search/kanji_search_radical.dart';
+import 'package:jadb/models/kanji_search/kanji_search_result.dart';
+import 'package:test/test.dart';
+
+Object? _roundTripJson(Object? value) => jsonDecode(jsonEncode(value));
+
+Map<String, dynamic> _roundTripMap(Object? json) =>
+    Map<String, dynamic>.from(_roundTripJson(json) as Map);
+
+void main() {
+  test('KanjiSearchRadical JSON serialization roundtrip', () {
+    const radical = KanjiSearchRadical(
+      symbol: '人',
+      names: ['ひと', 'にんべん'],
+      forms: ['亻'],
+      meanings: ['person', 'human'],
+    );
+
+    final restored = KanjiSearchRadical.fromJson(
+      _roundTripMap(radical.toJson()),
+    );
+
+    expect(restored, equals(radical));
+  });
+
+  test('KanjiSearchResult JSON serialization roundtrip', () {
+    const result = KanjiSearchResult(
+      kanji: '休',
+      taughtIn: 1,
+      jlptLevel: 'N5',
+      newspaperFrequencyRank: 1234,
+      strokeCount: 6,
+      meanings: ['rest', 'day off'],
+      kunyomi: ['やす.む', 'やす.まる'],
+      onyomi: ['キュウ'],
+      radical: KanjiSearchRadical(
+        symbol: '人',
+        names: ['ひと', 'にんべん'],
+        forms: ['亻'],
+        meanings: ['person', 'human'],
+      ),
+      parts: ['亻', '木'],
+      codepoints: {'ucs': '4F11', 'jis208': '1-22-57'},
+      nanori: ['やす'],
+      alternativeLanguageReadings: {
+        'korean': ['휴'],
+        'pinyin': ['xiū'],
+      },
+      strokeMiscounts: [5, 7],
+      queryCodes: {
+        'skip': ['1-2-4'],
+        'fourCorner': ['2429.0'],
+      },
+      dictionaryReferences: {'nelson_c': '122', 'heisig': '457'},
+    );
+
+    final restored = KanjiSearchResult.fromJson(_roundTripMap(result.toJson()));
+
+    expect(restored, equals(result));
+  });
+
+  test(
+    'KanjiSearchResult JSON serialization roundtrip - nullable and empty fields',
+    () {
+      const result = KanjiSearchResult(
+        kanji: '々',
+        taughtIn: null,
+        jlptLevel: null,
+        newspaperFrequencyRank: null,
+        strokeCount: 3,
+        meanings: ['iteration mark'],
+        kunyomi: [],
+        onyomi: [],
+        radical: null,
+        parts: [],
+        codepoints: {'ucs': '3005'},
+        nanori: [],
+        alternativeLanguageReadings: {},
+        strokeMiscounts: [],
+        queryCodes: {},
+        dictionaryReferences: {},
+      );
+
+      final restored = KanjiSearchResult.fromJson(
+        _roundTripMap(result.toJson()),
+      );
+
+      expect(restored, equals(result));
+    },
+  );
+}

test/models/word_search_test.dart

@@ -0,0 +1,282 @@
+import 'dart:convert';
+
+import 'package:jadb/models/common/jlpt_level.dart';
+import 'package:jadb/models/jmdict/jmdict_dialect.dart';
+import 'package:jadb/models/jmdict/jmdict_field.dart';
+import 'package:jadb/models/jmdict/jmdict_kanji_info.dart';
+import 'package:jadb/models/jmdict/jmdict_misc.dart';
+import 'package:jadb/models/jmdict/jmdict_pos.dart';
+import 'package:jadb/models/jmdict/jmdict_reading_info.dart';
+import 'package:jadb/models/word_search/word_search_match_span.dart';
+import 'package:jadb/models/word_search/word_search_result.dart';
+import 'package:jadb/models/word_search/word_search_ruby.dart';
+import 'package:jadb/models/word_search/word_search_sense.dart';
+import 'package:jadb/models/word_search/word_search_sense_language_source.dart';
+import 'package:jadb/models/word_search/word_search_sources.dart';
+import 'package:jadb/models/word_search/word_search_xref_entry.dart';
+import 'package:test/test.dart';
+
+Object? _roundTripJson(Object? value) => jsonDecode(jsonEncode(value));
+
+Map<String, dynamic> _roundTripMap(Object? json) =>
+    Map<String, dynamic>.from(_roundTripJson(json) as Map);
+
+Map<String, Object?> _roundTripObjectMap(Object? json) =>
+    Map<String, Object?>.from(_roundTripJson(json) as Map);
+
+void _expectScalarRoundTrip<T>({
+  required Iterable<T> values,
+  required Object? Function(T value) toJson,
+  required T Function(Object? json) fromJson,
+}) {
+  for (final value in values) {
+    expect(
+      fromJson(_roundTripJson(toJson(value))),
+      equals(value),
+      reason: 'Roundtrip failed for $value',
+    );
+  }
+}
+
+void _expectMapRoundTrip<T>({
+  required Iterable<T> values,
+  required Map<String, Object?> Function(T value) toJson,
+  required T Function(Map<String, Object?> json) fromJson,
+}) {
+  for (final value in values) {
+    expect(
+      fromJson(_roundTripObjectMap(toJson(value))),
+      equals(value),
+      reason: 'Roundtrip failed for $value',
+    );
+  }
+}
+
+WordSearchResult _buildNestedXrefResult() => WordSearchResult(
+  score: 7,
+  entryId: 300,
+  isCommon: false,
+  japanese: [WordSearchRuby(base: '補助', furigana: 'ほじょ')],
+  kanjiInfo: const {},
+  readingInfo: const {},
+  senses: const [],
+  jlptLevel: JlptLevel.none,
+  sources: const WordSearchSources(jmdict: true, jmnedict: false),
+);
+
+WordSearchSense _buildSense() => WordSearchSense(
+  englishDefinitions: ['kana', 'syllabary'],
+  partsOfSpeech: [JMdictPOS.n],
+  seeAlso: [
+    WordSearchXrefEntry(
+      entryId: 300,
+      ambiguous: false,
+      baseWord: '仮名遣い',
+      furigana: 'かなづかい',
+      xrefResult: _buildNestedXrefResult(),
+    ),
+  ],
+  antonyms: const [
+    WordSearchXrefEntry(
+      entryId: 301,
+      ambiguous: true,
+      baseWord: '漢字',
+      furigana: 'かんじ',
+      xrefResult: null,
+    ),
+  ],
+  restrictedToReading: ['かな'],
+  restrictedToKanji: ['仮名'],
+  fields: [JMdictField.linguistics],
+  dialects: [JMdictDialect.kansai],
+  misc: [JMdictMisc.onlyKana, JMdictMisc.rare],
+  info: ['Typically written using kana alone.'],
+  languageSource: const [
+    WordSearchSenseLanguageSource(
+      language: 'por',
+      phrase: 'canoa',
+      fullyDescribesSense: false,
+      constructedFromSmallerWords: true,
+    ),
+  ],
+);
+
+WordSearchResult _buildWordSearchResult({
+  List<WordSearchMatchSpan>? matchSpans,
+}) => WordSearchResult(
+  score: 42,
+  entryId: 123,
+  isCommon: true,
+  japanese: [
+    WordSearchRuby(base: '仮名', furigana: 'かな'),
+    WordSearchRuby(base: 'かな'),
+  ],
+  kanjiInfo: {'仮名': JMdictKanjiInfo.rK, '仮': JMdictKanjiInfo.ateji},
+  readingInfo: {'かな': JMdictReadingInfo.gikun, 'カナ': JMdictReadingInfo.rk},
+  senses: [_buildSense()],
+  jlptLevel: JlptLevel.n5,
+  sources: const WordSearchSources(jmdict: true, jmnedict: true),
+  matchSpans: matchSpans,
+);
+
+void main() {
+  test('JlptLevel JSON serialization roundtrip', () {
+    _expectScalarRoundTrip<JlptLevel>(
+      values: JlptLevel.values,
+      toJson: (value) => value.toJson(),
+      fromJson: JlptLevel.fromJson,
+    );
+  });
+
+  test('JMdictDialect JSON serialization roundtrip', () {
+    _expectMapRoundTrip<JMdictDialect>(
+      values: JMdictDialect.values,
+      toJson: (value) => value.toJson(),
+      fromJson: JMdictDialect.fromJson,
+    );
+  });
+
+  test('JMdictField JSON serialization roundtrip', () {
+    _expectMapRoundTrip<JMdictField>(
+      values: JMdictField.values,
+      toJson: (value) => value.toJson(),
+      fromJson: JMdictField.fromJson,
+    );
+  });
+
+  test('JMdictKanjiInfo JSON serialization roundtrip', () {
+    _expectMapRoundTrip<JMdictKanjiInfo>(
+      values: JMdictKanjiInfo.values,
+      toJson: (value) => value.toJson(),
+      fromJson: JMdictKanjiInfo.fromJson,
+    );
+  });
+
+  test('JMdictMisc JSON serialization roundtrip', () {
+    _expectMapRoundTrip<JMdictMisc>(
+      values: JMdictMisc.values,
+      toJson: (value) => value.toJson(),
+      fromJson: JMdictMisc.fromJson,
+    );
+  });
+
+  test('JMdictPOS JSON serialization roundtrip', () {
+    _expectMapRoundTrip<JMdictPOS>(
+      values: JMdictPOS.values,
+      toJson: (value) => value.toJson(),
+      fromJson: JMdictPOS.fromJson,
+    );
+  });
+
+  test('JMdictReadingInfo JSON serialization roundtrip', () {
+    _expectMapRoundTrip<JMdictReadingInfo>(
+      values: JMdictReadingInfo.values,
+      toJson: (value) => value.toJson(),
+      fromJson: JMdictReadingInfo.fromJson,
+    );
+  });
+
+  test('WordSearchMatchSpan JSON serialization roundtrip', () {
+    final span = WordSearchMatchSpan(
+      spanType: WordSearchMatchSpanType.sense,
+      index: 2,
+      subIndex: 3,
+      start: 4,
+      end: 8,
+    );
+
+    final restored = WordSearchMatchSpan.fromJson(_roundTripMap(span.toJson()));
+
+    expect(restored, equals(span));
+    expect(restored.subIndex, equals(span.subIndex));
+  });
+
+  test('WordSearchRuby JSON serialization roundtrip', () {
+    final ruby = WordSearchRuby(base: '仮名', furigana: 'かな');
+
+    final restored = WordSearchRuby.fromJson(_roundTripMap(ruby.toJson()));
+
+    expect(restored.toJson(), equals(ruby.toJson()));
+  });
+
+  test('WordSearchSenseLanguageSource JSON serialization roundtrip', () {
+    const source = WordSearchSenseLanguageSource(
+      language: 'por',
+      phrase: 'canoa',
+      fullyDescribesSense: false,
+      constructedFromSmallerWords: true,
+    );
+
+    final restored = WordSearchSenseLanguageSource.fromJson(
+      _roundTripMap(source.toJson()),
+    );
+
+    expect(restored.toJson(), equals(source.toJson()));
+  });
+
+  test('WordSearchSources JSON serialization roundtrip', () {
+    const sources = WordSearchSources(jmdict: false, jmnedict: true);
+
+    final restored = WordSearchSources.fromJson(
+      _roundTripMap(sources.toJson()),
+    );
+
+    expect(restored.toJson(), equals(sources.toJson()));
+    expect(restored.sqlValue, equals(sources.sqlValue));
+  });
+
+  test('WordSearchXrefEntry JSON serialization roundtrip', () {
+    final entry = WordSearchXrefEntry(
+      entryId: 300,
+      ambiguous: false,
+      baseWord: '仮名遣い',
+      furigana: 'かなづかい',
+      xrefResult: _buildNestedXrefResult(),
+    );
+
+    final restored = WordSearchXrefEntry.fromJson(
+      _roundTripMap(entry.toJson()),
+    );
+
+    expect(restored.toJson(), equals(entry.toJson()));
+    expect(restored.xrefResult?.toJson(), equals(entry.xrefResult?.toJson()));
+  });
+
+  test('WordSearchSense JSON serialization roundtrip', () {
+    final sense = _buildSense();
+
+    final restored = WordSearchSense.fromJson(_roundTripMap(sense.toJson()));
+
+    expect(restored.toJson(), equals(sense.toJson()));
+  });
+
+  test('WordSearchResult JSON serialization roundtrip', () {
+    final result = _buildWordSearchResult();
+
+    final restored = WordSearchResult.fromJson(_roundTripMap(result.toJson()));
+
+    expect(restored.toJson(), equals(result.toJson()));
+    expect(restored.matchSpans, isNull);
+  });
+
+  test('WordSearchResult leaves matchSpans out of JSON', () {
+    final result = _buildWordSearchResult(
+      matchSpans: [
+        WordSearchMatchSpan(
+          spanType: WordSearchMatchSpanType.sense,
+          index: 0,
+          subIndex: 1,
+          start: 0,
+          end: 4,
+        ),
+      ],
+    );
+
+    final json = _roundTripMap(result.toJson());
+    final restored = WordSearchResult.fromJson(json);
+
+    expect(json.containsKey('matchSpans'), isFalse);
+    expect(restored.matchSpans, isNull);
+    expect(restored.toJson(), equals(result.toJson()));
+  });
+}

test/search/datasource_versions_test.dart

@@ -0,0 +1,12 @@
+import 'package:jadb/search.dart';
+import 'package:test/test.dart';
+
+import 'setup_database_connection.dart';
+
+void main() {
+  test('Retrieve datasource versions', () async {
+    final connection = await setupDatabaseConnection();
+    final result = await connection.jadbGetDatasourceVersions();
+    expect(result, isNotNull);
+  });
+}

test/search/radical_search_test.dart

@@ -0,0 +1,85 @@
+import 'package:collection/collection.dart';
+import 'package:jadb/const_data/radicals.dart';
+import 'package:jadb/search.dart';
+import 'package:jadb/table_names/radkfile.dart';
+import 'package:test/test.dart';
+
+import 'setup_database_connection.dart';
+
+void main() {
+  test('All constant radicals should exist in the database', () async {
+    final connection = await setupDatabaseConnection();
+    final allRadicalsInDb = await connection.query(
+      RADKFILETableNames.radkfile,
+      columns: ['radical'],
+      distinct: true,
+    );
+
+    final radicalsInDb = allRadicalsInDb
+        .map((e) => e['radical'] as String)
+        .toSet();
+
+    final missingRadicals = radicals.values.flattenedToSet
+        .map((e) => e.formalVariant)
+        .toSet()
+        .difference(radicalsInDb);
+
+    expect(
+      missingRadicals,
+      isEmpty,
+      reason: 'Missing radicals in database: $missingRadicals',
+    );
+  });
+
+  test(
+    'All radicals in database should be in the constant radical list',
+    () async {
+      final connection = await setupDatabaseConnection();
+      final allRadicalsInDb = await connection.query(
+        RADKFILETableNames.radkfile,
+        columns: ['radical'],
+        distinct: true,
+      );
+
+      final radicalsInDb = allRadicalsInDb
+          .map((e) => e['radical'] as String)
+          .toSet();
+
+      final extraRadicals = radicalsInDb.difference(
+        radicals.values.flattenedToSet.map((e) => e.formalVariant).toSet(),
+      );
+
+      expect(
+        extraRadicals,
+        isEmpty,
+        reason:
+            'Extra radicals in database missing in the constant list: $extraRadicals',
+      );
+    },
+  );
+
+  test('All constant radicals are located in the correct stroke count group', () {
+    for (final mapEntry in radicals.entries) {
+      final strokeCount = mapEntry.key;
+      final radicalsInGroup = mapEntry.value;
+      for (final radical in radicalsInGroup) {
+        expect(
+          strokeCount,
+          radical.strokeCount,
+          reason:
+              'Radical ${radical.formalVariant} should have stroke count $strokeCount but has ${radical.strokeCount}',
+        );
+      }
+    }
+  });
+
+  group('All radicals should return results', () {
+    for (final radical in radicals.values.flattened) {
+      test(' - $radical', () async {
+        final connection = await setupDatabaseConnection();
+        final result = await connection.jadbSearchKanjiByRadicals([radical]);
+        expect(result, isNotEmpty);
+      });
+    }
+  });
+}

test/util/romaji_transliteration_test.dart

@@ -8,7 +8,7 @@ void main() {
       expect(result, 'かたまり');
     });
 
-    test('Basic test with diacritics', () {
+    test('Basic test with dakuten', () {
       final result = transliterateLatinToHiragana('gadamari');
       expect(result, 'がだまり');
     });
@@ -54,7 +54,7 @@ void main() {
 
     test('Basic test', expectSpans('katamari', ['か', 'た', 'ま', 'り']));
     test(
-      'Basic test with diacritics',
+      'Basic test with dakuten',
       expectSpans('gadamari', ['が', 'だ', 'ま', 'り']),
     );
     test('wi and we', expectSpans('wiwe', ['うぃ', 'うぇ']));
@@ -72,7 +72,7 @@ void main() {
       expect(result, 'katamari');
     });
 
-    test('Basic test with diacritics', () {
+    test('Basic test with dakuten', () {
       final result = transliterateHiraganaToLatin('がだまり');
       expect(result, 'gadamari');
     });
@@ -91,6 +91,21 @@ void main() {
       final result = transliterateHiraganaToLatin('かっぱ');
       expect(result, 'kappa');
     });
+
+    test('Iteration mark', () {
+      final result = transliterateHiraganaToLatin('さゝき');
+      expect(result, 'sasaki');
+    }, skip: 'Not yet implemented');
+
+    test('Iteration mark with dakuten', () {
+      final result = transliterateHiraganaToLatin('あひゞき');
+      expect(result, 'ahibiki');
+    }, skip: 'Not yet implemented');
+
+    test('Yori', () {
+      final result = transliterateHiraganaToLatin('ゟ');
+      expect(result, 'yori');
+    }, skip: 'Not yet implemented');
   });
 
   group('Hiragana -> Romaji Spans', () {
@@ -110,7 +125,7 @@ void main() {
 
     test('Basic test', expectSpans('かたまり', ['ka', 'ta', 'ma', 'ri']));
     test(
-      'Basic test with diacritics',
+      'Basic test with dakuten',
       expectSpans('がだまり', ['ga', 'da', 'ma', 'ri']),
     );
     test('wi and we', expectSpans('うぃうぇ', ['whi', 'whe']));
@@ -118,5 +133,17 @@ void main() {
 
     // TODO: fix the implementation
     // test('Double consonant', expectSpans('かっぱ', ['ka', 'ppa']));
+
+    test(
+      'Iteration mark',
+      expectSpans('さゝき', ['sa', 'sa', 'ki']),
+      skip: 'Not yet implemented',
+    );
+    test(
+      'Iteration mark with dakuten',
+      expectSpans('あひゞき', ['a', 'hi', 'bi', 'ki']),
+      skip: 'Not yet implemented',
+    );
+    test('Yori', expectSpans('ゟ', ['yori']), skip: 'Not yet implemented');
   });
 }