WIP: kanjidic: split news ichi gai nf into separate tables

flake.lock

@@ -3,7 +3,7 @@
     "jmdict-src": {
       "flake": false,
       "locked": {
-        "narHash": "sha256-sLl+OrVBgc4QCOZ2cvWGLZBerHDLuApyQOQyDyLUHtk=",
+        "narHash": "sha256-84P7r/fFlBnawy6yChrD9WMHmOWcEGWUmoK70N4rdGQ=",
         "type": "file",
         "url": "http://ftp.edrdg.org/pub/Nihongo/JMdict_e.gz"
       },
@@ -15,7 +15,7 @@
     "jmdict-with-examples-src": {
       "flake": false,
       "locked": {
-        "narHash": "sha256-FQvkYXwgmCJ+ChVkoFzamlG8kyczHAgsJ3zJ6OvRLZc=",
+        "narHash": "sha256-PM0sv7VcsCya2Ek02CI7hVwB3Jawn6bICSI+dsJK0yo=",
         "type": "file",
         "url": "http://ftp.edrdg.org/pub/Nihongo/JMdict_e_examp.gz"
       },
@@ -27,7 +27,7 @@
     "kanjidic2-src": {
       "flake": false,
       "locked": {
-        "narHash": "sha256-vyMpRnN9O3vCpvfVDACKdTlapBVx6yXg0X2tgXF2t+U=",
+        "narHash": "sha256-Lc0wUPpuDKuMDv2t87//w3z20RX8SMJI2iIRtUJ8fn0=",
         "type": "file",
         "url": "https://www.edrdg.org/kanjidic/kanjidic2.xml.gz"
       },
@@ -38,11 +38,11 @@
     },
     "nixpkgs": {
       "locked": {
-        "lastModified": 1751792365,
-        "narHash": "sha256-J1kI6oAj25IG4EdVlg2hQz8NZTBNYvIS0l4wpr9KcUo=",
+        "lastModified": 1746904237,
+        "narHash": "sha256-3e+AVBczosP5dCLQmMoMEogM57gmZ2qrVSrmq9aResQ=",
         "owner": "NixOS",
         "repo": "nixpkgs",
-        "rev": "1fd8bada0b6117e6c7eb54aad5813023eed37ccb",
+        "rev": "d89fc19e405cb2d55ce7cc114356846a0ee5e956",
         "type": "github"
       },
       "original": {

lib/search/word_search/entry_id_query.dart

@@ -53,7 +53,7 @@ String _filterFTSSensitiveCharacters(String word) {
             SELECT DISTINCT
               "${tableName}"."entryId",
               100
-              + (("${tableName}FTS"."reading" = ?) * 10000)
+              + (("${tableName}FTS"."reading" = ?) * 50)
               + "JMdict_EntryScore"."score"
               AS "score"
             FROM "${tableName}FTS"
@@ -61,6 +61,8 @@ String _filterFTSSensitiveCharacters(String word) {
             JOIN "JMdict_EntryScore" USING ("elementId")
             WHERE "${tableName}FTS"."reading" MATCH ? || '*'
               AND "JMdict_EntryScore"."type" = '${tableName == JMdictTableNames.kanjiElement ? 'k' : 'r'}'
+            ORDER BY
+              "JMdict_EntryScore"."score" DESC
             ${!countOnly ? 'LIMIT ?' : ''}
           ),
           non_fts_results AS (
@@ -74,19 +76,18 @@ String _filterFTSSensitiveCharacters(String word) {
             WHERE "reading" LIKE '%' || ? || '%'
               AND "${tableName}"."entryId" NOT IN (SELECT "entryId" FROM "fts_results")
               AND "JMdict_EntryScore"."type" = '${tableName == JMdictTableNames.kanjiElement ? 'k' : 'r'}'
+            ORDER BY
+              "JMdict_EntryScore"."score" DESC,
+              "${tableName}"."entryId" ASC
             ${!countOnly ? 'LIMIT ?' : ''}
           )
 
-        ${countOnly ? 'SELECT COUNT("entryId") AS count' : 'SELECT "entryId", MAX("score") AS "score"'}
+        ${countOnly ? 'SELECT COUNT("entryId") AS count' : 'SELECT "entryId", "score"'}
         FROM (
           SELECT * FROM fts_results
-          UNION
+          UNION ALL
           SELECT * FROM non_fts_results
         )
-        GROUP BY "entryId"
-        ORDER BY
-          "score" DESC,
-          "entryId" ASC
       '''
           .trim(),
       [

migrations/0001_JMDict.sql

@@ -43,11 +43,6 @@ CREATE TABLE "JMdict_KanjiElement" (
   "entryId" INTEGER NOT NULL REFERENCES "JMdict_Entry"("entryId"),
   "orderNum" INTEGER NOT NULL,
   "reading" TEXT NOT NULL,
-  "news" INTEGER CHECK ("news" BETWEEN 1 AND 2),
-  "ichi" INTEGER CHECK ("ichi" BETWEEN 1 AND 2),
-  "spec" INTEGER CHECK ("spec" BETWEEN 1 AND 2),
-  "gai" INTEGER CHECK ("gai" BETWEEN 1 AND 2),
-  "nf" INTEGER CHECK ("nf" BETWEEN 1 AND 48),
   UNIQUE("entryId", "reading"),
   UNIQUE("entryId", "orderNum")
 ) WITHOUT ROWID;
@@ -55,6 +50,31 @@ CREATE TABLE "JMdict_KanjiElement" (
 CREATE INDEX "JMdict_KanjiElement_byEntryId_byOrderNum" ON "JMdict_KanjiElement"("entryId", "orderNum");
 CREATE INDEX "JMdict_KanjiElement_byReading" ON "JMdict_KanjiElement"("reading");
 
+CREATE TABLE "JMdict_KanjiElementNews" (
+  "elementId" INTEGER PRIMARY KEY REFERENCES "JMdict_KanjiElement"("elementId"),
+  "news" INTEGER NOT NULL CHECK ("news" BETWEEN 1 AND 2)
+);
+
+CREATE TABLE "JMdict_KanjiElementIchi" (
+  "elementId" INTEGER PRIMARY KEY REFERENCES "JMdict_KanjiElement"("elementId"),
+  "ichi" INTEGER NOT NULL CHECK ("ichi" BETWEEN 1 AND 2)
+);
+
+CREATE TABLE "JMdict_KanjiElementSpec" (
+  "elementId" INTEGER PRIMARY KEY REFERENCES "JMdict_KanjiElement"("elementId"),
+  "spec" INTEGER NOT NULL CHECK ("spec" BETWEEN 1 AND 2)
+);
+
+CREATE TABLE "JMdict_KanjiElementGai" (
+  "elementId" INTEGER PRIMARY KEY REFERENCES "JMdict_KanjiElement"("elementId"),
+  "gai" INTEGER NOT NULL CHECK ("gai" BETWEEN 1 AND 2)
+);
+
+CREATE TABLE "JMdict_KanjiElementNF" (
+  "elementId" INTEGER PRIMARY KEY REFERENCES "JMdict_KanjiElement"("elementId"),
+  "nf" INTEGER NOT NULL CHECK ("nf" BETWEEN 1 AND 48)
+);
+
 CREATE TABLE "JMdict_KanjiElementInfo" (
   "elementId" INTEGER NOT NULL REFERENCES "JMdict_KanjiElement"("elementId"),
   "info" TEXT NOT NULL REFERENCES "JMdict_InfoKanji"("id"),
@@ -81,6 +101,36 @@ CREATE TABLE "JMdict_ReadingElement" (
 CREATE INDEX "JMdict_ReadingElement_byEntryId_byOrderNum" ON "JMdict_ReadingElement"("entryId", "orderNum");
 CREATE INDEX "JMdict_ReadingElement_byReading" ON "JMdict_ReadingElement"("reading");
 
+CREATE TABLE "JMdict_ReadingElementReadingDoesNotMatchKanji" (
+  "elementId" INTEGER PRIMARY KEY REFERENCES "JMdict_ReadingElement"("elementId"),
+  "readingDoesNotMatchKanji" BOOLEAN NOT NULL,
+);
+
+CREATE TABLE "JMdict_ReadingElementNews" (
+  "elementId" INTEGER PRIMARY KEY REFERENCES "JMdict_ReadingElement"("elementId"),
+  "news" INTEGER NOT NULL CHECK ("news" BETWEEN 1 AND 2)
+);
+
+CREATE TABLE "JMdict_ReadingElementIchi" (
+  "elementId" INTEGER PRIMARY KEY REFERENCES "JMdict_ReadingElement"("elementId"),
+  "ichi" INTEGER NOT NULL CHECK ("ichi" BETWEEN 1 AND 2)
+);
+
+CREATE TABLE "JMdict_ReadingElementSpec" (
+  "elementId" INTEGER PRIMARY KEY REFERENCES "JMdict_ReadingElement"("elementId"),
+  "spec" INTEGER NOT NULL CHECK ("spec" BETWEEN 1 AND 2)
+);
+
+CREATE TABLE "JMdict_ReadingElementGai" (
+  "elementId" INTEGER PRIMARY KEY REFERENCES "JMdict_ReadingElement"("elementId"),
+  "gai" INTEGER NOT NULL CHECK ("gai" BETWEEN 1 AND 2)
+);
+
+CREATE TABLE "JMdict_ReadingElementNF" (
+  "elementId" INTEGER PRIMARY KEY REFERENCES "JMdict_ReadingElement"("elementId"),
+  "nf" INTEGER NOT NULL CHECK ("nf" BETWEEN 1 AND 48)
+);
+
 CREATE TABLE "JMdict_ReadingElementRestriction" (
   "elementId" INTEGER NOT NULL REFERENCES "JMdict_ReadingElement"("elementId"),
   "restriction" TEXT NOT NULL,

migrations/0005_JMDict_search_index_tables.sql

@@ -48,6 +48,10 @@ SELECT
   - (substr(COALESCE("JMdict_JLPTTag"."jlptLevel", 'N0'), 2) * -5)
   AS "score"
 FROM "JMdict_ReadingElement"
+LEFT JOIN "JMdict_ReadingElementNews" USING ("elementId")
+LEFT JOIN "JMdict_ReadingElementIchi" USING ("elementId")
+LEFT JOIN "JMdict_ReadingElementSpec" USING ("elementId")
+LEFT JOIN "JMdict_ReadingElementGai" USING ("elementId")
 LEFT JOIN "JMdict_JLPTTag" USING ("entryId");
 
 CREATE VIEW "JMdict_EntryScoreView_Kanji" AS
@@ -80,6 +84,10 @@ SELECT
   - (substr(COALESCE("JMdict_JLPTTag"."jlptLevel", 'N0'), 2) * -5)
   AS "score"
 FROM "JMdict_KanjiElement"
+LEFT JOIN "JMdict_KanjiElementNews" USING ("elementId")
+LEFT JOIN "JMdict_KanjiElementIchi" USING ("elementId")
+LEFT JOIN "JMdict_KanjiElementSpec" USING ("elementId")
+LEFT JOIN "JMdict_KanjiElementGai" USING ("elementId")
 LEFT JOIN "JMdict_JLPTTag" USING ("entryId");
 
 CREATE VIEW "JMdict_EntryScoreView" AS