mugiten/jadb
2
1
Fork 0
Code Issues 65 Pull Requests Actions Activity

jmdict: embed element type (k/r) into elementId

Browse Source
This commit is contained in:
Oystein Kristoffer Tveit
2026-04-08 19:05:02 +09:00
parent 114febbe02
commit 5d26b41524
5 changed files with 18 additions and 26 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
lib/_data_ingestion/jmdict/xml_parser.dart
View File

@@ -109,7 +109,7 @@ List<Entry> parseJMDictData(XmlElement root) {
orderNum < 100,
'Entry $entryId has more than 100 readings, which will break the elementId generation logic.',
);
final elementId = entryId * 100 + orderNum;
final elementId = 1_000_000_000 + entryId * 100 + orderNum;
final rePri = getPriorityValues(rEle, 're');
final readingDoesNotMatchKanji = rEle

4
lib/search/word_search/entry_id_query.dart
View File

@@ -74,7 +74,7 @@ String _filterFTSSensitiveCharacters(String word) {
JOIN "$tableName" USING ("elementId")
JOIN "JMdict_EntryScore" USING ("elementId")
WHERE "${tableName}FTS"."reading" MATCH ? || '*'
AND "JMdict_EntryScore"."type" = '${tableName == JMdictTableNames.kanjiElement ? 'k' : 'r'}'
AND "JMdict_EntryScore"."elementId" ${tableName == JMdictTableNames.kanjiElement ? '<' : '>='} 1000000000
),
non_fts_results AS (
SELECT DISTINCT
@@ -86,7 +86,7 @@ String _filterFTSSensitiveCharacters(String word) {
JOIN "JMdict_EntryScore" USING ("elementId")
WHERE "reading" LIKE '%' || ? || '%'
AND "$tableName"."entryId" NOT IN (SELECT "entryId" FROM "fts_results")
AND "JMdict_EntryScore"."type" = '${tableName == JMdictTableNames.kanjiElement ? 'k' : 'r'}'
AND "JMdict_EntryScore"."elementId" ${tableName == JMdictTableNames.kanjiElement ? '<' : '>='} 1000000000
)
SELECT ${countOnly ? 'COUNT(DISTINCT "entryId") AS count' : '"entryId", MAX("score") AS "score"'}

2
migrations/0001_JMDict.sql
View File

@@ -77,7 +77,7 @@ CREATE TABLE "JMdict_KanjiElementInfo" (
CREATE TABLE "JMdict_ReadingElement" (
"elementId" INTEGER PRIMARY KEY,
"entryId" INTEGER NOT NULL GENERATED ALWAYS AS ("elementId" / 100) STORED,
"entryId" INTEGER NOT NULL GENERATED ALWAYS AS (("elementId" / 100) % 10000000) STORED,
"orderNum" INTEGER NOT NULL GENERATED ALWAYS AS ("elementId" % 100) VIRTUAL,
"reading" TEXT NOT NULL,
"readingDoesNotMatchKanji" BOOLEAN NOT NULL DEFAULT FALSE,

32
migrations/0005_JMDict_search_index_tables.sql
View File

@@ -1,29 +1,28 @@
CREATE TABLE "JMdict_EntryScore" (
"type" CHAR(1) NOT NULL CHECK ("type" IN ('r', 'k')),
"elementId" INTEGER NOT NULL,
"elementId" INTEGER PRIMARY KEY,
"score" INTEGER NOT NULL DEFAULT 0,
"common" BOOLEAN NOT NULL DEFAULT FALSE,
"entryId" INTEGER NOT NULL GENERATED ALWAYS AS ("elementId" / 100) STORED,
"entryId" INTEGER NOT NULL GENERATED ALWAYS AS (("elementId" / 100) % 10000000) STORED,
"type" CHAR(1) NOT NULL GENERATED ALWAYS AS (CASE
WHEN "elementId" / 1000000000 = 0 THEN 'k'
ELSE 'r'
END) VIRTUAL,
FOREIGN KEY ("entryId") REFERENCES "JMdict_Entry"("entryId"),
PRIMARY KEY ("type", "elementId")
FOREIGN KEY ("entryId") REFERENCES "JMdict_Entry"("entryId")
) WITHOUT ROWID;
CREATE INDEX "JMdict_EntryScore_byElementId_byScore" ON "JMdict_EntryScore"("elementId", "score");
CREATE INDEX "JMdict_EntryScore_byScore" ON "JMdict_EntryScore"("score");
CREATE INDEX "JMdict_EntryScore_byCommon" ON "JMdict_EntryScore"("common");
CREATE INDEX "JMdict_EntryScore_byType_byElementId_byScore" ON "JMdict_EntryScore"("type", "elementId", "score");
CREATE INDEX "JMdict_EntryScore_byType_byScore" ON "JMdict_EntryScore"("type", "score");
CREATE INDEX "JMdict_EntryScore_byType_byCommon" ON "JMdict_EntryScore"("type", "common");
CREATE INDEX "JMdict_EntryScore_byElementId_byCommon" ON "JMdict_EntryScore"("elementId", "common");
CREATE INDEX "JMdict_EntryScore_byCommon" ON "JMdict_EntryScore"("common");
-- NOTE: these views are deduplicated in order not to perform an unnecessary
-- UNION on every trigger
CREATE VIEW "JMdict_EntryScoreView_Reading" AS
SELECT
'r' AS "type",
"JMdict_ReadingElement"."entryId",
"JMdict_ReadingElement"."elementId",
(
@@ -55,7 +54,6 @@ LEFT JOIN "JMdict_JLPTTag" USING ("entryId");
CREATE VIEW "JMdict_EntryScoreView_Kanji" AS
SELECT
'k' AS "type",
"JMdict_KanjiElement"."entryId",
"JMdict_KanjiElement"."elementId",
(
@@ -99,12 +97,11 @@ CREATE TRIGGER "JMdict_EntryScore_Insert_JMdict_ReadingElement"
AFTER INSERT ON "JMdict_ReadingElement"
BEGIN
INSERT INTO "JMdict_EntryScore" (
"type",
"elementId",
"score",
"common"
)
SELECT "type", "elementId", "score", "common"
SELECT "elementId", "score", "common"
FROM "JMdict_EntryScoreView_Reading"
WHERE "elementId" = NEW."elementId";
END;
@@ -125,8 +122,7 @@ CREATE TRIGGER "JMdict_EntryScore_Delete_JMdict_ReadingElement"
AFTER DELETE ON "JMdict_ReadingElement"
BEGIN
DELETE FROM "JMdict_EntryScore"
WHERE "type" = 'r'
AND "elementId" = OLD."elementId";
WHERE "elementId" = OLD."elementId";
END;
--- JMdict_KanjiElement triggers
@@ -135,12 +131,11 @@ CREATE TRIGGER "JMdict_EntryScore_Insert_JMdict_KanjiElement"
AFTER INSERT ON "JMdict_KanjiElement"
BEGIN
INSERT INTO "JMdict_EntryScore" (
"type",
"elementId",
"score",
"common"
)
SELECT "type", "elementId", "score", "common"
SELECT "elementId", "score", "common"
FROM "JMdict_EntryScoreView_Kanji"
WHERE "elementId" = NEW."elementId";
END;
@@ -161,8 +156,7 @@ CREATE TRIGGER "JMdict_EntryScore_Delete_JMdict_KanjiElement"
AFTER DELETE ON "JMdict_KanjiElement"
BEGIN
DELETE FROM "JMdict_EntryScore"
WHERE "type" = 'k'
AND "elementId" = OLD."elementId";
WHERE "elementId" = OLD."elementId";
END;
--- JMdict_JLPTTag triggers

4
migrations/0010_Views.sql
View File

@@ -65,9 +65,7 @@ JOIN "JMdict_KanjiElement"
ON "JMdict_KanjiElementFTS"."entryId" = "JMdict_KanjiElement"."entryId"
AND "JMdict_KanjiElementFTS"."reading" LIKE '%' || "JMdict_KanjiElement"."reading"
JOIN "JMdict_EntryScore"
ON "JMdict_EntryScore"."type" = 'k'
AND "JMdict_KanjiElement"."entryId" = "JMdict_EntryScore"."entryId"
AND "JMdict_KanjiElement"."reading" = "JMdict_EntryScore"."reading"
ON "JMdict_EntryScore"."elementId" = "JMdict_KanjiElement"."elementId"
WHERE "JMdict_EntryScore"."common" = 1;