diff --git a/lib/_data_ingestion/jmdict/xml_parser.dart b/lib/_data_ingestion/jmdict/xml_parser.dart index 85bb413..1135548 100644 --- a/lib/_data_ingestion/jmdict/xml_parser.dart +++ b/lib/_data_ingestion/jmdict/xml_parser.dart @@ -109,7 +109,7 @@ List parseJMDictData(XmlElement root) { orderNum < 100, 'Entry $entryId has more than 100 readings, which will break the elementId generation logic.', ); - final elementId = entryId * 100 + orderNum; + final elementId = 1_000_000_000 + entryId * 100 + orderNum; final rePri = getPriorityValues(rEle, 're'); final readingDoesNotMatchKanji = rEle diff --git a/lib/search/word_search/entry_id_query.dart b/lib/search/word_search/entry_id_query.dart index 14156cc..b2dc269 100644 --- a/lib/search/word_search/entry_id_query.dart +++ b/lib/search/word_search/entry_id_query.dart @@ -74,7 +74,7 @@ String _filterFTSSensitiveCharacters(String word) { JOIN "$tableName" USING ("elementId") JOIN "JMdict_EntryScore" USING ("elementId") WHERE "${tableName}FTS"."reading" MATCH ? || '*' - AND "JMdict_EntryScore"."type" = '${tableName == JMdictTableNames.kanjiElement ? 'k' : 'r'}' + AND "JMdict_EntryScore"."elementId" ${tableName == JMdictTableNames.kanjiElement ? '<' : '>='} 1000000000 ), non_fts_results AS ( SELECT DISTINCT @@ -86,7 +86,7 @@ String _filterFTSSensitiveCharacters(String word) { JOIN "JMdict_EntryScore" USING ("elementId") WHERE "reading" LIKE '%' || ? || '%' AND "$tableName"."entryId" NOT IN (SELECT "entryId" FROM "fts_results") - AND "JMdict_EntryScore"."type" = '${tableName == JMdictTableNames.kanjiElement ? 'k' : 'r'}' + AND "JMdict_EntryScore"."elementId" ${tableName == JMdictTableNames.kanjiElement ? '<' : '>='} 1000000000 ) SELECT ${countOnly ? 'COUNT(DISTINCT "entryId") AS count' : '"entryId", MAX("score") AS "score"'} diff --git a/migrations/0001_JMDict.sql b/migrations/0001_JMDict.sql index 36567ed..e246c03 100644 --- a/migrations/0001_JMDict.sql +++ b/migrations/0001_JMDict.sql @@ -77,7 +77,7 @@ CREATE TABLE "JMdict_KanjiElementInfo" ( CREATE TABLE "JMdict_ReadingElement" ( "elementId" INTEGER PRIMARY KEY, - "entryId" INTEGER NOT NULL GENERATED ALWAYS AS ("elementId" / 100) STORED, + "entryId" INTEGER NOT NULL GENERATED ALWAYS AS (("elementId" / 100) % 10000000) STORED, "orderNum" INTEGER NOT NULL GENERATED ALWAYS AS ("elementId" % 100) VIRTUAL, "reading" TEXT NOT NULL, "readingDoesNotMatchKanji" BOOLEAN NOT NULL DEFAULT FALSE, diff --git a/migrations/0005_JMDict_search_index_tables.sql b/migrations/0005_JMDict_search_index_tables.sql index 01075e9..961badb 100644 --- a/migrations/0005_JMDict_search_index_tables.sql +++ b/migrations/0005_JMDict_search_index_tables.sql @@ -1,29 +1,28 @@ CREATE TABLE "JMdict_EntryScore" ( - "type" CHAR(1) NOT NULL CHECK ("type" IN ('r', 'k')), - "elementId" INTEGER NOT NULL, + "elementId" INTEGER PRIMARY KEY, "score" INTEGER NOT NULL DEFAULT 0, "common" BOOLEAN NOT NULL DEFAULT FALSE, - "entryId" INTEGER NOT NULL GENERATED ALWAYS AS ("elementId" / 100) STORED, + "entryId" INTEGER NOT NULL GENERATED ALWAYS AS (("elementId" / 100) % 10000000) STORED, + "type" CHAR(1) NOT NULL GENERATED ALWAYS AS (CASE + WHEN "elementId" / 1000000000 = 0 THEN 'k' + ELSE 'r' + END) VIRTUAL, - FOREIGN KEY ("entryId") REFERENCES "JMdict_Entry"("entryId"), - PRIMARY KEY ("type", "elementId") + FOREIGN KEY ("entryId") REFERENCES "JMdict_Entry"("entryId") ) WITHOUT ROWID; CREATE INDEX "JMdict_EntryScore_byElementId_byScore" ON "JMdict_EntryScore"("elementId", "score"); CREATE INDEX "JMdict_EntryScore_byScore" ON "JMdict_EntryScore"("score"); -CREATE INDEX "JMdict_EntryScore_byCommon" ON "JMdict_EntryScore"("common"); -CREATE INDEX "JMdict_EntryScore_byType_byElementId_byScore" ON "JMdict_EntryScore"("type", "elementId", "score"); -CREATE INDEX "JMdict_EntryScore_byType_byScore" ON "JMdict_EntryScore"("type", "score"); -CREATE INDEX "JMdict_EntryScore_byType_byCommon" ON "JMdict_EntryScore"("type", "common"); +CREATE INDEX "JMdict_EntryScore_byElementId_byCommon" ON "JMdict_EntryScore"("elementId", "common"); +CREATE INDEX "JMdict_EntryScore_byCommon" ON "JMdict_EntryScore"("common"); -- NOTE: these views are deduplicated in order not to perform an unnecessary -- UNION on every trigger CREATE VIEW "JMdict_EntryScoreView_Reading" AS SELECT - 'r' AS "type", "JMdict_ReadingElement"."entryId", "JMdict_ReadingElement"."elementId", ( @@ -55,7 +54,6 @@ LEFT JOIN "JMdict_JLPTTag" USING ("entryId"); CREATE VIEW "JMdict_EntryScoreView_Kanji" AS SELECT - 'k' AS "type", "JMdict_KanjiElement"."entryId", "JMdict_KanjiElement"."elementId", ( @@ -99,12 +97,11 @@ CREATE TRIGGER "JMdict_EntryScore_Insert_JMdict_ReadingElement" AFTER INSERT ON "JMdict_ReadingElement" BEGIN INSERT INTO "JMdict_EntryScore" ( - "type", "elementId", "score", "common" ) - SELECT "type", "elementId", "score", "common" + SELECT "elementId", "score", "common" FROM "JMdict_EntryScoreView_Reading" WHERE "elementId" = NEW."elementId"; END; @@ -125,8 +122,7 @@ CREATE TRIGGER "JMdict_EntryScore_Delete_JMdict_ReadingElement" AFTER DELETE ON "JMdict_ReadingElement" BEGIN DELETE FROM "JMdict_EntryScore" - WHERE "type" = 'r' - AND "elementId" = OLD."elementId"; + WHERE "elementId" = OLD."elementId"; END; --- JMdict_KanjiElement triggers @@ -135,12 +131,11 @@ CREATE TRIGGER "JMdict_EntryScore_Insert_JMdict_KanjiElement" AFTER INSERT ON "JMdict_KanjiElement" BEGIN INSERT INTO "JMdict_EntryScore" ( - "type", "elementId", "score", "common" ) - SELECT "type", "elementId", "score", "common" + SELECT "elementId", "score", "common" FROM "JMdict_EntryScoreView_Kanji" WHERE "elementId" = NEW."elementId"; END; @@ -161,8 +156,7 @@ CREATE TRIGGER "JMdict_EntryScore_Delete_JMdict_KanjiElement" AFTER DELETE ON "JMdict_KanjiElement" BEGIN DELETE FROM "JMdict_EntryScore" - WHERE "type" = 'k' - AND "elementId" = OLD."elementId"; + WHERE "elementId" = OLD."elementId"; END; --- JMdict_JLPTTag triggers diff --git a/migrations/0010_Views.sql b/migrations/0010_Views.sql index 10e1d82..d65ba86 100644 --- a/migrations/0010_Views.sql +++ b/migrations/0010_Views.sql @@ -65,9 +65,7 @@ JOIN "JMdict_KanjiElement" ON "JMdict_KanjiElementFTS"."entryId" = "JMdict_KanjiElement"."entryId" AND "JMdict_KanjiElementFTS"."reading" LIKE '%' || "JMdict_KanjiElement"."reading" JOIN "JMdict_EntryScore" - ON "JMdict_EntryScore"."type" = 'k' - AND "JMdict_KanjiElement"."entryId" = "JMdict_EntryScore"."entryId" - AND "JMdict_KanjiElement"."reading" = "JMdict_EntryScore"."reading" + ON "JMdict_EntryScore"."elementId" = "JMdict_KanjiElement"."elementId" WHERE "JMdict_EntryScore"."common" = 1;