From ef491a097722190c4130c2d8143cc63f5cb4a3a0 Mon Sep 17 00:00:00 2001 From: h7x4 Date: Thu, 4 Jun 2026 02:00:00 +0900 Subject: [PATCH] Add kana independent search with tamerye extension --- flake.lock | 85 +++++++++++++++++++++- flake.nix | 28 +++++-- lib/search/word_search/entry_id_query.dart | 6 +- lib/version.dart | 2 +- migrations/0001_JMDict.sql | 2 + migrations/0003_JMDict_FTS5_Ttables.sql | 8 +- nix/database_tool.nix | 8 ++ 7 files changed, 124 insertions(+), 15 deletions(-) diff --git a/flake.lock b/flake.lock index 1e62e8c..942bd74 100644 --- a/flake.lock +++ b/flake.lock @@ -1,5 +1,20 @@ { "nodes": { + "crane": { + "locked": { + "lastModified": 1780099841, + "narHash": "sha256-EVZd2RsbpreRUDSi9rBwPY+ZxoyMaiEBbZxxhljbaS4=", + "owner": "ipetkov", + "repo": "crane", + "rev": "0532eb17955225173906d671fb36306bdeb1e2dc", + "type": "github" + }, + "original": { + "owner": "ipetkov", + "repo": "crane", + "type": "github" + } + }, "datasources": { "inputs": { "nixpkgs": [ @@ -20,6 +35,28 @@ "url": "https://git.pvv.ntnu.no/Mugiten/datasources.git" } }, + "nix-sqlite": { + "inputs": { + "nixpkgs": [ + "tamerye", + "nixpkgs" + ] + }, + "locked": { + "lastModified": 1780224621, + "narHash": "sha256-mGAHKHEzh+J83GzYCyuM6xdfBPFYTBjNAmDmbHKRZ5U=", + "ref": "main", + "rev": "8afa09b948ec5ccbe488c21d14d458060720d313", + "revCount": 28, + "type": "git", + "url": "https://git.pvv.ntnu.no/mugiten/nix-custom-sqlite.git" + }, + "original": { + "ref": "main", + "type": "git", + "url": "https://git.pvv.ntnu.no/mugiten/nix-custom-sqlite.git" + } + }, "nixpkgs": { "locked": { "lastModified": 1779560665, @@ -38,7 +75,53 @@ "root": { "inputs": { "datasources": "datasources", - "nixpkgs": "nixpkgs" + "nixpkgs": "nixpkgs", + "tamerye": "tamerye" + } + }, + "rust-overlay": { + "inputs": { + "nixpkgs": [ + "tamerye", + "nixpkgs" + ] + }, + "locked": { + "lastModified": 1780197589, + "narHash": "sha256-FVCr2Ij/jKf59a4LW481eeOF6rJRreOBrVgW/aUBTrw=", + "owner": "oxalica", + "repo": "rust-overlay", + "rev": "21632e942d89bf1cce4e5a63d7e58a215a0cbfcc", + "type": "github" + }, + "original": { + "owner": "oxalica", + "repo": "rust-overlay", + "type": "github" + } + }, + "tamerye": { + "inputs": { + "crane": "crane", + "nix-sqlite": "nix-sqlite", + "nixpkgs": [ + "nixpkgs" + ], + "rust-overlay": "rust-overlay" + }, + "locked": { + "lastModified": 1780505561, + "narHash": "sha256-534k5H8k6GcFUGy7ENsOvuZ2Lf+6WcOf3vsqfrsnNlo=", + "ref": "main", + "rev": "f9789bb582218218e29a7ac674c5edf0d3609a5d", + "revCount": 17, + "type": "git", + "url": "https://git.pvv.ntnu.no/Mugiten/tamerye.git" + }, + "original": { + "ref": "main", + "type": "git", + "url": "https://git.pvv.ntnu.no/Mugiten/tamerye.git" } } }, diff --git a/flake.nix b/flake.nix index b7c7f88..09c2019 100644 --- a/flake.nix +++ b/flake.nix @@ -4,6 +4,11 @@ inputs = { nixpkgs.url = "nixpkgs/nixos-unstable"; + tamerye = { + url = "git+https://git.pvv.ntnu.no/Mugiten/tamerye.git?ref=main"; + inputs.nixpkgs.follows = "nixpkgs"; + }; + datasources = { url = "git+https://git.pvv.ntnu.no/Mugiten/datasources.git"; inputs.nixpkgs.follows = "nixpkgs"; @@ -13,6 +18,7 @@ outputs = { self, nixpkgs, + tamerye, datasources, }: let inherit (nixpkgs) lib; @@ -24,7 +30,14 @@ "armv7l-linux" ]; - forAllSystems = f: lib.genAttrs systems (system: f system nixpkgs.legacyPackages.${system}); + forAllSystems = f: lib.genAttrs systems (system: let + pkgs = import nixpkgs { + inherit system; + overlays = [ + tamerye.overlays.default + ]; + }; + in f system pkgs); in { apps = forAllSystems (system: pkgs: { default = { @@ -35,7 +48,7 @@ runtimeEnv = { JADB_PATH = "${self.packages.${system}.database}/jadb.sqlite"; - LIBSQLITE_PATH = "${pkgs.sqlite.out}/lib/libsqlite3.so"; + LIBSQLITE_PATH = "${pkgs.tamerye-sqlite}/lib/libsqlite3.so"; }; text = '' @@ -63,18 +76,18 @@ gnumake lcov sqldiff - sqlite-interactive + tamerye-sqlite-cli ]; env = { - LIBSQLITE_PATH = "${pkgs.sqlite.out}/lib/libsqlite3.so"; + LIBSQLITE_PATH = "${pkgs.tamerye-sqlite}/lib/libsqlite3.so"; JADB_PATH = "result/jadb.sqlite"; - LD_LIBRARY_PATH = lib.makeLibraryPath [ pkgs.sqlite ]; + LD_LIBRARY_PATH = lib.makeLibraryPath [ pkgs.tamerye-sqlite ]; }; }; sqlite-debugging = pkgs.mkShell { packages = with pkgs; [ - sqlite-interactive + tamerye-sqlite-cli sqlite-analyzer sqlite-web # sqlint @@ -119,16 +132,19 @@ inherit (datasources.packages.${system}) jmdict radkfile kanjidic2; database-tool = pkgs.callPackage ./nix/database_tool.nix { + sqlite = pkgs.tamerye-sqlite; inherit src; }; database = pkgs.callPackage ./nix/database.nix { + sqlite = pkgs.tamerye-sqlite-cli; inherit (datasources.packages.${system}) jmdict radkfile kanjidic2 tanos-jlpt; inherit (self.packages.${system}) database-tool; inherit src; }; database-wal = pkgs.callPackage ./nix/database.nix { + sqlite = pkgs.tamerye-sqlite-cli; inherit (datasources.packages.${system}) jmdict radkfile kanjidic2 tanos-jlpt; inherit (self.packages.${system}) database-tool; inherit src; diff --git a/lib/search/word_search/entry_id_query.dart b/lib/search/word_search/entry_id_query.dart index 9b3ab86..a939aee 100644 --- a/lib/search/word_search/entry_id_query.dart +++ b/lib/search/word_search/entry_id_query.dart @@ -67,14 +67,14 @@ String _filterFTSSensitiveCharacters(String word) { SELECT DISTINCT "$tableName"."entryId", 100 - + (("${tableName}FTS"."reading" = ?) * 10000) + + (("${tableName}FTS"."reading" = normalize_jp(?)) * 10000) + (("$tableName"."orderNum" = 0) * 20) + COALESCE("JMdict_EntryScore"."score", 0) AS "score" FROM "${tableName}FTS" JOIN "$tableName" USING ("elementId") LEFT JOIN "JMdict_EntryScore" USING ("elementId") - WHERE "${tableName}FTS"."reading" MATCH ? || '*' + WHERE "${tableName}FTS"."reading" MATCH normalize_jp(?) || '*' ), non_fts_results AS ( SELECT DISTINCT @@ -85,7 +85,7 @@ String _filterFTSSensitiveCharacters(String word) { AS "score" FROM "$tableName" LEFT JOIN "JMdict_EntryScore" USING ("elementId") - WHERE "reading" LIKE '%' || ? || '%' + WHERE "reading" LIKE '%' || normalize_jp(?) || '%' AND "$tableName"."entryId" NOT IN (SELECT "entryId" FROM "fts_results") ) diff --git a/lib/version.dart b/lib/version.dart index ecddb63..c7e1e60 100644 --- a/lib/version.dart +++ b/lib/version.dart @@ -1 +1 @@ -const int jadbSchemaVersion = 1; +const int jadbSchemaVersion = 2; diff --git a/migrations/0001_JMDict.sql b/migrations/0001_JMDict.sql index c67fb9b..9367053 100644 --- a/migrations/0001_JMDict.sql +++ b/migrations/0001_JMDict.sql @@ -66,6 +66,7 @@ CREATE TABLE "JMdict_KanjiElement" ( ) WITHOUT ROWID; CREATE INDEX "JMdict_KanjiElement_byReading" ON "JMdict_KanjiElement"("reading"); +CREATE INDEX "JMdict_KanjiElement_byNormalizedReading" ON "JMdict_KanjiElement"(normalize_jp("reading")); CREATE TABLE "JMdict_KanjiElementInfo" ( "elementId" INTEGER NOT NULL REFERENCES "JMdict_KanjiElement"("elementId"), @@ -91,6 +92,7 @@ CREATE TABLE "JMdict_ReadingElement" ( ) WITHOUT ROWID; CREATE INDEX "JMdict_ReadingElement_byReading" ON "JMdict_ReadingElement"("reading"); +CREATE INDEX "JMdict_ReadingElement_byNormalizedReading" ON "JMdict_ReadingElement"(normalize_jp("reading")); CREATE TABLE "JMdict_ReadingElementRestriction" ( "elementId" INTEGER NOT NULL REFERENCES "JMdict_ReadingElement"("elementId"), diff --git a/migrations/0003_JMDict_FTS5_Ttables.sql b/migrations/0003_JMDict_FTS5_Ttables.sql index 1fcf540..1fccb76 100644 --- a/migrations/0003_JMDict_FTS5_Ttables.sql +++ b/migrations/0003_JMDict_FTS5_Ttables.sql @@ -4,7 +4,7 @@ CREATE TRIGGER "JMdict_KanjiElement_InsertFTS" AFTER INSERT ON "JMdict_KanjiElement" BEGIN INSERT INTO "JMdict_KanjiElementFTS"("elementId", "reading") - VALUES (NEW."elementId", NEW."reading"); + VALUES (NEW."elementId", normalize_jp(NEW."reading")); END; CREATE TRIGGER "JMdict_KanjiElement_UpdateFTS" @@ -14,7 +14,7 @@ BEGIN UPDATE "JMdict_KanjiElementFTS" SET "elementId" = NEW."elementId", - "reading" = NEW."reading" + "reading" = normalize_jp(NEW."reading") WHERE "elementId" = OLD."elementId"; END; @@ -33,7 +33,7 @@ CREATE TRIGGER "JMdict_ReadingElement_InsertFTS" AFTER INSERT ON "JMdict_ReadingElement" BEGIN INSERT INTO "JMdict_ReadingElementFTS"("elementId", "reading") - VALUES (NEW."elementId", NEW."reading"); + VALUES (NEW."elementId", normalize_jp(NEW."reading")); END; CREATE TRIGGER "JMdict_ReadingElement_UpdateFTS" @@ -43,7 +43,7 @@ BEGIN UPDATE "JMdict_ReadingElementFTS" SET "elementId" = NEW."elementId", - "reading" = NEW."reading" + "reading" = normalize_jp(NEW."reading") WHERE "elementId" = OLD."elementId"; END; diff --git a/nix/database_tool.nix b/nix/database_tool.nix index 3fffb74..531e9b2 100644 --- a/nix/database_tool.nix +++ b/nix/database_tool.nix @@ -1,6 +1,10 @@ { src, buildDartApplication, + sqlite, + + callPackage, + path, }: buildDartApplication { pname = "jadb-database-tool"; @@ -32,5 +36,9 @@ buildDartApplication { autoPubspecLock = ../pubspec.lock; + customSourceBuilders.sqlite3 = callPackage "${path}/pkgs/development/compilers/dart/package-source-builders/sqlite3/default.nix" { + inherit sqlite; + }; + meta.mainProgram = "jadb"; }