diff --git a/flake.lock b/flake.lock index 1e62e8c..555a52c 100644 --- a/flake.lock +++ b/flake.lock @@ -1,5 +1,20 @@ { "nodes": { + "crane": { + "locked": { + "lastModified": 1780532242, + "narHash": "sha256-D+BsdpxmtUwtqGoY0IXPhHgTlmqgcZKCEo1oMyn7ep0=", + "owner": "ipetkov", + "repo": "crane", + "rev": "59a82a1222dd3b2080b5cc52a1a2e8d5f1b77f37", + "type": "github" + }, + "original": { + "owner": "ipetkov", + "repo": "crane", + "type": "github" + } + }, "datasources": { "inputs": { "nixpkgs": [ @@ -20,6 +35,28 @@ "url": "https://git.pvv.ntnu.no/Mugiten/datasources.git" } }, + "nix-sqlite": { + "inputs": { + "nixpkgs": [ + "tamerye", + "nixpkgs" + ] + }, + "locked": { + "lastModified": 1780568901, + "narHash": "sha256-r1z2EW6b1uzwOr4c2dnh7aLplZOE6JFeix/BGCQWISs=", + "ref": "main", + "rev": "0458b14f481ddd626e03f7b1cd4be4344960fd29", + "revCount": 29, + "type": "git", + "url": "https://git.pvv.ntnu.no/mugiten/nix-custom-sqlite.git" + }, + "original": { + "ref": "main", + "type": "git", + "url": "https://git.pvv.ntnu.no/mugiten/nix-custom-sqlite.git" + } + }, "nixpkgs": { "locked": { "lastModified": 1779560665, @@ -38,7 +75,53 @@ "root": { "inputs": { "datasources": "datasources", - "nixpkgs": "nixpkgs" + "nixpkgs": "nixpkgs", + "tamerye": "tamerye" + } + }, + "rust-overlay": { + "inputs": { + "nixpkgs": [ + "tamerye", + "nixpkgs" + ] + }, + "locked": { + "lastModified": 1780543271, + "narHash": "sha256-oPJ7eJN1sM37v92Rp/eyQL7/rUm0BOvXEBAoq/zN0cM=", + "owner": "oxalica", + "repo": "rust-overlay", + "rev": "c30ca201c5093540cf792f6982f81ba1aa0f3514", + "type": "github" + }, + "original": { + "owner": "oxalica", + "repo": "rust-overlay", + "type": "github" + } + }, + "tamerye": { + "inputs": { + "crane": "crane", + "nix-sqlite": "nix-sqlite", + "nixpkgs": [ + "nixpkgs" + ], + "rust-overlay": "rust-overlay" + }, + "locked": { + "lastModified": 1780568945, + "narHash": "sha256-yXz+M1KrfFheEFYOe9ZtgOG1J+mHIvTPs1o36X3zhjw=", + "ref": "main", + "rev": "a609ea342e54b4302bfed6291912100903b95619", + "revCount": 31, + "type": "git", + "url": "https://git.pvv.ntnu.no/Mugiten/tamerye.git" + }, + "original": { + "ref": "main", + "type": "git", + "url": "https://git.pvv.ntnu.no/Mugiten/tamerye.git" } } }, diff --git a/flake.nix b/flake.nix index b7c7f88..09c2019 100644 --- a/flake.nix +++ b/flake.nix @@ -4,6 +4,11 @@ inputs = { nixpkgs.url = "nixpkgs/nixos-unstable"; + tamerye = { + url = "git+https://git.pvv.ntnu.no/Mugiten/tamerye.git?ref=main"; + inputs.nixpkgs.follows = "nixpkgs"; + }; + datasources = { url = "git+https://git.pvv.ntnu.no/Mugiten/datasources.git"; inputs.nixpkgs.follows = "nixpkgs"; @@ -13,6 +18,7 @@ outputs = { self, nixpkgs, + tamerye, datasources, }: let inherit (nixpkgs) lib; @@ -24,7 +30,14 @@ "armv7l-linux" ]; - forAllSystems = f: lib.genAttrs systems (system: f system nixpkgs.legacyPackages.${system}); + forAllSystems = f: lib.genAttrs systems (system: let + pkgs = import nixpkgs { + inherit system; + overlays = [ + tamerye.overlays.default + ]; + }; + in f system pkgs); in { apps = forAllSystems (system: pkgs: { default = { @@ -35,7 +48,7 @@ runtimeEnv = { JADB_PATH = "${self.packages.${system}.database}/jadb.sqlite"; - LIBSQLITE_PATH = "${pkgs.sqlite.out}/lib/libsqlite3.so"; + LIBSQLITE_PATH = "${pkgs.tamerye-sqlite}/lib/libsqlite3.so"; }; text = '' @@ -63,18 +76,18 @@ gnumake lcov sqldiff - sqlite-interactive + tamerye-sqlite-cli ]; env = { - LIBSQLITE_PATH = "${pkgs.sqlite.out}/lib/libsqlite3.so"; + LIBSQLITE_PATH = "${pkgs.tamerye-sqlite}/lib/libsqlite3.so"; JADB_PATH = "result/jadb.sqlite"; - LD_LIBRARY_PATH = lib.makeLibraryPath [ pkgs.sqlite ]; + LD_LIBRARY_PATH = lib.makeLibraryPath [ pkgs.tamerye-sqlite ]; }; }; sqlite-debugging = pkgs.mkShell { packages = with pkgs; [ - sqlite-interactive + tamerye-sqlite-cli sqlite-analyzer sqlite-web # sqlint @@ -119,16 +132,19 @@ inherit (datasources.packages.${system}) jmdict radkfile kanjidic2; database-tool = pkgs.callPackage ./nix/database_tool.nix { + sqlite = pkgs.tamerye-sqlite; inherit src; }; database = pkgs.callPackage ./nix/database.nix { + sqlite = pkgs.tamerye-sqlite-cli; inherit (datasources.packages.${system}) jmdict radkfile kanjidic2 tanos-jlpt; inherit (self.packages.${system}) database-tool; inherit src; }; database-wal = pkgs.callPackage ./nix/database.nix { + sqlite = pkgs.tamerye-sqlite-cli; inherit (datasources.packages.${system}) jmdict radkfile kanjidic2 tanos-jlpt; inherit (self.packages.${system}) database-tool; inherit src; diff --git a/lib/search/word_search/entry_id_query.dart b/lib/search/word_search/entry_id_query.dart index 9b3ab86..a939aee 100644 --- a/lib/search/word_search/entry_id_query.dart +++ b/lib/search/word_search/entry_id_query.dart @@ -67,14 +67,14 @@ String _filterFTSSensitiveCharacters(String word) { SELECT DISTINCT "$tableName"."entryId", 100 - + (("${tableName}FTS"."reading" = ?) * 10000) + + (("${tableName}FTS"."reading" = normalize_jp(?)) * 10000) + (("$tableName"."orderNum" = 0) * 20) + COALESCE("JMdict_EntryScore"."score", 0) AS "score" FROM "${tableName}FTS" JOIN "$tableName" USING ("elementId") LEFT JOIN "JMdict_EntryScore" USING ("elementId") - WHERE "${tableName}FTS"."reading" MATCH ? || '*' + WHERE "${tableName}FTS"."reading" MATCH normalize_jp(?) || '*' ), non_fts_results AS ( SELECT DISTINCT @@ -85,7 +85,7 @@ String _filterFTSSensitiveCharacters(String word) { AS "score" FROM "$tableName" LEFT JOIN "JMdict_EntryScore" USING ("elementId") - WHERE "reading" LIKE '%' || ? || '%' + WHERE "reading" LIKE '%' || normalize_jp(?) || '%' AND "$tableName"."entryId" NOT IN (SELECT "entryId" FROM "fts_results") ) diff --git a/lib/version.dart b/lib/version.dart index ecddb63..c7e1e60 100644 --- a/lib/version.dart +++ b/lib/version.dart @@ -1 +1 @@ -const int jadbSchemaVersion = 1; +const int jadbSchemaVersion = 2; diff --git a/migrations/0001_JMDict.sql b/migrations/0001_JMDict.sql index c67fb9b..9367053 100644 --- a/migrations/0001_JMDict.sql +++ b/migrations/0001_JMDict.sql @@ -66,6 +66,7 @@ CREATE TABLE "JMdict_KanjiElement" ( ) WITHOUT ROWID; CREATE INDEX "JMdict_KanjiElement_byReading" ON "JMdict_KanjiElement"("reading"); +CREATE INDEX "JMdict_KanjiElement_byNormalizedReading" ON "JMdict_KanjiElement"(normalize_jp("reading")); CREATE TABLE "JMdict_KanjiElementInfo" ( "elementId" INTEGER NOT NULL REFERENCES "JMdict_KanjiElement"("elementId"), @@ -91,6 +92,7 @@ CREATE TABLE "JMdict_ReadingElement" ( ) WITHOUT ROWID; CREATE INDEX "JMdict_ReadingElement_byReading" ON "JMdict_ReadingElement"("reading"); +CREATE INDEX "JMdict_ReadingElement_byNormalizedReading" ON "JMdict_ReadingElement"(normalize_jp("reading")); CREATE TABLE "JMdict_ReadingElementRestriction" ( "elementId" INTEGER NOT NULL REFERENCES "JMdict_ReadingElement"("elementId"), diff --git a/migrations/0003_JMDict_FTS5_Ttables.sql b/migrations/0003_JMDict_FTS5_Ttables.sql index 1fcf540..1fccb76 100644 --- a/migrations/0003_JMDict_FTS5_Ttables.sql +++ b/migrations/0003_JMDict_FTS5_Ttables.sql @@ -4,7 +4,7 @@ CREATE TRIGGER "JMdict_KanjiElement_InsertFTS" AFTER INSERT ON "JMdict_KanjiElement" BEGIN INSERT INTO "JMdict_KanjiElementFTS"("elementId", "reading") - VALUES (NEW."elementId", NEW."reading"); + VALUES (NEW."elementId", normalize_jp(NEW."reading")); END; CREATE TRIGGER "JMdict_KanjiElement_UpdateFTS" @@ -14,7 +14,7 @@ BEGIN UPDATE "JMdict_KanjiElementFTS" SET "elementId" = NEW."elementId", - "reading" = NEW."reading" + "reading" = normalize_jp(NEW."reading") WHERE "elementId" = OLD."elementId"; END; @@ -33,7 +33,7 @@ CREATE TRIGGER "JMdict_ReadingElement_InsertFTS" AFTER INSERT ON "JMdict_ReadingElement" BEGIN INSERT INTO "JMdict_ReadingElementFTS"("elementId", "reading") - VALUES (NEW."elementId", NEW."reading"); + VALUES (NEW."elementId", normalize_jp(NEW."reading")); END; CREATE TRIGGER "JMdict_ReadingElement_UpdateFTS" @@ -43,7 +43,7 @@ BEGIN UPDATE "JMdict_ReadingElementFTS" SET "elementId" = NEW."elementId", - "reading" = NEW."reading" + "reading" = normalize_jp(NEW."reading") WHERE "elementId" = OLD."elementId"; END; diff --git a/nix/database_tool.nix b/nix/database_tool.nix index 3fffb74..531e9b2 100644 --- a/nix/database_tool.nix +++ b/nix/database_tool.nix @@ -1,6 +1,10 @@ { src, buildDartApplication, + sqlite, + + callPackage, + path, }: buildDartApplication { pname = "jadb-database-tool"; @@ -32,5 +36,9 @@ buildDartApplication { autoPubspecLock = ../pubspec.lock; + customSourceBuilders.sqlite3 = callPackage "${path}/pkgs/development/compilers/dart/package-source-builders/sqlite3/default.nix" { + inherit sqlite; + }; + meta.mainProgram = "jadb"; }