search/word_search: score matching kana higher than converted kana

flake.lock

@@ -1,5 +1,20 @@
 {
   "nodes": {
+    "crane": {
+      "locked": {
+        "lastModified": 1780099841,
+        "narHash": "sha256-EVZd2RsbpreRUDSi9rBwPY+ZxoyMaiEBbZxxhljbaS4=",
+        "owner": "ipetkov",
+        "repo": "crane",
+        "rev": "0532eb17955225173906d671fb36306bdeb1e2dc",
+        "type": "github"
+      },
+      "original": {
+        "owner": "ipetkov",
+        "repo": "crane",
+        "type": "github"
+      }
+    },
     "datasources": {
       "inputs": {
         "nixpkgs": [
@@ -20,6 +35,28 @@
         "url": "https://git.pvv.ntnu.no/Mugiten/datasources.git"
       }
     },
+    "nix-sqlite": {
+      "inputs": {
+        "nixpkgs": [
+          "tamerye",
+          "nixpkgs"
+        ]
+      },
+      "locked": {
+        "lastModified": 1780224621,
+        "narHash": "sha256-mGAHKHEzh+J83GzYCyuM6xdfBPFYTBjNAmDmbHKRZ5U=",
+        "ref": "main",
+        "rev": "8afa09b948ec5ccbe488c21d14d458060720d313",
+        "revCount": 28,
+        "type": "git",
+        "url": "https://git.pvv.ntnu.no/mugiten/nix-custom-sqlite.git"
+      },
+      "original": {
+        "ref": "main",
+        "type": "git",
+        "url": "https://git.pvv.ntnu.no/mugiten/nix-custom-sqlite.git"
+      }
+    },
     "nixpkgs": {
       "locked": {
         "lastModified": 1779560665,
@@ -38,7 +75,53 @@
     "root": {
       "inputs": {
         "datasources": "datasources",
-        "nixpkgs": "nixpkgs"
+        "nixpkgs": "nixpkgs",
+        "tamerye": "tamerye"
+      }
+    },
+    "rust-overlay": {
+      "inputs": {
+        "nixpkgs": [
+          "tamerye",
+          "nixpkgs"
+        ]
+      },
+      "locked": {
+        "lastModified": 1780197589,
+        "narHash": "sha256-FVCr2Ij/jKf59a4LW481eeOF6rJRreOBrVgW/aUBTrw=",
+        "owner": "oxalica",
+        "repo": "rust-overlay",
+        "rev": "21632e942d89bf1cce4e5a63d7e58a215a0cbfcc",
+        "type": "github"
+      },
+      "original": {
+        "owner": "oxalica",
+        "repo": "rust-overlay",
+        "type": "github"
+      }
+    },
+    "tamerye": {
+      "inputs": {
+        "crane": "crane",
+        "nix-sqlite": "nix-sqlite",
+        "nixpkgs": [
+          "nixpkgs"
+        ],
+        "rust-overlay": "rust-overlay"
+      },
+      "locked": {
+        "lastModified": 1780505561,
+        "narHash": "sha256-534k5H8k6GcFUGy7ENsOvuZ2Lf+6WcOf3vsqfrsnNlo=",
+        "ref": "main",
+        "rev": "f9789bb582218218e29a7ac674c5edf0d3609a5d",
+        "revCount": 17,
+        "type": "git",
+        "url": "https://git.pvv.ntnu.no/Mugiten/tamerye.git"
+      },
+      "original": {
+        "ref": "main",
+        "type": "git",
+        "url": "https://git.pvv.ntnu.no/Mugiten/tamerye.git"
       }
     }
   },

flake.nix

@@ -4,6 +4,11 @@
   inputs = {
     nixpkgs.url = "nixpkgs/nixos-unstable";
 
+    tamerye = {
+      url = "git+https://git.pvv.ntnu.no/Mugiten/tamerye.git?ref=main";
+      inputs.nixpkgs.follows = "nixpkgs";
+    };
+
     datasources = {
       url = "git+https://git.pvv.ntnu.no/Mugiten/datasources.git";
       inputs.nixpkgs.follows = "nixpkgs";
@@ -13,6 +18,7 @@
   outputs = {
     self,
     nixpkgs,
+    tamerye,
     datasources,
   }: let
     inherit (nixpkgs) lib;
@@ -24,7 +30,14 @@
       "armv7l-linux"
     ];
 
-    forAllSystems = f: lib.genAttrs systems (system: f system nixpkgs.legacyPackages.${system});
+    forAllSystems = f: lib.genAttrs systems (system: let
+      pkgs = import nixpkgs {
+        inherit system;
+        overlays = [
+          tamerye.overlays.default
+        ];
+      };
+    in f system pkgs);
   in {
     apps = forAllSystems (system: pkgs: {
       default = {
@@ -35,7 +48,7 @@
 
             runtimeEnv = {
               JADB_PATH = "${self.packages.${system}.database}/jadb.sqlite";
-              LIBSQLITE_PATH = "${pkgs.sqlite.out}/lib/libsqlite3.so";
+              LIBSQLITE_PATH = "${pkgs.tamerye-sqlite}/lib/libsqlite3.so";
             };
 
             text = ''
@@ -63,18 +76,18 @@
           gnumake
           lcov
           sqldiff
-          sqlite-interactive
+          tamerye-sqlite-cli
         ];
         env = {
-          LIBSQLITE_PATH = "${pkgs.sqlite.out}/lib/libsqlite3.so";
+          LIBSQLITE_PATH = "${pkgs.tamerye-sqlite}/lib/libsqlite3.so";
           JADB_PATH = "result/jadb.sqlite";
-          LD_LIBRARY_PATH = lib.makeLibraryPath [ pkgs.sqlite ];
+          LD_LIBRARY_PATH = lib.makeLibraryPath [ pkgs.tamerye-sqlite ];
         };
       };
 
       sqlite-debugging = pkgs.mkShell {
         packages = with pkgs; [
-          sqlite-interactive
+          tamerye-sqlite-cli
           sqlite-analyzer
           sqlite-web
           # sqlint
@@ -119,16 +132,19 @@
       inherit (datasources.packages.${system}) jmdict radkfile kanjidic2;
 
       database-tool = pkgs.callPackage ./nix/database_tool.nix {
+        sqlite = pkgs.tamerye-sqlite;
         inherit src;
       };
 
       database = pkgs.callPackage ./nix/database.nix {
+        sqlite = pkgs.tamerye-sqlite-cli;
         inherit (datasources.packages.${system}) jmdict radkfile kanjidic2 tanos-jlpt;
         inherit (self.packages.${system}) database-tool;
         inherit src;
       };
 
       database-wal = pkgs.callPackage ./nix/database.nix {
+        sqlite = pkgs.tamerye-sqlite-cli;
         inherit (datasources.packages.${system}) jmdict radkfile kanjidic2 tanos-jlpt;
         inherit (self.packages.${system}) database-tool;
         inherit src;

lib/search/word_search/entry_id_query.dart

@@ -67,25 +67,28 @@ String _filterFTSSensitiveCharacters(String word) {
             SELECT DISTINCT
               "$tableName"."entryId",
               100
-              + (("${tableName}FTS"."reading" = ?) * 10000)
+              + (("$tableName"."reading" = ?1) * 100000)
+              + (("${tableName}FTS"."reading" = normalize_jp(?1)) * 10000)
+              + (("$tableName"."reading" LIKE ?1 || '%') * 20)
               + (("$tableName"."orderNum" = 0) * 20)
               + COALESCE("JMdict_EntryScore"."score", 0)
               AS "score"
             FROM "${tableName}FTS"
             JOIN "$tableName" USING ("elementId")
             LEFT JOIN "JMdict_EntryScore" USING ("elementId")
-            WHERE "${tableName}FTS"."reading" MATCH ? || '*'
+            WHERE "${tableName}FTS"."reading" MATCH normalize_jp(?1) || '*'
           ),
           non_fts_results AS (
             SELECT DISTINCT
               "$tableName"."entryId",
               50
+              + (("$tableName"."reading" LIKE '%' || ?1 || '%') * 20)
               + (("$tableName"."orderNum" = 0) * 20)
               + COALESCE("JMdict_EntryScore"."score", 0)
               AS "score"
             FROM "$tableName"
             LEFT JOIN "JMdict_EntryScore" USING ("elementId")
-            WHERE "reading" LIKE '%' || ? || '%'
+            WHERE "$tableName"."reading" LIKE '%' || normalize_jp(?1) || '%'
               AND "$tableName"."entryId" NOT IN (SELECT "entryId" FROM "fts_results")
           )
 
@@ -102,8 +105,6 @@ String _filterFTSSensitiveCharacters(String word) {
       '''
         .trim(),
     [
-      _filterFTSSensitiveCharacters(word),
-      _filterFTSSensitiveCharacters(word),
       _filterFTSSensitiveCharacters(word),
       ?pageSize,
       ?offset,

lib/version.dart

@@ -1 +1 @@
-const int jadbSchemaVersion = 1;
+const int jadbSchemaVersion = 2;

migrations/0001_JMDict.sql

@@ -66,6 +66,7 @@ CREATE TABLE "JMdict_KanjiElement" (
 ) WITHOUT ROWID;
 
 CREATE INDEX "JMdict_KanjiElement_byReading" ON "JMdict_KanjiElement"("reading");
+CREATE INDEX "JMdict_KanjiElement_byNormalizedReading" ON "JMdict_KanjiElement"(normalize_jp("reading"));
 
 CREATE TABLE "JMdict_KanjiElementInfo" (
   "elementId" INTEGER NOT NULL REFERENCES "JMdict_KanjiElement"("elementId"),
@@ -91,6 +92,7 @@ CREATE TABLE "JMdict_ReadingElement" (
 ) WITHOUT ROWID;
 
 CREATE INDEX "JMdict_ReadingElement_byReading" ON "JMdict_ReadingElement"("reading");
+CREATE INDEX "JMdict_ReadingElement_byNormalizedReading" ON "JMdict_ReadingElement"(normalize_jp("reading"));
 
 CREATE TABLE "JMdict_ReadingElementRestriction" (
   "elementId" INTEGER NOT NULL REFERENCES "JMdict_ReadingElement"("elementId"),

migrations/0003_JMDict_FTS5_Ttables.sql

@@ -4,7 +4,7 @@ CREATE TRIGGER "JMdict_KanjiElement_InsertFTS"
 AFTER INSERT ON "JMdict_KanjiElement"
 BEGIN
   INSERT INTO "JMdict_KanjiElementFTS"("elementId", "reading")
-  VALUES (NEW."elementId", NEW."reading");
+  VALUES (NEW."elementId", normalize_jp(NEW."reading"));
 END;
 
 CREATE TRIGGER "JMdict_KanjiElement_UpdateFTS"
@@ -14,7 +14,7 @@ BEGIN
   UPDATE "JMdict_KanjiElementFTS"
   SET
     "elementId" = NEW."elementId",
-    "reading" = NEW."reading"
+    "reading" = normalize_jp(NEW."reading")
   WHERE "elementId" = OLD."elementId";
 END;
 
@@ -33,7 +33,7 @@ CREATE TRIGGER "JMdict_ReadingElement_InsertFTS"
 AFTER INSERT ON "JMdict_ReadingElement"
 BEGIN
   INSERT INTO "JMdict_ReadingElementFTS"("elementId", "reading")
-  VALUES (NEW."elementId", NEW."reading");
+  VALUES (NEW."elementId", normalize_jp(NEW."reading"));
 END;
 
 CREATE TRIGGER "JMdict_ReadingElement_UpdateFTS"
@@ -43,7 +43,7 @@ BEGIN
   UPDATE "JMdict_ReadingElementFTS"
   SET
     "elementId" = NEW."elementId",
-    "reading" = NEW."reading"
+    "reading" = normalize_jp(NEW."reading")
   WHERE "elementId" = OLD."elementId";
 END;
 

nix/database_tool.nix

@@ -1,6 +1,10 @@
 {
   src,
   buildDartApplication,
+  sqlite,
+
+  callPackage,
+  path,
 }:
 buildDartApplication {
   pname = "jadb-database-tool";
@@ -32,5 +36,9 @@ buildDartApplication {
 
   autoPubspecLock = ../pubspec.lock;
 
+  customSourceBuilders.sqlite3 = callPackage "${path}/pkgs/development/compilers/dart/package-source-builders/sqlite3/default.nix" {
+    inherit sqlite;
+  };
+
   meta.mainProgram = "jadb";
 }