mugiten/mugiten
1
1
Fork 0
Code Issues 132 Pull Requests Packages Releases 6 Activity

WIP: services/database: rewrite upgrade/reset logic

Browse Source
This commit is contained in:
Oystein Kristoffer Tveit
2026-06-01 15:55:25 +09:00
parent fe00301526
commit 39c59e3a01
8 changed files with 223 additions and 143 deletions
Download Patch File Download Diff File Expand all files Collapse all files
lib/models/verify_tables.dart
+4 -8
View File
@@ -1,9 +1,10 @@
import 'package:mugiten/services/database/database.dart';
import 'package:mugiten/services/database/schemas/v2/table_names.dart';
import 'package:sqflite/sqflite.dart';
Future<void> verifyMugitenTablesWithDbConnection(
final DatabaseExecutor db,
) async {
final DatabaseExecutor db, {
final Set<String> expectedTables = allSchemaV2TableNames,
}) async {
final Set<String> tables = await db
.query(
'sqlite_master',
@@ -15,11 +16,6 @@ Future<void> verifyMugitenTablesWithDbConnection(
return result.map((final row) => row['name'] as String).toSet();
});
final Set<String> expectedTables = {
...HistoryTableNames.allTables,
...LibraryListTableNames.allTables,
};
final missingTables = expectedTables.difference(tables);
if (missingTables.isNotEmpty) {
lib/services/database/database.dart
+37 -123
View File
@@ -1,18 +1,20 @@
import 'dart:developer';
import 'dart:io';
import 'package:flutter/services.dart';
import 'package:get_it/get_it.dart';
import 'package:jadb/search.dart';
import 'package:jadb/version.dart';
import 'package:mugiten/models/verify_tables.dart';
import 'package:path/path.dart';
import 'package:path_provider/path_provider.dart';
import 'package:sqflite/sqflite.dart';
export 'package:mugiten/services/database/database_reset.dart' show resetDatabase;
export 'package:mugiten/services/database/schemas/v2/table_names.dart'
show HistoryTableNames, LibraryListTableNames;
const int expectedDatabaseVersion = 2;
const int mugitenSchemaVersion = 2;
const int schemaVersion = mugitenSchemaVersion << 16 | jadbSchemaVersion;
/// Returns the directory where mugiten's database file is stored.
Future<Directory> _databaseDir() async {
@@ -26,7 +28,10 @@ Future<String> databasePath() async {
return join((await _databaseDir()).path, 'mugiten.sqlite');
}
Future<bool> databaseNeedsInitialization() async {
/// Checks if the database needs to be reset.
///
/// This is the case if the database does not yet exist, or if it's using an old schema version.
Future<bool> databaseNeedsReset() async {
final String dbPath = await databasePath();
if (!File(dbPath).existsSync()) {
@@ -41,7 +46,7 @@ Future<bool> databaseNeedsInitialization() async {
final databaseVersion = await database.getVersion();
await database.close();
if (databaseVersion < expectedDatabaseVersion) {
if (databaseVersion < schemaVersion) {
return true;
}
@@ -53,75 +58,6 @@ Future<void> quickInitializeDatabase() async {
await setupDatabase();
}
// Migration logic and heavy initialization
class DatabaseMigration {
final String path;
final String content;
const DatabaseMigration({required this.path, required this.content});
int get version {
final String fileName = basenameWithoutExtension(path);
return int.parse(fileName.split('_')[0]);
}
@override
String toString() {
return 'DatabaseMigration(path: $path, content: ${content.length} chars)';
}
}
Future<List<DatabaseMigration>> readMigrationsFromAssets() async {
log('Reading migrations from assets...');
final assetManifest = await AssetManifest.loadFromAssetBundle(rootBundle);
final List<String> migrations = assetManifest
.listAssets()
.where(
(final assetPath) =>
RegExp(r'^migrations\/\d{4}.*\.sql$').hasMatch(assetPath),
)
.toList();
assert(migrations.isNotEmpty, 'No migration files found in assets');
migrations.sort();
log('Found ${migrations.length} migration files:');
for (final migration in migrations) {
log(' - $migration');
}
return Future.wait(
migrations.map((final migration) async {
final content = await rootBundle.loadString(migration, cache: false);
return DatabaseMigration(path: migration, content: content);
}),
);
}
/// Migrates the database from version `oldVersion` to `newVersion`.
Future<void> migrate(
final Database db,
final Iterable<DatabaseMigration> migrations,
) async {
for (final migration in migrations) {
log('Running migration ${migration.version} from ${migration.path}');
migration.content
.split(';')
.map(
(final s) => s
.split('\n')
.where((final l) => !l.startsWith(RegExp(r'\s*--')))
.join('\n')
.trim(),
)
.where((final s) => s != '')
.forEach(db.execute);
}
}
Future<Database> openDatabaseWithoutMigrations(
final String dbPath, {
@@ -131,7 +67,7 @@ Future<Database> openDatabaseWithoutMigrations(
log('Opening database at $dbPath');
final Database database = await openDatabase(
dbPath,
version: expectedDatabaseVersion,
version: mugitenSchemaVersion,
readOnly: readOnly,
onConfigure: (final db) async {
// Enable foreign key constraints
@@ -152,43 +88,32 @@ Future<Database> openDatabaseWithoutMigrations(
return database;
}
Future<Database> openAndMigrateDatabase(
final String dbPath,
final Iterable<DatabaseMigration> migrations,
) async {
log('Opening database at $dbPath');
final Database database = await openDatabase(
dbPath,
version: expectedDatabaseVersion,
readOnly: false,
onUpgrade: (final db, final oldVersion, final newVersion) async {
log('Migrating database from v$oldVersion to v$newVersion...');
final migrationsToRun = migrations
.where(
(final migration) =>
migration.version > oldVersion &&
migration.version <= newVersion,
)
.toList();
// Future<Database> openAndMigrateDatabase(
// final String dbPath,
// final Iterable<RawSQLMigration> migrations,
// ) async {
// log('Opening database at $dbPath');
// final Database database = await openDatabase(
// dbPath,
// version: mugitenSchemaVersion,
// readOnly: false,
// onUpgrade: sqfliteOnUpgradeHook,
// onConfigure: (final db) async {
// // Enable foreign key constraints
// await db.execute('PRAGMA foreign_keys=ON');
// },
// onOpen: (final db) async {
// log('Verifying jadb tables...');
// await db.jadbVerifyTables();
await migrate(db, migrationsToRun);
},
onConfigure: (final db) async {
// Enable foreign key constraints
await db.execute('PRAGMA foreign_keys=ON');
},
onOpen: (final db) async {
log('Verifying jadb tables...');
await db.jadbVerifyTables();
// log('Verifying jadb tables...');
// await verifyMugitenTablesWithDbConnection(db);
log('Verifying jadb tables...');
await verifyMugitenTablesWithDbConnection(db);
log('Database tables verified successfully');
},
);
return database;
}
// log('Database tables verified successfully');
// },
// );
// return database;
// }
/// Sets up the database, creating it if it does not exist.
Future<void> setupDatabase() async {
@@ -205,8 +130,8 @@ Future<void> setupDatabase() async {
);
assert(
await database.getVersion() == expectedDatabaseVersion,
'Database version should be $expectedDatabaseVersion',
await database.getVersion() == mugitenSchemaVersion,
'Database version should be $mugitenSchemaVersion',
);
log('Registering database in GetIt...');
@@ -214,7 +139,7 @@ Future<void> setupDatabase() async {
}
/// Resets the database by closing it, deleting the file, and setting it up again.
Future<void> resetDatabase() async {
Future<void> resetGetItDatabase() async {
log('Closing database...');
await GetIt.instance.get<Database>().close();
@@ -227,14 +152,3 @@ Future<void> resetDatabase() async {
log('Setting up database again...');
await setupDatabase();
}
/// Extracts the `jadb.sqlite` file from the assets into a writable directory
/// and returns its path.
Future<void> extractJadbFromAssets(final String path) async {
final File jadbFile = File(path)..createSync();
final ByteData data = await rootBundle.load('assets/jadb.sqlite');
await jadbFile.writeAsBytes(
data.buffer.asUint8List(data.offsetInBytes, data.lengthInBytes),
);
}
lib/services/database/database_reset.dart
+159
View File
@@ -0,0 +1,159 @@
import 'dart:async';
import 'dart:developer';
import 'dart:io';
import 'package:flutter/services.dart';
import 'package:jadb/search.dart';
import 'package:mugiten/services/database/database.dart';
import 'package:path/path.dart';
import 'package:sqflite/sqflite.dart';
/// Extracts the `jadb.sqlite` file from the assets into a writable directory
/// and returns its path.
Future<void> extractJadbFromAssets(final String path) async {
final File jadbFile = File(path)..createSync();
final ByteData data = await rootBundle.load('assets/jadb.sqlite');
await jadbFile.writeAsBytes(
data.buffer.asUint8List(data.offsetInBytes, data.lengthInBytes),
);
}
/// Dataclass representing a raw SQL migration file, containing its path and content.
class RawSQLMigration {
final String path;
final String content;
const RawSQLMigration({required this.path, required this.content});
String get name {
final String fileName = basenameWithoutExtension(path);
return fileName.split('_').sublist(1).join('_');
}
int get version {
final String fileName = basenameWithoutExtension(path);
return int.parse(fileName.split('_')[0]);
}
@override
String toString() {
return 'RawSQLMigration(version: $version, name: $name, size: ${content.length})';
}
}
/// Reads all migration files for the given database version from the assets and returns them as a list of [RawSQLMigration]s.
Future<List<RawSQLMigration>> readMigrationsForDatabaseVersionFromAssets(
final int databaseVersion,
) async {
log(
'Reading migrations for database version $databaseVersion from assets...',
);
final assetManifest = await AssetManifest.loadFromAssetBundle(rootBundle);
final List<String> migrations = assetManifest
.listAssets()
.where(
(final assetPath) => RegExp(
'^lib/services/database/schemas/v$databaseVersion/\\d{4}.*\\.sql\$',
).hasMatch(assetPath),
)
.toList();
assert(
migrations.isNotEmpty,
'No migration files found in assets for database version $databaseVersion',
);
migrations.sort();
log('Found ${migrations.length} migration files:');
for (final migration in migrations) {
log(' - $migration');
}
return Future.wait(
migrations.map((final migration) async {
final content = await rootBundle.loadString(migration, cache: false);
return RawSQLMigration(path: migration, content: content);
}),
);
}
/// Applies the given list of [RawSQLMigration]s to the provided [Database] in a single transaction.
Future<void> applyMigrations(
final Database db,
final List<RawSQLMigration> migrations,
) async {
for (final migration in migrations) {
log('Applying migration $migration');
await db.transaction((final txn) async {
migration.content
.split(';')
.map(
(final s) => s
.split('\n')
.where((final l) => !l.startsWith(RegExp(r'\s*--')))
.join('\n')
.trim(),
)
.where((final s) => s != '')
.forEach(txn.execute);
});
}
}
/// Resets the database at the given path by:
///
/// - Deleting the database file (if it exists)
/// - Extracting a fresh copy of `jadb.sqlite` from the assets
/// - Applying all schema migrations for the current schema version
Future<Database> resetDatabase(final String dbPath) async {
log('Resetting database at $dbPath...');
final File dbFile = File(dbPath);
if (dbFile.existsSync()) {
dbFile.delete();
log('Deleted existing database file at $dbPath');
}
await extractJadbFromAssets(dbPath);
log('Extracted jadb.sqlite to $dbPath');
final migrations = await readMigrationsForDatabaseVersionFromAssets(
mugitenSchemaVersion,
);
final Database database = await openDatabase(
dbPath,
version: mugitenSchemaVersion,
readOnly: false,
onUpgrade: (final db, final oldVersion, final newVersion) async {
assert(
oldVersion == 0,
'Expected oldVersion to be 0 during database reset, but got $oldVersion',
);
log('Setting up new database with schema version $newVersion...');
await applyMigrations(db, migrations);
log('Database upgrade complete');
},
onConfigure: (final db) async {
// Enable foreign key constraints
await db.execute('PRAGMA foreign_keys=ON');
},
onOpen: (final db) async {
log('Verifying jadb tables...');
await db.jadbVerifyTables();
log('Verifying mugiten tables...');
// TODO: verify mugiten tables for the exact schema version.
// await verifyMugitenTablesWithDbConnection(db);
log('Database tables verified successfully');
},
);
return database;
}
lib/services/database/schemas/v1/table_names.dart
+7 -2
View File
@@ -32,7 +32,7 @@ abstract class HistoryTableNames {
static const String historyEntryOrderedByTimestamp =
'Mugiten_HistoryEntry_orderedByTimestamp';
static Set<String> get allTables => {
static const Set<String> allTables = {
historyEntry,
historyEntryKanji,
historyEntryTimestamp,
@@ -64,9 +64,14 @@ abstract class LibraryListTableNames {
/// - name TEXT
static const String libraryListOrdered = 'Mugiten_LibraryList_Ordered';
static Set<String> get allTables => {
static const Set<String> allTables = {
libraryList,
libraryListEntry,
libraryListOrdered,
};
}
const Set<String> allSchemaV1TableNames = {
...HistoryTableNames.allTables,
...LibraryListTableNames.allTables,
};
lib/services/database/schemas/v2/table_names.dart
+7 -2
View File
@@ -32,7 +32,7 @@ abstract class HistoryTableNames {
static const String historyEntryOrderedByTimestamp =
'Mugiten_HistoryEntry_orderedByTimestamp';
static Set<String> get allTables => {
static const Set<String> allTables = {
historyEntry,
historyEntryKanji,
historyEntryTimestamp,
@@ -59,5 +59,10 @@ abstract class LibraryListTableNames {
// VIEWS //
///////////
static Set<String> get allTables => {libraryList, libraryListEntry};
static const Set<String> allTables = {libraryList, libraryListEntry};
}
const Set<String> allSchemaV2TableNames = {
...HistoryTableNames.allTables,
...LibraryListTableNames.allTables,
};
lib/services/initialization/initialization_cubit.dart
+2 -2
View File
@@ -7,7 +7,7 @@ import 'package:mugiten/services/archive/v1/format.dart'
import 'package:mugiten/services/database/database.dart'
show
DatabaseMigration,
databaseNeedsInitialization,
databaseNeedsReset,
databasePath,
extractJadbFromAssets,
openAndMigrateDatabase,
@@ -36,7 +36,7 @@ class InitializationCubit extends Cubit<InitializationStatus> {
emit(FinishDownloadMLKitDigitalInkModel());
emit(CheckDatabase());
if (deleteDatabase || await databaseNeedsInitialization()) {
if (deleteDatabase || await databaseNeedsReset()) {
final String dbPath = await databasePath();
final databaseAlreadyExists = File(dbPath).existsSync();
lib/services/initialization/initialization_logic.dart
+1 -1
View File
@@ -15,7 +15,7 @@ Future<bool> needsInitialization() async {
return true;
}
if (await databaseNeedsInitialization()) {
if (await databaseNeedsReset()) {
return true;
}
test/testutils.dart
+6 -5
View File
@@ -5,9 +5,9 @@ import 'package:flutter/material.dart';
import 'package:jadb/search.dart';
import 'package:jadb/table_names/jmdict.dart';
import 'package:jadb/table_names/kanjidic.dart';
import 'package:mugiten/services/database/database.dart';
import 'package:mugiten/models/history_entry.dart';
import 'package:mugiten/models/library_list.dart';
import 'package:mugiten/services/database/database.dart';
import 'package:sqflite_common_ffi/sqflite_ffi.dart';
Future<Database> createDatabaseCopy({
@@ -37,10 +37,11 @@ Future<Database> createDatabaseCopy({
WidgetsFlutterBinding.ensureInitialized();
return await openAndMigrateDatabase(
jadbCopyPath,
await readMigrationsFromAssets(),
);
final database = await resetDatabase(jadbCopyPath);
assert(database.isOpen, 'Failed to open database copy at $jadbCopyPath');
return database;
}
Future<List<LibraryListEntry>> createRandomLibraryListEntries({