fixup! WIP: services/database: rewrite upgrade/reset logic
This commit is contained in:
@@ -1,5 +1,290 @@
|
||||
// TODO: generate export functions for all database schema versions
|
||||
import 'dart:io';
|
||||
|
||||
// TODO: dispatch export based on current database schema version
|
||||
import 'package:archive/archive_io.dart';
|
||||
import 'package:mugiten/services/archive/sql/export_db_v1.dart' as schema_v1;
|
||||
import 'package:mugiten/services/archive/sql/export_db_v2.dart' as schema_v2;
|
||||
import 'package:mugiten/services/archive/v1/format.dart' as archive_v1;
|
||||
import 'package:mugiten/services/archive/v2/format.dart' as archive_v2;
|
||||
import 'package:sqflite/sqlite_api.dart';
|
||||
|
||||
// TODO: dispatch import based on detected archive version.
|
||||
class ArchiveTransferEvent {
|
||||
final String type;
|
||||
final int progress;
|
||||
final int total;
|
||||
|
||||
final String? name;
|
||||
final int? subProgress;
|
||||
final int? subTotal;
|
||||
|
||||
const ArchiveTransferEvent({
|
||||
required this.type,
|
||||
required this.progress,
|
||||
required this.total,
|
||||
this.name,
|
||||
this.subProgress,
|
||||
this.subTotal,
|
||||
}) : assert(
|
||||
progress > 0 && total > 0 && progress <= total,
|
||||
'0 < progress <= total must hold',
|
||||
),
|
||||
assert(
|
||||
(subProgress == null && subTotal == null) ||
|
||||
(subProgress != null &&
|
||||
subTotal != null &&
|
||||
subProgress > 0 &&
|
||||
subTotal > 0 &&
|
||||
subProgress <= subTotal),
|
||||
'subProgress and subTotal must both be null or both be positive integers with subProgress <= subTotal',
|
||||
);
|
||||
|
||||
bool get hasSubProgress => subProgress != null && subTotal != null;
|
||||
|
||||
factory ArchiveTransferEvent.fromArchiveV2StreamEvent(
|
||||
final archive_v2.ArchiveV2StreamEvent event,
|
||||
) => ArchiveTransferEvent(
|
||||
type: event.type,
|
||||
progress: event.progress,
|
||||
total: event.total,
|
||||
name: event.name,
|
||||
subProgress: event.subProgress,
|
||||
subTotal: event.subTotal,
|
||||
);
|
||||
|
||||
@override
|
||||
String toString() {
|
||||
if (!hasSubProgress) {
|
||||
return 'ArchiveTransferEvent(type: $type, progress: $progress/$total)';
|
||||
}
|
||||
|
||||
return 'ArchiveTransferEvent(type: $type, name: $name, progress: $progress/$total, subProgress: $subProgress/$subTotal)';
|
||||
}
|
||||
}
|
||||
|
||||
abstract interface class SchemaArchiveExportStrategy {
|
||||
int get schemaVersion;
|
||||
int get archiveFormatVersion;
|
||||
|
||||
Future<int> totalAmountOfChunks(final Database db);
|
||||
|
||||
Stream<ArchiveTransferEvent> export(
|
||||
final Database db,
|
||||
final File archiveFile,
|
||||
);
|
||||
}
|
||||
|
||||
abstract base class ArchiveV2SchemaExportStrategy
|
||||
implements SchemaArchiveExportStrategy {
|
||||
const ArchiveV2SchemaExportStrategy();
|
||||
|
||||
archive_v2.ArchiveV2ExportAdapter get adapter;
|
||||
|
||||
@override
|
||||
int get archiveFormatVersion => archive_v2.expectedDataFormatVersion;
|
||||
|
||||
@override
|
||||
Future<int> totalAmountOfChunks(final Database db) {
|
||||
return archive_v2.totalAmountOfChunksFromDatabaseWithAdapter(
|
||||
db,
|
||||
adapter: adapter,
|
||||
);
|
||||
}
|
||||
|
||||
@override
|
||||
Stream<ArchiveTransferEvent> export(
|
||||
final Database db,
|
||||
final File archiveFile,
|
||||
) {
|
||||
return archive_v2
|
||||
.exportDataWithAdapter(
|
||||
db,
|
||||
archiveFile,
|
||||
adapter: adapter,
|
||||
)
|
||||
.map(ArchiveTransferEvent.fromArchiveV2StreamEvent);
|
||||
}
|
||||
}
|
||||
|
||||
final class SchemaV1ToArchiveV2ExportStrategy
|
||||
extends ArchiveV2SchemaExportStrategy {
|
||||
const SchemaV1ToArchiveV2ExportStrategy();
|
||||
|
||||
@override
|
||||
int get schemaVersion => 1;
|
||||
|
||||
@override
|
||||
archive_v2.ArchiveV2ExportAdapter get adapter =>
|
||||
schema_v1.archiveExportAdapter;
|
||||
}
|
||||
|
||||
final class SchemaV2ToArchiveV2ExportStrategy
|
||||
extends ArchiveV2SchemaExportStrategy {
|
||||
const SchemaV2ToArchiveV2ExportStrategy();
|
||||
|
||||
@override
|
||||
int get schemaVersion => 2;
|
||||
|
||||
@override
|
||||
archive_v2.ArchiveV2ExportAdapter get adapter =>
|
||||
schema_v2.archiveExportAdapter;
|
||||
}
|
||||
|
||||
abstract interface class ArchiveImportStrategy {
|
||||
int get archiveFormatVersion;
|
||||
|
||||
Stream<ArchiveTransferEvent> importArchive(
|
||||
final DatabaseExecutor db,
|
||||
final File archiveFile,
|
||||
);
|
||||
}
|
||||
|
||||
final class ArchiveV1ImportStrategy implements ArchiveImportStrategy {
|
||||
const ArchiveV1ImportStrategy();
|
||||
|
||||
@override
|
||||
int get archiveFormatVersion => archive_v1.expectedDataFormatVersion;
|
||||
|
||||
@override
|
||||
Stream<ArchiveTransferEvent> importArchive(
|
||||
final DatabaseExecutor db,
|
||||
final File archiveFile,
|
||||
) async* {
|
||||
if (db is! Database) {
|
||||
throw ArgumentError.value(
|
||||
db,
|
||||
'db',
|
||||
'Archive V1 import requires a Database instance',
|
||||
);
|
||||
}
|
||||
|
||||
await archive_v1.importData(db, archiveFile);
|
||||
}
|
||||
}
|
||||
|
||||
final class ArchiveV2ImportStrategy implements ArchiveImportStrategy {
|
||||
const ArchiveV2ImportStrategy();
|
||||
|
||||
@override
|
||||
int get archiveFormatVersion => archive_v2.expectedDataFormatVersion;
|
||||
|
||||
@override
|
||||
Stream<ArchiveTransferEvent> importArchive(
|
||||
final DatabaseExecutor db,
|
||||
final File archiveFile,
|
||||
) {
|
||||
return archive_v2
|
||||
.importData(db, archiveFile)
|
||||
.map(ArchiveTransferEvent.fromArchiveV2StreamEvent);
|
||||
}
|
||||
}
|
||||
|
||||
final Map<int, SchemaArchiveExportStrategy> _latestExportStrategyBySchema = {
|
||||
1: const SchemaV1ToArchiveV2ExportStrategy(),
|
||||
2: const SchemaV2ToArchiveV2ExportStrategy(),
|
||||
};
|
||||
|
||||
final Map<int, ArchiveImportStrategy> _importStrategiesByArchiveVersion = {
|
||||
archive_v1.expectedDataFormatVersion: const ArchiveV1ImportStrategy(),
|
||||
archive_v2.expectedDataFormatVersion: const ArchiveV2ImportStrategy(),
|
||||
};
|
||||
|
||||
/// Extracts Mugiten's schema version from SQLite's `user_version`.
|
||||
///
|
||||
/// Older development databases may still store the plain Mugiten schema version
|
||||
/// directly instead of the packed Mugiten+jadb version. Those are treated as-is.
|
||||
Future<int> detectMugitenSchemaVersion(final Database db) async {
|
||||
final userVersion = await db.getVersion();
|
||||
|
||||
if (userVersion <= 0xFFFF) {
|
||||
return userVersion;
|
||||
}
|
||||
|
||||
return userVersion >> 16;
|
||||
}
|
||||
|
||||
SchemaArchiveExportStrategy _exportStrategyForSchema(final int schemaVersion) {
|
||||
final strategy = _latestExportStrategyBySchema[schemaVersion];
|
||||
if (strategy == null) {
|
||||
throw UnsupportedError(
|
||||
'No archive export strategy registered for Mugiten schema version $schemaVersion',
|
||||
);
|
||||
}
|
||||
|
||||
return strategy;
|
||||
}
|
||||
|
||||
Future<int> totalAmountOfChunksFromDatabase(final Database db) async {
|
||||
final schemaVersion = await detectMugitenSchemaVersion(db);
|
||||
final strategy = _exportStrategyForSchema(schemaVersion);
|
||||
return strategy.totalAmountOfChunks(db);
|
||||
}
|
||||
|
||||
Future<int> detectArchiveVersion(final File archiveFile) async {
|
||||
if (!archiveFile.existsSync()) {
|
||||
throw Exception('Archive file does not exist: ${archiveFile.path}');
|
||||
}
|
||||
|
||||
final archive = ZipDecoder().decodeStream(InputFileStream(archiveFile.path));
|
||||
|
||||
try {
|
||||
for (final file in archive) {
|
||||
if (!file.isFile || file.name != 'version.txt') {
|
||||
continue;
|
||||
}
|
||||
|
||||
final bytes = file.readBytes();
|
||||
if (bytes == null) {
|
||||
throw FormatException(
|
||||
'Archive version file is empty: ${archiveFile.path}',
|
||||
);
|
||||
}
|
||||
|
||||
return int.parse(String.fromCharCodes(bytes).trim());
|
||||
}
|
||||
} finally {
|
||||
for (final file in archive) {
|
||||
file.closeSync();
|
||||
}
|
||||
}
|
||||
|
||||
throw FormatException(
|
||||
'Archive is missing version.txt: ${archiveFile.path}',
|
||||
);
|
||||
}
|
||||
|
||||
ArchiveImportStrategy _importStrategyForArchiveVersion(final int archiveVersion) {
|
||||
final strategy = _importStrategiesByArchiveVersion[archiveVersion];
|
||||
if (strategy == null) {
|
||||
throw UnsupportedError(
|
||||
'No archive import strategy registered for archive version $archiveVersion',
|
||||
);
|
||||
}
|
||||
|
||||
return strategy;
|
||||
}
|
||||
|
||||
/// Exports a migration backup using the newest archive format that the source
|
||||
/// database schema supports.
|
||||
Stream<ArchiveTransferEvent> exportBackupArchive(
|
||||
final Database db,
|
||||
final File archiveFile,
|
||||
) async* {
|
||||
final schemaVersion = await detectMugitenSchemaVersion(db);
|
||||
final strategy = _exportStrategyForSchema(schemaVersion);
|
||||
|
||||
yield* strategy.export(db, archiveFile);
|
||||
}
|
||||
|
||||
/// Imports an archive into the latest database schema.
|
||||
///
|
||||
/// Migration backups are normally written using the latest archive format, but
|
||||
/// older archive versions are also accepted here so that old user archives can
|
||||
/// be restored through the same dispatcher.
|
||||
Stream<ArchiveTransferEvent> importBackupArchive(
|
||||
final DatabaseExecutor db,
|
||||
final File archiveFile,
|
||||
) async* {
|
||||
final archiveVersion = await detectArchiveVersion(archiveFile);
|
||||
final strategy = _importStrategyForArchiveVersion(archiveVersion);
|
||||
|
||||
yield* strategy.importArchive(db, archiveFile);
|
||||
}
|
||||
|
||||
@@ -1,6 +1,14 @@
|
||||
import 'package:mugiten/services/archive/v2/format.dart';
|
||||
import 'package:mugiten/services/database/schemas/v1/table_names.dart';
|
||||
import 'package:sqflite/sqflite.dart';
|
||||
import 'package:sqflite/sqlite_api.dart';
|
||||
|
||||
const archiveExportAdapter = ArchiveV2ExportAdapter(
|
||||
historyEntryCount: historyEntryCount,
|
||||
historyEntryGetAll: historyEntryGetAll,
|
||||
libraryListGetLibraryMetadata: libraryListGetLibraryMetadata,
|
||||
libraryListGetTotalCounts: libraryListGetTotalCounts,
|
||||
libraryListGetEntries: libraryListGetEntries,
|
||||
);
|
||||
|
||||
Future<int> historyEntryCount(final DatabaseExecutor db) async {
|
||||
final result = await db.rawQuery('''
|
||||
@@ -55,10 +63,10 @@ Future<List<ArchiveV2LibraryListMetadata>> libraryListGetLibraryMetadata({
|
||||
required final DatabaseExecutor db,
|
||||
}) async {
|
||||
final result = await db.query(
|
||||
LibraryListTableNames.libraryList,
|
||||
LibraryListTableNames.libraryListOrdered,
|
||||
columns: ['name'],
|
||||
orderBy: '"name" ASC',
|
||||
);
|
||||
|
||||
return result
|
||||
.map(
|
||||
(final row) =>
|
||||
@@ -72,13 +80,13 @@ Future<Map<String, int>> libraryListGetTotalCounts({
|
||||
}) async {
|
||||
final result = await db.rawQuery('''
|
||||
SELECT
|
||||
"listName",
|
||||
"lists"."name" AS "listName",
|
||||
(
|
||||
SELECT COUNT(*)
|
||||
FROM "${LibraryListTableNames.libraryListEntry}"
|
||||
WHERE "${LibraryListTableNames.libraryListEntry}"."listName" = "${LibraryListTableNames.libraryList}"."name"
|
||||
FROM "${LibraryListTableNames.libraryListEntry}" AS "entries"
|
||||
WHERE "entries"."listName" = "lists"."name"
|
||||
) AS "count"
|
||||
FROM "${LibraryListTableNames.libraryList}"
|
||||
FROM "${LibraryListTableNames.libraryListOrdered}" AS "lists"
|
||||
''');
|
||||
|
||||
final counts = {
|
||||
@@ -92,7 +100,6 @@ Future<List<ArchiveV2LibraryListEntry>> libraryListGetEntries({
|
||||
required final DatabaseExecutor db,
|
||||
required final String listName,
|
||||
required final int page,
|
||||
required final int pageSize,
|
||||
}) async {
|
||||
final result = await db.rawQuery(
|
||||
'''
|
||||
@@ -132,13 +139,15 @@ Future<List<ArchiveV2LibraryListEntry>> libraryListGetEntries({
|
||||
LIMIT ?
|
||||
OFFSET ?
|
||||
''',
|
||||
[listName, listName, pageSize, page * pageSize],
|
||||
[listName, listName, libraryListChunkSize, page * libraryListChunkSize],
|
||||
);
|
||||
|
||||
final entries = result
|
||||
.map(
|
||||
(final entry) => ArchiveV2LibraryListEntry(
|
||||
lastModified: DateTime.parse(entry['lastModified'] as String),
|
||||
lastModified: DateTime.fromMillisecondsSinceEpoch(
|
||||
entry['lastModified'] as int,
|
||||
),
|
||||
jmdictEntryId: entry['jmdictEntryId'] as int?,
|
||||
kanji: entry['kanji'] as String?,
|
||||
),
|
||||
|
||||
@@ -1,9 +1,14 @@
|
||||
import 'package:mugiten/services/archive/v2/format.dart';
|
||||
import 'package:mugiten/services/database/schemas/v2/table_names.dart';
|
||||
import 'package:sqflite/sqflite.dart';
|
||||
import 'package:sqflite/sqlite_api.dart';
|
||||
|
||||
const int basicListOrderNumInterval = 100;
|
||||
const int defaultLibraryListPageSize = 100;
|
||||
const archiveExportAdapter = ArchiveV2ExportAdapter(
|
||||
historyEntryCount: historyEntryCount,
|
||||
historyEntryGetAll: historyEntryGetAll,
|
||||
libraryListGetLibraryMetadata: libraryListGetLibraryMetadata,
|
||||
libraryListGetTotalCounts: libraryListGetTotalCounts,
|
||||
libraryListGetEntries: libraryListGetEntries,
|
||||
);
|
||||
|
||||
Future<int> historyEntryCount(final DatabaseExecutor db) async {
|
||||
final result = await db.rawQuery('''
|
||||
@@ -60,8 +65,9 @@ Future<List<ArchiveV2LibraryListMetadata>> libraryListGetLibraryMetadata({
|
||||
final result = await db.query(
|
||||
LibraryListTableNames.libraryList,
|
||||
columns: ['name'],
|
||||
orderBy: '"name" ASC',
|
||||
orderBy: '"orderNum" ASC',
|
||||
);
|
||||
|
||||
return result
|
||||
.map(
|
||||
(final row) =>
|
||||
@@ -75,13 +81,14 @@ Future<Map<String, int>> libraryListGetTotalCounts({
|
||||
}) async {
|
||||
final result = await db.rawQuery('''
|
||||
SELECT
|
||||
"listName",
|
||||
"lists"."name" AS "listName",
|
||||
(
|
||||
SELECT COUNT(*)
|
||||
FROM "${LibraryListTableNames.libraryListEntry}"
|
||||
WHERE "${LibraryListTableNames.libraryListEntry}"."listName" = "${LibraryListTableNames.libraryList}"."name"
|
||||
FROM "${LibraryListTableNames.libraryListEntry}" AS "entries"
|
||||
WHERE "entries"."listName" = "lists"."name"
|
||||
) AS "count"
|
||||
FROM "${LibraryListTableNames.libraryList}"
|
||||
FROM "${LibraryListTableNames.libraryList}" AS "lists"
|
||||
ORDER BY "lists"."orderNum" ASC
|
||||
''');
|
||||
|
||||
final counts = {
|
||||
@@ -96,9 +103,8 @@ Future<List<ArchiveV2LibraryListEntry>> libraryListGetEntries({
|
||||
required final String listName,
|
||||
required final int page,
|
||||
}) async {
|
||||
final offset = basicListOrderNumInterval * (basicListOrderNumInterval * page);
|
||||
final limit =
|
||||
offset + (basicListOrderNumInterval * defaultLibraryListPageSize);
|
||||
final offset = libraryListChunkSize * libraryListChunkSize * page;
|
||||
final limit = offset + (libraryListChunkSize * libraryListChunkSize);
|
||||
|
||||
final result = await db.rawQuery(
|
||||
'''
|
||||
@@ -118,7 +124,9 @@ Future<List<ArchiveV2LibraryListEntry>> libraryListGetEntries({
|
||||
final entries = result
|
||||
.map(
|
||||
(final entry) => ArchiveV2LibraryListEntry(
|
||||
lastModified: DateTime.parse(entry['lastModified'] as String),
|
||||
lastModified: DateTime.fromMillisecondsSinceEpoch(
|
||||
entry['lastModified'] as int,
|
||||
),
|
||||
jmdictEntryId: entry['jmdictEntryId'] as int?,
|
||||
kanji: entry['kanji'] as String?,
|
||||
),
|
||||
|
||||
@@ -20,6 +20,92 @@ const int expectedDataFormatVersion = 2;
|
||||
const int historyChunkSize = 100;
|
||||
const int libraryListChunkSize = defaultLibraryListPageSize;
|
||||
|
||||
typedef ArchiveV2HistoryEntryCountQuery =
|
||||
Future<int> Function(DatabaseExecutor db);
|
||||
|
||||
typedef ArchiveV2HistoryEntryGetAllQuery =
|
||||
Future<List<ArchiveV2HistoryEntry>> Function({
|
||||
required DatabaseExecutor db,
|
||||
required int page,
|
||||
required int pageSize,
|
||||
});
|
||||
|
||||
typedef ArchiveV2LibraryListGetLibraryMetadataQuery =
|
||||
Future<List<ArchiveV2LibraryListMetadata>> Function({
|
||||
required DatabaseExecutor db,
|
||||
});
|
||||
|
||||
typedef ArchiveV2LibraryListGetTotalCountsQuery =
|
||||
Future<Map<String, int>> Function({required DatabaseExecutor db});
|
||||
|
||||
typedef ArchiveV2LibraryListGetEntriesQuery =
|
||||
Future<List<ArchiveV2LibraryListEntry>> Function({
|
||||
required DatabaseExecutor db,
|
||||
required String listName,
|
||||
required int page,
|
||||
});
|
||||
|
||||
/// An adapter that provides the necessary functions to export data from the database
|
||||
/// in the format expected by version 2 of the data archive.
|
||||
class ArchiveV2ExportAdapter {
|
||||
final ArchiveV2HistoryEntryCountQuery historyEntryCount;
|
||||
final ArchiveV2HistoryEntryGetAllQuery historyEntryGetAll;
|
||||
final ArchiveV2LibraryListGetLibraryMetadataQuery
|
||||
libraryListGetLibraryMetadata;
|
||||
final ArchiveV2LibraryListGetTotalCountsQuery libraryListGetTotalCounts;
|
||||
final ArchiveV2LibraryListGetEntriesQuery libraryListGetEntries;
|
||||
|
||||
const ArchiveV2ExportAdapter({
|
||||
required this.historyEntryCount,
|
||||
required this.historyEntryGetAll,
|
||||
required this.libraryListGetLibraryMetadata,
|
||||
required this.libraryListGetTotalCounts,
|
||||
required this.libraryListGetEntries,
|
||||
});
|
||||
}
|
||||
|
||||
final ArchiveV2ExportAdapter latestSchemaExportAdapter = ArchiveV2ExportAdapter(
|
||||
historyEntryCount: (final db) => db.historyEntryAmount(),
|
||||
historyEntryGetAll:
|
||||
({
|
||||
required final DatabaseExecutor db,
|
||||
required final int page,
|
||||
required final int pageSize,
|
||||
}) async {
|
||||
return (await db.historyEntryGetAll(
|
||||
page: page,
|
||||
pageSize: pageSize,
|
||||
)).map(ArchiveV2HistoryEntry.fromHistoryEntry).toList();
|
||||
},
|
||||
libraryListGetLibraryMetadata: ({required final DatabaseExecutor db}) async {
|
||||
return (await db.libraryListGetLists())
|
||||
.map((final list) => ArchiveV2LibraryListMetadata(name: list.name))
|
||||
.toList();
|
||||
},
|
||||
libraryListGetTotalCounts: ({required final DatabaseExecutor db}) async {
|
||||
final lists = await db.libraryListGetLists();
|
||||
return {for (final list in lists) list.name: list.totalCount};
|
||||
},
|
||||
libraryListGetEntries:
|
||||
({
|
||||
required final DatabaseExecutor db,
|
||||
required final String listName,
|
||||
required final int page,
|
||||
}) async {
|
||||
final entryPage = await db.libraryListGetListEntries(
|
||||
listName,
|
||||
page: page,
|
||||
);
|
||||
if (entryPage == null) {
|
||||
return <ArchiveV2LibraryListEntry>[];
|
||||
}
|
||||
|
||||
return entryPage.entries
|
||||
.map(ArchiveV2LibraryListEntry.fromLibraryListEntry)
|
||||
.toList();
|
||||
},
|
||||
);
|
||||
|
||||
/// Functions and properties that makes up the format of version 2 of the data archive.
|
||||
/// This archive is used to back up user data and optionally to transfer data between devices.
|
||||
/// The main difference to version 1 is that the data is split into chunks, so that it can be
|
||||
@@ -77,15 +163,16 @@ extension ArchiveFormatV2 on Directory {
|
||||
File get libraryMetadataFile =>
|
||||
File(libraryDir.uri.resolve('metadata.json').toFilePath());
|
||||
|
||||
/// The metadata of all library lists
|
||||
/// The metadata of all library lists.
|
||||
///
|
||||
/// This is expected to be a list of objects, containing:
|
||||
/// - *order*: implicitly from the order of the json list, the index of the library list
|
||||
/// - name: the original name of the library list
|
||||
/// - slug: the slugified name of the library list, used for the directory name
|
||||
Map<String, Object?> get libraryMetadata =>
|
||||
jsonDecode(libraryMetadataFile.readAsStringSync())
|
||||
as Map<String, Object?>;
|
||||
List<Map<String, Object?>> get libraryMetadata =>
|
||||
(jsonDecode(libraryMetadataFile.readAsStringSync()) as List<dynamic>)
|
||||
.map((final entry) => entry as Map<String, Object?>)
|
||||
.toList();
|
||||
|
||||
List<Directory> get libraryListDirs =>
|
||||
libraryDir.listSync().whereType<Directory>().toList();
|
||||
@@ -102,24 +189,36 @@ extension ArchiveFormatV2 on Directory {
|
||||
);
|
||||
|
||||
List<int> get libraryListEntryCounts => libraryListDirs
|
||||
.map(
|
||||
(final d) =>
|
||||
d.listSync().whereType<File>().length -
|
||||
1, // Subtract 1 for metadata.json
|
||||
)
|
||||
.map((final d) => d.listSync().whereType<File>().length)
|
||||
.toList();
|
||||
}
|
||||
|
||||
String slugifyLibraryListFileName(final String name) =>
|
||||
name.toLowerCase().replaceAll(RegExp(r'\s+'), '_');
|
||||
|
||||
Future<int> totalAmountOfChunksFromDatabase(final DatabaseExecutor db) async {
|
||||
final historyCount = await db.historyEntryAmount();
|
||||
final libraryListCounts = (await db.libraryListGetLists())
|
||||
.map((final list) => (list.totalCount / libraryListChunkSize).ceil())
|
||||
Future<int> totalAmountOfChunksFromDatabase(final DatabaseExecutor db) {
|
||||
return totalAmountOfChunksFromDatabaseWithAdapter(
|
||||
db,
|
||||
adapter: latestSchemaExportAdapter,
|
||||
);
|
||||
}
|
||||
|
||||
Future<int> totalAmountOfChunksFromDatabaseWithAdapter(
|
||||
final DatabaseExecutor db, {
|
||||
required final ArchiveV2ExportAdapter adapter,
|
||||
}) async {
|
||||
final historyCount = await adapter.historyEntryCount(db);
|
||||
final libraryMetadata = await adapter.libraryListGetLibraryMetadata(db: db);
|
||||
final libraryListCounts = await adapter.libraryListGetTotalCounts(db: db);
|
||||
|
||||
final libraryChunks = libraryMetadata
|
||||
.map(
|
||||
(final list) =>
|
||||
((libraryListCounts[list.name] ?? 0) / libraryListChunkSize).ceil(),
|
||||
)
|
||||
.sum;
|
||||
|
||||
return (historyCount / historyChunkSize).ceil() + libraryListCounts;
|
||||
return (historyCount / historyChunkSize).ceil() + libraryChunks;
|
||||
}
|
||||
|
||||
// TODO: skip counting chunks where the library list already exists
|
||||
@@ -196,7 +295,19 @@ class ArchiveV2StreamEvent {
|
||||
Stream<ArchiveV2StreamEvent> exportData(
|
||||
final DatabaseExecutor db,
|
||||
final File archiveFile,
|
||||
) async* {
|
||||
) {
|
||||
return exportDataWithAdapter(
|
||||
db,
|
||||
archiveFile,
|
||||
adapter: latestSchemaExportAdapter,
|
||||
);
|
||||
}
|
||||
|
||||
Stream<ArchiveV2StreamEvent> exportDataWithAdapter(
|
||||
final DatabaseExecutor db,
|
||||
final File archiveFile, {
|
||||
required final ArchiveV2ExportAdapter adapter,
|
||||
}) async* {
|
||||
if (!archiveFile.existsSync()) {
|
||||
archiveFile.createSync();
|
||||
}
|
||||
@@ -207,8 +318,8 @@ Stream<ArchiveV2StreamEvent> exportData(
|
||||
archiveRoot,
|
||||
).versionFile.writeAsString(expectedDataFormatVersion.toString());
|
||||
|
||||
yield* exportHistory(db, archiveRoot);
|
||||
yield* exportLibraryLists(db, archiveRoot);
|
||||
yield* exportHistory(db, archiveRoot, adapter: adapter);
|
||||
yield* exportLibraryLists(db, archiveRoot, adapter: adapter);
|
||||
|
||||
await packZip(archiveRoot, outputFile: archiveFile);
|
||||
|
||||
|
||||
@@ -91,26 +91,39 @@ class ArchiveV2HistorySearchInstance {
|
||||
|
||||
/// Calculate the total number of chunks needed to export the history,
|
||||
/// needed for progress tracking during export.
|
||||
Future<int> exportHistoryChunkCount(final DatabaseExecutor db) async =>
|
||||
(await db.historyEntryAmount() / historyChunkSize).ceil();
|
||||
Future<int> exportHistoryChunkCount(
|
||||
final DatabaseExecutor db, {
|
||||
final ArchiveV2ExportAdapter? adapter,
|
||||
}) async =>
|
||||
(await (adapter ?? latestSchemaExportAdapter).historyEntryCount(db) /
|
||||
historyChunkSize)
|
||||
.ceil();
|
||||
|
||||
/// Exports the history into json files in the given directory.
|
||||
///
|
||||
/// Streams back the number of chunks that have been exported so far.
|
||||
Stream<ArchiveV2StreamEvent> exportHistory(
|
||||
final DatabaseExecutor db,
|
||||
final Directory archiveRoot,
|
||||
) async* {
|
||||
final int chunkCount = await exportHistoryChunkCount(db);
|
||||
final Directory archiveRoot, {
|
||||
final ArchiveV2ExportAdapter? adapter,
|
||||
}) async* {
|
||||
final exportAdapter = adapter ?? latestSchemaExportAdapter;
|
||||
final int chunkCount = await exportHistoryChunkCount(
|
||||
db,
|
||||
adapter: exportAdapter,
|
||||
);
|
||||
|
||||
archiveRoot.historyDir.createSync();
|
||||
|
||||
for (int i = 0; i < chunkCount; i++) {
|
||||
final List<Map<String, Object?>> jsonEntries =
|
||||
(await db.historyEntryGetAll(page: i, pageSize: historyChunkSize))
|
||||
.map(ArchiveV2HistoryEntry.fromHistoryEntry)
|
||||
.map((final e) => e.toJson())
|
||||
.toList();
|
||||
final List<Map<String, Object?>> jsonEntries = (await exportAdapter
|
||||
.historyEntryGetAll(
|
||||
db: db,
|
||||
page: i,
|
||||
pageSize: historyChunkSize,
|
||||
))
|
||||
.map((final e) => e.toJson())
|
||||
.toList();
|
||||
|
||||
archiveRoot.historyChunkFile(i)
|
||||
..createSync()
|
||||
|
||||
@@ -53,23 +53,44 @@ class ArchiveV2LibraryListEntry {
|
||||
);
|
||||
}
|
||||
|
||||
/// Exports metadata about library lists, such as their names and order, into the archive.
|
||||
Future<void> exportLibraryMetadata(
|
||||
final DatabaseExecutor db,
|
||||
final Directory archiveRoot,
|
||||
) async {
|
||||
final libraryLists = await db.libraryListGetLists();
|
||||
final List<ArchiveV2LibraryListMetadata> metadataList = libraryLists
|
||||
class _ArchiveV2ExportLibraryList {
|
||||
final ArchiveV2LibraryListMetadata metadata;
|
||||
final int totalCount;
|
||||
|
||||
const _ArchiveV2ExportLibraryList({
|
||||
required this.metadata,
|
||||
required this.totalCount,
|
||||
});
|
||||
}
|
||||
|
||||
Future<List<_ArchiveV2ExportLibraryList>> _getExportLibraryLists(
|
||||
final DatabaseExecutor db, {
|
||||
required final ArchiveV2ExportAdapter adapter,
|
||||
}) async {
|
||||
final metadata = await adapter.libraryListGetLibraryMetadata(db: db);
|
||||
final counts = await adapter.libraryListGetTotalCounts(db: db);
|
||||
|
||||
return metadata
|
||||
.map(
|
||||
(final libraryList) => ArchiveV2LibraryListMetadata(
|
||||
name: libraryList.name,
|
||||
slug: slugifyLibraryListFileName(libraryList.name),
|
||||
(final meta) => _ArchiveV2ExportLibraryList(
|
||||
metadata: meta,
|
||||
totalCount: counts[meta.name] ?? 0,
|
||||
),
|
||||
)
|
||||
.toList();
|
||||
}
|
||||
|
||||
/// Exports metadata about library lists, such as their names and order, into the archive.
|
||||
Future<void> exportLibraryMetadata(
|
||||
final Directory archiveRoot,
|
||||
final List<_ArchiveV2ExportLibraryList> libraryLists,
|
||||
) async {
|
||||
final metadataFile = archiveRoot.libraryMetadataFile..createSync();
|
||||
await metadataFile.writeAsString(jsonEncode(metadataList));
|
||||
await metadataFile.writeAsString(
|
||||
jsonEncode(
|
||||
libraryLists.map((final libraryList) => libraryList.metadata.toJson()).toList(),
|
||||
),
|
||||
);
|
||||
}
|
||||
|
||||
List<ArchiveV2LibraryListMetadata> importLibraryMetadata(
|
||||
@@ -89,13 +110,17 @@ List<ArchiveV2LibraryListMetadata> importLibraryMetadata(
|
||||
|
||||
/// Calculate the total number of chunks needed to export all library lists,
|
||||
/// needed for progress tracking during export.
|
||||
Future<int> exportLibraryListChunkCount(final DatabaseExecutor db) async =>
|
||||
(await db.libraryListGetLists())
|
||||
.map(
|
||||
(final libraryList) =>
|
||||
(libraryList.totalCount / libraryListChunkSize).ceil(),
|
||||
)
|
||||
.reduce((final a, final b) => a + b);
|
||||
Future<int> exportLibraryListChunkCount(
|
||||
final DatabaseExecutor db, {
|
||||
final ArchiveV2ExportAdapter? adapter,
|
||||
}) async =>
|
||||
(await _getExportLibraryLists(
|
||||
db,
|
||||
adapter: adapter ?? latestSchemaExportAdapter,
|
||||
)).map(
|
||||
(final libraryList) =>
|
||||
(libraryList.totalCount / libraryListChunkSize).ceil(),
|
||||
).sum;
|
||||
|
||||
/// Exports all library lists into json files in the given directory.
|
||||
///
|
||||
@@ -103,13 +128,19 @@ Future<int> exportLibraryListChunkCount(final DatabaseExecutor db) async =>
|
||||
/// See also [exportLibraryListChunkCount].
|
||||
Stream<ArchiveV2StreamEvent> exportLibraryLists(
|
||||
final DatabaseExecutor db,
|
||||
final Directory archiveRoot,
|
||||
) async* {
|
||||
final Directory archiveRoot, {
|
||||
final ArchiveV2ExportAdapter? adapter,
|
||||
}) async* {
|
||||
final exportAdapter = adapter ?? latestSchemaExportAdapter;
|
||||
|
||||
archiveRoot.libraryDir.createSync();
|
||||
|
||||
await exportLibraryMetadata(db, archiveRoot);
|
||||
final libraryLists = await _getExportLibraryLists(
|
||||
db,
|
||||
adapter: exportAdapter,
|
||||
);
|
||||
|
||||
final libraryLists = await db.libraryListGetLists();
|
||||
await exportLibraryMetadata(archiveRoot, libraryLists);
|
||||
|
||||
for (final (i, libraryList) in libraryLists.indexed) {
|
||||
yield* exportLibraryList(
|
||||
@@ -118,6 +149,7 @@ Stream<ArchiveV2StreamEvent> exportLibraryLists(
|
||||
libraryList,
|
||||
i + 1,
|
||||
libraryLists.length,
|
||||
adapter: exportAdapter,
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -128,34 +160,35 @@ Stream<ArchiveV2StreamEvent> exportLibraryLists(
|
||||
Stream<ArchiveV2StreamEvent> exportLibraryList(
|
||||
final DatabaseExecutor db,
|
||||
final Directory archiveRoot,
|
||||
final LibraryList libraryList,
|
||||
final _ArchiveV2ExportLibraryList libraryList,
|
||||
final int index,
|
||||
final int total,
|
||||
) async* {
|
||||
final int totalEntries = libraryList.totalCount;
|
||||
final int chunkCount = (totalEntries / libraryListChunkSize).ceil();
|
||||
final int total, {
|
||||
final ArchiveV2ExportAdapter? adapter,
|
||||
}) async* {
|
||||
final exportAdapter = adapter ?? latestSchemaExportAdapter;
|
||||
final listName = libraryList.metadata.name;
|
||||
final int chunkCount = (libraryList.totalCount / libraryListChunkSize).ceil();
|
||||
|
||||
archiveRoot.libraryListDir(libraryList.name).createSync();
|
||||
archiveRoot.libraryListDir(listName).createSync();
|
||||
|
||||
for (int i = 0; i < chunkCount; i++) {
|
||||
final entryPage = (await db.libraryListGetListEntries(
|
||||
libraryList.name,
|
||||
final archiveEntries = await exportAdapter.libraryListGetEntries(
|
||||
db: db,
|
||||
listName: listName,
|
||||
page: i,
|
||||
))!;
|
||||
);
|
||||
|
||||
final archiveEntries = entryPage.entries
|
||||
.map(ArchiveV2LibraryListEntry.fromLibraryListEntry)
|
||||
.toList();
|
||||
|
||||
archiveRoot.libraryListChunkFile(libraryList.name, i)
|
||||
archiveRoot.libraryListChunkFile(listName, i)
|
||||
..createSync()
|
||||
..writeAsStringSync(jsonEncode(archiveEntries));
|
||||
..writeAsStringSync(
|
||||
jsonEncode(archiveEntries.map((final e) => e.toJson()).toList()),
|
||||
);
|
||||
|
||||
yield ArchiveV2StreamEvent(
|
||||
type: 'library',
|
||||
progress: index,
|
||||
total: total,
|
||||
name: libraryList.name,
|
||||
name: listName,
|
||||
subProgress: i + 1,
|
||||
subTotal: chunkCount,
|
||||
);
|
||||
@@ -210,7 +243,13 @@ Stream<ArchiveV2StreamEvent> importLibraryList(
|
||||
final int index,
|
||||
final int total,
|
||||
) async* {
|
||||
final chunkFiles = libraryListDir.listSync().whereType<File>();
|
||||
final chunkFiles = libraryListDir.listSync().whereType<File>().sortedBy(
|
||||
(final file) =>
|
||||
int.tryParse(
|
||||
file.uri.pathSegments.last.replaceFirst(RegExp(r'\.json$'), ''),
|
||||
) ??
|
||||
0,
|
||||
);
|
||||
|
||||
for (final (i, chunkFile) in chunkFiles.indexed) {
|
||||
final chunkContent = chunkFile.readAsStringSync();
|
||||
|
||||
Reference in New Issue
Block a user