fixup! WIP: services/database: rewrite upgrade/reset logic

This commit is contained in:
2026-06-01 22:47:38 +09:00
parent 39c59e3a01
commit e669253ca9
6 changed files with 556 additions and 91 deletions
+288 -3
View File
@@ -1,5 +1,290 @@
// TODO: generate export functions for all database schema versions
import 'dart:io';
// TODO: dispatch export based on current database schema version
import 'package:archive/archive_io.dart';
import 'package:mugiten/services/archive/sql/export_db_v1.dart' as schema_v1;
import 'package:mugiten/services/archive/sql/export_db_v2.dart' as schema_v2;
import 'package:mugiten/services/archive/v1/format.dart' as archive_v1;
import 'package:mugiten/services/archive/v2/format.dart' as archive_v2;
import 'package:sqflite/sqlite_api.dart';
// TODO: dispatch import based on detected archive version.
class ArchiveTransferEvent {
final String type;
final int progress;
final int total;
final String? name;
final int? subProgress;
final int? subTotal;
const ArchiveTransferEvent({
required this.type,
required this.progress,
required this.total,
this.name,
this.subProgress,
this.subTotal,
}) : assert(
progress > 0 && total > 0 && progress <= total,
'0 < progress <= total must hold',
),
assert(
(subProgress == null && subTotal == null) ||
(subProgress != null &&
subTotal != null &&
subProgress > 0 &&
subTotal > 0 &&
subProgress <= subTotal),
'subProgress and subTotal must both be null or both be positive integers with subProgress <= subTotal',
);
bool get hasSubProgress => subProgress != null && subTotal != null;
factory ArchiveTransferEvent.fromArchiveV2StreamEvent(
final archive_v2.ArchiveV2StreamEvent event,
) => ArchiveTransferEvent(
type: event.type,
progress: event.progress,
total: event.total,
name: event.name,
subProgress: event.subProgress,
subTotal: event.subTotal,
);
@override
String toString() {
if (!hasSubProgress) {
return 'ArchiveTransferEvent(type: $type, progress: $progress/$total)';
}
return 'ArchiveTransferEvent(type: $type, name: $name, progress: $progress/$total, subProgress: $subProgress/$subTotal)';
}
}
abstract interface class SchemaArchiveExportStrategy {
int get schemaVersion;
int get archiveFormatVersion;
Future<int> totalAmountOfChunks(final Database db);
Stream<ArchiveTransferEvent> export(
final Database db,
final File archiveFile,
);
}
abstract base class ArchiveV2SchemaExportStrategy
implements SchemaArchiveExportStrategy {
const ArchiveV2SchemaExportStrategy();
archive_v2.ArchiveV2ExportAdapter get adapter;
@override
int get archiveFormatVersion => archive_v2.expectedDataFormatVersion;
@override
Future<int> totalAmountOfChunks(final Database db) {
return archive_v2.totalAmountOfChunksFromDatabaseWithAdapter(
db,
adapter: adapter,
);
}
@override
Stream<ArchiveTransferEvent> export(
final Database db,
final File archiveFile,
) {
return archive_v2
.exportDataWithAdapter(
db,
archiveFile,
adapter: adapter,
)
.map(ArchiveTransferEvent.fromArchiveV2StreamEvent);
}
}
final class SchemaV1ToArchiveV2ExportStrategy
extends ArchiveV2SchemaExportStrategy {
const SchemaV1ToArchiveV2ExportStrategy();
@override
int get schemaVersion => 1;
@override
archive_v2.ArchiveV2ExportAdapter get adapter =>
schema_v1.archiveExportAdapter;
}
final class SchemaV2ToArchiveV2ExportStrategy
extends ArchiveV2SchemaExportStrategy {
const SchemaV2ToArchiveV2ExportStrategy();
@override
int get schemaVersion => 2;
@override
archive_v2.ArchiveV2ExportAdapter get adapter =>
schema_v2.archiveExportAdapter;
}
abstract interface class ArchiveImportStrategy {
int get archiveFormatVersion;
Stream<ArchiveTransferEvent> importArchive(
final DatabaseExecutor db,
final File archiveFile,
);
}
final class ArchiveV1ImportStrategy implements ArchiveImportStrategy {
const ArchiveV1ImportStrategy();
@override
int get archiveFormatVersion => archive_v1.expectedDataFormatVersion;
@override
Stream<ArchiveTransferEvent> importArchive(
final DatabaseExecutor db,
final File archiveFile,
) async* {
if (db is! Database) {
throw ArgumentError.value(
db,
'db',
'Archive V1 import requires a Database instance',
);
}
await archive_v1.importData(db, archiveFile);
}
}
final class ArchiveV2ImportStrategy implements ArchiveImportStrategy {
const ArchiveV2ImportStrategy();
@override
int get archiveFormatVersion => archive_v2.expectedDataFormatVersion;
@override
Stream<ArchiveTransferEvent> importArchive(
final DatabaseExecutor db,
final File archiveFile,
) {
return archive_v2
.importData(db, archiveFile)
.map(ArchiveTransferEvent.fromArchiveV2StreamEvent);
}
}
final Map<int, SchemaArchiveExportStrategy> _latestExportStrategyBySchema = {
1: const SchemaV1ToArchiveV2ExportStrategy(),
2: const SchemaV2ToArchiveV2ExportStrategy(),
};
final Map<int, ArchiveImportStrategy> _importStrategiesByArchiveVersion = {
archive_v1.expectedDataFormatVersion: const ArchiveV1ImportStrategy(),
archive_v2.expectedDataFormatVersion: const ArchiveV2ImportStrategy(),
};
/// Extracts Mugiten's schema version from SQLite's `user_version`.
///
/// Older development databases may still store the plain Mugiten schema version
/// directly instead of the packed Mugiten+jadb version. Those are treated as-is.
Future<int> detectMugitenSchemaVersion(final Database db) async {
final userVersion = await db.getVersion();
if (userVersion <= 0xFFFF) {
return userVersion;
}
return userVersion >> 16;
}
SchemaArchiveExportStrategy _exportStrategyForSchema(final int schemaVersion) {
final strategy = _latestExportStrategyBySchema[schemaVersion];
if (strategy == null) {
throw UnsupportedError(
'No archive export strategy registered for Mugiten schema version $schemaVersion',
);
}
return strategy;
}
Future<int> totalAmountOfChunksFromDatabase(final Database db) async {
final schemaVersion = await detectMugitenSchemaVersion(db);
final strategy = _exportStrategyForSchema(schemaVersion);
return strategy.totalAmountOfChunks(db);
}
Future<int> detectArchiveVersion(final File archiveFile) async {
if (!archiveFile.existsSync()) {
throw Exception('Archive file does not exist: ${archiveFile.path}');
}
final archive = ZipDecoder().decodeStream(InputFileStream(archiveFile.path));
try {
for (final file in archive) {
if (!file.isFile || file.name != 'version.txt') {
continue;
}
final bytes = file.readBytes();
if (bytes == null) {
throw FormatException(
'Archive version file is empty: ${archiveFile.path}',
);
}
return int.parse(String.fromCharCodes(bytes).trim());
}
} finally {
for (final file in archive) {
file.closeSync();
}
}
throw FormatException(
'Archive is missing version.txt: ${archiveFile.path}',
);
}
ArchiveImportStrategy _importStrategyForArchiveVersion(final int archiveVersion) {
final strategy = _importStrategiesByArchiveVersion[archiveVersion];
if (strategy == null) {
throw UnsupportedError(
'No archive import strategy registered for archive version $archiveVersion',
);
}
return strategy;
}
/// Exports a migration backup using the newest archive format that the source
/// database schema supports.
Stream<ArchiveTransferEvent> exportBackupArchive(
final Database db,
final File archiveFile,
) async* {
final schemaVersion = await detectMugitenSchemaVersion(db);
final strategy = _exportStrategyForSchema(schemaVersion);
yield* strategy.export(db, archiveFile);
}
/// Imports an archive into the latest database schema.
///
/// Migration backups are normally written using the latest archive format, but
/// older archive versions are also accepted here so that old user archives can
/// be restored through the same dispatcher.
Stream<ArchiveTransferEvent> importBackupArchive(
final DatabaseExecutor db,
final File archiveFile,
) async* {
final archiveVersion = await detectArchiveVersion(archiveFile);
final strategy = _importStrategyForArchiveVersion(archiveVersion);
yield* strategy.importArchive(db, archiveFile);
}
+19 -10
View File
@@ -1,6 +1,14 @@
import 'package:mugiten/services/archive/v2/format.dart';
import 'package:mugiten/services/database/schemas/v1/table_names.dart';
import 'package:sqflite/sqflite.dart';
import 'package:sqflite/sqlite_api.dart';
const archiveExportAdapter = ArchiveV2ExportAdapter(
historyEntryCount: historyEntryCount,
historyEntryGetAll: historyEntryGetAll,
libraryListGetLibraryMetadata: libraryListGetLibraryMetadata,
libraryListGetTotalCounts: libraryListGetTotalCounts,
libraryListGetEntries: libraryListGetEntries,
);
Future<int> historyEntryCount(final DatabaseExecutor db) async {
final result = await db.rawQuery('''
@@ -55,10 +63,10 @@ Future<List<ArchiveV2LibraryListMetadata>> libraryListGetLibraryMetadata({
required final DatabaseExecutor db,
}) async {
final result = await db.query(
LibraryListTableNames.libraryList,
LibraryListTableNames.libraryListOrdered,
columns: ['name'],
orderBy: '"name" ASC',
);
return result
.map(
(final row) =>
@@ -72,13 +80,13 @@ Future<Map<String, int>> libraryListGetTotalCounts({
}) async {
final result = await db.rawQuery('''
SELECT
"listName",
"lists"."name" AS "listName",
(
SELECT COUNT(*)
FROM "${LibraryListTableNames.libraryListEntry}"
WHERE "${LibraryListTableNames.libraryListEntry}"."listName" = "${LibraryListTableNames.libraryList}"."name"
FROM "${LibraryListTableNames.libraryListEntry}" AS "entries"
WHERE "entries"."listName" = "lists"."name"
) AS "count"
FROM "${LibraryListTableNames.libraryList}"
FROM "${LibraryListTableNames.libraryListOrdered}" AS "lists"
''');
final counts = {
@@ -92,7 +100,6 @@ Future<List<ArchiveV2LibraryListEntry>> libraryListGetEntries({
required final DatabaseExecutor db,
required final String listName,
required final int page,
required final int pageSize,
}) async {
final result = await db.rawQuery(
'''
@@ -132,13 +139,15 @@ Future<List<ArchiveV2LibraryListEntry>> libraryListGetEntries({
LIMIT ?
OFFSET ?
''',
[listName, listName, pageSize, page * pageSize],
[listName, listName, libraryListChunkSize, page * libraryListChunkSize],
);
final entries = result
.map(
(final entry) => ArchiveV2LibraryListEntry(
lastModified: DateTime.parse(entry['lastModified'] as String),
lastModified: DateTime.fromMillisecondsSinceEpoch(
entry['lastModified'] as int,
),
jmdictEntryId: entry['jmdictEntryId'] as int?,
kanji: entry['kanji'] as String?,
),
+20 -12
View File
@@ -1,9 +1,14 @@
import 'package:mugiten/services/archive/v2/format.dart';
import 'package:mugiten/services/database/schemas/v2/table_names.dart';
import 'package:sqflite/sqflite.dart';
import 'package:sqflite/sqlite_api.dart';
const int basicListOrderNumInterval = 100;
const int defaultLibraryListPageSize = 100;
const archiveExportAdapter = ArchiveV2ExportAdapter(
historyEntryCount: historyEntryCount,
historyEntryGetAll: historyEntryGetAll,
libraryListGetLibraryMetadata: libraryListGetLibraryMetadata,
libraryListGetTotalCounts: libraryListGetTotalCounts,
libraryListGetEntries: libraryListGetEntries,
);
Future<int> historyEntryCount(final DatabaseExecutor db) async {
final result = await db.rawQuery('''
@@ -60,8 +65,9 @@ Future<List<ArchiveV2LibraryListMetadata>> libraryListGetLibraryMetadata({
final result = await db.query(
LibraryListTableNames.libraryList,
columns: ['name'],
orderBy: '"name" ASC',
orderBy: '"orderNum" ASC',
);
return result
.map(
(final row) =>
@@ -75,13 +81,14 @@ Future<Map<String, int>> libraryListGetTotalCounts({
}) async {
final result = await db.rawQuery('''
SELECT
"listName",
"lists"."name" AS "listName",
(
SELECT COUNT(*)
FROM "${LibraryListTableNames.libraryListEntry}"
WHERE "${LibraryListTableNames.libraryListEntry}"."listName" = "${LibraryListTableNames.libraryList}"."name"
FROM "${LibraryListTableNames.libraryListEntry}" AS "entries"
WHERE "entries"."listName" = "lists"."name"
) AS "count"
FROM "${LibraryListTableNames.libraryList}"
FROM "${LibraryListTableNames.libraryList}" AS "lists"
ORDER BY "lists"."orderNum" ASC
''');
final counts = {
@@ -96,9 +103,8 @@ Future<List<ArchiveV2LibraryListEntry>> libraryListGetEntries({
required final String listName,
required final int page,
}) async {
final offset = basicListOrderNumInterval * (basicListOrderNumInterval * page);
final limit =
offset + (basicListOrderNumInterval * defaultLibraryListPageSize);
final offset = libraryListChunkSize * libraryListChunkSize * page;
final limit = offset + (libraryListChunkSize * libraryListChunkSize);
final result = await db.rawQuery(
'''
@@ -118,7 +124,9 @@ Future<List<ArchiveV2LibraryListEntry>> libraryListGetEntries({
final entries = result
.map(
(final entry) => ArchiveV2LibraryListEntry(
lastModified: DateTime.parse(entry['lastModified'] as String),
lastModified: DateTime.fromMillisecondsSinceEpoch(
entry['lastModified'] as int,
),
jmdictEntryId: entry['jmdictEntryId'] as int?,
kanji: entry['kanji'] as String?,
),
+128 -17
View File
@@ -20,6 +20,92 @@ const int expectedDataFormatVersion = 2;
const int historyChunkSize = 100;
const int libraryListChunkSize = defaultLibraryListPageSize;
typedef ArchiveV2HistoryEntryCountQuery =
Future<int> Function(DatabaseExecutor db);
typedef ArchiveV2HistoryEntryGetAllQuery =
Future<List<ArchiveV2HistoryEntry>> Function({
required DatabaseExecutor db,
required int page,
required int pageSize,
});
typedef ArchiveV2LibraryListGetLibraryMetadataQuery =
Future<List<ArchiveV2LibraryListMetadata>> Function({
required DatabaseExecutor db,
});
typedef ArchiveV2LibraryListGetTotalCountsQuery =
Future<Map<String, int>> Function({required DatabaseExecutor db});
typedef ArchiveV2LibraryListGetEntriesQuery =
Future<List<ArchiveV2LibraryListEntry>> Function({
required DatabaseExecutor db,
required String listName,
required int page,
});
/// An adapter that provides the necessary functions to export data from the database
/// in the format expected by version 2 of the data archive.
class ArchiveV2ExportAdapter {
final ArchiveV2HistoryEntryCountQuery historyEntryCount;
final ArchiveV2HistoryEntryGetAllQuery historyEntryGetAll;
final ArchiveV2LibraryListGetLibraryMetadataQuery
libraryListGetLibraryMetadata;
final ArchiveV2LibraryListGetTotalCountsQuery libraryListGetTotalCounts;
final ArchiveV2LibraryListGetEntriesQuery libraryListGetEntries;
const ArchiveV2ExportAdapter({
required this.historyEntryCount,
required this.historyEntryGetAll,
required this.libraryListGetLibraryMetadata,
required this.libraryListGetTotalCounts,
required this.libraryListGetEntries,
});
}
final ArchiveV2ExportAdapter latestSchemaExportAdapter = ArchiveV2ExportAdapter(
historyEntryCount: (final db) => db.historyEntryAmount(),
historyEntryGetAll:
({
required final DatabaseExecutor db,
required final int page,
required final int pageSize,
}) async {
return (await db.historyEntryGetAll(
page: page,
pageSize: pageSize,
)).map(ArchiveV2HistoryEntry.fromHistoryEntry).toList();
},
libraryListGetLibraryMetadata: ({required final DatabaseExecutor db}) async {
return (await db.libraryListGetLists())
.map((final list) => ArchiveV2LibraryListMetadata(name: list.name))
.toList();
},
libraryListGetTotalCounts: ({required final DatabaseExecutor db}) async {
final lists = await db.libraryListGetLists();
return {for (final list in lists) list.name: list.totalCount};
},
libraryListGetEntries:
({
required final DatabaseExecutor db,
required final String listName,
required final int page,
}) async {
final entryPage = await db.libraryListGetListEntries(
listName,
page: page,
);
if (entryPage == null) {
return <ArchiveV2LibraryListEntry>[];
}
return entryPage.entries
.map(ArchiveV2LibraryListEntry.fromLibraryListEntry)
.toList();
},
);
/// Functions and properties that makes up the format of version 2 of the data archive.
/// This archive is used to back up user data and optionally to transfer data between devices.
/// The main difference to version 1 is that the data is split into chunks, so that it can be
@@ -77,15 +163,16 @@ extension ArchiveFormatV2 on Directory {
File get libraryMetadataFile =>
File(libraryDir.uri.resolve('metadata.json').toFilePath());
/// The metadata of all library lists
/// The metadata of all library lists.
///
/// This is expected to be a list of objects, containing:
/// - *order*: implicitly from the order of the json list, the index of the library list
/// - name: the original name of the library list
/// - slug: the slugified name of the library list, used for the directory name
Map<String, Object?> get libraryMetadata =>
jsonDecode(libraryMetadataFile.readAsStringSync())
as Map<String, Object?>;
List<Map<String, Object?>> get libraryMetadata =>
(jsonDecode(libraryMetadataFile.readAsStringSync()) as List<dynamic>)
.map((final entry) => entry as Map<String, Object?>)
.toList();
List<Directory> get libraryListDirs =>
libraryDir.listSync().whereType<Directory>().toList();
@@ -102,24 +189,36 @@ extension ArchiveFormatV2 on Directory {
);
List<int> get libraryListEntryCounts => libraryListDirs
.map(
(final d) =>
d.listSync().whereType<File>().length -
1, // Subtract 1 for metadata.json
)
.map((final d) => d.listSync().whereType<File>().length)
.toList();
}
String slugifyLibraryListFileName(final String name) =>
name.toLowerCase().replaceAll(RegExp(r'\s+'), '_');
Future<int> totalAmountOfChunksFromDatabase(final DatabaseExecutor db) async {
final historyCount = await db.historyEntryAmount();
final libraryListCounts = (await db.libraryListGetLists())
.map((final list) => (list.totalCount / libraryListChunkSize).ceil())
Future<int> totalAmountOfChunksFromDatabase(final DatabaseExecutor db) {
return totalAmountOfChunksFromDatabaseWithAdapter(
db,
adapter: latestSchemaExportAdapter,
);
}
Future<int> totalAmountOfChunksFromDatabaseWithAdapter(
final DatabaseExecutor db, {
required final ArchiveV2ExportAdapter adapter,
}) async {
final historyCount = await adapter.historyEntryCount(db);
final libraryMetadata = await adapter.libraryListGetLibraryMetadata(db: db);
final libraryListCounts = await adapter.libraryListGetTotalCounts(db: db);
final libraryChunks = libraryMetadata
.map(
(final list) =>
((libraryListCounts[list.name] ?? 0) / libraryListChunkSize).ceil(),
)
.sum;
return (historyCount / historyChunkSize).ceil() + libraryListCounts;
return (historyCount / historyChunkSize).ceil() + libraryChunks;
}
// TODO: skip counting chunks where the library list already exists
@@ -196,7 +295,19 @@ class ArchiveV2StreamEvent {
Stream<ArchiveV2StreamEvent> exportData(
final DatabaseExecutor db,
final File archiveFile,
) async* {
) {
return exportDataWithAdapter(
db,
archiveFile,
adapter: latestSchemaExportAdapter,
);
}
Stream<ArchiveV2StreamEvent> exportDataWithAdapter(
final DatabaseExecutor db,
final File archiveFile, {
required final ArchiveV2ExportAdapter adapter,
}) async* {
if (!archiveFile.existsSync()) {
archiveFile.createSync();
}
@@ -207,8 +318,8 @@ Stream<ArchiveV2StreamEvent> exportData(
archiveRoot,
).versionFile.writeAsString(expectedDataFormatVersion.toString());
yield* exportHistory(db, archiveRoot);
yield* exportLibraryLists(db, archiveRoot);
yield* exportHistory(db, archiveRoot, adapter: adapter);
yield* exportLibraryLists(db, archiveRoot, adapter: adapter);
await packZip(archiveRoot, outputFile: archiveFile);
+23 -10
View File
@@ -91,26 +91,39 @@ class ArchiveV2HistorySearchInstance {
/// Calculate the total number of chunks needed to export the history,
/// needed for progress tracking during export.
Future<int> exportHistoryChunkCount(final DatabaseExecutor db) async =>
(await db.historyEntryAmount() / historyChunkSize).ceil();
Future<int> exportHistoryChunkCount(
final DatabaseExecutor db, {
final ArchiveV2ExportAdapter? adapter,
}) async =>
(await (adapter ?? latestSchemaExportAdapter).historyEntryCount(db) /
historyChunkSize)
.ceil();
/// Exports the history into json files in the given directory.
///
/// Streams back the number of chunks that have been exported so far.
Stream<ArchiveV2StreamEvent> exportHistory(
final DatabaseExecutor db,
final Directory archiveRoot,
) async* {
final int chunkCount = await exportHistoryChunkCount(db);
final Directory archiveRoot, {
final ArchiveV2ExportAdapter? adapter,
}) async* {
final exportAdapter = adapter ?? latestSchemaExportAdapter;
final int chunkCount = await exportHistoryChunkCount(
db,
adapter: exportAdapter,
);
archiveRoot.historyDir.createSync();
for (int i = 0; i < chunkCount; i++) {
final List<Map<String, Object?>> jsonEntries =
(await db.historyEntryGetAll(page: i, pageSize: historyChunkSize))
.map(ArchiveV2HistoryEntry.fromHistoryEntry)
.map((final e) => e.toJson())
.toList();
final List<Map<String, Object?>> jsonEntries = (await exportAdapter
.historyEntryGetAll(
db: db,
page: i,
pageSize: historyChunkSize,
))
.map((final e) => e.toJson())
.toList();
archiveRoot.historyChunkFile(i)
..createSync()
+78 -39
View File
@@ -53,23 +53,44 @@ class ArchiveV2LibraryListEntry {
);
}
/// Exports metadata about library lists, such as their names and order, into the archive.
Future<void> exportLibraryMetadata(
final DatabaseExecutor db,
final Directory archiveRoot,
) async {
final libraryLists = await db.libraryListGetLists();
final List<ArchiveV2LibraryListMetadata> metadataList = libraryLists
class _ArchiveV2ExportLibraryList {
final ArchiveV2LibraryListMetadata metadata;
final int totalCount;
const _ArchiveV2ExportLibraryList({
required this.metadata,
required this.totalCount,
});
}
Future<List<_ArchiveV2ExportLibraryList>> _getExportLibraryLists(
final DatabaseExecutor db, {
required final ArchiveV2ExportAdapter adapter,
}) async {
final metadata = await adapter.libraryListGetLibraryMetadata(db: db);
final counts = await adapter.libraryListGetTotalCounts(db: db);
return metadata
.map(
(final libraryList) => ArchiveV2LibraryListMetadata(
name: libraryList.name,
slug: slugifyLibraryListFileName(libraryList.name),
(final meta) => _ArchiveV2ExportLibraryList(
metadata: meta,
totalCount: counts[meta.name] ?? 0,
),
)
.toList();
}
/// Exports metadata about library lists, such as their names and order, into the archive.
Future<void> exportLibraryMetadata(
final Directory archiveRoot,
final List<_ArchiveV2ExportLibraryList> libraryLists,
) async {
final metadataFile = archiveRoot.libraryMetadataFile..createSync();
await metadataFile.writeAsString(jsonEncode(metadataList));
await metadataFile.writeAsString(
jsonEncode(
libraryLists.map((final libraryList) => libraryList.metadata.toJson()).toList(),
),
);
}
List<ArchiveV2LibraryListMetadata> importLibraryMetadata(
@@ -89,13 +110,17 @@ List<ArchiveV2LibraryListMetadata> importLibraryMetadata(
/// Calculate the total number of chunks needed to export all library lists,
/// needed for progress tracking during export.
Future<int> exportLibraryListChunkCount(final DatabaseExecutor db) async =>
(await db.libraryListGetLists())
.map(
(final libraryList) =>
(libraryList.totalCount / libraryListChunkSize).ceil(),
)
.reduce((final a, final b) => a + b);
Future<int> exportLibraryListChunkCount(
final DatabaseExecutor db, {
final ArchiveV2ExportAdapter? adapter,
}) async =>
(await _getExportLibraryLists(
db,
adapter: adapter ?? latestSchemaExportAdapter,
)).map(
(final libraryList) =>
(libraryList.totalCount / libraryListChunkSize).ceil(),
).sum;
/// Exports all library lists into json files in the given directory.
///
@@ -103,13 +128,19 @@ Future<int> exportLibraryListChunkCount(final DatabaseExecutor db) async =>
/// See also [exportLibraryListChunkCount].
Stream<ArchiveV2StreamEvent> exportLibraryLists(
final DatabaseExecutor db,
final Directory archiveRoot,
) async* {
final Directory archiveRoot, {
final ArchiveV2ExportAdapter? adapter,
}) async* {
final exportAdapter = adapter ?? latestSchemaExportAdapter;
archiveRoot.libraryDir.createSync();
await exportLibraryMetadata(db, archiveRoot);
final libraryLists = await _getExportLibraryLists(
db,
adapter: exportAdapter,
);
final libraryLists = await db.libraryListGetLists();
await exportLibraryMetadata(archiveRoot, libraryLists);
for (final (i, libraryList) in libraryLists.indexed) {
yield* exportLibraryList(
@@ -118,6 +149,7 @@ Stream<ArchiveV2StreamEvent> exportLibraryLists(
libraryList,
i + 1,
libraryLists.length,
adapter: exportAdapter,
);
}
}
@@ -128,34 +160,35 @@ Stream<ArchiveV2StreamEvent> exportLibraryLists(
Stream<ArchiveV2StreamEvent> exportLibraryList(
final DatabaseExecutor db,
final Directory archiveRoot,
final LibraryList libraryList,
final _ArchiveV2ExportLibraryList libraryList,
final int index,
final int total,
) async* {
final int totalEntries = libraryList.totalCount;
final int chunkCount = (totalEntries / libraryListChunkSize).ceil();
final int total, {
final ArchiveV2ExportAdapter? adapter,
}) async* {
final exportAdapter = adapter ?? latestSchemaExportAdapter;
final listName = libraryList.metadata.name;
final int chunkCount = (libraryList.totalCount / libraryListChunkSize).ceil();
archiveRoot.libraryListDir(libraryList.name).createSync();
archiveRoot.libraryListDir(listName).createSync();
for (int i = 0; i < chunkCount; i++) {
final entryPage = (await db.libraryListGetListEntries(
libraryList.name,
final archiveEntries = await exportAdapter.libraryListGetEntries(
db: db,
listName: listName,
page: i,
))!;
);
final archiveEntries = entryPage.entries
.map(ArchiveV2LibraryListEntry.fromLibraryListEntry)
.toList();
archiveRoot.libraryListChunkFile(libraryList.name, i)
archiveRoot.libraryListChunkFile(listName, i)
..createSync()
..writeAsStringSync(jsonEncode(archiveEntries));
..writeAsStringSync(
jsonEncode(archiveEntries.map((final e) => e.toJson()).toList()),
);
yield ArchiveV2StreamEvent(
type: 'library',
progress: index,
total: total,
name: libraryList.name,
name: listName,
subProgress: i + 1,
subTotal: chunkCount,
);
@@ -210,7 +243,13 @@ Stream<ArchiveV2StreamEvent> importLibraryList(
final int index,
final int total,
) async* {
final chunkFiles = libraryListDir.listSync().whereType<File>();
final chunkFiles = libraryListDir.listSync().whereType<File>().sortedBy(
(final file) =>
int.tryParse(
file.uri.pathSegments.last.replaceFirst(RegExp(r'\.json$'), ''),
) ??
0,
);
for (final (i, chunkFile) in chunkFiles.indexed) {
final chunkContent = chunkFile.readAsStringSync();