diff --git a/CHANGELOG.md b/CHANGELOG.md index f7d67ee00..1dae28940 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -32,6 +32,9 @@ The format is a modified version of [Keep a Changelog](https://keepachangelog.co ### Changed - Use a new decoder for images Android can't decode itself, like AVIF, JPEG XL and HEIF, and support JPEG 2000 images ([@wwww-wwww](https://github.com/wwww-wwww)) ([#3786](https://github.com/mihonapp/mihon/pull/3786)) +### Improved +- Create and restore backups faster and with less memory ([@NGB-Was-Taken](https://github.com/NGB-Was-Taken), [@AntsyLich](https://github.com/AntsyLich)) + ### Removed - Remove "Custom hardware bitmap threshold" and "Custom display profile" advanced settings ([@wwww-wwww](https://github.com/wwww-wwww)) ([#3786](https://github.com/mihonapp/mihon/pull/3786)) diff --git a/app/src/main/java/eu/kanade/tachiyomi/data/backup/BackupDecoder.kt b/app/src/main/java/eu/kanade/tachiyomi/data/backup/BackupDecoder.kt index 5aa85c6f7..a2275a5db 100644 --- a/app/src/main/java/eu/kanade/tachiyomi/data/backup/BackupDecoder.kt +++ b/app/src/main/java/eu/kanade/tachiyomi/data/backup/BackupDecoder.kt @@ -4,8 +4,11 @@ import android.content.Context import android.net.Uri import dev.zacsweers.metro.Inject import eu.kanade.tachiyomi.data.backup.models.Backup -import kotlinx.serialization.SerializationException +import eu.kanade.tachiyomi.data.backup.models.BackupManga +import kotlinx.coroutines.flow.Flow +import kotlinx.coroutines.flow.flow import kotlinx.serialization.protobuf.ProtoBuf +import okio.BufferedSource import okio.buffer import okio.gzip import okio.source @@ -19,9 +22,38 @@ class BackupDecoder( private val parser: ProtoBuf, ) { /** - * Decode a potentially-gzipped backup. + * Decode everything but the entries of a potentially-gzipped backup, counting the entries without decoding them. */ - fun decode(uri: Uri): Backup { + fun decodeMetadata(uri: Uri): Metadata { + return read(uri) { reader -> + orInvalid { + var entryCount = 0 + while (reader.nextEntry()) { + reader.skipEntry() + entryCount++ + } + Metadata(entryCount, reader.readMetadata()) + } + } + } + + /** + * Decode the entries of a potentially-gzipped backup one at a time, skipping those whose index [include] rejects. + */ + fun decodeEntries(uri: Uri, include: (index: Int) -> Boolean = { true }): Flow = flow { + read(uri) { reader -> + var index = 0 + while (orInvalid { reader.nextEntry() }) { + if (include(index++)) { + emit(orInvalid { reader.readEntry() }) + } else { + orInvalid { reader.skipEntry() } + } + } + } + } + + private inline fun read(uri: Uri, block: (BackupReader) -> T): T { return context.contentResolver.openInputStream(uri)!!.use { inputStream -> val source = inputStream.source().buffer() @@ -29,22 +61,32 @@ class BackupDecoder( require(2) } val id1id2 = peeked.readShort() - val backupString = when (id1id2.toInt()) { + val backupSource: BufferedSource = when (id1id2.toInt()) { 0x1f8b -> source.gzip().buffer() // 0x1f8b is gzip magic bytes MAGIC_JSON_SIGNATURE1, MAGIC_JSON_SIGNATURE2, MAGIC_JSON_SIGNATURE3 -> { throw IOException(context.stringResource(MR.strings.invalid_backup_file_json)) } else -> source - }.use { it.readByteArray() } - - try { - parser.decodeFromByteArray(Backup.serializer(), backupString) - } catch (_: SerializationException) { - throw IOException(context.stringResource(MR.strings.invalid_backup_file_unknown)) } + + backupSource.use { block(BackupReader(it, parser)) } } } + // Only around reading, so the JSON error above and anything thrown downstream of emit keep their own messages + private inline fun orInvalid(block: () -> T): T { + return try { + block() + } catch (e: IOException) { + throw IOException(context.stringResource(MR.strings.invalid_backup_file_unknown), e) + } + } + + data class Metadata( + val entryCount: Int, + val backup: Backup, + ) + companion object { private const val MAGIC_JSON_SIGNATURE1 = 0x7b7d // `{}` private const val MAGIC_JSON_SIGNATURE2 = 0x7b22 // `{"` diff --git a/app/src/main/java/eu/kanade/tachiyomi/data/backup/BackupFileValidator.kt b/app/src/main/java/eu/kanade/tachiyomi/data/backup/BackupFileValidator.kt index 2a2bd6739..a5351c0d2 100644 --- a/app/src/main/java/eu/kanade/tachiyomi/data/backup/BackupFileValidator.kt +++ b/app/src/main/java/eu/kanade/tachiyomi/data/backup/BackupFileValidator.kt @@ -3,6 +3,7 @@ package eu.kanade.tachiyomi.data.backup import android.net.Uri import dev.zacsweers.metro.Inject import eu.kanade.tachiyomi.data.track.TrackerManager +import kotlinx.coroutines.CancellationException import tachiyomi.domain.source.service.SourceManager @Inject @@ -18,8 +19,14 @@ class BackupFileValidator( * @return List of missing sources or missing trackers. */ suspend fun validate(uri: Uri): Results { + val trackerIds = mutableSetOf() val backup = try { - backupDecoder.decode(uri) + backupDecoder.decodeEntries(uri).collect { manga -> + manga.tracking.mapTo(trackerIds) { it.syncId } + } + backupDecoder.decodeMetadata(uri).backup + } catch (e: CancellationException) { + throw e } catch (e: Exception) { throw IllegalStateException(e) } @@ -38,11 +45,7 @@ class BackupFileValidator( .distinct() .sorted() - val trackers = backup.backupManga - .flatMap { it.tracking } - .map { it.syncId } - .distinct() - val missingTrackers = trackers + val missingTrackers = trackerIds .mapNotNull { trackerManager.get(it.toLong()) } .filter { !it.isLoggedIn } .map { it.name } diff --git a/app/src/main/java/eu/kanade/tachiyomi/data/backup/BackupStream.kt b/app/src/main/java/eu/kanade/tachiyomi/data/backup/BackupStream.kt new file mode 100644 index 000000000..1500baa0f --- /dev/null +++ b/app/src/main/java/eu/kanade/tachiyomi/data/backup/BackupStream.kt @@ -0,0 +1,153 @@ +package eu.kanade.tachiyomi.data.backup + +import eu.kanade.tachiyomi.data.backup.models.Backup +import eu.kanade.tachiyomi.data.backup.models.BackupManga +import kotlinx.serialization.DeserializationStrategy +import kotlinx.serialization.SerializationException +import kotlinx.serialization.protobuf.ProtoBuf +import okio.Buffer +import okio.BufferedSink +import okio.BufferedSource +import java.io.IOException + +/** + * Writes the same bytes as encoding a whole [Backup], without holding all of its entries at once. + */ +class BackupWriter( + private val sink: BufferedSink, + private val parser: ProtoBuf, +) { + var isEmpty = true + private set + + fun writeEntry(entry: BackupManga) { + val bytes = parser.encodeToByteArray(BackupManga.serializer(), entry) + sink.writeVarint(ENTRY_TAG) + sink.writeVarint(bytes.size.toLong()) + sink.write(bytes) + isEmpty = false + } + + /** + * Writes the other fields after the entries, as [metadata] without entries of its own. + */ + fun writeMetadata(metadata: Backup) { + val bytes = parser.encodeToByteArray(Backup.serializer(), metadata) + sink.write(bytes) + if (bytes.isNotEmpty()) isEmpty = false + } +} + +/** + * Reads an encoded [Backup] one entry at a time. Every other field is set aside for [readMetadata], wherever it + * appears. + * + * Malformed or truncated input throws an [IOException]. + */ +class BackupReader( + private val source: BufferedSource, + private val parser: ProtoBuf, +) { + private val metadata = Buffer() + private var entryLength = NO_ENTRY + + /** + * Moves to the next entry, which [readEntry] or [skipEntry] then consumes. + * + * @return false once the input is exhausted. + */ + fun nextEntry(): Boolean { + check(entryLength == NO_ENTRY) + while (!source.exhausted()) { + val tag = source.readVarint() + val fieldNumber = tag ushr 3 + val wireType = (tag and 0x7).toInt() + if (fieldNumber == 0L) throw IOException("Invalid field number 0") + + if (tag == ENTRY_TAG) { + entryLength = source.readLength() + return true + } + + metadata.writeVarint(tag) + when (wireType) { + WIRE_TYPE_VARINT -> metadata.writeVarint(source.readVarint()) + WIRE_TYPE_FIXED64 -> metadata.write(source, 8) + WIRE_TYPE_LENGTH_DELIMITED -> { + val length = source.readLength() + metadata.writeVarint(length) + metadata.write(source, length) + } + WIRE_TYPE_FIXED32 -> metadata.write(source, 4) + else -> throw IOException("Unsupported wire type $wireType") + } + } + return false + } + + fun readEntry(): BackupManga { + val bytes = source.readByteArray(takeEntryLength()) + return decode(BackupManga.serializer(), bytes) + } + + fun skipEntry() { + source.skip(takeEntryLength()) + } + + /** + * Skips the remaining entries and decodes everything else as a [Backup] without entries. + */ + fun readMetadata(): Backup { + if (entryLength != NO_ENTRY) skipEntry() + while (nextEntry()) skipEntry() + return decode(Backup.serializer(), metadata.readByteArray()) + } + + private fun takeEntryLength(): Long { + check(entryLength != NO_ENTRY) + return entryLength.also { entryLength = NO_ENTRY } + } + + private fun decode(deserializer: DeserializationStrategy, bytes: ByteArray): T { + return try { + parser.decodeFromByteArray(deserializer, bytes) + } catch (e: SerializationException) { + throw IOException(e) + } + } +} + +private fun BufferedSource.readVarint(): Long { + var result = 0L + for (shift in 0 until Long.SIZE_BITS step 7) { + val byte = readByte().toInt() + result = result or ((byte and 0x7F).toLong() shl shift) + if ((byte and 0x80) == 0) return result + } + throw IOException("Malformed varint") +} + +private fun BufferedSource.readLength(): Long { + val length = readVarint() + if (length !in 0..Int.MAX_VALUE) throw IOException("Invalid length $length") + return length +} + +private fun BufferedSink.writeVarint(value: Long) { + var remaining = value + while ((remaining and 0x7FL.inv()) != 0L) { + writeByte((remaining and 0x7F).toInt() or 0x80) + remaining = remaining ushr 7 + } + writeByte(remaining.toInt()) +} + +private const val WIRE_TYPE_VARINT = 0 +private const val WIRE_TYPE_FIXED64 = 1 +private const val WIRE_TYPE_LENGTH_DELIMITED = 2 +private const val WIRE_TYPE_FIXED32 = 5 + +// Backup.backupManga is field 1, and each entry is length-delimited +private const val ENTRY_TAG = 1L shl 3 or WIRE_TYPE_LENGTH_DELIMITED.toLong() + +private const val NO_ENTRY = -1L diff --git a/app/src/main/java/eu/kanade/tachiyomi/data/backup/create/BackupCreator.kt b/app/src/main/java/eu/kanade/tachiyomi/data/backup/create/BackupCreator.kt index 5a8725a58..a24f6562f 100644 --- a/app/src/main/java/eu/kanade/tachiyomi/data/backup/create/BackupCreator.kt +++ b/app/src/main/java/eu/kanade/tachiyomi/data/backup/create/BackupCreator.kt @@ -8,6 +8,7 @@ import dev.zacsweers.metro.AssistedFactory import dev.zacsweers.metro.AssistedInject import eu.kanade.tachiyomi.BuildConfig import eu.kanade.tachiyomi.data.backup.BackupFileValidator +import eu.kanade.tachiyomi.data.backup.BackupWriter import eu.kanade.tachiyomi.data.backup.create.creators.CategoriesBackupCreator import eu.kanade.tachiyomi.data.backup.create.creators.ExtensionStoresBackupCreator import eu.kanade.tachiyomi.data.backup.create.creators.MangaBackupCreator @@ -20,6 +21,8 @@ import eu.kanade.tachiyomi.data.backup.models.BackupManga import eu.kanade.tachiyomi.data.backup.models.BackupPreference import eu.kanade.tachiyomi.data.backup.models.BackupSource import eu.kanade.tachiyomi.data.backup.models.BackupSourcePreferences +import kotlinx.coroutines.flow.Flow +import kotlinx.coroutines.flow.emptyFlow import kotlinx.serialization.protobuf.ProtoBuf import logcat.LogPriority import okio.buffer @@ -85,27 +88,31 @@ class BackupCreator( val nonFavoriteManga = if (options.readEntries) mangaRepository.getReadMangaNotInLibrary() else emptyList() val backupManga = backupMangas(getFavorites.await() + nonFavoriteManga, options) - val backup = Backup( - backupManga = backupManga, - backupCategories = backupCategories(options), - backupSources = backupSources(backupManga), - backupPreferences = backupAppPreferences(options), - backupExtensionStores = backupExtensionStores(options), - backupSourcePreferences = backupSourcePreferences(options), - ) - - val byteArray = parser.encodeToByteArray(Backup.serializer(), backup) - if (byteArray.isEmpty()) { - throw IllegalStateException(context.stringResource(MR.strings.empty_backup_error)) - } - file.openOutputStream() .also { // Force overwrite old file (it as? FileOutputStream)?.channel?.truncate(0) } .sink().gzip().buffer().use { - it.write(byteArray) + val writer = BackupWriter(it, parser) + val sourceIds = mutableSetOf() + backupManga.collect { manga -> + writer.writeEntry(manga) + sourceIds += manga.source + } + writer.writeMetadata( + Backup( + backupManga = emptyList(), + backupCategories = backupCategories(options), + backupSources = backupSources(sourceIds), + backupPreferences = backupAppPreferences(options), + backupExtensionStores = backupExtensionStores(options), + backupSourcePreferences = backupSourcePreferences(options), + ), + ) + if (writer.isEmpty) { + throw IllegalStateException(context.stringResource(MR.strings.empty_backup_error)) + } } val fileUri = file.uri @@ -130,14 +137,14 @@ class BackupCreator( return categoriesBackupCreator() } - private suspend fun backupMangas(mangas: List, options: BackupOptions): List { - if (!options.libraryEntries) return emptyList() + private fun backupMangas(mangas: List, options: BackupOptions): Flow { + if (!options.libraryEntries) return emptyFlow() return mangaBackupCreator(mangas, options) } - private suspend fun backupSources(mangas: List): List { - return sourcesBackupCreator(mangas) + private suspend fun backupSources(sourceIds: Set): List { + return sourcesBackupCreator(sourceIds) } private fun backupAppPreferences(options: BackupOptions): List { diff --git a/app/src/main/java/eu/kanade/tachiyomi/data/backup/create/creators/MangaBackupCreator.kt b/app/src/main/java/eu/kanade/tachiyomi/data/backup/create/creators/MangaBackupCreator.kt index 26d873cd0..e4e350de5 100644 --- a/app/src/main/java/eu/kanade/tachiyomi/data/backup/create/creators/MangaBackupCreator.kt +++ b/app/src/main/java/eu/kanade/tachiyomi/data/backup/create/creators/MangaBackupCreator.kt @@ -7,11 +7,13 @@ import eu.kanade.tachiyomi.data.backup.models.BackupManga import eu.kanade.tachiyomi.data.backup.models.toBackupChapter import eu.kanade.tachiyomi.data.backup.models.toBackupTracking import eu.kanade.tachiyomi.ui.reader.setting.ReadingMode +import kotlinx.coroutines.flow.Flow +import kotlinx.coroutines.flow.flow import mihon.core.common.extensions.toByteArray import mihon.core.common.extensions.toEpochMillisOrZero -import tachiyomi.domain.category.interactor.GetCategories +import tachiyomi.domain.category.repository.CategoryRepository import tachiyomi.domain.chapter.repository.ChapterRepository -import tachiyomi.domain.history.interactor.GetHistory +import tachiyomi.domain.history.repository.HistoryRepository import tachiyomi.domain.manga.model.Manga import tachiyomi.domain.manga.repository.MangaRepository import tachiyomi.domain.track.repository.TrackRepository @@ -20,62 +22,71 @@ import tachiyomi.domain.track.repository.TrackRepository class MangaBackupCreator( private val mangaRepository: MangaRepository, private val chapterRepository: ChapterRepository, + private val categoryRepository: CategoryRepository, + private val historyRepository: HistoryRepository, private val trackRepository: TrackRepository, - private val getCategories: GetCategories, - private val getHistory: GetHistory, ) { - suspend operator fun invoke(mangas: List, options: BackupOptions): List { - return mangas.map { - backupManga(it, options) + /** + * Builds the entries a chunk at a time, so only one chunk's chapters are held at once. + */ + operator fun invoke(mangas: List, options: BackupOptions): Flow = flow { + mangas.chunked(CHUNK_SIZE).forEach { chunk -> + backupMangas(chunk, options).forEach { emit(it) } } } - private suspend fun backupManga(manga: Manga, options: BackupOptions): BackupManga { - // Entry for this manga - val mangaObject = manga.toBackupManga() + private suspend fun backupMangas(mangas: List, options: BackupOptions): List { + val mangaIds = mangas.map { it.id } - mangaObject.excludedScanlators = mangaRepository.getExcludedScanlators(manga.id).toList() - - if (options.chapters) { - // Backup all the chapters - chapterRepository.getChapterByMangaId(manga.id, applyScanlatorFilter = false) - .map { it.toBackupChapter() } - .takeUnless { it.isEmpty() } - ?.let { mangaObject.chapters = it } + val excludedScanlatorsByMangaId = mangaRepository.getExcludedScanlatorsByMangaIds(mangaIds) + // History is backed up by chapter URL, so it needs the chapters too + val chaptersByMangaId = if (options.chapters || options.history) { + chapterRepository.getChaptersByMangaIds(mangaIds) + } else { + emptyMap() } + val categoriesByMangaId = if (options.categories) { + categoryRepository.getCategoriesByMangaIds(mangaIds) + } else { + emptyMap() + } + val tracksByMangaId = if (options.tracking) trackRepository.getTracksByMangaIds(mangaIds) else emptyMap() + val historyByMangaId = if (options.history) historyRepository.getHistoryByMangaIds(mangaIds) else emptyMap() - if (options.categories) { - // Backup categories for this manga - val categoriesForManga = getCategories.await(manga.id) - if (categoriesForManga.isNotEmpty()) { - mangaObject.categories = categoriesForManga.map { it.order } + return mangas.map { manga -> + val mangaObject = manga.toBackupManga() + val chapters = chaptersByMangaId[manga.id].orEmpty() + + mangaObject.excludedScanlators = excludedScanlatorsByMangaId[manga.id].orEmpty() + + if (options.chapters) { + mangaObject.chapters = chapters.map { it.toBackupChapter() } } - } - if (options.tracking) { - val tracks = trackRepository.getTracksByMangaId(manga.id).map { it.toBackupTracking() } - if (tracks.isNotEmpty()) { - mangaObject.tracking = tracks + if (options.categories) { + mangaObject.categories = categoriesByMangaId[manga.id].orEmpty().map { it.order } } - } - if (options.history) { - val historyByMangaId = getHistory.await(manga.id) - if (historyByMangaId.isNotEmpty()) { - val chapterUrlsById = chapterRepository.getChapterByMangaId(manga.id).associate { it.id to it.url } - val history = historyByMangaId.mapNotNull { history -> + if (options.tracking) { + mangaObject.tracking = tracksByMangaId[manga.id].orEmpty().map { it.toBackupTracking() } + } + + if (options.history) { + val chapterUrlsById = chapters.associate { it.id to it.url } + mangaObject.history = historyByMangaId[manga.id].orEmpty().mapNotNull { history -> // A chapter removed since its history was read takes that history with it val url = chapterUrlsById[history.chapterId] ?: return@mapNotNull null BackupHistory(url, history.readAt.toEpochMillisOrZero(), history.readDuration) } - if (history.isNotEmpty()) { - mangaObject.history = history - } } - } - return mangaObject + mangaObject + } + } + + private companion object { + const val CHUNK_SIZE = 100 } } diff --git a/app/src/main/java/eu/kanade/tachiyomi/data/backup/create/creators/SourcesBackupCreator.kt b/app/src/main/java/eu/kanade/tachiyomi/data/backup/create/creators/SourcesBackupCreator.kt index 2b8180ce9..e4ac3ae1b 100644 --- a/app/src/main/java/eu/kanade/tachiyomi/data/backup/create/creators/SourcesBackupCreator.kt +++ b/app/src/main/java/eu/kanade/tachiyomi/data/backup/create/creators/SourcesBackupCreator.kt @@ -1,7 +1,6 @@ package eu.kanade.tachiyomi.data.backup.create.creators import dev.zacsweers.metro.Inject -import eu.kanade.tachiyomi.data.backup.models.BackupManga import eu.kanade.tachiyomi.data.backup.models.BackupSource import eu.kanade.tachiyomi.source.Source import tachiyomi.domain.source.service.SourceManager @@ -11,11 +10,8 @@ class SourcesBackupCreator( private val sourceManager: SourceManager, ) { - suspend operator fun invoke(mangas: List): List { - return mangas - .map(BackupManga::source) - .distinct() - .map { sourceManager.getOrStub(it).toBackupSource() } + suspend operator fun invoke(sourceIds: Set): List { + return sourceIds.map { sourceManager.getOrStub(it).toBackupSource() } } } diff --git a/app/src/main/java/eu/kanade/tachiyomi/data/backup/restore/BackupRestorer.kt b/app/src/main/java/eu/kanade/tachiyomi/data/backup/restore/BackupRestorer.kt index d4e36afff..c8e3cf5ea 100644 --- a/app/src/main/java/eu/kanade/tachiyomi/data/backup/restore/BackupRestorer.kt +++ b/app/src/main/java/eu/kanade/tachiyomi/data/backup/restore/BackupRestorer.kt @@ -9,7 +9,6 @@ import eu.kanade.tachiyomi.data.backup.BackupDecoder import eu.kanade.tachiyomi.data.backup.BackupNotifier import eu.kanade.tachiyomi.data.backup.models.BackupCategory import eu.kanade.tachiyomi.data.backup.models.BackupExtensionStore -import eu.kanade.tachiyomi.data.backup.models.BackupManga import eu.kanade.tachiyomi.data.backup.models.BackupPreference import eu.kanade.tachiyomi.data.backup.models.BackupSource import eu.kanade.tachiyomi.data.backup.models.BackupSourcePreferences @@ -23,6 +22,12 @@ import kotlinx.coroutines.CoroutineScope import kotlinx.coroutines.Job import kotlinx.coroutines.coroutineScope import kotlinx.coroutines.ensureActive +import kotlinx.coroutines.flow.chunked +import kotlinx.coroutines.flow.emitAll +import kotlinx.coroutines.flow.filter +import kotlinx.coroutines.flow.flow +import kotlinx.coroutines.flow.map +import kotlinx.coroutines.flow.withIndex import kotlinx.coroutines.launch import logcat.LogPriority import tachiyomi.core.common.i18n.stringResource @@ -32,6 +37,7 @@ import tachiyomi.domain.source.service.SourceManager import tachiyomi.i18n.MR import java.io.File import java.text.SimpleDateFormat +import java.util.BitSet import java.util.Date import java.util.Locale import java.util.concurrent.CopyOnWriteArrayList @@ -97,7 +103,7 @@ class BackupRestorer( } private suspend fun restoreFromFile(uri: Uri, options: RestoreOptions) { - val backup = backupDecoder.decode(uri) + val (entryCount, backup) = backupDecoder.decodeMetadata(uri) // Store source mapping for error messages val backupMaps = backup.backupSources @@ -105,7 +111,7 @@ class BackupRestorer( if (options.libraryEntries) { restoreSourceNames(backupMaps) - restoreAmount += backup.backupManga.size + restoreAmount += entryCount } if (options.categories) { restoreAmount += 1 @@ -138,7 +144,7 @@ class BackupRestorer( } if (options.libraryEntries) { restoreManga( - backup.backupManga, + uri, if (options.categories) backup.backupCategories else emptyList(), restoreCategoriesJob, ) @@ -175,14 +181,31 @@ class BackupRestorer( } private fun CoroutineScope.restoreManga( - backupMangas: List, + uri: Uri, backupCategories: List, categoriesRestoreJob: Job?, ) = launch { categoriesRestoreJob?.join() - mangaRestorer.sortByNew(backupMangas) + + // New entries go first. Rather than hold the others back in memory, a second pass reads them again. + val isExisting = mangaRestorer.getExistingMatcher() + val existingIndices = BitSet() + val newMangas = backupDecoder.decodeEntries(uri) + .withIndex() + .filter { (index, manga) -> + val exists = isExisting(manga) + if (exists) existingIndices.set(index) + !exists + } + .map { it.value } + val backupMangas = flow { + emitAll(newMangas) + emitAll(backupDecoder.decodeEntries(uri, existingIndices::get)) + } + + backupMangas .chunked(100) - .forEach { chunk -> + .collect { chunk -> val restoredAsBatch = try { ensureActive() mangaRestorer.restore(chunk, backupCategories) diff --git a/app/src/main/java/eu/kanade/tachiyomi/data/backup/restore/restorers/MangaRestorer.kt b/app/src/main/java/eu/kanade/tachiyomi/data/backup/restore/restorers/MangaRestorer.kt index e2fbc81a7..8f43ac0b7 100644 --- a/app/src/main/java/eu/kanade/tachiyomi/data/backup/restore/restorers/MangaRestorer.kt +++ b/app/src/main/java/eu/kanade/tachiyomi/data/backup/restore/restorers/MangaRestorer.kt @@ -23,11 +23,13 @@ class MangaRestorer( private val now = Clock.System.now().toLocalDateTime(timeZone) private val currentFetchWindow = fetchInterval.getWindow(now.date, timeZone) - suspend fun sortByNew(backupMangas: List): List { + /** + * Returns whether an entry was already in the database at the time of this call. + */ + suspend fun getExistingMatcher(): (BackupManga) -> Boolean { val urlsBySource = restoreRepository.getMangaUrlsBySourceId() - return backupMangas - .sortedBy { it.url in urlsBySource[it.source].orEmpty() } + return { it.url in urlsBySource[it.source].orEmpty() } } /** diff --git a/app/src/test/java/eu/kanade/tachiyomi/data/backup/BackupStreamTest.kt b/app/src/test/java/eu/kanade/tachiyomi/data/backup/BackupStreamTest.kt new file mode 100644 index 000000000..3c84f019d --- /dev/null +++ b/app/src/test/java/eu/kanade/tachiyomi/data/backup/BackupStreamTest.kt @@ -0,0 +1,202 @@ +package eu.kanade.tachiyomi.data.backup + +import eu.kanade.tachiyomi.data.backup.models.Backup +import eu.kanade.tachiyomi.data.backup.models.BackupCategory +import eu.kanade.tachiyomi.data.backup.models.BackupChapter +import eu.kanade.tachiyomi.data.backup.models.BackupExtensionStore +import eu.kanade.tachiyomi.data.backup.models.BackupHistory +import eu.kanade.tachiyomi.data.backup.models.BackupManga +import eu.kanade.tachiyomi.data.backup.models.BackupPreference +import eu.kanade.tachiyomi.data.backup.models.BackupSource +import eu.kanade.tachiyomi.data.backup.models.BackupSourcePreferences +import eu.kanade.tachiyomi.data.backup.models.BackupTracking +import eu.kanade.tachiyomi.data.backup.models.BooleanPreferenceValue +import eu.kanade.tachiyomi.data.backup.models.StringPreferenceValue +import eu.kanade.tachiyomi.data.backup.models.StringSetPreferenceValue +import io.kotest.matchers.shouldBe +import kotlinx.serialization.protobuf.ProtoBuf +import okio.Buffer +import org.junit.jupiter.api.Test +import org.junit.jupiter.api.assertThrows +import org.junit.jupiter.api.parallel.Execution +import org.junit.jupiter.api.parallel.ExecutionMode +import java.io.IOException + +@Execution(ExecutionMode.CONCURRENT) +class BackupStreamTest { + + private val parser = ProtoBuf + + @Test + fun `writer output is the plain encoding`() { + val backup = backup() + + val bytes = write(backup) + + bytes shouldBe encode(backup) + encode(parser.decodeFromByteArray(Backup.serializer(), bytes)) shouldBe encode(backup) + } + + @Test + fun `reader reads the plain encoding`() { + val backup = backup() + + val (entries, metadata) = read(encode(backup)) + + entries.map(::encode) shouldBe backup.backupManga.map(::encode) + encode(metadata) shouldBe encode(backup.copy(backupManga = emptyList())) + } + + @Test + fun `reader counts entries without decoding them`() { + val backup = backup() + val reader = BackupReader(Buffer().write(encode(backup)), parser) + + var count = 0 + while (reader.nextEntry()) { + reader.skipEntry() + count++ + } + + count shouldBe backup.backupManga.size + encode(reader.readMetadata()) shouldBe encode(backup.copy(backupManga = emptyList())) + } + + @Test + fun `reader takes fields in any order`() { + val backup = backup() + val (first, second, third) = backup.backupManga + val input = Buffer() + .write(encode(Backup(emptyList(), backupSources = backup.backupSources))) + .write(encode(Backup(listOf(first)))) + // Fields Backup doesn't know, one of each remaining wire type + .writeByte(3 shl 3 or 0).writeByte(0x96).writeByte(0x01) + .writeByte(4 shl 3 or 1).writeLongLe(42) + .write(encode(Backup(listOf(second), backupCategories = backup.backupCategories))) + .writeByte(5 shl 3 or 5).writeIntLe(42) + .write(encode(Backup(emptyList(), backupPreferences = backup.backupPreferences))) + .write(encode(Backup(listOf(third), backupExtensionStores = backup.backupExtensionStores))) + .write(encode(Backup(emptyList(), backupSourcePreferences = backup.backupSourcePreferences))) + .readByteArray() + + val (entries, metadata) = read(input) + + entries.map(::encode) shouldBe backup.backupManga.map(::encode) + encode(metadata) shouldBe encode(backup.copy(backupManga = emptyList())) + } + + @Test + fun `empty backup`() { + val sink = Buffer() + val writer = BackupWriter(sink, parser) + writer.writeMetadata(Backup(emptyList())) + + writer.isEmpty shouldBe true + sink.size shouldBe 0L + + val (entries, metadata) = read(ByteArray(0)) + entries shouldBe emptyList() + encode(metadata) shouldBe ByteArray(0) + } + + @Test + fun `truncated input throws`() { + val bytes = encode(backup()) + + // Cut inside the last entry, inside the metadata, and right after a tag + val entryEnd = encode(Backup(backup().backupManga)).size + listOf(entryEnd - 1, bytes.size - 1, entryEnd + 1).forEach { size -> + assertThrows { read(bytes.copyOf(size)) } + } + } + + @Test + fun `garbage input throws`() { + listOf( + // Wire type 7 doesn't exist + byteArrayOf(0x0F), + // Field number 0 + byteArrayOf(0x02, 0x00), + // A varint that never ends + ByteArray(11) { 0xFF.toByte() }, + // An entry longer than the input + byteArrayOf(0x0A, 0x10, 0x01), + // An entry that isn't a BackupManga + byteArrayOf(0x0A, 0x02, 0x0F, 0x00), + ).forEach { input -> + assertThrows { read(input) } + } + } + + private fun write(backup: Backup): ByteArray { + val sink = Buffer() + val writer = BackupWriter(sink, parser) + backup.backupManga.forEach(writer::writeEntry) + writer.writeMetadata(backup.copy(backupManga = emptyList())) + writer.isEmpty shouldBe false + return sink.readByteArray() + } + + private fun read(bytes: ByteArray): Pair, Backup> { + val reader = BackupReader(Buffer().write(bytes), parser) + val entries = buildList { + while (reader.nextEntry()) add(reader.readEntry()) + } + return entries to reader.readMetadata() + } + + private fun encode(backup: Backup) = parser.encodeToByteArray(Backup.serializer(), backup) + + private fun encode(manga: BackupManga) = parser.encodeToByteArray(BackupManga.serializer(), manga) + + private fun backup() = Backup( + backupManga = listOf( + BackupManga( + source = 1L, + url = "/manga/1", + title = "First", + genre = listOf("Action", "Drama"), + chapters = listOf( + BackupChapter(url = "/chapter/1", name = "Chapter 1", read = true, chapterNumber = 1f), + BackupChapter(url = "/chapter/2", name = "Chapter 2", bookmark = true, lastPageRead = 4), + ), + categories = listOf(0L, 1L), + tracking = listOf(BackupTracking(syncId = 2, libraryId = 3L, title = "First", mediaId = 7L)), + history = listOf(BackupHistory(url = "/chapter/1", lastRead = 1_700_000_000_000L, readDuration = 60)), + excludedScanlators = listOf("Group"), + ), + BackupManga(source = 2L, url = "/manga/2", title = "Second", favorite = false), + BackupManga( + source = 1L, + url = "/manga/3", + title = "Third", + description = "x".repeat(300), + chapters = List(200) { BackupChapter(url = "/chapter/3/$it", name = "Chapter $it") }, + ), + ), + backupCategories = listOf( + BackupCategory(name = "Reading", order = 0), + BackupCategory(name = "Done", order = 1), + ), + backupSources = listOf(BackupSource(name = "One", sourceId = 1L), BackupSource(name = "Two", sourceId = 2L)), + backupPreferences = listOf( + BackupPreference("flag", BooleanPreferenceValue(true)), + BackupPreference("tags", StringSetPreferenceValue(setOf("a", "b"))), + ), + backupSourcePreferences = listOf( + BackupSourcePreferences("source_1", listOf(BackupPreference("lang", StringPreferenceValue("en")))), + ), + backupExtensionStores = listOf( + BackupExtensionStore( + indexUrl = "https://example.org/index.min.json", + name = "Store", + badgeLabel = null, + signingKey = "key", + contactWebsite = "https://example.org", + contactDiscord = null, + isLegacy = false, + extensionListUrl = null, + ), + ), + ) +} diff --git a/data/src/main/java/tachiyomi/data/category/CategoryRepositoryImpl.kt b/data/src/main/java/tachiyomi/data/category/CategoryRepositoryImpl.kt index 4ad24bf0d..c39501560 100644 --- a/data/src/main/java/tachiyomi/data/category/CategoryRepositoryImpl.kt +++ b/data/src/main/java/tachiyomi/data/category/CategoryRepositoryImpl.kt @@ -50,6 +50,15 @@ class CategoryRepositoryImpl( .subscribeToList() } + override suspend fun getCategoriesByMangaIds(mangaIds: List): Map> { + return database.categoryQueries + .getCategoriesByMangaIds(mangaIds) { mangaId, id, name, order, flags -> + mangaId to mapCategory(id, name, order, flags) + } + .awaitAsList() + .groupBy({ it.first }, { it.second }) + } + override suspend fun insert(category: NewCategory) { database.categoryQueries.insert(name = category.name, flags = category.flags) } diff --git a/data/src/main/java/tachiyomi/data/chapter/ChapterRepositoryImpl.kt b/data/src/main/java/tachiyomi/data/chapter/ChapterRepositoryImpl.kt index 980a58870..97efaf4f5 100644 --- a/data/src/main/java/tachiyomi/data/chapter/ChapterRepositoryImpl.kt +++ b/data/src/main/java/tachiyomi/data/chapter/ChapterRepositoryImpl.kt @@ -113,6 +113,13 @@ class ChapterRepositoryImpl( .awaitAsList() } + override suspend fun getChaptersByMangaIds(mangaIds: List): Map> { + return database.chapterQueries + .getChaptersByMangaIds(mangaIds, ::mapChapter) + .awaitAsList() + .groupBy { it.mangaId } + } + override suspend fun getChapterSummaryByMangaId(mangaId: Long): ChapterSummary { return database.chapterQueries .getChapterSummaryByMangaId(mangaId) { total, latestChapterNumber -> diff --git a/data/src/main/java/tachiyomi/data/history/HistoryRepositoryImpl.kt b/data/src/main/java/tachiyomi/data/history/HistoryRepositoryImpl.kt index 275325286..d036ae1dc 100644 --- a/data/src/main/java/tachiyomi/data/history/HistoryRepositoryImpl.kt +++ b/data/src/main/java/tachiyomi/data/history/HistoryRepositoryImpl.kt @@ -48,6 +48,15 @@ class HistoryRepositoryImpl( .awaitAsList() } + override suspend fun getHistoryByMangaIds(mangaIds: List): Map> { + return database.historyQueries + .getHistoryByMangaIds(mangaIds) { mangaId, id, chapterId, readAt, readDuration -> + mangaId to HistoryMapper.mapHistory(id, chapterId, readAt, readDuration) + } + .awaitAsList() + .groupBy({ it.first }, { it.second }) + } + override suspend fun resetHistory(historyId: Long) { try { database.historyQueries.resetHistoryById(historyId) diff --git a/data/src/main/java/tachiyomi/data/manga/MangaRepositoryImpl.kt b/data/src/main/java/tachiyomi/data/manga/MangaRepositoryImpl.kt index 82c1fb788..6a125a641 100644 --- a/data/src/main/java/tachiyomi/data/manga/MangaRepositoryImpl.kt +++ b/data/src/main/java/tachiyomi/data/manga/MangaRepositoryImpl.kt @@ -144,6 +144,13 @@ class MangaRepositoryImpl( .toSet() } + override suspend fun getExcludedScanlatorsByMangaIds(mangaIds: List): Map> { + return database.excluded_scanlatorQueries + .getExcludedScanlatorsByMangaIds(mangaIds) + .awaitAsList() + .groupBy({ it.manga_id }, { it.scanlator }) + } + override fun getExcludedScanlatorsAsFlow(mangaId: Long): Flow> { return database.excluded_scanlatorQueries .getExcludedScanlatorsByMangaId(mangaId) diff --git a/data/src/main/java/tachiyomi/data/track/TrackRepositoryImpl.kt b/data/src/main/java/tachiyomi/data/track/TrackRepositoryImpl.kt index 29c452868..5eb9fc609 100644 --- a/data/src/main/java/tachiyomi/data/track/TrackRepositoryImpl.kt +++ b/data/src/main/java/tachiyomi/data/track/TrackRepositoryImpl.kt @@ -31,6 +31,13 @@ class TrackRepositoryImpl( .awaitAsList() } + override suspend fun getTracksByMangaIds(mangaIds: List): Map> { + return database.manga_trackQueries + .getTracksByMangaIds(mangaIds, TrackMapper::mapTrack) + .awaitAsList() + .groupBy { it.mangaId } + } + override suspend fun getTracks(): List { return database.manga_trackQueries .getTracks(TrackMapper::mapTrack) diff --git a/data/src/main/sqldelight/tachiyomi/data/category.sq b/data/src/main/sqldelight/tachiyomi/data/category.sq index 512de4f82..2335ecc53 100644 --- a/data/src/main/sqldelight/tachiyomi/data/category.sq +++ b/data/src/main/sqldelight/tachiyomi/data/category.sq @@ -34,6 +34,14 @@ ON C.id = MC.category_id WHERE MC.manga_id = :mangaId ORDER BY C.`order`; +getCategoriesByMangaIds: +SELECT MC.manga_id, C.id, C.name, C.`order`, C.flags +FROM category C +JOIN manga_category MC +ON C.id = MC.category_id +WHERE MC.manga_id IN :mangaIds +ORDER BY MC.manga_id, C.`order`; + insert: -- The position is taken inside the insert, so two categories created at once cannot claim the same one INSERT INTO category(name, `order`, flags) diff --git a/data/src/main/sqldelight/tachiyomi/data/chapter.sq b/data/src/main/sqldelight/tachiyomi/data/chapter.sq index d96d63c49..ab43d9104 100644 --- a/data/src/main/sqldelight/tachiyomi/data/chapter.sq +++ b/data/src/main/sqldelight/tachiyomi/data/chapter.sq @@ -48,6 +48,16 @@ AND ( OR ES.scanlator IS NULL ); +getChaptersByMangaIds: +-- The unique index's order, which getChaptersByMangaId returns too, so backups keep their chapter order +SELECT + id, manga_id, remote_url, remote_name, remote_scanlator, remote_chapter_number, + remote_date_upload, remote_order, remote_memo, user_read, user_bookmark, + user_last_page_read, state_date_fetch +FROM chapter +WHERE manga_id IN :mangaIds +ORDER BY manga_id, remote_url; + getChapterSummaryByMangaId: SELECT count(*) AS total, max(remote_chapter_number) AS latestChapterNumber FROM chapter diff --git a/data/src/main/sqldelight/tachiyomi/data/excluded_scanlator.sq b/data/src/main/sqldelight/tachiyomi/data/excluded_scanlator.sq index 96758dbec..a044ee3fd 100644 --- a/data/src/main/sqldelight/tachiyomi/data/excluded_scanlator.sq +++ b/data/src/main/sqldelight/tachiyomi/data/excluded_scanlator.sq @@ -21,3 +21,9 @@ getExcludedScanlatorsByMangaId: SELECT scanlator FROM excluded_scanlator WHERE manga_id = :mangaId; + +getExcludedScanlatorsByMangaIds: +SELECT manga_id, scanlator +FROM excluded_scanlator +WHERE manga_id IN :mangaIds +ORDER BY manga_id, scanlator; diff --git a/data/src/main/sqldelight/tachiyomi/data/history.sq b/data/src/main/sqldelight/tachiyomi/data/history.sq index 189f112e4..36ba89422 100644 --- a/data/src/main/sqldelight/tachiyomi/data/history.sq +++ b/data/src/main/sqldelight/tachiyomi/data/history.sq @@ -20,6 +20,12 @@ SELECT id, chapter_id, read_at, read_duration FROM history WHERE manga_id = :mangaId; +getHistoryByMangaIds: +SELECT manga_id, id, chapter_id, read_at, read_duration +FROM history +WHERE manga_id IN :mangaIds +ORDER BY manga_id, id; + resetHistoryById: UPDATE history SET read_at = NULL diff --git a/data/src/main/sqldelight/tachiyomi/data/manga_track.sq b/data/src/main/sqldelight/tachiyomi/data/manga_track.sq index a41a596c2..dc2a3c5de 100644 --- a/data/src/main/sqldelight/tachiyomi/data/manga_track.sq +++ b/data/src/main/sqldelight/tachiyomi/data/manga_track.sq @@ -48,6 +48,14 @@ SELECT FROM manga_track WHERE manga_id = :mangaId; +getTracksByMangaIds: +SELECT + id, manga_id, tracker_id, remote_id, remote_url, library_id, title, last_chapter_read, + total_chapters, status, score, start_date, finish_date, private +FROM manga_track +WHERE manga_id IN :mangaIds +ORDER BY manga_id, tracker_id; + upsert: INSERT INTO manga_track( manga_id, tracker_id, remote_id, remote_url, library_id, title, last_chapter_read, diff --git a/domain/src/main/java/tachiyomi/domain/category/repository/CategoryRepository.kt b/domain/src/main/java/tachiyomi/domain/category/repository/CategoryRepository.kt index 548a92153..c12cab914 100644 --- a/domain/src/main/java/tachiyomi/domain/category/repository/CategoryRepository.kt +++ b/domain/src/main/java/tachiyomi/domain/category/repository/CategoryRepository.kt @@ -16,6 +16,8 @@ interface CategoryRepository { fun getCategoriesByMangaIdAsFlow(mangaId: Long): Flow> + suspend fun getCategoriesByMangaIds(mangaIds: List): Map> + suspend fun insert(category: NewCategory) suspend fun insertAll(categories: List) diff --git a/domain/src/main/java/tachiyomi/domain/chapter/repository/ChapterRepository.kt b/domain/src/main/java/tachiyomi/domain/chapter/repository/ChapterRepository.kt index e39aa1e72..7b452ba80 100644 --- a/domain/src/main/java/tachiyomi/domain/chapter/repository/ChapterRepository.kt +++ b/domain/src/main/java/tachiyomi/domain/chapter/repository/ChapterRepository.kt @@ -20,6 +20,8 @@ interface ChapterRepository { suspend fun getChapterByMangaId(mangaId: Long, applyScanlatorFilter: Boolean = false): List + suspend fun getChaptersByMangaIds(mangaIds: List): Map> + suspend fun getChapterSummaryByMangaId(mangaId: Long): ChapterSummary suspend fun getScanlatorsByMangaId(mangaId: Long): List diff --git a/domain/src/main/java/tachiyomi/domain/history/repository/HistoryRepository.kt b/domain/src/main/java/tachiyomi/domain/history/repository/HistoryRepository.kt index c4cb27fc2..1310ff5b1 100644 --- a/domain/src/main/java/tachiyomi/domain/history/repository/HistoryRepository.kt +++ b/domain/src/main/java/tachiyomi/domain/history/repository/HistoryRepository.kt @@ -15,6 +15,8 @@ interface HistoryRepository { suspend fun getHistoryByMangaId(mangaId: Long): List + suspend fun getHistoryByMangaIds(mangaIds: List): Map> + suspend fun resetHistory(historyId: Long) suspend fun resetHistoryByMangaId(mangaId: Long) diff --git a/domain/src/main/java/tachiyomi/domain/manga/repository/MangaRepository.kt b/domain/src/main/java/tachiyomi/domain/manga/repository/MangaRepository.kt index 29be510d4..283d32b29 100644 --- a/domain/src/main/java/tachiyomi/domain/manga/repository/MangaRepository.kt +++ b/domain/src/main/java/tachiyomi/domain/manga/repository/MangaRepository.kt @@ -43,6 +43,8 @@ interface MangaRepository { suspend fun getExcludedScanlators(mangaId: Long): Set + suspend fun getExcludedScanlatorsByMangaIds(mangaIds: List): Map> + fun getExcludedScanlatorsAsFlow(mangaId: Long): Flow> suspend fun setExcludedScanlators(mangaId: Long, excludedScanlators: Set) diff --git a/domain/src/main/java/tachiyomi/domain/track/repository/TrackRepository.kt b/domain/src/main/java/tachiyomi/domain/track/repository/TrackRepository.kt index a4cf534ae..08fa5a755 100644 --- a/domain/src/main/java/tachiyomi/domain/track/repository/TrackRepository.kt +++ b/domain/src/main/java/tachiyomi/domain/track/repository/TrackRepository.kt @@ -9,6 +9,8 @@ interface TrackRepository { suspend fun getTracksByMangaId(mangaId: Long): List + suspend fun getTracksByMangaIds(mangaIds: List): Map> + suspend fun getTracks(): List fun getTracksAsFlow(): Flow>