From f59dabd4d7e73cc73ea018a0ee86cf3e04549364 Mon Sep 17 00:00:00 2001 From: AntsyLich <59261191+AntsyLich@users.noreply.github.com> Date: Mon, 26 Oct 2026 00:55:01 +0600 Subject: [PATCH] Decode only needed backup fields and restore entries in one pass Closes #4168 (cherry picked from commit 3ed83a8d329a10afe0e2ef5a8f9c7bf526ff0c73) --- CHANGELOG.md | 1 + .../tachiyomi/data/backup/BackupDecoder.kt | 29 +++++++++++++------ .../data/backup/BackupFileValidator.kt | 17 +++++++---- .../tachiyomi/data/backup/BackupStream.kt | 9 ++++-- .../data/backup/restore/BackupRestorer.kt | 24 +-------------- .../backup/restore/restorers/MangaRestorer.kt | 9 ------ .../tachiyomi/data/backup/BackupStreamTest.kt | 26 +++++++++++++++++ .../data/backup/RestoreRepositoryImpl.kt | 8 ----- .../main/sqldelight/tachiyomi/data/manga.sq | 4 --- .../backup/repository/RestoreRepository.kt | 2 -- 10 files changed, 66 insertions(+), 63 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6ad75193f9..fde6b17cfd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,6 +20,7 @@ The format is a modified version of [Keep a Changelog](https://keepachangelog.co ### Improved - Create and restore backups faster and with less memory ([@NGB-Was-Taken](https://github.com/NGB-Was-Taken), [@AntsyLich](https://github.com/AntsyLich)) + - Read backups faster before restoring, and decode each entry only once while restoring ([@AntsyLich](https://github.com/AntsyLich)) ### Removed - Remove the "Split tall images" download setting ([@AntsyLich](https://github.com/AntsyLich)) diff --git a/app/src/main/java/eu/kanade/tachiyomi/data/backup/BackupDecoder.kt b/app/src/main/java/eu/kanade/tachiyomi/data/backup/BackupDecoder.kt index a2275a5dbd..50c3eb7954 100644 --- a/app/src/main/java/eu/kanade/tachiyomi/data/backup/BackupDecoder.kt +++ b/app/src/main/java/eu/kanade/tachiyomi/data/backup/BackupDecoder.kt @@ -7,6 +7,7 @@ import eu.kanade.tachiyomi.data.backup.models.Backup import eu.kanade.tachiyomi.data.backup.models.BackupManga import kotlinx.coroutines.flow.Flow import kotlinx.coroutines.flow.flow +import kotlinx.serialization.DeserializationStrategy import kotlinx.serialization.protobuf.ProtoBuf import okio.BufferedSource import okio.buffer @@ -38,17 +39,27 @@ class BackupDecoder( } /** - * Decode the entries of a potentially-gzipped backup one at a time, skipping those whose index [include] rejects. + * Like [decodeMetadata], but hands each entry to [onEntry] decoded as [entry], which can declare only the fields + * it needs so the rest are skipped. */ - fun decodeEntries(uri: Uri, include: (index: Int) -> Boolean = { true }): Flow = flow { - read(uri) { reader -> - var index = 0 + fun decodeMetadata(uri: Uri, entry: DeserializationStrategy, onEntry: (T) -> Unit): Metadata { + return read(uri) { reader -> + var entryCount = 0 while (orInvalid { reader.nextEntry() }) { - if (include(index++)) { - emit(orInvalid { reader.readEntry() }) - } else { - orInvalid { reader.skipEntry() } - } + onEntry(orInvalid { reader.readEntry(entry) }) + entryCount++ + } + Metadata(entryCount, orInvalid { reader.readMetadata() }) + } + } + + /** + * Decode the entries of a potentially-gzipped backup one at a time. + */ + fun decodeEntries(uri: Uri): Flow = flow { + read(uri) { reader -> + while (orInvalid { reader.nextEntry() }) { + emit(orInvalid { reader.readEntry() }) } } } diff --git a/app/src/main/java/eu/kanade/tachiyomi/data/backup/BackupFileValidator.kt b/app/src/main/java/eu/kanade/tachiyomi/data/backup/BackupFileValidator.kt index a5351c0d25..a0b3d9c569 100644 --- a/app/src/main/java/eu/kanade/tachiyomi/data/backup/BackupFileValidator.kt +++ b/app/src/main/java/eu/kanade/tachiyomi/data/backup/BackupFileValidator.kt @@ -2,8 +2,10 @@ package eu.kanade.tachiyomi.data.backup import android.net.Uri import dev.zacsweers.metro.Inject +import eu.kanade.tachiyomi.data.backup.models.BackupTracking import eu.kanade.tachiyomi.data.track.TrackerManager -import kotlinx.coroutines.CancellationException +import kotlinx.serialization.Serializable +import kotlinx.serialization.protobuf.ProtoNumber import tachiyomi.domain.source.service.SourceManager @Inject @@ -21,12 +23,9 @@ class BackupFileValidator( suspend fun validate(uri: Uri): Results { val trackerIds = mutableSetOf() val backup = try { - backupDecoder.decodeEntries(uri).collect { manga -> + backupDecoder.decodeMetadata(uri, TrackedManga.serializer()) { manga -> manga.tracking.mapTo(trackerIds) { it.syncId } - } - backupDecoder.decodeMetadata(uri).backup - } catch (e: CancellationException) { - throw e + }.backup } catch (e: Exception) { throw IllegalStateException(e) } @@ -58,4 +57,10 @@ class BackupFileValidator( val missingSources: List, val missingTrackers: List, ) + + // Only the field validation reads, so decoding skips the rest of each entry + @Serializable + private class TrackedManga( + @ProtoNumber(18) val tracking: List = emptyList(), + ) } diff --git a/app/src/main/java/eu/kanade/tachiyomi/data/backup/BackupStream.kt b/app/src/main/java/eu/kanade/tachiyomi/data/backup/BackupStream.kt index 1500baa0f0..bb1b8c3d74 100644 --- a/app/src/main/java/eu/kanade/tachiyomi/data/backup/BackupStream.kt +++ b/app/src/main/java/eu/kanade/tachiyomi/data/backup/BackupStream.kt @@ -85,9 +85,14 @@ class BackupReader( return false } - fun readEntry(): BackupManga { + fun readEntry(): BackupManga = readEntry(BackupManga.serializer()) + + /** + * Decodes the entry as [deserializer], which can declare only the fields it needs so the rest are skipped. + */ + fun readEntry(deserializer: DeserializationStrategy): T { val bytes = source.readByteArray(takeEntryLength()) - return decode(BackupManga.serializer(), bytes) + return decode(deserializer, bytes) } fun skipEntry() { diff --git a/app/src/main/java/eu/kanade/tachiyomi/data/backup/restore/BackupRestorer.kt b/app/src/main/java/eu/kanade/tachiyomi/data/backup/restore/BackupRestorer.kt index c8e3cf5ea8..c04cf16eae 100644 --- a/app/src/main/java/eu/kanade/tachiyomi/data/backup/restore/BackupRestorer.kt +++ b/app/src/main/java/eu/kanade/tachiyomi/data/backup/restore/BackupRestorer.kt @@ -23,11 +23,6 @@ import kotlinx.coroutines.Job import kotlinx.coroutines.coroutineScope import kotlinx.coroutines.ensureActive import kotlinx.coroutines.flow.chunked -import kotlinx.coroutines.flow.emitAll -import kotlinx.coroutines.flow.filter -import kotlinx.coroutines.flow.flow -import kotlinx.coroutines.flow.map -import kotlinx.coroutines.flow.withIndex import kotlinx.coroutines.launch import logcat.LogPriority import tachiyomi.core.common.i18n.stringResource @@ -37,7 +32,6 @@ import tachiyomi.domain.source.service.SourceManager import tachiyomi.i18n.MR import java.io.File import java.text.SimpleDateFormat -import java.util.BitSet import java.util.Date import java.util.Locale import java.util.concurrent.CopyOnWriteArrayList @@ -187,23 +181,7 @@ class BackupRestorer( ) = launch { categoriesRestoreJob?.join() - // New entries go first. Rather than hold the others back in memory, a second pass reads them again. - val isExisting = mangaRestorer.getExistingMatcher() - val existingIndices = BitSet() - val newMangas = backupDecoder.decodeEntries(uri) - .withIndex() - .filter { (index, manga) -> - val exists = isExisting(manga) - if (exists) existingIndices.set(index) - !exists - } - .map { it.value } - val backupMangas = flow { - emitAll(newMangas) - emitAll(backupDecoder.decodeEntries(uri, existingIndices::get)) - } - - backupMangas + backupDecoder.decodeEntries(uri) .chunked(100) .collect { chunk -> val restoredAsBatch = try { diff --git a/app/src/main/java/eu/kanade/tachiyomi/data/backup/restore/restorers/MangaRestorer.kt b/app/src/main/java/eu/kanade/tachiyomi/data/backup/restore/restorers/MangaRestorer.kt index 8f43ac0b72..8145cb72e0 100644 --- a/app/src/main/java/eu/kanade/tachiyomi/data/backup/restore/restorers/MangaRestorer.kt +++ b/app/src/main/java/eu/kanade/tachiyomi/data/backup/restore/restorers/MangaRestorer.kt @@ -23,15 +23,6 @@ class MangaRestorer( private val now = Clock.System.now().toLocalDateTime(timeZone) private val currentFetchWindow = fetchInterval.getWindow(now.date, timeZone) - /** - * Returns whether an entry was already in the database at the time of this call. - */ - suspend fun getExistingMatcher(): (BackupManga) -> Boolean { - val urlsBySource = restoreRepository.getMangaUrlsBySourceId() - - return { it.url in urlsBySource[it.source].orEmpty() } - } - /** * Restores [backupMangas] all together, so either every one of them is restored or none is. */ diff --git a/app/src/test/java/eu/kanade/tachiyomi/data/backup/BackupStreamTest.kt b/app/src/test/java/eu/kanade/tachiyomi/data/backup/BackupStreamTest.kt index 3c84f019db..f4d6b7ced4 100644 --- a/app/src/test/java/eu/kanade/tachiyomi/data/backup/BackupStreamTest.kt +++ b/app/src/test/java/eu/kanade/tachiyomi/data/backup/BackupStreamTest.kt @@ -14,7 +14,9 @@ import eu.kanade.tachiyomi.data.backup.models.BooleanPreferenceValue import eu.kanade.tachiyomi.data.backup.models.StringPreferenceValue import eu.kanade.tachiyomi.data.backup.models.StringSetPreferenceValue import io.kotest.matchers.shouldBe +import kotlinx.serialization.Serializable import kotlinx.serialization.protobuf.ProtoBuf +import kotlinx.serialization.protobuf.ProtoNumber import okio.Buffer import org.junit.jupiter.api.Test import org.junit.jupiter.api.assertThrows @@ -62,6 +64,21 @@ class BackupStreamTest { encode(reader.readMetadata()) shouldBe encode(backup.copy(backupManga = emptyList())) } + @Test + fun `reader decodes only the fields an entry class declares`() { + val backup = backup() + val reader = BackupReader(Buffer().write(encode(backup)), parser) + + val entries = buildList { + while (reader.nextEntry()) add(reader.readEntry(PartialManga.serializer())) + } + + entries.map { Triple(it.source, it.url, it.tracking.map(::encode)) } shouldBe backup.backupManga.map { + Triple(it.source, it.url, it.tracking.map(::encode)) + } + encode(reader.readMetadata()) shouldBe encode(backup.copy(backupManga = emptyList())) + } + @Test fun `reader takes fields in any order`() { val backup = backup() @@ -149,6 +166,15 @@ class BackupStreamTest { private fun encode(manga: BackupManga) = parser.encodeToByteArray(BackupManga.serializer(), manga) + private fun encode(tracking: BackupTracking) = parser.encodeToByteArray(BackupTracking.serializer(), tracking) + + @Serializable + private class PartialManga( + @ProtoNumber(1) val source: Long, + @ProtoNumber(2) val url: String, + @ProtoNumber(18) val tracking: List = emptyList(), + ) + private fun backup() = Backup( backupManga = listOf( BackupManga( diff --git a/data/src/main/java/tachiyomi/data/backup/RestoreRepositoryImpl.kt b/data/src/main/java/tachiyomi/data/backup/RestoreRepositoryImpl.kt index aeb6d80a81..900cae5768 100644 --- a/data/src/main/java/tachiyomi/data/backup/RestoreRepositoryImpl.kt +++ b/data/src/main/java/tachiyomi/data/backup/RestoreRepositoryImpl.kt @@ -34,14 +34,6 @@ class RestoreRepositoryImpl( private val trackRepository: TrackRepository, ) : RestoreRepository { - override suspend fun getMangaUrlsBySourceId(): Map> { - return database.mangaQueries - .getAllMangaSourceAndUrl() - .awaitAsList() - .groupBy({ it.source_id }, { it.remote_url }) - .mapValues { it.value.toSet() } - } - override suspend fun restoreManga(entries: List, update: suspend (Manga) -> MangaUpdate) { database.transaction { entries.forEach { entry -> diff --git a/data/src/main/sqldelight/tachiyomi/data/manga.sq b/data/src/main/sqldelight/tachiyomi/data/manga.sq index 89d821e718..baf5b5a13c 100644 --- a/data/src/main/sqldelight/tachiyomi/data/manga.sq +++ b/data/src/main/sqldelight/tachiyomi/data/manga.sq @@ -79,10 +79,6 @@ AND EXISTS ( AND (user_read = 1 OR user_last_page_read != 0) ); -getAllMangaSourceAndUrl: -SELECT source_id, remote_url -FROM manga; - getSourceIdWithFavoriteCount: SELECT source_id, count(*) FROM manga diff --git a/domain/src/main/java/tachiyomi/domain/backup/repository/RestoreRepository.kt b/domain/src/main/java/tachiyomi/domain/backup/repository/RestoreRepository.kt index 79413f6c4e..fbfc8cfc1b 100644 --- a/domain/src/main/java/tachiyomi/domain/backup/repository/RestoreRepository.kt +++ b/domain/src/main/java/tachiyomi/domain/backup/repository/RestoreRepository.kt @@ -6,8 +6,6 @@ import tachiyomi.domain.manga.model.MangaUpdate interface RestoreRepository { - suspend fun getMangaUrlsBySourceId(): Map> - /** * Restores [entries] all together: either every one of them is restored or none is. [update] is what to change on each * entry once its chapters and history are in place, and is part of the same all-or-nothing restore.