Decode only needed backup fields and restore entries in one pass

Closes #4168

CHANGELOG.md

@@ -36,6 +36,7 @@ The format is a modified version of [Keep a Changelog](https://keepachangelog.co
 
 ### Improved
 - Create and restore backups faster and with less memory ([@NGB-Was-Taken](https://github.com/NGB-Was-Taken), [@AntsyLich](https://github.com/AntsyLich)) ([`771ba33`](https://github.com/mihonapp/mihon/commit/771ba33))
+  - Read backups faster before restoring, and decode each entry only once while restoring ([@AntsyLich](https://github.com/AntsyLich))
 
 ### Removed
 - Remove "Custom hardware bitmap threshold" and "Custom display profile" advanced settings ([@wwww-wwww](https://github.com/wwww-wwww)) ([#3786](https://github.com/mihonapp/mihon/pull/3786))

app/src/main/java/eu/kanade/tachiyomi/data/backup/BackupDecoder.kt

@@ -7,6 +7,7 @@ import eu.kanade.tachiyomi.data.backup.models.Backup
 import eu.kanade.tachiyomi.data.backup.models.BackupManga
 import kotlinx.coroutines.flow.Flow
 import kotlinx.coroutines.flow.flow
+import kotlinx.serialization.DeserializationStrategy
 import kotlinx.serialization.protobuf.ProtoBuf
 import okio.BufferedSource
 import okio.buffer
@@ -38,17 +39,27 @@ class BackupDecoder(
     }
 
     /**
-     * Decode the entries of a potentially-gzipped backup one at a time, skipping those whose index [include] rejects.
+     * Like [decodeMetadata], but hands each entry to [onEntry] decoded as [entry], which can declare only the fields
+     * it needs so the rest are skipped.
      */
-    fun decodeEntries(uri: Uri, include: (index: Int) -> Boolean = { true }): Flow<BackupManga> = flow {
-        read(uri) { reader ->
-            var index = 0
+    fun <T> decodeMetadata(uri: Uri, entry: DeserializationStrategy<T>, onEntry: (T) -> Unit): Metadata {
+        return read(uri) { reader ->
+            var entryCount = 0
             while (orInvalid { reader.nextEntry() }) {
-                if (include(index++)) {
-                    emit(orInvalid { reader.readEntry() })
-                } else {
-                    orInvalid { reader.skipEntry() }
-                }
+                onEntry(orInvalid { reader.readEntry(entry) })
+                entryCount++
+            }
+            Metadata(entryCount, orInvalid { reader.readMetadata() })
+        }
+    }
+
+    /**
+     * Decode the entries of a potentially-gzipped backup one at a time.
+     */
+    fun decodeEntries(uri: Uri): Flow<BackupManga> = flow {
+        read(uri) { reader ->
+            while (orInvalid { reader.nextEntry() }) {
+                emit(orInvalid { reader.readEntry() })
             }
         }
     }

app/src/main/java/eu/kanade/tachiyomi/data/backup/BackupFileValidator.kt

@@ -2,8 +2,10 @@ package eu.kanade.tachiyomi.data.backup
 
 import android.net.Uri
 import dev.zacsweers.metro.Inject
+import eu.kanade.tachiyomi.data.backup.models.BackupTracking
 import eu.kanade.tachiyomi.data.track.TrackerManager
-import kotlinx.coroutines.CancellationException
+import kotlinx.serialization.Serializable
+import kotlinx.serialization.protobuf.ProtoNumber
 import tachiyomi.domain.source.service.SourceManager
 
 @Inject
@@ -21,12 +23,9 @@ class BackupFileValidator(
     suspend fun validate(uri: Uri): Results {
         val trackerIds = mutableSetOf<Int>()
         val backup = try {
-            backupDecoder.decodeEntries(uri).collect { manga ->
+            backupDecoder.decodeMetadata(uri, TrackedManga.serializer()) { manga ->
                 manga.tracking.mapTo(trackerIds) { it.syncId }
-            }
-            backupDecoder.decodeMetadata(uri).backup
-        } catch (e: CancellationException) {
-            throw e
+            }.backup
         } catch (e: Exception) {
             throw IllegalStateException(e)
         }
@@ -58,4 +57,10 @@ class BackupFileValidator(
         val missingSources: List<String>,
         val missingTrackers: List<String>,
     )
+
+    // Only the field validation reads, so decoding skips the rest of each entry
+    @Serializable
+    private class TrackedManga(
+        @ProtoNumber(18) val tracking: List<BackupTracking> = emptyList(),
+    )
 }

app/src/main/java/eu/kanade/tachiyomi/data/backup/BackupStream.kt

@@ -85,9 +85,14 @@ class BackupReader(
         return false
     }
 
-    fun readEntry(): BackupManga {
+    fun readEntry(): BackupManga = readEntry(BackupManga.serializer())
+
+    /**
+     * Decodes the entry as [deserializer], which can declare only the fields it needs so the rest are skipped.
+     */
+    fun <T> readEntry(deserializer: DeserializationStrategy<T>): T {
         val bytes = source.readByteArray(takeEntryLength())
-        return decode(BackupManga.serializer(), bytes)
+        return decode(deserializer, bytes)
     }
 
     fun skipEntry() {

app/src/main/java/eu/kanade/tachiyomi/data/backup/restore/BackupRestorer.kt

@@ -23,11 +23,6 @@ import kotlinx.coroutines.Job
 import kotlinx.coroutines.coroutineScope
 import kotlinx.coroutines.ensureActive
 import kotlinx.coroutines.flow.chunked
-import kotlinx.coroutines.flow.emitAll
-import kotlinx.coroutines.flow.filter
-import kotlinx.coroutines.flow.flow
-import kotlinx.coroutines.flow.map
-import kotlinx.coroutines.flow.withIndex
 import kotlinx.coroutines.launch
 import logcat.LogPriority
 import tachiyomi.core.common.i18n.stringResource
@@ -37,7 +32,6 @@ import tachiyomi.domain.source.service.SourceManager
 import tachiyomi.i18n.MR
 import java.io.File
 import java.text.SimpleDateFormat
-import java.util.BitSet
 import java.util.Date
 import java.util.Locale
 import java.util.concurrent.CopyOnWriteArrayList
@@ -187,23 +181,7 @@ class BackupRestorer(
     ) = launch {
         categoriesRestoreJob?.join()
 
-        // New entries go first. Rather than hold the others back in memory, a second pass reads them again.
-        val isExisting = mangaRestorer.getExistingMatcher()
-        val existingIndices = BitSet()
-        val newMangas = backupDecoder.decodeEntries(uri)
-            .withIndex()
-            .filter { (index, manga) ->
-                val exists = isExisting(manga)
-                if (exists) existingIndices.set(index)
-                !exists
-            }
-            .map { it.value }
-        val backupMangas = flow {
-            emitAll(newMangas)
-            emitAll(backupDecoder.decodeEntries(uri, existingIndices::get))
-        }
-
-        backupMangas
+        backupDecoder.decodeEntries(uri)
             .chunked(100)
             .collect { chunk ->
                 val restoredAsBatch = try {

app/src/main/java/eu/kanade/tachiyomi/data/backup/restore/restorers/MangaRestorer.kt

@@ -23,15 +23,6 @@ class MangaRestorer(
     private val now = Clock.System.now().toLocalDateTime(timeZone)
     private val currentFetchWindow = fetchInterval.getWindow(now.date, timeZone)
 
-    /**
-     * Returns whether an entry was already in the database at the time of this call.
-     */
-    suspend fun getExistingMatcher(): (BackupManga) -> Boolean {
-        val urlsBySource = restoreRepository.getMangaUrlsBySourceId()
-
-        return { it.url in urlsBySource[it.source].orEmpty() }
-    }
-
     /**
      * Restores [backupMangas] all together, so either every one of them is restored or none is.
      */

app/src/test/java/eu/kanade/tachiyomi/data/backup/BackupStreamTest.kt

@@ -14,7 +14,9 @@ import eu.kanade.tachiyomi.data.backup.models.BooleanPreferenceValue
 import eu.kanade.tachiyomi.data.backup.models.StringPreferenceValue
 import eu.kanade.tachiyomi.data.backup.models.StringSetPreferenceValue
 import io.kotest.matchers.shouldBe
+import kotlinx.serialization.Serializable
 import kotlinx.serialization.protobuf.ProtoBuf
+import kotlinx.serialization.protobuf.ProtoNumber
 import okio.Buffer
 import org.junit.jupiter.api.Test
 import org.junit.jupiter.api.assertThrows
@@ -62,6 +64,21 @@ class BackupStreamTest {
         encode(reader.readMetadata()) shouldBe encode(backup.copy(backupManga = emptyList()))
     }
 
+    @Test
+    fun `reader decodes only the fields an entry class declares`() {
+        val backup = backup()
+        val reader = BackupReader(Buffer().write(encode(backup)), parser)
+
+        val entries = buildList {
+            while (reader.nextEntry()) add(reader.readEntry(PartialManga.serializer()))
+        }
+
+        entries.map { Triple(it.source, it.url, it.tracking.map(::encode)) } shouldBe backup.backupManga.map {
+            Triple(it.source, it.url, it.tracking.map(::encode))
+        }
+        encode(reader.readMetadata()) shouldBe encode(backup.copy(backupManga = emptyList()))
+    }
+
     @Test
     fun `reader takes fields in any order`() {
         val backup = backup()
@@ -149,6 +166,15 @@ class BackupStreamTest {
 
     private fun encode(manga: BackupManga) = parser.encodeToByteArray(BackupManga.serializer(), manga)
 
+    private fun encode(tracking: BackupTracking) = parser.encodeToByteArray(BackupTracking.serializer(), tracking)
+
+    @Serializable
+    private class PartialManga(
+        @ProtoNumber(1) val source: Long,
+        @ProtoNumber(2) val url: String,
+        @ProtoNumber(18) val tracking: List<BackupTracking> = emptyList(),
+    )
+
     private fun backup() = Backup(
         backupManga = listOf(
             BackupManga(

data/src/main/java/tachiyomi/data/backup/RestoreRepositoryImpl.kt

@@ -34,14 +34,6 @@ class RestoreRepositoryImpl(
     private val trackRepository: TrackRepository,
 ) : RestoreRepository {
 
-    override suspend fun getMangaUrlsBySourceId(): Map<Long, Set<String>> {
-        return database.mangaQueries
-            .getAllMangaSourceAndUrl()
-            .awaitAsList()
-            .groupBy({ it.source_id }, { it.remote_url })
-            .mapValues { it.value.toSet() }
-    }
-
     override suspend fun restoreManga(entries: List<RestoredManga>, update: suspend (Manga) -> MangaUpdate) {
         database.transaction {
             entries.forEach { entry ->

data/src/main/sqldelight/tachiyomi/data/manga.sq

@@ -79,10 +79,6 @@ AND EXISTS (
     AND (user_read = 1 OR user_last_page_read != 0)
 );
 
-getAllMangaSourceAndUrl:
-SELECT source_id, remote_url
-FROM manga;
-
 getSourceIdWithFavoriteCount:
 SELECT source_id, count(*)
 FROM manga

domain/src/main/java/tachiyomi/domain/backup/repository/RestoreRepository.kt

@@ -6,8 +6,6 @@ import tachiyomi.domain.manga.model.MangaUpdate
 
 interface RestoreRepository {
 
-    suspend fun getMangaUrlsBySourceId(): Map<Long, Set<String>>
-
     /**
      * Restores [entries] all together: either every one of them is restored or none is. [update] is what to change on each
      * entry once its chapters and history are in place, and is part of the same all-or-nothing restore.