Stream entries when creating and restoring backups

Closes #3850

Co-authored-by: NGB-Was-Taken <76197326+NGB-Was-Taken@users.noreply.github.com>

CHANGELOG.md

@@ -32,6 +32,9 @@ The format is a modified version of [Keep a Changelog](https://keepachangelog.co
 ### Changed
 - Use a new decoder for images Android can't decode itself, like AVIF, JPEG XL and HEIF, and support JPEG 2000 images ([@wwww-wwww](https://github.com/wwww-wwww)) ([#3786](https://github.com/mihonapp/mihon/pull/3786))
 
+### Improved
+- Create and restore backups faster and with less memory ([@NGB-Was-Taken](https://github.com/NGB-Was-Taken), [@AntsyLich](https://github.com/AntsyLich))
+
 ### Removed
 - Remove "Custom hardware bitmap threshold" and "Custom display profile" advanced settings ([@wwww-wwww](https://github.com/wwww-wwww)) ([#3786](https://github.com/mihonapp/mihon/pull/3786))
 

app/src/main/java/eu/kanade/tachiyomi/data/backup/BackupDecoder.kt

@@ -4,8 +4,11 @@ import android.content.Context
 import android.net.Uri
 import dev.zacsweers.metro.Inject
 import eu.kanade.tachiyomi.data.backup.models.Backup
-import kotlinx.serialization.SerializationException
+import eu.kanade.tachiyomi.data.backup.models.BackupManga
+import kotlinx.coroutines.flow.Flow
+import kotlinx.coroutines.flow.flow
 import kotlinx.serialization.protobuf.ProtoBuf
+import okio.BufferedSource
 import okio.buffer
 import okio.gzip
 import okio.source
@@ -19,9 +22,38 @@ class BackupDecoder(
     private val parser: ProtoBuf,
 ) {
     /**
-     * Decode a potentially-gzipped backup.
+     * Decode everything but the entries of a potentially-gzipped backup, counting the entries without decoding them.
      */
-    fun decode(uri: Uri): Backup {
+    fun decodeMetadata(uri: Uri): Metadata {
+        return read(uri) { reader ->
+            orInvalid {
+                var entryCount = 0
+                while (reader.nextEntry()) {
+                    reader.skipEntry()
+                    entryCount++
+                }
+                Metadata(entryCount, reader.readMetadata())
+            }
+        }
+    }
+
+    /**
+     * Decode the entries of a potentially-gzipped backup one at a time, skipping those whose index [include] rejects.
+     */
+    fun decodeEntries(uri: Uri, include: (index: Int) -> Boolean = { true }): Flow<BackupManga> = flow {
+        read(uri) { reader ->
+            var index = 0
+            while (orInvalid { reader.nextEntry() }) {
+                if (include(index++)) {
+                    emit(orInvalid { reader.readEntry() })
+                } else {
+                    orInvalid { reader.skipEntry() }
+                }
+            }
+        }
+    }
+
+    private inline fun <T> read(uri: Uri, block: (BackupReader) -> T): T {
         return context.contentResolver.openInputStream(uri)!!.use { inputStream ->
             val source = inputStream.source().buffer()
 
@@ -29,21 +61,31 @@ class BackupDecoder(
                 require(2)
             }
             val id1id2 = peeked.readShort()
-            val backupString = when (id1id2.toInt()) {
+            val backupSource: BufferedSource = when (id1id2.toInt()) {
                 0x1f8b -> source.gzip().buffer() // 0x1f8b is gzip magic bytes
                 MAGIC_JSON_SIGNATURE1, MAGIC_JSON_SIGNATURE2, MAGIC_JSON_SIGNATURE3 -> {
                     throw IOException(context.stringResource(MR.strings.invalid_backup_file_json))
                 }
                 else -> source
-            }.use { it.readByteArray() }
+            }
 
-            try {
-                parser.decodeFromByteArray(Backup.serializer(), backupString)
-            } catch (_: SerializationException) {
-                throw IOException(context.stringResource(MR.strings.invalid_backup_file_unknown))
+            backupSource.use { block(BackupReader(it, parser)) }
         }
     }
+
+    // Only around reading, so the JSON error above and anything thrown downstream of emit keep their own messages
+    private inline fun <T> orInvalid(block: () -> T): T {
+        return try {
+            block()
+        } catch (e: IOException) {
+            throw IOException(context.stringResource(MR.strings.invalid_backup_file_unknown), e)
         }
+    }
+
+    data class Metadata(
+        val entryCount: Int,
+        val backup: Backup,
+    )
 
     companion object {
         private const val MAGIC_JSON_SIGNATURE1 = 0x7b7d // `{}`

app/src/main/java/eu/kanade/tachiyomi/data/backup/BackupFileValidator.kt

@@ -3,6 +3,7 @@ package eu.kanade.tachiyomi.data.backup
 import android.net.Uri
 import dev.zacsweers.metro.Inject
 import eu.kanade.tachiyomi.data.track.TrackerManager
+import kotlinx.coroutines.CancellationException
 import tachiyomi.domain.source.service.SourceManager
 
 @Inject
@@ -18,8 +19,14 @@ class BackupFileValidator(
      * @return List of missing sources or missing trackers.
      */
     suspend fun validate(uri: Uri): Results {
+        val trackerIds = mutableSetOf<Int>()
         val backup = try {
-            backupDecoder.decode(uri)
+            backupDecoder.decodeEntries(uri).collect { manga ->
+                manga.tracking.mapTo(trackerIds) { it.syncId }
+            }
+            backupDecoder.decodeMetadata(uri).backup
+        } catch (e: CancellationException) {
+            throw e
         } catch (e: Exception) {
             throw IllegalStateException(e)
         }
@@ -38,11 +45,7 @@ class BackupFileValidator(
             .distinct()
             .sorted()
 
-        val trackers = backup.backupManga
-            .flatMap { it.tracking }
-            .map { it.syncId }
-            .distinct()
-        val missingTrackers = trackers
+        val missingTrackers = trackerIds
             .mapNotNull { trackerManager.get(it.toLong()) }
             .filter { !it.isLoggedIn }
             .map { it.name }

app/src/main/java/eu/kanade/tachiyomi/data/backup/BackupStream.kt

@@ -0,0 +1,153 @@
+package eu.kanade.tachiyomi.data.backup
+
+import eu.kanade.tachiyomi.data.backup.models.Backup
+import eu.kanade.tachiyomi.data.backup.models.BackupManga
+import kotlinx.serialization.DeserializationStrategy
+import kotlinx.serialization.SerializationException
+import kotlinx.serialization.protobuf.ProtoBuf
+import okio.Buffer
+import okio.BufferedSink
+import okio.BufferedSource
+import java.io.IOException
+
+/**
+ * Writes the same bytes as encoding a whole [Backup], without holding all of its entries at once.
+ */
+class BackupWriter(
+    private val sink: BufferedSink,
+    private val parser: ProtoBuf,
+) {
+    var isEmpty = true
+        private set
+
+    fun writeEntry(entry: BackupManga) {
+        val bytes = parser.encodeToByteArray(BackupManga.serializer(), entry)
+        sink.writeVarint(ENTRY_TAG)
+        sink.writeVarint(bytes.size.toLong())
+        sink.write(bytes)
+        isEmpty = false
+    }
+
+    /**
+     * Writes the other fields after the entries, as [metadata] without entries of its own.
+     */
+    fun writeMetadata(metadata: Backup) {
+        val bytes = parser.encodeToByteArray(Backup.serializer(), metadata)
+        sink.write(bytes)
+        if (bytes.isNotEmpty()) isEmpty = false
+    }
+}
+
+/**
+ * Reads an encoded [Backup] one entry at a time. Every other field is set aside for [readMetadata], wherever it
+ * appears.
+ *
+ * Malformed or truncated input throws an [IOException].
+ */
+class BackupReader(
+    private val source: BufferedSource,
+    private val parser: ProtoBuf,
+) {
+    private val metadata = Buffer()
+    private var entryLength = NO_ENTRY
+
+    /**
+     * Moves to the next entry, which [readEntry] or [skipEntry] then consumes.
+     *
+     * @return false once the input is exhausted.
+     */
+    fun nextEntry(): Boolean {
+        check(entryLength == NO_ENTRY)
+        while (!source.exhausted()) {
+            val tag = source.readVarint()
+            val fieldNumber = tag ushr 3
+            val wireType = (tag and 0x7).toInt()
+            if (fieldNumber == 0L) throw IOException("Invalid field number 0")
+
+            if (tag == ENTRY_TAG) {
+                entryLength = source.readLength()
+                return true
+            }
+
+            metadata.writeVarint(tag)
+            when (wireType) {
+                WIRE_TYPE_VARINT -> metadata.writeVarint(source.readVarint())
+                WIRE_TYPE_FIXED64 -> metadata.write(source, 8)
+                WIRE_TYPE_LENGTH_DELIMITED -> {
+                    val length = source.readLength()
+                    metadata.writeVarint(length)
+                    metadata.write(source, length)
+                }
+                WIRE_TYPE_FIXED32 -> metadata.write(source, 4)
+                else -> throw IOException("Unsupported wire type $wireType")
+            }
+        }
+        return false
+    }
+
+    fun readEntry(): BackupManga {
+        val bytes = source.readByteArray(takeEntryLength())
+        return decode(BackupManga.serializer(), bytes)
+    }
+
+    fun skipEntry() {
+        source.skip(takeEntryLength())
+    }
+
+    /**
+     * Skips the remaining entries and decodes everything else as a [Backup] without entries.
+     */
+    fun readMetadata(): Backup {
+        if (entryLength != NO_ENTRY) skipEntry()
+        while (nextEntry()) skipEntry()
+        return decode(Backup.serializer(), metadata.readByteArray())
+    }
+
+    private fun takeEntryLength(): Long {
+        check(entryLength != NO_ENTRY)
+        return entryLength.also { entryLength = NO_ENTRY }
+    }
+
+    private fun <T> decode(deserializer: DeserializationStrategy<T>, bytes: ByteArray): T {
+        return try {
+            parser.decodeFromByteArray(deserializer, bytes)
+        } catch (e: SerializationException) {
+            throw IOException(e)
+        }
+    }
+}
+
+private fun BufferedSource.readVarint(): Long {
+    var result = 0L
+    for (shift in 0 until Long.SIZE_BITS step 7) {
+        val byte = readByte().toInt()
+        result = result or ((byte and 0x7F).toLong() shl shift)
+        if ((byte and 0x80) == 0) return result
+    }
+    throw IOException("Malformed varint")
+}
+
+private fun BufferedSource.readLength(): Long {
+    val length = readVarint()
+    if (length !in 0..Int.MAX_VALUE) throw IOException("Invalid length $length")
+    return length
+}
+
+private fun BufferedSink.writeVarint(value: Long) {
+    var remaining = value
+    while ((remaining and 0x7FL.inv()) != 0L) {
+        writeByte((remaining and 0x7F).toInt() or 0x80)
+        remaining = remaining ushr 7
+    }
+    writeByte(remaining.toInt())
+}
+
+private const val WIRE_TYPE_VARINT = 0
+private const val WIRE_TYPE_FIXED64 = 1
+private const val WIRE_TYPE_LENGTH_DELIMITED = 2
+private const val WIRE_TYPE_FIXED32 = 5
+
+// Backup.backupManga is field 1, and each entry is length-delimited
+private const val ENTRY_TAG = 1L shl 3 or WIRE_TYPE_LENGTH_DELIMITED.toLong()
+
+private const val NO_ENTRY = -1L

app/src/main/java/eu/kanade/tachiyomi/data/backup/create/BackupCreator.kt

@@ -8,6 +8,7 @@ import dev.zacsweers.metro.AssistedFactory
 import dev.zacsweers.metro.AssistedInject
 import eu.kanade.tachiyomi.BuildConfig
 import eu.kanade.tachiyomi.data.backup.BackupFileValidator
+import eu.kanade.tachiyomi.data.backup.BackupWriter
 import eu.kanade.tachiyomi.data.backup.create.creators.CategoriesBackupCreator
 import eu.kanade.tachiyomi.data.backup.create.creators.ExtensionStoresBackupCreator
 import eu.kanade.tachiyomi.data.backup.create.creators.MangaBackupCreator
@@ -20,6 +21,8 @@ import eu.kanade.tachiyomi.data.backup.models.BackupManga
 import eu.kanade.tachiyomi.data.backup.models.BackupPreference
 import eu.kanade.tachiyomi.data.backup.models.BackupSource
 import eu.kanade.tachiyomi.data.backup.models.BackupSourcePreferences
+import kotlinx.coroutines.flow.Flow
+import kotlinx.coroutines.flow.emptyFlow
 import kotlinx.serialization.protobuf.ProtoBuf
 import logcat.LogPriority
 import okio.buffer
@@ -85,27 +88,31 @@ class BackupCreator(
             val nonFavoriteManga = if (options.readEntries) mangaRepository.getReadMangaNotInLibrary() else emptyList()
             val backupManga = backupMangas(getFavorites.await() + nonFavoriteManga, options)
 
-            val backup = Backup(
-                backupManga = backupManga,
-                backupCategories = backupCategories(options),
-                backupSources = backupSources(backupManga),
-                backupPreferences = backupAppPreferences(options),
-                backupExtensionStores = backupExtensionStores(options),
-                backupSourcePreferences = backupSourcePreferences(options),
-            )
-
-            val byteArray = parser.encodeToByteArray(Backup.serializer(), backup)
-            if (byteArray.isEmpty()) {
-                throw IllegalStateException(context.stringResource(MR.strings.empty_backup_error))
-            }
-
             file.openOutputStream()
                 .also {
                     // Force overwrite old file
                     (it as? FileOutputStream)?.channel?.truncate(0)
                 }
                 .sink().gzip().buffer().use {
-                    it.write(byteArray)
+                    val writer = BackupWriter(it, parser)
+                    val sourceIds = mutableSetOf<Long>()
+                    backupManga.collect { manga ->
+                        writer.writeEntry(manga)
+                        sourceIds += manga.source
+                    }
+                    writer.writeMetadata(
+                        Backup(
+                            backupManga = emptyList(),
+                            backupCategories = backupCategories(options),
+                            backupSources = backupSources(sourceIds),
+                            backupPreferences = backupAppPreferences(options),
+                            backupExtensionStores = backupExtensionStores(options),
+                            backupSourcePreferences = backupSourcePreferences(options),
+                        ),
+                    )
+                    if (writer.isEmpty) {
+                        throw IllegalStateException(context.stringResource(MR.strings.empty_backup_error))
+                    }
                 }
             val fileUri = file.uri
 
@@ -130,14 +137,14 @@ class BackupCreator(
         return categoriesBackupCreator()
     }
 
-    private suspend fun backupMangas(mangas: List<Manga>, options: BackupOptions): List<BackupManga> {
-        if (!options.libraryEntries) return emptyList()
+    private fun backupMangas(mangas: List<Manga>, options: BackupOptions): Flow<BackupManga> {
+        if (!options.libraryEntries) return emptyFlow()
 
         return mangaBackupCreator(mangas, options)
     }
 
-    private suspend fun backupSources(mangas: List<BackupManga>): List<BackupSource> {
-        return sourcesBackupCreator(mangas)
+    private suspend fun backupSources(sourceIds: Set<Long>): List<BackupSource> {
+        return sourcesBackupCreator(sourceIds)
     }
 
     private fun backupAppPreferences(options: BackupOptions): List<BackupPreference> {

app/src/main/java/eu/kanade/tachiyomi/data/backup/create/creators/MangaBackupCreator.kt

@@ -7,11 +7,13 @@ import eu.kanade.tachiyomi.data.backup.models.BackupManga
 import eu.kanade.tachiyomi.data.backup.models.toBackupChapter
 import eu.kanade.tachiyomi.data.backup.models.toBackupTracking
 import eu.kanade.tachiyomi.ui.reader.setting.ReadingMode
+import kotlinx.coroutines.flow.Flow
+import kotlinx.coroutines.flow.flow
 import mihon.core.common.extensions.toByteArray
 import mihon.core.common.extensions.toEpochMillisOrZero
-import tachiyomi.domain.category.interactor.GetCategories
+import tachiyomi.domain.category.repository.CategoryRepository
 import tachiyomi.domain.chapter.repository.ChapterRepository
-import tachiyomi.domain.history.interactor.GetHistory
+import tachiyomi.domain.history.repository.HistoryRepository
 import tachiyomi.domain.manga.model.Manga
 import tachiyomi.domain.manga.repository.MangaRepository
 import tachiyomi.domain.track.repository.TrackRepository
@@ -20,62 +22,71 @@ import tachiyomi.domain.track.repository.TrackRepository
 class MangaBackupCreator(
     private val mangaRepository: MangaRepository,
     private val chapterRepository: ChapterRepository,
+    private val categoryRepository: CategoryRepository,
+    private val historyRepository: HistoryRepository,
     private val trackRepository: TrackRepository,
-    private val getCategories: GetCategories,
-    private val getHistory: GetHistory,
 ) {
 
-    suspend operator fun invoke(mangas: List<Manga>, options: BackupOptions): List<BackupManga> {
-        return mangas.map {
-            backupManga(it, options)
+    /**
+     * Builds the entries a chunk at a time, so only one chunk's chapters are held at once.
+     */
+    operator fun invoke(mangas: List<Manga>, options: BackupOptions): Flow<BackupManga> = flow {
+        mangas.chunked(CHUNK_SIZE).forEach { chunk ->
+            backupMangas(chunk, options).forEach { emit(it) }
         }
     }
 
-    private suspend fun backupManga(manga: Manga, options: BackupOptions): BackupManga {
-        // Entry for this manga
+    private suspend fun backupMangas(mangas: List<Manga>, options: BackupOptions): List<BackupManga> {
+        val mangaIds = mangas.map { it.id }
+
+        val excludedScanlatorsByMangaId = mangaRepository.getExcludedScanlatorsByMangaIds(mangaIds)
+        // History is backed up by chapter URL, so it needs the chapters too
+        val chaptersByMangaId = if (options.chapters || options.history) {
+            chapterRepository.getChaptersByMangaIds(mangaIds)
+        } else {
+            emptyMap()
+        }
+        val categoriesByMangaId = if (options.categories) {
+            categoryRepository.getCategoriesByMangaIds(mangaIds)
+        } else {
+            emptyMap()
+        }
+        val tracksByMangaId = if (options.tracking) trackRepository.getTracksByMangaIds(mangaIds) else emptyMap()
+        val historyByMangaId = if (options.history) historyRepository.getHistoryByMangaIds(mangaIds) else emptyMap()
+
+        return mangas.map { manga ->
             val mangaObject = manga.toBackupManga()
+            val chapters = chaptersByMangaId[manga.id].orEmpty()
 
-        mangaObject.excludedScanlators = mangaRepository.getExcludedScanlators(manga.id).toList()
+            mangaObject.excludedScanlators = excludedScanlatorsByMangaId[manga.id].orEmpty()
 
             if (options.chapters) {
-            // Backup all the chapters
-            chapterRepository.getChapterByMangaId(manga.id, applyScanlatorFilter = false)
-                .map { it.toBackupChapter() }
-                .takeUnless { it.isEmpty() }
-                ?.let { mangaObject.chapters = it }
+                mangaObject.chapters = chapters.map { it.toBackupChapter() }
             }
 
             if (options.categories) {
-            // Backup categories for this manga
-            val categoriesForManga = getCategories.await(manga.id)
-            if (categoriesForManga.isNotEmpty()) {
-                mangaObject.categories = categoriesForManga.map { it.order }
-            }
+                mangaObject.categories = categoriesByMangaId[manga.id].orEmpty().map { it.order }
             }
 
             if (options.tracking) {
-            val tracks = trackRepository.getTracksByMangaId(manga.id).map { it.toBackupTracking() }
-            if (tracks.isNotEmpty()) {
-                mangaObject.tracking = tracks
-            }
+                mangaObject.tracking = tracksByMangaId[manga.id].orEmpty().map { it.toBackupTracking() }
             }
 
             if (options.history) {
-            val historyByMangaId = getHistory.await(manga.id)
-            if (historyByMangaId.isNotEmpty()) {
-                val chapterUrlsById = chapterRepository.getChapterByMangaId(manga.id).associate { it.id to it.url }
-                val history = historyByMangaId.mapNotNull { history ->
+                val chapterUrlsById = chapters.associate { it.id to it.url }
+                mangaObject.history = historyByMangaId[manga.id].orEmpty().mapNotNull { history ->
                     // A chapter removed since its history was read takes that history with it
                     val url = chapterUrlsById[history.chapterId] ?: return@mapNotNull null
                     BackupHistory(url, history.readAt.toEpochMillisOrZero(), history.readDuration)
                 }
-                if (history.isNotEmpty()) {
-                    mangaObject.history = history
             }
+
+            mangaObject
         }
     }
 
-        return mangaObject
+    private companion object {
+        const val CHUNK_SIZE = 100
     }
 }
 

app/src/main/java/eu/kanade/tachiyomi/data/backup/create/creators/SourcesBackupCreator.kt

@@ -1,7 +1,6 @@
 package eu.kanade.tachiyomi.data.backup.create.creators
 
 import dev.zacsweers.metro.Inject
-import eu.kanade.tachiyomi.data.backup.models.BackupManga
 import eu.kanade.tachiyomi.data.backup.models.BackupSource
 import eu.kanade.tachiyomi.source.Source
 import tachiyomi.domain.source.service.SourceManager
@@ -11,11 +10,8 @@ class SourcesBackupCreator(
     private val sourceManager: SourceManager,
 ) {
 
-    suspend operator fun invoke(mangas: List<BackupManga>): List<BackupSource> {
-        return mangas
-            .map(BackupManga::source)
-            .distinct()
-            .map { sourceManager.getOrStub(it).toBackupSource() }
+    suspend operator fun invoke(sourceIds: Set<Long>): List<BackupSource> {
+        return sourceIds.map { sourceManager.getOrStub(it).toBackupSource() }
     }
 }
 

app/src/main/java/eu/kanade/tachiyomi/data/backup/restore/BackupRestorer.kt

@@ -9,7 +9,6 @@ import eu.kanade.tachiyomi.data.backup.BackupDecoder
 import eu.kanade.tachiyomi.data.backup.BackupNotifier
 import eu.kanade.tachiyomi.data.backup.models.BackupCategory
 import eu.kanade.tachiyomi.data.backup.models.BackupExtensionStore
-import eu.kanade.tachiyomi.data.backup.models.BackupManga
 import eu.kanade.tachiyomi.data.backup.models.BackupPreference
 import eu.kanade.tachiyomi.data.backup.models.BackupSource
 import eu.kanade.tachiyomi.data.backup.models.BackupSourcePreferences
@@ -23,6 +22,12 @@ import kotlinx.coroutines.CoroutineScope
 import kotlinx.coroutines.Job
 import kotlinx.coroutines.coroutineScope
 import kotlinx.coroutines.ensureActive
+import kotlinx.coroutines.flow.chunked
+import kotlinx.coroutines.flow.emitAll
+import kotlinx.coroutines.flow.filter
+import kotlinx.coroutines.flow.flow
+import kotlinx.coroutines.flow.map
+import kotlinx.coroutines.flow.withIndex
 import kotlinx.coroutines.launch
 import logcat.LogPriority
 import tachiyomi.core.common.i18n.stringResource
@@ -32,6 +37,7 @@ import tachiyomi.domain.source.service.SourceManager
 import tachiyomi.i18n.MR
 import java.io.File
 import java.text.SimpleDateFormat
+import java.util.BitSet
 import java.util.Date
 import java.util.Locale
 import java.util.concurrent.CopyOnWriteArrayList
@@ -97,7 +103,7 @@ class BackupRestorer(
     }
 
     private suspend fun restoreFromFile(uri: Uri, options: RestoreOptions) {
-        val backup = backupDecoder.decode(uri)
+        val (entryCount, backup) = backupDecoder.decodeMetadata(uri)
 
         // Store source mapping for error messages
         val backupMaps = backup.backupSources
@@ -105,7 +111,7 @@ class BackupRestorer(
 
         if (options.libraryEntries) {
             restoreSourceNames(backupMaps)
-            restoreAmount += backup.backupManga.size
+            restoreAmount += entryCount
         }
         if (options.categories) {
             restoreAmount += 1
@@ -138,7 +144,7 @@ class BackupRestorer(
             }
             if (options.libraryEntries) {
                 restoreManga(
-                    backup.backupManga,
+                    uri,
                     if (options.categories) backup.backupCategories else emptyList(),
                     restoreCategoriesJob,
                 )
@@ -175,14 +181,31 @@ class BackupRestorer(
     }
 
     private fun CoroutineScope.restoreManga(
-        backupMangas: List<BackupManga>,
+        uri: Uri,
         backupCategories: List<BackupCategory>,
         categoriesRestoreJob: Job?,
     ) = launch {
         categoriesRestoreJob?.join()
-        mangaRestorer.sortByNew(backupMangas)
+
+        // New entries go first. Rather than hold the others back in memory, a second pass reads them again.
+        val isExisting = mangaRestorer.getExistingMatcher()
+        val existingIndices = BitSet()
+        val newMangas = backupDecoder.decodeEntries(uri)
+            .withIndex()
+            .filter { (index, manga) ->
+                val exists = isExisting(manga)
+                if (exists) existingIndices.set(index)
+                !exists
+            }
+            .map { it.value }
+        val backupMangas = flow {
+            emitAll(newMangas)
+            emitAll(backupDecoder.decodeEntries(uri, existingIndices::get))
+        }
+
+        backupMangas
             .chunked(100)
-            .forEach { chunk ->
+            .collect { chunk ->
                 val restoredAsBatch = try {
                     ensureActive()
                     mangaRestorer.restore(chunk, backupCategories)

app/src/main/java/eu/kanade/tachiyomi/data/backup/restore/restorers/MangaRestorer.kt

@@ -23,11 +23,13 @@ class MangaRestorer(
     private val now = Clock.System.now().toLocalDateTime(timeZone)
     private val currentFetchWindow = fetchInterval.getWindow(now.date, timeZone)
 
-    suspend fun sortByNew(backupMangas: List<BackupManga>): List<BackupManga> {
+    /**
+     * Returns whether an entry was already in the database at the time of this call.
+     */
+    suspend fun getExistingMatcher(): (BackupManga) -> Boolean {
         val urlsBySource = restoreRepository.getMangaUrlsBySourceId()
 
-        return backupMangas
-            .sortedBy { it.url in urlsBySource[it.source].orEmpty() }
+        return { it.url in urlsBySource[it.source].orEmpty() }
     }
 
     /**

app/src/test/java/eu/kanade/tachiyomi/data/backup/BackupStreamTest.kt

@@ -0,0 +1,202 @@
+package eu.kanade.tachiyomi.data.backup
+
+import eu.kanade.tachiyomi.data.backup.models.Backup
+import eu.kanade.tachiyomi.data.backup.models.BackupCategory
+import eu.kanade.tachiyomi.data.backup.models.BackupChapter
+import eu.kanade.tachiyomi.data.backup.models.BackupExtensionStore
+import eu.kanade.tachiyomi.data.backup.models.BackupHistory
+import eu.kanade.tachiyomi.data.backup.models.BackupManga
+import eu.kanade.tachiyomi.data.backup.models.BackupPreference
+import eu.kanade.tachiyomi.data.backup.models.BackupSource
+import eu.kanade.tachiyomi.data.backup.models.BackupSourcePreferences
+import eu.kanade.tachiyomi.data.backup.models.BackupTracking
+import eu.kanade.tachiyomi.data.backup.models.BooleanPreferenceValue
+import eu.kanade.tachiyomi.data.backup.models.StringPreferenceValue
+import eu.kanade.tachiyomi.data.backup.models.StringSetPreferenceValue
+import io.kotest.matchers.shouldBe
+import kotlinx.serialization.protobuf.ProtoBuf
+import okio.Buffer
+import org.junit.jupiter.api.Test
+import org.junit.jupiter.api.assertThrows
+import org.junit.jupiter.api.parallel.Execution
+import org.junit.jupiter.api.parallel.ExecutionMode
+import java.io.IOException
+
+@Execution(ExecutionMode.CONCURRENT)
+class BackupStreamTest {
+
+    private val parser = ProtoBuf
+
+    @Test
+    fun `writer output is the plain encoding`() {
+        val backup = backup()
+
+        val bytes = write(backup)
+
+        bytes shouldBe encode(backup)
+        encode(parser.decodeFromByteArray(Backup.serializer(), bytes)) shouldBe encode(backup)
+    }
+
+    @Test
+    fun `reader reads the plain encoding`() {
+        val backup = backup()
+
+        val (entries, metadata) = read(encode(backup))
+
+        entries.map(::encode) shouldBe backup.backupManga.map(::encode)
+        encode(metadata) shouldBe encode(backup.copy(backupManga = emptyList()))
+    }
+
+    @Test
+    fun `reader counts entries without decoding them`() {
+        val backup = backup()
+        val reader = BackupReader(Buffer().write(encode(backup)), parser)
+
+        var count = 0
+        while (reader.nextEntry()) {
+            reader.skipEntry()
+            count++
+        }
+
+        count shouldBe backup.backupManga.size
+        encode(reader.readMetadata()) shouldBe encode(backup.copy(backupManga = emptyList()))
+    }
+
+    @Test
+    fun `reader takes fields in any order`() {
+        val backup = backup()
+        val (first, second, third) = backup.backupManga
+        val input = Buffer()
+            .write(encode(Backup(emptyList(), backupSources = backup.backupSources)))
+            .write(encode(Backup(listOf(first))))
+            // Fields Backup doesn't know, one of each remaining wire type
+            .writeByte(3 shl 3 or 0).writeByte(0x96).writeByte(0x01)
+            .writeByte(4 shl 3 or 1).writeLongLe(42)
+            .write(encode(Backup(listOf(second), backupCategories = backup.backupCategories)))
+            .writeByte(5 shl 3 or 5).writeIntLe(42)
+            .write(encode(Backup(emptyList(), backupPreferences = backup.backupPreferences)))
+            .write(encode(Backup(listOf(third), backupExtensionStores = backup.backupExtensionStores)))
+            .write(encode(Backup(emptyList(), backupSourcePreferences = backup.backupSourcePreferences)))
+            .readByteArray()
+
+        val (entries, metadata) = read(input)
+
+        entries.map(::encode) shouldBe backup.backupManga.map(::encode)
+        encode(metadata) shouldBe encode(backup.copy(backupManga = emptyList()))
+    }
+
+    @Test
+    fun `empty backup`() {
+        val sink = Buffer()
+        val writer = BackupWriter(sink, parser)
+        writer.writeMetadata(Backup(emptyList()))
+
+        writer.isEmpty shouldBe true
+        sink.size shouldBe 0L
+
+        val (entries, metadata) = read(ByteArray(0))
+        entries shouldBe emptyList()
+        encode(metadata) shouldBe ByteArray(0)
+    }
+
+    @Test
+    fun `truncated input throws`() {
+        val bytes = encode(backup())
+
+        // Cut inside the last entry, inside the metadata, and right after a tag
+        val entryEnd = encode(Backup(backup().backupManga)).size
+        listOf(entryEnd - 1, bytes.size - 1, entryEnd + 1).forEach { size ->
+            assertThrows<IOException> { read(bytes.copyOf(size)) }
+        }
+    }
+
+    @Test
+    fun `garbage input throws`() {
+        listOf(
+            // Wire type 7 doesn't exist
+            byteArrayOf(0x0F),
+            // Field number 0
+            byteArrayOf(0x02, 0x00),
+            // A varint that never ends
+            ByteArray(11) { 0xFF.toByte() },
+            // An entry longer than the input
+            byteArrayOf(0x0A, 0x10, 0x01),
+            // An entry that isn't a BackupManga
+            byteArrayOf(0x0A, 0x02, 0x0F, 0x00),
+        ).forEach { input ->
+            assertThrows<IOException> { read(input) }
+        }
+    }
+
+    private fun write(backup: Backup): ByteArray {
+        val sink = Buffer()
+        val writer = BackupWriter(sink, parser)
+        backup.backupManga.forEach(writer::writeEntry)
+        writer.writeMetadata(backup.copy(backupManga = emptyList()))
+        writer.isEmpty shouldBe false
+        return sink.readByteArray()
+    }
+
+    private fun read(bytes: ByteArray): Pair<List<BackupManga>, Backup> {
+        val reader = BackupReader(Buffer().write(bytes), parser)
+        val entries = buildList {
+            while (reader.nextEntry()) add(reader.readEntry())
+        }
+        return entries to reader.readMetadata()
+    }
+
+    private fun encode(backup: Backup) = parser.encodeToByteArray(Backup.serializer(), backup)
+
+    private fun encode(manga: BackupManga) = parser.encodeToByteArray(BackupManga.serializer(), manga)
+
+    private fun backup() = Backup(
+        backupManga = listOf(
+            BackupManga(
+                source = 1L,
+                url = "/manga/1",
+                title = "First",
+                genre = listOf("Action", "Drama"),
+                chapters = listOf(
+                    BackupChapter(url = "/chapter/1", name = "Chapter 1", read = true, chapterNumber = 1f),
+                    BackupChapter(url = "/chapter/2", name = "Chapter 2", bookmark = true, lastPageRead = 4),
+                ),
+                categories = listOf(0L, 1L),
+                tracking = listOf(BackupTracking(syncId = 2, libraryId = 3L, title = "First", mediaId = 7L)),
+                history = listOf(BackupHistory(url = "/chapter/1", lastRead = 1_700_000_000_000L, readDuration = 60)),
+                excludedScanlators = listOf("Group"),
+            ),
+            BackupManga(source = 2L, url = "/manga/2", title = "Second", favorite = false),
+            BackupManga(
+                source = 1L,
+                url = "/manga/3",
+                title = "Third",
+                description = "x".repeat(300),
+                chapters = List(200) { BackupChapter(url = "/chapter/3/$it", name = "Chapter $it") },
+            ),
+        ),
+        backupCategories = listOf(
+            BackupCategory(name = "Reading", order = 0),
+            BackupCategory(name = "Done", order = 1),
+        ),
+        backupSources = listOf(BackupSource(name = "One", sourceId = 1L), BackupSource(name = "Two", sourceId = 2L)),
+        backupPreferences = listOf(
+            BackupPreference("flag", BooleanPreferenceValue(true)),
+            BackupPreference("tags", StringSetPreferenceValue(setOf("a", "b"))),
+        ),
+        backupSourcePreferences = listOf(
+            BackupSourcePreferences("source_1", listOf(BackupPreference("lang", StringPreferenceValue("en")))),
+        ),
+        backupExtensionStores = listOf(
+            BackupExtensionStore(
+                indexUrl = "https://example.org/index.min.json",
+                name = "Store",
+                badgeLabel = null,
+                signingKey = "key",
+                contactWebsite = "https://example.org",
+                contactDiscord = null,
+                isLegacy = false,
+                extensionListUrl = null,
+            ),
+        ),
+    )
+}

data/src/main/java/tachiyomi/data/category/CategoryRepositoryImpl.kt

@@ -50,6 +50,15 @@ class CategoryRepositoryImpl(
             .subscribeToList()
     }
 
+    override suspend fun getCategoriesByMangaIds(mangaIds: List<Long>): Map<Long, List<Category>> {
+        return database.categoryQueries
+            .getCategoriesByMangaIds(mangaIds) { mangaId, id, name, order, flags ->
+                mangaId to mapCategory(id, name, order, flags)
+            }
+            .awaitAsList()
+            .groupBy({ it.first }, { it.second })
+    }
+
     override suspend fun insert(category: NewCategory) {
         database.categoryQueries.insert(name = category.name, flags = category.flags)
     }

data/src/main/java/tachiyomi/data/chapter/ChapterRepositoryImpl.kt

@@ -113,6 +113,13 @@ class ChapterRepositoryImpl(
             .awaitAsList()
     }
 
+    override suspend fun getChaptersByMangaIds(mangaIds: List<Long>): Map<Long, List<Chapter>> {
+        return database.chapterQueries
+            .getChaptersByMangaIds(mangaIds, ::mapChapter)
+            .awaitAsList()
+            .groupBy { it.mangaId }
+    }
+
     override suspend fun getChapterSummaryByMangaId(mangaId: Long): ChapterSummary {
         return database.chapterQueries
             .getChapterSummaryByMangaId(mangaId) { total, latestChapterNumber ->

data/src/main/java/tachiyomi/data/history/HistoryRepositoryImpl.kt

@@ -48,6 +48,15 @@ class HistoryRepositoryImpl(
             .awaitAsList()
     }
 
+    override suspend fun getHistoryByMangaIds(mangaIds: List<Long>): Map<Long, List<History>> {
+        return database.historyQueries
+            .getHistoryByMangaIds(mangaIds) { mangaId, id, chapterId, readAt, readDuration ->
+                mangaId to HistoryMapper.mapHistory(id, chapterId, readAt, readDuration)
+            }
+            .awaitAsList()
+            .groupBy({ it.first }, { it.second })
+    }
+
     override suspend fun resetHistory(historyId: Long) {
         try {
             database.historyQueries.resetHistoryById(historyId)

data/src/main/java/tachiyomi/data/manga/MangaRepositoryImpl.kt

@@ -144,6 +144,13 @@ class MangaRepositoryImpl(
             .toSet()
     }
 
+    override suspend fun getExcludedScanlatorsByMangaIds(mangaIds: List<Long>): Map<Long, List<String>> {
+        return database.excluded_scanlatorQueries
+            .getExcludedScanlatorsByMangaIds(mangaIds)
+            .awaitAsList()
+            .groupBy({ it.manga_id }, { it.scanlator })
+    }
+
     override fun getExcludedScanlatorsAsFlow(mangaId: Long): Flow<Set<String>> {
         return database.excluded_scanlatorQueries
             .getExcludedScanlatorsByMangaId(mangaId)

data/src/main/java/tachiyomi/data/track/TrackRepositoryImpl.kt

@@ -31,6 +31,13 @@ class TrackRepositoryImpl(
             .awaitAsList()
     }
 
+    override suspend fun getTracksByMangaIds(mangaIds: List<Long>): Map<Long, List<Track>> {
+        return database.manga_trackQueries
+            .getTracksByMangaIds(mangaIds, TrackMapper::mapTrack)
+            .awaitAsList()
+            .groupBy { it.mangaId }
+    }
+
     override suspend fun getTracks(): List<Track> {
         return database.manga_trackQueries
             .getTracks(TrackMapper::mapTrack)

data/src/main/sqldelight/tachiyomi/data/category.sq

@@ -34,6 +34,14 @@ ON C.id = MC.category_id
 WHERE MC.manga_id = :mangaId
 ORDER BY C.`order`;
 
+getCategoriesByMangaIds:
+SELECT MC.manga_id, C.id, C.name, C.`order`, C.flags
+FROM category C
+JOIN manga_category MC
+ON C.id = MC.category_id
+WHERE MC.manga_id IN :mangaIds
+ORDER BY MC.manga_id, C.`order`;
+
 insert:
 -- The position is taken inside the insert, so two categories created at once cannot claim the same one
 INSERT INTO category(name, `order`, flags)

data/src/main/sqldelight/tachiyomi/data/chapter.sq

@@ -48,6 +48,16 @@ AND (
     OR ES.scanlator IS NULL
 );
 
+getChaptersByMangaIds:
+-- The unique index's order, which getChaptersByMangaId returns too, so backups keep their chapter order
+SELECT
+    id, manga_id, remote_url, remote_name, remote_scanlator, remote_chapter_number,
+    remote_date_upload, remote_order, remote_memo, user_read, user_bookmark,
+    user_last_page_read, state_date_fetch
+FROM chapter
+WHERE manga_id IN :mangaIds
+ORDER BY manga_id, remote_url;
+
 getChapterSummaryByMangaId:
 SELECT count(*) AS total, max(remote_chapter_number) AS latestChapterNumber
 FROM chapter

data/src/main/sqldelight/tachiyomi/data/excluded_scanlator.sq

@@ -21,3 +21,9 @@ getExcludedScanlatorsByMangaId:
 SELECT scanlator
 FROM excluded_scanlator
 WHERE manga_id = :mangaId;
+
+getExcludedScanlatorsByMangaIds:
+SELECT manga_id, scanlator
+FROM excluded_scanlator
+WHERE manga_id IN :mangaIds
+ORDER BY manga_id, scanlator;

data/src/main/sqldelight/tachiyomi/data/history.sq

@@ -20,6 +20,12 @@ SELECT id, chapter_id, read_at, read_duration
 FROM history
 WHERE manga_id = :mangaId;
 
+getHistoryByMangaIds:
+SELECT manga_id, id, chapter_id, read_at, read_duration
+FROM history
+WHERE manga_id IN :mangaIds
+ORDER BY manga_id, id;
+
 resetHistoryById:
 UPDATE history
 SET read_at = NULL

data/src/main/sqldelight/tachiyomi/data/manga_track.sq

@@ -48,6 +48,14 @@ SELECT
 FROM manga_track
 WHERE manga_id = :mangaId;
 
+getTracksByMangaIds:
+SELECT
+    id, manga_id, tracker_id, remote_id, remote_url, library_id, title, last_chapter_read,
+    total_chapters, status, score, start_date, finish_date, private
+FROM manga_track
+WHERE manga_id IN :mangaIds
+ORDER BY manga_id, tracker_id;
+
 upsert:
 INSERT INTO manga_track(
     manga_id, tracker_id, remote_id, remote_url, library_id, title, last_chapter_read,

domain/src/main/java/tachiyomi/domain/category/repository/CategoryRepository.kt

@@ -16,6 +16,8 @@ interface CategoryRepository {
 
     fun getCategoriesByMangaIdAsFlow(mangaId: Long): Flow<List<Category>>
 
+    suspend fun getCategoriesByMangaIds(mangaIds: List<Long>): Map<Long, List<Category>>
+
     suspend fun insert(category: NewCategory)
 
     suspend fun insertAll(categories: List<NewCategory>)

domain/src/main/java/tachiyomi/domain/chapter/repository/ChapterRepository.kt

@@ -20,6 +20,8 @@ interface ChapterRepository {
 
     suspend fun getChapterByMangaId(mangaId: Long, applyScanlatorFilter: Boolean = false): List<Chapter>
 
+    suspend fun getChaptersByMangaIds(mangaIds: List<Long>): Map<Long, List<Chapter>>
+
     suspend fun getChapterSummaryByMangaId(mangaId: Long): ChapterSummary
 
     suspend fun getScanlatorsByMangaId(mangaId: Long): List<String>

domain/src/main/java/tachiyomi/domain/history/repository/HistoryRepository.kt

@@ -15,6 +15,8 @@ interface HistoryRepository {
 
     suspend fun getHistoryByMangaId(mangaId: Long): List<History>
 
+    suspend fun getHistoryByMangaIds(mangaIds: List<Long>): Map<Long, List<History>>
+
     suspend fun resetHistory(historyId: Long)
 
     suspend fun resetHistoryByMangaId(mangaId: Long)

domain/src/main/java/tachiyomi/domain/manga/repository/MangaRepository.kt

@@ -43,6 +43,8 @@ interface MangaRepository {
 
     suspend fun getExcludedScanlators(mangaId: Long): Set<String>
 
+    suspend fun getExcludedScanlatorsByMangaIds(mangaIds: List<Long>): Map<Long, List<String>>
+
     fun getExcludedScanlatorsAsFlow(mangaId: Long): Flow<Set<String>>
 
     suspend fun setExcludedScanlators(mangaId: Long, excludedScanlators: Set<String>)

domain/src/main/java/tachiyomi/domain/track/repository/TrackRepository.kt

@@ -9,6 +9,8 @@ interface TrackRepository {
 
     suspend fun getTracksByMangaId(mangaId: Long): List<Track>
 
+    suspend fun getTracksByMangaIds(mangaIds: List<Long>): Map<Long, List<Track>>
+
     suspend fun getTracks(): List<Track>
 
     fun getTracksAsFlow(): Flow<List<Track>>