mirror/mihon
1
0
Fork 0
mirror of https://github.com/mihonapp/mihon.git synced 2026-10-16 13:48:09 -07:00
Code Issues Packages Projects Releases Wiki Activity

Stream entries when creating and restoring backups

Browse Source 
Closes #3850

Co-authored-by: NGB-Was-Taken <76197326+NGB-Was-Taken@users.noreply.github.com>
This commit is contained in:
AntsyLich
2026-10-13 01:51:35 +06:00
co-authored by NGB-Was-Taken
parent 5a5ff22c37
commit 771ba33a33
25 changed files with 619 additions and 90 deletions
Download Patch File Download Diff File Expand all files Collapse all files
CHANGELOG.md
+3
View File
@@ -32,6 +32,9 @@ The format is a modified version of [Keep a Changelog](https://keepachangelog.co
### Changed
- Use a new decoder for images Android can't decode itself, like AVIF, JPEG XL and HEIF, and support JPEG 2000 images ([@wwww-wwww](https://github.com/wwww-wwww)) ([#3786](https://github.com/mihonapp/mihon/pull/3786))
### Improved
- Create and restore backups faster and with less memory ([@NGB-Was-Taken](https://github.com/NGB-Was-Taken), [@AntsyLich](https://github.com/AntsyLich))
### Removed
- Remove "Custom hardware bitmap threshold" and "Custom display profile" advanced settings ([@wwww-wwww](https://github.com/wwww-wwww)) ([#3786](https://github.com/mihonapp/mihon/pull/3786))
app/src/main/java/eu/kanade/tachiyomi/data/backup/BackupDecoder.kt
+52 -10
View File
@@ -4,8 +4,11 @@ import android.content.Context
import android.net.Uri
import dev.zacsweers.metro.Inject
import eu.kanade.tachiyomi.data.backup.models.Backup
import kotlinx.serialization.SerializationException
import eu.kanade.tachiyomi.data.backup.models.BackupManga
import kotlinx.coroutines.flow.Flow
import kotlinx.coroutines.flow.flow
import kotlinx.serialization.protobuf.ProtoBuf
import okio.BufferedSource
import okio.buffer
import okio.gzip
import okio.source
@@ -19,9 +22,38 @@ class BackupDecoder(
private val parser: ProtoBuf,
) {
/**
* Decode a potentially-gzipped backup.
* Decode everything but the entries of a potentially-gzipped backup, counting the entries without decoding them.
*/
fun decode(uri: Uri): Backup {
fun decodeMetadata(uri: Uri): Metadata {
return read(uri) { reader ->
orInvalid {
var entryCount = 0
while (reader.nextEntry()) {
reader.skipEntry()
entryCount++
}
Metadata(entryCount, reader.readMetadata())
}
}
}
/**
* Decode the entries of a potentially-gzipped backup one at a time, skipping those whose index [include] rejects.
*/
fun decodeEntries(uri: Uri, include: (index: Int) -> Boolean = { true }): Flow<BackupManga> = flow {
read(uri) { reader ->
var index = 0
while (orInvalid { reader.nextEntry() }) {
if (include(index++)) {
emit(orInvalid { reader.readEntry() })
} else {
orInvalid { reader.skipEntry() }
}
}
}
}
private inline fun <T> read(uri: Uri, block: (BackupReader) -> T): T {
return context.contentResolver.openInputStream(uri)!!.use { inputStream ->
val source = inputStream.source().buffer()
@@ -29,22 +61,32 @@ class BackupDecoder(
require(2)
}
val id1id2 = peeked.readShort()
val backupString = when (id1id2.toInt()) {
val backupSource: BufferedSource = when (id1id2.toInt()) {
0x1f8b -> source.gzip().buffer() // 0x1f8b is gzip magic bytes
MAGIC_JSON_SIGNATURE1, MAGIC_JSON_SIGNATURE2, MAGIC_JSON_SIGNATURE3 -> {
throw IOException(context.stringResource(MR.strings.invalid_backup_file_json))
}
else -> source
}.use { it.readByteArray() }
try {
parser.decodeFromByteArray(Backup.serializer(), backupString)
} catch (_: SerializationException) {
throw IOException(context.stringResource(MR.strings.invalid_backup_file_unknown))
}
backupSource.use { block(BackupReader(it, parser)) }
}
}
// Only around reading, so the JSON error above and anything thrown downstream of emit keep their own messages
private inline fun <T> orInvalid(block: () -> T): T {
return try {
block()
} catch (e: IOException) {
throw IOException(context.stringResource(MR.strings.invalid_backup_file_unknown), e)
}
}
data class Metadata(
val entryCount: Int,
val backup: Backup,
)
companion object {
private const val MAGIC_JSON_SIGNATURE1 = 0x7b7d // `{}`
private const val MAGIC_JSON_SIGNATURE2 = 0x7b22 // `{"`
app/src/main/java/eu/kanade/tachiyomi/data/backup/BackupFileValidator.kt
+9 -6
View File
@@ -3,6 +3,7 @@ package eu.kanade.tachiyomi.data.backup
import android.net.Uri
import dev.zacsweers.metro.Inject
import eu.kanade.tachiyomi.data.track.TrackerManager
import kotlinx.coroutines.CancellationException
import tachiyomi.domain.source.service.SourceManager
@Inject
@@ -18,8 +19,14 @@ class BackupFileValidator(
* @return List of missing sources or missing trackers.
*/
suspend fun validate(uri: Uri): Results {
val trackerIds = mutableSetOf<Int>()
val backup = try {
backupDecoder.decode(uri)
backupDecoder.decodeEntries(uri).collect { manga ->
manga.tracking.mapTo(trackerIds) { it.syncId }
}
backupDecoder.decodeMetadata(uri).backup
} catch (e: CancellationException) {
throw e
} catch (e: Exception) {
throw IllegalStateException(e)
}
@@ -38,11 +45,7 @@ class BackupFileValidator(
.distinct()
.sorted()
val trackers = backup.backupManga
.flatMap { it.tracking }
.map { it.syncId }
.distinct()
val missingTrackers = trackers
val missingTrackers = trackerIds
.mapNotNull { trackerManager.get(it.toLong()) }
.filter { !it.isLoggedIn }
.map { it.name }
app/src/main/java/eu/kanade/tachiyomi/data/backup/BackupStream.kt
+153
View File
@@ -0,0 +1,153 @@
package eu.kanade.tachiyomi.data.backup
import eu.kanade.tachiyomi.data.backup.models.Backup
import eu.kanade.tachiyomi.data.backup.models.BackupManga
import kotlinx.serialization.DeserializationStrategy
import kotlinx.serialization.SerializationException
import kotlinx.serialization.protobuf.ProtoBuf
import okio.Buffer
import okio.BufferedSink
import okio.BufferedSource
import java.io.IOException
/**
* Writes the same bytes as encoding a whole [Backup], without holding all of its entries at once.
*/
class BackupWriter(
private val sink: BufferedSink,
private val parser: ProtoBuf,
) {
var isEmpty = true
private set
fun writeEntry(entry: BackupManga) {
val bytes = parser.encodeToByteArray(BackupManga.serializer(), entry)
sink.writeVarint(ENTRY_TAG)
sink.writeVarint(bytes.size.toLong())
sink.write(bytes)
isEmpty = false
}
/**
* Writes the other fields after the entries, as [metadata] without entries of its own.
*/
fun writeMetadata(metadata: Backup) {
val bytes = parser.encodeToByteArray(Backup.serializer(), metadata)
sink.write(bytes)
if (bytes.isNotEmpty()) isEmpty = false
}
}
/**
* Reads an encoded [Backup] one entry at a time. Every other field is set aside for [readMetadata], wherever it
* appears.
*
* Malformed or truncated input throws an [IOException].
*/
class BackupReader(
private val source: BufferedSource,
private val parser: ProtoBuf,
) {
private val metadata = Buffer()
private var entryLength = NO_ENTRY
/**
* Moves to the next entry, which [readEntry] or [skipEntry] then consumes.
*
* @return false once the input is exhausted.
*/
fun nextEntry(): Boolean {
check(entryLength == NO_ENTRY)
while (!source.exhausted()) {
val tag = source.readVarint()
val fieldNumber = tag ushr 3
val wireType = (tag and 0x7).toInt()
if (fieldNumber == 0L) throw IOException("Invalid field number 0")
if (tag == ENTRY_TAG) {
entryLength = source.readLength()
return true
}
metadata.writeVarint(tag)
when (wireType) {
WIRE_TYPE_VARINT -> metadata.writeVarint(source.readVarint())
WIRE_TYPE_FIXED64 -> metadata.write(source, 8)
WIRE_TYPE_LENGTH_DELIMITED -> {
val length = source.readLength()
metadata.writeVarint(length)
metadata.write(source, length)
}
WIRE_TYPE_FIXED32 -> metadata.write(source, 4)
else -> throw IOException("Unsupported wire type $wireType")
}
}
return false
}
fun readEntry(): BackupManga {
val bytes = source.readByteArray(takeEntryLength())
return decode(BackupManga.serializer(), bytes)
}
fun skipEntry() {
source.skip(takeEntryLength())
}
/**
* Skips the remaining entries and decodes everything else as a [Backup] without entries.
*/
fun readMetadata(): Backup {
if (entryLength != NO_ENTRY) skipEntry()
while (nextEntry()) skipEntry()
return decode(Backup.serializer(), metadata.readByteArray())
}
private fun takeEntryLength(): Long {
check(entryLength != NO_ENTRY)
return entryLength.also { entryLength = NO_ENTRY }
}
private fun <T> decode(deserializer: DeserializationStrategy<T>, bytes: ByteArray): T {
return try {
parser.decodeFromByteArray(deserializer, bytes)
} catch (e: SerializationException) {
throw IOException(e)
}
}
}
private fun BufferedSource.readVarint(): Long {
var result = 0L
for (shift in 0 until Long.SIZE_BITS step 7) {
val byte = readByte().toInt()
result = result or ((byte and 0x7F).toLong() shl shift)
if ((byte and 0x80) == 0) return result
}
throw IOException("Malformed varint")
}
private fun BufferedSource.readLength(): Long {
val length = readVarint()
if (length !in 0..Int.MAX_VALUE) throw IOException("Invalid length $length")
return length
}
private fun BufferedSink.writeVarint(value: Long) {
var remaining = value
while ((remaining and 0x7FL.inv()) != 0L) {
writeByte((remaining and 0x7F).toInt() or 0x80)
remaining = remaining ushr 7
}
writeByte(remaining.toInt())
}
private const val WIRE_TYPE_VARINT = 0
private const val WIRE_TYPE_FIXED64 = 1
private const val WIRE_TYPE_LENGTH_DELIMITED = 2
private const val WIRE_TYPE_FIXED32 = 5
// Backup.backupManga is field 1, and each entry is length-delimited
private const val ENTRY_TAG = 1L shl 3 or WIRE_TYPE_LENGTH_DELIMITED.toLong()
private const val NO_ENTRY = -1L
app/src/main/java/eu/kanade/tachiyomi/data/backup/create/BackupCreator.kt
+26 -19
View File
@@ -8,6 +8,7 @@ import dev.zacsweers.metro.AssistedFactory
import dev.zacsweers.metro.AssistedInject
import eu.kanade.tachiyomi.BuildConfig
import eu.kanade.tachiyomi.data.backup.BackupFileValidator
import eu.kanade.tachiyomi.data.backup.BackupWriter
import eu.kanade.tachiyomi.data.backup.create.creators.CategoriesBackupCreator
import eu.kanade.tachiyomi.data.backup.create.creators.ExtensionStoresBackupCreator
import eu.kanade.tachiyomi.data.backup.create.creators.MangaBackupCreator
@@ -20,6 +21,8 @@ import eu.kanade.tachiyomi.data.backup.models.BackupManga
import eu.kanade.tachiyomi.data.backup.models.BackupPreference
import eu.kanade.tachiyomi.data.backup.models.BackupSource
import eu.kanade.tachiyomi.data.backup.models.BackupSourcePreferences
import kotlinx.coroutines.flow.Flow
import kotlinx.coroutines.flow.emptyFlow
import kotlinx.serialization.protobuf.ProtoBuf
import logcat.LogPriority
import okio.buffer
@@ -85,27 +88,31 @@ class BackupCreator(
val nonFavoriteManga = if (options.readEntries) mangaRepository.getReadMangaNotInLibrary() else emptyList()
val backupManga = backupMangas(getFavorites.await() + nonFavoriteManga, options)
val backup = Backup(
backupManga = backupManga,
backupCategories = backupCategories(options),
backupSources = backupSources(backupManga),
backupPreferences = backupAppPreferences(options),
backupExtensionStores = backupExtensionStores(options),
backupSourcePreferences = backupSourcePreferences(options),
)
val byteArray = parser.encodeToByteArray(Backup.serializer(), backup)
if (byteArray.isEmpty()) {
throw IllegalStateException(context.stringResource(MR.strings.empty_backup_error))
}
file.openOutputStream()
.also {
// Force overwrite old file
(it as? FileOutputStream)?.channel?.truncate(0)
}
.sink().gzip().buffer().use {
it.write(byteArray)
val writer = BackupWriter(it, parser)
val sourceIds = mutableSetOf<Long>()
backupManga.collect { manga ->
writer.writeEntry(manga)
sourceIds += manga.source
}
writer.writeMetadata(
Backup(
backupManga = emptyList(),
backupCategories = backupCategories(options),
backupSources = backupSources(sourceIds),
backupPreferences = backupAppPreferences(options),
backupExtensionStores = backupExtensionStores(options),
backupSourcePreferences = backupSourcePreferences(options),
),
)
if (writer.isEmpty) {
throw IllegalStateException(context.stringResource(MR.strings.empty_backup_error))
}
}
val fileUri = file.uri
@@ -130,14 +137,14 @@ class BackupCreator(
return categoriesBackupCreator()
}
private suspend fun backupMangas(mangas: List<Manga>, options: BackupOptions): List<BackupManga> {
if (!options.libraryEntries) return emptyList()
private fun backupMangas(mangas: List<Manga>, options: BackupOptions): Flow<BackupManga> {
if (!options.libraryEntries) return emptyFlow()
return mangaBackupCreator(mangas, options)
}
private suspend fun backupSources(mangas: List<BackupManga>): List<BackupSource> {
return sourcesBackupCreator(mangas)
private suspend fun backupSources(sourceIds: Set<Long>): List<BackupSource> {
return sourcesBackupCreator(sourceIds)
}
private fun backupAppPreferences(options: BackupOptions): List<BackupPreference> {
app/src/main/java/eu/kanade/tachiyomi/data/backup/create/creators/MangaBackupCreator.kt
+50 -39
View File
@@ -7,11 +7,13 @@ import eu.kanade.tachiyomi.data.backup.models.BackupManga
import eu.kanade.tachiyomi.data.backup.models.toBackupChapter
import eu.kanade.tachiyomi.data.backup.models.toBackupTracking
import eu.kanade.tachiyomi.ui.reader.setting.ReadingMode
import kotlinx.coroutines.flow.Flow
import kotlinx.coroutines.flow.flow
import mihon.core.common.extensions.toByteArray
import mihon.core.common.extensions.toEpochMillisOrZero
import tachiyomi.domain.category.interactor.GetCategories
import tachiyomi.domain.category.repository.CategoryRepository
import tachiyomi.domain.chapter.repository.ChapterRepository
import tachiyomi.domain.history.interactor.GetHistory
import tachiyomi.domain.history.repository.HistoryRepository
import tachiyomi.domain.manga.model.Manga
import tachiyomi.domain.manga.repository.MangaRepository
import tachiyomi.domain.track.repository.TrackRepository
@@ -20,62 +22,71 @@ import tachiyomi.domain.track.repository.TrackRepository
class MangaBackupCreator(
private val mangaRepository: MangaRepository,
private val chapterRepository: ChapterRepository,
private val categoryRepository: CategoryRepository,
private val historyRepository: HistoryRepository,
private val trackRepository: TrackRepository,
private val getCategories: GetCategories,
private val getHistory: GetHistory,
) {
suspend operator fun invoke(mangas: List<Manga>, options: BackupOptions): List<BackupManga> {
return mangas.map {
backupManga(it, options)
/**
* Builds the entries a chunk at a time, so only one chunk's chapters are held at once.
*/
operator fun invoke(mangas: List<Manga>, options: BackupOptions): Flow<BackupManga> = flow {
mangas.chunked(CHUNK_SIZE).forEach { chunk ->
backupMangas(chunk, options).forEach { emit(it) }
}
}
private suspend fun backupManga(manga: Manga, options: BackupOptions): BackupManga {
// Entry for this manga
val mangaObject = manga.toBackupManga()
private suspend fun backupMangas(mangas: List<Manga>, options: BackupOptions): List<BackupManga> {
val mangaIds = mangas.map { it.id }
mangaObject.excludedScanlators = mangaRepository.getExcludedScanlators(manga.id).toList()
if (options.chapters) {
// Backup all the chapters
chapterRepository.getChapterByMangaId(manga.id, applyScanlatorFilter = false)
.map { it.toBackupChapter() }
.takeUnless { it.isEmpty() }
?.let { mangaObject.chapters = it }
val excludedScanlatorsByMangaId = mangaRepository.getExcludedScanlatorsByMangaIds(mangaIds)
// History is backed up by chapter URL, so it needs the chapters too
val chaptersByMangaId = if (options.chapters || options.history) {
chapterRepository.getChaptersByMangaIds(mangaIds)
} else {
emptyMap()
}
val categoriesByMangaId = if (options.categories) {
categoryRepository.getCategoriesByMangaIds(mangaIds)
} else {
emptyMap()
}
val tracksByMangaId = if (options.tracking) trackRepository.getTracksByMangaIds(mangaIds) else emptyMap()
val historyByMangaId = if (options.history) historyRepository.getHistoryByMangaIds(mangaIds) else emptyMap()
if (options.categories) {
// Backup categories for this manga
val categoriesForManga = getCategories.await(manga.id)
if (categoriesForManga.isNotEmpty()) {
mangaObject.categories = categoriesForManga.map { it.order }
return mangas.map { manga ->
val mangaObject = manga.toBackupManga()
val chapters = chaptersByMangaId[manga.id].orEmpty()
mangaObject.excludedScanlators = excludedScanlatorsByMangaId[manga.id].orEmpty()
if (options.chapters) {
mangaObject.chapters = chapters.map { it.toBackupChapter() }
}
}
if (options.tracking) {
val tracks = trackRepository.getTracksByMangaId(manga.id).map { it.toBackupTracking() }
if (tracks.isNotEmpty()) {
mangaObject.tracking = tracks
if (options.categories) {
mangaObject.categories = categoriesByMangaId[manga.id].orEmpty().map { it.order }
}
}
if (options.history) {
val historyByMangaId = getHistory.await(manga.id)
if (historyByMangaId.isNotEmpty()) {
val chapterUrlsById = chapterRepository.getChapterByMangaId(manga.id).associate { it.id to it.url }
val history = historyByMangaId.mapNotNull { history ->
if (options.tracking) {
mangaObject.tracking = tracksByMangaId[manga.id].orEmpty().map { it.toBackupTracking() }
}
if (options.history) {
val chapterUrlsById = chapters.associate { it.id to it.url }
mangaObject.history = historyByMangaId[manga.id].orEmpty().mapNotNull { history ->
// A chapter removed since its history was read takes that history with it
val url = chapterUrlsById[history.chapterId] ?: return@mapNotNull null
BackupHistory(url, history.readAt.toEpochMillisOrZero(), history.readDuration)
}
if (history.isNotEmpty()) {
mangaObject.history = history
}
}
}
return mangaObject
mangaObject
}
}
private companion object {
const val CHUNK_SIZE = 100
}
}
app/src/main/java/eu/kanade/tachiyomi/data/backup/create/creators/SourcesBackupCreator.kt
+2 -6
View File
@@ -1,7 +1,6 @@
package eu.kanade.tachiyomi.data.backup.create.creators
import dev.zacsweers.metro.Inject
import eu.kanade.tachiyomi.data.backup.models.BackupManga
import eu.kanade.tachiyomi.data.backup.models.BackupSource
import eu.kanade.tachiyomi.source.Source
import tachiyomi.domain.source.service.SourceManager
@@ -11,11 +10,8 @@ class SourcesBackupCreator(
private val sourceManager: SourceManager,
) {
suspend operator fun invoke(mangas: List<BackupManga>): List<BackupSource> {
return mangas
.map(BackupManga::source)
.distinct()
.map { sourceManager.getOrStub(it).toBackupSource() }
suspend operator fun invoke(sourceIds: Set<Long>): List<BackupSource> {
return sourceIds.map { sourceManager.getOrStub(it).toBackupSource() }
}
}
app/src/main/java/eu/kanade/tachiyomi/data/backup/restore/BackupRestorer.kt
+30 -7
View File
@@ -9,7 +9,6 @@ import eu.kanade.tachiyomi.data.backup.BackupDecoder
import eu.kanade.tachiyomi.data.backup.BackupNotifier
import eu.kanade.tachiyomi.data.backup.models.BackupCategory
import eu.kanade.tachiyomi.data.backup.models.BackupExtensionStore
import eu.kanade.tachiyomi.data.backup.models.BackupManga
import eu.kanade.tachiyomi.data.backup.models.BackupPreference
import eu.kanade.tachiyomi.data.backup.models.BackupSource
import eu.kanade.tachiyomi.data.backup.models.BackupSourcePreferences
@@ -23,6 +22,12 @@ import kotlinx.coroutines.CoroutineScope
import kotlinx.coroutines.Job
import kotlinx.coroutines.coroutineScope
import kotlinx.coroutines.ensureActive
import kotlinx.coroutines.flow.chunked
import kotlinx.coroutines.flow.emitAll
import kotlinx.coroutines.flow.filter
import kotlinx.coroutines.flow.flow
import kotlinx.coroutines.flow.map
import kotlinx.coroutines.flow.withIndex
import kotlinx.coroutines.launch
import logcat.LogPriority
import tachiyomi.core.common.i18n.stringResource
@@ -32,6 +37,7 @@ import tachiyomi.domain.source.service.SourceManager
import tachiyomi.i18n.MR
import java.io.File
import java.text.SimpleDateFormat
import java.util.BitSet
import java.util.Date
import java.util.Locale
import java.util.concurrent.CopyOnWriteArrayList
@@ -97,7 +103,7 @@ class BackupRestorer(
}
private suspend fun restoreFromFile(uri: Uri, options: RestoreOptions) {
val backup = backupDecoder.decode(uri)
val (entryCount, backup) = backupDecoder.decodeMetadata(uri)
// Store source mapping for error messages
val backupMaps = backup.backupSources
@@ -105,7 +111,7 @@ class BackupRestorer(
if (options.libraryEntries) {
restoreSourceNames(backupMaps)
restoreAmount += backup.backupManga.size
restoreAmount += entryCount
}
if (options.categories) {
restoreAmount += 1
@@ -138,7 +144,7 @@ class BackupRestorer(
}
if (options.libraryEntries) {
restoreManga(
backup.backupManga,
uri,
if (options.categories) backup.backupCategories else emptyList(),
restoreCategoriesJob,
)
@@ -175,14 +181,31 @@ class BackupRestorer(
}
private fun CoroutineScope.restoreManga(
backupMangas: List<BackupManga>,
uri: Uri,
backupCategories: List<BackupCategory>,
categoriesRestoreJob: Job?,
) = launch {
categoriesRestoreJob?.join()
mangaRestorer.sortByNew(backupMangas)
// New entries go first. Rather than hold the others back in memory, a second pass reads them again.
val isExisting = mangaRestorer.getExistingMatcher()
val existingIndices = BitSet()
val newMangas = backupDecoder.decodeEntries(uri)
.withIndex()
.filter { (index, manga) ->
val exists = isExisting(manga)
if (exists) existingIndices.set(index)
!exists
}
.map { it.value }
val backupMangas = flow {
emitAll(newMangas)
emitAll(backupDecoder.decodeEntries(uri, existingIndices::get))
}
backupMangas
.chunked(100)
.forEach { chunk ->
.collect { chunk ->
val restoredAsBatch = try {
ensureActive()
mangaRestorer.restore(chunk, backupCategories)
app/src/main/java/eu/kanade/tachiyomi/data/backup/restore/restorers/MangaRestorer.kt
+5 -3
View File
@@ -23,11 +23,13 @@ class MangaRestorer(
private val now = Clock.System.now().toLocalDateTime(timeZone)
private val currentFetchWindow = fetchInterval.getWindow(now.date, timeZone)
suspend fun sortByNew(backupMangas: List<BackupManga>): List<BackupManga> {
/**
* Returns whether an entry was already in the database at the time of this call.
*/
suspend fun getExistingMatcher(): (BackupManga) -> Boolean {
val urlsBySource = restoreRepository.getMangaUrlsBySourceId()
return backupMangas
.sortedBy { it.url in urlsBySource[it.source].orEmpty() }
return { it.url in urlsBySource[it.source].orEmpty() }
}
/**
app/src/test/java/eu/kanade/tachiyomi/data/backup/BackupStreamTest.kt
+202
View File
@@ -0,0 +1,202 @@
package eu.kanade.tachiyomi.data.backup
import eu.kanade.tachiyomi.data.backup.models.Backup
import eu.kanade.tachiyomi.data.backup.models.BackupCategory
import eu.kanade.tachiyomi.data.backup.models.BackupChapter
import eu.kanade.tachiyomi.data.backup.models.BackupExtensionStore
import eu.kanade.tachiyomi.data.backup.models.BackupHistory
import eu.kanade.tachiyomi.data.backup.models.BackupManga
import eu.kanade.tachiyomi.data.backup.models.BackupPreference
import eu.kanade.tachiyomi.data.backup.models.BackupSource
import eu.kanade.tachiyomi.data.backup.models.BackupSourcePreferences
import eu.kanade.tachiyomi.data.backup.models.BackupTracking
import eu.kanade.tachiyomi.data.backup.models.BooleanPreferenceValue
import eu.kanade.tachiyomi.data.backup.models.StringPreferenceValue
import eu.kanade.tachiyomi.data.backup.models.StringSetPreferenceValue
import io.kotest.matchers.shouldBe
import kotlinx.serialization.protobuf.ProtoBuf
import okio.Buffer
import org.junit.jupiter.api.Test
import org.junit.jupiter.api.assertThrows
import org.junit.jupiter.api.parallel.Execution
import org.junit.jupiter.api.parallel.ExecutionMode
import java.io.IOException
@Execution(ExecutionMode.CONCURRENT)
class BackupStreamTest {
private val parser = ProtoBuf
@Test
fun `writer output is the plain encoding`() {
val backup = backup()
val bytes = write(backup)
bytes shouldBe encode(backup)
encode(parser.decodeFromByteArray(Backup.serializer(), bytes)) shouldBe encode(backup)
}
@Test
fun `reader reads the plain encoding`() {
val backup = backup()
val (entries, metadata) = read(encode(backup))
entries.map(::encode) shouldBe backup.backupManga.map(::encode)
encode(metadata) shouldBe encode(backup.copy(backupManga = emptyList()))
}
@Test
fun `reader counts entries without decoding them`() {
val backup = backup()
val reader = BackupReader(Buffer().write(encode(backup)), parser)
var count = 0
while (reader.nextEntry()) {
reader.skipEntry()
count++
}
count shouldBe backup.backupManga.size
encode(reader.readMetadata()) shouldBe encode(backup.copy(backupManga = emptyList()))
}
@Test
fun `reader takes fields in any order`() {
val backup = backup()
val (first, second, third) = backup.backupManga
val input = Buffer()
.write(encode(Backup(emptyList(), backupSources = backup.backupSources)))
.write(encode(Backup(listOf(first))))
// Fields Backup doesn't know, one of each remaining wire type
.writeByte(3 shl 3 or 0).writeByte(0x96).writeByte(0x01)
.writeByte(4 shl 3 or 1).writeLongLe(42)
.write(encode(Backup(listOf(second), backupCategories = backup.backupCategories)))
.writeByte(5 shl 3 or 5).writeIntLe(42)
.write(encode(Backup(emptyList(), backupPreferences = backup.backupPreferences)))
.write(encode(Backup(listOf(third), backupExtensionStores = backup.backupExtensionStores)))
.write(encode(Backup(emptyList(), backupSourcePreferences = backup.backupSourcePreferences)))
.readByteArray()
val (entries, metadata) = read(input)
entries.map(::encode) shouldBe backup.backupManga.map(::encode)
encode(metadata) shouldBe encode(backup.copy(backupManga = emptyList()))
}
@Test
fun `empty backup`() {
val sink = Buffer()
val writer = BackupWriter(sink, parser)
writer.writeMetadata(Backup(emptyList()))
writer.isEmpty shouldBe true
sink.size shouldBe 0L
val (entries, metadata) = read(ByteArray(0))
entries shouldBe emptyList()
encode(metadata) shouldBe ByteArray(0)
}
@Test
fun `truncated input throws`() {
val bytes = encode(backup())
// Cut inside the last entry, inside the metadata, and right after a tag
val entryEnd = encode(Backup(backup().backupManga)).size
listOf(entryEnd - 1, bytes.size - 1, entryEnd + 1).forEach { size ->
assertThrows<IOException> { read(bytes.copyOf(size)) }
}
}
@Test
fun `garbage input throws`() {
listOf(
// Wire type 7 doesn't exist
byteArrayOf(0x0F),
// Field number 0
byteArrayOf(0x02, 0x00),
// A varint that never ends
ByteArray(11) { 0xFF.toByte() },
// An entry longer than the input
byteArrayOf(0x0A, 0x10, 0x01),
// An entry that isn't a BackupManga
byteArrayOf(0x0A, 0x02, 0x0F, 0x00),
).forEach { input ->
assertThrows<IOException> { read(input) }
}
}
private fun write(backup: Backup): ByteArray {
val sink = Buffer()
val writer = BackupWriter(sink, parser)
backup.backupManga.forEach(writer::writeEntry)
writer.writeMetadata(backup.copy(backupManga = emptyList()))
writer.isEmpty shouldBe false
return sink.readByteArray()
}
private fun read(bytes: ByteArray): Pair<List<BackupManga>, Backup> {
val reader = BackupReader(Buffer().write(bytes), parser)
val entries = buildList {
while (reader.nextEntry()) add(reader.readEntry())
}
return entries to reader.readMetadata()
}
private fun encode(backup: Backup) = parser.encodeToByteArray(Backup.serializer(), backup)
private fun encode(manga: BackupManga) = parser.encodeToByteArray(BackupManga.serializer(), manga)
private fun backup() = Backup(
backupManga = listOf(
BackupManga(
source = 1L,
url = "/manga/1",
title = "First",
genre = listOf("Action", "Drama"),
chapters = listOf(
BackupChapter(url = "/chapter/1", name = "Chapter 1", read = true, chapterNumber = 1f),
BackupChapter(url = "/chapter/2", name = "Chapter 2", bookmark = true, lastPageRead = 4),
),
categories = listOf(0L, 1L),
tracking = listOf(BackupTracking(syncId = 2, libraryId = 3L, title = "First", mediaId = 7L)),
history = listOf(BackupHistory(url = "/chapter/1", lastRead = 1_700_000_000_000L, readDuration = 60)),
excludedScanlators = listOf("Group"),
),
BackupManga(source = 2L, url = "/manga/2", title = "Second", favorite = false),
BackupManga(
source = 1L,
url = "/manga/3",
title = "Third",
description = "x".repeat(300),
chapters = List(200) { BackupChapter(url = "/chapter/3/$it", name = "Chapter $it") },
),
),
backupCategories = listOf(
BackupCategory(name = "Reading", order = 0),
BackupCategory(name = "Done", order = 1),
),
backupSources = listOf(BackupSource(name = "One", sourceId = 1L), BackupSource(name = "Two", sourceId = 2L)),
backupPreferences = listOf(
BackupPreference("flag", BooleanPreferenceValue(true)),
BackupPreference("tags", StringSetPreferenceValue(setOf("a", "b"))),
),
backupSourcePreferences = listOf(
BackupSourcePreferences("source_1", listOf(BackupPreference("lang", StringPreferenceValue("en")))),
),
backupExtensionStores = listOf(
BackupExtensionStore(
indexUrl = "https://example.org/index.min.json",
name = "Store",
badgeLabel = null,
signingKey = "key",
contactWebsite = "https://example.org",
contactDiscord = null,
isLegacy = false,
extensionListUrl = null,
),
),
)
}
data/src/main/java/tachiyomi/data/category/CategoryRepositoryImpl.kt
+9
View File
@@ -50,6 +50,15 @@ class CategoryRepositoryImpl(
.subscribeToList()
}
override suspend fun getCategoriesByMangaIds(mangaIds: List<Long>): Map<Long, List<Category>> {
return database.categoryQueries
.getCategoriesByMangaIds(mangaIds) { mangaId, id, name, order, flags ->
mangaId to mapCategory(id, name, order, flags)
}
.awaitAsList()
.groupBy({ it.first }, { it.second })
}
override suspend fun insert(category: NewCategory) {
database.categoryQueries.insert(name = category.name, flags = category.flags)
}
data/src/main/java/tachiyomi/data/chapter/ChapterRepositoryImpl.kt
+7
View File
@@ -113,6 +113,13 @@ class ChapterRepositoryImpl(
.awaitAsList()
}
override suspend fun getChaptersByMangaIds(mangaIds: List<Long>): Map<Long, List<Chapter>> {
return database.chapterQueries
.getChaptersByMangaIds(mangaIds, ::mapChapter)
.awaitAsList()
.groupBy { it.mangaId }
}
override suspend fun getChapterSummaryByMangaId(mangaId: Long): ChapterSummary {
return database.chapterQueries
.getChapterSummaryByMangaId(mangaId) { total, latestChapterNumber ->
data/src/main/java/tachiyomi/data/history/HistoryRepositoryImpl.kt
+9
View File
@@ -48,6 +48,15 @@ class HistoryRepositoryImpl(
.awaitAsList()
}
override suspend fun getHistoryByMangaIds(mangaIds: List<Long>): Map<Long, List<History>> {
return database.historyQueries
.getHistoryByMangaIds(mangaIds) { mangaId, id, chapterId, readAt, readDuration ->
mangaId to HistoryMapper.mapHistory(id, chapterId, readAt, readDuration)
}
.awaitAsList()
.groupBy({ it.first }, { it.second })
}
override suspend fun resetHistory(historyId: Long) {
try {
database.historyQueries.resetHistoryById(historyId)
data/src/main/java/tachiyomi/data/manga/MangaRepositoryImpl.kt
+7
View File
@@ -144,6 +144,13 @@ class MangaRepositoryImpl(
.toSet()
}
override suspend fun getExcludedScanlatorsByMangaIds(mangaIds: List<Long>): Map<Long, List<String>> {
return database.excluded_scanlatorQueries
.getExcludedScanlatorsByMangaIds(mangaIds)
.awaitAsList()
.groupBy({ it.manga_id }, { it.scanlator })
}
override fun getExcludedScanlatorsAsFlow(mangaId: Long): Flow<Set<String>> {
return database.excluded_scanlatorQueries
.getExcludedScanlatorsByMangaId(mangaId)
data/src/main/java/tachiyomi/data/track/TrackRepositoryImpl.kt
+7
View File
@@ -31,6 +31,13 @@ class TrackRepositoryImpl(
.awaitAsList()
}
override suspend fun getTracksByMangaIds(mangaIds: List<Long>): Map<Long, List<Track>> {
return database.manga_trackQueries
.getTracksByMangaIds(mangaIds, TrackMapper::mapTrack)
.awaitAsList()
.groupBy { it.mangaId }
}
override suspend fun getTracks(): List<Track> {
return database.manga_trackQueries
.getTracks(TrackMapper::mapTrack)
data/src/main/sqldelight/tachiyomi/data/category.sq
+8
View File
@@ -34,6 +34,14 @@ ON C.id = MC.category_id
WHERE MC.manga_id = :mangaId
ORDER BY C.`order`;
getCategoriesByMangaIds:
SELECT MC.manga_id, C.id, C.name, C.`order`, C.flags
FROM category C
JOIN manga_category MC
ON C.id = MC.category_id
WHERE MC.manga_id IN :mangaIds
ORDER BY MC.manga_id, C.`order`;
insert:
-- The position is taken inside the insert, so two categories created at once cannot claim the same one
INSERT INTO category(name, `order`, flags)
data/src/main/sqldelight/tachiyomi/data/chapter.sq
+10
View File
@@ -48,6 +48,16 @@ AND (
OR ES.scanlator IS NULL
);
getChaptersByMangaIds:
-- The unique index's order, which getChaptersByMangaId returns too, so backups keep their chapter order
SELECT
id, manga_id, remote_url, remote_name, remote_scanlator, remote_chapter_number,
remote_date_upload, remote_order, remote_memo, user_read, user_bookmark,
user_last_page_read, state_date_fetch
FROM chapter
WHERE manga_id IN :mangaIds
ORDER BY manga_id, remote_url;
getChapterSummaryByMangaId:
SELECT count(*) AS total, max(remote_chapter_number) AS latestChapterNumber
FROM chapter
data/src/main/sqldelight/tachiyomi/data/excluded_scanlator.sq
+6
View File
@@ -21,3 +21,9 @@ getExcludedScanlatorsByMangaId:
SELECT scanlator
FROM excluded_scanlator
WHERE manga_id = :mangaId;
getExcludedScanlatorsByMangaIds:
SELECT manga_id, scanlator
FROM excluded_scanlator
WHERE manga_id IN :mangaIds
ORDER BY manga_id, scanlator;
data/src/main/sqldelight/tachiyomi/data/history.sq
+6
View File
@@ -20,6 +20,12 @@ SELECT id, chapter_id, read_at, read_duration
FROM history
WHERE manga_id = :mangaId;
getHistoryByMangaIds:
SELECT manga_id, id, chapter_id, read_at, read_duration
FROM history
WHERE manga_id IN :mangaIds
ORDER BY manga_id, id;
resetHistoryById:
UPDATE history
SET read_at = NULL
data/src/main/sqldelight/tachiyomi/data/manga_track.sq
+8
View File
@@ -48,6 +48,14 @@ SELECT
FROM manga_track
WHERE manga_id = :mangaId;
getTracksByMangaIds:
SELECT
id, manga_id, tracker_id, remote_id, remote_url, library_id, title, last_chapter_read,
total_chapters, status, score, start_date, finish_date, private
FROM manga_track
WHERE manga_id IN :mangaIds
ORDER BY manga_id, tracker_id;
upsert:
INSERT INTO manga_track(
manga_id, tracker_id, remote_id, remote_url, library_id, title, last_chapter_read,
domain/src/main/java/tachiyomi/domain/category/repository/CategoryRepository.kt
+2
View File
@@ -16,6 +16,8 @@ interface CategoryRepository {
fun getCategoriesByMangaIdAsFlow(mangaId: Long): Flow<List<Category>>
suspend fun getCategoriesByMangaIds(mangaIds: List<Long>): Map<Long, List<Category>>
suspend fun insert(category: NewCategory)
suspend fun insertAll(categories: List<NewCategory>)
domain/src/main/java/tachiyomi/domain/chapter/repository/ChapterRepository.kt
+2
View File
@@ -20,6 +20,8 @@ interface ChapterRepository {
suspend fun getChapterByMangaId(mangaId: Long, applyScanlatorFilter: Boolean = false): List<Chapter>
suspend fun getChaptersByMangaIds(mangaIds: List<Long>): Map<Long, List<Chapter>>
suspend fun getChapterSummaryByMangaId(mangaId: Long): ChapterSummary
suspend fun getScanlatorsByMangaId(mangaId: Long): List<String>
domain/src/main/java/tachiyomi/domain/history/repository/HistoryRepository.kt
+2
View File
@@ -15,6 +15,8 @@ interface HistoryRepository {
suspend fun getHistoryByMangaId(mangaId: Long): List<History>
suspend fun getHistoryByMangaIds(mangaIds: List<Long>): Map<Long, List<History>>
suspend fun resetHistory(historyId: Long)
suspend fun resetHistoryByMangaId(mangaId: Long)
domain/src/main/java/tachiyomi/domain/manga/repository/MangaRepository.kt
+2
View File
@@ -43,6 +43,8 @@ interface MangaRepository {
suspend fun getExcludedScanlators(mangaId: Long): Set<String>
suspend fun getExcludedScanlatorsByMangaIds(mangaIds: List<Long>): Map<Long, List<String>>
fun getExcludedScanlatorsAsFlow(mangaId: Long): Flow<Set<String>>
suspend fun setExcludedScanlators(mangaId: Long, excludedScanlators: Set<String>)
domain/src/main/java/tachiyomi/domain/track/repository/TrackRepository.kt
+2
View File
@@ -9,6 +9,8 @@ interface TrackRepository {
suspend fun getTracksByMangaId(mangaId: Long): List<Track>
suspend fun getTracksByMangaIds(mangaIds: List<Long>): Map<Long, List<Track>>
suspend fun getTracks(): List<Track>
fun getTracksAsFlow(): Flow<List<Track>>