diff --git a/CHANGELOG.md b/CHANGELOG.md index 67767e07bc..5d3aca4190 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -31,6 +31,7 @@ The format is a modified version of [Keep a Changelog](https://keepachangelog.co ### Changed - Use a new decoder for images Android can't decode itself, like AVIF, JPEG XL and HEIF, and support JPEG 2000 images ([@wwww-wwww](https://github.com/wwww-wwww)) ([#3786](https://github.com/mihonapp/mihon/pull/3786)) +- Keep online chapters cached for a day instead of up to a size limit ([@AntsyLich](https://github.com/AntsyLich)) ### Removed - Remove "Custom hardware bitmap threshold" and "Custom display profile" advanced settings ([@wwww-wwww](https://github.com/wwww-wwww)) ([#3786](https://github.com/mihonapp/mihon/pull/3786)) diff --git a/app/build.gradle.kts b/app/build.gradle.kts index cdf2c2c8ce..797416e44f 100644 --- a/app/build.gradle.kts +++ b/app/build.gradle.kts @@ -273,7 +273,6 @@ dependencies { implementation(libs.jsoup) // Disk - implementation(libs.diskLruCache) implementation(libs.unifile) // Preferences diff --git a/app/src/main/java/eu/kanade/presentation/more/settings/screen/SettingsDataScreen.kt b/app/src/main/java/eu/kanade/presentation/more/settings/screen/SettingsDataScreen.kt index b709e77b9d..491d21cbf0 100644 --- a/app/src/main/java/eu/kanade/presentation/more/settings/screen/SettingsDataScreen.kt +++ b/app/src/main/java/eu/kanade/presentation/more/settings/screen/SettingsDataScreen.kt @@ -57,7 +57,6 @@ import eu.kanade.tachiyomi.util.system.workManager import kotlinx.coroutines.CancellationException import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.launch -import kotlinx.coroutines.withContext import logcat.LogPriority import mihon.app.di.appGraph import mihon.icons.materialsymbols.MaterialSymbols @@ -287,7 +286,7 @@ object SettingsDataScreen : SearchableSettings { val chapterCache = remember { context.appGraph.chapterCache } var cacheReadableSizeSema by remember { mutableIntStateOf(0) } val cacheReadableSize by produceState("", cacheReadableSizeSema) { - value = withContext(Dispatchers.IO) { chapterCache.readableSize } + value = chapterCache.readableSize() } return Preference.PreferenceGroup( @@ -311,7 +310,7 @@ object SettingsDataScreen : SearchableSettings { onClick = { scope.launch { try { - val deletedFiles = withContext(Dispatchers.IO) { chapterCache.clear() } + val deletedFiles = chapterCache.clear() context.toast(context.stringResource(MR.strings.cache_deleted, deletedFiles)) cacheReadableSizeSema++ } catch (e: Exception) { diff --git a/app/src/main/java/eu/kanade/tachiyomi/data/cache/ChapterCache.kt b/app/src/main/java/eu/kanade/tachiyomi/data/cache/ChapterCache.kt deleted file mode 100644 index 9f3ec66113..0000000000 --- a/app/src/main/java/eu/kanade/tachiyomi/data/cache/ChapterCache.kt +++ /dev/null @@ -1,238 +0,0 @@ -package eu.kanade.tachiyomi.data.cache - -import android.content.Context -import android.text.format.Formatter -import com.jakewharton.disklrucache.DiskLruCache -import dev.zacsweers.metro.AppScope -import dev.zacsweers.metro.Inject -import dev.zacsweers.metro.SingleIn -import eu.kanade.tachiyomi.source.model.Page -import eu.kanade.tachiyomi.ui.reader.model.DownloadStream -import eu.kanade.tachiyomi.util.storage.DiskUtil -import eu.kanade.tachiyomi.util.storage.saveTo -import kotlinx.serialization.json.Json -import logcat.LogPriority -import okhttp3.Response -import okio.buffer -import okio.sink -import tachiyomi.core.common.util.system.logcat -import tachiyomi.domain.chapter.model.Chapter -import java.io.File -import java.io.IOException - -/** - * Class used to create chapter cache - * For each image in a chapter a file is created - * For each chapter a Json list is created and converted to a file. - * The files are in format *md5key*.0 - * - * @param context the application context. - */ -@Inject -@SingleIn(AppScope::class) -class ChapterCache( - private val context: Context, - private val json: Json, -) { - - /** Cache class used for cache management. */ - private val diskCache by lazy { - DiskLruCache.open( - File(context.cacheDir, "chapter_disk_cache"), - PARAMETER_APP_VERSION, - PARAMETER_VALUE_COUNT, - PARAMETER_CACHE_SIZE, - ) - } - - /** - * Returns directory of cache. - */ - private val cacheDir: File - get() = diskCache.directory - - /** - * Returns real size of directory. - */ - private val realSize: Long - get() = DiskUtil.getDirectorySize(cacheDir) - - /** - * Returns real size of directory in human readable format. - */ - val readableSize: String - get() = Formatter.formatFileSize(context, realSize) - - /** - * Get page list from cache. - * - * @param chapter the chapter. - * @return the list of pages. - */ - fun getPageListFromCache(chapter: Chapter): List { - // Get the key for the chapter. - val key = DiskUtil.hashKeyForDisk(getKey(chapter)) - - // Convert JSON string to list of objects. Throws an exception if snapshot is null - return diskCache.get(key).use { - json.decodeFromString(it.getString(0)) - } - } - - /** - * Add page list to disk cache. - * - * @param chapter the chapter. - * @param pages list of pages. - */ - fun putPageListToCache(chapter: Chapter, pages: List) { - // Convert list of pages to json string. - val cachedValue = json.encodeToString(pages) - - // Initialize the editor (edits the values for an entry). - var editor: DiskLruCache.Editor? = null - - try { - // Get editor from md5 key. - val key = DiskUtil.hashKeyForDisk(getKey(chapter)) - editor = diskCache.edit(key) ?: return - - // Write chapter urls to cache. - editor.newOutputStream(0).sink().buffer().use { - it.write(cachedValue.toByteArray()) - it.flush() - } - - diskCache.flush() - editor.commit() - editor.abortUnlessCommitted() - } catch (e: Exception) { - logcat(LogPriority.WARN, e) { "Failed to put page list to cache" } - // Ignore. - } finally { - editor?.abortUnlessCommitted() - } - } - - /** - * Returns true if image is in cache. - * - * @param imageUrl url of image. - * @return true if in cache otherwise false. - */ - fun isImageInCache(imageUrl: String): Boolean { - return try { - val key = DiskUtil.hashKeyForDisk(imageUrl) - val inJournal = diskCache.get(key).use { it != null } - val fileExists = getImageFile(imageUrl).exists() - if (inJournal && !fileExists) { - logcat(LogPriority.WARN) { "Image is in journal but file is missing: $imageUrl" } - } - inJournal && fileExists - } catch (_: IOException) { - false - } - } - - /** - * Get image file from url. - * - * @param imageUrl url of image. - * @return path of image. - */ - fun getImageFile(imageUrl: String): File { - // Get file from md5 key. - val imageName = DiskUtil.hashKeyForDisk(imageUrl) + ".0" - return File(diskCache.directory, imageName) - } - - /** - * Add image to cache. - * - * @param imageUrl url of image. - * @param response http response from page. - * @param live also receives the bytes as they arrive. - * @throws IOException image error. - */ - @Throws(IOException::class) - fun putImageToCache(imageUrl: String, response: Response, live: DownloadStream? = null) { - // Initialize editor (edits the values for an entry). - var editor: DiskLruCache.Editor? = null - - try { - // Get editor from md5 key. - val key = DiskUtil.hashKeyForDisk(imageUrl) - editor = diskCache.edit(key) ?: return - - // Get OutputStream and write image with Okio. - if (live == null) { - response.body.source().saveTo(editor.newOutputStream(0)) - } else { - live.reserve(response.body.contentLength()) - response.body.source().use { source -> - editor.newOutputStream(0).use { out -> - val buffer = ByteArray(1 shl 16) - while (true) { - val n = source.read(buffer) - if (n < 0) break - out.write(buffer, 0, n) - live.append(buffer, 0, n) - } - } - } - } - - diskCache.flush() - editor.commit() - } finally { - response.body.close() - editor?.abortUnlessCommitted() - } - } - - fun clear(): Int { - var deletedFiles = 0 - cacheDir.listFiles()?.forEach { - if (removeFileFromCache(it.name)) { - deletedFiles++ - } - } - return deletedFiles - } - - /** - * Remove file from cache. - * - * @param file name of file "md5.0". - * @return status of deletion for the file. - */ - private fun removeFileFromCache(file: String): Boolean { - // Make sure we don't delete the journal file (keeps track of cache) - if (file == "journal" || file.startsWith("journal.")) { - return false - } - - return try { - // Remove the extension from the file to get the key of the cache - val key = file.substringBeforeLast(".") - // Remove file from cache - diskCache.remove(key) - } catch (e: Exception) { - logcat(LogPriority.WARN, e) { "Failed to remove file from cache" } - false - } - } - - private fun getKey(chapter: Chapter): String { - return "${chapter.mangaId}${chapter.url}" - } -} - -/** Application cache version. */ -private const val PARAMETER_APP_VERSION = 1 - -/** The number of values per cache entry. Must be positive. */ -private const val PARAMETER_VALUE_COUNT = 1 - -/** The maximum number of bytes this cache should use to store. */ -private const val PARAMETER_CACHE_SIZE = 100L * 1024 * 1024 diff --git a/app/src/main/java/eu/kanade/tachiyomi/data/download/Downloader.kt b/app/src/main/java/eu/kanade/tachiyomi/data/download/Downloader.kt index ca17339b89..a6ce2c63c7 100644 --- a/app/src/main/java/eu/kanade/tachiyomi/data/download/Downloader.kt +++ b/app/src/main/java/eu/kanade/tachiyomi/data/download/Downloader.kt @@ -7,7 +7,6 @@ import dev.zacsweers.metro.Inject import dev.zacsweers.metro.SingleIn import eu.kanade.domain.chapter.model.toSChapter import eu.kanade.domain.manga.model.getComicInfo -import eu.kanade.tachiyomi.data.cache.ChapterCache import eu.kanade.tachiyomi.data.download.model.Download import eu.kanade.tachiyomi.data.library.LibraryUpdateNotifier import eu.kanade.tachiyomi.data.notification.NotificationHandler @@ -50,6 +49,7 @@ import kotlinx.coroutines.withContext import logcat.LogPriority import mihon.core.archive.ZipWriter import mihon.core.metro.AppCoroutineScope +import mihon.data.cache.ChapterCache import nl.adaptivity.xmlutil.serialization.XML import okhttp3.Response import tachiyomi.core.common.i18n.stringResource @@ -64,7 +64,6 @@ import tachiyomi.domain.manga.model.Manga import tachiyomi.domain.source.service.SourceManager import tachiyomi.domain.track.interactor.GetTracks import tachiyomi.i18n.MR -import java.io.File import java.util.Locale import java.util.concurrent.ConcurrentHashMap import kotlin.time.Duration.Companion.seconds @@ -544,10 +543,8 @@ class Downloader( // If the image is already downloaded, do nothing. Otherwise download from network val file = when { imageFile != null -> imageFile - chapterCache.isImageInCache( - page.imageUrl!!, - ) -> copyImageFromCache(chapterCache.getImageFile(page.imageUrl!!), tmpDir, filename) - else -> downloadImage(page, download.source, tmpDir, filename) + else -> copyImageFromCache(page.imageUrl!!, tmpDir, filename) + ?: downloadImage(page, download.source, tmpDir, filename) } // When the page is ready, set page path, progress (just in case) and status @@ -610,24 +607,25 @@ class Downloader( } /** - * Copies the image from cache to file in tmpDir. + * Copies the image from cache to file in tmpDir, or returns null when it isn't cached. * - * @param cacheFile the file from cache. + * @param imageUrl the url of the image. * @param tmpDir the temporary directory of the download. * @param filename the filename of the image. */ - private fun copyImageFromCache(cacheFile: File, tmpDir: UniFile, filename: String): UniFile { - // Delete temp file if it exists - tmpDir.findFile("$filename.tmp")?.delete() - val tmpFile = tmpDir.createFile("$filename.tmp")!! - cacheFile.inputStream().use { input -> - tmpFile.openOutputStream().use { output -> - input.copyTo(output) + private suspend fun copyImageFromCache(imageUrl: String, tmpDir: UniFile, filename: String): UniFile? { + val tmpFile = chapterCache.openImage(imageUrl)?.use { input -> + // Delete temp file if it exists + tmpDir.findFile("$filename.tmp")?.delete() + tmpDir.createFile("$filename.tmp")!!.also { tmpFile -> + tmpFile.openOutputStream().use { output -> + input.copyTo(output) + } } - } - val extension = ImageUtil.findImageType(cacheFile.inputStream()) ?: return tmpFile + } ?: return null + chapterCache.removeImage(imageUrl) + val extension = tmpFile.openInputStream().use { ImageUtil.findImageType(it) } ?: return tmpFile tmpFile.renameTo("$filename.${extension.extension}") - cacheFile.delete() return tmpFile } diff --git a/app/src/main/java/eu/kanade/tachiyomi/ui/main/MainActivity.kt b/app/src/main/java/eu/kanade/tachiyomi/ui/main/MainActivity.kt index 3984efa4f2..30aa931a08 100644 --- a/app/src/main/java/eu/kanade/tachiyomi/ui/main/MainActivity.kt +++ b/app/src/main/java/eu/kanade/tachiyomi/ui/main/MainActivity.kt @@ -80,7 +80,6 @@ import eu.kanade.presentation.more.settings.screen.browse.ExtensionStoresScreen import eu.kanade.presentation.more.settings.screen.data.RestoreBackupScreen import eu.kanade.presentation.util.AssistContentScreen import eu.kanade.presentation.util.DefaultNavigatorScreenTransition -import eu.kanade.tachiyomi.data.cache.ChapterCache import eu.kanade.tachiyomi.data.download.DownloadCache import eu.kanade.tachiyomi.data.notification.NotificationReceiver import eu.kanade.tachiyomi.extension.ExtensionManager @@ -99,7 +98,6 @@ import eu.kanade.tachiyomi.util.system.isNavigationBarNeedsScrim import eu.kanade.tachiyomi.util.system.updaterEnabled import eu.kanade.tachiyomi.util.view.setComposeContent import kotlinx.coroutines.CancellationException -import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.channels.awaitClose import kotlinx.coroutines.delay import kotlinx.coroutines.flow.MutableStateFlow @@ -115,6 +113,7 @@ import mihon.app.di.AppGraph import mihon.app.di.appGraph import mihon.core.metro.metroGraph import mihon.core.migration.Migrator +import mihon.data.cache.ChapterCache import mihon.domain.database.repository.DatabaseRepository import mihon.feature.support.SupportUsScreen import mihon.icons.materialsymbols.MaterialSymbols @@ -325,9 +324,7 @@ class MainActivity : BaseActivity() { setSplashScreenExitAnimation(splashScreen) if (isLaunch && libraryPreferences.autoClearChapterCache.get()) { - lifecycleScope.launch(Dispatchers.IO) { - chapterCache.clear() - } + lifecycleScope.launch { chapterCache.clear() } } } diff --git a/app/src/main/java/eu/kanade/tachiyomi/ui/reader/ReaderViewModel.kt b/app/src/main/java/eu/kanade/tachiyomi/ui/reader/ReaderViewModel.kt index 4d60e14cdb..099a2771a7 100644 --- a/app/src/main/java/eu/kanade/tachiyomi/ui/reader/ReaderViewModel.kt +++ b/app/src/main/java/eu/kanade/tachiyomi/ui/reader/ReaderViewModel.kt @@ -25,7 +25,6 @@ import eu.kanade.domain.manga.model.readingMode import eu.kanade.domain.source.interactor.GetIncognitoState import eu.kanade.domain.track.interactor.TrackChapter import eu.kanade.domain.track.service.TrackPreferences -import eu.kanade.tachiyomi.data.cache.ChapterCache import eu.kanade.tachiyomi.data.cache.CoverCache import eu.kanade.tachiyomi.data.download.DownloadCache import eu.kanade.tachiyomi.data.download.DownloadManager @@ -77,6 +76,7 @@ import kotlinx.coroutines.sync.withLock import kotlinx.coroutines.withContext import logcat.LogPriority import mihon.core.metro.AppCoroutineScope +import mihon.data.cache.ChapterCache import tachiyomi.core.common.preference.toggle import tachiyomi.core.common.util.system.logcat import tachiyomi.domain.chapter.interactor.GetChaptersByMangaId diff --git a/app/src/main/java/eu/kanade/tachiyomi/ui/reader/loader/ChapterLoader.kt b/app/src/main/java/eu/kanade/tachiyomi/ui/reader/loader/ChapterLoader.kt index b4b1080d6a..2d3aee1f51 100644 --- a/app/src/main/java/eu/kanade/tachiyomi/ui/reader/loader/ChapterLoader.kt +++ b/app/src/main/java/eu/kanade/tachiyomi/ui/reader/loader/ChapterLoader.kt @@ -2,7 +2,6 @@ package eu.kanade.tachiyomi.ui.reader.loader import android.content.Context import eu.kanade.domain.chapter.model.toSChapter -import eu.kanade.tachiyomi.data.cache.ChapterCache import eu.kanade.tachiyomi.data.download.DownloadManager import eu.kanade.tachiyomi.data.download.DownloadProvider import eu.kanade.tachiyomi.source.Source @@ -13,6 +12,7 @@ import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.withContext import mihon.core.archive.archiveReader import mihon.core.archive.epubReader +import mihon.data.cache.ChapterCache import tachiyomi.core.common.i18n.stringResource import tachiyomi.core.common.util.system.logcat import tachiyomi.domain.manga.model.Manga diff --git a/app/src/main/java/eu/kanade/tachiyomi/ui/reader/loader/HttpPageLoader.kt b/app/src/main/java/eu/kanade/tachiyomi/ui/reader/loader/HttpPageLoader.kt index be6a68fc6f..8f216abed1 100644 --- a/app/src/main/java/eu/kanade/tachiyomi/ui/reader/loader/HttpPageLoader.kt +++ b/app/src/main/java/eu/kanade/tachiyomi/ui/reader/loader/HttpPageLoader.kt @@ -1,7 +1,6 @@ package eu.kanade.tachiyomi.ui.reader.loader import eu.kanade.domain.chapter.model.toSChapter -import eu.kanade.tachiyomi.data.cache.ChapterCache import eu.kanade.tachiyomi.source.model.Page import eu.kanade.tachiyomi.source.online.HttpSource import eu.kanade.tachiyomi.ui.reader.model.ReaderChapter @@ -16,6 +15,8 @@ import kotlinx.coroutines.cancel import kotlinx.coroutines.joinAll import kotlinx.coroutines.launch import kotlinx.coroutines.withContext +import mihon.data.cache.ChapterCache +import java.io.FileNotFoundException import java.util.concurrent.ConcurrentHashMap /** @@ -41,14 +42,8 @@ internal class HttpPageLoader( * otherwise fallbacks to network. */ override suspend fun getPages(): List { - val pages = try { - chapterCache.getPageListFromCache(chapter.chapter) - } catch (e: Throwable) { - if (e is CancellationException) { - throw e - } - source.getPageList(chapter.chapter.toSChapter()) - } + val pages = chapterCache.getPageList(chapter.chapter) + ?: source.getPageList(chapter.chapter.toSChapter()) return pages.mapIndexed { index, page -> // Don't trust sources and use our own indexing ReaderPage(index, page.url, page.imageUrl).also { @@ -114,7 +109,7 @@ internal class HttpPageLoader( try { // Convert to pages without reader information val pagesToSave = pages.map { Page(it.index, it.url, it.imageUrl) } - chapterCache.putPageListToCache(chapter.chapter, pagesToSave) + chapterCache.putPageList(chapter.chapter, pagesToSave) } catch (e: Throwable) { if (e is CancellationException) { throw e @@ -230,13 +225,12 @@ internal class HttpPageLoader( } val imageUrl = page.imageUrl!! - if (force || !chapterCache.isImageInCache(imageUrl)) { + val file = chapterCache.getImageFile(imageUrl).takeUnless { force } ?: run { val live = page.downloadStream page.status = Page.State.DownloadImage try { - val imageResponse = source.getImage(page) - chapterCache.putImageToCache(imageUrl, imageResponse, live) - live?.finish() + chapterCache.putImage(imageUrl, source.getImage(page), live) + .also { live?.finish() } } catch (e: Throwable) { live?.finish(e) throw e @@ -244,7 +238,16 @@ internal class HttpPageLoader( } page.downloadStream = null - page.stream = { chapterCache.getImageFile(imageUrl).inputStream() } + page.stream = { + try { + file.inputStream() + } catch (e: FileNotFoundException) { + // Gone from the cache since, so it's fetched again and shown once it's back + page.status = Page.State.Queue + launchLoadOnce(page) + throw e + } + } page.status = Page.State.Ready } catch (e: Throwable) { page.downloadStream = null diff --git a/app/src/main/java/mihon/app/di/AppGraph.kt b/app/src/main/java/mihon/app/di/AppGraph.kt index c1da46d481..91d5e975ff 100644 --- a/app/src/main/java/mihon/app/di/AppGraph.kt +++ b/app/src/main/java/mihon/app/di/AppGraph.kt @@ -18,7 +18,6 @@ import eu.kanade.tachiyomi.core.security.PrivacyPreferences import eu.kanade.tachiyomi.core.security.SecurityPreferences import eu.kanade.tachiyomi.data.backup.create.BackupCreateWorker import eu.kanade.tachiyomi.data.backup.restore.BackupRestoreWorker -import eu.kanade.tachiyomi.data.cache.ChapterCache import eu.kanade.tachiyomi.data.cache.CoverCache import eu.kanade.tachiyomi.data.download.DownloadCache import eu.kanade.tachiyomi.data.download.DownloadManager @@ -43,6 +42,7 @@ import eu.kanade.tachiyomi.util.CrashLogUtil import kotlinx.serialization.json.Json import kotlinx.serialization.protobuf.ProtoBuf import mihon.core.metro.IsDebugBuild +import mihon.data.cache.ChapterCache import mihon.domain.extension.interactor.GetExtensionStoreCountAsFlow import mihon.domain.extension.repository.ExtensionStoreRepository import nl.adaptivity.xmlutil.serialization.XML diff --git a/app/src/main/java/mihon/data/cache/ChapterCache.kt b/app/src/main/java/mihon/data/cache/ChapterCache.kt new file mode 100644 index 0000000000..9edba14a2c --- /dev/null +++ b/app/src/main/java/mihon/data/cache/ChapterCache.kt @@ -0,0 +1,127 @@ +package mihon.data.cache + +import android.content.Context +import android.text.format.Formatter +import dev.zacsweers.metro.AppScope +import dev.zacsweers.metro.Inject +import dev.zacsweers.metro.SingleIn +import eu.kanade.tachiyomi.source.model.Page +import eu.kanade.tachiyomi.ui.reader.model.DownloadStream +import kotlinx.coroutines.CancellationException +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.launch +import kotlinx.coroutines.withContext +import kotlinx.serialization.json.Json +import logcat.LogPriority +import mihon.core.common.cache.TtlDiskCache +import mihon.core.metro.AppCoroutineScope +import okhttp3.Response +import tachiyomi.core.common.util.system.logcat +import tachiyomi.domain.chapter.model.Chapter +import java.io.File +import java.io.IOException +import java.io.InputStream +import kotlin.time.Duration.Companion.days +import kotlin.time.Duration.Companion.hours + +/** + * The page lists and images of chapters read online. + * + * Page lists expire soon, as sources sign or rotate their image URLs. Images last longer. + */ +@Inject +@SingleIn(AppScope::class) +class ChapterCache( + private val context: Context, + private val json: Json, + @AppCoroutineScope scope: CoroutineScope, +) { + + private val root = File(context.cacheDir, "chapter_cache") + + // Out of the cache directory, so the system clearing that leaves the indexes + private val indexes = File(context.noBackupFilesDir, "chapter_cache") + + private val pageLists = TtlDiskCache(File(root, "pages"), File(indexes, "pages.index"), 1.hours, scope) + + private val images = TtlDiskCache(File(root, "images"), File(indexes, "images.index"), 1.days, scope) + + init { + scope.launch { + pageLists.warmUp() + images.warmUp() + } + } + + suspend fun readableSize(): String = Formatter.formatFileSize(context, pageLists.totalSize() + images.totalSize()) + + suspend fun getPageList(chapter: Chapter): List? { + val input = pageLists.open(chapter.key) ?: return null + return try { + json.decodeFromString(withContext(Dispatchers.IO) { input.bufferedReader().use { it.readText() } }) + } catch (e: Exception) { + if (e is CancellationException) throw e + logcat(LogPriority.WARN, e) { "Failed to read page list from cache" } + pageLists.remove(chapter.key) + null + } + } + + suspend fun putPageList(chapter: Chapter, pages: List) { + try { + val value = json.encodeToString(pages).toByteArray() + pageLists.put(chapter.key) { it.write(value) } + } catch (e: Exception) { + if (e is CancellationException) throw e + logcat(LogPriority.WARN, e) { "Failed to put page list to cache" } + } + } + + suspend fun isImageInCache(imageUrl: String): Boolean = imageUrl in images + + /** + * The image's file, or null when it isn't cached. + */ + suspend fun getImageFile(imageUrl: String): File? = images.get(imageUrl) + + /** + * Opens the image, or returns null when it isn't cached. + */ + suspend fun openImage(imageUrl: String): InputStream? = images.open(imageUrl) + + suspend fun removeImage(imageUrl: String) { + images.remove(imageUrl) + } + + /** + * Stores the image [response] holds, and closes it. + * + * @param live also receives the bytes as they arrive. + */ + @Throws(IOException::class) + suspend fun putImage(imageUrl: String, response: Response, live: DownloadStream? = null): File { + response.body.use { body -> + live?.reserve(body.contentLength()) + return images.put(imageUrl) { out -> + body.source().use { source -> + val buffer = ByteArray(1 shl 16) + while (true) { + val n = source.read(buffer) + if (n < 0) break + out.write(buffer, 0, n) + live?.append(buffer, 0, n) + } + } + } + } + } + + /** + * Removes everything, returning how many entries there were. + */ + suspend fun clear(): Int = pageLists.clear() + images.clear() + + private val Chapter.key: String + get() = "$mangaId$url" +} diff --git a/core/common/build.gradle.kts b/core/common/build.gradle.kts index 23beaaae77..9bef38c4ee 100644 --- a/core/common/build.gradle.kts +++ b/core/common/build.gradle.kts @@ -56,6 +56,7 @@ dependencies { implementation(libs.quickJs) testImplementation(libs.bundles.test) + testImplementation(libs.kotlinx.coroutines.test) testRuntimeOnly(libs.junit.platform.launcher) implementation(libs.metro.runtime) diff --git a/core/common/src/main/kotlin/mihon/core/common/cache/TtlDiskCache.kt b/core/common/src/main/kotlin/mihon/core/common/cache/TtlDiskCache.kt new file mode 100644 index 0000000000..032015163b --- /dev/null +++ b/core/common/src/main/kotlin/mihon/core/common/cache/TtlDiskCache.kt @@ -0,0 +1,315 @@ +package mihon.core.common.cache + +import kotlinx.coroutines.CoroutineDispatcher +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.channels.Channel +import kotlinx.coroutines.delay +import kotlinx.coroutines.launch +import kotlinx.coroutines.sync.Mutex +import kotlinx.coroutines.sync.withLock +import kotlinx.coroutines.withContext +import okio.ByteString.Companion.encodeUtf8 +import java.io.File +import java.io.FileNotFoundException +import java.io.IOException +import java.io.InputStream +import java.io.OutputStream +import java.nio.file.Files +import java.nio.file.StandardCopyOption +import java.util.UUID +import kotlin.time.Duration +import kotlin.time.Duration.Companion.seconds + +/** + * A disk cache whose entries expire [ttl] after they're written, which is all that bounds its size. + * + * An entry is one file named after its key's hash. It's written to a temporary file and moved into place, so a + * crash or a failed write leaves either the old entry or none, never a partial one. + * + * The [indexFile] records when each entry was written. It's kept apart from the entries, so that clearing them, as + * the system does with cache directories, leaves it in place, and it's saved in [scope] [saveDelay] after a change + * so a burst of writes saves it once. Loading reconciles it with the files there are, so an index that's lost or out + * of date only costs the write times it missed, which then come from the files' modification times. + */ +class TtlDiskCache( + val directory: File, + private val indexFile: File, + private val ttl: Duration, + scope: CoroutineScope, + private val saveDelay: Duration = 2.seconds, + private val dispatcher: CoroutineDispatcher = Dispatchers.IO, + private val now: () -> Long = System::currentTimeMillis, +) { + + private class Entry(val file: File, val size: Long, val writtenAt: Long) + + private val mutex = Mutex() + + // In the order they were written, so the expired ones come first + private val entries = LinkedHashMap() + + private var loaded = false + + private var size = 0L + + private var latestWrite = 0L + + private val saveRequests = Channel(Channel.CONFLATED) + + init { + scope.launch(dispatcher) { + for (request in saveRequests) { + delay(saveDelay) + mutex.withLock { save() } + } + } + } + + /** + * Loads the entries now, rather than on first use. + */ + suspend fun warmUp() = locked {} + + /** + * Bytes the entries take. + */ + suspend fun totalSize(): Long = locked { + // The system deletes cache files when storage runs low + entries.values.filterNot { it.file.exists() }.forEach { remove(it) } + size + } + + /** + * The file holding [key]'s value, or null when there's none, it has expired or it was deleted. + */ + suspend fun get(key: String): File? = locked { + val entry = entries[fileName(key)] ?: return@locked null + if (entry.isExpired() || !entry.file.exists()) { + remove(entry) + return@locked null + } + entry.file + } + + /** + * Opens [key]'s value, or returns null when there's none, it has expired or it was deleted. + */ + suspend fun open(key: String): InputStream? { + val file = get(key) ?: return null + return withContext(dispatcher) { + try { + file.inputStream() + } catch (_: FileNotFoundException) { + // Deleted since it was looked up + mutex.withLock { entries[file.name]?.takeUnless { it.file.exists() }?.let { remove(it) } } + null + } + } + } + + suspend operator fun contains(key: String): Boolean = get(key) != null + + /** + * Stores what [write] writes as [key]'s value, replacing any there is. Nothing is stored when [write] throws. + */ + @Throws(IOException::class) + suspend fun put(key: String, write: (OutputStream) -> Unit): File = withContext(dispatcher) { + // Before the temporary file exists, as loading deletes those left over + mutex.withLock { load() } + val name = fileName(key) + val temp = tempFile(directory, name) + try { + val out = try { + directory.mkdirs() + temp.outputStream() + } catch (_: FileNotFoundException) { + // The system deleted the directory in between + directory.mkdirs() + temp.outputStream() + } + out.use(write) + val file = File(directory, name) + mutex.withLock { + move(temp, file) + entries.remove(name)?.let { size -= it.size } + val entry = Entry(file, file.length(), now()) + entries[name] = entry + size += entry.size + latestWrite = maxOf(latestWrite, entry.writtenAt) + removeExpired() + changed() + } + file + } finally { + temp.delete() + } + } + + suspend fun remove(key: String): Boolean = locked { + val entry = entries[fileName(key)] ?: return@locked false + remove(entry) + true + } + + /** + * Removes every entry, returning how many there were. + */ + suspend fun clear(): Int = locked { + val count = entries.size + entries.values.forEach { it.file.delete() } + entries.clear() + size = 0 + save() + count + } + + /** + * Runs [block] on the [dispatcher] holding the lock, once the entries are loaded. + */ + private suspend inline fun locked(crossinline block: () -> T): T = withContext(dispatcher) { + mutex.withLock { + load() + settleClock() + block() + } + } + + private fun load() { + if (loaded) return + loaded = true + + // Left by a save that never finished + indexFile.parentFile + ?.listFiles { file -> file.name.startsWith("${indexFile.name}.$TEMP_INFIX") } + ?.forEach { it.delete() } + + val indexed = readIndex() + var unindexed = false + directory.listFiles() + .orEmpty() + .mapNotNull { file -> + when { + // Left by a write that never finished + TEMP_INFIX in file.name -> null.also { file.delete() } + // Written after the index was last saved + else -> indexed[file.name] + ?: Entry(file, file.length(), file.lastModified()).also { unindexed = true } + } + } + .sortedBy { it.writtenAt } + .forEach { + entries[it.file.name] = it + size += it.size + latestWrite = maxOf(latestWrite, it.writtenAt) + } + + if (unindexed || entries.size != indexed.size) changed() + removeExpired() + } + + /** + * Treats entries written after the current time, which the clock going back leaves, as written now. They then + * expire [ttl] from now rather than also after the skew, and stay in write order for [removeExpired]. + */ + private fun settleClock() { + val time = now() + if (time >= latestWrite) return + for (item in entries.entries) { + if (item.value.writtenAt > time) item.setValue(Entry(item.value.file, item.value.size, time)) + } + latestWrite = time + changed() + } + + /** + * The entries the index holds. Lines it can't read are skipped. + */ + private fun readIndex(): Map { + val indexed = HashMap() + try { + indexFile.useLines { lines -> + val iterator = lines.iterator() + if (!iterator.hasNext() || iterator.next() != HEADER) return indexed + iterator.forEach { line -> + val parts = line.split(' ') + if (parts.size != 3) return@forEach + val writtenAt = parts[1].toLongOrNull() ?: return@forEach + val size = parts[2].toLongOrNull() ?: return@forEach + indexed[parts[0]] = Entry(File(directory, parts[0]), size, writtenAt) + } + } + } catch (_: IOException) { + // No index yet, or one that can't be read; the files' modification times stand in + } + return indexed + } + + private fun removeExpired() { + settleClock() + val iterator = entries.values.iterator() + var removed = false + while (iterator.hasNext()) { + val entry = iterator.next() + if (!entry.isExpired()) break + iterator.remove() + entry.file.delete() + size -= entry.size + removed = true + } + if (removed) changed() + } + + private fun remove(entry: Entry) { + entries.remove(entry.file.name) + entry.file.delete() + size -= entry.size + changed() + } + + private fun changed() { + if (saveDelay <= Duration.ZERO) { + save() + } else { + saveRequests.trySend(Unit) + } + } + + /** + * Writes the index as the entries there are now. + */ + private fun save() { + val parent = indexFile.parentFile ?: return + val temp = tempFile(parent, indexFile.name) + try { + parent.mkdirs() + temp.bufferedWriter().use { writer -> + writer.write(HEADER) + writer.write("\n") + entries.values.forEach { + writer.write("${it.file.name} ${it.writtenAt} ${it.size}\n") + } + } + move(temp, indexFile) + } catch (_: IOException) { + // The old index stays, and the next load reconciles it with the files + } finally { + temp.delete() + } + } + + private fun move(from: File, to: File) { + Files.move(from.toPath(), to.toPath(), StandardCopyOption.ATOMIC_MOVE, StandardCopyOption.REPLACE_EXISTING) + } + + private fun tempFile(parent: File, name: String) = File(parent, "$name.$TEMP_INFIX${UUID.randomUUID()}") + + private fun Entry.isExpired() = now() - writtenAt >= ttl.inWholeMilliseconds + + private fun fileName(key: String) = key.encodeUtf8().sha256().hex() + + private companion object { + const val HEADER = "mihon.TtlDiskCache 1" + const val TEMP_INFIX = "tmp-" + } +} diff --git a/core/common/src/test/kotlin/mihon/core/common/cache/TtlDiskCacheTest.kt b/core/common/src/test/kotlin/mihon/core/common/cache/TtlDiskCacheTest.kt new file mode 100644 index 0000000000..fec10930cc --- /dev/null +++ b/core/common/src/test/kotlin/mihon/core/common/cache/TtlDiskCacheTest.kt @@ -0,0 +1,341 @@ +package mihon.core.common.cache + +import io.kotest.matchers.shouldBe +import io.kotest.matchers.shouldNotBe +import kotlinx.coroutines.CoroutineDispatcher +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.joinAll +import kotlinx.coroutines.launch +import kotlinx.coroutines.test.StandardTestDispatcher +import kotlinx.coroutines.test.TestScope +import kotlinx.coroutines.test.advanceTimeBy +import kotlinx.coroutines.test.runCurrent +import kotlinx.coroutines.test.runTest +import org.junit.jupiter.api.Test +import org.junit.jupiter.api.io.TempDir +import java.io.File +import java.io.IOException +import kotlin.time.Duration +import kotlin.time.Duration.Companion.hours +import kotlin.time.Duration.Companion.milliseconds +import kotlin.time.Duration.Companion.seconds + +class TtlDiskCacheTest { + + @TempDir + lateinit var root: File + + private val directory get() = File(root, "cache") + + private val index get() = File(root, "index/cache.index") + + private var time = 1_000_000_000L + + private fun TestScope.cache( + ttl: Duration = 1.hours, + saveDelay: Duration = Duration.ZERO, + dispatcher: CoroutineDispatcher = StandardTestDispatcher(testScheduler), + ) = TtlDiskCache(directory, index, ttl, backgroundScope, saveDelay, dispatcher, now = { time }) + + private suspend fun TtlDiskCache.put(key: String, value: String) = put(key) { it.write(value.toByteArray()) } + + private suspend fun TtlDiskCache.read(key: String) = get(key)?.readText() + + private fun entryFiles() = directory.listFiles().orEmpty().toList() + + @Test + fun `returns what was put`() = runTest { + val cache = cache() + cache.put("a", "one") + + cache.read("a") shouldBe "one" + cache.open("a")?.use { it.readBytes().decodeToString() } shouldBe "one" + cache.contains("a") shouldBe true + cache.contains("b") shouldBe false + cache.open("b") shouldBe null + } + + @Test + fun `replaces a key's value`() = runTest { + val cache = cache() + cache.put("a", "one") + cache.put("a", "two!") + + cache.read("a") shouldBe "two!" + cache.totalSize() shouldBe 4 + } + + @Test + fun `expires entries ttl after they were written`() = runTest { + val cache = cache(ttl = 10.seconds) + cache.put("a", "one") + + time += 9_000 + cache.read("a") shouldBe "one" + + time += 1_000 + cache.read("a") shouldBe null + cache.totalSize() shouldBe 0 + entryFiles().size shouldBe 0 + } + + @Test + fun `removes expired entries as others are written`() = runTest { + val cache = cache(ttl = 10.seconds) + cache.put("a", "aaaa") + time += 5_000 + cache.put("b", "bbbb") + time += 5_000 + cache.put("c", "cccc") + + cache.totalSize() shouldBe 8 + entryFiles().size shouldBe 2 + cache.read("b") shouldBe "bbbb" + } + + @Test + fun `counts a replaced entry's age from its last write`() = runTest { + val cache = cache(ttl = 10.seconds) + cache.put("a", "one") + time += 5_000 + cache.put("a", "two") + time += 5_000 + + cache.read("a") shouldBe "two" + } + + @Test + fun `keeps entries and their age across instances`() = runTest { + cache(ttl = 10.seconds).put("a", "one") + time += 5_000 + + val reopened = cache(ttl = 10.seconds) + reopened.read("a") shouldBe "one" + + time += 5_000 + reopened.read("a") shouldBe null + } + + @Test + fun `takes write times from the index rather than the files`() = runTest { + val file = cache(ttl = 10.seconds).put("a", "one") + // As a backup restore or a file system that keeps no times might leave it + file.setLastModified(0) + + cache(ttl = 10.seconds).read("a") shouldBe "one" + } + + @Test + fun `falls back to the files' times without an index`() = runTest { + val file = cache(ttl = 10.seconds).put("a", "one") + index.delete() + + file.setLastModified(time) + cache(ttl = 10.seconds).read("a") shouldBe "one" + + index.delete() + file.setLastModified(time - 10_000) + cache(ttl = 10.seconds).read("a") shouldBe null + } + + @Test + fun `skips index lines it can't read`() = runTest { + val cache = cache() + cache.put("a", "one") + cache.put("b", "two") + index.appendText("cut-sho") + + val reopened = cache() + reopened.read("a") shouldBe "one" + reopened.read("b") shouldBe "two" + reopened.totalSize() shouldBe 6 + } + + @Test + fun `forgets removed entries`() = runTest { + val cache = cache() + cache.put("a", "one") + cache.put("b", "two") + cache.remove("a") + + val reopened = cache() + reopened.read("a") shouldBe null + reopened.read("b") shouldBe "two" + } + + @Test + fun `saves a burst of changes once, after the delay`() = runTest { + val cache = cache(saveDelay = 200.milliseconds) + cache.put("a", "one") + cache.put("b", "two") + cache.put("c", "three") + + index.exists() shouldBe false + advanceTimeBy(199.milliseconds) + index.exists() shouldBe false + advanceTimeBy(1.milliseconds) + runCurrent() + index.readLines().size shouldBe 4 + } + + @Test + fun `handles files deleted behind its back`() = runTest { + val cache = cache() + val a = cache.put("a", "one") + cache.put("b", "two") + a.delete() + + cache.get("a") shouldBe null + cache.open("a") shouldBe null + cache.totalSize() shouldBe 3 + + directory.deleteRecursively() + cache.open("b") shouldBe null + cache.totalSize() shouldBe 0 + + cache.put("c", "three") + cache.read("c") shouldBe "three" + cache().read("c") shouldBe "three" + } + + @Test + fun `stores nothing when the write fails`() = runTest { + val cache = cache() + cache.put("a", "one") + + try { + cache.put("a") { + it.write("partial".toByteArray()) + throw IOException() + } + } catch (_: IOException) { + } + + cache.read("a") shouldBe "one" + entryFiles().size shouldBe 1 + } + + @Test + fun `deletes writes left unfinished`() = runTest { + directory.mkdirs() + File(directory, "abc.tmp-1").writeText("partial") + index.parentFile.mkdirs() + val unfinished = File(index.parentFile, "cache.index.tmp-1").apply { writeText("partial") } + + cache().totalSize() shouldBe 0 + entryFiles().size shouldBe 0 + unfinished.exists() shouldBe false + } + + @Test + fun `keeps the index when the entries are cleared from outside`() = runTest { + val cache = cache(ttl = 10.seconds) + cache.put("a", "one") + time += 5_000 + cache.put("b", "two") + directory.deleteRecursively() + + val reopened = cache(ttl = 10.seconds) + reopened.totalSize() shouldBe 0 + index.exists() shouldBe true + + reopened.put("c", "three") + time += 5_000 + cache(ttl = 10.seconds).read("c") shouldBe "three" + // The header and c, a and b dropped as their files were gone + index.readLines().size shouldBe 2 + } + + @Test + fun `removes and clears`() = runTest { + val cache = cache() + cache.put("a", "one") + cache.put("b", "two") + cache.put("c", "three") + + cache.remove("a") shouldBe true + cache.remove("a") shouldBe false + cache.clear() shouldBe 2 + cache.get("b") shouldBe null + cache.totalSize() shouldBe 0 + cache().totalSize() shouldBe 0 + } + + @Test + fun `keeps every entry written from several threads`() = runTest { + val cache = cache(dispatcher = Dispatchers.IO) + (0..<8).map { thread -> + launch(Dispatchers.Default) { + repeat(50) { cache.put("$thread-$it", "$thread-$it") } + } + }.joinAll() + + (0..<8).forEach { thread -> + repeat(50) { cache.read("$thread-$it") shouldBe "$thread-$it" } + } + entryFiles().size shouldBe 400 + cache.totalSize() shouldBe entryFiles().sumOf { it.length() } + cache(dispatcher = Dispatchers.IO).totalSize() shouldBe cache.totalSize() + } + + @Test + fun `names files after a hash of the key`() = runTest { + val cache = cache() + val file = cache.put("https://example.org/a/b.jpg", "x") + + file.name.length shouldBe 64 + file.parentFile shouldBe directory + cache.get("https://example.org/a/b.jpg") shouldNotBe null + } + + @Test + fun `warming up loads the entries before first use`() = runTest { + cache(ttl = 10.seconds).put("a", "one") + directory.mkdirs() + File(directory, "abc.tmp-1").writeText("partial") + time += 10_000 + + cache(ttl = 10.seconds).warmUp() + entryFiles().size shouldBe 0 + } + + @Test + fun `expires entries written before the clock went back ttl after it did`() = runTest { + val cache = cache(ttl = 10.seconds) + cache.put("a", "aaaa") + time -= 5_000 + cache.put("b", "bbbb") + + time += 9_000 + cache.read("a") shouldBe "aaaa" + time += 1_000 + cache.put("c", "cccc") + + cache.read("a") shouldBe null + cache.read("b") shouldBe null + entryFiles().size shouldBe 1 + cache.totalSize() shouldBe 4 + } + + @Test + fun `doesn't rewrite an index that matches the files`() = runTest { + cache().put("a", "one") + index.setLastModified(1_000) + + cache().warmUp() + index.lastModified() shouldBe 1_000 + } + + @Test + fun `rewrites an index that misses files`() = runTest { + cache().put("a", "one") + File(directory, "abc").writeText("two") + index.setLastModified(1_000) + + val reopened = cache() + reopened.warmUp() + index.lastModified() shouldNotBe 1_000 + index.readLines().size shouldBe 3 + reopened.totalSize() shouldBe 6 + } +} diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index 895c1f7995..bfcfa845a8 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -37,7 +37,6 @@ composeRichEditor = "1.2.1" composeWebview = "0.33.6" conscrypt = "2.8.0" directionalViewPager = "1.0.0" -diskLruCache = "2.0.2" firebase-bom = "35.1.0" firebase-crashlytics = "3.0.8" google-services = "4.5.0" @@ -136,7 +135,6 @@ composeRichEditor = { module = "com.mohamedrejeb.richeditor:richeditor-compose", composeWebview = { module = "io.github.kevinnzou:compose-webview", version.ref = "composeWebview" } conscrypt = { module = "org.conscrypt:conscrypt-android", version.ref = "conscrypt" } directionalViewPager = { module = "com.github.tachiyomiorg:DirectionalViewPager", version.ref = "directionalViewPager" } -diskLruCache = { module = "com.jakewharton:disklrucache", version.ref = "diskLruCache" } firebase-analytics = { module = "com.google.firebase:firebase-analytics" } firebase-bom = { module = "com.google.firebase:firebase-bom", version.ref = "firebase-bom" } firebase-crashlytics = { module = "com.google.firebase:firebase-crashlytics" }