Queue many chapters without probing each one on disk

app/src/main/java/eu/kanade/tachiyomi/data/download/DownloadProvider.kt

@@ -1,5 +1,6 @@
 package eu.kanade.tachiyomi.data.download
 
+import android.content.ContentResolver
 import android.content.Context
 import com.hippo.unifile.UniFile
 import dev.zacsweers.metro.AppScope
@@ -9,6 +10,9 @@ import eu.kanade.tachiyomi.source.Source
 import eu.kanade.tachiyomi.util.lang.Hash.md5
 import eu.kanade.tachiyomi.util.storage.DiskUtil
 import kotlinx.coroutines.Dispatchers
+import kotlinx.coroutines.async
+import kotlinx.coroutines.awaitAll
+import kotlinx.coroutines.coroutineScope
 import kotlinx.coroutines.withContext
 import logcat.LogPriority
 import tachiyomi.core.common.i18n.stringResource
@@ -132,11 +136,60 @@ class DownloadProvider(
         source: Source,
     ): Pair<UniFile?, List<UniFile>> = withContext(Dispatchers.IO) {
         val mangaDir = findMangaDir(manga.title, source) ?: return@withContext null to emptyList()
-        mangaDir to chapters.mapNotNull { chapter ->
-            getValidChapterDirNames(chapter.name, chapter.scanlator, chapter.url).asSequence()
-                .mapNotNull { mangaDir.findFile(it) }
-                .firstOrNull()
+        mangaDir to findChapterDirs(chapters, mangaDir).values.toList()
+    }
+
+    /**
+     * Returns the ids of the chapters whose download directory exists.
+     *
+     * @param chapters the chapters to query.
+     * @param manga the manga of the chapters.
+     * @param source the source of the chapters.
+     */
+    suspend fun findDownloadedChapterIds(
+        chapters: List<Chapter>,
+        manga: Manga,
+        source: Source,
+    ): Set<Long> = withContext(Dispatchers.IO) {
+        val mangaDir = findMangaDir(manga.title, source) ?: return@withContext emptySet()
+        findChapterDirs(chapters, mangaDir).keys
+    }
+
+    /**
+     * Returns the download directory of each chapter that has one, keyed by chapter id.
+     *
+     * A chapter is looked for under every name it may have been downloaded with, with the chapters looked up in
+     * parallel. On a storage access framework directory each lookup is a content resolver query, so from
+     * [LIST_CHAPTER_DIRS_MIN_CHAPTERS] chapters on the directory is listed once instead and the names are matched
+     * against the listing, ignoring case as the lookups do. Plain file directories are always looked up by name, as
+     * their listing can leave out files another app created.
+     *
+     * @param chapters the chapters to query.
+     * @param mangaDir the download directory of the chapters' manga.
+     */
+    private suspend fun findChapterDirs(chapters: List<Chapter>, mangaDir: UniFile): Map<Long, UniFile> {
+        val isDocumentTree = mangaDir.uri.scheme == ContentResolver.SCHEME_CONTENT
+        if (isDocumentTree && chapters.size >= LIST_CHAPTER_DIRS_MIN_CHAPTERS) {
+            val chapterDirs = mangaDir.listFiles().orEmpty()
+                .filter { it.name != null }
+                .associateByTo(sortedMapOf(String.CASE_INSENSITIVE_ORDER)) { it.name!! }
+            return chapters
+                .mapNotNull { chapter -> chapter.findDir { chapterDirs[it] }?.let { chapter.id to it } }
+                .toMap()
         }
+        return coroutineScope {
+            chapters
+                .map { chapter ->
+                    async(chapterDirLookupDispatcher) { chapter.findDir(mangaDir::findFile)?.let { chapter.id to it } }
+                }
+                .awaitAll()
+                .filterNotNull()
+                .toMap()
+        }
+    }
+
+    private fun Chapter.findDir(find: (String) -> UniFile?): UniFile? {
+        return getValidChapterDirNames(name, scanlator, url).firstNotNullOfOrNull(find)
     }
 
     /**
@@ -257,6 +310,12 @@ class DownloadProvider(
     }
 }
 
+private const val LIST_CHAPTER_DIRS_MIN_CHAPTERS = 50
+
+private const val CHAPTER_DIR_LOOKUP_PARALLELISM = 8
+
+private val chapterDirLookupDispatcher = Dispatchers.IO.limitedParallelism(CHAPTER_DIR_LOOKUP_PARALLELISM)
+
 /**
  * The parts of a chapter's directory name, each computed once however many name variations are built from them.
  */

app/src/main/java/eu/kanade/tachiyomi/data/download/Downloader.kt

@@ -357,14 +357,16 @@ class Downloader(
         val source = sourceManager.get(manga.source) as? HttpSource ?: return false
         val wasEmpty = queueState.value.isEmpty()
         val chaptersToQueue = withContext(Dispatchers.IO) {
+            val downloadedChapterIds = provider.findDownloadedChapterIds(chapters, manga, source)
+            val queuedChapterIds = queueState.value.mapTo(HashSet()) { it.chapter.id }
             chapters
                 // Filter out those already downloaded.
-                .filter { provider.findChapterDir(it.name, it.scanlator, it.url, manga.title, source) == null }
+                .filter { it.id !in downloadedChapterIds }
                 .asSequence()
                 // Add chapters to queue from the start.
                 .sortedByDescending { it.sourceOrder }
                 // Filter out those already enqueued.
-                .filter { chapter -> queueState.value.none { it.chapter.id == chapter.id } }
+                .filter { it.id !in queuedChapterIds }
                 // Create a download for each one.
                 .map { Download(source, manga, it) }
                 .toList()
@@ -765,7 +767,8 @@ class Downloader(
     private fun addAllToQueue(downloads: List<Download>) {
         _queueState.update { queue ->
             // Callers filter off the queue's thread, so a chapter may have been queued in the meantime
-            val toAdd = downloads.filter { download -> queue.none { it.chapter.id == download.chapter.id } }
+            val queuedChapterIds = queue.mapTo(HashSet()) { it.chapter.id }
+            val toAdd = downloads.filter { it.chapter.id !in queuedChapterIds }
             toAdd.forEach { download ->
                 download.status = Download.State.QUEUE
             }

app/src/main/java/eu/kanade/tachiyomi/ui/library/LibraryViewModel.kt

@@ -513,16 +513,7 @@ class LibraryViewModel(
         appScope.launch {
             mangas.forEach { manga ->
                 val chapters = getNextChapters.await(manga.id)
-                    .fastFilterNot { chapter ->
-                        downloadManager.getQueuedDownloadOrNull(chapter.id) != null ||
-                            downloadManager.isChapterDownloaded(
-                                chapter.name,
-                                chapter.scanlator,
-                                chapter.url,
-                                manga.title,
-                                manga.source,
-                            )
-                    }
+                    .filterNotQueuedOrDownloaded(manga)
                     .let { if (amount != null) it.take(amount) else it }
 
                 downloadManager.downloadChapters(manga, chapters)
@@ -535,21 +526,18 @@ class LibraryViewModel(
         appScope.launch {
             mangas.forEach { manga ->
                 val chapters = getBookmarkedChaptersByMangaId.await(manga.id)
-                    .fastFilterNot { chapter ->
-                        downloadManager.getQueuedDownloadOrNull(chapter.id) != null ||
-                            downloadManager.isChapterDownloaded(
-                                chapter.name,
-                                chapter.scanlator,
-                                chapter.url,
-                                manga.title,
-                                manga.source,
-                            )
-                    }
+                    .filterNotQueuedOrDownloaded(manga)
                 downloadManager.downloadChapters(manga, chapters)
             }
         }
     }
 
+    private fun List<Chapter>.filterNotQueuedOrDownloaded(manga: Manga): List<Chapter> {
+        val queuedDownloads = downloadManager.getQueuedDownloadsByChapterId()
+        val downloadedChapterIds = downloadManager.getDownloadedChapterIds(this, manga)
+        return fastFilterNot { it.id in queuedDownloads || it.id in downloadedChapterIds }
+    }
+
     /**
      * Marks mangas' chapters read status.
      */