diff --git a/app/src/main/java/eu/kanade/tachiyomi/data/backup/create/creators/MangaBackupCreator.kt b/app/src/main/java/eu/kanade/tachiyomi/data/backup/create/creators/MangaBackupCreator.kt index 22706e871..26d873cd0 100644 --- a/app/src/main/java/eu/kanade/tachiyomi/data/backup/create/creators/MangaBackupCreator.kt +++ b/app/src/main/java/eu/kanade/tachiyomi/data/backup/create/creators/MangaBackupCreator.kt @@ -99,4 +99,5 @@ private fun Manga.toBackupManga() = notes = this.notes, initialized = this.initialized, memo = this.memo.toByteArray(), + nextUpdate = this.nextUpdate.toEpochMillisOrZero(), ) diff --git a/app/src/main/java/eu/kanade/tachiyomi/data/backup/models/BackupManga.kt b/app/src/main/java/eu/kanade/tachiyomi/data/backup/models/BackupManga.kt index 97132cb0a..74a27fa01 100644 --- a/app/src/main/java/eu/kanade/tachiyomi/data/backup/models/BackupManga.kt +++ b/app/src/main/java/eu/kanade/tachiyomi/data/backup/models/BackupManga.kt @@ -48,6 +48,7 @@ class BackupManga( @ProtoNumber(110) var notes: String = "", @ProtoNumber(111) var initialized: Boolean = false, @ProtoNumber(112) var memo: ByteArray = JsonObjectEmptyBytes, + @ProtoNumber(113) var nextUpdate: Long = 0, ) { fun getMangaImpl(): Manga { return Manga.create().copy( @@ -75,6 +76,7 @@ class BackupManga( notes = this@BackupManga.notes, initialized = this@BackupManga.initialized, memo = this@BackupManga.memo.toJsonObject(), + nextUpdate = this@BackupManga.nextUpdate.toInstantOrNull(), ) } } diff --git a/app/src/main/java/eu/kanade/tachiyomi/data/backup/restore/restorers/MangaRestorer.kt b/app/src/main/java/eu/kanade/tachiyomi/data/backup/restore/restorers/MangaRestorer.kt index 184b50415..e2fbc81a7 100644 --- a/app/src/main/java/eu/kanade/tachiyomi/data/backup/restore/restorers/MangaRestorer.kt +++ b/app/src/main/java/eu/kanade/tachiyomi/data/backup/restore/restorers/MangaRestorer.kt @@ -58,8 +58,13 @@ class MangaRestorer( ) } - restoreRepository.restoreManga(entries) { - fetchInterval.withFetchInterval(it, now, timeZone, currentFetchWindow) + restoreRepository.restoreManga(entries) { manga -> + fetchInterval.withFetchInterval(manga, now, timeZone, currentFetchWindow).apply { + // A next update still ahead was worked out with the chapters the backup came from + manga.nextUpdate + ?.takeIf { it >= currentFetchWindow.start } + ?.let { nextUpdate = it } + } } } } diff --git a/data/src/main/java/tachiyomi/data/backup/RestoreRepositoryImpl.kt b/data/src/main/java/tachiyomi/data/backup/RestoreRepositoryImpl.kt index cf9151748..aeb6d80a8 100644 --- a/data/src/main/java/tachiyomi/data/backup/RestoreRepositoryImpl.kt +++ b/data/src/main/java/tachiyomi/data/backup/RestoreRepositoryImpl.kt @@ -92,6 +92,7 @@ class RestoreRepositoryImpl( initialized = dbManga.initialized || manga.initialized, // Merge both backup and local data with local winning memo = JsonObject(manga.memo + dbManga.memo), + nextUpdate = dbManga.nextUpdate ?: manga.nextUpdate, ) } diff --git a/domain/src/main/java/tachiyomi/domain/manga/interactor/FetchInterval.kt b/domain/src/main/java/tachiyomi/domain/manga/interactor/FetchInterval.kt index f08866a6f..0fd9cc03b 100644 --- a/domain/src/main/java/tachiyomi/domain/manga/interactor/FetchInterval.kt +++ b/domain/src/main/java/tachiyomi/domain/manga/interactor/FetchInterval.kt @@ -1,12 +1,15 @@ package tachiyomi.domain.manga.interactor import dev.zacsweers.metro.Inject +import kotlinx.datetime.DatePeriod +import kotlinx.datetime.DayOfWeek import kotlinx.datetime.LocalDate import kotlinx.datetime.LocalDateTime import kotlinx.datetime.TimeZone import kotlinx.datetime.atStartOfDayIn import kotlinx.datetime.daysUntil -import kotlinx.datetime.toInstant +import kotlinx.datetime.minus +import kotlinx.datetime.plus import kotlinx.datetime.toLocalDateTime import tachiyomi.domain.chapter.interactor.GetChaptersByMangaId import tachiyomi.domain.chapter.model.Chapter @@ -29,12 +32,18 @@ class FetchInterval( timeZone: TimeZone, window: ClosedRange?, ): MangaUpdate { - val interval = manga.fetchInterval.takeIf { it < 0 } ?: calculateInterval( - chapters = getChaptersByMangaId.await(manga.id, applyScanlatorFilter = true), - zone = timeZone, - ) val currentWindow = window ?: getWindow(Clock.System.now().toLocalDateTime(timeZone).date, timeZone) - val nextUpdate = calculateNextUpdate(manga, interval, dateTime, timeZone, currentWindow) + val nextUpdate: Instant + val interval: Int + if (manga.fetchInterval < 0) { + interval = manga.fetchInterval + nextUpdate = calculateCustomNextUpdate(manga, interval.absoluteValue, dateTime, timeZone, currentWindow) + } else { + val chapters = getChaptersByMangaId.await(manga.id, applyScanlatorFilter = true) + val schedule = estimateSchedule(chapters, timeZone, dateTime.date) + interval = schedule.interval + nextUpdate = calculateNextUpdate(manga, schedule, dateTime.date, timeZone, currentWindow) + } return MangaUpdate(manga.id) { this.nextUpdate = nextUpdate @@ -48,51 +57,113 @@ class FetchInterval( } internal fun calculateInterval(chapters: List, zone: TimeZone): Int { + return estimateSchedule(chapters, zone, Clock.System.now().toLocalDateTime(zone).date).interval + } + + internal fun estimateSchedule(chapters: List, zone: TimeZone, today: LocalDate): ReleaseSchedule { val chapterWindow = if (chapters.size <= 8) 3 else 10 - val uploadDates = chapters.asSequence() - .mapNotNull { it.dateUpload } - .sortedDescending() - .map { - it.toLocalDateTime(zone) - .date - .atStartOfDayIn(zone) - } - .distinct() - .take(chapterWindow) - .toList() - - val fetchDates = chapters.asSequence() - .map { it.dateFetch } - .sortedDescending() - .map { - it.toLocalDateTime(zone) - .date - .atStartOfDayIn(zone) - } - .distinct() - .take(chapterWindow) - .toList() - - val interval = when { - // Enough upload date from source - uploadDates.size >= 3 -> { - val ranges = uploadDates.windowed(2).map { x -> x[1].daysUntil(x[0], zone) }.sorted() - ranges[(ranges.size - 1) / 2] - } - // Enough fetch date from client - fetchDates.size >= 3 -> { - val ranges = fetchDates.windowed(2).map { x -> x[1].daysUntil(x[0], zone) }.sorted() - ranges[(ranges.size - 1) / 2] - } - // Default to 7 days - else -> 7 + // Sources that date chapters ahead of their release would otherwise pull the schedule into the future + val latestPlausibleDay = today.plus(DatePeriod(days = 1)) + fun releaseDays(dates: Sequence): List { + return dates + .map { it.toLocalDateTime(zone).date } + .filter { it <= latestPlausibleDay } + .distinct() + .sortedDescending() + .take(chapterWindow) + .toList() } - return interval.coerceIn(1, MAX_INTERVAL) + val uploadDays = releaseDays(chapters.asSequence().mapNotNull { it.dateUpload }) + val days = uploadDays.takeIf { it.size >= 3 } + ?: releaseDays(chapters.asSequence().map { it.dateFetch }).takeIf { it.size >= 3 } + ?: return ReleaseSchedule( + interval = DEFAULT_INTERVAL, + lastRelease = uploadDays.firstOrNull(), + pattern = ReleasePattern.EveryDays(DEFAULT_INTERVAL), + ) + + val gaps = days.zipWithNext { newer, older -> older.daysUntil(newer) }.sorted() + val medianGap = gaps[(gaps.size - 1) / 2] + val interval = medianGap.coerceIn(1, MAX_INTERVAL) + + return ReleaseSchedule( + interval = interval, + lastRelease = days.first(), + pattern = detectPattern(days, medianGap) ?: ReleasePattern.EveryDays(interval), + ) + } + + /** + * Finds a calendar pattern a plain day interval can't follow: a few fixed weekdays, or a day of the + * month. Both need a few releases to be told apart from chance. + */ + private fun detectPattern(days: List, medianGap: Int): ReleasePattern? { + if (days.size < MIN_RELEASES_FOR_PATTERN) return null + + if (medianGap in MONTHLY_GAPS) { + val daysOfMonth = days.map { it.day } + return when { + daysOfMonth.max() - daysOfMonth.min() <= DAY_OF_MONTH_TOLERANCE -> { + ReleasePattern.DayOfMonth(daysOfMonth.sorted()[(daysOfMonth.size - 1) / 2]) + } + // The last day of the month moves between the 28th and the 31st + daysOfMonth.all { it >= 28 } -> ReleasePattern.DayOfMonth(31) + else -> null + } + } + + val byFrequency = days.groupingBy { it.dayOfWeek }.eachCount().entries.sortedByDescending { it.value } + val weekdays = mutableSetOf() + var covered = 0 + for ((dayOfWeek, count) in byFrequency) { + if (covered >= days.size * WEEKDAY_COVERAGE) break + weekdays += dayOfWeek + covered += count + } + // One weekday is a weekly interval, and an irregular schedule spreads over most of the week + if (weekdays.size !in 2..MAX_WEEKDAYS || covered < days.size * WEEKDAY_COVERAGE) return null + // Releasing on several weekdays means more than one release a week + if (medianGap > 7 / weekdays.size + 1) return null + return ReleasePattern.Weekdays(weekdays) } private fun calculateNextUpdate( + manga: Manga, + schedule: ReleaseSchedule, + today: LocalDate, + timeZone: TimeZone, + window: ClosedRange, + ): Instant { + val lastRelease = listOfNotNull(schedule.lastRelease, manga.lastUpdate?.toLocalDateTime(timeZone)?.date) + .maxOrNull() + + // A release on or after the expected day means the next one should be counted from it instead + val nextUpdate = manga.nextUpdate + if ( + nextUpdate != null && + nextUpdate in window.start..window.endInclusive + 1.milliseconds && + (lastRelease == null || lastRelease < nextUpdate.toLocalDateTime(timeZone).date) + ) { + return nextUpdate + } + + val anchor = schedule.lastRelease ?: lastRelease ?: today + val daysSinceRelease = anchor.daysUntil(today) + val expected = if (daysSinceRelease > schedule.interval * HIATUS_CYCLES) { + // Past a few missed releases the schedule no longer holds, so checks spread out until it resumes + val interval = backOff(schedule.interval, daysSinceRelease) + anchor.plus(DatePeriod(days = ceilDiv(daysSinceRelease, interval) * interval)) + } else { + // Today counts, so a release expected today keeps being checked for until the window passes it + generateSequence(schedule.pattern.nextAfter(anchor)) { schedule.pattern.nextAfter(it) } + .first { it >= today } + } + return expected.atStartOfDayIn(timeZone) + } + + private fun calculateCustomNextUpdate( manga: Manga, interval: Int, dateTime: LocalDateTime, @@ -103,34 +174,71 @@ class FetchInterval( return manga.nextUpdate } - val instant = manga.lastUpdate ?: Clock.System.now() - val latestDate = instant.toLocalDateTime(timeZone).date.atStartOfDayIn(timeZone) - - val daysSinceLatest = (dateTime.toInstant(timeZone) - latestDate).inWholeDays - val cycle = daysSinceLatest.floorDiv( - interval.absoluteValue.takeIf { interval < 0 } - ?: increaseInterval(interval, daysSinceLatest, increaseWhenOver = 10), - ) - - val offsetDays = ((cycle + 1) * interval.absoluteValue.toLong()).days - return latestDate.plus(offsetDays) + val latestDate = (manga.lastUpdate ?: Clock.System.now()).toLocalDateTime(timeZone).date + val cycle = latestDate.daysUntil(dateTime.date).floorDiv(interval) + return latestDate.plus(DatePeriod(days = (cycle + 1) * interval)).atStartOfDayIn(timeZone) } - private fun increaseInterval(delta: Int, daysSinceLatest: Long, increaseWhenOver: Int): Int { - if (delta >= MAX_INTERVAL) return MAX_INTERVAL - - // double delta again if missed more than 9 check in new delta - val cycle = daysSinceLatest.floorDiv(delta) + 1 - return if (cycle > increaseWhenOver) { - increaseInterval(delta * 2, daysSinceLatest, increaseWhenOver) - } else { - delta + private fun backOff(interval: Int, daysSinceRelease: Int): Int { + var backedOff = interval + while (backedOff < MAX_INTERVAL && daysSinceRelease > backedOff * HIATUS_CYCLES) { + backedOff *= 2 } + return backedOff.coerceAtMost(MAX_INTERVAL) } + private fun ceilDiv(a: Int, b: Int) = -Math.floorDiv(-a, b) + companion object { const val MAX_INTERVAL = 28 + private const val DEFAULT_INTERVAL = 7 private const val GRACE_PERIOD = 1L + private const val HIATUS_CYCLES = 4 + private const val MIN_RELEASES_FOR_PATTERN = 4 + private const val MAX_WEEKDAYS = 3 + private const val WEEKDAY_COVERAGE = 0.8 + private const val DAY_OF_MONTH_TOLERANCE = 3 + private val MONTHLY_GAPS = 26..35 + } +} + +internal data class ReleaseSchedule( + /** Days between releases, which is what the fetch interval shows. */ + val interval: Int, + /** The most recent release day the next ones are counted from, if any chapter has a date. */ + val lastRelease: LocalDate?, + val pattern: ReleasePattern, +) + +internal sealed interface ReleasePattern { + + /** The first expected release day after [date]. */ + fun nextAfter(date: LocalDate): LocalDate + + data class EveryDays(val days: Int) : ReleasePattern { + override fun nextAfter(date: LocalDate) = date.plus(DatePeriod(days = days)) + } + + data class Weekdays(val days: Set) : ReleasePattern { + override fun nextAfter(date: LocalDate): LocalDate { + return generateSequence(date.plus(DatePeriod(days = 1))) { it.plus(DatePeriod(days = 1)) } + .first { it.dayOfWeek in days } + } + } + + /** Releases on [day] of every month, or its last day in months too short for it. */ + data class DayOfMonth(val day: Int) : ReleasePattern { + override fun nextAfter(date: LocalDate): LocalDate { + val sameMonth = date.inMonthClamped(day) + if (sameMonth > date) return sameMonth + return date.plus(DatePeriod(months = 1)).inMonthClamped(day) + } + + private fun LocalDate.inMonthClamped(day: Int): LocalDate { + val firstOfMonth = LocalDate(year, month, 1) + val lastDay = firstOfMonth.plus(DatePeriod(months = 1)).minus(DatePeriod(days = 1)).day + return LocalDate(year, month, day.coerceAtMost(lastDay)) + } } } diff --git a/domain/src/test/java/tachiyomi/domain/manga/interactor/FetchIntervalTest.kt b/domain/src/test/java/tachiyomi/domain/manga/interactor/FetchIntervalTest.kt index d17b98178..285afa74b 100644 --- a/domain/src/test/java/tachiyomi/domain/manga/interactor/FetchIntervalTest.kt +++ b/domain/src/test/java/tachiyomi/domain/manga/interactor/FetchIntervalTest.kt @@ -1,18 +1,28 @@ package tachiyomi.domain.manga.interactor import io.kotest.matchers.shouldBe +import io.mockk.coEvery import io.mockk.mockk +import kotlinx.coroutines.test.runTest +import kotlinx.datetime.DatePeriod +import kotlinx.datetime.DayOfWeek +import kotlinx.datetime.LocalDate import kotlinx.datetime.LocalDateTime import kotlinx.datetime.TimeZone +import kotlinx.datetime.atStartOfDayIn +import kotlinx.datetime.atTime +import kotlinx.datetime.plus import kotlinx.datetime.toInstant +import kotlinx.datetime.toLocalDateTime import org.junit.jupiter.api.Test import org.junit.jupiter.api.parallel.Execution import org.junit.jupiter.api.parallel.ExecutionMode +import tachiyomi.domain.chapter.interactor.GetChaptersByMangaId import tachiyomi.domain.chapter.model.Chapter +import tachiyomi.domain.manga.model.Manga import kotlin.time.Duration import kotlin.time.Duration.Companion.days import kotlin.time.Duration.Companion.hours -import kotlin.time.Instant @Execution(ExecutionMode.CONCURRENT) class FetchIntervalTest { @@ -131,11 +141,112 @@ class FetchIntervalTest { fetchInterval.calculateInterval(chapters, testTimeZone) shouldBe 2 } + @Test + fun `follows releases on a few fixed weekdays`() = runTest { + val mondaysAndThursdays = generateSequence(LocalDate.parse("2019-12-02")) { it.plus(DatePeriod(days = 1)) } + .takeWhile { it <= LocalDate.parse("2020-01-20") } + .filter { it.dayOfWeek == DayOfWeek.MONDAY || it.dayOfWeek == DayOfWeek.THURSDAY } + .toList() + val chapters = chaptersOn(mondaysAndThursdays) + + fetchInterval.estimateSchedule(chapters, testTimeZone, LocalDate.parse("2020-01-21")).pattern shouldBe + ReleasePattern.Weekdays(setOf(DayOfWeek.MONDAY, DayOfWeek.THURSDAY)) + nextUpdate(today = "2020-01-21", chapters) shouldBe LocalDate.parse("2020-01-23") + } + + @Test + fun `follows releases on a day of the month`() = runTest { + val chapters = chaptersOn( + listOf("2019-09-15", "2019-10-15", "2019-11-14", "2019-12-15", "2020-01-16").map(LocalDate::parse), + perDay = 2, + ) + + nextUpdate(today = "2020-01-20", chapters) shouldBe LocalDate.parse("2020-02-15") + } + + @Test + fun `follows releases on the last day of the month`() = runTest { + val chapters = chaptersOn( + listOf("2019-09-30", "2019-10-31", "2019-11-30", "2019-12-31", "2020-01-31").map(LocalDate::parse), + perDay = 2, + ) + + nextUpdate(today = "2020-02-05", chapters) shouldBe LocalDate.parse("2020-02-29") + } + + @Test + fun `keeps checking on the expected day`() = runTest { + val chapters = chaptersOn(weekly(from = "2019-12-04", to = "2019-12-25")) + + nextUpdate(today = "2020-01-01", chapters) shouldBe LocalDate.parse("2020-01-01") + } + + @Test + fun `keeps a next update in the window while nothing new came out`() = runTest { + val chapters = chaptersOn(weekly(from = "2019-12-04", to = "2019-12-25")) + val manga = Manga.create().copy(nextUpdate = LocalDate.parse("2020-01-01").atStartOfDayIn(testTimeZone)) + + nextUpdate(today = "2020-01-02", chapters, manga) shouldBe LocalDate.parse("2020-01-01") + } + + @Test + fun `counts from a release on the expected day`() = runTest { + val chapters = chaptersOn(weekly(from = "2019-12-04", to = "2020-01-01")) + val manga = Manga.create().copy(nextUpdate = LocalDate.parse("2020-01-01").atStartOfDayIn(testTimeZone)) + + nextUpdate(today = "2020-01-01", chapters, manga) shouldBe LocalDate.parse("2020-01-08") + } + + @Test + fun `spreads out checks during a hiatus`() = runTest { + val chapters = chaptersOn(weekly(from = "2019-09-11", to = "2019-10-02")) + + nextUpdate(today = "2020-01-01", chapters) shouldBe LocalDate.parse("2020-01-22") + } + + @Test + fun `ignores chapters dated in the future`() = runTest { + val chapters = chaptersOn(weekly(from = "2019-12-04", to = "2019-12-25") + LocalDate.parse("2020-06-01")) + + nextUpdate(today = "2020-01-01", chapters) shouldBe LocalDate.parse("2020-01-01") + } + + private suspend fun nextUpdate( + today: String, + chapters: List, + manga: Manga = Manga.create(), + ): LocalDate { + val getChaptersByMangaId = mockk() + coEvery { getChaptersByMangaId.await(any(), any()) } returns chapters + val fetchInterval = FetchInterval(getChaptersByMangaId) + val date = LocalDate.parse(today) + val update = fetchInterval.withFetchInterval( + manga = manga, + dateTime = date.atTime(10, 0), + timeZone = testTimeZone, + window = fetchInterval.getWindow(date, testTimeZone), + ) + return update.nextUpdate!!.toLocalDateTime(testTimeZone).date + } + + private fun weekly(from: String, to: String): List { + return generateSequence(LocalDate.parse(from)) { it.plus(DatePeriod(days = 7)) } + .takeWhile { it <= LocalDate.parse(to) } + .toList() + } + + private fun chaptersOn(days: List, perDay: Int = 1): List { + return days.flatMap { day -> + val time = day.atStartOfDayIn(testTimeZone) + 12.hours + List(perDay) { chapter.copy(dateFetch = time, dateUpload = time) } + } + } + private fun chapterWithTime(chapter: Chapter, duration: Duration): Chapter { val newTime = testTime.toInstant(testTimeZone) + duration return chapter.copy(dateFetch = newTime, dateUpload = newTime) } private fun List.lastUploadDate() = - last().dateUpload!! - Instant.fromEpochMilliseconds(0) + last().dateUpload!! - testTime.toInstant(testTimeZone) }