diff --git a/app/src/main/java/eu/kanade/domain/chapter/interactor/SyncChaptersWithSource.kt b/app/src/main/java/eu/kanade/domain/chapter/interactor/SyncChaptersWithSource.kt index eb073ffa6..8529a7928 100644 --- a/app/src/main/java/eu/kanade/domain/chapter/interactor/SyncChaptersWithSource.kt +++ b/app/src/main/java/eu/kanade/domain/chapter/interactor/SyncChaptersWithSource.kt @@ -226,11 +226,18 @@ class SyncChaptersWithSource( updated = updatedChapters, ) + val excludedScanlators = getExcludedScanlators.await(manga.id).toHashSet() + + // A first fetch lists the back catalogue, and chapters coming back under a deleted number were out already + val foundRelease = dbChapters.isNotEmpty() && + toAdd.any { chapter -> + chapter.scanlator !in excludedScanlators && + !(chapter.isRecognizedNumber && chapter.chapterNumber in deletedChapterNumbers) + } + // Set this manga as updated since chapters were changed // Note that last_update actually represents last time the chapter list changed at all - updateManga.awaitUpdateFetchIntervalAndLastUpdate(manga, timeZone, now, fetchWindow) - - val excludedScanlators = getExcludedScanlators.await(manga.id).toHashSet() + updateManga.awaitUpdateFetchIntervalAndLastUpdate(manga, timeZone, now, fetchWindow, foundRelease) return added.filterNot { it.url in changedOrDuplicateReadUrls || it.scanlator in excludedScanlators } } diff --git a/app/src/main/java/eu/kanade/domain/manga/interactor/UpdateManga.kt b/app/src/main/java/eu/kanade/domain/manga/interactor/UpdateManga.kt index 7f9c2e4ab..eeed3eba9 100644 --- a/app/src/main/java/eu/kanade/domain/manga/interactor/UpdateManga.kt +++ b/app/src/main/java/eu/kanade/domain/manga/interactor/UpdateManga.kt @@ -44,9 +44,10 @@ class UpdateManga( timeZone: TimeZone, dateTime: LocalDateTime, window: ClosedRange?, + foundRelease: Boolean, ): Boolean { return mangaRepository.update( - fetchInterval.withFetchInterval(manga, dateTime, timeZone, window).apply { + fetchInterval.withFetchInterval(manga, dateTime, timeZone, window, foundRelease).apply { lastUpdate = Clock.System.now() }, ) diff --git a/domain/src/main/java/tachiyomi/domain/manga/interactor/FetchInterval.kt b/domain/src/main/java/tachiyomi/domain/manga/interactor/FetchInterval.kt index 0fd9cc03b..1316012a6 100644 --- a/domain/src/main/java/tachiyomi/domain/manga/interactor/FetchInterval.kt +++ b/domain/src/main/java/tachiyomi/domain/manga/interactor/FetchInterval.kt @@ -1,21 +1,14 @@ package tachiyomi.domain.manga.interactor import dev.zacsweers.metro.Inject -import kotlinx.datetime.DatePeriod -import kotlinx.datetime.DayOfWeek import kotlinx.datetime.LocalDate import kotlinx.datetime.LocalDateTime import kotlinx.datetime.TimeZone import kotlinx.datetime.atStartOfDayIn -import kotlinx.datetime.daysUntil -import kotlinx.datetime.minus -import kotlinx.datetime.plus import kotlinx.datetime.toLocalDateTime import tachiyomi.domain.chapter.interactor.GetChaptersByMangaId -import tachiyomi.domain.chapter.model.Chapter import tachiyomi.domain.manga.model.Manga import tachiyomi.domain.manga.model.MangaUpdate -import kotlin.math.absoluteValue import kotlin.time.Clock import kotlin.time.Duration.Companion.days import kotlin.time.Duration.Companion.milliseconds @@ -26,28 +19,45 @@ class FetchInterval( private val getChaptersByMangaId: GetChaptersByMangaId, ) { + /** + * Works out the entry's fetch interval and the next day to check it for new chapters. + * + * @param foundRelease whether the check that led here found new chapters. + */ suspend fun withFetchInterval( manga: Manga, dateTime: LocalDateTime, timeZone: TimeZone, window: ClosedRange?, + foundRelease: Boolean = false, ): MangaUpdate { + val today = dateTime.date val currentWindow = window ?: getWindow(Clock.System.now().toLocalDateTime(timeZone).date, timeZone) - val nextUpdate: Instant - val interval: Int - if (manga.fetchInterval < 0) { - interval = manga.fetchInterval - nextUpdate = calculateCustomNextUpdate(manga, interval.absoluteValue, dateTime, timeZone, currentWindow) + val schedule = if (manga.fetchInterval < 0) { + // The check that found a release is the entry's last update + val lastUpdate = if (foundRelease) today else manga.lastUpdate?.toLocalDateTime(timeZone)?.date + ReleaseSchedule.fixed(-manga.fetchInterval, lastUpdate ?: today) } else { val chapters = getChaptersByMangaId.await(manga.id, applyScanlatorFilter = true) - val schedule = estimateSchedule(chapters, timeZone, dateTime.date) - interval = schedule.interval - nextUpdate = calculateNextUpdate(manga, schedule, dateTime.date, timeZone, currentWindow) + ReleaseSchedule.estimate(chapters, timeZone, today) + } + + val expected = manga.nextUpdate + val expectedDay = expected?.toLocalDateTime(timeZone)?.date + // A release on or after the expected day means the next one is counted from it instead + val isAwaitingRelease = expected != null && + expectedDay != null && + expected in currentWindow.start..currentWindow.endInclusive + 1.milliseconds && + (schedule.lastRelease == null || schedule.lastRelease < expectedDay) + val nextUpdate = when { + foundRelease -> schedule.nextCheckAfterRelease(today).atStartOfDayIn(timeZone) + isAwaitingRelease -> expected + else -> schedule.nextCheck(today).atStartOfDayIn(timeZone) } return MangaUpdate(manga.id) { this.nextUpdate = nextUpdate - fetchInterval = interval + fetchInterval = if (manga.fetchInterval < 0) manga.fetchInterval else schedule.interval } } @@ -56,189 +66,9 @@ class FetchInterval( return (today - GRACE_PERIOD.days)..(today + GRACE_PERIOD.days) } - internal fun calculateInterval(chapters: List, zone: TimeZone): Int { - return estimateSchedule(chapters, zone, Clock.System.now().toLocalDateTime(zone).date).interval - } - - internal fun estimateSchedule(chapters: List, zone: TimeZone, today: LocalDate): ReleaseSchedule { - val chapterWindow = if (chapters.size <= 8) 3 else 10 - - // Sources that date chapters ahead of their release would otherwise pull the schedule into the future - val latestPlausibleDay = today.plus(DatePeriod(days = 1)) - fun releaseDays(dates: Sequence): List { - return dates - .map { it.toLocalDateTime(zone).date } - .filter { it <= latestPlausibleDay } - .distinct() - .sortedDescending() - .take(chapterWindow) - .toList() - } - - val uploadDays = releaseDays(chapters.asSequence().mapNotNull { it.dateUpload }) - val days = uploadDays.takeIf { it.size >= 3 } - ?: releaseDays(chapters.asSequence().map { it.dateFetch }).takeIf { it.size >= 3 } - ?: return ReleaseSchedule( - interval = DEFAULT_INTERVAL, - lastRelease = uploadDays.firstOrNull(), - pattern = ReleasePattern.EveryDays(DEFAULT_INTERVAL), - ) - - val gaps = days.zipWithNext { newer, older -> older.daysUntil(newer) }.sorted() - val medianGap = gaps[(gaps.size - 1) / 2] - val interval = medianGap.coerceIn(1, MAX_INTERVAL) - - return ReleaseSchedule( - interval = interval, - lastRelease = days.first(), - pattern = detectPattern(days, medianGap) ?: ReleasePattern.EveryDays(interval), - ) - } - - /** - * Finds a calendar pattern a plain day interval can't follow: a few fixed weekdays, or a day of the - * month. Both need a few releases to be told apart from chance. - */ - private fun detectPattern(days: List, medianGap: Int): ReleasePattern? { - if (days.size < MIN_RELEASES_FOR_PATTERN) return null - - if (medianGap in MONTHLY_GAPS) { - val daysOfMonth = days.map { it.day } - return when { - daysOfMonth.max() - daysOfMonth.min() <= DAY_OF_MONTH_TOLERANCE -> { - ReleasePattern.DayOfMonth(daysOfMonth.sorted()[(daysOfMonth.size - 1) / 2]) - } - // The last day of the month moves between the 28th and the 31st - daysOfMonth.all { it >= 28 } -> ReleasePattern.DayOfMonth(31) - else -> null - } - } - - val byFrequency = days.groupingBy { it.dayOfWeek }.eachCount().entries.sortedByDescending { it.value } - val weekdays = mutableSetOf() - var covered = 0 - for ((dayOfWeek, count) in byFrequency) { - if (covered >= days.size * WEEKDAY_COVERAGE) break - weekdays += dayOfWeek - covered += count - } - // One weekday is a weekly interval, and an irregular schedule spreads over most of the week - if (weekdays.size !in 2..MAX_WEEKDAYS || covered < days.size * WEEKDAY_COVERAGE) return null - // Releasing on several weekdays means more than one release a week - if (medianGap > 7 / weekdays.size + 1) return null - return ReleasePattern.Weekdays(weekdays) - } - - private fun calculateNextUpdate( - manga: Manga, - schedule: ReleaseSchedule, - today: LocalDate, - timeZone: TimeZone, - window: ClosedRange, - ): Instant { - val lastRelease = listOfNotNull(schedule.lastRelease, manga.lastUpdate?.toLocalDateTime(timeZone)?.date) - .maxOrNull() - - // A release on or after the expected day means the next one should be counted from it instead - val nextUpdate = manga.nextUpdate - if ( - nextUpdate != null && - nextUpdate in window.start..window.endInclusive + 1.milliseconds && - (lastRelease == null || lastRelease < nextUpdate.toLocalDateTime(timeZone).date) - ) { - return nextUpdate - } - - val anchor = schedule.lastRelease ?: lastRelease ?: today - val daysSinceRelease = anchor.daysUntil(today) - val expected = if (daysSinceRelease > schedule.interval * HIATUS_CYCLES) { - // Past a few missed releases the schedule no longer holds, so checks spread out until it resumes - val interval = backOff(schedule.interval, daysSinceRelease) - anchor.plus(DatePeriod(days = ceilDiv(daysSinceRelease, interval) * interval)) - } else { - // Today counts, so a release expected today keeps being checked for until the window passes it - generateSequence(schedule.pattern.nextAfter(anchor)) { schedule.pattern.nextAfter(it) } - .first { it >= today } - } - return expected.atStartOfDayIn(timeZone) - } - - private fun calculateCustomNextUpdate( - manga: Manga, - interval: Int, - dateTime: LocalDateTime, - timeZone: TimeZone, - window: ClosedRange, - ): Instant { - if (manga.nextUpdate != null && manga.nextUpdate in window.start..window.endInclusive + 1.milliseconds) { - return manga.nextUpdate - } - - val latestDate = (manga.lastUpdate ?: Clock.System.now()).toLocalDateTime(timeZone).date - val cycle = latestDate.daysUntil(dateTime.date).floorDiv(interval) - return latestDate.plus(DatePeriod(days = (cycle + 1) * interval)).atStartOfDayIn(timeZone) - } - - private fun backOff(interval: Int, daysSinceRelease: Int): Int { - var backedOff = interval - while (backedOff < MAX_INTERVAL && daysSinceRelease > backedOff * HIATUS_CYCLES) { - backedOff *= 2 - } - return backedOff.coerceAtMost(MAX_INTERVAL) - } - - private fun ceilDiv(a: Int, b: Int) = -Math.floorDiv(-a, b) - companion object { const val MAX_INTERVAL = 28 - private const val DEFAULT_INTERVAL = 7 private const val GRACE_PERIOD = 1L - private const val HIATUS_CYCLES = 4 - private const val MIN_RELEASES_FOR_PATTERN = 4 - private const val MAX_WEEKDAYS = 3 - private const val WEEKDAY_COVERAGE = 0.8 - private const val DAY_OF_MONTH_TOLERANCE = 3 - private val MONTHLY_GAPS = 26..35 - } -} - -internal data class ReleaseSchedule( - /** Days between releases, which is what the fetch interval shows. */ - val interval: Int, - /** The most recent release day the next ones are counted from, if any chapter has a date. */ - val lastRelease: LocalDate?, - val pattern: ReleasePattern, -) - -internal sealed interface ReleasePattern { - - /** The first expected release day after [date]. */ - fun nextAfter(date: LocalDate): LocalDate - - data class EveryDays(val days: Int) : ReleasePattern { - override fun nextAfter(date: LocalDate) = date.plus(DatePeriod(days = days)) - } - - data class Weekdays(val days: Set) : ReleasePattern { - override fun nextAfter(date: LocalDate): LocalDate { - return generateSequence(date.plus(DatePeriod(days = 1))) { it.plus(DatePeriod(days = 1)) } - .first { it.dayOfWeek in days } - } - } - - /** Releases on [day] of every month, or its last day in months too short for it. */ - data class DayOfMonth(val day: Int) : ReleasePattern { - override fun nextAfter(date: LocalDate): LocalDate { - val sameMonth = date.inMonthClamped(day) - if (sameMonth > date) return sameMonth - return date.plus(DatePeriod(months = 1)).inMonthClamped(day) - } - - private fun LocalDate.inMonthClamped(day: Int): LocalDate { - val firstOfMonth = LocalDate(year, month, 1) - val lastDay = firstOfMonth.plus(DatePeriod(months = 1)).minus(DatePeriod(days = 1)).day - return LocalDate(year, month, day.coerceAtMost(lastDay)) - } } } diff --git a/domain/src/main/java/tachiyomi/domain/manga/interactor/ReleasePattern.kt b/domain/src/main/java/tachiyomi/domain/manga/interactor/ReleasePattern.kt new file mode 100644 index 000000000..12ab46888 --- /dev/null +++ b/domain/src/main/java/tachiyomi/domain/manga/interactor/ReleasePattern.kt @@ -0,0 +1,212 @@ +package tachiyomi.domain.manga.interactor + +import kotlinx.datetime.DatePeriod +import kotlinx.datetime.DayOfWeek +import kotlinx.datetime.LocalDate +import kotlinx.datetime.daysUntil +import kotlinx.datetime.isoDayNumber +import kotlinx.datetime.minus +import kotlinx.datetime.plus +import kotlin.math.abs + +/** The days an entry is expected to release on. */ +internal sealed interface ReleasePattern { + + /** The first expected release day after [date]. */ + fun nextAfter(date: LocalDate): LocalDate + + /** The last expected release day before [date]. */ + fun previousBefore(date: LocalDate): LocalDate + + /** The expected release day closest to [date], which a release on [date] stands in for. */ + fun nearest(date: LocalDate): LocalDate { + val before = previousBefore(date.plus(DatePeriod(days = 1))) + val after = nextAfter(date.minus(DatePeriod(days = 1))) + // A release halfway between is taken as late, so the next check comes sooner + return if (before.daysUntil(date) <= date.daysUntil(after)) before else after + } + + /** Releases every [days] days, in step with [onSchedule]. */ + data class EveryDays(val days: Int, val onSchedule: LocalDate) : ReleasePattern { + override fun nextAfter(date: LocalDate): LocalDate { + return date.plus(DatePeriod(days = days - onSchedule.daysUntil(date).mod(days))) + } + + override fun previousBefore(date: LocalDate): LocalDate { + return date.minus(DatePeriod(days = (onSchedule.daysUntil(date) - 1).mod(days) + 1)) + } + } + + data class Weekdays(val days: Set) : ReleasePattern { + override fun nextAfter(date: LocalDate): LocalDate { + return generateSequence(date.plus(DatePeriod(days = 1))) { it.plus(DatePeriod(days = 1)) } + .first { it.dayOfWeek in days } + } + + override fun previousBefore(date: LocalDate): LocalDate { + return generateSequence(date.minus(DatePeriod(days = 1))) { it.minus(DatePeriod(days = 1)) } + .first { it.dayOfWeek in days } + } + + // Release days are a day or two apart, so a release between them is more likely an extra than an early one + override fun nearest(date: LocalDate) = date + } + + /** Releases for [length] days in a row, every [every] days in step with [onSchedule]. */ + data class Batches(val every: Int, val onSchedule: LocalDate, val length: Int) : ReleasePattern { + override fun nextAfter(date: LocalDate): LocalDate { + val offset = onSchedule.daysUntil(date).mod(every) + return date.plus(DatePeriod(days = if (offset + 1 < length) 1 else every - offset)) + } + + override fun previousBefore(date: LocalDate): LocalDate { + val dayBefore = date.minus(DatePeriod(days = 1)) + val offset = onSchedule.daysUntil(dayBefore).mod(every) + return dayBefore.minus(DatePeriod(days = (offset - length + 1).coerceAtLeast(0))) + } + } + + /** A pattern that repeats every month. */ + sealed interface Monthly : ReleasePattern { + /** The release days in the month starting on [firstOfMonth]. */ + fun daysIn(firstOfMonth: LocalDate): List + + override fun nextAfter(date: LocalDate): LocalDate { + return generateSequence(date.firstOfMonth()) { it.plus(DatePeriod(months = 1)) } + .flatMap { daysIn(it).sorted() } + .first { it > date } + } + + override fun previousBefore(date: LocalDate): LocalDate { + return generateSequence(date.firstOfMonth()) { it.minus(DatePeriod(months = 1)) } + .flatMap { daysIn(it).sortedDescending() } + .first { it < date } + } + } + + /** Releases on [days] of every month, or a month's last day for those past it. */ + data class DaysOfMonth(val days: Set) : Monthly { + override fun daysIn(firstOfMonth: LocalDate): List { + val lastDay = firstOfMonth.lastOfMonth().day + return days.map { firstOfMonth.plus(DatePeriod(days = it.coerceAtMost(lastDay) - 1)) }.distinct() + } + } + + /** Releases on [dayOfWeek] of the [week]th week of every month, or its last week for [LAST_WEEK]. */ + data class WeekdayOfMonth(val week: Int, val dayOfWeek: DayOfWeek) : Monthly { + override fun daysIn(firstOfMonth: LocalDate): List { + if (week == LAST_WEEK) { + val lastOfMonth = firstOfMonth.lastOfMonth() + val daysBack = (lastOfMonth.dayOfWeek.isoDayNumber - dayOfWeek.isoDayNumber).mod(7) + return listOf(lastOfMonth.minus(DatePeriod(days = daysBack))) + } + val daysAhead = (dayOfWeek.isoDayNumber - firstOfMonth.dayOfWeek.isoDayNumber).mod(7) + return listOf(firstOfMonth.plus(DatePeriod(days = daysAhead + (week - 1) * 7))) + } + } + + companion object { + const val LAST_WEEK = -1 + + private const val MIN_RELEASES = 4 + private const val MAX_WEEKDAYS = 3 + private const val COVERAGE = 0.8 + private const val DAY_OF_MONTH_TOLERANCE = 3 + private val MONTHLY_GAPS = 26..35 + private val TWICE_MONTHLY_GAPS = 12..18 + + private const val BATCH_HISTORY = 40 + private const val BATCH_GAP = 2 + private const val BATCHES = 6 + private const val MIN_BATCHES = 3 + + /** + * Finds a calendar pattern a plain day interval can't follow in [days], newest first: a few fixed weekdays, + * days or a weekday of the month, or releases in batches, which can take longer than [days] to show up in + * [history]. They all need a few releases to be told apart from chance. + */ + fun detect(days: List, history: List, medianGap: Int): ReleasePattern? { + if (days.size < MIN_RELEASES) return null + // Batches come first, since their days can look like a few weekdays or a monthly schedule + return batches(history) ?: when (medianGap) { + in MONTHLY_GAPS -> monthly(days) + in TWICE_MONTHLY_GAPS -> twiceMonthly(days) + else -> weekdays(days, medianGap) + } + } + + private fun monthly(days: List): ReleasePattern? { + val daysOfMonth = days.map { it.day } + return when { + daysOfMonth.max() - daysOfMonth.min() <= DAY_OF_MONTH_TOLERANCE -> DaysOfMonth( + setOf(daysOfMonth.median()), + ) + // The last day of the month moves between the 28th and the 31st + daysOfMonth.all { it >= 28 } -> DaysOfMonth(setOf(31)) + else -> weekdayOfMonth(days) + } + } + + private fun weekdayOfMonth(days: List): ReleasePattern? { + val dayOfWeek = days.mostCommon { it.dayOfWeek } ?: return null + // A fifth week isn't in every month, so those days only count as the last week + val week = days.mostCommon { (it.day - 1) / 7 + 1 }?.takeIf { it < 5 } + if (week != null) return WeekdayOfMonth(week, dayOfWeek) + val isLastWeek = days.count { it.day + 7 > it.lastOfMonth().day } >= days.size * COVERAGE + return if (isLastWeek) WeekdayOfMonth(LAST_WEEK, dayOfWeek) else null + } + + private fun twiceMonthly(days: List): ReleasePattern? { + val daysOfMonth = days.map { it.day }.sorted() + val split = (1.. DAY_OF_MONTH_TOLERANCE }) return null + return DaysOfMonth(halves.map { it.median() }.toSet()) + } + + private fun weekdays(days: List, medianGap: Int): ReleasePattern? { + val byFrequency = days.groupingBy { it.dayOfWeek }.eachCount().entries.sortedByDescending { it.value } + val weekdays = mutableSetOf() + var covered = 0 + for ((dayOfWeek, count) in byFrequency) { + if (covered >= days.size * COVERAGE) break + weekdays += dayOfWeek + covered += count + } + // One weekday is a weekly interval, and an irregular schedule spreads over most of the week + if (weekdays.size !in 2..MAX_WEEKDAYS || covered < days.size * COVERAGE) return null + // Releasing on several weekdays means more than one release a week + if (medianGap > 7 / weekdays.size + 1) return null + return Weekdays(weekdays) + } + + /** Runs of releases a day or two apart, starting at a steady interval and mostly longer than a day. */ + private fun batches(history: List): ReleasePattern? { + val runs = mutableListOf>() + for (day in history.take(BATCH_HISTORY).asReversed()) { + val run = runs.lastOrNull() + if (run != null && run.last().daysUntil(day) <= BATCH_GAP) run += day else runs += mutableListOf(day) + } + val recent = runs.takeLast(BATCHES) + if (recent.size < MIN_BATCHES || recent.count { it.size > 1 } * 2 < recent.size) return null + + val gaps = recent.zipWithNext { older, newer -> older.first().daysUntil(newer.first()) } + val every = gaps.median() + val length = recent.map { it.first().daysUntil(it.last()) + 1 }.median() + if (every < length * 2 || gaps.any { abs(it - every) > every / 4 }) return null + return Batches(every, recent.last().first(), length) + } + + private fun List.median() = sorted()[(size - 1) / 2] + + /** The most common key of [selector], if at least [COVERAGE] of the days share it. */ + private fun List.mostCommon(selector: (LocalDate) -> T): T? { + val (key, count) = groupingBy(selector).eachCount().maxBy { it.value } + return key.takeIf { count >= size * COVERAGE } + } + } +} + +private fun LocalDate.firstOfMonth() = LocalDate(year, month, 1) + +private fun LocalDate.lastOfMonth() = firstOfMonth().plus(DatePeriod(months = 1)).minus(DatePeriod(days = 1)) diff --git a/domain/src/main/java/tachiyomi/domain/manga/interactor/ReleaseSchedule.kt b/domain/src/main/java/tachiyomi/domain/manga/interactor/ReleaseSchedule.kt new file mode 100644 index 000000000..64d5fdc12 --- /dev/null +++ b/domain/src/main/java/tachiyomi/domain/manga/interactor/ReleaseSchedule.kt @@ -0,0 +1,151 @@ +package tachiyomi.domain.manga.interactor + +import kotlinx.datetime.DatePeriod +import kotlinx.datetime.LocalDate +import kotlinx.datetime.TimeZone +import kotlinx.datetime.daysUntil +import kotlinx.datetime.plus +import kotlinx.datetime.toLocalDateTime +import tachiyomi.domain.chapter.model.Chapter +import kotlin.time.Instant + +/** How often an entry releases, and so when to check it for the next release. */ +internal data class ReleaseSchedule( + /** Days between releases, which is what the fetch interval shows. */ + val interval: Int, + /** The most recent release day, which the next ones are counted from. */ + val lastRelease: LocalDate?, + val pattern: ReleasePattern, + /** Days past the expected day a release can still come, from how late recent ones were. */ + val lateness: Int = 0, + /** Whether checks spread out once releases stop coming, which a schedule set by the user doesn't. */ + val backsOff: Boolean = true, + /** Lengths in days of earlier breaks in releases, which a break like a season's end is expected to last again. */ + val pastBreaks: List = emptyList(), +) { + + /** The next day to check for a release, as of [today]. Today counts, so a due release keeps being checked for. */ + fun nextCheck(today: LocalDate): LocalDate { + val anchor = lastRelease ?: today + val daysSinceRelease = anchor.daysUntil(today) + val isOnBreak = backsOff && daysSinceRelease > interval * HIATUS_CYCLES + // Around when earlier breaks ended, this one likely ends too, so the schedule is followed again + if (isOnBreak && pastBreaks.none { daysSinceRelease in it - interval..it + interval }) { + // Past a few missed releases the schedule no longer holds, so checks spread out until it resumes + val backedOff = backOff(daysSinceRelease) + val next = anchor.plus(DatePeriod(days = ceilDiv(daysSinceRelease, backedOff) * backedOff)) + val breakEnd = pastBreaks.map { + anchor.plus(DatePeriod(days = it - interval)) + }.filter { it > today }.minOrNull() + return minOf(next, breakEnd ?: next) + } + + val due = pattern.nextAfter(pattern.nearest(anchor)) + if (today in due..due.plus(DatePeriod(days = lateness))) return today + return generateSequence(due, pattern::nextAfter).first { it >= today }.cappedFrom(today) + } + + /** + * The next day to check after a check [today] found a release. + * + * The release is dated by its newest chapter, since a check can turn up one that came out on an earlier day. It + * stands in for the expected day nearest to it, so an early, late or extra release leaves the schedule in place. + */ + fun nextCheckAfterRelease(today: LocalDate): LocalDate { + val release = lastRelease?.coerceAtMost(today) ?: today + // What was found is out already, so checking again today would only find it again + val next = pattern.nextAfter(pattern.nearest(release)) + return generateSequence(next, pattern::nextAfter).first { it > today }.cappedFrom(today) + } + + // Months and batches can be further apart than a check should ever wait + private fun LocalDate.cappedFrom(today: LocalDate): LocalDate { + return coerceAtMost(today.plus(DatePeriod(days = FetchInterval.MAX_INTERVAL))) + } + + private fun backOff(daysSinceRelease: Int): Int { + var backedOff = interval + while (backedOff < FetchInterval.MAX_INTERVAL && daysSinceRelease > backedOff * HIATUS_CYCLES) { + backedOff *= 2 + } + return backedOff.coerceAtMost(FetchInterval.MAX_INTERVAL) + } + + private fun ceilDiv(a: Int, b: Int) = -Math.floorDiv(-a, b) + + companion object { + private const val DEFAULT_INTERVAL = 7 + private const val HIATUS_CYCLES = 4 + private const val MIN_RELEASES = 3 + private const val FEW_CHAPTERS = 8 + private const val FEW_CHAPTERS_RELEASES = 3 + private const val RECENT_RELEASES = 10 + private const val PHASE_RELEASES = 5 + + /** A schedule the user set to every [interval] days, counted from [lastRelease]. */ + fun fixed(interval: Int, lastRelease: LocalDate): ReleaseSchedule { + val pattern = ReleasePattern.EveryDays(interval, onSchedule = lastRelease) + return ReleaseSchedule(interval, lastRelease, pattern, backsOff = false) + } + + /** Estimates the schedule from the recent release days of [chapters], by upload date or else fetch date. */ + fun estimate(chapters: List, zone: TimeZone, today: LocalDate): ReleaseSchedule { + val recentReleases = if (chapters.size <= FEW_CHAPTERS) FEW_CHAPTERS_RELEASES else RECENT_RELEASES + // Sources that date chapters ahead of their release would otherwise pull the schedule into the future + val latestPlausibleDay = today.plus(DatePeriod(days = 1)) + fun releaseDays(dates: Sequence): List { + return dates + .map { it.toLocalDateTime(zone).date } + .filter { it <= latestPlausibleDay } + .distinct() + .sortedDescending() + .toList() + } + + val uploadDays = releaseDays(chapters.asSequence().mapNotNull { it.dateUpload }) + val fetchDays = releaseDays(chapters.asSequence().map { it.dateFetch }) + val history = uploadDays.takeIf { it.size >= MIN_RELEASES } + ?: fetchDays.takeIf { it.size >= MIN_RELEASES } + ?: run { + val lastRelease = uploadDays.firstOrNull() ?: fetchDays.firstOrNull() + val pattern = ReleasePattern.EveryDays(DEFAULT_INTERVAL, onSchedule = lastRelease ?: today) + return ReleaseSchedule(DEFAULT_INTERVAL, lastRelease, pattern) + } + val days = history.take(recentReleases) + + val gaps = days.zipWithNext { newer, older -> older.daysUntil(newer) }.sorted() + val medianGap = gaps[(gaps.size - 1) / 2] + val detected = ReleasePattern.detect(days, history, medianGap) + // Within a batch releases are a day apart, but it's the batches that come every so often + val interval = ((detected as? ReleasePattern.Batches)?.every ?: medianGap) + .coerceIn(1, FetchInterval.MAX_INTERVAL) + return ReleaseSchedule( + interval = interval, + lastRelease = days.first(), + pattern = detected + ?: ReleasePattern.EveryDays( + interval, + onSchedule = medianPhase(days.take(PHASE_RELEASES), interval), + ), + lateness = (gaps.last() - interval).coerceIn(0, interval / 2), + pastBreaks = history + .zipWithNext { newer, older -> older.daysUntil(newer) } + .filter { it > interval * HIATUS_CYCLES }, + ) + } + + /** + * The release in [days], newest first, that the others land closest to in step with [interval], so one early + * or late release doesn't shift the schedule. Ties go to the newest, which follows a schedule that moved. + */ + private fun medianPhase(days: List, interval: Int): LocalDate { + fun offset( + from: LocalDate, + to: LocalDate, + ) = from.daysUntil(to).mod(interval).let { minOf(it, interval - it) } + val phase = days.minBy { candidate -> days.sumOf { offset(candidate, it) } } + val onPhase = days.count { offset(phase, it) == 0 } + return if (onPhase * 5 >= days.size * 3) phase else days.first() + } + } +} diff --git a/domain/src/test/java/tachiyomi/domain/manga/interactor/FetchIntervalTest.kt b/domain/src/test/java/tachiyomi/domain/manga/interactor/FetchIntervalTest.kt index 285afa74b..2034d5d03 100644 --- a/domain/src/test/java/tachiyomi/domain/manga/interactor/FetchIntervalTest.kt +++ b/domain/src/test/java/tachiyomi/domain/manga/interactor/FetchIntervalTest.kt @@ -20,6 +20,7 @@ import org.junit.jupiter.api.parallel.ExecutionMode import tachiyomi.domain.chapter.interactor.GetChaptersByMangaId import tachiyomi.domain.chapter.model.Chapter import tachiyomi.domain.manga.model.Manga +import kotlin.time.Clock import kotlin.time.Duration import kotlin.time.Duration.Companion.days import kotlin.time.Duration.Companion.hours @@ -34,19 +35,17 @@ class FetchIntervalTest { dateUpload = testTime.toInstant(testTimeZone), ) - private val fetchInterval = FetchInterval(mockk()) - @Test fun `returns default interval of 7 days when not enough distinct days`() { val chaptersWithUploadDate = (1..50).map { chapterWithTime(chapter, 1.days) } - fetchInterval.calculateInterval(chaptersWithUploadDate, testTimeZone) shouldBe 7 + interval(chaptersWithUploadDate) shouldBe 7 val chaptersWithoutUploadDate = chaptersWithUploadDate.map { it.copy(dateUpload = null) } - fetchInterval.calculateInterval(chaptersWithoutUploadDate, testTimeZone) shouldBe 7 + interval(chaptersWithoutUploadDate) shouldBe 7 } @Test @@ -60,7 +59,7 @@ class FetchIntervalTest { val chapters = oldChapters + newChapters - fetchInterval.calculateInterval(chapters, testTimeZone) shouldBe 1 + interval(chapters) shouldBe 1 } @Test @@ -75,7 +74,7 @@ class FetchIntervalTest { val chapters = oldChapters + newChapters - fetchInterval.calculateInterval(chapters, testTimeZone) shouldBe 7 + interval(chapters) shouldBe 7 } @Test @@ -83,7 +82,7 @@ class FetchIntervalTest { val chapters = (1..10).map { chapterWithTime(chapter, 10.hours) } - fetchInterval.calculateInterval(chapters, testTimeZone) shouldBe 7 + interval(chapters) shouldBe 7 } @Test @@ -93,7 +92,7 @@ class FetchIntervalTest { } + (1..5).map { chapterWithTime(chapter, 2.days) } - fetchInterval.calculateInterval(chapters, testTimeZone) shouldBe 7 + interval(chapters) shouldBe 7 } @Test @@ -101,7 +100,7 @@ class FetchIntervalTest { val chapters = (1..20).map { chapterWithTime(chapter, it.days) } - fetchInterval.calculateInterval(chapters, testTimeZone) shouldBe 1 + interval(chapters) shouldBe 1 } @Test @@ -109,7 +108,7 @@ class FetchIntervalTest { val chapters = (1..20).map { chapterWithTime(chapter, (15 * it).hours) } - fetchInterval.calculateInterval(chapters, testTimeZone) shouldBe 1 + interval(chapters) shouldBe 1 } @Test @@ -117,7 +116,7 @@ class FetchIntervalTest { val chapters = (1..20).map { chapterWithTime(chapter, (2 * it).days) } - fetchInterval.calculateInterval(chapters, testTimeZone) shouldBe 2 + interval(chapters) shouldBe 2 } @Test @@ -125,12 +124,12 @@ class FetchIntervalTest { val chaptersWithUploadDate = (1..5).map { chapterWithTime(chapter, (25 * it).hours) } - fetchInterval.calculateInterval(chaptersWithUploadDate, testTimeZone) shouldBe 1 + interval(chaptersWithUploadDate) shouldBe 1 val chaptersWithoutUploadDate = chaptersWithUploadDate.map { it.copy(dateUpload = null) } - fetchInterval.calculateInterval(chaptersWithoutUploadDate, testTimeZone) shouldBe 1 + interval(chaptersWithoutUploadDate) shouldBe 1 } @Test @@ -138,7 +137,7 @@ class FetchIntervalTest { val chapters = (1..20).map { chapterWithTime(chapter, (43 * it).hours) } - fetchInterval.calculateInterval(chapters, testTimeZone) shouldBe 2 + interval(chapters) shouldBe 2 } @Test @@ -149,7 +148,7 @@ class FetchIntervalTest { .toList() val chapters = chaptersOn(mondaysAndThursdays) - fetchInterval.estimateSchedule(chapters, testTimeZone, LocalDate.parse("2020-01-21")).pattern shouldBe + ReleaseSchedule.estimate(chapters, testTimeZone, LocalDate.parse("2020-01-21")).pattern shouldBe ReleasePattern.Weekdays(setOf(DayOfWeek.MONDAY, DayOfWeek.THURSDAY)) nextUpdate(today = "2020-01-21", chapters) shouldBe LocalDate.parse("2020-01-23") } @@ -174,6 +173,62 @@ class FetchIntervalTest { nextUpdate(today = "2020-02-05", chapters) shouldBe LocalDate.parse("2020-02-29") } + @Test + fun `follows releases on two days of the month`() = runTest { + val days = + listOf("2019-10-01", "2019-10-15", "2019-11-01", "2019-11-16", "2019-12-01", "2019-12-15", "2020-01-01") + val chapters = chaptersOn((days + "2020-01-15").map(LocalDate::parse), perDay = 2) + + ReleaseSchedule.estimate(chapters, testTimeZone, LocalDate.parse("2020-01-20")).pattern shouldBe + ReleasePattern.DaysOfMonth(setOf(1, 15)) + nextUpdate(today = "2020-01-20", chapters) shouldBe LocalDate.parse("2020-02-01") + } + + @Test + fun `follows releases on a weekday of the month`() = runTest { + val firstSaturdays = listOf("2019-09-07", "2019-10-05", "2019-11-02", "2019-12-07", "2020-01-04") + val chapters = chaptersOn(firstSaturdays.map(LocalDate::parse), perDay = 2) + + nextUpdate(today = "2020-01-10", chapters) shouldBe LocalDate.parse("2020-02-01") + } + + @Test + fun `follows releases on a weekday of the last week of the month`() = runTest { + val lastFridays = listOf("2019-09-27", "2019-10-25", "2019-11-29", "2019-12-27", "2020-01-31") + val chapters = chaptersOn(lastFridays.map(LocalDate::parse), perDay = 2) + + nextUpdate(today = "2020-02-03", chapters) shouldBe LocalDate.parse("2020-02-28") + } + + @Test + fun `follows releases in batches`() = runTest { + // Three days of releases every three weeks + val batches = every(days = 21, from = "2019-09-02", to = "2019-12-16") + .flatMap { start -> (0..2).map { start.plus(DatePeriod(days = it)) } } + val chapters = chaptersOn(batches.filter { it <= LocalDate.parse("2019-12-17") }) + + nextUpdate(today = "2019-12-17", chapters, foundRelease = true) shouldBe LocalDate.parse("2019-12-18") + nextUpdate(today = "2019-12-18", chaptersOn(batches), foundRelease = true) shouldBe + LocalDate.parse("2020-01-06") + } + + @Test + fun `checks again around when an earlier break ended`() = runTest { + // A season of weekly releases, a 98 day break, and another season + val chapters = chaptersOn(weekly(from = "2019-01-02", to = "2019-03-27") + weekly("2019-07-03", "2019-09-25")) + + nextUpdate(today = "2019-12-20", chapters) shouldBe LocalDate.parse("2019-12-25") + nextUpdate(today = "2019-12-26", chapters) shouldBe LocalDate.parse("2020-01-01") + } + + @Test + fun `never waits more than 28 days between checks`() = runTest { + val lastDays = listOf("2019-09-30", "2019-10-31", "2019-11-30", "2019-12-31", "2020-01-31") + val chapters = chaptersOn(lastDays.map(LocalDate::parse), perDay = 2) + + nextUpdate(today = "2020-01-31", chapters, foundRelease = true) shouldBe LocalDate.parse("2020-02-28") + } + @Test fun `keeps checking on the expected day`() = runTest { val chapters = chaptersOn(weekly(from = "2019-12-04", to = "2019-12-25")) @@ -197,6 +252,71 @@ class FetchIntervalTest { nextUpdate(today = "2020-01-01", chapters, manga) shouldBe LocalDate.parse("2020-01-08") } + @Test + fun `keeps the schedule for a release uploaded the day before it was found`() = runTest { + val chapters = chaptersOn(weekly(from = "2019-10-02", to = "2019-12-25") + LocalDate.parse("2019-12-31")) + + nextUpdate(today = "2020-01-01", chapters, expecting("2020-01-01"), foundRelease = true) shouldBe + LocalDate.parse("2020-01-08") + } + + @Test + fun `keeps the schedule after a release shortly before the expected day`() = runTest { + val chapters = chaptersOn(weekly(from = "2019-10-02", to = "2019-12-25") + LocalDate.parse("2019-12-30")) + + nextUpdate(today = "2019-12-30", chapters, expecting("2020-01-01"), foundRelease = true) shouldBe + LocalDate.parse("2020-01-08") + } + + @Test + fun `keeps the schedule after a late release`() = runTest { + val chapters = chaptersOn(weekly(from = "2019-12-04", to = "2019-12-25") + LocalDate.parse("2020-01-03")) + + nextUpdate(today = "2020-01-03", chapters, expecting("2020-01-01"), foundRelease = true) shouldBe + LocalDate.parse("2020-01-08") + } + + @Test + fun `keeps expecting the release after an extra one well before it`() = runTest { + val chapters = chaptersOn(weekly(from = "2019-10-02", to = "2020-01-01") + LocalDate.parse("2020-01-02")) + + nextUpdate(today = "2020-01-02", chapters, expecting("2020-01-08"), foundRelease = true) shouldBe + LocalDate.parse("2020-01-08") + } + + @Test + fun `follows the schedule again when releases resume after a hiatus`() = runTest { + val chapters = chaptersOn(weekly(from = "2019-09-11", to = "2019-10-02") + LocalDate.parse("2020-01-01")) + + nextUpdate(today = "2020-01-01", chapters, expecting("2020-01-22"), foundRelease = true) shouldBe + LocalDate.parse("2020-01-08") + } + + @Test + fun `follows the schedule once releases keep coming early`() = runTest { + val history = every(days = 5, from = "2019-10-01", to = "2019-12-15") + val chapters = chaptersOn(history + every(days = 5, from = "2019-12-19", to = "2019-12-29")) + + nextUpdate(today = "2019-12-29", chapters, expecting("2019-12-30"), foundRelease = true) shouldBe + LocalDate.parse("2020-01-03") + } + + @Test + fun `keeps checking for an overdue release while recent ones came that late`() = runTest { + val chapters = chaptersOn(afterGaps("2019-10-18", 6, 5, 8, 6, 5, 6, 8, 6, 5)) + + // Due on the 18th from a median gap of 6, and gaps have run to 8 + nextUpdate(today = "2019-12-20", chapters, expecting("2019-12-18")) shouldBe LocalDate.parse("2019-12-20") + } + + @Test + fun `counts from an early release with a custom interval`() = runTest { + val manga = expecting("2020-01-01").copy(fetchInterval = -7) + + nextUpdate(today = "2019-12-31", emptyList(), manga, foundRelease = true) shouldBe + LocalDate.parse("2020-01-07") + } + @Test fun `spreads out checks during a hiatus`() = runTest { val chapters = chaptersOn(weekly(from = "2019-09-11", to = "2019-10-02")) @@ -215,6 +335,7 @@ class FetchIntervalTest { today: String, chapters: List, manga: Manga = Manga.create(), + foundRelease: Boolean = false, ): LocalDate { val getChaptersByMangaId = mockk() coEvery { getChaptersByMangaId.await(any(), any()) } returns chapters @@ -225,16 +346,35 @@ class FetchIntervalTest { dateTime = date.atTime(10, 0), timeZone = testTimeZone, window = fetchInterval.getWindow(date, testTimeZone), + foundRelease = foundRelease, ) return update.nextUpdate!!.toLocalDateTime(testTimeZone).date } - private fun weekly(from: String, to: String): List { - return generateSequence(LocalDate.parse(from)) { it.plus(DatePeriod(days = 7)) } + private fun expecting(day: String): Manga { + return Manga.create().copy(nextUpdate = LocalDate.parse(day).atStartOfDayIn(testTimeZone)) + } + + private fun interval(chapters: List): Int { + return ReleaseSchedule.estimate( + chapters, + testTimeZone, + Clock.System.now().toLocalDateTime(testTimeZone).date, + ).interval + } + + private fun weekly(from: String, to: String) = every(days = 7, from, to) + + private fun every(days: Int, from: String, to: String): List { + return generateSequence(LocalDate.parse(from)) { it.plus(DatePeriod(days = days)) } .takeWhile { it <= LocalDate.parse(to) } .toList() } + private fun afterGaps(from: String, vararg gaps: Int): List { + return gaps.runningFold(LocalDate.parse(from)) { day, gap -> day.plus(DatePeriod(days = gap)) } + } + private fun chaptersOn(days: List, perDay: Int = 1): List { return days.flatMap { day -> val time = day.atStartOfDayIn(testTimeZone) + 12.hours