Follow weekday and monthly release patterns for next update and back it up

app/src/main/java/eu/kanade/tachiyomi/data/backup/create/creators/MangaBackupCreator.kt

@@ -99,4 +99,5 @@ private fun Manga.toBackupManga() =
         notes = this.notes,
         initialized = this.initialized,
         memo = this.memo.toByteArray(),
+        nextUpdate = this.nextUpdate.toEpochMillisOrZero(),
     )

app/src/main/java/eu/kanade/tachiyomi/data/backup/models/BackupManga.kt

@@ -48,6 +48,7 @@ class BackupManga(
     @ProtoNumber(110) var notes: String = "",
     @ProtoNumber(111) var initialized: Boolean = false,
     @ProtoNumber(112) var memo: ByteArray = JsonObjectEmptyBytes,
+    @ProtoNumber(113) var nextUpdate: Long = 0,
 ) {
     fun getMangaImpl(): Manga {
         return Manga.create().copy(
@@ -75,6 +76,7 @@ class BackupManga(
             notes = this@BackupManga.notes,
             initialized = this@BackupManga.initialized,
             memo = this@BackupManga.memo.toJsonObject(),
+            nextUpdate = this@BackupManga.nextUpdate.toInstantOrNull(),
         )
     }
 }

app/src/main/java/eu/kanade/tachiyomi/data/backup/restore/restorers/MangaRestorer.kt

@@ -58,8 +58,13 @@ class MangaRestorer(
             )
         }
 
-        restoreRepository.restoreManga(entries) {
+        restoreRepository.restoreManga(entries) { manga ->
-            fetchInterval.withFetchInterval(it, now, timeZone, currentFetchWindow)
+            fetchInterval.withFetchInterval(manga, now, timeZone, currentFetchWindow).apply {
+                // A next update still ahead was worked out with the chapters the backup came from
+                manga.nextUpdate
+                    ?.takeIf { it >= currentFetchWindow.start }
+                    ?.let { nextUpdate = it }
+            }
         }
     }
 }

data/src/main/java/tachiyomi/data/backup/RestoreRepositoryImpl.kt

@@ -92,6 +92,7 @@ class RestoreRepositoryImpl(
             initialized = dbManga.initialized || manga.initialized,
             // Merge both backup and local data with local winning
             memo = JsonObject(manga.memo + dbManga.memo),
+            nextUpdate = dbManga.nextUpdate ?: manga.nextUpdate,
         )
     }
 

domain/src/main/java/tachiyomi/domain/manga/interactor/FetchInterval.kt

@@ -1,12 +1,15 @@
 package tachiyomi.domain.manga.interactor
 
 import dev.zacsweers.metro.Inject
+import kotlinx.datetime.DatePeriod
+import kotlinx.datetime.DayOfWeek
 import kotlinx.datetime.LocalDate
 import kotlinx.datetime.LocalDateTime
 import kotlinx.datetime.TimeZone
 import kotlinx.datetime.atStartOfDayIn
 import kotlinx.datetime.daysUntil
-import kotlinx.datetime.toInstant
+import kotlinx.datetime.minus
+import kotlinx.datetime.plus
 import kotlinx.datetime.toLocalDateTime
 import tachiyomi.domain.chapter.interactor.GetChaptersByMangaId
 import tachiyomi.domain.chapter.model.Chapter
@@ -29,12 +32,18 @@ class FetchInterval(
         timeZone: TimeZone,
         window: ClosedRange<Instant>?,
     ): MangaUpdate {
-        val interval = manga.fetchInterval.takeIf { it < 0 } ?: calculateInterval(
-            chapters = getChaptersByMangaId.await(manga.id, applyScanlatorFilter = true),
-            zone = timeZone,
-        )
         val currentWindow = window ?: getWindow(Clock.System.now().toLocalDateTime(timeZone).date, timeZone)
-        val nextUpdate = calculateNextUpdate(manga, interval, dateTime, timeZone, currentWindow)
+        val nextUpdate: Instant
+        val interval: Int
+        if (manga.fetchInterval < 0) {
+            interval = manga.fetchInterval
+            nextUpdate = calculateCustomNextUpdate(manga, interval.absoluteValue, dateTime, timeZone, currentWindow)
+        } else {
+            val chapters = getChaptersByMangaId.await(manga.id, applyScanlatorFilter = true)
+            val schedule = estimateSchedule(chapters, timeZone, dateTime.date)
+            interval = schedule.interval
+            nextUpdate = calculateNextUpdate(manga, schedule, dateTime.date, timeZone, currentWindow)
+        }
 
         return MangaUpdate(manga.id) {
             this.nextUpdate = nextUpdate
@@ -48,51 +57,113 @@ class FetchInterval(
     }
 
     internal fun calculateInterval(chapters: List<Chapter>, zone: TimeZone): Int {
+        return estimateSchedule(chapters, zone, Clock.System.now().toLocalDateTime(zone).date).interval
+    }
+
+    internal fun estimateSchedule(chapters: List<Chapter>, zone: TimeZone, today: LocalDate): ReleaseSchedule {
         val chapterWindow = if (chapters.size <= 8) 3 else 10
 
-        val uploadDates = chapters.asSequence()
+        // Sources that date chapters ahead of their release would otherwise pull the schedule into the future
-            .mapNotNull { it.dateUpload }
+        val latestPlausibleDay = today.plus(DatePeriod(days = 1))
-            .sortedDescending()
+        fun releaseDays(dates: Sequence<Instant>): List<LocalDate> {
-            .map {
+            return dates
-                it.toLocalDateTime(zone)
+                .map { it.toLocalDateTime(zone).date }
-                    .date
+                .filter { it <= latestPlausibleDay }
-                    .atStartOfDayIn(zone)
+                .distinct()
-            }
+                .sortedDescending()
-            .distinct()
+                .take(chapterWindow)
-            .take(chapterWindow)
+                .toList()
-            .toList()
-
-        val fetchDates = chapters.asSequence()
-            .map { it.dateFetch }
-            .sortedDescending()
-            .map {
-                it.toLocalDateTime(zone)
-                    .date
-                    .atStartOfDayIn(zone)
-            }
-            .distinct()
-            .take(chapterWindow)
-            .toList()
-
-        val interval = when {
-            // Enough upload date from source
-            uploadDates.size >= 3 -> {
-                val ranges = uploadDates.windowed(2).map { x -> x[1].daysUntil(x[0], zone) }.sorted()
-                ranges[(ranges.size - 1) / 2]
-            }
-            // Enough fetch date from client
-            fetchDates.size >= 3 -> {
-                val ranges = fetchDates.windowed(2).map { x -> x[1].daysUntil(x[0], zone) }.sorted()
-                ranges[(ranges.size - 1) / 2]
-            }
-            // Default to 7 days
-            else -> 7
         }
 
-        return interval.coerceIn(1, MAX_INTERVAL)
+        val uploadDays = releaseDays(chapters.asSequence().mapNotNull { it.dateUpload })
+        val days = uploadDays.takeIf { it.size >= 3 }
+            ?: releaseDays(chapters.asSequence().map { it.dateFetch }).takeIf { it.size >= 3 }
+            ?: return ReleaseSchedule(
+                interval = DEFAULT_INTERVAL,
+                lastRelease = uploadDays.firstOrNull(),
+                pattern = ReleasePattern.EveryDays(DEFAULT_INTERVAL),
+            )
+
+        val gaps = days.zipWithNext { newer, older -> older.daysUntil(newer) }.sorted()
+        val medianGap = gaps[(gaps.size - 1) / 2]
+        val interval = medianGap.coerceIn(1, MAX_INTERVAL)
+
+        return ReleaseSchedule(
+            interval = interval,
+            lastRelease = days.first(),
+            pattern = detectPattern(days, medianGap) ?: ReleasePattern.EveryDays(interval),
+        )
+    }
+
+    /**
+     * Finds a calendar pattern a plain day interval can't follow: a few fixed weekdays, or a day of the
+     * month. Both need a few releases to be told apart from chance.
+     */
+    private fun detectPattern(days: List<LocalDate>, medianGap: Int): ReleasePattern? {
+        if (days.size < MIN_RELEASES_FOR_PATTERN) return null
+
+        if (medianGap in MONTHLY_GAPS) {
+            val daysOfMonth = days.map { it.day }
+            return when {
+                daysOfMonth.max() - daysOfMonth.min() <= DAY_OF_MONTH_TOLERANCE -> {
+                    ReleasePattern.DayOfMonth(daysOfMonth.sorted()[(daysOfMonth.size - 1) / 2])
+                }
+                // The last day of the month moves between the 28th and the 31st
+                daysOfMonth.all { it >= 28 } -> ReleasePattern.DayOfMonth(31)
+                else -> null
+            }
+        }
+
+        val byFrequency = days.groupingBy { it.dayOfWeek }.eachCount().entries.sortedByDescending { it.value }
+        val weekdays = mutableSetOf<DayOfWeek>()
+        var covered = 0
+        for ((dayOfWeek, count) in byFrequency) {
+            if (covered >= days.size * WEEKDAY_COVERAGE) break
+            weekdays += dayOfWeek
+            covered += count
+        }
+        // One weekday is a weekly interval, and an irregular schedule spreads over most of the week
+        if (weekdays.size !in 2..MAX_WEEKDAYS || covered < days.size * WEEKDAY_COVERAGE) return null
+        // Releasing on several weekdays means more than one release a week
+        if (medianGap > 7 / weekdays.size + 1) return null
+        return ReleasePattern.Weekdays(weekdays)
     }
 
     private fun calculateNextUpdate(
+        manga: Manga,
+        schedule: ReleaseSchedule,
+        today: LocalDate,
+        timeZone: TimeZone,
+        window: ClosedRange<Instant>,
+    ): Instant {
+        val lastRelease = listOfNotNull(schedule.lastRelease, manga.lastUpdate?.toLocalDateTime(timeZone)?.date)
+            .maxOrNull()
+
+        // A release on or after the expected day means the next one should be counted from it instead
+        val nextUpdate = manga.nextUpdate
+        if (
+            nextUpdate != null &&
+            nextUpdate in window.start..window.endInclusive + 1.milliseconds &&
+            (lastRelease == null || lastRelease < nextUpdate.toLocalDateTime(timeZone).date)
+        ) {
+            return nextUpdate
+        }
+
+        val anchor = schedule.lastRelease ?: lastRelease ?: today
+        val daysSinceRelease = anchor.daysUntil(today)
+        val expected = if (daysSinceRelease > schedule.interval * HIATUS_CYCLES) {
+            // Past a few missed releases the schedule no longer holds, so checks spread out until it resumes
+            val interval = backOff(schedule.interval, daysSinceRelease)
+            anchor.plus(DatePeriod(days = ceilDiv(daysSinceRelease, interval) * interval))
+        } else {
+            // Today counts, so a release expected today keeps being checked for until the window passes it
+            generateSequence(schedule.pattern.nextAfter(anchor)) { schedule.pattern.nextAfter(it) }
+                .first { it >= today }
+        }
+        return expected.atStartOfDayIn(timeZone)
+    }
+
+    private fun calculateCustomNextUpdate(
         manga: Manga,
         interval: Int,
         dateTime: LocalDateTime,
@@ -103,34 +174,71 @@ class FetchInterval(
             return manga.nextUpdate
         }
 
-        val instant = manga.lastUpdate ?: Clock.System.now()
+        val latestDate = (manga.lastUpdate ?: Clock.System.now()).toLocalDateTime(timeZone).date
-        val latestDate = instant.toLocalDateTime(timeZone).date.atStartOfDayIn(timeZone)
+        val cycle = latestDate.daysUntil(dateTime.date).floorDiv(interval)
-
+        return latestDate.plus(DatePeriod(days = (cycle + 1) * interval)).atStartOfDayIn(timeZone)
-        val daysSinceLatest = (dateTime.toInstant(timeZone) - latestDate).inWholeDays
-        val cycle = daysSinceLatest.floorDiv(
-            interval.absoluteValue.takeIf { interval < 0 }
-                ?: increaseInterval(interval, daysSinceLatest, increaseWhenOver = 10),
-        )
-
-        val offsetDays = ((cycle + 1) * interval.absoluteValue.toLong()).days
-        return latestDate.plus(offsetDays)
     }
 
-    private fun increaseInterval(delta: Int, daysSinceLatest: Long, increaseWhenOver: Int): Int {
+    private fun backOff(interval: Int, daysSinceRelease: Int): Int {
-        if (delta >= MAX_INTERVAL) return MAX_INTERVAL
+        var backedOff = interval
-
+        while (backedOff < MAX_INTERVAL && daysSinceRelease > backedOff * HIATUS_CYCLES) {
-        // double delta again if missed more than 9 check in new delta
+            backedOff *= 2
-        val cycle = daysSinceLatest.floorDiv(delta) + 1
-        return if (cycle > increaseWhenOver) {
-            increaseInterval(delta * 2, daysSinceLatest, increaseWhenOver)
-        } else {
-            delta
         }
+        return backedOff.coerceAtMost(MAX_INTERVAL)
     }
 
+    private fun ceilDiv(a: Int, b: Int) = -Math.floorDiv(-a, b)
+
     companion object {
         const val MAX_INTERVAL = 28
 
+        private const val DEFAULT_INTERVAL = 7
         private const val GRACE_PERIOD = 1L
+        private const val HIATUS_CYCLES = 4
+        private const val MIN_RELEASES_FOR_PATTERN = 4
+        private const val MAX_WEEKDAYS = 3
+        private const val WEEKDAY_COVERAGE = 0.8
+        private const val DAY_OF_MONTH_TOLERANCE = 3
+        private val MONTHLY_GAPS = 26..35
+    }
+}
+
+internal data class ReleaseSchedule(
+    /** Days between releases, which is what the fetch interval shows. */
+    val interval: Int,
+    /** The most recent release day the next ones are counted from, if any chapter has a date. */
+    val lastRelease: LocalDate?,
+    val pattern: ReleasePattern,
+)
+
+internal sealed interface ReleasePattern {
+
+    /** The first expected release day after [date]. */
+    fun nextAfter(date: LocalDate): LocalDate
+
+    data class EveryDays(val days: Int) : ReleasePattern {
+        override fun nextAfter(date: LocalDate) = date.plus(DatePeriod(days = days))
+    }
+
+    data class Weekdays(val days: Set<DayOfWeek>) : ReleasePattern {
+        override fun nextAfter(date: LocalDate): LocalDate {
+            return generateSequence(date.plus(DatePeriod(days = 1))) { it.plus(DatePeriod(days = 1)) }
+                .first { it.dayOfWeek in days }
+        }
+    }
+
+    /** Releases on [day] of every month, or its last day in months too short for it. */
+    data class DayOfMonth(val day: Int) : ReleasePattern {
+        override fun nextAfter(date: LocalDate): LocalDate {
+            val sameMonth = date.inMonthClamped(day)
+            if (sameMonth > date) return sameMonth
+            return date.plus(DatePeriod(months = 1)).inMonthClamped(day)
+        }
+
+        private fun LocalDate.inMonthClamped(day: Int): LocalDate {
+            val firstOfMonth = LocalDate(year, month, 1)
+            val lastDay = firstOfMonth.plus(DatePeriod(months = 1)).minus(DatePeriod(days = 1)).day
+            return LocalDate(year, month, day.coerceAtMost(lastDay))
+        }
     }
 }

domain/src/test/java/tachiyomi/domain/manga/interactor/FetchIntervalTest.kt

@@ -1,18 +1,28 @@
 package tachiyomi.domain.manga.interactor
 
 import io.kotest.matchers.shouldBe
+import io.mockk.coEvery
 import io.mockk.mockk
+import kotlinx.coroutines.test.runTest
+import kotlinx.datetime.DatePeriod
+import kotlinx.datetime.DayOfWeek
+import kotlinx.datetime.LocalDate
 import kotlinx.datetime.LocalDateTime
 import kotlinx.datetime.TimeZone
+import kotlinx.datetime.atStartOfDayIn
+import kotlinx.datetime.atTime
+import kotlinx.datetime.plus
 import kotlinx.datetime.toInstant
+import kotlinx.datetime.toLocalDateTime
 import org.junit.jupiter.api.Test
 import org.junit.jupiter.api.parallel.Execution
 import org.junit.jupiter.api.parallel.ExecutionMode
+import tachiyomi.domain.chapter.interactor.GetChaptersByMangaId
 import tachiyomi.domain.chapter.model.Chapter
+import tachiyomi.domain.manga.model.Manga
 import kotlin.time.Duration
 import kotlin.time.Duration.Companion.days
 import kotlin.time.Duration.Companion.hours
-import kotlin.time.Instant
 
 @Execution(ExecutionMode.CONCURRENT)
 class FetchIntervalTest {
@@ -131,11 +141,112 @@ class FetchIntervalTest {
         fetchInterval.calculateInterval(chapters, testTimeZone) shouldBe 2
     }
 
+    @Test
+    fun `follows releases on a few fixed weekdays`() = runTest {
+        val mondaysAndThursdays = generateSequence(LocalDate.parse("2019-12-02")) { it.plus(DatePeriod(days = 1)) }
+            .takeWhile { it <= LocalDate.parse("2020-01-20") }
+            .filter { it.dayOfWeek == DayOfWeek.MONDAY || it.dayOfWeek == DayOfWeek.THURSDAY }
+            .toList()
+        val chapters = chaptersOn(mondaysAndThursdays)
+
+        fetchInterval.estimateSchedule(chapters, testTimeZone, LocalDate.parse("2020-01-21")).pattern shouldBe
+            ReleasePattern.Weekdays(setOf(DayOfWeek.MONDAY, DayOfWeek.THURSDAY))
+        nextUpdate(today = "2020-01-21", chapters) shouldBe LocalDate.parse("2020-01-23")
+    }
+
+    @Test
+    fun `follows releases on a day of the month`() = runTest {
+        val chapters = chaptersOn(
+            listOf("2019-09-15", "2019-10-15", "2019-11-14", "2019-12-15", "2020-01-16").map(LocalDate::parse),
+            perDay = 2,
+        )
+
+        nextUpdate(today = "2020-01-20", chapters) shouldBe LocalDate.parse("2020-02-15")
+    }
+
+    @Test
+    fun `follows releases on the last day of the month`() = runTest {
+        val chapters = chaptersOn(
+            listOf("2019-09-30", "2019-10-31", "2019-11-30", "2019-12-31", "2020-01-31").map(LocalDate::parse),
+            perDay = 2,
+        )
+
+        nextUpdate(today = "2020-02-05", chapters) shouldBe LocalDate.parse("2020-02-29")
+    }
+
+    @Test
+    fun `keeps checking on the expected day`() = runTest {
+        val chapters = chaptersOn(weekly(from = "2019-12-04", to = "2019-12-25"))
+
+        nextUpdate(today = "2020-01-01", chapters) shouldBe LocalDate.parse("2020-01-01")
+    }
+
+    @Test
+    fun `keeps a next update in the window while nothing new came out`() = runTest {
+        val chapters = chaptersOn(weekly(from = "2019-12-04", to = "2019-12-25"))
+        val manga = Manga.create().copy(nextUpdate = LocalDate.parse("2020-01-01").atStartOfDayIn(testTimeZone))
+
+        nextUpdate(today = "2020-01-02", chapters, manga) shouldBe LocalDate.parse("2020-01-01")
+    }
+
+    @Test
+    fun `counts from a release on the expected day`() = runTest {
+        val chapters = chaptersOn(weekly(from = "2019-12-04", to = "2020-01-01"))
+        val manga = Manga.create().copy(nextUpdate = LocalDate.parse("2020-01-01").atStartOfDayIn(testTimeZone))
+
+        nextUpdate(today = "2020-01-01", chapters, manga) shouldBe LocalDate.parse("2020-01-08")
+    }
+
+    @Test
+    fun `spreads out checks during a hiatus`() = runTest {
+        val chapters = chaptersOn(weekly(from = "2019-09-11", to = "2019-10-02"))
+
+        nextUpdate(today = "2020-01-01", chapters) shouldBe LocalDate.parse("2020-01-22")
+    }
+
+    @Test
+    fun `ignores chapters dated in the future`() = runTest {
+        val chapters = chaptersOn(weekly(from = "2019-12-04", to = "2019-12-25") + LocalDate.parse("2020-06-01"))
+
+        nextUpdate(today = "2020-01-01", chapters) shouldBe LocalDate.parse("2020-01-01")
+    }
+
+    private suspend fun nextUpdate(
+        today: String,
+        chapters: List<Chapter>,
+        manga: Manga = Manga.create(),
+    ): LocalDate {
+        val getChaptersByMangaId = mockk<GetChaptersByMangaId>()
+        coEvery { getChaptersByMangaId.await(any(), any()) } returns chapters
+        val fetchInterval = FetchInterval(getChaptersByMangaId)
+        val date = LocalDate.parse(today)
+        val update = fetchInterval.withFetchInterval(
+            manga = manga,
+            dateTime = date.atTime(10, 0),
+            timeZone = testTimeZone,
+            window = fetchInterval.getWindow(date, testTimeZone),
+        )
+        return update.nextUpdate!!.toLocalDateTime(testTimeZone).date
+    }
+
+    private fun weekly(from: String, to: String): List<LocalDate> {
+        return generateSequence(LocalDate.parse(from)) { it.plus(DatePeriod(days = 7)) }
+            .takeWhile { it <= LocalDate.parse(to) }
+            .toList()
+    }
+
+    private fun chaptersOn(days: List<LocalDate>, perDay: Int = 1): List<Chapter> {
+        return days.flatMap { day ->
+            val time = day.atStartOfDayIn(testTimeZone) + 12.hours
+            List(perDay) { chapter.copy(dateFetch = time, dateUpload = time) }
+        }
+    }
+
     private fun chapterWithTime(chapter: Chapter, duration: Duration): Chapter {
         val newTime = testTime.toInstant(testTimeZone) + duration
         return chapter.copy(dateFetch = newTime, dateUpload = newTime)
     }
 
     private fun List<Chapter>.lastUploadDate() =
-        last().dateUpload!! - Instant.fromEpochMilliseconds(0)
+        last().dateUpload!! - testTime.toInstant(testTimeZone)
 }