Peach: Local-first video metadata scraper and media manager.
feat(follow): 定时追更按来源活跃度放慢,限流跳过不记失败
longmeidao35345a69cc24
diff --git a/src/peach/follow_check.py b/src/peach/follow_check.pyindex 9022b3201..91e5d102b 100644--- a/src/peach/follow_check.py+++ b/src/peach/follow_check.py@@ -56,16 +56,26 @@ class CheckResult:def plan_check(store, credentials, *, source_id: int | None = None,older: bool = False,- backfill_providers: frozenset[str] = frozenset()) -> list[dict]:+ backfill_providers: frozenset[str] = frozenset(),+ scheduled: timedelta | None = None,+ moment: datetime | None = None) -> list[dict]:"""要检查哪些来源,以及每条要不要绕过条件请求游标。只读,不联网。`force_media_reparse` 单独在这里算好:凭据已经存在、但旧候选还标着`media_needs_credential` 时,条件请求的 304 会让旧解析结果永久不变——显式检查得无条件重取一次,凭据才真正生效。往回翻页时不算这个,那一页本来就没游标。++ `scheduled` 是定时检查的基础间隔:给了就只留这一跳到期的来源(`due_for_check`),+ 久没更新的按档放慢。手动检查不给,照旧全查。"""rows = [dict(row) for row in store.sources(enabled_only=True)if (source_id is None or row["id"] == source_id)and (not older or row["provider"] in backfill_providers)]+ if scheduled is not None:+ moment = moment or datetime.now(timezone.utc)+ latest = {int(source): _instant(value) for source, value in store.latest_item_stamps()}+ rows = [row for row in rows+ if due_for_check(row, latest.get(int(row["id"])), scheduled, moment)]for row in rows:row["force_media_reparse"] = (not older and credentials.load(row["provider"]) is not None@@ -80,6 +90,43 @@ def plan_check(store, credentials, *, source_id: int | None = None,return rows+#: 定时检查按来源活跃度放慢(ADR-0132):最近一条距今不到这么久,就按基础间隔的这个倍数查。+#: 基础间隔是设置里的「自动检查间隔」。越过最后一档的、一条都没有的,一天查一次。+ACTIVITY_TIERS = ((timedelta(days=7), 1), (timedelta(days=30), 3), (timedelta(days=90), 12))+DORMANT_INTERVAL = timedelta(days=1)+++def check_interval(base: timedelta, latest: datetime | None, moment: datetime) -> timedelta:+ """最近一条条目是 `latest` 的来源,定时检查隔多久查一次。不会比基础间隔更密。"""+ if latest is not None:+ for within, factor in ACTIVITY_TIERS:+ if moment - latest < within:+ return max(base, min(base * factor, DORMANT_INTERVAL))+ return max(base, DORMANT_INTERVAL)+++def due_for_check(row: Mapping, latest: datetime | None, base: timedelta,+ moment: datetime) -> bool:+ """定时检查这一跳要不要查这条来源。没查过的总要查。++ 判据放宽半个基础间隔:调度每隔一个基础间隔跳一次,上次检查总比这一跳晚一点,+ 卡死在「满一个间隔」上的话每档都会多等一跳。+ """+ checked = _instant(row["last_checked_at"])+ if checked is None:+ return True+ return checked + check_interval(base, latest, moment) - base / 2 <= moment+++def _instant(value) -> datetime | None:+ """账本里的 ISO 时间;认不出就当没有。"""+ try:+ parsed = datetime.fromisoformat(str(value or "").replace("Z", "+00:00"))+ except ValueError:+ return None+ return parsed if parsed.tzinfo else parsed.replace(tzinfo=timezone.utc)++def build_connector_for(provider: str, credentials, connector_factory, *,enrich_skip: frozenset[str] = frozenset()):"""按凭据仓库里现有的凭据造一个连接器。@@ -120,8 +167,9 @@ def run_check(row: Mapping, *, credentials, writer, connector_factory,"label": str(row["label"] or ""), "page": page, "older": older}remaining = cooldown.cooling(provider) if cooldown is not None else 0if remaining:- return CheckResult(**base, ok=False, status="error",- error=f"{provider} 限流冷却中,{int(remaining) + 1} 秒后再试,本次跳过")+ # 没问就不算这条来源失败:上次的结论与检查时间都不动,冷却一过下一跳照常到期。+ return CheckResult(**base, status="skipped",+ message=f"{provider} 限流冷却中,{int(remaining) + 1} 秒后再试,本次跳过")force = bool(row.get("force_media_reparse")) or replay_firstcutoff, floor = _history_limits(row, metadata, writer, moment,older=older, initial_days=initial_days)diff --git a/src/peach/follow_store.py b/src/peach/follow_store.pyindex 07f17aa07..eb65c497a 100644--- a/src/peach/follow_store.py+++ b/src/peach/follow_store.py@@ -617,6 +617,16 @@ class FollowStore:links.append({"service": service, "handle": handle, "url": source})self.merge_source_metadata(source_id, {"official_links": links}, moment=moment)+ def latest_item_stamps(self) -> list[tuple[int, str]]:+ """每条来源最近一条条目的时间文本:发布时间,没有就用第一次见到的时间。++ 定时检查按它判来源活不活跃(`follow_check.check_interval`)。一条都没有的来源不在结果里。+ """+ return [(int(row[0]), str(row[1])) for row in self._connect().execute(+ "SELECT source_id, max(COALESCE(NULLIF(published_at,''), first_seen_at))"+ " FROM follow_item GROUP BY source_id HAVING max(COALESCE(NULLIF(published_at,''),"+ " first_seen_at)) IS NOT NULL")]+def enriched_external_ids(self, source_id: int, mark: str) -> frozenset[str]:"""这条来源里细节已经补齐、不必再打详情页的条目 id。diff --git a/src/peach/web_follow.py b/src/peach/web_follow.pyindex 14a0e3257..9f0e0bb6c 100644--- a/src/peach/web_follow.py+++ b/src/peach/web_follow.py@@ -18,6 +18,7 @@ import threadingimport uuidimport urllib.parse+from datetime import timedeltafrom pathlib import Pathfrom typing import NamedTuple@@ -2319,6 +2320,9 @@ def _check_payload(result) -> dict:if not result.ok:payload.update({"status": result.status, "error": result.error})return payload+ if result.status == "skipped":+ payload.update({"deferred": True, "message": result.message})+ return payloadif result.exhausted:payload.update({"exhausted": True, "message": result.message})return payload@@ -2352,12 +2356,19 @@ class _FollowCheckSession:self.writer = _check_writer(contract)self.initial_days = follow_initial_days(contract.database)self.cooldown = getattr(contract, 'follow_media_resolver', None)+ #: 站点限流冷却中、这一轮没问的来源数。它们不进结果:没问就谈不上成败,+ #: 下一跳照常到期(ADR-0132)。+ self.deferred = 0def checkpoint(self, **fields):if self.job_id and not self.contract.follow_job.checkpoint(self.job_id, **fields):raise InterruptedError('关注检查已停止')def remember(self, row, result, *, finished):+ if result.get('deferred'):+ self.deferred += 1+ self.checkpoint(deferred=self.deferred)+ returnif result.get('ok') and isinstance(result.get('page'), int):self.pages[str(row['id'])] = result['page']if finished:@@ -2408,10 +2419,17 @@ def _run_follow_check(contract, body, job_id=None) -> dict:# 不会因为重走而倒退。补齐标记缺键的行在重走时各自重探一次详情。rewind = bool(body.get("rewind"))credentials = _credential_store(contract)+ # 定时那一跳只查到期的来源,久没更新的按档放慢(ADR-0132);点名的、往回翻的、+ # 手动「检查全部」和续跑冻结下来的那一批都不按到期筛。+ scheduled = None+ if (body.get("automatic") and source_id is None and "sources" not in body and not older+ and 'source_ids' not in previous):+ scheduler = getattr(contract, "follow_scheduler", None)+ scheduled = timedelta(minutes=scheduler.config.interval_minutes if scheduler else 60)with contract.database.read_connection() as connection:rows = plan_check(_store(contract, connection), credentials,source_id=source_id, older=older,- backfill_providers=_BACKFILL_PROVIDERS)+ backfill_providers=_BACKFILL_PROVIDERS, scheduled=scheduled)if "sources" in body:rows = [row for row in rows if row["id"] in body["sources"]]if 'source_ids' in previous:@@ -2433,7 +2451,8 @@ def _run_follow_check(contract, body, job_id=None) -> dict:followups = follow_creators.declare(connection)except sqlite3.Error:followups = []- return {"ok": True, "checked": len(session.results), "results": session.results, "followups": followups}+ return {"ok": True, "checked": len(session.results), "results": session.results,+ "deferred": session.deferred, "followups": followups}def q_follow_schedule(contract, _args) -> dict:diff --git a/tests/test_follow_check.py b/tests/test_follow_check.pyindex e4648aafb..a2a8ed2ae 100644--- a/tests/test_follow_check.py+++ b/tests/test_follow_check.py@@ -322,6 +322,44 @@ class PlanCheckTests(_CheckCase):frozenset())+class ScheduledCheckTests(_CheckCase):+ """定时检查按来源活跃度放慢;手动检查照旧全查(ADR-0132)。"""++ def _checked_with_latest(self, ref, age):+ source_id = self._register(ref=ref, url=f"https://{ref}.fanbox.cc/")+ published = (MOMENT - age).isoformat().replace("+00:00", "Z")+ self.store.record(source_id, _fetch(ref=ref, candidates=(+ FollowCandidate(provider="fanbox", external_id=ref, title=ref,+ published_at=published),)), moment=MOMENT)+ return source_id++ def _due(self, after):+ return {row["id"] for row in plan_check(self.store, _Credentials(),+ scheduled=timedelta(hours=1),+ moment=MOMENT + after)}++ def test_sources_that_stopped_updating_are_checked_less_often(self):+ active = self._checked_with_latest("active", timedelta(days=2))+ monthly = self._checked_with_latest("monthly", timedelta(days=20))+ quarterly = self._checked_with_latest("quarterly", timedelta(days=60))+ dormant = self._checked_with_latest("dormant", timedelta(days=200))+ empty = self._register(ref="empty", url="https://empty.fanbox.cc/")+ self.store.record(empty, _fetch(ref="empty"), moment=MOMENT)+ never = self._register(ref="never", url="https://never.fanbox.cc/")+ self.assertEqual(self._due(timedelta(hours=1)), {active, never})+ self.assertEqual(self._due(timedelta(hours=3)), {active, monthly, never})+ self.assertEqual(self._due(timedelta(hours=12)), {active, monthly, quarterly, never})+ self.assertEqual(self._due(timedelta(hours=24)),+ {active, monthly, quarterly, dormant, empty, never})+ self.assertEqual(len(plan_check(self.store, _Credentials())), 6)++ def test_a_tick_that_lands_just_before_the_interval_still_checks(self):+ """调度一跳一个间隔,上次检查总比这一跳晚一点;判据放宽半个间隔,不让它多等一跳。"""+ active = self._checked_with_latest("active", timedelta(days=2))+ self.assertEqual(self._due(timedelta(minutes=58)), {active})+ self.assertEqual(self._due(timedelta(minutes=20)), set())++class RunCheckTests(_CheckCase):def test_a_normal_check_reads_the_first_page_with_the_stored_cursors(self):source_id = self._register()@@ -388,8 +426,9 @@ class RunCheckTests(_CheckCase):self.assertEqual(self._row(first)["last_status"], "error")connector = _Connector(_fetch(ref="another"))skipped = self._run(second, connector, cooldown=cooldown)- self.assertFalse(skipped.ok)- self.assertIn("冷却", skipped.error)+ self.assertTrue(skipped.ok)+ self.assertEqual(skipped.status, "skipped")+ self.assertIn("冷却", skipped.message)self.assertEqual(connector.calls, [])self.assertIsNone(self._row(second)["last_status"])clock[0] += 121diff --git a/tests/test_follow_web.py b/tests/test_follow_web.pyindex f52be6723..dd697c1c1 100644--- a/tests/test_follow_web.py+++ b/tests/test_follow_web.py@@ -174,6 +174,32 @@ class FollowContractTests(unittest.TestCase):self.assertEqual([row['source'] for row in result['results']], [selected])self.assertEqual(factory.call_count, 1)+ def test_sources_skipped_for_a_site_cooldown_are_not_reported_as_failures(self):+ """站点限流冷却中:同站的来源这一轮不问,也不进失败清单(ADR-0132)。"""+ from peach.follow_stream import FollowMediaResolver+ self._seed(ref='first')+ self._seed(ref='second')+ self.contract.follow_media_resolver = FollowMediaResolver(transport=None)+ self.contract.follow_media_resolver.pause('rule34video', 60)+ with mock.patch.object(web_follow, 'build_connector') as factory:+ result = self._post('/api/follow/check', {})+ self.assertEqual(factory.call_count, 0)+ self.assertEqual((result['results'], result['deferred']), ([], 2))++ def test_a_scheduled_round_checks_only_the_sources_that_are_due(self):+ """定时那一跳跳过刚查过、又久未更新的来源;手动检查照旧全查。"""+ stale = self._seed(ref='stale')+ recent = self._seed(ref='recent')+ with self.contract.database.write_transaction() as connection:+ connection.execute("UPDATE follow_source SET last_checked_at=? WHERE id=?",+ (datetime.now(timezone.utc).isoformat(), recent))+ with mock.patch.object(web_follow, 'build_connector') as factory:+ factory.return_value.fetch.side_effect = FollowSourceError('offline')+ scheduled = self._post('/api/follow/check', {'automatic': True})+ manual = self._post('/api/follow/check', {})+ self.assertEqual([row['source'] for row in scheduled['results']], [stale])+ self.assertEqual({row['source'] for row in manual['results']}, {stale, recent})+def test_failed_numeric_source_has_a_readable_author(self):source = self._seed(ref='patreon/12387984', label='Pantsushi · patreon')with mock.patch.object(web_follow, 'build_connector') as factory: