Skip to content

armgit / peach

Public Imported from GitHub
0

Peach: Local-first video metadata scraper and media manager.

feat(follow): 定时追更按来源活跃度放慢,限流跳过不记失败

longmeidao35345a69cc24
diff --git a/src/peach/follow_check.py b/src/peach/follow_check.py
index 9022b3201..91e5d102b 100644
--- a/src/peach/follow_check.py
+++ b/src/peach/follow_check.py
@@ -56,16 +56,26 @@ class CheckResult:
def plan_check(store, credentials, *, source_id: int | None = None,
older: bool = False,
- backfill_providers: frozenset[str] = frozenset()) -> list[dict]:
+ backfill_providers: frozenset[str] = frozenset(),
+ scheduled: timedelta | None = None,
+ moment: datetime | None = None) -> list[dict]:
"""要检查哪些来源,以及每条要不要绕过条件请求游标。只读,不联网。
`force_media_reparse` 单独在这里算好:凭据已经存在、但旧候选还标着
`media_needs_credential` 时,条件请求的 304 会让旧解析结果永久不变——显式检查
得无条件重取一次,凭据才真正生效。往回翻页时不算这个,那一页本来就没游标。
+
+ `scheduled` 是定时检查的基础间隔:给了就只留这一跳到期的来源(`due_for_check`),
+ 久没更新的按档放慢。手动检查不给,照旧全查。
"""
rows = [dict(row) for row in store.sources(enabled_only=True)
if (source_id is None or row["id"] == source_id)
and (not older or row["provider"] in backfill_providers)]
+ if scheduled is not None:
+ moment = moment or datetime.now(timezone.utc)
+ latest = {int(source): _instant(value) for source, value in store.latest_item_stamps()}
+ rows = [row for row in rows
+ if due_for_check(row, latest.get(int(row["id"])), scheduled, moment)]
for row in rows:
row["force_media_reparse"] = (
not older and credentials.load(row["provider"]) is not None
@@ -80,6 +90,43 @@ def plan_check(store, credentials, *, source_id: int | None = None,
return rows
+#: 定时检查按来源活跃度放慢(ADR-0132):最近一条距今不到这么久,就按基础间隔的这个倍数查。
+#: 基础间隔是设置里的「自动检查间隔」。越过最后一档的、一条都没有的,一天查一次。
+ACTIVITY_TIERS = ((timedelta(days=7), 1), (timedelta(days=30), 3), (timedelta(days=90), 12))
+DORMANT_INTERVAL = timedelta(days=1)
+
+
+def check_interval(base: timedelta, latest: datetime | None, moment: datetime) -> timedelta:
+ """最近一条条目是 `latest` 的来源,定时检查隔多久查一次。不会比基础间隔更密。"""
+ if latest is not None:
+ for within, factor in ACTIVITY_TIERS:
+ if moment - latest < within:
+ return max(base, min(base * factor, DORMANT_INTERVAL))
+ return max(base, DORMANT_INTERVAL)
+
+
+def due_for_check(row: Mapping, latest: datetime | None, base: timedelta,
+ moment: datetime) -> bool:
+ """定时检查这一跳要不要查这条来源。没查过的总要查。
+
+ 判据放宽半个基础间隔:调度每隔一个基础间隔跳一次,上次检查总比这一跳晚一点,
+ 卡死在「满一个间隔」上的话每档都会多等一跳。
+ """
+ checked = _instant(row["last_checked_at"])
+ if checked is None:
+ return True
+ return checked + check_interval(base, latest, moment) - base / 2 <= moment
+
+
+def _instant(value) -> datetime | None:
+ """账本里的 ISO 时间;认不出就当没有。"""
+ try:
+ parsed = datetime.fromisoformat(str(value or "").replace("Z", "+00:00"))
+ except ValueError:
+ return None
+ return parsed if parsed.tzinfo else parsed.replace(tzinfo=timezone.utc)
+
+
def build_connector_for(provider: str, credentials, connector_factory, *,
enrich_skip: frozenset[str] = frozenset()):
"""按凭据仓库里现有的凭据造一个连接器。
@@ -120,8 +167,9 @@ def run_check(row: Mapping, *, credentials, writer, connector_factory,
"label": str(row["label"] or ""), "page": page, "older": older}
remaining = cooldown.cooling(provider) if cooldown is not None else 0
if remaining:
- return CheckResult(**base, ok=False, status="error",
- error=f"{provider} 限流冷却中,{int(remaining) + 1} 秒后再试,本次跳过")
+ # 没问就不算这条来源失败:上次的结论与检查时间都不动,冷却一过下一跳照常到期。
+ return CheckResult(**base, status="skipped",
+ message=f"{provider} 限流冷却中,{int(remaining) + 1} 秒后再试,本次跳过")
force = bool(row.get("force_media_reparse")) or replay_first
cutoff, floor = _history_limits(row, metadata, writer, moment,
older=older, initial_days=initial_days)
diff --git a/src/peach/follow_store.py b/src/peach/follow_store.py
index 07f17aa07..eb65c497a 100644
--- a/src/peach/follow_store.py
+++ b/src/peach/follow_store.py
@@ -617,6 +617,16 @@ class FollowStore:
links.append({"service": service, "handle": handle, "url": source})
self.merge_source_metadata(source_id, {"official_links": links}, moment=moment)
+ def latest_item_stamps(self) -> list[tuple[int, str]]:
+ """每条来源最近一条条目的时间文本:发布时间,没有就用第一次见到的时间。
+
+ 定时检查按它判来源活不活跃(`follow_check.check_interval`)。一条都没有的来源不在结果里。
+ """
+ return [(int(row[0]), str(row[1])) for row in self._connect().execute(
+ "SELECT source_id, max(COALESCE(NULLIF(published_at,''), first_seen_at))"
+ " FROM follow_item GROUP BY source_id HAVING max(COALESCE(NULLIF(published_at,''),"
+ " first_seen_at)) IS NOT NULL")]
+
def enriched_external_ids(self, source_id: int, mark: str) -> frozenset[str]:
"""这条来源里细节已经补齐、不必再打详情页的条目 id。
diff --git a/src/peach/web_follow.py b/src/peach/web_follow.py
index 14a0e3257..9f0e0bb6c 100644
--- a/src/peach/web_follow.py
+++ b/src/peach/web_follow.py
@@ -18,6 +18,7 @@ import threading
import uuid
import urllib.parse
+from datetime import timedelta
from pathlib import Path
from typing import NamedTuple
@@ -2319,6 +2320,9 @@ def _check_payload(result) -> dict:
if not result.ok:
payload.update({"status": result.status, "error": result.error})
return payload
+ if result.status == "skipped":
+ payload.update({"deferred": True, "message": result.message})
+ return payload
if result.exhausted:
payload.update({"exhausted": True, "message": result.message})
return payload
@@ -2352,12 +2356,19 @@ class _FollowCheckSession:
self.writer = _check_writer(contract)
self.initial_days = follow_initial_days(contract.database)
self.cooldown = getattr(contract, 'follow_media_resolver', None)
+ #: 站点限流冷却中、这一轮没问的来源数。它们不进结果:没问就谈不上成败,
+ #: 下一跳照常到期(ADR-0132)。
+ self.deferred = 0
def checkpoint(self, **fields):
if self.job_id and not self.contract.follow_job.checkpoint(self.job_id, **fields):
raise InterruptedError('关注检查已停止')
def remember(self, row, result, *, finished):
+ if result.get('deferred'):
+ self.deferred += 1
+ self.checkpoint(deferred=self.deferred)
+ return
if result.get('ok') and isinstance(result.get('page'), int):
self.pages[str(row['id'])] = result['page']
if finished:
@@ -2408,10 +2419,17 @@ def _run_follow_check(contract, body, job_id=None) -> dict:
# 不会因为重走而倒退。补齐标记缺键的行在重走时各自重探一次详情。
rewind = bool(body.get("rewind"))
credentials = _credential_store(contract)
+ # 定时那一跳只查到期的来源,久没更新的按档放慢(ADR-0132);点名的、往回翻的、
+ # 手动「检查全部」和续跑冻结下来的那一批都不按到期筛。
+ scheduled = None
+ if (body.get("automatic") and source_id is None and "sources" not in body and not older
+ and 'source_ids' not in previous):
+ scheduler = getattr(contract, "follow_scheduler", None)
+ scheduled = timedelta(minutes=scheduler.config.interval_minutes if scheduler else 60)
with contract.database.read_connection() as connection:
rows = plan_check(_store(contract, connection), credentials,
source_id=source_id, older=older,
- backfill_providers=_BACKFILL_PROVIDERS)
+ backfill_providers=_BACKFILL_PROVIDERS, scheduled=scheduled)
if "sources" in body:
rows = [row for row in rows if row["id"] in body["sources"]]
if 'source_ids' in previous:
@@ -2433,7 +2451,8 @@ def _run_follow_check(contract, body, job_id=None) -> dict:
followups = follow_creators.declare(connection)
except sqlite3.Error:
followups = []
- return {"ok": True, "checked": len(session.results), "results": session.results, "followups": followups}
+ return {"ok": True, "checked": len(session.results), "results": session.results,
+ "deferred": session.deferred, "followups": followups}
def q_follow_schedule(contract, _args) -> dict:
diff --git a/tests/test_follow_check.py b/tests/test_follow_check.py
index e4648aafb..a2a8ed2ae 100644
--- a/tests/test_follow_check.py
+++ b/tests/test_follow_check.py
@@ -322,6 +322,44 @@ class PlanCheckTests(_CheckCase):
frozenset())
+class ScheduledCheckTests(_CheckCase):
+ """定时检查按来源活跃度放慢;手动检查照旧全查(ADR-0132)。"""
+
+ def _checked_with_latest(self, ref, age):
+ source_id = self._register(ref=ref, url=f"https://{ref}.fanbox.cc/")
+ published = (MOMENT - age).isoformat().replace("+00:00", "Z")
+ self.store.record(source_id, _fetch(ref=ref, candidates=(
+ FollowCandidate(provider="fanbox", external_id=ref, title=ref,
+ published_at=published),)), moment=MOMENT)
+ return source_id
+
+ def _due(self, after):
+ return {row["id"] for row in plan_check(self.store, _Credentials(),
+ scheduled=timedelta(hours=1),
+ moment=MOMENT + after)}
+
+ def test_sources_that_stopped_updating_are_checked_less_often(self):
+ active = self._checked_with_latest("active", timedelta(days=2))
+ monthly = self._checked_with_latest("monthly", timedelta(days=20))
+ quarterly = self._checked_with_latest("quarterly", timedelta(days=60))
+ dormant = self._checked_with_latest("dormant", timedelta(days=200))
+ empty = self._register(ref="empty", url="https://empty.fanbox.cc/")
+ self.store.record(empty, _fetch(ref="empty"), moment=MOMENT)
+ never = self._register(ref="never", url="https://never.fanbox.cc/")
+ self.assertEqual(self._due(timedelta(hours=1)), {active, never})
+ self.assertEqual(self._due(timedelta(hours=3)), {active, monthly, never})
+ self.assertEqual(self._due(timedelta(hours=12)), {active, monthly, quarterly, never})
+ self.assertEqual(self._due(timedelta(hours=24)),
+ {active, monthly, quarterly, dormant, empty, never})
+ self.assertEqual(len(plan_check(self.store, _Credentials())), 6)
+
+ def test_a_tick_that_lands_just_before_the_interval_still_checks(self):
+ """调度一跳一个间隔,上次检查总比这一跳晚一点;判据放宽半个间隔,不让它多等一跳。"""
+ active = self._checked_with_latest("active", timedelta(days=2))
+ self.assertEqual(self._due(timedelta(minutes=58)), {active})
+ self.assertEqual(self._due(timedelta(minutes=20)), set())
+
+
class RunCheckTests(_CheckCase):
def test_a_normal_check_reads_the_first_page_with_the_stored_cursors(self):
source_id = self._register()
@@ -388,8 +426,9 @@ class RunCheckTests(_CheckCase):
self.assertEqual(self._row(first)["last_status"], "error")
connector = _Connector(_fetch(ref="another"))
skipped = self._run(second, connector, cooldown=cooldown)
- self.assertFalse(skipped.ok)
- self.assertIn("冷却", skipped.error)
+ self.assertTrue(skipped.ok)
+ self.assertEqual(skipped.status, "skipped")
+ self.assertIn("冷却", skipped.message)
self.assertEqual(connector.calls, [])
self.assertIsNone(self._row(second)["last_status"])
clock[0] += 121
diff --git a/tests/test_follow_web.py b/tests/test_follow_web.py
index f52be6723..dd697c1c1 100644
--- a/tests/test_follow_web.py
+++ b/tests/test_follow_web.py
@@ -174,6 +174,32 @@ class FollowContractTests(unittest.TestCase):
self.assertEqual([row['source'] for row in result['results']], [selected])
self.assertEqual(factory.call_count, 1)
+ def test_sources_skipped_for_a_site_cooldown_are_not_reported_as_failures(self):
+ """站点限流冷却中:同站的来源这一轮不问,也不进失败清单(ADR-0132)。"""
+ from peach.follow_stream import FollowMediaResolver
+ self._seed(ref='first')
+ self._seed(ref='second')
+ self.contract.follow_media_resolver = FollowMediaResolver(transport=None)
+ self.contract.follow_media_resolver.pause('rule34video', 60)
+ with mock.patch.object(web_follow, 'build_connector') as factory:
+ result = self._post('/api/follow/check', {})
+ self.assertEqual(factory.call_count, 0)
+ self.assertEqual((result['results'], result['deferred']), ([], 2))
+
+ def test_a_scheduled_round_checks_only_the_sources_that_are_due(self):
+ """定时那一跳跳过刚查过、又久未更新的来源;手动检查照旧全查。"""
+ stale = self._seed(ref='stale')
+ recent = self._seed(ref='recent')
+ with self.contract.database.write_transaction() as connection:
+ connection.execute("UPDATE follow_source SET last_checked_at=? WHERE id=?",
+ (datetime.now(timezone.utc).isoformat(), recent))
+ with mock.patch.object(web_follow, 'build_connector') as factory:
+ factory.return_value.fetch.side_effect = FollowSourceError('offline')
+ scheduled = self._post('/api/follow/check', {'automatic': True})
+ manual = self._post('/api/follow/check', {})
+ self.assertEqual([row['source'] for row in scheduled['results']], [stale])
+ self.assertEqual({row['source'] for row in manual['results']}, {stale, recent})
+
def test_failed_numeric_source_has_a_readable_author(self):
source = self._seed(ref='patreon/12387984', label='Pantsushi · patreon')
with mock.patch.object(web_follow, 'build_connector') as factory: