Peach: Local-first video metadata scraper and media manager.
feat(feeds): 订阅源按相位错峰拉取,来源冷却整轮只记一次
longmeidaob0a300ccca7d
diff --git a/src/peach/api.py b/src/peach/api.pyindex f966d315a..625d52826 100644--- a/src/peach/api.py+++ b/src/peach/api.py@@ -458,7 +458,8 @@ def create_app(contract.follow_scheduler = follow_scheduler# 订阅源拉取(ADR-0042)用同一个调度实现,只换 job id、状态文件与默认间隔。# 默认 6 小时:一位女优的 JavDB 演员页一天更新几条,比这更密只是把 JavDB 的配额- # 花在一张没变的页面上。+ # 花在一张没变的页面上。6 小时是每个源的周期;调度每半小时看一次谁到期,各源按+ # 自己的相位错开(ADR-0132)。feed_scheduler = FollowUpdateScheduler(settings.follow_state_root,lambda: web_feeds.w_feed_check(contract, {"automatic": True}),@@ -467,6 +468,7 @@ def create_app(filename="feed-schedule.json",default=FollowScheduleConfig(enabled=True,interval_minutes=feeds.DEFAULT_INTERVAL_MINUTES),+ tick_minutes=feeds.TICK_MINUTES,unavailable_message="订阅源拉取只在写入端可用",)contract.feed_scheduler = feed_schedulerdiff --git a/src/peach/feeds.py b/src/peach/feeds.pyindex 019b7fd35..1fd97f06b 100644--- a/src/peach/feeds.py+++ b/src/peach/feeds.py@@ -30,6 +30,13 @@ KIND_LABELS = {KIND_JAVDB_ACTOR: "JAV 订阅"}#: 拉取间隔。JavDB 按出口 IP 计配额,一张一天才更新几条的页面拉得再密也只是花配额。DEFAULT_INTERVAL_MINUTES = 360+#: 定时拉取多久看一次有没有到期的源(ADR-0132)。每个源仍按自己的间隔拉,这一档只决定+#: 到期之后最多再等多久;源在间隔窗口里各占一个相位,一跳只拉落在这半小时里的那几个。+TICK_MINUTES = 30++#: 相位散列用的黄金比例小数部分:连续的 id 落在窗口里也是均匀铺开的,几个源时也不扎堆。+_GOLDEN = 0.6180339887498949+#: 一次拉取最多解析多少条。源正常一页几十条,异常的那种(被替换成别的页面、#: 被镜像站塞进整站索引)会给出几千条,那不该变成几千个壳。MAX_ENTRIES = 200@@ -328,31 +335,56 @@ def sources(connection: sqlite3.Connection) -> list[dict]:def due_sources(connection: sqlite3.Connection, now: str | None = None) -> list[sqlite3.Row]:- """到期该拉的源。从没拉过的(`next_fetch_at` 为空)总是到期。"""+ """到期该拉的源,等得最久的在前。从没拉过的(`next_fetch_at` 为空)总是到期,排最前。"""moment = now or stamp()return list(connection.execute("SELECT * FROM feed_source WHERE enabled=1"- " AND (next_fetch_at IS NULL OR next_fetch_at<=?) ORDER BY id", (moment,)))+ " AND (next_fetch_at IS NULL OR next_fetch_at<=?) ORDER BY next_fetch_at, id", (moment,)))+++def next_fetch(source_id: int, moment: datetime, interval_minutes: int) -> datetime:+ """这个源下一次该拉的时刻:「现在 + 间隔」附近、落在它自己相位上的那一刻(ADR-0132)。++ 不直接取「现在 + 间隔」:调度每跳之间隔着 `TICK_MINUTES`,拉完那一刻总比到期晚一点,+ 按它往后排会一轮轮漂,同一时刻加的源也永远在同一跳一起到期。相位由 id 定,过渡那一轮+ 间隔落在半个到一个半间隔之间,之后每次都回到同一个相位。+ """+ period = max(1, int(interval_minutes)) * 60+ phase = (int(source_id) * _GOLDEN) % 1 * period+ base = moment + timedelta(seconds=period)+ offset = (phase - base.timestamp()) % period+ if offset > period / 2:+ offset -= period+ return base + timedelta(seconds=offset)def settle(connection: sqlite3.Connection, source_id: int, *, error: str | None = None,etag: str | None = None, last_modified: str | None = None,interval_minutes: int, seen: int = 0, new: int = 0,- page_name: str | None = None, now: datetime | None = None) -> None:+ page_name: str | None = None, now: datetime | None = None,+ retry_at: datetime | None = None) -> None:"""结算一次拉取。`page_name` 是这一页在站上挂的名字,解不出时留着上次的。失败也按同一间隔排下一次,不做指数退避——退避要么在这里写第二套判据,要么让一个临时挡回来的源沉默半天。下次时间在所有返回路径上都写,任何一种结局都不会让源卡住。+ `retry_at` 是来源冷却的结束时刻:被整站挡回来的不是这个源的错,冷却一过就该再拉。"""moment = now or datetime.now(timezone.utc)+ upcoming = retry_at or next_fetch(source_id, moment, interval_minutes)connection.execute("UPDATE feed_source SET last_error=?,last_fetched_at=?,next_fetch_at=?,""etag=COALESCE(?,etag),last_modified=COALESCE(?,last_modified),""name=COALESCE(?,name),last_seen_count=?,last_new_count=? WHERE id=?",- (error, stamp(moment), stamp(moment + timedelta(minutes=interval_minutes)),+ (error, stamp(moment), stamp(upcoming),etag, last_modified, page_name or None, int(seen), int(new), int(source_id)))+def defer(connection: sqlite3.Connection, source_id: int, until: datetime) -> None:+ """这一轮没轮到问就撞上了来源冷却:只把下次时间挪到冷却结束,上次的结论原样留着。"""+ connection.execute("UPDATE feed_source SET next_fetch_at=? WHERE id=?",+ (stamp(until), int(source_id)))++# -- 条目与壳 --------------------------------------------------------------diff --git a/src/peach/follow_scheduler.py b/src/peach/follow_scheduler.pyindex 9ac63aded..29154ce46 100644--- a/src/peach/follow_scheduler.py+++ b/src/peach/follow_scheduler.py@@ -88,9 +88,13 @@ class FollowUpdateScheduler:def __init__(self, state_root: Path, run_check: Callable[[], dict], *, available: bool,job_id: str = JOB_ID, filename: str = "follow-schedule.json",default: FollowScheduleConfig | None = None,+ tick_minutes: int | None = None,unavailable_message: str ="automatic follow updates are available only on the ledger writer"):self.job_id = job_id+ #: 多久跑一次。不给就等于设置里的间隔;订阅源给的是 `feeds.TICK_MINUTES`:+ #: 那边设置里的间隔是每个源的周期,到期由源自己判(ADR-0132)。+ self.tick_minutes = tick_minutesself.unavailable_message = unavailable_messageself.store = FollowScheduleStore(state_root, filename, default)self.run_check = run_check@@ -156,12 +160,17 @@ class FollowUpdateScheduler:raise ValueError("自动运行没有开启") from Nonereturn self.status()+ @property+ def tick(self) -> int:+ """调度任务的实际间隔(分钟)。"""+ return self.tick_minutes or self.config.interval_minutes+def _apply_job(self) -> None:if not self.config.enabled:if self.scheduler.get_job(self.job_id) is not None:self.scheduler.remove_job(self.job_id)return- interval = self.config.interval_minutes+ interval = self.tickself.scheduler.add_job(self._run,"interval",@@ -181,9 +190,9 @@ class FollowUpdateScheduler:到 `MAX_BACKOFF_FACTOR` 倍为止,且不超过允许的最大间隔。"""if failures <= 1:- return self.config.interval_minutes+ return self.tickfactor = min(2 ** (failures - 1), MAX_BACKOFF_FACTOR)- return min(self.config.interval_minutes * factor, MAX_INTERVAL_MINUTES)+ return min(self.tick * factor, MAX_INTERVAL_MINUTES)def _run(self) -> None:with self._state_lock:@@ -248,7 +257,7 @@ class FollowUpdateScheduler:if not failed or not self._started:returndelay = self.backoff_minutes(failures)- if delay <= self.config.interval_minutes:+ if delay <= self.tick:returntry:self.scheduler.modify_job(diff --git a/src/peach/web_feeds.py b/src/peach/web_feeds.pyindex 8020b5832..df10ec17b 100644--- a/src/peach/web_feeds.py+++ b/src/peach/web_feeds.py@@ -11,7 +11,7 @@ import reimport threadingimport timeimport uuid-from datetime import timedelta+from datetime import datetime, timedelta, timezonefrom urllib.parse import quotefrom . import (@@ -33,9 +33,30 @@ FETCH_MAX_BYTES = 2 * 1024 * 1024#: 一轮里最多拉几个源。源多的时候分几轮拉完,而不是让一轮跑到限流上去。MAX_SOURCES_PER_RUN = 20+#: 定时那几轮多久派一次存量后继:旧壳重试、补封面、缺头像的存量女优与想要作品重试+#: (ADR-0132)。定时拉取每 `feeds.TICK_MINUTES` 一跳,存量跟着每跳派一次就成了原来的十几倍;+#: 本轮新发现的番号不受它管,照旧当场派取资料。+STOCK_INTERVAL = timedelta(hours=6)+_NOT_MODIFIED = 304+def _cooling_until(contract, url: str) -> datetime | None:+ """这个地址所属的来源还在冷却就返回冷却结束的时刻(UTC),否则 None。"""+ from .scraping_access import paused_until, source_for++ source = source_for(url)+ until = paused_until(contract.follow_secrets_root, source) if source else 0.0+ return datetime.fromtimestamp(until, timezone.utc) if until else None+++def _paused_by_source(error: Exception) -> bool:+ """来源冷却挡下的这一次;本趟预算耗尽(`SourceBudgetExhausted`)不算来源冷却。"""+ from .scraping_access import SourceBudgetExhausted, SourcePaused++ return isinstance(error, SourcePaused) and not isinstance(error, SourceBudgetExhausted)++def _parse_source(kind: str, body: bytes, url: str):"""演员页受登录限制时返回可操作的原因。"""if kind == feeds.KIND_JAVDB_ACTOR and javdb.LOGIN.search(body.decode("utf-8", "replace")):@@ -85,9 +106,13 @@ def poll_source(contract, transport, source) -> dict:last_modified=source["last_modified"])except Exception as error: # noqa: BLE001 - 原因写到源那一行上给人看report.update(ok=False, error=str(error))+ # 整站冷却挡回来的,冷却一过就重拉,不跟着间隔再等一整轮。+ retry_at = _cooling_until(contract, source["url"]) if _paused_by_source(error) else None+ if retry_at is not None:+ report["paused"] = Truewith contract.database.write_transaction() as connection:feeds.settle(connection, source_id, error=str(error),- interval_minutes=interval)+ interval_minutes=interval, retry_at=retry_at)return reportif response.status == _NOT_MODIFIED:with contract.database.write_transaction() as connection:@@ -189,10 +214,19 @@ def _execute_check(contract, body, job_id: str) -> dict:codes = [code for row in results for code in row.get('codes') or []]if not contract.feed_job.checkpoint(job_id, source_ids=[int(row['id']) for row in rows]):raise InterruptedError('订阅检查已停止')+ deferred = 0try:for index, source in enumerate(rows):if int(source['id']) in completed:continue+ until = _cooling_until(contract, source["url"])+ if until is not None:+ pending = [row for row in rows[index:] if int(row['id']) not in completed]+ results, deferred = _defer_round(contract, pending, until, results)+ if not contract.feed_job.checkpoint(job_id, results=results, checked=len(results),+ total=len(rows), added=len(set(codes))):+ raise InterruptedError('订阅检查已停止')+ breakif not contract.feed_job.update(job_id, checked=len(results), total=len(rows),current=source["name"] or source["url"]):raise InterruptedError('订阅检查已停止')@@ -210,31 +244,80 @@ def _execute_check(contract, body, job_id: str) -> dict:contract.cache_bust()# 取资料:新发现的先排,没取到资料的旧壳留着保底名额,同一批一条后继。只缺封面的另成一批# 补封面。定时那一轮只带到了重试时间的,手动点的那一轮全带;已在队里的番号都不再带。- retry_after = feed_followup.RETRY_AFTER if body.get("automatic") else timedelta(0)+ # 存量部分(旧壳、封面、头像)定时那几轮隔 `STOCK_INTERVAL` 才派一次,新番号每轮都派。+ automatic = bool(body.get("automatic"))+ stock = _stock_due(contract) if automatic else True+ retry_after = feed_followup.RETRY_AFTER if automatic else timedelta(0)hidden = web_settings.hidden_compilations(contract.database)busy = feed_followup.in_flight(contract.task_runs)fresh = [code for code in dict.fromkeys(codes) if normalise_code_key(code) not in busy]with contract.database.read_connection() as connection:- batch = feed_followup.scrape_batch(- connection, fresh, exclude=[*codes, *busy], retry_after=retry_after, hidden=hidden)- covers = feed_followup.cover_backlog(- connection, contract.cover_root, exclude=[*codes, *batch, *busy],- retry_after=retry_after, hidden=hidden)- planned = feed_followup.plan(batch) + feed_followup.plan_covers(covers)- # 余下的名额给缺头像的存量女优(ADR-0097)。资料处理收尾也派这一份,但没有新片- # 入库的日子它一轮都不跑,存量就一直停在原地;定时拉取每天都跑。- planned += avatar_followup.stock(- connection, contract.avatar_root, Attempts(attempts_root(contract.candidate_root)),- limit=MAX_FOLLOWUPS - len(planned))- return {"ok": True, "checked": len(rows), "total": len(rows), "results": results,- "added": len(codes),+ if not stock:+ planned = feed_followup.plan(fresh[:feed_followup.MAX_BATCH])+ else:+ batch = feed_followup.scrape_batch(+ connection, fresh, exclude=[*codes, *busy], retry_after=retry_after, hidden=hidden)+ covers = feed_followup.cover_backlog(+ connection, contract.cover_root, exclude=[*codes, *batch, *busy],+ retry_after=retry_after, hidden=hidden)+ planned = feed_followup.plan(batch) + feed_followup.plan_covers(covers)+ # 余下的名额给缺头像的存量女优(ADR-0097)。资料处理收尾也派这一份,但没有新片+ # 入库的日子它一轮都不跑,存量就一直停在原地;定时拉取每天都跑。+ planned += avatar_followup.stock(+ connection, contract.avatar_root, Attempts(attempts_root(contract.candidate_root)),+ limit=MAX_FOLLOWUPS - len(planned))+ if stock:+ contract.feed_stock_planned_at = time.monotonic()+ return {"ok": True, "checked": len(results), "total": len(rows), "results": results,+ "added": len(codes), "deferred": deferred, "stock": stock,# 后继由结果声明,调度端统一派(ADR-0040):这里只把清单交出去。"followups": [dict(key=item.key, task_key=item.task_key, label=item.label)for item in planned]}+def _defer_round(contract, pending, until: datetime, results: list) -> tuple[list, int]:+ """来源在冷却:这一轮剩下的源都顺延到冷却结束,整轮只记一条失败(ADR-0132)。++ 冷却是整个来源的状态,不是哪个订阅的毛病;每个源各记一条「正在冷却」只会把真正的+ 原因淹掉。本轮撞上 403 的那一行已经写了原因,就在它后面补一句顺延了几个;本轮一开始+ 来源就在冷却(别的任务撞出来的),就用第一个源那一行说明,它的上次结论不改。+ """+ with contract.database.write_transaction() as connection:+ for row in pending:+ feeds.defer(connection, int(row["id"]), until)+ results = list(results)+ trigger = next((row for row in reversed(results) if row.get("paused")), None)+ rest = len(pending)+ if trigger is None:+ from .scraping_access import SOURCES, source_for+ first = pending[0]+ label = SOURCES.get(source_for(first["url"]) or "", {}).get("label", "来源")+ trigger = {"source": int(first["id"]), "name": first["name"] or first["url"],+ "ok": False, "seen": 0, "added": 0, "without_code": 0, "paused": True,+ "error": f"{label} 正在冷却,这一轮没有拉"}+ results.append(trigger)+ rest -= 1+ if rest:+ trigger["error"] = f"{trigger['error']};其余 {rest} 个源顺延到冷却结束后再拉"+ return results, len(pending)+++def _stock_due(contract) -> bool:+ """定时这一轮该不该派存量后继:本进程里还没派过,或上次派过已经满 `STOCK_INTERVAL`。"""+ last = getattr(contract, "feed_stock_planned_at", None)+ return last is None or time.monotonic() - last >= STOCK_INTERVAL.total_seconds()++def w_feed_check(contract, body) -> dict:- """拉一轮订阅。不带参数只拉到期的源,`all` 拉全部启用的源。"""+ """拉一轮订阅。不带参数只拉到期的源,`all` 拉全部启用的源。++ 定时那一跳没有到期的源就什么也不起:每半小时一跳(`feeds.TICK_MINUTES`),任务中心+ 不该每半小时多一行空跑。+ """+ if body.get("automatic") and "sources" not in body and not body.get("all"):+ with contract.database.read_connection() as connection:+ if not feeds.due_sources(connection):+ return {"ok": True, "idle": True, "checked": 0, "total": 0, "results": []}finished = threading.Event()request_id = uuid.uuid4().hex@@ -243,8 +326,9 @@ def w_feed_check(contract, body) -> dict:result = _execute_check(contract, body, job_id)contract.feed_job.update(job_id, **result, status="complete",current=None, completed_at=time.time())- # 「想要」里取不到资料或封面的,跟着每一轮拉取再试(隔 `web_wants.RETRY_AFTER`)。- web_wants.start_scrape(contract, trigger="scheduled" if body.get("automatic") else "manual")+ # 「想要」里取不到资料或封面的,跟着派存量的那几轮再试(隔 `web_wants.RETRY_AFTER`)。+ if result["stock"]:+ web_wants.start_scrape(contract, trigger="scheduled" if body.get("automatic") else "manual")finally:finished.set()diff --git a/src/peach/web_state.py b/src/peach/web_state.pyindex 3c87a2f65..e85c1f2d1 100644--- a/src/peach/web_state.py+++ b/src/peach/web_state.py@@ -223,6 +223,9 @@ class WebContract:#: 订阅源的定时拉取(ADR-0042)。与追更共用 `follow_scheduler` 那一个实现,#: 只是各占一个 job id 与一份状态文件;由 `api` 在装配时接上。self.feed_scheduler = None+ #: 定时拉取上一次派存量后继的时刻(`time.monotonic()`);None 是本进程还没派过+ #: (`web_feeds.STOCK_INTERVAL`)。+ self.feed_stock_planned_at = None#: 云下载(`downloads.DownloadService`),由 `api` 在装配时接上;没接上时#: `/api/downloads` 回「云下载没有启用」。self.downloads = Nonediff --git a/tests/test_feeds.py b/tests/test_feeds.pyindex 62600dbe8..6dd38b3cc 100644--- a/tests/test_feeds.py+++ b/tests/test_feeds.py@@ -4,6 +4,7 @@ from __future__ import annotationsimport sysimport tempfileimport unittest+from datetime import datetime, timedelta, timezonefrom pathlib import Pathsys.path.insert(0, str(Path(__file__).resolve().parents[1] / "src"))@@ -162,6 +163,35 @@ class CompilationTest(unittest.TestCase):self.assertIsNone(feeds.compilation_kind("愛する妻が…", "篠田ゆう", "マドンナ"))+class FeedPhaseTest(unittest.TestCase):+ """下次拉取按源的相位排(ADR-0132):同时加的源错开,跳与跳之间的延迟不累积。"""++ MOMENT = datetime(2026, 11, 3, 3, 28, 57, tzinfo=timezone.utc)+ PERIOD = timedelta(minutes=feeds.DEFAULT_INTERVAL_MINUTES)++ def test_sources_fetched_together_spread_over_the_window(self):+ upcoming = sorted(feeds.next_fetch(source_id, self.MOMENT, feeds.DEFAULT_INTERVAL_MINUTES)+ for source_id in range(1, 12))+ for moment in upcoming:+ self.assertGreaterEqual(moment, self.MOMENT + self.PERIOD / 2)+ self.assertLessEqual(moment, self.MOMENT + self.PERIOD * 1.5)+ slots = {int((moment.timestamp() % self.PERIOD.total_seconds())+ // timedelta(minutes=feeds.TICK_MINUTES).total_seconds()) for moment in upcoming}+ # 11 个源落进 12 个半小时档里的至少 9 个:一跳最多拉两三个,而不是 11 个一起。+ self.assertGreaterEqual(len(slots), 9)++ def test_tick_lag_does_not_drift_the_schedule(self):+ """到期之后要等到下一跳才拉,拉完排的下次仍回到同一个相位,周期保持一个间隔。"""+ moment = self.MOMENT+ due = feeds.next_fetch(7, moment, feeds.DEFAULT_INTERVAL_MINUTES)+ for lag in (timedelta(minutes=29), timedelta(minutes=3), timedelta(minutes=17)):+ fetched = due + lag+ following = feeds.next_fetch(7, fetched, feeds.DEFAULT_INTERVAL_MINUTES)+ self.assertAlmostEqual((following - due).total_seconds(),+ self.PERIOD.total_seconds(), places=2)+ due = following++class FeedStoreTest(unittest.TestCase):def setUp(self):self._tmp = tempfile.TemporaryDirectory()@@ -278,6 +308,27 @@ class FeedStoreTest(unittest.TestCase):with self.database.read_connection() as connection:self.assertEqual(feeds.due_sources(connection), [])+ def test_the_longest_waiting_source_is_due_first(self):+ first = self._source(url="https://javdb.com/actors/a1")+ second = self._source(url="https://javdb.com/actors/a2")+ fresh = self._source(url="https://javdb.com/actors/a3")+ with self.database.write_transaction() as connection:+ feeds.defer(connection, first, datetime(2026, 11, 3, 2, tzinfo=timezone.utc))+ feeds.defer(connection, second, datetime(2026, 11, 3, 1, tzinfo=timezone.utc))+ with self.database.read_connection() as connection:+ due = [int(row["id"]) for row in feeds.due_sources(connection, "2026-11-03T03:00:00Z")]+ self.assertEqual(due, [fresh, second, first])++ def test_settle_after_a_cooldown_retries_at_its_end(self):+ source_id = self._source()+ until = datetime(2026, 11, 3, 3, 15, tzinfo=timezone.utc)+ with self.database.write_transaction() as connection:+ feeds.settle(connection, source_id, error="来源拒绝访问", interval_minutes=360,+ now=datetime(2026, 11, 3, 3, tzinfo=timezone.utc), retry_at=until)+ row = connection.execute("SELECT next_fetch_at FROM feed_source WHERE id=?",+ (source_id,)).fetchone()+ self.assertEqual(row[0], feeds.stamp(until))+def test_performer_names_match_existing_entities_only(self):with self.database.write_transaction() as connection:connection.execute(diff --git a/tests/test_feeds_web.py b/tests/test_feeds_web.pyindex a968dea60..4236103b3 100644--- a/tests/test_feeds_web.py+++ b/tests/test_feeds_web.py@@ -9,7 +9,7 @@ import tempfileimport threadingimport timeimport unittest-from datetime import timedelta+from datetime import datetime, timedelta, timezonefrom pathlib import Pathfrom types import SimpleNamespacefrom unittest.mock import patch@@ -80,6 +80,8 @@ class FeedWebFixture(unittest.TestCase):# assert 之后才写完,也可能在临时目录删掉之后才醒。这里关掉自动领取,# 要跑后继的用例自己 `drain()`。self.contract.followups.stop()+ # 来源冷却记录读写这里:每轮拉取前都要看 JavDB 是否在冷却,不能读到本机真实的那份。+ self.contract.follow_secrets_root = Path(self.temporary.name) / "secrets"self.url = "https://javdb.com/actors/Smp1"sample = Path(self.temporary.name) / "feed-sample.html"sample.write_text(SAMPLE, encoding="utf-8")@@ -141,6 +143,10 @@ class FeedWebFixture(unittest.TestCase):from peach.http import HttpRequestreturn transport(HttpRequest("GET", url, headers), 30.0, 1 << 21)+ def _stock_window_passes(self):+ """定时拉取派存量后继的窗口(`web_feeds.STOCK_INTERVAL`)已经过去。"""+ self.contract.feed_stock_planned_at = None+def _add(self, entity_id=None):with self.contract.database.write_transaction() as connection:return feeds.add_source(connection, url=self.url, name="示例源",@@ -423,6 +429,66 @@ class FeedWebTest(FeedWebFixture):self.assertEqual(row["last_error"], "来源回了 HTTP 503")self.assertIsNotNone(row["next_fetch_at"])+ def _sources_named(self, *names):+ """再加几个演员页源,各回一份同样的样本页。返回全部源 id,第一个是 `_add` 那个。"""+ ids = [self._add()]+ for name in names:+ url = f"https://javdb.com/actors/{name}"+ self.transport.responses[url] = self.transport.responses[self.url]+ with self.contract.database.write_transaction() as connection:+ ids.append(feeds.add_source(connection, url=url, name=name))+ return ids++ def _rows(self):+ with self.contract.database.read_connection() as connection:+ return {int(row["id"]): dict(row) for row in connection.execute("SELECT * FROM feed_source")}++ def test_a_refused_round_records_one_failure_and_defers_the_rest_to_the_cooldown_end(self):+ """一轮里撞上 403:只记撞上的那一条,其余源不再问、顺延到冷却结束(ADR-0132)。"""+ from peach import scraping_access+ first, *rest = self._sources_named("Smp2", "Smp3")+ served = self.transport++ def refused(request, timeout, max_bytes):+ served.requests.append(request)+ scraping_access.pause_source(self.contract.follow_secrets_root, "javdb", refused=True)+ raise scraping_access.SourcePaused("来源拒绝访问,暂停向它请求一段时间")++ self.transport = refused+ result = self._check()+ self.assertEqual(len(served.requests), 1)+ self.assertEqual(result["deferred"], 2)+ [row] = result["results"]+ self.assertFalse(row["ok"])+ self.assertIn("其余 2 个源顺延到冷却结束后再拉", row["error"])+ until = scraping_access.paused_until(self.contract.follow_secrets_root, "javdb")+ expected = feeds.stamp(datetime.fromtimestamp(until, timezone.utc))+ rows = self._rows()+ self.assertEqual({rows[source]["next_fetch_at"] for source in (first, *rest)}, {expected})+ self.assertTrue(all(rows[source]["last_fetched_at"] is None for source in rest))++ def test_a_round_that_starts_in_cooldown_asks_nothing(self):+ """别的任务撞出来的冷却还没过:这一轮一个请求都不发,整轮只说一句。"""+ from peach import scraping_access+ self._sources_named("Smp2")+ scraping_access.pause_source(self.contract.follow_secrets_root, "javdb", refused=True)+ result = self._check()+ self.assertEqual(self.transport.requests, [])+ [row] = result["results"]+ self.assertEqual(row["error"], "JavDB 正在冷却,这一轮没有拉;其余 1 个源顺延到冷却结束后再拉")+ self.assertTrue(all(row["last_fetched_at"] is None and row["last_error"] is None+ for row in self._rows().values()))++ def test_a_scheduled_tick_without_due_sources_starts_nothing(self):+ """定时每半小时一跳;没有到期的源时不起任务,任务中心里不留空跑的一行。"""+ self._add()+ self._check()+ runs = len(self.contract.task_runs.query(task_key="feed-check", limit=50))+ result = web_feeds.w_feed_check(self.contract, {"automatic": True})+ self.assertTrue(result["idle"])+ self.assertEqual(len(self.contract.task_runs.query(task_key="feed-check", limit=50)), runs)+ self.assertEqual(len(self.transport.requests), 1)+def test_an_empty_alias_page_settles_quietly_and_names_its_row(self):"""同一个人的第二页挂在旧艺名下、一部作品都没有:不算失败,行上说出是哪个名字。"""with self.contract.database.write_transaction() as connection:@@ -906,8 +972,10 @@ class FeedWebTest(FeedWebFixture):self._drain(1)self.transport.responses[self.url] = HttpResponse(304, {}, b"", self.url)# 定时那一轮:刚试过的不再排;已经有封面、资料也齐的那部永远不再排。+ self._stock_window_passes()self.assertEqual(self._check(automatic=True)["followups"], [])self._attempted("HMN-071", feed_followup.RETRY_AFTER.total_seconds() + 60)+ self._stock_window_passes()self.assertEqual([item["key"] for item in self._check(automatic=True)["followups"]],["feed-cover:HMN-071"])# 人说了不想看的,不再替它花配额。@@ -956,10 +1024,30 @@ class FeedWebTest(FeedWebFixture):with self.contract.database.read_connection() as connection:current = avatar_followup.fingerprint(connection, entity_id, providers_root)Attempts(attempts_root(self.contract.candidate_root)).record(key, current, "图库 6 张认不准")+ self._stock_window_passes()self.assertNotIn(key, [item["key"] for item in self._check(automatic=True)["followups"]])index_path.write_text(json.dumps({"Content": {"1-S1": {"二の宮すずか.jpg": "二の宮すずか.jpg"}}}), encoding="utf-8")+ self._stock_window_passes()self.assertIn(key, [item["key"] for item in self._check(automatic=True)["followups"]])+ def test_scheduled_rounds_carry_the_stock_only_once_per_window(self):+ """定时每半小时一跳:新番号每跳都派,存量(旧壳、封面、头像、想要)一个窗口只派一次。"""+ self.cover_misses.add("HMN-071")+ self._add()+ first = self._check(automatic=True)+ self.assertTrue(first["stock"])+ self.assertEqual([item["key"] for item in first["followups"]],+ ["feed-scrape:SSIS-950,HMN-071"])+ self._drain(1)+ self._attempted("HMN-071", feed_followup.RETRY_AFTER.total_seconds() + 60)+ self.transport.responses[self.url] = HttpResponse(304, {}, b"", self.url)+ quiet = self._check(automatic=True)+ self.assertFalse(quiet["stock"])+ self.assertEqual(quiet["followups"], [])+ self._stock_window_passes()+ self.assertEqual([item["key"] for item in self._check(automatic=True)["followups"]],+ ["feed-cover:HMN-071"])+def test_an_unknown_action_is_refused(self):with self.assertRaises(ValueError):dispatch_api_post(self.contract, "/api/feeds/discovery",diff --git a/tests/test_follow_scheduler.py b/tests/test_follow_scheduler.pyindex a85e931c7..8c458d17f 100644--- a/tests/test_follow_scheduler.py+++ b/tests/test_follow_scheduler.py@@ -196,6 +196,21 @@ class FollowUpdateSchedulerTests(unittest.TestCase):self.assertGreater(datetime.fromisoformat(next_run), planned,"连续失败后应当推后,而不是照原间隔继续敲上游")+ def test_a_tick_shorter_than_the_period_runs_often_and_backs_off_from_the_tick(self):+ """订阅源:设置里的间隔是每个源的周期,调度按更短的一跳看谁到期(ADR-0132)。"""+ with tempfile.TemporaryDirectory() as temporary:+ scheduler = FollowUpdateScheduler(+ Path(temporary), lambda: {"ok": True, "checked": 0, "results": []},+ available=True, default=FollowScheduleConfig(True, 360), tick_minutes=30)+ self.addCleanup(scheduler.stop)+ before = datetime.now(timezone.utc)+ scheduler.start()+ status = scheduler.status()+ self.assertEqual(status["interval_minutes"], 360)+ self.assertLess(datetime.fromisoformat(status["next_run_at"]),+ before + timedelta(minutes=31))+ self.assertEqual(scheduler.backoff_minutes(2), 60)+def test_reader_never_starts_or_accepts_schedule_changes(self):with tempfile.TemporaryDirectory() as temporary:scheduler = FollowUpdateScheduler(Path(temporary), lambda: {}, available=False)