Peach: Local-first video metadata scraper and media manager.
fix(feeds): 来源冷却只顺延同一来源的订阅,DMM 冷却也按结束时刻重拉
longmeidao7ffd2bdb2c7a
diff --git a/docs/adr/0135-dmm-actress-catalogue-as-second-feed-source.md b/docs/adr/0135-dmm-actress-catalogue-as-second-feed-source.mdindex 2f9215db6..79fef6cb3 100644--- a/docs/adr/0135-dmm-actress-catalogue-as-second-feed-source.md+++ b/docs/adr/0135-dmm-actress-catalogue-as-second-feed-source.md@@ -63,6 +63,8 @@ provider 为 `dmm`、`r18dev`,`external_kind='performer'`,纯数字的编号**七、请求、限流与失败分类沿用刮削链。** 走同一条 `SourceTransport` 加主机限流,包在 `SiteSource.holding` 里:验证页、封禁、地区限制与限流让 DMM 整站进冷却;任何失败按原因写进那一行的 `last_error`,下次时间照写。+[ADR-0134](0134-feed-and-follow-checks-stagger-by-source.md) 的错峰与冷却顺延照用,只是顺延按来源分:+一轮里 JavDB 在冷却,只顺延 JavDB 演员页那几条,DMM 片单照拉,反过来也一样。片单为零是正常结果(这位在 DMM 上没有配信作品),不当失败。结果只产生番号壳,取资料、落库规则不变(ADR-0052)。## 被否决的方案diff --git a/src/peach/web_feeds.py b/src/peach/web_feeds.pyindex 3174becd3..483c25ba3 100644--- a/src/peach/web_feeds.py+++ b/src/peach/web_feeds.py@@ -50,6 +50,13 @@ def _cooling_until(contract, url: str) -> datetime | None:return datetime.fromtimestamp(until, timezone.utc) if until else None+def _same_source(url: str, other: str) -> bool:+ """两个订阅地址是不是同一个来源的:冷却按来源记,顺延也只顺延同一来源的那几个。"""+ from .scraping_access import source_for++ return source_for(url) == source_for(other)++def _paused_by_source(error: Exception) -> bool:"""来源冷却挡下的这一次;本趟预算耗尽(`SourceBudgetExhausted`)不算来源冷却。"""from .scraping_access import SourceBudgetExhausted, SourcePaused@@ -174,8 +181,13 @@ def _poll_dmm(contract, transport, source, report: dict) -> dict:message = error.message if isinstance(error, SourceFailure) else str(error)report.update(ok=False, error=message,reason=str(error.reason) if isinstance(error, SourceFailure) else None)+ # 和演员页同一条:整站冷却挡回来的,冷却一过就重拉(ADR-0134)。+ retry_at = _cooling_until(contract, source["url"]) if _paused_by_source(error) else None+ if retry_at is not None:+ report["paused"] = Truewith contract.database.write_transaction() as connection:- feeds.settle(connection, source_id, error=message, interval_minutes=interval)+ feeds.settle(connection, source_id, error=message, interval_minutes=interval,+ retry_at=retry_at)return reportparsed = feeds.parse_dmm_actress(works)hidden = web_settings.hidden_compilations(contract.database)@@ -259,19 +271,23 @@ def _execute_check(contract, body, job_id: str) -> dict:codes = [code for row in results for code in row.get('codes') or []]if not contract.feed_job.checkpoint(job_id, source_ids=[int(row['id']) for row in rows]):raise InterruptedError('订阅检查已停止')- deferred = 0+ deferred, held = 0, set()try:for index, source in enumerate(rows):- if int(source['id']) in completed:+ if int(source['id']) in completed | held:continueuntil = _cooling_until(contract, source["url"])if until is not None:- pending = [row for row in rows[index:] if int(row['id']) not in completed]- results, deferred = _defer_round(contract, pending, until, results)+ # 只顺延同一来源的:JavDB 冷却挡不住 DMM 片单,反过来也一样(ADR-0135)。+ pending = [row for row in rows[index:] if int(row['id']) not in completed | held+ and _same_source(row["url"], source["url"])]+ results, count = _defer_round(contract, pending, until, results)+ deferred += count+ held.update(int(row['id']) for row in pending)if not contract.feed_job.checkpoint(job_id, results=results, checked=len(results),total=len(rows), added=len(set(codes))):raise InterruptedError('订阅检查已停止')- break+ continueif not contract.feed_job.update(job_id, checked=len(results), total=len(rows),current=source["name"] or source["url"]):raise InterruptedError('订阅检查已停止')diff --git a/tests/test_feeds_web.py b/tests/test_feeds_web.pyindex 403763161..947e14ec5 100644--- a/tests/test_feeds_web.py+++ b/tests/test_feeds_web.py@@ -506,6 +506,22 @@ class FeedWebTest(FeedWebFixture):self.assertTrue(all(row["last_fetched_at"] is None and row["last_error"] is Nonefor row in self._rows().values()))+ def test_a_javdb_cooldown_does_not_defer_the_dmm_catalogue(self):+ """冷却按来源记:JavDB 在冷却,DMM 片单照拉(ADR-0135)。"""+ from peach import scraping_access+ javdb = self._add()+ with self.contract.database.write_transaction() as connection:+ dmm = feeds.add_source(connection, url=feeds.dmm_actress_url("1051912"))+ self.transport.responses[DMM_ENDPOINT] = dmm_catalogue(+ [("mikr00133", "2026-11-13", True, 1)], {"mikr00133": "MIKR-133"})+ scraping_access.pause_source(self.contract.follow_secrets_root, "javdb", refused=True)+ result = self._check()+ reports = {row["source"]: row for row in result["results"]}+ self.assertEqual(result["deferred"], 1)+ self.assertEqual(reports[javdb]["error"], "JavDB 正在冷却,这一轮没有拉")+ self.assertEqual((reports[dmm]["ok"], reports[dmm]["codes"]), (True, ["MIKR-133"]))+ self.assertTrue(all(request.url == DMM_ENDPOINT for request in self.transport.requests))+def test_a_scheduled_tick_without_due_sources_starts_nothing(self):"""定时每半小时一跳;没有到期的源时不起任务,任务中心里不留空跑的一行。"""self._add()