Skip to content

armgit / peach

Public Imported from GitHub
0

Peach: Local-first video metadata scraper and media manager.

fix(feeds): 来源冷却只顺延同一来源的订阅,DMM 冷却也按结束时刻重拉

longmeidao7ffd2bdb2c7a
diff --git a/docs/adr/0135-dmm-actress-catalogue-as-second-feed-source.md b/docs/adr/0135-dmm-actress-catalogue-as-second-feed-source.md
index 2f9215db6..79fef6cb3 100644
--- a/docs/adr/0135-dmm-actress-catalogue-as-second-feed-source.md
+++ b/docs/adr/0135-dmm-actress-catalogue-as-second-feed-source.md
@@ -63,6 +63,8 @@ provider 为 `dmm`、`r18dev`,`external_kind='performer'`,纯数字的编号
**七、请求、限流与失败分类沿用刮削链。** 走同一条 `SourceTransport` 加主机限流,包在 `SiteSource.holding` 里:
验证页、封禁、地区限制与限流让 DMM 整站进冷却;任何失败按原因写进那一行的 `last_error`,下次时间照写。
+[ADR-0134](0134-feed-and-follow-checks-stagger-by-source.md) 的错峰与冷却顺延照用,只是顺延按来源分:
+一轮里 JavDB 在冷却,只顺延 JavDB 演员页那几条,DMM 片单照拉,反过来也一样。
片单为零是正常结果(这位在 DMM 上没有配信作品),不当失败。结果只产生番号壳,取资料、落库规则不变(ADR-0052)。
## 被否决的方案
diff --git a/src/peach/web_feeds.py b/src/peach/web_feeds.py
index 3174becd3..483c25ba3 100644
--- a/src/peach/web_feeds.py
+++ b/src/peach/web_feeds.py
@@ -50,6 +50,13 @@ def _cooling_until(contract, url: str) -> datetime | None:
return datetime.fromtimestamp(until, timezone.utc) if until else None
+def _same_source(url: str, other: str) -> bool:
+ """两个订阅地址是不是同一个来源的:冷却按来源记,顺延也只顺延同一来源的那几个。"""
+ from .scraping_access import source_for
+
+ return source_for(url) == source_for(other)
+
+
def _paused_by_source(error: Exception) -> bool:
"""来源冷却挡下的这一次;本趟预算耗尽(`SourceBudgetExhausted`)不算来源冷却。"""
from .scraping_access import SourceBudgetExhausted, SourcePaused
@@ -174,8 +181,13 @@ def _poll_dmm(contract, transport, source, report: dict) -> dict:
message = error.message if isinstance(error, SourceFailure) else str(error)
report.update(ok=False, error=message,
reason=str(error.reason) if isinstance(error, SourceFailure) else None)
+ # 和演员页同一条:整站冷却挡回来的,冷却一过就重拉(ADR-0134)。
+ retry_at = _cooling_until(contract, source["url"]) if _paused_by_source(error) else None
+ if retry_at is not None:
+ report["paused"] = True
with contract.database.write_transaction() as connection:
- feeds.settle(connection, source_id, error=message, interval_minutes=interval)
+ feeds.settle(connection, source_id, error=message, interval_minutes=interval,
+ retry_at=retry_at)
return report
parsed = feeds.parse_dmm_actress(works)
hidden = web_settings.hidden_compilations(contract.database)
@@ -259,19 +271,23 @@ def _execute_check(contract, body, job_id: str) -> dict:
codes = [code for row in results for code in row.get('codes') or []]
if not contract.feed_job.checkpoint(job_id, source_ids=[int(row['id']) for row in rows]):
raise InterruptedError('订阅检查已停止')
- deferred = 0
+ deferred, held = 0, set()
try:
for index, source in enumerate(rows):
- if int(source['id']) in completed:
+ if int(source['id']) in completed | held:
continue
until = _cooling_until(contract, source["url"])
if until is not None:
- pending = [row for row in rows[index:] if int(row['id']) not in completed]
- results, deferred = _defer_round(contract, pending, until, results)
+ # 只顺延同一来源的:JavDB 冷却挡不住 DMM 片单,反过来也一样(ADR-0135)。
+ pending = [row for row in rows[index:] if int(row['id']) not in completed | held
+ and _same_source(row["url"], source["url"])]
+ results, count = _defer_round(contract, pending, until, results)
+ deferred += count
+ held.update(int(row['id']) for row in pending)
if not contract.feed_job.checkpoint(job_id, results=results, checked=len(results),
total=len(rows), added=len(set(codes))):
raise InterruptedError('订阅检查已停止')
- break
+ continue
if not contract.feed_job.update(job_id, checked=len(results), total=len(rows),
current=source["name"] or source["url"]):
raise InterruptedError('订阅检查已停止')
diff --git a/tests/test_feeds_web.py b/tests/test_feeds_web.py
index 403763161..947e14ec5 100644
--- a/tests/test_feeds_web.py
+++ b/tests/test_feeds_web.py
@@ -506,6 +506,22 @@ class FeedWebTest(FeedWebFixture):
self.assertTrue(all(row["last_fetched_at"] is None and row["last_error"] is None
for row in self._rows().values()))
+ def test_a_javdb_cooldown_does_not_defer_the_dmm_catalogue(self):
+ """冷却按来源记:JavDB 在冷却,DMM 片单照拉(ADR-0135)。"""
+ from peach import scraping_access
+ javdb = self._add()
+ with self.contract.database.write_transaction() as connection:
+ dmm = feeds.add_source(connection, url=feeds.dmm_actress_url("1051912"))
+ self.transport.responses[DMM_ENDPOINT] = dmm_catalogue(
+ [("mikr00133", "2026-11-13", True, 1)], {"mikr00133": "MIKR-133"})
+ scraping_access.pause_source(self.contract.follow_secrets_root, "javdb", refused=True)
+ result = self._check()
+ reports = {row["source"]: row for row in result["results"]}
+ self.assertEqual(result["deferred"], 1)
+ self.assertEqual(reports[javdb]["error"], "JavDB 正在冷却,这一轮没有拉")
+ self.assertEqual((reports[dmm]["ok"], reports[dmm]["codes"]), (True, ["MIKR-133"]))
+ self.assertTrue(all(request.url == DMM_ENDPOINT for request in self.transport.requests))
+
def test_a_scheduled_tick_without_due_sources_starts_nothing(self):
"""定时每半小时一跳;没有到期的源时不起任务,任务中心里不留空跑的一行。"""
self._add()