Skip to content

armgit / peach

Public Imported from GitHub
0

Peach: Local-first video metadata scraper and media manager.

feat(feeds): 订阅源按相位错峰拉取,来源冷却整轮只记一次

longmeidaob0a300ccca7d
diff --git a/src/peach/api.py b/src/peach/api.py
index f966d315a..625d52826 100644
--- a/src/peach/api.py
+++ b/src/peach/api.py
@@ -458,7 +458,8 @@ def create_app(
contract.follow_scheduler = follow_scheduler
# 订阅源拉取(ADR-0042)用同一个调度实现,只换 job id、状态文件与默认间隔。
# 默认 6 小时:一位女优的 JavDB 演员页一天更新几条,比这更密只是把 JavDB 的配额
- # 花在一张没变的页面上。
+ # 花在一张没变的页面上。6 小时是每个源的周期;调度每半小时看一次谁到期,各源按
+ # 自己的相位错开(ADR-0132)。
feed_scheduler = FollowUpdateScheduler(
settings.follow_state_root,
lambda: web_feeds.w_feed_check(contract, {"automatic": True}),
@@ -467,6 +468,7 @@ def create_app(
filename="feed-schedule.json",
default=FollowScheduleConfig(enabled=True,
interval_minutes=feeds.DEFAULT_INTERVAL_MINUTES),
+ tick_minutes=feeds.TICK_MINUTES,
unavailable_message="订阅源拉取只在写入端可用",
)
contract.feed_scheduler = feed_scheduler
diff --git a/src/peach/feeds.py b/src/peach/feeds.py
index 019b7fd35..1fd97f06b 100644
--- a/src/peach/feeds.py
+++ b/src/peach/feeds.py
@@ -30,6 +30,13 @@ KIND_LABELS = {KIND_JAVDB_ACTOR: "JAV 订阅"}
#: 拉取间隔。JavDB 按出口 IP 计配额,一张一天才更新几条的页面拉得再密也只是花配额。
DEFAULT_INTERVAL_MINUTES = 360
+#: 定时拉取多久看一次有没有到期的源(ADR-0132)。每个源仍按自己的间隔拉,这一档只决定
+#: 到期之后最多再等多久;源在间隔窗口里各占一个相位,一跳只拉落在这半小时里的那几个。
+TICK_MINUTES = 30
+
+#: 相位散列用的黄金比例小数部分:连续的 id 落在窗口里也是均匀铺开的,几个源时也不扎堆。
+_GOLDEN = 0.6180339887498949
+
#: 一次拉取最多解析多少条。源正常一页几十条,异常的那种(被替换成别的页面、
#: 被镜像站塞进整站索引)会给出几千条,那不该变成几千个壳。
MAX_ENTRIES = 200
@@ -328,31 +335,56 @@ def sources(connection: sqlite3.Connection) -> list[dict]:
def due_sources(connection: sqlite3.Connection, now: str | None = None) -> list[sqlite3.Row]:
- """到期该拉的源。从没拉过的(`next_fetch_at` 为空)总是到期。"""
+ """到期该拉的源,等得最久的在前。从没拉过的(`next_fetch_at` 为空)总是到期,排最前。"""
moment = now or stamp()
return list(connection.execute(
"SELECT * FROM feed_source WHERE enabled=1"
- " AND (next_fetch_at IS NULL OR next_fetch_at<=?) ORDER BY id", (moment,)))
+ " AND (next_fetch_at IS NULL OR next_fetch_at<=?) ORDER BY next_fetch_at, id", (moment,)))
+
+
+def next_fetch(source_id: int, moment: datetime, interval_minutes: int) -> datetime:
+ """这个源下一次该拉的时刻:「现在 + 间隔」附近、落在它自己相位上的那一刻(ADR-0132)。
+
+ 不直接取「现在 + 间隔」:调度每跳之间隔着 `TICK_MINUTES`,拉完那一刻总比到期晚一点,
+ 按它往后排会一轮轮漂,同一时刻加的源也永远在同一跳一起到期。相位由 id 定,过渡那一轮
+ 间隔落在半个到一个半间隔之间,之后每次都回到同一个相位。
+ """
+ period = max(1, int(interval_minutes)) * 60
+ phase = (int(source_id) * _GOLDEN) % 1 * period
+ base = moment + timedelta(seconds=period)
+ offset = (phase - base.timestamp()) % period
+ if offset > period / 2:
+ offset -= period
+ return base + timedelta(seconds=offset)
def settle(connection: sqlite3.Connection, source_id: int, *, error: str | None = None,
etag: str | None = None, last_modified: str | None = None,
interval_minutes: int, seen: int = 0, new: int = 0,
- page_name: str | None = None, now: datetime | None = None) -> None:
+ page_name: str | None = None, now: datetime | None = None,
+ retry_at: datetime | None = None) -> None:
"""结算一次拉取。`page_name` 是这一页在站上挂的名字,解不出时留着上次的。
失败也按同一间隔排下一次,不做指数退避——退避要么在这里写第二套判据,要么让一个
临时挡回来的源沉默半天。下次时间在所有返回路径上都写,任何一种结局都不会让源卡住。
+ `retry_at` 是来源冷却的结束时刻:被整站挡回来的不是这个源的错,冷却一过就该再拉。
"""
moment = now or datetime.now(timezone.utc)
+ upcoming = retry_at or next_fetch(source_id, moment, interval_minutes)
connection.execute(
"UPDATE feed_source SET last_error=?,last_fetched_at=?,next_fetch_at=?,"
"etag=COALESCE(?,etag),last_modified=COALESCE(?,last_modified),"
"name=COALESCE(?,name),last_seen_count=?,last_new_count=? WHERE id=?",
- (error, stamp(moment), stamp(moment + timedelta(minutes=interval_minutes)),
+ (error, stamp(moment), stamp(upcoming),
etag, last_modified, page_name or None, int(seen), int(new), int(source_id)))
+def defer(connection: sqlite3.Connection, source_id: int, until: datetime) -> None:
+ """这一轮没轮到问就撞上了来源冷却:只把下次时间挪到冷却结束,上次的结论原样留着。"""
+ connection.execute("UPDATE feed_source SET next_fetch_at=? WHERE id=?",
+ (stamp(until), int(source_id)))
+
+
# -- 条目与壳 --------------------------------------------------------------
diff --git a/src/peach/follow_scheduler.py b/src/peach/follow_scheduler.py
index 9ac63aded..29154ce46 100644
--- a/src/peach/follow_scheduler.py
+++ b/src/peach/follow_scheduler.py
@@ -88,9 +88,13 @@ class FollowUpdateScheduler:
def __init__(self, state_root: Path, run_check: Callable[[], dict], *, available: bool,
job_id: str = JOB_ID, filename: str = "follow-schedule.json",
default: FollowScheduleConfig | None = None,
+ tick_minutes: int | None = None,
unavailable_message: str =
"automatic follow updates are available only on the ledger writer"):
self.job_id = job_id
+ #: 多久跑一次。不给就等于设置里的间隔;订阅源给的是 `feeds.TICK_MINUTES`:
+ #: 那边设置里的间隔是每个源的周期,到期由源自己判(ADR-0132)。
+ self.tick_minutes = tick_minutes
self.unavailable_message = unavailable_message
self.store = FollowScheduleStore(state_root, filename, default)
self.run_check = run_check
@@ -156,12 +160,17 @@ class FollowUpdateScheduler:
raise ValueError("自动运行没有开启") from None
return self.status()
+ @property
+ def tick(self) -> int:
+ """调度任务的实际间隔(分钟)。"""
+ return self.tick_minutes or self.config.interval_minutes
+
def _apply_job(self) -> None:
if not self.config.enabled:
if self.scheduler.get_job(self.job_id) is not None:
self.scheduler.remove_job(self.job_id)
return
- interval = self.config.interval_minutes
+ interval = self.tick
self.scheduler.add_job(
self._run,
"interval",
@@ -181,9 +190,9 @@ class FollowUpdateScheduler:
到 `MAX_BACKOFF_FACTOR` 倍为止,且不超过允许的最大间隔。
"""
if failures <= 1:
- return self.config.interval_minutes
+ return self.tick
factor = min(2 ** (failures - 1), MAX_BACKOFF_FACTOR)
- return min(self.config.interval_minutes * factor, MAX_INTERVAL_MINUTES)
+ return min(self.tick * factor, MAX_INTERVAL_MINUTES)
def _run(self) -> None:
with self._state_lock:
@@ -248,7 +257,7 @@ class FollowUpdateScheduler:
if not failed or not self._started:
return
delay = self.backoff_minutes(failures)
- if delay <= self.config.interval_minutes:
+ if delay <= self.tick:
return
try:
self.scheduler.modify_job(
diff --git a/src/peach/web_feeds.py b/src/peach/web_feeds.py
index 8020b5832..df10ec17b 100644
--- a/src/peach/web_feeds.py
+++ b/src/peach/web_feeds.py
@@ -11,7 +11,7 @@ import re
import threading
import time
import uuid
-from datetime import timedelta
+from datetime import datetime, timedelta, timezone
from urllib.parse import quote
from . import (
@@ -33,9 +33,30 @@ FETCH_MAX_BYTES = 2 * 1024 * 1024
#: 一轮里最多拉几个源。源多的时候分几轮拉完,而不是让一轮跑到限流上去。
MAX_SOURCES_PER_RUN = 20
+#: 定时那几轮多久派一次存量后继:旧壳重试、补封面、缺头像的存量女优与想要作品重试
+#: (ADR-0132)。定时拉取每 `feeds.TICK_MINUTES` 一跳,存量跟着每跳派一次就成了原来的十几倍;
+#: 本轮新发现的番号不受它管,照旧当场派取资料。
+STOCK_INTERVAL = timedelta(hours=6)
+
_NOT_MODIFIED = 304
+def _cooling_until(contract, url: str) -> datetime | None:
+ """这个地址所属的来源还在冷却就返回冷却结束的时刻(UTC),否则 None。"""
+ from .scraping_access import paused_until, source_for
+
+ source = source_for(url)
+ until = paused_until(contract.follow_secrets_root, source) if source else 0.0
+ return datetime.fromtimestamp(until, timezone.utc) if until else None
+
+
+def _paused_by_source(error: Exception) -> bool:
+ """来源冷却挡下的这一次;本趟预算耗尽(`SourceBudgetExhausted`)不算来源冷却。"""
+ from .scraping_access import SourceBudgetExhausted, SourcePaused
+
+ return isinstance(error, SourcePaused) and not isinstance(error, SourceBudgetExhausted)
+
+
def _parse_source(kind: str, body: bytes, url: str):
"""演员页受登录限制时返回可操作的原因。"""
if kind == feeds.KIND_JAVDB_ACTOR and javdb.LOGIN.search(body.decode("utf-8", "replace")):
@@ -85,9 +106,13 @@ def poll_source(contract, transport, source) -> dict:
last_modified=source["last_modified"])
except Exception as error: # noqa: BLE001 - 原因写到源那一行上给人看
report.update(ok=False, error=str(error))
+ # 整站冷却挡回来的,冷却一过就重拉,不跟着间隔再等一整轮。
+ retry_at = _cooling_until(contract, source["url"]) if _paused_by_source(error) else None
+ if retry_at is not None:
+ report["paused"] = True
with contract.database.write_transaction() as connection:
feeds.settle(connection, source_id, error=str(error),
- interval_minutes=interval)
+ interval_minutes=interval, retry_at=retry_at)
return report
if response.status == _NOT_MODIFIED:
with contract.database.write_transaction() as connection:
@@ -189,10 +214,19 @@ def _execute_check(contract, body, job_id: str) -> dict:
codes = [code for row in results for code in row.get('codes') or []]
if not contract.feed_job.checkpoint(job_id, source_ids=[int(row['id']) for row in rows]):
raise InterruptedError('订阅检查已停止')
+ deferred = 0
try:
for index, source in enumerate(rows):
if int(source['id']) in completed:
continue
+ until = _cooling_until(contract, source["url"])
+ if until is not None:
+ pending = [row for row in rows[index:] if int(row['id']) not in completed]
+ results, deferred = _defer_round(contract, pending, until, results)
+ if not contract.feed_job.checkpoint(job_id, results=results, checked=len(results),
+ total=len(rows), added=len(set(codes))):
+ raise InterruptedError('订阅检查已停止')
+ break
if not contract.feed_job.update(job_id, checked=len(results), total=len(rows),
current=source["name"] or source["url"]):
raise InterruptedError('订阅检查已停止')
@@ -210,31 +244,80 @@ def _execute_check(contract, body, job_id: str) -> dict:
contract.cache_bust()
# 取资料:新发现的先排,没取到资料的旧壳留着保底名额,同一批一条后继。只缺封面的另成一批
# 补封面。定时那一轮只带到了重试时间的,手动点的那一轮全带;已在队里的番号都不再带。
- retry_after = feed_followup.RETRY_AFTER if body.get("automatic") else timedelta(0)
+ # 存量部分(旧壳、封面、头像)定时那几轮隔 `STOCK_INTERVAL` 才派一次,新番号每轮都派。
+ automatic = bool(body.get("automatic"))
+ stock = _stock_due(contract) if automatic else True
+ retry_after = feed_followup.RETRY_AFTER if automatic else timedelta(0)
hidden = web_settings.hidden_compilations(contract.database)
busy = feed_followup.in_flight(contract.task_runs)
fresh = [code for code in dict.fromkeys(codes) if normalise_code_key(code) not in busy]
with contract.database.read_connection() as connection:
- batch = feed_followup.scrape_batch(
- connection, fresh, exclude=[*codes, *busy], retry_after=retry_after, hidden=hidden)
- covers = feed_followup.cover_backlog(
- connection, contract.cover_root, exclude=[*codes, *batch, *busy],
- retry_after=retry_after, hidden=hidden)
- planned = feed_followup.plan(batch) + feed_followup.plan_covers(covers)
- # 余下的名额给缺头像的存量女优(ADR-0097)。资料处理收尾也派这一份,但没有新片
- # 入库的日子它一轮都不跑,存量就一直停在原地;定时拉取每天都跑。
- planned += avatar_followup.stock(
- connection, contract.avatar_root, Attempts(attempts_root(contract.candidate_root)),
- limit=MAX_FOLLOWUPS - len(planned))
- return {"ok": True, "checked": len(rows), "total": len(rows), "results": results,
- "added": len(codes),
+ if not stock:
+ planned = feed_followup.plan(fresh[:feed_followup.MAX_BATCH])
+ else:
+ batch = feed_followup.scrape_batch(
+ connection, fresh, exclude=[*codes, *busy], retry_after=retry_after, hidden=hidden)
+ covers = feed_followup.cover_backlog(
+ connection, contract.cover_root, exclude=[*codes, *batch, *busy],
+ retry_after=retry_after, hidden=hidden)
+ planned = feed_followup.plan(batch) + feed_followup.plan_covers(covers)
+ # 余下的名额给缺头像的存量女优(ADR-0097)。资料处理收尾也派这一份,但没有新片
+ # 入库的日子它一轮都不跑,存量就一直停在原地;定时拉取每天都跑。
+ planned += avatar_followup.stock(
+ connection, contract.avatar_root, Attempts(attempts_root(contract.candidate_root)),
+ limit=MAX_FOLLOWUPS - len(planned))
+ if stock:
+ contract.feed_stock_planned_at = time.monotonic()
+ return {"ok": True, "checked": len(results), "total": len(rows), "results": results,
+ "added": len(codes), "deferred": deferred, "stock": stock,
# 后继由结果声明,调度端统一派(ADR-0040):这里只把清单交出去。
"followups": [dict(key=item.key, task_key=item.task_key, label=item.label)
for item in planned]}
+def _defer_round(contract, pending, until: datetime, results: list) -> tuple[list, int]:
+ """来源在冷却:这一轮剩下的源都顺延到冷却结束,整轮只记一条失败(ADR-0132)。
+
+ 冷却是整个来源的状态,不是哪个订阅的毛病;每个源各记一条「正在冷却」只会把真正的
+ 原因淹掉。本轮撞上 403 的那一行已经写了原因,就在它后面补一句顺延了几个;本轮一开始
+ 来源就在冷却(别的任务撞出来的),就用第一个源那一行说明,它的上次结论不改。
+ """
+ with contract.database.write_transaction() as connection:
+ for row in pending:
+ feeds.defer(connection, int(row["id"]), until)
+ results = list(results)
+ trigger = next((row for row in reversed(results) if row.get("paused")), None)
+ rest = len(pending)
+ if trigger is None:
+ from .scraping_access import SOURCES, source_for
+ first = pending[0]
+ label = SOURCES.get(source_for(first["url"]) or "", {}).get("label", "来源")
+ trigger = {"source": int(first["id"]), "name": first["name"] or first["url"],
+ "ok": False, "seen": 0, "added": 0, "without_code": 0, "paused": True,
+ "error": f"{label} 正在冷却,这一轮没有拉"}
+ results.append(trigger)
+ rest -= 1
+ if rest:
+ trigger["error"] = f"{trigger['error']};其余 {rest} 个源顺延到冷却结束后再拉"
+ return results, len(pending)
+
+
+def _stock_due(contract) -> bool:
+ """定时这一轮该不该派存量后继:本进程里还没派过,或上次派过已经满 `STOCK_INTERVAL`。"""
+ last = getattr(contract, "feed_stock_planned_at", None)
+ return last is None or time.monotonic() - last >= STOCK_INTERVAL.total_seconds()
+
+
def w_feed_check(contract, body) -> dict:
- """拉一轮订阅。不带参数只拉到期的源,`all` 拉全部启用的源。"""
+ """拉一轮订阅。不带参数只拉到期的源,`all` 拉全部启用的源。
+
+ 定时那一跳没有到期的源就什么也不起:每半小时一跳(`feeds.TICK_MINUTES`),任务中心
+ 不该每半小时多一行空跑。
+ """
+ if body.get("automatic") and "sources" not in body and not body.get("all"):
+ with contract.database.read_connection() as connection:
+ if not feeds.due_sources(connection):
+ return {"ok": True, "idle": True, "checked": 0, "total": 0, "results": []}
finished = threading.Event()
request_id = uuid.uuid4().hex
@@ -243,8 +326,9 @@ def w_feed_check(contract, body) -> dict:
result = _execute_check(contract, body, job_id)
contract.feed_job.update(job_id, **result, status="complete",
current=None, completed_at=time.time())
- # 「想要」里取不到资料或封面的,跟着每一轮拉取再试(隔 `web_wants.RETRY_AFTER`)。
- web_wants.start_scrape(contract, trigger="scheduled" if body.get("automatic") else "manual")
+ # 「想要」里取不到资料或封面的,跟着派存量的那几轮再试(隔 `web_wants.RETRY_AFTER`)。
+ if result["stock"]:
+ web_wants.start_scrape(contract, trigger="scheduled" if body.get("automatic") else "manual")
finally:
finished.set()
diff --git a/src/peach/web_state.py b/src/peach/web_state.py
index 3c87a2f65..e85c1f2d1 100644
--- a/src/peach/web_state.py
+++ b/src/peach/web_state.py
@@ -223,6 +223,9 @@ class WebContract:
#: 订阅源的定时拉取(ADR-0042)。与追更共用 `follow_scheduler` 那一个实现,
#: 只是各占一个 job id 与一份状态文件;由 `api` 在装配时接上。
self.feed_scheduler = None
+ #: 定时拉取上一次派存量后继的时刻(`time.monotonic()`);None 是本进程还没派过
+ #: (`web_feeds.STOCK_INTERVAL`)。
+ self.feed_stock_planned_at = None
#: 云下载(`downloads.DownloadService`),由 `api` 在装配时接上;没接上时
#: `/api/downloads` 回「云下载没有启用」。
self.downloads = None
diff --git a/tests/test_feeds.py b/tests/test_feeds.py
index 62600dbe8..6dd38b3cc 100644
--- a/tests/test_feeds.py
+++ b/tests/test_feeds.py
@@ -4,6 +4,7 @@ from __future__ import annotations
import sys
import tempfile
import unittest
+from datetime import datetime, timedelta, timezone
from pathlib import Path
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "src"))
@@ -162,6 +163,35 @@ class CompilationTest(unittest.TestCase):
self.assertIsNone(feeds.compilation_kind("愛する妻が…", "篠田ゆう", "マドンナ"))
+class FeedPhaseTest(unittest.TestCase):
+ """下次拉取按源的相位排(ADR-0132):同时加的源错开,跳与跳之间的延迟不累积。"""
+
+ MOMENT = datetime(2026, 11, 3, 3, 28, 57, tzinfo=timezone.utc)
+ PERIOD = timedelta(minutes=feeds.DEFAULT_INTERVAL_MINUTES)
+
+ def test_sources_fetched_together_spread_over_the_window(self):
+ upcoming = sorted(feeds.next_fetch(source_id, self.MOMENT, feeds.DEFAULT_INTERVAL_MINUTES)
+ for source_id in range(1, 12))
+ for moment in upcoming:
+ self.assertGreaterEqual(moment, self.MOMENT + self.PERIOD / 2)
+ self.assertLessEqual(moment, self.MOMENT + self.PERIOD * 1.5)
+ slots = {int((moment.timestamp() % self.PERIOD.total_seconds())
+ // timedelta(minutes=feeds.TICK_MINUTES).total_seconds()) for moment in upcoming}
+ # 11 个源落进 12 个半小时档里的至少 9 个:一跳最多拉两三个,而不是 11 个一起。
+ self.assertGreaterEqual(len(slots), 9)
+
+ def test_tick_lag_does_not_drift_the_schedule(self):
+ """到期之后要等到下一跳才拉,拉完排的下次仍回到同一个相位,周期保持一个间隔。"""
+ moment = self.MOMENT
+ due = feeds.next_fetch(7, moment, feeds.DEFAULT_INTERVAL_MINUTES)
+ for lag in (timedelta(minutes=29), timedelta(minutes=3), timedelta(minutes=17)):
+ fetched = due + lag
+ following = feeds.next_fetch(7, fetched, feeds.DEFAULT_INTERVAL_MINUTES)
+ self.assertAlmostEqual((following - due).total_seconds(),
+ self.PERIOD.total_seconds(), places=2)
+ due = following
+
+
class FeedStoreTest(unittest.TestCase):
def setUp(self):
self._tmp = tempfile.TemporaryDirectory()
@@ -278,6 +308,27 @@ class FeedStoreTest(unittest.TestCase):
with self.database.read_connection() as connection:
self.assertEqual(feeds.due_sources(connection), [])
+ def test_the_longest_waiting_source_is_due_first(self):
+ first = self._source(url="https://javdb.com/actors/a1")
+ second = self._source(url="https://javdb.com/actors/a2")
+ fresh = self._source(url="https://javdb.com/actors/a3")
+ with self.database.write_transaction() as connection:
+ feeds.defer(connection, first, datetime(2026, 11, 3, 2, tzinfo=timezone.utc))
+ feeds.defer(connection, second, datetime(2026, 11, 3, 1, tzinfo=timezone.utc))
+ with self.database.read_connection() as connection:
+ due = [int(row["id"]) for row in feeds.due_sources(connection, "2026-11-03T03:00:00Z")]
+ self.assertEqual(due, [fresh, second, first])
+
+ def test_settle_after_a_cooldown_retries_at_its_end(self):
+ source_id = self._source()
+ until = datetime(2026, 11, 3, 3, 15, tzinfo=timezone.utc)
+ with self.database.write_transaction() as connection:
+ feeds.settle(connection, source_id, error="来源拒绝访问", interval_minutes=360,
+ now=datetime(2026, 11, 3, 3, tzinfo=timezone.utc), retry_at=until)
+ row = connection.execute("SELECT next_fetch_at FROM feed_source WHERE id=?",
+ (source_id,)).fetchone()
+ self.assertEqual(row[0], feeds.stamp(until))
+
def test_performer_names_match_existing_entities_only(self):
with self.database.write_transaction() as connection:
connection.execute(
diff --git a/tests/test_feeds_web.py b/tests/test_feeds_web.py
index a968dea60..4236103b3 100644
--- a/tests/test_feeds_web.py
+++ b/tests/test_feeds_web.py
@@ -9,7 +9,7 @@ import tempfile
import threading
import time
import unittest
-from datetime import timedelta
+from datetime import datetime, timedelta, timezone
from pathlib import Path
from types import SimpleNamespace
from unittest.mock import patch
@@ -80,6 +80,8 @@ class FeedWebFixture(unittest.TestCase):
# assert 之后才写完,也可能在临时目录删掉之后才醒。这里关掉自动领取,
# 要跑后继的用例自己 `drain()`。
self.contract.followups.stop()
+ # 来源冷却记录读写这里:每轮拉取前都要看 JavDB 是否在冷却,不能读到本机真实的那份。
+ self.contract.follow_secrets_root = Path(self.temporary.name) / "secrets"
self.url = "https://javdb.com/actors/Smp1"
sample = Path(self.temporary.name) / "feed-sample.html"
sample.write_text(SAMPLE, encoding="utf-8")
@@ -141,6 +143,10 @@ class FeedWebFixture(unittest.TestCase):
from peach.http import HttpRequest
return transport(HttpRequest("GET", url, headers), 30.0, 1 << 21)
+ def _stock_window_passes(self):
+ """定时拉取派存量后继的窗口(`web_feeds.STOCK_INTERVAL`)已经过去。"""
+ self.contract.feed_stock_planned_at = None
+
def _add(self, entity_id=None):
with self.contract.database.write_transaction() as connection:
return feeds.add_source(connection, url=self.url, name="示例源",
@@ -423,6 +429,66 @@ class FeedWebTest(FeedWebFixture):
self.assertEqual(row["last_error"], "来源回了 HTTP 503")
self.assertIsNotNone(row["next_fetch_at"])
+ def _sources_named(self, *names):
+ """再加几个演员页源,各回一份同样的样本页。返回全部源 id,第一个是 `_add` 那个。"""
+ ids = [self._add()]
+ for name in names:
+ url = f"https://javdb.com/actors/{name}"
+ self.transport.responses[url] = self.transport.responses[self.url]
+ with self.contract.database.write_transaction() as connection:
+ ids.append(feeds.add_source(connection, url=url, name=name))
+ return ids
+
+ def _rows(self):
+ with self.contract.database.read_connection() as connection:
+ return {int(row["id"]): dict(row) for row in connection.execute("SELECT * FROM feed_source")}
+
+ def test_a_refused_round_records_one_failure_and_defers_the_rest_to_the_cooldown_end(self):
+ """一轮里撞上 403:只记撞上的那一条,其余源不再问、顺延到冷却结束(ADR-0132)。"""
+ from peach import scraping_access
+ first, *rest = self._sources_named("Smp2", "Smp3")
+ served = self.transport
+
+ def refused(request, timeout, max_bytes):
+ served.requests.append(request)
+ scraping_access.pause_source(self.contract.follow_secrets_root, "javdb", refused=True)
+ raise scraping_access.SourcePaused("来源拒绝访问,暂停向它请求一段时间")
+
+ self.transport = refused
+ result = self._check()
+ self.assertEqual(len(served.requests), 1)
+ self.assertEqual(result["deferred"], 2)
+ [row] = result["results"]
+ self.assertFalse(row["ok"])
+ self.assertIn("其余 2 个源顺延到冷却结束后再拉", row["error"])
+ until = scraping_access.paused_until(self.contract.follow_secrets_root, "javdb")
+ expected = feeds.stamp(datetime.fromtimestamp(until, timezone.utc))
+ rows = self._rows()
+ self.assertEqual({rows[source]["next_fetch_at"] for source in (first, *rest)}, {expected})
+ self.assertTrue(all(rows[source]["last_fetched_at"] is None for source in rest))
+
+ def test_a_round_that_starts_in_cooldown_asks_nothing(self):
+ """别的任务撞出来的冷却还没过:这一轮一个请求都不发,整轮只说一句。"""
+ from peach import scraping_access
+ self._sources_named("Smp2")
+ scraping_access.pause_source(self.contract.follow_secrets_root, "javdb", refused=True)
+ result = self._check()
+ self.assertEqual(self.transport.requests, [])
+ [row] = result["results"]
+ self.assertEqual(row["error"], "JavDB 正在冷却,这一轮没有拉;其余 1 个源顺延到冷却结束后再拉")
+ self.assertTrue(all(row["last_fetched_at"] is None and row["last_error"] is None
+ for row in self._rows().values()))
+
+ def test_a_scheduled_tick_without_due_sources_starts_nothing(self):
+ """定时每半小时一跳;没有到期的源时不起任务,任务中心里不留空跑的一行。"""
+ self._add()
+ self._check()
+ runs = len(self.contract.task_runs.query(task_key="feed-check", limit=50))
+ result = web_feeds.w_feed_check(self.contract, {"automatic": True})
+ self.assertTrue(result["idle"])
+ self.assertEqual(len(self.contract.task_runs.query(task_key="feed-check", limit=50)), runs)
+ self.assertEqual(len(self.transport.requests), 1)
+
def test_an_empty_alias_page_settles_quietly_and_names_its_row(self):
"""同一个人的第二页挂在旧艺名下、一部作品都没有:不算失败,行上说出是哪个名字。"""
with self.contract.database.write_transaction() as connection:
@@ -906,8 +972,10 @@ class FeedWebTest(FeedWebFixture):
self._drain(1)
self.transport.responses[self.url] = HttpResponse(304, {}, b"", self.url)
# 定时那一轮:刚试过的不再排;已经有封面、资料也齐的那部永远不再排。
+ self._stock_window_passes()
self.assertEqual(self._check(automatic=True)["followups"], [])
self._attempted("HMN-071", feed_followup.RETRY_AFTER.total_seconds() + 60)
+ self._stock_window_passes()
self.assertEqual([item["key"] for item in self._check(automatic=True)["followups"]],
["feed-cover:HMN-071"])
# 人说了不想看的,不再替它花配额。
@@ -956,10 +1024,30 @@ class FeedWebTest(FeedWebFixture):
with self.contract.database.read_connection() as connection:
current = avatar_followup.fingerprint(connection, entity_id, providers_root)
Attempts(attempts_root(self.contract.candidate_root)).record(key, current, "图库 6 张认不准")
+ self._stock_window_passes()
self.assertNotIn(key, [item["key"] for item in self._check(automatic=True)["followups"]])
index_path.write_text(json.dumps({"Content": {"1-S1": {"二の宮すずか.jpg": "二の宮すずか.jpg"}}}), encoding="utf-8")
+ self._stock_window_passes()
self.assertIn(key, [item["key"] for item in self._check(automatic=True)["followups"]])
+ def test_scheduled_rounds_carry_the_stock_only_once_per_window(self):
+ """定时每半小时一跳:新番号每跳都派,存量(旧壳、封面、头像、想要)一个窗口只派一次。"""
+ self.cover_misses.add("HMN-071")
+ self._add()
+ first = self._check(automatic=True)
+ self.assertTrue(first["stock"])
+ self.assertEqual([item["key"] for item in first["followups"]],
+ ["feed-scrape:SSIS-950,HMN-071"])
+ self._drain(1)
+ self._attempted("HMN-071", feed_followup.RETRY_AFTER.total_seconds() + 60)
+ self.transport.responses[self.url] = HttpResponse(304, {}, b"", self.url)
+ quiet = self._check(automatic=True)
+ self.assertFalse(quiet["stock"])
+ self.assertEqual(quiet["followups"], [])
+ self._stock_window_passes()
+ self.assertEqual([item["key"] for item in self._check(automatic=True)["followups"]],
+ ["feed-cover:HMN-071"])
+
def test_an_unknown_action_is_refused(self):
with self.assertRaises(ValueError):
dispatch_api_post(self.contract, "/api/feeds/discovery",
diff --git a/tests/test_follow_scheduler.py b/tests/test_follow_scheduler.py
index a85e931c7..8c458d17f 100644
--- a/tests/test_follow_scheduler.py
+++ b/tests/test_follow_scheduler.py
@@ -196,6 +196,21 @@ class FollowUpdateSchedulerTests(unittest.TestCase):
self.assertGreater(datetime.fromisoformat(next_run), planned,
"连续失败后应当推后,而不是照原间隔继续敲上游")
+ def test_a_tick_shorter_than_the_period_runs_often_and_backs_off_from_the_tick(self):
+ """订阅源:设置里的间隔是每个源的周期,调度按更短的一跳看谁到期(ADR-0132)。"""
+ with tempfile.TemporaryDirectory() as temporary:
+ scheduler = FollowUpdateScheduler(
+ Path(temporary), lambda: {"ok": True, "checked": 0, "results": []},
+ available=True, default=FollowScheduleConfig(True, 360), tick_minutes=30)
+ self.addCleanup(scheduler.stop)
+ before = datetime.now(timezone.utc)
+ scheduler.start()
+ status = scheduler.status()
+ self.assertEqual(status["interval_minutes"], 360)
+ self.assertLess(datetime.fromisoformat(status["next_run_at"]),
+ before + timedelta(minutes=31))
+ self.assertEqual(scheduler.backoff_minutes(2), 60)
+
def test_reader_never_starts_or_accepts_schedule_changes(self):
with tempfile.TemporaryDirectory() as temporary:
scheduler = FollowUpdateScheduler(Path(temporary), lambda: {}, available=False)