Skip to content

armgit / peach

Public Imported from GitHub
0

Peach: Local-first video metadata scraper and media manager.

fix(feeds): 空跳保留上一轮的失败,冷却跳过不在补全旧页时空转

longmeidaoaadbbf23ee4c
diff --git a/docs/adr/0132-feed-and-follow-checks-stagger-by-source.md b/docs/adr/0132-feed-and-follow-checks-stagger-by-source.md
index 33efd9b12..3dd768018 100644
--- a/docs/adr/0132-feed-and-follow-checks-stagger-by-source.md
+++ b/docs/adr/0132-feed-and-follow-checks-stagger-by-source.md
@@ -43,6 +43,6 @@
## 后果
- 订阅源对 JavDB 的请求回到设计频率:11 个源每小时约 1.8 次,均匀分在各跳里,不再一次 11 个;漂移修掉后比改动前实际的约 0.9 次多。订阅 100 个源时约每小时 17 次,仍远低于 3 秒间隔的每分钟 20 次。
-- 定时调度状态里的「上次运行」每 30 分钟更新一次;`last_checked` 是这一跳拉了几个源,空跳为 0。
+- 定时调度状态里的「上次运行」只记真正拉了源的那一跳。空跳不改上次运行时间、上次失败与连续失败计数:按成功结算的话,一次真实失败半小时后就从界面上消失,失败的源却还要等到下一次到期才重拉。
- 设置页「自动检查更新」一行说明了按活跃度放慢的规则。
- rule34xxx 每轮会重复补全约 281 条始终补不齐的详情,是追更请求最多的地方,不在本 ADR 范围。
diff --git a/src/peach/follow_scheduler.py b/src/peach/follow_scheduler.py
index 29154ce46..e99299eff 100644
--- a/src/peach/follow_scheduler.py
+++ b/src/peach/follow_scheduler.py
@@ -197,6 +197,7 @@ class FollowUpdateScheduler:
def _run(self) -> None:
with self._state_lock:
started_before = self._last_started_at
+ error_before = self._last_error
self._running = True
self._last_started_at = _stamp()
self._last_error = None
@@ -221,6 +222,15 @@ class FollowUpdateScheduler:
self._last_skipped_at = _stamp()
self._last_started_at = started_before
return
+ if result.get("idle"):
+ # 这一跳没有到期的来源(订阅源每半小时一跳,ADR-0132):什么也没做,
+ # 上一轮的结论原样留着。按成功结算的话,一次真实失败半小时后就从界面上消失,
+ # 而失败的那几个源还要等到下一次到期才重拉。
+ skipped = True
+ with self._state_lock:
+ self._last_started_at = started_before
+ self._last_error = error_before
+ return
rows = result.get("results") or []
failures = [row for row in rows if not row.get("ok")]
with self._state_lock:
diff --git a/src/peach/web_follow.py b/src/peach/web_follow.py
index 9f0e0bb6c..a35ee5228 100644
--- a/src/peach/web_follow.py
+++ b/src/peach/web_follow.py
@@ -2395,8 +2395,8 @@ class _FollowCheckSession:
connector_factory=build_connector, older=older, progress=progress,
initial_days=self.initial_days, cooldown=self.cooldown))
more = (older and self.body.get('backfill_all') and result.get('ok')
- and not result.get('exhausted') and isinstance(result.get('page'), int)
- and round_index < 500)
+ and not result.get('exhausted') and not result.get('deferred')
+ and isinstance(result.get('page'), int) and round_index < 500)
self.remember(row, result, finished=not more)
if not more:
break
diff --git a/tests/test_follow_scheduler.py b/tests/test_follow_scheduler.py
index 8c458d17f..8af1dc352 100644
--- a/tests/test_follow_scheduler.py
+++ b/tests/test_follow_scheduler.py
@@ -211,6 +211,23 @@ class FollowUpdateSchedulerTests(unittest.TestCase):
before + timedelta(minutes=31))
self.assertEqual(scheduler.backoff_minutes(2), 60)
+ def test_an_idle_tick_keeps_the_last_failure_on_record(self):
+ """没有到期源的一跳什么也没做:上一轮的失败、连续失败计数和上次运行时间都留着。"""
+ outcomes = [{"ok": True, "checked": 1, "results": [{"ok": False, "added": 0}]},
+ {"ok": True, "idle": True, "checked": 0, "results": []}]
+ with tempfile.TemporaryDirectory() as temporary:
+ scheduler = FollowUpdateScheduler(Path(temporary), lambda: outcomes.pop(0),
+ available=True, tick_minutes=30)
+ scheduler._run()
+ failed = scheduler.status()
+ scheduler._run()
+ status = scheduler.status()
+ self.assertEqual(status["last_error"], "1 个来源检查失败")
+ self.assertEqual(status["consecutive_failures"], 1)
+ self.assertEqual((status["last_started_at"], status["last_finished_at"], status["last_checked"]),
+ (failed["last_started_at"], failed["last_finished_at"], 1))
+ self.assertFalse(status["running"])
+
def test_reader_never_starts_or_accepts_schedule_changes(self):
with tempfile.TemporaryDirectory() as temporary:
scheduler = FollowUpdateScheduler(Path(temporary), lambda: {}, available=False)
diff --git a/tests/test_follow_web.py b/tests/test_follow_web.py
index dd697c1c1..fee9b55bd 100644
--- a/tests/test_follow_web.py
+++ b/tests/test_follow_web.py
@@ -183,8 +183,11 @@ class FollowContractTests(unittest.TestCase):
self.contract.follow_media_resolver.pause('rule34video', 60)
with mock.patch.object(web_follow, 'build_connector') as factory:
result = self._post('/api/follow/check', {})
+ # 往回一路抓到底也一样:冷却中的那条只跳过一次,不在原地空转翻页。
+ backfill = self._post('/api/follow/check', {'older': True, 'backfill_all': True})
self.assertEqual(factory.call_count, 0)
self.assertEqual((result['results'], result['deferred']), ([], 2))
+ self.assertEqual((backfill['results'], backfill['deferred']), ([], 2))
def test_a_scheduled_round_checks_only_the_sources_that_are_due(self):
"""定时那一跳跳过刚查过、又久未更新的来源;手动检查照旧全查。"""