Skip to content

armgit / peach

Public Imported from GitHub
0

Peach: Local-first video metadata scraper and media manager.

fix(feeds): 取资料续跑只接着试余下的番号

longmeidao414b4a4bae7f
diff --git a/src/peach/feed_followup.py b/src/peach/feed_followup.py
index a85f1934d..effdc17d5 100644
--- a/src/peach/feed_followup.py
+++ b/src/peach/feed_followup.py
@@ -376,6 +376,10 @@ def run(contract, key: str, handle) -> dict:
封面断在连接上的几部另记 `cover_network`:这一类换条线路就能取到,新作页据此提示
去配来源的连接方式,而不是让人以为官方没出图。
+
+ 检查点记的是这一批里已经试过的番号,不论取没取到:服务重启后这一条重新领取时只接着
+ 试余下的。没取到的那几部由 `scrape_batch`、`cover_backlog` 到了重试时间再排,不在
+ 同一批的续跑里每次重启都重耗一遍。
"""
import httpx
@@ -386,11 +390,11 @@ def run(contract, key: str, handle) -> dict:
provider = None
saved = getattr(handle, 'saved', {})
saved = saved if isinstance(saved, dict) else {}
- completed = set(saved.get('completed_codes') or [])
+ done = set(saved.get('done_codes') or [])
fetched, covers = saved.get('fetched', 0), saved.get('covers', 0)
- missed = unreachable = 0
+ missed, unreachable = saved.get('miss', 0), saved.get('cover_network', 0)
for index, code in enumerate(codes):
- if code in completed:
+ if code in done:
continue
handle.progress(current=index, total=len(codes),
label=f"{TASK_LABEL}:{code}", throttle=0)
@@ -401,12 +405,10 @@ def run(contract, key: str, handle) -> dict:
if row is None:
# 壳被删了,或者这个番号这中间已经入库。两种都不该再取一遍。
continue
- metadata_ok = row['scraped_at'] is not None and row['scrape_error'] is None
if row["scraped_at"] is None or row["scrape_error"] is not None:
provider = provider or LibraryMetadataProvider(contract.follow_secrets_root)
if _scrape_fields(contract, provider, row):
fetched += 1
- metadata_ok = True
else:
missed += 1
# 封面地址在取资料那一步才写上壳,所以取完再读一遍,不用循环开头那份。
@@ -414,10 +416,7 @@ def run(contract, key: str, handle) -> dict:
cover_url = connection.execute(
"SELECT cover_url FROM feed_discovery WHERE id=?", (int(row["id"]),)).fetchone()
try:
- covered = ensure_cover(contract, code, cover_url[0] if cover_url else None)
- covers += covered
- if metadata_ok and covered:
- completed.add(code)
+ covers += ensure_cover(contract, code, cover_url[0] if cover_url else None)
except (CoverConnectError, httpx.TransportError):
unreachable += 1
except Exception: # noqa: BLE001 - 取不到封面这一部照样有资料,下一轮再试
@@ -426,7 +425,9 @@ def run(contract, key: str, handle) -> dict:
with contract.database.write_transaction() as connection:
connection.execute("UPDATE feed_discovery SET scraped_at=? WHERE id=?",
(feeds.stamp(), int(row["id"])))
- handle.checkpoint(completed_codes=sorted(completed), fetched=fetched, covers=covers)
+ done.add(code)
+ handle.checkpoint(done_codes=sorted(done), fetched=fetched, covers=covers,
+ miss=missed, cover_network=unreachable)
handle.progress(current=len(codes), total=len(codes), label=TASK_LABEL, throttle=0)
contract.cache_bust()
summary = {"total": len(codes), "ok": fetched, "miss": missed, "covers": covers}
diff --git a/tests/test_feeds_web.py b/tests/test_feeds_web.py
index 0f14b0691..07f60ba52 100644
--- a/tests/test_feeds_web.py
+++ b/tests/test_feeds_web.py
@@ -10,6 +10,7 @@ import time
import unittest
from datetime import timedelta
from pathlib import Path
+from types import SimpleNamespace
from unittest.mock import patch
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "src"))
@@ -728,6 +729,24 @@ class FeedWebTest(FeedWebFixture):
[run] = self.contract.task_runs.query(task_key=feed_followup.COVER_TASK_KEY)
self.assertEqual(run.result_summary, {"total": 1, "covers": 1})
+ def test_a_resumed_batch_does_not_retry_a_code_it_already_tried(self):
+ """服务重启后续跑这一批:上次试过没取到的那部不再在批首重耗一遍,计数接着上次的。"""
+ self._add()
+ self._check()
+ thread = self.contract.feed_job.thread
+ if thread is not None:
+ thread.join(5)
+ asked = []
+ collect = feed_followup.collect
+ feed_followup.collect = lambda provider, code: asked.append(code) or collect(provider, code)
+ saved = {"done_codes": ["SSIS-950"], "fetched": 0, "covers": 0, "miss": 1}
+ handle = SimpleNamespace(saved=saved, progress=lambda **_: None,
+ checkpoint=lambda **fields: saved.update(fields))
+ summary = feed_followup.run(self.contract, "feed-scrape:SSIS-950,HMN-071", handle)
+ self.assertEqual(asked, ["HMN-071"])
+ self.assertEqual(summary, {"total": 2, "ok": 1, "miss": 1, "covers": 1})
+ self.assertEqual(saved["done_codes"], ["HMN-071", "SSIS-950"])
+
def test_covers_cut_off_at_the_connection_are_counted_for_the_new_release_row(self):
"""连不上图片主机的几部单独记数,列表把最近一轮的数带给页面去提示换线路;
官方没图的那几部不算进去。"""