Peach: Local-first video metadata scraper and media manager.
fix(feeds): 取资料续跑只接着试余下的番号
longmeidao414b4a4bae7f
diff --git a/src/peach/feed_followup.py b/src/peach/feed_followup.pyindex a85f1934d..effdc17d5 100644--- a/src/peach/feed_followup.py+++ b/src/peach/feed_followup.py@@ -376,6 +376,10 @@ def run(contract, key: str, handle) -> dict:封面断在连接上的几部另记 `cover_network`:这一类换条线路就能取到,新作页据此提示去配来源的连接方式,而不是让人以为官方没出图。++ 检查点记的是这一批里已经试过的番号,不论取没取到:服务重启后这一条重新领取时只接着+ 试余下的。没取到的那几部由 `scrape_batch`、`cover_backlog` 到了重试时间再排,不在+ 同一批的续跑里每次重启都重耗一遍。"""import httpx@@ -386,11 +390,11 @@ def run(contract, key: str, handle) -> dict:provider = Nonesaved = getattr(handle, 'saved', {})saved = saved if isinstance(saved, dict) else {}- completed = set(saved.get('completed_codes') or [])+ done = set(saved.get('done_codes') or [])fetched, covers = saved.get('fetched', 0), saved.get('covers', 0)- missed = unreachable = 0+ missed, unreachable = saved.get('miss', 0), saved.get('cover_network', 0)for index, code in enumerate(codes):- if code in completed:+ if code in done:continuehandle.progress(current=index, total=len(codes),label=f"{TASK_LABEL}:{code}", throttle=0)@@ -401,12 +405,10 @@ def run(contract, key: str, handle) -> dict:if row is None:# 壳被删了,或者这个番号这中间已经入库。两种都不该再取一遍。continue- metadata_ok = row['scraped_at'] is not None and row['scrape_error'] is Noneif row["scraped_at"] is None or row["scrape_error"] is not None:provider = provider or LibraryMetadataProvider(contract.follow_secrets_root)if _scrape_fields(contract, provider, row):fetched += 1- metadata_ok = Trueelse:missed += 1# 封面地址在取资料那一步才写上壳,所以取完再读一遍,不用循环开头那份。@@ -414,10 +416,7 @@ def run(contract, key: str, handle) -> dict:cover_url = connection.execute("SELECT cover_url FROM feed_discovery WHERE id=?", (int(row["id"]),)).fetchone()try:- covered = ensure_cover(contract, code, cover_url[0] if cover_url else None)- covers += covered- if metadata_ok and covered:- completed.add(code)+ covers += ensure_cover(contract, code, cover_url[0] if cover_url else None)except (CoverConnectError, httpx.TransportError):unreachable += 1except Exception: # noqa: BLE001 - 取不到封面这一部照样有资料,下一轮再试@@ -426,7 +425,9 @@ def run(contract, key: str, handle) -> dict:with contract.database.write_transaction() as connection:connection.execute("UPDATE feed_discovery SET scraped_at=? WHERE id=?",(feeds.stamp(), int(row["id"])))- handle.checkpoint(completed_codes=sorted(completed), fetched=fetched, covers=covers)+ done.add(code)+ handle.checkpoint(done_codes=sorted(done), fetched=fetched, covers=covers,+ miss=missed, cover_network=unreachable)handle.progress(current=len(codes), total=len(codes), label=TASK_LABEL, throttle=0)contract.cache_bust()summary = {"total": len(codes), "ok": fetched, "miss": missed, "covers": covers}diff --git a/tests/test_feeds_web.py b/tests/test_feeds_web.pyindex 0f14b0691..07f60ba52 100644--- a/tests/test_feeds_web.py+++ b/tests/test_feeds_web.py@@ -10,6 +10,7 @@ import timeimport unittestfrom datetime import timedeltafrom pathlib import Path+from types import SimpleNamespacefrom unittest.mock import patchsys.path.insert(0, str(Path(__file__).resolve().parents[1] / "src"))@@ -728,6 +729,24 @@ class FeedWebTest(FeedWebFixture):[run] = self.contract.task_runs.query(task_key=feed_followup.COVER_TASK_KEY)self.assertEqual(run.result_summary, {"total": 1, "covers": 1})+ def test_a_resumed_batch_does_not_retry_a_code_it_already_tried(self):+ """服务重启后续跑这一批:上次试过没取到的那部不再在批首重耗一遍,计数接着上次的。"""+ self._add()+ self._check()+ thread = self.contract.feed_job.thread+ if thread is not None:+ thread.join(5)+ asked = []+ collect = feed_followup.collect+ feed_followup.collect = lambda provider, code: asked.append(code) or collect(provider, code)+ saved = {"done_codes": ["SSIS-950"], "fetched": 0, "covers": 0, "miss": 1}+ handle = SimpleNamespace(saved=saved, progress=lambda **_: None,+ checkpoint=lambda **fields: saved.update(fields))+ summary = feed_followup.run(self.contract, "feed-scrape:SSIS-950,HMN-071", handle)+ self.assertEqual(asked, ["HMN-071"])+ self.assertEqual(summary, {"total": 2, "ok": 1, "miss": 1, "covers": 1})+ self.assertEqual(saved["done_codes"], ["HMN-071", "SSIS-950"])+def test_covers_cut_off_at_the_connection_are_counted_for_the_new_release_row(self):"""连不上图片主机的几部单独记数,列表把最近一轮的数带给页面去提示换线路;官方没图的那几部不算进去。"""