Peach: Local-first video metadata scraper and media manager.
fix(follow): 逐项保存关注采集结果并保留中断进度
longmeidao891a55b8adce
diff --git a/src/peach/follow_check.py b/src/peach/follow_check.pyindex 5b94ef741..24918082a 100644--- a/src/peach/follow_check.py+++ b/src/peach/follow_check.py@@ -4,8 +4,8 @@ Web 与命令行共用这一份。各写一遍必然分岔:其中一份漏掉之后的强制重取、不学官方渠道的作者别名,于是同一句「检查更新」在网页上做的事比在终端里多,而没有任何地方说明这一点。-网络与落库都在这里,事务边界由调用方传进来的 `writer` 决定:Web 每条来源一个写-事务,命令行是一条连接加显式 commit。展示不在这里,`CheckResult` 交回给调用方。+网络与落库都在这里,已确认条目各自交给调用方传入的 `writer` 提交;整页完成后+才推进来源游标。展示不在这里,`CheckResult` 交回给调用方。"""from __future__ import annotations@@ -125,6 +125,7 @@ def run_check(row: Mapping, *, credentials, writer, connector_factory,force = bool(row.get("force_media_reparse")) or replay_firstcutoff, floor = _history_limits(row, metadata, writer, moment,older=older, initial_days=initial_days)+ recorder = _PageRecorder(row, writer, moment, cutoff)try:connector = build_connector_for(provider, credentials, connector_factory,@@ -132,6 +133,7 @@ def run_check(row: Mapping, *, credentials, writer, connector_factory,connector.history_after = cutoffconnector.history_skipped = 0connector.history_floor = floor+ connector.candidate_ready = recorder.publishif progress is not None:connector.progress = progressfetch = connector.fetch(@@ -158,12 +160,11 @@ def run_check(row: Mapping, *, credentials, writer, connector_factory,store.merge_source_metadata(source_id, {'initial_history_first_page_pending': False}, moment)elif history_skipped and page == 0 and 'initial_history_first_page_pending' not in metadata:store.merge_source_metadata(source_id, {'initial_history_first_page_pending': True}, moment)- outcome = store.record(- source_id, fetch,- creator_aliases=store.creator_aliases(row["entity_id"]),- moment=moment, page=page)+ outcome = recorder.record_in(store, fetch, page)if floor and not fetch.not_modified:- fetch, outcome = _fill_floor(row, connector, writer, fetch, outcome, cutoff, floor, moment)+ recorder.cutoff = None+ fetch, outcome = _fill_floor(row, connector, writer, fetch, outcome, cutoff, floor, moment,+ recorder=recorder)with writer() as store:learned = store.learn_official_author_alias(provider, ref, fetch.candidates)return CheckResult(**base, fetch=fetch, outcome=outcome,@@ -196,7 +197,7 @@ def _history_limits(row, metadata: dict, writer, moment: datetime, *,return datetime.fromisoformat(history_after), floor-def _fill_floor(row, connector, writer, fetch, outcome, cutoff, floor, moment):+def _fill_floor(row, connector, writer, fetch, outcome, cutoff, floor, moment, *, recorder):"""首次采集不足下限时往前翻页补足,再把边界挪到实际收下的最早一条。往前翻的页整页收下,回填游标因此和「已经抓到第几页」一致,之后手动加载更早@@ -219,10 +220,7 @@ def _fill_floor(row, connector, writer, fetch, outcome, cutoff, floor, moment):except (CredentialError, FollowSourceError):# 首页已经落库,补页失败只是这次补得少,之后手动加载更早还能接着翻。break- with writer() as store:- extra = store.record(source_id, more,- creator_aliases=store.creator_aliases(row["entity_id"]),- moment=moment, page=page)+ extra = recorder.record(more, page)kept.extend(more.candidates)for key in totals:totals[key] += getattr(extra, key)@@ -243,3 +241,39 @@ def _record_failure(writer, base: dict, error: Exception, moment: datetime,with writer() as store:store.record_error(base["source_id"], str(error), moment, status=status)return CheckResult(**base, ok=False, status=status, error=str(error))+++class _PageRecorder:+ """逐项提交可展示的结果,整页完成后才保存来源游标与完成状态。"""++ def __init__(self, row, writer, moment, cutoff):+ self.row, self.writer, self.moment, self.cutoff = row, writer, moment, cutoff+ self.added_ids: set[str] = set()++ def publish(self, candidate):+ # 有历史边界时,详情可能给出更早的日期;列表行由整页最终筛选接纳。+ if self.cutoff is not None and candidate.partial:+ return+ if not history_window((candidate,), self.cutoff):+ return+ fetch = SourceFetch(provider=str(self.row['provider']), ref=str(self.row['ref']),+ request_url=str(self.row['url']), semantics=str(self.row['semantics']),+ candidates=(candidate,))+ with self.writer() as store:+ outcome = store.record(int(self.row['id']), fetch,+ creator_aliases=store.creator_aliases(self.row['entity_id']),+ moment=self.moment, complete=False)+ if outcome.added:+ self.added_ids.add(str(candidate.external_id))++ def record(self, fetch, page):+ with self.writer() as store:+ return self.record_in(store, fetch, page)++ def record_in(self, store, fetch, page):+ outcome = store.record(int(self.row['id']), fetch,+ creator_aliases=store.creator_aliases(self.row['entity_id']),+ moment=self.moment, page=page)+ added = len(self.added_ids.intersection(str(c.external_id) for c in fetch.candidates))+ self.added_ids.clear()+ return replace(outcome, added=outcome.added + added, updated=outcome.updated - added)diff --git a/src/peach/follow_sources.py b/src/peach/follow_sources.pyindex e7b0d62ae..eb5b6b703 100644--- a/src/peach/follow_sources.py+++ b/src/peach/follow_sources.py@@ -474,6 +474,7 @@ class _BaseConnector:#: 可断言又不真的把测试拖成几十秒。self.sleeper = sleeperself.progress = None+ self.candidate_ready = Noneself.history_after = Noneself.history_skipped = 0#: 首次采集的条数下限。边界之外最多放行这么多条,精确裁剪由 `run_check`@@ -682,18 +683,24 @@ class _BaseConnector:limit = self.enrich_budgetskipped = self.enrich_skipcandidates = tuple(candidate for candidate in candidates if self.within_history(candidate))- if not limit:- return tuple(candidates), 0+ for candidate in candidates:+ self.publish_candidate(candidate)spent = 0result: list[FollowCandidate] = []for candidate in candidates:if spent >= limit or str(candidate.external_id) in skipped:result.append(candidate)- continue- spent += 1- result.append(self._enrich_one(candidate))+ else:+ spent += 1+ result.append(self._enrich_one(candidate))+ self.publish_candidate(result[-1])return tuple(result), spent+ def publish_candidate(self, candidate: FollowCandidate) -> None:+ """把已通过当前条目判据的结果交回调用方,分页完成由 fetch 回执声明。"""+ if self.candidate_ready is not None:+ self.candidate_ready(candidate)+def _enrich_one(self, candidate: FollowCandidate) -> FollowCandidate:"""补全一条候选。取不到就原样返回(仍是 `partial`),不猜、不写死。"""return candidate@@ -904,6 +911,7 @@ class KemonoConnector(_BaseConnector):skipped += 1continuekept.append(self._candidate(post, service, user))+ self.publish_candidate(kept[-1])return SourceFetch(candidates=tuple(kept), skipped=skipped, probed=probed,raw_body=response.body, **common)@@ -1233,6 +1241,7 @@ class Rule34VideoConnector(_BaseConnector):skipped_compilations += 1continuecandidates.append(enriched)+ self.publish_candidate(enriched)return SourceFetch(candidates=tuple(candidates), skipped=skipped_compilations,skipped_compilations=skipped_compilations, probed=probed,@@ -2194,6 +2203,7 @@ class F95ZoneConnector(_BaseConnector):"gofile_video_count": sum(item.get("media_kind") == "video" for item in media_items)},))+ self.publish_candidate(enriched[-1])return SourceFetch(candidates=tuple(enriched), skipped=skipped,raw_body=response.body, **common)diff --git a/src/peach/follow_store.py b/src/peach/follow_store.pyindex 87f603878..217e9524b 100644--- a/src/peach/follow_store.py+++ b/src/peach/follow_store.py@@ -420,7 +420,8 @@ class FollowStore:def record(self, source_id: int, fetch: SourceFetch, *,creator_aliases: tuple[str, ...] = (),- moment: datetime | None = None, page: int = 0) -> RecordOutcome:+ moment: datetime | None = None, page: int = 0,+ complete: bool = True) -> RecordOutcome:stamp = _now_text(moment)connection = self._connect()if fetch.not_modified:@@ -429,8 +430,9 @@ class FollowStore:" last_error=NULL, updated_at=? WHERE id=?", (stamp, stamp, source_id))return RecordOutcome(source_id, not_modified=True)- evidence, evidence_error = self._persist_evidence(fetch, moment)- self._replace_candidate_profile_links(source_id, fetch, moment=moment, page=page)+ evidence, evidence_error = self._persist_evidence(fetch, moment) if complete else (None, None)+ if complete:+ self._replace_candidate_profile_links(source_id, fetch, moment=moment, page=page)added = updated = 0for candidate in fetch.candidates:verdict = classify(candidate.title, creator_aliases=creator_aliases,@@ -478,13 +480,16 @@ class FollowStore:" ON CONFLICT(source_id,external_id) DO UPDATE SET"# status、asset_id 与 first_seen_at 刻意不在两个 SET 里:用户处理过的# 条目不因为再次抓到就退回 new,首见时间也不该被重写。- + (_PARTIAL_UPDATE if candidate.partial else _FULL_UPDATE),+ + (_PARTIAL_UPDATE if candidate.partial else _FULL_UPDATE)+ + ", evidence_path=COALESCE(follow_item.evidence_path, excluded.evidence_path)",values,)if existed:updated += 1else:added += 1+ if not complete:+ return RecordOutcome(source_id, len(fetch.candidates), added, updated)if page:# 往回抓:**不动 etag/last_modified**。那两个值是第一页的条件请求凭据,# 拿第 3 页的 etag 覆盖掉,下次常规检查就会拿它去问第一页,diff --git a/tests/test_follow_check.py b/tests/test_follow_check.pyindex 97986d77e..ca725d854 100644--- a/tests/test_follow_check.py+++ b/tests/test_follow_check.py@@ -8,6 +8,7 @@ import jsonimport sqlite3import tempfileimport unittest+from dataclasses import replacefrom datetime import datetime, timedelta, timezonefrom pathlib import Path@@ -15,7 +16,7 @@ from peach.follow import FollowHistoryEnd, FollowSourceError, FollowSourceRateLifrom peach.follow_check import plan_check, run_checkfrom peach.follow_stream import FollowMediaResolverfrom peach.follow_secrets import CredentialError-from peach.follow_sources import FollowCandidate, SourceFetch+from peach.follow_sources import FollowCandidate, SourceFetch, _BaseConnectorfrom peach.follow_store import FollowStorefrom support.ledger import fresh_ledger@@ -86,6 +87,117 @@ class _CheckCase(unittest.TestCase):moment=MOMENT, **kwargs)+class IncrementalCheckTests(_CheckCase):+ @contextlib.contextmanager+ def writer(self):+ with self.connection:+ yield self.store++ def _reader(self):+ database = self.connection.execute('PRAGMA database_list').fetchone()[2]+ connection = sqlite3.connect(database)+ connection.row_factory = sqlite3.Row+ self.addCleanup(connection.close)+ return connection++ def _connector(self, enrich_one):+ class Connector(_BaseConnector):+ def fetch(self, ref, **kwargs):+ candidates, probed = self.enrich(tuple(+ FollowCandidate(provider='fanbox', external_id=str(index),+ title=f'作品 {index}', partial=True)+ for index in (1, 2)))+ return _fetch(candidates=candidates, probed=probed, etag='finished', raw_body=b'page')++ def _enrich_one(self, candidate):+ return enrich_one(candidate)++ return Connector(enrich_budget=2)++ def test_a_completed_item_is_visible_while_the_next_item_is_being_enriched(self):+ source = self._register()+ self.connection.commit()+ reader = self._reader()++ def enrich(candidate):+ if candidate.external_id == '1':+ saved = reader.execute('SELECT * FROM follow_item ORDER BY external_id').fetchall()+ self.assertEqual([row['external_id'] for row in saved], ['1', '2'])+ self.assertTrue(all('post_type' not in json.loads(row['metadata_json']) for row in saved))+ if candidate.external_id == '2':+ saved = reader.execute('SELECT * FROM follow_item ORDER BY external_id').fetchall()+ self.assertEqual([row['external_id'] for row in saved], ['1', '2'])+ self.assertEqual(json.loads(saved[0]['metadata_json'])['post_type'], 'image')+ self.assertNotIn('post_type', json.loads(saved[1]['metadata_json']))+ self.assertIsNone(saved[0]['evidence_path'])+ pending = reader.execute('SELECT * FROM follow_source WHERE id=?', (source,)).fetchone()+ self.assertIsNone(pending['last_checked_at'])+ self.assertIsNone(pending['etag'])+ self.assertEqual(pending['backfill_page'], 0)+ return replace(candidate, partial=False, extra={'post_type': 'image'})++ result = self._run(source, self._connector(enrich))+ self.assertEqual((result.outcome.discovered, result.outcome.added, result.outcome.updated),+ (2, 2, 0))+ self.assertEqual(self._row(source)['etag'], 'finished')+ self.assertTrue(all(row[0] for row in reader.execute('SELECT evidence_path FROM follow_item')))+ again = self._run(source, self._connector(lambda candidate: candidate))+ self.assertEqual((again.outcome.added, again.outcome.updated), (0, 2))++ def test_a_rate_limit_keeps_completed_items_and_the_page_can_be_retried(self):+ source = self._register()+ self.connection.commit()++ def enrich(candidate):+ if candidate.external_id == '2':+ raise FollowSourceRateLimited('稍后再试', 120)+ return replace(candidate, partial=False, extra={'post_type': 'image'})++ result = self._run(source, self._connector(enrich))+ self.assertFalse(result.ok)+ reader = self._reader()+ saved = reader.execute('SELECT * FROM follow_item ORDER BY external_id').fetchall()+ self.assertEqual([row['external_id'] for row in saved], ['1', '2'])+ self.assertEqual(json.loads(saved[0]['metadata_json'])['post_type'], 'image')+ self.assertNotIn('post_type', json.loads(saved[1]['metadata_json']))+ row = self._row(source)+ self.assertEqual(row['last_status'], 'error')+ self.assertIsNone(row['etag'])+ self.assertEqual(row['backfill_page'], 0)+ retry = self._run(source, self._connector(lambda candidate: candidate))+ self.assertEqual((retry.outcome.added, retry.outcome.updated), (0, 2))+ self.assertEqual(len(self.store.items(source_id=source)), 2)++ def test_an_interrupted_history_page_keeps_items_without_advancing_its_cursor(self):+ source = self._register()+ self.connection.commit()++ def enrich(candidate):+ if candidate.external_id == '2':+ raise InterruptedError('任务中断')+ return candidate++ with self.assertRaises(InterruptedError):+ self._run(source, self._connector(enrich), older=True)+ self.assertEqual(len(self.store.items(source_id=source)), 2)+ self.assertEqual(self._row(source)['backfill_page'], 0)+ self.assertIsNone(self._row(source)['last_checked_at'])++ def test_detail_dates_enforce_the_persisted_history_boundary(self):+ source = self._register()+ self.store.merge_source_metadata(source, {+ 'initial_history_after': '2026-08-04T00:00:00Z', 'initial_history_floor': 0}, MOMENT)+ self.connection.commit()++ def enrich(candidate):+ self.assertEqual(len(self.store.items(source_id=source)), 0)+ return replace(candidate, partial=False, published_at='2025-01-01T00:00:00Z')++ result = self._run(source, self._connector(enrich))+ self.assertEqual(result.outcome.discovered, 0)+ self.assertEqual(len(self.store.items(source_id=source)), 0)++class SecondPhaseHandoffTests(_CheckCase):def test_the_skip_set_reaches_the_connector(self):"""算出来的跳过集合要真的传给连接器,否则第二阶段照样按条目数付请求。"""diff --git a/tests/test_follow_sources.py b/tests/test_follow_sources.pyindex 1196de025..ce574405c 100644--- a/tests/test_follow_sources.py+++ b/tests/test_follow_sources.py@@ -1046,7 +1046,7 @@ class Rule34VideoConnectorTests(unittest.TestCase):self.assertTrue(first.thumb_url.endswith("/preview.jpg"))self.assertIn("4542721_360.mp4", first.media_url)- def _detail_with_credits(self, names):+ def _detail_with_credits(self, names, *, ready=None):credits = "".join(f'<a class="item btn_link video_meta_pill"'f' href="https://rule34video.com/models/m{n}/">{name}</a>'@@ -1058,12 +1058,16 @@ class Rule34VideoConnectorTests(unittest.TestCase):return HttpResponse(200, {},RULE34VIDEO_HTML if "/models/" in request.url else detail)- return Rule34VideoConnector(transport=transport).fetch("lazyprocrastinator")+ connector = Rule34VideoConnector(transport=transport)+ connector.candidate_ready = ready+ return connector.fetch("lazyprocrastinator")def test_detail_with_many_credited_models_is_not_collected(self):# 样本自带 LazyProcrastinator 一位,这里再添三位就是四位画面作者。- result = self._detail_with_credits(["M1", "M2", "M3"])+ ready = []+ result = self._detail_with_credits(["M1", "M2", "M3"], ready=ready.append)self.assertEqual(result.candidates, ())+ self.assertEqual(ready, [])self.assertEqual(result.skipped, 2)self.assertEqual(result.skipped_compilations, 2)@@ -2214,6 +2218,21 @@ class EnrichPhaseTests(unittest.TestCase):self.assertEqual(result.probed, 0)self.assertTrue(all(item.partial for item in result.candidates))+ def test_each_enriched_item_is_published_before_the_next_detail_request(self):+ seen, ready = [], []+ connector = self._rule34xxx(seen, enrich_budget=2)++ def publish(candidate):+ ready.append(candidate)+ enriched = sum(not item.partial for item in ready)+ self.assertEqual(sum('s=view' in url for url in seen), enriched)++ connector.candidate_ready = publish+ result = connector.fetch('lazyprocrastinator')+ self.assertEqual([item.external_id for item in ready[:4]],+ [item.external_id for item in result.candidates])+ self.assertEqual([item.partial for item in ready], [True, True, True, True, False, False])+def test_the_skip_set_is_spent_on_the_condition_not_on_the_budget(self):"""跳过的条目不占额度:额度是为「还没补齐的那几条」留的。"""seen = []