Skip to content

armgit / peach

Public Imported from GitHub
0

Peach: Local-first video metadata scraper and media manager.

fix(follow): 逐项保存关注采集结果并保留中断进度

longmeidao891a55b8adce
diff --git a/src/peach/follow_check.py b/src/peach/follow_check.py
index 5b94ef741..24918082a 100644
--- a/src/peach/follow_check.py
+++ b/src/peach/follow_check.py
@@ -4,8 +4,8 @@ Web 与命令行共用这一份。各写一遍必然分岔:其中一份漏掉
之后的强制重取、不学官方渠道的作者别名,于是同一句「检查更新」在网页上做的事比
在终端里多,而没有任何地方说明这一点。
-网络与落库都在这里,事务边界由调用方传进来的 `writer` 决定:Web 每条来源一个写
-事务,命令行是一条连接加显式 commit。展示不在这里,`CheckResult` 交回给调用方。
+网络与落库都在这里,已确认条目各自交给调用方传入的 `writer` 提交;整页完成后
+才推进来源游标。展示不在这里,`CheckResult` 交回给调用方。
"""
from __future__ import annotations
@@ -125,6 +125,7 @@ def run_check(row: Mapping, *, credentials, writer, connector_factory,
force = bool(row.get("force_media_reparse")) or replay_first
cutoff, floor = _history_limits(row, metadata, writer, moment,
older=older, initial_days=initial_days)
+ recorder = _PageRecorder(row, writer, moment, cutoff)
try:
connector = build_connector_for(
provider, credentials, connector_factory,
@@ -132,6 +133,7 @@ def run_check(row: Mapping, *, credentials, writer, connector_factory,
connector.history_after = cutoff
connector.history_skipped = 0
connector.history_floor = floor
+ connector.candidate_ready = recorder.publish
if progress is not None:
connector.progress = progress
fetch = connector.fetch(
@@ -158,12 +160,11 @@ def run_check(row: Mapping, *, credentials, writer, connector_factory,
store.merge_source_metadata(source_id, {'initial_history_first_page_pending': False}, moment)
elif history_skipped and page == 0 and 'initial_history_first_page_pending' not in metadata:
store.merge_source_metadata(source_id, {'initial_history_first_page_pending': True}, moment)
- outcome = store.record(
- source_id, fetch,
- creator_aliases=store.creator_aliases(row["entity_id"]),
- moment=moment, page=page)
+ outcome = recorder.record_in(store, fetch, page)
if floor and not fetch.not_modified:
- fetch, outcome = _fill_floor(row, connector, writer, fetch, outcome, cutoff, floor, moment)
+ recorder.cutoff = None
+ fetch, outcome = _fill_floor(row, connector, writer, fetch, outcome, cutoff, floor, moment,
+ recorder=recorder)
with writer() as store:
learned = store.learn_official_author_alias(provider, ref, fetch.candidates)
return CheckResult(**base, fetch=fetch, outcome=outcome,
@@ -196,7 +197,7 @@ def _history_limits(row, metadata: dict, writer, moment: datetime, *,
return datetime.fromisoformat(history_after), floor
-def _fill_floor(row, connector, writer, fetch, outcome, cutoff, floor, moment):
+def _fill_floor(row, connector, writer, fetch, outcome, cutoff, floor, moment, *, recorder):
"""首次采集不足下限时往前翻页补足,再把边界挪到实际收下的最早一条。
往前翻的页整页收下,回填游标因此和「已经抓到第几页」一致,之后手动加载更早
@@ -219,10 +220,7 @@ def _fill_floor(row, connector, writer, fetch, outcome, cutoff, floor, moment):
except (CredentialError, FollowSourceError):
# 首页已经落库,补页失败只是这次补得少,之后手动加载更早还能接着翻。
break
- with writer() as store:
- extra = store.record(source_id, more,
- creator_aliases=store.creator_aliases(row["entity_id"]),
- moment=moment, page=page)
+ extra = recorder.record(more, page)
kept.extend(more.candidates)
for key in totals:
totals[key] += getattr(extra, key)
@@ -243,3 +241,39 @@ def _record_failure(writer, base: dict, error: Exception, moment: datetime,
with writer() as store:
store.record_error(base["source_id"], str(error), moment, status=status)
return CheckResult(**base, ok=False, status=status, error=str(error))
+
+
+class _PageRecorder:
+ """逐项提交可展示的结果,整页完成后才保存来源游标与完成状态。"""
+
+ def __init__(self, row, writer, moment, cutoff):
+ self.row, self.writer, self.moment, self.cutoff = row, writer, moment, cutoff
+ self.added_ids: set[str] = set()
+
+ def publish(self, candidate):
+ # 有历史边界时,详情可能给出更早的日期;列表行由整页最终筛选接纳。
+ if self.cutoff is not None and candidate.partial:
+ return
+ if not history_window((candidate,), self.cutoff):
+ return
+ fetch = SourceFetch(provider=str(self.row['provider']), ref=str(self.row['ref']),
+ request_url=str(self.row['url']), semantics=str(self.row['semantics']),
+ candidates=(candidate,))
+ with self.writer() as store:
+ outcome = store.record(int(self.row['id']), fetch,
+ creator_aliases=store.creator_aliases(self.row['entity_id']),
+ moment=self.moment, complete=False)
+ if outcome.added:
+ self.added_ids.add(str(candidate.external_id))
+
+ def record(self, fetch, page):
+ with self.writer() as store:
+ return self.record_in(store, fetch, page)
+
+ def record_in(self, store, fetch, page):
+ outcome = store.record(int(self.row['id']), fetch,
+ creator_aliases=store.creator_aliases(self.row['entity_id']),
+ moment=self.moment, page=page)
+ added = len(self.added_ids.intersection(str(c.external_id) for c in fetch.candidates))
+ self.added_ids.clear()
+ return replace(outcome, added=outcome.added + added, updated=outcome.updated - added)
diff --git a/src/peach/follow_sources.py b/src/peach/follow_sources.py
index e7b0d62ae..eb5b6b703 100644
--- a/src/peach/follow_sources.py
+++ b/src/peach/follow_sources.py
@@ -474,6 +474,7 @@ class _BaseConnector:
#: 可断言又不真的把测试拖成几十秒。
self.sleeper = sleeper
self.progress = None
+ self.candidate_ready = None
self.history_after = None
self.history_skipped = 0
#: 首次采集的条数下限。边界之外最多放行这么多条,精确裁剪由 `run_check`
@@ -682,18 +683,24 @@ class _BaseConnector:
limit = self.enrich_budget
skipped = self.enrich_skip
candidates = tuple(candidate for candidate in candidates if self.within_history(candidate))
- if not limit:
- return tuple(candidates), 0
+ for candidate in candidates:
+ self.publish_candidate(candidate)
spent = 0
result: list[FollowCandidate] = []
for candidate in candidates:
if spent >= limit or str(candidate.external_id) in skipped:
result.append(candidate)
- continue
- spent += 1
- result.append(self._enrich_one(candidate))
+ else:
+ spent += 1
+ result.append(self._enrich_one(candidate))
+ self.publish_candidate(result[-1])
return tuple(result), spent
+ def publish_candidate(self, candidate: FollowCandidate) -> None:
+ """把已通过当前条目判据的结果交回调用方,分页完成由 fetch 回执声明。"""
+ if self.candidate_ready is not None:
+ self.candidate_ready(candidate)
+
def _enrich_one(self, candidate: FollowCandidate) -> FollowCandidate:
"""补全一条候选。取不到就原样返回(仍是 `partial`),不猜、不写死。"""
return candidate
@@ -904,6 +911,7 @@ class KemonoConnector(_BaseConnector):
skipped += 1
continue
kept.append(self._candidate(post, service, user))
+ self.publish_candidate(kept[-1])
return SourceFetch(candidates=tuple(kept), skipped=skipped, probed=probed,
raw_body=response.body, **common)
@@ -1233,6 +1241,7 @@ class Rule34VideoConnector(_BaseConnector):
skipped_compilations += 1
continue
candidates.append(enriched)
+ self.publish_candidate(enriched)
return SourceFetch(
candidates=tuple(candidates), skipped=skipped_compilations,
skipped_compilations=skipped_compilations, probed=probed,
@@ -2194,6 +2203,7 @@ class F95ZoneConnector(_BaseConnector):
"gofile_video_count": sum(
item.get("media_kind") == "video" for item in media_items)},
))
+ self.publish_candidate(enriched[-1])
return SourceFetch(candidates=tuple(enriched), skipped=skipped,
raw_body=response.body, **common)
diff --git a/src/peach/follow_store.py b/src/peach/follow_store.py
index 87f603878..217e9524b 100644
--- a/src/peach/follow_store.py
+++ b/src/peach/follow_store.py
@@ -420,7 +420,8 @@ class FollowStore:
def record(self, source_id: int, fetch: SourceFetch, *,
creator_aliases: tuple[str, ...] = (),
- moment: datetime | None = None, page: int = 0) -> RecordOutcome:
+ moment: datetime | None = None, page: int = 0,
+ complete: bool = True) -> RecordOutcome:
stamp = _now_text(moment)
connection = self._connect()
if fetch.not_modified:
@@ -429,8 +430,9 @@ class FollowStore:
" last_error=NULL, updated_at=? WHERE id=?", (stamp, stamp, source_id))
return RecordOutcome(source_id, not_modified=True)
- evidence, evidence_error = self._persist_evidence(fetch, moment)
- self._replace_candidate_profile_links(source_id, fetch, moment=moment, page=page)
+ evidence, evidence_error = self._persist_evidence(fetch, moment) if complete else (None, None)
+ if complete:
+ self._replace_candidate_profile_links(source_id, fetch, moment=moment, page=page)
added = updated = 0
for candidate in fetch.candidates:
verdict = classify(candidate.title, creator_aliases=creator_aliases,
@@ -478,13 +480,16 @@ class FollowStore:
" ON CONFLICT(source_id,external_id) DO UPDATE SET"
# status、asset_id 与 first_seen_at 刻意不在两个 SET 里:用户处理过的
# 条目不因为再次抓到就退回 new,首见时间也不该被重写。
- + (_PARTIAL_UPDATE if candidate.partial else _FULL_UPDATE),
+ + (_PARTIAL_UPDATE if candidate.partial else _FULL_UPDATE)
+ + ", evidence_path=COALESCE(follow_item.evidence_path, excluded.evidence_path)",
values,
)
if existed:
updated += 1
else:
added += 1
+ if not complete:
+ return RecordOutcome(source_id, len(fetch.candidates), added, updated)
if page:
# 往回抓:**不动 etag/last_modified**。那两个值是第一页的条件请求凭据,
# 拿第 3 页的 etag 覆盖掉,下次常规检查就会拿它去问第一页,
diff --git a/tests/test_follow_check.py b/tests/test_follow_check.py
index 97986d77e..ca725d854 100644
--- a/tests/test_follow_check.py
+++ b/tests/test_follow_check.py
@@ -8,6 +8,7 @@ import json
import sqlite3
import tempfile
import unittest
+from dataclasses import replace
from datetime import datetime, timedelta, timezone
from pathlib import Path
@@ -15,7 +16,7 @@ from peach.follow import FollowHistoryEnd, FollowSourceError, FollowSourceRateLi
from peach.follow_check import plan_check, run_check
from peach.follow_stream import FollowMediaResolver
from peach.follow_secrets import CredentialError
-from peach.follow_sources import FollowCandidate, SourceFetch
+from peach.follow_sources import FollowCandidate, SourceFetch, _BaseConnector
from peach.follow_store import FollowStore
from support.ledger import fresh_ledger
@@ -86,6 +87,117 @@ class _CheckCase(unittest.TestCase):
moment=MOMENT, **kwargs)
+class IncrementalCheckTests(_CheckCase):
+ @contextlib.contextmanager
+ def writer(self):
+ with self.connection:
+ yield self.store
+
+ def _reader(self):
+ database = self.connection.execute('PRAGMA database_list').fetchone()[2]
+ connection = sqlite3.connect(database)
+ connection.row_factory = sqlite3.Row
+ self.addCleanup(connection.close)
+ return connection
+
+ def _connector(self, enrich_one):
+ class Connector(_BaseConnector):
+ def fetch(self, ref, **kwargs):
+ candidates, probed = self.enrich(tuple(
+ FollowCandidate(provider='fanbox', external_id=str(index),
+ title=f'作品 {index}', partial=True)
+ for index in (1, 2)))
+ return _fetch(candidates=candidates, probed=probed, etag='finished', raw_body=b'page')
+
+ def _enrich_one(self, candidate):
+ return enrich_one(candidate)
+
+ return Connector(enrich_budget=2)
+
+ def test_a_completed_item_is_visible_while_the_next_item_is_being_enriched(self):
+ source = self._register()
+ self.connection.commit()
+ reader = self._reader()
+
+ def enrich(candidate):
+ if candidate.external_id == '1':
+ saved = reader.execute('SELECT * FROM follow_item ORDER BY external_id').fetchall()
+ self.assertEqual([row['external_id'] for row in saved], ['1', '2'])
+ self.assertTrue(all('post_type' not in json.loads(row['metadata_json']) for row in saved))
+ if candidate.external_id == '2':
+ saved = reader.execute('SELECT * FROM follow_item ORDER BY external_id').fetchall()
+ self.assertEqual([row['external_id'] for row in saved], ['1', '2'])
+ self.assertEqual(json.loads(saved[0]['metadata_json'])['post_type'], 'image')
+ self.assertNotIn('post_type', json.loads(saved[1]['metadata_json']))
+ self.assertIsNone(saved[0]['evidence_path'])
+ pending = reader.execute('SELECT * FROM follow_source WHERE id=?', (source,)).fetchone()
+ self.assertIsNone(pending['last_checked_at'])
+ self.assertIsNone(pending['etag'])
+ self.assertEqual(pending['backfill_page'], 0)
+ return replace(candidate, partial=False, extra={'post_type': 'image'})
+
+ result = self._run(source, self._connector(enrich))
+ self.assertEqual((result.outcome.discovered, result.outcome.added, result.outcome.updated),
+ (2, 2, 0))
+ self.assertEqual(self._row(source)['etag'], 'finished')
+ self.assertTrue(all(row[0] for row in reader.execute('SELECT evidence_path FROM follow_item')))
+ again = self._run(source, self._connector(lambda candidate: candidate))
+ self.assertEqual((again.outcome.added, again.outcome.updated), (0, 2))
+
+ def test_a_rate_limit_keeps_completed_items_and_the_page_can_be_retried(self):
+ source = self._register()
+ self.connection.commit()
+
+ def enrich(candidate):
+ if candidate.external_id == '2':
+ raise FollowSourceRateLimited('稍后再试', 120)
+ return replace(candidate, partial=False, extra={'post_type': 'image'})
+
+ result = self._run(source, self._connector(enrich))
+ self.assertFalse(result.ok)
+ reader = self._reader()
+ saved = reader.execute('SELECT * FROM follow_item ORDER BY external_id').fetchall()
+ self.assertEqual([row['external_id'] for row in saved], ['1', '2'])
+ self.assertEqual(json.loads(saved[0]['metadata_json'])['post_type'], 'image')
+ self.assertNotIn('post_type', json.loads(saved[1]['metadata_json']))
+ row = self._row(source)
+ self.assertEqual(row['last_status'], 'error')
+ self.assertIsNone(row['etag'])
+ self.assertEqual(row['backfill_page'], 0)
+ retry = self._run(source, self._connector(lambda candidate: candidate))
+ self.assertEqual((retry.outcome.added, retry.outcome.updated), (0, 2))
+ self.assertEqual(len(self.store.items(source_id=source)), 2)
+
+ def test_an_interrupted_history_page_keeps_items_without_advancing_its_cursor(self):
+ source = self._register()
+ self.connection.commit()
+
+ def enrich(candidate):
+ if candidate.external_id == '2':
+ raise InterruptedError('任务中断')
+ return candidate
+
+ with self.assertRaises(InterruptedError):
+ self._run(source, self._connector(enrich), older=True)
+ self.assertEqual(len(self.store.items(source_id=source)), 2)
+ self.assertEqual(self._row(source)['backfill_page'], 0)
+ self.assertIsNone(self._row(source)['last_checked_at'])
+
+ def test_detail_dates_enforce_the_persisted_history_boundary(self):
+ source = self._register()
+ self.store.merge_source_metadata(source, {
+ 'initial_history_after': '2026-08-04T00:00:00Z', 'initial_history_floor': 0}, MOMENT)
+ self.connection.commit()
+
+ def enrich(candidate):
+ self.assertEqual(len(self.store.items(source_id=source)), 0)
+ return replace(candidate, partial=False, published_at='2025-01-01T00:00:00Z')
+
+ result = self._run(source, self._connector(enrich))
+ self.assertEqual(result.outcome.discovered, 0)
+ self.assertEqual(len(self.store.items(source_id=source)), 0)
+
+
class SecondPhaseHandoffTests(_CheckCase):
def test_the_skip_set_reaches_the_connector(self):
"""算出来的跳过集合要真的传给连接器,否则第二阶段照样按条目数付请求。"""
diff --git a/tests/test_follow_sources.py b/tests/test_follow_sources.py
index 1196de025..ce574405c 100644
--- a/tests/test_follow_sources.py
+++ b/tests/test_follow_sources.py
@@ -1046,7 +1046,7 @@ class Rule34VideoConnectorTests(unittest.TestCase):
self.assertTrue(first.thumb_url.endswith("/preview.jpg"))
self.assertIn("4542721_360.mp4", first.media_url)
- def _detail_with_credits(self, names):
+ def _detail_with_credits(self, names, *, ready=None):
credits = "".join(
f'<a class="item btn_link video_meta_pill"'
f' href="https://rule34video.com/models/m{n}/">{name}</a>'
@@ -1058,12 +1058,16 @@ class Rule34VideoConnectorTests(unittest.TestCase):
return HttpResponse(200, {},
RULE34VIDEO_HTML if "/models/" in request.url else detail)
- return Rule34VideoConnector(transport=transport).fetch("lazyprocrastinator")
+ connector = Rule34VideoConnector(transport=transport)
+ connector.candidate_ready = ready
+ return connector.fetch("lazyprocrastinator")
def test_detail_with_many_credited_models_is_not_collected(self):
# 样本自带 LazyProcrastinator 一位,这里再添三位就是四位画面作者。
- result = self._detail_with_credits(["M1", "M2", "M3"])
+ ready = []
+ result = self._detail_with_credits(["M1", "M2", "M3"], ready=ready.append)
self.assertEqual(result.candidates, ())
+ self.assertEqual(ready, [])
self.assertEqual(result.skipped, 2)
self.assertEqual(result.skipped_compilations, 2)
@@ -2214,6 +2218,21 @@ class EnrichPhaseTests(unittest.TestCase):
self.assertEqual(result.probed, 0)
self.assertTrue(all(item.partial for item in result.candidates))
+ def test_each_enriched_item_is_published_before_the_next_detail_request(self):
+ seen, ready = [], []
+ connector = self._rule34xxx(seen, enrich_budget=2)
+
+ def publish(candidate):
+ ready.append(candidate)
+ enriched = sum(not item.partial for item in ready)
+ self.assertEqual(sum('s=view' in url for url in seen), enriched)
+
+ connector.candidate_ready = publish
+ result = connector.fetch('lazyprocrastinator')
+ self.assertEqual([item.external_id for item in ready[:4]],
+ [item.external_id for item in result.candidates])
+ self.assertEqual([item.partial for item in ready], [True, True, True, True, False, False])
+
def test_the_skip_set_is_spent_on_the_condition_not_on_the_budget(self):
"""跳过的条目不占额度:额度是为「还没补齐的那几条」留的。"""
seen = []