Skip to content

armgit / peach

Public Imported from GitHub
0

Peach: Local-first video metadata scraper and media manager.

feat(tasks): 长时任务从持久检查点续跑

longmeidao9fbebdd64cf8
diff --git a/scripts/scrape_codes.py b/scripts/scrape_codes.py
index 249223b52..e41f87d42 100644
--- a/scripts/scrape_codes.py
+++ b/scripts/scrape_codes.py
@@ -902,7 +902,8 @@ def main(argv: list[str] | None = None, *, provider=None) -> int:
return code
-def _scrape(parser, args, handle, *, provider=None, resources) -> int:
+def _scrape_batch(parser, args):
+ """按参数签名定位同一批次及其输出。"""
sources = None
if args.sources is not None:
try:
@@ -924,6 +925,27 @@ def _scrape(parser, args, handle, *, provider=None, resources) -> int:
completed = set(saved.get('completed') or [])
if completed and not output.is_file():
parser.error('续跑候选文件未取得,请恢复原文件或使用 --no-resume')
+ return sources, profile, signature, active_path, output, checkpoint_path, saved, completed
+
+
+def _prepare_scrape_output(parser, args, connection, saved, signature, output, errors_path, completed):
+ """固定原批次番号,并保留成功番号的候选行。"""
+ codes = _load_codes(connection, args, parser)
+ if saved and saved.get('signature') != signature:
+ parser.error('续跑参数与原批次不一致,请使用原参数或 --no-resume')
+ if saved.get('codes'):
+ by_code = {normalise_code_key(row[0]): row for row in codes}
+ codes = [by_code[code] for code in saved['codes'] if code in by_code]
+ previous_rows = read_rows(output) if saved and output.is_file() else []
+ previous_rows = [row for row in previous_rows if row['query'] in completed]
+ write_rows(output, FIELDS, previous_rows, atomic=True)
+ previous_errors = read_rows(errors_path) if saved and errors_path.is_file() else []
+ write_rows(errors_path, ERROR_FIELDS, previous_errors, atomic=True)
+ return codes, len(previous_rows), len(previous_errors)
+
+
+def _scrape(parser, args, handle, *, provider=None, resources) -> int:
+ sources, profile, signature, active_path, output, checkpoint_path, saved, completed = _scrape_batch(parser, args)
error_name = output.name.replace("metadata-field-candidates-", "metadata-source-errors-", 1)
if error_name == output.name:
error_name = output.stem + "-errors.csv"
@@ -954,30 +976,18 @@ def _scrape(parser, args, handle, *, provider=None, resources) -> int:
resources.callback(connection.close)
# 用户在复核页收录过的 genre 这一批就当已知词,不再作为未收录回来问一遍。
genre_decisions = load_genre_decisions(connection)
- codes = _load_codes(connection, args, parser)
- if saved and saved.get('signature') != signature:
- connection.close()
- close_log()
- parser.error('续跑参数与原批次不一致,请使用原参数或 --no-resume')
- if saved.get('codes'):
- by_code = {normalise_code_key(row[0]): row for row in codes}
- codes = [by_code[code] for code in saved['codes'] if code in by_code]
+ codes, groups_written, errors_written = _prepare_scrape_output(
+ parser, args, connection, saved, signature, output, errors_path, completed)
def save_progress(status='active'):
record = dict(signature=signature, output=str(output.resolve()), status=status,
completed=sorted(completed), codes=[normalise_code_key(row[0]) for row in codes],
health=health, unmapped=list(unmapped_genres.items()))
atomic_write_text(checkpoint_path, json.dumps(record, ensure_ascii=False))
atomic_write_text(active_path, json.dumps(record, ensure_ascii=False))
- previous_rows = read_rows(output) if saved and output.is_file() else []
- previous_rows = [row for row in previous_rows if row['query'] in completed]
- write_rows(output, FIELDS, previous_rows, atomic=True)
- previous_errors = read_rows(errors_path) if saved and errors_path.is_file() else []
- write_rows(errors_path, ERROR_FIELDS, previous_errors, atomic=True)
save_progress()
log(f"字段候选批次:profile {profile},番号 {len(codes)},来源 {','.join(covered)};只读查询,不写 ledger")
throttle = _Throttle(health)
- groups_written, errors_written = len(previous_rows), len(previous_errors)
stopped: JobPolicyError | None = None
# 流式写:两个文件同时开着,行在长循环里边跑边落盘,中途还有 guard.check()
# 打断点。write_rows 收的是完整行集合,改用它等于「跑完才落盘」,被打断就全丢。
diff --git a/src/peach/jav_discovery_metadata.py b/src/peach/jav_discovery_metadata.py
index 08857dd33..112aff4e2 100644
--- a/src/peach/jav_discovery_metadata.py
+++ b/src/peach/jav_discovery_metadata.py
@@ -119,6 +119,14 @@ class DiscoveryMetadata:
return movie, {"metadata_state": state, "metadata_fields": fields,
"metadata_message": record.get("error", ""), "metadata_checked_at": record.get("checked_at")}
+ def _pending_codes(self, codes, successful, refresh):
+ """排除本批成功项、冷却项与仍新鲜的缓存。"""
+ with self.lock:
+ return [code for code in codes if code not in successful
+ and self.records.get(code, {}).get('retry_after', 0) <= self.clock()
+ and (refresh or not cache_fresh(self.records.get(code, {}).get('checked_at', -1),
+ self.records.get(code, {}).get('movie', {}).get('release_date'), self.clock()))]
+
def ensure(self, codes, *, refresh=False, job_id=None, _resume=None):
snapshot = None if _resume else self.job.snapshot() or self.last_job
if _resume:
@@ -133,10 +141,7 @@ class DiscoveryMetadata:
if snapshot and not refresh:
codes = snapshot['codes']
codes = list(dict.fromkeys(normalise_code_key(code) for code in codes if code))[:MAX_ITEMS]
- with self.lock:
- pending = [code for code in codes if code not in successful and self.records.get(code, {}).get('retry_after', 0) <= self.clock()
- and (refresh or not cache_fresh(self.records.get(code, {}).get('checked_at', -1),
- self.records.get(code, {}).get('movie', {}).get('release_date'), self.clock()))]
+ pending = self._pending_codes(codes, successful, refresh)
if snapshot and not pending:
return snapshot
budget = PublicRequestBudget()
@@ -147,46 +152,17 @@ class DiscoveryMetadata:
started = time.monotonic()
initial = {"total": len(codes), "completed": (_resume or {}).get('completed', len(codes) - len(pending)),
"failed": len(failures), 'refresh': refresh,
+ 'completed_codes': sorted(successful), 'attempted_codes': sorted(attempted),
+ 'failed_codes': sorted(failures), 'active_seconds': previous_elapsed,
+ 'started_at': (_resume or {}).get('started_at', self.clock()),
"requests": budget.requests, "bytes": budget.bytes, "stop_reason": "", "codes": codes,
"budget": {"max_items": MAX_ITEMS, "max_requests": budget.max_requests,
"max_bytes": budget.max_bytes, "max_seconds": budget.max_seconds}}
def work(current_id):
- completed = initial['completed']
- stop_reason = ""
- with public_cache_scope(self.path.parent / 'jav-public'), self.api.request_budget(budget):
- for code in pending:
- if not self.job.snapshot() or self.job.snapshot()["job_id"] != current_id:
- return
- try:
- movie = self.api.movie(code, refresh=True) if refresh else self.api.movie(code)
- self.remember(code, movie, checked_at=cache_checked_at('movie:' + code, exact=True))
- if cache_error('movie:' + code):
- raise PublicSourceUnavailable(cache_error('movie:' + code))
- successful.add(code)
- failures.discard(code)
- except PublicBudgetExceeded:
- stop_reason = budget.stop_reason
- break
- except PublicSourceUnavailable as error:
- self.remember(code, error=str(error))
- failures.add(code)
- if self.api.paused_until > self.api.clock():
- stop_reason = "source_cooldown"
- if code not in attempted:
- completed += 1
- attempted.add(code)
- if not self.job.update(current_id, completed=completed, failed=len(failures),
- completed_codes=sorted(successful), attempted_codes=sorted(attempted), failed_codes=sorted(failures),
- active_seconds=previous_elapsed + time.monotonic() - started,
- requests=budget.requests, bytes=budget.bytes, batch=(completed + 9) // 10):
- return
- self._save_job()
- if stop_reason:
- break
- self.job.update(current_id, status="stopped" if stop_reason else "complete", stop_reason=stop_reason,
- completed=completed, failed=len(failures), requests=budget.requests, bytes=budget.bytes)
- self._save_job()
+ self._run_batch(current_id, initial=initial, pending=pending, refresh=refresh, budget=budget,
+ previous_elapsed=previous_elapsed, started=started, successful=successful,
+ attempted=attempted, failures=failures)
from .job_recovery import current_job_id
token = current_job_id.set(_resume.get('job_id')) if _resume else None
@@ -198,6 +174,45 @@ class DiscoveryMetadata:
if token is not None:
current_job_id.reset(token)
+ def _run_batch(self, current_id, *, initial, pending, refresh, budget, previous_elapsed, started,
+ successful, attempted, failures):
+ completed = initial['completed']
+ stop_reason = ""
+ with public_cache_scope(self.path.parent / 'jav-public'), self.api.request_budget(budget):
+ for code in pending:
+ if not self.job.snapshot() or self.job.snapshot()["job_id"] != current_id:
+ return
+ try:
+ movie = self.api.movie(code, refresh=True) if refresh else self.api.movie(code)
+ self.remember(code, movie, checked_at=cache_checked_at('movie:' + code, exact=True))
+ if cache_error('movie:' + code):
+ raise PublicSourceUnavailable(cache_error('movie:' + code))
+ successful.add(code)
+ failures.discard(code)
+ except PublicBudgetExceeded:
+ stop_reason = budget.stop_reason
+ break
+ except PublicSourceUnavailable as error:
+ self.remember(code, error=str(error))
+ failures.add(code)
+ if self.api.paused_until > self.api.clock():
+ stop_reason = "source_cooldown"
+ if code not in attempted:
+ completed += 1
+ attempted.add(code)
+ if not self.job.update(current_id, completed=completed, failed=len(failures),
+ completed_codes=sorted(successful), attempted_codes=sorted(attempted), failed_codes=sorted(failures),
+ active_seconds=previous_elapsed + time.monotonic() - started,
+ requests=budget.requests, bytes=budget.bytes, batch=(completed + 9) // 10):
+ return
+ self._save_job()
+ if stop_reason:
+ break
+ self.job.update(current_id, status="stopped" if stop_reason else "complete", stop_reason=stop_reason,
+ completed=completed, failed=len(failures), requests=budget.requests, bytes=budget.bytes,
+ active_seconds=previous_elapsed + time.monotonic() - started)
+ self._save_job()
+
def stop(self):
self._save_job()
self.job.stop()
diff --git a/src/peach/web_follow.py b/src/peach/web_follow.py
index 1dcdd4a6b..70d3f7498 100644
--- a/src/peach/web_follow.py
+++ b/src/peach/web_follow.py
@@ -2142,6 +2142,60 @@ def _check_payload(result) -> dict:
return payload
+class _FollowCheckSession:
+ """固定来源集合,在每页落地后保存游标和来源结果。"""
+
+ def __init__(self, contract, body, job_id, previous, rows, credentials):
+ self.contract, self.body, self.job_id = contract, body, job_id
+ self.rows, self.credentials = rows, credentials
+ self.results = list(previous.get('results') or [])
+ self.pages = dict(previous.get('backfill_pages') or {})
+ self.writer = _check_writer(contract)
+ self.initial_days = follow_initial_days(contract.database)
+ self.cooldown = getattr(contract, 'follow_media_resolver', None)
+
+ def checkpoint(self, **fields):
+ if self.job_id and not self.contract.follow_job.checkpoint(self.job_id, **fields):
+ raise InterruptedError('关注检查已停止')
+
+ def remember(self, row, result, *, finished):
+ if result.get('ok') and isinstance(result.get('page'), int):
+ self.pages[str(row['id'])] = result['page']
+ if finished:
+ result['author'] = author_display_name(row)
+ self.results[:] = [item for item in self.results if item['source'] != row['id']]
+ self.results.append(result)
+ self.checkpoint(backfill_pages=dict(self.pages), results=self.results.copy(),
+ checked=len(self.results), total=len(self.rows))
+
+ def check(self, row):
+ current = {'source': row['id'], 'label': author_display_name(row),
+ 'provider': PROVIDER_LABELS.get(row['provider'], row['provider'])}
+ def progress(**fields):
+ if self.job_id:
+ self.contract.follow_job.update(self.job_id, total=len(self.rows), checked=len(self.results),
+ results=self.results.copy(), current={**current, **fields})
+ progress(attempt=1, max_attempts=5, retry_in=0)
+ _backfill_profile_links(row, self.credentials, self.writer)
+ older = bool(self.body.get('older'))
+ for round_index in range(501):
+ if self.job_id and self.contract.follow_job.snapshot() is None:
+ raise InterruptedError('关注检查已停止')
+ result = _check_payload(run_check(row, credentials=self.credentials, writer=self.writer,
+ connector_factory=build_connector, older=older, progress=progress,
+ initial_days=self.initial_days, cooldown=self.cooldown))
+ more = (older and self.body.get('backfill_all') and result.get('ok')
+ and not result.get('exhausted') and isinstance(result.get('page'), int)
+ and round_index < 500)
+ self.remember(row, result, finished=not more)
+ if not more:
+ break
+ row['backfill_page'] = result['page']
+ row['metadata_json'] = json.dumps({**json.loads(row.get('metadata_json') or '{}'),
+ 'initial_history_first_page_pending': False})
+ progress()
+
+
def _run_follow_check(contract, body, job_id=None) -> dict:
previous = (contract.follow_job.snapshot() or {}) if job_id else {}
requested = body.get("source")
@@ -2155,7 +2209,6 @@ def _run_follow_check(contract, body, job_id=None) -> dict:
# 不会因为重走而倒退。补齐标记缺键的行在重走时各自重探一次详情。
rewind = bool(body.get("rewind"))
credentials = _credential_store(contract)
- initial_days = follow_initial_days(contract.database)
with contract.database.read_connection() as connection:
rows = plan_check(_store(contract, connection), credentials,
source_id=source_id, older=older,
@@ -2164,66 +2217,16 @@ def _run_follow_check(contract, body, job_id=None) -> dict:
rows = [row for row in rows if row["id"] in body["sources"]]
if 'source_ids' in previous:
rows = [row for row in rows if row['id'] in previous['source_ids']]
- results = list(previous.get('results') or [])
- completed = {row['source'] for row in results if row.get('ok')}
- pages = dict(previous.get('backfill_pages') or {})
+ session = _FollowCheckSession(contract, body, job_id, previous, rows, credentials)
+ completed = {row['source'] for row in session.results if row.get('ok')}
if rewind and older:
for row in rows:
- row["backfill_page"] = pages.get(str(row['id']), 0)
- writer = _check_writer(contract)
- cooldown = getattr(contract, "follow_media_resolver", None)
- if job_id and not contract.follow_job.checkpoint(job_id, source_ids=[row['id'] for row in rows]):
- raise InterruptedError('关注检查已停止')
+ row["backfill_page"] = session.pages.get(str(row['id']), 0)
+ session.checkpoint(source_ids=[row['id'] for row in rows])
for row in rows:
if row['id'] in completed:
continue
- if job_id and contract.follow_job.snapshot() is None:
- raise InterruptedError('关注检查已停止')
- current = {"source": row["id"], "label": author_display_name(row),
- "provider": PROVIDER_LABELS.get(row["provider"], row["provider"])}
- def progress(*, current=current, **fields):
- if job_id:
- contract.follow_job.update(job_id, total=len(rows), checked=len(results),
- results=results.copy(), current={**current, **fields})
- progress(attempt=1, max_attempts=5, retry_in=0)
- _backfill_profile_links(row, credentials, writer)
- result = _check_payload(run_check(
- row, credentials=credentials, writer=writer,
- connector_factory=build_connector, older=older, progress=progress,
- initial_days=initial_days, cooldown=cooldown))
- def remember_page():
- if result.get('ok') and isinstance(result.get('page'), int):
- pages[str(row['id'])] = result['page']
- if job_id and not contract.follow_job.checkpoint(job_id, backfill_pages=dict(pages)):
- raise InterruptedError('关注检查已停止')
- remember_page()
- if older and body.get("backfill_all"):
- # 全量回抓:一页记完接着抓下一页,直到站点说没有更多、某一页失败,
- # 或满 500 轮。每轮把结果页写回工作行,`run_check` 从它加一起算,页码
- # 只增不减、不会原地打转;ledger 里的游标落在 `backfill_page`。
- rounds = 0
- while (result.get("ok") and not result.get("exhausted")
- and isinstance(result.get("page"), int) and rounds < 500):
- if job_id and contract.follow_job.snapshot() is None:
- raise InterruptedError('关注检查已停止')
- row["backfill_page"] = result["page"]
- # 首页重放只该有一次:`run_check` 已把标记写回 ledger,工作行里的旧
- # metadata 还带着它,不摘掉的话每一轮都判成重放,第 0 页连抓到轮数上限。
- row["metadata_json"] = json.dumps({
- **json.loads(row.get("metadata_json") or "{}"),
- "initial_history_first_page_pending": False})
- result = _check_payload(run_check(
- row, credentials=credentials, writer=writer,
- connector_factory=build_connector, older=True, progress=progress,
- initial_days=initial_days, cooldown=cooldown))
- rounds += 1
- remember_page()
- result["author"] = author_display_name(row)
- results = [item for item in results if item['source'] != row['id']]
- results.append(result)
- if job_id and not contract.follow_job.checkpoint(job_id, results=results.copy(), checked=len(results), total=len(rows)):
- raise InterruptedError('关注检查已停止')
- progress()
+ session.check(row)
# 还没建档的关注作者交给建档后继(ADR-0096)。后继由结果声明、调度端统一派
# (ADR-0040);这里判不出来只是这一轮不派,检查本身的结论不受影响。
try:
@@ -2231,7 +2234,7 @@ def _run_follow_check(contract, body, job_id=None) -> dict:
followups = follow_creators.declare(connection)
except sqlite3.Error:
followups = []
- return {"ok": True, "checked": len(results), "results": results, "followups": followups}
+ return {"ok": True, "checked": len(session.results), "results": session.results, "followups": followups}
def q_follow_schedule(contract, _args) -> dict:
@@ -2483,6 +2486,30 @@ def _external_search_payload(search) -> dict:
"url": search.url, "evidence": search.evidence}
+def _resolve_follow_line(contract, line, known, on_progress):
+ """解析一行链接或检索词,并标记已登记来源。"""
+ try:
+ parsed = parse_source_url(line)
+ except FollowSourceError as url_error:
+ if '://' in line or '/' in line:
+ return {'line': line, 'kind': 'error', 'error': str(url_error)}
+ try:
+ found = discover(line, secrets_root=contract.follow_secrets_root,
+ shared_root=contract.follow_shared_root,
+ state_root=contract.follow_state_root, on_progress=on_progress)
+ except (FollowSourceError, CredentialError) as term_error:
+ return {'line': line, 'kind': 'error', 'error': str(term_error)}
+ return {'line': line, 'kind': 'term',
+ 'candidates': [{**_candidate_payload(c, author=line),
+ 'known': (c.provider, canonical_source_ref(c.provider, c.ref)) in known}
+ for c in found.candidates],
+ 'failures': found.failures,
+ 'external_searches': [_external_search_payload(search) for search in found.external_searches]}
+ return {'line': line, 'kind': 'url',
+ 'candidates': [{**_candidate_payload(parsed),
+ 'known': (parsed.provider, canonical_source_ref(parsed.provider, parsed.ref)) in known}]}
+
+
def w_follow_resolve(contract, body, *, progress=None, checkpoint=None, resume_state=None) -> dict:
"""把粘进来的每一行解析成「可以添加什么」,但**不添加**。
@@ -2539,42 +2566,13 @@ def w_follow_resolve(contract, body, *, progress=None, checkpoint=None, resume_s
return note
- def resolve_line(index: int, line: str) -> dict:
- try:
- parsed = parse_source_url(line)
- except FollowSourceError as url_error:
- if "://" in line or "/" in line:
- # 看着就是链接,那就照链接的错误报,不要再拿去当名字查一遍。
- return {"line": line, "kind": "error", "error": str(url_error)}
- try:
- found = discover(line, secrets_root=contract.follow_secrets_root,
- shared_root=contract.follow_shared_root,
- state_root=contract.follow_state_root,
- on_progress=source_progress(index))
- except (FollowSourceError, CredentialError) as term_error:
- return {"line": line, "kind": "error", "error": str(term_error)}
- return {
- "line": line, "kind": "term",
- "candidates": [{**_candidate_payload(c, author=line),
- "known": (c.provider, canonical_source_ref(
- c.provider, c.ref)) in known}
- for c in found.candidates],
- "failures": found.failures,
- "external_searches": [_external_search_payload(search)
- for search in found.external_searches],
- }
- return {"line": line, "kind": "url",
- "candidates": [{**_candidate_payload(parsed),
- "known": (parsed.provider, canonical_source_ref(
- parsed.provider, parsed.ref)) in known}]}
-
results = list((resume_state or {}).get('results') or [])
completed = {row['line'] for row in results if row.get('kind') != 'error' and not row.get('failures')}
for index, line in enumerate(lines):
# 分母是 plan_units 算出来的,分子就照同一把尺子加:一行查失败也占掉了它那
# 几个来源,不加进去的话后面每一行都低这么多,环最后停在半路。
if line not in completed:
- result = resolve_line(index, line)
+ result = _resolve_follow_line(contract, line, known, source_progress(index))
results = [row for row in results if row['line'] != line]
results.append(result)
if checkpoint and not checkpoint(results=results.copy(), checked=done_units + plan_units(line), total=total_units):
diff --git a/src/peach/web_links.py b/src/peach/web_links.py
index 61bb8da61..dc3bfaa51 100644
--- a/src/peach/web_links.py
+++ b/src/peach/web_links.py
@@ -266,6 +266,34 @@ def _prune_selection(planned: list[dict], body: dict) -> list[dict]:
return [item for item in planned if item["id"] in requested]
+def _recheck_prune_links(planned, progress, checkpoint, resume_state):
+ """保存逐条重验结果,临时无法判定的链接留待重试。"""
+ saved = (resume_state or {}).get('prune_checks') or {}
+ confirmed = list(saved.get('confirmed') or [])
+ recovered = list(saved.get('recovered') or [])
+ completed = set(saved.get('completed') or [])
+ for index, item in enumerate(planned):
+ if item['id'] in completed:
+ continue
+ if progress:
+ progress(checked=index, total=len(planned),
+ message=f"重验失效链接:已检查 {index} / {len(planned)} 条")
+ status, note = _probe(item['url'])
+ verdict = link_verdict(status, note)
+ confirmed[:] = [row for row in confirmed if row['id'] != item['id']]
+ recovered[:] = [row for row in recovered if row['id'] != item['id']]
+ if verdict == 'gone':
+ confirmed.append({**item, 'note': _note(status, note)})
+ else:
+ recovered.append(item)
+ if verdict != 'unclear':
+ completed.add(item['id'])
+ if checkpoint and not checkpoint(prune_checks=dict(confirmed=confirmed, recovered=recovered,
+ completed=sorted(completed)), checked=index + 1, total=len(planned)):
+ raise InterruptedError('链接清理已停止')
+ return confirmed, recovered
+
+
def w_links_prune(contract: LinkContract, body, *, progress=None, checkpoint=None, resume_state=None):
"""删掉上一次检查判定为 gone 的链接。
@@ -295,18 +323,7 @@ def w_links_prune(contract: LinkContract, body, *, progress=None, checkpoint=Non
except ValueError as error:
return {"ok": False, "error": str(error)}
- confirmed, recovered = [], []
- for index, item in enumerate(planned):
- if progress:
- progress(checked=max(index, (resume_state or {}).get('checked', 0)), total=len(planned),
- message=f"重验失效链接:已检查 {index} / {len(planned)} 条")
- status, note = _probe(item["url"])
- if link_verdict(status, note) == "gone":
- confirmed.append({**item, "note": _note(status, note)})
- else:
- recovered.append(item)
- if checkpoint and not checkpoint(checked=max(index + 1, (resume_state or {}).get('checked', 0)), total=len(planned)):
- raise InterruptedError('链接清理已停止')
+ confirmed, recovered = _recheck_prune_links(planned, progress, checkpoint, resume_state)
removed = marked = 0
if progress:
diff --git a/src/peach/web_media_repair.py b/src/peach/web_media_repair.py
index 94155cbfa..4a6b09b88 100644
--- a/src/peach/web_media_repair.py
+++ b/src/peach/web_media_repair.py
@@ -139,6 +139,10 @@ def _advance(job, job_id: str, checked: int, *, skipped: bool) -> None:
state["checked"] = max(state.get('checked', 0), checked)
if skipped and checked > state.get('survey_checked', 0):
state["skipped"] = state.get("skipped", 0) + 1
+ state['survey_checked'] = max(state.get('survey_checked', 0), checked)
+ saved = {key: state.get(key, 0) for key in ('checked', 'survey_checked', 'skipped')}
+ if not job.checkpoint(job_id, **saved):
+ raise InterruptedError('媒体检查已停止')
def _reference_candidates(contract, asset_id: int, raw: str) -> list[str]:
diff --git a/src/peach/web_resource_sync.py b/src/peach/web_resource_sync.py
index 1617514f8..8bfb7cb2e 100644
--- a/src/peach/web_resource_sync.py
+++ b/src/peach/web_resource_sync.py
@@ -658,6 +658,33 @@ def _mark_record_holders(contract: ResourceSyncContract,
return marked, reattached, [row for row in rows if int(row["id"]) not in kept]
+def _apply_missing_records(contract, state, report, checkpoint, resume_state):
+ """逐批复核失效记录,保存完成集合与累计回执。"""
+ saved = (resume_state or {}).get('apply_checkpoint') or {}
+ processed = set(saved.get('processed') or [])
+ report(checked=len(processed), total=len(state['vanished_ids']), message="正在逐条复核失效记录")
+ marked, reattached = list(saved.get('marked') or []), list(saved.get('reattached') or [])
+ purge = dict(saved.get('purge') or {'purged': 0, 'blocked': [], 'empty_dirs_removed': 0})
+ pending = [value for value in state['vanished_ids'] if value not in processed]
+ for offset in range(0, len(pending), 100):
+ selected = pending[offset:offset + 100]
+ added, attached, confirmed = _mark_record_holders(contract, _recheck_vanished(contract, selected))
+ marked.extend(added)
+ reattached.extend(attached)
+ if confirmed:
+ from .web_batch import purge_vanished_rows
+ result = purge_vanished_rows(contract, confirmed)
+ for field in ('purged', 'empty_dirs_removed'):
+ purge[field] += int(result.get(field) or 0)
+ purge['blocked'].extend(result.get('blocked') or [])
+ processed.update(selected)
+ report(checked=len(processed), total=len(state['vanished_ids']), message='正在处理失效记录')
+ if checkpoint and not checkpoint(apply_checkpoint=dict(processed=sorted(processed), marked=marked,
+ reattached=reattached, purge=purge)):
+ raise InterruptedError('资源清理已停止')
+ return marked, reattached, purge
+
+
def w_resource_sync_apply(contract: ResourceSyncContract, body, *, progress=None, checkpoint=None, resume_state=None):
"""处理这一轮检查报出的失效记录与空文件夹,再清孤儿缓存。
@@ -687,28 +714,8 @@ def w_resource_sync_apply(contract: ResourceSyncContract, body, *, progress=None
return resume_state['apply_receipt']
state = _completed_scan(contract, scan_id)
report = progress or (lambda **_fields: None)
- saved = (resume_state or {}).get('apply_checkpoint') or {}
- processed = set(saved.get('processed') or [])
- report(checked=len(processed), total=len(state['vanished_ids']), message="正在逐条复核失效记录")
- marked, reattached = list(saved.get('marked') or []), list(saved.get('reattached') or [])
- purge = dict(saved.get('purge') or {'purged': 0, 'blocked': [], 'empty_dirs_removed': 0})
- pending = [value for value in state['vanished_ids'] if value not in processed]
- for offset in range(0, len(pending), 100):
- selected = pending[offset:offset + 100]
- added, attached, confirmed = _mark_record_holders(contract, _recheck_vanished(contract, selected))
- marked.extend(added)
- reattached.extend(attached)
- if confirmed:
- from .web_batch import purge_vanished_rows
- result = purge_vanished_rows(contract, confirmed)
- for field in ('purged', 'empty_dirs_removed'):
- purge[field] += int(result.get(field) or 0)
- purge['blocked'].extend(result.get('blocked') or [])
- processed.update(selected)
- report(checked=len(processed), total=len(state['vanished_ids']), message='正在处理失效记录')
- if checkpoint and not checkpoint(apply_checkpoint=dict(processed=sorted(processed), marked=marked,
- reattached=reattached, purge=purge)):
- raise InterruptedError('资源清理已停止')
+ marked, reattached, purge = _apply_missing_records(
+ contract, state, report, checkpoint, resume_state)
report(checked=0, total=None, message="正在删除空文件夹")
dirs_removed, dir_errors = 0, 0
for location, paths in state["empty_dirs"].items():
diff --git a/tests/test_follow_web.py b/tests/test_follow_web.py
index 25cc1745f..85c4c8979 100644
--- a/tests/test_follow_web.py
+++ b/tests/test_follow_web.py
@@ -105,6 +105,32 @@ class FollowContractTests(unittest.TestCase):
self.assertFalse(final['results'][0]['ok'])
self.assertEqual(factory.call_count, 1)
+ def test_finished_page_checkpoint_resumes_without_another_page_request(self):
+ source = self._seed()
+ for backfill_all in (False, True):
+ with self.subTest(backfill_all=backfill_all):
+ saved = {}
+ def checkpoint(_job_id, *, saved=saved, **fields):
+ saved.update(fields)
+ if fields.get('results'):
+ raise InterruptedError('停止于检查点')
+ return True
+ job = SimpleNamespace(snapshot=lambda saved=saved: saved, checkpoint=checkpoint,
+ update=lambda *a, **k: True)
+ body = {'older': True, 'rewind': True, 'backfill_all': backfill_all}
+ result = {'source': source, 'ok': True, 'page': 3, 'exhausted': backfill_all}
+ with mock.patch.object(self.contract, 'follow_job', job), \
+ mock.patch.object(web_follow, 'run_check', return_value=result) as check, \
+ mock.patch.object(web_follow, '_check_payload', side_effect=lambda value: value):
+ with self.assertRaises(InterruptedError):
+ web_follow._run_follow_check(self.contract, body, 'original')
+ self.assertEqual(saved['backfill_pages'], {str(source): 3})
+ self.assertEqual(saved['results'][0]['source'], source)
+ job.checkpoint = lambda *a, **k: True
+ resumed = web_follow._run_follow_check(self.contract, body, 'original')
+ self.assertEqual(check.call_count, 1)
+ self.assertEqual(resumed['checked'], 1)
+
def test_discovery_result_can_be_read_after_request_returns(self):
started = self._post('/api/follow/resolve', {'background': True,
'lines': ['https://rule34video.com/models/sample/']})
diff --git a/tests/test_jav_discovery_metadata.py b/tests/test_jav_discovery_metadata.py
index 513c40a6d..a873bd7e2 100644
--- a/tests/test_jav_discovery_metadata.py
+++ b/tests/test_jav_discovery_metadata.py
@@ -35,6 +35,25 @@ class DiscoveryMetadataTests(unittest.TestCase):
self.cache = DiscoveryMetadata(self.path, api=self.api, clock=lambda: self.now[0])
self.addCleanup(self.cache.stop)
+ def test_repeated_restart_keeps_checkpoint_before_the_next_item_finishes(self):
+ self.path.with_suffix('.job.json').write_text(json.dumps(dict(
+ job_id='original', status='running', codes=['EXM-001', 'EXM-002'], refresh=True,
+ started_at=100, completed=1, completed_codes=['EXM-001'], attempted_codes=['EXM-001'],
+ failed_codes=[], requests=3, bytes=120, active_seconds=10)), encoding='utf-8')
+ with patch.object(DiscoveryMetadata, '_run_batch'):
+ first = DiscoveryMetadata(self.path, api=self.api, clock=lambda: self.now[0])
+ first.job.thread.join(5)
+ first.stop()
+ saved = json.loads(self.path.with_suffix('.job.json').read_text(encoding='utf-8'))
+ self.assertEqual(saved['completed_codes'], ['EXM-001'])
+ self.assertEqual(saved['started_at'], 100)
+ with patch.object(self.api, 'movie', return_value={'number': 'EXM-002', 'tags': [], 'score': 4, 'duration': 120}) as movie:
+ second = DiscoveryMetadata(self.path, api=self.api, clock=lambda: self.now[0])
+ self.addCleanup(second.stop)
+ second.job.thread.join(5)
+ movie.assert_called_once_with('EXM-002', refresh=True)
+ self.assertEqual(second.job.snapshot()['completed'], 2)
+
def test_sidecar_survives_restart_and_distinguishes_empty_unknown_and_stale(self):
movie = {"number": "EXM-001", "tags": [], "score": "4.23", "duration": 180,
"preview_video_url": "https://example.invalid/secret",
diff --git a/tests/test_web_links.py b/tests/test_web_links.py
index cd9cd9136..788305b5c 100644
--- a/tests/test_web_links.py
+++ b/tests/test_web_links.py
@@ -3,6 +3,7 @@ import sqlite3
import sys
import tempfile
import unittest
+from unittest import mock
from contextlib import contextmanager
from pathlib import Path
@@ -56,6 +57,24 @@ class FakeContract:
class VerdictTests(unittest.TestCase):
+ def test_prune_checks_retain_definitive_results_across_interruption(self):
+ planned = [{'id': index, 'url': f'https://example.invalid/{index}'} for index in (1, 2, 3)]
+ saved = {}
+ def remember(**fields):
+ saved.update(fields)
+ if fields['checked'] == 2:
+ raise InterruptedError('停止于检查点')
+ return True
+ with mock.patch.object(web_links, '_probe', side_effect=[(404, ''), (503, '')]):
+ with self.assertRaises(InterruptedError):
+ web_links._recheck_prune_links(planned, None, remember, {})
+ with mock.patch.object(web_links, '_probe', side_effect=[(200, ''), (404, '')]) as probe:
+ confirmed, recovered = web_links._recheck_prune_links(planned, None, None, saved)
+ self.assertEqual([call.args[0] for call in probe.call_args_list],
+ ['https://example.invalid/2', 'https://example.invalid/3'])
+ self.assertEqual([row['id'] for row in confirmed], [1, 3])
+ self.assertEqual([row['id'] for row in recovered], [2])
+
def test_only_a_gone_status_is_deletable(self):
"""取不到不等于没了,这两件事只有 404/410 能划清。
diff --git a/tests/test_web_resource_sync.py b/tests/test_web_resource_sync.py
index cec7d18fb..35aab85af 100644
--- a/tests/test_web_resource_sync.py
+++ b/tests/test_web_resource_sync.py
@@ -317,6 +317,24 @@ class PurgeMissingTests(unittest.TestCase):
self.assertEqual((result['purged'], result['vanished']), (2, 1))
self.assertEqual(self.ids(), [1, 2, 4])
+ def test_apply_resumes_after_a_committed_chunk_with_cumulative_receipt(self):
+ saved = {}
+ def remember(**fields):
+ saved.update(fields)
+ if 'apply_checkpoint' in fields:
+ raise InterruptedError('停止于检查点')
+ return True
+ with self._synced():
+ scan = finish_scan(self.contract)
+ body = {'confirm': True, 'clean_cache': False, 'scan_id': scan['scan_id']}
+ with self.assertRaises(InterruptedError):
+ rm_sync.w_resource_sync_apply(self.contract, body, checkpoint=remember)
+ self.assertEqual(self.ids(), [1, 2, 4])
+ with mock.patch.object(rm_sync, '_recheck_vanished', wraps=rm_sync._recheck_vanished) as recheck:
+ result = rm_sync.w_resource_sync_apply(self.contract, body, resume_state=saved)
+ recheck.assert_not_called()
+ self.assertEqual((result['purged'], result['vanished']), (1, 1))
+
def test_local_only_configuration_has_resource_check(self):
with mock.patch.object(rm_sync, 'LOCATION_ROOT_DECLARATIONS', {'local': ('R:\\',)}):
self.assertEqual(rm_sync.configured_resource_locations(), ('local',))