Peach: Local-first video metadata scraper and media manager.
feat(tasks): 长时任务从持久检查点续跑
longmeidao9fbebdd64cf8
diff --git a/scripts/scrape_codes.py b/scripts/scrape_codes.pyindex 249223b52..e41f87d42 100644--- a/scripts/scrape_codes.py+++ b/scripts/scrape_codes.py@@ -902,7 +902,8 @@ def main(argv: list[str] | None = None, *, provider=None) -> int:return code-def _scrape(parser, args, handle, *, provider=None, resources) -> int:+def _scrape_batch(parser, args):+ """按参数签名定位同一批次及其输出。"""sources = Noneif args.sources is not None:try:@@ -924,6 +925,27 @@ def _scrape(parser, args, handle, *, provider=None, resources) -> int:completed = set(saved.get('completed') or [])if completed and not output.is_file():parser.error('续跑候选文件未取得,请恢复原文件或使用 --no-resume')+ return sources, profile, signature, active_path, output, checkpoint_path, saved, completed+++def _prepare_scrape_output(parser, args, connection, saved, signature, output, errors_path, completed):+ """固定原批次番号,并保留成功番号的候选行。"""+ codes = _load_codes(connection, args, parser)+ if saved and saved.get('signature') != signature:+ parser.error('续跑参数与原批次不一致,请使用原参数或 --no-resume')+ if saved.get('codes'):+ by_code = {normalise_code_key(row[0]): row for row in codes}+ codes = [by_code[code] for code in saved['codes'] if code in by_code]+ previous_rows = read_rows(output) if saved and output.is_file() else []+ previous_rows = [row for row in previous_rows if row['query'] in completed]+ write_rows(output, FIELDS, previous_rows, atomic=True)+ previous_errors = read_rows(errors_path) if saved and errors_path.is_file() else []+ write_rows(errors_path, ERROR_FIELDS, previous_errors, atomic=True)+ return codes, len(previous_rows), len(previous_errors)+++def _scrape(parser, args, handle, *, provider=None, resources) -> int:+ sources, profile, signature, active_path, output, checkpoint_path, saved, completed = _scrape_batch(parser, args)error_name = output.name.replace("metadata-field-candidates-", "metadata-source-errors-", 1)if error_name == output.name:error_name = output.stem + "-errors.csv"@@ -954,30 +976,18 @@ def _scrape(parser, args, handle, *, provider=None, resources) -> int:resources.callback(connection.close)# 用户在复核页收录过的 genre 这一批就当已知词,不再作为未收录回来问一遍。genre_decisions = load_genre_decisions(connection)- codes = _load_codes(connection, args, parser)- if saved and saved.get('signature') != signature:- connection.close()- close_log()- parser.error('续跑参数与原批次不一致,请使用原参数或 --no-resume')- if saved.get('codes'):- by_code = {normalise_code_key(row[0]): row for row in codes}- codes = [by_code[code] for code in saved['codes'] if code in by_code]+ codes, groups_written, errors_written = _prepare_scrape_output(+ parser, args, connection, saved, signature, output, errors_path, completed)def save_progress(status='active'):record = dict(signature=signature, output=str(output.resolve()), status=status,completed=sorted(completed), codes=[normalise_code_key(row[0]) for row in codes],health=health, unmapped=list(unmapped_genres.items()))atomic_write_text(checkpoint_path, json.dumps(record, ensure_ascii=False))atomic_write_text(active_path, json.dumps(record, ensure_ascii=False))- previous_rows = read_rows(output) if saved and output.is_file() else []- previous_rows = [row for row in previous_rows if row['query'] in completed]- write_rows(output, FIELDS, previous_rows, atomic=True)- previous_errors = read_rows(errors_path) if saved and errors_path.is_file() else []- write_rows(errors_path, ERROR_FIELDS, previous_errors, atomic=True)save_progress()log(f"字段候选批次:profile {profile},番号 {len(codes)},来源 {','.join(covered)};只读查询,不写 ledger")throttle = _Throttle(health)- groups_written, errors_written = len(previous_rows), len(previous_errors)stopped: JobPolicyError | None = None# 流式写:两个文件同时开着,行在长循环里边跑边落盘,中途还有 guard.check()# 打断点。write_rows 收的是完整行集合,改用它等于「跑完才落盘」,被打断就全丢。diff --git a/src/peach/jav_discovery_metadata.py b/src/peach/jav_discovery_metadata.pyindex 08857dd33..112aff4e2 100644--- a/src/peach/jav_discovery_metadata.py+++ b/src/peach/jav_discovery_metadata.py@@ -119,6 +119,14 @@ class DiscoveryMetadata:return movie, {"metadata_state": state, "metadata_fields": fields,"metadata_message": record.get("error", ""), "metadata_checked_at": record.get("checked_at")}+ def _pending_codes(self, codes, successful, refresh):+ """排除本批成功项、冷却项与仍新鲜的缓存。"""+ with self.lock:+ return [code for code in codes if code not in successful+ and self.records.get(code, {}).get('retry_after', 0) <= self.clock()+ and (refresh or not cache_fresh(self.records.get(code, {}).get('checked_at', -1),+ self.records.get(code, {}).get('movie', {}).get('release_date'), self.clock()))]+def ensure(self, codes, *, refresh=False, job_id=None, _resume=None):snapshot = None if _resume else self.job.snapshot() or self.last_jobif _resume:@@ -133,10 +141,7 @@ class DiscoveryMetadata:if snapshot and not refresh:codes = snapshot['codes']codes = list(dict.fromkeys(normalise_code_key(code) for code in codes if code))[:MAX_ITEMS]- with self.lock:- pending = [code for code in codes if code not in successful and self.records.get(code, {}).get('retry_after', 0) <= self.clock()- and (refresh or not cache_fresh(self.records.get(code, {}).get('checked_at', -1),- self.records.get(code, {}).get('movie', {}).get('release_date'), self.clock()))]+ pending = self._pending_codes(codes, successful, refresh)if snapshot and not pending:return snapshotbudget = PublicRequestBudget()@@ -147,46 +152,17 @@ class DiscoveryMetadata:started = time.monotonic()initial = {"total": len(codes), "completed": (_resume or {}).get('completed', len(codes) - len(pending)),"failed": len(failures), 'refresh': refresh,+ 'completed_codes': sorted(successful), 'attempted_codes': sorted(attempted),+ 'failed_codes': sorted(failures), 'active_seconds': previous_elapsed,+ 'started_at': (_resume or {}).get('started_at', self.clock()),"requests": budget.requests, "bytes": budget.bytes, "stop_reason": "", "codes": codes,"budget": {"max_items": MAX_ITEMS, "max_requests": budget.max_requests,"max_bytes": budget.max_bytes, "max_seconds": budget.max_seconds}}def work(current_id):- completed = initial['completed']- stop_reason = ""- with public_cache_scope(self.path.parent / 'jav-public'), self.api.request_budget(budget):- for code in pending:- if not self.job.snapshot() or self.job.snapshot()["job_id"] != current_id:- return- try:- movie = self.api.movie(code, refresh=True) if refresh else self.api.movie(code)- self.remember(code, movie, checked_at=cache_checked_at('movie:' + code, exact=True))- if cache_error('movie:' + code):- raise PublicSourceUnavailable(cache_error('movie:' + code))- successful.add(code)- failures.discard(code)- except PublicBudgetExceeded:- stop_reason = budget.stop_reason- break- except PublicSourceUnavailable as error:- self.remember(code, error=str(error))- failures.add(code)- if self.api.paused_until > self.api.clock():- stop_reason = "source_cooldown"- if code not in attempted:- completed += 1- attempted.add(code)- if not self.job.update(current_id, completed=completed, failed=len(failures),- completed_codes=sorted(successful), attempted_codes=sorted(attempted), failed_codes=sorted(failures),- active_seconds=previous_elapsed + time.monotonic() - started,- requests=budget.requests, bytes=budget.bytes, batch=(completed + 9) // 10):- return- self._save_job()- if stop_reason:- break- self.job.update(current_id, status="stopped" if stop_reason else "complete", stop_reason=stop_reason,- completed=completed, failed=len(failures), requests=budget.requests, bytes=budget.bytes)- self._save_job()+ self._run_batch(current_id, initial=initial, pending=pending, refresh=refresh, budget=budget,+ previous_elapsed=previous_elapsed, started=started, successful=successful,+ attempted=attempted, failures=failures)from .job_recovery import current_job_idtoken = current_job_id.set(_resume.get('job_id')) if _resume else None@@ -198,6 +174,45 @@ class DiscoveryMetadata:if token is not None:current_job_id.reset(token)+ def _run_batch(self, current_id, *, initial, pending, refresh, budget, previous_elapsed, started,+ successful, attempted, failures):+ completed = initial['completed']+ stop_reason = ""+ with public_cache_scope(self.path.parent / 'jav-public'), self.api.request_budget(budget):+ for code in pending:+ if not self.job.snapshot() or self.job.snapshot()["job_id"] != current_id:+ return+ try:+ movie = self.api.movie(code, refresh=True) if refresh else self.api.movie(code)+ self.remember(code, movie, checked_at=cache_checked_at('movie:' + code, exact=True))+ if cache_error('movie:' + code):+ raise PublicSourceUnavailable(cache_error('movie:' + code))+ successful.add(code)+ failures.discard(code)+ except PublicBudgetExceeded:+ stop_reason = budget.stop_reason+ break+ except PublicSourceUnavailable as error:+ self.remember(code, error=str(error))+ failures.add(code)+ if self.api.paused_until > self.api.clock():+ stop_reason = "source_cooldown"+ if code not in attempted:+ completed += 1+ attempted.add(code)+ if not self.job.update(current_id, completed=completed, failed=len(failures),+ completed_codes=sorted(successful), attempted_codes=sorted(attempted), failed_codes=sorted(failures),+ active_seconds=previous_elapsed + time.monotonic() - started,+ requests=budget.requests, bytes=budget.bytes, batch=(completed + 9) // 10):+ return+ self._save_job()+ if stop_reason:+ break+ self.job.update(current_id, status="stopped" if stop_reason else "complete", stop_reason=stop_reason,+ completed=completed, failed=len(failures), requests=budget.requests, bytes=budget.bytes,+ active_seconds=previous_elapsed + time.monotonic() - started)+ self._save_job()+def stop(self):self._save_job()self.job.stop()diff --git a/src/peach/web_follow.py b/src/peach/web_follow.pyindex 1dcdd4a6b..70d3f7498 100644--- a/src/peach/web_follow.py+++ b/src/peach/web_follow.py@@ -2142,6 +2142,60 @@ def _check_payload(result) -> dict:return payload+class _FollowCheckSession:+ """固定来源集合,在每页落地后保存游标和来源结果。"""++ def __init__(self, contract, body, job_id, previous, rows, credentials):+ self.contract, self.body, self.job_id = contract, body, job_id+ self.rows, self.credentials = rows, credentials+ self.results = list(previous.get('results') or [])+ self.pages = dict(previous.get('backfill_pages') or {})+ self.writer = _check_writer(contract)+ self.initial_days = follow_initial_days(contract.database)+ self.cooldown = getattr(contract, 'follow_media_resolver', None)++ def checkpoint(self, **fields):+ if self.job_id and not self.contract.follow_job.checkpoint(self.job_id, **fields):+ raise InterruptedError('关注检查已停止')++ def remember(self, row, result, *, finished):+ if result.get('ok') and isinstance(result.get('page'), int):+ self.pages[str(row['id'])] = result['page']+ if finished:+ result['author'] = author_display_name(row)+ self.results[:] = [item for item in self.results if item['source'] != row['id']]+ self.results.append(result)+ self.checkpoint(backfill_pages=dict(self.pages), results=self.results.copy(),+ checked=len(self.results), total=len(self.rows))++ def check(self, row):+ current = {'source': row['id'], 'label': author_display_name(row),+ 'provider': PROVIDER_LABELS.get(row['provider'], row['provider'])}+ def progress(**fields):+ if self.job_id:+ self.contract.follow_job.update(self.job_id, total=len(self.rows), checked=len(self.results),+ results=self.results.copy(), current={**current, **fields})+ progress(attempt=1, max_attempts=5, retry_in=0)+ _backfill_profile_links(row, self.credentials, self.writer)+ older = bool(self.body.get('older'))+ for round_index in range(501):+ if self.job_id and self.contract.follow_job.snapshot() is None:+ raise InterruptedError('关注检查已停止')+ result = _check_payload(run_check(row, credentials=self.credentials, writer=self.writer,+ connector_factory=build_connector, older=older, progress=progress,+ initial_days=self.initial_days, cooldown=self.cooldown))+ more = (older and self.body.get('backfill_all') and result.get('ok')+ and not result.get('exhausted') and isinstance(result.get('page'), int)+ and round_index < 500)+ self.remember(row, result, finished=not more)+ if not more:+ break+ row['backfill_page'] = result['page']+ row['metadata_json'] = json.dumps({**json.loads(row.get('metadata_json') or '{}'),+ 'initial_history_first_page_pending': False})+ progress()++def _run_follow_check(contract, body, job_id=None) -> dict:previous = (contract.follow_job.snapshot() or {}) if job_id else {}requested = body.get("source")@@ -2155,7 +2209,6 @@ def _run_follow_check(contract, body, job_id=None) -> dict:# 不会因为重走而倒退。补齐标记缺键的行在重走时各自重探一次详情。rewind = bool(body.get("rewind"))credentials = _credential_store(contract)- initial_days = follow_initial_days(contract.database)with contract.database.read_connection() as connection:rows = plan_check(_store(contract, connection), credentials,source_id=source_id, older=older,@@ -2164,66 +2217,16 @@ def _run_follow_check(contract, body, job_id=None) -> dict:rows = [row for row in rows if row["id"] in body["sources"]]if 'source_ids' in previous:rows = [row for row in rows if row['id'] in previous['source_ids']]- results = list(previous.get('results') or [])- completed = {row['source'] for row in results if row.get('ok')}- pages = dict(previous.get('backfill_pages') or {})+ session = _FollowCheckSession(contract, body, job_id, previous, rows, credentials)+ completed = {row['source'] for row in session.results if row.get('ok')}if rewind and older:for row in rows:- row["backfill_page"] = pages.get(str(row['id']), 0)- writer = _check_writer(contract)- cooldown = getattr(contract, "follow_media_resolver", None)- if job_id and not contract.follow_job.checkpoint(job_id, source_ids=[row['id'] for row in rows]):- raise InterruptedError('关注检查已停止')+ row["backfill_page"] = session.pages.get(str(row['id']), 0)+ session.checkpoint(source_ids=[row['id'] for row in rows])for row in rows:if row['id'] in completed:continue- if job_id and contract.follow_job.snapshot() is None:- raise InterruptedError('关注检查已停止')- current = {"source": row["id"], "label": author_display_name(row),- "provider": PROVIDER_LABELS.get(row["provider"], row["provider"])}- def progress(*, current=current, **fields):- if job_id:- contract.follow_job.update(job_id, total=len(rows), checked=len(results),- results=results.copy(), current={**current, **fields})- progress(attempt=1, max_attempts=5, retry_in=0)- _backfill_profile_links(row, credentials, writer)- result = _check_payload(run_check(- row, credentials=credentials, writer=writer,- connector_factory=build_connector, older=older, progress=progress,- initial_days=initial_days, cooldown=cooldown))- def remember_page():- if result.get('ok') and isinstance(result.get('page'), int):- pages[str(row['id'])] = result['page']- if job_id and not contract.follow_job.checkpoint(job_id, backfill_pages=dict(pages)):- raise InterruptedError('关注检查已停止')- remember_page()- if older and body.get("backfill_all"):- # 全量回抓:一页记完接着抓下一页,直到站点说没有更多、某一页失败,- # 或满 500 轮。每轮把结果页写回工作行,`run_check` 从它加一起算,页码- # 只增不减、不会原地打转;ledger 里的游标落在 `backfill_page`。- rounds = 0- while (result.get("ok") and not result.get("exhausted")- and isinstance(result.get("page"), int) and rounds < 500):- if job_id and contract.follow_job.snapshot() is None:- raise InterruptedError('关注检查已停止')- row["backfill_page"] = result["page"]- # 首页重放只该有一次:`run_check` 已把标记写回 ledger,工作行里的旧- # metadata 还带着它,不摘掉的话每一轮都判成重放,第 0 页连抓到轮数上限。- row["metadata_json"] = json.dumps({- **json.loads(row.get("metadata_json") or "{}"),- "initial_history_first_page_pending": False})- result = _check_payload(run_check(- row, credentials=credentials, writer=writer,- connector_factory=build_connector, older=True, progress=progress,- initial_days=initial_days, cooldown=cooldown))- rounds += 1- remember_page()- result["author"] = author_display_name(row)- results = [item for item in results if item['source'] != row['id']]- results.append(result)- if job_id and not contract.follow_job.checkpoint(job_id, results=results.copy(), checked=len(results), total=len(rows)):- raise InterruptedError('关注检查已停止')- progress()+ session.check(row)# 还没建档的关注作者交给建档后继(ADR-0096)。后继由结果声明、调度端统一派# (ADR-0040);这里判不出来只是这一轮不派,检查本身的结论不受影响。try:@@ -2231,7 +2234,7 @@ def _run_follow_check(contract, body, job_id=None) -> dict:followups = follow_creators.declare(connection)except sqlite3.Error:followups = []- return {"ok": True, "checked": len(results), "results": results, "followups": followups}+ return {"ok": True, "checked": len(session.results), "results": session.results, "followups": followups}def q_follow_schedule(contract, _args) -> dict:@@ -2483,6 +2486,30 @@ def _external_search_payload(search) -> dict:"url": search.url, "evidence": search.evidence}+def _resolve_follow_line(contract, line, known, on_progress):+ """解析一行链接或检索词,并标记已登记来源。"""+ try:+ parsed = parse_source_url(line)+ except FollowSourceError as url_error:+ if '://' in line or '/' in line:+ return {'line': line, 'kind': 'error', 'error': str(url_error)}+ try:+ found = discover(line, secrets_root=contract.follow_secrets_root,+ shared_root=contract.follow_shared_root,+ state_root=contract.follow_state_root, on_progress=on_progress)+ except (FollowSourceError, CredentialError) as term_error:+ return {'line': line, 'kind': 'error', 'error': str(term_error)}+ return {'line': line, 'kind': 'term',+ 'candidates': [{**_candidate_payload(c, author=line),+ 'known': (c.provider, canonical_source_ref(c.provider, c.ref)) in known}+ for c in found.candidates],+ 'failures': found.failures,+ 'external_searches': [_external_search_payload(search) for search in found.external_searches]}+ return {'line': line, 'kind': 'url',+ 'candidates': [{**_candidate_payload(parsed),+ 'known': (parsed.provider, canonical_source_ref(parsed.provider, parsed.ref)) in known}]}++def w_follow_resolve(contract, body, *, progress=None, checkpoint=None, resume_state=None) -> dict:"""把粘进来的每一行解析成「可以添加什么」,但**不添加**。@@ -2539,42 +2566,13 @@ def w_follow_resolve(contract, body, *, progress=None, checkpoint=None, resume_sreturn note- def resolve_line(index: int, line: str) -> dict:- try:- parsed = parse_source_url(line)- except FollowSourceError as url_error:- if "://" in line or "/" in line:- # 看着就是链接,那就照链接的错误报,不要再拿去当名字查一遍。- return {"line": line, "kind": "error", "error": str(url_error)}- try:- found = discover(line, secrets_root=contract.follow_secrets_root,- shared_root=contract.follow_shared_root,- state_root=contract.follow_state_root,- on_progress=source_progress(index))- except (FollowSourceError, CredentialError) as term_error:- return {"line": line, "kind": "error", "error": str(term_error)}- return {- "line": line, "kind": "term",- "candidates": [{**_candidate_payload(c, author=line),- "known": (c.provider, canonical_source_ref(- c.provider, c.ref)) in known}- for c in found.candidates],- "failures": found.failures,- "external_searches": [_external_search_payload(search)- for search in found.external_searches],- }- return {"line": line, "kind": "url",- "candidates": [{**_candidate_payload(parsed),- "known": (parsed.provider, canonical_source_ref(- parsed.provider, parsed.ref)) in known}]}-results = list((resume_state or {}).get('results') or [])completed = {row['line'] for row in results if row.get('kind') != 'error' and not row.get('failures')}for index, line in enumerate(lines):# 分母是 plan_units 算出来的,分子就照同一把尺子加:一行查失败也占掉了它那# 几个来源,不加进去的话后面每一行都低这么多,环最后停在半路。if line not in completed:- result = resolve_line(index, line)+ result = _resolve_follow_line(contract, line, known, source_progress(index))results = [row for row in results if row['line'] != line]results.append(result)if checkpoint and not checkpoint(results=results.copy(), checked=done_units + plan_units(line), total=total_units):diff --git a/src/peach/web_links.py b/src/peach/web_links.pyindex 61bb8da61..dc3bfaa51 100644--- a/src/peach/web_links.py+++ b/src/peach/web_links.py@@ -266,6 +266,34 @@ def _prune_selection(planned: list[dict], body: dict) -> list[dict]:return [item for item in planned if item["id"] in requested]+def _recheck_prune_links(planned, progress, checkpoint, resume_state):+ """保存逐条重验结果,临时无法判定的链接留待重试。"""+ saved = (resume_state or {}).get('prune_checks') or {}+ confirmed = list(saved.get('confirmed') or [])+ recovered = list(saved.get('recovered') or [])+ completed = set(saved.get('completed') or [])+ for index, item in enumerate(planned):+ if item['id'] in completed:+ continue+ if progress:+ progress(checked=index, total=len(planned),+ message=f"重验失效链接:已检查 {index} / {len(planned)} 条")+ status, note = _probe(item['url'])+ verdict = link_verdict(status, note)+ confirmed[:] = [row for row in confirmed if row['id'] != item['id']]+ recovered[:] = [row for row in recovered if row['id'] != item['id']]+ if verdict == 'gone':+ confirmed.append({**item, 'note': _note(status, note)})+ else:+ recovered.append(item)+ if verdict != 'unclear':+ completed.add(item['id'])+ if checkpoint and not checkpoint(prune_checks=dict(confirmed=confirmed, recovered=recovered,+ completed=sorted(completed)), checked=index + 1, total=len(planned)):+ raise InterruptedError('链接清理已停止')+ return confirmed, recovered++def w_links_prune(contract: LinkContract, body, *, progress=None, checkpoint=None, resume_state=None):"""删掉上一次检查判定为 gone 的链接。@@ -295,18 +323,7 @@ def w_links_prune(contract: LinkContract, body, *, progress=None, checkpoint=Nonexcept ValueError as error:return {"ok": False, "error": str(error)}- confirmed, recovered = [], []- for index, item in enumerate(planned):- if progress:- progress(checked=max(index, (resume_state or {}).get('checked', 0)), total=len(planned),- message=f"重验失效链接:已检查 {index} / {len(planned)} 条")- status, note = _probe(item["url"])- if link_verdict(status, note) == "gone":- confirmed.append({**item, "note": _note(status, note)})- else:- recovered.append(item)- if checkpoint and not checkpoint(checked=max(index + 1, (resume_state or {}).get('checked', 0)), total=len(planned)):- raise InterruptedError('链接清理已停止')+ confirmed, recovered = _recheck_prune_links(planned, progress, checkpoint, resume_state)removed = marked = 0if progress:diff --git a/src/peach/web_media_repair.py b/src/peach/web_media_repair.pyindex 94155cbfa..4a6b09b88 100644--- a/src/peach/web_media_repair.py+++ b/src/peach/web_media_repair.py@@ -139,6 +139,10 @@ def _advance(job, job_id: str, checked: int, *, skipped: bool) -> None:state["checked"] = max(state.get('checked', 0), checked)if skipped and checked > state.get('survey_checked', 0):state["skipped"] = state.get("skipped", 0) + 1+ state['survey_checked'] = max(state.get('survey_checked', 0), checked)+ saved = {key: state.get(key, 0) for key in ('checked', 'survey_checked', 'skipped')}+ if not job.checkpoint(job_id, **saved):+ raise InterruptedError('媒体检查已停止')def _reference_candidates(contract, asset_id: int, raw: str) -> list[str]:diff --git a/src/peach/web_resource_sync.py b/src/peach/web_resource_sync.pyindex 1617514f8..8bfb7cb2e 100644--- a/src/peach/web_resource_sync.py+++ b/src/peach/web_resource_sync.py@@ -658,6 +658,33 @@ def _mark_record_holders(contract: ResourceSyncContract,return marked, reattached, [row for row in rows if int(row["id"]) not in kept]+def _apply_missing_records(contract, state, report, checkpoint, resume_state):+ """逐批复核失效记录,保存完成集合与累计回执。"""+ saved = (resume_state or {}).get('apply_checkpoint') or {}+ processed = set(saved.get('processed') or [])+ report(checked=len(processed), total=len(state['vanished_ids']), message="正在逐条复核失效记录")+ marked, reattached = list(saved.get('marked') or []), list(saved.get('reattached') or [])+ purge = dict(saved.get('purge') or {'purged': 0, 'blocked': [], 'empty_dirs_removed': 0})+ pending = [value for value in state['vanished_ids'] if value not in processed]+ for offset in range(0, len(pending), 100):+ selected = pending[offset:offset + 100]+ added, attached, confirmed = _mark_record_holders(contract, _recheck_vanished(contract, selected))+ marked.extend(added)+ reattached.extend(attached)+ if confirmed:+ from .web_batch import purge_vanished_rows+ result = purge_vanished_rows(contract, confirmed)+ for field in ('purged', 'empty_dirs_removed'):+ purge[field] += int(result.get(field) or 0)+ purge['blocked'].extend(result.get('blocked') or [])+ processed.update(selected)+ report(checked=len(processed), total=len(state['vanished_ids']), message='正在处理失效记录')+ if checkpoint and not checkpoint(apply_checkpoint=dict(processed=sorted(processed), marked=marked,+ reattached=reattached, purge=purge)):+ raise InterruptedError('资源清理已停止')+ return marked, reattached, purge++def w_resource_sync_apply(contract: ResourceSyncContract, body, *, progress=None, checkpoint=None, resume_state=None):"""处理这一轮检查报出的失效记录与空文件夹,再清孤儿缓存。@@ -687,28 +714,8 @@ def w_resource_sync_apply(contract: ResourceSyncContract, body, *, progress=Nonereturn resume_state['apply_receipt']state = _completed_scan(contract, scan_id)report = progress or (lambda **_fields: None)- saved = (resume_state or {}).get('apply_checkpoint') or {}- processed = set(saved.get('processed') or [])- report(checked=len(processed), total=len(state['vanished_ids']), message="正在逐条复核失效记录")- marked, reattached = list(saved.get('marked') or []), list(saved.get('reattached') or [])- purge = dict(saved.get('purge') or {'purged': 0, 'blocked': [], 'empty_dirs_removed': 0})- pending = [value for value in state['vanished_ids'] if value not in processed]- for offset in range(0, len(pending), 100):- selected = pending[offset:offset + 100]- added, attached, confirmed = _mark_record_holders(contract, _recheck_vanished(contract, selected))- marked.extend(added)- reattached.extend(attached)- if confirmed:- from .web_batch import purge_vanished_rows- result = purge_vanished_rows(contract, confirmed)- for field in ('purged', 'empty_dirs_removed'):- purge[field] += int(result.get(field) or 0)- purge['blocked'].extend(result.get('blocked') or [])- processed.update(selected)- report(checked=len(processed), total=len(state['vanished_ids']), message='正在处理失效记录')- if checkpoint and not checkpoint(apply_checkpoint=dict(processed=sorted(processed), marked=marked,- reattached=reattached, purge=purge)):- raise InterruptedError('资源清理已停止')+ marked, reattached, purge = _apply_missing_records(+ contract, state, report, checkpoint, resume_state)report(checked=0, total=None, message="正在删除空文件夹")dirs_removed, dir_errors = 0, 0for location, paths in state["empty_dirs"].items():diff --git a/tests/test_follow_web.py b/tests/test_follow_web.pyindex 25cc1745f..85c4c8979 100644--- a/tests/test_follow_web.py+++ b/tests/test_follow_web.py@@ -105,6 +105,32 @@ class FollowContractTests(unittest.TestCase):self.assertFalse(final['results'][0]['ok'])self.assertEqual(factory.call_count, 1)+ def test_finished_page_checkpoint_resumes_without_another_page_request(self):+ source = self._seed()+ for backfill_all in (False, True):+ with self.subTest(backfill_all=backfill_all):+ saved = {}+ def checkpoint(_job_id, *, saved=saved, **fields):+ saved.update(fields)+ if fields.get('results'):+ raise InterruptedError('停止于检查点')+ return True+ job = SimpleNamespace(snapshot=lambda saved=saved: saved, checkpoint=checkpoint,+ update=lambda *a, **k: True)+ body = {'older': True, 'rewind': True, 'backfill_all': backfill_all}+ result = {'source': source, 'ok': True, 'page': 3, 'exhausted': backfill_all}+ with mock.patch.object(self.contract, 'follow_job', job), \+ mock.patch.object(web_follow, 'run_check', return_value=result) as check, \+ mock.patch.object(web_follow, '_check_payload', side_effect=lambda value: value):+ with self.assertRaises(InterruptedError):+ web_follow._run_follow_check(self.contract, body, 'original')+ self.assertEqual(saved['backfill_pages'], {str(source): 3})+ self.assertEqual(saved['results'][0]['source'], source)+ job.checkpoint = lambda *a, **k: True+ resumed = web_follow._run_follow_check(self.contract, body, 'original')+ self.assertEqual(check.call_count, 1)+ self.assertEqual(resumed['checked'], 1)+def test_discovery_result_can_be_read_after_request_returns(self):started = self._post('/api/follow/resolve', {'background': True,'lines': ['https://rule34video.com/models/sample/']})diff --git a/tests/test_jav_discovery_metadata.py b/tests/test_jav_discovery_metadata.pyindex 513c40a6d..a873bd7e2 100644--- a/tests/test_jav_discovery_metadata.py+++ b/tests/test_jav_discovery_metadata.py@@ -35,6 +35,25 @@ class DiscoveryMetadataTests(unittest.TestCase):self.cache = DiscoveryMetadata(self.path, api=self.api, clock=lambda: self.now[0])self.addCleanup(self.cache.stop)+ def test_repeated_restart_keeps_checkpoint_before_the_next_item_finishes(self):+ self.path.with_suffix('.job.json').write_text(json.dumps(dict(+ job_id='original', status='running', codes=['EXM-001', 'EXM-002'], refresh=True,+ started_at=100, completed=1, completed_codes=['EXM-001'], attempted_codes=['EXM-001'],+ failed_codes=[], requests=3, bytes=120, active_seconds=10)), encoding='utf-8')+ with patch.object(DiscoveryMetadata, '_run_batch'):+ first = DiscoveryMetadata(self.path, api=self.api, clock=lambda: self.now[0])+ first.job.thread.join(5)+ first.stop()+ saved = json.loads(self.path.with_suffix('.job.json').read_text(encoding='utf-8'))+ self.assertEqual(saved['completed_codes'], ['EXM-001'])+ self.assertEqual(saved['started_at'], 100)+ with patch.object(self.api, 'movie', return_value={'number': 'EXM-002', 'tags': [], 'score': 4, 'duration': 120}) as movie:+ second = DiscoveryMetadata(self.path, api=self.api, clock=lambda: self.now[0])+ self.addCleanup(second.stop)+ second.job.thread.join(5)+ movie.assert_called_once_with('EXM-002', refresh=True)+ self.assertEqual(second.job.snapshot()['completed'], 2)+def test_sidecar_survives_restart_and_distinguishes_empty_unknown_and_stale(self):movie = {"number": "EXM-001", "tags": [], "score": "4.23", "duration": 180,"preview_video_url": "https://example.invalid/secret",diff --git a/tests/test_web_links.py b/tests/test_web_links.pyindex cd9cd9136..788305b5c 100644--- a/tests/test_web_links.py+++ b/tests/test_web_links.py@@ -3,6 +3,7 @@ import sqlite3import sysimport tempfileimport unittest+from unittest import mockfrom contextlib import contextmanagerfrom pathlib import Path@@ -56,6 +57,24 @@ class FakeContract:class VerdictTests(unittest.TestCase):+ def test_prune_checks_retain_definitive_results_across_interruption(self):+ planned = [{'id': index, 'url': f'https://example.invalid/{index}'} for index in (1, 2, 3)]+ saved = {}+ def remember(**fields):+ saved.update(fields)+ if fields['checked'] == 2:+ raise InterruptedError('停止于检查点')+ return True+ with mock.patch.object(web_links, '_probe', side_effect=[(404, ''), (503, '')]):+ with self.assertRaises(InterruptedError):+ web_links._recheck_prune_links(planned, None, remember, {})+ with mock.patch.object(web_links, '_probe', side_effect=[(200, ''), (404, '')]) as probe:+ confirmed, recovered = web_links._recheck_prune_links(planned, None, None, saved)+ self.assertEqual([call.args[0] for call in probe.call_args_list],+ ['https://example.invalid/2', 'https://example.invalid/3'])+ self.assertEqual([row['id'] for row in confirmed], [1, 3])+ self.assertEqual([row['id'] for row in recovered], [2])+def test_only_a_gone_status_is_deletable(self):"""取不到不等于没了,这两件事只有 404/410 能划清。diff --git a/tests/test_web_resource_sync.py b/tests/test_web_resource_sync.pyindex cec7d18fb..35aab85af 100644--- a/tests/test_web_resource_sync.py+++ b/tests/test_web_resource_sync.py@@ -317,6 +317,24 @@ class PurgeMissingTests(unittest.TestCase):self.assertEqual((result['purged'], result['vanished']), (2, 1))self.assertEqual(self.ids(), [1, 2, 4])+ def test_apply_resumes_after_a_committed_chunk_with_cumulative_receipt(self):+ saved = {}+ def remember(**fields):+ saved.update(fields)+ if 'apply_checkpoint' in fields:+ raise InterruptedError('停止于检查点')+ return True+ with self._synced():+ scan = finish_scan(self.contract)+ body = {'confirm': True, 'clean_cache': False, 'scan_id': scan['scan_id']}+ with self.assertRaises(InterruptedError):+ rm_sync.w_resource_sync_apply(self.contract, body, checkpoint=remember)+ self.assertEqual(self.ids(), [1, 2, 4])+ with mock.patch.object(rm_sync, '_recheck_vanished', wraps=rm_sync._recheck_vanished) as recheck:+ result = rm_sync.w_resource_sync_apply(self.contract, body, resume_state=saved)+ recheck.assert_not_called()+ self.assertEqual((result['purged'], result['vanished']), (1, 1))+def test_local_only_configuration_has_resource_check(self):with mock.patch.object(rm_sync, 'LOCATION_ROOT_DECLARATIONS', {'local': ('R:\\',)}):self.assertEqual(rm_sync.configured_resource_locations(), ('local',))