Skip to content

armgit / peach

Public Imported from GitHub
0

Peach: Local-first video metadata scraper and media manager.

feat(library): 支持处理暂停、低占用模式与启动耗时粗估

longmeidao357d1c3625ec
diff --git a/docs/REUSE.md b/docs/REUSE.md
index df8f5a195..8083f7a13 100644
--- a/docs/REUSE.md
+++ b/docs/REUSE.md
@@ -609,3 +609,9 @@ Python 复用 MIT 许可的 [Ruff](https://docs.astral.sh/ruff/) 0.17.0,版本
`scripts/dev/copy-editor.js` 复用官网本地预览的 `edit-client.js` 交互:文字范围提示、原位编辑、按钮文字就近输入、回车和失焦保存、Esc 取消。
应用的 React 与动态页面使用现有静态文案定位、摘要校验和备份入口;演示站使用浏览器草稿。两者只在 `?edit` 启用,无新增依赖。
+
+## 馆藏处理控制与耗时预估
+
+扫描、探测、资料请求和九宫格复用 `process_library`、`scan_location`、`SourceTransport`、`frame_capture` 与已有影片检查点;任务恢复沿用 `BackgroundJob`、`job_recovery` 和 `task_run`,用户暂停意图单独持久保存。当前树及 `library_processing.py` 的 Git 历史已核对,安装暂停只用于安装事务,不能直接代表用户暂停。Python 标准库 `threading.Event` 与 `contextvars.ContextVar` 提供等待唤醒和当前任务隔离,沿用项目 Python 版本,跨 Windows 与 macOS,无额外依赖体积;许可证为 PSF,官方文档与实现见 [threading](https://docs.python.org/3/library/threading.html) 和 [contextvars](https://docs.python.org/3/library/contextvars.html)。现有 APScheduler 3.11.3 用于定时派发,不能暂停已进入媒体读取的函数,因此不承担安全停点。隔离测试使用临时账本、暂停线程与模拟时钟验证安全停点、跨重启意图、低占用间隔和暂停预算补偿,不读取真实媒体、不发外部请求。
+
+`processing_control` 保留 Peach 的阶段、模式与估时策略:启动仅汇总账本文件数和大小,优先使用本机同类完成耗时;未知新增明确不在初始范围内。阶段样本形成后按实际耗时校准,并返回估时范围、依据和范围所指阶段。低占用在单项读盘、探测、下载和抽帧之间留出间隔,FFmpeg 使用单线程解码与滤镜;只作用于当前任务。持久状态不修改 ledger schema,不改变生产入口。
diff --git a/src/peach/frame_capture.py b/src/peach/frame_capture.py
index 77078b1d2..99ca4c615 100644
--- a/src/peach/frame_capture.py
+++ b/src/peach/frame_capture.py
@@ -11,6 +11,7 @@ import os
import re
import subprocess
from pathlib import Path
+from .processing_control import checkpoint, low_mode
#: 部分网盘视频把色彩原色写成 reserved(非法值),swscale 会拒绝缩放;声明为 bt709
#: 只是覆盖坏的元数据,不改动像素。
@@ -34,11 +35,15 @@ def capture_frame(ffmpeg: str, path: str, timestamp: float, destination: Path, *
那个时间点,两小时的片子抽最后一帧要几分钟。
"""
command = [ffmpeg, "-y", "-v", "error", "-rw_timeout", "8000000"]
+ checkpoint('frame')
+ if low_mode():
+ command += ['-threads', '1', '-filter_threads', '1']
if color_override:
command += COLOR_OVERRIDE
command += [
"-ss", f"{timestamp:.2f}", "-i", path, "-frames:v", "1",
- "-vf", f"scale={int(width)}:-1", "-q:v", "4", str(destination),
+ "-vf", f"scale={int(width)}:-1", "-q:v", "4",
+ *(['-threads', '1'] if low_mode() else []), str(destination),
]
try:
completed = subprocess.run(command, capture_output=True, timeout=timeout,
@@ -68,9 +73,12 @@ def tile_frames(ffmpeg: str, pattern: str, columns: int, rows: int, destination:
于是拼出来的图少了后半段,而命令本身照样退出 0。
"""
destination.parent.mkdir(parents=True, exist_ok=True)
+ checkpoint('frame')
+ options = ['-threads', '1', '-filter_complex_threads', '1'] if low_mode() else []
subprocess.run(
- [ffmpeg, "-y", "-v", "error", "-i", pattern,
- "-filter_complex", f"tile={int(columns)}x{int(rows)}", "-q:v", "4", str(destination)],
+ [ffmpeg, "-y", "-v", "error", *options, "-i", pattern,
+ "-filter_complex", f"tile={int(columns)}x{int(rows)}", "-q:v", "4",
+ *(['-threads', '1'] if low_mode() else []), str(destination)],
capture_output=True, timeout=timeout,
creationflags=subprocess.CREATE_NO_WINDOW if os.name == "nt" else 0,
)
diff --git a/src/peach/job_recovery.py b/src/peach/job_recovery.py
index 24430319d..787a09ce8 100644
--- a/src/peach/job_recovery.py
+++ b/src/peach/job_recovery.py
@@ -94,6 +94,10 @@ def resume(db: Path, handlers: dict) -> list[str]:
with admission(db):
data = read(db, key)
pid = data.get("pid")
+ if key == 'library-processing':
+ from .processing_control import user_paused
+ if user_paused(db, data.get('job_id')):
+ continue
if (data.get("status") != "active" or not isinstance(data.get("request"), dict)
or (isinstance(pid, int) and pid != os.getpid() and process_alive(pid))):
continue
diff --git a/src/peach/jobs.py b/src/peach/jobs.py
index 0241d99d9..5a9a1690c 100644
--- a/src/peach/jobs.py
+++ b/src/peach/jobs.py
@@ -386,6 +386,8 @@ class BackgroundJob:
@staticmethod
def _progress_label(state: dict) -> str:
"""一行「此刻在做什么」。域各有各的字段名,这里按可读性从具体到笼统取第一个。"""
+ if state.get('status') in ('paused', 'pausing'):
+ return ('已暂停' if state['status'] == 'paused' else '正在暂停') + ' · ' + str(state.get('stage') or '')
current = state.get("current")
if isinstance(current, dict):
for key in ("label", "provider"):
@@ -548,7 +550,7 @@ class BackgroundJob:
self.state = None
thread = self.thread
self.thread = None
- if job_id and state is not None and state["status"] == "running":
+ if job_id and state is not None and state["status"] in ('running', 'pausing', 'paused'):
self._stop_run(job_id, resume=resume)
if thread is not None and thread.is_alive():
thread.join(timeout)
@@ -556,14 +558,23 @@ class BackgroundJob:
def _stop_run(self, job_id, *, resume):
if resume and job_id in self._recoverable:
from .job_recovery import read
- if read(self._database_path(), self.task_key).get('pause_requested'):
+ from .processing_control import user_paused
+ if (read(self._database_path(), self.task_key).get('pause_requested') or
+ self.task_key == 'library-processing' and user_paused(self._database_path(), job_id)):
self._paused_stops.add(job_id)
run_id = self._run_ids.pop(job_id, None)
if self.runs is not None and run_id is not None:
- self.runs.suspend(run_id)
+ self._suspend_run(job_id, run_id)
return
self._close_run(job_id, 'cancelled', error='服务关停,这一轮没有跑完')
+ def _suspend_run(self, job_id, run_id):
+ from .processing_control import user_paused
+ if self.task_key == 'library-processing' and user_paused(self._database_path(), job_id):
+ self.runs.suspend(run_id, label='已暂停,等待继续')
+ else:
+ self.runs.suspend(run_id)
+
def start_result(self, fn: Callable[[], dict], *, recovery: dict | None = None) -> dict:
"""执行一次操作并保存终态回执;重复提交只返回进行中的同一任务。"""
def work(job_id):
@@ -636,7 +647,7 @@ class BackgroundJob:
from .job_recovery import waiting
run_id = self._run_ids.pop(job_id, None)
if self.runs is not None and run_id is not None:
- self.runs.suspend(run_id)
+ self._suspend_run(job_id, run_id)
waiting(self._database_path(), self.task_key, job_id)
return
if state.get(self.id_key) != job_id:
diff --git a/src/peach/library_processing.py b/src/peach/library_processing.py
index 1506a0c90..1a900663d 100644
--- a/src/peach/library_processing.py
+++ b/src/peach/library_processing.py
@@ -40,6 +40,9 @@ from .media_probe import probe_unmeasured
from .media_processing_policy import metadata_only
from .strm import skip_background_read
from .record_rehome import reattach_in
+from .processing_control import (task_control, bind_control, checkpoint_values,
+ initial_estimate, ProcessingEstimate, interrupted_state,
+ deadline_after, resume_fields)
FIELDS = ('item_key', 'code', 'query', 'asset_id', 'asset_path', 'field', 'field_label', 'current_value',
'candidates_json', 'source_count', 'source_profile', 'policy_version', 'status',
@@ -354,6 +357,8 @@ class LibraryMetadataProvider:
from .config import TOOLS_DIR
from .jav_cover_fetch import Unavailable
from .scraping_access import SourcePaused, pause_source, values_for, record_source_check
+ from .processing_control import checkpoint
+ checkpoint('network')
timeout = None
if deadline is not None:
timeout = deadline - time.monotonic()
@@ -1367,7 +1372,7 @@ class _RemoteSession:
budget = ACTION_BUDGETS[action]
update(stage='采集缺失资料', current_action=action,
current_started_at=time.time(), current_deadline_at=time.time() + budget)
- deadline = time.monotonic() + budget
+ deadline = deadline_after(budget)
mark = _excused(self._provider)
evidence = _studio_evidence(row)
chain = self._chain(code, *evidence)
@@ -1466,7 +1471,7 @@ class _RemoteSession:
update(stage='采集缺失封面', current_action=action,
current_started_at=time.time(), current_deadline_at=time.time() + budget)
try:
- return int(self.provider().cover(code, cover_root, deadline=time.monotonic() + budget,
+ return int(self.provider().cover(code, cover_root, deadline=deadline_after(budget),
evidence=_studio_evidence(row), snapshots=snapshots,
absent=absent))
except DeadlineExceeded:
@@ -1904,7 +1909,7 @@ def process_library(config, db_path, candidate_root, cover_root, *, location='co
report=lambda state: None, provider_factory=None, job_id=None,
retry_ids=None, active=lambda: True, stage=ALL_STAGES,
database=None, route_overrides=None, state_file=None, asset_ids=None,
- resume_state=None):
+ resume_state=None, speed_mode=None):
"""登记文件与确定的番号,外部资料保留为可复核候选。
`retry_ids` 为 `None` 时处理整个馆藏;给定时只处理这些项目(上一任务记录的
@@ -1929,9 +1934,20 @@ def process_library(config, db_path, candidate_root, cover_root, *, location='co
targeted = retrying or selected_ids is not None
chosen_ids = selected_ids if selected_ids is not None else list(dict.fromkeys(map(int, retry_ids or [])))
scope_ids = chosen_ids if targeted else None
+ control = task_control(config, state_file)
+ identity = job_id or uuid.uuid4().hex
+ def control_report(**values):
+ resume_fields(state, values)
+ state.update(values)
+ estimate.update(state)
+ flush_state(force=True)
+ report(dict(state))
from .update_tasks import activity
- with activity(Path(db_path), 'library-processing'), FileLock(str(state_path(config)) + '.lock', timeout=0):
- state = dict(job_id=job_id or uuid.uuid4().hex, status='running',
+ with (activity(Path(db_path), 'library-processing'),
+ FileLock(str(state_path(config)) + '.lock', timeout=0),
+ bind_control(control, identity, publish=control_report, active=active, db_path=Path(db_path),
+ mode=speed_mode)):
+ state = dict(job_id=identity, status='running',
stage='读取本地资料' if retrying else '扫描文件',
checked=0, total=0, scanned=0, probed=0, identified=0, candidates=0, covers=0, previews=0,
requested_stage=stage, asset_ids=scope_ids,
@@ -1942,6 +1958,8 @@ def process_library(config, db_path, candidate_root, cover_root, *, location='co
current_started_at=None, current_deadline_at=None,
followups=[],
started_at=time.time(), error='')
+ state.update(initial_estimate(config, db_path, stage, scope_ids, speed_mode))
+ estimate = ProcessingEstimate(config, state, control)
# 实体表的水位在开工前记一次:比它大的实体就是这一轮建出来的(`_entity_followups`)。
checkpoints = _LibraryCheckpoints(state, resume_state, database)
entity_watermark = checkpoints.entity_watermark
@@ -1964,7 +1982,9 @@ def process_library(config, db_path, candidate_root, cover_root, *, location='co
_require_writer(config, db_path)
if not active():
raise InterruptedError('处理任务已停止')
+ checkpoint_values(values)
state.update(values)
+ estimate.update(state)
state['last_progress_at'] = time.time()
state['progress_seq'] += 1
flush_state(force='status' in values)
@@ -2126,6 +2146,7 @@ def process_library(config, db_path, candidate_root, cover_root, *, location='co
current_action='', current_started_at=None, current_deadline_at=None)
except Exception:
state.update(status='failed', error='处理被中断。核对媒体目录后重试。', completed_at=time.time())
+ state.update(interrupted_state(control))
_save(path, state)
raise
finally:
diff --git a/src/peach/media_probe.py b/src/peach/media_probe.py
index 2bc15491f..e73ee6c7f 100644
--- a/src/peach/media_probe.py
+++ b/src/peach/media_probe.py
@@ -25,6 +25,7 @@ from .jobs import ACTIVE_ASSET_SQL
from .media import resolve_case_insensitive
from .media_sources import cost_for
from .strm import skip_background_read
+from .processing_control import CURRENT, checkpoint
#: 一次探测最多等多久。ffprobe 只读文件头,网盘上实测一两秒。
TIMEOUT_SECONDS = 20.0
@@ -142,6 +143,18 @@ def _record(db_path, ffprobe: str, rows: Sequence, report: Callable[[int, int],
if not rows:
return 0
done = 0
+ if CURRENT.get() is not None:
+ # 馆藏任务逐项领取探测,不预先提交整库,安全停点没有仍在读盘的排队任务。
+ with closing(sqlite3.connect(db_path, timeout=120)) as connection:
+ report(0, len(rows))
+ for row in rows:
+ checkpoint('probe')
+ measured = measure(ffprobe, row[0], row[1])
+ connection.execute(UPDATE, measured)
+ connection.commit()
+ done += 1
+ report(done, len(rows))
+ return done
executor = ThreadPoolExecutor(max_workers=min(WORKERS, len(rows)), thread_name_prefix="PeachProbe")
try:
with closing(sqlite3.connect(db_path, timeout=120)) as connection:
diff --git a/src/peach/metadata_amane.py b/src/peach/metadata_amane.py
index f4e7fbcdf..8b6c2a88d 100644
--- a/src/peach/metadata_amane.py
+++ b/src/peach/metadata_amane.py
@@ -20,6 +20,7 @@ import os
import re
import shutil
import subprocess
+import time
import tomllib
from dataclasses import dataclass
from pathlib import Path
@@ -284,6 +285,11 @@ class AmaneBridge:
raise ValueError("未开放的 amane 站点:" + "、".join(unknown or ["(空)"]))
if set(direct_sites) - set(names):
raise ValueError("直连站点必须属于本次查询来源")
+ from .processing_control import low_mode, checkpoint
+ if low_mode() and len(names) > 1:
+ return self._serial_query(code, names, language=language, proxy_options=proxy_options,
+ direct_sites=direct_sites, timeout=timeout)
+ checkpoint('network')
proxy_args, env = proxy_argument(proxy_options or {"trust_env": True})
env["PYTHONIOENCODING"] = "utf-8"
command = [str(self.python), "-X", "utf8", str(self.script), "--number", number,
@@ -312,6 +318,21 @@ class AmaneBridge:
raise MetadataProviderError(str(message or f"amane 桥查询失败(exit {completed.returncode})"))
return report
+ def _serial_query(self, code, sites, *, language, proxy_options, direct_sites, timeout):
+ """低占用逐站启动桥查询,保留统一的整档时间预算。"""
+ from .processing_control import checkpoint, deadline_after
+ deadline = deadline_after(timeout) if timeout else None
+ records = {}
+ for site in sites:
+ checkpoint('network')
+ remaining = deadline - time.monotonic() if deadline is not None else None
+ if remaining is not None and remaining <= 1:
+ raise MetadataProviderError('amane 桥查询超时', kind='unavailable', retryable=True, temporary=True)
+ report = self.query(code, [site], language=language, proxy_options=proxy_options,
+ direct_sites=[site] if site in direct_sites else [], timeout=remaining)
+ records.update(report.get('sites') or {})
+ return {'number': code, 'sites': records}
+
def failure_error(site: str, record: Mapping[str, object]) -> MetadataProviderError:
"""桥记录里一站的失败 → Peach 的错误对象。上游 reason 留在 `detail`,措辞里也带着。"""
diff --git a/src/peach/processing_control.py b/src/peach/processing_control.py
new file mode 100644
index 000000000..ef42a199d
--- /dev/null
+++ b/src/peach/processing_control.py
@@ -0,0 +1,321 @@
+"""馆藏任务的安全停点、处理偏好和带依据的耗时范围。"""
+from __future__ import annotations
+
+import json
+import math
+import sqlite3
+import threading
+import time
+from contextlib import contextmanager, closing, nullcontext
+from contextvars import ContextVar
+
+from .fsutil import atomic_write_text
+
+CURRENT = ContextVar('peach_library_processing_control', default=None)
+_CONTROLS = {}
+_LOCK = threading.Lock()
+MODES = ('standard', 'low')
+
+
+def valid_mode(value):
+ if value not in MODES:
+ raise ValueError('请选择标准或低占用模式')
+ return value
+
+
+def _read(path):
+ try:
+ data = json.loads(path.read_text(encoding='utf-8'))
+ return data if isinstance(data, dict) else {}
+ except (OSError, ValueError):
+ return {}
+
+
+def control_for(config):
+ path = (config.directory('state') / 'library-processing-control.json').resolve()
+ with _LOCK:
+ if path not in _CONTROLS:
+ _CONTROLS[path] = ProcessingControl(path)
+ return _CONTROLS[path]
+
+
+def task_control(config, state_file):
+ return control_for(config) if state_file is None else None
+
+
+def bind_control(control, *args, **kwargs):
+ return control.bind(*args, **kwargs) if control is not None else nullcontext()
+
+
+def checkpoint_values(values):
+ before = time.monotonic()
+ checkpoint()
+ waited = time.monotonic() - before
+ for key in ('current_started_at', 'current_deadline_at'):
+ if values.get(key):
+ values[key] += waited
+
+
+def interrupted_state(control):
+ return dict(status='paused', error='', completed_at=None) if control is not None and control.paused else {}
+
+
+def resume_fields(state, values):
+ waited = values.pop('resumed_after', 0)
+ for key in ('current_started_at', 'current_deadline_at', 'last_progress_at'):
+ if waited and state.get(key):
+ state[key] += waited
+
+
+def checkpoint(kind='work'):
+ control = CURRENT.get()
+ if control is not None:
+ control.checkpoint(kind)
+
+
+def low_mode():
+ control = CURRENT.get()
+ return control is not None and control.mode == 'low'
+
+
+class ActiveDeadline(float):
+ """单调时钟截止点随本任务的用户暂停时长平移。"""
+ def __new__(cls, value, control, paused=None):
+ result = super().__new__(cls, value)
+ result.control = control
+ result.paused = control.paused_seconds if paused is None else paused
+ return result
+
+ def effective(self):
+ return float(self) + self.control.paused_seconds - self.paused
+
+ def __sub__(self, other):
+ return self.effective() - other
+
+ def __rsub__(self, other):
+ return other - self.effective()
+
+ def __add__(self, other):
+ return ActiveDeadline(float(self) + other, self.control, self.paused)
+
+ __radd__ = __add__
+
+ def __lt__(self, other):
+ return self.effective() < other
+
+ def __le__(self, other):
+ return self.effective() <= other
+
+ def __gt__(self, other):
+ return self.effective() > other
+
+ def __ge__(self, other):
+ return self.effective() >= other
+
+
+def deadline_after(seconds):
+ value, control = time.monotonic() + seconds, CURRENT.get()
+ return ActiveDeadline(value, control) if control is not None else value
+
+
+class ProcessingControl:
+ def __init__(self, path):
+ self.path = path
+ self.lock = threading.RLock()
+ data = _read(path)
+ self.mode = data.get('speed_mode') if data.get('speed_mode') in MODES else 'standard'
+ self.job_id = data.get('job_id')
+ self.paused = data.get('paused') is True
+ self.event = threading.Event()
+ self.event.set()
+ self.publish = None
+ self.active = lambda: True
+ self.db_path = None
+ self.next_work = {}
+ self.paused_seconds = 0.0
+ self.reloaded_at = 0.0
+
+ def refresh(self, force=False):
+ with self.lock:
+ now = time.monotonic()
+ if not force and now - self.reloaded_at < 0.5:
+ return
+ self.reloaded_at = now
+ data = _read(self.path)
+ if data.get('job_id') == self.job_id or self.publish is None:
+ self.job_id = data.get('job_id', self.job_id)
+ self.paused = data.get('paused', self.paused)
+ mode = data.get('speed_mode', self.mode)
+ if mode in MODES and self.mode != mode:
+ self.mode = mode
+ self.next_work.clear()
+
+ def _save(self):
+ atomic_write_text(self.path, json.dumps(dict(
+ speed_mode=self.mode, job_id=self.job_id, paused=self.paused)) + '\n')
+
+ def request(self, action, job_id=None, mode=None):
+ with self.lock:
+ if job_id is not None and job_id != self.job_id:
+ raise ValueError('任务已变化,请刷新页面后重试')
+ if action == 'set_speed':
+ self.mode = valid_mode(mode)
+ self.next_work.clear()
+ else:
+ self.paused = action == 'pause'
+ self._save()
+ self.event.set()
+
+ def select_job(self, job_id):
+ """调用方完成任务身份校验后,承接尚未进入首个安全停点的任务。"""
+ with self.lock:
+ if self.job_id != job_id:
+ self.job_id, self.paused, self.paused_seconds = job_id, False, 0.0
+
+ @contextmanager
+ def bind(self, job_id, *, publish, active, db_path, mode=None):
+ with self.lock:
+ self.select_job(job_id)
+ if mode is not None:
+ self.mode = valid_mode(mode)
+ self.publish, self.active, self.db_path = publish, active, db_path
+ self.next_work.clear()
+ self._save()
+ token = CURRENT.set(self)
+ try:
+ yield self
+ finally:
+ CURRENT.reset(token)
+ self.publish = None
+
+ def checkpoint(self, kind='work'):
+ from .update_tasks import check_pause
+ started = None
+ while True:
+ check_pause(self.db_path)
+ self.refresh()
+ if not self.active():
+ raise InterruptedError('处理任务已停止')
+ with self.lock:
+ paused, mode = self.paused, self.mode
+ self.event.clear()
+ if paused:
+ if started is None:
+ started = time.monotonic()
+ self.publish(status='paused', speed_mode=mode)
+ self.event.wait(0.25)
+ continue
+ if started is not None:
+ spent = time.monotonic() - started
+ self.paused_seconds += spent
+ self.publish(status='running', speed_mode=mode, resumed_after=spent)
+ started = None
+ # 控制只作用于当前馆藏任务;在单项 I/O 之间留出空档。
+ delay = {'scan': 0.008, 'probe': 0.75, 'network': 1.5, 'frame': 0.75,
+ 'work': 0.15}.get(kind, 0.15) if mode == 'low' else 0
+ remaining = self.next_work.get(kind, 0) - time.monotonic()
+ if remaining > 0:
+ self.event.wait(min(0.25, remaining))
+ continue
+ self.next_work[kind] = time.monotonic() + delay
+ return
+
+
+def user_paused(db_path, job_id):
+ from .update_tasks import state_for
+ data = _read(state_for(db_path).parent / 'library-processing-control.json')
+ return data.get('job_id') == job_id and data.get('paused') is True
+
+
+def initial_estimate(config, db_path, stage, asset_ids=None, mode=None):
+ """仅使用账本统计,不在启动 HTTP 请求里遍历媒体目录。"""
+ mode = mode or control_for(config).mode
+ query = "SELECT COUNT(*), COALESCE(SUM(size),0) FROM asset WHERE medium='video' AND disposal IS NULL"
+ parameters = []
+ if asset_ids is not None:
+ query += ' AND id IN (' + ','.join('?' * len(asset_ids)) + ')'
+ parameters += asset_ids
+ if stage == 'previews':
+ query += " AND location='local' AND (snapshot_path IS NULL OR snapshot_path='')"
+ if config.locations:
+ query += ' AND location IN (' + ','.join('?' * len(config.locations)) + ')'
+ parameters += list(config.locations)
+ with closing(sqlite3.connect(db_path, timeout=30)) as connection:
+ count, size = connection.execute(query, parameters).fetchone()
+ # 抽帧只读取局部数据;文件总大小仅作为编解码复杂度的温和修正,不按全量读取估时。
+ size_factor = min(2.0, max(0.8, (size / max(1, count) / 1024**3) ** 0.15))
+ seconds = {'scan': 0.2, 'metadata': 8.0, 'previews': 12.0 * size_factor,
+ 'collect': 20.0 * size_factor, 'all': 20.2 * size_factor}[stage]
+ history = _read(config.directory('state') / 'library-processing-timing.json')
+ sample = history.get(stage)
+ historical = isinstance(sample, (int, float)) and math.isfinite(sample) and sample > 0
+ if historical:
+ seconds = sample
+ factor = 1.8 if mode == 'low' else 1.0
+ known = bool(count) or asset_ids is not None or stage not in ('scan', 'all')
+ estimate = count * seconds * factor
+ return dict(speed_mode=mode,
+ eta_seconds_min=max(1, math.ceil(estimate * 0.4)) if known and count else 0 if known else None,
+ eta_seconds_max=max(1, math.ceil(estimate * 2.5)) if known and count else 0 if known else None,
+ eta_kind='historical' if historical else 'estimating', eta_initial=True,
+ eta_scope='本趟处理(已登记文件,未含未知新增)' if stage in ('scan', 'all') else '本趟处理',
+ eta_basis=f'根据已登记的 {count} 个视频、{size / 1024**3:.1f} GiB'
+ + ('及本机同类历史耗时粗估' if historical else '粗估,运行后校准')
+ if known else '正在统计新目录的文件数量和大小',
+ estimate_file_count=count, estimate_bytes=size)
+
+
+class ProcessingEstimate:
+ def __init__(self, config, state, control):
+ self.config, self.control = config, control
+ self.started = time.monotonic()
+ self.phase = None
+ self.phase_started = self.started
+ self.base_done = 0
+ self.paused_at_start = control.paused_seconds if control is not None else 0
+ self.mode = state['speed_mode']
+
+ def paused_elapsed(self):
+ return self.control.paused_seconds - self.paused_at_start if self.control is not None else 0
+
+ def update(self, state):
+ mode = self.control.mode if self.control is not None else self.mode
+ if mode != self.mode:
+ ratio = 1.8 if mode == 'low' else 1 / 1.8
+ for key in ('eta_seconds_min', 'eta_seconds_max'):
+ if isinstance(state.get(key), (int, float)):
+ state[key] = math.ceil(state[key] * ratio)
+ self.mode, self.phase = mode, None
+ state['eta_initial'] = True
+ state['speed_mode'] = mode
+ if state.get('status') in ('complete', 'failed'):
+ state.update(eta_seconds_min=0, eta_seconds_max=0)
+ if state.get('status') == 'complete' and state.get('estimate_file_count', 0) > 0:
+ elapsed = max(0.01, time.monotonic() - self.started - self.paused_elapsed())
+ path = self.config.directory('state') / 'library-processing-timing.json'
+ history = _read(path)
+ history[state['requested_stage']] = elapsed / state['estimate_file_count'] / (1.8 if mode == 'low' else 1)
+ atomic_write_text(path, json.dumps(history) + '\n')
+ return
+ if state.get('status') != 'running':
+ return
+ stage = state.get('stage')
+ phase = ('previews' if stage == '生成预览图' else
+ 'metadata' if stage in ('读取本地资料', '采集缺失资料', '采集缺失封面', '保存资料候选') else
+ 'probe' if stage == '读取时长与分辨率' else None)
+ done, total = state.get('checked', 0), state.get('total', 0)
+ now = time.monotonic() - self.paused_elapsed()
+ if phase != self.phase:
+ self.phase, self.phase_started, self.base_done = phase, now, done
+ completed = done - self.base_done
+ if phase is None or completed < 2 or total <= done:
+ return
+ elapsed = max(0.01, now - self.phase_started)
+ per_item = elapsed / completed
+ # 采样增多时收窄当前阶段的范围,范围所指阶段随结果一同返回。
+ uncertainty = max(0.2, 1 / math.sqrt(completed))
+ remaining = (total - done) * per_item
+ state.update(eta_seconds_min=max(1, math.ceil(remaining * (1 - uncertainty))),
+ eta_seconds_max=max(1, math.ceil(remaining * (1 + uncertainty))),
+ eta_kind='measured', eta_initial=False, eta_scope=f'当前阶段:{stage}',
+ eta_basis=f'根据当前阶段已完成的 {completed} 个视频耗时校准')
diff --git a/src/peach/scan.py b/src/peach/scan.py
index a69e8486a..2fbc4f3d3 100644
--- a/src/peach/scan.py
+++ b/src/peach/scan.py
@@ -40,6 +40,7 @@ from pathlib import Path, PureWindowsPath
from . import folder_rules, record_rehome, subtitles, wants
from .platform import is_windows_path, resolve_location, resolve_root
+from .processing_control import checkpoint
VIDEO = {".mp4", ".m4v", ".mkv", ".avi", ".wmv", ".mov", ".ts", ".flv", ".rmvb", ".mpg",
".m2ts", ".strm", ".iso"}
@@ -344,6 +345,7 @@ def _walk(top: Path):
"""
stack = [os.fspath(top)]
while stack:
+ checkpoint('scan')
directory = stack.pop()
try:
with os.scandir(directory) as scanner:
@@ -400,6 +402,7 @@ def scan_location(
try:
connection.execute("PRAGMA journal_mode=WAL")
for directory, entries in _walk(walk_root):
+ checkpoint('scan')
relative = Path(directory).relative_to(walk_root).parts
here: dict[str, tuple[int, str]] = {}
videos = video_stems(entry.name for entry in entries)
@@ -409,6 +412,7 @@ def scan_location(
artwork_dir = (os.path.basename(directory).casefold() in ARTWORK_DIRS
and os.path.dirname(directory) in video_dirs)
for entry in entries:
+ checkpoint('scan')
name = entry.name
if is_sidecar(name, videos, artwork_dir=artwork_dir):
skipped += 1
diff --git a/src/peach/scraping_access.py b/src/peach/scraping_access.py
index 85fc7babd..d0d04aba1 100644
--- a/src/peach/scraping_access.py
+++ b/src/peach/scraping_access.py
@@ -404,11 +404,14 @@ class SourceTransport:
self.root = secrets_root
self.transports: dict[str | None, HttpxTransport] = {}
self.max_requests, self.max_bytes = max_requests, max_bytes
- self.deadline = time.monotonic() + max_seconds if max_seconds else 0
+ from .processing_control import deadline_after
+ self.deadline = deadline_after(max_seconds) if max_seconds else 0
self.requests = self.bytes = 0
def __call__(self, request: HttpRequest, timeout: float, max_bytes: int):
for _ in range(6):
+ from .processing_control import checkpoint
+ checkpoint('network')
response = self._request(request, timeout, max_bytes)
if response.status not in {301, 302, 303, 307, 308} or not response.headers.get("location"):
return response
diff --git a/src/peach/task_runs.py b/src/peach/task_runs.py
index 34f0d8c04..5b7461dbe 100644
--- a/src/peach/task_runs.py
+++ b/src/peach/task_runs.py
@@ -276,14 +276,14 @@ class TaskRunStore:
return self._start(task_key, trigger=trigger, mutex_key=mutex_key,
conflict=conflict, total=total, label=label, pid=pid)
- def suspend(self, run_id: int) -> None:
+ def suspend(self, run_id: int, *, label: str = '等待安装完成后继续') -> None:
if not self.enabled:
return
with self.database.write_transaction(notify=False) as connection:
connection.execute(
"UPDATE task_run SET status='pending',pid=0,progress_label=?,result_summary=? "
"WHERE id=? AND status='running'",
- ('等待安装完成后继续', json.dumps({'paused_for_update': True}), run_id))
+ (label, json.dumps({'paused_for_update': True}), run_id))
def _resume(self, run_id, task_key):
if run_id is None or not self.enabled:
diff --git a/src/peach/web_library_processing.py b/src/peach/web_library_processing.py
index 30ffebdb2..5732e8f23 100644
--- a/src/peach/web_library_processing.py
+++ b/src/peach/web_library_processing.py
@@ -5,20 +5,26 @@ from . import settings_file
from .library_processing import (ALL_STAGES, COLLECT_STAGE, SCAN_STAGE, STAGES,
METADATA_STAGE, PREVIEWS_STAGE, _asset_selection,
decorate, issues_path, process_library, snapshot)
+from .processing_control import control_for, initial_estimate, valid_mode
+
+ACTIVE = ('running', 'pausing', 'paused')
def _current(contract):
live = contract.library_processing_job.snapshot()
- return live if live and live.get('status') == 'running' else snapshot(settings_file.active())
+ return live if live and live.get('status') in ACTIVE else snapshot(settings_file.active())
def q_library_processing(contract, _args):
state = decorate(_current(contract))
+ control = control_for(settings_file.active())
+ control.refresh(force=True)
+ state['speed_mode'] = control.mode
from .media_processing_policy import metadata_only
state['metadata_only'] = metadata_only(settings_file.active().directory('state'))
from .job_recovery import read
recovery = read(contract.db_path, 'library-processing')
- if state.get('status') != 'running' and recovery.get('status') == 'blocked':
+ if state.get('status') not in ACTIVE and recovery.get('status') == 'blocked':
state.update(status='failed', error=f"任务未能自动恢复:{recovery.get('error') or '请重新发起任务'}")
# 上一趟任务的状态文件里没有这个字段,而它那份完整清单还在磁盘上:按 job_id
# 推出地址补回去,界面才有得可给。
@@ -29,6 +35,74 @@ def q_library_processing(contract, _args):
return state
+def _control_action(contract, config, body):
+ action = body['action']
+ if action not in ('pause', 'resume', 'set_speed'):
+ raise ValueError('处理动作无效')
+ previous = _current(contract)
+ job_id = body.get('job_id')
+ if action != 'set_speed' and not job_id:
+ raise ValueError('请选择正在处理的任务')
+ if job_id and str(job_id) != str(previous.get('job_id') or ''):
+ raise ValueError('任务已变化,请刷新页面后重试')
+ if action != 'set_speed' and previous.get('status') not in ACTIVE:
+ raise ValueError('当前没有可暂停或继续的任务')
+ control = control_for(config)
+ control.refresh(force=True)
+ if job_id:
+ control.select_job(str(job_id))
+ if action == 'set_speed':
+ mode = valid_mode(body.get('speed_mode'))
+ control.request(action, mode=mode)
+ if previous.get('status') in ACTIVE:
+ ratio = 1.8 if mode == 'low' else 1 / 1.8
+ fields = dict(speed_mode=mode, eta_initial=True)
+ if previous.get('speed_mode') != mode:
+ for key in ('eta_seconds_min', 'eta_seconds_max'):
+ if isinstance(previous.get(key), (int, float)):
+ fields[key] = max(0, int(previous[key] * ratio))
+ contract.library_processing_job.update(previous['job_id'], **fields)
+ return q_library_processing(contract, {})
+ status = ('paused' if previous.get('status') == 'paused' else 'pausing') if action == 'pause' else 'running'
+ _announce_control(contract.library_processing_job, str(job_id), status)
+ control.request(action, str(job_id))
+ live = contract.library_processing_job.snapshot()
+ thread = getattr(contract.library_processing_job, 'thread', None)
+ if action == 'resume' and not (live and thread is not None and thread.is_alive()) and not _writer_active(config):
+ from .job_recovery import read, current_job_id, current_run
+ recovery = read(contract.db_path, 'library-processing')
+ identity = current_job_id.set(str(job_id))
+ run = current_run.set(recovery.get('run_id'))
+ try:
+ return w_library_processing(contract, {
+ 'stage': previous.get('requested_stage') or ALL_STAGES,
+ **({'asset_ids': previous['asset_ids']} if previous.get('asset_ids') else {}),
+ 'speed_mode': control.mode}, resume=True)
+ finally:
+ current_job_id.reset(identity)
+ current_run.reset(run)
+ result = q_library_processing(contract, {})
+ if not live and result.get('status') in ACTIVE:
+ result['status'] = status
+ return result
+
+
+def _announce_control(job, job_id, status):
+ with job.editing(job_id) as state:
+ if state is not None and state.get('status') in ACTIVE:
+ state['status'] = status
+
+
+def _writer_active(config):
+ from filelock import FileLock, Timeout
+ from .library_processing import state_path
+ try:
+ with FileLock(str(state_path(config)) + '.lock', timeout=0):
+ return False
+ except Timeout:
+ return True
+
+
def q_library_processing_issues(contract, args):
state = _current(contract)
job_id = str(args.get('job_id') or '')
@@ -73,15 +147,12 @@ def _retry_ids(previous, body):
return chosen
-def w_library_processing(contract, body, *, resume=False):
- config = settings_file.active()
+def _processing_request(contract, config, previous, body):
+ """校验馆藏、阶段与互斥的处理范围。"""
if not config.configured or not config.locations:
raise ValueError('请先添加媒体文件夹')
if contract.db_path.resolve() != (config.directory('database') / 'ledger.db').resolve():
raise ValueError('当前馆藏与媒体配置不一致')
- previous = snapshot(config)
- if previous.get('status') == 'running':
- return previous
retry_ids = _retry_ids(previous, body)
if 'asset_ids' in (body or {}) and body['asset_ids'] is None:
raise ValueError('请选择有效的视频项目')
@@ -95,8 +166,22 @@ def w_library_processing(contract, body, *, resume=False):
from .media_processing_policy import metadata_only
if stage == PREVIEWS_STAGE and metadata_only(config.directory('state')):
raise ValueError('视频预览生成已关闭,请先在媒体库设置中开启')
+ return retry_ids, asset_ids, stage
+
+
+def w_library_processing(contract, body, *, resume=False):
+ config = settings_file.active()
+ if (body or {}).get('action'):
+ return _control_action(contract, config, body)
+ previous = _current(contract)
+ if previous.get('status') in ACTIVE and not resume:
+ return previous
+ retry_ids, asset_ids, stage = _processing_request(contract, config, previous, body)
if retry_ids is not None and not retry_ids:
return previous
+ mode = valid_mode(control_for(config).mode if resume else
+ (body or {}).get('speed_mode') or control_for(config).mode)
+ control_for(config).request('set_speed', mode=mode)
def work(job_id):
try:
result = process_library(config, contract.db_path, contract.candidate_root, contract.cover_root,
@@ -105,6 +190,7 @@ def w_library_processing(contract, body, *, resume=False):
# 另建一个就是在同一个进程里开第二条写入路径。
database=contract.database,
resume_state=previous if resume else None,
+ speed_mode=control_for(config).mode,
active=lambda: (contract.library_processing_job.snapshot() or {}).get('job_id') == job_id,
report=lambda state: contract.library_processing_job.update(job_id, **{
key: value for key, value in state.items() if key != 'job_id'}))
@@ -116,6 +202,7 @@ def w_library_processing(contract, body, *, resume=False):
labels = {ALL_STAGES: '准备处理', SCAN_STAGE: '准备扫描文件', COLLECT_STAGE: '准备采集资料',
METADATA_STAGE: '准备采集资料', PREVIEWS_STAGE: '准备生成预览图'}
return contract.library_processing_job.start(
- work, restart=True, initial={'stage': labels[stage], 'requested_stage': stage},
- recovery={'stage': stage, **({'asset_ids': asset_ids or retry_ids}
+ work, restart=True, initial={'stage': labels[stage], 'requested_stage': stage,
+ **initial_estimate(config, contract.db_path, stage, asset_ids or retry_ids, mode)},
+ recovery={'stage': stage, 'speed_mode': mode, **({'asset_ids': asset_ids or retry_ids}
if asset_ids is not None or retry_ids is not None else {})})
diff --git a/tests/test_processing_control.py b/tests/test_processing_control.py
new file mode 100644
index 000000000..b30067500
--- /dev/null
+++ b/tests/test_processing_control.py
@@ -0,0 +1,282 @@
+"""馆藏处理控制与启动粗估的隔离行为回归。"""
+import json
+import tempfile
+import threading
+import unittest
+from contextlib import closing
+from pathlib import Path
+from types import SimpleNamespace
+from unittest.mock import Mock, patch
+
+from peach import job_recovery
+from peach.processing_control import (CURRENT, ProcessingControl, ProcessingEstimate,
+ checkpoint, control_for, initial_estimate, user_paused, deadline_after)
+from peach.settings_file import PeachConfig
+from peach.web_library_processing import w_library_processing
+from support.ledger import fresh_ledger
+
+
+class ProcessingControlTests(unittest.TestCase):
+ def setUp(self):
+ temporary = tempfile.TemporaryDirectory()
+ self.addCleanup(temporary.cleanup)
+ self.root = Path(temporary.name).resolve()
+ self.config = PeachConfig(self.root, self.root / 'config.toml', present=True,
+ locations={'local': ('R:\\Media',)})
+ self.db = fresh_ledger(self.root)
+
+ def seed(self, count=10):
+ import sqlite3
+ with closing(sqlite3.connect(self.db)) as connection, connection:
+ for index in range(count):
+ connection.execute('INSERT INTO asset(location,path,name,medium,size) VALUES(?,?,?,?,?)',
+ ('local', f'R:\\Media\\{index}.mp4', f'{index}.mp4', 'video', 1024**3))
+
+ def test_start_estimate_uses_known_count_size_and_scoped_selection(self):
+ self.seed()
+ estimate = initial_estimate(self.config, self.db, 'all')
+ chosen = initial_estimate(self.config, self.db, 'all', [1, 2])
+ low = initial_estimate(self.config, self.db, 'all', mode='low')
+ self.assertEqual(estimate['estimate_file_count'], 10)
+ self.assertEqual(estimate['estimate_bytes'], 10 * 1024**3)
+ self.assertGreater(estimate['eta_seconds_max'], estimate['eta_seconds_min'])
+ self.assertLess(chosen['eta_seconds_max'], estimate['eta_seconds_max'])
+ self.assertGreater(low['eta_seconds_max'], estimate['eta_seconds_max'])
+ self.assertTrue(estimate['eta_initial'])
+ self.assertIn('未含未知新增', estimate['eta_scope'])
+
+ def test_new_directory_reports_counting_without_invented_time(self):
+ estimate = initial_estimate(self.config, self.db, 'scan')
+ self.assertIsNone(estimate['eta_seconds_min'])
+ self.assertIsNone(estimate['eta_seconds_max'])
+ self.assertIn('统计', estimate['eta_basis'])
+
+ def test_progress_calibrates_range_and_paused_time_does_not_count(self):
+ self.seed()
+ state = dict(initial_estimate(self.config, self.db, 'metadata'),
+ requested_stage='metadata', status='running', stage='读取本地资料', checked=0, total=10)
+ with patch('peach.processing_control.time.monotonic', return_value=100):
+ estimate = ProcessingEstimate(self.config, state, None)
+ estimate.update(state)
+ state['checked'] = 2
+ with patch('peach.processing_control.time.monotonic', return_value=110):
+ estimate.update(state)
+ self.assertEqual(state['eta_kind'], 'measured')
+ self.assertFalse(state['eta_initial'])
+ self.assertLessEqual(state['eta_seconds_min'], 40)
+ self.assertGreaterEqual(state['eta_seconds_max'], 40)
+ frozen = (state['eta_seconds_min'], state['eta_seconds_max'])
+ state['status'] = 'paused'
+ with patch('peach.processing_control.time.monotonic', return_value=9999):
+ estimate.update(state)
+ self.assertEqual((state['eta_seconds_min'], state['eta_seconds_max']), frozen)
+
+ def test_pause_waits_at_checkpoint_and_resume_releases_same_job(self):
+ control = control_for(self.config)
+ entered, paused, finished = threading.Event(), threading.Event(), threading.Event()
+ errors = []
+ def publish(**values):
+ if values['status'] == 'paused':
+ paused.set()
+ def worker():
+ try:
+ with control.bind('one', publish=publish, active=lambda: True, db_path=self.db):
+ entered.set()
+ control.request('pause', 'one')
+ checkpoint('probe')
+ finished.set()
+ except BaseException as exc:
+ errors.append(exc)
+ thread = threading.Thread(target=worker)
+ thread.start()
+ self.addCleanup(lambda: control.request('resume', 'one'))
+ self.addCleanup(lambda: thread.join(2))
+ self.assertTrue(entered.wait(2))
+ self.assertTrue(paused.wait(2))
+ self.assertFalse(finished.is_set())
+ self.assertTrue(user_paused(self.db, 'one'))
+ control.request('resume', 'one')
+ thread.join(2)
+ self.assertFalse(thread.is_alive())
+ self.assertTrue(finished.is_set())
+ self.assertEqual(errors, [])
+ self.assertFalse(user_paused(self.db, 'one'))
+
+ def test_saved_pause_intent_prevents_automatic_restart(self):
+ control = control_for(self.config)
+ with control.bind('one', publish=lambda **_: None, active=lambda: True, db_path=self.db):
+ control.request('pause', 'one')
+ restored = ProcessingControl(control.path)
+ self.assertTrue(restored.paused)
+ job_recovery.save(self.db, 'library-processing', 'one', {'stage': 'scan'}, waiting=True)
+ handler = Mock()
+ self.assertEqual(job_recovery.resume(self.db, {'library-processing': handler}), [])
+ handler.assert_not_called()
+
+ def test_low_mode_leaves_io_gaps_and_preserves_unrelated_context(self):
+ control = control_for(self.config)
+ with control.bind('one', publish=lambda **_: None, active=lambda: True, db_path=self.db, mode='low'), \
+ patch('peach.processing_control.time.monotonic', return_value=10), \
+ patch.object(control.event, 'wait', side_effect=lambda _: control.request('set_speed', mode='standard')) as wait:
+ checkpoint('network')
+ checkpoint('network')
+ wait.assert_called_once()
+ self.assertIsNone(CURRENT.get())
+ checkpoint('network')
+ restored = ProcessingControl(control.path)
+ self.assertEqual(restored.mode, 'standard')
+
+ def test_stale_job_control_is_rejected(self):
+ contract = SimpleNamespace(library_processing_job=Mock())
+ contract.library_processing_job.snapshot.return_value = {'job_id': 'current', 'status': 'running'}
+ with patch('peach.web_library_processing.settings_file.active', return_value=self.config):
+ with self.assertRaisesRegex(ValueError, '任务已变化'):
+ w_library_processing(contract, {'action': 'pause', 'job_id': 'stale'})
+ with self.assertRaisesRegex(ValueError, '任务已变化'):
+ w_library_processing(contract, {'action': 'set_speed', 'job_id': 'stale', 'speed_mode': 'low'})
+
+ def test_completed_timing_is_available_for_next_initial_estimate(self):
+ self.seed()
+ state = dict(initial_estimate(self.config, self.db, 'metadata'),
+ requested_stage='metadata', status='running')
+ with patch('peach.processing_control.time.monotonic', return_value=100):
+ estimate = ProcessingEstimate(self.config, state, None)
+ state['status'] = 'complete'
+ with patch('peach.processing_control.time.monotonic', return_value=200):
+ estimate.update(state)
+ next_run = initial_estimate(self.config, self.db, 'metadata')
+ self.assertEqual(next_run['eta_kind'], 'historical')
+ self.assertTrue(next_run['eta_initial'])
+ self.assertEqual(json.loads((self.config.directory('state') / 'library-processing-timing.json').read_text()),
+ {'metadata': 10.0})
+
+ def test_action_deadline_keeps_the_same_budget_after_user_pause(self):
+ control = control_for(self.config)
+ with control.bind('one', publish=lambda **_: None, active=lambda: True, db_path=self.db), \
+ patch('peach.processing_control.time.monotonic', return_value=100):
+ deadline = deadline_after(90)
+ self.assertEqual(deadline - 110, 80)
+ control.paused_seconds += 3600
+ self.assertEqual(deadline - 3710, 80)
+ self.assertEqual((deadline + 5) - 3710, 85)
+ self.assertTrue(3710 < deadline)
+ self.assertTrue(3800 > deadline)
+
+ def test_web_pause_and_resume_preserve_running_job_and_initial_estimate(self):
+ from peach.jobs import BackgroundJob
+ from peach.library_processing import _save, state_path
+ from peach.web_library_processing import q_library_processing
+ from peach.repository import LedgerDatabase
+ from peach.task_runs import TaskRunStore
+ database_root = self.config.directory('database')
+ database_root.mkdir(parents=True)
+ db = fresh_ledger(database_root)
+ job = BackgroundJob('library-processing', runs=TaskRunStore(LedgerDatabase(db)))
+ contract = SimpleNamespace(db_path=db, library_processing_job=job,
+ candidate_root=self.root / 'candidates', cover_root=self.root / 'covers',
+ database=job.runs.database, cache_bust=lambda: None)
+ entered, advance, paused, done = (threading.Event() for _ in range(4))
+ calls, errors = [], []
+ def process(*_args, **kwargs):
+ identity = kwargs['job_id']
+ calls.append(identity)
+ def report(**values):
+ kwargs['report'](values)
+ if values.get('status') == 'paused':
+ paused.set()
+ try:
+ with control_for(self.config).bind(identity, publish=report, active=lambda: True, db_path=db):
+ _save(state_path(self.config), dict(job_id=identity, status='running', requested_stage='metadata'))
+ entered.set()
+ advance.wait(2)
+ checkpoint()
+ done.set()
+ return {'status': 'complete'}
+ except BaseException as exc:
+ errors.append(exc)
+ raise
+ with patch('peach.web_library_processing.settings_file.active', return_value=self.config), \
+ patch('peach.web_library_processing.process_library', side_effect=process):
+ initial = w_library_processing(contract, {'stage': 'metadata'})
+ self.assertIn('eta_seconds_max', initial)
+ self.assertTrue(entered.wait(2))
+ run_id = initial['run_id']
+ identity = initial['job_id']
+ reply = w_library_processing(contract, {'action': 'pause', 'job_id': identity})
+ self.assertEqual(reply['status'], 'pausing')
+ advance.set()
+ self.assertTrue(paused.wait(2))
+ self.assertEqual(q_library_processing(contract, {})['status'], 'paused')
+ self.assertFalse(done.is_set())
+ w_library_processing(contract, {'action': 'resume', 'job_id': identity})
+ job.thread.join(2)
+ self.assertTrue(done.is_set())
+ self.assertEqual(job.snapshot()['status'], 'complete')
+ self.assertEqual(job.snapshot()['run_id'], run_id)
+ self.assertEqual(calls, [identity])
+ self.assertEqual(errors, [])
+ self.assertEqual(job.runs.get(run_id).status, 'succeeded')
+
+ def test_low_mode_ffmpeg_uses_single_decode_filter_and_encode_threads(self):
+ from peach.frame_capture import capture_frame
+ control = control_for(self.config)
+ with control.bind('one', publish=lambda **_: None, active=lambda: True, db_path=self.db, mode='low'), \
+ patch('peach.frame_capture.subprocess.run', return_value=SimpleNamespace(stderr=b'')) as run:
+ capture_frame('ffmpeg', 'video.mp4', 1, self.root / 'frame.jpg')
+ command = run.call_args.args[0]
+ self.assertEqual(command.count('-threads'), 2)
+ self.assertEqual(command[command.index('-filter_threads') + 1], '1')
+
+ def test_user_paused_job_keeps_identity_and_progress_through_service_restart(self):
+ from peach.jobs import BackgroundJob
+ from peach.library_processing import _save, state_path
+ from peach.repository import LedgerDatabase
+ from peach.task_runs import TaskRunStore
+ database_root = self.config.directory('database')
+ database_root.mkdir(parents=True)
+ db = fresh_ledger(database_root)
+ runs = TaskRunStore(LedgerDatabase(db))
+ first = BackgroundJob('library-processing', runs=runs)
+ contract = SimpleNamespace(db_path=db, library_processing_job=first,
+ candidate_root=self.root / 'candidates', cover_root=self.root / 'covers',
+ database=runs.database, cache_bust=lambda: None)
+ entered, advance, paused = (threading.Event() for _ in range(3))
+ identities, resumes = [], []
+ def process(*_args, **kwargs):
+ identity = kwargs['job_id']
+ identities.append(identity)
+ resumes.append(kwargs['resume_state'])
+ state = dict(job_id=identity, requested_stage='metadata', status='running', completed_metadata=['1:R:\\a.mp4'])
+ def report(**values):
+ values.pop('resumed_after', None)
+ state.update(values)
+ _save(state_path(self.config), state)
+ kwargs['report'](dict(state))
+ if state['status'] == 'paused':
+ paused.set()
+ with control_for(self.config).bind(identity, publish=report, active=kwargs['active'], db_path=db):
+ _save(state_path(self.config), state)
+ if len(identities) == 1:
+ entered.set()
+ advance.wait(2)
+ checkpoint()
+ return {'status': 'complete'}
+ with patch('peach.web_library_processing.settings_file.active', return_value=self.config), \
+ patch('peach.web_library_processing.process_library', side_effect=process):
+ initial = w_library_processing(contract, {'stage': 'metadata'})
+ self.assertTrue(entered.wait(2))
+ w_library_processing(contract, {'action': 'pause', 'job_id': initial['job_id']})
+ advance.set()
+ self.assertTrue(paused.wait(2))
+ first.stop()
+ self.assertEqual(runs.get(initial['run_id']).status, 'pending')
+ self.assertEqual(runs.get(initial['run_id']).progress_label, '已暂停,等待继续')
+ second = BackgroundJob('library-processing', runs=runs)
+ contract.library_processing_job = second
+ w_library_processing(contract, {'action': 'resume', 'job_id': initial['job_id']})
+ second.thread.join(2)
+ self.assertEqual(identities, [initial['job_id'], initial['job_id']])
+ self.assertEqual(resumes[1]['completed_metadata'], ['1:R:\\a.mp4'])
+ self.assertEqual(second.snapshot()['status'], 'complete')
+ self.assertEqual(second.snapshot()['run_id'], initial['run_id'])
+ self.assertEqual(runs.get(initial['run_id']).status, 'succeeded')