Peach: Local-first video metadata scraper and media manager.
feat(library): 提供三档处理强度与对应耗时估算
longmeidaodf3f180ce622
diff --git a/docs/REUSE.md b/docs/REUSE.mdindex 8083f7a13..79ca8ed2e 100644--- a/docs/REUSE.md+++ b/docs/REUSE.md@@ -614,4 +614,4 @@ Python 复用 MIT 许可的 [Ruff](https://docs.astral.sh/ruff/) 0.17.0,版本扫描、探测、资料请求和九宫格复用 `process_library`、`scan_location`、`SourceTransport`、`frame_capture` 与已有影片检查点;任务恢复沿用 `BackgroundJob`、`job_recovery` 和 `task_run`,用户暂停意图单独持久保存。当前树及 `library_processing.py` 的 Git 历史已核对,安装暂停只用于安装事务,不能直接代表用户暂停。Python 标准库 `threading.Event` 与 `contextvars.ContextVar` 提供等待唤醒和当前任务隔离,沿用项目 Python 版本,跨 Windows 与 macOS,无额外依赖体积;许可证为 PSF,官方文档与实现见 [threading](https://docs.python.org/3/library/threading.html) 和 [contextvars](https://docs.python.org/3/library/contextvars.html)。现有 APScheduler 3.11.3 用于定时派发,不能暂停已进入媒体读取的函数,因此不承担安全停点。隔离测试使用临时账本、暂停线程与模拟时钟验证安全停点、跨重启意图、低占用间隔和暂停预算补偿,不读取真实媒体、不发外部请求。-`processing_control` 保留 Peach 的阶段、模式与估时策略:启动仅汇总账本文件数和大小,优先使用本机同类完成耗时;未知新增明确不在初始范围内。阶段样本形成后按实际耗时校准,并返回估时范围、依据和范围所指阶段。低占用在单项读盘、探测、下载和抽帧之间留出间隔,FFmpeg 使用单线程解码与滤镜;只作用于当前任务。持久状态不修改 ledger schema,不改变生产入口。+`processing_control` 保留 Peach 的阶段、模式与估时策略:启动仅汇总账本文件数和大小,优先使用本机同类完成耗时;未知新增明确不在初始范围内。阶段样本形成后按实际耗时校准,并返回估时范围、依据和范围所指阶段。处理强度为低占用、标准和高强度:低占用采用单线程 FFmpeg 与较大单项间隔,标准采用两个解码、滤镜和编码线程与短间隔,高强度采用工具自动线程且没有额外间隔。amane 低占用逐站查询,其余沿用现有调度;各档均遵守来源限流,单资产流水线与暂停检查点共用。强度只作用于当前任务,历史耗时按档位系数归一,初始与切换时的范围属于粗估,实际样本继续校准。持久状态不修改 ledger schema,不改变生产入口。diff --git a/src/peach/frame_capture.py b/src/peach/frame_capture.pyindex 99ca4c615..ddd4d6be7 100644--- a/src/peach/frame_capture.py+++ b/src/peach/frame_capture.py@@ -11,7 +11,7 @@ import osimport reimport subprocessfrom pathlib import Path-from .processing_control import checkpoint, low_mode+from .processing_control import checkpoint, ffmpeg_options#: 部分网盘视频把色彩原色写成 reserved(非法值),swscale 会拒绝缩放;声明为 bt709#: 只是覆盖坏的元数据,不改动像素。@@ -36,14 +36,13 @@ def capture_frame(ffmpeg: str, path: str, timestamp: float, destination: Path, *"""command = [ffmpeg, "-y", "-v", "error", "-rw_timeout", "8000000"]checkpoint('frame')- if low_mode():- command += ['-threads', '1', '-filter_threads', '1']+ command += ffmpeg_options(filter_option='-filter_threads')if color_override:command += COLOR_OVERRIDEcommand += ["-ss", f"{timestamp:.2f}", "-i", path, "-frames:v", "1","-vf", f"scale={int(width)}:-1", "-q:v", "4",- *(['-threads', '1'] if low_mode() else []), str(destination),+ *ffmpeg_options(), str(destination),]try:completed = subprocess.run(command, capture_output=True, timeout=timeout,@@ -74,11 +73,11 @@ def tile_frames(ffmpeg: str, pattern: str, columns: int, rows: int, destination:"""destination.parent.mkdir(parents=True, exist_ok=True)checkpoint('frame')- options = ['-threads', '1', '-filter_complex_threads', '1'] if low_mode() else []+ options = ffmpeg_options(filter_option='-filter_complex_threads')subprocess.run([ffmpeg, "-y", "-v", "error", *options, "-i", pattern,"-filter_complex", f"tile={int(columns)}x{int(rows)}", "-q:v", "4",- *(['-threads', '1'] if low_mode() else []), str(destination)],+ *ffmpeg_options(), str(destination)],capture_output=True, timeout=timeout,creationflags=subprocess.CREATE_NO_WINDOW if os.name == "nt" else 0,)diff --git a/src/peach/processing_control.py b/src/peach/processing_control.pyindex ef42a199d..ef8000f40 100644--- a/src/peach/processing_control.py+++ b/src/peach/processing_control.py@@ -14,12 +14,18 @@ from .fsutil import atomic_write_textCURRENT = ContextVar('peach_library_processing_control', default=None)_CONTROLS = {}_LOCK = threading.Lock()-MODES = ('standard', 'low')+MODES = ('low', 'standard', 'high')+MODE_FACTORS = {'low': 1.8, 'standard': 1.0, 'high': 0.75}+IO_INTERVALS = {+ 'low': {'scan': 0.008, 'probe': 0.75, 'network': 1.5, 'frame': 0.75, 'work': 0.15},+ 'standard': {'scan': 0.002, 'probe': 0.2, 'network': 0.25, 'frame': 0.2, 'work': 0.03},+ 'high': {},+}def valid_mode(value):if value not in MODES:- raise ValueError('请选择标准或低占用模式')+ raise ValueError('请选择低占用、标准或高强度模式')return value@@ -78,6 +84,20 @@ def low_mode():return control is not None and control.mode == 'low'+def mode_ratio(target, previous='standard'):+ return MODE_FACTORS[valid_mode(target)] / MODE_FACTORS[valid_mode(previous)]+++def ffmpeg_options(*, filter_option=None):+ """当前馆藏任务的 FFmpeg 线程选项;高强度与独立调用使用工具默认值。"""+ control = CURRENT.get()+ threads = {'low': 1, 'standard': 2}.get(control.mode) if control is not None else None+ if threads is None:+ return []+ value = str(threads)+ return ['-threads', value, *([filter_option, value] if filter_option else [])]++class ActiveDeadline(float):"""单调时钟截止点随本任务的用户暂停时长平移。"""def __new__(cls, value, control, paused=None):@@ -211,8 +231,7 @@ class ProcessingControl:self.publish(status='running', speed_mode=mode, resumed_after=spent)started = None# 控制只作用于当前馆藏任务;在单项 I/O 之间留出空档。- delay = {'scan': 0.008, 'probe': 0.75, 'network': 1.5, 'frame': 0.75,- 'work': 0.15}.get(kind, 0.15) if mode == 'low' else 0+ delay = IO_INTERVALS[mode].get(kind, 0)remaining = self.next_work.get(kind, 0) - time.monotonic()if remaining > 0:self.event.wait(min(0.25, remaining))@@ -251,7 +270,7 @@ def initial_estimate(config, db_path, stage, asset_ids=None, mode=None):historical = isinstance(sample, (int, float)) and math.isfinite(sample) and sample > 0if historical:seconds = sample- factor = 1.8 if mode == 'low' else 1.0+ factor = MODE_FACTORS[valid_mode(mode)]known = bool(count) or asset_ids is not None or stage not in ('scan', 'all')estimate = count * seconds * factorreturn dict(speed_mode=mode,@@ -281,7 +300,7 @@ class ProcessingEstimate:def update(self, state):mode = self.control.mode if self.control is not None else self.modeif mode != self.mode:- ratio = 1.8 if mode == 'low' else 1 / 1.8+ ratio = mode_ratio(mode, self.mode)for key in ('eta_seconds_min', 'eta_seconds_max'):if isinstance(state.get(key), (int, float)):state[key] = math.ceil(state[key] * ratio)@@ -294,7 +313,7 @@ class ProcessingEstimate:elapsed = max(0.01, time.monotonic() - self.started - self.paused_elapsed())path = self.config.directory('state') / 'library-processing-timing.json'history = _read(path)- history[state['requested_stage']] = elapsed / state['estimate_file_count'] / (1.8 if mode == 'low' else 1)+ history[state['requested_stage']] = elapsed / state['estimate_file_count'] / MODE_FACTORS[mode]atomic_write_text(path, json.dumps(history) + '\n')returnif state.get('status') != 'running':diff --git a/src/peach/web_library_processing.py b/src/peach/web_library_processing.pyindex 5732e8f23..5361c0a4e 100644--- a/src/peach/web_library_processing.py+++ b/src/peach/web_library_processing.py@@ -5,7 +5,7 @@ from . import settings_filefrom .library_processing import (ALL_STAGES, COLLECT_STAGE, SCAN_STAGE, STAGES,METADATA_STAGE, PREVIEWS_STAGE, _asset_selection,decorate, issues_path, process_library, snapshot)-from .processing_control import control_for, initial_estimate, valid_mode+from .processing_control import control_for, initial_estimate, valid_mode, mode_ratioACTIVE = ('running', 'pausing', 'paused')@@ -55,7 +55,7 @@ def _control_action(contract, config, body):mode = valid_mode(body.get('speed_mode'))control.request(action, mode=mode)if previous.get('status') in ACTIVE:- ratio = 1.8 if mode == 'low' else 1 / 1.8+ ratio = mode_ratio(mode, previous.get('speed_mode') or 'standard')fields = dict(speed_mode=mode, eta_initial=True)if previous.get('speed_mode') != mode:for key in ('eta_seconds_min', 'eta_seconds_max'):diff --git a/tests/test_processing_control.py b/tests/test_processing_control.pyindex 2004bd2cd..0d3063713 100644--- a/tests/test_processing_control.py+++ b/tests/test_processing_control.py@@ -37,11 +37,13 @@ class ProcessingControlTests(unittest.TestCase):estimate = initial_estimate(self.config, self.db, 'all')chosen = initial_estimate(self.config, self.db, 'all', [1, 2])low = initial_estimate(self.config, self.db, 'all', mode='low')+ high = initial_estimate(self.config, self.db, 'all', mode='high')self.assertEqual(estimate['estimate_file_count'], 10)self.assertEqual(estimate['estimate_bytes'], 10 * 1024**3)self.assertGreater(estimate['eta_seconds_max'], estimate['eta_seconds_min'])self.assertLess(chosen['eta_seconds_max'], estimate['eta_seconds_max'])self.assertGreater(low['eta_seconds_max'], estimate['eta_seconds_max'])+ self.assertLess(high['eta_seconds_max'], estimate['eta_seconds_max'])self.assertTrue(estimate['eta_initial'])self.assertIn('未含未知新增', estimate['eta_scope'])@@ -80,7 +82,7 @@ class ProcessingControlTests(unittest.TestCase):paused.set()def worker():try:- with control.bind('one', publish=publish, active=lambda: True, db_path=self.db):+ with control.bind('one', publish=publish, active=lambda: True, db_path=self.db, mode='high'):entered.set()control.request('pause', 'one')checkpoint('probe')@@ -104,10 +106,11 @@ class ProcessingControlTests(unittest.TestCase):def test_saved_pause_intent_prevents_automatic_restart(self):control = control_for(self.config)- with control.bind('one', publish=lambda **_: None, active=lambda: True, db_path=self.db):+ with control.bind('one', publish=lambda **_: None, active=lambda: True, db_path=self.db, mode='high'):control.request('pause', 'one')restored = ProcessingControl(control.path)self.assertTrue(restored.paused)+ self.assertEqual(restored.mode, 'high')job_recovery.save(self.db, 'library-processing', 'one', {'stage': 'scan'}, waiting=True)handler = Mock()self.assertEqual(job_recovery.resume(self.db, {'library-processing': handler}), [])@@ -197,8 +200,9 @@ class ProcessingControlTests(unittest.TestCase):raisewith patch('peach.web_library_processing.settings_file.active', return_value=self.config), \patch('peach.web_library_processing.process_library', side_effect=process):- initial = w_library_processing(contract, {'stage': 'metadata'})+ initial = w_library_processing(contract, {'stage': 'metadata', 'speed_mode': 'high'})self.assertIn('eta_seconds_max', initial)+ self.assertEqual(initial['speed_mode'], 'high')self.assertTrue(entered.wait(2))run_id = initial['run_id']identity = initial['job_id']@@ -263,7 +267,7 @@ class ProcessingControlTests(unittest.TestCase):return {'status': 'complete'}with patch('peach.web_library_processing.settings_file.active', return_value=self.config), \patch('peach.web_library_processing.process_library', side_effect=process):- initial = w_library_processing(contract, {'stage': 'metadata'})+ initial = w_library_processing(contract, {'stage': 'metadata', 'speed_mode': 'high'})self.assertTrue(entered.wait(2))w_library_processing(contract, {'action': 'pause', 'job_id': initial['job_id']})advance.set()@@ -279,8 +283,127 @@ class ProcessingControlTests(unittest.TestCase):self.assertEqual(resumes[1]['completed_metadata'], ['1:R:\\a.mp4'])self.assertEqual(second.snapshot()['status'], 'complete')self.assertEqual(second.snapshot()['run_id'], initial['run_id'])+ self.assertEqual(control_for(self.config).mode, 'high')self.assertEqual(runs.get(initial['run_id']).status, 'succeeded')+ def test_three_intensities_set_distinct_io_start_intervals(self):+ control = control_for(self.config)+ intervals = {}+ for mode in ('low', 'standard', 'high'):+ with self.subTest(mode=mode), \+ control.bind(mode, publish=lambda **_: None, active=lambda: True, db_path=self.db, mode=mode), \+ patch('peach.processing_control.time.monotonic', return_value=10):+ checkpoint('network')+ intervals[mode] = control.next_work['network'] - 10+ self.assertGreater(intervals['low'], intervals['standard'])+ self.assertGreater(intervals['standard'], intervals['high'])+ self.assertEqual(intervals['high'], 0)++ def test_three_intensities_set_decode_filter_and_encode_threads(self):+ from peach.frame_capture import capture_frame, tile_frames+ control = control_for(self.config)+ for mode, threads in (('low', '1'), ('standard', '2'), ('high', None)):+ with self.subTest(mode=mode), \+ control.bind(mode, publish=lambda **_: None, active=lambda: True, db_path=self.db, mode=mode), \+ patch('peach.frame_capture.checkpoint'), \+ patch('peach.frame_capture.subprocess.run', return_value=SimpleNamespace(stderr=b'')) as run:+ capture_frame('ffmpeg', 'video.mp4', 1, self.root / 'frame.jpg')+ tile_frames('ffmpeg', 'frame%02d.jpg', 3, 3, self.root / 'tile.jpg')+ for call, filter_option in zip(run.call_args_list, ('-filter_threads', '-filter_complex_threads')):+ command = call.args[0]+ if threads is None:+ self.assertNotIn('-threads', command)+ self.assertNotIn(filter_option, command)+ else:+ self.assertEqual(command.count('-threads'), 2)+ self.assertEqual(command[command.index(filter_option) + 1], threads)+ self.assertEqual([command[index + 1] for index, value in enumerate(command)+ if value == '-threads'], [threads, threads])++ def test_intensity_switch_recalibrates_estimate_in_every_direction(self):+ self.seed()+ control = control_for(self.config)+ for initial, target in (('low', 'standard'), ('standard', 'high'), ('high', 'low')):+ with self.subTest(initial=initial, target=target), \+ control.bind(initial, publish=lambda **_: None, active=lambda: True,+ db_path=self.db, mode=initial):+ state = dict(initial_estimate(self.config, self.db, 'metadata', mode=initial),+ status='paused', requested_stage='metadata')+ estimate = ProcessingEstimate(self.config, state, control)+ old = state['eta_seconds_max']+ control.request('set_speed', mode=target)+ estimate.update(state)+ self.assertEqual(state['speed_mode'], target)+ if target == 'low':+ self.assertGreater(state['eta_seconds_max'], old)+ else:+ self.assertLess(state['eta_seconds_max'], old)++ def test_high_intensity_honors_installation_pause(self):+ from peach.jobs import JobPaused+ from peach import update_tasks+ control = control_for(self.config)+ with control.bind('one', publish=lambda **_: None, active=lambda: True, db_path=self.db, mode='high'):+ token = update_tasks.reserve(self.db)+ try:+ with self.assertRaises(JobPaused):+ checkpoint('frame')+ finally:+ update_tasks.release(self.db, token)+ checkpoint('frame')+ self.assertEqual(control.mode, 'high')++ def test_high_history_is_normalized_for_every_initial_intensity(self):+ self.seed()+ control = control_for(self.config)+ with control.bind('one', publish=lambda **_: None, active=lambda: True, db_path=self.db, mode='high'):+ state = dict(initial_estimate(self.config, self.db, 'metadata', mode='high'),+ requested_stage='metadata', status='running')+ with patch('peach.processing_control.time.monotonic', return_value=100):+ estimate = ProcessingEstimate(self.config, state, control)+ state['status'] = 'complete'+ with patch('peach.processing_control.time.monotonic', return_value=175):+ estimate.update(state)+ history = json.loads((self.config.directory('state') / 'library-processing-timing.json').read_text())+ self.assertEqual(history['metadata'], 10.0)+ low, standard, high = [initial_estimate(self.config, self.db, 'metadata', mode=mode)+ for mode in ('low', 'standard', 'high')]+ self.assertEqual(high['eta_kind'], 'historical')+ self.assertGreater(low['eta_seconds_max'], standard['eta_seconds_max'])+ self.assertGreater(standard['eta_seconds_max'], high['eta_seconds_max'])++ def test_high_intensity_and_pause_intent_are_shared_between_control_instances(self):+ control = control_for(self.config)+ with control.bind('one', publish=lambda **_: None, active=lambda: True, db_path=self.db, mode='standard'):+ external = ProcessingControl(control.path)+ external.request('set_speed', 'one', mode='high')+ external.request('pause', 'one')+ control.refresh(force=True)+ self.assertEqual(control.mode, 'high')+ self.assertTrue(control.paused)+ external.request('resume', 'one')+ control.refresh(force=True)+ self.assertFalse(control.paused)+ self.assertEqual(control.mode, 'high')++ def test_api_switches_between_all_intensities_with_proportional_paused_estimates(self):+ from peach.jobs import BackgroundJob+ control = control_for(self.config)+ job = BackgroundJob('library-processing')+ job.state = dict(job_id='one', status='paused', speed_mode='low', eta_seconds_min=180, eta_seconds_max=360)+ contract = SimpleNamespace(db_path=self.db, library_processing_job=job)+ with control.bind('one', publish=lambda **_: None, active=lambda: True, db_path=self.db, mode='low'), \+ patch('peach.web_library_processing.settings_file.active', return_value=self.config):+ control.request('pause', 'one')+ high = w_library_processing(contract, {'action': 'set_speed', 'job_id': 'one', 'speed_mode': 'high'})+ self.assertEqual((high['speed_mode'], high['status']), ('high', 'paused'))+ self.assertEqual((high['eta_seconds_min'], high['eta_seconds_max']), (75, 150))+ standard = w_library_processing(contract, {'action': 'set_speed', 'job_id': 'one', 'speed_mode': 'standard'})+ self.assertEqual((standard['eta_seconds_min'], standard['eta_seconds_max']), (100, 200))+ low = w_library_processing(contract, {'action': 'set_speed', 'job_id': 'one', 'speed_mode': 'low'})+ self.assertEqual((low['eta_seconds_min'], low['eta_seconds_max']), (180, 360))+ self.assertTrue(control.paused)+def test_scan_progress_accumulates_discovered_counts_across_roots(self):from peach.library_processing import _scan_sourcesreports = []