Skip to content

armgit / peach

Public Imported from GitHub
0

Peach: Local-first video metadata scraper and media manager.

feat(library): 提供三档处理强度与对应耗时估算

longmeidaodf3f180ce622
diff --git a/docs/REUSE.md b/docs/REUSE.md
index 8083f7a13..79ca8ed2e 100644
--- a/docs/REUSE.md
+++ b/docs/REUSE.md
@@ -614,4 +614,4 @@ Python 复用 MIT 许可的 [Ruff](https://docs.astral.sh/ruff/) 0.17.0,版本
扫描、探测、资料请求和九宫格复用 `process_library`、`scan_location`、`SourceTransport`、`frame_capture` 与已有影片检查点;任务恢复沿用 `BackgroundJob`、`job_recovery` 和 `task_run`,用户暂停意图单独持久保存。当前树及 `library_processing.py` 的 Git 历史已核对,安装暂停只用于安装事务,不能直接代表用户暂停。Python 标准库 `threading.Event` 与 `contextvars.ContextVar` 提供等待唤醒和当前任务隔离,沿用项目 Python 版本,跨 Windows 与 macOS,无额外依赖体积;许可证为 PSF,官方文档与实现见 [threading](https://docs.python.org/3/library/threading.html) 和 [contextvars](https://docs.python.org/3/library/contextvars.html)。现有 APScheduler 3.11.3 用于定时派发,不能暂停已进入媒体读取的函数,因此不承担安全停点。隔离测试使用临时账本、暂停线程与模拟时钟验证安全停点、跨重启意图、低占用间隔和暂停预算补偿,不读取真实媒体、不发外部请求。
-`processing_control` 保留 Peach 的阶段、模式与估时策略:启动仅汇总账本文件数和大小,优先使用本机同类完成耗时;未知新增明确不在初始范围内。阶段样本形成后按实际耗时校准,并返回估时范围、依据和范围所指阶段。低占用在单项读盘、探测、下载和抽帧之间留出间隔,FFmpeg 使用单线程解码与滤镜;只作用于当前任务。持久状态不修改 ledger schema,不改变生产入口。
+`processing_control` 保留 Peach 的阶段、模式与估时策略:启动仅汇总账本文件数和大小,优先使用本机同类完成耗时;未知新增明确不在初始范围内。阶段样本形成后按实际耗时校准,并返回估时范围、依据和范围所指阶段。处理强度为低占用、标准和高强度:低占用采用单线程 FFmpeg 与较大单项间隔,标准采用两个解码、滤镜和编码线程与短间隔,高强度采用工具自动线程且没有额外间隔。amane 低占用逐站查询,其余沿用现有调度;各档均遵守来源限流,单资产流水线与暂停检查点共用。强度只作用于当前任务,历史耗时按档位系数归一,初始与切换时的范围属于粗估,实际样本继续校准。持久状态不修改 ledger schema,不改变生产入口。
diff --git a/src/peach/frame_capture.py b/src/peach/frame_capture.py
index 99ca4c615..ddd4d6be7 100644
--- a/src/peach/frame_capture.py
+++ b/src/peach/frame_capture.py
@@ -11,7 +11,7 @@ import os
import re
import subprocess
from pathlib import Path
-from .processing_control import checkpoint, low_mode
+from .processing_control import checkpoint, ffmpeg_options
#: 部分网盘视频把色彩原色写成 reserved(非法值),swscale 会拒绝缩放;声明为 bt709
#: 只是覆盖坏的元数据,不改动像素。
@@ -36,14 +36,13 @@ def capture_frame(ffmpeg: str, path: str, timestamp: float, destination: Path, *
"""
command = [ffmpeg, "-y", "-v", "error", "-rw_timeout", "8000000"]
checkpoint('frame')
- if low_mode():
- command += ['-threads', '1', '-filter_threads', '1']
+ command += ffmpeg_options(filter_option='-filter_threads')
if color_override:
command += COLOR_OVERRIDE
command += [
"-ss", f"{timestamp:.2f}", "-i", path, "-frames:v", "1",
"-vf", f"scale={int(width)}:-1", "-q:v", "4",
- *(['-threads', '1'] if low_mode() else []), str(destination),
+ *ffmpeg_options(), str(destination),
]
try:
completed = subprocess.run(command, capture_output=True, timeout=timeout,
@@ -74,11 +73,11 @@ def tile_frames(ffmpeg: str, pattern: str, columns: int, rows: int, destination:
"""
destination.parent.mkdir(parents=True, exist_ok=True)
checkpoint('frame')
- options = ['-threads', '1', '-filter_complex_threads', '1'] if low_mode() else []
+ options = ffmpeg_options(filter_option='-filter_complex_threads')
subprocess.run(
[ffmpeg, "-y", "-v", "error", *options, "-i", pattern,
"-filter_complex", f"tile={int(columns)}x{int(rows)}", "-q:v", "4",
- *(['-threads', '1'] if low_mode() else []), str(destination)],
+ *ffmpeg_options(), str(destination)],
capture_output=True, timeout=timeout,
creationflags=subprocess.CREATE_NO_WINDOW if os.name == "nt" else 0,
)
diff --git a/src/peach/processing_control.py b/src/peach/processing_control.py
index ef42a199d..ef8000f40 100644
--- a/src/peach/processing_control.py
+++ b/src/peach/processing_control.py
@@ -14,12 +14,18 @@ from .fsutil import atomic_write_text
CURRENT = ContextVar('peach_library_processing_control', default=None)
_CONTROLS = {}
_LOCK = threading.Lock()
-MODES = ('standard', 'low')
+MODES = ('low', 'standard', 'high')
+MODE_FACTORS = {'low': 1.8, 'standard': 1.0, 'high': 0.75}
+IO_INTERVALS = {
+ 'low': {'scan': 0.008, 'probe': 0.75, 'network': 1.5, 'frame': 0.75, 'work': 0.15},
+ 'standard': {'scan': 0.002, 'probe': 0.2, 'network': 0.25, 'frame': 0.2, 'work': 0.03},
+ 'high': {},
+}
def valid_mode(value):
if value not in MODES:
- raise ValueError('请选择标准或低占用模式')
+ raise ValueError('请选择低占用、标准或高强度模式')
return value
@@ -78,6 +84,20 @@ def low_mode():
return control is not None and control.mode == 'low'
+def mode_ratio(target, previous='standard'):
+ return MODE_FACTORS[valid_mode(target)] / MODE_FACTORS[valid_mode(previous)]
+
+
+def ffmpeg_options(*, filter_option=None):
+ """当前馆藏任务的 FFmpeg 线程选项;高强度与独立调用使用工具默认值。"""
+ control = CURRENT.get()
+ threads = {'low': 1, 'standard': 2}.get(control.mode) if control is not None else None
+ if threads is None:
+ return []
+ value = str(threads)
+ return ['-threads', value, *([filter_option, value] if filter_option else [])]
+
+
class ActiveDeadline(float):
"""单调时钟截止点随本任务的用户暂停时长平移。"""
def __new__(cls, value, control, paused=None):
@@ -211,8 +231,7 @@ class ProcessingControl:
self.publish(status='running', speed_mode=mode, resumed_after=spent)
started = None
# 控制只作用于当前馆藏任务;在单项 I/O 之间留出空档。
- delay = {'scan': 0.008, 'probe': 0.75, 'network': 1.5, 'frame': 0.75,
- 'work': 0.15}.get(kind, 0.15) if mode == 'low' else 0
+ delay = IO_INTERVALS[mode].get(kind, 0)
remaining = self.next_work.get(kind, 0) - time.monotonic()
if remaining > 0:
self.event.wait(min(0.25, remaining))
@@ -251,7 +270,7 @@ def initial_estimate(config, db_path, stage, asset_ids=None, mode=None):
historical = isinstance(sample, (int, float)) and math.isfinite(sample) and sample > 0
if historical:
seconds = sample
- factor = 1.8 if mode == 'low' else 1.0
+ factor = MODE_FACTORS[valid_mode(mode)]
known = bool(count) or asset_ids is not None or stage not in ('scan', 'all')
estimate = count * seconds * factor
return dict(speed_mode=mode,
@@ -281,7 +300,7 @@ class ProcessingEstimate:
def update(self, state):
mode = self.control.mode if self.control is not None else self.mode
if mode != self.mode:
- ratio = 1.8 if mode == 'low' else 1 / 1.8
+ ratio = mode_ratio(mode, self.mode)
for key in ('eta_seconds_min', 'eta_seconds_max'):
if isinstance(state.get(key), (int, float)):
state[key] = math.ceil(state[key] * ratio)
@@ -294,7 +313,7 @@ class ProcessingEstimate:
elapsed = max(0.01, time.monotonic() - self.started - self.paused_elapsed())
path = self.config.directory('state') / 'library-processing-timing.json'
history = _read(path)
- history[state['requested_stage']] = elapsed / state['estimate_file_count'] / (1.8 if mode == 'low' else 1)
+ history[state['requested_stage']] = elapsed / state['estimate_file_count'] / MODE_FACTORS[mode]
atomic_write_text(path, json.dumps(history) + '\n')
return
if state.get('status') != 'running':
diff --git a/src/peach/web_library_processing.py b/src/peach/web_library_processing.py
index 5732e8f23..5361c0a4e 100644
--- a/src/peach/web_library_processing.py
+++ b/src/peach/web_library_processing.py
@@ -5,7 +5,7 @@ from . import settings_file
from .library_processing import (ALL_STAGES, COLLECT_STAGE, SCAN_STAGE, STAGES,
METADATA_STAGE, PREVIEWS_STAGE, _asset_selection,
decorate, issues_path, process_library, snapshot)
-from .processing_control import control_for, initial_estimate, valid_mode
+from .processing_control import control_for, initial_estimate, valid_mode, mode_ratio
ACTIVE = ('running', 'pausing', 'paused')
@@ -55,7 +55,7 @@ def _control_action(contract, config, body):
mode = valid_mode(body.get('speed_mode'))
control.request(action, mode=mode)
if previous.get('status') in ACTIVE:
- ratio = 1.8 if mode == 'low' else 1 / 1.8
+ ratio = mode_ratio(mode, previous.get('speed_mode') or 'standard')
fields = dict(speed_mode=mode, eta_initial=True)
if previous.get('speed_mode') != mode:
for key in ('eta_seconds_min', 'eta_seconds_max'):
diff --git a/tests/test_processing_control.py b/tests/test_processing_control.py
index 2004bd2cd..0d3063713 100644
--- a/tests/test_processing_control.py
+++ b/tests/test_processing_control.py
@@ -37,11 +37,13 @@ class ProcessingControlTests(unittest.TestCase):
estimate = initial_estimate(self.config, self.db, 'all')
chosen = initial_estimate(self.config, self.db, 'all', [1, 2])
low = initial_estimate(self.config, self.db, 'all', mode='low')
+ high = initial_estimate(self.config, self.db, 'all', mode='high')
self.assertEqual(estimate['estimate_file_count'], 10)
self.assertEqual(estimate['estimate_bytes'], 10 * 1024**3)
self.assertGreater(estimate['eta_seconds_max'], estimate['eta_seconds_min'])
self.assertLess(chosen['eta_seconds_max'], estimate['eta_seconds_max'])
self.assertGreater(low['eta_seconds_max'], estimate['eta_seconds_max'])
+ self.assertLess(high['eta_seconds_max'], estimate['eta_seconds_max'])
self.assertTrue(estimate['eta_initial'])
self.assertIn('未含未知新增', estimate['eta_scope'])
@@ -80,7 +82,7 @@ class ProcessingControlTests(unittest.TestCase):
paused.set()
def worker():
try:
- with control.bind('one', publish=publish, active=lambda: True, db_path=self.db):
+ with control.bind('one', publish=publish, active=lambda: True, db_path=self.db, mode='high'):
entered.set()
control.request('pause', 'one')
checkpoint('probe')
@@ -104,10 +106,11 @@ class ProcessingControlTests(unittest.TestCase):
def test_saved_pause_intent_prevents_automatic_restart(self):
control = control_for(self.config)
- with control.bind('one', publish=lambda **_: None, active=lambda: True, db_path=self.db):
+ with control.bind('one', publish=lambda **_: None, active=lambda: True, db_path=self.db, mode='high'):
control.request('pause', 'one')
restored = ProcessingControl(control.path)
self.assertTrue(restored.paused)
+ self.assertEqual(restored.mode, 'high')
job_recovery.save(self.db, 'library-processing', 'one', {'stage': 'scan'}, waiting=True)
handler = Mock()
self.assertEqual(job_recovery.resume(self.db, {'library-processing': handler}), [])
@@ -197,8 +200,9 @@ class ProcessingControlTests(unittest.TestCase):
raise
with patch('peach.web_library_processing.settings_file.active', return_value=self.config), \
patch('peach.web_library_processing.process_library', side_effect=process):
- initial = w_library_processing(contract, {'stage': 'metadata'})
+ initial = w_library_processing(contract, {'stage': 'metadata', 'speed_mode': 'high'})
self.assertIn('eta_seconds_max', initial)
+ self.assertEqual(initial['speed_mode'], 'high')
self.assertTrue(entered.wait(2))
run_id = initial['run_id']
identity = initial['job_id']
@@ -263,7 +267,7 @@ class ProcessingControlTests(unittest.TestCase):
return {'status': 'complete'}
with patch('peach.web_library_processing.settings_file.active', return_value=self.config), \
patch('peach.web_library_processing.process_library', side_effect=process):
- initial = w_library_processing(contract, {'stage': 'metadata'})
+ initial = w_library_processing(contract, {'stage': 'metadata', 'speed_mode': 'high'})
self.assertTrue(entered.wait(2))
w_library_processing(contract, {'action': 'pause', 'job_id': initial['job_id']})
advance.set()
@@ -279,8 +283,127 @@ class ProcessingControlTests(unittest.TestCase):
self.assertEqual(resumes[1]['completed_metadata'], ['1:R:\\a.mp4'])
self.assertEqual(second.snapshot()['status'], 'complete')
self.assertEqual(second.snapshot()['run_id'], initial['run_id'])
+ self.assertEqual(control_for(self.config).mode, 'high')
self.assertEqual(runs.get(initial['run_id']).status, 'succeeded')
+ def test_three_intensities_set_distinct_io_start_intervals(self):
+ control = control_for(self.config)
+ intervals = {}
+ for mode in ('low', 'standard', 'high'):
+ with self.subTest(mode=mode), \
+ control.bind(mode, publish=lambda **_: None, active=lambda: True, db_path=self.db, mode=mode), \
+ patch('peach.processing_control.time.monotonic', return_value=10):
+ checkpoint('network')
+ intervals[mode] = control.next_work['network'] - 10
+ self.assertGreater(intervals['low'], intervals['standard'])
+ self.assertGreater(intervals['standard'], intervals['high'])
+ self.assertEqual(intervals['high'], 0)
+
+ def test_three_intensities_set_decode_filter_and_encode_threads(self):
+ from peach.frame_capture import capture_frame, tile_frames
+ control = control_for(self.config)
+ for mode, threads in (('low', '1'), ('standard', '2'), ('high', None)):
+ with self.subTest(mode=mode), \
+ control.bind(mode, publish=lambda **_: None, active=lambda: True, db_path=self.db, mode=mode), \
+ patch('peach.frame_capture.checkpoint'), \
+ patch('peach.frame_capture.subprocess.run', return_value=SimpleNamespace(stderr=b'')) as run:
+ capture_frame('ffmpeg', 'video.mp4', 1, self.root / 'frame.jpg')
+ tile_frames('ffmpeg', 'frame%02d.jpg', 3, 3, self.root / 'tile.jpg')
+ for call, filter_option in zip(run.call_args_list, ('-filter_threads', '-filter_complex_threads')):
+ command = call.args[0]
+ if threads is None:
+ self.assertNotIn('-threads', command)
+ self.assertNotIn(filter_option, command)
+ else:
+ self.assertEqual(command.count('-threads'), 2)
+ self.assertEqual(command[command.index(filter_option) + 1], threads)
+ self.assertEqual([command[index + 1] for index, value in enumerate(command)
+ if value == '-threads'], [threads, threads])
+
+ def test_intensity_switch_recalibrates_estimate_in_every_direction(self):
+ self.seed()
+ control = control_for(self.config)
+ for initial, target in (('low', 'standard'), ('standard', 'high'), ('high', 'low')):
+ with self.subTest(initial=initial, target=target), \
+ control.bind(initial, publish=lambda **_: None, active=lambda: True,
+ db_path=self.db, mode=initial):
+ state = dict(initial_estimate(self.config, self.db, 'metadata', mode=initial),
+ status='paused', requested_stage='metadata')
+ estimate = ProcessingEstimate(self.config, state, control)
+ old = state['eta_seconds_max']
+ control.request('set_speed', mode=target)
+ estimate.update(state)
+ self.assertEqual(state['speed_mode'], target)
+ if target == 'low':
+ self.assertGreater(state['eta_seconds_max'], old)
+ else:
+ self.assertLess(state['eta_seconds_max'], old)
+
+ def test_high_intensity_honors_installation_pause(self):
+ from peach.jobs import JobPaused
+ from peach import update_tasks
+ control = control_for(self.config)
+ with control.bind('one', publish=lambda **_: None, active=lambda: True, db_path=self.db, mode='high'):
+ token = update_tasks.reserve(self.db)
+ try:
+ with self.assertRaises(JobPaused):
+ checkpoint('frame')
+ finally:
+ update_tasks.release(self.db, token)
+ checkpoint('frame')
+ self.assertEqual(control.mode, 'high')
+
+ def test_high_history_is_normalized_for_every_initial_intensity(self):
+ self.seed()
+ control = control_for(self.config)
+ with control.bind('one', publish=lambda **_: None, active=lambda: True, db_path=self.db, mode='high'):
+ state = dict(initial_estimate(self.config, self.db, 'metadata', mode='high'),
+ requested_stage='metadata', status='running')
+ with patch('peach.processing_control.time.monotonic', return_value=100):
+ estimate = ProcessingEstimate(self.config, state, control)
+ state['status'] = 'complete'
+ with patch('peach.processing_control.time.monotonic', return_value=175):
+ estimate.update(state)
+ history = json.loads((self.config.directory('state') / 'library-processing-timing.json').read_text())
+ self.assertEqual(history['metadata'], 10.0)
+ low, standard, high = [initial_estimate(self.config, self.db, 'metadata', mode=mode)
+ for mode in ('low', 'standard', 'high')]
+ self.assertEqual(high['eta_kind'], 'historical')
+ self.assertGreater(low['eta_seconds_max'], standard['eta_seconds_max'])
+ self.assertGreater(standard['eta_seconds_max'], high['eta_seconds_max'])
+
+ def test_high_intensity_and_pause_intent_are_shared_between_control_instances(self):
+ control = control_for(self.config)
+ with control.bind('one', publish=lambda **_: None, active=lambda: True, db_path=self.db, mode='standard'):
+ external = ProcessingControl(control.path)
+ external.request('set_speed', 'one', mode='high')
+ external.request('pause', 'one')
+ control.refresh(force=True)
+ self.assertEqual(control.mode, 'high')
+ self.assertTrue(control.paused)
+ external.request('resume', 'one')
+ control.refresh(force=True)
+ self.assertFalse(control.paused)
+ self.assertEqual(control.mode, 'high')
+
+ def test_api_switches_between_all_intensities_with_proportional_paused_estimates(self):
+ from peach.jobs import BackgroundJob
+ control = control_for(self.config)
+ job = BackgroundJob('library-processing')
+ job.state = dict(job_id='one', status='paused', speed_mode='low', eta_seconds_min=180, eta_seconds_max=360)
+ contract = SimpleNamespace(db_path=self.db, library_processing_job=job)
+ with control.bind('one', publish=lambda **_: None, active=lambda: True, db_path=self.db, mode='low'), \
+ patch('peach.web_library_processing.settings_file.active', return_value=self.config):
+ control.request('pause', 'one')
+ high = w_library_processing(contract, {'action': 'set_speed', 'job_id': 'one', 'speed_mode': 'high'})
+ self.assertEqual((high['speed_mode'], high['status']), ('high', 'paused'))
+ self.assertEqual((high['eta_seconds_min'], high['eta_seconds_max']), (75, 150))
+ standard = w_library_processing(contract, {'action': 'set_speed', 'job_id': 'one', 'speed_mode': 'standard'})
+ self.assertEqual((standard['eta_seconds_min'], standard['eta_seconds_max']), (100, 200))
+ low = w_library_processing(contract, {'action': 'set_speed', 'job_id': 'one', 'speed_mode': 'low'})
+ self.assertEqual((low['eta_seconds_min'], low['eta_seconds_max']), (180, 360))
+ self.assertTrue(control.paused)
+
def test_scan_progress_accumulates_discovered_counts_across_roots(self):
from peach.library_processing import _scan_sources
reports = []