Peach: Local-first video metadata scraper and media manager.
fix(processing): 扫描与采集等待后台补全让路
RGB-loopbbdf5833f543
diff --git a/src/peach/download_postprocess.py b/src/peach/download_postprocess.pyindex e1b4c1c74..3128fa3d8 100644--- a/src/peach/download_postprocess.py+++ b/src/peach/download_postprocess.py@@ -12,6 +12,7 @@ from .downloads import (INGESTED, POSTPROCESS_COMPLETE, POSTPROCESS_FAILED,DownloadStore, iso)from .field_owners import SCAN_FILENAME, write_owned_fieldsfrom .followups import FollowupType, register+from .jobs import JobDeferredfrom .library_processing import COLLECT_STAGE, _ffprobe_path, process_libraryfrom .media_probe import probe_unmeasuredfrom .task_runs import declared_rows@@ -184,6 +185,10 @@ def run(contract, key: str, handle) -> dict:rows = declared_rows(metadata.get('followups'))if rows and getattr(handle, 'run_id', None) is not None:contract.task_runs.enqueue_followups(handle.run_id, rows)+ except JobDeferred as error:+ # 给扫描与采集让路:后继回到队列,路让回来后从头续跑,这次不算失败。+ store.postprocess(task_id, POSTPROCESS_QUEUED, detail=str(error))+ raiseexcept Exception as error:store.postprocess(task_id, POSTPROCESS_FAILED,detail=f"{type(error).__name__}: {error}")diff --git a/src/peach/followups.py b/src/peach/followups.pyindex f865dcf75..4968697d4 100644--- a/src/peach/followups.py+++ b/src/peach/followups.py@@ -31,6 +31,8 @@ LOGGER = logging.getLogger(__name__)#: 写账本的后继共用的通道名。它不是任务种类,是「这条路上一次只许过一个」。LEDGER_LANE = "ledger"+#: 后继让路以后,通道隔多久看一眼路让回来没有。+DEFER_POLL_SECONDS = 1.0@dataclass(frozen=True)@@ -212,8 +214,11 @@ class FollowupRunner:run = self.store.claim_followup(keys, first=_ahead(keys))if run is None:break- self._execute(run)+ deferred = self._execute(run)done += 1+ if deferred is not None:+ # 让路的那条已回到队列,同步排空时没人会把路让回来,留给后台通道。+ breakreturn done# -- 执行 --------------------------------------------------------------@@ -231,13 +236,17 @@ class FollowupRunner:with self._lock:self._active[thread] = run.task_keytry:- self._execute(run)+ deferred = self._execute(run)finally:with self._lock:self._active.pop(thread, None)+ # 让路的那条已回到队列:路让回来之前再领,领到的还是它。+ while deferred is not None and not self._stopping and not deferred.ready():+ time.sleep(DEFER_POLL_SECONDS)- def _execute(self, run) -> None:- from .jobs import JobPaused+ def _execute(self, run):+ """跑一条后继;它给别的任务让路时返回那个 `JobDeferred`,否则返回 None。"""+ from .jobs import JobDeferred, JobPausedfollowup_type = REGISTRY.get(run.task_key)handle = TaskRunHandle(self.store, run.id)if followup_type is None:@@ -247,6 +256,9 @@ class FollowupRunner:returntry:summary = followup_type.run(self.contract, run.followup_key, handle)+ except JobDeferred as error:+ self.store.suspend(run.id, label=str(error), paused_for_update=False)+ return errorexcept JobPaused as error:self.store.suspend(run.id, label=str(error) or '等待安装完成后继续')self._paused_for_update = Truediff --git a/src/peach/jobs.py b/src/peach/jobs.pyindex cda74a7e4..3b8d1abf3 100644--- a/src/peach/jobs.py+++ b/src/peach/jobs.py@@ -35,6 +35,13 @@ class JobPaused(JobPolicyError):exit_code = 3+class JobDeferred(RuntimeError):+ """后台任务这会儿跑不了,得给别的任务让路;回到队列,`ready()` 为真时再领。"""++ def ready(self) -> bool:+ return True++class JobCancelled(InterruptedError):"""用户停止任务;已保存结果保留,不再自动恢复。"""diff --git a/src/peach/library_processing.py b/src/peach/library_processing.pyindex 0b1a0d2a7..c6632d059 100644--- a/src/peach/library_processing.py+++ b/src/peach/library_processing.py@@ -7,10 +7,11 @@ import reimport sqlite3import subprocessimport tempfile+import threadingimport timeimport uuidimport xml.etree.ElementTree as ET-from contextlib import closing+from contextlib import closing, contextmanagerfrom contextvars import copy_contextfrom concurrent.futures import ThreadPoolExecutor, wait, FIRST_COMPLETEDfrom dataclasses import dataclass@@ -44,7 +45,7 @@ from .scan import scan_locationfrom .fsutil import atomic_write_textfrom . import artifact_rules, upgrade_rulesfrom .ffmpeg import FFmpegResolver-from .jobs import DiskGuard, JobPolicyError+from .jobs import DiskGuard, JobDeferred, JobPolicyErrorfrom .media_probe import probe_unmeasuredfrom .media_sources import cost_forfrom .media_processing_policy import metadata_only@@ -740,6 +741,83 @@ def state_path(config):return config.directory('state') / 'library-processing.json'+#: 等着全库处理锁的前台处理有几轮。后台定向任务看到它不为零就让路。+_waiting_foreground = 0+_waiting_guard = threading.Lock()+#: 前台处理等锁时多久再试一次。+LOCK_RETRY_SECONDS = 0.5+++def _foreground_waiting() -> bool:+ with _waiting_guard:+ return _waiting_foreground > 0+++def _lock_free(config) -> bool:+ try:+ with FileLock(str(state_path(config)) + '.lock', timeout=0):+ return True+ except Timeout:+ return False+++class ProcessingBusy(JobDeferred):+ """全库处理锁被占着,或者有用户发起的处理在等它;后台定向任务回到队列。"""++ def __init__(self, config):+ super().__init__('等待扫描与采集完成后继续')+ self.config = config++ def ready(self) -> bool:+ return not _foreground_waiting() and _lock_free(self.config)+++@contextmanager+def _processing_lock(config, *, background, active):+ """持有全库处理锁,返回本轮该用的 `active`。++ 共享候选文件整份读改写,同一时刻只能有一轮处理在写。用户发起的处理(不带+ `state_file`)等锁,并让在跑的后台定向任务在下一个检查点退出;后台任务碰上锁被占+ 或有人在等,就抛 `ProcessingBusy` 回到队列,凭各自的进度文件续跑。+ """+ global _waiting_foreground+ lock = FileLock(str(state_path(config)) + '.lock', timeout=0)+ if background:+ if _foreground_waiting():+ raise ProcessingBusy(config)+ try:+ lock.acquire()+ except Timeout:+ raise ProcessingBusy(config) from None+ try:+ yield lambda: active() and not _foreground_waiting()+ except InterruptedError:+ if active() and _foreground_waiting():+ raise ProcessingBusy(config) from None+ raise+ finally:+ lock.release()+ return+ with _waiting_guard:+ _waiting_foreground += 1+ try:+ while True:+ try:+ lock.acquire()+ break+ except Timeout:+ if not active():+ raise InterruptedError('处理任务已停止') from None+ time.sleep(LOCK_RETRY_SECONDS)+ try:+ yield active+ finally:+ lock.release()+ finally:+ with _waiting_guard:+ _waiting_foreground -= 1++def issues_path(config, job_id):return config.directory('state') / f'library-processing-{job_id}.issues.jsonl'@@ -2638,8 +2716,9 @@ def process_library(config, db_path, candidate_root, cover_root, *, location='coflush_state(force=True)report(dict(state))from .update_tasks import activity+ # 后台任务要让路时,`as active` 换上的那个会转为假;后面的检查点与闭包都读这一个。with (activity(Path(db_path), 'library-processing'), traffic.bind(),- FileLock(str(state_path(config)) + '.lock', timeout=0),+ _processing_lock(config, background=state_file is not None, active=active) as active,bind_control(control, identity, publish=control_report, active=active, db_path=Path(db_path),mode=speed_mode)):state = dict(job_id=identity, status='running',diff --git a/src/peach/task_runs.py b/src/peach/task_runs.pyindex 19f69ca10..b6707249b 100644--- a/src/peach/task_runs.py+++ b/src/peach/task_runs.py@@ -284,14 +284,15 @@ class TaskRunStore:return self._start(task_key, trigger=trigger, mutex_key=mutex_key,conflict=conflict, total=total, label=label, pid=pid)- def suspend(self, run_id: int, *, label: str = '等待安装完成后继续') -> None:+ def suspend(self, run_id: int, *, label: str = '等待安装完成后继续',+ paused_for_update: bool = True) -> None:if not self.enabled:returnwith self.database.write_transaction(notify=False) as connection:connection.execute("UPDATE task_run SET status='pending',pid=0,progress_label=?,result_summary=? ""WHERE id=? AND status='running'",- (label, json.dumps({'paused_for_update': True}), run_id))+ (label, json.dumps({'paused_for_update': True} if paused_for_update else {}), run_id))def _resume(self, run_id, task_key):if run_id is None or not self.enabled:diff --git a/tests/test_downloads_postprocess.py b/tests/test_downloads_postprocess.pyindex 8f92b768b..9c10eaf10 100644--- a/tests/test_downloads_postprocess.py+++ b/tests/test_downloads_postprocess.py@@ -10,6 +10,7 @@ from unittest.mock import Mock, patchfrom peach import download_postprocess as postfrom peach import ledger_backupsfrom peach.downloads import DownloadStore, DownloadTask, INGESTED, POSTPROCESS_PENDING+from peach.jobs import JobDeferredfrom peach.repository import LedgerDatabasefrom support.ledger import fresh_ledger@@ -150,6 +151,16 @@ class DownloadPostprocessTests(unittest.TestCase):self.assertEqual(self.run_task(), {'task_id': self.task.id, 'already_complete': True})self.contract.task_runs.enqueue_followups.assert_called_once()+ def test_collection_that_steps_aside_stays_queued_instead_of_failed(self):+ self.handle.run_id = 123+ self.contract.task_runs = Mock()+ with patch.object(post, '_collect', side_effect=JobDeferred('等待扫描与采集完成后继续')):+ with self.assertRaises(JobDeferred):+ self.run_task()+ task = self.store.get(self.task.id)+ self.assertEqual((task.postprocess_state, task.postprocess_detail), ('queued', '等待扫描与采集完成后继续'))+ self.contract.task_runs.enqueue_followups.assert_not_called()+def test_metadata_or_organization_failure_never_dispatches_followups(self):self.handle.run_id = 123self.contract.task_runs = Mock()diff --git a/tests/test_followups.py b/tests/test_followups.pyindex 220c62cab..31b2bac53 100644--- a/tests/test_followups.py+++ b/tests/test_followups.py@@ -16,7 +16,7 @@ from peach.avatar_followup import TASK_KEY as AVATAR_TASK_KEYfrom peach.avatar_followup import followup_key, parse_key, planfrom peach.followups import FollowupRunner, FollowupType, lanesfrom peach.jav_cover_fetch import NotFound-from peach.jobs import BackgroundJob+from peach.jobs import BackgroundJob, JobDeferredfrom peach.library_processing import process_libraryfrom peach.repository import LedgerDatabasefrom peach.task_runs import MAX_FOLLOWUPS, TaskRunStore@@ -210,6 +210,55 @@ class RunnerTests(LedgerTestCase):self.assertEqual({row.status for row in self.store.children(parent.id)}, {"succeeded"})+ def test_a_followup_that_steps_aside_waits_in_the_queue_and_runs_once_the_way_is_clear(self):+ """给扫描与采集让路的后继回到队列,不判失败;路让回来以后同一条接着跑完。"""+ clear = threading.Event()+ attempts = []++ class Blocked(JobDeferred):+ def ready(self):+ return clear.is_set()++ def busy_until_clear(_contract, key, _handle):+ attempts.append(key)+ if not clear.is_set():+ raise Blocked("等待扫描与采集完成后继续")+ return {"outcome": key}++ parent = self.parent()+ with registered(echo_type("ledger-a", run=busy_until_clear)), \+ mock.patch.object(followups_module, "DEFER_POLL_SECONDS", 0.01):+ child = self.store.enqueue_followups(parent.id, [("ledger-a:1", "ledger-a", "甲")])["queued"][0]+ runner = FollowupRunner(self.contract())+ runner.wake()+ deadline = time.monotonic() + 10+ while not attempts and time.monotonic() < deadline:+ time.sleep(0.01)+ time.sleep(0.05)+ waiting = self.store.get(child)+ self.assertEqual(waiting.status, "pending")+ self.assertEqual(waiting.progress_label, "等待扫描与采集完成后继续")+ self.assertEqual(attempts, ["ledger-a:1"])+ clear.set()+ self.wait_until_settled(parent.id)+ runner.stop(timeout=10)+ self.assertEqual(self.store.get(child).status, "succeeded")+ self.assertEqual(attempts, ["ledger-a:1", "ledger-a:1"])++ def test_draining_leaves_a_followup_that_steps_aside_in_the_queue(self):+ attempts = []++ def busy(_contract, key, _handle):+ attempts.append(key)+ raise JobDeferred("等待扫描与采集完成后继续")++ parent = self.parent()+ with registered(echo_type("ledger-a", run=busy)):+ child = self.store.enqueue_followups(parent.id, [("ledger-a:1", "ledger-a", "甲")])["queued"][0]+ self.assertEqual(FollowupRunner(self.contract()).drain(), 1)+ self.assertEqual(attempts, ["ledger-a:1"])+ self.assertEqual(self.store.get(child).status, "pending")+def test_a_type_that_jumps_the_queue_is_claimed_before_earlier_ones_in_its_lane(self):"""下载后处理这类人在等的后继,排在已经在排的后台批次前面;同一档里仍是先排先跑。"""order = []diff --git a/tests/test_metadata_library.py b/tests/test_metadata_library.pyindex 5ccb7734f..bec1e6993 100644--- a/tests/test_metadata_library.py+++ b/tests/test_metadata_library.py@@ -15,15 +15,15 @@ from contextlib import closingfrom filelock import FileLockfrom PIL import Image-from peach import artifact_rules, upgrade_rules+from peach import artifact_rules, library_processing, upgrade_rulesfrom peach.field_owners import auto_owner, owner_of, review_ownerfrom peach.genre_taxonomy import map_genresfrom peach.jav_cover_fetch import NotFoundfrom peach.http import HttpResponsefrom peach.library_nfo import read_nfo, sidecars, local_art-from peach.library_processing import (STALL_AFTER_SECONDS, _candidate_identity, _save, decorate,- issues_path, _merge_local_performer_profiles,+from peach.library_processing import (STALL_AFTER_SECONDS, ProcessingBusy, _candidate_identity, _save,+ decorate, issues_path, _merge_local_performer_profiles,process_library, snapshot, state_path, _fields)from peach.metadata_auto_apply import _apply_metadata_candidatefrom peach.repository import LedgerDatabase@@ -3585,3 +3585,122 @@ class LibraryPreviewCompletionTests(unittest.TestCase):generated / 'posters', generated / 'avatars', generated / 'logos')with Image.open(service.poster(1)) as image:self.assertEqual(image.size, (480, 270))+++class ProcessingLockTests(unittest.TestCase):+ """全库处理锁:用户发起的处理等锁,后台定向任务让路而不是报错。"""++ def setUp(self):+ temporary = tempfile.TemporaryDirectory()+ self.addCleanup(temporary.cleanup)+ self.root = Path(temporary.name).resolve()+ media = self.root / 'media'+ media.mkdir()+ prefix = str(media) if os.name == 'nt' else 'R:\\Media'+ self.config = PeachConfig(self.root, self.root / 'config.toml', present=True,+ locations={'local': (prefix,)}, mounts={'local': (str(media),)})+ self.enterContext(patch('peach.settings_file.active', return_value=self.config))+ self.db = fresh_ledger(self.root)+ self.database = LedgerDatabase(self.db)+ with self.database.write_transaction() as connection:+ for aid, code in ((1, 'ABW-358'), (2, 'ABW-359')):+ video = media / f'{code}.mp4'+ video.write_bytes(b'video')+ connection.execute('INSERT INTO asset(id,location,path,name,medium,code) VALUES(?,?,?,?,?,?)',+ (aid, 'local', str(PureWindowsPath(prefix) / video.name), video.name,+ 'video', code))++ def test_a_background_collection_steps_aside_while_the_processing_lock_is_held(self):+ config, db, database = self.config, self.db, self.database+ provider = stub_provider()+ provider.query.return_value = {}+ output = self.root / 'state/upgrade-repairs/1.json'+ holder = FileLock(str(state_path(config)) + '.lock', timeout=0)+ holder.acquire()+ try:+ with self.assertRaises(ProcessingBusy) as caught:+ process_library(config, db, self.root / 'generated', self.root / 'covers', asset_ids=[1],+ stage='collect', state_file=output, provider_factory=lambda: provider,+ database=database)+ self.assertFalse(caught.exception.ready())+ finally:+ holder.release()+ self.assertTrue(caught.exception.ready())+ provider.query.assert_not_called()++ def test_a_user_run_waits_for_the_processing_lock_and_holds_background_work_back(self):+ config, db, database = self.config, self.db, self.database+ provider = stub_provider()+ provider.query.return_value = {}+ holder = FileLock(str(state_path(config)) + '.lock', timeout=0)+ holder.acquire()+ results, errors = [], []++ def user_run():+ try:+ results.append(process_library(config, db, self.root / 'generated', self.root / 'covers',+ provider_factory=lambda: provider, database=database))+ except Exception as error:+ errors.append(error)++ with patch('peach.library_processing.LOCK_RETRY_SECONDS', 0.01), \+ patch('peach.library_processing._enrich_finished_performers',+ return_value={'aliases': 0, 'avatars': 0, 'conflicts': 0, 'failed': 0}):+ thread = threading.Thread(target=user_run)+ thread.start()+ try:+ deadline = time.monotonic() + 10+ while not library_processing._foreground_waiting() and time.monotonic() < deadline:+ time.sleep(0.01)+ self.assertTrue(library_processing._foreground_waiting())+ with self.assertRaises(ProcessingBusy):+ process_library(config, db, self.root / 'generated', self.root / 'covers', asset_ids=[1],+ stage='collect', state_file=self.root / 'state/upgrade-repairs/1.json',+ provider_factory=lambda: provider, database=database)+ finally:+ holder.release()+ thread.join(30)+ self.assertEqual(errors, [])+ self.assertNotEqual(results[0].get('outcome'), 'interrupted')+ self.assertEqual(sorted({call.args[0] for call in provider.query.call_args_list}), ['ABW-358', 'ABW-359'])+ self.assertFalse(library_processing._foreground_waiting())++ def test_a_running_background_collection_yields_to_a_user_run_at_its_next_checkpoint(self):+ config, db, database = self.config, self.db, self.database+ entered, proceed = threading.Event(), threading.Event()++ def query(*_args, **_kwargs):+ if not entered.is_set():+ entered.set()+ proceed.wait(10)+ return {}++ provider = stub_provider()+ provider.query.side_effect = query+ background, user = [], []++ def run(sink, **options):+ try:+ sink.append(process_library(config, db, self.root / 'generated', self.root / 'covers',+ provider_factory=lambda: provider, database=database, **options))+ except Exception as error:+ sink.append(error)++ with patch('peach.library_processing.LOCK_RETRY_SECONDS', 0.01), \+ patch('peach.library_processing._enrich_finished_performers',+ return_value={'aliases': 0, 'avatars': 0, 'conflicts': 0, 'failed': 0}):+ first = threading.Thread(target=run, args=(background,), kwargs=dict(+ asset_ids=[1, 2], stage='collect', state_file=self.root / 'state/upgrade-repairs/1.json'))+ first.start()+ self.assertTrue(entered.wait(10))+ second = threading.Thread(target=run, args=(user,))+ second.start()+ deadline = time.monotonic() + 10+ while not library_processing._foreground_waiting() and time.monotonic() < deadline:+ time.sleep(0.01)+ proceed.set()+ first.join(30)+ second.join(30)+ self.assertIsInstance(background[0], ProcessingBusy)+ self.assertIsInstance(user[0], dict)+ self.assertNotEqual(user[0].get('outcome'), 'interrupted')