Peach: Local-first video metadata scraper and media manager.
fix(tasks): 校验活动任务记录的进程创建时间
zouyongheafcf9d224c26
diff --git a/src/peach/job_recovery.py b/src/peach/job_recovery.pyindex a2017d16a..3ed1a99e9 100644--- a/src/peach/job_recovery.py+++ b/src/peach/job_recovery.py@@ -21,6 +21,20 @@ def _path(db: Path, key: str) -> Path:return state_for(db).parent / "job-recovery" / (key + ".json")+def _worker_alive(db: Path, data: dict) -> bool:+ from .jobs import process_alive, process_matches+ pid = data.get('pid')+ if not isinstance(pid, int) or pid <= 0:+ return False+ run_id = data.get('run_id')+ if isinstance(run_id, int):+ from .task_runs import TaskRunStore+ run = TaskRunStore(db).get(run_id)+ if run is not None and run.pid == pid:+ return process_matches(pid, run.started_at)+ return process_alive(pid)++def read(db: Path, key: str) -> dict:try:data = json.loads(_path(db, key).read_text(encoding="utf-8"))@@ -84,14 +98,12 @@ def request_pause(db: Path) -> None:def reconcile_paused(db: Path) -> None:- from .jobs import process_alivefrom .task_runs import TaskRunStorewith _mutex(state_for(db)):for path in (state_for(db).parent / 'job-recovery').glob('*.json'):data = read(db, path.stem)- pid = data.get('pid')if (data.get('status') == 'active' and data.get('pause_requested')- and isinstance(pid, int) and not process_alive(pid)):+ and not _worker_alive(db, data)):if data.get('run_id') is not None:TaskRunStore(db).suspend(data['run_id'])waiting(db, path.stem, data['job_id'])@@ -99,7 +111,6 @@ def reconcile_paused(db: Path) -> None:def resume(db: Path, handlers: dict) -> list[str]:"""领取与起跑在同一把锁内;失败保留参数但不循环重试。"""- from .jobs import process_aliveresumed = []for key, handler in handlers.items():try:@@ -111,10 +122,10 @@ def resume(db: Path, handlers: dict) -> list[str]:if user_paused(db, data.get('job_id')):continueif (data.get("status") != "active" or not isinstance(data.get("request"), dict)- or (isinstance(pid, int) and pid != os.getpid() and process_alive(pid))):+ or _worker_alive(db, data)):continue# 当前进程自己的新任务不属于启动恢复。- if pid == os.getpid():+ if pid == os.getpid() and not isinstance(data.get('run_id'), int):continueif key == 'library-processing' and user_cancelled(db, data.get('job_id')):if data.get('run_id') is not None:diff --git a/src/peach/jobs.py b/src/peach/jobs.pyindex 3b8d1abf3..53e25091a 100644--- a/src/peach/jobs.py+++ b/src/peach/jobs.py@@ -199,6 +199,33 @@ def process_alive(pid: int) -> bool:return True+def process_creation_time(pid: int) -> float | None:+ """返回 PID 当前指向进程的 Unix 创建时间;权限不足时保守返回未知。"""+ import psutil+ try:+ return psutil.Process(pid).create_time()+ except psutil.NoSuchProcess:+ return None+ except (psutil.AccessDenied, psutil.ZombieProcess):+ return None+++def process_matches(pid: int, started_at: str | None, *, alive=process_alive) -> bool:+ """核对活 PID 是否就是任务启动时的进程,避免 PID 复用永久占锁。"""+ if not alive(pid):+ return False+ if not started_at:+ return True+ from datetime import datetime+ try:+ task_started = datetime.fromisoformat(started_at.replace('Z', '+00:00')).timestamp()+ except (TypeError, ValueError, OverflowError):+ return True+ created = process_creation_time(pid)+ # 无权读取创建时间时不误回收真实任务;允许时间戳及 Win32 精度的小幅偏差。+ return created is None or created <= task_started + 2.0++def _process_alive_windows(pid: int) -> bool:"""用 OpenProcess 查存活,绝不能用 `os.kill(pid, 0)`。diff --git a/src/peach/task_runs.py b/src/peach/task_runs.pyindex b6707249b..ef8c027ee 100644--- a/src/peach/task_runs.py+++ b/src/peach/task_runs.py@@ -33,7 +33,7 @@ from datetime import datetime, timedelta, timezonefrom pathlib import Pathfrom typing import Iterator-from .jobs import TaskRunConflict, process_alive+from .jobs import TaskRunConflict, process_alive, process_matchesfrom .repository import LedgerDatabase__all__ = [@@ -641,7 +641,7 @@ class TaskRunStore:continuelocal = run.host == self.hostalive = (local and run.pid is not None and run.pid != os.getpid()- and process_alive(run.pid))+ and process_matches(run.pid, run.started_at, alive=process_alive))beat = _parse(run.heartbeat_at) or _parse(run.started_at)expired = beat is None or beat < deadlineif alive or (not local and not expired):diff --git a/tests/test_task_runs.py b/tests/test_task_runs.pyindex af704791a..c658643f5 100644--- a/tests/test_task_runs.py+++ b/tests/test_task_runs.py@@ -189,6 +189,18 @@ class TaskRunStoreTests(unittest.TestCase):self.assertEqual(self.store.recover_interrupted(), [])self.assertEqual(self.store.get(run.id).status, "running")+ def test_recovery_does_not_treat_a_reused_pid_as_the_original_worker(self):+ run = self.store.start("library-processing", trigger="manual")+ old_start = stamp(datetime.now(timezone.utc) - timedelta(days=1))+ with self.database.write_transaction(notify=False) as connection:+ connection.execute("UPDATE task_run SET pid=?,started_at=?,heartbeat_at=? WHERE id=?",+ (os.getpid() + 1, old_start, old_start, run.id))+ created_after_task = datetime.now(timezone.utc).timestamp()+ with mock.patch("peach.task_runs.process_alive", return_value=True), \+ mock.patch("peach.jobs.process_creation_time", return_value=created_after_task):+ self.assertEqual(self.store.recover_interrupted(), [run.id])+ self.assertEqual(self.store.get(run.id).status, "interrupted")+def test_recovery_of_another_machine_waits_for_the_lease_to_expire(self):run = self.store.start("demo", trigger="manual")stale = stamp(datetime.now(timezone.utc) - timedelta(seconds=30))diff --git a/tests/test_update_tasks.py b/tests/test_update_tasks.pyindex e35c59866..e471512b6 100644--- a/tests/test_update_tasks.py+++ b/tests/test_update_tasks.py@@ -8,11 +8,12 @@ import sysimport tempfileimport threadingimport unittest+from datetime import datetime, timedelta, timezonefrom unittest.mock import patchfrom peach import desktop_installer, job_recovery, settings_file, update_tasksfrom peach.jobs import BackgroundJob-from peach.task_runs import TaskRunStore+from peach.task_runs import TaskRunStore, stampfrom support.ledger import fresh_ledger@@ -66,6 +67,24 @@ class UpdateTaskTests(unittest.TestCase):pending = self.store.get(run.id)self.assertEqual((pending.status, pending.progress_current, pending.progress_total), ('pending', 3, 5))+ def test_recovery_resumes_when_saved_pid_was_reused_by_another_process(self):+ run = self.store.start('library-processing', trigger='manual')+ old_start = stamp(datetime.now(timezone.utc) - timedelta(days=1))+ with self.store.database.write_transaction(notify=False) as connection:+ connection.execute('UPDATE task_run SET started_at=?,heartbeat_at=? WHERE id=?',+ (old_start, old_start, run.id))+ job_recovery.save(self.db, 'library-processing', 'old-job', {'stage': 'collect'}, run_id=run.id)+ path = update_tasks.state_for(self.db).parent / 'job-recovery/library-processing.json'+ data = json.loads(path.read_text(encoding='utf-8'))+ data['pid'] = os.getpid()+ path.write_text(json.dumps(data), encoding='utf-8')+ observed = []+ handler = {'library-processing': lambda request: (observed.append(request), {'resumed': True})[1]}+ with patch('peach.jobs.process_creation_time', return_value=datetime.now(timezone.utc).timestamp()):+ self.assertEqual(job_recovery.resume(self.db, handler), ['library-processing'])+ self.assertEqual(observed, [{'stage': 'collect'}])+ self.assertEqual(self.store.get(run.id).status, 'running')+def test_terminal_checkpoint_is_readable_by_a_new_job_instance(self):first = BackgroundJob('resume-test', runs=self.store, task_key='link-check')original = first.start(lambda identity: first.update(identity, status='complete', checked=2, total=2), recovery={})