Skip to content

armgit / peach

Public Imported from GitHub
0

Peach: Local-first video metadata scraper and media manager.

fix(tasks): 校验活动任务记录的进程创建时间

zouyongheafcf9d224c26
diff --git a/src/peach/job_recovery.py b/src/peach/job_recovery.py
index a2017d16a..3ed1a99e9 100644
--- a/src/peach/job_recovery.py
+++ b/src/peach/job_recovery.py
@@ -21,6 +21,20 @@ def _path(db: Path, key: str) -> Path:
return state_for(db).parent / "job-recovery" / (key + ".json")
+def _worker_alive(db: Path, data: dict) -> bool:
+ from .jobs import process_alive, process_matches
+ pid = data.get('pid')
+ if not isinstance(pid, int) or pid <= 0:
+ return False
+ run_id = data.get('run_id')
+ if isinstance(run_id, int):
+ from .task_runs import TaskRunStore
+ run = TaskRunStore(db).get(run_id)
+ if run is not None and run.pid == pid:
+ return process_matches(pid, run.started_at)
+ return process_alive(pid)
+
+
def read(db: Path, key: str) -> dict:
try:
data = json.loads(_path(db, key).read_text(encoding="utf-8"))
@@ -84,14 +98,12 @@ def request_pause(db: Path) -> None:
def reconcile_paused(db: Path) -> None:
- from .jobs import process_alive
from .task_runs import TaskRunStore
with _mutex(state_for(db)):
for path in (state_for(db).parent / 'job-recovery').glob('*.json'):
data = read(db, path.stem)
- pid = data.get('pid')
if (data.get('status') == 'active' and data.get('pause_requested')
- and isinstance(pid, int) and not process_alive(pid)):
+ and not _worker_alive(db, data)):
if data.get('run_id') is not None:
TaskRunStore(db).suspend(data['run_id'])
waiting(db, path.stem, data['job_id'])
@@ -99,7 +111,6 @@ def reconcile_paused(db: Path) -> None:
def resume(db: Path, handlers: dict) -> list[str]:
"""领取与起跑在同一把锁内;失败保留参数但不循环重试。"""
- from .jobs import process_alive
resumed = []
for key, handler in handlers.items():
try:
@@ -111,10 +122,10 @@ def resume(db: Path, handlers: dict) -> list[str]:
if user_paused(db, data.get('job_id')):
continue
if (data.get("status") != "active" or not isinstance(data.get("request"), dict)
- or (isinstance(pid, int) and pid != os.getpid() and process_alive(pid))):
+ or _worker_alive(db, data)):
continue
# 当前进程自己的新任务不属于启动恢复。
- if pid == os.getpid():
+ if pid == os.getpid() and not isinstance(data.get('run_id'), int):
continue
if key == 'library-processing' and user_cancelled(db, data.get('job_id')):
if data.get('run_id') is not None:
diff --git a/src/peach/jobs.py b/src/peach/jobs.py
index 3b8d1abf3..53e25091a 100644
--- a/src/peach/jobs.py
+++ b/src/peach/jobs.py
@@ -199,6 +199,33 @@ def process_alive(pid: int) -> bool:
return True
+def process_creation_time(pid: int) -> float | None:
+ """返回 PID 当前指向进程的 Unix 创建时间;权限不足时保守返回未知。"""
+ import psutil
+ try:
+ return psutil.Process(pid).create_time()
+ except psutil.NoSuchProcess:
+ return None
+ except (psutil.AccessDenied, psutil.ZombieProcess):
+ return None
+
+
+def process_matches(pid: int, started_at: str | None, *, alive=process_alive) -> bool:
+ """核对活 PID 是否就是任务启动时的进程,避免 PID 复用永久占锁。"""
+ if not alive(pid):
+ return False
+ if not started_at:
+ return True
+ from datetime import datetime
+ try:
+ task_started = datetime.fromisoformat(started_at.replace('Z', '+00:00')).timestamp()
+ except (TypeError, ValueError, OverflowError):
+ return True
+ created = process_creation_time(pid)
+ # 无权读取创建时间时不误回收真实任务;允许时间戳及 Win32 精度的小幅偏差。
+ return created is None or created <= task_started + 2.0
+
+
def _process_alive_windows(pid: int) -> bool:
"""用 OpenProcess 查存活,绝不能用 `os.kill(pid, 0)`。
diff --git a/src/peach/task_runs.py b/src/peach/task_runs.py
index b6707249b..ef8c027ee 100644
--- a/src/peach/task_runs.py
+++ b/src/peach/task_runs.py
@@ -33,7 +33,7 @@ from datetime import datetime, timedelta, timezone
from pathlib import Path
from typing import Iterator
-from .jobs import TaskRunConflict, process_alive
+from .jobs import TaskRunConflict, process_alive, process_matches
from .repository import LedgerDatabase
__all__ = [
@@ -641,7 +641,7 @@ class TaskRunStore:
continue
local = run.host == self.host
alive = (local and run.pid is not None and run.pid != os.getpid()
- and process_alive(run.pid))
+ and process_matches(run.pid, run.started_at, alive=process_alive))
beat = _parse(run.heartbeat_at) or _parse(run.started_at)
expired = beat is None or beat < deadline
if alive or (not local and not expired):
diff --git a/tests/test_task_runs.py b/tests/test_task_runs.py
index af704791a..c658643f5 100644
--- a/tests/test_task_runs.py
+++ b/tests/test_task_runs.py
@@ -189,6 +189,18 @@ class TaskRunStoreTests(unittest.TestCase):
self.assertEqual(self.store.recover_interrupted(), [])
self.assertEqual(self.store.get(run.id).status, "running")
+ def test_recovery_does_not_treat_a_reused_pid_as_the_original_worker(self):
+ run = self.store.start("library-processing", trigger="manual")
+ old_start = stamp(datetime.now(timezone.utc) - timedelta(days=1))
+ with self.database.write_transaction(notify=False) as connection:
+ connection.execute("UPDATE task_run SET pid=?,started_at=?,heartbeat_at=? WHERE id=?",
+ (os.getpid() + 1, old_start, old_start, run.id))
+ created_after_task = datetime.now(timezone.utc).timestamp()
+ with mock.patch("peach.task_runs.process_alive", return_value=True), \
+ mock.patch("peach.jobs.process_creation_time", return_value=created_after_task):
+ self.assertEqual(self.store.recover_interrupted(), [run.id])
+ self.assertEqual(self.store.get(run.id).status, "interrupted")
+
def test_recovery_of_another_machine_waits_for_the_lease_to_expire(self):
run = self.store.start("demo", trigger="manual")
stale = stamp(datetime.now(timezone.utc) - timedelta(seconds=30))
diff --git a/tests/test_update_tasks.py b/tests/test_update_tasks.py
index e35c59866..e471512b6 100644
--- a/tests/test_update_tasks.py
+++ b/tests/test_update_tasks.py
@@ -8,11 +8,12 @@ import sys
import tempfile
import threading
import unittest
+from datetime import datetime, timedelta, timezone
from unittest.mock import patch
from peach import desktop_installer, job_recovery, settings_file, update_tasks
from peach.jobs import BackgroundJob
-from peach.task_runs import TaskRunStore
+from peach.task_runs import TaskRunStore, stamp
from support.ledger import fresh_ledger
@@ -66,6 +67,24 @@ class UpdateTaskTests(unittest.TestCase):
pending = self.store.get(run.id)
self.assertEqual((pending.status, pending.progress_current, pending.progress_total), ('pending', 3, 5))
+ def test_recovery_resumes_when_saved_pid_was_reused_by_another_process(self):
+ run = self.store.start('library-processing', trigger='manual')
+ old_start = stamp(datetime.now(timezone.utc) - timedelta(days=1))
+ with self.store.database.write_transaction(notify=False) as connection:
+ connection.execute('UPDATE task_run SET started_at=?,heartbeat_at=? WHERE id=?',
+ (old_start, old_start, run.id))
+ job_recovery.save(self.db, 'library-processing', 'old-job', {'stage': 'collect'}, run_id=run.id)
+ path = update_tasks.state_for(self.db).parent / 'job-recovery/library-processing.json'
+ data = json.loads(path.read_text(encoding='utf-8'))
+ data['pid'] = os.getpid()
+ path.write_text(json.dumps(data), encoding='utf-8')
+ observed = []
+ handler = {'library-processing': lambda request: (observed.append(request), {'resumed': True})[1]}
+ with patch('peach.jobs.process_creation_time', return_value=datetime.now(timezone.utc).timestamp()):
+ self.assertEqual(job_recovery.resume(self.db, handler), ['library-processing'])
+ self.assertEqual(observed, [{'stage': 'collect'}])
+ self.assertEqual(self.store.get(run.id).status, 'running')
+
def test_terminal_checkpoint_is_readable_by_a_new_job_instance(self):
first = BackgroundJob('resume-test', runs=self.store, task_key='link-check')
original = first.start(lambda identity: first.update(identity, status='complete', checked=2, total=2), recovery={})