Skip to content

armgit / peach

Public Imported from GitHub
0

Peach: Local-first video metadata scraper and media manager.

fix(processing): 扫描与采集等待后台补全让路

RGB-loopbbdf5833f543
diff --git a/src/peach/download_postprocess.py b/src/peach/download_postprocess.py
index e1b4c1c74..3128fa3d8 100644
--- a/src/peach/download_postprocess.py
+++ b/src/peach/download_postprocess.py
@@ -12,6 +12,7 @@ from .downloads import (INGESTED, POSTPROCESS_COMPLETE, POSTPROCESS_FAILED,
DownloadStore, iso)
from .field_owners import SCAN_FILENAME, write_owned_fields
from .followups import FollowupType, register
+from .jobs import JobDeferred
from .library_processing import COLLECT_STAGE, _ffprobe_path, process_library
from .media_probe import probe_unmeasured
from .task_runs import declared_rows
@@ -184,6 +185,10 @@ def run(contract, key: str, handle) -> dict:
rows = declared_rows(metadata.get('followups'))
if rows and getattr(handle, 'run_id', None) is not None:
contract.task_runs.enqueue_followups(handle.run_id, rows)
+ except JobDeferred as error:
+ # 给扫描与采集让路:后继回到队列,路让回来后从头续跑,这次不算失败。
+ store.postprocess(task_id, POSTPROCESS_QUEUED, detail=str(error))
+ raise
except Exception as error:
store.postprocess(task_id, POSTPROCESS_FAILED,
detail=f"{type(error).__name__}: {error}")
diff --git a/src/peach/followups.py b/src/peach/followups.py
index f865dcf75..4968697d4 100644
--- a/src/peach/followups.py
+++ b/src/peach/followups.py
@@ -31,6 +31,8 @@ LOGGER = logging.getLogger(__name__)
#: 写账本的后继共用的通道名。它不是任务种类,是「这条路上一次只许过一个」。
LEDGER_LANE = "ledger"
+#: 后继让路以后,通道隔多久看一眼路让回来没有。
+DEFER_POLL_SECONDS = 1.0
@dataclass(frozen=True)
@@ -212,8 +214,11 @@ class FollowupRunner:
run = self.store.claim_followup(keys, first=_ahead(keys))
if run is None:
break
- self._execute(run)
+ deferred = self._execute(run)
done += 1
+ if deferred is not None:
+ # 让路的那条已回到队列,同步排空时没人会把路让回来,留给后台通道。
+ break
return done
# -- 执行 --------------------------------------------------------------
@@ -231,13 +236,17 @@ class FollowupRunner:
with self._lock:
self._active[thread] = run.task_key
try:
- self._execute(run)
+ deferred = self._execute(run)
finally:
with self._lock:
self._active.pop(thread, None)
+ # 让路的那条已回到队列:路让回来之前再领,领到的还是它。
+ while deferred is not None and not self._stopping and not deferred.ready():
+ time.sleep(DEFER_POLL_SECONDS)
- def _execute(self, run) -> None:
- from .jobs import JobPaused
+ def _execute(self, run):
+ """跑一条后继;它给别的任务让路时返回那个 `JobDeferred`,否则返回 None。"""
+ from .jobs import JobDeferred, JobPaused
followup_type = REGISTRY.get(run.task_key)
handle = TaskRunHandle(self.store, run.id)
if followup_type is None:
@@ -247,6 +256,9 @@ class FollowupRunner:
return
try:
summary = followup_type.run(self.contract, run.followup_key, handle)
+ except JobDeferred as error:
+ self.store.suspend(run.id, label=str(error), paused_for_update=False)
+ return error
except JobPaused as error:
self.store.suspend(run.id, label=str(error) or '等待安装完成后继续')
self._paused_for_update = True
diff --git a/src/peach/jobs.py b/src/peach/jobs.py
index cda74a7e4..3b8d1abf3 100644
--- a/src/peach/jobs.py
+++ b/src/peach/jobs.py
@@ -35,6 +35,13 @@ class JobPaused(JobPolicyError):
exit_code = 3
+class JobDeferred(RuntimeError):
+ """后台任务这会儿跑不了,得给别的任务让路;回到队列,`ready()` 为真时再领。"""
+
+ def ready(self) -> bool:
+ return True
+
+
class JobCancelled(InterruptedError):
"""用户停止任务;已保存结果保留,不再自动恢复。"""
diff --git a/src/peach/library_processing.py b/src/peach/library_processing.py
index 0b1a0d2a7..c6632d059 100644
--- a/src/peach/library_processing.py
+++ b/src/peach/library_processing.py
@@ -7,10 +7,11 @@ import re
import sqlite3
import subprocess
import tempfile
+import threading
import time
import uuid
import xml.etree.ElementTree as ET
-from contextlib import closing
+from contextlib import closing, contextmanager
from contextvars import copy_context
from concurrent.futures import ThreadPoolExecutor, wait, FIRST_COMPLETED
from dataclasses import dataclass
@@ -44,7 +45,7 @@ from .scan import scan_location
from .fsutil import atomic_write_text
from . import artifact_rules, upgrade_rules
from .ffmpeg import FFmpegResolver
-from .jobs import DiskGuard, JobPolicyError
+from .jobs import DiskGuard, JobDeferred, JobPolicyError
from .media_probe import probe_unmeasured
from .media_sources import cost_for
from .media_processing_policy import metadata_only
@@ -740,6 +741,83 @@ def state_path(config):
return config.directory('state') / 'library-processing.json'
+#: 等着全库处理锁的前台处理有几轮。后台定向任务看到它不为零就让路。
+_waiting_foreground = 0
+_waiting_guard = threading.Lock()
+#: 前台处理等锁时多久再试一次。
+LOCK_RETRY_SECONDS = 0.5
+
+
+def _foreground_waiting() -> bool:
+ with _waiting_guard:
+ return _waiting_foreground > 0
+
+
+def _lock_free(config) -> bool:
+ try:
+ with FileLock(str(state_path(config)) + '.lock', timeout=0):
+ return True
+ except Timeout:
+ return False
+
+
+class ProcessingBusy(JobDeferred):
+ """全库处理锁被占着,或者有用户发起的处理在等它;后台定向任务回到队列。"""
+
+ def __init__(self, config):
+ super().__init__('等待扫描与采集完成后继续')
+ self.config = config
+
+ def ready(self) -> bool:
+ return not _foreground_waiting() and _lock_free(self.config)
+
+
+@contextmanager
+def _processing_lock(config, *, background, active):
+ """持有全库处理锁,返回本轮该用的 `active`。
+
+ 共享候选文件整份读改写,同一时刻只能有一轮处理在写。用户发起的处理(不带
+ `state_file`)等锁,并让在跑的后台定向任务在下一个检查点退出;后台任务碰上锁被占
+ 或有人在等,就抛 `ProcessingBusy` 回到队列,凭各自的进度文件续跑。
+ """
+ global _waiting_foreground
+ lock = FileLock(str(state_path(config)) + '.lock', timeout=0)
+ if background:
+ if _foreground_waiting():
+ raise ProcessingBusy(config)
+ try:
+ lock.acquire()
+ except Timeout:
+ raise ProcessingBusy(config) from None
+ try:
+ yield lambda: active() and not _foreground_waiting()
+ except InterruptedError:
+ if active() and _foreground_waiting():
+ raise ProcessingBusy(config) from None
+ raise
+ finally:
+ lock.release()
+ return
+ with _waiting_guard:
+ _waiting_foreground += 1
+ try:
+ while True:
+ try:
+ lock.acquire()
+ break
+ except Timeout:
+ if not active():
+ raise InterruptedError('处理任务已停止') from None
+ time.sleep(LOCK_RETRY_SECONDS)
+ try:
+ yield active
+ finally:
+ lock.release()
+ finally:
+ with _waiting_guard:
+ _waiting_foreground -= 1
+
+
def issues_path(config, job_id):
return config.directory('state') / f'library-processing-{job_id}.issues.jsonl'
@@ -2638,8 +2716,9 @@ def process_library(config, db_path, candidate_root, cover_root, *, location='co
flush_state(force=True)
report(dict(state))
from .update_tasks import activity
+ # 后台任务要让路时,`as active` 换上的那个会转为假;后面的检查点与闭包都读这一个。
with (activity(Path(db_path), 'library-processing'), traffic.bind(),
- FileLock(str(state_path(config)) + '.lock', timeout=0),
+ _processing_lock(config, background=state_file is not None, active=active) as active,
bind_control(control, identity, publish=control_report, active=active, db_path=Path(db_path),
mode=speed_mode)):
state = dict(job_id=identity, status='running',
diff --git a/src/peach/task_runs.py b/src/peach/task_runs.py
index 19f69ca10..b6707249b 100644
--- a/src/peach/task_runs.py
+++ b/src/peach/task_runs.py
@@ -284,14 +284,15 @@ class TaskRunStore:
return self._start(task_key, trigger=trigger, mutex_key=mutex_key,
conflict=conflict, total=total, label=label, pid=pid)
- def suspend(self, run_id: int, *, label: str = '等待安装完成后继续') -> None:
+ def suspend(self, run_id: int, *, label: str = '等待安装完成后继续',
+ paused_for_update: bool = True) -> None:
if not self.enabled:
return
with self.database.write_transaction(notify=False) as connection:
connection.execute(
"UPDATE task_run SET status='pending',pid=0,progress_label=?,result_summary=? "
"WHERE id=? AND status='running'",
- (label, json.dumps({'paused_for_update': True}), run_id))
+ (label, json.dumps({'paused_for_update': True} if paused_for_update else {}), run_id))
def _resume(self, run_id, task_key):
if run_id is None or not self.enabled:
diff --git a/tests/test_downloads_postprocess.py b/tests/test_downloads_postprocess.py
index 8f92b768b..9c10eaf10 100644
--- a/tests/test_downloads_postprocess.py
+++ b/tests/test_downloads_postprocess.py
@@ -10,6 +10,7 @@ from unittest.mock import Mock, patch
from peach import download_postprocess as post
from peach import ledger_backups
from peach.downloads import DownloadStore, DownloadTask, INGESTED, POSTPROCESS_PENDING
+from peach.jobs import JobDeferred
from peach.repository import LedgerDatabase
from support.ledger import fresh_ledger
@@ -150,6 +151,16 @@ class DownloadPostprocessTests(unittest.TestCase):
self.assertEqual(self.run_task(), {'task_id': self.task.id, 'already_complete': True})
self.contract.task_runs.enqueue_followups.assert_called_once()
+ def test_collection_that_steps_aside_stays_queued_instead_of_failed(self):
+ self.handle.run_id = 123
+ self.contract.task_runs = Mock()
+ with patch.object(post, '_collect', side_effect=JobDeferred('等待扫描与采集完成后继续')):
+ with self.assertRaises(JobDeferred):
+ self.run_task()
+ task = self.store.get(self.task.id)
+ self.assertEqual((task.postprocess_state, task.postprocess_detail), ('queued', '等待扫描与采集完成后继续'))
+ self.contract.task_runs.enqueue_followups.assert_not_called()
+
def test_metadata_or_organization_failure_never_dispatches_followups(self):
self.handle.run_id = 123
self.contract.task_runs = Mock()
diff --git a/tests/test_followups.py b/tests/test_followups.py
index 220c62cab..31b2bac53 100644
--- a/tests/test_followups.py
+++ b/tests/test_followups.py
@@ -16,7 +16,7 @@ from peach.avatar_followup import TASK_KEY as AVATAR_TASK_KEY
from peach.avatar_followup import followup_key, parse_key, plan
from peach.followups import FollowupRunner, FollowupType, lanes
from peach.jav_cover_fetch import NotFound
-from peach.jobs import BackgroundJob
+from peach.jobs import BackgroundJob, JobDeferred
from peach.library_processing import process_library
from peach.repository import LedgerDatabase
from peach.task_runs import MAX_FOLLOWUPS, TaskRunStore
@@ -210,6 +210,55 @@ class RunnerTests(LedgerTestCase):
self.assertEqual(
{row.status for row in self.store.children(parent.id)}, {"succeeded"})
+ def test_a_followup_that_steps_aside_waits_in_the_queue_and_runs_once_the_way_is_clear(self):
+ """给扫描与采集让路的后继回到队列,不判失败;路让回来以后同一条接着跑完。"""
+ clear = threading.Event()
+ attempts = []
+
+ class Blocked(JobDeferred):
+ def ready(self):
+ return clear.is_set()
+
+ def busy_until_clear(_contract, key, _handle):
+ attempts.append(key)
+ if not clear.is_set():
+ raise Blocked("等待扫描与采集完成后继续")
+ return {"outcome": key}
+
+ parent = self.parent()
+ with registered(echo_type("ledger-a", run=busy_until_clear)), \
+ mock.patch.object(followups_module, "DEFER_POLL_SECONDS", 0.01):
+ child = self.store.enqueue_followups(parent.id, [("ledger-a:1", "ledger-a", "甲")])["queued"][0]
+ runner = FollowupRunner(self.contract())
+ runner.wake()
+ deadline = time.monotonic() + 10
+ while not attempts and time.monotonic() < deadline:
+ time.sleep(0.01)
+ time.sleep(0.05)
+ waiting = self.store.get(child)
+ self.assertEqual(waiting.status, "pending")
+ self.assertEqual(waiting.progress_label, "等待扫描与采集完成后继续")
+ self.assertEqual(attempts, ["ledger-a:1"])
+ clear.set()
+ self.wait_until_settled(parent.id)
+ runner.stop(timeout=10)
+ self.assertEqual(self.store.get(child).status, "succeeded")
+ self.assertEqual(attempts, ["ledger-a:1", "ledger-a:1"])
+
+ def test_draining_leaves_a_followup_that_steps_aside_in_the_queue(self):
+ attempts = []
+
+ def busy(_contract, key, _handle):
+ attempts.append(key)
+ raise JobDeferred("等待扫描与采集完成后继续")
+
+ parent = self.parent()
+ with registered(echo_type("ledger-a", run=busy)):
+ child = self.store.enqueue_followups(parent.id, [("ledger-a:1", "ledger-a", "甲")])["queued"][0]
+ self.assertEqual(FollowupRunner(self.contract()).drain(), 1)
+ self.assertEqual(attempts, ["ledger-a:1"])
+ self.assertEqual(self.store.get(child).status, "pending")
+
def test_a_type_that_jumps_the_queue_is_claimed_before_earlier_ones_in_its_lane(self):
"""下载后处理这类人在等的后继,排在已经在排的后台批次前面;同一档里仍是先排先跑。"""
order = []
diff --git a/tests/test_metadata_library.py b/tests/test_metadata_library.py
index 5ccb7734f..bec1e6993 100644
--- a/tests/test_metadata_library.py
+++ b/tests/test_metadata_library.py
@@ -15,15 +15,15 @@ from contextlib import closing
from filelock import FileLock
from PIL import Image
-from peach import artifact_rules, upgrade_rules
+from peach import artifact_rules, library_processing, upgrade_rules
from peach.field_owners import auto_owner, owner_of, review_owner
from peach.genre_taxonomy import map_genres
from peach.jav_cover_fetch import NotFound
from peach.http import HttpResponse
from peach.library_nfo import read_nfo, sidecars, local_art
-from peach.library_processing import (STALL_AFTER_SECONDS, _candidate_identity, _save, decorate,
- issues_path, _merge_local_performer_profiles,
+from peach.library_processing import (STALL_AFTER_SECONDS, ProcessingBusy, _candidate_identity, _save,
+ decorate, issues_path, _merge_local_performer_profiles,
process_library, snapshot, state_path, _fields)
from peach.metadata_auto_apply import _apply_metadata_candidate
from peach.repository import LedgerDatabase
@@ -3585,3 +3585,122 @@ class LibraryPreviewCompletionTests(unittest.TestCase):
generated / 'posters', generated / 'avatars', generated / 'logos')
with Image.open(service.poster(1)) as image:
self.assertEqual(image.size, (480, 270))
+
+
+class ProcessingLockTests(unittest.TestCase):
+ """全库处理锁:用户发起的处理等锁,后台定向任务让路而不是报错。"""
+
+ def setUp(self):
+ temporary = tempfile.TemporaryDirectory()
+ self.addCleanup(temporary.cleanup)
+ self.root = Path(temporary.name).resolve()
+ media = self.root / 'media'
+ media.mkdir()
+ prefix = str(media) if os.name == 'nt' else 'R:\\Media'
+ self.config = PeachConfig(self.root, self.root / 'config.toml', present=True,
+ locations={'local': (prefix,)}, mounts={'local': (str(media),)})
+ self.enterContext(patch('peach.settings_file.active', return_value=self.config))
+ self.db = fresh_ledger(self.root)
+ self.database = LedgerDatabase(self.db)
+ with self.database.write_transaction() as connection:
+ for aid, code in ((1, 'ABW-358'), (2, 'ABW-359')):
+ video = media / f'{code}.mp4'
+ video.write_bytes(b'video')
+ connection.execute('INSERT INTO asset(id,location,path,name,medium,code) VALUES(?,?,?,?,?,?)',
+ (aid, 'local', str(PureWindowsPath(prefix) / video.name), video.name,
+ 'video', code))
+
+ def test_a_background_collection_steps_aside_while_the_processing_lock_is_held(self):
+ config, db, database = self.config, self.db, self.database
+ provider = stub_provider()
+ provider.query.return_value = {}
+ output = self.root / 'state/upgrade-repairs/1.json'
+ holder = FileLock(str(state_path(config)) + '.lock', timeout=0)
+ holder.acquire()
+ try:
+ with self.assertRaises(ProcessingBusy) as caught:
+ process_library(config, db, self.root / 'generated', self.root / 'covers', asset_ids=[1],
+ stage='collect', state_file=output, provider_factory=lambda: provider,
+ database=database)
+ self.assertFalse(caught.exception.ready())
+ finally:
+ holder.release()
+ self.assertTrue(caught.exception.ready())
+ provider.query.assert_not_called()
+
+ def test_a_user_run_waits_for_the_processing_lock_and_holds_background_work_back(self):
+ config, db, database = self.config, self.db, self.database
+ provider = stub_provider()
+ provider.query.return_value = {}
+ holder = FileLock(str(state_path(config)) + '.lock', timeout=0)
+ holder.acquire()
+ results, errors = [], []
+
+ def user_run():
+ try:
+ results.append(process_library(config, db, self.root / 'generated', self.root / 'covers',
+ provider_factory=lambda: provider, database=database))
+ except Exception as error:
+ errors.append(error)
+
+ with patch('peach.library_processing.LOCK_RETRY_SECONDS', 0.01), \
+ patch('peach.library_processing._enrich_finished_performers',
+ return_value={'aliases': 0, 'avatars': 0, 'conflicts': 0, 'failed': 0}):
+ thread = threading.Thread(target=user_run)
+ thread.start()
+ try:
+ deadline = time.monotonic() + 10
+ while not library_processing._foreground_waiting() and time.monotonic() < deadline:
+ time.sleep(0.01)
+ self.assertTrue(library_processing._foreground_waiting())
+ with self.assertRaises(ProcessingBusy):
+ process_library(config, db, self.root / 'generated', self.root / 'covers', asset_ids=[1],
+ stage='collect', state_file=self.root / 'state/upgrade-repairs/1.json',
+ provider_factory=lambda: provider, database=database)
+ finally:
+ holder.release()
+ thread.join(30)
+ self.assertEqual(errors, [])
+ self.assertNotEqual(results[0].get('outcome'), 'interrupted')
+ self.assertEqual(sorted({call.args[0] for call in provider.query.call_args_list}), ['ABW-358', 'ABW-359'])
+ self.assertFalse(library_processing._foreground_waiting())
+
+ def test_a_running_background_collection_yields_to_a_user_run_at_its_next_checkpoint(self):
+ config, db, database = self.config, self.db, self.database
+ entered, proceed = threading.Event(), threading.Event()
+
+ def query(*_args, **_kwargs):
+ if not entered.is_set():
+ entered.set()
+ proceed.wait(10)
+ return {}
+
+ provider = stub_provider()
+ provider.query.side_effect = query
+ background, user = [], []
+
+ def run(sink, **options):
+ try:
+ sink.append(process_library(config, db, self.root / 'generated', self.root / 'covers',
+ provider_factory=lambda: provider, database=database, **options))
+ except Exception as error:
+ sink.append(error)
+
+ with patch('peach.library_processing.LOCK_RETRY_SECONDS', 0.01), \
+ patch('peach.library_processing._enrich_finished_performers',
+ return_value={'aliases': 0, 'avatars': 0, 'conflicts': 0, 'failed': 0}):
+ first = threading.Thread(target=run, args=(background,), kwargs=dict(
+ asset_ids=[1, 2], stage='collect', state_file=self.root / 'state/upgrade-repairs/1.json'))
+ first.start()
+ self.assertTrue(entered.wait(10))
+ second = threading.Thread(target=run, args=(user,))
+ second.start()
+ deadline = time.monotonic() + 10
+ while not library_processing._foreground_waiting() and time.monotonic() < deadline:
+ time.sleep(0.01)
+ proceed.set()
+ first.join(30)
+ second.join(30)
+ self.assertIsInstance(background[0], ProcessingBusy)
+ self.assertIsInstance(user[0], dict)
+ self.assertNotEqual(user[0].get('outcome'), 'interrupted')