PRINTFILM: AI short-video marketing and AI short-drama creation platform
"""Tests for task runtime watchdog, heartbeat, and orphan recovery guards."""
from __future__ import annotations
import asyncio
from types import SimpleNamespace
from unittest.mock import AsyncMock, patch
import pytest
from app.services.tasks import poller as poller_mod
from app.services.tasks import runtime as runtime_mod
from app.services.tasks import scheduler as scheduler_mod
@pytest.fixture(autouse=True)
async def _reset_runtime_state(monkeypatch):
"""每个用例前后停掉调度/轮询/看门狗,避免泄漏协程。"""
monkeypatch.setattr(runtime_mod, "get_settings", lambda: SimpleNamespace(
task_runtime_enabled=True, task_runtime_watchdog_interval_sec=2,
))
monkeypatch.setattr(scheduler_mod, "_run_maintenance_operation", AsyncMock())
await runtime_mod.stop_task_runtime()
scheduler_mod._running_jobs.clear()
poller_mod._poll_inflight.clear()
yield
await runtime_mod.stop_task_runtime()
scheduler_mod._running_jobs.clear()
poller_mod._poll_inflight.clear()
@pytest.mark.asyncio
async def test_scheduler_status_and_heartbeat():
"""启动后调度状态为 running,心跳年龄应接近 0。"""
with (
patch.object(scheduler_mod, "recover_orphaned_tasks", new=AsyncMock(return_value=0)),
patch.object(scheduler_mod, "_tick", new=AsyncMock()),
):
await scheduler_mod.start_scheduler()
assert scheduler_mod.scheduler_status() == "running"
assert scheduler_mod.scheduler_tick_age_sec() < 5
assert not scheduler_mod.scheduler_tick_stale()
await scheduler_mod.stop_scheduler()
assert scheduler_mod.scheduler_status() == "stopped"
@pytest.mark.asyncio
async def test_watchdog_restarts_dead_scheduler():
"""调度循环退出后,看门狗应重新拉起。"""
with (
patch.object(scheduler_mod, "recover_orphaned_tasks", new=AsyncMock(return_value=0)),
patch.object(scheduler_mod, "_tick", new=AsyncMock()),
patch.object(poller_mod, "_select_and_poll_due", new=AsyncMock()),
patch.object(poller_mod, "_poll_ephemeral_deferred_tasks", new=AsyncMock()),
patch(
"app.services.tasks.runtime.get_settings",
return_value=SimpleNamespace(task_runtime_watchdog_interval_sec=0.05),
),
):
await runtime_mod.start_task_runtime()
first = scheduler_mod._scheduler_task
assert first is not None and not first.done()
# 人为结束调度循环(不走 stop,模拟异常退出)
scheduler_mod._intentionally_stopped = False
first.cancel()
try:
await first
except asyncio.CancelledError:
pass
for _ in range(40):
current = scheduler_mod._scheduler_task
if (
scheduler_mod.scheduler_status() == "running"
and current is not None
and current is not first
and not current.done()
):
break
await asyncio.sleep(0.05)
current = scheduler_mod._scheduler_task
assert scheduler_mod.scheduler_status() == "running"
assert current is not None and current is not first
assert runtime_mod.runtime_summary()["healthy"] is True
@pytest.mark.asyncio
async def test_recover_skips_local_alive_jobs():
"""本进程仍持有执行协程的 running 任务不应被孤儿恢复改写。"""
alive = asyncio.create_task(asyncio.sleep(60))
scheduler_mod._running_jobs[42] = alive
fake_task = SimpleNamespace(
id=42,
status="running",
cancel_requested=False,
current_step_key="step",
next_action_at=None,
lease_token="x",
lease_until=None,
)
class _FakeResult:
def scalars(self):
return self
def all(self):
return [fake_task]
class _FakeSession:
async def __aenter__(self):
return self
async def __aexit__(self, *args):
return False
async def execute(self, *_args, **_kwargs):
return _FakeResult()
async def commit(self):
return None
with (
patch.object(scheduler_mod, "AsyncSessionLocal", return_value=_FakeSession()),
patch.object(scheduler_mod, "reconcile_sequential_batches", new=AsyncMock()),
patch.object(scheduler_mod, "reconcile_stale_pending_tasks", new=AsyncMock()),
patch.object(scheduler_mod, "append_task_event", new=AsyncMock()),
patch(
"app.services.tasks.scheduler.get_settings",
return_value=SimpleNamespace(task_runtime_recover_grace_sec=1),
),
):
changed = await scheduler_mod.recover_orphaned_tasks()
assert changed == 0
assert fake_task.status == "running"
alive.cancel()
with pytest.raises(asyncio.CancelledError):
await alive
@pytest.mark.asyncio
async def test_runtime_summary_reports_components():
"""runtime_summary 应暴露 scheduler/poller/watchdog 与 healthy。"""
with (
patch.object(scheduler_mod, "recover_orphaned_tasks", new=AsyncMock(return_value=0)),
patch.object(scheduler_mod, "_tick", new=AsyncMock()),
patch.object(poller_mod, "_select_and_poll_due", new=AsyncMock()),
patch.object(poller_mod, "_poll_ephemeral_deferred_tasks", new=AsyncMock()),
):
await runtime_mod.start_task_runtime()
summary = runtime_mod.runtime_summary()
assert summary["scheduler"] == "running"
assert summary["poller"] == "running"
assert summary["watchdog"] == "running"
assert summary["healthy"] is True
assert "scheduler_tick_age_sec" in summary
@pytest.mark.asyncio
async def test_poller_not_stale_while_download_inflight():
"""拉 TokenFree 成片时心跳会停,不能被看门狗当成死循环掐断。"""
import time
poller_mod._intentionally_stopped = False
poller_mod._last_poll_mono = time.monotonic() - 120
poller_mod._poll_inflight.add(3244)
try:
assert poller_mod.poller_tick_stale() is False
poller_mod._last_poll_mono = time.monotonic() - 999
assert poller_mod.poller_tick_stale() is True
finally:
poller_mod._poll_inflight.clear()
poller_mod._intentionally_stopped = True