Self-improving AI Employees for everyone
Python ai-agentsai-employeeai-employeesassistantautonomous-agentsco-employeescompanygrok-botinstinctloop-agentsmuseown-your-datapersonalself-hosted
"""Worker self-restart in TemporalWorkerManager / TemporalWorkerPool.
The Temporal worker shuts down on a transient poll failure rather than
auto-retrying, and the run task is detached (main.py's startup retry loop
has already returned). The SDK ``Worker`` is single-use — a second
``run()`` on the same instance raises ``RuntimeError("Already started")``
— so every restart attempt must REBUILD a fresh worker. The previous
restart loop re-ran the same instance, which meant one crash left the
task queue permanently unpolled while the loop logged "restarting"
forever; the previous version of this test masked that by mocking a
re-runnable worker.
Cancellation (from ``stop()``) always wins so shutdown is never delayed
by a restart. Backoff knobs come from Settings (env-driven); the conftest
stubs Settings, so we patch it to a SimpleNamespace with tiny real
backoffs.
"""
from __future__ import annotations
import asyncio
from types import SimpleNamespace
from unittest.mock import MagicMock, patch
import pytest
from services.temporal.worker import TemporalWorkerManager, TemporalWorkerPool
def _patch_backoff():
return patch(
"core.config.Settings",
side_effect=lambda: SimpleNamespace(
temporal_worker_restart_backoff_seconds=0.001,
temporal_worker_restart_backoff_max_seconds=0.002,
),
)
class _SingleUseWorker:
"""Mimics the SDK contract: a second run() on one instance raises."""
def __init__(self, run_impl):
self._run_impl = run_impl
self._started = False
async def run(self):
if self._started:
raise RuntimeError("Already started")
self._started = True
await self._run_impl()
def _manager_with_builds(run_impls) -> tuple[TemporalWorkerManager, dict]:
"""Manager whose _build_worker yields single-use workers in sequence."""
mgr = TemporalWorkerManager(client=MagicMock(), task_queue="q")
state = {"builds": 0, "runs": 0}
impls = list(run_impls)
async def _tracked(impl):
state["runs"] += 1
await impl()
async def fake_build_worker():
impl = impls[min(state["builds"], len(impls) - 1)]
state["builds"] += 1
return _SingleUseWorker(lambda impl=impl: _tracked(impl))
mgr._build_worker = fake_build_worker # type: ignore[method-assign]
return mgr, state
async def test_worker_restarts_with_fresh_instance_after_transient_crash():
async def crash():
raise RuntimeError("transient poll failure")
async def healthy():
await asyncio.Event().wait() # healthy run blocks
mgr, state = _manager_with_builds([crash, healthy])
mgr._worker = await mgr._build_worker()
with _patch_backoff():
task = asyncio.create_task(mgr._run_worker())
await asyncio.sleep(0.05) # allow crash + rebuild + restart
# First build happened before the loop; the restart rebuilt once.
assert state["builds"] == 2
assert state["runs"] == 2
task.cancel()
with pytest.raises(asyncio.CancelledError):
await task
async def test_restart_never_reuses_the_single_use_instance():
"""Locks the regression the old mock hid: re-running the same SDK
worker raises "Already started" instantly. The restart loop must
rebuild before every retry, so the crashed instance is run once."""
crashed = _SingleUseWorker(run_impl=None)
async def crash():
raise RuntimeError("transient poll failure")
async def healthy():
await asyncio.Event().wait()
mgr, state = _manager_with_builds([healthy])
crashed._run_impl = crash
mgr._worker = crashed
with _patch_backoff():
task = asyncio.create_task(mgr._run_worker())
await asyncio.sleep(0.05)
# The crashed single-use instance was replaced, not re-run.
assert mgr._worker is not crashed
assert state["builds"] == 1
task.cancel()
with pytest.raises(asyncio.CancelledError):
await task
async def test_cancellation_does_not_restart():
calls = {"n": 0}
async def healthy():
calls["n"] += 1
await asyncio.Event().wait() # healthy run blocks until cancelled
mgr, state = _manager_with_builds([healthy])
mgr._worker = await mgr._build_worker()
with _patch_backoff():
task = asyncio.create_task(mgr._run_worker())
await asyncio.sleep(0.02)
assert calls["n"] == 1
task.cancel()
with pytest.raises(asyncio.CancelledError):
await task
assert calls["n"] == 1 # cancellation wins; not restarted
assert state["builds"] == 1 # and nothing was rebuilt
async def test_pool_queue_worker_restarts_with_fresh_instance():
"""Pool workers previously ran as bare worker.run() tasks with no
restart machinery — a crash silently killed the only worker for the
queue and its activities pended forever."""
pool = TemporalWorkerPool(client=MagicMock(), queues=["code-exec"])
state = {"builds": 0}
async def crash():
raise RuntimeError("poll failure")
async def healthy():
await asyncio.Event().wait()
def fake_build(queue):
impl = crash if state["builds"] == 0 else healthy
state["builds"] += 1
return _SingleUseWorker(impl)
pool._build_queue_worker = fake_build # type: ignore[method-assign]
with _patch_backoff():
first = pool._build_queue_worker("code-exec")
task = asyncio.create_task(pool._run_queue_worker("code-exec", first))
await asyncio.sleep(0.05)
assert state["builds"] == 2 # crash triggered one rebuild
task.cancel()
with pytest.raises(asyncio.CancelledError):
await task
async def test_pool_queue_worker_stops_when_queue_has_no_activities():
pool = TemporalWorkerPool(client=MagicMock(), queues=["code-exec"])
async def crash():
raise RuntimeError("poll failure")
pool._build_queue_worker = lambda queue: None # type: ignore[method-assign]
with _patch_backoff():
task = asyncio.create_task(
pool._run_queue_worker("code-exec", _SingleUseWorker(crash))
)
# Crash, then rebuild returns None -> the supervisor task ends.
await asyncio.wait_for(task, timeout=1.0)