Build and run agents you can see, understand and trust.
Python agentchatbotlarge-language-modelsllmllm-agentmcpmulti-agentmulti-modalreact-agentrealtime-agentvoice-agentvoice-assistant
# -*- coding: utf-8 -*-
# pylint: disable=protected-access
"""Regression tests for shared background task lifecycle and endpoints."""
import asyncio
import json
from dataclasses import fields, is_dataclass, replace
from datetime import datetime, timezone
from unittest.async_case import IsolatedAsyncioTestCase
from unittest.mock import AsyncMock, MagicMock, patch
import fakeredis.aioredis
import httpx
from agentscope.app import create_app
from agentscope.app._manager import (
BackgroundTaskManager,
CancelDispatcher,
ChatRunRegistry,
TaskStatus,
)
from agentscope.app._manager._background_task_manager import (
_MAX_RETAINED_TASKS,
BackgroundTask,
BackgroundTaskRecord,
)
from agentscope.app.message_bus import (
InMemoryMessageBus,
MessageBus,
MessageBusKeys,
RedisMessageBus,
)
from agentscope.app.middleware._tool_offload_middleware import (
_make_summary,
)
from agentscope.app._router._schema._background_task import to_info
_MANAGER_MODULE = BackgroundTaskManager.__module__
class TestBackgroundTaskCompatibility(IsolatedAsyncioTestCase):
"""Preserve the task dataclass API that existed before this feature."""
async def test_legacy_construction_and_dataclass_operations(self) -> None:
"""Old positional and keyword construction keep their field order."""
task = asyncio.create_task(asyncio.sleep(0))
await task
positional = BackgroundTask(task, "s1", "a1", "alice", "Tool", "id")
keyword = BackgroundTask(
asyncio_task=task,
session_id="s1",
agent_id="a1",
user_id="alice",
tool_name="Tool",
id="id",
started_at=positional.started_at,
)
expected = {
"asyncio_task": task,
"session_id": "s1",
"agent_id": "a1",
"user_id": "alice",
"tool_name": "Tool",
"id": "id",
"summary": "",
"started_at": positional.started_at,
"completed_at": None,
"status": TaskStatus.RUNNING,
"error_summary": None,
"execution_started_at": None,
}
self.assertTrue(is_dataclass(positional))
self.assertEqual(
[item.name for item in fields(BackgroundTask)],
list(expected),
)
self.assertEqual(vars(positional), expected)
self.assertEqual(positional, keyword)
self.assertEqual(
vars(replace(positional, tool_name="Other")),
{**expected, "tool_name": "Other"},
)
self.assertEqual(vars(positional), expected)
generated = BackgroundTask(task, "s1", "a1", "alice", "Tool")
self.assertIsInstance(generated.id, str)
self.assertNotEqual(generated.id, positional.id)
async def test_shared_record_is_an_independent_snapshot(self) -> None:
"""Serialization excludes handles and cannot mutate the local task."""
task = asyncio.create_task(asyncio.sleep(0))
await task
local = BackgroundTask(
task,
"s1",
"a1",
"alice",
"Tool",
"id",
summary="Tool(x=1)",
started_at=123.0,
)
record = BackgroundTaskRecord.model_validate(local)
expected = {
"id": "id",
"session_id": "s1",
"agent_id": "a1",
"user_id": "alice",
"tool_name": "Tool",
"summary": "Tool(x=1)",
"started_at": 123.0,
"completed_at": None,
"status": "running",
"error_summary": None,
"execution_started_at": None,
}
self.assertEqual(json.loads(record.model_dump_json()), expected)
local.status = TaskStatus.COMPLETED
local.completed_at = 124.0
self.assertEqual(json.loads(record.model_dump_json()), expected)
record.summary = "Changed"
self.assertEqual(local.summary, "Tool(x=1)")
async def test_api_preserves_execution_and_background_times(self) -> None:
"""Total time includes the foreground interval for every outcome."""
for status in TaskStatus:
with self.subTest(status=status):
completed_at = None if status == TaskStatus.RUNNING else 130
record = BackgroundTaskRecord(
id="id",
session_id="s1",
agent_id="a1",
user_id="alice",
tool_name="Bash",
summary="Bash(command=sleep 30)",
started_at=110,
execution_started_at=100,
completed_at=completed_at,
status=status,
error_summary=None,
)
self.assertEqual(
to_info(record).model_dump(mode="json"),
{
"task_id": "id",
"session_id": "s1",
"agent_id": "a1",
"tool_name": "Bash",
"summary": "Bash(command=sleep 30)",
"status": status.value,
"started_at": "1970-01-01T00:01:50Z",
"execution_started_at": "1970-01-01T00:01:40Z",
"completed_at": (
"1970-01-01T00:02:10Z"
if completed_at is not None
else None
),
"error_summary": None,
},
)
class TestBackgroundTaskRecords( # pylint: disable=too-many-public-methods
IsolatedAsyncioTestCase,
):
"""Exercise two workers sharing task metadata and cancellation signals."""
def _make_bus(self) -> MessageBus:
"""Provide the shared backend for both managers."""
return InMemoryMessageBus()
async def asyncSetUp(self) -> None:
"""Keep the bus alive until both managers have finished cleanup."""
self.bus = await self.enterAsyncContext(self._make_bus())
self.owner = await self.enterAsyncContext(
BackgroundTaskManager(self.bus),
)
self.reader = await self.enterAsyncContext(
BackgroundTaskManager(self.bus),
)
async def _register(
self,
manager: BackgroundTaskManager,
user_id: str = "alice",
session_id: str = "s1",
execution_started_at: float | None = None,
) -> tuple[asyncio.Event, asyncio.Task, str]:
"""Register a tool that finishes only when explicitly released."""
gate = asyncio.Event()
task = asyncio.create_task(gate.wait())
task_id = await manager.register_task(
task,
session_id,
"a1",
user_id,
"Tool",
summary="Tool(x=1)",
execution_started_at=execution_started_at,
)
return gate, task, task_id
async def _settle(self, *tasks: asyncio.Task) -> None:
"""Wait for execution and all metadata writes without timing sleeps."""
await asyncio.gather(*tasks, return_exceptions=True)
for manager in (self.owner, self.reader):
while manager._completion_tasks:
await asyncio.gather(*manager._completion_tasks)
# Done callbacks may still be queued after gather returns.
await asyncio.sleep(0)
async def _renew_once(self) -> None:
"""Run one heartbeat pass without waiting for the lease interval."""
self.owner._renewal_task.cancel()
await asyncio.gather(
self.owner._renewal_task,
return_exceptions=True,
)
with patch(
f"{_MANAGER_MODULE}.asyncio",
wraps=asyncio,
sleep=AsyncMock(side_effect=[None, asyncio.CancelledError]),
):
try:
await self.owner._renew_running_records()
except asyncio.CancelledError:
pass
async def test_settle_waits_for_pending_done_callbacks(self) -> None:
"""Completed tasks must be discarded before the helper returns."""
completed = asyncio.create_task(asyncio.sleep(0))
await completed
self.owner._completion_tasks.add(completed)
completed.add_done_callback(self.owner._completion_tasks.discard)
await self._settle()
self.assertEqual(self.owner._completion_tasks, set())
self.assertEqual(self.reader._completion_tasks, set())
async def test_execution_start_survives_renewal_and_completion(
self,
) -> None:
"""Preserve the supplied start through shared lifecycle writes."""
gate, task, task_id = await self._register(
self.owner,
execution_started_at=123.0,
)
expected = BackgroundTaskRecord.model_validate(
self.owner.tasks[task_id],
).model_dump()
expected["execution_started_at"] = 123.0
await self._renew_once()
record = await self.reader.get_task(task_id, user_id="alice")
self.assertEqual(record.model_dump(), expected)
gate.set()
await self._settle(task)
record = await self.reader.get_task(task_id, user_id="alice")
expected.update(
status=TaskStatus.COMPLETED,
completed_at=record.completed_at,
)
self.assertEqual(record.model_dump(), expected)
async def test_old_snapshot_has_unknown_execution_start(self) -> None:
"""Old snapshots remain readable without inventing a total duration."""
gate, task, task_id = await self._register(self.owner)
namespace = MessageBusKeys.bg_task_records("alice")
payload = json.loads(
await self.bus.registry_get(namespace, task_id),
)
del payload["execution_started_at"]
await self.bus.registry_set(
namespace,
task_id,
json.dumps(payload),
ttl_secs=MessageBusKeys.BG_TASKS_TTL_SECS,
)
expected = BackgroundTaskRecord.model_validate(
self.owner.tasks[task_id],
).model_dump()
expected["execution_started_at"] = None
record = await self.reader.get_task(task_id, user_id="alice")
self.assertEqual(record.model_dump(), expected)
self.assertIsNone(to_info(record).execution_started_at)
gate.set()
await self._settle(task)
async def test_shared_lifecycle_and_isolation(self) -> None:
"""Running and completed records are identical on either worker."""
gate, task, task_id = await self._register(self.owner)
expected = BackgroundTaskRecord.model_validate(
self.owner.tasks[task_id],
).model_dump()
record = await self.reader.get_task(task_id, user_id="alice")
self.assertEqual(record.model_dump(), expected)
self.assertEqual(
[
r.model_dump()
for r in await self.reader.list_tasks(
user_id="alice",
session_id="s1",
)
],
[expected],
)
self.assertEqual(
await self.reader.list_tasks(user_id="alice", session_id="s2"),
[],
)
self.assertEqual(await self.reader.list_tasks(user_id="bob"), [])
self.assertIsNone(
await self.reader.get_task(task_id, user_id="bob"),
)
self.assertIsNone(
await self.reader.get_task("missing", user_id="alice"),
)
gate.set()
await self._settle(task)
record = await self.reader.get_task(task_id, user_id="alice")
self.assertIsNotNone(record.completed_at)
expected.update(
status=TaskStatus.COMPLETED,
completed_at=record.completed_at,
)
self.assertEqual(record.model_dump(), expected)
self.assertEqual(self.owner.tasks, {})
self.assertEqual(
await self.bus.registry_getall(MessageBusKeys.bg_tasks("s1")),
{},
)
async def test_exception_records_failure(self) -> None:
"""An exception preserves its bounded error summary on any worker."""
gate = asyncio.Event()
async def fail() -> None:
await gate.wait()
raise ValueError("x" * 600)
task = asyncio.create_task(fail())
task_id = await self.owner.register_task(task, "s1", "a1", "alice")
expected = BackgroundTaskRecord.model_validate(
self.owner.tasks[task_id],
).model_dump()
gate.set()
await self._settle(task)
record = await self.reader.get_task(task_id, user_id="alice")
expected.update(
status=TaskStatus.FAILED,
error_summary="x" * 512,
completed_at=record.completed_at,
)
self.assertIsNotNone(record.completed_at)
self.assertEqual(record.model_dump(), expected)
async def test_local_and_remote_toolstop(self) -> None:
"""Both cancellation paths retain status and remove live entries."""
registry = await self.enterAsyncContext(ChatRunRegistry())
await self.enterAsyncContext(
CancelDispatcher(self.bus, registry, self.owner),
)
for caller in (self.owner, self.reader):
with self.subTest(local=caller is self.owner):
_, task, task_id = await self._register(self.owner)
expected = BackgroundTaskRecord.model_validate(
self.owner.tasks[task_id],
).model_dump()
stop = (await caller.list_tools("s1"))[0]
await stop(task_id=task_id)
await asyncio.wait_for(self._settle(task), timeout=5)
record = await self.reader.get_task(
task_id,
user_id="alice",
)
self.assertTrue(task.cancelled())
self.assertIsNotNone(record.completed_at)
expected.update(
status=TaskStatus.CANCELLED,
completed_at=record.completed_at,
)
self.assertEqual(record.model_dump(), expected)
self.assertEqual(self.owner.tasks, {})
self.assertEqual(
await self.bus.registry_getall(
MessageBusKeys.bg_tasks("s1"),
),
{},
)
self.assertFalse(self.owner.cancel_task(task_id))
async def test_shutdown_waits_for_shared_cleanup(self) -> None:
"""Shutdown returns only after cancelled states and cleanup persist."""
_, task, task_id = await self._register(self.owner)
expected = BackgroundTaskRecord.model_validate(
self.owner.tasks[task_id],
).model_dump()
await self.owner.__aexit__()
record = await self.reader.get_task(task_id, user_id="alice")
expected.update(
status=TaskStatus.CANCELLED,
completed_at=record.completed_at,
)
self.assertTrue(task.cancelled())
self.assertEqual(record.model_dump(), expected)
self.assertEqual(self.owner.tasks, {})
self.assertEqual(self.owner._completion_tasks, set())
self.assertEqual(
await self.bus.registry_getall(MessageBusKeys.bg_tasks("s1")),
{},
)
async def test_retention_is_shared_and_scoped_per_user(self) -> None:
"""Concurrent completions evict oldest outcomes, never running work."""
_, _, running_id = await self._register(self.owner)
running = BackgroundTaskRecord.model_validate(
self.owner.tasks[running_id],
).model_dump()
bob_gate, bob_task, bob_id = await self._register(
self.owner,
user_id="bob",
)
work = []
records = []
for index in range(_MAX_RETAINED_TASKS + 5):
manager = (self.owner, self.reader)[index % 2]
gate, task, task_id = await self._register(manager)
records.append(manager.tasks[task_id])
work.append((gate, task))
for gate, _ in work:
gate.set()
bob_gate.set()
await self._settle(bob_task, *(task for _, task in work))
retained = sorted(
records,
key=lambda record: (record.completed_at, record.id),
)[-_MAX_RETAINED_TASKS:]
expected = [running] + [
BackgroundTaskRecord.model_validate(record).model_dump()
for record in retained
]
expected.sort(key=lambda record: (record["started_at"], record["id"]))
self.assertEqual(
[
record.model_dump()
for record in await self.reader.list_tasks(
user_id="alice",
)
],
expected,
)
bob_record = await self.reader.get_task(bob_id, user_id="bob")
self.assertEqual(
[
record.model_dump()
for record in await self.reader.list_tasks(
user_id="bob",
)
],
[bob_record.model_dump()],
)
self.assertEqual(bob_record.status, TaskStatus.COMPLETED)
self.assertEqual(set(self.owner.tasks), {running_id})
self.assertEqual(self.reader.tasks, {})
async def test_failed_initial_write_still_cleans_up(self) -> None:
"""A partial registration cannot leave an untracked running task."""
original_set = self.bus.registry_set
async def fail_running_record(
namespace: str,
field: str,
value: str,
**kwargs: int,
) -> None:
if (
namespace == MessageBusKeys.bg_task_records("alice")
and json.loads(value)["status"] == "running"
):
raise ConnectionError("record write failed")
await original_set(namespace, field, value, **kwargs)
task = asyncio.create_task(asyncio.Event().wait())
with patch.object(
self.bus,
"registry_set",
side_effect=fail_running_record,
):
with self.assertRaisesRegex(ConnectionError, "record write"):
await self.owner.register_task(task, "s1", "a1", "alice")
await self._settle(task)
self.assertTrue(task.cancelled())
self.assertEqual(self.owner.tasks, {})
self.assertEqual(
await self.bus.registry_getall(MessageBusKeys.bg_tasks("s1")),
{},
)
async def test_initial_write_timeout_cleans_up_all_outcomes(self) -> None:
"""Either stalled initial write releases registration and handles."""
original_set = self.bus.registry_set
for namespace in (
MessageBusKeys.bg_tasks("s1"),
MessageBusKeys.bg_task_records("alice"),
):
for completed in (False, True):
with self.subTest(namespace=namespace, completed=completed):
started = asyncio.Event()
cancelled = asyncio.Event()
gate = asyncio.Event()
task = asyncio.create_task(gate.wait())
async def stall_initial_write(
ns: str,
field: str,
value: str,
*,
ttl_secs: int | None = None,
) -> None:
# Each subtest finishes this write before advancing.
# pylint: disable=cell-var-from-loop
payload = json.loads(value)
if (
ns == namespace
and payload.get("status", "running") == "running"
):
# The server may commit before the reply stalls.
await original_set(
ns,
field,
value,
ttl_secs=ttl_secs,
)
started.set()
try:
await asyncio.Event().wait()
finally:
cancelled.set()
await original_set(
ns,
field,
value,
ttl_secs=ttl_secs,
)
with (
patch.object(
self.bus,
"registry_set",
side_effect=stall_initial_write,
),
patch(f"{_MANAGER_MODULE}._IO_TIMEOUT_SECS", 0.1),
):
registration = asyncio.create_task(
self.owner.register_task(
task,
"s1",
"a1",
"alice",
),
)
try:
await asyncio.wait_for(started.wait(), timeout=1)
(task_id,) = self.owner.tasks
expected = BackgroundTaskRecord.model_validate(
self.owner.tasks[task_id],
).model_dump()
if completed:
gate.set()
await task
done, _ = await asyncio.wait(
{registration},
timeout=0.5,
)
self.assertEqual(done, {registration})
with self.assertRaises(TimeoutError):
await registration
self.assertTrue(cancelled.is_set())
await asyncio.wait_for(self._settle(task), 1)
finally:
registration.cancel()
await asyncio.gather(
registration,
return_exceptions=True,
)
self.assertEqual(self.owner.tasks, {})
self.assertEqual(task.cancelled(), not completed)
record = await self.reader.get_task(
task_id,
user_id="alice",
)
expected.update(
status=TaskStatus.COMPLETED
if completed
else TaskStatus.CANCELLED,
completed_at=record.completed_at,
)
self.assertEqual(record.model_dump(), expected)
self.assertEqual(
await self.bus.registry_getall(
MessageBusKeys.bg_tasks("s1"),
),
{},
)
async def test_failed_outcome_write_recovers_shared_status(self) -> None:
"""A failed write remains pending and is retried after recovery."""
gate, task, task_id = await self._register(self.owner)
expected = BackgroundTaskRecord.model_validate(
self.owner.tasks[task_id],
).model_dump()
failed = asyncio.Event()
async def unavailable(_record: BackgroundTaskRecord) -> None:
failed.set()
raise ConnectionError("unavailable")
with patch.object(
self.owner,
"_store_record",
side_effect=unavailable,
):
gate.set()
await asyncio.wait_for(failed.wait(), timeout=1)
self.assertEqual(self.owner.tasks, {})
self.assertTrue(self.owner._completion_tasks)
record = await self.reader.get_task(task_id, user_id="alice")
self.assertEqual(record.model_dump(), expected)
await asyncio.wait_for(self._settle(task), timeout=2)
record = await self.reader.get_task(task_id, user_id="alice")
self.assertIsNotNone(record.completed_at)
expected.update(
status=TaskStatus.COMPLETED,
completed_at=record.completed_at,
)
self.assertEqual(record.model_dump(), expected)
self.assertEqual(
await self.bus.registry_getall(MessageBusKeys.bg_tasks("s1")),
{},
)
self.assertEqual(self.owner._completion_tasks, set())
async def test_failed_shutdown_record_expires_despite_new_tasks(
self,
) -> None:
"""A lost outcome cannot remain running through user-hash renewal."""
gate, task, task_id = await self._register(self.owner)
namespace = MessageBusKeys.bg_task_records("alice")
value = await self.bus.registry_get(namespace, task_id)
expires_at = json.loads(value)["_expires_at"]
failed = asyncio.Event()
async def unavailable(_record: BackgroundTaskRecord) -> None:
failed.set()
raise ConnectionError("unavailable")
with (
patch.object(self.owner, "_store_record", side_effect=unavailable),
patch(f"{_MANAGER_MODULE}._SHUTDOWN_TIMEOUT_SECS", 0.1),
):
gate.set()
await asyncio.wait_for(failed.wait(), timeout=1)
await asyncio.wait_for(self.owner.__aexit__(), timeout=0.5)
self.assertTrue(task.done())
self.assertEqual(self.owner._completion_tasks, set())
self.assertEqual(
await self.bus.registry_get(namespace, task_id),
value,
)
with patch(
f"{_MANAGER_MODULE}.time.time",
return_value=expires_at + 1,
):
_, _, new_id = await self._register(self.reader)
expected = BackgroundTaskRecord.model_validate(
self.reader.tasks[new_id],
).model_dump()
self.assertIsNone(
await self.reader.get_task(task_id, user_id="alice"),
)
self.assertEqual(
[
record.model_dump()
for record in await self.reader.list_tasks(
user_id="alice",
)
],
[expected],
)
# Querying removes the expired field despite hash TTL renewal.
self.assertIsNone(
await self.bus.registry_get(namespace, task_id),
)
async def test_lists_remove_expired_records_as_user_activity_continues(
self,
) -> None:
"""Repeated user activity cannot accumulate invisible running rows."""
namespace = MessageBusKeys.bg_task_records("alice")
expected = {}
for batch in range(3):
_, _, task_id = await self._register(self.owner)
value = await self.bus.registry_get(namespace, task_id)
expected[task_id] = value
for index in range(5):
stale_id = f"stale-{batch}-{index}"
payload = json.loads(value)
payload.update(id=stale_id, _expires_at=0)
await self.bus.registry_set(
namespace,
stale_id,
json.dumps(payload),
ttl_secs=MessageBusKeys.BG_TASKS_TTL_SECS,
)
records = await self.reader.list_tasks(user_id="alice")
self.assertEqual(
{record.id: record.model_dump() for record in records},
{
task_id: BackgroundTaskRecord.model_validate_json(
value,
).model_dump()
for task_id, value in expected.items()
},
)
self.assertEqual(
await self.bus.registry_getall(namespace),
expected,
)
async def test_completion_evicts_expired_running_and_terminal_records(
self,
) -> None:
"""Finalization physically clears expired records of every status."""
gate, task, task_id = await self._register(self.owner)
namespace = MessageBusKeys.bg_task_records("alice")
value = await self.bus.registry_get(namespace, task_id)
for status in TaskStatus:
stale_id = f"expired-{status.value}"
payload = json.loads(value)
payload.update(
id=stale_id,
status=status.value,
_expires_at=0,
completed_at=None if status == TaskStatus.RUNNING else 1.0,
)
await self.bus.registry_set(
namespace,
stale_id,
json.dumps(payload),
)
gate.set()
await self._settle(task)
self.assertEqual(
await self.bus.registry_getall(namespace),
{task_id: await self.bus.registry_get(namespace, task_id)},
)
async def test_expiry_cleanup_preserves_concurrent_updates(self) -> None:
"""Queries and eviction cannot delete renewed or completed records."""
namespace = MessageBusKeys.bg_task_records("alice")
original_delete = self.bus.registry_del_if
original_set = self.bus.registry_set
for operation in ("list", "detail", "eviction"):
for status in (TaskStatus.RUNNING, TaskStatus.COMPLETED):
with self.subTest(operation=operation, status=status):
task_id = f"race-{operation}-{status.value}"
payload = {
"id": task_id,
"session_id": "s1",
"agent_id": "a1",
"user_id": "alice",
"tool_name": "Tool",
"summary": "Tool(x=1)",
"started_at": 0,
"completed_at": None,
"status": "running",
"error_summary": None,
"_expires_at": 0,
}
await original_set(namespace, task_id, json.dumps(payload))
payload.update(
status=status.value,
completed_at=1.0
if status == TaskStatus.COMPLETED
else None,
_expires_at=32503680000,
)
updated = json.dumps(payload)
async def concurrent_update(
ns: str,
field: str,
*,
expected: str,
new_value: str = updated,
) -> bool:
await original_set(ns, field, new_value)
return await original_delete(
ns,
field,
expected=expected,
)
with patch.object(
self.bus,
"registry_del_if",
side_effect=concurrent_update,
):
if operation == "list":
await self.reader.list_tasks(user_id="alice")
elif operation == "detail":
await self.reader.get_task(
task_id,
user_id="alice",
)
else:
await self.reader._evict_completed("alice")
self.assertEqual(
await self.bus.registry_get(namespace, task_id),
updated,
)
record = await self.reader.get_task(
task_id,
user_id="alice",
)
self.assertEqual(
record.model_dump(),
BackgroundTaskRecord.model_validate(
payload,
).model_dump(),
)
async def test_retention_lock_does_not_block_outcome_or_shutdown(
self,
) -> None:
"""A held retention lock cannot delay state writes or exhaust exit."""
gate, task, task_id = await self._register(self.owner)
expected = BackgroundTaskRecord.model_validate(
self.owner.tasks[task_id],
).model_dump()
cleaned = asyncio.Event()
original_del = self.bus.registry_del
async def observe_cleanup(namespace: str, field: str) -> None:
await original_del(namespace, field)
if namespace == MessageBusKeys.bg_tasks("s1"):
cleaned.set()
async with self.bus.acquire_lock(
MessageBusKeys.bg_task_records_lock("alice"),
ttl_secs=600,
):
with (
patch.object(
self.bus,
"registry_del",
side_effect=observe_cleanup,
),
patch(f"{_MANAGER_MODULE}._SHUTDOWN_TIMEOUT_SECS", 0.1),
patch(f"{_MANAGER_MODULE}._IO_TIMEOUT_SECS", 0.02),
):
gate.set()
await asyncio.wait_for(cleaned.wait(), timeout=1)
self.assertTrue(task.done())
self.assertEqual(self.owner.tasks, {})
record = await self.reader.get_task(task_id, user_id="alice")
expected.update(
status=TaskStatus.COMPLETED,
completed_at=record.completed_at,
)
self.assertEqual(record.model_dump(), expected)
await asyncio.wait_for(self.owner.__aexit__(), timeout=0.5)
self.assertEqual(self.owner._completion_tasks, set())
async def test_stalled_outcome_io_times_out_and_retries(self) -> None:
"""A nonresponsive write gets a deadline and can recover on retry."""
gate, task, task_id = await self._register(self.owner)
stalled = asyncio.Event()
original_store = self.owner._store_record
async def stall_once(record: BackgroundTaskRecord) -> None:
if not stalled.is_set():
stalled.set()
await asyncio.Event().wait()
await original_store(record)
with (
patch.object(self.owner, "_store_record", side_effect=stall_once),
patch(f"{_MANAGER_MODULE}._RETRY_DELAY_SECS", 0),
):
gate.set()
await asyncio.wait_for(stalled.wait(), timeout=1)
await asyncio.wait_for(self._settle(task), timeout=5)
record = await self.reader.get_task(task_id, user_id="alice")
self.assertEqual(record.status, TaskStatus.COMPLETED)
self.assertEqual(
await self.bus.registry_getall(MessageBusKeys.bg_tasks("s1")),
{},
)
async def test_shutdown_cancels_stalled_cleanup_io(self) -> None:
"""Shutdown is bounded even if a registry operation never responds."""
gate, task, task_id = await self._register(self.owner)
blocked = asyncio.Event()
cancelled = asyncio.Event()
async def stall_delete(_namespace: str, _field: str) -> None:
blocked.set()
try:
await asyncio.Event().wait()
finally:
cancelled.set()
with (
patch.object(self.bus, "registry_del", side_effect=stall_delete),
patch(f"{_MANAGER_MODULE}._SHUTDOWN_TIMEOUT_SECS", 0.1),
):
gate.set()
await asyncio.wait_for(blocked.wait(), timeout=1)
self.assertTrue(task.done())
await asyncio.wait_for(self.owner.__aexit__(), timeout=0.5)
self.assertTrue(cancelled.is_set())
self.assertEqual(self.owner._completion_tasks, set())
record = await self.reader.get_task(task_id, user_id="alice")
self.assertEqual(record.status, TaskStatus.COMPLETED)
async def test_running_task_renews_its_own_lease(self) -> None:
"""Long-running tasks stay visible beyond their initial lease."""
renewed = asyncio.Event()
original_set_if = self.bus.registry_set_if
async def observe_renewal(
namespace: str,
field: str,
value: str,
*,
expected: str,
ttl_secs: int | None = None,
) -> bool:
result = await original_set_if(
namespace,
field,
value,
expected=expected,
ttl_secs=ttl_secs,
)
renewed.set()
return result
with patch(f"{_MANAGER_MODULE}._RECORD_LEASE_SECS", 0.04):
gate, task, task_id = await self._register(self.owner)
value = await self.bus.registry_get(
MessageBusKeys.bg_task_records("alice"),
task_id,
)
future_time = json.loads(value)["_expires_at"] + 1
with (
patch(
f"{_MANAGER_MODULE}.time.time",
return_value=future_time,
),
patch.object(
self.bus,
"registry_set_if",
side_effect=observe_renewal,
),
):
await asyncio.wait_for(renewed.wait(), timeout=1)
expected = BackgroundTaskRecord.model_validate(
self.owner.tasks[task_id],
).model_dump()
record = await self.reader.get_task(task_id, user_id="alice")
self.assertEqual(record.model_dump(), expected)
gate.set()
await self._settle(task)
async def test_slow_io_keeps_all_running_tasks_visible(self) -> None:
"""Renewal latency must not accumulate across every running task."""
for _ in range(8):
await self._register(self.owner)
expected = {
task.id: BackgroundTaskRecord.model_validate(task).model_dump()
for task in self.owner.tasks.values()
}
task_ids = set(expected)
original_get = self.bus.registry_get
original_set_if = self.bus.registry_set_if
reads_started: set[str] = set()
writes_started: set[str] = set()
all_reads_started = asyncio.Event()
all_writes_started = asyncio.Event()
async def delayed_get(namespace: str, field: str) -> str | None:
reads_started.add(field)
if reads_started == task_ids:
all_reads_started.set()
await all_reads_started.wait()
return await original_get(namespace, field)
async def delayed_set_if(
namespace: str,
field: str,
value: str,
*,
expected: str | None,
ttl_secs: int | None = None,
) -> bool:
writes_started.add(field)
if writes_started == task_ids:
all_writes_started.set()
await all_writes_started.wait()
return await original_set_if(
namespace,
field,
value,
expected=expected,
ttl_secs=ttl_secs,
)
with (
patch.object(
self.bus,
"registry_get",
side_effect=delayed_get,
),
patch.object(
self.bus,
"registry_set_if",
side_effect=delayed_set_if,
),
):
for _ in range(2):
reads_started.clear()
writes_started.clear()
all_reads_started.clear()
all_writes_started.clear()
values = await self.bus.registry_getall(
MessageBusKeys.bg_task_records("alice"),
)
expires_at = max(
json.loads(value)["_expires_at"]
for value in values.values()
)
with patch(
f"{_MANAGER_MODULE}.time.time",
return_value=expires_at - 1,
) as clock:
await asyncio.wait_for(self._renew_once(), timeout=5)
clock.return_value = expires_at + 1
records = await self.reader.list_tasks(user_id="alice")
self.assertEqual(
{record.id: record.model_dump() for record in records},
expected,
)
self.assertEqual(
[
task.asyncio_task.done()
for task in self.owner.tasks.values()
],
[False] * 8,
)
async def test_renewal_budget_preserves_fairness_and_cancels_io(
self,
) -> None:
"""Slow tasks cannot starve the queue or leave I/O after shutdown."""
original_get = self.bus.registry_get
active: set[asyncio.Task] = set()
seen: set[str] = set()
peak = 0
cancellations = 0
reached_tail = asyncio.Event()
start_renewal = asyncio.Event()
original_renew = self.owner._renew_running_records
async def renew_after_setup() -> None:
"""Keep registration latency out of the renewal queue order."""
await start_renewal.wait()
await original_renew()
with (
patch(f"{_MANAGER_MODULE}._RECORD_LEASE_SECS", 0.2),
patch(f"{_MANAGER_MODULE}._RENEWAL_CONCURRENCY", 2),
patch.object(
self.owner,
"_renew_running_records",
side_effect=renew_after_setup,
),
):
for _ in range(6):
await self._register(self.owner)
task_ids = list(self.owner.tasks)
async def stalled_head(namespace: str, field: str) -> str | None:
nonlocal peak, cancellations
current = asyncio.current_task()
assert current is not None
active.add(current)
peak = max(peak, len(active))
try:
seen.add(field)
if seen == set(task_ids):
reached_tail.set()
if field in task_ids[:2]:
try:
await asyncio.Event().wait()
except asyncio.CancelledError:
cancellations += 1
raise
return await original_get(namespace, field)
finally:
active.remove(current)
with patch.object(
self.bus,
"registry_get",
side_effect=stalled_head,
):
start_renewal.set()
await asyncio.wait_for(reached_tail.wait(), timeout=0.5)
self.assertEqual(peak, 2)
self.assertGreaterEqual(cancellations, 2)
self.assertEqual(seen, set(task_ids))
await asyncio.wait_for(self.owner.__aexit__(), timeout=1)
self.assertEqual(active, set())
self.assertTrue(self.owner._renewal_task.done())
async def test_renewal_stops_after_io_swallows_cancellation(self) -> None:
"""A cancelled worker must not start another queued record."""
for _ in range(2):
await self._register(self.owner)
task_ids = list(self.owner.tasks)
original_get = self.bus.registry_get
reading = asyncio.Event()
seen: list[str] = []
async def finish_cancelled_read(
namespace: str,
field: str,
) -> str | None:
value = await original_get(namespace, field)
seen.append(field)
if field == task_ids[0]:
reading.set()
try:
await asyncio.Event().wait()
except asyncio.CancelledError:
# Model a response winning a Python 3.11 wait_for race.
return value
return value
with (
patch(f"{_MANAGER_MODULE}._RENEWAL_CONCURRENCY", 1),
patch.object(
self.bus,
"registry_get",
side_effect=finish_cancelled_read,
),
):
scheduler = asyncio.create_task(self._renew_once())
try:
await asyncio.wait_for(reading.wait(), timeout=2)
scheduler.cancel()
await asyncio.wait_for(scheduler, timeout=2)
self.assertEqual(seen, task_ids[:1])
finally:
scheduler.cancel()
await asyncio.gather(scheduler, return_exceptions=True)
async def test_registration_during_round_timeout_keeps_one_scheduler(
self,
) -> None:
"""Temporary timeout cancellation must not start a second scheduler."""
cancelling = asyncio.Event()
allow_cleanup = asyncio.Event()
resumed = asyncio.Event()
original_renew = self.owner._renew_running_record
async def delayed_cleanup(task: BackgroundTask) -> None:
if not allow_cleanup.is_set():
try:
await asyncio.Event().wait()
except asyncio.CancelledError:
cancelling.set()
await allow_cleanup.wait()
raise
await original_renew(task)
resumed.set()
with (
patch(f"{_MANAGER_MODULE}._RECORD_LEASE_SECS", 0.2),
patch(f"{_MANAGER_MODULE}._RENEWAL_CONCURRENCY", 2),
patch.object(
self.owner,
"_renew_running_record",
side_effect=delayed_cleanup,
),
):
await self._register(self.owner)
scheduler = self.owner._renewal_task
try:
await asyncio.wait_for(cancelling.wait(), timeout=2)
self.assertGreater(scheduler.cancelling(), 0)
self.assertFalse(scheduler.done())
for _ in range(3):
await self._register(self.owner)
self.assertIs(self.owner._renewal_task, scheduler)
allow_cleanup.set()
await asyncio.wait_for(resumed.wait(), timeout=2)
self.assertIs(self.owner._renewal_task, scheduler)
self.assertFalse(scheduler.done())
await self.owner.__aexit__()
self.assertTrue(scheduler.done())
self.assertIs(self.owner._renewal_task, scheduler)
finally:
allow_cleanup.set()
scheduler.cancel()
await asyncio.gather(scheduler, return_exceptions=True)
async def test_renewal_reuses_live_scheduler_and_restarts_after_exit(
self,
) -> None:
"""New work reuses a live scheduler, including its idle interval."""
sleeping = asyncio.Event()
release_sleep = asyncio.Event()
original_sleep = asyncio.sleep
async def hold_first_sleep(delay: float) -> None:
if not sleeping.is_set():
sleeping.set()
try:
await release_sleep.wait()
except asyncio.CancelledError:
await release_sleep.wait()
raise
else:
await original_sleep(delay)
with (
patch(f"{_MANAGER_MODULE}._RECORD_LEASE_SECS", 0.2),
patch(
f"{_MANAGER_MODULE}.asyncio",
wraps=asyncio,
sleep=AsyncMock(side_effect=hold_first_sleep),
),
):
gate, task, _ = await self._register(self.owner)
scheduler = self.owner._renewal_task
try:
await asyncio.wait_for(sleeping.wait(), timeout=2)
gate.set()
await self._settle(task)
self.assertEqual(scheduler.cancelling(), 0)
self.assertFalse(scheduler.done())
gate, task, _ = await self._register(self.owner)
self.assertIs(self.owner._renewal_task, scheduler)
gate.set()
await self._settle(task)
release_sleep.set()
await asyncio.wait_for(asyncio.shield(scheduler), timeout=2)
await self._register(self.owner)
replacement = self.owner._renewal_task
self.assertIsNot(replacement, scheduler)
self.assertFalse(replacement.done())
await self.owner.__aexit__()
self.assertTrue(replacement.done())
finally:
release_sleep.set()
scheduler.cancel()
await asyncio.gather(scheduler, return_exceptions=True)
async def test_heartbeat_restores_a_cleaned_running_record(self) -> None:
"""A live task reappears after its expired snapshot is deleted."""
_, _, task_id = await self._register(self.owner)
namespace = MessageBusKeys.bg_task_records("alice")
value = await self.bus.registry_get(namespace, task_id)
payload = json.loads(value)
payload["_expires_at"] = 0
await self.bus.registry_set(namespace, task_id, json.dumps(payload))
self.assertEqual(await self.reader.list_tasks(user_id="alice"), [])
self.assertIsNone(await self.bus.registry_get(namespace, task_id))
await self._renew_once()
expected = BackgroundTaskRecord.model_validate(
self.owner.tasks[task_id],
).model_dump()
record = await self.reader.get_task(task_id, user_id="alice")
self.assertIsNotNone(record)
self.assertEqual(record.model_dump(), expected)
self.assertEqual(
[
r.model_dump()
for r in await self.reader.list_tasks(
user_id="alice",
)
],
[expected],
)
async def test_heartbeat_restore_preserves_concurrent_outcome(
self,
) -> None:
"""An outcome written after a missing read wins over recovery."""
_, _, task_id = await self._register(self.owner)
namespace = MessageBusKeys.bg_task_records("alice")
await self.bus.registry_del(namespace, task_id)
terminal = BackgroundTaskRecord.model_validate(
self.owner.tasks[task_id],
)
terminal.status = TaskStatus.COMPLETED
terminal.completed_at = terminal.started_at + 1
original_set_if = self.bus.registry_set_if
async def complete_before_write(
namespace: str,
field: str,
value: str,
*,
expected: str | None,
ttl_secs: int | None = None,
) -> bool:
await self.owner._store_record(terminal)
return await original_set_if(
namespace,
field,
value,
expected=expected,
ttl_secs=ttl_secs,
)
with patch.object(
self.bus,
"registry_set_if",
side_effect=complete_before_write,
) as restore:
await self._renew_once()
restore.assert_awaited_once()
self.assertIsNone(restore.call_args.kwargs["expected"])
record = await self.reader.get_task(task_id, user_id="alice")
self.assertEqual(record.model_dump(), terminal.model_dump())
async def test_heartbeat_skips_task_completed_during_read(self) -> None:
"""A missing read cannot revive a task that has already finished."""
gate, task, task_id = await self._register(self.owner)
namespace = MessageBusKeys.bg_task_records("alice")
await self.bus.registry_del(namespace, task_id)
original_get = self.bus.registry_get
async def complete_during_read(
namespace: str,
field: str,
) -> str | None:
value = await original_get(namespace, field)
gate.set()
await self._settle(task)
return value
with (
patch.object(
self.bus,
"registry_get",
side_effect=complete_during_read,
),
patch.object(
self.bus,
"registry_set_if",
wraps=self.bus.registry_set_if,
) as restore,
):
await self._renew_once()
restore.assert_not_awaited()
record = await self.reader.get_task(task_id, user_id="alice")
self.assertEqual(record.status, TaskStatus.COMPLETED)
async def test_http_routes_read_another_workers_records(self) -> None:
"""The actual app exposes filtered lists, details and isolated 404s."""
gate, task, task_id = await self._register(
self.owner,
execution_started_at=100.0,
)
record = await self.reader.get_task(task_id, user_id="alice")
expected = {
"task_id": task_id,
"session_id": "s1",
"agent_id": "a1",
"tool_name": "Tool",
"summary": "Tool(x=1)",
"status": "running",
"started_at": datetime.fromtimestamp(
record.started_at,
tz=timezone.utc,
)
.isoformat()
.replace("+00:00", "Z"),
"execution_started_at": "1970-01-01T00:01:40Z",
"completed_at": None,
"error_summary": None,
}
app = create_app(
storage=MagicMock(),
message_bus=self.bus,
workspace_manager=MagicMock(),
)
app.state.background_task_manager = self.reader
client = await self.enterAsyncContext(
httpx.AsyncClient(
transport=httpx.ASGITransport(app=app),
base_url="http://test",
),
)
headers = {"X-User-ID": "alice"}
for query, body in (
("", {"tasks": [expected], "total": 1}),
("?session_id=s1", {"tasks": [expected], "total": 1}),
("?session_id=s2", {"tasks": [], "total": 0}),
):
response = await client.get(
f"/background-tasks{query}",
headers=headers,
)
self.assertEqual(
(response.status_code, response.json()),
(
200,
body,
),
)
response = await client.get(
f"/background-tasks/{task_id}",
headers=headers,
)
self.assertEqual(
(response.status_code, response.json()),
(
200,
expected,
),
)
for missing_id, user in ((task_id, "bob"), ("missing", "alice")):
response = await client.get(
f"/background-tasks/{missing_id}",
headers={"X-User-ID": user},
)
self.assertEqual(
(response.status_code, response.json()),
(
404,
{"detail": f"Background task '{missing_id}' not found."},
),
)
response = await client.get("/background-tasks")
self.assertEqual(response.status_code, 422)
gate.set()
await self._settle(task)
record = await self.reader.get_task(task_id, user_id="alice")
expected.update(
status="completed",
completed_at=datetime.fromtimestamp(
record.completed_at,
tz=timezone.utc,
)
.isoformat()
.replace("+00:00", "Z"),
)
response = await client.get(
f"/background-tasks/{task_id}",
headers=headers,
)
self.assertEqual(
(response.status_code, response.json()),
(
200,
expected,
),
)
class TestRedisBackgroundTaskRecords(TestBackgroundTaskRecords):
"""Run the same worker and lifecycle regressions through Redis APIs."""
def _make_bus(self) -> MessageBus:
"""Use a fake Redis server without opening network connections."""
client = fakeredis.aioredis.FakeRedis(decode_responses=True)
self.addAsyncCleanup(client.aclose)
return RedisMessageBus(connection_pool=client.connection_pool)
class TestMakeSummary(IsolatedAsyncioTestCase):
"""Test the _make_summary helper function."""
async def test_empty_input_returns_tool_name(self) -> None:
"""Empty or None input returns just the tool name."""
self.assertEqual(_make_summary("Bash", None), "Bash")
self.assertEqual(_make_summary("Bash", ""), "Bash")
async def test_dict_input_includes_key_values(self) -> None:
"""JSON dict input produces key=value summary."""
raw = json.dumps({"command": "ls -la", "timeout": 30})
result = _make_summary("Bash", raw)
self.assertIn("Bash(", result)
self.assertIn("command=ls -la", result)
self.assertIn("timeout=30", result)
self.assertTrue(result.endswith(")"))
async def test_respects_max_length(self) -> None:
"""Summary never exceeds _SUMMARY_MAX_LEN characters."""
raw = json.dumps({"key": "x" * 200, "other": "y" * 200})
result = _make_summary("VeryLongToolName", raw)
self.assertLessEqual(len(result), 128)
async def test_non_json_input_is_truncated(self) -> None:
"""Non-JSON input is included as-is with truncation."""
raw = "a" * 100
result = _make_summary("Tool", raw)
self.assertIn("Tool(", result)
self.assertIn("...", result)
self.assertLessEqual(len(result), 128)
async def test_non_json_short_input_no_ellipsis(self) -> None:
"""Short non-JSON input doesn't get a misleading ellipsis."""
result = _make_summary("Tool", "hi")
self.assertEqual(result, "Tool(hi)")
async def test_empty_dict_returns_tool_name(self) -> None:
"""Empty JSON dict returns just the tool name."""
result = _make_summary("Tool", "{}")
self.assertEqual(result, "Tool")
async def test_non_dict_json_returns_tool_name(self) -> None:
"""JSON that's not a dict returns just the tool name."""
result = _make_summary("Tool", "[1, 2, 3]")
self.assertEqual(result, "Tool")
async def test_long_values_are_truncated(self) -> None:
"""Individual values longer than 40 chars are truncated."""
raw = json.dumps({"file": "/" + "a" * 100})
result = _make_summary("Read", raw)
self.assertIn("...", result)
self.assertLessEqual(len(result), 128)
async def test_multiple_keys_included_within_budget(self) -> None:
"""Multiple short key-values are all included."""
raw = json.dumps({"a": "1", "b": "2", "c": "3"})
result = _make_summary("T", raw)
self.assertIn("a=1", result)
self.assertIn("b=2", result)
self.assertIn("c=3", result)