Skip to content

Build and run agents you can see, understand and trust.

feat(service): expose background tasks via REST endpoints (#1865)

XiuShenAlcfd3d99982cd
diff --git a/src/agentscope/app/_app.py b/src/agentscope/app/_app.py
index 7f004ac08..f97133bde 100644
--- a/src/agentscope/app/_app.py
+++ b/src/agentscope/app/_app.py
@@ -17,6 +17,7 @@ from .workspace_manager import WorkspaceManagerBase
from ._router import (
asr_router,
agent_router,
+ background_task_router,
channel_router,
chat_router,
credential_router,
@@ -439,6 +440,7 @@ def create_app(
for router in (
asr_router,
agent_router,
+ background_task_router,
chat_router,
credential_router,
health_router,
diff --git a/src/agentscope/app/_manager/__init__.py b/src/agentscope/app/_manager/__init__.py
index f4facb0e2..5409c6b49 100644
--- a/src/agentscope/app/_manager/__init__.py
+++ b/src/agentscope/app/_manager/__init__.py
@@ -6,12 +6,13 @@ from ._scheduler import SchedulerManager
from ._wakeup_dispatcher import WakeupDispatcher
from ._cancel_dispatcher import CancelDispatcher
from ._chat_run_registry import ChatRunRegistry
-from ._background_task_manager import BackgroundTaskManager
+from ._background_task_manager import BackgroundTaskManager, TaskStatus
__all__ = [
"BackgroundTaskManager",
"CancelDispatcher",
"ChatRunRegistry",
"SchedulerManager",
+ "TaskStatus",
"WakeupDispatcher",
]
diff --git a/src/agentscope/app/_manager/_background_task_manager.py b/src/agentscope/app/_manager/_background_task_manager.py
index f26acd066..48d2ca04f 100644
--- a/src/agentscope/app/_manager/_background_task_manager.py
+++ b/src/agentscope/app/_manager/_background_task_manager.py
@@ -5,25 +5,44 @@ import json
import time
from collections import OrderedDict
from dataclasses import dataclass, field
+from enum import Enum
from typing import Any, Self, TYPE_CHECKING
import shortuuid
-from pydantic import BaseModel, Field
+from pydantic import BaseModel, ConfigDict, Field
-from agentscope.message import TextBlock, ToolResultState
-from agentscope.permission import (
+from ...message import TextBlock, ToolResultState
+from ...permission import (
PermissionContext,
PermissionDecision,
PermissionBehavior,
)
-from agentscope.tool import ToolBase, ToolChunk
-from agentscope._logging import logger
+from ...tool import ToolBase, ToolChunk, ToolResponse
+from ..._logging import logger
from ..message_bus import MessageBusKeys
if TYPE_CHECKING:
from ..message_bus import MessageBus
+class TaskStatus(str, Enum):
+ """Status of a background task throughout its lifecycle."""
+
+ RUNNING = "running"
+ COMPLETED = "completed"
+ FAILED = "failed"
+ CANCELLED = "cancelled"
+
+
+_MAX_RETAINED_TASKS = 50
+# Maximum number of terminal-state tasks retained per user.
+_RECORD_LEASE_SECS = 120
+_RENEWAL_CONCURRENCY = 16
+_IO_TIMEOUT_SECS = 2.0
+_RETRY_DELAY_SECS = 1.0
+_SHUTDOWN_TIMEOUT_SECS = 5.0
+
+
@dataclass
class BackgroundTask:
"""Metadata for a single background task.
@@ -41,6 +60,19 @@ class BackgroundTask:
The name of the tool that was offloaded.
id (`str`):
Auto-generated unique task identifier.
+ summary (`str`):
+ A short human-readable summary of the tool invocation
+ for frontend display (at most 128 characters).
+ started_at (`float`):
+ Unix epoch (UTC) when the task was registered.
+ completed_at (`float | None`):
+ Unix epoch (UTC) when the task reached a terminal state,
+ or ``None`` while still running.
+ status (`TaskStatus`):
+ Current lifecycle status of the task.
+ error_summary (`str | None`):
+ Brief description of the failure reason when
+ ``status == FAILED``.
"""
asyncio_task: asyncio.Task
@@ -61,6 +93,38 @@ class BackgroundTask:
id: str = field(default_factory=shortuuid.uuid)
"""The background task id."""
+ summary: str = ""
+ """Short human-readable summary for frontend display."""
+
+ started_at: float = field(default_factory=time.time)
+ """Unix epoch (UTC) when the task was registered."""
+
+ completed_at: float | None = None
+ """Unix epoch (UTC) when the task finished, or None if running."""
+
+ status: TaskStatus = TaskStatus.RUNNING
+ """Current lifecycle status."""
+
+ error_summary: str | None = None
+ """Brief failure description (populated only for FAILED tasks)."""
+
+
+class BackgroundTaskRecord(BaseModel):
+ """Serializable task snapshot, independent of the local task handle."""
+
+ model_config = ConfigDict(from_attributes=True)
+
+ id: str
+ session_id: str
+ agent_id: str
+ user_id: str
+ tool_name: str
+ summary: str
+ started_at: float
+ completed_at: float | None
+ status: TaskStatus
+ error_summary: str | None
+
class _ToolStopParams(BaseModel):
"""The params of the stop tool."""
@@ -158,8 +222,8 @@ class ToolStop(ToolBase):
if (
local_task is not None
and local_task.session_id == self._session_id
+ and not local_task.asyncio_task.done()
):
- self.background_tasks.pop(task_id, None)
local_task.asyncio_task.cancel()
logger.info(
"Background task stopped via ToolStop (local): task_id=%s, "
@@ -241,6 +305,9 @@ class BackgroundTaskManager:
"""
self._message_bus = message_bus
self.tasks: OrderedDict[str, BackgroundTask] = OrderedDict()
+ self._completion_tasks: set[asyncio.Task] = set()
+ self._renewal_task: asyncio.Task | None = None
+ self._closed = False
# ------------------------------------------------------------------
# Task registration
@@ -253,12 +320,14 @@ class BackgroundTaskManager:
agent_id: str,
user_id: str,
tool_name: str = "",
+ *,
+ summary: str = "",
) -> str:
"""Register an already-running asyncio task.
- Writes to both the local handle cache and the global Redis
- registry. The task auto-removes from both when it finishes
- (via ``add_done_callback``).
+ Tracks running handles locally and publishes metadata globally.
+ On completion, releases the handle and retains terminal metadata
+ for polling from any worker.
Args:
asyncio_task (`asyncio.Task`):
@@ -271,6 +340,9 @@ class BackgroundTaskManager:
The user id of the originating request.
tool_name (`str`, optional):
The name of the offloaded tool.
+ summary (`str`, optional):
+ Short human-readable summary of the tool invocation
+ for frontend display.
Returns:
`str`:
@@ -282,39 +354,44 @@ class BackgroundTaskManager:
agent_id=agent_id,
user_id=user_id,
tool_name=tool_name,
+ summary=summary,
)
task_id = bg_task.id
self.tasks[task_id] = bg_task
- def _on_done(_t: asyncio.Task) -> None:
- self.tasks.pop(task_id, None)
- # Clean up asynchronously; the helper logs registry errors.
- try:
- asyncio.ensure_future(
- self._safe_bg_task_unregister(session_id, task_id),
- )
- except RuntimeError:
- # Event loop already closed during shutdown.
- pass
-
- metadata = json.dumps(
- {
- "tool_name": tool_name,
- "agent_id": agent_id,
- "started_at": time.time(),
- },
- ensure_ascii=False,
- )
+ # Install cleanup even if a registry write fails or is cancelled.
+ # The callback is attached after writes settle, so a fast task cannot
+ # publish a terminal state before its initial running state.
try:
- await self._message_bus.registry_set(
- MessageBusKeys.bg_tasks(session_id),
- task_id,
- metadata,
- ttl_secs=MessageBusKeys.BG_TASKS_TTL_SECS,
- )
+ async with asyncio.timeout(_IO_TIMEOUT_SECS):
+ await self._message_bus.registry_set(
+ MessageBusKeys.bg_tasks(session_id),
+ task_id,
+ json.dumps(
+ {
+ "tool_name": tool_name,
+ "agent_id": agent_id,
+ "started_at": bg_task.started_at,
+ },
+ ensure_ascii=False,
+ ),
+ ttl_secs=MessageBusKeys.BG_TASKS_TTL_SECS,
+ )
+ await self._store_record(
+ BackgroundTaskRecord.model_validate(bg_task),
+ )
+ # A round timeout can temporarily mark the scheduler cancelling.
+ if self._renewal_task is None or self._renewal_task.done():
+ self._renewal_task = asyncio.create_task(
+ self._renew_running_records(),
+ )
+ except BaseException:
+ asyncio_task.cancel()
+ raise
finally:
- # Always install cleanup, after the write to avoid early deletion.
- asyncio_task.add_done_callback(_on_done)
+ asyncio_task.add_done_callback(
+ lambda task: self._on_done(bg_task, task),
+ )
logger.info(
"Background task registered: task_id=%s, session_id=%s, "
@@ -324,35 +401,266 @@ class BackgroundTaskManager:
agent_id,
tool_name,
)
-
return task_id
- async def _safe_bg_task_unregister(
+ def _on_done(
self,
- session_id: str,
- task_id: str,
+ bg_task: BackgroundTask,
+ task: asyncio.Task,
) -> None:
- """Unregister a finished background task, logging any failure.
+ """Record the actual outcome and schedule shared-state cleanup."""
+ bg_task.completed_at = time.time()
+ if task.cancelled():
+ bg_task.status = TaskStatus.CANCELLED
+ elif task.exception() is not None:
+ bg_task.status = TaskStatus.FAILED
+ bg_task.error_summary = str(task.exception())[:512]
+ else:
+ result = task.result()
+ bg_task.status = TaskStatus.COMPLETED
+ if isinstance(result, ToolResponse):
+ if result.state == ToolResultState.ERROR:
+ bg_task.status = TaskStatus.FAILED
+ bg_task.error_summary = "\n".join(
+ block.text
+ for block in result.content
+ if isinstance(block, TextBlock)
+ )[:512]
+ elif result.state == ToolResultState.INTERRUPTED:
+ bg_task.status = TaskStatus.CANCELLED
+
+ self.tasks.pop(bg_task.id, None)
+ # Let renewal exit when idle so new registrations can reuse it safely.
+ if self._closed:
+ return
+ completion = asyncio.create_task(
+ self._finish_task(BackgroundTaskRecord.model_validate(bg_task)),
+ )
+ self._completion_tasks.add(completion)
+ completion.add_done_callback(self._completion_tasks.discard)
+
+ async def _store_record(self, record: BackgroundTaskRecord) -> None:
+ """Publish a snapshot with an expiry independent of the user hash."""
+ payload = record.model_dump(mode="json")
+ if record.status == TaskStatus.RUNNING:
+ payload["_expires_at"] = time.time() + _RECORD_LEASE_SECS
+ else:
+ assert record.completed_at is not None
+ payload["_expires_at"] = (
+ record.completed_at + MessageBusKeys.BG_TASKS_TTL_SECS
+ )
+ await self._message_bus.registry_set(
+ MessageBusKeys.bg_task_records(record.user_id),
+ record.id,
+ json.dumps(payload, ensure_ascii=False),
+ ttl_secs=MessageBusKeys.BG_TASKS_TTL_SECS,
+ )
- Args:
- session_id (`str`):
- The session id of the finished task.
- task_id (`str`):
- The task id to unregister from the global registry.
- """
+ async def _renew_running_records(self) -> None:
+ """Renew leases in bounded, fair rounds without scheduling drift."""
+ pending: OrderedDict[str, BackgroundTask] = OrderedDict()
+ loop = asyncio.get_running_loop()
+ interval = _RECORD_LEASE_SECS / 4
+ deadline = loop.time() + interval
+
+ async def renew_pending() -> None:
+ """Share the queue without creating a worker for every task."""
+ while pending:
+ _, task = pending.popitem(last=False)
+ await self._renew_running_record(task)
+
+ while self.tasks and not self._closed:
+ await asyncio.sleep(max(0, deadline - loop.time()))
+ deadline = loop.time() + interval
+ # Unvisited tasks stay ahead of retries and newly registered work.
+ pending.update(self.tasks)
+ try:
+ async with asyncio.timeout_at(deadline):
+ async with asyncio.TaskGroup() as group:
+ for _ in range(
+ min(_RENEWAL_CONCURRENCY, len(pending)),
+ ):
+ group.create_task(renew_pending())
+ except TimeoutError:
+ logger.warning(
+ "Background task renewal round exceeded its budget: "
+ "unvisited=%d",
+ len(pending),
+ )
+
+ async def _renew_running_record(self, task: BackgroundTask) -> None:
+ """Renew or restore one live record without overwriting outcomes."""
+ if task.asyncio_task.done():
+ return
try:
+ async with asyncio.timeout(_IO_TIMEOUT_SECS):
+ namespace = MessageBusKeys.bg_task_records(task.user_id)
+ value = await self._message_bus.registry_get(
+ namespace,
+ task.id,
+ )
+ if task.asyncio_task.done():
+ return
+ if value is None:
+ payload = BackgroundTaskRecord.model_validate(
+ task,
+ ).model_dump(mode="json")
+ else:
+ payload = json.loads(value)
+ if payload["status"] != TaskStatus.RUNNING:
+ return
+ payload["_expires_at"] = time.time() + _RECORD_LEASE_SECS
+ # A concurrent completion must win over this lease.
+ await self._message_bus.registry_set_if(
+ namespace,
+ task.id,
+ json.dumps(payload, ensure_ascii=False),
+ expected=value,
+ ttl_secs=MessageBusKeys.BG_TASKS_TTL_SECS,
+ )
+ except Exception:
+ logger.exception(
+ "Failed to renew background task lease: task_id=%s",
+ task.id,
+ )
+
+ async def _finish_task(self, record: BackgroundTaskRecord) -> None:
+ """Retry pending outcomes and cleanup until success or shutdown."""
+ stored = False
+ unregistered = False
+ while not self._closed:
+ try:
+ # Outcome persistence never waits for the retention lock.
+ async with asyncio.timeout(_IO_TIMEOUT_SECS):
+ if not stored:
+ await self._store_record(record)
+ stored = True
+ if not unregistered:
+ await self._message_bus.registry_del(
+ MessageBusKeys.bg_tasks(record.session_id),
+ record.id,
+ )
+ unregistered = True
+ async with asyncio.timeout(_IO_TIMEOUT_SECS):
+ async with self._message_bus.acquire_lock(
+ MessageBusKeys.bg_task_records_lock(record.user_id),
+ ttl_secs=30,
+ ):
+ await self._evict_completed(record.user_id)
+ return
+ except Exception:
+ logger.warning(
+ "Retrying background task finalization: task_id=%s",
+ record.id,
+ exc_info=True,
+ )
+ await asyncio.sleep(_RETRY_DELAY_SECS)
+
+ async def _evict_completed(self, user_id: str) -> None:
+ """Delete expired snapshots and bound terminal history."""
+ namespace = MessageBusKeys.bg_task_records(user_id)
+ values = await self._message_bus.registry_getall(namespace)
+ completed = []
+ for task_id, value in values.items():
+ record = await self._read_record(namespace, task_id, value)
+ if record is not None and record.status != TaskStatus.RUNNING:
+ completed.append(record)
+ completed.sort(key=lambda record: (record.completed_at, record.id))
+ for record in completed[:-_MAX_RETAINED_TASKS]:
await self._message_bus.registry_del(
- MessageBusKeys.bg_tasks(session_id),
- task_id,
+ MessageBusKeys.bg_task_records(user_id),
+ record.id,
)
- except Exception as e: # pylint: disable=broad-except
- logger.exception(
- "Failed to unregister background task from the global "
- "registry: task_id=%s, session_id=%s, error=%s",
+
+ async def _read_record(
+ self,
+ namespace: str,
+ task_id: str,
+ value: str,
+ ) -> BackgroundTaskRecord | None:
+ """Remove an expired snapshot only if its value is unchanged."""
+ payload = json.loads(value)
+ # Older snapshots have no per-record lease. Bound their lifetime too.
+ expires_at = payload.get(
+ "_expires_at",
+ payload["started_at"] + MessageBusKeys.BG_TASKS_TTL_SECS,
+ )
+ if expires_at <= time.time():
+ await self._message_bus.registry_del_if(
+ namespace,
task_id,
- session_id,
- str(e),
+ expected=value,
)
+ return None
+ return BackgroundTaskRecord.model_validate(payload)
+
+ # ------------------------------------------------------------------
+ # Task querying (for REST endpoints)
+ # ------------------------------------------------------------------
+
+ async def list_tasks(
+ self,
+ *,
+ user_id: str,
+ session_id: str | None = None,
+ ) -> list[BackgroundTaskRecord]:
+ """List shared running and retained terminal tasks for a user.
+
+ Args:
+ user_id (`str`):
+ Only return tasks belonging to this user.
+ session_id (`str | None`, optional):
+ If provided, further filter by session.
+
+ Returns:
+ `list[BackgroundTaskRecord]`:
+ Matching tasks ordered by registration time.
+ """
+ namespace = MessageBusKeys.bg_task_records(user_id)
+ values = await self._message_bus.registry_getall(namespace)
+ records = []
+ for task_id, value in values.items():
+ record = await self._read_record(namespace, task_id, value)
+ if record is not None:
+ records.append(record)
+ return sorted(
+ (
+ record
+ for record in records
+ if record.user_id == user_id
+ and (session_id is None or record.session_id == session_id)
+ ),
+ key=lambda record: (record.started_at, record.id),
+ )
+
+ async def get_task(
+ self,
+ task_id: str,
+ *,
+ user_id: str,
+ ) -> BackgroundTaskRecord | None:
+ """Get shared task metadata, scoped to the requesting user.
+
+ Args:
+ task_id (`str`):
+ The task identifier.
+ user_id (`str`):
+ The requesting user's id.
+
+ Returns:
+ `BackgroundTaskRecord | None`:
+ The task if owned by the user, else `None`.
+ """
+ namespace = MessageBusKeys.bg_task_records(user_id)
+ value = await self._message_bus.registry_get(namespace, task_id)
+ if value is None:
+ return None
+ record = await self._read_record(namespace, task_id, value)
+ return (
+ record
+ if record is not None and record.user_id == user_id
+ else None
+ )
# ------------------------------------------------------------------
# Tool listing
@@ -393,7 +701,7 @@ class BackgroundTaskManager:
"""
cancelled = 0
for bg_task in list(self.tasks.values()):
- if bg_task.session_id != session_id:
+ if bg_task.session_id != session_id or bg_task.asyncio_task.done():
continue
logger.info(
"Cancelling background task for session cancel: "
@@ -426,7 +734,7 @@ class BackgroundTaskManager:
``True`` if the task was found locally and cancelled.
"""
bg_task = self.tasks.get(task_id)
- if bg_task is None:
+ if bg_task is None or bg_task.asyncio_task.done():
return False
logger.info(
"Cancelling background task via bus signal: "
@@ -451,12 +759,18 @@ class BackgroundTaskManager:
return self
async def __aexit__(self, *exc: object) -> None:
- """Cancel all running background tasks on context exit."""
+ """Cancel work and flush outcomes within a bounded shutdown budget."""
count = len(self.tasks)
logger.info(
"Shutting down BackgroundTaskManager: cancelling %d task(s).",
count,
)
+ tasks = [bg.asyncio_task for bg in self.tasks.values()]
+ if self._renewal_task is not None:
+ self._renewal_task.cancel()
+ loop = asyncio.get_running_loop()
+ deadline = loop.time() + _SHUTDOWN_TIMEOUT_SECS
+ # Reserve the last 10% of the budget for cancellation cleanup.
for bg_task in list(self.tasks.values()):
logger.info(
"Cancelling background task on shutdown: task_id=%s, "
@@ -466,4 +780,46 @@ class BackgroundTaskManager:
bg_task.agent_id,
)
bg_task.asyncio_task.cancel()
- self.tasks.clear()
+ try:
+ if tasks:
+ await asyncio.wait(
+ tasks,
+ timeout=max(
+ 0,
+ deadline - loop.time() - _SHUTDOWN_TIMEOUT_SECS / 10,
+ ),
+ )
+ if self._completion_tasks:
+ await asyncio.wait(
+ self._completion_tasks,
+ timeout=max(
+ 0,
+ deadline - loop.time() - _SHUTDOWN_TIMEOUT_SECS / 10,
+ ),
+ )
+ finally:
+ # Late execution callbacks must not start I/O after the bus closes.
+ self._closed = True
+ pending = {
+ task
+ for task in [
+ *tasks,
+ *self._completion_tasks,
+ self._renewal_task,
+ ]
+ if task is not None and not task.done()
+ }
+ for task in pending:
+ task.cancel()
+ if pending:
+ _, unfinished = await asyncio.wait(
+ pending,
+ timeout=max(0, deadline - loop.time()),
+ )
+ logger.warning(
+ "Background task shutdown budget exhausted: "
+ "cancelled=%d, unfinished=%d",
+ len(pending),
+ len(unfinished),
+ )
+ self.tasks.clear()
diff --git a/src/agentscope/app/_router/__init__.py b/src/agentscope/app/_router/__init__.py
index 02aaa2eec..8a1220bcb 100644
--- a/src/agentscope/app/_router/__init__.py
+++ b/src/agentscope/app/_router/__init__.py
@@ -2,6 +2,7 @@
"""App routers."""
from ._asr import asr_router
from ._agent import agent_router
+from ._background_task import background_task_router
from ._channel import channel_router
from ._chat import chat_router
from ._credential import credential_router
@@ -23,6 +24,7 @@ from ._workspace import workspace_router
__all__ = [
"asr_router",
"agent_router",
+ "background_task_router",
"channel_router",
"model_router",
"realtime_router",
diff --git a/src/agentscope/app/_router/_background_task.py b/src/agentscope/app/_router/_background_task.py
new file mode 100644
index 000000000..8d05b064b
--- /dev/null
+++ b/src/agentscope/app/_router/_background_task.py
@@ -0,0 +1,98 @@
+# -*- coding: utf-8 -*-
+"""Background-task router — read-only endpoints for task observability."""
+from fastapi import APIRouter, Depends, HTTPException, Query, status
+
+from ..deps import get_background_task_manager, get_current_user_id
+from .._manager import BackgroundTaskManager
+from ._schema import (
+ BackgroundTaskInfo,
+ ListBackgroundTasksResponse,
+)
+from ._schema._background_task import to_info
+
+background_task_router = APIRouter(
+ prefix="/background-tasks",
+ tags=["background-tasks"],
+ responses={404: {"description": "Not found"}},
+)
+
+
+@background_task_router.get(
+ "",
+ response_model=ListBackgroundTasksResponse,
+ summary="List background tasks for the current user",
+)
+async def list_background_tasks(
+ session_id: str
+ | None = Query(
+ default=None,
+ description="Optional session filter.",
+ ),
+ user_id: str = Depends(get_current_user_id),
+ bg_manager: BackgroundTaskManager = Depends(
+ get_background_task_manager,
+ ),
+) -> ListBackgroundTasksResponse:
+ """List all background tasks (running and recently-completed) visible
+ to the authenticated user.
+
+ Args:
+ session_id (`str | None`, optional):
+ If provided, only return tasks belonging to this session.
+ user_id (`str`):
+ The authenticated user id (from ``X-User-ID`` header).
+ bg_manager (`BackgroundTaskManager`):
+ The application-wide background task manager.
+
+ Returns:
+ `ListBackgroundTasksResponse`:
+ Tasks and their total count.
+ """
+ tasks = await bg_manager.list_tasks(
+ user_id=user_id,
+ session_id=session_id,
+ )
+ infos = [to_info(t) for t in tasks]
+ return ListBackgroundTasksResponse(tasks=infos, total=len(infos))
+
+
+@background_task_router.get(
+ "/{task_id}",
+ response_model=BackgroundTaskInfo,
+ summary="Get a single background task by id",
+)
+async def get_background_task(
+ task_id: str,
+ user_id: str = Depends(get_current_user_id),
+ bg_manager: BackgroundTaskManager = Depends(
+ get_background_task_manager,
+ ),
+) -> BackgroundTaskInfo:
+ """Retrieve details of a single background task.
+
+ Returns 404 when the task does not exist or belongs to a different
+ user (no information leakage about task existence).
+
+ Args:
+ task_id (`str`):
+ The unique task identifier.
+ user_id (`str`):
+ The authenticated user id (from ``X-User-ID`` header).
+ bg_manager (`BackgroundTaskManager`):
+ The application-wide background task manager.
+
+ Returns:
+ `BackgroundTaskInfo`:
+ The task details.
+
+ Raises:
+ `HTTPException`:
+ 404 if the task is not found or not owned by the user.
+ """
+ task = await bg_manager.get_task(task_id, user_id=user_id)
+ if task is None:
+ raise HTTPException(
+ status_code=status.HTTP_404_NOT_FOUND,
+ detail=f"Background task '{task_id}' not found.",
+ )
+ return to_info(task)
diff --git a/src/agentscope/app/_router/_schema/__init__.py b/src/agentscope/app/_router/_schema/__init__.py
index e6057fdfb..c291f6bea 100644
--- a/src/agentscope/app/_router/_schema/__init__.py
+++ b/src/agentscope/app/_router/_schema/__init__.py
@@ -1,6 +1,10 @@
# -*- coding: utf-8 -*-
"""Schema models for the agent service."""
+from ._background_task import (
+ BackgroundTaskInfo,
+ ListBackgroundTasksResponse,
+)
from ._channel import (
StartCredentialBindingRequest,
ChannelActionResponse,
@@ -170,6 +174,9 @@ __all__ = [
"CreateAgentResponse",
"UpdateAgentRequest",
"ListSchedulesResponse",
+ # Background Task
+ "BackgroundTaskInfo",
+ "ListBackgroundTasksResponse",
# Channel
"ChannelActionResponse",
"ChannelChatId",
diff --git a/src/agentscope/app/_router/_schema/_background_task.py b/src/agentscope/app/_router/_schema/_background_task.py
new file mode 100644
index 000000000..eee507069
--- /dev/null
+++ b/src/agentscope/app/_router/_schema/_background_task.py
@@ -0,0 +1,76 @@
+# -*- coding: utf-8 -*-
+"""Request / response schemas for the background-task router."""
+from datetime import datetime, timezone
+
+from pydantic import BaseModel, Field
+
+from ..._manager._background_task_manager import (
+ BackgroundTaskRecord,
+ TaskStatus,
+)
+
+
+class BackgroundTaskInfo(BaseModel):
+ """Summary of a single background task for API responses."""
+
+ task_id: str = Field(description="Unique task identifier.")
+ session_id: str = Field(description="Session that owns this task.")
+ agent_id: str = Field(description="Agent that triggered this task.")
+ tool_name: str = Field(description="Name of the offloaded tool.")
+ summary: str = Field(
+ description="Short human-readable summary of the tool invocation. "
+ "May include unredacted tool arguments stored in shared task records.",
+ )
+ status: TaskStatus = Field(description="Current lifecycle status.")
+ started_at: datetime = Field(
+ description="UTC timestamp when the task was registered.",
+ )
+ completed_at: datetime | None = Field(
+ default=None,
+ description="UTC timestamp when the task reached a terminal "
+ "state, or null while still running.",
+ )
+ error_summary: str | None = Field(
+ default=None,
+ description="Brief failure description (only for failed tasks).",
+ )
+
+
+class ListBackgroundTasksResponse(BaseModel):
+ """Response body for listing background tasks."""
+
+ tasks: list[BackgroundTaskInfo] = Field(
+ description="Background tasks for the requested scope.",
+ )
+ total: int = Field(description="Total number of tasks returned.")
+
+
+def to_info(task: BackgroundTaskRecord) -> BackgroundTaskInfo:
+ """Convert shared task metadata to an API-facing schema.
+
+ Args:
+ task (`BackgroundTaskRecord`):
+ The shared background task metadata.
+
+ Returns:
+ `BackgroundTaskInfo`:
+ The serializable API response model.
+ """
+ return BackgroundTaskInfo(
+ task_id=task.id,
+ session_id=task.session_id,
+ agent_id=task.agent_id,
+ tool_name=task.tool_name,
+ summary=task.summary,
+ status=task.status,
+ started_at=datetime.fromtimestamp(
+ task.started_at,
+ tz=timezone.utc,
+ ),
+ completed_at=(
+ datetime.fromtimestamp(task.completed_at, tz=timezone.utc)
+ if task.completed_at is not None
+ else None
+ ),
+ error_summary=task.error_summary,
+ )
diff --git a/src/agentscope/app/message_bus/_base.py b/src/agentscope/app/message_bus/_base.py
index a943d46b5..f0989b4d9 100644
--- a/src/agentscope/app/message_bus/_base.py
+++ b/src/agentscope/app/message_bus/_base.py
@@ -514,7 +514,7 @@ class MessageBus(ABC): # pylint: disable=too-many-public-methods
field: str,
value: str,
*,
- expected: str,
+ expected: str | None,
ttl_secs: int | None = None,
) -> bool:
"""Set ``field`` only while it still holds ``expected``.
@@ -531,9 +531,10 @@ class MessageBus(ABC): # pylint: disable=too-many-public-methods
Field name within the registry.
value (`str`):
Serialized value to store.
- expected (`str`):
+ expected (`str | None`):
The value the caller last read. The write is skipped
- when the stored value differs.
+ when the stored value differs. ``None`` creates the field
+ only if it does not exist.
ttl_secs (`int | None`, optional):
Refresh the namespace expiry when the write happens.
@@ -542,6 +543,29 @@ class MessageBus(ABC): # pylint: disable=too-many-public-methods
``True`` when the value was written.
"""
+ async def registry_del_if( # pylint: disable=unused-argument
+ self,
+ namespace: str,
+ field: str,
+ *,
+ expected: str,
+ ) -> bool:
+ """Delete a field atomically only if its value still matches.
+
+ Backends may override this optional operation. The default leaves
+ data intact so existing adapters remain usable without an unsafe
+ read-then-delete fallback.
+
+ Args:
+ namespace (`str`): The registry key.
+ field (`str`): The field to remove.
+ expected (`str`): The previously observed value.
+
+ Returns:
+ `bool`: True if deleted; False if absent, changed, or unsupported.
+ """
+ return False
+
@abstractmethod
async def registry_pop(self, namespace: str, field: str) -> str | None:
"""Read ``field`` and remove it in one step.
diff --git a/src/agentscope/app/message_bus/_in_memory_message_bus.py b/src/agentscope/app/message_bus/_in_memory_message_bus.py
index 85385a2f3..bf5c370b5 100644
--- a/src/agentscope/app/message_bus/_in_memory_message_bus.py
+++ b/src/agentscope/app/message_bus/_in_memory_message_bus.py
@@ -506,7 +506,7 @@ class InMemoryMessageBus(
field: str,
value: str,
*,
- expected: str,
+ expected: str | None,
ttl_secs: int | None = None,
) -> bool:
"""Compare-and-set one field; ``ttl_secs`` ignored. See base."""
@@ -516,6 +516,20 @@ class InMemoryMessageBus(
self._registries.setdefault(namespace, {})[field] = value
return True
+ async def registry_del_if(
+ self,
+ namespace: str,
+ field: str,
+ *,
+ expected: str,
+ ) -> bool:
+ """Compare and delete without yielding between the two operations."""
+ registry = self._registries.get(namespace, {})
+ if registry.get(field) != expected:
+ return False
+ del registry[field]
+ return True
+
async def registry_pop(self, namespace: str, field: str) -> str | None:
"""Read and remove one field. See base."""
return self._registries.get(namespace, {}).pop(field, None)
diff --git a/src/agentscope/app/message_bus/_keys.py b/src/agentscope/app/message_bus/_keys.py
index 19c57a475..52e67e104 100644
--- a/src/agentscope/app/message_bus/_keys.py
+++ b/src/agentscope/app/message_bus/_keys.py
@@ -473,6 +473,16 @@ class MessageBusKeys: # pylint: disable=too-many-public-methods
"""Per-session background task registry key."""
return cls._BG_TASKS.format(sid=session_id)
+ @staticmethod
+ def bg_task_records(user_id: str) -> str:
+ """Shared running and terminal background task metadata per user."""
+ return f"agentscope:bg_task_records:{user_id}"
+
+ @staticmethod
+ def bg_task_records_lock(user_id: str) -> str:
+ """Serialize terminal-state retention across a user's workers."""
+ return f"agentscope:bg_task_records_lock:{user_id}"
+
# ------------------------------------------------------------------
# Knowledge-base indexing pipeline
# ------------------------------------------------------------------
diff --git a/src/agentscope/app/message_bus/_redis_message_bus.py b/src/agentscope/app/message_bus/_redis_message_bus.py
index 475b96c8c..afed73d69 100644
--- a/src/agentscope/app/message_bus/_redis_message_bus.py
+++ b/src/agentscope/app/message_bus/_redis_message_bus.py
@@ -31,7 +31,9 @@ return entries
# Compare-and-set on one hash field, so a caller can write back a value
# it decided on earlier without holding a lock across the think time.
_REGISTRY_SET_IF_LUA = """
-if redis.call('HGET', KEYS[1], ARGV[1]) ~= ARGV[3] then return 0 end
+local expected = ARGV[3]
+if ARGV[5] == '1' then expected = false end
+if redis.call('HGET', KEYS[1], ARGV[1]) ~= expected then return 0 end
redis.call('HSET', KEYS[1], ARGV[1], ARGV[2])
if tonumber(ARGV[4]) > 0 then redis.call('EXPIRE', KEYS[1], ARGV[4]) end
return 1
@@ -45,6 +47,11 @@ if value then redis.call('HDEL', KEYS[1], ARGV[1]) end
return value
"""
+_REGISTRY_DEL_IF_LUA = """
+if redis.call('HGET', KEYS[1], ARGV[1]) ~= ARGV[2] then return 0 end
+return redis.call('HDEL', KEYS[1], ARGV[1])
+"""
+
_LOCK_RELEASE_LUA = """
if redis.call('GET', KEYS[1]) == ARGV[1] then
return redis.call('DEL', KEYS[1])
@@ -517,7 +524,7 @@ class RedisMessageBus(MessageBus): # pylint: disable=too-many-public-methods
field: str,
value: str,
*,
- expected: str,
+ expected: str | None,
ttl_secs: int | None = None,
) -> bool:
"""Compare-and-set one Hash field via Lua. See base."""
@@ -527,11 +534,29 @@ class RedisMessageBus(MessageBus): # pylint: disable=too-many-public-methods
namespace,
field,
value,
- expected,
+ expected if expected is not None else "",
ttl_secs or 0,
+ int(expected is None),
)
return bool(written)
+ async def registry_del_if(
+ self,
+ namespace: str,
+ field: str,
+ *,
+ expected: str,
+ ) -> bool:
+ """Compare and delete one Hash field in a single Lua operation."""
+ deleted = await self._client.eval(
+ _REGISTRY_DEL_IF_LUA,
+ 1,
+ namespace,
+ field,
+ expected,
+ )
+ return bool(deleted)
+
async def registry_pop(self, namespace: str, field: str) -> str | None:
"""``HGET`` + ``HDEL`` as one Lua step. See base."""
return await self._client.eval(
diff --git a/src/agentscope/app/middleware/_tool_offload_middleware.py b/src/agentscope/app/middleware/_tool_offload_middleware.py
index 257e37316..e5fe4deeb 100644
--- a/src/agentscope/app/middleware/_tool_offload_middleware.py
+++ b/src/agentscope/app/middleware/_tool_offload_middleware.py
@@ -22,7 +22,7 @@ bus/wakeup infrastructure, which works correctly across processes.
import asyncio
import json
from copy import deepcopy
-from typing import AsyncGenerator, Callable
+from typing import Any, AsyncGenerator, Callable
from .._manager import BackgroundTaskManager
from ...middleware import MiddlewareBase
@@ -42,6 +42,69 @@ from ..._logging import logger
# Sentinel object used to signal end-of-stream in the drain queue.
_QUEUE_SENTINEL = object()
+_SUMMARY_MAX_LEN = 128
+
+
+def _make_summary(tool_name: str, raw_input: str | None) -> str:
+ """Generate a short human-readable summary for frontend display.
+
+ The result is guaranteed to be at most :data:`_SUMMARY_MAX_LEN`
+ characters. For dict-shaped JSON inputs, multiple key=value pairs
+ are included up to the length budget.
+
+ This truncates inputs without redacting secrets. Argument values,
+ nested values, commands, and malformed-input snippets may contain
+ sensitive data. The summary is stored in shared task snapshots and
+ returned by the user-scoped background-task REST endpoints. Redis
+ stores this field as plaintext; persistence to disk and backups depends
+ on the deployment configuration.
+
+ Args:
+ tool_name (`str`):
+ The tool name (always included as prefix).
+ raw_input (`str | None`):
+ The raw JSON-encoded tool input string.
+
+ Returns:
+ `str`:
+ A concise summary such as ``Bash(command=ls -la, ...)``.
+ """
+ if not raw_input:
+ return tool_name[:_SUMMARY_MAX_LEN]
+
+ try:
+ parsed: Any = json.loads(raw_input)
+ except (json.JSONDecodeError, TypeError):
+ snippet = raw_input[:64]
+ suffix = "..." if len(raw_input) > 64 else ""
+ result = f"{tool_name}({snippet}{suffix})"
+ return result[:_SUMMARY_MAX_LEN]
+
+ if not isinstance(parsed, dict) or not parsed:
+ return tool_name[:_SUMMARY_MAX_LEN]
+
+ prefix = f"{tool_name}("
+ suffix = ")"
+ budget = _SUMMARY_MAX_LEN - len(prefix) - len(suffix)
+ if budget <= 0:
+ return tool_name[:_SUMMARY_MAX_LEN]
+
+ parts: list[str] = []
+ used = 0
+ for key, val in parsed.items():
+ val_str = str(val)
+ if len(val_str) > 40:
+ val_str = val_str[:37] + "..."
+ part = f"{key}={val_str}"
+ separator_len = 2 if parts else 0
+ if used + len(part) + separator_len > budget:
+ parts.append("...")
+ break
+ parts.append(part)
+ used += len(part) + separator_len
+
+ return (prefix + ", ".join(parts) + suffix)[:_SUMMARY_MAX_LEN]
+
class ToolOffloadMiddleware(MiddlewareBase): # pylint: disable=abstract-method
"""Middleware that offloads timed-out tool calls to background tasks.
@@ -166,15 +229,24 @@ class ToolOffloadMiddleware(MiddlewareBase): # pylint: disable=abstract-method
# ----------------------------------------------------------------
queue: asyncio.Queue = asyncio.Queue()
- async def _drain_to_queue() -> None:
- """Drain next_handler output into *queue*."""
+ async def _drain_to_queue() -> ToolResponse | None:
+ """Drain output and return the terminal tool outcome."""
+ response = None
try:
async for item in next_handler(**input_kwargs):
+ if isinstance(item, ToolResponse):
+ response = item
await queue.put(item)
except Exception as exc: # pylint: disable=broad-except
await queue.put(exc)
+ response = ToolResponse(
+ id=tool_call.id,
+ state=ToolResultState.ERROR,
+ content=[TextBlock(text=str(exc))],
+ )
finally:
await queue.put(_QUEUE_SENTINEL)
+ return response
drain_task: asyncio.Task = asyncio.create_task(_drain_to_queue())
@@ -225,7 +297,6 @@ class ToolOffloadMiddleware(MiddlewareBase): # pylint: disable=abstract-method
# ----------------------------------------------------------------
session_id = agent.state.session_id
tool_name = tool_call.name
- snapshot = list(pre_collected)
logger.info(
"Tool '%s' timed out after %.1fs, offloading to background: "
@@ -245,7 +316,7 @@ class ToolOffloadMiddleware(MiddlewareBase): # pylint: disable=abstract-method
completion notification in those edge cases.
"""
try:
- await drain_task
+ response: ToolResponse | None = await drain_task
except asyncio.CancelledError:
logger.info(
"Background tool '%s' cancelled, skipping delivery: "
@@ -264,21 +335,12 @@ class ToolOffloadMiddleware(MiddlewareBase): # pylint: disable=abstract-method
)
return
- remaining_items: list = []
- while not queue.empty():
- try:
- item = queue.get_nowait()
- if isinstance(item, ToolResponse):
- remaining_items.append(item)
- break
- except asyncio.QueueEmpty:
- break
-
- all_items = snapshot + remaining_items
- response: ToolResponse | None = next(
- (i for i in all_items if isinstance(i, ToolResponse)),
- None,
- )
+ outcome = "completed"
+ if response is not None:
+ if response.state == ToolResultState.ERROR:
+ outcome = "failed"
+ elif response.state == ToolResultState.INTERRUPTED:
+ outcome = "been cancelled"
tool_call_id = tool_call.id
@@ -295,7 +357,7 @@ class ToolOffloadMiddleware(MiddlewareBase): # pylint: disable=abstract-method
hint=(
f"<system-notification>"
f"Tool '{tool_name}' running in background "
- f"(id={tool_call_id}) has completed with no output."
+ f"(id={tool_call_id}) has {outcome} with no output."
f"</system-notification>"
),
source=hint_source,
@@ -311,7 +373,7 @@ class ToolOffloadMiddleware(MiddlewareBase): # pylint: disable=abstract-method
prefix = (
f"<system-notification>"
f"Tool '{tool_name}' running in background "
- f"(id={tool_call_id}) has completed.\n\n"
+ f"(id={tool_call_id}) has {outcome}.\n\n"
f"Result:\n\n"
)
@@ -340,9 +402,10 @@ class ToolOffloadMiddleware(MiddlewareBase): # pylint: disable=abstract-method
# reasoning step, and WakeupDispatcher kicks an idle session
# on whichever process picks up the wakeup.
logger.info(
- "Background tool '%s' completed, pushing result to inbox "
+ "Background tool '%s' has %s, pushing result to inbox "
"and enqueueing wakeup: session_id=%s",
tool_name,
+ outcome,
session_id,
)
await deliver_to_inbox(
@@ -360,6 +423,7 @@ class ToolOffloadMiddleware(MiddlewareBase): # pylint: disable=abstract-method
agent_id=self._agent_id,
user_id=self._user_id,
tool_name=tool_name,
+ summary=_make_summary(tool_name, tool_call.input),
)
logger.info(
diff --git a/tests/background_task_endpoint_test.py b/tests/background_task_endpoint_test.py
new file mode 100644
index 000000000..e5093a55f
--- /dev/null
+++ b/tests/background_task_endpoint_test.py
@@ -0,0 +1,1377 @@
+# -*- coding: utf-8 -*-
+# pylint: disable=protected-access
+"""Regression tests for shared background task lifecycle and endpoints."""
+
+import asyncio
+import json
+from dataclasses import fields, is_dataclass, replace
+from datetime import datetime, timezone
+from unittest.async_case import IsolatedAsyncioTestCase
+from unittest.mock import AsyncMock, MagicMock, patch
+
+import fakeredis.aioredis
+import httpx
+
+from agentscope.app import create_app
+from agentscope.app._manager import (
+ BackgroundTaskManager,
+ CancelDispatcher,
+ ChatRunRegistry,
+ TaskStatus,
+)
+from agentscope.app._manager._background_task_manager import (
+ _MAX_RETAINED_TASKS,
+ BackgroundTask,
+ BackgroundTaskRecord,
+)
+from agentscope.app.message_bus import (
+ InMemoryMessageBus,
+ MessageBus,
+ MessageBusKeys,
+ RedisMessageBus,
+)
+from agentscope.app.middleware._tool_offload_middleware import (
+ _make_summary,
+)
+
+_MANAGER_MODULE = BackgroundTaskManager.__module__
+
+
+class TestBackgroundTaskCompatibility(IsolatedAsyncioTestCase):
+ """Preserve the task dataclass API that existed before this feature."""
+
+ async def test_legacy_construction_and_dataclass_operations(self) -> None:
+ """Old positional and keyword construction keep their field order."""
+ task = asyncio.create_task(asyncio.sleep(0))
+ await task
+ positional = BackgroundTask(task, "s1", "a1", "alice", "Tool", "id")
+ keyword = BackgroundTask(
+ asyncio_task=task,
+ session_id="s1",
+ agent_id="a1",
+ user_id="alice",
+ tool_name="Tool",
+ id="id",
+ started_at=positional.started_at,
+ )
+ expected = {
+ "asyncio_task": task,
+ "session_id": "s1",
+ "agent_id": "a1",
+ "user_id": "alice",
+ "tool_name": "Tool",
+ "id": "id",
+ "summary": "",
+ "started_at": positional.started_at,
+ "completed_at": None,
+ "status": TaskStatus.RUNNING,
+ "error_summary": None,
+ }
+ self.assertTrue(is_dataclass(positional))
+ self.assertEqual(
+ [item.name for item in fields(BackgroundTask)],
+ list(expected),
+ )
+ self.assertEqual(vars(positional), expected)
+ self.assertEqual(positional, keyword)
+ self.assertEqual(
+ vars(replace(positional, tool_name="Other")),
+ {**expected, "tool_name": "Other"},
+ )
+ self.assertEqual(vars(positional), expected)
+ generated = BackgroundTask(task, "s1", "a1", "alice", "Tool")
+ self.assertIsInstance(generated.id, str)
+ self.assertNotEqual(generated.id, positional.id)
+
+ async def test_shared_record_is_an_independent_snapshot(self) -> None:
+ """Serialization excludes handles and cannot mutate the local task."""
+ task = asyncio.create_task(asyncio.sleep(0))
+ await task
+ local = BackgroundTask(
+ task,
+ "s1",
+ "a1",
+ "alice",
+ "Tool",
+ "id",
+ summary="Tool(x=1)",
+ started_at=123.0,
+ )
+ record = BackgroundTaskRecord.model_validate(local)
+ expected = {
+ "id": "id",
+ "session_id": "s1",
+ "agent_id": "a1",
+ "user_id": "alice",
+ "tool_name": "Tool",
+ "summary": "Tool(x=1)",
+ "started_at": 123.0,
+ "completed_at": None,
+ "status": "running",
+ "error_summary": None,
+ }
+ self.assertEqual(json.loads(record.model_dump_json()), expected)
+ local.status = TaskStatus.COMPLETED
+ local.completed_at = 124.0
+ self.assertEqual(json.loads(record.model_dump_json()), expected)
+ record.summary = "Changed"
+ self.assertEqual(local.summary, "Tool(x=1)")
+
+
+class TestBackgroundTaskRecords( # pylint: disable=too-many-public-methods
+ IsolatedAsyncioTestCase,
+):
+ """Exercise two workers sharing task metadata and cancellation signals."""
+
+ def _make_bus(self) -> MessageBus:
+ """Provide the shared backend for both managers."""
+ return InMemoryMessageBus()
+
+ async def asyncSetUp(self) -> None:
+ """Keep the bus alive until both managers have finished cleanup."""
+ self.bus = await self.enterAsyncContext(self._make_bus())
+ self.owner = await self.enterAsyncContext(
+ BackgroundTaskManager(self.bus),
+ )
+ self.reader = await self.enterAsyncContext(
+ BackgroundTaskManager(self.bus),
+ )
+
+ async def _register(
+ self,
+ manager: BackgroundTaskManager,
+ user_id: str = "alice",
+ session_id: str = "s1",
+ ) -> tuple[asyncio.Event, asyncio.Task, str]:
+ """Register a tool that finishes only when explicitly released."""
+ gate = asyncio.Event()
+ task = asyncio.create_task(gate.wait())
+ task_id = await manager.register_task(
+ task,
+ session_id,
+ "a1",
+ user_id,
+ "Tool",
+ summary="Tool(x=1)",
+ )
+ return gate, task, task_id
+
+ async def _settle(self, *tasks: asyncio.Task) -> None:
+ """Wait for execution and all metadata writes without timing sleeps."""
+ await asyncio.gather(*tasks, return_exceptions=True)
+ for manager in (self.owner, self.reader):
+ while manager._completion_tasks:
+ await asyncio.gather(*manager._completion_tasks)
+ # Done callbacks may still be queued after gather returns.
+ await asyncio.sleep(0)
+
+ async def _renew_once(self) -> None:
+ """Run one heartbeat pass without waiting for the lease interval."""
+ self.owner._renewal_task.cancel()
+ await asyncio.gather(
+ self.owner._renewal_task,
+ return_exceptions=True,
+ )
+ with patch(
+ f"{_MANAGER_MODULE}.asyncio",
+ wraps=asyncio,
+ sleep=AsyncMock(side_effect=[None, asyncio.CancelledError]),
+ ):
+ try:
+ await self.owner._renew_running_records()
+ except asyncio.CancelledError:
+ pass
+
+ async def test_settle_waits_for_pending_done_callbacks(self) -> None:
+ """Completed tasks must be discarded before the helper returns."""
+ completed = asyncio.create_task(asyncio.sleep(0))
+ await completed
+ self.owner._completion_tasks.add(completed)
+ completed.add_done_callback(self.owner._completion_tasks.discard)
+
+ await self._settle()
+
+ self.assertEqual(self.owner._completion_tasks, set())
+ self.assertEqual(self.reader._completion_tasks, set())
+
+ async def test_shared_lifecycle_and_isolation(self) -> None:
+ """Running and completed records are identical on either worker."""
+ gate, task, task_id = await self._register(self.owner)
+ expected = BackgroundTaskRecord.model_validate(
+ self.owner.tasks[task_id],
+ ).model_dump()
+ record = await self.reader.get_task(task_id, user_id="alice")
+ self.assertEqual(record.model_dump(), expected)
+ self.assertEqual(
+ [
+ r.model_dump()
+ for r in await self.reader.list_tasks(
+ user_id="alice",
+ session_id="s1",
+ )
+ ],
+ [expected],
+ )
+ self.assertEqual(
+ await self.reader.list_tasks(user_id="alice", session_id="s2"),
+ [],
+ )
+ self.assertEqual(await self.reader.list_tasks(user_id="bob"), [])
+ self.assertIsNone(
+ await self.reader.get_task(task_id, user_id="bob"),
+ )
+ self.assertIsNone(
+ await self.reader.get_task("missing", user_id="alice"),
+ )
+
+ gate.set()
+ await self._settle(task)
+ record = await self.reader.get_task(task_id, user_id="alice")
+ self.assertIsNotNone(record.completed_at)
+ expected.update(
+ status=TaskStatus.COMPLETED,
+ completed_at=record.completed_at,
+ )
+ self.assertEqual(record.model_dump(), expected)
+ self.assertEqual(self.owner.tasks, {})
+ self.assertEqual(
+ await self.bus.registry_getall(MessageBusKeys.bg_tasks("s1")),
+ {},
+ )
+
+ async def test_exception_records_failure(self) -> None:
+ """An exception preserves its bounded error summary on any worker."""
+ gate = asyncio.Event()
+
+ async def fail() -> None:
+ await gate.wait()
+ raise ValueError("x" * 600)
+
+ task = asyncio.create_task(fail())
+ task_id = await self.owner.register_task(task, "s1", "a1", "alice")
+ expected = BackgroundTaskRecord.model_validate(
+ self.owner.tasks[task_id],
+ ).model_dump()
+ gate.set()
+ await self._settle(task)
+ record = await self.reader.get_task(task_id, user_id="alice")
+ expected.update(
+ status=TaskStatus.FAILED,
+ error_summary="x" * 512,
+ completed_at=record.completed_at,
+ )
+ self.assertIsNotNone(record.completed_at)
+ self.assertEqual(record.model_dump(), expected)
+
+ async def test_local_and_remote_toolstop(self) -> None:
+ """Both cancellation paths retain status and remove live entries."""
+ registry = await self.enterAsyncContext(ChatRunRegistry())
+ await self.enterAsyncContext(
+ CancelDispatcher(self.bus, registry, self.owner),
+ )
+ for caller in (self.owner, self.reader):
+ with self.subTest(local=caller is self.owner):
+ _, task, task_id = await self._register(self.owner)
+ expected = BackgroundTaskRecord.model_validate(
+ self.owner.tasks[task_id],
+ ).model_dump()
+ stop = (await caller.list_tools("s1"))[0]
+ await stop(task_id=task_id)
+ await asyncio.wait_for(self._settle(task), timeout=5)
+ record = await self.reader.get_task(
+ task_id,
+ user_id="alice",
+ )
+ self.assertTrue(task.cancelled())
+ self.assertIsNotNone(record.completed_at)
+ expected.update(
+ status=TaskStatus.CANCELLED,
+ completed_at=record.completed_at,
+ )
+ self.assertEqual(record.model_dump(), expected)
+ self.assertEqual(self.owner.tasks, {})
+ self.assertEqual(
+ await self.bus.registry_getall(
+ MessageBusKeys.bg_tasks("s1"),
+ ),
+ {},
+ )
+ self.assertFalse(self.owner.cancel_task(task_id))
+
+ async def test_shutdown_waits_for_shared_cleanup(self) -> None:
+ """Shutdown returns only after cancelled states and cleanup persist."""
+ _, task, task_id = await self._register(self.owner)
+ expected = BackgroundTaskRecord.model_validate(
+ self.owner.tasks[task_id],
+ ).model_dump()
+ await self.owner.__aexit__()
+ record = await self.reader.get_task(task_id, user_id="alice")
+ expected.update(
+ status=TaskStatus.CANCELLED,
+ completed_at=record.completed_at,
+ )
+ self.assertTrue(task.cancelled())
+ self.assertEqual(record.model_dump(), expected)
+ self.assertEqual(self.owner.tasks, {})
+ self.assertEqual(self.owner._completion_tasks, set())
+ self.assertEqual(
+ await self.bus.registry_getall(MessageBusKeys.bg_tasks("s1")),
+ {},
+ )
+
+ async def test_retention_is_shared_and_scoped_per_user(self) -> None:
+ """Concurrent completions evict oldest outcomes, never running work."""
+ _, _, running_id = await self._register(self.owner)
+ running = BackgroundTaskRecord.model_validate(
+ self.owner.tasks[running_id],
+ ).model_dump()
+ bob_gate, bob_task, bob_id = await self._register(
+ self.owner,
+ user_id="bob",
+ )
+ work = []
+ records = []
+ for index in range(_MAX_RETAINED_TASKS + 5):
+ manager = (self.owner, self.reader)[index % 2]
+ gate, task, task_id = await self._register(manager)
+ records.append(manager.tasks[task_id])
+ work.append((gate, task))
+ for gate, _ in work:
+ gate.set()
+ bob_gate.set()
+ await self._settle(bob_task, *(task for _, task in work))
+
+ retained = sorted(
+ records,
+ key=lambda record: (record.completed_at, record.id),
+ )[-_MAX_RETAINED_TASKS:]
+ expected = [running] + [
+ BackgroundTaskRecord.model_validate(record).model_dump()
+ for record in retained
+ ]
+ expected.sort(key=lambda record: (record["started_at"], record["id"]))
+ self.assertEqual(
+ [
+ record.model_dump()
+ for record in await self.reader.list_tasks(
+ user_id="alice",
+ )
+ ],
+ expected,
+ )
+ bob_record = await self.reader.get_task(bob_id, user_id="bob")
+ self.assertEqual(
+ [
+ record.model_dump()
+ for record in await self.reader.list_tasks(
+ user_id="bob",
+ )
+ ],
+ [bob_record.model_dump()],
+ )
+ self.assertEqual(bob_record.status, TaskStatus.COMPLETED)
+ self.assertEqual(set(self.owner.tasks), {running_id})
+ self.assertEqual(self.reader.tasks, {})
+
+ async def test_failed_initial_write_still_cleans_up(self) -> None:
+ """A partial registration cannot leave an untracked running task."""
+ original_set = self.bus.registry_set
+
+ async def fail_running_record(
+ namespace: str,
+ field: str,
+ value: str,
+ **kwargs: int,
+ ) -> None:
+ if (
+ namespace == MessageBusKeys.bg_task_records("alice")
+ and json.loads(value)["status"] == "running"
+ ):
+ raise ConnectionError("record write failed")
+ await original_set(namespace, field, value, **kwargs)
+
+ task = asyncio.create_task(asyncio.Event().wait())
+ with patch.object(
+ self.bus,
+ "registry_set",
+ side_effect=fail_running_record,
+ ):
+ with self.assertRaisesRegex(ConnectionError, "record write"):
+ await self.owner.register_task(task, "s1", "a1", "alice")
+ await self._settle(task)
+ self.assertTrue(task.cancelled())
+ self.assertEqual(self.owner.tasks, {})
+ self.assertEqual(
+ await self.bus.registry_getall(MessageBusKeys.bg_tasks("s1")),
+ {},
+ )
+
+ async def test_initial_write_timeout_cleans_up_all_outcomes(self) -> None:
+ """Either stalled initial write releases registration and handles."""
+ original_set = self.bus.registry_set
+ for namespace in (
+ MessageBusKeys.bg_tasks("s1"),
+ MessageBusKeys.bg_task_records("alice"),
+ ):
+ for completed in (False, True):
+ with self.subTest(namespace=namespace, completed=completed):
+ started = asyncio.Event()
+ cancelled = asyncio.Event()
+ gate = asyncio.Event()
+ task = asyncio.create_task(gate.wait())
+
+ async def stall_initial_write(
+ ns: str,
+ field: str,
+ value: str,
+ *,
+ ttl_secs: int | None = None,
+ ) -> None:
+ # Each subtest finishes this write before advancing.
+ # pylint: disable=cell-var-from-loop
+ payload = json.loads(value)
+ if (
+ ns == namespace
+ and payload.get("status", "running") == "running"
+ ):
+ # The server may commit before the reply stalls.
+ await original_set(
+ ns,
+ field,
+ value,
+ ttl_secs=ttl_secs,
+ )
+ started.set()
+ try:
+ await asyncio.Event().wait()
+ finally:
+ cancelled.set()
+ await original_set(
+ ns,
+ field,
+ value,
+ ttl_secs=ttl_secs,
+ )
+
+ with (
+ patch.object(
+ self.bus,
+ "registry_set",
+ side_effect=stall_initial_write,
+ ),
+ patch(f"{_MANAGER_MODULE}._IO_TIMEOUT_SECS", 0.1),
+ ):
+ registration = asyncio.create_task(
+ self.owner.register_task(
+ task,
+ "s1",
+ "a1",
+ "alice",
+ ),
+ )
+ try:
+ await asyncio.wait_for(started.wait(), timeout=1)
+ (task_id,) = self.owner.tasks
+ expected = BackgroundTaskRecord.model_validate(
+ self.owner.tasks[task_id],
+ ).model_dump()
+ if completed:
+ gate.set()
+ await task
+ done, _ = await asyncio.wait(
+ {registration},
+ timeout=0.5,
+ )
+ self.assertEqual(done, {registration})
+ with self.assertRaises(TimeoutError):
+ await registration
+ self.assertTrue(cancelled.is_set())
+ await asyncio.wait_for(self._settle(task), 1)
+ finally:
+ registration.cancel()
+ await asyncio.gather(
+ registration,
+ return_exceptions=True,
+ )
+ self.assertEqual(self.owner.tasks, {})
+ self.assertEqual(task.cancelled(), not completed)
+ record = await self.reader.get_task(
+ task_id,
+ user_id="alice",
+ )
+ expected.update(
+ status=TaskStatus.COMPLETED
+ if completed
+ else TaskStatus.CANCELLED,
+ completed_at=record.completed_at,
+ )
+ self.assertEqual(record.model_dump(), expected)
+ self.assertEqual(
+ await self.bus.registry_getall(
+ MessageBusKeys.bg_tasks("s1"),
+ ),
+ {},
+ )
+
+ async def test_failed_outcome_write_recovers_shared_status(self) -> None:
+ """A failed write remains pending and is retried after recovery."""
+ gate, task, task_id = await self._register(self.owner)
+ expected = BackgroundTaskRecord.model_validate(
+ self.owner.tasks[task_id],
+ ).model_dump()
+ failed = asyncio.Event()
+
+ async def unavailable(_record: BackgroundTaskRecord) -> None:
+ failed.set()
+ raise ConnectionError("unavailable")
+
+ with patch.object(
+ self.owner,
+ "_store_record",
+ side_effect=unavailable,
+ ):
+ gate.set()
+ await asyncio.wait_for(failed.wait(), timeout=1)
+ self.assertEqual(self.owner.tasks, {})
+ self.assertTrue(self.owner._completion_tasks)
+ record = await self.reader.get_task(task_id, user_id="alice")
+ self.assertEqual(record.model_dump(), expected)
+ await asyncio.wait_for(self._settle(task), timeout=2)
+ record = await self.reader.get_task(task_id, user_id="alice")
+ self.assertIsNotNone(record.completed_at)
+ expected.update(
+ status=TaskStatus.COMPLETED,
+ completed_at=record.completed_at,
+ )
+ self.assertEqual(record.model_dump(), expected)
+ self.assertEqual(
+ await self.bus.registry_getall(MessageBusKeys.bg_tasks("s1")),
+ {},
+ )
+ self.assertEqual(self.owner._completion_tasks, set())
+
+ async def test_failed_shutdown_record_expires_despite_new_tasks(
+ self,
+ ) -> None:
+ """A lost outcome cannot remain running through user-hash renewal."""
+ gate, task, task_id = await self._register(self.owner)
+ namespace = MessageBusKeys.bg_task_records("alice")
+ value = await self.bus.registry_get(namespace, task_id)
+ expires_at = json.loads(value)["_expires_at"]
+ failed = asyncio.Event()
+
+ async def unavailable(_record: BackgroundTaskRecord) -> None:
+ failed.set()
+ raise ConnectionError("unavailable")
+
+ with (
+ patch.object(self.owner, "_store_record", side_effect=unavailable),
+ patch(f"{_MANAGER_MODULE}._SHUTDOWN_TIMEOUT_SECS", 0.1),
+ ):
+ gate.set()
+ await asyncio.wait_for(failed.wait(), timeout=1)
+ await asyncio.wait_for(self.owner.__aexit__(), timeout=0.5)
+ self.assertTrue(task.done())
+ self.assertEqual(self.owner._completion_tasks, set())
+ self.assertEqual(
+ await self.bus.registry_get(namespace, task_id),
+ value,
+ )
+
+ with patch(
+ f"{_MANAGER_MODULE}.time.time",
+ return_value=expires_at + 1,
+ ):
+ _, _, new_id = await self._register(self.reader)
+ expected = BackgroundTaskRecord.model_validate(
+ self.reader.tasks[new_id],
+ ).model_dump()
+ self.assertIsNone(
+ await self.reader.get_task(task_id, user_id="alice"),
+ )
+ self.assertEqual(
+ [
+ record.model_dump()
+ for record in await self.reader.list_tasks(
+ user_id="alice",
+ )
+ ],
+ [expected],
+ )
+ # Querying removes the expired field despite hash TTL renewal.
+ self.assertIsNone(
+ await self.bus.registry_get(namespace, task_id),
+ )
+
+ async def test_lists_remove_expired_records_as_user_activity_continues(
+ self,
+ ) -> None:
+ """Repeated user activity cannot accumulate invisible running rows."""
+ namespace = MessageBusKeys.bg_task_records("alice")
+ expected = {}
+ for batch in range(3):
+ _, _, task_id = await self._register(self.owner)
+ value = await self.bus.registry_get(namespace, task_id)
+ expected[task_id] = value
+ for index in range(5):
+ stale_id = f"stale-{batch}-{index}"
+ payload = json.loads(value)
+ payload.update(id=stale_id, _expires_at=0)
+ await self.bus.registry_set(
+ namespace,
+ stale_id,
+ json.dumps(payload),
+ ttl_secs=MessageBusKeys.BG_TASKS_TTL_SECS,
+ )
+ records = await self.reader.list_tasks(user_id="alice")
+ self.assertEqual(
+ {record.id: record.model_dump() for record in records},
+ {
+ task_id: BackgroundTaskRecord.model_validate_json(
+ value,
+ ).model_dump()
+ for task_id, value in expected.items()
+ },
+ )
+ self.assertEqual(
+ await self.bus.registry_getall(namespace),
+ expected,
+ )
+
+ async def test_completion_evicts_expired_running_and_terminal_records(
+ self,
+ ) -> None:
+ """Finalization physically clears expired records of every status."""
+ gate, task, task_id = await self._register(self.owner)
+ namespace = MessageBusKeys.bg_task_records("alice")
+ value = await self.bus.registry_get(namespace, task_id)
+ for status in TaskStatus:
+ stale_id = f"expired-{status.value}"
+ payload = json.loads(value)
+ payload.update(
+ id=stale_id,
+ status=status.value,
+ _expires_at=0,
+ completed_at=None if status == TaskStatus.RUNNING else 1.0,
+ )
+ await self.bus.registry_set(
+ namespace,
+ stale_id,
+ json.dumps(payload),
+ )
+ gate.set()
+ await self._settle(task)
+ self.assertEqual(
+ await self.bus.registry_getall(namespace),
+ {task_id: await self.bus.registry_get(namespace, task_id)},
+ )
+
+ async def test_expiry_cleanup_preserves_concurrent_updates(self) -> None:
+ """Queries and eviction cannot delete renewed or completed records."""
+ namespace = MessageBusKeys.bg_task_records("alice")
+ original_delete = self.bus.registry_del_if
+ original_set = self.bus.registry_set
+ for operation in ("list", "detail", "eviction"):
+ for status in (TaskStatus.RUNNING, TaskStatus.COMPLETED):
+ with self.subTest(operation=operation, status=status):
+ task_id = f"race-{operation}-{status.value}"
+ payload = {
+ "id": task_id,
+ "session_id": "s1",
+ "agent_id": "a1",
+ "user_id": "alice",
+ "tool_name": "Tool",
+ "summary": "Tool(x=1)",
+ "started_at": 0,
+ "completed_at": None,
+ "status": "running",
+ "error_summary": None,
+ "_expires_at": 0,
+ }
+ await original_set(namespace, task_id, json.dumps(payload))
+ payload.update(
+ status=status.value,
+ completed_at=1.0
+ if status == TaskStatus.COMPLETED
+ else None,
+ _expires_at=32503680000,
+ )
+ updated = json.dumps(payload)
+
+ async def concurrent_update(
+ ns: str,
+ field: str,
+ *,
+ expected: str,
+ new_value: str = updated,
+ ) -> bool:
+ await original_set(ns, field, new_value)
+ return await original_delete(
+ ns,
+ field,
+ expected=expected,
+ )
+
+ with patch.object(
+ self.bus,
+ "registry_del_if",
+ side_effect=concurrent_update,
+ ):
+ if operation == "list":
+ await self.reader.list_tasks(user_id="alice")
+ elif operation == "detail":
+ await self.reader.get_task(
+ task_id,
+ user_id="alice",
+ )
+ else:
+ await self.reader._evict_completed("alice")
+ self.assertEqual(
+ await self.bus.registry_get(namespace, task_id),
+ updated,
+ )
+ record = await self.reader.get_task(
+ task_id,
+ user_id="alice",
+ )
+ self.assertEqual(
+ record.model_dump(),
+ BackgroundTaskRecord.model_validate(
+ payload,
+ ).model_dump(),
+ )
+
+ async def test_retention_lock_does_not_block_outcome_or_shutdown(
+ self,
+ ) -> None:
+ """A held retention lock cannot delay state writes or exhaust exit."""
+ gate, task, task_id = await self._register(self.owner)
+ expected = BackgroundTaskRecord.model_validate(
+ self.owner.tasks[task_id],
+ ).model_dump()
+ cleaned = asyncio.Event()
+ original_del = self.bus.registry_del
+
+ async def observe_cleanup(namespace: str, field: str) -> None:
+ await original_del(namespace, field)
+ if namespace == MessageBusKeys.bg_tasks("s1"):
+ cleaned.set()
+
+ async with self.bus.acquire_lock(
+ MessageBusKeys.bg_task_records_lock("alice"),
+ ttl_secs=600,
+ ):
+ with (
+ patch.object(
+ self.bus,
+ "registry_del",
+ side_effect=observe_cleanup,
+ ),
+ patch(f"{_MANAGER_MODULE}._SHUTDOWN_TIMEOUT_SECS", 0.1),
+ patch(f"{_MANAGER_MODULE}._IO_TIMEOUT_SECS", 0.02),
+ ):
+ gate.set()
+ await asyncio.wait_for(cleaned.wait(), timeout=1)
+ self.assertTrue(task.done())
+ self.assertEqual(self.owner.tasks, {})
+ record = await self.reader.get_task(task_id, user_id="alice")
+ expected.update(
+ status=TaskStatus.COMPLETED,
+ completed_at=record.completed_at,
+ )
+ self.assertEqual(record.model_dump(), expected)
+ await asyncio.wait_for(self.owner.__aexit__(), timeout=0.5)
+ self.assertEqual(self.owner._completion_tasks, set())
+
+ async def test_stalled_outcome_io_times_out_and_retries(self) -> None:
+ """A nonresponsive write gets a deadline and can recover on retry."""
+ gate, task, task_id = await self._register(self.owner)
+ stalled = asyncio.Event()
+ original_store = self.owner._store_record
+
+ async def stall_once(record: BackgroundTaskRecord) -> None:
+ if not stalled.is_set():
+ stalled.set()
+ await asyncio.Event().wait()
+ await original_store(record)
+
+ with (
+ patch.object(self.owner, "_store_record", side_effect=stall_once),
+ patch(f"{_MANAGER_MODULE}._IO_TIMEOUT_SECS", 0.02),
+ ):
+ gate.set()
+ await asyncio.wait_for(stalled.wait(), timeout=1)
+ await asyncio.wait_for(self._settle(task), timeout=2)
+ record = await self.reader.get_task(task_id, user_id="alice")
+ self.assertEqual(record.status, TaskStatus.COMPLETED)
+ self.assertEqual(
+ await self.bus.registry_getall(MessageBusKeys.bg_tasks("s1")),
+ {},
+ )
+
+ async def test_shutdown_cancels_stalled_cleanup_io(self) -> None:
+ """Shutdown is bounded even if a registry operation never responds."""
+ gate, task, task_id = await self._register(self.owner)
+ blocked = asyncio.Event()
+ cancelled = asyncio.Event()
+
+ async def stall_delete(_namespace: str, _field: str) -> None:
+ blocked.set()
+ try:
+ await asyncio.Event().wait()
+ finally:
+ cancelled.set()
+
+ with (
+ patch.object(self.bus, "registry_del", side_effect=stall_delete),
+ patch(f"{_MANAGER_MODULE}._SHUTDOWN_TIMEOUT_SECS", 0.1),
+ ):
+ gate.set()
+ await asyncio.wait_for(blocked.wait(), timeout=1)
+ self.assertTrue(task.done())
+ await asyncio.wait_for(self.owner.__aexit__(), timeout=0.5)
+ self.assertTrue(cancelled.is_set())
+ self.assertEqual(self.owner._completion_tasks, set())
+ record = await self.reader.get_task(task_id, user_id="alice")
+ self.assertEqual(record.status, TaskStatus.COMPLETED)
+
+ async def test_running_task_renews_its_own_lease(self) -> None:
+ """Long-running tasks stay visible beyond their initial lease."""
+ renewed = asyncio.Event()
+ original_set_if = self.bus.registry_set_if
+
+ async def observe_renewal(
+ namespace: str,
+ field: str,
+ value: str,
+ *,
+ expected: str,
+ ttl_secs: int | None = None,
+ ) -> bool:
+ result = await original_set_if(
+ namespace,
+ field,
+ value,
+ expected=expected,
+ ttl_secs=ttl_secs,
+ )
+ renewed.set()
+ return result
+
+ with patch(f"{_MANAGER_MODULE}._RECORD_LEASE_SECS", 0.04):
+ gate, task, task_id = await self._register(self.owner)
+ value = await self.bus.registry_get(
+ MessageBusKeys.bg_task_records("alice"),
+ task_id,
+ )
+ future_time = json.loads(value)["_expires_at"] + 1
+ with (
+ patch(
+ f"{_MANAGER_MODULE}.time.time",
+ return_value=future_time,
+ ),
+ patch.object(
+ self.bus,
+ "registry_set_if",
+ side_effect=observe_renewal,
+ ),
+ ):
+ await asyncio.wait_for(renewed.wait(), timeout=1)
+ expected = BackgroundTaskRecord.model_validate(
+ self.owner.tasks[task_id],
+ ).model_dump()
+ record = await self.reader.get_task(task_id, user_id="alice")
+ self.assertEqual(record.model_dump(), expected)
+ gate.set()
+ await self._settle(task)
+
+ async def test_slow_io_keeps_all_running_tasks_visible(self) -> None:
+ """Renewal latency must not accumulate across every running task."""
+ original_get = self.bus.registry_get
+ original_set_if = self.bus.registry_set_if
+
+ async def delayed_get(namespace: str, field: str) -> str | None:
+ await asyncio.sleep(0.04)
+ return await original_get(namespace, field)
+
+ async def delayed_set_if(
+ namespace: str,
+ field: str,
+ value: str,
+ *,
+ expected: str | None,
+ ttl_secs: int | None = None,
+ ) -> bool:
+ await asyncio.sleep(0.04)
+ return await original_set_if(
+ namespace,
+ field,
+ value,
+ expected=expected,
+ ttl_secs=ttl_secs,
+ )
+
+ with patch(f"{_MANAGER_MODULE}._RECORD_LEASE_SECS", 0.6):
+ for _ in range(8):
+ await self._register(self.owner)
+ expected = {
+ task.id: BackgroundTaskRecord.model_validate(task).model_dump()
+ for task in self.owner.tasks.values()
+ }
+ with (
+ patch.object(
+ self.bus,
+ "registry_get",
+ side_effect=delayed_get,
+ ),
+ patch.object(
+ self.bus,
+ "registry_set_if",
+ side_effect=delayed_set_if,
+ ),
+ ):
+ await asyncio.sleep(0.65)
+ for _ in range(4):
+ records = await self.reader.list_tasks(user_id="alice")
+ self.assertEqual(
+ {record.id: record.model_dump() for record in records},
+ expected,
+ )
+ self.assertEqual(
+ [
+ task.asyncio_task.done()
+ for task in self.owner.tasks.values()
+ ],
+ [False] * 8,
+ )
+ await asyncio.sleep(0.1)
+
+ async def test_renewal_budget_preserves_fairness_and_cancels_io(
+ self,
+ ) -> None:
+ """Slow tasks cannot starve the queue or leave I/O after shutdown."""
+ original_get = self.bus.registry_get
+ active: set[asyncio.Task] = set()
+ seen: set[str] = set()
+ peak = 0
+ cancellations = 0
+ reached_tail = asyncio.Event()
+ with (
+ patch(f"{_MANAGER_MODULE}._RECORD_LEASE_SECS", 0.2),
+ patch(f"{_MANAGER_MODULE}._RENEWAL_CONCURRENCY", 2),
+ ):
+ for _ in range(6):
+ await self._register(self.owner)
+ task_ids = list(self.owner.tasks)
+
+ async def stalled_head(namespace: str, field: str) -> str | None:
+ nonlocal peak, cancellations
+ current = asyncio.current_task()
+ assert current is not None
+ active.add(current)
+ peak = max(peak, len(active))
+ try:
+ seen.add(field)
+ if seen == set(task_ids):
+ reached_tail.set()
+ if field in task_ids[:2]:
+ try:
+ await asyncio.Event().wait()
+ except asyncio.CancelledError:
+ cancellations += 1
+ raise
+ return await original_get(namespace, field)
+ finally:
+ active.remove(current)
+
+ with patch.object(
+ self.bus,
+ "registry_get",
+ side_effect=stalled_head,
+ ):
+ await asyncio.wait_for(reached_tail.wait(), timeout=0.5)
+ self.assertEqual(peak, 2)
+ self.assertGreaterEqual(cancellations, 2)
+ self.assertEqual(seen, set(task_ids))
+ await asyncio.wait_for(self.owner.__aexit__(), timeout=1)
+ self.assertEqual(active, set())
+ self.assertTrue(self.owner._renewal_task.done())
+
+ async def test_registration_during_round_timeout_keeps_one_scheduler(
+ self,
+ ) -> None:
+ """Temporary timeout cancellation must not start a second scheduler."""
+ cancelling = asyncio.Event()
+ allow_cleanup = asyncio.Event()
+ resumed = asyncio.Event()
+ original_renew = self.owner._renew_running_record
+
+ async def delayed_cleanup(task: BackgroundTask) -> None:
+ if not allow_cleanup.is_set():
+ try:
+ await asyncio.Event().wait()
+ except asyncio.CancelledError:
+ cancelling.set()
+ await allow_cleanup.wait()
+ raise
+ await original_renew(task)
+ resumed.set()
+
+ with (
+ patch(f"{_MANAGER_MODULE}._RECORD_LEASE_SECS", 0.2),
+ patch(f"{_MANAGER_MODULE}._RENEWAL_CONCURRENCY", 2),
+ patch.object(
+ self.owner,
+ "_renew_running_record",
+ side_effect=delayed_cleanup,
+ ),
+ ):
+ await self._register(self.owner)
+ scheduler = self.owner._renewal_task
+ try:
+ await asyncio.wait_for(cancelling.wait(), timeout=2)
+ self.assertGreater(scheduler.cancelling(), 0)
+ self.assertFalse(scheduler.done())
+ for _ in range(3):
+ await self._register(self.owner)
+ self.assertIs(self.owner._renewal_task, scheduler)
+
+ allow_cleanup.set()
+ await asyncio.wait_for(resumed.wait(), timeout=2)
+ self.assertIs(self.owner._renewal_task, scheduler)
+ self.assertFalse(scheduler.done())
+ await self.owner.__aexit__()
+ self.assertTrue(scheduler.done())
+ self.assertIs(self.owner._renewal_task, scheduler)
+ finally:
+ allow_cleanup.set()
+ scheduler.cancel()
+ await asyncio.gather(scheduler, return_exceptions=True)
+
+ async def test_renewal_reuses_live_scheduler_and_restarts_after_exit(
+ self,
+ ) -> None:
+ """New work reuses a live scheduler, including its idle interval."""
+ sleeping = asyncio.Event()
+ release_sleep = asyncio.Event()
+ original_sleep = asyncio.sleep
+
+ async def hold_first_sleep(delay: float) -> None:
+ if not sleeping.is_set():
+ sleeping.set()
+ try:
+ await release_sleep.wait()
+ except asyncio.CancelledError:
+ await release_sleep.wait()
+ raise
+ else:
+ await original_sleep(delay)
+
+ with (
+ patch(f"{_MANAGER_MODULE}._RECORD_LEASE_SECS", 0.2),
+ patch(
+ f"{_MANAGER_MODULE}.asyncio",
+ wraps=asyncio,
+ sleep=AsyncMock(side_effect=hold_first_sleep),
+ ),
+ ):
+ gate, task, _ = await self._register(self.owner)
+ scheduler = self.owner._renewal_task
+ try:
+ await asyncio.wait_for(sleeping.wait(), timeout=2)
+ gate.set()
+ await self._settle(task)
+ self.assertEqual(scheduler.cancelling(), 0)
+ self.assertFalse(scheduler.done())
+
+ gate, task, _ = await self._register(self.owner)
+ self.assertIs(self.owner._renewal_task, scheduler)
+ gate.set()
+ await self._settle(task)
+ release_sleep.set()
+ await asyncio.wait_for(asyncio.shield(scheduler), timeout=2)
+
+ await self._register(self.owner)
+ replacement = self.owner._renewal_task
+ self.assertIsNot(replacement, scheduler)
+ self.assertFalse(replacement.done())
+ await self.owner.__aexit__()
+ self.assertTrue(replacement.done())
+ finally:
+ release_sleep.set()
+ scheduler.cancel()
+ await asyncio.gather(scheduler, return_exceptions=True)
+
+ async def test_heartbeat_restores_a_cleaned_running_record(self) -> None:
+ """A live task reappears after its expired snapshot is deleted."""
+ _, _, task_id = await self._register(self.owner)
+ namespace = MessageBusKeys.bg_task_records("alice")
+ value = await self.bus.registry_get(namespace, task_id)
+ payload = json.loads(value)
+ payload["_expires_at"] = 0
+ await self.bus.registry_set(namespace, task_id, json.dumps(payload))
+ self.assertEqual(await self.reader.list_tasks(user_id="alice"), [])
+ self.assertIsNone(await self.bus.registry_get(namespace, task_id))
+
+ await self._renew_once()
+
+ expected = BackgroundTaskRecord.model_validate(
+ self.owner.tasks[task_id],
+ ).model_dump()
+ record = await self.reader.get_task(task_id, user_id="alice")
+ self.assertIsNotNone(record)
+ self.assertEqual(record.model_dump(), expected)
+ self.assertEqual(
+ [
+ r.model_dump()
+ for r in await self.reader.list_tasks(
+ user_id="alice",
+ )
+ ],
+ [expected],
+ )
+
+ async def test_heartbeat_restore_preserves_concurrent_outcome(
+ self,
+ ) -> None:
+ """An outcome written after a missing read wins over recovery."""
+ _, _, task_id = await self._register(self.owner)
+ namespace = MessageBusKeys.bg_task_records("alice")
+ await self.bus.registry_del(namespace, task_id)
+ terminal = BackgroundTaskRecord.model_validate(
+ self.owner.tasks[task_id],
+ )
+ terminal.status = TaskStatus.COMPLETED
+ terminal.completed_at = terminal.started_at + 1
+ original_set_if = self.bus.registry_set_if
+
+ async def complete_before_write(
+ namespace: str,
+ field: str,
+ value: str,
+ *,
+ expected: str | None,
+ ttl_secs: int | None = None,
+ ) -> bool:
+ await self.owner._store_record(terminal)
+ return await original_set_if(
+ namespace,
+ field,
+ value,
+ expected=expected,
+ ttl_secs=ttl_secs,
+ )
+
+ with patch.object(
+ self.bus,
+ "registry_set_if",
+ side_effect=complete_before_write,
+ ) as restore:
+ await self._renew_once()
+ restore.assert_awaited_once()
+ self.assertIsNone(restore.call_args.kwargs["expected"])
+ record = await self.reader.get_task(task_id, user_id="alice")
+ self.assertEqual(record.model_dump(), terminal.model_dump())
+
+ async def test_heartbeat_skips_task_completed_during_read(self) -> None:
+ """A missing read cannot revive a task that has already finished."""
+ gate, task, task_id = await self._register(self.owner)
+ namespace = MessageBusKeys.bg_task_records("alice")
+ await self.bus.registry_del(namespace, task_id)
+ original_get = self.bus.registry_get
+
+ async def complete_during_read(
+ namespace: str,
+ field: str,
+ ) -> str | None:
+ value = await original_get(namespace, field)
+ gate.set()
+ await self._settle(task)
+ return value
+
+ with (
+ patch.object(
+ self.bus,
+ "registry_get",
+ side_effect=complete_during_read,
+ ),
+ patch.object(
+ self.bus,
+ "registry_set_if",
+ wraps=self.bus.registry_set_if,
+ ) as restore,
+ ):
+ await self._renew_once()
+ restore.assert_not_awaited()
+ record = await self.reader.get_task(task_id, user_id="alice")
+ self.assertEqual(record.status, TaskStatus.COMPLETED)
+
+ async def test_http_routes_read_another_workers_records(self) -> None:
+ """The actual app exposes filtered lists, details and isolated 404s."""
+ gate, task, task_id = await self._register(self.owner)
+ record = await self.reader.get_task(task_id, user_id="alice")
+ expected = {
+ "task_id": task_id,
+ "session_id": "s1",
+ "agent_id": "a1",
+ "tool_name": "Tool",
+ "summary": "Tool(x=1)",
+ "status": "running",
+ "started_at": datetime.fromtimestamp(
+ record.started_at,
+ tz=timezone.utc,
+ )
+ .isoformat()
+ .replace("+00:00", "Z"),
+ "completed_at": None,
+ "error_summary": None,
+ }
+ app = create_app(
+ storage=MagicMock(),
+ message_bus=self.bus,
+ workspace_manager=MagicMock(),
+ )
+ app.state.background_task_manager = self.reader
+ client = await self.enterAsyncContext(
+ httpx.AsyncClient(
+ transport=httpx.ASGITransport(app=app),
+ base_url="http://test",
+ ),
+ )
+ headers = {"X-User-ID": "alice"}
+ for query, body in (
+ ("", {"tasks": [expected], "total": 1}),
+ ("?session_id=s1", {"tasks": [expected], "total": 1}),
+ ("?session_id=s2", {"tasks": [], "total": 0}),
+ ):
+ response = await client.get(
+ f"/background-tasks{query}",
+ headers=headers,
+ )
+ self.assertEqual(
+ (response.status_code, response.json()),
+ (
+ 200,
+ body,
+ ),
+ )
+ response = await client.get(
+ f"/background-tasks/{task_id}",
+ headers=headers,
+ )
+ self.assertEqual(
+ (response.status_code, response.json()),
+ (
+ 200,
+ expected,
+ ),
+ )
+ for missing_id, user in ((task_id, "bob"), ("missing", "alice")):
+ response = await client.get(
+ f"/background-tasks/{missing_id}",
+ headers={"X-User-ID": user},
+ )
+ self.assertEqual(
+ (response.status_code, response.json()),
+ (
+ 404,
+ {"detail": f"Background task '{missing_id}' not found."},
+ ),
+ )
+ response = await client.get("/background-tasks")
+ self.assertEqual(response.status_code, 422)
+ gate.set()
+ await self._settle(task)
+ record = await self.reader.get_task(task_id, user_id="alice")
+ expected.update(
+ status="completed",
+ completed_at=datetime.fromtimestamp(
+ record.completed_at,
+ tz=timezone.utc,
+ )
+ .isoformat()
+ .replace("+00:00", "Z"),
+ )
+ response = await client.get(
+ f"/background-tasks/{task_id}",
+ headers=headers,
+ )
+ self.assertEqual(
+ (response.status_code, response.json()),
+ (
+ 200,
+ expected,
+ ),
+ )
+
+
+class TestRedisBackgroundTaskRecords(TestBackgroundTaskRecords):
+ """Run the same worker and lifecycle regressions through Redis APIs."""
+
+ def _make_bus(self) -> MessageBus:
+ """Use a fake Redis server without opening network connections."""
+ client = fakeredis.aioredis.FakeRedis(decode_responses=True)
+ self.addAsyncCleanup(client.aclose)
+ return RedisMessageBus(connection_pool=client.connection_pool)
+
+
+class TestMakeSummary(IsolatedAsyncioTestCase):
+ """Test the _make_summary helper function."""
+
+ async def test_empty_input_returns_tool_name(self) -> None:
+ """Empty or None input returns just the tool name."""
+ self.assertEqual(_make_summary("Bash", None), "Bash")
+ self.assertEqual(_make_summary("Bash", ""), "Bash")
+
+ async def test_dict_input_includes_key_values(self) -> None:
+ """JSON dict input produces key=value summary."""
+ raw = json.dumps({"command": "ls -la", "timeout": 30})
+ result = _make_summary("Bash", raw)
+ self.assertIn("Bash(", result)
+ self.assertIn("command=ls -la", result)
+ self.assertIn("timeout=30", result)
+ self.assertTrue(result.endswith(")"))
+
+ async def test_respects_max_length(self) -> None:
+ """Summary never exceeds _SUMMARY_MAX_LEN characters."""
+ raw = json.dumps({"key": "x" * 200, "other": "y" * 200})
+ result = _make_summary("VeryLongToolName", raw)
+ self.assertLessEqual(len(result), 128)
+
+ async def test_non_json_input_is_truncated(self) -> None:
+ """Non-JSON input is included as-is with truncation."""
+ raw = "a" * 100
+ result = _make_summary("Tool", raw)
+ self.assertIn("Tool(", result)
+ self.assertIn("...", result)
+ self.assertLessEqual(len(result), 128)
+
+ async def test_non_json_short_input_no_ellipsis(self) -> None:
+ """Short non-JSON input doesn't get a misleading ellipsis."""
+ result = _make_summary("Tool", "hi")
+ self.assertEqual(result, "Tool(hi)")
+
+ async def test_empty_dict_returns_tool_name(self) -> None:
+ """Empty JSON dict returns just the tool name."""
+ result = _make_summary("Tool", "{}")
+ self.assertEqual(result, "Tool")
+
+ async def test_non_dict_json_returns_tool_name(self) -> None:
+ """JSON that's not a dict returns just the tool name."""
+ result = _make_summary("Tool", "[1, 2, 3]")
+ self.assertEqual(result, "Tool")
+
+ async def test_long_values_are_truncated(self) -> None:
+ """Individual values longer than 40 chars are truncated."""
+ raw = json.dumps({"file": "/" + "a" * 100})
+ result = _make_summary("Read", raw)
+ self.assertIn("...", result)
+ self.assertLessEqual(len(result), 128)
+
+ async def test_multiple_keys_included_within_budget(self) -> None:
+ """Multiple short key-values are all included."""
+ raw = json.dumps({"a": "1", "b": "2", "c": "3"})
+ result = _make_summary("T", raw)
+ self.assertIn("a=1", result)
+ self.assertIn("b=2", result)
+ self.assertIn("c=3", result)
diff --git a/tests/background_task_manager_test.py b/tests/background_task_manager_test.py
index d65c49bd8..2858d277d 100644
--- a/tests/background_task_manager_test.py
+++ b/tests/background_task_manager_test.py
@@ -33,6 +33,14 @@ class _RegistrationBus(InMemoryMessageBus):
ttl_secs: int | None = None,
) -> None:
"""Write before or after the acknowledgement barrier."""
+ if namespace != MessageBusKeys.bg_tasks("session-a"):
+ await super().registry_set(
+ namespace,
+ field,
+ value,
+ ttl_secs=ttl_secs,
+ )
+ return
if self.commit_before_wait:
await super().registry_set(
namespace,
@@ -79,6 +87,7 @@ class BackgroundTaskManagerTest(IsolatedAsyncioTestCase):
task.cancel()
await asyncio.gather(*tasks, return_exceptions=True)
await self._checkpoint()
+ await self.manager.__aexit__()
@staticmethod
async def _checkpoint() -> None:
@@ -110,6 +119,7 @@ class BackgroundTaskManagerTest(IsolatedAsyncioTestCase):
user_id="user-a",
tool_name="slow_tool",
id=task_id,
+ started_at=self.manager.tasks[task_id].started_at,
),
},
)
@@ -137,7 +147,7 @@ class BackgroundTaskManagerTest(IsolatedAsyncioTestCase):
cancelled: bool,
complete_during_write: bool = False,
) -> None:
- """Verify faults preserve running tasks and clean up finished ones."""
+ """Verify faults cancel running tasks and clean up all outcomes."""
self.bus.commit_before_wait = committed
error = ConnectionError("registry acknowledgement failed")
self.bus.write_error = error
@@ -157,12 +167,11 @@ class BackgroundTaskManagerTest(IsolatedAsyncioTestCase):
with self.assertRaises(ConnectionError) as raised:
await self.registration
self.assertIs(raised.exception, error)
- self.assertEqual(self.task.done(), complete_during_write)
- if not complete_during_write:
- self.assertEqual(list(self.manager.tasks), [task_id])
- self.assertEqual(self.bus.delete_calls, [])
- self.finish.set()
- self.assertTrue(await self.task)
+ if complete_during_write:
+ self.assertTrue(await self.task)
+ else:
+ with self.assertRaises(asyncio.CancelledError):
+ await self.task
await self._assert_cleaned(task_id)
async def test_error_before_write(self) -> None:
@@ -174,7 +183,7 @@ class BackgroundTaskManagerTest(IsolatedAsyncioTestCase):
await self._assert_registration_fault(committed=True, cancelled=False)
async def test_cancel_before_write(self) -> None:
- """Cancelling registration preserves and eventually cleans the task."""
+ """Cancelling registration cancels and cleans the running task."""
await self._assert_registration_fault(committed=False, cancelled=True)
async def test_cancel_after_write(self) -> None:
diff --git a/tests/service_cancel_dispatcher_test.py b/tests/service_cancel_dispatcher_test.py
index acca2745c..cd202180f 100644
--- a/tests/service_cancel_dispatcher_test.py
+++ b/tests/service_cancel_dispatcher_test.py
@@ -278,11 +278,12 @@ class TestCancelDispatcher(IsolatedAsyncioTestCase):
break
await asyncio.sleep(0.01)
+ # Other sessions keep running until manager shutdown.
+ self.assertFalse(bg_task_b.cancelled())
+
self.assertTrue(bg_task_a1.cancelled() or bg_task_a1.done())
self.assertTrue(bg_task_a2.cancelled() or bg_task_a2.done())
- # sess-B BG task is left running until shutdown cancels it.
- self.assertFalse(bg_task_b.cancelled())
- bg_task_b.cancel()
+ self.assertTrue(bg_task_b.cancelled())
async def test_cancel_signal_closes_local_realtime_run(self) -> None:
"""A session cancel reaches the local WebRTC runner."""
diff --git a/tests/service_message_bus_test.py b/tests/service_message_bus_test.py
index 5bdb04e80..0d82b5ed9 100644
--- a/tests/service_message_bus_test.py
+++ b/tests/service_message_bus_test.py
@@ -172,6 +172,72 @@ class TestRegistryConditionalPrimitives(IsolatedAsyncioTestCase):
)
self.assertEqual(await self.bus.registry_get("ns", "f"), "v2")
+ async def test_set_if_none_creates_only_missing_fields(self) -> None:
+ """Missing and empty fields differ; concurrent creators cannot win."""
+ for bus in (self.bus, InMemoryMessageBus()):
+ with self.subTest(backend=type(bus).__name__):
+ await bus.registry_set("ns", "empty", "")
+ self.assertFalse(
+ await bus.registry_set_if(
+ "ns",
+ "empty",
+ "replacement",
+ expected=None,
+ ),
+ )
+ results = await asyncio.gather(
+ bus.registry_set_if("ns", "new", "v", expected=None),
+ bus.registry_set_if("ns", "new", "v", expected=None),
+ )
+ self.assertEqual(sorted(results), [False, True])
+ self.assertTrue(
+ await bus.registry_set_if(
+ "ns",
+ "empty",
+ "updated",
+ expected="",
+ ),
+ )
+ self.assertFalse(
+ await bus.registry_set_if(
+ "ns",
+ "missing",
+ "v",
+ expected="",
+ ),
+ )
+ self.assertEqual(
+ await bus.registry_getall("ns"),
+ {"empty": "updated", "new": "v"},
+ )
+
+ async def test_set_if_none_refreshes_expiry_only_on_creation(self) -> None:
+ """Recovery sets hash TTL without renewing a competing outcome."""
+ self.assertTrue(
+ await self.bus.registry_set_if(
+ "ns",
+ "f",
+ "terminal",
+ expected=None,
+ ttl_secs=1000,
+ ),
+ )
+ self.assertGreater(await self.fr.ttl("ns"), 0)
+ self.assertFalse(
+ await self.bus.registry_set_if(
+ "ns",
+ "f",
+ "running",
+ expected=None,
+ ttl_secs=5000,
+ ),
+ )
+ self.assertLessEqual(await self.fr.ttl("ns"), 1000)
+ self.assertEqual(
+ await self.bus.registry_getall("ns"),
+ {"f": "terminal"},
+ )
+
async def test_set_if_refreshes_the_expiry_only_when_it_writes(
self,
) -> None:
@@ -208,6 +274,48 @@ class TestRegistryConditionalPrimitives(IsolatedAsyncioTestCase):
"""An expired session reads as gone, not as a failure."""
self.assertIsNone(await self.bus.registry_pop("missing", "f"))
+ async def test_conditional_delete_preserves_newer_values(self) -> None:
+ """Both backends delete exactly one unchanged field, atomically."""
+ for bus in (self.bus, InMemoryMessageBus()):
+ with self.subTest(backend=type(bus).__name__):
+ await bus.registry_set("ns", "f", "new")
+ await bus.registry_set("ns", "keep", "value")
+ self.assertFalse(
+ await bus.registry_del_if("ns", "f", expected="old"),
+ )
+ self.assertEqual(
+ await bus.registry_getall("ns"),
+ {"f": "new", "keep": "value"},
+ )
+ results = await asyncio.gather(
+ bus.registry_del_if("ns", "f", expected="new"),
+ bus.registry_del_if("ns", "f", expected="new"),
+ )
+ self.assertEqual(sorted(results), [False, True])
+ self.assertEqual(
+ await bus.registry_getall("ns"),
+ {"keep": "value"},
+ )
+ self.assertFalse(
+ await bus.registry_del_if("missing", "f", expected=""),
+ )
+
+ async def test_default_conditional_delete_leaves_data_intact(self) -> None:
+ """Existing adapters inherit a safe no-op for conditional deletion."""
+ await self.bus.registry_set("ns", "f", "value")
+ self.assertFalse(
+ await MessageBus.registry_del_if(
+ self.bus,
+ "ns",
+ "f",
+ expected="value",
+ ),
+ )
+ self.assertEqual(
+ await self.bus.registry_getall("ns"),
+ {"f": "value"},
+ )
+
class TestLogPrimitive(IsolatedAsyncioTestCase):
"""Mode C — replay log: append / read with cursor / trim."""
diff --git a/tests/tool_offload_middleware_test.py b/tests/tool_offload_middleware_test.py
index 7bb6bfab6..86d245222 100644
--- a/tests/tool_offload_middleware_test.py
+++ b/tests/tool_offload_middleware_test.py
@@ -3,7 +3,7 @@
"""Unit tests for ToolOffloadMiddleware."""
import asyncio
import json
-from typing import Any
+from typing import Any, AsyncGenerator
from unittest.async_case import IsolatedAsyncioTestCase
from unittest.mock import AsyncMock, MagicMock
@@ -13,10 +13,17 @@ from pydantic import BaseModel
from utils import AnyString, MockModel
from agentscope.agent import Agent
-from agentscope.app.message_bus import MessageBus, MessageBusKeys
+from agentscope.app.message_bus import (
+ InMemoryMessageBus,
+ MessageBus,
+ MessageBusKeys,
+)
from agentscope.app.middleware import ToolOffloadMiddleware
-from agentscope.app._manager import BackgroundTaskManager
-from agentscope.message import TextBlock, ToolCallBlock
+from agentscope.app._manager import BackgroundTaskManager, TaskStatus
+from agentscope.app._manager._background_task_manager import (
+ BackgroundTaskRecord,
+)
+from agentscope.message import TextBlock, ToolCallBlock, ToolResultState
from agentscope.permission import (
PermissionContext,
PermissionDecision,
@@ -152,8 +159,8 @@ class ToolOffloadMiddlewareTest(IsolatedAsyncioTestCase):
async def asyncSetUp(self) -> None:
"""Set up test fixtures."""
self.mock_model = MockModel()
- self.bg_manager = BackgroundTaskManager(
- message_bus=MagicMock(spec=MessageBus),
+ self.bg_manager = await self.enterAsyncContext(
+ BackgroundTaskManager(message_bus=InMemoryMessageBus()),
)
# ------------------------------------------------------------------
@@ -374,6 +381,158 @@ class ToolOffloadMiddlewareTest(IsolatedAsyncioTestCase):
self.assertEqual(payload["session_id"], agent.state.session_id)
self.assertEqual(payload["agent_id"], "a")
+ async def test_background_failures_retain_error_status(self) -> None:
+ """Error responses and raised exceptions both retain failed state."""
+ for raises in (False, True):
+ with self.subTest(raises=raises):
+ agent, middleware = self._make_agent(
+ Toolkit(tools=[SlowTool()]),
+ timeout_secs=0.01,
+ )
+ gate = asyncio.Event()
+
+ async def handler(
+ *,
+ event: asyncio.Event = gate,
+ should_raise: bool = raises,
+ **_kwargs: Any,
+ ) -> AsyncGenerator:
+ """Fail only after the tool has been offloaded."""
+ await event.wait()
+ if should_raise:
+ raise ValueError("tool execution failed")
+ yield ToolResponse(
+ id="call_failure",
+ state=ToolResultState.ERROR,
+ content=[TextBlock(text="tool execution failed")],
+ )
+
+ call = ToolCallBlock(
+ id="call_failure",
+ name="slow_tool",
+ input="{}",
+ )
+ async for _ in middleware.on_acting(
+ agent,
+ {"tool_call": call},
+ handler,
+ ):
+ pass
+ bg = next(iter(self.bg_manager.tasks.values()))
+ expected = BackgroundTaskRecord.model_validate(bg).model_dump()
+ gate.set()
+ await bg.asyncio_task
+ await asyncio.gather(*self.bg_manager._completion_tasks)
+ record = await self.bg_manager.get_task(bg.id, user_id="u")
+ self.assertIsNotNone(record.completed_at)
+ expected.update(
+ status=TaskStatus.FAILED,
+ error_summary="tool execution failed",
+ completed_at=record.completed_at,
+ )
+ self.assertEqual(record.model_dump(), expected)
+ self.assertEqual(self.bg_manager.tasks, {})
+
+ async def test_interrupted_response_matches_record_and_inbox(self) -> None:
+ """Interrupted results keep their output and report cancellation."""
+ for output in ("", "partial output"):
+ with self.subTest(output=output):
+ agent, middleware = self._make_agent(
+ Toolkit(tools=[SlowTool()]),
+ timeout_secs=0.01,
+ )
+ gate = asyncio.Event()
+ delivered = asyncio.Event()
+ mock_bus = middleware._message_bus
+ mock_bus.publish.side_effect = (
+ lambda *_args, event=delivered, **_kwargs: event.set()
+ )
+ response = ToolResponse(
+ id="call_interrupted",
+ state=ToolResultState.INTERRUPTED,
+ content=[TextBlock(text=output)] if output else [],
+ )
+ original = response.model_dump(mode="json")
+
+ async def handler(
+ *,
+ event: asyncio.Event = gate,
+ result: ToolResponse = response,
+ **_kwargs: Any,
+ ) -> AsyncGenerator:
+ await event.wait()
+ yield result
+
+ call = ToolCallBlock(
+ id="call_interrupted",
+ name="slow_tool",
+ input="{}",
+ )
+ async for _ in middleware.on_acting(
+ agent,
+ {"tool_call": call},
+ handler,
+ ):
+ pass
+ bg = next(iter(self.bg_manager.tasks.values()))
+ expected = BackgroundTaskRecord.model_validate(bg).model_dump()
+ gate.set()
+ await bg.asyncio_task
+ await asyncio.gather(*self.bg_manager._completion_tasks)
+ await asyncio.wait_for(delivered.wait(), timeout=1)
+ record = await self.bg_manager.get_task(bg.id, user_id="u")
+ expected.update(
+ status=TaskStatus.CANCELLED,
+ completed_at=record.completed_at,
+ )
+ self.assertEqual(record.model_dump(), expected)
+
+ prefix = (
+ "<system-notification>Tool 'slow_tool' running in "
+ "background (id=call_interrupted) has been cancelled"
+ )
+ expected_hint: str | list[dict]
+ if output:
+ expected_hint = [
+ dict(
+ original["content"][0],
+ text=f"{prefix}.\n\nResult:\n\n{output}"
+ "</system-notification>",
+ ),
+ ]
+ else:
+ expected_hint = (
+ f"{prefix} with no output.</system-notification>"
+ )
+ inbox = [
+ call.args[1]
+ for call in mock_bus.queue_push.call_args_list
+ if call.args[0]
+ == MessageBusKeys.inbox(
+ agent.state.session_id,
+ )
+ ]
+ self.assertEqual(
+ inbox,
+ [
+ {
+ "type": "hint",
+ "id": AnyString(),
+ "created_at": AnyString(),
+ "finished_at": AnyString(),
+ "source": json.dumps(
+ {
+ "label": "tool_output",
+ "sublabel": "slow_tool · call_interrupted",
+ },
+ ensure_ascii=False,
+ ),
+ "hint": expected_hint,
+ },
+ ],
+ )
+ self.assertEqual(response.model_dump(mode="json"), original)
+
async def test_tool_stop_cancels_background_task(self) -> None:
"""ToolStop tool cancels the running background asyncio task."""
@@ -408,5 +567,9 @@ class ToolOffloadMiddlewareTest(IsolatedAsyncioTestCase):
# The asyncio task should be cancelling
self.assertTrue(asyncio_task.cancelled() or asyncio_task.cancelling())
- # Removed from manager
+ await asyncio.gather(asyncio_task, return_exceptions=True)
+ await asyncio.gather(*self.bg_manager._completion_tasks)
+ # The handle is released, but cancelled metadata remains queryable.
self.assertEqual(len(self.bg_manager.tasks), 0)
+ record = await self.bg_manager.get_task(task_id, user_id="u")
+ self.assertEqual(record.status, TaskStatus.CANCELLED)