Build and run agents you can see, understand and trust.
Python agentchatbotlarge-language-modelsllmllm-agentmcpmulti-agentmulti-modalreact-agentrealtime-agentvoice-agentvoice-assistant
feat(service): expose background tasks via REST endpoints (#1865)
XiuShenAlcfd3d99982cd
diff --git a/src/agentscope/app/_app.py b/src/agentscope/app/_app.pyindex 7f004ac08..f97133bde 100644--- a/src/agentscope/app/_app.py+++ b/src/agentscope/app/_app.py@@ -17,6 +17,7 @@ from .workspace_manager import WorkspaceManagerBasefrom ._router import (asr_router,agent_router,+ background_task_router,channel_router,chat_router,credential_router,@@ -439,6 +440,7 @@ def create_app(for router in (asr_router,agent_router,+ background_task_router,chat_router,credential_router,health_router,diff --git a/src/agentscope/app/_manager/__init__.py b/src/agentscope/app/_manager/__init__.pyindex f4facb0e2..5409c6b49 100644--- a/src/agentscope/app/_manager/__init__.py+++ b/src/agentscope/app/_manager/__init__.py@@ -6,12 +6,13 @@ from ._scheduler import SchedulerManagerfrom ._wakeup_dispatcher import WakeupDispatcherfrom ._cancel_dispatcher import CancelDispatcherfrom ._chat_run_registry import ChatRunRegistry-from ._background_task_manager import BackgroundTaskManager+from ._background_task_manager import BackgroundTaskManager, TaskStatus__all__ = ["BackgroundTaskManager","CancelDispatcher","ChatRunRegistry","SchedulerManager",+ "TaskStatus","WakeupDispatcher",]diff --git a/src/agentscope/app/_manager/_background_task_manager.py b/src/agentscope/app/_manager/_background_task_manager.pyindex f26acd066..48d2ca04f 100644--- a/src/agentscope/app/_manager/_background_task_manager.py+++ b/src/agentscope/app/_manager/_background_task_manager.py@@ -5,25 +5,44 @@ import jsonimport timefrom collections import OrderedDictfrom dataclasses import dataclass, field+from enum import Enumfrom typing import Any, Self, TYPE_CHECKINGimport shortuuid-from pydantic import BaseModel, Field+from pydantic import BaseModel, ConfigDict, Field-from agentscope.message import TextBlock, ToolResultState-from agentscope.permission import (+from ...message import TextBlock, ToolResultState+from ...permission import (PermissionContext,PermissionDecision,PermissionBehavior,)-from agentscope.tool import ToolBase, ToolChunk-from agentscope._logging import logger+from ...tool import ToolBase, ToolChunk, ToolResponse+from ..._logging import loggerfrom ..message_bus import MessageBusKeysif TYPE_CHECKING:from ..message_bus import MessageBus+class TaskStatus(str, Enum):+ """Status of a background task throughout its lifecycle."""++ RUNNING = "running"+ COMPLETED = "completed"+ FAILED = "failed"+ CANCELLED = "cancelled"+++_MAX_RETAINED_TASKS = 50+# Maximum number of terminal-state tasks retained per user.+_RECORD_LEASE_SECS = 120+_RENEWAL_CONCURRENCY = 16+_IO_TIMEOUT_SECS = 2.0+_RETRY_DELAY_SECS = 1.0+_SHUTDOWN_TIMEOUT_SECS = 5.0++@dataclassclass BackgroundTask:"""Metadata for a single background task.@@ -41,6 +60,19 @@ class BackgroundTask:The name of the tool that was offloaded.id (`str`):Auto-generated unique task identifier.+ summary (`str`):+ A short human-readable summary of the tool invocation+ for frontend display (at most 128 characters).+ started_at (`float`):+ Unix epoch (UTC) when the task was registered.+ completed_at (`float | None`):+ Unix epoch (UTC) when the task reached a terminal state,+ or ``None`` while still running.+ status (`TaskStatus`):+ Current lifecycle status of the task.+ error_summary (`str | None`):+ Brief description of the failure reason when+ ``status == FAILED``."""asyncio_task: asyncio.Task@@ -61,6 +93,38 @@ class BackgroundTask:id: str = field(default_factory=shortuuid.uuid)"""The background task id."""+ summary: str = ""+ """Short human-readable summary for frontend display."""++ started_at: float = field(default_factory=time.time)+ """Unix epoch (UTC) when the task was registered."""++ completed_at: float | None = None+ """Unix epoch (UTC) when the task finished, or None if running."""++ status: TaskStatus = TaskStatus.RUNNING+ """Current lifecycle status."""++ error_summary: str | None = None+ """Brief failure description (populated only for FAILED tasks)."""+++class BackgroundTaskRecord(BaseModel):+ """Serializable task snapshot, independent of the local task handle."""++ model_config = ConfigDict(from_attributes=True)++ id: str+ session_id: str+ agent_id: str+ user_id: str+ tool_name: str+ summary: str+ started_at: float+ completed_at: float | None+ status: TaskStatus+ error_summary: str | None+class _ToolStopParams(BaseModel):"""The params of the stop tool."""@@ -158,8 +222,8 @@ class ToolStop(ToolBase):if (local_task is not Noneand local_task.session_id == self._session_id+ and not local_task.asyncio_task.done()):- self.background_tasks.pop(task_id, None)local_task.asyncio_task.cancel()logger.info("Background task stopped via ToolStop (local): task_id=%s, "@@ -241,6 +305,9 @@ class BackgroundTaskManager:"""self._message_bus = message_busself.tasks: OrderedDict[str, BackgroundTask] = OrderedDict()+ self._completion_tasks: set[asyncio.Task] = set()+ self._renewal_task: asyncio.Task | None = None+ self._closed = False# ------------------------------------------------------------------# Task registration@@ -253,12 +320,14 @@ class BackgroundTaskManager:agent_id: str,user_id: str,tool_name: str = "",+ *,+ summary: str = "",) -> str:"""Register an already-running asyncio task.- Writes to both the local handle cache and the global Redis- registry. The task auto-removes from both when it finishes- (via ``add_done_callback``).+ Tracks running handles locally and publishes metadata globally.+ On completion, releases the handle and retains terminal metadata+ for polling from any worker.Args:asyncio_task (`asyncio.Task`):@@ -271,6 +340,9 @@ class BackgroundTaskManager:The user id of the originating request.tool_name (`str`, optional):The name of the offloaded tool.+ summary (`str`, optional):+ Short human-readable summary of the tool invocation+ for frontend display.Returns:`str`:@@ -282,39 +354,44 @@ class BackgroundTaskManager:agent_id=agent_id,user_id=user_id,tool_name=tool_name,+ summary=summary,)task_id = bg_task.idself.tasks[task_id] = bg_task- def _on_done(_t: asyncio.Task) -> None:- self.tasks.pop(task_id, None)- # Clean up asynchronously; the helper logs registry errors.- try:- asyncio.ensure_future(- self._safe_bg_task_unregister(session_id, task_id),- )- except RuntimeError:- # Event loop already closed during shutdown.- pass-- metadata = json.dumps(- {- "tool_name": tool_name,- "agent_id": agent_id,- "started_at": time.time(),- },- ensure_ascii=False,- )+ # Install cleanup even if a registry write fails or is cancelled.+ # The callback is attached after writes settle, so a fast task cannot+ # publish a terminal state before its initial running state.try:- await self._message_bus.registry_set(- MessageBusKeys.bg_tasks(session_id),- task_id,- metadata,- ttl_secs=MessageBusKeys.BG_TASKS_TTL_SECS,- )+ async with asyncio.timeout(_IO_TIMEOUT_SECS):+ await self._message_bus.registry_set(+ MessageBusKeys.bg_tasks(session_id),+ task_id,+ json.dumps(+ {+ "tool_name": tool_name,+ "agent_id": agent_id,+ "started_at": bg_task.started_at,+ },+ ensure_ascii=False,+ ),+ ttl_secs=MessageBusKeys.BG_TASKS_TTL_SECS,+ )+ await self._store_record(+ BackgroundTaskRecord.model_validate(bg_task),+ )+ # A round timeout can temporarily mark the scheduler cancelling.+ if self._renewal_task is None or self._renewal_task.done():+ self._renewal_task = asyncio.create_task(+ self._renew_running_records(),+ )+ except BaseException:+ asyncio_task.cancel()+ raisefinally:- # Always install cleanup, after the write to avoid early deletion.- asyncio_task.add_done_callback(_on_done)+ asyncio_task.add_done_callback(+ lambda task: self._on_done(bg_task, task),+ )logger.info("Background task registered: task_id=%s, session_id=%s, "@@ -324,35 +401,266 @@ class BackgroundTaskManager:agent_id,tool_name,)-return task_id- async def _safe_bg_task_unregister(+ def _on_done(self,- session_id: str,- task_id: str,+ bg_task: BackgroundTask,+ task: asyncio.Task,) -> None:- """Unregister a finished background task, logging any failure.+ """Record the actual outcome and schedule shared-state cleanup."""+ bg_task.completed_at = time.time()+ if task.cancelled():+ bg_task.status = TaskStatus.CANCELLED+ elif task.exception() is not None:+ bg_task.status = TaskStatus.FAILED+ bg_task.error_summary = str(task.exception())[:512]+ else:+ result = task.result()+ bg_task.status = TaskStatus.COMPLETED+ if isinstance(result, ToolResponse):+ if result.state == ToolResultState.ERROR:+ bg_task.status = TaskStatus.FAILED+ bg_task.error_summary = "\n".join(+ block.text+ for block in result.content+ if isinstance(block, TextBlock)+ )[:512]+ elif result.state == ToolResultState.INTERRUPTED:+ bg_task.status = TaskStatus.CANCELLED++ self.tasks.pop(bg_task.id, None)+ # Let renewal exit when idle so new registrations can reuse it safely.+ if self._closed:+ return+ completion = asyncio.create_task(+ self._finish_task(BackgroundTaskRecord.model_validate(bg_task)),+ )+ self._completion_tasks.add(completion)+ completion.add_done_callback(self._completion_tasks.discard)++ async def _store_record(self, record: BackgroundTaskRecord) -> None:+ """Publish a snapshot with an expiry independent of the user hash."""+ payload = record.model_dump(mode="json")+ if record.status == TaskStatus.RUNNING:+ payload["_expires_at"] = time.time() + _RECORD_LEASE_SECS+ else:+ assert record.completed_at is not None+ payload["_expires_at"] = (+ record.completed_at + MessageBusKeys.BG_TASKS_TTL_SECS+ )+ await self._message_bus.registry_set(+ MessageBusKeys.bg_task_records(record.user_id),+ record.id,+ json.dumps(payload, ensure_ascii=False),+ ttl_secs=MessageBusKeys.BG_TASKS_TTL_SECS,+ )- Args:- session_id (`str`):- The session id of the finished task.- task_id (`str`):- The task id to unregister from the global registry.- """+ async def _renew_running_records(self) -> None:+ """Renew leases in bounded, fair rounds without scheduling drift."""+ pending: OrderedDict[str, BackgroundTask] = OrderedDict()+ loop = asyncio.get_running_loop()+ interval = _RECORD_LEASE_SECS / 4+ deadline = loop.time() + interval++ async def renew_pending() -> None:+ """Share the queue without creating a worker for every task."""+ while pending:+ _, task = pending.popitem(last=False)+ await self._renew_running_record(task)++ while self.tasks and not self._closed:+ await asyncio.sleep(max(0, deadline - loop.time()))+ deadline = loop.time() + interval+ # Unvisited tasks stay ahead of retries and newly registered work.+ pending.update(self.tasks)+ try:+ async with asyncio.timeout_at(deadline):+ async with asyncio.TaskGroup() as group:+ for _ in range(+ min(_RENEWAL_CONCURRENCY, len(pending)),+ ):+ group.create_task(renew_pending())+ except TimeoutError:+ logger.warning(+ "Background task renewal round exceeded its budget: "+ "unvisited=%d",+ len(pending),+ )++ async def _renew_running_record(self, task: BackgroundTask) -> None:+ """Renew or restore one live record without overwriting outcomes."""+ if task.asyncio_task.done():+ returntry:+ async with asyncio.timeout(_IO_TIMEOUT_SECS):+ namespace = MessageBusKeys.bg_task_records(task.user_id)+ value = await self._message_bus.registry_get(+ namespace,+ task.id,+ )+ if task.asyncio_task.done():+ return+ if value is None:+ payload = BackgroundTaskRecord.model_validate(+ task,+ ).model_dump(mode="json")+ else:+ payload = json.loads(value)+ if payload["status"] != TaskStatus.RUNNING:+ return+ payload["_expires_at"] = time.time() + _RECORD_LEASE_SECS+ # A concurrent completion must win over this lease.+ await self._message_bus.registry_set_if(+ namespace,+ task.id,+ json.dumps(payload, ensure_ascii=False),+ expected=value,+ ttl_secs=MessageBusKeys.BG_TASKS_TTL_SECS,+ )+ except Exception:+ logger.exception(+ "Failed to renew background task lease: task_id=%s",+ task.id,+ )++ async def _finish_task(self, record: BackgroundTaskRecord) -> None:+ """Retry pending outcomes and cleanup until success or shutdown."""+ stored = False+ unregistered = False+ while not self._closed:+ try:+ # Outcome persistence never waits for the retention lock.+ async with asyncio.timeout(_IO_TIMEOUT_SECS):+ if not stored:+ await self._store_record(record)+ stored = True+ if not unregistered:+ await self._message_bus.registry_del(+ MessageBusKeys.bg_tasks(record.session_id),+ record.id,+ )+ unregistered = True+ async with asyncio.timeout(_IO_TIMEOUT_SECS):+ async with self._message_bus.acquire_lock(+ MessageBusKeys.bg_task_records_lock(record.user_id),+ ttl_secs=30,+ ):+ await self._evict_completed(record.user_id)+ return+ except Exception:+ logger.warning(+ "Retrying background task finalization: task_id=%s",+ record.id,+ exc_info=True,+ )+ await asyncio.sleep(_RETRY_DELAY_SECS)++ async def _evict_completed(self, user_id: str) -> None:+ """Delete expired snapshots and bound terminal history."""+ namespace = MessageBusKeys.bg_task_records(user_id)+ values = await self._message_bus.registry_getall(namespace)+ completed = []+ for task_id, value in values.items():+ record = await self._read_record(namespace, task_id, value)+ if record is not None and record.status != TaskStatus.RUNNING:+ completed.append(record)+ completed.sort(key=lambda record: (record.completed_at, record.id))+ for record in completed[:-_MAX_RETAINED_TASKS]:await self._message_bus.registry_del(- MessageBusKeys.bg_tasks(session_id),- task_id,+ MessageBusKeys.bg_task_records(user_id),+ record.id,)- except Exception as e: # pylint: disable=broad-except- logger.exception(- "Failed to unregister background task from the global "- "registry: task_id=%s, session_id=%s, error=%s",++ async def _read_record(+ self,+ namespace: str,+ task_id: str,+ value: str,+ ) -> BackgroundTaskRecord | None:+ """Remove an expired snapshot only if its value is unchanged."""+ payload = json.loads(value)+ # Older snapshots have no per-record lease. Bound their lifetime too.+ expires_at = payload.get(+ "_expires_at",+ payload["started_at"] + MessageBusKeys.BG_TASKS_TTL_SECS,+ )+ if expires_at <= time.time():+ await self._message_bus.registry_del_if(+ namespace,task_id,- session_id,- str(e),+ expected=value,)+ return None+ return BackgroundTaskRecord.model_validate(payload)++ # ------------------------------------------------------------------+ # Task querying (for REST endpoints)+ # ------------------------------------------------------------------++ async def list_tasks(+ self,+ *,+ user_id: str,+ session_id: str | None = None,+ ) -> list[BackgroundTaskRecord]:+ """List shared running and retained terminal tasks for a user.++ Args:+ user_id (`str`):+ Only return tasks belonging to this user.+ session_id (`str | None`, optional):+ If provided, further filter by session.++ Returns:+ `list[BackgroundTaskRecord]`:+ Matching tasks ordered by registration time.+ """+ namespace = MessageBusKeys.bg_task_records(user_id)+ values = await self._message_bus.registry_getall(namespace)+ records = []+ for task_id, value in values.items():+ record = await self._read_record(namespace, task_id, value)+ if record is not None:+ records.append(record)+ return sorted(+ (+ record+ for record in records+ if record.user_id == user_id+ and (session_id is None or record.session_id == session_id)+ ),+ key=lambda record: (record.started_at, record.id),+ )++ async def get_task(+ self,+ task_id: str,+ *,+ user_id: str,+ ) -> BackgroundTaskRecord | None:+ """Get shared task metadata, scoped to the requesting user.++ Args:+ task_id (`str`):+ The task identifier.+ user_id (`str`):+ The requesting user's id.++ Returns:+ `BackgroundTaskRecord | None`:+ The task if owned by the user, else `None`.+ """+ namespace = MessageBusKeys.bg_task_records(user_id)+ value = await self._message_bus.registry_get(namespace, task_id)+ if value is None:+ return None+ record = await self._read_record(namespace, task_id, value)+ return (+ record+ if record is not None and record.user_id == user_id+ else None+ )# ------------------------------------------------------------------# Tool listing@@ -393,7 +701,7 @@ class BackgroundTaskManager:"""cancelled = 0for bg_task in list(self.tasks.values()):- if bg_task.session_id != session_id:+ if bg_task.session_id != session_id or bg_task.asyncio_task.done():continuelogger.info("Cancelling background task for session cancel: "@@ -426,7 +734,7 @@ class BackgroundTaskManager:``True`` if the task was found locally and cancelled."""bg_task = self.tasks.get(task_id)- if bg_task is None:+ if bg_task is None or bg_task.asyncio_task.done():return Falselogger.info("Cancelling background task via bus signal: "@@ -451,12 +759,18 @@ class BackgroundTaskManager:return selfasync def __aexit__(self, *exc: object) -> None:- """Cancel all running background tasks on context exit."""+ """Cancel work and flush outcomes within a bounded shutdown budget."""count = len(self.tasks)logger.info("Shutting down BackgroundTaskManager: cancelling %d task(s).",count,)+ tasks = [bg.asyncio_task for bg in self.tasks.values()]+ if self._renewal_task is not None:+ self._renewal_task.cancel()+ loop = asyncio.get_running_loop()+ deadline = loop.time() + _SHUTDOWN_TIMEOUT_SECS+ # Reserve the last 10% of the budget for cancellation cleanup.for bg_task in list(self.tasks.values()):logger.info("Cancelling background task on shutdown: task_id=%s, "@@ -466,4 +780,46 @@ class BackgroundTaskManager:bg_task.agent_id,)bg_task.asyncio_task.cancel()- self.tasks.clear()+ try:+ if tasks:+ await asyncio.wait(+ tasks,+ timeout=max(+ 0,+ deadline - loop.time() - _SHUTDOWN_TIMEOUT_SECS / 10,+ ),+ )+ if self._completion_tasks:+ await asyncio.wait(+ self._completion_tasks,+ timeout=max(+ 0,+ deadline - loop.time() - _SHUTDOWN_TIMEOUT_SECS / 10,+ ),+ )+ finally:+ # Late execution callbacks must not start I/O after the bus closes.+ self._closed = True+ pending = {+ task+ for task in [+ *tasks,+ *self._completion_tasks,+ self._renewal_task,+ ]+ if task is not None and not task.done()+ }+ for task in pending:+ task.cancel()+ if pending:+ _, unfinished = await asyncio.wait(+ pending,+ timeout=max(0, deadline - loop.time()),+ )+ logger.warning(+ "Background task shutdown budget exhausted: "+ "cancelled=%d, unfinished=%d",+ len(pending),+ len(unfinished),+ )+ self.tasks.clear()diff --git a/src/agentscope/app/_router/__init__.py b/src/agentscope/app/_router/__init__.pyindex 02aaa2eec..8a1220bcb 100644--- a/src/agentscope/app/_router/__init__.py+++ b/src/agentscope/app/_router/__init__.py@@ -2,6 +2,7 @@"""App routers."""from ._asr import asr_routerfrom ._agent import agent_router+from ._background_task import background_task_routerfrom ._channel import channel_routerfrom ._chat import chat_routerfrom ._credential import credential_router@@ -23,6 +24,7 @@ from ._workspace import workspace_router__all__ = ["asr_router","agent_router",+ "background_task_router","channel_router","model_router","realtime_router",diff --git a/src/agentscope/app/_router/_background_task.py b/src/agentscope/app/_router/_background_task.pynew file mode 100644index 000000000..8d05b064b--- /dev/null+++ b/src/agentscope/app/_router/_background_task.py@@ -0,0 +1,98 @@+# -*- coding: utf-8 -*-+"""Background-task router — read-only endpoints for task observability."""+from fastapi import APIRouter, Depends, HTTPException, Query, status++from ..deps import get_background_task_manager, get_current_user_id+from .._manager import BackgroundTaskManager+from ._schema import (+ BackgroundTaskInfo,+ ListBackgroundTasksResponse,+)+from ._schema._background_task import to_info++background_task_router = APIRouter(+ prefix="/background-tasks",+ tags=["background-tasks"],+ responses={404: {"description": "Not found"}},+)+++@background_task_router.get(+ "",+ response_model=ListBackgroundTasksResponse,+ summary="List background tasks for the current user",+)+async def list_background_tasks(+ session_id: str+ | None = Query(+ default=None,+ description="Optional session filter.",+ ),+ user_id: str = Depends(get_current_user_id),+ bg_manager: BackgroundTaskManager = Depends(+ get_background_task_manager,+ ),+) -> ListBackgroundTasksResponse:+ """List all background tasks (running and recently-completed) visible+ to the authenticated user.++ Args:+ session_id (`str | None`, optional):+ If provided, only return tasks belonging to this session.+ user_id (`str`):+ The authenticated user id (from ``X-User-ID`` header).+ bg_manager (`BackgroundTaskManager`):+ The application-wide background task manager.++ Returns:+ `ListBackgroundTasksResponse`:+ Tasks and their total count.+ """+ tasks = await bg_manager.list_tasks(+ user_id=user_id,+ session_id=session_id,+ )+ infos = [to_info(t) for t in tasks]+ return ListBackgroundTasksResponse(tasks=infos, total=len(infos))+++@background_task_router.get(+ "/{task_id}",+ response_model=BackgroundTaskInfo,+ summary="Get a single background task by id",+)+async def get_background_task(+ task_id: str,+ user_id: str = Depends(get_current_user_id),+ bg_manager: BackgroundTaskManager = Depends(+ get_background_task_manager,+ ),+) -> BackgroundTaskInfo:+ """Retrieve details of a single background task.++ Returns 404 when the task does not exist or belongs to a different+ user (no information leakage about task existence).++ Args:+ task_id (`str`):+ The unique task identifier.+ user_id (`str`):+ The authenticated user id (from ``X-User-ID`` header).+ bg_manager (`BackgroundTaskManager`):+ The application-wide background task manager.++ Returns:+ `BackgroundTaskInfo`:+ The task details.++ Raises:+ `HTTPException`:+ 404 if the task is not found or not owned by the user.+ """+ task = await bg_manager.get_task(task_id, user_id=user_id)+ if task is None:+ raise HTTPException(+ status_code=status.HTTP_404_NOT_FOUND,+ detail=f"Background task '{task_id}' not found.",+ )+ return to_info(task)diff --git a/src/agentscope/app/_router/_schema/__init__.py b/src/agentscope/app/_router/_schema/__init__.pyindex e6057fdfb..c291f6bea 100644--- a/src/agentscope/app/_router/_schema/__init__.py+++ b/src/agentscope/app/_router/_schema/__init__.py@@ -1,6 +1,10 @@# -*- coding: utf-8 -*-"""Schema models for the agent service."""+from ._background_task import (+ BackgroundTaskInfo,+ ListBackgroundTasksResponse,+)from ._channel import (StartCredentialBindingRequest,ChannelActionResponse,@@ -170,6 +174,9 @@ __all__ = ["CreateAgentResponse","UpdateAgentRequest","ListSchedulesResponse",+ # Background Task+ "BackgroundTaskInfo",+ "ListBackgroundTasksResponse",# Channel"ChannelActionResponse","ChannelChatId",diff --git a/src/agentscope/app/_router/_schema/_background_task.py b/src/agentscope/app/_router/_schema/_background_task.pynew file mode 100644index 000000000..eee507069--- /dev/null+++ b/src/agentscope/app/_router/_schema/_background_task.py@@ -0,0 +1,76 @@+# -*- coding: utf-8 -*-+"""Request / response schemas for the background-task router."""+from datetime import datetime, timezone++from pydantic import BaseModel, Field++from ..._manager._background_task_manager import (+ BackgroundTaskRecord,+ TaskStatus,+)+++class BackgroundTaskInfo(BaseModel):+ """Summary of a single background task for API responses."""++ task_id: str = Field(description="Unique task identifier.")+ session_id: str = Field(description="Session that owns this task.")+ agent_id: str = Field(description="Agent that triggered this task.")+ tool_name: str = Field(description="Name of the offloaded tool.")+ summary: str = Field(+ description="Short human-readable summary of the tool invocation. "+ "May include unredacted tool arguments stored in shared task records.",+ )+ status: TaskStatus = Field(description="Current lifecycle status.")+ started_at: datetime = Field(+ description="UTC timestamp when the task was registered.",+ )+ completed_at: datetime | None = Field(+ default=None,+ description="UTC timestamp when the task reached a terminal "+ "state, or null while still running.",+ )+ error_summary: str | None = Field(+ default=None,+ description="Brief failure description (only for failed tasks).",+ )+++class ListBackgroundTasksResponse(BaseModel):+ """Response body for listing background tasks."""++ tasks: list[BackgroundTaskInfo] = Field(+ description="Background tasks for the requested scope.",+ )+ total: int = Field(description="Total number of tasks returned.")+++def to_info(task: BackgroundTaskRecord) -> BackgroundTaskInfo:+ """Convert shared task metadata to an API-facing schema.++ Args:+ task (`BackgroundTaskRecord`):+ The shared background task metadata.++ Returns:+ `BackgroundTaskInfo`:+ The serializable API response model.+ """+ return BackgroundTaskInfo(+ task_id=task.id,+ session_id=task.session_id,+ agent_id=task.agent_id,+ tool_name=task.tool_name,+ summary=task.summary,+ status=task.status,+ started_at=datetime.fromtimestamp(+ task.started_at,+ tz=timezone.utc,+ ),+ completed_at=(+ datetime.fromtimestamp(task.completed_at, tz=timezone.utc)+ if task.completed_at is not None+ else None+ ),+ error_summary=task.error_summary,+ )diff --git a/src/agentscope/app/message_bus/_base.py b/src/agentscope/app/message_bus/_base.pyindex a943d46b5..f0989b4d9 100644--- a/src/agentscope/app/message_bus/_base.py+++ b/src/agentscope/app/message_bus/_base.py@@ -514,7 +514,7 @@ class MessageBus(ABC): # pylint: disable=too-many-public-methodsfield: str,value: str,*,- expected: str,+ expected: str | None,ttl_secs: int | None = None,) -> bool:"""Set ``field`` only while it still holds ``expected``.@@ -531,9 +531,10 @@ class MessageBus(ABC): # pylint: disable=too-many-public-methodsField name within the registry.value (`str`):Serialized value to store.- expected (`str`):+ expected (`str | None`):The value the caller last read. The write is skipped- when the stored value differs.+ when the stored value differs. ``None`` creates the field+ only if it does not exist.ttl_secs (`int | None`, optional):Refresh the namespace expiry when the write happens.@@ -542,6 +543,29 @@ class MessageBus(ABC): # pylint: disable=too-many-public-methods``True`` when the value was written."""+ async def registry_del_if( # pylint: disable=unused-argument+ self,+ namespace: str,+ field: str,+ *,+ expected: str,+ ) -> bool:+ """Delete a field atomically only if its value still matches.++ Backends may override this optional operation. The default leaves+ data intact so existing adapters remain usable without an unsafe+ read-then-delete fallback.++ Args:+ namespace (`str`): The registry key.+ field (`str`): The field to remove.+ expected (`str`): The previously observed value.++ Returns:+ `bool`: True if deleted; False if absent, changed, or unsupported.+ """+ return False+@abstractmethodasync def registry_pop(self, namespace: str, field: str) -> str | None:"""Read ``field`` and remove it in one step.diff --git a/src/agentscope/app/message_bus/_in_memory_message_bus.py b/src/agentscope/app/message_bus/_in_memory_message_bus.pyindex 85385a2f3..bf5c370b5 100644--- a/src/agentscope/app/message_bus/_in_memory_message_bus.py+++ b/src/agentscope/app/message_bus/_in_memory_message_bus.py@@ -506,7 +506,7 @@ class InMemoryMessageBus(field: str,value: str,*,- expected: str,+ expected: str | None,ttl_secs: int | None = None,) -> bool:"""Compare-and-set one field; ``ttl_secs`` ignored. See base."""@@ -516,6 +516,20 @@ class InMemoryMessageBus(self._registries.setdefault(namespace, {})[field] = valuereturn True+ async def registry_del_if(+ self,+ namespace: str,+ field: str,+ *,+ expected: str,+ ) -> bool:+ """Compare and delete without yielding between the two operations."""+ registry = self._registries.get(namespace, {})+ if registry.get(field) != expected:+ return False+ del registry[field]+ return True+async def registry_pop(self, namespace: str, field: str) -> str | None:"""Read and remove one field. See base."""return self._registries.get(namespace, {}).pop(field, None)diff --git a/src/agentscope/app/message_bus/_keys.py b/src/agentscope/app/message_bus/_keys.pyindex 19c57a475..52e67e104 100644--- a/src/agentscope/app/message_bus/_keys.py+++ b/src/agentscope/app/message_bus/_keys.py@@ -473,6 +473,16 @@ class MessageBusKeys: # pylint: disable=too-many-public-methods"""Per-session background task registry key."""return cls._BG_TASKS.format(sid=session_id)+ @staticmethod+ def bg_task_records(user_id: str) -> str:+ """Shared running and terminal background task metadata per user."""+ return f"agentscope:bg_task_records:{user_id}"++ @staticmethod+ def bg_task_records_lock(user_id: str) -> str:+ """Serialize terminal-state retention across a user's workers."""+ return f"agentscope:bg_task_records_lock:{user_id}"+# ------------------------------------------------------------------# Knowledge-base indexing pipeline# ------------------------------------------------------------------diff --git a/src/agentscope/app/message_bus/_redis_message_bus.py b/src/agentscope/app/message_bus/_redis_message_bus.pyindex 475b96c8c..afed73d69 100644--- a/src/agentscope/app/message_bus/_redis_message_bus.py+++ b/src/agentscope/app/message_bus/_redis_message_bus.py@@ -31,7 +31,9 @@ return entries# Compare-and-set on one hash field, so a caller can write back a value# it decided on earlier without holding a lock across the think time._REGISTRY_SET_IF_LUA = """-if redis.call('HGET', KEYS[1], ARGV[1]) ~= ARGV[3] then return 0 end+local expected = ARGV[3]+if ARGV[5] == '1' then expected = false end+if redis.call('HGET', KEYS[1], ARGV[1]) ~= expected then return 0 endredis.call('HSET', KEYS[1], ARGV[1], ARGV[2])if tonumber(ARGV[4]) > 0 then redis.call('EXPIRE', KEYS[1], ARGV[4]) endreturn 1@@ -45,6 +47,11 @@ if value then redis.call('HDEL', KEYS[1], ARGV[1]) endreturn value"""+_REGISTRY_DEL_IF_LUA = """+if redis.call('HGET', KEYS[1], ARGV[1]) ~= ARGV[2] then return 0 end+return redis.call('HDEL', KEYS[1], ARGV[1])+"""+_LOCK_RELEASE_LUA = """if redis.call('GET', KEYS[1]) == ARGV[1] thenreturn redis.call('DEL', KEYS[1])@@ -517,7 +524,7 @@ class RedisMessageBus(MessageBus): # pylint: disable=too-many-public-methodsfield: str,value: str,*,- expected: str,+ expected: str | None,ttl_secs: int | None = None,) -> bool:"""Compare-and-set one Hash field via Lua. See base."""@@ -527,11 +534,29 @@ class RedisMessageBus(MessageBus): # pylint: disable=too-many-public-methodsnamespace,field,value,- expected,+ expected if expected is not None else "",ttl_secs or 0,+ int(expected is None),)return bool(written)+ async def registry_del_if(+ self,+ namespace: str,+ field: str,+ *,+ expected: str,+ ) -> bool:+ """Compare and delete one Hash field in a single Lua operation."""+ deleted = await self._client.eval(+ _REGISTRY_DEL_IF_LUA,+ 1,+ namespace,+ field,+ expected,+ )+ return bool(deleted)+async def registry_pop(self, namespace: str, field: str) -> str | None:"""``HGET`` + ``HDEL`` as one Lua step. See base."""return await self._client.eval(diff --git a/src/agentscope/app/middleware/_tool_offload_middleware.py b/src/agentscope/app/middleware/_tool_offload_middleware.pyindex 257e37316..e5fe4deeb 100644--- a/src/agentscope/app/middleware/_tool_offload_middleware.py+++ b/src/agentscope/app/middleware/_tool_offload_middleware.py@@ -22,7 +22,7 @@ bus/wakeup infrastructure, which works correctly across processes.import asyncioimport jsonfrom copy import deepcopy-from typing import AsyncGenerator, Callable+from typing import Any, AsyncGenerator, Callablefrom .._manager import BackgroundTaskManagerfrom ...middleware import MiddlewareBase@@ -42,6 +42,69 @@ from ..._logging import logger# Sentinel object used to signal end-of-stream in the drain queue._QUEUE_SENTINEL = object()+_SUMMARY_MAX_LEN = 128+++def _make_summary(tool_name: str, raw_input: str | None) -> str:+ """Generate a short human-readable summary for frontend display.++ The result is guaranteed to be at most :data:`_SUMMARY_MAX_LEN`+ characters. For dict-shaped JSON inputs, multiple key=value pairs+ are included up to the length budget.++ This truncates inputs without redacting secrets. Argument values,+ nested values, commands, and malformed-input snippets may contain+ sensitive data. The summary is stored in shared task snapshots and+ returned by the user-scoped background-task REST endpoints. Redis+ stores this field as plaintext; persistence to disk and backups depends+ on the deployment configuration.++ Args:+ tool_name (`str`):+ The tool name (always included as prefix).+ raw_input (`str | None`):+ The raw JSON-encoded tool input string.++ Returns:+ `str`:+ A concise summary such as ``Bash(command=ls -la, ...)``.+ """+ if not raw_input:+ return tool_name[:_SUMMARY_MAX_LEN]++ try:+ parsed: Any = json.loads(raw_input)+ except (json.JSONDecodeError, TypeError):+ snippet = raw_input[:64]+ suffix = "..." if len(raw_input) > 64 else ""+ result = f"{tool_name}({snippet}{suffix})"+ return result[:_SUMMARY_MAX_LEN]++ if not isinstance(parsed, dict) or not parsed:+ return tool_name[:_SUMMARY_MAX_LEN]++ prefix = f"{tool_name}("+ suffix = ")"+ budget = _SUMMARY_MAX_LEN - len(prefix) - len(suffix)+ if budget <= 0:+ return tool_name[:_SUMMARY_MAX_LEN]++ parts: list[str] = []+ used = 0+ for key, val in parsed.items():+ val_str = str(val)+ if len(val_str) > 40:+ val_str = val_str[:37] + "..."+ part = f"{key}={val_str}"+ separator_len = 2 if parts else 0+ if used + len(part) + separator_len > budget:+ parts.append("...")+ break+ parts.append(part)+ used += len(part) + separator_len++ return (prefix + ", ".join(parts) + suffix)[:_SUMMARY_MAX_LEN]+class ToolOffloadMiddleware(MiddlewareBase): # pylint: disable=abstract-method"""Middleware that offloads timed-out tool calls to background tasks.@@ -166,15 +229,24 @@ class ToolOffloadMiddleware(MiddlewareBase): # pylint: disable=abstract-method# ----------------------------------------------------------------queue: asyncio.Queue = asyncio.Queue()- async def _drain_to_queue() -> None:- """Drain next_handler output into *queue*."""+ async def _drain_to_queue() -> ToolResponse | None:+ """Drain output and return the terminal tool outcome."""+ response = Nonetry:async for item in next_handler(**input_kwargs):+ if isinstance(item, ToolResponse):+ response = itemawait queue.put(item)except Exception as exc: # pylint: disable=broad-exceptawait queue.put(exc)+ response = ToolResponse(+ id=tool_call.id,+ state=ToolResultState.ERROR,+ content=[TextBlock(text=str(exc))],+ )finally:await queue.put(_QUEUE_SENTINEL)+ return responsedrain_task: asyncio.Task = asyncio.create_task(_drain_to_queue())@@ -225,7 +297,6 @@ class ToolOffloadMiddleware(MiddlewareBase): # pylint: disable=abstract-method# ----------------------------------------------------------------session_id = agent.state.session_idtool_name = tool_call.name- snapshot = list(pre_collected)logger.info("Tool '%s' timed out after %.1fs, offloading to background: "@@ -245,7 +316,7 @@ class ToolOffloadMiddleware(MiddlewareBase): # pylint: disable=abstract-methodcompletion notification in those edge cases."""try:- await drain_task+ response: ToolResponse | None = await drain_taskexcept asyncio.CancelledError:logger.info("Background tool '%s' cancelled, skipping delivery: "@@ -264,21 +335,12 @@ class ToolOffloadMiddleware(MiddlewareBase): # pylint: disable=abstract-method)return- remaining_items: list = []- while not queue.empty():- try:- item = queue.get_nowait()- if isinstance(item, ToolResponse):- remaining_items.append(item)- break- except asyncio.QueueEmpty:- break-- all_items = snapshot + remaining_items- response: ToolResponse | None = next(- (i for i in all_items if isinstance(i, ToolResponse)),- None,- )+ outcome = "completed"+ if response is not None:+ if response.state == ToolResultState.ERROR:+ outcome = "failed"+ elif response.state == ToolResultState.INTERRUPTED:+ outcome = "been cancelled"tool_call_id = tool_call.id@@ -295,7 +357,7 @@ class ToolOffloadMiddleware(MiddlewareBase): # pylint: disable=abstract-methodhint=(f"<system-notification>"f"Tool '{tool_name}' running in background "- f"(id={tool_call_id}) has completed with no output."+ f"(id={tool_call_id}) has {outcome} with no output."f"</system-notification>"),source=hint_source,@@ -311,7 +373,7 @@ class ToolOffloadMiddleware(MiddlewareBase): # pylint: disable=abstract-methodprefix = (f"<system-notification>"f"Tool '{tool_name}' running in background "- f"(id={tool_call_id}) has completed.\n\n"+ f"(id={tool_call_id}) has {outcome}.\n\n"f"Result:\n\n")@@ -340,9 +402,10 @@ class ToolOffloadMiddleware(MiddlewareBase): # pylint: disable=abstract-method# reasoning step, and WakeupDispatcher kicks an idle session# on whichever process picks up the wakeup.logger.info(- "Background tool '%s' completed, pushing result to inbox "+ "Background tool '%s' has %s, pushing result to inbox ""and enqueueing wakeup: session_id=%s",tool_name,+ outcome,session_id,)await deliver_to_inbox(@@ -360,6 +423,7 @@ class ToolOffloadMiddleware(MiddlewareBase): # pylint: disable=abstract-methodagent_id=self._agent_id,user_id=self._user_id,tool_name=tool_name,+ summary=_make_summary(tool_name, tool_call.input),)logger.info(diff --git a/tests/background_task_endpoint_test.py b/tests/background_task_endpoint_test.pynew file mode 100644index 000000000..e5093a55f--- /dev/null+++ b/tests/background_task_endpoint_test.py@@ -0,0 +1,1377 @@+# -*- coding: utf-8 -*-+# pylint: disable=protected-access+"""Regression tests for shared background task lifecycle and endpoints."""++import asyncio+import json+from dataclasses import fields, is_dataclass, replace+from datetime import datetime, timezone+from unittest.async_case import IsolatedAsyncioTestCase+from unittest.mock import AsyncMock, MagicMock, patch++import fakeredis.aioredis+import httpx++from agentscope.app import create_app+from agentscope.app._manager import (+ BackgroundTaskManager,+ CancelDispatcher,+ ChatRunRegistry,+ TaskStatus,+)+from agentscope.app._manager._background_task_manager import (+ _MAX_RETAINED_TASKS,+ BackgroundTask,+ BackgroundTaskRecord,+)+from agentscope.app.message_bus import (+ InMemoryMessageBus,+ MessageBus,+ MessageBusKeys,+ RedisMessageBus,+)+from agentscope.app.middleware._tool_offload_middleware import (+ _make_summary,+)++_MANAGER_MODULE = BackgroundTaskManager.__module__+++class TestBackgroundTaskCompatibility(IsolatedAsyncioTestCase):+ """Preserve the task dataclass API that existed before this feature."""++ async def test_legacy_construction_and_dataclass_operations(self) -> None:+ """Old positional and keyword construction keep their field order."""+ task = asyncio.create_task(asyncio.sleep(0))+ await task+ positional = BackgroundTask(task, "s1", "a1", "alice", "Tool", "id")+ keyword = BackgroundTask(+ asyncio_task=task,+ session_id="s1",+ agent_id="a1",+ user_id="alice",+ tool_name="Tool",+ id="id",+ started_at=positional.started_at,+ )+ expected = {+ "asyncio_task": task,+ "session_id": "s1",+ "agent_id": "a1",+ "user_id": "alice",+ "tool_name": "Tool",+ "id": "id",+ "summary": "",+ "started_at": positional.started_at,+ "completed_at": None,+ "status": TaskStatus.RUNNING,+ "error_summary": None,+ }+ self.assertTrue(is_dataclass(positional))+ self.assertEqual(+ [item.name for item in fields(BackgroundTask)],+ list(expected),+ )+ self.assertEqual(vars(positional), expected)+ self.assertEqual(positional, keyword)+ self.assertEqual(+ vars(replace(positional, tool_name="Other")),+ {**expected, "tool_name": "Other"},+ )+ self.assertEqual(vars(positional), expected)+ generated = BackgroundTask(task, "s1", "a1", "alice", "Tool")+ self.assertIsInstance(generated.id, str)+ self.assertNotEqual(generated.id, positional.id)++ async def test_shared_record_is_an_independent_snapshot(self) -> None:+ """Serialization excludes handles and cannot mutate the local task."""+ task = asyncio.create_task(asyncio.sleep(0))+ await task+ local = BackgroundTask(+ task,+ "s1",+ "a1",+ "alice",+ "Tool",+ "id",+ summary="Tool(x=1)",+ started_at=123.0,+ )+ record = BackgroundTaskRecord.model_validate(local)+ expected = {+ "id": "id",+ "session_id": "s1",+ "agent_id": "a1",+ "user_id": "alice",+ "tool_name": "Tool",+ "summary": "Tool(x=1)",+ "started_at": 123.0,+ "completed_at": None,+ "status": "running",+ "error_summary": None,+ }+ self.assertEqual(json.loads(record.model_dump_json()), expected)+ local.status = TaskStatus.COMPLETED+ local.completed_at = 124.0+ self.assertEqual(json.loads(record.model_dump_json()), expected)+ record.summary = "Changed"+ self.assertEqual(local.summary, "Tool(x=1)")+++class TestBackgroundTaskRecords( # pylint: disable=too-many-public-methods+ IsolatedAsyncioTestCase,+):+ """Exercise two workers sharing task metadata and cancellation signals."""++ def _make_bus(self) -> MessageBus:+ """Provide the shared backend for both managers."""+ return InMemoryMessageBus()++ async def asyncSetUp(self) -> None:+ """Keep the bus alive until both managers have finished cleanup."""+ self.bus = await self.enterAsyncContext(self._make_bus())+ self.owner = await self.enterAsyncContext(+ BackgroundTaskManager(self.bus),+ )+ self.reader = await self.enterAsyncContext(+ BackgroundTaskManager(self.bus),+ )++ async def _register(+ self,+ manager: BackgroundTaskManager,+ user_id: str = "alice",+ session_id: str = "s1",+ ) -> tuple[asyncio.Event, asyncio.Task, str]:+ """Register a tool that finishes only when explicitly released."""+ gate = asyncio.Event()+ task = asyncio.create_task(gate.wait())+ task_id = await manager.register_task(+ task,+ session_id,+ "a1",+ user_id,+ "Tool",+ summary="Tool(x=1)",+ )+ return gate, task, task_id++ async def _settle(self, *tasks: asyncio.Task) -> None:+ """Wait for execution and all metadata writes without timing sleeps."""+ await asyncio.gather(*tasks, return_exceptions=True)+ for manager in (self.owner, self.reader):+ while manager._completion_tasks:+ await asyncio.gather(*manager._completion_tasks)+ # Done callbacks may still be queued after gather returns.+ await asyncio.sleep(0)++ async def _renew_once(self) -> None:+ """Run one heartbeat pass without waiting for the lease interval."""+ self.owner._renewal_task.cancel()+ await asyncio.gather(+ self.owner._renewal_task,+ return_exceptions=True,+ )+ with patch(+ f"{_MANAGER_MODULE}.asyncio",+ wraps=asyncio,+ sleep=AsyncMock(side_effect=[None, asyncio.CancelledError]),+ ):+ try:+ await self.owner._renew_running_records()+ except asyncio.CancelledError:+ pass++ async def test_settle_waits_for_pending_done_callbacks(self) -> None:+ """Completed tasks must be discarded before the helper returns."""+ completed = asyncio.create_task(asyncio.sleep(0))+ await completed+ self.owner._completion_tasks.add(completed)+ completed.add_done_callback(self.owner._completion_tasks.discard)++ await self._settle()++ self.assertEqual(self.owner._completion_tasks, set())+ self.assertEqual(self.reader._completion_tasks, set())++ async def test_shared_lifecycle_and_isolation(self) -> None:+ """Running and completed records are identical on either worker."""+ gate, task, task_id = await self._register(self.owner)+ expected = BackgroundTaskRecord.model_validate(+ self.owner.tasks[task_id],+ ).model_dump()+ record = await self.reader.get_task(task_id, user_id="alice")+ self.assertEqual(record.model_dump(), expected)+ self.assertEqual(+ [+ r.model_dump()+ for r in await self.reader.list_tasks(+ user_id="alice",+ session_id="s1",+ )+ ],+ [expected],+ )+ self.assertEqual(+ await self.reader.list_tasks(user_id="alice", session_id="s2"),+ [],+ )+ self.assertEqual(await self.reader.list_tasks(user_id="bob"), [])+ self.assertIsNone(+ await self.reader.get_task(task_id, user_id="bob"),+ )+ self.assertIsNone(+ await self.reader.get_task("missing", user_id="alice"),+ )++ gate.set()+ await self._settle(task)+ record = await self.reader.get_task(task_id, user_id="alice")+ self.assertIsNotNone(record.completed_at)+ expected.update(+ status=TaskStatus.COMPLETED,+ completed_at=record.completed_at,+ )+ self.assertEqual(record.model_dump(), expected)+ self.assertEqual(self.owner.tasks, {})+ self.assertEqual(+ await self.bus.registry_getall(MessageBusKeys.bg_tasks("s1")),+ {},+ )++ async def test_exception_records_failure(self) -> None:+ """An exception preserves its bounded error summary on any worker."""+ gate = asyncio.Event()++ async def fail() -> None:+ await gate.wait()+ raise ValueError("x" * 600)++ task = asyncio.create_task(fail())+ task_id = await self.owner.register_task(task, "s1", "a1", "alice")+ expected = BackgroundTaskRecord.model_validate(+ self.owner.tasks[task_id],+ ).model_dump()+ gate.set()+ await self._settle(task)+ record = await self.reader.get_task(task_id, user_id="alice")+ expected.update(+ status=TaskStatus.FAILED,+ error_summary="x" * 512,+ completed_at=record.completed_at,+ )+ self.assertIsNotNone(record.completed_at)+ self.assertEqual(record.model_dump(), expected)++ async def test_local_and_remote_toolstop(self) -> None:+ """Both cancellation paths retain status and remove live entries."""+ registry = await self.enterAsyncContext(ChatRunRegistry())+ await self.enterAsyncContext(+ CancelDispatcher(self.bus, registry, self.owner),+ )+ for caller in (self.owner, self.reader):+ with self.subTest(local=caller is self.owner):+ _, task, task_id = await self._register(self.owner)+ expected = BackgroundTaskRecord.model_validate(+ self.owner.tasks[task_id],+ ).model_dump()+ stop = (await caller.list_tools("s1"))[0]+ await stop(task_id=task_id)+ await asyncio.wait_for(self._settle(task), timeout=5)+ record = await self.reader.get_task(+ task_id,+ user_id="alice",+ )+ self.assertTrue(task.cancelled())+ self.assertIsNotNone(record.completed_at)+ expected.update(+ status=TaskStatus.CANCELLED,+ completed_at=record.completed_at,+ )+ self.assertEqual(record.model_dump(), expected)+ self.assertEqual(self.owner.tasks, {})+ self.assertEqual(+ await self.bus.registry_getall(+ MessageBusKeys.bg_tasks("s1"),+ ),+ {},+ )+ self.assertFalse(self.owner.cancel_task(task_id))++ async def test_shutdown_waits_for_shared_cleanup(self) -> None:+ """Shutdown returns only after cancelled states and cleanup persist."""+ _, task, task_id = await self._register(self.owner)+ expected = BackgroundTaskRecord.model_validate(+ self.owner.tasks[task_id],+ ).model_dump()+ await self.owner.__aexit__()+ record = await self.reader.get_task(task_id, user_id="alice")+ expected.update(+ status=TaskStatus.CANCELLED,+ completed_at=record.completed_at,+ )+ self.assertTrue(task.cancelled())+ self.assertEqual(record.model_dump(), expected)+ self.assertEqual(self.owner.tasks, {})+ self.assertEqual(self.owner._completion_tasks, set())+ self.assertEqual(+ await self.bus.registry_getall(MessageBusKeys.bg_tasks("s1")),+ {},+ )++ async def test_retention_is_shared_and_scoped_per_user(self) -> None:+ """Concurrent completions evict oldest outcomes, never running work."""+ _, _, running_id = await self._register(self.owner)+ running = BackgroundTaskRecord.model_validate(+ self.owner.tasks[running_id],+ ).model_dump()+ bob_gate, bob_task, bob_id = await self._register(+ self.owner,+ user_id="bob",+ )+ work = []+ records = []+ for index in range(_MAX_RETAINED_TASKS + 5):+ manager = (self.owner, self.reader)[index % 2]+ gate, task, task_id = await self._register(manager)+ records.append(manager.tasks[task_id])+ work.append((gate, task))+ for gate, _ in work:+ gate.set()+ bob_gate.set()+ await self._settle(bob_task, *(task for _, task in work))++ retained = sorted(+ records,+ key=lambda record: (record.completed_at, record.id),+ )[-_MAX_RETAINED_TASKS:]+ expected = [running] + [+ BackgroundTaskRecord.model_validate(record).model_dump()+ for record in retained+ ]+ expected.sort(key=lambda record: (record["started_at"], record["id"]))+ self.assertEqual(+ [+ record.model_dump()+ for record in await self.reader.list_tasks(+ user_id="alice",+ )+ ],+ expected,+ )+ bob_record = await self.reader.get_task(bob_id, user_id="bob")+ self.assertEqual(+ [+ record.model_dump()+ for record in await self.reader.list_tasks(+ user_id="bob",+ )+ ],+ [bob_record.model_dump()],+ )+ self.assertEqual(bob_record.status, TaskStatus.COMPLETED)+ self.assertEqual(set(self.owner.tasks), {running_id})+ self.assertEqual(self.reader.tasks, {})++ async def test_failed_initial_write_still_cleans_up(self) -> None:+ """A partial registration cannot leave an untracked running task."""+ original_set = self.bus.registry_set++ async def fail_running_record(+ namespace: str,+ field: str,+ value: str,+ **kwargs: int,+ ) -> None:+ if (+ namespace == MessageBusKeys.bg_task_records("alice")+ and json.loads(value)["status"] == "running"+ ):+ raise ConnectionError("record write failed")+ await original_set(namespace, field, value, **kwargs)++ task = asyncio.create_task(asyncio.Event().wait())+ with patch.object(+ self.bus,+ "registry_set",+ side_effect=fail_running_record,+ ):+ with self.assertRaisesRegex(ConnectionError, "record write"):+ await self.owner.register_task(task, "s1", "a1", "alice")+ await self._settle(task)+ self.assertTrue(task.cancelled())+ self.assertEqual(self.owner.tasks, {})+ self.assertEqual(+ await self.bus.registry_getall(MessageBusKeys.bg_tasks("s1")),+ {},+ )++ async def test_initial_write_timeout_cleans_up_all_outcomes(self) -> None:+ """Either stalled initial write releases registration and handles."""+ original_set = self.bus.registry_set+ for namespace in (+ MessageBusKeys.bg_tasks("s1"),+ MessageBusKeys.bg_task_records("alice"),+ ):+ for completed in (False, True):+ with self.subTest(namespace=namespace, completed=completed):+ started = asyncio.Event()+ cancelled = asyncio.Event()+ gate = asyncio.Event()+ task = asyncio.create_task(gate.wait())++ async def stall_initial_write(+ ns: str,+ field: str,+ value: str,+ *,+ ttl_secs: int | None = None,+ ) -> None:+ # Each subtest finishes this write before advancing.+ # pylint: disable=cell-var-from-loop+ payload = json.loads(value)+ if (+ ns == namespace+ and payload.get("status", "running") == "running"+ ):+ # The server may commit before the reply stalls.+ await original_set(+ ns,+ field,+ value,+ ttl_secs=ttl_secs,+ )+ started.set()+ try:+ await asyncio.Event().wait()+ finally:+ cancelled.set()+ await original_set(+ ns,+ field,+ value,+ ttl_secs=ttl_secs,+ )++ with (+ patch.object(+ self.bus,+ "registry_set",+ side_effect=stall_initial_write,+ ),+ patch(f"{_MANAGER_MODULE}._IO_TIMEOUT_SECS", 0.1),+ ):+ registration = asyncio.create_task(+ self.owner.register_task(+ task,+ "s1",+ "a1",+ "alice",+ ),+ )+ try:+ await asyncio.wait_for(started.wait(), timeout=1)+ (task_id,) = self.owner.tasks+ expected = BackgroundTaskRecord.model_validate(+ self.owner.tasks[task_id],+ ).model_dump()+ if completed:+ gate.set()+ await task+ done, _ = await asyncio.wait(+ {registration},+ timeout=0.5,+ )+ self.assertEqual(done, {registration})+ with self.assertRaises(TimeoutError):+ await registration+ self.assertTrue(cancelled.is_set())+ await asyncio.wait_for(self._settle(task), 1)+ finally:+ registration.cancel()+ await asyncio.gather(+ registration,+ return_exceptions=True,+ )+ self.assertEqual(self.owner.tasks, {})+ self.assertEqual(task.cancelled(), not completed)+ record = await self.reader.get_task(+ task_id,+ user_id="alice",+ )+ expected.update(+ status=TaskStatus.COMPLETED+ if completed+ else TaskStatus.CANCELLED,+ completed_at=record.completed_at,+ )+ self.assertEqual(record.model_dump(), expected)+ self.assertEqual(+ await self.bus.registry_getall(+ MessageBusKeys.bg_tasks("s1"),+ ),+ {},+ )++ async def test_failed_outcome_write_recovers_shared_status(self) -> None:+ """A failed write remains pending and is retried after recovery."""+ gate, task, task_id = await self._register(self.owner)+ expected = BackgroundTaskRecord.model_validate(+ self.owner.tasks[task_id],+ ).model_dump()+ failed = asyncio.Event()++ async def unavailable(_record: BackgroundTaskRecord) -> None:+ failed.set()+ raise ConnectionError("unavailable")++ with patch.object(+ self.owner,+ "_store_record",+ side_effect=unavailable,+ ):+ gate.set()+ await asyncio.wait_for(failed.wait(), timeout=1)+ self.assertEqual(self.owner.tasks, {})+ self.assertTrue(self.owner._completion_tasks)+ record = await self.reader.get_task(task_id, user_id="alice")+ self.assertEqual(record.model_dump(), expected)+ await asyncio.wait_for(self._settle(task), timeout=2)+ record = await self.reader.get_task(task_id, user_id="alice")+ self.assertIsNotNone(record.completed_at)+ expected.update(+ status=TaskStatus.COMPLETED,+ completed_at=record.completed_at,+ )+ self.assertEqual(record.model_dump(), expected)+ self.assertEqual(+ await self.bus.registry_getall(MessageBusKeys.bg_tasks("s1")),+ {},+ )+ self.assertEqual(self.owner._completion_tasks, set())++ async def test_failed_shutdown_record_expires_despite_new_tasks(+ self,+ ) -> None:+ """A lost outcome cannot remain running through user-hash renewal."""+ gate, task, task_id = await self._register(self.owner)+ namespace = MessageBusKeys.bg_task_records("alice")+ value = await self.bus.registry_get(namespace, task_id)+ expires_at = json.loads(value)["_expires_at"]+ failed = asyncio.Event()++ async def unavailable(_record: BackgroundTaskRecord) -> None:+ failed.set()+ raise ConnectionError("unavailable")++ with (+ patch.object(self.owner, "_store_record", side_effect=unavailable),+ patch(f"{_MANAGER_MODULE}._SHUTDOWN_TIMEOUT_SECS", 0.1),+ ):+ gate.set()+ await asyncio.wait_for(failed.wait(), timeout=1)+ await asyncio.wait_for(self.owner.__aexit__(), timeout=0.5)+ self.assertTrue(task.done())+ self.assertEqual(self.owner._completion_tasks, set())+ self.assertEqual(+ await self.bus.registry_get(namespace, task_id),+ value,+ )++ with patch(+ f"{_MANAGER_MODULE}.time.time",+ return_value=expires_at + 1,+ ):+ _, _, new_id = await self._register(self.reader)+ expected = BackgroundTaskRecord.model_validate(+ self.reader.tasks[new_id],+ ).model_dump()+ self.assertIsNone(+ await self.reader.get_task(task_id, user_id="alice"),+ )+ self.assertEqual(+ [+ record.model_dump()+ for record in await self.reader.list_tasks(+ user_id="alice",+ )+ ],+ [expected],+ )+ # Querying removes the expired field despite hash TTL renewal.+ self.assertIsNone(+ await self.bus.registry_get(namespace, task_id),+ )++ async def test_lists_remove_expired_records_as_user_activity_continues(+ self,+ ) -> None:+ """Repeated user activity cannot accumulate invisible running rows."""+ namespace = MessageBusKeys.bg_task_records("alice")+ expected = {}+ for batch in range(3):+ _, _, task_id = await self._register(self.owner)+ value = await self.bus.registry_get(namespace, task_id)+ expected[task_id] = value+ for index in range(5):+ stale_id = f"stale-{batch}-{index}"+ payload = json.loads(value)+ payload.update(id=stale_id, _expires_at=0)+ await self.bus.registry_set(+ namespace,+ stale_id,+ json.dumps(payload),+ ttl_secs=MessageBusKeys.BG_TASKS_TTL_SECS,+ )+ records = await self.reader.list_tasks(user_id="alice")+ self.assertEqual(+ {record.id: record.model_dump() for record in records},+ {+ task_id: BackgroundTaskRecord.model_validate_json(+ value,+ ).model_dump()+ for task_id, value in expected.items()+ },+ )+ self.assertEqual(+ await self.bus.registry_getall(namespace),+ expected,+ )++ async def test_completion_evicts_expired_running_and_terminal_records(+ self,+ ) -> None:+ """Finalization physically clears expired records of every status."""+ gate, task, task_id = await self._register(self.owner)+ namespace = MessageBusKeys.bg_task_records("alice")+ value = await self.bus.registry_get(namespace, task_id)+ for status in TaskStatus:+ stale_id = f"expired-{status.value}"+ payload = json.loads(value)+ payload.update(+ id=stale_id,+ status=status.value,+ _expires_at=0,+ completed_at=None if status == TaskStatus.RUNNING else 1.0,+ )+ await self.bus.registry_set(+ namespace,+ stale_id,+ json.dumps(payload),+ )+ gate.set()+ await self._settle(task)+ self.assertEqual(+ await self.bus.registry_getall(namespace),+ {task_id: await self.bus.registry_get(namespace, task_id)},+ )++ async def test_expiry_cleanup_preserves_concurrent_updates(self) -> None:+ """Queries and eviction cannot delete renewed or completed records."""+ namespace = MessageBusKeys.bg_task_records("alice")+ original_delete = self.bus.registry_del_if+ original_set = self.bus.registry_set+ for operation in ("list", "detail", "eviction"):+ for status in (TaskStatus.RUNNING, TaskStatus.COMPLETED):+ with self.subTest(operation=operation, status=status):+ task_id = f"race-{operation}-{status.value}"+ payload = {+ "id": task_id,+ "session_id": "s1",+ "agent_id": "a1",+ "user_id": "alice",+ "tool_name": "Tool",+ "summary": "Tool(x=1)",+ "started_at": 0,+ "completed_at": None,+ "status": "running",+ "error_summary": None,+ "_expires_at": 0,+ }+ await original_set(namespace, task_id, json.dumps(payload))+ payload.update(+ status=status.value,+ completed_at=1.0+ if status == TaskStatus.COMPLETED+ else None,+ _expires_at=32503680000,+ )+ updated = json.dumps(payload)++ async def concurrent_update(+ ns: str,+ field: str,+ *,+ expected: str,+ new_value: str = updated,+ ) -> bool:+ await original_set(ns, field, new_value)+ return await original_delete(+ ns,+ field,+ expected=expected,+ )++ with patch.object(+ self.bus,+ "registry_del_if",+ side_effect=concurrent_update,+ ):+ if operation == "list":+ await self.reader.list_tasks(user_id="alice")+ elif operation == "detail":+ await self.reader.get_task(+ task_id,+ user_id="alice",+ )+ else:+ await self.reader._evict_completed("alice")+ self.assertEqual(+ await self.bus.registry_get(namespace, task_id),+ updated,+ )+ record = await self.reader.get_task(+ task_id,+ user_id="alice",+ )+ self.assertEqual(+ record.model_dump(),+ BackgroundTaskRecord.model_validate(+ payload,+ ).model_dump(),+ )++ async def test_retention_lock_does_not_block_outcome_or_shutdown(+ self,+ ) -> None:+ """A held retention lock cannot delay state writes or exhaust exit."""+ gate, task, task_id = await self._register(self.owner)+ expected = BackgroundTaskRecord.model_validate(+ self.owner.tasks[task_id],+ ).model_dump()+ cleaned = asyncio.Event()+ original_del = self.bus.registry_del++ async def observe_cleanup(namespace: str, field: str) -> None:+ await original_del(namespace, field)+ if namespace == MessageBusKeys.bg_tasks("s1"):+ cleaned.set()++ async with self.bus.acquire_lock(+ MessageBusKeys.bg_task_records_lock("alice"),+ ttl_secs=600,+ ):+ with (+ patch.object(+ self.bus,+ "registry_del",+ side_effect=observe_cleanup,+ ),+ patch(f"{_MANAGER_MODULE}._SHUTDOWN_TIMEOUT_SECS", 0.1),+ patch(f"{_MANAGER_MODULE}._IO_TIMEOUT_SECS", 0.02),+ ):+ gate.set()+ await asyncio.wait_for(cleaned.wait(), timeout=1)+ self.assertTrue(task.done())+ self.assertEqual(self.owner.tasks, {})+ record = await self.reader.get_task(task_id, user_id="alice")+ expected.update(+ status=TaskStatus.COMPLETED,+ completed_at=record.completed_at,+ )+ self.assertEqual(record.model_dump(), expected)+ await asyncio.wait_for(self.owner.__aexit__(), timeout=0.5)+ self.assertEqual(self.owner._completion_tasks, set())++ async def test_stalled_outcome_io_times_out_and_retries(self) -> None:+ """A nonresponsive write gets a deadline and can recover on retry."""+ gate, task, task_id = await self._register(self.owner)+ stalled = asyncio.Event()+ original_store = self.owner._store_record++ async def stall_once(record: BackgroundTaskRecord) -> None:+ if not stalled.is_set():+ stalled.set()+ await asyncio.Event().wait()+ await original_store(record)++ with (+ patch.object(self.owner, "_store_record", side_effect=stall_once),+ patch(f"{_MANAGER_MODULE}._IO_TIMEOUT_SECS", 0.02),+ ):+ gate.set()+ await asyncio.wait_for(stalled.wait(), timeout=1)+ await asyncio.wait_for(self._settle(task), timeout=2)+ record = await self.reader.get_task(task_id, user_id="alice")+ self.assertEqual(record.status, TaskStatus.COMPLETED)+ self.assertEqual(+ await self.bus.registry_getall(MessageBusKeys.bg_tasks("s1")),+ {},+ )++ async def test_shutdown_cancels_stalled_cleanup_io(self) -> None:+ """Shutdown is bounded even if a registry operation never responds."""+ gate, task, task_id = await self._register(self.owner)+ blocked = asyncio.Event()+ cancelled = asyncio.Event()++ async def stall_delete(_namespace: str, _field: str) -> None:+ blocked.set()+ try:+ await asyncio.Event().wait()+ finally:+ cancelled.set()++ with (+ patch.object(self.bus, "registry_del", side_effect=stall_delete),+ patch(f"{_MANAGER_MODULE}._SHUTDOWN_TIMEOUT_SECS", 0.1),+ ):+ gate.set()+ await asyncio.wait_for(blocked.wait(), timeout=1)+ self.assertTrue(task.done())+ await asyncio.wait_for(self.owner.__aexit__(), timeout=0.5)+ self.assertTrue(cancelled.is_set())+ self.assertEqual(self.owner._completion_tasks, set())+ record = await self.reader.get_task(task_id, user_id="alice")+ self.assertEqual(record.status, TaskStatus.COMPLETED)++ async def test_running_task_renews_its_own_lease(self) -> None:+ """Long-running tasks stay visible beyond their initial lease."""+ renewed = asyncio.Event()+ original_set_if = self.bus.registry_set_if++ async def observe_renewal(+ namespace: str,+ field: str,+ value: str,+ *,+ expected: str,+ ttl_secs: int | None = None,+ ) -> bool:+ result = await original_set_if(+ namespace,+ field,+ value,+ expected=expected,+ ttl_secs=ttl_secs,+ )+ renewed.set()+ return result++ with patch(f"{_MANAGER_MODULE}._RECORD_LEASE_SECS", 0.04):+ gate, task, task_id = await self._register(self.owner)+ value = await self.bus.registry_get(+ MessageBusKeys.bg_task_records("alice"),+ task_id,+ )+ future_time = json.loads(value)["_expires_at"] + 1+ with (+ patch(+ f"{_MANAGER_MODULE}.time.time",+ return_value=future_time,+ ),+ patch.object(+ self.bus,+ "registry_set_if",+ side_effect=observe_renewal,+ ),+ ):+ await asyncio.wait_for(renewed.wait(), timeout=1)+ expected = BackgroundTaskRecord.model_validate(+ self.owner.tasks[task_id],+ ).model_dump()+ record = await self.reader.get_task(task_id, user_id="alice")+ self.assertEqual(record.model_dump(), expected)+ gate.set()+ await self._settle(task)++ async def test_slow_io_keeps_all_running_tasks_visible(self) -> None:+ """Renewal latency must not accumulate across every running task."""+ original_get = self.bus.registry_get+ original_set_if = self.bus.registry_set_if++ async def delayed_get(namespace: str, field: str) -> str | None:+ await asyncio.sleep(0.04)+ return await original_get(namespace, field)++ async def delayed_set_if(+ namespace: str,+ field: str,+ value: str,+ *,+ expected: str | None,+ ttl_secs: int | None = None,+ ) -> bool:+ await asyncio.sleep(0.04)+ return await original_set_if(+ namespace,+ field,+ value,+ expected=expected,+ ttl_secs=ttl_secs,+ )++ with patch(f"{_MANAGER_MODULE}._RECORD_LEASE_SECS", 0.6):+ for _ in range(8):+ await self._register(self.owner)+ expected = {+ task.id: BackgroundTaskRecord.model_validate(task).model_dump()+ for task in self.owner.tasks.values()+ }+ with (+ patch.object(+ self.bus,+ "registry_get",+ side_effect=delayed_get,+ ),+ patch.object(+ self.bus,+ "registry_set_if",+ side_effect=delayed_set_if,+ ),+ ):+ await asyncio.sleep(0.65)+ for _ in range(4):+ records = await self.reader.list_tasks(user_id="alice")+ self.assertEqual(+ {record.id: record.model_dump() for record in records},+ expected,+ )+ self.assertEqual(+ [+ task.asyncio_task.done()+ for task in self.owner.tasks.values()+ ],+ [False] * 8,+ )+ await asyncio.sleep(0.1)++ async def test_renewal_budget_preserves_fairness_and_cancels_io(+ self,+ ) -> None:+ """Slow tasks cannot starve the queue or leave I/O after shutdown."""+ original_get = self.bus.registry_get+ active: set[asyncio.Task] = set()+ seen: set[str] = set()+ peak = 0+ cancellations = 0+ reached_tail = asyncio.Event()+ with (+ patch(f"{_MANAGER_MODULE}._RECORD_LEASE_SECS", 0.2),+ patch(f"{_MANAGER_MODULE}._RENEWAL_CONCURRENCY", 2),+ ):+ for _ in range(6):+ await self._register(self.owner)+ task_ids = list(self.owner.tasks)++ async def stalled_head(namespace: str, field: str) -> str | None:+ nonlocal peak, cancellations+ current = asyncio.current_task()+ assert current is not None+ active.add(current)+ peak = max(peak, len(active))+ try:+ seen.add(field)+ if seen == set(task_ids):+ reached_tail.set()+ if field in task_ids[:2]:+ try:+ await asyncio.Event().wait()+ except asyncio.CancelledError:+ cancellations += 1+ raise+ return await original_get(namespace, field)+ finally:+ active.remove(current)++ with patch.object(+ self.bus,+ "registry_get",+ side_effect=stalled_head,+ ):+ await asyncio.wait_for(reached_tail.wait(), timeout=0.5)+ self.assertEqual(peak, 2)+ self.assertGreaterEqual(cancellations, 2)+ self.assertEqual(seen, set(task_ids))+ await asyncio.wait_for(self.owner.__aexit__(), timeout=1)+ self.assertEqual(active, set())+ self.assertTrue(self.owner._renewal_task.done())++ async def test_registration_during_round_timeout_keeps_one_scheduler(+ self,+ ) -> None:+ """Temporary timeout cancellation must not start a second scheduler."""+ cancelling = asyncio.Event()+ allow_cleanup = asyncio.Event()+ resumed = asyncio.Event()+ original_renew = self.owner._renew_running_record++ async def delayed_cleanup(task: BackgroundTask) -> None:+ if not allow_cleanup.is_set():+ try:+ await asyncio.Event().wait()+ except asyncio.CancelledError:+ cancelling.set()+ await allow_cleanup.wait()+ raise+ await original_renew(task)+ resumed.set()++ with (+ patch(f"{_MANAGER_MODULE}._RECORD_LEASE_SECS", 0.2),+ patch(f"{_MANAGER_MODULE}._RENEWAL_CONCURRENCY", 2),+ patch.object(+ self.owner,+ "_renew_running_record",+ side_effect=delayed_cleanup,+ ),+ ):+ await self._register(self.owner)+ scheduler = self.owner._renewal_task+ try:+ await asyncio.wait_for(cancelling.wait(), timeout=2)+ self.assertGreater(scheduler.cancelling(), 0)+ self.assertFalse(scheduler.done())+ for _ in range(3):+ await self._register(self.owner)+ self.assertIs(self.owner._renewal_task, scheduler)++ allow_cleanup.set()+ await asyncio.wait_for(resumed.wait(), timeout=2)+ self.assertIs(self.owner._renewal_task, scheduler)+ self.assertFalse(scheduler.done())+ await self.owner.__aexit__()+ self.assertTrue(scheduler.done())+ self.assertIs(self.owner._renewal_task, scheduler)+ finally:+ allow_cleanup.set()+ scheduler.cancel()+ await asyncio.gather(scheduler, return_exceptions=True)++ async def test_renewal_reuses_live_scheduler_and_restarts_after_exit(+ self,+ ) -> None:+ """New work reuses a live scheduler, including its idle interval."""+ sleeping = asyncio.Event()+ release_sleep = asyncio.Event()+ original_sleep = asyncio.sleep++ async def hold_first_sleep(delay: float) -> None:+ if not sleeping.is_set():+ sleeping.set()+ try:+ await release_sleep.wait()+ except asyncio.CancelledError:+ await release_sleep.wait()+ raise+ else:+ await original_sleep(delay)++ with (+ patch(f"{_MANAGER_MODULE}._RECORD_LEASE_SECS", 0.2),+ patch(+ f"{_MANAGER_MODULE}.asyncio",+ wraps=asyncio,+ sleep=AsyncMock(side_effect=hold_first_sleep),+ ),+ ):+ gate, task, _ = await self._register(self.owner)+ scheduler = self.owner._renewal_task+ try:+ await asyncio.wait_for(sleeping.wait(), timeout=2)+ gate.set()+ await self._settle(task)+ self.assertEqual(scheduler.cancelling(), 0)+ self.assertFalse(scheduler.done())++ gate, task, _ = await self._register(self.owner)+ self.assertIs(self.owner._renewal_task, scheduler)+ gate.set()+ await self._settle(task)+ release_sleep.set()+ await asyncio.wait_for(asyncio.shield(scheduler), timeout=2)++ await self._register(self.owner)+ replacement = self.owner._renewal_task+ self.assertIsNot(replacement, scheduler)+ self.assertFalse(replacement.done())+ await self.owner.__aexit__()+ self.assertTrue(replacement.done())+ finally:+ release_sleep.set()+ scheduler.cancel()+ await asyncio.gather(scheduler, return_exceptions=True)++ async def test_heartbeat_restores_a_cleaned_running_record(self) -> None:+ """A live task reappears after its expired snapshot is deleted."""+ _, _, task_id = await self._register(self.owner)+ namespace = MessageBusKeys.bg_task_records("alice")+ value = await self.bus.registry_get(namespace, task_id)+ payload = json.loads(value)+ payload["_expires_at"] = 0+ await self.bus.registry_set(namespace, task_id, json.dumps(payload))+ self.assertEqual(await self.reader.list_tasks(user_id="alice"), [])+ self.assertIsNone(await self.bus.registry_get(namespace, task_id))++ await self._renew_once()++ expected = BackgroundTaskRecord.model_validate(+ self.owner.tasks[task_id],+ ).model_dump()+ record = await self.reader.get_task(task_id, user_id="alice")+ self.assertIsNotNone(record)+ self.assertEqual(record.model_dump(), expected)+ self.assertEqual(+ [+ r.model_dump()+ for r in await self.reader.list_tasks(+ user_id="alice",+ )+ ],+ [expected],+ )++ async def test_heartbeat_restore_preserves_concurrent_outcome(+ self,+ ) -> None:+ """An outcome written after a missing read wins over recovery."""+ _, _, task_id = await self._register(self.owner)+ namespace = MessageBusKeys.bg_task_records("alice")+ await self.bus.registry_del(namespace, task_id)+ terminal = BackgroundTaskRecord.model_validate(+ self.owner.tasks[task_id],+ )+ terminal.status = TaskStatus.COMPLETED+ terminal.completed_at = terminal.started_at + 1+ original_set_if = self.bus.registry_set_if++ async def complete_before_write(+ namespace: str,+ field: str,+ value: str,+ *,+ expected: str | None,+ ttl_secs: int | None = None,+ ) -> bool:+ await self.owner._store_record(terminal)+ return await original_set_if(+ namespace,+ field,+ value,+ expected=expected,+ ttl_secs=ttl_secs,+ )++ with patch.object(+ self.bus,+ "registry_set_if",+ side_effect=complete_before_write,+ ) as restore:+ await self._renew_once()+ restore.assert_awaited_once()+ self.assertIsNone(restore.call_args.kwargs["expected"])+ record = await self.reader.get_task(task_id, user_id="alice")+ self.assertEqual(record.model_dump(), terminal.model_dump())++ async def test_heartbeat_skips_task_completed_during_read(self) -> None:+ """A missing read cannot revive a task that has already finished."""+ gate, task, task_id = await self._register(self.owner)+ namespace = MessageBusKeys.bg_task_records("alice")+ await self.bus.registry_del(namespace, task_id)+ original_get = self.bus.registry_get++ async def complete_during_read(+ namespace: str,+ field: str,+ ) -> str | None:+ value = await original_get(namespace, field)+ gate.set()+ await self._settle(task)+ return value++ with (+ patch.object(+ self.bus,+ "registry_get",+ side_effect=complete_during_read,+ ),+ patch.object(+ self.bus,+ "registry_set_if",+ wraps=self.bus.registry_set_if,+ ) as restore,+ ):+ await self._renew_once()+ restore.assert_not_awaited()+ record = await self.reader.get_task(task_id, user_id="alice")+ self.assertEqual(record.status, TaskStatus.COMPLETED)++ async def test_http_routes_read_another_workers_records(self) -> None:+ """The actual app exposes filtered lists, details and isolated 404s."""+ gate, task, task_id = await self._register(self.owner)+ record = await self.reader.get_task(task_id, user_id="alice")+ expected = {+ "task_id": task_id,+ "session_id": "s1",+ "agent_id": "a1",+ "tool_name": "Tool",+ "summary": "Tool(x=1)",+ "status": "running",+ "started_at": datetime.fromtimestamp(+ record.started_at,+ tz=timezone.utc,+ )+ .isoformat()+ .replace("+00:00", "Z"),+ "completed_at": None,+ "error_summary": None,+ }+ app = create_app(+ storage=MagicMock(),+ message_bus=self.bus,+ workspace_manager=MagicMock(),+ )+ app.state.background_task_manager = self.reader+ client = await self.enterAsyncContext(+ httpx.AsyncClient(+ transport=httpx.ASGITransport(app=app),+ base_url="http://test",+ ),+ )+ headers = {"X-User-ID": "alice"}+ for query, body in (+ ("", {"tasks": [expected], "total": 1}),+ ("?session_id=s1", {"tasks": [expected], "total": 1}),+ ("?session_id=s2", {"tasks": [], "total": 0}),+ ):+ response = await client.get(+ f"/background-tasks{query}",+ headers=headers,+ )+ self.assertEqual(+ (response.status_code, response.json()),+ (+ 200,+ body,+ ),+ )+ response = await client.get(+ f"/background-tasks/{task_id}",+ headers=headers,+ )+ self.assertEqual(+ (response.status_code, response.json()),+ (+ 200,+ expected,+ ),+ )+ for missing_id, user in ((task_id, "bob"), ("missing", "alice")):+ response = await client.get(+ f"/background-tasks/{missing_id}",+ headers={"X-User-ID": user},+ )+ self.assertEqual(+ (response.status_code, response.json()),+ (+ 404,+ {"detail": f"Background task '{missing_id}' not found."},+ ),+ )+ response = await client.get("/background-tasks")+ self.assertEqual(response.status_code, 422)+ gate.set()+ await self._settle(task)+ record = await self.reader.get_task(task_id, user_id="alice")+ expected.update(+ status="completed",+ completed_at=datetime.fromtimestamp(+ record.completed_at,+ tz=timezone.utc,+ )+ .isoformat()+ .replace("+00:00", "Z"),+ )+ response = await client.get(+ f"/background-tasks/{task_id}",+ headers=headers,+ )+ self.assertEqual(+ (response.status_code, response.json()),+ (+ 200,+ expected,+ ),+ )+++class TestRedisBackgroundTaskRecords(TestBackgroundTaskRecords):+ """Run the same worker and lifecycle regressions through Redis APIs."""++ def _make_bus(self) -> MessageBus:+ """Use a fake Redis server without opening network connections."""+ client = fakeredis.aioredis.FakeRedis(decode_responses=True)+ self.addAsyncCleanup(client.aclose)+ return RedisMessageBus(connection_pool=client.connection_pool)+++class TestMakeSummary(IsolatedAsyncioTestCase):+ """Test the _make_summary helper function."""++ async def test_empty_input_returns_tool_name(self) -> None:+ """Empty or None input returns just the tool name."""+ self.assertEqual(_make_summary("Bash", None), "Bash")+ self.assertEqual(_make_summary("Bash", ""), "Bash")++ async def test_dict_input_includes_key_values(self) -> None:+ """JSON dict input produces key=value summary."""+ raw = json.dumps({"command": "ls -la", "timeout": 30})+ result = _make_summary("Bash", raw)+ self.assertIn("Bash(", result)+ self.assertIn("command=ls -la", result)+ self.assertIn("timeout=30", result)+ self.assertTrue(result.endswith(")"))++ async def test_respects_max_length(self) -> None:+ """Summary never exceeds _SUMMARY_MAX_LEN characters."""+ raw = json.dumps({"key": "x" * 200, "other": "y" * 200})+ result = _make_summary("VeryLongToolName", raw)+ self.assertLessEqual(len(result), 128)++ async def test_non_json_input_is_truncated(self) -> None:+ """Non-JSON input is included as-is with truncation."""+ raw = "a" * 100+ result = _make_summary("Tool", raw)+ self.assertIn("Tool(", result)+ self.assertIn("...", result)+ self.assertLessEqual(len(result), 128)++ async def test_non_json_short_input_no_ellipsis(self) -> None:+ """Short non-JSON input doesn't get a misleading ellipsis."""+ result = _make_summary("Tool", "hi")+ self.assertEqual(result, "Tool(hi)")++ async def test_empty_dict_returns_tool_name(self) -> None:+ """Empty JSON dict returns just the tool name."""+ result = _make_summary("Tool", "{}")+ self.assertEqual(result, "Tool")++ async def test_non_dict_json_returns_tool_name(self) -> None:+ """JSON that's not a dict returns just the tool name."""+ result = _make_summary("Tool", "[1, 2, 3]")+ self.assertEqual(result, "Tool")++ async def test_long_values_are_truncated(self) -> None:+ """Individual values longer than 40 chars are truncated."""+ raw = json.dumps({"file": "/" + "a" * 100})+ result = _make_summary("Read", raw)+ self.assertIn("...", result)+ self.assertLessEqual(len(result), 128)++ async def test_multiple_keys_included_within_budget(self) -> None:+ """Multiple short key-values are all included."""+ raw = json.dumps({"a": "1", "b": "2", "c": "3"})+ result = _make_summary("T", raw)+ self.assertIn("a=1", result)+ self.assertIn("b=2", result)+ self.assertIn("c=3", result)diff --git a/tests/background_task_manager_test.py b/tests/background_task_manager_test.pyindex d65c49bd8..2858d277d 100644--- a/tests/background_task_manager_test.py+++ b/tests/background_task_manager_test.py@@ -33,6 +33,14 @@ class _RegistrationBus(InMemoryMessageBus):ttl_secs: int | None = None,) -> None:"""Write before or after the acknowledgement barrier."""+ if namespace != MessageBusKeys.bg_tasks("session-a"):+ await super().registry_set(+ namespace,+ field,+ value,+ ttl_secs=ttl_secs,+ )+ returnif self.commit_before_wait:await super().registry_set(namespace,@@ -79,6 +87,7 @@ class BackgroundTaskManagerTest(IsolatedAsyncioTestCase):task.cancel()await asyncio.gather(*tasks, return_exceptions=True)await self._checkpoint()+ await self.manager.__aexit__()@staticmethodasync def _checkpoint() -> None:@@ -110,6 +119,7 @@ class BackgroundTaskManagerTest(IsolatedAsyncioTestCase):user_id="user-a",tool_name="slow_tool",id=task_id,+ started_at=self.manager.tasks[task_id].started_at,),},)@@ -137,7 +147,7 @@ class BackgroundTaskManagerTest(IsolatedAsyncioTestCase):cancelled: bool,complete_during_write: bool = False,) -> None:- """Verify faults preserve running tasks and clean up finished ones."""+ """Verify faults cancel running tasks and clean up all outcomes."""self.bus.commit_before_wait = committederror = ConnectionError("registry acknowledgement failed")self.bus.write_error = error@@ -157,12 +167,11 @@ class BackgroundTaskManagerTest(IsolatedAsyncioTestCase):with self.assertRaises(ConnectionError) as raised:await self.registrationself.assertIs(raised.exception, error)- self.assertEqual(self.task.done(), complete_during_write)- if not complete_during_write:- self.assertEqual(list(self.manager.tasks), [task_id])- self.assertEqual(self.bus.delete_calls, [])- self.finish.set()- self.assertTrue(await self.task)+ if complete_during_write:+ self.assertTrue(await self.task)+ else:+ with self.assertRaises(asyncio.CancelledError):+ await self.taskawait self._assert_cleaned(task_id)async def test_error_before_write(self) -> None:@@ -174,7 +183,7 @@ class BackgroundTaskManagerTest(IsolatedAsyncioTestCase):await self._assert_registration_fault(committed=True, cancelled=False)async def test_cancel_before_write(self) -> None:- """Cancelling registration preserves and eventually cleans the task."""+ """Cancelling registration cancels and cleans the running task."""await self._assert_registration_fault(committed=False, cancelled=True)async def test_cancel_after_write(self) -> None:diff --git a/tests/service_cancel_dispatcher_test.py b/tests/service_cancel_dispatcher_test.pyindex acca2745c..cd202180f 100644--- a/tests/service_cancel_dispatcher_test.py+++ b/tests/service_cancel_dispatcher_test.py@@ -278,11 +278,12 @@ class TestCancelDispatcher(IsolatedAsyncioTestCase):breakawait asyncio.sleep(0.01)+ # Other sessions keep running until manager shutdown.+ self.assertFalse(bg_task_b.cancelled())+self.assertTrue(bg_task_a1.cancelled() or bg_task_a1.done())self.assertTrue(bg_task_a2.cancelled() or bg_task_a2.done())- # sess-B BG task is left running until shutdown cancels it.- self.assertFalse(bg_task_b.cancelled())- bg_task_b.cancel()+ self.assertTrue(bg_task_b.cancelled())async def test_cancel_signal_closes_local_realtime_run(self) -> None:"""A session cancel reaches the local WebRTC runner."""diff --git a/tests/service_message_bus_test.py b/tests/service_message_bus_test.pyindex 5bdb04e80..0d82b5ed9 100644--- a/tests/service_message_bus_test.py+++ b/tests/service_message_bus_test.py@@ -172,6 +172,72 @@ class TestRegistryConditionalPrimitives(IsolatedAsyncioTestCase):)self.assertEqual(await self.bus.registry_get("ns", "f"), "v2")+ async def test_set_if_none_creates_only_missing_fields(self) -> None:+ """Missing and empty fields differ; concurrent creators cannot win."""+ for bus in (self.bus, InMemoryMessageBus()):+ with self.subTest(backend=type(bus).__name__):+ await bus.registry_set("ns", "empty", "")+ self.assertFalse(+ await bus.registry_set_if(+ "ns",+ "empty",+ "replacement",+ expected=None,+ ),+ )+ results = await asyncio.gather(+ bus.registry_set_if("ns", "new", "v", expected=None),+ bus.registry_set_if("ns", "new", "v", expected=None),+ )+ self.assertEqual(sorted(results), [False, True])+ self.assertTrue(+ await bus.registry_set_if(+ "ns",+ "empty",+ "updated",+ expected="",+ ),+ )+ self.assertFalse(+ await bus.registry_set_if(+ "ns",+ "missing",+ "v",+ expected="",+ ),+ )+ self.assertEqual(+ await bus.registry_getall("ns"),+ {"empty": "updated", "new": "v"},+ )++ async def test_set_if_none_refreshes_expiry_only_on_creation(self) -> None:+ """Recovery sets hash TTL without renewing a competing outcome."""+ self.assertTrue(+ await self.bus.registry_set_if(+ "ns",+ "f",+ "terminal",+ expected=None,+ ttl_secs=1000,+ ),+ )+ self.assertGreater(await self.fr.ttl("ns"), 0)+ self.assertFalse(+ await self.bus.registry_set_if(+ "ns",+ "f",+ "running",+ expected=None,+ ttl_secs=5000,+ ),+ )+ self.assertLessEqual(await self.fr.ttl("ns"), 1000)+ self.assertEqual(+ await self.bus.registry_getall("ns"),+ {"f": "terminal"},+ )+async def test_set_if_refreshes_the_expiry_only_when_it_writes(self,) -> None:@@ -208,6 +274,48 @@ class TestRegistryConditionalPrimitives(IsolatedAsyncioTestCase):"""An expired session reads as gone, not as a failure."""self.assertIsNone(await self.bus.registry_pop("missing", "f"))+ async def test_conditional_delete_preserves_newer_values(self) -> None:+ """Both backends delete exactly one unchanged field, atomically."""+ for bus in (self.bus, InMemoryMessageBus()):+ with self.subTest(backend=type(bus).__name__):+ await bus.registry_set("ns", "f", "new")+ await bus.registry_set("ns", "keep", "value")+ self.assertFalse(+ await bus.registry_del_if("ns", "f", expected="old"),+ )+ self.assertEqual(+ await bus.registry_getall("ns"),+ {"f": "new", "keep": "value"},+ )+ results = await asyncio.gather(+ bus.registry_del_if("ns", "f", expected="new"),+ bus.registry_del_if("ns", "f", expected="new"),+ )+ self.assertEqual(sorted(results), [False, True])+ self.assertEqual(+ await bus.registry_getall("ns"),+ {"keep": "value"},+ )+ self.assertFalse(+ await bus.registry_del_if("missing", "f", expected=""),+ )++ async def test_default_conditional_delete_leaves_data_intact(self) -> None:+ """Existing adapters inherit a safe no-op for conditional deletion."""+ await self.bus.registry_set("ns", "f", "value")+ self.assertFalse(+ await MessageBus.registry_del_if(+ self.bus,+ "ns",+ "f",+ expected="value",+ ),+ )+ self.assertEqual(+ await self.bus.registry_getall("ns"),+ {"f": "value"},+ )+class TestLogPrimitive(IsolatedAsyncioTestCase):"""Mode C — replay log: append / read with cursor / trim."""diff --git a/tests/tool_offload_middleware_test.py b/tests/tool_offload_middleware_test.pyindex 7bb6bfab6..86d245222 100644--- a/tests/tool_offload_middleware_test.py+++ b/tests/tool_offload_middleware_test.py@@ -3,7 +3,7 @@"""Unit tests for ToolOffloadMiddleware."""import asyncioimport json-from typing import Any+from typing import Any, AsyncGeneratorfrom unittest.async_case import IsolatedAsyncioTestCasefrom unittest.mock import AsyncMock, MagicMock@@ -13,10 +13,17 @@ from pydantic import BaseModelfrom utils import AnyString, MockModelfrom agentscope.agent import Agent-from agentscope.app.message_bus import MessageBus, MessageBusKeys+from agentscope.app.message_bus import (+ InMemoryMessageBus,+ MessageBus,+ MessageBusKeys,+)from agentscope.app.middleware import ToolOffloadMiddleware-from agentscope.app._manager import BackgroundTaskManager-from agentscope.message import TextBlock, ToolCallBlock+from agentscope.app._manager import BackgroundTaskManager, TaskStatus+from agentscope.app._manager._background_task_manager import (+ BackgroundTaskRecord,+)+from agentscope.message import TextBlock, ToolCallBlock, ToolResultStatefrom agentscope.permission import (PermissionContext,PermissionDecision,@@ -152,8 +159,8 @@ class ToolOffloadMiddlewareTest(IsolatedAsyncioTestCase):async def asyncSetUp(self) -> None:"""Set up test fixtures."""self.mock_model = MockModel()- self.bg_manager = BackgroundTaskManager(- message_bus=MagicMock(spec=MessageBus),+ self.bg_manager = await self.enterAsyncContext(+ BackgroundTaskManager(message_bus=InMemoryMessageBus()),)# ------------------------------------------------------------------@@ -374,6 +381,158 @@ class ToolOffloadMiddlewareTest(IsolatedAsyncioTestCase):self.assertEqual(payload["session_id"], agent.state.session_id)self.assertEqual(payload["agent_id"], "a")+ async def test_background_failures_retain_error_status(self) -> None:+ """Error responses and raised exceptions both retain failed state."""+ for raises in (False, True):+ with self.subTest(raises=raises):+ agent, middleware = self._make_agent(+ Toolkit(tools=[SlowTool()]),+ timeout_secs=0.01,+ )+ gate = asyncio.Event()++ async def handler(+ *,+ event: asyncio.Event = gate,+ should_raise: bool = raises,+ **_kwargs: Any,+ ) -> AsyncGenerator:+ """Fail only after the tool has been offloaded."""+ await event.wait()+ if should_raise:+ raise ValueError("tool execution failed")+ yield ToolResponse(+ id="call_failure",+ state=ToolResultState.ERROR,+ content=[TextBlock(text="tool execution failed")],+ )++ call = ToolCallBlock(+ id="call_failure",+ name="slow_tool",+ input="{}",+ )+ async for _ in middleware.on_acting(+ agent,+ {"tool_call": call},+ handler,+ ):+ pass+ bg = next(iter(self.bg_manager.tasks.values()))+ expected = BackgroundTaskRecord.model_validate(bg).model_dump()+ gate.set()+ await bg.asyncio_task+ await asyncio.gather(*self.bg_manager._completion_tasks)+ record = await self.bg_manager.get_task(bg.id, user_id="u")+ self.assertIsNotNone(record.completed_at)+ expected.update(+ status=TaskStatus.FAILED,+ error_summary="tool execution failed",+ completed_at=record.completed_at,+ )+ self.assertEqual(record.model_dump(), expected)+ self.assertEqual(self.bg_manager.tasks, {})++ async def test_interrupted_response_matches_record_and_inbox(self) -> None:+ """Interrupted results keep their output and report cancellation."""+ for output in ("", "partial output"):+ with self.subTest(output=output):+ agent, middleware = self._make_agent(+ Toolkit(tools=[SlowTool()]),+ timeout_secs=0.01,+ )+ gate = asyncio.Event()+ delivered = asyncio.Event()+ mock_bus = middleware._message_bus+ mock_bus.publish.side_effect = (+ lambda *_args, event=delivered, **_kwargs: event.set()+ )+ response = ToolResponse(+ id="call_interrupted",+ state=ToolResultState.INTERRUPTED,+ content=[TextBlock(text=output)] if output else [],+ )+ original = response.model_dump(mode="json")++ async def handler(+ *,+ event: asyncio.Event = gate,+ result: ToolResponse = response,+ **_kwargs: Any,+ ) -> AsyncGenerator:+ await event.wait()+ yield result++ call = ToolCallBlock(+ id="call_interrupted",+ name="slow_tool",+ input="{}",+ )+ async for _ in middleware.on_acting(+ agent,+ {"tool_call": call},+ handler,+ ):+ pass+ bg = next(iter(self.bg_manager.tasks.values()))+ expected = BackgroundTaskRecord.model_validate(bg).model_dump()+ gate.set()+ await bg.asyncio_task+ await asyncio.gather(*self.bg_manager._completion_tasks)+ await asyncio.wait_for(delivered.wait(), timeout=1)+ record = await self.bg_manager.get_task(bg.id, user_id="u")+ expected.update(+ status=TaskStatus.CANCELLED,+ completed_at=record.completed_at,+ )+ self.assertEqual(record.model_dump(), expected)++ prefix = (+ "<system-notification>Tool 'slow_tool' running in "+ "background (id=call_interrupted) has been cancelled"+ )+ expected_hint: str | list[dict]+ if output:+ expected_hint = [+ dict(+ original["content"][0],+ text=f"{prefix}.\n\nResult:\n\n{output}"+ "</system-notification>",+ ),+ ]+ else:+ expected_hint = (+ f"{prefix} with no output.</system-notification>"+ )+ inbox = [+ call.args[1]+ for call in mock_bus.queue_push.call_args_list+ if call.args[0]+ == MessageBusKeys.inbox(+ agent.state.session_id,+ )+ ]+ self.assertEqual(+ inbox,+ [+ {+ "type": "hint",+ "id": AnyString(),+ "created_at": AnyString(),+ "finished_at": AnyString(),+ "source": json.dumps(+ {+ "label": "tool_output",+ "sublabel": "slow_tool · call_interrupted",+ },+ ensure_ascii=False,+ ),+ "hint": expected_hint,+ },+ ],+ )+ self.assertEqual(response.model_dump(mode="json"), original)+async def test_tool_stop_cancels_background_task(self) -> None:"""ToolStop tool cancels the running background asyncio task."""@@ -408,5 +567,9 @@ class ToolOffloadMiddlewareTest(IsolatedAsyncioTestCase):# The asyncio task should be cancellingself.assertTrue(asyncio_task.cancelled() or asyncio_task.cancelling())- # Removed from manager+ await asyncio.gather(asyncio_task, return_exceptions=True)+ await asyncio.gather(*self.bg_manager._completion_tasks)+ # The handle is released, but cancelled metadata remains queryable.self.assertEqual(len(self.bg_manager.tasks), 0)+ record = await self.bg_manager.get_task(task_id, user_id="u")+ self.assertEqual(record.status, TaskStatus.CANCELLED)