Enterprise agent platform for building, deploying, and operating AI teammates.
feat(v1): cancel a running task with POST /v1/chat/tasks/{task_id}/cancel (#3777)
Alex Liubbd7d1a23612
diff --git a/docs/deployment.md b/docs/deployment.mdindex 88679939b..045d6b3da 100644--- a/docs/deployment.md+++ b/docs/deployment.md@@ -991,3 +991,23 @@ With automatic resume on, and `XAGENT_TASK_INFRA_FAILURE_PAUSE_ENABLED` not set### Rollback- Nothing stored needs to be undone. Reads show `paused` again. SDK tasks paused by a settlement interruption keep being resumed automatically and rest `paused` when automatic resume stops; deploying again makes them read `failed`.++## 2026-11-13 — SDK tasks: `POST /v1/chat/tasks/{task_id}/cancel`, and `error_code` on task reads++A running task created through `/v1` can now be cancelled. The cancelled task reads `failed` with a new `error_code` of `task_cancelled`, and accepts a new message like any failed task.++### Client-visible changes++- `POST /v1/chat/tasks/{task_id}/cancel` answers 200 once the task is cancelled, including when an earlier call already cancelled it, and 202 when the cancel is recorded but not applied yet. If a new turn has started by the time 200 is returned, `status` and `run_id` describe the new turn and `cancelled_run_id` names the cancelled one.+- 409 `task_not_cancellable` means the run ended without being cancelled, including one that ended on its own, or was replaced by another turn, while the cancel was being applied. 409 `task_busy` means retry: the task is waiting for an answer, paused, has a turn queued to start, or changed state while the cancel was applied. 503 means the cancel could not be applied; retry.+- `GET /v1/chat/tasks/{task_id}` and the `task.completed` frame of `GET /v1/chat/tasks/{task_id}/events` carry `error_code`, `task_cancelled` for a cancelled task and `null` otherwise.++### Deployment impact++- No migration and no configuration.+- Mixed versions: a run owned by a process that predates this change rejects the cancel command as an unsupported scope, so the endpoint answers 503 until every process that executes tasks has been upgraded; calling again then succeeds.+- The Python SDK `tasks.cancel()` ships once waiting and paused tasks can be cancelled as well; until then only direct HTTP callers reach this endpoint.++### Rollback++- A task cancelled before the rollback stays `failed`; an older process does not report `error_code`, so that field is absent from its reads. The cancel endpoint answers 404 again.diff --git a/src/xagent/web/api/v1/_events_stream.py b/src/xagent/web/api/v1/_events_stream.pyindex 04da969d8..5096ef403 100644--- a/src/xagent/web/api/v1/_events_stream.py+++ b/src/xagent/web/api/v1/_events_stream.py@@ -413,6 +413,7 @@ def completed_frame(status: str,output: str | None,error: str | None,+ error_code: str | None,structured_output: dict[str, Any] | None = None,structured_output_status: str | None = None,structured_output_reason: dict[str, Any] | None = None,@@ -424,6 +425,7 @@ def completed_frame("status": status,"output": output,"error": error,+ "error_code": error_code,"structured_output": structured_output,"structured_output_status": structured_output_status,"structured_output_reason": structured_output_reason,@@ -1502,6 +1504,7 @@ async def watchdog_check_once(status=status.value,output=snapshot.output,error=snapshot.error,+ error_code=snapshot.error_code,**_structured_output_fields(snapshot),))@@ -2144,6 +2147,7 @@ def _terminal_snapshot_stream(status=snapshot.status.value,output=snapshot.output,error=snapshot.error,+ error_code=snapshot.error_code,**_structured_output_fields(snapshot),snapshot_total_steps=snapshot_total_steps,)diff --git a/src/xagent/web/api/v1/errors.py b/src/xagent/web/api/v1/errors.pyindex 807743f42..24d20be1f 100644--- a/src/xagent/web/api/v1/errors.py+++ b/src/xagent/web/api/v1/errors.py@@ -216,6 +216,14 @@ class V1ErrorCode(str, Enum):# current run (pending / paused / completed normally / failed). 409.TASK_NOT_WAITING = "task_not_waiting"+ # ``POST /v1/chat/tasks/{id}/cancel`` was called on a task that already+ # ended without being cancelled (completed, or failed on its own), or+ # whose run the cancel was aimed at ended while the cancel was being+ # applied and another turn took its place. 409,+ # not retryable -- a task that can be cancelled later answers+ # ``task_busy`` instead.+ TASK_NOT_CANCELLABLE = "task_not_cancellable"+# A reply's saved progress could not be read due to a transient# infrastructure failure. 503, retryable -- distinct from# ``interaction_not_resumable`` (which means resuming can never@@ -321,6 +329,9 @@ _DEFAULT_MESSAGES: dict[V1ErrorCode, str] = {V1ErrorCode.TASK_NOT_WAITING: ("This task is not waiting for an answer to a question."),+ V1ErrorCode.TASK_NOT_CANCELLABLE: (+ "This task has already ended and cannot be cancelled."+ ),V1ErrorCode.REPLY_OUTCOME_UNKNOWN: ("Reply was accepted but its outcome is not yet known. ""Repeat the same command_id to check its outcome; do not send a new reply."diff --git a/src/xagent/web/api/v1/tasks.py b/src/xagent/web/api/v1/tasks.pyindex 2a30efd5d..d2eb7af85 100644--- a/src/xagent/web/api/v1/tasks.py+++ b/src/xagent/web/api/v1/tasks.py@@ -45,6 +45,7 @@ from ...schemas.v1 import (AppendMessageRequest,AppendMessageResponse,CancelQuestionResponse,+ CancelTaskResponse,CreateTaskRequest,CreateTaskResponse,DeleteTaskResponse,@@ -82,6 +83,7 @@ from ...services.response_format import (normalize_response_format,violation_message,)+from ...services.sdk_task_cancel import sdk_cancel_error_codefrom ...services.structured_output_projection import (project_structured_output,response_format_configured,@@ -590,6 +592,10 @@ class _TaskInfoSnapshot:# ``project_structured_output`` applied to the effective status.structured_output_record: Any = Noneformat_configured: bool = False+ # ``task_cancelled`` when the stored row is the outcome of a cancel+ # request, ``None`` for every other row. Cleared like ``error`` wherever+ # the status shown is not the stored one.+ error_code: str | None = None# Terminal statuses an append is accepted from. With shared execution the row@@ -618,6 +624,7 @@ def effective_task_snapshot(snapshot: _TaskInfoSnapshot) -> _TaskInfoSnapshot:status=TaskStatus.PENDING,output=None,error=None,+ error_code=None,)view = snapshot.viewif view is None:@@ -628,6 +635,7 @@ def effective_task_snapshot(snapshot: _TaskInfoSnapshot) -> _TaskInfoSnapshot:control_state=TaskStatus.FAILED.value,output=None,error=view.error,+ error_code=None,)@@ -864,6 +872,9 @@ def _load_task_info_snapshot(start_queued=start_queued,structured_output_record=task.structured_output_record,format_configured=response_format_configured(task.agent_config),+ error_code=sdk_cancel_error_code(+ task.status, cast(str | None, task.error_message)+ ),)@@ -1030,6 +1041,7 @@ def _get_chat_task_sync(status=effective.status.value,output=effective.output,error=effective.error,+ error_code=effective.error_code,completed_at=None,)if task.view is not None:@@ -1039,6 +1051,7 @@ def _get_chat_task_sync(control_state=effective.control_state,output=effective.output,error=effective.error,+ error_code=effective.error_code,completed_at=task.view.stopped_at,)# Applied on every read, never cached: the record can change without the@@ -1072,6 +1085,7 @@ def _get_chat_task_sync(input=task.input,output=task.output,error=task.error,+ error_code=task.error_code,created_at=task.created_at,completed_at=completed_at,)@@ -1478,6 +1492,96 @@ async def cancel_waiting_task_question()+ "/chat/tasks/{task_id}/cancel",+ response_model=CancelTaskResponse,+ responses={+ 202: {+ "model": CancelTaskResponse,+ "description": "The cancel is recorded but not applied yet.",+ }+ },+)+async def cancel_chat_task(+ task_id: int,+ response: Response,+ principal: ApiKeyPrincipal = Depends(get_principal_from_api_key),+) -> CancelTaskResponse:+ """Stop a task that is still running.++ Reuses ``GET``'s ownership rule (another agent's task, another user's+ agent, a missing id, or a non-SDK task all answer 404) before anything+ about the task's state is read. The cancelled task reads ``failed``+ with ``error_code='task_cancelled'``, and accepts+ ``POST /v1/chat/tasks/{id}/messages`` like any failed task. A workforce+ run is cancelled as one task.++ The call polls up to about 10 seconds for the cancel to be applied;+ when this process applies the cancel itself it also waits for the run+ to stop. 200 means the task is cancelled, including when an earlier+ call already cancelled it; if a new turn has started since, ``status``+ and ``run_id`` describe that turn and ``cancelled_run_id`` names the+ cancelled one. 202 means the cancel is recorded but not applied yet,+ and calling again reaches the same request.++ Only a running task is cancelled. One that is waiting for an answer,+ paused, or whose turn is still queued to start answers 409+ ``task_busy`` and is left untouched.++ Raises:+ V1ApiError 401: missing / invalid / revoked key.+ V1ApiError 404: task missing, not owned by the key, or not SDK-created.+ V1ApiError 409: ``task_not_cancellable`` for a task that already+ ended without being cancelled, including one whose run ended+ on its own, or was replaced by another turn, while the cancel+ was being applied; ``task_busy`` when the task cannot be+ cancelled in its current state or changed state while the+ cancel was applied -- retry.+ V1ApiError 410: ``task_expired`` for a retention-expired task.+ V1ApiError 503: the cancel could not be applied; retry.+ """+ from ...services.sdk_task_cancel import (+ SdkTaskCancelBusy,+ SdkTaskCancelFailed,+ SdkTaskNotCancellable,+ request_sdk_task_cancel,+ )++ try:+ result = await request_sdk_task_cancel(+ task_id=task_id,+ scope=_sdk_task_scope(principal),+ )+ except task_start_service.TaskStartRejected as exc:+ _raise_start_rejection(exc)+ except SdkTaskNotCancellable as exc:+ raise V1ApiError(+ V1ErrorCode.TASK_NOT_CANCELLABLE, 409, message=str(exc) or None+ ) from exc+ except SdkTaskCancelBusy as exc:+ raise V1ApiError(V1ErrorCode.TASK_BUSY, 409, message=str(exc)) from exc+ except SdkTaskCancelFailed as exc:+ raise V1ApiError(+ V1ErrorCode.INTERNAL_ERROR,+ 503,+ message="The cancel could not be applied; retry.",+ ) from exc++ await record_principal_usage(+ principal, route="cancel_task", task_id=result.task_id, agent_id=None+ )+ if not result.cancelled:+ response.status_code = 202+ return CancelTaskResponse(+ task_id=result.task_id,+ status=result.status,+ run_id=result.run_id,+ state_version=result.state_version,+ cancelled_run_id=result.cancelled_run_id,+ error_code=result.error_code,+ )++@router.get("/chat/tasks/{task_id}", response_model=TaskInfoResponse)async def get_chat_task(task_id: int,diff --git a/src/xagent/web/schemas/v1.py b/src/xagent/web/schemas/v1.pyindex 7a4d74f40..ac81d2d2d 100644--- a/src/xagent/web/schemas/v1.py+++ b/src/xagent/web/schemas/v1.py@@ -1109,6 +1109,48 @@ class CancelQuestionResponse(BaseModel):)+class CancelTaskResponse(BaseModel):+ """``POST /v1/chat/tasks/{task_id}/cancel`` -> 200 / 202 response.++ 200: the run named by ``cancelled_run_id`` is cancelled, whether this+ call did it or an earlier one already had. The other fields describe+ the task now: it reads ``failed`` with ``error_code='task_cancelled'``+ unless a new turn has started since, in which case ``status``,+ ``run_id``, ``state_version`` and ``error_code`` describe that turn.+ 202: the cancel was recorded but not applied yet; ``status`` is still+ the task's current one and ``cancelled_run_id`` is null. Calling again+ reaches the same request.+ """++ task_id: int = Field(..., description="Existing task primary key.")+ status: str = Field(+ ...,+ description=(+ "Task status as stored: 'failed' once cancelled, the status of a "+ "turn that started after the cancel, or, on 202, the status the "+ "cancel is still being applied to."+ ),+ )+ run_id: Optional[str] = Field(..., description="Current execution run identity.")+ state_version: int = Field(+ ..., description="Monotonic version of the task control state."+ )+ cancelled_run_id: Optional[str] = Field(+ ...,+ description=(+ "The run that was cancelled. Null while the cancel has not been applied."+ ),+ )+ error_code: Optional[str] = Field(+ ...,+ description=(+ "'task_cancelled' while the task reads as cancelled, otherwise "+ "null (before the cancel is applied, or once a later turn has "+ "started); the same value GET /v1/chat/tasks/{task_id} reports."+ ),+ )++class CreateWorkforceRunRequest(BaseModel):"""Body for ``POST /v1/workforces/{workforce_id}/runs``.@@ -1281,6 +1323,16 @@ class TaskInfoResponse(BaseModel):"a fixed text saying why automatic resume stopped."),)+ error_code: Optional[str] = Field(+ None,+ description=(+ "Machine-readable reason for status='failed', when one is "+ "defined: 'task_cancelled' for a task stopped through "+ "POST /v1/chat/tasks/{task_id}/cancel. Null for every other "+ "failure, for a derived failed, and whenever status is not "+ "'failed'."+ ),+ )created_at: datetimecompleted_at: Optional[datetime] = Field(None,diff --git a/src/xagent/web/services/sdk_task_cancel.py b/src/xagent/web/services/sdk_task_cancel.pynew file mode 100644index 000000000..3d4ddf779--- /dev/null+++ b/src/xagent/web/services/sdk_task_cancel.py@@ -0,0 +1,757 @@+"""Cancel one SDK-source task through the durable command channel.++``POST /v1/chat/tasks/{task_id}/cancel`` is split in two halves that live+here. The request half authorizes the caller, refuses a target that cannot+be cancelled, persists a CANCEL command and polls a bounded time for it.+The execution half runs wherever that command is applied -- the process+that owns the task -- and is the SDK counterpart of the A2A and external+cores, which load their targets by ``Task.source`` and so cannot reach an+SDK task.++One judgement decides whether a task was cancelled:+``is_sdk_cancelled`` -- FAILED carrying a sentence only a cancellation+writes. The cancelled run's own settlement writes that sentence when the+run was stopped by an explicit ``cancel_task`` call+(``sdk_cancel_settlement_text``); this core writes the same row when no+settlement did. Whichever writer lands first owns the terminal row, and+the other one changes nothing.++Only a RUNNING task is cancelled here. A task with a turn still queued to+start, one waiting for the caller's answer and a paused one are refused as+busy, without a write.+"""++from __future__ import annotations++import asyncio+import logging+from dataclasses import dataclass, replace+from time import monotonic+from typing import TYPE_CHECKING, Any++from sqlalchemy import func, update+from sqlalchemy.orm import Session++from ..models.chat_message import TaskChatMessage+from ..models.database import get_session_local+from ..models.task import Task, TaskStatus+from .db_runtime import run_db_io_cancellation_safe+from .task_command_transport import (+ COMMAND_COMPLETED,+ COMMAND_FAILED,+ EnqueuedTaskCommand,+ TaskCommandKind,+ TaskCommandRejected,+ TaskCommandTaskMissing,+ dispatch_one_task_command,+ enqueue_task_command,+ load_task_command,+ retry_failed_task_command,+)+from .task_execution_controller import (+ StaleTaskRunError,+ TaskControlState,+ task_control_snapshot,+)++if TYPE_CHECKING:+ # ``task_start`` imports the orchestrator, which imports this module.+ from .task_start import SdkTaskScope++logger = logging.getLogger(__name__)++SDK_TASK_SOURCE = "sdk"++# Command payload scope that routes a CANCEL command to this core. Only a+# server-side producer writes it; client frames naming a scope are refused.+SDK_COMMAND_SCOPE = "sdk"++# The durable text of a cancelled SDK task, read back by the caller as the+# task's ``error``. It is also the cancellation judgement, so rewording it+# means adding the new sentence to ``_SDK_CANCEL_MESSAGES`` and keeping+# this one there: rows and processes carrying the old text must still be+# recognised.+SDK_CANCEL_ERROR_MESSAGE = "Task was cancelled by the caller."++_SDK_CANCEL_MESSAGES = frozenset({SDK_CANCEL_ERROR_MESSAGE})++# Machine-readable companion of the text on the public read surface.+SDK_CANCEL_ERROR_CODE = "task_cancelled"++# Passed to ``BackgroundTaskManager.cancel_task``. Nominal, not a bound:+# that call returns once the cancelled coroutine has unwound, and its+# cleanup drains database work that cannot be interrupted.+SDK_CANCEL_WAIT_SECONDS = 5.0++# How long the request waits for its command before answering "accepted".+SDK_CANCEL_REQUEST_WAIT_SECONDS = 10.0++_COMMAND_POLL_SECONDS = 0.05++# Read by a caller whose cancel was aimed at a run that is no longer the+# task's current one.+_AIMED_RUN_ENDED_MESSAGE = "The run this cancel was aimed at has already ended."++_TERMINAL_STATUSES = frozenset({TaskStatus.COMPLETED, TaskStatus.FAILED})+++def is_sdk_cancelled(status: object, error_message: object) -> bool:+ """Whether a task row is the outcome of an SDK cancellation.++ The row's ``error_message`` always describes its current run: a new run+ clears it when it starts and every other terminal write replaces it, so+ the sentence cannot outlive the run it was written for.+ """++ return status == TaskStatus.FAILED and error_message in _SDK_CANCEL_MESSAGES+++def sdk_cancel_error_code(status: object, error_message: object) -> str | None:+ """The public ``error_code`` of a task row: set only for a cancellation."""++ return SDK_CANCEL_ERROR_CODE if is_sdk_cancelled(status, error_message) else None+++def sdk_cancel_settlement_text(+ task_source: str | None, *, explicit_cancel: bool+) -> str | None:+ """Settlement text for an SDK run stopped by an explicit ``cancel_task``.++ ``None`` for every other source and for a cancellation nobody asked for+ (a host shutting down), which keep the wording they have always had.+ """++ if task_source == SDK_TASK_SOURCE and explicit_cancel:+ return SDK_CANCEL_ERROR_MESSAGE+ return None+++def sdk_cancel_result_fact(error_message: str | None) -> dict[str, Any]:+ """Result fact for a failed run, naming a cancellation as one.++ The next turn's context reports ``cancelled`` as such instead of as a+ failure.+ """++ if error_message in _SDK_CANCEL_MESSAGES:+ return {"status": "cancelled", "error": error_message}+ return {"error": error_message}+++class SdkTaskCancelBusy(Exception):+ """Nothing was cancelled, and retrying can succeed."""+++class SdkTaskNotCancellable(Exception):+ """The run already ended on its own; no retry can cancel it.++ The message, when set, is what the caller reads.+ """+++class SdkTaskCancelFailed(Exception):+ """The command could not be applied; the task state is unchanged by it."""+++@dataclass(frozen=True)+class SdkTaskCancelResult:+ task_id: int+ status: str+ run_id: str | None+ state_version: int+ cancelled_run_id: str | None+ error_code: str | None+ # False while the command is still being applied.+ cancelled: bool+++@dataclass(frozen=True)+class _CancelCommand:+ command_db_id: int+ # The run the command was persisted against: the one this request+ # answers for, whatever run the row shows by the time it is read back.+ aimed_run_id: str | None+++def _task_run_id(task: Task) -> str | None:+ run_id = getattr(task, "run_id", None)+ return str(run_id) if run_id is not None else None+++def _result(task: Task) -> SdkTaskCancelResult:+ cancelled = is_sdk_cancelled(task.status, task.error_message)+ run_id = _task_run_id(task)+ return SdkTaskCancelResult(+ task_id=int(task.id),+ status=task.status.value,+ run_id=run_id,+ state_version=int(task.state_version or 0),+ cancelled_run_id=run_id if cancelled else None,+ error_code=sdk_cancel_error_code(task.status, task.error_message),+ cancelled=cancelled,+ )+++def _ended_without_cancel(db: Session, task: Task) -> bool:+ """Whether the task's run ended and was not an SDK cancellation.++ Covers a COMPLETED or FAILED row and a paused row the read surface+ reports as failed because automatic resume gave up.+ """++ from .task_auto_resume import status_only_terminal_view++ if is_sdk_cancelled(task.status, task.error_message):+ return False+ if task.status in _TERMINAL_STATUSES:+ return True+ if (+ task.status == TaskStatus.PAUSED+ and task.control_state == TaskControlState.PAUSED.value+ ):+ return status_only_terminal_view(db, int(task.id)) is not None+ return False+++def _prepare_cancel_sync(+ *, task_id: int, scope: SdkTaskScope+) -> SdkTaskCancelResult | _CancelCommand:+ """Authorize, classify the target and persist its command, in one read."""++ from .task_start import TaskStartRejected, resolve_sdk_task+ from .task_start_protocol import has_queued_start++ SessionLocal = get_session_local()+ with SessionLocal() as db:+ # Read in the order the public task read uses: until a queued START+ # is handed off, the row still shows the previous turn, so judging+ # the row alone would answer for a turn the caller is not asking+ # about.+ queued = has_queued_start(db, task_id)+ task = resolve_sdk_task(task_id, scope, db)+ if queued:+ raise SdkTaskCancelBusy(+ "A turn of this task is queued to start and cannot be "+ "cancelled yet; retry once it is running."+ )+ if is_sdk_cancelled(task.status, task.error_message):+ return _result(task)+ if _ended_without_cancel(db, task):+ raise SdkTaskNotCancellable+ if task.status != TaskStatus.RUNNING:+ raise SdkTaskCancelBusy(+ f"A task that is {task.status.value} cannot be cancelled yet."+ )+ target_state_version = int(task.state_version or 0)+ # Read before the insert commits and expires ``task``: this is the+ # run the command targets.+ aimed_run_id = _task_run_id(task)+ try:+ enqueued = _enqueue_cancel(db, task, target_state_version)+ except TaskCommandTaskMissing as exc:+ # Deleted between the read above and the insert.+ raise TaskStartRejected("task_not_found") from exc+ if not enqueued.payload_matches:+ raise SdkTaskCancelBusy(+ "Another command holds this cancel identity; retry."+ )+ if enqueued.status == COMMAND_FAILED:+ retry_failed_task_command(db, enqueued.command_id)+ return _CancelCommand(+ command_db_id=enqueued.command_id, aimed_run_id=aimed_run_id+ )+++def _enqueue_cancel(+ db: Session, task: Task, target_state_version: int+) -> EnqueuedTaskCommand:+ task_id = int(task.id)+ return enqueue_task_command(+ db,+ task_id=task_id,+ actor_user_id=int(task.user_id) if task.user_id is not None else None,+ # Keyed on the state read here: a repeat against the same state+ # is the same command, and a repeat after the state moved is a+ # new one aimed at the new state.+ command_id=f"cancel:{task_id}:{target_state_version}",+ kind=TaskCommandKind.CANCEL,+ payload={+ "agent_id": int(task.agent_id),+ "target_state_version": target_state_version,+ "scope": SDK_COMMAND_SCOPE,+ },+ )+++def _load_result_sync(+ *,+ task_id: int,+ scope: SdkTaskScope,+ aimed_run_id: str | None,+ command_completed: bool,+) -> SdkTaskCancelResult:+ """Read the task as it stands once the command is over or overdue.++ ``aimed_run_id`` is the run the command was persisted against. A+ completed command cancelled that run, so the answer names it as+ ``cancelled_run_id`` while every other field describes the row now,+ which may already show a later run. Otherwise the answer is about the+ aimed run only. A row that shows another run means the aimed one is+ over, and retrying would aim at the new one, so it raises+ ``SdkTaskNotCancellable`` with ``_AIMED_RUN_ENDED_MESSAGE``. An aimed+ run that ended on its own raises ``SdkTaskNotCancellable`` exactly as+ the request would have on arrival, whether or not a later turn is+ queued behind it.+ """++ from .task_start import resolve_sdk_task++ SessionLocal = get_session_local()+ with SessionLocal() as db:+ task = resolve_sdk_task(task_id, scope, db)+ if command_completed:+ return replace(_result(task), cancelled=True, cancelled_run_id=aimed_run_id)+ if _task_run_id(task) != aimed_run_id:+ raise SdkTaskNotCancellable(_AIMED_RUN_ENDED_MESSAGE)+ if _ended_without_cancel(db, task):+ raise SdkTaskNotCancellable+ return _result(task)+++async def request_sdk_task_cancel(+ *, task_id: int, scope: SdkTaskScope+) -> SdkTaskCancelResult:+ """Authorize and cancel one SDK task, polling a bounded time for it.++ The poll is bounded; a cancel applied in this process first waits for+ the cancelled run to stop, which has no upper bound.++ ``scope`` is the calling key's ``SdkTaskScope``. A task outside it, a+ missing one and a non-SDK one raise the same ``TaskStartRejected`` the+ other task endpoints answer 404 with, before anything is read about the+ task's state.++ Returns ``cancelled=True`` once the task is the outcome of a+ cancellation, including when it already was, and once the command that+ cancelled the aimed run completed (``cancelled_run_id``). Returns+ ``cancelled=False`` when the command was persisted but not applied+ within ``SDK_CANCEL_REQUEST_WAIT_SECONDS``; calling again reaches the+ same command. Raises ``SdkTaskNotCancellable`` when the aimed run ended+ without being cancelled, whichever way the wait ends.+ """++ prepared = await run_db_io_cancellation_safe(+ lambda: _prepare_cancel_sync(task_id=task_id, scope=scope)+ )+ if isinstance(prepared, SdkTaskCancelResult):+ return prepared++ from .task_command_execution import execute_durable_task_command++ aimed_run_id = prepared.aimed_run_id+ deadline = monotonic() + SDK_CANCEL_REQUEST_WAIT_SECONDS+ # Applied here when this process owns the run; otherwise the owner's+ # dispatcher picks the durable row up and the poll below observes it.+ await dispatch_one_task_command(+ execute_durable_task_command,+ command_db_id=prepared.command_db_id,+ )+ while True:+ stored = await run_db_io_cancellation_safe(+ lambda: load_task_command(prepared.command_db_id)+ )+ if stored is not None and stored.status in {COMMAND_COMPLETED, COMMAND_FAILED}:+ command_completed = stored.status == COMMAND_COMPLETED+ result = await run_db_io_cancellation_safe(+ lambda: _load_result_sync(+ task_id=task_id,+ scope=scope,+ aimed_run_id=aimed_run_id,+ command_completed=command_completed,+ )+ )+ if command_completed:+ return result+ # The row decides, not the command: a command that lost to+ # another cancel of the same run still finds the task cancelled.+ if result.cancelled:+ return result+ if stored.rejection_reason == "stale_run":+ raise SdkTaskCancelBusy(+ "The task changed state while the cancel was applied; retry."+ )+ raise SdkTaskCancelFailed(stored.error or "cancel command failed")+ if monotonic() >= deadline:+ return await run_db_io_cancellation_safe(+ lambda: _load_result_sync(+ task_id=task_id,+ scope=scope,+ aimed_run_id=aimed_run_id,+ command_completed=False,+ )+ )+ await asyncio.sleep(_COMMAND_POLL_SECONDS)+++def _load_sdk_task(db: Session, *, task_id: int, agent_id: int) -> Task:+ task = (+ db.query(Task)+ .filter(+ Task.id == task_id,+ Task.agent_id == agent_id,+ Task.source == SDK_TASK_SOURCE,+ )+ .populate_existing()+ .first()+ )+ if task is None:+ raise TaskCommandRejected(+ f"task {task_id} is not an SDK task of agent {agent_id}",+ reason="task_not_found",+ )+ return task+++def _is_settled_sdk_cancel_target(+ task: Task,+ *,+ expected_run_id: str | None,+ expected_state_version: int,+) -> bool:+ """Whether this exact target already reached its cancel outcome.++ True for the command's own finalize replayed, and for the settlement of+ the run this command cancelled: both leave the target run FAILED at the+ command's state version or one past it, carrying the cancellation+ sentence. A run that failed on its own carries another text and is not+ this command's to report.+ """++ return (+ is_sdk_cancelled(task.status, task.error_message)+ and _task_run_id(task) == expected_run_id+ and int(task.state_version or 0)+ in {expected_state_version, expected_state_version + 1}+ )+++def _is_exact_cancel_target(+ task: Task, *, expected_run_id: str | None, expected_state_version: int+) -> bool:+ """Whether the row is still the running target this command was aimed at."""++ return (+ _task_run_id(task) == expected_run_id+ and int(task.state_version or 0) == expected_state_version+ and task.status == TaskStatus.RUNNING+ )+++def _stale(task: Task, expected_run_id: str | None, expected_state_version: int) -> str:+ return (+ f"task {task.id} changed from run/version "+ f"{expected_run_id}/{expected_state_version} to "+ f"{_task_run_id(task)}/{int(task.state_version or 0)}/{task.status.value}"+ )+++def _load_cancelable_sdk_task_sync(+ *,+ task_id: int,+ agent_id: int,+ expected_run_id: str | None,+ expected_state_version: int,+ terminal_event_state: dict[str, Any],+) -> bool:+ """Return whether the exact cancel target is already settled.++ A settled target fills ``terminal_event_state`` with the control+ snapshot its terminal broadcast carries.+ """++ SessionLocal = get_session_local()+ with SessionLocal() as db:+ task = _load_sdk_task(db, task_id=task_id, agent_id=agent_id)+ if _is_settled_sdk_cancel_target(+ task,+ expected_run_id=expected_run_id,+ expected_state_version=expected_state_version,+ ):+ if task.conversation_storage_version == 2:+ terminal_event_state.update(task_control_snapshot(task).as_dict())+ return True+ if not _is_exact_cancel_target(+ task,+ expected_run_id=expected_run_id,+ expected_state_version=expected_state_version,+ ):+ raise TaskCommandRejected(+ _stale(task, expected_run_id, expected_state_version),+ reason="stale_run",+ )+ return False+++def _close_cancelled_turn_no_commit(db: Session, task: Task) -> str | None:+ """Stage what a failed run's settlement leaves beside the terminal row.++ The assistant line mirrors the settlement's own -- the generic failure+ text under the task-failure provenance -- so a reader of the transcript+ cannot tell which writer ended the run. The delivery row of the turn is+ closed as ``dispatched``: its message may have been consumed, so it must+ not be offered for a resend.++ The running turn is the newest pending user row, which is sound only+ because a RUNNING task accepts no new turn. Returns that turn's id.+ """++ from .assistant_history_safety import TASK_FAILURE_MESSAGE_TYPE+ from .chat_history_service import (+ DELIVERY_DISPATCHED,+ DELIVERY_PENDING,+ mark_user_message_delivery,+ persist_assistant_message_no_commit,+ )+ from .client_error_messages import CLIENT_SAFE_TASK_FAILURE++ task_id = int(task.id)+ row = (+ db.query(TaskChatMessage.turn_id)+ .filter(+ TaskChatMessage.task_id == task_id,+ TaskChatMessage.role == "user",+ TaskChatMessage.delivery_status == DELIVERY_PENDING,+ )+ .order_by(TaskChatMessage.id.desc())+ .first()+ )+ turn_id = str(row[0]) if row is not None and row[0] is not None else None+ if task.user_id is not None:+ persist_assistant_message_no_commit(+ db,+ task_id=task_id,+ user_id=int(task.user_id),+ content=CLIENT_SAFE_TASK_FAILURE,+ message_type=TASK_FAILURE_MESSAGE_TYPE,+ turn_id=turn_id,+ )+ if turn_id is not None:+ mark_user_message_delivery(+ db, task_id=task_id, turn_id=turn_id, status=DELIVERY_DISPATCHED+ )+ return turn_id+++def _finalize_sdk_cancel_sync(+ *,+ task_id: int,+ agent_id: int,+ expected_run_id: str | None,+ expected_state_version: int,+ terminal_event_state: dict[str, Any],+) -> bool:+ """Persist cancellation for one exact target; return whether this wrote it.++ Raises ``StaleTaskRunError`` before any write, or after a rollback, when+ the target is neither still running as read nor already settled. When+ the target is settled or written here, it fills ``terminal_event_state``+ with the control snapshot the terminal broadcast carries, once the row+ is durable. The task's cached+ read is invalidated after this call's own commit.+ """++ from .task_coordinator_runtime import current_task_coordinator+ from .task_coordinator_service import (+ lock_task_lease_no_commit,+ task_lease_predicate,+ )+ from .task_execution_event_writer import stage_result_fact_no_commit+ from .task_lease_service import task_settlement_ownership_values+ from .task_orchestrator import (+ invalidate_task_cache_best_effort,+ sync_trigger_run_status,+ )+ from .workforce_runtime import sync_workforce_run_status++ SessionLocal = get_session_local()+ with SessionLocal() as db:+ coordinator = current_task_coordinator(task_id)+ if coordinator is not None:+ assert coordinator.lease is not None+ if not lock_task_lease_no_commit(db, coordinator.lease):+ raise StaleTaskRunError(f"task {task_id} cancellation owner changed")+ task = _load_sdk_task(db, task_id=task_id, agent_id=agent_id)+ if _is_settled_sdk_cancel_target(+ task,+ expected_run_id=expected_run_id,+ expected_state_version=expected_state_version,+ ):+ if task.conversation_storage_version == 2:+ settled_state = task_control_snapshot(task).as_dict()+ else:+ settled_state = {}+ db.rollback()+ terminal_event_state.update(settled_state)+ return False+ if not _is_exact_cancel_target(+ task,+ expected_run_id=expected_run_id,+ expected_state_version=expected_state_version,+ ):+ db.rollback()+ raise StaleTaskRunError(+ _stale(task, expected_run_id, expected_state_version)+ )++ statement = (+ update(Task)+ .where(+ Task.id == task_id,+ Task.agent_id == agent_id,+ Task.source == SDK_TASK_SOURCE,+ func.coalesce(Task.state_version, 0) == expected_state_version,+ Task.status == TaskStatus.RUNNING,+ )+ .values(+ status=TaskStatus.FAILED,+ control_state=TaskControlState.FAILED.value,+ state_version=expected_state_version + 1,+ **task_settlement_ownership_values(task_id),+ output=None,+ error_message=SDK_CANCEL_ERROR_MESSAGE,+ )+ )+ if expected_run_id is None:+ statement = statement.where(Task.run_id.is_(None))+ else:+ statement = statement.where(Task.run_id == expected_run_id)+ if coordinator is not None:+ assert coordinator.lease is not None+ statement = statement.where(task_lease_predicate(coordinator.lease))++ # ``task`` is already in the identity map, so RETURNING hands back+ # that same object; populate_existing refreshes its attributes.+ updated = db.execute(+ statement.returning(Task).execution_options(+ synchronize_session=False,+ populate_existing=True,+ )+ ).scalar_one_or_none()+ if updated is None:+ db.rollback()+ raise StaleTaskRunError(+ f"task {task_id} changed while finalizing cancel target "+ f"{expected_run_id}/{expected_state_version}"+ )+ sync_workforce_run_status(db, updated, TaskStatus.FAILED)+ sync_trigger_run_status(db, updated, TaskStatus.FAILED)+ turn_id = _close_cancelled_turn_no_commit(db, updated)+ stage_result_fact_no_commit(+ db,+ updated,+ sdk_cancel_result_fact(SDK_CANCEL_ERROR_MESSAGE),+ turn_id=turn_id,+ )+ event_state = (+ task_control_snapshot(updated).as_dict()+ if updated.conversation_storage_version == 2+ else {}+ )+ db.commit()+ terminal_event_state.update(event_state)+ invalidate_task_cache_best_effort(task_id)+ return True+++async def _broadcast_sdk_cancel_terminal_event(+ task_id: int, terminal_event_state: dict[str, Any]+) -> None:+ """Wake the task's event readers once its cancellation is durable.++ The cancelled run's settlement does not announce a cancellation, and a+ reader of the task's event stream would otherwise learn of it only from+ its own periodic row read. ``terminal_event_state`` is the control+ snapshot of a version-2 task; its presence is what makes the broadcast+ carry the turn's display events.+ """++ from .task_event_display import publish_task_result+ from .task_execution import create_terminal_task_error_event++ try:+ await publish_task_result(+ {+ **create_terminal_task_error_event(task_id, SDK_CANCEL_ERROR_MESSAGE),+ **terminal_event_state,+ },+ task_id,+ )+ except Exception:+ # The terminal row is already committed; a failed notification must+ # not turn that durable success into a retried command.+ logger.warning(+ "task %s cancellation was committed but its terminal broadcast failed",+ task_id,+ exc_info=True,+ )+++async def cancel_sdk_task_unserialized(+ *,+ task_id: int,+ agent_id: int,+ expected_run_id: str | None,+ expected_state_version: int,+) -> None:+ """Cancel one exact durable-command target while the caller owns its gate.++ Every expected failure leaves as ``TaskCommandRejected`` or+ ``StaleTaskRunError`` (which the dispatcher rejects as ``stale_run``),+ so the command ends at once with its reason instead of being retried.+ """++ terminal_event_state: dict[str, Any] = {}+ already_settled = await run_db_io_cancellation_safe(+ lambda: _load_cancelable_sdk_task_sync(+ task_id=task_id,+ agent_id=agent_id,+ expected_run_id=expected_run_id,+ expected_state_version=expected_state_version,+ terminal_event_state=terminal_event_state,+ )+ )+ if not already_settled:+ from .task_execution import background_task_manager++ # Returns once the run coroutine has unwound, which normally means+ # its own settlement already wrote the cancelled row.+ await background_task_manager.cancel_task(+ task_id, timeout_seconds=SDK_CANCEL_WAIT_SECONDS+ )++ def finalize() -> bool:+ return _finalize_sdk_cancel_sync(+ task_id=task_id,+ agent_id=agent_id,+ expected_run_id=expected_run_id,+ expected_state_version=expected_state_version,+ terminal_event_state=terminal_event_state,+ )++ try:+ wrote = await run_db_io_cancellation_safe(finalize)+ except StaleTaskRunError:+ # A settlement can commit between this command's read and its+ # conditional UPDATE. Every stale result is raised before any+ # write or after a rollback, so one more pass applies the same+ # exact checks: a settled cancel now matches, and any other+ # change to the target is still rejected.+ wrote = await run_db_io_cancellation_safe(finalize)+ if wrote:+ # No settlement wrote the row, so no coroutine had unwound here.+ # One that registered since the call above is stopped now; its+ # settlement finds the row terminal and writes nothing.+ await background_task_manager.cancel_task(+ task_id, timeout_seconds=SDK_CANCEL_WAIT_SECONDS+ )+ await _broadcast_sdk_cancel_terminal_event(task_id, terminal_event_state)diff --git a/src/xagent/web/services/task_auto_resume.py b/src/xagent/web/services/task_auto_resume.pyindex a8e13005e..bb4bea442 100644--- a/src/xagent/web/services/task_auto_resume.py+++ b/src/xagent/web/services/task_auto_resume.py@@ -2779,10 +2779,11 @@ def _terminal_view_snapshot(db: Session, task_id: int) -> Any:def status_only_terminal_view(db: Session, task_id: int) -> TerminalView | None:"""Whether a status-only caller should read ``task_id`` as failed, and why.- A read model for the public ``/v1`` task reads only: its one caller is- ``_load_task_info_snapshot``, which calls it only for a task it loaded- PAUSED with control state ``paused``; it is never written anywhere, and- the task row stays PAUSED. A caller+ A read model for the public ``/v1`` task reads: ``_load_task_info_snapshot``+ calls it only for a task it loaded PAUSED with control state ``paused``,+ and the SDK cancel request consults it for the same rows so that it+ refuses exactly the tasks those reads report as failed. It is never+ written anywhere, and the task row stays PAUSED. A callerthat sees nothing but the status cannot tell a task waiting for its nextautomatic resume from one that stopped for good, so it would wait on thesecond forever. The answer is a :class:`TerminalView` when all hold:diff --git a/src/xagent/web/services/task_command_execution.py b/src/xagent/web/services/task_command_execution.pyindex 8b642b92a..161d8ec2a 100644--- a/src/xagent/web/services/task_command_execution.py+++ b/src/xagent/web/services/task_command_execution.py@@ -149,6 +149,7 @@ from .managed_file_ref import (DurableStorageOperationError,log_durable_storage_fault,)+from .sdk_task_cancel import SDK_COMMAND_SCOPE, cancel_sdk_task_unserializedfrom .task_command_terminal_events import (TerminalTaskEventDraft,TerminalTaskEventMessageCode,@@ -4943,6 +4944,7 @@ async def _execute_durable_task_command(if (scope_value is not EXTERNAL_COMMAND_SCOPE_ABSENTand scope_value != EXTERNAL_COMMAND_SCOPE+ and scope_value != SDK_COMMAND_SCOPE):raise TaskCommandRejected(f"Cancel command {command.command_id} names task scope "@@ -4950,7 +4952,14 @@ async def _execute_durable_task_command(reason="unsupported_scope",)async with task_execution_controller.command(command.task_id):- if scope_value == EXTERNAL_COMMAND_SCOPE:+ if scope_value == SDK_COMMAND_SCOPE:+ await cancel_sdk_task_unserialized(+ task_id=command.task_id,+ agent_id=agent_id,+ expected_run_id=command.target_run_id,+ expected_state_version=target_state_version,+ )+ elif scope_value == EXTERNAL_COMMAND_SCOPE:await cancel_external_task_unserialized(task_id=command.task_id,agent_id=agent_id,diff --git a/src/xagent/web/services/task_execution.py b/src/xagent/web/services/task_execution.pyindex 5ca11edec..c7b2ad853 100644--- a/src/xagent/web/services/task_execution.py+++ b/src/xagent/web/services/task_execution.py@@ -536,20 +536,28 @@ class _DeferredInjectionRejectedRetryableError(RuntimeError):self.fenced = fenced-def _resume_cancel_settlement_error(task_source: str | None) -> str:+def _resume_cancel_settlement_error(+ task_source: str | None, *, explicit_cancel: bool+) -> str:"""Settlement text for a cancelled resume, matching a cancelled run.An external visitor reads this text in the transcript, and the external- cancel command recognizes it as its own settled outcome.+ cancel command recognizes it as its own settled outcome. An SDK resume+ stopped by an explicit cancel (``explicit_cancel``) settles with the+ text the SDK cancel command recognizes."""from .external_task_cancel import (EXTERNAL_TASK_SOURCE,EXTERNAL_TURN_INTERRUPTED_MESSAGE,)+ from .sdk_task_cancel import sdk_cancel_settlement_textif task_source == EXTERNAL_TASK_SOURCE:return EXTERNAL_TURN_INTERRUPTED_MESSAGE- return "resume execution cancelled"+ return (+ sdk_cancel_settlement_text(task_source, explicit_cancel=explicit_cancel)+ or "resume execution cancelled"+ )def client_safe_error_message(@@ -4132,6 +4140,8 @@ async def execute_resume_background()if lease is None:+ from .sdk_task_cancel import sdk_cancel_settlement_text+prior_status_box: list[TaskStatus] = []run_not_resumable_box: list[bool] = []ended_status_box: list[TaskStatus] = []@@ -4147,9 +4157,21 @@ async def execute_resume_background(ended_status_out=ended_status_box,history_turn_id_out=history_turn_id_box,),+ # Evaluated when the cleanup runs, after the cancellation+ # that triggers it has been recorded as explicit or not.lambda acquired: _settle_resumed_task_lease(acquired,- error_message="resume cancelled during lease acquisition",+ error_message=(+ sdk_cancel_settlement_text(+ trusted_task_source,+ explicit_cancel=(+ background_task_manager.cancel_was_requested(+ resume_owner_task+ )+ ),+ )+ or "resume cancelled during lease acquisition"+ ),turn_id=delivery_turn_id,),)@@ -4881,7 +4903,12 @@ async def execute_resume_background()returnexcept asyncio.CancelledError:- settlement_error = _resume_cancel_settlement_error(trusted_task_source)+ settlement_error = _resume_cancel_settlement_error(+ trusted_task_source,+ explicit_cancel=background_task_manager.cancel_was_requested(+ resume_owner_task+ ),+ )logger.info(f"V2 resume background task {task_id} cancelled")await record_accepted_delivery()if delivery_turn_id is not None and not delivery_was_dispatched:@@ -5243,7 +5270,10 @@ async def execute_resume_background(# one is never resendable).settlement_error = (settlement_error- or _resume_cancel_settlement_error(trusted_task_source)+ or _resume_cancel_settlement_error(+ trusted_task_source,+ explicit_cancel=explicit_cancel,+ ))settlement_report = SettlementReport()if pause_for_input and not explicit_cancel:diff --git a/src/xagent/web/services/task_orchestrator.py b/src/xagent/web/services/task_orchestrator.pyindex e320c1cce..f31c63872 100644--- a/src/xagent/web/services/task_orchestrator.py+++ b/src/xagent/web/services/task_orchestrator.py@@ -110,6 +110,7 @@ from .mcp_runtime import (MCPBuiltinOAuthActorPolicyRequiredError,)from .release_hold import release_hold+from .sdk_task_cancel import sdk_cancel_result_fact, sdk_cancel_settlement_textfrom .task_auto_recovery import (InterruptionDecision,InterruptionOutcome,@@ -2249,7 +2250,7 @@ def _settle_task_lease_attempt(witness = stage_result_fact_no_commit(settle_db,task,- {"error": error_message},+ sdk_cancel_result_fact(error_message),turn_id=history_turn_id,)if attempt is not None:@@ -2861,7 +2862,19 @@ def _schedule_bg(client_history_error_message = settlement_errorclient_history_message_type = CLIENT_SAFE_FAILURE_MESSAGE_TYPEelse:- settlement_error = "task execution cancelled"+ # An SDK run stopped by an explicit cancel settles with+ # the text its cancel command recognizes as its outcome.+ settlement_error = (+ sdk_cancel_settlement_text(+ task_source,+ explicit_cancel=(+ background_task_manager.cancel_was_requested(+ asyncio.current_task()+ )+ ),+ )+ or "task execution cancelled"+ )shutdown_handoff_candidate = (get_shutdown_handoff_enabled()and not background_task_manager.cancel_was_requested(diff --git a/tests/web/api/test_sdk_task_cancel.py b/tests/web/api/test_sdk_task_cancel.pynew file mode 100644index 000000000..8d95c8735--- /dev/null+++ b/tests/web/api/test_sdk_task_cancel.py@@ -0,0 +1,905 @@+"""SDK-scope cancel: its execution core and the settlement text it relies on.++Every test here drives one invariant of the SDK cancel path: which+cancellations write the cancellation sentence, that the cancelled run's+settlement and this core leave the same row, that neither claims an outcome+it did not cause, the ownership fence, and the routing that sends only an+``sdk``-scoped command to this core.+"""++import asyncio+import time+from datetime import UTC, datetime, timedelta+from types import SimpleNamespace+from typing import Any+from unittest.mock import AsyncMock, MagicMock++import pytest++from tests.shared.async_waits import DB_PROGRESS_TIMEOUT+from xagent.web.api import websocket as websocket_api+from xagent.web.models.agent import Agent+from xagent.web.models.chat_message import TaskChatMessage+from xagent.web.models.task import Task, TaskStatus+from xagent.web.models.task_execution_event import TaskExecutionEvent+from xagent.web.services import a2a_task_cancel as a2a_cancel_service+from xagent.web.services import sdk_task_cancel+from xagent.web.services import task_command_execution as command_execution_service+from xagent.web.services import task_coordinator_runtime, task_coordinator_service+from xagent.web.services import task_execution as task_execution_service+from xagent.web.services import task_orchestrator+from xagent.web.services.assistant_history_safety import TASK_FAILURE_MESSAGE_TYPE+from xagent.web.services.chat_history_service import (+ DELIVERY_DISPATCHED,+ DELIVERY_PENDING,+)+from xagent.web.services.client_error_messages import CLIENT_SAFE_TASK_FAILURE+from xagent.web.services.external_task_cancel import EXTERNAL_TURN_INTERRUPTED_MESSAGE+from xagent.web.services.sdk_task_cancel import (+ SDK_CANCEL_ERROR_MESSAGE,+ cancel_sdk_task_unserialized,+ is_sdk_cancelled,+ sdk_cancel_result_fact,+)+from xagent.web.services.task_command_transport import (+ ClaimedTaskCommand,+ TaskCommandKind,+ TaskCommandRejected,+)+from xagent.web.services.task_execution_controller import (+ StaleTaskRunError,+ TaskControlState,+)+from xagent.web.services.task_orchestrator import TaskTurnPayload++from .conftest import _admin_headers, _direct_db_session, client++pytestmark = pytest.mark.usefixtures("_test_db")++_ROW_COLUMNS = (+ "status",+ "control_state",+ "run_id",+ "state_version",+ "output",+ "error_message",+ "runner_id",+)+++def _create_agent() -> tuple[int, int]:+ response = client.post(+ "/api/agents",+ headers=_admin_headers(),+ json={+ "name": "SDK cancel agent",+ "description": "SDK cancel test agent",+ "instructions": "You are an SDK cancel test agent.",+ "execution_mode": "balanced",+ },+ )+ assert response.status_code == 200, response.text+ agent_id = int(response.json()["id"])+ db = _direct_db_session()+ try:+ owner_user_id = int(db.query(Agent).filter(Agent.id == agent_id).one().user_id)+ finally:+ db.close()+ return agent_id, owner_user_id+++def _create_task(+ *,+ source: str = "sdk",+ status: TaskStatus = TaskStatus.RUNNING,+ control_state: str = TaskControlState.RUNNING.value,+ run_id: str | None = "run-sdk",+ state_version: int = 4,+ error_message: str | None = None,+ output: str | None = None,+ storage_version: int = 1,+) -> tuple[int, int, int]:+ agent_id, owner_user_id = _create_agent()+ db = _direct_db_session()+ try:+ task = Task(+ user_id=owner_user_id,+ title="sdk cancel",+ status=status,+ control_state=control_state,+ run_id=run_id,+ state_version=state_version,+ agent_id=agent_id,+ source=source,+ is_visible=False,+ error_message=error_message,+ output=output,+ conversation_storage_version=storage_version,+ )+ db.add(task)+ db.commit()+ return int(task.id), agent_id, owner_user_id+ finally:+ db.close()+++def _load_task(task_id: int) -> Task:+ db = _direct_db_session()+ try:+ return db.query(Task).filter(Task.id == task_id).one()+ finally:+ db.close()+++def _row(task_id: int) -> dict[str, Any]:+ task = _load_task(task_id)+ return {column: getattr(task, column) for column in _ROW_COLUMNS}+++def _assistant_rows(task_id: int) -> list[tuple[str, str]]:+ db = _direct_db_session()+ try:+ return [+ (str(row.content), str(row.message_type))+ for row in db.query(TaskChatMessage)+ .filter(+ TaskChatMessage.task_id == task_id,+ TaskChatMessage.role == "assistant",+ )+ .order_by(TaskChatMessage.id)+ ]+ finally:+ db.close()+++def _result_facts(task_id: int) -> list[dict[str, Any]]:+ db = _direct_db_session()+ try:+ return [+ dict(event.payload)+ for event in db.query(TaskExecutionEvent)+ .filter(+ TaskExecutionEvent.task_id == task_id,+ TaskExecutionEvent.kind == "execution_settled",+ )+ .order_by(TaskExecutionEvent.id)+ ]+ finally:+ db.close()+++def _seed_pending_user_message(task_id: int, owner_user_id: int, turn_id: str) -> None:+ db = _direct_db_session()+ try:+ db.add(+ TaskChatMessage(+ task_id=task_id,+ user_id=owner_user_id,+ role="user",+ content="stop this one",+ message_type="user_message",+ turn_id=turn_id,+ delivery_status=DELIVERY_PENDING,+ )+ )+ db.commit()+ finally:+ db.close()+++def _delivery_status(task_id: int, turn_id: str) -> str:+ db = _direct_db_session()+ try:+ row = (+ db.query(TaskChatMessage)+ .filter(+ TaskChatMessage.task_id == task_id,+ TaskChatMessage.turn_id == turn_id,+ TaskChatMessage.role == "user",+ )+ .one()+ )+ return str(row.delivery_status)+ finally:+ db.close()+++def _broadcast_manager(monkeypatch: pytest.MonkeyPatch) -> MagicMock:+ manager = MagicMock()+ manager.broadcast_to_task = AsyncMock()+ monkeypatch.setattr(websocket_api, "manager", manager)+ return manager+++def _assert_cancelled(task_id: int, *, before: Task, storage_version: int) -> None:+ """The one row both writers must leave, whichever of them wrote it."""+ task = _load_task(task_id)+ assert is_sdk_cancelled(task.status, task.error_message)+ assert task.control_state == TaskControlState.FAILED.value+ assert task.error_message == SDK_CANCEL_ERROR_MESSAGE+ assert task.output is None+ assert task.run_id == before.run_id+ assert task.state_version == before.state_version + 1+ assert task.runner_id is None and task.lease_expires_at is None+ assert _assistant_rows(task_id) == [+ (CLIENT_SAFE_TASK_FAILURE, TASK_FAILURE_MESSAGE_TYPE)+ ]+ facts = _result_facts(task_id)+ if storage_version == 2:+ assert [fact["result"] for fact in facts] == [+ {"status": "cancelled", "error": SDK_CANCEL_ERROR_MESSAGE}+ ]+ else:+ assert facts == []+++async def _start_blocked_run(+ monkeypatch: pytest.MonkeyPatch, *, task_id: int, owner_user_id: int, source: str+) -> asyncio.Task:+ """A real background run parked inside its execution, ready to be cancelled."""+ started = asyncio.Event()++ async def block_until_cancelled(**_kwargs: object) -> None:+ started.set()+ await asyncio.Event().wait()++ monkeypatch.setattr(+ task_execution_service, "execute_task_background", block_until_cancelled+ )+ monkeypatch.setattr(+ task_orchestrator,+ "load_task_setup_snapshot_sync",+ lambda *_args, **_kwargs: SimpleNamespace(+ task=SimpleNamespace(id=task_id, conversation_storage_version=1)+ ),+ )+ monkeypatch.setattr(task_orchestrator, "_get_agent_manager", MagicMock())+ bg_task = task_orchestrator._schedule_bg(+ task_id=task_id,+ task_owner_user_id=owner_user_id,+ task_source=source,+ payload=TaskTurnPayload("stop me"),+ force_fresh=False,+ context=None,+ )+ await asyncio.wait_for(started.wait(), timeout=DB_PROGRESS_TIMEOUT)+ return bg_task+++ ("source", "explicit", "expected"),+ [+ ("sdk", True, SDK_CANCEL_ERROR_MESSAGE),+ ("sdk", False, "task execution cancelled"),+ ("widget", True, "task execution cancelled"),+ ("external", True, EXTERNAL_TURN_INTERRUPTED_MESSAGE),+ ],+ ids=["sdk_explicit", "sdk_shutdown", "other_source", "external"],+)+async def test_settlement_text_names_only_an_explicit_sdk_cancel(+ monkeypatch: pytest.MonkeyPatch, source: str, explicit: bool, expected: str+) -> None:+ """Only a run stopped through ``cancel_task`` settles as cancelled.++ A host shutting down cancels the same coroutine without anyone asking,+ and every other source keeps the wording it has always had.+ """+ task_id, _agent_id, owner_user_id = _create_task(+ source=source, run_id=None, state_version=0+ )+ _broadcast_manager(monkeypatch)+ bg_task = await _start_blocked_run(+ monkeypatch, task_id=task_id, owner_user_id=owner_user_id, source=source+ )+ manager = task_execution_service.background_task_manager+ try:+ if explicit:+ await manager.cancel_task(task_id, timeout_seconds=DB_PROGRESS_TIMEOUT)+ else:+ bg_task.cancel()+ with pytest.raises(asyncio.CancelledError):+ await bg_task+ finally:+ manager.running_tasks.pop(task_id, None)++ assert _load_task(task_id).error_message == expected+++ ("source", "explicit", "expected"),+ [+ ("sdk", True, SDK_CANCEL_ERROR_MESSAGE),+ ("sdk", False, "resume execution cancelled"),+ ("widget", True, "resume execution cancelled"),+ ("external", False, EXTERNAL_TURN_INTERRUPTED_MESSAGE),+ ],+)+def test_resume_settlement_text_names_only_an_explicit_sdk_cancel(+ source: str, explicit: bool, expected: str+) -> None:+ assert (+ task_execution_service._resume_cancel_settlement_error(+ source, explicit_cancel=explicit+ )+ == expected+ )+++ ("error_message", "expected"),+ [+ (+ SDK_CANCEL_ERROR_MESSAGE,+ {"status": "cancelled", "error": SDK_CANCEL_ERROR_MESSAGE},+ ),+ ("task execution cancelled", {"error": "task execution cancelled"}),+ (None, {"error": None}),+ ],+)+def test_result_fact_names_only_a_cancellation(error_message, expected) -> None:+ assert sdk_cancel_result_fact(error_message) == expected++[email protected]("storage_version", [1, 2])+async def test_running_cancel_is_settled_by_the_cancelled_run(+ monkeypatch: pytest.MonkeyPatch, storage_version: int+) -> None:+ """The cancelled run's own settlement writes the row; the core adds nothing.++ A replay of the same command afterwards finds its target settled and+ leaves the row, the transcript and the facts as they are.+ """+ task_id, agent_id, owner_user_id = _create_task(+ run_id=None, state_version=0, storage_version=storage_version+ )+ manager = _broadcast_manager(monkeypatch)+ await _start_blocked_run(+ monkeypatch, task_id=task_id, owner_user_id=owner_user_id, source="sdk"+ )+ before = _load_task(task_id)+ assert before.status == TaskStatus.RUNNING++ try:+ for _attempt in range(2):+ await cancel_sdk_task_unserialized(+ task_id=task_id,+ agent_id=agent_id,+ expected_run_id=before.run_id,+ expected_state_version=before.state_version,+ )+ _assert_cancelled(task_id, before=before, storage_version=storage_version)+ finally:+ task_execution_service.background_task_manager.running_tasks.pop(task_id, None)+ # One terminal announcement per call; a version-2 task also replays the+ # turn's display events ahead of each.+ announcements = [+ call.args[0]+ for call in manager.broadcast_to_task.await_args_list+ if call.args[0].get("type") == "task_error"+ ]+ assert len(announcements) == 2++[email protected]("storage_version", [1, 2])+async def test_running_cancel_is_finalized_by_the_core_without_a_local_run(+ monkeypatch: pytest.MonkeyPatch, storage_version: int+) -> None:+ """With no coroutine here to settle the run, the core writes the same row."""+ task_id, agent_id, owner_user_id = _create_task(+ output="stale output", storage_version=storage_version+ )+ _seed_pending_user_message(task_id, owner_user_id, "turn-running")+ _broadcast_manager(monkeypatch)+ synced: list[str] = []+ monkeypatch.setattr(+ "xagent.web.services.workforce_runtime.sync_workforce_run_status",+ lambda _db, _task, status: synced.append(f"workforce:{status.value}"),+ )+ monkeypatch.setattr(+ task_orchestrator,+ "sync_trigger_run_status",+ lambda _db, _task, status: synced.append(f"trigger:{status.value}"),+ )+ before = _load_task(task_id)++ await cancel_sdk_task_unserialized(+ task_id=task_id,+ agent_id=agent_id,+ expected_run_id=before.run_id,+ expected_state_version=before.state_version,+ )++ _assert_cancelled(task_id, before=before, storage_version=storage_version)+ assert _delivery_status(task_id, "turn-running") == DELIVERY_DISPATCHED+ assert synced == ["workforce:failed", "trigger:failed"]+++async def test_a_run_registered_after_the_first_cancel_is_stopped_after_the_write(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ """The core wrote the row because no coroutine was here to settle it, so+ one that registered meanwhile is cancelled once the row is terminal."""+ task_id, agent_id, _owner = _create_task()+ _broadcast_manager(monkeypatch)+ statuses: list[TaskStatus] = []++ async def record_status(_task_id: int, **_kwargs: object) -> SimpleNamespace:+ statuses.append(_load_task(task_id).status)+ return SimpleNamespace(requested=False)++ monkeypatch.setattr(+ task_execution_service.background_task_manager, "cancel_task", record_status+ )++ await cancel_sdk_task_unserialized(+ task_id=task_id,+ agent_id=agent_id,+ expected_run_id="run-sdk",+ expected_state_version=4,+ )++ assert statuses == [TaskStatus.RUNNING, TaskStatus.FAILED]+++ "moved",+ [+ {"status": TaskStatus.FAILED, "error_message": "boom", "state_version": 5},+ {"status": TaskStatus.COMPLETED, "output": "done", "state_version": 5},+ {"status": TaskStatus.PAUSED, "state_version": 5},+ {"run_id": "run-next"},+ {"state_version": 6},+ {"status": TaskStatus.WAITING_FOR_USER},+ ],+ ids=["failed_itself", "completed", "paused", "new_run", "moved", "waiting"],+)+async def test_cancel_never_claims_an_outcome_it_did_not_cause(+ monkeypatch: pytest.MonkeyPatch, moved: dict[str, Any]+) -> None:+ """A target that ended or moved on its own is rejected and left as it is."""+ task_id, agent_id, _owner = _create_task()+ db = _direct_db_session()+ try:+ db.query(Task).filter(Task.id == task_id).update(moved)+ db.commit()+ finally:+ db.close()+ manager = _broadcast_manager(monkeypatch)+ before = _row(task_id)++ with pytest.raises(TaskCommandRejected) as raised:+ await cancel_sdk_task_unserialized(+ task_id=task_id,+ agent_id=agent_id,+ expected_run_id="run-sdk",+ expected_state_version=4,+ )++ assert raised.value.reason == "stale_run"+ assert _row(task_id) == before+ assert _assistant_rows(task_id) == []+ assert manager.broadcast_to_task.await_count == 0++[email protected]("source", ["a2a", "external", "internal", "trigger"])+async def test_cancel_reaches_only_sdk_tasks(+ monkeypatch: pytest.MonkeyPatch, source: str+) -> None:+ task_id, agent_id, _owner = _create_task(source=source)+ _broadcast_manager(monkeypatch)+ before = _row(task_id)++ with pytest.raises(TaskCommandRejected) as raised:+ await cancel_sdk_task_unserialized(+ task_id=task_id,+ agent_id=agent_id,+ expected_run_id="run-sdk",+ expected_state_version=4,+ )++ assert raised.value.reason == "task_not_found"+ assert _row(task_id) == before+++async def test_finalize_accepts_a_settlement_that_lands_mid_cancel(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ """A stale finalize is rechecked once, and a settled cancel then matches."""+ task_id, agent_id, _owner = _create_task()+ _broadcast_manager(monkeypatch)+ real_finalize = sdk_task_cancel._finalize_sdk_cancel_sync+ calls = 0++ def settle_then_report_stale(**kwargs: Any) -> bool:+ nonlocal calls+ calls += 1+ if calls == 1:+ db = _direct_db_session()+ try:+ db.query(Task).filter(Task.id == task_id).update(+ {+ "status": TaskStatus.FAILED,+ "control_state": TaskControlState.FAILED.value,+ "state_version": 5,+ "error_message": SDK_CANCEL_ERROR_MESSAGE,+ }+ )+ db.commit()+ finally:+ db.close()+ raise StaleTaskRunError("settled between the read and the update")+ return real_finalize(**kwargs)++ monkeypatch.setattr(+ sdk_task_cancel, "_finalize_sdk_cancel_sync", settle_then_report_stale+ )++ await cancel_sdk_task_unserialized(+ task_id=task_id,+ agent_id=agent_id,+ expected_run_id="run-sdk",+ expected_state_version=4,+ )++ assert calls == 2+ assert _load_task(task_id).state_version == 5+ assert _assistant_rows(task_id) == []++[email protected]("fence", ["lock", "update"])+def test_replaced_attempt_rejects_cancel_and_preserves_task(+ monkeypatch: pytest.MonkeyPatch, fence: str+) -> None:+ """An old cancel command cannot settle a replacement lease attempt."""+ task_id, agent_id, _owner = _create_task()+ db = _direct_db_session()+ try:+ lease = task_coordinator_service.acquire_task_lease_no_commit(+ db, task_id, runner_id="worker"+ )+ db.commit()+ db.query(Task).filter(Task.id == task_id).update(+ {"lease_attempt_id": "replacement-attempt"}+ )+ db.commit()+ finally:+ db.close()+ monkeypatch.setattr(+ task_coordinator_runtime,+ "current_task_coordinator",+ lambda _task_id: SimpleNamespace(lease=lease, task_id=task_id),+ )+ if fence == "update":+ # Isolate the UPDATE predicate from the lock check ahead of it.+ monkeypatch.setattr(+ task_coordinator_service, "lock_task_lease_no_commit", lambda *_a: True+ )+ before = _row(task_id)++ with pytest.raises(StaleTaskRunError):+ sdk_task_cancel._finalize_sdk_cancel_sync(+ task_id=task_id,+ agent_id=agent_id,+ expected_run_id=before["run_id"],+ expected_state_version=before["state_version"],+ terminal_event_state={},+ )++ assert _row(task_id) == before+ assert _load_task(task_id).lease_attempt_id == "replacement-attempt"++[email protected]("delivery_fails", [False, True])+async def test_cancel_announces_after_commit_and_survives_a_failed_broadcast(+ monkeypatch: pytest.MonkeyPatch, delivery_fails: bool+) -> None:+ task_id, agent_id, _owner = _create_task()+ manager = _broadcast_manager(monkeypatch)+ seen: list[TaskStatus] = []++ async def record_committed_status(*_args: object, **_kwargs: object) -> None:+ seen.append(_load_task(task_id).status)+ if delivery_fails:+ raise RuntimeError("socket gone")++ manager.broadcast_to_task.side_effect = record_committed_status++ await cancel_sdk_task_unserialized(+ task_id=task_id,+ agent_id=agent_id,+ expected_run_id="run-sdk",+ expected_state_version=4,+ )++ assert seen == [TaskStatus.FAILED]+ assert _load_task(task_id).error_message == SDK_CANCEL_ERROR_MESSAGE+++async def test_the_cache_is_invalidated_once_after_the_commit_and_off_the_event_loop(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ """The invalidation is a blocking round trip with some cache backends, so+ it belongs to the finalize that runs in a worker thread."""+ task_id, agent_id, _owner = _create_task()+ _broadcast_manager(monkeypatch)+ calls: list[tuple[int, bool, TaskStatus]] = []++ def record(invalidated_task_id: int) -> None:+ try:+ asyncio.get_running_loop()+ on_event_loop = True+ except RuntimeError:+ on_event_loop = False+ calls.append((invalidated_task_id, on_event_loop, _load_task(task_id).status))++ monkeypatch.setattr(task_orchestrator, "invalidate_task_cache_best_effort", record)++ await cancel_sdk_task_unserialized(+ task_id=task_id,+ agent_id=agent_id,+ expected_run_id="run-sdk",+ expected_state_version=4,+ )++ assert calls == [(task_id, False, TaskStatus.FAILED)]+++def _terminal_frame(manager: MagicMock) -> dict[str, Any]:+ frames = [call.args[0] for call in manager.broadcast_to_task.await_args_list]+ return next(frame for frame in frames if frame.get("type") == "task_error")++[email protected]("storage_version", [1, 2])[email protected]("writer", ["core", "run_settlement"])+async def test_the_terminal_frame_carries_the_control_snapshot_of_a_v2_task(+ monkeypatch: pytest.MonkeyPatch, writer: str, storage_version: int+) -> None:+ """Without the snapshot, readers of a version-2 task's live frames never+ get the turn's last display events. Whichever writer lands the terminal+ row, the frame carries the state it left, and a version-1 task has none."""+ task_id, agent_id, owner_user_id = _create_task(+ run_id=None, state_version=0, storage_version=storage_version+ )+ _seed_pending_user_message(task_id, owner_user_id, "turn-running")+ manager = _broadcast_manager(monkeypatch)+ if writer == "run_settlement":+ await _start_blocked_run(+ monkeypatch, task_id=task_id, owner_user_id=owner_user_id, source="sdk"+ )+ before = _load_task(task_id)++ try:+ await cancel_sdk_task_unserialized(+ task_id=task_id,+ agent_id=agent_id,+ expected_run_id=before.run_id,+ expected_state_version=before.state_version,+ )+ finally:+ task_execution_service.background_task_manager.running_tasks.pop(task_id, None)++ frame = _terminal_frame(manager)+ if storage_version == 2:+ assert frame["state_version"] == before.state_version + 1+ assert frame["task"]["status"] == "failed"+ frames = [call.args[0] for call in manager.broadcast_to_task.await_args_list]+ assert frames.index(frame) > 0, "display events precede the terminal frame"+ else:+ assert "state_version" not in frame+++async def _start_resume(+ monkeypatch: pytest.MonkeyPatch,+ *,+ task_id: int,+ owner_user_id: int,+ run_id: str,+ phase: str,+ source: str = "sdk",+) -> asyncio.Task:+ """A real resume coordinator parked in ``phase``.++ It is registered the way the resume handoff registers it, in+ ``resume_tasks``, so ``cancel_task`` reaches it. ``execution`` parks it+ inside the agent's resumed execution; ``lease_acquisition`` parks it in+ the worker thread that claims the run's lease, which then claims it.+ """+ started = asyncio.Event()+ loop = asyncio.get_running_loop()+ real_acquire = task_execution_service._acquire_resume_task_lease++ def acquire_slowly(*args: Any, **kwargs: Any) -> Any:+ loop.call_soon_threadsafe(started.set)+ time.sleep(0.3)+ return real_acquire(*args, **kwargs)++ if phase == "lease_acquisition":+ monkeypatch.setattr(+ task_execution_service, "_acquire_resume_task_lease", acquire_slowly+ )++ async def block_until_cancelled(*_args: object, **_kwargs: object) -> None:+ started.set()+ await asyncio.Event().wait()++ agent_service = MagicMock()+ agent_service.resume_execution_by_id = AsyncMock(side_effect=block_until_cancelled)+ resume = asyncio.create_task(+ task_execution_service.execute_resume_background(+ task_id,+ agent_service,+ owner_user_id,+ expected_run_id=run_id,+ trusted_task_source=source,+ )+ )+ task_execution_service.background_task_manager.resume_tasks[task_id] = resume+ await asyncio.wait_for(started.wait(), timeout=DB_PROGRESS_TIMEOUT)+ return resume+++ ("phase", "explicit", "expected"),+ [+ ("execution", True, SDK_CANCEL_ERROR_MESSAGE),+ ("execution", False, "resume execution cancelled"),+ ("lease_acquisition", True, SDK_CANCEL_ERROR_MESSAGE),+ ("lease_acquisition", False, "resume cancelled during lease acquisition"),+ ],+ ids=[+ "execution-explicit_cancel",+ "execution-direct_cancel",+ "lease_acquisition-explicit_cancel",+ "lease_acquisition-direct_cancel",+ ],+)+async def test_a_cancelled_resume_settles_as_cancelled_only_when_asked(+ monkeypatch: pytest.MonkeyPatch, phase: str, explicit: bool, expected: str+) -> None:+ """A resumed SDK run takes the same two ways out as a first run: an+ explicit ``cancel_task`` settles it as cancelled, any other cancellation+ keeps the wording it has always had."""+ task_id, _agent_id, owner_user_id = _create_task(+ status=TaskStatus.WAITING_FOR_USER,+ control_state=TaskControlState.WAITING_FOR_USER.value,+ )+ _broadcast_manager(monkeypatch)+ resume = await _start_resume(+ monkeypatch,+ task_id=task_id,+ owner_user_id=owner_user_id,+ run_id="run-sdk",+ phase=phase,+ )+ manager = task_execution_service.background_task_manager+ try:+ if explicit:+ await manager.cancel_task(task_id, timeout_seconds=DB_PROGRESS_TIMEOUT)+ else:+ resume.cancel()+ with pytest.raises(asyncio.CancelledError):+ await resume+ finally:+ manager.running_tasks.pop(task_id, None)+ manager.resume_tasks.pop(task_id, None)++ row = _load_task(task_id)+ assert row.status == TaskStatus.FAILED+ assert row.error_message == expected+++def _cancel_command(task_id: int, agent_id: int, **payload: Any) -> ClaimedTaskCommand:+ return ClaimedTaskCommand(+ id=5,+ task_id=task_id,+ actor_user_id=None,+ command_id=f"cancel:{task_id}:4",+ kind=TaskCommandKind.CANCEL,+ payload={"agent_id": agent_id, "target_state_version": 4, **payload},+ target_run_id="run-sdk",+ attempt_count=1,+ )+++async def test_sdk_scope_routes_to_the_sdk_core_only(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ task_id, agent_id, _owner = _create_task()+ _broadcast_manager(monkeypatch)+ sdk_core = AsyncMock()+ external_core = AsyncMock()+ a2a_core = AsyncMock()+ monkeypatch.setattr(+ command_execution_service, "cancel_sdk_task_unserialized", sdk_core+ )+ monkeypatch.setattr(+ command_execution_service, "cancel_external_task_unserialized", external_core+ )+ monkeypatch.setattr(a2a_cancel_service, "cancel_a2a_task", a2a_core)++ await command_execution_service._execute_durable_task_command(+ _cancel_command(task_id, agent_id, scope="sdk")+ )++ sdk_core.assert_awaited_once_with(+ task_id=task_id,+ agent_id=agent_id,+ expected_run_id="run-sdk",+ expected_state_version=4,+ )+ assert external_core.await_count == 0+ assert a2a_core.await_count == 0++[email protected]("scope", [None, "external"], ids=["a2a", "external"])+async def test_other_scopes_never_reach_the_sdk_core(+ monkeypatch: pytest.MonkeyPatch, scope: str | None+) -> None:+ task_id, agent_id, _owner = _create_task()+ _broadcast_manager(monkeypatch)+ sdk_core = AsyncMock()+ monkeypatch.setattr(+ command_execution_service, "cancel_sdk_task_unserialized", sdk_core+ )+ monkeypatch.setattr(+ command_execution_service, "cancel_external_task_unserialized", AsyncMock()+ )+ monkeypatch.setattr(a2a_cancel_service, "cancel_a2a_task", AsyncMock())+ payload = {} if scope is None else {"scope": scope}++ await command_execution_service._execute_durable_task_command(+ _cancel_command(task_id, agent_id, **payload)+ )++ assert sdk_core.await_count == 0+++def test_lease_fields_are_cleared_only_without_a_coordinator(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ """Shared execution leaves ownership to the coordinator that is draining."""+ task_id, agent_id, _owner = _create_task()+ db = _direct_db_session()+ try:+ lease = task_coordinator_service.acquire_task_lease_no_commit(+ db, task_id, runner_id="worker"+ )+ db.commit()+ finally:+ db.close()+ monkeypatch.setattr(+ task_coordinator_runtime,+ "current_task_coordinator",+ lambda _task_id: SimpleNamespace(lease=lease, task_id=task_id),+ )+ before = _load_task(task_id)++ assert sdk_task_cancel._finalize_sdk_cancel_sync(+ task_id=task_id,+ agent_id=agent_id,+ expected_run_id=before.run_id,+ expected_state_version=before.state_version,+ terminal_event_state={},+ )++ after = _load_task(task_id)+ assert after.error_message == SDK_CANCEL_ERROR_MESSAGE+ assert after.runner_id == "worker"+ assert after.lease_attempt_id == lease.attempt_id+ assert after.lease_expires_at is not None+ assert after.lease_expires_at.replace(tzinfo=UTC) > datetime.now(UTC) - timedelta(+ minutes=5+ )diff --git a/tests/web/api/v1/test_events_stream.py b/tests/web/api/v1/test_events_stream.pyindex 0e535afd8..03afa5e1d 100644--- a/tests/web/api/v1/test_events_stream.py+++ b/tests/web/api/v1/test_events_stream.py@@ -100,10 +100,15 @@ def mock_start_task():yield mocked-# The raw structured output values of a snapshot that has none, and the fields-# a conclusion frame then carries.-NO_STRUCTURED_OUTPUT = {"structured_output_record": None, "format_configured": False}+# The raw structured output values of a snapshot that has none and was not+# cancelled, and the fields a conclusion frame then carries.+NO_STRUCTURED_OUTPUT = {+ "structured_output_record": None,+ "format_configured": False,+ "error_code": None,+}NULL_STRUCTURED_OUTPUT_FIELDS = {+ "error_code": None,"structured_output": None,"structured_output_status": None,"structured_output_reason": None,@@ -947,7 +952,7 @@ async def test_watchdog_terminal_task_row_closes_with_completed():)assert closed is Trueassert sink.queue.get_nowait() == (- es.completed_frame(status="failed", output=None, error="boom"),+ es.completed_frame(status="failed", output=None, error="boom", error_code=None),True,)@@ -1437,7 +1442,9 @@ async def test_close_frame_delivered_when_enqueued_between_two_yields():# awaiting the socket write while other tasks run.won = sink.enqueue_close(- es.completed_frame(status="completed", output="done", error=None)+ es.completed_frame(+ status="completed", output="done", error=None, error_code=None+ ))assert won is Truediff --git a/tests/web/api/v1/test_expired_tasks.py b/tests/web/api/v1/test_expired_tasks.pyindex 588715653..f2268b03f 100644--- a/tests/web/api/v1/test_expired_tasks.py+++ b/tests/web/api/v1/test_expired_tasks.py@@ -197,6 +197,8 @@ def _call(route: str, task_id: int, full_key: str, owner: dict):)if route == "cancel":return client.post(f"/v1/chat/tasks/{task_id}/cancel-question", headers=headers)+ if route == "cancel_task":+ return client.post(f"/v1/chat/tasks/{task_id}/cancel", headers=headers)if route == "upload":return client.post(f"/v1/chat/files?task_id={task_id}",@@ -206,7 +208,16 @@ def _call(route: str, task_id: int, full_key: str, owner: dict):raise AssertionError(route)-TASK_ROUTES = ["get", "steps", "events", "append", "reply", "cancel", "upload"]+TASK_ROUTES = [+ "get",+ "steps",+ "events",+ "append",+ "reply",+ "cancel",+ "cancel_task",+ "upload",+]def _assert_expired(resp, task_id: int, expired_at: datetime = EXPIRED_AT) -> None:diff --git a/tests/web/api/v1/test_personal_key_principal_policy.py b/tests/web/api/v1/test_personal_key_principal_policy.pyindex 39dfe396b..2fd0228dd 100644--- a/tests/web/api/v1/test_personal_key_principal_policy.py+++ b/tests/web/api/v1/test_personal_key_principal_policy.py@@ -129,9 +129,9 @@ def test_personal_key_policy_runs_only_after_the_key_verifies() -> None:@pytest.mark.usefixtures("_template_manager")def test_personal_key_policy_applies_to_every_personal_key_route() -> None:routes = _personal_key_routes()- # The 22 management routes (me, agents, templates, skills, MCP connectors), the 11+ # The 22 management routes (me, agents, templates, skills, MCP connectors), the 12# /v1/chat task routes and the features route.- assert len(routes) == 34, routes+ assert len(routes) == 35, routesheaders = _bearer(_create_personal_key()[0])for method, template in routes:diff --git a/tests/web/api/v1/test_personal_key_tasks.py b/tests/web/api/v1/test_personal_key_tasks.pyindex d907dc874..6a81cd7b2 100644--- a/tests/web/api/v1/test_personal_key_tasks.py+++ b/tests/web/api/v1/test_personal_key_tasks.py@@ -264,6 +264,8 @@ def _call(route: str, task_id: int, full_key: str, owner: dict):)if route == "cancel":return client.post(f"/v1/chat/tasks/{task_id}/cancel-question", headers=headers)+ if route == "cancel_task":+ return client.post(f"/v1/chat/tasks/{task_id}/cancel", headers=headers)if route == "delete":return client.delete(f"/v1/chat/tasks/{task_id}", headers=headers)raise AssertionError(route)@@ -388,8 +390,16 @@ def test_credential_kind_column_mapping(world) -> None:# The write routes that start work on the task re-check that the holder can-# still run its agent; reading, cancelling a question and deleting do not.-ALLOWED = {"append": 202, "reply": 409, "cancel": 409, "delete": 200, "get": 200}+# still run its agent; reading, cancelling a question, cancelling the task and+# deleting do not.+ALLOWED = {+ "append": 202,+ "reply": 409,+ "cancel": 409,+ "cancel_task": 409,+ "delete": 200,+ "get": 200,+}@pytest.mark.parametrize("reachable", [True, False], ids=["in_team", "removed"])diff --git a/tests/web/api/v1/test_status_only_projection.py b/tests/web/api/v1/test_status_only_projection.pyindex a675e28f3..2b62912ed 100644--- a/tests/web/api/v1/test_status_only_projection.py+++ b/tests/web/api/v1/test_status_only_projection.py@@ -49,7 +49,7 @@ from xagent.web.models.database import get_dbfrom xagent.web.models.task import Task, TaskStatusfrom xagent.web.models.task_auto_recovery import TaskAutoRecoveryfrom xagent.web.models.user import User-from xagent.web.services import task_auto_resume, task_event_bridge+from xagent.web.services import sdk_task_cancel, task_auto_resume, task_event_bridgefrom xagent.web.services.hot_path_cache import (InMemoryTTLCache,cache_get,@@ -294,7 +294,15 @@ def test_view_call_sites():defining = pathlib.Path(task_auto_resume.__file__).resolve()computed = [c for c in _calls("status_only_terminal_view") if c[0] != defining]- assert computed == [(tasks_py, "_load_task_info_snapshot")]+ # The read surface derives the failed status; the cancel request consults+ # the same view so it refuses exactly the tasks that read as failed.+ cancel_py = pathlib.Path(sdk_task_cancel.__file__).resolve()+ assert sorted(computed) == sorted(+ [+ (tasks_py, "_load_task_info_snapshot"),+ (cancel_py, "_ended_without_cancel"),+ ]+ )overlaid = _calls("effective_task_snapshot")assert overlaid, "the events endpoint must overlay its snapshots"@@ -311,6 +319,7 @@ ALLOWLIST = {("POST", "/v1/chat/tasks/{task_id}/messages"),("POST", "/v1/chat/tasks/{task_id}/reply"),("POST", "/v1/chat/tasks/{task_id}/cancel-question"),+ ("POST", "/v1/chat/tasks/{task_id}/cancel"),("POST", "/v1/workforces/{workforce_id}/runs"),}EXCLUDE = {diff --git a/tests/web/api/v1/test_structured_output_events.py b/tests/web/api/v1/test_structured_output_events.pyindex 1e1103646..590b4fa43 100644--- a/tests/web/api/v1/test_structured_output_events.py+++ b/tests/web/api/v1/test_structured_output_events.py@@ -182,7 +182,9 @@ def test_the_stream_reads_the_record_with_the_row_in_one_select():def test_a_frame_built_without_fields_carries_null_for_each():- frame = es.completed_frame(status=TaskStatus.FAILED.value, output=None, error="x")+ frame = es.completed_frame(+ status=TaskStatus.FAILED.value, output=None, error="x", error_code=None+ )data = json.loads(frame.split("data: ", 1)[1])assert [data[name] for name in NEW_KEYS] == [None, None, None]diff --git a/tests/web/api/v1/test_structured_output_get.py b/tests/web/api/v1/test_structured_output_get.pyindex c9f0566a3..7de5b4d0b 100644--- a/tests/web/api/v1/test_structured_output_get.py+++ b/tests/web/api/v1/test_structured_output_get.py@@ -143,6 +143,8 @@ def test_the_existing_fields_are_byte_identical_to_the_recorded_baseline():assert all(name in body for name in NEW_KEYS)for name in NEW_KEYS:del body[name]+ # Added after the baseline was recorded; null for these tasks.+ assert body.pop("error_code") is Noneassert json.dumps(body, ensure_ascii=False, separators=(",", ":")) == expecteddiff --git a/tests/web/api/v1/test_v1_cancel_task.py b/tests/web/api/v1/test_v1_cancel_task.pynew file mode 100644index 000000000..88c1e25a6--- /dev/null+++ b/tests/web/api/v1/test_v1_cancel_task.py@@ -0,0 +1,827 @@+"""Integration tests for ``POST /v1/chat/tasks/{task_id}/cancel``.++The route's contract: which task states it cancels, refuses for good, or+asks the caller to retry; that a refusal writes nothing; and that the+``error_code`` it reports is the one the task reads report.+"""++from __future__ import annotations++import asyncio+import json+import threading+from types import SimpleNamespace+from typing import Any+from unittest.mock import AsyncMock, MagicMock, patch++import pytest++from xagent.web.models.task import Task, TaskStatus+from xagent.web.models.task_command import TaskExecutionCommand+from xagent.web.services import sdk_task_cancel, task_auto_resume+from xagent.web.services import task_command_execution as command_execution_service+from xagent.web.services.sdk_task_cancel import SDK_CANCEL_ERROR_MESSAGE+from xagent.web.services.task_command_transport import (+ COMMAND_COMPLETED,+ TaskCommandKind,+ TaskCommandRejected,+ TaskCommandTaskMissing,+ enqueue_task_command,+)+from xagent.web.services.task_execution_controller import TaskControlState++from ..conftest import (+ _admin_headers,+ _direct_db_session,+ _register_second_user,+ client,+)+from .test_workforces import _create_active_workforce, _create_workforce_key++pytestmark = pytest.mark.usefixtures("_test_db")++_agent_name_counter = 0++_ROW_COLUMNS = ("status", "control_state", "run_id", "state_version", "error_message")+++def _bearer(full_key: str) -> dict[str, str]:+ return {"Authorization": f"Bearer {full_key}"}++[email protected](autouse=True)+def no_background_run():+ """Tasks are accepted and left RUNNING with no coroutine behind them."""+ with patch(+ "xagent.web.services.task_orchestrator._schedule_bg",+ new=MagicMock(),+ ):+ yield++[email protected](autouse=True)+def no_live_broadcast(monkeypatch: pytest.MonkeyPatch):+ from xagent.web.api import websocket as websocket_api++ manager = MagicMock()+ manager.broadcast_to_task = AsyncMock()+ monkeypatch.setattr(websocket_api, "manager", manager)+++def _create_agent_with_key(headers: dict[str, str] | None = None) -> tuple[int, str]:+ global _agent_name_counter+ _agent_name_counter += 1+ headers = headers or _admin_headers()+ agent_resp = client.post(+ "/api/agents",+ headers=headers,+ json={+ "name": f"v1 cancel test agent {_agent_name_counter}",+ "description": "test",+ "instructions": "you are a test agent",+ "execution_mode": "balanced",+ },+ )+ assert agent_resp.status_code == 200, agent_resp.text+ agent_id = agent_resp.json()["id"]+ key_resp = client.post(f"/api/agents/{agent_id}/api-key", headers=headers)+ assert key_resp.status_code == 200, key_resp.text+ return agent_id, key_resp.json()["full_key"]+++def _create_task(full_key: str, agent_id: int) -> int:+ response = client.post(+ "/v1/chat/tasks",+ headers=_bearer(full_key),+ json={"agent_id": agent_id, "message": {"role": "user", "content": "hello"}},+ )+ assert response.status_code == 202, response.text+ return response.json()["task_id"]+++def _running_task() -> tuple[int, int, str]:+ agent_id, key = _create_agent_with_key()+ task_id = _create_task(key, agent_id)+ assert _row(task_id)["status"] == TaskStatus.RUNNING+ return task_id, agent_id, key+++def _update(task_id: int, **values: Any) -> None:+ db = _direct_db_session()+ try:+ db.query(Task).filter(Task.id == task_id).update(values)+ db.commit()+ finally:+ db.close()+++def _row(task_id: int) -> dict[str, Any]:+ db = _direct_db_session()+ try:+ task = db.query(Task).filter(Task.id == task_id).one()+ return {column: getattr(task, column) for column in _ROW_COLUMNS}+ finally:+ db.close()+++def _cancel_commands(task_id: int) -> list[TaskExecutionCommand]:+ db = _direct_db_session()+ try:+ return (+ db.query(TaskExecutionCommand)+ .filter(+ TaskExecutionCommand.task_id == task_id,+ TaskExecutionCommand.kind == TaskCommandKind.CANCEL.value,+ )+ .all()+ )+ finally:+ db.close()+++def _queue_start(task_id: int) -> None:+ """Leave an accepted START that no executor has handed off yet."""+ db = _direct_db_session()+ try:+ task = db.query(Task).filter(Task.id == task_id).one()+ enqueue_task_command(+ db,+ task_id=task_id,+ actor_user_id=int(task.user_id),+ command_id="turn-queued",+ kind=TaskCommandKind.START,+ payload={},+ )+ finally:+ db.close()+++def _cancel(task_id: int, key: str):+ return client.post(f"/v1/chat/tasks/{task_id}/cancel", headers=_bearer(key))+++def _get(task_id: int, key: str) -> dict[str, Any]:+ response = client.get(f"/v1/chat/tasks/{task_id}", headers=_bearer(key))+ assert response.status_code == 200, response.text+ return response.json()+++def _error(response) -> str:+ return response.json()["error"]["code"]+++def _completed_frame(task_id: int, key: str) -> dict[str, Any]:+ response = client.get(f"/v1/chat/tasks/{task_id}/events", headers=_bearer(key))+ assert response.status_code == 200, response.text+ blocks = [block for block in response.text.split("\n\n") if block.strip()]+ for block in blocks:+ lines = block.splitlines()+ if "event: task.completed" in lines:+ data = next(line for line in lines if line.startswith("data: "))+ return json.loads(data[len("data: ") :])+ raise AssertionError(response.text)+++def test_cancel_running_task_then_repeat_then_append():+ """The whole caller-visible arc of one cancellation."""+ task_id, agent_id, key = _running_task()+ before = _row(task_id)++ first = _cancel(task_id, key)++ assert first.status_code == 200, first.text+ assert first.json() == {+ "task_id": task_id,+ "status": "failed",+ "run_id": before["run_id"],+ "state_version": before["state_version"] + 1,+ "cancelled_run_id": before["run_id"],+ "error_code": "task_cancelled",+ }+ read = _get(task_id, key)+ assert read["status"] == "failed"+ assert read["error"] == SDK_CANCEL_ERROR_MESSAGE+ assert read["error_code"] == "task_cancelled"+ assert read["output"] is None+ assert _completed_frame(task_id, key)["error_code"] == "task_cancelled"++ # A repeat reports the same outcome and adds no second command.+ again = _cancel(task_id, key)+ assert again.status_code == 200, again.text+ assert again.json() == first.json()+ assert len(_cancel_commands(task_id)) == 1++ # A cancelled task takes a new turn like any failed one, and the code+ # goes with the run it described.+ appended = client.post(+ f"/v1/chat/tasks/{task_id}/messages",+ headers=_bearer(key),+ json={"agent_id": agent_id, "message": {"role": "user", "content": "again"}},+ )+ assert appended.status_code == 202, appended.text+ read = _get(task_id, key)+ assert read["status"] == "running"+ assert read["error_code"] is None+++ ("values", "expected_code"),+ [+ (+ {+ "status": TaskStatus.COMPLETED,+ "control_state": TaskControlState.COMPLETED.value,+ "output": "done",+ },+ "task_not_cancellable",+ ),+ (+ {+ "status": TaskStatus.FAILED,+ "control_state": TaskControlState.FAILED.value,+ "error_message": "boom",+ },+ "task_not_cancellable",+ ),+ (+ {"status": TaskStatus.PENDING, "control_state": "idle"},+ "task_busy",+ ),+ (+ {+ "status": TaskStatus.PAUSED,+ "control_state": TaskControlState.PAUSED.value,+ },+ "task_busy",+ ),+ (+ {+ "status": TaskStatus.WAITING_FOR_USER,+ "control_state": TaskControlState.WAITING_FOR_USER.value,+ },+ "task_busy",+ ),+ ],+ ids=["completed", "failed_itself", "pending", "paused", "waiting"],+)+def test_states_that_are_not_running_are_refused_without_a_write(values, expected_code):+ task_id, _agent_id, key = _running_task()+ _update(task_id, runner_id=None, lease_expires_at=None, **values)+ before = _row(task_id)++ response = _cancel(task_id, key)++ assert response.status_code == 409, response.text+ assert _error(response) == expected_code+ assert _row(task_id) == before+ assert _cancel_commands(task_id) == []+++ "previous",+ [+ {+ "status": TaskStatus.FAILED,+ "control_state": TaskControlState.FAILED.value,+ "error_message": SDK_CANCEL_ERROR_MESSAGE,+ },+ {+ "status": TaskStatus.COMPLETED,+ "control_state": TaskControlState.COMPLETED.value,+ "output": "done",+ },+ ],+ ids=["previous_run_cancelled", "previous_run_completed"],+)+def test_a_queued_turn_is_never_answered_from_the_previous_run(previous):+ """While an appended START is queued the row still shows the run before+ it, so answering from the row would report on the wrong turn: a+ cancelled previous run must not read as "already cancelled"."""+ task_id, _agent_id, key = _running_task()+ _update(task_id, runner_id=None, lease_expires_at=None, **previous)+ _queue_start(task_id)+ before = _row(task_id)++ response = _cancel(task_id, key)++ assert response.status_code == 409, response.text+ assert _error(response) == "task_busy"+ assert _row(task_id) == before+ assert _cancel_commands(task_id) == []+ # The read surface agrees: the queued turn is pending and carries no code.+ read = _get(task_id, key)+ assert read["status"] == "pending"+ assert read["error_code"] is None+++def test_a_task_that_reads_failed_after_automatic_resume_gave_up_is_not_cancellable(+ monkeypatch: pytest.MonkeyPatch,+):+ task_id, _agent_id, key = _running_task()+ _update(+ task_id,+ status=TaskStatus.PAUSED,+ control_state=TaskControlState.PAUSED.value,+ runner_id=None,+ lease_expires_at=None,+ )+ monkeypatch.setattr(+ task_auto_resume,+ "status_only_terminal_view",+ lambda _db, _task_id: SimpleNamespace(state="exhausted"),+ )+ before = _row(task_id)++ response = _cancel(task_id, key)++ assert response.status_code == 409, response.text+ assert _error(response) == "task_not_cancellable"+ assert _row(task_id) == before+ assert _cancel_commands(task_id) == []++[email protected]("case", ["other_agent", "other_user", "missing", "non_sdk"])+def test_tasks_outside_the_key_scope_are_404_and_untouched(case):+ task_id, _agent_id, key = _running_task()+ target = task_id+ if case == "other_agent":+ _other_agent, key = _create_agent_with_key()+ elif case == "other_user":+ headers = _register_second_user("mallory", "mallorypass1")+ _other_agent, key = _create_agent_with_key(headers)+ elif case == "missing":+ target = task_id + 10_000+ else:+ _update(task_id, source="internal")+ before = _row(task_id)++ response = _cancel(target, key)++ assert response.status_code == 404, response.text+ assert _error(response) == "task_not_found"+ assert _row(task_id) == before+ assert _cancel_commands(task_id) == []+++def test_workforce_key_cancels_its_own_run():+ headers = _admin_headers()+ workforce_id = _create_active_workforce(headers, name="Cancel WF")+ key = _create_workforce_key(headers, workforce_id)+ run = client.post(+ f"/v1/workforces/{workforce_id}/runs",+ headers=_bearer(key),+ json={"message": {"role": "user", "content": "go"}},+ )+ assert run.status_code == 202, run.text+ task_id = run.json()["task_id"]++ response = _cancel(task_id, key)++ assert response.status_code == 200, response.text+ assert response.json()["error_code"] == "task_cancelled"+ assert _get(task_id, key)["error_code"] == "task_cancelled"+++def test_a_failure_that_was_not_a_cancellation_carries_no_code():+ task_id, _agent_id, key = _running_task()+ _update(+ task_id,+ status=TaskStatus.FAILED,+ control_state=TaskControlState.FAILED.value,+ error_message="task execution cancelled",+ runner_id=None,+ lease_expires_at=None,+ )++ assert _get(task_id, key)["error_code"] is None+ assert _completed_frame(task_id, key)["error_code"] is None+++def test_an_unapplied_cancel_is_accepted_and_repeats_reach_the_same_command(+ monkeypatch: pytest.MonkeyPatch,+):+ task_id, _agent_id, key = _running_task()+ monkeypatch.setattr(sdk_task_cancel, "dispatch_one_task_command", AsyncMock())+ monkeypatch.setattr(sdk_task_cancel, "SDK_CANCEL_REQUEST_WAIT_SECONDS", 0.1)+ before = _row(task_id)++ first = _cancel(task_id, key)+ second = _cancel(task_id, key)++ for response in (first, second):+ assert response.status_code == 202, response.text+ assert response.json() == {+ "task_id": task_id,+ "status": "running",+ "run_id": before["run_id"],+ "state_version": before["state_version"],+ "cancelled_run_id": None,+ "error_code": None,+ }+ assert _row(task_id) == before+ assert len(_cancel_commands(task_id)) == 1+++ ("reason", "status_code", "code"),+ [+ ("stale_run", 409, "task_busy"),+ ("unsupported_scope", 503, "internal_error"),+ (None, 503, "internal_error"),+ ],+)+def test_a_rejected_command_tells_the_caller_whether_to_retry(+ monkeypatch: pytest.MonkeyPatch, reason, status_code, code+):+ task_id, _agent_id, key = _running_task()+ before = _row(task_id)+ rejecting = AsyncMock(side_effect=TaskCommandRejected("rejected", reason=reason))+ monkeypatch.setattr(+ command_execution_service, "cancel_sdk_task_unserialized", rejecting+ )++ response = _cancel(task_id, key)++ assert response.status_code == status_code, response.text+ assert _error(response) == code+ # The command row's own error text stays on the server.+ assert "rejected" not in response.text+ assert _row(task_id) == before++ # The failed command is put back in the queue by the next call, which+ # then applies it.+ monkeypatch.undo()+ retried = _cancel(task_id, key)+ assert retried.status_code == 200, retried.text+ assert len(_cancel_commands(task_id)) == 1+++def test_a_task_deleted_before_the_command_is_stored_is_404(+ monkeypatch: pytest.MonkeyPatch,+):+ task_id, _agent_id, key = _running_task()+ monkeypatch.setattr(+ sdk_task_cancel,+ "enqueue_task_command",+ MagicMock(side_effect=TaskCommandTaskMissing(f"Task {task_id} not found")),+ )++ response = _cancel(task_id, key)++ assert response.status_code == 404, response.text+ assert _error(response) == "task_not_found"+++_ENDED_ROWS = {+ "completed": {+ "status": TaskStatus.COMPLETED,+ "control_state": TaskControlState.COMPLETED.value,+ "output": "done",+ },+ "failed_itself": {+ "status": TaskStatus.FAILED,+ "control_state": TaskControlState.FAILED.value,+ "error_message": "boom",+ },+}+++def _cancelled_row_values() -> dict[str, Any]:+ return {+ "status": TaskStatus.FAILED,+ "control_state": TaskControlState.FAILED.value,+ "error_message": SDK_CANCEL_ERROR_MESSAGE,+ }+++def _replace_cancel_core(monkeypatch: pytest.MonkeyPatch, core: Any) -> None:+ monkeypatch.setattr(command_execution_service, "cancel_sdk_task_unserialized", core)++[email protected]("ended", list(_ENDED_ROWS))[email protected]("exit_path", ["stale_run", "deadline"])+def test_a_run_that_ended_on_its_own_is_never_reported_as_busy_or_accepted(+ monkeypatch: pytest.MonkeyPatch, ended: str, exit_path: str+):+ """Whichever way the wait ends, a run that ended without a cancel+ answers what the request would have answered on arrival."""+ task_id, _agent_id, key = _running_task()+ values = {**_ENDED_ROWS[ended], "runner_id": None, "lease_expires_at": None}+ after_the_run_ended: list[dict[str, Any]] = []++ def end_the_run() -> None:+ _update(task_id, **values)+ after_the_run_ended.append(_row(task_id))++ if exit_path == "stale_run":++ async def ended_before_it_was_applied(**_kwargs: Any) -> None:+ end_the_run()+ raise TaskCommandRejected("moved", reason="stale_run")++ _replace_cancel_core(monkeypatch, ended_before_it_was_applied)+ else:++ async def ended_while_the_command_waits(*_args: Any, **_kwargs: Any) -> bool:+ end_the_run()+ return False++ monkeypatch.setattr(+ sdk_task_cancel,+ "dispatch_one_task_command",+ AsyncMock(side_effect=ended_while_the_command_waits),+ )+ monkeypatch.setattr(sdk_task_cancel, "SDK_CANCEL_REQUEST_WAIT_SECONDS", 0.1)++ response = _cancel(task_id, key)++ assert response.status_code == 409, response.text+ assert _error(response) == "task_not_cancellable"+ assert _row(task_id) == after_the_run_ended[0]+ assert len(_cancel_commands(task_id)) == 1+++def test_a_run_that_ended_on_its_own_with_a_turn_queued_behind_it_is_not_cancellable(+ monkeypatch: pytest.MonkeyPatch,+):+ """A turn queued after the aimed run ended does not hide that the aimed+ run ended without a cancel."""+ task_id, _agent_id, key = _running_task()++ async def ended_then_a_turn_was_queued(**_kwargs: Any) -> None:+ _update(+ task_id,+ **_ENDED_ROWS["completed"],+ runner_id=None,+ lease_expires_at=None,+ )+ _queue_start(task_id)+ raise TaskCommandRejected("moved", reason="stale_run")++ _replace_cancel_core(monkeypatch, ended_then_a_turn_was_queued)++ response = _cancel(task_id, key)++ assert response.status_code == 409, response.text+ assert _error(response) == "task_not_cancellable"+ assert len(_cancel_commands(task_id)) == 1+++def test_a_cancel_that_settled_as_paused_is_still_reported_busy(+ monkeypatch: pytest.MonkeyPatch,+):+ task_id, _agent_id, key = _running_task()+ after_pause: list[dict[str, Any]] = []++ async def paused_before_it_was_applied(**_kwargs: Any) -> None:+ _update(+ task_id,+ status=TaskStatus.PAUSED,+ control_state=TaskControlState.PAUSED.value,+ runner_id=None,+ lease_expires_at=None,+ )+ after_pause.append(_row(task_id))+ raise TaskCommandRejected("moved", reason="stale_run")++ _replace_cancel_core(monkeypatch, paused_before_it_was_applied)++ response = _cancel(task_id, key)++ assert response.status_code == 409, response.text+ assert _error(response) == "task_busy"+ assert _row(task_id) == after_pause[0]++[email protected]("replacement_ended", [False, True], ids=["running", "ended"])+def test_a_cancel_aimed_at_a_run_that_was_replaced_is_not_cancellable(+ monkeypatch: pytest.MonkeyPatch, replacement_ended: bool+):+ """The aimed run is over and the row shows another one, whatever state+ that one is in: retrying would cancel it, so the caller is told no retry+ can cancel this run."""+ task_id, _agent_id, key = _running_task()+ after_replacement: list[dict[str, Any]] = []+ replacement: dict[str, Any] = {"run_id": "run-next", "state_version": 50}+ if replacement_ended:+ replacement.update(_ENDED_ROWS["completed"], runner_id=None)++ async def replaced_before_it_was_applied(**_kwargs: Any) -> None:+ _update(task_id, **replacement)+ after_replacement.append(_row(task_id))+ raise TaskCommandRejected("moved", reason="stale_run")++ _replace_cancel_core(monkeypatch, replaced_before_it_was_applied)++ response = _cancel(task_id, key)++ assert response.status_code == 409, response.text+ assert _error(response) == "task_not_cancellable"+ assert (+ response.json()["error"]["message"]+ == "The run this cancel was aimed at has already ended."+ )+ assert _row(task_id) == after_replacement[0]+ assert len(_cancel_commands(task_id)) == 1+++def test_a_cancel_still_pending_when_its_run_was_replaced_is_not_cancellable(+ monkeypatch: pytest.MonkeyPatch,+):+ """The wait ends with the command still queued and the row already on+ another run: the aimed run is over, so the answer is not an accepted 202+ that a retry would turn into a cancel of the new run."""+ task_id, _agent_id, key = _running_task()++ async def replaced_while_nothing_applies(*_args: Any, **_kwargs: Any) -> bool:+ _update(task_id, run_id="run-next", state_version=50)+ return False++ monkeypatch.setattr(+ sdk_task_cancel, "dispatch_one_task_command", replaced_while_nothing_applies+ )+ monkeypatch.setattr(sdk_task_cancel, "SDK_CANCEL_REQUEST_WAIT_SECONDS", 0.1)++ response = _cancel(task_id, key)++ assert response.status_code == 409, response.text+ assert _error(response) == "task_not_cancellable"+ assert (+ response.json()["error"]["message"]+ == "The run this cancel was aimed at has already ended."+ )+ assert _row(task_id)["run_id"] == "run-next"+++def test_a_turn_queued_while_the_cancel_was_applied_still_reports_the_cancel(+ monkeypatch: pytest.MonkeyPatch,+):+ """The command ended, the task was cancelled by another request and a+ new turn was queued before this request read the outcome: the run this+ call aimed at is cancelled, so the answer says so."""+ task_id, _agent_id, key = _running_task()+ before = _row(task_id)++ async def cancelled_elsewhere_then_appended(**_kwargs: Any) -> None:+ _update(+ task_id,+ runner_id=None,+ lease_expires_at=None,+ **_cancelled_row_values(),+ )+ _queue_start(task_id)+ raise TaskCommandRejected("moved", reason="stale_run")++ _replace_cancel_core(monkeypatch, cancelled_elsewhere_then_appended)++ response = _cancel(task_id, key)++ assert response.status_code == 200, response.text+ assert response.json() == {+ "task_id": task_id,+ "status": "failed",+ "run_id": before["run_id"],+ "state_version": before["state_version"],+ "cancelled_run_id": before["run_id"],+ "error_code": "task_cancelled",+ }+++def test_a_completed_command_is_a_cancel_of_its_aimed_run_even_if_a_turn_is_queued(+ monkeypatch: pytest.MonkeyPatch,+):+ task_id, _agent_id, key = _running_task()+ before = _row(task_id)++ async def cancelled_then_appended(**_kwargs: Any) -> None:+ _update(+ task_id,+ state_version=before["state_version"] + 1,+ runner_id=None,+ lease_expires_at=None,+ **_cancelled_row_values(),+ )+ _queue_start(task_id)++ _replace_cancel_core(monkeypatch, cancelled_then_appended)++ response = _cancel(task_id, key)++ assert response.status_code == 200, response.text+ assert response.json() == {+ "task_id": task_id,+ "status": "failed",+ "run_id": before["run_id"],+ "state_version": before["state_version"] + 1,+ "cancelled_run_id": before["run_id"],+ "error_code": "task_cancelled",+ }+++def test_a_completed_command_names_its_own_run_when_a_new_run_already_runs(+ monkeypatch: pytest.MonkeyPatch,+):+ """The answer describes the row as it is now; ``cancelled_run_id`` is the+ run the command cancelled, and nothing of the new run is touched."""+ task_id, _agent_id, key = _running_task()+ before = _row(task_id)+ after_the_new_run_started: list[dict[str, Any]] = []++ async def cancelled_then_next_run_started(**_kwargs: Any) -> None:+ _update(+ task_id,+ status=TaskStatus.RUNNING,+ control_state=TaskControlState.RUNNING.value,+ run_id="run-next",+ state_version=before["state_version"] + 2,+ error_message=None,+ )+ after_the_new_run_started.append(_row(task_id))++ _replace_cancel_core(monkeypatch, cancelled_then_next_run_started)++ response = _cancel(task_id, key)++ assert response.status_code == 200, response.text+ assert response.json() == {+ "task_id": task_id,+ "status": "running",+ "run_id": "run-next",+ "state_version": before["state_version"] + 2,+ "cancelled_run_id": before["run_id"],+ "error_code": None,+ }+ assert _row(task_id) == after_the_new_run_started[0]+ assert len(_cancel_commands(task_id)) == 1+++def test_a_command_applied_by_another_process_while_polling_is_reported(+ monkeypatch: pytest.MonkeyPatch,+):+ task_id, _agent_id, key = _running_task()+ before = _row(task_id)++ def apply_elsewhere() -> None:+ _update(+ task_id,+ state_version=before["state_version"] + 1,+ runner_id=None,+ lease_expires_at=None,+ **_cancelled_row_values(),+ )+ db = _direct_db_session()+ try:+ db.query(TaskExecutionCommand).filter(+ TaskExecutionCommand.task_id == task_id,+ TaskExecutionCommand.kind == TaskCommandKind.CANCEL.value,+ ).update({"status": COMMAND_COMPLETED})+ db.commit()+ finally:+ db.close()++ async def left_to_the_owning_process(*_args: Any, **_kwargs: Any) -> bool:+ threading.Timer(0.2, apply_elsewhere).start()+ return False++ monkeypatch.setattr(+ sdk_task_cancel,+ "dispatch_one_task_command",+ AsyncMock(side_effect=left_to_the_owning_process),+ )++ response = _cancel(task_id, key)++ assert response.status_code == 200, response.text+ assert response.json()["cancelled_run_id"] == before["run_id"]+ assert response.json()["error_code"] == "task_cancelled"+++def test_the_wait_is_timed_from_before_the_command_is_dispatched(+ monkeypatch: pytest.MonkeyPatch,+):+ """A dispatch that outlasts the wait leaves nothing to wait for: the+ first poll after it already finds the deadline passed."""+ task_id, _agent_id, key = _running_task()+ polls: list[int] = []+ real_load = sdk_task_cancel.load_task_command++ def counting_load(command_db_id: int):+ polls.append(command_db_id)+ return real_load(command_db_id)++ async def slow_dispatch_that_applies_nothing(*_args: Any, **_kwargs: Any) -> bool:+ await asyncio.sleep(0.4)+ return False++ monkeypatch.setattr(+ sdk_task_cancel,+ "dispatch_one_task_command",+ AsyncMock(side_effect=slow_dispatch_that_applies_nothing),+ )+ monkeypatch.setattr(sdk_task_cancel, "load_task_command", counting_load)+ monkeypatch.setattr(sdk_task_cancel, "SDK_CANCEL_REQUEST_WAIT_SECONDS", 0.1)++ response = _cancel(task_id, key)++ assert response.status_code == 202, response.text+ assert len(polls) == 1