Skip to content

Enterprise agent platform for building, deploying, and operating AI teammates.

feat(v1): cancel a running task with POST /v1/chat/tasks/{task_id}/cancel (#3777)

Alex Liubbd7d1a23612
diff --git a/docs/deployment.md b/docs/deployment.md
index 88679939b..045d6b3da 100644
--- a/docs/deployment.md
+++ b/docs/deployment.md
@@ -991,3 +991,23 @@ With automatic resume on, and `XAGENT_TASK_INFRA_FAILURE_PAUSE_ENABLED` not set
### Rollback
- Nothing stored needs to be undone. Reads show `paused` again. SDK tasks paused by a settlement interruption keep being resumed automatically and rest `paused` when automatic resume stops; deploying again makes them read `failed`.
+
+## 2026-11-13 — SDK tasks: `POST /v1/chat/tasks/{task_id}/cancel`, and `error_code` on task reads
+
+A running task created through `/v1` can now be cancelled. The cancelled task reads `failed` with a new `error_code` of `task_cancelled`, and accepts a new message like any failed task.
+
+### Client-visible changes
+
+- `POST /v1/chat/tasks/{task_id}/cancel` answers 200 once the task is cancelled, including when an earlier call already cancelled it, and 202 when the cancel is recorded but not applied yet. If a new turn has started by the time 200 is returned, `status` and `run_id` describe the new turn and `cancelled_run_id` names the cancelled one.
+- 409 `task_not_cancellable` means the run ended without being cancelled, including one that ended on its own, or was replaced by another turn, while the cancel was being applied. 409 `task_busy` means retry: the task is waiting for an answer, paused, has a turn queued to start, or changed state while the cancel was applied. 503 means the cancel could not be applied; retry.
+- `GET /v1/chat/tasks/{task_id}` and the `task.completed` frame of `GET /v1/chat/tasks/{task_id}/events` carry `error_code`, `task_cancelled` for a cancelled task and `null` otherwise.
+
+### Deployment impact
+
+- No migration and no configuration.
+- Mixed versions: a run owned by a process that predates this change rejects the cancel command as an unsupported scope, so the endpoint answers 503 until every process that executes tasks has been upgraded; calling again then succeeds.
+- The Python SDK `tasks.cancel()` ships once waiting and paused tasks can be cancelled as well; until then only direct HTTP callers reach this endpoint.
+
+### Rollback
+
+- A task cancelled before the rollback stays `failed`; an older process does not report `error_code`, so that field is absent from its reads. The cancel endpoint answers 404 again.
diff --git a/src/xagent/web/api/v1/_events_stream.py b/src/xagent/web/api/v1/_events_stream.py
index 04da969d8..5096ef403 100644
--- a/src/xagent/web/api/v1/_events_stream.py
+++ b/src/xagent/web/api/v1/_events_stream.py
@@ -413,6 +413,7 @@ def completed_frame(
status: str,
output: str | None,
error: str | None,
+ error_code: str | None,
structured_output: dict[str, Any] | None = None,
structured_output_status: str | None = None,
structured_output_reason: dict[str, Any] | None = None,
@@ -424,6 +425,7 @@ def completed_frame(
"status": status,
"output": output,
"error": error,
+ "error_code": error_code,
"structured_output": structured_output,
"structured_output_status": structured_output_status,
"structured_output_reason": structured_output_reason,
@@ -1502,6 +1504,7 @@ async def watchdog_check_once(
status=status.value,
output=snapshot.output,
error=snapshot.error,
+ error_code=snapshot.error_code,
**_structured_output_fields(snapshot),
)
)
@@ -2144,6 +2147,7 @@ def _terminal_snapshot_stream(
status=snapshot.status.value,
output=snapshot.output,
error=snapshot.error,
+ error_code=snapshot.error_code,
**_structured_output_fields(snapshot),
snapshot_total_steps=snapshot_total_steps,
)
diff --git a/src/xagent/web/api/v1/errors.py b/src/xagent/web/api/v1/errors.py
index 807743f42..24d20be1f 100644
--- a/src/xagent/web/api/v1/errors.py
+++ b/src/xagent/web/api/v1/errors.py
@@ -216,6 +216,14 @@ class V1ErrorCode(str, Enum):
# current run (pending / paused / completed normally / failed). 409.
TASK_NOT_WAITING = "task_not_waiting"
+ # ``POST /v1/chat/tasks/{id}/cancel`` was called on a task that already
+ # ended without being cancelled (completed, or failed on its own), or
+ # whose run the cancel was aimed at ended while the cancel was being
+ # applied and another turn took its place. 409,
+ # not retryable -- a task that can be cancelled later answers
+ # ``task_busy`` instead.
+ TASK_NOT_CANCELLABLE = "task_not_cancellable"
+
# A reply's saved progress could not be read due to a transient
# infrastructure failure. 503, retryable -- distinct from
# ``interaction_not_resumable`` (which means resuming can never
@@ -321,6 +329,9 @@ _DEFAULT_MESSAGES: dict[V1ErrorCode, str] = {
V1ErrorCode.TASK_NOT_WAITING: (
"This task is not waiting for an answer to a question."
),
+ V1ErrorCode.TASK_NOT_CANCELLABLE: (
+ "This task has already ended and cannot be cancelled."
+ ),
V1ErrorCode.REPLY_OUTCOME_UNKNOWN: (
"Reply was accepted but its outcome is not yet known. "
"Repeat the same command_id to check its outcome; do not send a new reply."
diff --git a/src/xagent/web/api/v1/tasks.py b/src/xagent/web/api/v1/tasks.py
index 2a30efd5d..d2eb7af85 100644
--- a/src/xagent/web/api/v1/tasks.py
+++ b/src/xagent/web/api/v1/tasks.py
@@ -45,6 +45,7 @@ from ...schemas.v1 import (
AppendMessageRequest,
AppendMessageResponse,
CancelQuestionResponse,
+ CancelTaskResponse,
CreateTaskRequest,
CreateTaskResponse,
DeleteTaskResponse,
@@ -82,6 +83,7 @@ from ...services.response_format import (
normalize_response_format,
violation_message,
)
+from ...services.sdk_task_cancel import sdk_cancel_error_code
from ...services.structured_output_projection import (
project_structured_output,
response_format_configured,
@@ -590,6 +592,10 @@ class _TaskInfoSnapshot:
# ``project_structured_output`` applied to the effective status.
structured_output_record: Any = None
format_configured: bool = False
+ # ``task_cancelled`` when the stored row is the outcome of a cancel
+ # request, ``None`` for every other row. Cleared like ``error`` wherever
+ # the status shown is not the stored one.
+ error_code: str | None = None
# Terminal statuses an append is accepted from. With shared execution the row
@@ -618,6 +624,7 @@ def effective_task_snapshot(snapshot: _TaskInfoSnapshot) -> _TaskInfoSnapshot:
status=TaskStatus.PENDING,
output=None,
error=None,
+ error_code=None,
)
view = snapshot.view
if view is None:
@@ -628,6 +635,7 @@ def effective_task_snapshot(snapshot: _TaskInfoSnapshot) -> _TaskInfoSnapshot:
control_state=TaskStatus.FAILED.value,
output=None,
error=view.error,
+ error_code=None,
)
@@ -864,6 +872,9 @@ def _load_task_info_snapshot(
start_queued=start_queued,
structured_output_record=task.structured_output_record,
format_configured=response_format_configured(task.agent_config),
+ error_code=sdk_cancel_error_code(
+ task.status, cast(str | None, task.error_message)
+ ),
)
@@ -1030,6 +1041,7 @@ def _get_chat_task_sync(
status=effective.status.value,
output=effective.output,
error=effective.error,
+ error_code=effective.error_code,
completed_at=None,
)
if task.view is not None:
@@ -1039,6 +1051,7 @@ def _get_chat_task_sync(
control_state=effective.control_state,
output=effective.output,
error=effective.error,
+ error_code=effective.error_code,
completed_at=task.view.stopped_at,
)
# Applied on every read, never cached: the record can change without the
@@ -1072,6 +1085,7 @@ def _get_chat_task_sync(
input=task.input,
output=task.output,
error=task.error,
+ error_code=task.error_code,
created_at=task.created_at,
completed_at=completed_at,
)
@@ -1478,6 +1492,96 @@ async def cancel_waiting_task_question(
)
+ "/chat/tasks/{task_id}/cancel",
+ response_model=CancelTaskResponse,
+ responses={
+ 202: {
+ "model": CancelTaskResponse,
+ "description": "The cancel is recorded but not applied yet.",
+ }
+ },
+)
+async def cancel_chat_task(
+ task_id: int,
+ response: Response,
+ principal: ApiKeyPrincipal = Depends(get_principal_from_api_key),
+) -> CancelTaskResponse:
+ """Stop a task that is still running.
+
+ Reuses ``GET``'s ownership rule (another agent's task, another user's
+ agent, a missing id, or a non-SDK task all answer 404) before anything
+ about the task's state is read. The cancelled task reads ``failed``
+ with ``error_code='task_cancelled'``, and accepts
+ ``POST /v1/chat/tasks/{id}/messages`` like any failed task. A workforce
+ run is cancelled as one task.
+
+ The call polls up to about 10 seconds for the cancel to be applied;
+ when this process applies the cancel itself it also waits for the run
+ to stop. 200 means the task is cancelled, including when an earlier
+ call already cancelled it; if a new turn has started since, ``status``
+ and ``run_id`` describe that turn and ``cancelled_run_id`` names the
+ cancelled one. 202 means the cancel is recorded but not applied yet,
+ and calling again reaches the same request.
+
+ Only a running task is cancelled. One that is waiting for an answer,
+ paused, or whose turn is still queued to start answers 409
+ ``task_busy`` and is left untouched.
+
+ Raises:
+ V1ApiError 401: missing / invalid / revoked key.
+ V1ApiError 404: task missing, not owned by the key, or not SDK-created.
+ V1ApiError 409: ``task_not_cancellable`` for a task that already
+ ended without being cancelled, including one whose run ended
+ on its own, or was replaced by another turn, while the cancel
+ was being applied; ``task_busy`` when the task cannot be
+ cancelled in its current state or changed state while the
+ cancel was applied -- retry.
+ V1ApiError 410: ``task_expired`` for a retention-expired task.
+ V1ApiError 503: the cancel could not be applied; retry.
+ """
+ from ...services.sdk_task_cancel import (
+ SdkTaskCancelBusy,
+ SdkTaskCancelFailed,
+ SdkTaskNotCancellable,
+ request_sdk_task_cancel,
+ )
+
+ try:
+ result = await request_sdk_task_cancel(
+ task_id=task_id,
+ scope=_sdk_task_scope(principal),
+ )
+ except task_start_service.TaskStartRejected as exc:
+ _raise_start_rejection(exc)
+ except SdkTaskNotCancellable as exc:
+ raise V1ApiError(
+ V1ErrorCode.TASK_NOT_CANCELLABLE, 409, message=str(exc) or None
+ ) from exc
+ except SdkTaskCancelBusy as exc:
+ raise V1ApiError(V1ErrorCode.TASK_BUSY, 409, message=str(exc)) from exc
+ except SdkTaskCancelFailed as exc:
+ raise V1ApiError(
+ V1ErrorCode.INTERNAL_ERROR,
+ 503,
+ message="The cancel could not be applied; retry.",
+ ) from exc
+
+ await record_principal_usage(
+ principal, route="cancel_task", task_id=result.task_id, agent_id=None
+ )
+ if not result.cancelled:
+ response.status_code = 202
+ return CancelTaskResponse(
+ task_id=result.task_id,
+ status=result.status,
+ run_id=result.run_id,
+ state_version=result.state_version,
+ cancelled_run_id=result.cancelled_run_id,
+ error_code=result.error_code,
+ )
+
+
@router.get("/chat/tasks/{task_id}", response_model=TaskInfoResponse)
async def get_chat_task(
task_id: int,
diff --git a/src/xagent/web/schemas/v1.py b/src/xagent/web/schemas/v1.py
index 7a4d74f40..ac81d2d2d 100644
--- a/src/xagent/web/schemas/v1.py
+++ b/src/xagent/web/schemas/v1.py
@@ -1109,6 +1109,48 @@ class CancelQuestionResponse(BaseModel):
)
+class CancelTaskResponse(BaseModel):
+ """``POST /v1/chat/tasks/{task_id}/cancel`` -> 200 / 202 response.
+
+ 200: the run named by ``cancelled_run_id`` is cancelled, whether this
+ call did it or an earlier one already had. The other fields describe
+ the task now: it reads ``failed`` with ``error_code='task_cancelled'``
+ unless a new turn has started since, in which case ``status``,
+ ``run_id``, ``state_version`` and ``error_code`` describe that turn.
+ 202: the cancel was recorded but not applied yet; ``status`` is still
+ the task's current one and ``cancelled_run_id`` is null. Calling again
+ reaches the same request.
+ """
+
+ task_id: int = Field(..., description="Existing task primary key.")
+ status: str = Field(
+ ...,
+ description=(
+ "Task status as stored: 'failed' once cancelled, the status of a "
+ "turn that started after the cancel, or, on 202, the status the "
+ "cancel is still being applied to."
+ ),
+ )
+ run_id: Optional[str] = Field(..., description="Current execution run identity.")
+ state_version: int = Field(
+ ..., description="Monotonic version of the task control state."
+ )
+ cancelled_run_id: Optional[str] = Field(
+ ...,
+ description=(
+ "The run that was cancelled. Null while the cancel has not been applied."
+ ),
+ )
+ error_code: Optional[str] = Field(
+ ...,
+ description=(
+ "'task_cancelled' while the task reads as cancelled, otherwise "
+ "null (before the cancel is applied, or once a later turn has "
+ "started); the same value GET /v1/chat/tasks/{task_id} reports."
+ ),
+ )
+
+
class CreateWorkforceRunRequest(BaseModel):
"""Body for ``POST /v1/workforces/{workforce_id}/runs``.
@@ -1281,6 +1323,16 @@ class TaskInfoResponse(BaseModel):
"a fixed text saying why automatic resume stopped."
),
)
+ error_code: Optional[str] = Field(
+ None,
+ description=(
+ "Machine-readable reason for status='failed', when one is "
+ "defined: 'task_cancelled' for a task stopped through "
+ "POST /v1/chat/tasks/{task_id}/cancel. Null for every other "
+ "failure, for a derived failed, and whenever status is not "
+ "'failed'."
+ ),
+ )
created_at: datetime
completed_at: Optional[datetime] = Field(
None,
diff --git a/src/xagent/web/services/sdk_task_cancel.py b/src/xagent/web/services/sdk_task_cancel.py
new file mode 100644
index 000000000..3d4ddf779
--- /dev/null
+++ b/src/xagent/web/services/sdk_task_cancel.py
@@ -0,0 +1,757 @@
+"""Cancel one SDK-source task through the durable command channel.
+
+``POST /v1/chat/tasks/{task_id}/cancel`` is split in two halves that live
+here. The request half authorizes the caller, refuses a target that cannot
+be cancelled, persists a CANCEL command and polls a bounded time for it.
+The execution half runs wherever that command is applied -- the process
+that owns the task -- and is the SDK counterpart of the A2A and external
+cores, which load their targets by ``Task.source`` and so cannot reach an
+SDK task.
+
+One judgement decides whether a task was cancelled:
+``is_sdk_cancelled`` -- FAILED carrying a sentence only a cancellation
+writes. The cancelled run's own settlement writes that sentence when the
+run was stopped by an explicit ``cancel_task`` call
+(``sdk_cancel_settlement_text``); this core writes the same row when no
+settlement did. Whichever writer lands first owns the terminal row, and
+the other one changes nothing.
+
+Only a RUNNING task is cancelled here. A task with a turn still queued to
+start, one waiting for the caller's answer and a paused one are refused as
+busy, without a write.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import logging
+from dataclasses import dataclass, replace
+from time import monotonic
+from typing import TYPE_CHECKING, Any
+
+from sqlalchemy import func, update
+from sqlalchemy.orm import Session
+
+from ..models.chat_message import TaskChatMessage
+from ..models.database import get_session_local
+from ..models.task import Task, TaskStatus
+from .db_runtime import run_db_io_cancellation_safe
+from .task_command_transport import (
+ COMMAND_COMPLETED,
+ COMMAND_FAILED,
+ EnqueuedTaskCommand,
+ TaskCommandKind,
+ TaskCommandRejected,
+ TaskCommandTaskMissing,
+ dispatch_one_task_command,
+ enqueue_task_command,
+ load_task_command,
+ retry_failed_task_command,
+)
+from .task_execution_controller import (
+ StaleTaskRunError,
+ TaskControlState,
+ task_control_snapshot,
+)
+
+if TYPE_CHECKING:
+ # ``task_start`` imports the orchestrator, which imports this module.
+ from .task_start import SdkTaskScope
+
+logger = logging.getLogger(__name__)
+
+SDK_TASK_SOURCE = "sdk"
+
+# Command payload scope that routes a CANCEL command to this core. Only a
+# server-side producer writes it; client frames naming a scope are refused.
+SDK_COMMAND_SCOPE = "sdk"
+
+# The durable text of a cancelled SDK task, read back by the caller as the
+# task's ``error``. It is also the cancellation judgement, so rewording it
+# means adding the new sentence to ``_SDK_CANCEL_MESSAGES`` and keeping
+# this one there: rows and processes carrying the old text must still be
+# recognised.
+SDK_CANCEL_ERROR_MESSAGE = "Task was cancelled by the caller."
+
+_SDK_CANCEL_MESSAGES = frozenset({SDK_CANCEL_ERROR_MESSAGE})
+
+# Machine-readable companion of the text on the public read surface.
+SDK_CANCEL_ERROR_CODE = "task_cancelled"
+
+# Passed to ``BackgroundTaskManager.cancel_task``. Nominal, not a bound:
+# that call returns once the cancelled coroutine has unwound, and its
+# cleanup drains database work that cannot be interrupted.
+SDK_CANCEL_WAIT_SECONDS = 5.0
+
+# How long the request waits for its command before answering "accepted".
+SDK_CANCEL_REQUEST_WAIT_SECONDS = 10.0
+
+_COMMAND_POLL_SECONDS = 0.05
+
+# Read by a caller whose cancel was aimed at a run that is no longer the
+# task's current one.
+_AIMED_RUN_ENDED_MESSAGE = "The run this cancel was aimed at has already ended."
+
+_TERMINAL_STATUSES = frozenset({TaskStatus.COMPLETED, TaskStatus.FAILED})
+
+
+def is_sdk_cancelled(status: object, error_message: object) -> bool:
+ """Whether a task row is the outcome of an SDK cancellation.
+
+ The row's ``error_message`` always describes its current run: a new run
+ clears it when it starts and every other terminal write replaces it, so
+ the sentence cannot outlive the run it was written for.
+ """
+
+ return status == TaskStatus.FAILED and error_message in _SDK_CANCEL_MESSAGES
+
+
+def sdk_cancel_error_code(status: object, error_message: object) -> str | None:
+ """The public ``error_code`` of a task row: set only for a cancellation."""
+
+ return SDK_CANCEL_ERROR_CODE if is_sdk_cancelled(status, error_message) else None
+
+
+def sdk_cancel_settlement_text(
+ task_source: str | None, *, explicit_cancel: bool
+) -> str | None:
+ """Settlement text for an SDK run stopped by an explicit ``cancel_task``.
+
+ ``None`` for every other source and for a cancellation nobody asked for
+ (a host shutting down), which keep the wording they have always had.
+ """
+
+ if task_source == SDK_TASK_SOURCE and explicit_cancel:
+ return SDK_CANCEL_ERROR_MESSAGE
+ return None
+
+
+def sdk_cancel_result_fact(error_message: str | None) -> dict[str, Any]:
+ """Result fact for a failed run, naming a cancellation as one.
+
+ The next turn's context reports ``cancelled`` as such instead of as a
+ failure.
+ """
+
+ if error_message in _SDK_CANCEL_MESSAGES:
+ return {"status": "cancelled", "error": error_message}
+ return {"error": error_message}
+
+
+class SdkTaskCancelBusy(Exception):
+ """Nothing was cancelled, and retrying can succeed."""
+
+
+class SdkTaskNotCancellable(Exception):
+ """The run already ended on its own; no retry can cancel it.
+
+ The message, when set, is what the caller reads.
+ """
+
+
+class SdkTaskCancelFailed(Exception):
+ """The command could not be applied; the task state is unchanged by it."""
+
+
+@dataclass(frozen=True)
+class SdkTaskCancelResult:
+ task_id: int
+ status: str
+ run_id: str | None
+ state_version: int
+ cancelled_run_id: str | None
+ error_code: str | None
+ # False while the command is still being applied.
+ cancelled: bool
+
+
+@dataclass(frozen=True)
+class _CancelCommand:
+ command_db_id: int
+ # The run the command was persisted against: the one this request
+ # answers for, whatever run the row shows by the time it is read back.
+ aimed_run_id: str | None
+
+
+def _task_run_id(task: Task) -> str | None:
+ run_id = getattr(task, "run_id", None)
+ return str(run_id) if run_id is not None else None
+
+
+def _result(task: Task) -> SdkTaskCancelResult:
+ cancelled = is_sdk_cancelled(task.status, task.error_message)
+ run_id = _task_run_id(task)
+ return SdkTaskCancelResult(
+ task_id=int(task.id),
+ status=task.status.value,
+ run_id=run_id,
+ state_version=int(task.state_version or 0),
+ cancelled_run_id=run_id if cancelled else None,
+ error_code=sdk_cancel_error_code(task.status, task.error_message),
+ cancelled=cancelled,
+ )
+
+
+def _ended_without_cancel(db: Session, task: Task) -> bool:
+ """Whether the task's run ended and was not an SDK cancellation.
+
+ Covers a COMPLETED or FAILED row and a paused row the read surface
+ reports as failed because automatic resume gave up.
+ """
+
+ from .task_auto_resume import status_only_terminal_view
+
+ if is_sdk_cancelled(task.status, task.error_message):
+ return False
+ if task.status in _TERMINAL_STATUSES:
+ return True
+ if (
+ task.status == TaskStatus.PAUSED
+ and task.control_state == TaskControlState.PAUSED.value
+ ):
+ return status_only_terminal_view(db, int(task.id)) is not None
+ return False
+
+
+def _prepare_cancel_sync(
+ *, task_id: int, scope: SdkTaskScope
+) -> SdkTaskCancelResult | _CancelCommand:
+ """Authorize, classify the target and persist its command, in one read."""
+
+ from .task_start import TaskStartRejected, resolve_sdk_task
+ from .task_start_protocol import has_queued_start
+
+ SessionLocal = get_session_local()
+ with SessionLocal() as db:
+ # Read in the order the public task read uses: until a queued START
+ # is handed off, the row still shows the previous turn, so judging
+ # the row alone would answer for a turn the caller is not asking
+ # about.
+ queued = has_queued_start(db, task_id)
+ task = resolve_sdk_task(task_id, scope, db)
+ if queued:
+ raise SdkTaskCancelBusy(
+ "A turn of this task is queued to start and cannot be "
+ "cancelled yet; retry once it is running."
+ )
+ if is_sdk_cancelled(task.status, task.error_message):
+ return _result(task)
+ if _ended_without_cancel(db, task):
+ raise SdkTaskNotCancellable
+ if task.status != TaskStatus.RUNNING:
+ raise SdkTaskCancelBusy(
+ f"A task that is {task.status.value} cannot be cancelled yet."
+ )
+ target_state_version = int(task.state_version or 0)
+ # Read before the insert commits and expires ``task``: this is the
+ # run the command targets.
+ aimed_run_id = _task_run_id(task)
+ try:
+ enqueued = _enqueue_cancel(db, task, target_state_version)
+ except TaskCommandTaskMissing as exc:
+ # Deleted between the read above and the insert.
+ raise TaskStartRejected("task_not_found") from exc
+ if not enqueued.payload_matches:
+ raise SdkTaskCancelBusy(
+ "Another command holds this cancel identity; retry."
+ )
+ if enqueued.status == COMMAND_FAILED:
+ retry_failed_task_command(db, enqueued.command_id)
+ return _CancelCommand(
+ command_db_id=enqueued.command_id, aimed_run_id=aimed_run_id
+ )
+
+
+def _enqueue_cancel(
+ db: Session, task: Task, target_state_version: int
+) -> EnqueuedTaskCommand:
+ task_id = int(task.id)
+ return enqueue_task_command(
+ db,
+ task_id=task_id,
+ actor_user_id=int(task.user_id) if task.user_id is not None else None,
+ # Keyed on the state read here: a repeat against the same state
+ # is the same command, and a repeat after the state moved is a
+ # new one aimed at the new state.
+ command_id=f"cancel:{task_id}:{target_state_version}",
+ kind=TaskCommandKind.CANCEL,
+ payload={
+ "agent_id": int(task.agent_id),
+ "target_state_version": target_state_version,
+ "scope": SDK_COMMAND_SCOPE,
+ },
+ )
+
+
+def _load_result_sync(
+ *,
+ task_id: int,
+ scope: SdkTaskScope,
+ aimed_run_id: str | None,
+ command_completed: bool,
+) -> SdkTaskCancelResult:
+ """Read the task as it stands once the command is over or overdue.
+
+ ``aimed_run_id`` is the run the command was persisted against. A
+ completed command cancelled that run, so the answer names it as
+ ``cancelled_run_id`` while every other field describes the row now,
+ which may already show a later run. Otherwise the answer is about the
+ aimed run only. A row that shows another run means the aimed one is
+ over, and retrying would aim at the new one, so it raises
+ ``SdkTaskNotCancellable`` with ``_AIMED_RUN_ENDED_MESSAGE``. An aimed
+ run that ended on its own raises ``SdkTaskNotCancellable`` exactly as
+ the request would have on arrival, whether or not a later turn is
+ queued behind it.
+ """
+
+ from .task_start import resolve_sdk_task
+
+ SessionLocal = get_session_local()
+ with SessionLocal() as db:
+ task = resolve_sdk_task(task_id, scope, db)
+ if command_completed:
+ return replace(_result(task), cancelled=True, cancelled_run_id=aimed_run_id)
+ if _task_run_id(task) != aimed_run_id:
+ raise SdkTaskNotCancellable(_AIMED_RUN_ENDED_MESSAGE)
+ if _ended_without_cancel(db, task):
+ raise SdkTaskNotCancellable
+ return _result(task)
+
+
+async def request_sdk_task_cancel(
+ *, task_id: int, scope: SdkTaskScope
+) -> SdkTaskCancelResult:
+ """Authorize and cancel one SDK task, polling a bounded time for it.
+
+ The poll is bounded; a cancel applied in this process first waits for
+ the cancelled run to stop, which has no upper bound.
+
+ ``scope`` is the calling key's ``SdkTaskScope``. A task outside it, a
+ missing one and a non-SDK one raise the same ``TaskStartRejected`` the
+ other task endpoints answer 404 with, before anything is read about the
+ task's state.
+
+ Returns ``cancelled=True`` once the task is the outcome of a
+ cancellation, including when it already was, and once the command that
+ cancelled the aimed run completed (``cancelled_run_id``). Returns
+ ``cancelled=False`` when the command was persisted but not applied
+ within ``SDK_CANCEL_REQUEST_WAIT_SECONDS``; calling again reaches the
+ same command. Raises ``SdkTaskNotCancellable`` when the aimed run ended
+ without being cancelled, whichever way the wait ends.
+ """
+
+ prepared = await run_db_io_cancellation_safe(
+ lambda: _prepare_cancel_sync(task_id=task_id, scope=scope)
+ )
+ if isinstance(prepared, SdkTaskCancelResult):
+ return prepared
+
+ from .task_command_execution import execute_durable_task_command
+
+ aimed_run_id = prepared.aimed_run_id
+ deadline = monotonic() + SDK_CANCEL_REQUEST_WAIT_SECONDS
+ # Applied here when this process owns the run; otherwise the owner's
+ # dispatcher picks the durable row up and the poll below observes it.
+ await dispatch_one_task_command(
+ execute_durable_task_command,
+ command_db_id=prepared.command_db_id,
+ )
+ while True:
+ stored = await run_db_io_cancellation_safe(
+ lambda: load_task_command(prepared.command_db_id)
+ )
+ if stored is not None and stored.status in {COMMAND_COMPLETED, COMMAND_FAILED}:
+ command_completed = stored.status == COMMAND_COMPLETED
+ result = await run_db_io_cancellation_safe(
+ lambda: _load_result_sync(
+ task_id=task_id,
+ scope=scope,
+ aimed_run_id=aimed_run_id,
+ command_completed=command_completed,
+ )
+ )
+ if command_completed:
+ return result
+ # The row decides, not the command: a command that lost to
+ # another cancel of the same run still finds the task cancelled.
+ if result.cancelled:
+ return result
+ if stored.rejection_reason == "stale_run":
+ raise SdkTaskCancelBusy(
+ "The task changed state while the cancel was applied; retry."
+ )
+ raise SdkTaskCancelFailed(stored.error or "cancel command failed")
+ if monotonic() >= deadline:
+ return await run_db_io_cancellation_safe(
+ lambda: _load_result_sync(
+ task_id=task_id,
+ scope=scope,
+ aimed_run_id=aimed_run_id,
+ command_completed=False,
+ )
+ )
+ await asyncio.sleep(_COMMAND_POLL_SECONDS)
+
+
+def _load_sdk_task(db: Session, *, task_id: int, agent_id: int) -> Task:
+ task = (
+ db.query(Task)
+ .filter(
+ Task.id == task_id,
+ Task.agent_id == agent_id,
+ Task.source == SDK_TASK_SOURCE,
+ )
+ .populate_existing()
+ .first()
+ )
+ if task is None:
+ raise TaskCommandRejected(
+ f"task {task_id} is not an SDK task of agent {agent_id}",
+ reason="task_not_found",
+ )
+ return task
+
+
+def _is_settled_sdk_cancel_target(
+ task: Task,
+ *,
+ expected_run_id: str | None,
+ expected_state_version: int,
+) -> bool:
+ """Whether this exact target already reached its cancel outcome.
+
+ True for the command's own finalize replayed, and for the settlement of
+ the run this command cancelled: both leave the target run FAILED at the
+ command's state version or one past it, carrying the cancellation
+ sentence. A run that failed on its own carries another text and is not
+ this command's to report.
+ """
+
+ return (
+ is_sdk_cancelled(task.status, task.error_message)
+ and _task_run_id(task) == expected_run_id
+ and int(task.state_version or 0)
+ in {expected_state_version, expected_state_version + 1}
+ )
+
+
+def _is_exact_cancel_target(
+ task: Task, *, expected_run_id: str | None, expected_state_version: int
+) -> bool:
+ """Whether the row is still the running target this command was aimed at."""
+
+ return (
+ _task_run_id(task) == expected_run_id
+ and int(task.state_version or 0) == expected_state_version
+ and task.status == TaskStatus.RUNNING
+ )
+
+
+def _stale(task: Task, expected_run_id: str | None, expected_state_version: int) -> str:
+ return (
+ f"task {task.id} changed from run/version "
+ f"{expected_run_id}/{expected_state_version} to "
+ f"{_task_run_id(task)}/{int(task.state_version or 0)}/{task.status.value}"
+ )
+
+
+def _load_cancelable_sdk_task_sync(
+ *,
+ task_id: int,
+ agent_id: int,
+ expected_run_id: str | None,
+ expected_state_version: int,
+ terminal_event_state: dict[str, Any],
+) -> bool:
+ """Return whether the exact cancel target is already settled.
+
+ A settled target fills ``terminal_event_state`` with the control
+ snapshot its terminal broadcast carries.
+ """
+
+ SessionLocal = get_session_local()
+ with SessionLocal() as db:
+ task = _load_sdk_task(db, task_id=task_id, agent_id=agent_id)
+ if _is_settled_sdk_cancel_target(
+ task,
+ expected_run_id=expected_run_id,
+ expected_state_version=expected_state_version,
+ ):
+ if task.conversation_storage_version == 2:
+ terminal_event_state.update(task_control_snapshot(task).as_dict())
+ return True
+ if not _is_exact_cancel_target(
+ task,
+ expected_run_id=expected_run_id,
+ expected_state_version=expected_state_version,
+ ):
+ raise TaskCommandRejected(
+ _stale(task, expected_run_id, expected_state_version),
+ reason="stale_run",
+ )
+ return False
+
+
+def _close_cancelled_turn_no_commit(db: Session, task: Task) -> str | None:
+ """Stage what a failed run's settlement leaves beside the terminal row.
+
+ The assistant line mirrors the settlement's own -- the generic failure
+ text under the task-failure provenance -- so a reader of the transcript
+ cannot tell which writer ended the run. The delivery row of the turn is
+ closed as ``dispatched``: its message may have been consumed, so it must
+ not be offered for a resend.
+
+ The running turn is the newest pending user row, which is sound only
+ because a RUNNING task accepts no new turn. Returns that turn's id.
+ """
+
+ from .assistant_history_safety import TASK_FAILURE_MESSAGE_TYPE
+ from .chat_history_service import (
+ DELIVERY_DISPATCHED,
+ DELIVERY_PENDING,
+ mark_user_message_delivery,
+ persist_assistant_message_no_commit,
+ )
+ from .client_error_messages import CLIENT_SAFE_TASK_FAILURE
+
+ task_id = int(task.id)
+ row = (
+ db.query(TaskChatMessage.turn_id)
+ .filter(
+ TaskChatMessage.task_id == task_id,
+ TaskChatMessage.role == "user",
+ TaskChatMessage.delivery_status == DELIVERY_PENDING,
+ )
+ .order_by(TaskChatMessage.id.desc())
+ .first()
+ )
+ turn_id = str(row[0]) if row is not None and row[0] is not None else None
+ if task.user_id is not None:
+ persist_assistant_message_no_commit(
+ db,
+ task_id=task_id,
+ user_id=int(task.user_id),
+ content=CLIENT_SAFE_TASK_FAILURE,
+ message_type=TASK_FAILURE_MESSAGE_TYPE,
+ turn_id=turn_id,
+ )
+ if turn_id is not None:
+ mark_user_message_delivery(
+ db, task_id=task_id, turn_id=turn_id, status=DELIVERY_DISPATCHED
+ )
+ return turn_id
+
+
+def _finalize_sdk_cancel_sync(
+ *,
+ task_id: int,
+ agent_id: int,
+ expected_run_id: str | None,
+ expected_state_version: int,
+ terminal_event_state: dict[str, Any],
+) -> bool:
+ """Persist cancellation for one exact target; return whether this wrote it.
+
+ Raises ``StaleTaskRunError`` before any write, or after a rollback, when
+ the target is neither still running as read nor already settled. When
+ the target is settled or written here, it fills ``terminal_event_state``
+ with the control snapshot the terminal broadcast carries, once the row
+ is durable. The task's cached
+ read is invalidated after this call's own commit.
+ """
+
+ from .task_coordinator_runtime import current_task_coordinator
+ from .task_coordinator_service import (
+ lock_task_lease_no_commit,
+ task_lease_predicate,
+ )
+ from .task_execution_event_writer import stage_result_fact_no_commit
+ from .task_lease_service import task_settlement_ownership_values
+ from .task_orchestrator import (
+ invalidate_task_cache_best_effort,
+ sync_trigger_run_status,
+ )
+ from .workforce_runtime import sync_workforce_run_status
+
+ SessionLocal = get_session_local()
+ with SessionLocal() as db:
+ coordinator = current_task_coordinator(task_id)
+ if coordinator is not None:
+ assert coordinator.lease is not None
+ if not lock_task_lease_no_commit(db, coordinator.lease):
+ raise StaleTaskRunError(f"task {task_id} cancellation owner changed")
+ task = _load_sdk_task(db, task_id=task_id, agent_id=agent_id)
+ if _is_settled_sdk_cancel_target(
+ task,
+ expected_run_id=expected_run_id,
+ expected_state_version=expected_state_version,
+ ):
+ if task.conversation_storage_version == 2:
+ settled_state = task_control_snapshot(task).as_dict()
+ else:
+ settled_state = {}
+ db.rollback()
+ terminal_event_state.update(settled_state)
+ return False
+ if not _is_exact_cancel_target(
+ task,
+ expected_run_id=expected_run_id,
+ expected_state_version=expected_state_version,
+ ):
+ db.rollback()
+ raise StaleTaskRunError(
+ _stale(task, expected_run_id, expected_state_version)
+ )
+
+ statement = (
+ update(Task)
+ .where(
+ Task.id == task_id,
+ Task.agent_id == agent_id,
+ Task.source == SDK_TASK_SOURCE,
+ func.coalesce(Task.state_version, 0) == expected_state_version,
+ Task.status == TaskStatus.RUNNING,
+ )
+ .values(
+ status=TaskStatus.FAILED,
+ control_state=TaskControlState.FAILED.value,
+ state_version=expected_state_version + 1,
+ **task_settlement_ownership_values(task_id),
+ output=None,
+ error_message=SDK_CANCEL_ERROR_MESSAGE,
+ )
+ )
+ if expected_run_id is None:
+ statement = statement.where(Task.run_id.is_(None))
+ else:
+ statement = statement.where(Task.run_id == expected_run_id)
+ if coordinator is not None:
+ assert coordinator.lease is not None
+ statement = statement.where(task_lease_predicate(coordinator.lease))
+
+ # ``task`` is already in the identity map, so RETURNING hands back
+ # that same object; populate_existing refreshes its attributes.
+ updated = db.execute(
+ statement.returning(Task).execution_options(
+ synchronize_session=False,
+ populate_existing=True,
+ )
+ ).scalar_one_or_none()
+ if updated is None:
+ db.rollback()
+ raise StaleTaskRunError(
+ f"task {task_id} changed while finalizing cancel target "
+ f"{expected_run_id}/{expected_state_version}"
+ )
+ sync_workforce_run_status(db, updated, TaskStatus.FAILED)
+ sync_trigger_run_status(db, updated, TaskStatus.FAILED)
+ turn_id = _close_cancelled_turn_no_commit(db, updated)
+ stage_result_fact_no_commit(
+ db,
+ updated,
+ sdk_cancel_result_fact(SDK_CANCEL_ERROR_MESSAGE),
+ turn_id=turn_id,
+ )
+ event_state = (
+ task_control_snapshot(updated).as_dict()
+ if updated.conversation_storage_version == 2
+ else {}
+ )
+ db.commit()
+ terminal_event_state.update(event_state)
+ invalidate_task_cache_best_effort(task_id)
+ return True
+
+
+async def _broadcast_sdk_cancel_terminal_event(
+ task_id: int, terminal_event_state: dict[str, Any]
+) -> None:
+ """Wake the task's event readers once its cancellation is durable.
+
+ The cancelled run's settlement does not announce a cancellation, and a
+ reader of the task's event stream would otherwise learn of it only from
+ its own periodic row read. ``terminal_event_state`` is the control
+ snapshot of a version-2 task; its presence is what makes the broadcast
+ carry the turn's display events.
+ """
+
+ from .task_event_display import publish_task_result
+ from .task_execution import create_terminal_task_error_event
+
+ try:
+ await publish_task_result(
+ {
+ **create_terminal_task_error_event(task_id, SDK_CANCEL_ERROR_MESSAGE),
+ **terminal_event_state,
+ },
+ task_id,
+ )
+ except Exception:
+ # The terminal row is already committed; a failed notification must
+ # not turn that durable success into a retried command.
+ logger.warning(
+ "task %s cancellation was committed but its terminal broadcast failed",
+ task_id,
+ exc_info=True,
+ )
+
+
+async def cancel_sdk_task_unserialized(
+ *,
+ task_id: int,
+ agent_id: int,
+ expected_run_id: str | None,
+ expected_state_version: int,
+) -> None:
+ """Cancel one exact durable-command target while the caller owns its gate.
+
+ Every expected failure leaves as ``TaskCommandRejected`` or
+ ``StaleTaskRunError`` (which the dispatcher rejects as ``stale_run``),
+ so the command ends at once with its reason instead of being retried.
+ """
+
+ terminal_event_state: dict[str, Any] = {}
+ already_settled = await run_db_io_cancellation_safe(
+ lambda: _load_cancelable_sdk_task_sync(
+ task_id=task_id,
+ agent_id=agent_id,
+ expected_run_id=expected_run_id,
+ expected_state_version=expected_state_version,
+ terminal_event_state=terminal_event_state,
+ )
+ )
+ if not already_settled:
+ from .task_execution import background_task_manager
+
+ # Returns once the run coroutine has unwound, which normally means
+ # its own settlement already wrote the cancelled row.
+ await background_task_manager.cancel_task(
+ task_id, timeout_seconds=SDK_CANCEL_WAIT_SECONDS
+ )
+
+ def finalize() -> bool:
+ return _finalize_sdk_cancel_sync(
+ task_id=task_id,
+ agent_id=agent_id,
+ expected_run_id=expected_run_id,
+ expected_state_version=expected_state_version,
+ terminal_event_state=terminal_event_state,
+ )
+
+ try:
+ wrote = await run_db_io_cancellation_safe(finalize)
+ except StaleTaskRunError:
+ # A settlement can commit between this command's read and its
+ # conditional UPDATE. Every stale result is raised before any
+ # write or after a rollback, so one more pass applies the same
+ # exact checks: a settled cancel now matches, and any other
+ # change to the target is still rejected.
+ wrote = await run_db_io_cancellation_safe(finalize)
+ if wrote:
+ # No settlement wrote the row, so no coroutine had unwound here.
+ # One that registered since the call above is stopped now; its
+ # settlement finds the row terminal and writes nothing.
+ await background_task_manager.cancel_task(
+ task_id, timeout_seconds=SDK_CANCEL_WAIT_SECONDS
+ )
+ await _broadcast_sdk_cancel_terminal_event(task_id, terminal_event_state)
diff --git a/src/xagent/web/services/task_auto_resume.py b/src/xagent/web/services/task_auto_resume.py
index a8e13005e..bb4bea442 100644
--- a/src/xagent/web/services/task_auto_resume.py
+++ b/src/xagent/web/services/task_auto_resume.py
@@ -2779,10 +2779,11 @@ def _terminal_view_snapshot(db: Session, task_id: int) -> Any:
def status_only_terminal_view(db: Session, task_id: int) -> TerminalView | None:
"""Whether a status-only caller should read ``task_id`` as failed, and why.
- A read model for the public ``/v1`` task reads only: its one caller is
- ``_load_task_info_snapshot``, which calls it only for a task it loaded
- PAUSED with control state ``paused``; it is never written anywhere, and
- the task row stays PAUSED. A caller
+ A read model for the public ``/v1`` task reads: ``_load_task_info_snapshot``
+ calls it only for a task it loaded PAUSED with control state ``paused``,
+ and the SDK cancel request consults it for the same rows so that it
+ refuses exactly the tasks those reads report as failed. It is never
+ written anywhere, and the task row stays PAUSED. A caller
that sees nothing but the status cannot tell a task waiting for its next
automatic resume from one that stopped for good, so it would wait on the
second forever. The answer is a :class:`TerminalView` when all hold:
diff --git a/src/xagent/web/services/task_command_execution.py b/src/xagent/web/services/task_command_execution.py
index 8b642b92a..161d8ec2a 100644
--- a/src/xagent/web/services/task_command_execution.py
+++ b/src/xagent/web/services/task_command_execution.py
@@ -149,6 +149,7 @@ from .managed_file_ref import (
DurableStorageOperationError,
log_durable_storage_fault,
)
+from .sdk_task_cancel import SDK_COMMAND_SCOPE, cancel_sdk_task_unserialized
from .task_command_terminal_events import (
TerminalTaskEventDraft,
TerminalTaskEventMessageCode,
@@ -4943,6 +4944,7 @@ async def _execute_durable_task_command(
if (
scope_value is not EXTERNAL_COMMAND_SCOPE_ABSENT
and scope_value != EXTERNAL_COMMAND_SCOPE
+ and scope_value != SDK_COMMAND_SCOPE
):
raise TaskCommandRejected(
f"Cancel command {command.command_id} names task scope "
@@ -4950,7 +4952,14 @@ async def _execute_durable_task_command(
reason="unsupported_scope",
)
async with task_execution_controller.command(command.task_id):
- if scope_value == EXTERNAL_COMMAND_SCOPE:
+ if scope_value == SDK_COMMAND_SCOPE:
+ await cancel_sdk_task_unserialized(
+ task_id=command.task_id,
+ agent_id=agent_id,
+ expected_run_id=command.target_run_id,
+ expected_state_version=target_state_version,
+ )
+ elif scope_value == EXTERNAL_COMMAND_SCOPE:
await cancel_external_task_unserialized(
task_id=command.task_id,
agent_id=agent_id,
diff --git a/src/xagent/web/services/task_execution.py b/src/xagent/web/services/task_execution.py
index 5ca11edec..c7b2ad853 100644
--- a/src/xagent/web/services/task_execution.py
+++ b/src/xagent/web/services/task_execution.py
@@ -536,20 +536,28 @@ class _DeferredInjectionRejectedRetryableError(RuntimeError):
self.fenced = fenced
-def _resume_cancel_settlement_error(task_source: str | None) -> str:
+def _resume_cancel_settlement_error(
+ task_source: str | None, *, explicit_cancel: bool
+) -> str:
"""Settlement text for a cancelled resume, matching a cancelled run.
An external visitor reads this text in the transcript, and the external
- cancel command recognizes it as its own settled outcome.
+ cancel command recognizes it as its own settled outcome. An SDK resume
+ stopped by an explicit cancel (``explicit_cancel``) settles with the
+ text the SDK cancel command recognizes.
"""
from .external_task_cancel import (
EXTERNAL_TASK_SOURCE,
EXTERNAL_TURN_INTERRUPTED_MESSAGE,
)
+ from .sdk_task_cancel import sdk_cancel_settlement_text
if task_source == EXTERNAL_TASK_SOURCE:
return EXTERNAL_TURN_INTERRUPTED_MESSAGE
- return "resume execution cancelled"
+ return (
+ sdk_cancel_settlement_text(task_source, explicit_cancel=explicit_cancel)
+ or "resume execution cancelled"
+ )
def client_safe_error_message(
@@ -4132,6 +4140,8 @@ async def execute_resume_background(
)
if lease is None:
+ from .sdk_task_cancel import sdk_cancel_settlement_text
+
prior_status_box: list[TaskStatus] = []
run_not_resumable_box: list[bool] = []
ended_status_box: list[TaskStatus] = []
@@ -4147,9 +4157,21 @@ async def execute_resume_background(
ended_status_out=ended_status_box,
history_turn_id_out=history_turn_id_box,
),
+ # Evaluated when the cleanup runs, after the cancellation
+ # that triggers it has been recorded as explicit or not.
lambda acquired: _settle_resumed_task_lease(
acquired,
- error_message="resume cancelled during lease acquisition",
+ error_message=(
+ sdk_cancel_settlement_text(
+ trusted_task_source,
+ explicit_cancel=(
+ background_task_manager.cancel_was_requested(
+ resume_owner_task
+ )
+ ),
+ )
+ or "resume cancelled during lease acquisition"
+ ),
turn_id=delivery_turn_id,
),
)
@@ -4881,7 +4903,12 @@ async def execute_resume_background(
)
return
except asyncio.CancelledError:
- settlement_error = _resume_cancel_settlement_error(trusted_task_source)
+ settlement_error = _resume_cancel_settlement_error(
+ trusted_task_source,
+ explicit_cancel=background_task_manager.cancel_was_requested(
+ resume_owner_task
+ ),
+ )
logger.info(f"V2 resume background task {task_id} cancelled")
await record_accepted_delivery()
if delivery_turn_id is not None and not delivery_was_dispatched:
@@ -5243,7 +5270,10 @@ async def execute_resume_background(
# one is never resendable).
settlement_error = (
settlement_error
- or _resume_cancel_settlement_error(trusted_task_source)
+ or _resume_cancel_settlement_error(
+ trusted_task_source,
+ explicit_cancel=explicit_cancel,
+ )
)
settlement_report = SettlementReport()
if pause_for_input and not explicit_cancel:
diff --git a/src/xagent/web/services/task_orchestrator.py b/src/xagent/web/services/task_orchestrator.py
index e320c1cce..f31c63872 100644
--- a/src/xagent/web/services/task_orchestrator.py
+++ b/src/xagent/web/services/task_orchestrator.py
@@ -110,6 +110,7 @@ from .mcp_runtime import (
MCPBuiltinOAuthActorPolicyRequiredError,
)
from .release_hold import release_hold
+from .sdk_task_cancel import sdk_cancel_result_fact, sdk_cancel_settlement_text
from .task_auto_recovery import (
InterruptionDecision,
InterruptionOutcome,
@@ -2249,7 +2250,7 @@ def _settle_task_lease_attempt(
witness = stage_result_fact_no_commit(
settle_db,
task,
- {"error": error_message},
+ sdk_cancel_result_fact(error_message),
turn_id=history_turn_id,
)
if attempt is not None:
@@ -2861,7 +2862,19 @@ def _schedule_bg(
client_history_error_message = settlement_error
client_history_message_type = CLIENT_SAFE_FAILURE_MESSAGE_TYPE
else:
- settlement_error = "task execution cancelled"
+ # An SDK run stopped by an explicit cancel settles with
+ # the text its cancel command recognizes as its outcome.
+ settlement_error = (
+ sdk_cancel_settlement_text(
+ task_source,
+ explicit_cancel=(
+ background_task_manager.cancel_was_requested(
+ asyncio.current_task()
+ )
+ ),
+ )
+ or "task execution cancelled"
+ )
shutdown_handoff_candidate = (
get_shutdown_handoff_enabled()
and not background_task_manager.cancel_was_requested(
diff --git a/tests/web/api/test_sdk_task_cancel.py b/tests/web/api/test_sdk_task_cancel.py
new file mode 100644
index 000000000..8d95c8735
--- /dev/null
+++ b/tests/web/api/test_sdk_task_cancel.py
@@ -0,0 +1,905 @@
+"""SDK-scope cancel: its execution core and the settlement text it relies on.
+
+Every test here drives one invariant of the SDK cancel path: which
+cancellations write the cancellation sentence, that the cancelled run's
+settlement and this core leave the same row, that neither claims an outcome
+it did not cause, the ownership fence, and the routing that sends only an
+``sdk``-scoped command to this core.
+"""
+
+import asyncio
+import time
+from datetime import UTC, datetime, timedelta
+from types import SimpleNamespace
+from typing import Any
+from unittest.mock import AsyncMock, MagicMock
+
+import pytest
+
+from tests.shared.async_waits import DB_PROGRESS_TIMEOUT
+from xagent.web.api import websocket as websocket_api
+from xagent.web.models.agent import Agent
+from xagent.web.models.chat_message import TaskChatMessage
+from xagent.web.models.task import Task, TaskStatus
+from xagent.web.models.task_execution_event import TaskExecutionEvent
+from xagent.web.services import a2a_task_cancel as a2a_cancel_service
+from xagent.web.services import sdk_task_cancel
+from xagent.web.services import task_command_execution as command_execution_service
+from xagent.web.services import task_coordinator_runtime, task_coordinator_service
+from xagent.web.services import task_execution as task_execution_service
+from xagent.web.services import task_orchestrator
+from xagent.web.services.assistant_history_safety import TASK_FAILURE_MESSAGE_TYPE
+from xagent.web.services.chat_history_service import (
+ DELIVERY_DISPATCHED,
+ DELIVERY_PENDING,
+)
+from xagent.web.services.client_error_messages import CLIENT_SAFE_TASK_FAILURE
+from xagent.web.services.external_task_cancel import EXTERNAL_TURN_INTERRUPTED_MESSAGE
+from xagent.web.services.sdk_task_cancel import (
+ SDK_CANCEL_ERROR_MESSAGE,
+ cancel_sdk_task_unserialized,
+ is_sdk_cancelled,
+ sdk_cancel_result_fact,
+)
+from xagent.web.services.task_command_transport import (
+ ClaimedTaskCommand,
+ TaskCommandKind,
+ TaskCommandRejected,
+)
+from xagent.web.services.task_execution_controller import (
+ StaleTaskRunError,
+ TaskControlState,
+)
+from xagent.web.services.task_orchestrator import TaskTurnPayload
+
+from .conftest import _admin_headers, _direct_db_session, client
+
+pytestmark = pytest.mark.usefixtures("_test_db")
+
+_ROW_COLUMNS = (
+ "status",
+ "control_state",
+ "run_id",
+ "state_version",
+ "output",
+ "error_message",
+ "runner_id",
+)
+
+
+def _create_agent() -> tuple[int, int]:
+ response = client.post(
+ "/api/agents",
+ headers=_admin_headers(),
+ json={
+ "name": "SDK cancel agent",
+ "description": "SDK cancel test agent",
+ "instructions": "You are an SDK cancel test agent.",
+ "execution_mode": "balanced",
+ },
+ )
+ assert response.status_code == 200, response.text
+ agent_id = int(response.json()["id"])
+ db = _direct_db_session()
+ try:
+ owner_user_id = int(db.query(Agent).filter(Agent.id == agent_id).one().user_id)
+ finally:
+ db.close()
+ return agent_id, owner_user_id
+
+
+def _create_task(
+ *,
+ source: str = "sdk",
+ status: TaskStatus = TaskStatus.RUNNING,
+ control_state: str = TaskControlState.RUNNING.value,
+ run_id: str | None = "run-sdk",
+ state_version: int = 4,
+ error_message: str | None = None,
+ output: str | None = None,
+ storage_version: int = 1,
+) -> tuple[int, int, int]:
+ agent_id, owner_user_id = _create_agent()
+ db = _direct_db_session()
+ try:
+ task = Task(
+ user_id=owner_user_id,
+ title="sdk cancel",
+ status=status,
+ control_state=control_state,
+ run_id=run_id,
+ state_version=state_version,
+ agent_id=agent_id,
+ source=source,
+ is_visible=False,
+ error_message=error_message,
+ output=output,
+ conversation_storage_version=storage_version,
+ )
+ db.add(task)
+ db.commit()
+ return int(task.id), agent_id, owner_user_id
+ finally:
+ db.close()
+
+
+def _load_task(task_id: int) -> Task:
+ db = _direct_db_session()
+ try:
+ return db.query(Task).filter(Task.id == task_id).one()
+ finally:
+ db.close()
+
+
+def _row(task_id: int) -> dict[str, Any]:
+ task = _load_task(task_id)
+ return {column: getattr(task, column) for column in _ROW_COLUMNS}
+
+
+def _assistant_rows(task_id: int) -> list[tuple[str, str]]:
+ db = _direct_db_session()
+ try:
+ return [
+ (str(row.content), str(row.message_type))
+ for row in db.query(TaskChatMessage)
+ .filter(
+ TaskChatMessage.task_id == task_id,
+ TaskChatMessage.role == "assistant",
+ )
+ .order_by(TaskChatMessage.id)
+ ]
+ finally:
+ db.close()
+
+
+def _result_facts(task_id: int) -> list[dict[str, Any]]:
+ db = _direct_db_session()
+ try:
+ return [
+ dict(event.payload)
+ for event in db.query(TaskExecutionEvent)
+ .filter(
+ TaskExecutionEvent.task_id == task_id,
+ TaskExecutionEvent.kind == "execution_settled",
+ )
+ .order_by(TaskExecutionEvent.id)
+ ]
+ finally:
+ db.close()
+
+
+def _seed_pending_user_message(task_id: int, owner_user_id: int, turn_id: str) -> None:
+ db = _direct_db_session()
+ try:
+ db.add(
+ TaskChatMessage(
+ task_id=task_id,
+ user_id=owner_user_id,
+ role="user",
+ content="stop this one",
+ message_type="user_message",
+ turn_id=turn_id,
+ delivery_status=DELIVERY_PENDING,
+ )
+ )
+ db.commit()
+ finally:
+ db.close()
+
+
+def _delivery_status(task_id: int, turn_id: str) -> str:
+ db = _direct_db_session()
+ try:
+ row = (
+ db.query(TaskChatMessage)
+ .filter(
+ TaskChatMessage.task_id == task_id,
+ TaskChatMessage.turn_id == turn_id,
+ TaskChatMessage.role == "user",
+ )
+ .one()
+ )
+ return str(row.delivery_status)
+ finally:
+ db.close()
+
+
+def _broadcast_manager(monkeypatch: pytest.MonkeyPatch) -> MagicMock:
+ manager = MagicMock()
+ manager.broadcast_to_task = AsyncMock()
+ monkeypatch.setattr(websocket_api, "manager", manager)
+ return manager
+
+
+def _assert_cancelled(task_id: int, *, before: Task, storage_version: int) -> None:
+ """The one row both writers must leave, whichever of them wrote it."""
+ task = _load_task(task_id)
+ assert is_sdk_cancelled(task.status, task.error_message)
+ assert task.control_state == TaskControlState.FAILED.value
+ assert task.error_message == SDK_CANCEL_ERROR_MESSAGE
+ assert task.output is None
+ assert task.run_id == before.run_id
+ assert task.state_version == before.state_version + 1
+ assert task.runner_id is None and task.lease_expires_at is None
+ assert _assistant_rows(task_id) == [
+ (CLIENT_SAFE_TASK_FAILURE, TASK_FAILURE_MESSAGE_TYPE)
+ ]
+ facts = _result_facts(task_id)
+ if storage_version == 2:
+ assert [fact["result"] for fact in facts] == [
+ {"status": "cancelled", "error": SDK_CANCEL_ERROR_MESSAGE}
+ ]
+ else:
+ assert facts == []
+
+
+async def _start_blocked_run(
+ monkeypatch: pytest.MonkeyPatch, *, task_id: int, owner_user_id: int, source: str
+) -> asyncio.Task:
+ """A real background run parked inside its execution, ready to be cancelled."""
+ started = asyncio.Event()
+
+ async def block_until_cancelled(**_kwargs: object) -> None:
+ started.set()
+ await asyncio.Event().wait()
+
+ monkeypatch.setattr(
+ task_execution_service, "execute_task_background", block_until_cancelled
+ )
+ monkeypatch.setattr(
+ task_orchestrator,
+ "load_task_setup_snapshot_sync",
+ lambda *_args, **_kwargs: SimpleNamespace(
+ task=SimpleNamespace(id=task_id, conversation_storage_version=1)
+ ),
+ )
+ monkeypatch.setattr(task_orchestrator, "_get_agent_manager", MagicMock())
+ bg_task = task_orchestrator._schedule_bg(
+ task_id=task_id,
+ task_owner_user_id=owner_user_id,
+ task_source=source,
+ payload=TaskTurnPayload("stop me"),
+ force_fresh=False,
+ context=None,
+ )
+ await asyncio.wait_for(started.wait(), timeout=DB_PROGRESS_TIMEOUT)
+ return bg_task
+
+
+ ("source", "explicit", "expected"),
+ [
+ ("sdk", True, SDK_CANCEL_ERROR_MESSAGE),
+ ("sdk", False, "task execution cancelled"),
+ ("widget", True, "task execution cancelled"),
+ ("external", True, EXTERNAL_TURN_INTERRUPTED_MESSAGE),
+ ],
+ ids=["sdk_explicit", "sdk_shutdown", "other_source", "external"],
+)
+async def test_settlement_text_names_only_an_explicit_sdk_cancel(
+ monkeypatch: pytest.MonkeyPatch, source: str, explicit: bool, expected: str
+) -> None:
+ """Only a run stopped through ``cancel_task`` settles as cancelled.
+
+ A host shutting down cancels the same coroutine without anyone asking,
+ and every other source keeps the wording it has always had.
+ """
+ task_id, _agent_id, owner_user_id = _create_task(
+ source=source, run_id=None, state_version=0
+ )
+ _broadcast_manager(monkeypatch)
+ bg_task = await _start_blocked_run(
+ monkeypatch, task_id=task_id, owner_user_id=owner_user_id, source=source
+ )
+ manager = task_execution_service.background_task_manager
+ try:
+ if explicit:
+ await manager.cancel_task(task_id, timeout_seconds=DB_PROGRESS_TIMEOUT)
+ else:
+ bg_task.cancel()
+ with pytest.raises(asyncio.CancelledError):
+ await bg_task
+ finally:
+ manager.running_tasks.pop(task_id, None)
+
+ assert _load_task(task_id).error_message == expected
+
+
+ ("source", "explicit", "expected"),
+ [
+ ("sdk", True, SDK_CANCEL_ERROR_MESSAGE),
+ ("sdk", False, "resume execution cancelled"),
+ ("widget", True, "resume execution cancelled"),
+ ("external", False, EXTERNAL_TURN_INTERRUPTED_MESSAGE),
+ ],
+)
+def test_resume_settlement_text_names_only_an_explicit_sdk_cancel(
+ source: str, explicit: bool, expected: str
+) -> None:
+ assert (
+ task_execution_service._resume_cancel_settlement_error(
+ source, explicit_cancel=explicit
+ )
+ == expected
+ )
+
+
+ ("error_message", "expected"),
+ [
+ (
+ SDK_CANCEL_ERROR_MESSAGE,
+ {"status": "cancelled", "error": SDK_CANCEL_ERROR_MESSAGE},
+ ),
+ ("task execution cancelled", {"error": "task execution cancelled"}),
+ (None, {"error": None}),
+ ],
+)
+def test_result_fact_names_only_a_cancellation(error_message, expected) -> None:
+ assert sdk_cancel_result_fact(error_message) == expected
+
+
[email protected]("storage_version", [1, 2])
+async def test_running_cancel_is_settled_by_the_cancelled_run(
+ monkeypatch: pytest.MonkeyPatch, storage_version: int
+) -> None:
+ """The cancelled run's own settlement writes the row; the core adds nothing.
+
+ A replay of the same command afterwards finds its target settled and
+ leaves the row, the transcript and the facts as they are.
+ """
+ task_id, agent_id, owner_user_id = _create_task(
+ run_id=None, state_version=0, storage_version=storage_version
+ )
+ manager = _broadcast_manager(monkeypatch)
+ await _start_blocked_run(
+ monkeypatch, task_id=task_id, owner_user_id=owner_user_id, source="sdk"
+ )
+ before = _load_task(task_id)
+ assert before.status == TaskStatus.RUNNING
+
+ try:
+ for _attempt in range(2):
+ await cancel_sdk_task_unserialized(
+ task_id=task_id,
+ agent_id=agent_id,
+ expected_run_id=before.run_id,
+ expected_state_version=before.state_version,
+ )
+ _assert_cancelled(task_id, before=before, storage_version=storage_version)
+ finally:
+ task_execution_service.background_task_manager.running_tasks.pop(task_id, None)
+ # One terminal announcement per call; a version-2 task also replays the
+ # turn's display events ahead of each.
+ announcements = [
+ call.args[0]
+ for call in manager.broadcast_to_task.await_args_list
+ if call.args[0].get("type") == "task_error"
+ ]
+ assert len(announcements) == 2
+
+
[email protected]("storage_version", [1, 2])
+async def test_running_cancel_is_finalized_by_the_core_without_a_local_run(
+ monkeypatch: pytest.MonkeyPatch, storage_version: int
+) -> None:
+ """With no coroutine here to settle the run, the core writes the same row."""
+ task_id, agent_id, owner_user_id = _create_task(
+ output="stale output", storage_version=storage_version
+ )
+ _seed_pending_user_message(task_id, owner_user_id, "turn-running")
+ _broadcast_manager(monkeypatch)
+ synced: list[str] = []
+ monkeypatch.setattr(
+ "xagent.web.services.workforce_runtime.sync_workforce_run_status",
+ lambda _db, _task, status: synced.append(f"workforce:{status.value}"),
+ )
+ monkeypatch.setattr(
+ task_orchestrator,
+ "sync_trigger_run_status",
+ lambda _db, _task, status: synced.append(f"trigger:{status.value}"),
+ )
+ before = _load_task(task_id)
+
+ await cancel_sdk_task_unserialized(
+ task_id=task_id,
+ agent_id=agent_id,
+ expected_run_id=before.run_id,
+ expected_state_version=before.state_version,
+ )
+
+ _assert_cancelled(task_id, before=before, storage_version=storage_version)
+ assert _delivery_status(task_id, "turn-running") == DELIVERY_DISPATCHED
+ assert synced == ["workforce:failed", "trigger:failed"]
+
+
+async def test_a_run_registered_after_the_first_cancel_is_stopped_after_the_write(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ """The core wrote the row because no coroutine was here to settle it, so
+ one that registered meanwhile is cancelled once the row is terminal."""
+ task_id, agent_id, _owner = _create_task()
+ _broadcast_manager(monkeypatch)
+ statuses: list[TaskStatus] = []
+
+ async def record_status(_task_id: int, **_kwargs: object) -> SimpleNamespace:
+ statuses.append(_load_task(task_id).status)
+ return SimpleNamespace(requested=False)
+
+ monkeypatch.setattr(
+ task_execution_service.background_task_manager, "cancel_task", record_status
+ )
+
+ await cancel_sdk_task_unserialized(
+ task_id=task_id,
+ agent_id=agent_id,
+ expected_run_id="run-sdk",
+ expected_state_version=4,
+ )
+
+ assert statuses == [TaskStatus.RUNNING, TaskStatus.FAILED]
+
+
+ "moved",
+ [
+ {"status": TaskStatus.FAILED, "error_message": "boom", "state_version": 5},
+ {"status": TaskStatus.COMPLETED, "output": "done", "state_version": 5},
+ {"status": TaskStatus.PAUSED, "state_version": 5},
+ {"run_id": "run-next"},
+ {"state_version": 6},
+ {"status": TaskStatus.WAITING_FOR_USER},
+ ],
+ ids=["failed_itself", "completed", "paused", "new_run", "moved", "waiting"],
+)
+async def test_cancel_never_claims_an_outcome_it_did_not_cause(
+ monkeypatch: pytest.MonkeyPatch, moved: dict[str, Any]
+) -> None:
+ """A target that ended or moved on its own is rejected and left as it is."""
+ task_id, agent_id, _owner = _create_task()
+ db = _direct_db_session()
+ try:
+ db.query(Task).filter(Task.id == task_id).update(moved)
+ db.commit()
+ finally:
+ db.close()
+ manager = _broadcast_manager(monkeypatch)
+ before = _row(task_id)
+
+ with pytest.raises(TaskCommandRejected) as raised:
+ await cancel_sdk_task_unserialized(
+ task_id=task_id,
+ agent_id=agent_id,
+ expected_run_id="run-sdk",
+ expected_state_version=4,
+ )
+
+ assert raised.value.reason == "stale_run"
+ assert _row(task_id) == before
+ assert _assistant_rows(task_id) == []
+ assert manager.broadcast_to_task.await_count == 0
+
+
[email protected]("source", ["a2a", "external", "internal", "trigger"])
+async def test_cancel_reaches_only_sdk_tasks(
+ monkeypatch: pytest.MonkeyPatch, source: str
+) -> None:
+ task_id, agent_id, _owner = _create_task(source=source)
+ _broadcast_manager(monkeypatch)
+ before = _row(task_id)
+
+ with pytest.raises(TaskCommandRejected) as raised:
+ await cancel_sdk_task_unserialized(
+ task_id=task_id,
+ agent_id=agent_id,
+ expected_run_id="run-sdk",
+ expected_state_version=4,
+ )
+
+ assert raised.value.reason == "task_not_found"
+ assert _row(task_id) == before
+
+
+async def test_finalize_accepts_a_settlement_that_lands_mid_cancel(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ """A stale finalize is rechecked once, and a settled cancel then matches."""
+ task_id, agent_id, _owner = _create_task()
+ _broadcast_manager(monkeypatch)
+ real_finalize = sdk_task_cancel._finalize_sdk_cancel_sync
+ calls = 0
+
+ def settle_then_report_stale(**kwargs: Any) -> bool:
+ nonlocal calls
+ calls += 1
+ if calls == 1:
+ db = _direct_db_session()
+ try:
+ db.query(Task).filter(Task.id == task_id).update(
+ {
+ "status": TaskStatus.FAILED,
+ "control_state": TaskControlState.FAILED.value,
+ "state_version": 5,
+ "error_message": SDK_CANCEL_ERROR_MESSAGE,
+ }
+ )
+ db.commit()
+ finally:
+ db.close()
+ raise StaleTaskRunError("settled between the read and the update")
+ return real_finalize(**kwargs)
+
+ monkeypatch.setattr(
+ sdk_task_cancel, "_finalize_sdk_cancel_sync", settle_then_report_stale
+ )
+
+ await cancel_sdk_task_unserialized(
+ task_id=task_id,
+ agent_id=agent_id,
+ expected_run_id="run-sdk",
+ expected_state_version=4,
+ )
+
+ assert calls == 2
+ assert _load_task(task_id).state_version == 5
+ assert _assistant_rows(task_id) == []
+
+
[email protected]("fence", ["lock", "update"])
+def test_replaced_attempt_rejects_cancel_and_preserves_task(
+ monkeypatch: pytest.MonkeyPatch, fence: str
+) -> None:
+ """An old cancel command cannot settle a replacement lease attempt."""
+ task_id, agent_id, _owner = _create_task()
+ db = _direct_db_session()
+ try:
+ lease = task_coordinator_service.acquire_task_lease_no_commit(
+ db, task_id, runner_id="worker"
+ )
+ db.commit()
+ db.query(Task).filter(Task.id == task_id).update(
+ {"lease_attempt_id": "replacement-attempt"}
+ )
+ db.commit()
+ finally:
+ db.close()
+ monkeypatch.setattr(
+ task_coordinator_runtime,
+ "current_task_coordinator",
+ lambda _task_id: SimpleNamespace(lease=lease, task_id=task_id),
+ )
+ if fence == "update":
+ # Isolate the UPDATE predicate from the lock check ahead of it.
+ monkeypatch.setattr(
+ task_coordinator_service, "lock_task_lease_no_commit", lambda *_a: True
+ )
+ before = _row(task_id)
+
+ with pytest.raises(StaleTaskRunError):
+ sdk_task_cancel._finalize_sdk_cancel_sync(
+ task_id=task_id,
+ agent_id=agent_id,
+ expected_run_id=before["run_id"],
+ expected_state_version=before["state_version"],
+ terminal_event_state={},
+ )
+
+ assert _row(task_id) == before
+ assert _load_task(task_id).lease_attempt_id == "replacement-attempt"
+
+
[email protected]("delivery_fails", [False, True])
+async def test_cancel_announces_after_commit_and_survives_a_failed_broadcast(
+ monkeypatch: pytest.MonkeyPatch, delivery_fails: bool
+) -> None:
+ task_id, agent_id, _owner = _create_task()
+ manager = _broadcast_manager(monkeypatch)
+ seen: list[TaskStatus] = []
+
+ async def record_committed_status(*_args: object, **_kwargs: object) -> None:
+ seen.append(_load_task(task_id).status)
+ if delivery_fails:
+ raise RuntimeError("socket gone")
+
+ manager.broadcast_to_task.side_effect = record_committed_status
+
+ await cancel_sdk_task_unserialized(
+ task_id=task_id,
+ agent_id=agent_id,
+ expected_run_id="run-sdk",
+ expected_state_version=4,
+ )
+
+ assert seen == [TaskStatus.FAILED]
+ assert _load_task(task_id).error_message == SDK_CANCEL_ERROR_MESSAGE
+
+
+async def test_the_cache_is_invalidated_once_after_the_commit_and_off_the_event_loop(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ """The invalidation is a blocking round trip with some cache backends, so
+ it belongs to the finalize that runs in a worker thread."""
+ task_id, agent_id, _owner = _create_task()
+ _broadcast_manager(monkeypatch)
+ calls: list[tuple[int, bool, TaskStatus]] = []
+
+ def record(invalidated_task_id: int) -> None:
+ try:
+ asyncio.get_running_loop()
+ on_event_loop = True
+ except RuntimeError:
+ on_event_loop = False
+ calls.append((invalidated_task_id, on_event_loop, _load_task(task_id).status))
+
+ monkeypatch.setattr(task_orchestrator, "invalidate_task_cache_best_effort", record)
+
+ await cancel_sdk_task_unserialized(
+ task_id=task_id,
+ agent_id=agent_id,
+ expected_run_id="run-sdk",
+ expected_state_version=4,
+ )
+
+ assert calls == [(task_id, False, TaskStatus.FAILED)]
+
+
+def _terminal_frame(manager: MagicMock) -> dict[str, Any]:
+ frames = [call.args[0] for call in manager.broadcast_to_task.await_args_list]
+ return next(frame for frame in frames if frame.get("type") == "task_error")
+
+
[email protected]("storage_version", [1, 2])
[email protected]("writer", ["core", "run_settlement"])
+async def test_the_terminal_frame_carries_the_control_snapshot_of_a_v2_task(
+ monkeypatch: pytest.MonkeyPatch, writer: str, storage_version: int
+) -> None:
+ """Without the snapshot, readers of a version-2 task's live frames never
+ get the turn's last display events. Whichever writer lands the terminal
+ row, the frame carries the state it left, and a version-1 task has none."""
+ task_id, agent_id, owner_user_id = _create_task(
+ run_id=None, state_version=0, storage_version=storage_version
+ )
+ _seed_pending_user_message(task_id, owner_user_id, "turn-running")
+ manager = _broadcast_manager(monkeypatch)
+ if writer == "run_settlement":
+ await _start_blocked_run(
+ monkeypatch, task_id=task_id, owner_user_id=owner_user_id, source="sdk"
+ )
+ before = _load_task(task_id)
+
+ try:
+ await cancel_sdk_task_unserialized(
+ task_id=task_id,
+ agent_id=agent_id,
+ expected_run_id=before.run_id,
+ expected_state_version=before.state_version,
+ )
+ finally:
+ task_execution_service.background_task_manager.running_tasks.pop(task_id, None)
+
+ frame = _terminal_frame(manager)
+ if storage_version == 2:
+ assert frame["state_version"] == before.state_version + 1
+ assert frame["task"]["status"] == "failed"
+ frames = [call.args[0] for call in manager.broadcast_to_task.await_args_list]
+ assert frames.index(frame) > 0, "display events precede the terminal frame"
+ else:
+ assert "state_version" not in frame
+
+
+async def _start_resume(
+ monkeypatch: pytest.MonkeyPatch,
+ *,
+ task_id: int,
+ owner_user_id: int,
+ run_id: str,
+ phase: str,
+ source: str = "sdk",
+) -> asyncio.Task:
+ """A real resume coordinator parked in ``phase``.
+
+ It is registered the way the resume handoff registers it, in
+ ``resume_tasks``, so ``cancel_task`` reaches it. ``execution`` parks it
+ inside the agent's resumed execution; ``lease_acquisition`` parks it in
+ the worker thread that claims the run's lease, which then claims it.
+ """
+ started = asyncio.Event()
+ loop = asyncio.get_running_loop()
+ real_acquire = task_execution_service._acquire_resume_task_lease
+
+ def acquire_slowly(*args: Any, **kwargs: Any) -> Any:
+ loop.call_soon_threadsafe(started.set)
+ time.sleep(0.3)
+ return real_acquire(*args, **kwargs)
+
+ if phase == "lease_acquisition":
+ monkeypatch.setattr(
+ task_execution_service, "_acquire_resume_task_lease", acquire_slowly
+ )
+
+ async def block_until_cancelled(*_args: object, **_kwargs: object) -> None:
+ started.set()
+ await asyncio.Event().wait()
+
+ agent_service = MagicMock()
+ agent_service.resume_execution_by_id = AsyncMock(side_effect=block_until_cancelled)
+ resume = asyncio.create_task(
+ task_execution_service.execute_resume_background(
+ task_id,
+ agent_service,
+ owner_user_id,
+ expected_run_id=run_id,
+ trusted_task_source=source,
+ )
+ )
+ task_execution_service.background_task_manager.resume_tasks[task_id] = resume
+ await asyncio.wait_for(started.wait(), timeout=DB_PROGRESS_TIMEOUT)
+ return resume
+
+
+ ("phase", "explicit", "expected"),
+ [
+ ("execution", True, SDK_CANCEL_ERROR_MESSAGE),
+ ("execution", False, "resume execution cancelled"),
+ ("lease_acquisition", True, SDK_CANCEL_ERROR_MESSAGE),
+ ("lease_acquisition", False, "resume cancelled during lease acquisition"),
+ ],
+ ids=[
+ "execution-explicit_cancel",
+ "execution-direct_cancel",
+ "lease_acquisition-explicit_cancel",
+ "lease_acquisition-direct_cancel",
+ ],
+)
+async def test_a_cancelled_resume_settles_as_cancelled_only_when_asked(
+ monkeypatch: pytest.MonkeyPatch, phase: str, explicit: bool, expected: str
+) -> None:
+ """A resumed SDK run takes the same two ways out as a first run: an
+ explicit ``cancel_task`` settles it as cancelled, any other cancellation
+ keeps the wording it has always had."""
+ task_id, _agent_id, owner_user_id = _create_task(
+ status=TaskStatus.WAITING_FOR_USER,
+ control_state=TaskControlState.WAITING_FOR_USER.value,
+ )
+ _broadcast_manager(monkeypatch)
+ resume = await _start_resume(
+ monkeypatch,
+ task_id=task_id,
+ owner_user_id=owner_user_id,
+ run_id="run-sdk",
+ phase=phase,
+ )
+ manager = task_execution_service.background_task_manager
+ try:
+ if explicit:
+ await manager.cancel_task(task_id, timeout_seconds=DB_PROGRESS_TIMEOUT)
+ else:
+ resume.cancel()
+ with pytest.raises(asyncio.CancelledError):
+ await resume
+ finally:
+ manager.running_tasks.pop(task_id, None)
+ manager.resume_tasks.pop(task_id, None)
+
+ row = _load_task(task_id)
+ assert row.status == TaskStatus.FAILED
+ assert row.error_message == expected
+
+
+def _cancel_command(task_id: int, agent_id: int, **payload: Any) -> ClaimedTaskCommand:
+ return ClaimedTaskCommand(
+ id=5,
+ task_id=task_id,
+ actor_user_id=None,
+ command_id=f"cancel:{task_id}:4",
+ kind=TaskCommandKind.CANCEL,
+ payload={"agent_id": agent_id, "target_state_version": 4, **payload},
+ target_run_id="run-sdk",
+ attempt_count=1,
+ )
+
+
+async def test_sdk_scope_routes_to_the_sdk_core_only(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ task_id, agent_id, _owner = _create_task()
+ _broadcast_manager(monkeypatch)
+ sdk_core = AsyncMock()
+ external_core = AsyncMock()
+ a2a_core = AsyncMock()
+ monkeypatch.setattr(
+ command_execution_service, "cancel_sdk_task_unserialized", sdk_core
+ )
+ monkeypatch.setattr(
+ command_execution_service, "cancel_external_task_unserialized", external_core
+ )
+ monkeypatch.setattr(a2a_cancel_service, "cancel_a2a_task", a2a_core)
+
+ await command_execution_service._execute_durable_task_command(
+ _cancel_command(task_id, agent_id, scope="sdk")
+ )
+
+ sdk_core.assert_awaited_once_with(
+ task_id=task_id,
+ agent_id=agent_id,
+ expected_run_id="run-sdk",
+ expected_state_version=4,
+ )
+ assert external_core.await_count == 0
+ assert a2a_core.await_count == 0
+
+
[email protected]("scope", [None, "external"], ids=["a2a", "external"])
+async def test_other_scopes_never_reach_the_sdk_core(
+ monkeypatch: pytest.MonkeyPatch, scope: str | None
+) -> None:
+ task_id, agent_id, _owner = _create_task()
+ _broadcast_manager(monkeypatch)
+ sdk_core = AsyncMock()
+ monkeypatch.setattr(
+ command_execution_service, "cancel_sdk_task_unserialized", sdk_core
+ )
+ monkeypatch.setattr(
+ command_execution_service, "cancel_external_task_unserialized", AsyncMock()
+ )
+ monkeypatch.setattr(a2a_cancel_service, "cancel_a2a_task", AsyncMock())
+ payload = {} if scope is None else {"scope": scope}
+
+ await command_execution_service._execute_durable_task_command(
+ _cancel_command(task_id, agent_id, **payload)
+ )
+
+ assert sdk_core.await_count == 0
+
+
+def test_lease_fields_are_cleared_only_without_a_coordinator(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ """Shared execution leaves ownership to the coordinator that is draining."""
+ task_id, agent_id, _owner = _create_task()
+ db = _direct_db_session()
+ try:
+ lease = task_coordinator_service.acquire_task_lease_no_commit(
+ db, task_id, runner_id="worker"
+ )
+ db.commit()
+ finally:
+ db.close()
+ monkeypatch.setattr(
+ task_coordinator_runtime,
+ "current_task_coordinator",
+ lambda _task_id: SimpleNamespace(lease=lease, task_id=task_id),
+ )
+ before = _load_task(task_id)
+
+ assert sdk_task_cancel._finalize_sdk_cancel_sync(
+ task_id=task_id,
+ agent_id=agent_id,
+ expected_run_id=before.run_id,
+ expected_state_version=before.state_version,
+ terminal_event_state={},
+ )
+
+ after = _load_task(task_id)
+ assert after.error_message == SDK_CANCEL_ERROR_MESSAGE
+ assert after.runner_id == "worker"
+ assert after.lease_attempt_id == lease.attempt_id
+ assert after.lease_expires_at is not None
+ assert after.lease_expires_at.replace(tzinfo=UTC) > datetime.now(UTC) - timedelta(
+ minutes=5
+ )
diff --git a/tests/web/api/v1/test_events_stream.py b/tests/web/api/v1/test_events_stream.py
index 0e535afd8..03afa5e1d 100644
--- a/tests/web/api/v1/test_events_stream.py
+++ b/tests/web/api/v1/test_events_stream.py
@@ -100,10 +100,15 @@ def mock_start_task():
yield mocked
-# The raw structured output values of a snapshot that has none, and the fields
-# a conclusion frame then carries.
-NO_STRUCTURED_OUTPUT = {"structured_output_record": None, "format_configured": False}
+# The raw structured output values of a snapshot that has none and was not
+# cancelled, and the fields a conclusion frame then carries.
+NO_STRUCTURED_OUTPUT = {
+ "structured_output_record": None,
+ "format_configured": False,
+ "error_code": None,
+}
NULL_STRUCTURED_OUTPUT_FIELDS = {
+ "error_code": None,
"structured_output": None,
"structured_output_status": None,
"structured_output_reason": None,
@@ -947,7 +952,7 @@ async def test_watchdog_terminal_task_row_closes_with_completed():
)
assert closed is True
assert sink.queue.get_nowait() == (
- es.completed_frame(status="failed", output=None, error="boom"),
+ es.completed_frame(status="failed", output=None, error="boom", error_code=None),
True,
)
@@ -1437,7 +1442,9 @@ async def test_close_frame_delivered_when_enqueued_between_two_yields():
# awaiting the socket write while other tasks run.
won = sink.enqueue_close(
- es.completed_frame(status="completed", output="done", error=None)
+ es.completed_frame(
+ status="completed", output="done", error=None, error_code=None
+ )
)
assert won is True
diff --git a/tests/web/api/v1/test_expired_tasks.py b/tests/web/api/v1/test_expired_tasks.py
index 588715653..f2268b03f 100644
--- a/tests/web/api/v1/test_expired_tasks.py
+++ b/tests/web/api/v1/test_expired_tasks.py
@@ -197,6 +197,8 @@ def _call(route: str, task_id: int, full_key: str, owner: dict):
)
if route == "cancel":
return client.post(f"/v1/chat/tasks/{task_id}/cancel-question", headers=headers)
+ if route == "cancel_task":
+ return client.post(f"/v1/chat/tasks/{task_id}/cancel", headers=headers)
if route == "upload":
return client.post(
f"/v1/chat/files?task_id={task_id}",
@@ -206,7 +208,16 @@ def _call(route: str, task_id: int, full_key: str, owner: dict):
raise AssertionError(route)
-TASK_ROUTES = ["get", "steps", "events", "append", "reply", "cancel", "upload"]
+TASK_ROUTES = [
+ "get",
+ "steps",
+ "events",
+ "append",
+ "reply",
+ "cancel",
+ "cancel_task",
+ "upload",
+]
def _assert_expired(resp, task_id: int, expired_at: datetime = EXPIRED_AT) -> None:
diff --git a/tests/web/api/v1/test_personal_key_principal_policy.py b/tests/web/api/v1/test_personal_key_principal_policy.py
index 39dfe396b..2fd0228dd 100644
--- a/tests/web/api/v1/test_personal_key_principal_policy.py
+++ b/tests/web/api/v1/test_personal_key_principal_policy.py
@@ -129,9 +129,9 @@ def test_personal_key_policy_runs_only_after_the_key_verifies() -> None:
@pytest.mark.usefixtures("_template_manager")
def test_personal_key_policy_applies_to_every_personal_key_route() -> None:
routes = _personal_key_routes()
- # The 22 management routes (me, agents, templates, skills, MCP connectors), the 11
+ # The 22 management routes (me, agents, templates, skills, MCP connectors), the 12
# /v1/chat task routes and the features route.
- assert len(routes) == 34, routes
+ assert len(routes) == 35, routes
headers = _bearer(_create_personal_key()[0])
for method, template in routes:
diff --git a/tests/web/api/v1/test_personal_key_tasks.py b/tests/web/api/v1/test_personal_key_tasks.py
index d907dc874..6a81cd7b2 100644
--- a/tests/web/api/v1/test_personal_key_tasks.py
+++ b/tests/web/api/v1/test_personal_key_tasks.py
@@ -264,6 +264,8 @@ def _call(route: str, task_id: int, full_key: str, owner: dict):
)
if route == "cancel":
return client.post(f"/v1/chat/tasks/{task_id}/cancel-question", headers=headers)
+ if route == "cancel_task":
+ return client.post(f"/v1/chat/tasks/{task_id}/cancel", headers=headers)
if route == "delete":
return client.delete(f"/v1/chat/tasks/{task_id}", headers=headers)
raise AssertionError(route)
@@ -388,8 +390,16 @@ def test_credential_kind_column_mapping(world) -> None:
# The write routes that start work on the task re-check that the holder can
-# still run its agent; reading, cancelling a question and deleting do not.
-ALLOWED = {"append": 202, "reply": 409, "cancel": 409, "delete": 200, "get": 200}
+# still run its agent; reading, cancelling a question, cancelling the task and
+# deleting do not.
+ALLOWED = {
+ "append": 202,
+ "reply": 409,
+ "cancel": 409,
+ "cancel_task": 409,
+ "delete": 200,
+ "get": 200,
+}
@pytest.mark.parametrize("reachable", [True, False], ids=["in_team", "removed"])
diff --git a/tests/web/api/v1/test_status_only_projection.py b/tests/web/api/v1/test_status_only_projection.py
index a675e28f3..2b62912ed 100644
--- a/tests/web/api/v1/test_status_only_projection.py
+++ b/tests/web/api/v1/test_status_only_projection.py
@@ -49,7 +49,7 @@ from xagent.web.models.database import get_db
from xagent.web.models.task import Task, TaskStatus
from xagent.web.models.task_auto_recovery import TaskAutoRecovery
from xagent.web.models.user import User
-from xagent.web.services import task_auto_resume, task_event_bridge
+from xagent.web.services import sdk_task_cancel, task_auto_resume, task_event_bridge
from xagent.web.services.hot_path_cache import (
InMemoryTTLCache,
cache_get,
@@ -294,7 +294,15 @@ def test_view_call_sites():
defining = pathlib.Path(task_auto_resume.__file__).resolve()
computed = [c for c in _calls("status_only_terminal_view") if c[0] != defining]
- assert computed == [(tasks_py, "_load_task_info_snapshot")]
+ # The read surface derives the failed status; the cancel request consults
+ # the same view so it refuses exactly the tasks that read as failed.
+ cancel_py = pathlib.Path(sdk_task_cancel.__file__).resolve()
+ assert sorted(computed) == sorted(
+ [
+ (tasks_py, "_load_task_info_snapshot"),
+ (cancel_py, "_ended_without_cancel"),
+ ]
+ )
overlaid = _calls("effective_task_snapshot")
assert overlaid, "the events endpoint must overlay its snapshots"
@@ -311,6 +319,7 @@ ALLOWLIST = {
("POST", "/v1/chat/tasks/{task_id}/messages"),
("POST", "/v1/chat/tasks/{task_id}/reply"),
("POST", "/v1/chat/tasks/{task_id}/cancel-question"),
+ ("POST", "/v1/chat/tasks/{task_id}/cancel"),
("POST", "/v1/workforces/{workforce_id}/runs"),
}
EXCLUDE = {
diff --git a/tests/web/api/v1/test_structured_output_events.py b/tests/web/api/v1/test_structured_output_events.py
index 1e1103646..590b4fa43 100644
--- a/tests/web/api/v1/test_structured_output_events.py
+++ b/tests/web/api/v1/test_structured_output_events.py
@@ -182,7 +182,9 @@ def test_the_stream_reads_the_record_with_the_row_in_one_select():
def test_a_frame_built_without_fields_carries_null_for_each():
- frame = es.completed_frame(status=TaskStatus.FAILED.value, output=None, error="x")
+ frame = es.completed_frame(
+ status=TaskStatus.FAILED.value, output=None, error="x", error_code=None
+ )
data = json.loads(frame.split("data: ", 1)[1])
assert [data[name] for name in NEW_KEYS] == [None, None, None]
diff --git a/tests/web/api/v1/test_structured_output_get.py b/tests/web/api/v1/test_structured_output_get.py
index c9f0566a3..7de5b4d0b 100644
--- a/tests/web/api/v1/test_structured_output_get.py
+++ b/tests/web/api/v1/test_structured_output_get.py
@@ -143,6 +143,8 @@ def test_the_existing_fields_are_byte_identical_to_the_recorded_baseline():
assert all(name in body for name in NEW_KEYS)
for name in NEW_KEYS:
del body[name]
+ # Added after the baseline was recorded; null for these tasks.
+ assert body.pop("error_code") is None
assert json.dumps(body, ensure_ascii=False, separators=(",", ":")) == expected
diff --git a/tests/web/api/v1/test_v1_cancel_task.py b/tests/web/api/v1/test_v1_cancel_task.py
new file mode 100644
index 000000000..88c1e25a6
--- /dev/null
+++ b/tests/web/api/v1/test_v1_cancel_task.py
@@ -0,0 +1,827 @@
+"""Integration tests for ``POST /v1/chat/tasks/{task_id}/cancel``.
+
+The route's contract: which task states it cancels, refuses for good, or
+asks the caller to retry; that a refusal writes nothing; and that the
+``error_code`` it reports is the one the task reads report.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import json
+import threading
+from types import SimpleNamespace
+from typing import Any
+from unittest.mock import AsyncMock, MagicMock, patch
+
+import pytest
+
+from xagent.web.models.task import Task, TaskStatus
+from xagent.web.models.task_command import TaskExecutionCommand
+from xagent.web.services import sdk_task_cancel, task_auto_resume
+from xagent.web.services import task_command_execution as command_execution_service
+from xagent.web.services.sdk_task_cancel import SDK_CANCEL_ERROR_MESSAGE
+from xagent.web.services.task_command_transport import (
+ COMMAND_COMPLETED,
+ TaskCommandKind,
+ TaskCommandRejected,
+ TaskCommandTaskMissing,
+ enqueue_task_command,
+)
+from xagent.web.services.task_execution_controller import TaskControlState
+
+from ..conftest import (
+ _admin_headers,
+ _direct_db_session,
+ _register_second_user,
+ client,
+)
+from .test_workforces import _create_active_workforce, _create_workforce_key
+
+pytestmark = pytest.mark.usefixtures("_test_db")
+
+_agent_name_counter = 0
+
+_ROW_COLUMNS = ("status", "control_state", "run_id", "state_version", "error_message")
+
+
+def _bearer(full_key: str) -> dict[str, str]:
+ return {"Authorization": f"Bearer {full_key}"}
+
+
[email protected](autouse=True)
+def no_background_run():
+ """Tasks are accepted and left RUNNING with no coroutine behind them."""
+ with patch(
+ "xagent.web.services.task_orchestrator._schedule_bg",
+ new=MagicMock(),
+ ):
+ yield
+
+
[email protected](autouse=True)
+def no_live_broadcast(monkeypatch: pytest.MonkeyPatch):
+ from xagent.web.api import websocket as websocket_api
+
+ manager = MagicMock()
+ manager.broadcast_to_task = AsyncMock()
+ monkeypatch.setattr(websocket_api, "manager", manager)
+
+
+def _create_agent_with_key(headers: dict[str, str] | None = None) -> tuple[int, str]:
+ global _agent_name_counter
+ _agent_name_counter += 1
+ headers = headers or _admin_headers()
+ agent_resp = client.post(
+ "/api/agents",
+ headers=headers,
+ json={
+ "name": f"v1 cancel test agent {_agent_name_counter}",
+ "description": "test",
+ "instructions": "you are a test agent",
+ "execution_mode": "balanced",
+ },
+ )
+ assert agent_resp.status_code == 200, agent_resp.text
+ agent_id = agent_resp.json()["id"]
+ key_resp = client.post(f"/api/agents/{agent_id}/api-key", headers=headers)
+ assert key_resp.status_code == 200, key_resp.text
+ return agent_id, key_resp.json()["full_key"]
+
+
+def _create_task(full_key: str, agent_id: int) -> int:
+ response = client.post(
+ "/v1/chat/tasks",
+ headers=_bearer(full_key),
+ json={"agent_id": agent_id, "message": {"role": "user", "content": "hello"}},
+ )
+ assert response.status_code == 202, response.text
+ return response.json()["task_id"]
+
+
+def _running_task() -> tuple[int, int, str]:
+ agent_id, key = _create_agent_with_key()
+ task_id = _create_task(key, agent_id)
+ assert _row(task_id)["status"] == TaskStatus.RUNNING
+ return task_id, agent_id, key
+
+
+def _update(task_id: int, **values: Any) -> None:
+ db = _direct_db_session()
+ try:
+ db.query(Task).filter(Task.id == task_id).update(values)
+ db.commit()
+ finally:
+ db.close()
+
+
+def _row(task_id: int) -> dict[str, Any]:
+ db = _direct_db_session()
+ try:
+ task = db.query(Task).filter(Task.id == task_id).one()
+ return {column: getattr(task, column) for column in _ROW_COLUMNS}
+ finally:
+ db.close()
+
+
+def _cancel_commands(task_id: int) -> list[TaskExecutionCommand]:
+ db = _direct_db_session()
+ try:
+ return (
+ db.query(TaskExecutionCommand)
+ .filter(
+ TaskExecutionCommand.task_id == task_id,
+ TaskExecutionCommand.kind == TaskCommandKind.CANCEL.value,
+ )
+ .all()
+ )
+ finally:
+ db.close()
+
+
+def _queue_start(task_id: int) -> None:
+ """Leave an accepted START that no executor has handed off yet."""
+ db = _direct_db_session()
+ try:
+ task = db.query(Task).filter(Task.id == task_id).one()
+ enqueue_task_command(
+ db,
+ task_id=task_id,
+ actor_user_id=int(task.user_id),
+ command_id="turn-queued",
+ kind=TaskCommandKind.START,
+ payload={},
+ )
+ finally:
+ db.close()
+
+
+def _cancel(task_id: int, key: str):
+ return client.post(f"/v1/chat/tasks/{task_id}/cancel", headers=_bearer(key))
+
+
+def _get(task_id: int, key: str) -> dict[str, Any]:
+ response = client.get(f"/v1/chat/tasks/{task_id}", headers=_bearer(key))
+ assert response.status_code == 200, response.text
+ return response.json()
+
+
+def _error(response) -> str:
+ return response.json()["error"]["code"]
+
+
+def _completed_frame(task_id: int, key: str) -> dict[str, Any]:
+ response = client.get(f"/v1/chat/tasks/{task_id}/events", headers=_bearer(key))
+ assert response.status_code == 200, response.text
+ blocks = [block for block in response.text.split("\n\n") if block.strip()]
+ for block in blocks:
+ lines = block.splitlines()
+ if "event: task.completed" in lines:
+ data = next(line for line in lines if line.startswith("data: "))
+ return json.loads(data[len("data: ") :])
+ raise AssertionError(response.text)
+
+
+def test_cancel_running_task_then_repeat_then_append():
+ """The whole caller-visible arc of one cancellation."""
+ task_id, agent_id, key = _running_task()
+ before = _row(task_id)
+
+ first = _cancel(task_id, key)
+
+ assert first.status_code == 200, first.text
+ assert first.json() == {
+ "task_id": task_id,
+ "status": "failed",
+ "run_id": before["run_id"],
+ "state_version": before["state_version"] + 1,
+ "cancelled_run_id": before["run_id"],
+ "error_code": "task_cancelled",
+ }
+ read = _get(task_id, key)
+ assert read["status"] == "failed"
+ assert read["error"] == SDK_CANCEL_ERROR_MESSAGE
+ assert read["error_code"] == "task_cancelled"
+ assert read["output"] is None
+ assert _completed_frame(task_id, key)["error_code"] == "task_cancelled"
+
+ # A repeat reports the same outcome and adds no second command.
+ again = _cancel(task_id, key)
+ assert again.status_code == 200, again.text
+ assert again.json() == first.json()
+ assert len(_cancel_commands(task_id)) == 1
+
+ # A cancelled task takes a new turn like any failed one, and the code
+ # goes with the run it described.
+ appended = client.post(
+ f"/v1/chat/tasks/{task_id}/messages",
+ headers=_bearer(key),
+ json={"agent_id": agent_id, "message": {"role": "user", "content": "again"}},
+ )
+ assert appended.status_code == 202, appended.text
+ read = _get(task_id, key)
+ assert read["status"] == "running"
+ assert read["error_code"] is None
+
+
+ ("values", "expected_code"),
+ [
+ (
+ {
+ "status": TaskStatus.COMPLETED,
+ "control_state": TaskControlState.COMPLETED.value,
+ "output": "done",
+ },
+ "task_not_cancellable",
+ ),
+ (
+ {
+ "status": TaskStatus.FAILED,
+ "control_state": TaskControlState.FAILED.value,
+ "error_message": "boom",
+ },
+ "task_not_cancellable",
+ ),
+ (
+ {"status": TaskStatus.PENDING, "control_state": "idle"},
+ "task_busy",
+ ),
+ (
+ {
+ "status": TaskStatus.PAUSED,
+ "control_state": TaskControlState.PAUSED.value,
+ },
+ "task_busy",
+ ),
+ (
+ {
+ "status": TaskStatus.WAITING_FOR_USER,
+ "control_state": TaskControlState.WAITING_FOR_USER.value,
+ },
+ "task_busy",
+ ),
+ ],
+ ids=["completed", "failed_itself", "pending", "paused", "waiting"],
+)
+def test_states_that_are_not_running_are_refused_without_a_write(values, expected_code):
+ task_id, _agent_id, key = _running_task()
+ _update(task_id, runner_id=None, lease_expires_at=None, **values)
+ before = _row(task_id)
+
+ response = _cancel(task_id, key)
+
+ assert response.status_code == 409, response.text
+ assert _error(response) == expected_code
+ assert _row(task_id) == before
+ assert _cancel_commands(task_id) == []
+
+
+ "previous",
+ [
+ {
+ "status": TaskStatus.FAILED,
+ "control_state": TaskControlState.FAILED.value,
+ "error_message": SDK_CANCEL_ERROR_MESSAGE,
+ },
+ {
+ "status": TaskStatus.COMPLETED,
+ "control_state": TaskControlState.COMPLETED.value,
+ "output": "done",
+ },
+ ],
+ ids=["previous_run_cancelled", "previous_run_completed"],
+)
+def test_a_queued_turn_is_never_answered_from_the_previous_run(previous):
+ """While an appended START is queued the row still shows the run before
+ it, so answering from the row would report on the wrong turn: a
+ cancelled previous run must not read as "already cancelled"."""
+ task_id, _agent_id, key = _running_task()
+ _update(task_id, runner_id=None, lease_expires_at=None, **previous)
+ _queue_start(task_id)
+ before = _row(task_id)
+
+ response = _cancel(task_id, key)
+
+ assert response.status_code == 409, response.text
+ assert _error(response) == "task_busy"
+ assert _row(task_id) == before
+ assert _cancel_commands(task_id) == []
+ # The read surface agrees: the queued turn is pending and carries no code.
+ read = _get(task_id, key)
+ assert read["status"] == "pending"
+ assert read["error_code"] is None
+
+
+def test_a_task_that_reads_failed_after_automatic_resume_gave_up_is_not_cancellable(
+ monkeypatch: pytest.MonkeyPatch,
+):
+ task_id, _agent_id, key = _running_task()
+ _update(
+ task_id,
+ status=TaskStatus.PAUSED,
+ control_state=TaskControlState.PAUSED.value,
+ runner_id=None,
+ lease_expires_at=None,
+ )
+ monkeypatch.setattr(
+ task_auto_resume,
+ "status_only_terminal_view",
+ lambda _db, _task_id: SimpleNamespace(state="exhausted"),
+ )
+ before = _row(task_id)
+
+ response = _cancel(task_id, key)
+
+ assert response.status_code == 409, response.text
+ assert _error(response) == "task_not_cancellable"
+ assert _row(task_id) == before
+ assert _cancel_commands(task_id) == []
+
+
[email protected]("case", ["other_agent", "other_user", "missing", "non_sdk"])
+def test_tasks_outside_the_key_scope_are_404_and_untouched(case):
+ task_id, _agent_id, key = _running_task()
+ target = task_id
+ if case == "other_agent":
+ _other_agent, key = _create_agent_with_key()
+ elif case == "other_user":
+ headers = _register_second_user("mallory", "mallorypass1")
+ _other_agent, key = _create_agent_with_key(headers)
+ elif case == "missing":
+ target = task_id + 10_000
+ else:
+ _update(task_id, source="internal")
+ before = _row(task_id)
+
+ response = _cancel(target, key)
+
+ assert response.status_code == 404, response.text
+ assert _error(response) == "task_not_found"
+ assert _row(task_id) == before
+ assert _cancel_commands(task_id) == []
+
+
+def test_workforce_key_cancels_its_own_run():
+ headers = _admin_headers()
+ workforce_id = _create_active_workforce(headers, name="Cancel WF")
+ key = _create_workforce_key(headers, workforce_id)
+ run = client.post(
+ f"/v1/workforces/{workforce_id}/runs",
+ headers=_bearer(key),
+ json={"message": {"role": "user", "content": "go"}},
+ )
+ assert run.status_code == 202, run.text
+ task_id = run.json()["task_id"]
+
+ response = _cancel(task_id, key)
+
+ assert response.status_code == 200, response.text
+ assert response.json()["error_code"] == "task_cancelled"
+ assert _get(task_id, key)["error_code"] == "task_cancelled"
+
+
+def test_a_failure_that_was_not_a_cancellation_carries_no_code():
+ task_id, _agent_id, key = _running_task()
+ _update(
+ task_id,
+ status=TaskStatus.FAILED,
+ control_state=TaskControlState.FAILED.value,
+ error_message="task execution cancelled",
+ runner_id=None,
+ lease_expires_at=None,
+ )
+
+ assert _get(task_id, key)["error_code"] is None
+ assert _completed_frame(task_id, key)["error_code"] is None
+
+
+def test_an_unapplied_cancel_is_accepted_and_repeats_reach_the_same_command(
+ monkeypatch: pytest.MonkeyPatch,
+):
+ task_id, _agent_id, key = _running_task()
+ monkeypatch.setattr(sdk_task_cancel, "dispatch_one_task_command", AsyncMock())
+ monkeypatch.setattr(sdk_task_cancel, "SDK_CANCEL_REQUEST_WAIT_SECONDS", 0.1)
+ before = _row(task_id)
+
+ first = _cancel(task_id, key)
+ second = _cancel(task_id, key)
+
+ for response in (first, second):
+ assert response.status_code == 202, response.text
+ assert response.json() == {
+ "task_id": task_id,
+ "status": "running",
+ "run_id": before["run_id"],
+ "state_version": before["state_version"],
+ "cancelled_run_id": None,
+ "error_code": None,
+ }
+ assert _row(task_id) == before
+ assert len(_cancel_commands(task_id)) == 1
+
+
+ ("reason", "status_code", "code"),
+ [
+ ("stale_run", 409, "task_busy"),
+ ("unsupported_scope", 503, "internal_error"),
+ (None, 503, "internal_error"),
+ ],
+)
+def test_a_rejected_command_tells_the_caller_whether_to_retry(
+ monkeypatch: pytest.MonkeyPatch, reason, status_code, code
+):
+ task_id, _agent_id, key = _running_task()
+ before = _row(task_id)
+ rejecting = AsyncMock(side_effect=TaskCommandRejected("rejected", reason=reason))
+ monkeypatch.setattr(
+ command_execution_service, "cancel_sdk_task_unserialized", rejecting
+ )
+
+ response = _cancel(task_id, key)
+
+ assert response.status_code == status_code, response.text
+ assert _error(response) == code
+ # The command row's own error text stays on the server.
+ assert "rejected" not in response.text
+ assert _row(task_id) == before
+
+ # The failed command is put back in the queue by the next call, which
+ # then applies it.
+ monkeypatch.undo()
+ retried = _cancel(task_id, key)
+ assert retried.status_code == 200, retried.text
+ assert len(_cancel_commands(task_id)) == 1
+
+
+def test_a_task_deleted_before_the_command_is_stored_is_404(
+ monkeypatch: pytest.MonkeyPatch,
+):
+ task_id, _agent_id, key = _running_task()
+ monkeypatch.setattr(
+ sdk_task_cancel,
+ "enqueue_task_command",
+ MagicMock(side_effect=TaskCommandTaskMissing(f"Task {task_id} not found")),
+ )
+
+ response = _cancel(task_id, key)
+
+ assert response.status_code == 404, response.text
+ assert _error(response) == "task_not_found"
+
+
+_ENDED_ROWS = {
+ "completed": {
+ "status": TaskStatus.COMPLETED,
+ "control_state": TaskControlState.COMPLETED.value,
+ "output": "done",
+ },
+ "failed_itself": {
+ "status": TaskStatus.FAILED,
+ "control_state": TaskControlState.FAILED.value,
+ "error_message": "boom",
+ },
+}
+
+
+def _cancelled_row_values() -> dict[str, Any]:
+ return {
+ "status": TaskStatus.FAILED,
+ "control_state": TaskControlState.FAILED.value,
+ "error_message": SDK_CANCEL_ERROR_MESSAGE,
+ }
+
+
+def _replace_cancel_core(monkeypatch: pytest.MonkeyPatch, core: Any) -> None:
+ monkeypatch.setattr(command_execution_service, "cancel_sdk_task_unserialized", core)
+
+
[email protected]("ended", list(_ENDED_ROWS))
[email protected]("exit_path", ["stale_run", "deadline"])
+def test_a_run_that_ended_on_its_own_is_never_reported_as_busy_or_accepted(
+ monkeypatch: pytest.MonkeyPatch, ended: str, exit_path: str
+):
+ """Whichever way the wait ends, a run that ended without a cancel
+ answers what the request would have answered on arrival."""
+ task_id, _agent_id, key = _running_task()
+ values = {**_ENDED_ROWS[ended], "runner_id": None, "lease_expires_at": None}
+ after_the_run_ended: list[dict[str, Any]] = []
+
+ def end_the_run() -> None:
+ _update(task_id, **values)
+ after_the_run_ended.append(_row(task_id))
+
+ if exit_path == "stale_run":
+
+ async def ended_before_it_was_applied(**_kwargs: Any) -> None:
+ end_the_run()
+ raise TaskCommandRejected("moved", reason="stale_run")
+
+ _replace_cancel_core(monkeypatch, ended_before_it_was_applied)
+ else:
+
+ async def ended_while_the_command_waits(*_args: Any, **_kwargs: Any) -> bool:
+ end_the_run()
+ return False
+
+ monkeypatch.setattr(
+ sdk_task_cancel,
+ "dispatch_one_task_command",
+ AsyncMock(side_effect=ended_while_the_command_waits),
+ )
+ monkeypatch.setattr(sdk_task_cancel, "SDK_CANCEL_REQUEST_WAIT_SECONDS", 0.1)
+
+ response = _cancel(task_id, key)
+
+ assert response.status_code == 409, response.text
+ assert _error(response) == "task_not_cancellable"
+ assert _row(task_id) == after_the_run_ended[0]
+ assert len(_cancel_commands(task_id)) == 1
+
+
+def test_a_run_that_ended_on_its_own_with_a_turn_queued_behind_it_is_not_cancellable(
+ monkeypatch: pytest.MonkeyPatch,
+):
+ """A turn queued after the aimed run ended does not hide that the aimed
+ run ended without a cancel."""
+ task_id, _agent_id, key = _running_task()
+
+ async def ended_then_a_turn_was_queued(**_kwargs: Any) -> None:
+ _update(
+ task_id,
+ **_ENDED_ROWS["completed"],
+ runner_id=None,
+ lease_expires_at=None,
+ )
+ _queue_start(task_id)
+ raise TaskCommandRejected("moved", reason="stale_run")
+
+ _replace_cancel_core(monkeypatch, ended_then_a_turn_was_queued)
+
+ response = _cancel(task_id, key)
+
+ assert response.status_code == 409, response.text
+ assert _error(response) == "task_not_cancellable"
+ assert len(_cancel_commands(task_id)) == 1
+
+
+def test_a_cancel_that_settled_as_paused_is_still_reported_busy(
+ monkeypatch: pytest.MonkeyPatch,
+):
+ task_id, _agent_id, key = _running_task()
+ after_pause: list[dict[str, Any]] = []
+
+ async def paused_before_it_was_applied(**_kwargs: Any) -> None:
+ _update(
+ task_id,
+ status=TaskStatus.PAUSED,
+ control_state=TaskControlState.PAUSED.value,
+ runner_id=None,
+ lease_expires_at=None,
+ )
+ after_pause.append(_row(task_id))
+ raise TaskCommandRejected("moved", reason="stale_run")
+
+ _replace_cancel_core(monkeypatch, paused_before_it_was_applied)
+
+ response = _cancel(task_id, key)
+
+ assert response.status_code == 409, response.text
+ assert _error(response) == "task_busy"
+ assert _row(task_id) == after_pause[0]
+
+
[email protected]("replacement_ended", [False, True], ids=["running", "ended"])
+def test_a_cancel_aimed_at_a_run_that_was_replaced_is_not_cancellable(
+ monkeypatch: pytest.MonkeyPatch, replacement_ended: bool
+):
+ """The aimed run is over and the row shows another one, whatever state
+ that one is in: retrying would cancel it, so the caller is told no retry
+ can cancel this run."""
+ task_id, _agent_id, key = _running_task()
+ after_replacement: list[dict[str, Any]] = []
+ replacement: dict[str, Any] = {"run_id": "run-next", "state_version": 50}
+ if replacement_ended:
+ replacement.update(_ENDED_ROWS["completed"], runner_id=None)
+
+ async def replaced_before_it_was_applied(**_kwargs: Any) -> None:
+ _update(task_id, **replacement)
+ after_replacement.append(_row(task_id))
+ raise TaskCommandRejected("moved", reason="stale_run")
+
+ _replace_cancel_core(monkeypatch, replaced_before_it_was_applied)
+
+ response = _cancel(task_id, key)
+
+ assert response.status_code == 409, response.text
+ assert _error(response) == "task_not_cancellable"
+ assert (
+ response.json()["error"]["message"]
+ == "The run this cancel was aimed at has already ended."
+ )
+ assert _row(task_id) == after_replacement[0]
+ assert len(_cancel_commands(task_id)) == 1
+
+
+def test_a_cancel_still_pending_when_its_run_was_replaced_is_not_cancellable(
+ monkeypatch: pytest.MonkeyPatch,
+):
+ """The wait ends with the command still queued and the row already on
+ another run: the aimed run is over, so the answer is not an accepted 202
+ that a retry would turn into a cancel of the new run."""
+ task_id, _agent_id, key = _running_task()
+
+ async def replaced_while_nothing_applies(*_args: Any, **_kwargs: Any) -> bool:
+ _update(task_id, run_id="run-next", state_version=50)
+ return False
+
+ monkeypatch.setattr(
+ sdk_task_cancel, "dispatch_one_task_command", replaced_while_nothing_applies
+ )
+ monkeypatch.setattr(sdk_task_cancel, "SDK_CANCEL_REQUEST_WAIT_SECONDS", 0.1)
+
+ response = _cancel(task_id, key)
+
+ assert response.status_code == 409, response.text
+ assert _error(response) == "task_not_cancellable"
+ assert (
+ response.json()["error"]["message"]
+ == "The run this cancel was aimed at has already ended."
+ )
+ assert _row(task_id)["run_id"] == "run-next"
+
+
+def test_a_turn_queued_while_the_cancel_was_applied_still_reports_the_cancel(
+ monkeypatch: pytest.MonkeyPatch,
+):
+ """The command ended, the task was cancelled by another request and a
+ new turn was queued before this request read the outcome: the run this
+ call aimed at is cancelled, so the answer says so."""
+ task_id, _agent_id, key = _running_task()
+ before = _row(task_id)
+
+ async def cancelled_elsewhere_then_appended(**_kwargs: Any) -> None:
+ _update(
+ task_id,
+ runner_id=None,
+ lease_expires_at=None,
+ **_cancelled_row_values(),
+ )
+ _queue_start(task_id)
+ raise TaskCommandRejected("moved", reason="stale_run")
+
+ _replace_cancel_core(monkeypatch, cancelled_elsewhere_then_appended)
+
+ response = _cancel(task_id, key)
+
+ assert response.status_code == 200, response.text
+ assert response.json() == {
+ "task_id": task_id,
+ "status": "failed",
+ "run_id": before["run_id"],
+ "state_version": before["state_version"],
+ "cancelled_run_id": before["run_id"],
+ "error_code": "task_cancelled",
+ }
+
+
+def test_a_completed_command_is_a_cancel_of_its_aimed_run_even_if_a_turn_is_queued(
+ monkeypatch: pytest.MonkeyPatch,
+):
+ task_id, _agent_id, key = _running_task()
+ before = _row(task_id)
+
+ async def cancelled_then_appended(**_kwargs: Any) -> None:
+ _update(
+ task_id,
+ state_version=before["state_version"] + 1,
+ runner_id=None,
+ lease_expires_at=None,
+ **_cancelled_row_values(),
+ )
+ _queue_start(task_id)
+
+ _replace_cancel_core(monkeypatch, cancelled_then_appended)
+
+ response = _cancel(task_id, key)
+
+ assert response.status_code == 200, response.text
+ assert response.json() == {
+ "task_id": task_id,
+ "status": "failed",
+ "run_id": before["run_id"],
+ "state_version": before["state_version"] + 1,
+ "cancelled_run_id": before["run_id"],
+ "error_code": "task_cancelled",
+ }
+
+
+def test_a_completed_command_names_its_own_run_when_a_new_run_already_runs(
+ monkeypatch: pytest.MonkeyPatch,
+):
+ """The answer describes the row as it is now; ``cancelled_run_id`` is the
+ run the command cancelled, and nothing of the new run is touched."""
+ task_id, _agent_id, key = _running_task()
+ before = _row(task_id)
+ after_the_new_run_started: list[dict[str, Any]] = []
+
+ async def cancelled_then_next_run_started(**_kwargs: Any) -> None:
+ _update(
+ task_id,
+ status=TaskStatus.RUNNING,
+ control_state=TaskControlState.RUNNING.value,
+ run_id="run-next",
+ state_version=before["state_version"] + 2,
+ error_message=None,
+ )
+ after_the_new_run_started.append(_row(task_id))
+
+ _replace_cancel_core(monkeypatch, cancelled_then_next_run_started)
+
+ response = _cancel(task_id, key)
+
+ assert response.status_code == 200, response.text
+ assert response.json() == {
+ "task_id": task_id,
+ "status": "running",
+ "run_id": "run-next",
+ "state_version": before["state_version"] + 2,
+ "cancelled_run_id": before["run_id"],
+ "error_code": None,
+ }
+ assert _row(task_id) == after_the_new_run_started[0]
+ assert len(_cancel_commands(task_id)) == 1
+
+
+def test_a_command_applied_by_another_process_while_polling_is_reported(
+ monkeypatch: pytest.MonkeyPatch,
+):
+ task_id, _agent_id, key = _running_task()
+ before = _row(task_id)
+
+ def apply_elsewhere() -> None:
+ _update(
+ task_id,
+ state_version=before["state_version"] + 1,
+ runner_id=None,
+ lease_expires_at=None,
+ **_cancelled_row_values(),
+ )
+ db = _direct_db_session()
+ try:
+ db.query(TaskExecutionCommand).filter(
+ TaskExecutionCommand.task_id == task_id,
+ TaskExecutionCommand.kind == TaskCommandKind.CANCEL.value,
+ ).update({"status": COMMAND_COMPLETED})
+ db.commit()
+ finally:
+ db.close()
+
+ async def left_to_the_owning_process(*_args: Any, **_kwargs: Any) -> bool:
+ threading.Timer(0.2, apply_elsewhere).start()
+ return False
+
+ monkeypatch.setattr(
+ sdk_task_cancel,
+ "dispatch_one_task_command",
+ AsyncMock(side_effect=left_to_the_owning_process),
+ )
+
+ response = _cancel(task_id, key)
+
+ assert response.status_code == 200, response.text
+ assert response.json()["cancelled_run_id"] == before["run_id"]
+ assert response.json()["error_code"] == "task_cancelled"
+
+
+def test_the_wait_is_timed_from_before_the_command_is_dispatched(
+ monkeypatch: pytest.MonkeyPatch,
+):
+ """A dispatch that outlasts the wait leaves nothing to wait for: the
+ first poll after it already finds the deadline passed."""
+ task_id, _agent_id, key = _running_task()
+ polls: list[int] = []
+ real_load = sdk_task_cancel.load_task_command
+
+ def counting_load(command_db_id: int):
+ polls.append(command_db_id)
+ return real_load(command_db_id)
+
+ async def slow_dispatch_that_applies_nothing(*_args: Any, **_kwargs: Any) -> bool:
+ await asyncio.sleep(0.4)
+ return False
+
+ monkeypatch.setattr(
+ sdk_task_cancel,
+ "dispatch_one_task_command",
+ AsyncMock(side_effect=slow_dispatch_that_applies_nothing),
+ )
+ monkeypatch.setattr(sdk_task_cancel, "load_task_command", counting_load)
+ monkeypatch.setattr(sdk_task_cancel, "SDK_CANCEL_REQUEST_WAIT_SECONDS", 0.1)
+
+ response = _cancel(task_id, key)
+
+ assert response.status_code == 202, response.text
+ assert len(polls) == 1