Skip to content

Enterprise agent platform for building, deploying, and operating AI teammates.

feat(v1): cancel a turn that is still queued to start (#3791)

Alex Liu00cb0355e0d0
diff --git a/docs/deployment.md b/docs/deployment.md
index 045d6b3da..8eb867f0c 100644
--- a/docs/deployment.md
+++ b/docs/deployment.md
@@ -1011,3 +1011,26 @@ A running task created through `/v1` can now be cancelled. The cancelled task re
### Rollback
- A task cancelled before the rollback stays `failed`; an older process does not report `error_code`, so that field is absent from its reads. The cancel endpoint answers 404 again.
+
+## 2026-11-13 — SDK tasks: cancelling a turn that is still queued to start
+
+`POST /v1/chat/tasks/{task_id}/cancel` now also cancels a turn whose start is still queued, instead of answering 409 `task_busy`.
+
+### Client-visible changes
+
+- A new task cancelled before it starts answers 200 and reads `failed` with `error_code` `task_cancelled` and no `run_id`.
+- A turn appended to a finished or paused task and cancelled before it starts is withdrawn: the call answers 200 with `cancelled_run_id` naming the withdrawn run, and the task keeps reading as the turn before it. Calling again answers the same 200. A task paused by an interruption whose appended turn was withdrawn reads `failed` on `GET`, as it does when that turn cannot be started, and the answer reads `failed` too; a task paused on request keeps reading `paused`.
+- When the task has moved since the turn was queued (another run, or a newer version of the same one), so the queued turn can no longer start, the cancel acts on the task as it is now: a running task has that running run cancelled, and the response names it as `cancelled_run_id`.
+- A task that is waiting for an answer or paused, with no turn queued to start, still answers 409 `task_busy`.
+- A cancel that reaches a queued turn after it started cancels it as a running task.
+- A turn queued before it has a run of its own still answers 409 `task_busy`.
+- While a later turn is queued after a completed or failed run, a 200 answer reads `status` `pending` with a null `error_code`, as `GET /v1/chat/tasks/{task_id}` does; `cancelled_run_id` still names the cancelled run. The earlier entry's 200 answered `failed` with `task_cancelled` in that case.
+
+### Deployment impact
+
+- No migration and no configuration.
+- Mixed versions: a cancel aimed at a queued turn names that turn's run, so an older process rejects it as a changed run without writing anything and the endpoint answers 409 `task_busy`, as before. Once the turn has started, a process that predates the cancel endpoint answers 503, as in the earlier entry.
+
+### Rollback
+
+- A queued turn answers 409 `task_busy` again. Turns already withdrawn stay withdrawn.
diff --git a/src/xagent/web/api/v1/tasks.py b/src/xagent/web/api/v1/tasks.py
index eba3e23c3..991f48430 100644
--- a/src/xagent/web/api/v1/tasks.py
+++ b/src/xagent/web/api/v1/tasks.py
@@ -107,7 +107,7 @@ from ...services.task_orchestrator import (
TaskTurnFileBindingError,
TaskTurnNotFoundError,
)
-from ...services.task_start_protocol import has_queued_start
+from ...services.task_start_protocol import QUEUED_START_STATUSES, has_queued_start
from ...services.uploaded_file_store import (
UploadedFileVersionSnapshot,
snapshot_uploaded_file_version,
@@ -586,7 +586,7 @@ class _TaskInfoSnapshot:
# :func:`effective_task_snapshot` and the GET response apply the view.
view: TerminalView | None = None
# Set when an appended turn's START is queued and the row still holds the
- # previous turn (see ``_QUEUED_START_STATUSES``). Applied like ``view``.
+ # previous turn (see ``QUEUED_START_STATUSES``). Applied like ``view``.
start_queued: bool = False
# Read from the same row as ``status`` and ``run_id``: the stored
# structured result of the run the row shows, and whether the task has a
@@ -605,15 +605,6 @@ class _TaskInfoSnapshot:
accept_partial: bool = False
-# Terminal statuses an append is accepted from. With shared execution the row
-# keeps one of them, and the previous turn's output and error, until the
-# queued START is handed off, so a caller waiting for a terminal status would
-# take the previous turn as the new one. A PAUSED row keeps reading as paused,
-# which callers already wait on. A RUNNING or WAITING_FOR_USER row has moved
-# since the START was queued, and a PENDING row already reads as pending.
-_QUEUED_START_STATUSES = (TaskStatus.COMPLETED, TaskStatus.FAILED)
-
-
def effective_task_snapshot(snapshot: _TaskInfoSnapshot) -> _TaskInfoSnapshot:
"""``snapshot`` as the caller must read it: ``pending`` with no output or
error while an appended turn's START is queued, ``failed`` when it carries
@@ -849,7 +840,7 @@ def _load_task_info_snapshot(
db, task
)
view: TerminalView | None = None
- start_queued = queued_before_row and task.status in _QUEUED_START_STATUSES
+ start_queued = queued_before_row and task.status in QUEUED_START_STATUSES
# The view re-checks both inside its own statement; this only skips
# the query for reads that cannot be terminal.
if (
@@ -1540,12 +1531,22 @@ async def cancel_chat_task(
to stop. 200 means the task is cancelled, including when an earlier
call already cancelled it; if a new turn has started since, ``status``
and ``run_id`` describe that turn and ``cancelled_run_id`` names the
- cancelled one. 202 means the cancel is recorded but not applied yet,
- and calling again reaches the same request.
-
- Only a running task is cancelled. One that is waiting for an answer,
- paused, or whose turn is still queued to start answers 409
- ``task_busy`` and is left untouched.
+ cancelled one; while a later turn is queued after a completed or failed
+ run the task reads ``pending`` with a null ``error_code``, as ``GET``
+ reads it. 202 means the cancel is recorded but not applied yet, and
+ calling again reaches the same request.
+
+ A turn still queued to start is cancelled too. For a new task that
+ leaves the task cancelled with no ``run_id``. For a turn appended to a
+ finished or paused task it only withdraws that turn: the task keeps
+ reading as the turn before it, and ``cancelled_run_id`` names the
+ withdrawn run. A queued turn that can no longer start, because the
+ task has moved since it was queued (another run, or a newer version of
+ the same one), is not the target: a running run is cancelled, and any
+ other state is answered as if no turn were queued.
+
+ A task that is waiting for an answer or paused, with no turn queued to
+ start, answers 409 ``task_busy`` and is left untouched.
Raises:
V1ApiError 401: missing / invalid / revoked key.
diff --git a/src/xagent/web/schemas/v1.py b/src/xagent/web/schemas/v1.py
index 4fd6e40c2..45daf489b 100644
--- a/src/xagent/web/schemas/v1.py
+++ b/src/xagent/web/schemas/v1.py
@@ -1244,9 +1244,14 @@ class CancelTaskResponse(BaseModel):
200: the run named by ``cancelled_run_id`` is cancelled, whether this
call did it or an earlier one already had. The other fields describe
- the task now: it reads ``failed`` with ``error_code='task_cancelled'``
- unless a new turn has started since, in which case ``status``,
- ``run_id``, ``state_version`` and ``error_code`` describe that turn.
+ the task now: ``failed`` with ``error_code='task_cancelled'``, unless a
+ new turn has started since, in which case ``status``, ``run_id``,
+ ``state_version`` and ``error_code`` describe that turn, or a later
+ turn is queued to start after a completed or failed run, in which case
+ it reads ``pending`` with a null ``error_code``, as
+ ``GET /v1/chat/tasks/{task_id}`` does. A turn
+ withdrawn before it started leaves the task showing the turn before
+ it, so ``cancelled_run_id`` differs from ``run_id``.
202: the cancel was recorded but not applied yet; ``status`` is still
the task's current one and ``cancelled_run_id`` is null. Calling again
reaches the same request.
@@ -1256,9 +1261,14 @@ class CancelTaskResponse(BaseModel):
status: str = Field(
...,
description=(
- "Task status as stored: 'failed' once cancelled, the status of a "
- "turn that started after the cancel, or, on 202, the status the "
- "cancel is still being applied to."
+ "Task status as stored: 'failed' once cancelled, the previous "
+ "turn's status when a queued turn was withdrawn before it started, "
+ "the status of a turn that started after the cancel, or, on 202, "
+ "the status the cancel is still being applied to. While a turn is "
+ "queued after a completed or failed run it reads 'pending', and a "
+ "task paused by an interruption that automatic resume will no "
+ "longer pick up (for example, after its appended turn was "
+ "withdrawn) reads 'failed', as GET /v1/chat/tasks/{task_id} does."
),
)
run_id: Optional[str] = Field(..., description="Current execution run identity.")
@@ -1268,15 +1278,18 @@ class CancelTaskResponse(BaseModel):
cancelled_run_id: Optional[str] = Field(
...,
description=(
- "The run that was cancelled. Null while the cancel has not been applied."
+ "The run that was cancelled. Differs from run_id when a queued "
+ "turn was withdrawn before it started: the task then still shows "
+ "the turn before it. Null while the cancel has not been applied."
),
)
error_code: Optional[str] = Field(
...,
description=(
"'task_cancelled' while the task reads as cancelled, otherwise "
- "null (before the cancel is applied, or once a later turn has "
- "started); the same value GET /v1/chat/tasks/{task_id} reports."
+ "null (before the cancel is applied, while a later turn is "
+ "queued, or once a later turn has started); the same value "
+ "GET /v1/chat/tasks/{task_id} reports."
),
)
diff --git a/src/xagent/web/services/sdk_task_cancel.py b/src/xagent/web/services/sdk_task_cancel.py
index 3d4ddf779..8d54dd9b3 100644
--- a/src/xagent/web/services/sdk_task_cancel.py
+++ b/src/xagent/web/services/sdk_task_cancel.py
@@ -16,30 +16,45 @@ run was stopped by an explicit ``cancel_task`` call
settlement did. Whichever writer lands first owns the terminal row, and
the other one changes nothing.
-Only a RUNNING task is cancelled here. A task with a turn still queued to
-start, one waiting for the caller's answer and a paused one are refused as
-busy, without a write.
+A turn still queued to start is a target of its own: until its START is
+handed off the task row shows the turn before it, so the request names the
+queued run and the core judges that run, not the row. Cancelling it before
+it starts fails its START and, for an appended turn, leaves the row -- the
+previous turn's result -- untouched. A queued START is a target only while
+it can still hand off (``start_can_hand_off``): one the START consumer would
+reject because the row moved on is treated as absent, and the row is judged
+as it stands. A repeat of a cancel that withdrew the newest turn rejoins
+that cancel's command and is answered for that turn
+(``_cancel_of_newest_turn``), since the row no longer shows it.
+
+Besides a queued turn only a RUNNING task is cancelled here. One waiting
+for the caller's answer and a paused one are refused as busy, without a
+write.
"""
from __future__ import annotations
import asyncio
import logging
-from dataclasses import dataclass, replace
+from dataclasses import dataclass
from time import monotonic
from typing import TYPE_CHECKING, Any
-from sqlalchemy import func, update
+from sqlalchemy import func, or_, update
from sqlalchemy.orm import Session
from ..models.chat_message import TaskChatMessage
from ..models.database import get_session_local
from ..models.task import Task, TaskStatus
+from ..models.task_command import TaskExecutionCommand
from .db_runtime import run_db_io_cancellation_safe
from .task_command_transport import (
COMMAND_COMPLETED,
COMMAND_FAILED,
+ COMMAND_PENDING,
+ COMMAND_PROCESSING,
EnqueuedTaskCommand,
+ TaskCommandDeferred,
TaskCommandKind,
TaskCommandRejected,
TaskCommandTaskMissing,
@@ -48,6 +63,11 @@ from .task_command_transport import (
load_task_command,
retry_failed_task_command,
)
+from .task_execution_admission import (
+ STOPPED_BEFORE_START,
+ STOPPED_BEFORE_START_REASON,
+ waiting_admission,
+)
from .task_execution_controller import (
StaleTaskRunError,
TaskControlState,
@@ -170,7 +190,10 @@ class _CancelCommand:
command_db_id: int
# The run the command was persisted against: the one this request
# answers for, whatever run the row shows by the time it is read back.
+ # For a turn queued to start it is that turn's run, which the row does
+ # not show until the START is handed off.
aimed_run_id: str | None
+ aimed_at_queued_turn: bool
def _task_run_id(task: Task) -> str | None:
@@ -178,48 +201,214 @@ def _task_run_id(task: Task) -> str | None:
return str(run_id) if run_id is not None else None
-def _result(task: Task) -> SdkTaskCancelResult:
- cancelled = is_sdk_cancelled(task.status, task.error_message)
- run_id = _task_run_id(task)
+def _paused_reads_failed(db: Session, task: Task) -> bool:
+ """Whether the task read reports this PAUSED row as ``failed``.
+
+ The row stays PAUSED; automatic resume gave up, or the turn appended to
+ it could not be started, and ``status_only_terminal_view`` is what the
+ read surface derives ``failed`` from.
+ """
+
+ from .task_auto_resume import status_only_terminal_view
+
+ if (
+ task.status != TaskStatus.PAUSED
+ or task.control_state != TaskControlState.PAUSED.value
+ ):
+ return False
+ return status_only_terminal_view(db, int(task.id)) is not None
+
+
+def _result(
+ db: Session,
+ task: Task,
+ *,
+ cancelled: bool,
+ cancelled_run_id: str | None = None,
+ queued: bool = False,
+) -> SdkTaskCancelResult:
+ """The task row as the cancel response reports it.
+
+ ``cancelled`` is the caller's verdict on the run it was asked about,
+ which is not always the run the row holds. The status is the one the task
+ read reports: a completed or failed row reads as ``pending`` with no code
+ while a START is waiting (``queued``; ``QUEUED_START_STATUSES``), and a
+ paused row reads as ``failed`` when ``_paused_reads_failed``.
+ """
+
+ from .task_start_protocol import QUEUED_START_STATUSES
+
+ shown_pending = queued and task.status in QUEUED_START_STATUSES
+ if shown_pending:
+ status = TaskStatus.PENDING.value
+ elif _paused_reads_failed(db, task):
+ status = TaskStatus.FAILED.value
+ else:
+ status = task.status.value
return SdkTaskCancelResult(
task_id=int(task.id),
- status=task.status.value,
- run_id=run_id,
+ status=status,
+ run_id=_task_run_id(task),
state_version=int(task.state_version or 0),
- cancelled_run_id=run_id if cancelled else None,
- error_code=sdk_cancel_error_code(task.status, task.error_message),
+ cancelled_run_id=cancelled_run_id if cancelled else None,
+ error_code=(
+ None
+ if shown_pending
+ else sdk_cancel_error_code(task.status, task.error_message)
+ ),
cancelled=cancelled,
)
+def _row_result(db: Session, task: Task) -> SdkTaskCancelResult:
+ """The result for a command aimed at the run the row holds."""
+
+ return _result(
+ db,
+ task,
+ cancelled=is_sdk_cancelled(task.status, task.error_message),
+ cancelled_run_id=_task_run_id(task),
+ )
+
+
+def _queued_start(db: Session, task_id: int) -> TaskExecutionCommand | None:
+ """The START of ``task_id`` not handed off yet, as ``has_queued_start``
+ counts it (at most one exists)."""
+
+ from .task_start_protocol import queued_start_conditions
+
+ return (
+ db.query(TaskExecutionCommand)
+ .filter(*queued_start_conditions(task_id))
+ .order_by(TaskExecutionCommand.id.desc())
+ .first()
+ )
+
+
+def _cancel_of_newest_turn(db: Session, task: Task) -> TaskExecutionCommand | None:
+ """This endpoint's cancel command aimed at the task's newest START, if any.
+
+ A cancel of a turn queued to start is persisted under an identity made of
+ that turn's run and the version the START was accepted against
+ (``_cancel_command_id``), so the newest START names the one command a
+ cancel of that turn can be. Once the command has withdrawn the turn, the
+ row still shows the turn before it (or, for a new task, holds no run) and
+ no START is queued, so a repeat finds the turn only through this
+ identity -- whatever state the command is in, including while it is
+ still being applied after the withdrawal was committed. A START stopped
+ some other way has no such command and is not found.
+
+ The command answers only while the row is still as that cancel found or
+ left it: withdrawing an appended turn does not move the row, so the
+ command's version is still the row's. A row that moved since -- for
+ example the turn before was resumed and runs again -- is answered from
+ the row, as any other call would be. A new task is the exception: its
+ cancel wrote the row itself, which then holds no run and reads as
+ cancelled.
+ """
+
+ newest = (
+ db.query(TaskExecutionCommand)
+ .filter(
+ TaskExecutionCommand.task_id == int(task.id),
+ TaskExecutionCommand.kind == TaskCommandKind.START.value,
+ )
+ .order_by(TaskExecutionCommand.id.desc())
+ .populate_existing()
+ .first()
+ )
+ if (
+ newest is None
+ or newest.target_run_id is None
+ or newest.target_state_version is None
+ ):
+ return None
+ run_id = str(newest.target_run_id)
+ if run_id == _task_run_id(task):
+ return None
+ accepted_version = int(newest.target_state_version)
+ cancelled_new_task = _task_run_id(task) is None and is_sdk_cancelled(
+ task.status, task.error_message
+ )
+ if not cancelled_new_task and accepted_version != int(task.state_version or 0):
+ return None
+ cancel = (
+ db.query(TaskExecutionCommand)
+ .filter(
+ TaskExecutionCommand.task_id == int(task.id),
+ TaskExecutionCommand.command_id
+ == _cancel_command_id(int(task.id), accepted_version, run_id),
+ TaskExecutionCommand.kind == TaskCommandKind.CANCEL.value,
+ )
+ .populate_existing()
+ .one_or_none()
+ )
+ if cancel is None:
+ return None
+ payload: dict[str, Any] = cancel.payload if isinstance(cancel.payload, dict) else {}
+ if (
+ payload.get("scope") != SDK_COMMAND_SCOPE
+ or payload.get("queued_run_id") != run_id
+ ):
+ return None
+ return cancel
+
+
+def _start_of_run(
+ db: Session, task_id: int, run_id: str
+) -> TaskExecutionCommand | None:
+ """The START that opens exactly ``run_id``, whatever its state.
+
+ Matched on the run so that a later turn's START is never taken for the
+ one a cancel was aimed at.
+ """
+
+ return (
+ db.query(TaskExecutionCommand)
+ .filter(
+ TaskExecutionCommand.task_id == task_id,
+ TaskExecutionCommand.kind == TaskCommandKind.START.value,
+ TaskExecutionCommand.target_run_id == run_id,
+ )
+ .order_by(TaskExecutionCommand.id.desc())
+ .populate_existing()
+ .first()
+ )
+
+
+def _start_was_stopped(start: TaskExecutionCommand | None) -> bool:
+ if start is None or start.status != COMMAND_FAILED:
+ return False
+ result: dict[str, Any] = start.result if isinstance(start.result, dict) else {}
+ return bool(result.get("rejection_reason") == STOPPED_BEFORE_START_REASON)
+
+
def _ended_without_cancel(db: Session, task: Task) -> bool:
"""Whether the task's run ended and was not an SDK cancellation.
Covers a COMPLETED or FAILED row and a paused row the read surface
- reports as failed because automatic resume gave up.
+ reports as failed (``_paused_reads_failed``).
"""
- from .task_auto_resume import status_only_terminal_view
-
if is_sdk_cancelled(task.status, task.error_message):
return False
- if task.status in _TERMINAL_STATUSES:
- return True
- if (
- task.status == TaskStatus.PAUSED
- and task.control_state == TaskControlState.PAUSED.value
- ):
- return status_only_terminal_view(db, int(task.id)) is not None
- return False
+ return task.status in _TERMINAL_STATUSES or _paused_reads_failed(db, task)
def _prepare_cancel_sync(
*, task_id: int, scope: SdkTaskScope
) -> SdkTaskCancelResult | _CancelCommand:
- """Authorize, classify the target and persist its command, in one read."""
+ """Authorize, classify the target and persist its command, in one read.
- from .task_start import TaskStartRejected, resolve_sdk_task
- from .task_start_protocol import has_queued_start
+ The target is the queued START that can still hand off, else the row. A
+ repeat of a cancel that already withdrew the newest turn finds no such
+ START and rejoins that cancel's command (``_cancel_of_newest_turn``):
+ the same command to poll while it is still being applied, the same
+ answer once it completed.
+ """
+
+ from .task_start import resolve_sdk_task
+ from .task_start_protocol import start_can_hand_off
SessionLocal = get_session_local()
with SessionLocal() as db:
@@ -227,59 +416,128 @@ def _prepare_cancel_sync(
# is handed off, the row still shows the previous turn, so judging
# the row alone would answer for a turn the caller is not asking
# about.
- queued = has_queued_start(db, task_id)
+ queued = _queued_start(db, task_id)
task = resolve_sdk_task(task_id, scope, db)
- if queued:
- raise SdkTaskCancelBusy(
- "A turn of this task is queued to start and cannot be "
- "cancelled yet; retry once it is running."
- )
+ if queued is not None and not start_can_hand_off(queued, task):
+ # Never starts: the row moved on after the START was accepted,
+ # so the turn it would open is not what a cancel can reach.
+ queued = None
+ if queued is not None:
+ if queued.target_run_id is None:
+ raise SdkTaskCancelBusy(
+ "A turn of this task is queued to start and cannot be "
+ "cancelled yet; retry once it is running."
+ )
+ return _stage_cancel(db, task, queued_run_id=str(queued.target_run_id))
+ # A repeat of the cancel that withdrew the newest turn rejoins that
+ # command and is answered for that turn, not for whatever the row
+ # shows of the turn before it: polled like the first call while the
+ # command is still being applied, answered as cancelled once it
+ # completed. A failed command is not a cancellation, so the row
+ # decides.
+ cancel = _cancel_of_newest_turn(db, task)
+ if cancel is not None:
+ withdrawn_run_id = str(cancel.payload["queued_run_id"])
+ if cancel.status == COMMAND_COMPLETED:
+ return _result(
+ db, task, cancelled=True, cancelled_run_id=withdrawn_run_id
+ )
+ if cancel.status in {COMMAND_PENDING, COMMAND_PROCESSING}:
+ return _CancelCommand(
+ command_db_id=int(cancel.id),
+ aimed_run_id=withdrawn_run_id,
+ aimed_at_queued_turn=True,
+ )
if is_sdk_cancelled(task.status, task.error_message):
- return _result(task)
+ return _row_result(db, task)
if _ended_without_cancel(db, task):
raise SdkTaskNotCancellable
if task.status != TaskStatus.RUNNING:
raise SdkTaskCancelBusy(
f"A task that is {task.status.value} cannot be cancelled yet."
)
- target_state_version = int(task.state_version or 0)
- # Read before the insert commits and expires ``task``: this is the
- # run the command targets.
- aimed_run_id = _task_run_id(task)
- try:
- enqueued = _enqueue_cancel(db, task, target_state_version)
- except TaskCommandTaskMissing as exc:
- # Deleted between the read above and the insert.
- raise TaskStartRejected("task_not_found") from exc
- if not enqueued.payload_matches:
- raise SdkTaskCancelBusy(
- "Another command holds this cancel identity; retry."
- )
- if enqueued.status == COMMAND_FAILED:
- retry_failed_task_command(db, enqueued.command_id)
- return _CancelCommand(
- command_db_id=enqueued.command_id, aimed_run_id=aimed_run_id
+ return _stage_cancel(db, task, queued_run_id=None)
+
+
+def _stage_cancel(
+ db: Session, task: Task, *, queued_run_id: str | None
+) -> _CancelCommand:
+ """Persist the cancel command for the state just read.
+
+ ``queued_run_id`` aims it at the turn queued to start as that run;
+ otherwise it is aimed at the run the row holds.
+ """
+
+ from .task_start import TaskStartRejected
+
+ target_state_version = int(task.state_version or 0)
+ # Read before the insert commits and expires ``task``: this is the run
+ # the command targets.
+ aimed_run_id = queued_run_id if queued_run_id is not None else _task_run_id(task)
+ try:
+ enqueued = _enqueue_cancel(
+ db, task, target_state_version, queued_run_id=queued_run_id
)
+ except TaskCommandTaskMissing as exc:
+ # Deleted between the caller's read and the insert.
+ raise TaskStartRejected("task_not_found") from exc
+ if not enqueued.payload_matches:
+ raise SdkTaskCancelBusy("Another command holds this cancel identity; retry.")
+ if enqueued.status == COMMAND_FAILED:
+ retry_failed_task_command(db, enqueued.command_id)
+ return _CancelCommand(
+ command_db_id=enqueued.command_id,
+ aimed_run_id=aimed_run_id,
+ aimed_at_queued_turn=queued_run_id is not None,
+ )
+
+
+def _cancel_command_id(
+ task_id: int, target_state_version: int, queued_run_id: str | None
+) -> str:
+ """The identity of the cancel command for the state it is aimed at.
+
+ Keyed on the state the request read: a repeat against the same state is
+ the same command, and a repeat after the state moved is a new one aimed
+ at the new state. An appended turn does not move the row while it is
+ queued, so its run is part of the key; once such a turn is withdrawn,
+ ``_cancel_of_newest_turn`` finds the command again by this key.
+ """
+
+ command_id = f"cancel:{task_id}:{target_state_version}"
+ if queued_run_id is not None:
+ command_id = f"{command_id}:{queued_run_id}"
+ return command_id
def _enqueue_cancel(
- db: Session, task: Task, target_state_version: int
+ db: Session,
+ task: Task,
+ target_state_version: int,
+ *,
+ queued_run_id: str | None,
) -> EnqueuedTaskCommand:
task_id = int(task.id)
+ command_id = _cancel_command_id(task_id, target_state_version, queued_run_id)
+ payload: dict[str, Any] = {
+ "agent_id": int(task.agent_id),
+ "target_state_version": target_state_version,
+ "scope": SDK_COMMAND_SCOPE,
+ }
+ if queued_run_id is not None:
+ payload["queued_run_id"] = queued_run_id
return enqueue_task_command(
db,
task_id=task_id,
actor_user_id=int(task.user_id) if task.user_id is not None else None,
- # Keyed on the state read here: a repeat against the same state
- # is the same command, and a repeat after the state moved is a
- # new one aimed at the new state.
- command_id=f"cancel:{task_id}:{target_state_version}",
+ command_id=command_id,
kind=TaskCommandKind.CANCEL,
- payload={
- "agent_id": int(task.agent_id),
- "target_state_version": target_state_version,
- "scope": SDK_COMMAND_SCOPE,
- },
+ payload=payload,
+ # A queued turn's cancel targets that turn's run, not the run the row
+ # still shows. A process that does not read ``queued_run_id`` then
+ # finds the run changed and rejects the command without a write,
+ # instead of taking an earlier cancelled run for this one.
+ target_run_id=queued_run_id,
)
@@ -288,34 +546,73 @@ def _load_result_sync(
task_id: int,
scope: SdkTaskScope,
aimed_run_id: str | None,
+ aimed_at_queued_turn: bool,
command_completed: bool,
) -> SdkTaskCancelResult:
- """Read the task as it stands once the command is over or overdue.
+ """Read the outcome once the command is over or overdue.
``aimed_run_id`` is the run the command was persisted against. A
completed command cancelled that run, so the answer names it as
- ``cancelled_run_id`` while every other field describes the row now,
+ ``cancelled_run_id`` while every other field describes the task now,
which may already show a later run. Otherwise the answer is about the
- aimed run only. A row that shows another run means the aimed one is
- over, and retrying would aim at the new one, so it raises
- ``SdkTaskNotCancellable`` with ``_AIMED_RUN_ENDED_MESSAGE``. An aimed
- run that ended on its own raises ``SdkTaskNotCancellable`` exactly as
- the request would have on arrival, whether or not a later turn is
- queued behind it.
+ aimed run only:
+
+ - a turn queued to start is not on the row until its START is handed
+ off. While its START waits the command is still pending; a START
+ stopped before it began is that turn cancelled; any other START
+ outcome means the turn is over. A START the row moved away from is
+ not stopped by the command (``stale_run``): while it is still pending
+ the call answers busy, or 202 at the deadline, and once its consumer
+ rejects it the turn reads as over.
+ - a row that shows another run means the aimed one is over, and
+ retrying would aim at the new one, so it raises
+ ``SdkTaskNotCancellable`` with ``_AIMED_RUN_ENDED_MESSAGE``.
+ - an aimed run that ended on its own raises ``SdkTaskNotCancellable``
+ exactly as the request would have on arrival, whether or not a later
+ turn is queued behind it.
+
+ A START waiting for a later turn reads as ``pending`` with no code, and
+ a paused row that has stopped for good reads as ``failed``, as on the
+ task read.
"""
from .task_start import resolve_sdk_task
SessionLocal = get_session_local()
with SessionLocal() as db:
+ # Same read order as ``_prepare_cancel_sync``.
+ queued = _queued_start(db, task_id) is not None
task = resolve_sdk_task(task_id, scope, db)
if command_completed:
- return replace(_result(task), cancelled=True, cancelled_run_id=aimed_run_id)
+ return _result(
+ db, task, cancelled=True, cancelled_run_id=aimed_run_id, queued=queued
+ )
if _task_run_id(task) != aimed_run_id:
+ if aimed_at_queued_turn and aimed_run_id is not None:
+ start = _start_of_run(db, task_id, aimed_run_id)
+ if _start_was_stopped(start):
+ return _result(
+ db,
+ task,
+ cancelled=True,
+ cancelled_run_id=aimed_run_id,
+ queued=queued,
+ )
+ if start is not None and start.status in {
+ COMMAND_PENDING,
+ COMMAND_PROCESSING,
+ }:
+ return _result(db, task, cancelled=False, queued=queued)
raise SdkTaskNotCancellable(_AIMED_RUN_ENDED_MESSAGE)
if _ended_without_cancel(db, task):
raise SdkTaskNotCancellable
- return _result(task)
+ return _result(
+ db,
+ task,
+ cancelled=is_sdk_cancelled(task.status, task.error_message),
+ cancelled_run_id=aimed_run_id,
+ queued=queued,
+ )
async def request_sdk_task_cancel(
@@ -332,12 +629,15 @@ async def request_sdk_task_cancel(
task's state.
Returns ``cancelled=True`` once the task is the outcome of a
- cancellation, including when it already was, and once the command that
- cancelled the aimed run completed (``cancelled_run_id``). Returns
- ``cancelled=False`` when the command was persisted but not applied
- within ``SDK_CANCEL_REQUEST_WAIT_SECONDS``; calling again reaches the
- same command. Raises ``SdkTaskNotCancellable`` when the aimed run ended
- without being cancelled, whichever way the wait ends.
+ cancellation, including when it already was, once the command that
+ cancelled the aimed run completed (``cancelled_run_id``), and when an
+ earlier call's command that withdrew the newest turn completed, which
+ names that turn. Returns ``cancelled=False`` when the command was
+ persisted but not applied within ``SDK_CANCEL_REQUEST_WAIT_SECONDS``;
+ calling again reaches the same command, including after it withdrew a
+ queued turn and before it is marked completed. Raises
+ ``SdkTaskNotCancellable`` when the aimed run ended without being
+ cancelled, whichever way the wait ends.
"""
prepared = await run_db_io_cancellation_safe(
@@ -367,6 +667,7 @@ async def request_sdk_task_cancel(
task_id=task_id,
scope=scope,
aimed_run_id=aimed_run_id,
+ aimed_at_queued_turn=prepared.aimed_at_queued_turn,
command_completed=command_completed,
)
)
@@ -387,6 +688,7 @@ async def request_sdk_task_cancel(
task_id=task_id,
scope=scope,
aimed_run_id=aimed_run_id,
+ aimed_at_queued_turn=prepared.aimed_at_queued_turn,
command_completed=False,
)
)
@@ -436,14 +738,18 @@ def _is_settled_sdk_cancel_target(
def _is_exact_cancel_target(
- task: Task, *, expected_run_id: str | None, expected_state_version: int
+ task: Task,
+ *,
+ expected_run_id: str | None,
+ expected_state_version: int,
+ expected_status: TaskStatus,
) -> bool:
- """Whether the row is still the running target this command was aimed at."""
+ """Whether the row is still the target this command was aimed at."""
return (
_task_run_id(task) == expected_run_id
and int(task.state_version or 0) == expected_state_version
- and task.status == TaskStatus.RUNNING
+ and task.status == expected_status
)
@@ -455,12 +761,26 @@ def _stale(task: Task, expected_run_id: str | None, expected_state_version: int)
)
+def _carries_display_events(task: Task, expected_status: TaskStatus) -> bool:
+ """Whether the terminal broadcast should carry the turn's display events.
+
+ Only a version-2 task has them, and only a run that ran has the result
+ fact they are read from: a new task cancelled before it started has
+ none, and publishing them without it fails.
+ """
+
+ if expected_status != TaskStatus.RUNNING:
+ return False
+ return bool(task.conversation_storage_version == 2)
+
+
def _load_cancelable_sdk_task_sync(
*,
task_id: int,
agent_id: int,
expected_run_id: str | None,
expected_state_version: int,
+ expected_status: TaskStatus,
terminal_event_state: dict[str, Any],
) -> bool:
"""Return whether the exact cancel target is already settled.
@@ -477,13 +797,14 @@ def _load_cancelable_sdk_task_sync(
expected_run_id=expected_run_id,
expected_state_version=expected_state_version,
):
- if task.conversation_storage_version == 2:
+ if _carries_display_events(task, expected_status):
terminal_event_state.update(task_control_snapshot(task).as_dict())
return True
if not _is_exact_cancel_target(
task,
expected_run_id=expected_run_id,
expected_state_version=expected_state_version,
+ expected_status=expected_status,
):
raise TaskCommandRejected(
_stale(task, expected_run_id, expected_state_version),
@@ -548,16 +869,21 @@ def _finalize_sdk_cancel_sync(
agent_id: int,
expected_run_id: str | None,
expected_state_version: int,
+ expected_status: TaskStatus,
terminal_event_state: dict[str, Any],
) -> bool:
"""Persist cancellation for one exact target; return whether this wrote it.
+ ``expected_status`` is RUNNING, or PENDING for a new task whose START is
+ still queued: that task never ran, so it gets the terminal row alone and
+ its START's own settlement closes the turn.
+
Raises ``StaleTaskRunError`` before any write, or after a rollback, when
- the target is neither still running as read nor already settled. When
- the target is settled or written here, it fills ``terminal_event_state``
+ the target is neither still as read nor already settled. When the
+ target is settled or written here, it fills ``terminal_event_state``
with the control snapshot the terminal broadcast carries, once the row
- is durable. The task's cached
- read is invalidated after this call's own commit.
+ is durable. The task's cached read is invalidated after this call's own
+ commit.
"""
from .task_coordinator_runtime import current_task_coordinator
@@ -586,7 +912,7 @@ def _finalize_sdk_cancel_sync(
expected_run_id=expected_run_id,
expected_state_version=expected_state_version,
):
- if task.conversation_storage_version == 2:
+ if _carries_display_events(task, expected_status):
settled_state = task_control_snapshot(task).as_dict()
else:
settled_state = {}
@@ -597,6 +923,7 @@ def _finalize_sdk_cancel_sync(
task,
expected_run_id=expected_run_id,
expected_state_version=expected_state_version,
+ expected_status=expected_status,
):
db.rollback()
raise StaleTaskRunError(
@@ -610,7 +937,7 @@ def _finalize_sdk_cancel_sync(
Task.agent_id == agent_id,
Task.source == SDK_TASK_SOURCE,
func.coalesce(Task.state_version, 0) == expected_state_version,
- Task.status == TaskStatus.RUNNING,
+ Task.status == expected_status,
)
.values(
status=TaskStatus.FAILED,
@@ -645,16 +972,17 @@ def _finalize_sdk_cancel_sync(
)
sync_workforce_run_status(db, updated, TaskStatus.FAILED)
sync_trigger_run_status(db, updated, TaskStatus.FAILED)
- turn_id = _close_cancelled_turn_no_commit(db, updated)
- stage_result_fact_no_commit(
- db,
- updated,
- sdk_cancel_result_fact(SDK_CANCEL_ERROR_MESSAGE),
- turn_id=turn_id,
- )
+ if expected_status == TaskStatus.RUNNING:
+ turn_id = _close_cancelled_turn_no_commit(db, updated)
+ stage_result_fact_no_commit(
+ db,
+ updated,
+ sdk_cancel_result_fact(SDK_CANCEL_ERROR_MESSAGE),
+ turn_id=turn_id,
+ )
event_state = (
task_control_snapshot(updated).as_dict()
- if updated.conversation_storage_version == 2
+ if _carries_display_events(updated, expected_status)
else {}
)
db.commit()
@@ -696,20 +1024,192 @@ async def _broadcast_sdk_cancel_terminal_event(
)
+@dataclass(frozen=True)
+class _CancelTarget:
+ run_id: str | None
+ state_version: int
+ status: TaskStatus
+
+
+def _stop_queued_start_no_commit(
+ db: Session, task_id: int, start: TaskExecutionCommand
+) -> bool:
+ """Fail a START that no executor has claimed, leaving the task row alone.
+
+ Stopped while it is pending and either never attempted or waiting for an
+ admission ticket, the same START a cancel is dispatched past
+ (``waiting_admission``): one sent back to the admission queue after an
+ attempt is still stopped, one an executor claimed and may have acted on
+ is not. An append's failure belongs to its command: the row keeps the
+ previous turn's result. The turn's delivery row and runtime values are
+ released by the START's own settlement. Returns False when the START
+ moved first.
+ """
+
+ from .task_command_terminal_events import (
+ TerminalTaskEventDraft,
+ stage_terminal_event,
+ )
+ from .task_coordinator_runtime import current_task_coordinator
+ from .task_coordinator_service import lock_task_lease_no_commit
+ from .task_start_consumer import settle_failed_start_no_commit
+
+ coordinator = current_task_coordinator(task_id)
+ if coordinator is not None:
+ assert coordinator.lease is not None
+ if not lock_task_lease_no_commit(db, coordinator.lease):
+ raise StaleTaskRunError(f"task {task_id} cancellation owner changed")
+ stopped = db.execute(
+ update(TaskExecutionCommand)
+ .where(
+ TaskExecutionCommand.id == start.id,
+ TaskExecutionCommand.status == COMMAND_PENDING,
+ or_(
+ TaskExecutionCommand.attempt_count == 0,
+ waiting_admission(TaskExecutionCommand.id),
+ ),
+ )
+ .values(
+ status=COMMAND_FAILED,
+ error=STOPPED_BEFORE_START,
+ completed_at=func.now(),
+ result={"rejection_reason": STOPPED_BEFORE_START_REASON},
+ )
+ .returning(TaskExecutionCommand.id)
+ .execution_options(synchronize_session=False)
+ ).scalar_one_or_none()
+ if stopped is None:
+ return False
+ db.refresh(start)
+ settle_failed_start_no_commit(db, start)
+ stage_terminal_event(
+ db,
+ command_db_id=int(start.id),
+ draft=TerminalTaskEventDraft(message_code=None, resend_safe=False),
+ )
+ return True
+
+
+def _resolve_queued_cancel_target_sync(
+ *, task_id: int, agent_id: int, queued_run_id: str
+) -> _CancelTarget | None:
+ """Turn "the turn queued as ``queued_run_id``" into what to cancel now.
+
+ Returns the task-row target to cancel, or ``None`` when nothing is left
+ to do: the turn was already cancelled, or its START was stopped before
+ it began (by this call, for an appended turn). Raises
+ ``TaskCommandRejected`` when the turn ended on its own or never started,
+ including when its START can no longer hand off because the row moved
+ after it was accepted (checked before anything is written), and
+ ``TaskCommandDeferred`` while its START is being handed off.
+ """
+
+ from .task_start_protocol import start_can_hand_off
+
+ stale = TaskCommandRejected(
+ f"queued run {queued_run_id} of task {task_id} is no longer cancellable",
+ reason="stale_run",
+ )
+ SessionLocal = get_session_local()
+ with SessionLocal() as db:
+ # One more pass covers a START that is handed off, or stopped by
+ # another control, between this read and the write below.
+ for _attempt in range(2):
+ task = _load_sdk_task(db, task_id=task_id, agent_id=agent_id)
+ if _task_run_id(task) == queued_run_id:
+ # The queued turn started. Fenced on the version read now:
+ # the handoff and the run itself moved it since the request.
+ if is_sdk_cancelled(task.status, task.error_message):
+ return None
+ if task.status != TaskStatus.RUNNING:
+ raise stale
+ return _CancelTarget(
+ queued_run_id, int(task.state_version or 0), TaskStatus.RUNNING
+ )
+ start = _start_of_run(db, task_id, queued_run_id)
+ if _start_was_stopped(start):
+ return None
+ if start is None or start.status == COMMAND_FAILED:
+ raise stale
+ if start.status == COMMAND_PROCESSING:
+ raise TaskCommandDeferred(
+ f"START of run {queued_run_id} is being handed off"
+ )
+ if start.status == COMMAND_PENDING:
+ if _new_task_already_cancelled(task, start):
+ # A new task's cancel wrote the row and the START is
+ # settled only once that command completes. Applied
+ # again before then -- its completion never committed --
+ # the cancel is already done; the row has moved past the
+ # START, which must not read as the turn having ended.
+ return None
+ if not start_can_hand_off(start, task):
+ # The row moved on after the START was accepted, so the
+ # START will never open this run and there is nothing
+ # of the turn to cancel. Nothing is written.
+ raise stale
+ if task.status == TaskStatus.PENDING:
+ # A new task that never ran: the row itself is cancelled
+ # and the completed command settles the START.
+ return _CancelTarget(
+ _task_run_id(task),
+ int(task.state_version or 0),
+ TaskStatus.PENDING,
+ )
+ if _stop_queued_start_no_commit(db, task_id, start):
+ db.commit()
+ return None
+ db.rollback()
+ raise stale
+
+
+def _new_task_already_cancelled(task: Task, start: TaskExecutionCommand) -> bool:
+ """Whether ``start`` opens a new task whose row a cancel already wrote."""
+
+ from .task_start_protocol import TaskStartPayload
+
+ if _task_run_id(task) is not None or not is_sdk_cancelled(
+ task.status, task.error_message
+ ):
+ return False
+ try:
+ payload = TaskStartPayload.model_validate(start.payload)
+ except ValueError:
+ return False
+ return bool(payload.kind == "create")
+
+
async def cancel_sdk_task_unserialized(
*,
task_id: int,
agent_id: int,
expected_run_id: str | None,
expected_state_version: int,
+ queued_run_id: str | None = None,
) -> None:
"""Cancel one exact durable-command target while the caller owns its gate.
+ ``queued_run_id`` aims the command at a turn that was queued when it
+ was requested; the run and version to cancel are then read here.
+
Every expected failure leaves as ``TaskCommandRejected`` or
``StaleTaskRunError`` (which the dispatcher rejects as ``stale_run``),
so the command ends at once with its reason instead of being retried.
"""
+ expected_status = TaskStatus.RUNNING
+ if queued_run_id is not None:
+ target = await run_db_io_cancellation_safe(
+ lambda: _resolve_queued_cancel_target_sync(
+ task_id=task_id, agent_id=agent_id, queued_run_id=queued_run_id
+ )
+ )
+ if target is None:
+ return
+ expected_run_id = target.run_id
+ expected_state_version = target.state_version
+ expected_status = target.status
+
terminal_event_state: dict[str, Any] = {}
already_settled = await run_db_io_cancellation_safe(
lambda: _load_cancelable_sdk_task_sync(
@@ -717,6 +1217,7 @@ async def cancel_sdk_task_unserialized(
agent_id=agent_id,
expected_run_id=expected_run_id,
expected_state_version=expected_state_version,
+ expected_status=expected_status,
terminal_event_state=terminal_event_state,
)
)
@@ -735,6 +1236,7 @@ async def cancel_sdk_task_unserialized(
agent_id=agent_id,
expected_run_id=expected_run_id,
expected_state_version=expected_state_version,
+ expected_status=expected_status,
terminal_event_state=terminal_event_state,
)
diff --git a/src/xagent/web/services/task_command_execution.py b/src/xagent/web/services/task_command_execution.py
index 161d8ec2a..ea2ff8299 100644
--- a/src/xagent/web/services/task_command_execution.py
+++ b/src/xagent/web/services/task_command_execution.py
@@ -4706,10 +4706,20 @@ async def _execute_durable_task_command(
auto_resume = command.kind == TaskCommandKind.RESUME and (
command.command_id.startswith(AUTO_RESUME_COMMAND_PREFIX)
)
+ # An SDK cancel aimed at a turn that was still queued when it was
+ # requested targets that turn's run while the row holds the run before
+ # it, or none. Its core compares the target itself, against the run the
+ # payload names.
+ sdk_queued_cancel = (
+ command.kind == TaskCommandKind.CANCEL
+ and command.payload.get("scope") == SDK_COMMAND_SCOPE
+ and _sdk_queued_run_id(command.payload) is not None
+ )
if (
command.kind != TaskCommandKind.MESSAGE
and command.target_run_id is not None
and not auto_resume
+ and not sdk_queued_cancel
):
current_run_id = await run_db_io_cancellation_safe(
lambda: _load_command_task_run_id(command.task_id)
@@ -4958,6 +4968,7 @@ async def _execute_durable_task_command(
agent_id=agent_id,
expected_run_id=command.target_run_id,
expected_state_version=target_state_version,
+ queued_run_id=_sdk_queued_run_id(message_data),
)
elif scope_value == EXTERNAL_COMMAND_SCOPE:
await cancel_external_task_unserialized(
@@ -4996,6 +5007,13 @@ async def _execute_durable_task_command(
return result
+def _sdk_queued_run_id(message_data: dict[str, Any]) -> str | None:
+ """The queued run an SDK cancel names, or ``None`` when it names none."""
+
+ raw = message_data.get("queued_run_id")
+ return raw if isinstance(raw, str) and raw else None
+
+
def _command_scope(command: ClaimedTaskCommand) -> str | None:
"""The scope a command payload names, or ``None`` when it names none."""
diff --git a/src/xagent/web/services/task_command_transport.py b/src/xagent/web/services/task_command_transport.py
index c3782908d..ad0f96970 100644
--- a/src/xagent/web/services/task_command_transport.py
+++ b/src/xagent/web/services/task_command_transport.py
@@ -864,9 +864,13 @@ def enqueue_task_command(
payload: dict[str, Any],
reply_host_id: str | None = None,
reply_origin: str | None = None,
+ target_run_id: str | None = None,
) -> EnqueuedTaskCommand:
"""Commit an idempotent command and return only after it is durable.
+ ``target_run_id`` names the run the command is aimed at when it is not
+ the run the task row shows now, as for a turn whose START is queued.
+
TaskCommandOwnerStateError from stage_task_command's owner pre-flush
propagates uncaught -- it is deliberately not caught alongside
IntegrityError below. A caller managing its own session lifetime must
@@ -884,6 +888,7 @@ def enqueue_task_command(
payload=payload,
reply_host_id=reply_host_id,
reply_origin=reply_origin,
+ target_run_id=target_run_id,
)
# Only a newly created row is worth committing, and the commit stays
# inside this try so that a constraint failure surfacing there rather
diff --git a/src/xagent/web/services/task_execution_admission.py b/src/xagent/web/services/task_execution_admission.py
index 0c7827913..47142427b 100644
--- a/src/xagent/web/services/task_execution_admission.py
+++ b/src/xagent/web/services/task_execution_admission.py
@@ -72,7 +72,12 @@ PRIORITY_MAX = 10
_EXECUTION_KINDS = frozenset({"start", "resume", "resume_input", "message"})
_TERMINAL = ("completed", "failed")
-_STOPPED_BEFORE_START = "Task stopped before execution started."
+# What a START stopped before it began records: ``STOPPED_BEFORE_START`` as
+# its error when it is the stopped command itself, and always
+# ``STOPPED_BEFORE_START_REASON`` as the ``rejection_reason`` of its result,
+# by which a reader recognises it.
+STOPPED_BEFORE_START = "Task stopped before execution started."
+STOPPED_BEFORE_START_REASON = "cancelled_before_admission"
_CONTROL_STOPPED_COMMAND = "Task command stopped by a control request."
@@ -871,9 +876,9 @@ def settle_queued_start_for_pause(
)
.values(
status="failed",
- error=_STOPPED_BEFORE_START,
+ error=STOPPED_BEFORE_START,
completed_at=func.now(),
- result={"rejection_reason": "cancelled_before_admission"},
+ result={"rejection_reason": STOPPED_BEFORE_START_REASON},
)
.returning(TaskExecutionCommand.id)
)
@@ -937,7 +942,7 @@ def settle_cancelled_admissions(db: Session, command_id: int) -> None:
status="failed",
error=_CONTROL_STOPPED_COMMAND,
completed_at=func.now(),
- result={"rejection_reason": "cancelled_before_admission"},
+ result={"rejection_reason": STOPPED_BEFORE_START_REASON},
)
.returning(TaskExecutionCommand.id)
).all()
@@ -1035,7 +1040,7 @@ def settle_cap_held_commands_no_commit(db: Session, control_command_id: int) ->
status="failed",
error=_CONTROL_STOPPED_COMMAND,
completed_at=func.now(),
- result={"rejection_reason": "cancelled_before_admission"},
+ result={"rejection_reason": STOPPED_BEFORE_START_REASON},
)
.returning(TaskExecutionCommand.id)
).all()
diff --git a/src/xagent/web/services/task_start_protocol.py b/src/xagent/web/services/task_start_protocol.py
index 8d7c7517a..46192aa15 100644
--- a/src/xagent/web/services/task_start_protocol.py
+++ b/src/xagent/web/services/task_start_protocol.py
@@ -11,6 +11,7 @@ from typing import Annotated, Literal, Self
from pydantic import BaseModel, ConfigDict, Field, JsonValue, model_validator
from sqlalchemy import select
from sqlalchemy.orm import Session
+from sqlalchemy.sql.elements import ColumnElement
from ..models.task import Task, TaskStatus
from ..models.task_command import TaskExecutionCommand
@@ -172,6 +173,29 @@ def stage_task_start_command(
)
+# Terminal statuses an append is accepted from. With shared execution the row
+# keeps one of them, and the previous turn's output and error, until the
+# queued START is handed off, so a caller waiting for a terminal status would
+# take the previous turn as the new one; such a row reads as ``pending``. A
+# PAUSED row keeps reading as paused, which callers already wait on. A RUNNING
+# or WAITING_FOR_USER row has moved since the START was queued, and a PENDING
+# row already reads as pending.
+QUEUED_START_STATUSES = (TaskStatus.COMPLETED, TaskStatus.FAILED)
+
+
+def queued_start_conditions(task_id: int) -> tuple[ColumnElement[bool], ...]:
+ """Row conditions that select the STARTs of ``task_id`` not handed off.
+
+ The one definition of "queued": ``has_queued_start`` and the SDK cancel's
+ lookup of the queued START both filter with it.
+ """
+ return (
+ TaskExecutionCommand.task_id == task_id,
+ TaskExecutionCommand.kind == TaskCommandKind.START.value,
+ TaskExecutionCommand.status.in_((COMMAND_PENDING, COMMAND_PROCESSING)),
+ )
+
+
def has_queued_start(db: Session, task_id: int) -> bool:
"""Whether an accepted START of ``task_id`` has not been handed off yet.
@@ -183,17 +207,46 @@ def has_queued_start(db: Session, task_id: int) -> bool:
return (
db.execute(
select(TaskExecutionCommand.id)
- .where(
- TaskExecutionCommand.task_id == task_id,
- TaskExecutionCommand.kind == TaskCommandKind.START.value,
- TaskExecutionCommand.status.in_((COMMAND_PENDING, COMMAND_PROCESSING)),
- )
+ .where(*queued_start_conditions(task_id))
.limit(1)
).first()
is not None
)
+def start_can_hand_off(command: TaskExecutionCommand, task: Task) -> bool:
+ """Whether the START ``command`` can still begin its run on ``task``.
+
+ The conditions the START consumer applies before it hands a START off
+ that depend on the command and the task row: the payload decodes and
+ matches its command's envelope (``read_task_start_command``), the
+ command's own version and the payload's agree, the row is still the run
+ and version the START was accepted against, nothing is running, and a
+ CREATE finds its task still pending. A START that fails them is rejected
+ when it is claimed and never starts, so a control that aims at it would
+ be aimed at nothing.
+
+ Two consumer checks are not repeated here. ``command_identity_matches_task``
+ re-validates the issuing user's access, and a single-process consumer
+ refuses a START that is not ``existing``. A START either of them rejects
+ never starts either, so treating it as one that can still start only
+ lets a cancel withdraw a turn that would not have run.
+ """
+ try:
+ start = TaskStartPayload.model_validate(command.payload)
+ except ValueError:
+ return False
+ return not (
+ start.run_id != command.target_run_id
+ or start.turn_id != command.command_id
+ or command.target_state_version != start.state_version
+ or task.run_id != start.expected_run_id
+ or task.state_version != start.state_version
+ or task.status == TaskStatus.RUNNING
+ or (start.kind == "create" and task.status != TaskStatus.PENDING)
+ )
+
+
def read_task_start_command(command: ClaimedTaskCommand) -> TaskStartPayload:
"""Decode persisted inputs and verify their immutable envelope identity.
diff --git a/tests/web/api/test_sdk_task_cancel.py b/tests/web/api/test_sdk_task_cancel.py
index 8d95c8735..2e7c248fc 100644
--- a/tests/web/api/test_sdk_task_cancel.py
+++ b/tests/web/api/test_sdk_task_cancel.py
@@ -9,6 +9,7 @@ it did not cause, the ownership fence, and the routing that sends only an
import asyncio
import time
+from dataclasses import replace
from datetime import UTC, datetime, timedelta
from types import SimpleNamespace
from typing import Any
@@ -21,6 +22,8 @@ from xagent.web.api import websocket as websocket_api
from xagent.web.models.agent import Agent
from xagent.web.models.chat_message import TaskChatMessage
from xagent.web.models.task import Task, TaskStatus
+from xagent.web.models.task_admission import TaskAdmissionBucket, TaskAdmissionTicket
+from xagent.web.models.task_command import TaskExecutionCommand
from xagent.web.models.task_execution_event import TaskExecutionEvent
from xagent.web.services import a2a_task_cancel as a2a_cancel_service
from xagent.web.services import sdk_task_cancel
@@ -43,14 +46,17 @@ from xagent.web.services.sdk_task_cancel import (
)
from xagent.web.services.task_command_transport import (
ClaimedTaskCommand,
+ TaskCommandDeferred,
TaskCommandKind,
TaskCommandRejected,
+ stage_task_command,
)
from xagent.web.services.task_execution_controller import (
StaleTaskRunError,
TaskControlState,
)
from xagent.web.services.task_orchestrator import TaskTurnPayload
+from xagent.web.services.task_start_protocol import TaskStartPayload
from .conftest import _admin_headers, _direct_db_session, client
@@ -589,6 +595,7 @@ def test_replaced_attempt_rejects_cancel_and_preserves_task(
agent_id=agent_id,
expected_run_id=before["run_id"],
expected_state_version=before["state_version"],
+ expected_status=TaskStatus.RUNNING,
terminal_event_state={},
)
@@ -838,6 +845,7 @@ async def test_sdk_scope_routes_to_the_sdk_core_only(
agent_id=agent_id,
expected_run_id="run-sdk",
expected_state_version=4,
+ queued_run_id=None,
)
assert external_core.await_count == 0
assert a2a_core.await_count == 0
@@ -892,6 +900,7 @@ def test_lease_fields_are_cleared_only_without_a_coordinator(
agent_id=agent_id,
expected_run_id=before.run_id,
expected_state_version=before.state_version,
+ expected_status=TaskStatus.RUNNING,
terminal_event_state={},
)
@@ -903,3 +912,633 @@ def test_lease_fields_are_cleared_only_without_a_coordinator(
assert after.lease_expires_at.replace(tzinfo=UTC) > datetime.now(UTC) - timedelta(
minutes=5
)
+
+
+# ----- a turn that was queued when its cancel was requested -----
+
+QUEUED_RUN = "run-queued"
+STOPPED_BEFORE_START = {"rejection_reason": "cancelled_before_admission"}
+
+
+def _stage_start(
+ task_id: int,
+ owner_user_id: int,
+ *,
+ run_id: str = QUEUED_RUN,
+ turn_id: str = "turn-queued",
+ **columns: Any,
+) -> int:
+ """An accepted START for ``run_id`` with its pending user message.
+
+ Its payload is the one the acceptance writes: the run and version the row
+ holds now, which is what the START consumer compares the row with.
+ """
+ db = _direct_db_session()
+ try:
+ task = db.query(Task).filter(Task.id == task_id).one()
+ start = TaskStartPayload(
+ version=1,
+ run_id=run_id,
+ state_version=int(task.state_version or 0),
+ expected_run_id=task.run_id,
+ turn_id=turn_id,
+ kind="create" if task.status == TaskStatus.PENDING else "append",
+ message="hello",
+ )
+ staged = stage_task_command(
+ db,
+ task_id=task_id,
+ actor_user_id=owner_user_id,
+ command_id=turn_id,
+ kind=TaskCommandKind.START,
+ payload=start.model_dump(mode="json"),
+ target_run_id=run_id,
+ )
+ db.commit()
+ start_id = int(staged.staged_db_id)
+ if columns:
+ db.query(TaskExecutionCommand).filter(
+ TaskExecutionCommand.id == start_id
+ ).update(columns)
+ db.commit()
+ finally:
+ db.close()
+ _seed_pending_user_message(task_id, owner_user_id, turn_id)
+ return start_id
+
+
+def _start(start_id: int) -> tuple[str, Any]:
+ db = _direct_db_session()
+ try:
+ row = db.get(TaskExecutionCommand, start_id)
+ return str(row.status), row.result
+ finally:
+ db.close()
+
+
+async def _cancel_queued(task_id: int, agent_id: int) -> None:
+ # The command was recorded against the row as it stood while the turn
+ # was queued; the core reads the target to cancel from the queued run.
+ await cancel_sdk_task_unserialized(
+ task_id=task_id,
+ agent_id=agent_id,
+ expected_run_id="run-before",
+ expected_state_version=4,
+ queued_run_id=QUEUED_RUN,
+ )
+
+
+ "previous",
+ [
+ {"status": TaskStatus.COMPLETED, "control_state": "completed", "output": "x"},
+ {"status": TaskStatus.FAILED, "control_state": "failed", "error_message": "e"},
+ {
+ "status": TaskStatus.FAILED,
+ "control_state": "failed",
+ "error_message": SDK_CANCEL_ERROR_MESSAGE,
+ },
+ {"status": TaskStatus.PAUSED, "control_state": "paused"},
+ ],
+ ids=["completed", "failed", "cancelled", "paused"],
+)
+async def test_an_appended_turn_stopped_before_it_starts_leaves_the_previous_result(
+ monkeypatch: pytest.MonkeyPatch, previous: dict[str, Any]
+) -> None:
+ """An append's failure belongs to its command, never to the row."""
+ task_id, agent_id, owner_user_id = _create_task(run_id="run-before", **previous)
+ start_id = _stage_start(task_id, owner_user_id)
+ manager = _broadcast_manager(monkeypatch)
+ before = _row(task_id)
+
+ for _attempt in range(2): # the second call is the command replayed
+ await _cancel_queued(task_id, agent_id)
+
+ assert _row(task_id) == before
+ assert _start(start_id) == ("failed", STOPPED_BEFORE_START)
+ assert _delivery_status(task_id, "turn-queued") == "failed"
+ assert _assistant_rows(task_id) == []
+ assert manager.broadcast_to_task.await_count == 0
+
+
+async def test_a_new_task_cancel_applied_again_before_it_completed_is_done(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ """The cancel wrote the new task's row, then its completion never
+ committed and the command is applied again while the START is still
+ pending. The row now holds a later version than the START, which must
+ not read as a turn that ended on its own: the cancel is already done."""
+ task_id, agent_id, owner_user_id = _create_task(
+ status=TaskStatus.PENDING, control_state="idle", run_id=None, state_version=0
+ )
+ start_id = _stage_start(task_id, owner_user_id)
+ _broadcast_manager(monkeypatch)
+
+ await _cancel_queued(task_id, agent_id)
+ after_first = _row(task_id)
+ await _cancel_queued(task_id, agent_id)
+
+ assert _row(task_id) == after_first
+ task = _load_task(task_id)
+ assert is_sdk_cancelled(task.status, task.error_message)
+ assert _start(start_id)[0] == "pending"
+
+
+async def test_a_turn_appended_to_a_cancelled_new_task_is_withdrawn(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ """A new task was cancelled before it ran, so its row holds no run and
+ reads as cancelled; a turn appended to it is queued. Cancelling that
+ turn withdraws its START: the row is not taken for the turn's own
+ cancel already being done."""
+ task_id, agent_id, owner_user_id = _create_task(
+ status=TaskStatus.FAILED,
+ control_state=TaskControlState.FAILED.value,
+ run_id=None,
+ state_version=1,
+ error_message=SDK_CANCEL_ERROR_MESSAGE,
+ )
+ start_id = _stage_start(task_id, owner_user_id)
+ _broadcast_manager(monkeypatch)
+ before = _row(task_id)
+
+ await _cancel_queued(task_id, agent_id)
+
+ assert _row(task_id) == before
+ assert _start(start_id) == ("failed", STOPPED_BEFORE_START)
+
+
[email protected]("storage_version", [1, 2])
+async def test_a_new_task_cancelled_before_it_starts_is_cancelled_as_a_row(
+ monkeypatch: pytest.MonkeyPatch, storage_version: int
+) -> None:
+ """A task that never ran gets the terminal row alone: no transcript line
+ and no delivery change, which its START's own settlement owns. It has no
+ result fact either, so its terminal frame carries no display events,
+ whatever its storage version: asking for them would fail the broadcast."""
+ task_id, agent_id, owner_user_id = _create_task(
+ status=TaskStatus.PENDING,
+ control_state="idle",
+ run_id=None,
+ state_version=0,
+ storage_version=storage_version,
+ )
+ _stage_start(task_id, owner_user_id)
+ manager = _broadcast_manager(monkeypatch)
+
+ await _cancel_queued(task_id, agent_id)
+
+ task = _load_task(task_id)
+ assert is_sdk_cancelled(task.status, task.error_message)
+ assert (task.run_id, task.state_version) == (None, 1)
+ assert _assistant_rows(task_id) == []
+ assert _delivery_status(task_id, "turn-queued") == DELIVERY_PENDING
+ assert "state_version" not in _terminal_frame(manager)
+
+
+async def test_a_queued_turn_that_started_first_is_cancelled_as_running(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ """The handoff moved the run and the version since the request; the
+ queued run names the target and the version is the one read now."""
+ task_id, agent_id, owner_user_id = _create_task(run_id=QUEUED_RUN, state_version=7)
+ _stage_start(task_id, owner_user_id, status="completed", attempt_count=1)
+ _broadcast_manager(monkeypatch)
+
+ await _cancel_queued(task_id, agent_id)
+
+ task = _load_task(task_id)
+ assert is_sdk_cancelled(task.status, task.error_message)
+ assert (task.run_id, task.state_version) == (QUEUED_RUN, 8)
+
+
+async def test_a_queued_turn_already_cancelled_is_left_alone(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ task_id, agent_id, owner_user_id = _create_task(
+ run_id=QUEUED_RUN,
+ status=TaskStatus.FAILED,
+ control_state="failed",
+ error_message=SDK_CANCEL_ERROR_MESSAGE,
+ state_version=9,
+ )
+ _stage_start(task_id, owner_user_id, status="completed", attempt_count=1)
+ _broadcast_manager(monkeypatch)
+ before = _row(task_id)
+
+ await _cancel_queued(task_id, agent_id)
+
+ assert _row(task_id) == before
+
+
+async def test_a_later_queued_turn_is_never_stopped_for_an_earlier_one(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ """The aimed turn ran to completion and another one was appended since:
+ the cancel is stale, and the newer START is not its target."""
+ task_id, agent_id, owner_user_id = _create_task(
+ run_id=QUEUED_RUN,
+ status=TaskStatus.COMPLETED,
+ control_state="completed",
+ output="done",
+ )
+ _stage_start(task_id, owner_user_id, status="completed", attempt_count=1)
+ later_id = _stage_start(
+ task_id, owner_user_id, run_id="run-later", turn_id="turn-later"
+ )
+ _broadcast_manager(monkeypatch)
+ before = _row(task_id)
+
+ with pytest.raises(TaskCommandRejected) as raised:
+ await _cancel_queued(task_id, agent_id)
+
+ assert raised.value.reason == "stale_run"
+ assert _row(task_id) == before
+ assert _start(later_id) == ("pending", None)
+ assert _delivery_status(task_id, "turn-later") == DELIVERY_PENDING
+
+
+ ("start", "error", "reason"),
+ [
+ ({"status": "processing", "attempt_count": 1}, TaskCommandDeferred, None),
+ ({"status": "pending", "attempt_count": 1}, TaskCommandRejected, "stale_run"),
+ (
+ {"status": "failed", "result": {"rejection_reason": "invalid_start"}},
+ TaskCommandRejected,
+ "stale_run",
+ ),
+ (None, TaskCommandRejected, "stale_run"),
+ ],
+ ids=["being_handed_off", "claimed_before", "failed_itself", "missing"],
+)
+async def test_a_queued_start_this_cancel_cannot_stop_is_left_as_it_is(
+ monkeypatch: pytest.MonkeyPatch, start, error, reason
+) -> None:
+ task_id, agent_id, owner_user_id = _create_task(
+ run_id="run-before", status=TaskStatus.COMPLETED, control_state="completed"
+ )
+ start_id = None if start is None else _stage_start(task_id, owner_user_id, **start)
+ _broadcast_manager(monkeypatch)
+ before = _row(task_id)
+ start_before = None if start_id is None else _start(start_id)
+
+ with pytest.raises(error) as raised:
+ await _cancel_queued(task_id, agent_id)
+
+ if reason is not None:
+ assert raised.value.reason == reason
+ assert _row(task_id) == before
+ if start_id is not None:
+ assert _start(start_id) == start_before
+
+
+async def test_a_start_is_matched_by_its_run_not_by_being_the_one_queued(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ """The aimed turn failed to start and another turn is queued now: the
+ queued START belongs to that other turn and stays as it is."""
+ task_id, agent_id, owner_user_id = _create_task(
+ run_id="run-before", status=TaskStatus.COMPLETED, control_state="completed"
+ )
+ _stage_start(
+ task_id,
+ owner_user_id,
+ status="failed",
+ result={"rejection_reason": "invalid_start"},
+ )
+ later_id = _stage_start(
+ task_id, owner_user_id, run_id="run-later", turn_id="turn-later"
+ )
+ _broadcast_manager(monkeypatch)
+
+ with pytest.raises(TaskCommandRejected) as raised:
+ await _cancel_queued(task_id, agent_id)
+
+ assert raised.value.reason == "stale_run"
+ assert _start(later_id) == ("pending", None)
+
+
+async def test_an_sdk_cancel_of_a_running_task_keeps_the_dispatcher_run_check(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ """Only a cancel aimed at a queued turn bypasses the dispatcher's run
+ comparison; one aimed at the running row is rejected there, as before."""
+ task_id, agent_id, _owner = _create_task(run_id="run-after")
+ _broadcast_manager(monkeypatch)
+ sdk_core = AsyncMock()
+ monkeypatch.setattr(
+ command_execution_service, "cancel_sdk_task_unserialized", sdk_core
+ )
+
+ with pytest.raises(TaskCommandRejected) as raised:
+ await command_execution_service._execute_durable_task_command(
+ _cancel_command(task_id, agent_id, scope="sdk")
+ )
+
+ assert raised.value.reason == "stale_run"
+ sdk_core.assert_not_awaited()
+
+
+async def test_a_queued_cancel_read_without_its_queued_run_is_rejected_unwritten(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ """A process that does not read ``queued_run_id`` sees a cancel aimed at
+ the queued turn's run while the row still holds an earlier, cancelled
+ run. The run comparison rejects it before any core, so that earlier
+ run is never reported as this cancel's outcome and the queued START is
+ left alone."""
+ task_id, agent_id, owner_user_id = _create_task(
+ run_id="run-before",
+ status=TaskStatus.FAILED,
+ control_state="failed",
+ error_message=SDK_CANCEL_ERROR_MESSAGE,
+ )
+ start_id = _stage_start(task_id, owner_user_id)
+ _broadcast_manager(monkeypatch)
+ sdk_core = AsyncMock()
+ monkeypatch.setattr(
+ command_execution_service, "cancel_sdk_task_unserialized", sdk_core
+ )
+ before = _row(task_id)
+ command = replace(
+ _cancel_command(task_id, agent_id, scope="sdk"), target_run_id=QUEUED_RUN
+ )
+
+ with pytest.raises(TaskCommandRejected) as raised:
+ await command_execution_service._execute_durable_task_command(command)
+
+ assert raised.value.reason == "stale_run"
+ sdk_core.assert_not_awaited()
+ assert _row(task_id) == before
+ assert _start(start_id) == ("pending", None)
+
+
+async def test_an_sdk_cancel_aimed_at_a_queued_turn_reaches_its_core_past_the_row_run(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ """The row holds the run before the queued one while the command targets
+ the queued turn's run, as ``_enqueue_cancel`` records it, so the
+ dispatcher's own run comparison must not reject it."""
+ task_id, agent_id, _owner = _create_task(run_id="run-before")
+ _broadcast_manager(monkeypatch)
+ sdk_core = AsyncMock()
+ monkeypatch.setattr(
+ command_execution_service, "cancel_sdk_task_unserialized", sdk_core
+ )
+ command = replace(
+ _cancel_command(task_id, agent_id, scope="sdk", queued_run_id=QUEUED_RUN),
+ target_run_id=QUEUED_RUN,
+ )
+
+ await command_execution_service._execute_durable_task_command(command)
+
+ sdk_core.assert_awaited_once_with(
+ task_id=task_id,
+ agent_id=agent_id,
+ expected_run_id=QUEUED_RUN,
+ expected_state_version=4,
+ queued_run_id=QUEUED_RUN,
+ )
+
+
+def _update_row(task_id: int, **values: Any) -> None:
+ db = _direct_db_session()
+ try:
+ db.query(Task).filter(Task.id == task_id).update(values)
+ db.commit()
+ finally:
+ db.close()
+
+
+def _wait_for_admission(start_id: int, task_id: int) -> None:
+ """Give the START an unclaimed admission ticket: it waits for capacity."""
+ db = _direct_db_session()
+ try:
+ if db.get(TaskAdmissionBucket, "test-bucket") is None:
+ db.add(TaskAdmissionBucket(key="test-bucket", capacity=1, max_pending=10))
+ db.flush()
+ db.add(
+ TaskAdmissionTicket(
+ command_id=start_id, task_id=task_id, bucket_key="test-bucket"
+ )
+ )
+ db.commit()
+ finally:
+ db.close()
+
+
+ ("attempts", "ticket", "stopped"),
+ [
+ # Sent back to the admission queue after an attempt: still waiting.
+ (1, True, True),
+ # Held below the capacity cap before any attempt.
+ (0, False, True),
+ # Claimed before and holding no ticket: it may already have had an
+ # effect, and nothing says it still waits.
+ (1, False, False),
+ ],
+ ids=["returned_to_the_admission_queue", "never_attempted", "claimed_before"],
+)
+async def test_a_queued_start_is_stopped_only_while_a_cancel_can_overtake_it(
+ monkeypatch: pytest.MonkeyPatch, attempts: int, ticket: bool, stopped: bool
+) -> None:
+ task_id, agent_id, owner_user_id = _create_task(
+ run_id="run-before", status=TaskStatus.COMPLETED, control_state="completed"
+ )
+ start_id = _stage_start(task_id, owner_user_id, attempt_count=attempts)
+ if ticket:
+ _wait_for_admission(start_id, task_id)
+ _broadcast_manager(monkeypatch)
+ before = _row(task_id)
+
+ if stopped:
+ await _cancel_queued(task_id, agent_id)
+ assert _start(start_id) == ("failed", STOPPED_BEFORE_START)
+ else:
+ with pytest.raises(TaskCommandRejected) as raised:
+ await _cancel_queued(task_id, agent_id)
+ assert raised.value.reason == "stale_run"
+ assert _start(start_id) == ("pending", None)
+ assert _row(task_id) == before
+
+
+ "moved_row",
+ [
+ {
+ "status": TaskStatus.RUNNING,
+ "control_state": "running",
+ "run_id": "run-first",
+ "state_version": 5,
+ },
+ {
+ "status": TaskStatus.WAITING_FOR_USER,
+ "control_state": "waiting_for_user",
+ "run_id": "run-first",
+ "state_version": 5,
+ },
+ # The same run, one version on: the START was accepted at the older one.
+ {
+ "status": TaskStatus.COMPLETED,
+ "control_state": "completed",
+ "state_version": 5,
+ },
+ ],
+ ids=["running_on_another_run", "waiting_on_another_run", "version_moved"],
+)
+async def test_a_queued_start_that_can_no_longer_hand_off_is_not_withdrawn(
+ monkeypatch: pytest.MonkeyPatch, moved_row: dict[str, Any]
+) -> None:
+ """The row moved after the START was accepted, so the consumer would
+ reject the START and the turn never starts. Withdrawing it would answer
+ that the turn was cancelled while the run the row holds goes on: the
+ command is rejected, with the START and the row as they were."""
+ task_id, agent_id, owner_user_id = _create_task(
+ run_id="run-before",
+ state_version=4,
+ status=TaskStatus.COMPLETED,
+ control_state="completed",
+ )
+ start_id = _stage_start(task_id, owner_user_id)
+ _update_row(task_id, **moved_row)
+ _broadcast_manager(monkeypatch)
+ before = _row(task_id)
+
+ with pytest.raises(TaskCommandRejected) as raised:
+ await _cancel_queued(task_id, agent_id)
+
+ assert raised.value.reason == "stale_run"
+ assert _row(task_id) == before
+ assert _start(start_id) == ("pending", None)
+ assert _delivery_status(task_id, "turn-queued") == DELIVERY_PENDING
+
+
+async def test_a_new_task_whose_row_moved_is_not_cancelled_for_its_queued_start(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ task_id, agent_id, owner_user_id = _create_task(
+ status=TaskStatus.PENDING,
+ control_state="idle",
+ run_id=None,
+ state_version=0,
+ )
+ start_id = _stage_start(task_id, owner_user_id)
+ _update_row(task_id, state_version=1)
+ _broadcast_manager(monkeypatch)
+ before = _row(task_id)
+
+ with pytest.raises(TaskCommandRejected) as raised:
+ await _cancel_queued(task_id, agent_id)
+
+ assert raised.value.reason == "stale_run"
+ assert _row(task_id) == before
+ assert _start(start_id) == ("pending", None)
+
+
+def test_a_start_handed_off_while_it_was_being_stopped_is_cancelled_as_running(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ """The START is handed off between the read and the write that stops it:
+ the second pass finds the turn running on the queued run and targets it
+ at the version read then."""
+ task_id, agent_id, owner_user_id = _create_task(
+ run_id="run-before", status=TaskStatus.COMPLETED, control_state="completed"
+ )
+ start_id = _stage_start(task_id, owner_user_id)
+ stops: list[int] = []
+
+ def handed_off_first(_db: Any, _task_id: int, start: Any) -> bool:
+ stops.append(int(start.id))
+ other = _direct_db_session()
+ try:
+ other.query(TaskExecutionCommand).filter(
+ TaskExecutionCommand.id == start.id
+ ).update({"status": "completed", "attempt_count": 1})
+ other.query(Task).filter(Task.id == task_id).update(
+ {
+ "status": TaskStatus.RUNNING,
+ "control_state": "running",
+ "run_id": QUEUED_RUN,
+ "state_version": 7,
+ }
+ )
+ other.commit()
+ finally:
+ other.close()
+ return False
+
+ monkeypatch.setattr(
+ sdk_task_cancel, "_stop_queued_start_no_commit", handed_off_first
+ )
+
+ target = sdk_task_cancel._resolve_queued_cancel_target_sync(
+ task_id=task_id, agent_id=agent_id, queued_run_id=QUEUED_RUN
+ )
+
+ assert stops == [start_id]
+ assert target == sdk_task_cancel._CancelTarget(QUEUED_RUN, 7, TaskStatus.RUNNING)
+
+
+ ("status", "queued"),
+ [("pending", True), ("processing", True), ("completed", False), ("failed", False)],
+)
+def test_the_queued_start_lookup_and_the_task_read_share_one_definition(
+ status: str, queued: bool
+) -> None:
+ from xagent.web.services.task_start_protocol import has_queued_start
+
+ task_id, _agent_id, owner_user_id = _create_task(
+ run_id="run-before", status=TaskStatus.COMPLETED, control_state="completed"
+ )
+ _stage_start(task_id, owner_user_id, status=status)
+ db = _direct_db_session()
+ try:
+ assert has_queued_start(db, task_id) is queued
+ assert (sdk_task_cancel._queued_start(db, task_id) is not None) is queued
+ finally:
+ db.close()
+
+
+def test_the_stopped_before_start_wording_has_one_definition() -> None:
+ """The cancel stops a START with the wording every other control uses, so
+ it imports it from where the others write it."""
+ import ast
+ import pathlib
+
+ from xagent.web.services import task_execution_admission
+
+ assert (
+ task_execution_admission.STOPPED_BEFORE_START_REASON
+ == "cancelled_before_admission"
+ )
+ tree = ast.parse(pathlib.Path(sdk_task_cancel.__file__).read_text())
+ assigned = {
+ target.id
+ for node in tree.body
+ if isinstance(node, ast.Assign)
+ for target in node.targets
+ if isinstance(target, ast.Name)
+ }
+ assert not assigned & {"STOPPED_BEFORE_START", "STOPPED_BEFORE_START_REASON"}
+ assert (
+ sdk_task_cancel.STOPPED_BEFORE_START_REASON
+ == task_execution_admission.STOPPED_BEFORE_START_REASON
+ )
diff --git a/tests/web/api/v1/test_status_only_projection.py b/tests/web/api/v1/test_status_only_projection.py
index 0ed6ed1b7..e40c2cac3 100644
--- a/tests/web/api/v1/test_status_only_projection.py
+++ b/tests/web/api/v1/test_status_only_projection.py
@@ -48,6 +48,7 @@ from xagent.web.models.auth_database import get_auth_db
from xagent.web.models.database import get_db
from xagent.web.models.task import Task, TaskStatus
from xagent.web.models.task_auto_recovery import TaskAutoRecovery
+from xagent.web.models.task_command import TaskExecutionCommand
from xagent.web.models.user import User
from xagent.web.services import sdk_task_cancel, task_auto_resume, task_event_bridge
from xagent.web.services.hot_path_cache import (
@@ -59,6 +60,7 @@ from xagent.web.services.hot_path_cache import (
from xagent.web.services.task_auto_recovery import current_auto_recovery_view
from xagent.web.services.task_auto_resume import TerminalView, run_auto_resume_tick
from xagent.web.services.task_lease_service import utc_now
+from xagent.web.services.task_start import SdkTaskScope
from ..conftest import (
_admin_headers,
@@ -295,12 +297,13 @@ def test_view_call_sites():
computed = [c for c in _calls("status_only_terminal_view") if c[0] != defining]
# The read surface derives the failed status; the cancel request consults
- # the same view so it refuses exactly the tasks that read as failed.
+ # the same view, in one place, so it refuses exactly the tasks that read
+ # as failed and answers with the status the read reports.
cancel_py = pathlib.Path(sdk_task_cancel.__file__).resolve()
assert sorted(computed) == sorted(
[
(tasks_py, "_load_task_info_snapshot"),
- (cancel_py, "_ended_without_cancel"),
+ (cancel_py, "_paused_reads_failed"),
]
)
@@ -734,3 +737,54 @@ def test_web_reads_unchanged_for_sdk(api_db):
assert info["data"]["auto_recovery"]["auto_resume"]["stop_reason"] == (
"limit_reached"
)
+
+
[email protected]("row", ["scheduled", "exhausted"])
+def test_a_withdrawn_append_reads_the_same_in_the_cancel_answer_and_the_task_read(
+ factory, shared_mode, row
+):
+ """Withdrawing an appended turn fails its START, and the task read then
+ reports the paused row as ``failed`` ("the new message could not be
+ started"). The cancel answer, first and repeated, reports that status
+ too, not the stored ``paused``."""
+ tid, principal = _sdk_task(factory, total=20)
+ if row == "exhausted":
+ run_auto_resume_tick()
+ prepared = _accept(factory, tid)
+ scope = SdkTaskScope(agent_id=principal.agent.id, workforce_id=None)
+ with factory() as db:
+ run_id = str(db.get(TaskExecutionCommand, prepared.command_db_id).target_run_id)
+ task = db.get(Task, tid)
+ cancel = sdk_task_cancel._enqueue_cancel(
+ db, task, int(task.state_version), queued_run_id=run_id
+ )
+ assert (
+ sdk_task_cancel._resolve_queued_cancel_target_sync(
+ task_id=tid, agent_id=principal.agent.id, queued_run_id=run_id
+ )
+ is None
+ )
+ with factory() as db:
+ db.get(TaskExecutionCommand, cancel.command_id).status = "completed"
+ db.commit()
+
+ read = v1_tasks._get_chat_task_sync(tid, principal)
+ first = sdk_task_cancel._load_result_sync(
+ task_id=tid,
+ scope=scope,
+ aimed_run_id=run_id,
+ aimed_at_queued_turn=True,
+ command_completed=True,
+ )
+ repeated = sdk_task_cancel._prepare_cancel_sync(task_id=tid, scope=scope)
+
+ assert read.status == "failed"
+ assert read.error == NEW_TURN
+ assert repeated == first
+ assert first.cancelled and first.cancelled_run_id == run_id
+ assert (first.status, first.run_id, first.state_version, first.error_code) == (
+ read.status,
+ read.run_id,
+ read.state_version,
+ read.error_code,
+ )
diff --git a/tests/web/api/v1/test_v1_cancel_task.py b/tests/web/api/v1/test_v1_cancel_task.py
index 88c1e25a6..50c342f89 100644
--- a/tests/web/api/v1/test_v1_cancel_task.py
+++ b/tests/web/api/v1/test_v1_cancel_task.py
@@ -26,9 +26,11 @@ from xagent.web.services.task_command_transport import (
TaskCommandKind,
TaskCommandRejected,
TaskCommandTaskMissing,
- enqueue_task_command,
+ stage_task_command,
)
from xagent.web.services.task_execution_controller import TaskControlState
+from xagent.web.services.task_start import SdkTaskScope
+from xagent.web.services.task_start_protocol import TaskStartPayload
from ..conftest import (
_admin_headers,
@@ -139,19 +141,56 @@ def _cancel_commands(task_id: int) -> list[TaskExecutionCommand]:
db.close()
-def _queue_start(task_id: int) -> None:
- """Leave an accepted START that no executor has handed off yet."""
+QUEUED_RUN = "run-queued"
+
+
+def _queue_start(
+ task_id: int, *, run_id: str = QUEUED_RUN, turn_id: str = "turn-queued"
+) -> None:
+ """Leave an accepted START that no executor has handed off yet.
+
+ Its payload is the one the acceptance writes: the run and version the row
+ holds now, which is what the START consumer compares the row with.
+ """
db = _direct_db_session()
try:
task = db.query(Task).filter(Task.id == task_id).one()
- enqueue_task_command(
+ start = TaskStartPayload(
+ version=1,
+ run_id=run_id,
+ state_version=int(task.state_version or 0),
+ expected_run_id=task.run_id,
+ turn_id=turn_id,
+ kind="create" if task.status == TaskStatus.PENDING else "append",
+ message="hello",
+ )
+ stage_task_command(
db,
task_id=task_id,
actor_user_id=int(task.user_id),
- command_id="turn-queued",
+ command_id=start.turn_id,
kind=TaskCommandKind.START,
- payload={},
+ payload=start.model_dump(mode="json"),
+ target_run_id=start.run_id,
)
+ db.commit()
+ finally:
+ db.close()
+
+
+def _stop_queued_start(task_id: int) -> None:
+ db = _direct_db_session()
+ try:
+ db.query(TaskExecutionCommand).filter(
+ TaskExecutionCommand.task_id == task_id,
+ TaskExecutionCommand.kind == TaskCommandKind.START.value,
+ ).update(
+ {
+ "status": "failed",
+ "result": {"rejection_reason": "cancelled_before_admission"},
+ }
+ )
+ db.commit()
finally:
db.close()
@@ -290,28 +329,211 @@ def test_states_that_are_not_running_are_refused_without_a_write(values, expecte
"control_state": TaskControlState.COMPLETED.value,
"output": "done",
},
+ {
+ "status": TaskStatus.PAUSED,
+ "control_state": TaskControlState.PAUSED.value,
+ },
],
- ids=["previous_run_cancelled", "previous_run_completed"],
+ ids=["previous_run_cancelled", "previous_run_completed", "previous_run_paused"],
)
-def test_a_queued_turn_is_never_answered_from_the_previous_run(previous):
+def test_a_queued_turn_is_the_target_not_the_run_the_row_still_shows(
+ monkeypatch: pytest.MonkeyPatch, previous
+):
"""While an appended START is queued the row still shows the run before
- it, so answering from the row would report on the wrong turn: a
- cancelled previous run must not read as "already cancelled"."""
+ it. The request aims at the queued run; it is never answered from the
+ row, so a cancelled previous run does not read as "already cancelled".
+ The status and code read as the task read shows them: ``pending`` after
+ a completed or failed run, ``paused`` after a paused one."""
task_id, _agent_id, key = _running_task()
_update(task_id, runner_id=None, lease_expires_at=None, **previous)
_queue_start(task_id)
+ # No executor takes the queued START here, so the command behind it waits.
+ monkeypatch.setattr(sdk_task_cancel, "dispatch_one_task_command", AsyncMock())
+ monkeypatch.setattr(sdk_task_cancel, "SDK_CANCEL_REQUEST_WAIT_SECONDS", 0.1)
+ before = _row(task_id)
+
+ response = _cancel(task_id, key)
+
+ assert response.status_code == 202, response.text
+ task_read = _get(task_id, key)
+ assert response.json() == {
+ "task_id": task_id,
+ "status": task_read["status"],
+ "run_id": before["run_id"],
+ "state_version": before["state_version"],
+ "cancelled_run_id": None,
+ "error_code": task_read["error_code"],
+ }
+ assert task_read["status"] == (
+ "paused" if before["status"] == TaskStatus.PAUSED else "pending"
+ )
+ assert _row(task_id) == before
+ (command,) = _cancel_commands(task_id)
+ assert command.payload["queued_run_id"] == QUEUED_RUN
+ assert command.target_run_id == QUEUED_RUN
+ assert command.command_id.endswith(f":{QUEUED_RUN}")
+
+ # Once the core has stopped the START, the same request reports that
+ # turn as cancelled while the task still reads as the run before it.
+ _stop_queued_start(task_id)
+ monkeypatch.setattr(sdk_task_cancel, "_queued_start", lambda _db, _task_id: None)
+ result = sdk_task_cancel._load_result_sync(
+ task_id=task_id,
+ scope=SdkTaskScope(agent_id=_agent_id, workforce_id=None),
+ aimed_run_id=QUEUED_RUN,
+ aimed_at_queued_turn=True,
+ command_completed=False,
+ )
+ assert result.cancelled and result.cancelled_run_id == QUEUED_RUN
+ assert result.status == before["status"].value
+ assert result.run_id == before["run_id"]
+ assert result.error_code == (
+ "task_cancelled"
+ if before["error_message"] == SDK_CANCEL_ERROR_MESSAGE
+ else None
+ )
+ assert _row(task_id) == before
+
+
+def test_a_queued_new_task_cancelled_by_its_command_answers_for_the_queued_run(
+ monkeypatch: pytest.MonkeyPatch,
+):
+ """The command completed against a new task that never ran: the answer
+ names the queued run as cancelled while the task has no run of its own,
+ and every other field reads as the task read does."""
+ task_id, _agent_id, key = _running_task()
+ _update(
+ task_id,
+ status=TaskStatus.PENDING,
+ control_state="idle",
+ run_id=None,
+ runner_id=None,
+ lease_expires_at=None,
+ )
+ _queue_start(task_id)
+ before = _row(task_id)
+
+ # A cancel is dispatched past a START only while that START waits for
+ # admission; this stands in for that dispatch. The core's write is the
+ # terminal row, and the dispatcher's settlement of the cancelled
+ # admissions then fails the START.
+ async def cancelled_as_a_row(_executor: Any, *, command_db_id: int) -> bool:
+ _update(
+ task_id,
+ state_version=before["state_version"] + 1,
+ **_cancelled_row_values(),
+ )
+ _stop_queued_start(task_id)
+ _settle_cancel_command(command_db_id, status="completed")
+ return True
+
+ monkeypatch.setattr(
+ sdk_task_cancel, "dispatch_one_task_command", cancelled_as_a_row
+ )
+
+ response = _cancel(task_id, key)
+
+ assert response.status_code == 200, response.text
+ assert response.json() == {
+ "task_id": task_id,
+ "status": "failed",
+ "run_id": None,
+ "state_version": before["state_version"] + 1,
+ "cancelled_run_id": QUEUED_RUN,
+ "error_code": "task_cancelled",
+ }
+ task_read = _get(task_id, key)
+ assert (task_read["status"], task_read["error_code"]) == (
+ "failed",
+ "task_cancelled",
+ )
+ assert task_read["run_id"] is None
+
+
+def test_a_queued_turn_whose_cancel_went_stale_while_it_waits_is_busy(
+ monkeypatch: pytest.MonkeyPatch,
+):
+ """The command was rejected because the task moved, and the aimed START
+ is still waiting: nothing was cancelled and a retry can succeed."""
+ task_id, _agent_id, key = _running_task()
+ _update(
+ task_id,
+ status=TaskStatus.COMPLETED,
+ control_state=TaskControlState.COMPLETED.value,
+ runner_id=None,
+ lease_expires_at=None,
+ )
+ _queue_start(task_id)
before = _row(task_id)
+ async def went_stale(_executor: Any, *, command_db_id: int) -> bool:
+ _settle_cancel_command(
+ command_db_id, status="failed", rejection_reason="stale_run"
+ )
+ return True
+
+ monkeypatch.setattr(sdk_task_cancel, "dispatch_one_task_command", went_stale)
+
response = _cancel(task_id, key)
assert response.status_code == 409, response.text
assert _error(response) == "task_busy"
assert _row(task_id) == before
- assert _cancel_commands(task_id) == []
- # The read surface agrees: the queued turn is pending and carries no code.
- read = _get(task_id, key)
- assert read["status"] == "pending"
- assert read["error_code"] is None
+
+
+def test_a_queued_turn_that_failed_to_start_is_not_reported_from_the_previous_run(
+ monkeypatch: pytest.MonkeyPatch,
+):
+ """The aimed turn never started and was not stopped by this cancel; the
+ row is still the cancelled run before it, which is not the answer. The
+ aimed turn is over, so a retry -- which would report that earlier run --
+ is not invited."""
+ task_id, _agent_id, key = _running_task()
+ _update(
+ task_id,
+ status=TaskStatus.FAILED,
+ control_state=TaskControlState.FAILED.value,
+ error_message=SDK_CANCEL_ERROR_MESSAGE,
+ runner_id=None,
+ lease_expires_at=None,
+ )
+ _queue_start(task_id)
+
+ async def start_fails_and_the_cancel_goes_stale(
+ _executor: Any, *, command_db_id: int
+ ) -> bool:
+ db = _direct_db_session()
+ try:
+ commands = db.query(TaskExecutionCommand).filter(
+ TaskExecutionCommand.task_id == task_id
+ )
+ commands.filter(
+ TaskExecutionCommand.kind == TaskCommandKind.START.value
+ ).update(
+ {"status": "failed", "result": {"rejection_reason": "invalid_start"}}
+ )
+ commands.filter(TaskExecutionCommand.id == command_db_id).update(
+ {"status": "failed", "result": {"rejection_reason": "stale_run"}}
+ )
+ db.commit()
+ finally:
+ db.close()
+ return True
+
+ monkeypatch.setattr(
+ sdk_task_cancel,
+ "dispatch_one_task_command",
+ start_fails_and_the_cancel_goes_stale,
+ )
+
+ response = _cancel(task_id, key)
+
+ assert response.status_code == 409, response.text
+ assert _error(response) == "task_not_cancellable"
+ assert (
+ response.json()["error"]["message"]
+ == "The run this cancel was aimed at has already ended."
+ )
def test_a_task_that_reads_failed_after_automatic_resume_gave_up_is_not_cancellable(
@@ -494,6 +716,29 @@ def _cancelled_row_values() -> dict[str, Any]:
}
+def _settle_cancel_command(
+ command_db_id: int, *, status: str, rejection_reason: str | None = None
+) -> None:
+ """End a CANCEL command the way the dispatcher records its outcome."""
+ db = _direct_db_session()
+ try:
+ db.query(TaskExecutionCommand).filter(
+ TaskExecutionCommand.id == command_db_id
+ ).update(
+ {
+ "status": status,
+ "result": (
+ {"rejection_reason": rejection_reason}
+ if rejection_reason is not None
+ else {}
+ ),
+ }
+ )
+ db.commit()
+ finally:
+ db.close()
+
+
def _replace_cancel_core(monkeypatch: pytest.MonkeyPatch, core: Any) -> None:
monkeypatch.setattr(command_execution_service, "cancel_sdk_task_unserialized", core)
@@ -658,7 +903,8 @@ def test_a_turn_queued_while_the_cancel_was_applied_still_reports_the_cancel(
):
"""The command ended, the task was cancelled by another request and a
new turn was queued before this request read the outcome: the run this
- call aimed at is cancelled, so the answer says so."""
+ call aimed at is cancelled, so the answer says so, while the other
+ fields read as the task read does with a turn queued."""
task_id, _agent_id, key = _running_task()
before = _row(task_id)
@@ -679,11 +925,11 @@ def test_a_turn_queued_while_the_cancel_was_applied_still_reports_the_cancel(
assert response.status_code == 200, response.text
assert response.json() == {
"task_id": task_id,
- "status": "failed",
+ "status": "pending",
"run_id": before["run_id"],
"state_version": before["state_version"],
"cancelled_run_id": before["run_id"],
- "error_code": "task_cancelled",
+ "error_code": None,
}
@@ -710,11 +956,11 @@ def test_a_completed_command_is_a_cancel_of_its_aimed_run_even_if_a_turn_is_queu
assert response.status_code == 200, response.text
assert response.json() == {
"task_id": task_id,
- "status": "failed",
+ "status": "pending",
"run_id": before["run_id"],
"state_version": before["state_version"] + 1,
"cancelled_run_id": before["run_id"],
- "error_code": "task_cancelled",
+ "error_code": None,
}
@@ -825,3 +1071,519 @@ def test_the_wait_is_timed_from_before_the_command_is_dispatched(
assert response.status_code == 202, response.text
assert len(polls) == 1
+
+
+# ----- a cancel repeated after it withdrew a queued turn -----
+
+_PREVIOUS_RUNS = {
+ "previous_run_completed": {
+ "status": TaskStatus.COMPLETED,
+ "control_state": TaskControlState.COMPLETED.value,
+ "output": "done",
+ },
+ "previous_run_cancelled": {
+ "status": TaskStatus.FAILED,
+ "control_state": TaskControlState.FAILED.value,
+ "error_message": SDK_CANCEL_ERROR_MESSAGE,
+ },
+ "previous_run_paused": {
+ "status": TaskStatus.PAUSED,
+ "control_state": TaskControlState.PAUSED.value,
+ },
+ "new_task": {"status": TaskStatus.PENDING, "control_state": "idle", "run_id": None},
+}
+
+
+def _dispatch_through_the_core(monkeypatch: pytest.MonkeyPatch, agent_id: int) -> None:
+ """Apply each cancel command with the real core, as its owner would.
+
+ A cancel is dispatched past a queued START only while that START waits for
+ admission, which these tests do not set up; the core and the settlement it
+ leaves behind are the real ones.
+ """
+
+ async def apply(_executor: Any, *, command_db_id: int) -> bool:
+ db = _direct_db_session()
+ try:
+ command = db.get(TaskExecutionCommand, command_db_id)
+ task_id = int(command.task_id)
+ run_id = command.target_run_id
+ version = int(command.target_state_version)
+ queued_run_id = command.payload.get("queued_run_id")
+ finally:
+ db.close()
+ await sdk_task_cancel.cancel_sdk_task_unserialized(
+ task_id=task_id,
+ agent_id=agent_id,
+ expected_run_id=run_id,
+ expected_state_version=version,
+ queued_run_id=queued_run_id,
+ )
+ # A new task's START is settled by the dispatcher once the cancel
+ # has cancelled the row; an appended turn's was stopped by the core.
+ db = _direct_db_session()
+ try:
+ db.query(TaskExecutionCommand).filter(
+ TaskExecutionCommand.task_id == task_id,
+ TaskExecutionCommand.kind == TaskCommandKind.START.value,
+ TaskExecutionCommand.status == "pending",
+ ).update(
+ {
+ "status": "failed",
+ "result": {"rejection_reason": "cancelled_before_admission"},
+ }
+ )
+ db.commit()
+ finally:
+ db.close()
+ _settle_cancel_command(command_db_id, status="completed")
+ return True
+
+ monkeypatch.setattr(sdk_task_cancel, "dispatch_one_task_command", apply)
+
+
[email protected]("previous", list(_PREVIOUS_RUNS))
+def test_a_repeated_cancel_answers_for_the_turn_the_first_one_withdrew(
+ monkeypatch: pytest.MonkeyPatch, previous: str
+):
+ """The withdrawn turn leaves the row on the turn before it, so the repeat
+ cannot read the turn from the row: it is answered from the record of the
+ withdrawal, identically, and adds no second command."""
+ task_id, agent_id, key = _running_task()
+ _update(task_id, runner_id=None, lease_expires_at=None, **_PREVIOUS_RUNS[previous])
+ _queue_start(task_id)
+ _dispatch_through_the_core(monkeypatch, agent_id)
+
+ first = _cancel(task_id, key)
+ again = _cancel(task_id, key)
+
+ assert first.status_code == 200, first.text
+ assert first.json()["cancelled_run_id"] == QUEUED_RUN
+ if previous != "new_task":
+ # The row still shows the turn before the withdrawn one.
+ assert first.json()["run_id"] not in (None, QUEUED_RUN)
+ assert again.status_code == 200, again.text
+ assert again.json() == first.json()
+ assert len(_cancel_commands(task_id)) == 1
+
+
+def _apply_without_completing(monkeypatch: pytest.MonkeyPatch, agent_id: int) -> None:
+ """Apply the cancel with the real core and stop before it is completed.
+
+ The command is claimed as the dispatcher claims it and the core commits
+ its effect -- the withdrawn START, or a new task's cancelled row -- but
+ the separate transaction that marks the command completed (and, for a
+ new task, settles its START) has not run yet.
+ """
+
+ async def apply(_executor: Any, *, command_db_id: int) -> bool:
+ db = _direct_db_session()
+ try:
+ command = db.get(TaskExecutionCommand, command_db_id)
+ command.status = "processing"
+ command.attempt_count = 1
+ db.commit()
+ task_id = int(command.task_id)
+ run_id = command.target_run_id
+ version = int(command.target_state_version)
+ queued_run_id = command.payload.get("queued_run_id")
+ finally:
+ db.close()
+ await sdk_task_cancel.cancel_sdk_task_unserialized(
+ task_id=task_id,
+ agent_id=agent_id,
+ expected_run_id=run_id,
+ expected_state_version=version,
+ queued_run_id=queued_run_id,
+ )
+ return True
+
+ monkeypatch.setattr(sdk_task_cancel, "dispatch_one_task_command", apply)
+
+
+def _complete_applied_cancel(task_id: int, command_db_id: int) -> None:
+ """The step ``_apply_without_completing`` left out: a new task's START is
+ settled and the command is marked completed."""
+ db = _direct_db_session()
+ try:
+ db.query(TaskExecutionCommand).filter(
+ TaskExecutionCommand.task_id == task_id,
+ TaskExecutionCommand.kind == TaskCommandKind.START.value,
+ TaskExecutionCommand.status == "pending",
+ ).update(
+ {
+ "status": "failed",
+ "result": {"rejection_reason": "cancelled_before_admission"},
+ }
+ )
+ db.commit()
+ finally:
+ db.close()
+ _settle_cancel_command(command_db_id, status="completed")
+
+
+def _withdrawn_before_completion(
+ monkeypatch: pytest.MonkeyPatch, previous: str
+) -> tuple[int, str, Any, TaskExecutionCommand]:
+ """Queue a turn, cancel it, and leave the cancel applied but processing."""
+ task_id, agent_id, key = _running_task()
+ _update(task_id, runner_id=None, lease_expires_at=None, **_PREVIOUS_RUNS[previous])
+ _queue_start(task_id)
+ _apply_without_completing(monkeypatch, agent_id)
+ monkeypatch.setattr(sdk_task_cancel, "SDK_CANCEL_REQUEST_WAIT_SECONDS", 0.2)
+ first = _cancel(task_id, key)
+ (command,) = _cancel_commands(task_id)
+ assert command.status == "processing"
+ return task_id, key, first, command
+
+
[email protected]("previous", list(_PREVIOUS_RUNS))
+def test_a_repeat_while_the_withdrawing_cancel_is_still_processing_rejoins_it(
+ monkeypatch: pytest.MonkeyPatch, previous: str
+):
+ """The cancel withdrew the queued turn but is not marked completed yet.
+ A repeat in that interval finds the same command, adds none, and answers
+ exactly as the first call did at the same point: an appended turn reads
+ as withdrawn, a new task as still being cancelled; once the command
+ completes the new task reads as cancelled for the queued run."""
+ task_id, key, first, command = _withdrawn_before_completion(monkeypatch, previous)
+ dispatched = AsyncMock()
+ monkeypatch.setattr(sdk_task_cancel, "dispatch_one_task_command", dispatched)
+
+ again = _cancel(task_id, key)
+
+ assert dispatched.await_args.kwargs["command_db_id"] == command.id
+ assert [c.id for c in _cancel_commands(task_id)] == [command.id]
+ assert again.status_code == first.status_code, again.text
+ assert again.json() == first.json()
+ if previous != "new_task":
+ assert first.status_code == 200, first.text
+ assert first.json()["cancelled_run_id"] == QUEUED_RUN
+ # The row still shows the turn before the withdrawn one.
+ assert first.json()["run_id"] not in (None, QUEUED_RUN)
+ return
+ assert first.status_code == 202, first.text
+ assert first.json()["cancelled_run_id"] is None
+ _complete_applied_cancel(task_id, command.id)
+
+ after_completion = _cancel(task_id, key)
+
+ assert after_completion.status_code == 200, after_completion.text
+ assert after_completion.json()["run_id"] is None
+ assert after_completion.json()["cancelled_run_id"] == QUEUED_RUN
+ assert after_completion.json()["error_code"] == "task_cancelled"
+ assert len(_cancel_commands(task_id)) == 1
+
+
+def test_a_repeat_while_processing_after_the_turn_before_resumed_cancels_that_turn(
+ monkeypatch: pytest.MonkeyPatch,
+):
+ """The withdrawing cancel is still processing when the paused turn before
+ it is resumed on a newer version. That command no longer describes the
+ row, so the repeat is not answered with the withdrawal: it aims a new
+ command at the running turn."""
+ task_id, key, _first, command = _withdrawn_before_completion(
+ monkeypatch, "previous_run_paused"
+ )
+ resumed = _row(task_id)
+ _update(
+ task_id,
+ status=TaskStatus.RUNNING,
+ control_state=TaskControlState.RUNNING.value,
+ state_version=resumed["state_version"] + 1,
+ )
+ monkeypatch.setattr(sdk_task_cancel, "dispatch_one_task_command", AsyncMock())
+ monkeypatch.setattr(sdk_task_cancel, "SDK_CANCEL_REQUEST_WAIT_SECONDS", 0.1)
+
+ response = _cancel(task_id, key)
+
+ assert response.status_code == 202, response.text
+ assert response.json()["status"] == "running"
+ assert response.json()["run_id"] == resumed["run_id"]
+ assert response.json()["cancelled_run_id"] is None
+ commands = sorted(_cancel_commands(task_id), key=lambda c: c.id)
+ assert len(commands) == 2
+ assert commands[0].id == command.id
+ assert commands[-1].target_run_id == resumed["run_id"]
+
+
+def test_a_withdrawal_does_not_answer_once_the_turn_before_runs_again(
+ monkeypatch: pytest.MonkeyPatch,
+):
+ """After the withdrawal the turn before it was resumed and runs again on
+ a newer version. The record of the withdrawal no longer describes the
+ row, so the cancel aims at the running turn instead of answering that
+ the withdrawn one is cancelled while the running one goes on."""
+ task_id, agent_id, key = _running_task()
+ _update(
+ task_id,
+ runner_id=None,
+ lease_expires_at=None,
+ status=TaskStatus.PAUSED,
+ control_state=TaskControlState.PAUSED.value,
+ )
+ _queue_start(task_id)
+ _dispatch_through_the_core(monkeypatch, agent_id)
+ assert _cancel(task_id, key).status_code == 200
+ resumed = _row(task_id)
+ _update(
+ task_id,
+ status=TaskStatus.RUNNING,
+ control_state=TaskControlState.RUNNING.value,
+ state_version=resumed["state_version"] + 1,
+ )
+ monkeypatch.setattr(sdk_task_cancel, "dispatch_one_task_command", AsyncMock())
+ monkeypatch.setattr(sdk_task_cancel, "SDK_CANCEL_REQUEST_WAIT_SECONDS", 0.1)
+
+ response = _cancel(task_id, key)
+
+ assert response.status_code == 202, response.text
+ assert response.json()["status"] == "running"
+ assert response.json()["run_id"] == resumed["run_id"]
+ assert response.json()["cancelled_run_id"] is None
+ commands = _cancel_commands(task_id)
+ assert len(commands) == 2
+ assert commands[-1].target_run_id == resumed["run_id"]
+
+
+def _fail_newest_start(task_id: int, outcome: dict[str, Any]) -> None:
+ db = _direct_db_session()
+ try:
+ db.query(TaskExecutionCommand).filter(
+ TaskExecutionCommand.task_id == task_id,
+ TaskExecutionCommand.kind == TaskCommandKind.START.value,
+ ).update(outcome)
+ db.commit()
+ finally:
+ db.close()
+
+
+ "start_outcome",
+ [
+ {"status": "failed", "result": {"rejection_reason": "invalid_start"}},
+ # Stopped by a control other than this endpoint's cancel.
+ {
+ "status": "failed",
+ "result": {"rejection_reason": "cancelled_before_admission"},
+ },
+ ],
+ ids=["failed_itself", "stopped_by_another_control"],
+)
+def test_a_start_that_no_cancel_of_this_endpoint_withdrew_is_not_a_cancellation(
+ start_outcome,
+):
+ """The newest START failed and the row shows the completed turn before
+ it. With no SDK cancel aimed at that START's run the failure is not a
+ cancellation, so the call is refused as it would be on arrival."""
+ task_id, _agent_id, key = _running_task()
+ _update(
+ task_id,
+ runner_id=None,
+ lease_expires_at=None,
+ **_PREVIOUS_RUNS["previous_run_completed"],
+ )
+ _queue_start(task_id)
+ _fail_newest_start(task_id, start_outcome)
+ before = _row(task_id)
+
+ response = _cancel(task_id, key)
+
+ assert response.status_code == 409, response.text
+ assert _error(response) == "task_not_cancellable"
+ assert _row(task_id) == before
+ assert _cancel_commands(task_id) == []
+
+
+def test_a_cancel_not_completed_yet_for_a_stopped_start_is_rejoined(
+ monkeypatch: pytest.MonkeyPatch,
+):
+ """The newest START was stopped and this endpoint's cancel aimed at its
+ run is still pending. The call rejoins that command instead of judging
+ the row or staging another one, and answers as the command's own wait
+ does: at the deadline a START stopped before it began is that turn
+ cancelled."""
+ task_id, _agent_id, key = _running_task()
+ _update(
+ task_id,
+ runner_id=None,
+ lease_expires_at=None,
+ **_PREVIOUS_RUNS["previous_run_completed"],
+ )
+ _queue_start(task_id)
+ db = _direct_db_session()
+ try:
+ task = db.get(Task, task_id)
+ pending = sdk_task_cancel._enqueue_cancel(
+ db, task, int(task.state_version), queued_run_id=QUEUED_RUN
+ )
+ finally:
+ db.close()
+ _fail_newest_start(
+ task_id,
+ {
+ "status": "failed",
+ "result": {"rejection_reason": "cancelled_before_admission"},
+ },
+ )
+ dispatched = AsyncMock()
+ monkeypatch.setattr(sdk_task_cancel, "dispatch_one_task_command", dispatched)
+ monkeypatch.setattr(sdk_task_cancel, "SDK_CANCEL_REQUEST_WAIT_SECONDS", 0.1)
+ before = _row(task_id)
+
+ response = _cancel(task_id, key)
+
+ assert response.status_code == 200, response.text
+ assert response.json()["run_id"] == before["run_id"]
+ assert response.json()["cancelled_run_id"] == QUEUED_RUN
+ assert dispatched.await_args.kwargs["command_db_id"] == pending.command_id
+ assert [c.id for c in _cancel_commands(task_id)] == [pending.command_id]
+ assert _row(task_id) == before
+
+
+def test_a_cancel_after_a_withdrawn_turn_and_a_newer_started_one_answers_for_the_row(
+ monkeypatch: pytest.MonkeyPatch,
+):
+ """A turn was withdrawn, then another was appended and is running: the
+ newest START is the row's own run, so the cancel aims at the running
+ turn and is not answered with the withdrawal."""
+ task_id, agent_id, key = _running_task()
+ _update(
+ task_id,
+ runner_id=None,
+ lease_expires_at=None,
+ **_PREVIOUS_RUNS["previous_run_completed"],
+ )
+ _queue_start(task_id)
+ _dispatch_through_the_core(monkeypatch, agent_id)
+ assert _cancel(task_id, key).status_code == 200
+ _queue_start(task_id, run_id="run-second", turn_id="turn-second")
+ _update(
+ task_id,
+ status=TaskStatus.RUNNING,
+ control_state=TaskControlState.RUNNING.value,
+ run_id="run-second",
+ )
+ db = _direct_db_session()
+ try:
+ db.query(TaskExecutionCommand).filter(
+ TaskExecutionCommand.task_id == task_id,
+ TaskExecutionCommand.command_id == "turn-second",
+ ).update({"status": "completed"})
+ db.commit()
+ finally:
+ db.close()
+ monkeypatch.setattr(sdk_task_cancel, "dispatch_one_task_command", AsyncMock())
+ monkeypatch.setattr(sdk_task_cancel, "SDK_CANCEL_REQUEST_WAIT_SECONDS", 0.1)
+
+ response = _cancel(task_id, key)
+
+ assert response.status_code == 202, response.text
+ assert response.json()["run_id"] == "run-second"
+ assert response.json()["cancelled_run_id"] is None
+ commands = _cancel_commands(task_id)
+ assert len(commands) == 2
+ assert commands[-1].target_run_id == "run-second"
+
+
+# ----- a queued START that can no longer hand off is not the target -----
+
+
+ "moved",
+ ["running_on_another_run", "waiting_on_another_run"],
+)
+def test_a_queued_start_that_can_no_longer_hand_off_is_not_the_cancel_target(
+ monkeypatch: pytest.MonkeyPatch, moved: str
+):
+ """The row was moved to another run after the START was accepted (an
+ automatic resume won the race), so the START never opens its run. A
+ running row is cancelled as the running run it is; one waiting for an
+ answer is refused as busy, as it would be with no START at all."""
+ task_id, _agent_id, key = _running_task()
+ _update(
+ task_id,
+ runner_id=None,
+ lease_expires_at=None,
+ **_PREVIOUS_RUNS["previous_run_completed"],
+ )
+ _queue_start(task_id)
+ row_values: dict[str, Any] = {
+ "run_id": "run-first",
+ "state_version": _row(task_id)["state_version"] + 1,
+ }
+ if moved == "running_on_another_run":
+ row_values.update(
+ status=TaskStatus.RUNNING, control_state=TaskControlState.RUNNING.value
+ )
+ else:
+ row_values.update(
+ status=TaskStatus.WAITING_FOR_USER,
+ control_state=TaskControlState.WAITING_FOR_USER.value,
+ )
+ _update(task_id, **row_values)
+ monkeypatch.setattr(sdk_task_cancel, "dispatch_one_task_command", AsyncMock())
+ monkeypatch.setattr(sdk_task_cancel, "SDK_CANCEL_REQUEST_WAIT_SECONDS", 0.1)
+ before = _row(task_id)
+
+ response = _cancel(task_id, key)
+
+ assert _row(task_id) == before
+ if moved == "waiting_on_another_run":
+ assert response.status_code == 409, response.text
+ assert _error(response) == "task_busy"
+ assert _cancel_commands(task_id) == []
+ return
+ assert response.status_code == 202, response.text
+ assert response.json()["run_id"] == "run-first"
+ (command,) = _cancel_commands(task_id)
+ assert "queued_run_id" not in command.payload
+ assert command.target_run_id == "run-first"
+
+
+# ----- how the wait ends for a queued turn -----
+
+
+def test_a_cancel_whose_queued_turn_started_before_the_deadline_is_accepted(
+ monkeypatch: pytest.MonkeyPatch,
+):
+ """At the deadline the START has been handed off and the row shows the
+ turn running on the aimed run: the command is not applied yet, so the
+ answer is 202 for the running turn, not "already ended"."""
+ task_id, _agent_id, key = _running_task()
+ _update(
+ task_id,
+ runner_id=None,
+ lease_expires_at=None,
+ **_PREVIOUS_RUNS["previous_run_completed"],
+ )
+ _queue_start(task_id)
+
+ async def handed_off(_executor: Any, *, command_db_id: int) -> bool:
+ _update(
+ task_id,
+ status=TaskStatus.RUNNING,
+ control_state=TaskControlState.RUNNING.value,
+ run_id=QUEUED_RUN,
+ state_version=_row(task_id)["state_version"] + 1,
+ )
+ db = _direct_db_session()
+ try:
+ db.query(TaskExecutionCommand).filter(
+ TaskExecutionCommand.task_id == task_id,
+ TaskExecutionCommand.kind == TaskCommandKind.START.value,
+ ).update({"status": "completed"})
+ db.commit()
+ finally:
+ db.close()
+ return False
+
+ monkeypatch.setattr(sdk_task_cancel, "dispatch_one_task_command", handed_off)
+ monkeypatch.setattr(sdk_task_cancel, "SDK_CANCEL_REQUEST_WAIT_SECONDS", 0.1)
+
+ response = _cancel(task_id, key)
+
+ assert response.status_code == 202, response.text
+ assert response.json()["status"] == "running"
+ assert response.json()["run_id"] == QUEUED_RUN
+ assert response.json()["cancelled_run_id"] is None
diff --git a/tests/web/services/test_queued_start_cancel_admission.py b/tests/web/services/test_queued_start_cancel_admission.py
index c1663464c..e88975cf6 100644
--- a/tests/web/services/test_queued_start_cancel_admission.py
+++ b/tests/web/services/test_queued_start_cancel_admission.py
@@ -19,13 +19,16 @@ from xagent.web.models.task_runtime_secret import TaskRuntimeSecret
from xagent.web.services import (
a2a_task_cancel,
external_task_cancel,
+ sdk_task_cancel,
task_command_execution,
)
from xagent.web.services import task_command_transport as transport
+from xagent.web.services import task_execution_admission as admission
from xagent.web.services.chat_history_service import (
DELIVERY_DISPATCHED,
DELIVERY_FAILED,
)
+from xagent.web.services.task_admission_execution import AdmissionWaiting
from xagent.web.services.task_orchestrator import TaskTurnOrchestrator, TaskTurnPayload
from xagent.web.services.task_runtime_secrets import stage_runtime_values
@@ -38,7 +41,7 @@ from .test_task_execution_admission import (
)
[email protected]("scope", ["a2a", "external"])
[email protected]("scope", ["a2a", "external", "sdk"])
async def test_cancel_settles_capacity_waiting_start(scope, host, monkeypatch):
"""External runtime input is the supported host contract for its START."""
monkeypatch.setenv("ENCRYPTION_KEY", Fernet.generate_key().decode())
@@ -49,6 +52,7 @@ async def test_cancel_settles_capacity_waiting_start(scope, host, monkeypatch):
monkeypatch.setattr(
external_task_cancel, "get_session_local", lambda: host.sessions
)
+ monkeypatch.setattr(sdk_task_cancel, "get_session_local", lambda: host.sessions)
monkeypatch.setattr(
"xagent.web.services.task_event_bridge.get_task_event_bridge", lambda: Mock()
)
@@ -116,18 +120,26 @@ async def test_cancel_settles_capacity_waiting_start(scope, host, monkeypatch):
)
assert executed == []
- payload = {"agent_id": agent_id, "target_state_version": 0}
- if scope == "external":
- payload.update({"scope": "external", "turn_id": turn_id})
with host.sessions() as db:
- cancel = transport.enqueue_task_command(
- db,
- task_id=task_id,
- actor_user_id=host.user,
- command_id=f"cancel:{task_id}:0",
- kind=transport.TaskCommandKind.CANCEL,
- payload=payload,
- )
+ if scope == "sdk":
+ # Built as the endpoint builds it, which aims the command at
+ # the queued turn's run.
+ queued_run_id = db.get(TaskExecutionCommand, start_id).target_run_id
+ cancel = sdk_task_cancel._enqueue_cancel(
+ db, db.get(Task, task_id), 0, queued_run_id=queued_run_id
+ )
+ else:
+ payload = {"agent_id": agent_id, "target_state_version": 0}
+ if scope == "external":
+ payload.update({"scope": "external", "turn_id": turn_id})
+ cancel = transport.enqueue_task_command(
+ db,
+ task_id=task_id,
+ actor_user_id=host.user,
+ command_id=f"cancel:{task_id}:0",
+ kind=transport.TaskCommandKind.CANCEL,
+ payload=payload,
+ )
assert await transport.dispatch_one_task_command(
task_command_execution.execute_durable_task_command,
@@ -152,11 +164,11 @@ async def test_cancel_settles_capacity_waiting_start(scope, host, monkeypatch):
}
assert (task.status, task.error_message) == (
TaskStatus.FAILED,
- (
- "Stopped by the visitor."
- if scope == "external"
- else "Task canceled by A2A client."
- ),
+ {
+ "external": "Stopped by the visitor.",
+ "a2a": "Task canceled by A2A client.",
+ "sdk": sdk_task_cancel.SDK_CANCEL_ERROR_MESSAGE,
+ }[scope],
)
assert queued_start.status == "failed"
assert queued_start.error == "Task command stopped by a control request."
@@ -181,3 +193,83 @@ async def test_cancel_settles_capacity_waiting_start(scope, host, monkeypatch):
execute, command_db_id=start_id
)
assert executed == []
+
+
+async def test_an_appended_turn_sent_back_to_the_admission_queue_is_withdrawn(
+ host, monkeypatch
+):
+ """A START that was attempted and sent back to wait for capacity still
+ waits for admission, so the cancel overtakes it and withdraws the turn,
+ leaving the task as the turn before it."""
+ monkeypatch.setenv("ENCRYPTION_KEY", Fernet.generate_key().decode())
+ monkeypatch.setattr(
+ task_command_execution, "get_session_local", lambda: host.sessions
+ )
+ monkeypatch.setattr(sdk_task_cancel, "get_session_local", lambda: host.sessions)
+ monkeypatch.setattr(
+ "xagent.web.services.task_event_bridge.get_task_event_bridge", lambda: Mock()
+ )
+ with host.sessions() as db:
+ agent = Agent(user_id=host.user, name="Appended sdk cancellation")
+ db.add(agent)
+ db.flush()
+ task = Task(
+ user_id=host.user,
+ agent_id=agent.id,
+ source="sdk",
+ title="Appended sdk task",
+ status=TaskStatus.COMPLETED,
+ control_state="completed",
+ run_id="run-before",
+ state_version=3,
+ )
+ db.add(task)
+ db.flush()
+ task_id = int(task.id)
+ accepted = TaskTurnOrchestrator.claim_append_turn_no_commit(
+ db,
+ task_id=task_id,
+ task_owner_user_id=host.user,
+ payload=TaskTurnPayload("Append the turn", turn_id="turn-appended"),
+ )
+ db.commit()
+ start_id = int(accepted.command_db_id)
+
+ async def no_capacity(_command):
+ raise AdmissionWaiting("Waiting for execution admission")
+
+ assert await transport.dispatch_one_task_command(
+ no_capacity, command_db_id=start_id
+ )
+ # The owner drains and releases its tickets: the command waits again.
+ with host.sessions() as db, db.begin():
+ admission._release_tickets(db, TaskAdmissionTicket.command_id == start_id)
+ with host.sessions() as db:
+ start = db.get(TaskExecutionCommand, start_id)
+ ticket = db.get(TaskAdmissionTicket, start_id)
+ assert (start.status, start.attempt_count) == ("pending", 1)
+ assert ticket.owner_attempt_id is None
+ queued_run_id = start.target_run_id
+ cancel = sdk_task_cancel._enqueue_cancel(
+ db, db.get(Task, task_id), 3, queued_run_id=queued_run_id
+ )
+
+ assert await transport.dispatch_one_task_command(
+ task_command_execution.execute_durable_task_command,
+ command_db_id=cancel.command_id,
+ )
+
+ with host.sessions() as db:
+ start = db.get(TaskExecutionCommand, start_id)
+ task = db.get(Task, task_id)
+ assert db.get(TaskExecutionCommand, cancel.command_id).status == "completed"
+ assert start.status == "failed"
+ assert start.result == {"rejection_reason": "cancelled_before_admission"}
+ assert (task.status, task.run_id, task.state_version, task.error_message) == (
+ TaskStatus.COMPLETED,
+ "run-before",
+ 3,
+ None,
+ )
+ delivery = db.query(TaskChatMessage).filter_by(turn_id="turn-appended").one()
+ assert delivery.delivery_status == DELIVERY_FAILED