Enterprise agent platform for building, deploying, and operating AI teammates.
feat(v1): cancel a turn that is still queued to start (#3791)
Alex Liu00cb0355e0d0
diff --git a/docs/deployment.md b/docs/deployment.mdindex 045d6b3da..8eb867f0c 100644--- a/docs/deployment.md+++ b/docs/deployment.md@@ -1011,3 +1011,26 @@ A running task created through `/v1` can now be cancelled. The cancelled task re### Rollback- A task cancelled before the rollback stays `failed`; an older process does not report `error_code`, so that field is absent from its reads. The cancel endpoint answers 404 again.++## 2026-11-13 — SDK tasks: cancelling a turn that is still queued to start++`POST /v1/chat/tasks/{task_id}/cancel` now also cancels a turn whose start is still queued, instead of answering 409 `task_busy`.++### Client-visible changes++- A new task cancelled before it starts answers 200 and reads `failed` with `error_code` `task_cancelled` and no `run_id`.+- A turn appended to a finished or paused task and cancelled before it starts is withdrawn: the call answers 200 with `cancelled_run_id` naming the withdrawn run, and the task keeps reading as the turn before it. Calling again answers the same 200. A task paused by an interruption whose appended turn was withdrawn reads `failed` on `GET`, as it does when that turn cannot be started, and the answer reads `failed` too; a task paused on request keeps reading `paused`.+- When the task has moved since the turn was queued (another run, or a newer version of the same one), so the queued turn can no longer start, the cancel acts on the task as it is now: a running task has that running run cancelled, and the response names it as `cancelled_run_id`.+- A task that is waiting for an answer or paused, with no turn queued to start, still answers 409 `task_busy`.+- A cancel that reaches a queued turn after it started cancels it as a running task.+- A turn queued before it has a run of its own still answers 409 `task_busy`.+- While a later turn is queued after a completed or failed run, a 200 answer reads `status` `pending` with a null `error_code`, as `GET /v1/chat/tasks/{task_id}` does; `cancelled_run_id` still names the cancelled run. The earlier entry's 200 answered `failed` with `task_cancelled` in that case.++### Deployment impact++- No migration and no configuration.+- Mixed versions: a cancel aimed at a queued turn names that turn's run, so an older process rejects it as a changed run without writing anything and the endpoint answers 409 `task_busy`, as before. Once the turn has started, a process that predates the cancel endpoint answers 503, as in the earlier entry.++### Rollback++- A queued turn answers 409 `task_busy` again. Turns already withdrawn stay withdrawn.diff --git a/src/xagent/web/api/v1/tasks.py b/src/xagent/web/api/v1/tasks.pyindex eba3e23c3..991f48430 100644--- a/src/xagent/web/api/v1/tasks.py+++ b/src/xagent/web/api/v1/tasks.py@@ -107,7 +107,7 @@ from ...services.task_orchestrator import (TaskTurnFileBindingError,TaskTurnNotFoundError,)-from ...services.task_start_protocol import has_queued_start+from ...services.task_start_protocol import QUEUED_START_STATUSES, has_queued_startfrom ...services.uploaded_file_store import (UploadedFileVersionSnapshot,snapshot_uploaded_file_version,@@ -586,7 +586,7 @@ class _TaskInfoSnapshot:# :func:`effective_task_snapshot` and the GET response apply the view.view: TerminalView | None = None# Set when an appended turn's START is queued and the row still holds the- # previous turn (see ``_QUEUED_START_STATUSES``). Applied like ``view``.+ # previous turn (see ``QUEUED_START_STATUSES``). Applied like ``view``.start_queued: bool = False# Read from the same row as ``status`` and ``run_id``: the stored# structured result of the run the row shows, and whether the task has a@@ -605,15 +605,6 @@ class _TaskInfoSnapshot:accept_partial: bool = False-# Terminal statuses an append is accepted from. With shared execution the row-# keeps one of them, and the previous turn's output and error, until the-# queued START is handed off, so a caller waiting for a terminal status would-# take the previous turn as the new one. A PAUSED row keeps reading as paused,-# which callers already wait on. A RUNNING or WAITING_FOR_USER row has moved-# since the START was queued, and a PENDING row already reads as pending.-_QUEUED_START_STATUSES = (TaskStatus.COMPLETED, TaskStatus.FAILED)--def effective_task_snapshot(snapshot: _TaskInfoSnapshot) -> _TaskInfoSnapshot:"""``snapshot`` as the caller must read it: ``pending`` with no output orerror while an appended turn's START is queued, ``failed`` when it carries@@ -849,7 +840,7 @@ def _load_task_info_snapshot(db, task)view: TerminalView | None = None- start_queued = queued_before_row and task.status in _QUEUED_START_STATUSES+ start_queued = queued_before_row and task.status in QUEUED_START_STATUSES# The view re-checks both inside its own statement; this only skips# the query for reads that cannot be terminal.if (@@ -1540,12 +1531,22 @@ async def cancel_chat_task(to stop. 200 means the task is cancelled, including when an earliercall already cancelled it; if a new turn has started since, ``status``and ``run_id`` describe that turn and ``cancelled_run_id`` names the- cancelled one. 202 means the cancel is recorded but not applied yet,- and calling again reaches the same request.-- Only a running task is cancelled. One that is waiting for an answer,- paused, or whose turn is still queued to start answers 409- ``task_busy`` and is left untouched.+ cancelled one; while a later turn is queued after a completed or failed+ run the task reads ``pending`` with a null ``error_code``, as ``GET``+ reads it. 202 means the cancel is recorded but not applied yet, and+ calling again reaches the same request.++ A turn still queued to start is cancelled too. For a new task that+ leaves the task cancelled with no ``run_id``. For a turn appended to a+ finished or paused task it only withdraws that turn: the task keeps+ reading as the turn before it, and ``cancelled_run_id`` names the+ withdrawn run. A queued turn that can no longer start, because the+ task has moved since it was queued (another run, or a newer version of+ the same one), is not the target: a running run is cancelled, and any+ other state is answered as if no turn were queued.++ A task that is waiting for an answer or paused, with no turn queued to+ start, answers 409 ``task_busy`` and is left untouched.Raises:V1ApiError 401: missing / invalid / revoked key.diff --git a/src/xagent/web/schemas/v1.py b/src/xagent/web/schemas/v1.pyindex 4fd6e40c2..45daf489b 100644--- a/src/xagent/web/schemas/v1.py+++ b/src/xagent/web/schemas/v1.py@@ -1244,9 +1244,14 @@ class CancelTaskResponse(BaseModel):200: the run named by ``cancelled_run_id`` is cancelled, whether thiscall did it or an earlier one already had. The other fields describe- the task now: it reads ``failed`` with ``error_code='task_cancelled'``- unless a new turn has started since, in which case ``status``,- ``run_id``, ``state_version`` and ``error_code`` describe that turn.+ the task now: ``failed`` with ``error_code='task_cancelled'``, unless a+ new turn has started since, in which case ``status``, ``run_id``,+ ``state_version`` and ``error_code`` describe that turn, or a later+ turn is queued to start after a completed or failed run, in which case+ it reads ``pending`` with a null ``error_code``, as+ ``GET /v1/chat/tasks/{task_id}`` does. A turn+ withdrawn before it started leaves the task showing the turn before+ it, so ``cancelled_run_id`` differs from ``run_id``.202: the cancel was recorded but not applied yet; ``status`` is stillthe task's current one and ``cancelled_run_id`` is null. Calling againreaches the same request.@@ -1256,9 +1261,14 @@ class CancelTaskResponse(BaseModel):status: str = Field(...,description=(- "Task status as stored: 'failed' once cancelled, the status of a "- "turn that started after the cancel, or, on 202, the status the "- "cancel is still being applied to."+ "Task status as stored: 'failed' once cancelled, the previous "+ "turn's status when a queued turn was withdrawn before it started, "+ "the status of a turn that started after the cancel, or, on 202, "+ "the status the cancel is still being applied to. While a turn is "+ "queued after a completed or failed run it reads 'pending', and a "+ "task paused by an interruption that automatic resume will no "+ "longer pick up (for example, after its appended turn was "+ "withdrawn) reads 'failed', as GET /v1/chat/tasks/{task_id} does."),)run_id: Optional[str] = Field(..., description="Current execution run identity.")@@ -1268,15 +1278,18 @@ class CancelTaskResponse(BaseModel):cancelled_run_id: Optional[str] = Field(...,description=(- "The run that was cancelled. Null while the cancel has not been applied."+ "The run that was cancelled. Differs from run_id when a queued "+ "turn was withdrawn before it started: the task then still shows "+ "the turn before it. Null while the cancel has not been applied."),)error_code: Optional[str] = Field(...,description=("'task_cancelled' while the task reads as cancelled, otherwise "- "null (before the cancel is applied, or once a later turn has "- "started); the same value GET /v1/chat/tasks/{task_id} reports."+ "null (before the cancel is applied, while a later turn is "+ "queued, or once a later turn has started); the same value "+ "GET /v1/chat/tasks/{task_id} reports."),)diff --git a/src/xagent/web/services/sdk_task_cancel.py b/src/xagent/web/services/sdk_task_cancel.pyindex 3d4ddf779..8d54dd9b3 100644--- a/src/xagent/web/services/sdk_task_cancel.py+++ b/src/xagent/web/services/sdk_task_cancel.py@@ -16,30 +16,45 @@ run was stopped by an explicit ``cancel_task`` callsettlement did. Whichever writer lands first owns the terminal row, andthe other one changes nothing.-Only a RUNNING task is cancelled here. A task with a turn still queued to-start, one waiting for the caller's answer and a paused one are refused as-busy, without a write.+A turn still queued to start is a target of its own: until its START is+handed off the task row shows the turn before it, so the request names the+queued run and the core judges that run, not the row. Cancelling it before+it starts fails its START and, for an appended turn, leaves the row -- the+previous turn's result -- untouched. A queued START is a target only while+it can still hand off (``start_can_hand_off``): one the START consumer would+reject because the row moved on is treated as absent, and the row is judged+as it stands. A repeat of a cancel that withdrew the newest turn rejoins+that cancel's command and is answered for that turn+(``_cancel_of_newest_turn``), since the row no longer shows it.++Besides a queued turn only a RUNNING task is cancelled here. One waiting+for the caller's answer and a paused one are refused as busy, without a+write."""from __future__ import annotationsimport asyncioimport logging-from dataclasses import dataclass, replace+from dataclasses import dataclassfrom time import monotonicfrom typing import TYPE_CHECKING, Any-from sqlalchemy import func, update+from sqlalchemy import func, or_, updatefrom sqlalchemy.orm import Sessionfrom ..models.chat_message import TaskChatMessagefrom ..models.database import get_session_localfrom ..models.task import Task, TaskStatus+from ..models.task_command import TaskExecutionCommandfrom .db_runtime import run_db_io_cancellation_safefrom .task_command_transport import (COMMAND_COMPLETED,COMMAND_FAILED,+ COMMAND_PENDING,+ COMMAND_PROCESSING,EnqueuedTaskCommand,+ TaskCommandDeferred,TaskCommandKind,TaskCommandRejected,TaskCommandTaskMissing,@@ -48,6 +63,11 @@ from .task_command_transport import (load_task_command,retry_failed_task_command,)+from .task_execution_admission import (+ STOPPED_BEFORE_START,+ STOPPED_BEFORE_START_REASON,+ waiting_admission,+)from .task_execution_controller import (StaleTaskRunError,TaskControlState,@@ -170,7 +190,10 @@ class _CancelCommand:command_db_id: int# The run the command was persisted against: the one this request# answers for, whatever run the row shows by the time it is read back.+ # For a turn queued to start it is that turn's run, which the row does+ # not show until the START is handed off.aimed_run_id: str | None+ aimed_at_queued_turn: booldef _task_run_id(task: Task) -> str | None:@@ -178,48 +201,214 @@ def _task_run_id(task: Task) -> str | None:return str(run_id) if run_id is not None else None-def _result(task: Task) -> SdkTaskCancelResult:- cancelled = is_sdk_cancelled(task.status, task.error_message)- run_id = _task_run_id(task)+def _paused_reads_failed(db: Session, task: Task) -> bool:+ """Whether the task read reports this PAUSED row as ``failed``.++ The row stays PAUSED; automatic resume gave up, or the turn appended to+ it could not be started, and ``status_only_terminal_view`` is what the+ read surface derives ``failed`` from.+ """++ from .task_auto_resume import status_only_terminal_view++ if (+ task.status != TaskStatus.PAUSED+ or task.control_state != TaskControlState.PAUSED.value+ ):+ return False+ return status_only_terminal_view(db, int(task.id)) is not None+++def _result(+ db: Session,+ task: Task,+ *,+ cancelled: bool,+ cancelled_run_id: str | None = None,+ queued: bool = False,+) -> SdkTaskCancelResult:+ """The task row as the cancel response reports it.++ ``cancelled`` is the caller's verdict on the run it was asked about,+ which is not always the run the row holds. The status is the one the task+ read reports: a completed or failed row reads as ``pending`` with no code+ while a START is waiting (``queued``; ``QUEUED_START_STATUSES``), and a+ paused row reads as ``failed`` when ``_paused_reads_failed``.+ """++ from .task_start_protocol import QUEUED_START_STATUSES++ shown_pending = queued and task.status in QUEUED_START_STATUSES+ if shown_pending:+ status = TaskStatus.PENDING.value+ elif _paused_reads_failed(db, task):+ status = TaskStatus.FAILED.value+ else:+ status = task.status.valuereturn SdkTaskCancelResult(task_id=int(task.id),- status=task.status.value,- run_id=run_id,+ status=status,+ run_id=_task_run_id(task),state_version=int(task.state_version or 0),- cancelled_run_id=run_id if cancelled else None,- error_code=sdk_cancel_error_code(task.status, task.error_message),+ cancelled_run_id=cancelled_run_id if cancelled else None,+ error_code=(+ None+ if shown_pending+ else sdk_cancel_error_code(task.status, task.error_message)+ ),cancelled=cancelled,)+def _row_result(db: Session, task: Task) -> SdkTaskCancelResult:+ """The result for a command aimed at the run the row holds."""++ return _result(+ db,+ task,+ cancelled=is_sdk_cancelled(task.status, task.error_message),+ cancelled_run_id=_task_run_id(task),+ )+++def _queued_start(db: Session, task_id: int) -> TaskExecutionCommand | None:+ """The START of ``task_id`` not handed off yet, as ``has_queued_start``+ counts it (at most one exists)."""++ from .task_start_protocol import queued_start_conditions++ return (+ db.query(TaskExecutionCommand)+ .filter(*queued_start_conditions(task_id))+ .order_by(TaskExecutionCommand.id.desc())+ .first()+ )+++def _cancel_of_newest_turn(db: Session, task: Task) -> TaskExecutionCommand | None:+ """This endpoint's cancel command aimed at the task's newest START, if any.++ A cancel of a turn queued to start is persisted under an identity made of+ that turn's run and the version the START was accepted against+ (``_cancel_command_id``), so the newest START names the one command a+ cancel of that turn can be. Once the command has withdrawn the turn, the+ row still shows the turn before it (or, for a new task, holds no run) and+ no START is queued, so a repeat finds the turn only through this+ identity -- whatever state the command is in, including while it is+ still being applied after the withdrawal was committed. A START stopped+ some other way has no such command and is not found.++ The command answers only while the row is still as that cancel found or+ left it: withdrawing an appended turn does not move the row, so the+ command's version is still the row's. A row that moved since -- for+ example the turn before was resumed and runs again -- is answered from+ the row, as any other call would be. A new task is the exception: its+ cancel wrote the row itself, which then holds no run and reads as+ cancelled.+ """++ newest = (+ db.query(TaskExecutionCommand)+ .filter(+ TaskExecutionCommand.task_id == int(task.id),+ TaskExecutionCommand.kind == TaskCommandKind.START.value,+ )+ .order_by(TaskExecutionCommand.id.desc())+ .populate_existing()+ .first()+ )+ if (+ newest is None+ or newest.target_run_id is None+ or newest.target_state_version is None+ ):+ return None+ run_id = str(newest.target_run_id)+ if run_id == _task_run_id(task):+ return None+ accepted_version = int(newest.target_state_version)+ cancelled_new_task = _task_run_id(task) is None and is_sdk_cancelled(+ task.status, task.error_message+ )+ if not cancelled_new_task and accepted_version != int(task.state_version or 0):+ return None+ cancel = (+ db.query(TaskExecutionCommand)+ .filter(+ TaskExecutionCommand.task_id == int(task.id),+ TaskExecutionCommand.command_id+ == _cancel_command_id(int(task.id), accepted_version, run_id),+ TaskExecutionCommand.kind == TaskCommandKind.CANCEL.value,+ )+ .populate_existing()+ .one_or_none()+ )+ if cancel is None:+ return None+ payload: dict[str, Any] = cancel.payload if isinstance(cancel.payload, dict) else {}+ if (+ payload.get("scope") != SDK_COMMAND_SCOPE+ or payload.get("queued_run_id") != run_id+ ):+ return None+ return cancel+++def _start_of_run(+ db: Session, task_id: int, run_id: str+) -> TaskExecutionCommand | None:+ """The START that opens exactly ``run_id``, whatever its state.++ Matched on the run so that a later turn's START is never taken for the+ one a cancel was aimed at.+ """++ return (+ db.query(TaskExecutionCommand)+ .filter(+ TaskExecutionCommand.task_id == task_id,+ TaskExecutionCommand.kind == TaskCommandKind.START.value,+ TaskExecutionCommand.target_run_id == run_id,+ )+ .order_by(TaskExecutionCommand.id.desc())+ .populate_existing()+ .first()+ )+++def _start_was_stopped(start: TaskExecutionCommand | None) -> bool:+ if start is None or start.status != COMMAND_FAILED:+ return False+ result: dict[str, Any] = start.result if isinstance(start.result, dict) else {}+ return bool(result.get("rejection_reason") == STOPPED_BEFORE_START_REASON)++def _ended_without_cancel(db: Session, task: Task) -> bool:"""Whether the task's run ended and was not an SDK cancellation.Covers a COMPLETED or FAILED row and a paused row the read surface- reports as failed because automatic resume gave up.+ reports as failed (``_paused_reads_failed``)."""- from .task_auto_resume import status_only_terminal_view-if is_sdk_cancelled(task.status, task.error_message):return False- if task.status in _TERMINAL_STATUSES:- return True- if (- task.status == TaskStatus.PAUSED- and task.control_state == TaskControlState.PAUSED.value- ):- return status_only_terminal_view(db, int(task.id)) is not None- return False+ return task.status in _TERMINAL_STATUSES or _paused_reads_failed(db, task)def _prepare_cancel_sync(*, task_id: int, scope: SdkTaskScope) -> SdkTaskCancelResult | _CancelCommand:- """Authorize, classify the target and persist its command, in one read."""+ """Authorize, classify the target and persist its command, in one read.- from .task_start import TaskStartRejected, resolve_sdk_task- from .task_start_protocol import has_queued_start+ The target is the queued START that can still hand off, else the row. A+ repeat of a cancel that already withdrew the newest turn finds no such+ START and rejoins that cancel's command (``_cancel_of_newest_turn``):+ the same command to poll while it is still being applied, the same+ answer once it completed.+ """++ from .task_start import resolve_sdk_task+ from .task_start_protocol import start_can_hand_offSessionLocal = get_session_local()with SessionLocal() as db:@@ -227,59 +416,128 @@ def _prepare_cancel_sync(# is handed off, the row still shows the previous turn, so judging# the row alone would answer for a turn the caller is not asking# about.- queued = has_queued_start(db, task_id)+ queued = _queued_start(db, task_id)task = resolve_sdk_task(task_id, scope, db)- if queued:- raise SdkTaskCancelBusy(- "A turn of this task is queued to start and cannot be "- "cancelled yet; retry once it is running."- )+ if queued is not None and not start_can_hand_off(queued, task):+ # Never starts: the row moved on after the START was accepted,+ # so the turn it would open is not what a cancel can reach.+ queued = None+ if queued is not None:+ if queued.target_run_id is None:+ raise SdkTaskCancelBusy(+ "A turn of this task is queued to start and cannot be "+ "cancelled yet; retry once it is running."+ )+ return _stage_cancel(db, task, queued_run_id=str(queued.target_run_id))+ # A repeat of the cancel that withdrew the newest turn rejoins that+ # command and is answered for that turn, not for whatever the row+ # shows of the turn before it: polled like the first call while the+ # command is still being applied, answered as cancelled once it+ # completed. A failed command is not a cancellation, so the row+ # decides.+ cancel = _cancel_of_newest_turn(db, task)+ if cancel is not None:+ withdrawn_run_id = str(cancel.payload["queued_run_id"])+ if cancel.status == COMMAND_COMPLETED:+ return _result(+ db, task, cancelled=True, cancelled_run_id=withdrawn_run_id+ )+ if cancel.status in {COMMAND_PENDING, COMMAND_PROCESSING}:+ return _CancelCommand(+ command_db_id=int(cancel.id),+ aimed_run_id=withdrawn_run_id,+ aimed_at_queued_turn=True,+ )if is_sdk_cancelled(task.status, task.error_message):- return _result(task)+ return _row_result(db, task)if _ended_without_cancel(db, task):raise SdkTaskNotCancellableif task.status != TaskStatus.RUNNING:raise SdkTaskCancelBusy(f"A task that is {task.status.value} cannot be cancelled yet.")- target_state_version = int(task.state_version or 0)- # Read before the insert commits and expires ``task``: this is the- # run the command targets.- aimed_run_id = _task_run_id(task)- try:- enqueued = _enqueue_cancel(db, task, target_state_version)- except TaskCommandTaskMissing as exc:- # Deleted between the read above and the insert.- raise TaskStartRejected("task_not_found") from exc- if not enqueued.payload_matches:- raise SdkTaskCancelBusy(- "Another command holds this cancel identity; retry."- )- if enqueued.status == COMMAND_FAILED:- retry_failed_task_command(db, enqueued.command_id)- return _CancelCommand(- command_db_id=enqueued.command_id, aimed_run_id=aimed_run_id+ return _stage_cancel(db, task, queued_run_id=None)+++def _stage_cancel(+ db: Session, task: Task, *, queued_run_id: str | None+) -> _CancelCommand:+ """Persist the cancel command for the state just read.++ ``queued_run_id`` aims it at the turn queued to start as that run;+ otherwise it is aimed at the run the row holds.+ """++ from .task_start import TaskStartRejected++ target_state_version = int(task.state_version or 0)+ # Read before the insert commits and expires ``task``: this is the run+ # the command targets.+ aimed_run_id = queued_run_id if queued_run_id is not None else _task_run_id(task)+ try:+ enqueued = _enqueue_cancel(+ db, task, target_state_version, queued_run_id=queued_run_id)+ except TaskCommandTaskMissing as exc:+ # Deleted between the caller's read and the insert.+ raise TaskStartRejected("task_not_found") from exc+ if not enqueued.payload_matches:+ raise SdkTaskCancelBusy("Another command holds this cancel identity; retry.")+ if enqueued.status == COMMAND_FAILED:+ retry_failed_task_command(db, enqueued.command_id)+ return _CancelCommand(+ command_db_id=enqueued.command_id,+ aimed_run_id=aimed_run_id,+ aimed_at_queued_turn=queued_run_id is not None,+ )+++def _cancel_command_id(+ task_id: int, target_state_version: int, queued_run_id: str | None+) -> str:+ """The identity of the cancel command for the state it is aimed at.++ Keyed on the state the request read: a repeat against the same state is+ the same command, and a repeat after the state moved is a new one aimed+ at the new state. An appended turn does not move the row while it is+ queued, so its run is part of the key; once such a turn is withdrawn,+ ``_cancel_of_newest_turn`` finds the command again by this key.+ """++ command_id = f"cancel:{task_id}:{target_state_version}"+ if queued_run_id is not None:+ command_id = f"{command_id}:{queued_run_id}"+ return command_iddef _enqueue_cancel(- db: Session, task: Task, target_state_version: int+ db: Session,+ task: Task,+ target_state_version: int,+ *,+ queued_run_id: str | None,) -> EnqueuedTaskCommand:task_id = int(task.id)+ command_id = _cancel_command_id(task_id, target_state_version, queued_run_id)+ payload: dict[str, Any] = {+ "agent_id": int(task.agent_id),+ "target_state_version": target_state_version,+ "scope": SDK_COMMAND_SCOPE,+ }+ if queued_run_id is not None:+ payload["queued_run_id"] = queued_run_idreturn enqueue_task_command(db,task_id=task_id,actor_user_id=int(task.user_id) if task.user_id is not None else None,- # Keyed on the state read here: a repeat against the same state- # is the same command, and a repeat after the state moved is a- # new one aimed at the new state.- command_id=f"cancel:{task_id}:{target_state_version}",+ command_id=command_id,kind=TaskCommandKind.CANCEL,- payload={- "agent_id": int(task.agent_id),- "target_state_version": target_state_version,- "scope": SDK_COMMAND_SCOPE,- },+ payload=payload,+ # A queued turn's cancel targets that turn's run, not the run the row+ # still shows. A process that does not read ``queued_run_id`` then+ # finds the run changed and rejects the command without a write,+ # instead of taking an earlier cancelled run for this one.+ target_run_id=queued_run_id,)@@ -288,34 +546,73 @@ def _load_result_sync(task_id: int,scope: SdkTaskScope,aimed_run_id: str | None,+ aimed_at_queued_turn: bool,command_completed: bool,) -> SdkTaskCancelResult:- """Read the task as it stands once the command is over or overdue.+ """Read the outcome once the command is over or overdue.``aimed_run_id`` is the run the command was persisted against. Acompleted command cancelled that run, so the answer names it as- ``cancelled_run_id`` while every other field describes the row now,+ ``cancelled_run_id`` while every other field describes the task now,which may already show a later run. Otherwise the answer is about the- aimed run only. A row that shows another run means the aimed one is- over, and retrying would aim at the new one, so it raises- ``SdkTaskNotCancellable`` with ``_AIMED_RUN_ENDED_MESSAGE``. An aimed- run that ended on its own raises ``SdkTaskNotCancellable`` exactly as- the request would have on arrival, whether or not a later turn is- queued behind it.+ aimed run only:++ - a turn queued to start is not on the row until its START is handed+ off. While its START waits the command is still pending; a START+ stopped before it began is that turn cancelled; any other START+ outcome means the turn is over. A START the row moved away from is+ not stopped by the command (``stale_run``): while it is still pending+ the call answers busy, or 202 at the deadline, and once its consumer+ rejects it the turn reads as over.+ - a row that shows another run means the aimed one is over, and+ retrying would aim at the new one, so it raises+ ``SdkTaskNotCancellable`` with ``_AIMED_RUN_ENDED_MESSAGE``.+ - an aimed run that ended on its own raises ``SdkTaskNotCancellable``+ exactly as the request would have on arrival, whether or not a later+ turn is queued behind it.++ A START waiting for a later turn reads as ``pending`` with no code, and+ a paused row that has stopped for good reads as ``failed``, as on the+ task read."""from .task_start import resolve_sdk_taskSessionLocal = get_session_local()with SessionLocal() as db:+ # Same read order as ``_prepare_cancel_sync``.+ queued = _queued_start(db, task_id) is not Nonetask = resolve_sdk_task(task_id, scope, db)if command_completed:- return replace(_result(task), cancelled=True, cancelled_run_id=aimed_run_id)+ return _result(+ db, task, cancelled=True, cancelled_run_id=aimed_run_id, queued=queued+ )if _task_run_id(task) != aimed_run_id:+ if aimed_at_queued_turn and aimed_run_id is not None:+ start = _start_of_run(db, task_id, aimed_run_id)+ if _start_was_stopped(start):+ return _result(+ db,+ task,+ cancelled=True,+ cancelled_run_id=aimed_run_id,+ queued=queued,+ )+ if start is not None and start.status in {+ COMMAND_PENDING,+ COMMAND_PROCESSING,+ }:+ return _result(db, task, cancelled=False, queued=queued)raise SdkTaskNotCancellable(_AIMED_RUN_ENDED_MESSAGE)if _ended_without_cancel(db, task):raise SdkTaskNotCancellable- return _result(task)+ return _result(+ db,+ task,+ cancelled=is_sdk_cancelled(task.status, task.error_message),+ cancelled_run_id=aimed_run_id,+ queued=queued,+ )async def request_sdk_task_cancel(@@ -332,12 +629,15 @@ async def request_sdk_task_cancel(task's state.Returns ``cancelled=True`` once the task is the outcome of a- cancellation, including when it already was, and once the command that- cancelled the aimed run completed (``cancelled_run_id``). Returns- ``cancelled=False`` when the command was persisted but not applied- within ``SDK_CANCEL_REQUEST_WAIT_SECONDS``; calling again reaches the- same command. Raises ``SdkTaskNotCancellable`` when the aimed run ended- without being cancelled, whichever way the wait ends.+ cancellation, including when it already was, once the command that+ cancelled the aimed run completed (``cancelled_run_id``), and when an+ earlier call's command that withdrew the newest turn completed, which+ names that turn. Returns ``cancelled=False`` when the command was+ persisted but not applied within ``SDK_CANCEL_REQUEST_WAIT_SECONDS``;+ calling again reaches the same command, including after it withdrew a+ queued turn and before it is marked completed. Raises+ ``SdkTaskNotCancellable`` when the aimed run ended without being+ cancelled, whichever way the wait ends."""prepared = await run_db_io_cancellation_safe(@@ -367,6 +667,7 @@ async def request_sdk_task_cancel(task_id=task_id,scope=scope,aimed_run_id=aimed_run_id,+ aimed_at_queued_turn=prepared.aimed_at_queued_turn,command_completed=command_completed,))@@ -387,6 +688,7 @@ async def request_sdk_task_cancel(task_id=task_id,scope=scope,aimed_run_id=aimed_run_id,+ aimed_at_queued_turn=prepared.aimed_at_queued_turn,command_completed=False,))@@ -436,14 +738,18 @@ def _is_settled_sdk_cancel_target(def _is_exact_cancel_target(- task: Task, *, expected_run_id: str | None, expected_state_version: int+ task: Task,+ *,+ expected_run_id: str | None,+ expected_state_version: int,+ expected_status: TaskStatus,) -> bool:- """Whether the row is still the running target this command was aimed at."""+ """Whether the row is still the target this command was aimed at."""return (_task_run_id(task) == expected_run_idand int(task.state_version or 0) == expected_state_version- and task.status == TaskStatus.RUNNING+ and task.status == expected_status)@@ -455,12 +761,26 @@ def _stale(task: Task, expected_run_id: str | None, expected_state_version: int))+def _carries_display_events(task: Task, expected_status: TaskStatus) -> bool:+ """Whether the terminal broadcast should carry the turn's display events.++ Only a version-2 task has them, and only a run that ran has the result+ fact they are read from: a new task cancelled before it started has+ none, and publishing them without it fails.+ """++ if expected_status != TaskStatus.RUNNING:+ return False+ return bool(task.conversation_storage_version == 2)++def _load_cancelable_sdk_task_sync(*,task_id: int,agent_id: int,expected_run_id: str | None,expected_state_version: int,+ expected_status: TaskStatus,terminal_event_state: dict[str, Any],) -> bool:"""Return whether the exact cancel target is already settled.@@ -477,13 +797,14 @@ def _load_cancelable_sdk_task_sync(expected_run_id=expected_run_id,expected_state_version=expected_state_version,):- if task.conversation_storage_version == 2:+ if _carries_display_events(task, expected_status):terminal_event_state.update(task_control_snapshot(task).as_dict())return Trueif not _is_exact_cancel_target(task,expected_run_id=expected_run_id,expected_state_version=expected_state_version,+ expected_status=expected_status,):raise TaskCommandRejected(_stale(task, expected_run_id, expected_state_version),@@ -548,16 +869,21 @@ def _finalize_sdk_cancel_sync(agent_id: int,expected_run_id: str | None,expected_state_version: int,+ expected_status: TaskStatus,terminal_event_state: dict[str, Any],) -> bool:"""Persist cancellation for one exact target; return whether this wrote it.+ ``expected_status`` is RUNNING, or PENDING for a new task whose START is+ still queued: that task never ran, so it gets the terminal row alone and+ its START's own settlement closes the turn.+Raises ``StaleTaskRunError`` before any write, or after a rollback, when- the target is neither still running as read nor already settled. When- the target is settled or written here, it fills ``terminal_event_state``+ the target is neither still as read nor already settled. When the+ target is settled or written here, it fills ``terminal_event_state``with the control snapshot the terminal broadcast carries, once the row- is durable. The task's cached- read is invalidated after this call's own commit.+ is durable. The task's cached read is invalidated after this call's own+ commit."""from .task_coordinator_runtime import current_task_coordinator@@ -586,7 +912,7 @@ def _finalize_sdk_cancel_sync(expected_run_id=expected_run_id,expected_state_version=expected_state_version,):- if task.conversation_storage_version == 2:+ if _carries_display_events(task, expected_status):settled_state = task_control_snapshot(task).as_dict()else:settled_state = {}@@ -597,6 +923,7 @@ def _finalize_sdk_cancel_sync(task,expected_run_id=expected_run_id,expected_state_version=expected_state_version,+ expected_status=expected_status,):db.rollback()raise StaleTaskRunError(@@ -610,7 +937,7 @@ def _finalize_sdk_cancel_sync(Task.agent_id == agent_id,Task.source == SDK_TASK_SOURCE,func.coalesce(Task.state_version, 0) == expected_state_version,- Task.status == TaskStatus.RUNNING,+ Task.status == expected_status,).values(status=TaskStatus.FAILED,@@ -645,16 +972,17 @@ def _finalize_sdk_cancel_sync()sync_workforce_run_status(db, updated, TaskStatus.FAILED)sync_trigger_run_status(db, updated, TaskStatus.FAILED)- turn_id = _close_cancelled_turn_no_commit(db, updated)- stage_result_fact_no_commit(- db,- updated,- sdk_cancel_result_fact(SDK_CANCEL_ERROR_MESSAGE),- turn_id=turn_id,- )+ if expected_status == TaskStatus.RUNNING:+ turn_id = _close_cancelled_turn_no_commit(db, updated)+ stage_result_fact_no_commit(+ db,+ updated,+ sdk_cancel_result_fact(SDK_CANCEL_ERROR_MESSAGE),+ turn_id=turn_id,+ )event_state = (task_control_snapshot(updated).as_dict()- if updated.conversation_storage_version == 2+ if _carries_display_events(updated, expected_status)else {})db.commit()@@ -696,20 +1024,192 @@ async def _broadcast_sdk_cancel_terminal_event()+@dataclass(frozen=True)+class _CancelTarget:+ run_id: str | None+ state_version: int+ status: TaskStatus+++def _stop_queued_start_no_commit(+ db: Session, task_id: int, start: TaskExecutionCommand+) -> bool:+ """Fail a START that no executor has claimed, leaving the task row alone.++ Stopped while it is pending and either never attempted or waiting for an+ admission ticket, the same START a cancel is dispatched past+ (``waiting_admission``): one sent back to the admission queue after an+ attempt is still stopped, one an executor claimed and may have acted on+ is not. An append's failure belongs to its command: the row keeps the+ previous turn's result. The turn's delivery row and runtime values are+ released by the START's own settlement. Returns False when the START+ moved first.+ """++ from .task_command_terminal_events import (+ TerminalTaskEventDraft,+ stage_terminal_event,+ )+ from .task_coordinator_runtime import current_task_coordinator+ from .task_coordinator_service import lock_task_lease_no_commit+ from .task_start_consumer import settle_failed_start_no_commit++ coordinator = current_task_coordinator(task_id)+ if coordinator is not None:+ assert coordinator.lease is not None+ if not lock_task_lease_no_commit(db, coordinator.lease):+ raise StaleTaskRunError(f"task {task_id} cancellation owner changed")+ stopped = db.execute(+ update(TaskExecutionCommand)+ .where(+ TaskExecutionCommand.id == start.id,+ TaskExecutionCommand.status == COMMAND_PENDING,+ or_(+ TaskExecutionCommand.attempt_count == 0,+ waiting_admission(TaskExecutionCommand.id),+ ),+ )+ .values(+ status=COMMAND_FAILED,+ error=STOPPED_BEFORE_START,+ completed_at=func.now(),+ result={"rejection_reason": STOPPED_BEFORE_START_REASON},+ )+ .returning(TaskExecutionCommand.id)+ .execution_options(synchronize_session=False)+ ).scalar_one_or_none()+ if stopped is None:+ return False+ db.refresh(start)+ settle_failed_start_no_commit(db, start)+ stage_terminal_event(+ db,+ command_db_id=int(start.id),+ draft=TerminalTaskEventDraft(message_code=None, resend_safe=False),+ )+ return True+++def _resolve_queued_cancel_target_sync(+ *, task_id: int, agent_id: int, queued_run_id: str+) -> _CancelTarget | None:+ """Turn "the turn queued as ``queued_run_id``" into what to cancel now.++ Returns the task-row target to cancel, or ``None`` when nothing is left+ to do: the turn was already cancelled, or its START was stopped before+ it began (by this call, for an appended turn). Raises+ ``TaskCommandRejected`` when the turn ended on its own or never started,+ including when its START can no longer hand off because the row moved+ after it was accepted (checked before anything is written), and+ ``TaskCommandDeferred`` while its START is being handed off.+ """++ from .task_start_protocol import start_can_hand_off++ stale = TaskCommandRejected(+ f"queued run {queued_run_id} of task {task_id} is no longer cancellable",+ reason="stale_run",+ )+ SessionLocal = get_session_local()+ with SessionLocal() as db:+ # One more pass covers a START that is handed off, or stopped by+ # another control, between this read and the write below.+ for _attempt in range(2):+ task = _load_sdk_task(db, task_id=task_id, agent_id=agent_id)+ if _task_run_id(task) == queued_run_id:+ # The queued turn started. Fenced on the version read now:+ # the handoff and the run itself moved it since the request.+ if is_sdk_cancelled(task.status, task.error_message):+ return None+ if task.status != TaskStatus.RUNNING:+ raise stale+ return _CancelTarget(+ queued_run_id, int(task.state_version or 0), TaskStatus.RUNNING+ )+ start = _start_of_run(db, task_id, queued_run_id)+ if _start_was_stopped(start):+ return None+ if start is None or start.status == COMMAND_FAILED:+ raise stale+ if start.status == COMMAND_PROCESSING:+ raise TaskCommandDeferred(+ f"START of run {queued_run_id} is being handed off"+ )+ if start.status == COMMAND_PENDING:+ if _new_task_already_cancelled(task, start):+ # A new task's cancel wrote the row and the START is+ # settled only once that command completes. Applied+ # again before then -- its completion never committed --+ # the cancel is already done; the row has moved past the+ # START, which must not read as the turn having ended.+ return None+ if not start_can_hand_off(start, task):+ # The row moved on after the START was accepted, so the+ # START will never open this run and there is nothing+ # of the turn to cancel. Nothing is written.+ raise stale+ if task.status == TaskStatus.PENDING:+ # A new task that never ran: the row itself is cancelled+ # and the completed command settles the START.+ return _CancelTarget(+ _task_run_id(task),+ int(task.state_version or 0),+ TaskStatus.PENDING,+ )+ if _stop_queued_start_no_commit(db, task_id, start):+ db.commit()+ return None+ db.rollback()+ raise stale+++def _new_task_already_cancelled(task: Task, start: TaskExecutionCommand) -> bool:+ """Whether ``start`` opens a new task whose row a cancel already wrote."""++ from .task_start_protocol import TaskStartPayload++ if _task_run_id(task) is not None or not is_sdk_cancelled(+ task.status, task.error_message+ ):+ return False+ try:+ payload = TaskStartPayload.model_validate(start.payload)+ except ValueError:+ return False+ return bool(payload.kind == "create")++async def cancel_sdk_task_unserialized(*,task_id: int,agent_id: int,expected_run_id: str | None,expected_state_version: int,+ queued_run_id: str | None = None,) -> None:"""Cancel one exact durable-command target while the caller owns its gate.+ ``queued_run_id`` aims the command at a turn that was queued when it+ was requested; the run and version to cancel are then read here.+Every expected failure leaves as ``TaskCommandRejected`` or``StaleTaskRunError`` (which the dispatcher rejects as ``stale_run``),so the command ends at once with its reason instead of being retried."""+ expected_status = TaskStatus.RUNNING+ if queued_run_id is not None:+ target = await run_db_io_cancellation_safe(+ lambda: _resolve_queued_cancel_target_sync(+ task_id=task_id, agent_id=agent_id, queued_run_id=queued_run_id+ )+ )+ if target is None:+ return+ expected_run_id = target.run_id+ expected_state_version = target.state_version+ expected_status = target.status+terminal_event_state: dict[str, Any] = {}already_settled = await run_db_io_cancellation_safe(lambda: _load_cancelable_sdk_task_sync(@@ -717,6 +1217,7 @@ async def cancel_sdk_task_unserialized(agent_id=agent_id,expected_run_id=expected_run_id,expected_state_version=expected_state_version,+ expected_status=expected_status,terminal_event_state=terminal_event_state,))@@ -735,6 +1236,7 @@ async def cancel_sdk_task_unserialized(agent_id=agent_id,expected_run_id=expected_run_id,expected_state_version=expected_state_version,+ expected_status=expected_status,terminal_event_state=terminal_event_state,)diff --git a/src/xagent/web/services/task_command_execution.py b/src/xagent/web/services/task_command_execution.pyindex 161d8ec2a..ea2ff8299 100644--- a/src/xagent/web/services/task_command_execution.py+++ b/src/xagent/web/services/task_command_execution.py@@ -4706,10 +4706,20 @@ async def _execute_durable_task_command(auto_resume = command.kind == TaskCommandKind.RESUME and (command.command_id.startswith(AUTO_RESUME_COMMAND_PREFIX))+ # An SDK cancel aimed at a turn that was still queued when it was+ # requested targets that turn's run while the row holds the run before+ # it, or none. Its core compares the target itself, against the run the+ # payload names.+ sdk_queued_cancel = (+ command.kind == TaskCommandKind.CANCEL+ and command.payload.get("scope") == SDK_COMMAND_SCOPE+ and _sdk_queued_run_id(command.payload) is not None+ )if (command.kind != TaskCommandKind.MESSAGEand command.target_run_id is not Noneand not auto_resume+ and not sdk_queued_cancel):current_run_id = await run_db_io_cancellation_safe(lambda: _load_command_task_run_id(command.task_id)@@ -4958,6 +4968,7 @@ async def _execute_durable_task_command(agent_id=agent_id,expected_run_id=command.target_run_id,expected_state_version=target_state_version,+ queued_run_id=_sdk_queued_run_id(message_data),)elif scope_value == EXTERNAL_COMMAND_SCOPE:await cancel_external_task_unserialized(@@ -4996,6 +5007,13 @@ async def _execute_durable_task_command(return result+def _sdk_queued_run_id(message_data: dict[str, Any]) -> str | None:+ """The queued run an SDK cancel names, or ``None`` when it names none."""++ raw = message_data.get("queued_run_id")+ return raw if isinstance(raw, str) and raw else None++def _command_scope(command: ClaimedTaskCommand) -> str | None:"""The scope a command payload names, or ``None`` when it names none."""diff --git a/src/xagent/web/services/task_command_transport.py b/src/xagent/web/services/task_command_transport.pyindex c3782908d..ad0f96970 100644--- a/src/xagent/web/services/task_command_transport.py+++ b/src/xagent/web/services/task_command_transport.py@@ -864,9 +864,13 @@ def enqueue_task_command(payload: dict[str, Any],reply_host_id: str | None = None,reply_origin: str | None = None,+ target_run_id: str | None = None,) -> EnqueuedTaskCommand:"""Commit an idempotent command and return only after it is durable.+ ``target_run_id`` names the run the command is aimed at when it is not+ the run the task row shows now, as for a turn whose START is queued.+TaskCommandOwnerStateError from stage_task_command's owner pre-flushpropagates uncaught -- it is deliberately not caught alongsideIntegrityError below. A caller managing its own session lifetime must@@ -884,6 +888,7 @@ def enqueue_task_command(payload=payload,reply_host_id=reply_host_id,reply_origin=reply_origin,+ target_run_id=target_run_id,)# Only a newly created row is worth committing, and the commit stays# inside this try so that a constraint failure surfacing there ratherdiff --git a/src/xagent/web/services/task_execution_admission.py b/src/xagent/web/services/task_execution_admission.pyindex 0c7827913..47142427b 100644--- a/src/xagent/web/services/task_execution_admission.py+++ b/src/xagent/web/services/task_execution_admission.py@@ -72,7 +72,12 @@ PRIORITY_MAX = 10_EXECUTION_KINDS = frozenset({"start", "resume", "resume_input", "message"})_TERMINAL = ("completed", "failed")-_STOPPED_BEFORE_START = "Task stopped before execution started."+# What a START stopped before it began records: ``STOPPED_BEFORE_START`` as+# its error when it is the stopped command itself, and always+# ``STOPPED_BEFORE_START_REASON`` as the ``rejection_reason`` of its result,+# by which a reader recognises it.+STOPPED_BEFORE_START = "Task stopped before execution started."+STOPPED_BEFORE_START_REASON = "cancelled_before_admission"_CONTROL_STOPPED_COMMAND = "Task command stopped by a control request."@@ -871,9 +876,9 @@ def settle_queued_start_for_pause().values(status="failed",- error=_STOPPED_BEFORE_START,+ error=STOPPED_BEFORE_START,completed_at=func.now(),- result={"rejection_reason": "cancelled_before_admission"},+ result={"rejection_reason": STOPPED_BEFORE_START_REASON},).returning(TaskExecutionCommand.id))@@ -937,7 +942,7 @@ def settle_cancelled_admissions(db: Session, command_id: int) -> None:status="failed",error=_CONTROL_STOPPED_COMMAND,completed_at=func.now(),- result={"rejection_reason": "cancelled_before_admission"},+ result={"rejection_reason": STOPPED_BEFORE_START_REASON},).returning(TaskExecutionCommand.id)).all()@@ -1035,7 +1040,7 @@ def settle_cap_held_commands_no_commit(db: Session, control_command_id: int) ->status="failed",error=_CONTROL_STOPPED_COMMAND,completed_at=func.now(),- result={"rejection_reason": "cancelled_before_admission"},+ result={"rejection_reason": STOPPED_BEFORE_START_REASON},).returning(TaskExecutionCommand.id)).all()diff --git a/src/xagent/web/services/task_start_protocol.py b/src/xagent/web/services/task_start_protocol.pyindex 8d7c7517a..46192aa15 100644--- a/src/xagent/web/services/task_start_protocol.py+++ b/src/xagent/web/services/task_start_protocol.py@@ -11,6 +11,7 @@ from typing import Annotated, Literal, Selffrom pydantic import BaseModel, ConfigDict, Field, JsonValue, model_validatorfrom sqlalchemy import selectfrom sqlalchemy.orm import Session+from sqlalchemy.sql.elements import ColumnElementfrom ..models.task import Task, TaskStatusfrom ..models.task_command import TaskExecutionCommand@@ -172,6 +173,29 @@ def stage_task_start_command()+# Terminal statuses an append is accepted from. With shared execution the row+# keeps one of them, and the previous turn's output and error, until the+# queued START is handed off, so a caller waiting for a terminal status would+# take the previous turn as the new one; such a row reads as ``pending``. A+# PAUSED row keeps reading as paused, which callers already wait on. A RUNNING+# or WAITING_FOR_USER row has moved since the START was queued, and a PENDING+# row already reads as pending.+QUEUED_START_STATUSES = (TaskStatus.COMPLETED, TaskStatus.FAILED)+++def queued_start_conditions(task_id: int) -> tuple[ColumnElement[bool], ...]:+ """Row conditions that select the STARTs of ``task_id`` not handed off.++ The one definition of "queued": ``has_queued_start`` and the SDK cancel's+ lookup of the queued START both filter with it.+ """+ return (+ TaskExecutionCommand.task_id == task_id,+ TaskExecutionCommand.kind == TaskCommandKind.START.value,+ TaskExecutionCommand.status.in_((COMMAND_PENDING, COMMAND_PROCESSING)),+ )++def has_queued_start(db: Session, task_id: int) -> bool:"""Whether an accepted START of ``task_id`` has not been handed off yet.@@ -183,17 +207,46 @@ def has_queued_start(db: Session, task_id: int) -> bool:return (db.execute(select(TaskExecutionCommand.id)- .where(- TaskExecutionCommand.task_id == task_id,- TaskExecutionCommand.kind == TaskCommandKind.START.value,- TaskExecutionCommand.status.in_((COMMAND_PENDING, COMMAND_PROCESSING)),- )+ .where(*queued_start_conditions(task_id)).limit(1)).first()is not None)+def start_can_hand_off(command: TaskExecutionCommand, task: Task) -> bool:+ """Whether the START ``command`` can still begin its run on ``task``.++ The conditions the START consumer applies before it hands a START off+ that depend on the command and the task row: the payload decodes and+ matches its command's envelope (``read_task_start_command``), the+ command's own version and the payload's agree, the row is still the run+ and version the START was accepted against, nothing is running, and a+ CREATE finds its task still pending. A START that fails them is rejected+ when it is claimed and never starts, so a control that aims at it would+ be aimed at nothing.++ Two consumer checks are not repeated here. ``command_identity_matches_task``+ re-validates the issuing user's access, and a single-process consumer+ refuses a START that is not ``existing``. A START either of them rejects+ never starts either, so treating it as one that can still start only+ lets a cancel withdraw a turn that would not have run.+ """+ try:+ start = TaskStartPayload.model_validate(command.payload)+ except ValueError:+ return False+ return not (+ start.run_id != command.target_run_id+ or start.turn_id != command.command_id+ or command.target_state_version != start.state_version+ or task.run_id != start.expected_run_id+ or task.state_version != start.state_version+ or task.status == TaskStatus.RUNNING+ or (start.kind == "create" and task.status != TaskStatus.PENDING)+ )++def read_task_start_command(command: ClaimedTaskCommand) -> TaskStartPayload:"""Decode persisted inputs and verify their immutable envelope identity.diff --git a/tests/web/api/test_sdk_task_cancel.py b/tests/web/api/test_sdk_task_cancel.pyindex 8d95c8735..2e7c248fc 100644--- a/tests/web/api/test_sdk_task_cancel.py+++ b/tests/web/api/test_sdk_task_cancel.py@@ -9,6 +9,7 @@ it did not cause, the ownership fence, and the routing that sends only animport asyncioimport time+from dataclasses import replacefrom datetime import UTC, datetime, timedeltafrom types import SimpleNamespacefrom typing import Any@@ -21,6 +22,8 @@ from xagent.web.api import websocket as websocket_apifrom xagent.web.models.agent import Agentfrom xagent.web.models.chat_message import TaskChatMessagefrom xagent.web.models.task import Task, TaskStatus+from xagent.web.models.task_admission import TaskAdmissionBucket, TaskAdmissionTicket+from xagent.web.models.task_command import TaskExecutionCommandfrom xagent.web.models.task_execution_event import TaskExecutionEventfrom xagent.web.services import a2a_task_cancel as a2a_cancel_servicefrom xagent.web.services import sdk_task_cancel@@ -43,14 +46,17 @@ from xagent.web.services.sdk_task_cancel import ()from xagent.web.services.task_command_transport import (ClaimedTaskCommand,+ TaskCommandDeferred,TaskCommandKind,TaskCommandRejected,+ stage_task_command,)from xagent.web.services.task_execution_controller import (StaleTaskRunError,TaskControlState,)from xagent.web.services.task_orchestrator import TaskTurnPayload+from xagent.web.services.task_start_protocol import TaskStartPayloadfrom .conftest import _admin_headers, _direct_db_session, client@@ -589,6 +595,7 @@ def test_replaced_attempt_rejects_cancel_and_preserves_task(agent_id=agent_id,expected_run_id=before["run_id"],expected_state_version=before["state_version"],+ expected_status=TaskStatus.RUNNING,terminal_event_state={},)@@ -838,6 +845,7 @@ async def test_sdk_scope_routes_to_the_sdk_core_only(agent_id=agent_id,expected_run_id="run-sdk",expected_state_version=4,+ queued_run_id=None,)assert external_core.await_count == 0assert a2a_core.await_count == 0@@ -892,6 +900,7 @@ def test_lease_fields_are_cleared_only_without_a_coordinator(agent_id=agent_id,expected_run_id=before.run_id,expected_state_version=before.state_version,+ expected_status=TaskStatus.RUNNING,terminal_event_state={},)@@ -903,3 +912,633 @@ def test_lease_fields_are_cleared_only_without_a_coordinator(assert after.lease_expires_at.replace(tzinfo=UTC) > datetime.now(UTC) - timedelta(minutes=5)+++# ----- a turn that was queued when its cancel was requested -----++QUEUED_RUN = "run-queued"+STOPPED_BEFORE_START = {"rejection_reason": "cancelled_before_admission"}+++def _stage_start(+ task_id: int,+ owner_user_id: int,+ *,+ run_id: str = QUEUED_RUN,+ turn_id: str = "turn-queued",+ **columns: Any,+) -> int:+ """An accepted START for ``run_id`` with its pending user message.++ Its payload is the one the acceptance writes: the run and version the row+ holds now, which is what the START consumer compares the row with.+ """+ db = _direct_db_session()+ try:+ task = db.query(Task).filter(Task.id == task_id).one()+ start = TaskStartPayload(+ version=1,+ run_id=run_id,+ state_version=int(task.state_version or 0),+ expected_run_id=task.run_id,+ turn_id=turn_id,+ kind="create" if task.status == TaskStatus.PENDING else "append",+ message="hello",+ )+ staged = stage_task_command(+ db,+ task_id=task_id,+ actor_user_id=owner_user_id,+ command_id=turn_id,+ kind=TaskCommandKind.START,+ payload=start.model_dump(mode="json"),+ target_run_id=run_id,+ )+ db.commit()+ start_id = int(staged.staged_db_id)+ if columns:+ db.query(TaskExecutionCommand).filter(+ TaskExecutionCommand.id == start_id+ ).update(columns)+ db.commit()+ finally:+ db.close()+ _seed_pending_user_message(task_id, owner_user_id, turn_id)+ return start_id+++def _start(start_id: int) -> tuple[str, Any]:+ db = _direct_db_session()+ try:+ row = db.get(TaskExecutionCommand, start_id)+ return str(row.status), row.result+ finally:+ db.close()+++async def _cancel_queued(task_id: int, agent_id: int) -> None:+ # The command was recorded against the row as it stood while the turn+ # was queued; the core reads the target to cancel from the queued run.+ await cancel_sdk_task_unserialized(+ task_id=task_id,+ agent_id=agent_id,+ expected_run_id="run-before",+ expected_state_version=4,+ queued_run_id=QUEUED_RUN,+ )+++ "previous",+ [+ {"status": TaskStatus.COMPLETED, "control_state": "completed", "output": "x"},+ {"status": TaskStatus.FAILED, "control_state": "failed", "error_message": "e"},+ {+ "status": TaskStatus.FAILED,+ "control_state": "failed",+ "error_message": SDK_CANCEL_ERROR_MESSAGE,+ },+ {"status": TaskStatus.PAUSED, "control_state": "paused"},+ ],+ ids=["completed", "failed", "cancelled", "paused"],+)+async def test_an_appended_turn_stopped_before_it_starts_leaves_the_previous_result(+ monkeypatch: pytest.MonkeyPatch, previous: dict[str, Any]+) -> None:+ """An append's failure belongs to its command, never to the row."""+ task_id, agent_id, owner_user_id = _create_task(run_id="run-before", **previous)+ start_id = _stage_start(task_id, owner_user_id)+ manager = _broadcast_manager(monkeypatch)+ before = _row(task_id)++ for _attempt in range(2): # the second call is the command replayed+ await _cancel_queued(task_id, agent_id)++ assert _row(task_id) == before+ assert _start(start_id) == ("failed", STOPPED_BEFORE_START)+ assert _delivery_status(task_id, "turn-queued") == "failed"+ assert _assistant_rows(task_id) == []+ assert manager.broadcast_to_task.await_count == 0+++async def test_a_new_task_cancel_applied_again_before_it_completed_is_done(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ """The cancel wrote the new task's row, then its completion never+ committed and the command is applied again while the START is still+ pending. The row now holds a later version than the START, which must+ not read as a turn that ended on its own: the cancel is already done."""+ task_id, agent_id, owner_user_id = _create_task(+ status=TaskStatus.PENDING, control_state="idle", run_id=None, state_version=0+ )+ start_id = _stage_start(task_id, owner_user_id)+ _broadcast_manager(monkeypatch)++ await _cancel_queued(task_id, agent_id)+ after_first = _row(task_id)+ await _cancel_queued(task_id, agent_id)++ assert _row(task_id) == after_first+ task = _load_task(task_id)+ assert is_sdk_cancelled(task.status, task.error_message)+ assert _start(start_id)[0] == "pending"+++async def test_a_turn_appended_to_a_cancelled_new_task_is_withdrawn(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ """A new task was cancelled before it ran, so its row holds no run and+ reads as cancelled; a turn appended to it is queued. Cancelling that+ turn withdraws its START: the row is not taken for the turn's own+ cancel already being done."""+ task_id, agent_id, owner_user_id = _create_task(+ status=TaskStatus.FAILED,+ control_state=TaskControlState.FAILED.value,+ run_id=None,+ state_version=1,+ error_message=SDK_CANCEL_ERROR_MESSAGE,+ )+ start_id = _stage_start(task_id, owner_user_id)+ _broadcast_manager(monkeypatch)+ before = _row(task_id)++ await _cancel_queued(task_id, agent_id)++ assert _row(task_id) == before+ assert _start(start_id) == ("failed", STOPPED_BEFORE_START)++[email protected]("storage_version", [1, 2])+async def test_a_new_task_cancelled_before_it_starts_is_cancelled_as_a_row(+ monkeypatch: pytest.MonkeyPatch, storage_version: int+) -> None:+ """A task that never ran gets the terminal row alone: no transcript line+ and no delivery change, which its START's own settlement owns. It has no+ result fact either, so its terminal frame carries no display events,+ whatever its storage version: asking for them would fail the broadcast."""+ task_id, agent_id, owner_user_id = _create_task(+ status=TaskStatus.PENDING,+ control_state="idle",+ run_id=None,+ state_version=0,+ storage_version=storage_version,+ )+ _stage_start(task_id, owner_user_id)+ manager = _broadcast_manager(monkeypatch)++ await _cancel_queued(task_id, agent_id)++ task = _load_task(task_id)+ assert is_sdk_cancelled(task.status, task.error_message)+ assert (task.run_id, task.state_version) == (None, 1)+ assert _assistant_rows(task_id) == []+ assert _delivery_status(task_id, "turn-queued") == DELIVERY_PENDING+ assert "state_version" not in _terminal_frame(manager)+++async def test_a_queued_turn_that_started_first_is_cancelled_as_running(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ """The handoff moved the run and the version since the request; the+ queued run names the target and the version is the one read now."""+ task_id, agent_id, owner_user_id = _create_task(run_id=QUEUED_RUN, state_version=7)+ _stage_start(task_id, owner_user_id, status="completed", attempt_count=1)+ _broadcast_manager(monkeypatch)++ await _cancel_queued(task_id, agent_id)++ task = _load_task(task_id)+ assert is_sdk_cancelled(task.status, task.error_message)+ assert (task.run_id, task.state_version) == (QUEUED_RUN, 8)+++async def test_a_queued_turn_already_cancelled_is_left_alone(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ task_id, agent_id, owner_user_id = _create_task(+ run_id=QUEUED_RUN,+ status=TaskStatus.FAILED,+ control_state="failed",+ error_message=SDK_CANCEL_ERROR_MESSAGE,+ state_version=9,+ )+ _stage_start(task_id, owner_user_id, status="completed", attempt_count=1)+ _broadcast_manager(monkeypatch)+ before = _row(task_id)++ await _cancel_queued(task_id, agent_id)++ assert _row(task_id) == before+++async def test_a_later_queued_turn_is_never_stopped_for_an_earlier_one(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ """The aimed turn ran to completion and another one was appended since:+ the cancel is stale, and the newer START is not its target."""+ task_id, agent_id, owner_user_id = _create_task(+ run_id=QUEUED_RUN,+ status=TaskStatus.COMPLETED,+ control_state="completed",+ output="done",+ )+ _stage_start(task_id, owner_user_id, status="completed", attempt_count=1)+ later_id = _stage_start(+ task_id, owner_user_id, run_id="run-later", turn_id="turn-later"+ )+ _broadcast_manager(monkeypatch)+ before = _row(task_id)++ with pytest.raises(TaskCommandRejected) as raised:+ await _cancel_queued(task_id, agent_id)++ assert raised.value.reason == "stale_run"+ assert _row(task_id) == before+ assert _start(later_id) == ("pending", None)+ assert _delivery_status(task_id, "turn-later") == DELIVERY_PENDING+++ ("start", "error", "reason"),+ [+ ({"status": "processing", "attempt_count": 1}, TaskCommandDeferred, None),+ ({"status": "pending", "attempt_count": 1}, TaskCommandRejected, "stale_run"),+ (+ {"status": "failed", "result": {"rejection_reason": "invalid_start"}},+ TaskCommandRejected,+ "stale_run",+ ),+ (None, TaskCommandRejected, "stale_run"),+ ],+ ids=["being_handed_off", "claimed_before", "failed_itself", "missing"],+)+async def test_a_queued_start_this_cancel_cannot_stop_is_left_as_it_is(+ monkeypatch: pytest.MonkeyPatch, start, error, reason+) -> None:+ task_id, agent_id, owner_user_id = _create_task(+ run_id="run-before", status=TaskStatus.COMPLETED, control_state="completed"+ )+ start_id = None if start is None else _stage_start(task_id, owner_user_id, **start)+ _broadcast_manager(monkeypatch)+ before = _row(task_id)+ start_before = None if start_id is None else _start(start_id)++ with pytest.raises(error) as raised:+ await _cancel_queued(task_id, agent_id)++ if reason is not None:+ assert raised.value.reason == reason+ assert _row(task_id) == before+ if start_id is not None:+ assert _start(start_id) == start_before+++async def test_a_start_is_matched_by_its_run_not_by_being_the_one_queued(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ """The aimed turn failed to start and another turn is queued now: the+ queued START belongs to that other turn and stays as it is."""+ task_id, agent_id, owner_user_id = _create_task(+ run_id="run-before", status=TaskStatus.COMPLETED, control_state="completed"+ )+ _stage_start(+ task_id,+ owner_user_id,+ status="failed",+ result={"rejection_reason": "invalid_start"},+ )+ later_id = _stage_start(+ task_id, owner_user_id, run_id="run-later", turn_id="turn-later"+ )+ _broadcast_manager(monkeypatch)++ with pytest.raises(TaskCommandRejected) as raised:+ await _cancel_queued(task_id, agent_id)++ assert raised.value.reason == "stale_run"+ assert _start(later_id) == ("pending", None)+++async def test_an_sdk_cancel_of_a_running_task_keeps_the_dispatcher_run_check(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ """Only a cancel aimed at a queued turn bypasses the dispatcher's run+ comparison; one aimed at the running row is rejected there, as before."""+ task_id, agent_id, _owner = _create_task(run_id="run-after")+ _broadcast_manager(monkeypatch)+ sdk_core = AsyncMock()+ monkeypatch.setattr(+ command_execution_service, "cancel_sdk_task_unserialized", sdk_core+ )++ with pytest.raises(TaskCommandRejected) as raised:+ await command_execution_service._execute_durable_task_command(+ _cancel_command(task_id, agent_id, scope="sdk")+ )++ assert raised.value.reason == "stale_run"+ sdk_core.assert_not_awaited()+++async def test_a_queued_cancel_read_without_its_queued_run_is_rejected_unwritten(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ """A process that does not read ``queued_run_id`` sees a cancel aimed at+ the queued turn's run while the row still holds an earlier, cancelled+ run. The run comparison rejects it before any core, so that earlier+ run is never reported as this cancel's outcome and the queued START is+ left alone."""+ task_id, agent_id, owner_user_id = _create_task(+ run_id="run-before",+ status=TaskStatus.FAILED,+ control_state="failed",+ error_message=SDK_CANCEL_ERROR_MESSAGE,+ )+ start_id = _stage_start(task_id, owner_user_id)+ _broadcast_manager(monkeypatch)+ sdk_core = AsyncMock()+ monkeypatch.setattr(+ command_execution_service, "cancel_sdk_task_unserialized", sdk_core+ )+ before = _row(task_id)+ command = replace(+ _cancel_command(task_id, agent_id, scope="sdk"), target_run_id=QUEUED_RUN+ )++ with pytest.raises(TaskCommandRejected) as raised:+ await command_execution_service._execute_durable_task_command(command)++ assert raised.value.reason == "stale_run"+ sdk_core.assert_not_awaited()+ assert _row(task_id) == before+ assert _start(start_id) == ("pending", None)+++async def test_an_sdk_cancel_aimed_at_a_queued_turn_reaches_its_core_past_the_row_run(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ """The row holds the run before the queued one while the command targets+ the queued turn's run, as ``_enqueue_cancel`` records it, so the+ dispatcher's own run comparison must not reject it."""+ task_id, agent_id, _owner = _create_task(run_id="run-before")+ _broadcast_manager(monkeypatch)+ sdk_core = AsyncMock()+ monkeypatch.setattr(+ command_execution_service, "cancel_sdk_task_unserialized", sdk_core+ )+ command = replace(+ _cancel_command(task_id, agent_id, scope="sdk", queued_run_id=QUEUED_RUN),+ target_run_id=QUEUED_RUN,+ )++ await command_execution_service._execute_durable_task_command(command)++ sdk_core.assert_awaited_once_with(+ task_id=task_id,+ agent_id=agent_id,+ expected_run_id=QUEUED_RUN,+ expected_state_version=4,+ queued_run_id=QUEUED_RUN,+ )+++def _update_row(task_id: int, **values: Any) -> None:+ db = _direct_db_session()+ try:+ db.query(Task).filter(Task.id == task_id).update(values)+ db.commit()+ finally:+ db.close()+++def _wait_for_admission(start_id: int, task_id: int) -> None:+ """Give the START an unclaimed admission ticket: it waits for capacity."""+ db = _direct_db_session()+ try:+ if db.get(TaskAdmissionBucket, "test-bucket") is None:+ db.add(TaskAdmissionBucket(key="test-bucket", capacity=1, max_pending=10))+ db.flush()+ db.add(+ TaskAdmissionTicket(+ command_id=start_id, task_id=task_id, bucket_key="test-bucket"+ )+ )+ db.commit()+ finally:+ db.close()+++ ("attempts", "ticket", "stopped"),+ [+ # Sent back to the admission queue after an attempt: still waiting.+ (1, True, True),+ # Held below the capacity cap before any attempt.+ (0, False, True),+ # Claimed before and holding no ticket: it may already have had an+ # effect, and nothing says it still waits.+ (1, False, False),+ ],+ ids=["returned_to_the_admission_queue", "never_attempted", "claimed_before"],+)+async def test_a_queued_start_is_stopped_only_while_a_cancel_can_overtake_it(+ monkeypatch: pytest.MonkeyPatch, attempts: int, ticket: bool, stopped: bool+) -> None:+ task_id, agent_id, owner_user_id = _create_task(+ run_id="run-before", status=TaskStatus.COMPLETED, control_state="completed"+ )+ start_id = _stage_start(task_id, owner_user_id, attempt_count=attempts)+ if ticket:+ _wait_for_admission(start_id, task_id)+ _broadcast_manager(monkeypatch)+ before = _row(task_id)++ if stopped:+ await _cancel_queued(task_id, agent_id)+ assert _start(start_id) == ("failed", STOPPED_BEFORE_START)+ else:+ with pytest.raises(TaskCommandRejected) as raised:+ await _cancel_queued(task_id, agent_id)+ assert raised.value.reason == "stale_run"+ assert _start(start_id) == ("pending", None)+ assert _row(task_id) == before+++ "moved_row",+ [+ {+ "status": TaskStatus.RUNNING,+ "control_state": "running",+ "run_id": "run-first",+ "state_version": 5,+ },+ {+ "status": TaskStatus.WAITING_FOR_USER,+ "control_state": "waiting_for_user",+ "run_id": "run-first",+ "state_version": 5,+ },+ # The same run, one version on: the START was accepted at the older one.+ {+ "status": TaskStatus.COMPLETED,+ "control_state": "completed",+ "state_version": 5,+ },+ ],+ ids=["running_on_another_run", "waiting_on_another_run", "version_moved"],+)+async def test_a_queued_start_that_can_no_longer_hand_off_is_not_withdrawn(+ monkeypatch: pytest.MonkeyPatch, moved_row: dict[str, Any]+) -> None:+ """The row moved after the START was accepted, so the consumer would+ reject the START and the turn never starts. Withdrawing it would answer+ that the turn was cancelled while the run the row holds goes on: the+ command is rejected, with the START and the row as they were."""+ task_id, agent_id, owner_user_id = _create_task(+ run_id="run-before",+ state_version=4,+ status=TaskStatus.COMPLETED,+ control_state="completed",+ )+ start_id = _stage_start(task_id, owner_user_id)+ _update_row(task_id, **moved_row)+ _broadcast_manager(monkeypatch)+ before = _row(task_id)++ with pytest.raises(TaskCommandRejected) as raised:+ await _cancel_queued(task_id, agent_id)++ assert raised.value.reason == "stale_run"+ assert _row(task_id) == before+ assert _start(start_id) == ("pending", None)+ assert _delivery_status(task_id, "turn-queued") == DELIVERY_PENDING+++async def test_a_new_task_whose_row_moved_is_not_cancelled_for_its_queued_start(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ task_id, agent_id, owner_user_id = _create_task(+ status=TaskStatus.PENDING,+ control_state="idle",+ run_id=None,+ state_version=0,+ )+ start_id = _stage_start(task_id, owner_user_id)+ _update_row(task_id, state_version=1)+ _broadcast_manager(monkeypatch)+ before = _row(task_id)++ with pytest.raises(TaskCommandRejected) as raised:+ await _cancel_queued(task_id, agent_id)++ assert raised.value.reason == "stale_run"+ assert _row(task_id) == before+ assert _start(start_id) == ("pending", None)+++def test_a_start_handed_off_while_it_was_being_stopped_is_cancelled_as_running(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ """The START is handed off between the read and the write that stops it:+ the second pass finds the turn running on the queued run and targets it+ at the version read then."""+ task_id, agent_id, owner_user_id = _create_task(+ run_id="run-before", status=TaskStatus.COMPLETED, control_state="completed"+ )+ start_id = _stage_start(task_id, owner_user_id)+ stops: list[int] = []++ def handed_off_first(_db: Any, _task_id: int, start: Any) -> bool:+ stops.append(int(start.id))+ other = _direct_db_session()+ try:+ other.query(TaskExecutionCommand).filter(+ TaskExecutionCommand.id == start.id+ ).update({"status": "completed", "attempt_count": 1})+ other.query(Task).filter(Task.id == task_id).update(+ {+ "status": TaskStatus.RUNNING,+ "control_state": "running",+ "run_id": QUEUED_RUN,+ "state_version": 7,+ }+ )+ other.commit()+ finally:+ other.close()+ return False++ monkeypatch.setattr(+ sdk_task_cancel, "_stop_queued_start_no_commit", handed_off_first+ )++ target = sdk_task_cancel._resolve_queued_cancel_target_sync(+ task_id=task_id, agent_id=agent_id, queued_run_id=QUEUED_RUN+ )++ assert stops == [start_id]+ assert target == sdk_task_cancel._CancelTarget(QUEUED_RUN, 7, TaskStatus.RUNNING)+++ ("status", "queued"),+ [("pending", True), ("processing", True), ("completed", False), ("failed", False)],+)+def test_the_queued_start_lookup_and_the_task_read_share_one_definition(+ status: str, queued: bool+) -> None:+ from xagent.web.services.task_start_protocol import has_queued_start++ task_id, _agent_id, owner_user_id = _create_task(+ run_id="run-before", status=TaskStatus.COMPLETED, control_state="completed"+ )+ _stage_start(task_id, owner_user_id, status=status)+ db = _direct_db_session()+ try:+ assert has_queued_start(db, task_id) is queued+ assert (sdk_task_cancel._queued_start(db, task_id) is not None) is queued+ finally:+ db.close()+++def test_the_stopped_before_start_wording_has_one_definition() -> None:+ """The cancel stops a START with the wording every other control uses, so+ it imports it from where the others write it."""+ import ast+ import pathlib++ from xagent.web.services import task_execution_admission++ assert (+ task_execution_admission.STOPPED_BEFORE_START_REASON+ == "cancelled_before_admission"+ )+ tree = ast.parse(pathlib.Path(sdk_task_cancel.__file__).read_text())+ assigned = {+ target.id+ for node in tree.body+ if isinstance(node, ast.Assign)+ for target in node.targets+ if isinstance(target, ast.Name)+ }+ assert not assigned & {"STOPPED_BEFORE_START", "STOPPED_BEFORE_START_REASON"}+ assert (+ sdk_task_cancel.STOPPED_BEFORE_START_REASON+ == task_execution_admission.STOPPED_BEFORE_START_REASON+ )diff --git a/tests/web/api/v1/test_status_only_projection.py b/tests/web/api/v1/test_status_only_projection.pyindex 0ed6ed1b7..e40c2cac3 100644--- a/tests/web/api/v1/test_status_only_projection.py+++ b/tests/web/api/v1/test_status_only_projection.py@@ -48,6 +48,7 @@ from xagent.web.models.auth_database import get_auth_dbfrom xagent.web.models.database import get_dbfrom xagent.web.models.task import Task, TaskStatusfrom xagent.web.models.task_auto_recovery import TaskAutoRecovery+from xagent.web.models.task_command import TaskExecutionCommandfrom xagent.web.models.user import Userfrom xagent.web.services import sdk_task_cancel, task_auto_resume, task_event_bridgefrom xagent.web.services.hot_path_cache import (@@ -59,6 +60,7 @@ from xagent.web.services.hot_path_cache import (from xagent.web.services.task_auto_recovery import current_auto_recovery_viewfrom xagent.web.services.task_auto_resume import TerminalView, run_auto_resume_tickfrom xagent.web.services.task_lease_service import utc_now+from xagent.web.services.task_start import SdkTaskScopefrom ..conftest import (_admin_headers,@@ -295,12 +297,13 @@ def test_view_call_sites():computed = [c for c in _calls("status_only_terminal_view") if c[0] != defining]# The read surface derives the failed status; the cancel request consults- # the same view so it refuses exactly the tasks that read as failed.+ # the same view, in one place, so it refuses exactly the tasks that read+ # as failed and answers with the status the read reports.cancel_py = pathlib.Path(sdk_task_cancel.__file__).resolve()assert sorted(computed) == sorted([(tasks_py, "_load_task_info_snapshot"),- (cancel_py, "_ended_without_cancel"),+ (cancel_py, "_paused_reads_failed"),])@@ -734,3 +737,54 @@ def test_web_reads_unchanged_for_sdk(api_db):assert info["data"]["auto_recovery"]["auto_resume"]["stop_reason"] == ("limit_reached")++[email protected]("row", ["scheduled", "exhausted"])+def test_a_withdrawn_append_reads_the_same_in_the_cancel_answer_and_the_task_read(+ factory, shared_mode, row+):+ """Withdrawing an appended turn fails its START, and the task read then+ reports the paused row as ``failed`` ("the new message could not be+ started"). The cancel answer, first and repeated, reports that status+ too, not the stored ``paused``."""+ tid, principal = _sdk_task(factory, total=20)+ if row == "exhausted":+ run_auto_resume_tick()+ prepared = _accept(factory, tid)+ scope = SdkTaskScope(agent_id=principal.agent.id, workforce_id=None)+ with factory() as db:+ run_id = str(db.get(TaskExecutionCommand, prepared.command_db_id).target_run_id)+ task = db.get(Task, tid)+ cancel = sdk_task_cancel._enqueue_cancel(+ db, task, int(task.state_version), queued_run_id=run_id+ )+ assert (+ sdk_task_cancel._resolve_queued_cancel_target_sync(+ task_id=tid, agent_id=principal.agent.id, queued_run_id=run_id+ )+ is None+ )+ with factory() as db:+ db.get(TaskExecutionCommand, cancel.command_id).status = "completed"+ db.commit()++ read = v1_tasks._get_chat_task_sync(tid, principal)+ first = sdk_task_cancel._load_result_sync(+ task_id=tid,+ scope=scope,+ aimed_run_id=run_id,+ aimed_at_queued_turn=True,+ command_completed=True,+ )+ repeated = sdk_task_cancel._prepare_cancel_sync(task_id=tid, scope=scope)++ assert read.status == "failed"+ assert read.error == NEW_TURN+ assert repeated == first+ assert first.cancelled and first.cancelled_run_id == run_id+ assert (first.status, first.run_id, first.state_version, first.error_code) == (+ read.status,+ read.run_id,+ read.state_version,+ read.error_code,+ )diff --git a/tests/web/api/v1/test_v1_cancel_task.py b/tests/web/api/v1/test_v1_cancel_task.pyindex 88c1e25a6..50c342f89 100644--- a/tests/web/api/v1/test_v1_cancel_task.py+++ b/tests/web/api/v1/test_v1_cancel_task.py@@ -26,9 +26,11 @@ from xagent.web.services.task_command_transport import (TaskCommandKind,TaskCommandRejected,TaskCommandTaskMissing,- enqueue_task_command,+ stage_task_command,)from xagent.web.services.task_execution_controller import TaskControlState+from xagent.web.services.task_start import SdkTaskScope+from xagent.web.services.task_start_protocol import TaskStartPayloadfrom ..conftest import (_admin_headers,@@ -139,19 +141,56 @@ def _cancel_commands(task_id: int) -> list[TaskExecutionCommand]:db.close()-def _queue_start(task_id: int) -> None:- """Leave an accepted START that no executor has handed off yet."""+QUEUED_RUN = "run-queued"+++def _queue_start(+ task_id: int, *, run_id: str = QUEUED_RUN, turn_id: str = "turn-queued"+) -> None:+ """Leave an accepted START that no executor has handed off yet.++ Its payload is the one the acceptance writes: the run and version the row+ holds now, which is what the START consumer compares the row with.+ """db = _direct_db_session()try:task = db.query(Task).filter(Task.id == task_id).one()- enqueue_task_command(+ start = TaskStartPayload(+ version=1,+ run_id=run_id,+ state_version=int(task.state_version or 0),+ expected_run_id=task.run_id,+ turn_id=turn_id,+ kind="create" if task.status == TaskStatus.PENDING else "append",+ message="hello",+ )+ stage_task_command(db,task_id=task_id,actor_user_id=int(task.user_id),- command_id="turn-queued",+ command_id=start.turn_id,kind=TaskCommandKind.START,- payload={},+ payload=start.model_dump(mode="json"),+ target_run_id=start.run_id,)+ db.commit()+ finally:+ db.close()+++def _stop_queued_start(task_id: int) -> None:+ db = _direct_db_session()+ try:+ db.query(TaskExecutionCommand).filter(+ TaskExecutionCommand.task_id == task_id,+ TaskExecutionCommand.kind == TaskCommandKind.START.value,+ ).update(+ {+ "status": "failed",+ "result": {"rejection_reason": "cancelled_before_admission"},+ }+ )+ db.commit()finally:db.close()@@ -290,28 +329,211 @@ def test_states_that_are_not_running_are_refused_without_a_write(values, expecte"control_state": TaskControlState.COMPLETED.value,"output": "done",},+ {+ "status": TaskStatus.PAUSED,+ "control_state": TaskControlState.PAUSED.value,+ },],- ids=["previous_run_cancelled", "previous_run_completed"],+ ids=["previous_run_cancelled", "previous_run_completed", "previous_run_paused"],)-def test_a_queued_turn_is_never_answered_from_the_previous_run(previous):+def test_a_queued_turn_is_the_target_not_the_run_the_row_still_shows(+ monkeypatch: pytest.MonkeyPatch, previous+):"""While an appended START is queued the row still shows the run before- it, so answering from the row would report on the wrong turn: a- cancelled previous run must not read as "already cancelled"."""+ it. The request aims at the queued run; it is never answered from the+ row, so a cancelled previous run does not read as "already cancelled".+ The status and code read as the task read shows them: ``pending`` after+ a completed or failed run, ``paused`` after a paused one."""task_id, _agent_id, key = _running_task()_update(task_id, runner_id=None, lease_expires_at=None, **previous)_queue_start(task_id)+ # No executor takes the queued START here, so the command behind it waits.+ monkeypatch.setattr(sdk_task_cancel, "dispatch_one_task_command", AsyncMock())+ monkeypatch.setattr(sdk_task_cancel, "SDK_CANCEL_REQUEST_WAIT_SECONDS", 0.1)+ before = _row(task_id)++ response = _cancel(task_id, key)++ assert response.status_code == 202, response.text+ task_read = _get(task_id, key)+ assert response.json() == {+ "task_id": task_id,+ "status": task_read["status"],+ "run_id": before["run_id"],+ "state_version": before["state_version"],+ "cancelled_run_id": None,+ "error_code": task_read["error_code"],+ }+ assert task_read["status"] == (+ "paused" if before["status"] == TaskStatus.PAUSED else "pending"+ )+ assert _row(task_id) == before+ (command,) = _cancel_commands(task_id)+ assert command.payload["queued_run_id"] == QUEUED_RUN+ assert command.target_run_id == QUEUED_RUN+ assert command.command_id.endswith(f":{QUEUED_RUN}")++ # Once the core has stopped the START, the same request reports that+ # turn as cancelled while the task still reads as the run before it.+ _stop_queued_start(task_id)+ monkeypatch.setattr(sdk_task_cancel, "_queued_start", lambda _db, _task_id: None)+ result = sdk_task_cancel._load_result_sync(+ task_id=task_id,+ scope=SdkTaskScope(agent_id=_agent_id, workforce_id=None),+ aimed_run_id=QUEUED_RUN,+ aimed_at_queued_turn=True,+ command_completed=False,+ )+ assert result.cancelled and result.cancelled_run_id == QUEUED_RUN+ assert result.status == before["status"].value+ assert result.run_id == before["run_id"]+ assert result.error_code == (+ "task_cancelled"+ if before["error_message"] == SDK_CANCEL_ERROR_MESSAGE+ else None+ )+ assert _row(task_id) == before+++def test_a_queued_new_task_cancelled_by_its_command_answers_for_the_queued_run(+ monkeypatch: pytest.MonkeyPatch,+):+ """The command completed against a new task that never ran: the answer+ names the queued run as cancelled while the task has no run of its own,+ and every other field reads as the task read does."""+ task_id, _agent_id, key = _running_task()+ _update(+ task_id,+ status=TaskStatus.PENDING,+ control_state="idle",+ run_id=None,+ runner_id=None,+ lease_expires_at=None,+ )+ _queue_start(task_id)+ before = _row(task_id)++ # A cancel is dispatched past a START only while that START waits for+ # admission; this stands in for that dispatch. The core's write is the+ # terminal row, and the dispatcher's settlement of the cancelled+ # admissions then fails the START.+ async def cancelled_as_a_row(_executor: Any, *, command_db_id: int) -> bool:+ _update(+ task_id,+ state_version=before["state_version"] + 1,+ **_cancelled_row_values(),+ )+ _stop_queued_start(task_id)+ _settle_cancel_command(command_db_id, status="completed")+ return True++ monkeypatch.setattr(+ sdk_task_cancel, "dispatch_one_task_command", cancelled_as_a_row+ )++ response = _cancel(task_id, key)++ assert response.status_code == 200, response.text+ assert response.json() == {+ "task_id": task_id,+ "status": "failed",+ "run_id": None,+ "state_version": before["state_version"] + 1,+ "cancelled_run_id": QUEUED_RUN,+ "error_code": "task_cancelled",+ }+ task_read = _get(task_id, key)+ assert (task_read["status"], task_read["error_code"]) == (+ "failed",+ "task_cancelled",+ )+ assert task_read["run_id"] is None+++def test_a_queued_turn_whose_cancel_went_stale_while_it_waits_is_busy(+ monkeypatch: pytest.MonkeyPatch,+):+ """The command was rejected because the task moved, and the aimed START+ is still waiting: nothing was cancelled and a retry can succeed."""+ task_id, _agent_id, key = _running_task()+ _update(+ task_id,+ status=TaskStatus.COMPLETED,+ control_state=TaskControlState.COMPLETED.value,+ runner_id=None,+ lease_expires_at=None,+ )+ _queue_start(task_id)before = _row(task_id)+ async def went_stale(_executor: Any, *, command_db_id: int) -> bool:+ _settle_cancel_command(+ command_db_id, status="failed", rejection_reason="stale_run"+ )+ return True++ monkeypatch.setattr(sdk_task_cancel, "dispatch_one_task_command", went_stale)+response = _cancel(task_id, key)assert response.status_code == 409, response.textassert _error(response) == "task_busy"assert _row(task_id) == before- assert _cancel_commands(task_id) == []- # The read surface agrees: the queued turn is pending and carries no code.- read = _get(task_id, key)- assert read["status"] == "pending"- assert read["error_code"] is None+++def test_a_queued_turn_that_failed_to_start_is_not_reported_from_the_previous_run(+ monkeypatch: pytest.MonkeyPatch,+):+ """The aimed turn never started and was not stopped by this cancel; the+ row is still the cancelled run before it, which is not the answer. The+ aimed turn is over, so a retry -- which would report that earlier run --+ is not invited."""+ task_id, _agent_id, key = _running_task()+ _update(+ task_id,+ status=TaskStatus.FAILED,+ control_state=TaskControlState.FAILED.value,+ error_message=SDK_CANCEL_ERROR_MESSAGE,+ runner_id=None,+ lease_expires_at=None,+ )+ _queue_start(task_id)++ async def start_fails_and_the_cancel_goes_stale(+ _executor: Any, *, command_db_id: int+ ) -> bool:+ db = _direct_db_session()+ try:+ commands = db.query(TaskExecutionCommand).filter(+ TaskExecutionCommand.task_id == task_id+ )+ commands.filter(+ TaskExecutionCommand.kind == TaskCommandKind.START.value+ ).update(+ {"status": "failed", "result": {"rejection_reason": "invalid_start"}}+ )+ commands.filter(TaskExecutionCommand.id == command_db_id).update(+ {"status": "failed", "result": {"rejection_reason": "stale_run"}}+ )+ db.commit()+ finally:+ db.close()+ return True++ monkeypatch.setattr(+ sdk_task_cancel,+ "dispatch_one_task_command",+ start_fails_and_the_cancel_goes_stale,+ )++ response = _cancel(task_id, key)++ assert response.status_code == 409, response.text+ assert _error(response) == "task_not_cancellable"+ assert (+ response.json()["error"]["message"]+ == "The run this cancel was aimed at has already ended."+ )def test_a_task_that_reads_failed_after_automatic_resume_gave_up_is_not_cancellable(@@ -494,6 +716,29 @@ def _cancelled_row_values() -> dict[str, Any]:}+def _settle_cancel_command(+ command_db_id: int, *, status: str, rejection_reason: str | None = None+) -> None:+ """End a CANCEL command the way the dispatcher records its outcome."""+ db = _direct_db_session()+ try:+ db.query(TaskExecutionCommand).filter(+ TaskExecutionCommand.id == command_db_id+ ).update(+ {+ "status": status,+ "result": (+ {"rejection_reason": rejection_reason}+ if rejection_reason is not None+ else {}+ ),+ }+ )+ db.commit()+ finally:+ db.close()++def _replace_cancel_core(monkeypatch: pytest.MonkeyPatch, core: Any) -> None:monkeypatch.setattr(command_execution_service, "cancel_sdk_task_unserialized", core)@@ -658,7 +903,8 @@ def test_a_turn_queued_while_the_cancel_was_applied_still_reports_the_cancel():"""The command ended, the task was cancelled by another request and anew turn was queued before this request read the outcome: the run this- call aimed at is cancelled, so the answer says so."""+ call aimed at is cancelled, so the answer says so, while the other+ fields read as the task read does with a turn queued."""task_id, _agent_id, key = _running_task()before = _row(task_id)@@ -679,11 +925,11 @@ def test_a_turn_queued_while_the_cancel_was_applied_still_reports_the_cancel(assert response.status_code == 200, response.textassert response.json() == {"task_id": task_id,- "status": "failed",+ "status": "pending","run_id": before["run_id"],"state_version": before["state_version"],"cancelled_run_id": before["run_id"],- "error_code": "task_cancelled",+ "error_code": None,}@@ -710,11 +956,11 @@ def test_a_completed_command_is_a_cancel_of_its_aimed_run_even_if_a_turn_is_queuassert response.status_code == 200, response.textassert response.json() == {"task_id": task_id,- "status": "failed",+ "status": "pending","run_id": before["run_id"],"state_version": before["state_version"] + 1,"cancelled_run_id": before["run_id"],- "error_code": "task_cancelled",+ "error_code": None,}@@ -825,3 +1071,519 @@ def test_the_wait_is_timed_from_before_the_command_is_dispatched(assert response.status_code == 202, response.textassert len(polls) == 1+++# ----- a cancel repeated after it withdrew a queued turn -----++_PREVIOUS_RUNS = {+ "previous_run_completed": {+ "status": TaskStatus.COMPLETED,+ "control_state": TaskControlState.COMPLETED.value,+ "output": "done",+ },+ "previous_run_cancelled": {+ "status": TaskStatus.FAILED,+ "control_state": TaskControlState.FAILED.value,+ "error_message": SDK_CANCEL_ERROR_MESSAGE,+ },+ "previous_run_paused": {+ "status": TaskStatus.PAUSED,+ "control_state": TaskControlState.PAUSED.value,+ },+ "new_task": {"status": TaskStatus.PENDING, "control_state": "idle", "run_id": None},+}+++def _dispatch_through_the_core(monkeypatch: pytest.MonkeyPatch, agent_id: int) -> None:+ """Apply each cancel command with the real core, as its owner would.++ A cancel is dispatched past a queued START only while that START waits for+ admission, which these tests do not set up; the core and the settlement it+ leaves behind are the real ones.+ """++ async def apply(_executor: Any, *, command_db_id: int) -> bool:+ db = _direct_db_session()+ try:+ command = db.get(TaskExecutionCommand, command_db_id)+ task_id = int(command.task_id)+ run_id = command.target_run_id+ version = int(command.target_state_version)+ queued_run_id = command.payload.get("queued_run_id")+ finally:+ db.close()+ await sdk_task_cancel.cancel_sdk_task_unserialized(+ task_id=task_id,+ agent_id=agent_id,+ expected_run_id=run_id,+ expected_state_version=version,+ queued_run_id=queued_run_id,+ )+ # A new task's START is settled by the dispatcher once the cancel+ # has cancelled the row; an appended turn's was stopped by the core.+ db = _direct_db_session()+ try:+ db.query(TaskExecutionCommand).filter(+ TaskExecutionCommand.task_id == task_id,+ TaskExecutionCommand.kind == TaskCommandKind.START.value,+ TaskExecutionCommand.status == "pending",+ ).update(+ {+ "status": "failed",+ "result": {"rejection_reason": "cancelled_before_admission"},+ }+ )+ db.commit()+ finally:+ db.close()+ _settle_cancel_command(command_db_id, status="completed")+ return True++ monkeypatch.setattr(sdk_task_cancel, "dispatch_one_task_command", apply)++[email protected]("previous", list(_PREVIOUS_RUNS))+def test_a_repeated_cancel_answers_for_the_turn_the_first_one_withdrew(+ monkeypatch: pytest.MonkeyPatch, previous: str+):+ """The withdrawn turn leaves the row on the turn before it, so the repeat+ cannot read the turn from the row: it is answered from the record of the+ withdrawal, identically, and adds no second command."""+ task_id, agent_id, key = _running_task()+ _update(task_id, runner_id=None, lease_expires_at=None, **_PREVIOUS_RUNS[previous])+ _queue_start(task_id)+ _dispatch_through_the_core(monkeypatch, agent_id)++ first = _cancel(task_id, key)+ again = _cancel(task_id, key)++ assert first.status_code == 200, first.text+ assert first.json()["cancelled_run_id"] == QUEUED_RUN+ if previous != "new_task":+ # The row still shows the turn before the withdrawn one.+ assert first.json()["run_id"] not in (None, QUEUED_RUN)+ assert again.status_code == 200, again.text+ assert again.json() == first.json()+ assert len(_cancel_commands(task_id)) == 1+++def _apply_without_completing(monkeypatch: pytest.MonkeyPatch, agent_id: int) -> None:+ """Apply the cancel with the real core and stop before it is completed.++ The command is claimed as the dispatcher claims it and the core commits+ its effect -- the withdrawn START, or a new task's cancelled row -- but+ the separate transaction that marks the command completed (and, for a+ new task, settles its START) has not run yet.+ """++ async def apply(_executor: Any, *, command_db_id: int) -> bool:+ db = _direct_db_session()+ try:+ command = db.get(TaskExecutionCommand, command_db_id)+ command.status = "processing"+ command.attempt_count = 1+ db.commit()+ task_id = int(command.task_id)+ run_id = command.target_run_id+ version = int(command.target_state_version)+ queued_run_id = command.payload.get("queued_run_id")+ finally:+ db.close()+ await sdk_task_cancel.cancel_sdk_task_unserialized(+ task_id=task_id,+ agent_id=agent_id,+ expected_run_id=run_id,+ expected_state_version=version,+ queued_run_id=queued_run_id,+ )+ return True++ monkeypatch.setattr(sdk_task_cancel, "dispatch_one_task_command", apply)+++def _complete_applied_cancel(task_id: int, command_db_id: int) -> None:+ """The step ``_apply_without_completing`` left out: a new task's START is+ settled and the command is marked completed."""+ db = _direct_db_session()+ try:+ db.query(TaskExecutionCommand).filter(+ TaskExecutionCommand.task_id == task_id,+ TaskExecutionCommand.kind == TaskCommandKind.START.value,+ TaskExecutionCommand.status == "pending",+ ).update(+ {+ "status": "failed",+ "result": {"rejection_reason": "cancelled_before_admission"},+ }+ )+ db.commit()+ finally:+ db.close()+ _settle_cancel_command(command_db_id, status="completed")+++def _withdrawn_before_completion(+ monkeypatch: pytest.MonkeyPatch, previous: str+) -> tuple[int, str, Any, TaskExecutionCommand]:+ """Queue a turn, cancel it, and leave the cancel applied but processing."""+ task_id, agent_id, key = _running_task()+ _update(task_id, runner_id=None, lease_expires_at=None, **_PREVIOUS_RUNS[previous])+ _queue_start(task_id)+ _apply_without_completing(monkeypatch, agent_id)+ monkeypatch.setattr(sdk_task_cancel, "SDK_CANCEL_REQUEST_WAIT_SECONDS", 0.2)+ first = _cancel(task_id, key)+ (command,) = _cancel_commands(task_id)+ assert command.status == "processing"+ return task_id, key, first, command++[email protected]("previous", list(_PREVIOUS_RUNS))+def test_a_repeat_while_the_withdrawing_cancel_is_still_processing_rejoins_it(+ monkeypatch: pytest.MonkeyPatch, previous: str+):+ """The cancel withdrew the queued turn but is not marked completed yet.+ A repeat in that interval finds the same command, adds none, and answers+ exactly as the first call did at the same point: an appended turn reads+ as withdrawn, a new task as still being cancelled; once the command+ completes the new task reads as cancelled for the queued run."""+ task_id, key, first, command = _withdrawn_before_completion(monkeypatch, previous)+ dispatched = AsyncMock()+ monkeypatch.setattr(sdk_task_cancel, "dispatch_one_task_command", dispatched)++ again = _cancel(task_id, key)++ assert dispatched.await_args.kwargs["command_db_id"] == command.id+ assert [c.id for c in _cancel_commands(task_id)] == [command.id]+ assert again.status_code == first.status_code, again.text+ assert again.json() == first.json()+ if previous != "new_task":+ assert first.status_code == 200, first.text+ assert first.json()["cancelled_run_id"] == QUEUED_RUN+ # The row still shows the turn before the withdrawn one.+ assert first.json()["run_id"] not in (None, QUEUED_RUN)+ return+ assert first.status_code == 202, first.text+ assert first.json()["cancelled_run_id"] is None+ _complete_applied_cancel(task_id, command.id)++ after_completion = _cancel(task_id, key)++ assert after_completion.status_code == 200, after_completion.text+ assert after_completion.json()["run_id"] is None+ assert after_completion.json()["cancelled_run_id"] == QUEUED_RUN+ assert after_completion.json()["error_code"] == "task_cancelled"+ assert len(_cancel_commands(task_id)) == 1+++def test_a_repeat_while_processing_after_the_turn_before_resumed_cancels_that_turn(+ monkeypatch: pytest.MonkeyPatch,+):+ """The withdrawing cancel is still processing when the paused turn before+ it is resumed on a newer version. That command no longer describes the+ row, so the repeat is not answered with the withdrawal: it aims a new+ command at the running turn."""+ task_id, key, _first, command = _withdrawn_before_completion(+ monkeypatch, "previous_run_paused"+ )+ resumed = _row(task_id)+ _update(+ task_id,+ status=TaskStatus.RUNNING,+ control_state=TaskControlState.RUNNING.value,+ state_version=resumed["state_version"] + 1,+ )+ monkeypatch.setattr(sdk_task_cancel, "dispatch_one_task_command", AsyncMock())+ monkeypatch.setattr(sdk_task_cancel, "SDK_CANCEL_REQUEST_WAIT_SECONDS", 0.1)++ response = _cancel(task_id, key)++ assert response.status_code == 202, response.text+ assert response.json()["status"] == "running"+ assert response.json()["run_id"] == resumed["run_id"]+ assert response.json()["cancelled_run_id"] is None+ commands = sorted(_cancel_commands(task_id), key=lambda c: c.id)+ assert len(commands) == 2+ assert commands[0].id == command.id+ assert commands[-1].target_run_id == resumed["run_id"]+++def test_a_withdrawal_does_not_answer_once_the_turn_before_runs_again(+ monkeypatch: pytest.MonkeyPatch,+):+ """After the withdrawal the turn before it was resumed and runs again on+ a newer version. The record of the withdrawal no longer describes the+ row, so the cancel aims at the running turn instead of answering that+ the withdrawn one is cancelled while the running one goes on."""+ task_id, agent_id, key = _running_task()+ _update(+ task_id,+ runner_id=None,+ lease_expires_at=None,+ status=TaskStatus.PAUSED,+ control_state=TaskControlState.PAUSED.value,+ )+ _queue_start(task_id)+ _dispatch_through_the_core(monkeypatch, agent_id)+ assert _cancel(task_id, key).status_code == 200+ resumed = _row(task_id)+ _update(+ task_id,+ status=TaskStatus.RUNNING,+ control_state=TaskControlState.RUNNING.value,+ state_version=resumed["state_version"] + 1,+ )+ monkeypatch.setattr(sdk_task_cancel, "dispatch_one_task_command", AsyncMock())+ monkeypatch.setattr(sdk_task_cancel, "SDK_CANCEL_REQUEST_WAIT_SECONDS", 0.1)++ response = _cancel(task_id, key)++ assert response.status_code == 202, response.text+ assert response.json()["status"] == "running"+ assert response.json()["run_id"] == resumed["run_id"]+ assert response.json()["cancelled_run_id"] is None+ commands = _cancel_commands(task_id)+ assert len(commands) == 2+ assert commands[-1].target_run_id == resumed["run_id"]+++def _fail_newest_start(task_id: int, outcome: dict[str, Any]) -> None:+ db = _direct_db_session()+ try:+ db.query(TaskExecutionCommand).filter(+ TaskExecutionCommand.task_id == task_id,+ TaskExecutionCommand.kind == TaskCommandKind.START.value,+ ).update(outcome)+ db.commit()+ finally:+ db.close()+++ "start_outcome",+ [+ {"status": "failed", "result": {"rejection_reason": "invalid_start"}},+ # Stopped by a control other than this endpoint's cancel.+ {+ "status": "failed",+ "result": {"rejection_reason": "cancelled_before_admission"},+ },+ ],+ ids=["failed_itself", "stopped_by_another_control"],+)+def test_a_start_that_no_cancel_of_this_endpoint_withdrew_is_not_a_cancellation(+ start_outcome,+):+ """The newest START failed and the row shows the completed turn before+ it. With no SDK cancel aimed at that START's run the failure is not a+ cancellation, so the call is refused as it would be on arrival."""+ task_id, _agent_id, key = _running_task()+ _update(+ task_id,+ runner_id=None,+ lease_expires_at=None,+ **_PREVIOUS_RUNS["previous_run_completed"],+ )+ _queue_start(task_id)+ _fail_newest_start(task_id, start_outcome)+ before = _row(task_id)++ response = _cancel(task_id, key)++ assert response.status_code == 409, response.text+ assert _error(response) == "task_not_cancellable"+ assert _row(task_id) == before+ assert _cancel_commands(task_id) == []+++def test_a_cancel_not_completed_yet_for_a_stopped_start_is_rejoined(+ monkeypatch: pytest.MonkeyPatch,+):+ """The newest START was stopped and this endpoint's cancel aimed at its+ run is still pending. The call rejoins that command instead of judging+ the row or staging another one, and answers as the command's own wait+ does: at the deadline a START stopped before it began is that turn+ cancelled."""+ task_id, _agent_id, key = _running_task()+ _update(+ task_id,+ runner_id=None,+ lease_expires_at=None,+ **_PREVIOUS_RUNS["previous_run_completed"],+ )+ _queue_start(task_id)+ db = _direct_db_session()+ try:+ task = db.get(Task, task_id)+ pending = sdk_task_cancel._enqueue_cancel(+ db, task, int(task.state_version), queued_run_id=QUEUED_RUN+ )+ finally:+ db.close()+ _fail_newest_start(+ task_id,+ {+ "status": "failed",+ "result": {"rejection_reason": "cancelled_before_admission"},+ },+ )+ dispatched = AsyncMock()+ monkeypatch.setattr(sdk_task_cancel, "dispatch_one_task_command", dispatched)+ monkeypatch.setattr(sdk_task_cancel, "SDK_CANCEL_REQUEST_WAIT_SECONDS", 0.1)+ before = _row(task_id)++ response = _cancel(task_id, key)++ assert response.status_code == 200, response.text+ assert response.json()["run_id"] == before["run_id"]+ assert response.json()["cancelled_run_id"] == QUEUED_RUN+ assert dispatched.await_args.kwargs["command_db_id"] == pending.command_id+ assert [c.id for c in _cancel_commands(task_id)] == [pending.command_id]+ assert _row(task_id) == before+++def test_a_cancel_after_a_withdrawn_turn_and_a_newer_started_one_answers_for_the_row(+ monkeypatch: pytest.MonkeyPatch,+):+ """A turn was withdrawn, then another was appended and is running: the+ newest START is the row's own run, so the cancel aims at the running+ turn and is not answered with the withdrawal."""+ task_id, agent_id, key = _running_task()+ _update(+ task_id,+ runner_id=None,+ lease_expires_at=None,+ **_PREVIOUS_RUNS["previous_run_completed"],+ )+ _queue_start(task_id)+ _dispatch_through_the_core(monkeypatch, agent_id)+ assert _cancel(task_id, key).status_code == 200+ _queue_start(task_id, run_id="run-second", turn_id="turn-second")+ _update(+ task_id,+ status=TaskStatus.RUNNING,+ control_state=TaskControlState.RUNNING.value,+ run_id="run-second",+ )+ db = _direct_db_session()+ try:+ db.query(TaskExecutionCommand).filter(+ TaskExecutionCommand.task_id == task_id,+ TaskExecutionCommand.command_id == "turn-second",+ ).update({"status": "completed"})+ db.commit()+ finally:+ db.close()+ monkeypatch.setattr(sdk_task_cancel, "dispatch_one_task_command", AsyncMock())+ monkeypatch.setattr(sdk_task_cancel, "SDK_CANCEL_REQUEST_WAIT_SECONDS", 0.1)++ response = _cancel(task_id, key)++ assert response.status_code == 202, response.text+ assert response.json()["run_id"] == "run-second"+ assert response.json()["cancelled_run_id"] is None+ commands = _cancel_commands(task_id)+ assert len(commands) == 2+ assert commands[-1].target_run_id == "run-second"+++# ----- a queued START that can no longer hand off is not the target -----+++ "moved",+ ["running_on_another_run", "waiting_on_another_run"],+)+def test_a_queued_start_that_can_no_longer_hand_off_is_not_the_cancel_target(+ monkeypatch: pytest.MonkeyPatch, moved: str+):+ """The row was moved to another run after the START was accepted (an+ automatic resume won the race), so the START never opens its run. A+ running row is cancelled as the running run it is; one waiting for an+ answer is refused as busy, as it would be with no START at all."""+ task_id, _agent_id, key = _running_task()+ _update(+ task_id,+ runner_id=None,+ lease_expires_at=None,+ **_PREVIOUS_RUNS["previous_run_completed"],+ )+ _queue_start(task_id)+ row_values: dict[str, Any] = {+ "run_id": "run-first",+ "state_version": _row(task_id)["state_version"] + 1,+ }+ if moved == "running_on_another_run":+ row_values.update(+ status=TaskStatus.RUNNING, control_state=TaskControlState.RUNNING.value+ )+ else:+ row_values.update(+ status=TaskStatus.WAITING_FOR_USER,+ control_state=TaskControlState.WAITING_FOR_USER.value,+ )+ _update(task_id, **row_values)+ monkeypatch.setattr(sdk_task_cancel, "dispatch_one_task_command", AsyncMock())+ monkeypatch.setattr(sdk_task_cancel, "SDK_CANCEL_REQUEST_WAIT_SECONDS", 0.1)+ before = _row(task_id)++ response = _cancel(task_id, key)++ assert _row(task_id) == before+ if moved == "waiting_on_another_run":+ assert response.status_code == 409, response.text+ assert _error(response) == "task_busy"+ assert _cancel_commands(task_id) == []+ return+ assert response.status_code == 202, response.text+ assert response.json()["run_id"] == "run-first"+ (command,) = _cancel_commands(task_id)+ assert "queued_run_id" not in command.payload+ assert command.target_run_id == "run-first"+++# ----- how the wait ends for a queued turn -----+++def test_a_cancel_whose_queued_turn_started_before_the_deadline_is_accepted(+ monkeypatch: pytest.MonkeyPatch,+):+ """At the deadline the START has been handed off and the row shows the+ turn running on the aimed run: the command is not applied yet, so the+ answer is 202 for the running turn, not "already ended"."""+ task_id, _agent_id, key = _running_task()+ _update(+ task_id,+ runner_id=None,+ lease_expires_at=None,+ **_PREVIOUS_RUNS["previous_run_completed"],+ )+ _queue_start(task_id)++ async def handed_off(_executor: Any, *, command_db_id: int) -> bool:+ _update(+ task_id,+ status=TaskStatus.RUNNING,+ control_state=TaskControlState.RUNNING.value,+ run_id=QUEUED_RUN,+ state_version=_row(task_id)["state_version"] + 1,+ )+ db = _direct_db_session()+ try:+ db.query(TaskExecutionCommand).filter(+ TaskExecutionCommand.task_id == task_id,+ TaskExecutionCommand.kind == TaskCommandKind.START.value,+ ).update({"status": "completed"})+ db.commit()+ finally:+ db.close()+ return False++ monkeypatch.setattr(sdk_task_cancel, "dispatch_one_task_command", handed_off)+ monkeypatch.setattr(sdk_task_cancel, "SDK_CANCEL_REQUEST_WAIT_SECONDS", 0.1)++ response = _cancel(task_id, key)++ assert response.status_code == 202, response.text+ assert response.json()["status"] == "running"+ assert response.json()["run_id"] == QUEUED_RUN+ assert response.json()["cancelled_run_id"] is Nonediff --git a/tests/web/services/test_queued_start_cancel_admission.py b/tests/web/services/test_queued_start_cancel_admission.pyindex c1663464c..e88975cf6 100644--- a/tests/web/services/test_queued_start_cancel_admission.py+++ b/tests/web/services/test_queued_start_cancel_admission.py@@ -19,13 +19,16 @@ from xagent.web.models.task_runtime_secret import TaskRuntimeSecretfrom xagent.web.services import (a2a_task_cancel,external_task_cancel,+ sdk_task_cancel,task_command_execution,)from xagent.web.services import task_command_transport as transport+from xagent.web.services import task_execution_admission as admissionfrom xagent.web.services.chat_history_service import (DELIVERY_DISPATCHED,DELIVERY_FAILED,)+from xagent.web.services.task_admission_execution import AdmissionWaitingfrom xagent.web.services.task_orchestrator import TaskTurnOrchestrator, TaskTurnPayloadfrom xagent.web.services.task_runtime_secrets import stage_runtime_values@@ -38,7 +41,7 @@ from .test_task_execution_admission import ()[email protected]("scope", ["a2a", "external"])[email protected]("scope", ["a2a", "external", "sdk"])async def test_cancel_settles_capacity_waiting_start(scope, host, monkeypatch):"""External runtime input is the supported host contract for its START."""monkeypatch.setenv("ENCRYPTION_KEY", Fernet.generate_key().decode())@@ -49,6 +52,7 @@ async def test_cancel_settles_capacity_waiting_start(scope, host, monkeypatch):monkeypatch.setattr(external_task_cancel, "get_session_local", lambda: host.sessions)+ monkeypatch.setattr(sdk_task_cancel, "get_session_local", lambda: host.sessions)monkeypatch.setattr("xagent.web.services.task_event_bridge.get_task_event_bridge", lambda: Mock())@@ -116,18 +120,26 @@ async def test_cancel_settles_capacity_waiting_start(scope, host, monkeypatch):)assert executed == []- payload = {"agent_id": agent_id, "target_state_version": 0}- if scope == "external":- payload.update({"scope": "external", "turn_id": turn_id})with host.sessions() as db:- cancel = transport.enqueue_task_command(- db,- task_id=task_id,- actor_user_id=host.user,- command_id=f"cancel:{task_id}:0",- kind=transport.TaskCommandKind.CANCEL,- payload=payload,- )+ if scope == "sdk":+ # Built as the endpoint builds it, which aims the command at+ # the queued turn's run.+ queued_run_id = db.get(TaskExecutionCommand, start_id).target_run_id+ cancel = sdk_task_cancel._enqueue_cancel(+ db, db.get(Task, task_id), 0, queued_run_id=queued_run_id+ )+ else:+ payload = {"agent_id": agent_id, "target_state_version": 0}+ if scope == "external":+ payload.update({"scope": "external", "turn_id": turn_id})+ cancel = transport.enqueue_task_command(+ db,+ task_id=task_id,+ actor_user_id=host.user,+ command_id=f"cancel:{task_id}:0",+ kind=transport.TaskCommandKind.CANCEL,+ payload=payload,+ )assert await transport.dispatch_one_task_command(task_command_execution.execute_durable_task_command,@@ -152,11 +164,11 @@ async def test_cancel_settles_capacity_waiting_start(scope, host, monkeypatch):}assert (task.status, task.error_message) == (TaskStatus.FAILED,- (- "Stopped by the visitor."- if scope == "external"- else "Task canceled by A2A client."- ),+ {+ "external": "Stopped by the visitor.",+ "a2a": "Task canceled by A2A client.",+ "sdk": sdk_task_cancel.SDK_CANCEL_ERROR_MESSAGE,+ }[scope],)assert queued_start.status == "failed"assert queued_start.error == "Task command stopped by a control request."@@ -181,3 +193,83 @@ async def test_cancel_settles_capacity_waiting_start(scope, host, monkeypatch):execute, command_db_id=start_id)assert executed == []+++async def test_an_appended_turn_sent_back_to_the_admission_queue_is_withdrawn(+ host, monkeypatch+):+ """A START that was attempted and sent back to wait for capacity still+ waits for admission, so the cancel overtakes it and withdraws the turn,+ leaving the task as the turn before it."""+ monkeypatch.setenv("ENCRYPTION_KEY", Fernet.generate_key().decode())+ monkeypatch.setattr(+ task_command_execution, "get_session_local", lambda: host.sessions+ )+ monkeypatch.setattr(sdk_task_cancel, "get_session_local", lambda: host.sessions)+ monkeypatch.setattr(+ "xagent.web.services.task_event_bridge.get_task_event_bridge", lambda: Mock()+ )+ with host.sessions() as db:+ agent = Agent(user_id=host.user, name="Appended sdk cancellation")+ db.add(agent)+ db.flush()+ task = Task(+ user_id=host.user,+ agent_id=agent.id,+ source="sdk",+ title="Appended sdk task",+ status=TaskStatus.COMPLETED,+ control_state="completed",+ run_id="run-before",+ state_version=3,+ )+ db.add(task)+ db.flush()+ task_id = int(task.id)+ accepted = TaskTurnOrchestrator.claim_append_turn_no_commit(+ db,+ task_id=task_id,+ task_owner_user_id=host.user,+ payload=TaskTurnPayload("Append the turn", turn_id="turn-appended"),+ )+ db.commit()+ start_id = int(accepted.command_db_id)++ async def no_capacity(_command):+ raise AdmissionWaiting("Waiting for execution admission")++ assert await transport.dispatch_one_task_command(+ no_capacity, command_db_id=start_id+ )+ # The owner drains and releases its tickets: the command waits again.+ with host.sessions() as db, db.begin():+ admission._release_tickets(db, TaskAdmissionTicket.command_id == start_id)+ with host.sessions() as db:+ start = db.get(TaskExecutionCommand, start_id)+ ticket = db.get(TaskAdmissionTicket, start_id)+ assert (start.status, start.attempt_count) == ("pending", 1)+ assert ticket.owner_attempt_id is None+ queued_run_id = start.target_run_id+ cancel = sdk_task_cancel._enqueue_cancel(+ db, db.get(Task, task_id), 3, queued_run_id=queued_run_id+ )++ assert await transport.dispatch_one_task_command(+ task_command_execution.execute_durable_task_command,+ command_db_id=cancel.command_id,+ )++ with host.sessions() as db:+ start = db.get(TaskExecutionCommand, start_id)+ task = db.get(Task, task_id)+ assert db.get(TaskExecutionCommand, cancel.command_id).status == "completed"+ assert start.status == "failed"+ assert start.result == {"rejection_reason": "cancelled_before_admission"}+ assert (task.status, task.run_id, task.state_version, task.error_message) == (+ TaskStatus.COMPLETED,+ "run-before",+ 3,+ None,+ )+ delivery = db.query(TaskChatMessage).filter_by(turn_id="turn-appended").one()+ assert delivery.delivery_status == DELIVERY_FAILED