Skip to content

Automate browser based workflows with AI

Assemble a shadow terminal outcome envelope per copilot turn (#7671)

pedrohsdb17fc4ea58d0e
diff --git a/skyvern/forge/sdk/copilot/agent.py b/skyvern/forge/sdk/copilot/agent.py
index 7850e27d0..1a3a15f6e 100644
--- a/skyvern/forge/sdk/copilot/agent.py
+++ b/skyvern/forge/sdk/copilot/agent.py
@@ -13,7 +13,7 @@ import json
import math
import re
import uuid
-from collections.abc import Callable, Iterator, Mapping
+from collections.abc import Callable, Iterator, Mapping, Sequence
from dataclasses import dataclass
from datetime import datetime, timezone
from io import BytesIO
@@ -186,6 +186,7 @@ from skyvern.forge.sdk.copilot.streaming_adapter import (
flush_goal_satisfied_tool_result,
maybe_emit_design_end,
)
+from skyvern.forge.sdk.copilot.terminal_envelope import assemble_terminal_envelope, reason_in_reply_shadow
from skyvern.forge.sdk.copilot.tracing_setup import _copilot_model_name, ensure_tracing_initialized, is_tracing_enabled
from skyvern.forge.sdk.copilot.turn_context import TurnContextAssembler, TurnContextInputs, TurnContextPacket
from skyvern.forge.sdk.copilot.turn_halt import (
@@ -1729,6 +1730,82 @@ def _should_use_built_unverified_completed_reply(
)
+def _terminal_envelope_run_outcomes(ctx: CopilotContext) -> list[RecordedRunOutcome]:
+ raw = ctx.terminal_envelope_run_outcomes
+ if isinstance(raw, Sequence) and not isinstance(raw, (str, bytes, bytearray)):
+ outcomes = [outcome for outcome in raw if isinstance(outcome, RecordedRunOutcome)]
+ if outcomes:
+ return outcomes
+ recorded = ctx.last_run_outcome
+ return [recorded] if isinstance(recorded, RecordedRunOutcome) else []
+
+
+def _terminal_halt_fields(ctx: CopilotContext) -> tuple[str | None, str | None]:
+ halt = getattr(ctx, "turn_halt", None)
+ if not isinstance(halt, TurnHalt):
+ return None, None
+ blocker_reason: str | None = None
+ signal = halt.blocker_signal
+ if isinstance(signal, CopilotToolBlockerSignal):
+ blocker_reason = signal.user_facing_reason
+ return blocker_reason, halt.kind.value
+
+
+def _attempted_summary(narrative_summary: object, narrative_payload: object) -> str | None:
+ if isinstance(narrative_summary, str) and narrative_summary.strip():
+ return narrative_summary.strip()
+ if isinstance(narrative_payload, Mapping):
+ payload_summary = narrative_payload.get("narrativeSummary")
+ if isinstance(payload_summary, str) and payload_summary.strip():
+ return payload_summary.strip()
+ return None
+
+
+def _assemble_terminal_envelope_safe(
+ *,
+ response_type: str,
+ verified: bool,
+ workflow_applied: bool,
+ proposal_disposition: str | None,
+ run_outcomes: Sequence[RecordedRunOutcome],
+ blocker_reason: str | None,
+ halt_kind: str | None,
+ attempted: str | None,
+ workflow_mutated: bool,
+ turn_outcome_response_kind: str | None,
+ final_message: str,
+) -> dict[str, Any] | None:
+ try:
+ envelope = assemble_terminal_envelope(
+ response_type=response_type,
+ verified=verified,
+ workflow_applied=workflow_applied,
+ proposal_disposition=proposal_disposition,
+ run_outcomes=run_outcomes,
+ blocker_reason=blocker_reason,
+ halt_kind=halt_kind,
+ attempted=attempted,
+ workflow_mutated=workflow_mutated,
+ turn_outcome_response_kind=turn_outcome_response_kind,
+ )
+ except Exception:
+ LOG.warning("copilot terminal envelope assembly failed", exc_info=True)
+ return None
+ if envelope is None:
+ return None
+ reason_in_reply = reason_in_reply_shadow(envelope.run_display_reason, final_message)
+ payload = envelope.model_dump(mode="json")
+ LOG.info(
+ "copilot_terminal_envelope",
+ **payload,
+ response_type=response_type,
+ envelope_response_kind=envelope.response_kind,
+ reason_in_reply=reason_in_reply,
+ finalized=False,
+ )
+ return payload
+
+
def _make_agent_result(
ctx: CopilotContext | None,
*,
@@ -1795,6 +1872,25 @@ def _make_agent_result(
payload_updates["deliveredUnverifiedObservedOutputs"] = _delivered_unverified_snapshot
if payload_updates or len(payload_base) != len(narrative_payload):
kwargs["narrative_payload"] = {**payload_base, **payload_updates}
+ terminal_envelope: dict[str, Any] | None = None
+ if ctx is not None:
+ blocker_reason, halt_kind = _terminal_halt_fields(ctx)
+ turn_outcome_response_kind = turn_outcome.response_kind.value if turn_outcome is not None else None
+ response_type_value = response_type if isinstance(response_type, str) else "REPLY"
+ terminal_envelope = _assemble_terminal_envelope_safe(
+ response_type=response_type_value,
+ verified=bool(verified_goal_claim_authorized(ctx)),
+ workflow_applied=False,
+ proposal_disposition=proposal_disposition if isinstance(proposal_disposition, str) else None,
+ run_outcomes=_terminal_envelope_run_outcomes(ctx),
+ blocker_reason=blocker_reason,
+ halt_kind=halt_kind,
+ attempted=_attempted_summary(kwargs.get("narrative_summary"), kwargs.get("narrative_payload")),
+ workflow_mutated=bool(kwargs.get("workflow_was_persisted")) or kwargs.get("updated_workflow") is not None,
+ turn_outcome_response_kind=turn_outcome_response_kind,
+ final_message=str(kwargs.get("user_response") or ""),
+ )
+ kwargs["terminal_envelope"] = terminal_envelope
result = AgentResult(global_llm_context=final_context, turn_outcome=turn_outcome, **kwargs)
if ctx is not None and result.turn_outcome is not None:
result.turn_outcome = with_copilot_code_mode_diagnostics(result.turn_outcome, ctx)
diff --git a/skyvern/forge/sdk/copilot/blocker_signal.py b/skyvern/forge/sdk/copilot/blocker_signal.py
index f1a4f0b07..530bf96c9 100644
--- a/skyvern/forge/sdk/copilot/blocker_signal.py
+++ b/skyvern/forge/sdk/copilot/blocker_signal.py
@@ -354,6 +354,7 @@ class _TerminalEvidenceResetCtx(Protocol):
last_run_blocks_block_labels: list[str]
last_run_outcome: RecordedRunOutcome | None
last_run_outcome_block_labels: list[str]
+ terminal_envelope_run_outcomes: list[RecordedRunOutcome]
last_outcome_gate_reason: str | None
last_outcome_gate_workflow_run_id: str | None
last_test_anti_bot: str | None
@@ -390,6 +391,7 @@ def clear_terminal_evidence_on_workflow_edit(ctx: _TerminalEvidenceResetCtx) ->
ctx.last_run_blocks_block_labels = []
ctx.last_run_outcome = None
ctx.last_run_outcome_block_labels = []
+ ctx.terminal_envelope_run_outcomes = []
ctx.last_outcome_gate_reason = None
ctx.last_outcome_gate_workflow_run_id = None
ctx.last_test_anti_bot = None
diff --git a/skyvern/forge/sdk/copilot/context.py b/skyvern/forge/sdk/copilot/context.py
index 8b0d06baa..d595dffbd 100644
--- a/skyvern/forge/sdk/copilot/context.py
+++ b/skyvern/forge/sdk/copilot/context.py
@@ -90,6 +90,7 @@ class TurnNarrativePayload(TypedDict):
proposalDisposition: NotRequired[ProposalDisposition]
# TurnOutcome.response_kind value: "build" | "clarify" | "diagnose" | "refuse" | "recover".
responseKind: NotRequired[str]
+ terminalEnvelope: NotRequired[dict[str, Any]]
# The ADR-0005 terminal adjudication (enforcement.verified_goal_claim_authorized):
# True only when outcome evidence authorizes a tested-success claim.
verifiedSuccess: NotRequired[bool]
@@ -128,6 +129,7 @@ if TYPE_CHECKING:
from skyvern.forge.sdk.copilot.narration import NarratorState
from skyvern.forge.sdk.copilot.output_extraction_plan import FrozenRequestedOutputExtractionCandidate
from skyvern.forge.sdk.copilot.request_policy import RequestPolicy
+ from skyvern.forge.sdk.copilot.run_outcome import RecordedRunOutcome
from skyvern.forge.sdk.copilot.schema_incompatibility import SchemaIncompatibility
from skyvern.forge.sdk.copilot.turn_context import TurnContextPacket
from skyvern.forge.sdk.copilot.turn_halt import TurnHalt
@@ -669,6 +671,8 @@ class AgentResult:
narrative_summary: str | None = None
# Persisted on the assistant chat message so the bubble survives a reload.
narrative_payload: TurnNarrativePayload | None = None
+ # Shadow-only typed terminal-state envelope persisted and streamed on terminal frames.
+ terminal_envelope: dict[str, Any] | None = None
staged_workflow_yaml: str | None = None
staged_workflow: Workflow | None = None
has_staged_proposal: bool = False
@@ -801,6 +805,11 @@ class CopilotContext(AgentContext):
last_run_blocks_workflow_run_id: str | None = None
last_successful_run_blocks_workflow_run_id: str | None = None
last_outcome_gate_workflow_run_id: str | None = None
+ # In-turn run-outcome trace derived from assignments to ``last_run_outcome``
+ # (the same source that powers run_outcome SSE frames). Append-only across
+ # per-run pointer resets (``last_run_outcome = None``); cleared only by the
+ # workflow-edit evidence reset, which invalidates pre-edit run evidence.
+ terminal_envelope_run_outcomes: list[RecordedRunOutcome] = field(default_factory=list)
delivered_unverified_terminal: bool = False
delivered_unverified_workflow_run_id: str | None = None
delivered_unverified_observed_outputs: dict[str, Any] = field(default_factory=DeliveredUnverifiedPublicOutputs)
@@ -959,6 +968,32 @@ class CopilotContext(AgentContext):
turn_started_at: str | None = None
turn_ended_at: str | None = None
+ def __post_init__(self) -> None:
+ parent_post_init = getattr(super(), "__post_init__", None)
+ if callable(parent_post_init):
+ parent_post_init()
+ from skyvern.forge.sdk.copilot.run_outcome import RecordedRunOutcome
+
+ if isinstance(self.last_run_outcome, RecordedRunOutcome):
+ super().__setattr__("terminal_envelope_run_outcomes", [self.last_run_outcome])
+
+ def __setattr__(self, name: str, value: Any) -> None:
+ super().__setattr__(name, value)
+ if name != "last_run_outcome":
+ return
+ from skyvern.forge.sdk.copilot.run_outcome import RecordedRunOutcome
+
+ if not isinstance(value, RecordedRunOutcome):
+ # ``last_run_outcome = None`` is a per-run pointer reset, not an
+ # evidence reset — the trace survives so a later run in the same
+ # turn cannot mask an earlier honest not_demonstrated.
+ return
+ outcomes = getattr(self, "terminal_envelope_run_outcomes", None)
+ if isinstance(outcomes, list):
+ outcomes.append(value)
+ else:
+ super().__setattr__("terminal_envelope_run_outcomes", [value])
+
def has_genuine_workflow_attempt(self) -> bool:
"""This turn persisted a workflow proposal or executed a real build-test run; excludes
``test_after_update_done``, which is stamped for any ``run_blocks_and_collect_debug`` scout
diff --git a/skyvern/forge/sdk/copilot/terminal_envelope.py b/skyvern/forge/sdk/copilot/terminal_envelope.py
new file mode 100644
index 000000000..7947785f6
--- /dev/null
+++ b/skyvern/forge/sdk/copilot/terminal_envelope.py
@@ -0,0 +1,189 @@
+from __future__ import annotations
+
+from collections.abc import Sequence
+from typing import Literal
+
+import structlog
+from pydantic import BaseModel
+
+from skyvern.forge.sdk.copilot.run_outcome import RecordedRunOutcome
+
+LOG = structlog.get_logger(__name__)
+
+TerminalNextState = Literal["completed", "proposal_pending", "awaiting_user_input", "stopped"]
+TerminalResponseKind = Literal["question", "update", "answer", "stopped"]
+_FINAL_RUN_VERDICTS = frozenset({"demonstrated", "not_demonstrated", "not_evaluated"})
+_REVIEW_PROPOSAL_DISPOSITIONS = frozenset({"review_untested", "review_tested"})
+_SHADOW_REASON_TRAILING_PUNCTUATION = ".,;:!?"
+
+
+class TerminalOutcomeEnvelope(BaseModel):
+ next_state: TerminalNextState
+ verified: bool
+ workflow_applied: bool = False
+ run_verdict: str | None = None
+ run_display_reason: str | None = None
+ blocker_reason: str | None = None
+ halt_kind: str | None = None
+ user_action_required: bool = False
+ attempted: str | None = None
+ response_kind: TerminalResponseKind
+ envelope_version: int = 1
+
+
+def assemble_terminal_envelope(
+ *,
+ response_type: str,
+ verified: bool,
+ workflow_applied: bool,
+ proposal_disposition: str | None,
+ run_outcomes: Sequence[RecordedRunOutcome],
+ blocker_reason: str | None,
+ halt_kind: str | None,
+ attempted: str | None,
+ workflow_mutated: bool,
+ turn_outcome_response_kind: str | None,
+) -> TerminalOutcomeEnvelope | None:
+ run_outcome = _select_run_outcome_anchor(run_outcomes)
+ superseding_outcome = _later_demonstrated_after_anchor(run_outcomes, run_outcome)
+ if run_outcome is not None and superseding_outcome is not None:
+ LOG.info(
+ "Terminal envelope anchored a not_demonstrated verdict past a later demonstrated run",
+ anchored_workflow_run_id=run_outcome.workflow_run_id,
+ anchored_reason_code=run_outcome.reason_code,
+ later_workflow_run_id=superseding_outcome.workflow_run_id,
+ )
+ run_verdict = run_outcome.verdict if run_outcome is not None else None
+ run_display_reason = _clean_text(run_outcome.display_reason) if run_outcome is not None else None
+ user_action_required = response_type == "ASK_QUESTION"
+ next_state = _derive_next_state(
+ user_action_required=user_action_required,
+ verified=verified,
+ workflow_applied=workflow_applied,
+ proposal_disposition=proposal_disposition,
+ )
+ response_kind = _derive_response_kind(
+ user_action_required=user_action_required,
+ next_state=next_state,
+ workflow_mutated=workflow_mutated,
+ turn_outcome_response_kind=turn_outcome_response_kind,
+ )
+ return TerminalOutcomeEnvelope(
+ next_state=next_state,
+ verified=verified,
+ workflow_applied=workflow_applied,
+ run_verdict=run_verdict,
+ run_display_reason=run_display_reason,
+ blocker_reason=_clean_text(blocker_reason),
+ halt_kind=_clean_text(halt_kind),
+ user_action_required=user_action_required,
+ attempted=_clean_text(attempted),
+ response_kind=response_kind,
+ )
+
+
+def finalize_applied_state(envelope: TerminalOutcomeEnvelope, *, applied: bool) -> TerminalOutcomeEnvelope:
+ if envelope.user_action_required:
+ next_state: TerminalNextState = "awaiting_user_input"
+ elif envelope.verified and applied:
+ next_state = "completed"
+ elif envelope.next_state == "proposal_pending":
+ next_state = "proposal_pending"
+ else:
+ next_state = "stopped"
+ response_kind = _derive_response_kind(
+ user_action_required=envelope.user_action_required,
+ next_state=next_state,
+ prior_response_kind=envelope.response_kind,
+ )
+ return envelope.model_copy(
+ update={"workflow_applied": applied, "next_state": next_state, "response_kind": response_kind}
+ )
+
+
+def _select_run_outcome_anchor(run_outcomes: Sequence[RecordedRunOutcome]) -> RecordedRunOutcome | None:
+ final_outcomes = [outcome for outcome in run_outcomes if outcome.verdict in _FINAL_RUN_VERDICTS]
+ if not final_outcomes:
+ return None
+ last_not_demonstrated = [outcome for outcome in final_outcomes if outcome.verdict == "not_demonstrated"]
+ if last_not_demonstrated:
+ return last_not_demonstrated[-1]
+ return final_outcomes[-1]
+
+
+def _later_demonstrated_after_anchor(
+ run_outcomes: Sequence[RecordedRunOutcome], anchor: RecordedRunOutcome | None
+) -> RecordedRunOutcome | None:
+ if anchor is None or anchor.verdict != "not_demonstrated":
+ return None
+ seen_anchor = False
+ for outcome in run_outcomes:
+ if outcome is anchor:
+ seen_anchor = True
+ continue
+ if seen_anchor and outcome.verdict == "demonstrated":
+ return outcome
+ return None
+
+
+def _derive_next_state(
+ *,
+ user_action_required: bool,
+ verified: bool,
+ workflow_applied: bool,
+ proposal_disposition: str | None,
+) -> TerminalNextState:
+ if user_action_required:
+ return "awaiting_user_input"
+ if verified and workflow_applied:
+ return "completed"
+ if _proposal_requires_review(proposal_disposition):
+ return "proposal_pending"
+ return "stopped"
+
+
+def _derive_response_kind(
+ *,
+ user_action_required: bool,
+ next_state: TerminalNextState,
+ workflow_mutated: bool | None = None,
+ turn_outcome_response_kind: str | None = None,
+ prior_response_kind: TerminalResponseKind | None = None,
+) -> TerminalResponseKind:
+ if user_action_required:
+ return "question"
+ if next_state in {"completed", "proposal_pending"}:
+ return "update"
+ if prior_response_kind == "answer":
+ return "answer"
+ if workflow_mutated is False and turn_outcome_response_kind == "diagnose":
+ return "answer"
+ return "stopped"
+
+
+def normalize_shadow_reason_text(text: object, *, strip_trailing_punctuation: bool = False) -> str | None:
+ if not isinstance(text, str):
+ return None
+ normalized = " ".join(text.lower().split())
+ if strip_trailing_punctuation:
+ normalized = normalized.rstrip(_SHADOW_REASON_TRAILING_PUNCTUATION).strip()
+ return normalized or None
+
+
+def reason_in_reply_shadow(run_display_reason: str | None, final_message: str) -> bool:
+ normalized_reason = normalize_shadow_reason_text(run_display_reason, strip_trailing_punctuation=True)
+ normalized_reply = normalize_shadow_reason_text(final_message)
+ return bool(normalized_reason and normalized_reply and normalized_reason in normalized_reply)
+
+
+def _proposal_requires_review(proposal_disposition: str | None) -> bool:
+ if not isinstance(proposal_disposition, str):
+ return False
+ return proposal_disposition.strip().lower() in _REVIEW_PROPOSAL_DISPOSITIONS
+
+
+def _clean_text(value: str | None) -> str | None:
+ if not isinstance(value, str):
+ return None
+ text = value.strip()
+ return text or None
diff --git a/skyvern/forge/sdk/routes/workflow_copilot.py b/skyvern/forge/sdk/routes/workflow_copilot.py
index 6ed907ecc..ed04dd601 100644
--- a/skyvern/forge/sdk/routes/workflow_copilot.py
+++ b/skyvern/forge/sdk/routes/workflow_copilot.py
@@ -58,6 +58,11 @@ from skyvern.forge.sdk.copilot.recoverable_failure import (
merge_failure_into_context,
)
from skyvern.forge.sdk.copilot.request_policy import is_defer_authoring_durable_fill_criterion
+from skyvern.forge.sdk.copilot.terminal_envelope import (
+ TerminalOutcomeEnvelope,
+ finalize_applied_state,
+ reason_in_reply_shadow,
+)
from skyvern.forge.sdk.copilot.turn_outcome import (
CopilotComposerMode,
build_minimal_turn_outcome,
@@ -462,14 +467,69 @@ def _with_terminal_narrative_metadata(
*,
cancelled: bool,
proposal_disposition: ProposalDisposition,
+ terminal_envelope: dict[str, Any] | None = None,
) -> TurnNarrativePayload | None:
if narrative_payload is None:
return None
- return {
+ payload: TurnNarrativePayload = {
**narrative_payload,
"cancelled": cancelled,
"proposalDisposition": proposal_disposition,
}
+ if terminal_envelope is not None:
+ payload["terminalEnvelope"] = terminal_envelope
+ return payload
+
+
+def _agent_terminal_envelope(agent_result: AgentResult | None) -> dict[str, Any] | None:
+ if agent_result is None:
+ return None
+ try:
+ payload = agent_result.terminal_envelope
+ except AttributeError:
+ return None
+ return payload if isinstance(payload, dict) else None
+
+
+def _finalized_terminal_envelope(
+ agent_result: AgentResult | None,
+ *,
+ workflow_applied: bool,
+ final_message: str,
+ response_type: str,
+) -> dict[str, Any] | None:
+ payload = _agent_terminal_envelope(agent_result)
+ if payload is None:
+ return None
+ # workflow_applied mirrors the legacy auto-accept SSE field, which reads
+ # true on no-proposal turns because AgentResult defaults its disposition
+ # to auto_applicable. The envelope must only claim applied when a
+ # proposal actually existed to commit.
+ proposal_present = getattr(agent_result, "updated_workflow", None) is not None or bool(
+ getattr(agent_result, "has_staged_proposal", False)
+ )
+ try:
+ finalized = finalize_applied_state(
+ TerminalOutcomeEnvelope.model_validate(payload),
+ applied=workflow_applied and proposal_present,
+ )
+ payload = finalized.model_dump(mode="json")
+ except Exception:
+ LOG.warning("copilot terminal envelope finalization failed", exc_info=True)
+ return None
+ reason_in_reply = reason_in_reply_shadow(
+ payload.get("run_display_reason") if isinstance(payload.get("run_display_reason"), str) else None,
+ final_message,
+ )
+ LOG.info(
+ "copilot_terminal_envelope",
+ **payload,
+ response_type=response_type,
+ envelope_response_kind=payload.get("response_kind"),
+ reason_in_reply=reason_in_reply,
+ finalized=True,
+ )
+ return payload
def _build_recoverable_route_agent_result(
@@ -698,6 +758,7 @@ async def _persist_cancel_turn(
the success path; pass ``None`` for pre-agent cancels.
"""
turn_outcome: TurnOutcome | None
+ workflow_applied = False
if agent_result is None:
user_response = "Cancelled by user."
updated_workflow = None
@@ -709,6 +770,7 @@ async def _persist_cancel_turn(
response_turn_id = turn_id
narrative_summary = None
narrative_payload = None
+ terminal_envelope = None
if chat.proposed_workflow is not None and not keep_pending_proposal:
await asyncio.shield(_clear_proposed_workflow(chat))
else:
@@ -751,12 +813,20 @@ async def _persist_cancel_turn(
response_turn_id = turn_id or agent_result.turn_id
narrative_summary = agent_result.narrative_summary
narrative_payload = agent_result.narrative_payload
+ workflow_applied = _effective_auto_accept(chat.auto_accept, agent_result)
+ terminal_envelope = _finalized_terminal_envelope(
+ agent_result,
+ workflow_applied=workflow_applied,
+ final_message=user_response,
+ response_type=response_type,
+ )
proposal_disposition = _proposal_disposition(agent_result)
narrative_payload = _with_terminal_narrative_metadata(
narrative_payload,
cancelled=True,
proposal_disposition=proposal_disposition,
+ terminal_envelope=terminal_envelope,
)
await asyncio.shield(
@@ -791,11 +861,13 @@ async def _persist_cancel_turn(
total_tokens=total_tokens,
response_type=response_type,
proposal_disposition=proposal_disposition,
+ workflow_applied=workflow_applied,
cancelled=True,
output_policy_diagnostics=output_policy_diagnostics,
turn_id=response_turn_id,
narrative_summary=narrative_summary,
narrative_payload=narrative_payload,
+ terminal_envelope=terminal_envelope,
)
)
)
@@ -872,10 +944,17 @@ async def _finalise_normal_turn(
await _persist_completion_criteria_state(chat, agent_result, chat_request.message)
proposal_disposition = _proposal_disposition(agent_result)
workflow_applied = _effective_auto_accept(chat.auto_accept, agent_result)
+ terminal_envelope = _finalized_terminal_envelope(
+ agent_result,
+ workflow_applied=workflow_applied,
+ final_message=user_response,
+ response_type=agent_result.response_type,
+ )
narrative_payload = _with_terminal_narrative_metadata(
agent_result.narrative_payload,
cancelled=False,
proposal_disposition=proposal_disposition,
+ terminal_envelope=terminal_envelope,
)
await app.DATABASE.workflow_params.create_workflow_copilot_chat_message(
@@ -911,6 +990,7 @@ async def _finalise_normal_turn(
turn_id=agent_result.turn_id,
narrative_summary=agent_result.narrative_summary,
narrative_payload=narrative_payload,
+ terminal_envelope=terminal_envelope,
)
)
diff --git a/skyvern/forge/sdk/schemas/workflow_copilot.py b/skyvern/forge/sdk/schemas/workflow_copilot.py
index 5f1b79881..68c05f188 100644
--- a/skyvern/forge/sdk/schemas/workflow_copilot.py
+++ b/skyvern/forge/sdk/schemas/workflow_copilot.py
@@ -282,6 +282,10 @@ class WorkflowCopilotStreamResponseUpdate(BaseModel):
None,
description="Terminal narrative bubble snapshot for live clients; mirrors the persisted assistant chat row.",
)
+ terminal_envelope: dict[str, Any] | None = Field(
+ None,
+ description="Shadow-only terminal outcome envelope for end-of-turn typed-state validation.",
+ )
class WorkflowCopilotStreamErrorUpdate(BaseModel):
diff --git a/tests/unit/test_copilot_terminal_envelope.py b/tests/unit/test_copilot_terminal_envelope.py
new file mode 100644
index 000000000..1c5529b7d
--- /dev/null
+++ b/tests/unit/test_copilot_terminal_envelope.py
@@ -0,0 +1,332 @@
+from __future__ import annotations
+
+from typing import Any
+
+import pytest
+import structlog.testing
+
+from skyvern.forge.sdk.copilot import agent as agent_module
+from skyvern.forge.sdk.copilot.blocker_signal import clear_terminal_evidence_on_workflow_edit
+from skyvern.forge.sdk.copilot.run_outcome import RecordedRunOutcome
+from skyvern.forge.sdk.copilot.terminal_envelope import (
+ assemble_terminal_envelope,
+ finalize_applied_state,
+ reason_in_reply_shadow,
+)
+from skyvern.forge.sdk.copilot.tools.run_execution import _stash_recorded_run_outcome
+from tests.unit.copilot_test_helpers import make_copilot_ctx
+
+
+def _run_outcome(verdict: str, display_reason: str | None = None) -> RecordedRunOutcome:
+ return RecordedRunOutcome(verdict=verdict, display_reason=display_reason)
+
+
+def _assemble(**overrides: Any):
+ defaults = {
+ "response_type": "REPLY",
+ "verified": False,
+ "workflow_applied": False,
+ "proposal_disposition": "no_proposal",
+ "run_outcomes": [],
+ "blocker_reason": None,
+ "halt_kind": None,
+ "attempted": None,
+ "workflow_mutated": False,
+ "turn_outcome_response_kind": None,
+ }
+ defaults.update(overrides)
+ envelope = assemble_terminal_envelope(**defaults)
+ assert envelope is not None
+ return envelope
+
+
+def test_run_anchor_prefers_last_not_demonstrated_even_if_later_run_demonstrated() -> None:
+ envelope = _assemble(
+ run_outcomes=[
+ _run_outcome("not_demonstrated", "The checkout did not reach confirmation."),
+ _run_outcome("demonstrated", "A later scout run succeeded."),
+ ]
+ )
+
+ assert envelope.run_verdict == "not_demonstrated"
+ assert envelope.run_display_reason == "The checkout did not reach confirmation."
+
+
+def test_run_anchor_falls_back_to_latest_final_verdict_when_no_not_demonstrated() -> None:
+ envelope = _assemble(
+ run_outcomes=[
+ _run_outcome("not_evaluated", "Could not evaluate."),
+ _run_outcome("demonstrated", "Confirmed."),
+ ]
+ )
+
+ assert envelope.run_verdict == "demonstrated"
+ assert envelope.run_display_reason == "Confirmed."
+
+
+def test_run_anchor_empty_when_no_recorded_outcomes() -> None:
+ envelope = _assemble(run_outcomes=[])
+ assert envelope.run_verdict is None
+ assert envelope.run_display_reason is None
+
+
+def test_unknown_halt_kind_degrades_to_stopped_never_question() -> None:
+ envelope = _assemble(
+ halt_kind="capture_obligation_reopen",
+ blocker_reason="Capture obligation reopened mid-turn.",
+ )
+
+ assert envelope.response_kind == "stopped"
+ assert envelope.next_state == "stopped"
+ assert envelope.halt_kind == "capture_obligation_reopen"
+
+ finalized = finalize_applied_state(envelope, applied=False)
+ assert finalized.response_kind == "stopped"
+ assert finalized.next_state == "stopped"
+
+
+def test_anchor_supersession_divergence_is_logged() -> None:
+ with structlog.testing.capture_logs() as logs:
+ _assemble(
+ run_outcomes=[
+ _run_outcome("not_demonstrated", "The checkout did not reach confirmation."),
+ _run_outcome("demonstrated", "A later scout run succeeded."),
+ ]
+ )
+ assert any("anchored a not_demonstrated verdict" in log["event"] for log in logs)
+
+ with structlog.testing.capture_logs() as logs:
+ _assemble(run_outcomes=[_run_outcome("not_demonstrated", "No later run.")])
+ assert not any("anchored a not_demonstrated verdict" in log["event"] for log in logs)
+
+
+ ("response_type", "verified", "workflow_applied", "proposal_disposition", "expected_next_state"),
+ [
+ ("ASK_QUESTION", False, False, "no_proposal", "awaiting_user_input"),
+ ("REPLY", True, True, "no_proposal", "completed"),
+ ("REPLY", False, False, "review_tested", "proposal_pending"),
+ ("REPLY", False, False, "review_required", "stopped"),
+ ("REPLY", True, False, "auto_applicable", "stopped"),
+ ],
+)
+def test_next_state_derivation(
+ response_type: str,
+ verified: bool,
+ workflow_applied: bool,
+ proposal_disposition: str,
+ expected_next_state: str,
+) -> None:
+ envelope = _assemble(
+ response_type=response_type,
+ verified=verified,
+ workflow_applied=workflow_applied,
+ proposal_disposition=proposal_disposition,
+ )
+ assert envelope.next_state == expected_next_state
+
+
+ ("kwargs", "expected_response_kind"),
+ [
+ ({"response_type": "ASK_QUESTION"}, "question"),
+ ({"verified": True, "workflow_applied": True}, "update"),
+ ({"proposal_disposition": "review_untested"}, "update"),
+ ({"turn_outcome_response_kind": "diagnose", "workflow_mutated": False}, "answer"),
+ ({"turn_outcome_response_kind": "diagnose", "workflow_mutated": True}, "stopped"),
+ ({"turn_outcome_response_kind": "build", "workflow_mutated": False}, "stopped"),
+ ],
+)
+def test_response_kind_derivation(kwargs: dict[str, Any], expected_response_kind: str) -> None:
+ envelope = _assemble(**kwargs)
+ assert envelope.response_kind == expected_response_kind
+
+
+def test_user_action_required_derivation() -> None:
+ assert _assemble(response_type="ASK_QUESTION").user_action_required is True
+ assert _assemble(response_type="REPLY").user_action_required is False
+
+
+def test_blocker_fields_attempted_and_envelope_version() -> None:
+ envelope = _assemble(
+ blocker_reason=" Need account credentials. ",
+ halt_kind=" loop_detected ",
+ attempted=" Attempted full checkout run. ",
+ )
+
+ assert envelope.blocker_reason == "Need account credentials."
+ assert envelope.halt_kind == "loop_detected"
+ assert envelope.attempted == "Attempted full checkout run."
+ assert envelope.envelope_version == 1
+
+
+def test_finalize_applied_state_promotes_completed_when_verified_and_applied() -> None:
+ envelope = _assemble(verified=True, workflow_applied=False, proposal_disposition="no_proposal")
+
+ finalized = finalize_applied_state(envelope, applied=True)
+
+ assert finalized.workflow_applied is True
+ assert finalized.next_state == "completed"
+ assert finalized.response_kind == "update"
+
+
+def test_finalize_applied_state_blocks_completed_when_not_applied() -> None:
+ envelope = _assemble(verified=True, workflow_applied=True, proposal_disposition="no_proposal")
+
+ finalized = finalize_applied_state(envelope, applied=False)
+
+ assert finalized.workflow_applied is False
+ assert finalized.next_state == "stopped"
+ assert finalized.response_kind == "stopped"
+
+
+def test_finalize_applied_state_keeps_question_for_user_action_required() -> None:
+ envelope = _assemble(
+ response_type="ASK_QUESTION", verified=True, workflow_applied=False, proposal_disposition="no_proposal"
+ )
+
+ finalized = finalize_applied_state(envelope, applied=True)
+
+ assert finalized.workflow_applied is True
+ assert finalized.next_state == "awaiting_user_input"
+ assert finalized.response_kind == "question"
+
+
+def test_finalize_applied_state_preserves_answer_when_not_promoted_to_update() -> None:
+ envelope = _assemble(turn_outcome_response_kind="diagnose", workflow_mutated=False)
+ assert envelope.response_kind == "answer"
+
+ finalized = finalize_applied_state(envelope, applied=False)
+
+ assert finalized.next_state == "stopped"
+ assert finalized.response_kind == "answer"
+
+
+def test_terminal_envelope_outcomes_survive_per_run_pointer_reset() -> None:
+ ctx = make_copilot_ctx()
+ first = RecordedRunOutcome(
+ verdict="not_demonstrated",
+ display_reason="Checkout never reached confirmation.",
+ workflow_run_id="wr_first",
+ )
+ second = RecordedRunOutcome(
+ verdict="demonstrated",
+ display_reason="A later scout replay succeeded.",
+ workflow_run_id="wr_second",
+ )
+
+ _stash_recorded_run_outcome(ctx, first)
+ # _record_run_blocks_result resets the pointer before processing each new
+ # run in the turn; the trace must survive it or the anchor never sees the
+ # earlier failure.
+ ctx.last_run_outcome = None
+ ctx.last_run_outcome_block_labels = []
+ _stash_recorded_run_outcome(ctx, second)
+ outcomes = agent_module._terminal_envelope_run_outcomes(ctx)
+
+ assert [outcome.verdict for outcome in outcomes] == ["not_demonstrated", "demonstrated"]
+ assert outcomes[0].display_reason == "Checkout never reached confirmation."
+ assert outcomes[1].display_reason == "A later scout replay succeeded."
+
+ envelope = _assemble(run_outcomes=outcomes)
+
+ assert envelope.run_verdict == "not_demonstrated"
+ assert envelope.run_display_reason == "Checkout never reached confirmation."
+
+
+def test_terminal_envelope_outcomes_seed_from_constructor_last_run_outcome() -> None:
+ first = RecordedRunOutcome(
+ verdict="not_demonstrated",
+ display_reason="Seeded from constructor.",
+ workflow_run_id="wr_ctor",
+ )
+ second = RecordedRunOutcome(
+ verdict="demonstrated",
+ display_reason="Appended after construction.",
+ workflow_run_id="wr_runtime",
+ )
+ ctx = make_copilot_ctx(last_run_outcome=first)
+
+ assert ctx.terminal_envelope_run_outcomes == [first]
+
+ ctx.last_run_outcome = second
+
+ assert ctx.terminal_envelope_run_outcomes == [first, second]
+
+
+def test_terminal_envelope_outcomes_clear_on_workflow_edit_evidence_reset() -> None:
+ ctx = make_copilot_ctx()
+ _stash_recorded_run_outcome(
+ ctx,
+ RecordedRunOutcome(
+ verdict="not_demonstrated",
+ display_reason="Checkout never reached confirmation.",
+ workflow_run_id="wr_before_reset",
+ ),
+ )
+
+ clear_terminal_evidence_on_workflow_edit(ctx)
+ outcomes = agent_module._terminal_envelope_run_outcomes(ctx)
+ envelope = _assemble(run_outcomes=outcomes)
+
+ assert ctx.terminal_envelope_run_outcomes == []
+ assert outcomes == []
+ assert envelope.run_verdict is None
+
+
+def test_terminal_envelope_outcomes_reanchor_to_new_outcome_after_workflow_edit() -> None:
+ ctx = make_copilot_ctx()
+ _stash_recorded_run_outcome(
+ ctx,
+ RecordedRunOutcome(
+ verdict="not_demonstrated",
+ display_reason="Old failed run.",
+ workflow_run_id="wr_old",
+ ),
+ )
+ clear_terminal_evidence_on_workflow_edit(ctx)
+
+ _stash_recorded_run_outcome(
+ ctx,
+ RecordedRunOutcome(
+ verdict="not_demonstrated",
+ display_reason="New failed run after edit.",
+ workflow_run_id="wr_new",
+ ),
+ )
+ outcomes = agent_module._terminal_envelope_run_outcomes(ctx)
+ envelope = _assemble(run_outcomes=outcomes)
+
+ assert len(outcomes) == 1
+ assert outcomes[0].workflow_run_id == "wr_new"
+ assert envelope.run_verdict == "not_demonstrated"
+ assert envelope.run_display_reason == "New failed run after edit."
+
+
+def test_reason_in_reply_shadow_normalization() -> None:
+ assert reason_in_reply_shadow(
+ "Run completed but did not demonstrate the requested outcome.",
+ "The latest run completed but did not demonstrate the requested outcome, so I paused.",
+ )
+
+
+def test_safe_wrapper_returns_none_when_assembly_raises(monkeypatch: pytest.MonkeyPatch) -> None:
+ def _raise(**_: Any) -> Any:
+ raise RuntimeError("boom")
+
+ monkeypatch.setattr(agent_module, "assemble_terminal_envelope", _raise)
+ envelope = agent_module._assemble_terminal_envelope_safe(
+ response_type="REPLY",
+ verified=False,
+ workflow_applied=False,
+ proposal_disposition="no_proposal",
+ run_outcomes=[],
+ blocker_reason=None,
+ halt_kind=None,
+ attempted=None,
+ workflow_mutated=False,
+ turn_outcome_response_kind=None,
+ final_message="reply",
+ )
+
+ assert envelope is None
diff --git a/tests/unit/test_workflow_copilot_route.py b/tests/unit/test_workflow_copilot_route.py
index fa290fd80..1deb685b6 100644
--- a/tests/unit/test_workflow_copilot_route.py
+++ b/tests/unit/test_workflow_copilot_route.py
@@ -20,6 +20,7 @@ import json
from datetime import datetime, timezone
from io import BytesIO
from types import SimpleNamespace
+from typing import Any
from unittest.mock import AsyncMock, MagicMock
import pytest
@@ -31,11 +32,12 @@ from skyvern.forge import app
from skyvern.forge.sdk.api.llm.exceptions import LLMProviderError
from skyvern.forge.sdk.artifact.models import ArtifactType
from skyvern.forge.sdk.copilot import agent as agent_module
-from skyvern.forge.sdk.copilot.context import DeliveredUnverifiedPublicOutputs
+from skyvern.forge.sdk.copilot.context import AgentResult, DeliveredUnverifiedPublicOutputs
from skyvern.forge.sdk.copilot.schema_incompatibility import (
SchemaIncompatibility,
render_schema_incompatibility_user_reason,
)
+from skyvern.forge.sdk.copilot.terminal_envelope import assemble_terminal_envelope
from skyvern.forge.sdk.copilot.turn_halt import TurnHaltKind
from skyvern.forge.sdk.copilot.turn_outcome import (
build_minimal_turn_outcome,
@@ -77,6 +79,304 @@ def _make_chat_request(
)
+def _terminal_payload(*, verified: bool, workflow_applied: bool) -> dict[str, Any]:
+ envelope = assemble_terminal_envelope(
+ response_type="REPLY",
+ verified=verified,
+ workflow_applied=workflow_applied,
+ proposal_disposition="no_proposal",
+ run_outcomes=[],
+ blocker_reason=None,
+ halt_kind=None,
+ attempted="Attempted full run.",
+ workflow_mutated=True,
+ turn_outcome_response_kind="build",
+ )
+ assert envelope is not None
+ return envelope.model_dump(mode="json")
+
+
+def _narrative_payload() -> dict[str, Any]:
+ return {
+ "turnId": "turn-1",
+ "turnIndex": 0,
+ "mode": "build",
+ "designStarted": True,
+ "designEnded": True,
+ "draft": None,
+ "blocks": [],
+ "terminal": "response",
+ "terminalMessage": "done",
+ "narrativeSummary": "done",
+ "priorBlockCount": None,
+ "designActivity": [],
+ "startedAt": None,
+ "endedAt": None,
+ }
+
+
+async def test_finalise_normal_turn_finalizes_terminal_envelope_without_auto_accept(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ chat = SimpleNamespace(
+ organization_id="org-1",
+ workflow_copilot_chat_id="chat-1",
+ proposed_workflow=None,
+ auto_accept=False,
+ )
+ original_workflow = SimpleNamespace(workflow_id="wf-canonical")
+ updated_workflow = MagicMock()
+ updated_workflow.model_dump.return_value = {"workflow_id": "wf-draft"}
+ agent_result = AgentResult(
+ user_response="done",
+ updated_workflow=updated_workflow,
+ global_llm_context=None,
+ response_type="REPLY",
+ proposal_disposition="auto_applicable",
+ narrative_payload=_narrative_payload(),
+ terminal_envelope=_terminal_payload(verified=True, workflow_applied=False),
+ )
+ _, workflow_params = setup_new_copilot_mocks(monkeypatch, chat, original_workflow, agent_result)
+ stream = MagicMock(send=AsyncMock(return_value=True))
+
+ await workflow_copilot_route._finalise_normal_turn(
+ stream=stream,
+ chat=chat,
+ organization_id="org-1",
+ original_workflow=original_workflow,
+ chat_request=_make_chat_request(),
+ agent_result=agent_result,
+ )
+
+ response_frame = stream.send.await_args.args[0]
+ assert isinstance(response_frame, WorkflowCopilotStreamResponseUpdate)
+ assert response_frame.terminal_envelope is not None
+ assert response_frame.terminal_envelope["workflow_applied"] is False
+ assert response_frame.terminal_envelope["next_state"] != "completed"
+ assert response_frame.terminal_envelope["response_kind"] == "stopped"
+
+ persisted_payload = workflow_params.create_workflow_copilot_chat_message.await_args_list[-1].kwargs[
+ "narrative_payload"
+ ]
+ assert persisted_payload is not None
+ assert persisted_payload["terminalEnvelope"]["workflow_applied"] is False
+ assert persisted_payload["terminalEnvelope"]["next_state"] != "completed"
+ assert persisted_payload["terminalEnvelope"]["response_kind"] == "stopped"
+
+
+async def test_finalise_normal_turn_finalizes_terminal_envelope_with_auto_accept(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ chat = SimpleNamespace(
+ organization_id="org-1",
+ workflow_copilot_chat_id="chat-1",
+ proposed_workflow=None,
+ auto_accept=True,
+ )
+ original_workflow = SimpleNamespace(workflow_id="wf-canonical")
+ updated_workflow = MagicMock()
+ updated_workflow.model_dump.return_value = {"workflow_id": "wf-draft"}
+ agent_result = AgentResult(
+ user_response="done",
+ updated_workflow=updated_workflow,
+ global_llm_context=None,
+ response_type="REPLY",
+ proposal_disposition="auto_applicable",
+ narrative_payload=_narrative_payload(),
+ terminal_envelope=_terminal_payload(verified=True, workflow_applied=False),
+ )
+ _, workflow_params = setup_new_copilot_mocks(monkeypatch, chat, original_workflow, agent_result)
+ stream = MagicMock(send=AsyncMock(return_value=True))
+
+ await workflow_copilot_route._finalise_normal_turn(
+ stream=stream,
+ chat=chat,
+ organization_id="org-1",
+ original_workflow=original_workflow,
+ chat_request=_make_chat_request(),
+ agent_result=agent_result,
+ )
+
+ response_frame = stream.send.await_args.args[0]
+ assert isinstance(response_frame, WorkflowCopilotStreamResponseUpdate)
+ assert response_frame.terminal_envelope is not None
+ assert response_frame.terminal_envelope["workflow_applied"] is True
+ assert response_frame.terminal_envelope["next_state"] == "completed"
+ assert response_frame.terminal_envelope["response_kind"] == "update"
+
+ persisted_payload = workflow_params.create_workflow_copilot_chat_message.await_args_list[-1].kwargs[
+ "narrative_payload"
+ ]
+ assert persisted_payload is not None
+ assert persisted_payload["terminalEnvelope"]["workflow_applied"] is True
+ assert persisted_payload["terminalEnvelope"]["next_state"] == "completed"
+ assert persisted_payload["terminalEnvelope"]["response_kind"] == "update"
+
+
+async def test_finalise_normal_turn_envelope_not_applied_without_proposal_on_auto_accept(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ chat = SimpleNamespace(
+ organization_id="org-1",
+ workflow_copilot_chat_id="chat-1",
+ proposed_workflow=None,
+ auto_accept=True,
+ )
+ original_workflow = SimpleNamespace(workflow_id="wf-canonical")
+ # No proposal at all, and proposal_disposition left at the AgentResult
+ # default (auto_applicable) — the shape ASK_QUESTION/answer builders emit.
+ agent_result = AgentResult(
+ user_response="done",
+ updated_workflow=None,
+ global_llm_context=None,
+ response_type="REPLY",
+ narrative_payload=_narrative_payload(),
+ terminal_envelope=_terminal_payload(verified=True, workflow_applied=False),
+ )
+ _, workflow_params = setup_new_copilot_mocks(monkeypatch, chat, original_workflow, agent_result)
+ stream = MagicMock(send=AsyncMock(return_value=True))
+
+ await workflow_copilot_route._finalise_normal_turn(
+ stream=stream,
+ chat=chat,
+ organization_id="org-1",
+ original_workflow=original_workflow,
+ chat_request=_make_chat_request(),
+ agent_result=agent_result,
+ )
+
+ response_frame = stream.send.await_args.args[0]
+ assert isinstance(response_frame, WorkflowCopilotStreamResponseUpdate)
+ assert response_frame.terminal_envelope is not None
+ assert response_frame.terminal_envelope["workflow_applied"] is False
+ assert response_frame.terminal_envelope["next_state"] != "completed"
+
+ persisted_payload = workflow_params.create_workflow_copilot_chat_message.await_args_list[-1].kwargs[
+ "narrative_payload"
+ ]
+ assert persisted_payload is not None
+ assert persisted_payload["terminalEnvelope"]["workflow_applied"] is False
+ assert persisted_payload["terminalEnvelope"]["next_state"] != "completed"
+
+
+async def test_cancel_turn_finalizes_terminal_envelope_to_non_completed_state(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ chat = SimpleNamespace(
+ organization_id="org-1",
+ workflow_copilot_chat_id="chat-1",
+ proposed_workflow=None,
+ auto_accept=True,
+ )
+ original_workflow = SimpleNamespace(workflow_id="wf-canonical")
+ agent_result = AgentResult(
+ user_response="Cancelled by user.",
+ updated_workflow=None,
+ global_llm_context=None,
+ response_type="REPLY",
+ proposal_disposition="auto_applicable",
+ cancelled=True,
+ narrative_payload=_narrative_payload(),
+ terminal_envelope=_terminal_payload(verified=True, workflow_applied=True),
+ )
+ _, workflow_params = setup_new_copilot_mocks(monkeypatch, chat, original_workflow, agent_result)
+ stream = MagicMock(send=AsyncMock(return_value=True))
+
+ await workflow_copilot_route._persist_cancel_turn(
+ stream=stream,
+ chat=chat,
+ organization_id="org-1",
+ original_workflow=original_workflow,
+ user_message="stop",
+ agent_result=agent_result,
+ )
+
+ response_frame = stream.send.await_args.args[0]
+ assert isinstance(response_frame, WorkflowCopilotStreamResponseUpdate)
+ assert response_frame.cancelled is True
+ assert response_frame.terminal_envelope is not None
+ assert response_frame.terminal_envelope["workflow_applied"] is False
+ assert response_frame.terminal_envelope["next_state"] != "completed"
+ assert response_frame.terminal_envelope["response_kind"] == "stopped"
+
+ persisted_payload = workflow_params.create_workflow_copilot_chat_message.await_args_list[-1].kwargs[
+ "narrative_payload"
+ ]
+ assert persisted_payload is not None
+ assert persisted_payload["terminalEnvelope"]["workflow_applied"] is False
+ assert persisted_payload["terminalEnvelope"]["next_state"] != "completed"
+ assert persisted_payload["terminalEnvelope"]["response_kind"] == "stopped"
+
+
[email protected]("finalizer", ["normal", "cancel"], ids=["normal-turn", "cancel-turn"])
+async def test_terminal_envelope_finalization_failure_omits_envelope(
+ monkeypatch: pytest.MonkeyPatch,
+ finalizer: str,
+) -> None:
+ chat = SimpleNamespace(
+ organization_id="org-1",
+ workflow_copilot_chat_id="chat-1",
+ proposed_workflow=None,
+ auto_accept=(finalizer == "cancel"),
+ )
+ original_workflow = SimpleNamespace(workflow_id="wf-canonical")
+ updated_workflow = MagicMock()
+ updated_workflow.model_dump.return_value = {"workflow_id": "wf-draft"}
+ agent_result = AgentResult(
+ user_response="done" if finalizer == "normal" else "Cancelled by user.",
+ updated_workflow=updated_workflow if finalizer == "normal" else None,
+ global_llm_context=None,
+ response_type="REPLY",
+ proposal_disposition="auto_applicable",
+ cancelled=(finalizer == "cancel"),
+ narrative_payload=_narrative_payload(),
+ terminal_envelope=_terminal_payload(verified=True, workflow_applied=False),
+ )
+ _, workflow_params = setup_new_copilot_mocks(monkeypatch, chat, original_workflow, agent_result)
+ stream = MagicMock(send=AsyncMock(return_value=True))
+
+ def _raise(*_: Any, **__: Any) -> Any:
+ raise RuntimeError("finalize failed")
+
+ monkeypatch.setattr(workflow_copilot_route, "finalize_applied_state", _raise)
+
+ if finalizer == "normal":
+ await workflow_copilot_route._finalise_normal_turn(
+ stream=stream,
+ chat=chat,
+ organization_id="org-1",
+ original_workflow=original_workflow,
+ chat_request=_make_chat_request(),
+ agent_result=agent_result,
+ )
+ else:
+ await workflow_copilot_route._persist_cancel_turn(
+ stream=stream,
+ chat=chat,
+ organization_id="org-1",
+ original_workflow=original_workflow,
+ user_message="stop",
+ agent_result=agent_result,
+ )
+
+ response_frame = stream.send.await_args.args[0]
+ assert isinstance(response_frame, WorkflowCopilotStreamResponseUpdate)
+ assert response_frame.terminal_envelope is None
+ if finalizer == "cancel":
+ assert response_frame.cancelled is True
+
+ persisted_payload = workflow_params.create_workflow_copilot_chat_message.await_args_list[-1].kwargs[
+ "narrative_payload"
+ ]
+ assert persisted_payload is not None
+ assert "terminalEnvelope" not in persisted_payload
+
+
@pytest.mark.asyncio
async def test_get_debug_run_info_bounds_visible_elements_html(monkeypatch: pytest.MonkeyPatch) -> None:
max_chars = workflow_copilot_route.WORKFLOW_COPILOT_DEBUG_HTML_MAX_CHARS