Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
Assemble a shadow terminal outcome envelope per copilot turn (#7671)
pedrohsdb17fc4ea58d0e
diff --git a/skyvern/forge/sdk/copilot/agent.py b/skyvern/forge/sdk/copilot/agent.pyindex 7850e27d0..1a3a15f6e 100644--- a/skyvern/forge/sdk/copilot/agent.py+++ b/skyvern/forge/sdk/copilot/agent.py@@ -13,7 +13,7 @@ import jsonimport mathimport reimport uuid-from collections.abc import Callable, Iterator, Mapping+from collections.abc import Callable, Iterator, Mapping, Sequencefrom dataclasses import dataclassfrom datetime import datetime, timezonefrom io import BytesIO@@ -186,6 +186,7 @@ from skyvern.forge.sdk.copilot.streaming_adapter import (flush_goal_satisfied_tool_result,maybe_emit_design_end,)+from skyvern.forge.sdk.copilot.terminal_envelope import assemble_terminal_envelope, reason_in_reply_shadowfrom skyvern.forge.sdk.copilot.tracing_setup import _copilot_model_name, ensure_tracing_initialized, is_tracing_enabledfrom skyvern.forge.sdk.copilot.turn_context import TurnContextAssembler, TurnContextInputs, TurnContextPacketfrom skyvern.forge.sdk.copilot.turn_halt import (@@ -1729,6 +1730,82 @@ def _should_use_built_unverified_completed_reply()+def _terminal_envelope_run_outcomes(ctx: CopilotContext) -> list[RecordedRunOutcome]:+ raw = ctx.terminal_envelope_run_outcomes+ if isinstance(raw, Sequence) and not isinstance(raw, (str, bytes, bytearray)):+ outcomes = [outcome for outcome in raw if isinstance(outcome, RecordedRunOutcome)]+ if outcomes:+ return outcomes+ recorded = ctx.last_run_outcome+ return [recorded] if isinstance(recorded, RecordedRunOutcome) else []+++def _terminal_halt_fields(ctx: CopilotContext) -> tuple[str | None, str | None]:+ halt = getattr(ctx, "turn_halt", None)+ if not isinstance(halt, TurnHalt):+ return None, None+ blocker_reason: str | None = None+ signal = halt.blocker_signal+ if isinstance(signal, CopilotToolBlockerSignal):+ blocker_reason = signal.user_facing_reason+ return blocker_reason, halt.kind.value+++def _attempted_summary(narrative_summary: object, narrative_payload: object) -> str | None:+ if isinstance(narrative_summary, str) and narrative_summary.strip():+ return narrative_summary.strip()+ if isinstance(narrative_payload, Mapping):+ payload_summary = narrative_payload.get("narrativeSummary")+ if isinstance(payload_summary, str) and payload_summary.strip():+ return payload_summary.strip()+ return None+++def _assemble_terminal_envelope_safe(+ *,+ response_type: str,+ verified: bool,+ workflow_applied: bool,+ proposal_disposition: str | None,+ run_outcomes: Sequence[RecordedRunOutcome],+ blocker_reason: str | None,+ halt_kind: str | None,+ attempted: str | None,+ workflow_mutated: bool,+ turn_outcome_response_kind: str | None,+ final_message: str,+) -> dict[str, Any] | None:+ try:+ envelope = assemble_terminal_envelope(+ response_type=response_type,+ verified=verified,+ workflow_applied=workflow_applied,+ proposal_disposition=proposal_disposition,+ run_outcomes=run_outcomes,+ blocker_reason=blocker_reason,+ halt_kind=halt_kind,+ attempted=attempted,+ workflow_mutated=workflow_mutated,+ turn_outcome_response_kind=turn_outcome_response_kind,+ )+ except Exception:+ LOG.warning("copilot terminal envelope assembly failed", exc_info=True)+ return None+ if envelope is None:+ return None+ reason_in_reply = reason_in_reply_shadow(envelope.run_display_reason, final_message)+ payload = envelope.model_dump(mode="json")+ LOG.info(+ "copilot_terminal_envelope",+ **payload,+ response_type=response_type,+ envelope_response_kind=envelope.response_kind,+ reason_in_reply=reason_in_reply,+ finalized=False,+ )+ return payload++def _make_agent_result(ctx: CopilotContext | None,*,@@ -1795,6 +1872,25 @@ def _make_agent_result(payload_updates["deliveredUnverifiedObservedOutputs"] = _delivered_unverified_snapshotif payload_updates or len(payload_base) != len(narrative_payload):kwargs["narrative_payload"] = {**payload_base, **payload_updates}+ terminal_envelope: dict[str, Any] | None = None+ if ctx is not None:+ blocker_reason, halt_kind = _terminal_halt_fields(ctx)+ turn_outcome_response_kind = turn_outcome.response_kind.value if turn_outcome is not None else None+ response_type_value = response_type if isinstance(response_type, str) else "REPLY"+ terminal_envelope = _assemble_terminal_envelope_safe(+ response_type=response_type_value,+ verified=bool(verified_goal_claim_authorized(ctx)),+ workflow_applied=False,+ proposal_disposition=proposal_disposition if isinstance(proposal_disposition, str) else None,+ run_outcomes=_terminal_envelope_run_outcomes(ctx),+ blocker_reason=blocker_reason,+ halt_kind=halt_kind,+ attempted=_attempted_summary(kwargs.get("narrative_summary"), kwargs.get("narrative_payload")),+ workflow_mutated=bool(kwargs.get("workflow_was_persisted")) or kwargs.get("updated_workflow") is not None,+ turn_outcome_response_kind=turn_outcome_response_kind,+ final_message=str(kwargs.get("user_response") or ""),+ )+ kwargs["terminal_envelope"] = terminal_enveloperesult = AgentResult(global_llm_context=final_context, turn_outcome=turn_outcome, **kwargs)if ctx is not None and result.turn_outcome is not None:result.turn_outcome = with_copilot_code_mode_diagnostics(result.turn_outcome, ctx)diff --git a/skyvern/forge/sdk/copilot/blocker_signal.py b/skyvern/forge/sdk/copilot/blocker_signal.pyindex f1a4f0b07..530bf96c9 100644--- a/skyvern/forge/sdk/copilot/blocker_signal.py+++ b/skyvern/forge/sdk/copilot/blocker_signal.py@@ -354,6 +354,7 @@ class _TerminalEvidenceResetCtx(Protocol):last_run_blocks_block_labels: list[str]last_run_outcome: RecordedRunOutcome | Nonelast_run_outcome_block_labels: list[str]+ terminal_envelope_run_outcomes: list[RecordedRunOutcome]last_outcome_gate_reason: str | Nonelast_outcome_gate_workflow_run_id: str | Nonelast_test_anti_bot: str | None@@ -390,6 +391,7 @@ def clear_terminal_evidence_on_workflow_edit(ctx: _TerminalEvidenceResetCtx) ->ctx.last_run_blocks_block_labels = []ctx.last_run_outcome = Nonectx.last_run_outcome_block_labels = []+ ctx.terminal_envelope_run_outcomes = []ctx.last_outcome_gate_reason = Nonectx.last_outcome_gate_workflow_run_id = Nonectx.last_test_anti_bot = Nonediff --git a/skyvern/forge/sdk/copilot/context.py b/skyvern/forge/sdk/copilot/context.pyindex 8b0d06baa..d595dffbd 100644--- a/skyvern/forge/sdk/copilot/context.py+++ b/skyvern/forge/sdk/copilot/context.py@@ -90,6 +90,7 @@ class TurnNarrativePayload(TypedDict):proposalDisposition: NotRequired[ProposalDisposition]# TurnOutcome.response_kind value: "build" | "clarify" | "diagnose" | "refuse" | "recover".responseKind: NotRequired[str]+ terminalEnvelope: NotRequired[dict[str, Any]]# The ADR-0005 terminal adjudication (enforcement.verified_goal_claim_authorized):# True only when outcome evidence authorizes a tested-success claim.verifiedSuccess: NotRequired[bool]@@ -128,6 +129,7 @@ if TYPE_CHECKING:from skyvern.forge.sdk.copilot.narration import NarratorStatefrom skyvern.forge.sdk.copilot.output_extraction_plan import FrozenRequestedOutputExtractionCandidatefrom skyvern.forge.sdk.copilot.request_policy import RequestPolicy+ from skyvern.forge.sdk.copilot.run_outcome import RecordedRunOutcomefrom skyvern.forge.sdk.copilot.schema_incompatibility import SchemaIncompatibilityfrom skyvern.forge.sdk.copilot.turn_context import TurnContextPacketfrom skyvern.forge.sdk.copilot.turn_halt import TurnHalt@@ -669,6 +671,8 @@ class AgentResult:narrative_summary: str | None = None# Persisted on the assistant chat message so the bubble survives a reload.narrative_payload: TurnNarrativePayload | None = None+ # Shadow-only typed terminal-state envelope persisted and streamed on terminal frames.+ terminal_envelope: dict[str, Any] | None = Nonestaged_workflow_yaml: str | None = Nonestaged_workflow: Workflow | None = Nonehas_staged_proposal: bool = False@@ -801,6 +805,11 @@ class CopilotContext(AgentContext):last_run_blocks_workflow_run_id: str | None = Nonelast_successful_run_blocks_workflow_run_id: str | None = Nonelast_outcome_gate_workflow_run_id: str | None = None+ # In-turn run-outcome trace derived from assignments to ``last_run_outcome``+ # (the same source that powers run_outcome SSE frames). Append-only across+ # per-run pointer resets (``last_run_outcome = None``); cleared only by the+ # workflow-edit evidence reset, which invalidates pre-edit run evidence.+ terminal_envelope_run_outcomes: list[RecordedRunOutcome] = field(default_factory=list)delivered_unverified_terminal: bool = Falsedelivered_unverified_workflow_run_id: str | None = Nonedelivered_unverified_observed_outputs: dict[str, Any] = field(default_factory=DeliveredUnverifiedPublicOutputs)@@ -959,6 +968,32 @@ class CopilotContext(AgentContext):turn_started_at: str | None = Noneturn_ended_at: str | None = None+ def __post_init__(self) -> None:+ parent_post_init = getattr(super(), "__post_init__", None)+ if callable(parent_post_init):+ parent_post_init()+ from skyvern.forge.sdk.copilot.run_outcome import RecordedRunOutcome++ if isinstance(self.last_run_outcome, RecordedRunOutcome):+ super().__setattr__("terminal_envelope_run_outcomes", [self.last_run_outcome])++ def __setattr__(self, name: str, value: Any) -> None:+ super().__setattr__(name, value)+ if name != "last_run_outcome":+ return+ from skyvern.forge.sdk.copilot.run_outcome import RecordedRunOutcome++ if not isinstance(value, RecordedRunOutcome):+ # ``last_run_outcome = None`` is a per-run pointer reset, not an+ # evidence reset — the trace survives so a later run in the same+ # turn cannot mask an earlier honest not_demonstrated.+ return+ outcomes = getattr(self, "terminal_envelope_run_outcomes", None)+ if isinstance(outcomes, list):+ outcomes.append(value)+ else:+ super().__setattr__("terminal_envelope_run_outcomes", [value])+def has_genuine_workflow_attempt(self) -> bool:"""This turn persisted a workflow proposal or executed a real build-test run; excludes``test_after_update_done``, which is stamped for any ``run_blocks_and_collect_debug`` scoutdiff --git a/skyvern/forge/sdk/copilot/terminal_envelope.py b/skyvern/forge/sdk/copilot/terminal_envelope.pynew file mode 100644index 000000000..7947785f6--- /dev/null+++ b/skyvern/forge/sdk/copilot/terminal_envelope.py@@ -0,0 +1,189 @@+from __future__ import annotations++from collections.abc import Sequence+from typing import Literal++import structlog+from pydantic import BaseModel++from skyvern.forge.sdk.copilot.run_outcome import RecordedRunOutcome++LOG = structlog.get_logger(__name__)++TerminalNextState = Literal["completed", "proposal_pending", "awaiting_user_input", "stopped"]+TerminalResponseKind = Literal["question", "update", "answer", "stopped"]+_FINAL_RUN_VERDICTS = frozenset({"demonstrated", "not_demonstrated", "not_evaluated"})+_REVIEW_PROPOSAL_DISPOSITIONS = frozenset({"review_untested", "review_tested"})+_SHADOW_REASON_TRAILING_PUNCTUATION = ".,;:!?"+++class TerminalOutcomeEnvelope(BaseModel):+ next_state: TerminalNextState+ verified: bool+ workflow_applied: bool = False+ run_verdict: str | None = None+ run_display_reason: str | None = None+ blocker_reason: str | None = None+ halt_kind: str | None = None+ user_action_required: bool = False+ attempted: str | None = None+ response_kind: TerminalResponseKind+ envelope_version: int = 1+++def assemble_terminal_envelope(+ *,+ response_type: str,+ verified: bool,+ workflow_applied: bool,+ proposal_disposition: str | None,+ run_outcomes: Sequence[RecordedRunOutcome],+ blocker_reason: str | None,+ halt_kind: str | None,+ attempted: str | None,+ workflow_mutated: bool,+ turn_outcome_response_kind: str | None,+) -> TerminalOutcomeEnvelope | None:+ run_outcome = _select_run_outcome_anchor(run_outcomes)+ superseding_outcome = _later_demonstrated_after_anchor(run_outcomes, run_outcome)+ if run_outcome is not None and superseding_outcome is not None:+ LOG.info(+ "Terminal envelope anchored a not_demonstrated verdict past a later demonstrated run",+ anchored_workflow_run_id=run_outcome.workflow_run_id,+ anchored_reason_code=run_outcome.reason_code,+ later_workflow_run_id=superseding_outcome.workflow_run_id,+ )+ run_verdict = run_outcome.verdict if run_outcome is not None else None+ run_display_reason = _clean_text(run_outcome.display_reason) if run_outcome is not None else None+ user_action_required = response_type == "ASK_QUESTION"+ next_state = _derive_next_state(+ user_action_required=user_action_required,+ verified=verified,+ workflow_applied=workflow_applied,+ proposal_disposition=proposal_disposition,+ )+ response_kind = _derive_response_kind(+ user_action_required=user_action_required,+ next_state=next_state,+ workflow_mutated=workflow_mutated,+ turn_outcome_response_kind=turn_outcome_response_kind,+ )+ return TerminalOutcomeEnvelope(+ next_state=next_state,+ verified=verified,+ workflow_applied=workflow_applied,+ run_verdict=run_verdict,+ run_display_reason=run_display_reason,+ blocker_reason=_clean_text(blocker_reason),+ halt_kind=_clean_text(halt_kind),+ user_action_required=user_action_required,+ attempted=_clean_text(attempted),+ response_kind=response_kind,+ )+++def finalize_applied_state(envelope: TerminalOutcomeEnvelope, *, applied: bool) -> TerminalOutcomeEnvelope:+ if envelope.user_action_required:+ next_state: TerminalNextState = "awaiting_user_input"+ elif envelope.verified and applied:+ next_state = "completed"+ elif envelope.next_state == "proposal_pending":+ next_state = "proposal_pending"+ else:+ next_state = "stopped"+ response_kind = _derive_response_kind(+ user_action_required=envelope.user_action_required,+ next_state=next_state,+ prior_response_kind=envelope.response_kind,+ )+ return envelope.model_copy(+ update={"workflow_applied": applied, "next_state": next_state, "response_kind": response_kind}+ )+++def _select_run_outcome_anchor(run_outcomes: Sequence[RecordedRunOutcome]) -> RecordedRunOutcome | None:+ final_outcomes = [outcome for outcome in run_outcomes if outcome.verdict in _FINAL_RUN_VERDICTS]+ if not final_outcomes:+ return None+ last_not_demonstrated = [outcome for outcome in final_outcomes if outcome.verdict == "not_demonstrated"]+ if last_not_demonstrated:+ return last_not_demonstrated[-1]+ return final_outcomes[-1]+++def _later_demonstrated_after_anchor(+ run_outcomes: Sequence[RecordedRunOutcome], anchor: RecordedRunOutcome | None+) -> RecordedRunOutcome | None:+ if anchor is None or anchor.verdict != "not_demonstrated":+ return None+ seen_anchor = False+ for outcome in run_outcomes:+ if outcome is anchor:+ seen_anchor = True+ continue+ if seen_anchor and outcome.verdict == "demonstrated":+ return outcome+ return None+++def _derive_next_state(+ *,+ user_action_required: bool,+ verified: bool,+ workflow_applied: bool,+ proposal_disposition: str | None,+) -> TerminalNextState:+ if user_action_required:+ return "awaiting_user_input"+ if verified and workflow_applied:+ return "completed"+ if _proposal_requires_review(proposal_disposition):+ return "proposal_pending"+ return "stopped"+++def _derive_response_kind(+ *,+ user_action_required: bool,+ next_state: TerminalNextState,+ workflow_mutated: bool | None = None,+ turn_outcome_response_kind: str | None = None,+ prior_response_kind: TerminalResponseKind | None = None,+) -> TerminalResponseKind:+ if user_action_required:+ return "question"+ if next_state in {"completed", "proposal_pending"}:+ return "update"+ if prior_response_kind == "answer":+ return "answer"+ if workflow_mutated is False and turn_outcome_response_kind == "diagnose":+ return "answer"+ return "stopped"+++def normalize_shadow_reason_text(text: object, *, strip_trailing_punctuation: bool = False) -> str | None:+ if not isinstance(text, str):+ return None+ normalized = " ".join(text.lower().split())+ if strip_trailing_punctuation:+ normalized = normalized.rstrip(_SHADOW_REASON_TRAILING_PUNCTUATION).strip()+ return normalized or None+++def reason_in_reply_shadow(run_display_reason: str | None, final_message: str) -> bool:+ normalized_reason = normalize_shadow_reason_text(run_display_reason, strip_trailing_punctuation=True)+ normalized_reply = normalize_shadow_reason_text(final_message)+ return bool(normalized_reason and normalized_reply and normalized_reason in normalized_reply)+++def _proposal_requires_review(proposal_disposition: str | None) -> bool:+ if not isinstance(proposal_disposition, str):+ return False+ return proposal_disposition.strip().lower() in _REVIEW_PROPOSAL_DISPOSITIONS+++def _clean_text(value: str | None) -> str | None:+ if not isinstance(value, str):+ return None+ text = value.strip()+ return text or Nonediff --git a/skyvern/forge/sdk/routes/workflow_copilot.py b/skyvern/forge/sdk/routes/workflow_copilot.pyindex 6ed907ecc..ed04dd601 100644--- a/skyvern/forge/sdk/routes/workflow_copilot.py+++ b/skyvern/forge/sdk/routes/workflow_copilot.py@@ -58,6 +58,11 @@ from skyvern.forge.sdk.copilot.recoverable_failure import (merge_failure_into_context,)from skyvern.forge.sdk.copilot.request_policy import is_defer_authoring_durable_fill_criterion+from skyvern.forge.sdk.copilot.terminal_envelope import (+ TerminalOutcomeEnvelope,+ finalize_applied_state,+ reason_in_reply_shadow,+)from skyvern.forge.sdk.copilot.turn_outcome import (CopilotComposerMode,build_minimal_turn_outcome,@@ -462,14 +467,69 @@ def _with_terminal_narrative_metadata(*,cancelled: bool,proposal_disposition: ProposalDisposition,+ terminal_envelope: dict[str, Any] | None = None,) -> TurnNarrativePayload | None:if narrative_payload is None:return None- return {+ payload: TurnNarrativePayload = {**narrative_payload,"cancelled": cancelled,"proposalDisposition": proposal_disposition,}+ if terminal_envelope is not None:+ payload["terminalEnvelope"] = terminal_envelope+ return payload+++def _agent_terminal_envelope(agent_result: AgentResult | None) -> dict[str, Any] | None:+ if agent_result is None:+ return None+ try:+ payload = agent_result.terminal_envelope+ except AttributeError:+ return None+ return payload if isinstance(payload, dict) else None+++def _finalized_terminal_envelope(+ agent_result: AgentResult | None,+ *,+ workflow_applied: bool,+ final_message: str,+ response_type: str,+) -> dict[str, Any] | None:+ payload = _agent_terminal_envelope(agent_result)+ if payload is None:+ return None+ # workflow_applied mirrors the legacy auto-accept SSE field, which reads+ # true on no-proposal turns because AgentResult defaults its disposition+ # to auto_applicable. The envelope must only claim applied when a+ # proposal actually existed to commit.+ proposal_present = getattr(agent_result, "updated_workflow", None) is not None or bool(+ getattr(agent_result, "has_staged_proposal", False)+ )+ try:+ finalized = finalize_applied_state(+ TerminalOutcomeEnvelope.model_validate(payload),+ applied=workflow_applied and proposal_present,+ )+ payload = finalized.model_dump(mode="json")+ except Exception:+ LOG.warning("copilot terminal envelope finalization failed", exc_info=True)+ return None+ reason_in_reply = reason_in_reply_shadow(+ payload.get("run_display_reason") if isinstance(payload.get("run_display_reason"), str) else None,+ final_message,+ )+ LOG.info(+ "copilot_terminal_envelope",+ **payload,+ response_type=response_type,+ envelope_response_kind=payload.get("response_kind"),+ reason_in_reply=reason_in_reply,+ finalized=True,+ )+ return payloaddef _build_recoverable_route_agent_result(@@ -698,6 +758,7 @@ async def _persist_cancel_turn(the success path; pass ``None`` for pre-agent cancels."""turn_outcome: TurnOutcome | None+ workflow_applied = Falseif agent_result is None:user_response = "Cancelled by user."updated_workflow = None@@ -709,6 +770,7 @@ async def _persist_cancel_turn(response_turn_id = turn_idnarrative_summary = Nonenarrative_payload = None+ terminal_envelope = Noneif chat.proposed_workflow is not None and not keep_pending_proposal:await asyncio.shield(_clear_proposed_workflow(chat))else:@@ -751,12 +813,20 @@ async def _persist_cancel_turn(response_turn_id = turn_id or agent_result.turn_idnarrative_summary = agent_result.narrative_summarynarrative_payload = agent_result.narrative_payload+ workflow_applied = _effective_auto_accept(chat.auto_accept, agent_result)+ terminal_envelope = _finalized_terminal_envelope(+ agent_result,+ workflow_applied=workflow_applied,+ final_message=user_response,+ response_type=response_type,+ )proposal_disposition = _proposal_disposition(agent_result)narrative_payload = _with_terminal_narrative_metadata(narrative_payload,cancelled=True,proposal_disposition=proposal_disposition,+ terminal_envelope=terminal_envelope,)await asyncio.shield(@@ -791,11 +861,13 @@ async def _persist_cancel_turn(total_tokens=total_tokens,response_type=response_type,proposal_disposition=proposal_disposition,+ workflow_applied=workflow_applied,cancelled=True,output_policy_diagnostics=output_policy_diagnostics,turn_id=response_turn_id,narrative_summary=narrative_summary,narrative_payload=narrative_payload,+ terminal_envelope=terminal_envelope,)))@@ -872,10 +944,17 @@ async def _finalise_normal_turn(await _persist_completion_criteria_state(chat, agent_result, chat_request.message)proposal_disposition = _proposal_disposition(agent_result)workflow_applied = _effective_auto_accept(chat.auto_accept, agent_result)+ terminal_envelope = _finalized_terminal_envelope(+ agent_result,+ workflow_applied=workflow_applied,+ final_message=user_response,+ response_type=agent_result.response_type,+ )narrative_payload = _with_terminal_narrative_metadata(agent_result.narrative_payload,cancelled=False,proposal_disposition=proposal_disposition,+ terminal_envelope=terminal_envelope,)await app.DATABASE.workflow_params.create_workflow_copilot_chat_message(@@ -911,6 +990,7 @@ async def _finalise_normal_turn(turn_id=agent_result.turn_id,narrative_summary=agent_result.narrative_summary,narrative_payload=narrative_payload,+ terminal_envelope=terminal_envelope,))diff --git a/skyvern/forge/sdk/schemas/workflow_copilot.py b/skyvern/forge/sdk/schemas/workflow_copilot.pyindex 5f1b79881..68c05f188 100644--- a/skyvern/forge/sdk/schemas/workflow_copilot.py+++ b/skyvern/forge/sdk/schemas/workflow_copilot.py@@ -282,6 +282,10 @@ class WorkflowCopilotStreamResponseUpdate(BaseModel):None,description="Terminal narrative bubble snapshot for live clients; mirrors the persisted assistant chat row.",)+ terminal_envelope: dict[str, Any] | None = Field(+ None,+ description="Shadow-only terminal outcome envelope for end-of-turn typed-state validation.",+ )class WorkflowCopilotStreamErrorUpdate(BaseModel):diff --git a/tests/unit/test_copilot_terminal_envelope.py b/tests/unit/test_copilot_terminal_envelope.pynew file mode 100644index 000000000..1c5529b7d--- /dev/null+++ b/tests/unit/test_copilot_terminal_envelope.py@@ -0,0 +1,332 @@+from __future__ import annotations++from typing import Any++import pytest+import structlog.testing++from skyvern.forge.sdk.copilot import agent as agent_module+from skyvern.forge.sdk.copilot.blocker_signal import clear_terminal_evidence_on_workflow_edit+from skyvern.forge.sdk.copilot.run_outcome import RecordedRunOutcome+from skyvern.forge.sdk.copilot.terminal_envelope import (+ assemble_terminal_envelope,+ finalize_applied_state,+ reason_in_reply_shadow,+)+from skyvern.forge.sdk.copilot.tools.run_execution import _stash_recorded_run_outcome+from tests.unit.copilot_test_helpers import make_copilot_ctx+++def _run_outcome(verdict: str, display_reason: str | None = None) -> RecordedRunOutcome:+ return RecordedRunOutcome(verdict=verdict, display_reason=display_reason)+++def _assemble(**overrides: Any):+ defaults = {+ "response_type": "REPLY",+ "verified": False,+ "workflow_applied": False,+ "proposal_disposition": "no_proposal",+ "run_outcomes": [],+ "blocker_reason": None,+ "halt_kind": None,+ "attempted": None,+ "workflow_mutated": False,+ "turn_outcome_response_kind": None,+ }+ defaults.update(overrides)+ envelope = assemble_terminal_envelope(**defaults)+ assert envelope is not None+ return envelope+++def test_run_anchor_prefers_last_not_demonstrated_even_if_later_run_demonstrated() -> None:+ envelope = _assemble(+ run_outcomes=[+ _run_outcome("not_demonstrated", "The checkout did not reach confirmation."),+ _run_outcome("demonstrated", "A later scout run succeeded."),+ ]+ )++ assert envelope.run_verdict == "not_demonstrated"+ assert envelope.run_display_reason == "The checkout did not reach confirmation."+++def test_run_anchor_falls_back_to_latest_final_verdict_when_no_not_demonstrated() -> None:+ envelope = _assemble(+ run_outcomes=[+ _run_outcome("not_evaluated", "Could not evaluate."),+ _run_outcome("demonstrated", "Confirmed."),+ ]+ )++ assert envelope.run_verdict == "demonstrated"+ assert envelope.run_display_reason == "Confirmed."+++def test_run_anchor_empty_when_no_recorded_outcomes() -> None:+ envelope = _assemble(run_outcomes=[])+ assert envelope.run_verdict is None+ assert envelope.run_display_reason is None+++def test_unknown_halt_kind_degrades_to_stopped_never_question() -> None:+ envelope = _assemble(+ halt_kind="capture_obligation_reopen",+ blocker_reason="Capture obligation reopened mid-turn.",+ )++ assert envelope.response_kind == "stopped"+ assert envelope.next_state == "stopped"+ assert envelope.halt_kind == "capture_obligation_reopen"++ finalized = finalize_applied_state(envelope, applied=False)+ assert finalized.response_kind == "stopped"+ assert finalized.next_state == "stopped"+++def test_anchor_supersession_divergence_is_logged() -> None:+ with structlog.testing.capture_logs() as logs:+ _assemble(+ run_outcomes=[+ _run_outcome("not_demonstrated", "The checkout did not reach confirmation."),+ _run_outcome("demonstrated", "A later scout run succeeded."),+ ]+ )+ assert any("anchored a not_demonstrated verdict" in log["event"] for log in logs)++ with structlog.testing.capture_logs() as logs:+ _assemble(run_outcomes=[_run_outcome("not_demonstrated", "No later run.")])+ assert not any("anchored a not_demonstrated verdict" in log["event"] for log in logs)+++ ("response_type", "verified", "workflow_applied", "proposal_disposition", "expected_next_state"),+ [+ ("ASK_QUESTION", False, False, "no_proposal", "awaiting_user_input"),+ ("REPLY", True, True, "no_proposal", "completed"),+ ("REPLY", False, False, "review_tested", "proposal_pending"),+ ("REPLY", False, False, "review_required", "stopped"),+ ("REPLY", True, False, "auto_applicable", "stopped"),+ ],+)+def test_next_state_derivation(+ response_type: str,+ verified: bool,+ workflow_applied: bool,+ proposal_disposition: str,+ expected_next_state: str,+) -> None:+ envelope = _assemble(+ response_type=response_type,+ verified=verified,+ workflow_applied=workflow_applied,+ proposal_disposition=proposal_disposition,+ )+ assert envelope.next_state == expected_next_state+++ ("kwargs", "expected_response_kind"),+ [+ ({"response_type": "ASK_QUESTION"}, "question"),+ ({"verified": True, "workflow_applied": True}, "update"),+ ({"proposal_disposition": "review_untested"}, "update"),+ ({"turn_outcome_response_kind": "diagnose", "workflow_mutated": False}, "answer"),+ ({"turn_outcome_response_kind": "diagnose", "workflow_mutated": True}, "stopped"),+ ({"turn_outcome_response_kind": "build", "workflow_mutated": False}, "stopped"),+ ],+)+def test_response_kind_derivation(kwargs: dict[str, Any], expected_response_kind: str) -> None:+ envelope = _assemble(**kwargs)+ assert envelope.response_kind == expected_response_kind+++def test_user_action_required_derivation() -> None:+ assert _assemble(response_type="ASK_QUESTION").user_action_required is True+ assert _assemble(response_type="REPLY").user_action_required is False+++def test_blocker_fields_attempted_and_envelope_version() -> None:+ envelope = _assemble(+ blocker_reason=" Need account credentials. ",+ halt_kind=" loop_detected ",+ attempted=" Attempted full checkout run. ",+ )++ assert envelope.blocker_reason == "Need account credentials."+ assert envelope.halt_kind == "loop_detected"+ assert envelope.attempted == "Attempted full checkout run."+ assert envelope.envelope_version == 1+++def test_finalize_applied_state_promotes_completed_when_verified_and_applied() -> None:+ envelope = _assemble(verified=True, workflow_applied=False, proposal_disposition="no_proposal")++ finalized = finalize_applied_state(envelope, applied=True)++ assert finalized.workflow_applied is True+ assert finalized.next_state == "completed"+ assert finalized.response_kind == "update"+++def test_finalize_applied_state_blocks_completed_when_not_applied() -> None:+ envelope = _assemble(verified=True, workflow_applied=True, proposal_disposition="no_proposal")++ finalized = finalize_applied_state(envelope, applied=False)++ assert finalized.workflow_applied is False+ assert finalized.next_state == "stopped"+ assert finalized.response_kind == "stopped"+++def test_finalize_applied_state_keeps_question_for_user_action_required() -> None:+ envelope = _assemble(+ response_type="ASK_QUESTION", verified=True, workflow_applied=False, proposal_disposition="no_proposal"+ )++ finalized = finalize_applied_state(envelope, applied=True)++ assert finalized.workflow_applied is True+ assert finalized.next_state == "awaiting_user_input"+ assert finalized.response_kind == "question"+++def test_finalize_applied_state_preserves_answer_when_not_promoted_to_update() -> None:+ envelope = _assemble(turn_outcome_response_kind="diagnose", workflow_mutated=False)+ assert envelope.response_kind == "answer"++ finalized = finalize_applied_state(envelope, applied=False)++ assert finalized.next_state == "stopped"+ assert finalized.response_kind == "answer"+++def test_terminal_envelope_outcomes_survive_per_run_pointer_reset() -> None:+ ctx = make_copilot_ctx()+ first = RecordedRunOutcome(+ verdict="not_demonstrated",+ display_reason="Checkout never reached confirmation.",+ workflow_run_id="wr_first",+ )+ second = RecordedRunOutcome(+ verdict="demonstrated",+ display_reason="A later scout replay succeeded.",+ workflow_run_id="wr_second",+ )++ _stash_recorded_run_outcome(ctx, first)+ # _record_run_blocks_result resets the pointer before processing each new+ # run in the turn; the trace must survive it or the anchor never sees the+ # earlier failure.+ ctx.last_run_outcome = None+ ctx.last_run_outcome_block_labels = []+ _stash_recorded_run_outcome(ctx, second)+ outcomes = agent_module._terminal_envelope_run_outcomes(ctx)++ assert [outcome.verdict for outcome in outcomes] == ["not_demonstrated", "demonstrated"]+ assert outcomes[0].display_reason == "Checkout never reached confirmation."+ assert outcomes[1].display_reason == "A later scout replay succeeded."++ envelope = _assemble(run_outcomes=outcomes)++ assert envelope.run_verdict == "not_demonstrated"+ assert envelope.run_display_reason == "Checkout never reached confirmation."+++def test_terminal_envelope_outcomes_seed_from_constructor_last_run_outcome() -> None:+ first = RecordedRunOutcome(+ verdict="not_demonstrated",+ display_reason="Seeded from constructor.",+ workflow_run_id="wr_ctor",+ )+ second = RecordedRunOutcome(+ verdict="demonstrated",+ display_reason="Appended after construction.",+ workflow_run_id="wr_runtime",+ )+ ctx = make_copilot_ctx(last_run_outcome=first)++ assert ctx.terminal_envelope_run_outcomes == [first]++ ctx.last_run_outcome = second++ assert ctx.terminal_envelope_run_outcomes == [first, second]+++def test_terminal_envelope_outcomes_clear_on_workflow_edit_evidence_reset() -> None:+ ctx = make_copilot_ctx()+ _stash_recorded_run_outcome(+ ctx,+ RecordedRunOutcome(+ verdict="not_demonstrated",+ display_reason="Checkout never reached confirmation.",+ workflow_run_id="wr_before_reset",+ ),+ )++ clear_terminal_evidence_on_workflow_edit(ctx)+ outcomes = agent_module._terminal_envelope_run_outcomes(ctx)+ envelope = _assemble(run_outcomes=outcomes)++ assert ctx.terminal_envelope_run_outcomes == []+ assert outcomes == []+ assert envelope.run_verdict is None+++def test_terminal_envelope_outcomes_reanchor_to_new_outcome_after_workflow_edit() -> None:+ ctx = make_copilot_ctx()+ _stash_recorded_run_outcome(+ ctx,+ RecordedRunOutcome(+ verdict="not_demonstrated",+ display_reason="Old failed run.",+ workflow_run_id="wr_old",+ ),+ )+ clear_terminal_evidence_on_workflow_edit(ctx)++ _stash_recorded_run_outcome(+ ctx,+ RecordedRunOutcome(+ verdict="not_demonstrated",+ display_reason="New failed run after edit.",+ workflow_run_id="wr_new",+ ),+ )+ outcomes = agent_module._terminal_envelope_run_outcomes(ctx)+ envelope = _assemble(run_outcomes=outcomes)++ assert len(outcomes) == 1+ assert outcomes[0].workflow_run_id == "wr_new"+ assert envelope.run_verdict == "not_demonstrated"+ assert envelope.run_display_reason == "New failed run after edit."+++def test_reason_in_reply_shadow_normalization() -> None:+ assert reason_in_reply_shadow(+ "Run completed but did not demonstrate the requested outcome.",+ "The latest run completed but did not demonstrate the requested outcome, so I paused.",+ )+++def test_safe_wrapper_returns_none_when_assembly_raises(monkeypatch: pytest.MonkeyPatch) -> None:+ def _raise(**_: Any) -> Any:+ raise RuntimeError("boom")++ monkeypatch.setattr(agent_module, "assemble_terminal_envelope", _raise)+ envelope = agent_module._assemble_terminal_envelope_safe(+ response_type="REPLY",+ verified=False,+ workflow_applied=False,+ proposal_disposition="no_proposal",+ run_outcomes=[],+ blocker_reason=None,+ halt_kind=None,+ attempted=None,+ workflow_mutated=False,+ turn_outcome_response_kind=None,+ final_message="reply",+ )++ assert envelope is Nonediff --git a/tests/unit/test_workflow_copilot_route.py b/tests/unit/test_workflow_copilot_route.pyindex fa290fd80..1deb685b6 100644--- a/tests/unit/test_workflow_copilot_route.py+++ b/tests/unit/test_workflow_copilot_route.py@@ -20,6 +20,7 @@ import jsonfrom datetime import datetime, timezonefrom io import BytesIOfrom types import SimpleNamespace+from typing import Anyfrom unittest.mock import AsyncMock, MagicMockimport pytest@@ -31,11 +32,12 @@ from skyvern.forge import appfrom skyvern.forge.sdk.api.llm.exceptions import LLMProviderErrorfrom skyvern.forge.sdk.artifact.models import ArtifactTypefrom skyvern.forge.sdk.copilot import agent as agent_module-from skyvern.forge.sdk.copilot.context import DeliveredUnverifiedPublicOutputs+from skyvern.forge.sdk.copilot.context import AgentResult, DeliveredUnverifiedPublicOutputsfrom skyvern.forge.sdk.copilot.schema_incompatibility import (SchemaIncompatibility,render_schema_incompatibility_user_reason,)+from skyvern.forge.sdk.copilot.terminal_envelope import assemble_terminal_envelopefrom skyvern.forge.sdk.copilot.turn_halt import TurnHaltKindfrom skyvern.forge.sdk.copilot.turn_outcome import (build_minimal_turn_outcome,@@ -77,6 +79,304 @@ def _make_chat_request()+def _terminal_payload(*, verified: bool, workflow_applied: bool) -> dict[str, Any]:+ envelope = assemble_terminal_envelope(+ response_type="REPLY",+ verified=verified,+ workflow_applied=workflow_applied,+ proposal_disposition="no_proposal",+ run_outcomes=[],+ blocker_reason=None,+ halt_kind=None,+ attempted="Attempted full run.",+ workflow_mutated=True,+ turn_outcome_response_kind="build",+ )+ assert envelope is not None+ return envelope.model_dump(mode="json")+++def _narrative_payload() -> dict[str, Any]:+ return {+ "turnId": "turn-1",+ "turnIndex": 0,+ "mode": "build",+ "designStarted": True,+ "designEnded": True,+ "draft": None,+ "blocks": [],+ "terminal": "response",+ "terminalMessage": "done",+ "narrativeSummary": "done",+ "priorBlockCount": None,+ "designActivity": [],+ "startedAt": None,+ "endedAt": None,+ }+++async def test_finalise_normal_turn_finalizes_terminal_envelope_without_auto_accept(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ chat = SimpleNamespace(+ organization_id="org-1",+ workflow_copilot_chat_id="chat-1",+ proposed_workflow=None,+ auto_accept=False,+ )+ original_workflow = SimpleNamespace(workflow_id="wf-canonical")+ updated_workflow = MagicMock()+ updated_workflow.model_dump.return_value = {"workflow_id": "wf-draft"}+ agent_result = AgentResult(+ user_response="done",+ updated_workflow=updated_workflow,+ global_llm_context=None,+ response_type="REPLY",+ proposal_disposition="auto_applicable",+ narrative_payload=_narrative_payload(),+ terminal_envelope=_terminal_payload(verified=True, workflow_applied=False),+ )+ _, workflow_params = setup_new_copilot_mocks(monkeypatch, chat, original_workflow, agent_result)+ stream = MagicMock(send=AsyncMock(return_value=True))++ await workflow_copilot_route._finalise_normal_turn(+ stream=stream,+ chat=chat,+ organization_id="org-1",+ original_workflow=original_workflow,+ chat_request=_make_chat_request(),+ agent_result=agent_result,+ )++ response_frame = stream.send.await_args.args[0]+ assert isinstance(response_frame, WorkflowCopilotStreamResponseUpdate)+ assert response_frame.terminal_envelope is not None+ assert response_frame.terminal_envelope["workflow_applied"] is False+ assert response_frame.terminal_envelope["next_state"] != "completed"+ assert response_frame.terminal_envelope["response_kind"] == "stopped"++ persisted_payload = workflow_params.create_workflow_copilot_chat_message.await_args_list[-1].kwargs[+ "narrative_payload"+ ]+ assert persisted_payload is not None+ assert persisted_payload["terminalEnvelope"]["workflow_applied"] is False+ assert persisted_payload["terminalEnvelope"]["next_state"] != "completed"+ assert persisted_payload["terminalEnvelope"]["response_kind"] == "stopped"+++async def test_finalise_normal_turn_finalizes_terminal_envelope_with_auto_accept(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ chat = SimpleNamespace(+ organization_id="org-1",+ workflow_copilot_chat_id="chat-1",+ proposed_workflow=None,+ auto_accept=True,+ )+ original_workflow = SimpleNamespace(workflow_id="wf-canonical")+ updated_workflow = MagicMock()+ updated_workflow.model_dump.return_value = {"workflow_id": "wf-draft"}+ agent_result = AgentResult(+ user_response="done",+ updated_workflow=updated_workflow,+ global_llm_context=None,+ response_type="REPLY",+ proposal_disposition="auto_applicable",+ narrative_payload=_narrative_payload(),+ terminal_envelope=_terminal_payload(verified=True, workflow_applied=False),+ )+ _, workflow_params = setup_new_copilot_mocks(monkeypatch, chat, original_workflow, agent_result)+ stream = MagicMock(send=AsyncMock(return_value=True))++ await workflow_copilot_route._finalise_normal_turn(+ stream=stream,+ chat=chat,+ organization_id="org-1",+ original_workflow=original_workflow,+ chat_request=_make_chat_request(),+ agent_result=agent_result,+ )++ response_frame = stream.send.await_args.args[0]+ assert isinstance(response_frame, WorkflowCopilotStreamResponseUpdate)+ assert response_frame.terminal_envelope is not None+ assert response_frame.terminal_envelope["workflow_applied"] is True+ assert response_frame.terminal_envelope["next_state"] == "completed"+ assert response_frame.terminal_envelope["response_kind"] == "update"++ persisted_payload = workflow_params.create_workflow_copilot_chat_message.await_args_list[-1].kwargs[+ "narrative_payload"+ ]+ assert persisted_payload is not None+ assert persisted_payload["terminalEnvelope"]["workflow_applied"] is True+ assert persisted_payload["terminalEnvelope"]["next_state"] == "completed"+ assert persisted_payload["terminalEnvelope"]["response_kind"] == "update"+++async def test_finalise_normal_turn_envelope_not_applied_without_proposal_on_auto_accept(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ chat = SimpleNamespace(+ organization_id="org-1",+ workflow_copilot_chat_id="chat-1",+ proposed_workflow=None,+ auto_accept=True,+ )+ original_workflow = SimpleNamespace(workflow_id="wf-canonical")+ # No proposal at all, and proposal_disposition left at the AgentResult+ # default (auto_applicable) — the shape ASK_QUESTION/answer builders emit.+ agent_result = AgentResult(+ user_response="done",+ updated_workflow=None,+ global_llm_context=None,+ response_type="REPLY",+ narrative_payload=_narrative_payload(),+ terminal_envelope=_terminal_payload(verified=True, workflow_applied=False),+ )+ _, workflow_params = setup_new_copilot_mocks(monkeypatch, chat, original_workflow, agent_result)+ stream = MagicMock(send=AsyncMock(return_value=True))++ await workflow_copilot_route._finalise_normal_turn(+ stream=stream,+ chat=chat,+ organization_id="org-1",+ original_workflow=original_workflow,+ chat_request=_make_chat_request(),+ agent_result=agent_result,+ )++ response_frame = stream.send.await_args.args[0]+ assert isinstance(response_frame, WorkflowCopilotStreamResponseUpdate)+ assert response_frame.terminal_envelope is not None+ assert response_frame.terminal_envelope["workflow_applied"] is False+ assert response_frame.terminal_envelope["next_state"] != "completed"++ persisted_payload = workflow_params.create_workflow_copilot_chat_message.await_args_list[-1].kwargs[+ "narrative_payload"+ ]+ assert persisted_payload is not None+ assert persisted_payload["terminalEnvelope"]["workflow_applied"] is False+ assert persisted_payload["terminalEnvelope"]["next_state"] != "completed"+++async def test_cancel_turn_finalizes_terminal_envelope_to_non_completed_state(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ chat = SimpleNamespace(+ organization_id="org-1",+ workflow_copilot_chat_id="chat-1",+ proposed_workflow=None,+ auto_accept=True,+ )+ original_workflow = SimpleNamespace(workflow_id="wf-canonical")+ agent_result = AgentResult(+ user_response="Cancelled by user.",+ updated_workflow=None,+ global_llm_context=None,+ response_type="REPLY",+ proposal_disposition="auto_applicable",+ cancelled=True,+ narrative_payload=_narrative_payload(),+ terminal_envelope=_terminal_payload(verified=True, workflow_applied=True),+ )+ _, workflow_params = setup_new_copilot_mocks(monkeypatch, chat, original_workflow, agent_result)+ stream = MagicMock(send=AsyncMock(return_value=True))++ await workflow_copilot_route._persist_cancel_turn(+ stream=stream,+ chat=chat,+ organization_id="org-1",+ original_workflow=original_workflow,+ user_message="stop",+ agent_result=agent_result,+ )++ response_frame = stream.send.await_args.args[0]+ assert isinstance(response_frame, WorkflowCopilotStreamResponseUpdate)+ assert response_frame.cancelled is True+ assert response_frame.terminal_envelope is not None+ assert response_frame.terminal_envelope["workflow_applied"] is False+ assert response_frame.terminal_envelope["next_state"] != "completed"+ assert response_frame.terminal_envelope["response_kind"] == "stopped"++ persisted_payload = workflow_params.create_workflow_copilot_chat_message.await_args_list[-1].kwargs[+ "narrative_payload"+ ]+ assert persisted_payload is not None+ assert persisted_payload["terminalEnvelope"]["workflow_applied"] is False+ assert persisted_payload["terminalEnvelope"]["next_state"] != "completed"+ assert persisted_payload["terminalEnvelope"]["response_kind"] == "stopped"++[email protected]("finalizer", ["normal", "cancel"], ids=["normal-turn", "cancel-turn"])+async def test_terminal_envelope_finalization_failure_omits_envelope(+ monkeypatch: pytest.MonkeyPatch,+ finalizer: str,+) -> None:+ chat = SimpleNamespace(+ organization_id="org-1",+ workflow_copilot_chat_id="chat-1",+ proposed_workflow=None,+ auto_accept=(finalizer == "cancel"),+ )+ original_workflow = SimpleNamespace(workflow_id="wf-canonical")+ updated_workflow = MagicMock()+ updated_workflow.model_dump.return_value = {"workflow_id": "wf-draft"}+ agent_result = AgentResult(+ user_response="done" if finalizer == "normal" else "Cancelled by user.",+ updated_workflow=updated_workflow if finalizer == "normal" else None,+ global_llm_context=None,+ response_type="REPLY",+ proposal_disposition="auto_applicable",+ cancelled=(finalizer == "cancel"),+ narrative_payload=_narrative_payload(),+ terminal_envelope=_terminal_payload(verified=True, workflow_applied=False),+ )+ _, workflow_params = setup_new_copilot_mocks(monkeypatch, chat, original_workflow, agent_result)+ stream = MagicMock(send=AsyncMock(return_value=True))++ def _raise(*_: Any, **__: Any) -> Any:+ raise RuntimeError("finalize failed")++ monkeypatch.setattr(workflow_copilot_route, "finalize_applied_state", _raise)++ if finalizer == "normal":+ await workflow_copilot_route._finalise_normal_turn(+ stream=stream,+ chat=chat,+ organization_id="org-1",+ original_workflow=original_workflow,+ chat_request=_make_chat_request(),+ agent_result=agent_result,+ )+ else:+ await workflow_copilot_route._persist_cancel_turn(+ stream=stream,+ chat=chat,+ organization_id="org-1",+ original_workflow=original_workflow,+ user_message="stop",+ agent_result=agent_result,+ )++ response_frame = stream.send.await_args.args[0]+ assert isinstance(response_frame, WorkflowCopilotStreamResponseUpdate)+ assert response_frame.terminal_envelope is None+ if finalizer == "cancel":+ assert response_frame.cancelled is True++ persisted_payload = workflow_params.create_workflow_copilot_chat_message.await_args_list[-1].kwargs[+ "narrative_payload"+ ]+ assert persisted_payload is not None+ assert "terminalEnvelope" not in persisted_payload++@pytest.mark.asyncioasync def test_get_debug_run_info_bounds_visible_elements_html(monkeypatch: pytest.MonkeyPatch) -> None:max_chars = workflow_copilot_route.WORKFLOW_COPILOT_DEBUG_HTML_MAX_CHARS