Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
SKY-10130 (2/3): populate TurnOutcome on every AgentResult + typed user-non-progress signal (#6149)
Andrew Neilson631f0fbca3b9
diff --git a/skyvern/forge/sdk/copilot/agent.py b/skyvern/forge/sdk/copilot/agent.pyindex b5ba8d845..e0596cbe5 100644--- a/skyvern/forge/sdk/copilot/agent.py+++ b/skyvern/forge/sdk/copilot/agent.py@@ -75,6 +75,12 @@ from skyvern.forge.sdk.copilot.turn_intent import (TurnIntentMode,build_turn_intent,)+from skyvern.forge.sdk.copilot.turn_outcome import (+ build_minimal_turn_outcome,+ build_turn_outcome,+ derive_response_kind,+)+from skyvern.forge.sdk.schemas.copilot_turn_outcome import ResponseKind, TurnOutcomefrom skyvern.forge.sdk.schemas.persistent_browser_sessions import is_final_statusfrom skyvern.forge.sdk.schemas.workflow_copilot import (WorkflowCopilotChatHistoryMessage,@@ -658,9 +664,15 @@ def _build_exit_result(user_response: str,global_llm_context: str | None,cancelled: bool = False,+ terminal_reason: str | None = None,) -> AgentResult:"""AgentResult for agent-loop exits that don't go through ``_translate_to_agent_result``."""verified_workflow, verified_yaml = _verified_workflow_or_none(ctx)+ outcome = build_minimal_turn_outcome(+ user_response,+ response_kind=ResponseKind.CLARIFY,+ terminal_reason=terminal_reason or ("cancel" if cancelled else None),+ )return AgentResult(user_response=user_response,updated_workflow=verified_workflow,@@ -669,6 +681,7 @@ def _build_exit_result(workflow_was_persisted=ctx.workflow_persisted,total_tokens=ctx.total_tokens_used,cancelled=cancelled,+ turn_outcome=outcome,)@@ -928,9 +941,19 @@ def _build_wip_exit_result(unvalidated_reply: str,tested_reply: str,cancelled: bool = False,+ terminal_reason: str | None = None,) -> AgentResult:"""Selected non-success exits surface the most recent successfully parsed workflow."""recorded_failure_reply = _recorded_failure_reply(ctx, cancelled=cancelled)+ effective_terminal = terminal_reason or ("cancel" if cancelled else None)++ def _outcome(text: str) -> TurnOutcome:+ return build_minimal_turn_outcome(+ text,+ response_kind=ResponseKind.CLARIFY,+ terminal_reason=effective_terminal,+ )+# When an unverified edit/run has overwritten ``last_workflow``, prefer the# verified shape while still forcing explicit review.if (@@ -939,8 +962,10 @@ def _build_wip_exit_result(and ctx.last_workflow is not ctx.last_good_workflowand not ctx.last_test_suspicious_success):+ reply = _last_good_failure_reply(ctx, tested_reply) if recorded_failure_reply else tested_reply+ outcome = _outcome(reply)return AgentResult(- user_response=_last_good_failure_reply(ctx, tested_reply) if recorded_failure_reply else tested_reply,+ user_response=reply,updated_workflow=ctx.last_good_workflow,global_llm_context=global_llm_context,workflow_yaml=ctx.last_good_workflow_yaml,@@ -948,6 +973,7 @@ def _build_wip_exit_result(total_tokens=ctx.total_tokens_used,proposal_disposition="review_tested",cancelled=cancelled,+ turn_outcome=outcome,)if (ctx.last_workflow is not None@@ -960,6 +986,7 @@ def _build_wip_exit_result(reply = recorded_failure_replyelse:reply = unvalidated_reply if unvalidated else tested_reply+ outcome = _outcome(reply)return AgentResult(user_response=reply,updated_workflow=ctx.last_workflow,@@ -969,8 +996,15 @@ def _build_wip_exit_result(total_tokens=ctx.total_tokens_used,proposal_disposition="review_untested" if unvalidated else "auto_applicable",cancelled=cancelled,+ turn_outcome=outcome,)- return _build_exit_result(ctx, recorded_failure_reply or default_reply, global_llm_context, cancelled=cancelled)+ return _build_exit_result(+ ctx,+ recorded_failure_reply or default_reply,+ global_llm_context,+ cancelled=cancelled,+ terminal_reason=effective_terminal,+ )def _merge_exit_context(@@ -990,6 +1024,7 @@ def _build_timeout_exit_result(ctx: CopilotContext, global_llm_context: str | Nodefault_reply=_TIMEOUT_REPLY_DEFAULT,unvalidated_reply=_TIMEOUT_REPLY_UNVALIDATED,tested_reply=_TIMEOUT_REPLY_TESTED,+ terminal_reason="timeout",)@@ -1007,6 +1042,7 @@ def _build_max_turns_exit_result(ctx: CopilotContext, global_llm_context: str |default_reply=_MAX_TURNS_REPLY_DEFAULT,unvalidated_reply=_MAX_TURNS_REPLY_UNVALIDATED,tested_reply=_MAX_TURNS_REPLY_TESTED,+ terminal_reason="max_turns",)@@ -1029,6 +1065,7 @@ def _build_unexpected_error_exit_result(default_reply=default_reply,unvalidated_reply=_UNEXPECTED_ERROR_REPLY_UNVALIDATED,tested_reply=_UNEXPECTED_ERROR_REPLY_TESTED,+ terminal_reason="unexpected_error",)LOG.warning("Copilot unexpected error translated to recoverable reply",@@ -1318,8 +1355,28 @@ def _translate_to_agent_result(output_policy_diagnostics=output_policy_diagnostics,)+ final_user_response = str(user_response)+ attempted_kind = derive_response_kind(ctx.turn_intent)+ tool_call_names = [+ str(entry.get("tool") or entry.get("name") or "") for entry in ctx.tool_activity if isinstance(entry, dict)+ ]+ reason_codes = (+ [code.value for code in ctx.turn_intent.reason_codes]+ if ctx.turn_intent and ctx.turn_intent.reason_codes+ else []+ )+ reason_code = ",".join(reason_codes)++ turn_outcome = build_turn_outcome(+ final_user_response,+ turn_intent=ctx.turn_intent,+ response_kind=attempted_kind,+ reason_code=reason_code,+ tool_calls=[name for name in tool_call_names if name],+ )+return AgentResult(- user_response=str(user_response),+ user_response=final_user_response,updated_workflow=last_workflow,global_llm_context=enriched_context or None,response_type=resp_type,@@ -1335,6 +1392,7 @@ def _translate_to_agent_result(else "auto_applicable"),output_policy_diagnostics=output_policy_diagnostics,+ turn_outcome=turn_outcome,)@@ -1367,6 +1425,11 @@ def _build_feasibility_clarification_result(workflow_yaml=prior_workflow_yaml or None,workflow_was_persisted=False,clear_proposed_workflow=True,+ turn_outcome=build_minimal_turn_outcome(+ question,+ response_kind=ResponseKind.CLARIFY,+ reason_code="feasibility_clarification",+ ),)@@ -1424,15 +1487,22 @@ def _build_request_policy_clarification_result(structured.decisions_made.append(f"request-policy clarification required: {policy.credential_input_kind}/{policy.clarification_reason}")+ clarification_text = (+ policy.clarification_question or "I need one more detail before I can build and test this workflow safely."+ )return AgentResult(- user_response=policy.clarification_question- or "I need one more detail before I can build and test this workflow safely.",+ user_response=clarification_text,updated_workflow=None,global_llm_context=structured.to_json_str(),response_type="ASK_QUESTION",workflow_yaml=prior_workflow_yaml or None,workflow_was_persisted=False,clear_proposed_workflow=True,+ turn_outcome=build_minimal_turn_outcome(+ clarification_text,+ response_kind=ResponseKind.CLARIFY,+ reason_code="request_policy_clarification",+ ),)@@ -1712,6 +1782,12 @@ def _build_output_policy_blocked_result(user_response = "I could not safely return that chat reply. Please adjust the request and try again."if preserved_workflow is not None and add_saved_draft_copy:user_response = f"{user_response} {_SAVED_DRAFT_OUTPUT_POLICY_SUFFIX}"+ output_policy_outcome = build_minimal_turn_outcome(+ user_response,+ response_kind=ResponseKind.CLARIFY,+ reason_code="output_policy_block",+ terminal_reason="output_policy_block",+ )return AgentResult(user_response=user_response,updated_workflow=preserved_workflow,@@ -1736,6 +1812,7 @@ def _build_output_policy_blocked_result(hard_block_reason_codes=list(verdict.reason_codes),soft_rewrite_reason_codes=[],),+ turn_outcome=output_policy_outcome,)@@ -1858,14 +1935,21 @@ async def _run_copilot_turn_impl(error=str(e),workflow_permanent_id=chat_request.workflow_permanent_id,)+ missing_sdk_reply = (+ "Copilot backend is missing the OpenAI Agents SDK dependency. "+ "Rebuild or redeploy the backend image so `openai-agents` is installed."+ )+ missing_sdk_outcome = build_minimal_turn_outcome(+ missing_sdk_reply,+ response_kind=ResponseKind.CLARIFY,+ terminal_reason="missing_sdk",+ )return AgentResult(- user_response=(- "Copilot backend is missing the OpenAI Agents SDK dependency. "- "Rebuild or redeploy the backend image so `openai-agents` is installed."- ),+ user_response=missing_sdk_reply,updated_workflow=None,global_llm_context=global_llm_context,workflow_yaml=chat_request.workflow_yaml or None,+ turn_outcome=missing_sdk_outcome,)raise@@ -2169,16 +2253,23 @@ async def _run_copilot_turn_impl()# Non-retriable nav errors prove the current workflow doesn't# work; zero the proposal even if other tools succeeded.+ nav_reply = (+ f"The target URL could not be reached. Error: {exc.error_message}. "+ "Please verify the URL and try again."+ )+ nav_outcome = build_minimal_turn_outcome(+ nav_reply,+ response_kind=ResponseKind.CLARIFY,+ terminal_reason="non_retriable_nav",+ )return AgentResult(- user_response=(- f"The target URL could not be reached. Error: {exc.error_message}. "- "Please verify the URL and try again."- ),+ user_response=nav_reply,updated_workflow=None,global_llm_context=global_llm_context,workflow_yaml=None,workflow_was_persisted=ctx.workflow_persisted,total_tokens=ctx.total_tokens_used,+ turn_outcome=nav_outcome,)except Exception as e:LOG.error("Copilot agent error", error=str(e), exc_info=True)diff --git a/skyvern/forge/sdk/copilot/context.py b/skyvern/forge/sdk/copilot/context.pyindex 977bef02c..079ce36f3 100644--- a/skyvern/forge/sdk/copilot/context.py+++ b/skyvern/forge/sdk/copilot/context.py@@ -21,6 +21,7 @@ if TYPE_CHECKING:from skyvern.forge.sdk.copilot.request_policy import RequestPolicyfrom skyvern.forge.sdk.copilot.turn_context import TurnContextPacketfrom skyvern.forge.sdk.copilot.turn_intent import TurnIntent+ from skyvern.forge.sdk.schemas.copilot_turn_outcome import TurnOutcomeclass UrlVisit(BaseModel):@@ -135,6 +136,7 @@ class AgentResult:# Controls whether the route may auto-apply the proposal or must force explicit review.proposal_disposition: ProposalDisposition = "auto_applicable"output_policy_diagnostics: dict[str, Any] | None = None+ turn_outcome: TurnOutcome | None = None@dataclassdiff --git a/skyvern/forge/sdk/copilot/signature.py b/skyvern/forge/sdk/copilot/signature.pynew file mode 100644index 000000000..bd3d610ae--- /dev/null+++ b/skyvern/forge/sdk/copilot/signature.py@@ -0,0 +1,45 @@+"""Bounded, deterministic reply signatures for cross-turn loop detection.++The signature is computed over the user-visible reply (post+``normalize_response_scaffolding``), with variable spans (URLs, Skyvern IDs,+datetimes, numbers, backtick-quoted identifiers) stripped so two replies that+differ only by run-specific values share a signature.+"""++from __future__ import annotations++import hashlib+import re++from skyvern.forge.sdk.copilot.request_policy import redact_raw_secrets_for_prompt++RAW_SLICE_CAP = 8_000+COMPARE_CAP = 4_000+_DIGEST_BYTES = 8++_WHITESPACE_RE = re.compile(r"\s+")+_URL_RE = re.compile(r"https?://\S+", re.IGNORECASE)+_SKYVERN_ID_RE = re.compile(+ r"\b(?:wpid|wcc|wccm|wr|w|tsk|o|block|step|cred|wf)_[A-Za-z0-9]+",+ re.IGNORECASE,+)+_ISO_DATETIME_RE = re.compile(r"\b\d{4}-\d{2}-\d{2}(?:[ T]\d{2}:\d{2}(?::\d{2}(?:\.\d+)?)?(?:Z|[+\-]\d{2}:?\d{2})?)?\b")+_NUMERIC_RUN_RE = re.compile(r"\b\d{3,}\b")+_BACKTICK_IDENT_RE = re.compile(r"`[A-Za-z][A-Za-z0-9_]*`")+++def normalize_for_compare(text: str) -> str:+ raw = text or ""+ if len(raw) > RAW_SLICE_CAP:+ half = RAW_SLICE_CAP // 2+ raw = raw[:half] + raw[-half:]+ redacted = redact_raw_secrets_for_prompt(raw)+ for pattern in (_URL_RE, _SKYVERN_ID_RE, _ISO_DATETIME_RE, _BACKTICK_IDENT_RE, _NUMERIC_RUN_RE):+ redacted = pattern.sub(" ", redacted)+ collapsed = _WHITESPACE_RE.sub(" ", redacted).strip().lower()+ return collapsed[:COMPARE_CAP]+++def compute_signature(text: str) -> str:+ digest = hashlib.blake2b(normalize_for_compare(text).encode("utf-8"), digest_size=_DIGEST_BYTES)+ return digest.hexdigest()diff --git a/skyvern/forge/sdk/copilot/turn_intent.py b/skyvern/forge/sdk/copilot/turn_intent.pyindex 4de3595f4..d2e490098 100644--- a/skyvern/forge/sdk/copilot/turn_intent.py+++ b/skyvern/forge/sdk/copilot/turn_intent.py@@ -2,6 +2,7 @@ from __future__ import annotationsimport functoolsimport re+from difflib import SequenceMatcherfrom enum import StrEnumfrom typing import Any@@ -71,6 +72,7 @@ class TurnIntentReasonCode(StrEnum):KEYWORD_HEURISTIC = "keyword_heuristic"CONFIRMATION_CARRYOVER = "confirmation_carryover"RAW_SECRET_REFUSAL = "raw_secret_refusal"+ USER_NON_PROGRESS = "user_non_progress"class TurnIntentAuthority(BaseModel):@@ -334,6 +336,65 @@ def _is_bare_affirmative(user_message: str) -> bool:return normalized in _AFFIRMATIVE_REPLIES+_NON_PROGRESS_MARKER_RE = re.compile(+ r"\b(?:"+ r"i\s+can'?t\s+see\s+(?:it|that|them|those|the)"+ r"|still\s+(?:not|isn'?t|doesn'?t)\s+working"+ r"|still\s+(?:nothing|nope|none|no\s+luck)"+ r"|(?:still|it|that|this)\s+doesn'?t\s+work\b"+ r"|same\s+(?:problem|issue|answer|reply)"+ r"|didn'?t\s+(?:work|help)"+ r"|doesn'?t\s+help"+ r"|(?:still\s+)?can'?t\s+find\s+(?:it|that|them|those|the)\b"+ r"|where\s+(?:is|are|'?s)\s+(?:it|that|they|them)\b"+ r"|i\s+already\s+looked\b"+ r"|i\s+looked\s+(?:everywhere|already|but|and)"+ r"|not\s+(?:there|here|working)"+ r"|no\s+luck"+ r"|that\s+didn'?t\s+(?:work|help)"+ r")\b",+ re.IGNORECASE,+)+_NON_PROGRESS_SHORT_LIMIT = 80+_NON_PROGRESS_RESTATE_THRESHOLD = 0.7+++def _user_signals_non_progress(user_message: str, chat_history: list[WorkflowCopilotChatHistoryMessage]) -> bool:+ """True when the current user turn re-engages on the same problem.++ Requires at least one prior AI reply — without that, marker phrases like+ "where is my X" on a first turn are genuine questions, not non-progress+ restatements of a stuck conversation.+ """+ text = (user_message or "").strip()+ if not text:+ return False+ has_prior_ai_reply = any(+ m.sender == WorkflowCopilotChatSender.AI and (m.content or "").strip() for m in chat_history+ )+ if not has_prior_ai_reply:+ return False+ if _NON_PROGRESS_MARKER_RE.search(text):+ return True+ if len(text) > _NON_PROGRESS_SHORT_LIMIT:+ return False+ prior = next(+ (+ (m.content or "").strip()+ for m in reversed(chat_history)+ if m.sender == WorkflowCopilotChatSender.USER and (m.content or "").strip()+ ),+ "",+ )+ if not prior:+ return False+ current_norm = re.sub(r"\s+", " ", text.lower()).strip()+ prior_norm = re.sub(r"\s+", " ", prior.lower()).strip()+ if not current_norm or not prior_norm:+ return False+ return SequenceMatcher(None, current_norm, prior_norm).ratio() >= _NON_PROGRESS_RESTATE_THRESHOLD++def _carryover_mode_from_prior_turn(chat_history: list[WorkflowCopilotChatHistoryMessage], *, has_workflow: bool) -> tuple[TurnIntentMode, TurnIntentExpectedOutput] | None:@@ -434,6 +495,9 @@ def build_turn_intent(confidence = 0.3reason_codes.append(TurnIntentReasonCode.CONFIRMATION_CARRYOVER)+ if _user_signals_non_progress(user_message, chat_history):+ reason_codes.append(TurnIntentReasonCode.USER_NON_PROGRESS)+if mode == TurnIntentMode.EDIT:unresolved_block_refs = _unresolved_explicit_block_refs(user_message, workflow_yaml)if unresolved_block_refs:diff --git a/skyvern/forge/sdk/copilot/turn_outcome.py b/skyvern/forge/sdk/copilot/turn_outcome.pynew file mode 100644index 000000000..c9b970678--- /dev/null+++ b/skyvern/forge/sdk/copilot/turn_outcome.py@@ -0,0 +1,106 @@+"""Builders + closed `TurnIntentMode -> ResponseKind` mapping for `TurnOutcome`.++Schema types live in ``schemas/copilot_turn_outcome.py`` so chat-history+schemas can embed ``TurnOutcome`` without importing copilot business logic.+This module imports both the schema types and ``TurnIntentMode`` — the only+direction allowed.+"""++from __future__ import annotations++from collections.abc import Iterable+from typing import Any++import structlog++from skyvern.forge.sdk.copilot.signature import compute_signature+from skyvern.forge.sdk.copilot.turn_intent import TurnIntent, TurnIntentMode+from skyvern.forge.sdk.schemas.copilot_turn_outcome import ResponseKind, TurnOutcome++LOG = structlog.get_logger()++_RESPONSE_KIND_BY_MODE: dict[TurnIntentMode, ResponseKind] = {+ TurnIntentMode.BUILD: ResponseKind.BUILD,+ TurnIntentMode.EDIT: ResponseKind.BUILD,+ TurnIntentMode.DRAFT_ONLY: ResponseKind.BUILD,+ TurnIntentMode.CLARIFY: ResponseKind.CLARIFY,+ TurnIntentMode.UNKNOWN: ResponseKind.CLARIFY,+ TurnIntentMode.DIAGNOSE: ResponseKind.DIAGNOSE,+ TurnIntentMode.DOCS_ANSWER: ResponseKind.DIAGNOSE,+ TurnIntentMode.REFUSE: ResponseKind.REFUSE,+}++# Catches the "added a TurnIntentMode but forgot to map it" foot-gun at import+# time rather than letting the new mode silently fall through to CLARIFY.+# Raises explicitly (not ``assert``) so the guard survives ``python -O``.+_missing_modes = set(TurnIntentMode) - set(_RESPONSE_KIND_BY_MODE)+if _missing_modes:+ raise RuntimeError(f"_RESPONSE_KIND_BY_MODE missing entries for: {sorted(m.value for m in _missing_modes)}")+++def derive_response_kind(turn_intent: TurnIntent | None) -> ResponseKind:+ """Closed mapping. ``RECOVER`` is set only by the enforcement guard."""+ mode = getattr(turn_intent, "mode", None)+ if isinstance(mode, TurnIntentMode):+ return _RESPONSE_KIND_BY_MODE[mode]+ return ResponseKind.CLARIFY+++def _dedup_signatures(signatures: Iterable[str]) -> list[str]:+ return sorted({sig for sig in signatures if isinstance(sig, str) and sig})+++def build_minimal_turn_outcome(+ final_text: str,+ response_kind: ResponseKind,+ reason_code: str = "",+ terminal_reason: str | None = None,+ inherited_blocked_signatures: Iterable[str] = (),+) -> TurnOutcome:+ """Used by every direct-return ``AgentResult`` site so the persisted AI row+ always carries a ``turn_outcome``. Callers that need ban-set inheritance+ pass it in via ``inherited_blocked_signatures``; the route-level+ ``apply_repeated_reply_guard`` is the typical source."""+ return TurnOutcome(+ response_kind=response_kind,+ reason_code=reason_code,+ normalized_reply_signature=compute_signature(final_text),+ terminal_reason=terminal_reason,+ blocked_signatures=_dedup_signatures(inherited_blocked_signatures),+ )+++def build_turn_outcome(+ final_text: str,+ *,+ turn_intent: TurnIntent | None,+ response_kind: ResponseKind | None = None,+ reason_code: str = "",+ tool_calls: Iterable[str] = (),+ terminal_reason: str | None = None,+ inherited_blocked_signatures: Iterable[str] = (),+ extra_blocked_signatures: Iterable[str] = (),+) -> TurnOutcome:+ """Used by the translation path. Resolves ``response_kind`` from the turn+ intent when not supplied; merges inherited + extra blocked signatures so+ the enforcement guard can record the original signature it just blocked."""+ resolved_kind = response_kind if response_kind is not None else derive_response_kind(turn_intent)+ intent_summary: dict[str, Any] = {}+ if turn_intent is not None:+ try:+ intent_summary = dict(turn_intent.to_trace_data())+ except Exception as exc:+ LOG.warning(+ "Failed to serialize TurnIntent trace data for TurnOutcome; using empty dict",+ exc_info=exc,+ )+ intent_summary = {}+ return TurnOutcome(+ turn_intent_summary=intent_summary,+ response_kind=resolved_kind,+ reason_code=reason_code,+ normalized_reply_signature=compute_signature(final_text),+ tool_calls=[str(call) for call in tool_calls if call],+ terminal_reason=terminal_reason,+ blocked_signatures=_dedup_signatures(list(inherited_blocked_signatures) + list(extra_blocked_signatures)),+ )diff --git a/skyvern/forge/sdk/routes/workflow_copilot.py b/skyvern/forge/sdk/routes/workflow_copilot.pyindex f374f8ef0..ed489b3f9 100644--- a/skyvern/forge/sdk/routes/workflow_copilot.py+++ b/skyvern/forge/sdk/routes/workflow_copilot.py@@ -38,6 +38,7 @@ from skyvern.forge.sdk.core import skyvern_contextfrom skyvern.forge.sdk.experimentation.llm_prompt_config import get_llm_handler_for_prompt_typefrom skyvern.forge.sdk.routes.event_source_stream import EventSourceStream, FastAPIEventSourceStreamfrom skyvern.forge.sdk.routes.routers import base_router+from skyvern.forge.sdk.schemas.copilot_turn_outcome import TurnOutcomefrom skyvern.forge.sdk.schemas.organizations import Organizationfrom skyvern.forge.sdk.schemas.workflow_copilot import (WorkflowCopilotApplyProposedWorkflowRequest,@@ -388,6 +389,7 @@ async def _persist_cancel_turn(rollback uses the same ``workflow_was_persisted`` source of truth asthe success path; pass ``None`` for pre-agent cancels."""+ turn_outcome: TurnOutcome | Noneif agent_result is None:user_response = "Cancelled by user."updated_workflow = None@@ -395,6 +397,7 @@ async def _persist_cancel_turn(total_tokens = Noneresponse_type = "REPLY"output_policy_diagnostics = None+ turn_outcome = Noneif chat.proposed_workflow is not None:await asyncio.shield(_clear_proposed_workflow(chat))else:@@ -411,6 +414,7 @@ async def _persist_cancel_turn(total_tokens = getattr(agent_result, "total_tokens", None)response_type = getattr(agent_result, "response_type", "REPLY")output_policy_diagnostics = getattr(agent_result, "output_policy_diagnostics", None)+ turn_outcome = agent_result.turn_outcomeawait asyncio.shield(app.DATABASE.workflow_params.create_workflow_copilot_chat_message(@@ -427,6 +431,7 @@ async def _persist_cancel_turn(sender=WorkflowCopilotChatSender.AI,content=user_response,global_llm_context=updated_global_llm_context,+ turn_outcome=turn_outcome,))proposal_disposition = _proposal_disposition(agent_result)@@ -504,6 +509,7 @@ async def _finalise_normal_turn(sender=WorkflowCopilotChatSender.AI,content=user_response,global_llm_context=updated_global_llm_context,+ turn_outcome=agent_result.turn_outcome,)proposal_disposition = _proposal_disposition(agent_result)diff --git a/tests/unit/test_copilot_signature.py b/tests/unit/test_copilot_signature.pynew file mode 100644index 000000000..91c1b0c4e--- /dev/null+++ b/tests/unit/test_copilot_signature.py@@ -0,0 +1,54 @@+from __future__ import annotations++from skyvern.forge.sdk.copilot.signature import (+ COMPARE_CAP,+ compute_signature,+ normalize_for_compare,+)+++def test_normalize_strips_urls() -> None:+ a = normalize_for_compare("See https://example.com/a/b for details.")+ b = normalize_for_compare("See https://other.example.org/x for details.")+ assert a == b+++def test_normalize_strips_skyvern_ids() -> None:+ a = normalize_for_compare("Block `block_1` failed during run wr_abc123.")+ b = normalize_for_compare("Block `block_44` failed during run wr_xyz999.")+ assert a == b+++def test_normalize_strips_numbers_and_dates() -> None:+ a = normalize_for_compare("Run took 1234 ms and completed at 2026-05-22T10:00:00Z.")+ b = normalize_for_compare("Run took 9876 ms and completed at 2026-05-21T18:30:00Z.")+ assert a == b+++def test_normalize_caps_to_compare_cap() -> None:+ body = "a " * 10_000+ assert len(normalize_for_compare(body)) <= COMPARE_CAP+++def test_signature_matches_for_variant_inputs() -> None:+ a = "The file is in the Artifacts section. URL: https://example.com/foo/123"+ b = "The file is in the Artifacts section. URL: https://other.io/baz/9999"+ assert compute_signature(a) == compute_signature(b)+++def test_signature_differs_for_distinct_text() -> None:+ a = "The file is in the Artifacts section."+ b = "I have emailed the downloaded file to your account."+ assert compute_signature(a) != compute_signature(b)+++def test_signature_is_16_hex_chars() -> None:+ sig = compute_signature("hello")+ assert len(sig) == 16+ assert all(c in "0123456789abcdef" for c in sig)+++def test_normalize_redacts_secrets_before_compare() -> None:+ a = normalize_for_compare("Use password: hunter2abc to sign in.")+ b = normalize_for_compare("Use password: secret9999 to sign in.")+ assert a == bdiff --git a/tests/unit/test_copilot_turn_intent.py b/tests/unit/test_copilot_turn_intent.pyindex 8c6928292..649ff8e18 100644--- a/tests/unit/test_copilot_turn_intent.py+++ b/tests/unit/test_copilot_turn_intent.py@@ -408,6 +408,82 @@ def test_build_turn_intent_omits_workflow_change_on_first_turn() -> None:assert RequiredContextKey.WORKFLOW_CHANGE not in intent.required_context+def _ai_message(text: str) -> WorkflowCopilotChatHistoryMessage:+ return WorkflowCopilotChatHistoryMessage(+ sender=WorkflowCopilotChatSender.AI,+ content=text,+ created_at=datetime.now(timezone.utc),+ )+++def test_user_non_progress_fires_on_stuck_marker() -> None:+ intent = build_turn_intent(+ user_message="i can't see it",+ workflow_yaml="",+ chat_history=[_user_message("where is my downloaded file"), _ai_message("Check the Artifacts panel.")],+ global_llm_context="",+ request_policy=RequestPolicy(),+ )+ assert TurnIntentReasonCode.USER_NON_PROGRESS in intent.reason_codes+++def test_user_non_progress_fires_on_short_restatement() -> None:+ intent = build_turn_intent(+ user_message="where's my downloaded file?",+ workflow_yaml="",+ chat_history=[_user_message("where is my downloaded file"), _ai_message("Check the Artifacts panel.")],+ global_llm_context="",+ request_policy=RequestPolicy(),+ )+ assert TurnIntentReasonCode.USER_NON_PROGRESS in intent.reason_codes+++def test_user_non_progress_does_not_fire_on_topic_switch_after_ai_reply() -> None:+ """Regression: 'where is my X' is a topic switch, not non-progress, even with prior AI."""+ intent = build_turn_intent(+ user_message="where is my API key stored?",+ workflow_yaml="",+ chat_history=[_user_message("build me a workflow"), _ai_message("Drafted v1.")],+ global_llm_context="",+ request_policy=RequestPolicy(),+ )+ assert TurnIntentReasonCode.USER_NON_PROGRESS not in intent.reason_codes+++def test_user_non_progress_does_not_fire_on_first_turn_even_when_marker_matches() -> None:+ """Regression: a first-turn 'where is my X' is a real question, not non-progress."""+ intent = build_turn_intent(+ user_message="where is my API key stored?",+ workflow_yaml="",+ chat_history=[],+ global_llm_context="",+ request_policy=RequestPolicy(),+ )+ assert TurnIntentReasonCode.USER_NON_PROGRESS not in intent.reason_codes+++def test_user_non_progress_does_not_fire_on_topic_switch() -> None:+ intent = build_turn_intent(+ user_message="now build me a workflow for invoicing",+ workflow_yaml="",+ chat_history=[_user_message("where is my downloaded file"), _ai_message("Check the Artifacts panel.")],+ global_llm_context="",+ request_policy=RequestPolicy(),+ )+ assert TurnIntentReasonCode.USER_NON_PROGRESS not in intent.reason_codes+++def test_user_non_progress_does_not_fire_on_legitimate_followup() -> None:+ intent = build_turn_intent(+ user_message="where is the documentation for loop blocks?",+ workflow_yaml="",+ chat_history=[_user_message("build me a workflow"), _ai_message("Drafted v1.")],+ global_llm_context="",+ request_policy=RequestPolicy(),+ )+ assert TurnIntentReasonCode.USER_NON_PROGRESS not in intent.reason_codes++def test_store_request_policy_attaches_turn_intent_to_context() -> None:ctx = CopilotContext(organization_id="org-1",diff --git a/tests/unit/test_copilot_turn_outcome.py b/tests/unit/test_copilot_turn_outcome.pynew file mode 100644index 000000000..132f53809--- /dev/null+++ b/tests/unit/test_copilot_turn_outcome.py@@ -0,0 +1,74 @@+from __future__ import annotations++import pytest++from skyvern.forge.sdk.copilot.signature import compute_signature+from skyvern.forge.sdk.copilot.turn_intent import TurnIntent, TurnIntentMode+from skyvern.forge.sdk.copilot.turn_outcome import (+ build_minimal_turn_outcome,+ build_turn_outcome,+ derive_response_kind,+)+from skyvern.forge.sdk.schemas.copilot_turn_outcome import ResponseKind, TurnOutcome+++ "mode,expected",+ [+ (TurnIntentMode.BUILD, ResponseKind.BUILD),+ (TurnIntentMode.EDIT, ResponseKind.BUILD),+ (TurnIntentMode.DRAFT_ONLY, ResponseKind.BUILD),+ (TurnIntentMode.CLARIFY, ResponseKind.CLARIFY),+ (TurnIntentMode.UNKNOWN, ResponseKind.CLARIFY),+ (TurnIntentMode.DIAGNOSE, ResponseKind.DIAGNOSE),+ (TurnIntentMode.DOCS_ANSWER, ResponseKind.DIAGNOSE),+ (TurnIntentMode.REFUSE, ResponseKind.REFUSE),+ ],+)+def test_derive_response_kind_covers_every_mode(mode: TurnIntentMode, expected: ResponseKind) -> None:+ intent = TurnIntent(mode=mode)+ assert derive_response_kind(intent) is expected+++def test_derive_response_kind_handles_none_intent() -> None:+ assert derive_response_kind(None) is ResponseKind.CLARIFY+++def test_build_minimal_turn_outcome_sets_signature_and_inherited() -> None:+ outcome = build_minimal_turn_outcome(+ "The file is in the Artifacts section.",+ response_kind=ResponseKind.DIAGNOSE,+ reason_code="diagnose_reply",+ terminal_reason=None,+ inherited_blocked_signatures=["aaaa", "bbbb", "aaaa"],+ )+ assert outcome.response_kind is ResponseKind.DIAGNOSE+ assert outcome.normalized_reply_signature == compute_signature("The file is in the Artifacts section.")+ assert outcome.blocked_signatures == ["aaaa", "bbbb"]+ assert outcome.reason_code == "diagnose_reply"+++def test_build_turn_outcome_merges_inherited_and_extra() -> None:+ intent = TurnIntent(mode=TurnIntentMode.BUILD)+ outcome = build_turn_outcome(+ "drafted v1",+ turn_intent=intent,+ tool_calls=["update_workflow", "run_blocks_and_collect_debug"],+ inherited_blocked_signatures=["aaaa"],+ extra_blocked_signatures=["bbbb"],+ )+ assert outcome.response_kind is ResponseKind.BUILD+ assert outcome.tool_calls == ["update_workflow", "run_blocks_and_collect_debug"]+ assert outcome.blocked_signatures == ["aaaa", "bbbb"]+ assert "mode" in outcome.turn_intent_summary+++def test_turn_outcome_json_round_trip() -> None:+ outcome = build_minimal_turn_outcome(+ "answer",+ response_kind=ResponseKind.CLARIFY,+ inherited_blocked_signatures=["xyz"],+ )+ payload = outcome.model_dump(mode="json")+ restored = TurnOutcome.model_validate(payload)+ assert restored == outcomediff --git a/tests/unit/test_workflow_copilot_cancel.py b/tests/unit/test_workflow_copilot_cancel.pyindex c80f36079..c35306e92 100644--- a/tests/unit/test_workflow_copilot_cancel.py+++ b/tests/unit/test_workflow_copilot_cancel.py@@ -302,6 +302,7 @@ async def test_route_cancel_branch_persists_user_and_cancelled_messages(cancelled=True,total_tokens=None,response_type="REPLY",+ turn_outcome=None,)restore_mock, workflow_params, sent_payloads = await _drive_cancel_route(monkeypatch, chat, original_workflow, agent_result@@ -352,6 +353,7 @@ async def test_route_cancel_branch_persists_wip_proposal_and_response_frame(total_tokens=123,response_type="REPLY",proposal_disposition="review_untested",+ turn_outcome=None,)restore_mock, workflow_params, sent_payloads = await _drive_cancel_route(monkeypatch, chat, original_workflow, agent_result@@ -407,6 +409,7 @@ async def test_route_cancel_tested_wip_with_auto_accept_still_persists_proposal(total_tokens=123,response_type="REPLY",proposal_disposition="auto_applicable",+ turn_outcome=None,)restore_mock, workflow_params, sent_payloads = await _drive_cancel_route(monkeypatch, chat, original_workflow, agent_result@@ -450,6 +453,7 @@ async def test_route_cancel_review_tested_persists_proposal_without_unvalidated_total_tokens=456,response_type="REPLY",proposal_disposition="review_tested",+ turn_outcome=None,)restore_mock, workflow_params, sent_payloads = await _drive_cancel_route(monkeypatch, chat, original_workflow, agent_result@@ -495,6 +499,7 @@ async def test_route_cancel_clears_stale_proposed_workflow_when_no_wip(total_tokens=None,response_type="REPLY",proposal_disposition="auto_applicable",+ turn_outcome=None,)restore_mock, workflow_params, _sent = await _drive_cancel_route(monkeypatch, chat, original_workflow, agent_result)@@ -524,6 +529,7 @@ async def test_route_cancel_clears_stale_proposed_workflow_when_no_wip_and_no_petotal_tokens=None,response_type="REPLY",proposal_disposition="auto_applicable",+ turn_outcome=None,)restore_mock, workflow_params, _sent = await _drive_cancel_route(monkeypatch, chat, original_workflow, agent_result)@@ -602,6 +608,7 @@ async def test_timeout_wip_result_streams_normal_response_frame(total_tokens=123,response_type="REPLY",proposal_disposition="review_untested",+ turn_outcome=None,)restore_mock, workflow_params = _setup_route_mocks(monkeypatch, chat, original_workflow, agent_result)@@ -679,6 +686,7 @@ async def test_timeout_wip_review_tested_propagates_to_response_frame(total_tokens=789,response_type="REPLY",proposal_disposition="review_tested",+ turn_outcome=None,)restore_mock, workflow_params = _setup_route_mocks(monkeypatch, chat, original_workflow, agent_result)diff --git a/tests/unit/test_workflow_copilot_route.py b/tests/unit/test_workflow_copilot_route.pyindex d5f680b5a..5f3ec8d36 100644--- a/tests/unit/test_workflow_copilot_route.py+++ b/tests/unit/test_workflow_copilot_route.py@@ -237,6 +237,7 @@ async def test_flag_on_mid_stream_disconnect_restores_when_persisted_and_not_autworkflow_was_persisted=workflow_was_persisted,clear_proposed_workflow=False,proposal_disposition="auto_applicable",+ turn_outcome=None,)restore_mock = _setup_new_copilot_mocks(monkeypatch, chat, original_workflow, agent_result)@@ -389,6 +390,7 @@ async def test_flag_on_route_error_after_chat_persists_recoverable_reply(workflow_was_persisted=False,clear_proposed_workflow=False,unvalidated=False,+ turn_outcome=None,)_setup_new_copilot_mocks(monkeypatch, chat, original_workflow, agent_result)monkeypatch.setattr(@@ -455,6 +457,7 @@ async def test_route_error_after_restore_reports_workflow_not_modified(workflow_was_persisted=True,clear_proposed_workflow=False,unvalidated=False,+ turn_outcome=None,)restore_mock = _setup_new_copilot_mocks(monkeypatch, chat, original_workflow, agent_result)finalise_results: list[object] = []@@ -580,6 +583,7 @@ async def test_proposed_workflow_cleared_on_restore(workflow_was_persisted=workflow_was_persisted,clear_proposed_workflow=clear_proposed_flag,proposal_disposition="auto_applicable",+ turn_outcome=None,)_setup_new_copilot_mocks(monkeypatch, chat, original_workflow, agent_result)@@ -652,6 +656,7 @@ async def test_output_policy_block_preserves_unvalidated_prior_proposal_under_au"final_output_policy_allowed": False,"hard_block_reason_codes": ["internal_tool_instruction_leak"],},+ turn_outcome=None,)_setup_new_copilot_mocks(monkeypatch, chat, original_workflow, agent_result)@@ -715,6 +720,7 @@ async def test_unvalidated_timeout_wip_overrides_auto_accept(monkeypatch: pytest"raw_would_have_failed": True,"contained_failure": True,},+ turn_outcome=None,)restore_mock = _setup_new_copilot_mocks(monkeypatch, chat, original_workflow, agent_result)