Skip to content

Automate browser based workflows with AI

SKY-10130 (2/3): populate TurnOutcome on every AgentResult + typed user-non-progress signal (#6149)

Andrew Neilson631f0fbca3b9
diff --git a/skyvern/forge/sdk/copilot/agent.py b/skyvern/forge/sdk/copilot/agent.py
index b5ba8d845..e0596cbe5 100644
--- a/skyvern/forge/sdk/copilot/agent.py
+++ b/skyvern/forge/sdk/copilot/agent.py
@@ -75,6 +75,12 @@ from skyvern.forge.sdk.copilot.turn_intent import (
TurnIntentMode,
build_turn_intent,
)
+from skyvern.forge.sdk.copilot.turn_outcome import (
+ build_minimal_turn_outcome,
+ build_turn_outcome,
+ derive_response_kind,
+)
+from skyvern.forge.sdk.schemas.copilot_turn_outcome import ResponseKind, TurnOutcome
from skyvern.forge.sdk.schemas.persistent_browser_sessions import is_final_status
from skyvern.forge.sdk.schemas.workflow_copilot import (
WorkflowCopilotChatHistoryMessage,
@@ -658,9 +664,15 @@ def _build_exit_result(
user_response: str,
global_llm_context: str | None,
cancelled: bool = False,
+ terminal_reason: str | None = None,
) -> AgentResult:
"""AgentResult for agent-loop exits that don't go through ``_translate_to_agent_result``."""
verified_workflow, verified_yaml = _verified_workflow_or_none(ctx)
+ outcome = build_minimal_turn_outcome(
+ user_response,
+ response_kind=ResponseKind.CLARIFY,
+ terminal_reason=terminal_reason or ("cancel" if cancelled else None),
+ )
return AgentResult(
user_response=user_response,
updated_workflow=verified_workflow,
@@ -669,6 +681,7 @@ def _build_exit_result(
workflow_was_persisted=ctx.workflow_persisted,
total_tokens=ctx.total_tokens_used,
cancelled=cancelled,
+ turn_outcome=outcome,
)
@@ -928,9 +941,19 @@ def _build_wip_exit_result(
unvalidated_reply: str,
tested_reply: str,
cancelled: bool = False,
+ terminal_reason: str | None = None,
) -> AgentResult:
"""Selected non-success exits surface the most recent successfully parsed workflow."""
recorded_failure_reply = _recorded_failure_reply(ctx, cancelled=cancelled)
+ effective_terminal = terminal_reason or ("cancel" if cancelled else None)
+
+ def _outcome(text: str) -> TurnOutcome:
+ return build_minimal_turn_outcome(
+ text,
+ response_kind=ResponseKind.CLARIFY,
+ terminal_reason=effective_terminal,
+ )
+
# When an unverified edit/run has overwritten ``last_workflow``, prefer the
# verified shape while still forcing explicit review.
if (
@@ -939,8 +962,10 @@ def _build_wip_exit_result(
and ctx.last_workflow is not ctx.last_good_workflow
and not ctx.last_test_suspicious_success
):
+ reply = _last_good_failure_reply(ctx, tested_reply) if recorded_failure_reply else tested_reply
+ outcome = _outcome(reply)
return AgentResult(
- user_response=_last_good_failure_reply(ctx, tested_reply) if recorded_failure_reply else tested_reply,
+ user_response=reply,
updated_workflow=ctx.last_good_workflow,
global_llm_context=global_llm_context,
workflow_yaml=ctx.last_good_workflow_yaml,
@@ -948,6 +973,7 @@ def _build_wip_exit_result(
total_tokens=ctx.total_tokens_used,
proposal_disposition="review_tested",
cancelled=cancelled,
+ turn_outcome=outcome,
)
if (
ctx.last_workflow is not None
@@ -960,6 +986,7 @@ def _build_wip_exit_result(
reply = recorded_failure_reply
else:
reply = unvalidated_reply if unvalidated else tested_reply
+ outcome = _outcome(reply)
return AgentResult(
user_response=reply,
updated_workflow=ctx.last_workflow,
@@ -969,8 +996,15 @@ def _build_wip_exit_result(
total_tokens=ctx.total_tokens_used,
proposal_disposition="review_untested" if unvalidated else "auto_applicable",
cancelled=cancelled,
+ turn_outcome=outcome,
)
- return _build_exit_result(ctx, recorded_failure_reply or default_reply, global_llm_context, cancelled=cancelled)
+ return _build_exit_result(
+ ctx,
+ recorded_failure_reply or default_reply,
+ global_llm_context,
+ cancelled=cancelled,
+ terminal_reason=effective_terminal,
+ )
def _merge_exit_context(
@@ -990,6 +1024,7 @@ def _build_timeout_exit_result(ctx: CopilotContext, global_llm_context: str | No
default_reply=_TIMEOUT_REPLY_DEFAULT,
unvalidated_reply=_TIMEOUT_REPLY_UNVALIDATED,
tested_reply=_TIMEOUT_REPLY_TESTED,
+ terminal_reason="timeout",
)
@@ -1007,6 +1042,7 @@ def _build_max_turns_exit_result(ctx: CopilotContext, global_llm_context: str |
default_reply=_MAX_TURNS_REPLY_DEFAULT,
unvalidated_reply=_MAX_TURNS_REPLY_UNVALIDATED,
tested_reply=_MAX_TURNS_REPLY_TESTED,
+ terminal_reason="max_turns",
)
@@ -1029,6 +1065,7 @@ def _build_unexpected_error_exit_result(
default_reply=default_reply,
unvalidated_reply=_UNEXPECTED_ERROR_REPLY_UNVALIDATED,
tested_reply=_UNEXPECTED_ERROR_REPLY_TESTED,
+ terminal_reason="unexpected_error",
)
LOG.warning(
"Copilot unexpected error translated to recoverable reply",
@@ -1318,8 +1355,28 @@ def _translate_to_agent_result(
output_policy_diagnostics=output_policy_diagnostics,
)
+ final_user_response = str(user_response)
+ attempted_kind = derive_response_kind(ctx.turn_intent)
+ tool_call_names = [
+ str(entry.get("tool") or entry.get("name") or "") for entry in ctx.tool_activity if isinstance(entry, dict)
+ ]
+ reason_codes = (
+ [code.value for code in ctx.turn_intent.reason_codes]
+ if ctx.turn_intent and ctx.turn_intent.reason_codes
+ else []
+ )
+ reason_code = ",".join(reason_codes)
+
+ turn_outcome = build_turn_outcome(
+ final_user_response,
+ turn_intent=ctx.turn_intent,
+ response_kind=attempted_kind,
+ reason_code=reason_code,
+ tool_calls=[name for name in tool_call_names if name],
+ )
+
return AgentResult(
- user_response=str(user_response),
+ user_response=final_user_response,
updated_workflow=last_workflow,
global_llm_context=enriched_context or None,
response_type=resp_type,
@@ -1335,6 +1392,7 @@ def _translate_to_agent_result(
else "auto_applicable"
),
output_policy_diagnostics=output_policy_diagnostics,
+ turn_outcome=turn_outcome,
)
@@ -1367,6 +1425,11 @@ def _build_feasibility_clarification_result(
workflow_yaml=prior_workflow_yaml or None,
workflow_was_persisted=False,
clear_proposed_workflow=True,
+ turn_outcome=build_minimal_turn_outcome(
+ question,
+ response_kind=ResponseKind.CLARIFY,
+ reason_code="feasibility_clarification",
+ ),
)
@@ -1424,15 +1487,22 @@ def _build_request_policy_clarification_result(
structured.decisions_made.append(
f"request-policy clarification required: {policy.credential_input_kind}/{policy.clarification_reason}"
)
+ clarification_text = (
+ policy.clarification_question or "I need one more detail before I can build and test this workflow safely."
+ )
return AgentResult(
- user_response=policy.clarification_question
- or "I need one more detail before I can build and test this workflow safely.",
+ user_response=clarification_text,
updated_workflow=None,
global_llm_context=structured.to_json_str(),
response_type="ASK_QUESTION",
workflow_yaml=prior_workflow_yaml or None,
workflow_was_persisted=False,
clear_proposed_workflow=True,
+ turn_outcome=build_minimal_turn_outcome(
+ clarification_text,
+ response_kind=ResponseKind.CLARIFY,
+ reason_code="request_policy_clarification",
+ ),
)
@@ -1712,6 +1782,12 @@ def _build_output_policy_blocked_result(
user_response = "I could not safely return that chat reply. Please adjust the request and try again."
if preserved_workflow is not None and add_saved_draft_copy:
user_response = f"{user_response} {_SAVED_DRAFT_OUTPUT_POLICY_SUFFIX}"
+ output_policy_outcome = build_minimal_turn_outcome(
+ user_response,
+ response_kind=ResponseKind.CLARIFY,
+ reason_code="output_policy_block",
+ terminal_reason="output_policy_block",
+ )
return AgentResult(
user_response=user_response,
updated_workflow=preserved_workflow,
@@ -1736,6 +1812,7 @@ def _build_output_policy_blocked_result(
hard_block_reason_codes=list(verdict.reason_codes),
soft_rewrite_reason_codes=[],
),
+ turn_outcome=output_policy_outcome,
)
@@ -1858,14 +1935,21 @@ async def _run_copilot_turn_impl(
error=str(e),
workflow_permanent_id=chat_request.workflow_permanent_id,
)
+ missing_sdk_reply = (
+ "Copilot backend is missing the OpenAI Agents SDK dependency. "
+ "Rebuild or redeploy the backend image so `openai-agents` is installed."
+ )
+ missing_sdk_outcome = build_minimal_turn_outcome(
+ missing_sdk_reply,
+ response_kind=ResponseKind.CLARIFY,
+ terminal_reason="missing_sdk",
+ )
return AgentResult(
- user_response=(
- "Copilot backend is missing the OpenAI Agents SDK dependency. "
- "Rebuild or redeploy the backend image so `openai-agents` is installed."
- ),
+ user_response=missing_sdk_reply,
updated_workflow=None,
global_llm_context=global_llm_context,
workflow_yaml=chat_request.workflow_yaml or None,
+ turn_outcome=missing_sdk_outcome,
)
raise
@@ -2169,16 +2253,23 @@ async def _run_copilot_turn_impl(
)
# Non-retriable nav errors prove the current workflow doesn't
# work; zero the proposal even if other tools succeeded.
+ nav_reply = (
+ f"The target URL could not be reached. Error: {exc.error_message}. "
+ "Please verify the URL and try again."
+ )
+ nav_outcome = build_minimal_turn_outcome(
+ nav_reply,
+ response_kind=ResponseKind.CLARIFY,
+ terminal_reason="non_retriable_nav",
+ )
return AgentResult(
- user_response=(
- f"The target URL could not be reached. Error: {exc.error_message}. "
- "Please verify the URL and try again."
- ),
+ user_response=nav_reply,
updated_workflow=None,
global_llm_context=global_llm_context,
workflow_yaml=None,
workflow_was_persisted=ctx.workflow_persisted,
total_tokens=ctx.total_tokens_used,
+ turn_outcome=nav_outcome,
)
except Exception as e:
LOG.error("Copilot agent error", error=str(e), exc_info=True)
diff --git a/skyvern/forge/sdk/copilot/context.py b/skyvern/forge/sdk/copilot/context.py
index 977bef02c..079ce36f3 100644
--- a/skyvern/forge/sdk/copilot/context.py
+++ b/skyvern/forge/sdk/copilot/context.py
@@ -21,6 +21,7 @@ if TYPE_CHECKING:
from skyvern.forge.sdk.copilot.request_policy import RequestPolicy
from skyvern.forge.sdk.copilot.turn_context import TurnContextPacket
from skyvern.forge.sdk.copilot.turn_intent import TurnIntent
+ from skyvern.forge.sdk.schemas.copilot_turn_outcome import TurnOutcome
class UrlVisit(BaseModel):
@@ -135,6 +136,7 @@ class AgentResult:
# Controls whether the route may auto-apply the proposal or must force explicit review.
proposal_disposition: ProposalDisposition = "auto_applicable"
output_policy_diagnostics: dict[str, Any] | None = None
+ turn_outcome: TurnOutcome | None = None
@dataclass
diff --git a/skyvern/forge/sdk/copilot/signature.py b/skyvern/forge/sdk/copilot/signature.py
new file mode 100644
index 000000000..bd3d610ae
--- /dev/null
+++ b/skyvern/forge/sdk/copilot/signature.py
@@ -0,0 +1,45 @@
+"""Bounded, deterministic reply signatures for cross-turn loop detection.
+
+The signature is computed over the user-visible reply (post
+``normalize_response_scaffolding``), with variable spans (URLs, Skyvern IDs,
+datetimes, numbers, backtick-quoted identifiers) stripped so two replies that
+differ only by run-specific values share a signature.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import re
+
+from skyvern.forge.sdk.copilot.request_policy import redact_raw_secrets_for_prompt
+
+RAW_SLICE_CAP = 8_000
+COMPARE_CAP = 4_000
+_DIGEST_BYTES = 8
+
+_WHITESPACE_RE = re.compile(r"\s+")
+_URL_RE = re.compile(r"https?://\S+", re.IGNORECASE)
+_SKYVERN_ID_RE = re.compile(
+ r"\b(?:wpid|wcc|wccm|wr|w|tsk|o|block|step|cred|wf)_[A-Za-z0-9]+",
+ re.IGNORECASE,
+)
+_ISO_DATETIME_RE = re.compile(r"\b\d{4}-\d{2}-\d{2}(?:[ T]\d{2}:\d{2}(?::\d{2}(?:\.\d+)?)?(?:Z|[+\-]\d{2}:?\d{2})?)?\b")
+_NUMERIC_RUN_RE = re.compile(r"\b\d{3,}\b")
+_BACKTICK_IDENT_RE = re.compile(r"`[A-Za-z][A-Za-z0-9_]*`")
+
+
+def normalize_for_compare(text: str) -> str:
+ raw = text or ""
+ if len(raw) > RAW_SLICE_CAP:
+ half = RAW_SLICE_CAP // 2
+ raw = raw[:half] + raw[-half:]
+ redacted = redact_raw_secrets_for_prompt(raw)
+ for pattern in (_URL_RE, _SKYVERN_ID_RE, _ISO_DATETIME_RE, _BACKTICK_IDENT_RE, _NUMERIC_RUN_RE):
+ redacted = pattern.sub(" ", redacted)
+ collapsed = _WHITESPACE_RE.sub(" ", redacted).strip().lower()
+ return collapsed[:COMPARE_CAP]
+
+
+def compute_signature(text: str) -> str:
+ digest = hashlib.blake2b(normalize_for_compare(text).encode("utf-8"), digest_size=_DIGEST_BYTES)
+ return digest.hexdigest()
diff --git a/skyvern/forge/sdk/copilot/turn_intent.py b/skyvern/forge/sdk/copilot/turn_intent.py
index 4de3595f4..d2e490098 100644
--- a/skyvern/forge/sdk/copilot/turn_intent.py
+++ b/skyvern/forge/sdk/copilot/turn_intent.py
@@ -2,6 +2,7 @@ from __future__ import annotations
import functools
import re
+from difflib import SequenceMatcher
from enum import StrEnum
from typing import Any
@@ -71,6 +72,7 @@ class TurnIntentReasonCode(StrEnum):
KEYWORD_HEURISTIC = "keyword_heuristic"
CONFIRMATION_CARRYOVER = "confirmation_carryover"
RAW_SECRET_REFUSAL = "raw_secret_refusal"
+ USER_NON_PROGRESS = "user_non_progress"
class TurnIntentAuthority(BaseModel):
@@ -334,6 +336,65 @@ def _is_bare_affirmative(user_message: str) -> bool:
return normalized in _AFFIRMATIVE_REPLIES
+_NON_PROGRESS_MARKER_RE = re.compile(
+ r"\b(?:"
+ r"i\s+can'?t\s+see\s+(?:it|that|them|those|the)"
+ r"|still\s+(?:not|isn'?t|doesn'?t)\s+working"
+ r"|still\s+(?:nothing|nope|none|no\s+luck)"
+ r"|(?:still|it|that|this)\s+doesn'?t\s+work\b"
+ r"|same\s+(?:problem|issue|answer|reply)"
+ r"|didn'?t\s+(?:work|help)"
+ r"|doesn'?t\s+help"
+ r"|(?:still\s+)?can'?t\s+find\s+(?:it|that|them|those|the)\b"
+ r"|where\s+(?:is|are|'?s)\s+(?:it|that|they|them)\b"
+ r"|i\s+already\s+looked\b"
+ r"|i\s+looked\s+(?:everywhere|already|but|and)"
+ r"|not\s+(?:there|here|working)"
+ r"|no\s+luck"
+ r"|that\s+didn'?t\s+(?:work|help)"
+ r")\b",
+ re.IGNORECASE,
+)
+_NON_PROGRESS_SHORT_LIMIT = 80
+_NON_PROGRESS_RESTATE_THRESHOLD = 0.7
+
+
+def _user_signals_non_progress(user_message: str, chat_history: list[WorkflowCopilotChatHistoryMessage]) -> bool:
+ """True when the current user turn re-engages on the same problem.
+
+ Requires at least one prior AI reply — without that, marker phrases like
+ "where is my X" on a first turn are genuine questions, not non-progress
+ restatements of a stuck conversation.
+ """
+ text = (user_message or "").strip()
+ if not text:
+ return False
+ has_prior_ai_reply = any(
+ m.sender == WorkflowCopilotChatSender.AI and (m.content or "").strip() for m in chat_history
+ )
+ if not has_prior_ai_reply:
+ return False
+ if _NON_PROGRESS_MARKER_RE.search(text):
+ return True
+ if len(text) > _NON_PROGRESS_SHORT_LIMIT:
+ return False
+ prior = next(
+ (
+ (m.content or "").strip()
+ for m in reversed(chat_history)
+ if m.sender == WorkflowCopilotChatSender.USER and (m.content or "").strip()
+ ),
+ "",
+ )
+ if not prior:
+ return False
+ current_norm = re.sub(r"\s+", " ", text.lower()).strip()
+ prior_norm = re.sub(r"\s+", " ", prior.lower()).strip()
+ if not current_norm or not prior_norm:
+ return False
+ return SequenceMatcher(None, current_norm, prior_norm).ratio() >= _NON_PROGRESS_RESTATE_THRESHOLD
+
+
def _carryover_mode_from_prior_turn(
chat_history: list[WorkflowCopilotChatHistoryMessage], *, has_workflow: bool
) -> tuple[TurnIntentMode, TurnIntentExpectedOutput] | None:
@@ -434,6 +495,9 @@ def build_turn_intent(
confidence = 0.3
reason_codes.append(TurnIntentReasonCode.CONFIRMATION_CARRYOVER)
+ if _user_signals_non_progress(user_message, chat_history):
+ reason_codes.append(TurnIntentReasonCode.USER_NON_PROGRESS)
+
if mode == TurnIntentMode.EDIT:
unresolved_block_refs = _unresolved_explicit_block_refs(user_message, workflow_yaml)
if unresolved_block_refs:
diff --git a/skyvern/forge/sdk/copilot/turn_outcome.py b/skyvern/forge/sdk/copilot/turn_outcome.py
new file mode 100644
index 000000000..c9b970678
--- /dev/null
+++ b/skyvern/forge/sdk/copilot/turn_outcome.py
@@ -0,0 +1,106 @@
+"""Builders + closed `TurnIntentMode -> ResponseKind` mapping for `TurnOutcome`.
+
+Schema types live in ``schemas/copilot_turn_outcome.py`` so chat-history
+schemas can embed ``TurnOutcome`` without importing copilot business logic.
+This module imports both the schema types and ``TurnIntentMode`` — the only
+direction allowed.
+"""
+
+from __future__ import annotations
+
+from collections.abc import Iterable
+from typing import Any
+
+import structlog
+
+from skyvern.forge.sdk.copilot.signature import compute_signature
+from skyvern.forge.sdk.copilot.turn_intent import TurnIntent, TurnIntentMode
+from skyvern.forge.sdk.schemas.copilot_turn_outcome import ResponseKind, TurnOutcome
+
+LOG = structlog.get_logger()
+
+_RESPONSE_KIND_BY_MODE: dict[TurnIntentMode, ResponseKind] = {
+ TurnIntentMode.BUILD: ResponseKind.BUILD,
+ TurnIntentMode.EDIT: ResponseKind.BUILD,
+ TurnIntentMode.DRAFT_ONLY: ResponseKind.BUILD,
+ TurnIntentMode.CLARIFY: ResponseKind.CLARIFY,
+ TurnIntentMode.UNKNOWN: ResponseKind.CLARIFY,
+ TurnIntentMode.DIAGNOSE: ResponseKind.DIAGNOSE,
+ TurnIntentMode.DOCS_ANSWER: ResponseKind.DIAGNOSE,
+ TurnIntentMode.REFUSE: ResponseKind.REFUSE,
+}
+
+# Catches the "added a TurnIntentMode but forgot to map it" foot-gun at import
+# time rather than letting the new mode silently fall through to CLARIFY.
+# Raises explicitly (not ``assert``) so the guard survives ``python -O``.
+_missing_modes = set(TurnIntentMode) - set(_RESPONSE_KIND_BY_MODE)
+if _missing_modes:
+ raise RuntimeError(f"_RESPONSE_KIND_BY_MODE missing entries for: {sorted(m.value for m in _missing_modes)}")
+
+
+def derive_response_kind(turn_intent: TurnIntent | None) -> ResponseKind:
+ """Closed mapping. ``RECOVER`` is set only by the enforcement guard."""
+ mode = getattr(turn_intent, "mode", None)
+ if isinstance(mode, TurnIntentMode):
+ return _RESPONSE_KIND_BY_MODE[mode]
+ return ResponseKind.CLARIFY
+
+
+def _dedup_signatures(signatures: Iterable[str]) -> list[str]:
+ return sorted({sig for sig in signatures if isinstance(sig, str) and sig})
+
+
+def build_minimal_turn_outcome(
+ final_text: str,
+ response_kind: ResponseKind,
+ reason_code: str = "",
+ terminal_reason: str | None = None,
+ inherited_blocked_signatures: Iterable[str] = (),
+) -> TurnOutcome:
+ """Used by every direct-return ``AgentResult`` site so the persisted AI row
+ always carries a ``turn_outcome``. Callers that need ban-set inheritance
+ pass it in via ``inherited_blocked_signatures``; the route-level
+ ``apply_repeated_reply_guard`` is the typical source."""
+ return TurnOutcome(
+ response_kind=response_kind,
+ reason_code=reason_code,
+ normalized_reply_signature=compute_signature(final_text),
+ terminal_reason=terminal_reason,
+ blocked_signatures=_dedup_signatures(inherited_blocked_signatures),
+ )
+
+
+def build_turn_outcome(
+ final_text: str,
+ *,
+ turn_intent: TurnIntent | None,
+ response_kind: ResponseKind | None = None,
+ reason_code: str = "",
+ tool_calls: Iterable[str] = (),
+ terminal_reason: str | None = None,
+ inherited_blocked_signatures: Iterable[str] = (),
+ extra_blocked_signatures: Iterable[str] = (),
+) -> TurnOutcome:
+ """Used by the translation path. Resolves ``response_kind`` from the turn
+ intent when not supplied; merges inherited + extra blocked signatures so
+ the enforcement guard can record the original signature it just blocked."""
+ resolved_kind = response_kind if response_kind is not None else derive_response_kind(turn_intent)
+ intent_summary: dict[str, Any] = {}
+ if turn_intent is not None:
+ try:
+ intent_summary = dict(turn_intent.to_trace_data())
+ except Exception as exc:
+ LOG.warning(
+ "Failed to serialize TurnIntent trace data for TurnOutcome; using empty dict",
+ exc_info=exc,
+ )
+ intent_summary = {}
+ return TurnOutcome(
+ turn_intent_summary=intent_summary,
+ response_kind=resolved_kind,
+ reason_code=reason_code,
+ normalized_reply_signature=compute_signature(final_text),
+ tool_calls=[str(call) for call in tool_calls if call],
+ terminal_reason=terminal_reason,
+ blocked_signatures=_dedup_signatures(list(inherited_blocked_signatures) + list(extra_blocked_signatures)),
+ )
diff --git a/skyvern/forge/sdk/routes/workflow_copilot.py b/skyvern/forge/sdk/routes/workflow_copilot.py
index f374f8ef0..ed489b3f9 100644
--- a/skyvern/forge/sdk/routes/workflow_copilot.py
+++ b/skyvern/forge/sdk/routes/workflow_copilot.py
@@ -38,6 +38,7 @@ from skyvern.forge.sdk.core import skyvern_context
from skyvern.forge.sdk.experimentation.llm_prompt_config import get_llm_handler_for_prompt_type
from skyvern.forge.sdk.routes.event_source_stream import EventSourceStream, FastAPIEventSourceStream
from skyvern.forge.sdk.routes.routers import base_router
+from skyvern.forge.sdk.schemas.copilot_turn_outcome import TurnOutcome
from skyvern.forge.sdk.schemas.organizations import Organization
from skyvern.forge.sdk.schemas.workflow_copilot import (
WorkflowCopilotApplyProposedWorkflowRequest,
@@ -388,6 +389,7 @@ async def _persist_cancel_turn(
rollback uses the same ``workflow_was_persisted`` source of truth as
the success path; pass ``None`` for pre-agent cancels.
"""
+ turn_outcome: TurnOutcome | None
if agent_result is None:
user_response = "Cancelled by user."
updated_workflow = None
@@ -395,6 +397,7 @@ async def _persist_cancel_turn(
total_tokens = None
response_type = "REPLY"
output_policy_diagnostics = None
+ turn_outcome = None
if chat.proposed_workflow is not None:
await asyncio.shield(_clear_proposed_workflow(chat))
else:
@@ -411,6 +414,7 @@ async def _persist_cancel_turn(
total_tokens = getattr(agent_result, "total_tokens", None)
response_type = getattr(agent_result, "response_type", "REPLY")
output_policy_diagnostics = getattr(agent_result, "output_policy_diagnostics", None)
+ turn_outcome = agent_result.turn_outcome
await asyncio.shield(
app.DATABASE.workflow_params.create_workflow_copilot_chat_message(
@@ -427,6 +431,7 @@ async def _persist_cancel_turn(
sender=WorkflowCopilotChatSender.AI,
content=user_response,
global_llm_context=updated_global_llm_context,
+ turn_outcome=turn_outcome,
)
)
proposal_disposition = _proposal_disposition(agent_result)
@@ -504,6 +509,7 @@ async def _finalise_normal_turn(
sender=WorkflowCopilotChatSender.AI,
content=user_response,
global_llm_context=updated_global_llm_context,
+ turn_outcome=agent_result.turn_outcome,
)
proposal_disposition = _proposal_disposition(agent_result)
diff --git a/tests/unit/test_copilot_signature.py b/tests/unit/test_copilot_signature.py
new file mode 100644
index 000000000..91c1b0c4e
--- /dev/null
+++ b/tests/unit/test_copilot_signature.py
@@ -0,0 +1,54 @@
+from __future__ import annotations
+
+from skyvern.forge.sdk.copilot.signature import (
+ COMPARE_CAP,
+ compute_signature,
+ normalize_for_compare,
+)
+
+
+def test_normalize_strips_urls() -> None:
+ a = normalize_for_compare("See https://example.com/a/b for details.")
+ b = normalize_for_compare("See https://other.example.org/x for details.")
+ assert a == b
+
+
+def test_normalize_strips_skyvern_ids() -> None:
+ a = normalize_for_compare("Block `block_1` failed during run wr_abc123.")
+ b = normalize_for_compare("Block `block_44` failed during run wr_xyz999.")
+ assert a == b
+
+
+def test_normalize_strips_numbers_and_dates() -> None:
+ a = normalize_for_compare("Run took 1234 ms and completed at 2026-05-22T10:00:00Z.")
+ b = normalize_for_compare("Run took 9876 ms and completed at 2026-05-21T18:30:00Z.")
+ assert a == b
+
+
+def test_normalize_caps_to_compare_cap() -> None:
+ body = "a " * 10_000
+ assert len(normalize_for_compare(body)) <= COMPARE_CAP
+
+
+def test_signature_matches_for_variant_inputs() -> None:
+ a = "The file is in the Artifacts section. URL: https://example.com/foo/123"
+ b = "The file is in the Artifacts section. URL: https://other.io/baz/9999"
+ assert compute_signature(a) == compute_signature(b)
+
+
+def test_signature_differs_for_distinct_text() -> None:
+ a = "The file is in the Artifacts section."
+ b = "I have emailed the downloaded file to your account."
+ assert compute_signature(a) != compute_signature(b)
+
+
+def test_signature_is_16_hex_chars() -> None:
+ sig = compute_signature("hello")
+ assert len(sig) == 16
+ assert all(c in "0123456789abcdef" for c in sig)
+
+
+def test_normalize_redacts_secrets_before_compare() -> None:
+ a = normalize_for_compare("Use password: hunter2abc to sign in.")
+ b = normalize_for_compare("Use password: secret9999 to sign in.")
+ assert a == b
diff --git a/tests/unit/test_copilot_turn_intent.py b/tests/unit/test_copilot_turn_intent.py
index 8c6928292..649ff8e18 100644
--- a/tests/unit/test_copilot_turn_intent.py
+++ b/tests/unit/test_copilot_turn_intent.py
@@ -408,6 +408,82 @@ def test_build_turn_intent_omits_workflow_change_on_first_turn() -> None:
assert RequiredContextKey.WORKFLOW_CHANGE not in intent.required_context
+def _ai_message(text: str) -> WorkflowCopilotChatHistoryMessage:
+ return WorkflowCopilotChatHistoryMessage(
+ sender=WorkflowCopilotChatSender.AI,
+ content=text,
+ created_at=datetime.now(timezone.utc),
+ )
+
+
+def test_user_non_progress_fires_on_stuck_marker() -> None:
+ intent = build_turn_intent(
+ user_message="i can't see it",
+ workflow_yaml="",
+ chat_history=[_user_message("where is my downloaded file"), _ai_message("Check the Artifacts panel.")],
+ global_llm_context="",
+ request_policy=RequestPolicy(),
+ )
+ assert TurnIntentReasonCode.USER_NON_PROGRESS in intent.reason_codes
+
+
+def test_user_non_progress_fires_on_short_restatement() -> None:
+ intent = build_turn_intent(
+ user_message="where's my downloaded file?",
+ workflow_yaml="",
+ chat_history=[_user_message("where is my downloaded file"), _ai_message("Check the Artifacts panel.")],
+ global_llm_context="",
+ request_policy=RequestPolicy(),
+ )
+ assert TurnIntentReasonCode.USER_NON_PROGRESS in intent.reason_codes
+
+
+def test_user_non_progress_does_not_fire_on_topic_switch_after_ai_reply() -> None:
+ """Regression: 'where is my X' is a topic switch, not non-progress, even with prior AI."""
+ intent = build_turn_intent(
+ user_message="where is my API key stored?",
+ workflow_yaml="",
+ chat_history=[_user_message("build me a workflow"), _ai_message("Drafted v1.")],
+ global_llm_context="",
+ request_policy=RequestPolicy(),
+ )
+ assert TurnIntentReasonCode.USER_NON_PROGRESS not in intent.reason_codes
+
+
+def test_user_non_progress_does_not_fire_on_first_turn_even_when_marker_matches() -> None:
+ """Regression: a first-turn 'where is my X' is a real question, not non-progress."""
+ intent = build_turn_intent(
+ user_message="where is my API key stored?",
+ workflow_yaml="",
+ chat_history=[],
+ global_llm_context="",
+ request_policy=RequestPolicy(),
+ )
+ assert TurnIntentReasonCode.USER_NON_PROGRESS not in intent.reason_codes
+
+
+def test_user_non_progress_does_not_fire_on_topic_switch() -> None:
+ intent = build_turn_intent(
+ user_message="now build me a workflow for invoicing",
+ workflow_yaml="",
+ chat_history=[_user_message("where is my downloaded file"), _ai_message("Check the Artifacts panel.")],
+ global_llm_context="",
+ request_policy=RequestPolicy(),
+ )
+ assert TurnIntentReasonCode.USER_NON_PROGRESS not in intent.reason_codes
+
+
+def test_user_non_progress_does_not_fire_on_legitimate_followup() -> None:
+ intent = build_turn_intent(
+ user_message="where is the documentation for loop blocks?",
+ workflow_yaml="",
+ chat_history=[_user_message("build me a workflow"), _ai_message("Drafted v1.")],
+ global_llm_context="",
+ request_policy=RequestPolicy(),
+ )
+ assert TurnIntentReasonCode.USER_NON_PROGRESS not in intent.reason_codes
+
+
def test_store_request_policy_attaches_turn_intent_to_context() -> None:
ctx = CopilotContext(
organization_id="org-1",
diff --git a/tests/unit/test_copilot_turn_outcome.py b/tests/unit/test_copilot_turn_outcome.py
new file mode 100644
index 000000000..132f53809
--- /dev/null
+++ b/tests/unit/test_copilot_turn_outcome.py
@@ -0,0 +1,74 @@
+from __future__ import annotations
+
+import pytest
+
+from skyvern.forge.sdk.copilot.signature import compute_signature
+from skyvern.forge.sdk.copilot.turn_intent import TurnIntent, TurnIntentMode
+from skyvern.forge.sdk.copilot.turn_outcome import (
+ build_minimal_turn_outcome,
+ build_turn_outcome,
+ derive_response_kind,
+)
+from skyvern.forge.sdk.schemas.copilot_turn_outcome import ResponseKind, TurnOutcome
+
+
+ "mode,expected",
+ [
+ (TurnIntentMode.BUILD, ResponseKind.BUILD),
+ (TurnIntentMode.EDIT, ResponseKind.BUILD),
+ (TurnIntentMode.DRAFT_ONLY, ResponseKind.BUILD),
+ (TurnIntentMode.CLARIFY, ResponseKind.CLARIFY),
+ (TurnIntentMode.UNKNOWN, ResponseKind.CLARIFY),
+ (TurnIntentMode.DIAGNOSE, ResponseKind.DIAGNOSE),
+ (TurnIntentMode.DOCS_ANSWER, ResponseKind.DIAGNOSE),
+ (TurnIntentMode.REFUSE, ResponseKind.REFUSE),
+ ],
+)
+def test_derive_response_kind_covers_every_mode(mode: TurnIntentMode, expected: ResponseKind) -> None:
+ intent = TurnIntent(mode=mode)
+ assert derive_response_kind(intent) is expected
+
+
+def test_derive_response_kind_handles_none_intent() -> None:
+ assert derive_response_kind(None) is ResponseKind.CLARIFY
+
+
+def test_build_minimal_turn_outcome_sets_signature_and_inherited() -> None:
+ outcome = build_minimal_turn_outcome(
+ "The file is in the Artifacts section.",
+ response_kind=ResponseKind.DIAGNOSE,
+ reason_code="diagnose_reply",
+ terminal_reason=None,
+ inherited_blocked_signatures=["aaaa", "bbbb", "aaaa"],
+ )
+ assert outcome.response_kind is ResponseKind.DIAGNOSE
+ assert outcome.normalized_reply_signature == compute_signature("The file is in the Artifacts section.")
+ assert outcome.blocked_signatures == ["aaaa", "bbbb"]
+ assert outcome.reason_code == "diagnose_reply"
+
+
+def test_build_turn_outcome_merges_inherited_and_extra() -> None:
+ intent = TurnIntent(mode=TurnIntentMode.BUILD)
+ outcome = build_turn_outcome(
+ "drafted v1",
+ turn_intent=intent,
+ tool_calls=["update_workflow", "run_blocks_and_collect_debug"],
+ inherited_blocked_signatures=["aaaa"],
+ extra_blocked_signatures=["bbbb"],
+ )
+ assert outcome.response_kind is ResponseKind.BUILD
+ assert outcome.tool_calls == ["update_workflow", "run_blocks_and_collect_debug"]
+ assert outcome.blocked_signatures == ["aaaa", "bbbb"]
+ assert "mode" in outcome.turn_intent_summary
+
+
+def test_turn_outcome_json_round_trip() -> None:
+ outcome = build_minimal_turn_outcome(
+ "answer",
+ response_kind=ResponseKind.CLARIFY,
+ inherited_blocked_signatures=["xyz"],
+ )
+ payload = outcome.model_dump(mode="json")
+ restored = TurnOutcome.model_validate(payload)
+ assert restored == outcome
diff --git a/tests/unit/test_workflow_copilot_cancel.py b/tests/unit/test_workflow_copilot_cancel.py
index c80f36079..c35306e92 100644
--- a/tests/unit/test_workflow_copilot_cancel.py
+++ b/tests/unit/test_workflow_copilot_cancel.py
@@ -302,6 +302,7 @@ async def test_route_cancel_branch_persists_user_and_cancelled_messages(
cancelled=True,
total_tokens=None,
response_type="REPLY",
+ turn_outcome=None,
)
restore_mock, workflow_params, sent_payloads = await _drive_cancel_route(
monkeypatch, chat, original_workflow, agent_result
@@ -352,6 +353,7 @@ async def test_route_cancel_branch_persists_wip_proposal_and_response_frame(
total_tokens=123,
response_type="REPLY",
proposal_disposition="review_untested",
+ turn_outcome=None,
)
restore_mock, workflow_params, sent_payloads = await _drive_cancel_route(
monkeypatch, chat, original_workflow, agent_result
@@ -407,6 +409,7 @@ async def test_route_cancel_tested_wip_with_auto_accept_still_persists_proposal(
total_tokens=123,
response_type="REPLY",
proposal_disposition="auto_applicable",
+ turn_outcome=None,
)
restore_mock, workflow_params, sent_payloads = await _drive_cancel_route(
monkeypatch, chat, original_workflow, agent_result
@@ -450,6 +453,7 @@ async def test_route_cancel_review_tested_persists_proposal_without_unvalidated_
total_tokens=456,
response_type="REPLY",
proposal_disposition="review_tested",
+ turn_outcome=None,
)
restore_mock, workflow_params, sent_payloads = await _drive_cancel_route(
monkeypatch, chat, original_workflow, agent_result
@@ -495,6 +499,7 @@ async def test_route_cancel_clears_stale_proposed_workflow_when_no_wip(
total_tokens=None,
response_type="REPLY",
proposal_disposition="auto_applicable",
+ turn_outcome=None,
)
restore_mock, workflow_params, _sent = await _drive_cancel_route(monkeypatch, chat, original_workflow, agent_result)
@@ -524,6 +529,7 @@ async def test_route_cancel_clears_stale_proposed_workflow_when_no_wip_and_no_pe
total_tokens=None,
response_type="REPLY",
proposal_disposition="auto_applicable",
+ turn_outcome=None,
)
restore_mock, workflow_params, _sent = await _drive_cancel_route(monkeypatch, chat, original_workflow, agent_result)
@@ -602,6 +608,7 @@ async def test_timeout_wip_result_streams_normal_response_frame(
total_tokens=123,
response_type="REPLY",
proposal_disposition="review_untested",
+ turn_outcome=None,
)
restore_mock, workflow_params = _setup_route_mocks(monkeypatch, chat, original_workflow, agent_result)
@@ -679,6 +686,7 @@ async def test_timeout_wip_review_tested_propagates_to_response_frame(
total_tokens=789,
response_type="REPLY",
proposal_disposition="review_tested",
+ turn_outcome=None,
)
restore_mock, workflow_params = _setup_route_mocks(monkeypatch, chat, original_workflow, agent_result)
diff --git a/tests/unit/test_workflow_copilot_route.py b/tests/unit/test_workflow_copilot_route.py
index d5f680b5a..5f3ec8d36 100644
--- a/tests/unit/test_workflow_copilot_route.py
+++ b/tests/unit/test_workflow_copilot_route.py
@@ -237,6 +237,7 @@ async def test_flag_on_mid_stream_disconnect_restores_when_persisted_and_not_aut
workflow_was_persisted=workflow_was_persisted,
clear_proposed_workflow=False,
proposal_disposition="auto_applicable",
+ turn_outcome=None,
)
restore_mock = _setup_new_copilot_mocks(monkeypatch, chat, original_workflow, agent_result)
@@ -389,6 +390,7 @@ async def test_flag_on_route_error_after_chat_persists_recoverable_reply(
workflow_was_persisted=False,
clear_proposed_workflow=False,
unvalidated=False,
+ turn_outcome=None,
)
_setup_new_copilot_mocks(monkeypatch, chat, original_workflow, agent_result)
monkeypatch.setattr(
@@ -455,6 +457,7 @@ async def test_route_error_after_restore_reports_workflow_not_modified(
workflow_was_persisted=True,
clear_proposed_workflow=False,
unvalidated=False,
+ turn_outcome=None,
)
restore_mock = _setup_new_copilot_mocks(monkeypatch, chat, original_workflow, agent_result)
finalise_results: list[object] = []
@@ -580,6 +583,7 @@ async def test_proposed_workflow_cleared_on_restore(
workflow_was_persisted=workflow_was_persisted,
clear_proposed_workflow=clear_proposed_flag,
proposal_disposition="auto_applicable",
+ turn_outcome=None,
)
_setup_new_copilot_mocks(monkeypatch, chat, original_workflow, agent_result)
@@ -652,6 +656,7 @@ async def test_output_policy_block_preserves_unvalidated_prior_proposal_under_au
"final_output_policy_allowed": False,
"hard_block_reason_codes": ["internal_tool_instruction_leak"],
},
+ turn_outcome=None,
)
_setup_new_copilot_mocks(monkeypatch, chat, original_workflow, agent_result)
@@ -715,6 +720,7 @@ async def test_unvalidated_timeout_wip_overrides_auto_accept(monkeypatch: pytest
"raw_would_have_failed": True,
"contained_failure": True,
},
+ turn_outcome=None,
)
restore_mock = _setup_new_copilot_mocks(monkeypatch, chat, original_workflow, agent_result)