Skip to content

Automate browser based workflows with AI

SKY-9975 Add TurnIntent shadow contract (#6063)

Andrew Neilson7808fb1c0dd9
diff --git a/skyvern/forge/sdk/copilot/agent.py b/skyvern/forge/sdk/copilot/agent.py
index 9a6180f87..8c76e7e66 100644
--- a/skyvern/forge/sdk/copilot/agent.py
+++ b/skyvern/forge/sdk/copilot/agent.py
@@ -53,6 +53,7 @@ from skyvern.forge.sdk.copilot.output_utils import (
)
from skyvern.forge.sdk.copilot.request_policy import RequestPolicy, build_request_policy, redact_raw_secrets_for_prompt
from skyvern.forge.sdk.copilot.tracing_setup import _copilot_model_name, ensure_tracing_initialized, is_tracing_enabled
+from skyvern.forge.sdk.copilot.turn_intent import TurnIntent, build_turn_intent
from skyvern.forge.sdk.schemas.persistent_browser_sessions import is_final_status
from skyvern.forge.sdk.schemas.workflow_copilot import (
WorkflowCopilotChatHistoryMessage,
@@ -130,6 +131,10 @@ class RequestPolicyGuardrailInputs:
organization_id: str
handler: Any
previous_user_message: str | None = None
+ workflow_id: str | None = None
+ workflow_permanent_id: str | None = None
+ workflow_run_id: str | None = None
+ browser_session_id: str | None = None
_BLOCK_GOAL_CONTEXT_PREAMBLE = (
@@ -272,6 +277,27 @@ def _store_request_policy_on_context(
chat_history_text=policy_chat_history_text,
global_llm_context=policy_inputs.global_llm_context,
)
+ ctx.turn_intent = build_turn_intent(
+ user_message=policy_inputs.user_message,
+ workflow_yaml=policy_inputs.workflow_yaml,
+ chat_history=policy_inputs.chat_history_messages,
+ global_llm_context=policy_inputs.global_llm_context,
+ request_policy=policy,
+ workflow_id=policy_inputs.workflow_id,
+ workflow_permanent_id=policy_inputs.workflow_permanent_id,
+ workflow_run_id=policy_inputs.workflow_run_id,
+ browser_session_id=policy_inputs.browser_session_id,
+ )
+
+
+def _turn_intent_log_fields(intent: TurnIntent | None) -> dict[str, Any]:
+ if not isinstance(intent, TurnIntent):
+ return {}
+ return {f"turn_intent_{key}": value for key, value in intent.to_trace_data().items()}
+
+
+def _turn_intent_trace_fields(intent: TurnIntent | None) -> dict[str, str]:
+ return {key: str(value) for key, value in _turn_intent_log_fields(intent).items()}
def _build_system_prompt(
@@ -1172,12 +1198,14 @@ def _build_copilot_input_guardrails(
_store_request_policy_on_context(ctx, policy, policy_inputs)
blocked = isinstance(policy, RequestPolicy) and policy.user_response_policy == "ask_clarification"
if isinstance(policy, RequestPolicy):
+ turn_intent = ctx.turn_intent if isinstance(ctx, CopilotContext) else None
trace_data = {
"surface": "agent_input",
"policy_present": True,
"blocked": blocked,
"user_response_policy": policy.user_response_policy,
**policy.to_trace_data(),
+ **_turn_intent_log_fields(turn_intent),
}
else:
trace_data = {"surface": "agent_input", "blocked": False, "policy_present": False}
@@ -1386,6 +1414,10 @@ async def _run_copilot_turn_impl(
organization_id=organization_id,
handler=_resolve_request_policy_handler(llm_api_handler),
previous_user_message=previous_user_message,
+ workflow_id=chat_request.workflow_id,
+ workflow_permanent_id=chat_request.workflow_permanent_id,
+ workflow_run_id=getattr(chat_request, "workflow_run_id", None),
+ browser_session_id=getattr(chat_request, "browser_session_id", None),
)
request_policy_guardrails = _build_copilot_input_guardrails(
InputGuardrail,
@@ -1502,6 +1534,7 @@ async def _run_copilot_turn_impl(
"llm_key": llm_key,
"user_message_len": str(len(user_message)),
**{f"request_policy_{key}": str(value) for key, value in request_policy.to_trace_data().items()},
+ **_turn_intent_trace_fields(ctx.turn_intent),
},
)
diff --git a/skyvern/forge/sdk/copilot/context.py b/skyvern/forge/sdk/copilot/context.py
index 5b5661d53..48ba0ea2b 100644
--- a/skyvern/forge/sdk/copilot/context.py
+++ b/skyvern/forge/sdk/copilot/context.py
@@ -17,6 +17,7 @@ COPILOT_RESPONSE_TYPES: tuple[ResponseType, ...] = get_args(ResponseType)
if TYPE_CHECKING:
from skyvern.forge.sdk.copilot.narration import NarratorState
from skyvern.forge.sdk.copilot.request_policy import RequestPolicy
+ from skyvern.forge.sdk.copilot.turn_intent import TurnIntent
class UrlVisit(BaseModel):
@@ -167,6 +168,7 @@ class CopilotContext(AgentContext):
block_goal_main_goal: str = ""
allow_untested_workflow_draft: bool = False
request_policy: RequestPolicy | None = None
+ turn_intent: TurnIntent | None = None
# Tool tracking
consecutive_tool_tracker: list[str] = field(default_factory=list)
diff --git a/skyvern/forge/sdk/copilot/turn_intent.py b/skyvern/forge/sdk/copilot/turn_intent.py
new file mode 100644
index 000000000..cca9c03f3
--- /dev/null
+++ b/skyvern/forge/sdk/copilot/turn_intent.py
@@ -0,0 +1,253 @@
+from __future__ import annotations
+
+from enum import StrEnum
+from typing import Any
+
+from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator
+
+from skyvern.forge.sdk.copilot.request_policy import RequestPolicy, redact_raw_secrets_for_prompt
+from skyvern.forge.sdk.schemas.workflow_copilot import (
+ WorkflowCopilotChatHistoryMessage,
+ WorkflowCopilotChatSender,
+)
+
+
+class TurnIntentMode(StrEnum):
+ BUILD = "build"
+ EDIT = "edit"
+ DIAGNOSE = "diagnose"
+ DOCS_ANSWER = "docs_answer"
+ DRAFT_ONLY = "draft_only"
+ CLARIFY = "clarify"
+ REFUSE = "refuse"
+ UNKNOWN = "unknown"
+
+
+class RequiredContextKey(StrEnum):
+ CURRENT_WORKFLOW = "current_workflow"
+ PROPOSED_WORKFLOW = "proposed_workflow"
+ LATEST_ASSISTANT_PROPOSAL = "latest_assistant_proposal"
+ LATEST_RUN_RESULT = "latest_run_result"
+ CREDENTIAL_METADATA = "credential_metadata"
+ DOCS_CONTEXT = "docs_context"
+ BROWSER_STATE = "browser_state"
+
+
+class TurnIntentExpectedOutput(StrEnum):
+ WORKFLOW_UPDATE = "workflow_update"
+ WORKFLOW_DRAFT = "workflow_draft"
+ RUN_RESULT = "run_result"
+ EXPLANATION = "explanation"
+ CLARIFICATION = "clarification"
+ REFUSAL = "refusal"
+
+
+class TurnIntentReasonCode(StrEnum):
+ DEFAULT_UNKNOWN = "default_unknown"
+ REQUEST_POLICY_DERIVED = "request_policy_derived"
+ REQUEST_POLICY_CLARIFICATION = "request_policy_clarification"
+ TESTING_INTENT_SKIP_TEST = "testing_intent_skip_test"
+ WORKFLOW_CONTEXT_PRESENT = "workflow_context_present"
+ CHAT_HISTORY_PRESENT = "chat_history_present"
+ RUN_CONTEXT_PRESENT = "run_context_present"
+ BROWSER_CONTEXT_PRESENT = "browser_context_present"
+ KEYWORD_HEURISTIC = "keyword_heuristic"
+
+
+class TurnIntentAuthority(BaseModel):
+ model_config = ConfigDict(extra="forbid")
+
+ may_update_workflow: bool = False
+ may_run_blocks: bool = False
+ may_answer_without_mutation: bool = True
+ requires_user_input: bool = False
+
+
+class TurnIntent(BaseModel):
+ model_config = ConfigDict(extra="forbid")
+
+ mode: TurnIntentMode = TurnIntentMode.UNKNOWN
+ user_goal: str = ""
+ target_entities: dict[str, list[str]] = Field(default_factory=dict)
+ required_context: list[RequiredContextKey] = Field(default_factory=list)
+ authority: TurnIntentAuthority = Field(default_factory=TurnIntentAuthority)
+ expected_output: TurnIntentExpectedOutput = TurnIntentExpectedOutput.EXPLANATION
+ confidence: float = Field(default=0.0, ge=0.0, le=1.0)
+ reason_codes: list[TurnIntentReasonCode] = Field(default_factory=lambda: [TurnIntentReasonCode.DEFAULT_UNKNOWN])
+ missing_context_question: str | None = None
+
+ @field_validator("required_context", mode="after")
+ @classmethod
+ def _dedupe_required_context(cls, value: list[RequiredContextKey]) -> list[RequiredContextKey]:
+ return list(dict.fromkeys(value))
+
+ @field_validator("reason_codes", mode="after")
+ @classmethod
+ def _dedupe_reason_codes(cls, value: list[TurnIntentReasonCode]) -> list[TurnIntentReasonCode]:
+ return list(dict.fromkeys(value)) or [TurnIntentReasonCode.DEFAULT_UNKNOWN]
+
+ @field_validator("target_entities", mode="after")
+ @classmethod
+ def _dedupe_target_entities(cls, value: dict[str, list[str]]) -> dict[str, list[str]]:
+ return {
+ str(entity_type): list(dict.fromkeys(str(entity).strip() for entity in entities if str(entity).strip()))
+ for entity_type, entities in value.items()
+ if str(entity_type).strip()
+ }
+
+ @model_validator(mode="after")
+ def _align_expected_output_with_mode(self) -> TurnIntent:
+ if self.expected_output != TurnIntentExpectedOutput.EXPLANATION:
+ return self
+ expected_output_by_mode = {
+ TurnIntentMode.BUILD: TurnIntentExpectedOutput.WORKFLOW_DRAFT,
+ TurnIntentMode.EDIT: TurnIntentExpectedOutput.WORKFLOW_UPDATE,
+ TurnIntentMode.DRAFT_ONLY: TurnIntentExpectedOutput.WORKFLOW_DRAFT,
+ TurnIntentMode.CLARIFY: TurnIntentExpectedOutput.CLARIFICATION,
+ TurnIntentMode.REFUSE: TurnIntentExpectedOutput.REFUSAL,
+ }
+ if mapped_expected_output := expected_output_by_mode.get(self.mode):
+ self.expected_output = mapped_expected_output
+ return self
+
+ def to_trace_data(self) -> dict[str, Any]:
+ data: dict[str, Any] = {
+ "mode": self.mode.value,
+ "expected_output": self.expected_output.value,
+ "required_context": [key.value for key in self.required_context],
+ "may_update_workflow": self.authority.may_update_workflow,
+ "may_run_blocks": self.authority.may_run_blocks,
+ "may_answer_without_mutation": self.authority.may_answer_without_mutation,
+ "requires_user_input": self.authority.requires_user_input,
+ "confidence": self.confidence,
+ "reason_codes": [reason.value for reason in self.reason_codes],
+ }
+ if self.target_entities:
+ data["target_entity_types"] = sorted(self.target_entities)
+ if self.missing_context_question:
+ data["has_missing_context_question"] = True
+ return data
+
+
+_GOAL_MAX_CHARS = 240
+_BUILD_TERMS = ("build", "create", "make", "generate")
+_EDIT_TERMS = ("edit", "update", "change", "modify", "replace", "fix")
+_DIAGNOSE_TERMS = ("debug", "diagnose", "failed", "failure", "error", "result")
+_DOCS_TERMS = ("explain", "how do", "how does", "what is", "what are", "why", "docs", "documentation")
+
+
+def _normalize_user_goal(user_message: str) -> str:
+ goal = redact_raw_secrets_for_prompt((user_message or "").strip())
+ if len(goal) <= _GOAL_MAX_CHARS:
+ return goal
+ return goal[: _GOAL_MAX_CHARS - 3].rstrip() + "..."
+
+
+def _has_latest_assistant_turn(chat_history: list[WorkflowCopilotChatHistoryMessage]) -> bool:
+ return any(
+ message.sender == WorkflowCopilotChatSender.AI and (message.content or "").strip() for message in chat_history
+ )
+
+
+def _contains_any(text: str, terms: tuple[str, ...]) -> bool:
+ normalized = text.lower()
+ return any(term in normalized for term in terms)
+
+
+def _mode_from_keywords(
+ user_message: str, *, has_workflow: bool
+) -> tuple[TurnIntentMode, TurnIntentExpectedOutput] | None:
+ if _contains_any(user_message, _DIAGNOSE_TERMS):
+ return TurnIntentMode.DIAGNOSE, TurnIntentExpectedOutput.RUN_RESULT
+ if _contains_any(user_message, _DOCS_TERMS):
+ return TurnIntentMode.DOCS_ANSWER, TurnIntentExpectedOutput.EXPLANATION
+ if has_workflow and _contains_any(user_message, _EDIT_TERMS):
+ return TurnIntentMode.EDIT, TurnIntentExpectedOutput.WORKFLOW_UPDATE
+ if _contains_any(user_message, _BUILD_TERMS):
+ return TurnIntentMode.BUILD, TurnIntentExpectedOutput.WORKFLOW_DRAFT
+ return None
+
+
+def build_turn_intent(
+ *,
+ user_message: str,
+ workflow_yaml: str,
+ chat_history: list[WorkflowCopilotChatHistoryMessage],
+ global_llm_context: str,
+ request_policy: RequestPolicy,
+ workflow_id: str | None = None,
+ workflow_permanent_id: str | None = None,
+ workflow_run_id: str | None = None,
+ browser_session_id: str | None = None,
+) -> TurnIntent:
+ has_workflow = bool((workflow_yaml or "").strip())
+ has_prior_context = bool((global_llm_context or "").strip())
+ target_entities: dict[str, list[str]] = {}
+ required_context: list[RequiredContextKey] = []
+ reason_codes: list[TurnIntentReasonCode] = [TurnIntentReasonCode.REQUEST_POLICY_DERIVED]
+
+ if workflow_id or workflow_permanent_id or has_workflow:
+ target_entities["workflow"] = [value for value in (workflow_permanent_id, workflow_id) if value]
+ if workflow_run_id:
+ target_entities["run"] = [workflow_run_id]
+ if request_policy.credential_refs:
+ target_entities["credential"] = list(request_policy.credential_refs)
+
+ if has_workflow:
+ required_context.append(RequiredContextKey.CURRENT_WORKFLOW)
+ reason_codes.append(TurnIntentReasonCode.WORKFLOW_CONTEXT_PRESENT)
+ if _has_latest_assistant_turn(chat_history):
+ required_context.append(RequiredContextKey.LATEST_ASSISTANT_PROPOSAL)
+ reason_codes.append(TurnIntentReasonCode.CHAT_HISTORY_PRESENT)
+ if workflow_run_id:
+ required_context.append(RequiredContextKey.LATEST_RUN_RESULT)
+ reason_codes.append(TurnIntentReasonCode.RUN_CONTEXT_PRESENT)
+ if browser_session_id:
+ required_context.append(RequiredContextKey.BROWSER_STATE)
+ reason_codes.append(TurnIntentReasonCode.BROWSER_CONTEXT_PRESENT)
+ if request_policy.credential_input_kind != "none" or request_policy.resolved_credentials:
+ required_context.append(RequiredContextKey.CREDENTIAL_METADATA)
+
+ authority = TurnIntentAuthority(
+ may_update_workflow=request_policy.allow_update_workflow,
+ may_run_blocks=request_policy.allow_run_blocks and request_policy.testing_intent != "skip_test",
+ may_answer_without_mutation=True,
+ requires_user_input=request_policy.user_response_policy == "ask_clarification",
+ )
+ mode = TurnIntentMode.UNKNOWN
+ expected_output = TurnIntentExpectedOutput.EXPLANATION
+ confidence = 0.2 if (has_workflow or has_prior_context or chat_history) else 0.0
+ missing_context_question = None
+
+ if request_policy.user_response_policy == "ask_clarification":
+ mode = TurnIntentMode.CLARIFY
+ expected_output = TurnIntentExpectedOutput.CLARIFICATION
+ confidence = 0.8
+ missing_context_question = request_policy.clarification_question
+ reason_codes.append(TurnIntentReasonCode.REQUEST_POLICY_CLARIFICATION)
+ elif request_policy.testing_intent == "skip_test":
+ mode = TurnIntentMode.DRAFT_ONLY
+ expected_output = TurnIntentExpectedOutput.WORKFLOW_DRAFT
+ confidence = 0.6
+ reason_codes.append(TurnIntentReasonCode.TESTING_INTENT_SKIP_TEST)
+ elif keyword_mode := _mode_from_keywords(user_message, has_workflow=has_workflow):
+ mode, expected_output = keyword_mode
+ confidence = 0.35
+ reason_codes.append(TurnIntentReasonCode.KEYWORD_HEURISTIC)
+
+ if mode == TurnIntentMode.DOCS_ANSWER:
+ required_context.append(RequiredContextKey.DOCS_CONTEXT)
+ elif mode == TurnIntentMode.DIAGNOSE and RequiredContextKey.LATEST_RUN_RESULT not in required_context:
+ required_context.append(RequiredContextKey.LATEST_RUN_RESULT)
+
+ return TurnIntent(
+ mode=mode,
+ user_goal=_normalize_user_goal(user_message),
+ target_entities=target_entities,
+ required_context=required_context,
+ authority=authority,
+ expected_output=expected_output,
+ confidence=confidence,
+ reason_codes=reason_codes,
+ missing_context_question=missing_context_question,
+ )
diff --git a/tests/unit/test_copilot_turn_intent.py b/tests/unit/test_copilot_turn_intent.py
new file mode 100644
index 000000000..274584ba8
--- /dev/null
+++ b/tests/unit/test_copilot_turn_intent.py
@@ -0,0 +1,170 @@
+from __future__ import annotations
+
+from datetime import datetime, timezone
+from unittest.mock import MagicMock
+
+import pytest
+from pydantic import ValidationError
+
+from skyvern.forge.sdk.copilot.agent import RequestPolicyGuardrailInputs, _store_request_policy_on_context
+from skyvern.forge.sdk.copilot.context import CopilotContext
+from skyvern.forge.sdk.copilot.request_policy import RequestPolicy
+from skyvern.forge.sdk.copilot.turn_intent import (
+ RequiredContextKey,
+ TurnIntent,
+ TurnIntentAuthority,
+ TurnIntentMode,
+ TurnIntentReasonCode,
+ build_turn_intent,
+)
+from skyvern.forge.sdk.schemas.workflow_copilot import (
+ WorkflowCopilotChatHistoryMessage,
+ WorkflowCopilotChatSender,
+)
+
+
+def _user_message(content: str) -> WorkflowCopilotChatHistoryMessage:
+ return WorkflowCopilotChatHistoryMessage(
+ sender=WorkflowCopilotChatSender.USER,
+ content=content,
+ created_at=datetime.now(timezone.utc),
+ )
+
+
+def test_turn_intent_defaults_to_unknown_shadow_contract() -> None:
+ intent = TurnIntent()
+
+ assert intent.mode == TurnIntentMode.UNKNOWN
+ assert intent.user_goal == ""
+ assert intent.authority == TurnIntentAuthority()
+ assert intent.required_context == []
+ assert intent.reason_codes == [TurnIntentReasonCode.DEFAULT_UNKNOWN]
+
+
+def test_turn_intent_validates_confidence_bounds() -> None:
+ with pytest.raises(ValidationError):
+ TurnIntent(confidence=1.1)
+
+
+def test_turn_intent_trace_data_omits_raw_goal_and_target_values() -> None:
+ intent = TurnIntent(
+ mode=TurnIntentMode.EDIT,
+ user_goal="Use password: hunter2 to update the workflow",
+ target_entities={"workflow": ["w_123"], "credential": ["cred_sensitive"]},
+ required_context=[RequiredContextKey.CURRENT_WORKFLOW, RequiredContextKey.CREDENTIAL_METADATA],
+ authority=TurnIntentAuthority(may_update_workflow=True, may_run_blocks=False),
+ confidence=0.75,
+ missing_context_question="Which saved credential should I use?",
+ reason_codes=[TurnIntentReasonCode.REQUEST_POLICY_DERIVED],
+ )
+
+ trace_data = intent.to_trace_data()
+
+ assert trace_data == {
+ "mode": "edit",
+ "expected_output": "workflow_update",
+ "required_context": ["current_workflow", "credential_metadata"],
+ "may_update_workflow": True,
+ "may_run_blocks": False,
+ "may_answer_without_mutation": True,
+ "requires_user_input": False,
+ "confidence": 0.75,
+ "reason_codes": ["request_policy_derived"],
+ "target_entity_types": ["credential", "workflow"],
+ "has_missing_context_question": True,
+ }
+ assert "hunter2" not in repr(trace_data)
+ assert "cred_sensitive" not in repr(trace_data)
+
+
+def test_build_turn_intent_uses_request_policy_clarification_without_changing_policy() -> None:
+ policy = RequestPolicy(
+ user_response_policy="ask_clarification",
+ allow_update_workflow=False,
+ allow_run_blocks=False,
+ clarification_question="Which page should I target?",
+ clarification_reason="missing_target_context",
+ )
+
+ intent = build_turn_intent(
+ user_message="Update this workflow",
+ workflow_yaml="blocks: []",
+ chat_history=[_user_message("Build a workflow")],
+ global_llm_context="",
+ request_policy=policy,
+ )
+
+ assert intent.mode == TurnIntentMode.CLARIFY
+ assert intent.authority.requires_user_input is True
+ assert intent.authority.may_update_workflow is False
+ assert intent.authority.may_run_blocks is False
+ assert intent.missing_context_question == "Which page should I target?"
+ assert RequiredContextKey.CURRENT_WORKFLOW in intent.required_context
+ assert TurnIntentReasonCode.REQUEST_POLICY_CLARIFICATION in intent.reason_codes
+
+
+def test_build_turn_intent_redacts_user_goal() -> None:
+ intent = build_turn_intent(
+ user_message="Use password: hunter2 and build the workflow",
+ workflow_yaml="",
+ chat_history=[],
+ global_llm_context="",
+ request_policy=RequestPolicy(),
+ )
+
+ assert "hunter2" not in intent.user_goal
+ assert "[REDACTED_SECRET]" in intent.user_goal
+
+
+def test_build_turn_intent_marks_docs_context_for_docs_answer() -> None:
+ intent = build_turn_intent(
+ user_message="Why does a loop need a condition?",
+ workflow_yaml="",
+ chat_history=[],
+ global_llm_context="",
+ request_policy=RequestPolicy(),
+ )
+
+ assert intent.mode == TurnIntentMode.DOCS_ANSWER
+ assert RequiredContextKey.DOCS_CONTEXT in intent.required_context
+
+
+def test_build_turn_intent_marks_run_context_for_diagnose() -> None:
+ intent = build_turn_intent(
+ user_message="Diagnose the failed run",
+ workflow_yaml="blocks: []",
+ chat_history=[],
+ global_llm_context="",
+ request_policy=RequestPolicy(),
+ )
+
+ assert intent.mode == TurnIntentMode.DIAGNOSE
+ assert RequiredContextKey.LATEST_RUN_RESULT in intent.required_context
+
+
+def test_store_request_policy_attaches_turn_intent_to_context() -> None:
+ ctx = CopilotContext(
+ organization_id="org-1",
+ workflow_id="wf-1",
+ workflow_permanent_id="wfp-1",
+ workflow_yaml="",
+ browser_session_id=None,
+ stream=MagicMock(),
+ )
+ policy = RequestPolicy(allow_update_workflow=True, allow_run_blocks=False)
+ inputs = RequestPolicyGuardrailInputs(
+ user_message="Explain why the last run failed",
+ workflow_yaml="blocks: []",
+ chat_history_text="",
+ chat_history_messages=[],
+ global_llm_context="",
+ organization_id="org-1",
+ handler=None,
+ previous_user_message=None,
+ )
+
+ _store_request_policy_on_context(ctx, policy, inputs)
+
+ assert ctx.turn_intent is not None
+ assert ctx.turn_intent.authority.may_update_workflow is True
+ assert ctx.turn_intent.authority.may_run_blocks is False