Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
SKY-9975 Add TurnIntent shadow contract (#6063)
Andrew Neilson7808fb1c0dd9
diff --git a/skyvern/forge/sdk/copilot/agent.py b/skyvern/forge/sdk/copilot/agent.pyindex 9a6180f87..8c76e7e66 100644--- a/skyvern/forge/sdk/copilot/agent.py+++ b/skyvern/forge/sdk/copilot/agent.py@@ -53,6 +53,7 @@ from skyvern.forge.sdk.copilot.output_utils import ()from skyvern.forge.sdk.copilot.request_policy import RequestPolicy, build_request_policy, redact_raw_secrets_for_promptfrom skyvern.forge.sdk.copilot.tracing_setup import _copilot_model_name, ensure_tracing_initialized, is_tracing_enabled+from skyvern.forge.sdk.copilot.turn_intent import TurnIntent, build_turn_intentfrom skyvern.forge.sdk.schemas.persistent_browser_sessions import is_final_statusfrom skyvern.forge.sdk.schemas.workflow_copilot import (WorkflowCopilotChatHistoryMessage,@@ -130,6 +131,10 @@ class RequestPolicyGuardrailInputs:organization_id: strhandler: Anyprevious_user_message: str | None = None+ workflow_id: str | None = None+ workflow_permanent_id: str | None = None+ workflow_run_id: str | None = None+ browser_session_id: str | None = None_BLOCK_GOAL_CONTEXT_PREAMBLE = (@@ -272,6 +277,27 @@ def _store_request_policy_on_context(chat_history_text=policy_chat_history_text,global_llm_context=policy_inputs.global_llm_context,)+ ctx.turn_intent = build_turn_intent(+ user_message=policy_inputs.user_message,+ workflow_yaml=policy_inputs.workflow_yaml,+ chat_history=policy_inputs.chat_history_messages,+ global_llm_context=policy_inputs.global_llm_context,+ request_policy=policy,+ workflow_id=policy_inputs.workflow_id,+ workflow_permanent_id=policy_inputs.workflow_permanent_id,+ workflow_run_id=policy_inputs.workflow_run_id,+ browser_session_id=policy_inputs.browser_session_id,+ )+++def _turn_intent_log_fields(intent: TurnIntent | None) -> dict[str, Any]:+ if not isinstance(intent, TurnIntent):+ return {}+ return {f"turn_intent_{key}": value for key, value in intent.to_trace_data().items()}+++def _turn_intent_trace_fields(intent: TurnIntent | None) -> dict[str, str]:+ return {key: str(value) for key, value in _turn_intent_log_fields(intent).items()}def _build_system_prompt(@@ -1172,12 +1198,14 @@ def _build_copilot_input_guardrails(_store_request_policy_on_context(ctx, policy, policy_inputs)blocked = isinstance(policy, RequestPolicy) and policy.user_response_policy == "ask_clarification"if isinstance(policy, RequestPolicy):+ turn_intent = ctx.turn_intent if isinstance(ctx, CopilotContext) else Nonetrace_data = {"surface": "agent_input","policy_present": True,"blocked": blocked,"user_response_policy": policy.user_response_policy,**policy.to_trace_data(),+ **_turn_intent_log_fields(turn_intent),}else:trace_data = {"surface": "agent_input", "blocked": False, "policy_present": False}@@ -1386,6 +1414,10 @@ async def _run_copilot_turn_impl(organization_id=organization_id,handler=_resolve_request_policy_handler(llm_api_handler),previous_user_message=previous_user_message,+ workflow_id=chat_request.workflow_id,+ workflow_permanent_id=chat_request.workflow_permanent_id,+ workflow_run_id=getattr(chat_request, "workflow_run_id", None),+ browser_session_id=getattr(chat_request, "browser_session_id", None),)request_policy_guardrails = _build_copilot_input_guardrails(InputGuardrail,@@ -1502,6 +1534,7 @@ async def _run_copilot_turn_impl("llm_key": llm_key,"user_message_len": str(len(user_message)),**{f"request_policy_{key}": str(value) for key, value in request_policy.to_trace_data().items()},+ **_turn_intent_trace_fields(ctx.turn_intent),},)diff --git a/skyvern/forge/sdk/copilot/context.py b/skyvern/forge/sdk/copilot/context.pyindex 5b5661d53..48ba0ea2b 100644--- a/skyvern/forge/sdk/copilot/context.py+++ b/skyvern/forge/sdk/copilot/context.py@@ -17,6 +17,7 @@ COPILOT_RESPONSE_TYPES: tuple[ResponseType, ...] = get_args(ResponseType)if TYPE_CHECKING:from skyvern.forge.sdk.copilot.narration import NarratorStatefrom skyvern.forge.sdk.copilot.request_policy import RequestPolicy+ from skyvern.forge.sdk.copilot.turn_intent import TurnIntentclass UrlVisit(BaseModel):@@ -167,6 +168,7 @@ class CopilotContext(AgentContext):block_goal_main_goal: str = ""allow_untested_workflow_draft: bool = Falserequest_policy: RequestPolicy | None = None+ turn_intent: TurnIntent | None = None# Tool trackingconsecutive_tool_tracker: list[str] = field(default_factory=list)diff --git a/skyvern/forge/sdk/copilot/turn_intent.py b/skyvern/forge/sdk/copilot/turn_intent.pynew file mode 100644index 000000000..cca9c03f3--- /dev/null+++ b/skyvern/forge/sdk/copilot/turn_intent.py@@ -0,0 +1,253 @@+from __future__ import annotations++from enum import StrEnum+from typing import Any++from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator++from skyvern.forge.sdk.copilot.request_policy import RequestPolicy, redact_raw_secrets_for_prompt+from skyvern.forge.sdk.schemas.workflow_copilot import (+ WorkflowCopilotChatHistoryMessage,+ WorkflowCopilotChatSender,+)+++class TurnIntentMode(StrEnum):+ BUILD = "build"+ EDIT = "edit"+ DIAGNOSE = "diagnose"+ DOCS_ANSWER = "docs_answer"+ DRAFT_ONLY = "draft_only"+ CLARIFY = "clarify"+ REFUSE = "refuse"+ UNKNOWN = "unknown"+++class RequiredContextKey(StrEnum):+ CURRENT_WORKFLOW = "current_workflow"+ PROPOSED_WORKFLOW = "proposed_workflow"+ LATEST_ASSISTANT_PROPOSAL = "latest_assistant_proposal"+ LATEST_RUN_RESULT = "latest_run_result"+ CREDENTIAL_METADATA = "credential_metadata"+ DOCS_CONTEXT = "docs_context"+ BROWSER_STATE = "browser_state"+++class TurnIntentExpectedOutput(StrEnum):+ WORKFLOW_UPDATE = "workflow_update"+ WORKFLOW_DRAFT = "workflow_draft"+ RUN_RESULT = "run_result"+ EXPLANATION = "explanation"+ CLARIFICATION = "clarification"+ REFUSAL = "refusal"+++class TurnIntentReasonCode(StrEnum):+ DEFAULT_UNKNOWN = "default_unknown"+ REQUEST_POLICY_DERIVED = "request_policy_derived"+ REQUEST_POLICY_CLARIFICATION = "request_policy_clarification"+ TESTING_INTENT_SKIP_TEST = "testing_intent_skip_test"+ WORKFLOW_CONTEXT_PRESENT = "workflow_context_present"+ CHAT_HISTORY_PRESENT = "chat_history_present"+ RUN_CONTEXT_PRESENT = "run_context_present"+ BROWSER_CONTEXT_PRESENT = "browser_context_present"+ KEYWORD_HEURISTIC = "keyword_heuristic"+++class TurnIntentAuthority(BaseModel):+ model_config = ConfigDict(extra="forbid")++ may_update_workflow: bool = False+ may_run_blocks: bool = False+ may_answer_without_mutation: bool = True+ requires_user_input: bool = False+++class TurnIntent(BaseModel):+ model_config = ConfigDict(extra="forbid")++ mode: TurnIntentMode = TurnIntentMode.UNKNOWN+ user_goal: str = ""+ target_entities: dict[str, list[str]] = Field(default_factory=dict)+ required_context: list[RequiredContextKey] = Field(default_factory=list)+ authority: TurnIntentAuthority = Field(default_factory=TurnIntentAuthority)+ expected_output: TurnIntentExpectedOutput = TurnIntentExpectedOutput.EXPLANATION+ confidence: float = Field(default=0.0, ge=0.0, le=1.0)+ reason_codes: list[TurnIntentReasonCode] = Field(default_factory=lambda: [TurnIntentReasonCode.DEFAULT_UNKNOWN])+ missing_context_question: str | None = None++ @field_validator("required_context", mode="after")+ @classmethod+ def _dedupe_required_context(cls, value: list[RequiredContextKey]) -> list[RequiredContextKey]:+ return list(dict.fromkeys(value))++ @field_validator("reason_codes", mode="after")+ @classmethod+ def _dedupe_reason_codes(cls, value: list[TurnIntentReasonCode]) -> list[TurnIntentReasonCode]:+ return list(dict.fromkeys(value)) or [TurnIntentReasonCode.DEFAULT_UNKNOWN]++ @field_validator("target_entities", mode="after")+ @classmethod+ def _dedupe_target_entities(cls, value: dict[str, list[str]]) -> dict[str, list[str]]:+ return {+ str(entity_type): list(dict.fromkeys(str(entity).strip() for entity in entities if str(entity).strip()))+ for entity_type, entities in value.items()+ if str(entity_type).strip()+ }++ @model_validator(mode="after")+ def _align_expected_output_with_mode(self) -> TurnIntent:+ if self.expected_output != TurnIntentExpectedOutput.EXPLANATION:+ return self+ expected_output_by_mode = {+ TurnIntentMode.BUILD: TurnIntentExpectedOutput.WORKFLOW_DRAFT,+ TurnIntentMode.EDIT: TurnIntentExpectedOutput.WORKFLOW_UPDATE,+ TurnIntentMode.DRAFT_ONLY: TurnIntentExpectedOutput.WORKFLOW_DRAFT,+ TurnIntentMode.CLARIFY: TurnIntentExpectedOutput.CLARIFICATION,+ TurnIntentMode.REFUSE: TurnIntentExpectedOutput.REFUSAL,+ }+ if mapped_expected_output := expected_output_by_mode.get(self.mode):+ self.expected_output = mapped_expected_output+ return self++ def to_trace_data(self) -> dict[str, Any]:+ data: dict[str, Any] = {+ "mode": self.mode.value,+ "expected_output": self.expected_output.value,+ "required_context": [key.value for key in self.required_context],+ "may_update_workflow": self.authority.may_update_workflow,+ "may_run_blocks": self.authority.may_run_blocks,+ "may_answer_without_mutation": self.authority.may_answer_without_mutation,+ "requires_user_input": self.authority.requires_user_input,+ "confidence": self.confidence,+ "reason_codes": [reason.value for reason in self.reason_codes],+ }+ if self.target_entities:+ data["target_entity_types"] = sorted(self.target_entities)+ if self.missing_context_question:+ data["has_missing_context_question"] = True+ return data+++_GOAL_MAX_CHARS = 240+_BUILD_TERMS = ("build", "create", "make", "generate")+_EDIT_TERMS = ("edit", "update", "change", "modify", "replace", "fix")+_DIAGNOSE_TERMS = ("debug", "diagnose", "failed", "failure", "error", "result")+_DOCS_TERMS = ("explain", "how do", "how does", "what is", "what are", "why", "docs", "documentation")+++def _normalize_user_goal(user_message: str) -> str:+ goal = redact_raw_secrets_for_prompt((user_message or "").strip())+ if len(goal) <= _GOAL_MAX_CHARS:+ return goal+ return goal[: _GOAL_MAX_CHARS - 3].rstrip() + "..."+++def _has_latest_assistant_turn(chat_history: list[WorkflowCopilotChatHistoryMessage]) -> bool:+ return any(+ message.sender == WorkflowCopilotChatSender.AI and (message.content or "").strip() for message in chat_history+ )+++def _contains_any(text: str, terms: tuple[str, ...]) -> bool:+ normalized = text.lower()+ return any(term in normalized for term in terms)+++def _mode_from_keywords(+ user_message: str, *, has_workflow: bool+) -> tuple[TurnIntentMode, TurnIntentExpectedOutput] | None:+ if _contains_any(user_message, _DIAGNOSE_TERMS):+ return TurnIntentMode.DIAGNOSE, TurnIntentExpectedOutput.RUN_RESULT+ if _contains_any(user_message, _DOCS_TERMS):+ return TurnIntentMode.DOCS_ANSWER, TurnIntentExpectedOutput.EXPLANATION+ if has_workflow and _contains_any(user_message, _EDIT_TERMS):+ return TurnIntentMode.EDIT, TurnIntentExpectedOutput.WORKFLOW_UPDATE+ if _contains_any(user_message, _BUILD_TERMS):+ return TurnIntentMode.BUILD, TurnIntentExpectedOutput.WORKFLOW_DRAFT+ return None+++def build_turn_intent(+ *,+ user_message: str,+ workflow_yaml: str,+ chat_history: list[WorkflowCopilotChatHistoryMessage],+ global_llm_context: str,+ request_policy: RequestPolicy,+ workflow_id: str | None = None,+ workflow_permanent_id: str | None = None,+ workflow_run_id: str | None = None,+ browser_session_id: str | None = None,+) -> TurnIntent:+ has_workflow = bool((workflow_yaml or "").strip())+ has_prior_context = bool((global_llm_context or "").strip())+ target_entities: dict[str, list[str]] = {}+ required_context: list[RequiredContextKey] = []+ reason_codes: list[TurnIntentReasonCode] = [TurnIntentReasonCode.REQUEST_POLICY_DERIVED]++ if workflow_id or workflow_permanent_id or has_workflow:+ target_entities["workflow"] = [value for value in (workflow_permanent_id, workflow_id) if value]+ if workflow_run_id:+ target_entities["run"] = [workflow_run_id]+ if request_policy.credential_refs:+ target_entities["credential"] = list(request_policy.credential_refs)++ if has_workflow:+ required_context.append(RequiredContextKey.CURRENT_WORKFLOW)+ reason_codes.append(TurnIntentReasonCode.WORKFLOW_CONTEXT_PRESENT)+ if _has_latest_assistant_turn(chat_history):+ required_context.append(RequiredContextKey.LATEST_ASSISTANT_PROPOSAL)+ reason_codes.append(TurnIntentReasonCode.CHAT_HISTORY_PRESENT)+ if workflow_run_id:+ required_context.append(RequiredContextKey.LATEST_RUN_RESULT)+ reason_codes.append(TurnIntentReasonCode.RUN_CONTEXT_PRESENT)+ if browser_session_id:+ required_context.append(RequiredContextKey.BROWSER_STATE)+ reason_codes.append(TurnIntentReasonCode.BROWSER_CONTEXT_PRESENT)+ if request_policy.credential_input_kind != "none" or request_policy.resolved_credentials:+ required_context.append(RequiredContextKey.CREDENTIAL_METADATA)++ authority = TurnIntentAuthority(+ may_update_workflow=request_policy.allow_update_workflow,+ may_run_blocks=request_policy.allow_run_blocks and request_policy.testing_intent != "skip_test",+ may_answer_without_mutation=True,+ requires_user_input=request_policy.user_response_policy == "ask_clarification",+ )+ mode = TurnIntentMode.UNKNOWN+ expected_output = TurnIntentExpectedOutput.EXPLANATION+ confidence = 0.2 if (has_workflow or has_prior_context or chat_history) else 0.0+ missing_context_question = None++ if request_policy.user_response_policy == "ask_clarification":+ mode = TurnIntentMode.CLARIFY+ expected_output = TurnIntentExpectedOutput.CLARIFICATION+ confidence = 0.8+ missing_context_question = request_policy.clarification_question+ reason_codes.append(TurnIntentReasonCode.REQUEST_POLICY_CLARIFICATION)+ elif request_policy.testing_intent == "skip_test":+ mode = TurnIntentMode.DRAFT_ONLY+ expected_output = TurnIntentExpectedOutput.WORKFLOW_DRAFT+ confidence = 0.6+ reason_codes.append(TurnIntentReasonCode.TESTING_INTENT_SKIP_TEST)+ elif keyword_mode := _mode_from_keywords(user_message, has_workflow=has_workflow):+ mode, expected_output = keyword_mode+ confidence = 0.35+ reason_codes.append(TurnIntentReasonCode.KEYWORD_HEURISTIC)++ if mode == TurnIntentMode.DOCS_ANSWER:+ required_context.append(RequiredContextKey.DOCS_CONTEXT)+ elif mode == TurnIntentMode.DIAGNOSE and RequiredContextKey.LATEST_RUN_RESULT not in required_context:+ required_context.append(RequiredContextKey.LATEST_RUN_RESULT)++ return TurnIntent(+ mode=mode,+ user_goal=_normalize_user_goal(user_message),+ target_entities=target_entities,+ required_context=required_context,+ authority=authority,+ expected_output=expected_output,+ confidence=confidence,+ reason_codes=reason_codes,+ missing_context_question=missing_context_question,+ )diff --git a/tests/unit/test_copilot_turn_intent.py b/tests/unit/test_copilot_turn_intent.pynew file mode 100644index 000000000..274584ba8--- /dev/null+++ b/tests/unit/test_copilot_turn_intent.py@@ -0,0 +1,170 @@+from __future__ import annotations++from datetime import datetime, timezone+from unittest.mock import MagicMock++import pytest+from pydantic import ValidationError++from skyvern.forge.sdk.copilot.agent import RequestPolicyGuardrailInputs, _store_request_policy_on_context+from skyvern.forge.sdk.copilot.context import CopilotContext+from skyvern.forge.sdk.copilot.request_policy import RequestPolicy+from skyvern.forge.sdk.copilot.turn_intent import (+ RequiredContextKey,+ TurnIntent,+ TurnIntentAuthority,+ TurnIntentMode,+ TurnIntentReasonCode,+ build_turn_intent,+)+from skyvern.forge.sdk.schemas.workflow_copilot import (+ WorkflowCopilotChatHistoryMessage,+ WorkflowCopilotChatSender,+)+++def _user_message(content: str) -> WorkflowCopilotChatHistoryMessage:+ return WorkflowCopilotChatHistoryMessage(+ sender=WorkflowCopilotChatSender.USER,+ content=content,+ created_at=datetime.now(timezone.utc),+ )+++def test_turn_intent_defaults_to_unknown_shadow_contract() -> None:+ intent = TurnIntent()++ assert intent.mode == TurnIntentMode.UNKNOWN+ assert intent.user_goal == ""+ assert intent.authority == TurnIntentAuthority()+ assert intent.required_context == []+ assert intent.reason_codes == [TurnIntentReasonCode.DEFAULT_UNKNOWN]+++def test_turn_intent_validates_confidence_bounds() -> None:+ with pytest.raises(ValidationError):+ TurnIntent(confidence=1.1)+++def test_turn_intent_trace_data_omits_raw_goal_and_target_values() -> None:+ intent = TurnIntent(+ mode=TurnIntentMode.EDIT,+ user_goal="Use password: hunter2 to update the workflow",+ target_entities={"workflow": ["w_123"], "credential": ["cred_sensitive"]},+ required_context=[RequiredContextKey.CURRENT_WORKFLOW, RequiredContextKey.CREDENTIAL_METADATA],+ authority=TurnIntentAuthority(may_update_workflow=True, may_run_blocks=False),+ confidence=0.75,+ missing_context_question="Which saved credential should I use?",+ reason_codes=[TurnIntentReasonCode.REQUEST_POLICY_DERIVED],+ )++ trace_data = intent.to_trace_data()++ assert trace_data == {+ "mode": "edit",+ "expected_output": "workflow_update",+ "required_context": ["current_workflow", "credential_metadata"],+ "may_update_workflow": True,+ "may_run_blocks": False,+ "may_answer_without_mutation": True,+ "requires_user_input": False,+ "confidence": 0.75,+ "reason_codes": ["request_policy_derived"],+ "target_entity_types": ["credential", "workflow"],+ "has_missing_context_question": True,+ }+ assert "hunter2" not in repr(trace_data)+ assert "cred_sensitive" not in repr(trace_data)+++def test_build_turn_intent_uses_request_policy_clarification_without_changing_policy() -> None:+ policy = RequestPolicy(+ user_response_policy="ask_clarification",+ allow_update_workflow=False,+ allow_run_blocks=False,+ clarification_question="Which page should I target?",+ clarification_reason="missing_target_context",+ )++ intent = build_turn_intent(+ user_message="Update this workflow",+ workflow_yaml="blocks: []",+ chat_history=[_user_message("Build a workflow")],+ global_llm_context="",+ request_policy=policy,+ )++ assert intent.mode == TurnIntentMode.CLARIFY+ assert intent.authority.requires_user_input is True+ assert intent.authority.may_update_workflow is False+ assert intent.authority.may_run_blocks is False+ assert intent.missing_context_question == "Which page should I target?"+ assert RequiredContextKey.CURRENT_WORKFLOW in intent.required_context+ assert TurnIntentReasonCode.REQUEST_POLICY_CLARIFICATION in intent.reason_codes+++def test_build_turn_intent_redacts_user_goal() -> None:+ intent = build_turn_intent(+ user_message="Use password: hunter2 and build the workflow",+ workflow_yaml="",+ chat_history=[],+ global_llm_context="",+ request_policy=RequestPolicy(),+ )++ assert "hunter2" not in intent.user_goal+ assert "[REDACTED_SECRET]" in intent.user_goal+++def test_build_turn_intent_marks_docs_context_for_docs_answer() -> None:+ intent = build_turn_intent(+ user_message="Why does a loop need a condition?",+ workflow_yaml="",+ chat_history=[],+ global_llm_context="",+ request_policy=RequestPolicy(),+ )++ assert intent.mode == TurnIntentMode.DOCS_ANSWER+ assert RequiredContextKey.DOCS_CONTEXT in intent.required_context+++def test_build_turn_intent_marks_run_context_for_diagnose() -> None:+ intent = build_turn_intent(+ user_message="Diagnose the failed run",+ workflow_yaml="blocks: []",+ chat_history=[],+ global_llm_context="",+ request_policy=RequestPolicy(),+ )++ assert intent.mode == TurnIntentMode.DIAGNOSE+ assert RequiredContextKey.LATEST_RUN_RESULT in intent.required_context+++def test_store_request_policy_attaches_turn_intent_to_context() -> None:+ ctx = CopilotContext(+ organization_id="org-1",+ workflow_id="wf-1",+ workflow_permanent_id="wfp-1",+ workflow_yaml="",+ browser_session_id=None,+ stream=MagicMock(),+ )+ policy = RequestPolicy(allow_update_workflow=True, allow_run_blocks=False)+ inputs = RequestPolicyGuardrailInputs(+ user_message="Explain why the last run failed",+ workflow_yaml="blocks: []",+ chat_history_text="",+ chat_history_messages=[],+ global_llm_context="",+ organization_id="org-1",+ handler=None,+ previous_user_message=None,+ )++ _store_request_policy_on_context(ctx, policy, inputs)++ assert ctx.turn_intent is not None+ assert ctx.turn_intent.authority.may_update_workflow is True+ assert ctx.turn_intent.authority.may_run_blocks is False