Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
SKY-10984: Preserve run authority on turn-intent timeouts (#6583)
Andrew Neilson93f333e767ed
diff --git a/skyvern/config.py b/skyvern/config.pyindex 8218dddac..598025101 100644--- a/skyvern/config.py+++ b/skyvern/config.py@@ -149,6 +149,7 @@ class Settings(BaseSettings):LOG_LEVEL: str = "INFO"COPILOT_FEASIBILITY_GATE_TIMEOUT_SECONDS: float = 12.0COPILOT_REQUEST_POLICY_CLASSIFIER_TIMEOUT_SECONDS: float = 6.0+ COPILOT_TURN_INTENT_CLASSIFIER_TIMEOUT_SECONDS: float = 12.0COPILOT_COMPLETION_JUDGE_TIMEOUT_SECONDS: float = 12.0# Gate copilot verified-success on per-criterion outcome evidence.# Off restores the prior run-status/suspicious-success gate and classifier prompt.diff --git a/skyvern/forge/sdk/copilot/agent.py b/skyvern/forge/sdk/copilot/agent.pyindex 8e5fec877..88f32a6d9 100644--- a/skyvern/forge/sdk/copilot/agent.py+++ b/skyvern/forge/sdk/copilot/agent.py@@ -122,7 +122,7 @@ from skyvern.forge.sdk.copilot.turn_intent import (NO_MUTATION_TURN_INTENT_MODES,RequiredContextKey,TurnIntent,- TurnIntentClassification,+ TurnIntentClassifierResult,TurnIntentMode,build_turn_intent,classify_turn_intent,@@ -399,7 +399,7 @@ def _store_request_policy_on_context(ctx: CopilotContext,policy: RequestPolicy,policy_inputs: RequestPolicyGuardrailInputs,- turn_intent_classification: TurnIntentClassification | None = None,+ turn_intent_classifier_result: TurnIntentClassifierResult | None = None,) -> None:agent_user_message, policy_chat_history_text = _request_policy_agent_inputs(policy,@@ -426,7 +426,7 @@ def _store_request_policy_on_context(workflow_permanent_id=policy_inputs.workflow_permanent_id,workflow_run_id=policy_inputs.workflow_run_id,browser_session_id=policy_inputs.browser_session_id,- classification=turn_intent_classification,+ classifier_result=turn_intent_classifier_result,)@@ -2595,9 +2595,9 @@ def _build_copilot_input_guardrails(active_criteria=_stored_active_completion_criteria(policy_inputs),)if isinstance(ctx, CopilotContext):- turn_intent_classification = None+ turn_intent_classifier_result = Noneif policy.user_response_policy != "ask_clarification" or policy.raw_secret_handling == "redacted_draft":- turn_intent_classification = await classify_turn_intent(+ turn_intent_classifier_result = await classify_turn_intent(user_message=policy_inputs.user_message,workflow_yaml=policy_inputs.workflow_yaml,chat_history=policy_inputs.chat_history_messages,@@ -2609,7 +2609,7 @@ def _build_copilot_input_guardrails(ctx,policy,policy_inputs,- turn_intent_classification=turn_intent_classification,+ turn_intent_classifier_result=turn_intent_classifier_result,)blocked = isinstance(policy, RequestPolicy) and policy.user_response_policy == "ask_clarification"if isinstance(policy, RequestPolicy):diff --git a/skyvern/forge/sdk/copilot/turn_intent.py b/skyvern/forge/sdk/copilot/turn_intent.pyindex 002abd2a6..8a580816d 100644--- a/skyvern/forge/sdk/copilot/turn_intent.py+++ b/skyvern/forge/sdk/copilot/turn_intent.py@@ -2,6 +2,7 @@ from __future__ import annotationsimport asyncioimport re+import timefrom collections.abc import Mappingfrom difflib import SequenceMatcherfrom enum import StrEnum@@ -113,6 +114,21 @@ class TurnIntentReasonCode(StrEnum):LOW_CONFIDENCE_CLARIFICATION = "low_confidence_clarification"TARGET_ENTITY_RESOLVED = "target_entity_resolved"MISSING_EDIT_TARGET = "missing_edit_target"+ TRANSIENT_CLASSIFIER_FALLBACK = "transient_classifier_fallback"+++class TurnIntentClassifierFailureKind(StrEnum):+ TIMEOUT = "timeout"+ PROVIDER_ERROR = "provider_error"+ MISSING_HANDLER = "missing_handler"+ EMPTY_MESSAGE = "empty_message"+ PROMPT_RENDER_ERROR = "prompt_render_error"+ MALFORMED_OUTPUT = "malformed_output"+++_CLASSIFIER_REASON_CODES = tuple(+ reason for reason in TurnIntentReasonCode if reason != TurnIntentReasonCode.TRANSIENT_CLASSIFIER_FALLBACK+)_DEFAULT_EXPECTED_OUTPUT_BY_MODE: dict[TurnIntentMode, TurnIntentExpectedOutput] = {@@ -252,6 +268,38 @@ class TurnIntentClassification(BaseModel):return data+class TurnIntentClassifierResult(BaseModel):+ model_config = ConfigDict(extra="forbid")++ classification: TurnIntentClassification | None = None+ failure_kind: TurnIntentClassifierFailureKind | None = None++ @model_validator(mode="after")+ def _validate_result_shape(self) -> TurnIntentClassifierResult:+ if (self.classification is None) == (self.failure_kind is None):+ raise ValueError("classifier result must contain exactly one of classification or failure_kind")+ return self++ @classmethod+ def success(cls, classification: TurnIntentClassification) -> TurnIntentClassifierResult:+ return cls(classification=classification)++ @classmethod+ def failure(cls, failure_kind: TurnIntentClassifierFailureKind) -> TurnIntentClassifierResult:+ return cls(failure_kind=failure_kind)++ @property+ def is_success(self) -> bool:+ return self.classification is not None++ @property+ def is_transient_failure(self) -> bool:+ return self.failure_kind in (+ TurnIntentClassifierFailureKind.TIMEOUT,+ TurnIntentClassifierFailureKind.PROVIDER_ERROR,+ )++_GOAL_MAX_CHARS = 240@@ -405,9 +453,11 @@ def _coerce_reason_codes(raw: object) -> list[TurnIntentReasonCode]:reason_codes: list[TurnIntentReasonCode] = []for value in values:try:- reason_codes.append(TurnIntentReasonCode(value))+ reason_code = TurnIntentReasonCode(value)except ValueError:continue+ if reason_code in _CLASSIFIER_REASON_CODES:+ reason_codes.append(reason_code)return list(dict.fromkeys(reason_codes))@@ -519,12 +569,13 @@ async def classify_turn_intent(global_llm_context: str,request_policy: RequestPolicy,handler: LLMAPIHandler | None,-) -> TurnIntentClassification | None:+) -> TurnIntentClassifierResult:if not isinstance(user_message, str) or not user_message.strip():- return None+ LOG.info("turn-intent classifier skipped empty user message")+ return TurnIntentClassifierResult.failure(TurnIntentClassifierFailureKind.EMPTY_MESSAGE)if handler is None:LOG.info("turn-intent classifier has no LLM handler available")- return None+ return TurnIntentClassifierResult.failure(TurnIntentClassifierFailureKind.MISSING_HANDLER)safe_user_message = redact_raw_secrets_for_prompt(user_message)transcript = build_transcript_context(chat_history, safe_user_message)@@ -534,7 +585,7 @@ async def classify_turn_intent(mode_values=", ".join(mode.value for mode in TurnIntentMode),expected_output_values=", ".join(output.value for output in TurnIntentExpectedOutput),required_context_values=", ".join(key.value for key in RequiredContextKey),- reason_code_values=", ".join(reason.value for reason in TurnIntentReasonCode),+ reason_code_values=", ".join(reason.value for reason in _CLASSIFIER_REASON_CODES),user_message=escape_code_fences(safe_user_message),request_policy_summary=escape_code_fences(request_policy.prompt_summary()),workflow_yaml=escape_code_fences(@@ -550,31 +601,53 @@ async def classify_turn_intent()except Exception as exc:LOG.warning("turn-intent classifier prompt render failed", error=str(exc))- return None+ return TurnIntentClassifierResult.failure(TurnIntentClassifierFailureKind.PROMPT_RENDER_ERROR)+ timeout = settings.COPILOT_TURN_INTENT_CLASSIFIER_TIMEOUT_SECONDS+ started_at = time.monotonic()try:raw = await asyncio.wait_for(handler(prompt=prompt, prompt_name=PROMPT_NAME),- timeout=settings.COPILOT_FEASIBILITY_GATE_TIMEOUT_SECONDS,+ timeout=timeout,)except asyncio.TimeoutError:LOG.warning("turn-intent classifier timed out",- timeout=settings.COPILOT_FEASIBILITY_GATE_TIMEOUT_SECONDS,+ failure_kind=TurnIntentClassifierFailureKind.TIMEOUT.value,+ timeout=timeout,)- return None+ return TurnIntentClassifierResult.failure(TurnIntentClassifierFailureKind.TIMEOUT)except Exception as exc:- LOG.warning("turn-intent classifier failed", error=str(exc))- return None+ elapsed_seconds = time.monotonic() - started_at+ timeout_margin = min(0.25, max(0.001, timeout * 0.05))+ if elapsed_seconds + timeout_margin >= timeout:+ LOG.warning(+ "turn-intent classifier timed out",+ failure_kind=TurnIntentClassifierFailureKind.TIMEOUT.value,+ timeout=timeout,+ elapsed_seconds=elapsed_seconds,+ converted_error_type=type(exc).__name__,+ error=str(exc),+ )+ return TurnIntentClassifierResult.failure(TurnIntentClassifierFailureKind.TIMEOUT)+ LOG.warning(+ "turn-intent classifier provider failed",+ failure_kind=TurnIntentClassifierFailureKind.PROVIDER_ERROR.value,+ error=str(exc),+ )+ return TurnIntentClassifierResult.failure(TurnIntentClassifierFailureKind.PROVIDER_ERROR)classification = _turn_intent_classification_from_raw(raw)if classification is None:- LOG.warning("turn-intent classifier returned malformed output")- return None+ LOG.warning(+ "turn-intent classifier returned malformed output",+ failure_kind=TurnIntentClassifierFailureKind.MALFORMED_OUTPUT.value,+ )+ return TurnIntentClassifierResult.failure(TurnIntentClassifierFailureKind.MALFORMED_OUTPUT)with copilot_span("turn_intent_classifier", data=classification.to_trace_data()):LOG.info("turn-intent classifier decision", **classification.to_trace_data())- return classification+ return TurnIntentClassifierResult.success(classification)_PRIOR_RUN_OUTPUT_PREFIX = " output:"@@ -678,7 +751,7 @@ def build_turn_intent(workflow_permanent_id: str | None = None,workflow_run_id: str | None = None,browser_session_id: str | None = None,- classification: TurnIntentClassification | None = None,+ classifier_result: TurnIntentClassifierResult | None = None,) -> TurnIntent:has_workflow = bool((workflow_yaml or "").strip())has_prior_context = bool((global_llm_context or "").strip())@@ -720,6 +793,7 @@ def build_turn_intent(confidence = 0.2 if (has_workflow or has_prior_context or chat_history) else 0.0missing_context_question = None+ classification = classifier_result.classification if classifier_result and classifier_result.is_success else Nonehas_prior_run_signal = workflow_run_id is not None or _has_structured_prior_run_signal(global_llm_context)if request_policy.raw_secret_detected and request_policy.raw_secret_handling != "redacted_draft":@@ -782,7 +856,24 @@ def build_turn_intent(if _user_signals_non_progress(user_message, chat_history):reason_codes.append(TurnIntentReasonCode.USER_NON_PROGRESS)- if mode == TurnIntentMode.UNKNOWN and has_prior_run_signal:+ has_request_policy_update_or_run_authority = authority.may_update_workflow or authority.may_run_blocks+ suppress_prior_run_recovery = (+ mode == TurnIntentMode.UNKNOWN+ and has_prior_run_signal+ and classifier_result is not None+ and classifier_result.is_transient_failure+ and has_request_policy_update_or_run_authority+ )+ if suppress_prior_run_recovery:+ mode = TurnIntentMode.BUILD+ expected_output = (+ TurnIntentExpectedOutput.WORKFLOW_UPDATE+ if has_workflow or has_prior_context or workflow_id or workflow_permanent_id+ else TurnIntentExpectedOutput.WORKFLOW_DRAFT+ )+ confidence = max(confidence, 0.6)+ reason_codes.append(TurnIntentReasonCode.TRANSIENT_CLASSIFIER_FALLBACK)+ elif mode == TurnIntentMode.UNKNOWN and has_prior_run_signal:mode = TurnIntentMode.DIAGNOSEexpected_output = TurnIntentExpectedOutput.RUN_RESULTreason_codes.append(TurnIntentReasonCode.RECOVERY_FROM_RUN_CONTEXT)@@ -803,7 +894,12 @@ def build_turn_intent(authority.requires_user_input = Trueelif mode == TurnIntentMode.DIAGNOSE:authority.may_update_workflow = False- retest_mandated = request_policy.testing_intent == "require_test" and request_policy.allow_run_blocks+ retest_mandated = (+ request_policy.testing_intent == "require_test"+ and request_policy.allow_run_blocks+ and classification is not None+ and classification.mode == TurnIntentMode.DIAGNOSE+ )if retest_mandated:# RequestPolicy owns testing policy; an explicit require_test re-run must not be inverted# by the diagnose classification.diff --git a/tests/unit/test_copilot_turn_intent.py b/tests/unit/test_copilot_turn_intent.pyindex 612eb59a0..0220ced9b 100644--- a/tests/unit/test_copilot_turn_intent.py+++ b/tests/unit/test_copilot_turn_intent.py@@ -1,5 +1,6 @@from __future__ import annotations+import asynciofrom datetime import datetime, timezonefrom types import SimpleNamespacefrom unittest.mock import MagicMock@@ -7,6 +8,7 @@ from unittest.mock import MagicMockimport pytestfrom pydantic import ValidationError+from skyvern.config import settingsfrom skyvern.forge.sdk.copilot.agent import (RequestPolicyGuardrailInputs,_docs_answer_turn_directive,@@ -22,6 +24,8 @@ from skyvern.forge.sdk.copilot.turn_intent import (TurnIntent,TurnIntentAuthority,TurnIntentClassification,+ TurnIntentClassifierFailureKind,+ TurnIntentClassifierResult,TurnIntentExpectedOutput,TurnIntentMode,TurnIntentReasonCode,@@ -61,8 +65,8 @@ def _classification(target_entities: dict[str, list[str]] | None = None,missing_context_question: str | None = None,reason_codes: list[TurnIntentReasonCode] | None = None,-) -> TurnIntentClassification:- return TurnIntentClassification(+) -> TurnIntentClassifierResult:+ classification = TurnIntentClassification(mode=mode,expected_output=expected_output,required_context=required_context or [],@@ -71,6 +75,11 @@ def _classification(missing_context_question=missing_context_question,reason_codes=reason_codes or [],)+ return TurnIntentClassifierResult.success(classification)+++def _classifier_failure(failure_kind: TurnIntentClassifierFailureKind) -> TurnIntentClassifierResult:+ return TurnIntentClassifierResult.failure(failure_kind)def test_turn_intent_defaults_to_unknown_shadow_contract() -> None:@@ -141,7 +150,7 @@ def test_build_turn_intent_applies_llm_build_classification() -> None:global_llm_context="",request_policy=RequestPolicy(),browser_session_id="pbs_123",- classification=_classification(+ classifier_result=_classification(TurnIntentMode.BUILD,required_context=[RequiredContextKey.BROWSER_STATE],confidence=0.84,@@ -164,7 +173,7 @@ def test_build_turn_intent_applies_docs_classification_without_mutation() -> Nonchat_history=[],global_llm_context="",request_policy=RequestPolicy(),- classification=_classification(TurnIntentMode.DOCS_ANSWER),+ classifier_result=_classification(TurnIntentMode.DOCS_ANSWER),)assert intent.mode == TurnIntentMode.DOCS_ANSWER@@ -182,7 +191,7 @@ def test_build_turn_intent_applies_diagnose_classification_and_read_authority()global_llm_context="",request_policy=RequestPolicy(allow_update_workflow=True, allow_run_blocks=True),workflow_run_id="wr_123",- classification=_classification(TurnIntentMode.DIAGNOSE),+ classifier_result=_classification(TurnIntentMode.DIAGNOSE),)assert intent.mode == TurnIntentMode.DIAGNOSE@@ -201,7 +210,7 @@ def test_build_turn_intent_applies_draft_only_classification_without_run_authorichat_history=[],global_llm_context="",request_policy=RequestPolicy(allow_update_workflow=True, allow_run_blocks=True),- classification=_classification(TurnIntentMode.DRAFT_ONLY),+ classifier_result=_classification(TurnIntentMode.DRAFT_ONLY),)assert intent.mode == TurnIntentMode.DRAFT_ONLY@@ -231,7 +240,7 @@ def test_build_turn_intent_llm_diagnose_outranks_skip_test_policy() -> None:global_llm_context="",request_policy=RequestPolicy(testing_intent="skip_test", allow_update_workflow=True, allow_run_blocks=False),workflow_run_id="wr_123",- classification=_classification(TurnIntentMode.DIAGNOSE),+ classifier_result=_classification(TurnIntentMode.DIAGNOSE),)assert intent.mode == TurnIntentMode.DIAGNOSE@@ -248,7 +257,7 @@ def test_build_turn_intent_diagnose_with_require_test_keeps_run_authority() -> Nglobal_llm_context="",request_policy=RequestPolicy(testing_intent="require_test", allow_update_workflow=True, allow_run_blocks=True),workflow_run_id="wr_123",- classification=_classification(TurnIntentMode.DIAGNOSE),+ classifier_result=_classification(TurnIntentMode.DIAGNOSE),)assert intent.mode == TurnIntentMode.DIAGNOSE@@ -267,7 +276,7 @@ def test_build_turn_intent_diagnose_without_require_test_stays_answer_only() ->global_llm_context="",request_policy=RequestPolicy(allow_update_workflow=False, allow_run_blocks=False),workflow_run_id="wr_123",- classification=_classification(TurnIntentMode.DIAGNOSE),+ classifier_result=_classification(TurnIntentMode.DIAGNOSE),)assert intent.mode == TurnIntentMode.DIAGNOSE@@ -292,7 +301,7 @@ def test_build_turn_intent_uses_request_policy_clarification_over_llm_classificachat_history=[_user_message("Build a workflow")],global_llm_context="",request_policy=policy,- classification=_classification(TurnIntentMode.EDIT),+ classifier_result=_classification(TurnIntentMode.EDIT),)assert intent.mode == TurnIntentMode.CLARIFY@@ -321,7 +330,7 @@ def test_build_turn_intent_routes_raw_secret_to_refuse_over_llm_classification()chat_history=[],global_llm_context="",request_policy=policy,- classification=_classification(TurnIntentMode.BUILD),+ classifier_result=_classification(TurnIntentMode.BUILD),)assert intent.mode == TurnIntentMode.REFUSE@@ -340,7 +349,7 @@ def test_build_turn_intent_redacts_user_goal() -> None:chat_history=[],global_llm_context="",request_policy=RequestPolicy(),- classification=_classification(TurnIntentMode.BUILD),+ classifier_result=_classification(TurnIntentMode.BUILD),)assert "hunter2" not in intent.user_goal@@ -355,7 +364,7 @@ def test_build_turn_intent_low_confidence_mutating_classification_clarifies(modechat_history=[],global_llm_context="",request_policy=RequestPolicy(),- classification=_classification(mode, confidence=0.35, target_entities={"workflow_change": ["update_it"]}),+ classifier_result=_classification(mode, confidence=0.35, target_entities={"workflow_change": ["update_it"]}),)assert intent.mode == TurnIntentMode.CLARIFY@@ -371,7 +380,7 @@ def test_build_turn_intent_targetless_edit_classification_clarifies() -> None:chat_history=[],global_llm_context="",request_policy=RequestPolicy(),- classification=_classification(TurnIntentMode.EDIT, confidence=0.82),+ classifier_result=_classification(TurnIntentMode.EDIT, confidence=0.82),)assert intent.mode == TurnIntentMode.CLARIFY@@ -388,7 +397,7 @@ def test_build_turn_intent_allows_clear_workflow_change_edit_classification() ->chat_history=[],global_llm_context="",request_policy=RequestPolicy(allow_run_blocks=False),- classification=_classification(+ classifier_result=_classification(TurnIntentMode.EDIT,confidence=0.82,target_entities={"workflow_change": ["add_invoice_download_step"]},@@ -423,7 +432,7 @@ def test_build_turn_intent_resolves_classifier_block_targets(block_ref: str, expchat_history=[],global_llm_context="",request_policy=RequestPolicy(allow_run_blocks=False),- classification=_classification(+ classifier_result=_classification(TurnIntentMode.EDIT,confidence=0.65,target_entities={"block": [block_ref]},@@ -449,7 +458,7 @@ def test_build_turn_intent_preserves_classifier_unresolved_block_targets() -> Nochat_history=[],global_llm_context="",request_policy=RequestPolicy(allow_run_blocks=False),- classification=_classification(+ classifier_result=_classification(TurnIntentMode.CLARIFY,confidence=0.35,target_entities={"block": ["login block"]},@@ -503,6 +512,144 @@ def test_build_turn_intent_unknown_with_persisted_prior_run_signal_upgrades_to_dassert intent.authority.may_read_run_context is True+ "failure_kind",+ [TurnIntentClassifierFailureKind.TIMEOUT, TurnIntentClassifierFailureKind.PROVIDER_ERROR],+)+def test_build_turn_intent_transient_classifier_failure_suppresses_prior_run_recovery(+ failure_kind: TurnIntentClassifierFailureKind,+) -> None:+ intent = build_turn_intent(+ user_message="please continue",+ workflow_yaml="",+ chat_history=[],+ global_llm_context=_structured_context_with_run_decision(),+ request_policy=RequestPolicy(testing_intent="require_test", allow_update_workflow=True, allow_run_blocks=True),+ classifier_result=_classifier_failure(failure_kind),+ )++ assert intent.mode == TurnIntentMode.BUILD+ assert intent.expected_output == TurnIntentExpectedOutput.WORKFLOW_UPDATE+ assert intent.confidence == 0.6+ assert intent.authority.may_update_workflow is True+ assert intent.authority.may_run_blocks is True+ assert intent.authority.may_read_run_context is False+ assert TurnIntentReasonCode.TRANSIENT_CLASSIFIER_FALLBACK in intent.reason_codes+ assert TurnIntentReasonCode.RECOVERY_FROM_RUN_CONTEXT not in intent.reason_codes+++ "failure_kind",+ [+ TurnIntentClassifierFailureKind.MISSING_HANDLER,+ TurnIntentClassifierFailureKind.EMPTY_MESSAGE,+ TurnIntentClassifierFailureKind.PROMPT_RENDER_ERROR,+ TurnIntentClassifierFailureKind.MALFORMED_OUTPUT,+ ],+)+def test_build_turn_intent_structural_classifier_failure_uses_prior_run_recovery(+ failure_kind: TurnIntentClassifierFailureKind,+) -> None:+ intent = build_turn_intent(+ user_message="please continue",+ workflow_yaml="",+ chat_history=[],+ global_llm_context=_structured_context_with_run_decision(),+ request_policy=RequestPolicy(testing_intent="require_test", allow_update_workflow=True, allow_run_blocks=True),+ classifier_result=_classifier_failure(failure_kind),+ )++ assert intent.mode == TurnIntentMode.DIAGNOSE+ assert intent.authority.may_update_workflow is False+ assert intent.authority.may_run_blocks is False+ assert intent.authority.may_read_run_context is True+ assert TurnIntentReasonCode.RECOVERY_FROM_RUN_CONTEXT in intent.reason_codes+ assert TurnIntentReasonCode.TRANSIENT_CLASSIFIER_FALLBACK not in intent.reason_codes+++def test_build_turn_intent_genuine_unknown_classification_uses_prior_run_recovery() -> None:+ intent = build_turn_intent(+ user_message="please continue",+ workflow_yaml="",+ chat_history=[],+ global_llm_context=_structured_context_with_run_decision(),+ request_policy=RequestPolicy(testing_intent="require_test", allow_update_workflow=True, allow_run_blocks=True),+ classifier_result=_classification(TurnIntentMode.UNKNOWN, confidence=0.34),+ )++ assert intent.mode == TurnIntentMode.DIAGNOSE+ assert intent.authority.may_update_workflow is False+ assert intent.authority.may_run_blocks is False+ assert TurnIntentReasonCode.LLM_CLASSIFIER in intent.reason_codes+ assert TurnIntentReasonCode.RECOVERY_FROM_RUN_CONTEXT in intent.reason_codes+ assert TurnIntentReasonCode.TRANSIENT_CLASSIFIER_FALLBACK not in intent.reason_codes+++def test_build_turn_intent_transient_classifier_failure_without_authority_uses_prior_run_recovery() -> None:+ intent = build_turn_intent(+ user_message="please continue",+ workflow_yaml="",+ chat_history=[],+ global_llm_context=_structured_context_with_run_decision(),+ request_policy=RequestPolicy(allow_update_workflow=False, allow_run_blocks=False),+ classifier_result=_classifier_failure(TurnIntentClassifierFailureKind.TIMEOUT),+ )++ assert intent.mode == TurnIntentMode.DIAGNOSE+ assert intent.authority.may_update_workflow is False+ assert intent.authority.may_run_blocks is False+ assert TurnIntentReasonCode.RECOVERY_FROM_RUN_CONTEXT in intent.reason_codes+ assert TurnIntentReasonCode.TRANSIENT_CLASSIFIER_FALLBACK not in intent.reason_codes+++def test_build_turn_intent_raw_secret_refusal_wins_over_transient_classifier_failure() -> None:+ intent = build_turn_intent(+ user_message="Use password: hunter2 to sign in.",+ workflow_yaml="",+ chat_history=[],+ global_llm_context=_structured_context_with_run_decision(),+ request_policy=RequestPolicy(+ credential_input_kind="raw_secret",+ raw_secret_detected=True,+ raw_secret_handling="block",+ allow_update_workflow=True,+ allow_run_blocks=True,+ ),+ classifier_result=_classifier_failure(TurnIntentClassifierFailureKind.TIMEOUT),+ )++ assert intent.mode == TurnIntentMode.REFUSE+ assert intent.authority.may_update_workflow is False+ assert intent.authority.may_run_blocks is False+ assert intent.authority.requires_user_input is True+ assert TurnIntentReasonCode.RAW_SECRET_REFUSAL in intent.reason_codes+ assert TurnIntentReasonCode.TRANSIENT_CLASSIFIER_FALLBACK not in intent.reason_codes+++def test_build_turn_intent_policy_clarification_wins_over_transient_classifier_failure() -> None:+ intent = build_turn_intent(+ user_message="Build and test it.",+ workflow_yaml="",+ chat_history=[],+ global_llm_context=_structured_context_with_run_decision(),+ request_policy=RequestPolicy(+ user_response_policy="ask_clarification",+ allow_update_workflow=True,+ allow_run_blocks=True,+ clarification_question="Which page should I target?",+ ),+ classifier_result=_classifier_failure(TurnIntentClassifierFailureKind.TIMEOUT),+ )++ assert intent.mode == TurnIntentMode.CLARIFY+ assert intent.authority.may_update_workflow is False+ assert intent.authority.may_run_blocks is False+ assert intent.authority.requires_user_input is True+ assert intent.missing_context_question == "Which page should I target?"+ assert TurnIntentReasonCode.REQUEST_POLICY_CLARIFICATION in intent.reason_codes+ assert TurnIntentReasonCode.TRANSIENT_CLASSIFIER_FALLBACK not in intent.reason_codes++def test_has_structured_prior_run_signal_ignores_workflow_state_only() -> None:payload = StructuredContext(workflow_state="updated workflow yaml diff summary").to_json_str()assert _has_structured_prior_run_signal(payload) is False@@ -568,7 +715,7 @@ def test_store_request_policy_attaches_classified_turn_intent_to_context() -> Noctx,policy,inputs,- turn_intent_classification=_classification(TurnIntentMode.DIAGNOSE),+ turn_intent_classifier_result=_classification(TurnIntentMode.DIAGNOSE),)assert ctx.turn_intent is not None@@ -627,6 +774,20 @@ def test_turn_intent_classification_parser_rejects_malformed_payload() -> None:assert _turn_intent_classification_from_raw("not json") is None+def test_turn_intent_classification_parser_ignores_deterministic_fallback_reason_code() -> None:+ classification = _turn_intent_classification_from_raw(+ {+ "mode": "build",+ "expected_output": "workflow_draft",+ "confidence": 0.8,+ "reason_codes": ["transient_classifier_fallback", "target_entity_resolved"],+ }+ )++ assert classification is not None+ assert classification.reason_codes == [TurnIntentReasonCode.TARGET_ENTITY_RESOLVED]++@pytest.mark.asyncioasync def test_classify_turn_intent_calls_llm_handler_with_prompt_contract() -> None:calls: list[dict[str, str]] = []@@ -643,7 +804,7 @@ async def test_classify_turn_intent_calls_llm_handler_with_prompt_contract() ->"reason_codes": [],}- classification = await classify_turn_intent(+ classifier_result = await classify_turn_intent(user_message="Create a workflow from the page I have open.",workflow_yaml="",chat_history=[],@@ -652,9 +813,140 @@ async def test_classify_turn_intent_calls_llm_handler_with_prompt_contract() ->handler=handler,)+ assert classifier_result.is_success+ classification = classifier_result.classificationassert classification is not Noneassert classification.mode == TurnIntentMode.BUILDassert classification.required_context == [RequiredContextKey.BROWSER_STATE]assert calls[0]["prompt_name"] == PROMPT_NAMEassert "Create a workflow from the page I have open." in calls[0]["prompt"]assert "Allowed modes" in calls[0]["prompt"]+ assert "transient_classifier_fallback" not in calls[0]["prompt"]+++async def test_classify_turn_intent_reports_missing_handler_and_empty_message() -> None:+ async def handler(prompt: str, prompt_name: str, **_: object) -> dict[str, object]:+ raise AssertionError("handler should not be called")++ empty_result = await classify_turn_intent(+ user_message="",+ workflow_yaml="",+ chat_history=[],+ global_llm_context="",+ request_policy=RequestPolicy(),+ handler=handler,+ )+ missing_handler_result = await classify_turn_intent(+ user_message="Build it",+ workflow_yaml="",+ chat_history=[],+ global_llm_context="",+ request_policy=RequestPolicy(),+ handler=None,+ )++ assert empty_result.failure_kind == TurnIntentClassifierFailureKind.EMPTY_MESSAGE+ assert missing_handler_result.failure_kind == TurnIntentClassifierFailureKind.MISSING_HANDLER+++async def test_classify_turn_intent_uses_turn_intent_timeout_setting(monkeypatch: pytest.MonkeyPatch) -> None:+ monkeypatch.setattr(settings, "COPILOT_TURN_INTENT_CLASSIFIER_TIMEOUT_SECONDS", 0.001)+ monkeypatch.setattr(settings, "COPILOT_FEASIBILITY_GATE_TIMEOUT_SECONDS", 30.0)++ async def handler(prompt: str, prompt_name: str, **_: object) -> dict[str, object]:+ await asyncio.sleep(0.05)+ return {"mode": "build"}++ result = await classify_turn_intent(+ user_message="Build it",+ workflow_yaml="",+ chat_history=[],+ global_llm_context="",+ request_policy=RequestPolicy(),+ handler=handler,+ )++ assert result.failure_kind == TurnIntentClassifierFailureKind.TIMEOUT+++async def test_classify_turn_intent_reports_converted_cancellation_as_timeout(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ monkeypatch.setattr(settings, "COPILOT_TURN_INTENT_CLASSIFIER_TIMEOUT_SECONDS", 0.001)++ async def handler(prompt: str, prompt_name: str, **_: object) -> dict[str, object]:+ try:+ await asyncio.sleep(0.05)+ except asyncio.CancelledError as exc:+ raise RuntimeError("LLM request got cancelled") from exc+ return {"mode": "build"}++ result = await classify_turn_intent(+ user_message="Build it",+ workflow_yaml="",+ chat_history=[],+ global_llm_context="",+ request_policy=RequestPolicy(),+ handler=handler,+ )++ assert result.failure_kind == TurnIntentClassifierFailureKind.TIMEOUT+++async def test_classify_turn_intent_reports_provider_error() -> None:+ async def handler(prompt: str, prompt_name: str, **_: object) -> dict[str, object]:+ raise RuntimeError("provider unavailable")++ result = await classify_turn_intent(+ user_message="Build it",+ workflow_yaml="",+ chat_history=[],+ global_llm_context="",+ request_policy=RequestPolicy(),+ handler=handler,+ )++ assert result.failure_kind == TurnIntentClassifierFailureKind.PROVIDER_ERROR+++async def test_classify_turn_intent_reports_malformed_output() -> None:+ async def handler(prompt: str, prompt_name: str, **_: object) -> dict[str, object]:+ return {"mode": "not_a_mode"}++ result = await classify_turn_intent(+ user_message="Build it",+ workflow_yaml="",+ chat_history=[],+ global_llm_context="",+ request_policy=RequestPolicy(),+ handler=handler,+ )++ assert result.failure_kind == TurnIntentClassifierFailureKind.MALFORMED_OUTPUT+++async def test_classify_turn_intent_reports_prompt_render_error(monkeypatch: pytest.MonkeyPatch) -> None:+ def raise_prompt_error(**_: object) -> str:+ raise RuntimeError("template missing")++ async def handler(prompt: str, prompt_name: str, **_: object) -> dict[str, object]:+ raise AssertionError("handler should not be called")++ monkeypatch.setattr("skyvern.forge.sdk.copilot.turn_intent.prompt_engine.load_prompt", raise_prompt_error)++ result = await classify_turn_intent(+ user_message="Build it",+ workflow_yaml="",+ chat_history=[],+ global_llm_context="",+ request_policy=RequestPolicy(),+ handler=handler,+ )++ assert result.failure_kind == TurnIntentClassifierFailureKind.PROMPT_RENDER_ERROR