Skip to content

Automate browser based workflows with AI

SKY-10984: Preserve run authority on turn-intent timeouts (#6583)

Andrew Neilson93f333e767ed
diff --git a/skyvern/config.py b/skyvern/config.py
index 8218dddac..598025101 100644
--- a/skyvern/config.py
+++ b/skyvern/config.py
@@ -149,6 +149,7 @@ class Settings(BaseSettings):
LOG_LEVEL: str = "INFO"
COPILOT_FEASIBILITY_GATE_TIMEOUT_SECONDS: float = 12.0
COPILOT_REQUEST_POLICY_CLASSIFIER_TIMEOUT_SECONDS: float = 6.0
+ COPILOT_TURN_INTENT_CLASSIFIER_TIMEOUT_SECONDS: float = 12.0
COPILOT_COMPLETION_JUDGE_TIMEOUT_SECONDS: float = 12.0
# Gate copilot verified-success on per-criterion outcome evidence.
# Off restores the prior run-status/suspicious-success gate and classifier prompt.
diff --git a/skyvern/forge/sdk/copilot/agent.py b/skyvern/forge/sdk/copilot/agent.py
index 8e5fec877..88f32a6d9 100644
--- a/skyvern/forge/sdk/copilot/agent.py
+++ b/skyvern/forge/sdk/copilot/agent.py
@@ -122,7 +122,7 @@ from skyvern.forge.sdk.copilot.turn_intent import (
NO_MUTATION_TURN_INTENT_MODES,
RequiredContextKey,
TurnIntent,
- TurnIntentClassification,
+ TurnIntentClassifierResult,
TurnIntentMode,
build_turn_intent,
classify_turn_intent,
@@ -399,7 +399,7 @@ def _store_request_policy_on_context(
ctx: CopilotContext,
policy: RequestPolicy,
policy_inputs: RequestPolicyGuardrailInputs,
- turn_intent_classification: TurnIntentClassification | None = None,
+ turn_intent_classifier_result: TurnIntentClassifierResult | None = None,
) -> None:
agent_user_message, policy_chat_history_text = _request_policy_agent_inputs(
policy,
@@ -426,7 +426,7 @@ def _store_request_policy_on_context(
workflow_permanent_id=policy_inputs.workflow_permanent_id,
workflow_run_id=policy_inputs.workflow_run_id,
browser_session_id=policy_inputs.browser_session_id,
- classification=turn_intent_classification,
+ classifier_result=turn_intent_classifier_result,
)
@@ -2595,9 +2595,9 @@ def _build_copilot_input_guardrails(
active_criteria=_stored_active_completion_criteria(policy_inputs),
)
if isinstance(ctx, CopilotContext):
- turn_intent_classification = None
+ turn_intent_classifier_result = None
if policy.user_response_policy != "ask_clarification" or policy.raw_secret_handling == "redacted_draft":
- turn_intent_classification = await classify_turn_intent(
+ turn_intent_classifier_result = await classify_turn_intent(
user_message=policy_inputs.user_message,
workflow_yaml=policy_inputs.workflow_yaml,
chat_history=policy_inputs.chat_history_messages,
@@ -2609,7 +2609,7 @@ def _build_copilot_input_guardrails(
ctx,
policy,
policy_inputs,
- turn_intent_classification=turn_intent_classification,
+ turn_intent_classifier_result=turn_intent_classifier_result,
)
blocked = isinstance(policy, RequestPolicy) and policy.user_response_policy == "ask_clarification"
if isinstance(policy, RequestPolicy):
diff --git a/skyvern/forge/sdk/copilot/turn_intent.py b/skyvern/forge/sdk/copilot/turn_intent.py
index 002abd2a6..8a580816d 100644
--- a/skyvern/forge/sdk/copilot/turn_intent.py
+++ b/skyvern/forge/sdk/copilot/turn_intent.py
@@ -2,6 +2,7 @@ from __future__ import annotations
import asyncio
import re
+import time
from collections.abc import Mapping
from difflib import SequenceMatcher
from enum import StrEnum
@@ -113,6 +114,21 @@ class TurnIntentReasonCode(StrEnum):
LOW_CONFIDENCE_CLARIFICATION = "low_confidence_clarification"
TARGET_ENTITY_RESOLVED = "target_entity_resolved"
MISSING_EDIT_TARGET = "missing_edit_target"
+ TRANSIENT_CLASSIFIER_FALLBACK = "transient_classifier_fallback"
+
+
+class TurnIntentClassifierFailureKind(StrEnum):
+ TIMEOUT = "timeout"
+ PROVIDER_ERROR = "provider_error"
+ MISSING_HANDLER = "missing_handler"
+ EMPTY_MESSAGE = "empty_message"
+ PROMPT_RENDER_ERROR = "prompt_render_error"
+ MALFORMED_OUTPUT = "malformed_output"
+
+
+_CLASSIFIER_REASON_CODES = tuple(
+ reason for reason in TurnIntentReasonCode if reason != TurnIntentReasonCode.TRANSIENT_CLASSIFIER_FALLBACK
+)
_DEFAULT_EXPECTED_OUTPUT_BY_MODE: dict[TurnIntentMode, TurnIntentExpectedOutput] = {
@@ -252,6 +268,38 @@ class TurnIntentClassification(BaseModel):
return data
+class TurnIntentClassifierResult(BaseModel):
+ model_config = ConfigDict(extra="forbid")
+
+ classification: TurnIntentClassification | None = None
+ failure_kind: TurnIntentClassifierFailureKind | None = None
+
+ @model_validator(mode="after")
+ def _validate_result_shape(self) -> TurnIntentClassifierResult:
+ if (self.classification is None) == (self.failure_kind is None):
+ raise ValueError("classifier result must contain exactly one of classification or failure_kind")
+ return self
+
+ @classmethod
+ def success(cls, classification: TurnIntentClassification) -> TurnIntentClassifierResult:
+ return cls(classification=classification)
+
+ @classmethod
+ def failure(cls, failure_kind: TurnIntentClassifierFailureKind) -> TurnIntentClassifierResult:
+ return cls(failure_kind=failure_kind)
+
+ @property
+ def is_success(self) -> bool:
+ return self.classification is not None
+
+ @property
+ def is_transient_failure(self) -> bool:
+ return self.failure_kind in (
+ TurnIntentClassifierFailureKind.TIMEOUT,
+ TurnIntentClassifierFailureKind.PROVIDER_ERROR,
+ )
+
+
_GOAL_MAX_CHARS = 240
@@ -405,9 +453,11 @@ def _coerce_reason_codes(raw: object) -> list[TurnIntentReasonCode]:
reason_codes: list[TurnIntentReasonCode] = []
for value in values:
try:
- reason_codes.append(TurnIntentReasonCode(value))
+ reason_code = TurnIntentReasonCode(value)
except ValueError:
continue
+ if reason_code in _CLASSIFIER_REASON_CODES:
+ reason_codes.append(reason_code)
return list(dict.fromkeys(reason_codes))
@@ -519,12 +569,13 @@ async def classify_turn_intent(
global_llm_context: str,
request_policy: RequestPolicy,
handler: LLMAPIHandler | None,
-) -> TurnIntentClassification | None:
+) -> TurnIntentClassifierResult:
if not isinstance(user_message, str) or not user_message.strip():
- return None
+ LOG.info("turn-intent classifier skipped empty user message")
+ return TurnIntentClassifierResult.failure(TurnIntentClassifierFailureKind.EMPTY_MESSAGE)
if handler is None:
LOG.info("turn-intent classifier has no LLM handler available")
- return None
+ return TurnIntentClassifierResult.failure(TurnIntentClassifierFailureKind.MISSING_HANDLER)
safe_user_message = redact_raw_secrets_for_prompt(user_message)
transcript = build_transcript_context(chat_history, safe_user_message)
@@ -534,7 +585,7 @@ async def classify_turn_intent(
mode_values=", ".join(mode.value for mode in TurnIntentMode),
expected_output_values=", ".join(output.value for output in TurnIntentExpectedOutput),
required_context_values=", ".join(key.value for key in RequiredContextKey),
- reason_code_values=", ".join(reason.value for reason in TurnIntentReasonCode),
+ reason_code_values=", ".join(reason.value for reason in _CLASSIFIER_REASON_CODES),
user_message=escape_code_fences(safe_user_message),
request_policy_summary=escape_code_fences(request_policy.prompt_summary()),
workflow_yaml=escape_code_fences(
@@ -550,31 +601,53 @@ async def classify_turn_intent(
)
except Exception as exc:
LOG.warning("turn-intent classifier prompt render failed", error=str(exc))
- return None
+ return TurnIntentClassifierResult.failure(TurnIntentClassifierFailureKind.PROMPT_RENDER_ERROR)
+ timeout = settings.COPILOT_TURN_INTENT_CLASSIFIER_TIMEOUT_SECONDS
+ started_at = time.monotonic()
try:
raw = await asyncio.wait_for(
handler(prompt=prompt, prompt_name=PROMPT_NAME),
- timeout=settings.COPILOT_FEASIBILITY_GATE_TIMEOUT_SECONDS,
+ timeout=timeout,
)
except asyncio.TimeoutError:
LOG.warning(
"turn-intent classifier timed out",
- timeout=settings.COPILOT_FEASIBILITY_GATE_TIMEOUT_SECONDS,
+ failure_kind=TurnIntentClassifierFailureKind.TIMEOUT.value,
+ timeout=timeout,
)
- return None
+ return TurnIntentClassifierResult.failure(TurnIntentClassifierFailureKind.TIMEOUT)
except Exception as exc:
- LOG.warning("turn-intent classifier failed", error=str(exc))
- return None
+ elapsed_seconds = time.monotonic() - started_at
+ timeout_margin = min(0.25, max(0.001, timeout * 0.05))
+ if elapsed_seconds + timeout_margin >= timeout:
+ LOG.warning(
+ "turn-intent classifier timed out",
+ failure_kind=TurnIntentClassifierFailureKind.TIMEOUT.value,
+ timeout=timeout,
+ elapsed_seconds=elapsed_seconds,
+ converted_error_type=type(exc).__name__,
+ error=str(exc),
+ )
+ return TurnIntentClassifierResult.failure(TurnIntentClassifierFailureKind.TIMEOUT)
+ LOG.warning(
+ "turn-intent classifier provider failed",
+ failure_kind=TurnIntentClassifierFailureKind.PROVIDER_ERROR.value,
+ error=str(exc),
+ )
+ return TurnIntentClassifierResult.failure(TurnIntentClassifierFailureKind.PROVIDER_ERROR)
classification = _turn_intent_classification_from_raw(raw)
if classification is None:
- LOG.warning("turn-intent classifier returned malformed output")
- return None
+ LOG.warning(
+ "turn-intent classifier returned malformed output",
+ failure_kind=TurnIntentClassifierFailureKind.MALFORMED_OUTPUT.value,
+ )
+ return TurnIntentClassifierResult.failure(TurnIntentClassifierFailureKind.MALFORMED_OUTPUT)
with copilot_span("turn_intent_classifier", data=classification.to_trace_data()):
LOG.info("turn-intent classifier decision", **classification.to_trace_data())
- return classification
+ return TurnIntentClassifierResult.success(classification)
_PRIOR_RUN_OUTPUT_PREFIX = " output:"
@@ -678,7 +751,7 @@ def build_turn_intent(
workflow_permanent_id: str | None = None,
workflow_run_id: str | None = None,
browser_session_id: str | None = None,
- classification: TurnIntentClassification | None = None,
+ classifier_result: TurnIntentClassifierResult | None = None,
) -> TurnIntent:
has_workflow = bool((workflow_yaml or "").strip())
has_prior_context = bool((global_llm_context or "").strip())
@@ -720,6 +793,7 @@ def build_turn_intent(
confidence = 0.2 if (has_workflow or has_prior_context or chat_history) else 0.0
missing_context_question = None
+ classification = classifier_result.classification if classifier_result and classifier_result.is_success else None
has_prior_run_signal = workflow_run_id is not None or _has_structured_prior_run_signal(global_llm_context)
if request_policy.raw_secret_detected and request_policy.raw_secret_handling != "redacted_draft":
@@ -782,7 +856,24 @@ def build_turn_intent(
if _user_signals_non_progress(user_message, chat_history):
reason_codes.append(TurnIntentReasonCode.USER_NON_PROGRESS)
- if mode == TurnIntentMode.UNKNOWN and has_prior_run_signal:
+ has_request_policy_update_or_run_authority = authority.may_update_workflow or authority.may_run_blocks
+ suppress_prior_run_recovery = (
+ mode == TurnIntentMode.UNKNOWN
+ and has_prior_run_signal
+ and classifier_result is not None
+ and classifier_result.is_transient_failure
+ and has_request_policy_update_or_run_authority
+ )
+ if suppress_prior_run_recovery:
+ mode = TurnIntentMode.BUILD
+ expected_output = (
+ TurnIntentExpectedOutput.WORKFLOW_UPDATE
+ if has_workflow or has_prior_context or workflow_id or workflow_permanent_id
+ else TurnIntentExpectedOutput.WORKFLOW_DRAFT
+ )
+ confidence = max(confidence, 0.6)
+ reason_codes.append(TurnIntentReasonCode.TRANSIENT_CLASSIFIER_FALLBACK)
+ elif mode == TurnIntentMode.UNKNOWN and has_prior_run_signal:
mode = TurnIntentMode.DIAGNOSE
expected_output = TurnIntentExpectedOutput.RUN_RESULT
reason_codes.append(TurnIntentReasonCode.RECOVERY_FROM_RUN_CONTEXT)
@@ -803,7 +894,12 @@ def build_turn_intent(
authority.requires_user_input = True
elif mode == TurnIntentMode.DIAGNOSE:
authority.may_update_workflow = False
- retest_mandated = request_policy.testing_intent == "require_test" and request_policy.allow_run_blocks
+ retest_mandated = (
+ request_policy.testing_intent == "require_test"
+ and request_policy.allow_run_blocks
+ and classification is not None
+ and classification.mode == TurnIntentMode.DIAGNOSE
+ )
if retest_mandated:
# RequestPolicy owns testing policy; an explicit require_test re-run must not be inverted
# by the diagnose classification.
diff --git a/tests/unit/test_copilot_turn_intent.py b/tests/unit/test_copilot_turn_intent.py
index 612eb59a0..0220ced9b 100644
--- a/tests/unit/test_copilot_turn_intent.py
+++ b/tests/unit/test_copilot_turn_intent.py
@@ -1,5 +1,6 @@
from __future__ import annotations
+import asyncio
from datetime import datetime, timezone
from types import SimpleNamespace
from unittest.mock import MagicMock
@@ -7,6 +8,7 @@ from unittest.mock import MagicMock
import pytest
from pydantic import ValidationError
+from skyvern.config import settings
from skyvern.forge.sdk.copilot.agent import (
RequestPolicyGuardrailInputs,
_docs_answer_turn_directive,
@@ -22,6 +24,8 @@ from skyvern.forge.sdk.copilot.turn_intent import (
TurnIntent,
TurnIntentAuthority,
TurnIntentClassification,
+ TurnIntentClassifierFailureKind,
+ TurnIntentClassifierResult,
TurnIntentExpectedOutput,
TurnIntentMode,
TurnIntentReasonCode,
@@ -61,8 +65,8 @@ def _classification(
target_entities: dict[str, list[str]] | None = None,
missing_context_question: str | None = None,
reason_codes: list[TurnIntentReasonCode] | None = None,
-) -> TurnIntentClassification:
- return TurnIntentClassification(
+) -> TurnIntentClassifierResult:
+ classification = TurnIntentClassification(
mode=mode,
expected_output=expected_output,
required_context=required_context or [],
@@ -71,6 +75,11 @@ def _classification(
missing_context_question=missing_context_question,
reason_codes=reason_codes or [],
)
+ return TurnIntentClassifierResult.success(classification)
+
+
+def _classifier_failure(failure_kind: TurnIntentClassifierFailureKind) -> TurnIntentClassifierResult:
+ return TurnIntentClassifierResult.failure(failure_kind)
def test_turn_intent_defaults_to_unknown_shadow_contract() -> None:
@@ -141,7 +150,7 @@ def test_build_turn_intent_applies_llm_build_classification() -> None:
global_llm_context="",
request_policy=RequestPolicy(),
browser_session_id="pbs_123",
- classification=_classification(
+ classifier_result=_classification(
TurnIntentMode.BUILD,
required_context=[RequiredContextKey.BROWSER_STATE],
confidence=0.84,
@@ -164,7 +173,7 @@ def test_build_turn_intent_applies_docs_classification_without_mutation() -> Non
chat_history=[],
global_llm_context="",
request_policy=RequestPolicy(),
- classification=_classification(TurnIntentMode.DOCS_ANSWER),
+ classifier_result=_classification(TurnIntentMode.DOCS_ANSWER),
)
assert intent.mode == TurnIntentMode.DOCS_ANSWER
@@ -182,7 +191,7 @@ def test_build_turn_intent_applies_diagnose_classification_and_read_authority()
global_llm_context="",
request_policy=RequestPolicy(allow_update_workflow=True, allow_run_blocks=True),
workflow_run_id="wr_123",
- classification=_classification(TurnIntentMode.DIAGNOSE),
+ classifier_result=_classification(TurnIntentMode.DIAGNOSE),
)
assert intent.mode == TurnIntentMode.DIAGNOSE
@@ -201,7 +210,7 @@ def test_build_turn_intent_applies_draft_only_classification_without_run_authori
chat_history=[],
global_llm_context="",
request_policy=RequestPolicy(allow_update_workflow=True, allow_run_blocks=True),
- classification=_classification(TurnIntentMode.DRAFT_ONLY),
+ classifier_result=_classification(TurnIntentMode.DRAFT_ONLY),
)
assert intent.mode == TurnIntentMode.DRAFT_ONLY
@@ -231,7 +240,7 @@ def test_build_turn_intent_llm_diagnose_outranks_skip_test_policy() -> None:
global_llm_context="",
request_policy=RequestPolicy(testing_intent="skip_test", allow_update_workflow=True, allow_run_blocks=False),
workflow_run_id="wr_123",
- classification=_classification(TurnIntentMode.DIAGNOSE),
+ classifier_result=_classification(TurnIntentMode.DIAGNOSE),
)
assert intent.mode == TurnIntentMode.DIAGNOSE
@@ -248,7 +257,7 @@ def test_build_turn_intent_diagnose_with_require_test_keeps_run_authority() -> N
global_llm_context="",
request_policy=RequestPolicy(testing_intent="require_test", allow_update_workflow=True, allow_run_blocks=True),
workflow_run_id="wr_123",
- classification=_classification(TurnIntentMode.DIAGNOSE),
+ classifier_result=_classification(TurnIntentMode.DIAGNOSE),
)
assert intent.mode == TurnIntentMode.DIAGNOSE
@@ -267,7 +276,7 @@ def test_build_turn_intent_diagnose_without_require_test_stays_answer_only() ->
global_llm_context="",
request_policy=RequestPolicy(allow_update_workflow=False, allow_run_blocks=False),
workflow_run_id="wr_123",
- classification=_classification(TurnIntentMode.DIAGNOSE),
+ classifier_result=_classification(TurnIntentMode.DIAGNOSE),
)
assert intent.mode == TurnIntentMode.DIAGNOSE
@@ -292,7 +301,7 @@ def test_build_turn_intent_uses_request_policy_clarification_over_llm_classifica
chat_history=[_user_message("Build a workflow")],
global_llm_context="",
request_policy=policy,
- classification=_classification(TurnIntentMode.EDIT),
+ classifier_result=_classification(TurnIntentMode.EDIT),
)
assert intent.mode == TurnIntentMode.CLARIFY
@@ -321,7 +330,7 @@ def test_build_turn_intent_routes_raw_secret_to_refuse_over_llm_classification()
chat_history=[],
global_llm_context="",
request_policy=policy,
- classification=_classification(TurnIntentMode.BUILD),
+ classifier_result=_classification(TurnIntentMode.BUILD),
)
assert intent.mode == TurnIntentMode.REFUSE
@@ -340,7 +349,7 @@ def test_build_turn_intent_redacts_user_goal() -> None:
chat_history=[],
global_llm_context="",
request_policy=RequestPolicy(),
- classification=_classification(TurnIntentMode.BUILD),
+ classifier_result=_classification(TurnIntentMode.BUILD),
)
assert "hunter2" not in intent.user_goal
@@ -355,7 +364,7 @@ def test_build_turn_intent_low_confidence_mutating_classification_clarifies(mode
chat_history=[],
global_llm_context="",
request_policy=RequestPolicy(),
- classification=_classification(mode, confidence=0.35, target_entities={"workflow_change": ["update_it"]}),
+ classifier_result=_classification(mode, confidence=0.35, target_entities={"workflow_change": ["update_it"]}),
)
assert intent.mode == TurnIntentMode.CLARIFY
@@ -371,7 +380,7 @@ def test_build_turn_intent_targetless_edit_classification_clarifies() -> None:
chat_history=[],
global_llm_context="",
request_policy=RequestPolicy(),
- classification=_classification(TurnIntentMode.EDIT, confidence=0.82),
+ classifier_result=_classification(TurnIntentMode.EDIT, confidence=0.82),
)
assert intent.mode == TurnIntentMode.CLARIFY
@@ -388,7 +397,7 @@ def test_build_turn_intent_allows_clear_workflow_change_edit_classification() ->
chat_history=[],
global_llm_context="",
request_policy=RequestPolicy(allow_run_blocks=False),
- classification=_classification(
+ classifier_result=_classification(
TurnIntentMode.EDIT,
confidence=0.82,
target_entities={"workflow_change": ["add_invoice_download_step"]},
@@ -423,7 +432,7 @@ def test_build_turn_intent_resolves_classifier_block_targets(block_ref: str, exp
chat_history=[],
global_llm_context="",
request_policy=RequestPolicy(allow_run_blocks=False),
- classification=_classification(
+ classifier_result=_classification(
TurnIntentMode.EDIT,
confidence=0.65,
target_entities={"block": [block_ref]},
@@ -449,7 +458,7 @@ def test_build_turn_intent_preserves_classifier_unresolved_block_targets() -> No
chat_history=[],
global_llm_context="",
request_policy=RequestPolicy(allow_run_blocks=False),
- classification=_classification(
+ classifier_result=_classification(
TurnIntentMode.CLARIFY,
confidence=0.35,
target_entities={"block": ["login block"]},
@@ -503,6 +512,144 @@ def test_build_turn_intent_unknown_with_persisted_prior_run_signal_upgrades_to_d
assert intent.authority.may_read_run_context is True
+ "failure_kind",
+ [TurnIntentClassifierFailureKind.TIMEOUT, TurnIntentClassifierFailureKind.PROVIDER_ERROR],
+)
+def test_build_turn_intent_transient_classifier_failure_suppresses_prior_run_recovery(
+ failure_kind: TurnIntentClassifierFailureKind,
+) -> None:
+ intent = build_turn_intent(
+ user_message="please continue",
+ workflow_yaml="",
+ chat_history=[],
+ global_llm_context=_structured_context_with_run_decision(),
+ request_policy=RequestPolicy(testing_intent="require_test", allow_update_workflow=True, allow_run_blocks=True),
+ classifier_result=_classifier_failure(failure_kind),
+ )
+
+ assert intent.mode == TurnIntentMode.BUILD
+ assert intent.expected_output == TurnIntentExpectedOutput.WORKFLOW_UPDATE
+ assert intent.confidence == 0.6
+ assert intent.authority.may_update_workflow is True
+ assert intent.authority.may_run_blocks is True
+ assert intent.authority.may_read_run_context is False
+ assert TurnIntentReasonCode.TRANSIENT_CLASSIFIER_FALLBACK in intent.reason_codes
+ assert TurnIntentReasonCode.RECOVERY_FROM_RUN_CONTEXT not in intent.reason_codes
+
+
+ "failure_kind",
+ [
+ TurnIntentClassifierFailureKind.MISSING_HANDLER,
+ TurnIntentClassifierFailureKind.EMPTY_MESSAGE,
+ TurnIntentClassifierFailureKind.PROMPT_RENDER_ERROR,
+ TurnIntentClassifierFailureKind.MALFORMED_OUTPUT,
+ ],
+)
+def test_build_turn_intent_structural_classifier_failure_uses_prior_run_recovery(
+ failure_kind: TurnIntentClassifierFailureKind,
+) -> None:
+ intent = build_turn_intent(
+ user_message="please continue",
+ workflow_yaml="",
+ chat_history=[],
+ global_llm_context=_structured_context_with_run_decision(),
+ request_policy=RequestPolicy(testing_intent="require_test", allow_update_workflow=True, allow_run_blocks=True),
+ classifier_result=_classifier_failure(failure_kind),
+ )
+
+ assert intent.mode == TurnIntentMode.DIAGNOSE
+ assert intent.authority.may_update_workflow is False
+ assert intent.authority.may_run_blocks is False
+ assert intent.authority.may_read_run_context is True
+ assert TurnIntentReasonCode.RECOVERY_FROM_RUN_CONTEXT in intent.reason_codes
+ assert TurnIntentReasonCode.TRANSIENT_CLASSIFIER_FALLBACK not in intent.reason_codes
+
+
+def test_build_turn_intent_genuine_unknown_classification_uses_prior_run_recovery() -> None:
+ intent = build_turn_intent(
+ user_message="please continue",
+ workflow_yaml="",
+ chat_history=[],
+ global_llm_context=_structured_context_with_run_decision(),
+ request_policy=RequestPolicy(testing_intent="require_test", allow_update_workflow=True, allow_run_blocks=True),
+ classifier_result=_classification(TurnIntentMode.UNKNOWN, confidence=0.34),
+ )
+
+ assert intent.mode == TurnIntentMode.DIAGNOSE
+ assert intent.authority.may_update_workflow is False
+ assert intent.authority.may_run_blocks is False
+ assert TurnIntentReasonCode.LLM_CLASSIFIER in intent.reason_codes
+ assert TurnIntentReasonCode.RECOVERY_FROM_RUN_CONTEXT in intent.reason_codes
+ assert TurnIntentReasonCode.TRANSIENT_CLASSIFIER_FALLBACK not in intent.reason_codes
+
+
+def test_build_turn_intent_transient_classifier_failure_without_authority_uses_prior_run_recovery() -> None:
+ intent = build_turn_intent(
+ user_message="please continue",
+ workflow_yaml="",
+ chat_history=[],
+ global_llm_context=_structured_context_with_run_decision(),
+ request_policy=RequestPolicy(allow_update_workflow=False, allow_run_blocks=False),
+ classifier_result=_classifier_failure(TurnIntentClassifierFailureKind.TIMEOUT),
+ )
+
+ assert intent.mode == TurnIntentMode.DIAGNOSE
+ assert intent.authority.may_update_workflow is False
+ assert intent.authority.may_run_blocks is False
+ assert TurnIntentReasonCode.RECOVERY_FROM_RUN_CONTEXT in intent.reason_codes
+ assert TurnIntentReasonCode.TRANSIENT_CLASSIFIER_FALLBACK not in intent.reason_codes
+
+
+def test_build_turn_intent_raw_secret_refusal_wins_over_transient_classifier_failure() -> None:
+ intent = build_turn_intent(
+ user_message="Use password: hunter2 to sign in.",
+ workflow_yaml="",
+ chat_history=[],
+ global_llm_context=_structured_context_with_run_decision(),
+ request_policy=RequestPolicy(
+ credential_input_kind="raw_secret",
+ raw_secret_detected=True,
+ raw_secret_handling="block",
+ allow_update_workflow=True,
+ allow_run_blocks=True,
+ ),
+ classifier_result=_classifier_failure(TurnIntentClassifierFailureKind.TIMEOUT),
+ )
+
+ assert intent.mode == TurnIntentMode.REFUSE
+ assert intent.authority.may_update_workflow is False
+ assert intent.authority.may_run_blocks is False
+ assert intent.authority.requires_user_input is True
+ assert TurnIntentReasonCode.RAW_SECRET_REFUSAL in intent.reason_codes
+ assert TurnIntentReasonCode.TRANSIENT_CLASSIFIER_FALLBACK not in intent.reason_codes
+
+
+def test_build_turn_intent_policy_clarification_wins_over_transient_classifier_failure() -> None:
+ intent = build_turn_intent(
+ user_message="Build and test it.",
+ workflow_yaml="",
+ chat_history=[],
+ global_llm_context=_structured_context_with_run_decision(),
+ request_policy=RequestPolicy(
+ user_response_policy="ask_clarification",
+ allow_update_workflow=True,
+ allow_run_blocks=True,
+ clarification_question="Which page should I target?",
+ ),
+ classifier_result=_classifier_failure(TurnIntentClassifierFailureKind.TIMEOUT),
+ )
+
+ assert intent.mode == TurnIntentMode.CLARIFY
+ assert intent.authority.may_update_workflow is False
+ assert intent.authority.may_run_blocks is False
+ assert intent.authority.requires_user_input is True
+ assert intent.missing_context_question == "Which page should I target?"
+ assert TurnIntentReasonCode.REQUEST_POLICY_CLARIFICATION in intent.reason_codes
+ assert TurnIntentReasonCode.TRANSIENT_CLASSIFIER_FALLBACK not in intent.reason_codes
+
+
def test_has_structured_prior_run_signal_ignores_workflow_state_only() -> None:
payload = StructuredContext(workflow_state="updated workflow yaml diff summary").to_json_str()
assert _has_structured_prior_run_signal(payload) is False
@@ -568,7 +715,7 @@ def test_store_request_policy_attaches_classified_turn_intent_to_context() -> No
ctx,
policy,
inputs,
- turn_intent_classification=_classification(TurnIntentMode.DIAGNOSE),
+ turn_intent_classifier_result=_classification(TurnIntentMode.DIAGNOSE),
)
assert ctx.turn_intent is not None
@@ -627,6 +774,20 @@ def test_turn_intent_classification_parser_rejects_malformed_payload() -> None:
assert _turn_intent_classification_from_raw("not json") is None
+def test_turn_intent_classification_parser_ignores_deterministic_fallback_reason_code() -> None:
+ classification = _turn_intent_classification_from_raw(
+ {
+ "mode": "build",
+ "expected_output": "workflow_draft",
+ "confidence": 0.8,
+ "reason_codes": ["transient_classifier_fallback", "target_entity_resolved"],
+ }
+ )
+
+ assert classification is not None
+ assert classification.reason_codes == [TurnIntentReasonCode.TARGET_ENTITY_RESOLVED]
+
+
@pytest.mark.asyncio
async def test_classify_turn_intent_calls_llm_handler_with_prompt_contract() -> None:
calls: list[dict[str, str]] = []
@@ -643,7 +804,7 @@ async def test_classify_turn_intent_calls_llm_handler_with_prompt_contract() ->
"reason_codes": [],
}
- classification = await classify_turn_intent(
+ classifier_result = await classify_turn_intent(
user_message="Create a workflow from the page I have open.",
workflow_yaml="",
chat_history=[],
@@ -652,9 +813,140 @@ async def test_classify_turn_intent_calls_llm_handler_with_prompt_contract() ->
handler=handler,
)
+ assert classifier_result.is_success
+ classification = classifier_result.classification
assert classification is not None
assert classification.mode == TurnIntentMode.BUILD
assert classification.required_context == [RequiredContextKey.BROWSER_STATE]
assert calls[0]["prompt_name"] == PROMPT_NAME
assert "Create a workflow from the page I have open." in calls[0]["prompt"]
assert "Allowed modes" in calls[0]["prompt"]
+ assert "transient_classifier_fallback" not in calls[0]["prompt"]
+
+
+async def test_classify_turn_intent_reports_missing_handler_and_empty_message() -> None:
+ async def handler(prompt: str, prompt_name: str, **_: object) -> dict[str, object]:
+ raise AssertionError("handler should not be called")
+
+ empty_result = await classify_turn_intent(
+ user_message="",
+ workflow_yaml="",
+ chat_history=[],
+ global_llm_context="",
+ request_policy=RequestPolicy(),
+ handler=handler,
+ )
+ missing_handler_result = await classify_turn_intent(
+ user_message="Build it",
+ workflow_yaml="",
+ chat_history=[],
+ global_llm_context="",
+ request_policy=RequestPolicy(),
+ handler=None,
+ )
+
+ assert empty_result.failure_kind == TurnIntentClassifierFailureKind.EMPTY_MESSAGE
+ assert missing_handler_result.failure_kind == TurnIntentClassifierFailureKind.MISSING_HANDLER
+
+
+async def test_classify_turn_intent_uses_turn_intent_timeout_setting(monkeypatch: pytest.MonkeyPatch) -> None:
+ monkeypatch.setattr(settings, "COPILOT_TURN_INTENT_CLASSIFIER_TIMEOUT_SECONDS", 0.001)
+ monkeypatch.setattr(settings, "COPILOT_FEASIBILITY_GATE_TIMEOUT_SECONDS", 30.0)
+
+ async def handler(prompt: str, prompt_name: str, **_: object) -> dict[str, object]:
+ await asyncio.sleep(0.05)
+ return {"mode": "build"}
+
+ result = await classify_turn_intent(
+ user_message="Build it",
+ workflow_yaml="",
+ chat_history=[],
+ global_llm_context="",
+ request_policy=RequestPolicy(),
+ handler=handler,
+ )
+
+ assert result.failure_kind == TurnIntentClassifierFailureKind.TIMEOUT
+
+
+async def test_classify_turn_intent_reports_converted_cancellation_as_timeout(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ monkeypatch.setattr(settings, "COPILOT_TURN_INTENT_CLASSIFIER_TIMEOUT_SECONDS", 0.001)
+
+ async def handler(prompt: str, prompt_name: str, **_: object) -> dict[str, object]:
+ try:
+ await asyncio.sleep(0.05)
+ except asyncio.CancelledError as exc:
+ raise RuntimeError("LLM request got cancelled") from exc
+ return {"mode": "build"}
+
+ result = await classify_turn_intent(
+ user_message="Build it",
+ workflow_yaml="",
+ chat_history=[],
+ global_llm_context="",
+ request_policy=RequestPolicy(),
+ handler=handler,
+ )
+
+ assert result.failure_kind == TurnIntentClassifierFailureKind.TIMEOUT
+
+
+async def test_classify_turn_intent_reports_provider_error() -> None:
+ async def handler(prompt: str, prompt_name: str, **_: object) -> dict[str, object]:
+ raise RuntimeError("provider unavailable")
+
+ result = await classify_turn_intent(
+ user_message="Build it",
+ workflow_yaml="",
+ chat_history=[],
+ global_llm_context="",
+ request_policy=RequestPolicy(),
+ handler=handler,
+ )
+
+ assert result.failure_kind == TurnIntentClassifierFailureKind.PROVIDER_ERROR
+
+
+async def test_classify_turn_intent_reports_malformed_output() -> None:
+ async def handler(prompt: str, prompt_name: str, **_: object) -> dict[str, object]:
+ return {"mode": "not_a_mode"}
+
+ result = await classify_turn_intent(
+ user_message="Build it",
+ workflow_yaml="",
+ chat_history=[],
+ global_llm_context="",
+ request_policy=RequestPolicy(),
+ handler=handler,
+ )
+
+ assert result.failure_kind == TurnIntentClassifierFailureKind.MALFORMED_OUTPUT
+
+
+async def test_classify_turn_intent_reports_prompt_render_error(monkeypatch: pytest.MonkeyPatch) -> None:
+ def raise_prompt_error(**_: object) -> str:
+ raise RuntimeError("template missing")
+
+ async def handler(prompt: str, prompt_name: str, **_: object) -> dict[str, object]:
+ raise AssertionError("handler should not be called")
+
+ monkeypatch.setattr("skyvern.forge.sdk.copilot.turn_intent.prompt_engine.load_prompt", raise_prompt_error)
+
+ result = await classify_turn_intent(
+ user_message="Build it",
+ workflow_yaml="",
+ chat_history=[],
+ global_llm_context="",
+ request_policy=RequestPolicy(),
+ handler=handler,
+ )
+
+ assert result.failure_kind == TurnIntentClassifierFailureKind.PROMPT_RENDER_ERROR