Skip to content

Automate browser based workflows with AI

SKY-10892: Harden request policy fallback (#6564)

Andrew Neilson0e97f5d8036d
diff --git a/skyvern/config.py b/skyvern/config.py
index a79a76448..da4f22024 100644
--- a/skyvern/config.py
+++ b/skyvern/config.py
@@ -145,6 +145,8 @@ class Settings(BaseSettings):
LOG_RAW_API_REQUESTS_SUCCESSFUL_READS: bool = False
LOG_LEVEL: str = "INFO"
COPILOT_FEASIBILITY_GATE_TIMEOUT_SECONDS: float = 12.0
+ COPILOT_REQUEST_POLICY_CLASSIFIER_TIMEOUT_SECONDS: float = 6.0
+ COPILOT_COMPLETION_JUDGE_TIMEOUT_SECONDS: float = 12.0
# Gate copilot verified-success on per-criterion outcome evidence.
# Off restores the prior run-status/suspicious-success gate and classifier prompt.
COPILOT_OUTCOME_VERIFICATION_ENABLED: bool = True
diff --git a/skyvern/forge/sdk/copilot/agent.py b/skyvern/forge/sdk/copilot/agent.py
index 2f2a8c13c..c14631657 100644
--- a/skyvern/forge/sdk/copilot/agent.py
+++ b/skyvern/forge/sdk/copilot/agent.py
@@ -65,6 +65,7 @@ from skyvern.forge.sdk.copilot.enforcement import (
verified_goal_claim_authorized,
)
from skyvern.forge.sdk.copilot.failure_tracking import PER_TOOL_BUDGET_FAILURE_CATEGORY
+from skyvern.forge.sdk.copilot.llm_errors import is_retriable_llm_error as _is_retriable_llm_error
from skyvern.forge.sdk.copilot.outcome_verification_trace import (
finalize_outcome_verification_trace,
record_criteria_lifecycle,
@@ -93,7 +94,6 @@ from skyvern.forge.sdk.copilot.recoverable_failure import (
build_recoverable_failure,
clean_recorded_failure_text,
format_recoverable_failure_reply,
- iter_exception_chain,
merge_failure_into_context,
)
from skyvern.forge.sdk.copilot.request_policy import (
@@ -2397,44 +2397,6 @@ def _build_feasibility_clarification_result(
)
-_RETRIABLE_LLM_ERROR_NAMES = {
- "APIConnectionError",
- "APITimeoutError",
- "APIError",
- "InternalServerError",
- "RateLimitError",
- "ServiceUnavailableError",
- "Timeout",
-}
-_RETRIABLE_LLM_ERROR_TEXT = (
- "rate limit",
- "timeout",
- "timed out",
- "temporarily unavailable",
- "service unavailable",
- "connection error",
- "connection reset",
- "internal server error",
- "server error",
- "overloaded",
-)
-_LLM_ERROR_MODULE_MARKERS = ("openai", "litellm", "anthropic")
-
-
-def _is_retriable_llm_error(exc: BaseException) -> bool:
- for item in iter_exception_chain(exc):
- module = type(item).__module__.lower()
- name = type(item).__name__
- text = str(item).lower()
- if name in _RETRIABLE_LLM_ERROR_NAMES and any(marker in module for marker in _LLM_ERROR_MODULE_MARKERS):
- return True
- if any(marker in module for marker in _LLM_ERROR_MODULE_MARKERS) and any(
- phrase in text for phrase in _RETRIABLE_LLM_ERROR_TEXT
- ):
- return True
- return False
-
-
def _fallback_llm_key(config: CopilotConfig, current_llm_key: str) -> str | None:
fallback_key = config.fallback_llm_key
if not fallback_key or fallback_key == current_llm_key:
diff --git a/skyvern/forge/sdk/copilot/completion_verification.py b/skyvern/forge/sdk/copilot/completion_verification.py
index 4a1f32e15..e5827bb05 100644
--- a/skyvern/forge/sdk/copilot/completion_verification.py
+++ b/skyvern/forge/sdk/copilot/completion_verification.py
@@ -406,7 +406,7 @@ async def evaluate_completion_criteria(
try:
raw = await asyncio.wait_for(
handler(prompt=prompt, prompt_name=PROMPT_TEMPLATE_NAME),
- timeout=settings.COPILOT_FEASIBILITY_GATE_TIMEOUT_SECONDS,
+ timeout=settings.COPILOT_COMPLETION_JUDGE_TIMEOUT_SECONDS,
)
except asyncio.TimeoutError:
LOG.warning("completion-verification judge timed out")
diff --git a/skyvern/forge/sdk/copilot/enforcement.py b/skyvern/forge/sdk/copilot/enforcement.py
index 4b8a6e856..9b03f8eec 100644
--- a/skyvern/forge/sdk/copilot/enforcement.py
+++ b/skyvern/forge/sdk/copilot/enforcement.py
@@ -560,6 +560,18 @@ def _request_completion_contract(ctx: Any) -> str | None:
return None
+def _request_completion_contract_status(ctx: Any) -> str:
+ request_policy = getattr(ctx, "request_policy", None)
+ status = getattr(request_policy, "completion_contract_status", None)
+ if status in ("present", "absent", "unknown"):
+ return status
+ return "present" if _request_completion_contract(ctx) else "absent"
+
+
+def _completion_contract_unknown_due_to_policy_fallback(ctx: Any) -> bool:
+ return _request_completion_contract_status(ctx) == "unknown"
+
+
def _nudge(config: CopilotConfig | None, key: str) -> str:
if config is None:
return DEFAULT_ENFORCEMENT_NUDGES[key]
@@ -721,8 +733,10 @@ def _response_coverage_nudge(ctx: Any, parsed: dict[str, Any], config: CopilotCo
# (06c). The getattr default keeps this gate working on partial stacks.
user_message = getattr(ctx, "user_message", "")
completion_contract = _request_completion_contract(ctx)
- if isinstance(block_count, int) and _goal_likely_needs_more_blocks(
- user_message, block_count, completion_contract
+ if (
+ isinstance(block_count, int)
+ and not _completion_contract_unknown_due_to_policy_fallback(ctx)
+ and _goal_likely_needs_more_blocks(user_message, block_count, completion_contract)
):
nudge_count = getattr(ctx, "coverage_nudge_count", 0)
if nudge_count < MAX_INTERMEDIATE_NUDGES:
diff --git a/skyvern/forge/sdk/copilot/llm_errors.py b/skyvern/forge/sdk/copilot/llm_errors.py
new file mode 100644
index 000000000..aac551311
--- /dev/null
+++ b/skyvern/forge/sdk/copilot/llm_errors.py
@@ -0,0 +1,49 @@
+from __future__ import annotations
+
+from collections.abc import Iterator
+
+
+def iter_exception_chain(exc: BaseException) -> Iterator[BaseException]:
+ seen: set[int] = set()
+ current: BaseException | None = exc
+ while current is not None and id(current) not in seen:
+ seen.add(id(current))
+ yield current
+ current = current.__cause__ or current.__context__
+
+
+_RETRIABLE_LLM_ERROR_NAMES = {
+ "APIConnectionError",
+ "APIError",
+ "APITimeoutError",
+ "InternalServerError",
+ "RateLimitError",
+ "ServiceUnavailableError",
+ "Timeout",
+}
+_RETRIABLE_LLM_ERROR_TEXT = (
+ "connection error",
+ "connection reset",
+ "internal server error",
+ "overloaded",
+ "rate limit",
+ "server error",
+ "service unavailable",
+ "temporarily unavailable",
+ "timed out",
+ "timeout",
+)
+_LLM_ERROR_MODULE_MARKERS = ("openai", "litellm", "anthropic")
+
+
+def is_retriable_llm_error(exc: BaseException) -> bool:
+ for item in iter_exception_chain(exc):
+ module = type(item).__module__.lower()
+ name = type(item).__name__
+ text = str(item).lower()
+ module_has_llm_marker = any(marker in module for marker in _LLM_ERROR_MODULE_MARKERS)
+ if name in _RETRIABLE_LLM_ERROR_NAMES and module_has_llm_marker:
+ return True
+ if module_has_llm_marker and any(phrase in text for phrase in _RETRIABLE_LLM_ERROR_TEXT):
+ return True
+ return False
diff --git a/skyvern/forge/sdk/copilot/request_policy.py b/skyvern/forge/sdk/copilot/request_policy.py
index bdae25793..81b21dbfe 100644
--- a/skyvern/forge/sdk/copilot/request_policy.py
+++ b/skyvern/forge/sdk/copilot/request_policy.py
@@ -14,6 +14,7 @@ from skyvern.config import settings
from skyvern.forge import app
from skyvern.forge.prompts import prompt_engine
from skyvern.forge.sdk.copilot.context import StructuredContext
+from skyvern.forge.sdk.copilot.llm_errors import is_retriable_llm_error
from skyvern.forge.sdk.copilot.output_utils import parse_final_response
from skyvern.forge.sdk.copilot.tracing_setup import copilot_span
from skyvern.forge.sdk.copilot.workflow_credential_utils import workflow_credential_ids, workflow_credential_origins
@@ -31,6 +32,27 @@ PROMPT_NAME = "workflow-copilot-request-policy"
_TESTING_INTENTS = {"require_test", "skip_test", "unspecified"}
_KINDS = {"none", "raw_secret", "credential_id", "credential_name", "website_stored_credential", "placeholder"}
_RAW_SECRET_HANDLINGS = {"none", "block", "redacted_draft"}
+_CLASSIFIER_FAILURE_KINDS = {
+ "none",
+ "missing_handler",
+ "raw_secret_no_handler",
+ "timeout",
+ "transient_error",
+ "provider_error",
+ "retry_exhausted",
+}
+_CLASSIFICATION_RESPONSE_FIELDS = {
+ "testing_intent",
+ "credential_input_kind",
+ "credential_refs",
+ "login_page_urls",
+ "requires_user_clarification",
+ "completion_contract",
+ "completion_criteria",
+ "raw_secret_evidence",
+ "raw_secret_handling",
+ "clarification_reason",
+}
ClarificationReason = Literal[
"none",
"raw_secret",
@@ -123,6 +145,11 @@ _INVALID_CONDITIONAL_CONTAINER_MARKERS = (
"inside conditional",
"within conditional",
)
+_QUOTED_CREDENTIAL_NAME_RE = re.compile(r"(?:`([^`]{1,100})`|\"([^\"]{1,100})\"|'([^']{1,100})')")
+_NAMED_CREDENTIAL_TOKEN_RE = re.compile(
+ r"\b(?:saved\s+credential|credential)\s+(?:named|called)\s+([A-Za-z0-9_.@:-]{2,100})\b",
+ re.I,
+)
_MAX_COMPLETION_CRITERIA = 8
@@ -169,6 +196,10 @@ class RequestPolicy:
existing_workflow_credential_ids: list[str] = field(default_factory=list)
# Sorted at the trace/JSON boundary; YAML traversal uses sets.
existing_workflow_credential_origins: dict[str, list[str]] = field(default_factory=dict)
+ classifier_status: str = "not_run"
+ classifier_failure_kind: str = "none"
+ classifier_retry_count: int = 0
+ completion_contract_status: str = "absent"
def to_trace_data(self) -> dict[str, Any]:
return {
@@ -190,6 +221,10 @@ class RequestPolicy:
"raw_secret_detected": self.raw_secret_detected,
"has_raw_secret_evidence": self.raw_secret_evidence is not None,
"raw_secret_handling": self.raw_secret_handling,
+ "classifier_status": self.classifier_status,
+ "classifier_failure_kind": self.classifier_failure_kind,
+ "classifier_retry_count": self.classifier_retry_count,
+ "completion_contract_status": self.completion_contract_status,
"existing_workflow_credential_id_count": len(self.existing_workflow_credential_ids),
"existing_workflow_credential_origin_count": sum(
len(origins) for origins in self.existing_workflow_credential_origins.values()
@@ -205,6 +240,8 @@ class RequestPolicy:
f"allow_run_blocks: {self.allow_run_blocks}",
f"allow_missing_credentials_in_draft: {self.allow_missing_credentials_in_draft}",
f"raw_secret_handling: {self.raw_secret_handling}",
+ f"classifier_status: {self.classifier_status}",
+ f"completion_contract_status: {self.completion_contract_status}",
]
if self.completion_contract:
lines.append(f"completion_contract: {self.completion_contract}")
@@ -468,10 +505,19 @@ def redact_raw_secrets_for_prompt(text: str) -> str:
return redacted
-def _classification_from_raw(raw: Any) -> RequestPolicy:
+def _coerce_classifier_payload(raw: Any) -> dict[str, Any] | None:
if isinstance(raw, str):
raw = parse_final_response(raw)
if not isinstance(raw, dict):
+ return None
+ if not any(field in raw for field in _CLASSIFICATION_RESPONSE_FIELDS):
+ return None
+ return raw
+
+
+def _classification_from_raw(raw: Any) -> RequestPolicy:
+ raw = _coerce_classifier_payload(raw)
+ if raw is None:
return RequestPolicy()
testing_intent = raw.get("testing_intent")
credential_input_kind = raw.get("credential_input_kind")
@@ -490,6 +536,8 @@ def _classification_from_raw(raw: Any) -> RequestPolicy:
raw_secret_evidence=raw_secret_evidence,
raw_secret_handling=_coerce_raw_secret_handling(raw.get("raw_secret_handling")),
clarification_reason=_coerce_clarification_reason(raw.get("clarification_reason")),
+ classifier_status="success",
+ classifier_failure_kind="none",
)
if policy.credential_input_kind == "raw_secret":
policy.clarification_reason = "raw_secret"
@@ -582,7 +630,15 @@ def _render_active_criteria_for_prompt(criteria: list[CompletionCriterion] | Non
)
-def _classifier_fallback_policy(ids: list[str], *, raw_secret_present: bool) -> RequestPolicy:
+def _classifier_fallback_policy(
+ ids: list[str],
+ *,
+ raw_secret_present: bool,
+ failure_kind: str,
+ retry_count: int = 0,
+) -> RequestPolicy:
+ if failure_kind not in _CLASSIFIER_FAILURE_KINDS:
+ failure_kind = "provider_error"
if raw_secret_present:
return RequestPolicy(
credential_input_kind="raw_secret",
@@ -590,8 +646,43 @@ def _classifier_fallback_policy(ids: list[str], *, raw_secret_present: bool) ->
raw_secret_detected=True,
raw_secret_handling="block",
clarification_reason="raw_secret",
+ classifier_status="fallback",
+ classifier_failure_kind=failure_kind,
+ classifier_retry_count=retry_count,
+ completion_contract_status="unknown",
)
- return RequestPolicy(credential_input_kind="credential_id" if ids else "none", credential_refs=ids)
+ return RequestPolicy(
+ credential_input_kind="credential_id" if ids else "none",
+ credential_refs=ids,
+ classifier_status="fallback",
+ classifier_failure_kind=failure_kind,
+ classifier_retry_count=retry_count,
+ completion_contract_status="unknown",
+ )
+
+
+async def _run_request_policy_classifier(handler: Any, prompt: str) -> tuple[Any | None, str, int]:
+ retry_count = 0
+ last_failure_kind = "none"
+ for attempt in range(2):
+ try:
+ raw = await asyncio.wait_for(
+ handler(prompt=prompt, prompt_name=PROMPT_NAME),
+ timeout=settings.COPILOT_REQUEST_POLICY_CLASSIFIER_TIMEOUT_SECONDS,
+ )
+ return raw, "none", retry_count
+ except asyncio.TimeoutError:
+ last_failure_kind = "timeout" if attempt == 0 else "retry_exhausted"
+ except Exception as exc:
+ if attempt == 0 and is_retriable_llm_error(exc):
+ last_failure_kind = "transient_error"
+ else:
+ failure_kind = "retry_exhausted" if retry_count and is_retriable_llm_error(exc) else "provider_error"
+ return None, failure_kind, retry_count
+ if attempt == 0:
+ retry_count += 1
+ continue
+ return None, last_failure_kind, retry_count
async def _classify_request(
@@ -606,7 +697,7 @@ async def _classify_request(
ids = _credential_ids(user_message)
raw_secret_present = _raw_secret_detected(user_message)
if raw_secret_present and handler is None:
- return _classifier_fallback_policy(ids, raw_secret_present=True)
+ return _classifier_fallback_policy(ids, raw_secret_present=True, failure_kind="raw_secret_no_handler")
structural_reason = _structural_clarification_reason(user_message)
if structural_reason != "none" and not raw_secret_present:
return RequestPolicy(
@@ -616,7 +707,7 @@ async def _classify_request(
clarification_reason=structural_reason,
)
if handler is None:
- return _classifier_fallback_policy(ids, raw_secret_present=False)
+ return _classifier_fallback_policy(ids, raw_secret_present=False, failure_kind="missing_handler")
# Raw-secret turns intentionally reach the classifier when available so it
# can distinguish unsafe secret use from redacted draft/spec conversion.
@@ -636,20 +727,32 @@ async def _classify_request(
global_llm_context=escape_code_fences(redact_raw_secrets_for_prompt(global_llm_context)[:2048]),
active_completion_criteria=escape_code_fences(_render_active_criteria_for_prompt(active_criteria)),
)
- try:
- raw = await asyncio.wait_for(
- handler(prompt=prompt, prompt_name=PROMPT_NAME),
- timeout=settings.COPILOT_FEASIBILITY_GATE_TIMEOUT_SECONDS,
+ raw, failure_kind, retry_count = await _run_request_policy_classifier(handler, prompt)
+ if raw is None:
+ LOG.warning("request-policy classifier failed", failure_kind=failure_kind, retry_count=retry_count)
+ return _classifier_fallback_policy(
+ ids,
+ raw_secret_present=raw_secret_present,
+ failure_kind=failure_kind,
+ retry_count=retry_count,
+ )
+
+ raw_payload = _coerce_classifier_payload(raw)
+ if raw_payload is None:
+ LOG.warning("request-policy classifier returned malformed payload")
+ return _classifier_fallback_policy(
+ ids,
+ raw_secret_present=raw_secret_present,
+ failure_kind="provider_error",
+ retry_count=retry_count,
)
- except asyncio.TimeoutError:
- LOG.warning("request-policy classifier timed out")
- return _classifier_fallback_policy(ids, raw_secret_present=raw_secret_present)
- except Exception as exc:
- LOG.warning("request-policy classifier failed", error=str(exc))
- return _classifier_fallback_policy(ids, raw_secret_present=raw_secret_present)
-
- policy = _classification_from_raw(raw)
+
+ policy = _classification_from_raw(raw_payload)
+ policy.classifier_retry_count = retry_count
policy.completion_contract = _ground_completion_contract(user_message, policy.completion_contract)
+ policy.completion_contract_status = (
+ "present" if policy.completion_contract or policy.completion_criteria else "absent"
+ )
classifier_credential_refs = [_canonicalize_credential_ref(ref) for ref in policy.credential_refs]
policy.credential_refs = _clean_list(classifier_credential_refs + ids)
if raw_secret_present:
@@ -704,6 +807,52 @@ async def _load_credentials(organization_id: str) -> list[Credential]:
page += 1
+def _fallback_credential_name_candidates(user_message: str) -> list[str]:
+ candidates: list[str] = []
+ for match in _QUOTED_CREDENTIAL_NAME_RE.finditer(user_message or ""):
+ value = next((group for group in match.groups() if group), "")
+ if value.strip():
+ candidates.append(value.strip())
+ for match in _NAMED_CREDENTIAL_TOKEN_RE.finditer(user_message or ""):
+ value = match.group(1).strip()
+ if value:
+ candidates.append(value)
+ return _clean_list(candidates)
+
+
+async def _apply_fallback_credential_name_scope(
+ policy: RequestPolicy,
+ *,
+ user_message: str,
+ organization_id: str,
+) -> None:
+ if policy.classifier_status != "fallback":
+ return
+ if policy.raw_secret_detected or policy.credential_input_kind not in ("none", "credential_name"):
+ return
+ candidates = _fallback_credential_name_candidates(user_message)
+ if not candidates:
+ return
+ credentials = await _load_credentials(organization_id)
+ matched_names = _clean_list(
+ [candidate for candidate in candidates if any(credential.name == candidate for credential in credentials)]
+ )
+ if len(matched_names) == 1:
+ policy.credential_input_kind = "credential_name"
+ policy.credential_refs = matched_names
+ policy.requires_user_clarification = False
+ policy.clarification_reason = "none"
+ policy.clarification_question = None
+ elif len(matched_names) > 1:
+ matches = [credential for credential in credentials if credential.name in matched_names]
+ _block(
+ policy,
+ "I found multiple saved credentials named in your request. Which one should I use?",
+ matches,
+ reason="credential_name_unresolved",
+ )
+
+
def _safe_label(credential: Credential) -> str:
parts = [f"`{credential.credential_id}`", credential.name]
parts += [f"Login Page URL: {credential.tested_url}"] if credential.tested_url else []
@@ -1128,6 +1277,18 @@ async def build_request_policy(
policy.existing_workflow_credential_origins = {
credential_id: sorted(origins) for credential_id, origins in workflow_credential_origins(workflow_yaml).items()
}
+ try:
+ await _apply_fallback_credential_name_scope(
+ policy,
+ user_message=user_message,
+ organization_id=organization_id,
+ )
+ except Exception:
+ LOG.warning(
+ "request-policy fallback credential-name extraction failed",
+ organization_id=organization_id,
+ exc_info=True,
+ )
try:
await _seed_discovered_credentials(
policy,
@@ -1221,6 +1382,9 @@ async def build_request_policy(
"I could not verify the requested credential metadata for this organization. Please provide a valid saved credential by exact name or a credential ID beginning with cred_.",
)
- with copilot_span("request_policy", data=policy.to_trace_data()):
- LOG.info("request-policy decision", **policy.to_trace_data())
+ trace_data = policy.to_trace_data()
+ if policy.classifier_status == "fallback":
+ LOG.warning("request-policy fallback policy used", **trace_data)
+ with copilot_span("request_policy", data=trace_data):
+ LOG.info("request-policy decision", **trace_data)
return policy
diff --git a/skyvern/forge/sdk/copilot/tools/completion.py b/skyvern/forge/sdk/copilot/tools/completion.py
index e3278181b..1892d1ac3 100644
--- a/skyvern/forge/sdk/copilot/tools/completion.py
+++ b/skyvern/forge/sdk/copilot/tools/completion.py
@@ -146,7 +146,7 @@ async def _maybe_run_completion_verification_from_page_observation(
if (
remaining is not None
and remaining
- <= settings.COPILOT_FEASIBILITY_GATE_TIMEOUT_SECONDS + _COMPLETION_VERIFICATION_BUDGET_MARGIN_SECONDS
+ <= settings.COPILOT_COMPLETION_JUDGE_TIMEOUT_SECONDS + _COMPLETION_VERIFICATION_BUDGET_MARGIN_SECONDS
):
verification = CompletionVerificationResult(status="unavailable")
else:
@@ -298,7 +298,7 @@ async def _maybe_run_completion_verification(
# Too little budget to verify a candidate run: fail closed (unavailable) rather
# than let the run-status proxy claim an unverified outcome as success.
remaining = RUN_BLOCKS_SAFETY_CEILING_SECONDS - (time.monotonic() - handler_start)
- if remaining <= settings.COPILOT_FEASIBILITY_GATE_TIMEOUT_SECONDS + _COMPLETION_VERIFICATION_BUDGET_MARGIN_SECONDS:
+ if remaining <= settings.COPILOT_COMPLETION_JUDGE_TIMEOUT_SECONDS + _COMPLETION_VERIFICATION_BUDGET_MARGIN_SECONDS:
return CompletionVerificationResult(status="unavailable")
snapshot = _build_run_evidence_snapshot(copilot_ctx, result)
if not snapshot.has_evidence():
diff --git a/skyvern/forge/sdk/copilot/tools/workflow_update.py b/skyvern/forge/sdk/copilot/tools/workflow_update.py
index fb91fac23..5376ec6ef 100644
--- a/skyvern/forge/sdk/copilot/tools/workflow_update.py
+++ b/skyvern/forge/sdk/copilot/tools/workflow_update.py
@@ -30,6 +30,7 @@ from skyvern.forge.sdk.copilot.config import BlockAuthoringPolicy
from skyvern.forge.sdk.copilot.context import CopilotContext
from skyvern.forge.sdk.copilot.enforcement import (
POST_INTERMEDIATE_SUCCESS_NUDGE,
+ _completion_contract_unknown_due_to_policy_fallback,
_goal_likely_needs_more_blocks,
)
from skyvern.forge.sdk.copilot.loop_detection import clear_failed_step_tracker_for_tools_in_ctx
@@ -1684,6 +1685,9 @@ def _pre_run_workflow_coverage_error(copilot_ctx: Any) -> str | None:
else:
completion_contract = None
+ if _completion_contract_unknown_due_to_policy_fallback(copilot_ctx):
+ return None
+
if not _goal_likely_needs_more_blocks(user_message, block_count, completion_contract):
return None
diff --git a/tests/unit/test_copilot_agent_helpers.py b/tests/unit/test_copilot_agent_helpers.py
index df7247283..b5dd9d589 100644
--- a/tests/unit/test_copilot_agent_helpers.py
+++ b/tests/unit/test_copilot_agent_helpers.py
@@ -122,6 +122,19 @@ class TestFailedTestResponseNormalization:
assert "has not been run" in error
assert ctx.coverage_nudge_count == 1
+ def test_pre_run_coverage_guard_skips_unknown_completion_contract(self) -> None:
+ from skyvern.forge.sdk.copilot.tools import _pre_run_workflow_coverage_error
+
+ ctx = _ctx(
+ user_message="Go to https://example.com/contact. Fill out the contact form and submit it.",
+ request_policy=SimpleNamespace(completion_contract_status="unknown"),
+ last_update_block_count=1,
+ coverage_nudge_count=0,
+ )
+
+ assert _pre_run_workflow_coverage_error(ctx) is None
+ assert ctx.coverage_nudge_count == 0
+
def test_pre_run_coverage_guard_allows_single_final_action_contract(self) -> None:
from skyvern.forge.sdk.copilot.tools import _pre_run_workflow_coverage_error
@@ -2088,6 +2101,88 @@ class TestNativeToolSurface:
class TestRequestPolicyCredentialResolution:
+ @pytest.mark.asyncio
+ async def test_request_policy_classifier_timeout_retries_once(self, monkeypatch) -> None:
+ from skyvern.config import settings
+
+ monkeypatch.setattr(settings, "COPILOT_REQUEST_POLICY_CLASSIFIER_TIMEOUT_SECONDS", 0.01)
+ calls = 0
+
+ async def handler(*, prompt: str, prompt_name: str) -> dict[str, object]:
+ nonlocal calls
+ calls += 1
+ if calls == 1:
+ await asyncio.sleep(0.05)
+ return {"credential_input_kind": "none", "completion_contract": None}
+
+ policy = await _classify_request(
+ user_message="Build a workflow for https://example.com.",
+ workflow_yaml="",
+ chat_history=[],
+ global_llm_context="",
+ handler=handler,
+ )
+
+ assert calls == 2
+ assert policy.classifier_status == "success"
+ assert policy.classifier_retry_count == 1
+ assert policy.completion_contract_status == "absent"
+
+ @pytest.mark.asyncio
+ async def test_request_policy_classifier_malformed_payload_falls_back_without_retry(self) -> None:
+ calls = 0
+
+ async def handler(*, prompt: str, prompt_name: str) -> str:
+ nonlocal calls
+ calls += 1
+ return "I can build that workflow."
+
+ policy = await _classify_request(
+ user_message="Build a workflow for https://example.com.",
+ workflow_yaml="",
+ chat_history=[],
+ global_llm_context="",
+ handler=handler,
+ )
+
+ assert calls == 1
+ assert policy.classifier_status == "fallback"
+ assert policy.classifier_failure_kind == "provider_error"
+ assert policy.classifier_retry_count == 0
+ assert policy.completion_contract_status == "unknown"
+
+ @pytest.mark.asyncio
+ async def test_request_policy_classifier_transient_provider_error_retries_once(self) -> None:
+ class RateLimitError(Exception):
+ __module__ = "openai"
+
+ calls = 0
+
+ async def handler(*, prompt: str, prompt_name: str) -> dict[str, object]:
+ nonlocal calls
+ calls += 1
+ if calls == 1:
+ raise RateLimitError("rate limit")
+ return {
+ "credential_input_kind": "none",
+ "completion_contract": "complete when the account page is visible",
+ }
+
+ policy = await _classify_request(
+ user_message=(
+ "Build a workflow for https://example.com/account. complete when the account page is visible"
+ ),
+ workflow_yaml="",
+ chat_history=[],
+ global_llm_context="",
+ handler=handler,
+ )
+
+ assert calls == 2
+ assert policy.classifier_status == "success"
+ assert policy.classifier_retry_count == 1
+ assert policy.completion_contract_status == "present"
+
@pytest.mark.asyncio
async def test_missing_user_supplied_credential_ids_ask_for_clarification(self, monkeypatch) -> None:
from skyvern.forge.sdk.copilot import request_policy as policy_module
@@ -2547,6 +2642,65 @@ workflow_definition:
assert policy.allow_missing_credentials_in_draft is True
assert policy.requires_user_clarification is False
+ @pytest.mark.asyncio
+ async def test_fallback_policy_resolves_explicit_saved_credential_name(self, monkeypatch) -> None:
+ from skyvern.forge.sdk.copilot import request_policy as policy_module
+ from skyvern.forge.sdk.copilot.request_policy import build_request_policy
+
+ credential = SimpleNamespace(
+ credential_id="cred_portal",
+ name="mock-portal-login",
+ tested_url="https://portal.example/login",
+ )
+ credentials = SimpleNamespace(get_credentials=AsyncMock(return_value=[credential]))
+ monkeypatch.setattr(policy_module.app, "DATABASE", SimpleNamespace(credentials=credentials))
+
+ async def handler(*, prompt: str, prompt_name: str) -> dict[str, object]:
+ raise ValueError("classifier unavailable")
+
+ policy = await build_request_policy(
+ user_message='Log into the portal with saved credential named "mock-portal-login".',
+ workflow_yaml="",
+ chat_history=[],
+ global_llm_context="",
+ organization_id="org-1",
+ handler=handler,
+ )
+
+ assert policy.classifier_status == "fallback"
+ assert policy.classifier_failure_kind == "provider_error"
+ assert policy.completion_contract_status == "unknown"
+ assert policy.credential_input_kind == "credential_name"
+ assert policy.credential_refs == ["mock-portal-login"]
+ assert policy.resolved_credentials == [credential]
+
+ @pytest.mark.asyncio
+ async def test_fallback_policy_does_not_substring_match_saved_credential_name(self, monkeypatch) -> None:
+ from skyvern.forge.sdk.copilot import request_policy as policy_module
+ from skyvern.forge.sdk.copilot.request_policy import build_request_policy
+
+ credential = SimpleNamespace(credential_id="cred_login", name="login", tested_url=None)
+ credentials = SimpleNamespace(get_credentials=AsyncMock(return_value=[credential]))
+ monkeypatch.setattr(policy_module.app, "DATABASE", SimpleNamespace(credentials=credentials))
+
+ async def handler(*, prompt: str, prompt_name: str) -> dict[str, object]:
+ raise ValueError("classifier unavailable")
+
+ policy = await build_request_policy(
+ user_message="Log in to the portal using my saved credential.",
+ workflow_yaml="",
+ chat_history=[],
+ global_llm_context="",
+ organization_id="org-1",
+ handler=handler,
+ )
+
+ assert policy.classifier_status == "fallback"
+ assert policy.completion_contract_status == "unknown"
+ assert policy.credential_input_kind == "none"
+ assert policy.credential_refs == []
+ credentials.get_credentials.assert_not_called()
+
@pytest.mark.asyncio
async def test_request_policy_url_matching_and_skip_draft(self, monkeypatch) -> None:
from skyvern.forge.sdk.copilot import request_policy as policy_module
diff --git a/tests/unit/test_copilot_completion_verification.py b/tests/unit/test_copilot_completion_verification.py
index 4a83308ba..7da1e7fef 100644
--- a/tests/unit/test_copilot_completion_verification.py
+++ b/tests/unit/test_copilot_completion_verification.py
@@ -1,5 +1,6 @@
from __future__ import annotations
+import asyncio
import time
from collections.abc import Awaitable, Callable
from types import SimpleNamespace
@@ -149,6 +150,21 @@ async def test_evaluate_handler_exception_is_unavailable() -> None:
assert result.status == "unavailable"
+async def test_evaluate_uses_completion_judge_timeout(monkeypatch: pytest.MonkeyPatch) -> None:
+ monkeypatch.setattr(settings, "COPILOT_COMPLETION_JUDGE_TIMEOUT_SECONDS", 0.01)
+ monkeypatch.setattr(settings, "COPILOT_FEASIBILITY_GATE_TIMEOUT_SECONDS", 10.0)
+
+ async def handler(**_: object) -> dict[str, object]:
+ await asyncio.sleep(0.05)
+ return {"verdicts": [{"criterion_id": "c0", "satisfied": True, "reason_code": "evidence_confirms"}]}
+
+ snapshot = RunEvidenceSnapshot(current_url="https://example.com/done")
+ result = await evaluate_completion_criteria([_criterion("c0", "done page visible")], snapshot, handler)
+
+ assert result.status == "unavailable"
+
+
@pytest.mark.asyncio
async def test_evaluate_happy_path_returns_evaluated() -> None:
async def handler(**_: object) -> dict:
diff --git a/tests/unit/test_copilot_response_coverage_gate.py b/tests/unit/test_copilot_response_coverage_gate.py
index fb0cceefe..39757c5e0 100644
--- a/tests/unit/test_copilot_response_coverage_gate.py
+++ b/tests/unit/test_copilot_response_coverage_gate.py
@@ -117,6 +117,15 @@ def test_reply_after_success_with_request_policy_completion_contract_passes_thro
assert ctx.coverage_nudge_count == 0
+def test_reply_after_success_with_unknown_fallback_contract_passes_through() -> None:
+ ctx = _post_success_ctx("Go to https://example.com/contact. Fill out the contact form and submit it.")
+ ctx.request_policy = SimpleNamespace(completion_contract_status="unknown")
+ parsed = {"type": "REPLY", "user_response": "I created and tested the workflow."}
+
+ assert _response_coverage_nudge(ctx, parsed) is None
+ assert ctx.coverage_nudge_count == 0
+
+
def test_coverage_nudge_respects_counter_cap() -> None:
ctx = _post_success_ctx("go to X and download Y")
parsed = {"type": "REPLY", "user_response": "one block draft"}