Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
SKY-10892: Harden request policy fallback (#6564)
Andrew Neilson0e97f5d8036d
diff --git a/skyvern/config.py b/skyvern/config.pyindex a79a76448..da4f22024 100644--- a/skyvern/config.py+++ b/skyvern/config.py@@ -145,6 +145,8 @@ class Settings(BaseSettings):LOG_RAW_API_REQUESTS_SUCCESSFUL_READS: bool = FalseLOG_LEVEL: str = "INFO"COPILOT_FEASIBILITY_GATE_TIMEOUT_SECONDS: float = 12.0+ COPILOT_REQUEST_POLICY_CLASSIFIER_TIMEOUT_SECONDS: float = 6.0+ COPILOT_COMPLETION_JUDGE_TIMEOUT_SECONDS: float = 12.0# Gate copilot verified-success on per-criterion outcome evidence.# Off restores the prior run-status/suspicious-success gate and classifier prompt.COPILOT_OUTCOME_VERIFICATION_ENABLED: bool = Truediff --git a/skyvern/forge/sdk/copilot/agent.py b/skyvern/forge/sdk/copilot/agent.pyindex 2f2a8c13c..c14631657 100644--- a/skyvern/forge/sdk/copilot/agent.py+++ b/skyvern/forge/sdk/copilot/agent.py@@ -65,6 +65,7 @@ from skyvern.forge.sdk.copilot.enforcement import (verified_goal_claim_authorized,)from skyvern.forge.sdk.copilot.failure_tracking import PER_TOOL_BUDGET_FAILURE_CATEGORY+from skyvern.forge.sdk.copilot.llm_errors import is_retriable_llm_error as _is_retriable_llm_errorfrom skyvern.forge.sdk.copilot.outcome_verification_trace import (finalize_outcome_verification_trace,record_criteria_lifecycle,@@ -93,7 +94,6 @@ from skyvern.forge.sdk.copilot.recoverable_failure import (build_recoverable_failure,clean_recorded_failure_text,format_recoverable_failure_reply,- iter_exception_chain,merge_failure_into_context,)from skyvern.forge.sdk.copilot.request_policy import (@@ -2397,44 +2397,6 @@ def _build_feasibility_clarification_result()-_RETRIABLE_LLM_ERROR_NAMES = {- "APIConnectionError",- "APITimeoutError",- "APIError",- "InternalServerError",- "RateLimitError",- "ServiceUnavailableError",- "Timeout",-}-_RETRIABLE_LLM_ERROR_TEXT = (- "rate limit",- "timeout",- "timed out",- "temporarily unavailable",- "service unavailable",- "connection error",- "connection reset",- "internal server error",- "server error",- "overloaded",-)-_LLM_ERROR_MODULE_MARKERS = ("openai", "litellm", "anthropic")---def _is_retriable_llm_error(exc: BaseException) -> bool:- for item in iter_exception_chain(exc):- module = type(item).__module__.lower()- name = type(item).__name__- text = str(item).lower()- if name in _RETRIABLE_LLM_ERROR_NAMES and any(marker in module for marker in _LLM_ERROR_MODULE_MARKERS):- return True- if any(marker in module for marker in _LLM_ERROR_MODULE_MARKERS) and any(- phrase in text for phrase in _RETRIABLE_LLM_ERROR_TEXT- ):- return True- return False--def _fallback_llm_key(config: CopilotConfig, current_llm_key: str) -> str | None:fallback_key = config.fallback_llm_keyif not fallback_key or fallback_key == current_llm_key:diff --git a/skyvern/forge/sdk/copilot/completion_verification.py b/skyvern/forge/sdk/copilot/completion_verification.pyindex 4a1f32e15..e5827bb05 100644--- a/skyvern/forge/sdk/copilot/completion_verification.py+++ b/skyvern/forge/sdk/copilot/completion_verification.py@@ -406,7 +406,7 @@ async def evaluate_completion_criteria(try:raw = await asyncio.wait_for(handler(prompt=prompt, prompt_name=PROMPT_TEMPLATE_NAME),- timeout=settings.COPILOT_FEASIBILITY_GATE_TIMEOUT_SECONDS,+ timeout=settings.COPILOT_COMPLETION_JUDGE_TIMEOUT_SECONDS,)except asyncio.TimeoutError:LOG.warning("completion-verification judge timed out")diff --git a/skyvern/forge/sdk/copilot/enforcement.py b/skyvern/forge/sdk/copilot/enforcement.pyindex 4b8a6e856..9b03f8eec 100644--- a/skyvern/forge/sdk/copilot/enforcement.py+++ b/skyvern/forge/sdk/copilot/enforcement.py@@ -560,6 +560,18 @@ def _request_completion_contract(ctx: Any) -> str | None:return None+def _request_completion_contract_status(ctx: Any) -> str:+ request_policy = getattr(ctx, "request_policy", None)+ status = getattr(request_policy, "completion_contract_status", None)+ if status in ("present", "absent", "unknown"):+ return status+ return "present" if _request_completion_contract(ctx) else "absent"+++def _completion_contract_unknown_due_to_policy_fallback(ctx: Any) -> bool:+ return _request_completion_contract_status(ctx) == "unknown"++def _nudge(config: CopilotConfig | None, key: str) -> str:if config is None:return DEFAULT_ENFORCEMENT_NUDGES[key]@@ -721,8 +733,10 @@ def _response_coverage_nudge(ctx: Any, parsed: dict[str, Any], config: CopilotCo# (06c). The getattr default keeps this gate working on partial stacks.user_message = getattr(ctx, "user_message", "")completion_contract = _request_completion_contract(ctx)- if isinstance(block_count, int) and _goal_likely_needs_more_blocks(- user_message, block_count, completion_contract+ if (+ isinstance(block_count, int)+ and not _completion_contract_unknown_due_to_policy_fallback(ctx)+ and _goal_likely_needs_more_blocks(user_message, block_count, completion_contract)):nudge_count = getattr(ctx, "coverage_nudge_count", 0)if nudge_count < MAX_INTERMEDIATE_NUDGES:diff --git a/skyvern/forge/sdk/copilot/llm_errors.py b/skyvern/forge/sdk/copilot/llm_errors.pynew file mode 100644index 000000000..aac551311--- /dev/null+++ b/skyvern/forge/sdk/copilot/llm_errors.py@@ -0,0 +1,49 @@+from __future__ import annotations++from collections.abc import Iterator+++def iter_exception_chain(exc: BaseException) -> Iterator[BaseException]:+ seen: set[int] = set()+ current: BaseException | None = exc+ while current is not None and id(current) not in seen:+ seen.add(id(current))+ yield current+ current = current.__cause__ or current.__context__+++_RETRIABLE_LLM_ERROR_NAMES = {+ "APIConnectionError",+ "APIError",+ "APITimeoutError",+ "InternalServerError",+ "RateLimitError",+ "ServiceUnavailableError",+ "Timeout",+}+_RETRIABLE_LLM_ERROR_TEXT = (+ "connection error",+ "connection reset",+ "internal server error",+ "overloaded",+ "rate limit",+ "server error",+ "service unavailable",+ "temporarily unavailable",+ "timed out",+ "timeout",+)+_LLM_ERROR_MODULE_MARKERS = ("openai", "litellm", "anthropic")+++def is_retriable_llm_error(exc: BaseException) -> bool:+ for item in iter_exception_chain(exc):+ module = type(item).__module__.lower()+ name = type(item).__name__+ text = str(item).lower()+ module_has_llm_marker = any(marker in module for marker in _LLM_ERROR_MODULE_MARKERS)+ if name in _RETRIABLE_LLM_ERROR_NAMES and module_has_llm_marker:+ return True+ if module_has_llm_marker and any(phrase in text for phrase in _RETRIABLE_LLM_ERROR_TEXT):+ return True+ return Falsediff --git a/skyvern/forge/sdk/copilot/request_policy.py b/skyvern/forge/sdk/copilot/request_policy.pyindex bdae25793..81b21dbfe 100644--- a/skyvern/forge/sdk/copilot/request_policy.py+++ b/skyvern/forge/sdk/copilot/request_policy.py@@ -14,6 +14,7 @@ from skyvern.config import settingsfrom skyvern.forge import appfrom skyvern.forge.prompts import prompt_enginefrom skyvern.forge.sdk.copilot.context import StructuredContext+from skyvern.forge.sdk.copilot.llm_errors import is_retriable_llm_errorfrom skyvern.forge.sdk.copilot.output_utils import parse_final_responsefrom skyvern.forge.sdk.copilot.tracing_setup import copilot_spanfrom skyvern.forge.sdk.copilot.workflow_credential_utils import workflow_credential_ids, workflow_credential_origins@@ -31,6 +32,27 @@ PROMPT_NAME = "workflow-copilot-request-policy"_TESTING_INTENTS = {"require_test", "skip_test", "unspecified"}_KINDS = {"none", "raw_secret", "credential_id", "credential_name", "website_stored_credential", "placeholder"}_RAW_SECRET_HANDLINGS = {"none", "block", "redacted_draft"}+_CLASSIFIER_FAILURE_KINDS = {+ "none",+ "missing_handler",+ "raw_secret_no_handler",+ "timeout",+ "transient_error",+ "provider_error",+ "retry_exhausted",+}+_CLASSIFICATION_RESPONSE_FIELDS = {+ "testing_intent",+ "credential_input_kind",+ "credential_refs",+ "login_page_urls",+ "requires_user_clarification",+ "completion_contract",+ "completion_criteria",+ "raw_secret_evidence",+ "raw_secret_handling",+ "clarification_reason",+}ClarificationReason = Literal["none","raw_secret",@@ -123,6 +145,11 @@ _INVALID_CONDITIONAL_CONTAINER_MARKERS = ("inside conditional","within conditional",)+_QUOTED_CREDENTIAL_NAME_RE = re.compile(r"(?:`([^`]{1,100})`|\"([^\"]{1,100})\"|'([^']{1,100})')")+_NAMED_CREDENTIAL_TOKEN_RE = re.compile(+ r"\b(?:saved\s+credential|credential)\s+(?:named|called)\s+([A-Za-z0-9_.@:-]{2,100})\b",+ re.I,+)_MAX_COMPLETION_CRITERIA = 8@@ -169,6 +196,10 @@ class RequestPolicy:existing_workflow_credential_ids: list[str] = field(default_factory=list)# Sorted at the trace/JSON boundary; YAML traversal uses sets.existing_workflow_credential_origins: dict[str, list[str]] = field(default_factory=dict)+ classifier_status: str = "not_run"+ classifier_failure_kind: str = "none"+ classifier_retry_count: int = 0+ completion_contract_status: str = "absent"def to_trace_data(self) -> dict[str, Any]:return {@@ -190,6 +221,10 @@ class RequestPolicy:"raw_secret_detected": self.raw_secret_detected,"has_raw_secret_evidence": self.raw_secret_evidence is not None,"raw_secret_handling": self.raw_secret_handling,+ "classifier_status": self.classifier_status,+ "classifier_failure_kind": self.classifier_failure_kind,+ "classifier_retry_count": self.classifier_retry_count,+ "completion_contract_status": self.completion_contract_status,"existing_workflow_credential_id_count": len(self.existing_workflow_credential_ids),"existing_workflow_credential_origin_count": sum(len(origins) for origins in self.existing_workflow_credential_origins.values()@@ -205,6 +240,8 @@ class RequestPolicy:f"allow_run_blocks: {self.allow_run_blocks}",f"allow_missing_credentials_in_draft: {self.allow_missing_credentials_in_draft}",f"raw_secret_handling: {self.raw_secret_handling}",+ f"classifier_status: {self.classifier_status}",+ f"completion_contract_status: {self.completion_contract_status}",]if self.completion_contract:lines.append(f"completion_contract: {self.completion_contract}")@@ -468,10 +505,19 @@ def redact_raw_secrets_for_prompt(text: str) -> str:return redacted-def _classification_from_raw(raw: Any) -> RequestPolicy:+def _coerce_classifier_payload(raw: Any) -> dict[str, Any] | None:if isinstance(raw, str):raw = parse_final_response(raw)if not isinstance(raw, dict):+ return None+ if not any(field in raw for field in _CLASSIFICATION_RESPONSE_FIELDS):+ return None+ return raw+++def _classification_from_raw(raw: Any) -> RequestPolicy:+ raw = _coerce_classifier_payload(raw)+ if raw is None:return RequestPolicy()testing_intent = raw.get("testing_intent")credential_input_kind = raw.get("credential_input_kind")@@ -490,6 +536,8 @@ def _classification_from_raw(raw: Any) -> RequestPolicy:raw_secret_evidence=raw_secret_evidence,raw_secret_handling=_coerce_raw_secret_handling(raw.get("raw_secret_handling")),clarification_reason=_coerce_clarification_reason(raw.get("clarification_reason")),+ classifier_status="success",+ classifier_failure_kind="none",)if policy.credential_input_kind == "raw_secret":policy.clarification_reason = "raw_secret"@@ -582,7 +630,15 @@ def _render_active_criteria_for_prompt(criteria: list[CompletionCriterion] | Non)-def _classifier_fallback_policy(ids: list[str], *, raw_secret_present: bool) -> RequestPolicy:+def _classifier_fallback_policy(+ ids: list[str],+ *,+ raw_secret_present: bool,+ failure_kind: str,+ retry_count: int = 0,+) -> RequestPolicy:+ if failure_kind not in _CLASSIFIER_FAILURE_KINDS:+ failure_kind = "provider_error"if raw_secret_present:return RequestPolicy(credential_input_kind="raw_secret",@@ -590,8 +646,43 @@ def _classifier_fallback_policy(ids: list[str], *, raw_secret_present: bool) ->raw_secret_detected=True,raw_secret_handling="block",clarification_reason="raw_secret",+ classifier_status="fallback",+ classifier_failure_kind=failure_kind,+ classifier_retry_count=retry_count,+ completion_contract_status="unknown",)- return RequestPolicy(credential_input_kind="credential_id" if ids else "none", credential_refs=ids)+ return RequestPolicy(+ credential_input_kind="credential_id" if ids else "none",+ credential_refs=ids,+ classifier_status="fallback",+ classifier_failure_kind=failure_kind,+ classifier_retry_count=retry_count,+ completion_contract_status="unknown",+ )+++async def _run_request_policy_classifier(handler: Any, prompt: str) -> tuple[Any | None, str, int]:+ retry_count = 0+ last_failure_kind = "none"+ for attempt in range(2):+ try:+ raw = await asyncio.wait_for(+ handler(prompt=prompt, prompt_name=PROMPT_NAME),+ timeout=settings.COPILOT_REQUEST_POLICY_CLASSIFIER_TIMEOUT_SECONDS,+ )+ return raw, "none", retry_count+ except asyncio.TimeoutError:+ last_failure_kind = "timeout" if attempt == 0 else "retry_exhausted"+ except Exception as exc:+ if attempt == 0 and is_retriable_llm_error(exc):+ last_failure_kind = "transient_error"+ else:+ failure_kind = "retry_exhausted" if retry_count and is_retriable_llm_error(exc) else "provider_error"+ return None, failure_kind, retry_count+ if attempt == 0:+ retry_count += 1+ continue+ return None, last_failure_kind, retry_countasync def _classify_request(@@ -606,7 +697,7 @@ async def _classify_request(ids = _credential_ids(user_message)raw_secret_present = _raw_secret_detected(user_message)if raw_secret_present and handler is None:- return _classifier_fallback_policy(ids, raw_secret_present=True)+ return _classifier_fallback_policy(ids, raw_secret_present=True, failure_kind="raw_secret_no_handler")structural_reason = _structural_clarification_reason(user_message)if structural_reason != "none" and not raw_secret_present:return RequestPolicy(@@ -616,7 +707,7 @@ async def _classify_request(clarification_reason=structural_reason,)if handler is None:- return _classifier_fallback_policy(ids, raw_secret_present=False)+ return _classifier_fallback_policy(ids, raw_secret_present=False, failure_kind="missing_handler")# Raw-secret turns intentionally reach the classifier when available so it# can distinguish unsafe secret use from redacted draft/spec conversion.@@ -636,20 +727,32 @@ async def _classify_request(global_llm_context=escape_code_fences(redact_raw_secrets_for_prompt(global_llm_context)[:2048]),active_completion_criteria=escape_code_fences(_render_active_criteria_for_prompt(active_criteria)),)- try:- raw = await asyncio.wait_for(- handler(prompt=prompt, prompt_name=PROMPT_NAME),- timeout=settings.COPILOT_FEASIBILITY_GATE_TIMEOUT_SECONDS,+ raw, failure_kind, retry_count = await _run_request_policy_classifier(handler, prompt)+ if raw is None:+ LOG.warning("request-policy classifier failed", failure_kind=failure_kind, retry_count=retry_count)+ return _classifier_fallback_policy(+ ids,+ raw_secret_present=raw_secret_present,+ failure_kind=failure_kind,+ retry_count=retry_count,+ )++ raw_payload = _coerce_classifier_payload(raw)+ if raw_payload is None:+ LOG.warning("request-policy classifier returned malformed payload")+ return _classifier_fallback_policy(+ ids,+ raw_secret_present=raw_secret_present,+ failure_kind="provider_error",+ retry_count=retry_count,)- except asyncio.TimeoutError:- LOG.warning("request-policy classifier timed out")- return _classifier_fallback_policy(ids, raw_secret_present=raw_secret_present)- except Exception as exc:- LOG.warning("request-policy classifier failed", error=str(exc))- return _classifier_fallback_policy(ids, raw_secret_present=raw_secret_present)-- policy = _classification_from_raw(raw)++ policy = _classification_from_raw(raw_payload)+ policy.classifier_retry_count = retry_countpolicy.completion_contract = _ground_completion_contract(user_message, policy.completion_contract)+ policy.completion_contract_status = (+ "present" if policy.completion_contract or policy.completion_criteria else "absent"+ )classifier_credential_refs = [_canonicalize_credential_ref(ref) for ref in policy.credential_refs]policy.credential_refs = _clean_list(classifier_credential_refs + ids)if raw_secret_present:@@ -704,6 +807,52 @@ async def _load_credentials(organization_id: str) -> list[Credential]:page += 1+def _fallback_credential_name_candidates(user_message: str) -> list[str]:+ candidates: list[str] = []+ for match in _QUOTED_CREDENTIAL_NAME_RE.finditer(user_message or ""):+ value = next((group for group in match.groups() if group), "")+ if value.strip():+ candidates.append(value.strip())+ for match in _NAMED_CREDENTIAL_TOKEN_RE.finditer(user_message or ""):+ value = match.group(1).strip()+ if value:+ candidates.append(value)+ return _clean_list(candidates)+++async def _apply_fallback_credential_name_scope(+ policy: RequestPolicy,+ *,+ user_message: str,+ organization_id: str,+) -> None:+ if policy.classifier_status != "fallback":+ return+ if policy.raw_secret_detected or policy.credential_input_kind not in ("none", "credential_name"):+ return+ candidates = _fallback_credential_name_candidates(user_message)+ if not candidates:+ return+ credentials = await _load_credentials(organization_id)+ matched_names = _clean_list(+ [candidate for candidate in candidates if any(credential.name == candidate for credential in credentials)]+ )+ if len(matched_names) == 1:+ policy.credential_input_kind = "credential_name"+ policy.credential_refs = matched_names+ policy.requires_user_clarification = False+ policy.clarification_reason = "none"+ policy.clarification_question = None+ elif len(matched_names) > 1:+ matches = [credential for credential in credentials if credential.name in matched_names]+ _block(+ policy,+ "I found multiple saved credentials named in your request. Which one should I use?",+ matches,+ reason="credential_name_unresolved",+ )++def _safe_label(credential: Credential) -> str:parts = [f"`{credential.credential_id}`", credential.name]parts += [f"Login Page URL: {credential.tested_url}"] if credential.tested_url else []@@ -1128,6 +1277,18 @@ async def build_request_policy(policy.existing_workflow_credential_origins = {credential_id: sorted(origins) for credential_id, origins in workflow_credential_origins(workflow_yaml).items()}+ try:+ await _apply_fallback_credential_name_scope(+ policy,+ user_message=user_message,+ organization_id=organization_id,+ )+ except Exception:+ LOG.warning(+ "request-policy fallback credential-name extraction failed",+ organization_id=organization_id,+ exc_info=True,+ )try:await _seed_discovered_credentials(policy,@@ -1221,6 +1382,9 @@ async def build_request_policy("I could not verify the requested credential metadata for this organization. Please provide a valid saved credential by exact name or a credential ID beginning with cred_.",)- with copilot_span("request_policy", data=policy.to_trace_data()):- LOG.info("request-policy decision", **policy.to_trace_data())+ trace_data = policy.to_trace_data()+ if policy.classifier_status == "fallback":+ LOG.warning("request-policy fallback policy used", **trace_data)+ with copilot_span("request_policy", data=trace_data):+ LOG.info("request-policy decision", **trace_data)return policydiff --git a/skyvern/forge/sdk/copilot/tools/completion.py b/skyvern/forge/sdk/copilot/tools/completion.pyindex e3278181b..1892d1ac3 100644--- a/skyvern/forge/sdk/copilot/tools/completion.py+++ b/skyvern/forge/sdk/copilot/tools/completion.py@@ -146,7 +146,7 @@ async def _maybe_run_completion_verification_from_page_observation(if (remaining is not Noneand remaining- <= settings.COPILOT_FEASIBILITY_GATE_TIMEOUT_SECONDS + _COMPLETION_VERIFICATION_BUDGET_MARGIN_SECONDS+ <= settings.COPILOT_COMPLETION_JUDGE_TIMEOUT_SECONDS + _COMPLETION_VERIFICATION_BUDGET_MARGIN_SECONDS):verification = CompletionVerificationResult(status="unavailable")else:@@ -298,7 +298,7 @@ async def _maybe_run_completion_verification(# Too little budget to verify a candidate run: fail closed (unavailable) rather# than let the run-status proxy claim an unverified outcome as success.remaining = RUN_BLOCKS_SAFETY_CEILING_SECONDS - (time.monotonic() - handler_start)- if remaining <= settings.COPILOT_FEASIBILITY_GATE_TIMEOUT_SECONDS + _COMPLETION_VERIFICATION_BUDGET_MARGIN_SECONDS:+ if remaining <= settings.COPILOT_COMPLETION_JUDGE_TIMEOUT_SECONDS + _COMPLETION_VERIFICATION_BUDGET_MARGIN_SECONDS:return CompletionVerificationResult(status="unavailable")snapshot = _build_run_evidence_snapshot(copilot_ctx, result)if not snapshot.has_evidence():diff --git a/skyvern/forge/sdk/copilot/tools/workflow_update.py b/skyvern/forge/sdk/copilot/tools/workflow_update.pyindex fb91fac23..5376ec6ef 100644--- a/skyvern/forge/sdk/copilot/tools/workflow_update.py+++ b/skyvern/forge/sdk/copilot/tools/workflow_update.py@@ -30,6 +30,7 @@ from skyvern.forge.sdk.copilot.config import BlockAuthoringPolicyfrom skyvern.forge.sdk.copilot.context import CopilotContextfrom skyvern.forge.sdk.copilot.enforcement import (POST_INTERMEDIATE_SUCCESS_NUDGE,+ _completion_contract_unknown_due_to_policy_fallback,_goal_likely_needs_more_blocks,)from skyvern.forge.sdk.copilot.loop_detection import clear_failed_step_tracker_for_tools_in_ctx@@ -1684,6 +1685,9 @@ def _pre_run_workflow_coverage_error(copilot_ctx: Any) -> str | None:else:completion_contract = None+ if _completion_contract_unknown_due_to_policy_fallback(copilot_ctx):+ return None+if not _goal_likely_needs_more_blocks(user_message, block_count, completion_contract):return Nonediff --git a/tests/unit/test_copilot_agent_helpers.py b/tests/unit/test_copilot_agent_helpers.pyindex df7247283..b5dd9d589 100644--- a/tests/unit/test_copilot_agent_helpers.py+++ b/tests/unit/test_copilot_agent_helpers.py@@ -122,6 +122,19 @@ class TestFailedTestResponseNormalization:assert "has not been run" in errorassert ctx.coverage_nudge_count == 1+ def test_pre_run_coverage_guard_skips_unknown_completion_contract(self) -> None:+ from skyvern.forge.sdk.copilot.tools import _pre_run_workflow_coverage_error++ ctx = _ctx(+ user_message="Go to https://example.com/contact. Fill out the contact form and submit it.",+ request_policy=SimpleNamespace(completion_contract_status="unknown"),+ last_update_block_count=1,+ coverage_nudge_count=0,+ )++ assert _pre_run_workflow_coverage_error(ctx) is None+ assert ctx.coverage_nudge_count == 0+def test_pre_run_coverage_guard_allows_single_final_action_contract(self) -> None:from skyvern.forge.sdk.copilot.tools import _pre_run_workflow_coverage_error@@ -2088,6 +2101,88 @@ class TestNativeToolSurface:class TestRequestPolicyCredentialResolution:+ @pytest.mark.asyncio+ async def test_request_policy_classifier_timeout_retries_once(self, monkeypatch) -> None:+ from skyvern.config import settings++ monkeypatch.setattr(settings, "COPILOT_REQUEST_POLICY_CLASSIFIER_TIMEOUT_SECONDS", 0.01)+ calls = 0++ async def handler(*, prompt: str, prompt_name: str) -> dict[str, object]:+ nonlocal calls+ calls += 1+ if calls == 1:+ await asyncio.sleep(0.05)+ return {"credential_input_kind": "none", "completion_contract": None}++ policy = await _classify_request(+ user_message="Build a workflow for https://example.com.",+ workflow_yaml="",+ chat_history=[],+ global_llm_context="",+ handler=handler,+ )++ assert calls == 2+ assert policy.classifier_status == "success"+ assert policy.classifier_retry_count == 1+ assert policy.completion_contract_status == "absent"++ @pytest.mark.asyncio+ async def test_request_policy_classifier_malformed_payload_falls_back_without_retry(self) -> None:+ calls = 0++ async def handler(*, prompt: str, prompt_name: str) -> str:+ nonlocal calls+ calls += 1+ return "I can build that workflow."++ policy = await _classify_request(+ user_message="Build a workflow for https://example.com.",+ workflow_yaml="",+ chat_history=[],+ global_llm_context="",+ handler=handler,+ )++ assert calls == 1+ assert policy.classifier_status == "fallback"+ assert policy.classifier_failure_kind == "provider_error"+ assert policy.classifier_retry_count == 0+ assert policy.completion_contract_status == "unknown"++ @pytest.mark.asyncio+ async def test_request_policy_classifier_transient_provider_error_retries_once(self) -> None:+ class RateLimitError(Exception):+ __module__ = "openai"++ calls = 0++ async def handler(*, prompt: str, prompt_name: str) -> dict[str, object]:+ nonlocal calls+ calls += 1+ if calls == 1:+ raise RateLimitError("rate limit")+ return {+ "credential_input_kind": "none",+ "completion_contract": "complete when the account page is visible",+ }++ policy = await _classify_request(+ user_message=(+ "Build a workflow for https://example.com/account. complete when the account page is visible"+ ),+ workflow_yaml="",+ chat_history=[],+ global_llm_context="",+ handler=handler,+ )++ assert calls == 2+ assert policy.classifier_status == "success"+ assert policy.classifier_retry_count == 1+ assert policy.completion_contract_status == "present"+@pytest.mark.asyncioasync def test_missing_user_supplied_credential_ids_ask_for_clarification(self, monkeypatch) -> None:from skyvern.forge.sdk.copilot import request_policy as policy_module@@ -2547,6 +2642,65 @@ workflow_definition:assert policy.allow_missing_credentials_in_draft is Trueassert policy.requires_user_clarification is False+ @pytest.mark.asyncio+ async def test_fallback_policy_resolves_explicit_saved_credential_name(self, monkeypatch) -> None:+ from skyvern.forge.sdk.copilot import request_policy as policy_module+ from skyvern.forge.sdk.copilot.request_policy import build_request_policy++ credential = SimpleNamespace(+ credential_id="cred_portal",+ name="mock-portal-login",+ tested_url="https://portal.example/login",+ )+ credentials = SimpleNamespace(get_credentials=AsyncMock(return_value=[credential]))+ monkeypatch.setattr(policy_module.app, "DATABASE", SimpleNamespace(credentials=credentials))++ async def handler(*, prompt: str, prompt_name: str) -> dict[str, object]:+ raise ValueError("classifier unavailable")++ policy = await build_request_policy(+ user_message='Log into the portal with saved credential named "mock-portal-login".',+ workflow_yaml="",+ chat_history=[],+ global_llm_context="",+ organization_id="org-1",+ handler=handler,+ )++ assert policy.classifier_status == "fallback"+ assert policy.classifier_failure_kind == "provider_error"+ assert policy.completion_contract_status == "unknown"+ assert policy.credential_input_kind == "credential_name"+ assert policy.credential_refs == ["mock-portal-login"]+ assert policy.resolved_credentials == [credential]++ @pytest.mark.asyncio+ async def test_fallback_policy_does_not_substring_match_saved_credential_name(self, monkeypatch) -> None:+ from skyvern.forge.sdk.copilot import request_policy as policy_module+ from skyvern.forge.sdk.copilot.request_policy import build_request_policy++ credential = SimpleNamespace(credential_id="cred_login", name="login", tested_url=None)+ credentials = SimpleNamespace(get_credentials=AsyncMock(return_value=[credential]))+ monkeypatch.setattr(policy_module.app, "DATABASE", SimpleNamespace(credentials=credentials))++ async def handler(*, prompt: str, prompt_name: str) -> dict[str, object]:+ raise ValueError("classifier unavailable")++ policy = await build_request_policy(+ user_message="Log in to the portal using my saved credential.",+ workflow_yaml="",+ chat_history=[],+ global_llm_context="",+ organization_id="org-1",+ handler=handler,+ )++ assert policy.classifier_status == "fallback"+ assert policy.completion_contract_status == "unknown"+ assert policy.credential_input_kind == "none"+ assert policy.credential_refs == []+ credentials.get_credentials.assert_not_called()+@pytest.mark.asyncioasync def test_request_policy_url_matching_and_skip_draft(self, monkeypatch) -> None:from skyvern.forge.sdk.copilot import request_policy as policy_modulediff --git a/tests/unit/test_copilot_completion_verification.py b/tests/unit/test_copilot_completion_verification.pyindex 4a83308ba..7da1e7fef 100644--- a/tests/unit/test_copilot_completion_verification.py+++ b/tests/unit/test_copilot_completion_verification.py@@ -1,5 +1,6 @@from __future__ import annotations+import asyncioimport timefrom collections.abc import Awaitable, Callablefrom types import SimpleNamespace@@ -149,6 +150,21 @@ async def test_evaluate_handler_exception_is_unavailable() -> None:assert result.status == "unavailable"+async def test_evaluate_uses_completion_judge_timeout(monkeypatch: pytest.MonkeyPatch) -> None:+ monkeypatch.setattr(settings, "COPILOT_COMPLETION_JUDGE_TIMEOUT_SECONDS", 0.01)+ monkeypatch.setattr(settings, "COPILOT_FEASIBILITY_GATE_TIMEOUT_SECONDS", 10.0)++ async def handler(**_: object) -> dict[str, object]:+ await asyncio.sleep(0.05)+ return {"verdicts": [{"criterion_id": "c0", "satisfied": True, "reason_code": "evidence_confirms"}]}++ snapshot = RunEvidenceSnapshot(current_url="https://example.com/done")+ result = await evaluate_completion_criteria([_criterion("c0", "done page visible")], snapshot, handler)++ assert result.status == "unavailable"++@pytest.mark.asyncioasync def test_evaluate_happy_path_returns_evaluated() -> None:async def handler(**_: object) -> dict:diff --git a/tests/unit/test_copilot_response_coverage_gate.py b/tests/unit/test_copilot_response_coverage_gate.pyindex fb0cceefe..39757c5e0 100644--- a/tests/unit/test_copilot_response_coverage_gate.py+++ b/tests/unit/test_copilot_response_coverage_gate.py@@ -117,6 +117,15 @@ def test_reply_after_success_with_request_policy_completion_contract_passes_throassert ctx.coverage_nudge_count == 0+def test_reply_after_success_with_unknown_fallback_contract_passes_through() -> None:+ ctx = _post_success_ctx("Go to https://example.com/contact. Fill out the contact form and submit it.")+ ctx.request_policy = SimpleNamespace(completion_contract_status="unknown")+ parsed = {"type": "REPLY", "user_response": "I created and tested the workflow."}++ assert _response_coverage_nudge(ctx, parsed) is None+ assert ctx.coverage_nudge_count == 0++def test_coverage_nudge_respects_counter_cap() -> None:ctx = _post_success_ctx("go to X and download Y")parsed = {"type": "REPLY", "user_response": "one block draft"}