Skip to content

Automate browser based workflows with AI

fix(copilot): type ResponseScaffoldingNormalization.response_type as ResponseType + sync OSS mypy overrides (#6041)

Shuchang Zheng52293136f4ee
diff --git a/skyvern/forge/prompts/skyvern/workflow-copilot-request-policy.j2 b/skyvern/forge/prompts/skyvern/workflow-copilot-request-policy.j2
index ec3c55b44..9ae4c4226 100644
--- a/skyvern/forge/prompts/skyvern/workflow-copilot-request-policy.j2
+++ b/skyvern/forge/prompts/skyvern/workflow-copilot-request-policy.j2
@@ -8,7 +8,7 @@ testing_intent:
- Do not use skip_test for a "done when", "complete when", or success-condition sentence; that is a completion_contract.
credential_input_kind:
-- raw_secret: the user pasted a literal credential value — a password string, an API key/bearer token value, a session/auth cookie value, a TOTP/OTP/MFA digit code, a private-key body, a card number, or a CVV. The value must look like the secret itself (e.g. `password: hunter2`, `eyJ…` JWT, `sk-…` API key, a numeric OTP). Identifier-shaped vault-pointer strings such as `customer-<uuid>-pass`, `prod-app-secret-id`, `<service>-key-vault-id`, or `https://<vault>.vault.azure.net/secrets/<name>` are POINTERS to where a secret is stored, not secret values; do NOT classify them as raw_secret on the basis of a `-pass`, `-secret`, `-key`, or `-token` suffix alone.
+- raw_secret: the user pasted a literal credential value — a password string, an API key/bearer token value, a session/auth cookie value, a TOTP/OTP/MFA digit code, a private-key body, a card number, or a CVV. The value must look like the secret itself (e.g. `password: hunter2`, `[email protected]:<password>`, bulk `[email protected]:<password>` account rows, `eyJ…` JWT, `sk-…` API key, a numeric OTP). Identifier-shaped vault-pointer strings such as `customer-<uuid>-pass`, `prod-app-secret-id`, `<service>-key-vault-id`, or `https://<vault>.vault.azure.net/secrets/<name>` are POINTERS to where a secret is stored, not secret values; do NOT classify them as raw_secret on the basis of a `-pass`, `-secret`, `-key`, or `-token` suffix alone.
- credential_id: the user provided one or more exact credential IDs beginning with `cred_`. Put every ID in credential_refs.
- credential_name: the user gave an exact saved credential name, or vaguely says to use "my credentials", "my saved credentials", "credentials saved already", or an entry in the Credentials UI without enough website context. Treat quoted names and bare identifier-like names tied to credential verbs as exact names, for example `using azure_credentials`, `get the code from mfa_email`, or `use Login Prod`. Put exact names in credential_refs; leave credential_refs empty when the reference is vague. Vault-handle-shaped strings (UUID/hex/dashed handles ending in `-pass`, `-secret`, `-key`, `-token`, `-id`, or `-user`) are NOT saved credential names — they are vault pointers; do not classify them as credential_name unless the user explicitly says "use my saved credential named `<handle>`".
- website_stored_credential: the user asks to use stored/saved credentials for a specific website or login page URL. Put URLs in login_page_urls.
diff --git a/skyvern/forge/sdk/copilot/agent.py b/skyvern/forge/sdk/copilot/agent.py
index f557bc5cf..9a6180f87 100644
--- a/skyvern/forge/sdk/copilot/agent.py
+++ b/skyvern/forge/sdk/copilot/agent.py
@@ -43,6 +43,7 @@ from skyvern.forge.sdk.copilot.output_policy import (
derive_output_kind,
evaluate_output_policy,
hard_block_output_policy_verdict,
+ normalize_response_scaffolding,
output_policy_verdict_from_trace_data,
output_policy_verdict_to_trace_data,
)
@@ -583,6 +584,11 @@ _UNBACKED_WORKFLOW_DELIVERY_REPLY = (
"I wasn't able to produce a workflow proposal in this turn. Please try again, or provide the missing details "
"so I can build and test it."
)
+_INTERNAL_BLOCK_TAXONOMY_REPLY = (
+ "Internal workflow names are not the right interface to use when building with Copilot. "
+ "Describe the page action, data to collect, sign-in step, or check you want, and I'll translate that into "
+ "a supported workflow update."
+)
_PROPOSAL_ACCEPT_UI_ACTION_RE = re.compile(r"\b(?:accept|always\s+accept)\b", re.IGNORECASE)
_PROPOSAL_REJECT_UI_ACTION_RE = re.compile(r"\b(?:reject|discard)\b", re.IGNORECASE)
_UNVALIDATED_PROPOSAL_AFFORDANCE = (
@@ -723,6 +729,9 @@ def _translate_to_agent_result(
resp_type = action_data.get("type", "REPLY")
if resp_type not in COPILOT_RESPONSE_TYPES:
resp_type = "REPLY"
+ normalized_scaffolding = normalize_response_scaffolding(resp_type, str(user_response))
+ resp_type = normalized_scaffolding.response_type
+ user_response = normalized_scaffolding.user_response or "Done."
last_workflow = ctx.last_workflow
last_workflow_yaml = ctx.last_workflow_yaml
@@ -896,21 +905,33 @@ def _translate_to_agent_result(
unvalidated=unvalidated,
output_kind=output_kind,
)
+ soft_rewrite_reasons: list[OutputPolicyReason] = []
+ if OutputPolicyReason.INTERNAL_BLOCK_TAXONOMY_LEAK in output_policy_verdict.reason_codes:
+ user_response = _INTERNAL_BLOCK_TAXONOMY_REPLY
+ soft_rewrite_reasons.append(OutputPolicyReason.INTERNAL_BLOCK_TAXONOMY_LEAK)
+ output_policy_verdict.remove(OutputPolicyReason.INTERNAL_BLOCK_TAXONOMY_LEAK)
+ # Preserve the unbacked-proposal correction when both soft rewrites apply:
+ # a reply must not imply a workflow exists when no proposal was produced.
if OutputPolicyReason.UNBACKED_WORKFLOW_DELIVERY_CLAIM in output_policy_verdict.reason_codes:
user_response = _UNBACKED_WORKFLOW_DELIVERY_REPLY
+ soft_rewrite_reasons.append(OutputPolicyReason.UNBACKED_WORKFLOW_DELIVERY_CLAIM)
output_policy_verdict.remove(OutputPolicyReason.UNBACKED_WORKFLOW_DELIVERY_CLAIM)
if OutputPolicyReason.MISSING_PROPOSAL_STATE in output_policy_verdict.reason_codes:
+ soft_rewrite_reasons.append(OutputPolicyReason.MISSING_PROPOSAL_STATE)
output_policy_verdict.remove(OutputPolicyReason.MISSING_PROPOSAL_STATE)
if OutputPolicyReason.MISSING_UNVALIDATED_PROPOSAL_AFFORDANCE in output_policy_verdict.reason_codes:
user_response = _ensure_unvalidated_proposal_affordance(str(user_response))
+ soft_rewrite_reasons.append(OutputPolicyReason.MISSING_UNVALIDATED_PROPOSAL_AFFORDANCE)
output_policy_verdict.remove(OutputPolicyReason.MISSING_UNVALIDATED_PROPOSAL_AFFORDANCE)
+ trace_data = output_policy_verdict_to_trace_data(
+ output_policy_verdict,
+ surface="final_translation",
+ response_type=resp_type,
+ )
+ trace_data["soft_rewrite_reason_codes"] = [reason.value for reason in soft_rewrite_reasons]
LOG.info(
"copilot output policy final verdict",
- **output_policy_verdict_to_trace_data(
- output_policy_verdict,
- surface="final_translation",
- response_type=resp_type,
- ),
+ **trace_data,
)
if not output_policy_verdict.allowed:
return _build_output_policy_blocked_result(
@@ -1078,6 +1099,10 @@ def _evaluate_copilot_final_output_policy(
response_type = action_data.get("type", "REPLY")
if response_type not in COPILOT_RESPONSE_TYPES:
response_type = "REPLY"
+ policy_user_response = str(action_data.get("user_response") or text)
+ normalized_scaffolding = normalize_response_scaffolding(response_type, policy_user_response)
+ response_type = normalized_scaffolding.response_type
+ policy_user_response = normalized_scaffolding.user_response or "Done."
workflow_yaml = None
if response_type == "REPLACE_WORKFLOW" and isinstance(action_data.get("workflow_yaml"), str):
@@ -1088,7 +1113,6 @@ def _evaluate_copilot_final_output_policy(
workflow_attempted = ctx.last_update_block_count is not None or ctx.last_test_ok is not None
surface_untested_draft = _should_surface_untested_draft_despite_question(ctx, response_type)
policy_response_type = "REPLY" if surface_untested_draft else response_type
- policy_user_response = str(action_data.get("user_response") or text)
if surface_untested_draft:
policy_user_response = _rewrite_failed_test_response(policy_user_response, ctx)
updated_workflow_for_kind = (
@@ -1230,6 +1254,18 @@ def _build_output_policy_blocked_result(
"Store credentials in the Skyvern Credentials UI and reply with the saved credential name or a "
"credential ID beginning with cred_. DO NOT PROVIDE RAW LOGIN/PASSWORD."
)
+ elif OutputPolicyReason.UNAPPROVED_CREDENTIAL_REFERENCE in verdict.reason_codes:
+ user_response = (
+ "I need you to confirm which saved credential should be used before I can continue. "
+ "Please reply with the credential name from the Credentials UI, or adjust the workflow to avoid "
+ "using credentials."
+ )
+ elif OutputPolicyReason.CREDENTIAL_SCOPE_BROADENED in verdict.reason_codes:
+ user_response = (
+ "The selected credential is not approved for one of the URLs in this workflow. "
+ "Please use a saved credential tested for that URL, update the block URL to match the credential's "
+ "tested site, or adjust the workflow to avoid using credentials."
+ )
else:
user_response = "I could not safely return that Copilot output. Please adjust the request and try again."
return AgentResult(
diff --git a/skyvern/forge/sdk/copilot/context.py b/skyvern/forge/sdk/copilot/context.py
index 9a4324e6b..5b5661d53 100644
--- a/skyvern/forge/sdk/copilot/context.py
+++ b/skyvern/forge/sdk/copilot/context.py
@@ -12,7 +12,7 @@ from skyvern.forge.sdk.copilot.runtime import AgentContext
from skyvern.forge.sdk.workflow.models.workflow import Workflow
ResponseType = Literal["REPLY", "ASK_QUESTION", "REPLACE_WORKFLOW"]
-COPILOT_RESPONSE_TYPES: tuple[str, ...] = get_args(ResponseType)
+COPILOT_RESPONSE_TYPES: tuple[ResponseType, ...] = get_args(ResponseType)
if TYPE_CHECKING:
from skyvern.forge.sdk.copilot.narration import NarratorState
diff --git a/skyvern/forge/sdk/copilot/enforcement.py b/skyvern/forge/sdk/copilot/enforcement.py
index fbee0e8a8..39713fa83 100644
--- a/skyvern/forge/sdk/copilot/enforcement.py
+++ b/skyvern/forge/sdk/copilot/enforcement.py
@@ -36,6 +36,7 @@ from skyvern.forge.sdk.copilot.config import (
)
from skyvern.forge.sdk.copilot.failure_tracking import PER_TOOL_BUDGET_FAILURE_CATEGORY, normalize_failure_reason
from skyvern.forge.sdk.copilot.narration import TransitionKind
+from skyvern.forge.sdk.copilot.output_policy import normalize_response_scaffolding
from skyvern.forge.sdk.copilot.output_utils import (
extract_final_text,
looks_like_workflow_delivery_claim,
@@ -575,6 +576,16 @@ def _check_enforcement(
# Only runs when no state-based nudge fired.
if result is not None:
parsed = parse_final_response(extract_final_text(result))
+ normalized_scaffolding = normalize_response_scaffolding(
+ str(parsed.get("type") or "REPLY"),
+ str(parsed.get("user_response") or ""),
+ )
+ if normalized_scaffolding.changed:
+ parsed = {
+ **parsed,
+ "type": normalized_scaffolding.response_type,
+ "user_response": normalized_scaffolding.user_response or "Done.",
+ }
return _response_coverage_nudge(ctx, parsed, config)
return None
diff --git a/skyvern/forge/sdk/copilot/output_policy.py b/skyvern/forge/sdk/copilot/output_policy.py
index 1c354cff6..e9807e659 100644
--- a/skyvern/forge/sdk/copilot/output_policy.py
+++ b/skyvern/forge/sdk/copilot/output_policy.py
@@ -1,14 +1,14 @@
from __future__ import annotations
-import json
import re
from dataclasses import dataclass, field
from enum import StrEnum
-from typing import Any
+from typing import Any, cast
from urllib.parse import urlparse
+from skyvern.forge.sdk.copilot.context import COPILOT_RESPONSE_TYPES, ResponseType
from skyvern.forge.sdk.copilot.output_utils import looks_like_workflow_delivery_claim
-from skyvern.forge.sdk.copilot.request_policy import RAW_SECRET_PATTERNS, RequestPolicy
+from skyvern.forge.sdk.copilot.request_policy import RAW_SECRET_PATTERNS, RequestPolicy, contains_email_password_pair
from skyvern.utils.yaml_loader import safe_load_no_dates
WORKFLOW_PRESENT_SENTINEL = object()
@@ -27,6 +27,58 @@ UNVALIDATED_DISCLOSURE_PHRASES = (
"hasn't been verified",
"unvalidated",
)
+_INTERNAL_TOOL_INSTRUCTION_MARKERS = (
+ "call get_run_results",
+ "call update_and_run_blocks",
+)
+_INTERNAL_TOOL_RETRY_PHRASES = ("do not retry", "do not re-invoke")
+_INTERNAL_TOOL_RETRY_CONTEXT_MARKERS = (
+ "get_run_results",
+ "update_and_run_blocks",
+ "tool call",
+ "this tool",
+ "the tool",
+ "workflow_run_id",
+)
+_INTERNAL_TOOL_INSTRUCTION_TRANSLATION = str.maketrans({char: " " for char in "`'\"()[]{}.,:;"})
+_INTERNAL_BLOCK_TYPE_TERMS = frozenset(
+ {
+ "navigation",
+ "extraction",
+ "validation",
+ "login",
+ "goto_url",
+ "file_download",
+ "file_upload",
+ "text_prompt",
+ "for_loop",
+ "conditional",
+ "action",
+ "wait",
+ }
+)
+_INTERNAL_BLOCK_TYPE_CONTEXT_MARKERS = (
+ "block type",
+ "block types",
+ "internal block",
+ "internal blocks",
+ "workflow block",
+ "workflow blocks",
+ "supported block",
+ "supported blocks",
+)
+# Three distinct internal names in an informational reply implies taxonomy
+# enumeration, while one or two may be incidental product-language prose.
+_INFORMATIONAL_TAXONOMY_TERM_THRESHOLD = 3
+_IDENTIFIER_TOKEN_RE = re.compile(r"[A-Za-z][A-Za-z0-9_]*")
+_IDENTIFIER_DELIMITERS = frozenset({"`", '"', "'"})
+
+
+@dataclass(frozen=True)
+class ResponseScaffoldingNormalization:
+ response_type: ResponseType
+ user_response: str | None
+ changed: bool = False
class CopilotOutputKind(StrEnum):
@@ -47,7 +99,9 @@ class OutputPolicyReason(StrEnum):
MISSING_UNVALIDATED_PROPOSAL_AFFORDANCE = "missing_unvalidated_proposal_affordance"
MISSING_PROPOSAL_STATE = "missing_proposal_state"
PERSISTENCE_STATE_MISMATCH = "persistence_state_mismatch"
+ INTERNAL_TOOL_INSTRUCTION_LEAK = "internal_tool_instruction_leak"
OUTPUT_POLICY_CONTEXT_MISSING = "output_policy_context_missing"
+ INTERNAL_BLOCK_TAXONOMY_LEAK = "internal_block_taxonomy_leak"
@dataclass
@@ -78,6 +132,7 @@ _FINAL_OUTPUT_HARD_BLOCK_REASONS: frozenset[OutputPolicyReason] = frozenset(
OutputPolicyReason.UNAPPROVED_CREDENTIAL_REFERENCE,
OutputPolicyReason.CREDENTIAL_SCOPE_BROADENED,
OutputPolicyReason.PERSISTENCE_STATE_MISMATCH,
+ OutputPolicyReason.INTERNAL_TOOL_INSTRUCTION_LEAK,
OutputPolicyReason.OUTPUT_POLICY_CONTEXT_MISSING,
}
)
@@ -118,6 +173,46 @@ def derive_output_kind(
return CopilotOutputKind.INFORMATIONAL_ANSWER
+def normalize_response_scaffolding(response_type: str, user_response: str | None) -> ResponseScaffoldingNormalization:
+ typed_response_type: ResponseType = (
+ cast(ResponseType, response_type) if response_type in COPILOT_RESPONSE_TYPES else "REPLY"
+ )
+ label, stripped = _split_leading_response_label(user_response)
+ if label is None:
+ return ResponseScaffoldingNormalization(response_type=typed_response_type, user_response=user_response)
+ normalized_type: ResponseType
+ if label == "REPLACE_WORKFLOW":
+ normalized_type = "REPLACE_WORKFLOW" if typed_response_type == "REPLACE_WORKFLOW" else "REPLY"
+ else:
+ normalized_type = label
+ return ResponseScaffoldingNormalization(response_type=normalized_type, user_response=stripped, changed=True)
+
+
+def _split_leading_response_label(text: str | None) -> tuple[ResponseType | None, str | None]:
+ if not isinstance(text, str):
+ return None, text
+ candidate = text.lstrip()
+ candidate_upper = candidate.upper()
+ for response_type in sorted(COPILOT_RESPONSE_TYPES, key=len, reverse=True):
+ if not candidate_upper.startswith(response_type):
+ continue
+ remainder = candidate[len(response_type) :]
+ if not remainder:
+ continue
+ stripped = remainder.lstrip()
+ if not stripped:
+ return response_type, ""
+ if stripped[0] in {":", ","}:
+ return response_type, stripped[1:].lstrip()
+ protocol_like_label = "_" in response_type or candidate[: len(response_type)].isupper()
+ leading_whitespace = remainder[: len(remainder) - len(stripped)]
+ if "\n" in leading_whitespace and not stripped.startswith(("{", "```")):
+ return response_type, stripped
+ if remainder[0].isspace() and protocol_like_label and not stripped.startswith(("{", "```")):
+ return response_type, stripped
+ return None, text
+
+
def output_policy_verdict_to_trace_data(
verdict: OutputPolicyVerdict,
*,
@@ -191,6 +286,8 @@ def evaluate_output_policy(
values = [user_response, workflow_yaml, tool_arguments]
if any(_contains_raw_secret(value) for value in values):
verdict.add(OutputPolicyReason.RAW_SECRET_LEAK)
+ if _contains_internal_tool_instruction(user_response):
+ verdict.add(OutputPolicyReason.INTERNAL_TOOL_INSTRUCTION_LEAK)
if (
response_type == "REPLY"
and not has_workflow_proposal
@@ -206,6 +303,8 @@ def evaluate_output_policy(
and not _has_unvalidated_affordance(user_response)
):
verdict.add(OutputPolicyReason.MISSING_UNVALIDATED_PROPOSAL_AFFORDANCE)
+ if _contains_internal_block_taxonomy_leak(user_response, output_kind):
+ verdict.add(OutputPolicyReason.INTERNAL_BLOCK_TAXONOMY_LEAK)
if isinstance(request_policy, RequestPolicy):
if request_policy.user_response_policy == "ask_clarification" and response_type != "ASK_QUESTION":
@@ -226,23 +325,44 @@ def format_output_policy_tool_error(verdict: OutputPolicyVerdict) -> str:
def _contains_raw_secret(value: Any) -> bool:
- if value is None:
- return False
- text = _policy_text(value)
- for pattern in RAW_SECRET_PATTERNS:
- for match in pattern.finditer(text):
- if not any(marker in match.group(0) for marker in _PLACEHOLDER_MARKERS):
- return True
+ for text in _policy_text_values(value):
+ if contains_email_password_pair(text):
+ return True
+ for pattern in RAW_SECRET_PATTERNS:
+ for match in pattern.finditer(text):
+ if not any(marker in match.group(0) for marker in _PLACEHOLDER_MARKERS):
+ return True
return False
-def _policy_text(value: Any) -> str:
+def _contains_internal_tool_instruction(user_response: str | None) -> bool:
+ if not isinstance(user_response, str):
+ return False
+ normalized = " ".join(user_response.lower().translate(_INTERNAL_TOOL_INSTRUCTION_TRANSLATION).split())
+ if any(marker in normalized for marker in _INTERNAL_TOOL_INSTRUCTION_MARKERS):
+ return True
+ return any(phrase in normalized for phrase in _INTERNAL_TOOL_RETRY_PHRASES) and any(
+ marker in normalized for marker in _INTERNAL_TOOL_RETRY_CONTEXT_MARKERS
+ )
+
+
+def _policy_text_values(value: Any) -> list[str]:
+ if value is None:
+ return []
if isinstance(value, str):
- return value
- try:
- return json.dumps(value, default=str, sort_keys=True)
- except TypeError:
- return str(value)
+ return [value]
+ if isinstance(value, dict):
+ values: list[str] = []
+ for key, item in value.items():
+ values.extend(_policy_text_values(key))
+ values.extend(_policy_text_values(item))
+ return values
+ if isinstance(value, (list, tuple, set)):
+ values = []
+ for item in value:
+ values.extend(_policy_text_values(item))
+ return values
+ return [str(value)]
def _has_unvalidated_affordance(user_response: str | None) -> bool:
@@ -253,6 +373,63 @@ def _has_unvalidated_affordance(user_response: str | None) -> bool:
return bool(_UNVALIDATED_PROPOSAL_AFFORDANCE_RE.search(user_response) and has_disclosure)
+def _contains_internal_block_taxonomy_leak(
+ user_response: str | None,
+ output_kind: CopilotOutputKind,
+) -> bool:
+ if not user_response:
+ return False
+ if _contains_deprecated_block_identifier(user_response):
+ return True
+ if output_kind != CopilotOutputKind.INFORMATIONAL_ANSWER:
+ return False
+ taxonomy_terms = _internal_block_taxonomy_terms(user_response)
+ return len(taxonomy_terms) >= _INFORMATIONAL_TAXONOMY_TERM_THRESHOLD
+
+
+def _contains_deprecated_block_identifier(text: str) -> bool:
+ tokens = [_compact_identifier_token(match.group(0)) for match in _IDENTIFIER_TOKEN_RE.finditer(text)]
+ if "taskv2" in tokens:
+ return True
+ return any(left == "task" and right == "v2" for left, right in zip(tokens, tokens[1:]))
+
+
+def _internal_block_taxonomy_terms(text: str) -> set[str]:
+ lower = text.lower()
+ has_taxonomy_context = any(marker in lower for marker in _INTERNAL_BLOCK_TYPE_CONTEXT_MARKERS)
+ matches = list(_IDENTIFIER_TOKEN_RE.finditer(text))
+ terms: set[str] = set()
+ for index, match in enumerate(matches):
+ term = _normalized_internal_block_term(match.group(0))
+ if term is None:
+ continue
+ if has_taxonomy_context or _is_delimited_identifier(text, match.start(), match.end()):
+ terms.add(term)
+ continue
+ next_token = _compact_identifier_token(matches[index + 1].group(0)) if index + 1 < len(matches) else None
+ if next_token in {"block", "blocks", "for"}:
+ terms.add(term)
+ return terms
+
+
+def _normalized_internal_block_term(raw: str) -> str | None:
+ term = raw.lower()
+ if term in _INTERNAL_BLOCK_TYPE_TERMS:
+ return term
+ return None
+
+
+def _compact_identifier_token(raw: str) -> str:
+ return raw.lower().replace("_", "")
+
+
+def _is_delimited_identifier(text: str, start: int, end: int) -> bool:
+ if start == 0 or end >= len(text):
+ return False
+ left = text[start - 1]
+ return left == text[end] and left in _IDENTIFIER_DELIMITERS
+
+
def _apply_credential_policy(
verdict: OutputPolicyVerdict,
request_policy: RequestPolicy,
@@ -294,7 +471,10 @@ def _allowed_unresolved_credential_ids(request_policy: RequestPolicy) -> set[str
def _credential_ids(value: Any) -> set[str]:
if value is None:
return set()
- return set(_CREDENTIAL_ID_RE.findall(_policy_text(value)))
+ found: set[str] = set()
+ for text in _policy_text_values(value):
+ found.update(_CREDENTIAL_ID_RE.findall(text))
+ return found
def _workflow_broadens_credential_scope(workflow_yaml: str, request_policy: RequestPolicy) -> bool:
diff --git a/skyvern/forge/sdk/copilot/output_utils.py b/skyvern/forge/sdk/copilot/output_utils.py
index 2eb68b933..f1acf112d 100644
--- a/skyvern/forge/sdk/copilot/output_utils.py
+++ b/skyvern/forge/sdk/copilot/output_utils.py
@@ -46,7 +46,6 @@ def extract_final_text(result: RunResultStreaming) -> str:
_TYPE_ALTERNATION = "|".join(COPILOT_RESPONSE_TYPES)
-_LEADING_LABEL_RE = re.compile(rf"^\s*({_TYPE_ALTERNATION})\s*[:,]?\s+", re.IGNORECASE)
_USER_RESPONSE_VALUE_RE = re.compile(r'"user_response"\s*:\s*"((?:[^"\\]|\\.)*)"')
_TYPE_VALUE_RE = re.compile(rf'"type"\s*:\s*"({_TYPE_ALTERNATION})"')
_WORKFLOW_DELIVERY_CLAIM_PATTERNS = [
@@ -67,6 +66,38 @@ def _try_loads_dict(text: str) -> dict[str, Any] | None:
return parsed if isinstance(parsed, dict) else None
+def _strip_markdown_code_fence(text: str) -> str:
+ cleaned = text.strip()
+ for prefix in ("```json", "```"):
+ if cleaned.startswith(prefix):
+ cleaned = cleaned[len(prefix) :]
+ break
+ if cleaned.endswith("```"):
+ cleaned = cleaned[:-3]
+ return cleaned.strip()
+
+
+def _strip_structured_response_label(text: str) -> str | None:
+ text_upper = text.upper()
+ for response_type in sorted(COPILOT_RESPONSE_TYPES, key=len, reverse=True):
+ if not text_upper.startswith(response_type):
+ continue
+ remainder = text[len(response_type) :]
+ if not remainder:
+ continue
+ stripped = remainder.lstrip()
+ if not stripped:
+ continue
+ if stripped[0] in {":", ","}:
+ stripped = stripped[1:].lstrip()
+ elif not remainder[0].isspace():
+ continue
+ candidate = _strip_markdown_code_fence(stripped)
+ if candidate.startswith("{"):
+ return candidate
+ return None
+
+
def _looks_like_envelope(parsed: dict[str, Any]) -> bool:
if "user_response" in parsed:
return True
@@ -97,24 +128,18 @@ def parse_final_response(text: str) -> dict[str, Any]:
control characters in string values. Falls back to regex-extracting
``user_response`` from envelope-shaped text so a malformed envelope never
reaches the chat bubble."""
- cleaned = text.strip()
- for prefix in ("```json", "```"):
- if cleaned.startswith(prefix):
- cleaned = cleaned[len(prefix) :]
- break
- if cleaned.endswith("```"):
- cleaned = cleaned[:-3]
- cleaned = cleaned.strip()
+ cleaned = _strip_markdown_code_fence(text)
parsed = _try_loads_dict(cleaned)
if parsed is not None:
return parsed
- label_stripped = _LEADING_LABEL_RE.sub("", cleaned, count=1)
- if label_stripped != cleaned:
+ label_stripped = _strip_structured_response_label(cleaned)
+ if label_stripped is not None:
parsed = _try_loads_dict(label_stripped)
if parsed is not None:
return parsed
+ cleaned = label_stripped
first = cleaned.find("{")
last = cleaned.rfind("}")
diff --git a/skyvern/forge/sdk/copilot/request_policy.py b/skyvern/forge/sdk/copilot/request_policy.py
index b86e0437a..51225a554 100644
--- a/skyvern/forge/sdk/copilot/request_policy.py
+++ b/skyvern/forge/sdk/copilot/request_policy.py
@@ -7,6 +7,7 @@ from typing import Any, Literal, cast, get_args
from urllib.parse import urlparse
import structlog
+from email_validator import EmailNotValidError, validate_email
from skyvern.config import settings
from skyvern.forge import app
@@ -88,6 +89,8 @@ _RAW_SECRET_PATTERNS = (
)
# Reused by output-policy guardrails as syntactic leak backstops.
RAW_SECRET_PATTERNS = _RAW_SECRET_PATTERNS
+_COLON_DELIMITED_SECRET_SEGMENT_SEPARATORS = (",", ";", "|")
+_COLON_DELIMITED_SECRET_EDGE_CHARS = "\"'`()[]{}<>"
@dataclass
@@ -273,7 +276,57 @@ def _credential_ids(text: str) -> list[str]:
def _raw_secret_detected(text: str) -> bool:
- return any(pattern.search(text or "") for pattern in _RAW_SECRET_PATTERNS)
+ return any(pattern.search(text or "") for pattern in _RAW_SECRET_PATTERNS) or contains_email_password_pair(text)
+
+
+def _candidate_secret_segments(text: str) -> list[str]:
+ segments: list[str] = []
+ for raw_token in (text or "").split():
+ token_segments = [raw_token]
+ for separator in _COLON_DELIMITED_SECRET_SEGMENT_SEPARATORS:
+ token_segments = [part for segment in token_segments for part in segment.split(separator)]
+ segments.extend(segment.strip(_COLON_DELIMITED_SECRET_EDGE_CHARS) for segment in token_segments)
+ return [segment for segment in segments if segment]
+
+
+def _is_valid_account_row_email(value: str) -> bool:
+ if any(char.isspace() for char in value) or "/" in value or ":" in value:
+ return False
+ try:
+ validate_email(value, check_deliverability=False, test_environment=True)
+ except EmailNotValidError:
+ return False
+ return True
+
+
+def _looks_like_colon_delimited_secret_value(value: str) -> bool:
+ if len(value) < 4:
+ return False
+ if any(char.isspace() for char in value):
+ return False
+ if any(char in value for char in ("/", "?", "#")):
+ return False
+ if value.isdigit() and len(value) <= 5:
+ return False
+ return True
+
+
+def _email_password_pair_segments(text: str) -> list[str]:
+ pairs: list[str] = []
+ for segment in _candidate_secret_segments(text):
+ email, separator, secret_value = segment.partition(":")
+ if not separator:
+ continue
+ if _is_valid_account_row_email(email) and _looks_like_colon_delimited_secret_value(secret_value):
+ pairs.append(segment)
+ return pairs
+
+
+def contains_email_password_pair(text: str) -> bool:
+ # Privacy backstop for pasted account dumps. The request-policy classifier
+ # owns ambiguous credential semantics; this parser keeps high-confidence raw
+ # values out of model prompts and output surfaces without a broad regex rule.
+ return bool(_email_password_pair_segments(text))
def _coerce_clarification_reason(value: Any) -> ClarificationReason:
@@ -286,6 +339,8 @@ def redact_raw_secrets_for_prompt(text: str) -> str:
redacted = text or ""
for pattern in _RAW_SECRET_PATTERNS:
redacted = pattern.sub("[REDACTED_SECRET]", redacted)
+ for segment in _email_password_pair_segments(redacted):
+ redacted = redacted.replace(segment, "[REDACTED_SECRET]")
return redacted
diff --git a/skyvern/forge/sdk/copilot/tools.py b/skyvern/forge/sdk/copilot/tools.py
index a7ca7389f..28dc8e0f2 100644
--- a/skyvern/forge/sdk/copilot/tools.py
+++ b/skyvern/forge/sdk/copilot/tools.py
@@ -1810,6 +1810,24 @@ async def _watchdog_error_message(
return message
+def _watchdog_user_failure_reason(
+ exit_reason: WatchdogExitReason,
+ workflow_run_id: str,
+ budget_seconds: int,
+ run: WorkflowRun | None,
+) -> str:
+ if exit_reason == "stagnation":
+ body = f"The run stopped after no observable progress for {RUN_BLOCKS_STAGNATION_WINDOW_SECONDS}s."
+ elif exit_reason == "per_tool_budget":
+ body = f"The run exceeded the {budget_seconds}s per-tool-call budget while still making progress."
+ elif exit_reason == "ceiling":
+ body = f"The run exceeded the {budget_seconds}s absolute ceiling while still showing progress."
+ else:
+ status = f" Last observed status: {run.status}." if run is not None else ""
+ body = "The run ended before recording a trustworthy terminal status." + status
+ return f"{body} Run ID: {workflow_run_id}. Outcome is uncertain."
+
+
async def _run_blocks_and_collect_debug(
params: dict[str, Any],
ctx: CopilotContext,
@@ -2092,26 +2110,32 @@ async def _run_blocks_and_collect_debug(
error_msg = await _watchdog_error_message(
exit_reason, ctx, workflow_run.workflow_run_id, run, budget_seconds
)
- result: dict[str, Any] = {"ok": False, "error": error_msg}
+ user_failure_reason = _watchdog_user_failure_reason(
+ exit_reason, workflow_run.workflow_run_id, budget_seconds, run
+ )
+ result: dict[str, Any] = {
+ "ok": False,
+ "error": error_msg,
+ "data": {
+ "workflow_run_id": workflow_run.workflow_run_id,
+ "overall_status": run.status if run is not None else None,
+ "failure_reason": user_failure_reason,
+ },
+ }
if exit_reason == "per_tool_budget":
# Stable failure_categories entry so consecutive budget trips
# hash to the same streak signature; without it the run_id in
# ``error_msg`` would make every trip unique.
- result["data"] = {
- "workflow_run_id": workflow_run.workflow_run_id,
- "overall_status": run.status if run is not None else None,
- "failure_reason": error_msg,
- "failure_categories": [
- {
- "category": PER_TOOL_BUDGET_FAILURE_CATEGORY,
- "confidence_float": 1.0,
- "reasoning": (
- f"Per-tool-call budget of {budget_seconds}s exceeded; "
- "the run was making progress but cannot fit in a single tool call."
- ),
- }
- ],
- }
+ result["data"]["failure_categories"] = [
+ {
+ "category": PER_TOOL_BUDGET_FAILURE_CATEGORY,
+ "confidence_float": 1.0,
+ "reasoning": (
+ f"Per-tool-call budget of {budget_seconds}s exceeded; "
+ "the run was making progress but cannot fit in a single tool call."
+ ),
+ }
+ ]
if run_cancelled_by_watchdog:
result[_INTERNAL_RUN_CANCELLED_BY_WATCHDOG_KEY] = True
return result
@@ -3685,6 +3709,8 @@ def _record_run_blocks_result(copilot_ctx: Any, result: dict[str, Any]) -> None:
if isinstance(block, dict) and block.get("failure_reason"):
copilot_ctx.last_test_failure_reason = str(block["failure_reason"])
break
+ if copilot_ctx.last_test_failure_reason is None:
+ copilot_ctx.last_test_failure_reason = next(iter_failure_reasons(result), None)
if result.get("error") and copilot_ctx.last_test_failure_reason is None:
copilot_ctx.last_test_failure_reason = str(result["error"])
update_repeated_failure_state(copilot_ctx, result)
diff --git a/skyvern/webeye/utils/page.py b/skyvern/webeye/utils/page.py
index c43aae885..a99abf086 100644
--- a/skyvern/webeye/utils/page.py
+++ b/skyvern/webeye/utils/page.py
@@ -789,12 +789,9 @@ class SkyvernFrame:
This is designed for cached action execution to ensure the page is ready
before attempting to interact with elements.
"""
- total_start_time = time.time()
_tracer = otel_trace.get_tracer("skyvern")
# 1. Wait for loading indicators to disappear (longest timeout first)
- loading_indicator_duration_ms = 0.0
- step_start_time = time.time()
loading_indicator_result = "success"
with _tracer.start_as_current_span("skyvern.browser.page_ready.loading_indicators") as _li_span:
apply_context_attrs(_li_span)
@@ -808,19 +805,9 @@ class SkyvernFrame:
loading_indicator_result = "error"
LOG.warning("Failed to check loading indicators, proceeding", exc_info=True)
finally:
- loading_indicator_duration_ms = (time.time() - step_start_time) * 1000
_li_span.set_attribute("result", loading_indicator_result)
- LOG.info(
- "page_readiness_check",
- step="loading_indicators",
- result=loading_indicator_result,
- duration_ms=loading_indicator_duration_ms,
- timeout_ms=loading_indicator_timeout_ms,
- )
# 2. Wait for network idle (with short timeout - some pages never go idle)
- network_idle_duration_ms = 0.0
- step_start_time = time.time()
network_idle_result = "success"
with _tracer.start_as_current_span("skyvern.browser.page_ready.network_idle") as _ni_span:
apply_context_attrs(_ni_span)
@@ -831,19 +818,9 @@ class SkyvernFrame:
network_idle_result = "timeout"
LOG.warning("Network idle timeout - page may have constant activity, proceeding")
finally:
- network_idle_duration_ms = (time.time() - step_start_time) * 1000
_ni_span.set_attribute("result", network_idle_result)
- LOG.info(
- "page_readiness_check",
- step="network_idle",
- result=network_idle_result,
- duration_ms=network_idle_duration_ms,
- timeout_ms=network_idle_timeout_ms,
- )
# 3. Wait for DOM to stabilize
- dom_stability_duration_ms = 0.0
- step_start_time = time.time()
dom_stability_result = "success"
with _tracer.start_as_current_span("skyvern.browser.page_ready.dom_stability") as _ds_span:
apply_context_attrs(_ds_span)
@@ -858,29 +835,7 @@ class SkyvernFrame:
dom_stability_result = "error"
LOG.warning("Failed to check DOM stability, proceeding", exc_info=True)
finally:
- dom_stability_duration_ms = (time.time() - step_start_time) * 1000
_ds_span.set_attribute("result", dom_stability_result)
- LOG.info(
- "page_readiness_check",
- step="dom_stability",
- result=dom_stability_result,
- duration_ms=dom_stability_duration_ms,
- timeout_ms=dom_stability_timeout_ms,
- stable_ms=dom_stable_ms,
- )
-
- # Log total page readiness check duration
- total_duration_ms = (time.time() - total_start_time) * 1000
- LOG.info(
- "page_readiness_check_complete",
- total_duration_ms=total_duration_ms,
- loading_indicator_duration_ms=loading_indicator_duration_ms,
- network_idle_duration_ms=network_idle_duration_ms,
- dom_stability_duration_ms=dom_stability_duration_ms,
- loading_indicator_result=loading_indicator_result,
- network_idle_result=network_idle_result,
- dom_stability_result=dom_stability_result,
- )
async def _wait_for_loading_indicators_gone(self, timeout_ms: float = 5000) -> None:
"""
diff --git a/tests/unit/test_copilot_agent_helpers.py b/tests/unit/test_copilot_agent_helpers.py
index 90d8d2859..97ec790e0 100644
--- a/tests/unit/test_copilot_agent_helpers.py
+++ b/tests/unit/test_copilot_agent_helpers.py
@@ -605,6 +605,17 @@ class TestBlockGoalMainGoal:
class TestTranslateToAgentResultGating:
"""Covers the three SKY-9143 invariants that live in _translate_to_agent_result."""
+ def test_plain_internal_ask_question_label_is_normalized_by_output_policy(self) -> None:
+ ctx = _ctx()
+ result = SimpleNamespace(final_output="ASK_QUESTION\nWhich account should I use?", new_items=[])
+
+ agent_result = agent_module._translate_to_agent_result(
+ result, ctx, global_llm_context=None, chat_request=_chat_request(), organization_id="org-1"
+ )
+
+ assert agent_result.response_type == "ASK_QUESTION"
+ assert agent_result.user_response == "Which account should I use?"
+
def test_inline_replace_workflow_resets_test_ok_after_prior_pass(self, monkeypatch) -> None:
# A prior run_blocks test passed for the old workflow (ctx.last_test_ok=True,
# ctx.last_workflow=old_wf). The agent then emits inline REPLACE_WORKFLOW
@@ -619,7 +630,11 @@ class TestTranslateToAgentResultGating:
ctx = _ctx(last_workflow=old_wf, last_workflow_yaml="old: yaml", last_test_ok=True)
result = _fake_run_result(
- {"type": "REPLACE_WORKFLOW", "user_response": "Here you go.", "workflow_yaml": "new: yaml"}
+ {
+ "type": "REPLACE_WORKFLOW",
+ "user_response": "REPLACE_WORKFLOW\nHere you go.",
+ "workflow_yaml": "new: yaml",
+ }
)
agent_result = agent_module._translate_to_agent_result(
result, ctx, global_llm_context=None, chat_request=_chat_request(), organization_id="org-1"
@@ -1282,6 +1297,40 @@ class TestRequestPolicyCredentialResolution:
handler.assert_not_awaited()
get_credentials_by_ids.assert_not_called()
+ @pytest.mark.asyncio
+ async def test_bulk_colon_delimited_credentials_refuse_before_model_classification(self, monkeypatch) -> None:
+ from skyvern.forge.sdk.copilot import request_policy as policy_module
+ from skyvern.forge.sdk.copilot.request_policy import build_request_policy
+
+ get_credentials_by_ids = AsyncMock()
+ monkeypatch.setattr(
+ policy_module.app,
+ "DATABASE",
+ SimpleNamespace(credentials=SimpleNamespace(get_credentials_by_ids=get_credentials_by_ids)),
+ )
+ handler = AsyncMock()
+
+ policy = await build_request_policy(
+ user_message=(
+ "Use these accounts:\[email protected]:FakePass123!\[email protected]:AnotherFakePass456!"
+ ),
+ workflow_yaml="",
+ chat_history=[],
+ global_llm_context="",
+ organization_id="org-1",
+ handler=handler,
+ )
+
+ assert policy.raw_secret_detected is True
+ assert policy.credential_input_kind == "raw_secret"
+ assert policy.user_response_policy == "ask_clarification"
+ assert policy.allow_update_workflow is False
+ assert policy.allow_run_blocks is False
+ assert "DO NOT PROVIDE RAW LOGIN/PASSWORD" in (policy.clarification_question or "")
+ assert "FakePass123" not in repr(policy.to_trace_data())
+ handler.assert_not_awaited()
+ get_credentials_by_ids.assert_not_called()
+
@pytest.mark.asyncio
async def test_request_policy_url_matching_and_skip_draft(self, monkeypatch) -> None:
from skyvern.forge.sdk.copilot import request_policy as policy_module
diff --git a/tests/unit/test_copilot_enforcement_pruning.py b/tests/unit/test_copilot_enforcement_pruning.py
index ae1b5813d..2ec09e0f8 100644
--- a/tests/unit/test_copilot_enforcement_pruning.py
+++ b/tests/unit/test_copilot_enforcement_pruning.py
@@ -624,6 +624,22 @@ class TestEnforcement:
ask.new_items = []
assert _check_enforcement(ctx, ask) is None
+ def test_plain_labeled_ask_question_passes_even_with_coverage_gap(self) -> None:
+ from skyvern.forge.sdk.copilot.enforcement import _check_enforcement
+
+ ctx = self._make_ctx(
+ update_workflow_called=True,
+ test_after_update_done=True,
+ last_test_ok=True,
+ last_update_block_count=1,
+ user_message="Go to france.fr and then download all french regulations",
+ coverage_nudge_count=0,
+ )
+ ask = MagicMock()
+ ask.final_output = "ASK_QUESTION\nWhich source?"
+ ask.new_items = []
+ assert _check_enforcement(ctx, ask) is None
+
def test_explore_without_workflow_nudge(self) -> None:
from skyvern.forge.sdk.copilot.enforcement import POST_EXPLORE_WITHOUT_WORKFLOW_NUDGE, _check_enforcement
diff --git a/tests/unit/test_copilot_output_policy.py b/tests/unit/test_copilot_output_policy.py
index 54a1fd723..1a6a651bb 100644
--- a/tests/unit/test_copilot_output_policy.py
+++ b/tests/unit/test_copilot_output_policy.py
@@ -2,7 +2,7 @@ from __future__ import annotations
import json
from types import SimpleNamespace
-from unittest.mock import MagicMock
+from unittest.mock import MagicMock, patch
import pytest
@@ -15,6 +15,7 @@ from skyvern.forge.sdk.copilot.output_policy import (
derive_output_kind,
evaluate_output_policy,
hard_block_output_policy_verdict,
+ normalize_response_scaffolding,
)
from skyvern.forge.sdk.copilot.request_policy import RequestPolicy
@@ -88,6 +89,90 @@ def test_rejects_raw_secret_echo_in_user_response() -> None:
assert OutputPolicyReason.RAW_SECRET_LEAK in verdict.reason_codes
+ ("response_type", "user_response", "expected_type", "expected_response"),
+ [
+ ("REPLY", "ASK_QUESTION\nWhich account should I use?", "ASK_QUESTION", "Which account should I use?"),
+ ("REPLY", " ASK_QUESTION\nWhich account should I use?", "ASK_QUESTION", "Which account should I use?"),
+ ("REPLY", "ASK_QUESTION Which account should I use?", "ASK_QUESTION", "Which account should I use?"),
+ ("REPLY", "ask_question\nWhich account should I use?", "ASK_QUESTION", "Which account should I use?"),
+ ("REPLY", "Ask_Question Which account should I use?", "ASK_QUESTION", "Which account should I use?"),
+ ("REPLY", "REPLY\nI can help with that.", "REPLY", "I can help with that."),
+ ("REPLY", "reply\nI can help with that.", "REPLY", "I can help with that."),
+ ("REPLY", "REPLACE_WORKFLOW\nI updated the workflow.", "REPLY", "I updated the workflow."),
+ ("REPLY", "REPLACE_WORKFLOW I updated the workflow.", "REPLY", "I updated the workflow."),
+ (
+ "REPLACE_WORKFLOW",
+ "REPLACE_WORKFLOW\nI updated the workflow.",
+ "REPLACE_WORKFLOW",
+ "I updated the workflow.",
+ ),
+ ],
+)
+def test_normalizes_plain_internal_response_label_scaffolding(
+ response_type: str,
+ user_response: str,
+ expected_type: str,
+ expected_response: str,
+) -> None:
+ normalized = normalize_response_scaffolding(response_type, user_response)
+
+ assert normalized.changed
+ assert normalized.response_type == expected_type
+ assert normalized.user_response == expected_response
+
+
+def test_normalize_scaffolding_preserves_ordinary_reply_sentence() -> None:
+ text = "Reply with the invoice number from the page."
+ normalized = normalize_response_scaffolding("REPLY", text)
+
+ assert not normalized.changed
+ assert normalized.response_type == "REPLY"
+ assert normalized.user_response == text
+
+
+def test_normalize_scaffolding_preserves_wrapped_ordinary_reply_sentence() -> None:
+ text = "Reply with\nthe invoice number from the page."
+ normalized = normalize_response_scaffolding("REPLY", text)
+
+ assert not normalized.changed
+ assert normalized.response_type == "REPLY"
+ assert normalized.user_response == text
+
+
+ "user_response",
+ [
+ "Next step: call get_run_results with this workflow_run_id.",
+ 'Call `get_run_results(workflow_run_id="wr_123")` first, then await user input.',
+ "Then call update_and_run_blocks with a smaller chain.",
+ "Do NOT retry this tool call; wait for the current run result.",
+ "Do NOT re-invoke the tool until the user responds.",
+ ],
+)
+def test_rejects_internal_tool_instruction_leak_in_user_response(user_response: str) -> None:
+ verdict = evaluate_output_policy(
+ request_policy=_policy(),
+ response_type="REPLY",
+ user_response=user_response,
+ )
+
+ assert not verdict.allowed
+ assert OutputPolicyReason.INTERNAL_TOOL_INSTRUCTION_LEAK in verdict.reason_codes
+ assert OutputPolicyReason.INTERNAL_TOOL_INSTRUCTION_LEAK in hard_block_output_policy_verdict(verdict).reason_codes
+
+
+def test_allows_benign_do_not_retry_user_guidance() -> None:
+ verdict = evaluate_output_policy(
+ request_policy=_policy(),
+ response_type="REPLY",
+ user_response="Do not retry until the account lockout clears.",
+ )
+
+ assert verdict.allowed
+ assert OutputPolicyReason.INTERNAL_TOOL_INSTRUCTION_LEAK not in verdict.reason_codes
+
+
@pytest.mark.parametrize(
"user_response",
[
@@ -124,6 +209,30 @@ def test_rejects_raw_secret_in_structured_tool_arguments() -> None:
assert OutputPolicyReason.RAW_SECRET_LEAK in verdict.reason_codes
+def test_rejects_bulk_colon_delimited_credentials_in_tool_arguments() -> None:
+ verdict = evaluate_output_policy(
+ request_policy=_policy(),
+ tool_arguments={
+ "parameters": {"account_list": "[email protected]:FakePass123!\[email protected]:AnotherFakePass456!"}
+ },
+ )
+
+ assert not verdict.allowed
+ assert OutputPolicyReason.RAW_SECRET_LEAK in verdict.reason_codes
+
+
+def test_allows_scp_style_paths_and_url_ports_in_tool_arguments() -> None:
+ verdict = evaluate_output_policy(
+ request_policy=_policy(),
+ tool_arguments={
+ "repository": "[email protected]:skyvern-ai/skyvern.git",
+ "local_url": "https://[email protected]:8080?org=1",
+ },
+ )
+
+ assert verdict.allowed
+
+
def test_does_not_hard_block_on_prior_global_context_secret() -> None:
verdict = evaluate_output_policy(
request_policy=_policy(),
@@ -243,6 +352,97 @@ def test_allows_backend_authored_request_policy_clarification() -> None:
assert verdict.output_kind == CopilotOutputKind.CLARIFICATION_REQUEST
+def test_rejects_deprecated_block_taxonomy_in_final_text() -> None:
+ verdict = evaluate_output_policy(
+ request_policy=_policy(),
+ response_type="REPLY",
+ user_response=(
+ "`task_v2` is deprecated. Use these newer block types instead: "
+ "`navigation`, `extraction`, `validation`, `login`, `goto_url`, and `file_download`."
+ ),
+ )
+
+ assert not verdict.allowed
+ assert OutputPolicyReason.INTERNAL_BLOCK_TAXONOMY_LEAK in verdict.reason_codes
+
+
[email protected]("deprecated_identifier", ["task_v2", "Task_V2", "task-v2", "task v2", "taskv2"])
+def test_rejects_deprecated_block_identifier_outside_informational_answer(deprecated_identifier: str) -> None:
+ verdict = evaluate_output_policy(
+ request_policy=_policy(),
+ response_type="REPLY",
+ output_kind=CopilotOutputKind.WORKFLOW_DRAFT_PROPOSAL,
+ user_response=f"The draft still contains `{deprecated_identifier}` from the older workflow.",
+ )
+
+ assert not verdict.allowed
+ assert OutputPolicyReason.INTERNAL_BLOCK_TAXONOMY_LEAK in verdict.reason_codes
+
+
+def test_rejects_informational_block_taxonomy_list_without_deprecated_name() -> None:
+ verdict = evaluate_output_policy(
+ request_policy=_policy(),
+ response_type="REPLY",
+ user_response=(
+ "Use these block types: navigation for page actions, extraction for data, validation for checks, "
+ "and goto_url for direct URLs."
+ ),
+ )
+
+ assert not verdict.allowed
+ assert OutputPolicyReason.INTERNAL_BLOCK_TAXONOMY_LEAK in verdict.reason_codes
+
+
+def test_allows_two_internal_block_type_terms_in_informational_answer() -> None:
+ verdict = evaluate_output_policy(
+ request_policy=_policy(),
+ response_type="REPLY",
+ user_response="Use a navigation block for the page action and an extraction block for the final data.",
+ )
+
+ assert verdict.allowed
+
+
+def test_allows_generic_navigation_validation_extraction_prose() -> None:
+ verdict = evaluate_output_policy(
+ request_policy=_policy(),
+ response_type="REPLY",
+ user_response=(
+ "After login, the workflow uses navigation to reach the form, validation of the input, and "
+ "extraction of the resulting data."
+ ),
+ )
+
+ assert verdict.allowed
+
+
+def test_allows_taxonomy_terms_outside_informational_answer_without_deprecated_identifier() -> None:
+ verdict = evaluate_output_policy(
+ request_policy=_policy(),
+ response_type="REPLY",
+ output_kind=CopilotOutputKind.WORKFLOW_DRAFT_PROPOSAL,
+ user_response=(
+ "The draft includes navigation for page actions, extraction for data, validation for checks, "
+ "and goto_url for direct URLs."
+ ),
+ )
+
+ assert verdict.allowed
+
+
+def test_allows_single_task_block_reference_without_deprecated_identifier() -> None:
+ verdict = evaluate_output_policy(
+ request_policy=_policy(),
+ response_type="REPLY",
+ user_response=(
+ "For that page action, use a navigation step as the task block and describe what the browser should do."
+ ),
+ )
+
+ assert verdict.allowed
+ assert OutputPolicyReason.INTERNAL_BLOCK_TAXONOMY_LEAK not in verdict.reason_codes
+
+
def test_rejects_unapproved_credential_id_in_final_text() -> None:
verdict = evaluate_output_policy(
request_policy=_policy(),
@@ -462,6 +662,79 @@ def test_sdk_output_guardrail_hard_blocks_raw_secret_final_text() -> None:
assert verdict.reason_codes == [OutputPolicyReason.RAW_SECRET_LEAK]
+ ("reason", "expected_terms"),
+ [
+ (OutputPolicyReason.UNAPPROVED_CREDENTIAL_REFERENCE, ("credential", "confirm")),
+ (OutputPolicyReason.CREDENTIAL_SCOPE_BROADENED, ("credential", "url")),
+ ],
+)
+def test_output_policy_credential_block_asks_for_credential_confirmation(
+ reason: OutputPolicyReason,
+ expected_terms: tuple[str, str],
+) -> None:
+ result = agent_module._build_output_policy_blocked_result(
+ _ctx(),
+ OutputPolicyVerdict(
+ reason_codes=[reason],
+ ),
+ prior_global_llm_context="{}",
+ prior_workflow_yaml="title: Prior",
+ )
+
+ assert result.response_type == "ASK_QUESTION"
+ assert result.clear_proposed_workflow is True
+ response = result.user_response.lower()
+ for term in expected_terms:
+ assert term in response
+ assert "I could not safely return" not in result.user_response
+
+
+def test_scheduling_credential_policy_block_does_not_use_safety_refusal() -> None:
+ scheduling_yaml = """
+workflow_definition:
+ parameters:
+ - parameter_type: workflow
+ workflow_parameter_type: credential_id
+ key: planning_credentials
+ default_value: cred_unapproved
+ blocks:
+ - block_type: navigation
+ label: export_current_month_time_clock_csv
+ url: https://scheduler.example.test/zeitstempel
+ navigation_goal: Waehle den aktuellen Monat aus und exportiere die Zeitstempel CSV fuer Team A und Team B.
+ parameter_keys:
+ - planning_credentials
+ - block_type: navigation
+ label: compare_on_call_matrix
+ url: https://scheduler.example.test/matrix
+ navigation_goal: Vergleiche Matrix und Rufbereitschaft mit der CSV und melde Zeitueberschneidungen.
+ parameter_keys:
+ - planning_credentials
+"""
+ verdict = evaluate_output_policy(
+ request_policy=_policy(),
+ response_type="ASK_QUESTION",
+ user_response="Welche URL des Planungs-Tools soll ich verwenden?",
+ workflow_yaml=scheduling_yaml,
+ has_workflow_proposal=True,
+ )
+
+ assert OutputPolicyReason.UNAPPROVED_CREDENTIAL_REFERENCE in verdict.reason_codes
+
+ result = agent_module._build_output_policy_blocked_result(
+ _ctx(),
+ hard_block_output_policy_verdict(verdict),
+ prior_global_llm_context="{}",
+ prior_workflow_yaml="title: Prior",
+ )
+
+ assert result.response_type == "ASK_QUESTION"
+ assert "credential" in result.user_response.lower()
+ assert "confirm" in result.user_response.lower()
+ assert "I could not safely return" not in result.user_response
+
+
def test_sdk_output_guardrail_ignores_internal_context_credential_ids() -> None:
ctx = _ctx(
request_policy=_policy(
@@ -724,6 +997,59 @@ def test_translate_to_agent_result_rewrites_unbacked_workflow_claim() -> None:
assert agent_result.updated_workflow is None
+def test_translate_to_agent_result_rewrites_deprecated_block_taxonomy() -> None:
+ result = _fake_run_result(
+ {
+ "type": "REPLY",
+ "user_response": (
+ "`task_v2` is deprecated. Use these newer block types instead: "
+ "`navigation`, `extraction`, `validation`, `login`, `goto_url`, and `file_download`."
+ ),
+ }
+ )
+
+ with patch("skyvern.forge.sdk.copilot.agent.LOG.info") as log_info:
+ agent_result = agent_module._translate_to_agent_result(
+ result,
+ _ctx(),
+ global_llm_context=None,
+ chat_request=_chat_request(),
+ organization_id="org-1",
+ )
+
+ assert "task_v2" not in agent_result.user_response
+ assert "`navigation`" not in agent_result.user_response
+ assert "`extraction`" not in agent_result.user_response
+ assert "Describe the page action" in agent_result.user_response
+ assert agent_result.response_type == "REPLY"
+ assert agent_result.updated_workflow is None
+ final_log = next(call for call in log_info.call_args_list if call.args[0] == "copilot output policy final verdict")
+ assert final_log.kwargs["allowed"] is True
+ assert final_log.kwargs["reason_codes"] == []
+ assert final_log.kwargs["soft_rewrite_reason_codes"] == ["internal_block_taxonomy_leak"]
+
+
+def test_translate_to_agent_result_prioritizes_unbacked_workflow_claim_over_taxonomy_rewrite() -> None:
+ result = _fake_run_result(
+ {
+ "type": "REPLY",
+ "user_response": "I've drafted a workflow for you using `task_v2`.",
+ }
+ )
+
+ agent_result = agent_module._translate_to_agent_result(
+ result,
+ _ctx(),
+ global_llm_context=None,
+ chat_request=_chat_request(),
+ organization_id="org-1",
+ )
+
+ assert "wasn't able to produce a workflow proposal" in agent_result.user_response
+ assert "task_v2" not in agent_result.user_response
+ assert agent_result.updated_workflow is None
+
+
def test_translate_to_agent_result_adds_unvalidated_affordance() -> None:
workflow = SimpleNamespace(name="draft")
result = _fake_run_result({"type": "REPLY", "user_response": "I drafted the workflow."})
diff --git a/tests/unit/test_copilot_output_utils.py b/tests/unit/test_copilot_output_utils.py
index a8d95b151..eb8db9167 100644
--- a/tests/unit/test_copilot_output_utils.py
+++ b/tests/unit/test_copilot_output_utils.py
@@ -674,6 +674,16 @@ class TestParseFinalResponse:
assert parsed["type"] == "ASK_QUESTION"
assert parsed["user_response"] == "what date?"
+ def test_plain_leading_label_falls_through_for_output_policy(self) -> None:
+ text = "ASK_QUESTION\nWhich account should I use?"
+ parsed = parse_final_response(text)
+ assert parsed == {"type": "REPLY", "user_response": text}
+
+ def test_sentence_starting_with_reply_is_not_stripped(self) -> None:
+ text = "Reply with the invoice number from the page."
+ parsed = parse_final_response(text)
+ assert parsed == {"type": "REPLY", "user_response": text}
+
def test_strips_leading_replace_workflow_label(self) -> None:
envelope = (
'REPLACE_WORKFLOW {"type": "REPLACE_WORKFLOW", "user_response": "updated", "workflow_yaml": "title: x"}'
@@ -682,6 +692,22 @@ class TestParseFinalResponse:
assert parsed["type"] == "REPLACE_WORKFLOW"
assert parsed["workflow_yaml"] == "title: x"
+ def test_strips_mixed_case_leading_structured_label(self) -> None:
+ envelope = 'ask_question {"type": "ASK_QUESTION", "user_response": "which account?"}'
+ parsed = parse_final_response(envelope)
+ assert parsed["type"] == "ASK_QUESTION"
+ assert parsed["user_response"] == "which account?"
+
+ def test_strips_leading_label_before_json_code_fence(self) -> None:
+ envelope = (
+ "REPLACE_WORKFLOW\n```json\n"
+ '{"type": "REPLACE_WORKFLOW", "user_response": "updated", "workflow_yaml": "title: x"}\n'
+ "```"
+ )
+ parsed = parse_final_response(envelope)
+ assert parsed["type"] == "REPLACE_WORKFLOW"
+ assert parsed["workflow_yaml"] == "title: x"
+
def test_extracts_json_after_prose_preamble(self) -> None:
envelope = 'Here\'s my response: {"type": "REPLY", "user_response": "ok"}'
parsed = parse_final_response(envelope)
diff --git a/tests/unit/test_copilot_per_tool_budget.py b/tests/unit/test_copilot_per_tool_budget.py
index c1ea40859..83f299751 100644
--- a/tests/unit/test_copilot_per_tool_budget.py
+++ b/tests/unit/test_copilot_per_tool_budget.py
@@ -34,12 +34,14 @@ from skyvern.forge.sdk.copilot.failure_tracking import (
compute_failure_signature,
)
from skyvern.forge.sdk.copilot.tools import (
+ WatchdogExitReason,
_mark_pending_reconciliation_run,
_maybe_clear_reconciliation_flag,
_record_per_tool_budget_problem_blocks_from_results,
_record_run_blocks_result,
_record_workflow_update_result,
_tool_loop_error,
+ _watchdog_user_failure_reason,
)
@@ -59,7 +61,7 @@ def _budget_trip_result(workflow_run_id: str = "wr_1") -> dict:
"ok": False,
"error": (
f"The run exceeded the 240s per-tool-call budget while still making progress. "
- f"... Run ID: {workflow_run_id}. ..."
+ f"Run ID: {workflow_run_id}. Next step: call get_run_results with this workflow_run_id."
),
"data": {
"workflow_run_id": workflow_run_id,
@@ -82,6 +84,25 @@ def test_record_sets_top_category_on_per_tool_budget_result() -> None:
assert ctx.last_failure_category_top == PER_TOOL_BUDGET_FAILURE_CATEGORY
+def test_record_uses_policy_failure_reason_not_llm_tool_instruction() -> None:
+ ctx = _fresh_context()
+
+ _record_run_blocks_result(ctx, _budget_trip_result("wr_safe"))
+
+ assert ctx.last_test_failure_reason == "per-tool-call budget exceeded (Run ID: wr_safe)"
+ assert "get_run_results" not in ctx.last_test_failure_reason
+
+
+def test_watchdog_user_failure_reason_excludes_next_tool_instruction() -> None:
+ exit_reason: WatchdogExitReason = "per_tool_budget"
+ reason = _watchdog_user_failure_reason(exit_reason, "wr_safe", 240, None)
+
+ assert "per-tool-call budget" in reason
+ assert "Run ID: wr_safe" in reason
+ assert "get_run_results" not in reason
+ assert "update_and_run_blocks" not in reason
+
+
def test_record_clears_top_category_on_run_with_different_category() -> None:
ctx = _fresh_context()
ctx.last_failure_category_top = PER_TOOL_BUDGET_FAILURE_CATEGORY
diff --git a/tests/unit/test_copilot_request_policy_prompt.py b/tests/unit/test_copilot_request_policy_prompt.py
index 4c237d385..59089cd2d 100644
--- a/tests/unit/test_copilot_request_policy_prompt.py
+++ b/tests/unit/test_copilot_request_policy_prompt.py
@@ -4,6 +4,8 @@ from skyvern.forge.prompts import prompt_engine
from skyvern.forge.sdk.copilot.request_policy import (
PROMPT_NAME,
_raw_secret_detected,
+ contains_email_password_pair,
+ redact_raw_secrets_for_prompt,
)
@@ -52,24 +54,64 @@ class TestRequestPolicyPromptStructure:
assert "customer-<uuid>-pass" in rendered
assert "do NOT classify them as raw_secret" in rendered
+ def test_raw_secret_definition_includes_bulk_email_password_rows(self) -> None:
+ rendered = _render()
+ assert "bulk `[email protected]:<password>` account rows" in rendered
+
-class TestRawSecretRegexBackstop:
- def test_keyvault_handle_does_not_trip_regex(self) -> None:
+class TestRawSecretBackstop:
+ def test_keyvault_handle_does_not_trip_backstop(self) -> None:
assert _raw_secret_detected("customer-aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee-pass") is False
assert _raw_secret_detected("customer-aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee-user") is False
- def test_azure_keyvault_url_does_not_trip_regex(self) -> None:
+ def test_azure_keyvault_url_does_not_trip_backstop(self) -> None:
assert _raw_secret_detected("https://example-vault.vault.azure.net/secrets/my-secret") is False
- def test_workflow_behavior_question_does_not_trip_regex(self) -> None:
+ def test_workflow_behavior_question_does_not_trip_backstop(self) -> None:
message = (
"trigger_login appears to have worked as anticipated but next_step "
"is not receiving an active browser session to work with."
)
assert _raw_secret_detected(message) is False
- def test_actual_raw_password_still_trips_regex(self) -> None:
+ def test_actual_raw_password_still_trips_backstop(self) -> None:
assert _raw_secret_detected("Use this password: hunter2 to sign in.") is True
- def test_actual_api_key_still_trips_regex(self) -> None:
+ def test_colon_delimited_email_password_pair_trips_backstop(self) -> None:
+ assert _raw_secret_detected("Use [email protected]:FakePass123! to sign in.") is True
+ assert contains_email_password_pair("Use [email protected]:FakePass123! to sign in.") is True
+
+ def test_bulk_colon_delimited_email_password_pairs_trip_backstop(self) -> None:
+ message = """
+ Use these accounts:
+ [email protected]:FakePass123!
+ [email protected]:AnotherFakePass456!
+ [email protected]:ThirdFakePass789!
+ """
+ assert _raw_secret_detected(message) is True
+ assert contains_email_password_pair(message) is True
+
+ def test_bulk_colon_delimited_email_password_pairs_are_redacted(self) -> None:
+ redacted = redact_raw_secrets_for_prompt(
+ "Use these accounts:\[email protected]:FakePass123!\[email protected]:AnotherFakePass456!"
+ )
+
+ assert "FakePass123" not in redacted
+ assert "AnotherFakePass456" not in redacted
+ assert redacted.count("[REDACTED_SECRET]") == 2
+
+ def test_plain_email_label_does_not_trip_backstop(self) -> None:
+ assert _raw_secret_detected("Email: [email protected]") is False
+
+ def test_scp_style_repository_path_does_not_trip_backstop(self) -> None:
+ assert _raw_secret_detected("Clone [email protected]:skyvern-ai/skyvern.git before running tests.") is False
+ assert (
+ contains_email_password_pair("Clone [email protected]:skyvern-ai/skyvern.git before running tests.") is False
+ )
+
+ def test_url_port_with_query_does_not_trip_backstop(self) -> None:
+ assert _raw_secret_detected("Use https://[email protected]:8080?org=1 for local testing.") is False
+ assert contains_email_password_pair("Use https://[email protected]:8080?org=1 for local testing.") is False
+
+ def test_actual_api_key_still_trips_backstop(self) -> None:
assert _raw_secret_detected("The api_key = sk-abcdefghijklmnopqrstuvwxyz1234567890.") is True