Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
Bind Copilot evaluation identity to the API environment (#8826)
Andrew Neilson3c5ef488c113
diff --git a/skyvern/forge/sdk/copilot/output_policy.py b/skyvern/forge/sdk/copilot/output_policy.pyindex d8609c3bb..a114817e0 100644--- a/skyvern/forge/sdk/copilot/output_policy.py+++ b/skyvern/forge/sdk/copilot/output_policy.py@@ -61,35 +61,6 @@ UNVALIDATED_DISCLOSURE_PHRASES = ("hasn't been verified","unvalidated",)-_INTERNAL_BLOCK_TYPE_TERMS = frozenset(- {- "navigation",- "extraction",- "validation",- "login",- "goto_url",- "file_download",- "file_upload",- "text_prompt",- "for_loop",- "conditional",- "action",- "wait",- }-)-_INTERNAL_BLOCK_TYPE_CONTEXT_MARKERS = (- "block type",- "block types",- "internal block",- "internal blocks",- "workflow block",- "workflow blocks",- "supported block",- "supported blocks",-)-# Three distinct internal names in an informational reply implies taxonomy-# enumeration, while one or two may be incidental product-language prose.-_INFORMATIONAL_TAXONOMY_TERM_THRESHOLD = 3_IDENTIFIER_TOKEN_RE = re.compile(r"[A-Za-z][A-Za-z0-9_]*")_IDENTIFIER_DELIMITERS = frozenset({"`", '"', "'"})@@ -386,7 +357,7 @@ def evaluate_output_policy(# reaches storage, and scanning a draft judged the YAML encoding rather than the value.if _contains_raw_secret(user_response):verdict.add(OutputPolicyReason.RAW_SECRET_LEAK)- if _contains_internal_block_taxonomy_leak(user_response, output_kind, response_type):+ if _contains_internal_block_taxonomy_leak(user_response, response_type):verdict.add(OutputPolicyReason.INTERNAL_BLOCK_TAXONOMY_LEAK)if response_type in _USER_VISIBLE_REPLY_TYPES and _contains_internal_classifier_vocab_leak(user_response):verdict.add(OutputPolicyReason.INTERNAL_CLASSIFIER_VOCAB_LEAK)@@ -561,23 +532,14 @@ def _has_unvalidated_affordance(user_response: str | None) -> bool:return bool(_UNVALIDATED_PROPOSAL_AFFORDANCE_RE.search(user_response) and has_disclosure)-def _contains_internal_block_taxonomy_leak(- user_response: str | None,- output_kind: CopilotOutputKind,- response_type: str,-) -> bool:+def _contains_internal_block_taxonomy_leak(user_response: str | None, response_type: str) -> bool:if not user_response:return Falseif _contains_deprecated_block_identifier(user_response):return Trueif _contains_yaml_authoring_vocab_leak(user_response, response_type):return True- if response_type in _USER_VISIBLE_REPLY_TYPES and _contains_internal_tool_vocab_leak(user_response):- return True- if output_kind != CopilotOutputKind.INFORMATIONAL_ANSWER:- return False- taxonomy_terms = _internal_block_taxonomy_terms(user_response)- return len(taxonomy_terms) >= _INFORMATIONAL_TAXONOMY_TERM_THRESHOLD+ return response_type in _USER_VISIBLE_REPLY_TYPES and _contains_internal_tool_vocab_leak(user_response)def _contains_internal_tool_vocab_leak(user_response: str) -> bool:@@ -594,31 +556,6 @@ def _contains_deprecated_block_identifier(text: str) -> bool:return any(left == "task" and right == "v2" for left, right in zip(tokens, tokens[1:]))-def _internal_block_taxonomy_terms(text: str) -> set[str]:- lower = text.lower()- has_taxonomy_context = any(marker in lower for marker in _INTERNAL_BLOCK_TYPE_CONTEXT_MARKERS)- matches = list(_IDENTIFIER_TOKEN_RE.finditer(text))- terms: set[str] = set()- for index, match in enumerate(matches):- term = _normalized_internal_block_term(match.group(0))- if term is None:- continue- if has_taxonomy_context or _is_delimited_identifier(text, match.start(), match.end()):- terms.add(term)- continue- next_token = _compact_identifier_token(matches[index + 1].group(0)) if index + 1 < len(matches) else None- if next_token in {"block", "blocks", "for"}:- terms.add(term)- return terms---def _normalized_internal_block_term(raw: str) -> str | None:- term = raw.lower()- if term in _INTERNAL_BLOCK_TYPE_TERMS:- return term- return None--def _compact_identifier_token(raw: str) -> str:return raw.lower().replace("_", "")diff --git a/tests/unit/test_copilot_output_policy.py b/tests/unit/test_copilot_output_policy.pyindex 4e7522aba..5742ffedc 100644--- a/tests/unit/test_copilot_output_policy.py+++ b/tests/unit/test_copilot_output_policy.py@@ -844,51 +844,13 @@ def test_rejects_deprecated_block_identifier_outside_informational_answer(deprecassert OutputPolicyReason.INTERNAL_BLOCK_TAXONOMY_LEAK in verdict.reason_codes-def test_rejects_informational_block_taxonomy_list_without_deprecated_name() -> None:+def test_allows_an_answer_that_names_several_block_types() -> None:verdict = evaluate_output_policy(request_policy=_policy(),response_type="REPLY",user_response=(- "Use these block types: navigation for page actions, extraction for data, validation for checks, "- "and goto_url for direct URLs."- ),- )-- assert not verdict.allowed- assert OutputPolicyReason.INTERNAL_BLOCK_TAXONOMY_LEAK in verdict.reason_codes---def test_allows_two_internal_block_type_terms_in_informational_answer() -> None:- verdict = evaluate_output_policy(- request_policy=_policy(),- response_type="REPLY",- user_response="Use a navigation block for the page action and an extraction block for the final data.",- )-- assert verdict.allowed---def test_allows_generic_navigation_validation_extraction_prose() -> None:- verdict = evaluate_output_policy(- request_policy=_policy(),- response_type="REPLY",- user_response=(- "After login, the workflow uses navigation to reach the form, validation of the input, and "- "extraction of the resulting data."- ),- )-- assert verdict.allowed---def test_allows_taxonomy_terms_outside_informational_answer_without_deprecated_identifier() -> None:- verdict = evaluate_output_policy(- request_policy=_policy(),- response_type="REPLY",- output_kind=CopilotOutputKind.WORKFLOW_DRAFT_PROPOSAL,- user_response=(- "The draft includes navigation for page actions, extraction for data, validation for checks, "- "and goto_url for direct URLs."+ "Use a `for_loop` block over the list of URLs. Inside it, a `goto_url` block opens each one and an "+ "`extraction` block reads the page title; add a `validation` block if the page must load first."),)