Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
SKY-11094: finish the leveled / tri-state completion-adjudication contract (#6639)
Andrew Neilson4d8da05e1caa
diff --git a/skyvern/forge/sdk/copilot/completion_verification.py b/skyvern/forge/sdk/copilot/completion_verification.pyindex ef406881f..a5c742497 100644--- a/skyvern/forge/sdk/copilot/completion_verification.py+++ b/skyvern/forge/sdk/copilot/completion_verification.py@@ -346,6 +346,110 @@ def grade_definition_criteria(criteria: list[CompletionCriterion], workflow_yamlreturn verdicts+# Quoted literals need >=4 chars: a 2-3 char quoted token (a state code, "id", "ok")+# collides with incidental prose, too low-specificity to credit on lexical presence.+_QUOTED_LITERAL_RE = re.compile(r"[\"'‘’“”]([^\"'‘’“”]{4,120})[\"'‘’“”]")+_CURRENCY_LITERAL_RE = re.compile(r"[$€£¥]\s?\d[\d,]*(?:\.\d+)?")+_ISO_DATE_LITERAL_RE = re.compile(r"\b\d{4}-\d{2}-\d{2}\b")+# A bare digit run qualifies as an identifier only at >=5 digits: a 4-digit run is+# usually a year and too low-specificity to credit on a coincidental output match.+_LONG_DIGIT_LITERAL_RE = re.compile(r"\b\d{5,}\b")+++def _normalize_present_value(text: str) -> str:+ return " ".join(text.casefold().split())+++def _extract_present_value_literals(outcome: str) -> list[str]:+ literals: list[str] = []+ for match in _QUOTED_LITERAL_RE.finditer(outcome):+ literals.append(match.group(1))+ for pattern in (_CURRENCY_LITERAL_RE, _ISO_DATE_LITERAL_RE, _LONG_DIGIT_LITERAL_RE):+ literals.extend(pattern.findall(outcome))+ normalized: list[str] = []+ seen: set[str] = set()+ for literal in literals:+ candidate = _normalize_present_value(literal)+ if len(candidate) >= 2 and candidate not in seen:+ seen.add(candidate)+ normalized.append(candidate)+ return normalized+++def _serialized_block_output_haystacks(block_outputs: dict[str, Any]) -> list[tuple[str, str]]:+ haystacks: list[tuple[str, str]] = []+ for label, payload in block_outputs.items():+ if payload is None:+ continue+ forms = [payload] if isinstance(payload, str) else [json.dumps(payload, default=str), str(payload)]+ normalized = " ".join(_normalize_present_value(form) for form in forms if form)+ if normalized:+ haystacks.append((str(label), normalized))+ return haystacks+++def _present_verbatim(literal: str, haystack: str) -> bool:+ # Boundary-aware: plain substring would over-credit a short/numeric literal against+ # a sibling value ('$10' in '$100', 'ca' in 'california') — require token boundaries.+ start = 0+ while (idx := haystack.find(literal, start)) != -1:+ before = haystack[idx - 1] if idx else ""+ before_prev = haystack[idx - 2] if idx >= 2 else ""+ end = idx + len(literal)+ after = haystack[end] if end < len(haystack) else ""+ after_next = haystack[end + 1] if end + 1 < len(haystack) else ""+ embedded = (+ before.isalnum()+ or after.isalnum()+ or (after in ".," and after_next.isdigit())+ or (before in ".," and before_prev.isdigit())+ )+ if not embedded:+ return True+ start = idx + 1+ return False+++def grade_present_value_criteria(+ criteria: list[CompletionCriterion], snapshot: RunEvidenceSnapshot+) -> list[CriterionVerdict]:+ """Deterministically credit a run-plane criterion whose explicitly named/quoted+ value appears verbatim in the run's own block outputs.++ Abstains (emits nothing) for any criterion lacking a high-specificity literal or+ whose literal is not present, so the judge keeps deciding and recall is never+ weakened; only ever upgrades to ``satisfied``.+ """+ haystacks = _serialized_block_output_haystacks(snapshot.block_outputs)+ if not haystacks:+ return []+ verdicts: list[CriterionVerdict] = []+ for criterion in criteria:+ literals = _extract_present_value_literals(criterion.outcome)+ if not literals:+ continue+ # Every named literal must appear in a SINGLE block output: a partial match+ # (e.g. a date present while the named total is not) is not the named outcome.+ match_label = next(+ (+ label+ for label, haystack in haystacks+ if all(_present_verbatim(literal, haystack) for literal in literals)+ ),+ None,+ )+ if match_label is not None:+ verdicts.append(+ CriterionVerdict(+ criterion_id=criterion.id,+ state="satisfied",+ reason_code="present_value_verbatim",+ evidence_ref=f"block_outputs:{match_label}",+ )+ )+ return verdicts++_DEFINITION_REASON_PREFIX = "definition_"diff --git a/skyvern/forge/sdk/copilot/tools/blockers.py b/skyvern/forge/sdk/copilot/tools/blockers.pyindex a99d21bfc..1cfe825bc 100644--- a/skyvern/forge/sdk/copilot/tools/blockers.py+++ b/skyvern/forge/sdk/copilot/tools/blockers.py@@ -835,6 +835,13 @@ _ANTI_BOT_BLOCKER_TERMS: tuple[str, ...] = ("human verification","verify you are human",)+# Multi-word anti-bot phrases only: the bare tokens ``captcha``/``challenge`` are+# excluded so business text mentioning them does not false-positive when a code-block+# value is scanned regardless of its key.+_BROAD_SINGLE_TOKEN_TERMS: frozenset[str] = frozenset({"captcha", "challenge"})+_ANTI_BOT_BLOCKER_PHRASES: tuple[str, ...] = tuple(+ term for term in _ANTI_BOT_BLOCKER_TERMS if term not in _BROAD_SINGLE_TOKEN_TERMS+)# Strict subset of ``_STRUCTURED_BLOCKER_KEY_TERMS`` for the flag/status rules that# scan arbitrary code-block JSON; broad terms like ``verification`` stay string-only._STRICT_BLOCKER_FLAG_TERMS: frozenset[str] = frozenset(@@ -864,6 +871,11 @@ def _looks_like_anti_bot_blocker(text: str) -> bool:return any(term in lowered for term in _ANTI_BOT_BLOCKER_TERMS)+def _looks_like_anti_bot_phrase(text: str) -> bool:+ lowered = text.lower()+ return any(phrase in lowered for phrase in _ANTI_BOT_BLOCKER_PHRASES)++def _structured_blocker_message(value: object,*,@@ -871,6 +883,7 @@ def _structured_blocker_message(include_flag_keys: bool = False,key_terms: frozenset[str] = _STRUCTURED_BLOCKER_KEY_TERMS,declared_keys: frozenset[str] = frozenset(),+ scan_all_values_for_anti_bot: bool = False,) -> str | None:if depth > 5:return None@@ -884,9 +897,13 @@ def _structured_blocker_message(has_blocker_key = normalized_key in _STRUCTURED_BLOCKER_MESSAGE_KEYS or any(term in normalized_key for term in key_terms)- if has_blocker_key or (- normalized_key in {"message", "error", "failure_reason", "reason"}- and _looks_like_anti_bot_blocker(item)+ if (+ has_blocker_key+ or (+ normalized_key in {"message", "error", "failure_reason", "reason"}+ and _looks_like_anti_bot_blocker(item)+ )+ or (scan_all_values_for_anti_bot and _looks_like_anti_bot_phrase(item))):return item.strip()[:240]if include_flag_keys:@@ -900,6 +917,7 @@ def _structured_blocker_message(include_flag_keys=include_flag_keys,key_terms=key_terms,declared_keys=declared_keys,+ scan_all_values_for_anti_bot=scan_all_values_for_anti_bot,)if nested:return nested@@ -911,6 +929,7 @@ def _structured_blocker_message(include_flag_keys=include_flag_keys,key_terms=key_terms,declared_keys=declared_keys,+ scan_all_values_for_anti_bot=scan_all_values_for_anti_bot,)if nested:return nested@@ -982,12 +1001,14 @@ def _run_blocks_structured_blocker_message(result: dict[str, Any], copilot_ctx:if _is_code_block_type(block_type):# Code-block outputs are arbitrary JSON the model authored: key matching# uses the strict term set (broad terms like ``verification`` belong to- # the page-text arms) and metadata-declared goal keys are exempt.+ # the page-text arms) and metadata-declared goal keys are exempt. A value+ # carrying a real anti-bot phrase is still caught regardless of its key.blocker = _structured_blocker_message(block.get("extracted_data"),include_flag_keys=True,key_terms=_STRICT_BLOCKER_FLAG_TERMS,declared_keys=_declared_code_output_keys(copilot_ctx, block.get("label")),+ scan_all_values_for_anti_bot=True,)elif block_type in _DATA_PRODUCING_BLOCK_TYPES:payload = _block_data_payload(block.get("extracted_data"), block_type)@@ -1574,6 +1595,7 @@ def _analyze_run_blocks(include_flag_keys=True,key_terms=_STRICT_BLOCKER_FLAG_TERMS,declared_keys=declared_keys,+ scan_all_values_for_anti_bot=True,)if structured_blocker:texts_to_scan.append(structured_blocker)diff --git a/skyvern/forge/sdk/copilot/tools/completion.py b/skyvern/forge/sdk/copilot/tools/completion.pyindex ca8349778..7e57d544a 100644--- a/skyvern/forge/sdk/copilot/tools/completion.py+++ b/skyvern/forge/sdk/copilot/tools/completion.py@@ -12,6 +12,7 @@ from skyvern.forge.sdk.copilot.completion_verification import (combine_verification_results,evaluate_completion_criteria,grade_definition_criteria,+ grade_present_value_criteria,summarize_unsatisfied_outcomes,)from skyvern.forge.sdk.copilot.enforcement import _goal_likely_needs_more_blocks@@ -162,6 +163,7 @@ async def _maybe_run_completion_verification_from_page_observation()else:run_result = await evaluate_completion_criteria(run_criteria, snapshot, handler)+ run_result = _apply_present_value_upgrades(run_result, run_criteria, snapshot)verification = combine_verification_results(criterion_ids, run_result, definition_verdicts)if (@@ -261,6 +263,31 @@ def _build_run_evidence_snapshot(copilot_ctx: Any, result: dict[str, Any]) -> Ru)+def _apply_present_value_upgrades(+ run_result: CompletionVerificationResult,+ run_criteria: list[CompletionCriterion],+ snapshot: RunEvidenceSnapshot,+) -> CompletionVerificationResult:+ """Upgrade a ``no_evidence``/``unknown`` run verdict to a deterministic present-value+ ``satisfied``. An ``evidence_contradicts`` verdict is left to the judge, a judge+ ``satisfied`` is never downgraded, and an unavailable result is never fabricated.+ """+ if run_result.status != "evaluated":+ return run_result+ upgrades = {verdict.criterion_id: verdict for verdict in grade_present_value_criteria(run_criteria, snapshot)}+ if not upgrades:+ return run_result+ verdicts = [+ upgrades[verdict.criterion_id]+ if verdict.criterion_id in upgrades and not verdict.satisfied and verdict.reason_code != "evidence_contradicts"+ else verdict+ for verdict in run_result.verdicts+ ]+ return CompletionVerificationResult(+ status="evaluated", criterion_ids=list(run_result.criterion_ids), verdicts=verdicts+ )++async def _maybe_run_completion_verification(copilot_ctx: Any, result: dict[str, Any], handler_start: float) -> CompletionVerificationResult | None:@@ -305,6 +332,7 @@ async def _maybe_run_completion_verification()else:run_result = await evaluate_completion_criteria(run_criteria, snapshot, handler)+ run_result = _apply_present_value_upgrades(run_result, run_criteria, snapshot)return combine_verification_results(criterion_ids, run_result, definition_verdicts)diff --git a/tests/unit/test_copilot_completion_criteria_lifecycle.py b/tests/unit/test_copilot_completion_criteria_lifecycle.pyindex ab2c87357..134b3d913 100644--- a/tests/unit/test_copilot_completion_criteria_lifecycle.py+++ b/tests/unit/test_copilot_completion_criteria_lifecycle.py@@ -26,9 +26,11 @@ from skyvern.forge.sdk.copilot.completion_criteria_store import (from skyvern.forge.sdk.copilot.completion_verification import (CompletionVerificationResult,CriterionVerdict,+ RunEvidenceSnapshot,_coerce_result,combine_verification_results,grade_definition_criteria,+ grade_present_value_criteria,run_plane_all_no_evidence,)from skyvern.forge.sdk.copilot.context import CopilotContext@@ -50,7 +52,11 @@ from skyvern.forge.sdk.copilot.request_policy import (_parse_completion_criteria,normalized_criterion_outcome_key,)-from skyvern.forge.sdk.copilot.tools.completion import _outcome_failure_warrants_repair, _split_criteria_by_plane+from skyvern.forge.sdk.copilot.tools.completion import (+ _apply_present_value_upgrades,+ _outcome_failure_warrants_repair,+ _split_criteria_by_plane,+)def _criterion(cid: str, outcome: str, *, level: str = "run") -> CompletionCriterion:@@ -437,6 +443,137 @@ def test_split_criteria_by_plane() -> None:assert [c.id for c in definition_criteria] == ["c1"]+def _snapshot(block_outputs: dict) -> RunEvidenceSnapshot:+ return RunEvidenceSnapshot(workflow_run_id="wr_test", block_outputs=block_outputs)+++def test_present_value_credits_quoted_currency_literal_verbatim() -> None:+ criteria = [_criterion("c0", "the May 2026 statement total reads $4,210.55")]+ snapshot = _snapshot({"read_statement": {"month": "May 2026", "total": "$4,210.55"}})+ (verdict,) = grade_present_value_criteria(criteria, snapshot)+ assert verdict.state == "satisfied"+ assert verdict.reason_code == "present_value_verbatim"+ assert verdict.evidence_ref == "block_outputs:read_statement"+++def test_present_value_abstains_when_literal_absent_from_outputs() -> None:+ criteria = [_criterion("c0", "the May 2026 statement total reads $4,210.55")]+ snapshot = _snapshot({"read_statement": {"total": "$1,000.00"}})+ assert grade_present_value_criteria(criteria, snapshot) == []+++def test_present_value_abstains_on_empty_block_outputs() -> None:+ criteria = [_criterion("c0", "the May 2026 statement total reads $4,210.55")]+ assert grade_present_value_criteria(criteria, _snapshot({})) == []+++def test_present_value_abstains_on_blocked_run_without_target_literal() -> None:+ criteria = [_criterion("c0", "the statement total reads $4,210.55")]+ snapshot = _snapshot({"download": {"blocker": "verify you are human to continue"}})+ assert grade_present_value_criteria(criteria, snapshot) == []+++def test_present_value_abstains_when_no_high_specificity_literal() -> None:+ criteria = [_criterion("c0", "the invoice was downloaded successfully")]+ snapshot = _snapshot({"download": {"file": "invoice downloaded successfully"}})+ assert grade_present_value_criteria(criteria, snapshot) == []+++def test_present_value_does_not_credit_on_bare_year_token() -> None:+ # A 4-digit year is too low-specificity: a coincidental output mention of the+ # year must not credit the criterion.+ criteria = [_criterion("c0", "the 2026 billing summary is shown")]+ snapshot = _snapshot({"footer": {"copyright": "© 2026 Example Corp"}})+ assert grade_present_value_criteria(criteria, snapshot) == []+++def test_present_value_credits_multi_digit_account_identifier() -> None:+ criteria = [_criterion("c0", "Billing is open for account 100245")]+ snapshot = _snapshot({"open_billing": {"account_number": "100245", "tab": "Billing"}})+ (verdict,) = grade_present_value_criteria(criteria, snapshot)+ assert verdict.state == "satisfied"+ assert verdict.reason_code == "present_value_verbatim"+++def test_present_value_requires_every_named_literal_present() -> None:+ # A criterion naming both a date and a total must not be credited when only the+ # date is present: a partial match is not the named outcome.+ criteria = [_criterion("c0", "the 2026-05-05 statement totaling $4,210.55 is downloaded")]+ partial = _snapshot({"observe": {"statement_date": "2026-05-05"}})+ assert grade_present_value_criteria(criteria, partial) == []++ full = _snapshot({"download": {"date": "2026-05-05", "total": "$4,210.55"}})+ (verdict,) = grade_present_value_criteria(criteria, full)+ assert verdict.state == "satisfied"++ # Both literals must land in the SAME block output, not split across two.+ split = _snapshot({"observe": {"date": "2026-05-05"}, "summary": {"total": "$4,210.55"}})+ assert grade_present_value_criteria(criteria, split) == []+++def test_present_value_rejects_substring_of_longer_token() -> None:+ # Plain containment would over-credit a short/numeric literal against a sibling+ # value; a bounded match must reject these.+ embedded_quote = [_criterion("c0", 'the vendor is "acme"')]+ assert grade_present_value_criteria(embedded_quote, _snapshot({"x": {"vendor": "acmecorp"}})) == []++ currency = [_criterion("c0", "the total is $10")]+ assert grade_present_value_criteria(currency, _snapshot({"x": {"total": "$100"}})) == []+ assert grade_present_value_criteria(currency, _snapshot({"x": {"total": "$10.50"}})) == []++ identifier = [_criterion("c0", "account 100245")]+ assert grade_present_value_criteria(identifier, _snapshot({"x": {"account": "1002450"}})) == []+ assert grade_present_value_criteria(identifier, _snapshot({"x": {"account": "9100245"}})) == []+ assert grade_present_value_criteria(identifier, _snapshot({"x": {"ratio": "0.100245"}})) == []+++def test_present_value_credits_exact_bounded_literal() -> None:+ currency = [_criterion("c0", "the total is $10")]+ (verdict,) = grade_present_value_criteria(currency, _snapshot({"x": {"total": "$10"}}))+ assert verdict.reason_code == "present_value_verbatim"++ identifier = [_criterion("c0", "account 100245")]+ (verdict2,) = grade_present_value_criteria(identifier, _snapshot({"x": {"account": "100245"}}))+ assert verdict2.reason_code == "present_value_verbatim"+++def test_present_value_quoted_literal_specificity_floor() -> None:+ # A 2-3 char quoted literal is too low-specificity to credit even when present+ # verbatim (it collides with incidental prose); >=4 chars credits.+ two_char = [_criterion("c0", 'the state is "ca"')]+ assert grade_present_value_criteria(two_char, _snapshot({"x": {"state": "ca"}})) == []++ four_char = [_criterion("c0", 'the vendor is "acme"')]+ (verdict,) = grade_present_value_criteria(four_char, _snapshot({"x": {"vendor": "acme"}}))+ assert verdict.reason_code == "present_value_verbatim"+++def test_present_value_upgrade_is_upgrade_only() -> None:+ criteria = [_criterion("c0", 'the confirmation number is "ABC12345"')]+ snapshot = _snapshot({"confirm": {"number": "ABC12345"}})++ run_result = _evaluated(_verdict("c0", "unsatisfied", "no_evidence"))+ upgraded = _apply_present_value_upgrades(run_result, criteria, snapshot)+ assert upgraded.verdicts[0].state == "satisfied"+ assert upgraded.verdicts[0].reason_code == "present_value_verbatim"++ already_satisfied = _evaluated(_verdict("c0", "satisfied", "evidence_confirms"))+ kept = _apply_present_value_upgrades(already_satisfied, criteria, snapshot)+ assert kept.verdicts[0].reason_code == "evidence_confirms"++ contradicts = _evaluated(_verdict("c0", "unsatisfied", "evidence_contradicts"))+ absent_snapshot = _snapshot({"confirm": {"number": "ZZZ00000"}})+ unchanged = _apply_present_value_upgrades(contradicts, criteria, absent_snapshot)+ assert unchanged.verdicts[0].reason_code == "evidence_contradicts"++ present_contradicts = _evaluated(_verdict("c0", "unsatisfied", "evidence_contradicts"))+ kept_contradicts = _apply_present_value_upgrades(present_contradicts, criteria, snapshot)+ assert kept_contradicts.verdicts[0].reason_code == "evidence_contradicts"++ unavailable = CompletionVerificationResult(status="unavailable")+ assert _apply_present_value_upgrades(unavailable, criteria, snapshot).status == "unavailable"++def test_parse_completion_criteria_coerces_level() -> None:parsed = _parse_completion_criteria([diff --git a/tests/unit/test_copilot_run_output_content_gate.py b/tests/unit/test_copilot_run_output_content_gate.pyindex 137448ca8..1c3c583dd 100644--- a/tests/unit/test_copilot_run_output_content_gate.py+++ b/tests/unit/test_copilot_run_output_content_gate.py@@ -502,6 +502,33 @@ def test_verification_key_counts_as_goal_content_for_code_outputs() -> None:assert empty_data_blocks is False+def test_anti_bot_value_under_broad_key_trips_code_block_blocker() -> None:+ # SKY-10916: a broad/descriptive key carrying a real anti-bot phrase is caught+ # for code outputs even though the key stays out of the strict term set.+ result = _run_result([_code_block("check_access", {"verification": "verify you are human to continue"})])+ assert _run_blocks_structured_blocker_message(result) == "verify you are human to continue"++ status_detail = _run_result([_code_block("check_access", {"status_detail": "human verification required"})])+ assert _run_blocks_structured_blocker_message(status_detail) == "human verification required"+++def test_descriptive_verification_key_with_benign_value_stays_exempt() -> None:+ benign = _run_result([_code_block("plan_step", {"verification_method": "fill the login form"})])+ assert _run_blocks_structured_blocker_message(benign) is None++ boolean_flag = _run_result([_code_block("notify", {"verification_passed": True})])+ assert _run_blocks_structured_blocker_message(boolean_flag) is None+++def test_anti_bot_value_scan_gated_on_phrases_not_bare_tokens() -> None:+ # Extracted business text mentioning ``verification``/``challenge`` must not+ # false-positive under the phrase-set gate.+ result = _run_result(+ [_code_block("summary", {"notes": "we need verification of the challenge results before May"})]+ )+ assert _run_blocks_structured_blocker_message(result) is None++class _MetadataCtx:def __init__(self, metadata: dict) -> None:self.code_artifact_metadata = metadata