Skip to content

Automate browser based workflows with AI

SKY-11094: finish the leveled / tri-state completion-adjudication contract (#6639)

Andrew Neilson4d8da05e1caa
diff --git a/skyvern/forge/sdk/copilot/completion_verification.py b/skyvern/forge/sdk/copilot/completion_verification.py
index ef406881f..a5c742497 100644
--- a/skyvern/forge/sdk/copilot/completion_verification.py
+++ b/skyvern/forge/sdk/copilot/completion_verification.py
@@ -346,6 +346,110 @@ def grade_definition_criteria(criteria: list[CompletionCriterion], workflow_yaml
return verdicts
+# Quoted literals need >=4 chars: a 2-3 char quoted token (a state code, "id", "ok")
+# collides with incidental prose, too low-specificity to credit on lexical presence.
+_QUOTED_LITERAL_RE = re.compile(r"[\"'‘’“”]([^\"'‘’“”]{4,120})[\"'‘’“”]")
+_CURRENCY_LITERAL_RE = re.compile(r"[$€£¥]\s?\d[\d,]*(?:\.\d+)?")
+_ISO_DATE_LITERAL_RE = re.compile(r"\b\d{4}-\d{2}-\d{2}\b")
+# A bare digit run qualifies as an identifier only at >=5 digits: a 4-digit run is
+# usually a year and too low-specificity to credit on a coincidental output match.
+_LONG_DIGIT_LITERAL_RE = re.compile(r"\b\d{5,}\b")
+
+
+def _normalize_present_value(text: str) -> str:
+ return " ".join(text.casefold().split())
+
+
+def _extract_present_value_literals(outcome: str) -> list[str]:
+ literals: list[str] = []
+ for match in _QUOTED_LITERAL_RE.finditer(outcome):
+ literals.append(match.group(1))
+ for pattern in (_CURRENCY_LITERAL_RE, _ISO_DATE_LITERAL_RE, _LONG_DIGIT_LITERAL_RE):
+ literals.extend(pattern.findall(outcome))
+ normalized: list[str] = []
+ seen: set[str] = set()
+ for literal in literals:
+ candidate = _normalize_present_value(literal)
+ if len(candidate) >= 2 and candidate not in seen:
+ seen.add(candidate)
+ normalized.append(candidate)
+ return normalized
+
+
+def _serialized_block_output_haystacks(block_outputs: dict[str, Any]) -> list[tuple[str, str]]:
+ haystacks: list[tuple[str, str]] = []
+ for label, payload in block_outputs.items():
+ if payload is None:
+ continue
+ forms = [payload] if isinstance(payload, str) else [json.dumps(payload, default=str), str(payload)]
+ normalized = " ".join(_normalize_present_value(form) for form in forms if form)
+ if normalized:
+ haystacks.append((str(label), normalized))
+ return haystacks
+
+
+def _present_verbatim(literal: str, haystack: str) -> bool:
+ # Boundary-aware: plain substring would over-credit a short/numeric literal against
+ # a sibling value ('$10' in '$100', 'ca' in 'california') — require token boundaries.
+ start = 0
+ while (idx := haystack.find(literal, start)) != -1:
+ before = haystack[idx - 1] if idx else ""
+ before_prev = haystack[idx - 2] if idx >= 2 else ""
+ end = idx + len(literal)
+ after = haystack[end] if end < len(haystack) else ""
+ after_next = haystack[end + 1] if end + 1 < len(haystack) else ""
+ embedded = (
+ before.isalnum()
+ or after.isalnum()
+ or (after in ".," and after_next.isdigit())
+ or (before in ".," and before_prev.isdigit())
+ )
+ if not embedded:
+ return True
+ start = idx + 1
+ return False
+
+
+def grade_present_value_criteria(
+ criteria: list[CompletionCriterion], snapshot: RunEvidenceSnapshot
+) -> list[CriterionVerdict]:
+ """Deterministically credit a run-plane criterion whose explicitly named/quoted
+ value appears verbatim in the run's own block outputs.
+
+ Abstains (emits nothing) for any criterion lacking a high-specificity literal or
+ whose literal is not present, so the judge keeps deciding and recall is never
+ weakened; only ever upgrades to ``satisfied``.
+ """
+ haystacks = _serialized_block_output_haystacks(snapshot.block_outputs)
+ if not haystacks:
+ return []
+ verdicts: list[CriterionVerdict] = []
+ for criterion in criteria:
+ literals = _extract_present_value_literals(criterion.outcome)
+ if not literals:
+ continue
+ # Every named literal must appear in a SINGLE block output: a partial match
+ # (e.g. a date present while the named total is not) is not the named outcome.
+ match_label = next(
+ (
+ label
+ for label, haystack in haystacks
+ if all(_present_verbatim(literal, haystack) for literal in literals)
+ ),
+ None,
+ )
+ if match_label is not None:
+ verdicts.append(
+ CriterionVerdict(
+ criterion_id=criterion.id,
+ state="satisfied",
+ reason_code="present_value_verbatim",
+ evidence_ref=f"block_outputs:{match_label}",
+ )
+ )
+ return verdicts
+
+
_DEFINITION_REASON_PREFIX = "definition_"
diff --git a/skyvern/forge/sdk/copilot/tools/blockers.py b/skyvern/forge/sdk/copilot/tools/blockers.py
index a99d21bfc..1cfe825bc 100644
--- a/skyvern/forge/sdk/copilot/tools/blockers.py
+++ b/skyvern/forge/sdk/copilot/tools/blockers.py
@@ -835,6 +835,13 @@ _ANTI_BOT_BLOCKER_TERMS: tuple[str, ...] = (
"human verification",
"verify you are human",
)
+# Multi-word anti-bot phrases only: the bare tokens ``captcha``/``challenge`` are
+# excluded so business text mentioning them does not false-positive when a code-block
+# value is scanned regardless of its key.
+_BROAD_SINGLE_TOKEN_TERMS: frozenset[str] = frozenset({"captcha", "challenge"})
+_ANTI_BOT_BLOCKER_PHRASES: tuple[str, ...] = tuple(
+ term for term in _ANTI_BOT_BLOCKER_TERMS if term not in _BROAD_SINGLE_TOKEN_TERMS
+)
# Strict subset of ``_STRUCTURED_BLOCKER_KEY_TERMS`` for the flag/status rules that
# scan arbitrary code-block JSON; broad terms like ``verification`` stay string-only.
_STRICT_BLOCKER_FLAG_TERMS: frozenset[str] = frozenset(
@@ -864,6 +871,11 @@ def _looks_like_anti_bot_blocker(text: str) -> bool:
return any(term in lowered for term in _ANTI_BOT_BLOCKER_TERMS)
+def _looks_like_anti_bot_phrase(text: str) -> bool:
+ lowered = text.lower()
+ return any(phrase in lowered for phrase in _ANTI_BOT_BLOCKER_PHRASES)
+
+
def _structured_blocker_message(
value: object,
*,
@@ -871,6 +883,7 @@ def _structured_blocker_message(
include_flag_keys: bool = False,
key_terms: frozenset[str] = _STRUCTURED_BLOCKER_KEY_TERMS,
declared_keys: frozenset[str] = frozenset(),
+ scan_all_values_for_anti_bot: bool = False,
) -> str | None:
if depth > 5:
return None
@@ -884,9 +897,13 @@ def _structured_blocker_message(
has_blocker_key = normalized_key in _STRUCTURED_BLOCKER_MESSAGE_KEYS or any(
term in normalized_key for term in key_terms
)
- if has_blocker_key or (
- normalized_key in {"message", "error", "failure_reason", "reason"}
- and _looks_like_anti_bot_blocker(item)
+ if (
+ has_blocker_key
+ or (
+ normalized_key in {"message", "error", "failure_reason", "reason"}
+ and _looks_like_anti_bot_blocker(item)
+ )
+ or (scan_all_values_for_anti_bot and _looks_like_anti_bot_phrase(item))
):
return item.strip()[:240]
if include_flag_keys:
@@ -900,6 +917,7 @@ def _structured_blocker_message(
include_flag_keys=include_flag_keys,
key_terms=key_terms,
declared_keys=declared_keys,
+ scan_all_values_for_anti_bot=scan_all_values_for_anti_bot,
)
if nested:
return nested
@@ -911,6 +929,7 @@ def _structured_blocker_message(
include_flag_keys=include_flag_keys,
key_terms=key_terms,
declared_keys=declared_keys,
+ scan_all_values_for_anti_bot=scan_all_values_for_anti_bot,
)
if nested:
return nested
@@ -982,12 +1001,14 @@ def _run_blocks_structured_blocker_message(result: dict[str, Any], copilot_ctx:
if _is_code_block_type(block_type):
# Code-block outputs are arbitrary JSON the model authored: key matching
# uses the strict term set (broad terms like ``verification`` belong to
- # the page-text arms) and metadata-declared goal keys are exempt.
+ # the page-text arms) and metadata-declared goal keys are exempt. A value
+ # carrying a real anti-bot phrase is still caught regardless of its key.
blocker = _structured_blocker_message(
block.get("extracted_data"),
include_flag_keys=True,
key_terms=_STRICT_BLOCKER_FLAG_TERMS,
declared_keys=_declared_code_output_keys(copilot_ctx, block.get("label")),
+ scan_all_values_for_anti_bot=True,
)
elif block_type in _DATA_PRODUCING_BLOCK_TYPES:
payload = _block_data_payload(block.get("extracted_data"), block_type)
@@ -1574,6 +1595,7 @@ def _analyze_run_blocks(
include_flag_keys=True,
key_terms=_STRICT_BLOCKER_FLAG_TERMS,
declared_keys=declared_keys,
+ scan_all_values_for_anti_bot=True,
)
if structured_blocker:
texts_to_scan.append(structured_blocker)
diff --git a/skyvern/forge/sdk/copilot/tools/completion.py b/skyvern/forge/sdk/copilot/tools/completion.py
index ca8349778..7e57d544a 100644
--- a/skyvern/forge/sdk/copilot/tools/completion.py
+++ b/skyvern/forge/sdk/copilot/tools/completion.py
@@ -12,6 +12,7 @@ from skyvern.forge.sdk.copilot.completion_verification import (
combine_verification_results,
evaluate_completion_criteria,
grade_definition_criteria,
+ grade_present_value_criteria,
summarize_unsatisfied_outcomes,
)
from skyvern.forge.sdk.copilot.enforcement import _goal_likely_needs_more_blocks
@@ -162,6 +163,7 @@ async def _maybe_run_completion_verification_from_page_observation(
)
else:
run_result = await evaluate_completion_criteria(run_criteria, snapshot, handler)
+ run_result = _apply_present_value_upgrades(run_result, run_criteria, snapshot)
verification = combine_verification_results(criterion_ids, run_result, definition_verdicts)
if (
@@ -261,6 +263,31 @@ def _build_run_evidence_snapshot(copilot_ctx: Any, result: dict[str, Any]) -> Ru
)
+def _apply_present_value_upgrades(
+ run_result: CompletionVerificationResult,
+ run_criteria: list[CompletionCriterion],
+ snapshot: RunEvidenceSnapshot,
+) -> CompletionVerificationResult:
+ """Upgrade a ``no_evidence``/``unknown`` run verdict to a deterministic present-value
+ ``satisfied``. An ``evidence_contradicts`` verdict is left to the judge, a judge
+ ``satisfied`` is never downgraded, and an unavailable result is never fabricated.
+ """
+ if run_result.status != "evaluated":
+ return run_result
+ upgrades = {verdict.criterion_id: verdict for verdict in grade_present_value_criteria(run_criteria, snapshot)}
+ if not upgrades:
+ return run_result
+ verdicts = [
+ upgrades[verdict.criterion_id]
+ if verdict.criterion_id in upgrades and not verdict.satisfied and verdict.reason_code != "evidence_contradicts"
+ else verdict
+ for verdict in run_result.verdicts
+ ]
+ return CompletionVerificationResult(
+ status="evaluated", criterion_ids=list(run_result.criterion_ids), verdicts=verdicts
+ )
+
+
async def _maybe_run_completion_verification(
copilot_ctx: Any, result: dict[str, Any], handler_start: float
) -> CompletionVerificationResult | None:
@@ -305,6 +332,7 @@ async def _maybe_run_completion_verification(
)
else:
run_result = await evaluate_completion_criteria(run_criteria, snapshot, handler)
+ run_result = _apply_present_value_upgrades(run_result, run_criteria, snapshot)
return combine_verification_results(criterion_ids, run_result, definition_verdicts)
diff --git a/tests/unit/test_copilot_completion_criteria_lifecycle.py b/tests/unit/test_copilot_completion_criteria_lifecycle.py
index ab2c87357..134b3d913 100644
--- a/tests/unit/test_copilot_completion_criteria_lifecycle.py
+++ b/tests/unit/test_copilot_completion_criteria_lifecycle.py
@@ -26,9 +26,11 @@ from skyvern.forge.sdk.copilot.completion_criteria_store import (
from skyvern.forge.sdk.copilot.completion_verification import (
CompletionVerificationResult,
CriterionVerdict,
+ RunEvidenceSnapshot,
_coerce_result,
combine_verification_results,
grade_definition_criteria,
+ grade_present_value_criteria,
run_plane_all_no_evidence,
)
from skyvern.forge.sdk.copilot.context import CopilotContext
@@ -50,7 +52,11 @@ from skyvern.forge.sdk.copilot.request_policy import (
_parse_completion_criteria,
normalized_criterion_outcome_key,
)
-from skyvern.forge.sdk.copilot.tools.completion import _outcome_failure_warrants_repair, _split_criteria_by_plane
+from skyvern.forge.sdk.copilot.tools.completion import (
+ _apply_present_value_upgrades,
+ _outcome_failure_warrants_repair,
+ _split_criteria_by_plane,
+)
def _criterion(cid: str, outcome: str, *, level: str = "run") -> CompletionCriterion:
@@ -437,6 +443,137 @@ def test_split_criteria_by_plane() -> None:
assert [c.id for c in definition_criteria] == ["c1"]
+def _snapshot(block_outputs: dict) -> RunEvidenceSnapshot:
+ return RunEvidenceSnapshot(workflow_run_id="wr_test", block_outputs=block_outputs)
+
+
+def test_present_value_credits_quoted_currency_literal_verbatim() -> None:
+ criteria = [_criterion("c0", "the May 2026 statement total reads $4,210.55")]
+ snapshot = _snapshot({"read_statement": {"month": "May 2026", "total": "$4,210.55"}})
+ (verdict,) = grade_present_value_criteria(criteria, snapshot)
+ assert verdict.state == "satisfied"
+ assert verdict.reason_code == "present_value_verbatim"
+ assert verdict.evidence_ref == "block_outputs:read_statement"
+
+
+def test_present_value_abstains_when_literal_absent_from_outputs() -> None:
+ criteria = [_criterion("c0", "the May 2026 statement total reads $4,210.55")]
+ snapshot = _snapshot({"read_statement": {"total": "$1,000.00"}})
+ assert grade_present_value_criteria(criteria, snapshot) == []
+
+
+def test_present_value_abstains_on_empty_block_outputs() -> None:
+ criteria = [_criterion("c0", "the May 2026 statement total reads $4,210.55")]
+ assert grade_present_value_criteria(criteria, _snapshot({})) == []
+
+
+def test_present_value_abstains_on_blocked_run_without_target_literal() -> None:
+ criteria = [_criterion("c0", "the statement total reads $4,210.55")]
+ snapshot = _snapshot({"download": {"blocker": "verify you are human to continue"}})
+ assert grade_present_value_criteria(criteria, snapshot) == []
+
+
+def test_present_value_abstains_when_no_high_specificity_literal() -> None:
+ criteria = [_criterion("c0", "the invoice was downloaded successfully")]
+ snapshot = _snapshot({"download": {"file": "invoice downloaded successfully"}})
+ assert grade_present_value_criteria(criteria, snapshot) == []
+
+
+def test_present_value_does_not_credit_on_bare_year_token() -> None:
+ # A 4-digit year is too low-specificity: a coincidental output mention of the
+ # year must not credit the criterion.
+ criteria = [_criterion("c0", "the 2026 billing summary is shown")]
+ snapshot = _snapshot({"footer": {"copyright": "© 2026 Example Corp"}})
+ assert grade_present_value_criteria(criteria, snapshot) == []
+
+
+def test_present_value_credits_multi_digit_account_identifier() -> None:
+ criteria = [_criterion("c0", "Billing is open for account 100245")]
+ snapshot = _snapshot({"open_billing": {"account_number": "100245", "tab": "Billing"}})
+ (verdict,) = grade_present_value_criteria(criteria, snapshot)
+ assert verdict.state == "satisfied"
+ assert verdict.reason_code == "present_value_verbatim"
+
+
+def test_present_value_requires_every_named_literal_present() -> None:
+ # A criterion naming both a date and a total must not be credited when only the
+ # date is present: a partial match is not the named outcome.
+ criteria = [_criterion("c0", "the 2026-05-05 statement totaling $4,210.55 is downloaded")]
+ partial = _snapshot({"observe": {"statement_date": "2026-05-05"}})
+ assert grade_present_value_criteria(criteria, partial) == []
+
+ full = _snapshot({"download": {"date": "2026-05-05", "total": "$4,210.55"}})
+ (verdict,) = grade_present_value_criteria(criteria, full)
+ assert verdict.state == "satisfied"
+
+ # Both literals must land in the SAME block output, not split across two.
+ split = _snapshot({"observe": {"date": "2026-05-05"}, "summary": {"total": "$4,210.55"}})
+ assert grade_present_value_criteria(criteria, split) == []
+
+
+def test_present_value_rejects_substring_of_longer_token() -> None:
+ # Plain containment would over-credit a short/numeric literal against a sibling
+ # value; a bounded match must reject these.
+ embedded_quote = [_criterion("c0", 'the vendor is "acme"')]
+ assert grade_present_value_criteria(embedded_quote, _snapshot({"x": {"vendor": "acmecorp"}})) == []
+
+ currency = [_criterion("c0", "the total is $10")]
+ assert grade_present_value_criteria(currency, _snapshot({"x": {"total": "$100"}})) == []
+ assert grade_present_value_criteria(currency, _snapshot({"x": {"total": "$10.50"}})) == []
+
+ identifier = [_criterion("c0", "account 100245")]
+ assert grade_present_value_criteria(identifier, _snapshot({"x": {"account": "1002450"}})) == []
+ assert grade_present_value_criteria(identifier, _snapshot({"x": {"account": "9100245"}})) == []
+ assert grade_present_value_criteria(identifier, _snapshot({"x": {"ratio": "0.100245"}})) == []
+
+
+def test_present_value_credits_exact_bounded_literal() -> None:
+ currency = [_criterion("c0", "the total is $10")]
+ (verdict,) = grade_present_value_criteria(currency, _snapshot({"x": {"total": "$10"}}))
+ assert verdict.reason_code == "present_value_verbatim"
+
+ identifier = [_criterion("c0", "account 100245")]
+ (verdict2,) = grade_present_value_criteria(identifier, _snapshot({"x": {"account": "100245"}}))
+ assert verdict2.reason_code == "present_value_verbatim"
+
+
+def test_present_value_quoted_literal_specificity_floor() -> None:
+ # A 2-3 char quoted literal is too low-specificity to credit even when present
+ # verbatim (it collides with incidental prose); >=4 chars credits.
+ two_char = [_criterion("c0", 'the state is "ca"')]
+ assert grade_present_value_criteria(two_char, _snapshot({"x": {"state": "ca"}})) == []
+
+ four_char = [_criterion("c0", 'the vendor is "acme"')]
+ (verdict,) = grade_present_value_criteria(four_char, _snapshot({"x": {"vendor": "acme"}}))
+ assert verdict.reason_code == "present_value_verbatim"
+
+
+def test_present_value_upgrade_is_upgrade_only() -> None:
+ criteria = [_criterion("c0", 'the confirmation number is "ABC12345"')]
+ snapshot = _snapshot({"confirm": {"number": "ABC12345"}})
+
+ run_result = _evaluated(_verdict("c0", "unsatisfied", "no_evidence"))
+ upgraded = _apply_present_value_upgrades(run_result, criteria, snapshot)
+ assert upgraded.verdicts[0].state == "satisfied"
+ assert upgraded.verdicts[0].reason_code == "present_value_verbatim"
+
+ already_satisfied = _evaluated(_verdict("c0", "satisfied", "evidence_confirms"))
+ kept = _apply_present_value_upgrades(already_satisfied, criteria, snapshot)
+ assert kept.verdicts[0].reason_code == "evidence_confirms"
+
+ contradicts = _evaluated(_verdict("c0", "unsatisfied", "evidence_contradicts"))
+ absent_snapshot = _snapshot({"confirm": {"number": "ZZZ00000"}})
+ unchanged = _apply_present_value_upgrades(contradicts, criteria, absent_snapshot)
+ assert unchanged.verdicts[0].reason_code == "evidence_contradicts"
+
+ present_contradicts = _evaluated(_verdict("c0", "unsatisfied", "evidence_contradicts"))
+ kept_contradicts = _apply_present_value_upgrades(present_contradicts, criteria, snapshot)
+ assert kept_contradicts.verdicts[0].reason_code == "evidence_contradicts"
+
+ unavailable = CompletionVerificationResult(status="unavailable")
+ assert _apply_present_value_upgrades(unavailable, criteria, snapshot).status == "unavailable"
+
+
def test_parse_completion_criteria_coerces_level() -> None:
parsed = _parse_completion_criteria(
[
diff --git a/tests/unit/test_copilot_run_output_content_gate.py b/tests/unit/test_copilot_run_output_content_gate.py
index 137448ca8..1c3c583dd 100644
--- a/tests/unit/test_copilot_run_output_content_gate.py
+++ b/tests/unit/test_copilot_run_output_content_gate.py
@@ -502,6 +502,33 @@ def test_verification_key_counts_as_goal_content_for_code_outputs() -> None:
assert empty_data_blocks is False
+def test_anti_bot_value_under_broad_key_trips_code_block_blocker() -> None:
+ # SKY-10916: a broad/descriptive key carrying a real anti-bot phrase is caught
+ # for code outputs even though the key stays out of the strict term set.
+ result = _run_result([_code_block("check_access", {"verification": "verify you are human to continue"})])
+ assert _run_blocks_structured_blocker_message(result) == "verify you are human to continue"
+
+ status_detail = _run_result([_code_block("check_access", {"status_detail": "human verification required"})])
+ assert _run_blocks_structured_blocker_message(status_detail) == "human verification required"
+
+
+def test_descriptive_verification_key_with_benign_value_stays_exempt() -> None:
+ benign = _run_result([_code_block("plan_step", {"verification_method": "fill the login form"})])
+ assert _run_blocks_structured_blocker_message(benign) is None
+
+ boolean_flag = _run_result([_code_block("notify", {"verification_passed": True})])
+ assert _run_blocks_structured_blocker_message(boolean_flag) is None
+
+
+def test_anti_bot_value_scan_gated_on_phrases_not_bare_tokens() -> None:
+ # Extracted business text mentioning ``verification``/``challenge`` must not
+ # false-positive under the phrase-set gate.
+ result = _run_result(
+ [_code_block("summary", {"notes": "we need verification of the challenge results before May"})]
+ )
+ assert _run_blocks_structured_blocker_message(result) is None
+
+
class _MetadataCtx:
def __init__(self, metadata: dict) -> None:
self.code_artifact_metadata = metadata