Skip to content

Automate browser based workflows with AI

SKY-11357: make completion recognizer reliably-reached and authoritative (#6789)

Andrew Neilson1b696b163938
diff --git a/skyvern/forge/sdk/copilot/completion_verification.py b/skyvern/forge/sdk/copilot/completion_verification.py
index aeda47aca..f35d4b0ae 100644
--- a/skyvern/forge/sdk/copilot/completion_verification.py
+++ b/skyvern/forge/sdk/copilot/completion_verification.py
@@ -79,8 +79,19 @@ class CompletionVerificationResult:
def is_fully_satisfied(self) -> bool:
if self.status != "evaluated" or not self.criterion_ids:
return False
- satisfied = {verdict.criterion_id for verdict in self.verdicts if verdict.satisfied}
- return all(criterion_id in satisfied for criterion_id in self.criterion_ids)
+ verdict_by_id = {verdict.criterion_id: verdict for verdict in self.verdicts}
+ satisfied_count = 0
+ for criterion_id in self.criterion_ids:
+ verdict = verdict_by_id.get(criterion_id)
+ if verdict is not None and verdict.satisfied:
+ satisfied_count += 1
+ continue
+ # A definition-plane ``unknown`` is a YAML-grader abstention, not a refutation,
+ # so it must not veto a run whose observable outcome evidence is fully confirmed.
+ if verdict is not None and _is_definition_plane_abstention(verdict):
+ continue
+ return False
+ return satisfied_count > 0
def to_trace_data(self) -> dict[str, Any]:
unmet = [verdict for verdict in self.verdicts if not verdict.satisfied]
@@ -389,12 +400,12 @@ def grade_definition_criteria(criteria: list[CompletionCriterion], workflow_yaml
)
continue
defined, referenced = reference_state
+ named = _named_definition_inputs(criterion.outcome)
if defined and referenced:
# When the criterion names specific inputs, each must match a defined
# parameter key; one stray parameter must not satisfy a multi-input ask.
# An unmatchable name degrades to unknown, not unsatisfied — extraction
# is heuristic and a false negative would drive repair of correct YAML.
- named = _named_definition_inputs(criterion.outcome)
normalized_keys = [_normalize_input_phrase(key) for key in defined]
if named and not all(_input_matches_any_key(candidate, normalized_keys) for candidate in named):
verdicts.append(
@@ -411,6 +422,13 @@ def grade_definition_criteria(criteria: list[CompletionCriterion], workflow_yaml
evidence_ref="workflow_yaml:" + ",".join(referenced[:8]),
)
)
+ elif not named:
+ # A reusable-inputs hint with no specific named inputs cannot be proven
+ # false: a read-only/validation workflow legitimately defines none, so a
+ # missing/unreferenced parameter set abstains rather than sinking the run.
+ verdicts.append(
+ CriterionVerdict(criterion_id=criterion.id, state="unknown", reason_code="definition_parameters_absent")
+ )
elif defined:
verdicts.append(
CriterionVerdict(
@@ -434,18 +452,25 @@ _ISO_DATE_LITERAL_RE = re.compile(r"\b\d{4}-\d{2}-\d{2}\b")
# A bare digit run qualifies as an identifier only at >=5 digits: a 4-digit run is
# usually a year and too low-specificity to credit on a coincidental output match.
_LONG_DIGIT_LITERAL_RE = re.compile(r"\b\d{5,}\b")
+# A mixed alphanumeric code (e.g. WTR-1842-DEMO, ABC12345) credits only via the
+# letter+digit, >=6 alnum filter below, which excludes bare words and bare years.
+_STRUCTURED_ID_LITERAL_RE = re.compile(r"\b[A-Za-z0-9]+(?:[-_][A-Za-z0-9]+)*\b")
def _normalize_present_value(text: str) -> str:
return " ".join(text.casefold().split())
+def _is_structured_identifier(token: str) -> bool:
+ alnum = [ch for ch in token if ch.isalnum()]
+ return len(alnum) >= 6 and any(ch.isalpha() for ch in alnum) and any(ch.isdigit() for ch in alnum)
+
+
def _extract_present_value_literals(outcome: str) -> list[str]:
- literals: list[str] = []
- for match in _QUOTED_LITERAL_RE.finditer(outcome):
- literals.append(match.group(1))
+ literals: list[str] = [match.group(1) for match in _QUOTED_LITERAL_RE.finditer(outcome)]
for pattern in (_CURRENCY_LITERAL_RE, _ISO_DATE_LITERAL_RE, _LONG_DIGIT_LITERAL_RE):
literals.extend(pattern.findall(outcome))
+ literals.extend(token for token in _STRUCTURED_ID_LITERAL_RE.findall(outcome) if _is_structured_identifier(token))
normalized: list[str] = []
seen: set[str] = set()
for literal in literals:
@@ -893,6 +918,10 @@ def _is_meaningful_record_value(value: Any) -> bool:
_DEFINITION_REASON_PREFIX = "definition_"
+def _is_definition_plane_abstention(verdict: CriterionVerdict) -> bool:
+ return verdict.state == "unknown" and verdict.reason_code.startswith(_DEFINITION_REASON_PREFIX)
+
+
def run_plane_all_no_evidence(result: CompletionVerificationResult) -> bool:
"""Whether every run-plane verdict in an evaluated result is ``no_evidence``.
diff --git a/skyvern/forge/sdk/copilot/config.py b/skyvern/forge/sdk/copilot/config.py
index 5c2af53c6..4398a6617 100644
--- a/skyvern/forge/sdk/copilot/config.py
+++ b/skyvern/forge/sdk/copilot/config.py
@@ -126,24 +126,6 @@ POST_SUSPICIOUS_SUCCESS_NUDGE = (
"status alone. Verify the actual extracted data answers the user's question."
)
-POST_REPEATED_NULL_DATA_NUDGE = (
- "STOP — you have now produced multiple consecutive test runs where "
- "extraction/text_prompt blocks returned all-null or empty data. "
- "Re-prompting the extractor is not working — the problem is almost "
- "certainly NOT how the extraction goal is worded.\n"
- "You MUST now do ONE of the following before another update_workflow call:\n"
- "1. Call get_browser_screenshot on the workflow's browser session to see "
- "exactly what page the workflow is actually loading (it may differ from "
- "what you expect — e.g. a 'no results' fallback, cookie wall, or bot block).\n"
- "2. Call evaluate with JavaScript that searches for the expected content "
- "on the workflow's browser — confirm whether the data is even present.\n"
- "3. If the page the workflow loads genuinely does not contain the data, "
- "pivot to a different URL or source entirely — do NOT keep retrying "
- "extraction against the same failing page.\n"
- "Do NOT call update_and_run_blocks again until you have concrete evidence "
- "about what the workflow browser is actually seeing."
-)
-
POST_REPEATED_FRONTIER_FAILURE_WARN_NUDGE = (
"STOP — this is the second run with the same frontier and the same failure "
"signature. Re-running the same change again is unlikely to help.\n"
@@ -349,7 +331,6 @@ DEFAULT_ENFORCEMENT_NUDGES: dict[str, str] = {
"post_failed_test_inspect_first": POST_FAILED_TEST_INSPECT_FIRST_NUDGE,
"post_explore_without_workflow": POST_EXPLORE_WITHOUT_WORKFLOW_NUDGE,
"post_suspicious_success": POST_SUSPICIOUS_SUCCESS_NUDGE,
- "post_repeated_null_data": POST_REPEATED_NULL_DATA_NUDGE,
"post_repeated_frontier_failure_warn": POST_REPEATED_FRONTIER_FAILURE_WARN_NUDGE,
"post_repeated_frontier_failure_stop": POST_REPEATED_FRONTIER_FAILURE_STOP_NUDGE,
"post_parameter_binding_warn": POST_PARAMETER_BINDING_WARN_NUDGE,
diff --git a/skyvern/forge/sdk/copilot/context.py b/skyvern/forge/sdk/copilot/context.py
index 545b0d966..06f30869d 100644
--- a/skyvern/forge/sdk/copilot/context.py
+++ b/skyvern/forge/sdk/copilot/context.py
@@ -432,11 +432,6 @@ class CopilotContext(AgentContext):
last_run_blocks_workflow_run_id: str | None = None
last_successful_run_blocks_workflow_run_id: str | None = None
last_outcome_gate_workflow_run_id: str | None = None
- # Consecutive test runs whose data-producing blocks completed with no
- # meaningful output (missing, empty, or all-null fields). Resets when a
- # run produces real data. Used to escalate when the agent is stuck
- # retrying extraction against a page that doesn't contain the data.
- null_data_streak_count: int = 0
# Consecutive failed runs where navigation completed but the scraper
# could not read the page (generic "failed to load the website" template).
# Resets on any non-matching run outcome. Streak crosses workflow-shape
diff --git a/skyvern/forge/sdk/copilot/diagnosis_repair_contract.py b/skyvern/forge/sdk/copilot/diagnosis_repair_contract.py
index 563984813..688198c4c 100644
--- a/skyvern/forge/sdk/copilot/diagnosis_repair_contract.py
+++ b/skyvern/forge/sdk/copilot/diagnosis_repair_contract.py
@@ -573,7 +573,6 @@ def _prior_repair_attempts(ctx: Any) -> dict[str, int]:
keys = (
"repeated_failure_streak_count",
"failed_test_nudge_count",
- "null_data_streak_count",
"probable_site_block_streak_count",
"per_tool_budget_nudge_count",
"repeated_action_fingerprint_streak_count",
diff --git a/skyvern/forge/sdk/copilot/enforcement.py b/skyvern/forge/sdk/copilot/enforcement.py
index e9dc26e30..1ad3fe6b1 100644
--- a/skyvern/forge/sdk/copilot/enforcement.py
+++ b/skyvern/forge/sdk/copilot/enforcement.py
@@ -45,7 +45,6 @@ from skyvern.forge.sdk.copilot.config import (
POST_PROBABLE_SITE_BLOCK_STOP_NUDGE,
POST_REPEATED_FRONTIER_FAILURE_STOP_NUDGE,
POST_REPEATED_FRONTIER_FAILURE_WARN_NUDGE,
- POST_REPEATED_NULL_DATA_NUDGE,
POST_SUSPICIOUS_SUCCESS_NUDGE,
POST_UPDATE_NUDGE,
PRE_DISCOVERY_URL_QUESTION_NUDGE,
@@ -114,9 +113,6 @@ MAX_EXPLORE_WITHOUT_WORKFLOW_NUDGES = 2
# correctly diagnosed an unrecoverable block (anti-bot, paywall) and is no
# longer willing to re-run extraction.
MAX_SUSPICIOUS_SUCCESS_NUDGES = 2
-# Escalate after this many consecutive all-null extraction runs so the agent
-# inspects browser state instead of re-prompting the extractor.
-NULL_DATA_STREAK_ESCALATE_AT = 2
# Streak levels for repeated-failure (same frontier + same failure signature).
REPEATED_FRONTIER_STREAK_ESCALATE_AT = 2
REPEATED_FRONTIER_STREAK_STOP_AT = 3
@@ -1171,17 +1167,6 @@ def _needs_probable_site_block_stop_nudge(ctx: Any) -> bool:
return _get_int(ctx, "probable_site_block_stop_nudge_count") < MAX_PROBABLE_SITE_BLOCK_STOP_NUDGES
-def _needs_repeated_null_data_nudge(ctx: Any) -> bool:
- """Return True when suspicious-success has happened enough times to escalate."""
- # Same as above: non-retriable nav state never belongs on this branch.
- if getattr(ctx, "last_test_non_retriable_nav_error", None):
- return False
- if not getattr(ctx, "last_test_suspicious_success", False):
- return False
- streak = getattr(ctx, "null_data_streak_count", 0)
- return streak >= NULL_DATA_STREAK_ESCALATE_AT
-
-
def _get_int(ctx: Any, name: str, default: int = 0) -> int:
value = getattr(ctx, name, default)
return value if isinstance(value, int) else default
@@ -1334,9 +1319,6 @@ def _check_enforcement(
# Do NOT clear last_test_suspicious_success here. tools._record_run_blocks_result
# resets it on every new run; if the agent ignores the nudge and answers
# without rerunning, we want _check_enforcement to re-emit the nudge.
- if _needs_repeated_null_data_nudge(ctx):
- return _nudge(config, "post_repeated_null_data")
-
if _needs_suspicious_success_nudge(ctx):
ctx.suspicious_success_nudge_count = getattr(ctx, "suspicious_success_nudge_count", 0) + 1
return _nudge(config, "post_suspicious_success")
@@ -1657,7 +1639,6 @@ _NUDGE_TYPE_BY_MESSAGE: dict[str, str] = {
POST_NAVIGATE_NUDGE: "post_navigate",
POST_EXPLORE_WITHOUT_WORKFLOW_NUDGE: "explore_without_workflow",
POST_SUSPICIOUS_SUCCESS_NUDGE: "suspicious_success",
- POST_REPEATED_NULL_DATA_NUDGE: "repeated_null_data",
POST_REPEATED_FRONTIER_FAILURE_WARN_NUDGE: "repeated_frontier_failure_warn",
POST_REPEATED_FRONTIER_FAILURE_STOP_NUDGE: "repeated_frontier_failure_stop",
POST_NON_RETRIABLE_NAV_ERROR_STOP_NUDGE: "non_retriable_nav_error_stop",
@@ -1681,7 +1662,6 @@ _NUDGE_TYPE_BY_KEY: dict[str, str] = {
"post_navigate": "post_navigate",
"post_explore_without_workflow": "explore_without_workflow",
"post_suspicious_success": "suspicious_success",
- "post_repeated_null_data": "repeated_null_data",
"post_repeated_frontier_failure_warn": "repeated_frontier_failure_warn",
"post_repeated_frontier_failure_stop": "repeated_frontier_failure_stop",
"post_non_retriable_nav_error_stop": "non_retriable_nav_error_stop",
diff --git a/skyvern/forge/sdk/copilot/runtime.py b/skyvern/forge/sdk/copilot/runtime.py
index f2c121b31..c74f62752 100644
--- a/skyvern/forge/sdk/copilot/runtime.py
+++ b/skyvern/forge/sdk/copilot/runtime.py
@@ -184,7 +184,6 @@ class AgentContext:
copilot_total_timeout_exceeded: bool = False
failed_test_nudge_count: int = 0
explore_without_workflow_nudge_count: int = 0
- null_data_streak_count: int = 0
last_test_ok: bool | None = None
last_test_suspicious_success: bool = False
last_test_anti_bot: str | None = None
diff --git a/skyvern/forge/sdk/copilot/tools/completion.py b/skyvern/forge/sdk/copilot/tools/completion.py
index 365bd21b8..3e29b7d84 100644
--- a/skyvern/forge/sdk/copilot/tools/completion.py
+++ b/skyvern/forge/sdk/copilot/tools/completion.py
@@ -267,19 +267,18 @@ def _is_outcome_evidence_candidate(copilot_ctx: Any, result: dict[str, Any]) ->
"""A clean ok=True run worth judging on its whole-workflow outcome.
Recognition is governed by the outcome evidence the user can observe, not by
- whether every block was verified as an end-to-end prefix (SKY-10576). The judge
- requires positive evidence for every criterion, and ``empty_data_blocks`` rejects
- a run whose outcome block produced nothing, so an incomplete run never passes --
- this only lets an already-reached goal be recognized without a redundant
- full-prefix re-run.
+ whether the run produced any data. The judge requires positive evidence for
+ every criterion, so a genuinely-empty run still grades unsatisfied; only a
+ terminal anti-bot blocker keeps a completed run out of the judge, because that
+ evidence makes the apparent success unusable.
"""
if not bool(result.get("ok", False)):
return False
structured_blocker = _run_blocks_structured_blocker_message(result, copilot_ctx)
- anti_bot, empty_data_blocks, _categories = _analyze_run_blocks(result, copilot_ctx)
+ anti_bot, _empty_data_blocks, _categories = _analyze_run_blocks(result, copilot_ctx)
if structured_blocker and (anti_bot or _looks_like_anti_bot_blocker(structured_blocker)):
return False
- return not empty_data_blocks
+ return True
def _is_unfinished_run_verification_candidate(copilot_ctx: Any, result: dict[str, Any]) -> bool:
diff --git a/skyvern/forge/sdk/copilot/tools/run_execution.py b/skyvern/forge/sdk/copilot/tools/run_execution.py
index fb91b5983..f961c6311 100644
--- a/skyvern/forge/sdk/copilot/tools/run_execution.py
+++ b/skyvern/forge/sdk/copilot/tools/run_execution.py
@@ -82,6 +82,7 @@ from skyvern.forge.sdk.copilot.runtime import (
AgentContext,
ensure_browser_session,
)
+from skyvern.forge.sdk.copilot.terminal_predicates import outcome_fully_verified
from skyvern.forge.sdk.copilot.tracing_setup import copilot_span
from skyvern.forge.sdk.copilot.turn_halt import (
stash_repair_ceiling_turn_halt,
@@ -2110,6 +2111,9 @@ def _record_run_blocks_result(
if run_ok:
_mark_page_inspected(copilot_ctx)
+ completion_verification_evaluated = (
+ completion_verification is not None and completion_verification.status == "evaluated"
+ )
completion_fully_satisfied = (
completion_verification is not None
and completion_verification.status == "evaluated"
@@ -2146,12 +2150,10 @@ def _record_run_blocks_result(
copilot_ctx.verified_terminal_proposal_ready = True
copilot_ctx.last_test_suspicious_success = False
copilot_ctx.last_test_failure_reason = None
- copilot_ctx.null_data_streak_count = 0
copilot_ctx.suspicious_success_nudge_count = 0
- if empty_data_blocks and not completion_fully_satisfied:
+ if empty_data_blocks and not completion_verification_evaluated:
copilot_ctx.last_test_ok = None
copilot_ctx.last_test_suspicious_success = True
- copilot_ctx.null_data_streak_count = getattr(copilot_ctx, "null_data_streak_count", 0) + 1
copilot_ctx.last_test_failure_reason = (
"All blocks completed but data-producing blocks "
"produced no meaningful output "
@@ -2189,7 +2191,6 @@ def _record_run_blocks_result(
data.setdefault("failure_reason", outcome_unverified_reason)
else:
copilot_ctx.failed_test_nudge_count = 0
- copilot_ctx.null_data_streak_count = 0
copilot_ctx.probable_site_block_streak_count = 0
copilot_ctx.last_failed_workflow_yaml = None
# Real success: clear the signature latch so a subsequent bad URL in
@@ -2215,6 +2216,21 @@ def _record_run_blocks_result(
_update_verification_evidence_from_run_result(copilot_ctx, result)
return _stash_recorded_run_outcome(copilot_ctx, _adjudicated_run_outcome(copilot_ctx, completion_verification))
+ if outcome_fully_verified(copilot_ctx):
+ copilot_ctx.last_test_suspicious_success = False
+ copilot_ctx.last_test_failure_reason = None
+ copilot_ctx.suspicious_success_nudge_count = 0
+ copilot_ctx.failed_test_nudge_count = 0
+ copilot_ctx.probable_site_block_streak_count = 0
+ copilot_ctx.last_failed_workflow_yaml = None
+ copilot_ctx.last_full_workflow_test_ok = True
+ copilot_ctx.last_unverified_block_labels = []
+ copilot_ctx.last_good_workflow = copilot_ctx.last_workflow
+ copilot_ctx.last_good_workflow_yaml = copilot_ctx.last_workflow_yaml
+ update_repeated_failure_state(copilot_ctx, result)
+ _update_verification_evidence_from_run_result(copilot_ctx, result)
+ return _stash_recorded_run_outcome(copilot_ctx, _adjudicated_run_outcome(copilot_ctx, completion_verification))
+
copilot_ctx.last_failed_workflow_yaml = getattr(copilot_ctx, "workflow_yaml", None)
copilot_ctx.last_test_non_retriable_nav_error = _detect_non_retriable_nav_error(result)
if _detect_probable_site_block_wall(result):
diff --git a/tests/unit/test_copilot_completion_criteria_lifecycle.py b/tests/unit/test_copilot_completion_criteria_lifecycle.py
index 134b3d913..0291595c0 100644
--- a/tests/unit/test_copilot_completion_criteria_lifecycle.py
+++ b/tests/unit/test_copilot_completion_criteria_lifecycle.py
@@ -356,7 +356,7 @@ def test_definition_grading_satisfied_when_parameters_exist_and_are_referenced()
def test_definition_grading_unsatisfied_when_parameters_missing_or_unreferenced() -> None:
- criteria = [_criterion("c0", "the workflow accepts reusable inputs", level="definition")]
+ criteria = [_criterion("c0", "the workflow accepts first name and NPI as reusable inputs", level="definition")]
(missing,) = grade_definition_criteria(criteria, _NO_PARAMS_YAML)
assert missing.state == "unsatisfied"
assert missing.reason_code == "definition_parameters_missing"
@@ -365,6 +365,16 @@ def test_definition_grading_unsatisfied_when_parameters_missing_or_unreferenced(
assert unreferenced.reason_code == "definition_parameters_unreferenced"
+def test_definition_grading_abstains_when_no_specific_inputs_named() -> None:
+ criteria = [_criterion("c0", "the workflow accepts reusable inputs", level="definition")]
+ (no_params,) = grade_definition_criteria(criteria, _NO_PARAMS_YAML)
+ assert no_params.state == "unknown"
+ assert no_params.reason_code == "definition_parameters_absent"
+ (unreferenced,) = grade_definition_criteria(criteria, _UNREFERENCED_YAML)
+ assert unreferenced.state == "unknown"
+ assert unreferenced.reason_code == "definition_parameters_absent"
+
+
def test_definition_grading_requires_every_named_input_to_match() -> None:
criteria = [
_criterion(
@@ -557,21 +567,48 @@ def test_present_value_upgrade_is_upgrade_only() -> None:
assert upgraded.verdicts[0].state == "satisfied"
assert upgraded.verdicts[0].reason_code == "present_value_verbatim"
- already_satisfied = _evaluated(_verdict("c0", "satisfied", "evidence_confirms"))
- kept = _apply_present_value_upgrades(already_satisfied, criteria, snapshot)
- assert kept.verdicts[0].reason_code == "evidence_confirms"
- contradicts = _evaluated(_verdict("c0", "unsatisfied", "evidence_contradicts"))
- absent_snapshot = _snapshot({"confirm": {"number": "ZZZ00000"}})
- unchanged = _apply_present_value_upgrades(contradicts, criteria, absent_snapshot)
- assert unchanged.verdicts[0].reason_code == "evidence_contradicts"
+def test_present_value_credits_unquoted_structured_identifier() -> None:
+ criteria = [_criterion("c0", "the submitted request returns confirmation number WTR-1842-DEMO")]
+ snapshot = _snapshot({"submit_request": {"confirmation_number": "WTR-1842-DEMO", "status": "submitted"}})
+ (verdict,) = grade_present_value_criteria(criteria, snapshot)
+ assert verdict.state == "satisfied"
+ assert verdict.reason_code == "present_value_verbatim"
+ assert verdict.evidence_ref == "block_outputs:submit_request"
- present_contradicts = _evaluated(_verdict("c0", "unsatisfied", "evidence_contradicts"))
- kept_contradicts = _apply_present_value_upgrades(present_contradicts, criteria, snapshot)
- assert kept_contradicts.verdicts[0].reason_code == "evidence_contradicts"
- unavailable = CompletionVerificationResult(status="unavailable")
- assert _apply_present_value_upgrades(unavailable, criteria, snapshot).status == "unavailable"
+def test_present_value_structured_identifier_abstains_when_absent() -> None:
+ criteria = [_criterion("c0", "the submitted request returns confirmation number WTR-1842-DEMO")]
+ snapshot = _snapshot({"submit_request": {"confirmation_number": "WTR-9999-PROD"}})
+ assert grade_present_value_criteria(criteria, snapshot) == []
+
+
+def test_present_value_structured_identifier_requires_letter_and_digit() -> None:
+ # A bare word or a bare year-length token is not a high-specificity identifier.
+ plain_word = [_criterion("c0", "the page shows submitted")]
+ assert grade_present_value_criteria(plain_word, _snapshot({"x": {"state": "submitted"}})) == []
+
+ hyphen_word = [_criterion("c0", "the request is read-only")]
+ assert grade_present_value_criteria(hyphen_word, _snapshot({"x": {"mode": "read-only"}})) == []
+
+
+def test_present_value_structured_identifier_rejects_substring_of_longer_token() -> None:
+ criteria = [_criterion("c0", "confirmation number WTR-1842-DEMO")]
+ assert grade_present_value_criteria(criteria, _snapshot({"x": {"number": "WTR-1842-DEMO2"}})) == []
+
+
+def test_present_value_structured_identifier_upgrade_only() -> None:
+ criteria = [_criterion("c0", "confirmation number WTR-1842-DEMO")]
+ snapshot = _snapshot({"confirm": {"number": "WTR-1842-DEMO"}})
+
+ judge_unknown = _evaluated(_verdict("c0", "unknown", "unknown"))
+ upgraded = _apply_present_value_upgrades(judge_unknown, criteria, snapshot)
+ assert upgraded.verdicts[0].state == "satisfied"
+ assert upgraded.verdicts[0].reason_code == "present_value_verbatim"
+
+ contradicts = _evaluated(_verdict("c0", "unsatisfied", "evidence_contradicts"))
+ kept = _apply_present_value_upgrades(contradicts, criteria, snapshot)
+ assert kept.verdicts[0].reason_code == "evidence_contradicts"
def test_parse_completion_criteria_coerces_level() -> None:
diff --git a/tests/unit/test_copilot_completion_verification.py b/tests/unit/test_copilot_completion_verification.py
index 124d9cc10..ab0f8e835 100644
--- a/tests/unit/test_copilot_completion_verification.py
+++ b/tests/unit/test_copilot_completion_verification.py
@@ -183,6 +183,38 @@ def test_is_fully_satisfied_requires_every_criterion() -> None:
assert _evaluated(("c0", True), ("c1", False)).is_fully_satisfied() is False
+def _mixed(*verdicts: CriterionVerdict) -> CompletionVerificationResult:
+ return CompletionVerificationResult(
+ status="evaluated", criterion_ids=[v.criterion_id for v in verdicts], verdicts=list(verdicts)
+ )
+
+
+def test_definition_plane_abstention_does_not_sink_evidence_confirmed_run() -> None:
+ definition_unknown = CriterionVerdict(
+ criterion_id="c0", state="unknown", reason_code="definition_parameters_absent"
+ )
+ confirmed = CriterionVerdict(criterion_id="c1", state="satisfied", reason_code="evidence_confirms")
+ assert _mixed(definition_unknown, confirmed).is_fully_satisfied() is True
+
+
+def test_run_plane_unknown_still_blocks() -> None:
+ run_unknown = CriterionVerdict(criterion_id="c0", state="unknown", reason_code="unknown")
+ confirmed = CriterionVerdict(criterion_id="c1", state="satisfied", reason_code="evidence_confirms")
+ assert _mixed(run_unknown, confirmed).is_fully_satisfied() is False
+
+
+def test_all_definition_abstentions_do_not_vacuously_satisfy() -> None:
+ abstain_a = CriterionVerdict(criterion_id="c0", state="unknown", reason_code="definition_unknown")
+ abstain_b = CriterionVerdict(criterion_id="c1", state="unknown", reason_code="definition_parameters_absent")
+ assert _mixed(abstain_a, abstain_b).is_fully_satisfied() is False
+
+
+def test_definition_plane_unsatisfied_still_blocks() -> None:
+ unreferenced = CriterionVerdict(criterion_id="c0", state="unsatisfied", reason_code="definition_parameters_missing")
+ confirmed = CriterionVerdict(criterion_id="c1", state="satisfied", reason_code="evidence_confirms")
+ assert _mixed(unreferenced, confirmed).is_fully_satisfied() is False
+
+
def test_empty_verdicts_with_criteria_is_not_vacuously_satisfied() -> None:
result = CompletionVerificationResult(status="evaluated", criterion_ids=["c0"], verdicts=[])
assert result.is_fully_satisfied() is False
@@ -901,6 +933,22 @@ def test_record_run_blocks_downgrades_on_contradiction_without_confirmation_bloc
assert ctx.last_full_workflow_test_ok is False
+def test_record_run_blocks_demonstrated_when_lone_definition_abstention_with_confirmed_run() -> None:
+ ctx = _ctx_with_blocks("extraction")
+ verification = _mixed(
+ CriterionVerdict(criterion_id="c0", state="unknown", reason_code="definition_parameters_absent"),
+ CriterionVerdict(criterion_id="c1", state="satisfied", reason_code="evidence_confirms"),
+ )
+
+ recorded = _record_run_blocks_result(ctx, _clean_success_result(), completion_verification=verification)
+
+ assert recorded is not None
+ assert recorded.verdict == "demonstrated"
+ assert ctx.last_test_suspicious_success is False
+ assert ctx.last_full_workflow_test_ok is True
+ assert verified_goal_satisfied_context(ctx) is True
+
+
def _goto_only_result() -> dict:
return {
"ok": True,
@@ -1786,8 +1834,9 @@ def test_outcome_evidence_candidate_admits_clean_run_despite_unverified_prefix()
# A clean run is admitted for the judge even though b0/b1 are not in the verified
# prefix -- recognition is governed by the outcome judge, not the per-block prefix.
assert _is_outcome_evidence_candidate(ctx, _clean_success_result()) is True
- # It still rejects empty-data runs (no clean outcome to judge) and ok=False runs.
- assert _is_outcome_evidence_candidate(ctx, _empty_data_result()) is False
+ # An empty-data completed run is admitted for the judge (the judge requires positive
+ # evidence per criterion, so it grades unsatisfied); only ok=False runs are rejected.
+ assert _is_outcome_evidence_candidate(ctx, _empty_data_result()) is True
assert _is_outcome_evidence_candidate(ctx, {"ok": False, "data": {}}) is False
diff --git a/tests/unit/test_copilot_enforcement_hardening.py b/tests/unit/test_copilot_enforcement_hardening.py
index 5cfe76032..e2311fc97 100644
--- a/tests/unit/test_copilot_enforcement_hardening.py
+++ b/tests/unit/test_copilot_enforcement_hardening.py
@@ -218,7 +218,6 @@ def test_suspicious_success_nudge_refires_on_subsequent_turn() -> None:
ctx = _fresh_context()
ctx.last_test_ok = None
ctx.last_test_suspicious_success = True
- ctx.null_data_streak_count = 1
first = _check_enforcement(ctx)
assert first == POST_SUSPICIOUS_SUCCESS_NUDGE
diff --git a/tests/unit/test_copilot_enforcement_pruning.py b/tests/unit/test_copilot_enforcement_pruning.py
index 70280fb3a..bbd66b3b2 100644
--- a/tests/unit/test_copilot_enforcement_pruning.py
+++ b/tests/unit/test_copilot_enforcement_pruning.py
@@ -19,12 +19,9 @@ from skyvern.forge.sdk.copilot.blocker_signal import CopilotToolBlockerSignal
from skyvern.forge.sdk.copilot.completion_verification import CompletionVerificationResult, CriterionVerdict
from skyvern.forge.sdk.copilot.enforcement import (
KEEP_RECENT_TOOL_OUTPUTS,
- NULL_DATA_STREAK_ESCALATE_AT,
- POST_REPEATED_NULL_DATA_NUDGE,
POST_SUSPICIOUS_SUCCESS_NUDGE,
CopilotGoalSatisfied,
_check_enforcement,
- _needs_repeated_null_data_nudge,
_needs_suspicious_success_nudge,
_prune_input_list,
_summarize_tool_output,
@@ -64,7 +61,6 @@ class _Ctx:
self.last_failure_category_top = None
self.failed_test_nudge_count = 0
self.explore_without_workflow_nudge_count = 0
- self.null_data_streak_count = 0
self.repeated_failure_streak_count = 0
self.repeated_failure_nudge_emitted_at_streak = 0
self.verified_terminal_proposal_ready = False
@@ -335,7 +331,6 @@ def _fresh_ctx_for_record() -> SimpleNamespace:
last_test_anti_bot=None,
last_failure_category_top=None,
last_test_non_retriable_nav_error=None,
- null_data_streak_count=0,
failed_test_nudge_count=0,
last_failed_workflow_yaml=None,
last_good_workflow=None,
@@ -610,49 +605,23 @@ def test_record_run_blocks_result_sets_last_test_ok_none_on_watchdog_cancel_at_t
# ---------------------------------------------------------------------------
-# Repeated null-data escalation
+# Suspicious-success nudge
# ---------------------------------------------------------------------------
def test_suspicious_success_fires_when_flag_set() -> None:
ctx = _Ctx()
ctx.last_test_suspicious_success = True
- ctx.null_data_streak_count = 1
assert _needs_suspicious_success_nudge(ctx) is True
- assert _needs_repeated_null_data_nudge(ctx) is False
-def test_repeated_null_data_fires_at_threshold() -> None:
+def test_check_enforcement_returns_suspicious_nudge_when_flag_set() -> None:
ctx = _Ctx()
ctx.last_test_suspicious_success = True
- ctx.null_data_streak_count = NULL_DATA_STREAK_ESCALATE_AT
- assert _needs_repeated_null_data_nudge(ctx) is True
-
-
-def test_check_enforcement_returns_repeated_nudge_at_threshold() -> None:
- ctx = _Ctx()
- ctx.last_test_suspicious_success = True
- ctx.null_data_streak_count = NULL_DATA_STREAK_ESCALATE_AT
- nudge = _check_enforcement(ctx)
- assert nudge == POST_REPEATED_NULL_DATA_NUDGE
-
-
-def test_check_enforcement_returns_regular_suspicious_nudge_below_threshold() -> None:
- ctx = _Ctx()
- ctx.last_test_suspicious_success = True
- ctx.null_data_streak_count = 1
nudge = _check_enforcement(ctx)
assert nudge == POST_SUSPICIOUS_SUCCESS_NUDGE
-def test_repeated_null_data_requires_suspicious_flag() -> None:
- # If the current test wasn't a suspicious success, don't fire even with a high streak.
- ctx = _Ctx()
- ctx.last_test_suspicious_success = False
- ctx.null_data_streak_count = 99
- assert _needs_repeated_null_data_nudge(ctx) is False
-
-
# ---------------------------------------------------------------------------
# Tool-output pruning
# ---------------------------------------------------------------------------
diff --git a/tests/unit/test_copilot_non_retriable_nav.py b/tests/unit/test_copilot_non_retriable_nav.py
index 54484227f..fadfc665a 100644
--- a/tests/unit/test_copilot_non_retriable_nav.py
+++ b/tests/unit/test_copilot_non_retriable_nav.py
@@ -22,7 +22,6 @@ from skyvern.forge.sdk.copilot.enforcement import (
_extract_url_from_nav_error,
_maybe_raise_non_retriable_nav,
_needs_failed_test_nudge,
- _needs_repeated_null_data_nudge,
_needs_suspicious_success_nudge,
_non_retriable_nav_error_nudge,
_repeated_frontier_failure_nudge,
@@ -387,14 +386,6 @@ def test_needs_suspicious_success_nudge_suppressed_when_flag_set() -> None:
assert _needs_suspicious_success_nudge(ctx) is False
-def test_needs_repeated_null_data_nudge_suppressed_when_flag_set() -> None:
- ctx = _fresh_context()
- ctx.last_test_suspicious_success = True
- ctx.null_data_streak_count = 5
- ctx.last_test_non_retriable_nav_error = _DNS_FAILURE_REASON
- assert _needs_repeated_null_data_nudge(ctx) is False
-
-
def test_repeated_frontier_failure_nudge_suppressed_when_flag_set() -> None:
ctx = _fresh_context()
ctx.repeated_failure_streak_count = 5
@@ -492,18 +483,16 @@ def test_all_competing_branches_silent_after_latch() -> None:
# Reproduce the full steady state after the one-shot stop nudge has
# latched: ctx has a non-retriable error, last_test_ok=False,
# test_after_update_done=True, and all counters are set such that the
- # other branches would normally fire. Assert all four helpers are silent.
+ # other branches would normally fire. Assert each helper is silent.
ctx = _fresh_context()
ctx.test_after_update_done = True
ctx.last_test_ok = False
ctx.last_test_suspicious_success = True
- ctx.null_data_streak_count = 5
ctx.repeated_failure_streak_count = 5
ctx.last_test_non_retriable_nav_error = _DNS_FAILURE_REASON
assert _needs_failed_test_nudge(ctx) is False
assert _needs_suspicious_success_nudge(ctx) is False
- assert _needs_repeated_null_data_nudge(ctx) is False
assert _repeated_frontier_failure_nudge(ctx) is None
diff --git a/tests/unit/test_copilot_response_coverage_gate.py b/tests/unit/test_copilot_response_coverage_gate.py
index 683b17e63..a9aae5951 100644
--- a/tests/unit/test_copilot_response_coverage_gate.py
+++ b/tests/unit/test_copilot_response_coverage_gate.py
@@ -58,7 +58,6 @@ class _Ctx:
self.last_test_anti_bot = None
self.failed_test_nudge_count = 0
self.explore_without_workflow_nudge_count = 0
- self.null_data_streak_count = 0
self.repeated_failure_streak_count = 0
self.repeated_failure_nudge_emitted_at_streak = 0
self.last_artifact_health_blocker_reason = None
diff --git a/tests/unit/test_copilot_run_output_content_gate.py b/tests/unit/test_copilot_run_output_content_gate.py
index 099eddced..7554a799e 100644
--- a/tests/unit/test_copilot_run_output_content_gate.py
+++ b/tests/unit/test_copilot_run_output_content_gate.py
@@ -16,6 +16,7 @@ from skyvern.forge.sdk.copilot.completion_verification import CompletionVerifica
from skyvern.forge.sdk.copilot.context import CopilotContext
from skyvern.forge.sdk.copilot.enforcement import verified_goal_satisfied_context
from skyvern.forge.sdk.copilot.request_policy import CompletionCriterion, RequestPolicy
+from skyvern.forge.sdk.copilot.terminal_predicates import outcome_fully_verified
from skyvern.forge.sdk.copilot.tools import (
_analyze_run_blocks,
_current_workflow_has_evidence_block,
@@ -26,6 +27,7 @@ from skyvern.forge.sdk.copilot.tools import (
)
from skyvern.forge.sdk.copilot.tools._shared import _registered_output_parameter_payloads
from skyvern.forge.sdk.copilot.tools.blockers import _code_output_has_goal_content
+from skyvern.forge.sdk.copilot.tools.completion import _apply_present_value_upgrades, _build_run_evidence_snapshot
from skyvern.forge.sdk.copilot.tools.run_execution import _attach_registered_output_parameter_values
from skyvern.forge.sdk.copilot.turn_halt import TurnHaltKind
@@ -780,13 +782,15 @@ def test_all_null_metadata_goal_fields_are_flagged_as_no_goal_content() -> None:
assert _run_blocks_structured_blocker_message(result) is None
_, empty_data_blocks, _ = _analyze_run_blocks(result, ctx)
assert empty_data_blocks is True
- assert _is_outcome_evidence_candidate(ctx, result) is False
+ # The candidate gate no longer rejects empty output: a completed run reaches the
+ # judge, which requires positive evidence per criterion. With no verifier verdict
+ # (no criteria) the empty-output floor still marks the run suspicious.
+ assert _is_outcome_evidence_candidate(ctx, result) is True
_record_run_blocks_result(ctx, result, completion_verification=None)
assert ctx.last_test_ok is None
assert ctx.last_test_suspicious_success is True
- assert ctx.null_data_streak_count == 1
assert ctx.last_full_workflow_test_ok is False
assert getattr(ctx, "last_good_workflow", None) is None
@@ -834,7 +838,7 @@ def test_goal_path_alias_without_exact_declared_path_is_flagged_as_no_goal_conte
_, empty_data_blocks, _ = _analyze_run_blocks(result, ctx)
assert empty_data_blocks is True
- assert _is_outcome_evidence_candidate(ctx, result) is False
+ assert _is_outcome_evidence_candidate(ctx, result) is True
def test_null_goal_path_value_does_not_fall_back_to_alias_field() -> None:
@@ -844,7 +848,7 @@ def test_null_goal_path_value_does_not_fall_back_to_alias_field() -> None:
_, empty_data_blocks, _ = _analyze_run_blocks(result, ctx)
assert empty_data_blocks is True
- assert _is_outcome_evidence_candidate(ctx, result) is False
+ assert _is_outcome_evidence_candidate(ctx, result) is True
def test_partial_metadata_goal_fields_are_flagged_as_no_goal_content() -> None:
@@ -854,7 +858,7 @@ def test_partial_metadata_goal_fields_are_flagged_as_no_goal_content() -> None:
_, empty_data_blocks, _ = _analyze_run_blocks(result, ctx)
assert empty_data_blocks is True
- assert _is_outcome_evidence_candidate(ctx, result) is False
+ assert _is_outcome_evidence_candidate(ctx, result) is True
def test_undeclared_boolean_flags_are_not_goal_content() -> None:
@@ -912,7 +916,7 @@ def test_array_goal_value_path_does_not_match_scalar_root() -> None:
_, empty_data_blocks, _ = _analyze_run_blocks(result, ctx)
assert empty_data_blocks is True
- assert _is_outcome_evidence_candidate(ctx, result) is False
+ assert _is_outcome_evidence_candidate(ctx, result) is True
def test_candidacy_and_recording_agree_on_all_null_metadata_goal_fields() -> None:
@@ -920,7 +924,7 @@ def test_candidacy_and_recording_agree_on_all_null_metadata_goal_fields() -> Non
ctx = _ctx(result["data"]["blocks"])
ctx.code_artifact_metadata = {"expand_result_rows": _terminal_metadata_with_goal_fields()}
- assert _is_outcome_evidence_candidate(ctx, result) is False
+ assert _is_outcome_evidence_candidate(ctx, result) is True
_record_run_blocks_result(ctx, result, completion_verification=None)
assert ctx.last_test_ok is None
@@ -934,7 +938,7 @@ def test_empty_goal_collections_without_blocker_are_flagged() -> None:
assert _run_blocks_structured_blocker_message(result) is None
_, empty_data_blocks, _ = _analyze_run_blocks(result)
assert empty_data_blocks is True
- assert _is_outcome_evidence_candidate(ctx, result) is False
+ assert _is_outcome_evidence_candidate(ctx, result) is True
_record_run_blocks_result(ctx, result, completion_verification=None)
assert ctx.last_test_ok is None
@@ -1125,13 +1129,65 @@ def test_judge_unmet_on_detector_clean_run_does_not_reset_streaks_or_promote(met
entry["completion_criteria"][0]["terminal"] = False
ctx.code_artifact_metadata = {"search_registry_person": entry}
ctx.failed_test_nudge_count = 2
- ctx.null_data_streak_count = 3
ctx.probable_site_block_streak_count = 4
_record_run_blocks_result(ctx, result, completion_verification=_no_evidence("c0"))
assert ctx.failed_test_nudge_count == 2
- assert ctx.null_data_streak_count == 3
assert ctx.probable_site_block_streak_count == 4
assert ctx.last_full_workflow_test_ok is False
+
+
+def test_reached_goal_unfinished_run_is_recognized_when_verifier_fully_satisfied() -> None:
+ result = _run_result([_code_block("submit_request", {"confirmation_number": "WTR-1842-DEMO"})], ok=False)
+ ctx = _ctx(result["data"]["blocks"])
+
+ assert _is_unfinished_run_verification_candidate(ctx, result) is True
+
+ recorded = _record_run_blocks_result(ctx, result, completion_verification=_satisfied("c0"))
+
+ assert outcome_fully_verified(ctx) is True
+ assert recorded is not None
+ assert recorded.verdict == "demonstrated"
+ assert ctx.last_full_workflow_test_ok is True
+ assert ctx.last_test_suspicious_success is False
+
+
+def test_unfinished_run_with_unsatisfied_verifier_stays_unrecognized() -> None:
+ result = _run_result([_code_block("submit_request", {"confirmation_number": "WTR-1842-DEMO"})], ok=False)
+ ctx = _ctx(result["data"]["blocks"])
+
+ recorded = _record_run_blocks_result(ctx, result, completion_verification=_no_evidence("c0"))
+
+ assert outcome_fully_verified(ctx) is False
+ assert ctx.last_full_workflow_test_ok is False
+ assert recorded is None or recorded.verdict != "demonstrated"
assert getattr(ctx, "last_good_workflow", None) is None
+
+
+def test_present_value_upgrade_flips_lone_judge_unknown_to_demonstrated_on_ok_run() -> None:
+ result = _run_result([_code_block("submit_request", {"confirmation_number": "WTR-1842-DEMO"})], ok=True)
+ ctx = _ctx(result["data"]["blocks"])
+ ctx.request_policy = RequestPolicy(
+ completion_criteria=[
+ CompletionCriterion(id="c0", outcome="the submitted request returns confirmation number WTR-1842-DEMO")
+ ]
+ )
+
+ run_criteria = ctx.request_policy.completion_criteria
+ snapshot = _build_run_evidence_snapshot(ctx, result)
+ judge_result = CompletionVerificationResult(
+ status="evaluated",
+ criterion_ids=["c0"],
+ verdicts=[CriterionVerdict(criterion_id="c0", state="unknown", reason_code="unknown")],
+ )
+ upgraded = _apply_present_value_upgrades(judge_result, run_criteria, snapshot)
+ assert upgraded.is_fully_satisfied() is True
+
+ recorded = _record_run_blocks_result(ctx, result, completion_verification=upgraded)
+
+ assert recorded is not None
+ assert recorded.verdict == "demonstrated"
+ assert outcome_fully_verified(ctx) is True
+ assert ctx.last_test_suspicious_success is False
+ assert ctx.last_full_workflow_test_ok is True