Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
SKY-11357: make completion recognizer reliably-reached and authoritative (#6789)
Andrew Neilson1b696b163938
diff --git a/skyvern/forge/sdk/copilot/completion_verification.py b/skyvern/forge/sdk/copilot/completion_verification.pyindex aeda47aca..f35d4b0ae 100644--- a/skyvern/forge/sdk/copilot/completion_verification.py+++ b/skyvern/forge/sdk/copilot/completion_verification.py@@ -79,8 +79,19 @@ class CompletionVerificationResult:def is_fully_satisfied(self) -> bool:if self.status != "evaluated" or not self.criterion_ids:return False- satisfied = {verdict.criterion_id for verdict in self.verdicts if verdict.satisfied}- return all(criterion_id in satisfied for criterion_id in self.criterion_ids)+ verdict_by_id = {verdict.criterion_id: verdict for verdict in self.verdicts}+ satisfied_count = 0+ for criterion_id in self.criterion_ids:+ verdict = verdict_by_id.get(criterion_id)+ if verdict is not None and verdict.satisfied:+ satisfied_count += 1+ continue+ # A definition-plane ``unknown`` is a YAML-grader abstention, not a refutation,+ # so it must not veto a run whose observable outcome evidence is fully confirmed.+ if verdict is not None and _is_definition_plane_abstention(verdict):+ continue+ return False+ return satisfied_count > 0def to_trace_data(self) -> dict[str, Any]:unmet = [verdict for verdict in self.verdicts if not verdict.satisfied]@@ -389,12 +400,12 @@ def grade_definition_criteria(criteria: list[CompletionCriterion], workflow_yaml)continuedefined, referenced = reference_state+ named = _named_definition_inputs(criterion.outcome)if defined and referenced:# When the criterion names specific inputs, each must match a defined# parameter key; one stray parameter must not satisfy a multi-input ask.# An unmatchable name degrades to unknown, not unsatisfied — extraction# is heuristic and a false negative would drive repair of correct YAML.- named = _named_definition_inputs(criterion.outcome)normalized_keys = [_normalize_input_phrase(key) for key in defined]if named and not all(_input_matches_any_key(candidate, normalized_keys) for candidate in named):verdicts.append(@@ -411,6 +422,13 @@ def grade_definition_criteria(criteria: list[CompletionCriterion], workflow_yamlevidence_ref="workflow_yaml:" + ",".join(referenced[:8]),))+ elif not named:+ # A reusable-inputs hint with no specific named inputs cannot be proven+ # false: a read-only/validation workflow legitimately defines none, so a+ # missing/unreferenced parameter set abstains rather than sinking the run.+ verdicts.append(+ CriterionVerdict(criterion_id=criterion.id, state="unknown", reason_code="definition_parameters_absent")+ )elif defined:verdicts.append(CriterionVerdict(@@ -434,18 +452,25 @@ _ISO_DATE_LITERAL_RE = re.compile(r"\b\d{4}-\d{2}-\d{2}\b")# A bare digit run qualifies as an identifier only at >=5 digits: a 4-digit run is# usually a year and too low-specificity to credit on a coincidental output match._LONG_DIGIT_LITERAL_RE = re.compile(r"\b\d{5,}\b")+# A mixed alphanumeric code (e.g. WTR-1842-DEMO, ABC12345) credits only via the+# letter+digit, >=6 alnum filter below, which excludes bare words and bare years.+_STRUCTURED_ID_LITERAL_RE = re.compile(r"\b[A-Za-z0-9]+(?:[-_][A-Za-z0-9]+)*\b")def _normalize_present_value(text: str) -> str:return " ".join(text.casefold().split())+def _is_structured_identifier(token: str) -> bool:+ alnum = [ch for ch in token if ch.isalnum()]+ return len(alnum) >= 6 and any(ch.isalpha() for ch in alnum) and any(ch.isdigit() for ch in alnum)++def _extract_present_value_literals(outcome: str) -> list[str]:- literals: list[str] = []- for match in _QUOTED_LITERAL_RE.finditer(outcome):- literals.append(match.group(1))+ literals: list[str] = [match.group(1) for match in _QUOTED_LITERAL_RE.finditer(outcome)]for pattern in (_CURRENCY_LITERAL_RE, _ISO_DATE_LITERAL_RE, _LONG_DIGIT_LITERAL_RE):literals.extend(pattern.findall(outcome))+ literals.extend(token for token in _STRUCTURED_ID_LITERAL_RE.findall(outcome) if _is_structured_identifier(token))normalized: list[str] = []seen: set[str] = set()for literal in literals:@@ -893,6 +918,10 @@ def _is_meaningful_record_value(value: Any) -> bool:_DEFINITION_REASON_PREFIX = "definition_"+def _is_definition_plane_abstention(verdict: CriterionVerdict) -> bool:+ return verdict.state == "unknown" and verdict.reason_code.startswith(_DEFINITION_REASON_PREFIX)++def run_plane_all_no_evidence(result: CompletionVerificationResult) -> bool:"""Whether every run-plane verdict in an evaluated result is ``no_evidence``.diff --git a/skyvern/forge/sdk/copilot/config.py b/skyvern/forge/sdk/copilot/config.pyindex 5c2af53c6..4398a6617 100644--- a/skyvern/forge/sdk/copilot/config.py+++ b/skyvern/forge/sdk/copilot/config.py@@ -126,24 +126,6 @@ POST_SUSPICIOUS_SUCCESS_NUDGE = ("status alone. Verify the actual extracted data answers the user's question.")-POST_REPEATED_NULL_DATA_NUDGE = (- "STOP — you have now produced multiple consecutive test runs where "- "extraction/text_prompt blocks returned all-null or empty data. "- "Re-prompting the extractor is not working — the problem is almost "- "certainly NOT how the extraction goal is worded.\n"- "You MUST now do ONE of the following before another update_workflow call:\n"- "1. Call get_browser_screenshot on the workflow's browser session to see "- "exactly what page the workflow is actually loading (it may differ from "- "what you expect — e.g. a 'no results' fallback, cookie wall, or bot block).\n"- "2. Call evaluate with JavaScript that searches for the expected content "- "on the workflow's browser — confirm whether the data is even present.\n"- "3. If the page the workflow loads genuinely does not contain the data, "- "pivot to a different URL or source entirely — do NOT keep retrying "- "extraction against the same failing page.\n"- "Do NOT call update_and_run_blocks again until you have concrete evidence "- "about what the workflow browser is actually seeing."-)-POST_REPEATED_FRONTIER_FAILURE_WARN_NUDGE = ("STOP — this is the second run with the same frontier and the same failure ""signature. Re-running the same change again is unlikely to help.\n"@@ -349,7 +331,6 @@ DEFAULT_ENFORCEMENT_NUDGES: dict[str, str] = {"post_failed_test_inspect_first": POST_FAILED_TEST_INSPECT_FIRST_NUDGE,"post_explore_without_workflow": POST_EXPLORE_WITHOUT_WORKFLOW_NUDGE,"post_suspicious_success": POST_SUSPICIOUS_SUCCESS_NUDGE,- "post_repeated_null_data": POST_REPEATED_NULL_DATA_NUDGE,"post_repeated_frontier_failure_warn": POST_REPEATED_FRONTIER_FAILURE_WARN_NUDGE,"post_repeated_frontier_failure_stop": POST_REPEATED_FRONTIER_FAILURE_STOP_NUDGE,"post_parameter_binding_warn": POST_PARAMETER_BINDING_WARN_NUDGE,diff --git a/skyvern/forge/sdk/copilot/context.py b/skyvern/forge/sdk/copilot/context.pyindex 545b0d966..06f30869d 100644--- a/skyvern/forge/sdk/copilot/context.py+++ b/skyvern/forge/sdk/copilot/context.py@@ -432,11 +432,6 @@ class CopilotContext(AgentContext):last_run_blocks_workflow_run_id: str | None = Nonelast_successful_run_blocks_workflow_run_id: str | None = Nonelast_outcome_gate_workflow_run_id: str | None = None- # Consecutive test runs whose data-producing blocks completed with no- # meaningful output (missing, empty, or all-null fields). Resets when a- # run produces real data. Used to escalate when the agent is stuck- # retrying extraction against a page that doesn't contain the data.- null_data_streak_count: int = 0# Consecutive failed runs where navigation completed but the scraper# could not read the page (generic "failed to load the website" template).# Resets on any non-matching run outcome. Streak crosses workflow-shapediff --git a/skyvern/forge/sdk/copilot/diagnosis_repair_contract.py b/skyvern/forge/sdk/copilot/diagnosis_repair_contract.pyindex 563984813..688198c4c 100644--- a/skyvern/forge/sdk/copilot/diagnosis_repair_contract.py+++ b/skyvern/forge/sdk/copilot/diagnosis_repair_contract.py@@ -573,7 +573,6 @@ def _prior_repair_attempts(ctx: Any) -> dict[str, int]:keys = ("repeated_failure_streak_count","failed_test_nudge_count",- "null_data_streak_count","probable_site_block_streak_count","per_tool_budget_nudge_count","repeated_action_fingerprint_streak_count",diff --git a/skyvern/forge/sdk/copilot/enforcement.py b/skyvern/forge/sdk/copilot/enforcement.pyindex e9dc26e30..1ad3fe6b1 100644--- a/skyvern/forge/sdk/copilot/enforcement.py+++ b/skyvern/forge/sdk/copilot/enforcement.py@@ -45,7 +45,6 @@ from skyvern.forge.sdk.copilot.config import (POST_PROBABLE_SITE_BLOCK_STOP_NUDGE,POST_REPEATED_FRONTIER_FAILURE_STOP_NUDGE,POST_REPEATED_FRONTIER_FAILURE_WARN_NUDGE,- POST_REPEATED_NULL_DATA_NUDGE,POST_SUSPICIOUS_SUCCESS_NUDGE,POST_UPDATE_NUDGE,PRE_DISCOVERY_URL_QUESTION_NUDGE,@@ -114,9 +113,6 @@ MAX_EXPLORE_WITHOUT_WORKFLOW_NUDGES = 2# correctly diagnosed an unrecoverable block (anti-bot, paywall) and is no# longer willing to re-run extraction.MAX_SUSPICIOUS_SUCCESS_NUDGES = 2-# Escalate after this many consecutive all-null extraction runs so the agent-# inspects browser state instead of re-prompting the extractor.-NULL_DATA_STREAK_ESCALATE_AT = 2# Streak levels for repeated-failure (same frontier + same failure signature).REPEATED_FRONTIER_STREAK_ESCALATE_AT = 2REPEATED_FRONTIER_STREAK_STOP_AT = 3@@ -1171,17 +1167,6 @@ def _needs_probable_site_block_stop_nudge(ctx: Any) -> bool:return _get_int(ctx, "probable_site_block_stop_nudge_count") < MAX_PROBABLE_SITE_BLOCK_STOP_NUDGES-def _needs_repeated_null_data_nudge(ctx: Any) -> bool:- """Return True when suspicious-success has happened enough times to escalate."""- # Same as above: non-retriable nav state never belongs on this branch.- if getattr(ctx, "last_test_non_retriable_nav_error", None):- return False- if not getattr(ctx, "last_test_suspicious_success", False):- return False- streak = getattr(ctx, "null_data_streak_count", 0)- return streak >= NULL_DATA_STREAK_ESCALATE_AT--def _get_int(ctx: Any, name: str, default: int = 0) -> int:value = getattr(ctx, name, default)return value if isinstance(value, int) else default@@ -1334,9 +1319,6 @@ def _check_enforcement(# Do NOT clear last_test_suspicious_success here. tools._record_run_blocks_result# resets it on every new run; if the agent ignores the nudge and answers# without rerunning, we want _check_enforcement to re-emit the nudge.- if _needs_repeated_null_data_nudge(ctx):- return _nudge(config, "post_repeated_null_data")-if _needs_suspicious_success_nudge(ctx):ctx.suspicious_success_nudge_count = getattr(ctx, "suspicious_success_nudge_count", 0) + 1return _nudge(config, "post_suspicious_success")@@ -1657,7 +1639,6 @@ _NUDGE_TYPE_BY_MESSAGE: dict[str, str] = {POST_NAVIGATE_NUDGE: "post_navigate",POST_EXPLORE_WITHOUT_WORKFLOW_NUDGE: "explore_without_workflow",POST_SUSPICIOUS_SUCCESS_NUDGE: "suspicious_success",- POST_REPEATED_NULL_DATA_NUDGE: "repeated_null_data",POST_REPEATED_FRONTIER_FAILURE_WARN_NUDGE: "repeated_frontier_failure_warn",POST_REPEATED_FRONTIER_FAILURE_STOP_NUDGE: "repeated_frontier_failure_stop",POST_NON_RETRIABLE_NAV_ERROR_STOP_NUDGE: "non_retriable_nav_error_stop",@@ -1681,7 +1662,6 @@ _NUDGE_TYPE_BY_KEY: dict[str, str] = {"post_navigate": "post_navigate","post_explore_without_workflow": "explore_without_workflow","post_suspicious_success": "suspicious_success",- "post_repeated_null_data": "repeated_null_data","post_repeated_frontier_failure_warn": "repeated_frontier_failure_warn","post_repeated_frontier_failure_stop": "repeated_frontier_failure_stop","post_non_retriable_nav_error_stop": "non_retriable_nav_error_stop",diff --git a/skyvern/forge/sdk/copilot/runtime.py b/skyvern/forge/sdk/copilot/runtime.pyindex f2c121b31..c74f62752 100644--- a/skyvern/forge/sdk/copilot/runtime.py+++ b/skyvern/forge/sdk/copilot/runtime.py@@ -184,7 +184,6 @@ class AgentContext:copilot_total_timeout_exceeded: bool = Falsefailed_test_nudge_count: int = 0explore_without_workflow_nudge_count: int = 0- null_data_streak_count: int = 0last_test_ok: bool | None = Nonelast_test_suspicious_success: bool = Falselast_test_anti_bot: str | None = Nonediff --git a/skyvern/forge/sdk/copilot/tools/completion.py b/skyvern/forge/sdk/copilot/tools/completion.pyindex 365bd21b8..3e29b7d84 100644--- a/skyvern/forge/sdk/copilot/tools/completion.py+++ b/skyvern/forge/sdk/copilot/tools/completion.py@@ -267,19 +267,18 @@ def _is_outcome_evidence_candidate(copilot_ctx: Any, result: dict[str, Any]) ->"""A clean ok=True run worth judging on its whole-workflow outcome.Recognition is governed by the outcome evidence the user can observe, not by- whether every block was verified as an end-to-end prefix (SKY-10576). The judge- requires positive evidence for every criterion, and ``empty_data_blocks`` rejects- a run whose outcome block produced nothing, so an incomplete run never passes --- this only lets an already-reached goal be recognized without a redundant- full-prefix re-run.+ whether the run produced any data. The judge requires positive evidence for+ every criterion, so a genuinely-empty run still grades unsatisfied; only a+ terminal anti-bot blocker keeps a completed run out of the judge, because that+ evidence makes the apparent success unusable."""if not bool(result.get("ok", False)):return Falsestructured_blocker = _run_blocks_structured_blocker_message(result, copilot_ctx)- anti_bot, empty_data_blocks, _categories = _analyze_run_blocks(result, copilot_ctx)+ anti_bot, _empty_data_blocks, _categories = _analyze_run_blocks(result, copilot_ctx)if structured_blocker and (anti_bot or _looks_like_anti_bot_blocker(structured_blocker)):return False- return not empty_data_blocks+ return Truedef _is_unfinished_run_verification_candidate(copilot_ctx: Any, result: dict[str, Any]) -> bool:diff --git a/skyvern/forge/sdk/copilot/tools/run_execution.py b/skyvern/forge/sdk/copilot/tools/run_execution.pyindex fb91b5983..f961c6311 100644--- a/skyvern/forge/sdk/copilot/tools/run_execution.py+++ b/skyvern/forge/sdk/copilot/tools/run_execution.py@@ -82,6 +82,7 @@ from skyvern.forge.sdk.copilot.runtime import (AgentContext,ensure_browser_session,)+from skyvern.forge.sdk.copilot.terminal_predicates import outcome_fully_verifiedfrom skyvern.forge.sdk.copilot.tracing_setup import copilot_spanfrom skyvern.forge.sdk.copilot.turn_halt import (stash_repair_ceiling_turn_halt,@@ -2110,6 +2111,9 @@ def _record_run_blocks_result(if run_ok:_mark_page_inspected(copilot_ctx)+ completion_verification_evaluated = (+ completion_verification is not None and completion_verification.status == "evaluated"+ )completion_fully_satisfied = (completion_verification is not Noneand completion_verification.status == "evaluated"@@ -2146,12 +2150,10 @@ def _record_run_blocks_result(copilot_ctx.verified_terminal_proposal_ready = Truecopilot_ctx.last_test_suspicious_success = Falsecopilot_ctx.last_test_failure_reason = None- copilot_ctx.null_data_streak_count = 0copilot_ctx.suspicious_success_nudge_count = 0- if empty_data_blocks and not completion_fully_satisfied:+ if empty_data_blocks and not completion_verification_evaluated:copilot_ctx.last_test_ok = Nonecopilot_ctx.last_test_suspicious_success = True- copilot_ctx.null_data_streak_count = getattr(copilot_ctx, "null_data_streak_count", 0) + 1copilot_ctx.last_test_failure_reason = ("All blocks completed but data-producing blocks ""produced no meaningful output "@@ -2189,7 +2191,6 @@ def _record_run_blocks_result(data.setdefault("failure_reason", outcome_unverified_reason)else:copilot_ctx.failed_test_nudge_count = 0- copilot_ctx.null_data_streak_count = 0copilot_ctx.probable_site_block_streak_count = 0copilot_ctx.last_failed_workflow_yaml = None# Real success: clear the signature latch so a subsequent bad URL in@@ -2215,6 +2216,21 @@ def _record_run_blocks_result(_update_verification_evidence_from_run_result(copilot_ctx, result)return _stash_recorded_run_outcome(copilot_ctx, _adjudicated_run_outcome(copilot_ctx, completion_verification))+ if outcome_fully_verified(copilot_ctx):+ copilot_ctx.last_test_suspicious_success = False+ copilot_ctx.last_test_failure_reason = None+ copilot_ctx.suspicious_success_nudge_count = 0+ copilot_ctx.failed_test_nudge_count = 0+ copilot_ctx.probable_site_block_streak_count = 0+ copilot_ctx.last_failed_workflow_yaml = None+ copilot_ctx.last_full_workflow_test_ok = True+ copilot_ctx.last_unverified_block_labels = []+ copilot_ctx.last_good_workflow = copilot_ctx.last_workflow+ copilot_ctx.last_good_workflow_yaml = copilot_ctx.last_workflow_yaml+ update_repeated_failure_state(copilot_ctx, result)+ _update_verification_evidence_from_run_result(copilot_ctx, result)+ return _stash_recorded_run_outcome(copilot_ctx, _adjudicated_run_outcome(copilot_ctx, completion_verification))+copilot_ctx.last_failed_workflow_yaml = getattr(copilot_ctx, "workflow_yaml", None)copilot_ctx.last_test_non_retriable_nav_error = _detect_non_retriable_nav_error(result)if _detect_probable_site_block_wall(result):diff --git a/tests/unit/test_copilot_completion_criteria_lifecycle.py b/tests/unit/test_copilot_completion_criteria_lifecycle.pyindex 134b3d913..0291595c0 100644--- a/tests/unit/test_copilot_completion_criteria_lifecycle.py+++ b/tests/unit/test_copilot_completion_criteria_lifecycle.py@@ -356,7 +356,7 @@ def test_definition_grading_satisfied_when_parameters_exist_and_are_referenced()def test_definition_grading_unsatisfied_when_parameters_missing_or_unreferenced() -> None:- criteria = [_criterion("c0", "the workflow accepts reusable inputs", level="definition")]+ criteria = [_criterion("c0", "the workflow accepts first name and NPI as reusable inputs", level="definition")](missing,) = grade_definition_criteria(criteria, _NO_PARAMS_YAML)assert missing.state == "unsatisfied"assert missing.reason_code == "definition_parameters_missing"@@ -365,6 +365,16 @@ def test_definition_grading_unsatisfied_when_parameters_missing_or_unreferenced(assert unreferenced.reason_code == "definition_parameters_unreferenced"+def test_definition_grading_abstains_when_no_specific_inputs_named() -> None:+ criteria = [_criterion("c0", "the workflow accepts reusable inputs", level="definition")]+ (no_params,) = grade_definition_criteria(criteria, _NO_PARAMS_YAML)+ assert no_params.state == "unknown"+ assert no_params.reason_code == "definition_parameters_absent"+ (unreferenced,) = grade_definition_criteria(criteria, _UNREFERENCED_YAML)+ assert unreferenced.state == "unknown"+ assert unreferenced.reason_code == "definition_parameters_absent"++def test_definition_grading_requires_every_named_input_to_match() -> None:criteria = [_criterion(@@ -557,21 +567,48 @@ def test_present_value_upgrade_is_upgrade_only() -> None:assert upgraded.verdicts[0].state == "satisfied"assert upgraded.verdicts[0].reason_code == "present_value_verbatim"- already_satisfied = _evaluated(_verdict("c0", "satisfied", "evidence_confirms"))- kept = _apply_present_value_upgrades(already_satisfied, criteria, snapshot)- assert kept.verdicts[0].reason_code == "evidence_confirms"- contradicts = _evaluated(_verdict("c0", "unsatisfied", "evidence_contradicts"))- absent_snapshot = _snapshot({"confirm": {"number": "ZZZ00000"}})- unchanged = _apply_present_value_upgrades(contradicts, criteria, absent_snapshot)- assert unchanged.verdicts[0].reason_code == "evidence_contradicts"+def test_present_value_credits_unquoted_structured_identifier() -> None:+ criteria = [_criterion("c0", "the submitted request returns confirmation number WTR-1842-DEMO")]+ snapshot = _snapshot({"submit_request": {"confirmation_number": "WTR-1842-DEMO", "status": "submitted"}})+ (verdict,) = grade_present_value_criteria(criteria, snapshot)+ assert verdict.state == "satisfied"+ assert verdict.reason_code == "present_value_verbatim"+ assert verdict.evidence_ref == "block_outputs:submit_request"- present_contradicts = _evaluated(_verdict("c0", "unsatisfied", "evidence_contradicts"))- kept_contradicts = _apply_present_value_upgrades(present_contradicts, criteria, snapshot)- assert kept_contradicts.verdicts[0].reason_code == "evidence_contradicts"- unavailable = CompletionVerificationResult(status="unavailable")- assert _apply_present_value_upgrades(unavailable, criteria, snapshot).status == "unavailable"+def test_present_value_structured_identifier_abstains_when_absent() -> None:+ criteria = [_criterion("c0", "the submitted request returns confirmation number WTR-1842-DEMO")]+ snapshot = _snapshot({"submit_request": {"confirmation_number": "WTR-9999-PROD"}})+ assert grade_present_value_criteria(criteria, snapshot) == []+++def test_present_value_structured_identifier_requires_letter_and_digit() -> None:+ # A bare word or a bare year-length token is not a high-specificity identifier.+ plain_word = [_criterion("c0", "the page shows submitted")]+ assert grade_present_value_criteria(plain_word, _snapshot({"x": {"state": "submitted"}})) == []++ hyphen_word = [_criterion("c0", "the request is read-only")]+ assert grade_present_value_criteria(hyphen_word, _snapshot({"x": {"mode": "read-only"}})) == []+++def test_present_value_structured_identifier_rejects_substring_of_longer_token() -> None:+ criteria = [_criterion("c0", "confirmation number WTR-1842-DEMO")]+ assert grade_present_value_criteria(criteria, _snapshot({"x": {"number": "WTR-1842-DEMO2"}})) == []+++def test_present_value_structured_identifier_upgrade_only() -> None:+ criteria = [_criterion("c0", "confirmation number WTR-1842-DEMO")]+ snapshot = _snapshot({"confirm": {"number": "WTR-1842-DEMO"}})++ judge_unknown = _evaluated(_verdict("c0", "unknown", "unknown"))+ upgraded = _apply_present_value_upgrades(judge_unknown, criteria, snapshot)+ assert upgraded.verdicts[0].state == "satisfied"+ assert upgraded.verdicts[0].reason_code == "present_value_verbatim"++ contradicts = _evaluated(_verdict("c0", "unsatisfied", "evidence_contradicts"))+ kept = _apply_present_value_upgrades(contradicts, criteria, snapshot)+ assert kept.verdicts[0].reason_code == "evidence_contradicts"def test_parse_completion_criteria_coerces_level() -> None:diff --git a/tests/unit/test_copilot_completion_verification.py b/tests/unit/test_copilot_completion_verification.pyindex 124d9cc10..ab0f8e835 100644--- a/tests/unit/test_copilot_completion_verification.py+++ b/tests/unit/test_copilot_completion_verification.py@@ -183,6 +183,38 @@ def test_is_fully_satisfied_requires_every_criterion() -> None:assert _evaluated(("c0", True), ("c1", False)).is_fully_satisfied() is False+def _mixed(*verdicts: CriterionVerdict) -> CompletionVerificationResult:+ return CompletionVerificationResult(+ status="evaluated", criterion_ids=[v.criterion_id for v in verdicts], verdicts=list(verdicts)+ )+++def test_definition_plane_abstention_does_not_sink_evidence_confirmed_run() -> None:+ definition_unknown = CriterionVerdict(+ criterion_id="c0", state="unknown", reason_code="definition_parameters_absent"+ )+ confirmed = CriterionVerdict(criterion_id="c1", state="satisfied", reason_code="evidence_confirms")+ assert _mixed(definition_unknown, confirmed).is_fully_satisfied() is True+++def test_run_plane_unknown_still_blocks() -> None:+ run_unknown = CriterionVerdict(criterion_id="c0", state="unknown", reason_code="unknown")+ confirmed = CriterionVerdict(criterion_id="c1", state="satisfied", reason_code="evidence_confirms")+ assert _mixed(run_unknown, confirmed).is_fully_satisfied() is False+++def test_all_definition_abstentions_do_not_vacuously_satisfy() -> None:+ abstain_a = CriterionVerdict(criterion_id="c0", state="unknown", reason_code="definition_unknown")+ abstain_b = CriterionVerdict(criterion_id="c1", state="unknown", reason_code="definition_parameters_absent")+ assert _mixed(abstain_a, abstain_b).is_fully_satisfied() is False+++def test_definition_plane_unsatisfied_still_blocks() -> None:+ unreferenced = CriterionVerdict(criterion_id="c0", state="unsatisfied", reason_code="definition_parameters_missing")+ confirmed = CriterionVerdict(criterion_id="c1", state="satisfied", reason_code="evidence_confirms")+ assert _mixed(unreferenced, confirmed).is_fully_satisfied() is False++def test_empty_verdicts_with_criteria_is_not_vacuously_satisfied() -> None:result = CompletionVerificationResult(status="evaluated", criterion_ids=["c0"], verdicts=[])assert result.is_fully_satisfied() is False@@ -901,6 +933,22 @@ def test_record_run_blocks_downgrades_on_contradiction_without_confirmation_blocassert ctx.last_full_workflow_test_ok is False+def test_record_run_blocks_demonstrated_when_lone_definition_abstention_with_confirmed_run() -> None:+ ctx = _ctx_with_blocks("extraction")+ verification = _mixed(+ CriterionVerdict(criterion_id="c0", state="unknown", reason_code="definition_parameters_absent"),+ CriterionVerdict(criterion_id="c1", state="satisfied", reason_code="evidence_confirms"),+ )++ recorded = _record_run_blocks_result(ctx, _clean_success_result(), completion_verification=verification)++ assert recorded is not None+ assert recorded.verdict == "demonstrated"+ assert ctx.last_test_suspicious_success is False+ assert ctx.last_full_workflow_test_ok is True+ assert verified_goal_satisfied_context(ctx) is True++def _goto_only_result() -> dict:return {"ok": True,@@ -1786,8 +1834,9 @@ def test_outcome_evidence_candidate_admits_clean_run_despite_unverified_prefix()# A clean run is admitted for the judge even though b0/b1 are not in the verified# prefix -- recognition is governed by the outcome judge, not the per-block prefix.assert _is_outcome_evidence_candidate(ctx, _clean_success_result()) is True- # It still rejects empty-data runs (no clean outcome to judge) and ok=False runs.- assert _is_outcome_evidence_candidate(ctx, _empty_data_result()) is False+ # An empty-data completed run is admitted for the judge (the judge requires positive+ # evidence per criterion, so it grades unsatisfied); only ok=False runs are rejected.+ assert _is_outcome_evidence_candidate(ctx, _empty_data_result()) is Trueassert _is_outcome_evidence_candidate(ctx, {"ok": False, "data": {}}) is Falsediff --git a/tests/unit/test_copilot_enforcement_hardening.py b/tests/unit/test_copilot_enforcement_hardening.pyindex 5cfe76032..e2311fc97 100644--- a/tests/unit/test_copilot_enforcement_hardening.py+++ b/tests/unit/test_copilot_enforcement_hardening.py@@ -218,7 +218,6 @@ def test_suspicious_success_nudge_refires_on_subsequent_turn() -> None:ctx = _fresh_context()ctx.last_test_ok = Nonectx.last_test_suspicious_success = True- ctx.null_data_streak_count = 1first = _check_enforcement(ctx)assert first == POST_SUSPICIOUS_SUCCESS_NUDGEdiff --git a/tests/unit/test_copilot_enforcement_pruning.py b/tests/unit/test_copilot_enforcement_pruning.pyindex 70280fb3a..bbd66b3b2 100644--- a/tests/unit/test_copilot_enforcement_pruning.py+++ b/tests/unit/test_copilot_enforcement_pruning.py@@ -19,12 +19,9 @@ from skyvern.forge.sdk.copilot.blocker_signal import CopilotToolBlockerSignalfrom skyvern.forge.sdk.copilot.completion_verification import CompletionVerificationResult, CriterionVerdictfrom skyvern.forge.sdk.copilot.enforcement import (KEEP_RECENT_TOOL_OUTPUTS,- NULL_DATA_STREAK_ESCALATE_AT,- POST_REPEATED_NULL_DATA_NUDGE,POST_SUSPICIOUS_SUCCESS_NUDGE,CopilotGoalSatisfied,_check_enforcement,- _needs_repeated_null_data_nudge,_needs_suspicious_success_nudge,_prune_input_list,_summarize_tool_output,@@ -64,7 +61,6 @@ class _Ctx:self.last_failure_category_top = Noneself.failed_test_nudge_count = 0self.explore_without_workflow_nudge_count = 0- self.null_data_streak_count = 0self.repeated_failure_streak_count = 0self.repeated_failure_nudge_emitted_at_streak = 0self.verified_terminal_proposal_ready = False@@ -335,7 +331,6 @@ def _fresh_ctx_for_record() -> SimpleNamespace:last_test_anti_bot=None,last_failure_category_top=None,last_test_non_retriable_nav_error=None,- null_data_streak_count=0,failed_test_nudge_count=0,last_failed_workflow_yaml=None,last_good_workflow=None,@@ -610,49 +605,23 @@ def test_record_run_blocks_result_sets_last_test_ok_none_on_watchdog_cancel_at_t# ----------------------------------------------------------------------------# Repeated null-data escalation+# Suspicious-success nudge# ---------------------------------------------------------------------------def test_suspicious_success_fires_when_flag_set() -> None:ctx = _Ctx()ctx.last_test_suspicious_success = True- ctx.null_data_streak_count = 1assert _needs_suspicious_success_nudge(ctx) is True- assert _needs_repeated_null_data_nudge(ctx) is False-def test_repeated_null_data_fires_at_threshold() -> None:+def test_check_enforcement_returns_suspicious_nudge_when_flag_set() -> None:ctx = _Ctx()ctx.last_test_suspicious_success = True- ctx.null_data_streak_count = NULL_DATA_STREAK_ESCALATE_AT- assert _needs_repeated_null_data_nudge(ctx) is True---def test_check_enforcement_returns_repeated_nudge_at_threshold() -> None:- ctx = _Ctx()- ctx.last_test_suspicious_success = True- ctx.null_data_streak_count = NULL_DATA_STREAK_ESCALATE_AT- nudge = _check_enforcement(ctx)- assert nudge == POST_REPEATED_NULL_DATA_NUDGE---def test_check_enforcement_returns_regular_suspicious_nudge_below_threshold() -> None:- ctx = _Ctx()- ctx.last_test_suspicious_success = True- ctx.null_data_streak_count = 1nudge = _check_enforcement(ctx)assert nudge == POST_SUSPICIOUS_SUCCESS_NUDGE-def test_repeated_null_data_requires_suspicious_flag() -> None:- # If the current test wasn't a suspicious success, don't fire even with a high streak.- ctx = _Ctx()- ctx.last_test_suspicious_success = False- ctx.null_data_streak_count = 99- assert _needs_repeated_null_data_nudge(ctx) is False--# ---------------------------------------------------------------------------# Tool-output pruning# ---------------------------------------------------------------------------diff --git a/tests/unit/test_copilot_non_retriable_nav.py b/tests/unit/test_copilot_non_retriable_nav.pyindex 54484227f..fadfc665a 100644--- a/tests/unit/test_copilot_non_retriable_nav.py+++ b/tests/unit/test_copilot_non_retriable_nav.py@@ -22,7 +22,6 @@ from skyvern.forge.sdk.copilot.enforcement import (_extract_url_from_nav_error,_maybe_raise_non_retriable_nav,_needs_failed_test_nudge,- _needs_repeated_null_data_nudge,_needs_suspicious_success_nudge,_non_retriable_nav_error_nudge,_repeated_frontier_failure_nudge,@@ -387,14 +386,6 @@ def test_needs_suspicious_success_nudge_suppressed_when_flag_set() -> None:assert _needs_suspicious_success_nudge(ctx) is False-def test_needs_repeated_null_data_nudge_suppressed_when_flag_set() -> None:- ctx = _fresh_context()- ctx.last_test_suspicious_success = True- ctx.null_data_streak_count = 5- ctx.last_test_non_retriable_nav_error = _DNS_FAILURE_REASON- assert _needs_repeated_null_data_nudge(ctx) is False--def test_repeated_frontier_failure_nudge_suppressed_when_flag_set() -> None:ctx = _fresh_context()ctx.repeated_failure_streak_count = 5@@ -492,18 +483,16 @@ def test_all_competing_branches_silent_after_latch() -> None:# Reproduce the full steady state after the one-shot stop nudge has# latched: ctx has a non-retriable error, last_test_ok=False,# test_after_update_done=True, and all counters are set such that the- # other branches would normally fire. Assert all four helpers are silent.+ # other branches would normally fire. Assert each helper is silent.ctx = _fresh_context()ctx.test_after_update_done = Truectx.last_test_ok = Falsectx.last_test_suspicious_success = True- ctx.null_data_streak_count = 5ctx.repeated_failure_streak_count = 5ctx.last_test_non_retriable_nav_error = _DNS_FAILURE_REASONassert _needs_failed_test_nudge(ctx) is Falseassert _needs_suspicious_success_nudge(ctx) is False- assert _needs_repeated_null_data_nudge(ctx) is Falseassert _repeated_frontier_failure_nudge(ctx) is Nonediff --git a/tests/unit/test_copilot_response_coverage_gate.py b/tests/unit/test_copilot_response_coverage_gate.pyindex 683b17e63..a9aae5951 100644--- a/tests/unit/test_copilot_response_coverage_gate.py+++ b/tests/unit/test_copilot_response_coverage_gate.py@@ -58,7 +58,6 @@ class _Ctx:self.last_test_anti_bot = Noneself.failed_test_nudge_count = 0self.explore_without_workflow_nudge_count = 0- self.null_data_streak_count = 0self.repeated_failure_streak_count = 0self.repeated_failure_nudge_emitted_at_streak = 0self.last_artifact_health_blocker_reason = Nonediff --git a/tests/unit/test_copilot_run_output_content_gate.py b/tests/unit/test_copilot_run_output_content_gate.pyindex 099eddced..7554a799e 100644--- a/tests/unit/test_copilot_run_output_content_gate.py+++ b/tests/unit/test_copilot_run_output_content_gate.py@@ -16,6 +16,7 @@ from skyvern.forge.sdk.copilot.completion_verification import CompletionVerificafrom skyvern.forge.sdk.copilot.context import CopilotContextfrom skyvern.forge.sdk.copilot.enforcement import verified_goal_satisfied_contextfrom skyvern.forge.sdk.copilot.request_policy import CompletionCriterion, RequestPolicy+from skyvern.forge.sdk.copilot.terminal_predicates import outcome_fully_verifiedfrom skyvern.forge.sdk.copilot.tools import (_analyze_run_blocks,_current_workflow_has_evidence_block,@@ -26,6 +27,7 @@ from skyvern.forge.sdk.copilot.tools import ()from skyvern.forge.sdk.copilot.tools._shared import _registered_output_parameter_payloadsfrom skyvern.forge.sdk.copilot.tools.blockers import _code_output_has_goal_content+from skyvern.forge.sdk.copilot.tools.completion import _apply_present_value_upgrades, _build_run_evidence_snapshotfrom skyvern.forge.sdk.copilot.tools.run_execution import _attach_registered_output_parameter_valuesfrom skyvern.forge.sdk.copilot.turn_halt import TurnHaltKind@@ -780,13 +782,15 @@ def test_all_null_metadata_goal_fields_are_flagged_as_no_goal_content() -> None:assert _run_blocks_structured_blocker_message(result) is None_, empty_data_blocks, _ = _analyze_run_blocks(result, ctx)assert empty_data_blocks is True- assert _is_outcome_evidence_candidate(ctx, result) is False+ # The candidate gate no longer rejects empty output: a completed run reaches the+ # judge, which requires positive evidence per criterion. With no verifier verdict+ # (no criteria) the empty-output floor still marks the run suspicious.+ assert _is_outcome_evidence_candidate(ctx, result) is True_record_run_blocks_result(ctx, result, completion_verification=None)assert ctx.last_test_ok is Noneassert ctx.last_test_suspicious_success is True- assert ctx.null_data_streak_count == 1assert ctx.last_full_workflow_test_ok is Falseassert getattr(ctx, "last_good_workflow", None) is None@@ -834,7 +838,7 @@ def test_goal_path_alias_without_exact_declared_path_is_flagged_as_no_goal_conte_, empty_data_blocks, _ = _analyze_run_blocks(result, ctx)assert empty_data_blocks is True- assert _is_outcome_evidence_candidate(ctx, result) is False+ assert _is_outcome_evidence_candidate(ctx, result) is Truedef test_null_goal_path_value_does_not_fall_back_to_alias_field() -> None:@@ -844,7 +848,7 @@ def test_null_goal_path_value_does_not_fall_back_to_alias_field() -> None:_, empty_data_blocks, _ = _analyze_run_blocks(result, ctx)assert empty_data_blocks is True- assert _is_outcome_evidence_candidate(ctx, result) is False+ assert _is_outcome_evidence_candidate(ctx, result) is Truedef test_partial_metadata_goal_fields_are_flagged_as_no_goal_content() -> None:@@ -854,7 +858,7 @@ def test_partial_metadata_goal_fields_are_flagged_as_no_goal_content() -> None:_, empty_data_blocks, _ = _analyze_run_blocks(result, ctx)assert empty_data_blocks is True- assert _is_outcome_evidence_candidate(ctx, result) is False+ assert _is_outcome_evidence_candidate(ctx, result) is Truedef test_undeclared_boolean_flags_are_not_goal_content() -> None:@@ -912,7 +916,7 @@ def test_array_goal_value_path_does_not_match_scalar_root() -> None:_, empty_data_blocks, _ = _analyze_run_blocks(result, ctx)assert empty_data_blocks is True- assert _is_outcome_evidence_candidate(ctx, result) is False+ assert _is_outcome_evidence_candidate(ctx, result) is Truedef test_candidacy_and_recording_agree_on_all_null_metadata_goal_fields() -> None:@@ -920,7 +924,7 @@ def test_candidacy_and_recording_agree_on_all_null_metadata_goal_fields() -> Nonctx = _ctx(result["data"]["blocks"])ctx.code_artifact_metadata = {"expand_result_rows": _terminal_metadata_with_goal_fields()}- assert _is_outcome_evidence_candidate(ctx, result) is False+ assert _is_outcome_evidence_candidate(ctx, result) is True_record_run_blocks_result(ctx, result, completion_verification=None)assert ctx.last_test_ok is None@@ -934,7 +938,7 @@ def test_empty_goal_collections_without_blocker_are_flagged() -> None:assert _run_blocks_structured_blocker_message(result) is None_, empty_data_blocks, _ = _analyze_run_blocks(result)assert empty_data_blocks is True- assert _is_outcome_evidence_candidate(ctx, result) is False+ assert _is_outcome_evidence_candidate(ctx, result) is True_record_run_blocks_result(ctx, result, completion_verification=None)assert ctx.last_test_ok is None@@ -1125,13 +1129,65 @@ def test_judge_unmet_on_detector_clean_run_does_not_reset_streaks_or_promote(metentry["completion_criteria"][0]["terminal"] = Falsectx.code_artifact_metadata = {"search_registry_person": entry}ctx.failed_test_nudge_count = 2- ctx.null_data_streak_count = 3ctx.probable_site_block_streak_count = 4_record_run_blocks_result(ctx, result, completion_verification=_no_evidence("c0"))assert ctx.failed_test_nudge_count == 2- assert ctx.null_data_streak_count == 3assert ctx.probable_site_block_streak_count == 4assert ctx.last_full_workflow_test_ok is False+++def test_reached_goal_unfinished_run_is_recognized_when_verifier_fully_satisfied() -> None:+ result = _run_result([_code_block("submit_request", {"confirmation_number": "WTR-1842-DEMO"})], ok=False)+ ctx = _ctx(result["data"]["blocks"])++ assert _is_unfinished_run_verification_candidate(ctx, result) is True++ recorded = _record_run_blocks_result(ctx, result, completion_verification=_satisfied("c0"))++ assert outcome_fully_verified(ctx) is True+ assert recorded is not None+ assert recorded.verdict == "demonstrated"+ assert ctx.last_full_workflow_test_ok is True+ assert ctx.last_test_suspicious_success is False+++def test_unfinished_run_with_unsatisfied_verifier_stays_unrecognized() -> None:+ result = _run_result([_code_block("submit_request", {"confirmation_number": "WTR-1842-DEMO"})], ok=False)+ ctx = _ctx(result["data"]["blocks"])++ recorded = _record_run_blocks_result(ctx, result, completion_verification=_no_evidence("c0"))++ assert outcome_fully_verified(ctx) is False+ assert ctx.last_full_workflow_test_ok is False+ assert recorded is None or recorded.verdict != "demonstrated"assert getattr(ctx, "last_good_workflow", None) is None+++def test_present_value_upgrade_flips_lone_judge_unknown_to_demonstrated_on_ok_run() -> None:+ result = _run_result([_code_block("submit_request", {"confirmation_number": "WTR-1842-DEMO"})], ok=True)+ ctx = _ctx(result["data"]["blocks"])+ ctx.request_policy = RequestPolicy(+ completion_criteria=[+ CompletionCriterion(id="c0", outcome="the submitted request returns confirmation number WTR-1842-DEMO")+ ]+ )++ run_criteria = ctx.request_policy.completion_criteria+ snapshot = _build_run_evidence_snapshot(ctx, result)+ judge_result = CompletionVerificationResult(+ status="evaluated",+ criterion_ids=["c0"],+ verdicts=[CriterionVerdict(criterion_id="c0", state="unknown", reason_code="unknown")],+ )+ upgraded = _apply_present_value_upgrades(judge_result, run_criteria, snapshot)+ assert upgraded.is_fully_satisfied() is True++ recorded = _record_run_blocks_result(ctx, result, completion_verification=upgraded)++ assert recorded is not None+ assert recorded.verdict == "demonstrated"+ assert outcome_fully_verified(ctx) is True+ assert ctx.last_test_suspicious_success is False+ assert ctx.last_full_workflow_test_ok is True