Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
SKY-10964 Block verified success on artifact-health failures (#6672)
Andrew Neilson5e66b1692d29
diff --git a/skyvern/forge/sdk/copilot/agent.py b/skyvern/forge/sdk/copilot/agent.pyindex 43283a441..c522c08bd 100644--- a/skyvern/forge/sdk/copilot/agent.py+++ b/skyvern/forge/sdk/copilot/agent.py@@ -65,6 +65,7 @@ from skyvern.forge.sdk.copilot.context import (finalize_discovery_counter_in_global_llm_context,)from skyvern.forge.sdk.copilot.enforcement import (+ artifact_health_blocked,outcome_fully_verified,verified_goal_claim_authorized,)@@ -908,6 +909,8 @@ def _shape_ask_question_response(user_response: str, ctx: CopilotContext) -> strdef _completion_contract_not_violated(ctx: CopilotContext) -> bool:+ if artifact_health_blocked(ctx):+ return Falseresult = ctx.completion_verification_resultif result is None:return True@@ -1180,13 +1183,21 @@ async def _build_goal_satisfied_exit_result(ctx: CopilotContext, global_llm_contexcept Exception as flush_err:LOG.warning("copilot_goal_satisfied_tool_result_flush_failed", error=str(flush_err))verified_workflow, verified_yaml = _verified_workflow_or_none(ctx)- if verified_goal_claim_authorized(ctx):+ clean_test = ctx.last_test_ok is True and ctx.last_full_workflow_test_ok is True+ if clean_test and verified_goal_claim_authorized(ctx):user_response = "I created and tested the workflow successfully."- else:+ elif clean_test:user_response = ("I built the workflow and the test run completed, but the goal outcome was not ""independently verified. Review the draft to confirm it does what you need.")+ elif ctx.last_test_ok is False:+ user_response = "I reached the requested outcome, but the workflow test did not finish successfully."+ else:+ user_response = (+ "I reached the requested outcome, but the workflow has not been tested end-to-end. "+ "Review the draft before using it."+ )final_text, outcome = apply_repeated_reply_guard(final_text=user_response,attempted_kind=derive_response_kind(ctx.turn_intent),@@ -1249,7 +1260,7 @@ async def _build_goal_satisfied_exit_result(ctx: CopilotContext, global_llm_contworkflow_yaml=verified_yaml,workflow_was_persisted=ctx.workflow_persisted,total_tokens=ctx.total_tokens_used,- proposal_disposition="auto_applicable",+ proposal_disposition="auto_applicable" if verified_workflow is not None else "no_proposal",turn_outcome=outcome,turn_id=ctx.turn_id,narrative_summary=ctx.narrative_summary,diff --git a/skyvern/forge/sdk/copilot/completion_verification.py b/skyvern/forge/sdk/copilot/completion_verification.pyindex 01a553e8e..e64affb60 100644--- a/skyvern/forge/sdk/copilot/completion_verification.py+++ b/skyvern/forge/sdk/copilot/completion_verification.py@@ -126,12 +126,24 @@ class RunEvidenceSnapshot:block_outputs: dict[str, Any] = field(default_factory=dict)current_url: str | None = Nonepage_title: str | None = None+ run_terminal_status: str | None = Noneexecuted_block_labels: list[str] = field(default_factory=list)verified_context_block_labels: list[str] = field(default_factory=list)+ failed_block_labels: list[str] = field(default_factory=list)+ failure_classes: list[str] = field(default_factory=list)+ failure_reasons: list[str] = field(default_factory=list)page_evidence: dict[str, Any] = field(default_factory=dict)def has_evidence(self) -> bool:- return bool(self.block_outputs or self.current_url or self.page_title or self.page_evidence)+ return bool(+ self.block_outputs+ or self.current_url+ or self.page_title+ or self.failed_block_labels+ or self.failure_classes+ or self.failure_reasons+ or self.page_evidence+ )def render_prompt_block(self) -> str:lines: list[str] = []@@ -141,12 +153,23 @@ class RunEvidenceSnapshot:lines.append(f"observed_end_state_url: {self.current_url}")if self.page_title:lines.append(f"observed_end_state_page_title: {self.page_title}")+ if self.run_terminal_status:+ lines.append(f"run_terminal_status: {self.run_terminal_status}")if self.verified_context_block_labels:lines.append("verified_context_block_labels: " + ", ".join(self.verified_context_block_labels[:_MAX_BLOCK_OUTPUTS]))if self.executed_block_labels:lines.append("executed_block_labels: " + ", ".join(self.executed_block_labels[:_MAX_BLOCK_OUTPUTS]))+ if self.failed_block_labels:+ lines.append("failed_block_labels: " + ", ".join(self.failed_block_labels[:_MAX_BLOCK_OUTPUTS]))+ if self.failure_classes:+ lines.append("failure_classes: " + ", ".join(self.failure_classes[:_MAX_BLOCK_OUTPUTS]))+ if self.failure_reasons:+ lines.append("failure_reasons:")+ for reason in self.failure_reasons[:_MAX_BLOCK_OUTPUTS]:+ rendered_reason = " ".join(str(reason).split())[:_EVIDENCE_VALUE_MAX_CHARS]+ lines.append(f" - {rendered_reason}")if self.block_outputs:lines.append("produced_block_outputs:")for label, payload in list(self.block_outputs.items())[:_MAX_BLOCK_OUTPUTS]:diff --git a/skyvern/forge/sdk/copilot/context.py b/skyvern/forge/sdk/copilot/context.pyindex bff83b40f..9c08a3e2a 100644--- a/skyvern/forge/sdk/copilot/context.py+++ b/skyvern/forge/sdk/copilot/context.py@@ -403,6 +403,9 @@ class CopilotContext(AgentContext):last_update_block_count: int | None = Nonelast_test_ok: bool | None = Nonelast_test_failure_reason: str | None = None+ last_artifact_health_blocker_reason: str | None = None+ last_artifact_health_blocker_labels: list[str] = field(default_factory=list)+ last_artifact_health_failure_classes: list[str] = field(default_factory=list)failed_test_nudge_count: int = 0explore_without_workflow_nudge_count: int = 0code_only_code_schema_seen: bool = Falsediff --git a/skyvern/forge/sdk/copilot/enforcement.py b/skyvern/forge/sdk/copilot/enforcement.pyindex b6a3dc8b3..99d09b0cb 100644--- a/skyvern/forge/sdk/copilot/enforcement.py+++ b/skyvern/forge/sdk/copilot/enforcement.py@@ -323,6 +323,11 @@ def _outcome_criteria_evaluated(ctx: CopilotContext) -> bool:return result is not None and result.status == "evaluated"+def artifact_health_blocked(ctx: CopilotContext) -> bool:+ reason = ctx.last_artifact_health_blocker_reason+ return isinstance(reason, str) and bool(reason.strip())++def outcome_fully_verified(ctx: CopilotContext) -> bool:"""The judge confirmed every outcome criterion from the evidence this run produced.@@ -330,6 +335,8 @@ def outcome_fully_verified(ctx: CopilotContext) -> bool:recognized even when it was canceled or only partially completed. Run status mustnever suppress recognition of an outcome the user can observe was achieved."""+ if artifact_health_blocked(ctx):+ return Falseif not _outcome_criteria_evaluated(ctx):return Falseresult = ctx.completion_verification_result@@ -377,6 +384,7 @@ def gate_decision_trace_fields(ctx: CopilotContext) -> dict[str, bool]:"gate_last_full_workflow_test_ok": ctx.last_full_workflow_test_ok is True,"gate_diagnosis_contract_satisfies_goal": latest_diagnosis_contract_satisfies_goal(ctx),"gate_outcome_criteria_evaluated": _outcome_criteria_evaluated(ctx),+ "gate_artifact_health_blocked": artifact_health_blocked(ctx),"gate_likely_needs_more_work": _verified_goal_likely_needs_more_work(ctx),"gate_evaluated_this_turn": True,}diff --git a/skyvern/forge/sdk/copilot/runtime.py b/skyvern/forge/sdk/copilot/runtime.pyindex 525684c29..a91853578 100644--- a/skyvern/forge/sdk/copilot/runtime.py+++ b/skyvern/forge/sdk/copilot/runtime.py@@ -223,6 +223,9 @@ class AgentContext:last_good_workflow: Any | None = Nonelast_good_workflow_yaml: str | None = Nonelast_run_blocks_workflow_run_id: str | None = None+ last_artifact_health_blocker_reason: str | None = None+ last_artifact_health_blocker_labels: list[str] = field(default_factory=list)+ last_artifact_health_failure_classes: list[str] = field(default_factory=list)last_run_blocks_block_ids: list[str] = field(default_factory=list)last_run_blocks_block_labels: list[str] = field(default_factory=list)last_run_outcome: RecordedRunOutcome | None = Nonediff --git a/skyvern/forge/sdk/copilot/tools/completion.py b/skyvern/forge/sdk/copilot/tools/completion.pyindex 5032f98a8..4ca20edb2 100644--- a/skyvern/forge/sdk/copilot/tools/completion.py+++ b/skyvern/forge/sdk/copilot/tools/completion.py@@ -19,6 +19,7 @@ from skyvern.forge.sdk.copilot.completion_verification import (from skyvern.forge.sdk.copilot.enforcement import _goal_likely_needs_more_blocksfrom skyvern.forge.sdk.copilot.llm_config import resolve_main_copilot_handlerfrom skyvern.forge.sdk.copilot.outcome_verification_trace import record_completion_verification+from skyvern.forge.sdk.copilot.output_utils import iter_failure_reasonsfrom skyvern.forge.sdk.copilot.request_policy import CompletionCriterionfrom skyvern.forge.sdk.copilot.tracing_setup import copilot_span@@ -27,6 +28,7 @@ from ._shared import (_copilot_seconds_remaining,_current_workflow_block_labels,_current_workflow_has_evidence_block,+ _failed_run_block_labels,_is_meaningful_extracted_data,_valid_runtime_anchor_url,)@@ -184,6 +186,28 @@ async def _maybe_run_completion_verification_from_page_observation(_COMPLETION_VERIFICATION_BUDGET_MARGIN_SECONDS = 5.0+_ARTIFACT_HEALTH_EXCLUDED_CATEGORIES = frozenset(+ {+ "ACTIVE_RUN_TERMINAL_EVIDENCE",+ "ANTI_BOT_DETECTION",+ "AUTH_FAILURE",+ "BROWSER_ERROR",+ "CREDENTIAL_ERROR",+ "INFRASTRUCTURE_ERROR",+ "NAVIGATION_FAILURE",+ "PER_TOOL_BUDGET",+ "PROXY_ERROR",+ }+)++_TYPE_ERROR_GENERATED_CALL_MARKERS = (+ "wait_for_function",+ "positional argument",+ "positional arguments",+ "got an unexpected keyword",+ "missing 1 required",+)+async def _completion_verification_handler(copilot_ctx: Any) -> Any | None:return await resolve_main_copilot_handler(@@ -228,6 +252,67 @@ def _is_unfinished_run_verification_candidate(copilot_ctx: Any, result: dict[strreturn _valid_runtime_anchor_url(data.get("current_url")) is not None+def _failure_category_names(result: dict[str, Any]) -> list[str]:+ data = result.get("data")+ data = data if isinstance(data, dict) else {}+ raw_categories = data.get("failure_categories")+ if not isinstance(raw_categories, list):+ return []+ categories: list[str] = []+ for item in raw_categories:+ if not isinstance(item, dict):+ continue+ category = item.get("category")+ if isinstance(category, str) and category.strip():+ categories.append(category.strip())+ return list(dict.fromkeys(categories))+++def _artifact_health_failure_class(reason: str) -> str | None:+ lowered = reason.lower()+ if "syntaxerror" in lowered:+ return "SyntaxError"+ if "nameerror" in lowered:+ return "NameError"+ if "typeerror" in lowered and any(marker in lowered for marker in _TYPE_ERROR_GENERATED_CALL_MARKERS):+ return "TypeError"+ return None+++def _artifact_health_blocker_from_result(+ result: dict[str, Any], failure_reasons: list[str] | None = None+) -> tuple[str | None, list[str], list[str]]:+ if bool(result.get("ok", False)):+ return None, [], []+ categories = _failure_category_names(result)+ if categories and all(category in _ARTIFACT_HEALTH_EXCLUDED_CATEGORIES for category in categories):+ return None, [], []++ failure_reasons = list(+ dict.fromkeys(failure_reasons if failure_reasons is not None else iter_failure_reasons(result))+ )+ failure_classes = [+ failure_class+ for failure_class in dict.fromkeys(+ _artifact_health_failure_class(reason) for reason in failure_reasons if isinstance(reason, str)+ )+ if failure_class is not None+ ]+ if not failure_classes:+ return None, [], []++ data = result.get("data")+ data = data if isinstance(data, dict) else {}+ failed_labels = _failed_run_block_labels(data)+ label_detail = f" in block(s) {', '.join(failed_labels)}" if failed_labels else ""+ reason_preview = " ".join(str(failure_reasons[0]).split())[:240] if failure_reasons else "unknown failure"+ reason = (+ f"Artifact-health blocker{label_detail}: deterministic generated-code/runtime "+ f"{'/'.join(failure_classes)} failure: {reason_preview}"+ )+ return reason, failed_labels, failure_classes++def _build_run_evidence_snapshot(copilot_ctx: Any, result: dict[str, Any]) -> RunEvidenceSnapshot:data = result.get("data")data = data if isinstance(data, dict) else {}@@ -250,17 +335,30 @@ def _build_run_evidence_snapshot(copilot_ctx: Any, result: dict[str, Any]) -> Ruexecuted_block_labels = [str(label) for label in executed] if isinstance(executed, list) else []page_title = data.get("page_title")run_id = data.get("workflow_run_id")+ run_terminal_status = data.get("overall_status")+ failure_reasons = [" ".join(reason.split()) for reason in iter_failure_reasons(result)]+ _artifact_reason, artifact_failed_labels, artifact_failure_classes = _artifact_health_blocker_from_result(+ result,+ failure_reasons=failure_reasons,+ )+ failed_block_labels = artifact_failed_labels or _failed_run_block_labels(data)return RunEvidenceSnapshot(workflow_run_id=run_id if isinstance(run_id, str) else None,block_outputs=block_outputs,current_url=_valid_runtime_anchor_url(data.get("current_url")),page_title=page_title if isinstance(page_title, str) and page_title.strip() else None,+ run_terminal_status=run_terminal_status+ if isinstance(run_terminal_status, str) and run_terminal_status+ else None,executed_block_labels=executed_block_labels,verified_context_block_labels=_verified_context_block_labels_for_snapshot(copilot_ctx,current_label_order,executed_block_labels,),+ failed_block_labels=failed_block_labels,+ failure_classes=artifact_failure_classes,+ failure_reasons=failure_reasons,)diff --git a/skyvern/forge/sdk/copilot/tools/run_execution.py b/skyvern/forge/sdk/copilot/tools/run_execution.pyindex 9a4d27f3f..bb71e38ca 100644--- a/skyvern/forge/sdk/copilot/tools/run_execution.py+++ b/skyvern/forge/sdk/copilot/tools/run_execution.py@@ -109,6 +109,7 @@ from .blockers import (_trusted_post_drain_status,)from .completion import (+ _artifact_health_blocker_from_result,_emit_completion_verification_trace,_maybe_run_completion_verification,_outcome_failure_warrants_repair,@@ -1562,6 +1563,9 @@ def _record_run_blocks_result(copilot_ctx.last_full_workflow_test_ok = Falsecopilot_ctx.last_unverified_block_labels = _unverified_current_workflow_labels(copilot_ctx)copilot_ctx.last_test_failure_reason = None+ copilot_ctx.last_artifact_health_blocker_reason = None+ copilot_ctx.last_artifact_health_blocker_labels = []+ copilot_ctx.last_artifact_health_failure_classes = []if completion_verification is not None and completion_verification.status == "evaluated":copilot_ctx.last_outcome_gate_reason = _outcome_unverified_reason(copilot_ctx, completion_verification)copilot_ctx.last_outcome_gate_workflow_run_id = copilot_ctx.last_run_blocks_workflow_run_id@@ -1618,6 +1622,19 @@ def _record_run_blocks_result(if isinstance(data, dict):data["failure_categories"] = failure_categories+ artifact_reason, artifact_labels, artifact_classes = _artifact_health_blocker_from_result(result)+ if artifact_reason is not None:+ copilot_ctx.last_artifact_health_blocker_reason = artifact_reason+ copilot_ctx.last_artifact_health_blocker_labels = artifact_labels+ copilot_ctx.last_artifact_health_failure_classes = artifact_classes+ data = result.get("data")+ if isinstance(data, dict):+ data["artifact_health_blocker"] = {+ "reason": artifact_reason,+ "failed_block_labels": artifact_labels,+ "failure_classes": artifact_classes,+ }+if _active_run_terminal_evidence_detected(result):_update_verification_evidence_from_run_result(copilot_ctx, result)signal = _active_run_terminal_evidence_signal(copilot_ctx, "update_and_run_blocks")diff --git a/tests/unit/test_copilot_agent_helpers.py b/tests/unit/test_copilot_agent_helpers.pyindex 5afe35381..5ec85ea0c 100644--- a/tests/unit/test_copilot_agent_helpers.py+++ b/tests/unit/test_copilot_agent_helpers.py@@ -874,6 +874,60 @@ class TestVerifiedGoalSatisfiedStop:assert adjudication["criteriaLifecycleReason"] == "not_subset"assert adjudication["satisfiedCount"] == 2+ @pytest.mark.asyncio+ async def test_goal_satisfied_exit_result_does_not_claim_success_after_failed_test(self) -> None:+ from skyvern.forge.sdk.copilot.agent import _build_goal_satisfied_exit_result+ from skyvern.forge.sdk.copilot.completion_verification import CompletionVerificationResult, CriterionVerdict++ workflow = object()+ ctx = _ctx(+ last_workflow=workflow,+ last_workflow_yaml="workflow_definition:\n blocks: []\n",+ last_test_ok=False,+ last_full_workflow_test_ok=False,+ last_artifact_health_blocker_reason=(+ "Artifact-health blocker in block(s) extract_results: deterministic generated-code/runtime SyntaxError"+ ),+ last_artifact_health_blocker_labels=["extract_results"],+ last_artifact_health_failure_classes=["SyntaxError"],+ completion_verification_result=CompletionVerificationResult(+ status="evaluated",+ criterion_ids=["c0"],+ verdicts=[CriterionVerdict(criterion_id="c0", state="satisfied", reason_code="evidence_confirms")],+ ),+ tool_activity=[{"tool": "update_and_run_blocks", "summary": "failed"}],+ )++ result = await _build_goal_satisfied_exit_result(ctx, global_llm_context=None)++ assert "tested successfully" not in result.user_response.lower()+ assert "did not finish successfully" in result.user_response.lower()+ assert result.updated_workflow is None+ assert result.proposal_disposition == "no_proposal"++ @pytest.mark.asyncio+ async def test_goal_satisfied_exit_result_does_not_claim_failed_test_when_not_tested(self) -> None:+ from skyvern.forge.sdk.copilot.agent import _build_goal_satisfied_exit_result+ from skyvern.forge.sdk.copilot.completion_verification import CompletionVerificationResult, CriterionVerdict++ ctx = _ctx(+ last_workflow=None,+ last_workflow_yaml=None,+ last_test_ok=None,+ last_full_workflow_test_ok=False,+ completion_verification_result=CompletionVerificationResult(+ status="evaluated",+ criterion_ids=["c0"],+ verdicts=[CriterionVerdict(criterion_id="c0", state="satisfied", reason_code="evidence_confirms")],+ ),+ )++ result = await _build_goal_satisfied_exit_result(ctx, global_llm_context=None)++ assert "tested successfully" not in result.user_response.lower()+ assert "did not finish successfully" not in result.user_response.lower()+ assert "not been tested end-to-end" in result.user_response.lower()+class TestSupersededAgentIntentGates:def test_agent_no_longer_owns_request_policy_classification(self) -> None:diff --git a/tests/unit/test_copilot_completion_verification.py b/tests/unit/test_copilot_completion_verification.pyindex 847ede98e..ce37bea81 100644--- a/tests/unit/test_copilot_completion_verification.py+++ b/tests/unit/test_copilot_completion_verification.py@@ -57,6 +57,7 @@ from skyvern.forge.sdk.copilot.tools import (_tool_visible_result_after_completion_verification,_watchdog_exit_allows_terminal_promotion,)+from skyvern.forge.sdk.copilot.tools.completion import _artifact_health_blocker_from_resultdef _criterion(cid: str, outcome: str, *, method_mandated: bool = False) -> CompletionCriterion:@@ -267,8 +268,12 @@ async def test_evaluate_happy_path_returns_evaluated() -> None:def test_snapshot_has_evidence() -> None:assert RunEvidenceSnapshot().has_evidence() is False+ assert RunEvidenceSnapshot(run_terminal_status="failed").has_evidence() is Falseassert RunEvidenceSnapshot(current_url="https://example.com").has_evidence() is Trueassert RunEvidenceSnapshot(block_outputs={"a": 1}).has_evidence() is True+ assert RunEvidenceSnapshot(failed_block_labels=["extract"]).has_evidence() is True+ assert RunEvidenceSnapshot(failure_classes=["SyntaxError"]).has_evidence() is True+ assert RunEvidenceSnapshot(failure_reasons=["SyntaxError: bad generated code"]).has_evidence() is Trueassert RunEvidenceSnapshot(page_evidence={"visible_text_excerpt": "cart item PART-001-TEST"}).has_evidence() is True@@ -301,6 +306,25 @@ def test_snapshot_renders_bounded_page_evidence() -> None:assert "raw_html" not in rendered+def test_snapshot_renders_failed_run_artifact_health_signal() -> None:+ snapshot = RunEvidenceSnapshot(+ workflow_run_id="wr_failed",+ block_outputs={"extract_results": {"extracted_information": ["goal text"]}},+ current_url="https://example.com/results",+ run_terminal_status="failed",+ failed_block_labels=["extract_results"],+ failure_classes=["SyntaxError"],+ failure_reasons=["Page.evaluate: SyntaxError: Unexpected token ')'"],+ )++ rendered = snapshot.render_prompt_block()++ assert "run_terminal_status: failed" in rendered+ assert "failed_block_labels: extract_results" in rendered+ assert "failure_classes: SyntaxError" in rendered+ assert "Page.evaluate: SyntaxError" in rendered++def test_active_run_terminal_visual_fallback_uses_screenshot_when_missing() -> None:assert (_active_run_terminal_evidence_needs_visual_fallback(@@ -460,7 +484,7 @@ def test_gate_withholds_on_evaluated_unconfirmed_even_with_clean_run_status() ->def test_completion_contract_not_violated() -> None:- ctx = SimpleNamespace(completion_verification_result=None)+ ctx = SimpleNamespace(completion_verification_result=None, last_artifact_health_blocker_reason=None)assert _completion_contract_not_violated(ctx) is True # type: ignore[arg-type]ctx.completion_verification_result = _evaluated(("c0", True))assert _completion_contract_not_violated(ctx) is True # type: ignore[arg-type]@@ -829,6 +853,118 @@ def _canceled_gate_ctx() -> CopilotContext:return ctx+def _failed_generated_code_result() -> dict:+ return {+ "ok": False,+ "data": {+ "workflow_run_id": "wr_failed_code",+ "overall_status": "failed",+ "executed_block_labels": ["extract_results"],+ "current_url": "https://example.com/results",+ "blocks": [+ {+ "label": "extract_results",+ "block_type": "EXTRACTION",+ "status": "failed",+ "extracted_data": {"extracted_information": ["goal text from partial output"]},+ "failure_reason": "Page.evaluate: SyntaxError: Unexpected token ')'",+ }+ ],+ },+ }+++def test_artifact_health_type_error_is_not_masked_by_timeout_category() -> None:+ result = {+ "ok": False,+ "data": {+ "workflow_run_id": "wr_failed_code",+ "overall_status": "failed",+ "failure_categories": [+ {+ "category": "PAGE_LOAD_TIMEOUT",+ "confidence_float": 0.8,+ "reasoning": "Timeout in failure reason",+ }+ ],+ "blocks": [+ {+ "label": "wait_for_results",+ "block_type": "ACTION",+ "status": "failed",+ "failure_reason": (+ "TypeError: Page.wait_for_function() got an unexpected keyword argument 'timeout_ms'"+ ),+ }+ ],+ },+ }++ reason, failed_labels, failure_classes = _artifact_health_blocker_from_result(result)++ assert reason is not None+ assert "TypeError" in reason+ assert failed_labels == ["wait_for_results"]+ assert failure_classes == ["TypeError"]+++def test_artifact_health_not_masked_by_mixed_excluded_category() -> None:+ result = {+ "ok": False,+ "data": {+ "workflow_run_id": "wr_failed_code",+ "overall_status": "failed",+ "failure_categories": [+ {"category": "AUTH_FAILURE", "confidence_float": 0.8},+ {"category": "SCRIPT_ERROR", "confidence_float": 0.9},+ ],+ "blocks": [+ {+ "label": "extract_results",+ "block_type": "EXTRACTION",+ "status": "failed",+ "failure_reason": "Page.evaluate: SyntaxError: Unexpected token ')'",+ }+ ],+ },+ }++ reason, failed_labels, failure_classes = _artifact_health_blocker_from_result(result)++ assert reason is not None+ assert "SyntaxError" in reason+ assert failed_labels == ["extract_results"]+ assert failure_classes == ["SyntaxError"]+++def test_artifact_health_skips_when_all_categories_are_excluded() -> None:+ result = {+ "ok": False,+ "data": {+ "workflow_run_id": "wr_auth_failed",+ "overall_status": "failed",+ "failure_categories": [+ {"category": "AUTH_FAILURE", "confidence_float": 0.8},+ {"category": "CREDENTIAL_ERROR", "confidence_float": 0.7},+ ],+ "blocks": [+ {+ "label": "extract_results",+ "block_type": "EXTRACTION",+ "status": "failed",+ "failure_reason": "Page.evaluate: SyntaxError: Unexpected token ')'",+ }+ ],+ },+ }++ reason, failed_labels, failure_classes = _artifact_health_blocker_from_result(result)++ assert reason is None+ assert failed_labels == []+ assert failure_classes == []++def test_unfinished_run_verification_candidate_admits_canceled_with_evidence() -> None:ctx = _run_ctx()assert _is_unfinished_run_verification_candidate(ctx, _canceled_budget_result()) is True@@ -838,6 +974,29 @@ def test_unfinished_run_verification_candidate_admits_canceled_with_evidence() -assert _is_unfinished_run_verification_candidate(ctx, {"ok": False, "data": {}}) is False+def test_artifact_health_blocks_fully_satisfied_failed_run() -> None:+ result = _failed_generated_code_result()+ ctx = _run_ctx()+ ctx.last_workflow = SimpleNamespace(workflow_definition=SimpleNamespace(blocks=[]))+ ctx.last_workflow_yaml = "workflow: {}"++ snapshot = _build_run_evidence_snapshot(ctx, result)+ rendered = snapshot.render_prompt_block()+ assert "run_terminal_status: failed" in rendered+ assert "failure_classes: SyntaxError" in rendered+ assert "failed_block_labels: extract_results" in rendered++ _record_run_blocks_result(ctx, result, completion_verification=_evaluated(("c0", True)))++ assert ctx.last_artifact_health_blocker_reason is not None+ assert "SyntaxError" in ctx.last_artifact_health_blocker_reason+ assert ctx.last_artifact_health_blocker_labels == ["extract_results"]+ assert ctx.last_artifact_health_failure_classes == ["SyntaxError"]+ assert outcome_fully_verified(ctx) is False+ assert verified_goal_satisfied_context(ctx) is False+ assert _verified_workflow_or_none(ctx) == (None, None)++@pytest.mark.asyncioasync def test_maybe_run_completion_verification_runs_on_canceled_run(monkeypatch: pytest.MonkeyPatch) -> None:async def handler(**_: object) -> dict:@@ -1573,7 +1732,10 @@ async def test_maybe_run_completion_verification_unavailable_on_low_budget(monkedef test_completion_contract_not_violated_unavailable_blocks_surfacing() -> None:- ctx = SimpleNamespace(completion_verification_result=CompletionVerificationResult("unavailable"))+ ctx = SimpleNamespace(+ completion_verification_result=CompletionVerificationResult("unavailable"),+ last_artifact_health_blocker_reason=None,+ )# An unavailable verdict means the outcome could not be verified: do not surface# the workflow as verified on run status alone.assert _completion_contract_not_violated(ctx) is False # type: ignore[arg-type]