Skip to content

Automate browser based workflows with AI

SKY-10964 Block verified success on artifact-health failures (#6672)

Andrew Neilson5e66b1692d29
diff --git a/skyvern/forge/sdk/copilot/agent.py b/skyvern/forge/sdk/copilot/agent.py
index 43283a441..c522c08bd 100644
--- a/skyvern/forge/sdk/copilot/agent.py
+++ b/skyvern/forge/sdk/copilot/agent.py
@@ -65,6 +65,7 @@ from skyvern.forge.sdk.copilot.context import (
finalize_discovery_counter_in_global_llm_context,
)
from skyvern.forge.sdk.copilot.enforcement import (
+ artifact_health_blocked,
outcome_fully_verified,
verified_goal_claim_authorized,
)
@@ -908,6 +909,8 @@ def _shape_ask_question_response(user_response: str, ctx: CopilotContext) -> str
def _completion_contract_not_violated(ctx: CopilotContext) -> bool:
+ if artifact_health_blocked(ctx):
+ return False
result = ctx.completion_verification_result
if result is None:
return True
@@ -1180,13 +1183,21 @@ async def _build_goal_satisfied_exit_result(ctx: CopilotContext, global_llm_cont
except Exception as flush_err:
LOG.warning("copilot_goal_satisfied_tool_result_flush_failed", error=str(flush_err))
verified_workflow, verified_yaml = _verified_workflow_or_none(ctx)
- if verified_goal_claim_authorized(ctx):
+ clean_test = ctx.last_test_ok is True and ctx.last_full_workflow_test_ok is True
+ if clean_test and verified_goal_claim_authorized(ctx):
user_response = "I created and tested the workflow successfully."
- else:
+ elif clean_test:
user_response = (
"I built the workflow and the test run completed, but the goal outcome was not "
"independently verified. Review the draft to confirm it does what you need."
)
+ elif ctx.last_test_ok is False:
+ user_response = "I reached the requested outcome, but the workflow test did not finish successfully."
+ else:
+ user_response = (
+ "I reached the requested outcome, but the workflow has not been tested end-to-end. "
+ "Review the draft before using it."
+ )
final_text, outcome = apply_repeated_reply_guard(
final_text=user_response,
attempted_kind=derive_response_kind(ctx.turn_intent),
@@ -1249,7 +1260,7 @@ async def _build_goal_satisfied_exit_result(ctx: CopilotContext, global_llm_cont
workflow_yaml=verified_yaml,
workflow_was_persisted=ctx.workflow_persisted,
total_tokens=ctx.total_tokens_used,
- proposal_disposition="auto_applicable",
+ proposal_disposition="auto_applicable" if verified_workflow is not None else "no_proposal",
turn_outcome=outcome,
turn_id=ctx.turn_id,
narrative_summary=ctx.narrative_summary,
diff --git a/skyvern/forge/sdk/copilot/completion_verification.py b/skyvern/forge/sdk/copilot/completion_verification.py
index 01a553e8e..e64affb60 100644
--- a/skyvern/forge/sdk/copilot/completion_verification.py
+++ b/skyvern/forge/sdk/copilot/completion_verification.py
@@ -126,12 +126,24 @@ class RunEvidenceSnapshot:
block_outputs: dict[str, Any] = field(default_factory=dict)
current_url: str | None = None
page_title: str | None = None
+ run_terminal_status: str | None = None
executed_block_labels: list[str] = field(default_factory=list)
verified_context_block_labels: list[str] = field(default_factory=list)
+ failed_block_labels: list[str] = field(default_factory=list)
+ failure_classes: list[str] = field(default_factory=list)
+ failure_reasons: list[str] = field(default_factory=list)
page_evidence: dict[str, Any] = field(default_factory=dict)
def has_evidence(self) -> bool:
- return bool(self.block_outputs or self.current_url or self.page_title or self.page_evidence)
+ return bool(
+ self.block_outputs
+ or self.current_url
+ or self.page_title
+ or self.failed_block_labels
+ or self.failure_classes
+ or self.failure_reasons
+ or self.page_evidence
+ )
def render_prompt_block(self) -> str:
lines: list[str] = []
@@ -141,12 +153,23 @@ class RunEvidenceSnapshot:
lines.append(f"observed_end_state_url: {self.current_url}")
if self.page_title:
lines.append(f"observed_end_state_page_title: {self.page_title}")
+ if self.run_terminal_status:
+ lines.append(f"run_terminal_status: {self.run_terminal_status}")
if self.verified_context_block_labels:
lines.append(
"verified_context_block_labels: " + ", ".join(self.verified_context_block_labels[:_MAX_BLOCK_OUTPUTS])
)
if self.executed_block_labels:
lines.append("executed_block_labels: " + ", ".join(self.executed_block_labels[:_MAX_BLOCK_OUTPUTS]))
+ if self.failed_block_labels:
+ lines.append("failed_block_labels: " + ", ".join(self.failed_block_labels[:_MAX_BLOCK_OUTPUTS]))
+ if self.failure_classes:
+ lines.append("failure_classes: " + ", ".join(self.failure_classes[:_MAX_BLOCK_OUTPUTS]))
+ if self.failure_reasons:
+ lines.append("failure_reasons:")
+ for reason in self.failure_reasons[:_MAX_BLOCK_OUTPUTS]:
+ rendered_reason = " ".join(str(reason).split())[:_EVIDENCE_VALUE_MAX_CHARS]
+ lines.append(f" - {rendered_reason}")
if self.block_outputs:
lines.append("produced_block_outputs:")
for label, payload in list(self.block_outputs.items())[:_MAX_BLOCK_OUTPUTS]:
diff --git a/skyvern/forge/sdk/copilot/context.py b/skyvern/forge/sdk/copilot/context.py
index bff83b40f..9c08a3e2a 100644
--- a/skyvern/forge/sdk/copilot/context.py
+++ b/skyvern/forge/sdk/copilot/context.py
@@ -403,6 +403,9 @@ class CopilotContext(AgentContext):
last_update_block_count: int | None = None
last_test_ok: bool | None = None
last_test_failure_reason: str | None = None
+ last_artifact_health_blocker_reason: str | None = None
+ last_artifact_health_blocker_labels: list[str] = field(default_factory=list)
+ last_artifact_health_failure_classes: list[str] = field(default_factory=list)
failed_test_nudge_count: int = 0
explore_without_workflow_nudge_count: int = 0
code_only_code_schema_seen: bool = False
diff --git a/skyvern/forge/sdk/copilot/enforcement.py b/skyvern/forge/sdk/copilot/enforcement.py
index b6a3dc8b3..99d09b0cb 100644
--- a/skyvern/forge/sdk/copilot/enforcement.py
+++ b/skyvern/forge/sdk/copilot/enforcement.py
@@ -323,6 +323,11 @@ def _outcome_criteria_evaluated(ctx: CopilotContext) -> bool:
return result is not None and result.status == "evaluated"
+def artifact_health_blocked(ctx: CopilotContext) -> bool:
+ reason = ctx.last_artifact_health_blocker_reason
+ return isinstance(reason, str) and bool(reason.strip())
+
+
def outcome_fully_verified(ctx: CopilotContext) -> bool:
"""The judge confirmed every outcome criterion from the evidence this run produced.
@@ -330,6 +335,8 @@ def outcome_fully_verified(ctx: CopilotContext) -> bool:
recognized even when it was canceled or only partially completed. Run status must
never suppress recognition of an outcome the user can observe was achieved.
"""
+ if artifact_health_blocked(ctx):
+ return False
if not _outcome_criteria_evaluated(ctx):
return False
result = ctx.completion_verification_result
@@ -377,6 +384,7 @@ def gate_decision_trace_fields(ctx: CopilotContext) -> dict[str, bool]:
"gate_last_full_workflow_test_ok": ctx.last_full_workflow_test_ok is True,
"gate_diagnosis_contract_satisfies_goal": latest_diagnosis_contract_satisfies_goal(ctx),
"gate_outcome_criteria_evaluated": _outcome_criteria_evaluated(ctx),
+ "gate_artifact_health_blocked": artifact_health_blocked(ctx),
"gate_likely_needs_more_work": _verified_goal_likely_needs_more_work(ctx),
"gate_evaluated_this_turn": True,
}
diff --git a/skyvern/forge/sdk/copilot/runtime.py b/skyvern/forge/sdk/copilot/runtime.py
index 525684c29..a91853578 100644
--- a/skyvern/forge/sdk/copilot/runtime.py
+++ b/skyvern/forge/sdk/copilot/runtime.py
@@ -223,6 +223,9 @@ class AgentContext:
last_good_workflow: Any | None = None
last_good_workflow_yaml: str | None = None
last_run_blocks_workflow_run_id: str | None = None
+ last_artifact_health_blocker_reason: str | None = None
+ last_artifact_health_blocker_labels: list[str] = field(default_factory=list)
+ last_artifact_health_failure_classes: list[str] = field(default_factory=list)
last_run_blocks_block_ids: list[str] = field(default_factory=list)
last_run_blocks_block_labels: list[str] = field(default_factory=list)
last_run_outcome: RecordedRunOutcome | None = None
diff --git a/skyvern/forge/sdk/copilot/tools/completion.py b/skyvern/forge/sdk/copilot/tools/completion.py
index 5032f98a8..4ca20edb2 100644
--- a/skyvern/forge/sdk/copilot/tools/completion.py
+++ b/skyvern/forge/sdk/copilot/tools/completion.py
@@ -19,6 +19,7 @@ from skyvern.forge.sdk.copilot.completion_verification import (
from skyvern.forge.sdk.copilot.enforcement import _goal_likely_needs_more_blocks
from skyvern.forge.sdk.copilot.llm_config import resolve_main_copilot_handler
from skyvern.forge.sdk.copilot.outcome_verification_trace import record_completion_verification
+from skyvern.forge.sdk.copilot.output_utils import iter_failure_reasons
from skyvern.forge.sdk.copilot.request_policy import CompletionCriterion
from skyvern.forge.sdk.copilot.tracing_setup import copilot_span
@@ -27,6 +28,7 @@ from ._shared import (
_copilot_seconds_remaining,
_current_workflow_block_labels,
_current_workflow_has_evidence_block,
+ _failed_run_block_labels,
_is_meaningful_extracted_data,
_valid_runtime_anchor_url,
)
@@ -184,6 +186,28 @@ async def _maybe_run_completion_verification_from_page_observation(
_COMPLETION_VERIFICATION_BUDGET_MARGIN_SECONDS = 5.0
+_ARTIFACT_HEALTH_EXCLUDED_CATEGORIES = frozenset(
+ {
+ "ACTIVE_RUN_TERMINAL_EVIDENCE",
+ "ANTI_BOT_DETECTION",
+ "AUTH_FAILURE",
+ "BROWSER_ERROR",
+ "CREDENTIAL_ERROR",
+ "INFRASTRUCTURE_ERROR",
+ "NAVIGATION_FAILURE",
+ "PER_TOOL_BUDGET",
+ "PROXY_ERROR",
+ }
+)
+
+_TYPE_ERROR_GENERATED_CALL_MARKERS = (
+ "wait_for_function",
+ "positional argument",
+ "positional arguments",
+ "got an unexpected keyword",
+ "missing 1 required",
+)
+
async def _completion_verification_handler(copilot_ctx: Any) -> Any | None:
return await resolve_main_copilot_handler(
@@ -228,6 +252,67 @@ def _is_unfinished_run_verification_candidate(copilot_ctx: Any, result: dict[str
return _valid_runtime_anchor_url(data.get("current_url")) is not None
+def _failure_category_names(result: dict[str, Any]) -> list[str]:
+ data = result.get("data")
+ data = data if isinstance(data, dict) else {}
+ raw_categories = data.get("failure_categories")
+ if not isinstance(raw_categories, list):
+ return []
+ categories: list[str] = []
+ for item in raw_categories:
+ if not isinstance(item, dict):
+ continue
+ category = item.get("category")
+ if isinstance(category, str) and category.strip():
+ categories.append(category.strip())
+ return list(dict.fromkeys(categories))
+
+
+def _artifact_health_failure_class(reason: str) -> str | None:
+ lowered = reason.lower()
+ if "syntaxerror" in lowered:
+ return "SyntaxError"
+ if "nameerror" in lowered:
+ return "NameError"
+ if "typeerror" in lowered and any(marker in lowered for marker in _TYPE_ERROR_GENERATED_CALL_MARKERS):
+ return "TypeError"
+ return None
+
+
+def _artifact_health_blocker_from_result(
+ result: dict[str, Any], failure_reasons: list[str] | None = None
+) -> tuple[str | None, list[str], list[str]]:
+ if bool(result.get("ok", False)):
+ return None, [], []
+ categories = _failure_category_names(result)
+ if categories and all(category in _ARTIFACT_HEALTH_EXCLUDED_CATEGORIES for category in categories):
+ return None, [], []
+
+ failure_reasons = list(
+ dict.fromkeys(failure_reasons if failure_reasons is not None else iter_failure_reasons(result))
+ )
+ failure_classes = [
+ failure_class
+ for failure_class in dict.fromkeys(
+ _artifact_health_failure_class(reason) for reason in failure_reasons if isinstance(reason, str)
+ )
+ if failure_class is not None
+ ]
+ if not failure_classes:
+ return None, [], []
+
+ data = result.get("data")
+ data = data if isinstance(data, dict) else {}
+ failed_labels = _failed_run_block_labels(data)
+ label_detail = f" in block(s) {', '.join(failed_labels)}" if failed_labels else ""
+ reason_preview = " ".join(str(failure_reasons[0]).split())[:240] if failure_reasons else "unknown failure"
+ reason = (
+ f"Artifact-health blocker{label_detail}: deterministic generated-code/runtime "
+ f"{'/'.join(failure_classes)} failure: {reason_preview}"
+ )
+ return reason, failed_labels, failure_classes
+
+
def _build_run_evidence_snapshot(copilot_ctx: Any, result: dict[str, Any]) -> RunEvidenceSnapshot:
data = result.get("data")
data = data if isinstance(data, dict) else {}
@@ -250,17 +335,30 @@ def _build_run_evidence_snapshot(copilot_ctx: Any, result: dict[str, Any]) -> Ru
executed_block_labels = [str(label) for label in executed] if isinstance(executed, list) else []
page_title = data.get("page_title")
run_id = data.get("workflow_run_id")
+ run_terminal_status = data.get("overall_status")
+ failure_reasons = [" ".join(reason.split()) for reason in iter_failure_reasons(result)]
+ _artifact_reason, artifact_failed_labels, artifact_failure_classes = _artifact_health_blocker_from_result(
+ result,
+ failure_reasons=failure_reasons,
+ )
+ failed_block_labels = artifact_failed_labels or _failed_run_block_labels(data)
return RunEvidenceSnapshot(
workflow_run_id=run_id if isinstance(run_id, str) else None,
block_outputs=block_outputs,
current_url=_valid_runtime_anchor_url(data.get("current_url")),
page_title=page_title if isinstance(page_title, str) and page_title.strip() else None,
+ run_terminal_status=run_terminal_status
+ if isinstance(run_terminal_status, str) and run_terminal_status
+ else None,
executed_block_labels=executed_block_labels,
verified_context_block_labels=_verified_context_block_labels_for_snapshot(
copilot_ctx,
current_label_order,
executed_block_labels,
),
+ failed_block_labels=failed_block_labels,
+ failure_classes=artifact_failure_classes,
+ failure_reasons=failure_reasons,
)
diff --git a/skyvern/forge/sdk/copilot/tools/run_execution.py b/skyvern/forge/sdk/copilot/tools/run_execution.py
index 9a4d27f3f..bb71e38ca 100644
--- a/skyvern/forge/sdk/copilot/tools/run_execution.py
+++ b/skyvern/forge/sdk/copilot/tools/run_execution.py
@@ -109,6 +109,7 @@ from .blockers import (
_trusted_post_drain_status,
)
from .completion import (
+ _artifact_health_blocker_from_result,
_emit_completion_verification_trace,
_maybe_run_completion_verification,
_outcome_failure_warrants_repair,
@@ -1562,6 +1563,9 @@ def _record_run_blocks_result(
copilot_ctx.last_full_workflow_test_ok = False
copilot_ctx.last_unverified_block_labels = _unverified_current_workflow_labels(copilot_ctx)
copilot_ctx.last_test_failure_reason = None
+ copilot_ctx.last_artifact_health_blocker_reason = None
+ copilot_ctx.last_artifact_health_blocker_labels = []
+ copilot_ctx.last_artifact_health_failure_classes = []
if completion_verification is not None and completion_verification.status == "evaluated":
copilot_ctx.last_outcome_gate_reason = _outcome_unverified_reason(copilot_ctx, completion_verification)
copilot_ctx.last_outcome_gate_workflow_run_id = copilot_ctx.last_run_blocks_workflow_run_id
@@ -1618,6 +1622,19 @@ def _record_run_blocks_result(
if isinstance(data, dict):
data["failure_categories"] = failure_categories
+ artifact_reason, artifact_labels, artifact_classes = _artifact_health_blocker_from_result(result)
+ if artifact_reason is not None:
+ copilot_ctx.last_artifact_health_blocker_reason = artifact_reason
+ copilot_ctx.last_artifact_health_blocker_labels = artifact_labels
+ copilot_ctx.last_artifact_health_failure_classes = artifact_classes
+ data = result.get("data")
+ if isinstance(data, dict):
+ data["artifact_health_blocker"] = {
+ "reason": artifact_reason,
+ "failed_block_labels": artifact_labels,
+ "failure_classes": artifact_classes,
+ }
+
if _active_run_terminal_evidence_detected(result):
_update_verification_evidence_from_run_result(copilot_ctx, result)
signal = _active_run_terminal_evidence_signal(copilot_ctx, "update_and_run_blocks")
diff --git a/tests/unit/test_copilot_agent_helpers.py b/tests/unit/test_copilot_agent_helpers.py
index 5afe35381..5ec85ea0c 100644
--- a/tests/unit/test_copilot_agent_helpers.py
+++ b/tests/unit/test_copilot_agent_helpers.py
@@ -874,6 +874,60 @@ class TestVerifiedGoalSatisfiedStop:
assert adjudication["criteriaLifecycleReason"] == "not_subset"
assert adjudication["satisfiedCount"] == 2
+ @pytest.mark.asyncio
+ async def test_goal_satisfied_exit_result_does_not_claim_success_after_failed_test(self) -> None:
+ from skyvern.forge.sdk.copilot.agent import _build_goal_satisfied_exit_result
+ from skyvern.forge.sdk.copilot.completion_verification import CompletionVerificationResult, CriterionVerdict
+
+ workflow = object()
+ ctx = _ctx(
+ last_workflow=workflow,
+ last_workflow_yaml="workflow_definition:\n blocks: []\n",
+ last_test_ok=False,
+ last_full_workflow_test_ok=False,
+ last_artifact_health_blocker_reason=(
+ "Artifact-health blocker in block(s) extract_results: deterministic generated-code/runtime SyntaxError"
+ ),
+ last_artifact_health_blocker_labels=["extract_results"],
+ last_artifact_health_failure_classes=["SyntaxError"],
+ completion_verification_result=CompletionVerificationResult(
+ status="evaluated",
+ criterion_ids=["c0"],
+ verdicts=[CriterionVerdict(criterion_id="c0", state="satisfied", reason_code="evidence_confirms")],
+ ),
+ tool_activity=[{"tool": "update_and_run_blocks", "summary": "failed"}],
+ )
+
+ result = await _build_goal_satisfied_exit_result(ctx, global_llm_context=None)
+
+ assert "tested successfully" not in result.user_response.lower()
+ assert "did not finish successfully" in result.user_response.lower()
+ assert result.updated_workflow is None
+ assert result.proposal_disposition == "no_proposal"
+
+ @pytest.mark.asyncio
+ async def test_goal_satisfied_exit_result_does_not_claim_failed_test_when_not_tested(self) -> None:
+ from skyvern.forge.sdk.copilot.agent import _build_goal_satisfied_exit_result
+ from skyvern.forge.sdk.copilot.completion_verification import CompletionVerificationResult, CriterionVerdict
+
+ ctx = _ctx(
+ last_workflow=None,
+ last_workflow_yaml=None,
+ last_test_ok=None,
+ last_full_workflow_test_ok=False,
+ completion_verification_result=CompletionVerificationResult(
+ status="evaluated",
+ criterion_ids=["c0"],
+ verdicts=[CriterionVerdict(criterion_id="c0", state="satisfied", reason_code="evidence_confirms")],
+ ),
+ )
+
+ result = await _build_goal_satisfied_exit_result(ctx, global_llm_context=None)
+
+ assert "tested successfully" not in result.user_response.lower()
+ assert "did not finish successfully" not in result.user_response.lower()
+ assert "not been tested end-to-end" in result.user_response.lower()
+
class TestSupersededAgentIntentGates:
def test_agent_no_longer_owns_request_policy_classification(self) -> None:
diff --git a/tests/unit/test_copilot_completion_verification.py b/tests/unit/test_copilot_completion_verification.py
index 847ede98e..ce37bea81 100644
--- a/tests/unit/test_copilot_completion_verification.py
+++ b/tests/unit/test_copilot_completion_verification.py
@@ -57,6 +57,7 @@ from skyvern.forge.sdk.copilot.tools import (
_tool_visible_result_after_completion_verification,
_watchdog_exit_allows_terminal_promotion,
)
+from skyvern.forge.sdk.copilot.tools.completion import _artifact_health_blocker_from_result
def _criterion(cid: str, outcome: str, *, method_mandated: bool = False) -> CompletionCriterion:
@@ -267,8 +268,12 @@ async def test_evaluate_happy_path_returns_evaluated() -> None:
def test_snapshot_has_evidence() -> None:
assert RunEvidenceSnapshot().has_evidence() is False
+ assert RunEvidenceSnapshot(run_terminal_status="failed").has_evidence() is False
assert RunEvidenceSnapshot(current_url="https://example.com").has_evidence() is True
assert RunEvidenceSnapshot(block_outputs={"a": 1}).has_evidence() is True
+ assert RunEvidenceSnapshot(failed_block_labels=["extract"]).has_evidence() is True
+ assert RunEvidenceSnapshot(failure_classes=["SyntaxError"]).has_evidence() is True
+ assert RunEvidenceSnapshot(failure_reasons=["SyntaxError: bad generated code"]).has_evidence() is True
assert RunEvidenceSnapshot(page_evidence={"visible_text_excerpt": "cart item PART-001-TEST"}).has_evidence() is True
@@ -301,6 +306,25 @@ def test_snapshot_renders_bounded_page_evidence() -> None:
assert "raw_html" not in rendered
+def test_snapshot_renders_failed_run_artifact_health_signal() -> None:
+ snapshot = RunEvidenceSnapshot(
+ workflow_run_id="wr_failed",
+ block_outputs={"extract_results": {"extracted_information": ["goal text"]}},
+ current_url="https://example.com/results",
+ run_terminal_status="failed",
+ failed_block_labels=["extract_results"],
+ failure_classes=["SyntaxError"],
+ failure_reasons=["Page.evaluate: SyntaxError: Unexpected token ')'"],
+ )
+
+ rendered = snapshot.render_prompt_block()
+
+ assert "run_terminal_status: failed" in rendered
+ assert "failed_block_labels: extract_results" in rendered
+ assert "failure_classes: SyntaxError" in rendered
+ assert "Page.evaluate: SyntaxError" in rendered
+
+
def test_active_run_terminal_visual_fallback_uses_screenshot_when_missing() -> None:
assert (
_active_run_terminal_evidence_needs_visual_fallback(
@@ -460,7 +484,7 @@ def test_gate_withholds_on_evaluated_unconfirmed_even_with_clean_run_status() ->
def test_completion_contract_not_violated() -> None:
- ctx = SimpleNamespace(completion_verification_result=None)
+ ctx = SimpleNamespace(completion_verification_result=None, last_artifact_health_blocker_reason=None)
assert _completion_contract_not_violated(ctx) is True # type: ignore[arg-type]
ctx.completion_verification_result = _evaluated(("c0", True))
assert _completion_contract_not_violated(ctx) is True # type: ignore[arg-type]
@@ -829,6 +853,118 @@ def _canceled_gate_ctx() -> CopilotContext:
return ctx
+def _failed_generated_code_result() -> dict:
+ return {
+ "ok": False,
+ "data": {
+ "workflow_run_id": "wr_failed_code",
+ "overall_status": "failed",
+ "executed_block_labels": ["extract_results"],
+ "current_url": "https://example.com/results",
+ "blocks": [
+ {
+ "label": "extract_results",
+ "block_type": "EXTRACTION",
+ "status": "failed",
+ "extracted_data": {"extracted_information": ["goal text from partial output"]},
+ "failure_reason": "Page.evaluate: SyntaxError: Unexpected token ')'",
+ }
+ ],
+ },
+ }
+
+
+def test_artifact_health_type_error_is_not_masked_by_timeout_category() -> None:
+ result = {
+ "ok": False,
+ "data": {
+ "workflow_run_id": "wr_failed_code",
+ "overall_status": "failed",
+ "failure_categories": [
+ {
+ "category": "PAGE_LOAD_TIMEOUT",
+ "confidence_float": 0.8,
+ "reasoning": "Timeout in failure reason",
+ }
+ ],
+ "blocks": [
+ {
+ "label": "wait_for_results",
+ "block_type": "ACTION",
+ "status": "failed",
+ "failure_reason": (
+ "TypeError: Page.wait_for_function() got an unexpected keyword argument 'timeout_ms'"
+ ),
+ }
+ ],
+ },
+ }
+
+ reason, failed_labels, failure_classes = _artifact_health_blocker_from_result(result)
+
+ assert reason is not None
+ assert "TypeError" in reason
+ assert failed_labels == ["wait_for_results"]
+ assert failure_classes == ["TypeError"]
+
+
+def test_artifact_health_not_masked_by_mixed_excluded_category() -> None:
+ result = {
+ "ok": False,
+ "data": {
+ "workflow_run_id": "wr_failed_code",
+ "overall_status": "failed",
+ "failure_categories": [
+ {"category": "AUTH_FAILURE", "confidence_float": 0.8},
+ {"category": "SCRIPT_ERROR", "confidence_float": 0.9},
+ ],
+ "blocks": [
+ {
+ "label": "extract_results",
+ "block_type": "EXTRACTION",
+ "status": "failed",
+ "failure_reason": "Page.evaluate: SyntaxError: Unexpected token ')'",
+ }
+ ],
+ },
+ }
+
+ reason, failed_labels, failure_classes = _artifact_health_blocker_from_result(result)
+
+ assert reason is not None
+ assert "SyntaxError" in reason
+ assert failed_labels == ["extract_results"]
+ assert failure_classes == ["SyntaxError"]
+
+
+def test_artifact_health_skips_when_all_categories_are_excluded() -> None:
+ result = {
+ "ok": False,
+ "data": {
+ "workflow_run_id": "wr_auth_failed",
+ "overall_status": "failed",
+ "failure_categories": [
+ {"category": "AUTH_FAILURE", "confidence_float": 0.8},
+ {"category": "CREDENTIAL_ERROR", "confidence_float": 0.7},
+ ],
+ "blocks": [
+ {
+ "label": "extract_results",
+ "block_type": "EXTRACTION",
+ "status": "failed",
+ "failure_reason": "Page.evaluate: SyntaxError: Unexpected token ')'",
+ }
+ ],
+ },
+ }
+
+ reason, failed_labels, failure_classes = _artifact_health_blocker_from_result(result)
+
+ assert reason is None
+ assert failed_labels == []
+ assert failure_classes == []
+
+
def test_unfinished_run_verification_candidate_admits_canceled_with_evidence() -> None:
ctx = _run_ctx()
assert _is_unfinished_run_verification_candidate(ctx, _canceled_budget_result()) is True
@@ -838,6 +974,29 @@ def test_unfinished_run_verification_candidate_admits_canceled_with_evidence() -
assert _is_unfinished_run_verification_candidate(ctx, {"ok": False, "data": {}}) is False
+def test_artifact_health_blocks_fully_satisfied_failed_run() -> None:
+ result = _failed_generated_code_result()
+ ctx = _run_ctx()
+ ctx.last_workflow = SimpleNamespace(workflow_definition=SimpleNamespace(blocks=[]))
+ ctx.last_workflow_yaml = "workflow: {}"
+
+ snapshot = _build_run_evidence_snapshot(ctx, result)
+ rendered = snapshot.render_prompt_block()
+ assert "run_terminal_status: failed" in rendered
+ assert "failure_classes: SyntaxError" in rendered
+ assert "failed_block_labels: extract_results" in rendered
+
+ _record_run_blocks_result(ctx, result, completion_verification=_evaluated(("c0", True)))
+
+ assert ctx.last_artifact_health_blocker_reason is not None
+ assert "SyntaxError" in ctx.last_artifact_health_blocker_reason
+ assert ctx.last_artifact_health_blocker_labels == ["extract_results"]
+ assert ctx.last_artifact_health_failure_classes == ["SyntaxError"]
+ assert outcome_fully_verified(ctx) is False
+ assert verified_goal_satisfied_context(ctx) is False
+ assert _verified_workflow_or_none(ctx) == (None, None)
+
+
@pytest.mark.asyncio
async def test_maybe_run_completion_verification_runs_on_canceled_run(monkeypatch: pytest.MonkeyPatch) -> None:
async def handler(**_: object) -> dict:
@@ -1573,7 +1732,10 @@ async def test_maybe_run_completion_verification_unavailable_on_low_budget(monke
def test_completion_contract_not_violated_unavailable_blocks_surfacing() -> None:
- ctx = SimpleNamespace(completion_verification_result=CompletionVerificationResult("unavailable"))
+ ctx = SimpleNamespace(
+ completion_verification_result=CompletionVerificationResult("unavailable"),
+ last_artifact_health_blocker_reason=None,
+ )
# An unavailable verdict means the outcome could not be verified: do not surface
# the workflow as verified on run status alone.
assert _completion_contract_not_violated(ctx) is False # type: ignore[arg-type]