Skip to content

Automate browser based workflows with AI

fix(SKY-11958): fail closed on ambiguous output-contract owner resolution (#7223)

Andrew Neilsonbb7fe7158f55
diff --git a/skyvern/forge/sdk/copilot/build_test_outcome.py b/skyvern/forge/sdk/copilot/build_test_outcome.py
index c7b53c72d..b0e17d970 100644
--- a/skyvern/forge/sdk/copilot/build_test_outcome.py
+++ b/skyvern/forge/sdk/copilot/build_test_outcome.py
@@ -1561,7 +1561,7 @@ def _runtime_output_repair_facts(
return []
facts: list[dict[str, object]] = []
for verdict in completion_verification.verdicts:
- if verdict.satisfied or not verdict.output_path:
+ if not verdict.output_path:
continue
output_path = _bounded_ref(verdict.output_path)
if not _output_path_has_child(output_path):
@@ -1575,7 +1575,13 @@ def _runtime_output_repair_facts(
workflow_run_id,
output_path,
)
- value_status = _runtime_output_value_status(values, verdict)
+ owner_labels = _runtime_output_owner_labels(blocks, block_labels, verdict)
+ if verdict.satisfied:
+ if not owner_labels:
+ continue
+ value_status = "satisfied"
+ else:
+ value_status = _runtime_output_value_status(values, verdict)
fact: dict[str, object] = {
"workflow_run_id": _bounded_ref(workflow_run_id),
"output_path": output_path,
@@ -1584,8 +1590,10 @@ def _runtime_output_repair_facts(
"reason_code": _bounded_ref(verdict.reason_code),
"value_status": value_status,
}
- if len(block_labels) == 1:
- fact["block_label"] = block_labels[0]
+ if verdict.satisfied or len(owner_labels) > 1:
+ fact["owner_labels"] = owner_labels
+ if len(owner_labels) == 1:
+ fact["block_label"] = owner_labels[0]
if verdict.grounding_mode:
fact["grounding_mode"] = verdict.grounding_mode
if verdict.expected_output_shape:
@@ -1646,6 +1654,7 @@ def _runtime_output_values_for_path(
values: list[object] = []
evidence_refs: list[str] = []
block_labels: list[str] = []
+ current_labels = {label for block in blocks for label in [_bounded_ref(block.get("label"))] if label}
for item in registered_output_parameter_payloads:
item_run_id = _safe_str(item.get("workflow_run_id"))
if item_run_id != workflow_run_id:
@@ -1654,7 +1663,7 @@ def _runtime_output_values_for_path(
if not present:
continue
values.append(value)
- label = _bounded_ref(item.get("block_label"))
+ label = _registered_output_owner_label(item, current_labels)
key = _bounded_ref(item.get("output_parameter_key"))
if label:
block_labels.append(label)
@@ -1675,6 +1684,34 @@ def _runtime_output_values_for_path(
return values, list(dict.fromkeys(evidence_refs)), sorted(dict.fromkeys(block_labels))
+def _registered_output_owner_label(item: Mapping[str, object], current_labels: set[str]) -> str:
+ label = _bounded_ref(item.get("block_label"))
+ if label in current_labels:
+ return label
+ return ""
+
+
+def _runtime_output_owner_labels(
+ blocks: Sequence[Mapping[str, object]],
+ block_labels: Sequence[str],
+ verdict: CriterionVerdict,
+) -> list[str]:
+ if not verdict.satisfied and verdict.requested_output_evidence_source == "independent_run_evidence":
+ return []
+ current_labels = {label for block in blocks for label in [_bounded_ref(block.get("label"))] if label}
+ labels = {label for label in block_labels if label in current_labels}
+ evidence_label = _block_output_evidence_ref_label(verdict.evidence_ref)
+ if evidence_label in current_labels:
+ labels.add(evidence_label)
+ return sorted(labels)
+
+
+def _block_output_evidence_ref_label(evidence_ref: str | None) -> str:
+ if not evidence_ref or not evidence_ref.startswith("block_outputs:"):
+ return ""
+ return _bounded_ref(evidence_ref.removeprefix("block_outputs:").split(".", 1)[0])
+
+
def _registered_output_value_for_path(item: Mapping[str, object], output_path: str) -> tuple[object | None, bool]:
value = item.get("value")
key = _safe_str(item.get("output_parameter_key"))
@@ -1683,6 +1720,10 @@ def _registered_output_value_for_path(item: Mapping[str, object], output_path: s
if output_path.startswith("output.") and key == output_path.split(".", 1)[1]:
return value, True
if isinstance(value, Mapping):
+ if output_path.startswith("output."):
+ unwrapped_value, unwrapped_present = _value_at_output_path(value, output_path.split(".", 1)[1])
+ if unwrapped_present:
+ return unwrapped_value, True
return _value_at_output_path(value, output_path)
return None, False
diff --git a/skyvern/forge/sdk/copilot/tools/workflow_update.py b/skyvern/forge/sdk/copilot/tools/workflow_update.py
index 25bdc6ff6..f3cd77416 100644
--- a/skyvern/forge/sdk/copilot/tools/workflow_update.py
+++ b/skyvern/forge/sdk/copilot/tools/workflow_update.py
@@ -1848,6 +1848,8 @@ class _RuntimeOutputRepairContract:
required_paths: set[str]
facts: list[dict[str, Any]]
workflow_run_id: str
+ owner_labels: list[str]
+ owner_labels_by_path: dict[str, list[str]]
source: str = "runtime_output_repair"
reason_code: str = "runtime_output_repair_required"
@@ -2145,6 +2147,8 @@ def _runtime_output_contract_signature(runtime_contract: _RuntimeOutputRepairCon
payload = {
"workflow_run_id": runtime_contract.workflow_run_id,
"required_paths": sorted(runtime_contract.required_paths),
+ "owner_labels": runtime_contract.owner_labels,
+ "owner_labels_by_path": runtime_contract.owner_labels_by_path,
"facts": runtime_contract.facts,
}
return hashlib.sha256(json.dumps(payload, sort_keys=True, default=str).encode("utf-8")).hexdigest()
@@ -2199,6 +2203,35 @@ def _target_output_contract_block_label(
raw_code_artifact_metadata: object,
required_paths: set[str],
) -> tuple[str, list[str]]:
+ runtime_contract = _runtime_output_repair_contract_from_recorded_outcome(ctx)
+ if runtime_contract is not None:
+ code_blocks = _workflow_yaml_code_blocks_by_label(workflow_yaml)
+ runtime_owner_labels: set[str] = set()
+ missing_owner = False
+ ambiguous_owner = False
+ for path in sorted(runtime_contract.required_paths):
+ raw_path_owners = sorted(runtime_contract.owner_labels_by_path.get(path, []))
+ path_owner_labels = sorted(label for label in raw_path_owners if label in code_blocks)
+ # Ambiguity is judged before filtering: dropping stale labels must not resolve a
+ # contested path down to a lone survivor.
+ if len(raw_path_owners) > 1:
+ ambiguous_owner = True
+ runtime_owner_labels.update(path_owner_labels)
+ continue
+ if len(path_owner_labels) != 1:
+ missing_owner = missing_owner or not path_owner_labels
+ runtime_owner_labels.update(path_owner_labels)
+ continue
+ runtime_owner_labels.add(path_owner_labels[0])
+ if missing_owner:
+ return "", []
+ current_owner_labels = sorted(runtime_owner_labels)
+ if ambiguous_owner:
+ return "", current_owner_labels
+ if len(current_owner_labels) == 1:
+ _pin_output_contract_block_label(ctx, workflow_yaml, required_paths, current_owner_labels[0])
+ return current_owner_labels[0], current_owner_labels
+ return "", current_owner_labels
pinned_label = _pinned_output_contract_block_label(ctx, workflow_yaml, required_paths)
if pinned_label:
return pinned_label, [pinned_label]
@@ -2226,7 +2259,8 @@ def _runtime_output_repair_contract_from_recorded_outcome(ctx: AgentContext) ->
return None
facts: list[dict[str, Any]] = []
required_paths: set[str] = set()
- block_labels: set[str] = set()
+ owner_labels: set[str] = set()
+ owner_labels_by_path: dict[str, set[str]] = {}
for raw_fact in outcome.runtime_output_repair_facts:
if not isinstance(raw_fact, Mapping):
return None
@@ -2238,17 +2272,24 @@ def _runtime_output_repair_contract_from_recorded_outcome(ctx: AgentContext) ->
fact = dict(raw_fact)
fact["output_path"] = path
fact["output_root"] = _output_path_root(path)
+ path_owner_labels = owner_labels_by_path.setdefault(path, set())
+ raw_owner_labels = fact.get("owner_labels")
+ if isinstance(raw_owner_labels, list):
+ path_owner_labels.update(str(label).strip() for label in raw_owner_labels if str(label).strip())
label = str(fact.get("block_label") or "").strip()
if label:
- block_labels.add(label)
+ path_owner_labels.add(label)
+ owner_labels.update(path_owner_labels)
required_paths.add(path)
facts.append(fact)
- if not facts or not required_paths or len(block_labels) > 1:
+ if not facts or not required_paths:
return None
return _RuntimeOutputRepairContract(
required_paths=required_paths,
facts=sorted(facts, key=lambda item: str(item.get("output_path") or "")),
workflow_run_id=outcome.workflow_run_id,
+ owner_labels=sorted(owner_labels),
+ owner_labels_by_path={path: sorted(labels) for path, labels in sorted(owner_labels_by_path.items())},
)
diff --git a/tests/unit/test_copilot_code_block_persist_seam.py b/tests/unit/test_copilot_code_block_persist_seam.py
index e6dbea27f..6643c3afd 100644
--- a/tests/unit/test_copilot_code_block_persist_seam.py
+++ b/tests/unit/test_copilot_code_block_persist_seam.py
@@ -4261,6 +4261,354 @@ class TestCodeRepairProgressClassification:
assert source == "runtime_output_repair"
assert reason_code == "runtime_output_repair_required"
+ def test_runtime_output_facts_preserve_satisfied_output_owner_label(self) -> None:
+ result = {
+ "ok": True,
+ "data": {
+ "workflow_run_id": "wr_current",
+ "blocks": [
+ {
+ "label": "download_statement",
+ "status": "completed",
+ "extracted_data": {"output": {"statement_pdf": "statement.pdf"}},
+ }
+ ],
+ },
+ }
+ verification = CompletionVerificationResult(
+ status="evaluated",
+ criterion_ids=["__copilot_fallback_floor__run", "requested_statement_pdf"],
+ verdicts=[
+ CriterionVerdict(
+ criterion_id="__copilot_fallback_floor__run",
+ state="unsatisfied",
+ reason_code="no_evidence",
+ ),
+ CriterionVerdict(
+ criterion_id="requested_statement_pdf",
+ state="satisfied",
+ reason_code="evidence_confirms",
+ output_path="output.statement_pdf",
+ grounding_mode="exact_value",
+ expected_output_shape="string",
+ has_exact_value=True,
+ evidence_ref="block_outputs:download_statement.output.statement_pdf",
+ ),
+ ],
+ )
+
+ outcome = recorded_outcome_from_run_blocks_result(
+ result,
+ recorded_run_outcome=RecordedRunOutcome(
+ verdict="not_demonstrated",
+ reason_code="outcome_not_demonstrated",
+ workflow_run_id="wr_current",
+ ),
+ completion_verification=verification,
+ )
+
+ assert outcome is not None
+ assert outcome.reason_code == "outcome_not_demonstrated"
+ assert outcome.runtime_output_repair_facts == [
+ {
+ "workflow_run_id": "wr_current",
+ "block_label": "download_statement",
+ "owner_labels": ["download_statement"],
+ "output_path": "output.statement_pdf",
+ "output_root": "output",
+ "criterion_id": "requested_statement_pdf",
+ "reason_code": "evidence_confirms",
+ "grounding_mode": "exact_value",
+ "expected_output_shape": "string",
+ "value_status": "satisfied",
+ "evidence_refs": ["output:download_statement"],
+ }
+ ]
+
+ def test_runtime_output_facts_preserve_flat_registered_output_owner_label(self) -> None:
+ result = {
+ "ok": True,
+ "data": {
+ "workflow_run_id": "wr_current",
+ "blocks": [
+ {"label": "apex_portal_login", "status": "completed", "extracted_data": {}},
+ {"label": "apex_open_monthly_statement", "status": "completed", "extracted_data": {}},
+ {"label": "apex_download_invoice_pdf", "status": "completed", "extracted_data": {}},
+ ],
+ },
+ }
+ verification = CompletionVerificationResult(
+ status="evaluated",
+ criterion_ids=["__copilot_authored_output__output_file_name"],
+ verdicts=[
+ CriterionVerdict(
+ criterion_id="__copilot_authored_output__output_file_name",
+ state="unsatisfied",
+ reason_code="no_evidence",
+ output_path="output.file_name",
+ grounding_mode="missing",
+ requested_output_evidence_source="runtime_output",
+ )
+ ],
+ )
+
+ outcome = recorded_outcome_from_run_blocks_result(
+ result,
+ recorded_run_outcome=RecordedRunOutcome(
+ verdict="not_demonstrated",
+ reason_code="outcome_not_demonstrated",
+ workflow_run_id="wr_current",
+ ),
+ completion_verification=verification,
+ registered_output_parameter_payloads=[
+ {
+ "workflow_run_id": "wr_current",
+ "block_label": "apex_download_invoice_pdf",
+ "output_parameter_key": "apex_download_invoice_pdf_output",
+ "value": {
+ "file_name": "statement.pdf",
+ "downloaded_files": [{"filename": "statement.pdf"}],
+ },
+ }
+ ],
+ )
+
+ assert outcome is not None
+ assert outcome.runtime_output_repair_facts == [
+ {
+ "workflow_run_id": "wr_current",
+ "block_label": "apex_download_invoice_pdf",
+ "output_path": "output.file_name",
+ "output_root": "output",
+ "criterion_id": "__copilot_authored_output__output_file_name",
+ "reason_code": "no_evidence",
+ "grounding_mode": "missing",
+ "value_status": "no_typed_value",
+ "evidence_refs": ["registered_output:apex_download_invoice_pdf:apex_download_invoice_pdf_output"],
+ }
+ ]
+ ctx = _code_only_ctx()
+ ctx.latest_recorded_build_test_outcome = outcome
+ workflow_yaml = _yaml(
+ """
+ title: Statement download
+ workflow_definition:
+ blocks:
+ - block_type: code
+ label: apex_portal_login
+ code: |
+ return {"logged_in": True}
+ - block_type: code
+ label: apex_open_monthly_statement
+ code: |
+ return {"matched": True}
+ - block_type: code
+ label: apex_download_invoice_pdf
+ code: |
+ return {"file_name": "statement.pdf"}
+ """
+ )
+
+ evaluation = workflow_update_module._evaluate_output_contract_for_code_block(ctx, workflow_yaml, [])
+
+ assert evaluation is not None
+ assert evaluation.block_label == "apex_download_invoice_pdf"
+ assert evaluation.payload["output_owner_labels"] == ["apex_download_invoice_pdf"]
+ assert "missing_output_owner" not in evaluation.shape_violations
+
+ def test_runtime_output_facts_do_not_infer_registered_owner_from_parameter_key(self) -> None:
+ result = {
+ "ok": True,
+ "data": {
+ "workflow_run_id": "wr_current",
+ "blocks": [
+ {"label": "download_invoice", "status": "completed", "extracted_data": {}},
+ ],
+ },
+ }
+ verification = CompletionVerificationResult(
+ status="evaluated",
+ criterion_ids=["__copilot_authored_output__output_file_name"],
+ verdicts=[
+ CriterionVerdict(
+ criterion_id="__copilot_authored_output__output_file_name",
+ state="unsatisfied",
+ reason_code="no_evidence",
+ output_path="output.file_name",
+ grounding_mode="missing",
+ requested_output_evidence_source="runtime_output",
+ )
+ ],
+ )
+
+ outcome = recorded_outcome_from_run_blocks_result(
+ result,
+ recorded_run_outcome=RecordedRunOutcome(
+ verdict="not_demonstrated",
+ reason_code="outcome_not_demonstrated",
+ workflow_run_id="wr_current",
+ ),
+ completion_verification=verification,
+ registered_output_parameter_payloads=[
+ {
+ "workflow_run_id": "wr_current",
+ "output_parameter_key": "download_invoice_output",
+ "value": {"file_name": "statement.pdf"},
+ }
+ ],
+ )
+
+ assert outcome is not None
+ assert outcome.runtime_output_repair_facts == [
+ {
+ "workflow_run_id": "wr_current",
+ "output_path": "output.file_name",
+ "output_root": "output",
+ "criterion_id": "__copilot_authored_output__output_file_name",
+ "reason_code": "no_evidence",
+ "grounding_mode": "missing",
+ "value_status": "no_typed_value",
+ "evidence_refs": ["registered_output:unknown:download_invoice_output"],
+ }
+ ]
+ ctx = _code_only_ctx()
+ ctx.latest_recorded_build_test_outcome = outcome
+ workflow_yaml = _yaml(
+ """
+ title: Statement download
+ workflow_definition:
+ blocks:
+ - block_type: code
+ label: download_invoice
+ code: |
+ return {"file_name": "statement.pdf"}
+ """
+ )
+
+ evaluation = workflow_update_module._evaluate_output_contract_for_code_block(ctx, workflow_yaml, [])
+
+ assert evaluation is not None
+ assert evaluation.block_label == ""
+ assert evaluation.payload["output_owner_labels"] == []
+ assert "missing_output_owner" in evaluation.shape_violations
+
+ def test_runtime_output_owner_ignores_unsatisfied_independent_self_emitted_fields(self) -> None:
+ result = {
+ "ok": True,
+ "data": {
+ "workflow_run_id": "wr_current",
+ "blocks": [
+ {
+ "label": "login_to_service",
+ "status": "completed",
+ "extracted_data": {"output": {"logged_in": True}},
+ },
+ {
+ "label": "open_statement",
+ "status": "completed",
+ "extracted_data": {
+ "output": {
+ "matched": True,
+ "statement_date": "2026-05",
+ "visible_page_label": "Statement details",
+ }
+ },
+ },
+ ],
+ },
+ }
+ verification = CompletionVerificationResult(
+ status="evaluated",
+ criterion_ids=[
+ "__copilot_authored_output__output_logged_in",
+ "__copilot_authored_output__output_matched",
+ "__copilot_authored_output__output_statement_date",
+ "__copilot_authored_output__output_visible_page_label",
+ "__copilot_authored_output__output_downloaded",
+ ],
+ verdicts=[
+ CriterionVerdict(
+ criterion_id="__copilot_authored_output__output_logged_in",
+ state="unsatisfied",
+ reason_code="structurally_abstained",
+ output_path="output.logged_in",
+ grounding_mode="missing",
+ requested_output_evidence_source="independent_run_evidence",
+ evidence_ref="block_outputs:login_to_service.output.logged_in",
+ ),
+ CriterionVerdict(
+ criterion_id="__copilot_authored_output__output_matched",
+ state="unsatisfied",
+ reason_code="structurally_abstained",
+ output_path="output.matched",
+ grounding_mode="missing",
+ requested_output_evidence_source="independent_run_evidence",
+ evidence_ref="block_outputs:open_statement.output.matched",
+ ),
+ CriterionVerdict(
+ criterion_id="__copilot_authored_output__output_statement_date",
+ state="unsatisfied",
+ reason_code="structurally_abstained",
+ output_path="output.statement_date",
+ grounding_mode="missing",
+ requested_output_evidence_source="runtime_output",
+ evidence_ref="block_outputs:open_statement.output.statement_date",
+ ),
+ CriterionVerdict(
+ criterion_id="__copilot_authored_output__output_visible_page_label",
+ state="unsatisfied",
+ reason_code="structurally_abstained",
+ output_path="output.visible_page_label",
+ grounding_mode="missing",
+ requested_output_evidence_source="runtime_output",
+ evidence_ref="block_outputs:open_statement.output.visible_page_label",
+ ),
+ CriterionVerdict(
+ criterion_id="__copilot_authored_output__output_downloaded",
+ state="unsatisfied",
+ reason_code="no_evidence",
+ output_path="output.downloaded",
+ grounding_mode="missing",
+ requested_output_evidence_source="independent_run_evidence",
+ ),
+ ],
+ )
+
+ outcome = recorded_outcome_from_run_blocks_result(
+ result,
+ recorded_run_outcome=RecordedRunOutcome(
+ verdict="not_demonstrated",
+ reason_code="outcome_not_demonstrated",
+ workflow_run_id="wr_current",
+ ),
+ completion_verification=verification,
+ )
+ ctx = _code_only_ctx()
+ ctx.latest_recorded_build_test_outcome = outcome
+ workflow_yaml = _yaml(
+ """
+ title: Statement workflow
+ workflow_definition:
+ blocks:
+ - block_type: code
+ label: login_to_service
+ code: |
+ return {"output": {"logged_in": True}}
+ - block_type: code
+ label: open_statement
+ code: |
+ return {"output": {"statement_date": "2026-05"}}
+ """
+ )
+
+ evaluation = workflow_update_module._evaluate_output_contract_for_code_block(ctx, workflow_yaml, [])
+
+ assert outcome is not None
+ assert evaluation is not None
+ assert evaluation.block_label == ""
+ assert evaluation.payload["output_owner_labels"] == []
+ assert "missing_output_owner" in evaluation.shape_violations
+
def test_runtime_output_facts_ignore_other_run_registered_values(self) -> None:
result = {
"ok": True,
@@ -4467,6 +4815,313 @@ class TestCodeRepairProgressClassification:
key=lambda item: str(item.get("output_path") or ""),
)
+ def test_runtime_output_owner_selects_current_block_without_metadata(self) -> None:
+ ctx = _code_only_ctx()
+ ctx.latest_recorded_build_test_outcome = RecordedBuildTestOutcome(
+ phase="persisted_block_run",
+ attempted_tool="update_and_run_blocks",
+ verdict="repairable_failure",
+ reason_code="outcome_not_demonstrated",
+ workflow_run_id="wr_current",
+ structural_failure_identity="completion:runtime-output",
+ runtime_output_repair_facts=[
+ {
+ "workflow_run_id": "wr_current",
+ "owner_labels": ["download_statement"],
+ "output_path": "output.statement_pdf",
+ "output_root": "output",
+ "criterion_id": "requested_statement_pdf",
+ "reason_code": "evidence_confirms",
+ "value_status": "satisfied",
+ }
+ ],
+ )
+ workflow_yaml = _yaml(
+ """
+ title: Statement download
+ workflow_definition:
+ blocks:
+ - block_type: code
+ label: login
+ code: |
+ return {"logged_in": True}
+ - block_type: code
+ label: download_statement
+ code: |
+ return {"output": {"statement_pdf": "statement.pdf"}}
+ """
+ )
+
+ evaluation = workflow_update_module._evaluate_output_contract_for_code_block(ctx, workflow_yaml, [])
+
+ assert evaluation is not None
+ assert evaluation.block_label == "download_statement"
+ assert evaluation.payload["output_owner_labels"] == ["download_statement"]
+ assert "missing_output_owner" not in evaluation.shape_violations
+
+ def test_runtime_output_zero_owner_does_not_fall_through_to_single_block_default(self) -> None:
+ ctx = _code_only_ctx()
+ ctx.latest_recorded_build_test_outcome = RecordedBuildTestOutcome(
+ phase="persisted_block_run",
+ attempted_tool="update_and_run_blocks",
+ verdict="repairable_failure",
+ reason_code="outcome_not_demonstrated",
+ workflow_run_id="wr_current",
+ structural_failure_identity="completion:runtime-output",
+ runtime_output_repair_facts=[
+ {
+ "workflow_run_id": "wr_current",
+ "output_path": "output.statement_pdf",
+ "output_root": "output",
+ "criterion_id": "requested_statement_pdf",
+ "reason_code": "evidence_confirms",
+ "value_status": "satisfied",
+ }
+ ],
+ )
+ workflow_yaml = _yaml(
+ """
+ title: Statement download
+ workflow_definition:
+ blocks:
+ - block_type: code
+ label: only_block
+ code: |
+ return {"output": {"statement_pdf": "statement.pdf"}}
+ """
+ )
+
+ evaluation = workflow_update_module._evaluate_output_contract_for_code_block(ctx, workflow_yaml, [])
+
+ assert evaluation is not None
+ assert evaluation.block_label == ""
+ assert evaluation.payload["output_owner_labels"] == []
+ assert "missing_output_owner" in evaluation.shape_violations
+
+ def test_runtime_output_multi_owner_rejects_without_picking_metadata_owner(self) -> None:
+ ctx = _code_only_ctx()
+ ctx.latest_recorded_build_test_outcome = RecordedBuildTestOutcome(
+ phase="persisted_block_run",
+ attempted_tool="update_and_run_blocks",
+ verdict="repairable_failure",
+ reason_code="outcome_not_demonstrated",
+ workflow_run_id="wr_current",
+ structural_failure_identity="completion:runtime-output",
+ runtime_output_repair_facts=[
+ {
+ "workflow_run_id": "wr_current",
+ "owner_labels": ["download_a", "download_b"],
+ "output_path": "output.statement_pdf",
+ "output_root": "output",
+ "criterion_id": "requested_statement_pdf",
+ "reason_code": "evidence_confirms",
+ "value_status": "satisfied",
+ }
+ ],
+ )
+ workflow_yaml = _yaml(
+ """
+ title: Statement download
+ workflow_definition:
+ blocks:
+ - block_type: code
+ label: download_a
+ code: |
+ return {"output": {"statement_pdf": "a.pdf"}}
+ - block_type: code
+ label: download_b
+ code: |
+ return {"output": {"statement_pdf": "b.pdf"}}
+ """
+ )
+ metadata = [{"block_label": "download_a", "claimed_outcomes": [{"goal_value_paths": ["output.statement_pdf"]}]}]
+
+ evaluation = workflow_update_module._evaluate_output_contract_for_code_block(ctx, workflow_yaml, metadata)
+
+ assert evaluation is not None
+ assert evaluation.block_label == ""
+ assert evaluation.payload["output_owner_labels"] == ["download_a", "download_b"]
+ assert "ambiguous_output_owner" in evaluation.shape_violations
+
+ def test_runtime_output_stale_multi_owner_rejects_when_one_owner_current(self) -> None:
+ ctx = _code_only_ctx()
+ ctx.latest_recorded_build_test_outcome = RecordedBuildTestOutcome(
+ phase="persisted_block_run",
+ attempted_tool="update_and_run_blocks",
+ verdict="repairable_failure",
+ reason_code="outcome_not_demonstrated",
+ workflow_run_id="wr_current",
+ structural_failure_identity="completion:runtime-output",
+ runtime_output_repair_facts=[
+ {
+ "workflow_run_id": "wr_current",
+ "owner_labels": ["current_download", "stale_download"],
+ "output_path": "output.statement_pdf",
+ "output_root": "output",
+ "criterion_id": "requested_statement_pdf",
+ "reason_code": "evidence_confirms",
+ "value_status": "satisfied",
+ }
+ ],
+ )
+ workflow_yaml = _yaml(
+ """
+ title: Statement download
+ workflow_definition:
+ blocks:
+ - block_type: code
+ label: current_download
+ code: |
+ return {"output": {"statement_pdf": "a.pdf"}}
+ """
+ )
+
+ evaluation = workflow_update_module._evaluate_output_contract_for_code_block(ctx, workflow_yaml, [])
+
+ assert evaluation is not None
+ assert evaluation.block_label == ""
+ assert evaluation.payload["output_owner_labels"] == ["current_download"]
+ assert "ambiguous_output_owner" in evaluation.shape_violations
+
+ def test_runtime_output_multi_owner_all_stale_rejects_as_missing_owner(self) -> None:
+ ctx = _code_only_ctx()
+ ctx.latest_recorded_build_test_outcome = RecordedBuildTestOutcome(
+ phase="persisted_block_run",
+ attempted_tool="update_and_run_blocks",
+ verdict="repairable_failure",
+ reason_code="outcome_not_demonstrated",
+ workflow_run_id="wr_current",
+ structural_failure_identity="completion:runtime-output",
+ runtime_output_repair_facts=[
+ {
+ "workflow_run_id": "wr_current",
+ "owner_labels": ["stale_one", "stale_two"],
+ "output_path": "output.statement_pdf",
+ "output_root": "output",
+ "criterion_id": "requested_statement_pdf",
+ "reason_code": "evidence_confirms",
+ "value_status": "satisfied",
+ }
+ ],
+ )
+ workflow_yaml = _yaml(
+ """
+ title: Statement download
+ workflow_definition:
+ blocks:
+ - block_type: code
+ label: current_download
+ code: |
+ return {"output": {"statement_pdf": "a.pdf"}}
+ """
+ )
+
+ evaluation = workflow_update_module._evaluate_output_contract_for_code_block(ctx, workflow_yaml, [])
+
+ assert evaluation is not None
+ assert evaluation.block_label == ""
+ assert evaluation.payload["output_owner_labels"] == []
+ assert "missing_output_owner" in evaluation.shape_violations
+
+ def test_runtime_output_paths_with_disagreeing_single_owners_reject(self) -> None:
+ ctx = _code_only_ctx()
+ ctx.latest_recorded_build_test_outcome = RecordedBuildTestOutcome(
+ phase="persisted_block_run",
+ attempted_tool="update_and_run_blocks",
+ verdict="repairable_failure",
+ reason_code="outcome_not_demonstrated",
+ workflow_run_id="wr_current",
+ structural_failure_identity="completion:runtime-output",
+ runtime_output_repair_facts=[
+ {
+ "workflow_run_id": "wr_current",
+ "owner_labels": ["download_pdf"],
+ "output_path": "output.statement_pdf",
+ "output_root": "output",
+ "criterion_id": "requested_statement_pdf",
+ "reason_code": "evidence_confirms",
+ "value_status": "satisfied",
+ },
+ {
+ "workflow_run_id": "wr_current",
+ "owner_labels": ["extract_total"],
+ "output_path": "output.statement_total",
+ "output_root": "output",
+ "criterion_id": "requested_statement_total",
+ "reason_code": "evidence_confirms",
+ "value_status": "satisfied",
+ },
+ ],
+ )
+ workflow_yaml = _yaml(
+ """
+ title: Statement download
+ workflow_definition:
+ blocks:
+ - block_type: code
+ label: download_pdf
+ code: |
+ return {"output": {"statement_pdf": "a.pdf"}}
+ - block_type: code
+ label: extract_total
+ code: |
+ return {"output": {"statement_total": "12.00"}}
+ """
+ )
+
+ evaluation = workflow_update_module._evaluate_output_contract_for_code_block(ctx, workflow_yaml, [])
+
+ assert evaluation is not None
+ assert evaluation.block_label == ""
+ assert evaluation.payload["output_owner_labels"] == ["download_pdf", "extract_total"]
+ assert "ambiguous_output_owner" in evaluation.shape_violations
+
+ def test_runtime_output_owner_overrides_stale_pin(self) -> None:
+ ctx = _code_only_ctx()
+ ctx.turn_id = "runtime-owner-pin"
+ ctx.latest_recorded_build_test_outcome = RecordedBuildTestOutcome(
+ phase="persisted_block_run",
+ attempted_tool="update_and_run_blocks",
+ verdict="repairable_failure",
+ reason_code="outcome_not_demonstrated",
+ workflow_run_id="wr_current",
+ structural_failure_identity="completion:runtime-output",
+ runtime_output_repair_facts=[
+ {
+ "workflow_run_id": "wr_current",
+ "owner_labels": ["new_owner"],
+ "output_path": "output.statement_pdf",
+ "output_root": "output",
+ "criterion_id": "requested_statement_pdf",
+ "reason_code": "evidence_confirms",
+ "value_status": "satisfied",
+ }
+ ],
+ )
+ workflow_yaml = _yaml(
+ """
+ title: Statement download
+ workflow_definition:
+ blocks:
+ - block_type: code
+ label: stale_owner
+ code: |
+ return {"output": {"statement_pdf": "old.pdf"}}
+ - block_type: code
+ label: new_owner
+ code: |
+ return {"output": {"statement_pdf": "new.pdf"}}
+ """
+ )
+ pin_key = workflow_update_module._output_contract_pin_key(ctx, workflow_yaml, {"output.statement_pdf"})
+ ctx.output_contract_pinned_block_label_by_signature = {pin_key: "stale_owner"}
+
+ evaluation = workflow_update_module._evaluate_output_contract_for_code_block(ctx, workflow_yaml, [])
+
+ assert evaluation is not None
+ assert evaluation.block_label == "new_owner"
+ assert evaluation.payload["output_owner_labels"] == ["new_owner"]
+
@pytest.mark.asyncio
async def test_runtime_output_repair_facts_trigger_one_envelope_attempt_before_budget(
self, monkeypatch: pytest.MonkeyPatch
diff --git a/tests/unit/test_copilot_output_contract_actuation.py b/tests/unit/test_copilot_output_contract_actuation.py
index 5d83ae933..d40cc5890 100644
--- a/tests/unit/test_copilot_output_contract_actuation.py
+++ b/tests/unit/test_copilot_output_contract_actuation.py
@@ -525,6 +525,7 @@ def _advisory_ctx() -> SimpleNamespace:
recorded_build_test_outcome_history=[],
scouted_output_covered_paths=set(),
composition_page_evidence=None,
+ recorded_outcome_binding_constraint=None,
)