Skip to content

Automate browser based workflows with AI

Refuse Copilot drafts whose entire output contract is a null blocker (SKY-12539) (#7497)

Andrew Neilson3fcac5d83187
diff --git a/skyvern/forge/sdk/copilot/tools/run_execution.py b/skyvern/forge/sdk/copilot/tools/run_execution.py
index 6787cf94e..f500e4a41 100644
--- a/skyvern/forge/sdk/copilot/tools/run_execution.py
+++ b/skyvern/forge/sdk/copilot/tools/run_execution.py
@@ -206,7 +206,7 @@ from .guardrails import (
_placeholder_for_parameter_type,
)
from .scouting import _mark_page_inspected, _mark_post_run_page_observed
-from .workflow_update import record_output_contract_run_output_evidence
+from .workflow_update import output_contract_value_bearing_run_reject, record_output_contract_run_output_evidence
LOG = structlog.get_logger()
@@ -1545,6 +1545,23 @@ async def _run_blocks_and_collect_debug(
ctx.last_executed_block_labels = []
return runtime_security_failure
+ # The lane asserts a saved-workflow property, so its evidence set is every saved code
+ # block, never just the selected subset (security lanes above stay selection-scoped).
+ value_bearing_reject = output_contract_value_bearing_run_reject(
+ ctx,
+ {
+ code_input.label: code_input.code
+ for code_input in _selected_code_security_inputs(
+ _workflow_definition_blocks_for_code_security(workflow.workflow_definition),
+ selected_labels=set(),
+ include_descendants=True,
+ )
+ },
+ )
+ if value_bearing_reject is not None:
+ ctx.last_executed_block_labels = []
+ return value_bearing_reject
+
credential_ids = list(
dict.fromkeys(
_extract_credential_ids_from_tool_value(params.get("parameters") or {})
diff --git a/skyvern/forge/sdk/copilot/tools/workflow_update.py b/skyvern/forge/sdk/copilot/tools/workflow_update.py
index 9ed260568..1a4176dc6 100644
--- a/skyvern/forge/sdk/copilot/tools/workflow_update.py
+++ b/skyvern/forge/sdk/copilot/tools/workflow_update.py
@@ -2472,6 +2472,7 @@ _METADATA_CONTRACT_REQUIRED_BEFORE_RUN_REASON_CODE = "metadata_contract_required
_SEPARATED_SPINE_SHAPE_REQUIRED_REASON_CODE = "separated_spine_shape_required"
_SEPARATED_BROWSER_SPINE_PLUS_EXTRACTION_STRUCTURE = "separated_browser_spine_plus_extraction"
_OUTPUT_CONTRACT_REJECT_REASON_CODE = "output_contract_required"
+_OUTPUT_CONTRACT_VALUE_REQUIRED_REASON_CODE = "value_bearing_output_required"
_OUTPUT_CONTRACT_REJECT_BUDGET_REASON_CODE = "output_contract_reject_budget_exhausted"
_MAX_OUTPUT_CONTRACT_REJECTS = 4
_MAX_OUTPUT_CONTRACT_DEFERRALS = 3
@@ -2816,14 +2817,20 @@ def _output_contract_scope_key(ctx: AgentContext | None) -> str:
return ""
-def _stable_output_contract_key(scope_key: str, required_paths: set[str]) -> str:
+def _stable_output_contract_key(
+ scope_key: str,
+ required_paths: set[str],
+ request_slot_identity: Sequence[tuple[str, str]] = (),
+) -> str:
scope_key = scope_key.strip()
if not scope_key:
return ""
- payload = {
+ payload: dict[str, str | list[str] | list[tuple[str, str]]] = {
"scope": scope_key,
"required_paths": sorted(required_paths),
}
+ if request_slot_identity:
+ payload["request_slot_identity"] = sorted(request_slot_identity)
return hashlib.sha256(json.dumps(payload, sort_keys=True).encode("utf-8")).hexdigest()
@@ -2834,7 +2841,7 @@ def _output_contract_author_time_structural_payload(
block_label: str = "",
deficiency_family: str = "output_contract_unsatisfied",
) -> Mapping[str, object] | None:
- signature = _stable_output_contract_key(_output_contract_scope_key(ctx), required_paths)
+ signature = _output_contract_signature(ctx=ctx, required_paths=required_paths)
if not signature:
return None
payload: dict[str, object] = {
@@ -2852,12 +2859,18 @@ def _output_contract_signature(
*,
ctx: AgentContext | None = None,
scope_key: str = "",
- workflow_yaml: str,
- source: str,
- reason_code: str,
required_paths: set[str],
) -> str:
- return _stable_output_contract_key(scope_key or _output_contract_scope_key(ctx), required_paths)
+ """Sole producer of canonical output-contract keys: the degraded request-slot identity is
+ derived from ctx here so one logical contract keeps one key at every consuming seam."""
+ request_slot_identity: tuple[tuple[str, str], ...] = (
+ tuple(diagnostic.identity for diagnostic in _degraded_request_slot_diagnostics(ctx)) if ctx is not None else ()
+ )
+ return _stable_output_contract_key(
+ scope_key or _output_contract_scope_key(ctx),
+ required_paths,
+ request_slot_identity,
+ )
def _runtime_output_contract_signature(runtime_contract: _RuntimeOutputRepairContract | None) -> str:
@@ -2879,7 +2892,7 @@ def _default_output_contract_block_label(workflow_yaml: str) -> str:
def _output_contract_pin_key(ctx: AgentContext, workflow_yaml: str, required_paths: set[str]) -> str:
- return _stable_output_contract_key(_output_contract_scope_key(ctx), required_paths)
+ return _output_contract_signature(ctx=ctx, required_paths=required_paths)
def _pinned_output_contract_block_label(
@@ -3012,6 +3025,36 @@ def _runtime_output_repair_contract_from_recorded_outcome(ctx: AgentContext) ->
)
+class _OutputContractLiveness(StrEnum):
+ ABSENT = "absent"
+ VALUE_REQUIRED = "value_required"
+ DEGRADED_EMPTY = "degraded_empty"
+
+
+@dataclass(frozen=True)
+class _DegradedRequestSlotDiagnostic:
+ request_slot_id: str
+ floor_rekeyed_from_path: str
+ pinability: str
+ mint_disposition: str
+ mint_degrade: str
+ request_slot_failure_kind: str
+
+ @property
+ def identity(self) -> tuple[str, str]:
+ return self.request_slot_id, self.floor_rekeyed_from_path
+
+ def to_payload(self) -> dict[str, str]:
+ return {
+ "request_slot_id": self.request_slot_id,
+ "floor_rekeyed_from_path": self.floor_rekeyed_from_path,
+ "pinability": self.pinability,
+ "mint_disposition": self.mint_disposition,
+ "mint_degrade": self.mint_degrade,
+ "request_slot_failure_kind": self.request_slot_failure_kind,
+ }
+
+
@dataclass(frozen=True)
class _OutputContractRequiredPaths:
"""Two-lane contract: observation paths must be sourced from the page/run; declaration paths
@@ -3021,15 +3064,49 @@ class _OutputContractRequiredPaths:
declaration_paths: set[str]
source: str
reason_code: str
+ degraded_request_slots: tuple[_DegradedRequestSlotDiagnostic, ...] = ()
@property
def union(self) -> set[str]:
return self.observation_paths | self.declaration_paths
+ @property
+ def liveness(self) -> _OutputContractLiveness:
+ if self.observation_paths:
+ return _OutputContractLiveness.VALUE_REQUIRED
+ if self.degraded_request_slots:
+ return _OutputContractLiveness.DEGRADED_EMPTY
+ return _OutputContractLiveness.ABSENT
+
+
+def _degraded_request_slot_diagnostics(ctx: AgentContext) -> tuple[_DegradedRequestSlotDiagnostic, ...]:
+ diagnostics: list[_DegradedRequestSlotDiagnostic] = []
+ request_policy = ctx.request_policy
+ request_slot_failure_kind = request_policy.request_slot_failure_kind if request_policy is not None else None
+ for criterion in _active_completion_criteria(ctx):
+ if not isinstance(criterion, CompletionCriterion):
+ continue
+ if not (criterion.mint_disposition == "degraded" or criterion.mint_degrade is not None):
+ continue
+ if not criterion.request_slot_id and request_slot_failure_kind is None:
+ continue
+ diagnostics.append(
+ _DegradedRequestSlotDiagnostic(
+ request_slot_id=criterion.request_slot_id or "",
+ floor_rekeyed_from_path=_canonical_requested_output_path(criterion.floor_rekeyed_from_path),
+ pinability=str(criterion.pinability or ""),
+ mint_disposition=criterion.mint_disposition,
+ mint_degrade=str(criterion.mint_degrade or ""),
+ request_slot_failure_kind=request_slot_failure_kind or "",
+ )
+ )
+ return tuple(sorted(diagnostics, key=lambda item: item.identity))
+
def _output_contract_required_paths_source(ctx: AgentContext) -> _OutputContractRequiredPaths:
runtime_contract = _runtime_output_repair_contract_from_recorded_outcome(ctx)
antecedent_paths = _contingent_antecedent_child_paths(ctx)
+ degraded_request_slots = _degraded_request_slot_diagnostics(ctx)
if runtime_contract is not None:
runtime_observation_paths = runtime_contract.required_paths - _judgment_output_paths(ctx)
return _OutputContractRequiredPaths(
@@ -3037,6 +3114,7 @@ def _output_contract_required_paths_source(ctx: AgentContext) -> _OutputContract
declaration_paths=antecedent_paths - runtime_observation_paths,
source=runtime_contract.source,
reason_code=runtime_contract.reason_code,
+ degraded_request_slots=degraded_request_slots,
)
observation_paths, source, reason_code = _required_child_output_paths_for_authoring(ctx)
observation_paths = observation_paths - _judgment_output_paths(ctx)
@@ -3069,6 +3147,7 @@ def _output_contract_required_paths_source(ctx: AgentContext) -> _OutputContract
declaration_paths=antecedent_paths - observation_paths,
source=source,
reason_code=reason_code,
+ degraded_request_slots=degraded_request_slots,
)
@@ -3076,12 +3155,230 @@ def _declaration_envelope_paths(declaration_paths: set[str]) -> set[str]:
return declaration_paths | {_output_path_root(path) for path in declaration_paths}
+def _mutation_root_name(expression: ast.expr) -> str:
+ node = expression
+ while isinstance(node, (ast.Attribute, ast.Starred, ast.Subscript)):
+ node = node.value
+ return node.id if isinstance(node, ast.Name) else ""
+
+
+def _statement_mutation_root_names(statement: ast.stmt) -> set[str]:
+ """Every name the statement could rebind or mutate, walking any target shape to its root
+ name; method receivers and call arguments count because the callee may write through them."""
+ names: set[str] = set()
+ for node in ast.walk(statement):
+ if isinstance(node, (ast.Name, ast.Subscript, ast.Attribute)) and isinstance(node.ctx, (ast.Store, ast.Del)):
+ if root := _mutation_root_name(node):
+ names.add(root)
+ elif isinstance(node, ast.ExceptHandler) and node.name:
+ names.add(node.name)
+ elif isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef)):
+ names.add(node.name)
+ elif isinstance(node, (ast.Import, ast.ImportFrom)):
+ names.update((alias.asname or alias.name).split(".", 1)[0] for alias in node.names)
+ elif isinstance(node, (ast.Global, ast.Nonlocal)):
+ names.update(node.names)
+ elif isinstance(node, ast.Call):
+ receivers = [node.func.value] if isinstance(node.func, ast.Attribute) else []
+ receivers.extend(node.args)
+ receivers.extend(keyword.value for keyword in node.keywords)
+ for receiver in receivers:
+ if root := _mutation_root_name(receiver):
+ names.add(root)
+ elif isinstance(node, (ast.Assign, ast.AnnAssign, ast.AugAssign)) and node.value is not None:
+ # Assigning into a subscript/attribute target aliases the RHS object into that
+ # container, so a later mutation through the container mutates the RHS too. Taint every
+ # RHS name so the alias is not resolved to its pre-mutation value (over-taint is fail-open).
+ targets = node.targets if isinstance(node, ast.Assign) else [node.target]
+ if any(isinstance(target, (ast.Subscript, ast.Attribute)) for target in targets):
+ names.update(child.id for child in ast.walk(node.value) if isinstance(child, ast.Name))
+ return names
+
+
+def _top_level_static_assignments(tree: ast.Module) -> tuple[dict[str, ast.expr], set[str]]:
+ """Only a single-name top-level assignment resolves statically; every other write or
+ mutation shape marks its root name uncertain, and uncertain is terminal (fail-open).
+ Mutating an alias mutates the aliased object, so the taint spreads to every name the
+ marked assignment references."""
+ assignments: dict[str, ast.expr] = {}
+ uncertain_names: set[str] = set()
+
+ def mark_uncertain(names: set[str]) -> None:
+ pending = list(names)
+ while pending:
+ name = pending.pop()
+ if name in uncertain_names:
+ continue
+ uncertain_names.add(name)
+ assigned = assignments.pop(name, None)
+ if assigned is not None:
+ pending.extend(node.id for node in ast.walk(assigned) if isinstance(node, ast.Name))
+
+ for node in tree.body:
+ target_name = ""
+ value: ast.expr | None = None
+ if isinstance(node, ast.Assign) and len(node.targets) == 1 and isinstance(node.targets[0], ast.Name):
+ target_name, value = node.targets[0].id, node.value
+ elif isinstance(node, ast.AnnAssign) and isinstance(node.target, ast.Name) and node.value is not None:
+ target_name, value = node.target.id, node.value
+ written = _statement_mutation_root_names(node)
+ if not target_name or value is None:
+ mark_uncertain(written)
+ continue
+ mark_uncertain(written - {target_name})
+ if target_name in assignments or target_name in uncertain_names:
+ mark_uncertain({target_name})
+ else:
+ assignments[target_name] = value
+ return assignments, uncertain_names
+
+
+def _resolve_static_expression(
+ expression: ast.expr,
+ assignments: Mapping[str, ast.expr],
+ uncertain_names: set[str],
+ seen_names: frozenset[str] = frozenset(),
+) -> ast.expr | None:
+ if isinstance(expression, ast.Await):
+ return _resolve_static_expression(expression.value, assignments, uncertain_names, seen_names)
+ if not isinstance(expression, ast.Name):
+ return expression
+ if expression.id in uncertain_names or expression.id in seen_names:
+ return None
+ assigned = assignments.get(expression.id)
+ if assigned is None:
+ return None
+ return _resolve_static_expression(assigned, assignments, uncertain_names, seen_names | {expression.id})
+
+
+_StaticOutputPathState = Literal["value", "empty", "absent", "unknown"]
+
+
+def _static_leaf_value_state(expression: ast.expr) -> _StaticOutputPathState:
+ if isinstance(expression, ast.Constant):
+ if expression.value is None or (isinstance(expression.value, str) and not expression.value.strip()):
+ return "empty"
+ return "value"
+ if isinstance(expression, (ast.List, ast.Tuple, ast.Set)):
+ element_states = {_static_leaf_value_state(element) for element in expression.elts}
+ return "value" if element_states & {"value", "unknown"} else "empty"
+ if isinstance(expression, ast.Dict):
+ value_states = {_static_leaf_value_state(value) for value in expression.values}
+ return "value" if value_states & {"value", "unknown"} else "empty"
+ return "unknown"
+
+
+def _static_output_path_state(
+ expression: ast.expr,
+ segments: list[tuple[str, bool]],
+ assignments: Mapping[str, ast.expr],
+ uncertain_names: set[str],
+) -> _StaticOutputPathState:
+ resolved = _resolve_static_expression(expression, assignments, uncertain_names)
+ if resolved is None:
+ return "unknown"
+ if not segments:
+ return _static_leaf_value_state(resolved)
+ key, is_array = segments[0]
+ if not isinstance(resolved, ast.Dict):
+ return "unknown"
+ matching_values: list[ast.expr] = []
+ dynamic_key = False
+ dynamic_key_after_match = False
+ for raw_key, value in zip(resolved.keys, resolved.values):
+ if isinstance(raw_key, ast.Constant) and raw_key.value == key:
+ matching_values.append(value)
+ dynamic_key_after_match = False
+ elif not isinstance(raw_key, ast.Constant) or not isinstance(raw_key.value, str):
+ dynamic_key = True
+ dynamic_key_after_match = bool(matching_values)
+ if not matching_values:
+ return "unknown" if dynamic_key else "absent"
+ # A dynamic entry after the last matching literal can shadow it at runtime.
+ if dynamic_key_after_match:
+ return "unknown"
+ value = matching_values[-1]
+ remaining = segments[1:]
+ if not is_array:
+ return _static_output_path_state(value, remaining, assignments, uncertain_names)
+ resolved_value = _resolve_static_expression(value, assignments, uncertain_names)
+ if not isinstance(resolved_value, (ast.List, ast.Tuple, ast.Set)):
+ return "unknown"
+ if not resolved_value.elts:
+ return "empty"
+ if not remaining:
+ return "value"
+ states = [
+ _static_output_path_state(element, remaining, assignments, uncertain_names) for element in resolved_value.elts
+ ]
+ if "value" in states:
+ return "value"
+ return "unknown" if "unknown" in states else "empty"
+
+
+def _statically_lacks_value_bearing_observation_paths(code: str, observation_paths: set[str]) -> bool:
+ if not observation_paths:
+ return False
+ try:
+ tree = ast.parse(textwrap.dedent(code).strip() or "pass")
+ return_expressions = [
+ node.value
+ for node in _iter_top_level_scope(tree.body)
+ if isinstance(node, ast.Return) and node.value is not None
+ ]
+ if not return_expressions:
+ return False
+ assignments, uncertain_names = _top_level_static_assignments(tree)
+ for expression in return_expressions:
+ if not isinstance(_resolve_static_expression(expression, assignments, uncertain_names), ast.Dict):
+ return False
+ states = {
+ _static_output_path_state(expression, _path_segments(path), assignments, uncertain_names)
+ for path in observation_paths
+ }
+ if states & {"value", "unknown"}:
+ return False
+ return True
+ except (RecursionError, SyntaxError, ValueError):
+ return False
+
+
+def _statically_valueless_return_envelope(code: str) -> bool:
+ try:
+ tree = ast.parse(textwrap.dedent(code).strip() or "pass")
+ return_nodes = [node for node in _iter_top_level_scope(tree.body) if isinstance(node, ast.Return)]
+ if not return_nodes:
+ return False
+ assignments, uncertain_names = _top_level_static_assignments(tree)
+ for node in return_nodes:
+ if node.value is None:
+ return False
+ resolved = _resolve_static_expression(node.value, assignments, uncertain_names)
+ if not isinstance(resolved, ast.Dict):
+ return False
+ if _static_leaf_value_state(resolved) != "empty":
+ return False
+ return True
+ except (RecursionError, SyntaxError, ValueError):
+ return False
+
+
+def _static_value_bearing_violations(code: str, observation_paths: set[str]) -> list[str]:
+ if not _statically_lacks_value_bearing_observation_paths(code, observation_paths):
+ return []
+ return [
+ "Unable to impose synthesized code block: selected output extraction returns only statically "
+ f"empty value(s) for required output path(s): {', '.join(sorted(observation_paths))}."
+ ]
+
+
def _evaluate_output_contract_for_code_block(
ctx: AgentContext,
workflow_yaml: str,
raw_code_artifact_metadata: object,
*,
allow_static_return_advisory: bool = False,
+ enforce_value_bearing_liveness: bool = False,
) -> _OutputContractEvaluation | None:
if _copilot_block_authoring_policy(ctx) != BlockAuthoringPolicy.CODE_ONLY_BROWSER:
return None
@@ -3092,7 +3389,7 @@ def _evaluate_output_contract_for_code_block(
declaration_paths = contract.declaration_paths
source = contract.source
reason_code = contract.reason_code
- if not required_paths:
+ if not required_paths and contract.liveness is not _OutputContractLiveness.DEGRADED_EMPTY:
return None
effective_metadata = raw_code_artifact_metadata
if not _metadata_has_mapping_item(effective_metadata):
@@ -3110,15 +3407,22 @@ def _evaluate_output_contract_for_code_block(
target_metadata = _metadata_item_for_block_label(effective_metadata, block_label) if block_label else None
submitted_goal_paths = _metadata_item_goal_value_paths(target_metadata)
submitted_schema_paths = _metadata_item_extraction_schema_paths(target_metadata) if target_metadata else set()
- submitted_code_paths = (
- _code_block_produced_output_paths(str(target_block.get("code") or "")) if target_block is not None else set()
- )
+ target_code = str(target_block.get("code") or "") if target_block is not None else ""
+ submitted_code_paths = _code_block_produced_output_paths(target_code)
missing_metadata_paths = sorted(observation_paths - submitted_goal_paths)
missing_schema_paths = sorted(required_paths - submitted_schema_paths)
missing_return_paths = sorted(required_paths - submitted_code_paths)
missing_observation_return_paths = sorted(observation_paths - submitted_code_paths)
missing_declaration_return_paths = sorted(declaration_paths - submitted_code_paths)
shape_violations: list[str] = []
+ declaration_only_contract = bool(declaration_paths) and not observation_paths
+ if enforce_value_bearing_liveness:
+ if contract.liveness is _OutputContractLiveness.DEGRADED_EMPTY or declaration_only_contract:
+ shape_violations.append(_OUTPUT_CONTRACT_VALUE_REQUIRED_REASON_CODE)
+ elif target_block is not None and _statically_lacks_value_bearing_observation_paths(
+ target_code, observation_paths
+ ):
+ shape_violations.append(_OUTPUT_CONTRACT_VALUE_REQUIRED_REASON_CODE)
if not block_label:
shape_violations.append("ambiguous_output_owner" if owner_labels else "missing_output_owner")
elif target_block is None:
@@ -3134,13 +3438,7 @@ def _evaluate_output_contract_for_code_block(
):
shape_violations.append(_SEPARATED_SPINE_SHAPE_REQUIRED_REASON_CODE)
- signature = _output_contract_signature(
- ctx=ctx,
- workflow_yaml=workflow_yaml,
- source=source,
- reason_code=reason_code,
- required_paths=required_paths,
- )
+ signature = _output_contract_signature(ctx=ctx, required_paths=required_paths)
separated_spine_advisory_run = (
allow_static_return_advisory
and target_block is not None
@@ -3209,12 +3507,19 @@ def _evaluate_output_contract_for_code_block(
| set(effective_missing_return_paths)
| (required_paths if shape_violations else set())
)
+ value_bearing_output_required = _OUTPUT_CONTRACT_VALUE_REQUIRED_REASON_CODE in shape_violations
payload: dict[str, Any] = {
- "reason_code": _OUTPUT_CONTRACT_REJECT_REASON_CODE,
+ "reason_code": (
+ _OUTPUT_CONTRACT_VALUE_REQUIRED_REASON_CODE
+ if value_bearing_output_required
+ else _OUTPUT_CONTRACT_REJECT_REASON_CODE
+ ),
"block_label": block_label,
"artifact_id": artifact_id,
"canonical_required_child_paths": sorted(required_paths),
"declaration_only_child_paths": sorted(declaration_paths),
+ "contract_liveness": contract.liveness.value,
+ "degraded_request_slots": [slot.to_payload() for slot in contract.degraded_request_slots],
"source": source,
"metadata_contract_source": source,
"metadata_contract_reason_code": reason_code,
@@ -3225,7 +3530,13 @@ def _evaluate_output_contract_for_code_block(
"actuated_static_return_advisory": actuated_static_return_advisory,
"shape_violations": shape_violations,
"can_attempt_run": run_eligible,
- "reject_reason": "" if run_eligible else _OUTPUT_CONTRACT_REJECT_REASON_CODE,
+ "reject_reason": ""
+ if run_eligible
+ else (
+ _OUTPUT_CONTRACT_VALUE_REQUIRED_REASON_CODE
+ if value_bearing_output_required
+ else _OUTPUT_CONTRACT_REJECT_REASON_CODE
+ ),
"canonical_output_contract_signature": signature,
"canonical_runtime_output_contract_signature": runtime_signature,
"runtime_output_workflow_run_id": runtime_contract.workflow_run_id if runtime_contract is not None else "",
@@ -3316,6 +3627,10 @@ def _adjudicate_output_contract_ladder_after_reject(
owner-directive path, not granted an inert run."""
if ctx.turn_halt is not None or ctx.output_contract_bail_actuated_this_call:
return None
+ # A statically valueless contract cannot be adjudicated by dispatching a run, so a
+ # value-bearing reject never mints an advisory grant that would yield the preflight.
+ if _OUTPUT_CONTRACT_VALUE_REQUIRED_REASON_CODE in evaluation.shape_violations:
+ return None
signature = evaluation.canonical_signature
block_label = evaluation.block_label
if not signature or not block_label:
@@ -3441,7 +3756,7 @@ def _record_output_contract_family_reject(
if not required_paths:
return 0
scope_key = _output_contract_scope_key(ctx)
- signature = _stable_output_contract_key(scope_key, required_paths)
+ signature = _output_contract_signature(ctx=ctx, required_paths=required_paths)
if not signature:
LOG.info(
"copilot_output_contract_reject_count_unscoped",
@@ -3483,7 +3798,7 @@ def _record_output_contract_family_reject(
def _record_output_contract_deferral(ctx: AgentContext, required_paths: set[str]) -> int:
if not required_paths:
return 0
- signature = _stable_output_contract_key(_output_contract_scope_key(ctx), required_paths)
+ signature = _output_contract_signature(ctx=ctx, required_paths=required_paths)
if not signature:
return 0
count = int(ctx.output_contract_deferral_count_by_signature.get(signature, 0) or 0) + 1
@@ -3498,11 +3813,20 @@ def _output_contract_reject_result(
tool_name: str = "update_workflow",
) -> dict[str, Any]:
data = payload or evaluation.payload
+ observation_paths = {str(path) for path in data.get("canonical_required_child_paths", []) or []} - {
+ str(path) for path in data.get("declaration_only_child_paths", []) or []
+ }
if data.get("reason_code") == _OUTPUT_CONTRACT_REJECT_BUDGET_REASON_CODE:
error = (
"The workflow output contract repair budget is exhausted for this canonical requested-output contract. "
"Return with the typed output-contract payload instead of trying another variant."
)
+ elif data.get("reason_code") == _OUTPUT_CONTRACT_VALUE_REQUIRED_REASON_CODE and not observation_paths:
+ error = (
+ f"{tool_name} cannot proceed: every requested value-bearing output has degraded out of the "
+ "completion criteria, so no code revision can satisfy the contract. Re-establish at least one "
+ "requested value-bearing output before authoring again."
+ )
else:
path_text = ", ".join(str(path) for path in data.get("canonical_required_child_paths", []) or [])
path_suffix = f" Required requested output paths: {path_text}." if path_text else ""
@@ -3710,15 +4034,11 @@ def _scaffold_metadata_contract_for_update(
raw_code_artifact_metadata: object,
) -> tuple[object, bool]:
contract = _output_contract_required_paths_source(ctx)
+ if contract.union and contract.liveness is not _OutputContractLiveness.VALUE_REQUIRED:
+ return raw_code_artifact_metadata, False
if not contract.union:
return _scaffold_metadata_from_owned_carrier_produced_output(ctx, workflow_yaml, raw_code_artifact_metadata)
- signature = _output_contract_signature(
- ctx=ctx,
- workflow_yaml=workflow_yaml,
- source=contract.source,
- reason_code=contract.reason_code,
- required_paths=contract.union,
- )
+ signature = _output_contract_signature(ctx=ctx, required_paths=contract.union)
scaffolded = _apply_metadata_contract_scaffold(
ctx,
workflow_yaml,
@@ -3741,7 +4061,7 @@ def _apply_metadata_contract_schema_to_workflow_yaml(
return workflow_yaml
contract = _output_contract_required_paths_source(ctx)
required_paths = contract.union
- if not required_paths:
+ if not required_paths or contract.liveness is not _OutputContractLiveness.VALUE_REQUIRED:
return workflow_yaml
label, owner_labels = _target_output_contract_block_label(
ctx,
@@ -3811,25 +4131,13 @@ def _output_contract_reject_count(ctx: AgentContext, signature: str) -> int:
return 0
-def _runtime_output_repair_attempt_key(
- ctx: AgentContext,
- workflow_yaml: str,
- required_paths: set[str],
- source: str,
- reason_code: str,
-) -> str:
+def _runtime_output_repair_attempt_key(ctx: AgentContext, required_paths: set[str]) -> str:
runtime_contract = _runtime_output_repair_contract_from_recorded_outcome(ctx)
outcome = getattr(ctx, "latest_recorded_build_test_outcome", None)
if runtime_contract is None or not isinstance(outcome, RecordedBuildTestOutcome):
return ""
payload = {
- "output_contract_signature": _output_contract_signature(
- ctx=ctx,
- workflow_yaml=workflow_yaml,
- source=source,
- reason_code=reason_code,
- required_paths=required_paths,
- ),
+ "output_contract_signature": _output_contract_signature(ctx=ctx, required_paths=required_paths),
"runtime_output_contract_signature": _runtime_output_contract_signature(runtime_contract),
"recorded_structural_key": outcome.structural_key,
"authored_structure_signature": outcome.authored_structure_signature,
@@ -3906,7 +4214,7 @@ def _produce_split_extraction_envelope(
)
if envelope is None:
return None
- _, revalidation = _extraction_code_with_required_static_return(
+ _, revalidation = _extraction_code_with_value_bearing_static_return(
envelope.code, required_paths=scalar_paths, declaration_paths=declarations
)
if revalidation:
@@ -3958,7 +4266,7 @@ def _attempt_separated_spine_split(
if suffix_mutations or suffix_ambiguous:
return _SpineSplitOutcome(None, ["extraction_suffix_contains_browser_actions"], None)
- keyed_extraction, static_violations = _extraction_code_with_required_static_return(
+ keyed_extraction, static_violations = _extraction_code_with_value_bearing_static_return(
extraction_suffix, required_paths=required_paths, declaration_paths=declaration_paths
)
if static_violations:
@@ -4039,6 +4347,26 @@ def _arm_output_contract_output_owner_directive(
)
+def _arm_output_contract_value_bearing_directive(
+ ctx: AgentContext,
+ *,
+ signature: str,
+ workflow_yaml: str,
+ liveness: _OutputContractLiveness,
+) -> None:
+ fingerprint = _output_contract_structural_fingerprint(workflow_yaml, signature)
+ already_armed = ctx.output_contract_armed_directive_fingerprint_by_signature.get(signature) == fingerprint
+ _record_armed_directive_fingerprint(ctx, signature, fingerprint)
+ if already_armed:
+ return
+ LOG.info(
+ "copilot_output_contract_value_bearing_directive_emitted",
+ canonical_output_contract_signature=signature,
+ reason_code=_OUTPUT_CONTRACT_VALUE_REQUIRED_REASON_CODE,
+ contract_liveness=liveness.value,
+ )
+
+
def _output_owner_ambiguity_repair_context(
*,
required_paths: set[str],
@@ -4190,6 +4518,16 @@ def _output_contract_advisory_granted(ctx: AgentContext, signature: str) -> bool
return _output_contract_advisory_state(ctx, signature) == OutputContractAdvisoryState.GRANTED
+def _revoke_output_contract_advisory_run(ctx: AgentContext, signature: str) -> None:
+ if _output_contract_advisory_granted(ctx, signature):
+ ctx.output_contract_actuation_by_signature.pop(signature, None)
+ LOG.info(
+ "copilot_output_contract_advisory_run_revoked",
+ canonical_output_contract_signature=signature,
+ reason_code=_OUTPUT_CONTRACT_VALUE_REQUIRED_REASON_CODE,
+ )
+
+
def _grant_output_contract_advisory_run(ctx: AgentContext, signature: str) -> None:
if not signature:
return
@@ -4232,7 +4570,7 @@ def consume_output_contract_advisory_grant_for_run_result(
workflow_run_id = _workflow_run_id_from_run_result(run_result)
if workflow_run_id is None:
data = run_result.get("data")
- if isinstance(data, Mapping):
+ if isinstance(data, Mapping) and run_result.get("ok") is not False:
LOG.warning(
"copilot_output_contract_advisory_run_result_missing_workflow_run_id",
data_keys=sorted(str(key) for key in data),
@@ -4597,9 +4935,13 @@ def _stamp_declaration_contract_defaults(
target_block = _workflow_yaml_code_blocks_by_label(workflow_yaml).get(label)
if target_block is None:
return workflow_yaml, False
- stamped_code = _code_with_declared_contract_defaults(
- str(target_block.get("code") or ""), contract.declaration_paths
- )
+ target_code = str(target_block.get("code") or "")
+ produced_observation_paths = contract.observation_paths & _code_block_produced_output_paths(target_code)
+ if not produced_observation_paths:
+ return workflow_yaml, False
+ if _statically_lacks_value_bearing_observation_paths(target_code, produced_observation_paths):
+ return workflow_yaml, False
+ stamped_code = _code_with_declared_contract_defaults(target_code, contract.declaration_paths)
if not stamped_code:
return workflow_yaml, False
parsed = parse_workflow_yaml(workflow_yaml)
@@ -4683,7 +5025,7 @@ def _impose_covered_static_return_envelope(
)
if envelope is None:
return None
- _, revalidation = _extraction_code_with_required_static_return(
+ _, revalidation = _extraction_code_with_value_bearing_static_return(
envelope.code, required_paths=scalar_paths, declaration_paths=declaration_paths
)
if revalidation:
@@ -4740,13 +5082,15 @@ def _impose_output_contract_envelope_after_steering(
reason_code = contract.reason_code
if not required_paths:
return workflow_yaml, raw_code_artifact_metadata, False
- signature = _output_contract_signature(
- ctx=ctx,
- workflow_yaml=workflow_yaml,
- source=source,
- reason_code=reason_code,
- required_paths=required_paths,
- )
+ signature = _output_contract_signature(ctx=ctx, required_paths=required_paths)
+ if contract.liveness is not _OutputContractLiveness.VALUE_REQUIRED:
+ _arm_output_contract_value_bearing_directive(
+ ctx,
+ signature=signature,
+ workflow_yaml=workflow_yaml,
+ liveness=contract.liveness,
+ )
+ return workflow_yaml, raw_code_artifact_metadata, False
# The stamp precedes actuation and ignores advisory state so no
# acceptance route can persist a block that omits the declaration paths.
workflow_yaml, declaration_stamped = _stamp_declaration_contract_defaults(
@@ -4757,7 +5101,7 @@ def _impose_output_contract_envelope_after_steering(
signature,
)
_reopen_dispatch_lacked_bound_extraction(ctx, signature)
- runtime_attempt_key = _runtime_output_repair_attempt_key(ctx, workflow_yaml, required_paths, source, reason_code)
+ runtime_attempt_key = _runtime_output_repair_attempt_key(ctx, required_paths)
label, owner_labels = _target_output_contract_block_label(
ctx,
workflow_yaml,
@@ -4849,7 +5193,7 @@ def _impose_output_contract_envelope_after_steering(
)
_record_armed_directive_fingerprint(ctx, signature, current_fingerprint)
return workflow_yaml, raw_code_artifact_metadata, declaration_stamped
- keyed_code, violations = _extraction_code_with_required_static_return(
+ keyed_code, violations = _extraction_code_with_value_bearing_static_return(
target_code,
required_paths=required_paths,
declaration_paths=declaration_paths,
@@ -5066,9 +5410,12 @@ def _metadata_contract_run_preflight_reject(
workflow_yaml,
raw_code_artifact_metadata,
allow_static_return_advisory=True,
+ enforce_value_bearing_liveness=True,
)
if evaluation is None or not evaluation.has_deficiencies:
return None
+ if _OUTPUT_CONTRACT_VALUE_REQUIRED_REASON_CODE in evaluation.shape_violations:
+ _revoke_output_contract_advisory_run(ctx, evaluation.canonical_signature)
authored_fingerprint = _output_contract_structural_fingerprint(workflow_yaml, evaluation.canonical_signature)
advisory_granted = _output_contract_advisory_granted(ctx, evaluation.canonical_signature)
# A granted advisory must arm run-output evidence before the grant is consumed, even when the
@@ -5090,7 +5437,9 @@ def _metadata_contract_run_preflight_reject(
authored_structural_fingerprint=authored_fingerprint,
workflow_yaml=workflow_yaml,
)
- if advisory_granted or _output_contract_advisory_granted(ctx, evaluation.canonical_signature):
+ if (
+ evaluation.can_attempt_run or _OUTPUT_CONTRACT_VALUE_REQUIRED_REASON_CODE not in evaluation.shape_violations
+ ) and (advisory_granted or _output_contract_advisory_granted(ctx, evaluation.canonical_signature)):
_arm_pending_run_evidence(ctx, evaluation.canonical_signature, set(evaluation.observation_paths))
return None
if evaluation.repair_context is not None:
@@ -5113,6 +5462,75 @@ def _metadata_contract_run_preflight_reject(
}
+def output_contract_value_bearing_run_reject(
+ ctx: AgentContext,
+ code_by_label: Mapping[str, str],
+) -> dict[str, Any] | None:
+ """Refuses dispatch of a saved workflow whose author contract is statically provably
+ valueless (declaration-only, degraded-empty, or statically-all-empty); unknown stays fail-open."""
+ if _copilot_block_authoring_policy(ctx) != BlockAuthoringPolicy.CODE_ONLY_BROWSER:
+ return None
+ contract = _output_contract_required_paths_source(ctx)
+ declaration_only_contract = bool(contract.declaration_paths) and not contract.observation_paths
+ statically_valueless = contract.liveness is _OutputContractLiveness.DEGRADED_EMPTY or declaration_only_contract
+ if not statically_valueless and contract.liveness is _OutputContractLiveness.VALUE_REQUIRED and code_by_label:
+ statically_valueless = all(
+ _statically_lacks_value_bearing_observation_paths(code, contract.observation_paths)
+ for code in code_by_label.values()
+ )
+ # The code lane is criteria-independent so the invariant survives drift in the persisted
+ # criteria source (an unparseable row must not readmit a statically hollow envelope).
+ statically_empty_return_envelope = False
+ if not statically_valueless and code_by_label:
+ statically_empty_return_envelope = all(
+ _statically_valueless_return_envelope(code) for code in code_by_label.values()
+ )
+ statically_valueless = statically_empty_return_envelope
+ if not statically_valueless:
+ return None
+ LOG.info(
+ "copilot_value_bearing_dispatch_preflight_reject",
+ contract_liveness=contract.liveness.value,
+ canonical_required_child_paths=sorted(contract.union),
+ declaration_only_child_paths=sorted(contract.declaration_paths),
+ statically_empty_return_envelope=statically_empty_return_envelope,
+ block_labels=sorted(code_by_label),
+ )
+ if statically_empty_return_envelope and not contract.union:
+ error = (
+ "Cannot run the saved workflow: its code statically returns no values (every return "
+ "envelope is empty or null-only). Revise the code to return at least one real value "
+ "before running."
+ )
+ elif contract.liveness is _OutputContractLiveness.DEGRADED_EMPTY or declaration_only_contract:
+ error = (
+ "Cannot run the saved workflow: every requested value-bearing output has degraded out of "
+ "the completion criteria, so no code revision can satisfy the contract. Re-establish at "
+ "least one requested value-bearing output before running."
+ )
+ else:
+ error = (
+ "Cannot run the saved workflow: its output contract carries no value-bearing output "
+ "(only a blocker declaration or statically empty values). Revise the owning code block "
+ "to produce at least one requested value-bearing output before running."
+ )
+ return {
+ "ok": False,
+ "error": error,
+ "user_facing_summary": _compiled_authoring_user_summary(),
+ "data": {
+ "reason_code": _OUTPUT_CONTRACT_VALUE_REQUIRED_REASON_CODE,
+ "reject_reason": _OUTPUT_CONTRACT_VALUE_REQUIRED_REASON_CODE,
+ "contract_liveness": contract.liveness.value,
+ "canonical_required_child_paths": sorted(contract.union),
+ "declaration_only_child_paths": sorted(contract.declaration_paths),
+ "statically_empty_return_envelope": statically_empty_return_envelope,
+ "degraded_request_slots": [slot.to_payload() for slot in contract.degraded_request_slots],
+ "can_attempt_run": False,
+ },
+ }
+
+
def _output_path_root(path: str) -> str:
return path.split(".", 1)[0].split("[", 1)[0].strip()
@@ -5396,6 +5814,23 @@ def _extraction_code_with_required_static_return(
]
+def _extraction_code_with_value_bearing_static_return(
+ code: str,
+ *,
+ required_paths: set[str],
+ declaration_paths: set[str] | None = None,
+) -> tuple[str, list[str]]:
+ keyed_code, violations = _extraction_code_with_required_static_return(
+ code, required_paths=required_paths, declaration_paths=declaration_paths
+ )
+ if violations:
+ return keyed_code, violations
+ liveness_violations = _static_value_bearing_violations(keyed_code, required_paths - (declaration_paths or set()))
+ if liveness_violations:
+ return textwrap.dedent(code).strip(), liveness_violations
+ return keyed_code, []
+
+
def _single_mapping_local_static_return_candidate(code: str, root: str, required_paths: set[str]) -> str:
"""When the extraction suffix, on all top-level paths, assigns exactly one dict-literal
local and never returns, key that one mapping to the required root. Bounded to a single
@@ -9448,7 +9883,7 @@ def _maybe_impose_synthesized_code_block(
target_metadata = _metadata_item_for_block_label(raw_metadata, str(code_block.get("label") or ""))
required_split_paths = _metadata_item_goal_value_paths(target_metadata)
if required_split_paths:
- split_extraction_code, static_return_violations = _extraction_code_with_required_static_return(
+ split_extraction_code, static_return_violations = _extraction_code_with_value_bearing_static_return(
split_extraction_code,
required_paths=required_split_paths,
)
@@ -10826,7 +11261,13 @@ async def _update_workflow(
workflow_yaml,
params.get("code_artifact_metadata"),
allow_static_return_advisory=allow_static_output_uncertainty,
+ enforce_value_bearing_liveness=True,
)
+ if (
+ output_contract_evaluation is not None
+ and _OUTPUT_CONTRACT_VALUE_REQUIRED_REASON_CODE in output_contract_evaluation.shape_violations
+ ):
+ _revoke_output_contract_advisory_run(ctx, output_contract_evaluation.canonical_signature)
output_contract_static_advisory_allowed = (
output_contract_evaluation is not None and output_contract_evaluation.can_attempt_run
)
diff --git a/tests/unit/test_copilot_agent_helpers.py b/tests/unit/test_copilot_agent_helpers.py
index b76321a4b..124fdad54 100644
--- a/tests/unit/test_copilot_agent_helpers.py
+++ b/tests/unit/test_copilot_agent_helpers.py
@@ -2213,13 +2213,7 @@ workflow_definition:
last_code_authoring_repair_context=repair_context,
)
ctx.turn_id = "metadata-contract-run-preflight"
- signature = workflow_update_module._output_contract_signature(
- ctx=ctx,
- workflow_yaml=workflow_yaml,
- source="requested_output_contract",
- reason_code="requested_output_contract_missing_output_coverage",
- required_paths=required_paths,
- )
+ signature = workflow_update_module._output_contract_signature(ctx=ctx, required_paths=required_paths)
workflow_update_module._grant_output_contract_advisory_run(ctx, signature)
captured: dict[str, object] = {}
diff --git a/tests/unit/test_copilot_code_block_persist_seam.py b/tests/unit/test_copilot_code_block_persist_seam.py
index fbf17c7e7..ed8247c2c 100644
--- a/tests/unit/test_copilot_code_block_persist_seam.py
+++ b/tests/unit/test_copilot_code_block_persist_seam.py
@@ -10,7 +10,7 @@ import hashlib
import inspect
import json
import textwrap
-from types import SimpleNamespace
+from types import EllipsisType, SimpleNamespace
from unittest.mock import AsyncMock, Mock
import pytest
@@ -94,6 +94,7 @@ from skyvern.forge.sdk.copilot.tools import (
_detect_stale_block_metadata,
_update_workflow,
)
+from skyvern.forge.sdk.copilot.tools import run_execution as run_execution_module
from skyvern.forge.sdk.copilot.tools import scouting as scouting_module
from skyvern.forge.sdk.copilot.tools import workflow_update as workflow_update_module
from skyvern.forge.sdk.copilot.tools.workflow_update import (
@@ -4096,7 +4097,7 @@ class TestCodeRepairProgressClassification:
- block_type: code
label: extract_entry_output
code: |
- return build_output()
+ return {"output": build_output()}
"""
)
@@ -4167,13 +4168,7 @@ class TestCodeRepairProgressClassification:
await page.get_by_role("button", name="Submit").click()
"""
)
- signature = workflow_update_module._output_contract_signature(
- ctx=ctx,
- workflow_yaml=workflow_yaml,
- source="requested_output_contract",
- reason_code="requested_output_contract_missing_output_coverage",
- required_paths=required_paths,
- )
+ signature = workflow_update_module._output_contract_signature(ctx=ctx, required_paths=required_paths)
workflow_update_module._grant_output_contract_advisory_run(ctx, signature)
evaluation = workflow_update_module._evaluate_output_contract_for_code_block(
@@ -4253,13 +4248,7 @@ class TestCodeRepairProgressClassification:
"""
)
ctx.turn_id = "run-static-return-gap"
- signature = workflow_update_module._output_contract_signature(
- ctx=ctx,
- workflow_yaml=workflow_yaml,
- source="requested_output_contract",
- reason_code="requested_output_contract_missing_output_coverage",
- required_paths={"output.record_id"},
- )
+ signature = workflow_update_module._output_contract_signature(ctx=ctx, required_paths={"output.record_id"})
workflow_update_module._grant_output_contract_advisory_run(ctx, signature)
result = await _update_workflow(
@@ -4498,7 +4487,7 @@ class TestCodeRepairProgressClassification:
assert result is not None
assert result["ok"] is False
- assert result["data"]["reason_code"] == "output_contract_required"
+ assert result["data"]["reason_code"] == "value_bearing_output_required"
assert result["data"]["output_contract_reject_count"] == 4
assert result["data"]["canonical_required_child_paths"] == ["output.record_id"]
@@ -4507,43 +4496,16 @@ class TestCodeRepairProgressClassification:
ctx.turn_id = "contract-scope-a"
first = workflow_update_module._output_contract_signature(
ctx=ctx,
- workflow_yaml=_yaml(
- """
- title: First draft title
- workflow_definition:
- blocks: []
- """
- ),
- source="requested_output_contract",
- reason_code="requested_output_contract_missing_output_coverage",
required_paths={"output.locations[].address", "output.statuses"},
)
second = workflow_update_module._output_contract_signature(
ctx=ctx,
- workflow_yaml=_yaml(
- """
- title: Changed draft title
- workflow_definition:
- blocks: []
- """
- ),
- source="runtime_output_repair",
- reason_code="runtime_output_repair_required",
required_paths={"output.statuses", "output.locations[].address"},
)
other_ctx = _code_only_ctx()
other_ctx.turn_id = "contract-scope-b"
other_scope = workflow_update_module._output_contract_signature(
ctx=other_ctx,
- workflow_yaml=_yaml(
- """
- title: Changed draft title
- workflow_definition:
- blocks: []
- """
- ),
- source="runtime_output_repair",
- reason_code="runtime_output_repair_required",
required_paths={"output.statuses", "output.locations[].address"},
)
@@ -4578,13 +4540,7 @@ class TestCodeRepairProgressClassification:
reject_family="declared_output_return_shape",
)
- signature = workflow_update_module._output_contract_signature(
- ctx=ctx,
- workflow_yaml="title: Any\nworkflow_definition:\n blocks: []\n",
- source="different",
- reason_code="different",
- required_paths=required_paths,
- )
+ signature = workflow_update_module._output_contract_signature(ctx=ctx, required_paths=required_paths)
assert (first, second, third) == (1, 2, 3)
assert ctx.output_contract_reject_count_by_signature[signature] == 3
@@ -4954,7 +4910,7 @@ class TestCodeRepairProgressClassification:
assert result is not None
assert result["ok"] is False
assert result["data"]["reason_code"] == "metadata_contract_required_before_run"
- assert result["data"]["output_contract_reason_code"] == "output_contract_required"
+ assert result["data"]["output_contract_reason_code"] == "value_bearing_output_required"
assert result["data"]["block_label"] == "extract_entry_output"
assert result["data"]["canonical_required_child_paths"] == ["output.flags", "output.record_id"]
assert result["data"]["missing_goal_value_paths"] == ["output.flags", "output.record_id"]
@@ -5099,7 +5055,7 @@ class TestCodeRepairProgressClassification:
assert results[0] is not None
assert results[0]["ok"] is False
assert results[0]["data"]["reason_code"] == "metadata_contract_required_before_run"
- assert results[0]["data"]["output_contract_reason_code"] == "output_contract_required"
+ assert results[0]["data"]["output_contract_reason_code"] == "value_bearing_output_required"
assert results[0]["data"]["output_contract_reject_count"] == 1
assert results[-1] is not None
assert results[-1]["data"]["output_contract_reject_count"] == 4
@@ -8555,17 +8511,11 @@ class TestSeparatedSpineImpositionRunEligibility:
ctx, workflow_yaml, []
)
contract = workflow_update_module._output_contract_required_paths_source(ctx)
- required_paths, source, reason_code = contract.union, contract.source, contract.reason_code
+ required_paths = contract.union
read_label, _owner_labels = workflow_update_module._target_output_contract_block_label(
ctx, workflow_yaml, scaffolded_metadata, required_paths
)
- read_signature = workflow_update_module._output_contract_signature(
- ctx=ctx,
- workflow_yaml=workflow_yaml,
- source=source,
- reason_code=reason_code,
- required_paths=required_paths,
- )
+ read_signature = workflow_update_module._output_contract_signature(ctx=ctx, required_paths=required_paths)
read_key = workflow_update_module._output_contract_spine_directive_attempt_key(
signature=read_signature, block_label=read_label, workflow_yaml=workflow_yaml
)
@@ -14040,23 +13990,344 @@ def _declaration_stamp_ctx() -> CopilotContext:
return ctx
+def _degraded_declaration_stamp_ctx() -> CopilotContext:
+ ctx = _code_only_ctx()
+ ctx.turn_id = "t-degraded-decl"
+ ctx.scout_trajectory = []
+ requested_paths = (
+ "output.confirmation_number",
+ "output.account_number",
+ "output.start_date",
+ "output.deposit_amount",
+ "output.next_owner",
+ )
+ ctx.request_policy = RequestPolicy(
+ completion_criteria=[
+ *[
+ CompletionCriterion(
+ id=f"slot_{index}",
+ outcome=f"The returned record includes {path}.",
+ request_slot_id=f"slot_{index}",
+ pinability="unpinnable",
+ mint_disposition="degraded",
+ mint_degrade="undecidable_judgment",
+ requested_output_floor_rekeyed=True,
+ floor_rekeyed_from_path=path,
+ )
+ for index, path in enumerate(requested_paths)
+ ],
+ CompletionCriterion(
+ id="c_blocker",
+ outcome="A blocker is reported when the site blocks submission.",
+ contingent_on="the site blocks submission",
+ contingent_antecedent_output_path="output.blocker",
+ ),
+ ]
+ )
+ return ctx
+
+
class TestDeclarationContractStamp:
- def test_zero_output_block_persists_declared_blocker_on_first_contact(self) -> None:
+ @pytest.mark.parametrize("empty_value", ["None", "''", "' '", "[]", "{}", "[None]", "[{}]", "{'a': None}"])
+ def test_static_empty_observation_value_is_non_run_eligible(self, empty_value: str) -> None:
ctx = _declaration_stamp_ctx()
- workflow_yaml = _collapsed_spine_yaml('await page.click("#submit")')
+ workflow_yaml = _collapsed_spine_yaml(f'return {{"output": {{"record_id": {empty_value}, "blocker": None}}}}')
+ metadata = [
+ workflow_update_module._metadata_contract_template(
+ block_label="extract_record",
+ required_paths={"output.record_id"},
+ source="requested_output_contract",
+ reason_code="requested_output_contract_missing_output_coverage",
+ declaration_paths={"output.blocker"},
+ )
+ ]
- new_yaml, _metadata, applied = workflow_update_module._impose_output_contract_envelope_after_steering(
+ evaluation = workflow_update_module._evaluate_output_contract_for_code_block(
+ ctx,
+ workflow_yaml,
+ metadata,
+ allow_static_return_advisory=True,
+ enforce_value_bearing_liveness=True,
+ )
+
+ assert evaluation is not None
+ assert evaluation.missing_metadata_paths == []
+ assert evaluation.missing_schema_paths == []
+ assert evaluation.missing_return_paths == []
+ assert evaluation.payload["reason_code"] == "value_bearing_output_required"
+ assert evaluation.can_attempt_run is False
+
+ @pytest.mark.parametrize("value", ["0", "False", "'record-123'", "[None, 'x']", "{'a': 'x'}"])
+ def test_static_value_bearing_observation_remains_run_eligible(self, value: str) -> None:
+ ctx = _declaration_stamp_ctx()
+ workflow_yaml = _collapsed_spine_yaml(f'return {{"output": {{"record_id": {value}, "blocker": None}}}}')
+ metadata = [
+ workflow_update_module._metadata_contract_template(
+ block_label="extract_record",
+ required_paths={"output.record_id"},
+ source="requested_output_contract",
+ reason_code="requested_output_contract_missing_output_coverage",
+ declaration_paths={"output.blocker"},
+ )
+ ]
+
+ evaluation = workflow_update_module._evaluate_output_contract_for_code_block(
+ ctx, workflow_yaml, metadata, enforce_value_bearing_liveness=True
+ )
+
+ assert evaluation is not None
+ assert evaluation.has_deficiencies is False
+
+ def test_full_metadata_advisory_rejects_declaration_only_return(self) -> None:
+ ctx = _declaration_stamp_ctx()
+ required_paths = {"output.record_id", "output.blocker"}
+ signature = workflow_update_module._stable_output_contract_key(
+ workflow_update_module._output_contract_scope_key(ctx), required_paths
+ )
+ workflow_update_module._grant_output_contract_advisory_run(ctx, signature)
+ workflow_yaml = _collapsed_spine_yaml('await page.click("#submit")\nreturn {"output": {"blocker": None}}')
+ metadata = [
+ workflow_update_module._metadata_contract_template(
+ block_label="extract_record",
+ required_paths={"output.record_id"},
+ source="requested_output_contract",
+ reason_code="requested_output_contract_missing_output_coverage",
+ declaration_paths={"output.blocker"},
+ )
+ ]
+
+ evaluation = workflow_update_module._evaluate_output_contract_for_code_block(
+ ctx,
+ workflow_yaml,
+ metadata,
+ allow_static_return_advisory=True,
+ enforce_value_bearing_liveness=True,
+ )
+
+ assert evaluation is not None
+ assert evaluation.missing_metadata_paths == []
+ assert evaluation.missing_schema_paths == []
+ assert evaluation.missing_return_paths == ["output.record_id"]
+ assert evaluation.payload["static_return_advisory_paths"] == []
+ assert evaluation.payload["reason_code"] == "value_bearing_output_required"
+ assert evaluation.can_attempt_run is False
+
+ @pytest.mark.asyncio
+ async def test_full_metadata_declaration_only_contract_rejects_at_persist_seam(self) -> None:
+ ctx = _declaration_stamp_ctx()
+ workflow_yaml = _collapsed_spine_yaml('await page.click("#submit")\nreturn {"output": {"blocker": None}}')
+ metadata = [
+ workflow_update_module._metadata_contract_template(
+ block_label="extract_record",
+ required_paths={"output.record_id"},
+ source="requested_output_contract",
+ reason_code="requested_output_contract_missing_output_coverage",
+ declaration_paths={"output.blocker"},
+ )
+ ]
+
+ result = await _update_workflow(
+ {"workflow_yaml": workflow_yaml, "code_artifact_metadata": metadata},
+ ctx,
+ allow_missing_credentials=True,
+ )
+
+ assert result["ok"] is False
+ assert result["data"]["reason_code"] == "value_bearing_output_required"
+ assert result["data"]["contract_liveness"] == "value_required"
+ assert result["data"]["missing_goal_value_paths"] == []
+ assert result["data"]["missing_extraction_schema_paths"] == []
+ assert result["data"]["missing_code_return_paths"] == ["output.record_id"]
+
+ def test_grant_preflight_rejects_declaration_only_return(self) -> None:
+ ctx = _declaration_stamp_ctx()
+ workflow_yaml = _collapsed_spine_yaml('return {"output": {"blocker": None}}')
+ evaluation = workflow_update_module._evaluate_output_contract_for_code_block(
+ ctx,
+ workflow_yaml,
+ [],
+ allow_static_return_advisory=True,
+ enforce_value_bearing_liveness=True,
+ )
+ assert evaluation is not None
+ workflow_update_module._grant_output_contract_advisory_run(ctx, evaluation.canonical_signature)
+
+ preflight = workflow_update_module._metadata_contract_run_preflight_reject(ctx, workflow_yaml, [])
+
+ assert evaluation.payload["reason_code"] == "value_bearing_output_required"
+ assert evaluation.can_attempt_run is False
+ assert preflight is not None
+ assert preflight["data"]["output_contract_reason_code"] == "value_bearing_output_required"
+ assert (
+ workflow_update_module._output_contract_advisory_state(ctx, evaluation.canonical_signature)
+ == OutputContractAdvisoryState.UNUSED
+ )
+ assert evaluation.canonical_signature not in ctx.output_contract_pending_run_evidence
+
+ def test_degraded_requested_slots_make_declaration_only_contract_non_run_eligible(self) -> None:
+ ctx = _degraded_declaration_stamp_ctx()
+ workflow_yaml = _collapsed_spine_yaml('return {"output": {"blocker": None}}')
+
+ formed_yaml, metadata, applied = workflow_update_module._impose_output_contract_envelope_after_steering(
+ ctx, workflow_yaml, []
+ )
+ evaluation = workflow_update_module._evaluate_output_contract_for_code_block(
+ ctx,
+ formed_yaml,
+ metadata,
+ allow_static_return_advisory=True,
+ enforce_value_bearing_liveness=True,
+ )
+
+ assert applied is False
+ assert formed_yaml == workflow_yaml
+ assert evaluation is not None
+ signature = evaluation.canonical_signature
+ assert ctx.output_contract_armed_directive_fingerprint_by_signature.get(
+ signature
+ ) == workflow_update_module._output_contract_structural_fingerprint(workflow_yaml, signature)
+ assert evaluation.payload["contract_liveness"] == "degraded_empty"
+ assert evaluation.payload["reason_code"] == "value_bearing_output_required"
+ assert evaluation.can_attempt_run is False
+ assert [row["request_slot_id"] for row in evaluation.payload["degraded_request_slots"]] == [
+ "slot_0",
+ "slot_1",
+ "slot_2",
+ "slot_3",
+ "slot_4",
+ ]
+ assert [row["floor_rekeyed_from_path"] for row in evaluation.payload["degraded_request_slots"]] == [
+ "output.confirmation_number",
+ "output.account_number",
+ "output.start_date",
+ "output.deposit_amount",
+ "output.next_owner",
+ ]
+
+ def test_degraded_declaration_only_directive_arm_is_idempotent(self) -> None:
+ ctx = _degraded_declaration_stamp_ctx()
+ workflow_yaml = _collapsed_spine_yaml('return {"output": {"blocker": None}}')
+
+ first_yaml, _, first_applied = workflow_update_module._impose_output_contract_envelope_after_steering(
+ ctx, workflow_yaml, []
+ )
+ second_yaml, _, second_applied = workflow_update_module._impose_output_contract_envelope_after_steering(
+ ctx, first_yaml, []
+ )
+
+ assert first_applied is False
+ assert second_applied is False
+ assert first_yaml == workflow_yaml
+ assert second_yaml == workflow_yaml
+ assert len(ctx.output_contract_armed_directive_fingerprint_by_signature) == 1
+
+ def test_second_run_request_slot_producer_failure_rejects_blocker_only_return(self) -> None:
+ requested_paths = (
+ "output.confirmation_number",
+ "output.account_number",
+ "output.start_date",
+ "output.deposit_amount",
+ "output.next_owner",
+ )
+ ctx = _code_only_ctx()
+ ctx.turn_id = "t-request-slot-producer-failure"
+ ctx.scout_trajectory = []
+ ctx.request_policy = request_policy_module._classification_from_raw(
+ {
+ "completion_criteria": [
+ {
+ "outcome": f"The returned record includes {path}.",
+ "output_path": path,
+ "request_slot_source_id": "latest_request",
+ "request_slot_source_quote": path,
+ }
+ for path in requested_paths
+ ]
+ },
+ request_slot_failure_kind="invalid_output",
+ )
+ workflow_yaml = _collapsed_spine_yaml('return {"output": {"blocker": None}}')
+
+ evaluation = workflow_update_module._evaluate_output_contract_for_code_block(
+ ctx,
+ workflow_yaml,
+ [],
+ allow_static_return_advisory=True,
+ enforce_value_bearing_liveness=True,
+ )
+ preflight = workflow_update_module._metadata_contract_run_preflight_reject(ctx, workflow_yaml, [])
+
+ assert ctx.request_policy.request_slot_failure_kind == "invalid_output"
+ assert all(criterion.output_path is None for criterion in ctx.request_policy.completion_criteria)
+ assert all(criterion.mint_disposition == "degraded" for criterion in ctx.request_policy.completion_criteria)
+ assert evaluation is not None
+ assert evaluation.payload["contract_liveness"] == "degraded_empty"
+ assert evaluation.payload["reason_code"] == "value_bearing_output_required"
+ assert evaluation.can_attempt_run is False
+ assert [row["floor_rekeyed_from_path"] for row in evaluation.payload["degraded_request_slots"]] == sorted(
+ requested_paths
+ )
+ assert {row["request_slot_failure_kind"] for row in evaluation.payload["degraded_request_slots"]} == {
+ "invalid_output"
+ }
+ assert preflight is not None
+ assert preflight["data"]["output_contract_reason_code"] == "value_bearing_output_required"
+
+ @pytest.mark.asyncio
+ async def test_degraded_declaration_only_contract_never_persists_or_dispatches(
+ self, monkeypatch: pytest.MonkeyPatch
+ ) -> None:
+ ctx = _degraded_declaration_stamp_ctx()
+ workflow_yaml = _collapsed_spine_yaml('return {"output": {"blocker": None}}')
+ persisted = False
+ dispatched = False
+
+ async def _unexpected_update(*_args: object, **_kwargs: object) -> dict[str, object]:
+ nonlocal persisted
+ persisted = True
+ return {"ok": True}
+
+ async def _unexpected_dispatch(*_args: object, **_kwargs: object) -> dict[str, object]:
+ nonlocal dispatched
+ dispatched = True
+ return {"ok": True}
+
+ monkeypatch.setattr(tools_module, "_authority_tool_error", lambda *_args, **_kwargs: None)
+ monkeypatch.setattr(tools_module, "_update_workflow", _unexpected_update)
+ monkeypatch.setattr(tools_module, "_run_blocks_and_collect_debug", _unexpected_dispatch)
+
+ result = await tools_module.update_and_run_blocks_tool.on_invoke_tool(
+ SimpleNamespace(context=ctx, tool_name="update_and_run_blocks"),
+ json.dumps({"workflow_yaml": workflow_yaml, "block_labels": ["extract_entry_output"]}),
+ )
+
+ parsed = json.loads(result)
+ assert parsed["ok"] is False
+ assert parsed["data"]["reason_code"] == "metadata_contract_required_before_run"
+ assert parsed["data"]["output_contract_reason_code"] == "value_bearing_output_required"
+ assert parsed["data"]["contract_liveness"] == "degraded_empty"
+ assert persisted is False
+ assert dispatched is False
+
+ def test_mixed_value_and_declaration_return_remains_valid(self) -> None:
+ ctx = _declaration_stamp_ctx()
+ workflow_yaml = _collapsed_spine_yaml(
+ 'record_id = await page.locator("#record-id").text_content()\nreturn {"output": {"record_id": record_id}}'
+ )
+
+ formed_yaml, metadata, applied = workflow_update_module._impose_output_contract_envelope_after_steering(
ctx, workflow_yaml, []
)
+ evaluation = workflow_update_module._evaluate_output_contract_for_code_block(ctx, formed_yaml, metadata)
assert applied is True
- code = str(workflow_blocks(parse_workflow_yaml(new_yaml))[0].get("code") or "")
- produced = workflow_update_module._code_block_produced_output_paths(code)
- assert "output.blocker" in produced # nosemgrep: incomplete-url-substring-sanitization
- assert "output.record_id" not in produced
+ assert evaluation is not None
+ assert evaluation.has_deficiencies is False
+ code = str(workflow_blocks(parse_workflow_yaml(formed_yaml))[0].get("code") or "")
+ assert '"record_id": record_id' in code
assert '"blocker": None' in code
- def test_stamp_applies_before_output_contract_actuation(self) -> None:
+ def test_formation_does_not_create_declaration_only_scaffold(self) -> None:
ctx = _declaration_stamp_ctx()
workflow_yaml = _collapsed_spine_yaml('await page.click("#submit")\nreturn {}')
@@ -14065,8 +14336,12 @@ class TestDeclarationContractStamp:
)
assert applied is True
+ assert new_yaml == workflow_yaml
code = str(workflow_blocks(parse_workflow_yaml(new_yaml))[0].get("code") or "")
- assert 'return {"output": {"blocker": None}}' in code
+ assert "blocker" not in code
+ evaluation = workflow_update_module._evaluate_output_contract_for_code_block(ctx, new_yaml, _metadata)
+ assert evaluation is not None
+ assert evaluation.can_attempt_run is False
def test_stamp_is_idempotent_across_calls(self) -> None:
ctx = _declaration_stamp_ctx()
@@ -14104,6 +14379,589 @@ class TestDeclarationContractStamp:
assert applied is False
assert new_yaml == workflow_yaml
+ def test_stamp_refused_for_statically_empty_observation_code(self) -> None:
+ ctx = _declaration_stamp_ctx()
+ workflow_yaml = _collapsed_spine_yaml('return {"output": {"record_id": None}}')
+ contract = workflow_update_module._output_contract_required_paths_source(ctx)
+
+ stamped_yaml, applied = workflow_update_module._stamp_declaration_contract_defaults(
+ ctx, workflow_yaml, [], contract, "sig-declaration-stamp"
+ )
+
+ assert applied is False
+ assert stamped_yaml == workflow_yaml
+
+ def test_stamp_applies_for_runtime_populated_observation_code(self) -> None:
+ ctx = _declaration_stamp_ctx()
+ workflow_yaml = _collapsed_spine_yaml(
+ 'record_id = await page.inner_text("#record")\nreturn {"output": {"record_id": record_id}}'
+ )
+ contract = workflow_update_module._output_contract_required_paths_source(ctx)
+
+ stamped_yaml, applied = workflow_update_module._stamp_declaration_contract_defaults(
+ ctx, workflow_yaml, [], contract, "sig-declaration-stamp"
+ )
+
+ assert applied is True
+ code = str(workflow_blocks(parse_workflow_yaml(stamped_yaml))[0].get("code") or "")
+ assert '"blocker": None' in code
+
+
+def test_static_return_synthesis_refuses_statically_none_observation_local() -> None:
+ code = "record_id = None"
+
+ keyed, violations = workflow_update_module._extraction_code_with_value_bearing_static_return(
+ code, required_paths={"output.record_id"}, declaration_paths={"output.blocker"}
+ )
+
+ assert violations
+ assert keyed == code
+
+
+def test_static_return_synthesis_accepts_runtime_populated_observation_local() -> None:
+ code = 'record_id = await page.inner_text("#record")'
+
+ keyed, violations = workflow_update_module._extraction_code_with_value_bearing_static_return(
+ code, required_paths={"output.record_id"}, declaration_paths={"output.blocker"}
+ )
+
+ assert violations == []
+ assert {"output.record_id", "output.blocker"} <= workflow_update_module._code_block_produced_output_paths(keyed)
+
+
+def test_static_return_synthesis_refuses_covered_statically_empty_return() -> None:
+ code = 'return {"output": {"record_id": None, "blocker": None}}'
+
+ _, violations = workflow_update_module._extraction_code_with_value_bearing_static_return(
+ code, required_paths={"output.record_id"}, declaration_paths={"output.blocker"}
+ )
+
+ assert violations
+
+
+def test_value_bearing_lattice_unpack_after_empty_literal_is_fail_open() -> None:
+ code = 'extra = await collect()\nreturn {"output": {"record_id": None, **extra}}'
+
+ assert workflow_update_module._statically_lacks_value_bearing_observation_paths(code, {"output.record_id"}) is False
+
+
+def test_value_bearing_lattice_top_level_unpack_after_output_key_is_fail_open() -> None:
+ code = 'extra = await collect()\nreturn {"output": {"record_id": None}, **extra}'
+
+ assert workflow_update_module._statically_lacks_value_bearing_observation_paths(code, {"output.record_id"}) is False
+
+
+def test_value_bearing_lattice_empty_literal_after_unpack_stays_refused() -> None:
+ code = 'extra = await collect()\nreturn {"output": {**extra, "record_id": None}}'
+
+ assert workflow_update_module._statically_lacks_value_bearing_observation_paths(code, {"output.record_id"}) is True
+
+
+ "mutation",
+ [
+ "payload, extra = await split()",
+ "*rest, payload = await split()",
+ 'with open("audit.log") as payload:\n pass',
+ "try:\n pass\nexcept ValueError as payload:\n pass",
+ "if (payload := await build()):\n pass",
+ 'payload["output"]["record_id"] = await fetch()',
+ 'payload["output"].update(await fetch())',
+ 'for item in await rows():\n payload["output"]["items"].append(item)',
+ "populate(payload)",
+ "payload.output = await fetch()",
+ 'alias = payload\nalias["output"]["record_id"] = await fetch()',
+ 'alias = payload\nalias["output"].update(await fetch())',
+ 'inner = payload["output"]\ninner["record_id"] = await fetch()',
+ "alias = payload\npopulate(alias)",
+ "first = payload\nsecond = first\nsecond.output = await fetch()",
+ ],
+)
+def test_value_bearing_lattice_mutated_literal_is_fail_open(mutation: str) -> None:
+ code = f'payload = {{"output": {{"record_id": None}}}}\n{mutation}\nreturn payload'
+
+ assert workflow_update_module._statically_lacks_value_bearing_observation_paths(code, {"output.record_id"}) is False
+ assert workflow_update_module._statically_valueless_return_envelope(code) is False
+
+
+def test_value_bearing_lattice_unmutated_literal_stays_refused() -> None:
+ code = 'payload = {"output": {"record_id": None}}\nother = await fetch()\nreturn payload'
+
+ assert workflow_update_module._statically_lacks_value_bearing_observation_paths(code, {"output.record_id"}) is True
+ assert workflow_update_module._statically_valueless_return_envelope(code) is True
+
+
+def test_value_bearing_lattice_unmutated_alias_stays_refused() -> None:
+ code = 'payload = {"output": {"record_id": None}}\nalias = payload\nreturn payload'
+
+ assert workflow_update_module._statically_lacks_value_bearing_observation_paths(code, {"output.record_id"}) is True
+ assert workflow_update_module._statically_valueless_return_envelope(code) is True
+
+
+ "aliasing",
+ [
+ 'holder = {}\nholder["slot"] = payload',
+ 'holder = {}\nholder["slot"] = payload\nholder["slot"]["output"]["record_id"] = await fetch()',
+ 'holder = {}\nholder["slot"] = payload\nholder["slot"]["output"].update(await fetch())',
+ 'holder = obj()\nholder.slot = payload\nholder.slot["output"]["record_id"] = await fetch()',
+ ],
+)
+def test_value_bearing_lattice_alias_into_container_target_is_fail_open(aliasing: str) -> None:
+ # Assigning payload into a subscript/attribute target aliases it, so a mutation through the
+ # container could make the return value-bearing; the value name must taint (fail-open), not
+ # resolve to its pre-alias empty literal.
+ code = f'payload = {{"output": {{"record_id": None}}}}\n{aliasing}\nreturn payload'
+
+ assert workflow_update_module._statically_lacks_value_bearing_observation_paths(code, {"output.record_id"}) is False
+ assert workflow_update_module._statically_valueless_return_envelope(code) is False
+
+
+def test_value_bearing_lattice_nul_byte_source_is_fail_open() -> None:
+ code = 'return {"output": {"record_id": None}}\x00'
+
+ assert workflow_update_module._statically_lacks_value_bearing_observation_paths(code, {"output.record_id"}) is False
+ assert workflow_update_module._statically_valueless_return_envelope(code) is False
+
+
+def test_value_bearing_lattice_deep_alias_chain_is_fail_open() -> None:
+ lines = ['v0 = {"output": {"record_id": None}}']
+ lines.extend(f"v{index} = v{index - 1}" for index in range(1, 1500))
+ lines.append("return v1499")
+ code = "\n".join(lines)
+
+ assert workflow_update_module._statically_lacks_value_bearing_observation_paths(code, {"output.record_id"}) is False
+ assert workflow_update_module._statically_valueless_return_envelope(code) is False
+
+
+def test_canonical_output_contract_signature_is_single_keyed_under_degraded_slots() -> None:
+ ctx = _degraded_declaration_stamp_ctx()
+ workflow_yaml = _collapsed_spine_yaml('return {"output": {"blocker": None}}')
+ evaluation = workflow_update_module._evaluate_output_contract_for_code_block(
+ ctx,
+ workflow_yaml,
+ [],
+ allow_static_return_advisory=True,
+ enforce_value_bearing_liveness=True,
+ )
+ assert evaluation is not None
+ contract = workflow_update_module._output_contract_required_paths_source(ctx)
+ signature = workflow_update_module._output_contract_signature(ctx=ctx, required_paths=contract.union)
+
+ assert evaluation.canonical_signature == signature
+ assert evaluation.payload["canonical_output_contract_signature"] == signature
+ assert workflow_update_module._output_contract_pin_key(ctx, workflow_yaml, contract.union) == signature
+ structural = workflow_update_module._output_contract_author_time_structural_payload(ctx, contract.union)
+ assert structural is not None
+ assert structural["canonical_output_contract_signature"] == signature
+ workflow_update_module._record_output_contract_family_reject(
+ ctx, contract.union, reject_family="value_bearing_output_required"
+ )
+ assert ctx.output_contract_reject_count_by_signature == {signature: 1}
+ workflow_update_module._record_output_contract_deferral(ctx, contract.union)
+ assert ctx.output_contract_deferral_count_by_signature == {signature: 1}
+ identity_free = workflow_update_module._stable_output_contract_key(
+ workflow_update_module._output_contract_scope_key(ctx), contract.union
+ )
+ assert signature != identity_free
+
+
+def test_non_value_deficiency_keeps_granted_advisory_arming_at_preflight() -> None:
+ ctx = _declaration_stamp_ctx()
+ workflow_yaml = _collapsed_spine_yaml(
+ 'record_id = await page.locator("#record-id").text_content()\n'
+ 'return {"output": {"record_id": record_id, "blocker": None}}'
+ )
+ evaluation = workflow_update_module._evaluate_output_contract_for_code_block(
+ ctx,
+ workflow_yaml,
+ [],
+ allow_static_return_advisory=True,
+ enforce_value_bearing_liveness=True,
+ )
+ assert evaluation is not None
+ assert evaluation.has_deficiencies is True
+ assert evaluation.can_attempt_run is False
+ assert "value_bearing_output_required" not in evaluation.shape_violations
+ workflow_update_module._grant_output_contract_advisory_run(ctx, evaluation.canonical_signature)
+
+ preflight = workflow_update_module._metadata_contract_run_preflight_reject(ctx, workflow_yaml, [])
+
+ assert preflight is None
+ assert evaluation.canonical_signature in ctx.output_contract_pending_run_evidence
+ assert (
+ workflow_update_module._output_contract_advisory_state(ctx, evaluation.canonical_signature)
+ == OutputContractAdvisoryState.GRANTED
+ )
+
+
+def test_degraded_criteria_reject_steers_to_criteria_not_code_repair() -> None:
+ ctx = _degraded_declaration_stamp_ctx()
+ workflow_yaml = _collapsed_spine_yaml('return {"output": {"blocker": None}}')
+ evaluation = workflow_update_module._evaluate_output_contract_for_code_block(
+ ctx,
+ workflow_yaml,
+ [],
+ allow_static_return_advisory=True,
+ enforce_value_bearing_liveness=True,
+ )
+ assert evaluation is not None
+ assert evaluation.payload["contract_liveness"] == "degraded_empty"
+
+ result = workflow_update_module._output_contract_reject_result(evaluation)
+
+ assert result["ok"] is False
+ assert result["data"]["reason_code"] == "value_bearing_output_required"
+ assert "requested value-bearing output" in result["error"]
+ assert "code_artifact_metadata" not in result["error"]
+ assert "return skeleton" not in result["error"]
+
+
+def test_hollow_code_under_live_contract_keeps_code_repair_steering() -> None:
+ ctx = _declaration_stamp_ctx()
+ workflow_yaml = _collapsed_spine_yaml('return {"output": {"record_id": None, "blocker": None}}')
+ evaluation = workflow_update_module._evaluate_output_contract_for_code_block(
+ ctx,
+ workflow_yaml,
+ [],
+ allow_static_return_advisory=True,
+ enforce_value_bearing_liveness=True,
+ )
+ assert evaluation is not None
+ assert evaluation.payload["reason_code"] == "value_bearing_output_required"
+
+ result = workflow_update_module._output_contract_reject_result(evaluation)
+
+ assert "code_artifact_metadata" in result["error"]
+
+
+def _saved_code_workflow(
+ code: str, label: str = "finalize_service", extra: dict[str, str] | None = None
+) -> SimpleNamespace:
+ blocks = [{"label": label, "block_type": "code", "code": code}]
+ blocks.extend(
+ {"label": extra_label, "block_type": "code", "code": extra_code}
+ for extra_label, extra_code in (extra or {}).items()
+ )
+ labels = {str(block["label"]) for block in blocks}
+ return SimpleNamespace(
+ workflow_id="w",
+ workflow_definition={"parameters": [], "blocks": blocks},
+ get_output_parameter=lambda requested: SimpleNamespace(label=requested) if requested in labels else None,
+ )
+
+
+class TestValueBearingDispatchPreflight:
+ @staticmethod
+ def _guarded_app(monkeypatch: pytest.MonkeyPatch, *, organization: None | EllipsisType = ...) -> AsyncMock:
+ if organization is ...:
+ get_organization = AsyncMock(side_effect=AssertionError("org lookup called"))
+ else:
+ get_organization = AsyncMock(return_value=organization)
+ prepare_workflow = AsyncMock(side_effect=AssertionError("prepare_workflow called"))
+ monkeypatch.setattr(
+ run_execution_module.app,
+ "DATABASE",
+ SimpleNamespace(organizations=SimpleNamespace(get_organization=get_organization)),
+ )
+ monkeypatch.setattr(
+ run_execution_module.app,
+ "WORKFLOW_SERVICE",
+ SimpleNamespace(prepare_workflow=prepare_workflow),
+ )
+ return prepare_workflow
+
+ @pytest.mark.asyncio
+ async def test_dispatch_of_saved_degraded_blocker_only_workflow_is_refused(
+ self, monkeypatch: pytest.MonkeyPatch
+ ) -> None:
+ ctx = _degraded_declaration_stamp_ctx()
+ ctx.staged_workflow = _saved_code_workflow('return {"output": {"blocker": None}}')
+ prepare_workflow = self._guarded_app(monkeypatch)
+
+ result = await run_execution_module._run_blocks_and_collect_debug(
+ {"block_labels": ["finalize_service"], "parameters": {}}, ctx
+ )
+
+ assert result["ok"] is False
+ assert result["data"]["reason_code"] == "value_bearing_output_required"
+ assert result["data"]["contract_liveness"] == "degraded_empty"
+ assert result["data"]["can_attempt_run"] is False
+ assert [row["floor_rekeyed_from_path"] for row in result["data"]["degraded_request_slots"]] == [
+ "output.confirmation_number",
+ "output.account_number",
+ "output.start_date",
+ "output.deposit_amount",
+ "output.next_owner",
+ ]
+ assert result["user_facing_summary"]
+ assert ctx.last_executed_block_labels == []
+ prepare_workflow.assert_not_called()
+
+ @pytest.mark.asyncio
+ async def test_dispatch_of_statically_hollow_code_under_value_contract_is_refused(
+ self, monkeypatch: pytest.MonkeyPatch
+ ) -> None:
+ ctx = _declaration_stamp_ctx()
+ ctx.staged_workflow = _saved_code_workflow('return {"output": {"blocker": None}}')
+ prepare_workflow = self._guarded_app(monkeypatch)
+
+ result = await run_execution_module._run_blocks_and_collect_debug(
+ {"block_labels": ["finalize_service"], "parameters": {}}, ctx
+ )
+
+ assert result["ok"] is False
+ assert result["data"]["reason_code"] == "value_bearing_output_required"
+ assert result["data"]["contract_liveness"] == "value_required"
+ assert result["data"]["canonical_required_child_paths"] == ["output.blocker", "output.record_id"]
+ prepare_workflow.assert_not_called()
+
+ @pytest.mark.asyncio
+ async def test_dispatch_of_mixed_contract_workflow_passes_preflight(self, monkeypatch: pytest.MonkeyPatch) -> None:
+ ctx = _declaration_stamp_ctx()
+ ctx.staged_workflow = _saved_code_workflow(
+ 'record_id = await page.locator("#record-id").text_content()\n'
+ 'return {"output": {"record_id": record_id, "blocker": None}}'
+ )
+ self._guarded_app(monkeypatch, organization=None)
+
+ result = await run_execution_module._run_blocks_and_collect_debug(
+ {"block_labels": ["finalize_service"], "parameters": {}}, ctx
+ )
+
+ assert result == {"ok": False, "error": "Organization not found"}
+
+ @pytest.mark.asyncio
+ async def test_dispatch_without_output_contract_passes_preflight_for_value_bearing_code(
+ self, monkeypatch: pytest.MonkeyPatch
+ ) -> None:
+ ctx = _code_only_ctx()
+ ctx.staged_workflow = _saved_code_workflow(
+ 'record_id = await page.locator("#record-id").text_content()\nreturn {"output": {"record_id": record_id}}'
+ )
+ self._guarded_app(monkeypatch, organization=None)
+
+ result = await run_execution_module._run_blocks_and_collect_debug(
+ {"block_labels": ["finalize_service"], "parameters": {}}, ctx
+ )
+
+ assert result == {"ok": False, "error": "Organization not found"}
+
+ @pytest.mark.asyncio
+ async def test_owner_excluded_subset_run_dispatches_when_saved_workflow_bears_value(
+ self, monkeypatch: pytest.MonkeyPatch
+ ) -> None:
+ ctx = _declaration_stamp_ctx()
+ ctx.staged_workflow = _saved_code_workflow(
+ 'return {"logged_in": True}',
+ label="login",
+ extra={
+ "extract_record": (
+ 'record_id = await page.locator("#record-id").text_content()\n'
+ 'return {"output": {"record_id": record_id, "blocker": None}}'
+ )
+ },
+ )
+ self._guarded_app(monkeypatch, organization=None)
+
+ result = await run_execution_module._run_blocks_and_collect_debug(
+ {"block_labels": ["login"], "parameters": {}}, ctx
+ )
+
+ assert result == {"ok": False, "error": "Organization not found"}
+
+ @pytest.mark.asyncio
+ async def test_solo_empty_setup_block_dispatches_in_value_bearing_workflow(
+ self, monkeypatch: pytest.MonkeyPatch
+ ) -> None:
+ ctx = _code_only_ctx()
+ ctx.request_policy = RequestPolicy(completion_criteria=[])
+ ctx.staged_workflow = _saved_code_workflow(
+ "return {}",
+ label="setup",
+ extra={"extract_record": 'value = await fetch()\nreturn {"output": {"record_id": value}}'},
+ )
+ self._guarded_app(monkeypatch, organization=None)
+
+ result = await run_execution_module._run_blocks_and_collect_debug(
+ {"block_labels": ["setup"], "parameters": {}}, ctx
+ )
+
+ assert result == {"ok": False, "error": "Organization not found"}
+
+ @pytest.mark.asyncio
+ async def test_all_hollow_workflow_is_refused_on_any_selected_subset(self, monkeypatch: pytest.MonkeyPatch) -> None:
+ ctx = _declaration_stamp_ctx()
+ ctx.staged_workflow = _saved_code_workflow(
+ 'return {"output": {"blocker": None}}',
+ label="finalize_service",
+ extra={"extract_record": 'return {"output": {"record_id": None}}'},
+ )
+ prepare_workflow = self._guarded_app(monkeypatch)
+
+ result = await run_execution_module._run_blocks_and_collect_debug(
+ {"block_labels": ["extract_record"], "parameters": {}}, ctx
+ )
+
+ assert result["ok"] is False
+ assert result["data"]["reason_code"] == "value_bearing_output_required"
+ assert ctx.last_executed_block_labels == []
+ prepare_workflow.assert_not_called()
+
+ @pytest.mark.asyncio
+ async def test_dispatch_with_drifted_empty_criteria_source_refuses_hollow_code(
+ self, monkeypatch: pytest.MonkeyPatch
+ ) -> None:
+ ctx = _code_only_ctx()
+ ctx.request_policy = RequestPolicy(completion_criteria=[])
+ ctx.staged_workflow = _saved_code_workflow('return {"output": {"blocker": None}}')
+ prepare_workflow = self._guarded_app(monkeypatch)
+
+ result = await run_execution_module._run_blocks_and_collect_debug(
+ {"block_labels": ["finalize_service"], "parameters": {}}, ctx
+ )
+
+ assert result["ok"] is False
+ assert result["data"]["reason_code"] == "value_bearing_output_required"
+ assert result["data"]["contract_liveness"] == "absent"
+ assert result["data"]["statically_empty_return_envelope"] is True
+ assert result["data"]["can_attempt_run"] is False
+ assert result["user_facing_summary"]
+ prepare_workflow.assert_not_called()
+
+ def test_run_reject_fails_open_for_standard_policy(self) -> None:
+ ctx = _declaration_stamp_ctx()
+ ctx.block_authoring_policy = BlockAuthoringPolicy.STANDARD
+
+ reject = workflow_update_module.output_contract_value_bearing_run_reject(
+ ctx, {"finalize_service": 'return {"output": {"blocker": None}}'}
+ )
+
+ assert reject is None
+
+ def test_run_reject_fails_open_for_runtime_populated_code(self) -> None:
+ ctx = _declaration_stamp_ctx()
+
+ reject = workflow_update_module.output_contract_value_bearing_run_reject(
+ ctx,
+ {"finalize_service": 'record_id = await fetch()\nreturn {"output": {"record_id": record_id}}'},
+ )
+
+ assert reject is None
+
+ def test_run_reject_fails_open_when_any_selected_block_may_bear_value(self) -> None:
+ ctx = _declaration_stamp_ctx()
+
+ reject = workflow_update_module.output_contract_value_bearing_run_reject(
+ ctx,
+ {
+ "finalize_service": 'return {"output": {"blocker": None}}',
+ "extract_record": 'record_id = await fetch()\nreturn {"output": {"record_id": record_id}}',
+ },
+ )
+
+ assert reject is None
+
+ def test_run_reject_refuses_when_every_selected_block_is_statically_hollow(self) -> None:
+ ctx = _declaration_stamp_ctx()
+
+ reject = workflow_update_module.output_contract_value_bearing_run_reject(
+ ctx,
+ {
+ "finalize_service": 'return {"output": {"blocker": None}}',
+ "extract_record": 'return {"output": {"record_id": None, "blocker": None}}',
+ },
+ )
+
+ assert reject is not None
+ assert reject["ok"] is False
+ assert reject["data"]["reason_code"] == "value_bearing_output_required"
+
+ @pytest.mark.parametrize(
+ "code",
+ [
+ 'return {"output": {"blocker": None}}',
+ 'return {"output": {}}',
+ 'return {"output": None, "note": ""}',
+ 'payload = {"output": {"blocker": None}}\nreturn payload',
+ ],
+ )
+ def test_run_reject_refuses_statically_empty_envelope_with_zero_criteria(self, code: str) -> None:
+ ctx = _code_only_ctx()
+ ctx.request_policy = RequestPolicy(completion_criteria=[])
+
+ reject = workflow_update_module.output_contract_value_bearing_run_reject(ctx, {"finalize_service": code})
+
+ assert reject is not None
+ assert reject["data"]["reason_code"] == "value_bearing_output_required"
+ assert reject["data"]["statically_empty_return_envelope"] is True
+
+ @pytest.mark.parametrize(
+ "code",
+ [
+ 'return {"output": {"confirmation_number": "CN-1042"}}',
+ 'value = await fetch()\nreturn {"output": {"confirmation_number": value}}',
+ "return await collect()",
+ 'await page.goto("https://example.com/plans")',
+ ],
+ )
+ def test_run_reject_fails_open_with_zero_criteria_when_code_may_bear_value(self, code: str) -> None:
+ ctx = _code_only_ctx()
+ ctx.request_policy = RequestPolicy(completion_criteria=[])
+
+ reject = workflow_update_module.output_contract_value_bearing_run_reject(ctx, {"finalize_service": code})
+
+ assert reject is None
+
+ def test_run_reject_fails_open_with_zero_criteria_when_any_block_may_bear_value(self) -> None:
+ ctx = _code_only_ctx()
+ ctx.request_policy = RequestPolicy(completion_criteria=[])
+
+ reject = workflow_update_module.output_contract_value_bearing_run_reject(
+ ctx,
+ {
+ "finalize_service": 'return {"output": {"blocker": None}}',
+ "extract_record": 'value = await fetch()\nreturn {"output": {"record_id": value}}',
+ },
+ )
+
+ assert reject is None
+
+ def test_run_reject_degraded_criteria_message_names_criteria_side_cause(self) -> None:
+ ctx = _degraded_declaration_stamp_ctx()
+
+ reject = workflow_update_module.output_contract_value_bearing_run_reject(
+ ctx, {"finalize_service": 'return {"output": {"blocker": None}}'}
+ )
+
+ assert reject is not None
+ assert "requested value-bearing output" in reject["error"]
+ assert "Revise the owning code block" not in reject["error"]
+ assert reject["data"]["reason_code"] == "value_bearing_output_required"
+
+ def test_run_reject_zero_criteria_message_never_references_a_contract(self) -> None:
+ ctx = _code_only_ctx()
+ ctx.request_policy = RequestPolicy(completion_criteria=[])
+
+ reject = workflow_update_module.output_contract_value_bearing_run_reject(
+ ctx, {"finalize_service": 'return {"output": {}}'}
+ )
+
+ assert reject is not None
+ assert "output contract" not in reject["error"]
+ assert "statically returns no values" in reject["error"]
+ assert reject["data"]["reason_code"] == "value_bearing_output_required"
+
+ def test_run_reject_hollow_code_under_live_contract_keeps_code_side_steering(self) -> None:
+ ctx = _declaration_stamp_ctx()
+
+ reject = workflow_update_module.output_contract_value_bearing_run_reject(
+ ctx, {"finalize_service": 'return {"output": {"record_id": None, "blocker": None}}'}
+ )
+
+ assert reject is not None
+ assert "Revise the owning code block" in reject["error"]
+
def _gate_blocks(sibling_code: str) -> list[dict[str, object]]:
return [
diff --git a/tests/unit/test_copilot_output_contract_actuation.py b/tests/unit/test_copilot_output_contract_actuation.py
index ea3e72844..607203f71 100644
--- a/tests/unit/test_copilot_output_contract_actuation.py
+++ b/tests/unit/test_copilot_output_contract_actuation.py
@@ -12,6 +12,7 @@ import json
from types import SimpleNamespace
import pytest
+from structlog.testing import capture_logs
from skyvern.config import settings
from skyvern.forge.sdk.copilot import enforcement
@@ -33,7 +34,16 @@ from skyvern.forge.sdk.copilot.output_contracts import (
classify_output_contract_bail_family,
resolve_output_contract_actuation,
)
-from skyvern.forge.sdk.copilot.request_policy import CompletionCriterion, RequestPolicy
+from skyvern.forge.sdk.copilot.request_policy import (
+ CompletionCriterion,
+ RequestPolicy,
+ _bind_criterion_to_request_slot,
+)
+from skyvern.forge.sdk.copilot.request_slots import (
+ CanonicalRequestSlotV1,
+ RequestSlotPinability,
+ RequestSlotPlane,
+)
from skyvern.forge.sdk.copilot.result_evidence import loaded_result_composition_evidence_from_page
from skyvern.forge.sdk.copilot.runtime import (
output_contract_ladder_unresolved,
@@ -471,6 +481,7 @@ def test_option_a_falls_through_on_two_mapping_locals() -> None:
def _counter_ctx() -> SimpleNamespace:
return SimpleNamespace(
turn_id="turn_example",
+ request_policy=None,
output_contract_reject_count_by_signature={},
output_contract_last_reject_fingerprint_by_signature={},
output_contract_imposed_since_last_reject_by_signature={},
@@ -527,6 +538,7 @@ def _advisory_ctx() -> SimpleNamespace:
return SimpleNamespace(
turn_ownership=None,
blocker_signal_claimant=None,
+ request_policy=None,
gate_precedence_conflict_events=[],
output_contract_reject_count_by_signature={},
output_contract_imposed_since_last_reject_by_signature={},
@@ -913,6 +925,33 @@ def test_advisory_grant_is_not_consumed_without_workflow_run_id() -> None:
assert ctx.output_contract_actuation_by_signature[signature] == OutputContractAdvisoryState.GRANTED
+def test_reject_shaped_run_result_suppresses_missing_run_id_warning() -> None:
+ signature = "sig_reject_shape"
+ ctx = _arm_static_return_advisory_ctx(signature)
+ reject = {"ok": False, "error": "refused", "data": {"reason_code": "value_bearing_output_required"}}
+
+ with capture_logs() as logs:
+ consumed = wu.consume_output_contract_advisory_grant_for_run_result(ctx, reject)
+
+ assert consumed == []
+ assert not [
+ log for log in logs if log["event"] == "copilot_output_contract_advisory_run_result_missing_workflow_run_id"
+ ]
+
+
+def test_non_reject_run_result_missing_run_id_still_warns() -> None:
+ signature = "sig_missing_run_id"
+ ctx = _arm_static_return_advisory_ctx(signature)
+
+ with capture_logs() as logs:
+ consumed = wu.consume_output_contract_advisory_grant_for_run_result(ctx, {"ok": True, "data": {"note": "x"}})
+
+ assert consumed == []
+ assert [
+ log for log in logs if log["event"] == "copilot_output_contract_advisory_run_result_missing_workflow_run_id"
+ ]
+
+
def test_advisory_grant_arms_pending_run_output_evidence() -> None:
signature = "sig_pending_output"
ctx = _arm_static_return_advisory_ctx(signature)
@@ -1926,6 +1965,44 @@ def test_face_c_degraded_mint_preserves_satisfiable_value_bearing_authoring_path
assert "None" not in skeleton
+def test_canonically_bound_shapeless_valid_criterion_stays_dispatchable() -> None:
+ slot_hash = "a" * 48
+ slot = CanonicalRequestSlotV1(
+ slot_id="a" * 64,
+ canonical_path=f"output.request_slot_{slot_hash}_00",
+ path_segments=("output", f"request_slot_{slot_hash}_00"),
+ ordinal=0,
+ source_id="u0",
+ source_start=0,
+ source_end=10,
+ plane=RequestSlotPlane.RUN,
+ pinability=RequestSlotPinability.SHAPELESS_VALID,
+ )
+ bound = _bind_criterion_to_request_slot(
+ CompletionCriterion(
+ id="c1",
+ outcome="The visible action label is returned.",
+ output_path="output.visible_action_label",
+ ),
+ slot=slot,
+ source_quote="visible action label",
+ )
+
+ assert bound.request_slot_id == slot.slot_id
+ assert bound.output_path is None
+ assert bound.floor_rekeyed_from_path == "output.visible_action_label"
+ assert bound.mint_disposition != "degraded"
+ assert bound.mint_degrade is None
+
+ ctx = _antecedent_ctx(bound)
+ contract = wu._output_contract_required_paths_source(ctx)
+
+ assert contract.degraded_request_slots == ()
+ assert contract.liveness is not wu._OutputContractLiveness.DEGRADED_EMPTY
+ code = 'label = await page.inner_text("#action")\nreturn {"output": {"visible_action_label": label}}\n'
+ assert wu.output_contract_value_bearing_run_reject(ctx, {"b1": code}) is None
+
+
def test_classifier_minted_download_path_excluded_from_author_coverage() -> None:
ctx = _antecedent_ctx(
CompletionCriterion(