Skip to content

Automate browser based workflows with AI

ci: restore all-files pre-commit checks on pull requests (#7328)

Aaron Perez9e0221f586bb
diff --git a/skyvern/forge/sdk/copilot/code_block_synthesis.py b/skyvern/forge/sdk/copilot/code_block_synthesis.py
index eb731f629..905bf89b3 100644
--- a/skyvern/forge/sdk/copilot/code_block_synthesis.py
+++ b/skyvern/forge/sdk/copilot/code_block_synthesis.py
@@ -1421,6 +1421,35 @@ def _array_prefix(binding: LiveReadBinding) -> tuple[tuple[str, bool], ...]:
return ()
+def _key_value_scalar_read_statements(binding: LiveReadBinding, variable: str, *, guard_empty: bool) -> list[str]:
+ container = f"page.locator({json.dumps(binding.selector)})"
+ target = f"{container}.nth({binding.selector_index})"
+ children = f'{target}.locator(":scope > *")'
+ statements = [
+ f"if await {container}.count() != {binding.selector_count}:",
+ f'{_INDENT}raise ValueError("Observed scalar selector cardinality changed")',
+ f"if not await {target}.is_visible():",
+ f'{_INDENT}raise ValueError("Observed scalar relation is no longer visible")',
+ f"if await {children}.count() != {binding.child_count}:",
+ f'{_INDENT}raise ValueError("Observed scalar direct-child shape changed")',
+ f"if not await {children}.nth(0).is_visible():",
+ f'{_INDENT}raise ValueError("Observed scalar label is no longer visible")',
+ f"if (await {children}.nth(0).inner_text()).strip() != {json.dumps(binding.relation_label)}:",
+ f'{_INDENT}raise ValueError("Observed scalar label changed")',
+ f"if not await {children}.nth({binding.child_index}).is_visible():",
+ f'{_INDENT}raise ValueError("Observed scalar value is no longer visible")',
+ f"{variable} = (await {children}.nth({binding.child_index}).inner_text()).strip()",
+ ]
+ if guard_empty:
+ statements.extend(
+ [
+ f"if not {variable}:",
+ f'{_INDENT}raise ValueError("Observed scalar value is empty")',
+ ]
+ )
+ return statements
+
+
def synthesize_extraction_suffix(plan: RequestedOutputExtractionPlan) -> SynthesizedExtractionSuffix | None:
if not plan.live_reads:
return None
@@ -1429,26 +1458,7 @@ def synthesize_extraction_suffix(plan: RequestedOutputExtractionPlan) -> Synthes
scalar_bindings = [binding for binding in plan.live_reads if binding.kind == LiveReadKind.KEY_VALUE]
for index, binding in enumerate(scalar_bindings):
variable = f"_extraction_value_{index}"
- container = f"page.locator({json.dumps(binding.selector)})"
- target = f"{container}.nth({binding.selector_index})"
- children = f'{target}.locator(":scope > *")'
- lines.extend(
- [
- f"if await {container}.count() != {binding.selector_count}:",
- f'{_INDENT}raise ValueError("Observed scalar selector cardinality changed")',
- f"if not await {target}.is_visible():",
- f'{_INDENT}raise ValueError("Observed scalar relation is no longer visible")',
- f"if await {children}.count() != {binding.child_count}:",
- f'{_INDENT}raise ValueError("Observed scalar direct-child shape changed")',
- f"if not await {children}.nth(0).is_visible():",
- f'{_INDENT}raise ValueError("Observed scalar label is no longer visible")',
- f"if (await {children}.nth(0).inner_text()).strip() != {json.dumps(binding.relation_label)}:",
- f'{_INDENT}raise ValueError("Observed scalar label changed")',
- f"if not await {children}.nth({binding.child_index}).is_visible():",
- f'{_INDENT}raise ValueError("Observed scalar value is no longer visible")',
- f"{variable} = (await {children}.nth({binding.child_index}).inner_text()).strip()",
- ]
- )
+ lines.extend(_key_value_scalar_read_statements(binding, variable, guard_empty=False))
_set_return_expression(return_root, output_path_segments(binding.output_path), variable)
table_groups: dict[tuple[str, int, tuple[tuple[str, bool], ...]], list[LiveReadBinding]] = {}
@@ -1513,6 +1523,139 @@ def synthesize_extraction_suffix(plan: RequestedOutputExtractionPlan) -> Synthes
return SynthesizedExtractionSuffix(code=code, fingerprint=hashlib.sha256(fingerprint_material.encode()).hexdigest())
+_ENVELOPE_SCALAR_VAR_BASE = "_envelope_value"
+
+
+@dataclass(frozen=True, slots=True)
+class ProducedStaticReturnEnvelope:
+ code: str
+ keyed_paths: tuple[str, ...]
+
+
+def _snippet_scope_returns(statements: Sequence[ast.stmt]) -> list[ast.Return]:
+ found: list[ast.Return] = []
+ for statement in statements:
+ if isinstance(statement, ast.Return):
+ found.append(statement)
+ if isinstance(statement, (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef)):
+ continue
+ for child in ast.iter_child_nodes(statement):
+ if isinstance(child, ast.stmt):
+ found.extend(_snippet_scope_returns([child]))
+ elif isinstance(child, (ast.ExceptHandler, ast.match_case)):
+ found.extend(_snippet_scope_returns(child.body))
+ return found
+
+
+def _bound_or_referenced_identifiers(tree: ast.AST) -> set[str]:
+ names: set[str] = set()
+ for node in ast.walk(tree):
+ if isinstance(node, ast.Name):
+ names.add(node.id)
+ elif isinstance(node, ast.arg):
+ names.add(node.arg)
+ elif isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef)):
+ names.add(node.name)
+ elif isinstance(node, ast.ExceptHandler):
+ if node.name:
+ names.add(node.name)
+ elif isinstance(node, ast.alias):
+ if node.asname:
+ names.add(node.asname)
+ elif isinstance(node, (ast.Global, ast.Nonlocal)):
+ names.update(node.names)
+ elif isinstance(node, ast.MatchAs):
+ if node.name:
+ names.add(node.name)
+ return names
+
+
+def produce_covered_static_return_envelope(
+ code: str,
+ *,
+ plan: RequestedOutputExtractionPlan | None,
+ scalar_required_paths: set[str],
+ declaration_paths: set[str],
+ download_required_paths: set[str],
+ expects_download: bool,
+) -> ProducedStaticReturnEnvelope | None:
+ """Author the unique terminal keyed return covering every scout-bound scalar path, plus
+ None-default declaration children and the resolved download descriptor for the mixed shape.
+
+ Fail-closed: emits only when it owns the sole terminal return (zero returns, or the single
+ generator-owned download-idiom return), when every scalar path is plan-bound, and when no
+ minted local collides with an identifier already in the block."""
+ if download_required_paths and not expects_download:
+ return None
+ if not scalar_required_paths:
+ return None
+ if plan is None:
+ return None
+ stripped_code = textwrap.dedent(code).strip()
+ if not stripped_code:
+ return None
+ try:
+ tree = ast.parse(stripped_code)
+ except SyntaxError:
+ return None
+
+ bindings_by_path = {
+ binding.output_path: binding for binding in plan.live_reads if binding.kind == LiveReadKind.KEY_VALUE
+ }
+ ordered_scalar_paths = sorted(scalar_required_paths)
+ if any(path not in bindings_by_path for path in ordered_scalar_paths):
+ return None
+
+ minted_vars = [f"{_ENVELOPE_SCALAR_VAR_BASE}_{index}" for index in range(len(ordered_scalar_paths))]
+ existing_names = _bound_or_referenced_identifiers(tree)
+ if any(var in existing_names for var in minted_vars):
+ return None
+
+ returns = _snippet_scope_returns(tree.body)
+ download_descriptor_key = ""
+ download_descriptor_expr = ""
+ if download_required_paths:
+ if len(returns) != 1 or returns[0] not in tree.body:
+ return None
+ idiom_return = returns[0]
+ if idiom_return.col_offset != 0:
+ return None
+ if not isinstance(idiom_return.value, ast.Dict) or len(idiom_return.value.keys) != 1:
+ return None
+ descriptor_key_node = idiom_return.value.keys[0]
+ if not isinstance(descriptor_key_node, ast.Constant) or not isinstance(descriptor_key_node.value, str):
+ return None
+ descriptor_value = idiom_return.value.values[0]
+ if not isinstance(descriptor_value, (ast.Name, ast.Attribute)):
+ return None
+ download_descriptor_key = descriptor_key_node.value
+ download_descriptor_expr = ast.unparse(descriptor_value)
+ preserved_lines = stripped_code.splitlines()[: idiom_return.lineno - 1]
+ else:
+ if returns:
+ return None
+ preserved_lines = stripped_code.splitlines()
+
+ return_root = _ExtractionReturnNode()
+ scalar_statements: list[str] = []
+ for path, variable in zip(ordered_scalar_paths, minted_vars):
+ scalar_statements.extend(_key_value_scalar_read_statements(bindings_by_path[path], variable, guard_empty=True))
+ _set_return_expression(return_root, output_path_segments(path), variable)
+ for path in sorted(declaration_paths):
+ _set_return_expression(return_root, output_path_segments(path), "None")
+ if download_descriptor_key:
+ return_root.children.setdefault(
+ download_descriptor_key, _ExtractionReturnNode()
+ ).value_expression = download_descriptor_expr
+
+ body_lines = list(preserved_lines)
+ body_lines.extend(scalar_statements)
+ body_lines.append(f"return {_return_node_expression(return_root)}")
+ produced_code = "\n".join(body_lines).strip() + "\n"
+ keyed_paths = tuple(sorted(scalar_required_paths | declaration_paths))
+ return ProducedStaticReturnEnvelope(code=produced_code, keyed_paths=keyed_paths)
+
+
def _trajectory_contains_reveal(trajectory: Sequence[Mapping[str, Any]], plan: RequestedOutputExtractionPlan) -> bool:
return any(
str(interaction.get("tool_name") or "") == "click"
diff --git a/skyvern/forge/sdk/copilot/completion_criteria_store.py b/skyvern/forge/sdk/copilot/completion_criteria_store.py
index af1738fd6..ad2f5556d 100644
--- a/skyvern/forge/sdk/copilot/completion_criteria_store.py
+++ b/skyvern/forge/sdk/copilot/completion_criteria_store.py
@@ -136,6 +136,7 @@ def criteria_to_json(criteria: tuple[CompletionCriterion, ...] | list[Completion
"contingent_on": criterion.contingent_on,
"contingent_antecedent_output_path": criterion.contingent_antecedent_output_path,
"deliverable_kind": criterion.deliverable_kind,
+ "declared_deliverable_kind": criterion.declared_deliverable_kind,
"implicit": criterion.implicit,
"method_mandated": criterion.method_mandated,
"level": criterion.level,
@@ -216,6 +217,7 @@ def criteria_from_json(raw: Any) -> tuple[CompletionCriterion, ...]:
contingent_on=contingent_on,
contingent_antecedent_output_path=contingent_antecedent_output_path,
deliverable_kind=_normalize_deliverable_kind(item.get("deliverable_kind")),
+ declared_deliverable_kind=_normalize_deliverable_kind(item.get("declared_deliverable_kind")),
implicit=bool(item.get("implicit")),
method_mandated=bool(item.get("method_mandated")),
level=level if isinstance(level, str) and level in _CRITERION_LEVELS else "run", # type: ignore[arg-type]
@@ -243,7 +245,8 @@ def _criterion_reconcile_key(criterion: CompletionCriterion) -> str:
contingent_key = criterion.contingent_on or ""
contingent_path_key = criterion.contingent_antecedent_output_path or ""
deliverable_kind_key = (
- f"{criterion.deliverable_kind or ''}\x1fmint_degrade:{criterion.mint_degrade or ''}"
+ f"{criterion.deliverable_kind or ''}\x1fdeclared:{criterion.declared_deliverable_kind or ''}"
+ f"\x1fmint_degrade:{criterion.mint_degrade or ''}"
f"\x1fjudgment:{judgment_truth_condition_key(criterion.judgment_truth_condition)}"
)
expected_output_value_key = typed_expected_output_value_key(criterion.expected_output_value)
diff --git a/skyvern/forge/sdk/copilot/completion_output_grounding.py b/skyvern/forge/sdk/copilot/completion_output_grounding.py
index 85229a3cb..593f22de2 100644
--- a/skyvern/forge/sdk/copilot/completion_output_grounding.py
+++ b/skyvern/forge/sdk/copilot/completion_output_grounding.py
@@ -4,7 +4,8 @@ import ast
import json
import re
import textwrap
-from collections.abc import Mapping
+from collections.abc import Collection, Mapping
+from dataclasses import replace
from typing import Any, Literal, Protocol
import structlog
@@ -22,6 +23,7 @@ from skyvern.forge.sdk.copilot.request_policy import (
lookup_requested_output_path_alias,
schema_output_path_aliases_from_criteria,
)
+from skyvern.forge.sdk.copilot.task_output_envelope import _TASK_ENVELOPE_BLOCK_TYPES, _TASK_OUTPUT_PAYLOAD_FIELDS
from skyvern.utils.yaml_loader import safe_load_no_dates
LOG = structlog.get_logger()
@@ -289,6 +291,92 @@ def grade_requested_output_criteria(
return verdicts
+def floor_rekeyed_path_backing(
+ copilot_ctx: _GroundingCtx,
+ criteria: list[CompletionCriterion],
+ emission_block_outputs: Mapping[str, Any],
+ emission_block_output_sources: Mapping[str, EvidenceSourceKind],
+ emission_block_types: Mapping[str, str | None],
+ runtime_envelope_labels: Collection[str] = frozenset(),
+) -> dict[str, bool]:
+ """Which floor-rekeyed markers have a meaningful runtime/registered value at their original
+ ``floor_rekeyed_from_path`` in an emission-only evidence view (page and artifact observations
+ excluded by construction), resolved with the requested-output grader's accepted-producer-label,
+ alias, and projection machinery. The rule is identical whether or not the artifact emitted
+ anything: absent emission reads as unbacked."""
+ marked = [criterion for criterion in criteria if criterion.requested_output_floor_rekeyed]
+ if not marked:
+ return {}
+ effective_criteria = [
+ replace(criterion, output_path=criterion.floor_rekeyed_from_path)
+ for criterion in marked
+ if criterion.floor_rekeyed_from_path
+ ]
+ aliases = schema_output_path_aliases_from_criteria(effective_criteria)
+ metadata = _grounding_code_artifact_metadata(copilot_ctx)
+ raw_authored_paths_by_label = _authored_output_contract_paths_by_label(copilot_ctx, metadata=metadata)
+ authored_paths_by_label = _authored_output_contract_paths_by_label(copilot_ctx, aliases, metadata=metadata)
+ independent_labels = {
+ label for label, source in emission_block_output_sources.items() if source in _INDEPENDENT_EVIDENCE_SOURCES
+ }
+ envelope_labels = {
+ label
+ for label in emission_block_outputs
+ if (emission_block_types.get(label) or "").upper() in _TASK_ENVELOPE_BLOCK_TYPES
+ }
+ backing: dict[str, bool] = {}
+ for criterion in marked:
+ path = _normalize_output_path(criterion.floor_rekeyed_from_path)
+ backing[criterion.id] = _floor_rekeyed_path_emission_present(
+ path,
+ emission_block_outputs,
+ authored_paths_by_label,
+ raw_authored_paths_by_label,
+ independent_labels,
+ envelope_labels,
+ runtime_envelope_labels,
+ )
+ return backing
+
+
+def _floor_rekeyed_path_emission_present(
+ path: str,
+ block_outputs: Mapping[str, Any],
+ authored_paths_by_label: dict[str, set[str]],
+ raw_authored_paths_by_label: dict[str, set[str]],
+ independent_labels: set[str],
+ envelope_labels: set[str],
+ runtime_envelope_labels: Collection[str],
+) -> bool:
+ if not path:
+ return False
+ accepted_labels = {label for label, paths in authored_paths_by_label.items() if path in paths}
+ if accepted_labels:
+ candidate_labels = accepted_labels
+ accepted_authored_paths = set().union(*(authored_paths_by_label[label] for label in accepted_labels))
+ accepted_raw_authored_paths = set().union(
+ *(raw_authored_paths_by_label.get(label, set()) for label in accepted_labels)
+ )
+ projection_roots = _runtime_projection_roots(accepted_authored_paths | accepted_raw_authored_paths)
+ if accepted_labels & envelope_labels:
+ projection_roots |= set(_TASK_OUTPUT_PAYLOAD_FIELDS)
+ elif envelope_labels:
+ if len(runtime_envelope_labels) > 1:
+ # No authored contract names the producer, so with more than one task-envelope block the
+ # emission is unattributable and a coincidental match in an unrelated block must not back
+ # it. A single envelope block is the unambiguous producer.
+ return False
+ candidate_labels = envelope_labels
+ projection_roots = set(_TASK_OUTPUT_PAYLOAD_FIELDS)
+ else:
+ return False
+ accepted_runtime_output_items = _accepted_runtime_outputs(block_outputs, candidate_labels, independent_labels)
+ if not accepted_runtime_output_items:
+ return False
+ match_state, _ref = _runtime_output_path_presence(dict(accepted_runtime_output_items), path, projection_roots)
+ return match_state == "present"
+
+
def _carrier_confirmation(
criterion: CompletionCriterion,
path: str,
diff --git a/skyvern/forge/sdk/copilot/completion_verification.py b/skyvern/forge/sdk/copilot/completion_verification.py
index 2bddf38b6..9036b0a4a 100644
--- a/skyvern/forge/sdk/copilot/completion_verification.py
+++ b/skyvern/forge/sdk/copilot/completion_verification.py
@@ -116,6 +116,7 @@ class CompletionVerificationResult:
degraded_criterion_ids: list[str] = field(default_factory=list)
floor_rekeyed_criterion_ids: list[str] = field(default_factory=list)
floor_rekeyed_output_path_by_criterion_id: dict[str, str] = field(default_factory=dict)
+ floor_rekeyed_backed_by_criterion_id: dict[str, bool] = field(default_factory=dict)
contingent_degraded_criterion_ids: list[str] = field(default_factory=list)
requested_output_criteria_count: int = 0
@@ -2218,6 +2219,7 @@ def combine_verification_results(
degraded_ids: list[str] = []
floor_rekeyed_ids: list[str] = []
floor_rekeyed_path_by_id: dict[str, str] = {}
+ floor_rekeyed_backed_by_id: dict[str, bool] = {}
contingent_degraded_ids: list[str] = []
if run_result is not None:
contingent_ids = list(dict.fromkeys([*contingent_ids, *run_result.contingent_criterion_ids]))
@@ -2229,6 +2231,7 @@ def combine_verification_results(
degraded_ids = list(dict.fromkeys([*degraded_ids, *run_result.degraded_criterion_ids]))
floor_rekeyed_ids = list(dict.fromkeys([*floor_rekeyed_ids, *run_result.floor_rekeyed_criterion_ids]))
floor_rekeyed_path_by_id.update(run_result.floor_rekeyed_output_path_by_criterion_id)
+ floor_rekeyed_backed_by_id.update(run_result.floor_rekeyed_backed_by_criterion_id)
contingent_degraded_ids = list(
dict.fromkeys([*contingent_degraded_ids, *run_result.contingent_degraded_criterion_ids])
)
@@ -2244,6 +2247,7 @@ def combine_verification_results(
degraded_criterion_ids=degraded_ids,
floor_rekeyed_criterion_ids=floor_rekeyed_ids,
floor_rekeyed_output_path_by_criterion_id=floor_rekeyed_path_by_id,
+ floor_rekeyed_backed_by_criterion_id=floor_rekeyed_backed_by_id,
contingent_degraded_criterion_ids=contingent_degraded_ids,
requested_output_criteria_count=requested_output_criteria_count,
)
@@ -2480,6 +2484,86 @@ def carry_floor_rekeyed_criterion_ids(
)
+def carry_floor_rekeyed_path_backing(
+ result: CompletionVerificationResult,
+ backing_by_criterion_id: Mapping[str, bool],
+) -> CompletionVerificationResult:
+ if not backing_by_criterion_id and not result.floor_rekeyed_backed_by_criterion_id:
+ return result
+ merged = dict(result.floor_rekeyed_backed_by_criterion_id)
+ merged.update(backing_by_criterion_id)
+ return replace(result, floor_rekeyed_backed_by_criterion_id=merged)
+
+
+@dataclass(frozen=True)
+class FloorRekeyedEmissionWithhold:
+ criterion_ids: tuple[str, ...]
+ unbacked_output_paths: tuple[str, ...]
+ backed_output_paths: tuple[str, ...]
+
+
+def floor_rekeyed_effective_marked_ids(result: CompletionVerificationResult) -> list[str]:
+ """Floor-rekeyed markers that required an emission this run: the marked set minus the
+ structurally-unfired contingent (abstained) markers, which needed no emission to fire."""
+ abstained = result.abstained_criterion_ids()
+ return [criterion_id for criterion_id in result.floor_rekeyed_criterion_ids if criterion_id not in abstained]
+
+
+def floor_rekeyed_emission_withhold(
+ result: CompletionVerificationResult | None,
+) -> FloorRekeyedEmissionWithhold | None:
+ """A fully satisfied run whose floor-rekeyed emission markers lack a meaningful runtime value at
+ their original path delivered no emission that corroboration can substitute for. Keyed on the typed
+ marker id-set (never payload emptiness) so never-minted reach-state and structurally-unfired
+ contingent markers are untouched, and a missing backing entry reads as unbacked to fail closed."""
+ if result is None or result.status != "evaluated" or not result.is_fully_satisfied():
+ return None
+ effective_ids = floor_rekeyed_effective_marked_ids(result)
+ if not effective_ids:
+ return None
+ unbacked_ids: list[str] = []
+ unbacked_paths: list[str] = []
+ backed_paths: list[str] = []
+ for criterion_id in effective_ids:
+ path = result.floor_rekeyed_output_path_by_criterion_id.get(criterion_id, "")
+ if result.floor_rekeyed_backed_by_criterion_id.get(criterion_id, False):
+ backed_paths.append(path)
+ else:
+ unbacked_ids.append(criterion_id)
+ unbacked_paths.append(path)
+ if not unbacked_ids:
+ return None
+ return FloorRekeyedEmissionWithhold(
+ criterion_ids=tuple(unbacked_ids),
+ unbacked_output_paths=tuple(unbacked_paths),
+ backed_output_paths=tuple(backed_paths),
+ )
+
+
+def floor_rekeyed_emission_lane_fields(result: CompletionVerificationResult | None) -> dict[str, Any] | None:
+ """Structured fingerprint fields for the floor-rekeyed emission lane, emitted whenever an
+ evaluated result carries markers — on satisfied and unsatisfied results alike, independent of
+ outcome."""
+ if result is None or result.status != "evaluated" or not result.floor_rekeyed_criterion_ids:
+ return None
+ effective_ids = floor_rekeyed_effective_marked_ids(result)
+ backed_ids = [cid for cid in effective_ids if result.floor_rekeyed_backed_by_criterion_id.get(cid, False)]
+ unbacked_ids = [cid for cid in effective_ids if cid not in backed_ids]
+ withhold = floor_rekeyed_emission_withhold(result)
+ return {
+ "criterion_ids": list(result.floor_rekeyed_criterion_ids),
+ "effective_criterion_ids": list(effective_ids),
+ "abstained_excluded_criterion_ids": sorted(set(result.floor_rekeyed_criterion_ids) - set(effective_ids)),
+ "backed_criterion_ids": backed_ids,
+ "unbacked_criterion_ids": unbacked_ids,
+ "backed_output_paths": [result.floor_rekeyed_output_path_by_criterion_id.get(cid, "") for cid in backed_ids],
+ "unbacked_output_paths": [
+ result.floor_rekeyed_output_path_by_criterion_id.get(cid, "") for cid in unbacked_ids
+ ],
+ "engaged": withhold is not None,
+ }
+
+
async def evaluate_completion_criteria(
criteria: list[CompletionCriterion],
snapshot: RunEvidenceSnapshot,
diff --git a/skyvern/forge/sdk/copilot/enforcement.py b/skyvern/forge/sdk/copilot/enforcement.py
index 69b8ee796..6604096c9 100644
--- a/skyvern/forge/sdk/copilot/enforcement.py
+++ b/skyvern/forge/sdk/copilot/enforcement.py
@@ -2425,6 +2425,30 @@ def _requested_output_coverage_tokens(ctx: AgentContext) -> dict[str, frozenset[
}
+def _registered_download_deliverable_paths(ctx: AgentContext) -> set[str]:
+ return {
+ criterion.output_path
+ for criterion in _pre_run_gated_completion_criteria(ctx)
+ if criterion.declared_deliverable_kind == "registered_download" and criterion.output_path
+ }
+
+
+def download_satisfied_requested_output_paths(ctx: AgentContext) -> set[str]:
+ """Requested-output paths a reached download registration satisfies at runtime rather than a
+ page-scalar read: the registered-download alias paths plus the paths the classifier declared as
+ ``registered_download`` deliverables. Empty unless a download target with a captured selector
+ was reached. Author-time seam classification only — it never credits scout coverage."""
+ download = ctx.reached_download_target
+ if download is None or not download.selector:
+ return set()
+ requested = _requested_output_paths_for_ctx(ctx)
+ # The scout reads page scalars; it can never read a file that exists only once a download fires.
+ # So a declared download kind on a path the scout DID cover is a classifier false positive, and the
+ # path stays a live-read scalar. The canonical alias paths are download-registered by definition.
+ declared = _registered_download_deliverable_paths(ctx) - set(ctx.scouted_output_covered_paths)
+ return requested & (REGISTERED_DOWNLOAD_REQUESTED_OUTPUT_PATHS | declared)
+
+
def uncovered_requested_output_paths(ctx: AgentContext) -> set[str]:
"""Requested-output paths not yet credited by scouted evidence. A path whose identifying
tokens are all generic (e.g. ``output.data``) is uncoverable by token match and is exempted,
@@ -2433,10 +2457,7 @@ def uncovered_requested_output_paths(ctx: AgentContext) -> set[str]:
if not requested:
return set()
tokens_by_path = _requested_output_coverage_tokens(ctx)
- covered: set[str] = set(ctx.scouted_output_covered_paths)
- download = ctx.reached_download_target
- if download is not None and download.selector:
- covered |= REGISTERED_DOWNLOAD_REQUESTED_OUTPUT_PATHS
+ covered: set[str] = set(ctx.scouted_output_covered_paths) | download_satisfied_requested_output_paths(ctx)
return {path for path in requested if path not in covered and tokens_by_path.get(path)}
@@ -2457,6 +2478,26 @@ def requested_output_extraction_plan(ctx: AgentContext) -> RequestedOutputExtrac
)
+def requested_scalar_output_extraction_plan(ctx: AgentContext) -> RequestedOutputExtractionPlan | None:
+ """Extraction plan over the page-scalar subset of requested outputs (requested minus the
+ download-registered paths), for the mixed download+scalar shape whose download half is
+ satisfied by execution registration rather than a static keyed read."""
+ requested_paths = _requested_output_paths_for_ctx(ctx) - download_satisfied_requested_output_paths(ctx)
+ if not requested_paths:
+ return None
+ labels_by_path: dict[str, tuple[str, ...]] = {}
+ for criterion in _pre_run_gated_completion_criteria(ctx):
+ if criterion.output_path in requested_paths and criterion.outcome.strip():
+ labels_by_path.setdefault(criterion.output_path, ())
+ labels_by_path[criterion.output_path] += (criterion.outcome.strip(),)
+ if set(labels_by_path) != requested_paths:
+ return None
+ return derive_requested_output_extraction_plan(
+ flow_evidence=ctx.flow_evidence,
+ labels_by_path=labels_by_path,
+ )
+
+
def requested_output_extraction_plan_changed(ctx: AgentContext, current: RequestedOutputExtractionPlan | None) -> bool:
if current is None or len(ctx.flow_evidence) < 2:
return False
@@ -2598,10 +2639,11 @@ def synthesized_trajectory_is_goal_complete(ctx: AgentContext) -> bool:
return False
if _request_expects_unreached_download(ctx):
return False
- requested_paths = _requested_output_paths_for_ctx(ctx)
- plan = requested_output_extraction_plan(ctx)
- if requested_paths and (plan is None or not requested_paths.issubset(set(plan.requested_output_paths))):
- return False
+ scalar_paths = _requested_output_paths_for_ctx(ctx) - download_satisfied_requested_output_paths(ctx)
+ if scalar_paths:
+ plan = requested_scalar_output_extraction_plan(ctx)
+ if plan is None or not scalar_paths.issubset(set(plan.requested_output_paths)):
+ return False
if _credential_flow_scout_gap_incomplete(ctx, ctx.scout_trajectory):
return False
return synthesized_trajectory_reaches_goal(ctx)
diff --git a/skyvern/forge/sdk/copilot/request_policy.py b/skyvern/forge/sdk/copilot/request_policy.py
index 3f7b8acad..32c2c280d 100644
--- a/skyvern/forge/sdk/copilot/request_policy.py
+++ b/skyvern/forge/sdk/copilot/request_policy.py
@@ -281,6 +281,9 @@ class CompletionCriterion:
contingent_on: str | None = None
contingent_antecedent_output_path: str | None = None
deliverable_kind: Literal["registered_download"] | None = None
+ # Author-time seam signal only: unlike ``deliverable_kind`` it survives canonicalization onto
+ # non-canonical output paths, so it is never rendered to the completion verifier.
+ declared_deliverable_kind: Literal["registered_download"] | None = None
implicit: bool = False
method_mandated: bool = False
# "definition": a property of the workflow definition itself, graded against the
@@ -925,6 +928,7 @@ def _parse_completion_criteria(raw: Any) -> list[CompletionCriterion]:
contingent_on=contingent_on,
contingent_antecedent_output_path=contingent_antecedent_output_path,
deliverable_kind=deliverable_kind,
+ declared_deliverable_kind=deliverable_kind,
implicit=bool(item.get("implicit")),
method_mandated=bool(item.get("method_mandated")),
level=cast(CriterionLevel, level_raw)
@@ -1560,6 +1564,7 @@ def _apply_requested_output_completion_criteria(
)
metadata_by_output_path: dict[str, tuple[str | None, str | None, Literal["registered_download"] | None]] = {}
+ declared_kind_by_output_path: dict[str, Literal["registered_download"]] = {}
for criterion in policy.completion_criteria:
if criterion.level == "definition" or criterion.method_mandated:
continue
@@ -1567,6 +1572,7 @@ def _apply_requested_output_completion_criteria(
not criterion.contingent_on
and not criterion.contingent_antecedent_output_path
and not criterion.deliverable_kind
+ and not criterion.declared_deliverable_kind
):
continue
for field_name, output_path, _field_label in requested_specs:
@@ -1574,6 +1580,8 @@ def _apply_requested_output_completion_criteria(
deliverable_kind = (
criterion.deliverable_kind if output_path in REGISTERED_DOWNLOAD_REQUESTED_OUTPUT_PATHS else None
)
+ if criterion.output_path == output_path and criterion.declared_deliverable_kind:
+ declared_kind_by_output_path.setdefault(output_path, criterion.declared_deliverable_kind)
metadata_by_output_path.setdefault(
output_path,
(
@@ -1606,6 +1614,7 @@ def _apply_requested_output_completion_criteria(
contingent_on=metadata_by_output_path.get(output_path, (None, None, None))[0],
contingent_antecedent_output_path=metadata_by_output_path.get(output_path, (None, None, None))[1],
deliverable_kind=metadata_by_output_path.get(output_path, (None, None, None))[2],
+ declared_deliverable_kind=declared_kind_by_output_path.get(output_path),
judgment_truth_condition=judgment_condition_by_output_path.get(output_path),
)
for _field_name, output_path, field_label in requested_specs
@@ -1658,6 +1667,7 @@ def _apply_validation_classification_completion_criteria(policy: RequestPolicy)
expected_output_shape=None,
requested_output_evidence_source="runtime_output",
deliverable_kind=None,
+ declared_deliverable_kind=None,
terminal_action_family=None,
classification_output_key=output_key,
expected_classification=expected,
diff --git a/skyvern/forge/sdk/copilot/task_output_envelope.py b/skyvern/forge/sdk/copilot/task_output_envelope.py
new file mode 100644
index 000000000..0c25c5d27
--- /dev/null
+++ b/skyvern/forge/sdk/copilot/task_output_envelope.py
@@ -0,0 +1,35 @@
+from __future__ import annotations
+
+from skyvern.forge.sdk.workflow.models.block import BaseTaskBlock, Block, TaskV2Block
+from skyvern.schemas.workflows import BlockType
+
+
+def _task_output_envelope_block_types() -> frozenset[str]:
+ # Almost every envelope block extends BaseTaskBlock; TaskV2Block subclasses Block
+ # directly but emits the same TaskOutput envelope, so seed the walk with both roots.
+ block_types: set[str] = set()
+ pending: list[type[Block]] = [BaseTaskBlock, TaskV2Block]
+ while pending:
+ cls = pending.pop()
+ pending.extend(cls.__subclasses__())
+ field = cls.model_fields.get("block_type")
+ default = field.default if field is not None else None
+ if isinstance(default, BlockType):
+ block_types.add(default.value.upper())
+ return frozenset(block_types)
+
+
+# Block types whose ``block.output`` is a ``TaskOutput.from_task()`` envelope rather than the raw
+# payload, so meaningful-data and floor-rekeyed backing checks slice/project only
+# ``_TASK_OUTPUT_PAYLOAD_FIELDS`` and always-populated metadata (task_id, status) reads no signal.
+# Derived from the block subclass tree so a newly added task-backed block can't fall out of sync.
+_TASK_ENVELOPE_BLOCK_TYPES: frozenset[str] = _task_output_envelope_block_types()
+
+
+# Payload fields inside a ``TaskOutput.from_task()`` envelope carrying real produced content; the
+# rest is always-populated metadata.
+_TASK_OUTPUT_PAYLOAD_FIELDS: tuple[str, ...] = (
+ "extracted_information",
+ "downloaded_files",
+ "downloaded_file_urls",
+)
diff --git a/skyvern/forge/sdk/copilot/tools/_shared.py b/skyvern/forge/sdk/copilot/tools/_shared.py
index 2b16ce82c..378c04a9f 100644
--- a/skyvern/forge/sdk/copilot/tools/_shared.py
+++ b/skyvern/forge/sdk/copilot/tools/_shared.py
@@ -28,11 +28,14 @@ from skyvern.forge.sdk.copilot.composition_evidence import has_bounded_page_sche
from skyvern.forge.sdk.copilot.context import CopilotContext
from skyvern.forge.sdk.copilot.enforcement import TOTAL_TIMEOUT_SECONDS
from skyvern.forge.sdk.copilot.runtime import AgentContext
+from skyvern.forge.sdk.copilot.task_output_envelope import (
+ _TASK_ENVELOPE_BLOCK_TYPES,
+ _TASK_OUTPUT_PAYLOAD_FIELDS,
+)
from skyvern.forge.sdk.copilot.tracing_setup import copilot_span
from skyvern.forge.sdk.copilot.turn_halt import stash_turn_halt_from_blocker_signal
from skyvern.forge.sdk.copilot.turn_ownership import emit_blocker_signal_payload
from skyvern.forge.sdk.copilot.verification_evidence import WorkflowVerificationEvidence
-from skyvern.forge.sdk.workflow.models.block import BaseTaskBlock, Block, TaskV2Block
from skyvern.forge.sdk.workflow.models.workflow import WorkflowRunStatus
from skyvern.schemas.workflows import BlockType
from skyvern.utils.yaml_loader import safe_load_no_dates
@@ -59,32 +62,6 @@ _DATA_PRODUCING_BLOCK_TYPES = frozenset({"EXTRACTION", "TEXT_PROMPT"})
_OUTCOME_EVIDENCE_BLOCK_TYPES = frozenset({BlockType.EXTRACTION.value, BlockType.VALIDATION.value})
-def _task_output_envelope_block_types() -> frozenset[str]:
- # Almost every envelope block extends BaseTaskBlock; TaskV2Block subclasses Block
- # directly but emits the same TaskOutput envelope, so seed the walk with both roots.
- block_types: set[str] = set()
- pending: list[type[Block]] = [BaseTaskBlock, TaskV2Block]
- while pending:
- cls = pending.pop()
- pending.extend(cls.__subclasses__())
- field = cls.model_fields.get("block_type")
- default = field.default if field is not None else None
- if isinstance(default, BlockType):
- block_types.add(default.value.upper())
- return frozenset(block_types)
-
-
-# Block types whose ``block.output`` is a ``TaskOutput.from_task()`` envelope
-# (schemas/tasks.py:TaskOutput) rather than the raw payload; the meaningful-data
-# check slices these to ``_TASK_OUTPUT_PAYLOAD_FIELDS`` so always-populated
-# envelope metadata (task_id, status, artifact IDs) can't read a reach-state task
-# (login, navigation) that produced no content as meaningful. Envelope shape is
-# orthogonal to data production, so a raw-value block (CODE, TEXT_PROMPT) whose
-# value happens to carry a ``task_id`` is excluded here and passes through unsliced.
-# Derived from the envelope block roots so a newly added task-backed block can't fall out of sync.
-_TASK_ENVELOPE_BLOCK_TYPES: frozenset[str] = _task_output_envelope_block_types()
-
-
# Absolute upper bound on a single ``run_blocks`` tool invocation. Exists only
# as a last-resort trip wire for runaway loops — progressing runs should never
# approach this. The OpenAI Agents SDK wraps the tool in
@@ -185,16 +162,6 @@ def _is_meaningful_extracted_data(extracted: Any) -> bool:
return True
-# Payload fields inside a ``TaskOutput.from_task()`` envelope
-# (schemas/tasks.py:TaskOutput). Only these carry "did the block produce
-# something useful?" signal; the rest (task_id, status, artifact IDs, etc.)
-# are always populated on a completed run and would short-circuit
-# _is_meaningful_extracted_data to True even when nothing useful was produced.
-_TASK_OUTPUT_PAYLOAD_FIELDS: tuple[str, ...] = (
- "extracted_information",
- "downloaded_files",
- "downloaded_file_urls",
-)
_TASK_OUTPUT_PARAMETER_SUFFIX = "_output"
diff --git a/skyvern/forge/sdk/copilot/tools/completion.py b/skyvern/forge/sdk/copilot/tools/completion.py
index 88ce7b3b4..08f834b97 100644
--- a/skyvern/forge/sdk/copilot/tools/completion.py
+++ b/skyvern/forge/sdk/copilot/tools/completion.py
@@ -8,6 +8,8 @@ from skyvern.config import settings
from skyvern.forge.sdk.copilot.challenge_evidence import carrier_backed_anti_bot_categories
from skyvern.forge.sdk.copilot.completion_criteria_store import note_adjudication_on_turn_state
from skyvern.forge.sdk.copilot.completion_output_grounding import (
+ _GroundingCtx,
+ floor_rekeyed_path_backing,
grade_requested_output_criteria,
split_requested_output_criteria,
)
@@ -22,6 +24,7 @@ from skyvern.forge.sdk.copilot.completion_verification import (
_is_structural_requested_output_abstention,
carry_degraded_criterion_ids,
carry_floor_rekeyed_criterion_ids,
+ carry_floor_rekeyed_path_backing,
combine_verification_results,
evaluate_completion_criteria,
grade_definition_criteria,
@@ -63,6 +66,7 @@ from skyvern.forge.sdk.copilot.terminal_predicates import outcome_fully_verified
from skyvern.forge.sdk.copilot.tracing_setup import copilot_span
from ._shared import (
+ _TASK_ENVELOPE_BLOCK_TYPES,
RUN_BLOCKS_SAFETY_CEILING_SECONDS,
_copilot_seconds_remaining,
_current_workflow_block_labels,
@@ -70,6 +74,7 @@ from ._shared import (
_failed_run_block_labels,
_is_meaningful_extracted_data,
_registered_output_parameter_payloads,
+ _registered_output_payload_view,
_valid_runtime_anchor_url,
_workflow_output_parameter_payloads,
)
@@ -849,6 +854,99 @@ def _pre_run_page_reference_text(reference: PreRunPageReference | None, run_id:
return reference.text or None
+def _floor_rekeyed_emission_evidence(
+ copilot_ctx: _GroundingCtx, run_data: Mapping[str, Any] | None
+) -> tuple[dict[str, Any], dict[str, EvidenceSourceKind], dict[str, str | None], set[str]]:
+ """Emission-only evidence view for floor-rekeyed backing: this run's block runtime outputs and
+ registered output parameters, each envelope-sliced with its producer block's type so metadata-only
+ envelopes carry no signal. Page and registered-artifact observations never enter by construction.
+ The fourth element is the labels of task-envelope blocks that ran this run — a producer that
+ emitted nothing drops from the evidence view, so this is the only signal of how many candidate
+ producers exist when no authored contract names one."""
+ block_outputs: dict[str, Any] = {}
+ block_output_sources: dict[str, EvidenceSourceKind] = {}
+ block_types: dict[str, str | None] = {}
+ runtime_envelope_labels: set[str] = set()
+ if not isinstance(run_data, Mapping):
+ return block_outputs, block_output_sources, block_types, runtime_envelope_labels
+ current_labels = set(_current_workflow_block_labels(copilot_ctx))
+ blocks = run_data.get("blocks")
+ if isinstance(blocks, list):
+ for block in blocks:
+ if not isinstance(block, dict):
+ continue
+ label = block.get("label")
+ block_type = block.get("block_type")
+ extracted = block.get("extracted_data")
+ if isinstance(label, str) and label in current_labels:
+ if (block_type or "").upper() in _TASK_ENVELOPE_BLOCK_TYPES:
+ runtime_envelope_labels.add(label)
+ sliced = _registered_output_payload_view(extracted, block_type)
+ if _is_meaningful_extracted_data(sliced):
+ block_outputs[label] = sliced
+ block_output_sources[label] = "runtime_output"
+ block_types[label] = block_type
+ for output_key, output_value in _workflow_output_parameter_payloads(extracted).items():
+ block_outputs[output_key] = _registered_output_payload_view(output_value, block_type)
+ block_output_sources[output_key] = "registered_output_parameter"
+ block_types[output_key] = block_type
+ for output_key, output_value in _workflow_output_parameter_payloads(run_data.get("output")).items():
+ if not _is_meaningful_extracted_data(_registered_output_payload_view(output_value, None)):
+ continue
+ block_outputs.setdefault(output_key, output_value)
+ block_output_sources.setdefault(output_key, "registered_output_parameter")
+ for registered in _registered_output_parameter_payloads(run_data):
+ registered_output_key = registered.get("output_parameter_key")
+ registered_block_type = registered.get("block_type")
+ registered_output_value = _registered_output_payload_view(registered.get("value"), registered_block_type)
+ registered_block_label = registered.get("block_label")
+ if isinstance(registered_output_key, str) and registered_output_key:
+ block_outputs[registered_output_key] = registered_output_value
+ block_output_sources[registered_output_key] = "registered_output_parameter"
+ block_types[registered_output_key] = registered_block_type
+ if isinstance(registered_block_label, str) and registered_block_label in current_labels:
+ if isinstance(registered_output_key, str) and registered_output_key:
+ existing = block_outputs.get(registered_block_label)
+ if isinstance(existing, dict):
+ merged = dict(existing)
+ merged.setdefault(registered_output_key, registered_output_value)
+ block_outputs[registered_block_label] = merged
+ else:
+ block_outputs[registered_block_label] = {registered_output_key: registered_output_value}
+ block_output_sources.setdefault(registered_block_label, "registered_output_parameter")
+ block_types.setdefault(registered_block_label, registered_block_type)
+ else:
+ block_outputs[registered_block_label] = registered_output_value
+ block_output_sources[registered_block_label] = "registered_output_parameter"
+ block_types[registered_block_label] = registered_block_type
+ return block_outputs, block_output_sources, block_types, runtime_envelope_labels
+
+
+def _carry_floor_rekeyed_backing(
+ copilot_ctx: _GroundingCtx,
+ verification: CompletionVerificationResult,
+ criteria: list[CompletionCriterion],
+ run_data: Mapping[str, Any] | None,
+) -> CompletionVerificationResult:
+ if not verification.floor_rekeyed_criterion_ids:
+ return verification
+ (
+ emission_block_outputs,
+ emission_block_output_sources,
+ emission_block_types,
+ runtime_envelope_labels,
+ ) = _floor_rekeyed_emission_evidence(copilot_ctx, run_data)
+ backing = floor_rekeyed_path_backing(
+ copilot_ctx,
+ criteria,
+ emission_block_outputs,
+ emission_block_output_sources,
+ emission_block_types,
+ runtime_envelope_labels=runtime_envelope_labels,
+ )
+ return carry_floor_rekeyed_path_backing(verification, backing)
+
+
def _build_run_evidence_snapshot(copilot_ctx: Any, result: dict[str, Any]) -> RunEvidenceSnapshot:
data = result.get("data")
data = data if isinstance(data, dict) else {}
@@ -871,13 +969,23 @@ def _build_run_evidence_snapshot(copilot_ctx: Any, result: dict[str, Any]) -> Ru
block_outputs[label] = evidence_output
block_output_sources[label] = "runtime_output"
for output_key, output_value in _workflow_output_parameter_payloads(output).items():
+ if not _is_meaningful_extracted_data(
+ _registered_output_payload_view(output_value, block.get("block_type"))
+ ):
+ continue
block_outputs[output_key] = output_value
block_output_sources[output_key] = "registered_output_parameter"
for output_key, output_value in _workflow_output_parameter_payloads(data.get("output")).items():
+ if not _is_meaningful_extracted_data(_registered_output_payload_view(output_value, None)):
+ continue
block_outputs[output_key] = output_value
block_output_sources[output_key] = "registered_output_parameter"
for registered in _registered_output_parameter_payloads(data):
registered_output_key = registered.get("output_parameter_key")
+ if not _is_meaningful_extracted_data(
+ _registered_output_payload_view(registered.get("value"), registered.get("block_type"))
+ ):
+ continue
registered_output_value = _completion_evidence_payload(registered.get("value"))
registered_block_label = registered.get("block_label")
if isinstance(registered_output_key, str) and registered_output_key:
@@ -1208,7 +1316,11 @@ async def _maybe_run_completion_verification(
if verification is None:
return None
verification = carry_degraded_criterion_ids(verification, criteria)
- return carry_floor_rekeyed_criterion_ids(verification, criteria)
+ verification = carry_floor_rekeyed_criterion_ids(verification, criteria)
+ run_data = result.get("data")
+ return _carry_floor_rekeyed_backing(
+ copilot_ctx, verification, criteria, run_data if isinstance(run_data, dict) else None
+ )
async def _completion_verification_from_run_result(
diff --git a/skyvern/forge/sdk/copilot/tools/run_execution.py b/skyvern/forge/sdk/copilot/tools/run_execution.py
index 3a05a913b..50d3f6e82 100644
--- a/skyvern/forge/sdk/copilot/tools/run_execution.py
+++ b/skyvern/forge/sdk/copilot/tools/run_execution.py
@@ -59,6 +59,8 @@ from skyvern.forge.sdk.copilot.completion_verification import (
DeliveredUnverifiedTerminalState,
degraded_contract_delivered_unverified_terminal_state,
floor_rekeyed_deliverable_credit,
+ floor_rekeyed_emission_lane_fields,
+ floor_rekeyed_emission_withhold,
only_structural_requested_output_abstentions,
zero_requested_output_criteria_credit,
)
@@ -2669,6 +2671,15 @@ def _delivered_unverified_registered_outputs(data: Mapping[str, Any]) -> dict[st
return observed
+def _emit_floor_rekeyed_emission_lane(
+ completion_verification: CompletionVerificationResult | None, workflow_run_id: str | None
+) -> None:
+ fields = floor_rekeyed_emission_lane_fields(completion_verification)
+ if fields is None:
+ return
+ LOG.info("copilot.completion.floor_rekeyed_emission_lane", workflow_run_id=workflow_run_id, **fields)
+
+
def _record_run_blocks_result(
copilot_ctx: Any, result: dict[str, Any], completion_verification: CompletionVerificationResult | None = None
) -> RecordedRunOutcome | None:
@@ -2700,6 +2711,7 @@ def _record_run_blocks_result(
_record_adjudication_on_turn_state(copilot_ctx, completion_verification)
if completion_verification is not None and completion_verification.status == "evaluated":
_emit_completion_verification_trace(copilot_ctx, completion_verification)
+ _emit_floor_rekeyed_emission_lane(completion_verification, run_id if isinstance(run_id, str) else None)
copilot_ctx.last_run_blocks_workflow_run_id = run_id if isinstance(run_id, str) else None
copilot_ctx.last_successful_run_blocks_workflow_run_id = run_id if run_ok and isinstance(run_id, str) else None
copilot_ctx.delivered_unverified_terminal = False
@@ -3175,41 +3187,46 @@ def _zero_requested_output_criteria_delivered_unverified(
data: Mapping[str, Any],
workflow_run_id: str | None,
) -> RecordedRunOutcome | None:
- if not zero_requested_output_criteria_credit(
+ emission_withhold = floor_rekeyed_emission_withhold(completion_verification)
+ registered_output_meaningful = _has_meaningful_registered_output_payload(data)
+ credit_withhold = zero_requested_output_criteria_credit(
completion_verification,
- has_meaningful_registered_output=_has_meaningful_registered_output_payload(data),
- ):
+ has_meaningful_registered_output=registered_output_meaningful,
+ )
+ if emission_withhold is None and not credit_withhold:
return None
- credit = floor_rekeyed_deliverable_credit(completion_verification)
- if credit is not None:
- payloads = _registered_output_parameter_payloads(data)
- payload_keys = sorted(
- {key for item in payloads for key in [item.get("output_parameter_key")] if isinstance(key, str) and key}
- )
- unbound_paths = [
- output_path
- for output_path in credit.output_paths
- if output_path and not registered_output_payload_binds_output_path(payloads, output_path)
- ]
- if unbound_paths:
- LOG.info(
- "copilot.completion.floor_rekeyed_credit_payload_unbound",
- workflow_run_id=workflow_run_id,
- criterion_ids=list(credit.criterion_ids),
- unbound_output_paths=unbound_paths,
- registered_output_keys=payload_keys,
- )
- else:
- LOG.info(
- "copilot.completion.floor_rekeyed_deliverable_credit",
- workflow_run_id=workflow_run_id,
- criterion_ids=list(credit.criterion_ids),
- evidence_sources=list(credit.evidence_sources),
- evidence_refs=list(credit.evidence_refs),
- credited_output_paths=list(credit.output_paths),
- registered_output_keys=payload_keys,
+ if emission_withhold is None:
+ credit = floor_rekeyed_deliverable_credit(completion_verification)
+ if credit is not None:
+ payloads = _registered_output_parameter_payloads(data)
+ payload_keys = sorted(
+ {key for item in payloads for key in [item.get("output_parameter_key")] if isinstance(key, str) and key}
)
- return None
+ unbound_paths = [
+ output_path
+ for output_path in credit.output_paths
+ if output_path and not registered_output_payload_binds_output_path(payloads, output_path)
+ ]
+ if unbound_paths:
+ LOG.info(
+ "copilot.completion.floor_rekeyed_credit_payload_unbound",
+ workflow_run_id=workflow_run_id,
+ criterion_ids=list(credit.criterion_ids),
+ unbound_output_paths=unbound_paths,
+ backed_by_criterion_id=dict(completion_verification.floor_rekeyed_backed_by_criterion_id),
+ registered_output_keys=payload_keys,
+ )
+ else:
+ LOG.info(
+ "copilot.completion.floor_rekeyed_deliverable_credit",
+ workflow_run_id=workflow_run_id,
+ criterion_ids=list(credit.criterion_ids),
+ evidence_sources=list(credit.evidence_sources),
+ evidence_refs=list(credit.evidence_refs),
+ credited_output_paths=list(credit.output_paths),
+ registered_output_keys=payload_keys,
+ )
+ return None
copilot_ctx.last_test_suspicious_success = False
copilot_ctx.last_test_failure_reason = None
copilot_ctx.suspicious_success_nudge_count = 0
@@ -3219,17 +3236,27 @@ def _zero_requested_output_criteria_delivered_unverified(
copilot_ctx.delivered_unverified_workflow_run_id = workflow_run_id
copilot_ctx.delivered_unverified_observed_outputs = _delivered_unverified_registered_outputs(data)
stash_delivered_unverified_turn_halt(copilot_ctx, workflow_run_id=workflow_run_id)
- LOG.info(
- "copilot.completion.zero_requested_output_credit_withheld",
- workflow_run_id=workflow_run_id,
- requested_output_criteria_count=0,
- registered_output_meaningful=True,
- )
+ if emission_withhold is not None:
+ LOG.info(
+ "copilot.completion.floor_rekeyed_emission_withheld",
+ workflow_run_id=workflow_run_id,
+ requested_output_criteria_count=completion_verification.requested_output_criteria_count,
+ registered_output_meaningful=registered_output_meaningful,
+ floor_rekeyed_unbacked_criterion_ids=list(emission_withhold.criterion_ids),
+ floor_rekeyed_unbacked_output_paths=list(emission_withhold.unbacked_output_paths),
+ )
+ display_reason = "The latest run did not produce the requested output, so it was not verified."
+ else:
+ LOG.info(
+ "copilot.completion.zero_requested_output_credit_withheld",
+ workflow_run_id=workflow_run_id,
+ requested_output_criteria_count=0,
+ registered_output_meaningful=True,
+ )
+ display_reason = "The latest run returned output, but it was not independently verified."
return RecordedRunOutcome(
verdict="not_evaluated",
- display_reason=run_outcome_display_reason(
- "The latest run returned output, but it was not independently verified."
- ),
+ display_reason=run_outcome_display_reason(display_reason),
workflow_run_id=workflow_run_id,
)
diff --git a/skyvern/forge/sdk/copilot/tools/workflow_update.py b/skyvern/forge/sdk/copilot/tools/workflow_update.py
index 0ea892c53..fedaafaa2 100644
--- a/skyvern/forge/sdk/copilot/tools/workflow_update.py
+++ b/skyvern/forge/sdk/copilot/tools/workflow_update.py
@@ -72,6 +72,7 @@ from skyvern.forge.sdk.copilot.code_block_synthesis import (
locator_selector_literals,
missing_rung_text,
normalized_locator_expr,
+ produce_covered_static_return_envelope,
render_missing_rung_call_sources,
synthesize_code_block,
synthesize_code_block_with_extraction,
@@ -95,8 +96,10 @@ from skyvern.forge.sdk.copilot.enforcement import (
_record_code_authoring_guardrail_reject,
_scouted_spine_open_obligation,
arm_credential_scout_reopen,
+ download_satisfied_requested_output_paths,
requested_output_extraction_plan,
requested_output_extraction_plan_changed,
+ requested_scalar_output_extraction_plan,
synthesized_goal_completion_landing_pending,
synthesized_persistence_reopened,
synthesized_persistence_reopened_after_failed_run,
@@ -1639,7 +1642,7 @@ def _requested_output_child_paths(ctx: AgentContext) -> set[str]:
return set()
paths: set[str] = set()
for criterion in _active_completion_criteria(ctx):
- if isinstance(criterion, CompletionCriterion) and _independent_judgment_output_criterion(criterion):
+ if isinstance(criterion, CompletionCriterion) and _is_judgment_boolean_criterion(criterion):
continue
if getattr(criterion, "level", None) == "definition":
continue
@@ -1681,18 +1684,16 @@ def _active_completion_criteria(ctx: AgentContext) -> list[CompletionCriterion]:
return request_policy.graded_completion_criteria()
-def _independent_judgment_output_criterion(criterion: CompletionCriterion) -> bool:
- return criterion.requested_output_evidence_source == "independent_run_evidence" and _is_judgment_boolean_criterion(
- criterion
- )
-
-
-def _independent_judgment_output_paths(ctx: AgentContext) -> set[str]:
+def _judgment_output_paths(ctx: AgentContext) -> set[str]:
+ # Judgment-boolean paths are dropped from the producer's static-bind set, not moved to the
+ # declaration lane; declaring one would fabricate a None where the run's own judgment belongs. The
+ # requested-output gate reads the completion criteria directly rather than this bind set, so dropping
+ # cannot pin it open.
paths: set[str] = set()
for criterion in _active_completion_criteria(ctx):
if not isinstance(criterion, CompletionCriterion):
continue
- if not _independent_judgment_output_criterion(criterion):
+ if not _is_judgment_boolean_criterion(criterion):
continue
path = _canonical_requested_output_path(criterion.output_path)
if path and _output_path_has_child(path):
@@ -2389,7 +2390,7 @@ def _output_contract_required_paths_source(ctx: AgentContext) -> _OutputContract
runtime_contract = _runtime_output_repair_contract_from_recorded_outcome(ctx)
antecedent_paths = _contingent_antecedent_child_paths(ctx)
if runtime_contract is not None:
- runtime_observation_paths = runtime_contract.required_paths - _independent_judgment_output_paths(ctx)
+ runtime_observation_paths = runtime_contract.required_paths - _judgment_output_paths(ctx)
return _OutputContractRequiredPaths(
observation_paths=runtime_observation_paths,
declaration_paths=antecedent_paths - runtime_observation_paths,
@@ -2397,6 +2398,7 @@ def _output_contract_required_paths_source(ctx: AgentContext) -> _OutputContract
reason_code=runtime_contract.reason_code,
)
observation_paths, source, reason_code = _required_child_output_paths_for_authoring(ctx)
+ observation_paths = observation_paths - _judgment_output_paths(ctx)
repair_context = getattr(ctx, "last_code_authoring_repair_context", None)
if (
not observation_paths
@@ -2411,7 +2413,7 @@ def _output_contract_required_paths_source(ctx: AgentContext) -> _OutputContract
*repair_context.required_code_return_paths,
]
)
- rehydrated -= _independent_judgment_output_paths(ctx)
+ rehydrated -= _judgment_output_paths(ctx)
# An antecedent the repair contract carried only in schema/return roles stays in the
# declaration lane on rehydration; the goal role is the observation-lane record.
observation_paths = rehydrated - (antecedent_paths - goal_paths)
@@ -3190,6 +3192,46 @@ class _SpineSplitOutcome(NamedTuple):
stage_count: int | None
+def _produce_split_extraction_envelope(
+ ctx: AgentContext,
+ extraction_suffix: str,
+ *,
+ required_paths: set[str],
+ declaration_paths: set[str] | None,
+ label: str,
+ signature: str,
+) -> str | None:
+ """Scalar-only producer for the spine-split seam: the extraction suffix carries no browser
+ actions, so the mixed download shape abstains here and the split bail stays byte-identical."""
+ declarations = declaration_paths or set()
+ if required_paths & download_satisfied_requested_output_paths(ctx):
+ return None
+ scalar_paths = required_paths - declarations
+ envelope = produce_covered_static_return_envelope(
+ extraction_suffix,
+ plan=requested_scalar_output_extraction_plan(ctx),
+ scalar_required_paths=scalar_paths,
+ declaration_paths=declarations,
+ download_required_paths=set(),
+ expects_download=False,
+ )
+ if envelope is None:
+ return None
+ _, revalidation = _extraction_code_with_required_static_return(
+ envelope.code, required_paths=scalar_paths, declaration_paths=declarations
+ )
+ if revalidation:
+ return None
+ LOG.info(
+ "copilot_output_contract_keyed_static_return_produced",
+ block_label=label,
+ keyed_paths=list(envelope.keyed_paths),
+ download_registration_paths=[],
+ canonical_output_contract_signature=signature,
+ )
+ return envelope.code
+
+
def _attempt_separated_spine_split(
*,
ctx: AgentContext,
@@ -3198,6 +3240,7 @@ def _attempt_separated_spine_split(
target_code: str,
synthesized: SynthesizedCodeBlock,
required_paths: set[str],
+ signature: str,
declaration_paths: set[str] | None = None,
) -> _SpineSplitOutcome:
code_block = next(
@@ -3230,7 +3273,17 @@ def _attempt_separated_spine_split(
extraction_suffix, required_paths=required_paths, declaration_paths=declaration_paths
)
if static_violations:
- return _SpineSplitOutcome(None, ["static_return_envelope_unavailable"], None)
+ produced_extraction = _produce_split_extraction_envelope(
+ ctx,
+ extraction_suffix,
+ required_paths=required_paths,
+ declaration_paths=declaration_paths,
+ label=label,
+ signature=signature,
+ )
+ if produced_extraction is None:
+ return _SpineSplitOutcome(None, ["static_return_envelope_unavailable"], None)
+ keyed_extraction = produced_extraction
if _browser_surface_contains_full_action_spine(keyed_extraction, reconciled_synthesized):
return _SpineSplitOutcome(None, ["extraction_retains_full_spine"], None)
@@ -3880,6 +3933,110 @@ def _stamp_declaration_contract_defaults(
return yaml.safe_dump(parsed, sort_keys=False), True
+def _impose_covered_static_return_envelope(
+ ctx: AgentContext,
+ *,
+ parsed: dict[str, Any],
+ workflow_yaml: str,
+ raw_code_artifact_metadata: object,
+ label: str,
+ target_code: str,
+ observation_paths: set[str],
+ declaration_paths: set[str],
+ source: str,
+ reason_code: str,
+ signature: str,
+ runtime_attempt_key: str,
+) -> tuple[str, object, bool] | None:
+ """Deterministic evidence-derived envelope on the whole-block scaffold route: author the scalar
+ live-reads (mixed shape merges the resolved download descriptor into one return), or admit a
+ download-only block, so ``static_return_envelope_unavailable`` never fires for scout-covered goals.
+ Returns None to leave today's bail ladder byte-identical."""
+ download_registered = download_satisfied_requested_output_paths(ctx)
+ scalar_paths = observation_paths - download_registered
+ download_paths = observation_paths & download_registered
+ download = ctx.reached_download_target
+ expects_download = download is not None and bool(download.selector) and _code_uses_expect_download(target_code)
+ block_metadata = _metadata_item_for_block_label(raw_code_artifact_metadata, label) or {}
+
+ if not scalar_paths:
+ if not (download_paths and expects_download):
+ return None
+ if _download_return_shape_error(label, block_metadata, target_code) is not None:
+ return None
+ scaffolded_metadata = _apply_metadata_contract_scaffold(
+ ctx,
+ workflow_yaml,
+ raw_code_artifact_metadata,
+ required_paths=observation_paths,
+ source=source,
+ reason_code=reason_code,
+ declaration_paths=declaration_paths,
+ allow_missing_static_return=True,
+ )
+ _mark_output_contract_imposed(ctx, signature)
+ _record_runtime_output_repair_attempt(ctx, runtime_attempt_key)
+ LOG.info(
+ "copilot_output_contract_download_registration_admitted",
+ block_label=label,
+ download_registration_paths=sorted(download_paths),
+ canonical_output_contract_signature=signature,
+ )
+ return workflow_yaml, scaffolded_metadata, scaffolded_metadata is not raw_code_artifact_metadata
+
+ envelope = produce_covered_static_return_envelope(
+ target_code,
+ plan=requested_scalar_output_extraction_plan(ctx),
+ scalar_required_paths=scalar_paths,
+ declaration_paths=declaration_paths,
+ download_required_paths=download_paths,
+ expects_download=expects_download,
+ )
+ if envelope is None:
+ return None
+ _, revalidation = _extraction_code_with_required_static_return(
+ envelope.code, required_paths=scalar_paths, declaration_paths=declaration_paths
+ )
+ if revalidation:
+ return None
+ if download_paths and _download_return_shape_error(label, block_metadata, envelope.code) is not None:
+ return None
+
+ for block in _workflow_code_blocks(parsed):
+ if str(block.get("label") or "").strip() == label:
+ block["code"] = envelope.code.rstrip() + "\n"
+ break
+ imposed_yaml = yaml.safe_dump(parsed, sort_keys=False)
+ scaffolded_metadata = _apply_metadata_contract_scaffold(
+ ctx,
+ imposed_yaml,
+ raw_code_artifact_metadata,
+ required_paths=observation_paths,
+ source=source,
+ reason_code=reason_code,
+ declaration_paths=declaration_paths,
+ allow_missing_static_return=bool(download_paths),
+ )
+ _mark_output_contract_imposed(ctx, signature)
+ _record_runtime_output_repair_attempt(ctx, runtime_attempt_key)
+ LOG.info(
+ "copilot_output_contract_keyed_static_return_produced",
+ block_label=label,
+ keyed_paths=list(envelope.keyed_paths),
+ download_registration_paths=sorted(download_paths),
+ canonical_output_contract_signature=signature,
+ )
+ LOG.info(
+ "copilot_output_contract_envelope_imposed_after_steering",
+ block_label=label,
+ canonical_output_contract_signature=signature,
+ steering_reject_count=_output_contract_reject_count(ctx, signature),
+ return_envelope_applied=True,
+ metadata_scaffold_applied=scaffolded_metadata is not raw_code_artifact_metadata,
+ )
+ return imposed_yaml, scaffolded_metadata, True
+
+
def _impose_output_contract_envelope_after_steering(
ctx: AgentContext,
workflow_yaml: str,
@@ -3944,6 +4101,7 @@ def _impose_output_contract_envelope_after_steering(
target_code=target_code,
synthesized=synthesized,
required_paths=required_paths,
+ signature=signature,
declaration_paths=declaration_paths,
)
if split.imposed_yaml is not None:
@@ -4008,6 +4166,23 @@ def _impose_output_contract_envelope_after_steering(
declaration_paths=declaration_paths,
)
if violations:
+ produced = _impose_covered_static_return_envelope(
+ ctx,
+ parsed=parsed,
+ workflow_yaml=workflow_yaml,
+ raw_code_artifact_metadata=raw_code_artifact_metadata,
+ label=label,
+ target_code=target_code,
+ observation_paths=observation_paths,
+ declaration_paths=declaration_paths,
+ source=source,
+ reason_code=reason_code,
+ signature=signature,
+ runtime_attempt_key=runtime_attempt_key,
+ )
+ if produced is not None:
+ produced_yaml, produced_metadata, produced_changed = produced
+ return produced_yaml, produced_metadata, produced_changed or declaration_stamped
scaffolded_metadata = _apply_metadata_contract_scaffold(
ctx,
workflow_yaml,
diff --git a/skyvern/forge/sdk/forge_log.py b/skyvern/forge/sdk/forge_log.py
index 34ddcb20a..cfc183202 100644
--- a/skyvern/forge/sdk/forge_log.py
+++ b/skyvern/forge/sdk/forge_log.py
@@ -461,13 +461,17 @@ def setup_logger() -> None:
handler = logging.StreamHandler()
handler.setFormatter(
structlog.stdlib.ProcessorFormatter(
+ foreign_pre_chain=[
+ add_error_processor,
+ structlog.processors.format_exc_info,
+ ],
processors=[
structlog.stdlib.add_log_level,
structlog.stdlib.add_logger_name,
structlog.stdlib.ProcessorFormatter.remove_processors_meta,
structlog.processors.TimeStamper(fmt="iso"),
renderer,
- ]
+ ],
)
)
root_logger = logging.getLogger()
diff --git a/tests/unit/test_copilot_code_block_persist_seam.py b/tests/unit/test_copilot_code_block_persist_seam.py
index 1b91f6b43..dfca744ae 100644
--- a/tests/unit/test_copilot_code_block_persist_seam.py
+++ b/tests/unit/test_copilot_code_block_persist_seam.py
@@ -17,6 +17,7 @@ from structlog.testing import capture_logs
from skyvern.forge.sdk.copilot import agent as agent_module
from skyvern.forge.sdk.copilot import enforcement as enforcement_module
+from skyvern.forge.sdk.copilot import request_policy as request_policy_module
from skyvern.forge.sdk.copilot.blocker_signal import (
CREDENTIAL_SCOUT_VERIFY_REPLY,
CopilotToolBlockerSignal,
@@ -41,12 +42,14 @@ from skyvern.forge.sdk.copilot.code_block_preflight import (
)
from skyvern.forge.sdk.copilot.code_block_security import CodeBlockSecurityError
from skyvern.forge.sdk.copilot.code_block_synthesis import (
+ ProducedStaticReturnEnvelope,
ScoutGap,
SynthesisDiagnostics,
SynthesizedCodeBlock,
_get_by_role_expr,
_get_by_role_expr_strict,
credential_scout_gap,
+ produce_covered_static_return_envelope,
)
from skyvern.forge.sdk.copilot.completion_verification import CompletionVerificationResult, CriterionVerdict
from skyvern.forge.sdk.copilot.config import BlockAuthoringPolicy, CopilotConfig
@@ -60,6 +63,12 @@ from skyvern.forge.sdk.copilot.output_contracts import (
OutputContractAdvisoryState,
code_block_available_binding_keys_by_label,
)
+from skyvern.forge.sdk.copilot.output_extraction_plan import (
+ LiveReadBinding,
+ LiveReadKind,
+ RequestedOutputExtractionPlan,
+ RevealAnchor,
+)
from skyvern.forge.sdk.copilot.output_utils import sanitize_tool_result_for_llm
from skyvern.forge.sdk.copilot.reached_download_target import ReachedDownloadTarget
from skyvern.forge.sdk.copilot.request_policy import (
@@ -4373,6 +4382,94 @@ class TestCodeRepairProgressClassification:
assert source == "requested_output_contract"
assert reason_code == "requested_output_contract_missing_output_coverage"
+ def test_runtime_output_judgment_is_excluded_from_scalar_bind_set(self) -> None:
+ ctx = _code_only_ctx()
+ ctx.request_policy = RequestPolicy(
+ completion_criteria=[
+ CompletionCriterion(
+ id="login_gate",
+ outcome="the target path is blocked by a login gate",
+ output_path="output.login_gate_blocks_target",
+ expected_output_value=True,
+ expected_output_shape="goal_judgment_boolean",
+ requested_output_evidence_source="runtime_output",
+ ),
+ CompletionCriterion(
+ id="record_id",
+ outcome="the record id is returned",
+ output_path="output.record_id",
+ ),
+ ]
+ )
+
+ contract = workflow_update_module._output_contract_required_paths_source(ctx)
+
+ assert contract.observation_paths == {"output.record_id"}
+ assert contract.union == {"output.record_id"}
+ assert "output.login_gate_blocks_target" not in contract.declaration_paths
+
+ def test_runtime_output_judgment_alone_yields_empty_bind_set(self) -> None:
+ ctx = _code_only_ctx()
+ ctx.request_policy = RequestPolicy(
+ completion_criteria=[
+ CompletionCriterion(
+ id="login_gate",
+ outcome="the target path is blocked by a login gate",
+ output_path="output.login_gate_blocks_target",
+ expected_output_value=True,
+ expected_output_shape="goal_judgment_boolean",
+ requested_output_evidence_source="runtime_output",
+ )
+ ]
+ )
+
+ assert workflow_update_module._output_contract_required_paths_source(ctx).union == set()
+
+ def test_non_judgment_scalar_bind_set_is_unchanged(self) -> None:
+ ctx = _code_only_ctx()
+ ctx.request_policy = RequestPolicy(
+ completion_criteria=[
+ CompletionCriterion(
+ id="record_id",
+ outcome="the record id is returned",
+ output_path="output.record_id",
+ )
+ ]
+ )
+
+ assert workflow_update_module._output_contract_required_paths_source(ctx).observation_paths == {
+ "output.record_id"
+ }
+
+ def test_judgment_output_paths_widens_independent_predicate(self) -> None:
+ ctx = _code_only_ctx()
+ ctx.request_policy = RequestPolicy(
+ completion_criteria=[
+ CompletionCriterion(
+ id="independent_gate",
+ outcome="the independent path is blocked by a login gate",
+ output_path="output.independent_gate_blocks_target",
+ expected_output_shape="goal_judgment_boolean",
+ requested_output_evidence_source="independent_run_evidence",
+ ),
+ CompletionCriterion(
+ id="runtime_gate",
+ outcome="the runtime path is blocked by a login gate",
+ output_path="output.runtime_gate_blocks_target",
+ expected_output_value=True,
+ expected_output_shape="goal_judgment_boolean",
+ requested_output_evidence_source="runtime_output",
+ ),
+ ]
+ )
+
+ all_judgment = workflow_update_module._judgment_output_paths(ctx)
+
+ assert all_judgment == {
+ "output.independent_gate_blocks_target",
+ "output.runtime_gate_blocks_target",
+ }
+
def test_independent_judgment_runtime_repair_fact_is_not_rehydrated_as_code_return_path(self) -> None:
ctx = _code_only_ctx()
ctx.request_policy = RequestPolicy(
@@ -15006,3 +15103,431 @@ class TestSequencedDownloadTerminalCoverage:
assert guarded is not None
assert any("scouted_spine_under_build" in violation for violation in guarded.violations)
+
+
+def _scalar_binding(output_path: str, label: str, selector: str) -> LiveReadBinding:
+ return LiveReadBinding(
+ output_path=output_path,
+ kind=LiveReadKind.KEY_VALUE,
+ selector=selector,
+ selector_count=1,
+ selector_index=0,
+ child_index=1,
+ child_count=2,
+ relation_label=label,
+ )
+
+
+def _scalar_plan(*bindings: LiveReadBinding) -> RequestedOutputExtractionPlan:
+ return RequestedOutputExtractionPlan(
+ requested_output_paths=tuple(binding.output_path for binding in bindings),
+ observation_step=1,
+ observation_identity="obs-identity",
+ reveal=RevealAnchor(selector="#reveal"),
+ live_reads=tuple(bindings),
+ identity="plan-identity",
+ )
+
+
+_MIXED_DOWNLOAD_CODE = (
+ "async with page.expect_download() as dl_info:\n"
+ ' await page.locator("#download-invoice").click()\n'
+ "dl_info_file = await dl_info.value\n"
+ "downloaded_file_name = dl_info_file.suggested_filename\n"
+ "await dl_info_file.path()\n"
+ "return {\n"
+ ' "downloaded_file_name": downloaded_file_name,\n'
+ "}\n"
+)
+
+
+def test_producer_scalar_coverage_single_return_over_all_paths() -> None:
+ plan = _scalar_plan(
+ _scalar_binding("output.account_number", "Account Number", ".acct"),
+ _scalar_binding("output.confirmation_number", "Confirmation Number", ".conf"),
+ )
+ envelope = produce_covered_static_return_envelope(
+ 'await page.locator("#lookup").click()\n',
+ plan=plan,
+ scalar_required_paths={"output.account_number", "output.confirmation_number"},
+ declaration_paths=set(),
+ download_required_paths=set(),
+ expects_download=False,
+ )
+ assert isinstance(envelope, ProducedStaticReturnEnvelope)
+ assert envelope.keyed_paths == ("output.account_number", "output.confirmation_number")
+ assert envelope.code.count("return {") == 1
+ produced = workflow_update_module._code_block_produced_output_paths(envelope.code)
+ assert {"output.account_number", "output.confirmation_number"} <= produced
+ _, violations = workflow_update_module._extraction_code_with_required_static_return(
+ envelope.code, required_paths={"output.account_number", "output.confirmation_number"}
+ )
+ assert violations == []
+
+
+def test_producer_mixed_merged_single_return_passes_download_gate() -> None:
+ plan = _scalar_plan(
+ _scalar_binding("output.statement_amount", "Amount Due", ".amt"),
+ _scalar_binding("output.statement_date", "Statement Date", ".dt"),
+ )
+ envelope = produce_covered_static_return_envelope(
+ _MIXED_DOWNLOAD_CODE,
+ plan=plan,
+ scalar_required_paths={"output.statement_amount", "output.statement_date"},
+ declaration_paths=set(),
+ download_required_paths={"output.downloaded_invoice_pdf"},
+ expects_download=True,
+ )
+ assert isinstance(envelope, ProducedStaticReturnEnvelope)
+ assert envelope.code.count("return {") == 1
+ assert '"downloaded_file_name": downloaded_file_name' in envelope.code
+ assert workflow_update_module._code_uses_expect_download(envelope.code)
+ artifact = {"claimed_outcomes": [{"id": "claim", "goal_value_paths": ["output.downloaded_invoice_pdf"]}]}
+ assert workflow_update_module._download_return_shape_error("block", artifact, envelope.code) is None
+ _, violations = workflow_update_module._extraction_code_with_required_static_return(
+ envelope.code, required_paths={"output.statement_amount", "output.statement_date"}
+ )
+ assert violations == []
+
+
+def test_producer_no_coverage_abstains() -> None:
+ plan = _scalar_plan(_scalar_binding("output.account_number", "Account Number", ".acct"))
+ envelope = produce_covered_static_return_envelope(
+ 'await page.locator("#lookup").click()\n',
+ plan=plan,
+ scalar_required_paths={"output.account_number", "output.confirmation_number"},
+ declaration_paths=set(),
+ download_required_paths=set(),
+ expects_download=False,
+ )
+ assert envelope is None
+
+
+def test_producer_output_passes_keyer_on_reduced_set() -> None:
+ plan = _scalar_plan(_scalar_binding("output.amount", "Amount", ".amt"))
+ envelope = produce_covered_static_return_envelope(
+ 'await page.locator("#lookup").click()\n',
+ plan=plan,
+ scalar_required_paths={"output.amount"},
+ declaration_paths={"output.optional_note"},
+ download_required_paths=set(),
+ expects_download=False,
+ )
+ assert envelope is not None
+ _, violations = workflow_update_module._extraction_code_with_required_static_return(
+ envelope.code, required_paths={"output.amount"}, declaration_paths={"output.optional_note"}
+ )
+ assert violations == []
+
+
+def test_producer_abstains_when_unbindable_judgment_path_kept_in_scalar_set() -> None:
+ plan = _scalar_plan(_scalar_binding("output.record_id", "Record Identifier", ".record-kv"))
+ envelope = produce_covered_static_return_envelope(
+ 'await page.locator("#lookup").click()\n',
+ plan=plan,
+ scalar_required_paths={"output.record_id", "output.login_gate_blocks_target"},
+ declaration_paths=set(),
+ download_required_paths=set(),
+ expects_download=False,
+ )
+ assert envelope is None
+
+
+def test_producer_emits_keyed_scalar_when_judgment_path_excluded() -> None:
+ plan = _scalar_plan(_scalar_binding("output.record_id", "Record Identifier", ".record-kv"))
+ envelope = produce_covered_static_return_envelope(
+ 'await page.locator("#lookup").click()\n',
+ plan=plan,
+ scalar_required_paths={"output.record_id"},
+ declaration_paths=set(),
+ download_required_paths=set(),
+ expects_download=False,
+ )
+ assert isinstance(envelope, ProducedStaticReturnEnvelope)
+ assert envelope.keyed_paths == ("output.record_id",)
+ assert "login_gate_blocks_target" not in envelope.code
+ produced = workflow_update_module._code_block_produced_output_paths(envelope.code)
+ assert "output.record_id" in produced
+ assert "output.login_gate_blocks_target" not in produced
+
+
+def test_producer_branchy_return_abstains() -> None:
+ plan = _scalar_plan(_scalar_binding("output.amount", "Amount", ".amt"))
+ envelope = produce_covered_static_return_envelope(
+ 'if await page.locator("#done").is_visible():\n return {"output": {}}\n',
+ plan=plan,
+ scalar_required_paths={"output.amount"},
+ declaration_paths=set(),
+ download_required_paths=set(),
+ expects_download=False,
+ )
+ assert envelope is None
+
+
+def test_producer_download_without_idiom_abstains() -> None:
+ plan = _scalar_plan(_scalar_binding("output.statement_amount", "Amount Due", ".amt"))
+ envelope = produce_covered_static_return_envelope(
+ _MIXED_DOWNLOAD_CODE,
+ plan=plan,
+ scalar_required_paths={"output.statement_amount"},
+ declaration_paths=set(),
+ download_required_paths={"output.downloaded_invoice_pdf"},
+ expects_download=False,
+ )
+ assert envelope is None
+
+
+def test_producer_identifier_collision_abstains() -> None:
+ plan = _scalar_plan(_scalar_binding("output.amount", "Amount", ".amt"))
+ envelope = produce_covered_static_return_envelope(
+ '_envelope_value_0 = 1\nawait page.locator("#lookup").click()\n',
+ plan=plan,
+ scalar_required_paths={"output.amount"},
+ declaration_paths=set(),
+ download_required_paths=set(),
+ expects_download=False,
+ )
+ assert envelope is None
+
+
+def test_producer_declaration_paths_are_none_defaults() -> None:
+ plan = _scalar_plan(_scalar_binding("output.amount", "Amount", ".amt"))
+ envelope = produce_covered_static_return_envelope(
+ 'await page.locator("#lookup").click()\n',
+ plan=plan,
+ scalar_required_paths={"output.amount"},
+ declaration_paths={"output.optional_note"},
+ download_required_paths=set(),
+ expects_download=False,
+ )
+ assert envelope is not None
+ assert '"optional_note": None' in envelope.code
+ assert "optional_note" not in envelope.code.split('"optional_note": None')[0]
+
+
+def test_producer_generated_code_raises_on_empty_scalar_text() -> None:
+ plan = _scalar_plan(_scalar_binding("output.amount", "Amount", ".amt"))
+ envelope = produce_covered_static_return_envelope(
+ 'await page.locator("#lookup").click()\n',
+ plan=plan,
+ scalar_required_paths={"output.amount"},
+ declaration_paths=set(),
+ download_required_paths=set(),
+ expects_download=False,
+ )
+ assert envelope is not None
+ tree = ast.parse(envelope.code)
+ empty_guards = [
+ node
+ for node in ast.walk(tree)
+ if isinstance(node, ast.If)
+ and isinstance(node.test, ast.UnaryOp)
+ and isinstance(node.test.op, ast.Not)
+ and any(isinstance(child, ast.Raise) for child in node.body)
+ ]
+ assert empty_guards
+
+
+def test_produce_split_extraction_envelope_mixed_abstains(monkeypatch: pytest.MonkeyPatch) -> None:
+ ctx = SimpleNamespace(
+ reached_download_target=SimpleNamespace(selector="#download-invoice"),
+ scouted_output_covered_paths={"output.statement_amount"},
+ )
+ monkeypatch.setattr(
+ workflow_update_module,
+ "download_satisfied_requested_output_paths",
+ lambda _ctx: {"output.downloaded_invoice_pdf"},
+ )
+ result = workflow_update_module._produce_split_extraction_envelope(
+ ctx,
+ 'value = "x"\n',
+ required_paths={"output.statement_amount", "output.downloaded_invoice_pdf"},
+ declaration_paths=set(),
+ label="extract_block",
+ signature="sig",
+ )
+ assert result is None
+
+
+def _download_classification_ctx(
+ *criteria: CompletionCriterion,
+ covered_paths: set[str] | None = None,
+ download_selector: str | None = "#download-invoice",
+) -> SimpleNamespace:
+ download = (
+ ReachedDownloadTarget(
+ selector=download_selector,
+ affordance_text="Download Invoice",
+ download_kind="attribute",
+ source_step="trajectory_recency",
+ already_registered=False,
+ )
+ if download_selector
+ else None
+ )
+ return SimpleNamespace(
+ completion_criteria_turn_state=SimpleNamespace(decision=SimpleNamespace(criteria=tuple(criteria))),
+ last_code_authoring_repair_context=None,
+ scouted_output_covered_paths=set(covered_paths or set()),
+ reached_download_target=download,
+ flow_evidence=[],
+ )
+
+
+def test_uncovered_scalar_path_is_not_classified_as_a_download_path() -> None:
+ ctx = _download_classification_ctx(
+ CompletionCriterion(
+ id="statement_amount",
+ outcome="The returned record includes the statement amount.",
+ output_path="output.statement_amount",
+ ),
+ covered_paths=set(),
+ )
+
+ assert enforcement_module.download_satisfied_requested_output_paths(ctx) == set()
+ assert enforcement_module.uncovered_requested_output_paths(ctx) == {"output.statement_amount"}
+ assert (
+ workflow_update_module._impose_covered_static_return_envelope(
+ ctx,
+ parsed={},
+ workflow_yaml="",
+ raw_code_artifact_metadata=[],
+ label="download_block",
+ target_code=_MIXED_DOWNLOAD_CODE,
+ observation_paths={"output.statement_amount"},
+ declaration_paths=set(),
+ source="scout",
+ reason_code="reason",
+ signature="sig",
+ runtime_attempt_key="key",
+ )
+ is None
+ )
+
+
+def test_declared_registered_download_kind_on_uncovered_custom_path_is_a_download_path() -> None:
+ ctx = _download_classification_ctx(
+ CompletionCriterion(
+ id="downloaded_invoice_pdf",
+ outcome="The returned record includes the downloaded invoice pdf.",
+ output_path="output.downloaded_invoice_pdf",
+ declared_deliverable_kind="registered_download",
+ ),
+ covered_paths=set(),
+ )
+
+ assert enforcement_module.download_satisfied_requested_output_paths(ctx) == {"output.downloaded_invoice_pdf"}
+ assert enforcement_module.uncovered_requested_output_paths(ctx) == set()
+
+
+def test_declared_registered_download_kind_on_scout_covered_path_is_not_a_download_path() -> None:
+ ctx = _download_classification_ctx(
+ CompletionCriterion(
+ id="npi",
+ outcome="The returned record includes NPI.",
+ output_path="output.npi",
+ declared_deliverable_kind="registered_download",
+ ),
+ CompletionCriterion(
+ id="downloaded_invoice_pdf",
+ outcome="The returned record includes the downloaded invoice pdf.",
+ output_path="output.downloaded_invoice_pdf",
+ declared_deliverable_kind="registered_download",
+ ),
+ covered_paths={"output.npi"},
+ )
+
+ assert enforcement_module.download_satisfied_requested_output_paths(ctx) == {"output.downloaded_invoice_pdf"}
+
+
+def test_declared_registered_download_kind_survives_requested_output_canonicalization() -> None:
+ policy = RequestPolicy(
+ completion_criteria=[
+ CompletionCriterion(
+ id="download",
+ outcome="The workflow downloads the invoice file.",
+ output_path="output.downloaded_invoice_pdf",
+ deliverable_kind="registered_download",
+ declared_deliverable_kind="registered_download",
+ )
+ ]
+ )
+
+ request_policy_module._apply_requested_output_completion_criteria(
+ policy, "return the downloaded_invoice_pdf when you are done"
+ )
+
+ canonical = {criterion.output_path: criterion for criterion in policy.completion_criteria if criterion.output_path}
+ criterion = canonical["output.downloaded_invoice_pdf"]
+ assert criterion.declared_deliverable_kind == "registered_download"
+ assert criterion.deliverable_kind is None
+
+
+def test_split_seam_declaration_path_is_none_defaulted_not_live_read(monkeypatch: pytest.MonkeyPatch) -> None:
+ plan = _scalar_plan(
+ _scalar_binding("output.amount", "Amount", ".amt"),
+ _scalar_binding("output.optional_note", "Optional Note", ".note"),
+ )
+ monkeypatch.setattr(workflow_update_module, "download_satisfied_requested_output_paths", lambda _ctx: set())
+ monkeypatch.setattr(workflow_update_module, "requested_scalar_output_extraction_plan", lambda _ctx: plan)
+
+ produced = workflow_update_module._produce_split_extraction_envelope(
+ SimpleNamespace(),
+ 'await page.locator("#lookup").click()\n',
+ required_paths={"output.amount", "output.optional_note"},
+ declaration_paths={"output.optional_note"},
+ label="extract_block",
+ signature="sig",
+ )
+
+ assert produced is not None
+ assert '"optional_note": None' in produced
+ assert ".note" not in produced
+ assert "Optional Note" not in produced
+
+
+def test_producer_abstains_when_terminal_return_shares_a_line() -> None:
+ plan = _scalar_plan(_scalar_binding("output.statement_amount", "Amount Due", ".amt"))
+ code = (
+ "async with page.expect_download() as dl_info:\n"
+ ' await page.locator("#download-invoice").click()\n'
+ "dl_info_file = await dl_info.value\n"
+ 'name = dl_info_file.suggested_filename; return {"downloaded_file_name": name}\n'
+ )
+
+ envelope = produce_covered_static_return_envelope(
+ code,
+ plan=plan,
+ scalar_required_paths={"output.statement_amount"},
+ declaration_paths=set(),
+ download_required_paths={"output.downloaded_invoice_pdf"},
+ expects_download=True,
+ )
+
+ assert envelope is None
+
+
+def test_producer_preserves_the_models_download_descriptor_key() -> None:
+ plan = _scalar_plan(_scalar_binding("output.statement_amount", "Amount Due", ".amt"))
+ code = (
+ "async with page.expect_download() as dl_info:\n"
+ ' await page.locator("#download-invoice").click()\n'
+ 'return {"saved_as": dl_info.value.suggested_filename}\n'
+ )
+
+ envelope = produce_covered_static_return_envelope(
+ code,
+ plan=plan,
+ scalar_required_paths={"output.statement_amount"},
+ declaration_paths=set(),
+ download_required_paths={"output.downloaded_invoice_pdf"},
+ expects_download=True,
+ )
+
+ assert envelope is not None
+ assert envelope.code.count("return {") == 1
+ assert '"saved_as": dl_info.value.suggested_filename' in envelope.code
+ assert "downloaded_file_name" not in envelope.code
+ artifact = {"claimed_outcomes": [{"id": "claim", "goal_value_paths": ["output.downloaded_invoice_pdf"]}]}
+ assert workflow_update_module._download_return_shape_error("block", artifact, envelope.code) is None
+ assert {"output.statement_amount"} <= workflow_update_module._code_block_produced_output_paths(envelope.code)
diff --git a/tests/unit/test_copilot_completion_verification.py b/tests/unit/test_copilot_completion_verification.py
index 592a9029f..e851ad6fe 100644
--- a/tests/unit/test_copilot_completion_verification.py
+++ b/tests/unit/test_copilot_completion_verification.py
@@ -29,6 +29,7 @@ from skyvern.forge.sdk.copilot.completion_criteria_store import (
from skyvern.forge.sdk.copilot.completion_output_grounding import (
_schema_boolean_output_paths,
_value_matches_expected,
+ floor_rekeyed_path_backing,
grade_requested_output_criteria,
split_requested_output_criteria,
)
@@ -39,6 +40,7 @@ from skyvern.forge.sdk.copilot.completion_verification import (
DeliveredUnverifiedTerminalState,
EvidenceSourceKind,
FloorRekeyedDeliverableCredit,
+ FloorRekeyedEmissionWithhold,
RunEvidenceSnapshot,
_coerce_result,
_structured_record_has_identifier,
@@ -48,6 +50,8 @@ from skyvern.forge.sdk.copilot.completion_verification import (
degraded_contract_delivered_unverified_terminal_state,
evaluate_completion_criteria,
floor_rekeyed_deliverable_credit,
+ floor_rekeyed_emission_lane_fields,
+ floor_rekeyed_emission_withhold,
grade_definition_criteria,
grade_fallback_floor_reached_end_state_criteria,
grade_present_value_criteria,
@@ -126,6 +130,7 @@ from skyvern.forge.sdk.copilot.tools.completion import (
_POST_RUN_PAGE_OBSERVATION_LABEL,
_artifact_health_blocker_from_result,
_completion_verification_from_run_result,
+ _floor_rekeyed_emission_evidence,
_reconcile_download_completion_criterion,
)
from skyvern.forge.sdk.copilot.tools.composition_capture import _active_run_terminal_monitor_enabled
@@ -3448,6 +3453,7 @@ def _floor_rekeyed_result(
verdicts: list[CriterionVerdict],
marked_ids: list[str],
output_path_by_id: dict[str, str] | None = None,
+ backed_by_id: dict[str, bool] | None = None,
) -> CompletionVerificationResult:
return CompletionVerificationResult(
status="evaluated",
@@ -3455,6 +3461,7 @@ def _floor_rekeyed_result(
verdicts=list(verdicts),
floor_rekeyed_criterion_ids=list(marked_ids),
floor_rekeyed_output_path_by_criterion_id=dict(output_path_by_id or {}),
+ floor_rekeyed_backed_by_criterion_id=dict(backed_by_id or {}),
)
@@ -3670,6 +3677,7 @@ def test_floor_rekeyed_credit_grants_when_floor_path_binds_delivered_payload() -
_floored_run_plane_verdicts(corroborator_source="independent_page_evidence"),
["deliverable"],
{"deliverable": "output.document_name"},
+ backed_by_id={"deliverable": True},
)
with capture_logs() as logs:
@@ -3692,6 +3700,7 @@ def test_floor_rekeyed_credit_withheld_when_floor_path_absent_from_payload() ->
_floored_run_plane_verdicts(corroborator_source="independent_page_evidence"),
["deliverable"],
{"deliverable": "output.document_name"},
+ backed_by_id={"deliverable": True},
)
with capture_logs() as logs:
@@ -3709,7 +3718,7 @@ def test_floor_rekeyed_credit_withheld_when_floor_path_absent_from_payload() ->
assert any(entry.get("event") == "copilot.completion.zero_requested_output_credit_withheld" for entry in logs)
-def test_floor_rekeyed_deliverable_reaches_verified_success() -> None:
+def test_floor_rekeyed_deliverable_withholds_without_runtime_backing() -> None:
ctx = _ctx_with_blocks("code")
verification = _floor_rekeyed_result(
_corroborated_abstention_verdicts(
@@ -3717,6 +3726,7 @@ def test_floor_rekeyed_deliverable_reaches_verified_success() -> None:
corroborator_evidence_ref=_PAGE_EVIDENCE_DELIVERABLE_REF,
),
["deliverable"],
+ {"deliverable": "output.document_name"},
)
with capture_logs() as logs:
@@ -3725,13 +3735,15 @@ def test_floor_rekeyed_deliverable_reaches_verified_success() -> None:
)
assert recorded is not None
- assert recorded.verdict == "demonstrated"
- assert ctx.delivered_unverified_terminal is False
- assert ctx.turn_halt is None
- granted = [entry for entry in logs if entry.get("event") == "copilot.completion.floor_rekeyed_deliverable_credit"]
- assert granted and granted[0]["criterion_ids"] == ["deliverable"]
- assert granted[0]["evidence_sources"] == ["independent_page_evidence"]
- assert not any(entry.get("event") == "copilot.completion.zero_requested_output_credit_withheld" for entry in logs)
+ assert recorded.verdict == "not_evaluated"
+ assert ctx.delivered_unverified_terminal is True
+ assert ctx.turn_halt is not None
+ assert ctx.turn_halt.kind.value == "delivered_unverified"
+ assert not any(entry.get("event") == "copilot.completion.floor_rekeyed_deliverable_credit" for entry in logs)
+ lane = [entry for entry in logs if entry.get("event") == "copilot.completion.floor_rekeyed_emission_lane"]
+ assert lane and lane[0]["engaged"] is True
+ assert lane[0]["unbacked_criterion_ids"] == ["deliverable"]
+ assert any(entry.get("event") == "copilot.completion.floor_rekeyed_emission_withheld" for entry in logs)
def _floored_policy_criteria(criteria: list[CompletionCriterion], user_message: str) -> list[CompletionCriterion]:
@@ -3821,6 +3833,16 @@ async def test_floor_rekeyed_deliverable_is_credited_through_the_verification_pr
)
ctx = _ctx_with_blocks("code")
ctx.request_policy = policy
+ ctx.last_workflow_yaml = textwrap.dedent(
+ """
+ workflow_definition:
+ blocks:
+ - block_type: code
+ label: extract_bill
+ code: |
+ return {"doc_status": "Delivered"}
+ """
+ ).strip()
ctx.composition_page_evidence = {
"workflow_run_id": "wr_x",
"observed_after_workflow_run": True,
@@ -3862,10 +3884,456 @@ def test_floor_rekeyed_runtime_output_only_still_withholds() -> None:
assert ctx.delivered_unverified_terminal is True
assert ctx.turn_halt is not None
assert ctx.turn_halt.kind.value == "delivered_unverified"
- assert any(entry.get("event") == "copilot.completion.zero_requested_output_credit_withheld" for entry in logs)
+ assert any(entry.get("event") == "copilot.completion.floor_rekeyed_emission_withheld" for entry in logs)
+ assert not any(entry.get("event") == "copilot.completion.floor_rekeyed_deliverable_credit" for entry in logs)
+
+
+def _backing_ctx(returns_literal: str, block_label: str = "extract_bill") -> CopilotContext:
+ ctx = _run_ctx()
+ ctx.last_workflow = SimpleNamespace(
+ workflow_definition=SimpleNamespace(blocks=[SimpleNamespace(label=block_label, block_type="code")])
+ )
+ ctx.last_workflow_yaml = textwrap.dedent(
+ f"""
+ workflow_definition:
+ blocks:
+ - block_type: code
+ label: {block_label}
+ code: |
+ return {returns_literal}
+ """
+ ).strip()
+ return ctx
+
+
+def _marked_output_criterion(path: str, cid: str = "c1") -> CompletionCriterion:
+ return CompletionCriterion(
+ id=cid,
+ outcome="The requested deliverable is returned.",
+ level="run",
+ requested_output_floor_rekeyed=True,
+ floor_rekeyed_from_path=path,
+ )
+
+
+def _backing_for(ctx: CopilotContext, criteria: list[CompletionCriterion], run_data: dict) -> dict[str, bool]:
+ block_outputs, sources, block_types, runtime_envelope_labels = _floor_rekeyed_emission_evidence(ctx, run_data)
+ return floor_rekeyed_path_backing(
+ ctx, criteria, block_outputs, sources, block_types, runtime_envelope_labels=runtime_envelope_labels
+ )
+
+
+def test_backing_backs_registered_scalar_at_floor_path() -> None:
+ ctx = _backing_ctx('{"doc_status": "Delivered"}')
+ run_data = _registered_output_result({"doc_status": "Delivered"})["data"]
+
+ backing = _backing_for(ctx, [_marked_output_criterion("output.doc_status")], run_data)
+
+ assert backing == {"c1": True}
+
+
+def test_backing_does_not_back_metadata_only_envelope() -> None:
+ envelope_type = next(iter(_TASK_ENVELOPE_BLOCK_TYPES))
+ ctx = _backing_ctx('{"doc_status": "Delivered"}')
+ envelope = {
+ "task_id": "t1",
+ "status": "completed",
+ "extracted_information": None,
+ "downloaded_files": None,
+ "downloaded_file_urls": None,
+ }
+ run_data = _registered_output_result(envelope, block_type=envelope_type)["data"]
+
+ backing = _backing_for(ctx, [_marked_output_criterion("output.doc_status")], run_data)
+
+ assert backing == {"c1": False}
+
+
+def _task_envelope_ctx(label: str = "run_task") -> CopilotContext:
+ ctx = _run_ctx()
+ ctx.last_workflow = SimpleNamespace(
+ workflow_definition=SimpleNamespace(blocks=[SimpleNamespace(label=label, block_type="task_v2")])
+ )
+ ctx.last_workflow_yaml = textwrap.dedent(
+ f"""
+ workflow_definition:
+ blocks:
+ - block_type: task_v2
+ label: {label}
+ """
+ ).strip()
+ return ctx
+
+
+def _task_envelope_run_data(label: str, extracted_information: dict[str, Any] | None) -> dict:
+ return {
+ "workflow_run_id": "wr_x",
+ "overall_status": "completed",
+ "executed_block_labels": [label],
+ "current_url": "https://example.com/account",
+ "blocks": [
+ {
+ "label": label,
+ "block_type": "task_v2",
+ "status": "completed",
+ "extracted_data": {
+ "task_id": "t1",
+ "status": "completed",
+ "extracted_information": extracted_information,
+ "downloaded_files": None,
+ "downloaded_file_urls": None,
+ },
+ }
+ ],
+ "registered_output_parameter_values": [],
+ }
+
+
+def test_backing_backs_contract_less_task_envelope_extracted_information() -> None:
+ ctx = _task_envelope_ctx()
+ run_data = _task_envelope_run_data("run_task", {"job_title": "Staff Engineer"})
+
+ backing = _backing_for(ctx, [_marked_output_criterion("output.job_title")], run_data)
+
+ assert backing == {"c1": True}
+
+
+def test_backing_does_not_back_metadata_only_task_envelope() -> None:
+ ctx = _task_envelope_ctx()
+ run_data = _task_envelope_run_data("run_task", None)
+
+ backing = _backing_for(ctx, [_marked_output_criterion("output.job_title")], run_data)
+
+ assert backing == {"c1": False}
+
+
+def _mixed_authored_and_envelope_ctx() -> CopilotContext:
+ ctx = _run_ctx()
+ ctx.last_workflow = SimpleNamespace(
+ workflow_definition=SimpleNamespace(
+ blocks=[
+ SimpleNamespace(label="extract_bill", block_type="code"),
+ SimpleNamespace(label="run_task", block_type="task_v2"),
+ ]
+ )
+ )
+ ctx.last_workflow_yaml = textwrap.dedent(
+ """
+ workflow_definition:
+ blocks:
+ - block_type: code
+ label: extract_bill
+ code: |
+ return {"job_title": "authored placeholder"}
+ - block_type: task_v2
+ label: run_task
+ """
+ ).strip()
+ return ctx
+
+
+def test_backing_does_not_cross_back_authored_marker_from_foreign_task_envelope() -> None:
+ ctx = _mixed_authored_and_envelope_ctx()
+ run_data = {
+ "workflow_run_id": "wr_x",
+ "overall_status": "completed",
+ "executed_block_labels": ["extract_bill", "run_task"],
+ "current_url": "https://example.com/account",
+ "blocks": [
+ {
+ "label": "extract_bill",
+ "block_type": "code",
+ "status": "completed",
+ "extracted_data": {"note": "done"},
+ },
+ {
+ "label": "run_task",
+ "block_type": "task_v2",
+ "status": "completed",
+ "extracted_data": {
+ "task_id": "t1",
+ "status": "completed",
+ "extracted_information": {"job_title": "Staff Engineer"},
+ "downloaded_files": None,
+ "downloaded_file_urls": None,
+ },
+ },
+ ],
+ "registered_output_parameter_values": [],
+ }
+
+ backing = _backing_for(ctx, [_marked_output_criterion("output.job_title")], run_data)
+
+ assert backing == {"c1": False}
+
+
+def test_backing_does_not_back_unrelated_field() -> None:
+ ctx = _backing_ctx('{"doc_status": "Delivered"}')
+ run_data = _registered_output_result({"doc_status": "Delivered"})["data"]
+
+ backing = _backing_for(ctx, [_marked_output_criterion("output.unrelated_field")], run_data)
+
+ assert backing == {"c1": False}
+
+
+def _two_contract_less_task_envelope_ctx() -> CopilotContext:
+ ctx = _run_ctx()
+ ctx.last_workflow = SimpleNamespace(
+ workflow_definition=SimpleNamespace(
+ blocks=[
+ SimpleNamespace(label="run_task", block_type="task_v2"),
+ SimpleNamespace(label="run_task_2", block_type="task_v2"),
+ ]
+ )
+ )
+ ctx.last_workflow_yaml = textwrap.dedent(
+ """
+ workflow_definition:
+ blocks:
+ - block_type: task_v2
+ label: run_task
+ - block_type: task_v2
+ label: run_task_2
+ """
+ ).strip()
+ return ctx
+
+
+def test_backing_fails_closed_across_ambiguous_contract_less_task_envelopes() -> None:
+ # The criterion's real producer (run_task) emits nothing at the path; an unrelated task
+ # envelope (run_task_2) coincidentally emits it. With no authored contract naming the
+ # producer and more than one candidate envelope, the emission is unattributable and must
+ # fail closed rather than pool the coincidental match.
+ ctx = _two_contract_less_task_envelope_ctx()
+ run_data = {
+ "workflow_run_id": "wr_x",
+ "overall_status": "completed",
+ "executed_block_labels": ["run_task", "run_task_2"],
+ "current_url": "https://example.com/account",
+ "blocks": [
+ {
+ "label": "run_task",
+ "block_type": "task_v2",
+ "status": "completed",
+ "extracted_data": {
+ "task_id": "t1",
+ "status": "completed",
+ "extracted_information": None,
+ "downloaded_files": None,
+ "downloaded_file_urls": None,
+ },
+ },
+ {
+ "label": "run_task_2",
+ "block_type": "task_v2",
+ "status": "completed",
+ "extracted_data": {
+ "task_id": "t2",
+ "status": "completed",
+ "extracted_information": {"job_title": "Staff Engineer"},
+ "downloaded_files": None,
+ "downloaded_file_urls": None,
+ },
+ },
+ ],
+ "registered_output_parameter_values": [],
+ }
+
+ backing = _backing_for(ctx, [_marked_output_criterion("output.job_title")], run_data)
+
+ assert backing == {"c1": False}
+
+
+def test_emission_evidence_gates_hollow_top_level_output() -> None:
+ # A hollow top-level ``*_output`` payload must not enter the emission view as backing
+ # evidence — the same meaningfulness gate _build_run_evidence_snapshot applies.
+ ctx = _backing_ctx('{"doc_status": "Delivered"}')
+ run_data = {
+ "workflow_run_id": "wr_x",
+ "overall_status": "completed",
+ "executed_block_labels": [],
+ "current_url": "https://example.com/account",
+ "blocks": [],
+ "output": {"extract_bill_output": {"extracted_information": None}},
+ "registered_output_parameter_values": [],
+ }
+
+ block_outputs, _sources, _types, _envs = _floor_rekeyed_emission_evidence(ctx, run_data)
+
+ assert "extract_bill_output" not in block_outputs
+
+
+def test_backing_empty_artifact_is_unbacked() -> None:
+ ctx = _backing_ctx('{"doc_status": "Delivered"}')
+
+ backing = _backing_for(ctx, [_marked_output_criterion("output.doc_status")], None)
+
+ assert backing == {"c1": False}
+
+
+def test_emission_evidence_excludes_page_observation_sources() -> None:
+ ctx = _backing_ctx('{"doc_status": "Delivered"}')
+ run_data = _registered_output_result({"doc_status": "Delivered"})["data"]
+
+ _, sources, _types, _envs = _floor_rekeyed_emission_evidence(ctx, run_data)
+
+ assert sources
+ assert set(sources.values()) <= {"runtime_output", "registered_output_parameter"}
+
+
+def _emission_result(
+ *,
+ backed: dict[str, bool],
+ paths: dict[str, str] | None = None,
+ marked: tuple[str, ...] = ("c1",),
+ contingent: tuple[str, ...] = (),
+ structural_unfired: tuple[str, ...] = (),
+ extra_verdicts: tuple[CriterionVerdict, ...] = (),
+) -> CompletionVerificationResult:
+ verdicts = [
+ CriterionVerdict(
+ criterion_id=cid, state="satisfied", reason_code="evidence_confirms", evidence_ref="observed_end_state_url"
+ )
+ for cid in marked
+ ]
+ criterion_ids = [*marked, *(verdict.criterion_id for verdict in extra_verdicts)]
+ return CompletionVerificationResult(
+ status="evaluated",
+ criterion_ids=criterion_ids,
+ verdicts=[*verdicts, *extra_verdicts],
+ floor_rekeyed_criterion_ids=list(marked),
+ floor_rekeyed_output_path_by_criterion_id=dict(paths or {}),
+ floor_rekeyed_backed_by_criterion_id=dict(backed),
+ contingent_criterion_ids=list(contingent),
+ structural_unfired_criterion_ids=list(structural_unfired),
+ )
+
+
+def test_floor_rekeyed_emission_withhold_engages_when_marker_unbacked() -> None:
+ withhold = floor_rekeyed_emission_withhold(
+ _emission_result(backed={"c1": False}, paths={"c1": "output.doc_status"})
+ )
+
+ assert isinstance(withhold, FloorRekeyedEmissionWithhold)
+ assert withhold.criterion_ids == ("c1",)
+ assert withhold.unbacked_output_paths == ("output.doc_status",)
+
+
+def test_floor_rekeyed_emission_withhold_none_when_backed() -> None:
+ assert floor_rekeyed_emission_withhold(_emission_result(backed={"c1": True})) is None
+
+
+def test_floor_rekeyed_emission_withhold_missing_entry_fails_closed() -> None:
+ assert floor_rekeyed_emission_withhold(_emission_result(backed={})) is not None
+
+
+def test_floor_rekeyed_emission_withhold_ignores_empty_marker_set() -> None:
+ reach_state = _evaluated(("login", True))
+
+ assert floor_rekeyed_emission_withhold(reach_state) is None
+
+
+def test_floor_rekeyed_emission_withhold_excludes_structurally_unfired_contingent_marker() -> None:
+ result = _emission_result(
+ backed={},
+ marked=("c1",),
+ contingent=("c1",),
+ structural_unfired=("c1",),
+ extra_verdicts=(
+ CriterionVerdict(
+ criterion_id="ok", state="satisfied", reason_code="evidence_confirms", evidence_ref="observed_end_state"
+ ),
+ ),
+ )
+
+ assert result.is_fully_satisfied() is True
+ assert floor_rekeyed_emission_withhold(result) is None
+
+
+def test_floor_rekeyed_emission_withhold_keyed_on_marker_not_artifact_richness() -> None:
+ lean = _emission_result(backed={"c1": False})
+ rich = _emission_result(
+ backed={"c1": False},
+ extra_verdicts=(
+ CriterionVerdict(
+ criterion_id="ok", state="satisfied", reason_code="evidence_confirms", evidence_ref="observed_end_state"
+ ),
+ ),
+ )
+
+ assert floor_rekeyed_emission_withhold(lean) is not None
+ assert floor_rekeyed_emission_withhold(rich) is not None
+
+
+def test_floor_rekeyed_emission_lane_fields_fire_on_satisfied_and_unsatisfied_results() -> None:
+ satisfied = _emission_result(backed={"c1": False}, paths={"c1": "output.doc_status"})
+ unsatisfied = CompletionVerificationResult(
+ status="evaluated",
+ criterion_ids=["c1"],
+ verdicts=[CriterionVerdict(criterion_id="c1", state="unsatisfied", reason_code="no_evidence")],
+ floor_rekeyed_criterion_ids=["c1"],
+ floor_rekeyed_backed_by_criterion_id={"c1": False},
+ )
+
+ satisfied_fields = floor_rekeyed_emission_lane_fields(satisfied)
+ unsatisfied_fields = floor_rekeyed_emission_lane_fields(unsatisfied)
+
+ assert satisfied_fields is not None and satisfied_fields["engaged"] is True
+ assert unsatisfied_fields is not None and unsatisfied_fields["engaged"] is False
+
+
+def test_floor_rekeyed_emission_shallow_bound_but_unbacked_withholds() -> None:
+ ctx = _ctx_with_blocks("code")
+ verification = _floor_rekeyed_result(
+ _floored_run_plane_verdicts(corroborator_source="independent_page_evidence"),
+ ["deliverable"],
+ {"deliverable": "output.document_name"},
+ backed_by_id={"deliverable": False},
+ )
+
+ with capture_logs() as logs:
+ recorded = _record_run_blocks_result(
+ ctx,
+ _registered_output_result({"document_name": "Resale certificate"}),
+ completion_verification=verification,
+ )
+
+ assert recorded is not None
+ assert recorded.verdict == "not_evaluated"
+ assert ctx.delivered_unverified_terminal is True
+ lane = [entry for entry in logs if entry.get("event") == "copilot.completion.floor_rekeyed_emission_lane"]
+ assert lane and lane[0]["engaged"] is True
assert not any(entry.get("event") == "copilot.completion.floor_rekeyed_deliverable_credit" for entry in logs)
+def test_registered_output_meaningfulness_gate_drops_hollow_envelope() -> None:
+ envelope_type = next(iter(_TASK_ENVELOPE_BLOCK_TYPES))
+ ctx = _ctx_with_blocks("code")
+ hollow = _registered_output_result(
+ {
+ "task_id": "t",
+ "status": "completed",
+ "extracted_information": None,
+ "downloaded_files": None,
+ "downloaded_file_urls": None,
+ },
+ block_type=envelope_type,
+ )
+
+ snapshot = _build_run_evidence_snapshot(ctx, hollow)
+
+ assert "bill_statement" not in snapshot.block_outputs
+
+
+def test_registered_output_meaningfulness_gate_keeps_real_scalar() -> None:
+ ctx = _ctx_with_blocks("code")
+ real = _registered_output_result({"doc_status": "Delivered"})
+
+ snapshot = _build_run_evidence_snapshot(ctx, real)
+
+ assert snapshot.block_outputs.get("bill_statement") == {"doc_status": "Delivered"}
+
+
def test_zero_requested_output_criteria_withholds_verified_success() -> None:
ctx = _ctx_with_blocks("code")
diff --git a/tests/unit/test_forge_log_foreign_tracebacks.py b/tests/unit/test_forge_log_foreign_tracebacks.py
new file mode 100644
index 000000000..41c29ef06
--- /dev/null
+++ b/tests/unit/test_forge_log_foreign_tracebacks.py
@@ -0,0 +1,73 @@
+from __future__ import annotations
+
+import io
+import json
+import logging
+from collections.abc import Iterator
+
+import pytest
+import structlog
+
+from skyvern.config import settings
+from skyvern.forge.sdk.forge_log import setup_logger
+
+
+def json_stream(monkeypatch: pytest.MonkeyPatch) -> Iterator[io.StringIO]:
+ monkeypatch.setattr(settings, "JSON_LOGGING", True)
+ setup_logger()
+ handler = logging.getLogger().handlers[0]
+ assert isinstance(handler, logging.StreamHandler)
+ stream = io.StringIO()
+ previous = handler.setStream(stream)
+ try:
+ yield stream
+ finally:
+ handler.setStream(previous)
+
+
+def _raise_through_wrapper() -> None:
+ def async_wrapper() -> None:
+ raise ValueError("kaboom from an activity")
+
+ async_wrapper()
+
+
+def test_foreign_stdlib_exception_renders_single_structured_line(json_stream: io.StringIO) -> None:
+ """temporalio/asyncio emit stdlib records; their exc_info must collapse to one JSON entry."""
+ logger = logging.getLogger("temporalio.activity")
+ try:
+ _raise_through_wrapper()
+ except ValueError:
+ logger.warning("Completing activity as failed", exc_info=True)
+
+ lines = json_stream.getvalue().strip().splitlines()
+ assert len(lines) == 1
+ record = json.loads(lines[0])
+
+ assert record["logger"] == "temporalio.activity"
+ assert "exc_info" not in record # the raw (traceback, ...) tuple is never dumped
+ assert "<traceback object at" not in json_stream.getvalue()
+ assert "Traceback (most recent call last)" in record["exception"]
+ assert "ValueError: kaboom from an activity" in record["exception"]
+ assert "async_wrapper" in record["exception"]
+ assert record["error_type"] == "builtins.ValueError"
+ assert record["error_category"] == "ERROR"
+ assert record["exception_hash"]
+
+
+def test_native_structlog_exception_not_double_processed(json_stream: io.StringIO) -> None:
+ logger = structlog.get_logger("skyvern.foreign_traceback_test")
+ try:
+ _raise_through_wrapper()
+ except ValueError:
+ logger.exception("native boom")
+
+ lines = json_stream.getvalue().strip().splitlines()
+ assert len(lines) == 1
+ record = json.loads(lines[0])
+
+ assert "exc_info" not in record
+ assert "Traceback (most recent call last)" in record["exception"]
+ assert record["exception"].count("Traceback (most recent call last)") == 1
+ assert record["error_type"] == "builtins.ValueError"