Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
ci: restore all-files pre-commit checks on pull requests (#7328)
Aaron Perez9e0221f586bb
diff --git a/skyvern/forge/sdk/copilot/code_block_synthesis.py b/skyvern/forge/sdk/copilot/code_block_synthesis.pyindex eb731f629..905bf89b3 100644--- a/skyvern/forge/sdk/copilot/code_block_synthesis.py+++ b/skyvern/forge/sdk/copilot/code_block_synthesis.py@@ -1421,6 +1421,35 @@ def _array_prefix(binding: LiveReadBinding) -> tuple[tuple[str, bool], ...]:return ()+def _key_value_scalar_read_statements(binding: LiveReadBinding, variable: str, *, guard_empty: bool) -> list[str]:+ container = f"page.locator({json.dumps(binding.selector)})"+ target = f"{container}.nth({binding.selector_index})"+ children = f'{target}.locator(":scope > *")'+ statements = [+ f"if await {container}.count() != {binding.selector_count}:",+ f'{_INDENT}raise ValueError("Observed scalar selector cardinality changed")',+ f"if not await {target}.is_visible():",+ f'{_INDENT}raise ValueError("Observed scalar relation is no longer visible")',+ f"if await {children}.count() != {binding.child_count}:",+ f'{_INDENT}raise ValueError("Observed scalar direct-child shape changed")',+ f"if not await {children}.nth(0).is_visible():",+ f'{_INDENT}raise ValueError("Observed scalar label is no longer visible")',+ f"if (await {children}.nth(0).inner_text()).strip() != {json.dumps(binding.relation_label)}:",+ f'{_INDENT}raise ValueError("Observed scalar label changed")',+ f"if not await {children}.nth({binding.child_index}).is_visible():",+ f'{_INDENT}raise ValueError("Observed scalar value is no longer visible")',+ f"{variable} = (await {children}.nth({binding.child_index}).inner_text()).strip()",+ ]+ if guard_empty:+ statements.extend(+ [+ f"if not {variable}:",+ f'{_INDENT}raise ValueError("Observed scalar value is empty")',+ ]+ )+ return statements++def synthesize_extraction_suffix(plan: RequestedOutputExtractionPlan) -> SynthesizedExtractionSuffix | None:if not plan.live_reads:return None@@ -1429,26 +1458,7 @@ def synthesize_extraction_suffix(plan: RequestedOutputExtractionPlan) -> Synthesscalar_bindings = [binding for binding in plan.live_reads if binding.kind == LiveReadKind.KEY_VALUE]for index, binding in enumerate(scalar_bindings):variable = f"_extraction_value_{index}"- container = f"page.locator({json.dumps(binding.selector)})"- target = f"{container}.nth({binding.selector_index})"- children = f'{target}.locator(":scope > *")'- lines.extend(- [- f"if await {container}.count() != {binding.selector_count}:",- f'{_INDENT}raise ValueError("Observed scalar selector cardinality changed")',- f"if not await {target}.is_visible():",- f'{_INDENT}raise ValueError("Observed scalar relation is no longer visible")',- f"if await {children}.count() != {binding.child_count}:",- f'{_INDENT}raise ValueError("Observed scalar direct-child shape changed")',- f"if not await {children}.nth(0).is_visible():",- f'{_INDENT}raise ValueError("Observed scalar label is no longer visible")',- f"if (await {children}.nth(0).inner_text()).strip() != {json.dumps(binding.relation_label)}:",- f'{_INDENT}raise ValueError("Observed scalar label changed")',- f"if not await {children}.nth({binding.child_index}).is_visible():",- f'{_INDENT}raise ValueError("Observed scalar value is no longer visible")',- f"{variable} = (await {children}.nth({binding.child_index}).inner_text()).strip()",- ]- )+ lines.extend(_key_value_scalar_read_statements(binding, variable, guard_empty=False))_set_return_expression(return_root, output_path_segments(binding.output_path), variable)table_groups: dict[tuple[str, int, tuple[tuple[str, bool], ...]], list[LiveReadBinding]] = {}@@ -1513,6 +1523,139 @@ def synthesize_extraction_suffix(plan: RequestedOutputExtractionPlan) -> Synthesreturn SynthesizedExtractionSuffix(code=code, fingerprint=hashlib.sha256(fingerprint_material.encode()).hexdigest())+_ENVELOPE_SCALAR_VAR_BASE = "_envelope_value"+++@dataclass(frozen=True, slots=True)+class ProducedStaticReturnEnvelope:+ code: str+ keyed_paths: tuple[str, ...]+++def _snippet_scope_returns(statements: Sequence[ast.stmt]) -> list[ast.Return]:+ found: list[ast.Return] = []+ for statement in statements:+ if isinstance(statement, ast.Return):+ found.append(statement)+ if isinstance(statement, (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef)):+ continue+ for child in ast.iter_child_nodes(statement):+ if isinstance(child, ast.stmt):+ found.extend(_snippet_scope_returns([child]))+ elif isinstance(child, (ast.ExceptHandler, ast.match_case)):+ found.extend(_snippet_scope_returns(child.body))+ return found+++def _bound_or_referenced_identifiers(tree: ast.AST) -> set[str]:+ names: set[str] = set()+ for node in ast.walk(tree):+ if isinstance(node, ast.Name):+ names.add(node.id)+ elif isinstance(node, ast.arg):+ names.add(node.arg)+ elif isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef)):+ names.add(node.name)+ elif isinstance(node, ast.ExceptHandler):+ if node.name:+ names.add(node.name)+ elif isinstance(node, ast.alias):+ if node.asname:+ names.add(node.asname)+ elif isinstance(node, (ast.Global, ast.Nonlocal)):+ names.update(node.names)+ elif isinstance(node, ast.MatchAs):+ if node.name:+ names.add(node.name)+ return names+++def produce_covered_static_return_envelope(+ code: str,+ *,+ plan: RequestedOutputExtractionPlan | None,+ scalar_required_paths: set[str],+ declaration_paths: set[str],+ download_required_paths: set[str],+ expects_download: bool,+) -> ProducedStaticReturnEnvelope | None:+ """Author the unique terminal keyed return covering every scout-bound scalar path, plus+ None-default declaration children and the resolved download descriptor for the mixed shape.++ Fail-closed: emits only when it owns the sole terminal return (zero returns, or the single+ generator-owned download-idiom return), when every scalar path is plan-bound, and when no+ minted local collides with an identifier already in the block."""+ if download_required_paths and not expects_download:+ return None+ if not scalar_required_paths:+ return None+ if plan is None:+ return None+ stripped_code = textwrap.dedent(code).strip()+ if not stripped_code:+ return None+ try:+ tree = ast.parse(stripped_code)+ except SyntaxError:+ return None++ bindings_by_path = {+ binding.output_path: binding for binding in plan.live_reads if binding.kind == LiveReadKind.KEY_VALUE+ }+ ordered_scalar_paths = sorted(scalar_required_paths)+ if any(path not in bindings_by_path for path in ordered_scalar_paths):+ return None++ minted_vars = [f"{_ENVELOPE_SCALAR_VAR_BASE}_{index}" for index in range(len(ordered_scalar_paths))]+ existing_names = _bound_or_referenced_identifiers(tree)+ if any(var in existing_names for var in minted_vars):+ return None++ returns = _snippet_scope_returns(tree.body)+ download_descriptor_key = ""+ download_descriptor_expr = ""+ if download_required_paths:+ if len(returns) != 1 or returns[0] not in tree.body:+ return None+ idiom_return = returns[0]+ if idiom_return.col_offset != 0:+ return None+ if not isinstance(idiom_return.value, ast.Dict) or len(idiom_return.value.keys) != 1:+ return None+ descriptor_key_node = idiom_return.value.keys[0]+ if not isinstance(descriptor_key_node, ast.Constant) or not isinstance(descriptor_key_node.value, str):+ return None+ descriptor_value = idiom_return.value.values[0]+ if not isinstance(descriptor_value, (ast.Name, ast.Attribute)):+ return None+ download_descriptor_key = descriptor_key_node.value+ download_descriptor_expr = ast.unparse(descriptor_value)+ preserved_lines = stripped_code.splitlines()[: idiom_return.lineno - 1]+ else:+ if returns:+ return None+ preserved_lines = stripped_code.splitlines()++ return_root = _ExtractionReturnNode()+ scalar_statements: list[str] = []+ for path, variable in zip(ordered_scalar_paths, minted_vars):+ scalar_statements.extend(_key_value_scalar_read_statements(bindings_by_path[path], variable, guard_empty=True))+ _set_return_expression(return_root, output_path_segments(path), variable)+ for path in sorted(declaration_paths):+ _set_return_expression(return_root, output_path_segments(path), "None")+ if download_descriptor_key:+ return_root.children.setdefault(+ download_descriptor_key, _ExtractionReturnNode()+ ).value_expression = download_descriptor_expr++ body_lines = list(preserved_lines)+ body_lines.extend(scalar_statements)+ body_lines.append(f"return {_return_node_expression(return_root)}")+ produced_code = "\n".join(body_lines).strip() + "\n"+ keyed_paths = tuple(sorted(scalar_required_paths | declaration_paths))+ return ProducedStaticReturnEnvelope(code=produced_code, keyed_paths=keyed_paths)++def _trajectory_contains_reveal(trajectory: Sequence[Mapping[str, Any]], plan: RequestedOutputExtractionPlan) -> bool:return any(str(interaction.get("tool_name") or "") == "click"diff --git a/skyvern/forge/sdk/copilot/completion_criteria_store.py b/skyvern/forge/sdk/copilot/completion_criteria_store.pyindex af1738fd6..ad2f5556d 100644--- a/skyvern/forge/sdk/copilot/completion_criteria_store.py+++ b/skyvern/forge/sdk/copilot/completion_criteria_store.py@@ -136,6 +136,7 @@ def criteria_to_json(criteria: tuple[CompletionCriterion, ...] | list[Completion"contingent_on": criterion.contingent_on,"contingent_antecedent_output_path": criterion.contingent_antecedent_output_path,"deliverable_kind": criterion.deliverable_kind,+ "declared_deliverable_kind": criterion.declared_deliverable_kind,"implicit": criterion.implicit,"method_mandated": criterion.method_mandated,"level": criterion.level,@@ -216,6 +217,7 @@ def criteria_from_json(raw: Any) -> tuple[CompletionCriterion, ...]:contingent_on=contingent_on,contingent_antecedent_output_path=contingent_antecedent_output_path,deliverable_kind=_normalize_deliverable_kind(item.get("deliverable_kind")),+ declared_deliverable_kind=_normalize_deliverable_kind(item.get("declared_deliverable_kind")),implicit=bool(item.get("implicit")),method_mandated=bool(item.get("method_mandated")),level=level if isinstance(level, str) and level in _CRITERION_LEVELS else "run", # type: ignore[arg-type]@@ -243,7 +245,8 @@ def _criterion_reconcile_key(criterion: CompletionCriterion) -> str:contingent_key = criterion.contingent_on or ""contingent_path_key = criterion.contingent_antecedent_output_path or ""deliverable_kind_key = (- f"{criterion.deliverable_kind or ''}\x1fmint_degrade:{criterion.mint_degrade or ''}"+ f"{criterion.deliverable_kind or ''}\x1fdeclared:{criterion.declared_deliverable_kind or ''}"+ f"\x1fmint_degrade:{criterion.mint_degrade or ''}"f"\x1fjudgment:{judgment_truth_condition_key(criterion.judgment_truth_condition)}")expected_output_value_key = typed_expected_output_value_key(criterion.expected_output_value)diff --git a/skyvern/forge/sdk/copilot/completion_output_grounding.py b/skyvern/forge/sdk/copilot/completion_output_grounding.pyindex 85229a3cb..593f22de2 100644--- a/skyvern/forge/sdk/copilot/completion_output_grounding.py+++ b/skyvern/forge/sdk/copilot/completion_output_grounding.py@@ -4,7 +4,8 @@ import astimport jsonimport reimport textwrap-from collections.abc import Mapping+from collections.abc import Collection, Mapping+from dataclasses import replacefrom typing import Any, Literal, Protocolimport structlog@@ -22,6 +23,7 @@ from skyvern.forge.sdk.copilot.request_policy import (lookup_requested_output_path_alias,schema_output_path_aliases_from_criteria,)+from skyvern.forge.sdk.copilot.task_output_envelope import _TASK_ENVELOPE_BLOCK_TYPES, _TASK_OUTPUT_PAYLOAD_FIELDSfrom skyvern.utils.yaml_loader import safe_load_no_datesLOG = structlog.get_logger()@@ -289,6 +291,92 @@ def grade_requested_output_criteria(return verdicts+def floor_rekeyed_path_backing(+ copilot_ctx: _GroundingCtx,+ criteria: list[CompletionCriterion],+ emission_block_outputs: Mapping[str, Any],+ emission_block_output_sources: Mapping[str, EvidenceSourceKind],+ emission_block_types: Mapping[str, str | None],+ runtime_envelope_labels: Collection[str] = frozenset(),+) -> dict[str, bool]:+ """Which floor-rekeyed markers have a meaningful runtime/registered value at their original+ ``floor_rekeyed_from_path`` in an emission-only evidence view (page and artifact observations+ excluded by construction), resolved with the requested-output grader's accepted-producer-label,+ alias, and projection machinery. The rule is identical whether or not the artifact emitted+ anything: absent emission reads as unbacked."""+ marked = [criterion for criterion in criteria if criterion.requested_output_floor_rekeyed]+ if not marked:+ return {}+ effective_criteria = [+ replace(criterion, output_path=criterion.floor_rekeyed_from_path)+ for criterion in marked+ if criterion.floor_rekeyed_from_path+ ]+ aliases = schema_output_path_aliases_from_criteria(effective_criteria)+ metadata = _grounding_code_artifact_metadata(copilot_ctx)+ raw_authored_paths_by_label = _authored_output_contract_paths_by_label(copilot_ctx, metadata=metadata)+ authored_paths_by_label = _authored_output_contract_paths_by_label(copilot_ctx, aliases, metadata=metadata)+ independent_labels = {+ label for label, source in emission_block_output_sources.items() if source in _INDEPENDENT_EVIDENCE_SOURCES+ }+ envelope_labels = {+ label+ for label in emission_block_outputs+ if (emission_block_types.get(label) or "").upper() in _TASK_ENVELOPE_BLOCK_TYPES+ }+ backing: dict[str, bool] = {}+ for criterion in marked:+ path = _normalize_output_path(criterion.floor_rekeyed_from_path)+ backing[criterion.id] = _floor_rekeyed_path_emission_present(+ path,+ emission_block_outputs,+ authored_paths_by_label,+ raw_authored_paths_by_label,+ independent_labels,+ envelope_labels,+ runtime_envelope_labels,+ )+ return backing+++def _floor_rekeyed_path_emission_present(+ path: str,+ block_outputs: Mapping[str, Any],+ authored_paths_by_label: dict[str, set[str]],+ raw_authored_paths_by_label: dict[str, set[str]],+ independent_labels: set[str],+ envelope_labels: set[str],+ runtime_envelope_labels: Collection[str],+) -> bool:+ if not path:+ return False+ accepted_labels = {label for label, paths in authored_paths_by_label.items() if path in paths}+ if accepted_labels:+ candidate_labels = accepted_labels+ accepted_authored_paths = set().union(*(authored_paths_by_label[label] for label in accepted_labels))+ accepted_raw_authored_paths = set().union(+ *(raw_authored_paths_by_label.get(label, set()) for label in accepted_labels)+ )+ projection_roots = _runtime_projection_roots(accepted_authored_paths | accepted_raw_authored_paths)+ if accepted_labels & envelope_labels:+ projection_roots |= set(_TASK_OUTPUT_PAYLOAD_FIELDS)+ elif envelope_labels:+ if len(runtime_envelope_labels) > 1:+ # No authored contract names the producer, so with more than one task-envelope block the+ # emission is unattributable and a coincidental match in an unrelated block must not back+ # it. A single envelope block is the unambiguous producer.+ return False+ candidate_labels = envelope_labels+ projection_roots = set(_TASK_OUTPUT_PAYLOAD_FIELDS)+ else:+ return False+ accepted_runtime_output_items = _accepted_runtime_outputs(block_outputs, candidate_labels, independent_labels)+ if not accepted_runtime_output_items:+ return False+ match_state, _ref = _runtime_output_path_presence(dict(accepted_runtime_output_items), path, projection_roots)+ return match_state == "present"++def _carrier_confirmation(criterion: CompletionCriterion,path: str,diff --git a/skyvern/forge/sdk/copilot/completion_verification.py b/skyvern/forge/sdk/copilot/completion_verification.pyindex 2bddf38b6..9036b0a4a 100644--- a/skyvern/forge/sdk/copilot/completion_verification.py+++ b/skyvern/forge/sdk/copilot/completion_verification.py@@ -116,6 +116,7 @@ class CompletionVerificationResult:degraded_criterion_ids: list[str] = field(default_factory=list)floor_rekeyed_criterion_ids: list[str] = field(default_factory=list)floor_rekeyed_output_path_by_criterion_id: dict[str, str] = field(default_factory=dict)+ floor_rekeyed_backed_by_criterion_id: dict[str, bool] = field(default_factory=dict)contingent_degraded_criterion_ids: list[str] = field(default_factory=list)requested_output_criteria_count: int = 0@@ -2218,6 +2219,7 @@ def combine_verification_results(degraded_ids: list[str] = []floor_rekeyed_ids: list[str] = []floor_rekeyed_path_by_id: dict[str, str] = {}+ floor_rekeyed_backed_by_id: dict[str, bool] = {}contingent_degraded_ids: list[str] = []if run_result is not None:contingent_ids = list(dict.fromkeys([*contingent_ids, *run_result.contingent_criterion_ids]))@@ -2229,6 +2231,7 @@ def combine_verification_results(degraded_ids = list(dict.fromkeys([*degraded_ids, *run_result.degraded_criterion_ids]))floor_rekeyed_ids = list(dict.fromkeys([*floor_rekeyed_ids, *run_result.floor_rekeyed_criterion_ids]))floor_rekeyed_path_by_id.update(run_result.floor_rekeyed_output_path_by_criterion_id)+ floor_rekeyed_backed_by_id.update(run_result.floor_rekeyed_backed_by_criterion_id)contingent_degraded_ids = list(dict.fromkeys([*contingent_degraded_ids, *run_result.contingent_degraded_criterion_ids]))@@ -2244,6 +2247,7 @@ def combine_verification_results(degraded_criterion_ids=degraded_ids,floor_rekeyed_criterion_ids=floor_rekeyed_ids,floor_rekeyed_output_path_by_criterion_id=floor_rekeyed_path_by_id,+ floor_rekeyed_backed_by_criterion_id=floor_rekeyed_backed_by_id,contingent_degraded_criterion_ids=contingent_degraded_ids,requested_output_criteria_count=requested_output_criteria_count,)@@ -2480,6 +2484,86 @@ def carry_floor_rekeyed_criterion_ids()+def carry_floor_rekeyed_path_backing(+ result: CompletionVerificationResult,+ backing_by_criterion_id: Mapping[str, bool],+) -> CompletionVerificationResult:+ if not backing_by_criterion_id and not result.floor_rekeyed_backed_by_criterion_id:+ return result+ merged = dict(result.floor_rekeyed_backed_by_criterion_id)+ merged.update(backing_by_criterion_id)+ return replace(result, floor_rekeyed_backed_by_criterion_id=merged)+++@dataclass(frozen=True)+class FloorRekeyedEmissionWithhold:+ criterion_ids: tuple[str, ...]+ unbacked_output_paths: tuple[str, ...]+ backed_output_paths: tuple[str, ...]+++def floor_rekeyed_effective_marked_ids(result: CompletionVerificationResult) -> list[str]:+ """Floor-rekeyed markers that required an emission this run: the marked set minus the+ structurally-unfired contingent (abstained) markers, which needed no emission to fire."""+ abstained = result.abstained_criterion_ids()+ return [criterion_id for criterion_id in result.floor_rekeyed_criterion_ids if criterion_id not in abstained]+++def floor_rekeyed_emission_withhold(+ result: CompletionVerificationResult | None,+) -> FloorRekeyedEmissionWithhold | None:+ """A fully satisfied run whose floor-rekeyed emission markers lack a meaningful runtime value at+ their original path delivered no emission that corroboration can substitute for. Keyed on the typed+ marker id-set (never payload emptiness) so never-minted reach-state and structurally-unfired+ contingent markers are untouched, and a missing backing entry reads as unbacked to fail closed."""+ if result is None or result.status != "evaluated" or not result.is_fully_satisfied():+ return None+ effective_ids = floor_rekeyed_effective_marked_ids(result)+ if not effective_ids:+ return None+ unbacked_ids: list[str] = []+ unbacked_paths: list[str] = []+ backed_paths: list[str] = []+ for criterion_id in effective_ids:+ path = result.floor_rekeyed_output_path_by_criterion_id.get(criterion_id, "")+ if result.floor_rekeyed_backed_by_criterion_id.get(criterion_id, False):+ backed_paths.append(path)+ else:+ unbacked_ids.append(criterion_id)+ unbacked_paths.append(path)+ if not unbacked_ids:+ return None+ return FloorRekeyedEmissionWithhold(+ criterion_ids=tuple(unbacked_ids),+ unbacked_output_paths=tuple(unbacked_paths),+ backed_output_paths=tuple(backed_paths),+ )+++def floor_rekeyed_emission_lane_fields(result: CompletionVerificationResult | None) -> dict[str, Any] | None:+ """Structured fingerprint fields for the floor-rekeyed emission lane, emitted whenever an+ evaluated result carries markers — on satisfied and unsatisfied results alike, independent of+ outcome."""+ if result is None or result.status != "evaluated" or not result.floor_rekeyed_criterion_ids:+ return None+ effective_ids = floor_rekeyed_effective_marked_ids(result)+ backed_ids = [cid for cid in effective_ids if result.floor_rekeyed_backed_by_criterion_id.get(cid, False)]+ unbacked_ids = [cid for cid in effective_ids if cid not in backed_ids]+ withhold = floor_rekeyed_emission_withhold(result)+ return {+ "criterion_ids": list(result.floor_rekeyed_criterion_ids),+ "effective_criterion_ids": list(effective_ids),+ "abstained_excluded_criterion_ids": sorted(set(result.floor_rekeyed_criterion_ids) - set(effective_ids)),+ "backed_criterion_ids": backed_ids,+ "unbacked_criterion_ids": unbacked_ids,+ "backed_output_paths": [result.floor_rekeyed_output_path_by_criterion_id.get(cid, "") for cid in backed_ids],+ "unbacked_output_paths": [+ result.floor_rekeyed_output_path_by_criterion_id.get(cid, "") for cid in unbacked_ids+ ],+ "engaged": withhold is not None,+ }++async def evaluate_completion_criteria(criteria: list[CompletionCriterion],snapshot: RunEvidenceSnapshot,diff --git a/skyvern/forge/sdk/copilot/enforcement.py b/skyvern/forge/sdk/copilot/enforcement.pyindex 69b8ee796..6604096c9 100644--- a/skyvern/forge/sdk/copilot/enforcement.py+++ b/skyvern/forge/sdk/copilot/enforcement.py@@ -2425,6 +2425,30 @@ def _requested_output_coverage_tokens(ctx: AgentContext) -> dict[str, frozenset[}+def _registered_download_deliverable_paths(ctx: AgentContext) -> set[str]:+ return {+ criterion.output_path+ for criterion in _pre_run_gated_completion_criteria(ctx)+ if criterion.declared_deliverable_kind == "registered_download" and criterion.output_path+ }+++def download_satisfied_requested_output_paths(ctx: AgentContext) -> set[str]:+ """Requested-output paths a reached download registration satisfies at runtime rather than a+ page-scalar read: the registered-download alias paths plus the paths the classifier declared as+ ``registered_download`` deliverables. Empty unless a download target with a captured selector+ was reached. Author-time seam classification only — it never credits scout coverage."""+ download = ctx.reached_download_target+ if download is None or not download.selector:+ return set()+ requested = _requested_output_paths_for_ctx(ctx)+ # The scout reads page scalars; it can never read a file that exists only once a download fires.+ # So a declared download kind on a path the scout DID cover is a classifier false positive, and the+ # path stays a live-read scalar. The canonical alias paths are download-registered by definition.+ declared = _registered_download_deliverable_paths(ctx) - set(ctx.scouted_output_covered_paths)+ return requested & (REGISTERED_DOWNLOAD_REQUESTED_OUTPUT_PATHS | declared)++def uncovered_requested_output_paths(ctx: AgentContext) -> set[str]:"""Requested-output paths not yet credited by scouted evidence. A path whose identifyingtokens are all generic (e.g. ``output.data``) is uncoverable by token match and is exempted,@@ -2433,10 +2457,7 @@ def uncovered_requested_output_paths(ctx: AgentContext) -> set[str]:if not requested:return set()tokens_by_path = _requested_output_coverage_tokens(ctx)- covered: set[str] = set(ctx.scouted_output_covered_paths)- download = ctx.reached_download_target- if download is not None and download.selector:- covered |= REGISTERED_DOWNLOAD_REQUESTED_OUTPUT_PATHS+ covered: set[str] = set(ctx.scouted_output_covered_paths) | download_satisfied_requested_output_paths(ctx)return {path for path in requested if path not in covered and tokens_by_path.get(path)}@@ -2457,6 +2478,26 @@ def requested_output_extraction_plan(ctx: AgentContext) -> RequestedOutputExtrac)+def requested_scalar_output_extraction_plan(ctx: AgentContext) -> RequestedOutputExtractionPlan | None:+ """Extraction plan over the page-scalar subset of requested outputs (requested minus the+ download-registered paths), for the mixed download+scalar shape whose download half is+ satisfied by execution registration rather than a static keyed read."""+ requested_paths = _requested_output_paths_for_ctx(ctx) - download_satisfied_requested_output_paths(ctx)+ if not requested_paths:+ return None+ labels_by_path: dict[str, tuple[str, ...]] = {}+ for criterion in _pre_run_gated_completion_criteria(ctx):+ if criterion.output_path in requested_paths and criterion.outcome.strip():+ labels_by_path.setdefault(criterion.output_path, ())+ labels_by_path[criterion.output_path] += (criterion.outcome.strip(),)+ if set(labels_by_path) != requested_paths:+ return None+ return derive_requested_output_extraction_plan(+ flow_evidence=ctx.flow_evidence,+ labels_by_path=labels_by_path,+ )++def requested_output_extraction_plan_changed(ctx: AgentContext, current: RequestedOutputExtractionPlan | None) -> bool:if current is None or len(ctx.flow_evidence) < 2:return False@@ -2598,10 +2639,11 @@ def synthesized_trajectory_is_goal_complete(ctx: AgentContext) -> bool:return Falseif _request_expects_unreached_download(ctx):return False- requested_paths = _requested_output_paths_for_ctx(ctx)- plan = requested_output_extraction_plan(ctx)- if requested_paths and (plan is None or not requested_paths.issubset(set(plan.requested_output_paths))):- return False+ scalar_paths = _requested_output_paths_for_ctx(ctx) - download_satisfied_requested_output_paths(ctx)+ if scalar_paths:+ plan = requested_scalar_output_extraction_plan(ctx)+ if plan is None or not scalar_paths.issubset(set(plan.requested_output_paths)):+ return Falseif _credential_flow_scout_gap_incomplete(ctx, ctx.scout_trajectory):return Falsereturn synthesized_trajectory_reaches_goal(ctx)diff --git a/skyvern/forge/sdk/copilot/request_policy.py b/skyvern/forge/sdk/copilot/request_policy.pyindex 3f7b8acad..32c2c280d 100644--- a/skyvern/forge/sdk/copilot/request_policy.py+++ b/skyvern/forge/sdk/copilot/request_policy.py@@ -281,6 +281,9 @@ class CompletionCriterion:contingent_on: str | None = Nonecontingent_antecedent_output_path: str | None = Nonedeliverable_kind: Literal["registered_download"] | None = None+ # Author-time seam signal only: unlike ``deliverable_kind`` it survives canonicalization onto+ # non-canonical output paths, so it is never rendered to the completion verifier.+ declared_deliverable_kind: Literal["registered_download"] | None = Noneimplicit: bool = Falsemethod_mandated: bool = False# "definition": a property of the workflow definition itself, graded against the@@ -925,6 +928,7 @@ def _parse_completion_criteria(raw: Any) -> list[CompletionCriterion]:contingent_on=contingent_on,contingent_antecedent_output_path=contingent_antecedent_output_path,deliverable_kind=deliverable_kind,+ declared_deliverable_kind=deliverable_kind,implicit=bool(item.get("implicit")),method_mandated=bool(item.get("method_mandated")),level=cast(CriterionLevel, level_raw)@@ -1560,6 +1564,7 @@ def _apply_requested_output_completion_criteria()metadata_by_output_path: dict[str, tuple[str | None, str | None, Literal["registered_download"] | None]] = {}+ declared_kind_by_output_path: dict[str, Literal["registered_download"]] = {}for criterion in policy.completion_criteria:if criterion.level == "definition" or criterion.method_mandated:continue@@ -1567,6 +1572,7 @@ def _apply_requested_output_completion_criteria(not criterion.contingent_onand not criterion.contingent_antecedent_output_pathand not criterion.deliverable_kind+ and not criterion.declared_deliverable_kind):continuefor field_name, output_path, _field_label in requested_specs:@@ -1574,6 +1580,8 @@ def _apply_requested_output_completion_criteria(deliverable_kind = (criterion.deliverable_kind if output_path in REGISTERED_DOWNLOAD_REQUESTED_OUTPUT_PATHS else None)+ if criterion.output_path == output_path and criterion.declared_deliverable_kind:+ declared_kind_by_output_path.setdefault(output_path, criterion.declared_deliverable_kind)metadata_by_output_path.setdefault(output_path,(@@ -1606,6 +1614,7 @@ def _apply_requested_output_completion_criteria(contingent_on=metadata_by_output_path.get(output_path, (None, None, None))[0],contingent_antecedent_output_path=metadata_by_output_path.get(output_path, (None, None, None))[1],deliverable_kind=metadata_by_output_path.get(output_path, (None, None, None))[2],+ declared_deliverable_kind=declared_kind_by_output_path.get(output_path),judgment_truth_condition=judgment_condition_by_output_path.get(output_path),)for _field_name, output_path, field_label in requested_specs@@ -1658,6 +1667,7 @@ def _apply_validation_classification_completion_criteria(policy: RequestPolicy)expected_output_shape=None,requested_output_evidence_source="runtime_output",deliverable_kind=None,+ declared_deliverable_kind=None,terminal_action_family=None,classification_output_key=output_key,expected_classification=expected,diff --git a/skyvern/forge/sdk/copilot/task_output_envelope.py b/skyvern/forge/sdk/copilot/task_output_envelope.pynew file mode 100644index 000000000..0c25c5d27--- /dev/null+++ b/skyvern/forge/sdk/copilot/task_output_envelope.py@@ -0,0 +1,35 @@+from __future__ import annotations++from skyvern.forge.sdk.workflow.models.block import BaseTaskBlock, Block, TaskV2Block+from skyvern.schemas.workflows import BlockType+++def _task_output_envelope_block_types() -> frozenset[str]:+ # Almost every envelope block extends BaseTaskBlock; TaskV2Block subclasses Block+ # directly but emits the same TaskOutput envelope, so seed the walk with both roots.+ block_types: set[str] = set()+ pending: list[type[Block]] = [BaseTaskBlock, TaskV2Block]+ while pending:+ cls = pending.pop()+ pending.extend(cls.__subclasses__())+ field = cls.model_fields.get("block_type")+ default = field.default if field is not None else None+ if isinstance(default, BlockType):+ block_types.add(default.value.upper())+ return frozenset(block_types)+++# Block types whose ``block.output`` is a ``TaskOutput.from_task()`` envelope rather than the raw+# payload, so meaningful-data and floor-rekeyed backing checks slice/project only+# ``_TASK_OUTPUT_PAYLOAD_FIELDS`` and always-populated metadata (task_id, status) reads no signal.+# Derived from the block subclass tree so a newly added task-backed block can't fall out of sync.+_TASK_ENVELOPE_BLOCK_TYPES: frozenset[str] = _task_output_envelope_block_types()+++# Payload fields inside a ``TaskOutput.from_task()`` envelope carrying real produced content; the+# rest is always-populated metadata.+_TASK_OUTPUT_PAYLOAD_FIELDS: tuple[str, ...] = (+ "extracted_information",+ "downloaded_files",+ "downloaded_file_urls",+)diff --git a/skyvern/forge/sdk/copilot/tools/_shared.py b/skyvern/forge/sdk/copilot/tools/_shared.pyindex 2b16ce82c..378c04a9f 100644--- a/skyvern/forge/sdk/copilot/tools/_shared.py+++ b/skyvern/forge/sdk/copilot/tools/_shared.py@@ -28,11 +28,14 @@ from skyvern.forge.sdk.copilot.composition_evidence import has_bounded_page_schefrom skyvern.forge.sdk.copilot.context import CopilotContextfrom skyvern.forge.sdk.copilot.enforcement import TOTAL_TIMEOUT_SECONDSfrom skyvern.forge.sdk.copilot.runtime import AgentContext+from skyvern.forge.sdk.copilot.task_output_envelope import (+ _TASK_ENVELOPE_BLOCK_TYPES,+ _TASK_OUTPUT_PAYLOAD_FIELDS,+)from skyvern.forge.sdk.copilot.tracing_setup import copilot_spanfrom skyvern.forge.sdk.copilot.turn_halt import stash_turn_halt_from_blocker_signalfrom skyvern.forge.sdk.copilot.turn_ownership import emit_blocker_signal_payloadfrom skyvern.forge.sdk.copilot.verification_evidence import WorkflowVerificationEvidence-from skyvern.forge.sdk.workflow.models.block import BaseTaskBlock, Block, TaskV2Blockfrom skyvern.forge.sdk.workflow.models.workflow import WorkflowRunStatusfrom skyvern.schemas.workflows import BlockTypefrom skyvern.utils.yaml_loader import safe_load_no_dates@@ -59,32 +62,6 @@ _DATA_PRODUCING_BLOCK_TYPES = frozenset({"EXTRACTION", "TEXT_PROMPT"})_OUTCOME_EVIDENCE_BLOCK_TYPES = frozenset({BlockType.EXTRACTION.value, BlockType.VALIDATION.value})-def _task_output_envelope_block_types() -> frozenset[str]:- # Almost every envelope block extends BaseTaskBlock; TaskV2Block subclasses Block- # directly but emits the same TaskOutput envelope, so seed the walk with both roots.- block_types: set[str] = set()- pending: list[type[Block]] = [BaseTaskBlock, TaskV2Block]- while pending:- cls = pending.pop()- pending.extend(cls.__subclasses__())- field = cls.model_fields.get("block_type")- default = field.default if field is not None else None- if isinstance(default, BlockType):- block_types.add(default.value.upper())- return frozenset(block_types)---# Block types whose ``block.output`` is a ``TaskOutput.from_task()`` envelope-# (schemas/tasks.py:TaskOutput) rather than the raw payload; the meaningful-data-# check slices these to ``_TASK_OUTPUT_PAYLOAD_FIELDS`` so always-populated-# envelope metadata (task_id, status, artifact IDs) can't read a reach-state task-# (login, navigation) that produced no content as meaningful. Envelope shape is-# orthogonal to data production, so a raw-value block (CODE, TEXT_PROMPT) whose-# value happens to carry a ``task_id`` is excluded here and passes through unsliced.-# Derived from the envelope block roots so a newly added task-backed block can't fall out of sync.-_TASK_ENVELOPE_BLOCK_TYPES: frozenset[str] = _task_output_envelope_block_types()--# Absolute upper bound on a single ``run_blocks`` tool invocation. Exists only# as a last-resort trip wire for runaway loops — progressing runs should never# approach this. The OpenAI Agents SDK wraps the tool in@@ -185,16 +162,6 @@ def _is_meaningful_extracted_data(extracted: Any) -> bool:return True-# Payload fields inside a ``TaskOutput.from_task()`` envelope-# (schemas/tasks.py:TaskOutput). Only these carry "did the block produce-# something useful?" signal; the rest (task_id, status, artifact IDs, etc.)-# are always populated on a completed run and would short-circuit-# _is_meaningful_extracted_data to True even when nothing useful was produced.-_TASK_OUTPUT_PAYLOAD_FIELDS: tuple[str, ...] = (- "extracted_information",- "downloaded_files",- "downloaded_file_urls",-)_TASK_OUTPUT_PARAMETER_SUFFIX = "_output"diff --git a/skyvern/forge/sdk/copilot/tools/completion.py b/skyvern/forge/sdk/copilot/tools/completion.pyindex 88ce7b3b4..08f834b97 100644--- a/skyvern/forge/sdk/copilot/tools/completion.py+++ b/skyvern/forge/sdk/copilot/tools/completion.py@@ -8,6 +8,8 @@ from skyvern.config import settingsfrom skyvern.forge.sdk.copilot.challenge_evidence import carrier_backed_anti_bot_categoriesfrom skyvern.forge.sdk.copilot.completion_criteria_store import note_adjudication_on_turn_statefrom skyvern.forge.sdk.copilot.completion_output_grounding import (+ _GroundingCtx,+ floor_rekeyed_path_backing,grade_requested_output_criteria,split_requested_output_criteria,)@@ -22,6 +24,7 @@ from skyvern.forge.sdk.copilot.completion_verification import (_is_structural_requested_output_abstention,carry_degraded_criterion_ids,carry_floor_rekeyed_criterion_ids,+ carry_floor_rekeyed_path_backing,combine_verification_results,evaluate_completion_criteria,grade_definition_criteria,@@ -63,6 +66,7 @@ from skyvern.forge.sdk.copilot.terminal_predicates import outcome_fully_verifiedfrom skyvern.forge.sdk.copilot.tracing_setup import copilot_spanfrom ._shared import (+ _TASK_ENVELOPE_BLOCK_TYPES,RUN_BLOCKS_SAFETY_CEILING_SECONDS,_copilot_seconds_remaining,_current_workflow_block_labels,@@ -70,6 +74,7 @@ from ._shared import (_failed_run_block_labels,_is_meaningful_extracted_data,_registered_output_parameter_payloads,+ _registered_output_payload_view,_valid_runtime_anchor_url,_workflow_output_parameter_payloads,)@@ -849,6 +854,99 @@ def _pre_run_page_reference_text(reference: PreRunPageReference | None, run_id:return reference.text or None+def _floor_rekeyed_emission_evidence(+ copilot_ctx: _GroundingCtx, run_data: Mapping[str, Any] | None+) -> tuple[dict[str, Any], dict[str, EvidenceSourceKind], dict[str, str | None], set[str]]:+ """Emission-only evidence view for floor-rekeyed backing: this run's block runtime outputs and+ registered output parameters, each envelope-sliced with its producer block's type so metadata-only+ envelopes carry no signal. Page and registered-artifact observations never enter by construction.+ The fourth element is the labels of task-envelope blocks that ran this run — a producer that+ emitted nothing drops from the evidence view, so this is the only signal of how many candidate+ producers exist when no authored contract names one."""+ block_outputs: dict[str, Any] = {}+ block_output_sources: dict[str, EvidenceSourceKind] = {}+ block_types: dict[str, str | None] = {}+ runtime_envelope_labels: set[str] = set()+ if not isinstance(run_data, Mapping):+ return block_outputs, block_output_sources, block_types, runtime_envelope_labels+ current_labels = set(_current_workflow_block_labels(copilot_ctx))+ blocks = run_data.get("blocks")+ if isinstance(blocks, list):+ for block in blocks:+ if not isinstance(block, dict):+ continue+ label = block.get("label")+ block_type = block.get("block_type")+ extracted = block.get("extracted_data")+ if isinstance(label, str) and label in current_labels:+ if (block_type or "").upper() in _TASK_ENVELOPE_BLOCK_TYPES:+ runtime_envelope_labels.add(label)+ sliced = _registered_output_payload_view(extracted, block_type)+ if _is_meaningful_extracted_data(sliced):+ block_outputs[label] = sliced+ block_output_sources[label] = "runtime_output"+ block_types[label] = block_type+ for output_key, output_value in _workflow_output_parameter_payloads(extracted).items():+ block_outputs[output_key] = _registered_output_payload_view(output_value, block_type)+ block_output_sources[output_key] = "registered_output_parameter"+ block_types[output_key] = block_type+ for output_key, output_value in _workflow_output_parameter_payloads(run_data.get("output")).items():+ if not _is_meaningful_extracted_data(_registered_output_payload_view(output_value, None)):+ continue+ block_outputs.setdefault(output_key, output_value)+ block_output_sources.setdefault(output_key, "registered_output_parameter")+ for registered in _registered_output_parameter_payloads(run_data):+ registered_output_key = registered.get("output_parameter_key")+ registered_block_type = registered.get("block_type")+ registered_output_value = _registered_output_payload_view(registered.get("value"), registered_block_type)+ registered_block_label = registered.get("block_label")+ if isinstance(registered_output_key, str) and registered_output_key:+ block_outputs[registered_output_key] = registered_output_value+ block_output_sources[registered_output_key] = "registered_output_parameter"+ block_types[registered_output_key] = registered_block_type+ if isinstance(registered_block_label, str) and registered_block_label in current_labels:+ if isinstance(registered_output_key, str) and registered_output_key:+ existing = block_outputs.get(registered_block_label)+ if isinstance(existing, dict):+ merged = dict(existing)+ merged.setdefault(registered_output_key, registered_output_value)+ block_outputs[registered_block_label] = merged+ else:+ block_outputs[registered_block_label] = {registered_output_key: registered_output_value}+ block_output_sources.setdefault(registered_block_label, "registered_output_parameter")+ block_types.setdefault(registered_block_label, registered_block_type)+ else:+ block_outputs[registered_block_label] = registered_output_value+ block_output_sources[registered_block_label] = "registered_output_parameter"+ block_types[registered_block_label] = registered_block_type+ return block_outputs, block_output_sources, block_types, runtime_envelope_labels+++def _carry_floor_rekeyed_backing(+ copilot_ctx: _GroundingCtx,+ verification: CompletionVerificationResult,+ criteria: list[CompletionCriterion],+ run_data: Mapping[str, Any] | None,+) -> CompletionVerificationResult:+ if not verification.floor_rekeyed_criterion_ids:+ return verification+ (+ emission_block_outputs,+ emission_block_output_sources,+ emission_block_types,+ runtime_envelope_labels,+ ) = _floor_rekeyed_emission_evidence(copilot_ctx, run_data)+ backing = floor_rekeyed_path_backing(+ copilot_ctx,+ criteria,+ emission_block_outputs,+ emission_block_output_sources,+ emission_block_types,+ runtime_envelope_labels=runtime_envelope_labels,+ )+ return carry_floor_rekeyed_path_backing(verification, backing)++def _build_run_evidence_snapshot(copilot_ctx: Any, result: dict[str, Any]) -> RunEvidenceSnapshot:data = result.get("data")data = data if isinstance(data, dict) else {}@@ -871,13 +969,23 @@ def _build_run_evidence_snapshot(copilot_ctx: Any, result: dict[str, Any]) -> Rublock_outputs[label] = evidence_outputblock_output_sources[label] = "runtime_output"for output_key, output_value in _workflow_output_parameter_payloads(output).items():+ if not _is_meaningful_extracted_data(+ _registered_output_payload_view(output_value, block.get("block_type"))+ ):+ continueblock_outputs[output_key] = output_valueblock_output_sources[output_key] = "registered_output_parameter"for output_key, output_value in _workflow_output_parameter_payloads(data.get("output")).items():+ if not _is_meaningful_extracted_data(_registered_output_payload_view(output_value, None)):+ continueblock_outputs[output_key] = output_valueblock_output_sources[output_key] = "registered_output_parameter"for registered in _registered_output_parameter_payloads(data):registered_output_key = registered.get("output_parameter_key")+ if not _is_meaningful_extracted_data(+ _registered_output_payload_view(registered.get("value"), registered.get("block_type"))+ ):+ continueregistered_output_value = _completion_evidence_payload(registered.get("value"))registered_block_label = registered.get("block_label")if isinstance(registered_output_key, str) and registered_output_key:@@ -1208,7 +1316,11 @@ async def _maybe_run_completion_verification(if verification is None:return Noneverification = carry_degraded_criterion_ids(verification, criteria)- return carry_floor_rekeyed_criterion_ids(verification, criteria)+ verification = carry_floor_rekeyed_criterion_ids(verification, criteria)+ run_data = result.get("data")+ return _carry_floor_rekeyed_backing(+ copilot_ctx, verification, criteria, run_data if isinstance(run_data, dict) else None+ )async def _completion_verification_from_run_result(diff --git a/skyvern/forge/sdk/copilot/tools/run_execution.py b/skyvern/forge/sdk/copilot/tools/run_execution.pyindex 3a05a913b..50d3f6e82 100644--- a/skyvern/forge/sdk/copilot/tools/run_execution.py+++ b/skyvern/forge/sdk/copilot/tools/run_execution.py@@ -59,6 +59,8 @@ from skyvern.forge.sdk.copilot.completion_verification import (DeliveredUnverifiedTerminalState,degraded_contract_delivered_unverified_terminal_state,floor_rekeyed_deliverable_credit,+ floor_rekeyed_emission_lane_fields,+ floor_rekeyed_emission_withhold,only_structural_requested_output_abstentions,zero_requested_output_criteria_credit,)@@ -2669,6 +2671,15 @@ def _delivered_unverified_registered_outputs(data: Mapping[str, Any]) -> dict[streturn observed+def _emit_floor_rekeyed_emission_lane(+ completion_verification: CompletionVerificationResult | None, workflow_run_id: str | None+) -> None:+ fields = floor_rekeyed_emission_lane_fields(completion_verification)+ if fields is None:+ return+ LOG.info("copilot.completion.floor_rekeyed_emission_lane", workflow_run_id=workflow_run_id, **fields)++def _record_run_blocks_result(copilot_ctx: Any, result: dict[str, Any], completion_verification: CompletionVerificationResult | None = None) -> RecordedRunOutcome | None:@@ -2700,6 +2711,7 @@ def _record_run_blocks_result(_record_adjudication_on_turn_state(copilot_ctx, completion_verification)if completion_verification is not None and completion_verification.status == "evaluated":_emit_completion_verification_trace(copilot_ctx, completion_verification)+ _emit_floor_rekeyed_emission_lane(completion_verification, run_id if isinstance(run_id, str) else None)copilot_ctx.last_run_blocks_workflow_run_id = run_id if isinstance(run_id, str) else Nonecopilot_ctx.last_successful_run_blocks_workflow_run_id = run_id if run_ok and isinstance(run_id, str) else Nonecopilot_ctx.delivered_unverified_terminal = False@@ -3175,41 +3187,46 @@ def _zero_requested_output_criteria_delivered_unverified(data: Mapping[str, Any],workflow_run_id: str | None,) -> RecordedRunOutcome | None:- if not zero_requested_output_criteria_credit(+ emission_withhold = floor_rekeyed_emission_withhold(completion_verification)+ registered_output_meaningful = _has_meaningful_registered_output_payload(data)+ credit_withhold = zero_requested_output_criteria_credit(completion_verification,- has_meaningful_registered_output=_has_meaningful_registered_output_payload(data),- ):+ has_meaningful_registered_output=registered_output_meaningful,+ )+ if emission_withhold is None and not credit_withhold:return None- credit = floor_rekeyed_deliverable_credit(completion_verification)- if credit is not None:- payloads = _registered_output_parameter_payloads(data)- payload_keys = sorted(- {key for item in payloads for key in [item.get("output_parameter_key")] if isinstance(key, str) and key}- )- unbound_paths = [- output_path- for output_path in credit.output_paths- if output_path and not registered_output_payload_binds_output_path(payloads, output_path)- ]- if unbound_paths:- LOG.info(- "copilot.completion.floor_rekeyed_credit_payload_unbound",- workflow_run_id=workflow_run_id,- criterion_ids=list(credit.criterion_ids),- unbound_output_paths=unbound_paths,- registered_output_keys=payload_keys,- )- else:- LOG.info(- "copilot.completion.floor_rekeyed_deliverable_credit",- workflow_run_id=workflow_run_id,- criterion_ids=list(credit.criterion_ids),- evidence_sources=list(credit.evidence_sources),- evidence_refs=list(credit.evidence_refs),- credited_output_paths=list(credit.output_paths),- registered_output_keys=payload_keys,+ if emission_withhold is None:+ credit = floor_rekeyed_deliverable_credit(completion_verification)+ if credit is not None:+ payloads = _registered_output_parameter_payloads(data)+ payload_keys = sorted(+ {key for item in payloads for key in [item.get("output_parameter_key")] if isinstance(key, str) and key})- return None+ unbound_paths = [+ output_path+ for output_path in credit.output_paths+ if output_path and not registered_output_payload_binds_output_path(payloads, output_path)+ ]+ if unbound_paths:+ LOG.info(+ "copilot.completion.floor_rekeyed_credit_payload_unbound",+ workflow_run_id=workflow_run_id,+ criterion_ids=list(credit.criterion_ids),+ unbound_output_paths=unbound_paths,+ backed_by_criterion_id=dict(completion_verification.floor_rekeyed_backed_by_criterion_id),+ registered_output_keys=payload_keys,+ )+ else:+ LOG.info(+ "copilot.completion.floor_rekeyed_deliverable_credit",+ workflow_run_id=workflow_run_id,+ criterion_ids=list(credit.criterion_ids),+ evidence_sources=list(credit.evidence_sources),+ evidence_refs=list(credit.evidence_refs),+ credited_output_paths=list(credit.output_paths),+ registered_output_keys=payload_keys,+ )+ return Nonecopilot_ctx.last_test_suspicious_success = Falsecopilot_ctx.last_test_failure_reason = Nonecopilot_ctx.suspicious_success_nudge_count = 0@@ -3219,17 +3236,27 @@ def _zero_requested_output_criteria_delivered_unverified(copilot_ctx.delivered_unverified_workflow_run_id = workflow_run_idcopilot_ctx.delivered_unverified_observed_outputs = _delivered_unverified_registered_outputs(data)stash_delivered_unverified_turn_halt(copilot_ctx, workflow_run_id=workflow_run_id)- LOG.info(- "copilot.completion.zero_requested_output_credit_withheld",- workflow_run_id=workflow_run_id,- requested_output_criteria_count=0,- registered_output_meaningful=True,- )+ if emission_withhold is not None:+ LOG.info(+ "copilot.completion.floor_rekeyed_emission_withheld",+ workflow_run_id=workflow_run_id,+ requested_output_criteria_count=completion_verification.requested_output_criteria_count,+ registered_output_meaningful=registered_output_meaningful,+ floor_rekeyed_unbacked_criterion_ids=list(emission_withhold.criterion_ids),+ floor_rekeyed_unbacked_output_paths=list(emission_withhold.unbacked_output_paths),+ )+ display_reason = "The latest run did not produce the requested output, so it was not verified."+ else:+ LOG.info(+ "copilot.completion.zero_requested_output_credit_withheld",+ workflow_run_id=workflow_run_id,+ requested_output_criteria_count=0,+ registered_output_meaningful=True,+ )+ display_reason = "The latest run returned output, but it was not independently verified."return RecordedRunOutcome(verdict="not_evaluated",- display_reason=run_outcome_display_reason(- "The latest run returned output, but it was not independently verified."- ),+ display_reason=run_outcome_display_reason(display_reason),workflow_run_id=workflow_run_id,)diff --git a/skyvern/forge/sdk/copilot/tools/workflow_update.py b/skyvern/forge/sdk/copilot/tools/workflow_update.pyindex 0ea892c53..fedaafaa2 100644--- a/skyvern/forge/sdk/copilot/tools/workflow_update.py+++ b/skyvern/forge/sdk/copilot/tools/workflow_update.py@@ -72,6 +72,7 @@ from skyvern.forge.sdk.copilot.code_block_synthesis import (locator_selector_literals,missing_rung_text,normalized_locator_expr,+ produce_covered_static_return_envelope,render_missing_rung_call_sources,synthesize_code_block,synthesize_code_block_with_extraction,@@ -95,8 +96,10 @@ from skyvern.forge.sdk.copilot.enforcement import (_record_code_authoring_guardrail_reject,_scouted_spine_open_obligation,arm_credential_scout_reopen,+ download_satisfied_requested_output_paths,requested_output_extraction_plan,requested_output_extraction_plan_changed,+ requested_scalar_output_extraction_plan,synthesized_goal_completion_landing_pending,synthesized_persistence_reopened,synthesized_persistence_reopened_after_failed_run,@@ -1639,7 +1642,7 @@ def _requested_output_child_paths(ctx: AgentContext) -> set[str]:return set()paths: set[str] = set()for criterion in _active_completion_criteria(ctx):- if isinstance(criterion, CompletionCriterion) and _independent_judgment_output_criterion(criterion):+ if isinstance(criterion, CompletionCriterion) and _is_judgment_boolean_criterion(criterion):continueif getattr(criterion, "level", None) == "definition":continue@@ -1681,18 +1684,16 @@ def _active_completion_criteria(ctx: AgentContext) -> list[CompletionCriterion]:return request_policy.graded_completion_criteria()-def _independent_judgment_output_criterion(criterion: CompletionCriterion) -> bool:- return criterion.requested_output_evidence_source == "independent_run_evidence" and _is_judgment_boolean_criterion(- criterion- )---def _independent_judgment_output_paths(ctx: AgentContext) -> set[str]:+def _judgment_output_paths(ctx: AgentContext) -> set[str]:+ # Judgment-boolean paths are dropped from the producer's static-bind set, not moved to the+ # declaration lane; declaring one would fabricate a None where the run's own judgment belongs. The+ # requested-output gate reads the completion criteria directly rather than this bind set, so dropping+ # cannot pin it open.paths: set[str] = set()for criterion in _active_completion_criteria(ctx):if not isinstance(criterion, CompletionCriterion):continue- if not _independent_judgment_output_criterion(criterion):+ if not _is_judgment_boolean_criterion(criterion):continuepath = _canonical_requested_output_path(criterion.output_path)if path and _output_path_has_child(path):@@ -2389,7 +2390,7 @@ def _output_contract_required_paths_source(ctx: AgentContext) -> _OutputContractruntime_contract = _runtime_output_repair_contract_from_recorded_outcome(ctx)antecedent_paths = _contingent_antecedent_child_paths(ctx)if runtime_contract is not None:- runtime_observation_paths = runtime_contract.required_paths - _independent_judgment_output_paths(ctx)+ runtime_observation_paths = runtime_contract.required_paths - _judgment_output_paths(ctx)return _OutputContractRequiredPaths(observation_paths=runtime_observation_paths,declaration_paths=antecedent_paths - runtime_observation_paths,@@ -2397,6 +2398,7 @@ def _output_contract_required_paths_source(ctx: AgentContext) -> _OutputContractreason_code=runtime_contract.reason_code,)observation_paths, source, reason_code = _required_child_output_paths_for_authoring(ctx)+ observation_paths = observation_paths - _judgment_output_paths(ctx)repair_context = getattr(ctx, "last_code_authoring_repair_context", None)if (not observation_paths@@ -2411,7 +2413,7 @@ def _output_contract_required_paths_source(ctx: AgentContext) -> _OutputContract*repair_context.required_code_return_paths,])- rehydrated -= _independent_judgment_output_paths(ctx)+ rehydrated -= _judgment_output_paths(ctx)# An antecedent the repair contract carried only in schema/return roles stays in the# declaration lane on rehydration; the goal role is the observation-lane record.observation_paths = rehydrated - (antecedent_paths - goal_paths)@@ -3190,6 +3192,46 @@ class _SpineSplitOutcome(NamedTuple):stage_count: int | None+def _produce_split_extraction_envelope(+ ctx: AgentContext,+ extraction_suffix: str,+ *,+ required_paths: set[str],+ declaration_paths: set[str] | None,+ label: str,+ signature: str,+) -> str | None:+ """Scalar-only producer for the spine-split seam: the extraction suffix carries no browser+ actions, so the mixed download shape abstains here and the split bail stays byte-identical."""+ declarations = declaration_paths or set()+ if required_paths & download_satisfied_requested_output_paths(ctx):+ return None+ scalar_paths = required_paths - declarations+ envelope = produce_covered_static_return_envelope(+ extraction_suffix,+ plan=requested_scalar_output_extraction_plan(ctx),+ scalar_required_paths=scalar_paths,+ declaration_paths=declarations,+ download_required_paths=set(),+ expects_download=False,+ )+ if envelope is None:+ return None+ _, revalidation = _extraction_code_with_required_static_return(+ envelope.code, required_paths=scalar_paths, declaration_paths=declarations+ )+ if revalidation:+ return None+ LOG.info(+ "copilot_output_contract_keyed_static_return_produced",+ block_label=label,+ keyed_paths=list(envelope.keyed_paths),+ download_registration_paths=[],+ canonical_output_contract_signature=signature,+ )+ return envelope.code++def _attempt_separated_spine_split(*,ctx: AgentContext,@@ -3198,6 +3240,7 @@ def _attempt_separated_spine_split(target_code: str,synthesized: SynthesizedCodeBlock,required_paths: set[str],+ signature: str,declaration_paths: set[str] | None = None,) -> _SpineSplitOutcome:code_block = next(@@ -3230,7 +3273,17 @@ def _attempt_separated_spine_split(extraction_suffix, required_paths=required_paths, declaration_paths=declaration_paths)if static_violations:- return _SpineSplitOutcome(None, ["static_return_envelope_unavailable"], None)+ produced_extraction = _produce_split_extraction_envelope(+ ctx,+ extraction_suffix,+ required_paths=required_paths,+ declaration_paths=declaration_paths,+ label=label,+ signature=signature,+ )+ if produced_extraction is None:+ return _SpineSplitOutcome(None, ["static_return_envelope_unavailable"], None)+ keyed_extraction = produced_extractionif _browser_surface_contains_full_action_spine(keyed_extraction, reconciled_synthesized):return _SpineSplitOutcome(None, ["extraction_retains_full_spine"], None)@@ -3880,6 +3933,110 @@ def _stamp_declaration_contract_defaults(return yaml.safe_dump(parsed, sort_keys=False), True+def _impose_covered_static_return_envelope(+ ctx: AgentContext,+ *,+ parsed: dict[str, Any],+ workflow_yaml: str,+ raw_code_artifact_metadata: object,+ label: str,+ target_code: str,+ observation_paths: set[str],+ declaration_paths: set[str],+ source: str,+ reason_code: str,+ signature: str,+ runtime_attempt_key: str,+) -> tuple[str, object, bool] | None:+ """Deterministic evidence-derived envelope on the whole-block scaffold route: author the scalar+ live-reads (mixed shape merges the resolved download descriptor into one return), or admit a+ download-only block, so ``static_return_envelope_unavailable`` never fires for scout-covered goals.+ Returns None to leave today's bail ladder byte-identical."""+ download_registered = download_satisfied_requested_output_paths(ctx)+ scalar_paths = observation_paths - download_registered+ download_paths = observation_paths & download_registered+ download = ctx.reached_download_target+ expects_download = download is not None and bool(download.selector) and _code_uses_expect_download(target_code)+ block_metadata = _metadata_item_for_block_label(raw_code_artifact_metadata, label) or {}++ if not scalar_paths:+ if not (download_paths and expects_download):+ return None+ if _download_return_shape_error(label, block_metadata, target_code) is not None:+ return None+ scaffolded_metadata = _apply_metadata_contract_scaffold(+ ctx,+ workflow_yaml,+ raw_code_artifact_metadata,+ required_paths=observation_paths,+ source=source,+ reason_code=reason_code,+ declaration_paths=declaration_paths,+ allow_missing_static_return=True,+ )+ _mark_output_contract_imposed(ctx, signature)+ _record_runtime_output_repair_attempt(ctx, runtime_attempt_key)+ LOG.info(+ "copilot_output_contract_download_registration_admitted",+ block_label=label,+ download_registration_paths=sorted(download_paths),+ canonical_output_contract_signature=signature,+ )+ return workflow_yaml, scaffolded_metadata, scaffolded_metadata is not raw_code_artifact_metadata++ envelope = produce_covered_static_return_envelope(+ target_code,+ plan=requested_scalar_output_extraction_plan(ctx),+ scalar_required_paths=scalar_paths,+ declaration_paths=declaration_paths,+ download_required_paths=download_paths,+ expects_download=expects_download,+ )+ if envelope is None:+ return None+ _, revalidation = _extraction_code_with_required_static_return(+ envelope.code, required_paths=scalar_paths, declaration_paths=declaration_paths+ )+ if revalidation:+ return None+ if download_paths and _download_return_shape_error(label, block_metadata, envelope.code) is not None:+ return None++ for block in _workflow_code_blocks(parsed):+ if str(block.get("label") or "").strip() == label:+ block["code"] = envelope.code.rstrip() + "\n"+ break+ imposed_yaml = yaml.safe_dump(parsed, sort_keys=False)+ scaffolded_metadata = _apply_metadata_contract_scaffold(+ ctx,+ imposed_yaml,+ raw_code_artifact_metadata,+ required_paths=observation_paths,+ source=source,+ reason_code=reason_code,+ declaration_paths=declaration_paths,+ allow_missing_static_return=bool(download_paths),+ )+ _mark_output_contract_imposed(ctx, signature)+ _record_runtime_output_repair_attempt(ctx, runtime_attempt_key)+ LOG.info(+ "copilot_output_contract_keyed_static_return_produced",+ block_label=label,+ keyed_paths=list(envelope.keyed_paths),+ download_registration_paths=sorted(download_paths),+ canonical_output_contract_signature=signature,+ )+ LOG.info(+ "copilot_output_contract_envelope_imposed_after_steering",+ block_label=label,+ canonical_output_contract_signature=signature,+ steering_reject_count=_output_contract_reject_count(ctx, signature),+ return_envelope_applied=True,+ metadata_scaffold_applied=scaffolded_metadata is not raw_code_artifact_metadata,+ )+ return imposed_yaml, scaffolded_metadata, True++def _impose_output_contract_envelope_after_steering(ctx: AgentContext,workflow_yaml: str,@@ -3944,6 +4101,7 @@ def _impose_output_contract_envelope_after_steering(target_code=target_code,synthesized=synthesized,required_paths=required_paths,+ signature=signature,declaration_paths=declaration_paths,)if split.imposed_yaml is not None:@@ -4008,6 +4166,23 @@ def _impose_output_contract_envelope_after_steering(declaration_paths=declaration_paths,)if violations:+ produced = _impose_covered_static_return_envelope(+ ctx,+ parsed=parsed,+ workflow_yaml=workflow_yaml,+ raw_code_artifact_metadata=raw_code_artifact_metadata,+ label=label,+ target_code=target_code,+ observation_paths=observation_paths,+ declaration_paths=declaration_paths,+ source=source,+ reason_code=reason_code,+ signature=signature,+ runtime_attempt_key=runtime_attempt_key,+ )+ if produced is not None:+ produced_yaml, produced_metadata, produced_changed = produced+ return produced_yaml, produced_metadata, produced_changed or declaration_stampedscaffolded_metadata = _apply_metadata_contract_scaffold(ctx,workflow_yaml,diff --git a/skyvern/forge/sdk/forge_log.py b/skyvern/forge/sdk/forge_log.pyindex 34ddcb20a..cfc183202 100644--- a/skyvern/forge/sdk/forge_log.py+++ b/skyvern/forge/sdk/forge_log.py@@ -461,13 +461,17 @@ def setup_logger() -> None:handler = logging.StreamHandler()handler.setFormatter(structlog.stdlib.ProcessorFormatter(+ foreign_pre_chain=[+ add_error_processor,+ structlog.processors.format_exc_info,+ ],processors=[structlog.stdlib.add_log_level,structlog.stdlib.add_logger_name,structlog.stdlib.ProcessorFormatter.remove_processors_meta,structlog.processors.TimeStamper(fmt="iso"),renderer,- ]+ ],))root_logger = logging.getLogger()diff --git a/tests/unit/test_copilot_code_block_persist_seam.py b/tests/unit/test_copilot_code_block_persist_seam.pyindex 1b91f6b43..dfca744ae 100644--- a/tests/unit/test_copilot_code_block_persist_seam.py+++ b/tests/unit/test_copilot_code_block_persist_seam.py@@ -17,6 +17,7 @@ from structlog.testing import capture_logsfrom skyvern.forge.sdk.copilot import agent as agent_modulefrom skyvern.forge.sdk.copilot import enforcement as enforcement_module+from skyvern.forge.sdk.copilot import request_policy as request_policy_modulefrom skyvern.forge.sdk.copilot.blocker_signal import (CREDENTIAL_SCOUT_VERIFY_REPLY,CopilotToolBlockerSignal,@@ -41,12 +42,14 @@ from skyvern.forge.sdk.copilot.code_block_preflight import ()from skyvern.forge.sdk.copilot.code_block_security import CodeBlockSecurityErrorfrom skyvern.forge.sdk.copilot.code_block_synthesis import (+ ProducedStaticReturnEnvelope,ScoutGap,SynthesisDiagnostics,SynthesizedCodeBlock,_get_by_role_expr,_get_by_role_expr_strict,credential_scout_gap,+ produce_covered_static_return_envelope,)from skyvern.forge.sdk.copilot.completion_verification import CompletionVerificationResult, CriterionVerdictfrom skyvern.forge.sdk.copilot.config import BlockAuthoringPolicy, CopilotConfig@@ -60,6 +63,12 @@ from skyvern.forge.sdk.copilot.output_contracts import (OutputContractAdvisoryState,code_block_available_binding_keys_by_label,)+from skyvern.forge.sdk.copilot.output_extraction_plan import (+ LiveReadBinding,+ LiveReadKind,+ RequestedOutputExtractionPlan,+ RevealAnchor,+)from skyvern.forge.sdk.copilot.output_utils import sanitize_tool_result_for_llmfrom skyvern.forge.sdk.copilot.reached_download_target import ReachedDownloadTargetfrom skyvern.forge.sdk.copilot.request_policy import (@@ -4373,6 +4382,94 @@ class TestCodeRepairProgressClassification:assert source == "requested_output_contract"assert reason_code == "requested_output_contract_missing_output_coverage"+ def test_runtime_output_judgment_is_excluded_from_scalar_bind_set(self) -> None:+ ctx = _code_only_ctx()+ ctx.request_policy = RequestPolicy(+ completion_criteria=[+ CompletionCriterion(+ id="login_gate",+ outcome="the target path is blocked by a login gate",+ output_path="output.login_gate_blocks_target",+ expected_output_value=True,+ expected_output_shape="goal_judgment_boolean",+ requested_output_evidence_source="runtime_output",+ ),+ CompletionCriterion(+ id="record_id",+ outcome="the record id is returned",+ output_path="output.record_id",+ ),+ ]+ )++ contract = workflow_update_module._output_contract_required_paths_source(ctx)++ assert contract.observation_paths == {"output.record_id"}+ assert contract.union == {"output.record_id"}+ assert "output.login_gate_blocks_target" not in contract.declaration_paths++ def test_runtime_output_judgment_alone_yields_empty_bind_set(self) -> None:+ ctx = _code_only_ctx()+ ctx.request_policy = RequestPolicy(+ completion_criteria=[+ CompletionCriterion(+ id="login_gate",+ outcome="the target path is blocked by a login gate",+ output_path="output.login_gate_blocks_target",+ expected_output_value=True,+ expected_output_shape="goal_judgment_boolean",+ requested_output_evidence_source="runtime_output",+ )+ ]+ )++ assert workflow_update_module._output_contract_required_paths_source(ctx).union == set()++ def test_non_judgment_scalar_bind_set_is_unchanged(self) -> None:+ ctx = _code_only_ctx()+ ctx.request_policy = RequestPolicy(+ completion_criteria=[+ CompletionCriterion(+ id="record_id",+ outcome="the record id is returned",+ output_path="output.record_id",+ )+ ]+ )++ assert workflow_update_module._output_contract_required_paths_source(ctx).observation_paths == {+ "output.record_id"+ }++ def test_judgment_output_paths_widens_independent_predicate(self) -> None:+ ctx = _code_only_ctx()+ ctx.request_policy = RequestPolicy(+ completion_criteria=[+ CompletionCriterion(+ id="independent_gate",+ outcome="the independent path is blocked by a login gate",+ output_path="output.independent_gate_blocks_target",+ expected_output_shape="goal_judgment_boolean",+ requested_output_evidence_source="independent_run_evidence",+ ),+ CompletionCriterion(+ id="runtime_gate",+ outcome="the runtime path is blocked by a login gate",+ output_path="output.runtime_gate_blocks_target",+ expected_output_value=True,+ expected_output_shape="goal_judgment_boolean",+ requested_output_evidence_source="runtime_output",+ ),+ ]+ )++ all_judgment = workflow_update_module._judgment_output_paths(ctx)++ assert all_judgment == {+ "output.independent_gate_blocks_target",+ "output.runtime_gate_blocks_target",+ }+def test_independent_judgment_runtime_repair_fact_is_not_rehydrated_as_code_return_path(self) -> None:ctx = _code_only_ctx()ctx.request_policy = RequestPolicy(@@ -15006,3 +15103,431 @@ class TestSequencedDownloadTerminalCoverage:assert guarded is not Noneassert any("scouted_spine_under_build" in violation for violation in guarded.violations)+++def _scalar_binding(output_path: str, label: str, selector: str) -> LiveReadBinding:+ return LiveReadBinding(+ output_path=output_path,+ kind=LiveReadKind.KEY_VALUE,+ selector=selector,+ selector_count=1,+ selector_index=0,+ child_index=1,+ child_count=2,+ relation_label=label,+ )+++def _scalar_plan(*bindings: LiveReadBinding) -> RequestedOutputExtractionPlan:+ return RequestedOutputExtractionPlan(+ requested_output_paths=tuple(binding.output_path for binding in bindings),+ observation_step=1,+ observation_identity="obs-identity",+ reveal=RevealAnchor(selector="#reveal"),+ live_reads=tuple(bindings),+ identity="plan-identity",+ )+++_MIXED_DOWNLOAD_CODE = (+ "async with page.expect_download() as dl_info:\n"+ ' await page.locator("#download-invoice").click()\n'+ "dl_info_file = await dl_info.value\n"+ "downloaded_file_name = dl_info_file.suggested_filename\n"+ "await dl_info_file.path()\n"+ "return {\n"+ ' "downloaded_file_name": downloaded_file_name,\n'+ "}\n"+)+++def test_producer_scalar_coverage_single_return_over_all_paths() -> None:+ plan = _scalar_plan(+ _scalar_binding("output.account_number", "Account Number", ".acct"),+ _scalar_binding("output.confirmation_number", "Confirmation Number", ".conf"),+ )+ envelope = produce_covered_static_return_envelope(+ 'await page.locator("#lookup").click()\n',+ plan=plan,+ scalar_required_paths={"output.account_number", "output.confirmation_number"},+ declaration_paths=set(),+ download_required_paths=set(),+ expects_download=False,+ )+ assert isinstance(envelope, ProducedStaticReturnEnvelope)+ assert envelope.keyed_paths == ("output.account_number", "output.confirmation_number")+ assert envelope.code.count("return {") == 1+ produced = workflow_update_module._code_block_produced_output_paths(envelope.code)+ assert {"output.account_number", "output.confirmation_number"} <= produced+ _, violations = workflow_update_module._extraction_code_with_required_static_return(+ envelope.code, required_paths={"output.account_number", "output.confirmation_number"}+ )+ assert violations == []+++def test_producer_mixed_merged_single_return_passes_download_gate() -> None:+ plan = _scalar_plan(+ _scalar_binding("output.statement_amount", "Amount Due", ".amt"),+ _scalar_binding("output.statement_date", "Statement Date", ".dt"),+ )+ envelope = produce_covered_static_return_envelope(+ _MIXED_DOWNLOAD_CODE,+ plan=plan,+ scalar_required_paths={"output.statement_amount", "output.statement_date"},+ declaration_paths=set(),+ download_required_paths={"output.downloaded_invoice_pdf"},+ expects_download=True,+ )+ assert isinstance(envelope, ProducedStaticReturnEnvelope)+ assert envelope.code.count("return {") == 1+ assert '"downloaded_file_name": downloaded_file_name' in envelope.code+ assert workflow_update_module._code_uses_expect_download(envelope.code)+ artifact = {"claimed_outcomes": [{"id": "claim", "goal_value_paths": ["output.downloaded_invoice_pdf"]}]}+ assert workflow_update_module._download_return_shape_error("block", artifact, envelope.code) is None+ _, violations = workflow_update_module._extraction_code_with_required_static_return(+ envelope.code, required_paths={"output.statement_amount", "output.statement_date"}+ )+ assert violations == []+++def test_producer_no_coverage_abstains() -> None:+ plan = _scalar_plan(_scalar_binding("output.account_number", "Account Number", ".acct"))+ envelope = produce_covered_static_return_envelope(+ 'await page.locator("#lookup").click()\n',+ plan=plan,+ scalar_required_paths={"output.account_number", "output.confirmation_number"},+ declaration_paths=set(),+ download_required_paths=set(),+ expects_download=False,+ )+ assert envelope is None+++def test_producer_output_passes_keyer_on_reduced_set() -> None:+ plan = _scalar_plan(_scalar_binding("output.amount", "Amount", ".amt"))+ envelope = produce_covered_static_return_envelope(+ 'await page.locator("#lookup").click()\n',+ plan=plan,+ scalar_required_paths={"output.amount"},+ declaration_paths={"output.optional_note"},+ download_required_paths=set(),+ expects_download=False,+ )+ assert envelope is not None+ _, violations = workflow_update_module._extraction_code_with_required_static_return(+ envelope.code, required_paths={"output.amount"}, declaration_paths={"output.optional_note"}+ )+ assert violations == []+++def test_producer_abstains_when_unbindable_judgment_path_kept_in_scalar_set() -> None:+ plan = _scalar_plan(_scalar_binding("output.record_id", "Record Identifier", ".record-kv"))+ envelope = produce_covered_static_return_envelope(+ 'await page.locator("#lookup").click()\n',+ plan=plan,+ scalar_required_paths={"output.record_id", "output.login_gate_blocks_target"},+ declaration_paths=set(),+ download_required_paths=set(),+ expects_download=False,+ )+ assert envelope is None+++def test_producer_emits_keyed_scalar_when_judgment_path_excluded() -> None:+ plan = _scalar_plan(_scalar_binding("output.record_id", "Record Identifier", ".record-kv"))+ envelope = produce_covered_static_return_envelope(+ 'await page.locator("#lookup").click()\n',+ plan=plan,+ scalar_required_paths={"output.record_id"},+ declaration_paths=set(),+ download_required_paths=set(),+ expects_download=False,+ )+ assert isinstance(envelope, ProducedStaticReturnEnvelope)+ assert envelope.keyed_paths == ("output.record_id",)+ assert "login_gate_blocks_target" not in envelope.code+ produced = workflow_update_module._code_block_produced_output_paths(envelope.code)+ assert "output.record_id" in produced+ assert "output.login_gate_blocks_target" not in produced+++def test_producer_branchy_return_abstains() -> None:+ plan = _scalar_plan(_scalar_binding("output.amount", "Amount", ".amt"))+ envelope = produce_covered_static_return_envelope(+ 'if await page.locator("#done").is_visible():\n return {"output": {}}\n',+ plan=plan,+ scalar_required_paths={"output.amount"},+ declaration_paths=set(),+ download_required_paths=set(),+ expects_download=False,+ )+ assert envelope is None+++def test_producer_download_without_idiom_abstains() -> None:+ plan = _scalar_plan(_scalar_binding("output.statement_amount", "Amount Due", ".amt"))+ envelope = produce_covered_static_return_envelope(+ _MIXED_DOWNLOAD_CODE,+ plan=plan,+ scalar_required_paths={"output.statement_amount"},+ declaration_paths=set(),+ download_required_paths={"output.downloaded_invoice_pdf"},+ expects_download=False,+ )+ assert envelope is None+++def test_producer_identifier_collision_abstains() -> None:+ plan = _scalar_plan(_scalar_binding("output.amount", "Amount", ".amt"))+ envelope = produce_covered_static_return_envelope(+ '_envelope_value_0 = 1\nawait page.locator("#lookup").click()\n',+ plan=plan,+ scalar_required_paths={"output.amount"},+ declaration_paths=set(),+ download_required_paths=set(),+ expects_download=False,+ )+ assert envelope is None+++def test_producer_declaration_paths_are_none_defaults() -> None:+ plan = _scalar_plan(_scalar_binding("output.amount", "Amount", ".amt"))+ envelope = produce_covered_static_return_envelope(+ 'await page.locator("#lookup").click()\n',+ plan=plan,+ scalar_required_paths={"output.amount"},+ declaration_paths={"output.optional_note"},+ download_required_paths=set(),+ expects_download=False,+ )+ assert envelope is not None+ assert '"optional_note": None' in envelope.code+ assert "optional_note" not in envelope.code.split('"optional_note": None')[0]+++def test_producer_generated_code_raises_on_empty_scalar_text() -> None:+ plan = _scalar_plan(_scalar_binding("output.amount", "Amount", ".amt"))+ envelope = produce_covered_static_return_envelope(+ 'await page.locator("#lookup").click()\n',+ plan=plan,+ scalar_required_paths={"output.amount"},+ declaration_paths=set(),+ download_required_paths=set(),+ expects_download=False,+ )+ assert envelope is not None+ tree = ast.parse(envelope.code)+ empty_guards = [+ node+ for node in ast.walk(tree)+ if isinstance(node, ast.If)+ and isinstance(node.test, ast.UnaryOp)+ and isinstance(node.test.op, ast.Not)+ and any(isinstance(child, ast.Raise) for child in node.body)+ ]+ assert empty_guards+++def test_produce_split_extraction_envelope_mixed_abstains(monkeypatch: pytest.MonkeyPatch) -> None:+ ctx = SimpleNamespace(+ reached_download_target=SimpleNamespace(selector="#download-invoice"),+ scouted_output_covered_paths={"output.statement_amount"},+ )+ monkeypatch.setattr(+ workflow_update_module,+ "download_satisfied_requested_output_paths",+ lambda _ctx: {"output.downloaded_invoice_pdf"},+ )+ result = workflow_update_module._produce_split_extraction_envelope(+ ctx,+ 'value = "x"\n',+ required_paths={"output.statement_amount", "output.downloaded_invoice_pdf"},+ declaration_paths=set(),+ label="extract_block",+ signature="sig",+ )+ assert result is None+++def _download_classification_ctx(+ *criteria: CompletionCriterion,+ covered_paths: set[str] | None = None,+ download_selector: str | None = "#download-invoice",+) -> SimpleNamespace:+ download = (+ ReachedDownloadTarget(+ selector=download_selector,+ affordance_text="Download Invoice",+ download_kind="attribute",+ source_step="trajectory_recency",+ already_registered=False,+ )+ if download_selector+ else None+ )+ return SimpleNamespace(+ completion_criteria_turn_state=SimpleNamespace(decision=SimpleNamespace(criteria=tuple(criteria))),+ last_code_authoring_repair_context=None,+ scouted_output_covered_paths=set(covered_paths or set()),+ reached_download_target=download,+ flow_evidence=[],+ )+++def test_uncovered_scalar_path_is_not_classified_as_a_download_path() -> None:+ ctx = _download_classification_ctx(+ CompletionCriterion(+ id="statement_amount",+ outcome="The returned record includes the statement amount.",+ output_path="output.statement_amount",+ ),+ covered_paths=set(),+ )++ assert enforcement_module.download_satisfied_requested_output_paths(ctx) == set()+ assert enforcement_module.uncovered_requested_output_paths(ctx) == {"output.statement_amount"}+ assert (+ workflow_update_module._impose_covered_static_return_envelope(+ ctx,+ parsed={},+ workflow_yaml="",+ raw_code_artifact_metadata=[],+ label="download_block",+ target_code=_MIXED_DOWNLOAD_CODE,+ observation_paths={"output.statement_amount"},+ declaration_paths=set(),+ source="scout",+ reason_code="reason",+ signature="sig",+ runtime_attempt_key="key",+ )+ is None+ )+++def test_declared_registered_download_kind_on_uncovered_custom_path_is_a_download_path() -> None:+ ctx = _download_classification_ctx(+ CompletionCriterion(+ id="downloaded_invoice_pdf",+ outcome="The returned record includes the downloaded invoice pdf.",+ output_path="output.downloaded_invoice_pdf",+ declared_deliverable_kind="registered_download",+ ),+ covered_paths=set(),+ )++ assert enforcement_module.download_satisfied_requested_output_paths(ctx) == {"output.downloaded_invoice_pdf"}+ assert enforcement_module.uncovered_requested_output_paths(ctx) == set()+++def test_declared_registered_download_kind_on_scout_covered_path_is_not_a_download_path() -> None:+ ctx = _download_classification_ctx(+ CompletionCriterion(+ id="npi",+ outcome="The returned record includes NPI.",+ output_path="output.npi",+ declared_deliverable_kind="registered_download",+ ),+ CompletionCriterion(+ id="downloaded_invoice_pdf",+ outcome="The returned record includes the downloaded invoice pdf.",+ output_path="output.downloaded_invoice_pdf",+ declared_deliverable_kind="registered_download",+ ),+ covered_paths={"output.npi"},+ )++ assert enforcement_module.download_satisfied_requested_output_paths(ctx) == {"output.downloaded_invoice_pdf"}+++def test_declared_registered_download_kind_survives_requested_output_canonicalization() -> None:+ policy = RequestPolicy(+ completion_criteria=[+ CompletionCriterion(+ id="download",+ outcome="The workflow downloads the invoice file.",+ output_path="output.downloaded_invoice_pdf",+ deliverable_kind="registered_download",+ declared_deliverable_kind="registered_download",+ )+ ]+ )++ request_policy_module._apply_requested_output_completion_criteria(+ policy, "return the downloaded_invoice_pdf when you are done"+ )++ canonical = {criterion.output_path: criterion for criterion in policy.completion_criteria if criterion.output_path}+ criterion = canonical["output.downloaded_invoice_pdf"]+ assert criterion.declared_deliverable_kind == "registered_download"+ assert criterion.deliverable_kind is None+++def test_split_seam_declaration_path_is_none_defaulted_not_live_read(monkeypatch: pytest.MonkeyPatch) -> None:+ plan = _scalar_plan(+ _scalar_binding("output.amount", "Amount", ".amt"),+ _scalar_binding("output.optional_note", "Optional Note", ".note"),+ )+ monkeypatch.setattr(workflow_update_module, "download_satisfied_requested_output_paths", lambda _ctx: set())+ monkeypatch.setattr(workflow_update_module, "requested_scalar_output_extraction_plan", lambda _ctx: plan)++ produced = workflow_update_module._produce_split_extraction_envelope(+ SimpleNamespace(),+ 'await page.locator("#lookup").click()\n',+ required_paths={"output.amount", "output.optional_note"},+ declaration_paths={"output.optional_note"},+ label="extract_block",+ signature="sig",+ )++ assert produced is not None+ assert '"optional_note": None' in produced+ assert ".note" not in produced+ assert "Optional Note" not in produced+++def test_producer_abstains_when_terminal_return_shares_a_line() -> None:+ plan = _scalar_plan(_scalar_binding("output.statement_amount", "Amount Due", ".amt"))+ code = (+ "async with page.expect_download() as dl_info:\n"+ ' await page.locator("#download-invoice").click()\n'+ "dl_info_file = await dl_info.value\n"+ 'name = dl_info_file.suggested_filename; return {"downloaded_file_name": name}\n'+ )++ envelope = produce_covered_static_return_envelope(+ code,+ plan=plan,+ scalar_required_paths={"output.statement_amount"},+ declaration_paths=set(),+ download_required_paths={"output.downloaded_invoice_pdf"},+ expects_download=True,+ )++ assert envelope is None+++def test_producer_preserves_the_models_download_descriptor_key() -> None:+ plan = _scalar_plan(_scalar_binding("output.statement_amount", "Amount Due", ".amt"))+ code = (+ "async with page.expect_download() as dl_info:\n"+ ' await page.locator("#download-invoice").click()\n'+ 'return {"saved_as": dl_info.value.suggested_filename}\n'+ )++ envelope = produce_covered_static_return_envelope(+ code,+ plan=plan,+ scalar_required_paths={"output.statement_amount"},+ declaration_paths=set(),+ download_required_paths={"output.downloaded_invoice_pdf"},+ expects_download=True,+ )++ assert envelope is not None+ assert envelope.code.count("return {") == 1+ assert '"saved_as": dl_info.value.suggested_filename' in envelope.code+ assert "downloaded_file_name" not in envelope.code+ artifact = {"claimed_outcomes": [{"id": "claim", "goal_value_paths": ["output.downloaded_invoice_pdf"]}]}+ assert workflow_update_module._download_return_shape_error("block", artifact, envelope.code) is None+ assert {"output.statement_amount"} <= workflow_update_module._code_block_produced_output_paths(envelope.code)diff --git a/tests/unit/test_copilot_completion_verification.py b/tests/unit/test_copilot_completion_verification.pyindex 592a9029f..e851ad6fe 100644--- a/tests/unit/test_copilot_completion_verification.py+++ b/tests/unit/test_copilot_completion_verification.py@@ -29,6 +29,7 @@ from skyvern.forge.sdk.copilot.completion_criteria_store import (from skyvern.forge.sdk.copilot.completion_output_grounding import (_schema_boolean_output_paths,_value_matches_expected,+ floor_rekeyed_path_backing,grade_requested_output_criteria,split_requested_output_criteria,)@@ -39,6 +40,7 @@ from skyvern.forge.sdk.copilot.completion_verification import (DeliveredUnverifiedTerminalState,EvidenceSourceKind,FloorRekeyedDeliverableCredit,+ FloorRekeyedEmissionWithhold,RunEvidenceSnapshot,_coerce_result,_structured_record_has_identifier,@@ -48,6 +50,8 @@ from skyvern.forge.sdk.copilot.completion_verification import (degraded_contract_delivered_unverified_terminal_state,evaluate_completion_criteria,floor_rekeyed_deliverable_credit,+ floor_rekeyed_emission_lane_fields,+ floor_rekeyed_emission_withhold,grade_definition_criteria,grade_fallback_floor_reached_end_state_criteria,grade_present_value_criteria,@@ -126,6 +130,7 @@ from skyvern.forge.sdk.copilot.tools.completion import (_POST_RUN_PAGE_OBSERVATION_LABEL,_artifact_health_blocker_from_result,_completion_verification_from_run_result,+ _floor_rekeyed_emission_evidence,_reconcile_download_completion_criterion,)from skyvern.forge.sdk.copilot.tools.composition_capture import _active_run_terminal_monitor_enabled@@ -3448,6 +3453,7 @@ def _floor_rekeyed_result(verdicts: list[CriterionVerdict],marked_ids: list[str],output_path_by_id: dict[str, str] | None = None,+ backed_by_id: dict[str, bool] | None = None,) -> CompletionVerificationResult:return CompletionVerificationResult(status="evaluated",@@ -3455,6 +3461,7 @@ def _floor_rekeyed_result(verdicts=list(verdicts),floor_rekeyed_criterion_ids=list(marked_ids),floor_rekeyed_output_path_by_criterion_id=dict(output_path_by_id or {}),+ floor_rekeyed_backed_by_criterion_id=dict(backed_by_id or {}),)@@ -3670,6 +3677,7 @@ def test_floor_rekeyed_credit_grants_when_floor_path_binds_delivered_payload() -_floored_run_plane_verdicts(corroborator_source="independent_page_evidence"),["deliverable"],{"deliverable": "output.document_name"},+ backed_by_id={"deliverable": True},)with capture_logs() as logs:@@ -3692,6 +3700,7 @@ def test_floor_rekeyed_credit_withheld_when_floor_path_absent_from_payload() ->_floored_run_plane_verdicts(corroborator_source="independent_page_evidence"),["deliverable"],{"deliverable": "output.document_name"},+ backed_by_id={"deliverable": True},)with capture_logs() as logs:@@ -3709,7 +3718,7 @@ def test_floor_rekeyed_credit_withheld_when_floor_path_absent_from_payload() ->assert any(entry.get("event") == "copilot.completion.zero_requested_output_credit_withheld" for entry in logs)-def test_floor_rekeyed_deliverable_reaches_verified_success() -> None:+def test_floor_rekeyed_deliverable_withholds_without_runtime_backing() -> None:ctx = _ctx_with_blocks("code")verification = _floor_rekeyed_result(_corroborated_abstention_verdicts(@@ -3717,6 +3726,7 @@ def test_floor_rekeyed_deliverable_reaches_verified_success() -> None:corroborator_evidence_ref=_PAGE_EVIDENCE_DELIVERABLE_REF,),["deliverable"],+ {"deliverable": "output.document_name"},)with capture_logs() as logs:@@ -3725,13 +3735,15 @@ def test_floor_rekeyed_deliverable_reaches_verified_success() -> None:)assert recorded is not None- assert recorded.verdict == "demonstrated"- assert ctx.delivered_unverified_terminal is False- assert ctx.turn_halt is None- granted = [entry for entry in logs if entry.get("event") == "copilot.completion.floor_rekeyed_deliverable_credit"]- assert granted and granted[0]["criterion_ids"] == ["deliverable"]- assert granted[0]["evidence_sources"] == ["independent_page_evidence"]- assert not any(entry.get("event") == "copilot.completion.zero_requested_output_credit_withheld" for entry in logs)+ assert recorded.verdict == "not_evaluated"+ assert ctx.delivered_unverified_terminal is True+ assert ctx.turn_halt is not None+ assert ctx.turn_halt.kind.value == "delivered_unverified"+ assert not any(entry.get("event") == "copilot.completion.floor_rekeyed_deliverable_credit" for entry in logs)+ lane = [entry for entry in logs if entry.get("event") == "copilot.completion.floor_rekeyed_emission_lane"]+ assert lane and lane[0]["engaged"] is True+ assert lane[0]["unbacked_criterion_ids"] == ["deliverable"]+ assert any(entry.get("event") == "copilot.completion.floor_rekeyed_emission_withheld" for entry in logs)def _floored_policy_criteria(criteria: list[CompletionCriterion], user_message: str) -> list[CompletionCriterion]:@@ -3821,6 +3833,16 @@ async def test_floor_rekeyed_deliverable_is_credited_through_the_verification_pr)ctx = _ctx_with_blocks("code")ctx.request_policy = policy+ ctx.last_workflow_yaml = textwrap.dedent(+ """+ workflow_definition:+ blocks:+ - block_type: code+ label: extract_bill+ code: |+ return {"doc_status": "Delivered"}+ """+ ).strip()ctx.composition_page_evidence = {"workflow_run_id": "wr_x","observed_after_workflow_run": True,@@ -3862,10 +3884,456 @@ def test_floor_rekeyed_runtime_output_only_still_withholds() -> None:assert ctx.delivered_unverified_terminal is Trueassert ctx.turn_halt is not Noneassert ctx.turn_halt.kind.value == "delivered_unverified"- assert any(entry.get("event") == "copilot.completion.zero_requested_output_credit_withheld" for entry in logs)+ assert any(entry.get("event") == "copilot.completion.floor_rekeyed_emission_withheld" for entry in logs)+ assert not any(entry.get("event") == "copilot.completion.floor_rekeyed_deliverable_credit" for entry in logs)+++def _backing_ctx(returns_literal: str, block_label: str = "extract_bill") -> CopilotContext:+ ctx = _run_ctx()+ ctx.last_workflow = SimpleNamespace(+ workflow_definition=SimpleNamespace(blocks=[SimpleNamespace(label=block_label, block_type="code")])+ )+ ctx.last_workflow_yaml = textwrap.dedent(+ f"""+ workflow_definition:+ blocks:+ - block_type: code+ label: {block_label}+ code: |+ return {returns_literal}+ """+ ).strip()+ return ctx+++def _marked_output_criterion(path: str, cid: str = "c1") -> CompletionCriterion:+ return CompletionCriterion(+ id=cid,+ outcome="The requested deliverable is returned.",+ level="run",+ requested_output_floor_rekeyed=True,+ floor_rekeyed_from_path=path,+ )+++def _backing_for(ctx: CopilotContext, criteria: list[CompletionCriterion], run_data: dict) -> dict[str, bool]:+ block_outputs, sources, block_types, runtime_envelope_labels = _floor_rekeyed_emission_evidence(ctx, run_data)+ return floor_rekeyed_path_backing(+ ctx, criteria, block_outputs, sources, block_types, runtime_envelope_labels=runtime_envelope_labels+ )+++def test_backing_backs_registered_scalar_at_floor_path() -> None:+ ctx = _backing_ctx('{"doc_status": "Delivered"}')+ run_data = _registered_output_result({"doc_status": "Delivered"})["data"]++ backing = _backing_for(ctx, [_marked_output_criterion("output.doc_status")], run_data)++ assert backing == {"c1": True}+++def test_backing_does_not_back_metadata_only_envelope() -> None:+ envelope_type = next(iter(_TASK_ENVELOPE_BLOCK_TYPES))+ ctx = _backing_ctx('{"doc_status": "Delivered"}')+ envelope = {+ "task_id": "t1",+ "status": "completed",+ "extracted_information": None,+ "downloaded_files": None,+ "downloaded_file_urls": None,+ }+ run_data = _registered_output_result(envelope, block_type=envelope_type)["data"]++ backing = _backing_for(ctx, [_marked_output_criterion("output.doc_status")], run_data)++ assert backing == {"c1": False}+++def _task_envelope_ctx(label: str = "run_task") -> CopilotContext:+ ctx = _run_ctx()+ ctx.last_workflow = SimpleNamespace(+ workflow_definition=SimpleNamespace(blocks=[SimpleNamespace(label=label, block_type="task_v2")])+ )+ ctx.last_workflow_yaml = textwrap.dedent(+ f"""+ workflow_definition:+ blocks:+ - block_type: task_v2+ label: {label}+ """+ ).strip()+ return ctx+++def _task_envelope_run_data(label: str, extracted_information: dict[str, Any] | None) -> dict:+ return {+ "workflow_run_id": "wr_x",+ "overall_status": "completed",+ "executed_block_labels": [label],+ "current_url": "https://example.com/account",+ "blocks": [+ {+ "label": label,+ "block_type": "task_v2",+ "status": "completed",+ "extracted_data": {+ "task_id": "t1",+ "status": "completed",+ "extracted_information": extracted_information,+ "downloaded_files": None,+ "downloaded_file_urls": None,+ },+ }+ ],+ "registered_output_parameter_values": [],+ }+++def test_backing_backs_contract_less_task_envelope_extracted_information() -> None:+ ctx = _task_envelope_ctx()+ run_data = _task_envelope_run_data("run_task", {"job_title": "Staff Engineer"})++ backing = _backing_for(ctx, [_marked_output_criterion("output.job_title")], run_data)++ assert backing == {"c1": True}+++def test_backing_does_not_back_metadata_only_task_envelope() -> None:+ ctx = _task_envelope_ctx()+ run_data = _task_envelope_run_data("run_task", None)++ backing = _backing_for(ctx, [_marked_output_criterion("output.job_title")], run_data)++ assert backing == {"c1": False}+++def _mixed_authored_and_envelope_ctx() -> CopilotContext:+ ctx = _run_ctx()+ ctx.last_workflow = SimpleNamespace(+ workflow_definition=SimpleNamespace(+ blocks=[+ SimpleNamespace(label="extract_bill", block_type="code"),+ SimpleNamespace(label="run_task", block_type="task_v2"),+ ]+ )+ )+ ctx.last_workflow_yaml = textwrap.dedent(+ """+ workflow_definition:+ blocks:+ - block_type: code+ label: extract_bill+ code: |+ return {"job_title": "authored placeholder"}+ - block_type: task_v2+ label: run_task+ """+ ).strip()+ return ctx+++def test_backing_does_not_cross_back_authored_marker_from_foreign_task_envelope() -> None:+ ctx = _mixed_authored_and_envelope_ctx()+ run_data = {+ "workflow_run_id": "wr_x",+ "overall_status": "completed",+ "executed_block_labels": ["extract_bill", "run_task"],+ "current_url": "https://example.com/account",+ "blocks": [+ {+ "label": "extract_bill",+ "block_type": "code",+ "status": "completed",+ "extracted_data": {"note": "done"},+ },+ {+ "label": "run_task",+ "block_type": "task_v2",+ "status": "completed",+ "extracted_data": {+ "task_id": "t1",+ "status": "completed",+ "extracted_information": {"job_title": "Staff Engineer"},+ "downloaded_files": None,+ "downloaded_file_urls": None,+ },+ },+ ],+ "registered_output_parameter_values": [],+ }++ backing = _backing_for(ctx, [_marked_output_criterion("output.job_title")], run_data)++ assert backing == {"c1": False}+++def test_backing_does_not_back_unrelated_field() -> None:+ ctx = _backing_ctx('{"doc_status": "Delivered"}')+ run_data = _registered_output_result({"doc_status": "Delivered"})["data"]++ backing = _backing_for(ctx, [_marked_output_criterion("output.unrelated_field")], run_data)++ assert backing == {"c1": False}+++def _two_contract_less_task_envelope_ctx() -> CopilotContext:+ ctx = _run_ctx()+ ctx.last_workflow = SimpleNamespace(+ workflow_definition=SimpleNamespace(+ blocks=[+ SimpleNamespace(label="run_task", block_type="task_v2"),+ SimpleNamespace(label="run_task_2", block_type="task_v2"),+ ]+ )+ )+ ctx.last_workflow_yaml = textwrap.dedent(+ """+ workflow_definition:+ blocks:+ - block_type: task_v2+ label: run_task+ - block_type: task_v2+ label: run_task_2+ """+ ).strip()+ return ctx+++def test_backing_fails_closed_across_ambiguous_contract_less_task_envelopes() -> None:+ # The criterion's real producer (run_task) emits nothing at the path; an unrelated task+ # envelope (run_task_2) coincidentally emits it. With no authored contract naming the+ # producer and more than one candidate envelope, the emission is unattributable and must+ # fail closed rather than pool the coincidental match.+ ctx = _two_contract_less_task_envelope_ctx()+ run_data = {+ "workflow_run_id": "wr_x",+ "overall_status": "completed",+ "executed_block_labels": ["run_task", "run_task_2"],+ "current_url": "https://example.com/account",+ "blocks": [+ {+ "label": "run_task",+ "block_type": "task_v2",+ "status": "completed",+ "extracted_data": {+ "task_id": "t1",+ "status": "completed",+ "extracted_information": None,+ "downloaded_files": None,+ "downloaded_file_urls": None,+ },+ },+ {+ "label": "run_task_2",+ "block_type": "task_v2",+ "status": "completed",+ "extracted_data": {+ "task_id": "t2",+ "status": "completed",+ "extracted_information": {"job_title": "Staff Engineer"},+ "downloaded_files": None,+ "downloaded_file_urls": None,+ },+ },+ ],+ "registered_output_parameter_values": [],+ }++ backing = _backing_for(ctx, [_marked_output_criterion("output.job_title")], run_data)++ assert backing == {"c1": False}+++def test_emission_evidence_gates_hollow_top_level_output() -> None:+ # A hollow top-level ``*_output`` payload must not enter the emission view as backing+ # evidence — the same meaningfulness gate _build_run_evidence_snapshot applies.+ ctx = _backing_ctx('{"doc_status": "Delivered"}')+ run_data = {+ "workflow_run_id": "wr_x",+ "overall_status": "completed",+ "executed_block_labels": [],+ "current_url": "https://example.com/account",+ "blocks": [],+ "output": {"extract_bill_output": {"extracted_information": None}},+ "registered_output_parameter_values": [],+ }++ block_outputs, _sources, _types, _envs = _floor_rekeyed_emission_evidence(ctx, run_data)++ assert "extract_bill_output" not in block_outputs+++def test_backing_empty_artifact_is_unbacked() -> None:+ ctx = _backing_ctx('{"doc_status": "Delivered"}')++ backing = _backing_for(ctx, [_marked_output_criterion("output.doc_status")], None)++ assert backing == {"c1": False}+++def test_emission_evidence_excludes_page_observation_sources() -> None:+ ctx = _backing_ctx('{"doc_status": "Delivered"}')+ run_data = _registered_output_result({"doc_status": "Delivered"})["data"]++ _, sources, _types, _envs = _floor_rekeyed_emission_evidence(ctx, run_data)++ assert sources+ assert set(sources.values()) <= {"runtime_output", "registered_output_parameter"}+++def _emission_result(+ *,+ backed: dict[str, bool],+ paths: dict[str, str] | None = None,+ marked: tuple[str, ...] = ("c1",),+ contingent: tuple[str, ...] = (),+ structural_unfired: tuple[str, ...] = (),+ extra_verdicts: tuple[CriterionVerdict, ...] = (),+) -> CompletionVerificationResult:+ verdicts = [+ CriterionVerdict(+ criterion_id=cid, state="satisfied", reason_code="evidence_confirms", evidence_ref="observed_end_state_url"+ )+ for cid in marked+ ]+ criterion_ids = [*marked, *(verdict.criterion_id for verdict in extra_verdicts)]+ return CompletionVerificationResult(+ status="evaluated",+ criterion_ids=criterion_ids,+ verdicts=[*verdicts, *extra_verdicts],+ floor_rekeyed_criterion_ids=list(marked),+ floor_rekeyed_output_path_by_criterion_id=dict(paths or {}),+ floor_rekeyed_backed_by_criterion_id=dict(backed),+ contingent_criterion_ids=list(contingent),+ structural_unfired_criterion_ids=list(structural_unfired),+ )+++def test_floor_rekeyed_emission_withhold_engages_when_marker_unbacked() -> None:+ withhold = floor_rekeyed_emission_withhold(+ _emission_result(backed={"c1": False}, paths={"c1": "output.doc_status"})+ )++ assert isinstance(withhold, FloorRekeyedEmissionWithhold)+ assert withhold.criterion_ids == ("c1",)+ assert withhold.unbacked_output_paths == ("output.doc_status",)+++def test_floor_rekeyed_emission_withhold_none_when_backed() -> None:+ assert floor_rekeyed_emission_withhold(_emission_result(backed={"c1": True})) is None+++def test_floor_rekeyed_emission_withhold_missing_entry_fails_closed() -> None:+ assert floor_rekeyed_emission_withhold(_emission_result(backed={})) is not None+++def test_floor_rekeyed_emission_withhold_ignores_empty_marker_set() -> None:+ reach_state = _evaluated(("login", True))++ assert floor_rekeyed_emission_withhold(reach_state) is None+++def test_floor_rekeyed_emission_withhold_excludes_structurally_unfired_contingent_marker() -> None:+ result = _emission_result(+ backed={},+ marked=("c1",),+ contingent=("c1",),+ structural_unfired=("c1",),+ extra_verdicts=(+ CriterionVerdict(+ criterion_id="ok", state="satisfied", reason_code="evidence_confirms", evidence_ref="observed_end_state"+ ),+ ),+ )++ assert result.is_fully_satisfied() is True+ assert floor_rekeyed_emission_withhold(result) is None+++def test_floor_rekeyed_emission_withhold_keyed_on_marker_not_artifact_richness() -> None:+ lean = _emission_result(backed={"c1": False})+ rich = _emission_result(+ backed={"c1": False},+ extra_verdicts=(+ CriterionVerdict(+ criterion_id="ok", state="satisfied", reason_code="evidence_confirms", evidence_ref="observed_end_state"+ ),+ ),+ )++ assert floor_rekeyed_emission_withhold(lean) is not None+ assert floor_rekeyed_emission_withhold(rich) is not None+++def test_floor_rekeyed_emission_lane_fields_fire_on_satisfied_and_unsatisfied_results() -> None:+ satisfied = _emission_result(backed={"c1": False}, paths={"c1": "output.doc_status"})+ unsatisfied = CompletionVerificationResult(+ status="evaluated",+ criterion_ids=["c1"],+ verdicts=[CriterionVerdict(criterion_id="c1", state="unsatisfied", reason_code="no_evidence")],+ floor_rekeyed_criterion_ids=["c1"],+ floor_rekeyed_backed_by_criterion_id={"c1": False},+ )++ satisfied_fields = floor_rekeyed_emission_lane_fields(satisfied)+ unsatisfied_fields = floor_rekeyed_emission_lane_fields(unsatisfied)++ assert satisfied_fields is not None and satisfied_fields["engaged"] is True+ assert unsatisfied_fields is not None and unsatisfied_fields["engaged"] is False+++def test_floor_rekeyed_emission_shallow_bound_but_unbacked_withholds() -> None:+ ctx = _ctx_with_blocks("code")+ verification = _floor_rekeyed_result(+ _floored_run_plane_verdicts(corroborator_source="independent_page_evidence"),+ ["deliverable"],+ {"deliverable": "output.document_name"},+ backed_by_id={"deliverable": False},+ )++ with capture_logs() as logs:+ recorded = _record_run_blocks_result(+ ctx,+ _registered_output_result({"document_name": "Resale certificate"}),+ completion_verification=verification,+ )++ assert recorded is not None+ assert recorded.verdict == "not_evaluated"+ assert ctx.delivered_unverified_terminal is True+ lane = [entry for entry in logs if entry.get("event") == "copilot.completion.floor_rekeyed_emission_lane"]+ assert lane and lane[0]["engaged"] is Trueassert not any(entry.get("event") == "copilot.completion.floor_rekeyed_deliverable_credit" for entry in logs)+def test_registered_output_meaningfulness_gate_drops_hollow_envelope() -> None:+ envelope_type = next(iter(_TASK_ENVELOPE_BLOCK_TYPES))+ ctx = _ctx_with_blocks("code")+ hollow = _registered_output_result(+ {+ "task_id": "t",+ "status": "completed",+ "extracted_information": None,+ "downloaded_files": None,+ "downloaded_file_urls": None,+ },+ block_type=envelope_type,+ )++ snapshot = _build_run_evidence_snapshot(ctx, hollow)++ assert "bill_statement" not in snapshot.block_outputs+++def test_registered_output_meaningfulness_gate_keeps_real_scalar() -> None:+ ctx = _ctx_with_blocks("code")+ real = _registered_output_result({"doc_status": "Delivered"})++ snapshot = _build_run_evidence_snapshot(ctx, real)++ assert snapshot.block_outputs.get("bill_statement") == {"doc_status": "Delivered"}++def test_zero_requested_output_criteria_withholds_verified_success() -> None:ctx = _ctx_with_blocks("code")diff --git a/tests/unit/test_forge_log_foreign_tracebacks.py b/tests/unit/test_forge_log_foreign_tracebacks.pynew file mode 100644index 000000000..41c29ef06--- /dev/null+++ b/tests/unit/test_forge_log_foreign_tracebacks.py@@ -0,0 +1,73 @@+from __future__ import annotations++import io+import json+import logging+from collections.abc import Iterator++import pytest+import structlog++from skyvern.config import settings+from skyvern.forge.sdk.forge_log import setup_logger+++def json_stream(monkeypatch: pytest.MonkeyPatch) -> Iterator[io.StringIO]:+ monkeypatch.setattr(settings, "JSON_LOGGING", True)+ setup_logger()+ handler = logging.getLogger().handlers[0]+ assert isinstance(handler, logging.StreamHandler)+ stream = io.StringIO()+ previous = handler.setStream(stream)+ try:+ yield stream+ finally:+ handler.setStream(previous)+++def _raise_through_wrapper() -> None:+ def async_wrapper() -> None:+ raise ValueError("kaboom from an activity")++ async_wrapper()+++def test_foreign_stdlib_exception_renders_single_structured_line(json_stream: io.StringIO) -> None:+ """temporalio/asyncio emit stdlib records; their exc_info must collapse to one JSON entry."""+ logger = logging.getLogger("temporalio.activity")+ try:+ _raise_through_wrapper()+ except ValueError:+ logger.warning("Completing activity as failed", exc_info=True)++ lines = json_stream.getvalue().strip().splitlines()+ assert len(lines) == 1+ record = json.loads(lines[0])++ assert record["logger"] == "temporalio.activity"+ assert "exc_info" not in record # the raw (traceback, ...) tuple is never dumped+ assert "<traceback object at" not in json_stream.getvalue()+ assert "Traceback (most recent call last)" in record["exception"]+ assert "ValueError: kaboom from an activity" in record["exception"]+ assert "async_wrapper" in record["exception"]+ assert record["error_type"] == "builtins.ValueError"+ assert record["error_category"] == "ERROR"+ assert record["exception_hash"]+++def test_native_structlog_exception_not_double_processed(json_stream: io.StringIO) -> None:+ logger = structlog.get_logger("skyvern.foreign_traceback_test")+ try:+ _raise_through_wrapper()+ except ValueError:+ logger.exception("native boom")++ lines = json_stream.getvalue().strip().splitlines()+ assert len(lines) == 1+ record = json.loads(lines[0])++ assert "exc_info" not in record+ assert "Traceback (most recent call last)" in record["exception"]+ assert record["exception"].count("Traceback (most recent call last)") == 1+ assert record["error_type"] == "builtins.ValueError"