Skip to content

Automate browser based workflows with AI

Don't lose the record of a downloaded file when a code block fails or auto-recovers (SKY-13694) (#8167)

Andrew Neilsona043dbb09ad7
diff --git a/skyvern/forge/sdk/copilot/tools/completion.py b/skyvern/forge/sdk/copilot/tools/completion.py
index ea6a39e35..5f0261d8a 100644
--- a/skyvern/forge/sdk/copilot/tools/completion.py
+++ b/skyvern/forge/sdk/copilot/tools/completion.py
@@ -808,14 +808,12 @@ def _download_file_name(value: Any) -> str | None:
def _completion_evidence_payload(output: Any) -> Any:
if not isinstance(output, dict):
return output
- nested_output = output.get("output")
+ # Root scope only: the execution layer binds registration keys at the root of every block
+ # output, so keys found only in a nested scope are the code's own claim, not registration.
has_download_output = any(output.get(key) for key in REGISTERED_DOWNLOAD_OUTPUT_KEYS)
- has_nested_download_output = isinstance(nested_output, dict) and any(
- nested_output.get(key) for key in REGISTERED_DOWNLOAD_OUTPUT_KEYS
- )
- if not has_download_output and not has_nested_download_output:
+ if not has_download_output:
return output
- download_output = output if has_download_output or not isinstance(nested_output, dict) else nested_output
+ download_output = output
names: list[str] = []
if name := _download_file_name(download_output.get("downloaded_file_name")):
names.append(name)
@@ -835,14 +833,6 @@ def _completion_evidence_payload(output: Any) -> Any:
payload["downloaded_file_artifact_count"] = len(artifacts)
if names:
payload["downloaded_file_names"] = list(dict.fromkeys(names))[:_MAX_EVIDENCE_FILE_NAMES]
- if has_nested_download_output and not has_download_output and isinstance(nested_output, dict):
- preserved_output = {
- key: value
- for key, value in nested_output.items()
- if key not in REGISTERED_DOWNLOAD_OUTPUT_KEYS and key not in {"page", "download"}
- }
- if preserved_output:
- payload["output"] = preserved_output
return payload
diff --git a/skyvern/forge/sdk/copilot/tools/run_execution.py b/skyvern/forge/sdk/copilot/tools/run_execution.py
index ed041ad38..ed6658a59 100644
--- a/skyvern/forge/sdk/copilot/tools/run_execution.py
+++ b/skyvern/forge/sdk/copilot/tools/run_execution.py
@@ -135,6 +135,8 @@ from skyvern.forge.sdk.utils.pdf_parser import extract_pdf_file
from skyvern.forge.sdk.workflow.models.block import CodeBlock
from skyvern.forge.sdk.workflow.models.parameter import OutputParameter, WorkflowParameter
from skyvern.forge.sdk.workflow.models.workflow import Workflow, WorkflowRun, WorkflowRunStatus
+from skyvern.forge.sdk.workflow.runtime_completion import contract_from_request_criteria
+from skyvern.forge.sdk.workflow.service import run_selection_is_partial
from skyvern.schemas.workflows import BlockType
from skyvern.utils.files import initialize_skyvern_state_file
from skyvern.webeye.navigation import is_skip_inner_retry_error
@@ -1054,6 +1056,24 @@ def _selected_code_security_inputs(
return code_blocks
+def _requested_completion_contract(
+ ctx: CopilotContext,
+ runtime_workflow: Workflow,
+ labels_to_execute: list[str],
+) -> dict[str, object] | None:
+ """The turn's own deliverable obligation, for a test run that executes the whole workflow.
+
+ The obligation attaches to the workflow only when a proposal is accepted, which is after the
+ test run this grades; a selection that runs only part of the workflow stays ungraded."""
+ if ctx.request_policy is None:
+ return None
+ if not runtime_workflow.workflow_definition.blocks:
+ return None
+ if run_selection_is_partial(runtime_workflow, labels_to_execute):
+ return None
+ return contract_from_request_criteria(ctx.request_policy.graded_completion_criteria())
+
+
def _selected_blocks_require_sandbox(
blocks: list[Any],
*,
@@ -1852,6 +1872,16 @@ async def _run_blocks_and_collect_debug(
runtime_frontier_starter_url_seeded = seeded_runtime_workflow is not runtime_workflow
runtime_workflow = seeded_runtime_workflow
+ requested_completion_contract = _requested_completion_contract(ctx, runtime_workflow, labels_to_execute)
+ if requested_completion_contract is not None:
+ runtime_workflow = runtime_workflow.model_copy(
+ update={
+ "workflow_definition": runtime_workflow.workflow_definition.model_copy(
+ update={"completion_contract": requested_completion_contract}
+ )
+ }
+ )
+
# Snapshot version persisted for a worker-dispatched run or an inline run sourced from an
# unsaved prior draft. The run is created against its exact workflow_id so prepare_workflow
# reads parameter rows from the same definition execute_workflow receives. Without the inline
@@ -2004,6 +2034,7 @@ async def _run_blocks_and_collect_debug(
block_labels=labels_to_execute,
block_outputs=block_outputs_to_seed or None,
workflow_override=runtime_workflow,
+ requested_completion_contract=requested_completion_contract,
)
)
except BaseException:
diff --git a/skyvern/forge/sdk/workflow/models/block.py b/skyvern/forge/sdk/workflow/models/block.py
index fe7be9ca2..bb7e68b83 100644
--- a/skyvern/forge/sdk/workflow/models/block.py
+++ b/skyvern/forge/sdk/workflow/models/block.py
@@ -114,6 +114,7 @@ from skyvern.forge.sdk.api.llm.schema_validator import validate_schema
from skyvern.forge.sdk.artifact.models import ArtifactType
from skyvern.forge.sdk.copilot.block_goal_wrapping import compose_mini_goal
from skyvern.forge.sdk.copilot.reached_download_target import (
+ REGISTERED_DOWNLOAD_OUTPUT_KEYS,
block_output_has_registered_download,
code_is_download_intent,
)
@@ -309,6 +310,61 @@ async def capture_block_download_baseline(
)
+DOWNLOAD_BINDING_FAILURE_REASON = (
+ "A file downloaded into the run directory but no registered download landed in the block output."
+)
+UNBOUND_DOWNLOAD_OUTPUT_KEY = "download_registration_failed"
+
+
+def bind_downloaded_files_to_output(result: Any, downloaded_files: list[FileInfo]) -> Any:
+ """Attach registration evidence to whatever the block's code returned.
+
+ The host's ``FileInfo`` list is the evidence of record, so it overwrites any registration-shaped
+ keys the executed code authored for itself. With no host evidence those keys are dropped rather
+ than left to read as a registration that never happened. The caller's dict is never mutated —
+ the secure arm passes the sidecar result's own payload."""
+ if not downloaded_files:
+ # Only a truthy authored value is a claim to drop; a None or empty field is schema, and
+ # removing it would break templates that dereference it under strict rendering.
+ if isinstance(result, dict) and any(result.get(key) for key in REGISTERED_DOWNLOAD_OUTPUT_KEYS):
+ result = {
+ key: value for key, value in result.items() if not (key in REGISTERED_DOWNLOAD_OUTPUT_KEYS and value)
+ }
+ return result
+ if not isinstance(result, dict):
+ result = {"value": result} if result is not None else {}
+ else:
+ result = dict(result)
+ result["downloaded_files"] = [file_info.model_dump() for file_info in downloaded_files]
+ result["downloaded_file_urls"] = [file_info.url for file_info in downloaded_files]
+ result["downloaded_file_artifact_ids"] = [
+ file_info.artifact_id for file_info in downloaded_files if file_info.artifact_id
+ ]
+ return result
+
+
+def local_download_dir_file_names(download_run_id: str | None) -> set[str] | None:
+ """Names of the files currently sitting in the run's local download directory.
+
+ Host-side evidence the executing code cannot author: a snapshot taken before the block and one
+ after it discriminate a download this block produced from files earlier blocks left behind.
+ ``None`` means the directory could not be read — an unknown snapshot, never an empty one, so a
+ failed read cannot make pre-existing files look new."""
+ if not download_run_id:
+ return None
+ try:
+ return {entry.name for entry in Path(get_download_dir(download_run_id)).iterdir() if entry.is_file()}
+ except Exception:
+ return None
+
+
+def unbound_download_output(result: Any) -> dict[str, Any]:
+ """A non-null payload for a block whose download never bound, carrying no registration keys."""
+ payload = dict(result) if isinstance(result, dict) else ({"value": result} if result is not None else {})
+ payload[UNBOUND_DOWNLOAD_OUTPUT_KEY] = True
+ return payload
+
+
if settings.WORKFLOW_TEMPLATING_STRICTNESS == "strict":
jinja_sandbox_env = SandboxedEnvironment(undefined=StrictUndefined)
else:
@@ -4436,11 +4492,15 @@ async def wrapper({default_args}):
workflow_run_id: str,
workflow_run_block_id: str,
download_run_id: str | None = None,
- ) -> list[FileInfo]:
+ ) -> list[FileInfo] | None:
+ """Registered downloads, or ``None`` when registration could not complete.
+
+ ``None`` is not an empty registration: workflow finalization retries the save, so a caller
+ must not read a storage timeout as proof that nothing was downloaded."""
# Register up front so the block output carries downloaded_file_urls for
# downstream blocks in the same run; workflow finalization re-runs the save safely.
if not organization_id:
- return []
+ return None
storage_run_id = download_run_id or workflow_run_id
try:
async with asyncio.timeout(SAVE_DOWNLOADED_FILES_TIMEOUT):
@@ -4454,7 +4514,7 @@ async def wrapper({default_args}):
workflow_run_id=workflow_run_id,
workflow_run_block_id=workflow_run_block_id,
)
- return []
+ return None
except Exception:
LOG.warning(
"CodeBlock failed to register downloaded files; will retry at workflow finalization",
@@ -4462,12 +4522,30 @@ async def wrapper({default_args}):
workflow_run_block_id=workflow_run_block_id,
exc_info=True,
)
- return []
+ return None
+ return await self._read_back_downloaded_files(
+ organization_id=organization_id,
+ workflow_run_id=workflow_run_id,
+ workflow_run_block_id=workflow_run_block_id,
+ download_run_id=storage_run_id,
+ )
+
+ async def _read_back_downloaded_files(
+ self,
+ *,
+ organization_id: str | None,
+ workflow_run_id: str,
+ workflow_run_block_id: str,
+ download_run_id: str | None = None,
+ ) -> list[FileInfo] | None:
+ """Registered downloads read back from storage, or ``None`` when the read could not complete."""
+ if not organization_id:
+ return None
try:
async with asyncio.timeout(GET_DOWNLOADED_FILES_TIMEOUT):
return await app.STORAGE.get_downloaded_files(
organization_id=organization_id,
- run_id=storage_run_id,
+ run_id=download_run_id or workflow_run_id,
)
except asyncio.TimeoutError:
LOG.warning(
@@ -4475,7 +4553,7 @@ async def wrapper({default_args}):
workflow_run_id=workflow_run_id,
workflow_run_block_id=workflow_run_block_id,
)
- return []
+ return None
except Exception:
LOG.warning(
"CodeBlock failed to read back downloaded files; will retry at workflow finalization",
@@ -4483,7 +4561,95 @@ async def wrapper({default_args}):
workflow_run_block_id=workflow_run_block_id,
exc_info=True,
)
- return []
+ return None
+
+ async def _bind_and_grade_downloads(
+ self,
+ *,
+ engine: str,
+ result: Any,
+ downloaded_files: list[FileInfo] | None,
+ download_dir_before: set[str] | None,
+ resolved_download_id: str | None,
+ workflow_run_context: WorkflowRunContext,
+ workflow_run_id: str,
+ workflow_run_block_id: str,
+ organization_id: str | None,
+ authored_registration: bool = False,
+ ) -> tuple[Any, str | None]:
+ """Bind registration evidence into the block output, returning a failure reason if it cannot.
+
+ A file that appeared in the run download directory while this block ran, with nothing
+ registered to bind, is a download the run can never account for — the block reports that
+ rather than persisting an output that reads as if no download happened. The verdict needs
+ evidence on both sides: when registration or either directory snapshot could not be read,
+ the block is not accused of swallowing a download that storage merely failed to record."""
+ organization_id = organization_id or workflow_run_context.organization_id
+ if downloaded_files is None:
+ LOG.warning(
+ "codeblock.download_binding_verdict_skipped",
+ engine=engine,
+ reason="registration_incomplete",
+ workflow_run_id=workflow_run_id,
+ workflow_run_block_id=workflow_run_block_id,
+ block_label=self.label,
+ organization_id=organization_id,
+ )
+ # No host verdict at all, so the payload is left exactly as the code returned it:
+ # dropping its keys would assert an absence just as unproven as the claim.
+ # Storage trouble is exactly when this diagnostic matters, so it fires here too.
+ await self._record_unregistered_download_intent(
+ engine=engine,
+ registered=authored_registration,
+ output=result,
+ workflow_run_context=workflow_run_context,
+ workflow_run_id=workflow_run_id,
+ workflow_run_block_id=workflow_run_block_id,
+ organization_id=organization_id,
+ )
+ return result, None
+ current_context = skyvern_context.current()
+ downloaded_files = filter_downloaded_files_for_current_iteration(
+ downloaded_files,
+ current_context.loop_internal_state if current_context else None,
+ )
+ output = bind_downloaded_files_to_output(result, downloaded_files)
+ await self._record_unregistered_download_intent(
+ engine=engine,
+ registered=bool(downloaded_files) or authored_registration,
+ output=output,
+ workflow_run_context=workflow_run_context,
+ workflow_run_id=workflow_run_id,
+ workflow_run_block_id=workflow_run_block_id,
+ organization_id=organization_id,
+ )
+ if downloaded_files:
+ return output, None
+ download_dir_after = local_download_dir_file_names(resolved_download_id)
+ if download_dir_after is None or download_dir_before is None:
+ LOG.warning(
+ "codeblock.download_binding_verdict_skipped",
+ engine=engine,
+ reason="download_dir_snapshot_unreadable",
+ workflow_run_id=workflow_run_id,
+ workflow_run_block_id=workflow_run_block_id,
+ block_label=self.label,
+ organization_id=organization_id,
+ )
+ return output, None
+ new_file_names = download_dir_after - download_dir_before
+ if not new_file_names:
+ return output, None
+ LOG.error(
+ "codeblock.download_on_disk_unbound",
+ engine=engine,
+ workflow_run_id=workflow_run_id,
+ workflow_run_block_id=workflow_run_block_id,
+ block_label=self.label,
+ new_file_count=len(new_file_names),
+ organization_id=organization_id,
+ )
+ return unbound_download_output(output), DOWNLOAD_BINDING_FAILURE_REASON
async def _materialize_file_parameter_path(
self,
@@ -5243,6 +5409,62 @@ async def wrapper({default_args}):
exc_info=True,
)
+ async def _failure_output_with_downloads(
+ self,
+ *,
+ engine: str,
+ workflow_run_context: WorkflowRunContext,
+ workflow_run_id: str,
+ workflow_run_block_id: str,
+ organization_id: str | None,
+ resolved_download_id: str | None,
+ download_dir_before: set[str] | None,
+ result: dict[str, Any] | list | str | None = None,
+ ) -> dict[str, Any] | None:
+ """Registration evidence for a block that failed after a download already landed.
+
+ Returns None when nothing new reached the run directory, so a failure that downloaded
+ nothing keeps whatever output the caller supplied. The block is already failing for its own
+ reason, so the binding verdict is discarded here — only the evidence is kept, bound onto the
+ caller's failure payload when one exists."""
+ download_dir_after = local_download_dir_file_names(resolved_download_id)
+ if download_dir_after is None or download_dir_before is None:
+ return None
+ new_file_names = download_dir_after - download_dir_before
+ if not new_file_names:
+ return None
+ # Read back before saving so a sidecar run that already uploaded is not uploaded twice, but
+ # skip the save only when the read-back already accounts for what this block added — an
+ # earlier block's registration is not evidence for this one's file.
+ downloaded_files = await self._read_back_downloaded_files(
+ organization_id=organization_id or workflow_run_context.organization_id,
+ workflow_run_id=workflow_run_id,
+ workflow_run_block_id=workflow_run_block_id,
+ download_run_id=resolved_download_id,
+ )
+ registered_names = {file_info.filename for file_info in downloaded_files or []}
+ if not new_file_names.issubset(registered_names):
+ downloaded_files = await self._register_downloaded_files(
+ organization_id=organization_id or workflow_run_context.organization_id,
+ workflow_run_id=workflow_run_id,
+ workflow_run_block_id=workflow_run_block_id,
+ download_run_id=resolved_download_id,
+ )
+ output, _ = await self._bind_and_grade_downloads(
+ engine=engine,
+ result=result,
+ downloaded_files=downloaded_files,
+ download_dir_before=download_dir_before,
+ resolved_download_id=resolved_download_id,
+ workflow_run_context=workflow_run_context,
+ workflow_run_id=workflow_run_id,
+ workflow_run_block_id=workflow_run_block_id,
+ organization_id=organization_id,
+ )
+ if output is not None:
+ await self.record_output_parameter_value(workflow_run_context, workflow_run_id, output)
+ return output if isinstance(output, dict) else None
+
async def _failed_result_with_evidence(
self,
*,
@@ -5254,6 +5476,9 @@ async def wrapper({default_args}):
organization_id: str | None,
browser_state: BrowserState | None,
page: Page | None,
+ engine: str,
+ resolved_download_id: str | None,
+ download_dir_before: set[str] | None,
output_parameter_value: dict[str, Any] | list | str | None = None,
error_codes: list[str] | None = None,
) -> BlockResult:
@@ -5273,7 +5498,19 @@ async def wrapper({default_args}):
return await self.build_block_result(
success=False,
failure_reason=failure_reason,
- output_parameter_value=output_parameter_value,
+ output_parameter_value=(
+ await self._failure_output_with_downloads(
+ engine=engine,
+ workflow_run_context=workflow_run_context,
+ workflow_run_id=workflow_run_id,
+ workflow_run_block_id=workflow_run_block_id,
+ organization_id=organization_id,
+ resolved_download_id=resolved_download_id,
+ download_dir_before=download_dir_before,
+ result=output_parameter_value,
+ )
+ or output_parameter_value
+ ),
status=status,
workflow_run_block_id=workflow_run_block_id,
organization_id=organization_id,
@@ -5295,6 +5532,8 @@ async def wrapper({default_args}):
browser_session_id: str | None,
browser_state: BrowserState | None = None,
page: Page | None = None,
+ resolved_download_id: str | None = None,
+ download_dir_before: set[str] | None = None,
) -> BlockResult:
# Capture before the healable branch: a non-healable failure is exactly the case the
# repair loop has the least to go on.
@@ -5308,7 +5547,81 @@ async def wrapper({default_args}):
)
async def _finalize_heal_result(result: BlockResult | None) -> BlockResult:
+ # A healed block can still come out failed here: a download that landed with nothing
+ # registered fails the binding check exactly as it would on the normal success exit.
if result is not None:
+ if result.success:
+ # A healed block is still a block whose download must be accountable: the raise
+ # that triggered the heal fired after the file landed, and this path was the one
+ # exit that recorded the heal's own output without ever looking at the run
+ # directory (SKY-13694's completed-with-null arm).
+ try:
+ downloaded_files = await self._register_downloaded_files(
+ organization_id=organization_id or workflow_run_context.organization_id,
+ workflow_run_id=workflow_run_id,
+ workflow_run_block_id=workflow_run_block_id,
+ download_run_id=resolved_download_id,
+ )
+ bound_output, binding_failure_reason = await self._bind_and_grade_downloads(
+ engine="inline",
+ result=result.output_parameter_value,
+ downloaded_files=downloaded_files,
+ download_dir_before=download_dir_before,
+ resolved_download_id=resolved_download_id,
+ workflow_run_context=workflow_run_context,
+ workflow_run_id=workflow_run_id,
+ workflow_run_block_id=workflow_run_block_id,
+ organization_id=organization_id,
+ )
+ except asyncio.CancelledError:
+ # A cancel inside the storage round-trips must not book the healed block
+ # as failed or skip its billing hook; the verdict is moot on teardown.
+ await recorder.finalize(success=True)
+ raise
+ if binding_failure_reason is not None:
+ await self.record_output_parameter_value(workflow_run_context, workflow_run_id, bound_output)
+ await recorder.finalize(success=False)
+ return await self.build_block_result(
+ success=False,
+ failure_reason=binding_failure_reason,
+ output_parameter_value=bound_output,
+ status=BlockStatus.failed,
+ workflow_run_block_id=workflow_run_block_id,
+ organization_id=organization_id,
+ )
+ if bound_output is not result.output_parameter_value:
+ result = await self.build_block_result(
+ success=result.success,
+ failure_reason=result.failure_reason,
+ output_parameter_value=bound_output,
+ status=result.status,
+ workflow_run_block_id=workflow_run_block_id,
+ organization_id=organization_id,
+ error_codes=result.error_codes or None,
+ )
+ else:
+ # A heal that fired and still failed keeps its verdict, but a download that
+ # landed before the failure is bound like every other failing exit.
+ failure_output = await self._failure_output_with_downloads(
+ engine="inline",
+ workflow_run_context=workflow_run_context,
+ workflow_run_id=workflow_run_id,
+ workflow_run_block_id=workflow_run_block_id,
+ organization_id=organization_id,
+ resolved_download_id=resolved_download_id,
+ download_dir_before=download_dir_before,
+ result=result.output_parameter_value,
+ )
+ if failure_output is not None:
+ result = await self.build_block_result(
+ success=False,
+ failure_reason=result.failure_reason,
+ output_parameter_value=failure_output,
+ status=result.status or BlockStatus.failed,
+ workflow_run_block_id=workflow_run_block_id,
+ organization_id=organization_id,
+ error_codes=result.error_codes or None,
+ )
output_obligation = self._output_obligation_for_heal_result(result)
# Record output before finalizing so a failed write fails closed, never leaving a
# completed block without the output downstream consumers require.
@@ -5642,6 +5955,7 @@ async def wrapper({default_args}):
)
resolved_download_id = resolve_run_download_id(block_context, fallback_run_id=workflow_run_id)
+ download_dir_before = local_download_dir_file_names(resolved_download_id)
browser_state = await self.get_or_create_browser_state(
workflow_run_id=workflow_run_id,
organization_id=organization_id,
@@ -5871,6 +6185,8 @@ async def wrapper({default_args}):
browser_session_id=browser_session_id,
browser_state=browser_state,
page=page,
+ resolved_download_id=resolved_download_id,
+ download_dir_before=download_dir_before,
)
if secure_code_block_result.block_result is None:
LOG.warning(
@@ -5889,6 +6205,9 @@ async def wrapper({default_args}):
organization_id=organization_id,
browser_state=browser_state,
page=page,
+ engine="secure_runner",
+ resolved_download_id=resolved_download_id,
+ download_dir_before=download_dir_before,
)
# failure=None does not imply success: infra arms (no browser/page, runner raise,
# invalid output) return a failed block_result with no healable failure metadata.
@@ -5905,25 +6224,76 @@ async def wrapper({default_args}):
)
await recorder.finalize(success=False)
return secure_code_block_result.block_result
- await recorder.finalize(success=True)
secure_output = secure_code_block_result.block_result.output_parameter_value
- await self._record_unregistered_download_intent(
- engine="secure_runner",
- # The sidecar registers into the block output, so that payload is the
- # registration evidence here — the inline directory diff never runs.
- registered=block_output_has_registered_download(secure_output),
- output=secure_output,
- workflow_run_context=workflow_run_context,
- workflow_run_id=workflow_run_id,
- workflow_run_block_id=workflow_run_block_id,
- organization_id=organization_id,
- )
+ # The sidecar already saved what it downloaded, so read back rather than
+ # re-uploading; the save runs when the read-back does not account for the files
+ # this block added — an earlier block's registration is not evidence for them.
+ try:
+ host_files = await self._read_back_downloaded_files(
+ organization_id=organization_id or workflow_run_context.organization_id,
+ workflow_run_id=workflow_run_id,
+ workflow_run_block_id=workflow_run_block_id,
+ download_run_id=resolved_download_id,
+ )
+ secure_dir_after = local_download_dir_file_names(resolved_download_id)
+ secure_registered_names = {file_info.filename for file_info in host_files or []}
+ # An unreadable snapshot is unknown, never empty: coverage cannot be proven,
+ # so the save runs rather than being skipped on a coerced empty diff.
+ if secure_dir_after is None or download_dir_before is None:
+ secure_new_file_names = None
+ else:
+ secure_new_file_names = secure_dir_after - download_dir_before
+ if secure_new_file_names is None or not secure_new_file_names.issubset(secure_registered_names):
+ host_files = await self._register_downloaded_files(
+ organization_id=organization_id or workflow_run_context.organization_id,
+ workflow_run_id=workflow_run_id,
+ workflow_run_block_id=workflow_run_block_id,
+ download_run_id=resolved_download_id,
+ )
+ secure_output, binding_failure_reason = await self._bind_and_grade_downloads(
+ engine="secure_runner",
+ result=secure_output,
+ downloaded_files=host_files,
+ download_dir_before=download_dir_before,
+ resolved_download_id=resolved_download_id,
+ workflow_run_context=workflow_run_context,
+ workflow_run_id=workflow_run_id,
+ workflow_run_block_id=workflow_run_block_id,
+ organization_id=organization_id,
+ authored_registration=block_output_has_registered_download(secure_output),
+ )
+ except asyncio.CancelledError:
+ # A cancel inside the storage round-trips must not book the executed block
+ # as failed or skip its billing hook; the verdict is moot on teardown.
+ await recorder.finalize(success=True)
+ raise
+ await recorder.finalize(success=binding_failure_reason is None)
await self.record_output_parameter_value(
workflow_run_context,
workflow_run_id,
secure_output,
)
- return secure_code_block_result.block_result
+ if binding_failure_reason is not None:
+ return await self.build_block_result(
+ success=False,
+ failure_reason=binding_failure_reason,
+ output_parameter_value=secure_output,
+ status=BlockStatus.failed,
+ workflow_run_block_id=workflow_run_block_id,
+ organization_id=organization_id,
+ )
+ # The sidecar already wrote the block row with its own pre-binding return, and
+ # that row — not the output parameter — is what download-evidence readers use.
+ sidecar_result = secure_code_block_result.block_result
+ return await self.build_block_result(
+ success=sidecar_result.success,
+ failure_reason=sidecar_result.failure_reason,
+ output_parameter_value=secure_output,
+ status=sidecar_result.status,
+ workflow_run_block_id=workflow_run_block_id,
+ organization_id=organization_id,
+ error_codes=sidecar_result.error_codes or None,
+ )
LOG.warning(
"codeblock.secure_runner_downgrade",
selection_reason="override_returned_none",
@@ -5956,6 +6326,9 @@ async def wrapper({default_args}):
organization_id=organization_id,
browser_state=browser_state,
page=page,
+ engine="inline",
+ resolved_download_id=resolved_download_id,
+ download_dir_before=download_dir_before,
)
except asyncio.TimeoutError:
await recorder.persist(recorder.recorded_actions())
@@ -5972,6 +6345,9 @@ async def wrapper({default_args}):
organization_id=organization_id,
browser_state=browser_state,
page=page,
+ engine="inline",
+ resolved_download_id=resolved_download_id,
+ download_dir_before=download_dir_before,
)
except Exception as e:
# Exact type, not isinstance: the IllegitCompleteScriptTermination subclass means the
@@ -5988,6 +6364,9 @@ async def wrapper({default_args}):
organization_id=organization_id,
browser_state=browser_state,
page=page,
+ engine="inline",
+ resolved_download_id=resolved_download_id,
+ download_dir_before=download_dir_before,
)
if isinstance(e, ErrorCode) and self.error_code_mapping and e.error_code in self.error_code_mapping:
# A declared business error: skip self-heal entirely (it's an intentional
@@ -6007,6 +6386,9 @@ async def wrapper({default_args}):
organization_id=organization_id,
browser_state=browser_state,
page=page,
+ engine="inline",
+ resolved_download_id=resolved_download_id,
+ download_dir_before=download_dir_before,
output_parameter_value=failure_output,
error_codes=[e.error_code],
)
@@ -6050,7 +6432,15 @@ async def wrapper({default_args}):
return await self.build_block_result(
success=False,
failure_reason=failure_reason,
- output_parameter_value=None,
+ output_parameter_value=await self._failure_output_with_downloads(
+ engine="inline",
+ workflow_run_context=workflow_run_context,
+ workflow_run_id=workflow_run_id,
+ workflow_run_block_id=workflow_run_block_id,
+ organization_id=organization_id,
+ resolved_download_id=resolved_download_id,
+ download_dir_before=download_dir_before,
+ ),
status=BlockStatus.failed,
workflow_run_block_id=workflow_run_block_id,
organization_id=organization_id,
@@ -6069,65 +6459,61 @@ async def wrapper({default_args}):
browser_session_id=browser_session_id,
browser_state=browser_state,
page=page,
+ resolved_download_id=resolved_download_id,
+ download_dir_before=download_dir_before,
)
else:
await recorder.persist(recorder.recorded_actions())
- await recorder.finalize(success=True)
+ # A leaked recorder proxy (RecordingLocator/Page/Keyboard) is not JSON serializable and
+ # would either crash registration or, via the default= fallback, replace the value with a
+ # useless placeholder — normalize the wrapper family to its selector/marker first.
+ result = json_safe_recorder_output(result)
+ result = json.loads(
+ json.dumps(result, default=lambda value: f"Object '{type(value)}' is not JSON serializable")
+ )
+ # Mask resolved secrets (OTP codes, passwords) a user assigned to a local before they
+ # reach captured locals, the persisted output, or the logged value. Mirrors
+ # HttpRequestBlock and is stronger than the name-based excluded_parameter_keys filter.
+ result = workflow_run_context.mask_secrets_in_data(result)
+
+ try:
+ downloaded_files = await self._register_downloaded_files(
+ organization_id=organization_id or workflow_run_context.organization_id,
+ workflow_run_id=workflow_run_id,
+ workflow_run_block_id=workflow_run_block_id,
+ download_run_id=resolved_download_id,
+ )
+ result, binding_failure_reason = await self._bind_and_grade_downloads(
+ engine="inline",
+ result=result,
+ downloaded_files=downloaded_files,
+ download_dir_before=download_dir_before,
+ resolved_download_id=resolved_download_id,
+ workflow_run_context=workflow_run_context,
+ workflow_run_id=workflow_run_id,
+ workflow_run_block_id=workflow_run_block_id,
+ organization_id=organization_id,
+ )
+ except asyncio.CancelledError:
+ # A cancel inside the storage round-trips must not book the executed block as
+ # failed or skip its billing hook; the verdict is moot on teardown.
+ await recorder.finalize(success=True)
+ raise
+ await recorder.finalize(success=binding_failure_reason is None)
+ await self.record_output_parameter_value(workflow_run_context, workflow_run_id, result)
+ return await self.build_block_result(
+ success=binding_failure_reason is None,
+ failure_reason=binding_failure_reason,
+ output_parameter_value=result,
+ status=BlockStatus.completed if binding_failure_reason is None else BlockStatus.failed,
+ workflow_run_block_id=workflow_run_block_id,
+ organization_id=organization_id,
+ )
finally:
# Safety net for paths the except arms miss (CancelledError, link_block failure).
await recorder.finalize(success=False)
- # A leaked recorder proxy (RecordingLocator/Page/Keyboard) is not JSON serializable and
- # would either crash registration or, via the default= fallback, replace the value with a
- # useless placeholder — normalize the wrapper family to its selector/marker first.
- result = json_safe_recorder_output(result)
- result = json.loads(
- json.dumps(result, default=lambda value: f"Object '{type(value)}' is not JSON serializable")
- )
- # Mask resolved secrets (OTP codes, passwords) a user assigned to a local before they
- # reach captured locals, the persisted output, or the logged value. Mirrors
- # HttpRequestBlock and is stronger than the name-based excluded_parameter_keys filter.
- result = workflow_run_context.mask_secrets_in_data(result)
-
- downloaded_files = await self._register_downloaded_files(
- organization_id=organization_id or workflow_run_context.organization_id,
- workflow_run_id=workflow_run_id,
- workflow_run_block_id=workflow_run_block_id,
- download_run_id=resolved_download_id,
- )
- current_context = skyvern_context.current()
- downloaded_files = filter_downloaded_files_for_current_iteration(
- downloaded_files,
- current_context.loop_internal_state if current_context else None,
- )
- if downloaded_files and not isinstance(result, dict):
- result = {"value": result} if result is not None else {}
- if downloaded_files and isinstance(result, dict):
- result["downloaded_files"] = [fi.model_dump() for fi in downloaded_files]
- result["downloaded_file_urls"] = [fi.url for fi in downloaded_files]
- result["downloaded_file_artifact_ids"] = [fi.artifact_id for fi in downloaded_files if fi.artifact_id]
-
- await self._record_unregistered_download_intent(
- engine="inline",
- registered=bool(downloaded_files),
- output=result,
- workflow_run_context=workflow_run_context,
- workflow_run_id=workflow_run_id,
- workflow_run_block_id=workflow_run_block_id,
- organization_id=organization_id,
- )
-
- await self.record_output_parameter_value(workflow_run_context, workflow_run_id, result)
- return await self.build_block_result(
- success=True,
- failure_reason=None,
- output_parameter_value=result,
- status=BlockStatus.completed,
- workflow_run_block_id=workflow_run_block_id,
- organization_id=organization_id,
- )
-
SCHEMA_VALIDATION_MAX_ATTEMPTS = 2
SCHEMA_VALIDATION_MAX_ERRORS = 5
diff --git a/skyvern/forge/sdk/workflow/service.py b/skyvern/forge/sdk/workflow/service.py
index 91c9adf8a..b74962389 100644
--- a/skyvern/forge/sdk/workflow/service.py
+++ b/skyvern/forge/sdk/workflow/service.py
@@ -768,6 +768,19 @@ def _require_elapsed_timeout_failure_reason(timeout_failure_reason: str | None)
return timeout_failure_reason
+def run_selection_is_partial(workflow: Workflow, block_labels: list[str] | None) -> bool:
+ """Whether the executed selection left any of the workflow's blocks unrun.
+
+ A selection naming every top-level block runs the whole workflow, so it owes the workflow's
+ declared deliverable even though it was dispatched as a block selection. The finally block is
+ excluded because execute_workflow runs it on its own path, so a full selection never names it."""
+ if not block_labels:
+ return False
+ definition = workflow.workflow_definition
+ definition_labels = {block.label for block in definition.blocks} - {definition.finally_block_label}
+ return not definition_labels.issubset(set(block_labels))
+
+
def _collect_enterprise_gated_workflow_features(
workflow: Workflow,
*,
@@ -3742,6 +3755,7 @@ class WorkflowService:
browser_session_id: str | None = None,
need_call_webhook: bool = True,
workflow_override: Workflow | None = None,
+ requested_completion_contract: dict[str, Any] | None = None,
) -> WorkflowRun:
"""Execute a workflow.
@@ -4420,7 +4434,8 @@ class WorkflowService:
self._finalize_workflow_run_status(
workflow_run_id=workflow_run_id,
workflow_run=workflow_run,
- is_partial_run=bool(block_labels),
+ is_partial_run=run_selection_is_partial(workflow, block_labels),
+ requested_completion_contract=requested_completion_contract,
pre_finally_status=pre_finally_status,
pre_finally_failure_reason=pre_finally_failure_reason,
pre_finally_failure_category=pre_finally_failure_category,
@@ -8340,6 +8355,7 @@ class WorkflowService:
pre_finally_failure_reason: str | None,
pre_finally_failure_category: list[dict] | None = None,
is_partial_run: bool = False,
+ requested_completion_contract: dict[str, Any] | None = None,
) -> WorkflowRun:
"""
Set final workflow run status based on pre-finally state.
@@ -8356,7 +8372,11 @@ class WorkflowService:
WorkflowRunStatus.terminated,
WorkflowRunStatus.timed_out,
):
- contract_verdict = await self._grade_completion_contract(workflow_run, is_partial_run=is_partial_run)
+ contract_verdict = await self._grade_completion_contract(
+ workflow_run,
+ is_partial_run=is_partial_run,
+ requested_completion_contract=requested_completion_contract,
+ )
if contract_verdict is not None and not contract_verdict.satisfied:
updated = await self._update_workflow_run_status_if_not_final(
workflow_run_id=workflow_run_id,
@@ -8415,7 +8435,11 @@ class WorkflowService:
return 0
async def _grade_completion_contract(
- self, workflow_run: WorkflowRun, *, is_partial_run: bool = False
+ self,
+ workflow_run: WorkflowRun,
+ *,
+ is_partial_run: bool = False,
+ requested_completion_contract: dict[str, Any] | None = None,
) -> ContractVerdict | None:
"""Grade the workflow's declared completion contract, or None when it declares nothing.
@@ -8434,6 +8458,10 @@ class WorkflowService:
organization_id=workflow_run.organization_id,
)
criteria = parse_completion_contract(workflow.workflow_definition if workflow else None)
+ if not criteria and requested_completion_contract is not None:
+ # A copilot test run executes a version the request's obligation has not been
+ # written onto yet — it attaches when the proposal is accepted, after this run.
+ criteria = parse_completion_contract({"completion_contract": requested_completion_contract})
if not criteria:
return None
context = skyvern_context.current()
diff --git a/tests/unit/test_code_block_downloads.py b/tests/unit/test_code_block_downloads.py
index ab2718bc4..f70d08c77 100644
--- a/tests/unit/test_code_block_downloads.py
+++ b/tests/unit/test_code_block_downloads.py
@@ -9,7 +9,9 @@ Covers the three acceptance behaviors:
``set_input_files``; an empty URI leaves the value untouched.
"""
+import asyncio
import os
+from collections.abc import Callable
from datetime import UTC, datetime
from types import SimpleNamespace
from unittest.mock import AsyncMock
@@ -18,9 +20,13 @@ import pytest
from structlog.testing import capture_logs
from skyvern.exceptions import IllegitCompleteScriptTermination, ScriptTerminationException
-from skyvern.forge.sdk.copilot.reached_download_target import code_is_download_intent
+from skyvern.forge.sdk.copilot.reached_download_target import (
+ block_output_has_registered_download,
+ code_is_download_intent,
+)
from skyvern.forge.sdk.core import skyvern_context
from skyvern.forge.sdk.core.skyvern_context import SkyvernContext
+from skyvern.forge.sdk.db.utils import downloaded_file_count_from_output
from skyvern.forge.sdk.schemas.files import FileInfo
from skyvern.forge.sdk.workflow.models import block as block_module
from skyvern.forge.sdk.workflow.models.block import CodeBlock
@@ -98,6 +104,8 @@ def _wire_block_runtime(
context = SimpleNamespace(
organization_id="o_1",
workflow=workflow,
+ workflow_permanent_id="wpid_test",
+ workflow_id="w_test",
get_value=lambda key: (values or {}).get(key),
mask_secrets_in_data=lambda data, mask="*****": data,
)
@@ -106,8 +114,10 @@ def _wire_block_runtime(
monkeypatch.setattr(CodeBlock, "record_output_parameter_value", AsyncMock())
-def _wire_secure_runner(monkeypatch: pytest.MonkeyPatch, *, output: dict) -> None:
- """Route execute() down the secure sidecar arm, which returns before the inline directory diff."""
+def _wire_secure_runner(
+ monkeypatch: pytest.MonkeyPatch, *, output: dict, on_execute: Callable[[], None] | None = None
+) -> None:
+ """Route execute() down the secure sidecar arm, whose returned payload is what the host binds."""
block_result = BlockResult(
success=True,
output_parameter=_output_parameter("code_out"),
@@ -115,11 +125,26 @@ def _wire_secure_runner(monkeypatch: pytest.MonkeyPatch, *, output: dict) -> Non
status=BlockStatus.completed,
workflow_run_block_id="",
)
+
+ async def _execute_override(**kwargs: object) -> SimpleNamespace:
+ if on_execute is not None:
+ on_execute()
+ return SimpleNamespace(block_result=block_result, failure=None)
+
fake_app = block_module.app
fake_app.AGENT_FUNCTION.should_use_codeblock_runner = AsyncMock(return_value=True)
- fake_app.AGENT_FUNCTION.execute_code_block_override = AsyncMock(
- return_value=SimpleNamespace(block_result=block_result, failure=None)
- )
+ fake_app.AGENT_FUNCTION.execute_code_block_override = AsyncMock(side_effect=_execute_override)
+
+
+def _write_run_download(download_root: str, filename: str = "invoice.pdf") -> None:
+ run_dir = os.path.join(download_root, "wr_1")
+ os.makedirs(run_dir, exist_ok=True)
+ with open(os.path.join(run_dir, filename), "wb") as handle:
+ handle.write(b"%PDF-1.4 statement")
+
+
+def _persisted_output() -> object:
+ return CodeBlock.record_output_parameter_value.await_args.args[2]
def _fake_storage_app(monkeypatch: pytest.MonkeyPatch, *, save, get) -> None:
@@ -632,3 +657,786 @@ async def test_secure_runner_registered_download_is_not_counted(
assert _download_intent_events(logs) == []
assert result.success is True
+
+
+async def test_inline_file_on_disk_with_no_registration_fails_loudly(
+ monkeypatch: pytest.MonkeyPatch, _isolated_download_path: str
+) -> None:
+ """A file this block put in the run directory that never registered cannot persist as an output
+ that reads like nothing was downloaded."""
+ skyvern_context.set(SkyvernContext(organization_id="o_1", workflow_run_id="wr_1", run_id="wr_1"))
+
+ _fake_storage_app(monkeypatch, save=AsyncMock(), get=AsyncMock(return_value=[]))
+ _wire_block_runtime(monkeypatch)
+
+ async def _download_then_return_unexpected_shape(*args: object, **kwargs: object) -> object:
+ _write_run_download(_isolated_download_path)
+ return None
+
+ monkeypatch.setattr(
+ CodeBlock,
+ "execute_user_function_with_timeout",
+ AsyncMock(side_effect=_download_then_return_unexpected_shape),
+ )
+
+ block = CodeBlock(
+ label="download_invoice",
+ code="value = 'unused'",
+ output_parameter=_output_parameter("code_out"),
+ )
+ result = await block.execute(workflow_run_id="wr_1", workflow_run_block_id="", organization_id="o_1")
+
+ assert result.success is False
+ assert result.status == BlockStatus.failed
+ assert result.failure_reason == block_module.DOWNLOAD_BINDING_FAILURE_REASON
+ persisted = _persisted_output()
+ assert persisted is not None
+ assert persisted[block_module.UNBOUND_DOWNLOAD_OUTPUT_KEY] is True
+ assert "downloaded_files" not in persisted
+ assert persisted == result.output_parameter_value
+
+
+async def test_secure_runner_file_on_disk_with_no_registration_fails_loudly(
+ monkeypatch: pytest.MonkeyPatch, _isolated_download_path: str
+) -> None:
+ """Engine parity: authored registration-shaped keys are not registration evidence."""
+ skyvern_context.set(SkyvernContext(organization_id="o_1", workflow_run_id="wr_1", run_id="wr_1"))
+
+ _fake_storage_app(monkeypatch, save=AsyncMock(), get=AsyncMock(return_value=[]))
+ _wire_block_runtime(monkeypatch)
+ _wire_secure_runner(
+ monkeypatch,
+ output={"downloaded_files": [{"file_name": "invoice.pdf"}]},
+ on_execute=lambda: _write_run_download(_isolated_download_path),
+ )
+
+ block = CodeBlock(
+ label="download_invoice",
+ code=_SWALLOWED_DOWNLOAD_TERMINAL,
+ output_parameter=_output_parameter("code_out"),
+ )
+ result = await block.execute(workflow_run_id="wr_1", workflow_run_block_id="", organization_id="o_1")
+
+ assert result.success is False
+ assert result.status == BlockStatus.failed
+ assert result.failure_reason == block_module.DOWNLOAD_BINDING_FAILURE_REASON
+ persisted = _persisted_output()
+ assert persisted[block_module.UNBOUND_DOWNLOAD_OUTPUT_KEY] is True
+ # The authored key is the one that must not survive — asserting only on a key the code never
+ # wrote would pass no matter what the binder did.
+ assert "downloaded_files" not in persisted
+ assert "downloaded_file_urls" not in persisted
+ assert block_output_has_registered_download(persisted) is False
+ assert persisted == result.output_parameter_value
+
+
+async def test_secure_runner_binds_host_registration_over_authored_keys(
+ monkeypatch: pytest.MonkeyPatch, _isolated_download_path: str
+) -> None:
+ skyvern_context.set(SkyvernContext(organization_id="o_1", workflow_run_id="wr_1", run_id="wr_1"))
+
+ file_info = FileInfo(
+ url="https://api.example.com/v1/artifacts/a_dl_1/content?artifact_name=invoice.pdf",
+ filename="invoice.pdf",
+ checksum="deadbeef",
+ artifact_id="a_dl_1",
+ modified_at=datetime(2026, 6, 14, 12, 0, tzinfo=UTC),
+ )
+ save_mock = AsyncMock()
+ _fake_storage_app(monkeypatch, save=save_mock, get=AsyncMock(side_effect=[[], [file_info]]))
+ _wire_block_runtime(monkeypatch)
+ _wire_secure_runner(
+ monkeypatch,
+ output={"downloaded_files": [{"file_name": "not-the-registered-file.pdf"}], "status": "ok"},
+ on_execute=lambda: _write_run_download(_isolated_download_path),
+ )
+
+ block = CodeBlock(
+ label="download_invoice",
+ code=_SWALLOWED_DOWNLOAD_TERMINAL,
+ output_parameter=_output_parameter("code_out"),
+ )
+ result = await block.execute(workflow_run_id="wr_1", workflow_run_block_id="", organization_id="o_1")
+
+ assert result.success is True
+ output = result.output_parameter_value
+ assert output["downloaded_files"] == [file_info.model_dump()]
+ assert output["downloaded_file_artifact_ids"] == ["a_dl_1"]
+ assert output["status"] == "ok"
+ assert _persisted_output() == output
+ # The sidecar already uploaded what it downloaded; a host save here would double-upload.
+ save_mock.assert_not_awaited()
+
+
+async def test_loop_iteration_filter_emptying_the_list_is_not_a_binding_failure(
+ monkeypatch: pytest.MonkeyPatch, _isolated_download_path: str
+) -> None:
+ """A file carried in from an earlier iteration is filtered out legitimately, and this block put
+ nothing new in the run directory."""
+ prev_file = FileInfo(
+ url="https://api.example.com/v1/artifacts/a_prev/content?artifact_name=prev.pdf",
+ filename="prev.pdf",
+ checksum="abc",
+ artifact_id="a_prev",
+ )
+ skyvern_context.set(
+ SkyvernContext(
+ organization_id="o_1",
+ workflow_run_id="wr_1",
+ run_id="wr_1",
+ loop_internal_state={
+ "downloaded_file_signatures_before_iteration": [
+ block_module.to_downloaded_file_signature(prev_file),
+ ],
+ },
+ )
+ )
+ _write_run_download(_isolated_download_path, filename="prev.pdf")
+
+ _fake_storage_app(monkeypatch, save=AsyncMock(), get=AsyncMock(return_value=[prev_file]))
+ _wire_block_runtime(monkeypatch)
+
+ block = CodeBlock(
+ label="code_download",
+ code="saved = 'ok'",
+ output_parameter=_output_parameter("code_out"),
+ )
+ result = await block.execute(workflow_run_id="wr_1", workflow_run_block_id="", organization_id="o_1")
+
+ assert result.success is True
+ assert result.output_parameter_value == {"saved": "ok"}
+
+
+async def test_registration_binds_onto_an_unexpected_return_shape(
+ monkeypatch: pytest.MonkeyPatch, _isolated_download_path: str
+) -> None:
+ """The pass path: a legitimate download lands in the declared output whatever the generated code
+ chose to return."""
+ skyvern_context.set(SkyvernContext(organization_id="o_1", workflow_run_id="wr_1", run_id="wr_1"))
+
+ file_info = FileInfo(
+ url="https://api.example.com/v1/artifacts/a_dl_1/content?artifact_name=invoice.pdf",
+ filename="invoice.pdf",
+ checksum="deadbeef",
+ artifact_id="a_dl_1",
+ modified_at=datetime(2026, 6, 14, 12, 0, tzinfo=UTC),
+ )
+ _fake_storage_app(monkeypatch, save=AsyncMock(), get=AsyncMock(side_effect=[[], [file_info]]))
+ _wire_block_runtime(monkeypatch)
+
+ async def _download_then_return_none(*args: object, **kwargs: object) -> object:
+ _write_run_download(_isolated_download_path)
+ return None
+
+ monkeypatch.setattr(
+ CodeBlock,
+ "execute_user_function_with_timeout",
+ AsyncMock(side_effect=_download_then_return_none),
+ )
+
+ block = CodeBlock(
+ label="download_invoice",
+ code="value = 'unused'",
+ output_parameter=_output_parameter("code_out"),
+ )
+ result = await block.execute(workflow_run_id="wr_1", workflow_run_block_id="", organization_id="o_1")
+
+ assert result.success is True
+ output = result.output_parameter_value
+ assert output["downloaded_files"] == [file_info.model_dump()]
+ assert output["downloaded_file_urls"] == [file_info.url]
+ assert output["downloaded_file_artifact_ids"] == ["a_dl_1"]
+ assert _persisted_output() == output
+
+
+def _raise_after_download(download_root: str, error: Exception) -> Callable[..., object]:
+ """User code that lands a file in the run directory and then dies."""
+
+ async def _run(*args: object, **kwargs: object) -> object:
+ _write_run_download(download_root)
+ raise error
+
+ return _run
+
+
+async def test_raising_block_binds_download_that_already_landed(
+ monkeypatch: pytest.MonkeyPatch, _isolated_download_path: str
+) -> None:
+ """A block that downloads and then raises still carries the registration evidence out.
+
+ The download reached the run directory, so it is accountable regardless of how the block
+ exited; the block's own failure reason is what survives, not the binding verdict."""
+ skyvern_context.set(SkyvernContext(organization_id="o_1", workflow_run_id="wr_1", run_id="wr_1"))
+
+ file_info = FileInfo(
+ url="https://api.example.com/v1/artifacts/a_dl_1/content?artifact_name=invoice.pdf",
+ filename="invoice.pdf",
+ checksum="deadbeef",
+ artifact_id="a_dl_1",
+ modified_at=datetime(2026, 6, 14, 12, 0, tzinfo=UTC),
+ )
+ _fake_storage_app(monkeypatch, save=AsyncMock(), get=AsyncMock(side_effect=[[], [file_info]]))
+ _wire_block_runtime(monkeypatch)
+ monkeypatch.setattr(
+ CodeBlock,
+ "execute_user_function_with_timeout",
+ AsyncMock(side_effect=_raise_after_download(_isolated_download_path, RuntimeError("Download is starting"))),
+ )
+
+ block = CodeBlock(
+ label="download_invoice",
+ code="value = 'unused'",
+ output_parameter=_output_parameter("code_out"),
+ )
+ result = await block.execute(workflow_run_id="wr_1", workflow_run_block_id="", organization_id="o_1")
+
+ assert result.success is False
+ output = result.output_parameter_value
+ assert output is not None
+ assert output["downloaded_files"] == [file_info.model_dump()]
+ assert output["downloaded_file_urls"] == [file_info.url]
+ assert output["downloaded_file_artifact_ids"] == ["a_dl_1"]
+ assert "Download is starting" in (result.failure_reason or "")
+ assert result.failure_reason != block_module.DOWNLOAD_BINDING_FAILURE_REASON
+ assert _persisted_output() == output
+
+
+async def test_raising_block_with_unregistered_download_persists_non_null_output(
+ monkeypatch: pytest.MonkeyPatch, _isolated_download_path: str
+) -> None:
+ """The file landed but nothing registered: the output still cannot read as 'no download'."""
+ skyvern_context.set(SkyvernContext(organization_id="o_1", workflow_run_id="wr_1", run_id="wr_1"))
+
+ _fake_storage_app(monkeypatch, save=AsyncMock(), get=AsyncMock(return_value=[]))
+ _wire_block_runtime(monkeypatch)
+ monkeypatch.setattr(
+ CodeBlock,
+ "execute_user_function_with_timeout",
+ AsyncMock(side_effect=_raise_after_download(_isolated_download_path, RuntimeError("boom"))),
+ )
+
+ block = CodeBlock(
+ label="download_invoice",
+ code="value = 'unused'",
+ output_parameter=_output_parameter("code_out"),
+ )
+ result = await block.execute(workflow_run_id="wr_1", workflow_run_block_id="", organization_id="o_1")
+
+ assert result.success is False
+ persisted = _persisted_output()
+ assert persisted is not None
+ assert persisted[block_module.UNBOUND_DOWNLOAD_OUTPUT_KEY] is True
+ assert persisted == result.output_parameter_value
+
+
+async def test_raising_block_without_download_still_persists_no_output(
+ monkeypatch: pytest.MonkeyPatch, _isolated_download_path: str
+) -> None:
+ """A failure that downloaded nothing is unchanged: no registration, no output."""
+ skyvern_context.set(SkyvernContext(organization_id="o_1", workflow_run_id="wr_1", run_id="wr_1"))
+
+ _fake_storage_app(monkeypatch, save=AsyncMock(), get=AsyncMock(return_value=[]))
+ _wire_block_runtime(monkeypatch)
+
+ async def _just_raise(*args: object, **kwargs: object) -> object:
+ raise RuntimeError("boom")
+
+ monkeypatch.setattr(CodeBlock, "execute_user_function_with_timeout", AsyncMock(side_effect=_just_raise))
+
+ block = CodeBlock(
+ label="no_download",
+ code="value = 'unused'",
+ output_parameter=_output_parameter("code_out"),
+ )
+ result = await block.execute(workflow_run_id="wr_1", workflow_run_block_id="", organization_id="o_1")
+
+ assert result.success is False
+ assert result.output_parameter_value is None
+ CodeBlock.record_output_parameter_value.assert_not_awaited()
+
+
+async def test_timed_out_block_binds_download_that_already_landed(
+ monkeypatch: pytest.MonkeyPatch, _isolated_download_path: str
+) -> None:
+ """Engine parity for the timeout exit — the slow download that did land is still accountable."""
+ skyvern_context.set(SkyvernContext(organization_id="o_1", workflow_run_id="wr_1", run_id="wr_1"))
+
+ file_info = FileInfo(
+ url="https://api.example.com/v1/artifacts/a_dl_1/content?artifact_name=invoice.pdf",
+ filename="invoice.pdf",
+ checksum="deadbeef",
+ artifact_id="a_dl_1",
+ modified_at=datetime(2026, 6, 14, 12, 0, tzinfo=UTC),
+ )
+ _fake_storage_app(monkeypatch, save=AsyncMock(), get=AsyncMock(side_effect=[[], [file_info]]))
+ _wire_block_runtime(monkeypatch)
+ monkeypatch.setattr(
+ CodeBlock,
+ "execute_user_function_with_timeout",
+ AsyncMock(side_effect=_raise_after_download(_isolated_download_path, asyncio.TimeoutError())),
+ )
+
+ block = CodeBlock(
+ label="download_invoice",
+ code="value = 'unused'",
+ output_parameter=_output_parameter("code_out"),
+ )
+ result = await block.execute(workflow_run_id="wr_1", workflow_run_block_id="", organization_id="o_1")
+
+ assert result.success is False
+ output = result.output_parameter_value
+ assert output is not None
+ assert output["downloaded_file_urls"] == [file_info.url]
+ assert "exceeded" in (result.failure_reason or "")
+
+
+async def test_authored_registration_keys_are_dropped_without_host_evidence(
+ monkeypatch: pytest.MonkeyPatch, _isolated_download_path: str
+) -> None:
+ """Keys the executed code wrote for itself are not registration evidence.
+
+ The host registered nothing, so no consumer reading this output may find a download in it."""
+ skyvern_context.set(SkyvernContext(organization_id="o_1", workflow_run_id="wr_1", run_id="wr_1"))
+
+ _fake_storage_app(monkeypatch, save=AsyncMock(), get=AsyncMock(return_value=[]))
+ _wire_block_runtime(monkeypatch)
+
+ async def _claim_a_download_that_never_happened(*args: object, **kwargs: object) -> object:
+ return {
+ "downloaded_files": [{"filename": "invoice.pdf"}],
+ "downloaded_file_urls": ["file:///tmp/invoice.pdf"],
+ "downloaded_file_artifact_ids": ["a_made_up"],
+ "note": "model wrote these itself",
+ }
+
+ monkeypatch.setattr(
+ CodeBlock,
+ "execute_user_function_with_timeout",
+ AsyncMock(side_effect=_claim_a_download_that_never_happened),
+ )
+
+ block = CodeBlock(
+ label="download_invoice",
+ code="value = 'unused'",
+ output_parameter=_output_parameter("code_out"),
+ )
+ result = await block.execute(workflow_run_id="wr_1", workflow_run_block_id="", organization_id="o_1")
+
+ persisted = _persisted_output()
+ assert persisted["note"] == "model wrote these itself"
+ assert "downloaded_files" not in persisted
+ assert "downloaded_file_urls" not in persisted
+ assert "downloaded_file_artifact_ids" not in persisted
+ assert block_output_has_registered_download(persisted) is False
+ assert downloaded_file_count_from_output(result.output_parameter_value) is None
+
+
+async def test_registration_timeout_does_not_fail_a_block_that_downloaded(
+ monkeypatch: pytest.MonkeyPatch, _isolated_download_path: str
+) -> None:
+ """A storage timeout is not evidence that the code swallowed a download.
+
+ Workflow finalization retries the save, so failing the block here would destroy a run whose
+ download actually succeeded."""
+ skyvern_context.set(SkyvernContext(organization_id="o_1", workflow_run_id="wr_1", run_id="wr_1"))
+
+ _fake_storage_app(
+ monkeypatch,
+ save=AsyncMock(side_effect=asyncio.TimeoutError()),
+ get=AsyncMock(return_value=[]),
+ )
+ _wire_block_runtime(monkeypatch)
+
+ async def _download_then_return(*args: object, **kwargs: object) -> object:
+ _write_run_download(_isolated_download_path)
+ return {"ok": True}
+
+ monkeypatch.setattr(
+ CodeBlock,
+ "execute_user_function_with_timeout",
+ AsyncMock(side_effect=_download_then_return),
+ )
+
+ block = CodeBlock(
+ label="download_invoice",
+ code="value = 'unused'",
+ output_parameter=_output_parameter("code_out"),
+ )
+ result = await block.execute(workflow_run_id="wr_1", workflow_run_block_id="", organization_id="o_1")
+
+ assert result.success is True
+ assert result.status == BlockStatus.completed
+ assert result.failure_reason != block_module.DOWNLOAD_BINDING_FAILURE_REASON
+ assert block_module.UNBOUND_DOWNLOAD_OUTPUT_KEY not in (result.output_parameter_value or {})
+
+
+def test_unreadable_download_dir_is_an_unknown_snapshot_not_an_empty_one() -> None:
+ """An unreadable directory must not read as empty, or pre-existing files look new."""
+ assert block_module.local_download_dir_file_names(None) is None
+
+
+def test_none_valued_registration_fields_are_schema_not_claims() -> None:
+ """A None or empty registration field survives the strip — removing it breaks strict templates."""
+ payload = {"downloaded_files": None, "downloaded_file_urls": [], "note": "schema fields"}
+ bound = block_module.bind_downloaded_files_to_output(dict(payload), [])
+ assert bound == payload
+
+
+async def test_successful_self_heal_binds_the_download_that_preceded_the_raise(
+ monkeypatch: pytest.MonkeyPatch, _isolated_download_path: str
+) -> None:
+ """The completed-with-null arm: code downloads, raises, the heal succeeds with no output.
+
+ Recording the heal's own None without ever looking at the run directory is what persisted a
+ completed block with a null output while the file sat on disk (SKY-13694's failing arm)."""
+ skyvern_context.set(SkyvernContext(organization_id="o_1", workflow_run_id="wr_1", run_id="wr_1"))
+
+ file_info = FileInfo(
+ url="https://api.example.com/v1/artifacts/a_dl_1/content?artifact_name=invoice.pdf",
+ filename="invoice.pdf",
+ checksum="deadbeef",
+ artifact_id="a_dl_1",
+ modified_at=datetime(2026, 6, 14, 12, 0, tzinfo=UTC),
+ )
+ _fake_storage_app(monkeypatch, save=AsyncMock(), get=AsyncMock(side_effect=[[], [file_info]]))
+ block_module.app.AGENT_FUNCTION.resolve_self_heal_api_key = AsyncMock(return_value=None)
+ _wire_block_runtime(monkeypatch)
+ monkeypatch.setattr(
+ CodeBlock,
+ "execute_user_function_with_timeout",
+ AsyncMock(side_effect=_raise_after_download(_isolated_download_path, Exception("Download is starting"))),
+ )
+ monkeypatch.setattr(CodeBlock, "_self_heal_enabled", AsyncMock(return_value=True))
+ monkeypatch.setattr(CodeBlock, "_is_healable_page_failure", lambda self, e, page, engine_selection: True)
+
+ healed = BlockResult(
+ success=True,
+ output_parameter=_output_parameter("code_out"),
+ output_parameter_value=None,
+ status=BlockStatus.completed,
+ workflow_run_block_id="",
+ )
+ monkeypatch.setattr(CodeBlock, "_attempt_self_heal", AsyncMock(return_value=healed))
+
+ block = CodeBlock(
+ label="download_invoice",
+ code="value = 'unused'",
+ output_parameter=_output_parameter("code_out"),
+ )
+ result = await block.execute(workflow_run_id="wr_1", workflow_run_block_id="", organization_id="o_1")
+
+ assert result.success is True
+ output = result.output_parameter_value
+ assert output is not None
+ assert output["downloaded_files"] == [file_info.model_dump()]
+ assert output["downloaded_file_urls"] == [file_info.url]
+ assert _persisted_output() == output
+
+
+async def test_secure_success_registers_its_own_file_even_when_an_earlier_block_registered(
+ monkeypatch: pytest.MonkeyPatch, _isolated_download_path: str
+) -> None:
+ """An earlier block's registration is not evidence for the file this secure block added."""
+ skyvern_context.set(SkyvernContext(organization_id="o_1", workflow_run_id="wr_1", run_id="wr_1"))
+
+ earlier = FileInfo(
+ url="https://api.example.com/v1/artifacts/a_dl_0/content?artifact_name=earlier.pdf",
+ filename="earlier.pdf",
+ checksum="beefdead",
+ artifact_id="a_dl_0",
+ modified_at=datetime(2026, 6, 13, 12, 0, tzinfo=UTC),
+ )
+ this_block = FileInfo(
+ url="https://api.example.com/v1/artifacts/a_dl_1/content?artifact_name=invoice.pdf",
+ filename="invoice.pdf",
+ checksum="deadbeef",
+ artifact_id="a_dl_1",
+ modified_at=datetime(2026, 6, 14, 12, 0, tzinfo=UTC),
+ )
+ save_mock = AsyncMock()
+ # Baseline empty; read-back sees only the earlier block's file; post-save read sees both.
+ _fake_storage_app(
+ monkeypatch,
+ save=save_mock,
+ get=AsyncMock(side_effect=[[], [earlier], [earlier, this_block]]),
+ )
+ _wire_block_runtime(monkeypatch)
+ _wire_secure_runner(
+ monkeypatch,
+ output={"status": "ok"},
+ on_execute=lambda: _write_run_download(_isolated_download_path),
+ )
+
+ block = CodeBlock(
+ label="download_invoice",
+ code=_SWALLOWED_DOWNLOAD_TERMINAL,
+ output_parameter=_output_parameter("code_out"),
+ )
+ result = await block.execute(workflow_run_id="wr_1", workflow_run_block_id="", organization_id="o_1")
+
+ save_mock.assert_awaited()
+ assert result.success is True
+ assert result.failure_reason != block_module.DOWNLOAD_BINDING_FAILURE_REASON
+ output = result.output_parameter_value
+ assert this_block.url in output["downloaded_file_urls"]
+
+
+async def test_secure_backstop_runs_when_the_directory_snapshot_is_unreadable(
+ monkeypatch: pytest.MonkeyPatch, _isolated_download_path: str
+) -> None:
+ """An unreadable snapshot is unknown, never empty — coverage cannot be proven, so the save runs."""
+ skyvern_context.set(SkyvernContext(organization_id="o_1", workflow_run_id="wr_1", run_id="wr_1"))
+
+ earlier = FileInfo(
+ url="https://api.example.com/v1/artifacts/a_dl_0/content?artifact_name=earlier.pdf",
+ filename="earlier.pdf",
+ checksum="beefdead",
+ artifact_id="a_dl_0",
+ modified_at=datetime(2026, 6, 13, 12, 0, tzinfo=UTC),
+ )
+ save_mock = AsyncMock()
+ _fake_storage_app(monkeypatch, save=save_mock, get=AsyncMock(return_value=[earlier]))
+ _wire_block_runtime(monkeypatch)
+ _wire_secure_runner(monkeypatch, output={"status": "ok"})
+ monkeypatch.setattr(block_module, "local_download_dir_file_names", lambda download_run_id: None)
+
+ block = CodeBlock(
+ label="download_invoice",
+ code=_SWALLOWED_DOWNLOAD_TERMINAL,
+ output_parameter=_output_parameter("code_out"),
+ )
+ result = await block.execute(workflow_run_id="wr_1", workflow_run_block_id="", organization_id="o_1")
+
+ save_mock.assert_awaited()
+ assert result.success is True
+
+
+async def test_cancel_during_registration_does_not_book_the_block_failed(
+ monkeypatch: pytest.MonkeyPatch, _isolated_download_path: str
+) -> None:
+ """A cancel inside the storage round-trips keeps the executed block's seat successful.
+
+ Booking it failed would also skip the billing hook for code that ran to completion."""
+ skyvern_context.set(SkyvernContext(organization_id="o_1", workflow_run_id="wr_1", run_id="wr_1"))
+
+ _fake_storage_app(
+ monkeypatch,
+ save=AsyncMock(side_effect=asyncio.CancelledError()),
+ get=AsyncMock(return_value=[]),
+ )
+ _wire_block_runtime(monkeypatch)
+ monkeypatch.setattr(
+ CodeBlock,
+ "execute_user_function_with_timeout",
+ AsyncMock(return_value={"ok": True}),
+ )
+
+ finalize_calls: list[bool] = []
+ original_finalize = block_module.CodeBlockActionRecording.finalize
+
+ async def _spy_finalize(self, success: bool) -> None:
+ finalize_calls.append(success)
+ await original_finalize(self, success)
+
+ monkeypatch.setattr(block_module.CodeBlockActionRecording, "finalize", _spy_finalize)
+
+ block = CodeBlock(
+ label="download_invoice",
+ code="value = 'unused'",
+ output_parameter=_output_parameter("code_out"),
+ )
+ with pytest.raises(asyncio.CancelledError):
+ await block.execute(workflow_run_id="wr_1", workflow_run_block_id="", organization_id="o_1")
+
+ # The first finalize wins; it must book the executed block successful, not failed.
+ assert finalize_calls
+ assert finalize_calls[0] is True
+
+
+async def test_failed_self_heal_still_binds_the_download_that_preceded_the_raise(
+ monkeypatch: pytest.MonkeyPatch, _isolated_download_path: str
+) -> None:
+ """A heal that fired and lost keeps its failure verdict, but not the file's evidence."""
+ skyvern_context.set(SkyvernContext(organization_id="o_1", workflow_run_id="wr_1", run_id="wr_1"))
+
+ file_info = FileInfo(
+ url="https://api.example.com/v1/artifacts/a_dl_1/content?artifact_name=invoice.pdf",
+ filename="invoice.pdf",
+ checksum="deadbeef",
+ artifact_id="a_dl_1",
+ modified_at=datetime(2026, 6, 14, 12, 0, tzinfo=UTC),
+ )
+ _fake_storage_app(monkeypatch, save=AsyncMock(), get=AsyncMock(side_effect=[[], [file_info]]))
+ block_module.app.AGENT_FUNCTION.resolve_self_heal_api_key = AsyncMock(return_value=None)
+ _wire_block_runtime(monkeypatch)
+ monkeypatch.setattr(
+ CodeBlock,
+ "execute_user_function_with_timeout",
+ AsyncMock(side_effect=_raise_after_download(_isolated_download_path, Exception("Download is starting"))),
+ )
+ monkeypatch.setattr(CodeBlock, "_self_heal_enabled", AsyncMock(return_value=True))
+ monkeypatch.setattr(CodeBlock, "_is_healable_page_failure", lambda self, e, page, engine_selection: True)
+
+ healed = BlockResult(
+ success=False,
+ output_parameter=_output_parameter("code_out"),
+ output_parameter_value=None,
+ status=BlockStatus.failed,
+ failure_reason="heal fired and lost",
+ workflow_run_block_id="",
+ )
+ monkeypatch.setattr(CodeBlock, "_attempt_self_heal", AsyncMock(return_value=healed))
+
+ block = CodeBlock(
+ label="download_invoice",
+ code="value = 'unused'",
+ output_parameter=_output_parameter("code_out"),
+ )
+ result = await block.execute(workflow_run_id="wr_1", workflow_run_block_id="", organization_id="o_1")
+
+ assert result.success is False
+ assert result.failure_reason == "heal fired and lost"
+ output = result.output_parameter_value
+ assert output is not None
+ assert output["downloaded_files"] == [file_info.model_dump()]
+ assert output["downloaded_file_urls"] == [file_info.url]
+
+
+async def test_unreachable_storage_leaves_the_authored_payload_alone(
+ monkeypatch: pytest.MonkeyPatch, _isolated_download_path: str
+) -> None:
+ """With no host verdict, neither the claim nor its absence is proven, so nothing is asserted.
+
+ Stripping here would delete a real download whenever storage is briefly unreachable; the strip
+ belongs to the verdict that the host registered nothing, not to the absence of a verdict."""
+ skyvern_context.set(SkyvernContext(organization_id="o_1", workflow_run_id="wr_1", run_id="wr_1"))
+
+ _fake_storage_app(
+ monkeypatch,
+ save=AsyncMock(side_effect=asyncio.TimeoutError()),
+ get=AsyncMock(side_effect=asyncio.TimeoutError()),
+ )
+ _wire_block_runtime(monkeypatch)
+
+ async def _return_authored_registration(*args: object, **kwargs: object) -> object:
+ return {"downloaded_file_urls": ["https://api.example.com/real.pdf"], "note": "sidecar"}
+
+ monkeypatch.setattr(
+ CodeBlock,
+ "execute_user_function_with_timeout",
+ AsyncMock(side_effect=_return_authored_registration),
+ )
+
+ block = CodeBlock(
+ label="download_invoice",
+ code="value = 'unused'",
+ output_parameter=_output_parameter("code_out"),
+ )
+ result = await block.execute(workflow_run_id="wr_1", workflow_run_block_id="", organization_id="o_1")
+
+ assert result.success is True
+ assert result.output_parameter_value["downloaded_file_urls"] == ["https://api.example.com/real.pdf"]
+ assert result.output_parameter_value["note"] == "sidecar"
+
+
+async def test_a_failing_block_registers_its_own_file_even_when_an_earlier_one_registered(
+ monkeypatch: pytest.MonkeyPatch, _isolated_download_path: str
+) -> None:
+ """An earlier block's registration is not evidence for this block's file.
+
+ Reading back a non-empty list says nothing about whether the file this block just added is in
+ it, so the save cannot be skipped on that alone."""
+ skyvern_context.set(SkyvernContext(organization_id="o_1", workflow_run_id="wr_1", run_id="wr_1"))
+
+ earlier = FileInfo(
+ url="https://api.example.com/v1/artifacts/a_dl_0/content?artifact_name=earlier.pdf",
+ filename="earlier.pdf",
+ checksum="beefdead",
+ artifact_id="a_dl_0",
+ modified_at=datetime(2026, 6, 13, 12, 0, tzinfo=UTC),
+ )
+ save_mock = AsyncMock()
+ _fake_storage_app(monkeypatch, save=save_mock, get=AsyncMock(return_value=[earlier]))
+ _wire_block_runtime(monkeypatch)
+ monkeypatch.setattr(
+ CodeBlock,
+ "execute_user_function_with_timeout",
+ AsyncMock(side_effect=_raise_after_download(_isolated_download_path, Exception("Download is starting"))),
+ )
+
+ block = CodeBlock(
+ label="download_invoice",
+ code="value = 'unused'",
+ output_parameter=_output_parameter("code_out"),
+ )
+ await block.execute(workflow_run_id="wr_1", workflow_run_block_id="", organization_id="o_1")
+
+ save_mock.assert_awaited()
+
+
+async def test_secure_runner_bound_output_reaches_the_block_row(
+ monkeypatch: pytest.MonkeyPatch, _isolated_download_path: str
+) -> None:
+ """Download evidence is read from the block row, so the secure path must write it there.
+
+ Returning the sidecar's own result would leave the row holding its pre-binding value."""
+ skyvern_context.set(SkyvernContext(organization_id="o_1", workflow_run_id="wr_1", run_id="wr_1"))
+
+ file_info = FileInfo(
+ url="https://api.example.com/v1/artifacts/a_dl_1/content?artifact_name=invoice.pdf",
+ filename="invoice.pdf",
+ checksum="deadbeef",
+ artifact_id="a_dl_1",
+ modified_at=datetime(2026, 6, 14, 12, 0, tzinfo=UTC),
+ )
+ _fake_storage_app(monkeypatch, save=AsyncMock(), get=AsyncMock(side_effect=[[], [file_info]]))
+ _wire_block_runtime(monkeypatch)
+ _wire_secure_runner(
+ monkeypatch,
+ output={"downloaded_file_urls": ["file:///authored/by/the/code.pdf"], "note": "sidecar"},
+ on_execute=lambda: _write_run_download(_isolated_download_path),
+ )
+
+ row_writes: list[object] = []
+ original_build = CodeBlock.build_block_result
+
+ async def _record_row_write(self, *args: object, **kwargs: object) -> BlockResult:
+ row_writes.append(kwargs.get("output_parameter_value"))
+ return await original_build(self, *args, **kwargs)
+
+ monkeypatch.setattr(CodeBlock, "build_block_result", _record_row_write)
+
+ block = CodeBlock(
+ label="download_invoice",
+ code=_SWALLOWED_DOWNLOAD_TERMINAL,
+ output_parameter=_output_parameter("code_out"),
+ )
+ result = await block.execute(workflow_run_id="wr_1", workflow_run_block_id="", organization_id="o_1")
+
+ assert result.success is True
+ # build_block_result is the only writer of workflow_run_blocks.output.
+ assert row_writes, "the secure success path never wrote the block row"
+ row_output = row_writes[-1]
+ assert row_output["downloaded_files"] == [file_info.model_dump()]
+ assert row_output["downloaded_file_urls"] == [file_info.url]
+ assert block_output_has_registered_download(row_output) is True
+ assert row_output == result.output_parameter_value
diff --git a/tests/unit/test_copilot_completion_verification.py b/tests/unit/test_copilot_completion_verification.py
index 6d4c6c2c7..f86ff83cb 100644
--- a/tests/unit/test_copilot_completion_verification.py
+++ b/tests/unit/test_copilot_completion_verification.py
@@ -9552,11 +9552,14 @@ async def test_download_registered_output_parameter_injects_and_verifies_without
@pytest.mark.asyncio
-async def test_download_registered_nested_output_corroborates_requested_fields(
+async def test_download_registration_keys_nested_in_the_authored_payload_are_not_evidence(
monkeypatch: pytest.MonkeyPatch,
) -> None:
+ """The execution layer binds registration keys at the root of every block output, so keys found
+ only inside the code's own nested payload are its claim, never registration."""
+
async def fail_handler(**_: str) -> str:
- raise AssertionError("nested registered download evidence must bypass the judge")
+ raise AssertionError("authored-only download keys must fail deterministically, not reach the judge")
_patch_completion_handler(monkeypatch, fail_handler)
ctx = _run_ctx()
@@ -9607,11 +9610,11 @@ async def test_download_registered_nested_output_corroborates_requested_fields(
)
assert verification is not None
- assert verification.is_fully_satisfied() is True
+ assert verification.is_fully_satisfied() is False
verdicts = {verdict.criterion_id: verdict for verdict in verification.verdicts}
- assert verdicts["c_account_number"].reason_code == "structurally_abstained"
- assert verdicts["c_account_number"].evidence_ref == "block_outputs:extract_profile.output.account_number"
- assert verdicts[REGISTERED_DOWNLOAD_COMPLETION_CRITERION_ID].reason_code == "evidence_confirms"
+ download_verdict = verdicts[REGISTERED_DOWNLOAD_COMPLETION_CRITERION_ID]
+ assert download_verdict.satisfied is False
+ assert download_verdict.reason_code != "evidence_confirms"
@pytest.mark.asyncio
diff --git a/tests/unit/test_copilot_response_coverage_gate.py b/tests/unit/test_copilot_response_coverage_gate.py
index 889f9fe23..a2ae46d11 100644
--- a/tests/unit/test_copilot_response_coverage_gate.py
+++ b/tests/unit/test_copilot_response_coverage_gate.py
@@ -133,7 +133,10 @@ def test_ask_question_before_acting_on_discovery_candidate_fires_nudge() -> None
assert nudge is not None
assert nudge.rule == "post_discovery_entrypoint_url_question"
- assert nudge.message.rpartition("Resolved candidate_url: ")[2] == ctx.resolved_discovery_entrypoint_url
+ # nosemgrep false positive: asserts the advisory interpolates the resolved entrypoint.
+ assert (
+ nudge.message.rpartition("Resolved candidate_url: ")[2] == ctx.resolved_discovery_entrypoint_url
+ ) # nosemgrep: incomplete-url-substring-sanitization
assert ctx.discovery_entrypoint_url_question_nudge_count == 1
diff --git a/tests/unit/test_workflow_runtime_completion.py b/tests/unit/test_workflow_runtime_completion.py
index 3034219de..6a86b523a 100644
--- a/tests/unit/test_workflow_runtime_completion.py
+++ b/tests/unit/test_workflow_runtime_completion.py
@@ -21,6 +21,7 @@ from skyvern.forge.sdk.workflow.runtime_completion import (
parse_completion_contract,
with_contract,
)
+from skyvern.forge.sdk.workflow.service import run_selection_is_partial
_DOWNLOAD_CONTRACT = {
"completion_contract": {
@@ -266,6 +267,83 @@ async def test_finalize_skips_grading_a_partial_run(monkeypatch: pytest.MonkeyPa
assert statuses == [WorkflowRunStatus.completed]
+async def test_finalize_grades_a_test_run_against_the_requested_contract(monkeypatch: pytest.MonkeyPatch) -> None:
+ """A copilot test run executes a version the obligation has not been written onto yet, so a run
+ that registered nothing must still not finalize as completed."""
+ service, run, statuses = _wire_finalize(monkeypatch, contract=None, downloaded=[])
+
+ await service._finalize_workflow_run_status(
+ workflow_run_id=run.workflow_run_id,
+ workflow_run=run,
+ pre_finally_status=WorkflowRunStatus.running,
+ pre_finally_failure_reason=None,
+ requested_completion_contract=_DERIVED_CONTRACT,
+ )
+
+ assert statuses == [WorkflowRunStatus.terminated]
+
+
+async def test_finalize_completes_a_test_run_that_produced_the_requested_file(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ service, run, statuses = _wire_finalize(monkeypatch, contract=None, downloaded=["invoice.pdf"])
+
+ await service._finalize_workflow_run_status(
+ workflow_run_id=run.workflow_run_id,
+ workflow_run=run,
+ pre_finally_status=WorkflowRunStatus.running,
+ pre_finally_failure_reason=None,
+ requested_completion_contract=_DERIVED_CONTRACT,
+ )
+
+ assert statuses == [WorkflowRunStatus.completed]
+
+
+async def test_a_subset_run_stays_ungraded_even_with_a_requested_contract(monkeypatch: pytest.MonkeyPatch) -> None:
+ service, run, statuses = _wire_finalize(monkeypatch, contract=None, downloaded=[])
+
+ await service._finalize_workflow_run_status(
+ workflow_run_id=run.workflow_run_id,
+ workflow_run=run,
+ pre_finally_status=WorkflowRunStatus.running,
+ pre_finally_failure_reason=None,
+ is_partial_run=True,
+ requested_completion_contract=_DERIVED_CONTRACT,
+ )
+
+ assert statuses == [WorkflowRunStatus.completed]
+
+
+def _workflow_with_blocks(*labels: str, finally_block_label: str | None = None) -> SimpleNamespace:
+ return SimpleNamespace(
+ workflow_definition=SimpleNamespace(
+ blocks=[SimpleNamespace(label=label) for label in labels],
+ finally_block_label=finally_block_label,
+ )
+ )
+
+
+def test_a_selection_naming_every_block_is_not_a_partial_run() -> None:
+ workflow = _workflow_with_blocks("download_statement", "summarize")
+
+ assert run_selection_is_partial(workflow, None) is False
+ assert run_selection_is_partial(workflow, ["download_statement", "summarize"]) is False
+ assert run_selection_is_partial(workflow, ["download_statement"]) is True
+
+
+def test_the_finally_block_is_not_owed_by_a_full_selection() -> None:
+ """execute_workflow runs the finally block on its own path, so a full selection never names it.
+
+ Counting it as unrun would silently skip contract grading for every workflow that has one."""
+ workflow = _workflow_with_blocks("download_statement", "summarize", "cleanup", finally_block_label="cleanup")
+
+ assert run_selection_is_partial(workflow, ["download_statement", "summarize"]) is False
+ assert run_selection_is_partial(workflow, ["download_statement"]) is True
+
+
def test_a_stored_contract_survives_a_write_that_does_not_carry_one() -> None:
"""Non-copilot save paths rebuild the definition through models that omit the field."""
stored = {"completion_contract": _DERIVED_CONTRACT, "blocks": []}
diff --git a/tests/unit/workflow/test_code_block_recorder.py b/tests/unit/workflow/test_code_block_recorder.py
index ec50b90b7..3ca749fd0 100644
--- a/tests/unit/workflow/test_code_block_recorder.py
+++ b/tests/unit/workflow/test_code_block_recorder.py
@@ -683,11 +683,21 @@ async def test_self_heal_success_finalizes_seat_completed(monkeypatch: pytest.Mo
AsyncMock(return_value=1),
)
monkeypatch.setattr(app.AGENT_FUNCTION, "resolve_self_heal_api_key", AsyncMock(return_value=None))
- # Stub the heal to a success result; this tests execute()'s seat-finalization wiring, not the heal itself.
+ # Stub the heal to a success result; this tests execute()'s seat-finalization wiring, not the
+ # heal itself. The stub carries the full BlockResult surface the finalizer reads — heal
+ # finalization rebuilds the result when download binding changes its output.
monkeypatch.setattr(
CodeBlock,
"_attempt_self_heal",
- AsyncMock(return_value=SimpleNamespace(success=True, output_parameter_value=None, failure_reason=None)),
+ AsyncMock(
+ return_value=SimpleNamespace(
+ success=True,
+ output_parameter_value=None,
+ failure_reason=None,
+ status=BlockStatus.completed,
+ error_codes=[],
+ )
+ ),
)
block = _make_code_block("await page.locator('#x').click()", goal="go")