Skip to content

Automate browser based workflows with AI

Complete runs that publish the generated file they promised (SKY-18053) (#8866)

Andrew Neilson0b68d4d04c57
diff --git a/skyvern/forge/sdk/copilot/reached_download_target.py b/skyvern/forge/sdk/copilot/reached_download_target.py
index 1ceec8622..0fa7943d8 100644
--- a/skyvern/forge/sdk/copilot/reached_download_target.py
+++ b/skyvern/forge/sdk/copilot/reached_download_target.py
@@ -201,6 +201,13 @@ def generated_file_artifact_ids(outputs: Iterable[Any]) -> frozenset[str]:
)
+def without_generated_file_stamp(output: Any) -> Any:
+ """``output`` without the worker-owned key, for a code block row whose output the secure worker did not write."""
+ if not isinstance(output, dict) or GENERATED_FILE_ARTIFACT_IDS_KEY not in output:
+ return output
+ return {key: value for key, value in output.items() if key != GENERATED_FILE_ARTIFACT_IDS_KEY}
+
+
def registered_download_proof_view(output: Any, generated: frozenset[str]) -> Any:
"""``output`` with the rows of ``generated`` removed from its registration keys."""
if not isinstance(output, dict) or not generated:
diff --git a/skyvern/forge/sdk/copilot/tools/run_execution.py b/skyvern/forge/sdk/copilot/tools/run_execution.py
index bfaf95cd3..81c577b4a 100644
--- a/skyvern/forge/sdk/copilot/tools/run_execution.py
+++ b/skyvern/forge/sdk/copilot/tools/run_execution.py
@@ -273,9 +273,7 @@ from skyvern.forge.sdk.workflow.models.workflow import (
WorkflowRunStatus,
workflow_definition_sha256,
)
-from skyvern.forge.sdk.workflow.runtime_completion import contract_from_request_criteria
from skyvern.forge.sdk.workflow.runtime_secret_bridge import consume_copilot_runtime_secret_values
-from skyvern.forge.sdk.workflow.service import run_selection_is_partial
from skyvern.schemas.proxy_location import ProxyLocationInput, runtime_proxy_location
from skyvern.schemas.self_heal import HealEpisodeDetail, HealStatus
from skyvern.schemas.workflows import BlockStatus, BlockType
@@ -1522,24 +1520,6 @@ def _selected_code_security_inputs(
return code_blocks
-def _requested_completion_contract(
- ctx: CopilotContext,
- runtime_workflow: Workflow,
- labels_to_execute: list[str],
-) -> dict[str, object] | None:
- """The turn's own deliverable obligation, for a test run that executes the whole workflow.
-
- The obligation attaches to the workflow only when a proposal is accepted, which is after the
- test run this grades; a selection that runs only part of the workflow stays ungraded."""
- if ctx.request_policy is None:
- return None
- if not runtime_workflow.workflow_definition.blocks:
- return None
- if run_selection_is_partial(runtime_workflow, labels_to_execute):
- return None
- return contract_from_request_criteria(ctx.request_policy.graded_completion_criteria())
-
-
def _selected_blocks_require_sandbox(
blocks: list[Any],
*,
@@ -4214,16 +4194,6 @@ async def _run_blocks_and_collect_debug(
runtime_frontier_starter_url_seeded = seeded_runtime_workflow is not runtime_workflow
runtime_workflow = seeded_runtime_workflow
- requested_completion_contract = _requested_completion_contract(ctx, runtime_workflow, labels_to_execute)
- if requested_completion_contract is not None:
- runtime_workflow = runtime_workflow.model_copy(
- update={
- "workflow_definition": runtime_workflow.workflow_definition.model_copy(
- update={"completion_contract": requested_completion_contract}
- )
- }
- )
-
# Snapshot version persisted for a worker-dispatched run or an inline run of a definition that
# was never persisted. The run is created against its exact workflow_id so prepare_workflow
# reads parameter rows from the same definition execute_workflow receives. Without the inline
@@ -4601,7 +4571,6 @@ async def _run_blocks_and_collect_debug(
# the in-memory runtime copy instead registers block outputs against ids that
# were never persisted, and every consumer keyed on the definition drops them.
workflow_override=snapshot.workflow if dispatch_workflow is not None else runtime_workflow,
- requested_completion_contract=requested_completion_contract,
)
)
except BaseException:
diff --git a/skyvern/forge/sdk/copilot/tools/workflow_update.py b/skyvern/forge/sdk/copilot/tools/workflow_update.py
index c043049b1..57969a6e4 100644
--- a/skyvern/forge/sdk/copilot/tools/workflow_update.py
+++ b/skyvern/forge/sdk/copilot/tools/workflow_update.py
@@ -245,11 +245,14 @@ class CodeArtifactCompletionCriterion(BaseModel):
terminal: bool | None = None
output_path: str | None = None
requested_output_evidence_source: RequestedOutputEvidenceSource | None = None
- deliverable_kind: Literal["registered_download"] | None = Field(
+ deliverable_kind: Literal["registered_download", "generated_file"] | None = Field(
default=None,
description=(
- "Set to `registered_download` only when this criterion promises the user a downloaded file. "
- "Run finalization then verifies execution-layer file registration; authored code output cannot satisfy it."
+ "Set only when this criterion promises the user a file. `registered_download`: a file the site serves and "
+ "the browser downloads. `generated_file`: a file the workflow builds itself and publishes with "
+ "`publish_file`, such as a screenshot or report. Run finalization verifies the matching "
+ "execution-layer registration: a published file never satisfies `registered_download`, a site download "
+ "never satisfies `generated_file`, and authored code output satisfies neither."
),
)
judgment_predicate: JudgmentPredicate | None = Field(
diff --git a/skyvern/forge/sdk/workflow/models/block.py b/skyvern/forge/sdk/workflow/models/block.py
index 7b70b349e..ed39076ef 100644
--- a/skyvern/forge/sdk/workflow/models/block.py
+++ b/skyvern/forge/sdk/workflow/models/block.py
@@ -168,6 +168,7 @@ from skyvern.forge.sdk.copilot.reached_download_target import (
REGISTERED_DOWNLOAD_OUTPUT_KEYS,
block_output_has_registered_download,
code_is_download_intent,
+ without_generated_file_stamp,
)
from skyvern.forge.sdk.copilot.turn_origin import TurnOrigin
from skyvern.forge.sdk.core import skyvern_context
@@ -8279,7 +8280,9 @@ async def wrapper({default_args}):
if updated_task.status == TaskStatus.completed:
# The block's value is what its own return would have been; downloads are bound by
# the recorder (_finalize_heal_result), the same as the inline success exit.
- output_parameter_value = workflow_run_context.mask_secrets_in_data(updated_task.extracted_information)
+ output_parameter_value = without_generated_file_stamp(
+ workflow_run_context.mask_secrets_in_data(updated_task.extracted_information)
+ )
if record_output_parameter:
await self.record_output_parameter_value(
workflow_run_context, workflow_run_id, output_parameter_value
@@ -10377,7 +10380,7 @@ async def wrapper({default_args}):
# Mask resolved secrets (OTP codes, passwords) a user assigned to a local before they
# reach captured locals, the persisted output, or the logged value. Mirrors
# HttpRequestBlock and is stronger than the name-based excluded_parameter_keys filter.
- result = workflow_run_context.mask_secrets_in_data(result)
+ result = without_generated_file_stamp(workflow_run_context.mask_secrets_in_data(result))
try:
downloaded_files, skipped_file_names = await self._register_downloaded_files(
diff --git a/skyvern/forge/sdk/workflow/runtime_completion.py b/skyvern/forge/sdk/workflow/runtime_completion.py
index 8ad6ce356..1a1be7cd6 100644
--- a/skyvern/forge/sdk/workflow/runtime_completion.py
+++ b/skyvern/forge/sdk/workflow/runtime_completion.py
@@ -18,12 +18,17 @@ import structlog
LOG = structlog.get_logger()
-# The only criterion kind graded today. Its evidence is the execution layer's own file registration,
-# which no generated code can fake or swallow.
+# Each kind is graded only from the execution layer's own registration of that kind of file, which
+# no generated code can fake or swallow: a site download, or a file the secure worker published.
CRITERION_REGISTERED_DOWNLOAD = "registered_download"
+CRITERION_GENERATED_FILE = "generated_file"
_CONTRACT_KEY = "completion_contract"
-_SUPPORTED_KINDS = frozenset({CRITERION_REGISTERED_DOWNLOAD})
+_SUPPORTED_KINDS = frozenset({CRITERION_REGISTERED_DOWNLOAD, CRITERION_GENERATED_FILE})
+_UNMET_REASONS = {
+ CRITERION_REGISTERED_DOWNLOAD: "The workflow did not produce the file it is declared to download.",
+ CRITERION_GENERATED_FILE: "The workflow did not produce the file it is declared to generate.",
+}
# The field is public on the workflow schema, so a hand-written contract is bounded here.
_MAX_CRITERIA = 16
_MAX_ID_CHARS = 128
@@ -79,37 +84,31 @@ def grade_completion_contract(
criteria: tuple[CompletionCriterion, ...],
*,
registered_download_count: int,
+ generated_file_count: int = 0,
) -> ContractVerdict:
"""Grade declared criteria against execution-layer evidence."""
- unmet: list[str] = []
+ unmet: list[CompletionCriterion] = []
+ unclaimed_generated = generated_file_count
for criterion in criteria:
- if criterion.kind == CRITERION_REGISTERED_DOWNLOAD and registered_download_count < criterion.min_count:
- unmet.append(criterion.id)
+ if criterion.kind == CRITERION_REGISTERED_DOWNLOAD:
+ if registered_download_count < criterion.min_count:
+ unmet.append(criterion)
+ elif criterion.kind == CRITERION_GENERATED_FILE:
+ # A published file is not tied to a criterion, so each promise claims its own in declaration
+ # order: the unmet ids say how many promises are short, not which files are missing.
+ if unclaimed_generated < criterion.min_count:
+ unmet.append(criterion)
+ else:
+ unclaimed_generated -= criterion.min_count
if not unmet:
return ContractVerdict(satisfied=True, unmet_criterion_ids=(), reason=None)
return ContractVerdict(
satisfied=False,
- unmet_criterion_ids=tuple(unmet),
- reason="The workflow did not produce the file it is declared to download.",
+ unmet_criterion_ids=tuple(criterion.id for criterion in unmet),
+ reason=_UNMET_REASONS[unmet[0].kind],
)
-def contract_from_request_criteria(criteria: object) -> dict[str, object] | None:
- """Project the request's own typed completion criteria into a workflow-carried contract.
-
- The obligation comes from what the user asked for, never from the shape of the code the product
- generated: a block that merely reports a download-shaped result promises nothing, and a request
- that asked for a file promises one however the block is written."""
- if not isinstance(criteria, (list, tuple)):
- return None
- if not any(_criterion_requests_a_download(criterion) for criterion in criteria):
- return None
- return {
- "schema_version": 1,
- "criteria": [{"id": "requested_download", "kind": CRITERION_REGISTERED_DOWNLOAD, "min_count": 1}],
- }
-
-
def contract_from_code_artifact_metadata(metadata: object) -> dict[str, object] | None:
"""Project the acting model's structured deliverable declaration into the runtime contract.
@@ -119,7 +118,8 @@ def contract_from_code_artifact_metadata(metadata: object) -> dict[str, object]
"""
if not isinstance(metadata, Mapping):
return None
- seen_ids: set[str] = set()
+ # Ids are model-chosen per block, so two blocks can reuse one id for promises of different kinds.
+ seen: set[tuple[str, str]] = set()
criteria: list[dict[str, object]] = []
for artifact in metadata.values():
if not isinstance(artifact, Mapping):
@@ -128,15 +128,14 @@ def contract_from_code_artifact_metadata(metadata: object) -> dict[str, object]
if not isinstance(raw_criteria, list):
continue
for item in raw_criteria:
- if not isinstance(item, Mapping) or item.get("deliverable_kind") != CRITERION_REGISTERED_DOWNLOAD:
+ kind = item.get("deliverable_kind") if isinstance(item, Mapping) else None
+ if not isinstance(kind, str) or kind not in _SUPPORTED_KINDS:
continue
- identifier = (
- str(item.get("id") or CRITERION_REGISTERED_DOWNLOAD).strip() or CRITERION_REGISTERED_DOWNLOAD
- )[:_MAX_ID_CHARS]
- if identifier in seen_ids:
+ identifier = (str(item.get("id") or kind).strip() or kind)[:_MAX_ID_CHARS]
+ if (identifier, kind) in seen:
continue
- seen_ids.add(identifier)
- criteria.append({"id": identifier, "kind": CRITERION_REGISTERED_DOWNLOAD, "min_count": 1})
+ seen.add((identifier, kind))
+ criteria.append({"id": identifier, "kind": kind, "min_count": 1})
if not criteria:
return None
return {"schema_version": 1, "criteria": criteria}
@@ -161,21 +160,3 @@ def with_contract(definition: dict, carried: dict[str, object] | None) -> dict:
return definition
definition[_CONTRACT_KEY] = carried
return definition
-
-
-def _criterion_requests_a_download(criterion: object) -> bool:
- """Whether a request criterion asks for a registered download.
-
- The request classifier types this as ``deliverable_kind``/``output_path``; the synthetic id is
- the copilot's own internal marker for the same obligation. Both are request-derived."""
- from skyvern.forge.sdk.copilot.reached_download_target import REGISTERED_DOWNLOAD_REQUESTED_OUTPUT_PATHS
- from skyvern.forge.sdk.copilot.request_policy import REGISTERED_DOWNLOAD_COMPLETION_CRITERION_ID
-
- for attr in ("deliverable_kind", "declared_deliverable_kind"):
- if str(getattr(criterion, attr, "") or "").strip() == CRITERION_REGISTERED_DOWNLOAD:
- return True
- if str(getattr(criterion, "output_path", "") or "").strip() in REGISTERED_DOWNLOAD_REQUESTED_OUTPUT_PATHS:
- return True
- # The copilot's own synthetic marker for the same obligation, compared by id so this stays
- # readable from an untyped criterion.
- return str(getattr(criterion, "id", "") or "") == REGISTERED_DOWNLOAD_COMPLETION_CRITERION_ID
diff --git a/skyvern/forge/sdk/workflow/service.py b/skyvern/forge/sdk/workflow/service.py
index 540ac8775..32c2aa5dd 100644
--- a/skyvern/forge/sdk/workflow/service.py
+++ b/skyvern/forge/sdk/workflow/service.py
@@ -13753,7 +13753,7 @@ class WorkflowService:
attempt_rows=attempt_rows,
attempt_number=attempt_number,
)
- # A file the run's own code generated is a real download but not a delivered one.
+ # A file the run's own code generated is left out of the download count only.
run_blocks = await app.DATABASE.observer.get_workflow_run_blocks(
workflow_run_id=workflow_run.workflow_run_id,
organization_id=workflow_run.organization_id,
@@ -13767,7 +13767,15 @@ class WorkflowService:
# downloads), so they are separate files and both count.
registered_ids = {file.artifact_id for file in registered if file.artifact_id}
download_count = len(registered) + len(session_download_ids - registered_ids)
- verdict = grade_completion_contract(criteria, registered_download_count=download_count)
+ # A CODE row carries this key only from the secure worker: the in-process executor and the AI
+ # fallback drop it. On any other block type it is authored data.
+ worker_generated = generated_file_artifact_ids(
+ block.output for block in run_blocks if block.block_type == BlockType.CODE
+ )
+ generated_file_count = len({file.artifact_id for file in files if file.artifact_id in worker_generated})
+ verdict = grade_completion_contract(
+ criteria, registered_download_count=download_count, generated_file_count=generated_file_count
+ )
LOG.info(
"workflow_completion_contract_graded",
workflow_run_id=workflow_run.workflow_run_id,
@@ -13776,6 +13784,7 @@ class WorkflowService:
registered_download_count=len(registered),
session_download_count=len(session_download_ids),
graded_download_count=download_count,
+ generated_file_count=generated_file_count,
)
return verdict
except Exception:
diff --git a/tests/unit/test_code_block_downloads.py b/tests/unit/test_code_block_downloads.py
index c4a42077b..09df21738 100644
--- a/tests/unit/test_code_block_downloads.py
+++ b/tests/unit/test_code_block_downloads.py
@@ -45,6 +45,7 @@ from skyvern.forge.sdk.browser_network_egress_monitor import BrowserNetworkEgres
from skyvern.forge.sdk.copilot.reached_download_target import (
block_output_has_registered_download,
code_is_download_intent,
+ generated_file_artifact_ids,
)
from skyvern.forge.sdk.copilot.tools.run_execution import build_test_evidence_packet
from skyvern.forge.sdk.core import skyvern_context
@@ -1736,6 +1737,38 @@ async def test_authored_registration_keys_are_dropped_without_host_evidence(
assert downloaded_file_count_from_output(result.output_parameter_value) is None
+async def test_in_process_block_cannot_stamp_a_site_download_as_a_published_file(
+ monkeypatch: pytest.MonkeyPatch, _isolated_download_path: str
+) -> None:
+ """Only the secure worker writes the published-file key, so a block naming a real download under it earns nothing."""
+ skyvern_context.set(SkyvernContext(organization_id="o_1", workflow_run_id="wr_1", run_id="wr_1"))
+
+ site_download = FileInfo(
+ url="https://api.example.com/v1/artifacts/a_site/content?artifact_name=invoice.pdf",
+ filename="invoice.pdf",
+ checksum="deadbeef",
+ artifact_id="a_site",
+ modified_at=datetime(2026, 6, 14, 12, 0, tzinfo=UTC),
+ )
+ _fake_storage_app(monkeypatch, save=AsyncMock(), get=AsyncMock(side_effect=[[], [site_download]]))
+ _wire_block_runtime(monkeypatch)
+
+ block = CodeBlock(
+ label="code_download",
+ code="rows = 3\ngenerated_file_artifact_ids = ['a_site']",
+ output_parameter=_output_parameter("code_out"),
+ )
+ result = await block.execute(workflow_run_id="wr_1", workflow_run_block_id="", organization_id="o_1")
+
+ assert result.success is True
+ for output in (_persisted_output(), result.output_parameter_value):
+ assert output["rows"] == 3
+ assert output["downloaded_file_artifact_ids"] == ["a_site"]
+ assert "generated_file_artifact_ids" not in output
+ assert generated_file_artifact_ids([output]) == frozenset()
+
+
@pytest.mark.asyncio
async def test_registration_timeout_does_not_fail_a_block_that_downloaded(
monkeypatch: pytest.MonkeyPatch, _isolated_download_path: str
diff --git a/tests/unit/test_workflow_runtime_completion.py b/tests/unit/test_workflow_runtime_completion.py
index bd87a4dcd..01c3aafd3 100644
--- a/tests/unit/test_workflow_runtime_completion.py
+++ b/tests/unit/test_workflow_runtime_completion.py
@@ -6,12 +6,15 @@ verdict does not depend on which engine ran the blocks or how generated code des
from __future__ import annotations
+from datetime import datetime, timedelta, timezone
from types import SimpleNamespace
from unittest.mock import AsyncMock
import pytest
from skyvern.exceptions import WorkflowNotFound
+from skyvern.forge.sdk.copilot.tools.workflow_update import CodeArtifactCompletionCriterion
+from skyvern.forge.sdk.db.models import WorkflowRunAttemptModel
from skyvern.forge.sdk.schemas.files import FileInfo
from skyvern.forge.sdk.workflow import service as service_module
from skyvern.forge.sdk.workflow.models.workflow import WorkflowRunStatus
@@ -19,12 +22,13 @@ from skyvern.forge.sdk.workflow.runtime_completion import (
CompletionCriterion,
carried_contract,
contract_from_code_artifact_metadata,
- contract_from_request_criteria,
grade_completion_contract,
parse_completion_contract,
with_contract,
)
from skyvern.forge.sdk.workflow.service import run_selection_is_partial
+from skyvern.schemas.workflows import BlockType
+from tests.unit.conftest import FakeWorkflowRunAttemptsRepository
_DOWNLOAD_CONTRACT = {
"completion_contract": {
@@ -106,7 +110,7 @@ def _downloaded_file(name: str, *, artifact_id: str | None = None):
return SimpleNamespace(artifact_id=artifact_id, filename=name, checksum=None, file_size=None)
-def _wire_finalize(monkeypatch, *, contract, downloaded):
+def _wire_finalize(monkeypatch, *, contract, downloaded, run_blocks=(), attempts=None):
"""A WorkflowService with just enough wired to exercise the finalize status decision."""
from skyvern.forge.sdk.workflow.service import WorkflowService
@@ -144,6 +148,19 @@ def _wire_finalize(monkeypatch, *, contract, downloaded):
)
),
)
+ monkeypatch.setattr(
+ service_module.app.DATABASE.observer, "get_workflow_run_blocks", AsyncMock(return_value=list(run_blocks))
+ )
+ if attempts is not None:
+ run.failure_reason = None
+ run.started_at = attempts[-1].started_at
+ run.finished_at = None
+ monkeypatch.setattr(
+ service_module.app.DATABASE, "workflow_run_attempts", FakeWorkflowRunAttemptsRepository(attempts)
+ )
+ monkeypatch.setattr(
+ service_module.app.DATABASE.artifacts, "list_download_artifacts_for_attempt", AsyncMock(return_value=[])
+ )
return service, run, statuses
@@ -298,27 +315,6 @@ async def test_finalize_grades_the_version_execution_loaded_when_it_is_soft_dele
assert statuses == [WorkflowRunStatus.terminated]
-def test_contract_comes_from_the_request_not_the_code() -> None:
- """The obligation is what the user asked for, never the shape of the generated code."""
- from skyvern.forge.sdk.copilot.completion_verification import registered_download_completion_criterion
- from skyvern.forge.sdk.copilot.request_policy import CompletionCriterion as RequestCriterion
-
- assert contract_from_request_criteria([registered_download_completion_criterion()]) is not None
- assert contract_from_request_criteria([RequestCriterion(id="c0", outcome="something else")]) is None
- assert contract_from_request_criteria([]) is None
- assert contract_from_request_criteria(None) is None
-
-
-def test_requested_contract_round_trips_through_the_parser() -> None:
- from skyvern.forge.sdk.copilot.completion_verification import registered_download_completion_criterion
-
- contract = contract_from_request_criteria([registered_download_completion_criterion()])
- criteria = parse_completion_contract({"completion_contract": contract})
- assert [c.kind for c in criteria] == ["registered_download"]
- assert grade_completion_contract(criteria, registered_download_count=0).satisfied is False
- assert grade_completion_contract(criteria, registered_download_count=1).satisfied is True
-
-
def test_download_contract_comes_from_model_declared_artifact_metadata() -> None:
metadata = {
"download_statement": {
@@ -356,6 +352,31 @@ def test_ordinary_artifact_criterion_does_not_create_a_download_contract() -> No
assert contract_from_code_artifact_metadata(metadata) is None
+ ("published", "unmet"),
+ [(0, ("pdf", "xlsx", "zip")), (1, ("xlsx", "zip")), (2, ("zip",)), (3, ())],
+)
+def test_each_generated_promise_needs_its_own_published_file(published: int, unmet: tuple[str, ...]) -> None:
+ contract = contract_from_code_artifact_metadata(
+ {
+ "build_report": {
+ "completion_criteria": [
+ {"id": criterion_id, "deliverable_kind": "generated_file"}
+ for criterion_id in ("pdf", "xlsx", "zip")
+ ]
+ }
+ }
+ )
+
+ verdict = grade_completion_contract(
+ parse_completion_contract({"completion_contract": contract}),
+ registered_download_count=0,
+ generated_file_count=published,
+ )
+
+ assert (verdict.satisfied, verdict.unmet_criterion_ids) == (not unmet, unmet)
+
+
@pytest.mark.asyncio
async def test_finalize_skips_grading_a_partial_run(monkeypatch: pytest.MonkeyPatch) -> None:
"""A frontier run of a block subset was never asked to produce the whole deliverable."""
@@ -535,6 +556,213 @@ async def test_one_file_reported_by_both_download_sources_does_not_satisfy_a_two
assert statuses == [WorkflowRunStatus.terminated]
+_T0 = datetime(2026, 1, 1, 12, tzinfo=timezone.utc)
+_EARLIER_ATTEMPT_THEN_CURRENT = (
+ (1, _T0 - timedelta(minutes=20), _T0 - timedelta(minutes=5)),
+ (2, _T0 - timedelta(minutes=1), None),
+)
+_ONE_ATTEMPT = ((1, _T0 - timedelta(minutes=1), None),)
+
+
+def _declared_contract(*kinds: str) -> dict[str, object] | None:
+ """What the real producer carries when each kind is declared by its own block under one shared criterion id."""
+ return contract_from_code_artifact_metadata(
+ {
+ f"block_{index}": {
+ "completion_criteria": [
+ CodeArtifactCompletionCriterion.model_validate(
+ {"id": "capture", "deliverable_kind": kind}
+ ).model_dump(mode="json")
+ ]
+ }
+ for index, kind in enumerate(kinds)
+ }
+ )
+
+
+def _registered(artifact_id: str, *, minutes_from_t0: int = 0) -> FileInfo:
+ return FileInfo(
+ url=f"s3://b/{artifact_id}", artifact_id=artifact_id, modified_at=_T0 + timedelta(minutes=minutes_from_t0)
+ )
+
+
+def _block_row(output: dict[str, object], block_type: BlockType = BlockType.CODE) -> SimpleNamespace:
+ return SimpleNamespace(block_type=block_type, output=output)
+
+
+def _attempt_row(number: int, started_at: datetime, finished_at: datetime | None) -> WorkflowRunAttemptModel:
+ return WorkflowRunAttemptModel(
+ workflow_run_id="wr_1",
+ attempt_number=number,
+ organization_id="o_1",
+ status="running",
+ started_at=started_at,
+ created_at=started_at,
+ finished_at=finished_at,
+ )
+
+
+_STAMPED = {"generated_file_artifact_ids": ["a_gen"]}
+
+
+ ("kinds", "downloaded", "run_blocks", "attempts", "expected_status", "unmet_promise"),
+ [
+ pytest.param(
+ ("generated_file",),
+ [_registered("a_gen")],
+ [_block_row(_STAMPED)],
+ _ONE_ATTEMPT,
+ WorkflowRunStatus.completed,
+ None,
+ id="generated-promise-met-by-a-same-attempt-published-file",
+ ),
+ pytest.param(
+ ("generated_file",),
+ [_registered("a_gen")],
+ [_block_row(_STAMPED)],
+ _EARLIER_ATTEMPT_THEN_CURRENT,
+ WorkflowRunStatus.completed,
+ None,
+ id="generated-promise-met-by-a-current-attempt-published-file-after-a-retry",
+ ),
+ pytest.param(
+ ("generated_file",),
+ [],
+ [],
+ None,
+ WorkflowRunStatus.terminated,
+ "generate",
+ id="generated-promise-with-nothing-registered",
+ ),
+ pytest.param(
+ ("generated_file",),
+ [_registered("a_site")],
+ [_block_row({"rows": 3, "downloaded_file_artifact_ids": ["a_site"]})],
+ _ONE_ATTEMPT,
+ WorkflowRunStatus.terminated,
+ "generate",
+ id="generated-promise-not-met-by-a-site-download-bound-to-an-in-process-code-row",
+ ),
+ pytest.param(
+ ("generated_file",),
+ [_registered("a_gen")],
+ [],
+ _ONE_ATTEMPT,
+ WorkflowRunStatus.terminated,
+ "generate",
+ id="generated-promise-not-met-by-an-unstamped-site-download",
+ ),
+ pytest.param(
+ ("generated_file",),
+ [_registered("a_gen")],
+ [_block_row(_STAMPED, BlockType.TEXT_PROMPT)],
+ _ONE_ATTEMPT,
+ WorkflowRunStatus.terminated,
+ "generate",
+ id="generated-promise-not-met-by-a-stamp-on-a-non-code-row",
+ ),
+ pytest.param(
+ ("generated_file",),
+ [_registered("a_gen", minutes_from_t0=-10)],
+ [_block_row(_STAMPED)],
+ _EARLIER_ATTEMPT_THEN_CURRENT,
+ WorkflowRunStatus.terminated,
+ "generate",
+ id="generated-promise-not-met-by-an-earlier-attempt-published-file",
+ ),
+ pytest.param(
+ ("registered_download",),
+ [_registered("a_gen")],
+ [_block_row(_STAMPED)],
+ _ONE_ATTEMPT,
+ WorkflowRunStatus.terminated,
+ "download",
+ id="download-promise-not-met-by-a-published-file",
+ ),
+ pytest.param(
+ ("registered_download",),
+ [_registered("a_site", minutes_from_t0=-10)],
+ [],
+ _EARLIER_ATTEMPT_THEN_CURRENT,
+ WorkflowRunStatus.terminated,
+ "download",
+ id="download-promise-not-met-by-an-earlier-attempt-download",
+ ),
+ pytest.param(
+ ("registered_download",),
+ [_registered("a_site")],
+ [],
+ _EARLIER_ATTEMPT_THEN_CURRENT,
+ WorkflowRunStatus.completed,
+ None,
+ id="download-promise-met-by-a-current-attempt-download",
+ ),
+ pytest.param(
+ ("registered_download",),
+ [],
+ [_block_row({"downloaded_files": [{"file_name": "statement.pdf"}]})],
+ None,
+ WorkflowRunStatus.terminated,
+ "download",
+ id="download-promise-not-met-by-an-authored-filename",
+ ),
+ pytest.param(
+ ("registered_download",),
+ [],
+ [_block_row({"downloaded_file_urls": ["https://files.example/statement.pdf"]})],
+ None,
+ WorkflowRunStatus.terminated,
+ "download",
+ id="download-promise-not-met-by-an-authored-url",
+ ),
+ pytest.param(
+ ("registered_download", "generated_file"),
+ [_registered("a_site")],
+ [],
+ _ONE_ATTEMPT,
+ WorkflowRunStatus.terminated,
+ "generate",
+ id="both-promises-declared-and-only-a-site-download-registered",
+ ),
+ ],
+)
+async def test_each_declared_deliverable_kind_is_met_only_by_its_own_registered_evidence(
+ monkeypatch: pytest.MonkeyPatch,
+ kinds: tuple[str, ...],
+ downloaded: list[FileInfo],
+ run_blocks: list[SimpleNamespace],
+ attempts: tuple[tuple[int, datetime, datetime | None], ...] | None,
+ expected_status: WorkflowRunStatus,
+ unmet_promise: str | None,
+) -> None:
+ contract = _declared_contract(*kinds)
+ assert contract is not None
+ assert [criterion["kind"] for criterion in contract["criteria"]] == list(kinds)
+ service, run, statuses = _wire_finalize(
+ monkeypatch,
+ contract=contract,
+ downloaded=downloaded,
+ run_blocks=run_blocks,
+ attempts=None if attempts is None else [_attempt_row(*attempt) for attempt in attempts],
+ )
+
+ # `completed` is also what a swallowed grading error writes, so the verdict is read directly too.
+ verdict = await service._grade_completion_contract(run)
+ await service._finalize_workflow_run_status(
+ workflow_run_id=run.workflow_run_id,
+ workflow_run=run,
+ pre_finally_status=WorkflowRunStatus.running,
+ pre_finally_failure_reason=None,
+ )
+
+ assert verdict is not None
+ assert verdict.satisfied is (unmet_promise is None)
+ assert unmet_promise is None or (verdict.reason is not None and verdict.reason.endswith(f"to {unmet_promise}."))
+ assert statuses == [expected_status]
+
+
def test_interactive_copilot_routes_do_not_own_completion_contract_lifecycle() -> None:
from skyvern.forge.sdk.routes import workflow_copilot as route
@@ -552,24 +780,3 @@ def test_apply_proposed_workflow_route_is_bound_to_the_route_handler() -> None:
routes = [r for r in base_router.routes if getattr(r, "path", "") == "/workflow/copilot/apply-proposed-workflow"]
assert routes, "route not registered"
assert routes[0].endpoint.__name__ == "workflow_copilot_apply_proposed_workflow"
-
-
-def test_a_request_criterion_is_recognized_by_its_typed_deliverable_fields() -> None:
- """The persisted criterion carries deliverable_kind/output_path; the synthetic id is the
- copilot's separate internal marker, and keying on it alone misses every real request."""
- requested = SimpleNamespace(
- id="c0",
- outcome="the current electricity statement is downloaded as a PDF",
- deliverable_kind="registered_download",
- declared_deliverable_kind="registered_download",
- output_path="output.downloaded_files",
- )
- unrelated = SimpleNamespace(id="c1", outcome="a summary", deliverable_kind=None, output_path=None)
-
- assert contract_from_request_criteria([unrelated, requested]) is not None
- assert contract_from_request_criteria([unrelated]) is None
-
-
-def test_output_path_alone_identifies_a_requested_download() -> None:
- by_path = SimpleNamespace(id="c0", deliverable_kind=None, output_path="output.downloaded_files")
- assert contract_from_request_criteria([by_path]) is not None
diff --git a/tests/unit/workflow/test_code_block_escalation.py b/tests/unit/workflow/test_code_block_escalation.py
index 22707268e..67214d23c 100644
--- a/tests/unit/workflow/test_code_block_escalation.py
+++ b/tests/unit/workflow/test_code_block_escalation.py
@@ -31,7 +31,10 @@ from skyvern.forge.sdk.copilot.nav_attribution import (
proxy_owns_nav_codes,
target_owns_nav_codes,
)
-from skyvern.forge.sdk.copilot.reached_download_target import REGISTERED_DOWNLOAD_OUTPUT_KEYS
+from skyvern.forge.sdk.copilot.reached_download_target import (
+ GENERATED_FILE_ARTIFACT_IDS_KEY,
+ REGISTERED_DOWNLOAD_OUTPUT_KEYS,
+)
from skyvern.forge.sdk.core import skyvern_context
from skyvern.forge.sdk.core.skyvern_context import SkyvernContext
from skyvern.forge.sdk.db.repositories.workflow_runs import WorkflowRunsRepository
@@ -2419,6 +2422,45 @@ async def test_completed_heal_records_extracted_information(
assert "empty object" not in goal
+async def test_completed_heal_drops_the_published_file_key_from_extracted_information(
+ monkeypatch: pytest.MonkeyPatch, ai_fallback_flag: Callable[[str | None], None]
+) -> None:
+ """The fallback's answer is not the secure worker's, so it cannot carry the worker's published-file list."""
+ ai_fallback_flag("o_test")
+ record = AsyncMock(return_value=None)
+ monkeypatch.setattr(CodeBlock, "record_output_parameter_value", record)
+ state = _install_db_fakes(
+ monkeypatch,
+ final_status=TaskStatus.completed,
+ extracted_information={"order_total": "42.50", GENERATED_FILE_ARTIFACT_IDS_KEY: ["a_site"]},
+ )
+ block = _make_code_block(
+ steps=[CodeBlockStep(description="read the order total", line_start=1, line_end=1)],
+ prompt="Read the order total",
+ data_schema={
+ "type": "object",
+ "properties": {
+ "order_total": {"type": "string"},
+ GENERATED_FILE_ARTIFACT_IDS_KEY: {"type": "array", "items": {"type": "string"}},
+ },
+ },
+ )
+ exc = RuntimeError("rotted selector")
+
+ result = await _heal(block, _make_context(), exc, _recording_page(exc))
+
+ assert result is not None and result.success is True
+ code_row_outputs = [
+ update["output"]
+ for update in state["workflow_run_block_updates"]
+ if update["workflow_run_block_id"] == "wrb_test"
+ ]
+ assert code_row_outputs == [{"order_total": "42.50"}]
+ assert result.output_parameter_value == {"order_total": "42.50"}
+ assert record.await_args.args[2] == {"order_total": "42.50"}
+
+
@pytest.mark.parametrize("extracted", [{"total": "1"}, "oops", 3])
@pytest.mark.asyncio
async def test_a_non_list_answer_to_an_array_schema_fails_the_block(