Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
Complete runs that publish the generated file they promised (SKY-18053) (#8866)
Andrew Neilson0b68d4d04c57
diff --git a/skyvern/forge/sdk/copilot/reached_download_target.py b/skyvern/forge/sdk/copilot/reached_download_target.pyindex 1ceec8622..0fa7943d8 100644--- a/skyvern/forge/sdk/copilot/reached_download_target.py+++ b/skyvern/forge/sdk/copilot/reached_download_target.py@@ -201,6 +201,13 @@ def generated_file_artifact_ids(outputs: Iterable[Any]) -> frozenset[str]:)+def without_generated_file_stamp(output: Any) -> Any:+ """``output`` without the worker-owned key, for a code block row whose output the secure worker did not write."""+ if not isinstance(output, dict) or GENERATED_FILE_ARTIFACT_IDS_KEY not in output:+ return output+ return {key: value for key, value in output.items() if key != GENERATED_FILE_ARTIFACT_IDS_KEY}++def registered_download_proof_view(output: Any, generated: frozenset[str]) -> Any:"""``output`` with the rows of ``generated`` removed from its registration keys."""if not isinstance(output, dict) or not generated:diff --git a/skyvern/forge/sdk/copilot/tools/run_execution.py b/skyvern/forge/sdk/copilot/tools/run_execution.pyindex bfaf95cd3..81c577b4a 100644--- a/skyvern/forge/sdk/copilot/tools/run_execution.py+++ b/skyvern/forge/sdk/copilot/tools/run_execution.py@@ -273,9 +273,7 @@ from skyvern.forge.sdk.workflow.models.workflow import (WorkflowRunStatus,workflow_definition_sha256,)-from skyvern.forge.sdk.workflow.runtime_completion import contract_from_request_criteriafrom skyvern.forge.sdk.workflow.runtime_secret_bridge import consume_copilot_runtime_secret_values-from skyvern.forge.sdk.workflow.service import run_selection_is_partialfrom skyvern.schemas.proxy_location import ProxyLocationInput, runtime_proxy_locationfrom skyvern.schemas.self_heal import HealEpisodeDetail, HealStatusfrom skyvern.schemas.workflows import BlockStatus, BlockType@@ -1522,24 +1520,6 @@ def _selected_code_security_inputs(return code_blocks-def _requested_completion_contract(- ctx: CopilotContext,- runtime_workflow: Workflow,- labels_to_execute: list[str],-) -> dict[str, object] | None:- """The turn's own deliverable obligation, for a test run that executes the whole workflow.-- The obligation attaches to the workflow only when a proposal is accepted, which is after the- test run this grades; a selection that runs only part of the workflow stays ungraded."""- if ctx.request_policy is None:- return None- if not runtime_workflow.workflow_definition.blocks:- return None- if run_selection_is_partial(runtime_workflow, labels_to_execute):- return None- return contract_from_request_criteria(ctx.request_policy.graded_completion_criteria())--def _selected_blocks_require_sandbox(blocks: list[Any],*,@@ -4214,16 +4194,6 @@ async def _run_blocks_and_collect_debug(runtime_frontier_starter_url_seeded = seeded_runtime_workflow is not runtime_workflowruntime_workflow = seeded_runtime_workflow- requested_completion_contract = _requested_completion_contract(ctx, runtime_workflow, labels_to_execute)- if requested_completion_contract is not None:- runtime_workflow = runtime_workflow.model_copy(- update={- "workflow_definition": runtime_workflow.workflow_definition.model_copy(- update={"completion_contract": requested_completion_contract}- )- }- )-# Snapshot version persisted for a worker-dispatched run or an inline run of a definition that# was never persisted. The run is created against its exact workflow_id so prepare_workflow# reads parameter rows from the same definition execute_workflow receives. Without the inline@@ -4601,7 +4571,6 @@ async def _run_blocks_and_collect_debug(# the in-memory runtime copy instead registers block outputs against ids that# were never persisted, and every consumer keyed on the definition drops them.workflow_override=snapshot.workflow if dispatch_workflow is not None else runtime_workflow,- requested_completion_contract=requested_completion_contract,))except BaseException:diff --git a/skyvern/forge/sdk/copilot/tools/workflow_update.py b/skyvern/forge/sdk/copilot/tools/workflow_update.pyindex c043049b1..57969a6e4 100644--- a/skyvern/forge/sdk/copilot/tools/workflow_update.py+++ b/skyvern/forge/sdk/copilot/tools/workflow_update.py@@ -245,11 +245,14 @@ class CodeArtifactCompletionCriterion(BaseModel):terminal: bool | None = Noneoutput_path: str | None = Nonerequested_output_evidence_source: RequestedOutputEvidenceSource | None = None- deliverable_kind: Literal["registered_download"] | None = Field(+ deliverable_kind: Literal["registered_download", "generated_file"] | None = Field(default=None,description=(- "Set to `registered_download` only when this criterion promises the user a downloaded file. "- "Run finalization then verifies execution-layer file registration; authored code output cannot satisfy it."+ "Set only when this criterion promises the user a file. `registered_download`: a file the site serves and "+ "the browser downloads. `generated_file`: a file the workflow builds itself and publishes with "+ "`publish_file`, such as a screenshot or report. Run finalization verifies the matching "+ "execution-layer registration: a published file never satisfies `registered_download`, a site download "+ "never satisfies `generated_file`, and authored code output satisfies neither."),)judgment_predicate: JudgmentPredicate | None = Field(diff --git a/skyvern/forge/sdk/workflow/models/block.py b/skyvern/forge/sdk/workflow/models/block.pyindex 7b70b349e..ed39076ef 100644--- a/skyvern/forge/sdk/workflow/models/block.py+++ b/skyvern/forge/sdk/workflow/models/block.py@@ -168,6 +168,7 @@ from skyvern.forge.sdk.copilot.reached_download_target import (REGISTERED_DOWNLOAD_OUTPUT_KEYS,block_output_has_registered_download,code_is_download_intent,+ without_generated_file_stamp,)from skyvern.forge.sdk.copilot.turn_origin import TurnOriginfrom skyvern.forge.sdk.core import skyvern_context@@ -8279,7 +8280,9 @@ async def wrapper({default_args}):if updated_task.status == TaskStatus.completed:# The block's value is what its own return would have been; downloads are bound by# the recorder (_finalize_heal_result), the same as the inline success exit.- output_parameter_value = workflow_run_context.mask_secrets_in_data(updated_task.extracted_information)+ output_parameter_value = without_generated_file_stamp(+ workflow_run_context.mask_secrets_in_data(updated_task.extracted_information)+ )if record_output_parameter:await self.record_output_parameter_value(workflow_run_context, workflow_run_id, output_parameter_value@@ -10377,7 +10380,7 @@ async def wrapper({default_args}):# Mask resolved secrets (OTP codes, passwords) a user assigned to a local before they# reach captured locals, the persisted output, or the logged value. Mirrors# HttpRequestBlock and is stronger than the name-based excluded_parameter_keys filter.- result = workflow_run_context.mask_secrets_in_data(result)+ result = without_generated_file_stamp(workflow_run_context.mask_secrets_in_data(result))try:downloaded_files, skipped_file_names = await self._register_downloaded_files(diff --git a/skyvern/forge/sdk/workflow/runtime_completion.py b/skyvern/forge/sdk/workflow/runtime_completion.pyindex 8ad6ce356..1a1be7cd6 100644--- a/skyvern/forge/sdk/workflow/runtime_completion.py+++ b/skyvern/forge/sdk/workflow/runtime_completion.py@@ -18,12 +18,17 @@ import structlogLOG = structlog.get_logger()-# The only criterion kind graded today. Its evidence is the execution layer's own file registration,-# which no generated code can fake or swallow.+# Each kind is graded only from the execution layer's own registration of that kind of file, which+# no generated code can fake or swallow: a site download, or a file the secure worker published.CRITERION_REGISTERED_DOWNLOAD = "registered_download"+CRITERION_GENERATED_FILE = "generated_file"_CONTRACT_KEY = "completion_contract"-_SUPPORTED_KINDS = frozenset({CRITERION_REGISTERED_DOWNLOAD})+_SUPPORTED_KINDS = frozenset({CRITERION_REGISTERED_DOWNLOAD, CRITERION_GENERATED_FILE})+_UNMET_REASONS = {+ CRITERION_REGISTERED_DOWNLOAD: "The workflow did not produce the file it is declared to download.",+ CRITERION_GENERATED_FILE: "The workflow did not produce the file it is declared to generate.",+}# The field is public on the workflow schema, so a hand-written contract is bounded here._MAX_CRITERIA = 16_MAX_ID_CHARS = 128@@ -79,37 +84,31 @@ def grade_completion_contract(criteria: tuple[CompletionCriterion, ...],*,registered_download_count: int,+ generated_file_count: int = 0,) -> ContractVerdict:"""Grade declared criteria against execution-layer evidence."""- unmet: list[str] = []+ unmet: list[CompletionCriterion] = []+ unclaimed_generated = generated_file_countfor criterion in criteria:- if criterion.kind == CRITERION_REGISTERED_DOWNLOAD and registered_download_count < criterion.min_count:- unmet.append(criterion.id)+ if criterion.kind == CRITERION_REGISTERED_DOWNLOAD:+ if registered_download_count < criterion.min_count:+ unmet.append(criterion)+ elif criterion.kind == CRITERION_GENERATED_FILE:+ # A published file is not tied to a criterion, so each promise claims its own in declaration+ # order: the unmet ids say how many promises are short, not which files are missing.+ if unclaimed_generated < criterion.min_count:+ unmet.append(criterion)+ else:+ unclaimed_generated -= criterion.min_countif not unmet:return ContractVerdict(satisfied=True, unmet_criterion_ids=(), reason=None)return ContractVerdict(satisfied=False,- unmet_criterion_ids=tuple(unmet),- reason="The workflow did not produce the file it is declared to download.",+ unmet_criterion_ids=tuple(criterion.id for criterion in unmet),+ reason=_UNMET_REASONS[unmet[0].kind],)-def contract_from_request_criteria(criteria: object) -> dict[str, object] | None:- """Project the request's own typed completion criteria into a workflow-carried contract.-- The obligation comes from what the user asked for, never from the shape of the code the product- generated: a block that merely reports a download-shaped result promises nothing, and a request- that asked for a file promises one however the block is written."""- if not isinstance(criteria, (list, tuple)):- return None- if not any(_criterion_requests_a_download(criterion) for criterion in criteria):- return None- return {- "schema_version": 1,- "criteria": [{"id": "requested_download", "kind": CRITERION_REGISTERED_DOWNLOAD, "min_count": 1}],- }--def contract_from_code_artifact_metadata(metadata: object) -> dict[str, object] | None:"""Project the acting model's structured deliverable declaration into the runtime contract.@@ -119,7 +118,8 @@ def contract_from_code_artifact_metadata(metadata: object) -> dict[str, object]"""if not isinstance(metadata, Mapping):return None- seen_ids: set[str] = set()+ # Ids are model-chosen per block, so two blocks can reuse one id for promises of different kinds.+ seen: set[tuple[str, str]] = set()criteria: list[dict[str, object]] = []for artifact in metadata.values():if not isinstance(artifact, Mapping):@@ -128,15 +128,14 @@ def contract_from_code_artifact_metadata(metadata: object) -> dict[str, object]if not isinstance(raw_criteria, list):continuefor item in raw_criteria:- if not isinstance(item, Mapping) or item.get("deliverable_kind") != CRITERION_REGISTERED_DOWNLOAD:+ kind = item.get("deliverable_kind") if isinstance(item, Mapping) else None+ if not isinstance(kind, str) or kind not in _SUPPORTED_KINDS:continue- identifier = (- str(item.get("id") or CRITERION_REGISTERED_DOWNLOAD).strip() or CRITERION_REGISTERED_DOWNLOAD- )[:_MAX_ID_CHARS]- if identifier in seen_ids:+ identifier = (str(item.get("id") or kind).strip() or kind)[:_MAX_ID_CHARS]+ if (identifier, kind) in seen:continue- seen_ids.add(identifier)- criteria.append({"id": identifier, "kind": CRITERION_REGISTERED_DOWNLOAD, "min_count": 1})+ seen.add((identifier, kind))+ criteria.append({"id": identifier, "kind": kind, "min_count": 1})if not criteria:return Nonereturn {"schema_version": 1, "criteria": criteria}@@ -161,21 +160,3 @@ def with_contract(definition: dict, carried: dict[str, object] | None) -> dict:return definitiondefinition[_CONTRACT_KEY] = carriedreturn definition---def _criterion_requests_a_download(criterion: object) -> bool:- """Whether a request criterion asks for a registered download.-- The request classifier types this as ``deliverable_kind``/``output_path``; the synthetic id is- the copilot's own internal marker for the same obligation. Both are request-derived."""- from skyvern.forge.sdk.copilot.reached_download_target import REGISTERED_DOWNLOAD_REQUESTED_OUTPUT_PATHS- from skyvern.forge.sdk.copilot.request_policy import REGISTERED_DOWNLOAD_COMPLETION_CRITERION_ID-- for attr in ("deliverable_kind", "declared_deliverable_kind"):- if str(getattr(criterion, attr, "") or "").strip() == CRITERION_REGISTERED_DOWNLOAD:- return True- if str(getattr(criterion, "output_path", "") or "").strip() in REGISTERED_DOWNLOAD_REQUESTED_OUTPUT_PATHS:- return True- # The copilot's own synthetic marker for the same obligation, compared by id so this stays- # readable from an untyped criterion.- return str(getattr(criterion, "id", "") or "") == REGISTERED_DOWNLOAD_COMPLETION_CRITERION_IDdiff --git a/skyvern/forge/sdk/workflow/service.py b/skyvern/forge/sdk/workflow/service.pyindex 540ac8775..32c2aa5dd 100644--- a/skyvern/forge/sdk/workflow/service.py+++ b/skyvern/forge/sdk/workflow/service.py@@ -13753,7 +13753,7 @@ class WorkflowService:attempt_rows=attempt_rows,attempt_number=attempt_number,)- # A file the run's own code generated is a real download but not a delivered one.+ # A file the run's own code generated is left out of the download count only.run_blocks = await app.DATABASE.observer.get_workflow_run_blocks(workflow_run_id=workflow_run.workflow_run_id,organization_id=workflow_run.organization_id,@@ -13767,7 +13767,15 @@ class WorkflowService:# downloads), so they are separate files and both count.registered_ids = {file.artifact_id for file in registered if file.artifact_id}download_count = len(registered) + len(session_download_ids - registered_ids)- verdict = grade_completion_contract(criteria, registered_download_count=download_count)+ # A CODE row carries this key only from the secure worker: the in-process executor and the AI+ # fallback drop it. On any other block type it is authored data.+ worker_generated = generated_file_artifact_ids(+ block.output for block in run_blocks if block.block_type == BlockType.CODE+ )+ generated_file_count = len({file.artifact_id for file in files if file.artifact_id in worker_generated})+ verdict = grade_completion_contract(+ criteria, registered_download_count=download_count, generated_file_count=generated_file_count+ )LOG.info("workflow_completion_contract_graded",workflow_run_id=workflow_run.workflow_run_id,@@ -13776,6 +13784,7 @@ class WorkflowService:registered_download_count=len(registered),session_download_count=len(session_download_ids),graded_download_count=download_count,+ generated_file_count=generated_file_count,)return verdictexcept Exception:diff --git a/tests/unit/test_code_block_downloads.py b/tests/unit/test_code_block_downloads.pyindex c4a42077b..09df21738 100644--- a/tests/unit/test_code_block_downloads.py+++ b/tests/unit/test_code_block_downloads.py@@ -45,6 +45,7 @@ from skyvern.forge.sdk.browser_network_egress_monitor import BrowserNetworkEgresfrom skyvern.forge.sdk.copilot.reached_download_target import (block_output_has_registered_download,code_is_download_intent,+ generated_file_artifact_ids,)from skyvern.forge.sdk.copilot.tools.run_execution import build_test_evidence_packetfrom skyvern.forge.sdk.core import skyvern_context@@ -1736,6 +1737,38 @@ async def test_authored_registration_keys_are_dropped_without_host_evidence(assert downloaded_file_count_from_output(result.output_parameter_value) is None+async def test_in_process_block_cannot_stamp_a_site_download_as_a_published_file(+ monkeypatch: pytest.MonkeyPatch, _isolated_download_path: str+) -> None:+ """Only the secure worker writes the published-file key, so a block naming a real download under it earns nothing."""+ skyvern_context.set(SkyvernContext(organization_id="o_1", workflow_run_id="wr_1", run_id="wr_1"))++ site_download = FileInfo(+ url="https://api.example.com/v1/artifacts/a_site/content?artifact_name=invoice.pdf",+ filename="invoice.pdf",+ checksum="deadbeef",+ artifact_id="a_site",+ modified_at=datetime(2026, 6, 14, 12, 0, tzinfo=UTC),+ )+ _fake_storage_app(monkeypatch, save=AsyncMock(), get=AsyncMock(side_effect=[[], [site_download]]))+ _wire_block_runtime(monkeypatch)++ block = CodeBlock(+ label="code_download",+ code="rows = 3\ngenerated_file_artifact_ids = ['a_site']",+ output_parameter=_output_parameter("code_out"),+ )+ result = await block.execute(workflow_run_id="wr_1", workflow_run_block_id="", organization_id="o_1")++ assert result.success is True+ for output in (_persisted_output(), result.output_parameter_value):+ assert output["rows"] == 3+ assert output["downloaded_file_artifact_ids"] == ["a_site"]+ assert "generated_file_artifact_ids" not in output+ assert generated_file_artifact_ids([output]) == frozenset()++@pytest.mark.asyncioasync def test_registration_timeout_does_not_fail_a_block_that_downloaded(monkeypatch: pytest.MonkeyPatch, _isolated_download_path: strdiff --git a/tests/unit/test_workflow_runtime_completion.py b/tests/unit/test_workflow_runtime_completion.pyindex bd87a4dcd..01c3aafd3 100644--- a/tests/unit/test_workflow_runtime_completion.py+++ b/tests/unit/test_workflow_runtime_completion.py@@ -6,12 +6,15 @@ verdict does not depend on which engine ran the blocks or how generated code desfrom __future__ import annotations+from datetime import datetime, timedelta, timezonefrom types import SimpleNamespacefrom unittest.mock import AsyncMockimport pytestfrom skyvern.exceptions import WorkflowNotFound+from skyvern.forge.sdk.copilot.tools.workflow_update import CodeArtifactCompletionCriterion+from skyvern.forge.sdk.db.models import WorkflowRunAttemptModelfrom skyvern.forge.sdk.schemas.files import FileInfofrom skyvern.forge.sdk.workflow import service as service_modulefrom skyvern.forge.sdk.workflow.models.workflow import WorkflowRunStatus@@ -19,12 +22,13 @@ from skyvern.forge.sdk.workflow.runtime_completion import (CompletionCriterion,carried_contract,contract_from_code_artifact_metadata,- contract_from_request_criteria,grade_completion_contract,parse_completion_contract,with_contract,)from skyvern.forge.sdk.workflow.service import run_selection_is_partial+from skyvern.schemas.workflows import BlockType+from tests.unit.conftest import FakeWorkflowRunAttemptsRepository_DOWNLOAD_CONTRACT = {"completion_contract": {@@ -106,7 +110,7 @@ def _downloaded_file(name: str, *, artifact_id: str | None = None):return SimpleNamespace(artifact_id=artifact_id, filename=name, checksum=None, file_size=None)-def _wire_finalize(monkeypatch, *, contract, downloaded):+def _wire_finalize(monkeypatch, *, contract, downloaded, run_blocks=(), attempts=None):"""A WorkflowService with just enough wired to exercise the finalize status decision."""from skyvern.forge.sdk.workflow.service import WorkflowService@@ -144,6 +148,19 @@ def _wire_finalize(monkeypatch, *, contract, downloaded):)),)+ monkeypatch.setattr(+ service_module.app.DATABASE.observer, "get_workflow_run_blocks", AsyncMock(return_value=list(run_blocks))+ )+ if attempts is not None:+ run.failure_reason = None+ run.started_at = attempts[-1].started_at+ run.finished_at = None+ monkeypatch.setattr(+ service_module.app.DATABASE, "workflow_run_attempts", FakeWorkflowRunAttemptsRepository(attempts)+ )+ monkeypatch.setattr(+ service_module.app.DATABASE.artifacts, "list_download_artifacts_for_attempt", AsyncMock(return_value=[])+ )return service, run, statuses@@ -298,27 +315,6 @@ async def test_finalize_grades_the_version_execution_loaded_when_it_is_soft_deleassert statuses == [WorkflowRunStatus.terminated]-def test_contract_comes_from_the_request_not_the_code() -> None:- """The obligation is what the user asked for, never the shape of the generated code."""- from skyvern.forge.sdk.copilot.completion_verification import registered_download_completion_criterion- from skyvern.forge.sdk.copilot.request_policy import CompletionCriterion as RequestCriterion-- assert contract_from_request_criteria([registered_download_completion_criterion()]) is not None- assert contract_from_request_criteria([RequestCriterion(id="c0", outcome="something else")]) is None- assert contract_from_request_criteria([]) is None- assert contract_from_request_criteria(None) is None---def test_requested_contract_round_trips_through_the_parser() -> None:- from skyvern.forge.sdk.copilot.completion_verification import registered_download_completion_criterion-- contract = contract_from_request_criteria([registered_download_completion_criterion()])- criteria = parse_completion_contract({"completion_contract": contract})- assert [c.kind for c in criteria] == ["registered_download"]- assert grade_completion_contract(criteria, registered_download_count=0).satisfied is False- assert grade_completion_contract(criteria, registered_download_count=1).satisfied is True--def test_download_contract_comes_from_model_declared_artifact_metadata() -> None:metadata = {"download_statement": {@@ -356,6 +352,31 @@ def test_ordinary_artifact_criterion_does_not_create_a_download_contract() -> Noassert contract_from_code_artifact_metadata(metadata) is None+ ("published", "unmet"),+ [(0, ("pdf", "xlsx", "zip")), (1, ("xlsx", "zip")), (2, ("zip",)), (3, ())],+)+def test_each_generated_promise_needs_its_own_published_file(published: int, unmet: tuple[str, ...]) -> None:+ contract = contract_from_code_artifact_metadata(+ {+ "build_report": {+ "completion_criteria": [+ {"id": criterion_id, "deliverable_kind": "generated_file"}+ for criterion_id in ("pdf", "xlsx", "zip")+ ]+ }+ }+ )++ verdict = grade_completion_contract(+ parse_completion_contract({"completion_contract": contract}),+ registered_download_count=0,+ generated_file_count=published,+ )++ assert (verdict.satisfied, verdict.unmet_criterion_ids) == (not unmet, unmet)++@pytest.mark.asyncioasync def test_finalize_skips_grading_a_partial_run(monkeypatch: pytest.MonkeyPatch) -> None:"""A frontier run of a block subset was never asked to produce the whole deliverable."""@@ -535,6 +556,213 @@ async def test_one_file_reported_by_both_download_sources_does_not_satisfy_a_twoassert statuses == [WorkflowRunStatus.terminated]+_T0 = datetime(2026, 1, 1, 12, tzinfo=timezone.utc)+_EARLIER_ATTEMPT_THEN_CURRENT = (+ (1, _T0 - timedelta(minutes=20), _T0 - timedelta(minutes=5)),+ (2, _T0 - timedelta(minutes=1), None),+)+_ONE_ATTEMPT = ((1, _T0 - timedelta(minutes=1), None),)+++def _declared_contract(*kinds: str) -> dict[str, object] | None:+ """What the real producer carries when each kind is declared by its own block under one shared criterion id."""+ return contract_from_code_artifact_metadata(+ {+ f"block_{index}": {+ "completion_criteria": [+ CodeArtifactCompletionCriterion.model_validate(+ {"id": "capture", "deliverable_kind": kind}+ ).model_dump(mode="json")+ ]+ }+ for index, kind in enumerate(kinds)+ }+ )+++def _registered(artifact_id: str, *, minutes_from_t0: int = 0) -> FileInfo:+ return FileInfo(+ url=f"s3://b/{artifact_id}", artifact_id=artifact_id, modified_at=_T0 + timedelta(minutes=minutes_from_t0)+ )+++def _block_row(output: dict[str, object], block_type: BlockType = BlockType.CODE) -> SimpleNamespace:+ return SimpleNamespace(block_type=block_type, output=output)+++def _attempt_row(number: int, started_at: datetime, finished_at: datetime | None) -> WorkflowRunAttemptModel:+ return WorkflowRunAttemptModel(+ workflow_run_id="wr_1",+ attempt_number=number,+ organization_id="o_1",+ status="running",+ started_at=started_at,+ created_at=started_at,+ finished_at=finished_at,+ )+++_STAMPED = {"generated_file_artifact_ids": ["a_gen"]}+++ ("kinds", "downloaded", "run_blocks", "attempts", "expected_status", "unmet_promise"),+ [+ pytest.param(+ ("generated_file",),+ [_registered("a_gen")],+ [_block_row(_STAMPED)],+ _ONE_ATTEMPT,+ WorkflowRunStatus.completed,+ None,+ id="generated-promise-met-by-a-same-attempt-published-file",+ ),+ pytest.param(+ ("generated_file",),+ [_registered("a_gen")],+ [_block_row(_STAMPED)],+ _EARLIER_ATTEMPT_THEN_CURRENT,+ WorkflowRunStatus.completed,+ None,+ id="generated-promise-met-by-a-current-attempt-published-file-after-a-retry",+ ),+ pytest.param(+ ("generated_file",),+ [],+ [],+ None,+ WorkflowRunStatus.terminated,+ "generate",+ id="generated-promise-with-nothing-registered",+ ),+ pytest.param(+ ("generated_file",),+ [_registered("a_site")],+ [_block_row({"rows": 3, "downloaded_file_artifact_ids": ["a_site"]})],+ _ONE_ATTEMPT,+ WorkflowRunStatus.terminated,+ "generate",+ id="generated-promise-not-met-by-a-site-download-bound-to-an-in-process-code-row",+ ),+ pytest.param(+ ("generated_file",),+ [_registered("a_gen")],+ [],+ _ONE_ATTEMPT,+ WorkflowRunStatus.terminated,+ "generate",+ id="generated-promise-not-met-by-an-unstamped-site-download",+ ),+ pytest.param(+ ("generated_file",),+ [_registered("a_gen")],+ [_block_row(_STAMPED, BlockType.TEXT_PROMPT)],+ _ONE_ATTEMPT,+ WorkflowRunStatus.terminated,+ "generate",+ id="generated-promise-not-met-by-a-stamp-on-a-non-code-row",+ ),+ pytest.param(+ ("generated_file",),+ [_registered("a_gen", minutes_from_t0=-10)],+ [_block_row(_STAMPED)],+ _EARLIER_ATTEMPT_THEN_CURRENT,+ WorkflowRunStatus.terminated,+ "generate",+ id="generated-promise-not-met-by-an-earlier-attempt-published-file",+ ),+ pytest.param(+ ("registered_download",),+ [_registered("a_gen")],+ [_block_row(_STAMPED)],+ _ONE_ATTEMPT,+ WorkflowRunStatus.terminated,+ "download",+ id="download-promise-not-met-by-a-published-file",+ ),+ pytest.param(+ ("registered_download",),+ [_registered("a_site", minutes_from_t0=-10)],+ [],+ _EARLIER_ATTEMPT_THEN_CURRENT,+ WorkflowRunStatus.terminated,+ "download",+ id="download-promise-not-met-by-an-earlier-attempt-download",+ ),+ pytest.param(+ ("registered_download",),+ [_registered("a_site")],+ [],+ _EARLIER_ATTEMPT_THEN_CURRENT,+ WorkflowRunStatus.completed,+ None,+ id="download-promise-met-by-a-current-attempt-download",+ ),+ pytest.param(+ ("registered_download",),+ [],+ [_block_row({"downloaded_files": [{"file_name": "statement.pdf"}]})],+ None,+ WorkflowRunStatus.terminated,+ "download",+ id="download-promise-not-met-by-an-authored-filename",+ ),+ pytest.param(+ ("registered_download",),+ [],+ [_block_row({"downloaded_file_urls": ["https://files.example/statement.pdf"]})],+ None,+ WorkflowRunStatus.terminated,+ "download",+ id="download-promise-not-met-by-an-authored-url",+ ),+ pytest.param(+ ("registered_download", "generated_file"),+ [_registered("a_site")],+ [],+ _ONE_ATTEMPT,+ WorkflowRunStatus.terminated,+ "generate",+ id="both-promises-declared-and-only-a-site-download-registered",+ ),+ ],+)+async def test_each_declared_deliverable_kind_is_met_only_by_its_own_registered_evidence(+ monkeypatch: pytest.MonkeyPatch,+ kinds: tuple[str, ...],+ downloaded: list[FileInfo],+ run_blocks: list[SimpleNamespace],+ attempts: tuple[tuple[int, datetime, datetime | None], ...] | None,+ expected_status: WorkflowRunStatus,+ unmet_promise: str | None,+) -> None:+ contract = _declared_contract(*kinds)+ assert contract is not None+ assert [criterion["kind"] for criterion in contract["criteria"]] == list(kinds)+ service, run, statuses = _wire_finalize(+ monkeypatch,+ contract=contract,+ downloaded=downloaded,+ run_blocks=run_blocks,+ attempts=None if attempts is None else [_attempt_row(*attempt) for attempt in attempts],+ )++ # `completed` is also what a swallowed grading error writes, so the verdict is read directly too.+ verdict = await service._grade_completion_contract(run)+ await service._finalize_workflow_run_status(+ workflow_run_id=run.workflow_run_id,+ workflow_run=run,+ pre_finally_status=WorkflowRunStatus.running,+ pre_finally_failure_reason=None,+ )++ assert verdict is not None+ assert verdict.satisfied is (unmet_promise is None)+ assert unmet_promise is None or (verdict.reason is not None and verdict.reason.endswith(f"to {unmet_promise}."))+ assert statuses == [expected_status]++def test_interactive_copilot_routes_do_not_own_completion_contract_lifecycle() -> None:from skyvern.forge.sdk.routes import workflow_copilot as route@@ -552,24 +780,3 @@ def test_apply_proposed_workflow_route_is_bound_to_the_route_handler() -> None:routes = [r for r in base_router.routes if getattr(r, "path", "") == "/workflow/copilot/apply-proposed-workflow"]assert routes, "route not registered"assert routes[0].endpoint.__name__ == "workflow_copilot_apply_proposed_workflow"---def test_a_request_criterion_is_recognized_by_its_typed_deliverable_fields() -> None:- """The persisted criterion carries deliverable_kind/output_path; the synthetic id is the- copilot's separate internal marker, and keying on it alone misses every real request."""- requested = SimpleNamespace(- id="c0",- outcome="the current electricity statement is downloaded as a PDF",- deliverable_kind="registered_download",- declared_deliverable_kind="registered_download",- output_path="output.downloaded_files",- )- unrelated = SimpleNamespace(id="c1", outcome="a summary", deliverable_kind=None, output_path=None)-- assert contract_from_request_criteria([unrelated, requested]) is not None- assert contract_from_request_criteria([unrelated]) is None---def test_output_path_alone_identifies_a_requested_download() -> None:- by_path = SimpleNamespace(id="c0", deliverable_kind=None, output_path="output.downloaded_files")- assert contract_from_request_criteria([by_path]) is not Nonediff --git a/tests/unit/workflow/test_code_block_escalation.py b/tests/unit/workflow/test_code_block_escalation.pyindex 22707268e..67214d23c 100644--- a/tests/unit/workflow/test_code_block_escalation.py+++ b/tests/unit/workflow/test_code_block_escalation.py@@ -31,7 +31,10 @@ from skyvern.forge.sdk.copilot.nav_attribution import (proxy_owns_nav_codes,target_owns_nav_codes,)-from skyvern.forge.sdk.copilot.reached_download_target import REGISTERED_DOWNLOAD_OUTPUT_KEYS+from skyvern.forge.sdk.copilot.reached_download_target import (+ GENERATED_FILE_ARTIFACT_IDS_KEY,+ REGISTERED_DOWNLOAD_OUTPUT_KEYS,+)from skyvern.forge.sdk.core import skyvern_contextfrom skyvern.forge.sdk.core.skyvern_context import SkyvernContextfrom skyvern.forge.sdk.db.repositories.workflow_runs import WorkflowRunsRepository@@ -2419,6 +2422,45 @@ async def test_completed_heal_records_extracted_information(assert "empty object" not in goal+async def test_completed_heal_drops_the_published_file_key_from_extracted_information(+ monkeypatch: pytest.MonkeyPatch, ai_fallback_flag: Callable[[str | None], None]+) -> None:+ """The fallback's answer is not the secure worker's, so it cannot carry the worker's published-file list."""+ ai_fallback_flag("o_test")+ record = AsyncMock(return_value=None)+ monkeypatch.setattr(CodeBlock, "record_output_parameter_value", record)+ state = _install_db_fakes(+ monkeypatch,+ final_status=TaskStatus.completed,+ extracted_information={"order_total": "42.50", GENERATED_FILE_ARTIFACT_IDS_KEY: ["a_site"]},+ )+ block = _make_code_block(+ steps=[CodeBlockStep(description="read the order total", line_start=1, line_end=1)],+ prompt="Read the order total",+ data_schema={+ "type": "object",+ "properties": {+ "order_total": {"type": "string"},+ GENERATED_FILE_ARTIFACT_IDS_KEY: {"type": "array", "items": {"type": "string"}},+ },+ },+ )+ exc = RuntimeError("rotted selector")++ result = await _heal(block, _make_context(), exc, _recording_page(exc))++ assert result is not None and result.success is True+ code_row_outputs = [+ update["output"]+ for update in state["workflow_run_block_updates"]+ if update["workflow_run_block_id"] == "wrb_test"+ ]+ assert code_row_outputs == [{"order_total": "42.50"}]+ assert result.output_parameter_value == {"order_total": "42.50"}+ assert record.await_args.args[2] == {"order_total": "42.50"}++@pytest.mark.parametrize("extracted", [{"total": "1"}, "oops", 3])@pytest.mark.asyncioasync def test_a_non_list_answer_to_an_array_schema_fails_the_block(