Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
Deliver the download instead of shipping a draft the build test disproved (SKY-13230) (#7914)
Andrew Neilson4bb0105c2b41
diff --git a/docs/api-reference/openapi.json b/docs/api-reference/openapi.jsonindex 47850cca7..2949a05e9 100644--- a/docs/api-reference/openapi.json+++ b/docs/api-reference/openapi.json@@ -27487,6 +27487,19 @@}],"title": "Workflow System Prompt"+ },+ "completion_contract": {+ "anyOf": [+ {+ "additionalProperties": true,+ "type": "object"+ },+ {+ "type": "null"+ }+ ],+ "title": "Completion Contract",+ "description": "Copilot-managed: what a run of this workflow must produce, graded at run finalization. Derived from the request when a workflow is accepted; not intended to be authored by hand."}},"type": "object",@@ -27730,6 +27743,19 @@}],"title": "Workflow System Prompt"+ },+ "completion_contract": {+ "anyOf": [+ {+ "additionalProperties": true,+ "type": "object"+ },+ {+ "type": "null"+ }+ ],+ "title": "Completion Contract",+ "description": "Copilot-managed: what a run of this workflow must produce, graded at run finalization. Derived from the request when a workflow is accepted; not intended to be authored by hand."}},"type": "object",diff --git a/skyvern/forge/sdk/copilot/enforcement.py b/skyvern/forge/sdk/copilot/enforcement.pyindex 60b163a90..2430ba626 100644--- a/skyvern/forge/sdk/copilot/enforcement.py+++ b/skyvern/forge/sdk/copilot/enforcement.py@@ -130,6 +130,7 @@ from skyvern.forge.sdk.copilot.run_outcome import ()from skyvern.forge.sdk.copilot.runtime import (AgentContext,+ diagnosis_repair_obligation_open,)from skyvern.forge.sdk.copilot.screenshot_utils import ScreenshotEntryfrom skyvern.forge.sdk.copilot.terminal_predicates import (@@ -172,6 +173,9 @@ POST_INTERMEDIATE_SUCCESS_NUDGE = copilot_config_defaults.POST_INTERMEDIATE_SUCCMAX_POST_UPDATE_NUDGES = 2MAX_INTERMEDIATE_NUDGES = 8MAX_FAILED_TEST_NUDGES = 2+# Repair rounds the typed obligation may force past the ordinary nudge budget before a turn is+# allowed to report an unrepairable failure.+MAX_REPAIR_OBLIGATION_NUDGES = 6MAX_FORMAT_NUDGES = 2MAX_NO_WORKFLOW_NUDGES = 2MAX_DISCOVERY_ENTRYPOINT_URL_QUESTION_NUDGES = 2@@ -1155,6 +1159,29 @@ def _needs_explore_without_workflow_nudge(ctx: Any) -> bool:return nudge_count < MAX_EXPLORE_WITHOUT_WORKFLOW_NUDGES+def _repair_obligation_live(ctx: AgentContext) -> bool:+ """The typed obligation, bounded. It is discharged by evidence, but must not outlive the+ evidence that produced it: a failure that looks repairable and is not would otherwise be+ re-nudged until the turn budget dies, burying the blocker the model is trying to report."""+ if not diagnosis_repair_obligation_open(ctx):+ return False+ return _get_int(ctx, "repair_obligation_nudge_count") < MAX_REPAIR_OBLIGATION_NUDGES+++def _repair_obligation_blocks_finalize(ctx: AgentContext, result: RunResultStreaming | None) -> bool:+ """True when the turn is about to end while the typed contract still says the failure is repairable.++ The nudge counters bound *repetition of a nudge*; they were never evidence that the failure had been+ addressed, so once they ran out a turn could finalize a draft its own build test disproved. Ending is+ still bounded — by the total-turn timeout and max-turns, both of which exit as WIP rather than as an+ accept-ready proposal."""+ if not _repair_obligation_live(ctx):+ return False+ parsed = _parse_normalized_final_response(result)+ # ASK_QUESTION is a legitimate exit: it needs the user, not another repair round.+ return parsed is not None and parsed.get("type") == "REPLY"++def _needs_failed_test_nudge(ctx: Any) -> bool:"""Return True when the last test failed and the agent hasn't iterated yet."""# A permanent nav error cannot be 'fix the workflow and retry' material —@@ -1345,7 +1372,9 @@ def _check_enforcement():return _nudge(config, "post_update")- if _post_run_observed_reply_can_finalize(ctx, result):+ # Observing the reached page is the first repair move, not the last: while the typed contract+ # still says REPAIR, a reply that reports the failure instead of acting on it re-enters the loop.+ if not _repair_obligation_live(ctx) and _post_run_observed_reply_can_finalize(ctx, result):return None_maybe_stash_terminal_challenge_halt(ctx)@@ -1397,6 +1426,12 @@ def _check_enforcement(return _nudge(config, "post_failed_test_inspect_first")return _nudge(config, "post_failed_test")+ # Counters exhausted but the contract still says REPAIR: keep steering rather than finalize a+ # draft the build test disproved.+ if _repair_obligation_blocks_finalize(ctx, result):+ ctx.repair_obligation_nudge_count = _get_int(ctx, "repair_obligation_nudge_count") + 1+ return _nudge(config, "post_failed_test")+# Response-time gate: peek at the model's final output to tell ASK_QUESTION# (always allowed) from a REPLY with a coverage gap or progress-narration.# Only runs when no state-based nudge fired.diff --git a/skyvern/forge/sdk/copilot/reached_download_target.py b/skyvern/forge/sdk/copilot/reached_download_target.pyindex a7b5eb0e4..5416e44cf 100644--- a/skyvern/forge/sdk/copilot/reached_download_target.py+++ b/skyvern/forge/sdk/copilot/reached_download_target.py@@ -9,13 +9,17 @@ from dataclasses import dataclassfrom enum import Enumfrom typing import Any, Literal, cast-DownloadKind = Literal["registered", "attribute", "extension"]+DownloadKind = Literal["registered", "attribute", "extension", "observed"]-# ``registered`` is the only kind backed by an actual browser download having fired (S1);-# ``attribute``/``extension`` are S2 predictions from a scouted link.+# ``registered`` (S1) and ``observed`` (S3) are backed by an actual browser download having fired;+# ``attribute``/``extension`` are S2 predictions from a scouted link's href shape.DOWNLOAD_KIND_REGISTERED: DownloadKind = "registered"DOWNLOAD_KIND_ATTRIBUTE: DownloadKind = "attribute"DOWNLOAD_KIND_EXTENSION: DownloadKind = "extension"+# A download the scout's own click produced. Href shape cannot see these: a query-parameter URL+# serving a file via Content-Disposition has no extension and no ``download`` attribute, so the+# href-shape prediction is blind to it and only the fired download proves the affordance.+DOWNLOAD_KIND_OBSERVED: DownloadKind = "observed"# S2 may only mint a prediction; ``registered`` is S1-only and must never come from a nav target._PREDICTED_DOWNLOAD_KINDS: frozenset[str] = frozenset({DOWNLOAD_KIND_ATTRIBUTE, DOWNLOAD_KIND_EXTENSION})@@ -69,6 +73,7 @@ NAV_TARGET_DOWNLOAD_KIND_KEY = "download_kind"class _SourceStepKind(str, Enum):trajectory_recency = "trajectory_recency"registered_output = "registered_output"+ observed_download = "observed_download"@dataclass(frozen=True)@@ -150,6 +155,23 @@ def block_output_has_registered_download(block_output: Any) -> bool:return any(bool(block_output.get(key)) for key in REGISTERED_DOWNLOAD_OUTPUT_KEYS)+def derive_from_observed_download(*, selector: str, affordance_text: str = "") -> ReachedDownloadTarget | None:+ """S3: mint a target from a download the scout's own click just produced.++ Stronger evidence than S2 (the download fired) while still carrying the selector S1 lacks, so the+ synthesizer can compile its terminal ``expect_download`` step against the exercised affordance."""+ selector = _summary_str(selector)+ if not selector:+ return None+ return ReachedDownloadTarget(+ selector=selector,+ affordance_text=_summary_str(affordance_text),+ download_kind=DOWNLOAD_KIND_OBSERVED,+ source_step=_SourceStepKind.observed_download.value,+ already_registered=False,+ )++def derive_from_block_outputs(block_outputs_by_label: Any) -> ReachedDownloadTarget | None:"""S1: confirm a reached download from a browser download already registered into a block output.diff --git a/skyvern/forge/sdk/copilot/runtime.py b/skyvern/forge/sdk/copilot/runtime.pyindex de5fff6b2..e7389009a 100644--- a/skyvern/forge/sdk/copilot/runtime.py+++ b/skyvern/forge/sdk/copilot/runtime.py@@ -528,6 +528,10 @@ class AgentContext:# Selector of an in-flight click, captured pre-dispatch so a failed/timed-out click can gate a# settle re-perception on whether that selector still resolves to a live element.pending_scout_click_selector: str | None = None+ # Browser-session download filenames snapshotted before a scout click, so the post-hook can tell+ # a download this click produced from one an earlier click left behind.+ pending_scout_download_snapshot: frozenset[str] | None = None+ repair_obligation_nudge_count: int = 0# (selector, ambiguous) verdict from a pre-dispatch live count probe, applied to the recorded# interaction only when the post-action resolved selector matches the probed one.pending_scout_ambiguous: tuple[str, bool] | None = None@@ -569,6 +573,23 @@ class AgentContext:blocker_signal_claimant: TurnClaimant | None = None+def diagnosis_repair_obligation_open(ctx: AgentContext) -> bool:+ """True while the latest diagnosis contract says the failed run is repairable and no later run has+ discharged it. The build-test oracle already reported the failure; this keeps the loop on the hook for+ consuming it, so a turn cannot observe the reached page once and then finalize a draft the run disproved.++ Keyed on the typed decision, never on a nudge count: the contract's own ASK/STOP/NO_CHANGE transitions+ discharge it, and the operational total-turn timeout bounds it."""+ # Deferred: diagnosis_repair_contract reaches this module through context, so a module-level+ # import cycles.+ from skyvern.forge.sdk.copilot.diagnosis_repair_contract import RepairNextAction++ contract = getattr(ctx, "latest_diagnosis_repair_contract", None)+ if contract is None:+ return False+ return contract.repair_decision.next_action is RepairNextAction.REPAIR++def output_contract_ladder_unresolved(ctx: AgentContext) -> bool:"""True while an output-contract signature has a live actuation ladder — a landed actuation or a GRANTEDadvisory — that has not yet reached a typed terminal or a dispatched (CONSUMED) run. Loop and churn detectorsdiff --git a/skyvern/forge/sdk/copilot/tools/completion.py b/skyvern/forge/sdk/copilot/tools/completion.pyindex 0bf40dfab..645e56406 100644--- a/skyvern/forge/sdk/copilot/tools/completion.py+++ b/skyvern/forge/sdk/copilot/tools/completion.py@@ -58,6 +58,7 @@ from skyvern.forge.sdk.copilot.output_utils import iter_failure_reasonsfrom skyvern.forge.sdk.copilot.reached_download_target import (DOWNLOAD_KIND_ATTRIBUTE,DOWNLOAD_KIND_EXTENSION,+ DOWNLOAD_KIND_OBSERVED,DOWNLOAD_KIND_REGISTERED,REGISTERED_DOWNLOAD_OUTPUT_KEYS,REGISTERED_DOWNLOAD_REQUESTED_OUTPUT_PATHS,@@ -97,7 +98,9 @@ from .blockers import (LOG = structlog.get_logger()-_TYPED_DOWNLOAD_KINDS = frozenset({DOWNLOAD_KIND_REGISTERED, DOWNLOAD_KIND_ATTRIBUTE, DOWNLOAD_KIND_EXTENSION})+_TYPED_DOWNLOAD_KINDS = frozenset(+ {DOWNLOAD_KIND_REGISTERED, DOWNLOAD_KIND_ATTRIBUTE, DOWNLOAD_KIND_EXTENSION, DOWNLOAD_KIND_OBSERVED}+)_POST_RUN_PAGE_OBSERVATION_LABEL = "post_run_page_observation"_REGISTERED_ARTIFACT_OBSERVATION_LABEL = "registered_artifact_observation"# Stamp keys the same-run gate reads; they are dropped from the graded payload so the run iddiff --git a/skyvern/forge/sdk/copilot/tools/mcp_hooks.py b/skyvern/forge/sdk/copilot/tools/mcp_hooks.pyindex 9e29080be..9802703b1 100644--- a/skyvern/forge/sdk/copilot/tools/mcp_hooks.py+++ b/skyvern/forge/sdk/copilot/tools/mcp_hooks.py@@ -61,6 +61,7 @@ from .scouting import (_consume_scout_source_url,_mark_page_inspected,_mark_pending_browser_interaction_observation,+ _maybe_attach_observed_download_target,_maybe_attach_reached_download_target,_prenav_ambiguity_for_selector,_prenav_dynamic_row_for_selector,@@ -69,6 +70,7 @@ from .scouting import (_register_scout_interaction_observation,_reset_evaluate_tracker,_resolve_scout_role_name,+ _scout_session_download_names,_selector_live_match_count,_steer_evaluate_result,account_no_progress_interaction_click,@@ -348,6 +350,7 @@ async def _click_pre_hook(ctx.pending_scout_click_selector = Nonectx.pending_scout_ambiguous = Nonectx.pending_scout_dynamic_row = None+ ctx.pending_scout_download_snapshot = Noneawait _capture_scout_source_url(ctx)deterministic_result = _strip_intent_for_code_only_selector_action(params, ctx, tool_name="click")if deterministic_result is not None:@@ -374,6 +377,7 @@ async def _click_pre_hook(await _capture_scout_ambiguity(ctx, selector)if _copilot_block_authoring_policy(ctx) == BlockAuthoringPolicy.CODE_ONLY_BROWSER:await _capture_scout_dynamic_row(ctx, selector)+ ctx.pending_scout_download_snapshot = await _scout_session_download_names(ctx)return None@@ -529,6 +533,10 @@ async def _click_post_hook(if observation_step is not None:result["observation_step"] = observation_stepresult["data"]["observation_step"] = observation_step+ if _copilot_block_authoring_policy(ctx) == BlockAuthoringPolicy.CODE_ONLY_BROWSER:+ # A download this click produced is proof the affordance works, so it outranks the+ # href-shape prediction — and is the only source that sees a command-URL download.+ await _maybe_attach_observed_download_target(ctx, result, selector=selector, url=url)if page_evidence is not None:_attach_scout_page_summary(result, page_evidence)if _copilot_block_authoring_policy(ctx) == BlockAuthoringPolicy.CODE_ONLY_BROWSER:diff --git a/skyvern/forge/sdk/copilot/tools/scouting.py b/skyvern/forge/sdk/copilot/tools/scouting.pyindex 73ba96007..a1e18a398 100644--- a/skyvern/forge/sdk/copilot/tools/scouting.py+++ b/skyvern/forge/sdk/copilot/tools/scouting.py@@ -12,6 +12,7 @@ from urllib.parse import urlparseimport structlogfrom skyvern.config import settings+from skyvern.forge import appfrom skyvern.forge.sdk.copilot.build_test_outcome import (RecordedBuildTestOutcome,bind_post_run_page_path_failure,@@ -66,11 +67,15 @@ from skyvern.forge.sdk.copilot.enforcement import (reset_no_progress_interaction_count,)from skyvern.forge.sdk.copilot.reached_download_target import (+ DOWNLOAD_KIND_OBSERVED,ReachedDownloadTarget,)from skyvern.forge.sdk.copilot.reached_download_target import (derive_from_navigation_targets as _derive_reached_download_from_nav_targets,)+from skyvern.forge.sdk.copilot.reached_download_target import (+ derive_from_observed_download,+)from skyvern.forge.sdk.copilot.reached_download_target import guidance_for as _reached_download_guidance_forfrom skyvern.forge.sdk.copilot.result_evidence import (LoadedResultCompositionEvidence,@@ -1889,6 +1894,79 @@ def _with_trajectory_anchor(ctx: AgentContext, target: ReachedDownloadTarget) ->return replace(target, trajectory_anchor=anchor)+async def _scout_session_download_names(ctx: AgentContext) -> frozenset[str] | None:+ """Filenames currently registered in the scout's browser session, or empty when unavailable.++ Read-only and failure-tolerant: this only sharpens download detection, so a storage hiccup must+ never break a scout click."""+ browser_session_id = ctx.browser_session_id+ organization_id = ctx.organization_id+ if not browser_session_id or not organization_id:+ return None+ try:+ files = await app.STORAGE.list_downloaded_files_in_browser_session(+ organization_id=organization_id, browser_session_id=browser_session_id+ )+ except Exception:+ LOG.warning("copilot_scout_download_snapshot_failed", exc_info=True)+ return None+ return frozenset(str(name) for name in files or ())+++async def _maybe_attach_observed_download_target(+ ctx: AgentContext,+ result: dict[str, Any],+ *,+ selector: str,+ url: str,+) -> None:+ """S3: pin the clicked affordance when the scout's own click produced a new download.++ href shape cannot see a command-style download URL, so without this the model authors the+ download step freehand instead of receiving the synthesizer's uncaught terminal."""+ data = result.get("data")+ if not isinstance(data, dict) or not selector:+ return+ before = ctx.pending_scout_download_snapshot+ ctx.pending_scout_download_snapshot = None+ if before is None:+ return+ try:+ after = await _scout_session_download_names(ctx)+ if after is None or not (after - before):+ return+ target = derive_from_observed_download(selector=selector, affordance_text=_scout_click_text(result))+ if target is None:+ return+ data["reached_download_target"] = target.to_dict()+ data["reached_download_guidance"] = _reached_download_guidance_for(target)+ ctx.reached_download_target = _with_trajectory_anchor(ctx, target)+ if ctx.synthesized_block_offered and not ctx.update_workflow_called:+ ctx.synthesized_block_offered = False+ ctx.synthesized_block_offered_goal_complete = False+ _register_reached_download_scout_interaction(ctx, target, url=url)+ LOG.info(+ "copilot_reached_download_target_steer",+ url=url,+ download_kind=target.download_kind,+ already_registered=target.already_registered,+ )+ except Exception:+ LOG.warning("copilot_observed_download_target_attach_failed", exc_info=True)+++def _scout_click_text(result: dict[str, Any]) -> str:+ data = result.get("data")+ if not isinstance(data, dict):+ return ""+ text = data.get("text") or data.get("accessible_name") or ""+ return text if isinstance(text, str) else ""+++def _is_observed_download_target(target: object) -> bool:+ return getattr(target, "download_kind", None) == DOWNLOAD_KIND_OBSERVED++async def _maybe_attach_reached_download_target(ctx: AgentContext,result: dict[str, Any],@@ -1916,6 +1994,11 @@ async def _maybe_attach_reached_download_target(target = _derive_reached_download_from_nav_targets(parsed.get("navigation_targets"))if target is None:return+ if _is_observed_download_target(ctx.reached_download_target):+ # A download that actually fired outranks a prediction from link shape; without this the+ # later writer would repoint the target at an affordance nothing has exercised.+ LOG.info("copilot_reached_download_target_prediction_yielded_to_observed", url=url)+ returndata["reached_download_target"] = target.to_dict()data["reached_download_guidance"] = _reached_download_guidance_for(target)if not target.already_registered:diff --git a/skyvern/forge/sdk/db/repositories/artifacts.py b/skyvern/forge/sdk/db/repositories/artifacts.pyindex 56eade5f7..5559a787a 100644--- a/skyvern/forge/sdk/db/repositories/artifacts.py+++ b/skyvern/forge/sdk/db/repositories/artifacts.py@@ -4,7 +4,7 @@ import datetimefrom typing import TYPE_CHECKING, Callableimport structlog-from sqlalchemy import and_, delete, or_, select, update+from sqlalchemy import and_, delete, func, or_, select, updatefrom sqlalchemy.exc import SQLAlchemyErrorfrom skyvern.forge.sdk.artifact.models import Artifact, ArtifactType@@ -544,6 +544,30 @@ class ArtifactsRepository(BaseRepository):await session.commit()return result.rowcount or 0+ @db_operation("count_unclaimed_session_download_artifacts")+ async def count_unclaimed_session_download_artifacts(+ self,+ browser_session_id: str,+ organization_id: str,+ run_started_at: datetime.datetime,+ ) -> int:+ """Session-scoped DOWNLOAD artifacts this run produced but finalization has not claimed yet.++ Read-only counterpart of :meth:`claim_session_download_artifacts_for_run`, matched on the+ same filter so a grader reading before the claim sees this run's files and not a reused+ session's earlier ones."""+ async with self.Session() as session:+ result = await session.execute(+ select(func.count())+ .select_from(ArtifactModel)+ .where(ArtifactModel.browser_session_id == browser_session_id)+ .where(ArtifactModel.organization_id == organization_id)+ .where(ArtifactModel.artifact_type == ArtifactType.DOWNLOAD)+ .where(ArtifactModel.run_id.is_(None))+ .where(ArtifactModel.created_at >= run_started_at)+ )+ return int(result.scalar() or 0)+@db_operation("delete_artifact_for_browser_session")async def delete_artifact_for_browser_session(self,diff --git a/skyvern/forge/sdk/db/repositories/workflows.py b/skyvern/forge/sdk/db/repositories/workflows.pyindex a8af273cb..8e00565fa 100644--- a/skyvern/forge/sdk/db/repositories/workflows.py+++ b/skyvern/forge/sdk/db/repositories/workflows.py@@ -43,6 +43,7 @@ from skyvern.forge.sdk.workflow.browser_action_policy_enrollment import (from skyvern.forge.sdk.workflow.models.block import Block, ForLoopBlock, WhileLoopBlockfrom skyvern.forge.sdk.workflow.models.parameter import OutputParameterfrom skyvern.forge.sdk.workflow.models.workflow import Workflow, WorkflowDefinition+from skyvern.forge.sdk.workflow.runtime_completion import carried_contract, with_contractfrom skyvern.schemas.runs import ProxyLocationInputfrom skyvern.schemas.workflows import WorkflowStatus@@ -1165,8 +1166,11 @@ class WorkflowsRepository(BaseRepository):# not depend on caller-side object identity between the# top-level parameters list and each block's field._align_block_output_parameters(workflow_definition)+ definition_json = with_contract(+ workflow_definition.model_dump(mode="json"), carried_contract(workflow.workflow_definition)+ )workflow.workflow_definition = with_policy(- workflow_definition.model_dump(mode="json"), carried_policy(workflow.workflow_definition)+ definition_json, carried_policy(workflow.workflow_definition))if version is not None:workflow.version = versiondiff --git a/skyvern/forge/sdk/routes/workflow_copilot.py b/skyvern/forge/sdk/routes/workflow_copilot.pyindex e3b7a2af1..4b97c9add 100644--- a/skyvern/forge/sdk/routes/workflow_copilot.py+++ b/skyvern/forge/sdk/routes/workflow_copilot.py@@ -108,6 +108,7 @@ from skyvern.forge.sdk.services import org_auth_servicefrom skyvern.forge.sdk.workflow.exceptions import BaseWorkflowHTTPExceptionfrom skyvern.forge.sdk.workflow.models.parameter import ParameterTypefrom skyvern.forge.sdk.workflow.models.workflow import Workflow+from skyvern.forge.sdk.workflow.runtime_completion import contract_from_request_criteriafrom skyvern.forge.sdk.workflow.workflow_definition_converter import convert_workflow_definitionfrom skyvern.schemas.workflows import (WorkflowCreateYAMLRequest,@@ -1086,6 +1087,9 @@ async def _finalise_normal_turn(restore_failed = Trueif _should_commit_staged_workflow(chat.auto_accept, agent_result):+ # Both acceptance paths must carry the same contract, or the obligation depends on which+ # button the user pressed.+ await _attach_requested_completion_contract(chat, agent_result.staged_workflow, agent_result)try:await _commit_staged_workflow(organization_id=organization_id,@@ -2891,6 +2895,44 @@ async def workflow_copilot_clear_proposed_workflow(raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Chat not found")+async def _turn_completion_criteria(chat: Any, agent_result: Any) -> object | None:+ """This turn's typed criteria, preferring the in-memory turn state over the stored snapshot.++ Auto-accept commits before the turn's criteria are persisted, so a turn that first mints a+ download criterion would otherwise read a pre-turn snapshot and commit with no contract."""+ turn_state = getattr(agent_result, "completion_criteria_turn_state", None)+ decision = getattr(turn_state, "decision", None)+ criteria = getattr(decision, "criteria", None)+ if criteria:+ return criteria+ snapshot = await _load_completion_criteria_snapshot(chat)+ active = snapshot.active if snapshot is not None else None+ return active.criteria if active is not None else None+++async def _attach_requested_completion_contract(chat: Any, target: Any, agent_result: Any = None) -> None:+ """Carry the request's typed completion criteria onto the version being accepted.++ Read at accept time so the contract binds to the definition being accepted, and skipped silently+ when the chat has no active criteria — a workflow that promised nothing is graded as before.+ Applied on both acceptance paths so the obligation cannot depend on which one ran."""+ try:+ definition = getattr(target, "workflow_definition", None)+ if definition is None:+ return+ contract = contract_from_request_criteria(await _turn_completion_criteria(chat, agent_result))+ if contract is None:+ return+ definition.completion_contract = contract+ LOG.info(+ "copilot_completion_contract_attached_from_request",+ workflow_copilot_chat_id=chat.workflow_copilot_chat_id,+ workflow_permanent_id=chat.workflow_permanent_id,+ )+ except Exception:+ LOG.warning("copilot_completion_contract_attach_failed", exc_info=True)++@base_router.post("/workflow/copilot/apply-proposed-workflow", include_in_schema=False)async def workflow_copilot_apply_proposed_workflow(apply_request: WorkflowCopilotApplyProposedWorkflowRequest,@@ -2925,6 +2967,8 @@ async def workflow_copilot_apply_proposed_workflow(detail=f"Proposed copilot YAML is invalid: {e}",)+ await _attach_requested_completion_contract(chat, yaml_request)+current_workflow = await app.WORKFLOW_SERVICE.get_workflow_by_permanent_id(workflow_permanent_id=chat.workflow_permanent_id,organization_id=organization.organization_id,diff --git a/skyvern/forge/sdk/workflow/models/block.py b/skyvern/forge/sdk/workflow/models/block.pyindex e804d9d78..9fd06555a 100644--- a/skyvern/forge/sdk/workflow/models/block.py+++ b/skyvern/forge/sdk/workflow/models/block.py@@ -70,6 +70,7 @@ from skyvern.exceptions import (MissingStarterUrl,NoTOTPVerificationCodeFound,PDFParsingError,+ ScriptTerminationException,SkyvernException,SyncTriggeredSequentialCredentialUnsupported,TaskNotFound,@@ -104,6 +105,10 @@ from skyvern.forge.sdk.api.llm.exceptions import (from skyvern.forge.sdk.api.llm.schema_validator import validate_schemafrom skyvern.forge.sdk.artifact.models import ArtifactTypefrom skyvern.forge.sdk.copilot.block_goal_wrapping import compose_mini_goal+from skyvern.forge.sdk.copilot.reached_download_target import (+ block_output_has_registered_download,+ code_is_download_intent,+)from skyvern.forge.sdk.copilot.runtime import _browser_context_is_attachablefrom skyvern.forge.sdk.copilot.self_heal_recovery import SelfHealRecoveryResult, run_self_heal_recoveryfrom skyvern.forge.sdk.copilot.turn_origin import HealAdoptionFailed, TurnOrigin@@ -4500,21 +4505,48 @@ async def wrapper({default_args}):)return compose_mini_goal(main_goal=safe_main, mini_goal=safe_mini)- async def _self_heal_enabled(self, workflow_run_context: WorkflowRunContext) -> bool:- # User-facing per-workflow setting, restricted to copilot-authored workflows —- # pre-copilot code blocks must never gain agentic recovery from the toggle alone.- # The env default stays as the OSS/standalone and local-dev override.- if settings.ENABLE_CODE_BLOCK_SELF_HEALING:- return True+ async def _record_unregistered_download_intent(+ self,+ *,+ engine: str,+ registered: bool,+ output: object,+ workflow_run_context: WorkflowRunContext,+ workflow_run_id: str,+ workflow_run_block_id: str,+ organization_id: str | None,+ ) -> None:+ """Record a block that authors a browser download but registered no file.++ Telemetry only, and deliberately not outcome authority: the AST shape is evadable and+ authorship is not a promise. The outcome belongs to the workflow's persisted completion+ contract, graded at run finalization; this record's deletion condition is that coverage.+ Emitted from every execution engine so the prevalence it measures is not engine-biased.+ """+ if registered or not code_is_download_intent(self.code):+ return+ LOG.warning(+ "codeblock.download_intent_unregistered",+ engine=engine,+ copilot_authored=await self._workflow_is_copilot_authored(workflow_run_context),+ workflow_run_id=workflow_run_id,+ workflow_run_block_id=workflow_run_block_id,+ block_label=self.label,+ code_sha256=hashlib.sha256(self.code.encode()).hexdigest(),+ result_keys=sorted(output.keys()) if isinstance(output, dict) else None,+ organization_id=organization_id,+ )++ async def _workflow_is_copilot_authored(self, workflow_run_context: WorkflowRunContext) -> bool:workflow = workflow_run_context.workflow- if workflow is None or not workflow.enable_self_healing:+ if workflow is None:return Falseif "copilot" in (workflow.created_by, workflow.edited_by):return True# User saves re-stamp both fields with the user id, so the current version alone is# not durable; fall back to lineage (copilot stamps every version it writes and- # back-stamps v1 on copilot-born workflows). This runs inside the block's exception- # handler — a lookup failure must fail closed, never mask the original block failure.+ # back-stamps v1 on copilot-born workflows). A lookup failure must fail closed —+ # callers gate copilot-only behavior on this, and it must never mask a block failure.try:return await app.DATABASE.workflows.is_workflow_copilot_authored(workflow_permanent_id=workflow.workflow_permanent_id,@@ -4522,12 +4554,23 @@ async def wrapper({default_args}):)except Exception:LOG.warning(- "Self-heal copilot-lineage lookup failed; failing closed (no heal)",+ "Copilot-lineage lookup failed; failing closed",workflow_permanent_id=workflow.workflow_permanent_id,exc_info=True,)return False+ async def _self_heal_enabled(self, workflow_run_context: WorkflowRunContext) -> bool:+ # User-facing per-workflow setting, restricted to copilot-authored workflows —+ # pre-copilot code blocks must never gain agentic recovery from the toggle alone.+ # The env default stays as the OSS/standalone and local-dev override.+ if settings.ENABLE_CODE_BLOCK_SELF_HEALING:+ return True+ workflow = workflow_run_context.workflow+ if workflow is None or not workflow.enable_self_healing:+ return False+ return await self._workflow_is_copilot_authored(workflow_run_context)+def _is_healable_page_failure(self,exception: Exception,@@ -4841,9 +4884,9 @@ async def wrapper({default_args}):or failure.healability_hint is not None)if not has_structured:- # No healability_hint to disambiguate, so only the page-class code heals blind;+ # No healability_hint to disambiguate, so only the page-class codes heal blind;# user_code_error is ambiguous (could be a bare `raise`) and stays fail-closed.- if failure.error_code == "unsupported_page_operation":+ if failure.error_code in {"unsupported_page_operation", "browser_operation_failed"}:return HealClassification(healable=True, skip_reason=None)return HealClassification(healable=False,@@ -5542,10 +5585,22 @@ async def wrapper({default_args}):await recorder.finalize(success=False)return secure_code_block_result.block_resultawait recorder.finalize(success=True)+ secure_output = secure_code_block_result.block_result.output_parameter_value+ await self._record_unregistered_download_intent(+ engine="secure_runner",+ # The sidecar registers into the block output, so that payload is the+ # registration evidence here — the inline directory diff never runs.+ registered=block_output_has_registered_download(secure_output),+ output=secure_output,+ workflow_run_context=workflow_run_context,+ workflow_run_id=workflow_run_id,+ workflow_run_block_id=workflow_run_block_id,+ organization_id=organization_id,+ )await self.record_output_parameter_value(workflow_run_context,workflow_run_id,- secure_code_block_result.block_result.output_parameter_value,+ secure_output,)return secure_code_block_result.block_resultLOG.warning(@@ -5594,6 +5649,19 @@ async def wrapper({default_args}):organization_id=organization_id,)except Exception as e:+ # Exact type, not isinstance: the IllegitCompleteScriptTermination subclass means the+ # complete-verifier rejected the block, which is a failure to heal, not an intentional stop.+ if type(e) is ScriptTerminationException:+ await recorder.persist(recorder.recorded_actions())+ await recorder.finalize(success=False)+ return await self.build_block_result(+ success=False,+ failure_reason=workflow_run_context.mask_secrets_in_data(str(e)),+ output_parameter_value=None,+ status=BlockStatus.terminated,+ workflow_run_block_id=workflow_run_block_id,+ organization_id=organization_id,+ )exc = CustomizedCodeException(e)failing_line = user_code_line_from_exception(e)# User code can raise an exception carrying a resolved secret (e.g.@@ -5692,6 +5760,16 @@ async def wrapper({default_args}):result["downloaded_file_urls"] = [fi.url for fi in downloaded_files]result["downloaded_file_artifact_ids"] = [fi.artifact_id for fi in downloaded_files if fi.artifact_id]+ await self._record_unregistered_download_intent(+ engine="inline",+ registered=bool(downloaded_files),+ output=result,+ workflow_run_context=workflow_run_context,+ workflow_run_id=workflow_run_id,+ workflow_run_block_id=workflow_run_block_id,+ organization_id=organization_id,+ )+await self.record_output_parameter_value(workflow_run_context, workflow_run_id, result)return await self.build_block_result(success=True,diff --git a/skyvern/forge/sdk/workflow/models/workflow.py b/skyvern/forge/sdk/workflow/models/workflow.pyindex b8a9ac3a4..7038549f6 100644--- a/skyvern/forge/sdk/workflow/models/workflow.py+++ b/skyvern/forge/sdk/workflow/models/workflow.py@@ -116,6 +116,10 @@ class WorkflowDefinition(BaseModel):finally_block_label: str | None = Noneerror_code_mapping: dict[str, str] | None = Noneworkflow_system_prompt: str | None = None+ completion_contract: dict[str, Any] | None = Field(+ default=None,+ description="Copilot-managed: what a run of this workflow must produce, graded at run finalization. Derived from the request when a workflow is accepted; not intended to be authored by hand.",+ )def validate(self) -> None:all_labels: set[str] = set()diff --git a/skyvern/forge/sdk/workflow/runtime_completion.py b/skyvern/forge/sdk/workflow/runtime_completion.pynew file mode 100644index 000000000..47b8ffa1f--- /dev/null+++ b/skyvern/forge/sdk/workflow/runtime_completion.py@@ -0,0 +1,148 @@+"""Runtime grading of a workflow's declared completion contract.++A workflow version can carry a small typed statement of what a run must produce. Grading it at+finalization — after files register, before the status write — is what lets a run that produced+nothing report that, no matter which execution engine ran the blocks or how the block's own code+chose to describe its outcome.++Contract-less workflows are untouched: the grader has nothing to say about a run whose workflow+never declared an outcome.+"""++from __future__ import annotations++from dataclasses import dataclass++import structlog++LOG = structlog.get_logger()++# The only criterion kind graded today. Its evidence is the execution layer's own file registration,+# which no generated code can fake or swallow.+CRITERION_REGISTERED_DOWNLOAD = "registered_download"++_CONTRACT_KEY = "completion_contract"+_SUPPORTED_KINDS = frozenset({CRITERION_REGISTERED_DOWNLOAD})+# The field is public on the workflow schema, so a hand-written contract is bounded here.+_MAX_CRITERIA = 16+_MAX_ID_CHARS = 128+++@dataclass(frozen=True)+class CompletionCriterion:+ id: str+ kind: str+ min_count: int = 1+++@dataclass(frozen=True)+class ContractVerdict:+ satisfied: bool+ unmet_criterion_ids: tuple[str, ...]+ reason: str | None+++def parse_completion_contract(workflow_definition: object) -> tuple[CompletionCriterion, ...]:+ """Read the typed criteria a workflow version declares, ignoring anything unrecognized.++ Unknown kinds are dropped rather than failing the parse: an older worker must keep running a+ workflow authored by a newer one, and a criterion it cannot grade is not a criterion it may+ treat as unmet."""+ raw = (+ workflow_definition.get(_CONTRACT_KEY)+ if isinstance(workflow_definition, dict)+ else getattr(workflow_definition, _CONTRACT_KEY, None)+ )+ if not isinstance(raw, dict):+ return ()+ criteria = raw.get("criteria")+ if not isinstance(criteria, list):+ return ()+ parsed: list[CompletionCriterion] = []+ for item in criteria[:_MAX_CRITERIA]:+ if not isinstance(item, dict):+ continue+ kind = str(item.get("kind") or "").strip()+ if kind not in _SUPPORTED_KINDS:+ continue+ identifier = (str(item.get("id") or kind).strip() or kind)[:_MAX_ID_CHARS]+ try:+ min_count = int(item.get("min_count", 1))+ except (TypeError, ValueError):+ min_count = 1+ parsed.append(CompletionCriterion(id=identifier, kind=kind, min_count=max(1, min_count)))+ return tuple(parsed)+++def grade_completion_contract(+ criteria: tuple[CompletionCriterion, ...],+ *,+ registered_download_count: int,+) -> ContractVerdict:+ """Grade declared criteria against execution-layer evidence."""+ unmet: list[str] = []+ for criterion in criteria:+ if criterion.kind == CRITERION_REGISTERED_DOWNLOAD and registered_download_count < criterion.min_count:+ unmet.append(criterion.id)+ if not unmet:+ return ContractVerdict(satisfied=True, unmet_criterion_ids=(), reason=None)+ return ContractVerdict(+ satisfied=False,+ unmet_criterion_ids=tuple(unmet),+ reason="The workflow did not produce the file it is declared to download.",+ )+++def contract_from_request_criteria(criteria: object) -> dict[str, object] | None:+ """Project the request's own typed completion criteria into a workflow-carried contract.++ The obligation comes from what the user asked for, never from the shape of the code the product+ generated: a block that merely reports a download-shaped result promises nothing, and a request+ that asked for a file promises one however the block is written."""+ if not isinstance(criteria, (list, tuple)):+ return None+ if not any(_criterion_requests_a_download(criterion) for criterion in criteria):+ return None+ return {+ "schema_version": 1,+ "criteria": [{"id": "requested_download", "kind": CRITERION_REGISTERED_DOWNLOAD, "min_count": 1}],+ }+++def carried_contract(existing_definition: object) -> dict[str, object] | None:+ """The contract already stored on a workflow version, if any."""+ raw = (+ existing_definition.get(_CONTRACT_KEY)+ if isinstance(existing_definition, dict)+ else getattr(existing_definition, _CONTRACT_KEY, None)+ )+ return raw if isinstance(raw, dict) else None+++def with_contract(definition: dict, carried: dict[str, object] | None) -> dict:+ """Preserve a stored contract across a write that did not carry one.++ Every non-copilot save path rebuilds the definition through models that do not know this field,+ so without this a builder edit anywhere in the workflow would silently drop the obligation."""+ if carried is None or definition.get(_CONTRACT_KEY) is not None:+ return definition+ definition[_CONTRACT_KEY] = carried+ return definition+++def _criterion_requests_a_download(criterion: object) -> bool:+ """Whether a request criterion asks for a registered download.++ The request classifier types this as ``deliverable_kind``/``output_path``; the synthetic id is+ the copilot's own internal marker for the same obligation. Both are request-derived."""+ from skyvern.forge.sdk.copilot.reached_download_target import REGISTERED_DOWNLOAD_REQUESTED_OUTPUT_PATHS+ from skyvern.forge.sdk.copilot.request_policy import REGISTERED_DOWNLOAD_COMPLETION_CRITERION_ID++ for attr in ("deliverable_kind", "declared_deliverable_kind"):+ if str(getattr(criterion, attr, "") or "").strip() == CRITERION_REGISTERED_DOWNLOAD:+ return True+ if str(getattr(criterion, "output_path", "") or "").strip() in REGISTERED_DOWNLOAD_REQUESTED_OUTPUT_PATHS:+ return True+ # The copilot's own synthetic marker for the same obligation, compared by id so this stays+ # readable from an untyped criterion.+ return str(getattr(criterion, "id", "") or "") == REGISTERED_DOWNLOAD_COMPLETION_CRITERION_IDdiff --git a/skyvern/forge/sdk/workflow/service.py b/skyvern/forge/sdk/workflow/service.pyindex b4acddc8d..e35477ac6 100644--- a/skyvern/forge/sdk/workflow/service.py+++ b/skyvern/forge/sdk/workflow/service.py@@ -61,7 +61,7 @@ from skyvern.exceptions import (from skyvern.forge import appfrom skyvern.forge.failure_classifier import classify_from_failure_reasonfrom skyvern.forge.prompts import prompt_engine-from skyvern.forge.sdk.api.files import is_temp_working_dir+from skyvern.forge.sdk.api.files import is_temp_working_dir, resolve_run_download_idfrom skyvern.forge.sdk.artifact.models import Artifact, ArtifactTypefrom skyvern.forge.sdk.artifact.storage.base import _file_infos_from_download_artifactsfrom skyvern.forge.sdk.browser_action_policy import BrowserActionPolicy@@ -156,6 +156,11 @@ from skyvern.forge.sdk.workflow.models.workflow import (WorkflowRunStatus,is_adaptive_caching,)+from skyvern.forge.sdk.workflow.runtime_completion import (+ ContractVerdict,+ grade_completion_contract,+ parse_completion_contract,+)from skyvern.forge.sdk.workflow.secret_encryption import encrypt_workflow_definition_secretsfrom skyvern.forge.sdk.workflow.status_mapping import (BLOCK_STATUS_MAP,@@ -530,6 +535,8 @@ def _get_workflow_definition_core_data(workflow_definition: WorkflowDefinition)"credential_parameter_id","onepassword_credential_parameter_id","azure_vault_credential_parameter_id",+ # Graded at finalization, not executed: its presence must not invalidate cached scripts.+ "completion_contract","disable_cache","next_block_label","version",@@ -3712,6 +3719,7 @@ class WorkflowService:self._finalize_workflow_run_status(workflow_run_id=workflow_run_id,workflow_run=workflow_run,+ is_partial_run=bool(block_labels),pre_finally_status=pre_finally_status,pre_finally_failure_reason=pre_finally_failure_reason,pre_finally_failure_category=pre_finally_failure_category,@@ -7501,6 +7509,7 @@ class WorkflowService:pre_finally_status: WorkflowRunStatus,pre_finally_failure_reason: str | None,pre_finally_failure_category: list[dict] | None = None,+ is_partial_run: bool = False,) -> WorkflowRun:"""Set final workflow run status based on pre-finally state.@@ -7517,6 +7526,16 @@ class WorkflowService:WorkflowRunStatus.terminated,WorkflowRunStatus.timed_out,):+ contract_verdict = await self._grade_completion_contract(workflow_run, is_partial_run=is_partial_run)+ if contract_verdict is not None and not contract_verdict.satisfied:+ updated = await self._update_workflow_run_status_if_not_final(+ workflow_run_id=workflow_run_id,+ status=WorkflowRunStatus.terminated,+ failure_reason=contract_verdict.reason,+ )+ if updated is None:+ return await self._current_row_after_lost_finalize(workflow_run_id, workflow_run)+ return updatedupdated = await self._update_workflow_run_status_if_not_final(workflow_run_id=workflow_run_id,status=WorkflowRunStatus.completed,@@ -7541,6 +7560,81 @@ class WorkflowService:return workflow_run+ async def _session_download_count(self, workflow_run: WorkflowRun, context: object) -> int:+ """This run's downloads still scoped to the browser session, which cleanup tags with the run+ id after this grade runs. Failure-tolerant: an unavailable count must not manufacture a+ shortfall."""+ browser_session_id = getattr(context, "browser_session_id", None)+ if not browser_session_id:+ return 0+ try:+ # Scoped to this run's window and to rows finalization has not claimed: a reused session+ # carries earlier runs' downloads, which would otherwise satisfy the contract for a run+ # that produced nothing.+ return await app.DATABASE.artifacts.count_unclaimed_session_download_artifacts(+ browser_session_id=browser_session_id,+ organization_id=workflow_run.organization_id,+ run_started_at=workflow_run.created_at,+ )+ except Exception:+ LOG.warning(+ "Session download count failed while grading; counting zero from this source",+ workflow_run_id=workflow_run.workflow_run_id,+ exc_info=True,+ )+ return 0++ async def _grade_completion_contract(+ self, workflow_run: WorkflowRun, *, is_partial_run: bool = False+ ) -> ContractVerdict | None:+ """Grade the workflow's declared completion contract, or None when it declares nothing.++ Runs before the status write and after blocks have registered their files, so the verdict+ reflects what the run produced rather than what its code reported. Failure-tolerant: a+ grading error must never turn a good run into a bad one."""+ if is_partial_run:+ # A subset run was never asked to produce the whole workflow's deliverable.+ return None+ try:+ # The run is pinned to one workflow version; grading the permanent id would read+ # whatever version exists now, so an edit mid-run could judge this run by a contract+ # it never executed.+ workflow = await self.get_workflow(+ workflow_id=workflow_run.workflow_id,+ organization_id=workflow_run.organization_id,+ )+ criteria = parse_completion_contract(workflow.workflow_definition if workflow else None)+ if not criteria:+ return None+ context = skyvern_context.current()+ # Producer and consumer must resolve the same download key: a sub-workflow registers+ # files under the parent's run id, so the raw workflow_run_id would read an empty dir.+ files = await app.STORAGE.get_downloaded_files(+ organization_id=workflow_run.organization_id,+ run_id=resolve_run_download_id(context, fallback_run_id=workflow_run.workflow_run_id),+ )+ # Session-scoped downloads are only tagged with the run id during cleanup, which runs+ # after this grade. Counting them here keeps a real download from reading as zero;+ # double-counting one file is harmless because the grader only ever needs a floor.+ session_files = await self._session_download_count(workflow_run, context)+ verdict = grade_completion_contract(criteria, registered_download_count=len(files or []) + session_files)+ LOG.info(+ "workflow_completion_contract_graded",+ workflow_run_id=workflow_run.workflow_run_id,+ satisfied=verdict.satisfied,+ unmet_criterion_ids=list(verdict.unmet_criterion_ids),+ registered_download_count=len(files or []),+ session_download_count=session_files,+ )+ return verdict+ except Exception:+ LOG.warning(+ "Completion-contract grading failed; leaving the run status unchanged",+ workflow_run_id=workflow_run.workflow_run_id,+ exc_info=True,+ )+ return None+async def _current_row_after_lost_finalize(self, workflow_run_id: str, fallback: WorkflowRun) -> WorkflowRun:LOG.info("In-band finalize lost to an out-of-band finalizer; leaving terminal status intact",diff --git a/skyvern/forge/sdk/workflow/workflow_definition_converter.py b/skyvern/forge/sdk/workflow/workflow_definition_converter.pyindex 6ac6b26ec..8e48f209e 100644--- a/skyvern/forge/sdk/workflow/workflow_definition_converter.py+++ b/skyvern/forge/sdk/workflow/workflow_definition_converter.py@@ -343,6 +343,7 @@ def convert_workflow_definition(finally_block_label=workflow_definition_yaml.finally_block_label,error_code_mapping=workflow_definition_yaml.error_code_mapping,workflow_system_prompt=workflow_definition_yaml.workflow_system_prompt,+ completion_contract=workflow_definition_yaml.completion_contract,)LOG.info(diff --git a/skyvern/schemas/workflows.py b/skyvern/schemas/workflows.pyindex 15640f428..693faf083 100644--- a/skyvern/schemas/workflows.py+++ b/skyvern/schemas/workflows.py@@ -1406,6 +1406,10 @@ class WorkflowDefinitionYAML(BaseModel):finally_block_label: str | None = Noneerror_code_mapping: dict[str, str] | None = Noneworkflow_system_prompt: str | None = None+ completion_contract: dict[str, Any] | None = Field(+ default=None,+ description="Copilot-managed: what a run of this workflow must produce, graded at run finalization. Derived from the request when a workflow is accepted; not intended to be authored by hand.",+ )@model_validator(mode="after")def validate_unique_block_labels(self) -> "WorkflowDefinitionYAML":diff --git a/tests/unit/test_code_block_downloads.py b/tests/unit/test_code_block_downloads.pyindex 3c66b64dd..ab2718bc4 100644--- a/tests/unit/test_code_block_downloads.py+++ b/tests/unit/test_code_block_downloads.py@@ -15,7 +15,10 @@ from types import SimpleNamespacefrom unittest.mock import AsyncMockimport pytest+from structlog.testing import capture_logs+from skyvern.exceptions import IllegitCompleteScriptTermination, ScriptTerminationException+from skyvern.forge.sdk.copilot.reached_download_target import code_is_download_intentfrom skyvern.forge.sdk.core import skyvern_contextfrom skyvern.forge.sdk.core.skyvern_context import SkyvernContextfrom skyvern.forge.sdk.schemas.files import FileInfo@@ -27,6 +30,7 @@ from skyvern.forge.sdk.workflow.models.parameter import (WorkflowParameter,WorkflowParameterType,)+from skyvern.schemas.workflows import BlockResult, BlockStatusfrom skyvern.webeye.browser_artifacts import BrowserArtifacts@@ -72,10 +76,20 @@ def _isolated_download_path(tmp_path, monkeypatch: pytest.MonkeyPatch) -> str:return str(download_root)+def _copilot_workflow() -> SimpleNamespace:+ return SimpleNamespace(+ created_by="copilot",+ edited_by=None,+ workflow_permanent_id="wpid_test",+ organization_id="o_1",+ )++def _wire_block_runtime(monkeypatch: pytest.MonkeyPatch,*,values: dict[str, object] | None = None,+ workflow: SimpleNamespace | None = None,) -> None:page = SimpleNamespace()browser_state = SimpleNamespace(get_working_page=AsyncMock(return_value=page), browser_artifacts=BrowserArtifacts())@@ -83,6 +97,7 @@ def _wire_block_runtime(context = SimpleNamespace(organization_id="o_1",+ workflow=workflow,get_value=lambda key: (values or {}).get(key),mask_secrets_in_data=lambda data, mask="*****": data,)@@ -91,6 +106,22 @@ def _wire_block_runtime(monkeypatch.setattr(CodeBlock, "record_output_parameter_value", AsyncMock())+def _wire_secure_runner(monkeypatch: pytest.MonkeyPatch, *, output: dict) -> None:+ """Route execute() down the secure sidecar arm, which returns before the inline directory diff."""+ block_result = BlockResult(+ success=True,+ output_parameter=_output_parameter("code_out"),+ output_parameter_value=output,+ status=BlockStatus.completed,+ workflow_run_block_id="",+ )+ fake_app = block_module.app+ fake_app.AGENT_FUNCTION.should_use_codeblock_runner = AsyncMock(return_value=True)+ fake_app.AGENT_FUNCTION.execute_code_block_override = AsyncMock(+ return_value=SimpleNamespace(block_result=block_result, failure=None)+ )++def _fake_storage_app(monkeypatch: pytest.MonkeyPatch, *, save, get) -> None:fake_app = SimpleNamespace(STORAGE=SimpleNamespace(save_downloaded_files=save, get_downloaded_files=get),@@ -345,3 +376,259 @@ async def test_code_block_file_parameter_empty_uri_left_unchanged(assert result.success is Trueassert result.output_parameter_value["resolved"] == {"s3uri": ""}download_mock.assert_not_awaited()+++def _download_intent_events(logs: list[dict]) -> list[dict]:+ return [entry for entry in logs if entry.get("event") == "codeblock.download_intent_unregistered"]+++# The shape generated code actually takes: the download terminal is present and does raise on+# timeout, but a bare ``except`` converts that raise into a returned blocker string, so the block+# returns normally and reports success.+_SWALLOWED_DOWNLOAD_TERMINAL = """+downloaded_files = []+blocker = ""+try:+ async with page.expect_download(timeout=15000) as download_info:+ await latest.click()+ download = await download_info.value+ downloaded_files.append({"file_name": download.suggested_filename})+except Exception:+ blocker = "the link opens a delivery form rather than a downloadable file"+return {"downloaded_files": downloaded_files, "blocker": blocker}+"""+++async def test_download_intent_block_registering_nothing_is_counted(+ monkeypatch: pytest.MonkeyPatch, _isolated_download_path: str+) -> None:+ skyvern_context.set(SkyvernContext(organization_id="o_1", workflow_run_id="wr_1", run_id="wr_1"))++ _fake_storage_app(monkeypatch, save=AsyncMock(), get=AsyncMock(return_value=[]))+ _wire_block_runtime(monkeypatch)++ block = CodeBlock(+ label="download_invoice",+ code='result = {"downloaded_files": [], "blocker": "no downloadable file"}\nreturn result',+ output_parameter=_output_parameter("code_out"),+ )+ with capture_logs() as logs:+ result = await block.execute(workflow_run_id="wr_1", workflow_run_block_id="", organization_id="o_1")++ events = _download_intent_events(logs)+ assert len(events) == 1+ assert events[0]["block_label"] == "download_invoice"+ assert events[0]["copilot_authored"] is False+ # Telemetry, not outcome authority: the outcome contract is the workflow's persisted+ # completion criteria, graded at run finalization.+ assert result.success is True+++async def test_copilot_authored_download_block_registering_nothing_is_counted_with_lineage(+ monkeypatch: pytest.MonkeyPatch, _isolated_download_path: str+) -> None:+ skyvern_context.set(SkyvernContext(organization_id="o_1", workflow_run_id="wr_1", run_id="wr_1"))++ _fake_storage_app(monkeypatch, save=AsyncMock(), get=AsyncMock(return_value=[]))+ _wire_block_runtime(monkeypatch, workflow=_copilot_workflow())++ # The production shape, executed: page is a stub, so page.expect_download raises inside the+ # try, the bare except converts it into the blocker string, and the code returns normally.+ block = CodeBlock(+ label="download_invoice",+ code=_SWALLOWED_DOWNLOAD_TERMINAL,+ output_parameter=_output_parameter("code_out"),+ )+ with capture_logs() as logs:+ result = await block.execute(workflow_run_id="wr_1", workflow_run_block_id="", organization_id="o_1")++ events = _download_intent_events(logs)+ assert len(events) == 1+ assert events[0]["copilot_authored"] is True+ assert result.success is True+++async def test_copilot_authored_download_block_that_registers_a_file_succeeds(+ monkeypatch: pytest.MonkeyPatch, _isolated_download_path: str+) -> None:+ skyvern_context.set(SkyvernContext(organization_id="o_1", workflow_run_id="wr_1", run_id="wr_1"))++ file_info = FileInfo(+ url="https://api.example.com/v1/artifacts/a_dl_1/content?artifact_name=invoice.pdf",+ filename="invoice.pdf",+ checksum="deadbeef",+ artifact_id="a_dl_1",+ modified_at=datetime(2026, 6, 14, 12, 0, tzinfo=UTC),+ )+ _fake_storage_app(monkeypatch, save=AsyncMock(), get=AsyncMock(side_effect=[[], [file_info]]))+ _wire_block_runtime(monkeypatch, workflow=_copilot_workflow())++ block = CodeBlock(+ label="download_invoice",+ code='result = {"downloaded_files": []}\nreturn result',+ output_parameter=_output_parameter("code_out"),+ )+ with capture_logs() as logs:+ result = await block.execute(workflow_run_id="wr_1", workflow_run_block_id="", organization_id="o_1")++ assert _download_intent_events(logs) == []+ assert result.success is True+ assert result.output_parameter_value["downloaded_file_artifact_ids"] == ["a_dl_1"]+++async def test_download_intent_block_that_registers_a_file_is_not_counted(+ monkeypatch: pytest.MonkeyPatch, _isolated_download_path: str+) -> None:+ skyvern_context.set(SkyvernContext(organization_id="o_1", workflow_run_id="wr_1", run_id="wr_1"))++ file_info = FileInfo(+ url="https://api.example.com/v1/artifacts/a_dl_1/content?artifact_name=invoice.pdf",+ filename="invoice.pdf",+ checksum="deadbeef",+ artifact_id="a_dl_1",+ modified_at=datetime(2026, 6, 14, 12, 0, tzinfo=UTC),+ )+ _fake_storage_app(monkeypatch, save=AsyncMock(), get=AsyncMock(side_effect=[[], [file_info]]))+ _wire_block_runtime(monkeypatch)++ block = CodeBlock(+ label="download_invoice",+ code='result = {"downloaded_files": []}\nreturn result',+ output_parameter=_output_parameter("code_out"),+ )+ with capture_logs() as logs:+ result = await block.execute(workflow_run_id="wr_1", workflow_run_block_id="", organization_id="o_1")++ assert _download_intent_events(logs) == []+ assert result.success is True+ assert result.output_parameter_value["downloaded_file_artifact_ids"] == ["a_dl_1"]+++async def test_block_without_download_intent_is_not_counted(+ monkeypatch: pytest.MonkeyPatch, _isolated_download_path: str+) -> None:+ skyvern_context.set(SkyvernContext(organization_id="o_1", workflow_run_id="wr_1", run_id="wr_1"))++ _fake_storage_app(monkeypatch, save=AsyncMock(), get=AsyncMock(return_value=[]))+ _wire_block_runtime(monkeypatch)++ block = CodeBlock(+ label="extract_only",+ code='return {"blocker": "no downloadable file", "rows": []}',+ output_parameter=_output_parameter("code_out"),+ )+ with capture_logs() as logs:+ result = await block.execute(workflow_run_id="wr_1", workflow_run_block_id="", organization_id="o_1")++ assert _download_intent_events(logs) == []+ assert result.success is True+++async def test_terminate_maps_to_terminated_not_failed(+ monkeypatch: pytest.MonkeyPatch, _isolated_download_path: str+) -> None:+ """``page.terminate()`` is the sanctioned way for a block to report it cannot reach the file."""+ skyvern_context.set(SkyvernContext(organization_id="o_1", workflow_run_id="wr_1", run_id="wr_1"))++ _fake_storage_app(monkeypatch, save=AsyncMock(), get=AsyncMock(return_value=[]))+ _wire_block_runtime(monkeypatch)+ monkeypatch.setattr(+ CodeBlock,+ "execute_user_function_with_timeout",+ AsyncMock(side_effect=ScriptTerminationException("Terminate called: no downloadable file")),+ )++ block = CodeBlock(+ label="download_invoice",+ code="value = 'unused'",+ output_parameter=_output_parameter("code_out"),+ )+ result = await block.execute(workflow_run_id="wr_1", workflow_run_block_id="", organization_id="o_1")++ assert result.success is False+ assert result.status == BlockStatus.terminated+ assert "no downloadable file" in (result.failure_reason or "")+++async def test_complete_verify_rejection_still_fails(+ monkeypatch: pytest.MonkeyPatch, _isolated_download_path: str+) -> None:+ """The IllegitCompleteScriptTermination subclass must not be read as an intentional stop."""+ skyvern_context.set(SkyvernContext(organization_id="o_1", workflow_run_id="wr_1", run_id="wr_1"))++ _fake_storage_app(monkeypatch, save=AsyncMock(), get=AsyncMock(return_value=[]))+ _wire_block_runtime(monkeypatch)+ monkeypatch.setattr(+ CodeBlock,+ "execute_user_function_with_timeout",+ AsyncMock(side_effect=IllegitCompleteScriptTermination("verifier rejected")),+ )++ block = CodeBlock(+ label="download_invoice",+ code="value = 'unused'",+ output_parameter=_output_parameter("code_out"),+ )+ result = await block.execute(workflow_run_id="wr_1", workflow_run_block_id="", organization_id="o_1")++ assert result.success is False+ assert result.status != BlockStatus.terminated+++def test_swallowed_download_terminal_still_reads_as_download_intent() -> None:+ """A caught raise hides the failure from the block result but not from the AST predicate."""+ assert code_is_download_intent(_SWALLOWED_DOWNLOAD_TERMINAL) is True+++async def test_secure_runner_download_intent_is_counted_too(+ monkeypatch: pytest.MonkeyPatch, _isolated_download_path: str+) -> None:+ """Engine parity: the sidecar path returns before the inline diff, so it emits its own record."""+ skyvern_context.set(SkyvernContext(organization_id="o_1", workflow_run_id="wr_1", run_id="wr_1"))++ _fake_storage_app(monkeypatch, save=AsyncMock(), get=AsyncMock(return_value=[]))+ _wire_block_runtime(monkeypatch, workflow=_copilot_workflow())+ _wire_secure_runner(monkeypatch, output={"downloaded_files": [], "blocker": "no downloadable file"})++ block = CodeBlock(+ label="download_invoice",+ code=_SWALLOWED_DOWNLOAD_TERMINAL,+ output_parameter=_output_parameter("code_out"),+ )+ with capture_logs() as logs:+ result = await block.execute(workflow_run_id="wr_1", workflow_run_block_id="", organization_id="o_1")++ events = _download_intent_events(logs)+ assert len(events) == 1+ assert events[0]["engine"] == "secure_runner"+ assert result.success is True+++async def test_secure_runner_registered_download_is_not_counted(+ monkeypatch: pytest.MonkeyPatch, _isolated_download_path: str+) -> None:+ skyvern_context.set(SkyvernContext(organization_id="o_1", workflow_run_id="wr_1", run_id="wr_1"))++ _fake_storage_app(monkeypatch, save=AsyncMock(), get=AsyncMock(return_value=[]))+ _wire_block_runtime(monkeypatch, workflow=_copilot_workflow())+ _wire_secure_runner(monkeypatch, output={"downloaded_files": [{"filename": "invoice.pdf"}]})++ block = CodeBlock(+ label="download_invoice",+ code=_SWALLOWED_DOWNLOAD_TERMINAL,+ output_parameter=_output_parameter("code_out"),+ )+ with capture_logs() as logs:+ result = await block.execute(workflow_run_id="wr_1", workflow_run_block_id="", organization_id="o_1")++ assert _download_intent_events(logs) == []+ assert result.success is Truediff --git a/tests/unit/test_copilot_enforcement_hardening.py b/tests/unit/test_copilot_enforcement_hardening.pyindex 3fb4b999c..f5d96b5c2 100644--- a/tests/unit/test_copilot_enforcement_hardening.py+++ b/tests/unit/test_copilot_enforcement_hardening.py@@ -491,3 +491,101 @@ def test_reject_count_resets_only_on_non_repeat_without_frontier_unchanged() ->assert _code_authoring_reject_count_resets(False, True) is Falseassert _code_authoring_reject_count_resets(None, False) is Falseassert _code_authoring_reject_count_resets(True, False) is False+++# ---------------------------------------------------------------------------+# Repair obligation — a turn may not finalize a draft its own build test disproved+# ---------------------------------------------------------------------------+++class _FakeFinalResult:+ """Minimal stand-in for RunResultStreaming carrying one final model response."""++ def __init__(self, response_type: str, user_response: str) -> None:+ self._payload = json.dumps({"type": response_type, "user_response": user_response})++ @property+ def final_output(self) -> str:+ return self._payload+++def _failed_run_ctx(next_action: Any, *, observed: bool = True) -> Any:+ ctx = _fresh_context()+ ctx.test_after_update_done = True+ ctx.last_test_ok = False+ ctx.latest_diagnosis_repair_contract = _repair_contract(next_action)+ if observed:+ ctx.post_run_page_observation_after_failed_test = True+ ctx.post_run_page_observation_tool = "inspect_page_for_composition"+ ctx.post_run_page_observation_workflow_run_id = "wr_x"+ ctx.last_run_blocks_workflow_run_id = "wr_x"+ return ctx+++def test_observed_reply_cannot_finalize_while_repair_is_owed() -> None:+ """The production shape: failed run -> inspect once -> report the blocker -> turn ends."""+ from skyvern.forge.sdk.copilot.diagnosis_repair_contract import RepairNextAction++ ctx = _failed_run_ctx(RepairNextAction.REPAIR)+ result = _FakeFinalResult("REPLY", "The latest bill link opens an email-delivery form.")++ assert _check_enforcement(ctx, result) is not None+++def test_observed_reply_finalizes_once_repair_is_discharged() -> None:+ """A contract that no longer asks for repair releases the turn — the obligation is typed."""+ from skyvern.forge.sdk.copilot.diagnosis_repair_contract import RepairNextAction++ ctx = _failed_run_ctx(RepairNextAction.NO_CHANGE)+ result = _FakeFinalResult("REPLY", "Downloaded the latest invoice.")++ assert _check_enforcement(ctx, result) is None+++def test_ask_question_still_finalizes_while_repair_is_owed() -> None:+ """Needing the user is a legitimate exit; another repair round cannot supply the answer."""+ from skyvern.forge.sdk.copilot.diagnosis_repair_contract import RepairNextAction++ ctx = _failed_run_ctx(RepairNextAction.REPAIR)+ ctx.failed_test_nudge_count = 99 # counters exhausted+ result = _FakeFinalResult("ASK_QUESTION", "Which account should I use?")++ assert _check_enforcement(ctx, result) is None+++def test_exhausted_nudge_counters_do_not_release_an_open_repair_obligation() -> None:+ """Counters bound nudge repetition; they were never evidence the failure was addressed."""+ from skyvern.forge.sdk.copilot.diagnosis_repair_contract import RepairNextAction++ ctx = _failed_run_ctx(RepairNextAction.REPAIR)+ ctx.failed_test_nudge_count = 99+ result = _FakeFinalResult("REPLY", "I drafted the workflow; it attempts to download the bill.")++ assert _check_enforcement(ctx, result) == POST_FAILED_TEST_NUDGE+++def test_stop_decision_releases_the_turn_even_with_counters_exhausted() -> None:+ """Typed terminal evidence, not a counter, is what ends a repairable failure."""+ from skyvern.forge.sdk.copilot.diagnosis_repair_contract import RepairNextAction++ ctx = _failed_run_ctx(RepairNextAction.STOP)+ ctx.failed_test_nudge_count = 99+ result = _FakeFinalResult("REPLY", "This site requires a mailed statement request.")++ assert _check_enforcement(ctx, result) is None+++def test_repair_obligation_releases_the_turn_once_its_rounds_are_spent() -> None:+ """A failure that looks repairable but is not must still be reportable, not re-nudged forever."""+ from skyvern.forge.sdk.copilot.diagnosis_repair_contract import RepairNextAction+ from skyvern.forge.sdk.copilot.enforcement import MAX_REPAIR_OBLIGATION_NUDGES++ ctx = _failed_run_ctx(RepairNextAction.REPAIR)+ ctx.failed_test_nudge_count = 99+ result = _FakeFinalResult("REPLY", "This site offers no downloadable statement.")++ # Held open while rounds remain.+ assert _check_enforcement(ctx, result) == POST_FAILED_TEST_NUDGE++ ctx.repair_obligation_nudge_count = MAX_REPAIR_OBLIGATION_NUDGES+ assert _check_enforcement(ctx, result) is Nonediff --git a/tests/unit/test_copilot_hooks.py b/tests/unit/test_copilot_hooks.pyindex 55d94c41f..71689c06c 100644--- a/tests/unit/test_copilot_hooks.py+++ b/tests/unit/test_copilot_hooks.py@@ -990,6 +990,7 @@ class TestBrowserInteractionObservationHooks:pending_scout_ambiguous=None,pending_scout_reanchor=None,pending_scout_dynamic_row=None,+ pending_scout_download_snapshot=None,discovery_mcp_server=None,scouted_interactions=[],scout_trajectory=[],@@ -1027,6 +1028,7 @@ class TestBrowserInteractionObservationHooks:pending_scout_ambiguous=None,pending_scout_reanchor=None,pending_scout_dynamic_row=None,+ pending_scout_download_snapshot=None,discovery_mcp_server=None,scouted_interactions=[],scout_trajectory=[],@@ -1085,6 +1087,7 @@ class TestScoutedInteractionCapture:pending_scout_ambiguous=None,pending_scout_reanchor=None,pending_scout_dynamic_row=None,+ pending_scout_download_snapshot=None,discovery_mcp_server=None,browser_session_id=None,scouted_interactions=[],diff --git a/tests/unit/test_copilot_response_coverage_gate.py b/tests/unit/test_copilot_response_coverage_gate.pyindex 15505d001..3c6f6360f 100644--- a/tests/unit/test_copilot_response_coverage_gate.py+++ b/tests/unit/test_copilot_response_coverage_gate.py@@ -53,6 +53,7 @@ class _Ctx:self.composition_page_evidence = Noneself.last_update_block_count = Noneself.last_test_ok = None+ self.latest_diagnosis_repair_contract = Noneself.last_full_workflow_test_ok = Falseself.last_test_failure_reason = Noneself.last_test_suspicious_success = Falsediff --git a/tests/unit/test_workflow_runtime_completion.py b/tests/unit/test_workflow_runtime_completion.pynew file mode 100644index 000000000..3034219de--- /dev/null+++ b/tests/unit/test_workflow_runtime_completion.py@@ -0,0 +1,408 @@+"""Runtime grading of a workflow's declared completion contract.++The contract lives on the workflow version and is graded from execution-layer evidence, so the+verdict does not depend on which engine ran the blocks or how generated code described its outcome.+"""++from __future__ import annotations++from types import SimpleNamespace+from unittest.mock import AsyncMock++import pytest++from skyvern.forge.sdk.workflow import service as service_module+from skyvern.forge.sdk.workflow.models.workflow import WorkflowRunStatus+from skyvern.forge.sdk.workflow.runtime_completion import (+ CompletionCriterion,+ carried_contract,+ contract_from_request_criteria,+ grade_completion_contract,+ parse_completion_contract,+ with_contract,+)++_DOWNLOAD_CONTRACT = {+ "completion_contract": {+ "schema_version": 1,+ "criteria": [{"id": "must_download", "kind": "registered_download", "min_count": 1}],+ }+}+++def test_workflow_without_a_contract_declares_nothing() -> None:+ assert parse_completion_contract({}) == ()+ assert parse_completion_contract(None) == ()+ assert parse_completion_contract({"completion_contract": {"criteria": "nope"}}) == ()+++def test_download_contract_parses() -> None:+ (criterion,) = parse_completion_contract(_DOWNLOAD_CONTRACT)+ assert criterion == CompletionCriterion(id="must_download", kind="registered_download", min_count=1)+++def test_unknown_kinds_are_dropped_not_failed() -> None:+ """An older worker must keep running a newer workflow, and must never fail what it cannot grade."""+ contract = {+ "completion_contract": {+ "criteria": [+ {"id": "future", "kind": "some_future_kind"},+ {"id": "must_download", "kind": "registered_download"},+ ]+ }+ }+ parsed = parse_completion_contract(contract)+ assert [c.id for c in parsed] == ["must_download"]+++def test_a_run_that_registered_a_file_satisfies_the_contract() -> None:+ criteria = parse_completion_contract(_DOWNLOAD_CONTRACT)+ verdict = grade_completion_contract(criteria, registered_download_count=1)+ assert verdict.satisfied is True+ assert verdict.unmet_criterion_ids == ()+++def test_a_run_that_registered_nothing_is_unmet() -> None:+ """The production shape: the block returned cleanly, the run produced no file."""+ criteria = parse_completion_contract(_DOWNLOAD_CONTRACT)+ verdict = grade_completion_contract(criteria, registered_download_count=0)+ assert verdict.satisfied is False+ assert verdict.unmet_criterion_ids == ("must_download",)+ assert verdict.reason+++def test_no_criteria_grades_as_satisfied() -> None:+ """Contract-less workflows keep their existing outcome."""+ assert grade_completion_contract((), registered_download_count=0).satisfied is True+++def test_min_count_is_honored_and_floored_at_one() -> None:+ contract = {"completion_contract": {"criteria": [{"kind": "registered_download", "min_count": 2}]}}+ criteria = parse_completion_contract(contract)+ assert grade_completion_contract(criteria, registered_download_count=1).satisfied is False+ assert grade_completion_contract(criteria, registered_download_count=2).satisfied is True++ zero = parse_completion_contract(+ {"completion_contract": {"criteria": [{"kind": "registered_download", "min_count": 0}]}}+ )+ assert zero[0].min_count == 1+++_DERIVED_CONTRACT = {+ "schema_version": 1,+ "criteria": [{"id": "declared_download", "kind": "registered_download", "min_count": 1}],+}+++def _wire_finalize(monkeypatch, *, contract, downloaded):+ """A WorkflowService with just enough wired to exercise the finalize status decision."""+ from skyvern.forge.sdk.workflow.service import WorkflowService++ service = WorkflowService()+ run = SimpleNamespace(+ workflow_run_id="wr_1",+ workflow_id="w_pinned",+ workflow_permanent_id="wpid_1",+ organization_id="o_1",+ status=WorkflowRunStatus.running,+ )+ statuses: list[WorkflowRunStatus] = []++ async def _update(workflow_run_id, status, **kwargs):+ statuses.append(status)+ return run++ async def _get_workflow(workflow_id, organization_id=None):+ definition = dict(_DEFINITION_BASE)+ if contract is not None:+ definition["completion_contract"] = contract+ return SimpleNamespace(workflow_definition=definition)++ monkeypatch.setattr(service, "_update_workflow_run_status_if_not_final", _update)+ monkeypatch.setattr(service, "get_workflow", _get_workflow)+ monkeypatch.setattr(+ service_module.app,+ "STORAGE",+ SimpleNamespace(get_downloaded_files=AsyncMock(return_value=list(downloaded))),+ )+ return service, run, statuses+++_DEFINITION_BASE: dict = {"version": 2, "parameters": [], "blocks": []}+++_DOWNLOAD_CODE = (+ 'async with page.expect_download() as dl:\n await page.locator("a").click()\nreturn {"downloaded_files": []}'+)+++_DOWNLOAD_YAML = """title: Harbor bill+workflow_definition:+ version: 2+ parameters: []+ blocks:+ - block_type: code+ label: download_statement+ code: |+ async with page.expect_download(timeout=15000) as dl:+ await page.locator("#currentBill").click()+ download = await dl.value+ return {"downloaded_files": [{"file_name": download.suggested_filename}]}+"""++_PLAIN_YAML = """title: Plain+workflow_definition:+ version: 2+ parameters: []+ blocks:+ - block_type: code+ label: extract+ code: |+ return {"rows": []}+"""+++async def test_finalize_terminates_a_run_that_did_not_produce_its_declared_file(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ """The headline behavior: a run whose workflow declares a download and registered none must not+ finalize as completed."""+ service, run, statuses = _wire_finalize(monkeypatch, contract=_DERIVED_CONTRACT, downloaded=[])++ await service._finalize_workflow_run_status(+ workflow_run_id=run.workflow_run_id,+ workflow_run=run,+ pre_finally_status=WorkflowRunStatus.running,+ pre_finally_failure_reason=None,+ )++ assert statuses == [WorkflowRunStatus.terminated]+++async def test_finalize_completes_a_run_that_produced_its_declared_file(monkeypatch: pytest.MonkeyPatch) -> None:+ service, run, statuses = _wire_finalize(monkeypatch, contract=_DERIVED_CONTRACT, downloaded=["invoice.pdf"])++ await service._finalize_workflow_run_status(+ workflow_run_id=run.workflow_run_id,+ workflow_run=run,+ pre_finally_status=WorkflowRunStatus.running,+ pre_finally_failure_reason=None,+ )++ assert statuses == [WorkflowRunStatus.completed]+++async def test_finalize_leaves_a_contract_less_workflow_alone(monkeypatch: pytest.MonkeyPatch) -> None:+ service, run, statuses = _wire_finalize(monkeypatch, contract=None, downloaded=[])++ await service._finalize_workflow_run_status(+ workflow_run_id=run.workflow_run_id,+ workflow_run=run,+ pre_finally_status=WorkflowRunStatus.running,+ pre_finally_failure_reason=None,+ )++ assert statuses == [WorkflowRunStatus.completed]+++async def test_finalize_grades_the_version_the_run_executed(monkeypatch: pytest.MonkeyPatch) -> None:+ """An edit mid-run must not judge this run by a contract it never executed."""+ service, run, statuses = _wire_finalize(monkeypatch, contract=None, downloaded=[])+ seen: list[str] = []++ async def _get_workflow(workflow_id: str, organization_id: str | None = None):+ seen.append(workflow_id)+ return SimpleNamespace(workflow_definition={})++ monkeypatch.setattr(service, "get_workflow", _get_workflow)+ await service._finalize_workflow_run_status(+ workflow_run_id=run.workflow_run_id,+ workflow_run=run,+ pre_finally_status=WorkflowRunStatus.running,+ pre_finally_failure_reason=None,+ )++ assert seen == [run.workflow_id]+++def test_contract_comes_from_the_request_not_the_code() -> None:+ """The obligation is what the user asked for, never the shape of the generated code."""+ from skyvern.forge.sdk.copilot.completion_verification import registered_download_completion_criterion+ from skyvern.forge.sdk.copilot.request_policy import CompletionCriterion as RequestCriterion++ assert contract_from_request_criteria([registered_download_completion_criterion()]) is not None+ assert contract_from_request_criteria([RequestCriterion(id="c0", outcome="something else")]) is None+ assert contract_from_request_criteria([]) is None+ assert contract_from_request_criteria(None) is None+++def test_requested_contract_round_trips_through_the_parser() -> None:+ from skyvern.forge.sdk.copilot.completion_verification import registered_download_completion_criterion++ contract = contract_from_request_criteria([registered_download_completion_criterion()])+ criteria = parse_completion_contract({"completion_contract": contract})+ assert [c.kind for c in criteria] == ["registered_download"]+ assert grade_completion_contract(criteria, registered_download_count=0).satisfied is False+ assert grade_completion_contract(criteria, registered_download_count=1).satisfied is True+++async def test_finalize_skips_grading_a_partial_run(monkeypatch: pytest.MonkeyPatch) -> None:+ """A frontier run of a block subset was never asked to produce the whole deliverable."""+ service, run, statuses = _wire_finalize(monkeypatch, contract=_DERIVED_CONTRACT, downloaded=[])++ await service._finalize_workflow_run_status(+ workflow_run_id=run.workflow_run_id,+ workflow_run=run,+ pre_finally_status=WorkflowRunStatus.running,+ pre_finally_failure_reason=None,+ is_partial_run=True,+ )++ assert statuses == [WorkflowRunStatus.completed]+++def test_a_stored_contract_survives_a_write_that_does_not_carry_one() -> None:+ """Non-copilot save paths rebuild the definition through models that omit the field."""+ stored = {"completion_contract": _DERIVED_CONTRACT, "blocks": []}+ rebuilt = with_contract({"blocks": []}, carried_contract(stored))+ assert rebuilt["completion_contract"] == _DERIVED_CONTRACT+++def test_an_incoming_contract_is_not_overwritten_by_the_carried_one() -> None:+ incoming = {"completion_contract": {"schema_version": 1, "criteria": []}, "blocks": []}+ rebuilt = with_contract(dict(incoming), carried_contract({"completion_contract": _DERIVED_CONTRACT}))+ assert rebuilt["completion_contract"] == incoming["completion_contract"]+++def test_no_stored_contract_leaves_the_definition_untouched() -> None:+ assert "completion_contract" not in with_contract({"blocks": []}, carried_contract({"blocks": []}))+++async def test_finalize_counts_session_scoped_downloads_not_yet_claimed(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ """Session downloads are tagged with the run id during cleanup, after this grade runs."""+ service, run, statuses = _wire_finalize(monkeypatch, contract=_DERIVED_CONTRACT, downloaded=[])+ monkeypatch.setattr(service, "_session_download_count", AsyncMock(return_value=1))++ await service._finalize_workflow_run_status(+ workflow_run_id=run.workflow_run_id,+ workflow_run=run,+ pre_finally_status=WorkflowRunStatus.running,+ pre_finally_failure_reason=None,+ )++ assert statuses == [WorkflowRunStatus.completed]+++async def test_both_acceptance_paths_attach_the_same_contract(monkeypatch: pytest.MonkeyPatch) -> None:+ """The obligation must not depend on whether the user clicked Accept or had auto-accept on."""+ from skyvern.forge.sdk.copilot.completion_verification import registered_download_completion_criterion+ from skyvern.forge.sdk.routes import workflow_copilot as route++ snapshot = SimpleNamespace(active=SimpleNamespace(criteria=[registered_download_completion_criterion()]))+ monkeypatch.setattr(route, "_load_completion_criteria_snapshot", AsyncMock(return_value=snapshot))+ chat = SimpleNamespace(workflow_copilot_chat_id="wcc_1", workflow_permanent_id="wpid_1")++ for target in (+ SimpleNamespace(workflow_definition=SimpleNamespace(completion_contract=None)), # manual accept+ SimpleNamespace(workflow_definition=SimpleNamespace(completion_contract=None)), # auto accept+ ):+ await route._attach_requested_completion_contract(chat, target)+ assert target.workflow_definition.completion_contract["criteria"][0]["kind"] == "registered_download"+++async def test_no_active_criteria_attaches_nothing(monkeypatch: pytest.MonkeyPatch) -> None:+ from skyvern.forge.sdk.routes import workflow_copilot as route++ monkeypatch.setattr(+ route, "_load_completion_criteria_snapshot", AsyncMock(return_value=SimpleNamespace(active=None))+ )+ target = SimpleNamespace(workflow_definition=SimpleNamespace(completion_contract=None))+ await route._attach_requested_completion_contract(+ SimpleNamespace(workflow_copilot_chat_id="c", workflow_permanent_id="w"), target+ )+ assert target.workflow_definition.completion_contract is None+++def test_apply_proposed_workflow_route_is_bound_to_the_route_handler() -> None:+ """A helper inserted between the decorator and its function silently rebinds the endpoint to the+ helper, and the route then 422s on the helper's arguments."""+ from skyvern.forge.sdk.routes.workflow_copilot import base_router++ routes = [r for r in base_router.routes if getattr(r, "path", "") == "/workflow/copilot/apply-proposed-workflow"]+ assert routes, "route not registered"+ assert routes[0].endpoint.__name__ == "workflow_copilot_apply_proposed_workflow"+++def test_a_request_criterion_is_recognized_by_its_typed_deliverable_fields() -> None:+ """The persisted criterion carries deliverable_kind/output_path; the synthetic id is the+ copilot's separate internal marker, and keying on it alone misses every real request."""+ requested = SimpleNamespace(+ id="c0",+ outcome="the current electricity statement is downloaded as a PDF",+ deliverable_kind="registered_download",+ declared_deliverable_kind="registered_download",+ output_path="output.downloaded_files",+ )+ unrelated = SimpleNamespace(id="c1", outcome="a summary", deliverable_kind=None, output_path=None)++ assert contract_from_request_criteria([unrelated, requested]) is not None+ assert contract_from_request_criteria([unrelated]) is None+++def test_output_path_alone_identifies_a_requested_download() -> None:+ by_path = SimpleNamespace(id="c0", deliverable_kind=None, output_path="output.downloaded_files")+ assert contract_from_request_criteria([by_path]) is not None+++async def test_auto_accept_uses_this_turns_criteria_not_the_stored_snapshot(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ """Auto-accept commits before the turn's criteria are persisted, so a turn that first mints a+ download criterion must still carry the contract."""+ from skyvern.forge.sdk.copilot.completion_verification import registered_download_completion_criterion+ from skyvern.forge.sdk.routes import workflow_copilot as route++ # The stored snapshot is pre-turn and has nothing; this turn's state holds the new criterion.+ monkeypatch.setattr(+ route, "_load_completion_criteria_snapshot", AsyncMock(return_value=SimpleNamespace(active=None))+ )+ agent_result = SimpleNamespace(+ completion_criteria_turn_state=SimpleNamespace(+ decision=SimpleNamespace(criteria=(registered_download_completion_criterion(),))+ )+ )+ target = SimpleNamespace(workflow_definition=SimpleNamespace(completion_contract=None))++ await route._attach_requested_completion_contract(+ SimpleNamespace(workflow_copilot_chat_id="c", workflow_permanent_id="w"), target, agent_result+ )++ assert target.workflow_definition.completion_contract is not None+++async def test_manual_accept_still_falls_back_to_the_stored_snapshot(monkeypatch: pytest.MonkeyPatch) -> None:+ """Manual accept runs in a later request with no turn state, so the persisted set is the source."""+ from skyvern.forge.sdk.copilot.completion_verification import registered_download_completion_criterion+ from skyvern.forge.sdk.routes import workflow_copilot as route++ snapshot = SimpleNamespace(active=SimpleNamespace(criteria=[registered_download_completion_criterion()]))+ monkeypatch.setattr(route, "_load_completion_criteria_snapshot", AsyncMock(return_value=snapshot))+ target = SimpleNamespace(workflow_definition=SimpleNamespace(completion_contract=None))++ await route._attach_requested_completion_contract(+ SimpleNamespace(workflow_copilot_chat_id="c", workflow_permanent_id="w"), target+ )++ assert target.workflow_definition.completion_contract is not Nonediff --git a/tests/unit/workflow/test_code_block_escalation.py b/tests/unit/workflow/test_code_block_escalation.pyindex a0e7915e9..56fc255cb 100644--- a/tests/unit/workflow/test_code_block_escalation.py+++ b/tests/unit/workflow/test_code_block_escalation.py@@ -1274,6 +1274,7 @@ async def test_escalation_task_verifies_with_action_history(("error_code", "healable", "skip_reason"),[("unsupported_page_operation", True, None),+ ("browser_operation_failed", True, None),("timeout", False, HealSkipReason.timeout_class),("insecure_code_detected", False, HealSkipReason.insecure_code),("browser_disconnected", False, HealSkipReason.unclassifiable),