Skip to content

Automate browser based workflows with AI

Deliver the download instead of shipping a draft the build test disproved (SKY-13230) (#7914)

Andrew Neilson4bb0105c2b41
diff --git a/docs/api-reference/openapi.json b/docs/api-reference/openapi.json
index 47850cca7..2949a05e9 100644
--- a/docs/api-reference/openapi.json
+++ b/docs/api-reference/openapi.json
@@ -27487,6 +27487,19 @@
}
],
"title": "Workflow System Prompt"
+ },
+ "completion_contract": {
+ "anyOf": [
+ {
+ "additionalProperties": true,
+ "type": "object"
+ },
+ {
+ "type": "null"
+ }
+ ],
+ "title": "Completion Contract",
+ "description": "Copilot-managed: what a run of this workflow must produce, graded at run finalization. Derived from the request when a workflow is accepted; not intended to be authored by hand."
}
},
"type": "object",
@@ -27730,6 +27743,19 @@
}
],
"title": "Workflow System Prompt"
+ },
+ "completion_contract": {
+ "anyOf": [
+ {
+ "additionalProperties": true,
+ "type": "object"
+ },
+ {
+ "type": "null"
+ }
+ ],
+ "title": "Completion Contract",
+ "description": "Copilot-managed: what a run of this workflow must produce, graded at run finalization. Derived from the request when a workflow is accepted; not intended to be authored by hand."
}
},
"type": "object",
diff --git a/skyvern/forge/sdk/copilot/enforcement.py b/skyvern/forge/sdk/copilot/enforcement.py
index 60b163a90..2430ba626 100644
--- a/skyvern/forge/sdk/copilot/enforcement.py
+++ b/skyvern/forge/sdk/copilot/enforcement.py
@@ -130,6 +130,7 @@ from skyvern.forge.sdk.copilot.run_outcome import (
)
from skyvern.forge.sdk.copilot.runtime import (
AgentContext,
+ diagnosis_repair_obligation_open,
)
from skyvern.forge.sdk.copilot.screenshot_utils import ScreenshotEntry
from skyvern.forge.sdk.copilot.terminal_predicates import (
@@ -172,6 +173,9 @@ POST_INTERMEDIATE_SUCCESS_NUDGE = copilot_config_defaults.POST_INTERMEDIATE_SUCC
MAX_POST_UPDATE_NUDGES = 2
MAX_INTERMEDIATE_NUDGES = 8
MAX_FAILED_TEST_NUDGES = 2
+# Repair rounds the typed obligation may force past the ordinary nudge budget before a turn is
+# allowed to report an unrepairable failure.
+MAX_REPAIR_OBLIGATION_NUDGES = 6
MAX_FORMAT_NUDGES = 2
MAX_NO_WORKFLOW_NUDGES = 2
MAX_DISCOVERY_ENTRYPOINT_URL_QUESTION_NUDGES = 2
@@ -1155,6 +1159,29 @@ def _needs_explore_without_workflow_nudge(ctx: Any) -> bool:
return nudge_count < MAX_EXPLORE_WITHOUT_WORKFLOW_NUDGES
+def _repair_obligation_live(ctx: AgentContext) -> bool:
+ """The typed obligation, bounded. It is discharged by evidence, but must not outlive the
+ evidence that produced it: a failure that looks repairable and is not would otherwise be
+ re-nudged until the turn budget dies, burying the blocker the model is trying to report."""
+ if not diagnosis_repair_obligation_open(ctx):
+ return False
+ return _get_int(ctx, "repair_obligation_nudge_count") < MAX_REPAIR_OBLIGATION_NUDGES
+
+
+def _repair_obligation_blocks_finalize(ctx: AgentContext, result: RunResultStreaming | None) -> bool:
+ """True when the turn is about to end while the typed contract still says the failure is repairable.
+
+ The nudge counters bound *repetition of a nudge*; they were never evidence that the failure had been
+ addressed, so once they ran out a turn could finalize a draft its own build test disproved. Ending is
+ still bounded — by the total-turn timeout and max-turns, both of which exit as WIP rather than as an
+ accept-ready proposal."""
+ if not _repair_obligation_live(ctx):
+ return False
+ parsed = _parse_normalized_final_response(result)
+ # ASK_QUESTION is a legitimate exit: it needs the user, not another repair round.
+ return parsed is not None and parsed.get("type") == "REPLY"
+
+
def _needs_failed_test_nudge(ctx: Any) -> bool:
"""Return True when the last test failed and the agent hasn't iterated yet."""
# A permanent nav error cannot be 'fix the workflow and retry' material —
@@ -1345,7 +1372,9 @@ def _check_enforcement(
):
return _nudge(config, "post_update")
- if _post_run_observed_reply_can_finalize(ctx, result):
+ # Observing the reached page is the first repair move, not the last: while the typed contract
+ # still says REPAIR, a reply that reports the failure instead of acting on it re-enters the loop.
+ if not _repair_obligation_live(ctx) and _post_run_observed_reply_can_finalize(ctx, result):
return None
_maybe_stash_terminal_challenge_halt(ctx)
@@ -1397,6 +1426,12 @@ def _check_enforcement(
return _nudge(config, "post_failed_test_inspect_first")
return _nudge(config, "post_failed_test")
+ # Counters exhausted but the contract still says REPAIR: keep steering rather than finalize a
+ # draft the build test disproved.
+ if _repair_obligation_blocks_finalize(ctx, result):
+ ctx.repair_obligation_nudge_count = _get_int(ctx, "repair_obligation_nudge_count") + 1
+ return _nudge(config, "post_failed_test")
+
# Response-time gate: peek at the model's final output to tell ASK_QUESTION
# (always allowed) from a REPLY with a coverage gap or progress-narration.
# Only runs when no state-based nudge fired.
diff --git a/skyvern/forge/sdk/copilot/reached_download_target.py b/skyvern/forge/sdk/copilot/reached_download_target.py
index a7b5eb0e4..5416e44cf 100644
--- a/skyvern/forge/sdk/copilot/reached_download_target.py
+++ b/skyvern/forge/sdk/copilot/reached_download_target.py
@@ -9,13 +9,17 @@ from dataclasses import dataclass
from enum import Enum
from typing import Any, Literal, cast
-DownloadKind = Literal["registered", "attribute", "extension"]
+DownloadKind = Literal["registered", "attribute", "extension", "observed"]
-# ``registered`` is the only kind backed by an actual browser download having fired (S1);
-# ``attribute``/``extension`` are S2 predictions from a scouted link.
+# ``registered`` (S1) and ``observed`` (S3) are backed by an actual browser download having fired;
+# ``attribute``/``extension`` are S2 predictions from a scouted link's href shape.
DOWNLOAD_KIND_REGISTERED: DownloadKind = "registered"
DOWNLOAD_KIND_ATTRIBUTE: DownloadKind = "attribute"
DOWNLOAD_KIND_EXTENSION: DownloadKind = "extension"
+# A download the scout's own click produced. Href shape cannot see these: a query-parameter URL
+# serving a file via Content-Disposition has no extension and no ``download`` attribute, so the
+# href-shape prediction is blind to it and only the fired download proves the affordance.
+DOWNLOAD_KIND_OBSERVED: DownloadKind = "observed"
# S2 may only mint a prediction; ``registered`` is S1-only and must never come from a nav target.
_PREDICTED_DOWNLOAD_KINDS: frozenset[str] = frozenset({DOWNLOAD_KIND_ATTRIBUTE, DOWNLOAD_KIND_EXTENSION})
@@ -69,6 +73,7 @@ NAV_TARGET_DOWNLOAD_KIND_KEY = "download_kind"
class _SourceStepKind(str, Enum):
trajectory_recency = "trajectory_recency"
registered_output = "registered_output"
+ observed_download = "observed_download"
@dataclass(frozen=True)
@@ -150,6 +155,23 @@ def block_output_has_registered_download(block_output: Any) -> bool:
return any(bool(block_output.get(key)) for key in REGISTERED_DOWNLOAD_OUTPUT_KEYS)
+def derive_from_observed_download(*, selector: str, affordance_text: str = "") -> ReachedDownloadTarget | None:
+ """S3: mint a target from a download the scout's own click just produced.
+
+ Stronger evidence than S2 (the download fired) while still carrying the selector S1 lacks, so the
+ synthesizer can compile its terminal ``expect_download`` step against the exercised affordance."""
+ selector = _summary_str(selector)
+ if not selector:
+ return None
+ return ReachedDownloadTarget(
+ selector=selector,
+ affordance_text=_summary_str(affordance_text),
+ download_kind=DOWNLOAD_KIND_OBSERVED,
+ source_step=_SourceStepKind.observed_download.value,
+ already_registered=False,
+ )
+
+
def derive_from_block_outputs(block_outputs_by_label: Any) -> ReachedDownloadTarget | None:
"""S1: confirm a reached download from a browser download already registered into a block output.
diff --git a/skyvern/forge/sdk/copilot/runtime.py b/skyvern/forge/sdk/copilot/runtime.py
index de5fff6b2..e7389009a 100644
--- a/skyvern/forge/sdk/copilot/runtime.py
+++ b/skyvern/forge/sdk/copilot/runtime.py
@@ -528,6 +528,10 @@ class AgentContext:
# Selector of an in-flight click, captured pre-dispatch so a failed/timed-out click can gate a
# settle re-perception on whether that selector still resolves to a live element.
pending_scout_click_selector: str | None = None
+ # Browser-session download filenames snapshotted before a scout click, so the post-hook can tell
+ # a download this click produced from one an earlier click left behind.
+ pending_scout_download_snapshot: frozenset[str] | None = None
+ repair_obligation_nudge_count: int = 0
# (selector, ambiguous) verdict from a pre-dispatch live count probe, applied to the recorded
# interaction only when the post-action resolved selector matches the probed one.
pending_scout_ambiguous: tuple[str, bool] | None = None
@@ -569,6 +573,23 @@ class AgentContext:
blocker_signal_claimant: TurnClaimant | None = None
+def diagnosis_repair_obligation_open(ctx: AgentContext) -> bool:
+ """True while the latest diagnosis contract says the failed run is repairable and no later run has
+ discharged it. The build-test oracle already reported the failure; this keeps the loop on the hook for
+ consuming it, so a turn cannot observe the reached page once and then finalize a draft the run disproved.
+
+ Keyed on the typed decision, never on a nudge count: the contract's own ASK/STOP/NO_CHANGE transitions
+ discharge it, and the operational total-turn timeout bounds it."""
+ # Deferred: diagnosis_repair_contract reaches this module through context, so a module-level
+ # import cycles.
+ from skyvern.forge.sdk.copilot.diagnosis_repair_contract import RepairNextAction
+
+ contract = getattr(ctx, "latest_diagnosis_repair_contract", None)
+ if contract is None:
+ return False
+ return contract.repair_decision.next_action is RepairNextAction.REPAIR
+
+
def output_contract_ladder_unresolved(ctx: AgentContext) -> bool:
"""True while an output-contract signature has a live actuation ladder — a landed actuation or a GRANTED
advisory — that has not yet reached a typed terminal or a dispatched (CONSUMED) run. Loop and churn detectors
diff --git a/skyvern/forge/sdk/copilot/tools/completion.py b/skyvern/forge/sdk/copilot/tools/completion.py
index 0bf40dfab..645e56406 100644
--- a/skyvern/forge/sdk/copilot/tools/completion.py
+++ b/skyvern/forge/sdk/copilot/tools/completion.py
@@ -58,6 +58,7 @@ from skyvern.forge.sdk.copilot.output_utils import iter_failure_reasons
from skyvern.forge.sdk.copilot.reached_download_target import (
DOWNLOAD_KIND_ATTRIBUTE,
DOWNLOAD_KIND_EXTENSION,
+ DOWNLOAD_KIND_OBSERVED,
DOWNLOAD_KIND_REGISTERED,
REGISTERED_DOWNLOAD_OUTPUT_KEYS,
REGISTERED_DOWNLOAD_REQUESTED_OUTPUT_PATHS,
@@ -97,7 +98,9 @@ from .blockers import (
LOG = structlog.get_logger()
-_TYPED_DOWNLOAD_KINDS = frozenset({DOWNLOAD_KIND_REGISTERED, DOWNLOAD_KIND_ATTRIBUTE, DOWNLOAD_KIND_EXTENSION})
+_TYPED_DOWNLOAD_KINDS = frozenset(
+ {DOWNLOAD_KIND_REGISTERED, DOWNLOAD_KIND_ATTRIBUTE, DOWNLOAD_KIND_EXTENSION, DOWNLOAD_KIND_OBSERVED}
+)
_POST_RUN_PAGE_OBSERVATION_LABEL = "post_run_page_observation"
_REGISTERED_ARTIFACT_OBSERVATION_LABEL = "registered_artifact_observation"
# Stamp keys the same-run gate reads; they are dropped from the graded payload so the run id
diff --git a/skyvern/forge/sdk/copilot/tools/mcp_hooks.py b/skyvern/forge/sdk/copilot/tools/mcp_hooks.py
index 9e29080be..9802703b1 100644
--- a/skyvern/forge/sdk/copilot/tools/mcp_hooks.py
+++ b/skyvern/forge/sdk/copilot/tools/mcp_hooks.py
@@ -61,6 +61,7 @@ from .scouting import (
_consume_scout_source_url,
_mark_page_inspected,
_mark_pending_browser_interaction_observation,
+ _maybe_attach_observed_download_target,
_maybe_attach_reached_download_target,
_prenav_ambiguity_for_selector,
_prenav_dynamic_row_for_selector,
@@ -69,6 +70,7 @@ from .scouting import (
_register_scout_interaction_observation,
_reset_evaluate_tracker,
_resolve_scout_role_name,
+ _scout_session_download_names,
_selector_live_match_count,
_steer_evaluate_result,
account_no_progress_interaction_click,
@@ -348,6 +350,7 @@ async def _click_pre_hook(
ctx.pending_scout_click_selector = None
ctx.pending_scout_ambiguous = None
ctx.pending_scout_dynamic_row = None
+ ctx.pending_scout_download_snapshot = None
await _capture_scout_source_url(ctx)
deterministic_result = _strip_intent_for_code_only_selector_action(params, ctx, tool_name="click")
if deterministic_result is not None:
@@ -374,6 +377,7 @@ async def _click_pre_hook(
await _capture_scout_ambiguity(ctx, selector)
if _copilot_block_authoring_policy(ctx) == BlockAuthoringPolicy.CODE_ONLY_BROWSER:
await _capture_scout_dynamic_row(ctx, selector)
+ ctx.pending_scout_download_snapshot = await _scout_session_download_names(ctx)
return None
@@ -529,6 +533,10 @@ async def _click_post_hook(
if observation_step is not None:
result["observation_step"] = observation_step
result["data"]["observation_step"] = observation_step
+ if _copilot_block_authoring_policy(ctx) == BlockAuthoringPolicy.CODE_ONLY_BROWSER:
+ # A download this click produced is proof the affordance works, so it outranks the
+ # href-shape prediction — and is the only source that sees a command-URL download.
+ await _maybe_attach_observed_download_target(ctx, result, selector=selector, url=url)
if page_evidence is not None:
_attach_scout_page_summary(result, page_evidence)
if _copilot_block_authoring_policy(ctx) == BlockAuthoringPolicy.CODE_ONLY_BROWSER:
diff --git a/skyvern/forge/sdk/copilot/tools/scouting.py b/skyvern/forge/sdk/copilot/tools/scouting.py
index 73ba96007..a1e18a398 100644
--- a/skyvern/forge/sdk/copilot/tools/scouting.py
+++ b/skyvern/forge/sdk/copilot/tools/scouting.py
@@ -12,6 +12,7 @@ from urllib.parse import urlparse
import structlog
from skyvern.config import settings
+from skyvern.forge import app
from skyvern.forge.sdk.copilot.build_test_outcome import (
RecordedBuildTestOutcome,
bind_post_run_page_path_failure,
@@ -66,11 +67,15 @@ from skyvern.forge.sdk.copilot.enforcement import (
reset_no_progress_interaction_count,
)
from skyvern.forge.sdk.copilot.reached_download_target import (
+ DOWNLOAD_KIND_OBSERVED,
ReachedDownloadTarget,
)
from skyvern.forge.sdk.copilot.reached_download_target import (
derive_from_navigation_targets as _derive_reached_download_from_nav_targets,
)
+from skyvern.forge.sdk.copilot.reached_download_target import (
+ derive_from_observed_download,
+)
from skyvern.forge.sdk.copilot.reached_download_target import guidance_for as _reached_download_guidance_for
from skyvern.forge.sdk.copilot.result_evidence import (
LoadedResultCompositionEvidence,
@@ -1889,6 +1894,79 @@ def _with_trajectory_anchor(ctx: AgentContext, target: ReachedDownloadTarget) ->
return replace(target, trajectory_anchor=anchor)
+async def _scout_session_download_names(ctx: AgentContext) -> frozenset[str] | None:
+ """Filenames currently registered in the scout's browser session, or empty when unavailable.
+
+ Read-only and failure-tolerant: this only sharpens download detection, so a storage hiccup must
+ never break a scout click."""
+ browser_session_id = ctx.browser_session_id
+ organization_id = ctx.organization_id
+ if not browser_session_id or not organization_id:
+ return None
+ try:
+ files = await app.STORAGE.list_downloaded_files_in_browser_session(
+ organization_id=organization_id, browser_session_id=browser_session_id
+ )
+ except Exception:
+ LOG.warning("copilot_scout_download_snapshot_failed", exc_info=True)
+ return None
+ return frozenset(str(name) for name in files or ())
+
+
+async def _maybe_attach_observed_download_target(
+ ctx: AgentContext,
+ result: dict[str, Any],
+ *,
+ selector: str,
+ url: str,
+) -> None:
+ """S3: pin the clicked affordance when the scout's own click produced a new download.
+
+ href shape cannot see a command-style download URL, so without this the model authors the
+ download step freehand instead of receiving the synthesizer's uncaught terminal."""
+ data = result.get("data")
+ if not isinstance(data, dict) or not selector:
+ return
+ before = ctx.pending_scout_download_snapshot
+ ctx.pending_scout_download_snapshot = None
+ if before is None:
+ return
+ try:
+ after = await _scout_session_download_names(ctx)
+ if after is None or not (after - before):
+ return
+ target = derive_from_observed_download(selector=selector, affordance_text=_scout_click_text(result))
+ if target is None:
+ return
+ data["reached_download_target"] = target.to_dict()
+ data["reached_download_guidance"] = _reached_download_guidance_for(target)
+ ctx.reached_download_target = _with_trajectory_anchor(ctx, target)
+ if ctx.synthesized_block_offered and not ctx.update_workflow_called:
+ ctx.synthesized_block_offered = False
+ ctx.synthesized_block_offered_goal_complete = False
+ _register_reached_download_scout_interaction(ctx, target, url=url)
+ LOG.info(
+ "copilot_reached_download_target_steer",
+ url=url,
+ download_kind=target.download_kind,
+ already_registered=target.already_registered,
+ )
+ except Exception:
+ LOG.warning("copilot_observed_download_target_attach_failed", exc_info=True)
+
+
+def _scout_click_text(result: dict[str, Any]) -> str:
+ data = result.get("data")
+ if not isinstance(data, dict):
+ return ""
+ text = data.get("text") or data.get("accessible_name") or ""
+ return text if isinstance(text, str) else ""
+
+
+def _is_observed_download_target(target: object) -> bool:
+ return getattr(target, "download_kind", None) == DOWNLOAD_KIND_OBSERVED
+
+
async def _maybe_attach_reached_download_target(
ctx: AgentContext,
result: dict[str, Any],
@@ -1916,6 +1994,11 @@ async def _maybe_attach_reached_download_target(
target = _derive_reached_download_from_nav_targets(parsed.get("navigation_targets"))
if target is None:
return
+ if _is_observed_download_target(ctx.reached_download_target):
+ # A download that actually fired outranks a prediction from link shape; without this the
+ # later writer would repoint the target at an affordance nothing has exercised.
+ LOG.info("copilot_reached_download_target_prediction_yielded_to_observed", url=url)
+ return
data["reached_download_target"] = target.to_dict()
data["reached_download_guidance"] = _reached_download_guidance_for(target)
if not target.already_registered:
diff --git a/skyvern/forge/sdk/db/repositories/artifacts.py b/skyvern/forge/sdk/db/repositories/artifacts.py
index 56eade5f7..5559a787a 100644
--- a/skyvern/forge/sdk/db/repositories/artifacts.py
+++ b/skyvern/forge/sdk/db/repositories/artifacts.py
@@ -4,7 +4,7 @@ import datetime
from typing import TYPE_CHECKING, Callable
import structlog
-from sqlalchemy import and_, delete, or_, select, update
+from sqlalchemy import and_, delete, func, or_, select, update
from sqlalchemy.exc import SQLAlchemyError
from skyvern.forge.sdk.artifact.models import Artifact, ArtifactType
@@ -544,6 +544,30 @@ class ArtifactsRepository(BaseRepository):
await session.commit()
return result.rowcount or 0
+ @db_operation("count_unclaimed_session_download_artifacts")
+ async def count_unclaimed_session_download_artifacts(
+ self,
+ browser_session_id: str,
+ organization_id: str,
+ run_started_at: datetime.datetime,
+ ) -> int:
+ """Session-scoped DOWNLOAD artifacts this run produced but finalization has not claimed yet.
+
+ Read-only counterpart of :meth:`claim_session_download_artifacts_for_run`, matched on the
+ same filter so a grader reading before the claim sees this run's files and not a reused
+ session's earlier ones."""
+ async with self.Session() as session:
+ result = await session.execute(
+ select(func.count())
+ .select_from(ArtifactModel)
+ .where(ArtifactModel.browser_session_id == browser_session_id)
+ .where(ArtifactModel.organization_id == organization_id)
+ .where(ArtifactModel.artifact_type == ArtifactType.DOWNLOAD)
+ .where(ArtifactModel.run_id.is_(None))
+ .where(ArtifactModel.created_at >= run_started_at)
+ )
+ return int(result.scalar() or 0)
+
@db_operation("delete_artifact_for_browser_session")
async def delete_artifact_for_browser_session(
self,
diff --git a/skyvern/forge/sdk/db/repositories/workflows.py b/skyvern/forge/sdk/db/repositories/workflows.py
index a8af273cb..8e00565fa 100644
--- a/skyvern/forge/sdk/db/repositories/workflows.py
+++ b/skyvern/forge/sdk/db/repositories/workflows.py
@@ -43,6 +43,7 @@ from skyvern.forge.sdk.workflow.browser_action_policy_enrollment import (
from skyvern.forge.sdk.workflow.models.block import Block, ForLoopBlock, WhileLoopBlock
from skyvern.forge.sdk.workflow.models.parameter import OutputParameter
from skyvern.forge.sdk.workflow.models.workflow import Workflow, WorkflowDefinition
+from skyvern.forge.sdk.workflow.runtime_completion import carried_contract, with_contract
from skyvern.schemas.runs import ProxyLocationInput
from skyvern.schemas.workflows import WorkflowStatus
@@ -1165,8 +1166,11 @@ class WorkflowsRepository(BaseRepository):
# not depend on caller-side object identity between the
# top-level parameters list and each block's field.
_align_block_output_parameters(workflow_definition)
+ definition_json = with_contract(
+ workflow_definition.model_dump(mode="json"), carried_contract(workflow.workflow_definition)
+ )
workflow.workflow_definition = with_policy(
- workflow_definition.model_dump(mode="json"), carried_policy(workflow.workflow_definition)
+ definition_json, carried_policy(workflow.workflow_definition)
)
if version is not None:
workflow.version = version
diff --git a/skyvern/forge/sdk/routes/workflow_copilot.py b/skyvern/forge/sdk/routes/workflow_copilot.py
index e3b7a2af1..4b97c9add 100644
--- a/skyvern/forge/sdk/routes/workflow_copilot.py
+++ b/skyvern/forge/sdk/routes/workflow_copilot.py
@@ -108,6 +108,7 @@ from skyvern.forge.sdk.services import org_auth_service
from skyvern.forge.sdk.workflow.exceptions import BaseWorkflowHTTPException
from skyvern.forge.sdk.workflow.models.parameter import ParameterType
from skyvern.forge.sdk.workflow.models.workflow import Workflow
+from skyvern.forge.sdk.workflow.runtime_completion import contract_from_request_criteria
from skyvern.forge.sdk.workflow.workflow_definition_converter import convert_workflow_definition
from skyvern.schemas.workflows import (
WorkflowCreateYAMLRequest,
@@ -1086,6 +1087,9 @@ async def _finalise_normal_turn(
restore_failed = True
if _should_commit_staged_workflow(chat.auto_accept, agent_result):
+ # Both acceptance paths must carry the same contract, or the obligation depends on which
+ # button the user pressed.
+ await _attach_requested_completion_contract(chat, agent_result.staged_workflow, agent_result)
try:
await _commit_staged_workflow(
organization_id=organization_id,
@@ -2891,6 +2895,44 @@ async def workflow_copilot_clear_proposed_workflow(
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Chat not found")
+async def _turn_completion_criteria(chat: Any, agent_result: Any) -> object | None:
+ """This turn's typed criteria, preferring the in-memory turn state over the stored snapshot.
+
+ Auto-accept commits before the turn's criteria are persisted, so a turn that first mints a
+ download criterion would otherwise read a pre-turn snapshot and commit with no contract."""
+ turn_state = getattr(agent_result, "completion_criteria_turn_state", None)
+ decision = getattr(turn_state, "decision", None)
+ criteria = getattr(decision, "criteria", None)
+ if criteria:
+ return criteria
+ snapshot = await _load_completion_criteria_snapshot(chat)
+ active = snapshot.active if snapshot is not None else None
+ return active.criteria if active is not None else None
+
+
+async def _attach_requested_completion_contract(chat: Any, target: Any, agent_result: Any = None) -> None:
+ """Carry the request's typed completion criteria onto the version being accepted.
+
+ Read at accept time so the contract binds to the definition being accepted, and skipped silently
+ when the chat has no active criteria — a workflow that promised nothing is graded as before.
+ Applied on both acceptance paths so the obligation cannot depend on which one ran."""
+ try:
+ definition = getattr(target, "workflow_definition", None)
+ if definition is None:
+ return
+ contract = contract_from_request_criteria(await _turn_completion_criteria(chat, agent_result))
+ if contract is None:
+ return
+ definition.completion_contract = contract
+ LOG.info(
+ "copilot_completion_contract_attached_from_request",
+ workflow_copilot_chat_id=chat.workflow_copilot_chat_id,
+ workflow_permanent_id=chat.workflow_permanent_id,
+ )
+ except Exception:
+ LOG.warning("copilot_completion_contract_attach_failed", exc_info=True)
+
+
@base_router.post("/workflow/copilot/apply-proposed-workflow", include_in_schema=False)
async def workflow_copilot_apply_proposed_workflow(
apply_request: WorkflowCopilotApplyProposedWorkflowRequest,
@@ -2925,6 +2967,8 @@ async def workflow_copilot_apply_proposed_workflow(
detail=f"Proposed copilot YAML is invalid: {e}",
)
+ await _attach_requested_completion_contract(chat, yaml_request)
+
current_workflow = await app.WORKFLOW_SERVICE.get_workflow_by_permanent_id(
workflow_permanent_id=chat.workflow_permanent_id,
organization_id=organization.organization_id,
diff --git a/skyvern/forge/sdk/workflow/models/block.py b/skyvern/forge/sdk/workflow/models/block.py
index e804d9d78..9fd06555a 100644
--- a/skyvern/forge/sdk/workflow/models/block.py
+++ b/skyvern/forge/sdk/workflow/models/block.py
@@ -70,6 +70,7 @@ from skyvern.exceptions import (
MissingStarterUrl,
NoTOTPVerificationCodeFound,
PDFParsingError,
+ ScriptTerminationException,
SkyvernException,
SyncTriggeredSequentialCredentialUnsupported,
TaskNotFound,
@@ -104,6 +105,10 @@ from skyvern.forge.sdk.api.llm.exceptions import (
from skyvern.forge.sdk.api.llm.schema_validator import validate_schema
from skyvern.forge.sdk.artifact.models import ArtifactType
from skyvern.forge.sdk.copilot.block_goal_wrapping import compose_mini_goal
+from skyvern.forge.sdk.copilot.reached_download_target import (
+ block_output_has_registered_download,
+ code_is_download_intent,
+)
from skyvern.forge.sdk.copilot.runtime import _browser_context_is_attachable
from skyvern.forge.sdk.copilot.self_heal_recovery import SelfHealRecoveryResult, run_self_heal_recovery
from skyvern.forge.sdk.copilot.turn_origin import HealAdoptionFailed, TurnOrigin
@@ -4500,21 +4505,48 @@ async def wrapper({default_args}):
)
return compose_mini_goal(main_goal=safe_main, mini_goal=safe_mini)
- async def _self_heal_enabled(self, workflow_run_context: WorkflowRunContext) -> bool:
- # User-facing per-workflow setting, restricted to copilot-authored workflows —
- # pre-copilot code blocks must never gain agentic recovery from the toggle alone.
- # The env default stays as the OSS/standalone and local-dev override.
- if settings.ENABLE_CODE_BLOCK_SELF_HEALING:
- return True
+ async def _record_unregistered_download_intent(
+ self,
+ *,
+ engine: str,
+ registered: bool,
+ output: object,
+ workflow_run_context: WorkflowRunContext,
+ workflow_run_id: str,
+ workflow_run_block_id: str,
+ organization_id: str | None,
+ ) -> None:
+ """Record a block that authors a browser download but registered no file.
+
+ Telemetry only, and deliberately not outcome authority: the AST shape is evadable and
+ authorship is not a promise. The outcome belongs to the workflow's persisted completion
+ contract, graded at run finalization; this record's deletion condition is that coverage.
+ Emitted from every execution engine so the prevalence it measures is not engine-biased.
+ """
+ if registered or not code_is_download_intent(self.code):
+ return
+ LOG.warning(
+ "codeblock.download_intent_unregistered",
+ engine=engine,
+ copilot_authored=await self._workflow_is_copilot_authored(workflow_run_context),
+ workflow_run_id=workflow_run_id,
+ workflow_run_block_id=workflow_run_block_id,
+ block_label=self.label,
+ code_sha256=hashlib.sha256(self.code.encode()).hexdigest(),
+ result_keys=sorted(output.keys()) if isinstance(output, dict) else None,
+ organization_id=organization_id,
+ )
+
+ async def _workflow_is_copilot_authored(self, workflow_run_context: WorkflowRunContext) -> bool:
workflow = workflow_run_context.workflow
- if workflow is None or not workflow.enable_self_healing:
+ if workflow is None:
return False
if "copilot" in (workflow.created_by, workflow.edited_by):
return True
# User saves re-stamp both fields with the user id, so the current version alone is
# not durable; fall back to lineage (copilot stamps every version it writes and
- # back-stamps v1 on copilot-born workflows). This runs inside the block's exception
- # handler — a lookup failure must fail closed, never mask the original block failure.
+ # back-stamps v1 on copilot-born workflows). A lookup failure must fail closed —
+ # callers gate copilot-only behavior on this, and it must never mask a block failure.
try:
return await app.DATABASE.workflows.is_workflow_copilot_authored(
workflow_permanent_id=workflow.workflow_permanent_id,
@@ -4522,12 +4554,23 @@ async def wrapper({default_args}):
)
except Exception:
LOG.warning(
- "Self-heal copilot-lineage lookup failed; failing closed (no heal)",
+ "Copilot-lineage lookup failed; failing closed",
workflow_permanent_id=workflow.workflow_permanent_id,
exc_info=True,
)
return False
+ async def _self_heal_enabled(self, workflow_run_context: WorkflowRunContext) -> bool:
+ # User-facing per-workflow setting, restricted to copilot-authored workflows —
+ # pre-copilot code blocks must never gain agentic recovery from the toggle alone.
+ # The env default stays as the OSS/standalone and local-dev override.
+ if settings.ENABLE_CODE_BLOCK_SELF_HEALING:
+ return True
+ workflow = workflow_run_context.workflow
+ if workflow is None or not workflow.enable_self_healing:
+ return False
+ return await self._workflow_is_copilot_authored(workflow_run_context)
+
def _is_healable_page_failure(
self,
exception: Exception,
@@ -4841,9 +4884,9 @@ async def wrapper({default_args}):
or failure.healability_hint is not None
)
if not has_structured:
- # No healability_hint to disambiguate, so only the page-class code heals blind;
+ # No healability_hint to disambiguate, so only the page-class codes heal blind;
# user_code_error is ambiguous (could be a bare `raise`) and stays fail-closed.
- if failure.error_code == "unsupported_page_operation":
+ if failure.error_code in {"unsupported_page_operation", "browser_operation_failed"}:
return HealClassification(healable=True, skip_reason=None)
return HealClassification(
healable=False,
@@ -5542,10 +5585,22 @@ async def wrapper({default_args}):
await recorder.finalize(success=False)
return secure_code_block_result.block_result
await recorder.finalize(success=True)
+ secure_output = secure_code_block_result.block_result.output_parameter_value
+ await self._record_unregistered_download_intent(
+ engine="secure_runner",
+ # The sidecar registers into the block output, so that payload is the
+ # registration evidence here — the inline directory diff never runs.
+ registered=block_output_has_registered_download(secure_output),
+ output=secure_output,
+ workflow_run_context=workflow_run_context,
+ workflow_run_id=workflow_run_id,
+ workflow_run_block_id=workflow_run_block_id,
+ organization_id=organization_id,
+ )
await self.record_output_parameter_value(
workflow_run_context,
workflow_run_id,
- secure_code_block_result.block_result.output_parameter_value,
+ secure_output,
)
return secure_code_block_result.block_result
LOG.warning(
@@ -5594,6 +5649,19 @@ async def wrapper({default_args}):
organization_id=organization_id,
)
except Exception as e:
+ # Exact type, not isinstance: the IllegitCompleteScriptTermination subclass means the
+ # complete-verifier rejected the block, which is a failure to heal, not an intentional stop.
+ if type(e) is ScriptTerminationException:
+ await recorder.persist(recorder.recorded_actions())
+ await recorder.finalize(success=False)
+ return await self.build_block_result(
+ success=False,
+ failure_reason=workflow_run_context.mask_secrets_in_data(str(e)),
+ output_parameter_value=None,
+ status=BlockStatus.terminated,
+ workflow_run_block_id=workflow_run_block_id,
+ organization_id=organization_id,
+ )
exc = CustomizedCodeException(e)
failing_line = user_code_line_from_exception(e)
# User code can raise an exception carrying a resolved secret (e.g.
@@ -5692,6 +5760,16 @@ async def wrapper({default_args}):
result["downloaded_file_urls"] = [fi.url for fi in downloaded_files]
result["downloaded_file_artifact_ids"] = [fi.artifact_id for fi in downloaded_files if fi.artifact_id]
+ await self._record_unregistered_download_intent(
+ engine="inline",
+ registered=bool(downloaded_files),
+ output=result,
+ workflow_run_context=workflow_run_context,
+ workflow_run_id=workflow_run_id,
+ workflow_run_block_id=workflow_run_block_id,
+ organization_id=organization_id,
+ )
+
await self.record_output_parameter_value(workflow_run_context, workflow_run_id, result)
return await self.build_block_result(
success=True,
diff --git a/skyvern/forge/sdk/workflow/models/workflow.py b/skyvern/forge/sdk/workflow/models/workflow.py
index b8a9ac3a4..7038549f6 100644
--- a/skyvern/forge/sdk/workflow/models/workflow.py
+++ b/skyvern/forge/sdk/workflow/models/workflow.py
@@ -116,6 +116,10 @@ class WorkflowDefinition(BaseModel):
finally_block_label: str | None = None
error_code_mapping: dict[str, str] | None = None
workflow_system_prompt: str | None = None
+ completion_contract: dict[str, Any] | None = Field(
+ default=None,
+ description="Copilot-managed: what a run of this workflow must produce, graded at run finalization. Derived from the request when a workflow is accepted; not intended to be authored by hand.",
+ )
def validate(self) -> None:
all_labels: set[str] = set()
diff --git a/skyvern/forge/sdk/workflow/runtime_completion.py b/skyvern/forge/sdk/workflow/runtime_completion.py
new file mode 100644
index 000000000..47b8ffa1f
--- /dev/null
+++ b/skyvern/forge/sdk/workflow/runtime_completion.py
@@ -0,0 +1,148 @@
+"""Runtime grading of a workflow's declared completion contract.
+
+A workflow version can carry a small typed statement of what a run must produce. Grading it at
+finalization — after files register, before the status write — is what lets a run that produced
+nothing report that, no matter which execution engine ran the blocks or how the block's own code
+chose to describe its outcome.
+
+Contract-less workflows are untouched: the grader has nothing to say about a run whose workflow
+never declared an outcome.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+
+import structlog
+
+LOG = structlog.get_logger()
+
+# The only criterion kind graded today. Its evidence is the execution layer's own file registration,
+# which no generated code can fake or swallow.
+CRITERION_REGISTERED_DOWNLOAD = "registered_download"
+
+_CONTRACT_KEY = "completion_contract"
+_SUPPORTED_KINDS = frozenset({CRITERION_REGISTERED_DOWNLOAD})
+# The field is public on the workflow schema, so a hand-written contract is bounded here.
+_MAX_CRITERIA = 16
+_MAX_ID_CHARS = 128
+
+
+@dataclass(frozen=True)
+class CompletionCriterion:
+ id: str
+ kind: str
+ min_count: int = 1
+
+
+@dataclass(frozen=True)
+class ContractVerdict:
+ satisfied: bool
+ unmet_criterion_ids: tuple[str, ...]
+ reason: str | None
+
+
+def parse_completion_contract(workflow_definition: object) -> tuple[CompletionCriterion, ...]:
+ """Read the typed criteria a workflow version declares, ignoring anything unrecognized.
+
+ Unknown kinds are dropped rather than failing the parse: an older worker must keep running a
+ workflow authored by a newer one, and a criterion it cannot grade is not a criterion it may
+ treat as unmet."""
+ raw = (
+ workflow_definition.get(_CONTRACT_KEY)
+ if isinstance(workflow_definition, dict)
+ else getattr(workflow_definition, _CONTRACT_KEY, None)
+ )
+ if not isinstance(raw, dict):
+ return ()
+ criteria = raw.get("criteria")
+ if not isinstance(criteria, list):
+ return ()
+ parsed: list[CompletionCriterion] = []
+ for item in criteria[:_MAX_CRITERIA]:
+ if not isinstance(item, dict):
+ continue
+ kind = str(item.get("kind") or "").strip()
+ if kind not in _SUPPORTED_KINDS:
+ continue
+ identifier = (str(item.get("id") or kind).strip() or kind)[:_MAX_ID_CHARS]
+ try:
+ min_count = int(item.get("min_count", 1))
+ except (TypeError, ValueError):
+ min_count = 1
+ parsed.append(CompletionCriterion(id=identifier, kind=kind, min_count=max(1, min_count)))
+ return tuple(parsed)
+
+
+def grade_completion_contract(
+ criteria: tuple[CompletionCriterion, ...],
+ *,
+ registered_download_count: int,
+) -> ContractVerdict:
+ """Grade declared criteria against execution-layer evidence."""
+ unmet: list[str] = []
+ for criterion in criteria:
+ if criterion.kind == CRITERION_REGISTERED_DOWNLOAD and registered_download_count < criterion.min_count:
+ unmet.append(criterion.id)
+ if not unmet:
+ return ContractVerdict(satisfied=True, unmet_criterion_ids=(), reason=None)
+ return ContractVerdict(
+ satisfied=False,
+ unmet_criterion_ids=tuple(unmet),
+ reason="The workflow did not produce the file it is declared to download.",
+ )
+
+
+def contract_from_request_criteria(criteria: object) -> dict[str, object] | None:
+ """Project the request's own typed completion criteria into a workflow-carried contract.
+
+ The obligation comes from what the user asked for, never from the shape of the code the product
+ generated: a block that merely reports a download-shaped result promises nothing, and a request
+ that asked for a file promises one however the block is written."""
+ if not isinstance(criteria, (list, tuple)):
+ return None
+ if not any(_criterion_requests_a_download(criterion) for criterion in criteria):
+ return None
+ return {
+ "schema_version": 1,
+ "criteria": [{"id": "requested_download", "kind": CRITERION_REGISTERED_DOWNLOAD, "min_count": 1}],
+ }
+
+
+def carried_contract(existing_definition: object) -> dict[str, object] | None:
+ """The contract already stored on a workflow version, if any."""
+ raw = (
+ existing_definition.get(_CONTRACT_KEY)
+ if isinstance(existing_definition, dict)
+ else getattr(existing_definition, _CONTRACT_KEY, None)
+ )
+ return raw if isinstance(raw, dict) else None
+
+
+def with_contract(definition: dict, carried: dict[str, object] | None) -> dict:
+ """Preserve a stored contract across a write that did not carry one.
+
+ Every non-copilot save path rebuilds the definition through models that do not know this field,
+ so without this a builder edit anywhere in the workflow would silently drop the obligation."""
+ if carried is None or definition.get(_CONTRACT_KEY) is not None:
+ return definition
+ definition[_CONTRACT_KEY] = carried
+ return definition
+
+
+def _criterion_requests_a_download(criterion: object) -> bool:
+ """Whether a request criterion asks for a registered download.
+
+ The request classifier types this as ``deliverable_kind``/``output_path``; the synthetic id is
+ the copilot's own internal marker for the same obligation. Both are request-derived."""
+ from skyvern.forge.sdk.copilot.reached_download_target import REGISTERED_DOWNLOAD_REQUESTED_OUTPUT_PATHS
+ from skyvern.forge.sdk.copilot.request_policy import REGISTERED_DOWNLOAD_COMPLETION_CRITERION_ID
+
+ for attr in ("deliverable_kind", "declared_deliverable_kind"):
+ if str(getattr(criterion, attr, "") or "").strip() == CRITERION_REGISTERED_DOWNLOAD:
+ return True
+ if str(getattr(criterion, "output_path", "") or "").strip() in REGISTERED_DOWNLOAD_REQUESTED_OUTPUT_PATHS:
+ return True
+ # The copilot's own synthetic marker for the same obligation, compared by id so this stays
+ # readable from an untyped criterion.
+ return str(getattr(criterion, "id", "") or "") == REGISTERED_DOWNLOAD_COMPLETION_CRITERION_ID
diff --git a/skyvern/forge/sdk/workflow/service.py b/skyvern/forge/sdk/workflow/service.py
index b4acddc8d..e35477ac6 100644
--- a/skyvern/forge/sdk/workflow/service.py
+++ b/skyvern/forge/sdk/workflow/service.py
@@ -61,7 +61,7 @@ from skyvern.exceptions import (
from skyvern.forge import app
from skyvern.forge.failure_classifier import classify_from_failure_reason
from skyvern.forge.prompts import prompt_engine
-from skyvern.forge.sdk.api.files import is_temp_working_dir
+from skyvern.forge.sdk.api.files import is_temp_working_dir, resolve_run_download_id
from skyvern.forge.sdk.artifact.models import Artifact, ArtifactType
from skyvern.forge.sdk.artifact.storage.base import _file_infos_from_download_artifacts
from skyvern.forge.sdk.browser_action_policy import BrowserActionPolicy
@@ -156,6 +156,11 @@ from skyvern.forge.sdk.workflow.models.workflow import (
WorkflowRunStatus,
is_adaptive_caching,
)
+from skyvern.forge.sdk.workflow.runtime_completion import (
+ ContractVerdict,
+ grade_completion_contract,
+ parse_completion_contract,
+)
from skyvern.forge.sdk.workflow.secret_encryption import encrypt_workflow_definition_secrets
from skyvern.forge.sdk.workflow.status_mapping import (
BLOCK_STATUS_MAP,
@@ -530,6 +535,8 @@ def _get_workflow_definition_core_data(workflow_definition: WorkflowDefinition)
"credential_parameter_id",
"onepassword_credential_parameter_id",
"azure_vault_credential_parameter_id",
+ # Graded at finalization, not executed: its presence must not invalidate cached scripts.
+ "completion_contract",
"disable_cache",
"next_block_label",
"version",
@@ -3712,6 +3719,7 @@ class WorkflowService:
self._finalize_workflow_run_status(
workflow_run_id=workflow_run_id,
workflow_run=workflow_run,
+ is_partial_run=bool(block_labels),
pre_finally_status=pre_finally_status,
pre_finally_failure_reason=pre_finally_failure_reason,
pre_finally_failure_category=pre_finally_failure_category,
@@ -7501,6 +7509,7 @@ class WorkflowService:
pre_finally_status: WorkflowRunStatus,
pre_finally_failure_reason: str | None,
pre_finally_failure_category: list[dict] | None = None,
+ is_partial_run: bool = False,
) -> WorkflowRun:
"""
Set final workflow run status based on pre-finally state.
@@ -7517,6 +7526,16 @@ class WorkflowService:
WorkflowRunStatus.terminated,
WorkflowRunStatus.timed_out,
):
+ contract_verdict = await self._grade_completion_contract(workflow_run, is_partial_run=is_partial_run)
+ if contract_verdict is not None and not contract_verdict.satisfied:
+ updated = await self._update_workflow_run_status_if_not_final(
+ workflow_run_id=workflow_run_id,
+ status=WorkflowRunStatus.terminated,
+ failure_reason=contract_verdict.reason,
+ )
+ if updated is None:
+ return await self._current_row_after_lost_finalize(workflow_run_id, workflow_run)
+ return updated
updated = await self._update_workflow_run_status_if_not_final(
workflow_run_id=workflow_run_id,
status=WorkflowRunStatus.completed,
@@ -7541,6 +7560,81 @@ class WorkflowService:
return workflow_run
+ async def _session_download_count(self, workflow_run: WorkflowRun, context: object) -> int:
+ """This run's downloads still scoped to the browser session, which cleanup tags with the run
+ id after this grade runs. Failure-tolerant: an unavailable count must not manufacture a
+ shortfall."""
+ browser_session_id = getattr(context, "browser_session_id", None)
+ if not browser_session_id:
+ return 0
+ try:
+ # Scoped to this run's window and to rows finalization has not claimed: a reused session
+ # carries earlier runs' downloads, which would otherwise satisfy the contract for a run
+ # that produced nothing.
+ return await app.DATABASE.artifacts.count_unclaimed_session_download_artifacts(
+ browser_session_id=browser_session_id,
+ organization_id=workflow_run.organization_id,
+ run_started_at=workflow_run.created_at,
+ )
+ except Exception:
+ LOG.warning(
+ "Session download count failed while grading; counting zero from this source",
+ workflow_run_id=workflow_run.workflow_run_id,
+ exc_info=True,
+ )
+ return 0
+
+ async def _grade_completion_contract(
+ self, workflow_run: WorkflowRun, *, is_partial_run: bool = False
+ ) -> ContractVerdict | None:
+ """Grade the workflow's declared completion contract, or None when it declares nothing.
+
+ Runs before the status write and after blocks have registered their files, so the verdict
+ reflects what the run produced rather than what its code reported. Failure-tolerant: a
+ grading error must never turn a good run into a bad one."""
+ if is_partial_run:
+ # A subset run was never asked to produce the whole workflow's deliverable.
+ return None
+ try:
+ # The run is pinned to one workflow version; grading the permanent id would read
+ # whatever version exists now, so an edit mid-run could judge this run by a contract
+ # it never executed.
+ workflow = await self.get_workflow(
+ workflow_id=workflow_run.workflow_id,
+ organization_id=workflow_run.organization_id,
+ )
+ criteria = parse_completion_contract(workflow.workflow_definition if workflow else None)
+ if not criteria:
+ return None
+ context = skyvern_context.current()
+ # Producer and consumer must resolve the same download key: a sub-workflow registers
+ # files under the parent's run id, so the raw workflow_run_id would read an empty dir.
+ files = await app.STORAGE.get_downloaded_files(
+ organization_id=workflow_run.organization_id,
+ run_id=resolve_run_download_id(context, fallback_run_id=workflow_run.workflow_run_id),
+ )
+ # Session-scoped downloads are only tagged with the run id during cleanup, which runs
+ # after this grade. Counting them here keeps a real download from reading as zero;
+ # double-counting one file is harmless because the grader only ever needs a floor.
+ session_files = await self._session_download_count(workflow_run, context)
+ verdict = grade_completion_contract(criteria, registered_download_count=len(files or []) + session_files)
+ LOG.info(
+ "workflow_completion_contract_graded",
+ workflow_run_id=workflow_run.workflow_run_id,
+ satisfied=verdict.satisfied,
+ unmet_criterion_ids=list(verdict.unmet_criterion_ids),
+ registered_download_count=len(files or []),
+ session_download_count=session_files,
+ )
+ return verdict
+ except Exception:
+ LOG.warning(
+ "Completion-contract grading failed; leaving the run status unchanged",
+ workflow_run_id=workflow_run.workflow_run_id,
+ exc_info=True,
+ )
+ return None
+
async def _current_row_after_lost_finalize(self, workflow_run_id: str, fallback: WorkflowRun) -> WorkflowRun:
LOG.info(
"In-band finalize lost to an out-of-band finalizer; leaving terminal status intact",
diff --git a/skyvern/forge/sdk/workflow/workflow_definition_converter.py b/skyvern/forge/sdk/workflow/workflow_definition_converter.py
index 6ac6b26ec..8e48f209e 100644
--- a/skyvern/forge/sdk/workflow/workflow_definition_converter.py
+++ b/skyvern/forge/sdk/workflow/workflow_definition_converter.py
@@ -343,6 +343,7 @@ def convert_workflow_definition(
finally_block_label=workflow_definition_yaml.finally_block_label,
error_code_mapping=workflow_definition_yaml.error_code_mapping,
workflow_system_prompt=workflow_definition_yaml.workflow_system_prompt,
+ completion_contract=workflow_definition_yaml.completion_contract,
)
LOG.info(
diff --git a/skyvern/schemas/workflows.py b/skyvern/schemas/workflows.py
index 15640f428..693faf083 100644
--- a/skyvern/schemas/workflows.py
+++ b/skyvern/schemas/workflows.py
@@ -1406,6 +1406,10 @@ class WorkflowDefinitionYAML(BaseModel):
finally_block_label: str | None = None
error_code_mapping: dict[str, str] | None = None
workflow_system_prompt: str | None = None
+ completion_contract: dict[str, Any] | None = Field(
+ default=None,
+ description="Copilot-managed: what a run of this workflow must produce, graded at run finalization. Derived from the request when a workflow is accepted; not intended to be authored by hand.",
+ )
@model_validator(mode="after")
def validate_unique_block_labels(self) -> "WorkflowDefinitionYAML":
diff --git a/tests/unit/test_code_block_downloads.py b/tests/unit/test_code_block_downloads.py
index 3c66b64dd..ab2718bc4 100644
--- a/tests/unit/test_code_block_downloads.py
+++ b/tests/unit/test_code_block_downloads.py
@@ -15,7 +15,10 @@ from types import SimpleNamespace
from unittest.mock import AsyncMock
import pytest
+from structlog.testing import capture_logs
+from skyvern.exceptions import IllegitCompleteScriptTermination, ScriptTerminationException
+from skyvern.forge.sdk.copilot.reached_download_target import code_is_download_intent
from skyvern.forge.sdk.core import skyvern_context
from skyvern.forge.sdk.core.skyvern_context import SkyvernContext
from skyvern.forge.sdk.schemas.files import FileInfo
@@ -27,6 +30,7 @@ from skyvern.forge.sdk.workflow.models.parameter import (
WorkflowParameter,
WorkflowParameterType,
)
+from skyvern.schemas.workflows import BlockResult, BlockStatus
from skyvern.webeye.browser_artifacts import BrowserArtifacts
@@ -72,10 +76,20 @@ def _isolated_download_path(tmp_path, monkeypatch: pytest.MonkeyPatch) -> str:
return str(download_root)
+def _copilot_workflow() -> SimpleNamespace:
+ return SimpleNamespace(
+ created_by="copilot",
+ edited_by=None,
+ workflow_permanent_id="wpid_test",
+ organization_id="o_1",
+ )
+
+
def _wire_block_runtime(
monkeypatch: pytest.MonkeyPatch,
*,
values: dict[str, object] | None = None,
+ workflow: SimpleNamespace | None = None,
) -> None:
page = SimpleNamespace()
browser_state = SimpleNamespace(get_working_page=AsyncMock(return_value=page), browser_artifacts=BrowserArtifacts())
@@ -83,6 +97,7 @@ def _wire_block_runtime(
context = SimpleNamespace(
organization_id="o_1",
+ workflow=workflow,
get_value=lambda key: (values or {}).get(key),
mask_secrets_in_data=lambda data, mask="*****": data,
)
@@ -91,6 +106,22 @@ def _wire_block_runtime(
monkeypatch.setattr(CodeBlock, "record_output_parameter_value", AsyncMock())
+def _wire_secure_runner(monkeypatch: pytest.MonkeyPatch, *, output: dict) -> None:
+ """Route execute() down the secure sidecar arm, which returns before the inline directory diff."""
+ block_result = BlockResult(
+ success=True,
+ output_parameter=_output_parameter("code_out"),
+ output_parameter_value=output,
+ status=BlockStatus.completed,
+ workflow_run_block_id="",
+ )
+ fake_app = block_module.app
+ fake_app.AGENT_FUNCTION.should_use_codeblock_runner = AsyncMock(return_value=True)
+ fake_app.AGENT_FUNCTION.execute_code_block_override = AsyncMock(
+ return_value=SimpleNamespace(block_result=block_result, failure=None)
+ )
+
+
def _fake_storage_app(monkeypatch: pytest.MonkeyPatch, *, save, get) -> None:
fake_app = SimpleNamespace(
STORAGE=SimpleNamespace(save_downloaded_files=save, get_downloaded_files=get),
@@ -345,3 +376,259 @@ async def test_code_block_file_parameter_empty_uri_left_unchanged(
assert result.success is True
assert result.output_parameter_value["resolved"] == {"s3uri": ""}
download_mock.assert_not_awaited()
+
+
+def _download_intent_events(logs: list[dict]) -> list[dict]:
+ return [entry for entry in logs if entry.get("event") == "codeblock.download_intent_unregistered"]
+
+
+# The shape generated code actually takes: the download terminal is present and does raise on
+# timeout, but a bare ``except`` converts that raise into a returned blocker string, so the block
+# returns normally and reports success.
+_SWALLOWED_DOWNLOAD_TERMINAL = """
+downloaded_files = []
+blocker = ""
+try:
+ async with page.expect_download(timeout=15000) as download_info:
+ await latest.click()
+ download = await download_info.value
+ downloaded_files.append({"file_name": download.suggested_filename})
+except Exception:
+ blocker = "the link opens a delivery form rather than a downloadable file"
+return {"downloaded_files": downloaded_files, "blocker": blocker}
+"""
+
+
+async def test_download_intent_block_registering_nothing_is_counted(
+ monkeypatch: pytest.MonkeyPatch, _isolated_download_path: str
+) -> None:
+ skyvern_context.set(SkyvernContext(organization_id="o_1", workflow_run_id="wr_1", run_id="wr_1"))
+
+ _fake_storage_app(monkeypatch, save=AsyncMock(), get=AsyncMock(return_value=[]))
+ _wire_block_runtime(monkeypatch)
+
+ block = CodeBlock(
+ label="download_invoice",
+ code='result = {"downloaded_files": [], "blocker": "no downloadable file"}\nreturn result',
+ output_parameter=_output_parameter("code_out"),
+ )
+ with capture_logs() as logs:
+ result = await block.execute(workflow_run_id="wr_1", workflow_run_block_id="", organization_id="o_1")
+
+ events = _download_intent_events(logs)
+ assert len(events) == 1
+ assert events[0]["block_label"] == "download_invoice"
+ assert events[0]["copilot_authored"] is False
+ # Telemetry, not outcome authority: the outcome contract is the workflow's persisted
+ # completion criteria, graded at run finalization.
+ assert result.success is True
+
+
+async def test_copilot_authored_download_block_registering_nothing_is_counted_with_lineage(
+ monkeypatch: pytest.MonkeyPatch, _isolated_download_path: str
+) -> None:
+ skyvern_context.set(SkyvernContext(organization_id="o_1", workflow_run_id="wr_1", run_id="wr_1"))
+
+ _fake_storage_app(monkeypatch, save=AsyncMock(), get=AsyncMock(return_value=[]))
+ _wire_block_runtime(monkeypatch, workflow=_copilot_workflow())
+
+ # The production shape, executed: page is a stub, so page.expect_download raises inside the
+ # try, the bare except converts it into the blocker string, and the code returns normally.
+ block = CodeBlock(
+ label="download_invoice",
+ code=_SWALLOWED_DOWNLOAD_TERMINAL,
+ output_parameter=_output_parameter("code_out"),
+ )
+ with capture_logs() as logs:
+ result = await block.execute(workflow_run_id="wr_1", workflow_run_block_id="", organization_id="o_1")
+
+ events = _download_intent_events(logs)
+ assert len(events) == 1
+ assert events[0]["copilot_authored"] is True
+ assert result.success is True
+
+
+async def test_copilot_authored_download_block_that_registers_a_file_succeeds(
+ monkeypatch: pytest.MonkeyPatch, _isolated_download_path: str
+) -> None:
+ skyvern_context.set(SkyvernContext(organization_id="o_1", workflow_run_id="wr_1", run_id="wr_1"))
+
+ file_info = FileInfo(
+ url="https://api.example.com/v1/artifacts/a_dl_1/content?artifact_name=invoice.pdf",
+ filename="invoice.pdf",
+ checksum="deadbeef",
+ artifact_id="a_dl_1",
+ modified_at=datetime(2026, 6, 14, 12, 0, tzinfo=UTC),
+ )
+ _fake_storage_app(monkeypatch, save=AsyncMock(), get=AsyncMock(side_effect=[[], [file_info]]))
+ _wire_block_runtime(monkeypatch, workflow=_copilot_workflow())
+
+ block = CodeBlock(
+ label="download_invoice",
+ code='result = {"downloaded_files": []}\nreturn result',
+ output_parameter=_output_parameter("code_out"),
+ )
+ with capture_logs() as logs:
+ result = await block.execute(workflow_run_id="wr_1", workflow_run_block_id="", organization_id="o_1")
+
+ assert _download_intent_events(logs) == []
+ assert result.success is True
+ assert result.output_parameter_value["downloaded_file_artifact_ids"] == ["a_dl_1"]
+
+
+async def test_download_intent_block_that_registers_a_file_is_not_counted(
+ monkeypatch: pytest.MonkeyPatch, _isolated_download_path: str
+) -> None:
+ skyvern_context.set(SkyvernContext(organization_id="o_1", workflow_run_id="wr_1", run_id="wr_1"))
+
+ file_info = FileInfo(
+ url="https://api.example.com/v1/artifacts/a_dl_1/content?artifact_name=invoice.pdf",
+ filename="invoice.pdf",
+ checksum="deadbeef",
+ artifact_id="a_dl_1",
+ modified_at=datetime(2026, 6, 14, 12, 0, tzinfo=UTC),
+ )
+ _fake_storage_app(monkeypatch, save=AsyncMock(), get=AsyncMock(side_effect=[[], [file_info]]))
+ _wire_block_runtime(monkeypatch)
+
+ block = CodeBlock(
+ label="download_invoice",
+ code='result = {"downloaded_files": []}\nreturn result',
+ output_parameter=_output_parameter("code_out"),
+ )
+ with capture_logs() as logs:
+ result = await block.execute(workflow_run_id="wr_1", workflow_run_block_id="", organization_id="o_1")
+
+ assert _download_intent_events(logs) == []
+ assert result.success is True
+ assert result.output_parameter_value["downloaded_file_artifact_ids"] == ["a_dl_1"]
+
+
+async def test_block_without_download_intent_is_not_counted(
+ monkeypatch: pytest.MonkeyPatch, _isolated_download_path: str
+) -> None:
+ skyvern_context.set(SkyvernContext(organization_id="o_1", workflow_run_id="wr_1", run_id="wr_1"))
+
+ _fake_storage_app(monkeypatch, save=AsyncMock(), get=AsyncMock(return_value=[]))
+ _wire_block_runtime(monkeypatch)
+
+ block = CodeBlock(
+ label="extract_only",
+ code='return {"blocker": "no downloadable file", "rows": []}',
+ output_parameter=_output_parameter("code_out"),
+ )
+ with capture_logs() as logs:
+ result = await block.execute(workflow_run_id="wr_1", workflow_run_block_id="", organization_id="o_1")
+
+ assert _download_intent_events(logs) == []
+ assert result.success is True
+
+
+async def test_terminate_maps_to_terminated_not_failed(
+ monkeypatch: pytest.MonkeyPatch, _isolated_download_path: str
+) -> None:
+ """``page.terminate()`` is the sanctioned way for a block to report it cannot reach the file."""
+ skyvern_context.set(SkyvernContext(organization_id="o_1", workflow_run_id="wr_1", run_id="wr_1"))
+
+ _fake_storage_app(monkeypatch, save=AsyncMock(), get=AsyncMock(return_value=[]))
+ _wire_block_runtime(monkeypatch)
+ monkeypatch.setattr(
+ CodeBlock,
+ "execute_user_function_with_timeout",
+ AsyncMock(side_effect=ScriptTerminationException("Terminate called: no downloadable file")),
+ )
+
+ block = CodeBlock(
+ label="download_invoice",
+ code="value = 'unused'",
+ output_parameter=_output_parameter("code_out"),
+ )
+ result = await block.execute(workflow_run_id="wr_1", workflow_run_block_id="", organization_id="o_1")
+
+ assert result.success is False
+ assert result.status == BlockStatus.terminated
+ assert "no downloadable file" in (result.failure_reason or "")
+
+
+async def test_complete_verify_rejection_still_fails(
+ monkeypatch: pytest.MonkeyPatch, _isolated_download_path: str
+) -> None:
+ """The IllegitCompleteScriptTermination subclass must not be read as an intentional stop."""
+ skyvern_context.set(SkyvernContext(organization_id="o_1", workflow_run_id="wr_1", run_id="wr_1"))
+
+ _fake_storage_app(monkeypatch, save=AsyncMock(), get=AsyncMock(return_value=[]))
+ _wire_block_runtime(monkeypatch)
+ monkeypatch.setattr(
+ CodeBlock,
+ "execute_user_function_with_timeout",
+ AsyncMock(side_effect=IllegitCompleteScriptTermination("verifier rejected")),
+ )
+
+ block = CodeBlock(
+ label="download_invoice",
+ code="value = 'unused'",
+ output_parameter=_output_parameter("code_out"),
+ )
+ result = await block.execute(workflow_run_id="wr_1", workflow_run_block_id="", organization_id="o_1")
+
+ assert result.success is False
+ assert result.status != BlockStatus.terminated
+
+
+def test_swallowed_download_terminal_still_reads_as_download_intent() -> None:
+ """A caught raise hides the failure from the block result but not from the AST predicate."""
+ assert code_is_download_intent(_SWALLOWED_DOWNLOAD_TERMINAL) is True
+
+
+async def test_secure_runner_download_intent_is_counted_too(
+ monkeypatch: pytest.MonkeyPatch, _isolated_download_path: str
+) -> None:
+ """Engine parity: the sidecar path returns before the inline diff, so it emits its own record."""
+ skyvern_context.set(SkyvernContext(organization_id="o_1", workflow_run_id="wr_1", run_id="wr_1"))
+
+ _fake_storage_app(monkeypatch, save=AsyncMock(), get=AsyncMock(return_value=[]))
+ _wire_block_runtime(monkeypatch, workflow=_copilot_workflow())
+ _wire_secure_runner(monkeypatch, output={"downloaded_files": [], "blocker": "no downloadable file"})
+
+ block = CodeBlock(
+ label="download_invoice",
+ code=_SWALLOWED_DOWNLOAD_TERMINAL,
+ output_parameter=_output_parameter("code_out"),
+ )
+ with capture_logs() as logs:
+ result = await block.execute(workflow_run_id="wr_1", workflow_run_block_id="", organization_id="o_1")
+
+ events = _download_intent_events(logs)
+ assert len(events) == 1
+ assert events[0]["engine"] == "secure_runner"
+ assert result.success is True
+
+
+async def test_secure_runner_registered_download_is_not_counted(
+ monkeypatch: pytest.MonkeyPatch, _isolated_download_path: str
+) -> None:
+ skyvern_context.set(SkyvernContext(organization_id="o_1", workflow_run_id="wr_1", run_id="wr_1"))
+
+ _fake_storage_app(monkeypatch, save=AsyncMock(), get=AsyncMock(return_value=[]))
+ _wire_block_runtime(monkeypatch, workflow=_copilot_workflow())
+ _wire_secure_runner(monkeypatch, output={"downloaded_files": [{"filename": "invoice.pdf"}]})
+
+ block = CodeBlock(
+ label="download_invoice",
+ code=_SWALLOWED_DOWNLOAD_TERMINAL,
+ output_parameter=_output_parameter("code_out"),
+ )
+ with capture_logs() as logs:
+ result = await block.execute(workflow_run_id="wr_1", workflow_run_block_id="", organization_id="o_1")
+
+ assert _download_intent_events(logs) == []
+ assert result.success is True
diff --git a/tests/unit/test_copilot_enforcement_hardening.py b/tests/unit/test_copilot_enforcement_hardening.py
index 3fb4b999c..f5d96b5c2 100644
--- a/tests/unit/test_copilot_enforcement_hardening.py
+++ b/tests/unit/test_copilot_enforcement_hardening.py
@@ -491,3 +491,101 @@ def test_reject_count_resets_only_on_non_repeat_without_frontier_unchanged() ->
assert _code_authoring_reject_count_resets(False, True) is False
assert _code_authoring_reject_count_resets(None, False) is False
assert _code_authoring_reject_count_resets(True, False) is False
+
+
+# ---------------------------------------------------------------------------
+# Repair obligation — a turn may not finalize a draft its own build test disproved
+# ---------------------------------------------------------------------------
+
+
+class _FakeFinalResult:
+ """Minimal stand-in for RunResultStreaming carrying one final model response."""
+
+ def __init__(self, response_type: str, user_response: str) -> None:
+ self._payload = json.dumps({"type": response_type, "user_response": user_response})
+
+ @property
+ def final_output(self) -> str:
+ return self._payload
+
+
+def _failed_run_ctx(next_action: Any, *, observed: bool = True) -> Any:
+ ctx = _fresh_context()
+ ctx.test_after_update_done = True
+ ctx.last_test_ok = False
+ ctx.latest_diagnosis_repair_contract = _repair_contract(next_action)
+ if observed:
+ ctx.post_run_page_observation_after_failed_test = True
+ ctx.post_run_page_observation_tool = "inspect_page_for_composition"
+ ctx.post_run_page_observation_workflow_run_id = "wr_x"
+ ctx.last_run_blocks_workflow_run_id = "wr_x"
+ return ctx
+
+
+def test_observed_reply_cannot_finalize_while_repair_is_owed() -> None:
+ """The production shape: failed run -> inspect once -> report the blocker -> turn ends."""
+ from skyvern.forge.sdk.copilot.diagnosis_repair_contract import RepairNextAction
+
+ ctx = _failed_run_ctx(RepairNextAction.REPAIR)
+ result = _FakeFinalResult("REPLY", "The latest bill link opens an email-delivery form.")
+
+ assert _check_enforcement(ctx, result) is not None
+
+
+def test_observed_reply_finalizes_once_repair_is_discharged() -> None:
+ """A contract that no longer asks for repair releases the turn — the obligation is typed."""
+ from skyvern.forge.sdk.copilot.diagnosis_repair_contract import RepairNextAction
+
+ ctx = _failed_run_ctx(RepairNextAction.NO_CHANGE)
+ result = _FakeFinalResult("REPLY", "Downloaded the latest invoice.")
+
+ assert _check_enforcement(ctx, result) is None
+
+
+def test_ask_question_still_finalizes_while_repair_is_owed() -> None:
+ """Needing the user is a legitimate exit; another repair round cannot supply the answer."""
+ from skyvern.forge.sdk.copilot.diagnosis_repair_contract import RepairNextAction
+
+ ctx = _failed_run_ctx(RepairNextAction.REPAIR)
+ ctx.failed_test_nudge_count = 99 # counters exhausted
+ result = _FakeFinalResult("ASK_QUESTION", "Which account should I use?")
+
+ assert _check_enforcement(ctx, result) is None
+
+
+def test_exhausted_nudge_counters_do_not_release_an_open_repair_obligation() -> None:
+ """Counters bound nudge repetition; they were never evidence the failure was addressed."""
+ from skyvern.forge.sdk.copilot.diagnosis_repair_contract import RepairNextAction
+
+ ctx = _failed_run_ctx(RepairNextAction.REPAIR)
+ ctx.failed_test_nudge_count = 99
+ result = _FakeFinalResult("REPLY", "I drafted the workflow; it attempts to download the bill.")
+
+ assert _check_enforcement(ctx, result) == POST_FAILED_TEST_NUDGE
+
+
+def test_stop_decision_releases_the_turn_even_with_counters_exhausted() -> None:
+ """Typed terminal evidence, not a counter, is what ends a repairable failure."""
+ from skyvern.forge.sdk.copilot.diagnosis_repair_contract import RepairNextAction
+
+ ctx = _failed_run_ctx(RepairNextAction.STOP)
+ ctx.failed_test_nudge_count = 99
+ result = _FakeFinalResult("REPLY", "This site requires a mailed statement request.")
+
+ assert _check_enforcement(ctx, result) is None
+
+
+def test_repair_obligation_releases_the_turn_once_its_rounds_are_spent() -> None:
+ """A failure that looks repairable but is not must still be reportable, not re-nudged forever."""
+ from skyvern.forge.sdk.copilot.diagnosis_repair_contract import RepairNextAction
+ from skyvern.forge.sdk.copilot.enforcement import MAX_REPAIR_OBLIGATION_NUDGES
+
+ ctx = _failed_run_ctx(RepairNextAction.REPAIR)
+ ctx.failed_test_nudge_count = 99
+ result = _FakeFinalResult("REPLY", "This site offers no downloadable statement.")
+
+ # Held open while rounds remain.
+ assert _check_enforcement(ctx, result) == POST_FAILED_TEST_NUDGE
+
+ ctx.repair_obligation_nudge_count = MAX_REPAIR_OBLIGATION_NUDGES
+ assert _check_enforcement(ctx, result) is None
diff --git a/tests/unit/test_copilot_hooks.py b/tests/unit/test_copilot_hooks.py
index 55d94c41f..71689c06c 100644
--- a/tests/unit/test_copilot_hooks.py
+++ b/tests/unit/test_copilot_hooks.py
@@ -990,6 +990,7 @@ class TestBrowserInteractionObservationHooks:
pending_scout_ambiguous=None,
pending_scout_reanchor=None,
pending_scout_dynamic_row=None,
+ pending_scout_download_snapshot=None,
discovery_mcp_server=None,
scouted_interactions=[],
scout_trajectory=[],
@@ -1027,6 +1028,7 @@ class TestBrowserInteractionObservationHooks:
pending_scout_ambiguous=None,
pending_scout_reanchor=None,
pending_scout_dynamic_row=None,
+ pending_scout_download_snapshot=None,
discovery_mcp_server=None,
scouted_interactions=[],
scout_trajectory=[],
@@ -1085,6 +1087,7 @@ class TestScoutedInteractionCapture:
pending_scout_ambiguous=None,
pending_scout_reanchor=None,
pending_scout_dynamic_row=None,
+ pending_scout_download_snapshot=None,
discovery_mcp_server=None,
browser_session_id=None,
scouted_interactions=[],
diff --git a/tests/unit/test_copilot_response_coverage_gate.py b/tests/unit/test_copilot_response_coverage_gate.py
index 15505d001..3c6f6360f 100644
--- a/tests/unit/test_copilot_response_coverage_gate.py
+++ b/tests/unit/test_copilot_response_coverage_gate.py
@@ -53,6 +53,7 @@ class _Ctx:
self.composition_page_evidence = None
self.last_update_block_count = None
self.last_test_ok = None
+ self.latest_diagnosis_repair_contract = None
self.last_full_workflow_test_ok = False
self.last_test_failure_reason = None
self.last_test_suspicious_success = False
diff --git a/tests/unit/test_workflow_runtime_completion.py b/tests/unit/test_workflow_runtime_completion.py
new file mode 100644
index 000000000..3034219de
--- /dev/null
+++ b/tests/unit/test_workflow_runtime_completion.py
@@ -0,0 +1,408 @@
+"""Runtime grading of a workflow's declared completion contract.
+
+The contract lives on the workflow version and is graded from execution-layer evidence, so the
+verdict does not depend on which engine ran the blocks or how generated code described its outcome.
+"""
+
+from __future__ import annotations
+
+from types import SimpleNamespace
+from unittest.mock import AsyncMock
+
+import pytest
+
+from skyvern.forge.sdk.workflow import service as service_module
+from skyvern.forge.sdk.workflow.models.workflow import WorkflowRunStatus
+from skyvern.forge.sdk.workflow.runtime_completion import (
+ CompletionCriterion,
+ carried_contract,
+ contract_from_request_criteria,
+ grade_completion_contract,
+ parse_completion_contract,
+ with_contract,
+)
+
+_DOWNLOAD_CONTRACT = {
+ "completion_contract": {
+ "schema_version": 1,
+ "criteria": [{"id": "must_download", "kind": "registered_download", "min_count": 1}],
+ }
+}
+
+
+def test_workflow_without_a_contract_declares_nothing() -> None:
+ assert parse_completion_contract({}) == ()
+ assert parse_completion_contract(None) == ()
+ assert parse_completion_contract({"completion_contract": {"criteria": "nope"}}) == ()
+
+
+def test_download_contract_parses() -> None:
+ (criterion,) = parse_completion_contract(_DOWNLOAD_CONTRACT)
+ assert criterion == CompletionCriterion(id="must_download", kind="registered_download", min_count=1)
+
+
+def test_unknown_kinds_are_dropped_not_failed() -> None:
+ """An older worker must keep running a newer workflow, and must never fail what it cannot grade."""
+ contract = {
+ "completion_contract": {
+ "criteria": [
+ {"id": "future", "kind": "some_future_kind"},
+ {"id": "must_download", "kind": "registered_download"},
+ ]
+ }
+ }
+ parsed = parse_completion_contract(contract)
+ assert [c.id for c in parsed] == ["must_download"]
+
+
+def test_a_run_that_registered_a_file_satisfies_the_contract() -> None:
+ criteria = parse_completion_contract(_DOWNLOAD_CONTRACT)
+ verdict = grade_completion_contract(criteria, registered_download_count=1)
+ assert verdict.satisfied is True
+ assert verdict.unmet_criterion_ids == ()
+
+
+def test_a_run_that_registered_nothing_is_unmet() -> None:
+ """The production shape: the block returned cleanly, the run produced no file."""
+ criteria = parse_completion_contract(_DOWNLOAD_CONTRACT)
+ verdict = grade_completion_contract(criteria, registered_download_count=0)
+ assert verdict.satisfied is False
+ assert verdict.unmet_criterion_ids == ("must_download",)
+ assert verdict.reason
+
+
+def test_no_criteria_grades_as_satisfied() -> None:
+ """Contract-less workflows keep their existing outcome."""
+ assert grade_completion_contract((), registered_download_count=0).satisfied is True
+
+
+def test_min_count_is_honored_and_floored_at_one() -> None:
+ contract = {"completion_contract": {"criteria": [{"kind": "registered_download", "min_count": 2}]}}
+ criteria = parse_completion_contract(contract)
+ assert grade_completion_contract(criteria, registered_download_count=1).satisfied is False
+ assert grade_completion_contract(criteria, registered_download_count=2).satisfied is True
+
+ zero = parse_completion_contract(
+ {"completion_contract": {"criteria": [{"kind": "registered_download", "min_count": 0}]}}
+ )
+ assert zero[0].min_count == 1
+
+
+_DERIVED_CONTRACT = {
+ "schema_version": 1,
+ "criteria": [{"id": "declared_download", "kind": "registered_download", "min_count": 1}],
+}
+
+
+def _wire_finalize(monkeypatch, *, contract, downloaded):
+ """A WorkflowService with just enough wired to exercise the finalize status decision."""
+ from skyvern.forge.sdk.workflow.service import WorkflowService
+
+ service = WorkflowService()
+ run = SimpleNamespace(
+ workflow_run_id="wr_1",
+ workflow_id="w_pinned",
+ workflow_permanent_id="wpid_1",
+ organization_id="o_1",
+ status=WorkflowRunStatus.running,
+ )
+ statuses: list[WorkflowRunStatus] = []
+
+ async def _update(workflow_run_id, status, **kwargs):
+ statuses.append(status)
+ return run
+
+ async def _get_workflow(workflow_id, organization_id=None):
+ definition = dict(_DEFINITION_BASE)
+ if contract is not None:
+ definition["completion_contract"] = contract
+ return SimpleNamespace(workflow_definition=definition)
+
+ monkeypatch.setattr(service, "_update_workflow_run_status_if_not_final", _update)
+ monkeypatch.setattr(service, "get_workflow", _get_workflow)
+ monkeypatch.setattr(
+ service_module.app,
+ "STORAGE",
+ SimpleNamespace(get_downloaded_files=AsyncMock(return_value=list(downloaded))),
+ )
+ return service, run, statuses
+
+
+_DEFINITION_BASE: dict = {"version": 2, "parameters": [], "blocks": []}
+
+
+_DOWNLOAD_CODE = (
+ 'async with page.expect_download() as dl:\n await page.locator("a").click()\nreturn {"downloaded_files": []}'
+)
+
+
+_DOWNLOAD_YAML = """title: Harbor bill
+workflow_definition:
+ version: 2
+ parameters: []
+ blocks:
+ - block_type: code
+ label: download_statement
+ code: |
+ async with page.expect_download(timeout=15000) as dl:
+ await page.locator("#currentBill").click()
+ download = await dl.value
+ return {"downloaded_files": [{"file_name": download.suggested_filename}]}
+"""
+
+_PLAIN_YAML = """title: Plain
+workflow_definition:
+ version: 2
+ parameters: []
+ blocks:
+ - block_type: code
+ label: extract
+ code: |
+ return {"rows": []}
+"""
+
+
+async def test_finalize_terminates_a_run_that_did_not_produce_its_declared_file(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ """The headline behavior: a run whose workflow declares a download and registered none must not
+ finalize as completed."""
+ service, run, statuses = _wire_finalize(monkeypatch, contract=_DERIVED_CONTRACT, downloaded=[])
+
+ await service._finalize_workflow_run_status(
+ workflow_run_id=run.workflow_run_id,
+ workflow_run=run,
+ pre_finally_status=WorkflowRunStatus.running,
+ pre_finally_failure_reason=None,
+ )
+
+ assert statuses == [WorkflowRunStatus.terminated]
+
+
+async def test_finalize_completes_a_run_that_produced_its_declared_file(monkeypatch: pytest.MonkeyPatch) -> None:
+ service, run, statuses = _wire_finalize(monkeypatch, contract=_DERIVED_CONTRACT, downloaded=["invoice.pdf"])
+
+ await service._finalize_workflow_run_status(
+ workflow_run_id=run.workflow_run_id,
+ workflow_run=run,
+ pre_finally_status=WorkflowRunStatus.running,
+ pre_finally_failure_reason=None,
+ )
+
+ assert statuses == [WorkflowRunStatus.completed]
+
+
+async def test_finalize_leaves_a_contract_less_workflow_alone(monkeypatch: pytest.MonkeyPatch) -> None:
+ service, run, statuses = _wire_finalize(monkeypatch, contract=None, downloaded=[])
+
+ await service._finalize_workflow_run_status(
+ workflow_run_id=run.workflow_run_id,
+ workflow_run=run,
+ pre_finally_status=WorkflowRunStatus.running,
+ pre_finally_failure_reason=None,
+ )
+
+ assert statuses == [WorkflowRunStatus.completed]
+
+
+async def test_finalize_grades_the_version_the_run_executed(monkeypatch: pytest.MonkeyPatch) -> None:
+ """An edit mid-run must not judge this run by a contract it never executed."""
+ service, run, statuses = _wire_finalize(monkeypatch, contract=None, downloaded=[])
+ seen: list[str] = []
+
+ async def _get_workflow(workflow_id: str, organization_id: str | None = None):
+ seen.append(workflow_id)
+ return SimpleNamespace(workflow_definition={})
+
+ monkeypatch.setattr(service, "get_workflow", _get_workflow)
+ await service._finalize_workflow_run_status(
+ workflow_run_id=run.workflow_run_id,
+ workflow_run=run,
+ pre_finally_status=WorkflowRunStatus.running,
+ pre_finally_failure_reason=None,
+ )
+
+ assert seen == [run.workflow_id]
+
+
+def test_contract_comes_from_the_request_not_the_code() -> None:
+ """The obligation is what the user asked for, never the shape of the generated code."""
+ from skyvern.forge.sdk.copilot.completion_verification import registered_download_completion_criterion
+ from skyvern.forge.sdk.copilot.request_policy import CompletionCriterion as RequestCriterion
+
+ assert contract_from_request_criteria([registered_download_completion_criterion()]) is not None
+ assert contract_from_request_criteria([RequestCriterion(id="c0", outcome="something else")]) is None
+ assert contract_from_request_criteria([]) is None
+ assert contract_from_request_criteria(None) is None
+
+
+def test_requested_contract_round_trips_through_the_parser() -> None:
+ from skyvern.forge.sdk.copilot.completion_verification import registered_download_completion_criterion
+
+ contract = contract_from_request_criteria([registered_download_completion_criterion()])
+ criteria = parse_completion_contract({"completion_contract": contract})
+ assert [c.kind for c in criteria] == ["registered_download"]
+ assert grade_completion_contract(criteria, registered_download_count=0).satisfied is False
+ assert grade_completion_contract(criteria, registered_download_count=1).satisfied is True
+
+
+async def test_finalize_skips_grading_a_partial_run(monkeypatch: pytest.MonkeyPatch) -> None:
+ """A frontier run of a block subset was never asked to produce the whole deliverable."""
+ service, run, statuses = _wire_finalize(monkeypatch, contract=_DERIVED_CONTRACT, downloaded=[])
+
+ await service._finalize_workflow_run_status(
+ workflow_run_id=run.workflow_run_id,
+ workflow_run=run,
+ pre_finally_status=WorkflowRunStatus.running,
+ pre_finally_failure_reason=None,
+ is_partial_run=True,
+ )
+
+ assert statuses == [WorkflowRunStatus.completed]
+
+
+def test_a_stored_contract_survives_a_write_that_does_not_carry_one() -> None:
+ """Non-copilot save paths rebuild the definition through models that omit the field."""
+ stored = {"completion_contract": _DERIVED_CONTRACT, "blocks": []}
+ rebuilt = with_contract({"blocks": []}, carried_contract(stored))
+ assert rebuilt["completion_contract"] == _DERIVED_CONTRACT
+
+
+def test_an_incoming_contract_is_not_overwritten_by_the_carried_one() -> None:
+ incoming = {"completion_contract": {"schema_version": 1, "criteria": []}, "blocks": []}
+ rebuilt = with_contract(dict(incoming), carried_contract({"completion_contract": _DERIVED_CONTRACT}))
+ assert rebuilt["completion_contract"] == incoming["completion_contract"]
+
+
+def test_no_stored_contract_leaves_the_definition_untouched() -> None:
+ assert "completion_contract" not in with_contract({"blocks": []}, carried_contract({"blocks": []}))
+
+
+async def test_finalize_counts_session_scoped_downloads_not_yet_claimed(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ """Session downloads are tagged with the run id during cleanup, after this grade runs."""
+ service, run, statuses = _wire_finalize(monkeypatch, contract=_DERIVED_CONTRACT, downloaded=[])
+ monkeypatch.setattr(service, "_session_download_count", AsyncMock(return_value=1))
+
+ await service._finalize_workflow_run_status(
+ workflow_run_id=run.workflow_run_id,
+ workflow_run=run,
+ pre_finally_status=WorkflowRunStatus.running,
+ pre_finally_failure_reason=None,
+ )
+
+ assert statuses == [WorkflowRunStatus.completed]
+
+
+async def test_both_acceptance_paths_attach_the_same_contract(monkeypatch: pytest.MonkeyPatch) -> None:
+ """The obligation must not depend on whether the user clicked Accept or had auto-accept on."""
+ from skyvern.forge.sdk.copilot.completion_verification import registered_download_completion_criterion
+ from skyvern.forge.sdk.routes import workflow_copilot as route
+
+ snapshot = SimpleNamespace(active=SimpleNamespace(criteria=[registered_download_completion_criterion()]))
+ monkeypatch.setattr(route, "_load_completion_criteria_snapshot", AsyncMock(return_value=snapshot))
+ chat = SimpleNamespace(workflow_copilot_chat_id="wcc_1", workflow_permanent_id="wpid_1")
+
+ for target in (
+ SimpleNamespace(workflow_definition=SimpleNamespace(completion_contract=None)), # manual accept
+ SimpleNamespace(workflow_definition=SimpleNamespace(completion_contract=None)), # auto accept
+ ):
+ await route._attach_requested_completion_contract(chat, target)
+ assert target.workflow_definition.completion_contract["criteria"][0]["kind"] == "registered_download"
+
+
+async def test_no_active_criteria_attaches_nothing(monkeypatch: pytest.MonkeyPatch) -> None:
+ from skyvern.forge.sdk.routes import workflow_copilot as route
+
+ monkeypatch.setattr(
+ route, "_load_completion_criteria_snapshot", AsyncMock(return_value=SimpleNamespace(active=None))
+ )
+ target = SimpleNamespace(workflow_definition=SimpleNamespace(completion_contract=None))
+ await route._attach_requested_completion_contract(
+ SimpleNamespace(workflow_copilot_chat_id="c", workflow_permanent_id="w"), target
+ )
+ assert target.workflow_definition.completion_contract is None
+
+
+def test_apply_proposed_workflow_route_is_bound_to_the_route_handler() -> None:
+ """A helper inserted between the decorator and its function silently rebinds the endpoint to the
+ helper, and the route then 422s on the helper's arguments."""
+ from skyvern.forge.sdk.routes.workflow_copilot import base_router
+
+ routes = [r for r in base_router.routes if getattr(r, "path", "") == "/workflow/copilot/apply-proposed-workflow"]
+ assert routes, "route not registered"
+ assert routes[0].endpoint.__name__ == "workflow_copilot_apply_proposed_workflow"
+
+
+def test_a_request_criterion_is_recognized_by_its_typed_deliverable_fields() -> None:
+ """The persisted criterion carries deliverable_kind/output_path; the synthetic id is the
+ copilot's separate internal marker, and keying on it alone misses every real request."""
+ requested = SimpleNamespace(
+ id="c0",
+ outcome="the current electricity statement is downloaded as a PDF",
+ deliverable_kind="registered_download",
+ declared_deliverable_kind="registered_download",
+ output_path="output.downloaded_files",
+ )
+ unrelated = SimpleNamespace(id="c1", outcome="a summary", deliverable_kind=None, output_path=None)
+
+ assert contract_from_request_criteria([unrelated, requested]) is not None
+ assert contract_from_request_criteria([unrelated]) is None
+
+
+def test_output_path_alone_identifies_a_requested_download() -> None:
+ by_path = SimpleNamespace(id="c0", deliverable_kind=None, output_path="output.downloaded_files")
+ assert contract_from_request_criteria([by_path]) is not None
+
+
+async def test_auto_accept_uses_this_turns_criteria_not_the_stored_snapshot(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ """Auto-accept commits before the turn's criteria are persisted, so a turn that first mints a
+ download criterion must still carry the contract."""
+ from skyvern.forge.sdk.copilot.completion_verification import registered_download_completion_criterion
+ from skyvern.forge.sdk.routes import workflow_copilot as route
+
+ # The stored snapshot is pre-turn and has nothing; this turn's state holds the new criterion.
+ monkeypatch.setattr(
+ route, "_load_completion_criteria_snapshot", AsyncMock(return_value=SimpleNamespace(active=None))
+ )
+ agent_result = SimpleNamespace(
+ completion_criteria_turn_state=SimpleNamespace(
+ decision=SimpleNamespace(criteria=(registered_download_completion_criterion(),))
+ )
+ )
+ target = SimpleNamespace(workflow_definition=SimpleNamespace(completion_contract=None))
+
+ await route._attach_requested_completion_contract(
+ SimpleNamespace(workflow_copilot_chat_id="c", workflow_permanent_id="w"), target, agent_result
+ )
+
+ assert target.workflow_definition.completion_contract is not None
+
+
+async def test_manual_accept_still_falls_back_to_the_stored_snapshot(monkeypatch: pytest.MonkeyPatch) -> None:
+ """Manual accept runs in a later request with no turn state, so the persisted set is the source."""
+ from skyvern.forge.sdk.copilot.completion_verification import registered_download_completion_criterion
+ from skyvern.forge.sdk.routes import workflow_copilot as route
+
+ snapshot = SimpleNamespace(active=SimpleNamespace(criteria=[registered_download_completion_criterion()]))
+ monkeypatch.setattr(route, "_load_completion_criteria_snapshot", AsyncMock(return_value=snapshot))
+ target = SimpleNamespace(workflow_definition=SimpleNamespace(completion_contract=None))
+
+ await route._attach_requested_completion_contract(
+ SimpleNamespace(workflow_copilot_chat_id="c", workflow_permanent_id="w"), target
+ )
+
+ assert target.workflow_definition.completion_contract is not None
diff --git a/tests/unit/workflow/test_code_block_escalation.py b/tests/unit/workflow/test_code_block_escalation.py
index a0e7915e9..56fc255cb 100644
--- a/tests/unit/workflow/test_code_block_escalation.py
+++ b/tests/unit/workflow/test_code_block_escalation.py
@@ -1274,6 +1274,7 @@ async def test_escalation_task_verifies_with_action_history(
("error_code", "healable", "skip_reason"),
[
("unsupported_page_operation", True, None),
+ ("browser_operation_failed", True, None),
("timeout", False, HealSkipReason.timeout_class),
("insecure_code_detected", False, HealSkipReason.insecure_code),
("browser_disconnected", False, HealSkipReason.unclassifiable),