Skip to content

Automate browser based workflows with AI

SKY-12588: withhold block runs while declared workflow inputs are wired to nothing (stacked on #13549) (#7565)

Andrew Neilsonda5afc74f553
diff --git a/skyvern/forge/sdk/copilot/tools/__init__.py b/skyvern/forge/sdk/copilot/tools/__init__.py
index 676d22ff2..3b750f9d3 100644
--- a/skyvern/forge/sdk/copilot/tools/__init__.py
+++ b/skyvern/forge/sdk/copilot/tools/__init__.py
@@ -253,6 +253,7 @@ from .workflow_update import _metadata_contract_run_preflight_reject as _metadat
from .workflow_update import _normalize_code_artifact_metadata as _normalize_code_artifact_metadata
from .workflow_update import _record_workflow_proxy_location_span as _record_workflow_proxy_location_span
from .workflow_update import _record_workflow_update_result as _record_workflow_update_result
+from .workflow_update import _run_dispatch_definition_reject as _run_dispatch_definition_reject
from .workflow_update import _scaffold_metadata_contract_for_update as _scaffold_metadata_contract_for_update
from .workflow_update import _update_workflow as _update_workflow
from .workflow_update import (
@@ -456,6 +457,21 @@ async def run_blocks_tool(
if loop_error:
return _diagnosis_repair_tool_error(copilot_ctx, "run_blocks_and_collect_debug", loop_error)
+ definition_reject_result = _run_dispatch_definition_reject(copilot_ctx, copilot_ctx.workflow_yaml or "")
+ if definition_reject_result is not None:
+ record_tool_step_result_for_ctx(
+ copilot_ctx,
+ "run_blocks_and_collect_debug",
+ arguments,
+ definition_reject_result,
+ )
+ _record_diagnosis_repair_contract(
+ copilot_ctx,
+ source_tool="run_blocks_and_collect_debug",
+ result=definition_reject_result,
+ )
+ return json.dumps(definition_reject_result)
+
prior_definition = await _get_prior_workflow_definition(copilot_ctx)
labels_to_execute, block_outputs_to_seed, frontier_start_label = _plan_frontier(
copilot_ctx, block_labels, prior_definition, prior_definition
diff --git a/skyvern/forge/sdk/copilot/tools/workflow_update.py b/skyvern/forge/sdk/copilot/tools/workflow_update.py
index c001da09a..e0be04a71 100644
--- a/skyvern/forge/sdk/copilot/tools/workflow_update.py
+++ b/skyvern/forge/sdk/copilot/tools/workflow_update.py
@@ -6228,6 +6228,49 @@ def _metadata_preflight_reject_yields_to_ladder(ctx: AgentContext) -> bool:
)
+def _run_dispatch_definition_reject(ctx: AgentContext, workflow_yaml: str) -> dict[str, Any] | None:
+ # A keyless draft may persist (draft-first authoring), but a run must never dispatch
+ # while declared string keys are unreferenced — mirrors the update_and_run gate.
+ if not workflow_yaml:
+ return None
+ definition_reject = _definition_plane_preflight_reject(
+ ctx,
+ workflow_yaml,
+ enforce_untagged_declared_inputs=True,
+ )
+ if definition_reject is None:
+ return None
+ halted = _stash_unresolved_recorded_outcome_grounding_halt(
+ ctx,
+ definition_reject.unreferenced_parameter_keys,
+ )
+ if not halted:
+ if _record_definition_plane_ablation_event(
+ ctx,
+ workflow_yaml,
+ definition_reject,
+ code_artifact_metadata=ctx.code_artifact_metadata,
+ ):
+ return None
+ _record_definition_plane_reject(
+ ctx,
+ workflow_yaml,
+ definition_reject,
+ code_artifact_metadata=ctx.code_artifact_metadata,
+ )
+ return {
+ "ok": False,
+ "error": _definition_plane_reject_error(definition_reject),
+ "user_facing_summary": _compiled_authoring_user_summary(),
+ "data": {
+ "reason_code": "definition_contract_unsatisfied",
+ "definition_criterion_ids": list(definition_reject.criterion_ids),
+ "definition_reason_codes": list(definition_reject.reason_codes),
+ "unreferenced_parameter_keys": list(definition_reject.unreferenced_parameter_keys),
+ },
+ }
+
+
def _metadata_contract_run_preflight_reject(
ctx: AgentContext,
workflow_yaml: str,
diff --git a/tests/unit/test_copilot_code_block_persist_seam.py b/tests/unit/test_copilot_code_block_persist_seam.py
index d36361a49..4a54b4d19 100644
--- a/tests/unit/test_copilot_code_block_persist_seam.py
+++ b/tests/unit/test_copilot_code_block_persist_seam.py
@@ -1744,6 +1744,153 @@ class TestCodeRepairProgressClassification:
assert directive is not None
assert directive.parameter_binding_directive is not None
+ @pytest.mark.asyncio
+ async def test_keyless_draft_persists_but_run_blocks_is_rejected_when_criteria_dark(
+ self, monkeypatch: pytest.MonkeyPatch
+ ) -> None:
+ _stub_successful_update(monkeypatch)
+ ctx = _code_only_ctx()
+ ctx.request_policy = RequestPolicy(allow_run_blocks=True)
+ keyless_yaml = _yaml(
+ """
+ title: Download monthly statement
+ workflow_definition:
+ parameters:
+ - {parameter_type: workflow, key: account_number, workflow_parameter_type: string}
+ - {parameter_type: workflow, key: billing_start_date, workflow_parameter_type: string}
+ - {parameter_type: workflow, key: billing_end_date, workflow_parameter_type: string}
+ blocks:
+ - block_type: text_prompt
+ label: normalize_invoice_inputs
+ prompt: "Return these inputs unchanged: account number 100245, dates 2026-05-01 to 2026-05-31."
+ - block_type: code
+ label: fetch_invoice_pdf
+ parameter_keys: []
+ code: |
+ await page.locator("#current-statement-row").click()
+ await page.locator('a[href="/statements/100245_2026-05.pdf"]').click()
+ """
+ )
+
+ persist_result = await _update_workflow(
+ {"workflow_yaml": keyless_yaml},
+ ctx,
+ allow_missing_credentials=True,
+ )
+ assert persist_result["ok"] is True
+
+ dispatched: list[object] = []
+
+ async def _run(_params: object, _ctx: object, **_kwargs: object) -> dict[str, object]:
+ dispatched.append(_params)
+ return {"ok": True}
+
+ monkeypatch.setattr(tools_module, "_run_blocks_and_collect_debug", _run)
+ monkeypatch.setattr(tools_module, "_authority_tool_error", lambda *_args, **_kwargs: None)
+ run_result = json.loads(
+ await tools_module.run_blocks_tool.on_invoke_tool(
+ SimpleNamespace(context=ctx, tool_name="run_blocks_and_collect_debug"),
+ json.dumps({"block_labels": ["fetch_invoice_pdf"]}),
+ )
+ )
+
+ assert run_result["ok"] is False
+ assert run_result["data"]["reason_code"] == "definition_contract_unsatisfied"
+ assert run_result["data"]["unreferenced_parameter_keys"] == [
+ "account_number",
+ "billing_end_date",
+ "billing_start_date",
+ ]
+ assert dispatched == []
+
+ @pytest.mark.asyncio
+ async def test_keyless_run_blocks_dispatches_in_log_only_shadow_mode(self, monkeypatch: pytest.MonkeyPatch) -> None:
+ _stub_successful_update(monkeypatch)
+ ctx = _code_only_ctx()
+ ctx.request_policy = RequestPolicy(allow_run_blocks=True)
+ keyless_yaml = _yaml(
+ """
+ title: Download monthly statement
+ workflow_definition:
+ parameters:
+ - {parameter_type: workflow, key: account_number, workflow_parameter_type: string}
+ blocks:
+ - block_type: code
+ label: fetch_invoice_pdf
+ parameter_keys: []
+ code: |
+ await page.locator("#current-statement-row").click()
+ """
+ )
+ persist_result = await _update_workflow(
+ {"workflow_yaml": keyless_yaml},
+ ctx,
+ allow_missing_credentials=True,
+ )
+ assert persist_result["ok"] is True
+
+ ctx.author_time_gate_log_only_ids = {workflow_update_module.DEFINITION_CONTRACT_UNSATISFIED_GATE_ID}
+ dispatched: list[object] = []
+
+ async def _run(_params: object, _ctx: object, **_kwargs: object) -> dict[str, object]:
+ dispatched.append(_params)
+ return {"ok": True}
+
+ monkeypatch.setattr(tools_module, "_run_blocks_and_collect_debug", _run)
+ monkeypatch.setattr(tools_module, "_authority_tool_error", lambda *_args, **_kwargs: None)
+ await tools_module.run_blocks_tool.on_invoke_tool(
+ SimpleNamespace(context=ctx, tool_name="run_blocks_and_collect_debug"),
+ json.dumps({"block_labels": ["fetch_invoice_pdf"]}),
+ )
+
+ assert dispatched != []
+ assert [event.gate_id for event in ctx.author_time_gate_ablation_events] == [
+ workflow_update_module.DEFINITION_CONTRACT_UNSATISFIED_GATE_ID
+ ]
+
+ @pytest.mark.asyncio
+ async def test_bound_draft_run_blocks_dispatches_when_criteria_dark(self, monkeypatch: pytest.MonkeyPatch) -> None:
+ _stub_successful_update(monkeypatch)
+ ctx = _code_only_ctx()
+ ctx.request_policy = RequestPolicy(allow_run_blocks=True)
+ bound_yaml = _yaml(
+ """
+ title: Download monthly statement
+ workflow_definition:
+ parameters:
+ - {parameter_type: workflow, key: account_number, workflow_parameter_type: string}
+ blocks:
+ - block_type: code
+ label: fetch_invoice_pdf
+ parameter_keys: [account_number]
+ code: |
+ await page.locator("#account").fill(str(account_number))
+ await page.locator("#current-statement-row").click()
+ """
+ )
+
+ persist_result = await _update_workflow(
+ {"workflow_yaml": bound_yaml},
+ ctx,
+ allow_missing_credentials=True,
+ )
+ assert persist_result["ok"] is True
+
+ dispatched: list[object] = []
+
+ async def _run(_params: object, _ctx: object, **_kwargs: object) -> dict[str, object]:
+ dispatched.append(_params)
+ return {"ok": True, "run_status": "completed"}
+
+ monkeypatch.setattr(tools_module, "_run_blocks_and_collect_debug", _run)
+ monkeypatch.setattr(tools_module, "_authority_tool_error", lambda *_args, **_kwargs: None)
+ await tools_module.run_blocks_tool.on_invoke_tool(
+ SimpleNamespace(context=ctx, tool_name="run_blocks_and_collect_debug"),
+ json.dumps({"block_labels": ["fetch_invoice_pdf"]}),
+ )
+
+ assert dispatched != []
+
@pytest.mark.asyncio
async def test_recut_ambiguous_field_binding_returns_value_free_typed_directive(
self, monkeypatch: pytest.MonkeyPatch