Skip to content

Automate browser based workflows with AI

Scope the Copilot sandbox gate to code-bearing drafts and make the refusal terminal (#8005)

Aaron Pereza82b1ab6989e
diff --git a/skyvern/forge/sdk/copilot/agent.py b/skyvern/forge/sdk/copilot/agent.py
index a7fa24b93..c0fe03842 100644
--- a/skyvern/forge/sdk/copilot/agent.py
+++ b/skyvern/forge/sdk/copilot/agent.py
@@ -1760,6 +1760,14 @@ def _rewrite_failed_test_response(
block_word = "block" if positive_block_count == 1 else "blocks"
draft_phrase = f"a draft workflow with {positive_block_count} {block_word}"
+ # No run row means nothing executed, so claiming the draft was tested is false.
+ if ctx.last_failure_category_top == "UNRECOVERABLE_TOOL_ERROR" and ctx.last_run_blocks_workflow_run_id is None:
+ return (
+ f"I created {draft_phrase}, but I couldn't start a test run: "
+ f"{_normalize_failure_reason(ctx.last_test_failure_reason)}. "
+ f"Nothing was executed, so the draft is unverified.{keep_draft_affordance}"
+ )
+
failure_summary = _normalize_failure_reason(ctx.last_test_failure_reason)
follow_up = _FAILURE_FOLLOW_UP.get(ctx.last_failure_category_top or "", "")
return (
@@ -3041,7 +3049,14 @@ def _recorded_failure_reply(
test_failed = ctx.last_test_ok is False or run_status == "failed"
unrecoverable_stop = next_action == "stop" or failure_type == "unrecoverable_tool_error"
+ # test_after_update_done is stamped for the run tools regardless of ok, so
+ # test_attempted alone cannot distinguish "ran and failed" from "never started".
+ # Only the run id proves a row existed: pre-run refusals can carry a synthetic
+ # overall_status="failed" even though execution never started.
+ run_created = bool(getattr(diagnosis_input, "workflow_run_id", None))
if getattr(ctx, "last_workflow", None) is not None:
+ if unrecoverable_stop and not run_created:
+ return f"I built {block_phrase}, but I couldn't start a test run: {reason}.{status_sentence}"
if test_attempted and test_failed and not unrecoverable_stop:
return f"I built {block_phrase} and tested it, but the test failed: {reason}.{status_sentence}"
if test_attempted:
diff --git a/skyvern/forge/sdk/copilot/tools/run_execution.py b/skyvern/forge/sdk/copilot/tools/run_execution.py
index 441713309..08f231d7e 100644
--- a/skyvern/forge/sdk/copilot/tools/run_execution.py
+++ b/skyvern/forge/sdk/copilot/tools/run_execution.py
@@ -233,6 +233,20 @@ RUN_BLOCKS_POLL_INTERVAL_SECONDS = 5.0
COPILOT_SANDBOX_UNAVAILABLE_ERROR = "Sandboxed worker is unavailable; execution was not started."
+# Block types that can reach exec() in the API process, so a run containing one may
+# only proceed on the sandboxed worker. CODE compiles user code directly;
+# WORKFLOW_TRIGGER runs its child in-process with block_labels=None, which both
+# executes the child's own code blocks and re-opens the cached-script import path;
+# TaskV2 synthesizes a code block at runtime from planner output. The latter two
+# cannot be proven code-free by inspecting the draft, so they stay fail-closed.
+_SANDBOX_REQUIRED_BLOCK_TYPES = frozenset(
+ {
+ BlockType.CODE.value,
+ BlockType.WORKFLOW_TRIGGER.value,
+ BlockType.TaskV2.value,
+ }
+)
+
# Detached cleanup tasks held here so the garbage collector does not drop them
# while they still have work to do, and so the "task exception was never
# retrieved" warning cannot fire — each task adds a done-callback that logs
@@ -424,7 +438,27 @@ def _maybe_clear_reconciliation_flag(copilot_ctx: Any, result: Any) -> None:
def _copilot_sandbox_unavailable_result() -> dict[str, Any]:
- return {"ok": False, "error": COPILOT_SANDBOX_UNAVAILABLE_ERROR}
+ # No repair can make the sandbox reachable, so the result carries
+ # UNRECOVERABLE_TOOL_ERROR to reach the contract's STOP lane. Without it the
+ # refusal classifies as a generic FAILED_RUN and the enforcement loop nudges
+ # the model to retry a run that never started.
+ return {
+ "ok": False,
+ "error": COPILOT_SANDBOX_UNAVAILABLE_ERROR,
+ "data": {
+ "workflow_run_id": None,
+ "overall_status": "failed",
+ "failure_reason": COPILOT_SANDBOX_UNAVAILABLE_ERROR,
+ "blocks": [],
+ "failure_categories": [
+ {
+ "category": "UNRECOVERABLE_TOOL_ERROR",
+ "confidence_float": 1.0,
+ "reasoning": "Sandboxed execution was unavailable; no workflow run was created.",
+ }
+ ],
+ },
+ }
def _mark_pending_reconciliation_run(copilot_ctx: Any, workflow_run_id: str) -> None:
@@ -1004,6 +1038,33 @@ def _selected_code_security_inputs(
return code_blocks
+def _selected_blocks_require_sandbox(
+ blocks: list[Any],
+ *,
+ selected_labels: set[str],
+ include_descendants: bool = False,
+) -> bool:
+ for block in blocks:
+ if isinstance(block, Mapping):
+ label = str(block.get("label") or "")
+ block_type = str(block.get("block_type") or "").lower()
+ children = _mapping_child_blocks(block)
+ else:
+ label = str(getattr(block, "label", "") or "")
+ block_type = str(getattr(block, "block_type", "") or "").lower()
+ children = _typed_child_blocks(block)
+ selected = include_descendants or label in selected_labels
+ if selected and block_type in _SANDBOX_REQUIRED_BLOCK_TYPES:
+ return True
+ if children and _selected_blocks_require_sandbox(
+ children,
+ selected_labels=selected_labels,
+ include_descendants=selected,
+ ):
+ return True
+ return False
+
+
def _mapping_child_blocks(block: Mapping[str, Any]) -> list[Any]:
children: list[Any] = []
for key in ("loop_blocks", "blocks"):
@@ -1606,16 +1667,38 @@ async def _run_blocks_and_collect_debug(
if not workflow.get_output_parameter(label):
return {"ok": False, "error": f"Block label not found in saved workflow: {label!r}"}
+ workflow_definition = workflow.workflow_definition
+ finally_block_label = (
+ workflow_definition.get("finally_block_label")
+ if isinstance(workflow_definition, Mapping)
+ else getattr(workflow_definition, "finally_block_label", None)
+ )
+ labels_that_may_execute = list(labels_to_execute)
+ if (
+ isinstance(finally_block_label, str)
+ and finally_block_label
+ and finally_block_label not in labels_that_may_execute
+ ):
+ # WorkflowService executes this top-level block after every non-canceled
+ # body, independently of the partial-run block whitelist. Admission,
+ # runtime code security, and credential replay checks must all see it.
+ labels_that_may_execute.append(finally_block_label)
+
runtime_security_failure = _runtime_code_security_failure_for_selected_labels(
workflow,
block_labels=list(block_labels),
- labels_to_execute=labels_to_execute,
+ labels_to_execute=labels_that_may_execute,
frontier_start_label=frontier_start_label,
)
if runtime_security_failure is not None:
ctx.last_executed_block_labels = []
return runtime_security_failure
+ requires_sandbox = _selected_blocks_require_sandbox(
+ _workflow_definition_blocks_for_code_security(workflow_definition),
+ selected_labels=set(labels_that_may_execute),
+ )
+
credential_ids = list(
dict.fromkeys(
_extract_credential_ids_from_tool_value(params.get("parameters") or {})
@@ -1650,7 +1733,10 @@ async def _run_blocks_and_collect_debug(
)
# Compared against the literal True so anything other than an explicit opt-in — including a
# test double that auto-mocks the hook into a truthy object — still fails closed.
- if not dispatch_to_worker and app.AGENT_FUNCTION.allow_copilot_inline_code_execution() is not True:
+ allow_inline_code_execution = (
+ app.AGENT_FUNCTION.allow_copilot_inline_code_execution() is True if not dispatch_to_worker else False
+ )
+ if requires_sandbox and not dispatch_to_worker and not allow_inline_code_execution:
return _copilot_sandbox_unavailable_result()
runtime_workflow = _workflow_with_runtime_block_goal_context(workflow, ctx)
@@ -1702,7 +1788,7 @@ async def _run_blocks_and_collect_debug(
},
}
- use_fresh_session = _should_use_fresh_session_for_login_first_replay(ctx, labels_to_execute, workflow)
+ use_fresh_session = _should_use_fresh_session_for_login_first_replay(ctx, labels_that_may_execute, workflow)
# True when the run was threaded into a fresh session rather than the scout's debug session;
# gates the post-run rebind (~:1135) so the ephemeral run session is not adopted as the
# context session.
@@ -1858,14 +1944,15 @@ async def _run_blocks_and_collect_debug(
block_outputs=block_outputs_to_seed or None,
)
else:
- LOG.error(
- "UNSANDBOXED: executing copilot workflow code in the API process because "
- "COPILOT_ALLOW_INLINE_CODE_EXECUTION is enabled. This is a local-development path "
- "with no sandbox isolation; the run below is NOT a sandboxed run.",
- workflow_run_id=workflow_run.workflow_run_id,
- workflow_permanent_id=ctx.workflow_permanent_id,
- organization_id=ctx.organization_id,
- )
+ if allow_inline_code_execution:
+ LOG.error(
+ "UNSANDBOXED: executing copilot workflow code in the API process because "
+ "COPILOT_ALLOW_INLINE_CODE_EXECUTION is enabled. This is a local-development path "
+ "with no sandbox isolation; the run below is NOT a sandboxed run.",
+ workflow_run_id=workflow_run.workflow_run_id,
+ workflow_permanent_id=ctx.workflow_permanent_id,
+ organization_id=ctx.organization_id,
+ )
# prepare_workflow replaced the ambient context with this run's own, so the marker is
# scoped to this run and is inherited by the execution task created below.
inline_run_context = skyvern_context.current()
diff --git a/skyvern/forge/sdk/workflow/service.py b/skyvern/forge/sdk/workflow/service.py
index 143b1582c..e1bc75413 100644
--- a/skyvern/forge/sdk/workflow/service.py
+++ b/skyvern/forge/sdk/workflow/service.py
@@ -4091,9 +4091,12 @@ class WorkflowService:
loaded_script_module = None
# If no cached script exists, check if a static pre-built script
- # should be created for this platform (e.g., ATS). This persists the
+ # should be created for this platform (e.g., ATS). This persists the
# script to DB (pinned) on first run so it shows in the Code tab.
- if is_script_run and not script_blocks_by_label:
+ # Partial runs must stay agent-only, matching get_workflow_script's
+ # existing contract: bootstrapping here would re-open run_signature
+ # compile/exec after that lookup deliberately returned no script.
+ if is_script_run and not block_labels and not script_blocks_by_label:
try:
static_result = await app.AGENT_FUNCTION.ensure_static_script(
workflow=workflow,
diff --git a/tests/unit/test_copilot_timeout_wip.py b/tests/unit/test_copilot_timeout_wip.py
index 7a1aefbcd..0a47afb92 100644
--- a/tests/unit/test_copilot_timeout_wip.py
+++ b/tests/unit/test_copilot_timeout_wip.py
@@ -61,9 +61,18 @@ def _ctx(
return ctx
-def _blocker_contract(reason: str, *, run_status: str | None = "running") -> DiagnosisRepairContract:
+def _blocker_contract(
+ reason: str,
+ *,
+ run_status: str | None = "running",
+ workflow_run_id: str | None = None,
+) -> DiagnosisRepairContract:
return DiagnosisRepairContract(
- diagnosis_input=DiagnosisInput(source_tool="get_browser_screenshot", run_status=run_status),
+ diagnosis_input=DiagnosisInput(
+ source_tool="get_browser_screenshot",
+ workflow_run_id=workflow_run_id,
+ run_status=run_status,
+ ),
diagnosis_result=DiagnosisResult(root_cause_summary=reason, confidence=0.9),
repair_decision=RepairDecision(next_action="stop"),
verification_result=VerificationResult(
@@ -438,7 +447,10 @@ class TestBuildUnexpectedErrorExitResult:
wf = MagicMock(name="wf")
ctx = _ctx(last_workflow=wf, last_workflow_yaml="version: '1.0'", last_test_ok=False)
ctx.last_update_block_count = 3
- ctx.latest_diagnosis_repair_contract = _blocker_contract("Browser session was no longer reachable.")
+ ctx.latest_diagnosis_repair_contract = _blocker_contract(
+ "Browser session was no longer reachable.",
+ workflow_run_id="wr_test",
+ )
result = _build_unexpected_error_exit_result(ctx, global_llm_context=None)
@@ -474,6 +486,7 @@ class TestBuildUnexpectedErrorExitResult:
ctx.latest_diagnosis_repair_contract = _blocker_contract(
"The browser session disappeared before screenshot verification could complete.",
run_status="aborted",
+ workflow_run_id="wr_test",
)
result = _build_unexpected_error_exit_result(ctx, global_llm_context=None)
@@ -500,12 +513,32 @@ class TestBuildUnexpectedErrorExitResult:
assert result.updated_workflow is wf
assert result.proposal_disposition == "review_untested"
assert result.user_response == (
- "I built a 2-block draft, but I couldn't verify it: "
+ "I built a 2-block draft, but I couldn't start a test run: "
"Browser session not found while reading https://example.test. Last run status: aborted."
)
assert "pbs_" not in result.user_response
assert "token=secret" not in result.user_response
+ def test_sandbox_refusal_does_not_claim_a_test_ran(self) -> None:
+ """No run row means nothing executed, so the reply must not say it tested the draft."""
+ from skyvern.forge.sdk.copilot.diagnosis_repair_contract import build_diagnosis_repair_contract
+ from skyvern.forge.sdk.copilot.tools.run_execution import _copilot_sandbox_unavailable_result
+
+ wf = MagicMock(name="wf")
+ ctx = _ctx(last_workflow=wf, last_workflow_yaml="version: '1.0'", last_test_ok=False)
+ ctx.test_after_update_done = True
+ ctx.last_update_block_count = 6
+ ctx.latest_diagnosis_repair_contract = build_diagnosis_repair_contract(
+ source_tool="run_blocks_and_collect_debug",
+ result=_copilot_sandbox_unavailable_result(),
+ ctx=ctx,
+ )
+
+ result = _build_unexpected_error_exit_result(ctx, global_llm_context=None)
+
+ assert "tested it" not in result.user_response
+ assert "I couldn't start a test run" in result.user_response
+
def test_suspicious_success_drops_proposal(self) -> None:
wf = MagicMock(name="wf")
ctx = _ctx(