Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
Scope the Copilot sandbox gate to code-bearing drafts and make the refusal terminal (#8005)
Aaron Pereza82b1ab6989e
diff --git a/skyvern/forge/sdk/copilot/agent.py b/skyvern/forge/sdk/copilot/agent.pyindex a7fa24b93..c0fe03842 100644--- a/skyvern/forge/sdk/copilot/agent.py+++ b/skyvern/forge/sdk/copilot/agent.py@@ -1760,6 +1760,14 @@ def _rewrite_failed_test_response(block_word = "block" if positive_block_count == 1 else "blocks"draft_phrase = f"a draft workflow with {positive_block_count} {block_word}"+ # No run row means nothing executed, so claiming the draft was tested is false.+ if ctx.last_failure_category_top == "UNRECOVERABLE_TOOL_ERROR" and ctx.last_run_blocks_workflow_run_id is None:+ return (+ f"I created {draft_phrase}, but I couldn't start a test run: "+ f"{_normalize_failure_reason(ctx.last_test_failure_reason)}. "+ f"Nothing was executed, so the draft is unverified.{keep_draft_affordance}"+ )+failure_summary = _normalize_failure_reason(ctx.last_test_failure_reason)follow_up = _FAILURE_FOLLOW_UP.get(ctx.last_failure_category_top or "", "")return (@@ -3041,7 +3049,14 @@ def _recorded_failure_reply(test_failed = ctx.last_test_ok is False or run_status == "failed"unrecoverable_stop = next_action == "stop" or failure_type == "unrecoverable_tool_error"+ # test_after_update_done is stamped for the run tools regardless of ok, so+ # test_attempted alone cannot distinguish "ran and failed" from "never started".+ # Only the run id proves a row existed: pre-run refusals can carry a synthetic+ # overall_status="failed" even though execution never started.+ run_created = bool(getattr(diagnosis_input, "workflow_run_id", None))if getattr(ctx, "last_workflow", None) is not None:+ if unrecoverable_stop and not run_created:+ return f"I built {block_phrase}, but I couldn't start a test run: {reason}.{status_sentence}"if test_attempted and test_failed and not unrecoverable_stop:return f"I built {block_phrase} and tested it, but the test failed: {reason}.{status_sentence}"if test_attempted:diff --git a/skyvern/forge/sdk/copilot/tools/run_execution.py b/skyvern/forge/sdk/copilot/tools/run_execution.pyindex 441713309..08f231d7e 100644--- a/skyvern/forge/sdk/copilot/tools/run_execution.py+++ b/skyvern/forge/sdk/copilot/tools/run_execution.py@@ -233,6 +233,20 @@ RUN_BLOCKS_POLL_INTERVAL_SECONDS = 5.0COPILOT_SANDBOX_UNAVAILABLE_ERROR = "Sandboxed worker is unavailable; execution was not started."+# Block types that can reach exec() in the API process, so a run containing one may+# only proceed on the sandboxed worker. CODE compiles user code directly;+# WORKFLOW_TRIGGER runs its child in-process with block_labels=None, which both+# executes the child's own code blocks and re-opens the cached-script import path;+# TaskV2 synthesizes a code block at runtime from planner output. The latter two+# cannot be proven code-free by inspecting the draft, so they stay fail-closed.+_SANDBOX_REQUIRED_BLOCK_TYPES = frozenset(+ {+ BlockType.CODE.value,+ BlockType.WORKFLOW_TRIGGER.value,+ BlockType.TaskV2.value,+ }+)+# Detached cleanup tasks held here so the garbage collector does not drop them# while they still have work to do, and so the "task exception was never# retrieved" warning cannot fire — each task adds a done-callback that logs@@ -424,7 +438,27 @@ def _maybe_clear_reconciliation_flag(copilot_ctx: Any, result: Any) -> None:def _copilot_sandbox_unavailable_result() -> dict[str, Any]:- return {"ok": False, "error": COPILOT_SANDBOX_UNAVAILABLE_ERROR}+ # No repair can make the sandbox reachable, so the result carries+ # UNRECOVERABLE_TOOL_ERROR to reach the contract's STOP lane. Without it the+ # refusal classifies as a generic FAILED_RUN and the enforcement loop nudges+ # the model to retry a run that never started.+ return {+ "ok": False,+ "error": COPILOT_SANDBOX_UNAVAILABLE_ERROR,+ "data": {+ "workflow_run_id": None,+ "overall_status": "failed",+ "failure_reason": COPILOT_SANDBOX_UNAVAILABLE_ERROR,+ "blocks": [],+ "failure_categories": [+ {+ "category": "UNRECOVERABLE_TOOL_ERROR",+ "confidence_float": 1.0,+ "reasoning": "Sandboxed execution was unavailable; no workflow run was created.",+ }+ ],+ },+ }def _mark_pending_reconciliation_run(copilot_ctx: Any, workflow_run_id: str) -> None:@@ -1004,6 +1038,33 @@ def _selected_code_security_inputs(return code_blocks+def _selected_blocks_require_sandbox(+ blocks: list[Any],+ *,+ selected_labels: set[str],+ include_descendants: bool = False,+) -> bool:+ for block in blocks:+ if isinstance(block, Mapping):+ label = str(block.get("label") or "")+ block_type = str(block.get("block_type") or "").lower()+ children = _mapping_child_blocks(block)+ else:+ label = str(getattr(block, "label", "") or "")+ block_type = str(getattr(block, "block_type", "") or "").lower()+ children = _typed_child_blocks(block)+ selected = include_descendants or label in selected_labels+ if selected and block_type in _SANDBOX_REQUIRED_BLOCK_TYPES:+ return True+ if children and _selected_blocks_require_sandbox(+ children,+ selected_labels=selected_labels,+ include_descendants=selected,+ ):+ return True+ return False++def _mapping_child_blocks(block: Mapping[str, Any]) -> list[Any]:children: list[Any] = []for key in ("loop_blocks", "blocks"):@@ -1606,16 +1667,38 @@ async def _run_blocks_and_collect_debug(if not workflow.get_output_parameter(label):return {"ok": False, "error": f"Block label not found in saved workflow: {label!r}"}+ workflow_definition = workflow.workflow_definition+ finally_block_label = (+ workflow_definition.get("finally_block_label")+ if isinstance(workflow_definition, Mapping)+ else getattr(workflow_definition, "finally_block_label", None)+ )+ labels_that_may_execute = list(labels_to_execute)+ if (+ isinstance(finally_block_label, str)+ and finally_block_label+ and finally_block_label not in labels_that_may_execute+ ):+ # WorkflowService executes this top-level block after every non-canceled+ # body, independently of the partial-run block whitelist. Admission,+ # runtime code security, and credential replay checks must all see it.+ labels_that_may_execute.append(finally_block_label)+runtime_security_failure = _runtime_code_security_failure_for_selected_labels(workflow,block_labels=list(block_labels),- labels_to_execute=labels_to_execute,+ labels_to_execute=labels_that_may_execute,frontier_start_label=frontier_start_label,)if runtime_security_failure is not None:ctx.last_executed_block_labels = []return runtime_security_failure+ requires_sandbox = _selected_blocks_require_sandbox(+ _workflow_definition_blocks_for_code_security(workflow_definition),+ selected_labels=set(labels_that_may_execute),+ )+credential_ids = list(dict.fromkeys(_extract_credential_ids_from_tool_value(params.get("parameters") or {})@@ -1650,7 +1733,10 @@ async def _run_blocks_and_collect_debug()# Compared against the literal True so anything other than an explicit opt-in — including a# test double that auto-mocks the hook into a truthy object — still fails closed.- if not dispatch_to_worker and app.AGENT_FUNCTION.allow_copilot_inline_code_execution() is not True:+ allow_inline_code_execution = (+ app.AGENT_FUNCTION.allow_copilot_inline_code_execution() is True if not dispatch_to_worker else False+ )+ if requires_sandbox and not dispatch_to_worker and not allow_inline_code_execution:return _copilot_sandbox_unavailable_result()runtime_workflow = _workflow_with_runtime_block_goal_context(workflow, ctx)@@ -1702,7 +1788,7 @@ async def _run_blocks_and_collect_debug(},}- use_fresh_session = _should_use_fresh_session_for_login_first_replay(ctx, labels_to_execute, workflow)+ use_fresh_session = _should_use_fresh_session_for_login_first_replay(ctx, labels_that_may_execute, workflow)# True when the run was threaded into a fresh session rather than the scout's debug session;# gates the post-run rebind (~:1135) so the ephemeral run session is not adopted as the# context session.@@ -1858,14 +1944,15 @@ async def _run_blocks_and_collect_debug(block_outputs=block_outputs_to_seed or None,)else:- LOG.error(- "UNSANDBOXED: executing copilot workflow code in the API process because "- "COPILOT_ALLOW_INLINE_CODE_EXECUTION is enabled. This is a local-development path "- "with no sandbox isolation; the run below is NOT a sandboxed run.",- workflow_run_id=workflow_run.workflow_run_id,- workflow_permanent_id=ctx.workflow_permanent_id,- organization_id=ctx.organization_id,- )+ if allow_inline_code_execution:+ LOG.error(+ "UNSANDBOXED: executing copilot workflow code in the API process because "+ "COPILOT_ALLOW_INLINE_CODE_EXECUTION is enabled. This is a local-development path "+ "with no sandbox isolation; the run below is NOT a sandboxed run.",+ workflow_run_id=workflow_run.workflow_run_id,+ workflow_permanent_id=ctx.workflow_permanent_id,+ organization_id=ctx.organization_id,+ )# prepare_workflow replaced the ambient context with this run's own, so the marker is# scoped to this run and is inherited by the execution task created below.inline_run_context = skyvern_context.current()diff --git a/skyvern/forge/sdk/workflow/service.py b/skyvern/forge/sdk/workflow/service.pyindex 143b1582c..e1bc75413 100644--- a/skyvern/forge/sdk/workflow/service.py+++ b/skyvern/forge/sdk/workflow/service.py@@ -4091,9 +4091,12 @@ class WorkflowService:loaded_script_module = None# If no cached script exists, check if a static pre-built script- # should be created for this platform (e.g., ATS). This persists the+ # should be created for this platform (e.g., ATS). This persists the# script to DB (pinned) on first run so it shows in the Code tab.- if is_script_run and not script_blocks_by_label:+ # Partial runs must stay agent-only, matching get_workflow_script's+ # existing contract: bootstrapping here would re-open run_signature+ # compile/exec after that lookup deliberately returned no script.+ if is_script_run and not block_labels and not script_blocks_by_label:try:static_result = await app.AGENT_FUNCTION.ensure_static_script(workflow=workflow,diff --git a/tests/unit/test_copilot_timeout_wip.py b/tests/unit/test_copilot_timeout_wip.pyindex 7a1aefbcd..0a47afb92 100644--- a/tests/unit/test_copilot_timeout_wip.py+++ b/tests/unit/test_copilot_timeout_wip.py@@ -61,9 +61,18 @@ def _ctx(return ctx-def _blocker_contract(reason: str, *, run_status: str | None = "running") -> DiagnosisRepairContract:+def _blocker_contract(+ reason: str,+ *,+ run_status: str | None = "running",+ workflow_run_id: str | None = None,+) -> DiagnosisRepairContract:return DiagnosisRepairContract(- diagnosis_input=DiagnosisInput(source_tool="get_browser_screenshot", run_status=run_status),+ diagnosis_input=DiagnosisInput(+ source_tool="get_browser_screenshot",+ workflow_run_id=workflow_run_id,+ run_status=run_status,+ ),diagnosis_result=DiagnosisResult(root_cause_summary=reason, confidence=0.9),repair_decision=RepairDecision(next_action="stop"),verification_result=VerificationResult(@@ -438,7 +447,10 @@ class TestBuildUnexpectedErrorExitResult:wf = MagicMock(name="wf")ctx = _ctx(last_workflow=wf, last_workflow_yaml="version: '1.0'", last_test_ok=False)ctx.last_update_block_count = 3- ctx.latest_diagnosis_repair_contract = _blocker_contract("Browser session was no longer reachable.")+ ctx.latest_diagnosis_repair_contract = _blocker_contract(+ "Browser session was no longer reachable.",+ workflow_run_id="wr_test",+ )result = _build_unexpected_error_exit_result(ctx, global_llm_context=None)@@ -474,6 +486,7 @@ class TestBuildUnexpectedErrorExitResult:ctx.latest_diagnosis_repair_contract = _blocker_contract("The browser session disappeared before screenshot verification could complete.",run_status="aborted",+ workflow_run_id="wr_test",)result = _build_unexpected_error_exit_result(ctx, global_llm_context=None)@@ -500,12 +513,32 @@ class TestBuildUnexpectedErrorExitResult:assert result.updated_workflow is wfassert result.proposal_disposition == "review_untested"assert result.user_response == (- "I built a 2-block draft, but I couldn't verify it: "+ "I built a 2-block draft, but I couldn't start a test run: ""Browser session not found while reading https://example.test. Last run status: aborted.")assert "pbs_" not in result.user_responseassert "token=secret" not in result.user_response+ def test_sandbox_refusal_does_not_claim_a_test_ran(self) -> None:+ """No run row means nothing executed, so the reply must not say it tested the draft."""+ from skyvern.forge.sdk.copilot.diagnosis_repair_contract import build_diagnosis_repair_contract+ from skyvern.forge.sdk.copilot.tools.run_execution import _copilot_sandbox_unavailable_result++ wf = MagicMock(name="wf")+ ctx = _ctx(last_workflow=wf, last_workflow_yaml="version: '1.0'", last_test_ok=False)+ ctx.test_after_update_done = True+ ctx.last_update_block_count = 6+ ctx.latest_diagnosis_repair_contract = build_diagnosis_repair_contract(+ source_tool="run_blocks_and_collect_debug",+ result=_copilot_sandbox_unavailable_result(),+ ctx=ctx,+ )++ result = _build_unexpected_error_exit_result(ctx, global_llm_context=None)++ assert "tested it" not in result.user_response+ assert "I couldn't start a test run" in result.user_response+def test_suspicious_success_drops_proposal(self) -> None:wf = MagicMock(name="wf")ctx = _ctx(