Skip to content

Automate browser based workflows with AI

Carry the typed CodeBlock runner error code onto the block result (SKY-13469) (#8030)

Andrew Neilson8a28e97ee69f
diff --git a/skyvern/forge/sdk/copilot/build_test_outcome.py b/skyvern/forge/sdk/copilot/build_test_outcome.py
index 77947283d..f0b3f2737 100644
--- a/skyvern/forge/sdk/copilot/build_test_outcome.py
+++ b/skyvern/forge/sdk/copilot/build_test_outcome.py
@@ -61,6 +61,7 @@ BuildTestOutcomeReasonCode = Literal[
"blocker_reported",
"failed_run",
"run_completed_unevaluated",
+ "unrecoverable_tool_error",
"suspicious_success",
"missing_structural_evidence",
"unchanged_after_recorded_outcome",
@@ -90,6 +91,7 @@ _REF_TEXT_MAX = 96
_VALUE_EXCERPT_MAX = 700
_HISTORY_LIMIT = 8
_INSPECT_PAGE_SOURCE_TOOL = "inspect_page_for_composition"
+_UNRECOVERABLE_TOOL_ERROR_CATEGORY = "UNRECOVERABLE_TOOL_ERROR"
_PLAYWRIGHT_LOCATOR_WAIT_RE = re.compile(
r"waiting for locator\((?P<quote>['\"])(?P<selector>.*?)(?P=quote)\)"
r"(?P<locator_chain>(?:\.[A-Za-z_][A-Za-z0-9_]*(?:\([^)]*\))?)*)\s+to be (?P<state>[a-z_]+)",
@@ -1385,6 +1387,13 @@ def recorded_outcome_from_run_blocks_result(
if failed_block is not None or not bool(result.get("ok"))
else "run_completed_unevaluated"
)
+ if any(ref.split(":", 1)[0] == _UNRECOVERABLE_TOOL_ERROR_CATEGORY for ref in failure_categories):
+ # The run failed on the tool plane, not on what was authored, so it is not test
+ # signal: dropping the identity keys it out of outcome dedup and grounding too.
+ verdict = "not_authoritative"
+ reason_code = "unrecoverable_tool_error"
+ structural_identity = ""
+ page_refs = []
has_runtime_failure_evidence = bool(failure_categories or failure_type or runtime_failure_identity or failed_block)
if (
verdict == "repairable_failure"
diff --git a/skyvern/forge/sdk/copilot/context.py b/skyvern/forge/sdk/copilot/context.py
index 2d01ad74b..6a6b231b8 100644
--- a/skyvern/forge/sdk/copilot/context.py
+++ b/skyvern/forge/sdk/copilot/context.py
@@ -927,6 +927,9 @@ class CopilotContext(AgentContext):
# exception in run_with_enforcement. Cleared at the top of every call to
# _record_run_blocks_result so stale state can't leak across runs.
last_test_non_retriable_nav_error: str | None = None
+ # Secure-runner codes from the latest run that were faults of the sandbox itself, joined.
+ # Cleared per run in _record_run_blocks_result, so a later clean run releases the guard.
+ last_infrastructure_tool_error: str | None = None
# Normalized signature of the non-retriable nav error last nudged on.
# Lets the stop nudge re-fire if the user retries with a different bad URL
# (different signature) in the same session. Cleared on meaningful success.
diff --git a/skyvern/forge/sdk/copilot/runtime.py b/skyvern/forge/sdk/copilot/runtime.py
index b03a9f366..340d834e5 100644
--- a/skyvern/forge/sdk/copilot/runtime.py
+++ b/skyvern/forge/sdk/copilot/runtime.py
@@ -345,6 +345,7 @@ class AgentContext:
last_code_authoring_repair_context: CodeAuthoringRepairContext | None = None
challenge_gated_proxy_retry_count: int = 0
last_test_non_retriable_nav_error: str | None = None
+ last_infrastructure_tool_error: str | None = None
non_retriable_nav_error_last_emitted_signature: str | None = None
workflow_persisted: bool = False
last_workflow: Any | None = None
diff --git a/skyvern/forge/sdk/copilot/tools/blockers.py b/skyvern/forge/sdk/copilot/tools/blockers.py
index 45aa33954..bd73aef42 100644
--- a/skyvern/forge/sdk/copilot/tools/blockers.py
+++ b/skyvern/forge/sdk/copilot/tools/blockers.py
@@ -70,6 +70,9 @@ _CURRENT_PAGE_TERMINAL_CHALLENGE_TOOLS = (
| frozenset({"click", "navigate_browser", "press_key", "scroll", "select_option", "type_text"})
)
_OUTPUT_CONTRACT_LADDER_AUTHORING_TOOLS = frozenset({"update_workflow", "update_and_run_blocks"})
+_INFRASTRUCTURE_TOOL_ERROR_BLOCKED_TOOLS = BLOCK_RUNNING_TOOLS | frozenset(
+ {"update_workflow", "edit_block", "delete_block"}
+)
async def _safe_read_workflow_run(
@@ -1411,6 +1414,7 @@ LOOP_PLANE_REFUSAL_REASON_CODES: frozenset[str] = frozenset(
"tool_error_synthesized_block_persistence_required",
"tool_error_challenge_gated_submit_disabled",
"tool_error_non_retriable_nav",
+ "tool_error_infrastructure_unavailable",
"tool_error_pending_reconciliation_no_input",
"tool_error_pending_reconciliation_requires_input",
"tool_error_per_tool_budget_rerun",
@@ -1441,6 +1445,37 @@ def _tool_loop_error(ctx: AgentContext, tool_name: str, arguments: dict[str, Any
if current_page_challenge_signal is not None:
return _emit_loop_plane_refusal(ctx, current_page_challenge_signal)
+ # The diagnosis contract's STOP only runs BETWEEN agent turns, so without this entrypoint
+ # check the model can re-author around an unreachable sandbox before the nudge ever fires.
+ infrastructure_error = getattr(ctx, "last_infrastructure_tool_error", None)
+ if (
+ tool_name in _INFRASTRUCTURE_TOOL_ERROR_BLOCKED_TOOLS
+ and isinstance(infrastructure_error, str)
+ and infrastructure_error
+ ):
+ return _emit_loop_plane_refusal(
+ ctx,
+ CopilotToolBlockerSignal(
+ blocker_kind="tool_error",
+ agent_steering_text=(
+ f"The last run failed because the code sandbox was unreachable ({infrastructure_error[:200]}). "
+ "The user's code is not at fault. Do NOT edit, delete, or re-run any block — reply to the "
+ "user explaining that the code sandbox is unavailable and asking them to retry shortly."
+ ),
+ user_facing_reason=(
+ "The code sandbox is unavailable right now, so I couldn't test your code block. "
+ "Your code is unchanged — please try again in a few minutes."
+ ),
+ recovery_hint="ask_user_clarifying",
+ cleared_by_tools=frozenset(),
+ # The reply promises the user's code is unchanged, so the unpersisted draft
+ # has to survive the turn that says so.
+ preserves_workflow_draft=True,
+ internal_reason_code="tool_error_infrastructure_unavailable",
+ blocked_tool=tool_name,
+ ),
+ )
+
persistence_signal = synthesized_block_persistence_signal(ctx, tool_name, arguments)
if persistence_signal is not None:
return _emit_loop_plane_refusal(ctx, persistence_signal)
diff --git a/skyvern/forge/sdk/copilot/tools/run_execution.py b/skyvern/forge/sdk/copilot/tools/run_execution.py
index 08f231d7e..5eb0d543f 100644
--- a/skyvern/forge/sdk/copilot/tools/run_execution.py
+++ b/skyvern/forge/sdk/copilot/tools/run_execution.py
@@ -247,6 +247,21 @@ _SANDBOX_REQUIRED_BLOCK_TYPES = frozenset(
}
)
+# Sandbox-process faults, not authored-code faults. ``timeout`` and ``user_code_error`` stay
+# out: both are repairable despite also carrying ``runner_internal_error``. ``busy`` is in —
+# a saturated runner gate says nothing about the code, so rewriting it cannot help.
+INFRASTRUCTURE_RUNNER_ERROR_CODES: frozenset[str] = frozenset(
+ {
+ "runner_unavailable",
+ "protocol_error",
+ "internal_error",
+ "child_exited",
+ "child_no_request",
+ "child_malformed_request",
+ "busy",
+ }
+)
+
# Detached cleanup tasks held here so the garbage collector does not drop them
# while they still have work to do, and so the "task exception was never
# retrieved" warning cannot fire — each task adds a done-callback that logs
@@ -2264,6 +2279,8 @@ async def _run_blocks_and_collect_debug(
}
if block.failure_reason:
block_result["failure_reason"] = block.failure_reason
+ if block.error_codes:
+ block_result["error_codes"] = list(block.error_codes)
if hasattr(block, "output") and block.output:
block_result["extracted_data"] = block.output
if block.label is not None:
@@ -2752,6 +2769,24 @@ def _detect_non_retriable_nav_error(result: dict[str, Any]) -> str | None:
return next((reason for reason in iter_failure_reasons(result) if is_skip_inner_retry_error(reason)), None)
+def _infrastructure_runner_error_codes(result: dict[str, Any]) -> list[str]:
+ data = result.get("data")
+ blocks = data.get("blocks") if isinstance(data, dict) else None
+ if not isinstance(blocks, list):
+ return []
+ found: list[str] = []
+ for block in blocks:
+ if not isinstance(block, dict):
+ continue
+ codes = block.get("error_codes")
+ if not isinstance(codes, list):
+ continue
+ for code in codes:
+ if isinstance(code, str) and code in INFRASTRUCTURE_RUNNER_ERROR_CODES and code not in found:
+ found.append(code)
+ return found
+
+
def _update_verification_evidence_from_run_result(copilot_ctx: AgentContext, result: Mapping[str, object]) -> None:
evidence = _workflow_verification_evidence(copilot_ctx)
data_value = result.get("data")
@@ -2888,6 +2923,7 @@ def _record_run_blocks_result(
prior_budget_flag = copilot_ctx.last_failure_category_top == PER_TOOL_BUDGET_FAILURE_CATEGORY
copilot_ctx.last_failure_category_top = None
copilot_ctx.last_test_non_retriable_nav_error = None
+ copilot_ctx.last_infrastructure_tool_error = None
copilot_ctx.post_run_page_observation_tool = None
copilot_ctx.post_run_page_observation_url = None
copilot_ctx.post_run_page_observation_workflow_run_id = None
@@ -2897,6 +2933,22 @@ def _record_run_blocks_result(
structured_blocker = _run_blocks_structured_blocker_message(result, copilot_ctx)
anti_bot_match, empty_data_blocks, failure_categories = _analyze_run_blocks(result, copilot_ctx)
+ infrastructure_runner_codes = _infrastructure_runner_error_codes(result)
+ if infrastructure_runner_codes:
+ copilot_ctx.last_infrastructure_tool_error = ", ".join(infrastructure_runner_codes)
+ # Prepended, not appended: `last_failure_category_top` reads entry zero, and the
+ # infrastructure fault outranks whatever the block's prose was classified as.
+ failure_categories = [
+ {
+ "category": "UNRECOVERABLE_TOOL_ERROR",
+ "confidence_float": 1.0,
+ "reasoning": (
+ "The code sandbox was unreachable "
+ f"({copilot_ctx.last_infrastructure_tool_error}); no edit to the block can reach it."
+ ),
+ },
+ *(failure_categories or []),
+ ]
artifact_flag_key = _artifact_challenge_flag_from_result(result, copilot_ctx)
anti_bot_source = first_carrier_backed_anti_bot_source(failure_categories) if anti_bot_match else None
anti_bot_evidence_source = anti_bot_source.value if anti_bot_source else None
diff --git a/skyvern/forge/sdk/workflow/models/block.py b/skyvern/forge/sdk/workflow/models/block.py
index bc7b433b1..5ce111bba 100644
--- a/skyvern/forge/sdk/workflow/models/block.py
+++ b/skyvern/forge/sdk/workflow/models/block.py
@@ -21,6 +21,7 @@ import unicodedata
import uuid
import zipfile
from collections import defaultdict, deque
+from collections.abc import Sequence
from datetime import date, datetime, time, timezone
from email.message import EmailMessage
from functools import partial
@@ -440,6 +441,16 @@ def _maybe_truncate_loop_outputs(
outputs_with_loop_values.append(last)
+def build_block_failure_output(failure_reason: str, error_codes: Sequence[str]) -> dict[str, Any]:
+ # SKY-7939: also surface the failure in `outputs.<block_label>` so callers
+ # can tell which block failed without cross-referencing the timeline.
+ return {
+ "status": BlockStatus.failed.value,
+ "failure_reason": failure_reason,
+ "errors": [{"error_code": code, "reasoning": failure_reason, "confidence_float": 1.0} for code in error_codes],
+ }
+
+
class Block(BaseModel, abc.ABC):
"""Base class for workflow nodes (see branching spec [[s-4bnl]] for metadata semantics)."""
@@ -5683,17 +5694,30 @@ async def wrapper({default_args}):
secure_classification = self._classify_secure_runner_failure(secure_failure)
async def build_secure_failure_result() -> BlockResult:
- if secure_code_block_result.block_result is not None:
- return secure_code_block_result.block_result
+ engine_block_result = secure_code_block_result.block_result
+ if engine_block_result is not None:
+ await self.record_output_parameter_value(
+ workflow_run_context,
+ workflow_run_id,
+ engine_block_result.output_parameter_value,
+ )
+ return engine_block_result
+ secure_failure_reason = (
+ secure_failure.failure_reason or "Code block failed in the secure runner"
+ )
+ secure_error_codes = [secure_failure.error_code] if secure_failure.error_code else []
+ failure_output = build_block_failure_output(secure_failure_reason, secure_error_codes)
+ await self.record_output_parameter_value(
+ workflow_run_context, workflow_run_id, failure_output
+ )
return await self.build_block_result(
success=False,
- failure_reason=(
- secure_failure.failure_reason or "Code block failed in the secure runner"
- ),
- output_parameter_value=None,
+ failure_reason=secure_failure_reason,
+ output_parameter_value=failure_output,
status=BlockStatus.failed,
workflow_run_block_id=workflow_run_block_id,
organization_id=organization_id,
+ error_codes=secure_error_codes or None,
)
return await self._resolve_failure_with_heal(
@@ -8917,16 +8941,8 @@ class FileParserBlock(Block):
organization_id: str | None,
failure_reason: str,
) -> BlockResult:
- # SKY-7939: also surface the failure in `outputs.<block_label>` so callers
- # can tell which block failed without cross-referencing the timeline.
error_codes = self.get_failure_error_codes()
- failure_output: dict[str, Any] = {
- "status": BlockStatus.failed.value,
- "failure_reason": failure_reason,
- "errors": [
- {"error_code": code, "reasoning": failure_reason, "confidence_float": 1.0} for code in error_codes
- ],
- }
+ failure_output = build_block_failure_output(failure_reason, error_codes)
await self.record_output_parameter_value(workflow_run_context, workflow_run_id, failure_output)
return await self.build_block_result(
success=False,
diff --git a/tests/unit/test_copilot_agent_helpers.py b/tests/unit/test_copilot_agent_helpers.py
index 59a5501a3..7af21a6e7 100644
--- a/tests/unit/test_copilot_agent_helpers.py
+++ b/tests/unit/test_copilot_agent_helpers.py
@@ -23,6 +23,7 @@ from skyvern.forge.sdk.copilot.agent import (
_VERIFIED_WORKFLOW_SUCCESS_REPLY,
_build_goal_satisfied_exit_result,
_resolve_wrapped_exception_exit_result,
+ _rewrite_failed_test_response,
_synthesized_block_offer_prompt,
_verified_workflow_or_none,
)
@@ -7482,3 +7483,20 @@ def test_a_turn_that_forms_no_requested_output_still_dumps_its_trajectory(tmp_pa
recorded = json.loads(dumps[0].read_text())["scout_trajectory"]
assert [step["credential_field"] for step in recorded] == ["totp"]
+
+
+def test_rewrite_names_the_sandbox_outage_when_the_runner_was_unreachable() -> None:
+ ctx = _ctx(
+ last_update_block_count=1,
+ last_test_ok=False,
+ last_test_failure_reason="Secure CodeBlock runner is unavailable. Please retry.",
+ last_failure_category_top="UNRECOVERABLE_TOOL_ERROR",
+ last_run_blocks_workflow_run_id="wr_runner",
+ )
+
+ rewritten = _rewrite_failed_test_response("All set — the workflow is ready.", ctx)
+
+ assert rewritten == (
+ "I created a draft workflow with 1 block and tested it, but the test failed. "
+ "Failure: Secure CodeBlock runner is unavailable. Please retry.."
+ )
diff --git a/tests/unit/test_copilot_build_test_outcome.py b/tests/unit/test_copilot_build_test_outcome.py
index f341949e4..b91a013db 100644
--- a/tests/unit/test_copilot_build_test_outcome.py
+++ b/tests/unit/test_copilot_build_test_outcome.py
@@ -1747,3 +1747,53 @@ def test_an_all_literal_block_still_retires_when_the_call_is_gone() -> None:
record_build_test_outcome(ctx, passing_run("wr_2", ["sign_in_and_read"]))
assert unresolved_runtime_block_failure(ctx) is None
+
+
+def _secure_runner_failure_result(error_code: str, category: str | None) -> dict[str, object]:
+ data: dict[str, object] = {
+ "workflow_run_id": "wr_runner",
+ "overall_status": "failed",
+ "blocks": [
+ {
+ "label": "run_code",
+ "block_type": "CODE",
+ "status": "failed",
+ "failure_reason": "Secure CodeBlock runner is unavailable. Please retry.",
+ "error_codes": [error_code],
+ }
+ ],
+ }
+ if category is not None:
+ data["failure_categories"] = [{"category": category, "confidence_float": 1.0, "reasoning": "sandbox"}]
+ return {"ok": False, "data": data}
+
+
+def test_unrecoverable_tool_error_run_is_not_repairable_and_not_authoritative() -> None:
+ outcome = recorded_outcome_from_run_blocks_result(
+ _secure_runner_failure_result("runner_unavailable", "UNRECOVERABLE_TOOL_ERROR")
+ )
+
+ assert outcome is not None
+ assert outcome.verdict != "repairable_failure"
+ assert outcome.verdict == "not_authoritative"
+ assert outcome.reason_code == "unrecoverable_tool_error"
+ assert outcome.is_authoritative is False
+ assert outcome.structural_failure_identity == ""
+
+
+def test_user_code_error_run_stays_repairable() -> None:
+ outcome = recorded_outcome_from_run_blocks_result(
+ _secure_runner_failure_result("user_code_error", "CODE_BLOCK_FAILURE")
+ )
+
+ assert outcome is not None
+ assert outcome.verdict == "repairable_failure"
+ assert outcome.reason_code == "runtime_block_failure"
+
+
+def test_runner_timeout_run_stays_repairable() -> None:
+ outcome = recorded_outcome_from_run_blocks_result(_secure_runner_failure_result("timeout", "CODE_BLOCK_FAILURE"))
+
+ assert outcome is not None
+ assert outcome.verdict == "repairable_failure"
+ assert outcome.reason_code == "runtime_block_failure"
diff --git a/tests/unit/test_copilot_diagnosis_repair_contract.py b/tests/unit/test_copilot_diagnosis_repair_contract.py
index 1b9d539ea..58f9d1bc5 100644
--- a/tests/unit/test_copilot_diagnosis_repair_contract.py
+++ b/tests/unit/test_copilot_diagnosis_repair_contract.py
@@ -2377,3 +2377,67 @@ async def test_failed_seam_capture_preserves_clean_matching_evidence(
ctx, run_session_id="run_session", run_id="wr_failed", current_url="https://example.test/app"
)
assert ctx.composition_page_evidence is clean
+
+
+def _secure_runner_unavailable_result() -> dict[str, object]:
+ return {
+ "ok": False,
+ "error": "Run failed.",
+ "data": {
+ "workflow_run_id": "wr_runner_unavailable",
+ "overall_status": "failed",
+ "blocks": [
+ {
+ "label": "run_code",
+ "block_type": "CODE",
+ "status": "failed",
+ "failure_reason": "Secure CodeBlock runner is unavailable. Please retry.",
+ "error_codes": ["runner_unavailable"],
+ }
+ ],
+ },
+ }
+
+
+def test_runner_unavailable_stops_even_with_a_code_authoring_repair_context() -> None:
+ ctx = _ctx()
+ result = _secure_runner_unavailable_result()
+ run_execution_module._record_run_blocks_result(ctx, result, completion_verification=None)
+ data = result["data"]
+ assert isinstance(data, dict)
+ data["authoring_repair_context"] = CodeAuthoringRepairContext(
+ block_label="run_code",
+ reason_code="ambiguous_bare_selector",
+ selector="button",
+ refiner_selector="xpath=//button[normalize-space()='Download']",
+ ).model_dump(mode="json")
+
+ contract = build_diagnosis_repair_contract(
+ source_tool="update_and_run_blocks",
+ result=result,
+ ctx=ctx,
+ )
+
+ assert contract.diagnosis_result.suspected_failure_type == DiagnosisFailureType.UNRECOVERABLE_TOOL_ERROR
+ assert contract.repair_decision.next_action == RepairNextAction.STOP
+
+
+def test_user_code_error_still_repairs_through_the_contract() -> None:
+ ctx = _ctx()
+ result = _secure_runner_unavailable_result()
+ data = result["data"]
+ assert isinstance(data, dict)
+ blocks = data["blocks"]
+ assert isinstance(blocks, list)
+ blocks[0]["error_codes"] = ["user_code_error"]
+ blocks[0]["failure_reason"] = "NameError: name 'undefined_helper' is not defined"
+ run_execution_module._record_run_blocks_result(ctx, result, completion_verification=None)
+
+ contract = build_diagnosis_repair_contract(
+ source_tool="update_and_run_blocks",
+ result=result,
+ ctx=ctx,
+ )
+
+ assert contract.diagnosis_result.suspected_failure_type != DiagnosisFailureType.UNRECOVERABLE_TOOL_ERROR
+ assert contract.repair_decision.next_action != RepairNextAction.STOP
diff --git a/tests/unit/test_copilot_run_output_content_gate.py b/tests/unit/test_copilot_run_output_content_gate.py
index 0646512f4..e76974a5d 100644
--- a/tests/unit/test_copilot_run_output_content_gate.py
+++ b/tests/unit/test_copilot_run_output_content_gate.py
@@ -28,6 +28,7 @@ from skyvern.forge.sdk.copilot.tools import (
_is_unfinished_run_verification_candidate,
_record_run_blocks_result,
_run_blocks_structured_blocker_message,
+ _tool_loop_error,
run_execution,
)
from skyvern.forge.sdk.copilot.tools._shared import _registered_output_parameter_payloads
@@ -1459,3 +1460,60 @@ def test_present_value_upgrade_flips_lone_judge_unknown_to_demonstrated_on_ok_ru
assert outcome_fully_verified(ctx) is True
assert ctx.last_test_suspicious_success is False
assert ctx.last_full_workflow_test_ok is True
+
+
+def _failed_code_block_run_result(error_code: str) -> dict[str, Any]:
+ return {
+ "ok": False,
+ "data": {
+ "workflow_run_id": "wr_test",
+ "overall_status": "failed",
+ "blocks": [
+ {
+ "label": "run_code",
+ "block_type": "CODE",
+ "status": "failed",
+ "failure_reason": "Secure CodeBlock runner is unavailable. Please retry.",
+ "error_codes": [error_code],
+ }
+ ],
+ },
+ }
+
+
[email protected]("error_code", sorted(run_execution.INFRASTRUCTURE_RUNNER_ERROR_CODES))
+def test_infrastructure_runner_code_prepends_unrecoverable_tool_error(error_code: str) -> None:
+ result = _failed_code_block_run_result(error_code)
+ ctx = _ctx(result["data"]["blocks"])
+
+ _record_run_blocks_result(ctx, result, completion_verification=None)
+
+ categories = result["data"]["failure_categories"]
+ assert categories[0]["category"] == "UNRECOVERABLE_TOOL_ERROR"
+ assert ctx.last_failure_category_top == "UNRECOVERABLE_TOOL_ERROR"
+ assert ctx.last_infrastructure_tool_error == error_code
+
+
[email protected]("error_code", ["timeout", "user_code_error", "insecure_code_detected"])
+def test_repairable_runner_code_injects_no_unrecoverable_category(error_code: str) -> None:
+ result = _failed_code_block_run_result(error_code)
+ ctx = _ctx(result["data"]["blocks"])
+
+ _record_run_blocks_result(ctx, result, completion_verification=None)
+
+ categories = result["data"].get("failure_categories") or []
+ assert all(entry.get("category") != "UNRECOVERABLE_TOOL_ERROR" for entry in categories)
+ assert ctx.last_failure_category_top != "UNRECOVERABLE_TOOL_ERROR"
+ assert ctx.last_infrastructure_tool_error is None
+
+
+def test_pre_run_sandbox_refusal_leaves_the_within_turn_guard_disarmed() -> None:
+ result = run_execution._copilot_sandbox_unavailable_result()
+ ctx = _ctx()
+
+ _record_run_blocks_result(ctx, result, completion_verification=None)
+
+ assert result["data"]["workflow_run_id"] is None
+ assert ctx.last_failure_category_top == "UNRECOVERABLE_TOOL_ERROR"
+ assert ctx.last_infrastructure_tool_error is None
+ assert _tool_loop_error(ctx, "edit_block") is None
diff --git a/tests/unit/test_copilot_tool_loop_error_block_running.py b/tests/unit/test_copilot_tool_loop_error_block_running.py
index 3e3265d93..8e1f78885 100644
--- a/tests/unit/test_copilot_tool_loop_error_block_running.py
+++ b/tests/unit/test_copilot_tool_loop_error_block_running.py
@@ -13,6 +13,8 @@ import time
from types import SimpleNamespace
from typing import Any
+import pytest
+
from skyvern.forge.sdk.copilot.enforcement import TOTAL_TIMEOUT_SECONDS
from skyvern.forge.sdk.copilot.failure_tracking import compute_action_sequence_fingerprint
from skyvern.forge.sdk.copilot.loop_detection import record_consecutive_tool_result_boundary_for_ctx
@@ -21,7 +23,9 @@ from skyvern.forge.sdk.copilot.tools import (
PER_TOOL_CALL_BUDGET_SECONDS,
_active_block_run_budget_seconds,
_tool_loop_error,
+ blockers,
)
+from skyvern.forge.sdk.copilot.tools.run_execution import INFRASTRUCTURE_RUNNER_ERROR_CODES
def _ctx(**overrides: Any) -> Any:
@@ -174,3 +178,48 @@ def test_bypass_applies_to_both_block_running_tool_names() -> None:
ctx = _ctx()
for _ in range(5):
assert _tool_loop_error(ctx, "run_blocks_and_collect_debug") is None
+
+
+def test_infrastructure_runner_failure_blocks_authoring_and_running_tools() -> None:
+ ctx = _ctx(last_infrastructure_tool_error="runner_unavailable")
+
+ for tool in ("edit_block", "delete_block", "update_workflow", "update_and_run_blocks"):
+ msg = _tool_loop_error(ctx, tool)
+ assert msg is not None, f"{tool} should be refused"
+ assert "runner_unavailable" in msg
+ assert "code sandbox" in msg
+
+
+def test_infrastructure_runner_failure_leaves_reporting_tools_callable() -> None:
+ ctx = _ctx(last_infrastructure_tool_error="runner_unavailable")
+
+ for tool in ("get_run_results", "list_credentials"):
+ assert _tool_loop_error(ctx, tool) is None, f"{tool} should stay callable"
+
+
+def test_edit_block_is_not_refused_without_an_infrastructure_runner_failure() -> None:
+ ctx = _ctx(last_infrastructure_tool_error=None)
+
+ assert _tool_loop_error(ctx, "edit_block") is None
+
+
+def test_infrastructure_runner_failure_preserves_the_unpersisted_draft(monkeypatch: pytest.MonkeyPatch) -> None:
+ captured: list[Any] = []
+ monkeypatch.setattr(
+ blockers,
+ "_emit_loop_plane_refusal",
+ lambda ctx, signal: captured.append(signal) or "refused",
+ )
+ ctx = _ctx(last_infrastructure_tool_error="runner_unavailable", turn_halt=None)
+
+ _tool_loop_error(ctx, "edit_block")
+
+ (signal,) = captured
+ assert "unchanged" in signal.user_facing_reason
+ assert signal.preserves_workflow_draft is True
+
+
+def test_busy_runner_is_an_infrastructure_failure_not_an_authored_code_failure() -> None:
+ assert "busy" in INFRASTRUCTURE_RUNNER_ERROR_CODES
+ assert "timeout" not in INFRASTRUCTURE_RUNNER_ERROR_CODES
+ assert "user_code_error" not in INFRASTRUCTURE_RUNNER_ERROR_CODES