Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
Carry the typed CodeBlock runner error code onto the block result (SKY-13469) (#8030)
Andrew Neilson8a28e97ee69f
diff --git a/skyvern/forge/sdk/copilot/build_test_outcome.py b/skyvern/forge/sdk/copilot/build_test_outcome.pyindex 77947283d..f0b3f2737 100644--- a/skyvern/forge/sdk/copilot/build_test_outcome.py+++ b/skyvern/forge/sdk/copilot/build_test_outcome.py@@ -61,6 +61,7 @@ BuildTestOutcomeReasonCode = Literal["blocker_reported","failed_run","run_completed_unevaluated",+ "unrecoverable_tool_error","suspicious_success","missing_structural_evidence","unchanged_after_recorded_outcome",@@ -90,6 +91,7 @@ _REF_TEXT_MAX = 96_VALUE_EXCERPT_MAX = 700_HISTORY_LIMIT = 8_INSPECT_PAGE_SOURCE_TOOL = "inspect_page_for_composition"+_UNRECOVERABLE_TOOL_ERROR_CATEGORY = "UNRECOVERABLE_TOOL_ERROR"_PLAYWRIGHT_LOCATOR_WAIT_RE = re.compile(r"waiting for locator\((?P<quote>['\"])(?P<selector>.*?)(?P=quote)\)"r"(?P<locator_chain>(?:\.[A-Za-z_][A-Za-z0-9_]*(?:\([^)]*\))?)*)\s+to be (?P<state>[a-z_]+)",@@ -1385,6 +1387,13 @@ def recorded_outcome_from_run_blocks_result(if failed_block is not None or not bool(result.get("ok"))else "run_completed_unevaluated")+ if any(ref.split(":", 1)[0] == _UNRECOVERABLE_TOOL_ERROR_CATEGORY for ref in failure_categories):+ # The run failed on the tool plane, not on what was authored, so it is not test+ # signal: dropping the identity keys it out of outcome dedup and grounding too.+ verdict = "not_authoritative"+ reason_code = "unrecoverable_tool_error"+ structural_identity = ""+ page_refs = []has_runtime_failure_evidence = bool(failure_categories or failure_type or runtime_failure_identity or failed_block)if (verdict == "repairable_failure"diff --git a/skyvern/forge/sdk/copilot/context.py b/skyvern/forge/sdk/copilot/context.pyindex 2d01ad74b..6a6b231b8 100644--- a/skyvern/forge/sdk/copilot/context.py+++ b/skyvern/forge/sdk/copilot/context.py@@ -927,6 +927,9 @@ class CopilotContext(AgentContext):# exception in run_with_enforcement. Cleared at the top of every call to# _record_run_blocks_result so stale state can't leak across runs.last_test_non_retriable_nav_error: str | None = None+ # Secure-runner codes from the latest run that were faults of the sandbox itself, joined.+ # Cleared per run in _record_run_blocks_result, so a later clean run releases the guard.+ last_infrastructure_tool_error: str | None = None# Normalized signature of the non-retriable nav error last nudged on.# Lets the stop nudge re-fire if the user retries with a different bad URL# (different signature) in the same session. Cleared on meaningful success.diff --git a/skyvern/forge/sdk/copilot/runtime.py b/skyvern/forge/sdk/copilot/runtime.pyindex b03a9f366..340d834e5 100644--- a/skyvern/forge/sdk/copilot/runtime.py+++ b/skyvern/forge/sdk/copilot/runtime.py@@ -345,6 +345,7 @@ class AgentContext:last_code_authoring_repair_context: CodeAuthoringRepairContext | None = Nonechallenge_gated_proxy_retry_count: int = 0last_test_non_retriable_nav_error: str | None = None+ last_infrastructure_tool_error: str | None = Nonenon_retriable_nav_error_last_emitted_signature: str | None = Noneworkflow_persisted: bool = Falselast_workflow: Any | None = Nonediff --git a/skyvern/forge/sdk/copilot/tools/blockers.py b/skyvern/forge/sdk/copilot/tools/blockers.pyindex 45aa33954..bd73aef42 100644--- a/skyvern/forge/sdk/copilot/tools/blockers.py+++ b/skyvern/forge/sdk/copilot/tools/blockers.py@@ -70,6 +70,9 @@ _CURRENT_PAGE_TERMINAL_CHALLENGE_TOOLS = (| frozenset({"click", "navigate_browser", "press_key", "scroll", "select_option", "type_text"}))_OUTPUT_CONTRACT_LADDER_AUTHORING_TOOLS = frozenset({"update_workflow", "update_and_run_blocks"})+_INFRASTRUCTURE_TOOL_ERROR_BLOCKED_TOOLS = BLOCK_RUNNING_TOOLS | frozenset(+ {"update_workflow", "edit_block", "delete_block"}+)async def _safe_read_workflow_run(@@ -1411,6 +1414,7 @@ LOOP_PLANE_REFUSAL_REASON_CODES: frozenset[str] = frozenset("tool_error_synthesized_block_persistence_required","tool_error_challenge_gated_submit_disabled","tool_error_non_retriable_nav",+ "tool_error_infrastructure_unavailable","tool_error_pending_reconciliation_no_input","tool_error_pending_reconciliation_requires_input","tool_error_per_tool_budget_rerun",@@ -1441,6 +1445,37 @@ def _tool_loop_error(ctx: AgentContext, tool_name: str, arguments: dict[str, Anyif current_page_challenge_signal is not None:return _emit_loop_plane_refusal(ctx, current_page_challenge_signal)+ # The diagnosis contract's STOP only runs BETWEEN agent turns, so without this entrypoint+ # check the model can re-author around an unreachable sandbox before the nudge ever fires.+ infrastructure_error = getattr(ctx, "last_infrastructure_tool_error", None)+ if (+ tool_name in _INFRASTRUCTURE_TOOL_ERROR_BLOCKED_TOOLS+ and isinstance(infrastructure_error, str)+ and infrastructure_error+ ):+ return _emit_loop_plane_refusal(+ ctx,+ CopilotToolBlockerSignal(+ blocker_kind="tool_error",+ agent_steering_text=(+ f"The last run failed because the code sandbox was unreachable ({infrastructure_error[:200]}). "+ "The user's code is not at fault. Do NOT edit, delete, or re-run any block — reply to the "+ "user explaining that the code sandbox is unavailable and asking them to retry shortly."+ ),+ user_facing_reason=(+ "The code sandbox is unavailable right now, so I couldn't test your code block. "+ "Your code is unchanged — please try again in a few minutes."+ ),+ recovery_hint="ask_user_clarifying",+ cleared_by_tools=frozenset(),+ # The reply promises the user's code is unchanged, so the unpersisted draft+ # has to survive the turn that says so.+ preserves_workflow_draft=True,+ internal_reason_code="tool_error_infrastructure_unavailable",+ blocked_tool=tool_name,+ ),+ )+persistence_signal = synthesized_block_persistence_signal(ctx, tool_name, arguments)if persistence_signal is not None:return _emit_loop_plane_refusal(ctx, persistence_signal)diff --git a/skyvern/forge/sdk/copilot/tools/run_execution.py b/skyvern/forge/sdk/copilot/tools/run_execution.pyindex 08f231d7e..5eb0d543f 100644--- a/skyvern/forge/sdk/copilot/tools/run_execution.py+++ b/skyvern/forge/sdk/copilot/tools/run_execution.py@@ -247,6 +247,21 @@ _SANDBOX_REQUIRED_BLOCK_TYPES = frozenset(})+# Sandbox-process faults, not authored-code faults. ``timeout`` and ``user_code_error`` stay+# out: both are repairable despite also carrying ``runner_internal_error``. ``busy`` is in —+# a saturated runner gate says nothing about the code, so rewriting it cannot help.+INFRASTRUCTURE_RUNNER_ERROR_CODES: frozenset[str] = frozenset(+ {+ "runner_unavailable",+ "protocol_error",+ "internal_error",+ "child_exited",+ "child_no_request",+ "child_malformed_request",+ "busy",+ }+)+# Detached cleanup tasks held here so the garbage collector does not drop them# while they still have work to do, and so the "task exception was never# retrieved" warning cannot fire — each task adds a done-callback that logs@@ -2264,6 +2279,8 @@ async def _run_blocks_and_collect_debug(}if block.failure_reason:block_result["failure_reason"] = block.failure_reason+ if block.error_codes:+ block_result["error_codes"] = list(block.error_codes)if hasattr(block, "output") and block.output:block_result["extracted_data"] = block.outputif block.label is not None:@@ -2752,6 +2769,24 @@ def _detect_non_retriable_nav_error(result: dict[str, Any]) -> str | None:return next((reason for reason in iter_failure_reasons(result) if is_skip_inner_retry_error(reason)), None)+def _infrastructure_runner_error_codes(result: dict[str, Any]) -> list[str]:+ data = result.get("data")+ blocks = data.get("blocks") if isinstance(data, dict) else None+ if not isinstance(blocks, list):+ return []+ found: list[str] = []+ for block in blocks:+ if not isinstance(block, dict):+ continue+ codes = block.get("error_codes")+ if not isinstance(codes, list):+ continue+ for code in codes:+ if isinstance(code, str) and code in INFRASTRUCTURE_RUNNER_ERROR_CODES and code not in found:+ found.append(code)+ return found++def _update_verification_evidence_from_run_result(copilot_ctx: AgentContext, result: Mapping[str, object]) -> None:evidence = _workflow_verification_evidence(copilot_ctx)data_value = result.get("data")@@ -2888,6 +2923,7 @@ def _record_run_blocks_result(prior_budget_flag = copilot_ctx.last_failure_category_top == PER_TOOL_BUDGET_FAILURE_CATEGORYcopilot_ctx.last_failure_category_top = Nonecopilot_ctx.last_test_non_retriable_nav_error = None+ copilot_ctx.last_infrastructure_tool_error = Nonecopilot_ctx.post_run_page_observation_tool = Nonecopilot_ctx.post_run_page_observation_url = Nonecopilot_ctx.post_run_page_observation_workflow_run_id = None@@ -2897,6 +2933,22 @@ def _record_run_blocks_result(structured_blocker = _run_blocks_structured_blocker_message(result, copilot_ctx)anti_bot_match, empty_data_blocks, failure_categories = _analyze_run_blocks(result, copilot_ctx)+ infrastructure_runner_codes = _infrastructure_runner_error_codes(result)+ if infrastructure_runner_codes:+ copilot_ctx.last_infrastructure_tool_error = ", ".join(infrastructure_runner_codes)+ # Prepended, not appended: `last_failure_category_top` reads entry zero, and the+ # infrastructure fault outranks whatever the block's prose was classified as.+ failure_categories = [+ {+ "category": "UNRECOVERABLE_TOOL_ERROR",+ "confidence_float": 1.0,+ "reasoning": (+ "The code sandbox was unreachable "+ f"({copilot_ctx.last_infrastructure_tool_error}); no edit to the block can reach it."+ ),+ },+ *(failure_categories or []),+ ]artifact_flag_key = _artifact_challenge_flag_from_result(result, copilot_ctx)anti_bot_source = first_carrier_backed_anti_bot_source(failure_categories) if anti_bot_match else Noneanti_bot_evidence_source = anti_bot_source.value if anti_bot_source else Nonediff --git a/skyvern/forge/sdk/workflow/models/block.py b/skyvern/forge/sdk/workflow/models/block.pyindex bc7b433b1..5ce111bba 100644--- a/skyvern/forge/sdk/workflow/models/block.py+++ b/skyvern/forge/sdk/workflow/models/block.py@@ -21,6 +21,7 @@ import unicodedataimport uuidimport zipfilefrom collections import defaultdict, deque+from collections.abc import Sequencefrom datetime import date, datetime, time, timezonefrom email.message import EmailMessagefrom functools import partial@@ -440,6 +441,16 @@ def _maybe_truncate_loop_outputs(outputs_with_loop_values.append(last)+def build_block_failure_output(failure_reason: str, error_codes: Sequence[str]) -> dict[str, Any]:+ # SKY-7939: also surface the failure in `outputs.<block_label>` so callers+ # can tell which block failed without cross-referencing the timeline.+ return {+ "status": BlockStatus.failed.value,+ "failure_reason": failure_reason,+ "errors": [{"error_code": code, "reasoning": failure_reason, "confidence_float": 1.0} for code in error_codes],+ }++class Block(BaseModel, abc.ABC):"""Base class for workflow nodes (see branching spec [[s-4bnl]] for metadata semantics)."""@@ -5683,17 +5694,30 @@ async def wrapper({default_args}):secure_classification = self._classify_secure_runner_failure(secure_failure)async def build_secure_failure_result() -> BlockResult:- if secure_code_block_result.block_result is not None:- return secure_code_block_result.block_result+ engine_block_result = secure_code_block_result.block_result+ if engine_block_result is not None:+ await self.record_output_parameter_value(+ workflow_run_context,+ workflow_run_id,+ engine_block_result.output_parameter_value,+ )+ return engine_block_result+ secure_failure_reason = (+ secure_failure.failure_reason or "Code block failed in the secure runner"+ )+ secure_error_codes = [secure_failure.error_code] if secure_failure.error_code else []+ failure_output = build_block_failure_output(secure_failure_reason, secure_error_codes)+ await self.record_output_parameter_value(+ workflow_run_context, workflow_run_id, failure_output+ )return await self.build_block_result(success=False,- failure_reason=(- secure_failure.failure_reason or "Code block failed in the secure runner"- ),- output_parameter_value=None,+ failure_reason=secure_failure_reason,+ output_parameter_value=failure_output,status=BlockStatus.failed,workflow_run_block_id=workflow_run_block_id,organization_id=organization_id,+ error_codes=secure_error_codes or None,)return await self._resolve_failure_with_heal(@@ -8917,16 +8941,8 @@ class FileParserBlock(Block):organization_id: str | None,failure_reason: str,) -> BlockResult:- # SKY-7939: also surface the failure in `outputs.<block_label>` so callers- # can tell which block failed without cross-referencing the timeline.error_codes = self.get_failure_error_codes()- failure_output: dict[str, Any] = {- "status": BlockStatus.failed.value,- "failure_reason": failure_reason,- "errors": [- {"error_code": code, "reasoning": failure_reason, "confidence_float": 1.0} for code in error_codes- ],- }+ failure_output = build_block_failure_output(failure_reason, error_codes)await self.record_output_parameter_value(workflow_run_context, workflow_run_id, failure_output)return await self.build_block_result(success=False,diff --git a/tests/unit/test_copilot_agent_helpers.py b/tests/unit/test_copilot_agent_helpers.pyindex 59a5501a3..7af21a6e7 100644--- a/tests/unit/test_copilot_agent_helpers.py+++ b/tests/unit/test_copilot_agent_helpers.py@@ -23,6 +23,7 @@ from skyvern.forge.sdk.copilot.agent import (_VERIFIED_WORKFLOW_SUCCESS_REPLY,_build_goal_satisfied_exit_result,_resolve_wrapped_exception_exit_result,+ _rewrite_failed_test_response,_synthesized_block_offer_prompt,_verified_workflow_or_none,)@@ -7482,3 +7483,20 @@ def test_a_turn_that_forms_no_requested_output_still_dumps_its_trajectory(tmp_parecorded = json.loads(dumps[0].read_text())["scout_trajectory"]assert [step["credential_field"] for step in recorded] == ["totp"]+++def test_rewrite_names_the_sandbox_outage_when_the_runner_was_unreachable() -> None:+ ctx = _ctx(+ last_update_block_count=1,+ last_test_ok=False,+ last_test_failure_reason="Secure CodeBlock runner is unavailable. Please retry.",+ last_failure_category_top="UNRECOVERABLE_TOOL_ERROR",+ last_run_blocks_workflow_run_id="wr_runner",+ )++ rewritten = _rewrite_failed_test_response("All set — the workflow is ready.", ctx)++ assert rewritten == (+ "I created a draft workflow with 1 block and tested it, but the test failed. "+ "Failure: Secure CodeBlock runner is unavailable. Please retry.."+ )diff --git a/tests/unit/test_copilot_build_test_outcome.py b/tests/unit/test_copilot_build_test_outcome.pyindex f341949e4..b91a013db 100644--- a/tests/unit/test_copilot_build_test_outcome.py+++ b/tests/unit/test_copilot_build_test_outcome.py@@ -1747,3 +1747,53 @@ def test_an_all_literal_block_still_retires_when_the_call_is_gone() -> None:record_build_test_outcome(ctx, passing_run("wr_2", ["sign_in_and_read"]))assert unresolved_runtime_block_failure(ctx) is None+++def _secure_runner_failure_result(error_code: str, category: str | None) -> dict[str, object]:+ data: dict[str, object] = {+ "workflow_run_id": "wr_runner",+ "overall_status": "failed",+ "blocks": [+ {+ "label": "run_code",+ "block_type": "CODE",+ "status": "failed",+ "failure_reason": "Secure CodeBlock runner is unavailable. Please retry.",+ "error_codes": [error_code],+ }+ ],+ }+ if category is not None:+ data["failure_categories"] = [{"category": category, "confidence_float": 1.0, "reasoning": "sandbox"}]+ return {"ok": False, "data": data}+++def test_unrecoverable_tool_error_run_is_not_repairable_and_not_authoritative() -> None:+ outcome = recorded_outcome_from_run_blocks_result(+ _secure_runner_failure_result("runner_unavailable", "UNRECOVERABLE_TOOL_ERROR")+ )++ assert outcome is not None+ assert outcome.verdict != "repairable_failure"+ assert outcome.verdict == "not_authoritative"+ assert outcome.reason_code == "unrecoverable_tool_error"+ assert outcome.is_authoritative is False+ assert outcome.structural_failure_identity == ""+++def test_user_code_error_run_stays_repairable() -> None:+ outcome = recorded_outcome_from_run_blocks_result(+ _secure_runner_failure_result("user_code_error", "CODE_BLOCK_FAILURE")+ )++ assert outcome is not None+ assert outcome.verdict == "repairable_failure"+ assert outcome.reason_code == "runtime_block_failure"+++def test_runner_timeout_run_stays_repairable() -> None:+ outcome = recorded_outcome_from_run_blocks_result(_secure_runner_failure_result("timeout", "CODE_BLOCK_FAILURE"))++ assert outcome is not None+ assert outcome.verdict == "repairable_failure"+ assert outcome.reason_code == "runtime_block_failure"diff --git a/tests/unit/test_copilot_diagnosis_repair_contract.py b/tests/unit/test_copilot_diagnosis_repair_contract.pyindex 1b9d539ea..58f9d1bc5 100644--- a/tests/unit/test_copilot_diagnosis_repair_contract.py+++ b/tests/unit/test_copilot_diagnosis_repair_contract.py@@ -2377,3 +2377,67 @@ async def test_failed_seam_capture_preserves_clean_matching_evidence(ctx, run_session_id="run_session", run_id="wr_failed", current_url="https://example.test/app")assert ctx.composition_page_evidence is clean+++def _secure_runner_unavailable_result() -> dict[str, object]:+ return {+ "ok": False,+ "error": "Run failed.",+ "data": {+ "workflow_run_id": "wr_runner_unavailable",+ "overall_status": "failed",+ "blocks": [+ {+ "label": "run_code",+ "block_type": "CODE",+ "status": "failed",+ "failure_reason": "Secure CodeBlock runner is unavailable. Please retry.",+ "error_codes": ["runner_unavailable"],+ }+ ],+ },+ }+++def test_runner_unavailable_stops_even_with_a_code_authoring_repair_context() -> None:+ ctx = _ctx()+ result = _secure_runner_unavailable_result()+ run_execution_module._record_run_blocks_result(ctx, result, completion_verification=None)+ data = result["data"]+ assert isinstance(data, dict)+ data["authoring_repair_context"] = CodeAuthoringRepairContext(+ block_label="run_code",+ reason_code="ambiguous_bare_selector",+ selector="button",+ refiner_selector="xpath=//button[normalize-space()='Download']",+ ).model_dump(mode="json")++ contract = build_diagnosis_repair_contract(+ source_tool="update_and_run_blocks",+ result=result,+ ctx=ctx,+ )++ assert contract.diagnosis_result.suspected_failure_type == DiagnosisFailureType.UNRECOVERABLE_TOOL_ERROR+ assert contract.repair_decision.next_action == RepairNextAction.STOP+++def test_user_code_error_still_repairs_through_the_contract() -> None:+ ctx = _ctx()+ result = _secure_runner_unavailable_result()+ data = result["data"]+ assert isinstance(data, dict)+ blocks = data["blocks"]+ assert isinstance(blocks, list)+ blocks[0]["error_codes"] = ["user_code_error"]+ blocks[0]["failure_reason"] = "NameError: name 'undefined_helper' is not defined"+ run_execution_module._record_run_blocks_result(ctx, result, completion_verification=None)++ contract = build_diagnosis_repair_contract(+ source_tool="update_and_run_blocks",+ result=result,+ ctx=ctx,+ )++ assert contract.diagnosis_result.suspected_failure_type != DiagnosisFailureType.UNRECOVERABLE_TOOL_ERROR+ assert contract.repair_decision.next_action != RepairNextAction.STOPdiff --git a/tests/unit/test_copilot_run_output_content_gate.py b/tests/unit/test_copilot_run_output_content_gate.pyindex 0646512f4..e76974a5d 100644--- a/tests/unit/test_copilot_run_output_content_gate.py+++ b/tests/unit/test_copilot_run_output_content_gate.py@@ -28,6 +28,7 @@ from skyvern.forge.sdk.copilot.tools import (_is_unfinished_run_verification_candidate,_record_run_blocks_result,_run_blocks_structured_blocker_message,+ _tool_loop_error,run_execution,)from skyvern.forge.sdk.copilot.tools._shared import _registered_output_parameter_payloads@@ -1459,3 +1460,60 @@ def test_present_value_upgrade_flips_lone_judge_unknown_to_demonstrated_on_ok_ruassert outcome_fully_verified(ctx) is Trueassert ctx.last_test_suspicious_success is Falseassert ctx.last_full_workflow_test_ok is True+++def _failed_code_block_run_result(error_code: str) -> dict[str, Any]:+ return {+ "ok": False,+ "data": {+ "workflow_run_id": "wr_test",+ "overall_status": "failed",+ "blocks": [+ {+ "label": "run_code",+ "block_type": "CODE",+ "status": "failed",+ "failure_reason": "Secure CodeBlock runner is unavailable. Please retry.",+ "error_codes": [error_code],+ }+ ],+ },+ }++[email protected]("error_code", sorted(run_execution.INFRASTRUCTURE_RUNNER_ERROR_CODES))+def test_infrastructure_runner_code_prepends_unrecoverable_tool_error(error_code: str) -> None:+ result = _failed_code_block_run_result(error_code)+ ctx = _ctx(result["data"]["blocks"])++ _record_run_blocks_result(ctx, result, completion_verification=None)++ categories = result["data"]["failure_categories"]+ assert categories[0]["category"] == "UNRECOVERABLE_TOOL_ERROR"+ assert ctx.last_failure_category_top == "UNRECOVERABLE_TOOL_ERROR"+ assert ctx.last_infrastructure_tool_error == error_code++[email protected]("error_code", ["timeout", "user_code_error", "insecure_code_detected"])+def test_repairable_runner_code_injects_no_unrecoverable_category(error_code: str) -> None:+ result = _failed_code_block_run_result(error_code)+ ctx = _ctx(result["data"]["blocks"])++ _record_run_blocks_result(ctx, result, completion_verification=None)++ categories = result["data"].get("failure_categories") or []+ assert all(entry.get("category") != "UNRECOVERABLE_TOOL_ERROR" for entry in categories)+ assert ctx.last_failure_category_top != "UNRECOVERABLE_TOOL_ERROR"+ assert ctx.last_infrastructure_tool_error is None+++def test_pre_run_sandbox_refusal_leaves_the_within_turn_guard_disarmed() -> None:+ result = run_execution._copilot_sandbox_unavailable_result()+ ctx = _ctx()++ _record_run_blocks_result(ctx, result, completion_verification=None)++ assert result["data"]["workflow_run_id"] is None+ assert ctx.last_failure_category_top == "UNRECOVERABLE_TOOL_ERROR"+ assert ctx.last_infrastructure_tool_error is None+ assert _tool_loop_error(ctx, "edit_block") is Nonediff --git a/tests/unit/test_copilot_tool_loop_error_block_running.py b/tests/unit/test_copilot_tool_loop_error_block_running.pyindex 3e3265d93..8e1f78885 100644--- a/tests/unit/test_copilot_tool_loop_error_block_running.py+++ b/tests/unit/test_copilot_tool_loop_error_block_running.py@@ -13,6 +13,8 @@ import timefrom types import SimpleNamespacefrom typing import Any+import pytest+from skyvern.forge.sdk.copilot.enforcement import TOTAL_TIMEOUT_SECONDSfrom skyvern.forge.sdk.copilot.failure_tracking import compute_action_sequence_fingerprintfrom skyvern.forge.sdk.copilot.loop_detection import record_consecutive_tool_result_boundary_for_ctx@@ -21,7 +23,9 @@ from skyvern.forge.sdk.copilot.tools import (PER_TOOL_CALL_BUDGET_SECONDS,_active_block_run_budget_seconds,_tool_loop_error,+ blockers,)+from skyvern.forge.sdk.copilot.tools.run_execution import INFRASTRUCTURE_RUNNER_ERROR_CODESdef _ctx(**overrides: Any) -> Any:@@ -174,3 +178,48 @@ def test_bypass_applies_to_both_block_running_tool_names() -> None:ctx = _ctx()for _ in range(5):assert _tool_loop_error(ctx, "run_blocks_and_collect_debug") is None+++def test_infrastructure_runner_failure_blocks_authoring_and_running_tools() -> None:+ ctx = _ctx(last_infrastructure_tool_error="runner_unavailable")++ for tool in ("edit_block", "delete_block", "update_workflow", "update_and_run_blocks"):+ msg = _tool_loop_error(ctx, tool)+ assert msg is not None, f"{tool} should be refused"+ assert "runner_unavailable" in msg+ assert "code sandbox" in msg+++def test_infrastructure_runner_failure_leaves_reporting_tools_callable() -> None:+ ctx = _ctx(last_infrastructure_tool_error="runner_unavailable")++ for tool in ("get_run_results", "list_credentials"):+ assert _tool_loop_error(ctx, tool) is None, f"{tool} should stay callable"+++def test_edit_block_is_not_refused_without_an_infrastructure_runner_failure() -> None:+ ctx = _ctx(last_infrastructure_tool_error=None)++ assert _tool_loop_error(ctx, "edit_block") is None+++def test_infrastructure_runner_failure_preserves_the_unpersisted_draft(monkeypatch: pytest.MonkeyPatch) -> None:+ captured: list[Any] = []+ monkeypatch.setattr(+ blockers,+ "_emit_loop_plane_refusal",+ lambda ctx, signal: captured.append(signal) or "refused",+ )+ ctx = _ctx(last_infrastructure_tool_error="runner_unavailable", turn_halt=None)++ _tool_loop_error(ctx, "edit_block")++ (signal,) = captured+ assert "unchanged" in signal.user_facing_reason+ assert signal.preserves_workflow_draft is True+++def test_busy_runner_is_an_infrastructure_failure_not_an_authored_code_failure() -> None:+ assert "busy" in INFRASTRUCTURE_RUNNER_ERROR_CODES+ assert "timeout" not in INFRASTRUCTURE_RUNNER_ERROR_CODES+ assert "user_code_error" not in INFRASTRUCTURE_RUNNER_ERROR_CODES