Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
feat(SKY-12053): path-neutral heal chokepoint for both code-block engines (#7248)
pedrohsdb38b178f22f3a
diff --git a/skyvern/forge/agent.py b/skyvern/forge/agent.pyindex 0a7ebfeba..ef02e1fe9 100644--- a/skyvern/forge/agent.py+++ b/skyvern/forge/agent.py@@ -764,6 +764,7 @@ class ForgeAgent:cua_response: OpenAIResponse | None = None,llm_caller: LLMCaller | None = None,download_baseline_files: list[str] | None = None,+ pre_resolved_browser_state: BrowserState | None = None,) -> Tuple[Step, DetailedAgentStepOutput | None, Step | None]:# set the step_id and task_id in the contextcontext = skyvern_context.ensure_context()@@ -887,7 +888,13 @@ class ForgeAgent:step,browser_state,detailed_output,- ) = await self.initialize_execution_state(task, step, workflow_run, browser_session_id)+ ) = await self.initialize_execution_state(+ task,+ step,+ workflow_run,+ browser_session_id,+ pre_resolved_browser_state=pre_resolved_browser_state,+ )# mark step as completed and mark task as completedif (@@ -3346,8 +3353,11 @@ class ForgeAgent:step: Step,workflow_run: WorkflowRun | None = None,browser_session_id: str | None = None,+ pre_resolved_browser_state: BrowserState | None = None,) -> tuple[Step, BrowserState, DetailedAgentStepOutput]:- if workflow_run:+ if pre_resolved_browser_state is not None:+ browser_state = pre_resolved_browser_state+ elif workflow_run:browser_state = await app.BROWSER_MANAGER.get_or_create_for_workflow_run(workflow_run=workflow_run,url=task.url,diff --git a/skyvern/forge/agent_functions.py b/skyvern/forge/agent_functions.pyindex 9052db52f..e108a1610 100644--- a/skyvern/forge/agent_functions.py+++ b/skyvern/forge/agent_functions.py@@ -43,7 +43,7 @@ from skyvern.forge.sdk.services import google_drive_service, google_oauth_servicfrom skyvern.forge.sdk.services.credentials import AuthenticatorTotpParseResultfrom skyvern.forge.sdk.trace import tracedfrom skyvern.forge.sdk.workflow.models.block import BlockTypeVar-from skyvern.schemas.workflows import FileStorageType, FileUploadDestination+from skyvern.schemas.workflows import BlockResult, FileStorageType, FileUploadDestinationfrom skyvern.services.otp_gmail import GmailOTPVerificationContextfrom skyvern.webeye.actions.actions import Actionfrom skyvern.webeye.browser_state import BrowserState@@ -54,6 +54,7 @@ from skyvern.webeye.utils.page import SkyvernFrameif TYPE_CHECKING:from skyvern.forge.sdk.db.enums import WorkflowRunTriggerTypefrom skyvern.forge.sdk.workflow.context_manager import WorkflowRunContext+ from skyvern.forge.sdk.workflow.models.code_block_recorder import RecordingPagefrom skyvern.forge.sdk.workflow.models.workflow import Workflow, WorkflowRun, WorkflowRunStatusfrom skyvern.services.otp_service import OTPValue@@ -116,6 +117,22 @@ class FieldOptionResolution:fallback_to_llm: bool+@dataclass+class CodeBlockEngineFailure:+ error_code: str | None+ safe_message: str | None+ failure_reason: str | None+ exception_class: str | None+ failing_line: int | None+ healability_hint: bool | None+++@dataclass+class CodeBlockEngineResult:+ block_result: BlockResult | None+ failure: CodeBlockEngineFailure | None++def _remove_rect(element: dict) -> None:if "rect" in element:del element["rect"]@@ -819,7 +836,8 @@ class AgentFunction:workflow_run_context: "WorkflowRunContext",parameter_values: dict[str, Any],credential_parameter_keys: set[str],- ) -> Any | None:+ recording_page: "RecordingPage | None" = None,+ ) -> CodeBlockEngineResult | None:"""Run a CodeBlock through the secure runner sidecar, or return None for legacy.OSS no-op returns None so callers fall through to in-process execution. Clouddiff --git a/skyvern/forge/sdk/workflow/models/block.py b/skyvern/forge/sdk/workflow/models/block.pyindex 4eff7049a..3beb39e27 100644--- a/skyvern/forge/sdk/workflow/models/block.py+++ b/skyvern/forge/sdk/workflow/models/block.py@@ -21,7 +21,7 @@ from datetime import date, datetime, time, timezonefrom email.message import EmailMessagefrom pathlib import Path, PurePosixPathfrom types import SimpleNamespace-from typing import Annotated, Any, Awaitable, Callable, ClassVar, Literal, Union, cast+from typing import TYPE_CHECKING, Annotated, Any, Awaitable, Callable, ClassVar, Literal, Union, castfrom urllib.parse import quote, urlparseimport aiofiles@@ -159,6 +159,7 @@ from skyvern.forge.sdk.workflow.models.parameter import (WorkflowParameterType,)from skyvern.schemas.runs import RunEngine+from skyvern.schemas.self_heal import HealClassification, HealSkipReason, OutputObligationfrom skyvern.schemas.workflows import (AIFallbackMode,BlockResult,@@ -180,6 +181,9 @@ from skyvern.webeye.browser_factory import rebind_download_dirfrom skyvern.webeye.browser_state import BrowserStatefrom skyvern.webeye.utils.page import SkyvernFrame+if TYPE_CHECKING:+ from skyvern.forge.agent_functions import CodeBlockEngineFailure+LOG = structlog.get_logger()@@ -4132,15 +4136,21 @@ async def wrapper({default_args}):workflow_run_block_id: str,organization_id: str | None,browser_session_id: str | None,- browser_state: BrowserState,- page: Page,+ browser_state: BrowserState | None = None,+ page: Page | None = None,+ classification: HealClassification | None = None,+ record_output_parameter: bool = True,) -> BlockResult | None:"""Run one bounded agent mini-run on the same workflow-run browser to finish the block's goal(narrowed to the failing step when one is confidently matched). Returns a BlockResult when aheal was attempted, or None to fall through to the caller's fail-closed path."""if not await self._self_heal_enabled(workflow_run_context):return None- if not self._is_healable_page_failure(exception, recording_page):+ effective_classification = classification or HealClassification(+ healable=self._is_healable_page_failure(exception, recording_page),+ skip_reason=None,+ )+ if not effective_classification.healable:return Noneif not self.prompt:return None@@ -4190,7 +4200,7 @@ async def wrapper({default_args}):escalation_url = (self._derive_escalation_navigation_url(failing_line, recording_page) if failing_line is not None else "")- if escalation_url:+ if escalation_url and browser_state is not None and page is not None:# BROWSER_MANAGER can early-return a cached browser state without ever reading# task.url, so escalation_task.url alone is not reliable navigation recourse.# Drive the live browser_state/page the heal will run on directly instead.@@ -4260,6 +4270,7 @@ async def wrapper({default_args}):task_block=None,browser_session_id=browser_session_id,close_browser_on_completion=False,+ pre_resolved_browser_state=browser_state,)finally:current_context.task_id = previous_task_id@@ -4294,7 +4305,10 @@ async def wrapper({default_args}):)task_output = TaskOutput.from_task(updated_task, downloaded_files)output_parameter_value = workflow_run_context.mask_secrets_in_data(task_output.model_dump())- await self.record_output_parameter_value(workflow_run_context, workflow_run_id, output_parameter_value)+ if record_output_parameter:+ await self.record_output_parameter_value(+ workflow_run_context, workflow_run_id, output_parameter_value+ )await self._finalize_recovery_block(recovery_block_id, BlockStatus.completed, organization_id)return await self.build_block_result(success=True,@@ -4333,6 +4347,106 @@ async def wrapper({default_args}):await self._fail_escalation_task(escalation_task, escalation_step, recovery_block_id, organization_id)return None+ @staticmethod+ def _is_playwright_exception_class_name(exception_class: str | None) -> bool:+ if exception_class is None:+ return False+ lowered = exception_class.lower()+ if not lowered.startswith("playwright."):+ return False+ return lowered.rsplit(".", 1)[-1].endswith("error")++ @staticmethod+ def _secure_skip_reason_for_error_code(error_code: str | None) -> HealSkipReason:+ if error_code == "timeout":+ return HealSkipReason.timeout_class+ if error_code == "insecure_code_detected":+ return HealSkipReason.insecure_code+ return HealSkipReason.unclassifiable++ def _classify_secure_runner_failure(self, failure: CodeBlockEngineFailure | None) -> HealClassification:+ if failure is None:+ return HealClassification(healable=False, skip_reason=HealSkipReason.unclassifiable)+ has_structured = (+ failure.exception_class is not None+ or failure.failing_line is not None+ or failure.healability_hint is not None+ )+ if not has_structured:+ # No healability_hint to disambiguate, so only the page-class code heals blind;+ # user_code_error is ambiguous (could be a bare `raise`) and stays fail-closed.+ if failure.error_code == "unsupported_page_operation":+ return HealClassification(healable=True, skip_reason=None)+ return HealClassification(+ healable=False,+ skip_reason=self._secure_skip_reason_for_error_code(failure.error_code),+ )+ if failure.healability_hint is True:+ return HealClassification(healable=True, skip_reason=None)+ if failure.healability_hint is None and self._is_playwright_exception_class_name(failure.exception_class):+ return HealClassification(healable=True, skip_reason=None)+ return HealClassification(+ healable=False,+ skip_reason=self._secure_skip_reason_for_error_code(failure.error_code),+ )++ async def _resolve_failure_with_heal(+ self,+ *,+ exception: Exception | None,+ failing_line: int | None,+ build_failure_result: Callable[[], Awaitable[BlockResult]],+ classification: HealClassification,+ recorder: CodeBlockActionRecording,+ workflow_run_context: WorkflowRunContext,+ workflow_run_id: str,+ workflow_run_block_id: str,+ organization_id: str | None,+ browser_session_id: str | None,+ browser_state: BrowserState | None = None,+ page: Page | None = None,+ ) -> BlockResult:+ healed: BlockResult | None = None+ if classification.healable:+ live_browser_state = browser_state or app.BROWSER_MANAGER.get_for_workflow_run(+ workflow_run_id=workflow_run_id+ )+ exception_for_heal = exception or RuntimeError("CodeBlock failed")+ healed = await self._attempt_self_heal(+ exception=exception_for_heal,+ failing_line=failing_line,+ recording_page=recorder.recording_page,+ classification=classification,+ workflow_run_context=workflow_run_context,+ workflow_run_id=workflow_run_id,+ workflow_run_block_id=workflow_run_block_id,+ organization_id=organization_id,+ browser_session_id=browser_session_id,+ browser_state=live_browser_state,+ page=page,+ record_output_parameter=False,+ )+ if healed is not None:+ output_obligation = OutputObligation.none+ if healed.success:+ output_obligation = (+ OutputObligation.observed+ if healed.output_parameter_value is not None+ else OutputObligation.vestigial+ )+ # Record output before finalizing so a failed write fails closed, never leaving a+ # completed block without the output downstream consumers require.+ if output_obligation in {OutputObligation.observed, OutputObligation.vestigial}:+ await self.record_output_parameter_value(+ workflow_run_context,+ workflow_run_id,+ healed.output_parameter_value,+ )+ await recorder.finalize(success=healed.success)+ return healed+ await recorder.finalize(success=False)+ return await build_failure_result()+async def execute(self,workflow_run_id: str,@@ -4484,26 +4598,6 @@ async def wrapper({default_args}):workflow_run_block_id=workflow_run_block_id,block_label=self.label,)- if use_codeblock_runner:- secure_code_block_result = await app.AGENT_FUNCTION.execute_code_block_override(- block=self,- workflow_run_id=workflow_run_id,- workflow_run_block_id=workflow_run_block_id,- organization_id=organization_id,- browser_session_id=browser_session_id,- workflow_run_context=workflow_run_context,- parameter_values=parameter_values,- credential_parameter_keys=credential_parameter_keys,- )- LOG.info(- "Secure CodeBlock override returned",- override_returned_none=secure_code_block_result is None,- workflow_run_id=workflow_run_id,- workflow_run_block_id=workflow_run_block_id,- block_label=self.label,- )- if secure_code_block_result is not None:- return secure_code_block_result# A prompt-bearing code block gets a task v1 + step so its recorded calls render through# the standard action/artifact timeline and the agent can later take over on failure.@@ -4521,6 +4615,87 @@ async def wrapper({default_args}):try:await recorder.link_block()+ if use_codeblock_runner:+ secure_code_block_result = await app.AGENT_FUNCTION.execute_code_block_override(+ block=self,+ workflow_run_id=workflow_run_id,+ workflow_run_block_id=workflow_run_block_id,+ organization_id=organization_id,+ browser_session_id=browser_session_id,+ workflow_run_context=workflow_run_context,+ parameter_values=parameter_values,+ credential_parameter_keys=credential_parameter_keys,+ recording_page=recording_page,+ )+ LOG.info(+ "Secure CodeBlock override returned",+ override_returned_none=secure_code_block_result is None,+ workflow_run_id=workflow_run_id,+ workflow_run_block_id=workflow_run_block_id,+ block_label=self.label,+ )+ if secure_code_block_result is not None:+ await recorder.persist(recorder.recorded_actions())+ if secure_code_block_result.failure is not None:+ secure_failure = secure_code_block_result.failure+ secure_classification = self._classify_secure_runner_failure(secure_failure)++ async def build_secure_failure_result() -> BlockResult:+ if secure_code_block_result.block_result is not None:+ return secure_code_block_result.block_result+ return await self.build_block_result(+ success=False,+ failure_reason=(+ secure_failure.failure_reason or "Code block failed in the secure runner"+ ),+ output_parameter_value=None,+ status=BlockStatus.failed,+ workflow_run_block_id=workflow_run_block_id,+ organization_id=organization_id,+ )++ return await self._resolve_failure_with_heal(+ exception=None,+ failing_line=secure_failure.failing_line,+ build_failure_result=build_secure_failure_result,+ classification=secure_classification,+ recorder=recorder,+ workflow_run_context=workflow_run_context,+ workflow_run_id=workflow_run_id,+ workflow_run_block_id=workflow_run_block_id,+ organization_id=organization_id,+ browser_session_id=browser_session_id,+ browser_state=browser_state,+ page=page,+ )+ if secure_code_block_result.block_result is None:+ LOG.warning(+ "Secure CodeBlock runner returned success without block result",+ workflow_run_id=workflow_run_id,+ workflow_run_block_id=workflow_run_block_id,+ block_label=self.label,+ )+ await recorder.finalize(success=False)+ return await self.build_block_result(+ success=False,+ failure_reason="Secure code block runner returned no result",+ output_parameter_value=None,+ status=BlockStatus.failed,+ workflow_run_block_id=workflow_run_block_id,+ organization_id=organization_id,+ )+ # failure=None does not imply success: infra arms (no browser/page, runner raise,+ # invalid output) return a failed block_result with no healable failure metadata.+ if not secure_code_block_result.block_result.success:+ await recorder.finalize(success=False)+ return secure_code_block_result.block_result+ await recorder.finalize(success=True)+ await self.record_output_parameter_value(+ workflow_run_context,+ workflow_run_id,+ secure_code_block_result.block_result.output_parameter_value,+ )+ return secure_code_block_result.block_resultuser_function = self.generate_async_user_function(self.code, recording_page, parameter_values)result = await self.execute_user_function_with_timeout(user_function,@@ -4571,10 +4746,28 @@ async def wrapper({default_args}):))await recorder.persist(recorded)- healed = await self._attempt_self_heal(+ legacy_healable = self._is_healable_page_failure(e, recording_page)+ legacy_classification = HealClassification(+ healable=legacy_healable,+ skip_reason=None if legacy_healable else HealSkipReason.unclassifiable,+ )++ async def build_legacy_failure_result() -> BlockResult:+ return await self.build_block_result(+ success=False,+ failure_reason=failure_reason,+ output_parameter_value=None,+ status=BlockStatus.failed,+ workflow_run_block_id=workflow_run_block_id,+ organization_id=organization_id,+ )++ return await self._resolve_failure_with_heal(exception=e,failing_line=failing_line,- recording_page=recording_page,+ build_failure_result=build_legacy_failure_result,+ classification=legacy_classification,+ recorder=recorder,workflow_run_context=workflow_run_context,workflow_run_id=workflow_run_id,workflow_run_block_id=workflow_run_block_id,@@ -4583,20 +4776,6 @@ async def wrapper({default_args}):browser_state=browser_state,page=page,)- if healed is not None:- # Finalize the seat task to the heal outcome before the idempotent `finally` no-ops it,- # so a healed success no longer leaves the seat row failed under a completed block.- await recorder.finalize(success=healed.success)- return healed- await recorder.finalize(success=False)- return await self.build_block_result(- success=False,- failure_reason=failure_reason,- output_parameter_value=None,- status=BlockStatus.failed,- workflow_run_block_id=workflow_run_block_id,- organization_id=organization_id,- )else:await recorder.persist(recorder.recorded_actions())diff --git a/skyvern/schemas/self_heal.py b/skyvern/schemas/self_heal.pynew file mode 100644index 000000000..033c93940--- /dev/null+++ b/skyvern/schemas/self_heal.py@@ -0,0 +1,93 @@+from __future__ import annotations++from dataclasses import dataclass+from datetime import datetime+from enum import StrEnum+from typing import Literal++from pydantic import BaseModel, ConfigDict+++class HealStatus(StrEnum):+ # fired_unverified: the heal ran but its outcome could not be verified against the block's+ # goal — distinct from fired_completed/fired_failed so an unverifiable heal is never recorded+ # as a definite success or failure (a light-proxy trap flagged in review).+ fired_completed = "fired_completed"+ fired_failed = "fired_failed"+ fired_unverified = "fired_unverified"+ skipped = "skipped"+++class HealSkipReason(StrEnum):+ capped = "capped"+ adoption_failed = "adoption_failed"+ credential_unavailable = "credential_unavailable"+ timeout_class = "timeout_class"+ insecure_code = "insecure_code"+ unclassifiable = "unclassifiable"+++class OutputObligation(StrEnum):+ none = "none"+ vestigial = "vestigial"+ observed = "observed"+++@dataclass(frozen=True)+class HealClassification:+ healable: bool+ skip_reason: HealSkipReason | None+++class HealEpisode(BaseModel):+ model_config = ConfigDict(from_attributes=True)++ heal_episode_id: str+ organization_id: str+ workflow_permanent_id: str+ workflow_id: str+ workflow_run_id: str+ workflow_run_block_id: str+ block_label: str+ engine: str+ status: HealStatus+ skip_reason: HealSkipReason | None = None+ block_prompt: str | None = None+ block_code: str | None = None+ block_steps: list | dict | None = None+ snapshot_available: bool = False+ convergence_eligible: bool = False+ parameter_binding_keys: list[str] | dict | None = None+ exception_class: str | None = None+ failing_line: int | None = None+ matched_step_index: int | None = None+ failure_message: str | None = None+ escalation_task_id: str | None = None+ wall_clock_ms: int | None = None+ action_count: int | None = None+ output_obligation: OutputObligation | None = None+ dom_snapshot_artifact_id: str | None = None+ scout_transcript_artifact_id: str | None = None+ screenshot_artifact_id: str | None = None+ created_at: datetime+ modified_at: datetime+++class WorkflowHealProposal(BaseModel):+ model_config = ConfigDict(from_attributes=True)++ heal_proposal_id: str+ organization_id: str+ workflow_permanent_id: str+ block_label: str+ candidate_definition: dict | list+ provenance: dict | list | None = None+ episode_ids: list[str]+ rendered_diff: str | None = None+ base_version: int+ base_definition_hash: str+ status: Literal["proposed", "adopted", "rejected", "stale"] = "proposed"+ adopted_workflow_id: str | None = None+ episode_window: str | None = None+ created_at: datetime+ modified_at: datetimediff --git a/tests/unit/workflow/test_code_block_escalation.py b/tests/unit/workflow/test_code_block_escalation.pyindex 27e33fcdf..764eac346 100644--- a/tests/unit/workflow/test_code_block_escalation.py+++ b/tests/unit/workflow/test_code_block_escalation.py@@ -16,6 +16,7 @@ import pytestfrom playwright.async_api import TimeoutError as PlaywrightTimeoutErrorfrom skyvern.forge import app+from skyvern.forge.agent_functions import CodeBlockEngineFailure, CodeBlockEngineResultfrom skyvern.forge.sdk.core import skyvern_contextfrom skyvern.forge.sdk.core.skyvern_context import SkyvernContextfrom skyvern.forge.sdk.schemas.files import FileInfo@@ -23,6 +24,7 @@ from skyvern.forge.sdk.schemas.tasks import TaskStatusfrom skyvern.forge.sdk.workflow.context_manager import WorkflowRunContextfrom skyvern.forge.sdk.workflow.models.block import BlockResult, BlockStatus, BlockType, CodeBlock, CodeBlockStepfrom skyvern.forge.sdk.workflow.models.parameter import OutputParameter, ParameterType+from skyvern.schemas.self_heal import HealSkipReasonfrom skyvern.webeye.actions.actions import ActionSECRET_VALUE = "hunter2-super-secret"@@ -120,6 +122,7 @@ def _install_db_fakes("created_actions": [],"recovery_block_kwargs": None,"recovery_block_updates": [],+ "workflow_run_block_updates": [],}async def _create_task(**kwargs: object) -> _FakeTask:@@ -172,7 +175,9 @@ def _install_db_fakes(return SimpleNamespace(workflow_run_block_id="wrb_recovery")async def _update_workflow_run_block(**kwargs: object) -> None:- state["recovery_block_updates"].append(kwargs)+ state["workflow_run_block_updates"].append(kwargs)+ if kwargs.get("workflow_run_block_id") == "wrb_recovery":+ state["recovery_block_updates"].append(kwargs)monkeypatch.setattr(app.DATABASE.observer, "create_workflow_run_block", AsyncMock(side_effect=_create_workflow_run_block)@@ -200,6 +205,42 @@ def _browser_state() -> MagicMock:return browser_state+class FakeRecorder:+ instances: list[FakeRecorder] = []+ _next_last_exception: Exception | None = None++ @classmethod+ def reset(cls, *, last_recorded_exception: Exception | None = None) -> None:+ cls.instances = []+ cls._next_last_exception = last_recorded_exception++ def __init__(self, **kwargs: Any) -> None:+ self.recording_page = MagicMock()+ self.recording_page.last_recorded_exception = MagicMock(return_value=self._next_last_exception)+ self._actions: list[Any] = []+ self.finalized_success: bool | None = None+ self.__class__.instances.append(self)++ async def create_task_and_step(self) -> None:+ return None++ async def link_block(self) -> None:+ return None++ def recorded_actions(self) -> list[Any]:+ return list(self._actions)++ def last_recorded_exception(self) -> Exception | None:+ return self._next_last_exception++ async def persist(self, actions: list[Any]) -> None:+ return None++ async def finalize(self, success: bool) -> None:+ if self.finalized_success is None:+ self.finalized_success = success++async def _heal(block: CodeBlock,context: WorkflowRunContext,@@ -225,6 +266,36 @@ async def _heal()+def _statuses_for_block(+ state: dict[str, Any],+ workflow_run_block_id: str,+) -> list[BlockStatus]:+ statuses: list[BlockStatus] = []+ for update in state["workflow_run_block_updates"]:+ if update.get("workflow_run_block_id") != workflow_run_block_id:+ continue+ status = update.get("status")+ if isinstance(status, BlockStatus):+ statuses.append(status)+ return statuses+++def _patch_execute_chokepoint_environment(+ monkeypatch: pytest.MonkeyPatch,+ *,+ context: WorkflowRunContext,+ fake_browser_state: object,+ use_codeblock_runner: bool,+) -> None:+ monkeypatch.setattr(CodeBlock, "get_workflow_run_context", MagicMock(return_value=context))+ monkeypatch.setattr(CodeBlock, "get_or_create_browser_state", AsyncMock(return_value=fake_browser_state))+ monkeypatch.setattr(CodeBlock, "_ensure_run_recording_artifact", AsyncMock(return_value=None))+ monkeypatch.setattr(CodeBlock, "format_potential_template_parameters", MagicMock(return_value=None))+ monkeypatch.setattr(app.AGENT_FUNCTION, "validate_code_block", AsyncMock(return_value=None))+ monkeypatch.setattr(app.AGENT_FUNCTION, "should_use_codeblock_runner", AsyncMock(return_value=use_codeblock_runner))+ monkeypatch.setattr(app.BROWSER_MANAGER, "get_for_workflow_run", MagicMock(return_value=fake_browser_state))++@pytest.mark.asyncioasync def test_everything_off_is_no_op(monkeypatch: pytest.MonkeyPatch) -> None:monkeypatch.setattr("skyvern.config.settings.ENABLE_CODE_BLOCK_SELF_HEALING", False, raising=False)@@ -1188,3 +1259,485 @@ async def test_escalation_task_verifies_with_action_history(assert result is not None and result.success is Trueassert state["create_task_kwargs"]["include_action_history_in_verification"] is True+++ ("error_code", "healable", "skip_reason"),+ [+ ("unsupported_page_operation", True, None),+ ("timeout", False, HealSkipReason.timeout_class),+ ("insecure_code_detected", False, HealSkipReason.insecure_code),+ ("browser_disconnected", False, HealSkipReason.unclassifiable),+ ("user_code_error", False, HealSkipReason.unclassifiable),+ ("busy", False, HealSkipReason.unclassifiable),+ ],+)+def test_secure_runner_degraded_classification_from_error_code(+ error_code: str, healable: bool, skip_reason: HealSkipReason | None+) -> None:+ block = _make_code_block()+ classification = block._classify_secure_runner_failure(+ CodeBlockEngineFailure(+ error_code=error_code,+ safe_message=None,+ failure_reason=None,+ exception_class=None,+ failing_line=None,+ healability_hint=None,+ )+ )+ assert classification.healable is healable+ assert classification.skip_reason == skip_reason+++def test_secure_runner_classification_uses_playwright_class_when_hint_is_unknown() -> None:+ block = _make_code_block()+ classification = block._classify_secure_runner_failure(+ CodeBlockEngineFailure(+ error_code="busy",+ safe_message=None,+ failure_reason=None,+ exception_class="playwright._impl._errors.TimeoutError",+ failing_line=3,+ healability_hint=None,+ )+ )+ assert classification.healable is True+ assert classification.skip_reason is None+++def test_secure_runner_classification_treats_browser_disconnected_as_non_healable() -> None:+ block = _make_code_block()+ classification = block._classify_secure_runner_failure(+ CodeBlockEngineFailure(+ error_code="browser_disconnected",+ safe_message=None,+ failure_reason=None,+ exception_class="codeblock.page_operation_broker.BrowserDisconnectedError",+ failing_line=3,+ healability_hint=None,+ )+ )+ assert classification.healable is False+ assert classification.skip_reason == HealSkipReason.unclassifiable+++async def test_self_heal_passes_pre_resolved_browser_state_by_identity(monkeypatch: pytest.MonkeyPatch) -> None:+ monkeypatch.setattr("skyvern.config.settings.ENABLE_CODE_BLOCK_SELF_HEALING", True, raising=False)+ state = _install_db_fakes(monkeypatch, final_status=TaskStatus.completed)+ block = _make_code_block(steps=[CodeBlockStep(description="download", line_start=1, line_end=1)])+ browser_state = object()+ exc = RuntimeError("rotted selector")++ with skyvern_context.scoped(SkyvernContext(organization_id="o_test", workflow_run_id="wr_test")):+ result = await block._attempt_self_heal(+ exception=exc,+ failing_line=1,+ recording_page=_recording_page(exc),+ workflow_run_context=_make_context(enable_self_healing=True),+ workflow_run_id="wr_test",+ workflow_run_block_id="wrb_test",+ organization_id="o_test",+ browser_session_id=None,+ browser_state=browser_state,+ )++ assert result is not None+ assert state["execute_step_kwargs"]["pre_resolved_browser_state"] is browser_state+++async def test_legacy_heal_success_skips_failed_write_and_ends_completed(monkeypatch: pytest.MonkeyPatch) -> None:+ monkeypatch.setattr("skyvern.config.settings.ENABLE_CODE_BLOCK_SELF_HEALING", False, raising=False)+ state = _install_db_fakes(monkeypatch, final_status=TaskStatus.completed)+ block = _make_code_block(steps=[CodeBlockStep(description="download", line_start=1, line_end=1)])+ context = _make_context(enable_self_healing=True)+ fake_page = MagicMock()+ fake_browser_state = SimpleNamespace(get_working_page=AsyncMock(return_value=fake_page))+ _patch_execute_chokepoint_environment(+ monkeypatch,+ context=context,+ fake_browser_state=fake_browser_state,+ use_codeblock_runner=False,+ )+ raised = RuntimeError("rotted selector")+ monkeypatch.setattr(CodeBlock, "execute_user_function_with_timeout", AsyncMock(side_effect=raised))+ monkeypatch.setattr(app.AGENT_FUNCTION, "execute_code_block_override", AsyncMock(return_value=None))+ FakeRecorder.reset(last_recorded_exception=raised)+ monkeypatch.setattr("skyvern.forge.sdk.workflow.models.block.CodeBlockActionRecording", FakeRecorder)++ async def _heal_success(**kwargs: Any) -> BlockResult:+ return await block.build_block_result(+ success=True,+ failure_reason=None,+ output_parameter_value={"task_id": "tsk_escalation", "status": "completed"},+ status=BlockStatus.completed,+ workflow_run_block_id=kwargs["workflow_run_block_id"],+ organization_id=kwargs["organization_id"],+ )++ heal_mock = AsyncMock(side_effect=_heal_success)+ monkeypatch.setattr(block, "_attempt_self_heal", heal_mock)+ record_output_mock = AsyncMock(return_value=None)+ monkeypatch.setattr(CodeBlock, "record_output_parameter_value", record_output_mock)++ result = await block.execute(+ workflow_run_id="wr_test",+ workflow_run_block_id="wrb_test",+ organization_id="o_test",+ browser_session_id="pbs_test",+ )++ block_statuses = _statuses_for_block(state, "wrb_test")+ assert result.success is True+ assert heal_mock.await_count == 1+ assert BlockStatus.failed not in block_statuses+ assert block_statuses[-1] == BlockStatus.completed+ assert record_output_mock.await_count == 1+ assert len(FakeRecorder.instances) == 1+ assert FakeRecorder.instances[0].finalized_success is True+++async def test_heal_output_write_failure_does_not_finalize_completed(monkeypatch: pytest.MonkeyPatch) -> None:+ monkeypatch.setattr("skyvern.config.settings.ENABLE_CODE_BLOCK_SELF_HEALING", False, raising=False)+ _install_db_fakes(monkeypatch, final_status=TaskStatus.completed)+ block = _make_code_block(steps=[CodeBlockStep(description="download", line_start=1, line_end=1)])+ context = _make_context(enable_self_healing=True)+ fake_page = MagicMock()+ fake_browser_state = SimpleNamespace(get_working_page=AsyncMock(return_value=fake_page))+ _patch_execute_chokepoint_environment(+ monkeypatch,+ context=context,+ fake_browser_state=fake_browser_state,+ use_codeblock_runner=False,+ )+ raised = RuntimeError("rotted selector")+ monkeypatch.setattr(CodeBlock, "execute_user_function_with_timeout", AsyncMock(side_effect=raised))+ monkeypatch.setattr(app.AGENT_FUNCTION, "execute_code_block_override", AsyncMock(return_value=None))+ FakeRecorder.reset(last_recorded_exception=raised)+ monkeypatch.setattr("skyvern.forge.sdk.workflow.models.block.CodeBlockActionRecording", FakeRecorder)++ async def _heal_success(**kwargs: Any) -> BlockResult:+ return await block.build_block_result(+ success=True,+ failure_reason=None,+ output_parameter_value={"task_id": "tsk_escalation", "status": "completed"},+ status=BlockStatus.completed,+ workflow_run_block_id=kwargs["workflow_run_block_id"],+ organization_id=kwargs["organization_id"],+ )++ monkeypatch.setattr(block, "_attempt_self_heal", AsyncMock(side_effect=_heal_success))+ write_error = RuntimeError("output parameter write failed")+ record_output_mock = AsyncMock(side_effect=write_error)+ monkeypatch.setattr(CodeBlock, "record_output_parameter_value", record_output_mock)++ with pytest.raises(RuntimeError, match="output parameter write failed"):+ await block.execute(+ workflow_run_id="wr_test",+ workflow_run_block_id="wrb_test",+ organization_id="o_test",+ browser_session_id="pbs_test",+ )++ assert record_output_mock.await_count == 1+ assert len(FakeRecorder.instances) == 1+ assert FakeRecorder.instances[0].finalized_success is not True+++async def test_secure_heal_success_skips_failed_write_and_ends_completed(monkeypatch: pytest.MonkeyPatch) -> None:+ monkeypatch.setattr("skyvern.config.settings.ENABLE_CODE_BLOCK_SELF_HEALING", False, raising=False)+ state = _install_db_fakes(monkeypatch, final_status=TaskStatus.completed)+ block = _make_code_block(steps=[CodeBlockStep(description="download", line_start=1, line_end=1)])+ context = _make_context(enable_self_healing=True)+ fake_page = MagicMock()+ fake_browser_state = SimpleNamespace(get_working_page=AsyncMock(return_value=fake_page))+ _patch_execute_chokepoint_environment(+ monkeypatch,+ context=context,+ fake_browser_state=fake_browser_state,+ use_codeblock_runner=True,+ )+ monkeypatch.setattr(+ app.AGENT_FUNCTION,+ "execute_code_block_override",+ AsyncMock(+ return_value=CodeBlockEngineResult(+ block_result=None,+ failure=CodeBlockEngineFailure(+ error_code="user_code_error",+ safe_message=None,+ failure_reason="CodeBlock failed while running user code.",+ exception_class="playwright._impl._errors.TimeoutError",+ failing_line=2,+ healability_hint=True,+ ),+ )+ ),+ )+ FakeRecorder.reset(last_recorded_exception=None)+ monkeypatch.setattr("skyvern.forge.sdk.workflow.models.block.CodeBlockActionRecording", FakeRecorder)++ async def _heal_success(**kwargs: Any) -> BlockResult:+ return await block.build_block_result(+ success=True,+ failure_reason=None,+ output_parameter_value={"task_id": "tsk_escalation", "status": "completed"},+ status=BlockStatus.completed,+ workflow_run_block_id=kwargs["workflow_run_block_id"],+ organization_id=kwargs["organization_id"],+ )++ heal_mock = AsyncMock(side_effect=_heal_success)+ monkeypatch.setattr(block, "_attempt_self_heal", heal_mock)+ record_output_mock = AsyncMock(return_value=None)+ monkeypatch.setattr(CodeBlock, "record_output_parameter_value", record_output_mock)++ result = await block.execute(+ workflow_run_id="wr_test",+ workflow_run_block_id="wrb_test",+ organization_id="o_test",+ browser_session_id="pbs_test",+ )++ block_statuses = _statuses_for_block(state, "wrb_test")+ assert result.success is True+ assert heal_mock.await_count == 1+ assert heal_mock.await_args.kwargs["page"] is fake_page+ assert heal_mock.await_args.kwargs["browser_state"] is fake_browser_state+ assert BlockStatus.failed not in block_statuses+ assert block_statuses[-1] == BlockStatus.completed+ assert record_output_mock.await_count == 1+ assert len(FakeRecorder.instances) == 1+ assert FakeRecorder.instances[0].finalized_success is True+++async def test_legacy_heal_declined_writes_failed_once(monkeypatch: pytest.MonkeyPatch) -> None:+ monkeypatch.setattr("skyvern.config.settings.ENABLE_CODE_BLOCK_SELF_HEALING", False, raising=False)+ state = _install_db_fakes(monkeypatch, final_status=TaskStatus.completed)+ block = _make_code_block(steps=[CodeBlockStep(description="download", line_start=1, line_end=1)])+ context = _make_context(enable_self_healing=True)+ fake_page = MagicMock()+ fake_browser_state = SimpleNamespace(get_working_page=AsyncMock(return_value=fake_page))+ _patch_execute_chokepoint_environment(+ monkeypatch,+ context=context,+ fake_browser_state=fake_browser_state,+ use_codeblock_runner=False,+ )+ raised = RuntimeError("rotted selector")+ monkeypatch.setattr(CodeBlock, "execute_user_function_with_timeout", AsyncMock(side_effect=raised))+ monkeypatch.setattr(app.AGENT_FUNCTION, "execute_code_block_override", AsyncMock(return_value=None))+ FakeRecorder.reset(last_recorded_exception=raised)+ monkeypatch.setattr("skyvern.forge.sdk.workflow.models.block.CodeBlockActionRecording", FakeRecorder)+ monkeypatch.setattr(block, "_attempt_self_heal", AsyncMock(return_value=None))+ monkeypatch.setattr(CodeBlock, "record_output_parameter_value", AsyncMock(return_value=None))++ result = await block.execute(+ workflow_run_id="wr_test",+ workflow_run_block_id="wrb_test",+ organization_id="o_test",+ browser_session_id="pbs_test",+ )++ block_statuses = _statuses_for_block(state, "wrb_test")+ assert result.success is False+ assert block_statuses.count(BlockStatus.failed) == 1+ assert len(FakeRecorder.instances) == 1+ assert FakeRecorder.instances[0].finalized_success is False+++async def test_secure_heal_declined_writes_failed_once(monkeypatch: pytest.MonkeyPatch) -> None:+ monkeypatch.setattr("skyvern.config.settings.ENABLE_CODE_BLOCK_SELF_HEALING", False, raising=False)+ state = _install_db_fakes(monkeypatch, final_status=TaskStatus.completed)+ block = _make_code_block(steps=[CodeBlockStep(description="download", line_start=1, line_end=1)])+ context = _make_context(enable_self_healing=True)+ fake_page = MagicMock()+ fake_browser_state = SimpleNamespace(get_working_page=AsyncMock(return_value=fake_page))+ _patch_execute_chokepoint_environment(+ monkeypatch,+ context=context,+ fake_browser_state=fake_browser_state,+ use_codeblock_runner=True,+ )+ monkeypatch.setattr(+ app.AGENT_FUNCTION,+ "execute_code_block_override",+ AsyncMock(+ return_value=CodeBlockEngineResult(+ block_result=None,+ failure=CodeBlockEngineFailure(+ error_code="user_code_error",+ safe_message=None,+ failure_reason="CodeBlock failed while running user code.",+ exception_class="playwright._impl._errors.TimeoutError",+ failing_line=2,+ healability_hint=True,+ ),+ )+ ),+ )+ FakeRecorder.reset(last_recorded_exception=None)+ monkeypatch.setattr("skyvern.forge.sdk.workflow.models.block.CodeBlockActionRecording", FakeRecorder)+ monkeypatch.setattr(block, "_attempt_self_heal", AsyncMock(return_value=None))+ monkeypatch.setattr(CodeBlock, "record_output_parameter_value", AsyncMock(return_value=None))++ result = await block.execute(+ workflow_run_id="wr_test",+ workflow_run_block_id="wrb_test",+ organization_id="o_test",+ browser_session_id="pbs_test",+ )++ block_statuses = _statuses_for_block(state, "wrb_test")+ assert result.success is False+ assert block_statuses.count(BlockStatus.failed) == 1+ assert len(FakeRecorder.instances) == 1+ assert FakeRecorder.instances[0].finalized_success is False+++async def test_secure_unhealable_failure_without_result_builds_failed_once(monkeypatch: pytest.MonkeyPatch) -> None:+ monkeypatch.setattr("skyvern.config.settings.ENABLE_CODE_BLOCK_SELF_HEALING", False, raising=False)+ state = _install_db_fakes(monkeypatch, final_status=TaskStatus.completed)+ block = _make_code_block()+ context = _make_context(enable_self_healing=False)+ fake_page = MagicMock()+ fake_browser_state = SimpleNamespace(get_working_page=AsyncMock(return_value=fake_page))+ _patch_execute_chokepoint_environment(+ monkeypatch,+ context=context,+ fake_browser_state=fake_browser_state,+ use_codeblock_runner=True,+ )+ monkeypatch.setattr(+ app.AGENT_FUNCTION,+ "execute_code_block_override",+ AsyncMock(+ return_value=CodeBlockEngineResult(+ block_result=None,+ failure=CodeBlockEngineFailure(+ error_code="timeout",+ safe_message="CodeBlock execution timed out.",+ failure_reason="CodeBlock execution timed out.",+ exception_class=None,+ failing_line=None,+ healability_hint=False,+ ),+ )+ ),+ )+ FakeRecorder.reset(last_recorded_exception=None)+ monkeypatch.setattr("skyvern.forge.sdk.workflow.models.block.CodeBlockActionRecording", FakeRecorder)+ heal_mock = AsyncMock(return_value=None)+ monkeypatch.setattr(block, "_attempt_self_heal", heal_mock)+ monkeypatch.setattr(CodeBlock, "record_output_parameter_value", AsyncMock(return_value=None))++ result = await block.execute(+ workflow_run_id="wr_test",+ workflow_run_block_id="wrb_test",+ organization_id="o_test",+ browser_session_id="pbs_test",+ )++ block_statuses = _statuses_for_block(state, "wrb_test")+ assert result.success is False+ assert heal_mock.await_count == 0+ assert block_statuses.count(BlockStatus.failed) == 1+ assert len(FakeRecorder.instances) == 1+ assert FakeRecorder.instances[0].finalized_success is False+++async def test_secure_runner_missing_block_result_returns_generic_failure(monkeypatch: pytest.MonkeyPatch) -> None:+ monkeypatch.setattr("skyvern.config.settings.ENABLE_CODE_BLOCK_SELF_HEALING", False, raising=False)+ state = _install_db_fakes(monkeypatch, final_status=TaskStatus.completed)+ block = _make_code_block()+ context = _make_context(enable_self_healing=True)+ fake_page = MagicMock()+ fake_browser_state = SimpleNamespace(get_working_page=AsyncMock(return_value=fake_page))+ _patch_execute_chokepoint_environment(+ monkeypatch,+ context=context,+ fake_browser_state=fake_browser_state,+ use_codeblock_runner=True,+ )+ monkeypatch.setattr(+ app.AGENT_FUNCTION,+ "execute_code_block_override",+ AsyncMock(return_value=CodeBlockEngineResult(block_result=None, failure=None)),+ )+ FakeRecorder.reset(last_recorded_exception=None)+ monkeypatch.setattr("skyvern.forge.sdk.workflow.models.block.CodeBlockActionRecording", FakeRecorder)+ monkeypatch.setattr(CodeBlock, "record_output_parameter_value", AsyncMock(return_value=None))++ result = await block.execute(+ workflow_run_id="wr_test",+ workflow_run_block_id="wrb_test",+ organization_id="o_test",+ browser_session_id="pbs_test",+ )++ block_statuses = _statuses_for_block(state, "wrb_test")+ assert result.success is False+ assert result.failure_reason == "Secure code block runner returned no result"+ assert block_statuses.count(BlockStatus.failed) == 1+ assert len(FakeRecorder.instances) == 1+ assert FakeRecorder.instances[0].finalized_success is False+++async def test_secure_infra_failure_without_metadata_finalizes_failed_and_records_no_output(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ monkeypatch.setattr("skyvern.config.settings.ENABLE_CODE_BLOCK_SELF_HEALING", False, raising=False)+ state = _install_db_fakes(monkeypatch, final_status=TaskStatus.completed)+ block = _make_code_block()+ context = _make_context(enable_self_healing=True)+ fake_page = MagicMock()+ fake_browser_state = SimpleNamespace(get_working_page=AsyncMock(return_value=fake_page))+ _patch_execute_chokepoint_environment(+ monkeypatch,+ context=context,+ fake_browser_state=fake_browser_state,+ use_codeblock_runner=True,+ )+ failed_result = await block.build_block_result(+ success=False,+ failure_reason="infra failure",+ output_parameter_value={"raw": "not-recorded"},+ status=BlockStatus.failed,+ workflow_run_block_id="wrb_test",+ organization_id="o_test",+ )+ monkeypatch.setattr(+ app.AGENT_FUNCTION,+ "execute_code_block_override",+ AsyncMock(return_value=CodeBlockEngineResult(block_result=failed_result, failure=None)),+ )+ FakeRecorder.reset(last_recorded_exception=None)+ monkeypatch.setattr("skyvern.forge.sdk.workflow.models.block.CodeBlockActionRecording", FakeRecorder)+ heal_mock = AsyncMock(return_value=None)+ monkeypatch.setattr(block, "_attempt_self_heal", heal_mock)+ record_output_mock = AsyncMock(return_value=None)+ monkeypatch.setattr(CodeBlock, "record_output_parameter_value", record_output_mock)++ result = await block.execute(+ workflow_run_id="wr_test",+ workflow_run_block_id="wrb_test",+ organization_id="o_test",+ browser_session_id="pbs_test",+ )++ block_statuses = _statuses_for_block(state, "wrb_test")+ assert result.success is False+ assert len(FakeRecorder.instances) == 1+ assert FakeRecorder.instances[0].finalized_success is False+ assert record_output_mock.await_count == 0+ assert BlockStatus.completed not in block_statuses+ heal_mock.assert_not_awaited()diff --git a/tests/unit/workflow/test_code_block_recorder.py b/tests/unit/workflow/test_code_block_recorder.pyindex 4e64b32d1..373ee6fe2 100644--- a/tests/unit/workflow/test_code_block_recorder.py+++ b/tests/unit/workflow/test_code_block_recorder.py@@ -400,8 +400,10 @@ def _patch_execute_environment(async def validate_code_block(*args, **kwargs): # noqa: ANN002, ANN003, ANN201return None+ browser_state = FakeBrowserState()+async def get_browser_state(*args, **kwargs): # noqa: ANN002, ANN003, ANN201- return FakeBrowserState()+ return browser_stateasync def record_output(*args, **kwargs): # noqa: ANN002, ANN003, ANN201return None@@ -419,6 +421,7 @@ def _patch_execute_environment("skyvern.forge.sdk.workflow.models.block.app.AGENT_FUNCTION.validate_code_block", validate_code_block)monkeypatch.setattr(CodeBlock, "get_or_create_browser_state", get_browser_state)+ monkeypatch.setattr(app.BROWSER_MANAGER, "get_for_workflow_run", lambda *args, **kwargs: browser_state)monkeypatch.setattr(CodeBlock, "get_workflow_run_context", lambda *args: context)monkeypatch.setattr(CodeBlock, "record_output_parameter_value", record_output)monkeypatch.setattr(app.DATABASE.observer, "get_workflow_run_block", mocks["get_workflow_run_block"])@@ -587,7 +590,11 @@ async def test_self_heal_success_finalizes_seat_completed(monkeypatch: pytest.Mocontext = FakeWorkflowRunContext()mocks = _patch_execute_environment(monkeypatch, page, context)# Stub the heal to a success result; this tests execute()'s seat-finalization wiring, not the heal itself.- monkeypatch.setattr(CodeBlock, "_attempt_self_heal", AsyncMock(return_value=SimpleNamespace(success=True)))+ monkeypatch.setattr(+ CodeBlock,+ "_attempt_self_heal",+ AsyncMock(return_value=SimpleNamespace(success=True, output_parameter_value=None)),+ )block = _make_code_block("await page.locator('#x').click()", goal="go")result = await block.execute(workflow_run_id="wr_test", workflow_run_block_id="wrb_test", organization_id="o_test")