Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
Classify code-block heal errors by selected engine (RUS-5) (#7707)
LawyZhengdaa2505c19ed
diff --git a/skyvern/forge/sdk/workflow/models/block.py b/skyvern/forge/sdk/workflow/models/block.pyindex d9506c4cb..fe537afe1 100644--- a/skyvern/forge/sdk/workflow/models/block.py+++ b/skyvern/forge/sdk/workflow/models/block.py@@ -8,6 +8,7 @@ import copyimport csvimport hashlibimport html+import inspectimport jsonimport keywordimport os@@ -199,6 +200,7 @@ from skyvern.webeye.utils.page import SkyvernFrameif TYPE_CHECKING:from skyvern.forge.agent_functions import CodeBlockEngineFailure+ from skyvern.webeye.browser_engine import BrowserEngineSelectionLOG = structlog.get_logger()@@ -4135,11 +4137,18 @@ async def wrapper({default_args}):)return False- def _is_healable_page_failure(self, exception: Exception, recording_page: RecordingPage) -> bool:+ def _is_healable_page_failure(+ self,+ exception: Exception,+ recording_page: RecordingPage,+ engine_selection: BrowserEngineSelection | None = None,+ ) -> bool:"""Heal genuine page failures only: a recorded page call raised, or an (unmapped) Playwrightpage error surfaced. A deliberate non-Playwright raise in user logic stays non-healable."""if recording_page.last_recorded_exception() is exception:return True+ if engine_selection is not None:+ return engine_selection.is_engine_error(exception) is Truereturn isinstance(exception, PlaywrightError) # locator/timeout/navigation errors subclass thisasync def _finalize_recovery_block(@@ -4220,10 +4229,21 @@ async def wrapper({default_args}):heal was attempted, or None to fall through to the caller's fail-closed path."""if not await self._self_heal_enabled(workflow_run_context):return None- effective_classification = classification or HealClassification(- healable=self._is_healable_page_failure(exception, recording_page),- skip_reason=None,- )+ effective_classification = classification+ if effective_classification is None:+ engine_selection = (+ browser_state.engine_selection+ if browser_state and inspect.getattr_static(browser_state, "engine_selection", None) is not None+ else None+ )+ effective_classification = HealClassification(+ healable=self._is_healable_page_failure(+ exception,+ recording_page,+ engine_selection,+ ),+ skip_reason=None,+ )if not effective_classification.healable:return Noneif not self.prompt:@@ -5204,7 +5224,16 @@ async def wrapper({default_args}):))await recorder.persist(recorded)- legacy_healable = self._is_healable_page_failure(e, recording_page)+ engine_selection = (+ browser_state.engine_selection+ if browser_state and inspect.getattr_static(browser_state, "engine_selection", None) is not None+ else None+ )+ legacy_healable = self._is_healable_page_failure(+ e,+ recording_page,+ engine_selection,+ )legacy_classification = HealClassification(healable=legacy_healable,skip_reason=None if legacy_healable else HealSkipReason.unclassifiable,diff --git a/tests/unit/test_code_block_heal_engine_classification.py b/tests/unit/test_code_block_heal_engine_classification.pynew file mode 100644index 000000000..f586534d1--- /dev/null+++ b/tests/unit/test_code_block_heal_engine_classification.py@@ -0,0 +1,173 @@+from types import SimpleNamespace+from unittest.mock import AsyncMock, MagicMock++import pytest+from playwright.async_api import Error as PlaywrightError++from skyvern.forge import app+from skyvern.forge.sdk.workflow.models import block as block_module+from skyvern.forge.sdk.workflow.models.block import CodeBlock+from skyvern.schemas.self_heal import HealClassification, HealSkipReason+from skyvern.webeye.browser_engine import BrowserEngineMetadata, BrowserEngineSelection+++class _NativeError(Exception):+ pass+++class _NativeTimeoutError(_NativeError):+ pass+++class _ForeignError(Exception):+ pass+++class _ForeignTimeoutError(_ForeignError):+ pass+++def _selection(+ name: str = "native",+ error_type: type[BaseException] = _NativeError,+ timeout_error_type: type[BaseException] = _NativeTimeoutError,+) -> BrowserEngineSelection:+ return BrowserEngineSelection(+ name=name,+ start_driver=AsyncMock(),+ error_type=error_type,+ timeout_error_type=timeout_error_type,+ metadata=BrowserEngineMetadata(name=name),+ selection_reason="test",+ )+++def _block() -> CodeBlock:+ return CodeBlock.model_construct(+ label="code",+ code="raise Exception('boom')",+ parameters=[],+ output_parameter=MagicMock(),+ )+++def _recording_page(recorded_exception: Exception | None) -> MagicMock:+ page = MagicMock()+ page.last_recorded_exception.return_value = recorded_exception+ return page+++def test_recorded_exception_is_always_healable() -> None:+ block = _block()+ exception = PlaywrightError("foreign but recorded")++ assert block._is_healable_page_failure(exception, _recording_page(exception), _selection()) is True+++def test_none_selection_preserves_stock_playwright_behavior() -> None:+ block = _block()+ recording_page = _recording_page(None)++ assert block._is_healable_page_failure(PlaywrightError("stock"), recording_page) is True+ assert block._is_healable_page_failure(RuntimeError("user code"), recording_page) is False+++def test_selected_engine_native_error_is_healable() -> None:+ assert (+ _block()._is_healable_page_failure(+ _NativeError("native"),+ _recording_page(None),+ _selection(),+ )+ is True+ )++[email protected]("exception", [PlaywrightError("stock"), _ForeignError("foreign")])+def test_selected_engine_foreign_driver_error_is_not_healable(exception: Exception) -> None:+ assert _block()._is_healable_page_failure(exception, _recording_page(None), _selection()) is False+++async def test_live_legacy_path_uses_run_pinned_engine_as_authority(monkeypatch: pytest.MonkeyPatch) -> None:+ exception = PlaywrightError("stock error from a foreign driver")+ selection = _selection()+ page = MagicMock()+ browser_state = SimpleNamespace(+ engine_selection=selection,+ get_working_page=AsyncMock(return_value=page),+ )+ context = SimpleNamespace(+ organization_id="o_test",+ get_value=MagicMock(),+ mask_secrets_in_data=lambda value: value,+ )++ class _Recorder:+ def __init__(self, **kwargs: object) -> None:+ self.recording_page = _recording_page(None)++ async def create_task_and_step(self) -> None:+ return None++ async def link_block(self) -> None:+ return None++ def recorded_actions(self) -> list[object]:+ return []++ def last_recorded_exception(self) -> None:+ return None++ async def persist(self, actions: list[object]) -> None:+ return None++ async def finalize(self, success: bool) -> None:+ return None++ resolve_failure = AsyncMock(return_value=MagicMock())+ monkeypatch.setattr(app.AGENT_FUNCTION, "validate_code_block", AsyncMock(return_value=None))+ monkeypatch.setattr(app.AGENT_FUNCTION, "should_use_codeblock_runner", AsyncMock(return_value=False))+ monkeypatch.setattr(CodeBlock, "get_workflow_run_context", MagicMock(return_value=context))+ monkeypatch.setattr(CodeBlock, "get_or_create_browser_state", AsyncMock(return_value=browser_state))+ monkeypatch.setattr(CodeBlock, "_ensure_run_recording_artifact", AsyncMock(return_value=None))+ monkeypatch.setattr(CodeBlock, "format_potential_template_parameters", MagicMock(return_value=None))+ monkeypatch.setattr(CodeBlock, "generate_async_user_function", MagicMock(return_value=AsyncMock()))+ monkeypatch.setattr(CodeBlock, "execute_user_function_with_timeout", AsyncMock(side_effect=exception))+ monkeypatch.setattr(CodeBlock, "_resolve_failure_with_heal", resolve_failure)+ monkeypatch.setattr(block_module, "CodeBlockActionRecording", _Recorder)++ await _block().execute(+ workflow_run_id="wr_test",+ workflow_run_block_id="wrb_test",+ organization_id="o_test",+ )++ resolve_failure.assert_awaited_once()+ assert resolve_failure.await_args is not None+ classification = resolve_failure.await_args.kwargs["classification"]+ assert classification == HealClassification(healable=False, skip_reason=HealSkipReason.unclassifiable)+++async def test_attempt_self_heal_default_classification_uses_pinned_engine(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ block = _block()+ exception = PlaywrightError("foreign stock error")+ browser_state = SimpleNamespace(engine_selection=_selection())+ monkeypatch.setattr(CodeBlock, "_self_heal_enabled", AsyncMock(return_value=True))++ result = await block._attempt_self_heal(+ exception=exception,+ failing_line=None,+ recording_page=_recording_page(None),+ workflow_run_context=MagicMock(),+ workflow_run_id="wr_test",+ workflow_run_block_id="wrb_test",+ organization_id="o_test",+ browser_session_id=None,+ browser_state=browser_state,+ )++ assert result is None