Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
Log expected page-reload and missing-actions step failures at warning (#8855)
Shuchang Zheng1d849bfc7797
diff --git a/skyvern/forge/agent.py b/skyvern/forge/agent.pyindex 0379c27b6..411b16fad 100644--- a/skyvern/forge/agent.py+++ b/skyvern/forge/agent.py@@ -680,6 +680,10 @@ _LLM_STEP_EXCEPTIONS = frozenset(})+# Typed step failures that are an expected outcome: step_exception carries them to the run record and the failure+# summary, so agent_step logs them at warning instead of paging as an unexpected error.+_EXPECTED_STEP_FAILURES = (FailedToReloadPage, LLMResponseMissingActionsError)+def _llm_error_category(reasoning: str) -> list[dict]:return [{"category": FailureCategory.LLM_ERROR.value, "confidence_float": 0.9, "reasoning": reasoning}]@@ -4882,11 +4886,20 @@ class ForgeAgent:raiseexcept Exception as e:- LOG.exception(- "Unexpected exception in agent_step, marking step as failed",- step_order=step.order,- step_retry=step.retry_index,- )+ if isinstance(e, _EXPECTED_STEP_FAILURES):+ LOG.warning(+ "Expected exception in agent_step, marking step as failed",+ step_order=step.order,+ step_retry=step.retry_index,+ error_type=e.__class__.__name__,+ exc_info=True,+ )+ else:+ LOG.exception(+ "Unexpected exception in agent_step, marking step as failed",+ step_order=step.order,+ step_retry=step.retry_index,+ )detailed_agent_step_output.step_exception = e.__class__.__name__failed_step = await self.update_step(step=step,diff --git a/tests/unit/test_agent_step_characterization.py b/tests/unit/test_agent_step_characterization.pyindex a3ab5cd2c..d03ed8d4c 100644--- a/tests/unit/test_agent_step_characterization.py+++ b/tests/unit/test_agent_step_characterization.py@@ -10,9 +10,10 @@ from zoneinfo import ZoneInfoimport pytest-from skyvern.exceptions import NoTOTPVerificationCodeFound, ScrapingFailedBlankPage+from skyvern.exceptions import FailedToReloadPage, NoTOTPVerificationCodeFound, ScrapingFailedBlankPagefrom skyvern.forge import appfrom skyvern.forge.agent import ForgeAgent, StepPromptResult+from skyvern.forge.sdk.api.llm.exceptions import LLMResponseMissingActionsErrorfrom skyvern.forge.sdk.core import skyvern_contextfrom skyvern.forge.sdk.core.skyvern_context import SkyvernContextfrom skyvern.forge.sdk.models import Step, StepStatus@@ -438,6 +439,30 @@ async def test_unexpected_exception_returns_failed_step_instead_of_raising(monkeassert output.step_exception == "RuntimeError"+ ("raised", "expected_log_method"),+ [+ (FailedToReloadPage("https://example.test", "Page.reload: Timeout 60000ms exceeded."), "warning"),+ (LLMResponseMissingActionsError(["page_info"]), "warning"),+ (RuntimeError("browser exploded"), "exception"),+ ],+)+async def test_expected_step_failures_log_at_warning_and_other_exceptions_at_error(+ monkeypatch: pytest.MonkeyPatch, raised: Exception, expected_log_method: str+) -> None:+ log = MagicMock()+ monkeypatch.setattr("skyvern.forge.agent.LOG", log)+ rig = make_agent_step_rig(monkeypatch, action_handler=AsyncMock(side_effect=raised))++ step, output = await rig.run()++ assert step.status == StepStatus.failed+ assert output.step_exception == type(raised).__name__+ leveled = [call[0] for call in log.method_calls if call[0] in ("warning", "error", "exception")]+ assert leveled == [expected_log_method]++@pytest.mark.asyncioasync def test_successful_complete_action_with_extraction_goal_appends_extract(monkeypatch: pytest.MonkeyPatch,