Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
fix(SKY-10807): propagate CancelledError so run timeouts actually stop the run (#6677)
Shuchang Zhengc3faa4ca41dd
diff --git a/skyvern/forge/agent.py b/skyvern/forge/agent.pyindex 5fe0815f7..a3754864c 100644--- a/skyvern/forge/agent.py+++ b/skyvern/forge/agent.py@@ -2088,18 +2088,25 @@ class ForgeAgent:completed_step.speculative_original_status = StepStatus.completedreturn completed_step, detailed_agent_step_output.get_clean_detailed_output()except CancelledError:- LOG.exception(- "CancelledError in agent_step, marking step as failed",+ # A cancellation here is a deliberate stop (elapsed-time timeout / user cancel). Persist+ # the step as failed (shielded, so the write survives the cancel) for observability, then+ # re-raise so the cancellation actually halts the run. Returning instead would let the step+ # loop treat it as a retryable failure and keep executing past the timeout.+ LOG.info(+ "CancelledError in agent_step, marking step failed and re-raising",step_order=step.order,step_retry=step.retry_index,)detailed_agent_step_output.step_exception = "CancelledError"- failed_step = await self.update_step(- step=step,- status=StepStatus.failed,- output=detailed_agent_step_output.to_agent_step_output(),- )- return failed_step, detailed_agent_step_output.get_clean_detailed_output()+ with contextlib.suppress(Exception):+ await asyncio.shield(+ self.update_step(+ step=step,+ status=StepStatus.failed,+ output=detailed_agent_step_output.to_agent_step_output(),+ )+ )+ raiseexcept (UnsupportedActionType,UnsupportedTaskType,diff --git a/skyvern/forge/sdk/api/llm/api_handler_factory.py b/skyvern/forge/sdk/api/llm/api_handler_factory.pyindex 7c77b6865..4767560dc 100644--- a/skyvern/forge/sdk/api/llm/api_handler_factory.py+++ b/skyvern/forge/sdk/api/llm/api_handler_factory.py@@ -1381,9 +1381,12 @@ class LLMAPIHandlerFactory:)raise LLMProviderErrorRetryableTask(llm_key, cause=e) from eexcept CancelledError:+ # A cancellation here means the run is being stopped (elapsed-time timeout / user+ # cancel) or a speculative step was intentionally cancelled. Either way it must+ # propagate so the stop actually halts the run; never convert it to a provider error._duration = time.perf_counter() - start_time+ _llm_span.set_attribute("status", "cancelled")if is_speculative_step:- _llm_span.set_attribute("status", "cancelled")LOG.debug("LLM request cancelled (speculative step)",llm_key=llm_key,@@ -1391,17 +1394,15 @@ class LLMAPIHandlerFactory:prompt_name=prompt_name,duration_seconds=_duration,)- raiseelse:- _llm_span.set_attribute("status", "error")- LOG.error(- "LLM request got cancelled",+ LOG.warning(+ "LLM request cancelled",llm_key=llm_key,model=main_model_group,prompt_name=prompt_name,duration_seconds=_duration,)- raise LLMProviderError(llm_key) from None+ raiseexcept litellm.exceptions.RateLimitError as e:duration_seconds = time.perf_counter() - start_time_llm_span.set_attribute("status", "rate_limited")@@ -1939,11 +1940,12 @@ class LLMAPIHandlerFactory:)raise SkyvernContextWindowExceededError(model=model_name, prompt_name=prompt_name) from eexcept CancelledError:- # Speculative steps are intentionally cancelled when goal verification completes first,- # so we log at debug level. Non-speculative cancellations are unexpected errors.+ # A cancellation here means the run is being stopped (elapsed-time timeout / user+ # cancel) or a speculative step was intentionally cancelled. Either way it must+ # propagate so the stop actually halts the run; never convert it to a provider error.t_llm_cancelled = time.perf_counter()+ _llm_span.set_attribute("status", "cancelled")if is_speculative_step:- _llm_span.set_attribute("status", "cancelled")LOG.debug("LLM request cancelled (speculative step)",llm_key=llm_key,@@ -1951,17 +1953,15 @@ class LLMAPIHandlerFactory:prompt_name=prompt_name,duration=t_llm_cancelled - t_llm_request,)- raiseelse:- _llm_span.set_attribute("status", "error")- LOG.error(- "LLM request got cancelled",+ LOG.warning(+ "LLM request cancelled",llm_key=llm_key,model=model_name,prompt_name=prompt_name,duration=t_llm_cancelled - t_llm_request,)- raise LLMProviderError(llm_key) from None+ raiseexcept litellm.exceptions.RateLimitError as e:duration_seconds = time.perf_counter() - start_time_llm_span.set_attribute("status", "rate_limited")@@ -2513,27 +2513,26 @@ class LLMCaller:_llm_span.set_attribute("status", "error")raise LLMProviderErrorRetryableTask(self.llm_key, cause=e) from eexcept CancelledError:- # Speculative steps are intentionally cancelled when goal verification returns completed,- # so we log at debug level. Non-speculative cancellations are unexpected errors.+ # A cancellation here means the run is being stopped (elapsed-time timeout / user+ # cancel) or a speculative step was intentionally cancelled. Either way it must+ # propagate so the stop actually halts the run; never convert it to a provider error.t_llm_cancelled = time.perf_counter()+ _llm_span.set_attribute("status", "cancelled")if is_speculative_step:- _llm_span.set_attribute("status", "cancelled")LOG.debug("LLM request cancelled (speculative step)",llm_key=self.llm_key,model=self.llm_config.model_name,duration=t_llm_cancelled - t_llm_request,)- raiseelse:- _llm_span.set_attribute("status", "error")- LOG.error(- "LLM request got cancelled",+ LOG.warning(+ "LLM request cancelled",llm_key=self.llm_key,model=self.llm_config.model_name,duration=t_llm_cancelled - t_llm_request,)- raise LLMProviderError(self.llm_key) from None+ raiseexcept RateLimitError as e:_llm_span.set_attribute("status", "rate_limited")LOG.warning("LLM request rate limited", llm_key=self.llm_key)diff --git a/tests/unit/test_agent_step_cancellation.py b/tests/unit/test_agent_step_cancellation.pynew file mode 100644index 000000000..1a39c25e5--- /dev/null+++ b/tests/unit/test_agent_step_cancellation.py@@ -0,0 +1,117 @@+from asyncio import CancelledError+from datetime import UTC, datetime+from unittest.mock import AsyncMock, MagicMock+from zoneinfo import ZoneInfo++import pytest++from skyvern.forge.agent import ForgeAgent+from skyvern.forge.sdk.core import skyvern_context+from skyvern.forge.sdk.core.skyvern_context import SkyvernContext+from skyvern.forge.sdk.models import Step, StepStatus+from skyvern.webeye.actions.actions import ClickAction+from skyvern.webeye.scraper.scraped_page import ScrapedPage+from tests.unit.helpers import make_browser_state, make_organization, make_step, make_task+++async def test_agent_step_reraises_cancelled_error(monkeypatch: pytest.MonkeyPatch) -> None:+ """A CancelledError during a step (elapsed-time timeout / user stop) must propagate out of+ agent_step so the timeout actually halts the run, instead of being swallowed into a failed-step+ return that the step loop then retries. The step is still persisted as failed before re-raising."""+ agent = ForgeAgent()+ now = datetime.now(UTC)+ organization = make_organization(now)+ task = make_task(now, organization, navigation_goal="Reach confirmation page", workflow_run_id="workflow-1")+ step = make_step(now, task, step_id="step-cancel", status=StepStatus.created, order=0, output=None)++ browser_state, _, page = make_browser_state()+ browser_state.must_get_working_page = AsyncMock(return_value=page)+ browser_state.get_working_page = AsyncMock(return_value=page)++ async def _dummy_cleanup(*_args, **_kwargs) -> list[dict]:+ return []++ scraped_page = ScrapedPage(+ elements=[],+ element_tree=[],+ element_tree_trimmed=[],+ _browser_state=browser_state,+ _clean_up_func=_dummy_cleanup,+ _scrape_exclude=None,+ )+ scraped_page.screenshots = [b"image"]++ agent.build_and_record_step_prompt = AsyncMock(return_value=(scraped_page, "prompt", False, "extract-actions"))+ json_response: dict[str, object] = {"actions": [{"action_type": "CLICK", "element_id": "node-1"}]}+ agent.handle_potential_OTP_actions = AsyncMock(return_value=(json_response, []))++ click_action = ClickAction(+ element_id="node-1",+ organization_id=task.organization_id,+ workflow_run_id=task.workflow_run_id,+ task_id=task.task_id,+ step_id=step.step_id,+ step_order=step.order,+ action_order=0,+ )+ monkeypatch.setattr("skyvern.forge.agent.parse_actions", lambda *_, **__: [click_action])++ # The run-level cancellation lands while the action is executing.+ monkeypatch.setattr("skyvern.forge.agent.ActionHandler.handle_action", AsyncMock(side_effect=CancelledError()))+ agent.record_artifacts_after_action = AsyncMock()+ agent._is_multi_field_totp_sequence = MagicMock(return_value=False)+ agent.check_user_goal_complete = AsyncMock()++ llm_handler_mock = AsyncMock(return_value=json_response)+ monkeypatch.setattr(+ "skyvern.forge.agent.LLMAPIHandlerFactory.get_override_llm_api_handler",+ lambda *_args, **_kwargs: llm_handler_mock,+ )+ monkeypatch.setattr("skyvern.forge.agent.app.AGENT_FUNCTION.prepare_step_execution", AsyncMock(return_value=None))+ monkeypatch.setattr("skyvern.forge.agent.app.AGENT_FUNCTION.post_action_execution", AsyncMock())+ monkeypatch.setattr("skyvern.forge.agent.asyncio.sleep", AsyncMock(return_value=None))+ monkeypatch.setattr("skyvern.forge.agent.random.uniform", lambda *_args, **_kwargs: 0)+ monkeypatch.setattr(+ "skyvern.forge.agent.app.EXPERIMENTATION_PROVIDER.is_feature_enabled_cached",+ AsyncMock(return_value=False),+ )++ update_statuses: list[StepStatus | None] = []++ async def fake_update_step(+ step: Step,+ status: StepStatus | None = None,+ output=None,+ is_last: bool | None = None,+ retry_index: int | None = None,+ **_kwargs,+ ) -> Step:+ update_statuses.append(status)+ if status is not None:+ step.status = status+ return step++ agent.update_step = AsyncMock(side_effect=fake_update_step)++ context = SkyvernContext(+ task_id=task.task_id,+ step_id=None,+ organization_id=task.organization_id,+ workflow_run_id=task.workflow_run_id,+ tz_info=ZoneInfo("UTC"),+ )+ skyvern_context.set(context)+ try:+ with pytest.raises(CancelledError):+ await agent.agent_step(+ task=task,+ step=step,+ browser_state=browser_state,+ organization=organization,+ )+ finally:+ skyvern_context.reset()++ # The cancelled step is still recorded as failed (so it is not left orphaned as `running`).+ assert StepStatus.failed in update_statusesdiff --git a/tests/unit/test_api_handler_factory.py b/tests/unit/test_api_handler_factory.pyindex d3ea60ee8..61c5fda80 100644--- a/tests/unit/test_api_handler_factory.py+++ b/tests/unit/test_api_handler_factory.py@@ -1,5 +1,6 @@from __future__ import annotations+from asyncio import CancelledErrorfrom datetime import datetimefrom types import SimpleNamespacefrom typing import Any@@ -824,3 +825,42 @@ def test_completion_cost_returns_zero_when_litellm_raises(monkeypatch: pytest.Momonkeypatch.setattr(litellm, "completion_cost", _raise)resp = SimpleNamespace(_hidden_params={"provider_specific_fields": {"traffic_type": "ON_DEMAND_FLEX"}})assert LLMAPIHandlerFactory._completion_cost(resp) == 0.0+++async def test_non_speculative_cancelled_error_propagates(monkeypatch: pytest.MonkeyPatch) -> None:+ """A run-level cancellation (elapsed-time timeout / user stop) landing inside an LLM call must+ propagate as CancelledError so the timeout actually halts the run. It must NOT be converted into+ a retryable LLMProviderError, which the step loop would treat as a failure and retry."""+ context = MagicMock()+ context.vertex_cache_name = None+ context.use_prompt_caching = False+ context.cached_static_prompt = None+ context.hashed_href_map = {}++ llm_config = LLMConfig(+ model_name="gpt-4",+ required_env_vars=[],+ supports_vision=True,+ add_assistant_prefix=False,+ )+ monkeypatch.setattr(+ "skyvern.forge.sdk.api.llm.api_handler_factory.LLMConfigRegistry.get_config", lambda _: llm_config+ )+ monkeypatch.setattr(+ "skyvern.forge.sdk.api.llm.api_handler_factory.LLMConfigRegistry.is_router_config", lambda _: False+ )+ monkeypatch.setattr("skyvern.forge.sdk.api.llm.api_handler_factory.skyvern_context.current", lambda: context)+ monkeypatch.setattr(+ api_handler_factory, "llm_messages_builder", AsyncMock(return_value=[{"role": "user", "content": "test"}])+ )+ monkeypatch.setattr(api_handler_factory.litellm, "completion_cost", lambda _: 0.0)+ monkeypatch.setattr(api_handler_factory.litellm, "acompletion", AsyncMock(side_effect=CancelledError()))++ # No step is passed, so is_speculative_step is False (the non-speculative branch).+ handler = LLMAPIHandlerFactory.get_llm_api_handler("gpt-4")+ with pytest.raises(BaseException) as exc_info:+ await handler(prompt="test prompt", prompt_name=EXTRACT_ACTION_PROMPT_NAME)+ assert isinstance(exc_info.value, CancelledError), (+ f"expected CancelledError to propagate, got {type(exc_info.value).__name__}"+ )