Skip to content

Automate browser based workflows with AI

fix(SKY-10807): propagate CancelledError so run timeouts actually stop the run (#6677)

Shuchang Zhengc3faa4ca41dd
diff --git a/skyvern/forge/agent.py b/skyvern/forge/agent.py
index 5fe0815f7..a3754864c 100644
--- a/skyvern/forge/agent.py
+++ b/skyvern/forge/agent.py
@@ -2088,18 +2088,25 @@ class ForgeAgent:
completed_step.speculative_original_status = StepStatus.completed
return completed_step, detailed_agent_step_output.get_clean_detailed_output()
except CancelledError:
- LOG.exception(
- "CancelledError in agent_step, marking step as failed",
+ # A cancellation here is a deliberate stop (elapsed-time timeout / user cancel). Persist
+ # the step as failed (shielded, so the write survives the cancel) for observability, then
+ # re-raise so the cancellation actually halts the run. Returning instead would let the step
+ # loop treat it as a retryable failure and keep executing past the timeout.
+ LOG.info(
+ "CancelledError in agent_step, marking step failed and re-raising",
step_order=step.order,
step_retry=step.retry_index,
)
detailed_agent_step_output.step_exception = "CancelledError"
- failed_step = await self.update_step(
- step=step,
- status=StepStatus.failed,
- output=detailed_agent_step_output.to_agent_step_output(),
- )
- return failed_step, detailed_agent_step_output.get_clean_detailed_output()
+ with contextlib.suppress(Exception):
+ await asyncio.shield(
+ self.update_step(
+ step=step,
+ status=StepStatus.failed,
+ output=detailed_agent_step_output.to_agent_step_output(),
+ )
+ )
+ raise
except (
UnsupportedActionType,
UnsupportedTaskType,
diff --git a/skyvern/forge/sdk/api/llm/api_handler_factory.py b/skyvern/forge/sdk/api/llm/api_handler_factory.py
index 7c77b6865..4767560dc 100644
--- a/skyvern/forge/sdk/api/llm/api_handler_factory.py
+++ b/skyvern/forge/sdk/api/llm/api_handler_factory.py
@@ -1381,9 +1381,12 @@ class LLMAPIHandlerFactory:
)
raise LLMProviderErrorRetryableTask(llm_key, cause=e) from e
except CancelledError:
+ # A cancellation here means the run is being stopped (elapsed-time timeout / user
+ # cancel) or a speculative step was intentionally cancelled. Either way it must
+ # propagate so the stop actually halts the run; never convert it to a provider error.
_duration = time.perf_counter() - start_time
+ _llm_span.set_attribute("status", "cancelled")
if is_speculative_step:
- _llm_span.set_attribute("status", "cancelled")
LOG.debug(
"LLM request cancelled (speculative step)",
llm_key=llm_key,
@@ -1391,17 +1394,15 @@ class LLMAPIHandlerFactory:
prompt_name=prompt_name,
duration_seconds=_duration,
)
- raise
else:
- _llm_span.set_attribute("status", "error")
- LOG.error(
- "LLM request got cancelled",
+ LOG.warning(
+ "LLM request cancelled",
llm_key=llm_key,
model=main_model_group,
prompt_name=prompt_name,
duration_seconds=_duration,
)
- raise LLMProviderError(llm_key) from None
+ raise
except litellm.exceptions.RateLimitError as e:
duration_seconds = time.perf_counter() - start_time
_llm_span.set_attribute("status", "rate_limited")
@@ -1939,11 +1940,12 @@ class LLMAPIHandlerFactory:
)
raise SkyvernContextWindowExceededError(model=model_name, prompt_name=prompt_name) from e
except CancelledError:
- # Speculative steps are intentionally cancelled when goal verification completes first,
- # so we log at debug level. Non-speculative cancellations are unexpected errors.
+ # A cancellation here means the run is being stopped (elapsed-time timeout / user
+ # cancel) or a speculative step was intentionally cancelled. Either way it must
+ # propagate so the stop actually halts the run; never convert it to a provider error.
t_llm_cancelled = time.perf_counter()
+ _llm_span.set_attribute("status", "cancelled")
if is_speculative_step:
- _llm_span.set_attribute("status", "cancelled")
LOG.debug(
"LLM request cancelled (speculative step)",
llm_key=llm_key,
@@ -1951,17 +1953,15 @@ class LLMAPIHandlerFactory:
prompt_name=prompt_name,
duration=t_llm_cancelled - t_llm_request,
)
- raise
else:
- _llm_span.set_attribute("status", "error")
- LOG.error(
- "LLM request got cancelled",
+ LOG.warning(
+ "LLM request cancelled",
llm_key=llm_key,
model=model_name,
prompt_name=prompt_name,
duration=t_llm_cancelled - t_llm_request,
)
- raise LLMProviderError(llm_key) from None
+ raise
except litellm.exceptions.RateLimitError as e:
duration_seconds = time.perf_counter() - start_time
_llm_span.set_attribute("status", "rate_limited")
@@ -2513,27 +2513,26 @@ class LLMCaller:
_llm_span.set_attribute("status", "error")
raise LLMProviderErrorRetryableTask(self.llm_key, cause=e) from e
except CancelledError:
- # Speculative steps are intentionally cancelled when goal verification returns completed,
- # so we log at debug level. Non-speculative cancellations are unexpected errors.
+ # A cancellation here means the run is being stopped (elapsed-time timeout / user
+ # cancel) or a speculative step was intentionally cancelled. Either way it must
+ # propagate so the stop actually halts the run; never convert it to a provider error.
t_llm_cancelled = time.perf_counter()
+ _llm_span.set_attribute("status", "cancelled")
if is_speculative_step:
- _llm_span.set_attribute("status", "cancelled")
LOG.debug(
"LLM request cancelled (speculative step)",
llm_key=self.llm_key,
model=self.llm_config.model_name,
duration=t_llm_cancelled - t_llm_request,
)
- raise
else:
- _llm_span.set_attribute("status", "error")
- LOG.error(
- "LLM request got cancelled",
+ LOG.warning(
+ "LLM request cancelled",
llm_key=self.llm_key,
model=self.llm_config.model_name,
duration=t_llm_cancelled - t_llm_request,
)
- raise LLMProviderError(self.llm_key) from None
+ raise
except RateLimitError as e:
_llm_span.set_attribute("status", "rate_limited")
LOG.warning("LLM request rate limited", llm_key=self.llm_key)
diff --git a/tests/unit/test_agent_step_cancellation.py b/tests/unit/test_agent_step_cancellation.py
new file mode 100644
index 000000000..1a39c25e5
--- /dev/null
+++ b/tests/unit/test_agent_step_cancellation.py
@@ -0,0 +1,117 @@
+from asyncio import CancelledError
+from datetime import UTC, datetime
+from unittest.mock import AsyncMock, MagicMock
+from zoneinfo import ZoneInfo
+
+import pytest
+
+from skyvern.forge.agent import ForgeAgent
+from skyvern.forge.sdk.core import skyvern_context
+from skyvern.forge.sdk.core.skyvern_context import SkyvernContext
+from skyvern.forge.sdk.models import Step, StepStatus
+from skyvern.webeye.actions.actions import ClickAction
+from skyvern.webeye.scraper.scraped_page import ScrapedPage
+from tests.unit.helpers import make_browser_state, make_organization, make_step, make_task
+
+
+async def test_agent_step_reraises_cancelled_error(monkeypatch: pytest.MonkeyPatch) -> None:
+ """A CancelledError during a step (elapsed-time timeout / user stop) must propagate out of
+ agent_step so the timeout actually halts the run, instead of being swallowed into a failed-step
+ return that the step loop then retries. The step is still persisted as failed before re-raising."""
+ agent = ForgeAgent()
+ now = datetime.now(UTC)
+ organization = make_organization(now)
+ task = make_task(now, organization, navigation_goal="Reach confirmation page", workflow_run_id="workflow-1")
+ step = make_step(now, task, step_id="step-cancel", status=StepStatus.created, order=0, output=None)
+
+ browser_state, _, page = make_browser_state()
+ browser_state.must_get_working_page = AsyncMock(return_value=page)
+ browser_state.get_working_page = AsyncMock(return_value=page)
+
+ async def _dummy_cleanup(*_args, **_kwargs) -> list[dict]:
+ return []
+
+ scraped_page = ScrapedPage(
+ elements=[],
+ element_tree=[],
+ element_tree_trimmed=[],
+ _browser_state=browser_state,
+ _clean_up_func=_dummy_cleanup,
+ _scrape_exclude=None,
+ )
+ scraped_page.screenshots = [b"image"]
+
+ agent.build_and_record_step_prompt = AsyncMock(return_value=(scraped_page, "prompt", False, "extract-actions"))
+ json_response: dict[str, object] = {"actions": [{"action_type": "CLICK", "element_id": "node-1"}]}
+ agent.handle_potential_OTP_actions = AsyncMock(return_value=(json_response, []))
+
+ click_action = ClickAction(
+ element_id="node-1",
+ organization_id=task.organization_id,
+ workflow_run_id=task.workflow_run_id,
+ task_id=task.task_id,
+ step_id=step.step_id,
+ step_order=step.order,
+ action_order=0,
+ )
+ monkeypatch.setattr("skyvern.forge.agent.parse_actions", lambda *_, **__: [click_action])
+
+ # The run-level cancellation lands while the action is executing.
+ monkeypatch.setattr("skyvern.forge.agent.ActionHandler.handle_action", AsyncMock(side_effect=CancelledError()))
+ agent.record_artifacts_after_action = AsyncMock()
+ agent._is_multi_field_totp_sequence = MagicMock(return_value=False)
+ agent.check_user_goal_complete = AsyncMock()
+
+ llm_handler_mock = AsyncMock(return_value=json_response)
+ monkeypatch.setattr(
+ "skyvern.forge.agent.LLMAPIHandlerFactory.get_override_llm_api_handler",
+ lambda *_args, **_kwargs: llm_handler_mock,
+ )
+ monkeypatch.setattr("skyvern.forge.agent.app.AGENT_FUNCTION.prepare_step_execution", AsyncMock(return_value=None))
+ monkeypatch.setattr("skyvern.forge.agent.app.AGENT_FUNCTION.post_action_execution", AsyncMock())
+ monkeypatch.setattr("skyvern.forge.agent.asyncio.sleep", AsyncMock(return_value=None))
+ monkeypatch.setattr("skyvern.forge.agent.random.uniform", lambda *_args, **_kwargs: 0)
+ monkeypatch.setattr(
+ "skyvern.forge.agent.app.EXPERIMENTATION_PROVIDER.is_feature_enabled_cached",
+ AsyncMock(return_value=False),
+ )
+
+ update_statuses: list[StepStatus | None] = []
+
+ async def fake_update_step(
+ step: Step,
+ status: StepStatus | None = None,
+ output=None,
+ is_last: bool | None = None,
+ retry_index: int | None = None,
+ **_kwargs,
+ ) -> Step:
+ update_statuses.append(status)
+ if status is not None:
+ step.status = status
+ return step
+
+ agent.update_step = AsyncMock(side_effect=fake_update_step)
+
+ context = SkyvernContext(
+ task_id=task.task_id,
+ step_id=None,
+ organization_id=task.organization_id,
+ workflow_run_id=task.workflow_run_id,
+ tz_info=ZoneInfo("UTC"),
+ )
+ skyvern_context.set(context)
+ try:
+ with pytest.raises(CancelledError):
+ await agent.agent_step(
+ task=task,
+ step=step,
+ browser_state=browser_state,
+ organization=organization,
+ )
+ finally:
+ skyvern_context.reset()
+
+ # The cancelled step is still recorded as failed (so it is not left orphaned as `running`).
+ assert StepStatus.failed in update_statuses
diff --git a/tests/unit/test_api_handler_factory.py b/tests/unit/test_api_handler_factory.py
index d3ea60ee8..61c5fda80 100644
--- a/tests/unit/test_api_handler_factory.py
+++ b/tests/unit/test_api_handler_factory.py
@@ -1,5 +1,6 @@
from __future__ import annotations
+from asyncio import CancelledError
from datetime import datetime
from types import SimpleNamespace
from typing import Any
@@ -824,3 +825,42 @@ def test_completion_cost_returns_zero_when_litellm_raises(monkeypatch: pytest.Mo
monkeypatch.setattr(litellm, "completion_cost", _raise)
resp = SimpleNamespace(_hidden_params={"provider_specific_fields": {"traffic_type": "ON_DEMAND_FLEX"}})
assert LLMAPIHandlerFactory._completion_cost(resp) == 0.0
+
+
+async def test_non_speculative_cancelled_error_propagates(monkeypatch: pytest.MonkeyPatch) -> None:
+ """A run-level cancellation (elapsed-time timeout / user stop) landing inside an LLM call must
+ propagate as CancelledError so the timeout actually halts the run. It must NOT be converted into
+ a retryable LLMProviderError, which the step loop would treat as a failure and retry."""
+ context = MagicMock()
+ context.vertex_cache_name = None
+ context.use_prompt_caching = False
+ context.cached_static_prompt = None
+ context.hashed_href_map = {}
+
+ llm_config = LLMConfig(
+ model_name="gpt-4",
+ required_env_vars=[],
+ supports_vision=True,
+ add_assistant_prefix=False,
+ )
+ monkeypatch.setattr(
+ "skyvern.forge.sdk.api.llm.api_handler_factory.LLMConfigRegistry.get_config", lambda _: llm_config
+ )
+ monkeypatch.setattr(
+ "skyvern.forge.sdk.api.llm.api_handler_factory.LLMConfigRegistry.is_router_config", lambda _: False
+ )
+ monkeypatch.setattr("skyvern.forge.sdk.api.llm.api_handler_factory.skyvern_context.current", lambda: context)
+ monkeypatch.setattr(
+ api_handler_factory, "llm_messages_builder", AsyncMock(return_value=[{"role": "user", "content": "test"}])
+ )
+ monkeypatch.setattr(api_handler_factory.litellm, "completion_cost", lambda _: 0.0)
+ monkeypatch.setattr(api_handler_factory.litellm, "acompletion", AsyncMock(side_effect=CancelledError()))
+
+ # No step is passed, so is_speculative_step is False (the non-speculative branch).
+ handler = LLMAPIHandlerFactory.get_llm_api_handler("gpt-4")
+ with pytest.raises(BaseException) as exc_info:
+ await handler(prompt="test prompt", prompt_name=EXTRACT_ACTION_PROMPT_NAME)
+ assert isinstance(exc_info.value, CancelledError), (
+ f"expected CancelledError to propagate, got {type(exc_info.value).__name__}"
+ )