Skip to content

Automate browser based workflows with AI

Log where a step is waiting when it runs past 20 minutes (SKY-17603) (#8791)

Shuchang Zheng0b02150bb019
diff --git a/skyvern/forge/agent.py b/skyvern/forge/agent.py
index 0c1694751..326b2ccfb 100644
--- a/skyvern/forge/agent.py
+++ b/skyvern/forge/agent.py
@@ -247,6 +247,7 @@ from skyvern.utils.secret_redaction import (
redact_har_bytes,
redact_secrets_from_text,
)
+from skyvern.utils.stall_watch import log_if_stalled
from skyvern.utils.token_counter import count_tokens
from skyvern.utils.url_validators import strip_query_params
from skyvern.webeye.actions.action_types import ActionType
@@ -331,6 +332,11 @@ _CLAIMED_DOWNLOAD_GRACE_POLL_SECONDS = 1.0
_V3_BLANK_PAGE_SETTLE_POLLS = 8
_V3_BLANK_PAGE_SETTLE_POLL_SECONDS = 0.25
+# Healthy steps finish well inside 20 minutes, so a step still running after that is logged with where it is
+# waiting, then again every 30 minutes.
+STEP_STALL_LOG_FIRST_AFTER_SECONDS = 20 * 60
+STEP_STALL_LOG_REPEAT_EVERY_SECONDS = 30 * 60
+
EXTRACT_ACTION_TEMPLATE = "extract-action"
DECISIVE_CRITERION_VALIDATE_TEMPLATE = "decisive-criterion-validate"
@@ -3902,19 +3908,27 @@ class ForgeAgent:
if engine in [RunEngine.anthropic_cua, RunEngine.ui_tars, RunEngine.yutori_navigator] and llm_caller:
LLMCallerManager.set_llm_caller(task.task_id, llm_caller)
- step, detailed_output = await self.agent_step(
- task,
- step,
- browser_state,
- organization=organization,
- task_block=task_block,
- complete_verification=complete_verification,
- engine=engine,
- cua_response=cua_response,
- llm_caller=llm_caller,
- attempt_started_at=attempt_started_at,
- list_files_before=list_files_before,
- )
+ with log_if_stalled(
+ "Agent step still running",
+ first_after_seconds=STEP_STALL_LOG_FIRST_AFTER_SECONDS,
+ repeat_every_seconds=STEP_STALL_LOG_REPEAT_EVERY_SECONDS,
+ step_phase="agent_step",
+ step_id=step.step_id,
+ step_order=step.order,
+ ):
+ step, detailed_output = await self.agent_step(
+ task,
+ step,
+ browser_state,
+ organization=organization,
+ task_block=task_block,
+ complete_verification=complete_verification,
+ engine=engine,
+ cua_response=cua_response,
+ llm_caller=llm_caller,
+ attempt_started_at=attempt_started_at,
+ list_files_before=list_files_before,
+ )
await app.AGENT_FUNCTION.post_step_execution(task, step)
task = await self.update_task_errors_from_detailed_output(task, detailed_output) # type: ignore
# Shadow-only loop-stall observability; never raises, never terminates (see shadow.py).
@@ -4001,21 +4015,29 @@ class ForgeAgent:
return step, detailed_output, None
elif step.status == StepStatus.completed:
# TODO (kerem): keep the task object uptodate at all times so that clean_up_task can just use it
- (
- is_task_completed,
- maybe_last_step,
- maybe_next_step,
- ) = await self.handle_completed_step(
- organization=organization,
- task=task,
- step=step,
- page=await browser_state.get_working_page(),
- task_block=task_block,
- browser_state=browser_state,
- scraped_page=detailed_output.scraped_page if detailed_output else None,
- engine=engine,
- complete_verification=complete_verification,
- )
+ with log_if_stalled(
+ "Agent step still running",
+ first_after_seconds=STEP_STALL_LOG_FIRST_AFTER_SECONDS,
+ repeat_every_seconds=STEP_STALL_LOG_REPEAT_EVERY_SECONDS,
+ step_phase="handle_completed_step",
+ step_id=step.step_id,
+ step_order=step.order,
+ ):
+ (
+ is_task_completed,
+ maybe_last_step,
+ maybe_next_step,
+ ) = await self.handle_completed_step(
+ organization=organization,
+ task=task,
+ step=step,
+ page=await browser_state.get_working_page(),
+ task_block=task_block,
+ browser_state=browser_state,
+ scraped_page=detailed_output.scraped_page if detailed_output else None,
+ engine=engine,
+ complete_verification=complete_verification,
+ )
# Flush here (after handle_completed_step) so that verification LLM artifacts
# from check_user_goal_complete/complete_verify are included in the same
# step archive as the rest of the step data.
diff --git a/skyvern/utils/stall_watch.py b/skyvern/utils/stall_watch.py
new file mode 100644
index 000000000..eb1cd35f1
--- /dev/null
+++ b/skyvern/utils/stall_watch.py
@@ -0,0 +1,93 @@
+import asyncio
+import contextlib
+from collections.abc import Iterator
+from pathlib import PurePath
+from types import AsyncGeneratorType, CoroutineType, FrameType, GeneratorType
+from typing import Any
+
+import structlog
+
+LOG = structlog.get_logger()
+
+# Keep the innermost frames: the call that is actually waiting sits at the bottom of a deep chain.
+_MAX_REPORTED_FRAMES = 40
+_MAX_WALKED_FRAMES = 500
+
+
+def _frame_and_awaited(awaitable: object) -> tuple[FrameType | None, object]:
+ if isinstance(awaitable, CoroutineType):
+ return awaitable.cr_frame, awaitable.cr_await
+ if isinstance(awaitable, GeneratorType):
+ return awaitable.gi_frame, awaitable.gi_yieldfrom
+ if isinstance(awaitable, AsyncGeneratorType):
+ return awaitable.ag_frame, awaitable.ag_await
+ return None, None
+
+
+def describe_await_chain(task: asyncio.Task[Any]) -> list[str]:
+ """Where a suspended task is waiting, outermost first, as ``dir/file.py:line function`` entries."""
+ frames: list[str] = []
+ awaitable: object = task.get_coro()
+ while awaitable is not None and len(frames) < _MAX_WALKED_FRAMES:
+ frame, awaited = _frame_and_awaited(awaitable)
+ if frame is None:
+ # A future or another awaitable with no frame of its own ends the chain.
+ frames.append(type(awaitable).__name__)
+ break
+ path = PurePath(frame.f_code.co_filename)
+ frames.append(f"{path.parent.name}/{path.name}:{frame.f_lineno} {frame.f_code.co_name}")
+ awaitable = awaited
+ return frames[-_MAX_REPORTED_FRAMES:]
+
+
+async def _log_while_running(
+ target: asyncio.Task[Any],
+ message: str,
+ first_after_seconds: float,
+ repeat_every_seconds: float,
+ log_fields: dict[str, Any],
+) -> None:
+ loop = asyncio.get_running_loop()
+ started = loop.time()
+ delay = first_after_seconds
+ while True:
+ await asyncio.sleep(delay)
+ if target.done():
+ return
+ try:
+ LOG.warning(
+ message,
+ elapsed_seconds=round(loop.time() - started),
+ await_chain=describe_await_chain(target),
+ **log_fields,
+ )
+ except Exception:
+ with contextlib.suppress(Exception):
+ LOG.exception("Stall watcher stopped after failing to log", stall_message=message, **log_fields)
+ return
+ delay = repeat_every_seconds
+
+
+def log_if_stalled(
+ message: str,
+ *,
+ first_after_seconds: float,
+ repeat_every_seconds: float,
+ **log_fields: Any,
+) -> Iterator[None]:
+ """While the enclosed block runs longer than ``first_after_seconds``, log where the current task is waiting.
+
+ Diagnostics only: the block runs exactly as without it, and the watcher stops when the block exits.
+ """
+ target = asyncio.current_task()
+ if target is None:
+ yield
+ return
+ watcher = asyncio.create_task(
+ _log_while_running(target, message, first_after_seconds, repeat_every_seconds, log_fields)
+ )
+ try:
+ yield
+ finally:
+ watcher.cancel()
diff --git a/tests/unit/test_stall_watch.py b/tests/unit/test_stall_watch.py
new file mode 100644
index 000000000..5abb2fd4f
--- /dev/null
+++ b/tests/unit/test_stall_watch.py
@@ -0,0 +1,49 @@
+import asyncio
+
+import pytest
+from structlog.testing import capture_logs
+
+from skyvern.utils.stall_watch import log_if_stalled
+
+
+async def _waits_on_the_browser(fut: asyncio.Future[str]) -> str:
+ return await fut
+
+
+async def _run_step(fut: asyncio.Future[str], first_after_seconds: float) -> str:
+ with log_if_stalled(
+ "Agent step still running",
+ first_after_seconds=first_after_seconds,
+ repeat_every_seconds=60,
+ step_id="stp_1",
+ ):
+ return await _waits_on_the_browser(fut)
+
+
+async def test_a_stalled_step_logs_the_call_it_is_waiting_in() -> None:
+ fut: asyncio.Future[str] = asyncio.get_running_loop().create_future()
+ with capture_logs() as logs:
+ step = asyncio.create_task(_run_step(fut, first_after_seconds=0.01))
+ for _ in range(200):
+ await asyncio.sleep(0.01)
+ if logs:
+ break
+ [stall] = [log for log in logs if log["event"] == "Agent step still running"]
+ assert stall["step_id"] == "stp_1"
+ assert stall["log_level"] == "warning"
+ # The innermost awaiting function is the one the step is stuck in.
+ assert stall["await_chain"][-2].endswith("_waits_on_the_browser")
+ fut.set_result("done")
+ assert await step == "done"
+
+
+async def test_a_step_that_finishes_in_time_logs_nothing_and_stops_its_watcher() -> None:
+ fut: asyncio.Future[str] = asyncio.get_running_loop().create_future()
+ fut.set_result("done")
+ with capture_logs() as logs:
+ assert await _run_step(fut, first_after_seconds=60) == "done"
+ await asyncio.sleep(0)
+ assert logs == []
+ assert asyncio.all_tasks() == {asyncio.current_task()}