Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
Log where a step is waiting when it runs past 20 minutes (SKY-17603) (#8791)
Shuchang Zheng0b02150bb019
diff --git a/skyvern/forge/agent.py b/skyvern/forge/agent.pyindex 0c1694751..326b2ccfb 100644--- a/skyvern/forge/agent.py+++ b/skyvern/forge/agent.py@@ -247,6 +247,7 @@ from skyvern.utils.secret_redaction import (redact_har_bytes,redact_secrets_from_text,)+from skyvern.utils.stall_watch import log_if_stalledfrom skyvern.utils.token_counter import count_tokensfrom skyvern.utils.url_validators import strip_query_paramsfrom skyvern.webeye.actions.action_types import ActionType@@ -331,6 +332,11 @@ _CLAIMED_DOWNLOAD_GRACE_POLL_SECONDS = 1.0_V3_BLANK_PAGE_SETTLE_POLLS = 8_V3_BLANK_PAGE_SETTLE_POLL_SECONDS = 0.25+# Healthy steps finish well inside 20 minutes, so a step still running after that is logged with where it is+# waiting, then again every 30 minutes.+STEP_STALL_LOG_FIRST_AFTER_SECONDS = 20 * 60+STEP_STALL_LOG_REPEAT_EVERY_SECONDS = 30 * 60+EXTRACT_ACTION_TEMPLATE = "extract-action"DECISIVE_CRITERION_VALIDATE_TEMPLATE = "decisive-criterion-validate"@@ -3902,19 +3908,27 @@ class ForgeAgent:if engine in [RunEngine.anthropic_cua, RunEngine.ui_tars, RunEngine.yutori_navigator] and llm_caller:LLMCallerManager.set_llm_caller(task.task_id, llm_caller)- step, detailed_output = await self.agent_step(- task,- step,- browser_state,- organization=organization,- task_block=task_block,- complete_verification=complete_verification,- engine=engine,- cua_response=cua_response,- llm_caller=llm_caller,- attempt_started_at=attempt_started_at,- list_files_before=list_files_before,- )+ with log_if_stalled(+ "Agent step still running",+ first_after_seconds=STEP_STALL_LOG_FIRST_AFTER_SECONDS,+ repeat_every_seconds=STEP_STALL_LOG_REPEAT_EVERY_SECONDS,+ step_phase="agent_step",+ step_id=step.step_id,+ step_order=step.order,+ ):+ step, detailed_output = await self.agent_step(+ task,+ step,+ browser_state,+ organization=organization,+ task_block=task_block,+ complete_verification=complete_verification,+ engine=engine,+ cua_response=cua_response,+ llm_caller=llm_caller,+ attempt_started_at=attempt_started_at,+ list_files_before=list_files_before,+ )await app.AGENT_FUNCTION.post_step_execution(task, step)task = await self.update_task_errors_from_detailed_output(task, detailed_output) # type: ignore# Shadow-only loop-stall observability; never raises, never terminates (see shadow.py).@@ -4001,21 +4015,29 @@ class ForgeAgent:return step, detailed_output, Noneelif step.status == StepStatus.completed:# TODO (kerem): keep the task object uptodate at all times so that clean_up_task can just use it- (- is_task_completed,- maybe_last_step,- maybe_next_step,- ) = await self.handle_completed_step(- organization=organization,- task=task,- step=step,- page=await browser_state.get_working_page(),- task_block=task_block,- browser_state=browser_state,- scraped_page=detailed_output.scraped_page if detailed_output else None,- engine=engine,- complete_verification=complete_verification,- )+ with log_if_stalled(+ "Agent step still running",+ first_after_seconds=STEP_STALL_LOG_FIRST_AFTER_SECONDS,+ repeat_every_seconds=STEP_STALL_LOG_REPEAT_EVERY_SECONDS,+ step_phase="handle_completed_step",+ step_id=step.step_id,+ step_order=step.order,+ ):+ (+ is_task_completed,+ maybe_last_step,+ maybe_next_step,+ ) = await self.handle_completed_step(+ organization=organization,+ task=task,+ step=step,+ page=await browser_state.get_working_page(),+ task_block=task_block,+ browser_state=browser_state,+ scraped_page=detailed_output.scraped_page if detailed_output else None,+ engine=engine,+ complete_verification=complete_verification,+ )# Flush here (after handle_completed_step) so that verification LLM artifacts# from check_user_goal_complete/complete_verify are included in the same# step archive as the rest of the step data.diff --git a/skyvern/utils/stall_watch.py b/skyvern/utils/stall_watch.pynew file mode 100644index 000000000..eb1cd35f1--- /dev/null+++ b/skyvern/utils/stall_watch.py@@ -0,0 +1,93 @@+import asyncio+import contextlib+from collections.abc import Iterator+from pathlib import PurePath+from types import AsyncGeneratorType, CoroutineType, FrameType, GeneratorType+from typing import Any++import structlog++LOG = structlog.get_logger()++# Keep the innermost frames: the call that is actually waiting sits at the bottom of a deep chain.+_MAX_REPORTED_FRAMES = 40+_MAX_WALKED_FRAMES = 500+++def _frame_and_awaited(awaitable: object) -> tuple[FrameType | None, object]:+ if isinstance(awaitable, CoroutineType):+ return awaitable.cr_frame, awaitable.cr_await+ if isinstance(awaitable, GeneratorType):+ return awaitable.gi_frame, awaitable.gi_yieldfrom+ if isinstance(awaitable, AsyncGeneratorType):+ return awaitable.ag_frame, awaitable.ag_await+ return None, None+++def describe_await_chain(task: asyncio.Task[Any]) -> list[str]:+ """Where a suspended task is waiting, outermost first, as ``dir/file.py:line function`` entries."""+ frames: list[str] = []+ awaitable: object = task.get_coro()+ while awaitable is not None and len(frames) < _MAX_WALKED_FRAMES:+ frame, awaited = _frame_and_awaited(awaitable)+ if frame is None:+ # A future or another awaitable with no frame of its own ends the chain.+ frames.append(type(awaitable).__name__)+ break+ path = PurePath(frame.f_code.co_filename)+ frames.append(f"{path.parent.name}/{path.name}:{frame.f_lineno} {frame.f_code.co_name}")+ awaitable = awaited+ return frames[-_MAX_REPORTED_FRAMES:]+++async def _log_while_running(+ target: asyncio.Task[Any],+ message: str,+ first_after_seconds: float,+ repeat_every_seconds: float,+ log_fields: dict[str, Any],+) -> None:+ loop = asyncio.get_running_loop()+ started = loop.time()+ delay = first_after_seconds+ while True:+ await asyncio.sleep(delay)+ if target.done():+ return+ try:+ LOG.warning(+ message,+ elapsed_seconds=round(loop.time() - started),+ await_chain=describe_await_chain(target),+ **log_fields,+ )+ except Exception:+ with contextlib.suppress(Exception):+ LOG.exception("Stall watcher stopped after failing to log", stall_message=message, **log_fields)+ return+ delay = repeat_every_seconds+++def log_if_stalled(+ message: str,+ *,+ first_after_seconds: float,+ repeat_every_seconds: float,+ **log_fields: Any,+) -> Iterator[None]:+ """While the enclosed block runs longer than ``first_after_seconds``, log where the current task is waiting.++ Diagnostics only: the block runs exactly as without it, and the watcher stops when the block exits.+ """+ target = asyncio.current_task()+ if target is None:+ yield+ return+ watcher = asyncio.create_task(+ _log_while_running(target, message, first_after_seconds, repeat_every_seconds, log_fields)+ )+ try:+ yield+ finally:+ watcher.cancel()diff --git a/tests/unit/test_stall_watch.py b/tests/unit/test_stall_watch.pynew file mode 100644index 000000000..5abb2fd4f--- /dev/null+++ b/tests/unit/test_stall_watch.py@@ -0,0 +1,49 @@+import asyncio++import pytest+from structlog.testing import capture_logs++from skyvern.utils.stall_watch import log_if_stalled+++async def _waits_on_the_browser(fut: asyncio.Future[str]) -> str:+ return await fut+++async def _run_step(fut: asyncio.Future[str], first_after_seconds: float) -> str:+ with log_if_stalled(+ "Agent step still running",+ first_after_seconds=first_after_seconds,+ repeat_every_seconds=60,+ step_id="stp_1",+ ):+ return await _waits_on_the_browser(fut)+++async def test_a_stalled_step_logs_the_call_it_is_waiting_in() -> None:+ fut: asyncio.Future[str] = asyncio.get_running_loop().create_future()+ with capture_logs() as logs:+ step = asyncio.create_task(_run_step(fut, first_after_seconds=0.01))+ for _ in range(200):+ await asyncio.sleep(0.01)+ if logs:+ break+ [stall] = [log for log in logs if log["event"] == "Agent step still running"]+ assert stall["step_id"] == "stp_1"+ assert stall["log_level"] == "warning"+ # The innermost awaiting function is the one the step is stuck in.+ assert stall["await_chain"][-2].endswith("_waits_on_the_browser")+ fut.set_result("done")+ assert await step == "done"+++async def test_a_step_that_finishes_in_time_logs_nothing_and_stops_its_watcher() -> None:+ fut: asyncio.Future[str] = asyncio.get_running_loop().create_future()+ fut.set_result("done")+ with capture_logs() as logs:+ assert await _run_step(fut, first_after_seconds=60) == "done"+ await asyncio.sleep(0)+ assert logs == []+ assert asyncio.all_tasks() == {asyncio.current_task()}