Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
Run a task's skipped terminal cleanup once when its handler could not fail the row (SKY-15729) (#8552)
pedrohsdb678dda70482f
diff --git a/skyvern/forge/agent.py b/skyvern/forge/agent.pyindex eebc74857..33cadb0a7 100644--- a/skyvern/forge/agent.py+++ b/skyvern/forge/agent.py@@ -2904,6 +2904,11 @@ class ForgeAgent:detailed_output: DetailedAgentStepOutput | None = Nonelist_files_before: list[str] = download_baseline_files.copy() if download_baseline_files is not None else []browser_state: BrowserState | None = None+ # Set by a handler that ended the task but could not fail the row, so it skipped cleanup;+ # the outer `finally` then runs that cleanup once. It is the handler's own signal rather+ # than an inference from the task row, which cannot distinguish it from the unconditional+ # handlers that clean up on an already-final row too.+ cleanup_skipped_by_handler = Falsetry:if download_baseline_files is None and task.workflow_run_id:list_files_before = list_files_in_directory(@@ -3256,6 +3261,7 @@ class ForgeAgent:)else:LOG.warning("Task isn't marked as failed, after step termination. NOT clean up the task")+ cleanup_skipped_by_handler = Truereturn step, detailed_output, Noneexcept FailedToSendWebhook:LOG.exception(@@ -3288,6 +3294,7 @@ class ForgeAgent:)else:LOG.warning("Task isn't marked as failed, after navigation failure. NOT clean up the task")+ cleanup_skipped_by_handler = Truereturn step, detailed_output, next_stepexcept TaskAlreadyCanceled:LOG.info(@@ -3451,6 +3458,7 @@ class ForgeAgent:)else:LOG.warning("Task isn't marked as failed, after browser/session failure. NOT clean up the task")+ cleanup_skipped_by_handler = Truereturn step, detailed_output, Noneexcept Exception as e:LOG.exception("Got an unexpected exception in step, marking task as failed")@@ -3470,14 +3478,39 @@ class ForgeAgent:)else:LOG.warning("Task isn't marked as failed, after unexpected exception. NOT clean up the task")+ cleanup_skipped_by_handler = Truereturn step, detailed_output, Nonefinally:- # remove the step_id from the contextcontext = skyvern_context.ensure_context()- context.step_id = None- context.task_id = None- context.navigation_goal = None- context.navigation_payload = None+ # Placed after the handlers so `fail_task` has already read the live browser and before+ # the context is cleared; `contained_effect` keeps an ordinary failure here from turning+ # the handler's return into a raise, while a cancellation still propagates.+ # It recovers the download half only, so its kwargs are pinned to that scope rather than+ # inherited from the handler that skipped this cleanup.+ try:+ if cleanup_skipped_by_handler and not context.cleanup_downloads_recorded(task.task_id):+ with contained_effect("skipped task cleanup recovery", task_id=task.task_id):+ LOG.warning(+ "Task cleanup was skipped by its exception handler; running it once here",+ task_id=task.task_id,+ )+ await self.clean_up_task(+ task=task,+ last_step=step,+ api_key=api_key,+ need_call_webhook=False,+ need_final_screenshot=False,+ close_browser_on_completion=close_browser_on_completion,+ browser_session_id=browser_session_id,+ download_suffix=None,+ )+ finally:+ # A cancellation during the recovery's await propagates by design, so the reset runs+ # in its own `finally` to keep the context from outliving the task it describes.+ context.step_id = None+ context.task_id = None+ context.navigation_goal = None+ context.navigation_payload = None@staticmethoddef _latest_download_failure_status(steps: list[Step]) -> int | None:@@ -7422,6 +7455,14 @@ class ForgeAgent:workflow_run_id=task.workflow_run_id,)+ # Reaching this line means the download half has had its run: the save is wrapped in its own+ # try that swallows TimeoutError and Exception. Marked here rather than on entry because the+ # task refresh above raises TaskNotFound before the download half ever starts, and marking on+ # entry would let that raise suppress the recovery in execute_step and lose the download.+ _recording_context = skyvern_context.current()+ if _recording_context is not None:+ _recording_context.mark_cleanup_downloads_recorded(task.task_id)+# Last point common to every terminal path, and late enough that the screenshot and# download work above has already given the billed speculative call time to land.await drain_speculative_persist_tasks(skyvern_context.current())diff --git a/skyvern/forge/sdk/core/skyvern_context.py b/skyvern/forge/sdk/core/skyvern_context.pyindex 29a33ad85..5ceeeedb6 100644--- a/skyvern/forge/sdk/core/skyvern_context.py+++ b/skyvern/forge/sdk/core/skyvern_context.py@@ -479,6 +479,12 @@ class SkyvernContext:# task/run/persistent-session scope.download_popup_claims: dict[str, list[Page]] = field(default_factory=dict)+ # Tasks whose terminal cleanup has already run its download half -- settle, save, and artifact+ # outcome recording -- so a recovery path can tell "never recorded" from "recorded" without+ # inferring it from the task row. Keyed per task rather than per stack frame because cleanup is+ # also entered from callees (``_execute_task_v3``) whose progress a caller-local flag cannot see.+ cleanup_downloads_recorded_task_ids: set[str] = field(default_factory=builtins.set)+# parallel verification optimization# stores pre-scraped data for next step to avoid re-scrapingnext_step_pre_scraped_data: dict[str, Any] | None = None@@ -730,6 +736,12 @@ class SkyvernContext:def clear_download_popup_claims(self, task_id: str) -> None:self.download_popup_claims.pop(task_id, None)+ def mark_cleanup_downloads_recorded(self, task_id: str) -> None:+ self.cleanup_downloads_recorded_task_ids.add(task_id)++ def cleanup_downloads_recorded(self, task_id: str) -> bool:+ return task_id in self.cleanup_downloads_recorded_task_ids+def flush_feature_flags(self) -> None:if not self.feature_flag_entries:returndiff --git a/tests/unit/test_execute_step_exception_handling.py b/tests/unit/test_execute_step_exception_handling.pyindex b726a0424..2dcccb3a1 100644--- a/tests/unit/test_execute_step_exception_handling.py+++ b/tests/unit/test_execute_step_exception_handling.py@@ -24,6 +24,7 @@ passes — only a change in behavior fails.from __future__ import annotations+import asynciofrom dataclasses import dataclassfrom datetime import UTC, datetimefrom types import SimpleNamespace@@ -55,7 +56,8 @@ from skyvern.exceptions import (from skyvern.forge.agent import ForgeAgentfrom skyvern.forge.sdk.core import skyvern_contextfrom skyvern.forge.sdk.core.skyvern_context import SkyvernContext-from skyvern.forge.sdk.models import StepStatus+from skyvern.forge.sdk.models import Step, StepStatus+from skyvern.forge.sdk.schemas.tasks import Task, TaskStatusfrom skyvern.webeye.browser_errors import BrowserTargetClosedErrorfrom tests.unit.helpers import make_organization, make_step, make_task@@ -330,15 +332,23 @@ async def test_cleanup_handler_pins_webhook_screenshot_and_fail_task(async def test_cleanup_gating_when_fail_task_reports_not_failed(monkeypatch: pytest.MonkeyPatch, case: CleanupHandlerCase) -> None:- """Pin whether clean_up_task is skipped when fail_task reports the task was NOT failed.-- Only three handlers (step-termination, failed-to-navigate, generic Exception) gate- cleanup on that result; the rest clean up unconditionally.+ """Cleanup runs exactly once on every handler, whatever fail_task reports: the gated handlers+ skip their own ``clean_up_task`` and the outer ``finally`` runs it, while the unconditional+ handlers reach it directly and keep their own webhook/screenshot decisions. The recovery must+ not add a second call on top of either."""outcome = await _drive_execute_step(monkeypatch, case.exc_factory(), fail_task_result=False)assert outcome.raised is None- assert outcome.cleanup_called is not case.cleanup_gated_on_fail_task+ assert outcome.clean_up_task.await_count == 1+ if case.cleanup_gated_on_fail_task:+ # The recovery's call, not the handler's. It recovers the download half only: the row was+ # finalized by someone else who webhooks itself, and the browser may not be usable.+ assert outcome.effective_webhook is False+ assert outcome.effective_final_screenshot is False+ else:+ assert outcome.effective_webhook is case.effective_webhook+ assert outcome.effective_final_screenshot is case.effective_final_screenshot@pytest.mark.asyncio@@ -542,3 +552,271 @@ async def test_execute_step_releases_scraped_page_before_recursion(monkeypatch:assert fail_fast_spy.await_args.kwargs["scraped_page"] is scraped_pageif path == "execute_all_steps":assert handle_completed_step.await_args.kwargs["scraped_page"] is scraped_page+++# These drive the REAL ``fail_task`` and ``clean_up_task``, because mocking exactly those two hides+# that most handlers clean up unconditionally. Only the browser, storage and webhook edges are faked,+# and ``fail_task`` genuinely returns False because the row it is handed is already final.+++@dataclass+class RealCleanupOutcome:+ raised: BaseException | None+ saves: AsyncMock+ cleanup_entries: list[dict[str, Any]]+ downloads_recorded: bool+ context_cleared: bool+++async def _drive_execute_step_real_cleanup(+ monkeypatch: pytest.MonkeyPatch,+ *,+ in_try: Callable[[ForgeAgent, Task, Step, dict[str, bool]], Any],+ task_block: Any = None,+ cleanup_raises: BaseException | None = None,+) -> RealCleanupOutcome:+ """Run ``execute_step`` against an already-final task row with real fail_task/clean_up_task.++ ``in_try`` replaces the first in-``try`` await, standing in for whatever the try body did before+ raising -- including entering ``clean_up_task`` itself, which is the shape ``_execute_task_v3``+ has: its last statement before returning is a ``clean_up_task`` call. Its fourth argument is a+ knob dict; setting ``knobs["db_down"]`` makes the task-refresh read fail.+ """+ agent = ForgeAgent()+ now = datetime.now(UTC)+ organization = make_organization(now)+ task = make_task(now, organization)+ step = make_step(now, task, step_id="step-0", status=StepStatus.running, order=0, output=None)+ # The already-final row: Task.validate_update refuses completed -> failed with+ # InvalidTaskStatusTransition, which is what makes the real fail_task return False.+ final_task = make_task(now, organization, status=TaskStatus.completed, extracted_information="done")+ knobs: dict[str, bool] = {"db_down": False}++ async def get_task(*_args: Any, **_kwargs: Any) -> Task:+ if knobs["db_down"]:+ raise RuntimeError("database unavailable")+ return final_task++ monkeypatch.setattr("skyvern.forge.agent.app.DATABASE.tasks.get_task", get_task)+ monkeypatch.setattr("skyvern.forge.agent.app.DATABASE.tasks.get_task_steps", AsyncMock(return_value=[]))+ monkeypatch.setattr("skyvern.forge.agent.app.DATABASE.tasks.update_task", AsyncMock(return_value=final_task))+ monkeypatch.setattr("skyvern.forge.agent.app.DATABASE.tasks.update_step", AsyncMock(return_value=step))+ monkeypatch.setattr("skyvern.forge.agent.save_step_logs", AsyncMock(return_value=None))+ monkeypatch.setattr(+ "skyvern.forge.agent.app.WORKFLOW_CONTEXT_MANAGER.artifact_redaction_enabled", MagicMock(return_value=False)+ )+ monkeypatch.setattr(+ "skyvern.forge.agent.app.WORKFLOW_CONTEXT_MANAGER.runtime_secret_values_for_artifacts",+ MagicMock(return_value=[]),+ )+ # The browser, storage and webhook edges clean_up_task reaches out to, all downstream of the+ # download save this suite asserts on.+ monkeypatch.setattr("skyvern.forge.agent.app.BROWSER_MANAGER.get_for_task", MagicMock(return_value=None))+ monkeypatch.setattr("skyvern.forge.agent.analytics.capture", MagicMock(return_value=None))+ monkeypatch.setattr("skyvern.forge.agent.drain_speculative_persist_tasks", AsyncMock(return_value=None))+ monkeypatch.setattr(+ "skyvern.forge.agent.uploaded_file_service.delete_files_attached_to_run", AsyncMock(return_value=None)+ )+ monkeypatch.setattr("skyvern.forge.agent.app.ARTIFACT_MANAGER.wait_for_upload_aiotasks", AsyncMock())+ agent.async_operation_pool.remove_task = AsyncMock() # type: ignore[method-assign]+ agent.cleanup_browser_and_create_artifacts = AsyncMock() # type: ignore[method-assign]+ agent.execute_task_webhook = AsyncMock() # type: ignore[method-assign]+ # The observation point: the download save clean_up_task performs, which is the work the ticket+ # exists to stop losing.+ saves = AsyncMock(return_value=None)+ monkeypatch.setattr("skyvern.forge.agent.app.STORAGE.save_downloaded_files", saves)++ # A spy, not a mock: it counts entries and calls straight through, so the real clean_up_task --+ # and therefore the real marker -- still runs.+ cleanup_entries: list[dict[str, Any]] = []+ real_cleanup = agent.clean_up_task++ async def counting_cleanup(**kwargs: Any) -> None:+ cleanup_entries.append(kwargs)+ if cleanup_raises is not None:+ raise cleanup_raises+ return await real_cleanup(**kwargs)++ agent.clean_up_task = counting_cleanup # type: ignore[method-assign]++ async def first_in_try_await(*_args: Any, **_kwargs: Any) -> None:+ await in_try(agent, task, step, knobs)++ monkeypatch.setattr("skyvern.forge.agent.app.AGENT_FUNCTION.validate_step_execution", first_in_try_await)++ context = SkyvernContext(+ organization_id=organization.organization_id,+ task_id=task.task_id,+ step_id=None,+ tz_info=ZoneInfo("UTC"),+ )+ skyvern_context.set(context)+ raised: BaseException | None = None+ try:+ await agent.execute_step(+ organization=organization,+ task=task,+ step=step,+ api_key="api-key",+ task_block=task_block,+ download_baseline_files=[],+ )+ except BaseException as caught: # noqa: BLE001 - propagation is part of what is characterized+ raised = caught+ finally:+ downloads_recorded = context.cleanup_downloads_recorded(task.task_id)+ context_cleared = (+ context.step_id is None+ and context.task_id is None+ and context.navigation_goal is None+ and context.navigation_payload is None+ )+ skyvern_context.reset()++ return RealCleanupOutcome(+ raised=raised,+ saves=saves,+ cleanup_entries=cleanup_entries,+ downloads_recorded=downloads_recorded,+ context_cleared=context_cleared,+ )+++async def test_handler_that_cannot_fail_the_task_still_records_its_downloads(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ """The ticket's case: the row is already final, so the real fail_task returns False and the+ gated handler skips cleanup. The download half must still run, exactly once.++ RED against origin/main, where nothing recovers the skipped cleanup and the save never happens.+ """++ async def raise_against_a_final_row(*_args: Any) -> None:+ raise RuntimeError("failed after the task row was finalized")++ outcome = await _drive_execute_step_real_cleanup(monkeypatch, in_try=raise_against_a_final_row)++ assert outcome.raised is None+ assert len(outcome.cleanup_entries) == 1+ assert outcome.saves.await_count == 1+ assert outcome.downloads_recorded is True+ # The recovery takes the download half and declines the rest: the external finalizer that made+ # this row terminal webhooks itself, and finalizing without that caller's baseline would rename+ # files belonging to another block.+ recovery = outcome.cleanup_entries[0]+ assert recovery["need_call_webhook"] is False+ assert recovery["need_final_screenshot"] is False+ assert recovery["download_suffix"] is None+ assert "list_files_before" not in recovery+++async def test_recovery_does_not_rerun_a_cleanup_that_already_recorded(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ """Exactly once, not at least once.++ ``_execute_task_v3``'s last statement before returning is a ``clean_up_task`` call, so a raise+ just after it reaches the handlers with the downloads already recorded. A second pass would not+ duplicate the DOWNLOAD row -- ``save_downloaded_files`` dedupes on ``{base_uri}/{file}`` plus+ checksum, and the recovery does not rename -- but it would spend another settle-and-checksum+ budget on a teardown path and record the download outcome a second time.+ """++ async def clean_up_then_raise(agent: ForgeAgent, task: Task, step: Step, _knobs: dict[str, bool]) -> None:+ await agent.clean_up_task(task=task, last_step=step)+ raise RuntimeError("raised after cleanup had already recorded the downloads")++ outcome = await _drive_execute_step_real_cleanup(monkeypatch, in_try=clean_up_then_raise)++ assert outcome.raised is None+ # One entry, from the try body: the recovery saw the downloads recorded and stood down.+ assert len(outcome.cleanup_entries) == 1+ assert outcome.saves.await_count == 1+++async def test_cleanup_that_died_before_the_download_half_does_not_claim_it_recorded(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ """A cleanup that dies before the download half must leave the recovery free to run.++ ``clean_up_task``'s task refresh converts any db failure into ``TaskNotFound`` and raises it+ long before the download half starts. Recording the marker on entry would let that raise+ suppress the recovery and lose the download outright, so the marker records the download half+ having run rather than the function having been entered.++ Scope: this drives the cleanup that dies from inside the ``try`` body, which is the shape+ ``_execute_task_v3`` has. A ``clean_up_task`` that dies the same way from inside a *handler*+ propagates out of ``execute_step`` instead of returning, and is not recovered here -- a+ pre-existing gap shared by all sixteen handlers, called out in the PR rather than fixed.+ """++ async def clean_up_with_a_dead_db(agent: ForgeAgent, task: Task, step: Step, knobs: dict[str, bool]) -> None:+ knobs["db_down"] = True+ try:+ await agent.clean_up_task(task=task, last_step=step)+ finally:+ knobs["db_down"] = False++ outcome = await _drive_execute_step_real_cleanup(monkeypatch, in_try=clean_up_with_a_dead_db)++ assert outcome.raised is None+ # Two entries: the one that died at the refresh, and the recovery that actually recorded.+ assert len(outcome.cleanup_entries) == 2+ assert outcome.saves.await_count == 1+ assert outcome.downloads_recorded is True+++async def test_recovery_does_not_rename_a_block_download_to_its_suffix(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ """The recovery must not carry the block's ``download_suffix`` into cleanup's finalize.++ Finalizing renames every file the baseline did not already list. The baseline that makes that+ safe is built by the caller that ran the loop -- ``_execute_task_v3`` augments one with the+ files its own ``file_upload`` tool staged, and never receives this frame's. Finalizing against+ this frame's raw pre-step listing would rename staged inputs, and any earlier block's downloads,+ to this block's suffix. Leaving the download under the site's own name is the lesser outcome.++ Also the only test here that drives a gated handler other than the generic ``except Exception``.+ """+ task_block = MagicMock()+ task_block.download_suffix = "block-suffix"+ task_block.complete_on_download = False++ async def fail_to_navigate(*_args: Any) -> None:+ raise FailedToNavigateToUrl("https://example.com", "boom")++ outcome = await _drive_execute_step_real_cleanup(monkeypatch, in_try=fail_to_navigate, task_block=task_block)++ assert outcome.raised is None+ assert len(outcome.cleanup_entries) == 1+ assert outcome.saves.await_count == 1+ assert outcome.cleanup_entries[0]["download_suffix"] is None+++async def test_cancellation_inside_the_recovery_still_clears_the_context(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ """A cancellation during the recovery must propagate and still leave the context cleared.++ The recovery added an await to a ``finally`` that previously had none, so a ``CancelledError``+ -- a ``BaseException``, which ``contained_effect`` deliberately does not contain -- would+ otherwise exit before the context reset and leave the run describing a task that is over.+ """++ async def raise_against_a_final_row(*_args: Any) -> None:+ raise RuntimeError("failed after the task row was finalized")++ outcome = await _drive_execute_step_real_cleanup(+ monkeypatch,+ in_try=raise_against_a_final_row,+ cleanup_raises=asyncio.CancelledError(),+ )++ assert isinstance(outcome.raised, asyncio.CancelledError)+ assert outcome.context_cleared is True