Skip to content

Automate browser based workflows with AI

Record the finally block's minutes instead of the run's body twice (SKY-14606) (#8284)

Shuchang Zhengef39e0892514
diff --git a/skyvern/forge/sdk/workflow/service.py b/skyvern/forge/sdk/workflow/service.py
index 76ed865bb..740a2856a 100644
--- a/skyvern/forge/sdk/workflow/service.py
+++ b/skyvern/forge/sdk/workflow/service.py
@@ -5143,6 +5143,13 @@ class WorkflowService:
# A finally-only failure is a cleanup outcome, not evidence that replaying the
# workflow with another credential can help.
finally_block_set_terminal_outcome = False
+ # When a terminal row is re-opened so its finally block can run, the write that
+ # terminalized it already recorded the run's minutes up to that instant, and carrying
+ # that instant makes the in-band re-finalization below record only what ran after it.
+ # Out-of-band writers that terminalize the row mid-block (API cancel, the
+ # heartbeat-timeout activity, the durable fail finalizer) carry no such instant and
+ # still record now - started_at.
+ run_minutes_recorded_through: datetime | None = None
try:
effective_max_elapsed_minutes = get_effective_workflow_run_max_elapsed_time_minutes(
@@ -5347,6 +5354,12 @@ class WorkflowService:
WorkflowRunStatus.terminated,
WorkflowRunStatus.timed_out,
):
+ # A terminal write stamps ``finished_at`` in the same breath as it
+ # records run minutes, so the stamp is the instant those minutes
+ # were measured to. A row that reached a terminal status without one
+ # (bulk stuck-run cleanup writes status only) recorded nothing, and
+ # the re-finalization is then the run's first and only sample.
+ run_minutes_recorded_through = workflow_run.finished_at
workflow_run = await self._update_workflow_run_status(
workflow_run_id=workflow_run_id,
status=WorkflowRunStatus.running,
@@ -5518,6 +5531,7 @@ class WorkflowService:
pre_finally_status=pre_finally_status,
pre_finally_failure_reason=pre_finally_failure_reason,
pre_finally_failure_category=pre_finally_failure_category,
+ run_minutes_recorded_through=run_minutes_recorded_through,
)
)
except BaseException:
@@ -9416,6 +9430,7 @@ class WorkflowService:
failure_reason: str | None = None,
run_with: str | None = None,
failure_category: list[dict] | None = None,
+ run_minutes_recorded_through: datetime | None = None,
) -> WorkflowRun | None:
""":meth:`_update_workflow_run_status` for writers that must lose a race against the
run's own finalizer. Returns ``None`` when the row was already terminal."""
@@ -9428,7 +9443,11 @@ class WorkflowService:
)
if workflow_run is None:
return None
- await self._after_workflow_run_status_write(workflow_run, status)
+ await self._after_workflow_run_status_write(
+ workflow_run,
+ status,
+ run_minutes_recorded_through=run_minutes_recorded_through,
+ )
return workflow_run
async def _finish_preexisting_timed_out_workflow_run(
@@ -9452,25 +9471,40 @@ class WorkflowService:
return workflow_run
async def _after_workflow_run_status_write(
- self, workflow_run: WorkflowRun, status: WorkflowRunStatus, emit_run_minutes: bool = True
+ self,
+ workflow_run: WorkflowRun,
+ status: WorkflowRunStatus,
+ emit_run_minutes: bool = True,
+ run_minutes_recorded_through: datetime | None = None,
) -> None:
workflow_run_id = workflow_run.workflow_run_id
if status.is_final():
# Free extraction-cache entries for this run.
extraction_cache.clear_workflow_run(workflow_run_id)
+ now = datetime.now(UTC)
start_time = (
workflow_run.started_at.replace(tzinfo=UTC)
if workflow_run.started_at
else workflow_run.created_at.replace(tzinfo=UTC)
)
queued_seconds = (start_time - workflow_run.created_at.replace(tzinfo=UTC)).total_seconds()
- duration_seconds = (datetime.now(UTC) - start_time).total_seconds()
+ duration_seconds = (now - start_time).total_seconds()
+ # A run can reach a terminal status twice: the finally-block path terminalizes,
+ # re-opens the row to `running` so the block can execute, then terminalizes again.
+ # Both writes are real non-terminal -> terminal flips, so the second one records
+ # only the compute the first one had not measured yet.
+ recorded_seconds = (
+ duration_seconds
+ if run_minutes_recorded_through is None
+ else (now - run_minutes_recorded_through.replace(tzinfo=UTC)).total_seconds()
+ )
LOG.info(
"Workflow run duration metrics",
workflow_run_id=workflow_run_id,
workflow_id=workflow_run.workflow_id,
queued_seconds=queued_seconds,
duration_seconds=duration_seconds,
+ recorded_seconds=recorded_seconds,
workflow_run_status=workflow_run.status,
organization_id=workflow_run.organization_id,
run_with=workflow_run.run_with,
@@ -9486,7 +9520,7 @@ class WorkflowService:
await app.AGENT_FUNCTION.record_run_duration(
run_type="workflow_run",
status=str(status),
- duration_seconds=duration_seconds,
+ duration_seconds=recorded_seconds,
workflow_run_id=workflow_run_id,
organization_id=workflow_run.organization_id,
excluded_reason=None if workflow_run.started_at else "never_started",
@@ -9567,6 +9601,7 @@ class WorkflowService:
pre_finally_failure_category: list[dict] | None = None,
is_partial_run: bool = False,
requested_completion_contract: dict[str, Any] | None = None,
+ run_minutes_recorded_through: datetime | None = None,
) -> WorkflowRun:
"""
Set final workflow run status based on pre-finally state.
@@ -9612,6 +9647,7 @@ class WorkflowService:
status=pre_finally_status,
failure_reason=pre_finally_failure_reason,
failure_category=pre_finally_failure_category,
+ run_minutes_recorded_through=run_minutes_recorded_through,
)
if updated is None:
return await self._current_row_after_lost_finalize(workflow_run_id, workflow_run)
@@ -9979,12 +10015,15 @@ class WorkflowService:
)
queued_seconds = (start_time - updated.created_at.replace(tzinfo=UTC)).total_seconds()
duration_seconds = (datetime.now(UTC) - start_time).total_seconds()
+ # This path finalizes once and never re-opens the run, so the recorded sample is
+ # the whole duration. It is logged anyway so every emission site carries the field.
LOG.info(
"Workflow run duration metrics",
workflow_run_id=workflow_run_id,
workflow_id=updated.workflow_id,
queued_seconds=queued_seconds,
duration_seconds=duration_seconds,
+ recorded_seconds=duration_seconds,
workflow_run_status=updated.status,
organization_id=updated.organization_id,
run_with=updated.run_with,
diff --git a/tests/unit/test_workflow_elapsed_timeout.py b/tests/unit/test_workflow_elapsed_timeout.py
index 70911ba1a..5edb64cae 100644
--- a/tests/unit/test_workflow_elapsed_timeout.py
+++ b/tests/unit/test_workflow_elapsed_timeout.py
@@ -51,6 +51,7 @@ def _workflow_run(
max_elapsed_time_minutes=max_elapsed_time_minutes,
started_at=started_at,
created_at=started_at,
+ finished_at=None,
code_gen=False,
run_with="agent",
)
diff --git a/tests/unit/test_workflow_run_minutes_gate.py b/tests/unit/test_workflow_run_minutes_gate.py
index ac5dca0c7..7624a1d5b 100644
--- a/tests/unit/test_workflow_run_minutes_gate.py
+++ b/tests/unit/test_workflow_run_minutes_gate.py
@@ -16,7 +16,9 @@ only the post-claim row can say whether the task started.
from __future__ import annotations
-from datetime import UTC, datetime, timedelta
+import copy
+from datetime import UTC, datetime, timedelta, tzinfo
+from types import SimpleNamespace
from unittest.mock import AsyncMock, MagicMock
import pytest
@@ -25,6 +27,8 @@ from skyvern.forge import agent as agent_module
from skyvern.forge import app
from skyvern.forge.agent import ForgeAgent
from skyvern.forge.sdk.schemas.tasks import Task, TaskStatus
+from skyvern.forge.sdk.workflow import service as service_module
+from skyvern.forge.sdk.workflow.models.block import BlockType
from skyvern.forge.sdk.workflow.models.workflow import WorkflowRunStatus
from skyvern.forge.sdk.workflow.service import WorkflowService
@@ -145,3 +149,192 @@ async def test_task_v1_emission_reads_started_at_from_the_claim_not_the_entry_re
kwargs = record_run_duration.await_args.kwargs
assert kwargs.get("excluded_reason") is None
assert kwargs["duration_seconds"] == pytest.approx(9 * 60, abs=5)
+
+
+FINALLY_BLOCK_SECONDS = 5 * 60
+BODY_SECONDS = 20 * 60
+
+
+class _Clock:
+ """The service reads wall clock through ``service_module.datetime``. Driving it by hand is
+ what makes the body's minutes and the finally block's minutes two distinct intervals
+ rather than two reads of the same instant."""
+
+ def __init__(self, start: datetime) -> None:
+ self.now_value = start
+
+ def advance(self, seconds: float) -> None:
+ self.now_value += timedelta(seconds=seconds)
+
+ def now(self, tz: tzinfo | None = None) -> datetime:
+ return self.now_value if tz is None else self.now_value.astimezone(tz)
+
+
+class _FakeWorkflowRunStore:
+ """The two writers ``_update_workflow_run_status`` picks between, over one mutable row: a
+ conditional claim that refuses an already-terminal row, and the unconditional overwrite.
+ A terminal write stamps ``finished_at`` at the clock's current instant, as the real row does.
+ """
+
+ def __init__(self, row: SimpleNamespace, clock: _Clock) -> None:
+ self.row = row
+ self.clock = clock
+
+ def snapshot(self) -> SimpleNamespace:
+ return copy.copy(self.row)
+
+ async def get_workflow_run(self, workflow_run_id: str, organization_id: str | None = None) -> SimpleNamespace:
+ return self.snapshot()
+
+ async def update_workflow_run_if_not_final(
+ self, workflow_run_id: str, status: WorkflowRunStatus, **_: object
+ ) -> SimpleNamespace | None:
+ if self.row.status.is_final():
+ return None
+ self.row.status = status
+ if status.is_final():
+ self.row.finished_at = self.clock.now_value
+ return self.snapshot()
+
+ async def update_workflow_run(
+ self, workflow_run_id: str, status: WorkflowRunStatus | None = None, **_: object
+ ) -> SimpleNamespace:
+ if status is not None:
+ self.row.status = status
+ return self.snapshot()
+
+
+async def test_finally_block_re_finalization_records_only_the_minutes_it_added(
+ monkeypatch: pytest.MonkeyPatch,
+ record_run_duration: AsyncMock,
+) -> None:
+ """A run whose body terminalized it and whose workflow declares a finally block is written
+ back to ``running`` so the block can execute, then terminalized again. Both writes flip a
+ non-terminal row to a terminal one, so recording each one's full ``now - started_at``
+ bills the body twice (SKY-14606). The second write owes only the compute the first did
+ not measure -- and it does owe that, because the finally block is real work on the pod.
+ """
+ clock = _Clock(datetime.now(UTC))
+ started_at = clock.now_value
+ store = _FakeWorkflowRunStore(
+ SimpleNamespace(
+ workflow_run_id="wr_finally",
+ workflow_id="wf_finally",
+ workflow_permanent_id="wpid_finally",
+ organization_id="org_finally",
+ parent_workflow_run_id=None,
+ status=WorkflowRunStatus.running,
+ failure_reason=None,
+ failure_category=None,
+ created_at=started_at,
+ started_at=started_at,
+ finished_at=None,
+ run_with="agent",
+ ai_fallback=False,
+ trigger_type=None,
+ workflow_schedule_id=None,
+ browser_session_id=None,
+ browser_profile_id=None,
+ browser_address=None,
+ start_fresh_browser=None,
+ reuse_browser_session=None,
+ ignore_inherited_workflow_system_prompt=False,
+ proxy_location=None,
+ max_elapsed_time_minutes=None,
+ code_gen=False,
+ ),
+ clock,
+ )
+ workflow = SimpleNamespace(
+ workflow_id="wf_finally",
+ workflow_permanent_id="wpid_finally",
+ organization_id="org_finally",
+ title="Finally workflow",
+ persist_browser_session=False,
+ reuse_browser_session=False,
+ generate_script_on_terminal=False,
+ model=None,
+ workflow_definition=SimpleNamespace(
+ parameters=[],
+ finally_block_label="cleanup",
+ blocks=[SimpleNamespace(block_type=BlockType.TASK)],
+ ),
+ )
+ organization = SimpleNamespace(organization_id="org_finally")
+
+ monkeypatch.setattr(service_module, "datetime", SimpleNamespace(now=clock.now))
+ monkeypatch.setattr(
+ service_module.app,
+ "WORKFLOW_CONTEXT_MANAGER",
+ SimpleNamespace(
+ initialize_workflow_run_context=AsyncMock(),
+ get_workflow_run_context=lambda _workflow_run_id: SimpleNamespace(browser_session_id=None),
+ ),
+ )
+ monkeypatch.setattr(service_module.app, "DATABASE", SimpleNamespace(workflow_runs=store))
+ monkeypatch.setattr(service_module.workflow_script_service, "workflow_has_conditionals", lambda _workflow: False)
+ monkeypatch.setattr(
+ service_module.workflow_script_service,
+ "get_workflow_script",
+ AsyncMock(return_value=(None, None, False)),
+ )
+ monkeypatch.setattr(service_module.skyvern_context, "current", lambda: None)
+ monkeypatch.setattr(service_module, "is_adaptive_caching", lambda _workflow, _workflow_run: False)
+ monkeypatch.setattr(service_module, "_get_workflow_run_max_elapsed_timeout_seconds", lambda _workflow_run: 10.0)
+
+ svc = WorkflowService()
+
+ async def terminalize_inside_body(**_: object) -> tuple[SimpleNamespace, set[str]]:
+ clock.advance(BODY_SECONDS)
+ await svc.mark_workflow_run_as_terminated(
+ workflow_run_id="wr_finally",
+ failure_reason="terminate criterion matched",
+ )
+ return store.snapshot(), set()
+
+ statuses_seen_by_finally_block: list[WorkflowRunStatus] = []
+
+ async def observe_finally_block(**_: object) -> None:
+ statuses_seen_by_finally_block.append(store.row.status)
+ clock.advance(FINALLY_BLOCK_SECONDS)
+ return None
+
+ monkeypatch.setattr(svc, "get_workflow_run", AsyncMock(side_effect=lambda **_: store.snapshot()))
+ monkeypatch.setattr(svc, "get_workflow", AsyncMock(return_value=workflow))
+ monkeypatch.setattr(svc, "bind_browser_action_policy", AsyncMock(return_value=None))
+ monkeypatch.setattr(svc, "mark_workflow_run_as_running", AsyncMock(side_effect=lambda **_: store.snapshot()))
+ monkeypatch.setattr(svc, "get_workflow_run_parameter_tuples", AsyncMock(return_value=[]))
+ monkeypatch.setattr(svc, "get_workflow_output_parameters", AsyncMock(return_value=[]))
+ monkeypatch.setattr(svc, "_collect_inherited_workflow_system_prompt", AsyncMock(return_value=None))
+ monkeypatch.setattr(svc, "auto_create_browser_session_if_needed", AsyncMock(return_value=None))
+ monkeypatch.setattr(svc, "_browser_profile_is_managed", AsyncMock(return_value=False))
+ monkeypatch.setattr(svc, "_execute_workflow_blocks", AsyncMock(side_effect=terminalize_inside_body))
+ monkeypatch.setattr(svc, "generate_script_if_needed", AsyncMock())
+ monkeypatch.setattr(svc, "should_run_script", AsyncMock(return_value=False))
+ monkeypatch.setattr(svc, "_execute_finally_block_if_configured", AsyncMock(side_effect=observe_finally_block))
+ monkeypatch.setattr(svc, "clean_up_workflow", AsyncMock())
+
+ await svc.execute_workflow(workflow_run_id="wr_finally", api_key=None, organization=organization)
+
+ # The row really was re-opened and re-finalized. Without both flips there is nothing to
+ # double-count and the durations below would pass for the wrong reason.
+ assert statuses_seen_by_finally_block == [WorkflowRunStatus.running]
+ assert store.row.status == WorkflowRunStatus.terminated
+
+ # Two terminal writes, two samples: dropping the second would erase the finally block's
+ # own compute, which is as wrong as counting the body twice.
+ assert record_run_duration.await_count == 2
+ body_call, re_finalize_call = record_run_duration.await_args_list
+ assert [call.kwargs["status"] for call in (body_call, re_finalize_call)] == [str(WorkflowRunStatus.terminated)] * 2
+ assert body_call.kwargs["excluded_reason"] is None
+ assert body_call.kwargs["duration_seconds"] == pytest.approx(BODY_SECONDS)
+ assert re_finalize_call.kwargs["duration_seconds"] == pytest.approx(FINALLY_BLOCK_SECONDS)
+
+ # The invariant the delta form exists to hold: the samples partition the run's wall clock
+ # rather than overlapping on the body.
+ wall_clock_seconds = (clock.now_value - started_at).total_seconds()
+ assert wall_clock_seconds == pytest.approx(BODY_SECONDS + FINALLY_BLOCK_SECONDS)
+ assert sum(call.kwargs["duration_seconds"] for call in record_run_duration.await_args_list) == pytest.approx(
+ wall_clock_seconds
+ )
diff --git a/tests/unit/test_workflow_service_cancel.py b/tests/unit/test_workflow_service_cancel.py
index 9fd8417e6..70d058ca6 100644
--- a/tests/unit/test_workflow_service_cancel.py
+++ b/tests/unit/test_workflow_service_cancel.py
@@ -167,13 +167,15 @@ async def test_mark_canceled_if_not_final_logs_duration_metrics(
"""Terminal-transition parity with ``_update_workflow_run_status``: a
successful conditional cancel must emit the ``Workflow run duration metrics``
log with the same structured fields, so the metric stays comparable across
- statuses.
+ statuses. ``recorded_seconds`` must equal the sample handed to
+ ``record_run_duration`` so a log query keyed on it reconciles with the metric.
"""
from skyvern.forge.sdk.workflow import service as service_module
from skyvern.forge.sdk.workflow.service import WorkflowService
fixed_now = datetime(2026, 1, 1, tzinfo=UTC)
updated_row = _make_updated_row(fixed_now)
+ updated_row.parent_workflow_run_id = None
class FrozenDateTime(datetime):
@classmethod
@@ -193,6 +195,8 @@ async def test_mark_canceled_if_not_final_logs_duration_metrics(
"_sync_task_run_from_workflow_run",
AsyncMock(return_value=None),
)
+ recorder = AsyncMock()
+ monkeypatch.setattr(app.AGENT_FUNCTION, "record_run_duration", recorder)
info_calls: list[tuple[str, dict]] = []
@@ -214,6 +218,8 @@ async def test_mark_canceled_if_not_final_logs_duration_metrics(
assert metrics["workflow_run_status"] == WorkflowRunStatus.canceled
assert metrics["queued_seconds"] == pytest.approx(10.0, abs=1.0)
assert metrics["duration_seconds"] == pytest.approx(20.0, abs=1.0)
+ assert recorder.await_count == 1
+ assert metrics["recorded_seconds"] == recorder.await_args.kwargs["duration_seconds"]
@pytest.mark.asyncio