Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
Record the finally block's minutes instead of the run's body twice (SKY-14606) (#8284)
Shuchang Zhengef39e0892514
diff --git a/skyvern/forge/sdk/workflow/service.py b/skyvern/forge/sdk/workflow/service.pyindex 76ed865bb..740a2856a 100644--- a/skyvern/forge/sdk/workflow/service.py+++ b/skyvern/forge/sdk/workflow/service.py@@ -5143,6 +5143,13 @@ class WorkflowService:# A finally-only failure is a cleanup outcome, not evidence that replaying the# workflow with another credential can help.finally_block_set_terminal_outcome = False+ # When a terminal row is re-opened so its finally block can run, the write that+ # terminalized it already recorded the run's minutes up to that instant, and carrying+ # that instant makes the in-band re-finalization below record only what ran after it.+ # Out-of-band writers that terminalize the row mid-block (API cancel, the+ # heartbeat-timeout activity, the durable fail finalizer) carry no such instant and+ # still record now - started_at.+ run_minutes_recorded_through: datetime | None = Nonetry:effective_max_elapsed_minutes = get_effective_workflow_run_max_elapsed_time_minutes(@@ -5347,6 +5354,12 @@ class WorkflowService:WorkflowRunStatus.terminated,WorkflowRunStatus.timed_out,):+ # A terminal write stamps ``finished_at`` in the same breath as it+ # records run minutes, so the stamp is the instant those minutes+ # were measured to. A row that reached a terminal status without one+ # (bulk stuck-run cleanup writes status only) recorded nothing, and+ # the re-finalization is then the run's first and only sample.+ run_minutes_recorded_through = workflow_run.finished_atworkflow_run = await self._update_workflow_run_status(workflow_run_id=workflow_run_id,status=WorkflowRunStatus.running,@@ -5518,6 +5531,7 @@ class WorkflowService:pre_finally_status=pre_finally_status,pre_finally_failure_reason=pre_finally_failure_reason,pre_finally_failure_category=pre_finally_failure_category,+ run_minutes_recorded_through=run_minutes_recorded_through,))except BaseException:@@ -9416,6 +9430,7 @@ class WorkflowService:failure_reason: str | None = None,run_with: str | None = None,failure_category: list[dict] | None = None,+ run_minutes_recorded_through: datetime | None = None,) -> WorkflowRun | None:""":meth:`_update_workflow_run_status` for writers that must lose a race against therun's own finalizer. Returns ``None`` when the row was already terminal."""@@ -9428,7 +9443,11 @@ class WorkflowService:)if workflow_run is None:return None- await self._after_workflow_run_status_write(workflow_run, status)+ await self._after_workflow_run_status_write(+ workflow_run,+ status,+ run_minutes_recorded_through=run_minutes_recorded_through,+ )return workflow_runasync def _finish_preexisting_timed_out_workflow_run(@@ -9452,25 +9471,40 @@ class WorkflowService:return workflow_runasync def _after_workflow_run_status_write(- self, workflow_run: WorkflowRun, status: WorkflowRunStatus, emit_run_minutes: bool = True+ self,+ workflow_run: WorkflowRun,+ status: WorkflowRunStatus,+ emit_run_minutes: bool = True,+ run_minutes_recorded_through: datetime | None = None,) -> None:workflow_run_id = workflow_run.workflow_run_idif status.is_final():# Free extraction-cache entries for this run.extraction_cache.clear_workflow_run(workflow_run_id)+ now = datetime.now(UTC)start_time = (workflow_run.started_at.replace(tzinfo=UTC)if workflow_run.started_atelse workflow_run.created_at.replace(tzinfo=UTC))queued_seconds = (start_time - workflow_run.created_at.replace(tzinfo=UTC)).total_seconds()- duration_seconds = (datetime.now(UTC) - start_time).total_seconds()+ duration_seconds = (now - start_time).total_seconds()+ # A run can reach a terminal status twice: the finally-block path terminalizes,+ # re-opens the row to `running` so the block can execute, then terminalizes again.+ # Both writes are real non-terminal -> terminal flips, so the second one records+ # only the compute the first one had not measured yet.+ recorded_seconds = (+ duration_seconds+ if run_minutes_recorded_through is None+ else (now - run_minutes_recorded_through.replace(tzinfo=UTC)).total_seconds()+ )LOG.info("Workflow run duration metrics",workflow_run_id=workflow_run_id,workflow_id=workflow_run.workflow_id,queued_seconds=queued_seconds,duration_seconds=duration_seconds,+ recorded_seconds=recorded_seconds,workflow_run_status=workflow_run.status,organization_id=workflow_run.organization_id,run_with=workflow_run.run_with,@@ -9486,7 +9520,7 @@ class WorkflowService:await app.AGENT_FUNCTION.record_run_duration(run_type="workflow_run",status=str(status),- duration_seconds=duration_seconds,+ duration_seconds=recorded_seconds,workflow_run_id=workflow_run_id,organization_id=workflow_run.organization_id,excluded_reason=None if workflow_run.started_at else "never_started",@@ -9567,6 +9601,7 @@ class WorkflowService:pre_finally_failure_category: list[dict] | None = None,is_partial_run: bool = False,requested_completion_contract: dict[str, Any] | None = None,+ run_minutes_recorded_through: datetime | None = None,) -> WorkflowRun:"""Set final workflow run status based on pre-finally state.@@ -9612,6 +9647,7 @@ class WorkflowService:status=pre_finally_status,failure_reason=pre_finally_failure_reason,failure_category=pre_finally_failure_category,+ run_minutes_recorded_through=run_minutes_recorded_through,)if updated is None:return await self._current_row_after_lost_finalize(workflow_run_id, workflow_run)@@ -9979,12 +10015,15 @@ class WorkflowService:)queued_seconds = (start_time - updated.created_at.replace(tzinfo=UTC)).total_seconds()duration_seconds = (datetime.now(UTC) - start_time).total_seconds()+ # This path finalizes once and never re-opens the run, so the recorded sample is+ # the whole duration. It is logged anyway so every emission site carries the field.LOG.info("Workflow run duration metrics",workflow_run_id=workflow_run_id,workflow_id=updated.workflow_id,queued_seconds=queued_seconds,duration_seconds=duration_seconds,+ recorded_seconds=duration_seconds,workflow_run_status=updated.status,organization_id=updated.organization_id,run_with=updated.run_with,diff --git a/tests/unit/test_workflow_elapsed_timeout.py b/tests/unit/test_workflow_elapsed_timeout.pyindex 70911ba1a..5edb64cae 100644--- a/tests/unit/test_workflow_elapsed_timeout.py+++ b/tests/unit/test_workflow_elapsed_timeout.py@@ -51,6 +51,7 @@ def _workflow_run(max_elapsed_time_minutes=max_elapsed_time_minutes,started_at=started_at,created_at=started_at,+ finished_at=None,code_gen=False,run_with="agent",)diff --git a/tests/unit/test_workflow_run_minutes_gate.py b/tests/unit/test_workflow_run_minutes_gate.pyindex ac5dca0c7..7624a1d5b 100644--- a/tests/unit/test_workflow_run_minutes_gate.py+++ b/tests/unit/test_workflow_run_minutes_gate.py@@ -16,7 +16,9 @@ only the post-claim row can say whether the task started.from __future__ import annotations-from datetime import UTC, datetime, timedelta+import copy+from datetime import UTC, datetime, timedelta, tzinfo+from types import SimpleNamespacefrom unittest.mock import AsyncMock, MagicMockimport pytest@@ -25,6 +27,8 @@ from skyvern.forge import agent as agent_modulefrom skyvern.forge import appfrom skyvern.forge.agent import ForgeAgentfrom skyvern.forge.sdk.schemas.tasks import Task, TaskStatus+from skyvern.forge.sdk.workflow import service as service_module+from skyvern.forge.sdk.workflow.models.block import BlockTypefrom skyvern.forge.sdk.workflow.models.workflow import WorkflowRunStatusfrom skyvern.forge.sdk.workflow.service import WorkflowService@@ -145,3 +149,192 @@ async def test_task_v1_emission_reads_started_at_from_the_claim_not_the_entry_rekwargs = record_run_duration.await_args.kwargsassert kwargs.get("excluded_reason") is Noneassert kwargs["duration_seconds"] == pytest.approx(9 * 60, abs=5)+++FINALLY_BLOCK_SECONDS = 5 * 60+BODY_SECONDS = 20 * 60+++class _Clock:+ """The service reads wall clock through ``service_module.datetime``. Driving it by hand is+ what makes the body's minutes and the finally block's minutes two distinct intervals+ rather than two reads of the same instant."""++ def __init__(self, start: datetime) -> None:+ self.now_value = start++ def advance(self, seconds: float) -> None:+ self.now_value += timedelta(seconds=seconds)++ def now(self, tz: tzinfo | None = None) -> datetime:+ return self.now_value if tz is None else self.now_value.astimezone(tz)+++class _FakeWorkflowRunStore:+ """The two writers ``_update_workflow_run_status`` picks between, over one mutable row: a+ conditional claim that refuses an already-terminal row, and the unconditional overwrite.+ A terminal write stamps ``finished_at`` at the clock's current instant, as the real row does.+ """++ def __init__(self, row: SimpleNamespace, clock: _Clock) -> None:+ self.row = row+ self.clock = clock++ def snapshot(self) -> SimpleNamespace:+ return copy.copy(self.row)++ async def get_workflow_run(self, workflow_run_id: str, organization_id: str | None = None) -> SimpleNamespace:+ return self.snapshot()++ async def update_workflow_run_if_not_final(+ self, workflow_run_id: str, status: WorkflowRunStatus, **_: object+ ) -> SimpleNamespace | None:+ if self.row.status.is_final():+ return None+ self.row.status = status+ if status.is_final():+ self.row.finished_at = self.clock.now_value+ return self.snapshot()++ async def update_workflow_run(+ self, workflow_run_id: str, status: WorkflowRunStatus | None = None, **_: object+ ) -> SimpleNamespace:+ if status is not None:+ self.row.status = status+ return self.snapshot()+++async def test_finally_block_re_finalization_records_only_the_minutes_it_added(+ monkeypatch: pytest.MonkeyPatch,+ record_run_duration: AsyncMock,+) -> None:+ """A run whose body terminalized it and whose workflow declares a finally block is written+ back to ``running`` so the block can execute, then terminalized again. Both writes flip a+ non-terminal row to a terminal one, so recording each one's full ``now - started_at``+ bills the body twice (SKY-14606). The second write owes only the compute the first did+ not measure -- and it does owe that, because the finally block is real work on the pod.+ """+ clock = _Clock(datetime.now(UTC))+ started_at = clock.now_value+ store = _FakeWorkflowRunStore(+ SimpleNamespace(+ workflow_run_id="wr_finally",+ workflow_id="wf_finally",+ workflow_permanent_id="wpid_finally",+ organization_id="org_finally",+ parent_workflow_run_id=None,+ status=WorkflowRunStatus.running,+ failure_reason=None,+ failure_category=None,+ created_at=started_at,+ started_at=started_at,+ finished_at=None,+ run_with="agent",+ ai_fallback=False,+ trigger_type=None,+ workflow_schedule_id=None,+ browser_session_id=None,+ browser_profile_id=None,+ browser_address=None,+ start_fresh_browser=None,+ reuse_browser_session=None,+ ignore_inherited_workflow_system_prompt=False,+ proxy_location=None,+ max_elapsed_time_minutes=None,+ code_gen=False,+ ),+ clock,+ )+ workflow = SimpleNamespace(+ workflow_id="wf_finally",+ workflow_permanent_id="wpid_finally",+ organization_id="org_finally",+ title="Finally workflow",+ persist_browser_session=False,+ reuse_browser_session=False,+ generate_script_on_terminal=False,+ model=None,+ workflow_definition=SimpleNamespace(+ parameters=[],+ finally_block_label="cleanup",+ blocks=[SimpleNamespace(block_type=BlockType.TASK)],+ ),+ )+ organization = SimpleNamespace(organization_id="org_finally")++ monkeypatch.setattr(service_module, "datetime", SimpleNamespace(now=clock.now))+ monkeypatch.setattr(+ service_module.app,+ "WORKFLOW_CONTEXT_MANAGER",+ SimpleNamespace(+ initialize_workflow_run_context=AsyncMock(),+ get_workflow_run_context=lambda _workflow_run_id: SimpleNamespace(browser_session_id=None),+ ),+ )+ monkeypatch.setattr(service_module.app, "DATABASE", SimpleNamespace(workflow_runs=store))+ monkeypatch.setattr(service_module.workflow_script_service, "workflow_has_conditionals", lambda _workflow: False)+ monkeypatch.setattr(+ service_module.workflow_script_service,+ "get_workflow_script",+ AsyncMock(return_value=(None, None, False)),+ )+ monkeypatch.setattr(service_module.skyvern_context, "current", lambda: None)+ monkeypatch.setattr(service_module, "is_adaptive_caching", lambda _workflow, _workflow_run: False)+ monkeypatch.setattr(service_module, "_get_workflow_run_max_elapsed_timeout_seconds", lambda _workflow_run: 10.0)++ svc = WorkflowService()++ async def terminalize_inside_body(**_: object) -> tuple[SimpleNamespace, set[str]]:+ clock.advance(BODY_SECONDS)+ await svc.mark_workflow_run_as_terminated(+ workflow_run_id="wr_finally",+ failure_reason="terminate criterion matched",+ )+ return store.snapshot(), set()++ statuses_seen_by_finally_block: list[WorkflowRunStatus] = []++ async def observe_finally_block(**_: object) -> None:+ statuses_seen_by_finally_block.append(store.row.status)+ clock.advance(FINALLY_BLOCK_SECONDS)+ return None++ monkeypatch.setattr(svc, "get_workflow_run", AsyncMock(side_effect=lambda **_: store.snapshot()))+ monkeypatch.setattr(svc, "get_workflow", AsyncMock(return_value=workflow))+ monkeypatch.setattr(svc, "bind_browser_action_policy", AsyncMock(return_value=None))+ monkeypatch.setattr(svc, "mark_workflow_run_as_running", AsyncMock(side_effect=lambda **_: store.snapshot()))+ monkeypatch.setattr(svc, "get_workflow_run_parameter_tuples", AsyncMock(return_value=[]))+ monkeypatch.setattr(svc, "get_workflow_output_parameters", AsyncMock(return_value=[]))+ monkeypatch.setattr(svc, "_collect_inherited_workflow_system_prompt", AsyncMock(return_value=None))+ monkeypatch.setattr(svc, "auto_create_browser_session_if_needed", AsyncMock(return_value=None))+ monkeypatch.setattr(svc, "_browser_profile_is_managed", AsyncMock(return_value=False))+ monkeypatch.setattr(svc, "_execute_workflow_blocks", AsyncMock(side_effect=terminalize_inside_body))+ monkeypatch.setattr(svc, "generate_script_if_needed", AsyncMock())+ monkeypatch.setattr(svc, "should_run_script", AsyncMock(return_value=False))+ monkeypatch.setattr(svc, "_execute_finally_block_if_configured", AsyncMock(side_effect=observe_finally_block))+ monkeypatch.setattr(svc, "clean_up_workflow", AsyncMock())++ await svc.execute_workflow(workflow_run_id="wr_finally", api_key=None, organization=organization)++ # The row really was re-opened and re-finalized. Without both flips there is nothing to+ # double-count and the durations below would pass for the wrong reason.+ assert statuses_seen_by_finally_block == [WorkflowRunStatus.running]+ assert store.row.status == WorkflowRunStatus.terminated++ # Two terminal writes, two samples: dropping the second would erase the finally block's+ # own compute, which is as wrong as counting the body twice.+ assert record_run_duration.await_count == 2+ body_call, re_finalize_call = record_run_duration.await_args_list+ assert [call.kwargs["status"] for call in (body_call, re_finalize_call)] == [str(WorkflowRunStatus.terminated)] * 2+ assert body_call.kwargs["excluded_reason"] is None+ assert body_call.kwargs["duration_seconds"] == pytest.approx(BODY_SECONDS)+ assert re_finalize_call.kwargs["duration_seconds"] == pytest.approx(FINALLY_BLOCK_SECONDS)++ # The invariant the delta form exists to hold: the samples partition the run's wall clock+ # rather than overlapping on the body.+ wall_clock_seconds = (clock.now_value - started_at).total_seconds()+ assert wall_clock_seconds == pytest.approx(BODY_SECONDS + FINALLY_BLOCK_SECONDS)+ assert sum(call.kwargs["duration_seconds"] for call in record_run_duration.await_args_list) == pytest.approx(+ wall_clock_seconds+ )diff --git a/tests/unit/test_workflow_service_cancel.py b/tests/unit/test_workflow_service_cancel.pyindex 9fd8417e6..70d058ca6 100644--- a/tests/unit/test_workflow_service_cancel.py+++ b/tests/unit/test_workflow_service_cancel.py@@ -167,13 +167,15 @@ async def test_mark_canceled_if_not_final_logs_duration_metrics("""Terminal-transition parity with ``_update_workflow_run_status``: asuccessful conditional cancel must emit the ``Workflow run duration metrics``log with the same structured fields, so the metric stays comparable across- statuses.+ statuses. ``recorded_seconds`` must equal the sample handed to+ ``record_run_duration`` so a log query keyed on it reconciles with the metric."""from skyvern.forge.sdk.workflow import service as service_modulefrom skyvern.forge.sdk.workflow.service import WorkflowServicefixed_now = datetime(2026, 1, 1, tzinfo=UTC)updated_row = _make_updated_row(fixed_now)+ updated_row.parent_workflow_run_id = Noneclass FrozenDateTime(datetime):@classmethod@@ -193,6 +195,8 @@ async def test_mark_canceled_if_not_final_logs_duration_metrics("_sync_task_run_from_workflow_run",AsyncMock(return_value=None),)+ recorder = AsyncMock()+ monkeypatch.setattr(app.AGENT_FUNCTION, "record_run_duration", recorder)info_calls: list[tuple[str, dict]] = []@@ -214,6 +218,8 @@ async def test_mark_canceled_if_not_final_logs_duration_metrics(assert metrics["workflow_run_status"] == WorkflowRunStatus.canceledassert metrics["queued_seconds"] == pytest.approx(10.0, abs=1.0)assert metrics["duration_seconds"] == pytest.approx(20.0, abs=1.0)+ assert recorder.await_count == 1+ assert metrics["recorded_seconds"] == recorder.await_args.kwargs["duration_seconds"]@pytest.mark.asyncio