Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
Count only real compute in run minutes (SKY-14608, SKY-14609) (#8247)
Shuchang Zhengc68a662fb7ff
diff --git a/skyvern/forge/agent.py b/skyvern/forge/agent.pyindex 2ea4c8d04..0e37a791a 100644--- a/skyvern/forge/agent.py+++ b/skyvern/forge/agent.py@@ -6551,10 +6551,10 @@ class ForgeAgent:if getattr(task, key) != value}- start_time = task.started_at.replace(tzinfo=UTC) if task.started_at else task.created_at.replace(tzinfo=UTC)- duration_seconds = (datetime.now(UTC) - start_time).total_seconds()# Track task duration when task is completed, failed, or terminatedif status in [TaskStatus.completed, TaskStatus.failed, TaskStatus.terminated]:+ start_time = task.started_at.replace(tzinfo=UTC) if task.started_at else task.created_at.replace(tzinfo=UTC)+ duration_seconds = (datetime.now(UTC) - start_time).total_seconds()queued_seconds = (start_time - task.created_at.replace(tzinfo=UTC)).total_seconds()LOG.info("Task duration metrics",@@ -6568,19 +6568,33 @@ class ForgeAgent:)await save_task_logs(task.task_id)LOG.info("Updating task in db", task_id=task.task_id, diff=update_comparison, sampling=True)- updated_task = await app.DATABASE.tasks.update_task(+ updated_task, finish_claimed = await app.DATABASE.tasks.update_task_and_claim_finish(task.task_id,organization_id=task.organization_id,**updates,)- # Minutes emit after the terminal write lands and only on the non-final ->- # final transition (task holds the pre-write status), so neither a retried- # failed write nor a repeated terminal update double-counts the run;- # is_final() includes canceled and timed_out, matching the workflow-run gate.- if task.workflow_run_id is None and status.is_final() and not task.status.is_final():- await app.AGENT_FUNCTION.record_run_duration(- run_type="task_v1", status=str(status), duration_seconds=duration_seconds- )+ # Minutes emit only when THIS write atomically flipped finished_at from NULL+ # -- the shared claim every task_v1 finalizer rides -- so a concurrent+ # finalizer (the stuck-task sweep's CAS, the temporal timeout activity, a+ # retried write) cannot double-count the run. Both the duration and the+ # exclusion come from the post-claim row, never the entry read: a worker can+ # stamp started_at between the two, and the stale read would bill that+ # compute as never_started.+ if updated_task.workflow_run_id is None and finish_claimed:+ if updated_task.started_at and updated_task.finished_at:+ await app.AGENT_FUNCTION.record_run_duration(+ run_type="task_v1",+ status=str(status),+ duration_seconds=(updated_task.finished_at - updated_task.started_at).total_seconds(),+ )+ else:+ # Never started: no compute, but export the exclusion.+ await app.AGENT_FUNCTION.record_run_duration(+ run_type="task_v1",+ status=str(status),+ duration_seconds=0.0,+ excluded_reason="never_started",+ )return updated_taskasync def _handle_completed_step_with_parallel_verification(diff --git a/skyvern/forge/agent_functions.py b/skyvern/forge/agent_functions.pyindex 72b63c02e..d12a1784d 100644--- a/skyvern/forge/agent_functions.py+++ b/skyvern/forge/agent_functions.py@@ -886,11 +886,14 @@ class AgentFunction:duration_seconds: float,workflow_run_id: str | None = None,organization_id: str | None = None,+ excluded_reason: str | None = None,) -> None:"""Cloud overrides this to emit run-duration telemetry; the OSS default is a no-op.workflow_run_id/organization_id let the override refine run_type (e.g. aworkflow run that backs a task_v2) without the caller paying for the lookup.+ excluded_reason marks a run whose duration must not count as compute (e.g.+ it never started); the override records it as an exclusion, not as minutes."""return Nonediff --git a/skyvern/forge/sdk/db/repositories/tasks.py b/skyvern/forge/sdk/db/repositories/tasks.pyindex 1c7e6944b..f0456d9c7 100644--- a/skyvern/forge/sdk/db/repositories/tasks.py+++ b/skyvern/forge/sdk/db/repositories/tasks.py@@ -591,6 +591,64 @@ class TasksRepository(BaseRepository):organization_id: str | None = None,failure_category: list[dict[str, Any]] | None = None,) -> Task:+ updated_task, _ = await self._update_task_with_finish_claim(+ task_id,+ status=status,+ extracted_information=extracted_information,+ webhook_failure_reason=webhook_failure_reason,+ failure_reason=failure_reason,+ errors=errors,+ max_steps_per_run=max_steps_per_run,+ organization_id=organization_id,+ failure_category=failure_category,+ )+ return updated_task++ @traced(name="skyvern.db.update_task_and_claim_finish")+ @db_operation("update_task_and_claim_finish")+ async def update_task_and_claim_finish(+ self,+ task_id: str,+ status: TaskStatus | None = None,+ extracted_information: dict[str, Any] | list | str | None = None,+ webhook_failure_reason: str | None = None,+ failure_reason: str | None = None,+ errors: list[dict[str, Any]] | None = None,+ max_steps_per_run: int | None = None,+ organization_id: str | None = None,+ failure_category: list[dict[str, Any]] | None = None,+ ) -> tuple[Task, bool]:+ """``update_task``, plus whether THIS write flipped ``finished_at`` from NULL.++ The flip happens under a row lock and every finalizer stamps ``finished_at``+ atomically with its terminal status (here and in the bulk CAS paths), so at+ most one concurrent finalizer sees True. Callers use it as the exactly-once+ gate for per-task side effects such as the run-minutes emission.+ """+ return await self._update_task_with_finish_claim(+ task_id,+ status=status,+ extracted_information=extracted_information,+ webhook_failure_reason=webhook_failure_reason,+ failure_reason=failure_reason,+ errors=errors,+ max_steps_per_run=max_steps_per_run,+ organization_id=organization_id,+ failure_category=failure_category,+ )++ async def _update_task_with_finish_claim(+ self,+ task_id: str,+ status: TaskStatus | None = None,+ extracted_information: dict[str, Any] | list | str | None = None,+ webhook_failure_reason: str | None = None,+ failure_reason: str | None = None,+ errors: list[dict[str, Any]] | None = None,+ max_steps_per_run: int | None = None,+ organization_id: str | None = None,+ failure_category: list[dict[str, Any]] | None = None,+ ) -> tuple[Task, bool]:if (status is Noneand extracted_information is None@@ -603,10 +661,17 @@ class TasksRepository(BaseRepository):raise ValueError("At least one of status, extracted_information, or failure_reason must be provided to update the task")+ finish_claimed = Falseasync with self.Session() as session:if task := (await session.scalars(- select(TaskModel).filter_by(task_id=task_id).filter_by(organization_id=organization_id)+ select(TaskModel)+ .filter_by(task_id=task_id)+ .filter_by(organization_id=organization_id)+ # The row lock makes the finished_at NULL->set flip below an atomic+ # claim: concurrent finalizers serialize here, and only the first+ # sees finished_at still NULL.+ .with_for_update())).first():if status is not None:@@ -617,6 +682,7 @@ class TasksRepository(BaseRepository):task.started_at = naive_utc_now()if status.is_final() and task.finished_at is None:task.finished_at = naive_utc_now()+ finish_claimed = Trueif extracted_information is not None:task.extracted_information = extracted_informationif failure_reason is not None:@@ -649,10 +715,46 @@ class TasksRepository(BaseRepository):self._background_tasks.add(bg)bg.add_done_callback(self._background_tasks.discard)- return updated_task+ return updated_task, finish_claimedelse:raise NotFoundError("Task not found")+ @traced(name="skyvern.db.reset_task_for_rerun")+ @db_operation("reset_task_for_rerun")+ async def reset_task_for_rerun(self, task_id: str, organization_id: str) -> Task:+ """Return a task to ``created`` with its lifecycle timestamps cleared.++ ``update_task`` only ever stamps timestamps forward, so it cannot undo a finished+ run. Clearing ``finished_at`` here is what re-arms the exactly-once finish claim in+ ``_update_task_with_finish_claim``: leaving it set spends the rerun's claim before it+ begins, and the rerun's real compute never emits.+ """+ async with self.Session() as session:+ if task := (+ await session.scalars(+ select(TaskModel)+ .filter_by(task_id=task_id)+ .filter_by(organization_id=organization_id)+ .with_for_update()+ )+ ).first():+ task.status = TaskStatus.created+ task.queued_at = None+ task.started_at = None+ task.finished_at = None+ await session.commit()+ await session.refresh(task)+ reset_task = convert_to_task(task, debug_enabled=self.debug_enabled)+ else:+ raise NotFoundError("Task not found")++ await self.sync_task_run_status(+ organization_id=organization_id,+ run_id=task_id,+ status=TaskStatus.created.value,+ )+ return reset_task+@db_operation("update_task_2fa_state")async def update_task_2fa_state(self,@@ -711,9 +813,11 @@ class TasksRepository(BaseRepository):return []async with self.Session() as session:- update_values = {}+ update_values: dict[str, Any] = {}if status:update_values["status"] = status.value+ if status.is_final():+ update_values["finished_at"] = func.coalesce(TaskModel.finished_at, naive_utc_now())if failure_reason:update_values["failure_reason"] = failure_reasondiff --git a/skyvern/forge/sdk/workflow/service.py b/skyvern/forge/sdk/workflow/service.pyindex c1837b13c..11a65c913 100644--- a/skyvern/forge/sdk/workflow/service.py+++ b/skyvern/forge/sdk/workflow/service.py@@ -9478,6 +9478,10 @@ class WorkflowService:trigger_type=workflow_run.trigger_type,workflow_schedule_id=workflow_run.workflow_schedule_id,)+ # Run minutes measure compute. A run finalized without ever reaching+ # `running` held no pod, and the created_at fallback above would bill its+ # whole queue age instead -- so it records as an exclusion (a zero-minute+ # sample tagged excluded=never_started) rather than silently vanishing.if emit_run_minutes and workflow_run.parent_workflow_run_id is None:await app.AGENT_FUNCTION.record_run_duration(run_type="workflow_run",@@ -9485,6 +9489,7 @@ class WorkflowService:duration_seconds=duration_seconds,workflow_run_id=workflow_run_id,organization_id=workflow_run.organization_id,+ excluded_reason=None if workflow_run.started_at else "never_started",)await self._apply_completion_run_tags_best_effort(workflow_run)self._schedule_workflow_run_terminal_hooks(@@ -9987,6 +9992,9 @@ class WorkflowService:trigger_type=updated.trigger_type,workflow_schedule_id=updated.workflow_schedule_id,)+ # Same compute gate as ``_after_workflow_run_status_write``: cancelling a run+ # that never started bills queue age, not compute, so it records as a tagged+ # zero-minute exclusion instead.if updated.parent_workflow_run_id is None:await app.AGENT_FUNCTION.record_run_duration(run_type="workflow_run",@@ -9994,6 +10002,7 @@ class WorkflowService:duration_seconds=duration_seconds,workflow_run_id=workflow_run_id,organization_id=updated.organization_id,+ excluded_reason=None if updated.started_at else "never_started",)await self._apply_completion_run_tags_best_effort(updated)diff --git a/skyvern/webeye/utils/dom.py b/skyvern/webeye/utils/dom.pyindex 917c402e9..31f774bd6 100644--- a/skyvern/webeye/utils/dom.py+++ b/skyvern/webeye/utils/dom.py@@ -901,7 +901,11 @@ class SkyvernElement:if unique_id is None:return None- return await dom.get_skyvern_element_by_id(unique_id)+ try:+ return await dom.get_skyvern_element_by_id(unique_id)+ except MissingElementDict:+ # The for= control is live but was not scraped; an unmapped control is None, not a failed click.+ return None@staticmethodasync def _label_click_forwards_to_descendant(label_locator: Locator, *, fail_closed: bool = False) -> bool:diff --git a/tests/unit/forge/sdk/db/test_repositories.py b/tests/unit/forge/sdk/db/test_repositories.pyindex 66132c033..d244cdc3e 100644--- a/tests/unit/forge/sdk/db/test_repositories.py+++ b/tests/unit/forge/sdk/db/test_repositories.py@@ -8,8 +8,9 @@ from unittest.mock import MagicMock, patchimport pytestfrom sqlalchemy.ext.asyncio import AsyncEngine, async_sessionmaker-from skyvern.forge.sdk.db.models import OrganizationModel+from skyvern.forge.sdk.db.models import OrganizationModel, TaskModelfrom skyvern.forge.sdk.db.repositories.organizations import OrganizationsRepository+from skyvern.forge.sdk.db.repositories.tasks import TasksRepositoryfrom skyvern.forge.sdk.schemas.tasks import TaskStatusfrom tests.unit.conftest import MockAsyncSessionCtx, make_mock_session@@ -462,3 +463,114 @@ async def test_create_task_leaves_started_at_unset_for_other_statuses(monkeypatctask = await _create_task_with_status(monkeypatch, TaskStatus.created.value)assert task.started_at is None+++async def test_task_finish_claim_is_exactly_once_across_racing_finalizers(sqlite_engine: AsyncEngine):+ """Two finalizers landing on one task must produce exactly one finish claim.++ The arbitration is the finished_at NULL->set flip: bulk_update_tasks' status+ CAS performs it atomically with its claim, and update_task_and_claim_finish+ reports whether ITS write performed it. The first interleaving encodes the+ reproduced race where a concurrent-agent-style writer pre-read a non-final+ status, the cron sweep claimed the task, and the agent's write still landed:+ the write lands, but the claim -- and any per-task side effect gated on it --+ stays with the sweep.+ """+ factory = async_sessionmaker(sqlite_engine, expire_on_commit=False)+ repo = TasksRepository(session_factory=factory, debug_enabled=False)+ started = datetime.now(timezone.utc).replace(tzinfo=None)++ async def _seed(task_id: str) -> None:+ async with factory() as session:+ session.add(+ TaskModel(+ task_id=task_id,+ organization_id="o_race",+ status=TaskStatus.running.value,+ url="https://example.test/",+ started_at=started,+ errors=[],+ )+ )+ await session.commit()++ # Sweep first: its CAS claim IS the flip, so the racing agent-style write+ # gets claim=False even though its (stale) pre-read saw a non-final status.+ await _seed("tsk_sweep_first")+ swept = await repo.bulk_update_tasks(+ ["tsk_sweep_first"], status=TaskStatus.timed_out, only_if_status_in=[TaskStatus.running]+ )+ _, agent_claimed = await repo.update_task_and_claim_finish(+ "tsk_sweep_first", status=TaskStatus.completed, organization_id="o_race"+ )+ assert swept == ["tsk_sweep_first"]+ assert agent_claimed is False++ # Agent first: the sweep's CAS finds no non-final row and claims nothing.+ await _seed("tsk_agent_first")+ _, agent_claimed = await repo.update_task_and_claim_finish(+ "tsk_agent_first", status=TaskStatus.completed, organization_id="o_race"+ )+ swept = await repo.bulk_update_tasks(+ ["tsk_agent_first"], status=TaskStatus.timed_out, only_if_status_in=[TaskStatus.running]+ )+ assert agent_claimed is True+ assert swept == []++ # Same writer twice (an overlapping activity retry): one flip, one claim.+ await _seed("tsk_retry")+ _, first = await repo.update_task_and_claim_finish(+ "tsk_retry", status=TaskStatus.timed_out, organization_id="o_race"+ )+ _, second = await repo.update_task_and_claim_finish(+ "tsk_retry", status=TaskStatus.timed_out, organization_id="o_race"+ )+ assert (first, second) == (True, False)++ for background_sync in list(repo._background_tasks):+ await background_sync+++async def test_resetting_a_task_for_rerun_re_arms_its_finish_claim(sqlite_engine: AsyncEngine):+ """A rerun of a finished task must be able to claim its own finish.++ The claim is the finished_at NULL->set flip, so a reset that leaves finished_at+ set hands the rerun a spent claim: its real compute never emits, and a later+ sweep that does claim the row emits the PREVIOUS run's duration.+ """+ factory = async_sessionmaker(sqlite_engine, expire_on_commit=False)+ repo = TasksRepository(session_factory=factory, debug_enabled=False)++ async with factory() as session:+ session.add(+ TaskModel(+ task_id="tsk_rerun",+ organization_id="o_rerun",+ status=TaskStatus.running.value,+ url="https://example.test/",+ queued_at=datetime.now(timezone.utc).replace(tzinfo=None),+ started_at=datetime.now(timezone.utc).replace(tzinfo=None),+ errors=[],+ )+ )+ await session.commit()++ _, first_claim = await repo.update_task_and_claim_finish(+ "tsk_rerun", status=TaskStatus.completed, organization_id="o_rerun"+ )+ assert first_claim is True++ reset_task = await repo.reset_task_for_rerun(task_id="tsk_rerun", organization_id="o_rerun")++ assert reset_task.status == TaskStatus.created+ assert (reset_task.queued_at, reset_task.started_at, reset_task.finished_at) == (None, None, None)++ _, rerun_claim = await repo.update_task_and_claim_finish(+ "tsk_rerun", status=TaskStatus.completed, organization_id="o_rerun"+ )+ assert rerun_claim is True++ for background_sync in list(repo._background_tasks):+ await background_syncdiff --git a/tests/unit/test_stale_context_guards.py b/tests/unit/test_stale_context_guards.pyindex fa8e06f0f..828c11819 100644--- a/tests/unit/test_stale_context_guards.py+++ b/tests/unit/test_stale_context_guards.py@@ -6,7 +6,7 @@ import pytestfrom playwright.async_api import Error as PlaywrightErrorfrom playwright.async_api import TimeoutError as PlaywrightTimeoutError-from skyvern.exceptions import MissingElement, MultipleElementsFound+from skyvern.exceptions import MissingElement, MissingElementDict, MultipleElementsFoundfrom skyvern.webeye.utils.dom import SkyvernElement, is_element_detached_error, resolve_locator_DETACHED_ERROR = "ElementHandle.content_frame: Element is not attached to the DOM"@@ -202,3 +202,22 @@ async def test_press_fill_classifies_timeout_when_element_gone(monkeypatch: pytemonkeypatch.setattr("skyvern.webeye.utils.dom.EventStrategyFactory.type_text", _raise)with pytest.raises(MissingElement):await element.press_fill("hello")+++async def test_find_label_for_returns_none_when_for_control_was_not_scraped() -> None:+ # The label's for= control is live (found once, carries a unique id) but is absent from the scraped dict.+ control_locator = MagicMock()+ control_locator.count = AsyncMock(return_value=1)+ control_locator.get_attribute = AsyncMock(return_value="AACH")+ frame = MagicMock()+ frame.locator = MagicMock(return_value=control_locator)+ label = SkyvernElement(MagicMock(), frame, {"id": "AABQ", "tagName": "label", "attributes": {"for": "ctl"}})+ dom = MagicMock()+ dom.get_skyvern_element_by_id = AsyncMock(side_effect=MissingElementDict("AACH"))++ assert await label.find_label_for(dom) is None++ mapped = object()+ dom.get_skyvern_element_by_id = AsyncMock(return_value=mapped)+ assert await label.find_label_for(dom) is mappeddiff --git a/tests/unit/test_workflow_run_minutes_gate.py b/tests/unit/test_workflow_run_minutes_gate.pynew file mode 100644index 000000000..ac5dca0c7--- /dev/null+++ b/tests/unit/test_workflow_run_minutes_gate.py@@ -0,0 +1,147 @@+"""Run minutes count compute, so a workflow run that never reached ``running``+must contribute no minutes (SKY-14608) -- but the exclusion itself is exported,+tagged ``excluded_reason="never_started"``, so the removed cohort stays+observable instead of silently vanishing.++Both terminal writers derive ``duration_seconds`` from+``COALESCE(started_at, created_at)``. On a run finalized straight out of the+queue that fallback measures queue age, and the run held no pod at all -- which+is why the emission, not the duration, is what carries the exclusion. The two+writers carry independent copies of the logic, so both are covered here.++The same rule governs the task_v1 emitter in ``Agent.update_task``, which reads+the task once on entry and finalizes it later: it is covered here too, because+only the post-claim row can say whether the task started.+"""++from __future__ import annotations++from datetime import UTC, datetime, timedelta+from unittest.mock import AsyncMock, MagicMock++import pytest++from skyvern.forge import agent as agent_module+from skyvern.forge import app+from skyvern.forge.agent import ForgeAgent+from skyvern.forge.sdk.schemas.tasks import Task, TaskStatus+from skyvern.forge.sdk.workflow.models.workflow import WorkflowRunStatus+from skyvern.forge.sdk.workflow.service import WorkflowService+++def _make_row(*, started: bool) -> MagicMock:+ now = datetime.now(UTC)+ row = MagicMock()+ row.workflow_run_id = "wr_gate"+ row.workflow_id = "wf_gate"+ row.organization_id = "org_gate"+ row.parent_workflow_run_id = None+ row.created_at = now - timedelta(minutes=30)+ row.started_at = (now - timedelta(minutes=20)) if started else None+ row.status = WorkflowRunStatus.canceled+ row.run_with = None+ row.ai_fallback = False+ row.trigger_type = None+ row.workflow_schedule_id = None+ row.failure_category = None+ return row+++def record_run_duration(monkeypatch: pytest.MonkeyPatch) -> AsyncMock:+ emitter = AsyncMock()+ monkeypatch.setattr(app.AGENT_FUNCTION, "record_run_duration", emitter)+ monkeypatch.setattr(WorkflowService, "_apply_completion_run_tags_best_effort", AsyncMock())+ monkeypatch.setattr(WorkflowService, "_schedule_workflow_run_terminal_hooks", MagicMock())+ monkeypatch.setattr(WorkflowService, "_sync_task_run_from_workflow_run", AsyncMock())+ return emitter+++def _assert_emission(record_run_duration: AsyncMock, *, started: bool) -> None:+ assert record_run_duration.await_count == 1+ kwargs = record_run_duration.await_args.kwargs+ if started:+ assert kwargs["excluded_reason"] is None+ assert kwargs["duration_seconds"] == pytest.approx(20 * 60, abs=5)+ else:+ # Excluded, not silent: the recorder turns this into a zero-minute sample+ # tagged excluded=never_started, so sums stay compute-only while the+ # exclusion stays countable.+ assert kwargs["excluded_reason"] == "never_started"++[email protected]("started", [True, False])+async def test_terminal_write_emits_minutes_only_for_runs_that_started(+ record_run_duration: AsyncMock,+ started: bool,+) -> None:+ await WorkflowService()._after_workflow_run_status_write(_make_row(started=started), WorkflowRunStatus.canceled)++ _assert_emission(record_run_duration, started=started)++[email protected]("started", [True, False])+async def test_conditional_cancel_emits_minutes_only_for_runs_that_started(+ monkeypatch: pytest.MonkeyPatch,+ record_run_duration: AsyncMock,+ started: bool,+) -> None:+ row = _make_row(started=started)+ monkeypatch.setattr(+ app.DATABASE.workflow_runs,+ "update_workflow_run_if_not_final",+ AsyncMock(return_value=row),+ )++ await WorkflowService().mark_workflow_run_as_canceled_if_not_final(workflow_run_id="wr_gate")++ _assert_emission(record_run_duration, started=started)+++def _make_task(*, status: TaskStatus, started_at: datetime | None, finished_at: datetime | None = None) -> Task:+ now = datetime.now(UTC)+ return Task(+ task_id="tsk_gate",+ organization_id="org_gate",+ url="https://example.com",+ status=status,+ created_at=now - timedelta(minutes=30),+ modified_at=now,+ started_at=started_at,+ finished_at=finished_at,+ workflow_run_id=None,+ )+++async def test_task_v1_emission_reads_started_at_from_the_claim_not_the_entry_read(+ monkeypatch: pytest.MonkeyPatch,+ record_run_duration: AsyncMock,+) -> None:+ """A worker can stamp ``started_at`` between ``update_task``'s entry read and its+ finished_at claim. Deciding the exclusion off the entry read would bill that+ task's real compute as ``never_started`` and emit zero minutes for it.+ """+ now = datetime.now(UTC)+ entry_task = _make_task(status=TaskStatus.queued, started_at=None)+ claimed_task = _make_task(+ status=TaskStatus.canceled,+ started_at=now - timedelta(minutes=9),+ finished_at=now,+ )+ monkeypatch.setattr(app.DATABASE.tasks, "get_task", AsyncMock(return_value=entry_task))+ monkeypatch.setattr(+ app.DATABASE.tasks,+ "update_task_and_claim_finish",+ AsyncMock(return_value=(claimed_task, True)),+ )+ monkeypatch.setattr(agent_module, "save_task_logs", AsyncMock())++ await ForgeAgent().update_task(entry_task, status=TaskStatus.canceled)++ assert record_run_duration.await_count == 1+ kwargs = record_run_duration.await_args.kwargs+ assert kwargs.get("excluded_reason") is None+ assert kwargs["duration_seconds"] == pytest.approx(9 * 60, abs=5)