Skip to content

Automate browser based workflows with AI

Keep Task V3 turn steps out of the per-step pricing skill's step counts (SKY-18896) (#9038)

pedrohsdb7399babddd9d
diff --git a/skyvern/forge/agent_functions.py b/skyvern/forge/agent_functions.py
index 8475058ed..76e3f146f 100644
--- a/skyvern/forge/agent_functions.py
+++ b/skyvern/forge/agent_functions.py
@@ -1195,7 +1195,7 @@ class AgentFunction:
return ()
# An audit of the answers a run entered, consulted before the form moves on, or None (OSS: none).
- def task_v3_answer_basis_audit(
+ async def task_v3_answer_basis_audit(
self, parameters: dict[str, Any] | None, *, goal: str, organization_id: str | None, step: Step | None
) -> AnswerBasisAudit | None:
return None
diff --git a/skyvern/forge/taskv3/engine.py b/skyvern/forge/taskv3/engine.py
index 347d041a8..53d35c497 100644
--- a/skyvern/forge/taskv3/engine.py
+++ b/skyvern/forge/taskv3/engine.py
@@ -21,6 +21,7 @@ in-process adapter over ``do_observe``/``do_execute`` for shared hardening + act
from __future__ import annotations
+import copy
import dataclasses
import functools
import json
@@ -582,6 +583,10 @@ async def run_task_v3_agent_loop(
async def _terminate_premise(reason: str, prior_checks: int) -> None:
assert unlisted_reask_criteria is not None and tool_trail is not None
+ # The run's context object is shared and the next block rewrites its ids; a check that outlives the run must
+ # log and attribute under this run's. Set before the first await, inside the check's own context copy.
+ if (run_context := skyvern_context.current()) is not None:
+ skyvern_context.set(copy.copy(run_context))
try:
timeout, redact, caller, skip = _stop_check_setup()
if not terminate_premise_check:
@@ -613,7 +618,14 @@ async def run_task_v3_agent_loop(
)
except Exception:
LOG.warning("taskv3 terminate premise failed", exc_info=True)
+ # The run flushes its step archive once, as it ends; a check still running then lands in a fresh one.
+ if loop_ended and step is not None:
+ try:
+ await app.ARTIFACT_MANAGER.flush_step_archive(step.step_id)
+ except Exception:
+ LOG.warning("taskv3 terminate premise archive flush failed", exc_info=True)
+ loop_ended = False
finish_tool = make_finish_tool(
unseen_text=unseen_text,
page_fingerprint=None if page_free else (settle_fingerprint or page_fingerprint),
@@ -655,7 +667,7 @@ async def run_task_v3_agent_loop(
answer_audit = (
None
if page_free
- else app.AGENT_FUNCTION.task_v3_answer_basis_audit(
+ else await app.AGENT_FUNCTION.task_v3_answer_basis_audit(
refs.masked,
goal="\n\n".join(part for part in (model_goal, goal_instructions) if part),
organization_id=organization_id,
@@ -742,6 +754,7 @@ async def run_task_v3_agent_loop(
anthropic_cache_markers=anthropic_cache_markers,
)
finally:
+ loop_ended = True
if failure_watch is not None:
failure_watch.close()
# The context outlives this run; a signal raised as the loop was cancelled must not fire
diff --git a/skyvern/forge/taskv3/loop.py b/skyvern/forge/taskv3/loop.py
index 14c1a2037..0d9080551 100644
--- a/skyvern/forge/taskv3/loop.py
+++ b/skyvern/forge/taskv3/loop.py
@@ -62,6 +62,15 @@ FinishStatus = Literal["completed", "failed", "terminated"]
UnlistedReaskCheck = Callable[[NonCompletedStatus, str], Awaitable[UnlistedReask]]
# (reason, earlier checks this run); logs its own verdict and never raises.
TerminatePremiseCheck = Callable[[str, int], Awaitable[None]]
+# The loop only holds weak references to its tasks, so a shadow nobody awaits is kept here until it ends.
+_PREMISE_CHECKS: set[asyncio.Task[None]] = set()
+
+
+def _premise_check_done(task: asyncio.Task[None]) -> None:
+ _PREMISE_CHECKS.discard(task)
+ if not task.cancelled() and (exc := task.exception()) is not None:
+ LOG.warning("taskv3 terminate premise failed", exc_info=exc)
+
# Why a failing tool call failed, as one closed vocabulary. Spelled as a `Literal` rather than `str`
# because the whole value of the facet is that it is closed: it is written at ~33 literal sites in
@@ -2818,7 +2827,8 @@ def make_finish_tool(
`unseen_text` holds a completed verdict, on its own budget, while the page shows text no result the model read
carried, or a result in the finish's own batch carried page text or download lines.
- `terminate_premise` checks every terminated finish past the holds above, alongside any re-ask, and changes nothing.
+ `terminate_premise` checks every terminated finish past the holds above in the background, never awaited, and
+ changes nothing.
`intercepted_submissions` lists the submissions a page-level interceptor held this block. A failed or terminated
verdict is held once per submission whose response reached the page after the newest read the model received
@@ -3061,16 +3071,6 @@ def make_finish_tool(
},
)
- async def _premise_done(premise: asyncio.Future[None] | None) -> None:
- try:
- if premise is not None:
- await premise
- except asyncio.CancelledError:
- # A cancel raised inside the shadow's own call is not the run's; only the run's own cancel goes on.
- task = asyncio.current_task()
- if task is not None and task.cancelling():
- raise
-
async def handler(args: dict[str, Any]) -> ToolResult:
nonlocal deferrals, stale_holds, failure_deferrals, reask_asked, premise_checks
nonlocal submission_hold_turn, submission_hold_message
@@ -3402,7 +3402,11 @@ def make_finish_tool(
premise = None
if status == "terminated" and terminate_premise is not None:
premise = asyncio.ensure_future(terminate_premise(args.get("reason") or "", premise_checks))
+ _PREMISE_CHECKS.add(premise)
+ premise.add_done_callback(_premise_check_done)
premise_checks += 1
+ # One yield, so the check renders its prompt from the trail this verdict saw before anything else runs.
+ await asyncio.sleep(0)
if original is not None and unlisted_reask is not None and not reask_asked:
reask_asked = True
try:
@@ -3420,12 +3424,8 @@ def make_finish_tool(
premise.cancel()
# Defer: the loop's cancellation check persists `canceled` before another turn.
return ToolResult.error("the run was canceled while the verdict was being checked.")
- # After the cancel check, so the shadow's wait never widens the window in which a cancel changes the result.
- await _premise_done(premise)
if converted is not None:
return converted
- else:
- await _premise_done(premise)
candidates = download_facts if status == "completed" and download_settle is not None else None
if candidates is not None and (candidates.held_candidates is None or candidates.resolve_candidates is None):
candidates = None
diff --git a/tests/unit/test_taskv3_engine.py b/tests/unit/test_taskv3_engine.py
index 73107cc23..a95a63e51 100644
--- a/tests/unit/test_taskv3_engine.py
+++ b/tests/unit/test_taskv3_engine.py
@@ -1684,7 +1684,7 @@ async def test_the_answer_audit_wraps_the_run_tools_and_sees_the_instructions(mo
applied: list[tuple[list[str], object]] = []
audit = object()
- def hook(parameters: Any, *, goal: str, organization_id: Any, step: Any) -> object:
+ async def hook(parameters: Any, *, goal: str, organization_id: Any, step: Any) -> object:
hook_calls.append({"parameters": parameters, "goal": goal})
return audit
@@ -2682,9 +2682,7 @@ async def test_a_terminate_with_a_completion_criterion_asks_both_judges_once_eac
@pytest.mark.asyncio
-async def test_a_premise_check_that_fails_or_outlasts_its_timeout_still_ends_the_run_as_the_model_said(
- monkeypatch: pytest.MonkeyPatch,
-) -> None:
+async def test_a_premise_check_that_fails_still_ends_the_run_as_the_model_said() -> None:
def broken_redactor() -> Any:
raise RuntimeError("redactor down")
@@ -2700,23 +2698,57 @@ async def test_a_premise_check_that_fails_or_outlasts_its_timeout_still_ends_the
)
assert outcome.status == "terminated"
+
+async def test_a_slow_premise_check_logs_after_the_run_ends_under_the_runs_own_ids(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ release = asyncio.Event()
+ checked_under: list[str | None] = []
+
class _SlowCaller(_PremiseAnsweringCaller):
async def call(self, **kwargs: Any) -> dict[str, Any]: # type: ignore[override]
+ kwargs.pop("step", None)
if kwargs.get("prompt_name") == TERMINATE_PREMISE_PROMPT_NAME:
- await asyncio.sleep(5)
+ await release.wait()
+ checked_under.append(skyvern_context.ensure_context().task_id)
return await super().call(**kwargs)
- monkeypatch.setattr(engine_mod, "GOAL_CHECK_TIMEOUT_SECONDS", 0.05)
+ flushed: list[str] = []
+
+ async def flush_step_archive(step_id: str) -> None:
+ flushed.append(step_id)
+
+ monkeypatch.setattr(app, "ARTIFACT_MANAGER", SimpleNamespace(flush_step_archive=flush_step_archive))
slow = _SlowCaller([[("finish", {"status": "terminated", "reason": "Not listed."})]], answer={})
- with capture_logs() as logs:
- outcome = await run_task_v3_agent_loop(
- page_provider=_fixed_page_provider(_FakePage()),
- llm_caller=slow,
- goal="Open the statements.",
- unlisted_reask_criteria=("", "the statements link is not on the page"),
+
+ def premise_whys() -> list[str]:
+ return [log["why"] for log in logs if log["event"] == "taskv3 terminate premise"]
+
+ with skyvern_context.scoped(SkyvernContext(task_id="tsk_premise")) as run_context, capture_logs() as logs:
+ outcome = await asyncio.wait_for(
+ run_task_v3_agent_loop(
+ page_provider=_fixed_page_provider(_FakePage()),
+ llm_caller=slow,
+ goal="Open the statements.",
+ unlisted_reask_criteria=("", "the statements link is not on the page"),
+ step=SimpleNamespace(step_id="stp_premise"),
+ ),
+ timeout=1.0,
)
- assert outcome.status == "terminated"
- assert [log["why"] for log in logs if log["event"] == "taskv3 terminate premise"] == ["skipped:timeout"]
+ assert outcome.status == "terminated"
+ assert premise_whys() == []
+ # What the block does once it ends; the shared context then belongs to the next block.
+ run_context.task_id = None
+ release.set()
+ async with asyncio.timeout(5):
+ while not premise_whys():
+ await asyncio.sleep(0.01)
+
+ assert premise_whys() == ["skipped:unparseable"]
+ assert checked_under == ["tsk_premise"]
+ # The run's own archive flush has already happened, so the late check flushes what its call wrote.
+ assert flushed == ["stp_premise"]
def _fixed_read_tool(content: str, *, page: _FakePage | None = None, lands_at: str | None = None) -> ToolSpec:
diff --git a/tests/unit/test_taskv3_goal_check.py b/tests/unit/test_taskv3_goal_check.py
index 61e777635..a24df5a51 100644
--- a/tests/unit/test_taskv3_goal_check.py
+++ b/tests/unit/test_taskv3_goal_check.py
@@ -1,6 +1,7 @@
from __future__ import annotations
import asyncio
+import gc
import itertools
import re
import time
@@ -1195,44 +1196,6 @@ async def test_a_premise_quote_from_a_page_the_agent_has_left_does_not_ground_an
assert premise.why == "skipped:quote_not_in_evidence"
-async def test_a_cancel_landing_while_only_the_premise_check_runs_leaves_the_verdict_as_on_main() -> None:
- # On main the cancel check runs as soon as the re-ask answers; the shadow's wait must not widen that window. A
- # converting re-ask is left out: its settle window yields, and a cancel inside it is caught on main too.
- converts = False
- canceled = False
- answered = asyncio.Event()
- checked: list[str] = []
-
- class _AnsweringSpy(_ReaskSpy):
- async def __call__(self, status: NonCompletedStatus, reason: str) -> UnlistedReask:
- result = await super().__call__(status, reason)
- answered.set()
- return result
-
- async def premise(reason: str, prior_checks: int) -> None:
- nonlocal canceled
- await answered.wait()
- await asyncio.sleep(0)
- canceled = True
- checked.append(reason)
-
- async def should_cancel() -> bool:
- return canceled
-
- finish = make_finish_tool(
- **_gate_kwargs(pending=False, blocked=False, unsettled=False),
- should_cancel=should_cancel,
- unlisted_reask=_AnsweringSpy(converts=converts),
- terminate_premise=premise,
- )
- result = await finish.handler({"status": "terminated", "reason": "no PIN screen"})
-
- assert result.status == "ok" and result.data is not None
- assert result.data["status"] == ("completed" if converts else "terminated")
- assert checked == ["no PIN screen"]
-
-
@pytest.mark.asyncio
@pytest.mark.parametrize("reasks", [True, False])
async def test_a_cancel_raised_inside_the_premise_check_does_not_end_the_run(reasks: bool) -> None:
@@ -1251,23 +1214,31 @@ async def test_a_cancel_raised_inside_the_premise_check_does_not_end_the_run(rea
@pytest.mark.asyncio
-async def test_the_runs_own_cancel_still_ends_the_run_while_the_premise_check_runs() -> None:
- started = asyncio.Event()
+async def test_a_premise_check_that_raises_after_the_verdict_is_logged_not_left_unretrieved() -> None:
+ unhandled: list[dict[str, Any]] = []
+ asyncio.get_running_loop().set_exception_handler(lambda loop, context: unhandled.append(context))
+ release = asyncio.Event()
async def premise(reason: str, prior_checks: int) -> None:
- started.set()
- await asyncio.Event().wait()
+ await release.wait()
+ raise RuntimeError("judge exploded")
finish = make_finish_tool(
**_gate_kwargs(pending=False, blocked=False, unsettled=False),
terminate_premise=premise,
)
- run = asyncio.ensure_future(finish.handler({"status": "terminated", "reason": "no PIN screen"}))
- await started.wait()
- run.cancel()
-
- with pytest.raises(asyncio.CancelledError):
- await run
+ with capture_logs() as logs:
+ result = await asyncio.wait_for(finish.handler({"status": "terminated", "reason": "no PIN screen"}), 1)
+ assert result.status == "ok" and result.data is not None
+ assert result.data["status"] == "terminated"
+ release.set()
+ for _ in range(3):
+ await asyncio.sleep(0)
+ del finish
+ gc.collect()
+
+ assert unhandled == []
+ assert [log["event"] for log in logs if "premise" in log["event"]] == ["taskv3 terminate premise failed"]
@pytest.mark.asyncio