Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
Keep Task V3 turn steps out of the per-step pricing skill's step counts (SKY-18896) (#9038)
pedrohsdb7399babddd9d
diff --git a/skyvern/forge/agent_functions.py b/skyvern/forge/agent_functions.pyindex 8475058ed..76e3f146f 100644--- a/skyvern/forge/agent_functions.py+++ b/skyvern/forge/agent_functions.py@@ -1195,7 +1195,7 @@ class AgentFunction:return ()# An audit of the answers a run entered, consulted before the form moves on, or None (OSS: none).- def task_v3_answer_basis_audit(+ async def task_v3_answer_basis_audit(self, parameters: dict[str, Any] | None, *, goal: str, organization_id: str | None, step: Step | None) -> AnswerBasisAudit | None:return Nonediff --git a/skyvern/forge/taskv3/engine.py b/skyvern/forge/taskv3/engine.pyindex 347d041a8..53d35c497 100644--- a/skyvern/forge/taskv3/engine.py+++ b/skyvern/forge/taskv3/engine.py@@ -21,6 +21,7 @@ in-process adapter over ``do_observe``/``do_execute`` for shared hardening + actfrom __future__ import annotations+import copyimport dataclassesimport functoolsimport json@@ -582,6 +583,10 @@ async def run_task_v3_agent_loop(async def _terminate_premise(reason: str, prior_checks: int) -> None:assert unlisted_reask_criteria is not None and tool_trail is not None+ # The run's context object is shared and the next block rewrites its ids; a check that outlives the run must+ # log and attribute under this run's. Set before the first await, inside the check's own context copy.+ if (run_context := skyvern_context.current()) is not None:+ skyvern_context.set(copy.copy(run_context))try:timeout, redact, caller, skip = _stop_check_setup()if not terminate_premise_check:@@ -613,7 +618,14 @@ async def run_task_v3_agent_loop()except Exception:LOG.warning("taskv3 terminate premise failed", exc_info=True)+ # The run flushes its step archive once, as it ends; a check still running then lands in a fresh one.+ if loop_ended and step is not None:+ try:+ await app.ARTIFACT_MANAGER.flush_step_archive(step.step_id)+ except Exception:+ LOG.warning("taskv3 terminate premise archive flush failed", exc_info=True)+ loop_ended = Falsefinish_tool = make_finish_tool(unseen_text=unseen_text,page_fingerprint=None if page_free else (settle_fingerprint or page_fingerprint),@@ -655,7 +667,7 @@ async def run_task_v3_agent_loop(answer_audit = (Noneif page_free- else app.AGENT_FUNCTION.task_v3_answer_basis_audit(+ else await app.AGENT_FUNCTION.task_v3_answer_basis_audit(refs.masked,goal="\n\n".join(part for part in (model_goal, goal_instructions) if part),organization_id=organization_id,@@ -742,6 +754,7 @@ async def run_task_v3_agent_loop(anthropic_cache_markers=anthropic_cache_markers,)finally:+ loop_ended = Trueif failure_watch is not None:failure_watch.close()# The context outlives this run; a signal raised as the loop was cancelled must not firediff --git a/skyvern/forge/taskv3/loop.py b/skyvern/forge/taskv3/loop.pyindex 14c1a2037..0d9080551 100644--- a/skyvern/forge/taskv3/loop.py+++ b/skyvern/forge/taskv3/loop.py@@ -62,6 +62,15 @@ FinishStatus = Literal["completed", "failed", "terminated"]UnlistedReaskCheck = Callable[[NonCompletedStatus, str], Awaitable[UnlistedReask]]# (reason, earlier checks this run); logs its own verdict and never raises.TerminatePremiseCheck = Callable[[str, int], Awaitable[None]]+# The loop only holds weak references to its tasks, so a shadow nobody awaits is kept here until it ends.+_PREMISE_CHECKS: set[asyncio.Task[None]] = set()+++def _premise_check_done(task: asyncio.Task[None]) -> None:+ _PREMISE_CHECKS.discard(task)+ if not task.cancelled() and (exc := task.exception()) is not None:+ LOG.warning("taskv3 terminate premise failed", exc_info=exc)+# Why a failing tool call failed, as one closed vocabulary. Spelled as a `Literal` rather than `str`# because the whole value of the facet is that it is closed: it is written at ~33 literal sites in@@ -2818,7 +2827,8 @@ def make_finish_tool(`unseen_text` holds a completed verdict, on its own budget, while the page shows text no result the model readcarried, or a result in the finish's own batch carried page text or download lines.- `terminate_premise` checks every terminated finish past the holds above, alongside any re-ask, and changes nothing.+ `terminate_premise` checks every terminated finish past the holds above in the background, never awaited, and+ changes nothing.`intercepted_submissions` lists the submissions a page-level interceptor held this block. A failed or terminatedverdict is held once per submission whose response reached the page after the newest read the model received@@ -3061,16 +3071,6 @@ def make_finish_tool(},)- async def _premise_done(premise: asyncio.Future[None] | None) -> None:- try:- if premise is not None:- await premise- except asyncio.CancelledError:- # A cancel raised inside the shadow's own call is not the run's; only the run's own cancel goes on.- task = asyncio.current_task()- if task is not None and task.cancelling():- raise-async def handler(args: dict[str, Any]) -> ToolResult:nonlocal deferrals, stale_holds, failure_deferrals, reask_asked, premise_checksnonlocal submission_hold_turn, submission_hold_message@@ -3402,7 +3402,11 @@ def make_finish_tool(premise = Noneif status == "terminated" and terminate_premise is not None:premise = asyncio.ensure_future(terminate_premise(args.get("reason") or "", premise_checks))+ _PREMISE_CHECKS.add(premise)+ premise.add_done_callback(_premise_check_done)premise_checks += 1+ # One yield, so the check renders its prompt from the trail this verdict saw before anything else runs.+ await asyncio.sleep(0)if original is not None and unlisted_reask is not None and not reask_asked:reask_asked = Truetry:@@ -3420,12 +3424,8 @@ def make_finish_tool(premise.cancel()# Defer: the loop's cancellation check persists `canceled` before another turn.return ToolResult.error("the run was canceled while the verdict was being checked.")- # After the cancel check, so the shadow's wait never widens the window in which a cancel changes the result.- await _premise_done(premise)if converted is not None:return converted- else:- await _premise_done(premise)candidates = download_facts if status == "completed" and download_settle is not None else Noneif candidates is not None and (candidates.held_candidates is None or candidates.resolve_candidates is None):candidates = Nonediff --git a/tests/unit/test_taskv3_engine.py b/tests/unit/test_taskv3_engine.pyindex 73107cc23..a95a63e51 100644--- a/tests/unit/test_taskv3_engine.py+++ b/tests/unit/test_taskv3_engine.py@@ -1684,7 +1684,7 @@ async def test_the_answer_audit_wraps_the_run_tools_and_sees_the_instructions(moapplied: list[tuple[list[str], object]] = []audit = object()- def hook(parameters: Any, *, goal: str, organization_id: Any, step: Any) -> object:+ async def hook(parameters: Any, *, goal: str, organization_id: Any, step: Any) -> object:hook_calls.append({"parameters": parameters, "goal": goal})return audit@@ -2682,9 +2682,7 @@ async def test_a_terminate_with_a_completion_criterion_asks_both_judges_once_eac@pytest.mark.asyncio-async def test_a_premise_check_that_fails_or_outlasts_its_timeout_still_ends_the_run_as_the_model_said(- monkeypatch: pytest.MonkeyPatch,-) -> None:+async def test_a_premise_check_that_fails_still_ends_the_run_as_the_model_said() -> None:def broken_redactor() -> Any:raise RuntimeError("redactor down")@@ -2700,23 +2698,57 @@ async def test_a_premise_check_that_fails_or_outlasts_its_timeout_still_ends_the)assert outcome.status == "terminated"++async def test_a_slow_premise_check_logs_after_the_run_ends_under_the_runs_own_ids(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ release = asyncio.Event()+ checked_under: list[str | None] = []+class _SlowCaller(_PremiseAnsweringCaller):async def call(self, **kwargs: Any) -> dict[str, Any]: # type: ignore[override]+ kwargs.pop("step", None)if kwargs.get("prompt_name") == TERMINATE_PREMISE_PROMPT_NAME:- await asyncio.sleep(5)+ await release.wait()+ checked_under.append(skyvern_context.ensure_context().task_id)return await super().call(**kwargs)- monkeypatch.setattr(engine_mod, "GOAL_CHECK_TIMEOUT_SECONDS", 0.05)+ flushed: list[str] = []++ async def flush_step_archive(step_id: str) -> None:+ flushed.append(step_id)++ monkeypatch.setattr(app, "ARTIFACT_MANAGER", SimpleNamespace(flush_step_archive=flush_step_archive))slow = _SlowCaller([[("finish", {"status": "terminated", "reason": "Not listed."})]], answer={})- with capture_logs() as logs:- outcome = await run_task_v3_agent_loop(- page_provider=_fixed_page_provider(_FakePage()),- llm_caller=slow,- goal="Open the statements.",- unlisted_reask_criteria=("", "the statements link is not on the page"),++ def premise_whys() -> list[str]:+ return [log["why"] for log in logs if log["event"] == "taskv3 terminate premise"]++ with skyvern_context.scoped(SkyvernContext(task_id="tsk_premise")) as run_context, capture_logs() as logs:+ outcome = await asyncio.wait_for(+ run_task_v3_agent_loop(+ page_provider=_fixed_page_provider(_FakePage()),+ llm_caller=slow,+ goal="Open the statements.",+ unlisted_reask_criteria=("", "the statements link is not on the page"),+ step=SimpleNamespace(step_id="stp_premise"),+ ),+ timeout=1.0,)- assert outcome.status == "terminated"- assert [log["why"] for log in logs if log["event"] == "taskv3 terminate premise"] == ["skipped:timeout"]+ assert outcome.status == "terminated"+ assert premise_whys() == []+ # What the block does once it ends; the shared context then belongs to the next block.+ run_context.task_id = None+ release.set()+ async with asyncio.timeout(5):+ while not premise_whys():+ await asyncio.sleep(0.01)++ assert premise_whys() == ["skipped:unparseable"]+ assert checked_under == ["tsk_premise"]+ # The run's own archive flush has already happened, so the late check flushes what its call wrote.+ assert flushed == ["stp_premise"]def _fixed_read_tool(content: str, *, page: _FakePage | None = None, lands_at: str | None = None) -> ToolSpec:diff --git a/tests/unit/test_taskv3_goal_check.py b/tests/unit/test_taskv3_goal_check.pyindex 61e777635..a24df5a51 100644--- a/tests/unit/test_taskv3_goal_check.py+++ b/tests/unit/test_taskv3_goal_check.py@@ -1,6 +1,7 @@from __future__ import annotationsimport asyncio+import gcimport itertoolsimport reimport time@@ -1195,44 +1196,6 @@ async def test_a_premise_quote_from_a_page_the_agent_has_left_does_not_ground_anassert premise.why == "skipped:quote_not_in_evidence"-async def test_a_cancel_landing_while_only_the_premise_check_runs_leaves_the_verdict_as_on_main() -> None:- # On main the cancel check runs as soon as the re-ask answers; the shadow's wait must not widen that window. A- # converting re-ask is left out: its settle window yields, and a cancel inside it is caught on main too.- converts = False- canceled = False- answered = asyncio.Event()- checked: list[str] = []-- class _AnsweringSpy(_ReaskSpy):- async def __call__(self, status: NonCompletedStatus, reason: str) -> UnlistedReask:- result = await super().__call__(status, reason)- answered.set()- return result-- async def premise(reason: str, prior_checks: int) -> None:- nonlocal canceled- await answered.wait()- await asyncio.sleep(0)- canceled = True- checked.append(reason)-- async def should_cancel() -> bool:- return canceled-- finish = make_finish_tool(- **_gate_kwargs(pending=False, blocked=False, unsettled=False),- should_cancel=should_cancel,- unlisted_reask=_AnsweringSpy(converts=converts),- terminate_premise=premise,- )- result = await finish.handler({"status": "terminated", "reason": "no PIN screen"})-- assert result.status == "ok" and result.data is not None- assert result.data["status"] == ("completed" if converts else "terminated")- assert checked == ["no PIN screen"]--@pytest.mark.asyncio@pytest.mark.parametrize("reasks", [True, False])async def test_a_cancel_raised_inside_the_premise_check_does_not_end_the_run(reasks: bool) -> None:@@ -1251,23 +1214,31 @@ async def test_a_cancel_raised_inside_the_premise_check_does_not_end_the_run(rea@pytest.mark.asyncio-async def test_the_runs_own_cancel_still_ends_the_run_while_the_premise_check_runs() -> None:- started = asyncio.Event()+async def test_a_premise_check_that_raises_after_the_verdict_is_logged_not_left_unretrieved() -> None:+ unhandled: list[dict[str, Any]] = []+ asyncio.get_running_loop().set_exception_handler(lambda loop, context: unhandled.append(context))+ release = asyncio.Event()async def premise(reason: str, prior_checks: int) -> None:- started.set()- await asyncio.Event().wait()+ await release.wait()+ raise RuntimeError("judge exploded")finish = make_finish_tool(**_gate_kwargs(pending=False, blocked=False, unsettled=False),terminate_premise=premise,)- run = asyncio.ensure_future(finish.handler({"status": "terminated", "reason": "no PIN screen"}))- await started.wait()- run.cancel()-- with pytest.raises(asyncio.CancelledError):- await run+ with capture_logs() as logs:+ result = await asyncio.wait_for(finish.handler({"status": "terminated", "reason": "no PIN screen"}), 1)+ assert result.status == "ok" and result.data is not None+ assert result.data["status"] == "terminated"+ release.set()+ for _ in range(3):+ await asyncio.sleep(0)+ del finish+ gc.collect()++ assert unhandled == []+ assert [log["event"] for log in logs if "premise" in log["event"]] == ["taskv3 terminate premise failed"]@pytest.mark.asyncio