Skip to content

Automate browser based workflows with AI

Run every Task V3 test through production's agent setup (SKY-18395) (#8959)

Andrew Neilson1822679c68bf
diff --git a/tests/conftest.py b/tests/conftest.py
index 7b7b7568a..387a03444 100644
--- a/tests/conftest.py
+++ b/tests/conftest.py
@@ -1,5 +1,6 @@
"""Fixtures shared by every suite."""
+import functools
import gc
import inspect
import io
@@ -23,6 +24,7 @@ from skyvern.forge.sdk.experimentation.code_block_ai_fallback import CODE_BLOCK_
from skyvern.forge.sdk.experimentation.providers import NoOpExperimentationProvider
from skyvern.forge.sdk.forge_log import setup_logger
from skyvern.forge.sdk.workflow.models.block import CodeBlock
+from skyvern.forge.taskv3 import engine as taskv3_engine
from skyvern.services import organization_log_scope
_monkeypatch_setattr = pytest.MonkeyPatch.setattr
@@ -88,6 +90,41 @@ def _url_validation_enabled(monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.setattr(settings, "ENABLE_URL_VALIDATION", True)
+LOOP_OWNER = ("skyvern/forge/agent.py", "_execute_task_v3")
+# The loop's own contract tests call it directly by design.
+LOOP_OWNER_EXEMPT_TEST_FILES = frozenset({"test_taskv3_engine.py"})
+
+
+def loop_entered_through_executor() -> bool:
+ frame = sys._getframe(1)
+ while frame is not None:
+ code = frame.f_code
+ if code.co_name == LOOP_OWNER[1] and code.co_filename.endswith(LOOP_OWNER[0]):
+ return True
+ frame = frame.f_back
+ return False
+
+
[email protected](autouse=True)
+def _loop_runs_only_inside_the_executor(request: pytest.FixtureRequest, monkeypatch: pytest.MonkeyPatch) -> None:
+ """The runtime half of tests/cloud/test_taskv3_single_loop_call_site.py: it sees calls the static scan cannot,
+ such as a name computed at runtime."""
+ if request.node.path.name in LOOP_OWNER_EXEMPT_TEST_FILES:
+ return
+ loop = taskv3_engine.run_task_v3_agent_loop
+
+ @functools.wraps(loop)
+ async def guarded(*args: Any, **kwargs: Any) -> taskv3_engine.LoopOutcome:
+ if not loop_entered_through_executor():
+ raise AssertionError(
+ "run_task_v3_agent_loop ran outside ForgeAgent._execute_task_v3; run the task through the executor "
+ "(tests/browser_e2e/taskv3_fixture_helpers.run_production_task_v3)"
+ )
+ return await loop(*args, **kwargs)
+
+ monkeypatch.setattr(taskv3_engine, "run_task_v3_agent_loop", guarded)
+
+
@pytest.fixture(autouse=True)
def _isolate_organization_age_cache() -> Iterator[None]:
"""Loading an organization caches its creation time process-wide; no test may leak it into another's logs."""
diff --git a/tests/unit/helpers.py b/tests/unit/helpers.py
index 983045318..d20dc99b2 100644
--- a/tests/unit/helpers.py
+++ b/tests/unit/helpers.py
@@ -1,9 +1,10 @@
from __future__ import annotations
import json
+import secrets
from contextlib import contextmanager
from dataclasses import dataclass
-from datetime import datetime
+from datetime import UTC, datetime
from types import SimpleNamespace
from typing import Any, Iterator, Sequence
from unittest.mock import AsyncMock, MagicMock
@@ -346,6 +347,131 @@ def setup_parallel_verification_mocks(
)
+# The repository's update_step adds these deltas to the step's own running totals, as the SQL update does.
+_STEP_COUNTER_COLUMNS = {
+ "incremental_cost": "step_cost",
+ "incremental_input_tokens": "input_token_count",
+ "incremental_output_tokens": "output_token_count",
+ "incremental_reasoning_tokens": "reasoning_token_count",
+ "incremental_cached_tokens": "cached_token_count",
+}
+
+
+@dataclass
+class InMemoryTaskStepRepo:
+ """Backs app.DATABASE.tasks for one task and step, so the real ForgeAgent update methods read back their writes."""
+
+ task: Task | None = None
+ step: Step | None = None
+
+ async def create_task(self, *, url: str, organization_id: str | None, **fields: Any) -> Task:
+ fields["max_screenshot_scrolls"] = fields.pop("max_screenshot_scrolling_times", None)
+ now = datetime.now(UTC)
+ self.task = Task(
+ task_id=f"tsk_{secrets.token_hex(6)}",
+ url=url,
+ organization_id=organization_id,
+ status=TaskStatus.created,
+ created_at=now,
+ modified_at=now,
+ **{name: value for name, value in fields.items() if value is not None},
+ )
+ return self.task
+
+ async def create_step(self, task_id: str, *, order: int, retry_index: int, organization_id: str | None) -> Step:
+ now = datetime.now(UTC)
+ self.step = Step(
+ task_id=task_id,
+ step_id=f"stp_{secrets.token_hex(6)}",
+ status=StepStatus.created,
+ order=order,
+ retry_index=retry_index,
+ is_last=False,
+ organization_id=organization_id,
+ created_at=now,
+ modified_at=now,
+ )
+ return self.step
+
+ async def get_task(self, task_id: str, organization_id: str | None = None) -> Task | None:
+ return self.task if self.task is not None and self.task.task_id == task_id else None
+
+ async def update_task(self, task_id: str, organization_id: str | None = None, **updates: Any) -> Task:
+ updated, _ = await self.update_task_and_claim_finish(task_id, organization_id=organization_id, **updates)
+ return updated
+
+ async def update_task_and_claim_finish(
+ self,
+ task_id: str,
+ status: TaskStatus | None = None,
+ organization_id: str | None = None,
+ errors: list[dict[str, Any]] | None = None,
+ *,
+ keep_final_status: bool = False,
+ **updates: Any,
+ ) -> tuple[Task, bool]:
+ task = self.task
+ assert task is not None and task.task_id == task_id, f"no task {task_id}"
+ if keep_final_status and task.status.is_final():
+ return task, False
+ finish_claimed = False
+ if status is not None:
+ task.status = status
+ if status == TaskStatus.running and task.started_at is None:
+ task.started_at = datetime.now(UTC)
+ if status.is_final() and task.finished_at is None:
+ task.finished_at = datetime.now(UTC)
+ finish_claimed = True
+ if errors is not None:
+ task.errors = [*(task.errors or []), *errors]
+ for key, value in updates.items():
+ if value is not None:
+ setattr(task, key, value)
+ return task, finish_claimed
+
+ async def update_step(self, task_id: str, step_id: str, organization_id: str | None = None, **updates: Any) -> Step:
+ assert self.step is not None and self.step.step_id == step_id, f"no step {step_id}"
+ for key, value in updates.items():
+ if key in _STEP_COUNTER_COLUMNS:
+ column = _STEP_COUNTER_COLUMNS[key]
+ if value is not None:
+ setattr(self.step, column, (getattr(self.step, column) or 0) + value)
+ else:
+ setattr(self.step, key, value)
+ return self.step
+
+ async def get_first_step(self, task_id: str, organization_id: str | None = None) -> None:
+ return None
+
+ async def get_task_steps(self, task_id: str, organization_id: str | None = None) -> list[Step]:
+ return [self.step] if self.step is not None and self.step.task_id == task_id else []
+
+ async def get_latest_step(self, task_id: str, organization_id: str | None = None) -> Step | None:
+ steps = await self.get_task_steps(task_id, organization_id)
+ return steps[-1] if steps else None
+
+ async def get_run(self, run_id: str, organization_id: str | None = None) -> None:
+ return None
+
+ def install(self, monkeypatch: MonkeyPatch) -> None:
+ for name in (
+ "create_task",
+ "create_step",
+ "get_task",
+ "update_task",
+ "update_task_and_claim_finish",
+ "update_step",
+ "get_first_step",
+ "get_task_steps",
+ "get_latest_step",
+ "get_run",
+ ):
+ monkeypatch.setattr(app.DATABASE.tasks, name, getattr(self, name))
+ monkeypatch.setattr(
+ app.DATABASE.observer, "get_workflow_run_block_engine_by_task_id", AsyncMock(return_value=None)
+ )
+
+
def make_browser_state() -> tuple[MagicMock, MagicMock, MagicMock]:
return MagicMock(), MagicMock(), MagicMock()
diff --git a/tests/unit/test_agent_task_v3.py b/tests/unit/test_agent_task_v3.py
index a3f4847fd..b14544be7 100644
--- a/tests/unit/test_agent_task_v3.py
+++ b/tests/unit/test_agent_task_v3.py
@@ -69,16 +69,16 @@ from skyvern.forge.taskv3 import input_dispatch
from skyvern.forge.taskv3 import provider_downloads as taskv3_provider_downloads
from skyvern.forge.taskv3 import tools as taskv3_tools
from skyvern.forge.taskv3.auth_tools import VerificationFailure, VerificationState
-from skyvern.forge.taskv3.engine import DEFAULT_MAX_SETTLE_DEFERRALS, MIN_ACTION_STEPS, run_task_v3_agent_loop
-from skyvern.forge.taskv3.goal_check import BLOCK_COMPLETION_CHECK_PROMPT_NAME
+from skyvern.forge.taskv3.engine import DEFAULT_MAX_SETTLE_DEFERRALS, MIN_ACTION_STEPS
+from skyvern.forge.taskv3.goal_check import (
+ BLOCK_COMPLETION_CHECK_PROMPT_NAME,
+)
from skyvern.forge.taskv3.goal_composition import CodeProgressRecord, CodeTypedValue
from skyvern.forge.taskv3.handoff_redaction import pin_caller_authored_block_urls
from skyvern.forge.taskv3.loop import (
ACTION_BLOCK_TARGET_ACTION_RESERVE,
ACTION_LOOP_GUARD,
- COMPLETION_GATE_REFUSAL,
NAV_DEAD_END_GUARD,
- TOKEN_BUDGET_EXTENDED_EVENT,
LoopOutcome,
RoundAction,
ToolSpec,
@@ -114,11 +114,14 @@ from tests.unit._download_provider_fakes import (
_secret_bearing_validation_error,
_StallingActionDownloadObservation,
)
-from tests.unit.helpers import make_action_row, make_browser_state, make_organization, make_step, make_task
+from tests.unit.helpers import (
+ make_action_row,
+ make_browser_state,
+ make_organization,
+ make_step,
+ make_task,
+)
from tests.unit.scoped_asyncio import ScopedAsyncio
-from tests.unit.test_taskv3_engine import _fixed_read_tool, _ReaskAnsweringCaller
-from tests.unit.test_taskv3_loop import _ScriptedCaller
-from tests.unit.test_taskv3_tools import _FakePage, _fixed_page_provider
async def _run_execute_task_v3(
@@ -162,8 +165,6 @@ async def _run_execute_task_v3(
# Called with the loop's kwargs before the loop returns, to drive state the loop's tools own.
on_loop: Callable[[dict[str, Any]], None] | None = None,
on_loop_async: Callable[[dict[str, Any]], Awaitable[None]] | None = None,
- # Replaces the canned `outcome`: receives the kwargs agent.py built and returns the loop's outcome.
- loop_body: Callable[[dict[str, Any]], Awaitable[LoopOutcome]] | None = None,
# Leave the credential-TOTP candidate gate reading the real workflow-run context.
real_credential_totp_candidate: bool = False,
llm_caller_factory: Any = None,
@@ -234,8 +235,6 @@ async def _run_execute_task_v3(
on_loop(kwargs)
if on_loop_async is not None:
await on_loop_async(kwargs)
- if loop_body is not None:
- return await loop_body(kwargs)
if provider_probe_calls:
provider = kwargs["page_provider"]
loop_mock.resolved_pages = [await provider() for _ in range(provider_probe_calls)]
@@ -3158,75 +3157,6 @@ async def test_execute_task_v3_action_block_target_reserve_answers_to_the_run_po
assert kwargs["target_action_reserve"] == reserve
-class _AdvancingFormPage(_FakePage):
- """Every action advances the form, so the next observe is fresh page-change evidence."""
-
- # Takes the probe's argument, as Playwright's evaluate does, so the real settle sampler reads this page.
- async def evaluate(self, _js: str, _arg: Any = None) -> str:
- raw = await super().evaluate(_js)
- if "document.readyState" in _js:
- return raw
- section = sum(1 for name, _args in self.calls if name in ("click", "fill", "type"))
- data = json.loads(raw)
- data["title"] = f"Apply, section {section}"
- data["elements"][0]["label"] = f"Section {section} first name"
- return json.dumps(data)
-
-
-async def test_execute_task_v3_navigation_block_token_trip_with_progress_is_extended(
- monkeypatch: pytest.MonkeyPatch,
-) -> None:
- # A long form resends a growing transcript, so the token backstop trips while most of the step budget is
- # unspent. Driven through agent.py with the real engine and loop, so the step floor, ceilings and backstops
- # agent.py configures are the ones the grant must survive. The form is filled one field per round, and each
- # new field followed by a changed page renews the grant.
- script: list[list[tuple[str, dict[str, Any]]]] = []
- for i in range(5):
- script.append([("observe", {})])
- script.append([("type", {"selector": f"#field-{i}", "text": f"answer {i}"})])
- script.append([("observe", {})])
- script.append([("click", {"selector": "#submit"})])
- script.append([("finish", {"status": "completed", "reason": "submitted"})])
- # The first call is the normal-size fixed prompt; the burn after it is the transcript growing.
- caller = _ScriptedCaller(
- script,
- usage_for_call=lambda call: {"prompt_tokens": 15_000 if call == 0 else 200_000, "completion_tokens": 0},
- )
- page = _AdvancingFormPage()
- loop_kwargs: dict[str, Any] = {}
-
- async def _real_loop(kwargs: dict[str, Any]) -> LoopOutcome:
- loop_kwargs.update(kwargs)
- return await run_task_v3_agent_loop(
- # The scripted caller takes no step-scoped call kwargs.
- **{**kwargs, "page_provider": _fixed_page_provider(page), "llm_caller": caller, "step": None}
- )
-
- monkeypatch.setattr(ForgeAgent, "_check_workflow_run_step_budget", AsyncMock(return_value=None))
- with capture_logs() as logs:
- _step, task, _loop_mock, _post = await _run_execute_task_v3(
- monkeypatch,
- LoopOutcome(status="failed", reason="the canned outcome must not be used"),
- task_block=_make_block(NavigationBlock, navigation_goal="Fill and submit the application"),
- workflow_run_id="wr_long_form",
- max_steps_per_run=10,
- data_extraction_goal=None,
- extracted_information_schema=None,
- loop_body=_real_loop,
- working_page=page,
- )
- assert loop_kwargs["max_action_steps"] == MIN_ACTION_STEPS
- granted = [entry for entry in logs if entry["event"] == TOKEN_BUDGET_EXTENDED_EVENT]
- assert [(entry["guard"], entry["original_max_tokens"]) for entry in granted] == [
- ("max_tokens", loop_kwargs["max_tokens"]),
- ("max_tokens", granted[0]["max_tokens"]),
- ]
- assert granted[0]["action_steps"] < MIN_ACTION_STEPS
- assert caller.calls == len(script)
- assert task.status == TaskStatus.completed
-
-
@pytest.mark.asyncio
async def test_execute_task_v3_no_workflow_ceiling_without_a_pool(monkeypatch: pytest.MonkeyPatch) -> None:
# No org pool -> no hard ceiling: the loop's extension is bounded only by its own gate.
@@ -4211,97 +4141,6 @@ async def test_a_fifth_veto_on_a_reask_conversion_restores_the_models_own_verdic
assert task.failure_reason == "the site offers no way to continue"
-async def test_execute_task_v3_gate_veto_is_refused_in_loop_and_the_approved_retry_skips_the_post_loop_gate(
- monkeypatch: pytest.MonkeyPatch,
-) -> None:
- block = _make_block(NavigationBlock, navigation_goal="Submit the application")
- page = _AdvancingFormPage()
- caller = _ScriptedCaller(
- [
- [("finish", {"status": "completed", "reason": "looks done"})],
- [("observe", {})],
- [("finish", {"status": "completed", "reason": "confirmed"})],
- ]
- )
-
- async def _real_loop(kwargs: dict[str, Any]) -> LoopOutcome:
- return await run_task_v3_agent_loop(
- **{**kwargs, "page_provider": _fixed_page_provider(page), "llm_caller": caller, "step": None}
- )
-
- _step, task, loop_mock, _post = await _run_execute_task_v3(
- monkeypatch,
- LoopOutcome(status="failed", reason="the canned outcome must not be used"),
- task_block=block,
- loop_body=_real_loop,
- working_page=page,
- # A post-loop consult would draw the trailing veto and fail the task.
- completion_gate_raises=[False, True, False],
- data_extraction_goal=None,
- extracted_information_schema=None,
- )
-
- assert task.status == TaskStatus.completed
- assert loop_mock.completion_gate.await_args.kwargs["task_block"] is block
- assert sum(COMPLETION_GATE_REFUSAL in str(m.get("content")) for m in caller.message_history) == 1
-
-
-async def test_execute_task_v3_a_vetoed_reask_conversion_restores_the_models_verdict(
- monkeypatch: pytest.MonkeyPatch,
-) -> None:
- # The model terminated; the re-ask converted it; the deployment gate rejects the completion. The task must
- # end as the model's own terminated verdict, not as a failed completion the model never claimed.
- page = _FakePage()
- outcome = await run_task_v3_agent_loop(
- page_provider=_fixed_page_provider(page),
- llm_caller=_ReaskAnsweringCaller(
- [
- [("type", {"selector": "#email", "text": "[email protected]"})],
- [("click", {"selector": "#create"})],
- [("read_fixture", {})],
- [("observe", {})],
- [("finish", {"status": "terminated", "reason": "The PIN screen was never shown."})],
- ],
- answer={
- "verdict": "completed",
- "terminate_criterion_holds": False,
- "skipped_screen": "the PIN screen",
- "quote": "Email: [email protected]",
- "evidence": "The next form carries the account's email.",
- },
- ),
- goal="Create the account.",
- parameters={"email": "[email protected]"},
- extra_tools=[
- _fixed_read_tool(
- "Application\nEmail: [email protected]", page=page, lands_at="https://example.test/apply/section/1"
- )
- ],
- unlisted_reask_criteria=("a PIN screen is shown", "the create-account submission fails"),
- )
- assert outcome.status == "completed" and outcome.converted_from == "terminated"
- _step, task, loop_mock, _post = await _run_execute_task_v3(
- monkeypatch,
- outcome,
- task_block=_make_block(NavigationBlock, navigation_goal="Create the account"),
- completion_gate_vetoes=True,
- complete_criterion="a PIN screen is shown",
- terminate_criterion="the create-account submission fails",
- data_extraction_goal=None,
- extracted_information_schema=None,
- )
-
- assert loop_mock.await_args is not None
- assert loop_mock.await_args.kwargs["unlisted_reask_criteria"] == (
- "a PIN screen is shown",
- "the create-account submission fails",
- )
- assert task.status == TaskStatus.terminated
- assert task.failure_reason == "The PIN screen was never shown."
-
-
@pytest.mark.asyncio
@pytest.mark.parametrize(
("task_type", "custom_key", "criteria"),
@@ -6651,55 +6490,6 @@ async def test_execute_task_v3_reads_a_blocks_caller_urls_from_its_unrendered_de
assert off_the_page in kwargs["goal"] # nosemgrep: incomplete-url-substring-sanitization
[email protected]("initial", [False, True])
-async def test_execute_task_v3_logs_a_dead_task_url_with_a_query_as_task_supplied(
- monkeypatch: pytest.MonkeyPatch, initial: bool
-) -> None:
- # agent.py's task-target set and requested start URL, through the real engine and navigate tool: a
- # task URL whose query names the page is the task's own target, whether the in-loop navigate or the
- # setup navigation (redirected first) is the one that 404s.
- import skyvern.utils.url_validators as urlv
-
- monkeypatch.setattr(urlv, "validate_fetch_url", lambda url: url)
- task_url = "https://jobs.example.test/portal.action?page=apply&id=42"
- page = _FakePage()
-
- async def _goto(url: str, timeout: int | None = None, wait_until: str | None = None) -> Any:
- page.url = url
- return SimpleNamespace(status=404, url=url)
-
- page.goto = _goto # type: ignore[method-assign]
- script = [[("navigate", {"url": task_url})], [("finish", {"status": "completed", "reason": "never"})]]
-
- async def _real_loop(kwargs: dict[str, Any]) -> LoopOutcome:
- return await run_task_v3_agent_loop(
- **{
- **kwargs,
- "page_provider": _fixed_page_provider(page),
- "llm_caller": _ScriptedCaller(script),
- "step": None,
- }
- )
-
- with capture_logs() as logs:
- await _run_execute_task_v3(
- monkeypatch,
- LoopOutcome(status="failed", reason="the canned outcome must not be used"),
- url=task_url,
- navigation_goal="Apply.",
- navigation_status=404 if initial else None,
- landed_url="https://jobs.example.test/expired" if initial else None,
- data_extraction_goal=None,
- extracted_information_schema=None,
- loop_body=_real_loop,
- working_page=page,
- )
-
- event = "taskv3 loop initial navigation dead end" if initial else "taskv3 loop navigation dead end"
- assert [entry["provenance"] for entry in logs if entry["event"] == event] == ["task_supplied"]
-
-
@pytest.mark.asyncio
async def test_execute_task_v3_publishes_no_path_for_a_block_that_never_reached_the_pin(
monkeypatch: pytest.MonkeyPatch,