Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
Run every Task V3 test through production's agent setup (SKY-18395) (#8959)
Andrew Neilson1822679c68bf
diff --git a/tests/conftest.py b/tests/conftest.pyindex 7b7b7568a..387a03444 100644--- a/tests/conftest.py+++ b/tests/conftest.py@@ -1,5 +1,6 @@"""Fixtures shared by every suite."""+import functoolsimport gcimport inspectimport io@@ -23,6 +24,7 @@ from skyvern.forge.sdk.experimentation.code_block_ai_fallback import CODE_BLOCK_from skyvern.forge.sdk.experimentation.providers import NoOpExperimentationProviderfrom skyvern.forge.sdk.forge_log import setup_loggerfrom skyvern.forge.sdk.workflow.models.block import CodeBlock+from skyvern.forge.taskv3 import engine as taskv3_enginefrom skyvern.services import organization_log_scope_monkeypatch_setattr = pytest.MonkeyPatch.setattr@@ -88,6 +90,41 @@ def _url_validation_enabled(monkeypatch: pytest.MonkeyPatch) -> None:monkeypatch.setattr(settings, "ENABLE_URL_VALIDATION", True)+LOOP_OWNER = ("skyvern/forge/agent.py", "_execute_task_v3")+# The loop's own contract tests call it directly by design.+LOOP_OWNER_EXEMPT_TEST_FILES = frozenset({"test_taskv3_engine.py"})+++def loop_entered_through_executor() -> bool:+ frame = sys._getframe(1)+ while frame is not None:+ code = frame.f_code+ if code.co_name == LOOP_OWNER[1] and code.co_filename.endswith(LOOP_OWNER[0]):+ return True+ frame = frame.f_back+ return False++[email protected](autouse=True)+def _loop_runs_only_inside_the_executor(request: pytest.FixtureRequest, monkeypatch: pytest.MonkeyPatch) -> None:+ """The runtime half of tests/cloud/test_taskv3_single_loop_call_site.py: it sees calls the static scan cannot,+ such as a name computed at runtime."""+ if request.node.path.name in LOOP_OWNER_EXEMPT_TEST_FILES:+ return+ loop = taskv3_engine.run_task_v3_agent_loop++ @functools.wraps(loop)+ async def guarded(*args: Any, **kwargs: Any) -> taskv3_engine.LoopOutcome:+ if not loop_entered_through_executor():+ raise AssertionError(+ "run_task_v3_agent_loop ran outside ForgeAgent._execute_task_v3; run the task through the executor "+ "(tests/browser_e2e/taskv3_fixture_helpers.run_production_task_v3)"+ )+ return await loop(*args, **kwargs)++ monkeypatch.setattr(taskv3_engine, "run_task_v3_agent_loop", guarded)++@pytest.fixture(autouse=True)def _isolate_organization_age_cache() -> Iterator[None]:"""Loading an organization caches its creation time process-wide; no test may leak it into another's logs."""diff --git a/tests/unit/helpers.py b/tests/unit/helpers.pyindex 983045318..d20dc99b2 100644--- a/tests/unit/helpers.py+++ b/tests/unit/helpers.py@@ -1,9 +1,10 @@from __future__ import annotationsimport json+import secretsfrom contextlib import contextmanagerfrom dataclasses import dataclass-from datetime import datetime+from datetime import UTC, datetimefrom types import SimpleNamespacefrom typing import Any, Iterator, Sequencefrom unittest.mock import AsyncMock, MagicMock@@ -346,6 +347,131 @@ def setup_parallel_verification_mocks()+# The repository's update_step adds these deltas to the step's own running totals, as the SQL update does.+_STEP_COUNTER_COLUMNS = {+ "incremental_cost": "step_cost",+ "incremental_input_tokens": "input_token_count",+ "incremental_output_tokens": "output_token_count",+ "incremental_reasoning_tokens": "reasoning_token_count",+ "incremental_cached_tokens": "cached_token_count",+}+++@dataclass+class InMemoryTaskStepRepo:+ """Backs app.DATABASE.tasks for one task and step, so the real ForgeAgent update methods read back their writes."""++ task: Task | None = None+ step: Step | None = None++ async def create_task(self, *, url: str, organization_id: str | None, **fields: Any) -> Task:+ fields["max_screenshot_scrolls"] = fields.pop("max_screenshot_scrolling_times", None)+ now = datetime.now(UTC)+ self.task = Task(+ task_id=f"tsk_{secrets.token_hex(6)}",+ url=url,+ organization_id=organization_id,+ status=TaskStatus.created,+ created_at=now,+ modified_at=now,+ **{name: value for name, value in fields.items() if value is not None},+ )+ return self.task++ async def create_step(self, task_id: str, *, order: int, retry_index: int, organization_id: str | None) -> Step:+ now = datetime.now(UTC)+ self.step = Step(+ task_id=task_id,+ step_id=f"stp_{secrets.token_hex(6)}",+ status=StepStatus.created,+ order=order,+ retry_index=retry_index,+ is_last=False,+ organization_id=organization_id,+ created_at=now,+ modified_at=now,+ )+ return self.step++ async def get_task(self, task_id: str, organization_id: str | None = None) -> Task | None:+ return self.task if self.task is not None and self.task.task_id == task_id else None++ async def update_task(self, task_id: str, organization_id: str | None = None, **updates: Any) -> Task:+ updated, _ = await self.update_task_and_claim_finish(task_id, organization_id=organization_id, **updates)+ return updated++ async def update_task_and_claim_finish(+ self,+ task_id: str,+ status: TaskStatus | None = None,+ organization_id: str | None = None,+ errors: list[dict[str, Any]] | None = None,+ *,+ keep_final_status: bool = False,+ **updates: Any,+ ) -> tuple[Task, bool]:+ task = self.task+ assert task is not None and task.task_id == task_id, f"no task {task_id}"+ if keep_final_status and task.status.is_final():+ return task, False+ finish_claimed = False+ if status is not None:+ task.status = status+ if status == TaskStatus.running and task.started_at is None:+ task.started_at = datetime.now(UTC)+ if status.is_final() and task.finished_at is None:+ task.finished_at = datetime.now(UTC)+ finish_claimed = True+ if errors is not None:+ task.errors = [*(task.errors or []), *errors]+ for key, value in updates.items():+ if value is not None:+ setattr(task, key, value)+ return task, finish_claimed++ async def update_step(self, task_id: str, step_id: str, organization_id: str | None = None, **updates: Any) -> Step:+ assert self.step is not None and self.step.step_id == step_id, f"no step {step_id}"+ for key, value in updates.items():+ if key in _STEP_COUNTER_COLUMNS:+ column = _STEP_COUNTER_COLUMNS[key]+ if value is not None:+ setattr(self.step, column, (getattr(self.step, column) or 0) + value)+ else:+ setattr(self.step, key, value)+ return self.step++ async def get_first_step(self, task_id: str, organization_id: str | None = None) -> None:+ return None++ async def get_task_steps(self, task_id: str, organization_id: str | None = None) -> list[Step]:+ return [self.step] if self.step is not None and self.step.task_id == task_id else []++ async def get_latest_step(self, task_id: str, organization_id: str | None = None) -> Step | None:+ steps = await self.get_task_steps(task_id, organization_id)+ return steps[-1] if steps else None++ async def get_run(self, run_id: str, organization_id: str | None = None) -> None:+ return None++ def install(self, monkeypatch: MonkeyPatch) -> None:+ for name in (+ "create_task",+ "create_step",+ "get_task",+ "update_task",+ "update_task_and_claim_finish",+ "update_step",+ "get_first_step",+ "get_task_steps",+ "get_latest_step",+ "get_run",+ ):+ monkeypatch.setattr(app.DATABASE.tasks, name, getattr(self, name))+ monkeypatch.setattr(+ app.DATABASE.observer, "get_workflow_run_block_engine_by_task_id", AsyncMock(return_value=None)+ )++def make_browser_state() -> tuple[MagicMock, MagicMock, MagicMock]:return MagicMock(), MagicMock(), MagicMock()diff --git a/tests/unit/test_agent_task_v3.py b/tests/unit/test_agent_task_v3.pyindex a3f4847fd..b14544be7 100644--- a/tests/unit/test_agent_task_v3.py+++ b/tests/unit/test_agent_task_v3.py@@ -69,16 +69,16 @@ from skyvern.forge.taskv3 import input_dispatchfrom skyvern.forge.taskv3 import provider_downloads as taskv3_provider_downloadsfrom skyvern.forge.taskv3 import tools as taskv3_toolsfrom skyvern.forge.taskv3.auth_tools import VerificationFailure, VerificationState-from skyvern.forge.taskv3.engine import DEFAULT_MAX_SETTLE_DEFERRALS, MIN_ACTION_STEPS, run_task_v3_agent_loop-from skyvern.forge.taskv3.goal_check import BLOCK_COMPLETION_CHECK_PROMPT_NAME+from skyvern.forge.taskv3.engine import DEFAULT_MAX_SETTLE_DEFERRALS, MIN_ACTION_STEPS+from skyvern.forge.taskv3.goal_check import (+ BLOCK_COMPLETION_CHECK_PROMPT_NAME,+)from skyvern.forge.taskv3.goal_composition import CodeProgressRecord, CodeTypedValuefrom skyvern.forge.taskv3.handoff_redaction import pin_caller_authored_block_urlsfrom skyvern.forge.taskv3.loop import (ACTION_BLOCK_TARGET_ACTION_RESERVE,ACTION_LOOP_GUARD,- COMPLETION_GATE_REFUSAL,NAV_DEAD_END_GUARD,- TOKEN_BUDGET_EXTENDED_EVENT,LoopOutcome,RoundAction,ToolSpec,@@ -114,11 +114,14 @@ from tests.unit._download_provider_fakes import (_secret_bearing_validation_error,_StallingActionDownloadObservation,)-from tests.unit.helpers import make_action_row, make_browser_state, make_organization, make_step, make_task+from tests.unit.helpers import (+ make_action_row,+ make_browser_state,+ make_organization,+ make_step,+ make_task,+)from tests.unit.scoped_asyncio import ScopedAsyncio-from tests.unit.test_taskv3_engine import _fixed_read_tool, _ReaskAnsweringCaller-from tests.unit.test_taskv3_loop import _ScriptedCaller-from tests.unit.test_taskv3_tools import _FakePage, _fixed_page_providerasync def _run_execute_task_v3(@@ -162,8 +165,6 @@ async def _run_execute_task_v3(# Called with the loop's kwargs before the loop returns, to drive state the loop's tools own.on_loop: Callable[[dict[str, Any]], None] | None = None,on_loop_async: Callable[[dict[str, Any]], Awaitable[None]] | None = None,- # Replaces the canned `outcome`: receives the kwargs agent.py built and returns the loop's outcome.- loop_body: Callable[[dict[str, Any]], Awaitable[LoopOutcome]] | None = None,# Leave the credential-TOTP candidate gate reading the real workflow-run context.real_credential_totp_candidate: bool = False,llm_caller_factory: Any = None,@@ -234,8 +235,6 @@ async def _run_execute_task_v3(on_loop(kwargs)if on_loop_async is not None:await on_loop_async(kwargs)- if loop_body is not None:- return await loop_body(kwargs)if provider_probe_calls:provider = kwargs["page_provider"]loop_mock.resolved_pages = [await provider() for _ in range(provider_probe_calls)]@@ -3158,75 +3157,6 @@ async def test_execute_task_v3_action_block_target_reserve_answers_to_the_run_poassert kwargs["target_action_reserve"] == reserve-class _AdvancingFormPage(_FakePage):- """Every action advances the form, so the next observe is fresh page-change evidence."""-- # Takes the probe's argument, as Playwright's evaluate does, so the real settle sampler reads this page.- async def evaluate(self, _js: str, _arg: Any = None) -> str:- raw = await super().evaluate(_js)- if "document.readyState" in _js:- return raw- section = sum(1 for name, _args in self.calls if name in ("click", "fill", "type"))- data = json.loads(raw)- data["title"] = f"Apply, section {section}"- data["elements"][0]["label"] = f"Section {section} first name"- return json.dumps(data)---async def test_execute_task_v3_navigation_block_token_trip_with_progress_is_extended(- monkeypatch: pytest.MonkeyPatch,-) -> None:- # A long form resends a growing transcript, so the token backstop trips while most of the step budget is- # unspent. Driven through agent.py with the real engine and loop, so the step floor, ceilings and backstops- # agent.py configures are the ones the grant must survive. The form is filled one field per round, and each- # new field followed by a changed page renews the grant.- script: list[list[tuple[str, dict[str, Any]]]] = []- for i in range(5):- script.append([("observe", {})])- script.append([("type", {"selector": f"#field-{i}", "text": f"answer {i}"})])- script.append([("observe", {})])- script.append([("click", {"selector": "#submit"})])- script.append([("finish", {"status": "completed", "reason": "submitted"})])- # The first call is the normal-size fixed prompt; the burn after it is the transcript growing.- caller = _ScriptedCaller(- script,- usage_for_call=lambda call: {"prompt_tokens": 15_000 if call == 0 else 200_000, "completion_tokens": 0},- )- page = _AdvancingFormPage()- loop_kwargs: dict[str, Any] = {}-- async def _real_loop(kwargs: dict[str, Any]) -> LoopOutcome:- loop_kwargs.update(kwargs)- return await run_task_v3_agent_loop(- # The scripted caller takes no step-scoped call kwargs.- **{**kwargs, "page_provider": _fixed_page_provider(page), "llm_caller": caller, "step": None}- )-- monkeypatch.setattr(ForgeAgent, "_check_workflow_run_step_budget", AsyncMock(return_value=None))- with capture_logs() as logs:- _step, task, _loop_mock, _post = await _run_execute_task_v3(- monkeypatch,- LoopOutcome(status="failed", reason="the canned outcome must not be used"),- task_block=_make_block(NavigationBlock, navigation_goal="Fill and submit the application"),- workflow_run_id="wr_long_form",- max_steps_per_run=10,- data_extraction_goal=None,- extracted_information_schema=None,- loop_body=_real_loop,- working_page=page,- )- assert loop_kwargs["max_action_steps"] == MIN_ACTION_STEPS- granted = [entry for entry in logs if entry["event"] == TOKEN_BUDGET_EXTENDED_EVENT]- assert [(entry["guard"], entry["original_max_tokens"]) for entry in granted] == [- ("max_tokens", loop_kwargs["max_tokens"]),- ("max_tokens", granted[0]["max_tokens"]),- ]- assert granted[0]["action_steps"] < MIN_ACTION_STEPS- assert caller.calls == len(script)- assert task.status == TaskStatus.completed--@pytest.mark.asyncioasync def test_execute_task_v3_no_workflow_ceiling_without_a_pool(monkeypatch: pytest.MonkeyPatch) -> None:# No org pool -> no hard ceiling: the loop's extension is bounded only by its own gate.@@ -4211,97 +4141,6 @@ async def test_a_fifth_veto_on_a_reask_conversion_restores_the_models_own_verdicassert task.failure_reason == "the site offers no way to continue"-async def test_execute_task_v3_gate_veto_is_refused_in_loop_and_the_approved_retry_skips_the_post_loop_gate(- monkeypatch: pytest.MonkeyPatch,-) -> None:- block = _make_block(NavigationBlock, navigation_goal="Submit the application")- page = _AdvancingFormPage()- caller = _ScriptedCaller(- [- [("finish", {"status": "completed", "reason": "looks done"})],- [("observe", {})],- [("finish", {"status": "completed", "reason": "confirmed"})],- ]- )-- async def _real_loop(kwargs: dict[str, Any]) -> LoopOutcome:- return await run_task_v3_agent_loop(- **{**kwargs, "page_provider": _fixed_page_provider(page), "llm_caller": caller, "step": None}- )-- _step, task, loop_mock, _post = await _run_execute_task_v3(- monkeypatch,- LoopOutcome(status="failed", reason="the canned outcome must not be used"),- task_block=block,- loop_body=_real_loop,- working_page=page,- # A post-loop consult would draw the trailing veto and fail the task.- completion_gate_raises=[False, True, False],- data_extraction_goal=None,- extracted_information_schema=None,- )-- assert task.status == TaskStatus.completed- assert loop_mock.completion_gate.await_args.kwargs["task_block"] is block- assert sum(COMPLETION_GATE_REFUSAL in str(m.get("content")) for m in caller.message_history) == 1---async def test_execute_task_v3_a_vetoed_reask_conversion_restores_the_models_verdict(- monkeypatch: pytest.MonkeyPatch,-) -> None:- # The model terminated; the re-ask converted it; the deployment gate rejects the completion. The task must- # end as the model's own terminated verdict, not as a failed completion the model never claimed.- page = _FakePage()- outcome = await run_task_v3_agent_loop(- page_provider=_fixed_page_provider(page),- llm_caller=_ReaskAnsweringCaller(- [- [("type", {"selector": "#email", "text": "[email protected]"})],- [("click", {"selector": "#create"})],- [("read_fixture", {})],- [("observe", {})],- [("finish", {"status": "terminated", "reason": "The PIN screen was never shown."})],- ],- answer={- "verdict": "completed",- "terminate_criterion_holds": False,- "skipped_screen": "the PIN screen",- "quote": "Email: [email protected]",- "evidence": "The next form carries the account's email.",- },- ),- goal="Create the account.",- parameters={"email": "[email protected]"},- extra_tools=[- _fixed_read_tool(- "Application\nEmail: [email protected]", page=page, lands_at="https://example.test/apply/section/1"- )- ],- unlisted_reask_criteria=("a PIN screen is shown", "the create-account submission fails"),- )- assert outcome.status == "completed" and outcome.converted_from == "terminated"- _step, task, loop_mock, _post = await _run_execute_task_v3(- monkeypatch,- outcome,- task_block=_make_block(NavigationBlock, navigation_goal="Create the account"),- completion_gate_vetoes=True,- complete_criterion="a PIN screen is shown",- terminate_criterion="the create-account submission fails",- data_extraction_goal=None,- extracted_information_schema=None,- )-- assert loop_mock.await_args is not None- assert loop_mock.await_args.kwargs["unlisted_reask_criteria"] == (- "a PIN screen is shown",- "the create-account submission fails",- )- assert task.status == TaskStatus.terminated- assert task.failure_reason == "The PIN screen was never shown."--@pytest.mark.asyncio@pytest.mark.parametrize(("task_type", "custom_key", "criteria"),@@ -6651,55 +6490,6 @@ async def test_execute_task_v3_reads_a_blocks_caller_urls_from_its_unrendered_deassert off_the_page in kwargs["goal"] # nosemgrep: incomplete-url-substring-sanitization[email protected]("initial", [False, True])-async def test_execute_task_v3_logs_a_dead_task_url_with_a_query_as_task_supplied(- monkeypatch: pytest.MonkeyPatch, initial: bool-) -> None:- # agent.py's task-target set and requested start URL, through the real engine and navigate tool: a- # task URL whose query names the page is the task's own target, whether the in-loop navigate or the- # setup navigation (redirected first) is the one that 404s.- import skyvern.utils.url_validators as urlv-- monkeypatch.setattr(urlv, "validate_fetch_url", lambda url: url)- task_url = "https://jobs.example.test/portal.action?page=apply&id=42"- page = _FakePage()-- async def _goto(url: str, timeout: int | None = None, wait_until: str | None = None) -> Any:- page.url = url- return SimpleNamespace(status=404, url=url)-- page.goto = _goto # type: ignore[method-assign]- script = [[("navigate", {"url": task_url})], [("finish", {"status": "completed", "reason": "never"})]]-- async def _real_loop(kwargs: dict[str, Any]) -> LoopOutcome:- return await run_task_v3_agent_loop(- **{- **kwargs,- "page_provider": _fixed_page_provider(page),- "llm_caller": _ScriptedCaller(script),- "step": None,- }- )-- with capture_logs() as logs:- await _run_execute_task_v3(- monkeypatch,- LoopOutcome(status="failed", reason="the canned outcome must not be used"),- url=task_url,- navigation_goal="Apply.",- navigation_status=404 if initial else None,- landed_url="https://jobs.example.test/expired" if initial else None,- data_extraction_goal=None,- extracted_information_schema=None,- loop_body=_real_loop,- working_page=page,- )-- event = "taskv3 loop initial navigation dead end" if initial else "taskv3 loop navigation dead end"- assert [entry["provenance"] for entry in logs if entry["event"] == event] == ["task_supplied"]--@pytest.mark.asyncioasync def test_execute_task_v3_publishes_no_path_for_a_block_that_never_reached_the_pin(monkeypatch: pytest.MonkeyPatch,