Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
Fix cached login scripts falsely completing when 0 form fields found (#5270)
pedrohsdb88136906c8af
diff --git a/skyvern/core/script_generations/skyvern_page.py b/skyvern/core/script_generations/skyvern_page.pyindex f80ce83fc..568b728ee 100644--- a/skyvern/core/script_generations/skyvern_page.py+++ b/skyvern/core/script_generations/skyvern_page.py@@ -1145,10 +1145,16 @@ class SkyvernPage(Page):@action_wrap(ActionType.WAIT)async def wait(self,- seconds: float,+ seconds: float | None = None,**kwargs: Any,) -> None:- await asyncio.sleep(seconds)+ timeout_ms = kwargs.pop("timeout_ms", None)+ if seconds is not None:+ await asyncio.sleep(seconds)+ elif timeout_ms is not None:+ await asyncio.sleep(timeout_ms / 1000.0)+ else:+ await asyncio.sleep(0)@action_wrap(ActionType.NULL_ACTION)async def null_action(self, **kwargs: Any) -> None:@@ -1874,6 +1880,13 @@ class SkyvernPage(Page):data_keys=list(data.keys())[:10],)+ if not form_fields:+ raise RuntimeError(+ "fill_form found 0 form fields on the page. "+ "The page may not have finished rendering — try adding "+ "await page.wait(timeout_ms=5000) before fill_form()."+ )+mapping = await self.dynamic_field_map(form_fields, data, prompt=prompt)if not await self.validate_mapping(form_fields, mapping, prompt):diff --git a/skyvern/forge/prompts/skyvern/script-reviewer-form-filling.j2 b/skyvern/forge/prompts/skyvern/script-reviewer-form-filling.j2index 02546c6ed..ae7eaf3b1 100644--- a/skyvern/forge/prompts/skyvern/script-reviewer-form-filling.j2+++ b/skyvern/forge/prompts/skyvern/script-reviewer-form-filling.j2@@ -143,6 +143,13 @@ When the error or page text suggests the page wasn't ready for interaction, theIf the existing code already has a `page.wait()` and it still failed, increase the timeout (e.g., 5000→8000→10000→15000).+3. **Before `fill_form()` when 0 fields are found** — If the error message says "fill_form found 0 form fields", the page hasn't finished rendering its form inputs. Add `await page.wait(timeout_ms=5000)` **before** the `fill_form()` call:+ ```python+ await page.goto(url)+ await page.wait(timeout_ms=5000) # wait for form fields to render+ await page.fill_form(context.parameters, prompt='...')+ ```+## What to do1. Compare the episode's form fields to the existing FIELD_MAP.2. For each field in the episode, first check if it matches a standard category listed above. If it does, do NOT add it to FIELD_MAP — canonical matching handles it automatically.diff --git a/skyvern/forge/prompts/skyvern/script-reviewer.j2 b/skyvern/forge/prompts/skyvern/script-reviewer.j2index b2b973fce..e5d1cd253 100644--- a/skyvern/forge/prompts/skyvern/script-reviewer.j2+++ b/skyvern/forge/prompts/skyvern/script-reviewer.j2@@ -143,6 +143,13 @@ When the error or page text suggests the page wasn't ready for interaction, theIf the existing code already has a `page.wait()` and it still failed, increase the timeout (e.g., 5000→8000→10000→15000).+3. **Before `fill_form()` when 0 fields are found** — If the error message says "fill_form found 0 form fields", the page hasn't finished rendering its form inputs. Add `await page.wait(timeout_ms=5000)` **before** the `fill_form()` call:+ ```python+ await page.goto(url)+ await page.wait(timeout_ms=5000) # wait for form fields to render+ await page.fill_form(context.parameters, prompt='...')+ ```+## What to do{% for episode in episodes %}diff --git a/skyvern/services/script_service.py b/skyvern/services/script_service.pyindex 54c30205b..d88d7b94c 100644--- a/skyvern/services/script_service.py+++ b/skyvern/services/script_service.py@@ -18,7 +18,12 @@ from fastapi import BackgroundTasks, HTTPExceptionfrom jinja2.sandbox import SandboxedEnvironmentfrom skyvern.config import settings-from skyvern.constants import BROWSER_DOWNLOADING_SUFFIX, GET_DOWNLOADED_FILES_TIMEOUT, SAVE_DOWNLOADED_FILES_TIMEOUT+from skyvern.constants import (+ BROWSER_DOWNLOADING_SUFFIX,+ DEFAULT_LOGIN_COMPLETE_CRITERION,+ GET_DOWNLOADED_FILES_TIMEOUT,+ SAVE_DOWNLOADED_FILES_TIMEOUT,+)from skyvern.core.script_generations.constants import SCRIPT_TASK_BLOCKSfrom skyvern.core.script_generations.generate_script import _build_block_fn, create_or_update_script_blockfrom skyvern.core.script_generations.script_skyvern_page import script_run_context_manager@@ -473,11 +478,17 @@ async def _create_workflow_block_run_and_task(# Include script parameters as navigation_payload so handlers# (e.g. file upload) can find URLs like resume_link in the payload.nav_payload = context.script_run_parameters or None+ # Apply default complete_criterion for login blocks so cached scripts+ # get the same rigorous LLM verification as the agent path (SKY-8540).+ # Without this, the LLM only sees a generic navigation_goal and can+ # falsely mark login as complete when credentials were never entered.+ task_complete_criterion = DEFAULT_LOGIN_COMPLETE_CRITERION if block_type == BlockType.LOGIN else Nonetask = await app.DATABASE.create_task(# fix HACK: changed the type of url to str | None to support None url. url is not used in the script right now.url=url or "",title=f"Script {block_type.value} task",navigation_goal=prompt,+ complete_criterion=task_complete_criterion,data_extraction_goal=prompt if block_type == BlockType.EXTRACTION else None,extracted_information_schema=schema,navigation_payload=nav_payload,diff --git a/tests/unit/test_script_skyvern_page.py b/tests/unit/test_script_skyvern_page.pyindex 27e5b1f3b..4fd9f0736 100644--- a/tests/unit/test_script_skyvern_page.py+++ b/tests/unit/test_script_skyvern_page.py@@ -3,11 +3,13 @@ Unit tests for ScriptSkyvernPage.Tests _wait_for_page_ready_before_action (regression test for self._page bug, PR #8425),_ensure_element_ids_on_page (injects unique_id attrs after page navigation),-and terminate() (raises ScriptTerminationException for Code 2.0 cached execution).+terminate() (raises ScriptTerminationException for Code 2.0 cached execution),+and wait() (accepts both seconds= and timeout_ms= parameter styles)."""import inspectimport re+import timefrom unittest.mock import AsyncMock, MagicMock, patchimport pytest@@ -756,3 +758,103 @@ async def test_fill_autocomplete_value_none_proactive_unchanged(mock_scraped_pagassert result == "ai_value"script_page._ai.ai_input_text.assert_called_once()+++# =============================================================================+# Tests for wait() — timeout_ms support+# =============================================================================+++def _get_wait_inner_fn():+ """Extract the inner wait function from the action_wrap closure.++ action_wrap replaces the method with a wrapper. The original function+ is stored in the closure as the 'fn' free variable.+ """+ from skyvern.core.script_generations.skyvern_page import SkyvernPage++ wrapper = SkyvernPage.wait+ # closure vars are ('action', 'fn') per action_wrap implementation+ for var_name, cell in zip(wrapper.__code__.co_freevars, wrapper.__closure__):+ if var_name == "fn":+ return cell.cell_contents+ raise RuntimeError("Could not extract inner wait function from action_wrap closure")+++class TestWaitMethod:+ """Tests for SkyvernPage.wait() accepting both seconds= and timeout_ms=.++ The script reviewer prompt documents the API as page.wait(timeout_ms=5000),+ but the original implementation only accepted wait(seconds=5). This mismatch+ caused every LLM-generated wait call to raise TypeError at runtime, silently+ triggering agent fallback instead of actually waiting.+ """++ @pytest.mark.asyncio+ async def test_wait_with_seconds_kwarg(self):+ """wait(seconds=0.05) should sleep for ~0.05 seconds."""+ fn = _get_wait_inner_fn()+ t0 = time.monotonic()+ await fn(None, seconds=0.05)+ elapsed = time.monotonic() - t0+ assert elapsed >= 0.04, f"Expected ~0.05s sleep, got {elapsed:.3f}s"++ @pytest.mark.asyncio+ async def test_wait_with_seconds_positional(self):+ """wait(0.05) should sleep for ~0.05 seconds (positional arg)."""+ fn = _get_wait_inner_fn()+ t0 = time.monotonic()+ await fn(None, 0.05)+ elapsed = time.monotonic() - t0+ assert elapsed >= 0.04, f"Expected ~0.05s sleep, got {elapsed:.3f}s"++ @pytest.mark.asyncio+ async def test_wait_with_timeout_ms_kwarg(self):+ """wait(timeout_ms=50) should sleep for ~0.05 seconds."""+ fn = _get_wait_inner_fn()+ t0 = time.monotonic()+ await fn(None, timeout_ms=50)+ elapsed = time.monotonic() - t0+ assert elapsed >= 0.04, f"Expected ~0.05s sleep, got {elapsed:.3f}s"++ @pytest.mark.asyncio+ async def test_wait_with_timeout_ms_converts_correctly(self):+ """wait(timeout_ms=100) should sleep ~0.1s, not 100 seconds."""+ fn = _get_wait_inner_fn()+ t0 = time.monotonic()+ await fn(None, timeout_ms=100)+ elapsed = time.monotonic() - t0+ assert 0.08 <= elapsed <= 0.5, f"Expected ~0.1s, got {elapsed:.3f}s — ms→s conversion may be wrong"++ @pytest.mark.asyncio+ async def test_wait_seconds_takes_precedence_over_timeout_ms(self):+ """When both seconds= and timeout_ms= are provided, seconds= wins."""+ fn = _get_wait_inner_fn()+ t0 = time.monotonic()+ await fn(None, seconds=0.05, timeout_ms=10000)+ elapsed = time.monotonic() - t0+ assert elapsed < 1.0, f"seconds= should take precedence, but waited {elapsed:.3f}s"++ @pytest.mark.asyncio+ async def test_wait_no_args_returns_immediately(self):+ """wait() with no args should return immediately (sleep 0)."""+ fn = _get_wait_inner_fn()+ t0 = time.monotonic()+ await fn(None)+ elapsed = time.monotonic() - t0+ assert elapsed < 0.1, f"Expected immediate return, got {elapsed:.3f}s"++ def test_wait_signature_allows_timeout_ms(self):+ """The inner wait function must accept timeout_ms via **kwargs without TypeError.++ This is the core regression test: the old signature was wait(self, seconds: float, **kwargs)+ where seconds was required. Calling wait(timeout_ms=5000) raised TypeError because+ seconds had no default. The fix makes seconds optional (default None).+ """+ fn = _get_wait_inner_fn()+ sig = inspect.signature(fn)+ seconds_param = sig.parameters.get("seconds")+ assert seconds_param is not None, "wait() should have a 'seconds' parameter"+ assert seconds_param.default is None, (+ f"seconds should default to None so timeout_ms can be used instead, got default={seconds_param.default}"+ )