Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
fix: restrict form_filling classification to blocks already using fill_form (#5395)
pedrohsdbe0fba0b094eb
diff --git a/skyvern/services/script_reviewer.py b/skyvern/services/script_reviewer.pyindex 4d1e2c656..cb1287c01 100644--- a/skyvern/services/script_reviewer.py+++ b/skyvern/services/script_reviewer.py@@ -662,12 +662,7 @@ class ScriptReviewer:function_signature = self._extract_function_signature(existing_code)# Classify strategy for this block- strategy = self._classify_block_strategy(- block_label=block_label,- navigation_goal=navigation_goal,- existing_code=existing_code,- episodes=episodes,- )+ strategy = self._classify_block_strategy(existing_code=existing_code)# Choose template based on strategyif strategy == "extraction":@@ -1024,19 +1019,21 @@ class ScriptReviewer:def _classify_block_strategy(self,- block_label: str,- navigation_goal: str | None,existing_code: str | None,- episodes: list[ScriptFallbackEpisode],) -> Literal["form_filling", "sequential", "extraction"]:- """Classify a block's caching strategy based on its content.+ """Classify a block's caching strategy based on its existing code.Rules (heuristic, no LLM call):1. If existing_code contains "page.extract(" without "page.click(" → "extraction"2. If existing_code already uses "page.fill_form(" → "form_filling"- 3. If navigation_goal contains form-filling keywords AND- episodes have form_fields with 4+ fields → "form_filling"- 4. Otherwise → "sequential"+ (preserve classification for blocks already using the form-filling API)+ 3. Otherwise → "sequential"++ Note: form_filling is NOT inferred from page content or navigation goals.+ The form-filling template produces a FIELD_MAP dict response format that is+ incompatible with blocks that mix clicks, navigation, and downloads. Only+ blocks explicitly using page.fill_form() (static form scripts) should use+ the form-filling template."""code = existing_code or ""@@ -1044,45 +1041,10 @@ class ScriptReviewer:if "page.extract(" in code and "page.click(" not in code:return "extraction"- # Rule 2: Already uses fill_form (previous form-filling classification)+ # Rule 2: Already uses fill_form (preserve existing form-filling classification)if "page.fill_form(" in code:return "form_filling"- # Rule 3: Heuristic for form-filling based on navigation goal + episode data- goal = (navigation_goal or "").lower()- form_keywords = {"fill", "application", "form", "submit", "apply", "sign up", "register", "signup"}- has_form_goal = any(kw in goal for kw in form_keywords)-- # Check known form hosts in episode URLs- form_hosts = {"lever.co", "greenhouse.io", "workday.com", "myworkdayjobs.com", "icims.com", "ashbyhq.com"}- has_form_url = False- for ep in episodes:- if ep.page_url:- if any(host in ep.page_url.lower() for host in form_hosts):- has_form_url = True- break-- # Count form fields and form actions per episode (use max across episodes)- max_form_fields = 0- max_form_actions = 0- for ep in episodes:- if isinstance(ep.agent_actions, dict):- form_fields = ep.agent_actions.get("form_fields", [])- if isinstance(form_fields, list):- max_form_fields = max(max_form_fields, len(form_fields))-- ep_form_actions = 0- actions = ep.agent_actions.get("actions", [])- if isinstance(actions, list):- for a in actions:- if isinstance(a, dict) and a.get("action_type") in ("input_text", "select_option"):- ep_form_actions += 1- max_form_actions = max(max_form_actions, ep_form_actions)-- # Classify as form_filling if we see strong signals- if (has_form_goal or has_form_url) and (max_form_fields >= 4 or max_form_actions >= 4):- return "form_filling"-return "sequential"async def _load_block_code(diff --git a/tests/unit/test_script_reviewer_quality.py b/tests/unit/test_script_reviewer_quality.pyindex e57c6c4d1..8a6b1b127 100644--- a/tests/unit/test_script_reviewer_quality.py+++ b/tests/unit/test_script_reviewer_quality.py@@ -527,3 +527,53 @@ async def block_1(page, context):result = extract_cached_blocks_from_source(source)assert "only_block" in resultassert "pass" in result["only_block"]+++class TestClassifyBlockStrategy:+ """Tests for _classify_block_strategy template selection."""++ def setup_method(self):+ from skyvern.services.script_reviewer import ScriptReviewer++ self.reviewer = ScriptReviewer()++ def test_extraction_block(self):+ """Block with page.extract() and no page.click() → extraction."""+ code = "async def block(page, ctx):\n return await page.extract(prompt='...')\n"+ result = self.reviewer._classify_block_strategy(existing_code=code)+ assert result == "extraction"++ def test_extraction_with_click_is_sequential(self):+ """Block with both extract and click → sequential (not extraction)."""+ code = (+ "async def block(page, ctx):\n await page.click('#btn')\n return await page.extract(prompt='...')\n"+ )+ result = self.reviewer._classify_block_strategy(existing_code=code)+ assert result == "sequential"++ def test_existing_fill_form_stays_form_filling(self):+ """Block already using page.fill_form() stays form_filling."""+ code = "async def block(page, ctx):\n await page.fill_form(ctx.parameters, prompt='...')\n"+ result = self.reviewer._classify_block_strategy(existing_code=code)+ assert result == "form_filling"++ def test_download_block_is_sequential(self):+ """Download block should be sequential, never form_filling."""+ code = "async def download(page, ctx):\n await page.download_file(prompt='...')\n await page.complete()\n"+ result = self.reviewer._classify_block_strategy(existing_code=code)+ assert result == "sequential"++ def test_navigation_block_with_form_fields_is_sequential(self):+ """Block on a page with form fields should still be sequential (not form_filling).++ Regression guard: this data (5 form fields + form keywords in goal + form+ action types) would have triggered the removed heuristic. Verifies it's gone.+ """+ code = "async def block(page, ctx):\n await page.click('#btn')\n await page.complete()\n"+ result = self.reviewer._classify_block_strategy(existing_code=code)+ assert result == "sequential"++ def test_default_is_sequential(self):+ """Empty code, no special signals → sequential."""+ result = self.reviewer._classify_block_strategy(existing_code="")+ assert result == "sequential"