Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
feat(SKY-9570): turn classify-UNKNOWN into a productive learning signal (#5882)
pedrohsdbc587bfb6f5ba
diff --git a/skyvern/core/script_generations/real_skyvern_page_ai.py b/skyvern/core/script_generations/real_skyvern_page_ai.pyindex af157f3c1..4a49cb94d 100644--- a/skyvern/core/script_generations/real_skyvern_page_ai.py+++ b/skyvern/core/script_generations/real_skyvern_page_ai.py@@ -594,6 +594,12 @@ class RealSkyvernPageAi(SkyvernPageAi):"""current_url = self.page.urlresult = "UNKNOWN"+ # Defaults so the meta write below is always safe — the LLM-exception+ # path skips the assignments inside the try block, but UNKNOWN can+ # still be returned and the metadata fields must be defined.+ reasoning: str = ""+ confidence: float = 0.0+ extracted_text_for_meta: str = ""# Tier 0: URL pattern matching (FREE)if url_patterns:@@ -608,7 +614,12 @@ class RealSkyvernPageAi(SkyvernPageAi):)result = keyawait self._record_branch_hit(result)- self._store_classify_result(result)+ self._store_classify_result(+ result,+ current_url=current_url,+ options=options,+ block_label=self.current_label,+ )return resultexcept re.error:LOG.warning("page.classify: invalid URL regex pattern", key=key, pattern=pattern)@@ -633,18 +644,23 @@ class RealSkyvernPageAi(SkyvernPageAi):)result = keyawait self._record_branch_hit(result)- self._store_classify_result(result)+ self._store_classify_result(+ result,+ current_url=current_url,+ options=options,+ block_label=self.current_label,+ )return result# Tier 2: Mini-LLM classificationif not scraped:await self._refresh_scraped_page(take_screenshots=False)- extracted_text = (self.scraped_page.extracted_text or "")[:2000]+ extracted_text_for_meta = (self.scraped_page.extracted_text or "")[:2000]classify_prompt = prompt_engine.load_prompt(template="page-classify",current_url=current_url,- extracted_text=extracted_text,+ extracted_text=extracted_text_for_meta,options=options,)@@ -684,7 +700,15 @@ class RealSkyvernPageAi(SkyvernPageAi):result = "UNKNOWN"await self._record_branch_hit(result)- self._store_classify_result(result)+ self._store_classify_result(+ result,+ current_url=current_url,+ options=options,+ block_label=self.current_label,+ rejection_reasoning=reasoning,+ confidence=confidence,+ text_excerpt=extracted_text_for_meta,+ )return resultasync def _record_element_fallback_episode(@@ -794,14 +818,36 @@ class RealSkyvernPageAi(SkyvernPageAi):LOG.warning("Failed to record element fallback episode for selector failure", exc_info=True)@staticmethod- def _store_classify_result(result: str) -> None:- """Store the classify result on SkyvernContext for fallback episode recording."""+ def _store_classify_result(+ result: str,+ *,+ current_url: str,+ options: dict[str, str],+ block_label: str | None,+ rejection_reasoning: str = "",+ confidence: float = 0.0,+ text_excerpt: str = "",+ ) -> None:+ """Store classify result + (on UNKNOWN) rejection metadata for the next element_fallback to consume."""try:ctx = skyvern_context.current()- if ctx:- ctx.last_classify_result = result+ if ctx is None:+ return+ ctx.last_classify_result = result+ if result == "UNKNOWN":+ ctx.last_classify_meta = {+ "result": "UNKNOWN",+ "url_at_classify": current_url,+ "block_label_at_classify": block_label,+ "candidate_options": dict(options),+ "rejection_reasoning": rejection_reasoning,+ "confidence": confidence,+ "text_excerpt": (text_excerpt or "")[:500],+ }+ else:+ ctx.last_classify_meta = Noneexcept Exception:- pass+ LOG.debug("_store_classify_result: failed to persist classify meta", exc_info=True)async def _record_branch_hit(self, branch_key: str) -> None:"""Best-effort recording of a classify branch hit for TTL tracking."""@@ -824,16 +870,20 @@ class RealSkyvernPageAi(SkyvernPageAi):navigation_goal: str,max_steps: int = 5,) -> None:- """Activate the AI agent from the CURRENT page position to achieve a navigation goal.+ """Drive an AI agent from the current page state toward ``navigation_goal``.- Uses repeated ai_act calls with ai_validate checks to execute from the- current page state. Records all actions taken for later review by the- AI Script Reviewer.+ Records one fallback episode per invocation regardless of exit path; pops+ any preceding classify metadata at entry so subsequent fallbacks cannot+ inherit it."""context = skyvern_context.current()if not context or not context.organization_id:raise Exception("element_fallback requires an active context with organization_id")+ # Consume-at-entry — exception-safe pop.+ classify_meta: dict[str, Any] | None = context.last_classify_meta+ context.last_classify_meta = None+LOG.info("page.element_fallback: starting from current page",navigation_goal=navigation_goal,@@ -842,7 +892,6 @@ class RealSkyvernPageAi(SkyvernPageAi):workflow_run_id=context.workflow_run_id,)- # Capture page state before element fallback for episode recordingpage_url_at_entry = self.page.urlpage_text_at_entry: str | None = Nonetry:@@ -852,55 +901,63 @@ class RealSkyvernPageAi(SkyvernPageAi):steps_taken: list[dict] = []completed = False+ exception_summary: str | None = None+ captured_exception: BaseException | None = None++ try:+ for step_num in range(max_steps):+ is_complete = await self.ai_validate(+ prompt=f"Has the following goal been achieved? Goal: {navigation_goal}",+ )+ if is_complete:+ LOG.info(+ "page.element_fallback: goal achieved",+ step_num=step_num,+ navigation_goal=navigation_goal,+ )+ completed = True+ break- for step_num in range(max_steps):- # Check if the goal has been achieved- is_complete = await self.ai_validate(- prompt=f"Has the following goal been achieved? Goal: {navigation_goal}",- )- if is_complete:LOG.info(- "page.element_fallback: goal achieved",+ "page.element_fallback: executing step",step_num=step_num,- navigation_goal=navigation_goal,+ current_url=self.page.url,)- completed = True- break-- LOG.info(- "page.element_fallback: executing step",- step_num=step_num,- current_url=self.page.url,- )- # Let the AI agent take an action toward the goal- await self.ai_act(- prompt=f"Take the next action to achieve this goal: {navigation_goal}",+ context.current_step_actions = []+ url_before = self.page.url+ try:+ await self.ai_act(+ prompt=f"Take the next action to achieve this goal: {navigation_goal}",+ )+ actions_this_step = list(context.current_step_actions or [])+ finally:+ context.current_step_actions = None+ steps_taken.append(+ {+ "step": step_num,+ "url_before": url_before,+ "url_after": self.page.url,+ "actions": actions_this_step[:10],+ }+ )+ except Exception as exc:+ exception_summary = f"{type(exc).__name__}: {exc!s}"[:500]+ captured_exception = exc+ finally:+ await self._persist_classify_unknown_episode(+ context=context,+ classify_meta=classify_meta,+ page_url_at_entry=page_url_at_entry,+ page_text_at_entry=page_text_at_entry,+ steps_taken=steps_taken,+ navigation_goal=navigation_goal,+ completed=completed,+ exception_summary=exception_summary,)- steps_taken.append({"step": step_num, "url_after": self.page.url})- # Record an element fallback episode for the feedback loop- if context.workflow_run_id and context.workflow_permanent_id:- try:- await app.DATABASE.scripts.create_fallback_episode(- organization_id=context.organization_id,- workflow_permanent_id=context.workflow_permanent_id,- workflow_run_id=context.workflow_run_id,- block_label=self.current_label or "unknown",- fallback_type="element",- script_revision_id=context.script_revision_id,- error_message=f"classify returned UNKNOWN, element_fallback goal: {navigation_goal}",- page_url=page_url_at_entry,- page_text_snapshot=page_text_at_entry,- agent_actions={- "navigation_goal": navigation_goal,- "completed": completed,- "steps_taken": len(steps_taken),- "steps": steps_taken[:20],- },- )- except Exception:- LOG.debug("Failed to record element fallback episode", exc_info=True)+ if captured_exception is not None:+ raise captured_exceptionif not completed:LOG.warning(@@ -910,6 +967,80 @@ class RealSkyvernPageAi(SkyvernPageAi):)raise Exception(f"Element fallback did not complete within {max_steps} steps")+ async def _persist_classify_unknown_episode(+ self,+ *,+ context: skyvern_context.SkyvernContext,+ classify_meta: dict[str, Any] | None,+ page_url_at_entry: str,+ page_text_at_entry: str | None,+ steps_taken: list[dict],+ navigation_goal: str,+ completed: bool,+ exception_summary: str | None,+ ) -> None:+ """Record the fallback episode; attach classify_meta only on (URL, block_label) match."""+ if not (context.organization_id and context.workflow_run_id and context.workflow_permanent_id):+ LOG.debug("Skipping fallback episode: missing context identifiers")+ return++ # Single narrow: ``meta`` is the trustworthy classify metadata or None.+ # Carries the URL+block_label predecessor invariant.+ meta: dict[str, Any] | None = None+ if (+ classify_meta is not None+ and classify_meta.get("url_at_classify") == page_url_at_entry+ and classify_meta.get("block_label_at_classify") == self.current_label+ ):+ meta = classify_meta++ classify_result_for_episode: str | None = meta.get("result") if meta is not None else None++ # Flat ``actions`` matches the reviewer template iteration; ``steps`` keeps the per-iteration grouping.+ flat_actions: list[dict[str, Any]] = []+ for step in steps_taken:+ for act in step.get("actions") or []:+ flat_actions.append(act)+ agent_actions_payload: dict[str, Any] = {+ "navigation_goal": navigation_goal,+ "completed": completed,+ "steps_taken": len(steps_taken),+ "actions": flat_actions[:20],+ "steps": steps_taken[:20],+ }+ if exception_summary is not None:+ agent_actions_payload["exception_summary"] = exception_summary+ if meta is not None and meta.get("result") == "UNKNOWN":+ agent_actions_payload["classify_rejection"] = {+ "reasoning": meta.get("rejection_reasoning"),+ "confidence": meta.get("confidence"),+ "candidate_options": meta.get("candidate_options"),+ "text_excerpt": meta.get("text_excerpt"),+ }++ error_message = (+ f"classify returned UNKNOWN, element_fallback goal: {navigation_goal}"+ if classify_result_for_episode == "UNKNOWN"+ else f"element_fallback goal: {navigation_goal}"+ )++ try:+ await app.DATABASE.scripts.create_fallback_episode(+ organization_id=context.organization_id,+ workflow_permanent_id=context.workflow_permanent_id,+ workflow_run_id=context.workflow_run_id,+ block_label=self.current_label or "unknown",+ fallback_type="element",+ script_revision_id=context.script_revision_id,+ error_message=error_message,+ classify_result=classify_result_for_episode,+ page_url=page_url_at_entry,+ page_text_snapshot=page_text_at_entry,+ agent_actions=agent_actions_payload,+ )+ except Exception:+ LOG.debug("Failed to record element fallback episode", exc_info=True)+async def ai_extract(self,prompt: str,@@ -1387,6 +1518,58 @@ class RealSkyvernPageAi(SkyvernPageAi):)return+ # ``current_step_actions`` is None outside element_fallback frames; field names match the reviewer template.+ if context.current_step_actions is not None:+ # ai_act treats ``result is None or empty`` as success (no+ # exception raised below). Label "failed" only when there's+ # positive evidence of failure — otherwise "success" — to+ # match the existing ``if result and result[-1].success is+ # False: raise`` semantics two lines below.+ action_failed = bool(result) and result[-1].success is False+ action_record: dict[str, Any] = {+ "action_type": action_type,+ "intention": getattr(action, "intention", None),+ "reasoning": getattr(action, "reasoning", None),+ "page_url": self.page.url,+ "status": "failed" if action_failed else "success",+ }+ el_data = getattr(action, "skyvern_element_data", None)+ if el_data:+ action_record["element_text"] = (el_data.get("text") or "")[:200]+ action_record["element_tag"] = el_data.get("tagName", "")+ sel_options = compute_selector_options(el_data)+ if sel_options:+ action_record["selector_options"] = sel_options+ action_record["css_suggestion"] = sel_options[0][0]+ el_attrs = el_data.get("attributes") or {}+ if el_attrs:+ action_record["all_attributes"] = {+ k: v+ for k, v in el_attrs.items()+ if k+ in (+ "id",+ "name",+ "class",+ "aria-label",+ "placeholder",+ "type",+ "role",+ "data-testid",+ "href",+ "value",+ "title",+ )+ }+ if hasattr(action, "get_xpath"):+ try:+ xpath = action.get_xpath()+ except Exception:+ xpath = None+ if xpath:+ action_record["xpath"] = xpath+ context.current_step_actions.append(action_record)+if result and result[-1].success is False:raise Exception(result[-1].exception_message)diff --git a/skyvern/forge/prompts/skyvern/script-reviewer-extraction.j2 b/skyvern/forge/prompts/skyvern/script-reviewer-extraction.j2index 4dce563ef..5dca98631 100644--- a/skyvern/forge/prompts/skyvern/script-reviewer-extraction.j2+++ b/skyvern/forge/prompts/skyvern/script-reviewer-extraction.j2@@ -86,6 +86,22 @@ The cached code failed and the AI agent took over. Here is what happened:- **Fallback Type**: {{ episode.fallback_type }}- **Error**: {{ episode.error_message or "N/A" }}- **Classify Result**: {{ episode.classify_result or "N/A" }}+{% if episode.classify_result == "UNKNOWN" %}+ ⚠️ `page.classify()` returned UNKNOWN — the runtime LLM rejected every named option. Add a new branch with URL/text patterns derived from the page data below.+{% endif %}+{% if episode.agent_actions is mapping and episode.agent_actions.classify_rejection is defined %}+- **Why the runtime LLM rejected the options**:+ > {{ episode.agent_actions.classify_rejection.reasoning or "(no reasoning recorded)" }}+{% if episode.agent_actions.classify_rejection.candidate_options %}+ Options the LLM was given:+{%- for opt_key, opt_desc in episode.agent_actions.classify_rejection.candidate_options.items() %}+ - `{{ opt_key }}`: {{ opt_desc }}+{%- endfor %}+{% endif %}+{% endif %}+{% if episode.agent_actions is mapping and episode.agent_actions.exception_summary is defined and episode.agent_actions.exception_summary %}+- **Recovery agent crashed mid-fallback**: `{{ episode.agent_actions.exception_summary }}`+{% endif %}- **Page URL at failure**: {{ episode.page_url or "N/A" }}- **Page Text at failure** (use this to derive text_patterns):```diff --git a/skyvern/forge/prompts/skyvern/script-reviewer-form-filling.j2 b/skyvern/forge/prompts/skyvern/script-reviewer-form-filling.j2index 5acf33b46..16fee76e9 100644--- a/skyvern/forge/prompts/skyvern/script-reviewer-form-filling.j2+++ b/skyvern/forge/prompts/skyvern/script-reviewer-form-filling.j2@@ -91,6 +91,22 @@ The cached code failed and the AI agent took over. Here is what happened:- **Block Label**: {{ episode.block_label }}- **Fallback Type**: {{ episode.fallback_type }}- **Error**: {{ episode.error_message or "N/A" }}+{% if episode.classify_result == "UNKNOWN" %}+ ⚠️ `page.classify()` returned UNKNOWN — the runtime LLM rejected every named option. Add a new branch (prefer URL/text patterns over LLM-only options).+{% endif %}+{% if episode.agent_actions is mapping and episode.agent_actions.classify_rejection is defined %}+- **Why the runtime LLM rejected the options**:+ > {{ episode.agent_actions.classify_rejection.reasoning or "(no reasoning recorded)" }}+{% if episode.agent_actions.classify_rejection.candidate_options %}+ Options the LLM was given:+{%- for opt_key, opt_desc in episode.agent_actions.classify_rejection.candidate_options.items() %}+ - `{{ opt_key }}`: {{ opt_desc }}+{%- endfor %}+{% endif %}+{% endif %}+{% if episode.agent_actions is mapping and episode.agent_actions.exception_summary is defined and episode.agent_actions.exception_summary %}+- **Recovery agent crashed mid-fallback**: `{{ episode.agent_actions.exception_summary }}`+{% endif %}- **Page URL at failure**: {{ episode.page_url or "N/A" }}- **Page Text at failure**:```diff --git a/skyvern/forge/prompts/skyvern/script-reviewer.j2 b/skyvern/forge/prompts/skyvern/script-reviewer.j2index d339cc918..4488160bc 100644--- a/skyvern/forge/prompts/skyvern/script-reviewer.j2+++ b/skyvern/forge/prompts/skyvern/script-reviewer.j2@@ -122,9 +122,27 @@ The cached code failed and the AI agent took over. Here is what happened:- **Fallback Type**: {{ episode.fallback_type }}- **Error**: {{ episode.error_message or "N/A" }}- **Classify Result**: {{ episode.classify_result or "N/A" }}-{% if episode.classify_result and episode.classify_result != "N/A" %}+{% if episode.classify_result == "UNKNOWN" %}+ ⚠️ `page.classify()` returned UNKNOWN — the runtime LLM rejected every named option. This is the signal to ADD a new option/branch covering this page state. Use the rejection reasoning and candidate options below as the seed for the new option's description, and prefer URL/text patterns derived from the **Page URL at failure** and **Page Text at failure** fields below for deterministic dispatch.+{% elif episode.classify_result and episode.classify_result != "N/A" %}⚠️ The classify matched `{{ episode.classify_result }}` but its branch actions STILL FAILED. Update selectors for this branch using the episode data below.{% endif %}+{% if episode.agent_actions is mapping and episode.agent_actions.classify_rejection is defined %}+- **Why the runtime LLM rejected the options** (verbatim mini-LLM reasoning at classify time):+ > {{ episode.agent_actions.classify_rejection.reasoning or "(no reasoning recorded)" }}+{% if episode.agent_actions.classify_rejection.confidence is defined %}+ Confidence in rejection: {{ "%.2f"|format(episode.agent_actions.classify_rejection.confidence) }}.+{% endif %}+{% if episode.agent_actions.classify_rejection.candidate_options %}+ Options the LLM was given (and rejected):+{%- for opt_key, opt_desc in episode.agent_actions.classify_rejection.candidate_options.items() %}+ - `{{ opt_key }}`: {{ opt_desc }}+{%- endfor %}+{% endif %}+{% endif %}+{% if episode.agent_actions is mapping and episode.agent_actions.exception_summary is defined and episode.agent_actions.exception_summary %}+- **Recovery agent crashed mid-fallback**: `{{ episode.agent_actions.exception_summary }}` — treat the partial action trace below as a best-effort hint, not a complete recovery recipe.+{% endif %}- **Page URL at failure**: {{ episode.page_url or "N/A" }}- **Page Text at failure** (use this to derive text_patterns):```@@ -250,7 +268,15 @@ If the existing code already has a `page.wait()` and it still failed, increase t## What to do{% for episode in episodes %}-{% if episode.classify_result and episode.classify_result != "N/A" and episode.classify_result != "UNKNOWN" %}+{% if episode.classify_result == "UNKNOWN" %}+### Adding a new branch for this novel page state+The classify returned UNKNOWN — its candidate options (shown above) didn't fit the page. Add a new option that DOES fit:+1. **Prefer adding** a new `elif` branch with a `text_patterns` or `url_patterns` entry over rewording an existing option's description. The mini-LLM tier is brittle; URL/text matching is deterministic.+2. **Seed the new option's description** from the rejection reasoning shown above ("Why the runtime LLM rejected the options"). It already explains what page state is actually present.+3. **Derive the pattern** from the recorded **Page URL at failure** (URL fragments unique to this state) or **Page Text at failure** (a substring that uniquely identifies the page). Add the pattern to the `text_patterns=` or `url_patterns=` argument of the existing `page.classify()` call AND add the matching option to `options=`.+4. **Translate the recorded agent actions** into the new branch body, using selector/element data per the rules below.+5. **Do NOT remove the `else` branch.** It still needs to call `await page.element_fallback(navigation_goal="...")` to catch any genuinely-new states the new branch doesn't cover.+{% elif episode.classify_result and episode.classify_result != "N/A" %}### Handling episode with existing branch `{{ episode.classify_result }}`The classify routed to `{{ episode.classify_result }}` but its actions failed. Update the branch's selectors:1. Keep existing selectors — ADD new label variants with comma-separated CSS selectors.diff --git a/skyvern/forge/sdk/core/skyvern_context.py b/skyvern/forge/sdk/core/skyvern_context.pyindex d152baa8c..c8a4c8387 100644--- a/skyvern/forge/sdk/core/skyvern_context.py+++ b/skyvern/forge/sdk/core/skyvern_context.py@@ -90,6 +90,8 @@ class SkyvernContext:ai_mode_override: str | None = Nonescript_llm_call_count: int = 0last_classify_result: str | None = None+ last_classify_meta: dict[str, Any] | None = None+ current_step_actions: list[dict[str, Any]] | None = Noneskip_complete_verification: bool = False# magic link handlingdiff --git a/skyvern/services/script_reviewer.py b/skyvern/services/script_reviewer.pyindex ebcc85e4e..2fd82a7ac 100644--- a/skyvern/services/script_reviewer.py+++ b/skyvern/services/script_reviewer.py@@ -68,6 +68,10 @@ async def load_filtered_run_param_values(workflow_run_id: str) -> dict[str, str]return result+class AutoFixMissingElseGoalError(ValueError):+ """Raised when an auto-injected else-branch element_fallback would need a navigation_goal but none exists."""++@dataclasses.dataclass(frozen=True)class BlockReviewResult:"""Result of reviewing a single block, carrying the code and LLM artifacts."""@@ -1090,9 +1094,9 @@ class ScriptReviewer:marker_ids=[c.marker_id for c in recoverable_candidates[:10]],)- # Use provided navigation goal, or fall back to a generic description- if not navigation_goal:- navigation_goal = "Complete the navigation task for this block"+ # Empty string is safe for downstream re.findall / prompt rendering;+ # ``_auto_fix_missing_else`` raises if a real goal is actually required.+ navigation_goal_for_prompt = navigation_goal or ""# Infer function signature from existing codefunction_signature = self._extract_function_signature(existing_code)@@ -1127,7 +1131,7 @@ class ScriptReviewer:# Extract parameter keys from {{ param }} placeholders in the navigation goal# and merge with workflow-level parameter keys for a complete list.- goal_param_keys = set(re.findall(r"\{\{\s*(\w+)\s*\}\}", navigation_goal))+ goal_param_keys = set(re.findall(r"\{\{\s*(\w+)\s*\}\}", navigation_goal_for_prompt))parameter_keys = sorted(goal_param_keys | set(all_parameter_keys or []))# The filter strips `<block_label>_output` from the valid set so the# validator's primary mode rejects any ref to the current block's own@@ -1165,7 +1169,7 @@ class ScriptReviewer:# Build the reviewer promptreviewer_prompt = prompt_engine.load_prompt(template=template,- navigation_goal=navigation_goal,+ navigation_goal=navigation_goal_for_prompt,existing_code=existing_code,episodes=[{@@ -1326,8 +1330,18 @@ class ScriptReviewer:# Validate classify handling (every classify must have an else branch)classify_error = self._validate_classify_handling(updated_code)if classify_error is not None:- # Try auto-fixing by injecting missing else branches- fixed_code = self._auto_fix_missing_else(updated_code, navigation_goal or "")+ # Try auto-fixing by injecting missing else branches.+ fixed_code: str | None+ try:+ fixed_code = self._auto_fix_missing_else(updated_code, navigation_goal)+ except AutoFixMissingElseGoalError as exc:+ LOG.warning(+ "ScriptReviewer: skipped auto-fix-else for goal-less block",+ block_label=block_label,+ script_revision_id=script_revision_id,+ reason=str(exc),+ )+ fixed_code = Nonerevalidation_error = (self._validate_classify_handling(fixed_code) if fixed_code else "auto-fix returned None")@@ -2917,12 +2931,13 @@ class ScriptReviewer:+ " Each return statement must use a next_block_label and branch_index from the branch definitions.")- def _auto_fix_missing_else(self, code: str, navigation_goal: str) -> str | None:- """Auto-inject missing `else` branches after page.classify() if/elif chains.+ def _auto_fix_missing_else(self, code: str, navigation_goal: str | None) -> str | None:+ """Auto-inject missing ``else`` branches after page.classify() if/elif chains.- When the LLM generates a classify with if/elif but no else, this inserts- an else branch with element_fallback. Returns the fixed code, or None- if the code structure is too complex to safely auto-fix.+ Returns the fixed code, or None when no injection is performed (no+ classify calls present, or the structure is too complex to safely+ auto-fix). Raises ``AutoFixMissingElseGoalError`` if injection is+ needed but no ``navigation_goal`` is available."""lines = code.split("\n")classify_line_indices = []@@ -2933,6 +2948,14 @@ class ScriptReviewer:if not classify_line_indices:return None+ # Defer the goal check to injection sites — classify-already-has-else short-circuits without it.+ def _require_goal() -> str:+ if not navigation_goal or not navigation_goal.strip():+ raise AutoFixMissingElseGoalError(+ "Cannot auto-inject else-branch element_fallback: block has no navigation_goal."+ )+ return navigation_goal+# Process in reverse order so line insertions don't shift earlier indicesfor classify_line_idx in reversed(classify_line_indices):if_indent: int | None = None@@ -2973,8 +2996,9 @@ class ScriptReviewer:if last_branch_body_end is not None:indent_str = " " * if_indentbody_indent = " " * (if_indent + 4)+ _goal_str = _require_goal()goal = (- navigation_goal.replace("\\", "\\\\")+ _goal_str.replace("\\", "\\\\").replace('"', '\\"').replace("\n", " ").replace("\r", " ")[:200]@@ -2992,8 +3016,9 @@ class ScriptReviewer:if if_indent is not None and last_branch_body_end is not None:indent_str = " " * if_indentbody_indent = " " * (if_indent + 4)+ _goal_str = _require_goal()goal = (- navigation_goal.replace("\\", "\\\\")+ _goal_str.replace("\\", "\\\\").replace('"', '\\"').replace("\n", " ").replace("\r", " ")[:200]@@ -3010,11 +3035,9 @@ class ScriptReviewer:if if_indent is not None and last_branch_body_end is not None:indent_str = " " * if_indentbody_indent = " " * (if_indent + 4)+ _goal_str = _require_goal()goal = (- navigation_goal.replace("\\", "\\\\")- .replace('"', '\\"')- .replace("\n", " ")- .replace("\r", " ")[:200]+ _goal_str.replace("\\", "\\\\").replace('"', '\\"').replace("\n", " ").replace("\r", " ")[:200])else_block = [f"{indent_str}else:",diff --git a/tests/unit/test_auto_fix_missing_else_goal.py b/tests/unit/test_auto_fix_missing_else_goal.pynew file mode 100644index 000000000..87f66178f--- /dev/null+++ b/tests/unit/test_auto_fix_missing_else_goal.py@@ -0,0 +1,74 @@+"""Tests for ``_auto_fix_missing_else`` goal-required behavior."""++from __future__ import annotations++import pytest++from skyvern.services.script_reviewer import AutoFixMissingElseGoalError, ScriptReviewer++CLASSIFY_NO_ELSE_CODE = """+async def block_fn(page, context):+ state = await page.classify(options={"a": "page A"})+ if state == "a":+ await page.click(selector="#a-button")+"""+++CLASSIFY_WITH_ELSE_CODE = """+async def block_fn(page, context):+ state = await page.classify(options={"a": "page A"})+ if state == "a":+ await page.click(selector="#a-button")+ else:+ await page.element_fallback(navigation_goal="Click the A button")+"""+++NO_CLASSIFY_CODE = """+async def block_fn(page, context):+ await page.click(selector="#submit")+"""+++class TestAutoFixMissingElseGoal:+ def setup_method(self) -> None:+ self.reviewer = ScriptReviewer()++ def test_raises_on_missing_goal_when_else_needs_injection(self) -> None:+ with pytest.raises(AutoFixMissingElseGoalError):+ self.reviewer._auto_fix_missing_else(CLASSIFY_NO_ELSE_CODE, None)++ def test_raises_on_empty_string_goal(self) -> None:+ with pytest.raises(AutoFixMissingElseGoalError):+ self.reviewer._auto_fix_missing_else(CLASSIFY_NO_ELSE_CODE, "")++ def test_raises_on_whitespace_only_goal(self) -> None:+ with pytest.raises(AutoFixMissingElseGoalError):+ self.reviewer._auto_fix_missing_else(CLASSIFY_NO_ELSE_CODE, " \n\t ")++ def test_no_classify_returns_none_without_raising(self) -> None:+ # No classify in the code → nothing to inject → None, no goal needed.+ assert self.reviewer._auto_fix_missing_else(NO_CLASSIFY_CODE, None) is None+ assert self.reviewer._auto_fix_missing_else(NO_CLASSIFY_CODE, "") is None++ def test_real_goal_threaded_into_injected_else(self) -> None:+ fixed = self.reviewer._auto_fix_missing_else(+ CLASSIFY_NO_ELSE_CODE,+ "Click the A button on the search results page",+ )+ assert fixed is not None+ assert "else:" in fixed+ assert "page.element_fallback" in fixed+ assert "Click the A button on the search results page" in fixed+ # And the placeholder string must NOT appear.+ assert "Complete the navigation task for this block" not in fixed++ def test_existing_else_branch_is_untouched(self) -> None:+ # Even with a missing goal — there's no injection needed, so no raise.+ result = self.reviewer._auto_fix_missing_else(CLASSIFY_WITH_ELSE_CODE, None)+ # _auto_fix_missing_else returns the (possibly-edited) code; if there+ # was nothing to fix, the code is returned unchanged. Either None or+ # the original-equivalent string is acceptable as long as no raise+ # occurs.+ if result is not None:+ assert "Click the A button" in resultdiff --git a/tests/unit/test_classify_metadata_lifecycle.py b/tests/unit/test_classify_metadata_lifecycle.pynew file mode 100644index 000000000..afca60156--- /dev/null+++ b/tests/unit/test_classify_metadata_lifecycle.py@@ -0,0 +1,175 @@+"""Tests for ``_store_classify_result`` metadata lifecycle on SkyvernContext+and the consume-at-entry invariant in ``ai_element_fallback``.+"""++from __future__ import annotations++from types import SimpleNamespace+from unittest.mock import AsyncMock++import pytest++from skyvern.core.script_generations.real_skyvern_page_ai import RealSkyvernPageAi+from skyvern.forge import app+from skyvern.forge.sdk.core import skyvern_context+++def ctx() -> skyvern_context.SkyvernContext:+ skyvern_context.set(skyvern_context.SkyvernContext(organization_id="o_test", workflow_run_id="wr_test"))+ yield skyvern_context.current()+ skyvern_context.reset()+++def test_unknown_writes_full_meta(ctx: skyvern_context.SkyvernContext) -> None:+ RealSkyvernPageAi._store_classify_result(+ "UNKNOWN",+ current_url="https://example.com/search",+ options={"card_link": "single recipe card"},+ block_label="search_block",+ rejection_reasoning="page is search results, not a single card",+ confidence=1.0,+ text_excerpt="Recipe results for...",+ )+ meta = ctx.last_classify_meta+ assert meta is not None+ assert meta["result"] == "UNKNOWN"+ assert meta["url_at_classify"] == "https://example.com/search"+ assert meta["block_label_at_classify"] == "search_block"+ assert meta["candidate_options"] == {"card_link": "single recipe card"}+ assert meta["rejection_reasoning"] == "page is search results, not a single card"+ assert meta["confidence"] == 1.0+ assert meta["text_excerpt"] == "Recipe results for..."+ assert ctx.last_classify_result == "UNKNOWN"+++def test_matched_result_clears_meta(ctx: skyvern_context.SkyvernContext) -> None:+ ctx.last_classify_meta = {"result": "UNKNOWN", "stale": True}+ RealSkyvernPageAi._store_classify_result(+ "card_link",+ current_url="https://example.com/recipe/123",+ options={"card_link": "single recipe card"},+ block_label="search_block",+ )+ assert ctx.last_classify_meta is None+ assert ctx.last_classify_result == "card_link"+++def test_unknown_with_default_args_does_not_raise(ctx: skyvern_context.SkyvernContext) -> None:+ """LLM-exception path: reasoning='', confidence=0.0, text_excerpt=''."""+ RealSkyvernPageAi._store_classify_result(+ "UNKNOWN",+ current_url="https://example.com",+ options={"a": "x"},+ block_label=None,+ )+ meta = ctx.last_classify_meta+ assert meta is not None+ assert meta["rejection_reasoning"] == ""+ assert meta["confidence"] == 0.0+ assert meta["text_excerpt"] == ""+ assert meta["block_label_at_classify"] is None+++def test_text_excerpt_is_truncated_to_500(ctx: skyvern_context.SkyvernContext) -> None:+ long_text = "x" * 1000+ RealSkyvernPageAi._store_classify_result(+ "UNKNOWN",+ current_url="https://example.com",+ options={"a": "x"},+ block_label="b",+ text_excerpt=long_text,+ )+ assert ctx.last_classify_meta is not None+ assert len(ctx.last_classify_meta["text_excerpt"]) == 500+++def test_no_context_no_raise() -> None:+ """When there's no current SkyvernContext, the helper is a silent no-op."""+ RealSkyvernPageAi._store_classify_result(+ "UNKNOWN",+ current_url="https://example.com",+ options={"a": "x"},+ block_label=None,+ )+++def _make_page_stub(url: str = "https://example.com/x") -> SimpleNamespace:+ async def _inner_text(_selector: str) -> str:+ return "page text"++ return SimpleNamespace(url=url, inner_text=_inner_text)+++def _make_page_ai_stub(page: SimpleNamespace) -> RealSkyvernPageAi:+ inst = RealSkyvernPageAi.__new__(RealSkyvernPageAi)+ inst.page = page # type: ignore[assignment]+ inst.scraped_page = SimpleNamespace() # type: ignore[assignment]+ inst.current_label = "block_a"+ return inst+++def fallback_ctx() -> skyvern_context.SkyvernContext:+ skyvern_context.set(+ skyvern_context.SkyvernContext(+ organization_id="o_test",+ workflow_run_id="wr_test",+ workflow_permanent_id="wpid_test",+ )+ )+ yield skyvern_context.current()+ skyvern_context.reset()+++async def test_ai_element_fallback_consumes_meta_on_success(+ fallback_ctx: skyvern_context.SkyvernContext, monkeypatch: pytest.MonkeyPatch+) -> None:+ """Happy path: meta is popped at function entry; subsequent reads see None."""+ fallback_ctx.last_classify_meta = {+ "result": "UNKNOWN",+ "url_at_classify": "https://example.com/x",+ "block_label_at_classify": "block_a",+ "candidate_options": {"a": "x"},+ "rejection_reasoning": "n/a",+ "confidence": 1.0,+ "text_excerpt": "",+ }+ page = _make_page_stub()+ page_ai = _make_page_ai_stub(page)++ monkeypatch.setattr(page_ai, "ai_validate", AsyncMock(return_value=True))+ monkeypatch.setattr(page_ai, "ai_act", AsyncMock(return_value=None))+ monkeypatch.setattr(app.DATABASE.scripts, "create_fallback_episode", AsyncMock(return_value=None))++ await page_ai.ai_element_fallback(navigation_goal="real goal")++ assert fallback_ctx.last_classify_meta is None+++async def test_ai_element_fallback_consumes_meta_when_ai_act_raises(+ fallback_ctx: skyvern_context.SkyvernContext, monkeypatch: pytest.MonkeyPatch+) -> None:+ """Exception path: meta is popped at function entry, persists None even when ai_act raises mid-loop."""+ fallback_ctx.last_classify_meta = {+ "result": "UNKNOWN",+ "url_at_classify": "https://example.com/x",+ "block_label_at_classify": "block_a",+ "candidate_options": {"a": "x"},+ "rejection_reasoning": "n/a",+ "confidence": 1.0,+ "text_excerpt": "",+ }+ page = _make_page_stub()+ page_ai = _make_page_ai_stub(page)++ monkeypatch.setattr(page_ai, "ai_validate", AsyncMock(return_value=False))+ monkeypatch.setattr(page_ai, "ai_act", AsyncMock(side_effect=RuntimeError("boom")))+ monkeypatch.setattr(app.DATABASE.scripts, "create_fallback_episode", AsyncMock(return_value=None))++ with pytest.raises(RuntimeError, match="boom"):+ await page_ai.ai_element_fallback(navigation_goal="real goal", max_steps=2)++ assert fallback_ctx.last_classify_meta is None